1//===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// RISC-V.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/RISCVMatInt.h"
15#include "RISCVRegisterBankInfo.h"
16#include "RISCVSubtarget.h"
17#include "RISCVTargetMachine.h"
18#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
19#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
20#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
21#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
22#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/MachineJumpTableInfo.h"
25#include "llvm/IR/IntrinsicsRISCV.h"
26#include "llvm/Support/Debug.h"
27
28#define DEBUG_TYPE "riscv-isel"
29
30using namespace llvm;
31using namespace MIPatternMatch;
32
33#define GET_GLOBALISEL_PREDICATE_BITSET
34#include "RISCVGenGlobalISel.inc"
35#undef GET_GLOBALISEL_PREDICATE_BITSET
36
37namespace {
38
39class RISCVInstructionSelector : public InstructionSelector {
40public:
41 RISCVInstructionSelector(const RISCVTargetMachine &TM,
42 const RISCVSubtarget &STI,
43 const RISCVRegisterBankInfo &RBI);
44
45 bool select(MachineInstr &MI) override;
46
47 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
48 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
49 BlockFrequencyInfo *BFI) override {
50 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
51 MRI = &MF.getRegInfo();
52 }
53
54 static const char *getName() { return DEBUG_TYPE; }
55
56private:
57 static constexpr unsigned MaxRecursionDepth = 6;
58
59 bool hasAllNBitUsers(const MachineInstr &MI, unsigned Bits,
60 const unsigned Depth = 0) const;
61 bool hasAllHUsers(const MachineInstr &MI) const {
62 return hasAllNBitUsers(MI, Bits: 16);
63 }
64 bool hasAllWUsers(const MachineInstr &MI) const {
65 return hasAllNBitUsers(MI, Bits: 32);
66 }
67
68 bool isRegInGprb(Register Reg) const;
69 bool isRegInFprb(Register Reg) const;
70 bool isWorthFoldingAdd(Register AddResult) const;
71
72 // tblgen-erated 'select' implementation, used as the initial selector for
73 // the patterns that don't require complex C++.
74 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
75
76 // A lowering phase that runs before any selection attempts.
77 // Returns true if the instruction was modified.
78 void preISelLower(MachineInstr &MI);
79
80 bool replacePtrWithInt(MachineInstr &MI, unsigned OpIdx);
81
82 // Custom selection methods
83 bool selectCopy(MachineInstr &MI) const;
84 bool selectImplicitDef(MachineInstr &MI) const;
85 bool materializeImm(Register Reg, int64_t Imm, MachineInstr &MI) const;
86 // Emit a constant-materialization instruction sequence.
87 bool materializeInstSeq(Register DstReg, const RISCVMatInt::InstSeq &Seq,
88 MachineInstr &MI) const;
89 bool selectAddr(MachineInstr &MI, bool IsLocal = true,
90 bool IsExternWeak = false) const;
91 bool selectSelect(MachineInstr &MI) const;
92 bool selectFPCompare(MachineInstr &MI) const;
93 void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
94 MachineInstr &MI) const;
95 bool selectUnmergeValues(MachineInstr &MI) const;
96 void addVectorLoadStoreOperands(MachineInstr &I,
97 SmallVectorImpl<Register> &SrcOps,
98 unsigned &CurOp, bool IsMasked,
99 bool IsStridedOrIndexed,
100 LLT *IndexVT = nullptr) const;
101 bool selectIntrinsicWithSideEffects(MachineInstr &I) const;
102 bool selectIntrinsic(MachineInstr &I) const;
103 bool selectExtractSubvector(MachineInstr &MI) const;
104 bool selectInsertSubVector(MachineInstr &I) const;
105 ComplexRendererFns selectShiftMask(MachineOperand &Root,
106 unsigned ShiftWidth) const;
107 ComplexRendererFns selectShiftMaskXLen(MachineOperand &Root) const {
108 return selectShiftMask(Root, ShiftWidth: STI.getXLen());
109 }
110 ComplexRendererFns selectShiftMask32(MachineOperand &Root) const {
111 return selectShiftMask(Root, ShiftWidth: 32);
112 }
113 ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
114 ComplexRendererFns selectAddrRegImmLsb00000(MachineOperand &Root) const;
115
116 // Plan for materializing a constant address as (Hi materialization, Lo12
117 // offset). Lo12 is a simm12 that, for prefetch (IsPrefetch), must be
118 // a multiple of 32.
119 struct ConstAddrPlan {
120 enum { X0, LUI, InstSeq } Kind = X0;
121 int64_t Hi20 = 0;
122 RISCVMatInt::InstSeq Seq;
123 int64_t Lo12 = 0;
124 };
125 ComplexRendererFns computeConstAddr(int64_t CVal, bool IsPrefetch,
126 Register OrigBase) const;
127 // Materialize the high part of Plan into a register. If OrigBase is valid,
128 // ADD it to the materialized high part (for G_PTR_ADD + large constant).
129 Register materializeConstBase(MachineInstrBuilder &MIB,
130 const ConstAddrPlan &Plan,
131 Register OrigBase) const;
132
133 ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;
134 template <unsigned Bits>
135 ComplexRendererFns selectSExtBits(MachineOperand &Root) const {
136 return selectSExtBits(Root, Bits);
137 }
138
139 ComplexRendererFns selectZExtBits(MachineOperand &Root, unsigned Bits) const;
140 template <unsigned Bits>
141 ComplexRendererFns selectZExtBits(MachineOperand &Root) const {
142 return selectZExtBits(Root, Bits);
143 }
144
145 ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
146 template <unsigned ShAmt>
147 ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
148 return selectSHXADDOp(Root, ShAmt);
149 }
150
151 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
152 unsigned ShAmt) const;
153 template <unsigned ShAmt>
154 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
155 return selectSHXADD_UWOp(Root, ShAmt);
156 }
157
158 ComplexRendererFns renderVLOp(MachineOperand &Root) const;
159 ComplexRendererFns renderAddiPair(Register BaseReg, int64_t AddiImm,
160 int64_t OffsetImm) const;
161 // Custom renderers for tablegen
162 void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
163 int OpIdx) const;
164 void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
165 int OpIdx) const;
166 void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
167 int OpIdx) const;
168 void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
169 int OpIdx) const;
170
171 void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
172 int OpIdx) const;
173 void renderXLenSubTrailingOnes(MachineInstrBuilder &MIB,
174 const MachineInstr &MI, int OpIdx) const;
175
176 void renderAddiPairImmLarge(MachineInstrBuilder &MIB, const MachineInstr &MI,
177 int OpIdx) const;
178 void renderAddiPairImmSmall(MachineInstrBuilder &MIB, const MachineInstr &MI,
179 int OpIdx) const;
180
181 const RISCVSubtarget &STI;
182 const RISCVInstrInfo &TII;
183 const RISCVRegisterInfo &TRI;
184 const RISCVRegisterBankInfo &RBI;
185 const RISCVTargetMachine &TM;
186
187 MachineRegisterInfo *MRI = nullptr;
188
189 // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
190 // uses "STI." in the code generated by TableGen. We need to unify the name of
191 // Subtarget variable.
192 const RISCVSubtarget *Subtarget = &STI;
193
194#define GET_GLOBALISEL_PREDICATES_DECL
195#include "RISCVGenGlobalISel.inc"
196#undef GET_GLOBALISEL_PREDICATES_DECL
197
198#define GET_GLOBALISEL_TEMPORARIES_DECL
199#include "RISCVGenGlobalISel.inc"
200#undef GET_GLOBALISEL_TEMPORARIES_DECL
201};
202
203} // end anonymous namespace
204
205#define GET_GLOBALISEL_IMPL
206#include "RISCVGenGlobalISel.inc"
207#undef GET_GLOBALISEL_IMPL
208
209RISCVInstructionSelector::RISCVInstructionSelector(
210 const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
211 const RISCVRegisterBankInfo &RBI)
212 : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
213 TM(TM),
214
215#define GET_GLOBALISEL_PREDICATES_INIT
216#include "RISCVGenGlobalISel.inc"
217#undef GET_GLOBALISEL_PREDICATES_INIT
218#define GET_GLOBALISEL_TEMPORARIES_INIT
219#include "RISCVGenGlobalISel.inc"
220#undef GET_GLOBALISEL_TEMPORARIES_INIT
221{
222}
223
224// Mimics optimizations in ISel and RISCVOptWInst Pass
225bool RISCVInstructionSelector::hasAllNBitUsers(const MachineInstr &MI,
226 unsigned Bits,
227 const unsigned Depth) const {
228
229 assert((MI.getOpcode() == TargetOpcode::G_ADD ||
230 MI.getOpcode() == TargetOpcode::G_SUB ||
231 MI.getOpcode() == TargetOpcode::G_MUL ||
232 MI.getOpcode() == TargetOpcode::G_SHL ||
233 MI.getOpcode() == TargetOpcode::G_LSHR ||
234 MI.getOpcode() == TargetOpcode::G_AND ||
235 MI.getOpcode() == TargetOpcode::G_OR ||
236 MI.getOpcode() == TargetOpcode::G_XOR ||
237 MI.getOpcode() == TargetOpcode::G_SEXT_INREG || Depth != 0) &&
238 "Unexpected opcode");
239
240 if (Depth >= RISCVInstructionSelector::MaxRecursionDepth)
241 return false;
242
243 auto DestReg = MI.getOperand(i: 0).getReg();
244 for (auto &UserOp : MRI->use_nodbg_operands(Reg: DestReg)) {
245 assert(UserOp.getParent() && "UserOp must have a parent");
246 const MachineInstr &UserMI = *UserOp.getParent();
247 unsigned OpIdx = UserOp.getOperandNo();
248
249 switch (UserMI.getOpcode()) {
250 default:
251 return false;
252 case RISCV::ADDW:
253 case RISCV::ADDIW:
254 case RISCV::SUBW:
255 case RISCV::FCVT_D_W:
256 case RISCV::FCVT_S_W:
257 if (Bits >= 32)
258 break;
259 return false;
260 case RISCV::SLL:
261 case RISCV::SRA:
262 case RISCV::SRL:
263 // Shift amount operands only use log2(Xlen) bits.
264 if (OpIdx == 2 && Bits >= Log2_32(Value: Subtarget->getXLen()))
265 break;
266 return false;
267 case RISCV::SLLI:
268 // SLLI only uses the lower (XLen - ShAmt) bits.
269 if (Bits >= Subtarget->getXLen() - UserMI.getOperand(i: 2).getImm())
270 break;
271 return false;
272 case RISCV::ANDI:
273 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(
274 Value: (uint64_t)UserMI.getOperand(i: 2).getImm()))
275 break;
276 goto RecCheck;
277 case RISCV::AND:
278 case RISCV::OR:
279 case RISCV::XOR:
280 RecCheck:
281 if (hasAllNBitUsers(MI: UserMI, Bits, Depth: Depth + 1))
282 break;
283 return false;
284 case RISCV::SRLI: {
285 unsigned ShAmt = UserMI.getOperand(i: 2).getImm();
286 // If we are shifting right by less than Bits, and users don't demand any
287 // bits that were shifted into [Bits-1:0], then we can consider this as an
288 // N-Bit user.
289 if (Bits > ShAmt && hasAllNBitUsers(MI: UserMI, Bits: Bits - ShAmt, Depth: Depth + 1))
290 break;
291 return false;
292 }
293 }
294 }
295
296 return true;
297}
298
299InstructionSelector::ComplexRendererFns
300RISCVInstructionSelector::selectShiftMask(MachineOperand &Root,
301 unsigned ShiftWidth) const {
302 if (!Root.isReg())
303 return std::nullopt;
304
305 using namespace llvm::MIPatternMatch;
306
307 Register ShAmtReg = Root.getReg();
308 // Peek through zext.
309 Register ZExtSrcReg;
310 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrcReg))))
311 ShAmtReg = ZExtSrcReg;
312
313 APInt AndMask;
314 Register AndSrcReg;
315 // Try to combine the following pattern (applicable to other shift
316 // instructions as well as 32-bit ones):
317 //
318 // %4:gprb(s64) = G_AND %3, %2
319 // %5:gprb(s64) = G_LSHR %1, %4(s64)
320 //
321 // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
322 // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
323 // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
324 // then it can be eliminated. Given register rs1 or rs2 holding a constant
325 // (the and mask), there are two cases G_AND can be erased:
326 //
327 // 1. the lowest log2(XLEN) bits of the and mask are all set
328 // 2. the bits of the register being masked are already unset (zero set)
329 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: AndSrcReg), R: m_ICst(Cst&: AndMask)))) {
330 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
331 if (ShMask.isSubsetOf(RHS: AndMask)) {
332 ShAmtReg = AndSrcReg;
333 } else {
334 // SimplifyDemandedBits may have optimized the mask so try restoring any
335 // bits that are known zero.
336 KnownBits Known = VT->getKnownBits(R: AndSrcReg);
337 if (ShMask.isSubsetOf(RHS: AndMask | Known.Zero))
338 ShAmtReg = AndSrcReg;
339 }
340 }
341
342 APInt Imm;
343 Register Reg;
344 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAdd(L: m_Reg(R&: Reg), R: m_ICst(Cst&: Imm)))) {
345 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0)
346 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
347 // to avoid the ADD.
348 ShAmtReg = Reg;
349 } else if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GSub(L: m_ICst(Cst&: Imm), R: m_Reg(R&: Reg)))) {
350 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0) {
351 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
352 // to generate a NEG instead of a SUB of a constant.
353 ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
354 unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
355 return {{[=](MachineInstrBuilder &MIB) {
356 MachineIRBuilder(*MIB.getInstr())
357 .buildInstr(Opc: NegOpc, DstOps: {ShAmtReg}, SrcOps: {Register(RISCV::X0), Reg});
358 MIB.addReg(RegNo: ShAmtReg);
359 }}};
360 }
361 if (Imm.urem(RHS: ShiftWidth) == ShiftWidth - 1) {
362 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
363 // to generate a NOT instead of a SUB of a constant.
364 ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
365 return {{[=](MachineInstrBuilder &MIB) {
366 MachineIRBuilder(*MIB.getInstr())
367 .buildInstr(Opc: RISCV::XORI, DstOps: {ShAmtReg}, SrcOps: {Reg})
368 .addImm(Val: -1);
369 MIB.addReg(RegNo: ShAmtReg);
370 }}};
371 }
372 }
373
374 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ShAmtReg); }}};
375}
376
377InstructionSelector::ComplexRendererFns
378RISCVInstructionSelector::selectSExtBits(MachineOperand &Root,
379 unsigned Bits) const {
380 if (!Root.isReg())
381 return std::nullopt;
382 Register RootReg = Root.getReg();
383
384 Register SrcReg;
385 if (mi_match(R: RootReg, MRI: *MRI,
386 P: m_GSExtInReg(Src: m_Reg(R&: SrcReg), Imm: m_SpecificImm(RequestedValue: Bits)))) {
387 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SrcReg); }}};
388 }
389
390 unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits();
391 if ((Size - VT->computeNumSignBits(R: RootReg)) < Bits)
392 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}};
393
394 return std::nullopt;
395}
396
397InstructionSelector::ComplexRendererFns
398RISCVInstructionSelector::selectZExtBits(MachineOperand &Root,
399 unsigned Bits) const {
400 if (!Root.isReg())
401 return std::nullopt;
402 Register RootReg = Root.getReg();
403
404 Register RegX;
405 uint64_t Mask = maskTrailingOnes<uint64_t>(N: Bits);
406 if (mi_match(R: RootReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: RegX), R: m_SpecificICst(RequestedValue: Mask)))) {
407 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}};
408 }
409
410 if (mi_match(R: RootReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: RegX))) &&
411 MRI->getType(Reg: RegX).getScalarSizeInBits() == Bits)
412 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}};
413
414 unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits();
415 if (VT->maskedValueIsZero(Val: RootReg, Mask: APInt::getBitsSetFrom(numBits: Size, loBit: Bits)))
416 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}};
417
418 return std::nullopt;
419}
420
421InstructionSelector::ComplexRendererFns
422RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
423 unsigned ShAmt) const {
424 using namespace llvm::MIPatternMatch;
425
426 if (!Root.isReg())
427 return std::nullopt;
428 Register RootReg = Root.getReg();
429
430 const unsigned XLen = STI.getXLen();
431 APInt Mask, C2;
432 Register RegY;
433 std::optional<bool> LeftShift;
434 // (and (shl y, c2), mask)
435 if (mi_match(R: RootReg, MRI: *MRI,
436 P: m_GAnd(L: m_GShl(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
437 LeftShift = true;
438 // (and (lshr y, c2), mask)
439 else if (mi_match(R: RootReg, MRI: *MRI,
440 P: m_GAnd(L: m_GLShr(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
441 LeftShift = false;
442
443 if (LeftShift.has_value()) {
444 if (*LeftShift)
445 Mask &= maskTrailingZeros<uint64_t>(N: C2.getZExtValue());
446 else
447 Mask &= maskTrailingOnes<uint64_t>(N: XLen - C2.getZExtValue());
448
449 if (Mask.isShiftedMask()) {
450 unsigned Leading = XLen - Mask.getActiveBits();
451 unsigned Trailing = Mask.countr_zero();
452 // Given (and (shl y, c2), mask) in which mask has no leading zeros and
453 // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
454 if (*LeftShift && Leading == 0 && C2.ult(RHS: Trailing) && Trailing == ShAmt) {
455 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
456 return {{[=](MachineInstrBuilder &MIB) {
457 MachineIRBuilder(*MIB.getInstr())
458 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
459 .addImm(Val: Trailing - C2.getZExtValue());
460 MIB.addReg(RegNo: DstReg);
461 }}};
462 }
463
464 // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
465 // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
466 if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
467 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
468 return {{[=](MachineInstrBuilder &MIB) {
469 MachineIRBuilder(*MIB.getInstr())
470 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
471 .addImm(Val: Leading + Trailing);
472 MIB.addReg(RegNo: DstReg);
473 }}};
474 }
475 }
476 }
477
478 LeftShift.reset();
479
480 // (shl (and y, mask), c2)
481 if (mi_match(R: RootReg, MRI: *MRI,
482 P: m_GShl(L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
483 R: m_ICst(Cst&: C2))))
484 LeftShift = true;
485 // (lshr (and y, mask), c2)
486 else if (mi_match(R: RootReg, MRI: *MRI,
487 P: m_GLShr(L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
488 R: m_ICst(Cst&: C2))))
489 LeftShift = false;
490
491 if (LeftShift.has_value() && Mask.isShiftedMask()) {
492 unsigned Leading = XLen - Mask.getActiveBits();
493 unsigned Trailing = Mask.countr_zero();
494
495 // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
496 // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
497 bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&
498 (Trailing + C2.getZExtValue()) == ShAmt;
499 if (!Cond)
500 // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
501 // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
502 Cond = !*LeftShift && Leading == 32 && C2.ult(RHS: Trailing) &&
503 (Trailing - C2.getZExtValue()) == ShAmt;
504
505 if (Cond) {
506 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
507 return {{[=](MachineInstrBuilder &MIB) {
508 MachineIRBuilder(*MIB.getInstr())
509 .buildInstr(Opc: RISCV::SRLIW, DstOps: {DstReg}, SrcOps: {RegY})
510 .addImm(Val: Trailing);
511 MIB.addReg(RegNo: DstReg);
512 }}};
513 }
514 }
515
516 return std::nullopt;
517}
518
519InstructionSelector::ComplexRendererFns
520RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
521 unsigned ShAmt) const {
522 using namespace llvm::MIPatternMatch;
523
524 if (!Root.isReg())
525 return std::nullopt;
526 Register RootReg = Root.getReg();
527
528 // Given (and (shl x, c2), mask) in which mask is a shifted mask with
529 // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
530 // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
531 APInt Mask, C2;
532 Register RegX;
533 if (mi_match(
534 R: RootReg, MRI: *MRI,
535 P: m_OneNonDBGUse(SP: m_GAnd(L: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: RegX), R: m_ICst(Cst&: C2))),
536 R: m_ICst(Cst&: Mask))))) {
537 Mask &= maskTrailingZeros<uint64_t>(N: C2.getZExtValue());
538
539 if (Mask.isShiftedMask()) {
540 unsigned Leading = Mask.countl_zero();
541 unsigned Trailing = Mask.countr_zero();
542 if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
543 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
544 return {{[=](MachineInstrBuilder &MIB) {
545 MachineIRBuilder(*MIB.getInstr())
546 .buildInstr(Opc: RISCV::SLLI, DstOps: {DstReg}, SrcOps: {RegX})
547 .addImm(Val: C2.getZExtValue() - ShAmt);
548 MIB.addReg(RegNo: DstReg);
549 }}};
550 }
551 }
552 }
553
554 return std::nullopt;
555}
556
557InstructionSelector::ComplexRendererFns
558RISCVInstructionSelector::renderVLOp(MachineOperand &Root) const {
559 assert(Root.isReg() && "Expected operand to be a Register");
560 std::optional<ValueAndVReg> C;
561 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GCst(ValReg&: C))) {
562 if (C->Value.isAllOnes())
563 // If the operand is a G_CONSTANT with value of all ones it is larger than
564 // VLMAX. We convert it to an immediate with value VLMaxSentinel. This is
565 // recognized specially by the vsetvli insertion pass.
566 return {{[=](MachineInstrBuilder &MIB) {
567 MIB.addImm(Val: RISCV::VLMaxSentinel);
568 }}};
569
570 if (isUInt<5>(x: C->Value.getZExtValue())) {
571 uint64_t ZExtC = C->Value.getZExtValue();
572 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ZExtC); }}};
573 }
574 }
575 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Root.getReg()); }}};
576}
577
578InstructionSelector::ComplexRendererFns
579RISCVInstructionSelector::renderAddiPair(Register BaseReg, int64_t AddiImm,
580 int64_t OffsetImm) const {
581 return {{[=](MachineInstrBuilder &MIB) {
582 Register Tmp = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
583 MachineInstr *Addi =
584 BuildMI(BB&: *MIB->getParent(), I&: *MIB.getInstr(), MIMD: MIB->getDebugLoc(),
585 MCID: TII.get(Opcode: RISCV::ADDI), DestReg: Tmp)
586 .addReg(RegNo: BaseReg)
587 .addImm(Val: AddiImm);
588 constrainSelectedInstRegOperands(I&: *Addi, TII, TRI, RBI);
589 MIB.addReg(RegNo: Tmp);
590 },
591 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: OffsetImm); }}};
592}
593
594InstructionSelector::ComplexRendererFns
595RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
596 if (!Root.isReg())
597 return std::nullopt;
598
599 Register RootReg = Root.getReg();
600
601 // Frame index.
602 int FI;
603 if (mi_match(R: RootReg, MRI: *MRI, P: m_GFrameIndex(FI))) {
604 return {{
605 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); },
606 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
607 }};
608 }
609
610 // base + constant offset (G_PTR_ADD).
611 Register BaseReg;
612 int64_t RHSC;
613 if (mi_match(R: RootReg, MRI: *MRI, P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_ICst(Cst&: RHSC)))) {
614 if (isInt<12>(x: RHSC)) {
615 int BaseFI;
616 if (mi_match(R: BaseReg, MRI: *MRI, P: m_GFrameIndex(FI&: BaseFI)))
617 return {{
618 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: BaseFI); },
619 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
620 }};
621
622 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: BaseReg); },
623 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
624 }
625
626 // Large constant offset. Fold a -2048/2047 adjustment so the whole
627 // constant can be split across an ADDI and the load/store offset.
628 if (RHSC >= -4096 && RHSC <= 4094) {
629 int64_t Adj = RHSC < 0 ? -2048 : 2047;
630 return renderAddiPair(BaseReg, AddiImm: Adj, OffsetImm: RHSC - Adj);
631 }
632
633 if (isWorthFoldingAdd(AddResult: RootReg))
634 if (auto Fns = computeConstAddr(CVal: RHSC, /*IsPrefetch=*/false, OrigBase: BaseReg))
635 return Fns;
636 }
637
638 // Bare constant address. IRTranslator lowers inttoptr(C) to
639 // G_INTTOPTR(G_CONSTANT); look through it to reach the constant.
640 int64_t CVal;
641 if (mi_match(R: RootReg, MRI: *MRI, P: m_GIntToPtr(Src: m_ICst(Cst&: CVal))) ||
642 mi_match(R: RootReg, MRI: *MRI, P: m_ICst(Cst&: CVal))) {
643 if (auto Fns = computeConstAddr(CVal, /*IsPrefetch=*/false, OrigBase: Register()))
644 return Fns;
645 }
646
647 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); },
648 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
649}
650
651InstructionSelector::ComplexRendererFns
652RISCVInstructionSelector::selectAddrRegImmLsb00000(MachineOperand &Root) const {
653 if (!Root.isReg())
654 return std::nullopt;
655
656 Register RootReg = Root.getReg();
657
658 // Frame index.
659 int FI;
660 if (mi_match(R: RootReg, MRI: *MRI, P: m_GFrameIndex(FI))) {
661 return {{
662 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); },
663 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
664 }};
665 }
666
667 // base + constant offset (G_PTR_ADD).
668 Register BaseReg;
669 int64_t RHSC;
670 if (mi_match(R: RootReg, MRI: *MRI, P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_ICst(Cst&: RHSC)))) {
671 if (isInt<12>(x: RHSC)) {
672 // Not a multiple of 32: can't encode, use the address as-is.
673 if ((RHSC & 0b11111) != 0) {
674 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); },
675 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
676 }
677 // Fold the offset.
678 int BaseFI;
679 if (mi_match(R: BaseReg, MRI: *MRI, P: m_GFrameIndex(FI&: BaseFI)))
680 return {{
681 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: BaseFI); },
682 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
683 }};
684 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: BaseReg); },
685 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
686 }
687
688 // Large constant: fold a -2048/2016 adjustment to save an instruction.
689 if ((-2049 >= RHSC && RHSC >= -4096) || (4063 >= RHSC && RHSC >= 2017)) {
690 int64_t Adj = RHSC < 0 ? -2048 : 2016;
691 return renderAddiPair(BaseReg, AddiImm: RHSC - Adj, OffsetImm: Adj);
692 }
693
694 // Otherwise split the constant into Hi (materialized + added to the base)
695 // and Lo12 (folded offset).
696 if (auto Fns = computeConstAddr(CVal: RHSC, /*IsPrefetch=*/true, OrigBase: BaseReg))
697 return Fns;
698 }
699
700 // Bare constant address. IRTranslator emits inttoptr(C) as
701 // G_INTTOPTR(G_CONSTANT); look through the G_INTTOPTR to reach the constant.
702 int64_t CVal;
703 if (mi_match(R: RootReg, MRI: *MRI, P: m_GIntToPtr(Src: m_ICst(Cst&: CVal))) ||
704 mi_match(R: RootReg, MRI: *MRI, P: m_ICst(Cst&: CVal))) {
705 if (auto Fns = computeConstAddr(CVal, /*IsPrefetch=*/true, OrigBase: Register()))
706 return Fns;
707 }
708
709 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); },
710 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
711}
712
713/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
714/// CC Must be an ICMP Predicate.
715static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
716 switch (CC) {
717 default:
718 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
719 case CmpInst::Predicate::ICMP_EQ:
720 return RISCVCC::COND_EQ;
721 case CmpInst::Predicate::ICMP_NE:
722 return RISCVCC::COND_NE;
723 case CmpInst::Predicate::ICMP_ULT:
724 return RISCVCC::COND_LTU;
725 case CmpInst::Predicate::ICMP_SLT:
726 return RISCVCC::COND_LT;
727 case CmpInst::Predicate::ICMP_UGE:
728 return RISCVCC::COND_GEU;
729 case CmpInst::Predicate::ICMP_SGE:
730 return RISCVCC::COND_GE;
731 }
732}
733
734static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC,
735 Register &LHS, Register &RHS,
736 MachineRegisterInfo &MRI) {
737 // Try to fold an ICmp. If that fails, use a NE compare with X0.
738 CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
739 if (!mi_match(R: CondReg, MRI, P: m_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: LHS), R: m_Reg(R&: RHS)))) {
740 LHS = CondReg;
741 RHS = RISCV::X0;
742 CC = RISCVCC::COND_NE;
743 return;
744 }
745
746 // We found an ICmp, do some canonicalization.
747
748 // Adjust comparisons to use comparison with 0 if possible.
749 if (auto Constant = getIConstantVRegSExtVal(VReg: RHS, MRI)) {
750 switch (Pred) {
751 case CmpInst::Predicate::ICMP_SGT:
752 // Convert X > -1 to X >= 0
753 if (*Constant == -1) {
754 CC = RISCVCC::COND_GE;
755 RHS = RISCV::X0;
756 return;
757 }
758 break;
759 case CmpInst::Predicate::ICMP_SLT:
760 // Convert X < 1 to 0 >= X
761 if (*Constant == 1) {
762 CC = RISCVCC::COND_GE;
763 RHS = LHS;
764 LHS = RISCV::X0;
765 return;
766 }
767 break;
768 default:
769 break;
770 }
771 }
772
773 switch (Pred) {
774 default:
775 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
776 case CmpInst::Predicate::ICMP_EQ:
777 case CmpInst::Predicate::ICMP_NE:
778 case CmpInst::Predicate::ICMP_ULT:
779 case CmpInst::Predicate::ICMP_SLT:
780 case CmpInst::Predicate::ICMP_UGE:
781 case CmpInst::Predicate::ICMP_SGE:
782 // These CCs are supported directly by RISC-V branches.
783 break;
784 case CmpInst::Predicate::ICMP_SGT:
785 case CmpInst::Predicate::ICMP_SLE:
786 case CmpInst::Predicate::ICMP_UGT:
787 case CmpInst::Predicate::ICMP_ULE:
788 // These CCs are not supported directly by RISC-V branches, but changing the
789 // direction of the CC and swapping LHS and RHS are.
790 Pred = CmpInst::getSwappedPredicate(pred: Pred);
791 std::swap(a&: LHS, b&: RHS);
792 break;
793 }
794
795 CC = getRISCVCCFromICmp(CC: Pred);
796}
797
798/// Select the RISC-V Zalasr opcode for the G_LOAD or G_STORE operation
799/// \p GenericOpc, appropriate for the GPR register bank and of memory access
800/// size \p OpSize.
801static unsigned selectZalasrLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {
802 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
803 switch (OpSize) {
804 default:
805 llvm_unreachable("Unexpected memory size");
806 case 8:
807 return IsStore ? RISCV::SB_RL : RISCV::LB_AQ;
808 case 16:
809 return IsStore ? RISCV::SH_RL : RISCV::LH_AQ;
810 case 32:
811 return IsStore ? RISCV::SW_RL : RISCV::LW_AQ;
812 case 64:
813 return IsStore ? RISCV::SD_RL : RISCV::LD_AQ;
814 }
815}
816
817/// Select the RISC-V regimm opcode for the G_LOAD or G_STORE operation
818/// \p GenericOpc, appropriate for the GPR register bank and of memory access
819/// size \p OpSize. \returns \p GenericOpc if the combination is unsupported.
820static unsigned selectRegImmLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {
821 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
822 switch (OpSize) {
823 case 8:
824 // Prefer unsigned due to no c.lb in Zcb.
825 return IsStore ? RISCV::SB : RISCV::LBU;
826 case 16:
827 return IsStore ? RISCV::SH : RISCV::LH;
828 case 32:
829 return IsStore ? RISCV::SW : RISCV::LW;
830 case 64:
831 return IsStore ? RISCV::SD : RISCV::LD;
832 }
833
834 return GenericOpc;
835}
836
837void RISCVInstructionSelector::addVectorLoadStoreOperands(
838 MachineInstr &I, SmallVectorImpl<Register> &SrcOps, unsigned &CurOp,
839 bool IsMasked, bool IsStridedOrIndexed, LLT *IndexVT) const {
840 // Base Pointer
841 auto PtrReg = I.getOperand(i: CurOp++).getReg();
842 SrcOps.push_back(Elt: PtrReg);
843
844 // Stride or Index
845 if (IsStridedOrIndexed) {
846 auto StrideReg = I.getOperand(i: CurOp++).getReg();
847 SrcOps.push_back(Elt: StrideReg);
848 if (IndexVT)
849 *IndexVT = MRI->getType(Reg: StrideReg);
850 }
851
852 // Mask
853 if (IsMasked) {
854 auto MaskReg = I.getOperand(i: CurOp++).getReg();
855 SrcOps.push_back(Elt: MaskReg);
856 }
857}
858
859bool RISCVInstructionSelector::selectIntrinsicWithSideEffects(
860 MachineInstr &I) const {
861 // Find the intrinsic ID.
862 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
863 // Select the instruction.
864 switch (IntrinID) {
865 default:
866 return false;
867 case Intrinsic::riscv_vlm:
868 case Intrinsic::riscv_vle:
869 case Intrinsic::riscv_vle_mask:
870 case Intrinsic::riscv_vlse:
871 case Intrinsic::riscv_vlse_mask: {
872 bool IsMasked = IntrinID == Intrinsic::riscv_vle_mask ||
873 IntrinID == Intrinsic::riscv_vlse_mask;
874 bool IsStrided = IntrinID == Intrinsic::riscv_vlse ||
875 IntrinID == Intrinsic::riscv_vlse_mask;
876 LLT VT = MRI->getType(Reg: I.getOperand(i: 0).getReg());
877 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
878
879 // Result vector
880 const Register DstReg = I.getOperand(i: 0).getReg();
881
882 // Sources
883 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
884 unsigned CurOp = 2;
885 SmallVector<Register, 4> SrcOps; // Source registers.
886
887 // Passthru
888 if (HasPassthruOperand) {
889 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
890 SrcOps.push_back(Elt: PassthruReg);
891 } else {
892 SrcOps.push_back(Elt: Register(RISCV::NoRegister));
893 }
894
895 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: IsStrided);
896
897 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
898 const RISCV::VLEPseudo *P =
899 RISCV::getVLEPseudo(Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
900 LMUL: static_cast<unsigned>(LMUL));
901
902 MachineInstrBuilder PseudoMI =
903 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo), DestReg: DstReg);
904 for (Register Reg : SrcOps)
905 PseudoMI.addReg(RegNo: Reg);
906
907 // Select VL
908 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
909 for (auto &RenderFn : *VLOpFn)
910 RenderFn(PseudoMI);
911
912 // SEW
913 PseudoMI.addImm(Val: Log2SEW);
914
915 // Policy
916 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
917 if (IsMasked)
918 Policy = I.getOperand(i: CurOp++).getImm();
919 PseudoMI.addImm(Val: Policy);
920
921 // Memref
922 PseudoMI.cloneMemRefs(OtherMI: I);
923
924 I.eraseFromParent();
925 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
926 return true;
927 }
928 case Intrinsic::riscv_vloxei:
929 case Intrinsic::riscv_vloxei_mask:
930 case Intrinsic::riscv_vluxei:
931 case Intrinsic::riscv_vluxei_mask: {
932 bool IsMasked = IntrinID == Intrinsic::riscv_vloxei_mask ||
933 IntrinID == Intrinsic::riscv_vluxei_mask;
934 bool IsOrdered = IntrinID == Intrinsic::riscv_vloxei ||
935 IntrinID == Intrinsic::riscv_vloxei_mask;
936 LLT VT = MRI->getType(Reg: I.getOperand(i: 0).getReg());
937 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
938
939 // Result vector
940 const Register DstReg = I.getOperand(i: 0).getReg();
941
942 // Sources
943 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
944 unsigned CurOp = 2;
945 SmallVector<Register, 4> SrcOps; // Source registers.
946
947 // Passthru
948 if (HasPassthruOperand) {
949 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
950 SrcOps.push_back(Elt: PassthruReg);
951 } else {
952 // Use NoRegister if there is no specified passthru.
953 SrcOps.push_back(Elt: Register());
954 }
955 LLT IndexVT;
956 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: true, IndexVT: &IndexVT);
957
958 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
959 RISCVVType::VLMUL IndexLMUL =
960 RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: IndexVT));
961 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
962 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
963 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
964 "values when XLEN=32");
965 }
966 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
967 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
968 IndexLMUL: static_cast<unsigned>(IndexLMUL));
969
970 MachineInstrBuilder PseudoMI =
971 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo), DestReg: DstReg);
972 for (Register Reg : SrcOps)
973 PseudoMI.addReg(RegNo: Reg);
974
975 // Select VL
976 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
977 for (auto &RenderFn : *VLOpFn)
978 RenderFn(PseudoMI);
979
980 // SEW
981 PseudoMI.addImm(Val: Log2SEW);
982
983 // Policy
984 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
985 if (IsMasked)
986 Policy = I.getOperand(i: CurOp++).getImm();
987 PseudoMI.addImm(Val: Policy);
988
989 // Memref
990 PseudoMI.cloneMemRefs(OtherMI: I);
991
992 I.eraseFromParent();
993 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
994 return true;
995 }
996 case Intrinsic::riscv_vsm:
997 case Intrinsic::riscv_vse:
998 case Intrinsic::riscv_vse_mask:
999 case Intrinsic::riscv_vsse:
1000 case Intrinsic::riscv_vsse_mask: {
1001 bool IsMasked = IntrinID == Intrinsic::riscv_vse_mask ||
1002 IntrinID == Intrinsic::riscv_vsse_mask;
1003 bool IsStrided = IntrinID == Intrinsic::riscv_vsse ||
1004 IntrinID == Intrinsic::riscv_vsse_mask;
1005 LLT VT = MRI->getType(Reg: I.getOperand(i: 1).getReg());
1006 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
1007
1008 // Sources
1009 unsigned CurOp = 1;
1010 SmallVector<Register, 4> SrcOps; // Source registers.
1011
1012 // Store value
1013 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
1014 SrcOps.push_back(Elt: PassthruReg);
1015
1016 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: IsStrided);
1017
1018 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
1019 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
1020 Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
1021
1022 MachineInstrBuilder PseudoMI =
1023 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo));
1024 for (Register Reg : SrcOps)
1025 PseudoMI.addReg(RegNo: Reg);
1026
1027 // Select VL
1028 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
1029 for (auto &RenderFn : *VLOpFn)
1030 RenderFn(PseudoMI);
1031
1032 // SEW
1033 PseudoMI.addImm(Val: Log2SEW);
1034
1035 // Memref
1036 PseudoMI.cloneMemRefs(OtherMI: I);
1037
1038 I.eraseFromParent();
1039 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1040 return true;
1041 }
1042 case Intrinsic::riscv_vsoxei:
1043 case Intrinsic::riscv_vsoxei_mask:
1044 case Intrinsic::riscv_vsuxei:
1045 case Intrinsic::riscv_vsuxei_mask: {
1046 bool IsMasked = IntrinID == Intrinsic::riscv_vsoxei_mask ||
1047 IntrinID == Intrinsic::riscv_vsuxei_mask;
1048 bool IsOrdered = IntrinID == Intrinsic::riscv_vsoxei ||
1049 IntrinID == Intrinsic::riscv_vsoxei_mask;
1050 LLT VT = MRI->getType(Reg: I.getOperand(i: 1).getReg());
1051 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
1052
1053 // Sources
1054 unsigned CurOp = 1;
1055 SmallVector<Register, 4> SrcOps; // Source registers.
1056
1057 // Store value
1058 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
1059 SrcOps.push_back(Elt: PassthruReg);
1060
1061 LLT IndexVT;
1062 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: true, IndexVT: &IndexVT);
1063
1064 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
1065 RISCVVType::VLMUL IndexLMUL =
1066 RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: IndexVT));
1067 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
1068 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
1069 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
1070 "values when XLEN=32");
1071 }
1072 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
1073 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
1074 IndexLMUL: static_cast<unsigned>(IndexLMUL));
1075
1076 MachineInstrBuilder PseudoMI =
1077 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo));
1078 for (Register Reg : SrcOps)
1079 PseudoMI.addReg(RegNo: Reg);
1080
1081 // Select VL
1082 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
1083 for (auto &RenderFn : *VLOpFn)
1084 RenderFn(PseudoMI);
1085
1086 // SEW
1087 PseudoMI.addImm(Val: Log2SEW);
1088
1089 // Memref
1090 PseudoMI.cloneMemRefs(OtherMI: I);
1091
1092 I.eraseFromParent();
1093 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1094 return true;
1095 }
1096 }
1097}
1098
1099bool RISCVInstructionSelector::selectIntrinsic(MachineInstr &I) const {
1100 // Find the intrinsic ID.
1101 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
1102 // Select the instruction.
1103 switch (IntrinID) {
1104 default:
1105 return false;
1106 case Intrinsic::riscv_vsetvli:
1107 case Intrinsic::riscv_vsetvlimax: {
1108
1109 bool VLMax = IntrinID == Intrinsic::riscv_vsetvlimax;
1110
1111 unsigned Offset = VLMax ? 2 : 3;
1112 unsigned SEW = RISCVVType::decodeVSEW(VSEW: I.getOperand(i: Offset).getImm() & 0x7);
1113 RISCVVType::VLMUL VLMul =
1114 static_cast<RISCVVType::VLMUL>(I.getOperand(i: Offset + 1).getImm() & 0x7);
1115
1116 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMUL: VLMul, SEW, /*TailAgnostic*/ true,
1117 /*MaskAgnostic*/ true);
1118
1119 Register DstReg = I.getOperand(i: 0).getReg();
1120
1121 Register VLOperand;
1122 unsigned Opcode = RISCV::PseudoVSETVLI;
1123
1124 // Check if AVL is a constant that equals VLMAX.
1125 if (!VLMax) {
1126 Register AVLReg = I.getOperand(i: 2).getReg();
1127 if (auto AVLConst = getIConstantVRegValWithLookThrough(VReg: AVLReg, MRI: *MRI)) {
1128 uint64_t AVL = AVLConst->Value.getZExtValue();
1129 if (auto VLEN = Subtarget->getRealVLen()) {
1130 if (*VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == AVL)
1131 VLMax = true;
1132 }
1133 }
1134
1135 if (mi_match(R: AVLReg, MRI: *MRI, P: m_AllOnes()))
1136 VLMax = true;
1137 }
1138
1139 if (VLMax) {
1140 VLOperand = Register(RISCV::X0);
1141 Opcode = RISCV::PseudoVSETVLIX0;
1142 } else {
1143 Register AVLReg = I.getOperand(i: 2).getReg();
1144 VLOperand = AVLReg;
1145
1146 // Check if AVL is a small constant that can use PseudoVSETIVLI.
1147 if (auto AVLConst = getIConstantVRegValWithLookThrough(VReg: AVLReg, MRI: *MRI)) {
1148 uint64_t AVL = AVLConst->Value.getZExtValue();
1149 if (isUInt<5>(x: AVL)) {
1150 MachineInstr *PseudoMI =
1151 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1152 MCID: TII.get(Opcode: RISCV::PseudoVSETIVLI), DestReg: DstReg)
1153 .addImm(Val: AVL)
1154 .addImm(Val: VTypeI);
1155 I.eraseFromParent();
1156 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1157 return true;
1158 }
1159 }
1160 }
1161
1162 MachineInstr *PseudoMI =
1163 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode), DestReg: DstReg)
1164 .addReg(RegNo: VLOperand)
1165 .addImm(Val: VTypeI);
1166 I.eraseFromParent();
1167 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1168 return true;
1169 }
1170 }
1171}
1172
1173bool RISCVInstructionSelector::selectExtractSubvector(MachineInstr &MI) const {
1174 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_SUBVECTOR);
1175
1176 Register DstReg = MI.getOperand(i: 0).getReg();
1177 Register SrcReg = MI.getOperand(i: 1).getReg();
1178
1179 LLT DstTy = MRI->getType(Reg: DstReg);
1180 LLT SrcTy = MRI->getType(Reg: SrcReg);
1181
1182 unsigned Idx = static_cast<unsigned>(MI.getOperand(i: 2).getImm());
1183
1184 MVT DstMVT = getMVTForLLT(Ty: DstTy);
1185 MVT SrcMVT = getMVTForLLT(Ty: SrcTy);
1186
1187 unsigned SubRegIdx;
1188 std::tie(args&: SubRegIdx, args&: Idx) =
1189 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1190 VecVT: SrcMVT, SubVecVT: DstMVT, InsertExtractIdx: Idx, TRI: &TRI);
1191
1192 if (Idx != 0)
1193 return false;
1194
1195 unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: DstMVT);
1196 const TargetRegisterClass *DstRC = TRI.getRegClass(i: DstRegClassID);
1197 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1198 return false;
1199
1200 unsigned SrcRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: SrcMVT);
1201 const TargetRegisterClass *SrcRC = TRI.getRegClass(i: SrcRegClassID);
1202 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
1203 return false;
1204
1205 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1206 DestReg: DstReg)
1207 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubRegIdx);
1208
1209 MI.eraseFromParent();
1210 return true;
1211}
1212
1213bool RISCVInstructionSelector::selectInsertSubVector(MachineInstr &MI) const {
1214 assert(MI.getOpcode() == TargetOpcode::G_INSERT_SUBVECTOR);
1215
1216 Register DstReg = MI.getOperand(i: 0).getReg();
1217 Register VecReg = MI.getOperand(i: 1).getReg();
1218 Register SubVecReg = MI.getOperand(i: 2).getReg();
1219
1220 LLT VecTy = MRI->getType(Reg: VecReg);
1221 LLT SubVecTy = MRI->getType(Reg: SubVecReg);
1222
1223 MVT VecMVT = getMVTForLLT(Ty: VecTy);
1224 MVT SubVecMVT = getMVTForLLT(Ty: SubVecTy);
1225
1226 unsigned Idx = static_cast<unsigned>(MI.getOperand(i: 3).getImm());
1227
1228 unsigned SubRegIdx;
1229 std::tie(args&: SubRegIdx, args&: Idx) =
1230 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1231 VecVT: VecMVT, SubVecVT: SubVecMVT, InsertExtractIdx: Idx, TRI: &TRI);
1232
1233 // If the Idx hasn't been completely eliminated then this is a subvector
1234 // insert which doesn't naturally align to a vector register. These must
1235 // be handled using instructions to manipulate the vector registers.
1236 if (Idx != 0)
1237 return false;
1238
1239 // Constrain dst
1240 unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: VecMVT);
1241 const TargetRegisterClass *DstRC = TRI.getRegClass(i: DstRegClassID);
1242 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1243 return false;
1244
1245 // If we haven't set a SubRegIdx, then we must be going between
1246 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
1247 if (SubRegIdx == RISCV::NoSubRegister) {
1248 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecMVT) ==
1249 DstRegClassID &&
1250 "Unexpected subvector insert");
1251 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1252 DestReg: DstReg)
1253 .addReg(RegNo: SubVecReg);
1254 MI.eraseFromParent();
1255 return true;
1256 }
1257
1258 // Use INSERT_SUBREG to insert the subvector into the vector at the
1259 // appropriate subregister index.
1260 MachineInstr *Ins = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1261 MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DstReg)
1262 .addReg(RegNo: VecReg)
1263 .addReg(RegNo: SubVecReg)
1264 .addImm(Val: SubRegIdx);
1265
1266 MI.eraseFromParent();
1267 constrainSelectedInstRegOperands(I&: *Ins, TII, TRI, RBI);
1268 return true;
1269}
1270
1271bool RISCVInstructionSelector::select(MachineInstr &MI) {
1272 preISelLower(MI);
1273 const unsigned Opc = MI.getOpcode();
1274
1275 if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
1276 if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
1277 const Register DefReg = MI.getOperand(i: 0).getReg();
1278 const LLT DefTy = MRI->getType(Reg: DefReg);
1279
1280 const RegClassOrRegBank &RegClassOrBank =
1281 MRI->getRegClassOrRegBank(Reg: DefReg);
1282
1283 const TargetRegisterClass *DefRC =
1284 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
1285 if (!DefRC) {
1286 if (!DefTy.isValid()) {
1287 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
1288 return false;
1289 }
1290
1291 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
1292 DefRC = TRI.getRegClassForTypeOnBank(Ty: DefTy, RB, Is64Bit: STI.is64Bit());
1293 if (!DefRC) {
1294 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
1295 return false;
1296 }
1297 }
1298
1299 MI.setDesc(TII.get(Opcode: TargetOpcode::PHI));
1300 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI&: *MRI);
1301 }
1302
1303 // Certain non-generic instructions also need some special handling.
1304 if (MI.isCopy())
1305 return selectCopy(MI);
1306
1307 return true;
1308 }
1309
1310 if (selectImpl(I&: MI, CoverageInfo&: *CoverageInfo))
1311 return true;
1312
1313 switch (Opc) {
1314 case TargetOpcode::G_ANYEXT:
1315 case TargetOpcode::G_PTRTOINT:
1316 case TargetOpcode::G_INTTOPTR:
1317 case TargetOpcode::G_TRUNC:
1318 case TargetOpcode::G_FREEZE:
1319 return selectCopy(MI);
1320 case TargetOpcode::G_CONSTANT: {
1321 Register DstReg = MI.getOperand(i: 0).getReg();
1322 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
1323
1324 if (!materializeImm(Reg: DstReg, Imm, MI))
1325 return false;
1326
1327 MI.eraseFromParent();
1328 return true;
1329 }
1330 case TargetOpcode::G_ZEXT:
1331 case TargetOpcode::G_SEXT: {
1332 bool IsSigned = Opc != TargetOpcode::G_ZEXT;
1333 Register DstReg = MI.getOperand(i: 0).getReg();
1334 Register SrcReg = MI.getOperand(i: 1).getReg();
1335 LLT SrcTy = MRI->getType(Reg: SrcReg);
1336 unsigned SrcSize = SrcTy.getSizeInBits();
1337
1338 if (SrcTy.isVector())
1339 return false; // Should be handled by imported patterns.
1340
1341 assert((*RBI.getRegBank(DstReg, *MRI, TRI)).getID() ==
1342 RISCV::GPRBRegBankID &&
1343 "Unexpected ext regbank");
1344
1345 // Use addiw SrcReg, 0 (sext.w) for i32.
1346 if (IsSigned && SrcSize == 32) {
1347 MI.setDesc(TII.get(Opcode: RISCV::ADDIW));
1348 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
1349 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1350 return true;
1351 }
1352
1353 // Use add.uw SrcReg, X0 (zext.w) for i32 with Zba.
1354 if (!IsSigned && SrcSize == 32 && STI.hasStdExtZba()) {
1355 MI.setDesc(TII.get(Opcode: RISCV::ADD_UW));
1356 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::X0, /*isDef=*/false));
1357 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1358 return true;
1359 }
1360
1361 // Use sext.h/zext.h for i16 with Zbb.
1362 if (SrcSize == 16 &&
1363 (STI.hasStdExtZbb() || (!IsSigned && STI.hasStdExtZbkb()))) {
1364 MI.setDesc(TII.get(Opcode: IsSigned ? RISCV::SEXT_H
1365 : STI.is64Bit() ? RISCV::ZEXT_H_RV64
1366 : RISCV::ZEXT_H_RV32));
1367 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1368 return true;
1369 }
1370
1371 // Fall back to shift pair.
1372 Register ShiftLeftReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1373 MachineInstr *ShiftLeft = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1374 MCID: TII.get(Opcode: RISCV::SLLI), DestReg: ShiftLeftReg)
1375 .addReg(RegNo: SrcReg)
1376 .addImm(Val: STI.getXLen() - SrcSize);
1377 constrainSelectedInstRegOperands(I&: *ShiftLeft, TII, TRI, RBI);
1378 MachineInstr *ShiftRight =
1379 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1380 MCID: TII.get(Opcode: IsSigned ? RISCV::SRAI : RISCV::SRLI), DestReg: DstReg)
1381 .addReg(RegNo: ShiftLeftReg)
1382 .addImm(Val: STI.getXLen() - SrcSize);
1383 constrainSelectedInstRegOperands(I&: *ShiftRight, TII, TRI, RBI);
1384 MI.eraseFromParent();
1385 return true;
1386 }
1387 case TargetOpcode::G_FCONSTANT: {
1388 // TODO: Use constant pool for complex constants.
1389 Register DstReg = MI.getOperand(i: 0).getReg();
1390 const APFloat &FPimm = MI.getOperand(i: 1).getFPImm()->getValueAPF();
1391 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
1392 if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
1393 Register GPRReg;
1394 if (FPimm.isPosZero()) {
1395 GPRReg = RISCV::X0;
1396 } else {
1397 GPRReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1398 APInt Imm = FPimm.bitcastToAPInt();
1399 if (!materializeImm(Reg: GPRReg, Imm: Imm.getSExtValue(), MI))
1400 return false;
1401 }
1402
1403 unsigned Opcode = Size == 64 ? RISCV::FMV_D_X
1404 : Size == 32 ? RISCV::FMV_W_X
1405 : RISCV::FMV_H_X;
1406 MachineInstr *FMV = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1407 MCID: TII.get(Opcode), DestReg: DstReg)
1408 .addReg(RegNo: GPRReg);
1409 constrainSelectedInstRegOperands(I&: *FMV, TII, TRI, RBI);
1410 } else {
1411 // s64 on rv32
1412 assert(Size == 64 && !Subtarget->is64Bit() &&
1413 "Unexpected size or subtarget");
1414
1415 if (FPimm.isPosZero()) {
1416 // Optimize +0.0 to use fcvt.d.w
1417 MachineInstr *FCVT = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1418 MCID: TII.get(Opcode: RISCV::FCVT_D_W), DestReg: DstReg)
1419 .addReg(RegNo: RISCV::X0)
1420 .addImm(Val: RISCVFPRndMode::RNE);
1421 constrainSelectedInstRegOperands(I&: *FCVT, TII, TRI, RBI);
1422
1423 MI.eraseFromParent();
1424 return true;
1425 }
1426
1427 // Split into two pieces and build through the stack.
1428 Register GPRRegHigh = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1429 Register GPRRegLow = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1430 APInt Imm = FPimm.bitcastToAPInt();
1431 if (!materializeImm(Reg: GPRRegHigh, Imm: Imm.extractBits(numBits: 32, bitPosition: 32).getSExtValue(),
1432 MI))
1433 return false;
1434 if (!materializeImm(Reg: GPRRegLow, Imm: Imm.trunc(width: 32).getSExtValue(), MI))
1435 return false;
1436 MachineInstr *PairF64 =
1437 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1438 MCID: TII.get(Opcode: RISCV::BuildPairF64Pseudo), DestReg: DstReg)
1439 .addReg(RegNo: GPRRegLow)
1440 .addReg(RegNo: GPRRegHigh);
1441 constrainSelectedInstRegOperands(I&: *PairF64, TII, TRI, RBI);
1442 }
1443
1444 MI.eraseFromParent();
1445 return true;
1446 }
1447 case TargetOpcode::G_GLOBAL_VALUE: {
1448 auto *GV = MI.getOperand(i: 1).getGlobal();
1449 if (GV->isThreadLocal()) {
1450 // TODO: implement this case.
1451 return false;
1452 }
1453
1454 return selectAddr(MI, IsLocal: GV->isDSOLocal(), IsExternWeak: GV->hasExternalWeakLinkage());
1455 }
1456 case TargetOpcode::G_JUMP_TABLE:
1457 case TargetOpcode::G_CONSTANT_POOL:
1458 return selectAddr(MI);
1459 case TargetOpcode::G_BRCOND: {
1460 Register LHS, RHS;
1461 RISCVCC::CondCode CC;
1462 getOperandsForBranch(CondReg: MI.getOperand(i: 0).getReg(), CC, LHS, RHS, MRI&: *MRI);
1463
1464 MachineInstr *Bcc = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1465 MCID: TII.get(Opcode: RISCVCC::getBrCond(CC)))
1466 .addReg(RegNo: LHS)
1467 .addReg(RegNo: RHS)
1468 .addMBB(MBB: MI.getOperand(i: 1).getMBB());
1469 MI.eraseFromParent();
1470 constrainSelectedInstRegOperands(I&: *Bcc, TII, TRI, RBI);
1471 return true;
1472 }
1473 case TargetOpcode::G_BRINDIRECT:
1474 MI.setDesc(TII.get(Opcode: RISCV::PseudoBRIND));
1475 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
1476 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1477 return true;
1478 case TargetOpcode::G_SELECT:
1479 return selectSelect(MI);
1480 case TargetOpcode::G_FCMP:
1481 return selectFPCompare(MI);
1482 case TargetOpcode::G_FENCE: {
1483 AtomicOrdering FenceOrdering =
1484 static_cast<AtomicOrdering>(MI.getOperand(i: 0).getImm());
1485 SyncScope::ID FenceSSID =
1486 static_cast<SyncScope::ID>(MI.getOperand(i: 1).getImm());
1487 emitFence(FenceOrdering, FenceSSID, MI);
1488 MI.eraseFromParent();
1489 return true;
1490 }
1491 case TargetOpcode::G_IMPLICIT_DEF:
1492 return selectImplicitDef(MI);
1493 case TargetOpcode::G_UNMERGE_VALUES:
1494 return selectUnmergeValues(MI);
1495 case TargetOpcode::G_LOAD:
1496 case TargetOpcode::G_STORE: {
1497 GLoadStore &LdSt = cast<GLoadStore>(Val&: MI);
1498 const Register ValReg = LdSt.getReg(Idx: 0);
1499 const Register PtrReg = LdSt.getPointerReg();
1500 LLT PtrTy = MRI->getType(Reg: PtrReg);
1501
1502 const RegisterBank &RB = *RBI.getRegBank(Reg: ValReg, MRI: *MRI, TRI);
1503 if (RB.getID() != RISCV::GPRBRegBankID)
1504 return false;
1505
1506#ifndef NDEBUG
1507 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, *MRI, TRI);
1508 // Check that the pointer register is valid.
1509 assert(PtrRB.getID() == RISCV::GPRBRegBankID &&
1510 "Load/Store pointer operand isn't a GPR");
1511 assert(PtrTy.isPointer() && "Load/Store pointer operand isn't a pointer");
1512#endif
1513
1514 // Can only handle AddressSpace 0.
1515 if (PtrTy.getAddressSpace() != 0)
1516 return false;
1517
1518 unsigned MemSize = LdSt.getMemSizeInBits().getValue();
1519 AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();
1520
1521 if (isStrongerThanMonotonic(AO: Order)) {
1522 MI.setDesc(TII.get(Opcode: selectZalasrLoadStoreOp(GenericOpc: Opc, OpSize: MemSize)));
1523 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1524 return true;
1525 }
1526
1527 const unsigned NewOpc = selectRegImmLoadStoreOp(GenericOpc: MI.getOpcode(), OpSize: MemSize);
1528 if (NewOpc == MI.getOpcode())
1529 return false;
1530
1531 // Check if we can fold anything into the addressing mode.
1532 auto AddrModeFns = selectAddrRegImm(Root&: MI.getOperand(i: 1));
1533 if (!AddrModeFns)
1534 return false;
1535
1536 // Folded something. Create a new instruction and return it.
1537 MachineInstrBuilder NewInst =
1538 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: NewOpc));
1539 NewInst.setMIFlags(MI.getFlags());
1540 if (isa<GStore>(Val: MI))
1541 NewInst.addUse(RegNo: ValReg);
1542 else
1543 NewInst.addDef(RegNo: ValReg);
1544 NewInst.cloneMemRefs(OtherMI: MI);
1545 for (auto &Fn : *AddrModeFns)
1546 Fn(NewInst);
1547 MI.eraseFromParent();
1548
1549 constrainSelectedInstRegOperands(I&: *NewInst, TII, TRI, RBI);
1550 return true;
1551 }
1552 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1553 return selectIntrinsicWithSideEffects(I&: MI);
1554 case TargetOpcode::G_INTRINSIC:
1555 return selectIntrinsic(I&: MI);
1556 case TargetOpcode::G_EXTRACT_SUBVECTOR:
1557 return selectExtractSubvector(MI);
1558 case TargetOpcode::G_INSERT_SUBVECTOR:
1559 return selectInsertSubVector(MI);
1560 default:
1561 return false;
1562 }
1563}
1564
1565bool RISCVInstructionSelector::selectUnmergeValues(MachineInstr &MI) const {
1566 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
1567
1568 if (!Subtarget->hasStdExtZfa())
1569 return false;
1570
1571 // Split F64 Src into two s32 parts
1572 if (MI.getNumOperands() != 3)
1573 return false;
1574 Register Src = MI.getOperand(i: 2).getReg();
1575 Register Lo = MI.getOperand(i: 0).getReg();
1576 Register Hi = MI.getOperand(i: 1).getReg();
1577 if (!isRegInFprb(Reg: Src) || !isRegInGprb(Reg: Lo) || !isRegInGprb(Reg: Hi))
1578 return false;
1579
1580 MachineInstr *ExtractLo = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1581 MCID: TII.get(Opcode: RISCV::FMV_X_W_FPR64), DestReg: Lo)
1582 .addReg(RegNo: Src);
1583 constrainSelectedInstRegOperands(I&: *ExtractLo, TII, TRI, RBI);
1584
1585 MachineInstr *ExtractHi = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1586 MCID: TII.get(Opcode: RISCV::FMVH_X_D), DestReg: Hi)
1587 .addReg(RegNo: Src);
1588 constrainSelectedInstRegOperands(I&: *ExtractHi, TII, TRI, RBI);
1589
1590 MI.eraseFromParent();
1591 return true;
1592}
1593
1594bool RISCVInstructionSelector::replacePtrWithInt(MachineInstr &MI,
1595 unsigned OpIdx) {
1596 MachineOperand &Op = MI.getOperand(i: OpIdx);
1597 Register PtrReg = Op.getReg();
1598 assert(MRI->getType(PtrReg).isPointer() && "Operand is not a pointer!");
1599
1600 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1601 Register IntReg = MRI->createGenericVirtualRegister(Ty: sXLen);
1602 MRI->setRegBank(Reg: IntReg, RegBank: RBI.getRegBank(ID: RISCV::GPRBRegBankID));
1603 MachineInstr *PtrToInt = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1604 MCID: TII.get(Opcode: TargetOpcode::G_PTRTOINT), DestReg: IntReg)
1605 .addReg(RegNo: PtrReg);
1606 Op.setReg(IntReg);
1607 return select(MI&: *PtrToInt);
1608}
1609
1610void RISCVInstructionSelector::preISelLower(MachineInstr &MI) {
1611 switch (MI.getOpcode()) {
1612 case TargetOpcode::G_PTR_ADD: {
1613 Register DstReg = MI.getOperand(i: 0).getReg();
1614 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1615
1616 replacePtrWithInt(MI, OpIdx: 1);
1617 MI.setDesc(TII.get(Opcode: TargetOpcode::G_ADD));
1618 MRI->setType(VReg: DstReg, Ty: sXLen);
1619 break;
1620 }
1621 case TargetOpcode::G_PTRMASK: {
1622 Register DstReg = MI.getOperand(i: 0).getReg();
1623 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1624 replacePtrWithInt(MI, OpIdx: 1);
1625 MI.setDesc(TII.get(Opcode: TargetOpcode::G_AND));
1626 MRI->setType(VReg: DstReg, Ty: sXLen);
1627 break;
1628 }
1629 }
1630}
1631
1632void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
1633 const MachineInstr &MI,
1634 int OpIdx) const {
1635 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1636 "Expected G_CONSTANT");
1637 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
1638 MIB.addImm(Val: -CstVal);
1639}
1640
1641void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
1642 const MachineInstr &MI,
1643 int OpIdx) const {
1644 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1645 "Expected G_CONSTANT");
1646 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1647 MIB.addImm(Val: STI.getXLen() - CstVal);
1648}
1649
1650void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
1651 const MachineInstr &MI,
1652 int OpIdx) const {
1653 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1654 "Expected G_CONSTANT");
1655 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1656 MIB.addImm(Val: 32 - CstVal);
1657}
1658
1659void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
1660 const MachineInstr &MI,
1661 int OpIdx) const {
1662 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1663 "Expected G_CONSTANT");
1664 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
1665 MIB.addImm(Val: CstVal + 1);
1666}
1667
1668void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
1669 const MachineInstr &MI,
1670 int OpIdx) const {
1671 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1672 "Expected G_CONSTANT");
1673 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
1674 MIB.addImm(Val: llvm::countr_zero(Val: C));
1675}
1676
1677void RISCVInstructionSelector::renderXLenSubTrailingOnes(
1678 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
1679 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1680 "Expected G_CONSTANT");
1681 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
1682 MIB.addImm(Val: Subtarget->getXLen() - llvm::countr_one(Value: C));
1683}
1684
1685void RISCVInstructionSelector::renderAddiPairImmSmall(MachineInstrBuilder &MIB,
1686 const MachineInstr &MI,
1687 int OpIdx) const {
1688 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1689 "Expected G_CONSTANT");
1690 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
1691 int64_t Adj = Imm < 0 ? -2048 : 2047;
1692 MIB.addImm(Val: Imm - Adj);
1693}
1694
1695void RISCVInstructionSelector::renderAddiPairImmLarge(MachineInstrBuilder &MIB,
1696 const MachineInstr &MI,
1697 int OpIdx) const {
1698 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1699 "Expected G_CONSTANT");
1700 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue() < 0 ? -2048 : 2047;
1701 MIB.addImm(Val: Imm);
1702}
1703
1704bool RISCVInstructionSelector::isRegInGprb(Register Reg) const {
1705 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::GPRBRegBankID;
1706}
1707
1708bool RISCVInstructionSelector::isRegInFprb(Register Reg) const {
1709 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID;
1710}
1711
1712// A G_PTR_ADD result is worth splitting into Hi (materialized) +
1713// Lo12 (folded offset) only if every user is a plain scalar load/store
1714// using it as the address. Otherwise the ADD is selected on its own with
1715// the full materialized constant, making the Hi materialization here redundant.
1716bool RISCVInstructionSelector::isWorthFoldingAdd(Register AddResult) const {
1717 for (const MachineInstr &User : MRI->use_nodbg_instructions(Reg: AddResult)) {
1718 auto *LdSt = dyn_cast<GLoadStore>(Val: &User);
1719 if (!LdSt)
1720 return false;
1721 // Must be used as the pointer, not the stored value.
1722 if (LdSt->getPointerReg() != AddResult)
1723 return false;
1724 if (isStrongerThanMonotonic(AO: LdSt->getMMO().getSuccessOrdering()))
1725 return false;
1726 // Only scalar integer/f16/f32/f64 memory (exclude vectors, f128, ...).
1727 LLT Ty = MRI->getType(Reg: User.getOperand(i: 0).getReg());
1728 if (!Ty.isScalar() || Ty.getSizeInBits() > 64)
1729 return false;
1730 }
1731 return true;
1732}
1733
1734bool RISCVInstructionSelector::selectCopy(MachineInstr &MI) const {
1735 Register DstReg = MI.getOperand(i: 0).getReg();
1736
1737 if (DstReg.isPhysical())
1738 return true;
1739
1740 const TargetRegisterClass *DstRC =
1741 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
1742
1743 assert(DstRC &&
1744 "Register class not available for LLT, register bank combination");
1745
1746 // No need to constrain SrcReg. It will get constrained when
1747 // we hit another of its uses or its defs.
1748 // Copies do not have constraints.
1749 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
1750 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1751 << " operand\n");
1752 return false;
1753 }
1754
1755 MI.setDesc(TII.get(Opcode: RISCV::COPY));
1756 return true;
1757}
1758
1759bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI) const {
1760 assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
1761
1762 const Register DstReg = MI.getOperand(i: 0).getReg();
1763 const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(
1764 Ty: MRI->getType(Reg: DstReg), RB: *RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI), Is64Bit: STI.is64Bit());
1765
1766 assert(DstRC &&
1767 "Register class not available for LLT, register bank combination");
1768
1769 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
1770 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1771 << " operand\n");
1772 }
1773 MI.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
1774 return true;
1775}
1776
1777bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
1778 MachineInstr &MI) const {
1779 if (Imm == 0) {
1780 MachineBasicBlock &MBB = *MI.getParent();
1781 DebugLoc DL = MI.getDebugLoc();
1782 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg).addReg(RegNo: RISCV::X0);
1783 RBI.constrainGenericRegister(Reg: DstReg, RC: RISCV::GPRRegClass, MRI&: *MRI);
1784 return true;
1785 }
1786
1787 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: *Subtarget);
1788 return materializeInstSeq(DstReg, Seq, MI);
1789}
1790
1791bool RISCVInstructionSelector::materializeInstSeq(
1792 Register DstReg, const RISCVMatInt::InstSeq &Seq, MachineInstr &MI) const {
1793 assert(!Seq.empty() && "materializeInstSeq requires a non-empty sequence");
1794
1795 MachineBasicBlock &MBB = *MI.getParent();
1796 DebugLoc DL = MI.getDebugLoc();
1797 unsigned NumInsts = Seq.size();
1798 Register SrcReg = RISCV::X0;
1799
1800 for (unsigned i = 0; i < NumInsts; i++) {
1801 Register TmpReg = i < NumInsts - 1
1802 ? MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass)
1803 : DstReg;
1804 const RISCVMatInt::Inst &I = Seq[i];
1805 MachineInstr *Result;
1806
1807 switch (I.getOpndKind()) {
1808 case RISCVMatInt::Imm:
1809 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1810 .addImm(Val: I.getImm());
1811 break;
1812 case RISCVMatInt::RegX0:
1813 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1814 .addReg(RegNo: SrcReg)
1815 .addReg(RegNo: RISCV::X0);
1816 break;
1817 case RISCVMatInt::RegReg:
1818 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1819 .addReg(RegNo: SrcReg)
1820 .addReg(RegNo: SrcReg);
1821 break;
1822 case RISCVMatInt::RegImm:
1823 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1824 .addReg(RegNo: SrcReg)
1825 .addImm(Val: I.getImm());
1826 break;
1827 }
1828
1829 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1830
1831 SrcReg = TmpReg;
1832 }
1833
1834 return true;
1835}
1836
1837InstructionSelector::ComplexRendererFns
1838RISCVInstructionSelector::computeConstAddr(int64_t CVal, bool IsPrefetch,
1839 Register OrigBase) const {
1840 // Split the constant into a materialized high part (the base) and
1841 // a simm12 low part (the offset). For prefetch the low part
1842 // must additionally be a multiple of 32 (simm12_lsb00000).
1843 int64_t Lo12 = SignExtend64<12>(x: CVal);
1844 int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
1845 auto emit = [&](ConstAddrPlan Plan) -> ComplexRendererFns {
1846 return {{[=](MachineInstrBuilder &MIB) {
1847 MIB.addReg(RegNo: materializeConstBase(MIB, Plan, OrigBase));
1848 },
1849 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Plan.Lo12); }}};
1850 };
1851 if (!Subtarget->is64Bit() || isInt<32>(x: Hi)) {
1852 if (IsPrefetch && (Lo12 & 0b11111) != 0)
1853 return std::nullopt;
1854 ConstAddrPlan Plan;
1855 Plan.Lo12 = Lo12;
1856 if (Hi) {
1857 Plan.Kind = ConstAddrPlan::LUI;
1858 Plan.Hi20 = (Hi >> 12) & 0xfffff;
1859 }
1860 return emit(std::move(Plan));
1861 }
1862
1863 // Otherwise ask constant materialization how it would handle the constant
1864 // and fold the trailing ADDI into the offset.
1865 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: CVal, STI: *Subtarget);
1866 if (Seq.back().getOpcode() != RISCV::ADDI)
1867 return std::nullopt;
1868 Lo12 = Seq.back().getImm();
1869 if (IsPrefetch && (Lo12 & 0b11111) != 0)
1870 return std::nullopt;
1871 Seq.pop_back();
1872 if (Seq.empty())
1873 return std::nullopt;
1874 ConstAddrPlan Plan;
1875 Plan.Kind = ConstAddrPlan::InstSeq;
1876 Plan.Seq = std::move(Seq);
1877 Plan.Lo12 = Lo12;
1878 return emit(std::move(Plan));
1879}
1880
1881Register
1882RISCVInstructionSelector::materializeConstBase(MachineInstrBuilder &MIB,
1883 const ConstAddrPlan &Plan,
1884 Register OrigBase) const {
1885 MachineBasicBlock &MBB = *MIB->getParent();
1886 DebugLoc DL = MIB->getDebugLoc();
1887 MachineInstr &InsertPt = *MIB.getInstr();
1888
1889 Register HiReg = RISCV::X0;
1890 switch (Plan.Kind) {
1891 case ConstAddrPlan::X0:
1892 break;
1893 case ConstAddrPlan::LUI: {
1894 HiReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1895 MachineInstr *LUI = BuildMI(BB&: MBB, I&: InsertPt, MIMD: DL, MCID: TII.get(Opcode: RISCV::LUI), DestReg: HiReg)
1896 .addImm(Val: Plan.Hi20);
1897 constrainSelectedInstRegOperands(I&: *LUI, TII, TRI, RBI);
1898 break;
1899 }
1900 case ConstAddrPlan::InstSeq: {
1901 HiReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1902 materializeInstSeq(DstReg: HiReg, Seq: Plan.Seq, MI&: InsertPt);
1903 break;
1904 }
1905 }
1906
1907 // For G_PTR_ADD + large constant, add the original base to the materialized
1908 // high part.
1909 if (OrigBase.isValid() && HiReg != RISCV::X0) {
1910 Register BaseReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1911 MachineInstr *Add = BuildMI(BB&: MBB, I&: InsertPt, MIMD: DL, MCID: TII.get(Opcode: RISCV::ADD), DestReg: BaseReg)
1912 .addReg(RegNo: OrigBase)
1913 .addReg(RegNo: HiReg);
1914 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
1915 return BaseReg;
1916 }
1917 return OrigBase.isValid() ? OrigBase : HiReg;
1918}
1919
1920bool RISCVInstructionSelector::selectAddr(MachineInstr &MI, bool IsLocal,
1921 bool IsExternWeak) const {
1922 assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1923 MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
1924 MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
1925 "Unexpected opcode");
1926
1927 const MachineOperand &DispMO = MI.getOperand(i: 1);
1928
1929 Register DefReg = MI.getOperand(i: 0).getReg();
1930 const LLT DefTy = MRI->getType(Reg: DefReg);
1931
1932 // When HWASAN is used and tagging of global variables is enabled
1933 // they should be accessed via the GOT, since the tagged address of a global
1934 // is incompatible with existing code models. This also applies to non-pic
1935 // mode.
1936 if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
1937 if (IsLocal && !Subtarget->allowTaggedGlobals()) {
1938 // Use PC-relative addressing to access the symbol. This generates the
1939 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
1940 // %pcrel_lo(auipc)).
1941 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
1942 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1943 return true;
1944 }
1945
1946 // Use PC-relative addressing to access the GOT for this symbol, then
1947 // load the address from the GOT. This generates the pattern (PseudoLGA
1948 // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
1949 // %pcrel_lo(auipc))).
1950 MachineFunction &MF = *MI.getParent()->getParent();
1951 MachineMemOperand *MemOp = MF.getMachineMemOperand(
1952 PtrInfo: MachinePointerInfo::getGOT(MF),
1953 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1954 MachineMemOperand::MOInvariant,
1955 MemTy: DefTy, BaseAlignment: Align(DefTy.getSizeInBits() / 8));
1956
1957 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1958 MCID: TII.get(Opcode: RISCV::PseudoLGA), DestReg: DefReg)
1959 .addDisp(Disp: DispMO, off: 0)
1960 .addMemOperand(MMO: MemOp);
1961
1962 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1963
1964 MI.eraseFromParent();
1965 return true;
1966 }
1967
1968 switch (TM.getCodeModel()) {
1969 default: {
1970 reportGISelFailure(MF&: *MF, MORE&: *MORE, PassName: getName(),
1971 Msg: "Unsupported code model for lowering", MI);
1972 return false;
1973 }
1974 case CodeModel::Small: {
1975 // Must lie within a single 2 GiB address range and must lie between
1976 // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
1977 // (lui %hi(sym)) %lo(sym)).
1978 Register AddrHiDest = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1979 MachineInstr *AddrHi = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1980 MCID: TII.get(Opcode: RISCV::LUI), DestReg: AddrHiDest)
1981 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_HI);
1982
1983 constrainSelectedInstRegOperands(I&: *AddrHi, TII, TRI, RBI);
1984
1985 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1986 MCID: TII.get(Opcode: RISCV::ADDI), DestReg: DefReg)
1987 .addReg(RegNo: AddrHiDest)
1988 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_LO);
1989
1990 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1991
1992 MI.eraseFromParent();
1993 return true;
1994 }
1995 case CodeModel::Medium:
1996 // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
1997 // relocation needs to reference a label that points to the auipc
1998 // instruction itself, not the global. This cannot be done inside the
1999 // instruction selector.
2000 if (IsExternWeak) {
2001 // An extern weak symbol may be undefined, i.e. have value 0, which may
2002 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
2003 // symbol. This generates the pattern (PseudoLGA sym), which expands to
2004 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
2005 MachineFunction &MF = *MI.getParent()->getParent();
2006 MachineMemOperand *MemOp = MF.getMachineMemOperand(
2007 PtrInfo: MachinePointerInfo::getGOT(MF),
2008 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
2009 MachineMemOperand::MOInvariant,
2010 MemTy: DefTy, BaseAlignment: Align(DefTy.getSizeInBits() / 8));
2011
2012 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2013 MCID: TII.get(Opcode: RISCV::PseudoLGA), DestReg: DefReg)
2014 .addDisp(Disp: DispMO, off: 0)
2015 .addMemOperand(MMO: MemOp);
2016
2017 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
2018
2019 MI.eraseFromParent();
2020 return true;
2021 }
2022
2023 // Generate a sequence for accessing addresses within any 2GiB range
2024 // within the address space. This generates the pattern (PseudoLLA sym),
2025 // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
2026 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
2027 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
2028 return true;
2029 }
2030
2031 return false;
2032}
2033
2034bool RISCVInstructionSelector::selectSelect(MachineInstr &MI) const {
2035 auto &SelectMI = cast<GSelect>(Val&: MI);
2036
2037 Register LHS, RHS;
2038 RISCVCC::CondCode CC;
2039 getOperandsForBranch(CondReg: SelectMI.getCondReg(), CC, LHS, RHS, MRI&: *MRI);
2040
2041 Register DstReg = SelectMI.getReg(Idx: 0);
2042
2043 unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
2044 if (RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
2045 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
2046 Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
2047 : RISCV::Select_FPR64_Using_CC_GPR;
2048 }
2049
2050 MachineInstr *Result =
2051 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: Opc))
2052 .addDef(RegNo: DstReg)
2053 .addReg(RegNo: LHS)
2054 .addReg(RegNo: RHS)
2055 .addImm(Val: CC)
2056 .addReg(RegNo: SelectMI.getTrueReg())
2057 .addReg(RegNo: SelectMI.getFalseReg());
2058 MI.eraseFromParent();
2059 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
2060 return true;
2061}
2062
2063// Convert an FCMP predicate to one of the supported F or D instructions.
2064static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
2065 assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");
2066 switch (Pred) {
2067 default:
2068 llvm_unreachable("Unsupported predicate");
2069 case CmpInst::FCMP_OLT:
2070 return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
2071 case CmpInst::FCMP_OLE:
2072 return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
2073 case CmpInst::FCMP_OEQ:
2074 return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
2075 }
2076}
2077
2078// Try legalizing an FCMP by swapping or inverting the predicate to one that
2079// is supported.
2080static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,
2081 CmpInst::Predicate &Pred, bool &NeedInvert) {
2082 auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
2083 return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
2084 Pred == CmpInst::FCMP_OEQ;
2085 };
2086
2087 assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
2088
2089 CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(pred: Pred);
2090 if (isLegalFCmpPredicate(InvPred)) {
2091 Pred = InvPred;
2092 std::swap(a&: LHS, b&: RHS);
2093 return true;
2094 }
2095
2096 InvPred = CmpInst::getInversePredicate(pred: Pred);
2097 NeedInvert = true;
2098 if (isLegalFCmpPredicate(InvPred)) {
2099 Pred = InvPred;
2100 return true;
2101 }
2102 InvPred = CmpInst::getSwappedPredicate(pred: InvPred);
2103 if (isLegalFCmpPredicate(InvPred)) {
2104 Pred = InvPred;
2105 std::swap(a&: LHS, b&: RHS);
2106 return true;
2107 }
2108
2109 return false;
2110}
2111
2112// Emit a sequence of instructions to compare LHS and RHS using Pred. Return
2113// the result in DstReg.
2114// FIXME: Maybe we should expand this earlier.
2115bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI) const {
2116 auto &CmpMI = cast<GFCmp>(Val&: MI);
2117 CmpInst::Predicate Pred = CmpMI.getCond();
2118
2119 Register DstReg = CmpMI.getReg(Idx: 0);
2120 Register LHS = CmpMI.getLHSReg();
2121 Register RHS = CmpMI.getRHSReg();
2122
2123 unsigned Size = MRI->getType(Reg: LHS).getSizeInBits();
2124 assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
2125
2126 Register TmpReg = DstReg;
2127
2128 bool NeedInvert = false;
2129 // First try swapping operands or inverting.
2130 if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
2131 if (NeedInvert)
2132 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2133 MachineInstr *Cmp = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2134 MCID: TII.get(Opcode: getFCmpOpcode(Pred, Size)), DestReg: TmpReg)
2135 .addReg(RegNo: LHS)
2136 .addReg(RegNo: RHS);
2137 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
2138 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
2139 // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
2140 NeedInvert = Pred == CmpInst::FCMP_UEQ;
2141 Register Cmp1Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2142 MachineInstr *Cmp1 =
2143 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2144 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size)), DestReg: Cmp1Reg)
2145 .addReg(RegNo: LHS)
2146 .addReg(RegNo: RHS);
2147 constrainSelectedInstRegOperands(I&: *Cmp1, TII, TRI, RBI);
2148 Register Cmp2Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2149 MachineInstr *Cmp2 =
2150 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2151 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size)), DestReg: Cmp2Reg)
2152 .addReg(RegNo: RHS)
2153 .addReg(RegNo: LHS);
2154 constrainSelectedInstRegOperands(I&: *Cmp2, TII, TRI, RBI);
2155 if (NeedInvert)
2156 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2157 MachineInstr *Or = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2158 MCID: TII.get(Opcode: RISCV::OR), DestReg: TmpReg)
2159 .addReg(RegNo: Cmp1Reg)
2160 .addReg(RegNo: Cmp2Reg);
2161 constrainSelectedInstRegOperands(I&: *Or, TII, TRI, RBI);
2162 } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
2163 // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
2164 // If LHS and RHS are the same, a single FEQ suffices.
2165 NeedInvert = Pred == CmpInst::FCMP_UNO;
2166 if (NeedInvert)
2167 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2168 if (LHS == RHS) {
2169 MachineInstr *Cmp =
2170 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2171 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: TmpReg)
2172 .addReg(RegNo: LHS)
2173 .addReg(RegNo: LHS);
2174 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
2175 } else {
2176 Register Cmp1Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2177 MachineInstr *Cmp1 =
2178 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2179 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: Cmp1Reg)
2180 .addReg(RegNo: LHS)
2181 .addReg(RegNo: LHS);
2182 constrainSelectedInstRegOperands(I&: *Cmp1, TII, TRI, RBI);
2183 Register Cmp2Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2184 MachineInstr *Cmp2 =
2185 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2186 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: Cmp2Reg)
2187 .addReg(RegNo: RHS)
2188 .addReg(RegNo: RHS);
2189 constrainSelectedInstRegOperands(I&: *Cmp2, TII, TRI, RBI);
2190 MachineInstr *And = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2191 MCID: TII.get(Opcode: RISCV::AND), DestReg: TmpReg)
2192 .addReg(RegNo: Cmp1Reg)
2193 .addReg(RegNo: Cmp2Reg);
2194 constrainSelectedInstRegOperands(I&: *And, TII, TRI, RBI);
2195 }
2196 } else
2197 llvm_unreachable("Unhandled predicate");
2198
2199 // Emit an XORI to invert the result if needed.
2200 if (NeedInvert) {
2201 MachineInstr *Xor = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2202 MCID: TII.get(Opcode: RISCV::XORI), DestReg: DstReg)
2203 .addReg(RegNo: TmpReg)
2204 .addImm(Val: 1);
2205 constrainSelectedInstRegOperands(I&: *Xor, TII, TRI, RBI);
2206 }
2207
2208 MI.eraseFromParent();
2209 return true;
2210}
2211
2212void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
2213 SyncScope::ID FenceSSID,
2214 MachineInstr &MI) const {
2215 MachineBasicBlock &MBB = *MI.getParent();
2216 DebugLoc DL = MI.getDebugLoc();
2217
2218 if (STI.hasStdExtZtso()) {
2219 // The only fence that needs an instruction is a sequentially-consistent
2220 // cross-thread fence.
2221 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
2222 FenceSSID == SyncScope::System) {
2223 // fence rw, rw
2224 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE))
2225 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W)
2226 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W);
2227 return;
2228 }
2229
2230 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
2231 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER));
2232 return;
2233 }
2234
2235 // singlethread fences only synchronize with signal handlers on the same
2236 // thread and thus only need to preserve instruction order, not actually
2237 // enforce memory ordering.
2238 if (FenceSSID == SyncScope::SingleThread) {
2239 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER));
2240 return;
2241 }
2242
2243 // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
2244 // Manual: Volume I.
2245 unsigned Pred, Succ;
2246 switch (FenceOrdering) {
2247 default:
2248 llvm_unreachable("Unexpected ordering");
2249 case AtomicOrdering::AcquireRelease:
2250 // fence acq_rel -> fence.tso
2251 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE_TSO));
2252 return;
2253 case AtomicOrdering::Acquire:
2254 // fence acquire -> fence r, rw
2255 Pred = RISCVFenceField::R;
2256 Succ = RISCVFenceField::R | RISCVFenceField::W;
2257 break;
2258 case AtomicOrdering::Release:
2259 // fence release -> fence rw, w
2260 Pred = RISCVFenceField::R | RISCVFenceField::W;
2261 Succ = RISCVFenceField::W;
2262 break;
2263 case AtomicOrdering::SequentiallyConsistent:
2264 // fence seq_cst -> fence rw, rw
2265 Pred = RISCVFenceField::R | RISCVFenceField::W;
2266 Succ = RISCVFenceField::R | RISCVFenceField::W;
2267 break;
2268 }
2269 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE)).addImm(Val: Pred).addImm(Val: Succ);
2270}
2271
2272namespace llvm {
2273InstructionSelector *
2274createRISCVInstructionSelector(const RISCVTargetMachine &TM,
2275 const RISCVSubtarget &Subtarget,
2276 const RISCVRegisterBankInfo &RBI) {
2277 return new RISCVInstructionSelector(TM, Subtarget, RBI);
2278}
2279} // end namespace llvm
2280