1//===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// RISC-V.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/RISCVMatInt.h"
15#include "RISCVRegisterBankInfo.h"
16#include "RISCVSubtarget.h"
17#include "RISCVTargetMachine.h"
18#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
19#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
20#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
21#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
22#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/MachineJumpTableInfo.h"
25#include "llvm/IR/IntrinsicsRISCV.h"
26#include "llvm/Support/Debug.h"
27
28#define DEBUG_TYPE "riscv-isel"
29
30using namespace llvm;
31using namespace MIPatternMatch;
32
33#define GET_GLOBALISEL_PREDICATE_BITSET
34#include "RISCVGenGlobalISel.inc"
35#undef GET_GLOBALISEL_PREDICATE_BITSET
36
37namespace {
38
39class RISCVInstructionSelector : public InstructionSelector {
40public:
41 RISCVInstructionSelector(const RISCVTargetMachine &TM,
42 const RISCVSubtarget &STI,
43 const RISCVRegisterBankInfo &RBI);
44
45 bool select(MachineInstr &MI) override;
46
47 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
48 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
49 BlockFrequencyInfo *BFI) override {
50 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
51 MRI = &MF.getRegInfo();
52 }
53
54 static const char *getName() { return DEBUG_TYPE; }
55
56private:
57 static constexpr unsigned MaxRecursionDepth = 6;
58
59 bool hasAllNBitUsers(const MachineInstr &MI, unsigned Bits,
60 const unsigned Depth = 0) const;
61 bool hasAllHUsers(const MachineInstr &MI) const {
62 return hasAllNBitUsers(MI, Bits: 16);
63 }
64 bool hasAllWUsers(const MachineInstr &MI) const {
65 return hasAllNBitUsers(MI, Bits: 32);
66 }
67
68 bool isRegInGprb(Register Reg) const;
69 bool isRegInFprb(Register Reg) const;
70 bool isWorthFoldingAdd(Register AddResult) const;
71
72 // tblgen-erated 'select' implementation, used as the initial selector for
73 // the patterns that don't require complex C++.
74 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
75
76 // A lowering phase that runs before any selection attempts.
77 // Returns true if the instruction was modified.
78 void preISelLower(MachineInstr &MI);
79
80 bool replacePtrWithInt(MachineInstr &MI, unsigned OpIdx);
81
82 // Custom selection methods
83 bool selectCopy(MachineInstr &MI) const;
84 bool selectImplicitDef(MachineInstr &MI) const;
85 bool materializeImm(Register Reg, int64_t Imm, MachineInstr &MI) const;
86 // Emit a constant-materialization instruction sequence.
87 bool materializeInstSeq(Register DstReg, const RISCVMatInt::InstSeq &Seq,
88 MachineInstr &MI) const;
89 bool selectAddr(MachineInstr &MI, bool IsLocal = true,
90 bool IsExternWeak = false) const;
91 bool selectSelect(MachineInstr &MI) const;
92 bool selectFPCompare(MachineInstr &MI) const;
93 void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
94 MachineInstr &MI) const;
95 bool selectUnmergeValues(MachineInstr &MI) const;
96 bool selectMergeValues(MachineInstr &MI) const;
97 void addVectorLoadStoreOperands(MachineInstr &I,
98 SmallVectorImpl<Register> &SrcOps,
99 unsigned &CurOp, bool IsMasked,
100 bool IsStridedOrIndexed,
101 LLT *IndexVT = nullptr) const;
102 bool selectIntrinsicWithSideEffects(MachineInstr &I) const;
103 bool selectIntrinsic(MachineInstr &I) const;
104 bool selectExtractSubvector(MachineInstr &MI) const;
105 bool selectInsertSubVector(MachineInstr &I) const;
106 ComplexRendererFns selectShiftMask(MachineOperand &Root,
107 unsigned ShiftWidth) const;
108 ComplexRendererFns selectShiftMaskXLen(MachineOperand &Root) const {
109 return selectShiftMask(Root, ShiftWidth: STI.getXLen());
110 }
111 ComplexRendererFns selectShiftMask32(MachineOperand &Root) const {
112 return selectShiftMask(Root, ShiftWidth: 32);
113 }
114 ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
115 ComplexRendererFns selectBrindRegImm(MachineOperand &Root) const;
116 ComplexRendererFns selectAddrRegImmLsb00000(MachineOperand &Root) const;
117
118 // Plan for materializing a constant address as (Hi materialization, Lo12
119 // offset). Lo12 is a simm12 that, for prefetch (IsPrefetch), must be
120 // a multiple of 32.
121 struct ConstAddrPlan {
122 enum { X0, LUI, InstSeq } Kind = X0;
123 int64_t Hi20 = 0;
124 RISCVMatInt::InstSeq Seq;
125 int64_t Lo12 = 0;
126 };
127 ComplexRendererFns computeConstAddr(int64_t CVal, bool IsPrefetch,
128 Register OrigBase) const;
129 // Materialize the high part of Plan into a register. If OrigBase is valid,
130 // ADD it to the materialized high part (for G_PTR_ADD + large constant).
131 Register materializeConstBase(MachineInstrBuilder &MIB,
132 const ConstAddrPlan &Plan,
133 Register OrigBase) const;
134
135 ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;
136 template <unsigned Bits>
137 ComplexRendererFns selectSExtBits(MachineOperand &Root) const {
138 return selectSExtBits(Root, Bits);
139 }
140
141 ComplexRendererFns selectZExtBits(MachineOperand &Root, unsigned Bits) const;
142 template <unsigned Bits>
143 ComplexRendererFns selectZExtBits(MachineOperand &Root) const {
144 return selectZExtBits(Root, Bits);
145 }
146
147 ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
148 template <unsigned ShAmt>
149 ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
150 return selectSHXADDOp(Root, ShAmt);
151 }
152
153 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
154 unsigned ShAmt) const;
155 template <unsigned ShAmt>
156 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
157 return selectSHXADD_UWOp(Root, ShAmt);
158 }
159
160 ComplexRendererFns renderVLOp(MachineOperand &Root) const;
161 ComplexRendererFns renderAddiPair(Register BaseReg, int64_t AddiImm,
162 int64_t OffsetImm) const;
163 // Custom renderers for tablegen
164 void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
165 int OpIdx) const;
166 void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
167 int OpIdx) const;
168 void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
169 int OpIdx) const;
170 void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
171 int OpIdx) const;
172
173 void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
174 int OpIdx) const;
175 void renderXLenSubTrailingOnes(MachineInstrBuilder &MIB,
176 const MachineInstr &MI, int OpIdx) const;
177
178 void renderAddiPairImmLarge(MachineInstrBuilder &MIB, const MachineInstr &MI,
179 int OpIdx) const;
180 void renderAddiPairImmSmall(MachineInstrBuilder &MIB, const MachineInstr &MI,
181 int OpIdx) const;
182
183 const RISCVSubtarget &STI;
184 const RISCVInstrInfo &TII;
185 const RISCVRegisterInfo &TRI;
186 const RISCVRegisterBankInfo &RBI;
187 const RISCVTargetMachine &TM;
188
189 MachineRegisterInfo *MRI = nullptr;
190
191 // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
192 // uses "STI." in the code generated by TableGen. We need to unify the name of
193 // Subtarget variable.
194 const RISCVSubtarget *Subtarget = &STI;
195
196#define GET_GLOBALISEL_PREDICATES_DECL
197#include "RISCVGenGlobalISel.inc"
198#undef GET_GLOBALISEL_PREDICATES_DECL
199
200#define GET_GLOBALISEL_TEMPORARIES_DECL
201#include "RISCVGenGlobalISel.inc"
202#undef GET_GLOBALISEL_TEMPORARIES_DECL
203};
204
205} // end anonymous namespace
206
207#define GET_GLOBALISEL_IMPL
208#include "RISCVGenGlobalISel.inc"
209#undef GET_GLOBALISEL_IMPL
210
211RISCVInstructionSelector::RISCVInstructionSelector(
212 const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
213 const RISCVRegisterBankInfo &RBI)
214 : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
215 TM(TM),
216
217#define GET_GLOBALISEL_PREDICATES_INIT
218#include "RISCVGenGlobalISel.inc"
219#undef GET_GLOBALISEL_PREDICATES_INIT
220#define GET_GLOBALISEL_TEMPORARIES_INIT
221#include "RISCVGenGlobalISel.inc"
222#undef GET_GLOBALISEL_TEMPORARIES_INIT
223{
224}
225
226// Mimics optimizations in ISel and RISCVOptWInst Pass
227bool RISCVInstructionSelector::hasAllNBitUsers(const MachineInstr &MI,
228 unsigned Bits,
229 const unsigned Depth) const {
230
231 assert((MI.getOpcode() == TargetOpcode::G_ADD ||
232 MI.getOpcode() == TargetOpcode::G_SUB ||
233 MI.getOpcode() == TargetOpcode::G_MUL ||
234 MI.getOpcode() == TargetOpcode::G_SHL ||
235 MI.getOpcode() == TargetOpcode::G_LSHR ||
236 MI.getOpcode() == TargetOpcode::G_AND ||
237 MI.getOpcode() == TargetOpcode::G_OR ||
238 MI.getOpcode() == TargetOpcode::G_XOR ||
239 MI.getOpcode() == TargetOpcode::G_SEXT_INREG || Depth != 0) &&
240 "Unexpected opcode");
241
242 if (Depth >= RISCVInstructionSelector::MaxRecursionDepth)
243 return false;
244
245 auto DestReg = MI.getOperand(i: 0).getReg();
246 for (auto &UserOp : MRI->use_nodbg_operands(Reg: DestReg)) {
247 assert(UserOp.getParent() && "UserOp must have a parent");
248 const MachineInstr &UserMI = *UserOp.getParent();
249 unsigned OpIdx = UserOp.getOperandNo();
250
251 switch (UserMI.getOpcode()) {
252 default:
253 return false;
254 case RISCV::ADDW:
255 case RISCV::ADDIW:
256 case RISCV::SUBW:
257 case RISCV::FCVT_D_W:
258 case RISCV::FCVT_S_W:
259 if (Bits >= 32)
260 break;
261 return false;
262 case RISCV::SLL:
263 case RISCV::SRA:
264 case RISCV::SRL:
265 // Shift amount operands only use log2(Xlen) bits.
266 if (OpIdx == 2 && Bits >= Log2_32(Value: Subtarget->getXLen()))
267 break;
268 return false;
269 case RISCV::SLLI:
270 // SLLI only uses the lower (XLen - ShAmt) bits.
271 if (Bits >= Subtarget->getXLen() - UserMI.getOperand(i: 2).getImm())
272 break;
273 return false;
274 case RISCV::ANDI:
275 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(
276 Value: (uint64_t)UserMI.getOperand(i: 2).getImm()))
277 break;
278 goto RecCheck;
279 case RISCV::AND:
280 case RISCV::OR:
281 case RISCV::XOR:
282 RecCheck:
283 if (hasAllNBitUsers(MI: UserMI, Bits, Depth: Depth + 1))
284 break;
285 return false;
286 case RISCV::SRLI: {
287 unsigned ShAmt = UserMI.getOperand(i: 2).getImm();
288 // If we are shifting right by less than Bits, and users don't demand any
289 // bits that were shifted into [Bits-1:0], then we can consider this as an
290 // N-Bit user.
291 if (Bits > ShAmt && hasAllNBitUsers(MI: UserMI, Bits: Bits - ShAmt, Depth: Depth + 1))
292 break;
293 return false;
294 }
295 }
296 }
297
298 return true;
299}
300
301InstructionSelector::ComplexRendererFns
302RISCVInstructionSelector::selectShiftMask(MachineOperand &Root,
303 unsigned ShiftWidth) const {
304 if (!Root.isReg())
305 return std::nullopt;
306
307 using namespace llvm::MIPatternMatch;
308
309 Register ShAmtReg = Root.getReg();
310 // Peek through zext.
311 Register ZExtSrcReg;
312 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrcReg))))
313 ShAmtReg = ZExtSrcReg;
314
315 APInt AndMask;
316 Register AndSrcReg;
317 // Try to combine the following pattern (applicable to other shift
318 // instructions as well as 32-bit ones):
319 //
320 // %4:gprb(s64) = G_AND %3, %2
321 // %5:gprb(s64) = G_LSHR %1, %4(s64)
322 //
323 // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
324 // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
325 // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
326 // then it can be eliminated. Given register rs1 or rs2 holding a constant
327 // (the and mask), there are two cases G_AND can be erased:
328 //
329 // 1. the lowest log2(XLEN) bits of the and mask are all set
330 // 2. the bits of the register being masked are already unset (zero set)
331 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: AndSrcReg), R: m_ICst(Cst&: AndMask)))) {
332 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
333 if (ShMask.isSubsetOf(RHS: AndMask)) {
334 ShAmtReg = AndSrcReg;
335 } else {
336 // SimplifyDemandedBits may have optimized the mask so try restoring any
337 // bits that are known zero.
338 KnownBits Known = VT->getKnownBits(R: AndSrcReg);
339 if (ShMask.isSubsetOf(RHS: AndMask | Known.Zero))
340 ShAmtReg = AndSrcReg;
341 }
342 }
343
344 APInt Imm;
345 Register Reg;
346 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAdd(L: m_Reg(R&: Reg), R: m_ICst(Cst&: Imm)))) {
347 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0)
348 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
349 // to avoid the ADD.
350 ShAmtReg = Reg;
351 } else if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GSub(L: m_ICst(Cst&: Imm), R: m_Reg(R&: Reg)))) {
352 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0) {
353 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
354 // to generate a NEG instead of a SUB of a constant.
355 ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
356 unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
357 return {{[=](MachineInstrBuilder &MIB) {
358 MachineIRBuilder(*MIB.getInstr())
359 .buildInstr(Opc: NegOpc, DstOps: {ShAmtReg}, SrcOps: {Register(RISCV::X0), Reg});
360 MIB.addReg(RegNo: ShAmtReg);
361 }}};
362 }
363 if (Imm.urem(RHS: ShiftWidth) == ShiftWidth - 1) {
364 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
365 // to generate a NOT instead of a SUB of a constant.
366 ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
367 return {{[=](MachineInstrBuilder &MIB) {
368 MachineIRBuilder(*MIB.getInstr())
369 .buildInstr(Opc: RISCV::XORI, DstOps: {ShAmtReg}, SrcOps: {Reg})
370 .addImm(Val: -1);
371 MIB.addReg(RegNo: ShAmtReg);
372 }}};
373 }
374 }
375
376 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ShAmtReg); }}};
377}
378
379InstructionSelector::ComplexRendererFns
380RISCVInstructionSelector::selectSExtBits(MachineOperand &Root,
381 unsigned Bits) const {
382 if (!Root.isReg())
383 return std::nullopt;
384 Register RootReg = Root.getReg();
385
386 Register SrcReg;
387 if (mi_match(R: RootReg, MRI: *MRI,
388 P: m_GSExtInReg(Src: m_Reg(R&: SrcReg), Imm: m_SpecificImm(RequestedValue: Bits)))) {
389 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SrcReg); }}};
390 }
391
392 unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits();
393 if ((Size - VT->computeNumSignBits(R: RootReg)) < Bits)
394 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}};
395
396 return std::nullopt;
397}
398
399InstructionSelector::ComplexRendererFns
400RISCVInstructionSelector::selectZExtBits(MachineOperand &Root,
401 unsigned Bits) const {
402 if (!Root.isReg())
403 return std::nullopt;
404 Register RootReg = Root.getReg();
405
406 Register RegX;
407 uint64_t Mask = maskTrailingOnes<uint64_t>(N: Bits);
408 if (mi_match(R: RootReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: RegX), R: m_SpecificICst(RequestedValue: Mask)))) {
409 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}};
410 }
411
412 if (mi_match(R: RootReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: RegX))) &&
413 MRI->getType(Reg: RegX).getScalarSizeInBits() == Bits)
414 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}};
415
416 unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits();
417 if (VT->maskedValueIsZero(Val: RootReg, Mask: APInt::getBitsSetFrom(numBits: Size, loBit: Bits)))
418 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}};
419
420 return std::nullopt;
421}
422
423InstructionSelector::ComplexRendererFns
424RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
425 unsigned ShAmt) const {
426 using namespace llvm::MIPatternMatch;
427
428 if (!Root.isReg())
429 return std::nullopt;
430 Register RootReg = Root.getReg();
431
432 const unsigned XLen = STI.getXLen();
433 APInt Mask;
434 int64_t C2;
435 Register RegY;
436 std::optional<bool> LeftShift;
437 // (and (shl y, c2), mask)
438 if (mi_match(R: RootReg, MRI: *MRI,
439 P: m_GAnd(L: m_GShl(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
440 LeftShift = true;
441 // (and (lshr y, c2), mask)
442 else if (mi_match(R: RootReg, MRI: *MRI,
443 P: m_GAnd(L: m_GLShr(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
444 LeftShift = false;
445
446 if (LeftShift.has_value()) {
447 assert(Mask.getBitWidth() == XLen && "Unexpected mask size!");
448 assert(C2 > 0 && C2 < XLen && "Unexpected shift amount!");
449 if (*LeftShift)
450 Mask &= maskTrailingZeros<uint64_t>(N: C2);
451 else
452 Mask &= maskTrailingOnes<uint64_t>(N: XLen - C2);
453
454 if (Mask.isShiftedMask()) {
455 unsigned Leading = XLen - Mask.getActiveBits();
456 unsigned Trailing = Mask.countr_zero();
457 // Given (and (shl y, c2), mask) in which mask has no leading zeros and
458 // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
459 if (*LeftShift && Leading == 0 && C2 < Trailing && Trailing == ShAmt) {
460 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
461 return {{[=](MachineInstrBuilder &MIB) {
462 MachineIRBuilder(*MIB.getInstr())
463 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
464 .addImm(Val: Trailing - C2);
465 MIB.addReg(RegNo: DstReg);
466 }}};
467 }
468
469 // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
470 // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
471 if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
472 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
473 return {{[=](MachineInstrBuilder &MIB) {
474 MachineIRBuilder(*MIB.getInstr())
475 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
476 .addImm(Val: Leading + Trailing);
477 MIB.addReg(RegNo: DstReg);
478 }}};
479 }
480 }
481 }
482
483 LeftShift.reset();
484
485 // (shl (and y, mask), c2)
486 if (mi_match(R: RootReg, MRI: *MRI,
487 P: m_GShl(L: m_OneUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))), R: m_ICst(Cst&: C2))))
488 LeftShift = true;
489 // (lshr (and y, mask), c2)
490 else if (mi_match(R: RootReg, MRI: *MRI,
491 P: m_GLShr(L: m_OneUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
492 R: m_ICst(Cst&: C2))))
493 LeftShift = false;
494
495 if (LeftShift.has_value() && Mask.isShiftedMask()) {
496 unsigned Leading = XLen - Mask.getActiveBits();
497 unsigned Trailing = Mask.countr_zero();
498
499 // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
500 // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
501 bool Cond =
502 *LeftShift && Leading == 32 && Trailing > 0 && (Trailing + C2) == ShAmt;
503 if (!Cond)
504 // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
505 // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
506 Cond = !*LeftShift && Leading == 32 && C2 < Trailing &&
507 (Trailing - C2) == ShAmt;
508
509 if (Cond) {
510 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
511 return {{[=](MachineInstrBuilder &MIB) {
512 MachineIRBuilder(*MIB.getInstr())
513 .buildInstr(Opc: RISCV::SRLIW, DstOps: {DstReg}, SrcOps: {RegY})
514 .addImm(Val: Trailing);
515 MIB.addReg(RegNo: DstReg);
516 }}};
517 }
518 }
519
520 return std::nullopt;
521}
522
523InstructionSelector::ComplexRendererFns
524RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
525 unsigned ShAmt) const {
526 using namespace llvm::MIPatternMatch;
527
528 if (!Root.isReg())
529 return std::nullopt;
530 Register RootReg = Root.getReg();
531
532 // Given (and (shl x, c2), mask) in which mask is a shifted mask with
533 // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
534 // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
535 APInt Mask;
536 int64_t C2;
537 Register RegX;
538 if (mi_match(R: RootReg, MRI: *MRI,
539 P: m_OneUse(SP: m_GAnd(L: m_OneUse(SP: m_GShl(L: m_Reg(R&: RegX), R: m_ICst(Cst&: C2))),
540 R: m_ICst(Cst&: Mask))))) {
541 assert(C2 > 0 && C2 < Mask.getBitWidth() && "Unexpected shift amount!");
542 Mask &= maskTrailingZeros<uint64_t>(N: C2);
543
544 if (Mask.isShiftedMask()) {
545 unsigned Leading = Mask.countl_zero();
546 unsigned Trailing = Mask.countr_zero();
547 if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
548 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
549 return {{[=](MachineInstrBuilder &MIB) {
550 MachineIRBuilder(*MIB.getInstr())
551 .buildInstr(Opc: RISCV::SLLI, DstOps: {DstReg}, SrcOps: {RegX})
552 .addImm(Val: C2 - ShAmt);
553 MIB.addReg(RegNo: DstReg);
554 }}};
555 }
556 }
557 }
558
559 return std::nullopt;
560}
561
562InstructionSelector::ComplexRendererFns
563RISCVInstructionSelector::renderVLOp(MachineOperand &Root) const {
564 assert(Root.isReg() && "Expected operand to be a Register");
565 std::optional<ValueAndVReg> C;
566 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GCst(ValReg&: C))) {
567 if (C->Value.isAllOnes())
568 // If the operand is a G_CONSTANT with value of all ones it is larger than
569 // VLMAX. We convert it to an immediate with value VLMaxSentinel. This is
570 // recognized specially by the vsetvli insertion pass.
571 return {{[=](MachineInstrBuilder &MIB) {
572 MIB.addImm(Val: RISCV::VLMaxSentinel);
573 }}};
574
575 if (isUInt<5>(x: C->Value.getZExtValue())) {
576 uint64_t ZExtC = C->Value.getZExtValue();
577 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ZExtC); }}};
578 }
579 }
580 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Root.getReg()); }}};
581}
582
583InstructionSelector::ComplexRendererFns
584RISCVInstructionSelector::renderAddiPair(Register BaseReg, int64_t AddiImm,
585 int64_t OffsetImm) const {
586 return {{[=](MachineInstrBuilder &MIB) {
587 Register Tmp = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
588 MachineInstr *Addi =
589 BuildMI(BB&: *MIB->getParent(), I&: *MIB.getInstr(), MIMD: MIB->getDebugLoc(),
590 MCID: TII.get(Opcode: RISCV::ADDI), DestReg: Tmp)
591 .addReg(RegNo: BaseReg)
592 .addImm(Val: AddiImm);
593 constrainSelectedInstRegOperands(I&: *Addi, TII, TRI, RBI);
594 MIB.addReg(RegNo: Tmp);
595 },
596 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: OffsetImm); }}};
597}
598
599InstructionSelector::ComplexRendererFns
600RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
601 if (!Root.isReg())
602 return std::nullopt;
603
604 Register RootReg = Root.getReg();
605
606 // Frame index.
607 int FI;
608 if (mi_match(R: RootReg, MRI: *MRI, P: m_GFrameIndex(FI))) {
609 return {{
610 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); },
611 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
612 }};
613 }
614
615 // base + constant offset (G_PTR_ADD).
616 Register BaseReg;
617 int64_t RHSC;
618 if (mi_match(R: RootReg, MRI: *MRI, P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_ICst(Cst&: RHSC)))) {
619 if (isInt<12>(x: RHSC)) {
620 int BaseFI;
621 if (mi_match(R: BaseReg, MRI: *MRI, P: m_GFrameIndex(FI&: BaseFI)))
622 return {{
623 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: BaseFI); },
624 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
625 }};
626
627 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: BaseReg); },
628 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
629 }
630
631 // Large constant offset. Fold a -2048/2047 adjustment so the whole
632 // constant can be split across an ADDI and the load/store offset.
633 if (RHSC >= -4096 && RHSC <= 4094) {
634 int64_t Adj = RHSC < 0 ? -2048 : 2047;
635 return renderAddiPair(BaseReg, AddiImm: Adj, OffsetImm: RHSC - Adj);
636 }
637
638 if (isWorthFoldingAdd(AddResult: RootReg))
639 if (auto Fns = computeConstAddr(CVal: RHSC, /*IsPrefetch=*/false, OrigBase: BaseReg))
640 return Fns;
641 }
642
643 // Bare constant address. IRTranslator lowers inttoptr(C) to
644 // G_INTTOPTR(G_CONSTANT); look through it to reach the constant.
645 int64_t CVal;
646 if (mi_match(R: RootReg, MRI: *MRI, P: m_GIntToPtr(Src: m_ICst(Cst&: CVal))) ||
647 mi_match(R: RootReg, MRI: *MRI, P: m_ICst(Cst&: CVal))) {
648 if (auto Fns = computeConstAddr(CVal, /*IsPrefetch=*/false, OrigBase: Register()))
649 return Fns;
650 }
651
652 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); },
653 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
654}
655
656InstructionSelector::ComplexRendererFns
657RISCVInstructionSelector::selectBrindRegImm(MachineOperand &Root) const {
658 if (!Root.isReg())
659 return std::nullopt;
660
661 Register RootReg = Root.getReg();
662
663 // base + constant offset (G_PTR_ADD). Doesn't match a FrameIndex, unlike
664 // selectAddrRegImm, since the callers of this (e.g. indirect branches)
665 // can't take a FrameIndex or global address operand.
666 Register BaseReg;
667 int64_t RHSC;
668 if (mi_match(R: RootReg, MRI: *MRI, P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_ICst(Cst&: RHSC))) &&
669 isInt<12>(x: RHSC)) {
670 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: BaseReg); },
671 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
672 }
673
674 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); },
675 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
676}
677
678InstructionSelector::ComplexRendererFns
679RISCVInstructionSelector::selectAddrRegImmLsb00000(MachineOperand &Root) const {
680 if (!Root.isReg())
681 return std::nullopt;
682
683 Register RootReg = Root.getReg();
684
685 // Frame index.
686 int FI;
687 if (mi_match(R: RootReg, MRI: *MRI, P: m_GFrameIndex(FI))) {
688 return {{
689 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); },
690 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
691 }};
692 }
693
694 // base + constant offset (G_PTR_ADD).
695 Register BaseReg;
696 int64_t RHSC;
697 if (mi_match(R: RootReg, MRI: *MRI, P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_ICst(Cst&: RHSC)))) {
698 if (isInt<12>(x: RHSC)) {
699 // Not a multiple of 32: can't encode, use the address as-is.
700 if ((RHSC & 0b11111) != 0) {
701 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); },
702 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
703 }
704 // Fold the offset.
705 int BaseFI;
706 if (mi_match(R: BaseReg, MRI: *MRI, P: m_GFrameIndex(FI&: BaseFI)))
707 return {{
708 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: BaseFI); },
709 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
710 }};
711 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: BaseReg); },
712 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
713 }
714
715 // Large constant: fold a -2048/2016 adjustment to save an instruction.
716 if ((-2049 >= RHSC && RHSC >= -4096) || (4063 >= RHSC && RHSC >= 2017)) {
717 int64_t Adj = RHSC < 0 ? -2048 : 2016;
718 return renderAddiPair(BaseReg, AddiImm: RHSC - Adj, OffsetImm: Adj);
719 }
720
721 // Otherwise split the constant into Hi (materialized + added to the base)
722 // and Lo12 (folded offset).
723 if (auto Fns = computeConstAddr(CVal: RHSC, /*IsPrefetch=*/true, OrigBase: BaseReg))
724 return Fns;
725 }
726
727 // Bare constant address. IRTranslator emits inttoptr(C) as
728 // G_INTTOPTR(G_CONSTANT); look through the G_INTTOPTR to reach the constant.
729 int64_t CVal;
730 if (mi_match(R: RootReg, MRI: *MRI, P: m_GIntToPtr(Src: m_ICst(Cst&: CVal))) ||
731 mi_match(R: RootReg, MRI: *MRI, P: m_ICst(Cst&: CVal))) {
732 if (auto Fns = computeConstAddr(CVal, /*IsPrefetch=*/true, OrigBase: Register()))
733 return Fns;
734 }
735
736 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); },
737 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
738}
739
740/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
741/// CC Must be an ICMP Predicate.
742static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
743 switch (CC) {
744 default:
745 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
746 case CmpInst::Predicate::ICMP_EQ:
747 return RISCVCC::COND_EQ;
748 case CmpInst::Predicate::ICMP_NE:
749 return RISCVCC::COND_NE;
750 case CmpInst::Predicate::ICMP_ULT:
751 return RISCVCC::COND_LTU;
752 case CmpInst::Predicate::ICMP_SLT:
753 return RISCVCC::COND_LT;
754 case CmpInst::Predicate::ICMP_UGE:
755 return RISCVCC::COND_GEU;
756 case CmpInst::Predicate::ICMP_SGE:
757 return RISCVCC::COND_GE;
758 }
759}
760
761static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC,
762 Register &LHS, Register &RHS,
763 MachineRegisterInfo &MRI) {
764 // Try to fold an ICmp. If that fails, use a NE compare with X0.
765 CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
766 if (!mi_match(R: CondReg, MRI, P: m_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: LHS), R: m_Reg(R&: RHS)))) {
767 LHS = CondReg;
768 RHS = RISCV::X0;
769 CC = RISCVCC::COND_NE;
770 return;
771 }
772
773 // We found an ICmp, do some canonicalization.
774
775 // Adjust comparisons to use comparison with 0 if possible.
776 if (auto Constant = getIConstantVRegSExtVal(VReg: RHS, MRI)) {
777 switch (Pred) {
778 case CmpInst::Predicate::ICMP_SGT:
779 // Convert X > -1 to X >= 0
780 if (*Constant == -1) {
781 CC = RISCVCC::COND_GE;
782 RHS = RISCV::X0;
783 return;
784 }
785 break;
786 case CmpInst::Predicate::ICMP_SLT:
787 // Convert X < 1 to 0 >= X
788 if (*Constant == 1) {
789 CC = RISCVCC::COND_GE;
790 RHS = LHS;
791 LHS = RISCV::X0;
792 return;
793 }
794 break;
795 default:
796 break;
797 }
798 }
799
800 switch (Pred) {
801 default:
802 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
803 case CmpInst::Predicate::ICMP_EQ:
804 case CmpInst::Predicate::ICMP_NE:
805 case CmpInst::Predicate::ICMP_ULT:
806 case CmpInst::Predicate::ICMP_SLT:
807 case CmpInst::Predicate::ICMP_UGE:
808 case CmpInst::Predicate::ICMP_SGE:
809 // These CCs are supported directly by RISC-V branches.
810 break;
811 case CmpInst::Predicate::ICMP_SGT:
812 case CmpInst::Predicate::ICMP_SLE:
813 case CmpInst::Predicate::ICMP_UGT:
814 case CmpInst::Predicate::ICMP_ULE:
815 // These CCs are not supported directly by RISC-V branches, but changing the
816 // direction of the CC and swapping LHS and RHS are.
817 Pred = CmpInst::getSwappedPredicate(pred: Pred);
818 std::swap(a&: LHS, b&: RHS);
819 break;
820 }
821
822 CC = getRISCVCCFromICmp(CC: Pred);
823}
824
825/// Select the RISC-V Zalasr opcode for the G_LOAD or G_STORE operation
826/// \p GenericOpc, appropriate for the GPR register bank and of memory access
827/// size \p OpSize.
828static unsigned selectZalasrLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {
829 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
830 switch (OpSize) {
831 default:
832 llvm_unreachable("Unexpected memory size");
833 case 8:
834 return IsStore ? RISCV::SB_RL : RISCV::LB_AQ;
835 case 16:
836 return IsStore ? RISCV::SH_RL : RISCV::LH_AQ;
837 case 32:
838 return IsStore ? RISCV::SW_RL : RISCV::LW_AQ;
839 case 64:
840 return IsStore ? RISCV::SD_RL : RISCV::LD_AQ;
841 }
842}
843
844/// Select the RISC-V regimm opcode for the G_LOAD or G_STORE operation
845/// \p GenericOpc on \p STI, appropriate for the GPR register bank and of
846/// memory access size \p OpSize. \returns \p GenericOpc if the combination
847/// is unsupported.
848static unsigned selectRegImmLoadStoreOp(const RISCVSubtarget &STI,
849 unsigned GenericOpc, unsigned OpSize) {
850 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
851 switch (OpSize) {
852 case 8:
853 // Prefer unsigned due to no c.lb in Zcb.
854 return IsStore ? RISCV::SB : RISCV::LBU;
855 case 16:
856 return IsStore ? RISCV::SH : RISCV::LH;
857 case 32:
858 return IsStore ? RISCV::SW : RISCV::LW;
859 case 64:
860 // LD/SD are RV64-only. On RV32 a 64-bit access goes through an even/odd
861 // GPR pair, which Zilsd provides via the LD_RV32/SD_RV32 pseudos.
862 if (!STI.is64Bit() && STI.hasStdExtZilsd())
863 return IsStore ? RISCV::SD_RV32 : RISCV::LD_RV32;
864 return IsStore ? RISCV::SD : RISCV::LD;
865 }
866
867 return GenericOpc;
868}
869
870void RISCVInstructionSelector::addVectorLoadStoreOperands(
871 MachineInstr &I, SmallVectorImpl<Register> &SrcOps, unsigned &CurOp,
872 bool IsMasked, bool IsStridedOrIndexed, LLT *IndexVT) const {
873 // Base Pointer
874 auto PtrReg = I.getOperand(i: CurOp++).getReg();
875 SrcOps.push_back(Elt: PtrReg);
876
877 // Stride or Index
878 if (IsStridedOrIndexed) {
879 auto StrideReg = I.getOperand(i: CurOp++).getReg();
880 SrcOps.push_back(Elt: StrideReg);
881 if (IndexVT)
882 *IndexVT = MRI->getType(Reg: StrideReg);
883 }
884
885 // Mask
886 if (IsMasked) {
887 auto MaskReg = I.getOperand(i: CurOp++).getReg();
888 SrcOps.push_back(Elt: MaskReg);
889 }
890}
891
892bool RISCVInstructionSelector::selectIntrinsicWithSideEffects(
893 MachineInstr &I) const {
894 // Find the intrinsic ID.
895 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
896 // Select the instruction.
897 switch (IntrinID) {
898 default:
899 return false;
900 case Intrinsic::riscv_vlm:
901 case Intrinsic::riscv_vle:
902 case Intrinsic::riscv_vle_mask:
903 case Intrinsic::riscv_vlse:
904 case Intrinsic::riscv_vlse_mask: {
905 bool IsMasked = IntrinID == Intrinsic::riscv_vle_mask ||
906 IntrinID == Intrinsic::riscv_vlse_mask;
907 bool IsStrided = IntrinID == Intrinsic::riscv_vlse ||
908 IntrinID == Intrinsic::riscv_vlse_mask;
909 LLT VT = MRI->getType(Reg: I.getOperand(i: 0).getReg());
910 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
911
912 // Result vector
913 const Register DstReg = I.getOperand(i: 0).getReg();
914
915 // Sources
916 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
917 unsigned CurOp = 2;
918 SmallVector<Register, 4> SrcOps; // Source registers.
919
920 // Passthru
921 if (HasPassthruOperand) {
922 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
923 SrcOps.push_back(Elt: PassthruReg);
924 } else {
925 SrcOps.push_back(Elt: Register(RISCV::NoRegister));
926 }
927
928 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: IsStrided);
929
930 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
931 const RISCV::VLEPseudo *P =
932 RISCV::getVLEPseudo(Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
933 LMUL: static_cast<unsigned>(LMUL));
934
935 MachineInstrBuilder PseudoMI =
936 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo), DestReg: DstReg);
937 for (Register Reg : SrcOps)
938 PseudoMI.addReg(RegNo: Reg);
939
940 // Select VL
941 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
942 for (auto &RenderFn : *VLOpFn)
943 RenderFn(PseudoMI);
944
945 // SEW
946 PseudoMI.addImm(Val: Log2SEW);
947
948 // Policy
949 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
950 if (IsMasked)
951 Policy = I.getOperand(i: CurOp++).getImm();
952 PseudoMI.addImm(Val: Policy);
953
954 // Memref
955 PseudoMI.cloneMemRefs(OtherMI: I);
956
957 I.eraseFromParent();
958 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
959 return true;
960 }
961 case Intrinsic::riscv_vloxei:
962 case Intrinsic::riscv_vloxei_mask:
963 case Intrinsic::riscv_vluxei:
964 case Intrinsic::riscv_vluxei_mask: {
965 bool IsMasked = IntrinID == Intrinsic::riscv_vloxei_mask ||
966 IntrinID == Intrinsic::riscv_vluxei_mask;
967 bool IsOrdered = IntrinID == Intrinsic::riscv_vloxei ||
968 IntrinID == Intrinsic::riscv_vloxei_mask;
969 LLT VT = MRI->getType(Reg: I.getOperand(i: 0).getReg());
970 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
971
972 // Result vector
973 const Register DstReg = I.getOperand(i: 0).getReg();
974
975 // Sources
976 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
977 unsigned CurOp = 2;
978 SmallVector<Register, 4> SrcOps; // Source registers.
979
980 // Passthru
981 if (HasPassthruOperand) {
982 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
983 SrcOps.push_back(Elt: PassthruReg);
984 } else {
985 // Use NoRegister if there is no specified passthru.
986 SrcOps.push_back(Elt: Register());
987 }
988 LLT IndexVT;
989 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: true, IndexVT: &IndexVT);
990
991 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
992 RISCVVType::VLMUL IndexLMUL =
993 RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: IndexVT));
994 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
995 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
996 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
997 "values when XLEN=32");
998 }
999 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
1000 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
1001 IndexLMUL: static_cast<unsigned>(IndexLMUL));
1002
1003 MachineInstrBuilder PseudoMI =
1004 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo), DestReg: DstReg);
1005 for (Register Reg : SrcOps)
1006 PseudoMI.addReg(RegNo: Reg);
1007
1008 // Select VL
1009 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
1010 for (auto &RenderFn : *VLOpFn)
1011 RenderFn(PseudoMI);
1012
1013 // SEW
1014 PseudoMI.addImm(Val: Log2SEW);
1015
1016 // Policy
1017 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
1018 if (IsMasked)
1019 Policy = I.getOperand(i: CurOp++).getImm();
1020 PseudoMI.addImm(Val: Policy);
1021
1022 // Memref
1023 PseudoMI.cloneMemRefs(OtherMI: I);
1024
1025 I.eraseFromParent();
1026 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1027 return true;
1028 }
1029 case Intrinsic::riscv_vsm:
1030 case Intrinsic::riscv_vse:
1031 case Intrinsic::riscv_vse_mask:
1032 case Intrinsic::riscv_vsse:
1033 case Intrinsic::riscv_vsse_mask: {
1034 bool IsMasked = IntrinID == Intrinsic::riscv_vse_mask ||
1035 IntrinID == Intrinsic::riscv_vsse_mask;
1036 bool IsStrided = IntrinID == Intrinsic::riscv_vsse ||
1037 IntrinID == Intrinsic::riscv_vsse_mask;
1038 LLT VT = MRI->getType(Reg: I.getOperand(i: 1).getReg());
1039 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
1040
1041 // Sources
1042 unsigned CurOp = 1;
1043 SmallVector<Register, 4> SrcOps; // Source registers.
1044
1045 // Store value
1046 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
1047 SrcOps.push_back(Elt: PassthruReg);
1048
1049 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: IsStrided);
1050
1051 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
1052 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
1053 Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
1054
1055 MachineInstrBuilder PseudoMI =
1056 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo));
1057 for (Register Reg : SrcOps)
1058 PseudoMI.addReg(RegNo: Reg);
1059
1060 // Select VL
1061 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
1062 for (auto &RenderFn : *VLOpFn)
1063 RenderFn(PseudoMI);
1064
1065 // SEW
1066 PseudoMI.addImm(Val: Log2SEW);
1067
1068 // Memref
1069 PseudoMI.cloneMemRefs(OtherMI: I);
1070
1071 I.eraseFromParent();
1072 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1073 return true;
1074 }
1075 case Intrinsic::riscv_vsoxei:
1076 case Intrinsic::riscv_vsoxei_mask:
1077 case Intrinsic::riscv_vsuxei:
1078 case Intrinsic::riscv_vsuxei_mask: {
1079 bool IsMasked = IntrinID == Intrinsic::riscv_vsoxei_mask ||
1080 IntrinID == Intrinsic::riscv_vsuxei_mask;
1081 bool IsOrdered = IntrinID == Intrinsic::riscv_vsoxei ||
1082 IntrinID == Intrinsic::riscv_vsoxei_mask;
1083 LLT VT = MRI->getType(Reg: I.getOperand(i: 1).getReg());
1084 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
1085
1086 // Sources
1087 unsigned CurOp = 1;
1088 SmallVector<Register, 4> SrcOps; // Source registers.
1089
1090 // Store value
1091 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
1092 SrcOps.push_back(Elt: PassthruReg);
1093
1094 LLT IndexVT;
1095 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: true, IndexVT: &IndexVT);
1096
1097 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
1098 RISCVVType::VLMUL IndexLMUL =
1099 RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: IndexVT));
1100 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
1101 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
1102 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
1103 "values when XLEN=32");
1104 }
1105 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
1106 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
1107 IndexLMUL: static_cast<unsigned>(IndexLMUL));
1108
1109 MachineInstrBuilder PseudoMI =
1110 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo));
1111 for (Register Reg : SrcOps)
1112 PseudoMI.addReg(RegNo: Reg);
1113
1114 // Select VL
1115 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
1116 for (auto &RenderFn : *VLOpFn)
1117 RenderFn(PseudoMI);
1118
1119 // SEW
1120 PseudoMI.addImm(Val: Log2SEW);
1121
1122 // Memref
1123 PseudoMI.cloneMemRefs(OtherMI: I);
1124
1125 I.eraseFromParent();
1126 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1127 return true;
1128 }
1129 }
1130}
1131
1132bool RISCVInstructionSelector::selectIntrinsic(MachineInstr &I) const {
1133 // Find the intrinsic ID.
1134 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
1135 // Select the instruction.
1136 switch (IntrinID) {
1137 default:
1138 return false;
1139 case Intrinsic::riscv_vsetvli:
1140 case Intrinsic::riscv_vsetvlimax: {
1141
1142 bool VLMax = IntrinID == Intrinsic::riscv_vsetvlimax;
1143
1144 unsigned Offset = VLMax ? 2 : 3;
1145 unsigned SEW = RISCVVType::decodeVSEW(VSEW: I.getOperand(i: Offset).getImm() & 0x7);
1146 RISCVVType::VLMUL VLMul =
1147 static_cast<RISCVVType::VLMUL>(I.getOperand(i: Offset + 1).getImm() & 0x7);
1148
1149 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMUL: VLMul, SEW, /*TailAgnostic*/ true,
1150 /*MaskAgnostic*/ true);
1151
1152 Register DstReg = I.getOperand(i: 0).getReg();
1153
1154 Register VLOperand;
1155 unsigned Opcode = RISCV::PseudoVSETVLI;
1156
1157 // Check if AVL is a constant that equals VLMAX.
1158 if (!VLMax) {
1159 Register AVLReg = I.getOperand(i: 2).getReg();
1160 if (auto AVLConst = getIConstantVRegValWithLookThrough(VReg: AVLReg, MRI: *MRI)) {
1161 uint64_t AVL = AVLConst->Value.getZExtValue();
1162 if (auto VLEN = Subtarget->getRealVLen()) {
1163 if (*VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == AVL)
1164 VLMax = true;
1165 }
1166 }
1167
1168 if (mi_match(R: AVLReg, MRI: *MRI, P: m_AllOnes()))
1169 VLMax = true;
1170 }
1171
1172 if (VLMax) {
1173 VLOperand = Register(RISCV::X0);
1174 Opcode = RISCV::PseudoVSETVLIX0;
1175 } else {
1176 Register AVLReg = I.getOperand(i: 2).getReg();
1177 VLOperand = AVLReg;
1178
1179 // Check if AVL is a small constant that can use PseudoVSETIVLI.
1180 if (auto AVLConst = getIConstantVRegValWithLookThrough(VReg: AVLReg, MRI: *MRI)) {
1181 uint64_t AVL = AVLConst->Value.getZExtValue();
1182 if (isUInt<5>(x: AVL)) {
1183 MachineInstr *PseudoMI =
1184 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1185 MCID: TII.get(Opcode: RISCV::PseudoVSETIVLI), DestReg: DstReg)
1186 .addImm(Val: AVL)
1187 .addImm(Val: VTypeI);
1188 I.eraseFromParent();
1189 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1190 return true;
1191 }
1192 }
1193 }
1194
1195 MachineInstr *PseudoMI =
1196 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode), DestReg: DstReg)
1197 .addReg(RegNo: VLOperand)
1198 .addImm(Val: VTypeI);
1199 I.eraseFromParent();
1200 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1201 return true;
1202 }
1203 }
1204}
1205
1206bool RISCVInstructionSelector::selectExtractSubvector(MachineInstr &MI) const {
1207 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_SUBVECTOR);
1208
1209 Register DstReg = MI.getOperand(i: 0).getReg();
1210 Register SrcReg = MI.getOperand(i: 1).getReg();
1211
1212 LLT DstTy = MRI->getType(Reg: DstReg);
1213 LLT SrcTy = MRI->getType(Reg: SrcReg);
1214
1215 unsigned Idx = static_cast<unsigned>(MI.getOperand(i: 2).getImm());
1216
1217 MVT DstMVT = getMVTForLLT(Ty: DstTy);
1218 MVT SrcMVT = getMVTForLLT(Ty: SrcTy);
1219
1220 unsigned SubRegIdx;
1221 std::tie(args&: SubRegIdx, args&: Idx) =
1222 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1223 VecVT: SrcMVT, SubVecVT: DstMVT, InsertExtractIdx: Idx, TRI: &TRI);
1224
1225 if (Idx != 0)
1226 return false;
1227
1228 unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: DstMVT);
1229 const TargetRegisterClass *DstRC = TRI.getRegClass(i: DstRegClassID);
1230 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1231 return false;
1232
1233 unsigned SrcRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: SrcMVT);
1234 const TargetRegisterClass *SrcRC = TRI.getRegClass(i: SrcRegClassID);
1235 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
1236 return false;
1237
1238 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1239 DestReg: DstReg)
1240 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubRegIdx);
1241
1242 MI.eraseFromParent();
1243 return true;
1244}
1245
1246bool RISCVInstructionSelector::selectInsertSubVector(MachineInstr &MI) const {
1247 assert(MI.getOpcode() == TargetOpcode::G_INSERT_SUBVECTOR);
1248
1249 Register DstReg = MI.getOperand(i: 0).getReg();
1250 Register VecReg = MI.getOperand(i: 1).getReg();
1251 Register SubVecReg = MI.getOperand(i: 2).getReg();
1252
1253 LLT VecTy = MRI->getType(Reg: VecReg);
1254 LLT SubVecTy = MRI->getType(Reg: SubVecReg);
1255
1256 MVT VecMVT = getMVTForLLT(Ty: VecTy);
1257 MVT SubVecMVT = getMVTForLLT(Ty: SubVecTy);
1258
1259 unsigned Idx = static_cast<unsigned>(MI.getOperand(i: 3).getImm());
1260
1261 unsigned SubRegIdx;
1262 std::tie(args&: SubRegIdx, args&: Idx) =
1263 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1264 VecVT: VecMVT, SubVecVT: SubVecMVT, InsertExtractIdx: Idx, TRI: &TRI);
1265
1266 // If the Idx hasn't been completely eliminated then this is a subvector
1267 // insert which doesn't naturally align to a vector register. These must
1268 // be handled using instructions to manipulate the vector registers.
1269 if (Idx != 0)
1270 return false;
1271
1272 // Constrain dst
1273 unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: VecMVT);
1274 const TargetRegisterClass *DstRC = TRI.getRegClass(i: DstRegClassID);
1275 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1276 return false;
1277
1278 // If we haven't set a SubRegIdx, then we must be going between
1279 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
1280 if (SubRegIdx == RISCV::NoSubRegister) {
1281 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecMVT) ==
1282 DstRegClassID &&
1283 "Unexpected subvector insert");
1284 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1285 DestReg: DstReg)
1286 .addReg(RegNo: SubVecReg);
1287 MI.eraseFromParent();
1288 return true;
1289 }
1290
1291 // Use INSERT_SUBREG to insert the subvector into the vector at the
1292 // appropriate subregister index.
1293 MachineInstr *Ins = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1294 MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DstReg)
1295 .addReg(RegNo: VecReg)
1296 .addReg(RegNo: SubVecReg)
1297 .addImm(Val: SubRegIdx);
1298
1299 MI.eraseFromParent();
1300 constrainSelectedInstRegOperands(I&: *Ins, TII, TRI, RBI);
1301 return true;
1302}
1303
1304bool RISCVInstructionSelector::select(MachineInstr &MI) {
1305 preISelLower(MI);
1306 const unsigned Opc = MI.getOpcode();
1307
1308 if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
1309 if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
1310 const Register DefReg = MI.getOperand(i: 0).getReg();
1311 const LLT DefTy = MRI->getType(Reg: DefReg);
1312
1313 const RegClassOrRegBank &RegClassOrBank =
1314 MRI->getRegClassOrRegBank(Reg: DefReg);
1315
1316 const TargetRegisterClass *DefRC =
1317 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
1318 if (!DefRC) {
1319 if (!DefTy.isValid()) {
1320 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
1321 return false;
1322 }
1323
1324 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
1325 DefRC = TRI.getRegClassForTypeOnBank(Ty: DefTy, RB, Is64Bit: STI.is64Bit());
1326 if (!DefRC) {
1327 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
1328 return false;
1329 }
1330 }
1331
1332 MI.setDesc(TII.get(Opcode: TargetOpcode::PHI));
1333 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI&: *MRI);
1334 }
1335
1336 // Certain non-generic instructions also need some special handling.
1337 if (MI.isCopy())
1338 return selectCopy(MI);
1339
1340 return true;
1341 }
1342
1343 if (selectImpl(I&: MI, CoverageInfo&: *CoverageInfo))
1344 return true;
1345
1346 switch (Opc) {
1347 case TargetOpcode::G_BITCAST:
1348 case TargetOpcode::G_ANYEXT:
1349 case TargetOpcode::G_PTRTOINT:
1350 case TargetOpcode::G_INTTOPTR:
1351 case TargetOpcode::G_TRUNC:
1352 case TargetOpcode::G_FREEZE:
1353 return selectCopy(MI);
1354 case TargetOpcode::G_CONSTANT: {
1355 Register DstReg = MI.getOperand(i: 0).getReg();
1356 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
1357
1358 if (!materializeImm(Reg: DstReg, Imm, MI))
1359 return false;
1360
1361 MI.eraseFromParent();
1362 return true;
1363 }
1364 case TargetOpcode::G_ZEXT:
1365 case TargetOpcode::G_SEXT: {
1366 bool IsSigned = Opc != TargetOpcode::G_ZEXT;
1367 Register DstReg = MI.getOperand(i: 0).getReg();
1368 Register SrcReg = MI.getOperand(i: 1).getReg();
1369 LLT SrcTy = MRI->getType(Reg: SrcReg);
1370 unsigned SrcSize = SrcTy.getSizeInBits();
1371
1372 if (SrcTy.isVector())
1373 return false; // Should be handled by imported patterns.
1374
1375 assert((*RBI.getRegBank(DstReg, *MRI, TRI)).getID() ==
1376 RISCV::GPRBRegBankID &&
1377 "Unexpected ext regbank");
1378
1379 // Use addiw SrcReg, 0 (sext.w) for i32.
1380 if (IsSigned && SrcSize == 32) {
1381 MI.setDesc(TII.get(Opcode: RISCV::ADDIW));
1382 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
1383 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1384 return true;
1385 }
1386
1387 // Use add.uw SrcReg, X0 (zext.w) for i32 with Zba.
1388 if (!IsSigned && SrcSize == 32 && STI.hasStdExtZba()) {
1389 MI.setDesc(TII.get(Opcode: RISCV::ADD_UW));
1390 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::X0, /*isDef=*/false));
1391 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1392 return true;
1393 }
1394
1395 // Use sext.h/zext.h for i16 with Zbb.
1396 if (SrcSize == 16 &&
1397 (STI.hasStdExtZbb() || (!IsSigned && STI.hasStdExtZbkb()))) {
1398 MI.setDesc(TII.get(Opcode: IsSigned ? RISCV::SEXT_H
1399 : STI.is64Bit() ? RISCV::ZEXT_H_RV64
1400 : RISCV::ZEXT_H_RV32));
1401 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1402 return true;
1403 }
1404
1405 // Fall back to shift pair.
1406 Register ShiftLeftReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1407 MachineInstr *ShiftLeft = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1408 MCID: TII.get(Opcode: RISCV::SLLI), DestReg: ShiftLeftReg)
1409 .addReg(RegNo: SrcReg)
1410 .addImm(Val: STI.getXLen() - SrcSize);
1411 constrainSelectedInstRegOperands(I&: *ShiftLeft, TII, TRI, RBI);
1412 MachineInstr *ShiftRight =
1413 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1414 MCID: TII.get(Opcode: IsSigned ? RISCV::SRAI : RISCV::SRLI), DestReg: DstReg)
1415 .addReg(RegNo: ShiftLeftReg)
1416 .addImm(Val: STI.getXLen() - SrcSize);
1417 constrainSelectedInstRegOperands(I&: *ShiftRight, TII, TRI, RBI);
1418 MI.eraseFromParent();
1419 return true;
1420 }
1421 case TargetOpcode::G_FCONSTANT: {
1422 // TODO: Use constant pool for complex constants.
1423 Register DstReg = MI.getOperand(i: 0).getReg();
1424 const APFloat &FPimm = MI.getOperand(i: 1).getFPImm()->getValueAPF();
1425 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
1426 if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
1427 Register GPRReg;
1428 if (FPimm.isPosZero()) {
1429 GPRReg = RISCV::X0;
1430 } else {
1431 GPRReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1432 APInt Imm = FPimm.bitcastToAPInt();
1433 if (!materializeImm(Reg: GPRReg, Imm: Imm.getSExtValue(), MI))
1434 return false;
1435 }
1436
1437 unsigned Opcode = Size == 64 ? RISCV::FMV_D_X
1438 : Size == 32 ? RISCV::FMV_W_X
1439 : RISCV::FMV_H_X;
1440 MachineInstr *FMV = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1441 MCID: TII.get(Opcode), DestReg: DstReg)
1442 .addReg(RegNo: GPRReg);
1443 constrainSelectedInstRegOperands(I&: *FMV, TII, TRI, RBI);
1444 } else {
1445 // s64 on rv32
1446 assert(Size == 64 && !Subtarget->is64Bit() &&
1447 "Unexpected size or subtarget");
1448
1449 if (FPimm.isPosZero()) {
1450 // Optimize +0.0 to use fcvt.d.w
1451 MachineInstr *FCVT = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1452 MCID: TII.get(Opcode: RISCV::FCVT_D_W), DestReg: DstReg)
1453 .addReg(RegNo: RISCV::X0)
1454 .addImm(Val: RISCVFPRndMode::RNE);
1455 constrainSelectedInstRegOperands(I&: *FCVT, TII, TRI, RBI);
1456
1457 MI.eraseFromParent();
1458 return true;
1459 }
1460
1461 // Split into two pieces and build through the stack.
1462 Register GPRRegHigh = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1463 Register GPRRegLow = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1464 APInt Imm = FPimm.bitcastToAPInt();
1465 if (!materializeImm(Reg: GPRRegHigh, Imm: Imm.extractBits(numBits: 32, bitPosition: 32).getSExtValue(),
1466 MI))
1467 return false;
1468 if (!materializeImm(Reg: GPRRegLow, Imm: Imm.trunc(width: 32).getSExtValue(), MI))
1469 return false;
1470 MachineInstr *PairF64 =
1471 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1472 MCID: TII.get(Opcode: RISCV::BuildPairF64Pseudo), DestReg: DstReg)
1473 .addReg(RegNo: GPRRegLow)
1474 .addReg(RegNo: GPRRegHigh);
1475 constrainSelectedInstRegOperands(I&: *PairF64, TII, TRI, RBI);
1476 }
1477
1478 MI.eraseFromParent();
1479 return true;
1480 }
1481 case TargetOpcode::G_GLOBAL_VALUE: {
1482 auto *GV = MI.getOperand(i: 1).getGlobal();
1483 if (GV->isThreadLocal()) {
1484 // TODO: implement this case.
1485 return false;
1486 }
1487
1488 return selectAddr(MI, IsLocal: GV->isDSOLocal(), IsExternWeak: GV->hasExternalWeakLinkage());
1489 }
1490 case TargetOpcode::G_JUMP_TABLE:
1491 case TargetOpcode::G_CONSTANT_POOL:
1492 return selectAddr(MI);
1493 case TargetOpcode::G_BRCOND: {
1494 Register LHS, RHS;
1495 RISCVCC::CondCode CC;
1496 getOperandsForBranch(CondReg: MI.getOperand(i: 0).getReg(), CC, LHS, RHS, MRI&: *MRI);
1497
1498 MachineInstr *Bcc = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1499 MCID: TII.get(Opcode: RISCVCC::getBrCond(CC)))
1500 .addReg(RegNo: LHS)
1501 .addReg(RegNo: RHS)
1502 .addMBB(MBB: MI.getOperand(i: 1).getMBB());
1503 MI.eraseFromParent();
1504 constrainSelectedInstRegOperands(I&: *Bcc, TII, TRI, RBI);
1505 return true;
1506 }
1507 case TargetOpcode::G_SELECT:
1508 return selectSelect(MI);
1509 case TargetOpcode::G_FCMP:
1510 return selectFPCompare(MI);
1511 case TargetOpcode::G_FENCE: {
1512 AtomicOrdering FenceOrdering =
1513 static_cast<AtomicOrdering>(MI.getOperand(i: 0).getImm());
1514 SyncScope::ID FenceSSID =
1515 static_cast<SyncScope::ID>(MI.getOperand(i: 1).getImm());
1516 emitFence(FenceOrdering, FenceSSID, MI);
1517 MI.eraseFromParent();
1518 return true;
1519 }
1520 case TargetOpcode::G_IMPLICIT_DEF:
1521 return selectImplicitDef(MI);
1522 case TargetOpcode::G_UNMERGE_VALUES:
1523 return selectUnmergeValues(MI);
1524 case TargetOpcode::G_MERGE_VALUES:
1525 return selectMergeValues(MI);
1526 case TargetOpcode::G_LOAD:
1527 case TargetOpcode::G_STORE: {
1528 GLoadStore &LdSt = cast<GLoadStore>(Val&: MI);
1529 const Register ValReg = LdSt.getReg(Idx: 0);
1530 const Register PtrReg = LdSt.getPointerReg();
1531 LLT PtrTy = MRI->getType(Reg: PtrReg);
1532
1533 const RegisterBank &RB = *RBI.getRegBank(Reg: ValReg, MRI: *MRI, TRI);
1534 if (RB.getID() != RISCV::GPRBRegBankID)
1535 return false;
1536
1537#ifndef NDEBUG
1538 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, *MRI, TRI);
1539 // Check that the pointer register is valid.
1540 assert(PtrRB.getID() == RISCV::GPRBRegBankID &&
1541 "Load/Store pointer operand isn't a GPR");
1542 assert(PtrTy.isPointer() && "Load/Store pointer operand isn't a pointer");
1543#endif
1544
1545 // Can only handle AddressSpace 0.
1546 if (PtrTy.getAddressSpace() != 0)
1547 return false;
1548
1549 unsigned MemSize = LdSt.getMemSizeInBits().getValue();
1550 AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();
1551
1552 if (isStrongerThanMonotonic(AO: Order)) {
1553 MI.setDesc(TII.get(Opcode: selectZalasrLoadStoreOp(GenericOpc: Opc, OpSize: MemSize)));
1554 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1555 return true;
1556 }
1557
1558 const unsigned NewOpc =
1559 selectRegImmLoadStoreOp(STI, GenericOpc: MI.getOpcode(), OpSize: MemSize);
1560 if (NewOpc == MI.getOpcode())
1561 return false;
1562
1563 // Check if we can fold anything into the addressing mode.
1564 auto AddrModeFns = selectAddrRegImm(Root&: MI.getOperand(i: 1));
1565 if (!AddrModeFns)
1566 return false;
1567
1568 // Folded something. Create a new instruction and return it.
1569 MachineInstrBuilder NewInst =
1570 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: NewOpc));
1571 NewInst.setMIFlags(MI.getFlags());
1572 if (isa<GStore>(Val: MI))
1573 NewInst.addUse(RegNo: ValReg);
1574 else
1575 NewInst.addDef(RegNo: ValReg);
1576 NewInst.cloneMemRefs(OtherMI: MI);
1577 for (auto &Fn : *AddrModeFns)
1578 Fn(NewInst);
1579 MI.eraseFromParent();
1580
1581 constrainSelectedInstRegOperands(I&: *NewInst, TII, TRI, RBI);
1582 return true;
1583 }
1584 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1585 return selectIntrinsicWithSideEffects(I&: MI);
1586 case TargetOpcode::G_INTRINSIC:
1587 return selectIntrinsic(I&: MI);
1588 case TargetOpcode::G_EXTRACT_SUBVECTOR:
1589 return selectExtractSubvector(MI);
1590 case TargetOpcode::G_INSERT_SUBVECTOR:
1591 return selectInsertSubVector(MI);
1592 default:
1593 return false;
1594 }
1595}
1596
1597bool RISCVInstructionSelector::selectUnmergeValues(MachineInstr &MI) const {
1598 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
1599
1600 // Split Src into two s32 parts.
1601 if (MI.getNumOperands() != 3)
1602 return false;
1603 Register Src = MI.getOperand(i: 2).getReg();
1604 Register Lo = MI.getOperand(i: 0).getReg();
1605 Register Hi = MI.getOperand(i: 1).getReg();
1606
1607 // RV32+Zilsd: split a GPRPair (i64) into its two s32 GPR halves.
1608 if (!Subtarget->is64Bit() && Subtarget->hasStdExtZilsd() &&
1609 MRI->getType(Reg: Src) == LLT::scalar(SizeInBits: 64) && isRegInGprb(Reg: Src) &&
1610 isRegInGprb(Reg: Lo) && isRegInGprb(Reg: Hi)) {
1611 // GPRPairNoX0 rather than GPRPair: a sub-register index is only valid for
1612 // a class when every register in it supports the index, and X0_Pair has no
1613 // sub_gpr_odd. SelectionDAG gets the same effect through
1614 // InstrEmitter::ConstrainForSubReg.
1615 if (!RBI.constrainGenericRegister(Reg: Src, RC: RISCV::GPRPairNoX0RegClass, MRI&: *MRI))
1616 return false;
1617 // COPY with a sub-register index rather than EXTRACT_SUBREG, which is what
1618 // SelectionDAG emits too: InstrEmitter lowers an EXTRACT_SUBREG SDNode to
1619 // %dst = COPY %src:sub and never builds the pseudo itself. A literal
1620 // EXTRACT_SUBREG would also survive to ExpandPostRAPseudos, which treats
1621 // one as unreachable.
1622 auto ExtractHalf = [&](Register Dst, unsigned SubIdx) {
1623 RBI.constrainGenericRegister(Reg: Dst, RC: RISCV::GPRRegClass, MRI&: *MRI);
1624 MachineInstr &C = *BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1625 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Dst)
1626 .addReg(RegNo: Src, Flags: {}, SubReg: SubIdx);
1627 constrainSelectedInstRegOperands(I&: C, TII, TRI, RBI);
1628 };
1629 ExtractHalf(Lo, RISCV::sub_gpr_even);
1630 ExtractHalf(Hi, RISCV::sub_gpr_odd);
1631 MI.eraseFromParent();
1632 return true;
1633 }
1634
1635 if (!Subtarget->hasStdExtZfa())
1636 return false;
1637
1638 if (!isRegInFprb(Reg: Src) || !isRegInGprb(Reg: Lo) || !isRegInGprb(Reg: Hi))
1639 return false;
1640
1641 MachineInstr *ExtractLo = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1642 MCID: TII.get(Opcode: RISCV::FMV_X_W_FPR64), DestReg: Lo)
1643 .addReg(RegNo: Src);
1644 constrainSelectedInstRegOperands(I&: *ExtractLo, TII, TRI, RBI);
1645
1646 MachineInstr *ExtractHi = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1647 MCID: TII.get(Opcode: RISCV::FMVH_X_D), DestReg: Hi)
1648 .addReg(RegNo: Src);
1649 constrainSelectedInstRegOperands(I&: *ExtractHi, TII, TRI, RBI);
1650
1651 MI.eraseFromParent();
1652 return true;
1653}
1654
1655bool RISCVInstructionSelector::selectMergeValues(MachineInstr &MI) const {
1656 assert(MI.getOpcode() == TargetOpcode::G_MERGE_VALUES);
1657
1658 if (MI.getNumOperands() != 3)
1659 return false;
1660 Register Dst = MI.getOperand(i: 0).getReg();
1661 Register Lo = MI.getOperand(i: 1).getReg();
1662 Register Hi = MI.getOperand(i: 2).getReg();
1663
1664 // RV32+Zilsd: build a GPRPair (i64) from two s32 GPR halves.
1665 if (!Subtarget->is64Bit() && Subtarget->hasStdExtZilsd() &&
1666 MRI->getType(Reg: Dst) == LLT::scalar(SizeInBits: 64) && isRegInGprb(Reg: Dst) &&
1667 isRegInGprb(Reg: Lo) && isRegInGprb(Reg: Hi)) {
1668 // GPRPairNoX0 rather than GPRPair, because sub_gpr_odd is not valid for
1669 // X0_Pair. Constrain Dst up front: constrainSelectedInstRegOperands below
1670 // would otherwise infer GPRPair from the s64 type.
1671 if (!RBI.constrainGenericRegister(Reg: Dst, RC: RISCV::GPRPairNoX0RegClass, MRI&: *MRI))
1672 return false;
1673 MachineInstr &Seq = *BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1674 MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dst)
1675 .addReg(RegNo: Lo)
1676 .addImm(Val: RISCV::sub_gpr_even)
1677 .addReg(RegNo: Hi)
1678 .addImm(Val: RISCV::sub_gpr_odd);
1679 MI.eraseFromParent();
1680 constrainSelectedInstRegOperands(I&: Seq, TII, TRI, RBI);
1681 return true;
1682 }
1683 return false;
1684}
1685
1686bool RISCVInstructionSelector::replacePtrWithInt(MachineInstr &MI,
1687 unsigned OpIdx) {
1688 MachineOperand &Op = MI.getOperand(i: OpIdx);
1689 Register PtrReg = Op.getReg();
1690 assert(MRI->getType(PtrReg).isPointer() && "Operand is not a pointer!");
1691
1692 const LLT iXLen = LLT::integer(SizeInBits: STI.getXLen());
1693 Register IntReg = MRI->createGenericVirtualRegister(Ty: iXLen);
1694 MRI->setRegBank(Reg: IntReg, RegBank: RBI.getRegBank(ID: RISCV::GPRBRegBankID));
1695 MachineInstr *PtrToInt = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1696 MCID: TII.get(Opcode: TargetOpcode::G_PTRTOINT), DestReg: IntReg)
1697 .addReg(RegNo: PtrReg);
1698 Op.setReg(IntReg);
1699 return select(MI&: *PtrToInt);
1700}
1701
1702void RISCVInstructionSelector::preISelLower(MachineInstr &MI) {
1703 switch (MI.getOpcode()) {
1704 case TargetOpcode::G_PTR_ADD: {
1705 Register DstReg = MI.getOperand(i: 0).getReg();
1706 const LLT iXLen = LLT::integer(SizeInBits: STI.getXLen());
1707
1708 replacePtrWithInt(MI, OpIdx: 1);
1709 MI.setDesc(TII.get(Opcode: TargetOpcode::G_ADD));
1710 MRI->setType(VReg: DstReg, Ty: iXLen);
1711 break;
1712 }
1713 case TargetOpcode::G_PTRMASK: {
1714 Register DstReg = MI.getOperand(i: 0).getReg();
1715 const LLT iXLen = LLT::integer(SizeInBits: STI.getXLen());
1716 replacePtrWithInt(MI, OpIdx: 1);
1717 MI.setDesc(TII.get(Opcode: TargetOpcode::G_AND));
1718 MRI->setType(VReg: DstReg, Ty: iXLen);
1719 break;
1720 }
1721 }
1722}
1723
1724void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
1725 const MachineInstr &MI,
1726 int OpIdx) const {
1727 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1728 "Expected G_CONSTANT");
1729 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
1730 MIB.addImm(Val: -CstVal);
1731}
1732
1733void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
1734 const MachineInstr &MI,
1735 int OpIdx) const {
1736 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1737 "Expected G_CONSTANT");
1738 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1739 MIB.addImm(Val: STI.getXLen() - CstVal);
1740}
1741
1742void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
1743 const MachineInstr &MI,
1744 int OpIdx) const {
1745 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1746 "Expected G_CONSTANT");
1747 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1748 MIB.addImm(Val: 32 - CstVal);
1749}
1750
1751void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
1752 const MachineInstr &MI,
1753 int OpIdx) const {
1754 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1755 "Expected G_CONSTANT");
1756 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
1757 MIB.addImm(Val: CstVal + 1);
1758}
1759
1760void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
1761 const MachineInstr &MI,
1762 int OpIdx) const {
1763 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1764 "Expected G_CONSTANT");
1765 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
1766 MIB.addImm(Val: llvm::countr_zero(Val: C));
1767}
1768
1769void RISCVInstructionSelector::renderXLenSubTrailingOnes(
1770 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
1771 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1772 "Expected G_CONSTANT");
1773 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
1774 MIB.addImm(Val: Subtarget->getXLen() - llvm::countr_one(Value: C));
1775}
1776
1777void RISCVInstructionSelector::renderAddiPairImmSmall(MachineInstrBuilder &MIB,
1778 const MachineInstr &MI,
1779 int OpIdx) const {
1780 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1781 "Expected G_CONSTANT");
1782 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
1783 int64_t Adj = Imm < 0 ? -2048 : 2047;
1784 MIB.addImm(Val: Imm - Adj);
1785}
1786
1787void RISCVInstructionSelector::renderAddiPairImmLarge(MachineInstrBuilder &MIB,
1788 const MachineInstr &MI,
1789 int OpIdx) const {
1790 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1791 "Expected G_CONSTANT");
1792 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue() < 0 ? -2048 : 2047;
1793 MIB.addImm(Val: Imm);
1794}
1795
1796bool RISCVInstructionSelector::isRegInGprb(Register Reg) const {
1797 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::GPRBRegBankID;
1798}
1799
1800bool RISCVInstructionSelector::isRegInFprb(Register Reg) const {
1801 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID;
1802}
1803
1804// A G_PTR_ADD result is worth splitting into Hi (materialized) +
1805// Lo12 (folded offset) only if every user is a plain scalar load/store
1806// using it as the address. Otherwise the ADD is selected on its own with
1807// the full materialized constant, making the Hi materialization here redundant.
1808bool RISCVInstructionSelector::isWorthFoldingAdd(Register AddResult) const {
1809 for (const MachineInstr &User : MRI->use_nodbg_instructions(Reg: AddResult)) {
1810 auto *LdSt = dyn_cast<GLoadStore>(Val: &User);
1811 if (!LdSt)
1812 return false;
1813 // Must be used as the pointer, not the stored value.
1814 if (LdSt->getPointerReg() != AddResult)
1815 return false;
1816 if (isStrongerThanMonotonic(AO: LdSt->getMMO().getSuccessOrdering()))
1817 return false;
1818 // Only scalar integer/f16/f32/f64 memory (exclude vectors, f128, ...).
1819 LLT Ty = MRI->getType(Reg: User.getOperand(i: 0).getReg());
1820 if (!Ty.isScalar() || Ty.getSizeInBits() > 64)
1821 return false;
1822 }
1823 return true;
1824}
1825
1826bool RISCVInstructionSelector::selectCopy(MachineInstr &MI) const {
1827 Register DstReg = MI.getOperand(i: 0).getReg();
1828
1829 if (DstReg.isPhysical())
1830 return true;
1831
1832 const TargetRegisterClass *DstRC =
1833 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
1834
1835 assert(DstRC &&
1836 "Register class not available for LLT, register bank combination");
1837
1838 // No need to constrain SrcReg. It will get constrained when
1839 // we hit another of its uses or its defs.
1840 // Copies do not have constraints.
1841 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
1842 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1843 << " operand\n");
1844 return false;
1845 }
1846
1847 MI.setDesc(TII.get(Opcode: RISCV::COPY));
1848 return true;
1849}
1850
1851bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI) const {
1852 assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
1853
1854 const Register DstReg = MI.getOperand(i: 0).getReg();
1855 const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(
1856 Ty: MRI->getType(Reg: DstReg), RB: *RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI), Is64Bit: STI.is64Bit());
1857
1858 assert(DstRC &&
1859 "Register class not available for LLT, register bank combination");
1860
1861 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
1862 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1863 << " operand\n");
1864 }
1865 MI.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
1866 return true;
1867}
1868
1869bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
1870 MachineInstr &MI) const {
1871 if (Imm == 0) {
1872 MachineBasicBlock &MBB = *MI.getParent();
1873 DebugLoc DL = MI.getDebugLoc();
1874 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg).addReg(RegNo: RISCV::X0);
1875 RBI.constrainGenericRegister(Reg: DstReg, RC: RISCV::GPRRegClass, MRI&: *MRI);
1876 return true;
1877 }
1878
1879 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: *Subtarget);
1880 return materializeInstSeq(DstReg, Seq, MI);
1881}
1882
1883bool RISCVInstructionSelector::materializeInstSeq(
1884 Register DstReg, const RISCVMatInt::InstSeq &Seq, MachineInstr &MI) const {
1885 assert(!Seq.empty() && "materializeInstSeq requires a non-empty sequence");
1886
1887 MachineBasicBlock &MBB = *MI.getParent();
1888 DebugLoc DL = MI.getDebugLoc();
1889 unsigned NumInsts = Seq.size();
1890 Register SrcReg = RISCV::X0;
1891
1892 for (unsigned i = 0; i < NumInsts; i++) {
1893 Register TmpReg = i < NumInsts - 1
1894 ? MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass)
1895 : DstReg;
1896 const RISCVMatInt::Inst &I = Seq[i];
1897 MachineInstr *Result;
1898
1899 switch (I.getOpndKind()) {
1900 case RISCVMatInt::Imm:
1901 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1902 .addImm(Val: I.getImm());
1903 break;
1904 case RISCVMatInt::RegX0:
1905 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1906 .addReg(RegNo: SrcReg)
1907 .addReg(RegNo: RISCV::X0);
1908 break;
1909 case RISCVMatInt::RegReg:
1910 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1911 .addReg(RegNo: SrcReg)
1912 .addReg(RegNo: SrcReg);
1913 break;
1914 case RISCVMatInt::RegImm:
1915 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1916 .addReg(RegNo: SrcReg)
1917 .addImm(Val: I.getImm());
1918 break;
1919 }
1920
1921 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1922
1923 SrcReg = TmpReg;
1924 }
1925
1926 return true;
1927}
1928
1929InstructionSelector::ComplexRendererFns
1930RISCVInstructionSelector::computeConstAddr(int64_t CVal, bool IsPrefetch,
1931 Register OrigBase) const {
1932 // Split the constant into a materialized high part (the base) and
1933 // a simm12 low part (the offset). For prefetch the low part
1934 // must additionally be a multiple of 32 (simm12_lsb00000).
1935 int64_t Lo12 = SignExtend64<12>(x: CVal);
1936 int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
1937 auto emit = [&](ConstAddrPlan Plan) -> ComplexRendererFns {
1938 return {{[=](MachineInstrBuilder &MIB) {
1939 MIB.addReg(RegNo: materializeConstBase(MIB, Plan, OrigBase));
1940 },
1941 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Plan.Lo12); }}};
1942 };
1943 if (!Subtarget->is64Bit() || isInt<32>(x: Hi)) {
1944 if (IsPrefetch && (Lo12 & 0b11111) != 0)
1945 return std::nullopt;
1946 ConstAddrPlan Plan;
1947 Plan.Lo12 = Lo12;
1948 if (Hi) {
1949 Plan.Kind = ConstAddrPlan::LUI;
1950 Plan.Hi20 = (Hi >> 12) & 0xfffff;
1951 }
1952 return emit(std::move(Plan));
1953 }
1954
1955 // Otherwise ask constant materialization how it would handle the constant
1956 // and fold the trailing ADDI into the offset.
1957 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: CVal, STI: *Subtarget);
1958 if (Seq.back().getOpcode() != RISCV::ADDI)
1959 return std::nullopt;
1960 Lo12 = Seq.back().getImm();
1961 if (IsPrefetch && (Lo12 & 0b11111) != 0)
1962 return std::nullopt;
1963 Seq.pop_back();
1964 if (Seq.empty())
1965 return std::nullopt;
1966 ConstAddrPlan Plan;
1967 Plan.Kind = ConstAddrPlan::InstSeq;
1968 Plan.Seq = std::move(Seq);
1969 Plan.Lo12 = Lo12;
1970 return emit(std::move(Plan));
1971}
1972
1973Register
1974RISCVInstructionSelector::materializeConstBase(MachineInstrBuilder &MIB,
1975 const ConstAddrPlan &Plan,
1976 Register OrigBase) const {
1977 MachineBasicBlock &MBB = *MIB->getParent();
1978 DebugLoc DL = MIB->getDebugLoc();
1979 MachineInstr &InsertPt = *MIB.getInstr();
1980
1981 Register HiReg = RISCV::X0;
1982 switch (Plan.Kind) {
1983 case ConstAddrPlan::X0:
1984 break;
1985 case ConstAddrPlan::LUI: {
1986 HiReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1987 MachineInstr *LUI = BuildMI(BB&: MBB, I&: InsertPt, MIMD: DL, MCID: TII.get(Opcode: RISCV::LUI), DestReg: HiReg)
1988 .addImm(Val: Plan.Hi20);
1989 constrainSelectedInstRegOperands(I&: *LUI, TII, TRI, RBI);
1990 break;
1991 }
1992 case ConstAddrPlan::InstSeq: {
1993 HiReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1994 materializeInstSeq(DstReg: HiReg, Seq: Plan.Seq, MI&: InsertPt);
1995 break;
1996 }
1997 }
1998
1999 // For G_PTR_ADD + large constant, add the original base to the materialized
2000 // high part.
2001 if (OrigBase.isValid() && HiReg != RISCV::X0) {
2002 Register BaseReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2003 MachineInstr *Add = BuildMI(BB&: MBB, I&: InsertPt, MIMD: DL, MCID: TII.get(Opcode: RISCV::ADD), DestReg: BaseReg)
2004 .addReg(RegNo: OrigBase)
2005 .addReg(RegNo: HiReg);
2006 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
2007 return BaseReg;
2008 }
2009 return OrigBase.isValid() ? OrigBase : HiReg;
2010}
2011
2012bool RISCVInstructionSelector::selectAddr(MachineInstr &MI, bool IsLocal,
2013 bool IsExternWeak) const {
2014 assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
2015 MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
2016 MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
2017 "Unexpected opcode");
2018
2019 const MachineOperand &DispMO = MI.getOperand(i: 1);
2020
2021 Register DefReg = MI.getOperand(i: 0).getReg();
2022 const LLT DefTy = MRI->getType(Reg: DefReg);
2023
2024 // When HWASAN is used and tagging of global variables is enabled
2025 // they should be accessed via the GOT, since the tagged address of a global
2026 // is incompatible with existing code models. This also applies to non-pic
2027 // mode.
2028 if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
2029 if (IsLocal && !Subtarget->allowTaggedGlobals()) {
2030 // Use PC-relative addressing to access the symbol. This generates the
2031 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
2032 // %pcrel_lo(auipc)).
2033 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
2034 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
2035 return true;
2036 }
2037
2038 // Use PC-relative addressing to access the GOT for this symbol, then
2039 // load the address from the GOT. This generates the pattern (PseudoLGA
2040 // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
2041 // %pcrel_lo(auipc))).
2042 MachineFunction &MF = *MI.getParent()->getParent();
2043 MachineMemOperand *MemOp = MF.getMachineMemOperand(
2044 PtrInfo: MachinePointerInfo::getGOT(MF),
2045 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
2046 MachineMemOperand::MOInvariant,
2047 MemTy: DefTy, BaseAlignment: Align(DefTy.getSizeInBits() / 8));
2048
2049 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2050 MCID: TII.get(Opcode: RISCV::PseudoLGA), DestReg: DefReg)
2051 .addDisp(Disp: DispMO, off: 0)
2052 .addMemOperand(MMO: MemOp);
2053
2054 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
2055
2056 MI.eraseFromParent();
2057 return true;
2058 }
2059
2060 switch (TM.getCodeModel()) {
2061 default: {
2062 reportGISelFailure(MF&: *MF, MORE&: *MORE, PassName: getName(),
2063 Msg: "Unsupported code model for lowering", MI);
2064 return false;
2065 }
2066 case CodeModel::Small: {
2067 // Must lie within a single 2 GiB address range and must lie between
2068 // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
2069 // (lui %hi(sym)) %lo(sym)).
2070 Register AddrHiDest = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2071 MachineInstr *AddrHi = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2072 MCID: TII.get(Opcode: RISCV::LUI), DestReg: AddrHiDest)
2073 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_HI);
2074
2075 constrainSelectedInstRegOperands(I&: *AddrHi, TII, TRI, RBI);
2076
2077 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2078 MCID: TII.get(Opcode: RISCV::ADDI), DestReg: DefReg)
2079 .addReg(RegNo: AddrHiDest)
2080 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_LO);
2081
2082 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
2083
2084 MI.eraseFromParent();
2085 return true;
2086 }
2087 case CodeModel::Medium:
2088 // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
2089 // relocation needs to reference a label that points to the auipc
2090 // instruction itself, not the global. This cannot be done inside the
2091 // instruction selector.
2092 if (IsExternWeak) {
2093 // An extern weak symbol may be undefined, i.e. have value 0, which may
2094 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
2095 // symbol. This generates the pattern (PseudoLGA sym), which expands to
2096 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
2097 MachineFunction &MF = *MI.getParent()->getParent();
2098 MachineMemOperand *MemOp = MF.getMachineMemOperand(
2099 PtrInfo: MachinePointerInfo::getGOT(MF),
2100 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
2101 MachineMemOperand::MOInvariant,
2102 MemTy: DefTy, BaseAlignment: Align(DefTy.getSizeInBits() / 8));
2103
2104 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2105 MCID: TII.get(Opcode: RISCV::PseudoLGA), DestReg: DefReg)
2106 .addDisp(Disp: DispMO, off: 0)
2107 .addMemOperand(MMO: MemOp);
2108
2109 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
2110
2111 MI.eraseFromParent();
2112 return true;
2113 }
2114
2115 // Generate a sequence for accessing addresses within any 2GiB range
2116 // within the address space. This generates the pattern (PseudoLLA sym),
2117 // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
2118 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
2119 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
2120 return true;
2121 }
2122
2123 return false;
2124}
2125
2126bool RISCVInstructionSelector::selectSelect(MachineInstr &MI) const {
2127 auto &SelectMI = cast<GSelect>(Val&: MI);
2128
2129 Register LHS, RHS;
2130 RISCVCC::CondCode CC;
2131 getOperandsForBranch(CondReg: SelectMI.getCondReg(), CC, LHS, RHS, MRI&: *MRI);
2132
2133 Register DstReg = SelectMI.getReg(Idx: 0);
2134
2135 unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
2136 if (RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
2137 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
2138 Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
2139 : RISCV::Select_FPR64_Using_CC_GPR;
2140 }
2141
2142 MachineInstr *Result =
2143 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: Opc))
2144 .addDef(RegNo: DstReg)
2145 .addReg(RegNo: LHS)
2146 .addReg(RegNo: RHS)
2147 .addImm(Val: CC)
2148 .addReg(RegNo: SelectMI.getTrueReg())
2149 .addReg(RegNo: SelectMI.getFalseReg());
2150 MI.eraseFromParent();
2151 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
2152 return true;
2153}
2154
2155// Convert an FCMP predicate to one of the supported F or D instructions.
2156static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
2157 assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");
2158 switch (Pred) {
2159 default:
2160 llvm_unreachable("Unsupported predicate");
2161 case CmpInst::FCMP_OLT:
2162 return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
2163 case CmpInst::FCMP_OLE:
2164 return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
2165 case CmpInst::FCMP_OEQ:
2166 return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
2167 }
2168}
2169
2170// Try legalizing an FCMP by swapping or inverting the predicate to one that
2171// is supported.
2172static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,
2173 CmpInst::Predicate &Pred, bool &NeedInvert) {
2174 auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
2175 return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
2176 Pred == CmpInst::FCMP_OEQ;
2177 };
2178
2179 assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
2180
2181 CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(pred: Pred);
2182 if (isLegalFCmpPredicate(InvPred)) {
2183 Pred = InvPred;
2184 std::swap(a&: LHS, b&: RHS);
2185 return true;
2186 }
2187
2188 InvPred = CmpInst::getInversePredicate(pred: Pred);
2189 NeedInvert = true;
2190 if (isLegalFCmpPredicate(InvPred)) {
2191 Pred = InvPred;
2192 return true;
2193 }
2194 InvPred = CmpInst::getSwappedPredicate(pred: InvPred);
2195 if (isLegalFCmpPredicate(InvPred)) {
2196 Pred = InvPred;
2197 std::swap(a&: LHS, b&: RHS);
2198 return true;
2199 }
2200
2201 return false;
2202}
2203
2204// Emit a sequence of instructions to compare LHS and RHS using Pred. Return
2205// the result in DstReg.
2206// FIXME: Maybe we should expand this earlier.
2207bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI) const {
2208 auto &CmpMI = cast<GFCmp>(Val&: MI);
2209 CmpInst::Predicate Pred = CmpMI.getCond();
2210
2211 Register DstReg = CmpMI.getReg(Idx: 0);
2212 Register LHS = CmpMI.getLHSReg();
2213 Register RHS = CmpMI.getRHSReg();
2214
2215 unsigned Size = MRI->getType(Reg: LHS).getSizeInBits();
2216 assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
2217
2218 Register TmpReg = DstReg;
2219
2220 bool NeedInvert = false;
2221 // First try swapping operands or inverting.
2222 if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
2223 if (NeedInvert)
2224 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2225 MachineInstr *Cmp = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2226 MCID: TII.get(Opcode: getFCmpOpcode(Pred, Size)), DestReg: TmpReg)
2227 .setMIFlag(MachineInstr::NoFPExcept)
2228 .addReg(RegNo: LHS)
2229 .addReg(RegNo: RHS);
2230 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
2231 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
2232 // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
2233 NeedInvert = Pred == CmpInst::FCMP_UEQ;
2234 Register Cmp1Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2235 MachineInstr *Cmp1 =
2236 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2237 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size)), DestReg: Cmp1Reg)
2238 .setMIFlag(MachineInstr::NoFPExcept)
2239 .addReg(RegNo: LHS)
2240 .addReg(RegNo: RHS);
2241 constrainSelectedInstRegOperands(I&: *Cmp1, TII, TRI, RBI);
2242 Register Cmp2Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2243 MachineInstr *Cmp2 =
2244 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2245 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size)), DestReg: Cmp2Reg)
2246 .setMIFlag(MachineInstr::NoFPExcept)
2247 .addReg(RegNo: RHS)
2248 .addReg(RegNo: LHS);
2249 constrainSelectedInstRegOperands(I&: *Cmp2, TII, TRI, RBI);
2250 if (NeedInvert)
2251 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2252 MachineInstr *Or = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2253 MCID: TII.get(Opcode: RISCV::OR), DestReg: TmpReg)
2254 .addReg(RegNo: Cmp1Reg)
2255 .addReg(RegNo: Cmp2Reg);
2256 constrainSelectedInstRegOperands(I&: *Or, TII, TRI, RBI);
2257 } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
2258 // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
2259 // If LHS and RHS are the same, a single FEQ suffices.
2260 NeedInvert = Pred == CmpInst::FCMP_UNO;
2261 if (NeedInvert)
2262 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2263 if (LHS == RHS) {
2264 MachineInstr *Cmp =
2265 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2266 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: TmpReg)
2267 .setMIFlag(MachineInstr::NoFPExcept)
2268 .addReg(RegNo: LHS)
2269 .addReg(RegNo: LHS);
2270 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
2271 } else {
2272 Register Cmp1Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2273 MachineInstr *Cmp1 =
2274 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2275 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: Cmp1Reg)
2276 .setMIFlag(MachineInstr::NoFPExcept)
2277 .addReg(RegNo: LHS)
2278 .addReg(RegNo: LHS);
2279 constrainSelectedInstRegOperands(I&: *Cmp1, TII, TRI, RBI);
2280 Register Cmp2Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2281 MachineInstr *Cmp2 =
2282 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2283 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: Cmp2Reg)
2284 .setMIFlag(MachineInstr::NoFPExcept)
2285 .addReg(RegNo: RHS)
2286 .addReg(RegNo: RHS);
2287 constrainSelectedInstRegOperands(I&: *Cmp2, TII, TRI, RBI);
2288 MachineInstr *And = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2289 MCID: TII.get(Opcode: RISCV::AND), DestReg: TmpReg)
2290 .addReg(RegNo: Cmp1Reg)
2291 .addReg(RegNo: Cmp2Reg);
2292 constrainSelectedInstRegOperands(I&: *And, TII, TRI, RBI);
2293 }
2294 } else
2295 llvm_unreachable("Unhandled predicate");
2296
2297 // Emit an XORI to invert the result if needed.
2298 if (NeedInvert) {
2299 MachineInstr *Xor = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2300 MCID: TII.get(Opcode: RISCV::XORI), DestReg: DstReg)
2301 .addReg(RegNo: TmpReg)
2302 .addImm(Val: 1);
2303 constrainSelectedInstRegOperands(I&: *Xor, TII, TRI, RBI);
2304 }
2305
2306 MI.eraseFromParent();
2307 return true;
2308}
2309
2310void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
2311 SyncScope::ID FenceSSID,
2312 MachineInstr &MI) const {
2313 MachineBasicBlock &MBB = *MI.getParent();
2314 DebugLoc DL = MI.getDebugLoc();
2315
2316 if (STI.hasStdExtZtso()) {
2317 // The only fence that needs an instruction is a sequentially-consistent
2318 // cross-thread fence.
2319 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
2320 FenceSSID == SyncScope::System) {
2321 // fence rw, rw
2322 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE))
2323 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W)
2324 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W);
2325 return;
2326 }
2327
2328 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
2329 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER));
2330 return;
2331 }
2332
2333 // singlethread fences only synchronize with signal handlers on the same
2334 // thread and thus only need to preserve instruction order, not actually
2335 // enforce memory ordering.
2336 if (FenceSSID == SyncScope::SingleThread) {
2337 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER));
2338 return;
2339 }
2340
2341 // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
2342 // Manual: Volume I.
2343 unsigned Pred, Succ;
2344 switch (FenceOrdering) {
2345 default:
2346 llvm_unreachable("Unexpected ordering");
2347 case AtomicOrdering::AcquireRelease:
2348 // fence acq_rel -> fence.tso
2349 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE_TSO));
2350 return;
2351 case AtomicOrdering::Acquire:
2352 // fence acquire -> fence r, rw
2353 Pred = RISCVFenceField::R;
2354 Succ = RISCVFenceField::R | RISCVFenceField::W;
2355 break;
2356 case AtomicOrdering::Release:
2357 // fence release -> fence rw, w
2358 Pred = RISCVFenceField::R | RISCVFenceField::W;
2359 Succ = RISCVFenceField::W;
2360 break;
2361 case AtomicOrdering::SequentiallyConsistent:
2362 // fence seq_cst -> fence rw, rw
2363 Pred = RISCVFenceField::R | RISCVFenceField::W;
2364 Succ = RISCVFenceField::R | RISCVFenceField::W;
2365 break;
2366 }
2367 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE)).addImm(Val: Pred).addImm(Val: Succ);
2368}
2369
2370namespace llvm {
2371InstructionSelector *
2372createRISCVInstructionSelector(const RISCVTargetMachine &TM,
2373 const RISCVSubtarget &Subtarget,
2374 const RISCVRegisterBankInfo &RBI) {
2375 return new RISCVInstructionSelector(TM, Subtarget, RBI);
2376}
2377} // end namespace llvm
2378