1//===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// RISC-V.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/RISCVMatInt.h"
15#include "RISCVRegisterBankInfo.h"
16#include "RISCVSubtarget.h"
17#include "RISCVTargetMachine.h"
18#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
19#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
20#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
21#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
22#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/MachineJumpTableInfo.h"
25#include "llvm/IR/IntrinsicsRISCV.h"
26#include "llvm/Support/Debug.h"
27
28#define DEBUG_TYPE "riscv-isel"
29
30using namespace llvm;
31using namespace MIPatternMatch;
32
33#define GET_GLOBALISEL_PREDICATE_BITSET
34#include "RISCVGenGlobalISel.inc"
35#undef GET_GLOBALISEL_PREDICATE_BITSET
36
37namespace {
38
39class RISCVInstructionSelector : public InstructionSelector {
40public:
41 RISCVInstructionSelector(const RISCVTargetMachine &TM,
42 const RISCVSubtarget &STI,
43 const RISCVRegisterBankInfo &RBI);
44
45 bool select(MachineInstr &MI) override;
46
47 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
48 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
49 BlockFrequencyInfo *BFI) override {
50 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
51 MRI = &MF.getRegInfo();
52 }
53
54 static const char *getName() { return DEBUG_TYPE; }
55
56private:
57 const TargetRegisterClass *
58 getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB) const;
59
60 static constexpr unsigned MaxRecursionDepth = 6;
61
62 bool hasAllNBitUsers(const MachineInstr &MI, unsigned Bits,
63 const unsigned Depth = 0) const;
64 bool hasAllHUsers(const MachineInstr &MI) const {
65 return hasAllNBitUsers(MI, Bits: 16);
66 }
67 bool hasAllWUsers(const MachineInstr &MI) const {
68 return hasAllNBitUsers(MI, Bits: 32);
69 }
70
71 bool isRegInGprb(Register Reg) const;
72 bool isRegInFprb(Register Reg) const;
73
74 // tblgen-erated 'select' implementation, used as the initial selector for
75 // the patterns that don't require complex C++.
76 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
77
78 // A lowering phase that runs before any selection attempts.
79 // Returns true if the instruction was modified.
80 void preISelLower(MachineInstr &MI, MachineIRBuilder &MIB);
81
82 bool replacePtrWithInt(MachineOperand &Op, MachineIRBuilder &MIB);
83
84 // Custom selection methods
85 bool selectCopy(MachineInstr &MI) const;
86 bool selectImplicitDef(MachineInstr &MI, MachineIRBuilder &MIB) const;
87 bool materializeImm(Register Reg, int64_t Imm, MachineIRBuilder &MIB) const;
88 bool selectAddr(MachineInstr &MI, MachineIRBuilder &MIB, bool IsLocal = true,
89 bool IsExternWeak = false) const;
90 bool selectSelect(MachineInstr &MI, MachineIRBuilder &MIB) const;
91 bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB) const;
92 void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
93 MachineIRBuilder &MIB) const;
94 bool selectUnmergeValues(MachineInstr &MI, MachineIRBuilder &MIB) const;
95 void addVectorLoadStoreOperands(MachineInstr &I,
96 SmallVectorImpl<SrcOp> &SrcOps,
97 unsigned &CurOp, bool IsMasked,
98 bool IsStridedOrIndexed,
99 LLT *IndexVT = nullptr) const;
100 bool selectIntrinsicWithSideEffects(MachineInstr &I,
101 MachineIRBuilder &MIB) const;
102 bool selectIntrinsic(MachineInstr &I, MachineIRBuilder &MIB) const;
103 bool selectExtractSubvector(MachineInstr &MI, MachineIRBuilder &MIB) const;
104
105 ComplexRendererFns selectShiftMask(MachineOperand &Root,
106 unsigned ShiftWidth) const;
107 ComplexRendererFns selectShiftMaskXLen(MachineOperand &Root) const {
108 return selectShiftMask(Root, ShiftWidth: STI.getXLen());
109 }
110 ComplexRendererFns selectShiftMask32(MachineOperand &Root) const {
111 return selectShiftMask(Root, ShiftWidth: 32);
112 }
113 ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
114
115 ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;
116 template <unsigned Bits>
117 ComplexRendererFns selectSExtBits(MachineOperand &Root) const {
118 return selectSExtBits(Root, Bits);
119 }
120
121 ComplexRendererFns selectZExtBits(MachineOperand &Root, unsigned Bits) const;
122 template <unsigned Bits>
123 ComplexRendererFns selectZExtBits(MachineOperand &Root) const {
124 return selectZExtBits(Root, Bits);
125 }
126
127 ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
128 template <unsigned ShAmt>
129 ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
130 return selectSHXADDOp(Root, ShAmt);
131 }
132
133 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
134 unsigned ShAmt) const;
135 template <unsigned ShAmt>
136 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
137 return selectSHXADD_UWOp(Root, ShAmt);
138 }
139
140 ComplexRendererFns renderVLOp(MachineOperand &Root) const;
141
142 // Custom renderers for tablegen
143 void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
144 int OpIdx) const;
145 void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
146 int OpIdx) const;
147 void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
148 int OpIdx) const;
149 void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
150 int OpIdx) const;
151 void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI,
152 int OpIdx) const;
153
154 void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
155 int OpIdx) const;
156 void renderXLenSubTrailingOnes(MachineInstrBuilder &MIB,
157 const MachineInstr &MI, int OpIdx) const;
158
159 void renderAddiPairImmLarge(MachineInstrBuilder &MIB, const MachineInstr &MI,
160 int OpIdx) const;
161 void renderAddiPairImmSmall(MachineInstrBuilder &MIB, const MachineInstr &MI,
162 int OpIdx) const;
163
164 const RISCVSubtarget &STI;
165 const RISCVInstrInfo &TII;
166 const RISCVRegisterInfo &TRI;
167 const RISCVRegisterBankInfo &RBI;
168 const RISCVTargetMachine &TM;
169
170 MachineRegisterInfo *MRI = nullptr;
171
172 // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
173 // uses "STI." in the code generated by TableGen. We need to unify the name of
174 // Subtarget variable.
175 const RISCVSubtarget *Subtarget = &STI;
176
177#define GET_GLOBALISEL_PREDICATES_DECL
178#include "RISCVGenGlobalISel.inc"
179#undef GET_GLOBALISEL_PREDICATES_DECL
180
181#define GET_GLOBALISEL_TEMPORARIES_DECL
182#include "RISCVGenGlobalISel.inc"
183#undef GET_GLOBALISEL_TEMPORARIES_DECL
184};
185
186} // end anonymous namespace
187
188#define GET_GLOBALISEL_IMPL
189#include "RISCVGenGlobalISel.inc"
190#undef GET_GLOBALISEL_IMPL
191
192RISCVInstructionSelector::RISCVInstructionSelector(
193 const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
194 const RISCVRegisterBankInfo &RBI)
195 : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
196 TM(TM),
197
198#define GET_GLOBALISEL_PREDICATES_INIT
199#include "RISCVGenGlobalISel.inc"
200#undef GET_GLOBALISEL_PREDICATES_INIT
201#define GET_GLOBALISEL_TEMPORARIES_INIT
202#include "RISCVGenGlobalISel.inc"
203#undef GET_GLOBALISEL_TEMPORARIES_INIT
204{
205}
206
207// Mimics optimizations in ISel and RISCVOptWInst Pass
208bool RISCVInstructionSelector::hasAllNBitUsers(const MachineInstr &MI,
209 unsigned Bits,
210 const unsigned Depth) const {
211
212 assert((MI.getOpcode() == TargetOpcode::G_ADD ||
213 MI.getOpcode() == TargetOpcode::G_SUB ||
214 MI.getOpcode() == TargetOpcode::G_MUL ||
215 MI.getOpcode() == TargetOpcode::G_SHL ||
216 MI.getOpcode() == TargetOpcode::G_LSHR ||
217 MI.getOpcode() == TargetOpcode::G_AND ||
218 MI.getOpcode() == TargetOpcode::G_OR ||
219 MI.getOpcode() == TargetOpcode::G_XOR ||
220 MI.getOpcode() == TargetOpcode::G_SEXT_INREG || Depth != 0) &&
221 "Unexpected opcode");
222
223 if (Depth >= RISCVInstructionSelector::MaxRecursionDepth)
224 return false;
225
226 auto DestReg = MI.getOperand(i: 0).getReg();
227 for (auto &UserOp : MRI->use_nodbg_operands(Reg: DestReg)) {
228 assert(UserOp.getParent() && "UserOp must have a parent");
229 const MachineInstr &UserMI = *UserOp.getParent();
230 unsigned OpIdx = UserOp.getOperandNo();
231
232 switch (UserMI.getOpcode()) {
233 default:
234 return false;
235 case RISCV::ADDW:
236 case RISCV::ADDIW:
237 case RISCV::SUBW:
238 case RISCV::FCVT_D_W:
239 case RISCV::FCVT_S_W:
240 if (Bits >= 32)
241 break;
242 return false;
243 case RISCV::SLL:
244 case RISCV::SRA:
245 case RISCV::SRL:
246 // Shift amount operands only use log2(Xlen) bits.
247 if (OpIdx == 2 && Bits >= Log2_32(Value: Subtarget->getXLen()))
248 break;
249 return false;
250 case RISCV::SLLI:
251 // SLLI only uses the lower (XLen - ShAmt) bits.
252 if (Bits >= Subtarget->getXLen() - UserMI.getOperand(i: 2).getImm())
253 break;
254 return false;
255 case RISCV::ANDI:
256 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(
257 Value: (uint64_t)UserMI.getOperand(i: 2).getImm()))
258 break;
259 goto RecCheck;
260 case RISCV::AND:
261 case RISCV::OR:
262 case RISCV::XOR:
263 RecCheck:
264 if (hasAllNBitUsers(MI: UserMI, Bits, Depth: Depth + 1))
265 break;
266 return false;
267 case RISCV::SRLI: {
268 unsigned ShAmt = UserMI.getOperand(i: 2).getImm();
269 // If we are shifting right by less than Bits, and users don't demand any
270 // bits that were shifted into [Bits-1:0], then we can consider this as an
271 // N-Bit user.
272 if (Bits > ShAmt && hasAllNBitUsers(MI: UserMI, Bits: Bits - ShAmt, Depth: Depth + 1))
273 break;
274 return false;
275 }
276 }
277 }
278
279 return true;
280}
281
282InstructionSelector::ComplexRendererFns
283RISCVInstructionSelector::selectShiftMask(MachineOperand &Root,
284 unsigned ShiftWidth) const {
285 if (!Root.isReg())
286 return std::nullopt;
287
288 using namespace llvm::MIPatternMatch;
289
290 Register ShAmtReg = Root.getReg();
291 // Peek through zext.
292 Register ZExtSrcReg;
293 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrcReg))))
294 ShAmtReg = ZExtSrcReg;
295
296 APInt AndMask;
297 Register AndSrcReg;
298 // Try to combine the following pattern (applicable to other shift
299 // instructions as well as 32-bit ones):
300 //
301 // %4:gprb(s64) = G_AND %3, %2
302 // %5:gprb(s64) = G_LSHR %1, %4(s64)
303 //
304 // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
305 // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
306 // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
307 // then it can be eliminated. Given register rs1 or rs2 holding a constant
308 // (the and mask), there are two cases G_AND can be erased:
309 //
310 // 1. the lowest log2(XLEN) bits of the and mask are all set
311 // 2. the bits of the register being masked are already unset (zero set)
312 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: AndSrcReg), R: m_ICst(Cst&: AndMask)))) {
313 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
314 if (ShMask.isSubsetOf(RHS: AndMask)) {
315 ShAmtReg = AndSrcReg;
316 } else {
317 // SimplifyDemandedBits may have optimized the mask so try restoring any
318 // bits that are known zero.
319 KnownBits Known = VT->getKnownBits(R: AndSrcReg);
320 if (ShMask.isSubsetOf(RHS: AndMask | Known.Zero))
321 ShAmtReg = AndSrcReg;
322 }
323 }
324
325 APInt Imm;
326 Register Reg;
327 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAdd(L: m_Reg(R&: Reg), R: m_ICst(Cst&: Imm)))) {
328 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0)
329 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
330 // to avoid the ADD.
331 ShAmtReg = Reg;
332 } else if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GSub(L: m_ICst(Cst&: Imm), R: m_Reg(R&: Reg)))) {
333 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0) {
334 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
335 // to generate a NEG instead of a SUB of a constant.
336 ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
337 unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
338 return {{[=](MachineInstrBuilder &MIB) {
339 MachineIRBuilder(*MIB.getInstr())
340 .buildInstr(Opc: NegOpc, DstOps: {ShAmtReg}, SrcOps: {Register(RISCV::X0), Reg});
341 MIB.addReg(RegNo: ShAmtReg);
342 }}};
343 }
344 if (Imm.urem(RHS: ShiftWidth) == ShiftWidth - 1) {
345 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
346 // to generate a NOT instead of a SUB of a constant.
347 ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
348 return {{[=](MachineInstrBuilder &MIB) {
349 MachineIRBuilder(*MIB.getInstr())
350 .buildInstr(Opc: RISCV::XORI, DstOps: {ShAmtReg}, SrcOps: {Reg})
351 .addImm(Val: -1);
352 MIB.addReg(RegNo: ShAmtReg);
353 }}};
354 }
355 }
356
357 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ShAmtReg); }}};
358}
359
360InstructionSelector::ComplexRendererFns
361RISCVInstructionSelector::selectSExtBits(MachineOperand &Root,
362 unsigned Bits) const {
363 if (!Root.isReg())
364 return std::nullopt;
365 Register RootReg = Root.getReg();
366 MachineInstr *RootDef = MRI->getVRegDef(Reg: RootReg);
367
368 if (RootDef->getOpcode() == TargetOpcode::G_SEXT_INREG &&
369 RootDef->getOperand(i: 2).getImm() == Bits) {
370 return {
371 {[=](MachineInstrBuilder &MIB) { MIB.add(MO: RootDef->getOperand(i: 1)); }}};
372 }
373
374 unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits();
375 if ((Size - VT->computeNumSignBits(R: RootReg)) < Bits)
376 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}};
377
378 return std::nullopt;
379}
380
381InstructionSelector::ComplexRendererFns
382RISCVInstructionSelector::selectZExtBits(MachineOperand &Root,
383 unsigned Bits) const {
384 if (!Root.isReg())
385 return std::nullopt;
386 Register RootReg = Root.getReg();
387
388 Register RegX;
389 uint64_t Mask = maskTrailingOnes<uint64_t>(N: Bits);
390 if (mi_match(R: RootReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: RegX), R: m_SpecificICst(RequestedValue: Mask)))) {
391 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}};
392 }
393
394 if (mi_match(R: RootReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: RegX))) &&
395 MRI->getType(Reg: RegX).getScalarSizeInBits() == Bits)
396 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}};
397
398 unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits();
399 if (VT->maskedValueIsZero(Val: RootReg, Mask: APInt::getBitsSetFrom(numBits: Size, loBit: Bits)))
400 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}};
401
402 return std::nullopt;
403}
404
405InstructionSelector::ComplexRendererFns
406RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
407 unsigned ShAmt) const {
408 using namespace llvm::MIPatternMatch;
409
410 if (!Root.isReg())
411 return std::nullopt;
412 Register RootReg = Root.getReg();
413
414 const unsigned XLen = STI.getXLen();
415 APInt Mask, C2;
416 Register RegY;
417 std::optional<bool> LeftShift;
418 // (and (shl y, c2), mask)
419 if (mi_match(R: RootReg, MRI: *MRI,
420 P: m_GAnd(L: m_GShl(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
421 LeftShift = true;
422 // (and (lshr y, c2), mask)
423 else if (mi_match(R: RootReg, MRI: *MRI,
424 P: m_GAnd(L: m_GLShr(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
425 LeftShift = false;
426
427 if (LeftShift.has_value()) {
428 if (*LeftShift)
429 Mask &= maskTrailingZeros<uint64_t>(N: C2.getLimitedValue());
430 else
431 Mask &= maskTrailingOnes<uint64_t>(N: XLen - C2.getLimitedValue());
432
433 if (Mask.isShiftedMask()) {
434 unsigned Leading = XLen - Mask.getActiveBits();
435 unsigned Trailing = Mask.countr_zero();
436 // Given (and (shl y, c2), mask) in which mask has no leading zeros and
437 // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
438 if (*LeftShift && Leading == 0 && C2.ult(RHS: Trailing) && Trailing == ShAmt) {
439 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
440 return {{[=](MachineInstrBuilder &MIB) {
441 MachineIRBuilder(*MIB.getInstr())
442 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
443 .addImm(Val: Trailing - C2.getLimitedValue());
444 MIB.addReg(RegNo: DstReg);
445 }}};
446 }
447
448 // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
449 // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
450 if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
451 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
452 return {{[=](MachineInstrBuilder &MIB) {
453 MachineIRBuilder(*MIB.getInstr())
454 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
455 .addImm(Val: Leading + Trailing);
456 MIB.addReg(RegNo: DstReg);
457 }}};
458 }
459 }
460 }
461
462 LeftShift.reset();
463
464 // (shl (and y, mask), c2)
465 if (mi_match(R: RootReg, MRI: *MRI,
466 P: m_GShl(L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
467 R: m_ICst(Cst&: C2))))
468 LeftShift = true;
469 // (lshr (and y, mask), c2)
470 else if (mi_match(R: RootReg, MRI: *MRI,
471 P: m_GLShr(L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
472 R: m_ICst(Cst&: C2))))
473 LeftShift = false;
474
475 if (LeftShift.has_value() && Mask.isShiftedMask()) {
476 unsigned Leading = XLen - Mask.getActiveBits();
477 unsigned Trailing = Mask.countr_zero();
478
479 // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
480 // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
481 bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&
482 (Trailing + C2.getLimitedValue()) == ShAmt;
483 if (!Cond)
484 // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
485 // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
486 Cond = !*LeftShift && Leading == 32 && C2.ult(RHS: Trailing) &&
487 (Trailing - C2.getLimitedValue()) == ShAmt;
488
489 if (Cond) {
490 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
491 return {{[=](MachineInstrBuilder &MIB) {
492 MachineIRBuilder(*MIB.getInstr())
493 .buildInstr(Opc: RISCV::SRLIW, DstOps: {DstReg}, SrcOps: {RegY})
494 .addImm(Val: Trailing);
495 MIB.addReg(RegNo: DstReg);
496 }}};
497 }
498 }
499
500 return std::nullopt;
501}
502
503InstructionSelector::ComplexRendererFns
504RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
505 unsigned ShAmt) const {
506 using namespace llvm::MIPatternMatch;
507
508 if (!Root.isReg())
509 return std::nullopt;
510 Register RootReg = Root.getReg();
511
512 // Given (and (shl x, c2), mask) in which mask is a shifted mask with
513 // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
514 // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
515 APInt Mask, C2;
516 Register RegX;
517 if (mi_match(
518 R: RootReg, MRI: *MRI,
519 P: m_OneNonDBGUse(SP: m_GAnd(L: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: RegX), R: m_ICst(Cst&: C2))),
520 R: m_ICst(Cst&: Mask))))) {
521 Mask &= maskTrailingZeros<uint64_t>(N: C2.getLimitedValue());
522
523 if (Mask.isShiftedMask()) {
524 unsigned Leading = Mask.countl_zero();
525 unsigned Trailing = Mask.countr_zero();
526 if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
527 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
528 return {{[=](MachineInstrBuilder &MIB) {
529 MachineIRBuilder(*MIB.getInstr())
530 .buildInstr(Opc: RISCV::SLLI, DstOps: {DstReg}, SrcOps: {RegX})
531 .addImm(Val: C2.getLimitedValue() - ShAmt);
532 MIB.addReg(RegNo: DstReg);
533 }}};
534 }
535 }
536 }
537
538 return std::nullopt;
539}
540
541InstructionSelector::ComplexRendererFns
542RISCVInstructionSelector::renderVLOp(MachineOperand &Root) const {
543 assert(Root.isReg() && "Expected operand to be a Register");
544 MachineInstr *RootDef = MRI->getVRegDef(Reg: Root.getReg());
545
546 if (RootDef->getOpcode() == TargetOpcode::G_CONSTANT) {
547 auto C = RootDef->getOperand(i: 1).getCImm();
548 if (C->getValue().isAllOnes())
549 // If the operand is a G_CONSTANT with value of all ones it is larger than
550 // VLMAX. We convert it to an immediate with value VLMaxSentinel. This is
551 // recognized specially by the vsetvli insertion pass.
552 return {{[=](MachineInstrBuilder &MIB) {
553 MIB.addImm(Val: RISCV::VLMaxSentinel);
554 }}};
555
556 if (isUInt<5>(x: C->getZExtValue())) {
557 uint64_t ZExtC = C->getZExtValue();
558 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ZExtC); }}};
559 }
560 }
561 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Root.getReg()); }}};
562}
563
564InstructionSelector::ComplexRendererFns
565RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
566 if (!Root.isReg())
567 return std::nullopt;
568
569 MachineInstr *RootDef = MRI->getVRegDef(Reg: Root.getReg());
570 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
571 return {{
572 [=](MachineInstrBuilder &MIB) { MIB.add(MO: RootDef->getOperand(i: 1)); },
573 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
574 }};
575 }
576
577 if (isBaseWithConstantOffset(Root, MRI: *MRI)) {
578 MachineOperand &LHS = RootDef->getOperand(i: 1);
579 MachineOperand &RHS = RootDef->getOperand(i: 2);
580 MachineInstr *LHSDef = MRI->getVRegDef(Reg: LHS.getReg());
581 MachineInstr *RHSDef = MRI->getVRegDef(Reg: RHS.getReg());
582
583 int64_t RHSC = RHSDef->getOperand(i: 1).getCImm()->getSExtValue();
584 if (isInt<12>(x: RHSC)) {
585 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
586 return {{
587 [=](MachineInstrBuilder &MIB) { MIB.add(MO: LHSDef->getOperand(i: 1)); },
588 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
589 }};
590
591 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: LHS); },
592 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
593 }
594 }
595
596 // TODO: Need to get the immediate from a G_PTR_ADD. Should this be done in
597 // the combiner?
598 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Root.getReg()); },
599 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
600}
601
602/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
603/// CC Must be an ICMP Predicate.
604static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
605 switch (CC) {
606 default:
607 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
608 case CmpInst::Predicate::ICMP_EQ:
609 return RISCVCC::COND_EQ;
610 case CmpInst::Predicate::ICMP_NE:
611 return RISCVCC::COND_NE;
612 case CmpInst::Predicate::ICMP_ULT:
613 return RISCVCC::COND_LTU;
614 case CmpInst::Predicate::ICMP_SLT:
615 return RISCVCC::COND_LT;
616 case CmpInst::Predicate::ICMP_UGE:
617 return RISCVCC::COND_GEU;
618 case CmpInst::Predicate::ICMP_SGE:
619 return RISCVCC::COND_GE;
620 }
621}
622
623static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC,
624 Register &LHS, Register &RHS,
625 MachineRegisterInfo &MRI) {
626 // Try to fold an ICmp. If that fails, use a NE compare with X0.
627 CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
628 if (!mi_match(R: CondReg, MRI, P: m_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: LHS), R: m_Reg(R&: RHS)))) {
629 LHS = CondReg;
630 RHS = RISCV::X0;
631 CC = RISCVCC::COND_NE;
632 return;
633 }
634
635 // We found an ICmp, do some canonicalization.
636
637 // Adjust comparisons to use comparison with 0 if possible.
638 if (auto Constant = getIConstantVRegSExtVal(VReg: RHS, MRI)) {
639 switch (Pred) {
640 case CmpInst::Predicate::ICMP_SGT:
641 // Convert X > -1 to X >= 0
642 if (*Constant == -1) {
643 CC = RISCVCC::COND_GE;
644 RHS = RISCV::X0;
645 return;
646 }
647 break;
648 case CmpInst::Predicate::ICMP_SLT:
649 // Convert X < 1 to 0 >= X
650 if (*Constant == 1) {
651 CC = RISCVCC::COND_GE;
652 RHS = LHS;
653 LHS = RISCV::X0;
654 return;
655 }
656 break;
657 default:
658 break;
659 }
660 }
661
662 switch (Pred) {
663 default:
664 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
665 case CmpInst::Predicate::ICMP_EQ:
666 case CmpInst::Predicate::ICMP_NE:
667 case CmpInst::Predicate::ICMP_ULT:
668 case CmpInst::Predicate::ICMP_SLT:
669 case CmpInst::Predicate::ICMP_UGE:
670 case CmpInst::Predicate::ICMP_SGE:
671 // These CCs are supported directly by RISC-V branches.
672 break;
673 case CmpInst::Predicate::ICMP_SGT:
674 case CmpInst::Predicate::ICMP_SLE:
675 case CmpInst::Predicate::ICMP_UGT:
676 case CmpInst::Predicate::ICMP_ULE:
677 // These CCs are not supported directly by RISC-V branches, but changing the
678 // direction of the CC and swapping LHS and RHS are.
679 Pred = CmpInst::getSwappedPredicate(pred: Pred);
680 std::swap(a&: LHS, b&: RHS);
681 break;
682 }
683
684 CC = getRISCVCCFromICmp(CC: Pred);
685}
686
687/// Select the RISC-V Zalasr opcode for the G_LOAD or G_STORE operation
688/// \p GenericOpc, appropriate for the GPR register bank and of memory access
689/// size \p OpSize.
690static unsigned selectZalasrLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {
691 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
692 switch (OpSize) {
693 default:
694 llvm_unreachable("Unexpected memory size");
695 case 8:
696 return IsStore ? RISCV::SB_RL : RISCV::LB_AQ;
697 case 16:
698 return IsStore ? RISCV::SH_RL : RISCV::LH_AQ;
699 case 32:
700 return IsStore ? RISCV::SW_RL : RISCV::LW_AQ;
701 case 64:
702 return IsStore ? RISCV::SD_RL : RISCV::LD_AQ;
703 }
704}
705
706/// Select the RISC-V regimm opcode for the G_LOAD or G_STORE operation
707/// \p GenericOpc, appropriate for the GPR register bank and of memory access
708/// size \p OpSize. \returns \p GenericOpc if the combination is unsupported.
709static unsigned selectRegImmLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {
710 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
711 switch (OpSize) {
712 case 8:
713 // Prefer unsigned due to no c.lb in Zcb.
714 return IsStore ? RISCV::SB : RISCV::LBU;
715 case 16:
716 return IsStore ? RISCV::SH : RISCV::LH;
717 case 32:
718 return IsStore ? RISCV::SW : RISCV::LW;
719 case 64:
720 return IsStore ? RISCV::SD : RISCV::LD;
721 }
722
723 return GenericOpc;
724}
725
726void RISCVInstructionSelector::addVectorLoadStoreOperands(
727 MachineInstr &I, SmallVectorImpl<SrcOp> &SrcOps, unsigned &CurOp,
728 bool IsMasked, bool IsStridedOrIndexed, LLT *IndexVT) const {
729 // Base Pointer
730 auto PtrReg = I.getOperand(i: CurOp++).getReg();
731 SrcOps.push_back(Elt: PtrReg);
732
733 // Stride or Index
734 if (IsStridedOrIndexed) {
735 auto StrideReg = I.getOperand(i: CurOp++).getReg();
736 SrcOps.push_back(Elt: StrideReg);
737 if (IndexVT)
738 *IndexVT = MRI->getType(Reg: StrideReg);
739 }
740
741 // Mask
742 if (IsMasked) {
743 auto MaskReg = I.getOperand(i: CurOp++).getReg();
744 SrcOps.push_back(Elt: MaskReg);
745 }
746}
747
748bool RISCVInstructionSelector::selectIntrinsicWithSideEffects(
749 MachineInstr &I, MachineIRBuilder &MIB) const {
750 // Find the intrinsic ID.
751 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
752 // Select the instruction.
753 switch (IntrinID) {
754 default:
755 return false;
756 case Intrinsic::riscv_vlm:
757 case Intrinsic::riscv_vle:
758 case Intrinsic::riscv_vle_mask:
759 case Intrinsic::riscv_vlse:
760 case Intrinsic::riscv_vlse_mask: {
761 bool IsMasked = IntrinID == Intrinsic::riscv_vle_mask ||
762 IntrinID == Intrinsic::riscv_vlse_mask;
763 bool IsStrided = IntrinID == Intrinsic::riscv_vlse ||
764 IntrinID == Intrinsic::riscv_vlse_mask;
765 LLT VT = MRI->getType(Reg: I.getOperand(i: 0).getReg());
766 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
767
768 // Result vector
769 const Register DstReg = I.getOperand(i: 0).getReg();
770
771 // Sources
772 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
773 unsigned CurOp = 2;
774 SmallVector<SrcOp, 4> SrcOps; // Source registers.
775
776 // Passthru
777 if (HasPassthruOperand) {
778 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
779 SrcOps.push_back(Elt: PassthruReg);
780 } else {
781 SrcOps.push_back(Elt: Register(RISCV::NoRegister));
782 }
783
784 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: IsStrided);
785
786 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
787 const RISCV::VLEPseudo *P =
788 RISCV::getVLEPseudo(Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
789 LMUL: static_cast<unsigned>(LMUL));
790
791 auto PseudoMI = MIB.buildInstr(Opc: P->Pseudo, DstOps: {DstReg}, SrcOps);
792
793 // Select VL
794 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
795 for (auto &RenderFn : *VLOpFn)
796 RenderFn(PseudoMI);
797
798 // SEW
799 PseudoMI.addImm(Val: Log2SEW);
800
801 // Policy
802 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
803 if (IsMasked)
804 Policy = I.getOperand(i: CurOp++).getImm();
805 PseudoMI.addImm(Val: Policy);
806
807 // Memref
808 PseudoMI.cloneMemRefs(OtherMI: I);
809
810 I.eraseFromParent();
811 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
812 return true;
813 }
814 case Intrinsic::riscv_vloxei:
815 case Intrinsic::riscv_vloxei_mask:
816 case Intrinsic::riscv_vluxei:
817 case Intrinsic::riscv_vluxei_mask: {
818 bool IsMasked = IntrinID == Intrinsic::riscv_vloxei_mask ||
819 IntrinID == Intrinsic::riscv_vluxei_mask;
820 bool IsOrdered = IntrinID == Intrinsic::riscv_vloxei ||
821 IntrinID == Intrinsic::riscv_vloxei_mask;
822 LLT VT = MRI->getType(Reg: I.getOperand(i: 0).getReg());
823 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
824
825 // Result vector
826 const Register DstReg = I.getOperand(i: 0).getReg();
827
828 // Sources
829 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
830 unsigned CurOp = 2;
831 SmallVector<SrcOp, 4> SrcOps; // Source registers.
832
833 // Passthru
834 if (HasPassthruOperand) {
835 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
836 SrcOps.push_back(Elt: PassthruReg);
837 } else {
838 // Use NoRegister if there is no specified passthru.
839 SrcOps.push_back(Elt: Register());
840 }
841 LLT IndexVT;
842 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: true, IndexVT: &IndexVT);
843
844 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
845 RISCVVType::VLMUL IndexLMUL =
846 RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: IndexVT));
847 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
848 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
849 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
850 "values when XLEN=32");
851 }
852 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
853 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
854 IndexLMUL: static_cast<unsigned>(IndexLMUL));
855
856 auto PseudoMI = MIB.buildInstr(Opc: P->Pseudo, DstOps: {DstReg}, SrcOps);
857
858 // Select VL
859 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
860 for (auto &RenderFn : *VLOpFn)
861 RenderFn(PseudoMI);
862
863 // SEW
864 PseudoMI.addImm(Val: Log2SEW);
865
866 // Policy
867 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
868 if (IsMasked)
869 Policy = I.getOperand(i: CurOp++).getImm();
870 PseudoMI.addImm(Val: Policy);
871
872 // Memref
873 PseudoMI.cloneMemRefs(OtherMI: I);
874
875 I.eraseFromParent();
876 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
877 return true;
878 }
879 case Intrinsic::riscv_vsm:
880 case Intrinsic::riscv_vse:
881 case Intrinsic::riscv_vse_mask:
882 case Intrinsic::riscv_vsse:
883 case Intrinsic::riscv_vsse_mask: {
884 bool IsMasked = IntrinID == Intrinsic::riscv_vse_mask ||
885 IntrinID == Intrinsic::riscv_vsse_mask;
886 bool IsStrided = IntrinID == Intrinsic::riscv_vsse ||
887 IntrinID == Intrinsic::riscv_vsse_mask;
888 LLT VT = MRI->getType(Reg: I.getOperand(i: 1).getReg());
889 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
890
891 // Sources
892 unsigned CurOp = 1;
893 SmallVector<SrcOp, 4> SrcOps; // Source registers.
894
895 // Store value
896 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
897 SrcOps.push_back(Elt: PassthruReg);
898
899 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: IsStrided);
900
901 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
902 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
903 Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
904
905 auto PseudoMI = MIB.buildInstr(Opc: P->Pseudo, DstOps: {}, SrcOps);
906
907 // Select VL
908 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
909 for (auto &RenderFn : *VLOpFn)
910 RenderFn(PseudoMI);
911
912 // SEW
913 PseudoMI.addImm(Val: Log2SEW);
914
915 // Memref
916 PseudoMI.cloneMemRefs(OtherMI: I);
917
918 I.eraseFromParent();
919 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
920 return true;
921 }
922 case Intrinsic::riscv_vsoxei:
923 case Intrinsic::riscv_vsoxei_mask:
924 case Intrinsic::riscv_vsuxei:
925 case Intrinsic::riscv_vsuxei_mask: {
926 bool IsMasked = IntrinID == Intrinsic::riscv_vsoxei_mask ||
927 IntrinID == Intrinsic::riscv_vsuxei_mask;
928 bool IsOrdered = IntrinID == Intrinsic::riscv_vsoxei ||
929 IntrinID == Intrinsic::riscv_vsoxei_mask;
930 LLT VT = MRI->getType(Reg: I.getOperand(i: 1).getReg());
931 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
932
933 // Sources
934 unsigned CurOp = 1;
935 SmallVector<SrcOp, 4> SrcOps; // Source registers.
936
937 // Store value
938 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
939 SrcOps.push_back(Elt: PassthruReg);
940
941 LLT IndexVT;
942 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: true, IndexVT: &IndexVT);
943
944 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
945 RISCVVType::VLMUL IndexLMUL =
946 RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: IndexVT));
947 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
948 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
949 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
950 "values when XLEN=32");
951 }
952 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
953 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
954 IndexLMUL: static_cast<unsigned>(IndexLMUL));
955
956 auto PseudoMI = MIB.buildInstr(Opc: P->Pseudo, DstOps: {}, SrcOps);
957
958 // Select VL
959 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
960 for (auto &RenderFn : *VLOpFn)
961 RenderFn(PseudoMI);
962
963 // SEW
964 PseudoMI.addImm(Val: Log2SEW);
965
966 // Memref
967 PseudoMI.cloneMemRefs(OtherMI: I);
968
969 I.eraseFromParent();
970 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
971 return true;
972 }
973 }
974}
975
976bool RISCVInstructionSelector::selectIntrinsic(MachineInstr &I,
977 MachineIRBuilder &MIB) const {
978 // Find the intrinsic ID.
979 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
980 // Select the instruction.
981 switch (IntrinID) {
982 default:
983 return false;
984 case Intrinsic::riscv_vsetvli:
985 case Intrinsic::riscv_vsetvlimax: {
986
987 bool VLMax = IntrinID == Intrinsic::riscv_vsetvlimax;
988
989 unsigned Offset = VLMax ? 2 : 3;
990 unsigned SEW = RISCVVType::decodeVSEW(VSEW: I.getOperand(i: Offset).getImm() & 0x7);
991 RISCVVType::VLMUL VLMul =
992 static_cast<RISCVVType::VLMUL>(I.getOperand(i: Offset + 1).getImm() & 0x7);
993
994 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMUL: VLMul, SEW, /*TailAgnostic*/ true,
995 /*MaskAgnostic*/ true);
996
997 Register DstReg = I.getOperand(i: 0).getReg();
998
999 Register VLOperand;
1000 unsigned Opcode = RISCV::PseudoVSETVLI;
1001
1002 // Check if AVL is a constant that equals VLMAX.
1003 if (!VLMax) {
1004 Register AVLReg = I.getOperand(i: 2).getReg();
1005 if (auto AVLConst = getIConstantVRegValWithLookThrough(VReg: AVLReg, MRI: *MRI)) {
1006 uint64_t AVL = AVLConst->Value.getZExtValue();
1007 if (auto VLEN = Subtarget->getRealVLen()) {
1008 if (*VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == AVL)
1009 VLMax = true;
1010 }
1011 }
1012
1013 MachineInstr *AVLDef = MRI->getVRegDef(Reg: AVLReg);
1014 if (AVLDef && AVLDef->getOpcode() == TargetOpcode::G_CONSTANT) {
1015 const auto *C = AVLDef->getOperand(i: 1).getCImm();
1016 if (C->getValue().isAllOnes())
1017 VLMax = true;
1018 }
1019 }
1020
1021 if (VLMax) {
1022 VLOperand = Register(RISCV::X0);
1023 Opcode = RISCV::PseudoVSETVLIX0;
1024 } else {
1025 Register AVLReg = I.getOperand(i: 2).getReg();
1026 VLOperand = AVLReg;
1027
1028 // Check if AVL is a small constant that can use PseudoVSETIVLI.
1029 if (auto AVLConst = getIConstantVRegValWithLookThrough(VReg: AVLReg, MRI: *MRI)) {
1030 uint64_t AVL = AVLConst->Value.getZExtValue();
1031 if (isUInt<5>(x: AVL)) {
1032 auto PseudoMI = MIB.buildInstr(Opc: RISCV::PseudoVSETIVLI, DstOps: {DstReg}, SrcOps: {})
1033 .addImm(Val: AVL)
1034 .addImm(Val: VTypeI);
1035 I.eraseFromParent();
1036 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1037 return true;
1038 }
1039 }
1040 }
1041
1042 auto PseudoMI =
1043 MIB.buildInstr(Opc: Opcode, DstOps: {DstReg}, SrcOps: {VLOperand}).addImm(Val: VTypeI);
1044 I.eraseFromParent();
1045 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1046 return true;
1047 }
1048 }
1049}
1050
1051bool RISCVInstructionSelector::selectExtractSubvector(
1052 MachineInstr &MI, MachineIRBuilder &MIB) const {
1053 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_SUBVECTOR);
1054
1055 Register DstReg = MI.getOperand(i: 0).getReg();
1056 Register SrcReg = MI.getOperand(i: 1).getReg();
1057
1058 LLT DstTy = MRI->getType(Reg: DstReg);
1059 LLT SrcTy = MRI->getType(Reg: SrcReg);
1060
1061 unsigned Idx = static_cast<unsigned>(MI.getOperand(i: 2).getImm());
1062
1063 MVT DstMVT = getMVTForLLT(Ty: DstTy);
1064 MVT SrcMVT = getMVTForLLT(Ty: SrcTy);
1065
1066 unsigned SubRegIdx;
1067 std::tie(args&: SubRegIdx, args&: Idx) =
1068 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1069 VecVT: SrcMVT, SubVecVT: DstMVT, InsertExtractIdx: Idx, TRI: &TRI);
1070
1071 if (Idx != 0)
1072 return false;
1073
1074 unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: DstMVT);
1075 const TargetRegisterClass *DstRC = TRI.getRegClass(i: DstRegClassID);
1076 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1077 return false;
1078
1079 unsigned SrcRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: SrcMVT);
1080 const TargetRegisterClass *SrcRC = TRI.getRegClass(i: SrcRegClassID);
1081 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
1082 return false;
1083
1084 MIB.buildInstr(Opc: TargetOpcode::COPY, DstOps: {DstReg}, SrcOps: {})
1085 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubRegIdx);
1086
1087 MI.eraseFromParent();
1088 return true;
1089}
1090
1091bool RISCVInstructionSelector::select(MachineInstr &MI) {
1092 MachineIRBuilder MIB(MI);
1093
1094 preISelLower(MI, MIB);
1095 const unsigned Opc = MI.getOpcode();
1096
1097 if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
1098 if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
1099 const Register DefReg = MI.getOperand(i: 0).getReg();
1100 const LLT DefTy = MRI->getType(Reg: DefReg);
1101
1102 const RegClassOrRegBank &RegClassOrBank =
1103 MRI->getRegClassOrRegBank(Reg: DefReg);
1104
1105 const TargetRegisterClass *DefRC =
1106 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
1107 if (!DefRC) {
1108 if (!DefTy.isValid()) {
1109 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
1110 return false;
1111 }
1112
1113 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
1114 DefRC = getRegClassForTypeOnBank(Ty: DefTy, RB);
1115 if (!DefRC) {
1116 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
1117 return false;
1118 }
1119 }
1120
1121 MI.setDesc(TII.get(Opcode: TargetOpcode::PHI));
1122 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI&: *MRI);
1123 }
1124
1125 // Certain non-generic instructions also need some special handling.
1126 if (MI.isCopy())
1127 return selectCopy(MI);
1128
1129 return true;
1130 }
1131
1132 if (selectImpl(I&: MI, CoverageInfo&: *CoverageInfo))
1133 return true;
1134
1135 switch (Opc) {
1136 case TargetOpcode::G_ANYEXT:
1137 case TargetOpcode::G_PTRTOINT:
1138 case TargetOpcode::G_INTTOPTR:
1139 case TargetOpcode::G_TRUNC:
1140 case TargetOpcode::G_FREEZE:
1141 return selectCopy(MI);
1142 case TargetOpcode::G_CONSTANT: {
1143 Register DstReg = MI.getOperand(i: 0).getReg();
1144 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
1145
1146 if (!materializeImm(Reg: DstReg, Imm, MIB))
1147 return false;
1148
1149 MI.eraseFromParent();
1150 return true;
1151 }
1152 case TargetOpcode::G_ZEXT:
1153 case TargetOpcode::G_SEXT: {
1154 bool IsSigned = Opc != TargetOpcode::G_ZEXT;
1155 Register DstReg = MI.getOperand(i: 0).getReg();
1156 Register SrcReg = MI.getOperand(i: 1).getReg();
1157 LLT SrcTy = MRI->getType(Reg: SrcReg);
1158 unsigned SrcSize = SrcTy.getSizeInBits();
1159
1160 if (SrcTy.isVector())
1161 return false; // Should be handled by imported patterns.
1162
1163 assert((*RBI.getRegBank(DstReg, *MRI, TRI)).getID() ==
1164 RISCV::GPRBRegBankID &&
1165 "Unexpected ext regbank");
1166
1167 // Use addiw SrcReg, 0 (sext.w) for i32.
1168 if (IsSigned && SrcSize == 32) {
1169 MI.setDesc(TII.get(Opcode: RISCV::ADDIW));
1170 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
1171 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1172 return true;
1173 }
1174
1175 // Use add.uw SrcReg, X0 (zext.w) for i32 with Zba.
1176 if (!IsSigned && SrcSize == 32 && STI.hasStdExtZba()) {
1177 MI.setDesc(TII.get(Opcode: RISCV::ADD_UW));
1178 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::X0, /*isDef=*/false));
1179 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1180 return true;
1181 }
1182
1183 // Use sext.h/zext.h for i16 with Zbb.
1184 if (SrcSize == 16 && STI.hasStdExtZbb()) {
1185 MI.setDesc(TII.get(Opcode: IsSigned ? RISCV::SEXT_H
1186 : STI.isRV64() ? RISCV::ZEXT_H_RV64
1187 : RISCV::ZEXT_H_RV32));
1188 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1189 return true;
1190 }
1191
1192 // Use pack(w) SrcReg, X0 for i16 zext with Zbkb.
1193 if (!IsSigned && SrcSize == 16 && STI.hasStdExtZbkb()) {
1194 MI.setDesc(TII.get(Opcode: STI.is64Bit() ? RISCV::PACKW : RISCV::PACK));
1195 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::X0, /*isDef=*/false));
1196 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1197 return true;
1198 }
1199
1200 // Fall back to shift pair.
1201 auto ShiftLeft =
1202 MIB.buildInstr(Opc: RISCV::SLLI, DstOps: {&RISCV::GPRRegClass}, SrcOps: {SrcReg})
1203 .addImm(Val: STI.getXLen() - SrcSize);
1204 constrainSelectedInstRegOperands(I&: *ShiftLeft, TII, TRI, RBI);
1205 auto ShiftRight = MIB.buildInstr(Opc: IsSigned ? RISCV::SRAI : RISCV::SRLI,
1206 DstOps: {DstReg}, SrcOps: {ShiftLeft})
1207 .addImm(Val: STI.getXLen() - SrcSize);
1208 constrainSelectedInstRegOperands(I&: *ShiftRight, TII, TRI, RBI);
1209 MI.eraseFromParent();
1210 return true;
1211 }
1212 case TargetOpcode::G_FCONSTANT: {
1213 // TODO: Use constant pool for complex constants.
1214 Register DstReg = MI.getOperand(i: 0).getReg();
1215 const APFloat &FPimm = MI.getOperand(i: 1).getFPImm()->getValueAPF();
1216 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
1217 if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
1218 Register GPRReg;
1219 if (FPimm.isPosZero()) {
1220 GPRReg = RISCV::X0;
1221 } else {
1222 GPRReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1223 APInt Imm = FPimm.bitcastToAPInt();
1224 if (!materializeImm(Reg: GPRReg, Imm: Imm.getSExtValue(), MIB))
1225 return false;
1226 }
1227
1228 unsigned Opcode = Size == 64 ? RISCV::FMV_D_X
1229 : Size == 32 ? RISCV::FMV_W_X
1230 : RISCV::FMV_H_X;
1231 auto FMV = MIB.buildInstr(Opc: Opcode, DstOps: {DstReg}, SrcOps: {GPRReg});
1232 FMV.constrainAllUses(TII, TRI, RBI);
1233 } else {
1234 // s64 on rv32
1235 assert(Size == 64 && !Subtarget->is64Bit() &&
1236 "Unexpected size or subtarget");
1237
1238 if (FPimm.isPosZero()) {
1239 // Optimize +0.0 to use fcvt.d.w
1240 MachineInstrBuilder FCVT =
1241 MIB.buildInstr(Opc: RISCV::FCVT_D_W, DstOps: {DstReg}, SrcOps: {Register(RISCV::X0)})
1242 .addImm(Val: RISCVFPRndMode::RNE);
1243 FCVT.constrainAllUses(TII, TRI, RBI);
1244
1245 MI.eraseFromParent();
1246 return true;
1247 }
1248
1249 // Split into two pieces and build through the stack.
1250 Register GPRRegHigh = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1251 Register GPRRegLow = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1252 APInt Imm = FPimm.bitcastToAPInt();
1253 if (!materializeImm(Reg: GPRRegHigh, Imm: Imm.extractBits(numBits: 32, bitPosition: 32).getSExtValue(),
1254 MIB))
1255 return false;
1256 if (!materializeImm(Reg: GPRRegLow, Imm: Imm.trunc(width: 32).getSExtValue(), MIB))
1257 return false;
1258 MachineInstrBuilder PairF64 = MIB.buildInstr(
1259 Opc: RISCV::BuildPairF64Pseudo, DstOps: {DstReg}, SrcOps: {GPRRegLow, GPRRegHigh});
1260 PairF64.constrainAllUses(TII, TRI, RBI);
1261 }
1262
1263 MI.eraseFromParent();
1264 return true;
1265 }
1266 case TargetOpcode::G_GLOBAL_VALUE: {
1267 auto *GV = MI.getOperand(i: 1).getGlobal();
1268 if (GV->isThreadLocal()) {
1269 // TODO: implement this case.
1270 return false;
1271 }
1272
1273 return selectAddr(MI, MIB, IsLocal: GV->isDSOLocal(), IsExternWeak: GV->hasExternalWeakLinkage());
1274 }
1275 case TargetOpcode::G_JUMP_TABLE:
1276 case TargetOpcode::G_CONSTANT_POOL:
1277 return selectAddr(MI, MIB, IsLocal: MRI);
1278 case TargetOpcode::G_BRCOND: {
1279 Register LHS, RHS;
1280 RISCVCC::CondCode CC;
1281 getOperandsForBranch(CondReg: MI.getOperand(i: 0).getReg(), CC, LHS, RHS, MRI&: *MRI);
1282
1283 auto Bcc = MIB.buildInstr(Opc: RISCVCC::getBrCond(CC), DstOps: {}, SrcOps: {LHS, RHS})
1284 .addMBB(MBB: MI.getOperand(i: 1).getMBB());
1285 MI.eraseFromParent();
1286 constrainSelectedInstRegOperands(I&: *Bcc, TII, TRI, RBI);
1287 return true;
1288 }
1289 case TargetOpcode::G_BRINDIRECT:
1290 MI.setDesc(TII.get(Opcode: RISCV::PseudoBRIND));
1291 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
1292 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1293 return true;
1294 case TargetOpcode::G_SELECT:
1295 return selectSelect(MI, MIB);
1296 case TargetOpcode::G_FCMP:
1297 return selectFPCompare(MI, MIB);
1298 case TargetOpcode::G_FENCE: {
1299 AtomicOrdering FenceOrdering =
1300 static_cast<AtomicOrdering>(MI.getOperand(i: 0).getImm());
1301 SyncScope::ID FenceSSID =
1302 static_cast<SyncScope::ID>(MI.getOperand(i: 1).getImm());
1303 emitFence(FenceOrdering, FenceSSID, MIB);
1304 MI.eraseFromParent();
1305 return true;
1306 }
1307 case TargetOpcode::G_IMPLICIT_DEF:
1308 return selectImplicitDef(MI, MIB);
1309 case TargetOpcode::G_UNMERGE_VALUES:
1310 return selectUnmergeValues(MI, MIB);
1311 case TargetOpcode::G_LOAD:
1312 case TargetOpcode::G_STORE: {
1313 GLoadStore &LdSt = cast<GLoadStore>(Val&: MI);
1314 const Register ValReg = LdSt.getReg(Idx: 0);
1315 const Register PtrReg = LdSt.getPointerReg();
1316 LLT PtrTy = MRI->getType(Reg: PtrReg);
1317
1318 const RegisterBank &RB = *RBI.getRegBank(Reg: ValReg, MRI: *MRI, TRI);
1319 if (RB.getID() != RISCV::GPRBRegBankID)
1320 return false;
1321
1322#ifndef NDEBUG
1323 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, *MRI, TRI);
1324 // Check that the pointer register is valid.
1325 assert(PtrRB.getID() == RISCV::GPRBRegBankID &&
1326 "Load/Store pointer operand isn't a GPR");
1327 assert(PtrTy.isPointer() && "Load/Store pointer operand isn't a pointer");
1328#endif
1329
1330 // Can only handle AddressSpace 0.
1331 if (PtrTy.getAddressSpace() != 0)
1332 return false;
1333
1334 unsigned MemSize = LdSt.getMemSizeInBits().getValue();
1335 AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();
1336
1337 if (isStrongerThanMonotonic(AO: Order)) {
1338 MI.setDesc(TII.get(Opcode: selectZalasrLoadStoreOp(GenericOpc: Opc, OpSize: MemSize)));
1339 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1340 return true;
1341 }
1342
1343 const unsigned NewOpc = selectRegImmLoadStoreOp(GenericOpc: MI.getOpcode(), OpSize: MemSize);
1344 if (NewOpc == MI.getOpcode())
1345 return false;
1346
1347 // Check if we can fold anything into the addressing mode.
1348 auto AddrModeFns = selectAddrRegImm(Root&: MI.getOperand(i: 1));
1349 if (!AddrModeFns)
1350 return false;
1351
1352 // Folded something. Create a new instruction and return it.
1353 auto NewInst = MIB.buildInstr(Opc: NewOpc, DstOps: {}, SrcOps: {}, Flags: MI.getFlags());
1354 if (isa<GStore>(Val: MI))
1355 NewInst.addUse(RegNo: ValReg);
1356 else
1357 NewInst.addDef(RegNo: ValReg);
1358 NewInst.cloneMemRefs(OtherMI: MI);
1359 for (auto &Fn : *AddrModeFns)
1360 Fn(NewInst);
1361 MI.eraseFromParent();
1362
1363 constrainSelectedInstRegOperands(I&: *NewInst, TII, TRI, RBI);
1364 return true;
1365 }
1366 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1367 return selectIntrinsicWithSideEffects(I&: MI, MIB);
1368 case TargetOpcode::G_INTRINSIC:
1369 return selectIntrinsic(I&: MI, MIB);
1370 case TargetOpcode::G_EXTRACT_SUBVECTOR:
1371 return selectExtractSubvector(MI, MIB);
1372 default:
1373 return false;
1374 }
1375}
1376
1377bool RISCVInstructionSelector::selectUnmergeValues(
1378 MachineInstr &MI, MachineIRBuilder &MIB) const {
1379 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
1380
1381 if (!Subtarget->hasStdExtZfa())
1382 return false;
1383
1384 // Split F64 Src into two s32 parts
1385 if (MI.getNumOperands() != 3)
1386 return false;
1387 Register Src = MI.getOperand(i: 2).getReg();
1388 Register Lo = MI.getOperand(i: 0).getReg();
1389 Register Hi = MI.getOperand(i: 1).getReg();
1390 if (!isRegInFprb(Reg: Src) || !isRegInGprb(Reg: Lo) || !isRegInGprb(Reg: Hi))
1391 return false;
1392
1393 MachineInstr *ExtractLo = MIB.buildInstr(Opc: RISCV::FMV_X_W_FPR64, DstOps: {Lo}, SrcOps: {Src});
1394 constrainSelectedInstRegOperands(I&: *ExtractLo, TII, TRI, RBI);
1395
1396 MachineInstr *ExtractHi = MIB.buildInstr(Opc: RISCV::FMVH_X_D, DstOps: {Hi}, SrcOps: {Src});
1397 constrainSelectedInstRegOperands(I&: *ExtractHi, TII, TRI, RBI);
1398
1399 MI.eraseFromParent();
1400 return true;
1401}
1402
1403bool RISCVInstructionSelector::replacePtrWithInt(MachineOperand &Op,
1404 MachineIRBuilder &MIB) {
1405 Register PtrReg = Op.getReg();
1406 assert(MRI->getType(PtrReg).isPointer() && "Operand is not a pointer!");
1407
1408 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1409 auto PtrToInt = MIB.buildPtrToInt(Dst: sXLen, Src: PtrReg);
1410 MRI->setRegBank(Reg: PtrToInt.getReg(Idx: 0), RegBank: RBI.getRegBank(ID: RISCV::GPRBRegBankID));
1411 Op.setReg(PtrToInt.getReg(Idx: 0));
1412 return select(MI&: *PtrToInt);
1413}
1414
1415void RISCVInstructionSelector::preISelLower(MachineInstr &MI,
1416 MachineIRBuilder &MIB) {
1417 switch (MI.getOpcode()) {
1418 case TargetOpcode::G_PTR_ADD: {
1419 Register DstReg = MI.getOperand(i: 0).getReg();
1420 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1421
1422 replacePtrWithInt(Op&: MI.getOperand(i: 1), MIB);
1423 MI.setDesc(TII.get(Opcode: TargetOpcode::G_ADD));
1424 MRI->setType(VReg: DstReg, Ty: sXLen);
1425 break;
1426 }
1427 case TargetOpcode::G_PTRMASK: {
1428 Register DstReg = MI.getOperand(i: 0).getReg();
1429 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1430 replacePtrWithInt(Op&: MI.getOperand(i: 1), MIB);
1431 MI.setDesc(TII.get(Opcode: TargetOpcode::G_AND));
1432 MRI->setType(VReg: DstReg, Ty: sXLen);
1433 break;
1434 }
1435 }
1436}
1437
1438void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
1439 const MachineInstr &MI,
1440 int OpIdx) const {
1441 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1442 "Expected G_CONSTANT");
1443 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
1444 MIB.addImm(Val: -CstVal);
1445}
1446
1447void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
1448 const MachineInstr &MI,
1449 int OpIdx) const {
1450 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1451 "Expected G_CONSTANT");
1452 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1453 MIB.addImm(Val: STI.getXLen() - CstVal);
1454}
1455
1456void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
1457 const MachineInstr &MI,
1458 int OpIdx) const {
1459 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1460 "Expected G_CONSTANT");
1461 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1462 MIB.addImm(Val: 32 - CstVal);
1463}
1464
1465void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
1466 const MachineInstr &MI,
1467 int OpIdx) const {
1468 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1469 "Expected G_CONSTANT");
1470 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
1471 MIB.addImm(Val: CstVal + 1);
1472}
1473
1474void RISCVInstructionSelector::renderFrameIndex(MachineInstrBuilder &MIB,
1475 const MachineInstr &MI,
1476 int OpIdx) const {
1477 assert(MI.getOpcode() == TargetOpcode::G_FRAME_INDEX && OpIdx == -1 &&
1478 "Expected G_FRAME_INDEX");
1479 MIB.add(MO: MI.getOperand(i: 1));
1480}
1481
1482void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
1483 const MachineInstr &MI,
1484 int OpIdx) const {
1485 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1486 "Expected G_CONSTANT");
1487 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
1488 MIB.addImm(Val: llvm::countr_zero(Val: C));
1489}
1490
1491void RISCVInstructionSelector::renderXLenSubTrailingOnes(
1492 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
1493 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1494 "Expected G_CONSTANT");
1495 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
1496 MIB.addImm(Val: Subtarget->getXLen() - llvm::countr_one(Value: C));
1497}
1498
1499void RISCVInstructionSelector::renderAddiPairImmSmall(MachineInstrBuilder &MIB,
1500 const MachineInstr &MI,
1501 int OpIdx) const {
1502 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1503 "Expected G_CONSTANT");
1504 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
1505 int64_t Adj = Imm < 0 ? -2048 : 2047;
1506 MIB.addImm(Val: Imm - Adj);
1507}
1508
1509void RISCVInstructionSelector::renderAddiPairImmLarge(MachineInstrBuilder &MIB,
1510 const MachineInstr &MI,
1511 int OpIdx) const {
1512 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1513 "Expected G_CONSTANT");
1514 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue() < 0 ? -2048 : 2047;
1515 MIB.addImm(Val: Imm);
1516}
1517
1518const TargetRegisterClass *RISCVInstructionSelector::getRegClassForTypeOnBank(
1519 LLT Ty, const RegisterBank &RB) const {
1520 if (RB.getID() == RISCV::GPRBRegBankID) {
1521 if (Ty.getSizeInBits() <= 32 || (STI.is64Bit() && Ty.getSizeInBits() == 64))
1522 return &RISCV::GPRRegClass;
1523 }
1524
1525 if (RB.getID() == RISCV::FPRBRegBankID) {
1526 if (Ty.getSizeInBits() == 16)
1527 return &RISCV::FPR16RegClass;
1528 if (Ty.getSizeInBits() == 32)
1529 return &RISCV::FPR32RegClass;
1530 if (Ty.getSizeInBits() == 64)
1531 return &RISCV::FPR64RegClass;
1532 }
1533
1534 if (RB.getID() == RISCV::VRBRegBankID) {
1535 if (Ty.getSizeInBits().getKnownMinValue() <= 64)
1536 return &RISCV::VRRegClass;
1537
1538 if (Ty.getSizeInBits().getKnownMinValue() == 128)
1539 return &RISCV::VRM2RegClass;
1540
1541 if (Ty.getSizeInBits().getKnownMinValue() == 256)
1542 return &RISCV::VRM4RegClass;
1543
1544 if (Ty.getSizeInBits().getKnownMinValue() == 512)
1545 return &RISCV::VRM8RegClass;
1546 }
1547
1548 return nullptr;
1549}
1550
1551bool RISCVInstructionSelector::isRegInGprb(Register Reg) const {
1552 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::GPRBRegBankID;
1553}
1554
1555bool RISCVInstructionSelector::isRegInFprb(Register Reg) const {
1556 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID;
1557}
1558
1559bool RISCVInstructionSelector::selectCopy(MachineInstr &MI) const {
1560 Register DstReg = MI.getOperand(i: 0).getReg();
1561
1562 if (DstReg.isPhysical())
1563 return true;
1564
1565 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
1566 Ty: MRI->getType(Reg: DstReg), RB: *RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI));
1567 assert(DstRC &&
1568 "Register class not available for LLT, register bank combination");
1569
1570 // No need to constrain SrcReg. It will get constrained when
1571 // we hit another of its uses or its defs.
1572 // Copies do not have constraints.
1573 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
1574 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1575 << " operand\n");
1576 return false;
1577 }
1578
1579 MI.setDesc(TII.get(Opcode: RISCV::COPY));
1580 return true;
1581}
1582
1583bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI,
1584 MachineIRBuilder &MIB) const {
1585 assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
1586
1587 const Register DstReg = MI.getOperand(i: 0).getReg();
1588 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
1589 Ty: MRI->getType(Reg: DstReg), RB: *RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI));
1590
1591 assert(DstRC &&
1592 "Register class not available for LLT, register bank combination");
1593
1594 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
1595 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1596 << " operand\n");
1597 }
1598 MI.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
1599 return true;
1600}
1601
1602bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
1603 MachineIRBuilder &MIB) const {
1604 if (Imm == 0) {
1605 MIB.buildCopy(Res: DstReg, Op: Register(RISCV::X0));
1606 RBI.constrainGenericRegister(Reg: DstReg, RC: RISCV::GPRRegClass, MRI&: *MRI);
1607 return true;
1608 }
1609
1610 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: *Subtarget);
1611 unsigned NumInsts = Seq.size();
1612 Register SrcReg = RISCV::X0;
1613
1614 for (unsigned i = 0; i < NumInsts; i++) {
1615 Register TmpReg = i < NumInsts - 1
1616 ? MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass)
1617 : DstReg;
1618 const RISCVMatInt::Inst &I = Seq[i];
1619 MachineInstr *Result;
1620
1621 switch (I.getOpndKind()) {
1622 case RISCVMatInt::Imm:
1623 // clang-format off
1624 Result = MIB.buildInstr(Opc: I.getOpcode(), DstOps: {TmpReg}, SrcOps: {})
1625 .addImm(Val: I.getImm());
1626 // clang-format on
1627 break;
1628 case RISCVMatInt::RegX0:
1629 Result = MIB.buildInstr(Opc: I.getOpcode(), DstOps: {TmpReg},
1630 SrcOps: {SrcReg, Register(RISCV::X0)});
1631 break;
1632 case RISCVMatInt::RegReg:
1633 Result = MIB.buildInstr(Opc: I.getOpcode(), DstOps: {TmpReg}, SrcOps: {SrcReg, SrcReg});
1634 break;
1635 case RISCVMatInt::RegImm:
1636 Result =
1637 MIB.buildInstr(Opc: I.getOpcode(), DstOps: {TmpReg}, SrcOps: {SrcReg}).addImm(Val: I.getImm());
1638 break;
1639 }
1640
1641 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1642
1643 SrcReg = TmpReg;
1644 }
1645
1646 return true;
1647}
1648
1649bool RISCVInstructionSelector::selectAddr(MachineInstr &MI,
1650 MachineIRBuilder &MIB, bool IsLocal,
1651 bool IsExternWeak) const {
1652 assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1653 MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
1654 MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
1655 "Unexpected opcode");
1656
1657 const MachineOperand &DispMO = MI.getOperand(i: 1);
1658
1659 Register DefReg = MI.getOperand(i: 0).getReg();
1660 const LLT DefTy = MRI->getType(Reg: DefReg);
1661
1662 // When HWASAN is used and tagging of global variables is enabled
1663 // they should be accessed via the GOT, since the tagged address of a global
1664 // is incompatible with existing code models. This also applies to non-pic
1665 // mode.
1666 if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
1667 if (IsLocal && !Subtarget->allowTaggedGlobals()) {
1668 // Use PC-relative addressing to access the symbol. This generates the
1669 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
1670 // %pcrel_lo(auipc)).
1671 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
1672 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1673 return true;
1674 }
1675
1676 // Use PC-relative addressing to access the GOT for this symbol, then
1677 // load the address from the GOT. This generates the pattern (PseudoLGA
1678 // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
1679 // %pcrel_lo(auipc))).
1680 MachineFunction &MF = *MI.getParent()->getParent();
1681 MachineMemOperand *MemOp = MF.getMachineMemOperand(
1682 PtrInfo: MachinePointerInfo::getGOT(MF),
1683 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1684 MachineMemOperand::MOInvariant,
1685 MemTy: DefTy, base_alignment: Align(DefTy.getSizeInBits() / 8));
1686
1687 auto Result = MIB.buildInstr(Opc: RISCV::PseudoLGA, DstOps: {DefReg}, SrcOps: {})
1688 .addDisp(Disp: DispMO, off: 0)
1689 .addMemOperand(MMO: MemOp);
1690
1691 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1692
1693 MI.eraseFromParent();
1694 return true;
1695 }
1696
1697 switch (TM.getCodeModel()) {
1698 default: {
1699 reportGISelFailure(MF&: *MF, MORE&: *MORE, PassName: getName(),
1700 Msg: "Unsupported code model for lowering", MI);
1701 return false;
1702 }
1703 case CodeModel::Small: {
1704 // Must lie within a single 2 GiB address range and must lie between
1705 // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
1706 // (lui %hi(sym)) %lo(sym)).
1707 Register AddrHiDest = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1708 MachineInstr *AddrHi = MIB.buildInstr(Opc: RISCV::LUI, DstOps: {AddrHiDest}, SrcOps: {})
1709 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_HI);
1710
1711 constrainSelectedInstRegOperands(I&: *AddrHi, TII, TRI, RBI);
1712
1713 auto Result = MIB.buildInstr(Opc: RISCV::ADDI, DstOps: {DefReg}, SrcOps: {AddrHiDest})
1714 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_LO);
1715
1716 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1717
1718 MI.eraseFromParent();
1719 return true;
1720 }
1721 case CodeModel::Medium:
1722 // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
1723 // relocation needs to reference a label that points to the auipc
1724 // instruction itself, not the global. This cannot be done inside the
1725 // instruction selector.
1726 if (IsExternWeak) {
1727 // An extern weak symbol may be undefined, i.e. have value 0, which may
1728 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
1729 // symbol. This generates the pattern (PseudoLGA sym), which expands to
1730 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
1731 MachineFunction &MF = *MI.getParent()->getParent();
1732 MachineMemOperand *MemOp = MF.getMachineMemOperand(
1733 PtrInfo: MachinePointerInfo::getGOT(MF),
1734 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1735 MachineMemOperand::MOInvariant,
1736 MemTy: DefTy, base_alignment: Align(DefTy.getSizeInBits() / 8));
1737
1738 auto Result = MIB.buildInstr(Opc: RISCV::PseudoLGA, DstOps: {DefReg}, SrcOps: {})
1739 .addDisp(Disp: DispMO, off: 0)
1740 .addMemOperand(MMO: MemOp);
1741
1742 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1743
1744 MI.eraseFromParent();
1745 return true;
1746 }
1747
1748 // Generate a sequence for accessing addresses within any 2GiB range
1749 // within the address space. This generates the pattern (PseudoLLA sym),
1750 // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
1751 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
1752 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1753 return true;
1754 }
1755
1756 return false;
1757}
1758
1759bool RISCVInstructionSelector::selectSelect(MachineInstr &MI,
1760 MachineIRBuilder &MIB) const {
1761 auto &SelectMI = cast<GSelect>(Val&: MI);
1762
1763 Register LHS, RHS;
1764 RISCVCC::CondCode CC;
1765 getOperandsForBranch(CondReg: SelectMI.getCondReg(), CC, LHS, RHS, MRI&: *MRI);
1766
1767 Register DstReg = SelectMI.getReg(Idx: 0);
1768
1769 unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
1770 if (RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
1771 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
1772 Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
1773 : RISCV::Select_FPR64_Using_CC_GPR;
1774 }
1775
1776 MachineInstr *Result = MIB.buildInstr(Opcode: Opc)
1777 .addDef(RegNo: DstReg)
1778 .addReg(RegNo: LHS)
1779 .addReg(RegNo: RHS)
1780 .addImm(Val: CC)
1781 .addReg(RegNo: SelectMI.getTrueReg())
1782 .addReg(RegNo: SelectMI.getFalseReg());
1783 MI.eraseFromParent();
1784 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1785 return true;
1786}
1787
1788// Convert an FCMP predicate to one of the supported F or D instructions.
1789static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
1790 assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");
1791 switch (Pred) {
1792 default:
1793 llvm_unreachable("Unsupported predicate");
1794 case CmpInst::FCMP_OLT:
1795 return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
1796 case CmpInst::FCMP_OLE:
1797 return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
1798 case CmpInst::FCMP_OEQ:
1799 return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
1800 }
1801}
1802
1803// Try legalizing an FCMP by swapping or inverting the predicate to one that
1804// is supported.
1805static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,
1806 CmpInst::Predicate &Pred, bool &NeedInvert) {
1807 auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
1808 return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
1809 Pred == CmpInst::FCMP_OEQ;
1810 };
1811
1812 assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
1813
1814 CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(pred: Pred);
1815 if (isLegalFCmpPredicate(InvPred)) {
1816 Pred = InvPred;
1817 std::swap(a&: LHS, b&: RHS);
1818 return true;
1819 }
1820
1821 InvPred = CmpInst::getInversePredicate(pred: Pred);
1822 NeedInvert = true;
1823 if (isLegalFCmpPredicate(InvPred)) {
1824 Pred = InvPred;
1825 return true;
1826 }
1827 InvPred = CmpInst::getSwappedPredicate(pred: InvPred);
1828 if (isLegalFCmpPredicate(InvPred)) {
1829 Pred = InvPred;
1830 std::swap(a&: LHS, b&: RHS);
1831 return true;
1832 }
1833
1834 return false;
1835}
1836
1837// Emit a sequence of instructions to compare LHS and RHS using Pred. Return
1838// the result in DstReg.
1839// FIXME: Maybe we should expand this earlier.
1840bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
1841 MachineIRBuilder &MIB) const {
1842 auto &CmpMI = cast<GFCmp>(Val&: MI);
1843 CmpInst::Predicate Pred = CmpMI.getCond();
1844
1845 Register DstReg = CmpMI.getReg(Idx: 0);
1846 Register LHS = CmpMI.getLHSReg();
1847 Register RHS = CmpMI.getRHSReg();
1848
1849 unsigned Size = MRI->getType(Reg: LHS).getSizeInBits();
1850 assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
1851
1852 Register TmpReg = DstReg;
1853
1854 bool NeedInvert = false;
1855 // First try swapping operands or inverting.
1856 if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
1857 if (NeedInvert)
1858 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1859 auto Cmp = MIB.buildInstr(Opc: getFCmpOpcode(Pred, Size), DstOps: {TmpReg}, SrcOps: {LHS, RHS});
1860 Cmp.constrainAllUses(TII, TRI, RBI);
1861 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
1862 // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
1863 NeedInvert = Pred == CmpInst::FCMP_UEQ;
1864 auto Cmp1 = MIB.buildInstr(Opc: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size),
1865 DstOps: {&RISCV::GPRRegClass}, SrcOps: {LHS, RHS});
1866 Cmp1.constrainAllUses(TII, TRI, RBI);
1867 auto Cmp2 = MIB.buildInstr(Opc: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size),
1868 DstOps: {&RISCV::GPRRegClass}, SrcOps: {RHS, LHS});
1869 Cmp2.constrainAllUses(TII, TRI, RBI);
1870 if (NeedInvert)
1871 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1872 auto Or =
1873 MIB.buildInstr(Opc: RISCV::OR, DstOps: {TmpReg}, SrcOps: {Cmp1.getReg(Idx: 0), Cmp2.getReg(Idx: 0)});
1874 Or.constrainAllUses(TII, TRI, RBI);
1875 } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
1876 // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
1877 // FIXME: If LHS and RHS are the same we can use a single FEQ.
1878 NeedInvert = Pred == CmpInst::FCMP_UNO;
1879 auto Cmp1 = MIB.buildInstr(Opc: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size),
1880 DstOps: {&RISCV::GPRRegClass}, SrcOps: {LHS, LHS});
1881 Cmp1.constrainAllUses(TII, TRI, RBI);
1882 auto Cmp2 = MIB.buildInstr(Opc: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size),
1883 DstOps: {&RISCV::GPRRegClass}, SrcOps: {RHS, RHS});
1884 Cmp2.constrainAllUses(TII, TRI, RBI);
1885 if (NeedInvert)
1886 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1887 auto And =
1888 MIB.buildInstr(Opc: RISCV::AND, DstOps: {TmpReg}, SrcOps: {Cmp1.getReg(Idx: 0), Cmp2.getReg(Idx: 0)});
1889 And.constrainAllUses(TII, TRI, RBI);
1890 } else
1891 llvm_unreachable("Unhandled predicate");
1892
1893 // Emit an XORI to invert the result if needed.
1894 if (NeedInvert) {
1895 auto Xor = MIB.buildInstr(Opc: RISCV::XORI, DstOps: {DstReg}, SrcOps: {TmpReg}).addImm(Val: 1);
1896 Xor.constrainAllUses(TII, TRI, RBI);
1897 }
1898
1899 MI.eraseFromParent();
1900 return true;
1901}
1902
1903void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
1904 SyncScope::ID FenceSSID,
1905 MachineIRBuilder &MIB) const {
1906 if (STI.hasStdExtZtso()) {
1907 // The only fence that needs an instruction is a sequentially-consistent
1908 // cross-thread fence.
1909 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
1910 FenceSSID == SyncScope::System) {
1911 // fence rw, rw
1912 MIB.buildInstr(Opc: RISCV::FENCE, DstOps: {}, SrcOps: {})
1913 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W)
1914 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W);
1915 return;
1916 }
1917
1918 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
1919 MIB.buildInstr(Opc: TargetOpcode::MEMBARRIER, DstOps: {}, SrcOps: {});
1920 return;
1921 }
1922
1923 // singlethread fences only synchronize with signal handlers on the same
1924 // thread and thus only need to preserve instruction order, not actually
1925 // enforce memory ordering.
1926 if (FenceSSID == SyncScope::SingleThread) {
1927 MIB.buildInstr(Opc: TargetOpcode::MEMBARRIER, DstOps: {}, SrcOps: {});
1928 return;
1929 }
1930
1931 // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
1932 // Manual: Volume I.
1933 unsigned Pred, Succ;
1934 switch (FenceOrdering) {
1935 default:
1936 llvm_unreachable("Unexpected ordering");
1937 case AtomicOrdering::AcquireRelease:
1938 // fence acq_rel -> fence.tso
1939 MIB.buildInstr(Opc: RISCV::FENCE_TSO, DstOps: {}, SrcOps: {});
1940 return;
1941 case AtomicOrdering::Acquire:
1942 // fence acquire -> fence r, rw
1943 Pred = RISCVFenceField::R;
1944 Succ = RISCVFenceField::R | RISCVFenceField::W;
1945 break;
1946 case AtomicOrdering::Release:
1947 // fence release -> fence rw, w
1948 Pred = RISCVFenceField::R | RISCVFenceField::W;
1949 Succ = RISCVFenceField::W;
1950 break;
1951 case AtomicOrdering::SequentiallyConsistent:
1952 // fence seq_cst -> fence rw, rw
1953 Pred = RISCVFenceField::R | RISCVFenceField::W;
1954 Succ = RISCVFenceField::R | RISCVFenceField::W;
1955 break;
1956 }
1957 MIB.buildInstr(Opc: RISCV::FENCE, DstOps: {}, SrcOps: {}).addImm(Val: Pred).addImm(Val: Succ);
1958}
1959
1960namespace llvm {
1961InstructionSelector *
1962createRISCVInstructionSelector(const RISCVTargetMachine &TM,
1963 const RISCVSubtarget &Subtarget,
1964 const RISCVRegisterBankInfo &RBI) {
1965 return new RISCVInstructionSelector(TM, Subtarget, RBI);
1966}
1967} // end namespace llvm
1968