1//===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// RISC-V.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/RISCVMatInt.h"
15#include "RISCVRegisterBankInfo.h"
16#include "RISCVSubtarget.h"
17#include "RISCVTargetMachine.h"
18#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
19#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
20#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
21#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
22#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/MachineJumpTableInfo.h"
25#include "llvm/IR/IntrinsicsRISCV.h"
26#include "llvm/Support/Debug.h"
27
28#define DEBUG_TYPE "riscv-isel"
29
30using namespace llvm;
31using namespace MIPatternMatch;
32
33#define GET_GLOBALISEL_PREDICATE_BITSET
34#include "RISCVGenGlobalISel.inc"
35#undef GET_GLOBALISEL_PREDICATE_BITSET
36
37namespace {
38
39class RISCVInstructionSelector : public InstructionSelector {
40public:
41 RISCVInstructionSelector(const RISCVTargetMachine &TM,
42 const RISCVSubtarget &STI,
43 const RISCVRegisterBankInfo &RBI);
44
45 bool select(MachineInstr &MI) override;
46
47 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
48 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
49 BlockFrequencyInfo *BFI) override {
50 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
51 MRI = &MF.getRegInfo();
52 }
53
54 static const char *getName() { return DEBUG_TYPE; }
55
56private:
57 static constexpr unsigned MaxRecursionDepth = 6;
58
59 bool hasAllNBitUsers(const MachineInstr &MI, unsigned Bits,
60 const unsigned Depth = 0) const;
61 bool hasAllHUsers(const MachineInstr &MI) const {
62 return hasAllNBitUsers(MI, Bits: 16);
63 }
64 bool hasAllWUsers(const MachineInstr &MI) const {
65 return hasAllNBitUsers(MI, Bits: 32);
66 }
67
68 bool isRegInGprb(Register Reg) const;
69 bool isRegInFprb(Register Reg) const;
70
71 // tblgen-erated 'select' implementation, used as the initial selector for
72 // the patterns that don't require complex C++.
73 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
74
75 // A lowering phase that runs before any selection attempts.
76 // Returns true if the instruction was modified.
77 void preISelLower(MachineInstr &MI);
78
79 bool replacePtrWithInt(MachineOperand &Op);
80
81 // Custom selection methods
82 bool selectCopy(MachineInstr &MI) const;
83 bool selectImplicitDef(MachineInstr &MI) const;
84 bool materializeImm(Register Reg, int64_t Imm, MachineInstr &MI) const;
85 bool selectAddr(MachineInstr &MI, bool IsLocal = true,
86 bool IsExternWeak = false) const;
87 bool selectSelect(MachineInstr &MI) const;
88 bool selectFPCompare(MachineInstr &MI) const;
89 void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
90 MachineInstr &MI) const;
91 bool selectUnmergeValues(MachineInstr &MI) const;
92 void addVectorLoadStoreOperands(MachineInstr &I,
93 SmallVectorImpl<Register> &SrcOps,
94 unsigned &CurOp, bool IsMasked,
95 bool IsStridedOrIndexed,
96 LLT *IndexVT = nullptr) const;
97 bool selectIntrinsicWithSideEffects(MachineInstr &I) const;
98 bool selectIntrinsic(MachineInstr &I) const;
99 bool selectExtractSubvector(MachineInstr &MI) const;
100 bool selectInsertSubVector(MachineInstr &I) const;
101 ComplexRendererFns selectShiftMask(MachineOperand &Root,
102 unsigned ShiftWidth) const;
103 ComplexRendererFns selectShiftMaskXLen(MachineOperand &Root) const {
104 return selectShiftMask(Root, ShiftWidth: STI.getXLen());
105 }
106 ComplexRendererFns selectShiftMask32(MachineOperand &Root) const {
107 return selectShiftMask(Root, ShiftWidth: 32);
108 }
109 ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
110
111 ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;
112 template <unsigned Bits>
113 ComplexRendererFns selectSExtBits(MachineOperand &Root) const {
114 return selectSExtBits(Root, Bits);
115 }
116
117 ComplexRendererFns selectZExtBits(MachineOperand &Root, unsigned Bits) const;
118 template <unsigned Bits>
119 ComplexRendererFns selectZExtBits(MachineOperand &Root) const {
120 return selectZExtBits(Root, Bits);
121 }
122
123 ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
124 template <unsigned ShAmt>
125 ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
126 return selectSHXADDOp(Root, ShAmt);
127 }
128
129 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
130 unsigned ShAmt) const;
131 template <unsigned ShAmt>
132 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
133 return selectSHXADD_UWOp(Root, ShAmt);
134 }
135
136 ComplexRendererFns renderVLOp(MachineOperand &Root) const;
137
138 // Custom renderers for tablegen
139 void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
140 int OpIdx) const;
141 void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
142 int OpIdx) const;
143 void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
144 int OpIdx) const;
145 void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
146 int OpIdx) const;
147 void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI,
148 int OpIdx) const;
149
150 void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
151 int OpIdx) const;
152 void renderXLenSubTrailingOnes(MachineInstrBuilder &MIB,
153 const MachineInstr &MI, int OpIdx) const;
154
155 void renderAddiPairImmLarge(MachineInstrBuilder &MIB, const MachineInstr &MI,
156 int OpIdx) const;
157 void renderAddiPairImmSmall(MachineInstrBuilder &MIB, const MachineInstr &MI,
158 int OpIdx) const;
159
160 const RISCVSubtarget &STI;
161 const RISCVInstrInfo &TII;
162 const RISCVRegisterInfo &TRI;
163 const RISCVRegisterBankInfo &RBI;
164 const RISCVTargetMachine &TM;
165
166 MachineRegisterInfo *MRI = nullptr;
167
168 // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
169 // uses "STI." in the code generated by TableGen. We need to unify the name of
170 // Subtarget variable.
171 const RISCVSubtarget *Subtarget = &STI;
172
173#define GET_GLOBALISEL_PREDICATES_DECL
174#include "RISCVGenGlobalISel.inc"
175#undef GET_GLOBALISEL_PREDICATES_DECL
176
177#define GET_GLOBALISEL_TEMPORARIES_DECL
178#include "RISCVGenGlobalISel.inc"
179#undef GET_GLOBALISEL_TEMPORARIES_DECL
180};
181
182} // end anonymous namespace
183
184#define GET_GLOBALISEL_IMPL
185#include "RISCVGenGlobalISel.inc"
186#undef GET_GLOBALISEL_IMPL
187
188RISCVInstructionSelector::RISCVInstructionSelector(
189 const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
190 const RISCVRegisterBankInfo &RBI)
191 : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
192 TM(TM),
193
194#define GET_GLOBALISEL_PREDICATES_INIT
195#include "RISCVGenGlobalISel.inc"
196#undef GET_GLOBALISEL_PREDICATES_INIT
197#define GET_GLOBALISEL_TEMPORARIES_INIT
198#include "RISCVGenGlobalISel.inc"
199#undef GET_GLOBALISEL_TEMPORARIES_INIT
200{
201}
202
203// Mimics optimizations in ISel and RISCVOptWInst Pass
204bool RISCVInstructionSelector::hasAllNBitUsers(const MachineInstr &MI,
205 unsigned Bits,
206 const unsigned Depth) const {
207
208 assert((MI.getOpcode() == TargetOpcode::G_ADD ||
209 MI.getOpcode() == TargetOpcode::G_SUB ||
210 MI.getOpcode() == TargetOpcode::G_MUL ||
211 MI.getOpcode() == TargetOpcode::G_SHL ||
212 MI.getOpcode() == TargetOpcode::G_LSHR ||
213 MI.getOpcode() == TargetOpcode::G_AND ||
214 MI.getOpcode() == TargetOpcode::G_OR ||
215 MI.getOpcode() == TargetOpcode::G_XOR ||
216 MI.getOpcode() == TargetOpcode::G_SEXT_INREG || Depth != 0) &&
217 "Unexpected opcode");
218
219 if (Depth >= RISCVInstructionSelector::MaxRecursionDepth)
220 return false;
221
222 auto DestReg = MI.getOperand(i: 0).getReg();
223 for (auto &UserOp : MRI->use_nodbg_operands(Reg: DestReg)) {
224 assert(UserOp.getParent() && "UserOp must have a parent");
225 const MachineInstr &UserMI = *UserOp.getParent();
226 unsigned OpIdx = UserOp.getOperandNo();
227
228 switch (UserMI.getOpcode()) {
229 default:
230 return false;
231 case RISCV::ADDW:
232 case RISCV::ADDIW:
233 case RISCV::SUBW:
234 case RISCV::FCVT_D_W:
235 case RISCV::FCVT_S_W:
236 if (Bits >= 32)
237 break;
238 return false;
239 case RISCV::SLL:
240 case RISCV::SRA:
241 case RISCV::SRL:
242 // Shift amount operands only use log2(Xlen) bits.
243 if (OpIdx == 2 && Bits >= Log2_32(Value: Subtarget->getXLen()))
244 break;
245 return false;
246 case RISCV::SLLI:
247 // SLLI only uses the lower (XLen - ShAmt) bits.
248 if (Bits >= Subtarget->getXLen() - UserMI.getOperand(i: 2).getImm())
249 break;
250 return false;
251 case RISCV::ANDI:
252 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(
253 Value: (uint64_t)UserMI.getOperand(i: 2).getImm()))
254 break;
255 goto RecCheck;
256 case RISCV::AND:
257 case RISCV::OR:
258 case RISCV::XOR:
259 RecCheck:
260 if (hasAllNBitUsers(MI: UserMI, Bits, Depth: Depth + 1))
261 break;
262 return false;
263 case RISCV::SRLI: {
264 unsigned ShAmt = UserMI.getOperand(i: 2).getImm();
265 // If we are shifting right by less than Bits, and users don't demand any
266 // bits that were shifted into [Bits-1:0], then we can consider this as an
267 // N-Bit user.
268 if (Bits > ShAmt && hasAllNBitUsers(MI: UserMI, Bits: Bits - ShAmt, Depth: Depth + 1))
269 break;
270 return false;
271 }
272 }
273 }
274
275 return true;
276}
277
278InstructionSelector::ComplexRendererFns
279RISCVInstructionSelector::selectShiftMask(MachineOperand &Root,
280 unsigned ShiftWidth) const {
281 if (!Root.isReg())
282 return std::nullopt;
283
284 using namespace llvm::MIPatternMatch;
285
286 Register ShAmtReg = Root.getReg();
287 // Peek through zext.
288 Register ZExtSrcReg;
289 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrcReg))))
290 ShAmtReg = ZExtSrcReg;
291
292 APInt AndMask;
293 Register AndSrcReg;
294 // Try to combine the following pattern (applicable to other shift
295 // instructions as well as 32-bit ones):
296 //
297 // %4:gprb(s64) = G_AND %3, %2
298 // %5:gprb(s64) = G_LSHR %1, %4(s64)
299 //
300 // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
301 // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
302 // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
303 // then it can be eliminated. Given register rs1 or rs2 holding a constant
304 // (the and mask), there are two cases G_AND can be erased:
305 //
306 // 1. the lowest log2(XLEN) bits of the and mask are all set
307 // 2. the bits of the register being masked are already unset (zero set)
308 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: AndSrcReg), R: m_ICst(Cst&: AndMask)))) {
309 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
310 if (ShMask.isSubsetOf(RHS: AndMask)) {
311 ShAmtReg = AndSrcReg;
312 } else {
313 // SimplifyDemandedBits may have optimized the mask so try restoring any
314 // bits that are known zero.
315 KnownBits Known = VT->getKnownBits(R: AndSrcReg);
316 if (ShMask.isSubsetOf(RHS: AndMask | Known.Zero))
317 ShAmtReg = AndSrcReg;
318 }
319 }
320
321 APInt Imm;
322 Register Reg;
323 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAdd(L: m_Reg(R&: Reg), R: m_ICst(Cst&: Imm)))) {
324 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0)
325 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
326 // to avoid the ADD.
327 ShAmtReg = Reg;
328 } else if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GSub(L: m_ICst(Cst&: Imm), R: m_Reg(R&: Reg)))) {
329 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0) {
330 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
331 // to generate a NEG instead of a SUB of a constant.
332 ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
333 unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
334 return {{[=](MachineInstrBuilder &MIB) {
335 MachineIRBuilder(*MIB.getInstr())
336 .buildInstr(Opc: NegOpc, DstOps: {ShAmtReg}, SrcOps: {Register(RISCV::X0), Reg});
337 MIB.addReg(RegNo: ShAmtReg);
338 }}};
339 }
340 if (Imm.urem(RHS: ShiftWidth) == ShiftWidth - 1) {
341 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
342 // to generate a NOT instead of a SUB of a constant.
343 ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
344 return {{[=](MachineInstrBuilder &MIB) {
345 MachineIRBuilder(*MIB.getInstr())
346 .buildInstr(Opc: RISCV::XORI, DstOps: {ShAmtReg}, SrcOps: {Reg})
347 .addImm(Val: -1);
348 MIB.addReg(RegNo: ShAmtReg);
349 }}};
350 }
351 }
352
353 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ShAmtReg); }}};
354}
355
356InstructionSelector::ComplexRendererFns
357RISCVInstructionSelector::selectSExtBits(MachineOperand &Root,
358 unsigned Bits) const {
359 if (!Root.isReg())
360 return std::nullopt;
361 Register RootReg = Root.getReg();
362 MachineInstr *RootDef = MRI->getVRegDef(Reg: RootReg);
363
364 if (RootDef->getOpcode() == TargetOpcode::G_SEXT_INREG &&
365 RootDef->getOperand(i: 2).getImm() == Bits) {
366 return {
367 {[=](MachineInstrBuilder &MIB) { MIB.add(MO: RootDef->getOperand(i: 1)); }}};
368 }
369
370 unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits();
371 if ((Size - VT->computeNumSignBits(R: RootReg)) < Bits)
372 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}};
373
374 return std::nullopt;
375}
376
377InstructionSelector::ComplexRendererFns
378RISCVInstructionSelector::selectZExtBits(MachineOperand &Root,
379 unsigned Bits) const {
380 if (!Root.isReg())
381 return std::nullopt;
382 Register RootReg = Root.getReg();
383
384 Register RegX;
385 uint64_t Mask = maskTrailingOnes<uint64_t>(N: Bits);
386 if (mi_match(R: RootReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: RegX), R: m_SpecificICst(RequestedValue: Mask)))) {
387 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}};
388 }
389
390 if (mi_match(R: RootReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: RegX))) &&
391 MRI->getType(Reg: RegX).getScalarSizeInBits() == Bits)
392 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}};
393
394 unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits();
395 if (VT->maskedValueIsZero(Val: RootReg, Mask: APInt::getBitsSetFrom(numBits: Size, loBit: Bits)))
396 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}};
397
398 return std::nullopt;
399}
400
401InstructionSelector::ComplexRendererFns
402RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
403 unsigned ShAmt) const {
404 using namespace llvm::MIPatternMatch;
405
406 if (!Root.isReg())
407 return std::nullopt;
408 Register RootReg = Root.getReg();
409
410 const unsigned XLen = STI.getXLen();
411 APInt Mask, C2;
412 Register RegY;
413 std::optional<bool> LeftShift;
414 // (and (shl y, c2), mask)
415 if (mi_match(R: RootReg, MRI: *MRI,
416 P: m_GAnd(L: m_GShl(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
417 LeftShift = true;
418 // (and (lshr y, c2), mask)
419 else if (mi_match(R: RootReg, MRI: *MRI,
420 P: m_GAnd(L: m_GLShr(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
421 LeftShift = false;
422
423 if (LeftShift.has_value()) {
424 if (*LeftShift)
425 Mask &= maskTrailingZeros<uint64_t>(N: C2.getZExtValue());
426 else
427 Mask &= maskTrailingOnes<uint64_t>(N: XLen - C2.getZExtValue());
428
429 if (Mask.isShiftedMask()) {
430 unsigned Leading = XLen - Mask.getActiveBits();
431 unsigned Trailing = Mask.countr_zero();
432 // Given (and (shl y, c2), mask) in which mask has no leading zeros and
433 // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
434 if (*LeftShift && Leading == 0 && C2.ult(RHS: Trailing) && Trailing == ShAmt) {
435 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
436 return {{[=](MachineInstrBuilder &MIB) {
437 MachineIRBuilder(*MIB.getInstr())
438 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
439 .addImm(Val: Trailing - C2.getZExtValue());
440 MIB.addReg(RegNo: DstReg);
441 }}};
442 }
443
444 // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
445 // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
446 if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
447 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
448 return {{[=](MachineInstrBuilder &MIB) {
449 MachineIRBuilder(*MIB.getInstr())
450 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
451 .addImm(Val: Leading + Trailing);
452 MIB.addReg(RegNo: DstReg);
453 }}};
454 }
455 }
456 }
457
458 LeftShift.reset();
459
460 // (shl (and y, mask), c2)
461 if (mi_match(R: RootReg, MRI: *MRI,
462 P: m_GShl(L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
463 R: m_ICst(Cst&: C2))))
464 LeftShift = true;
465 // (lshr (and y, mask), c2)
466 else if (mi_match(R: RootReg, MRI: *MRI,
467 P: m_GLShr(L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
468 R: m_ICst(Cst&: C2))))
469 LeftShift = false;
470
471 if (LeftShift.has_value() && Mask.isShiftedMask()) {
472 unsigned Leading = XLen - Mask.getActiveBits();
473 unsigned Trailing = Mask.countr_zero();
474
475 // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
476 // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
477 bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&
478 (Trailing + C2.getZExtValue()) == ShAmt;
479 if (!Cond)
480 // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
481 // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
482 Cond = !*LeftShift && Leading == 32 && C2.ult(RHS: Trailing) &&
483 (Trailing - C2.getZExtValue()) == ShAmt;
484
485 if (Cond) {
486 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
487 return {{[=](MachineInstrBuilder &MIB) {
488 MachineIRBuilder(*MIB.getInstr())
489 .buildInstr(Opc: RISCV::SRLIW, DstOps: {DstReg}, SrcOps: {RegY})
490 .addImm(Val: Trailing);
491 MIB.addReg(RegNo: DstReg);
492 }}};
493 }
494 }
495
496 return std::nullopt;
497}
498
499InstructionSelector::ComplexRendererFns
500RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
501 unsigned ShAmt) const {
502 using namespace llvm::MIPatternMatch;
503
504 if (!Root.isReg())
505 return std::nullopt;
506 Register RootReg = Root.getReg();
507
508 // Given (and (shl x, c2), mask) in which mask is a shifted mask with
509 // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
510 // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
511 APInt Mask, C2;
512 Register RegX;
513 if (mi_match(
514 R: RootReg, MRI: *MRI,
515 P: m_OneNonDBGUse(SP: m_GAnd(L: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: RegX), R: m_ICst(Cst&: C2))),
516 R: m_ICst(Cst&: Mask))))) {
517 Mask &= maskTrailingZeros<uint64_t>(N: C2.getZExtValue());
518
519 if (Mask.isShiftedMask()) {
520 unsigned Leading = Mask.countl_zero();
521 unsigned Trailing = Mask.countr_zero();
522 if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
523 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
524 return {{[=](MachineInstrBuilder &MIB) {
525 MachineIRBuilder(*MIB.getInstr())
526 .buildInstr(Opc: RISCV::SLLI, DstOps: {DstReg}, SrcOps: {RegX})
527 .addImm(Val: C2.getZExtValue() - ShAmt);
528 MIB.addReg(RegNo: DstReg);
529 }}};
530 }
531 }
532 }
533
534 return std::nullopt;
535}
536
537InstructionSelector::ComplexRendererFns
538RISCVInstructionSelector::renderVLOp(MachineOperand &Root) const {
539 assert(Root.isReg() && "Expected operand to be a Register");
540 MachineInstr *RootDef = MRI->getVRegDef(Reg: Root.getReg());
541
542 if (RootDef->getOpcode() == TargetOpcode::G_CONSTANT) {
543 auto C = RootDef->getOperand(i: 1).getCImm();
544 if (C->getValue().isAllOnes())
545 // If the operand is a G_CONSTANT with value of all ones it is larger than
546 // VLMAX. We convert it to an immediate with value VLMaxSentinel. This is
547 // recognized specially by the vsetvli insertion pass.
548 return {{[=](MachineInstrBuilder &MIB) {
549 MIB.addImm(Val: RISCV::VLMaxSentinel);
550 }}};
551
552 if (isUInt<5>(x: C->getZExtValue())) {
553 uint64_t ZExtC = C->getZExtValue();
554 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ZExtC); }}};
555 }
556 }
557 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Root.getReg()); }}};
558}
559
560InstructionSelector::ComplexRendererFns
561RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
562 if (!Root.isReg())
563 return std::nullopt;
564
565 MachineInstr *RootDef = MRI->getVRegDef(Reg: Root.getReg());
566 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
567 return {{
568 [=](MachineInstrBuilder &MIB) { MIB.add(MO: RootDef->getOperand(i: 1)); },
569 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
570 }};
571 }
572
573 if (isBaseWithConstantOffset(Root, MRI: *MRI)) {
574 MachineOperand &LHS = RootDef->getOperand(i: 1);
575 MachineOperand &RHS = RootDef->getOperand(i: 2);
576 MachineInstr *LHSDef = MRI->getVRegDef(Reg: LHS.getReg());
577 MachineInstr *RHSDef = MRI->getVRegDef(Reg: RHS.getReg());
578
579 int64_t RHSC = RHSDef->getOperand(i: 1).getCImm()->getSExtValue();
580 if (isInt<12>(x: RHSC)) {
581 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
582 return {{
583 [=](MachineInstrBuilder &MIB) { MIB.add(MO: LHSDef->getOperand(i: 1)); },
584 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
585 }};
586
587 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: LHS); },
588 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
589 }
590 }
591
592 // TODO: Need to get the immediate from a G_PTR_ADD. Should this be done in
593 // the combiner?
594 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Root.getReg()); },
595 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
596}
597
598/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
599/// CC Must be an ICMP Predicate.
600static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
601 switch (CC) {
602 default:
603 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
604 case CmpInst::Predicate::ICMP_EQ:
605 return RISCVCC::COND_EQ;
606 case CmpInst::Predicate::ICMP_NE:
607 return RISCVCC::COND_NE;
608 case CmpInst::Predicate::ICMP_ULT:
609 return RISCVCC::COND_LTU;
610 case CmpInst::Predicate::ICMP_SLT:
611 return RISCVCC::COND_LT;
612 case CmpInst::Predicate::ICMP_UGE:
613 return RISCVCC::COND_GEU;
614 case CmpInst::Predicate::ICMP_SGE:
615 return RISCVCC::COND_GE;
616 }
617}
618
619static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC,
620 Register &LHS, Register &RHS,
621 MachineRegisterInfo &MRI) {
622 // Try to fold an ICmp. If that fails, use a NE compare with X0.
623 CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
624 if (!mi_match(R: CondReg, MRI, P: m_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: LHS), R: m_Reg(R&: RHS)))) {
625 LHS = CondReg;
626 RHS = RISCV::X0;
627 CC = RISCVCC::COND_NE;
628 return;
629 }
630
631 // We found an ICmp, do some canonicalization.
632
633 // Adjust comparisons to use comparison with 0 if possible.
634 if (auto Constant = getIConstantVRegSExtVal(VReg: RHS, MRI)) {
635 switch (Pred) {
636 case CmpInst::Predicate::ICMP_SGT:
637 // Convert X > -1 to X >= 0
638 if (*Constant == -1) {
639 CC = RISCVCC::COND_GE;
640 RHS = RISCV::X0;
641 return;
642 }
643 break;
644 case CmpInst::Predicate::ICMP_SLT:
645 // Convert X < 1 to 0 >= X
646 if (*Constant == 1) {
647 CC = RISCVCC::COND_GE;
648 RHS = LHS;
649 LHS = RISCV::X0;
650 return;
651 }
652 break;
653 default:
654 break;
655 }
656 }
657
658 switch (Pred) {
659 default:
660 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
661 case CmpInst::Predicate::ICMP_EQ:
662 case CmpInst::Predicate::ICMP_NE:
663 case CmpInst::Predicate::ICMP_ULT:
664 case CmpInst::Predicate::ICMP_SLT:
665 case CmpInst::Predicate::ICMP_UGE:
666 case CmpInst::Predicate::ICMP_SGE:
667 // These CCs are supported directly by RISC-V branches.
668 break;
669 case CmpInst::Predicate::ICMP_SGT:
670 case CmpInst::Predicate::ICMP_SLE:
671 case CmpInst::Predicate::ICMP_UGT:
672 case CmpInst::Predicate::ICMP_ULE:
673 // These CCs are not supported directly by RISC-V branches, but changing the
674 // direction of the CC and swapping LHS and RHS are.
675 Pred = CmpInst::getSwappedPredicate(pred: Pred);
676 std::swap(a&: LHS, b&: RHS);
677 break;
678 }
679
680 CC = getRISCVCCFromICmp(CC: Pred);
681}
682
683/// Select the RISC-V Zalasr opcode for the G_LOAD or G_STORE operation
684/// \p GenericOpc, appropriate for the GPR register bank and of memory access
685/// size \p OpSize.
686static unsigned selectZalasrLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {
687 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
688 switch (OpSize) {
689 default:
690 llvm_unreachable("Unexpected memory size");
691 case 8:
692 return IsStore ? RISCV::SB_RL : RISCV::LB_AQ;
693 case 16:
694 return IsStore ? RISCV::SH_RL : RISCV::LH_AQ;
695 case 32:
696 return IsStore ? RISCV::SW_RL : RISCV::LW_AQ;
697 case 64:
698 return IsStore ? RISCV::SD_RL : RISCV::LD_AQ;
699 }
700}
701
702/// Select the RISC-V regimm opcode for the G_LOAD or G_STORE operation
703/// \p GenericOpc, appropriate for the GPR register bank and of memory access
704/// size \p OpSize. \returns \p GenericOpc if the combination is unsupported.
705static unsigned selectRegImmLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {
706 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
707 switch (OpSize) {
708 case 8:
709 // Prefer unsigned due to no c.lb in Zcb.
710 return IsStore ? RISCV::SB : RISCV::LBU;
711 case 16:
712 return IsStore ? RISCV::SH : RISCV::LH;
713 case 32:
714 return IsStore ? RISCV::SW : RISCV::LW;
715 case 64:
716 return IsStore ? RISCV::SD : RISCV::LD;
717 }
718
719 return GenericOpc;
720}
721
722void RISCVInstructionSelector::addVectorLoadStoreOperands(
723 MachineInstr &I, SmallVectorImpl<Register> &SrcOps, unsigned &CurOp,
724 bool IsMasked, bool IsStridedOrIndexed, LLT *IndexVT) const {
725 // Base Pointer
726 auto PtrReg = I.getOperand(i: CurOp++).getReg();
727 SrcOps.push_back(Elt: PtrReg);
728
729 // Stride or Index
730 if (IsStridedOrIndexed) {
731 auto StrideReg = I.getOperand(i: CurOp++).getReg();
732 SrcOps.push_back(Elt: StrideReg);
733 if (IndexVT)
734 *IndexVT = MRI->getType(Reg: StrideReg);
735 }
736
737 // Mask
738 if (IsMasked) {
739 auto MaskReg = I.getOperand(i: CurOp++).getReg();
740 SrcOps.push_back(Elt: MaskReg);
741 }
742}
743
744bool RISCVInstructionSelector::selectIntrinsicWithSideEffects(
745 MachineInstr &I) const {
746 // Find the intrinsic ID.
747 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
748 // Select the instruction.
749 switch (IntrinID) {
750 default:
751 return false;
752 case Intrinsic::riscv_vlm:
753 case Intrinsic::riscv_vle:
754 case Intrinsic::riscv_vle_mask:
755 case Intrinsic::riscv_vlse:
756 case Intrinsic::riscv_vlse_mask: {
757 bool IsMasked = IntrinID == Intrinsic::riscv_vle_mask ||
758 IntrinID == Intrinsic::riscv_vlse_mask;
759 bool IsStrided = IntrinID == Intrinsic::riscv_vlse ||
760 IntrinID == Intrinsic::riscv_vlse_mask;
761 LLT VT = MRI->getType(Reg: I.getOperand(i: 0).getReg());
762 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
763
764 // Result vector
765 const Register DstReg = I.getOperand(i: 0).getReg();
766
767 // Sources
768 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
769 unsigned CurOp = 2;
770 SmallVector<Register, 4> SrcOps; // Source registers.
771
772 // Passthru
773 if (HasPassthruOperand) {
774 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
775 SrcOps.push_back(Elt: PassthruReg);
776 } else {
777 SrcOps.push_back(Elt: Register(RISCV::NoRegister));
778 }
779
780 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: IsStrided);
781
782 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
783 const RISCV::VLEPseudo *P =
784 RISCV::getVLEPseudo(Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
785 LMUL: static_cast<unsigned>(LMUL));
786
787 MachineInstrBuilder PseudoMI =
788 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo), DestReg: DstReg);
789 for (Register Reg : SrcOps)
790 PseudoMI.addReg(RegNo: Reg);
791
792 // Select VL
793 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
794 for (auto &RenderFn : *VLOpFn)
795 RenderFn(PseudoMI);
796
797 // SEW
798 PseudoMI.addImm(Val: Log2SEW);
799
800 // Policy
801 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
802 if (IsMasked)
803 Policy = I.getOperand(i: CurOp++).getImm();
804 PseudoMI.addImm(Val: Policy);
805
806 // Memref
807 PseudoMI.cloneMemRefs(OtherMI: I);
808
809 I.eraseFromParent();
810 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
811 return true;
812 }
813 case Intrinsic::riscv_vloxei:
814 case Intrinsic::riscv_vloxei_mask:
815 case Intrinsic::riscv_vluxei:
816 case Intrinsic::riscv_vluxei_mask: {
817 bool IsMasked = IntrinID == Intrinsic::riscv_vloxei_mask ||
818 IntrinID == Intrinsic::riscv_vluxei_mask;
819 bool IsOrdered = IntrinID == Intrinsic::riscv_vloxei ||
820 IntrinID == Intrinsic::riscv_vloxei_mask;
821 LLT VT = MRI->getType(Reg: I.getOperand(i: 0).getReg());
822 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
823
824 // Result vector
825 const Register DstReg = I.getOperand(i: 0).getReg();
826
827 // Sources
828 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
829 unsigned CurOp = 2;
830 SmallVector<Register, 4> SrcOps; // Source registers.
831
832 // Passthru
833 if (HasPassthruOperand) {
834 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
835 SrcOps.push_back(Elt: PassthruReg);
836 } else {
837 // Use NoRegister if there is no specified passthru.
838 SrcOps.push_back(Elt: Register());
839 }
840 LLT IndexVT;
841 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: true, IndexVT: &IndexVT);
842
843 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
844 RISCVVType::VLMUL IndexLMUL =
845 RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: IndexVT));
846 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
847 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
848 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
849 "values when XLEN=32");
850 }
851 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
852 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
853 IndexLMUL: static_cast<unsigned>(IndexLMUL));
854
855 MachineInstrBuilder PseudoMI =
856 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo), DestReg: DstReg);
857 for (Register Reg : SrcOps)
858 PseudoMI.addReg(RegNo: Reg);
859
860 // Select VL
861 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
862 for (auto &RenderFn : *VLOpFn)
863 RenderFn(PseudoMI);
864
865 // SEW
866 PseudoMI.addImm(Val: Log2SEW);
867
868 // Policy
869 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
870 if (IsMasked)
871 Policy = I.getOperand(i: CurOp++).getImm();
872 PseudoMI.addImm(Val: Policy);
873
874 // Memref
875 PseudoMI.cloneMemRefs(OtherMI: I);
876
877 I.eraseFromParent();
878 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
879 return true;
880 }
881 case Intrinsic::riscv_vsm:
882 case Intrinsic::riscv_vse:
883 case Intrinsic::riscv_vse_mask:
884 case Intrinsic::riscv_vsse:
885 case Intrinsic::riscv_vsse_mask: {
886 bool IsMasked = IntrinID == Intrinsic::riscv_vse_mask ||
887 IntrinID == Intrinsic::riscv_vsse_mask;
888 bool IsStrided = IntrinID == Intrinsic::riscv_vsse ||
889 IntrinID == Intrinsic::riscv_vsse_mask;
890 LLT VT = MRI->getType(Reg: I.getOperand(i: 1).getReg());
891 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
892
893 // Sources
894 unsigned CurOp = 1;
895 SmallVector<Register, 4> SrcOps; // Source registers.
896
897 // Store value
898 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
899 SrcOps.push_back(Elt: PassthruReg);
900
901 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: IsStrided);
902
903 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
904 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
905 Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
906
907 MachineInstrBuilder PseudoMI =
908 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo));
909 for (Register Reg : SrcOps)
910 PseudoMI.addReg(RegNo: Reg);
911
912 // Select VL
913 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
914 for (auto &RenderFn : *VLOpFn)
915 RenderFn(PseudoMI);
916
917 // SEW
918 PseudoMI.addImm(Val: Log2SEW);
919
920 // Memref
921 PseudoMI.cloneMemRefs(OtherMI: I);
922
923 I.eraseFromParent();
924 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
925 return true;
926 }
927 case Intrinsic::riscv_vsoxei:
928 case Intrinsic::riscv_vsoxei_mask:
929 case Intrinsic::riscv_vsuxei:
930 case Intrinsic::riscv_vsuxei_mask: {
931 bool IsMasked = IntrinID == Intrinsic::riscv_vsoxei_mask ||
932 IntrinID == Intrinsic::riscv_vsuxei_mask;
933 bool IsOrdered = IntrinID == Intrinsic::riscv_vsoxei ||
934 IntrinID == Intrinsic::riscv_vsoxei_mask;
935 LLT VT = MRI->getType(Reg: I.getOperand(i: 1).getReg());
936 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
937
938 // Sources
939 unsigned CurOp = 1;
940 SmallVector<Register, 4> SrcOps; // Source registers.
941
942 // Store value
943 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
944 SrcOps.push_back(Elt: PassthruReg);
945
946 LLT IndexVT;
947 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: true, IndexVT: &IndexVT);
948
949 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
950 RISCVVType::VLMUL IndexLMUL =
951 RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: IndexVT));
952 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
953 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
954 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
955 "values when XLEN=32");
956 }
957 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
958 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
959 IndexLMUL: static_cast<unsigned>(IndexLMUL));
960
961 MachineInstrBuilder PseudoMI =
962 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo));
963 for (Register Reg : SrcOps)
964 PseudoMI.addReg(RegNo: Reg);
965
966 // Select VL
967 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
968 for (auto &RenderFn : *VLOpFn)
969 RenderFn(PseudoMI);
970
971 // SEW
972 PseudoMI.addImm(Val: Log2SEW);
973
974 // Memref
975 PseudoMI.cloneMemRefs(OtherMI: I);
976
977 I.eraseFromParent();
978 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
979 return true;
980 }
981 }
982}
983
984bool RISCVInstructionSelector::selectIntrinsic(MachineInstr &I) const {
985 // Find the intrinsic ID.
986 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
987 // Select the instruction.
988 switch (IntrinID) {
989 default:
990 return false;
991 case Intrinsic::riscv_vsetvli:
992 case Intrinsic::riscv_vsetvlimax: {
993
994 bool VLMax = IntrinID == Intrinsic::riscv_vsetvlimax;
995
996 unsigned Offset = VLMax ? 2 : 3;
997 unsigned SEW = RISCVVType::decodeVSEW(VSEW: I.getOperand(i: Offset).getImm() & 0x7);
998 RISCVVType::VLMUL VLMul =
999 static_cast<RISCVVType::VLMUL>(I.getOperand(i: Offset + 1).getImm() & 0x7);
1000
1001 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMUL: VLMul, SEW, /*TailAgnostic*/ true,
1002 /*MaskAgnostic*/ true);
1003
1004 Register DstReg = I.getOperand(i: 0).getReg();
1005
1006 Register VLOperand;
1007 unsigned Opcode = RISCV::PseudoVSETVLI;
1008
1009 // Check if AVL is a constant that equals VLMAX.
1010 if (!VLMax) {
1011 Register AVLReg = I.getOperand(i: 2).getReg();
1012 if (auto AVLConst = getIConstantVRegValWithLookThrough(VReg: AVLReg, MRI: *MRI)) {
1013 uint64_t AVL = AVLConst->Value.getZExtValue();
1014 if (auto VLEN = Subtarget->getRealVLen()) {
1015 if (*VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == AVL)
1016 VLMax = true;
1017 }
1018 }
1019
1020 MachineInstr *AVLDef = MRI->getVRegDef(Reg: AVLReg);
1021 if (AVLDef && AVLDef->getOpcode() == TargetOpcode::G_CONSTANT) {
1022 const auto *C = AVLDef->getOperand(i: 1).getCImm();
1023 if (C->getValue().isAllOnes())
1024 VLMax = true;
1025 }
1026 }
1027
1028 if (VLMax) {
1029 VLOperand = Register(RISCV::X0);
1030 Opcode = RISCV::PseudoVSETVLIX0;
1031 } else {
1032 Register AVLReg = I.getOperand(i: 2).getReg();
1033 VLOperand = AVLReg;
1034
1035 // Check if AVL is a small constant that can use PseudoVSETIVLI.
1036 if (auto AVLConst = getIConstantVRegValWithLookThrough(VReg: AVLReg, MRI: *MRI)) {
1037 uint64_t AVL = AVLConst->Value.getZExtValue();
1038 if (isUInt<5>(x: AVL)) {
1039 MachineInstr *PseudoMI =
1040 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1041 MCID: TII.get(Opcode: RISCV::PseudoVSETIVLI), DestReg: DstReg)
1042 .addImm(Val: AVL)
1043 .addImm(Val: VTypeI);
1044 I.eraseFromParent();
1045 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1046 return true;
1047 }
1048 }
1049 }
1050
1051 MachineInstr *PseudoMI =
1052 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode), DestReg: DstReg)
1053 .addReg(RegNo: VLOperand)
1054 .addImm(Val: VTypeI);
1055 I.eraseFromParent();
1056 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1057 return true;
1058 }
1059 }
1060}
1061
1062bool RISCVInstructionSelector::selectExtractSubvector(MachineInstr &MI) const {
1063 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_SUBVECTOR);
1064
1065 Register DstReg = MI.getOperand(i: 0).getReg();
1066 Register SrcReg = MI.getOperand(i: 1).getReg();
1067
1068 LLT DstTy = MRI->getType(Reg: DstReg);
1069 LLT SrcTy = MRI->getType(Reg: SrcReg);
1070
1071 unsigned Idx = static_cast<unsigned>(MI.getOperand(i: 2).getImm());
1072
1073 MVT DstMVT = getMVTForLLT(Ty: DstTy);
1074 MVT SrcMVT = getMVTForLLT(Ty: SrcTy);
1075
1076 unsigned SubRegIdx;
1077 std::tie(args&: SubRegIdx, args&: Idx) =
1078 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1079 VecVT: SrcMVT, SubVecVT: DstMVT, InsertExtractIdx: Idx, TRI: &TRI);
1080
1081 if (Idx != 0)
1082 return false;
1083
1084 unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: DstMVT);
1085 const TargetRegisterClass *DstRC = TRI.getRegClass(i: DstRegClassID);
1086 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1087 return false;
1088
1089 unsigned SrcRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: SrcMVT);
1090 const TargetRegisterClass *SrcRC = TRI.getRegClass(i: SrcRegClassID);
1091 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
1092 return false;
1093
1094 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1095 DestReg: DstReg)
1096 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubRegIdx);
1097
1098 MI.eraseFromParent();
1099 return true;
1100}
1101
1102bool RISCVInstructionSelector::selectInsertSubVector(MachineInstr &MI) const {
1103 assert(MI.getOpcode() == TargetOpcode::G_INSERT_SUBVECTOR);
1104
1105 Register DstReg = MI.getOperand(i: 0).getReg();
1106 Register VecReg = MI.getOperand(i: 1).getReg();
1107 Register SubVecReg = MI.getOperand(i: 2).getReg();
1108
1109 LLT VecTy = MRI->getType(Reg: VecReg);
1110 LLT SubVecTy = MRI->getType(Reg: SubVecReg);
1111
1112 MVT VecMVT = getMVTForLLT(Ty: VecTy);
1113 MVT SubVecMVT = getMVTForLLT(Ty: SubVecTy);
1114
1115 unsigned Idx = static_cast<unsigned>(MI.getOperand(i: 3).getImm());
1116
1117 unsigned SubRegIdx;
1118 std::tie(args&: SubRegIdx, args&: Idx) =
1119 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1120 VecVT: VecMVT, SubVecVT: SubVecMVT, InsertExtractIdx: Idx, TRI: &TRI);
1121
1122 // If the Idx hasn't been completely eliminated then this is a subvector
1123 // insert which doesn't naturally align to a vector register. These must
1124 // be handled using instructions to manipulate the vector registers.
1125 if (Idx != 0)
1126 return false;
1127
1128 // Constrain dst
1129 unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: VecMVT);
1130 const TargetRegisterClass *DstRC = TRI.getRegClass(i: DstRegClassID);
1131 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1132 return false;
1133
1134 // If we haven't set a SubRegIdx, then we must be going between
1135 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
1136 if (SubRegIdx == RISCV::NoSubRegister) {
1137 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecMVT) ==
1138 DstRegClassID &&
1139 "Unexpected subvector insert");
1140 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1141 DestReg: DstReg)
1142 .addReg(RegNo: SubVecReg);
1143 MI.eraseFromParent();
1144 return true;
1145 }
1146
1147 // Use INSERT_SUBREG to insert the subvector into the vector at the
1148 // appropriate subregister index.
1149 MachineInstr *Ins = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1150 MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DstReg)
1151 .addReg(RegNo: VecReg)
1152 .addReg(RegNo: SubVecReg)
1153 .addImm(Val: SubRegIdx);
1154
1155 MI.eraseFromParent();
1156 constrainSelectedInstRegOperands(I&: *Ins, TII, TRI, RBI);
1157 return true;
1158}
1159
1160bool RISCVInstructionSelector::select(MachineInstr &MI) {
1161 preISelLower(MI);
1162 const unsigned Opc = MI.getOpcode();
1163
1164 if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
1165 if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
1166 const Register DefReg = MI.getOperand(i: 0).getReg();
1167 const LLT DefTy = MRI->getType(Reg: DefReg);
1168
1169 const RegClassOrRegBank &RegClassOrBank =
1170 MRI->getRegClassOrRegBank(Reg: DefReg);
1171
1172 const TargetRegisterClass *DefRC =
1173 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
1174 if (!DefRC) {
1175 if (!DefTy.isValid()) {
1176 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
1177 return false;
1178 }
1179
1180 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
1181 DefRC = TRI.getRegClassForTypeOnBank(Ty: DefTy, RB, Is64Bit: STI.is64Bit());
1182 if (!DefRC) {
1183 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
1184 return false;
1185 }
1186 }
1187
1188 MI.setDesc(TII.get(Opcode: TargetOpcode::PHI));
1189 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI&: *MRI);
1190 }
1191
1192 // Certain non-generic instructions also need some special handling.
1193 if (MI.isCopy())
1194 return selectCopy(MI);
1195
1196 return true;
1197 }
1198
1199 if (selectImpl(I&: MI, CoverageInfo&: *CoverageInfo))
1200 return true;
1201
1202 switch (Opc) {
1203 case TargetOpcode::G_ANYEXT:
1204 case TargetOpcode::G_PTRTOINT:
1205 case TargetOpcode::G_INTTOPTR:
1206 case TargetOpcode::G_TRUNC:
1207 case TargetOpcode::G_FREEZE:
1208 return selectCopy(MI);
1209 case TargetOpcode::G_CONSTANT: {
1210 Register DstReg = MI.getOperand(i: 0).getReg();
1211 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
1212
1213 if (!materializeImm(Reg: DstReg, Imm, MI))
1214 return false;
1215
1216 MI.eraseFromParent();
1217 return true;
1218 }
1219 case TargetOpcode::G_ZEXT:
1220 case TargetOpcode::G_SEXT: {
1221 bool IsSigned = Opc != TargetOpcode::G_ZEXT;
1222 Register DstReg = MI.getOperand(i: 0).getReg();
1223 Register SrcReg = MI.getOperand(i: 1).getReg();
1224 LLT SrcTy = MRI->getType(Reg: SrcReg);
1225 unsigned SrcSize = SrcTy.getSizeInBits();
1226
1227 if (SrcTy.isVector())
1228 return false; // Should be handled by imported patterns.
1229
1230 assert((*RBI.getRegBank(DstReg, *MRI, TRI)).getID() ==
1231 RISCV::GPRBRegBankID &&
1232 "Unexpected ext regbank");
1233
1234 // Use addiw SrcReg, 0 (sext.w) for i32.
1235 if (IsSigned && SrcSize == 32) {
1236 MI.setDesc(TII.get(Opcode: RISCV::ADDIW));
1237 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
1238 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1239 return true;
1240 }
1241
1242 // Use add.uw SrcReg, X0 (zext.w) for i32 with Zba.
1243 if (!IsSigned && SrcSize == 32 && STI.hasStdExtZba()) {
1244 MI.setDesc(TII.get(Opcode: RISCV::ADD_UW));
1245 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::X0, /*isDef=*/false));
1246 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1247 return true;
1248 }
1249
1250 // Use sext.h/zext.h for i16 with Zbb.
1251 if (SrcSize == 16 &&
1252 (STI.hasStdExtZbb() || (!IsSigned && STI.hasStdExtZbkb()))) {
1253 MI.setDesc(TII.get(Opcode: IsSigned ? RISCV::SEXT_H
1254 : STI.isRV64() ? RISCV::ZEXT_H_RV64
1255 : RISCV::ZEXT_H_RV32));
1256 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1257 return true;
1258 }
1259
1260 // Fall back to shift pair.
1261 Register ShiftLeftReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1262 MachineInstr *ShiftLeft = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1263 MCID: TII.get(Opcode: RISCV::SLLI), DestReg: ShiftLeftReg)
1264 .addReg(RegNo: SrcReg)
1265 .addImm(Val: STI.getXLen() - SrcSize);
1266 constrainSelectedInstRegOperands(I&: *ShiftLeft, TII, TRI, RBI);
1267 MachineInstr *ShiftRight =
1268 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1269 MCID: TII.get(Opcode: IsSigned ? RISCV::SRAI : RISCV::SRLI), DestReg: DstReg)
1270 .addReg(RegNo: ShiftLeftReg)
1271 .addImm(Val: STI.getXLen() - SrcSize);
1272 constrainSelectedInstRegOperands(I&: *ShiftRight, TII, TRI, RBI);
1273 MI.eraseFromParent();
1274 return true;
1275 }
1276 case TargetOpcode::G_FCONSTANT: {
1277 // TODO: Use constant pool for complex constants.
1278 Register DstReg = MI.getOperand(i: 0).getReg();
1279 const APFloat &FPimm = MI.getOperand(i: 1).getFPImm()->getValueAPF();
1280 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
1281 if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
1282 Register GPRReg;
1283 if (FPimm.isPosZero()) {
1284 GPRReg = RISCV::X0;
1285 } else {
1286 GPRReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1287 APInt Imm = FPimm.bitcastToAPInt();
1288 if (!materializeImm(Reg: GPRReg, Imm: Imm.getSExtValue(), MI))
1289 return false;
1290 }
1291
1292 unsigned Opcode = Size == 64 ? RISCV::FMV_D_X
1293 : Size == 32 ? RISCV::FMV_W_X
1294 : RISCV::FMV_H_X;
1295 MachineInstr *FMV = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1296 MCID: TII.get(Opcode), DestReg: DstReg)
1297 .addReg(RegNo: GPRReg);
1298 constrainSelectedInstRegOperands(I&: *FMV, TII, TRI, RBI);
1299 } else {
1300 // s64 on rv32
1301 assert(Size == 64 && !Subtarget->is64Bit() &&
1302 "Unexpected size or subtarget");
1303
1304 if (FPimm.isPosZero()) {
1305 // Optimize +0.0 to use fcvt.d.w
1306 MachineInstr *FCVT = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1307 MCID: TII.get(Opcode: RISCV::FCVT_D_W), DestReg: DstReg)
1308 .addReg(RegNo: RISCV::X0)
1309 .addImm(Val: RISCVFPRndMode::RNE);
1310 constrainSelectedInstRegOperands(I&: *FCVT, TII, TRI, RBI);
1311
1312 MI.eraseFromParent();
1313 return true;
1314 }
1315
1316 // Split into two pieces and build through the stack.
1317 Register GPRRegHigh = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1318 Register GPRRegLow = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1319 APInt Imm = FPimm.bitcastToAPInt();
1320 if (!materializeImm(Reg: GPRRegHigh, Imm: Imm.extractBits(numBits: 32, bitPosition: 32).getSExtValue(),
1321 MI))
1322 return false;
1323 if (!materializeImm(Reg: GPRRegLow, Imm: Imm.trunc(width: 32).getSExtValue(), MI))
1324 return false;
1325 MachineInstr *PairF64 =
1326 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1327 MCID: TII.get(Opcode: RISCV::BuildPairF64Pseudo), DestReg: DstReg)
1328 .addReg(RegNo: GPRRegLow)
1329 .addReg(RegNo: GPRRegHigh);
1330 constrainSelectedInstRegOperands(I&: *PairF64, TII, TRI, RBI);
1331 }
1332
1333 MI.eraseFromParent();
1334 return true;
1335 }
1336 case TargetOpcode::G_GLOBAL_VALUE: {
1337 auto *GV = MI.getOperand(i: 1).getGlobal();
1338 if (GV->isThreadLocal()) {
1339 // TODO: implement this case.
1340 return false;
1341 }
1342
1343 return selectAddr(MI, IsLocal: GV->isDSOLocal(), IsExternWeak: GV->hasExternalWeakLinkage());
1344 }
1345 case TargetOpcode::G_JUMP_TABLE:
1346 case TargetOpcode::G_CONSTANT_POOL:
1347 return selectAddr(MI);
1348 case TargetOpcode::G_BRCOND: {
1349 Register LHS, RHS;
1350 RISCVCC::CondCode CC;
1351 getOperandsForBranch(CondReg: MI.getOperand(i: 0).getReg(), CC, LHS, RHS, MRI&: *MRI);
1352
1353 MachineInstr *Bcc = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1354 MCID: TII.get(Opcode: RISCVCC::getBrCond(CC)))
1355 .addReg(RegNo: LHS)
1356 .addReg(RegNo: RHS)
1357 .addMBB(MBB: MI.getOperand(i: 1).getMBB());
1358 MI.eraseFromParent();
1359 constrainSelectedInstRegOperands(I&: *Bcc, TII, TRI, RBI);
1360 return true;
1361 }
1362 case TargetOpcode::G_BRINDIRECT:
1363 MI.setDesc(TII.get(Opcode: RISCV::PseudoBRIND));
1364 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
1365 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1366 return true;
1367 case TargetOpcode::G_SELECT:
1368 return selectSelect(MI);
1369 case TargetOpcode::G_FCMP:
1370 return selectFPCompare(MI);
1371 case TargetOpcode::G_FENCE: {
1372 AtomicOrdering FenceOrdering =
1373 static_cast<AtomicOrdering>(MI.getOperand(i: 0).getImm());
1374 SyncScope::ID FenceSSID =
1375 static_cast<SyncScope::ID>(MI.getOperand(i: 1).getImm());
1376 emitFence(FenceOrdering, FenceSSID, MI);
1377 MI.eraseFromParent();
1378 return true;
1379 }
1380 case TargetOpcode::G_IMPLICIT_DEF:
1381 return selectImplicitDef(MI);
1382 case TargetOpcode::G_UNMERGE_VALUES:
1383 return selectUnmergeValues(MI);
1384 case TargetOpcode::G_LOAD:
1385 case TargetOpcode::G_STORE: {
1386 GLoadStore &LdSt = cast<GLoadStore>(Val&: MI);
1387 const Register ValReg = LdSt.getReg(Idx: 0);
1388 const Register PtrReg = LdSt.getPointerReg();
1389 LLT PtrTy = MRI->getType(Reg: PtrReg);
1390
1391 const RegisterBank &RB = *RBI.getRegBank(Reg: ValReg, MRI: *MRI, TRI);
1392 if (RB.getID() != RISCV::GPRBRegBankID)
1393 return false;
1394
1395#ifndef NDEBUG
1396 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, *MRI, TRI);
1397 // Check that the pointer register is valid.
1398 assert(PtrRB.getID() == RISCV::GPRBRegBankID &&
1399 "Load/Store pointer operand isn't a GPR");
1400 assert(PtrTy.isPointer() && "Load/Store pointer operand isn't a pointer");
1401#endif
1402
1403 // Can only handle AddressSpace 0.
1404 if (PtrTy.getAddressSpace() != 0)
1405 return false;
1406
1407 unsigned MemSize = LdSt.getMemSizeInBits().getValue();
1408 AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();
1409
1410 if (isStrongerThanMonotonic(AO: Order)) {
1411 MI.setDesc(TII.get(Opcode: selectZalasrLoadStoreOp(GenericOpc: Opc, OpSize: MemSize)));
1412 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1413 return true;
1414 }
1415
1416 const unsigned NewOpc = selectRegImmLoadStoreOp(GenericOpc: MI.getOpcode(), OpSize: MemSize);
1417 if (NewOpc == MI.getOpcode())
1418 return false;
1419
1420 // Check if we can fold anything into the addressing mode.
1421 auto AddrModeFns = selectAddrRegImm(Root&: MI.getOperand(i: 1));
1422 if (!AddrModeFns)
1423 return false;
1424
1425 // Folded something. Create a new instruction and return it.
1426 MachineInstrBuilder NewInst =
1427 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: NewOpc));
1428 NewInst.setMIFlags(MI.getFlags());
1429 if (isa<GStore>(Val: MI))
1430 NewInst.addUse(RegNo: ValReg);
1431 else
1432 NewInst.addDef(RegNo: ValReg);
1433 NewInst.cloneMemRefs(OtherMI: MI);
1434 for (auto &Fn : *AddrModeFns)
1435 Fn(NewInst);
1436 MI.eraseFromParent();
1437
1438 constrainSelectedInstRegOperands(I&: *NewInst, TII, TRI, RBI);
1439 return true;
1440 }
1441 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1442 return selectIntrinsicWithSideEffects(I&: MI);
1443 case TargetOpcode::G_INTRINSIC:
1444 return selectIntrinsic(I&: MI);
1445 case TargetOpcode::G_EXTRACT_SUBVECTOR:
1446 return selectExtractSubvector(MI);
1447 case TargetOpcode::G_INSERT_SUBVECTOR:
1448 return selectInsertSubVector(MI);
1449 default:
1450 return false;
1451 }
1452}
1453
1454bool RISCVInstructionSelector::selectUnmergeValues(MachineInstr &MI) const {
1455 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
1456
1457 if (!Subtarget->hasStdExtZfa())
1458 return false;
1459
1460 // Split F64 Src into two s32 parts
1461 if (MI.getNumOperands() != 3)
1462 return false;
1463 Register Src = MI.getOperand(i: 2).getReg();
1464 Register Lo = MI.getOperand(i: 0).getReg();
1465 Register Hi = MI.getOperand(i: 1).getReg();
1466 if (!isRegInFprb(Reg: Src) || !isRegInGprb(Reg: Lo) || !isRegInGprb(Reg: Hi))
1467 return false;
1468
1469 MachineInstr *ExtractLo = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1470 MCID: TII.get(Opcode: RISCV::FMV_X_W_FPR64), DestReg: Lo)
1471 .addReg(RegNo: Src);
1472 constrainSelectedInstRegOperands(I&: *ExtractLo, TII, TRI, RBI);
1473
1474 MachineInstr *ExtractHi = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1475 MCID: TII.get(Opcode: RISCV::FMVH_X_D), DestReg: Hi)
1476 .addReg(RegNo: Src);
1477 constrainSelectedInstRegOperands(I&: *ExtractHi, TII, TRI, RBI);
1478
1479 MI.eraseFromParent();
1480 return true;
1481}
1482
1483bool RISCVInstructionSelector::replacePtrWithInt(MachineOperand &Op) {
1484 Register PtrReg = Op.getReg();
1485 assert(MRI->getType(PtrReg).isPointer() && "Operand is not a pointer!");
1486
1487 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1488 MachineInstr &ParentMI = *Op.getParent();
1489 Register IntReg = MRI->createGenericVirtualRegister(Ty: sXLen);
1490 MRI->setRegBank(Reg: IntReg, RegBank: RBI.getRegBank(ID: RISCV::GPRBRegBankID));
1491 MachineInstr *PtrToInt =
1492 BuildMI(BB&: *ParentMI.getParent(), I&: ParentMI, MIMD: ParentMI.getDebugLoc(),
1493 MCID: TII.get(Opcode: TargetOpcode::G_PTRTOINT), DestReg: IntReg)
1494 .addReg(RegNo: PtrReg);
1495 Op.setReg(IntReg);
1496 return select(MI&: *PtrToInt);
1497}
1498
1499void RISCVInstructionSelector::preISelLower(MachineInstr &MI) {
1500 switch (MI.getOpcode()) {
1501 case TargetOpcode::G_PTR_ADD: {
1502 Register DstReg = MI.getOperand(i: 0).getReg();
1503 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1504
1505 replacePtrWithInt(Op&: MI.getOperand(i: 1));
1506 MI.setDesc(TII.get(Opcode: TargetOpcode::G_ADD));
1507 MRI->setType(VReg: DstReg, Ty: sXLen);
1508 break;
1509 }
1510 case TargetOpcode::G_PTRMASK: {
1511 Register DstReg = MI.getOperand(i: 0).getReg();
1512 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1513 replacePtrWithInt(Op&: MI.getOperand(i: 1));
1514 MI.setDesc(TII.get(Opcode: TargetOpcode::G_AND));
1515 MRI->setType(VReg: DstReg, Ty: sXLen);
1516 break;
1517 }
1518 }
1519}
1520
1521void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
1522 const MachineInstr &MI,
1523 int OpIdx) const {
1524 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1525 "Expected G_CONSTANT");
1526 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
1527 MIB.addImm(Val: -CstVal);
1528}
1529
1530void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
1531 const MachineInstr &MI,
1532 int OpIdx) const {
1533 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1534 "Expected G_CONSTANT");
1535 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1536 MIB.addImm(Val: STI.getXLen() - CstVal);
1537}
1538
1539void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
1540 const MachineInstr &MI,
1541 int OpIdx) const {
1542 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1543 "Expected G_CONSTANT");
1544 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1545 MIB.addImm(Val: 32 - CstVal);
1546}
1547
1548void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
1549 const MachineInstr &MI,
1550 int OpIdx) const {
1551 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1552 "Expected G_CONSTANT");
1553 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
1554 MIB.addImm(Val: CstVal + 1);
1555}
1556
1557void RISCVInstructionSelector::renderFrameIndex(MachineInstrBuilder &MIB,
1558 const MachineInstr &MI,
1559 int OpIdx) const {
1560 assert(MI.getOpcode() == TargetOpcode::G_FRAME_INDEX && OpIdx == -1 &&
1561 "Expected G_FRAME_INDEX");
1562 MIB.add(MO: MI.getOperand(i: 1));
1563}
1564
1565void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
1566 const MachineInstr &MI,
1567 int OpIdx) const {
1568 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1569 "Expected G_CONSTANT");
1570 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
1571 MIB.addImm(Val: llvm::countr_zero(Val: C));
1572}
1573
1574void RISCVInstructionSelector::renderXLenSubTrailingOnes(
1575 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
1576 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1577 "Expected G_CONSTANT");
1578 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
1579 MIB.addImm(Val: Subtarget->getXLen() - llvm::countr_one(Value: C));
1580}
1581
1582void RISCVInstructionSelector::renderAddiPairImmSmall(MachineInstrBuilder &MIB,
1583 const MachineInstr &MI,
1584 int OpIdx) const {
1585 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1586 "Expected G_CONSTANT");
1587 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
1588 int64_t Adj = Imm < 0 ? -2048 : 2047;
1589 MIB.addImm(Val: Imm - Adj);
1590}
1591
1592void RISCVInstructionSelector::renderAddiPairImmLarge(MachineInstrBuilder &MIB,
1593 const MachineInstr &MI,
1594 int OpIdx) const {
1595 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1596 "Expected G_CONSTANT");
1597 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue() < 0 ? -2048 : 2047;
1598 MIB.addImm(Val: Imm);
1599}
1600
1601bool RISCVInstructionSelector::isRegInGprb(Register Reg) const {
1602 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::GPRBRegBankID;
1603}
1604
1605bool RISCVInstructionSelector::isRegInFprb(Register Reg) const {
1606 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID;
1607}
1608
1609bool RISCVInstructionSelector::selectCopy(MachineInstr &MI) const {
1610 MachineOperand Dst = MI.getOperand(i: 0);
1611 Register DstReg = MI.getOperand(i: 0).getReg();
1612
1613 if (DstReg.isPhysical())
1614 return true;
1615
1616 const TargetRegisterClass *DstRC =
1617 TRI.getConstrainedRegClassForOperand(MO: Dst, MRI: *MRI);
1618
1619 assert(DstRC &&
1620 "Register class not available for LLT, register bank combination");
1621
1622 // No need to constrain SrcReg. It will get constrained when
1623 // we hit another of its uses or its defs.
1624 // Copies do not have constraints.
1625 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
1626 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1627 << " operand\n");
1628 return false;
1629 }
1630
1631 MI.setDesc(TII.get(Opcode: RISCV::COPY));
1632 return true;
1633}
1634
1635bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI) const {
1636 assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
1637
1638 const Register DstReg = MI.getOperand(i: 0).getReg();
1639 const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(
1640 Ty: MRI->getType(Reg: DstReg), RB: *RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI), Is64Bit: STI.is64Bit());
1641
1642 assert(DstRC &&
1643 "Register class not available for LLT, register bank combination");
1644
1645 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
1646 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1647 << " operand\n");
1648 }
1649 MI.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
1650 return true;
1651}
1652
1653bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
1654 MachineInstr &MI) const {
1655 MachineBasicBlock &MBB = *MI.getParent();
1656 DebugLoc DL = MI.getDebugLoc();
1657
1658 if (Imm == 0) {
1659 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg).addReg(RegNo: RISCV::X0);
1660 RBI.constrainGenericRegister(Reg: DstReg, RC: RISCV::GPRRegClass, MRI&: *MRI);
1661 return true;
1662 }
1663
1664 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: *Subtarget);
1665 unsigned NumInsts = Seq.size();
1666 Register SrcReg = RISCV::X0;
1667
1668 for (unsigned i = 0; i < NumInsts; i++) {
1669 Register TmpReg = i < NumInsts - 1
1670 ? MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass)
1671 : DstReg;
1672 const RISCVMatInt::Inst &I = Seq[i];
1673 MachineInstr *Result;
1674
1675 switch (I.getOpndKind()) {
1676 case RISCVMatInt::Imm:
1677 // clang-format off
1678 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1679 .addImm(Val: I.getImm());
1680 // clang-format on
1681 break;
1682 case RISCVMatInt::RegX0:
1683 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1684 .addReg(RegNo: SrcReg)
1685 .addReg(RegNo: RISCV::X0);
1686 break;
1687 case RISCVMatInt::RegReg:
1688 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1689 .addReg(RegNo: SrcReg)
1690 .addReg(RegNo: SrcReg);
1691 break;
1692 case RISCVMatInt::RegImm:
1693 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1694 .addReg(RegNo: SrcReg)
1695 .addImm(Val: I.getImm());
1696 break;
1697 }
1698
1699 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1700
1701 SrcReg = TmpReg;
1702 }
1703
1704 return true;
1705}
1706
1707bool RISCVInstructionSelector::selectAddr(MachineInstr &MI, bool IsLocal,
1708 bool IsExternWeak) const {
1709 assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1710 MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
1711 MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
1712 "Unexpected opcode");
1713
1714 const MachineOperand &DispMO = MI.getOperand(i: 1);
1715
1716 Register DefReg = MI.getOperand(i: 0).getReg();
1717 const LLT DefTy = MRI->getType(Reg: DefReg);
1718
1719 // When HWASAN is used and tagging of global variables is enabled
1720 // they should be accessed via the GOT, since the tagged address of a global
1721 // is incompatible with existing code models. This also applies to non-pic
1722 // mode.
1723 if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
1724 if (IsLocal && !Subtarget->allowTaggedGlobals()) {
1725 // Use PC-relative addressing to access the symbol. This generates the
1726 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
1727 // %pcrel_lo(auipc)).
1728 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
1729 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1730 return true;
1731 }
1732
1733 // Use PC-relative addressing to access the GOT for this symbol, then
1734 // load the address from the GOT. This generates the pattern (PseudoLGA
1735 // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
1736 // %pcrel_lo(auipc))).
1737 MachineFunction &MF = *MI.getParent()->getParent();
1738 MachineMemOperand *MemOp = MF.getMachineMemOperand(
1739 PtrInfo: MachinePointerInfo::getGOT(MF),
1740 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1741 MachineMemOperand::MOInvariant,
1742 MemTy: DefTy, base_alignment: Align(DefTy.getSizeInBits() / 8));
1743
1744 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1745 MCID: TII.get(Opcode: RISCV::PseudoLGA), DestReg: DefReg)
1746 .addDisp(Disp: DispMO, off: 0)
1747 .addMemOperand(MMO: MemOp);
1748
1749 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1750
1751 MI.eraseFromParent();
1752 return true;
1753 }
1754
1755 switch (TM.getCodeModel()) {
1756 default: {
1757 reportGISelFailure(MF&: *MF, MORE&: *MORE, PassName: getName(),
1758 Msg: "Unsupported code model for lowering", MI);
1759 return false;
1760 }
1761 case CodeModel::Small: {
1762 // Must lie within a single 2 GiB address range and must lie between
1763 // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
1764 // (lui %hi(sym)) %lo(sym)).
1765 Register AddrHiDest = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1766 MachineInstr *AddrHi = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1767 MCID: TII.get(Opcode: RISCV::LUI), DestReg: AddrHiDest)
1768 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_HI);
1769
1770 constrainSelectedInstRegOperands(I&: *AddrHi, TII, TRI, RBI);
1771
1772 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1773 MCID: TII.get(Opcode: RISCV::ADDI), DestReg: DefReg)
1774 .addReg(RegNo: AddrHiDest)
1775 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_LO);
1776
1777 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1778
1779 MI.eraseFromParent();
1780 return true;
1781 }
1782 case CodeModel::Medium:
1783 // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
1784 // relocation needs to reference a label that points to the auipc
1785 // instruction itself, not the global. This cannot be done inside the
1786 // instruction selector.
1787 if (IsExternWeak) {
1788 // An extern weak symbol may be undefined, i.e. have value 0, which may
1789 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
1790 // symbol. This generates the pattern (PseudoLGA sym), which expands to
1791 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
1792 MachineFunction &MF = *MI.getParent()->getParent();
1793 MachineMemOperand *MemOp = MF.getMachineMemOperand(
1794 PtrInfo: MachinePointerInfo::getGOT(MF),
1795 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1796 MachineMemOperand::MOInvariant,
1797 MemTy: DefTy, base_alignment: Align(DefTy.getSizeInBits() / 8));
1798
1799 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1800 MCID: TII.get(Opcode: RISCV::PseudoLGA), DestReg: DefReg)
1801 .addDisp(Disp: DispMO, off: 0)
1802 .addMemOperand(MMO: MemOp);
1803
1804 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1805
1806 MI.eraseFromParent();
1807 return true;
1808 }
1809
1810 // Generate a sequence for accessing addresses within any 2GiB range
1811 // within the address space. This generates the pattern (PseudoLLA sym),
1812 // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
1813 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
1814 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1815 return true;
1816 }
1817
1818 return false;
1819}
1820
1821bool RISCVInstructionSelector::selectSelect(MachineInstr &MI) const {
1822 auto &SelectMI = cast<GSelect>(Val&: MI);
1823
1824 Register LHS, RHS;
1825 RISCVCC::CondCode CC;
1826 getOperandsForBranch(CondReg: SelectMI.getCondReg(), CC, LHS, RHS, MRI&: *MRI);
1827
1828 Register DstReg = SelectMI.getReg(Idx: 0);
1829
1830 unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
1831 if (RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
1832 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
1833 Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
1834 : RISCV::Select_FPR64_Using_CC_GPR;
1835 }
1836
1837 MachineInstr *Result =
1838 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: Opc))
1839 .addDef(RegNo: DstReg)
1840 .addReg(RegNo: LHS)
1841 .addReg(RegNo: RHS)
1842 .addImm(Val: CC)
1843 .addReg(RegNo: SelectMI.getTrueReg())
1844 .addReg(RegNo: SelectMI.getFalseReg());
1845 MI.eraseFromParent();
1846 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1847 return true;
1848}
1849
1850// Convert an FCMP predicate to one of the supported F or D instructions.
1851static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
1852 assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");
1853 switch (Pred) {
1854 default:
1855 llvm_unreachable("Unsupported predicate");
1856 case CmpInst::FCMP_OLT:
1857 return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
1858 case CmpInst::FCMP_OLE:
1859 return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
1860 case CmpInst::FCMP_OEQ:
1861 return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
1862 }
1863}
1864
1865// Try legalizing an FCMP by swapping or inverting the predicate to one that
1866// is supported.
1867static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,
1868 CmpInst::Predicate &Pred, bool &NeedInvert) {
1869 auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
1870 return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
1871 Pred == CmpInst::FCMP_OEQ;
1872 };
1873
1874 assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
1875
1876 CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(pred: Pred);
1877 if (isLegalFCmpPredicate(InvPred)) {
1878 Pred = InvPred;
1879 std::swap(a&: LHS, b&: RHS);
1880 return true;
1881 }
1882
1883 InvPred = CmpInst::getInversePredicate(pred: Pred);
1884 NeedInvert = true;
1885 if (isLegalFCmpPredicate(InvPred)) {
1886 Pred = InvPred;
1887 return true;
1888 }
1889 InvPred = CmpInst::getSwappedPredicate(pred: InvPred);
1890 if (isLegalFCmpPredicate(InvPred)) {
1891 Pred = InvPred;
1892 std::swap(a&: LHS, b&: RHS);
1893 return true;
1894 }
1895
1896 return false;
1897}
1898
1899// Emit a sequence of instructions to compare LHS and RHS using Pred. Return
1900// the result in DstReg.
1901// FIXME: Maybe we should expand this earlier.
1902bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI) const {
1903 auto &CmpMI = cast<GFCmp>(Val&: MI);
1904 CmpInst::Predicate Pred = CmpMI.getCond();
1905
1906 Register DstReg = CmpMI.getReg(Idx: 0);
1907 Register LHS = CmpMI.getLHSReg();
1908 Register RHS = CmpMI.getRHSReg();
1909
1910 unsigned Size = MRI->getType(Reg: LHS).getSizeInBits();
1911 assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
1912
1913 Register TmpReg = DstReg;
1914
1915 bool NeedInvert = false;
1916 // First try swapping operands or inverting.
1917 if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
1918 if (NeedInvert)
1919 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1920 MachineInstr *Cmp = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1921 MCID: TII.get(Opcode: getFCmpOpcode(Pred, Size)), DestReg: TmpReg)
1922 .addReg(RegNo: LHS)
1923 .addReg(RegNo: RHS);
1924 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
1925 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
1926 // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
1927 NeedInvert = Pred == CmpInst::FCMP_UEQ;
1928 Register Cmp1Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1929 MachineInstr *Cmp1 =
1930 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1931 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size)), DestReg: Cmp1Reg)
1932 .addReg(RegNo: LHS)
1933 .addReg(RegNo: RHS);
1934 constrainSelectedInstRegOperands(I&: *Cmp1, TII, TRI, RBI);
1935 Register Cmp2Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1936 MachineInstr *Cmp2 =
1937 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1938 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size)), DestReg: Cmp2Reg)
1939 .addReg(RegNo: RHS)
1940 .addReg(RegNo: LHS);
1941 constrainSelectedInstRegOperands(I&: *Cmp2, TII, TRI, RBI);
1942 if (NeedInvert)
1943 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1944 MachineInstr *Or = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1945 MCID: TII.get(Opcode: RISCV::OR), DestReg: TmpReg)
1946 .addReg(RegNo: Cmp1Reg)
1947 .addReg(RegNo: Cmp2Reg);
1948 constrainSelectedInstRegOperands(I&: *Or, TII, TRI, RBI);
1949 } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
1950 // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
1951 // If LHS and RHS are the same, a single FEQ suffices.
1952 NeedInvert = Pred == CmpInst::FCMP_UNO;
1953 if (NeedInvert)
1954 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1955 if (LHS == RHS) {
1956 MachineInstr *Cmp =
1957 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1958 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: TmpReg)
1959 .addReg(RegNo: LHS)
1960 .addReg(RegNo: LHS);
1961 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
1962 } else {
1963 Register Cmp1Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1964 MachineInstr *Cmp1 =
1965 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1966 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: Cmp1Reg)
1967 .addReg(RegNo: LHS)
1968 .addReg(RegNo: LHS);
1969 constrainSelectedInstRegOperands(I&: *Cmp1, TII, TRI, RBI);
1970 Register Cmp2Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1971 MachineInstr *Cmp2 =
1972 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1973 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: Cmp2Reg)
1974 .addReg(RegNo: RHS)
1975 .addReg(RegNo: RHS);
1976 constrainSelectedInstRegOperands(I&: *Cmp2, TII, TRI, RBI);
1977 MachineInstr *And = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1978 MCID: TII.get(Opcode: RISCV::AND), DestReg: TmpReg)
1979 .addReg(RegNo: Cmp1Reg)
1980 .addReg(RegNo: Cmp2Reg);
1981 constrainSelectedInstRegOperands(I&: *And, TII, TRI, RBI);
1982 }
1983 } else
1984 llvm_unreachable("Unhandled predicate");
1985
1986 // Emit an XORI to invert the result if needed.
1987 if (NeedInvert) {
1988 MachineInstr *Xor = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1989 MCID: TII.get(Opcode: RISCV::XORI), DestReg: DstReg)
1990 .addReg(RegNo: TmpReg)
1991 .addImm(Val: 1);
1992 constrainSelectedInstRegOperands(I&: *Xor, TII, TRI, RBI);
1993 }
1994
1995 MI.eraseFromParent();
1996 return true;
1997}
1998
1999void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
2000 SyncScope::ID FenceSSID,
2001 MachineInstr &MI) const {
2002 MachineBasicBlock &MBB = *MI.getParent();
2003 DebugLoc DL = MI.getDebugLoc();
2004
2005 if (STI.hasStdExtZtso()) {
2006 // The only fence that needs an instruction is a sequentially-consistent
2007 // cross-thread fence.
2008 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
2009 FenceSSID == SyncScope::System) {
2010 // fence rw, rw
2011 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE))
2012 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W)
2013 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W);
2014 return;
2015 }
2016
2017 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
2018 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER));
2019 return;
2020 }
2021
2022 // singlethread fences only synchronize with signal handlers on the same
2023 // thread and thus only need to preserve instruction order, not actually
2024 // enforce memory ordering.
2025 if (FenceSSID == SyncScope::SingleThread) {
2026 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER));
2027 return;
2028 }
2029
2030 // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
2031 // Manual: Volume I.
2032 unsigned Pred, Succ;
2033 switch (FenceOrdering) {
2034 default:
2035 llvm_unreachable("Unexpected ordering");
2036 case AtomicOrdering::AcquireRelease:
2037 // fence acq_rel -> fence.tso
2038 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE_TSO));
2039 return;
2040 case AtomicOrdering::Acquire:
2041 // fence acquire -> fence r, rw
2042 Pred = RISCVFenceField::R;
2043 Succ = RISCVFenceField::R | RISCVFenceField::W;
2044 break;
2045 case AtomicOrdering::Release:
2046 // fence release -> fence rw, w
2047 Pred = RISCVFenceField::R | RISCVFenceField::W;
2048 Succ = RISCVFenceField::W;
2049 break;
2050 case AtomicOrdering::SequentiallyConsistent:
2051 // fence seq_cst -> fence rw, rw
2052 Pred = RISCVFenceField::R | RISCVFenceField::W;
2053 Succ = RISCVFenceField::R | RISCVFenceField::W;
2054 break;
2055 }
2056 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE)).addImm(Val: Pred).addImm(Val: Succ);
2057}
2058
2059namespace llvm {
2060InstructionSelector *
2061createRISCVInstructionSelector(const RISCVTargetMachine &TM,
2062 const RISCVSubtarget &Subtarget,
2063 const RISCVRegisterBankInfo &RBI) {
2064 return new RISCVInstructionSelector(TM, Subtarget, RBI);
2065}
2066} // end namespace llvm
2067