1//===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// RISC-V.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/RISCVMatInt.h"
15#include "RISCVRegisterBankInfo.h"
16#include "RISCVSubtarget.h"
17#include "RISCVTargetMachine.h"
18#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
19#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
20#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
21#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
22#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/MachineJumpTableInfo.h"
25#include "llvm/IR/IntrinsicsRISCV.h"
26#include "llvm/Support/Debug.h"
27
28#define DEBUG_TYPE "riscv-isel"
29
30using namespace llvm;
31using namespace MIPatternMatch;
32
33#define GET_GLOBALISEL_PREDICATE_BITSET
34#include "RISCVGenGlobalISel.inc"
35#undef GET_GLOBALISEL_PREDICATE_BITSET
36
37namespace {
38
39class RISCVInstructionSelector : public InstructionSelector {
40public:
41 RISCVInstructionSelector(const RISCVTargetMachine &TM,
42 const RISCVSubtarget &STI,
43 const RISCVRegisterBankInfo &RBI);
44
45 bool select(MachineInstr &MI) override;
46
47 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
48 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
49 BlockFrequencyInfo *BFI) override {
50 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
51 MRI = &MF.getRegInfo();
52 }
53
54 static const char *getName() { return DEBUG_TYPE; }
55
56private:
57 static constexpr unsigned MaxRecursionDepth = 6;
58
59 bool hasAllNBitUsers(const MachineInstr &MI, unsigned Bits,
60 const unsigned Depth = 0) const;
61 bool hasAllHUsers(const MachineInstr &MI) const {
62 return hasAllNBitUsers(MI, Bits: 16);
63 }
64 bool hasAllWUsers(const MachineInstr &MI) const {
65 return hasAllNBitUsers(MI, Bits: 32);
66 }
67
68 bool isRegInGprb(Register Reg) const;
69 bool isRegInFprb(Register Reg) const;
70 bool isWorthFoldingAdd(Register AddResult) const;
71
72 // tblgen-erated 'select' implementation, used as the initial selector for
73 // the patterns that don't require complex C++.
74 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
75
76 // A lowering phase that runs before any selection attempts.
77 // Returns true if the instruction was modified.
78 void preISelLower(MachineInstr &MI);
79
80 bool replacePtrWithInt(MachineInstr &MI, unsigned OpIdx);
81
82 // Custom selection methods
83 bool selectCopy(MachineInstr &MI) const;
84 bool selectImplicitDef(MachineInstr &MI) const;
85 bool materializeImm(Register Reg, int64_t Imm, MachineInstr &MI) const;
86 // Emit a constant-materialization instruction sequence.
87 bool materializeInstSeq(Register DstReg, const RISCVMatInt::InstSeq &Seq,
88 MachineInstr &MI) const;
89 bool selectAddr(MachineInstr &MI, bool IsLocal = true,
90 bool IsExternWeak = false) const;
91 bool selectSelect(MachineInstr &MI) const;
92 bool selectFPCompare(MachineInstr &MI) const;
93 void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
94 MachineInstr &MI) const;
95 bool selectUnmergeValues(MachineInstr &MI) const;
96 void addVectorLoadStoreOperands(MachineInstr &I,
97 SmallVectorImpl<Register> &SrcOps,
98 unsigned &CurOp, bool IsMasked,
99 bool IsStridedOrIndexed,
100 LLT *IndexVT = nullptr) const;
101 bool selectIntrinsicWithSideEffects(MachineInstr &I) const;
102 bool selectIntrinsic(MachineInstr &I) const;
103 bool selectExtractSubvector(MachineInstr &MI) const;
104 bool selectInsertSubVector(MachineInstr &I) const;
105 ComplexRendererFns selectShiftMask(MachineOperand &Root,
106 unsigned ShiftWidth) const;
107 ComplexRendererFns selectShiftMaskXLen(MachineOperand &Root) const {
108 return selectShiftMask(Root, ShiftWidth: STI.getXLen());
109 }
110 ComplexRendererFns selectShiftMask32(MachineOperand &Root) const {
111 return selectShiftMask(Root, ShiftWidth: 32);
112 }
113 ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
114 ComplexRendererFns selectBrindRegImm(MachineOperand &Root) const;
115 ComplexRendererFns selectAddrRegImmLsb00000(MachineOperand &Root) const;
116
117 // Plan for materializing a constant address as (Hi materialization, Lo12
118 // offset). Lo12 is a simm12 that, for prefetch (IsPrefetch), must be
119 // a multiple of 32.
120 struct ConstAddrPlan {
121 enum { X0, LUI, InstSeq } Kind = X0;
122 int64_t Hi20 = 0;
123 RISCVMatInt::InstSeq Seq;
124 int64_t Lo12 = 0;
125 };
126 ComplexRendererFns computeConstAddr(int64_t CVal, bool IsPrefetch,
127 Register OrigBase) const;
128 // Materialize the high part of Plan into a register. If OrigBase is valid,
129 // ADD it to the materialized high part (for G_PTR_ADD + large constant).
130 Register materializeConstBase(MachineInstrBuilder &MIB,
131 const ConstAddrPlan &Plan,
132 Register OrigBase) const;
133
134 ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;
135 template <unsigned Bits>
136 ComplexRendererFns selectSExtBits(MachineOperand &Root) const {
137 return selectSExtBits(Root, Bits);
138 }
139
140 ComplexRendererFns selectZExtBits(MachineOperand &Root, unsigned Bits) const;
141 template <unsigned Bits>
142 ComplexRendererFns selectZExtBits(MachineOperand &Root) const {
143 return selectZExtBits(Root, Bits);
144 }
145
146 ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
147 template <unsigned ShAmt>
148 ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
149 return selectSHXADDOp(Root, ShAmt);
150 }
151
152 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
153 unsigned ShAmt) const;
154 template <unsigned ShAmt>
155 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
156 return selectSHXADD_UWOp(Root, ShAmt);
157 }
158
159 ComplexRendererFns renderVLOp(MachineOperand &Root) const;
160 ComplexRendererFns renderAddiPair(Register BaseReg, int64_t AddiImm,
161 int64_t OffsetImm) const;
162 // Custom renderers for tablegen
163 void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
164 int OpIdx) const;
165 void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
166 int OpIdx) const;
167 void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
168 int OpIdx) const;
169 void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
170 int OpIdx) const;
171
172 void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
173 int OpIdx) const;
174 void renderXLenSubTrailingOnes(MachineInstrBuilder &MIB,
175 const MachineInstr &MI, int OpIdx) const;
176
177 void renderAddiPairImmLarge(MachineInstrBuilder &MIB, const MachineInstr &MI,
178 int OpIdx) const;
179 void renderAddiPairImmSmall(MachineInstrBuilder &MIB, const MachineInstr &MI,
180 int OpIdx) const;
181
182 const RISCVSubtarget &STI;
183 const RISCVInstrInfo &TII;
184 const RISCVRegisterInfo &TRI;
185 const RISCVRegisterBankInfo &RBI;
186 const RISCVTargetMachine &TM;
187
188 MachineRegisterInfo *MRI = nullptr;
189
190 // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
191 // uses "STI." in the code generated by TableGen. We need to unify the name of
192 // Subtarget variable.
193 const RISCVSubtarget *Subtarget = &STI;
194
195#define GET_GLOBALISEL_PREDICATES_DECL
196#include "RISCVGenGlobalISel.inc"
197#undef GET_GLOBALISEL_PREDICATES_DECL
198
199#define GET_GLOBALISEL_TEMPORARIES_DECL
200#include "RISCVGenGlobalISel.inc"
201#undef GET_GLOBALISEL_TEMPORARIES_DECL
202};
203
204} // end anonymous namespace
205
206#define GET_GLOBALISEL_IMPL
207#include "RISCVGenGlobalISel.inc"
208#undef GET_GLOBALISEL_IMPL
209
210RISCVInstructionSelector::RISCVInstructionSelector(
211 const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
212 const RISCVRegisterBankInfo &RBI)
213 : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
214 TM(TM),
215
216#define GET_GLOBALISEL_PREDICATES_INIT
217#include "RISCVGenGlobalISel.inc"
218#undef GET_GLOBALISEL_PREDICATES_INIT
219#define GET_GLOBALISEL_TEMPORARIES_INIT
220#include "RISCVGenGlobalISel.inc"
221#undef GET_GLOBALISEL_TEMPORARIES_INIT
222{
223}
224
225// Mimics optimizations in ISel and RISCVOptWInst Pass
226bool RISCVInstructionSelector::hasAllNBitUsers(const MachineInstr &MI,
227 unsigned Bits,
228 const unsigned Depth) const {
229
230 assert((MI.getOpcode() == TargetOpcode::G_ADD ||
231 MI.getOpcode() == TargetOpcode::G_SUB ||
232 MI.getOpcode() == TargetOpcode::G_MUL ||
233 MI.getOpcode() == TargetOpcode::G_SHL ||
234 MI.getOpcode() == TargetOpcode::G_LSHR ||
235 MI.getOpcode() == TargetOpcode::G_AND ||
236 MI.getOpcode() == TargetOpcode::G_OR ||
237 MI.getOpcode() == TargetOpcode::G_XOR ||
238 MI.getOpcode() == TargetOpcode::G_SEXT_INREG || Depth != 0) &&
239 "Unexpected opcode");
240
241 if (Depth >= RISCVInstructionSelector::MaxRecursionDepth)
242 return false;
243
244 auto DestReg = MI.getOperand(i: 0).getReg();
245 for (auto &UserOp : MRI->use_nodbg_operands(Reg: DestReg)) {
246 assert(UserOp.getParent() && "UserOp must have a parent");
247 const MachineInstr &UserMI = *UserOp.getParent();
248 unsigned OpIdx = UserOp.getOperandNo();
249
250 switch (UserMI.getOpcode()) {
251 default:
252 return false;
253 case RISCV::ADDW:
254 case RISCV::ADDIW:
255 case RISCV::SUBW:
256 case RISCV::FCVT_D_W:
257 case RISCV::FCVT_S_W:
258 if (Bits >= 32)
259 break;
260 return false;
261 case RISCV::SLL:
262 case RISCV::SRA:
263 case RISCV::SRL:
264 // Shift amount operands only use log2(Xlen) bits.
265 if (OpIdx == 2 && Bits >= Log2_32(Value: Subtarget->getXLen()))
266 break;
267 return false;
268 case RISCV::SLLI:
269 // SLLI only uses the lower (XLen - ShAmt) bits.
270 if (Bits >= Subtarget->getXLen() - UserMI.getOperand(i: 2).getImm())
271 break;
272 return false;
273 case RISCV::ANDI:
274 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(
275 Value: (uint64_t)UserMI.getOperand(i: 2).getImm()))
276 break;
277 goto RecCheck;
278 case RISCV::AND:
279 case RISCV::OR:
280 case RISCV::XOR:
281 RecCheck:
282 if (hasAllNBitUsers(MI: UserMI, Bits, Depth: Depth + 1))
283 break;
284 return false;
285 case RISCV::SRLI: {
286 unsigned ShAmt = UserMI.getOperand(i: 2).getImm();
287 // If we are shifting right by less than Bits, and users don't demand any
288 // bits that were shifted into [Bits-1:0], then we can consider this as an
289 // N-Bit user.
290 if (Bits > ShAmt && hasAllNBitUsers(MI: UserMI, Bits: Bits - ShAmt, Depth: Depth + 1))
291 break;
292 return false;
293 }
294 }
295 }
296
297 return true;
298}
299
300InstructionSelector::ComplexRendererFns
301RISCVInstructionSelector::selectShiftMask(MachineOperand &Root,
302 unsigned ShiftWidth) const {
303 if (!Root.isReg())
304 return std::nullopt;
305
306 using namespace llvm::MIPatternMatch;
307
308 Register ShAmtReg = Root.getReg();
309 // Peek through zext.
310 Register ZExtSrcReg;
311 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrcReg))))
312 ShAmtReg = ZExtSrcReg;
313
314 APInt AndMask;
315 Register AndSrcReg;
316 // Try to combine the following pattern (applicable to other shift
317 // instructions as well as 32-bit ones):
318 //
319 // %4:gprb(s64) = G_AND %3, %2
320 // %5:gprb(s64) = G_LSHR %1, %4(s64)
321 //
322 // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
323 // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
324 // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
325 // then it can be eliminated. Given register rs1 or rs2 holding a constant
326 // (the and mask), there are two cases G_AND can be erased:
327 //
328 // 1. the lowest log2(XLEN) bits of the and mask are all set
329 // 2. the bits of the register being masked are already unset (zero set)
330 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: AndSrcReg), R: m_ICst(Cst&: AndMask)))) {
331 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
332 if (ShMask.isSubsetOf(RHS: AndMask)) {
333 ShAmtReg = AndSrcReg;
334 } else {
335 // SimplifyDemandedBits may have optimized the mask so try restoring any
336 // bits that are known zero.
337 KnownBits Known = VT->getKnownBits(R: AndSrcReg);
338 if (ShMask.isSubsetOf(RHS: AndMask | Known.Zero))
339 ShAmtReg = AndSrcReg;
340 }
341 }
342
343 APInt Imm;
344 Register Reg;
345 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAdd(L: m_Reg(R&: Reg), R: m_ICst(Cst&: Imm)))) {
346 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0)
347 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
348 // to avoid the ADD.
349 ShAmtReg = Reg;
350 } else if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GSub(L: m_ICst(Cst&: Imm), R: m_Reg(R&: Reg)))) {
351 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0) {
352 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
353 // to generate a NEG instead of a SUB of a constant.
354 ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
355 unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
356 return {{[=](MachineInstrBuilder &MIB) {
357 MachineIRBuilder(*MIB.getInstr())
358 .buildInstr(Opc: NegOpc, DstOps: {ShAmtReg}, SrcOps: {Register(RISCV::X0), Reg});
359 MIB.addReg(RegNo: ShAmtReg);
360 }}};
361 }
362 if (Imm.urem(RHS: ShiftWidth) == ShiftWidth - 1) {
363 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
364 // to generate a NOT instead of a SUB of a constant.
365 ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
366 return {{[=](MachineInstrBuilder &MIB) {
367 MachineIRBuilder(*MIB.getInstr())
368 .buildInstr(Opc: RISCV::XORI, DstOps: {ShAmtReg}, SrcOps: {Reg})
369 .addImm(Val: -1);
370 MIB.addReg(RegNo: ShAmtReg);
371 }}};
372 }
373 }
374
375 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ShAmtReg); }}};
376}
377
378InstructionSelector::ComplexRendererFns
379RISCVInstructionSelector::selectSExtBits(MachineOperand &Root,
380 unsigned Bits) const {
381 if (!Root.isReg())
382 return std::nullopt;
383 Register RootReg = Root.getReg();
384
385 Register SrcReg;
386 if (mi_match(R: RootReg, MRI: *MRI,
387 P: m_GSExtInReg(Src: m_Reg(R&: SrcReg), Imm: m_SpecificImm(RequestedValue: Bits)))) {
388 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SrcReg); }}};
389 }
390
391 unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits();
392 if ((Size - VT->computeNumSignBits(R: RootReg)) < Bits)
393 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}};
394
395 return std::nullopt;
396}
397
398InstructionSelector::ComplexRendererFns
399RISCVInstructionSelector::selectZExtBits(MachineOperand &Root,
400 unsigned Bits) const {
401 if (!Root.isReg())
402 return std::nullopt;
403 Register RootReg = Root.getReg();
404
405 Register RegX;
406 uint64_t Mask = maskTrailingOnes<uint64_t>(N: Bits);
407 if (mi_match(R: RootReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: RegX), R: m_SpecificICst(RequestedValue: Mask)))) {
408 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}};
409 }
410
411 if (mi_match(R: RootReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: RegX))) &&
412 MRI->getType(Reg: RegX).getScalarSizeInBits() == Bits)
413 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}};
414
415 unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits();
416 if (VT->maskedValueIsZero(Val: RootReg, Mask: APInt::getBitsSetFrom(numBits: Size, loBit: Bits)))
417 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}};
418
419 return std::nullopt;
420}
421
422InstructionSelector::ComplexRendererFns
423RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
424 unsigned ShAmt) const {
425 using namespace llvm::MIPatternMatch;
426
427 if (!Root.isReg())
428 return std::nullopt;
429 Register RootReg = Root.getReg();
430
431 const unsigned XLen = STI.getXLen();
432 APInt Mask, C2;
433 Register RegY;
434 std::optional<bool> LeftShift;
435 // (and (shl y, c2), mask)
436 if (mi_match(R: RootReg, MRI: *MRI,
437 P: m_GAnd(L: m_GShl(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
438 LeftShift = true;
439 // (and (lshr y, c2), mask)
440 else if (mi_match(R: RootReg, MRI: *MRI,
441 P: m_GAnd(L: m_GLShr(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
442 LeftShift = false;
443
444 if (LeftShift.has_value()) {
445 if (*LeftShift)
446 Mask &= maskTrailingZeros<uint64_t>(N: C2.getZExtValue());
447 else
448 Mask &= maskTrailingOnes<uint64_t>(N: XLen - C2.getZExtValue());
449
450 if (Mask.isShiftedMask()) {
451 unsigned Leading = XLen - Mask.getActiveBits();
452 unsigned Trailing = Mask.countr_zero();
453 // Given (and (shl y, c2), mask) in which mask has no leading zeros and
454 // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
455 if (*LeftShift && Leading == 0 && C2.ult(RHS: Trailing) && Trailing == ShAmt) {
456 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
457 return {{[=](MachineInstrBuilder &MIB) {
458 MachineIRBuilder(*MIB.getInstr())
459 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
460 .addImm(Val: Trailing - C2.getZExtValue());
461 MIB.addReg(RegNo: DstReg);
462 }}};
463 }
464
465 // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
466 // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
467 if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
468 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
469 return {{[=](MachineInstrBuilder &MIB) {
470 MachineIRBuilder(*MIB.getInstr())
471 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
472 .addImm(Val: Leading + Trailing);
473 MIB.addReg(RegNo: DstReg);
474 }}};
475 }
476 }
477 }
478
479 LeftShift.reset();
480
481 // (shl (and y, mask), c2)
482 if (mi_match(R: RootReg, MRI: *MRI,
483 P: m_GShl(L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
484 R: m_ICst(Cst&: C2))))
485 LeftShift = true;
486 // (lshr (and y, mask), c2)
487 else if (mi_match(R: RootReg, MRI: *MRI,
488 P: m_GLShr(L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
489 R: m_ICst(Cst&: C2))))
490 LeftShift = false;
491
492 if (LeftShift.has_value() && Mask.isShiftedMask()) {
493 unsigned Leading = XLen - Mask.getActiveBits();
494 unsigned Trailing = Mask.countr_zero();
495
496 // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
497 // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
498 bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&
499 (Trailing + C2.getZExtValue()) == ShAmt;
500 if (!Cond)
501 // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
502 // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
503 Cond = !*LeftShift && Leading == 32 && C2.ult(RHS: Trailing) &&
504 (Trailing - C2.getZExtValue()) == ShAmt;
505
506 if (Cond) {
507 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
508 return {{[=](MachineInstrBuilder &MIB) {
509 MachineIRBuilder(*MIB.getInstr())
510 .buildInstr(Opc: RISCV::SRLIW, DstOps: {DstReg}, SrcOps: {RegY})
511 .addImm(Val: Trailing);
512 MIB.addReg(RegNo: DstReg);
513 }}};
514 }
515 }
516
517 return std::nullopt;
518}
519
520InstructionSelector::ComplexRendererFns
521RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
522 unsigned ShAmt) const {
523 using namespace llvm::MIPatternMatch;
524
525 if (!Root.isReg())
526 return std::nullopt;
527 Register RootReg = Root.getReg();
528
529 // Given (and (shl x, c2), mask) in which mask is a shifted mask with
530 // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
531 // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
532 APInt Mask, C2;
533 Register RegX;
534 if (mi_match(
535 R: RootReg, MRI: *MRI,
536 P: m_OneNonDBGUse(SP: m_GAnd(L: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: RegX), R: m_ICst(Cst&: C2))),
537 R: m_ICst(Cst&: Mask))))) {
538 Mask &= maskTrailingZeros<uint64_t>(N: C2.getZExtValue());
539
540 if (Mask.isShiftedMask()) {
541 unsigned Leading = Mask.countl_zero();
542 unsigned Trailing = Mask.countr_zero();
543 if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
544 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
545 return {{[=](MachineInstrBuilder &MIB) {
546 MachineIRBuilder(*MIB.getInstr())
547 .buildInstr(Opc: RISCV::SLLI, DstOps: {DstReg}, SrcOps: {RegX})
548 .addImm(Val: C2.getZExtValue() - ShAmt);
549 MIB.addReg(RegNo: DstReg);
550 }}};
551 }
552 }
553 }
554
555 return std::nullopt;
556}
557
558InstructionSelector::ComplexRendererFns
559RISCVInstructionSelector::renderVLOp(MachineOperand &Root) const {
560 assert(Root.isReg() && "Expected operand to be a Register");
561 std::optional<ValueAndVReg> C;
562 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GCst(ValReg&: C))) {
563 if (C->Value.isAllOnes())
564 // If the operand is a G_CONSTANT with value of all ones it is larger than
565 // VLMAX. We convert it to an immediate with value VLMaxSentinel. This is
566 // recognized specially by the vsetvli insertion pass.
567 return {{[=](MachineInstrBuilder &MIB) {
568 MIB.addImm(Val: RISCV::VLMaxSentinel);
569 }}};
570
571 if (isUInt<5>(x: C->Value.getZExtValue())) {
572 uint64_t ZExtC = C->Value.getZExtValue();
573 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ZExtC); }}};
574 }
575 }
576 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Root.getReg()); }}};
577}
578
579InstructionSelector::ComplexRendererFns
580RISCVInstructionSelector::renderAddiPair(Register BaseReg, int64_t AddiImm,
581 int64_t OffsetImm) const {
582 return {{[=](MachineInstrBuilder &MIB) {
583 Register Tmp = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
584 MachineInstr *Addi =
585 BuildMI(BB&: *MIB->getParent(), I&: *MIB.getInstr(), MIMD: MIB->getDebugLoc(),
586 MCID: TII.get(Opcode: RISCV::ADDI), DestReg: Tmp)
587 .addReg(RegNo: BaseReg)
588 .addImm(Val: AddiImm);
589 constrainSelectedInstRegOperands(I&: *Addi, TII, TRI, RBI);
590 MIB.addReg(RegNo: Tmp);
591 },
592 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: OffsetImm); }}};
593}
594
595InstructionSelector::ComplexRendererFns
596RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
597 if (!Root.isReg())
598 return std::nullopt;
599
600 Register RootReg = Root.getReg();
601
602 // Frame index.
603 int FI;
604 if (mi_match(R: RootReg, MRI: *MRI, P: m_GFrameIndex(FI))) {
605 return {{
606 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); },
607 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
608 }};
609 }
610
611 // base + constant offset (G_PTR_ADD).
612 Register BaseReg;
613 int64_t RHSC;
614 if (mi_match(R: RootReg, MRI: *MRI, P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_ICst(Cst&: RHSC)))) {
615 if (isInt<12>(x: RHSC)) {
616 int BaseFI;
617 if (mi_match(R: BaseReg, MRI: *MRI, P: m_GFrameIndex(FI&: BaseFI)))
618 return {{
619 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: BaseFI); },
620 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
621 }};
622
623 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: BaseReg); },
624 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
625 }
626
627 // Large constant offset. Fold a -2048/2047 adjustment so the whole
628 // constant can be split across an ADDI and the load/store offset.
629 if (RHSC >= -4096 && RHSC <= 4094) {
630 int64_t Adj = RHSC < 0 ? -2048 : 2047;
631 return renderAddiPair(BaseReg, AddiImm: Adj, OffsetImm: RHSC - Adj);
632 }
633
634 if (isWorthFoldingAdd(AddResult: RootReg))
635 if (auto Fns = computeConstAddr(CVal: RHSC, /*IsPrefetch=*/false, OrigBase: BaseReg))
636 return Fns;
637 }
638
639 // Bare constant address. IRTranslator lowers inttoptr(C) to
640 // G_INTTOPTR(G_CONSTANT); look through it to reach the constant.
641 int64_t CVal;
642 if (mi_match(R: RootReg, MRI: *MRI, P: m_GIntToPtr(Src: m_ICst(Cst&: CVal))) ||
643 mi_match(R: RootReg, MRI: *MRI, P: m_ICst(Cst&: CVal))) {
644 if (auto Fns = computeConstAddr(CVal, /*IsPrefetch=*/false, OrigBase: Register()))
645 return Fns;
646 }
647
648 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); },
649 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
650}
651
652InstructionSelector::ComplexRendererFns
653RISCVInstructionSelector::selectBrindRegImm(MachineOperand &Root) const {
654 if (!Root.isReg())
655 return std::nullopt;
656
657 Register RootReg = Root.getReg();
658
659 // base + constant offset (G_PTR_ADD). Doesn't match a FrameIndex, unlike
660 // selectAddrRegImm, since the callers of this (e.g. indirect branches)
661 // can't take a FrameIndex or global address operand.
662 Register BaseReg;
663 int64_t RHSC;
664 if (mi_match(R: RootReg, MRI: *MRI, P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_ICst(Cst&: RHSC))) &&
665 isInt<12>(x: RHSC)) {
666 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: BaseReg); },
667 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
668 }
669
670 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); },
671 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
672}
673
674InstructionSelector::ComplexRendererFns
675RISCVInstructionSelector::selectAddrRegImmLsb00000(MachineOperand &Root) const {
676 if (!Root.isReg())
677 return std::nullopt;
678
679 Register RootReg = Root.getReg();
680
681 // Frame index.
682 int FI;
683 if (mi_match(R: RootReg, MRI: *MRI, P: m_GFrameIndex(FI))) {
684 return {{
685 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); },
686 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
687 }};
688 }
689
690 // base + constant offset (G_PTR_ADD).
691 Register BaseReg;
692 int64_t RHSC;
693 if (mi_match(R: RootReg, MRI: *MRI, P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_ICst(Cst&: RHSC)))) {
694 if (isInt<12>(x: RHSC)) {
695 // Not a multiple of 32: can't encode, use the address as-is.
696 if ((RHSC & 0b11111) != 0) {
697 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); },
698 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
699 }
700 // Fold the offset.
701 int BaseFI;
702 if (mi_match(R: BaseReg, MRI: *MRI, P: m_GFrameIndex(FI&: BaseFI)))
703 return {{
704 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: BaseFI); },
705 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
706 }};
707 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: BaseReg); },
708 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
709 }
710
711 // Large constant: fold a -2048/2016 adjustment to save an instruction.
712 if ((-2049 >= RHSC && RHSC >= -4096) || (4063 >= RHSC && RHSC >= 2017)) {
713 int64_t Adj = RHSC < 0 ? -2048 : 2016;
714 return renderAddiPair(BaseReg, AddiImm: RHSC - Adj, OffsetImm: Adj);
715 }
716
717 // Otherwise split the constant into Hi (materialized + added to the base)
718 // and Lo12 (folded offset).
719 if (auto Fns = computeConstAddr(CVal: RHSC, /*IsPrefetch=*/true, OrigBase: BaseReg))
720 return Fns;
721 }
722
723 // Bare constant address. IRTranslator emits inttoptr(C) as
724 // G_INTTOPTR(G_CONSTANT); look through the G_INTTOPTR to reach the constant.
725 int64_t CVal;
726 if (mi_match(R: RootReg, MRI: *MRI, P: m_GIntToPtr(Src: m_ICst(Cst&: CVal))) ||
727 mi_match(R: RootReg, MRI: *MRI, P: m_ICst(Cst&: CVal))) {
728 if (auto Fns = computeConstAddr(CVal, /*IsPrefetch=*/true, OrigBase: Register()))
729 return Fns;
730 }
731
732 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); },
733 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
734}
735
736/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
737/// CC Must be an ICMP Predicate.
738static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
739 switch (CC) {
740 default:
741 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
742 case CmpInst::Predicate::ICMP_EQ:
743 return RISCVCC::COND_EQ;
744 case CmpInst::Predicate::ICMP_NE:
745 return RISCVCC::COND_NE;
746 case CmpInst::Predicate::ICMP_ULT:
747 return RISCVCC::COND_LTU;
748 case CmpInst::Predicate::ICMP_SLT:
749 return RISCVCC::COND_LT;
750 case CmpInst::Predicate::ICMP_UGE:
751 return RISCVCC::COND_GEU;
752 case CmpInst::Predicate::ICMP_SGE:
753 return RISCVCC::COND_GE;
754 }
755}
756
757static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC,
758 Register &LHS, Register &RHS,
759 MachineRegisterInfo &MRI) {
760 // Try to fold an ICmp. If that fails, use a NE compare with X0.
761 CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
762 if (!mi_match(R: CondReg, MRI, P: m_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: LHS), R: m_Reg(R&: RHS)))) {
763 LHS = CondReg;
764 RHS = RISCV::X0;
765 CC = RISCVCC::COND_NE;
766 return;
767 }
768
769 // We found an ICmp, do some canonicalization.
770
771 // Adjust comparisons to use comparison with 0 if possible.
772 if (auto Constant = getIConstantVRegSExtVal(VReg: RHS, MRI)) {
773 switch (Pred) {
774 case CmpInst::Predicate::ICMP_SGT:
775 // Convert X > -1 to X >= 0
776 if (*Constant == -1) {
777 CC = RISCVCC::COND_GE;
778 RHS = RISCV::X0;
779 return;
780 }
781 break;
782 case CmpInst::Predicate::ICMP_SLT:
783 // Convert X < 1 to 0 >= X
784 if (*Constant == 1) {
785 CC = RISCVCC::COND_GE;
786 RHS = LHS;
787 LHS = RISCV::X0;
788 return;
789 }
790 break;
791 default:
792 break;
793 }
794 }
795
796 switch (Pred) {
797 default:
798 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
799 case CmpInst::Predicate::ICMP_EQ:
800 case CmpInst::Predicate::ICMP_NE:
801 case CmpInst::Predicate::ICMP_ULT:
802 case CmpInst::Predicate::ICMP_SLT:
803 case CmpInst::Predicate::ICMP_UGE:
804 case CmpInst::Predicate::ICMP_SGE:
805 // These CCs are supported directly by RISC-V branches.
806 break;
807 case CmpInst::Predicate::ICMP_SGT:
808 case CmpInst::Predicate::ICMP_SLE:
809 case CmpInst::Predicate::ICMP_UGT:
810 case CmpInst::Predicate::ICMP_ULE:
811 // These CCs are not supported directly by RISC-V branches, but changing the
812 // direction of the CC and swapping LHS and RHS are.
813 Pred = CmpInst::getSwappedPredicate(pred: Pred);
814 std::swap(a&: LHS, b&: RHS);
815 break;
816 }
817
818 CC = getRISCVCCFromICmp(CC: Pred);
819}
820
821/// Select the RISC-V Zalasr opcode for the G_LOAD or G_STORE operation
822/// \p GenericOpc, appropriate for the GPR register bank and of memory access
823/// size \p OpSize.
824static unsigned selectZalasrLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {
825 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
826 switch (OpSize) {
827 default:
828 llvm_unreachable("Unexpected memory size");
829 case 8:
830 return IsStore ? RISCV::SB_RL : RISCV::LB_AQ;
831 case 16:
832 return IsStore ? RISCV::SH_RL : RISCV::LH_AQ;
833 case 32:
834 return IsStore ? RISCV::SW_RL : RISCV::LW_AQ;
835 case 64:
836 return IsStore ? RISCV::SD_RL : RISCV::LD_AQ;
837 }
838}
839
840/// Select the RISC-V regimm opcode for the G_LOAD or G_STORE operation
841/// \p GenericOpc, appropriate for the GPR register bank and of memory access
842/// size \p OpSize. \returns \p GenericOpc if the combination is unsupported.
843static unsigned selectRegImmLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {
844 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
845 switch (OpSize) {
846 case 8:
847 // Prefer unsigned due to no c.lb in Zcb.
848 return IsStore ? RISCV::SB : RISCV::LBU;
849 case 16:
850 return IsStore ? RISCV::SH : RISCV::LH;
851 case 32:
852 return IsStore ? RISCV::SW : RISCV::LW;
853 case 64:
854 return IsStore ? RISCV::SD : RISCV::LD;
855 }
856
857 return GenericOpc;
858}
859
860void RISCVInstructionSelector::addVectorLoadStoreOperands(
861 MachineInstr &I, SmallVectorImpl<Register> &SrcOps, unsigned &CurOp,
862 bool IsMasked, bool IsStridedOrIndexed, LLT *IndexVT) const {
863 // Base Pointer
864 auto PtrReg = I.getOperand(i: CurOp++).getReg();
865 SrcOps.push_back(Elt: PtrReg);
866
867 // Stride or Index
868 if (IsStridedOrIndexed) {
869 auto StrideReg = I.getOperand(i: CurOp++).getReg();
870 SrcOps.push_back(Elt: StrideReg);
871 if (IndexVT)
872 *IndexVT = MRI->getType(Reg: StrideReg);
873 }
874
875 // Mask
876 if (IsMasked) {
877 auto MaskReg = I.getOperand(i: CurOp++).getReg();
878 SrcOps.push_back(Elt: MaskReg);
879 }
880}
881
882bool RISCVInstructionSelector::selectIntrinsicWithSideEffects(
883 MachineInstr &I) const {
884 // Find the intrinsic ID.
885 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
886 // Select the instruction.
887 switch (IntrinID) {
888 default:
889 return false;
890 case Intrinsic::riscv_vlm:
891 case Intrinsic::riscv_vle:
892 case Intrinsic::riscv_vle_mask:
893 case Intrinsic::riscv_vlse:
894 case Intrinsic::riscv_vlse_mask: {
895 bool IsMasked = IntrinID == Intrinsic::riscv_vle_mask ||
896 IntrinID == Intrinsic::riscv_vlse_mask;
897 bool IsStrided = IntrinID == Intrinsic::riscv_vlse ||
898 IntrinID == Intrinsic::riscv_vlse_mask;
899 LLT VT = MRI->getType(Reg: I.getOperand(i: 0).getReg());
900 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
901
902 // Result vector
903 const Register DstReg = I.getOperand(i: 0).getReg();
904
905 // Sources
906 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
907 unsigned CurOp = 2;
908 SmallVector<Register, 4> SrcOps; // Source registers.
909
910 // Passthru
911 if (HasPassthruOperand) {
912 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
913 SrcOps.push_back(Elt: PassthruReg);
914 } else {
915 SrcOps.push_back(Elt: Register(RISCV::NoRegister));
916 }
917
918 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: IsStrided);
919
920 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
921 const RISCV::VLEPseudo *P =
922 RISCV::getVLEPseudo(Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
923 LMUL: static_cast<unsigned>(LMUL));
924
925 MachineInstrBuilder PseudoMI =
926 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo), DestReg: DstReg);
927 for (Register Reg : SrcOps)
928 PseudoMI.addReg(RegNo: Reg);
929
930 // Select VL
931 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
932 for (auto &RenderFn : *VLOpFn)
933 RenderFn(PseudoMI);
934
935 // SEW
936 PseudoMI.addImm(Val: Log2SEW);
937
938 // Policy
939 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
940 if (IsMasked)
941 Policy = I.getOperand(i: CurOp++).getImm();
942 PseudoMI.addImm(Val: Policy);
943
944 // Memref
945 PseudoMI.cloneMemRefs(OtherMI: I);
946
947 I.eraseFromParent();
948 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
949 return true;
950 }
951 case Intrinsic::riscv_vloxei:
952 case Intrinsic::riscv_vloxei_mask:
953 case Intrinsic::riscv_vluxei:
954 case Intrinsic::riscv_vluxei_mask: {
955 bool IsMasked = IntrinID == Intrinsic::riscv_vloxei_mask ||
956 IntrinID == Intrinsic::riscv_vluxei_mask;
957 bool IsOrdered = IntrinID == Intrinsic::riscv_vloxei ||
958 IntrinID == Intrinsic::riscv_vloxei_mask;
959 LLT VT = MRI->getType(Reg: I.getOperand(i: 0).getReg());
960 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
961
962 // Result vector
963 const Register DstReg = I.getOperand(i: 0).getReg();
964
965 // Sources
966 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
967 unsigned CurOp = 2;
968 SmallVector<Register, 4> SrcOps; // Source registers.
969
970 // Passthru
971 if (HasPassthruOperand) {
972 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
973 SrcOps.push_back(Elt: PassthruReg);
974 } else {
975 // Use NoRegister if there is no specified passthru.
976 SrcOps.push_back(Elt: Register());
977 }
978 LLT IndexVT;
979 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: true, IndexVT: &IndexVT);
980
981 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
982 RISCVVType::VLMUL IndexLMUL =
983 RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: IndexVT));
984 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
985 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
986 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
987 "values when XLEN=32");
988 }
989 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
990 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
991 IndexLMUL: static_cast<unsigned>(IndexLMUL));
992
993 MachineInstrBuilder PseudoMI =
994 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo), DestReg: DstReg);
995 for (Register Reg : SrcOps)
996 PseudoMI.addReg(RegNo: Reg);
997
998 // Select VL
999 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
1000 for (auto &RenderFn : *VLOpFn)
1001 RenderFn(PseudoMI);
1002
1003 // SEW
1004 PseudoMI.addImm(Val: Log2SEW);
1005
1006 // Policy
1007 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
1008 if (IsMasked)
1009 Policy = I.getOperand(i: CurOp++).getImm();
1010 PseudoMI.addImm(Val: Policy);
1011
1012 // Memref
1013 PseudoMI.cloneMemRefs(OtherMI: I);
1014
1015 I.eraseFromParent();
1016 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1017 return true;
1018 }
1019 case Intrinsic::riscv_vsm:
1020 case Intrinsic::riscv_vse:
1021 case Intrinsic::riscv_vse_mask:
1022 case Intrinsic::riscv_vsse:
1023 case Intrinsic::riscv_vsse_mask: {
1024 bool IsMasked = IntrinID == Intrinsic::riscv_vse_mask ||
1025 IntrinID == Intrinsic::riscv_vsse_mask;
1026 bool IsStrided = IntrinID == Intrinsic::riscv_vsse ||
1027 IntrinID == Intrinsic::riscv_vsse_mask;
1028 LLT VT = MRI->getType(Reg: I.getOperand(i: 1).getReg());
1029 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
1030
1031 // Sources
1032 unsigned CurOp = 1;
1033 SmallVector<Register, 4> SrcOps; // Source registers.
1034
1035 // Store value
1036 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
1037 SrcOps.push_back(Elt: PassthruReg);
1038
1039 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: IsStrided);
1040
1041 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
1042 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
1043 Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
1044
1045 MachineInstrBuilder PseudoMI =
1046 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo));
1047 for (Register Reg : SrcOps)
1048 PseudoMI.addReg(RegNo: Reg);
1049
1050 // Select VL
1051 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
1052 for (auto &RenderFn : *VLOpFn)
1053 RenderFn(PseudoMI);
1054
1055 // SEW
1056 PseudoMI.addImm(Val: Log2SEW);
1057
1058 // Memref
1059 PseudoMI.cloneMemRefs(OtherMI: I);
1060
1061 I.eraseFromParent();
1062 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1063 return true;
1064 }
1065 case Intrinsic::riscv_vsoxei:
1066 case Intrinsic::riscv_vsoxei_mask:
1067 case Intrinsic::riscv_vsuxei:
1068 case Intrinsic::riscv_vsuxei_mask: {
1069 bool IsMasked = IntrinID == Intrinsic::riscv_vsoxei_mask ||
1070 IntrinID == Intrinsic::riscv_vsuxei_mask;
1071 bool IsOrdered = IntrinID == Intrinsic::riscv_vsoxei ||
1072 IntrinID == Intrinsic::riscv_vsoxei_mask;
1073 LLT VT = MRI->getType(Reg: I.getOperand(i: 1).getReg());
1074 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
1075
1076 // Sources
1077 unsigned CurOp = 1;
1078 SmallVector<Register, 4> SrcOps; // Source registers.
1079
1080 // Store value
1081 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
1082 SrcOps.push_back(Elt: PassthruReg);
1083
1084 LLT IndexVT;
1085 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: true, IndexVT: &IndexVT);
1086
1087 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
1088 RISCVVType::VLMUL IndexLMUL =
1089 RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: IndexVT));
1090 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
1091 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
1092 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
1093 "values when XLEN=32");
1094 }
1095 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
1096 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
1097 IndexLMUL: static_cast<unsigned>(IndexLMUL));
1098
1099 MachineInstrBuilder PseudoMI =
1100 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo));
1101 for (Register Reg : SrcOps)
1102 PseudoMI.addReg(RegNo: Reg);
1103
1104 // Select VL
1105 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
1106 for (auto &RenderFn : *VLOpFn)
1107 RenderFn(PseudoMI);
1108
1109 // SEW
1110 PseudoMI.addImm(Val: Log2SEW);
1111
1112 // Memref
1113 PseudoMI.cloneMemRefs(OtherMI: I);
1114
1115 I.eraseFromParent();
1116 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1117 return true;
1118 }
1119 }
1120}
1121
1122bool RISCVInstructionSelector::selectIntrinsic(MachineInstr &I) const {
1123 // Find the intrinsic ID.
1124 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
1125 // Select the instruction.
1126 switch (IntrinID) {
1127 default:
1128 return false;
1129 case Intrinsic::riscv_vsetvli:
1130 case Intrinsic::riscv_vsetvlimax: {
1131
1132 bool VLMax = IntrinID == Intrinsic::riscv_vsetvlimax;
1133
1134 unsigned Offset = VLMax ? 2 : 3;
1135 unsigned SEW = RISCVVType::decodeVSEW(VSEW: I.getOperand(i: Offset).getImm() & 0x7);
1136 RISCVVType::VLMUL VLMul =
1137 static_cast<RISCVVType::VLMUL>(I.getOperand(i: Offset + 1).getImm() & 0x7);
1138
1139 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMUL: VLMul, SEW, /*TailAgnostic*/ true,
1140 /*MaskAgnostic*/ true);
1141
1142 Register DstReg = I.getOperand(i: 0).getReg();
1143
1144 Register VLOperand;
1145 unsigned Opcode = RISCV::PseudoVSETVLI;
1146
1147 // Check if AVL is a constant that equals VLMAX.
1148 if (!VLMax) {
1149 Register AVLReg = I.getOperand(i: 2).getReg();
1150 if (auto AVLConst = getIConstantVRegValWithLookThrough(VReg: AVLReg, MRI: *MRI)) {
1151 uint64_t AVL = AVLConst->Value.getZExtValue();
1152 if (auto VLEN = Subtarget->getRealVLen()) {
1153 if (*VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == AVL)
1154 VLMax = true;
1155 }
1156 }
1157
1158 if (mi_match(R: AVLReg, MRI: *MRI, P: m_AllOnes()))
1159 VLMax = true;
1160 }
1161
1162 if (VLMax) {
1163 VLOperand = Register(RISCV::X0);
1164 Opcode = RISCV::PseudoVSETVLIX0;
1165 } else {
1166 Register AVLReg = I.getOperand(i: 2).getReg();
1167 VLOperand = AVLReg;
1168
1169 // Check if AVL is a small constant that can use PseudoVSETIVLI.
1170 if (auto AVLConst = getIConstantVRegValWithLookThrough(VReg: AVLReg, MRI: *MRI)) {
1171 uint64_t AVL = AVLConst->Value.getZExtValue();
1172 if (isUInt<5>(x: AVL)) {
1173 MachineInstr *PseudoMI =
1174 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1175 MCID: TII.get(Opcode: RISCV::PseudoVSETIVLI), DestReg: DstReg)
1176 .addImm(Val: AVL)
1177 .addImm(Val: VTypeI);
1178 I.eraseFromParent();
1179 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1180 return true;
1181 }
1182 }
1183 }
1184
1185 MachineInstr *PseudoMI =
1186 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode), DestReg: DstReg)
1187 .addReg(RegNo: VLOperand)
1188 .addImm(Val: VTypeI);
1189 I.eraseFromParent();
1190 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1191 return true;
1192 }
1193 }
1194}
1195
1196bool RISCVInstructionSelector::selectExtractSubvector(MachineInstr &MI) const {
1197 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_SUBVECTOR);
1198
1199 Register DstReg = MI.getOperand(i: 0).getReg();
1200 Register SrcReg = MI.getOperand(i: 1).getReg();
1201
1202 LLT DstTy = MRI->getType(Reg: DstReg);
1203 LLT SrcTy = MRI->getType(Reg: SrcReg);
1204
1205 unsigned Idx = static_cast<unsigned>(MI.getOperand(i: 2).getImm());
1206
1207 MVT DstMVT = getMVTForLLT(Ty: DstTy);
1208 MVT SrcMVT = getMVTForLLT(Ty: SrcTy);
1209
1210 unsigned SubRegIdx;
1211 std::tie(args&: SubRegIdx, args&: Idx) =
1212 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1213 VecVT: SrcMVT, SubVecVT: DstMVT, InsertExtractIdx: Idx, TRI: &TRI);
1214
1215 if (Idx != 0)
1216 return false;
1217
1218 unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: DstMVT);
1219 const TargetRegisterClass *DstRC = TRI.getRegClass(i: DstRegClassID);
1220 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1221 return false;
1222
1223 unsigned SrcRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: SrcMVT);
1224 const TargetRegisterClass *SrcRC = TRI.getRegClass(i: SrcRegClassID);
1225 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
1226 return false;
1227
1228 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1229 DestReg: DstReg)
1230 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubRegIdx);
1231
1232 MI.eraseFromParent();
1233 return true;
1234}
1235
1236bool RISCVInstructionSelector::selectInsertSubVector(MachineInstr &MI) const {
1237 assert(MI.getOpcode() == TargetOpcode::G_INSERT_SUBVECTOR);
1238
1239 Register DstReg = MI.getOperand(i: 0).getReg();
1240 Register VecReg = MI.getOperand(i: 1).getReg();
1241 Register SubVecReg = MI.getOperand(i: 2).getReg();
1242
1243 LLT VecTy = MRI->getType(Reg: VecReg);
1244 LLT SubVecTy = MRI->getType(Reg: SubVecReg);
1245
1246 MVT VecMVT = getMVTForLLT(Ty: VecTy);
1247 MVT SubVecMVT = getMVTForLLT(Ty: SubVecTy);
1248
1249 unsigned Idx = static_cast<unsigned>(MI.getOperand(i: 3).getImm());
1250
1251 unsigned SubRegIdx;
1252 std::tie(args&: SubRegIdx, args&: Idx) =
1253 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1254 VecVT: VecMVT, SubVecVT: SubVecMVT, InsertExtractIdx: Idx, TRI: &TRI);
1255
1256 // If the Idx hasn't been completely eliminated then this is a subvector
1257 // insert which doesn't naturally align to a vector register. These must
1258 // be handled using instructions to manipulate the vector registers.
1259 if (Idx != 0)
1260 return false;
1261
1262 // Constrain dst
1263 unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: VecMVT);
1264 const TargetRegisterClass *DstRC = TRI.getRegClass(i: DstRegClassID);
1265 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1266 return false;
1267
1268 // If we haven't set a SubRegIdx, then we must be going between
1269 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
1270 if (SubRegIdx == RISCV::NoSubRegister) {
1271 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecMVT) ==
1272 DstRegClassID &&
1273 "Unexpected subvector insert");
1274 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1275 DestReg: DstReg)
1276 .addReg(RegNo: SubVecReg);
1277 MI.eraseFromParent();
1278 return true;
1279 }
1280
1281 // Use INSERT_SUBREG to insert the subvector into the vector at the
1282 // appropriate subregister index.
1283 MachineInstr *Ins = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1284 MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DstReg)
1285 .addReg(RegNo: VecReg)
1286 .addReg(RegNo: SubVecReg)
1287 .addImm(Val: SubRegIdx);
1288
1289 MI.eraseFromParent();
1290 constrainSelectedInstRegOperands(I&: *Ins, TII, TRI, RBI);
1291 return true;
1292}
1293
1294bool RISCVInstructionSelector::select(MachineInstr &MI) {
1295 preISelLower(MI);
1296 const unsigned Opc = MI.getOpcode();
1297
1298 if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
1299 if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
1300 const Register DefReg = MI.getOperand(i: 0).getReg();
1301 const LLT DefTy = MRI->getType(Reg: DefReg);
1302
1303 const RegClassOrRegBank &RegClassOrBank =
1304 MRI->getRegClassOrRegBank(Reg: DefReg);
1305
1306 const TargetRegisterClass *DefRC =
1307 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
1308 if (!DefRC) {
1309 if (!DefTy.isValid()) {
1310 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
1311 return false;
1312 }
1313
1314 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
1315 DefRC = TRI.getRegClassForTypeOnBank(Ty: DefTy, RB, Is64Bit: STI.is64Bit());
1316 if (!DefRC) {
1317 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
1318 return false;
1319 }
1320 }
1321
1322 MI.setDesc(TII.get(Opcode: TargetOpcode::PHI));
1323 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI&: *MRI);
1324 }
1325
1326 // Certain non-generic instructions also need some special handling.
1327 if (MI.isCopy())
1328 return selectCopy(MI);
1329
1330 return true;
1331 }
1332
1333 if (selectImpl(I&: MI, CoverageInfo&: *CoverageInfo))
1334 return true;
1335
1336 switch (Opc) {
1337 case TargetOpcode::G_ANYEXT:
1338 case TargetOpcode::G_PTRTOINT:
1339 case TargetOpcode::G_INTTOPTR:
1340 case TargetOpcode::G_TRUNC:
1341 case TargetOpcode::G_FREEZE:
1342 return selectCopy(MI);
1343 case TargetOpcode::G_CONSTANT: {
1344 Register DstReg = MI.getOperand(i: 0).getReg();
1345 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
1346
1347 if (!materializeImm(Reg: DstReg, Imm, MI))
1348 return false;
1349
1350 MI.eraseFromParent();
1351 return true;
1352 }
1353 case TargetOpcode::G_ZEXT:
1354 case TargetOpcode::G_SEXT: {
1355 bool IsSigned = Opc != TargetOpcode::G_ZEXT;
1356 Register DstReg = MI.getOperand(i: 0).getReg();
1357 Register SrcReg = MI.getOperand(i: 1).getReg();
1358 LLT SrcTy = MRI->getType(Reg: SrcReg);
1359 unsigned SrcSize = SrcTy.getSizeInBits();
1360
1361 if (SrcTy.isVector())
1362 return false; // Should be handled by imported patterns.
1363
1364 assert((*RBI.getRegBank(DstReg, *MRI, TRI)).getID() ==
1365 RISCV::GPRBRegBankID &&
1366 "Unexpected ext regbank");
1367
1368 // Use addiw SrcReg, 0 (sext.w) for i32.
1369 if (IsSigned && SrcSize == 32) {
1370 MI.setDesc(TII.get(Opcode: RISCV::ADDIW));
1371 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
1372 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1373 return true;
1374 }
1375
1376 // Use add.uw SrcReg, X0 (zext.w) for i32 with Zba.
1377 if (!IsSigned && SrcSize == 32 && STI.hasStdExtZba()) {
1378 MI.setDesc(TII.get(Opcode: RISCV::ADD_UW));
1379 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::X0, /*isDef=*/false));
1380 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1381 return true;
1382 }
1383
1384 // Use sext.h/zext.h for i16 with Zbb.
1385 if (SrcSize == 16 &&
1386 (STI.hasStdExtZbb() || (!IsSigned && STI.hasStdExtZbkb()))) {
1387 MI.setDesc(TII.get(Opcode: IsSigned ? RISCV::SEXT_H
1388 : STI.is64Bit() ? RISCV::ZEXT_H_RV64
1389 : RISCV::ZEXT_H_RV32));
1390 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1391 return true;
1392 }
1393
1394 // Fall back to shift pair.
1395 Register ShiftLeftReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1396 MachineInstr *ShiftLeft = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1397 MCID: TII.get(Opcode: RISCV::SLLI), DestReg: ShiftLeftReg)
1398 .addReg(RegNo: SrcReg)
1399 .addImm(Val: STI.getXLen() - SrcSize);
1400 constrainSelectedInstRegOperands(I&: *ShiftLeft, TII, TRI, RBI);
1401 MachineInstr *ShiftRight =
1402 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1403 MCID: TII.get(Opcode: IsSigned ? RISCV::SRAI : RISCV::SRLI), DestReg: DstReg)
1404 .addReg(RegNo: ShiftLeftReg)
1405 .addImm(Val: STI.getXLen() - SrcSize);
1406 constrainSelectedInstRegOperands(I&: *ShiftRight, TII, TRI, RBI);
1407 MI.eraseFromParent();
1408 return true;
1409 }
1410 case TargetOpcode::G_FCONSTANT: {
1411 // TODO: Use constant pool for complex constants.
1412 Register DstReg = MI.getOperand(i: 0).getReg();
1413 const APFloat &FPimm = MI.getOperand(i: 1).getFPImm()->getValueAPF();
1414 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
1415 if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
1416 Register GPRReg;
1417 if (FPimm.isPosZero()) {
1418 GPRReg = RISCV::X0;
1419 } else {
1420 GPRReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1421 APInt Imm = FPimm.bitcastToAPInt();
1422 if (!materializeImm(Reg: GPRReg, Imm: Imm.getSExtValue(), MI))
1423 return false;
1424 }
1425
1426 unsigned Opcode = Size == 64 ? RISCV::FMV_D_X
1427 : Size == 32 ? RISCV::FMV_W_X
1428 : RISCV::FMV_H_X;
1429 MachineInstr *FMV = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1430 MCID: TII.get(Opcode), DestReg: DstReg)
1431 .addReg(RegNo: GPRReg);
1432 constrainSelectedInstRegOperands(I&: *FMV, TII, TRI, RBI);
1433 } else {
1434 // s64 on rv32
1435 assert(Size == 64 && !Subtarget->is64Bit() &&
1436 "Unexpected size or subtarget");
1437
1438 if (FPimm.isPosZero()) {
1439 // Optimize +0.0 to use fcvt.d.w
1440 MachineInstr *FCVT = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1441 MCID: TII.get(Opcode: RISCV::FCVT_D_W), DestReg: DstReg)
1442 .addReg(RegNo: RISCV::X0)
1443 .addImm(Val: RISCVFPRndMode::RNE);
1444 constrainSelectedInstRegOperands(I&: *FCVT, TII, TRI, RBI);
1445
1446 MI.eraseFromParent();
1447 return true;
1448 }
1449
1450 // Split into two pieces and build through the stack.
1451 Register GPRRegHigh = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1452 Register GPRRegLow = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1453 APInt Imm = FPimm.bitcastToAPInt();
1454 if (!materializeImm(Reg: GPRRegHigh, Imm: Imm.extractBits(numBits: 32, bitPosition: 32).getSExtValue(),
1455 MI))
1456 return false;
1457 if (!materializeImm(Reg: GPRRegLow, Imm: Imm.trunc(width: 32).getSExtValue(), MI))
1458 return false;
1459 MachineInstr *PairF64 =
1460 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1461 MCID: TII.get(Opcode: RISCV::BuildPairF64Pseudo), DestReg: DstReg)
1462 .addReg(RegNo: GPRRegLow)
1463 .addReg(RegNo: GPRRegHigh);
1464 constrainSelectedInstRegOperands(I&: *PairF64, TII, TRI, RBI);
1465 }
1466
1467 MI.eraseFromParent();
1468 return true;
1469 }
1470 case TargetOpcode::G_GLOBAL_VALUE: {
1471 auto *GV = MI.getOperand(i: 1).getGlobal();
1472 if (GV->isThreadLocal()) {
1473 // TODO: implement this case.
1474 return false;
1475 }
1476
1477 return selectAddr(MI, IsLocal: GV->isDSOLocal(), IsExternWeak: GV->hasExternalWeakLinkage());
1478 }
1479 case TargetOpcode::G_JUMP_TABLE:
1480 case TargetOpcode::G_CONSTANT_POOL:
1481 return selectAddr(MI);
1482 case TargetOpcode::G_BRCOND: {
1483 Register LHS, RHS;
1484 RISCVCC::CondCode CC;
1485 getOperandsForBranch(CondReg: MI.getOperand(i: 0).getReg(), CC, LHS, RHS, MRI&: *MRI);
1486
1487 MachineInstr *Bcc = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1488 MCID: TII.get(Opcode: RISCVCC::getBrCond(CC)))
1489 .addReg(RegNo: LHS)
1490 .addReg(RegNo: RHS)
1491 .addMBB(MBB: MI.getOperand(i: 1).getMBB());
1492 MI.eraseFromParent();
1493 constrainSelectedInstRegOperands(I&: *Bcc, TII, TRI, RBI);
1494 return true;
1495 }
1496 case TargetOpcode::G_SELECT:
1497 return selectSelect(MI);
1498 case TargetOpcode::G_FCMP:
1499 return selectFPCompare(MI);
1500 case TargetOpcode::G_FENCE: {
1501 AtomicOrdering FenceOrdering =
1502 static_cast<AtomicOrdering>(MI.getOperand(i: 0).getImm());
1503 SyncScope::ID FenceSSID =
1504 static_cast<SyncScope::ID>(MI.getOperand(i: 1).getImm());
1505 emitFence(FenceOrdering, FenceSSID, MI);
1506 MI.eraseFromParent();
1507 return true;
1508 }
1509 case TargetOpcode::G_IMPLICIT_DEF:
1510 return selectImplicitDef(MI);
1511 case TargetOpcode::G_UNMERGE_VALUES:
1512 return selectUnmergeValues(MI);
1513 case TargetOpcode::G_LOAD:
1514 case TargetOpcode::G_STORE: {
1515 GLoadStore &LdSt = cast<GLoadStore>(Val&: MI);
1516 const Register ValReg = LdSt.getReg(Idx: 0);
1517 const Register PtrReg = LdSt.getPointerReg();
1518 LLT PtrTy = MRI->getType(Reg: PtrReg);
1519
1520 const RegisterBank &RB = *RBI.getRegBank(Reg: ValReg, MRI: *MRI, TRI);
1521 if (RB.getID() != RISCV::GPRBRegBankID)
1522 return false;
1523
1524#ifndef NDEBUG
1525 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, *MRI, TRI);
1526 // Check that the pointer register is valid.
1527 assert(PtrRB.getID() == RISCV::GPRBRegBankID &&
1528 "Load/Store pointer operand isn't a GPR");
1529 assert(PtrTy.isPointer() && "Load/Store pointer operand isn't a pointer");
1530#endif
1531
1532 // Can only handle AddressSpace 0.
1533 if (PtrTy.getAddressSpace() != 0)
1534 return false;
1535
1536 unsigned MemSize = LdSt.getMemSizeInBits().getValue();
1537 AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();
1538
1539 if (isStrongerThanMonotonic(AO: Order)) {
1540 MI.setDesc(TII.get(Opcode: selectZalasrLoadStoreOp(GenericOpc: Opc, OpSize: MemSize)));
1541 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1542 return true;
1543 }
1544
1545 const unsigned NewOpc = selectRegImmLoadStoreOp(GenericOpc: MI.getOpcode(), OpSize: MemSize);
1546 if (NewOpc == MI.getOpcode())
1547 return false;
1548
1549 // Check if we can fold anything into the addressing mode.
1550 auto AddrModeFns = selectAddrRegImm(Root&: MI.getOperand(i: 1));
1551 if (!AddrModeFns)
1552 return false;
1553
1554 // Folded something. Create a new instruction and return it.
1555 MachineInstrBuilder NewInst =
1556 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: NewOpc));
1557 NewInst.setMIFlags(MI.getFlags());
1558 if (isa<GStore>(Val: MI))
1559 NewInst.addUse(RegNo: ValReg);
1560 else
1561 NewInst.addDef(RegNo: ValReg);
1562 NewInst.cloneMemRefs(OtherMI: MI);
1563 for (auto &Fn : *AddrModeFns)
1564 Fn(NewInst);
1565 MI.eraseFromParent();
1566
1567 constrainSelectedInstRegOperands(I&: *NewInst, TII, TRI, RBI);
1568 return true;
1569 }
1570 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1571 return selectIntrinsicWithSideEffects(I&: MI);
1572 case TargetOpcode::G_INTRINSIC:
1573 return selectIntrinsic(I&: MI);
1574 case TargetOpcode::G_EXTRACT_SUBVECTOR:
1575 return selectExtractSubvector(MI);
1576 case TargetOpcode::G_INSERT_SUBVECTOR:
1577 return selectInsertSubVector(MI);
1578 default:
1579 return false;
1580 }
1581}
1582
1583bool RISCVInstructionSelector::selectUnmergeValues(MachineInstr &MI) const {
1584 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
1585
1586 if (!Subtarget->hasStdExtZfa())
1587 return false;
1588
1589 // Split F64 Src into two s32 parts
1590 if (MI.getNumOperands() != 3)
1591 return false;
1592 Register Src = MI.getOperand(i: 2).getReg();
1593 Register Lo = MI.getOperand(i: 0).getReg();
1594 Register Hi = MI.getOperand(i: 1).getReg();
1595 if (!isRegInFprb(Reg: Src) || !isRegInGprb(Reg: Lo) || !isRegInGprb(Reg: Hi))
1596 return false;
1597
1598 MachineInstr *ExtractLo = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1599 MCID: TII.get(Opcode: RISCV::FMV_X_W_FPR64), DestReg: Lo)
1600 .addReg(RegNo: Src);
1601 constrainSelectedInstRegOperands(I&: *ExtractLo, TII, TRI, RBI);
1602
1603 MachineInstr *ExtractHi = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1604 MCID: TII.get(Opcode: RISCV::FMVH_X_D), DestReg: Hi)
1605 .addReg(RegNo: Src);
1606 constrainSelectedInstRegOperands(I&: *ExtractHi, TII, TRI, RBI);
1607
1608 MI.eraseFromParent();
1609 return true;
1610}
1611
1612bool RISCVInstructionSelector::replacePtrWithInt(MachineInstr &MI,
1613 unsigned OpIdx) {
1614 MachineOperand &Op = MI.getOperand(i: OpIdx);
1615 Register PtrReg = Op.getReg();
1616 assert(MRI->getType(PtrReg).isPointer() && "Operand is not a pointer!");
1617
1618 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1619 Register IntReg = MRI->createGenericVirtualRegister(Ty: sXLen);
1620 MRI->setRegBank(Reg: IntReg, RegBank: RBI.getRegBank(ID: RISCV::GPRBRegBankID));
1621 MachineInstr *PtrToInt = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1622 MCID: TII.get(Opcode: TargetOpcode::G_PTRTOINT), DestReg: IntReg)
1623 .addReg(RegNo: PtrReg);
1624 Op.setReg(IntReg);
1625 return select(MI&: *PtrToInt);
1626}
1627
1628void RISCVInstructionSelector::preISelLower(MachineInstr &MI) {
1629 switch (MI.getOpcode()) {
1630 case TargetOpcode::G_PTR_ADD: {
1631 Register DstReg = MI.getOperand(i: 0).getReg();
1632 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1633
1634 replacePtrWithInt(MI, OpIdx: 1);
1635 MI.setDesc(TII.get(Opcode: TargetOpcode::G_ADD));
1636 MRI->setType(VReg: DstReg, Ty: sXLen);
1637 break;
1638 }
1639 case TargetOpcode::G_PTRMASK: {
1640 Register DstReg = MI.getOperand(i: 0).getReg();
1641 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1642 replacePtrWithInt(MI, OpIdx: 1);
1643 MI.setDesc(TII.get(Opcode: TargetOpcode::G_AND));
1644 MRI->setType(VReg: DstReg, Ty: sXLen);
1645 break;
1646 }
1647 }
1648}
1649
1650void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
1651 const MachineInstr &MI,
1652 int OpIdx) const {
1653 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1654 "Expected G_CONSTANT");
1655 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
1656 MIB.addImm(Val: -CstVal);
1657}
1658
1659void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
1660 const MachineInstr &MI,
1661 int OpIdx) const {
1662 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1663 "Expected G_CONSTANT");
1664 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1665 MIB.addImm(Val: STI.getXLen() - CstVal);
1666}
1667
1668void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
1669 const MachineInstr &MI,
1670 int OpIdx) const {
1671 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1672 "Expected G_CONSTANT");
1673 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1674 MIB.addImm(Val: 32 - CstVal);
1675}
1676
1677void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
1678 const MachineInstr &MI,
1679 int OpIdx) const {
1680 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1681 "Expected G_CONSTANT");
1682 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
1683 MIB.addImm(Val: CstVal + 1);
1684}
1685
1686void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
1687 const MachineInstr &MI,
1688 int OpIdx) const {
1689 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1690 "Expected G_CONSTANT");
1691 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
1692 MIB.addImm(Val: llvm::countr_zero(Val: C));
1693}
1694
1695void RISCVInstructionSelector::renderXLenSubTrailingOnes(
1696 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
1697 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1698 "Expected G_CONSTANT");
1699 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
1700 MIB.addImm(Val: Subtarget->getXLen() - llvm::countr_one(Value: C));
1701}
1702
1703void RISCVInstructionSelector::renderAddiPairImmSmall(MachineInstrBuilder &MIB,
1704 const MachineInstr &MI,
1705 int OpIdx) const {
1706 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1707 "Expected G_CONSTANT");
1708 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
1709 int64_t Adj = Imm < 0 ? -2048 : 2047;
1710 MIB.addImm(Val: Imm - Adj);
1711}
1712
1713void RISCVInstructionSelector::renderAddiPairImmLarge(MachineInstrBuilder &MIB,
1714 const MachineInstr &MI,
1715 int OpIdx) const {
1716 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1717 "Expected G_CONSTANT");
1718 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue() < 0 ? -2048 : 2047;
1719 MIB.addImm(Val: Imm);
1720}
1721
1722bool RISCVInstructionSelector::isRegInGprb(Register Reg) const {
1723 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::GPRBRegBankID;
1724}
1725
1726bool RISCVInstructionSelector::isRegInFprb(Register Reg) const {
1727 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID;
1728}
1729
1730// A G_PTR_ADD result is worth splitting into Hi (materialized) +
1731// Lo12 (folded offset) only if every user is a plain scalar load/store
1732// using it as the address. Otherwise the ADD is selected on its own with
1733// the full materialized constant, making the Hi materialization here redundant.
1734bool RISCVInstructionSelector::isWorthFoldingAdd(Register AddResult) const {
1735 for (const MachineInstr &User : MRI->use_nodbg_instructions(Reg: AddResult)) {
1736 auto *LdSt = dyn_cast<GLoadStore>(Val: &User);
1737 if (!LdSt)
1738 return false;
1739 // Must be used as the pointer, not the stored value.
1740 if (LdSt->getPointerReg() != AddResult)
1741 return false;
1742 if (isStrongerThanMonotonic(AO: LdSt->getMMO().getSuccessOrdering()))
1743 return false;
1744 // Only scalar integer/f16/f32/f64 memory (exclude vectors, f128, ...).
1745 LLT Ty = MRI->getType(Reg: User.getOperand(i: 0).getReg());
1746 if (!Ty.isScalar() || Ty.getSizeInBits() > 64)
1747 return false;
1748 }
1749 return true;
1750}
1751
1752bool RISCVInstructionSelector::selectCopy(MachineInstr &MI) const {
1753 Register DstReg = MI.getOperand(i: 0).getReg();
1754
1755 if (DstReg.isPhysical())
1756 return true;
1757
1758 const TargetRegisterClass *DstRC =
1759 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
1760
1761 assert(DstRC &&
1762 "Register class not available for LLT, register bank combination");
1763
1764 // No need to constrain SrcReg. It will get constrained when
1765 // we hit another of its uses or its defs.
1766 // Copies do not have constraints.
1767 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
1768 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1769 << " operand\n");
1770 return false;
1771 }
1772
1773 MI.setDesc(TII.get(Opcode: RISCV::COPY));
1774 return true;
1775}
1776
1777bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI) const {
1778 assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
1779
1780 const Register DstReg = MI.getOperand(i: 0).getReg();
1781 const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(
1782 Ty: MRI->getType(Reg: DstReg), RB: *RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI), Is64Bit: STI.is64Bit());
1783
1784 assert(DstRC &&
1785 "Register class not available for LLT, register bank combination");
1786
1787 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
1788 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1789 << " operand\n");
1790 }
1791 MI.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
1792 return true;
1793}
1794
1795bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
1796 MachineInstr &MI) const {
1797 if (Imm == 0) {
1798 MachineBasicBlock &MBB = *MI.getParent();
1799 DebugLoc DL = MI.getDebugLoc();
1800 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg).addReg(RegNo: RISCV::X0);
1801 RBI.constrainGenericRegister(Reg: DstReg, RC: RISCV::GPRRegClass, MRI&: *MRI);
1802 return true;
1803 }
1804
1805 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: *Subtarget);
1806 return materializeInstSeq(DstReg, Seq, MI);
1807}
1808
1809bool RISCVInstructionSelector::materializeInstSeq(
1810 Register DstReg, const RISCVMatInt::InstSeq &Seq, MachineInstr &MI) const {
1811 assert(!Seq.empty() && "materializeInstSeq requires a non-empty sequence");
1812
1813 MachineBasicBlock &MBB = *MI.getParent();
1814 DebugLoc DL = MI.getDebugLoc();
1815 unsigned NumInsts = Seq.size();
1816 Register SrcReg = RISCV::X0;
1817
1818 for (unsigned i = 0; i < NumInsts; i++) {
1819 Register TmpReg = i < NumInsts - 1
1820 ? MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass)
1821 : DstReg;
1822 const RISCVMatInt::Inst &I = Seq[i];
1823 MachineInstr *Result;
1824
1825 switch (I.getOpndKind()) {
1826 case RISCVMatInt::Imm:
1827 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1828 .addImm(Val: I.getImm());
1829 break;
1830 case RISCVMatInt::RegX0:
1831 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1832 .addReg(RegNo: SrcReg)
1833 .addReg(RegNo: RISCV::X0);
1834 break;
1835 case RISCVMatInt::RegReg:
1836 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1837 .addReg(RegNo: SrcReg)
1838 .addReg(RegNo: SrcReg);
1839 break;
1840 case RISCVMatInt::RegImm:
1841 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1842 .addReg(RegNo: SrcReg)
1843 .addImm(Val: I.getImm());
1844 break;
1845 }
1846
1847 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1848
1849 SrcReg = TmpReg;
1850 }
1851
1852 return true;
1853}
1854
1855InstructionSelector::ComplexRendererFns
1856RISCVInstructionSelector::computeConstAddr(int64_t CVal, bool IsPrefetch,
1857 Register OrigBase) const {
1858 // Split the constant into a materialized high part (the base) and
1859 // a simm12 low part (the offset). For prefetch the low part
1860 // must additionally be a multiple of 32 (simm12_lsb00000).
1861 int64_t Lo12 = SignExtend64<12>(x: CVal);
1862 int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
1863 auto emit = [&](ConstAddrPlan Plan) -> ComplexRendererFns {
1864 return {{[=](MachineInstrBuilder &MIB) {
1865 MIB.addReg(RegNo: materializeConstBase(MIB, Plan, OrigBase));
1866 },
1867 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Plan.Lo12); }}};
1868 };
1869 if (!Subtarget->is64Bit() || isInt<32>(x: Hi)) {
1870 if (IsPrefetch && (Lo12 & 0b11111) != 0)
1871 return std::nullopt;
1872 ConstAddrPlan Plan;
1873 Plan.Lo12 = Lo12;
1874 if (Hi) {
1875 Plan.Kind = ConstAddrPlan::LUI;
1876 Plan.Hi20 = (Hi >> 12) & 0xfffff;
1877 }
1878 return emit(std::move(Plan));
1879 }
1880
1881 // Otherwise ask constant materialization how it would handle the constant
1882 // and fold the trailing ADDI into the offset.
1883 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: CVal, STI: *Subtarget);
1884 if (Seq.back().getOpcode() != RISCV::ADDI)
1885 return std::nullopt;
1886 Lo12 = Seq.back().getImm();
1887 if (IsPrefetch && (Lo12 & 0b11111) != 0)
1888 return std::nullopt;
1889 Seq.pop_back();
1890 if (Seq.empty())
1891 return std::nullopt;
1892 ConstAddrPlan Plan;
1893 Plan.Kind = ConstAddrPlan::InstSeq;
1894 Plan.Seq = std::move(Seq);
1895 Plan.Lo12 = Lo12;
1896 return emit(std::move(Plan));
1897}
1898
1899Register
1900RISCVInstructionSelector::materializeConstBase(MachineInstrBuilder &MIB,
1901 const ConstAddrPlan &Plan,
1902 Register OrigBase) const {
1903 MachineBasicBlock &MBB = *MIB->getParent();
1904 DebugLoc DL = MIB->getDebugLoc();
1905 MachineInstr &InsertPt = *MIB.getInstr();
1906
1907 Register HiReg = RISCV::X0;
1908 switch (Plan.Kind) {
1909 case ConstAddrPlan::X0:
1910 break;
1911 case ConstAddrPlan::LUI: {
1912 HiReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1913 MachineInstr *LUI = BuildMI(BB&: MBB, I&: InsertPt, MIMD: DL, MCID: TII.get(Opcode: RISCV::LUI), DestReg: HiReg)
1914 .addImm(Val: Plan.Hi20);
1915 constrainSelectedInstRegOperands(I&: *LUI, TII, TRI, RBI);
1916 break;
1917 }
1918 case ConstAddrPlan::InstSeq: {
1919 HiReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1920 materializeInstSeq(DstReg: HiReg, Seq: Plan.Seq, MI&: InsertPt);
1921 break;
1922 }
1923 }
1924
1925 // For G_PTR_ADD + large constant, add the original base to the materialized
1926 // high part.
1927 if (OrigBase.isValid() && HiReg != RISCV::X0) {
1928 Register BaseReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1929 MachineInstr *Add = BuildMI(BB&: MBB, I&: InsertPt, MIMD: DL, MCID: TII.get(Opcode: RISCV::ADD), DestReg: BaseReg)
1930 .addReg(RegNo: OrigBase)
1931 .addReg(RegNo: HiReg);
1932 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
1933 return BaseReg;
1934 }
1935 return OrigBase.isValid() ? OrigBase : HiReg;
1936}
1937
1938bool RISCVInstructionSelector::selectAddr(MachineInstr &MI, bool IsLocal,
1939 bool IsExternWeak) const {
1940 assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1941 MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
1942 MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
1943 "Unexpected opcode");
1944
1945 const MachineOperand &DispMO = MI.getOperand(i: 1);
1946
1947 Register DefReg = MI.getOperand(i: 0).getReg();
1948 const LLT DefTy = MRI->getType(Reg: DefReg);
1949
1950 // When HWASAN is used and tagging of global variables is enabled
1951 // they should be accessed via the GOT, since the tagged address of a global
1952 // is incompatible with existing code models. This also applies to non-pic
1953 // mode.
1954 if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
1955 if (IsLocal && !Subtarget->allowTaggedGlobals()) {
1956 // Use PC-relative addressing to access the symbol. This generates the
1957 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
1958 // %pcrel_lo(auipc)).
1959 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
1960 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1961 return true;
1962 }
1963
1964 // Use PC-relative addressing to access the GOT for this symbol, then
1965 // load the address from the GOT. This generates the pattern (PseudoLGA
1966 // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
1967 // %pcrel_lo(auipc))).
1968 MachineFunction &MF = *MI.getParent()->getParent();
1969 MachineMemOperand *MemOp = MF.getMachineMemOperand(
1970 PtrInfo: MachinePointerInfo::getGOT(MF),
1971 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1972 MachineMemOperand::MOInvariant,
1973 MemTy: DefTy, BaseAlignment: Align(DefTy.getSizeInBits() / 8));
1974
1975 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1976 MCID: TII.get(Opcode: RISCV::PseudoLGA), DestReg: DefReg)
1977 .addDisp(Disp: DispMO, off: 0)
1978 .addMemOperand(MMO: MemOp);
1979
1980 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1981
1982 MI.eraseFromParent();
1983 return true;
1984 }
1985
1986 switch (TM.getCodeModel()) {
1987 default: {
1988 reportGISelFailure(MF&: *MF, MORE&: *MORE, PassName: getName(),
1989 Msg: "Unsupported code model for lowering", MI);
1990 return false;
1991 }
1992 case CodeModel::Small: {
1993 // Must lie within a single 2 GiB address range and must lie between
1994 // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
1995 // (lui %hi(sym)) %lo(sym)).
1996 Register AddrHiDest = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1997 MachineInstr *AddrHi = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1998 MCID: TII.get(Opcode: RISCV::LUI), DestReg: AddrHiDest)
1999 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_HI);
2000
2001 constrainSelectedInstRegOperands(I&: *AddrHi, TII, TRI, RBI);
2002
2003 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2004 MCID: TII.get(Opcode: RISCV::ADDI), DestReg: DefReg)
2005 .addReg(RegNo: AddrHiDest)
2006 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_LO);
2007
2008 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
2009
2010 MI.eraseFromParent();
2011 return true;
2012 }
2013 case CodeModel::Medium:
2014 // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
2015 // relocation needs to reference a label that points to the auipc
2016 // instruction itself, not the global. This cannot be done inside the
2017 // instruction selector.
2018 if (IsExternWeak) {
2019 // An extern weak symbol may be undefined, i.e. have value 0, which may
2020 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
2021 // symbol. This generates the pattern (PseudoLGA sym), which expands to
2022 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
2023 MachineFunction &MF = *MI.getParent()->getParent();
2024 MachineMemOperand *MemOp = MF.getMachineMemOperand(
2025 PtrInfo: MachinePointerInfo::getGOT(MF),
2026 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
2027 MachineMemOperand::MOInvariant,
2028 MemTy: DefTy, BaseAlignment: Align(DefTy.getSizeInBits() / 8));
2029
2030 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2031 MCID: TII.get(Opcode: RISCV::PseudoLGA), DestReg: DefReg)
2032 .addDisp(Disp: DispMO, off: 0)
2033 .addMemOperand(MMO: MemOp);
2034
2035 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
2036
2037 MI.eraseFromParent();
2038 return true;
2039 }
2040
2041 // Generate a sequence for accessing addresses within any 2GiB range
2042 // within the address space. This generates the pattern (PseudoLLA sym),
2043 // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
2044 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
2045 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
2046 return true;
2047 }
2048
2049 return false;
2050}
2051
2052bool RISCVInstructionSelector::selectSelect(MachineInstr &MI) const {
2053 auto &SelectMI = cast<GSelect>(Val&: MI);
2054
2055 Register LHS, RHS;
2056 RISCVCC::CondCode CC;
2057 getOperandsForBranch(CondReg: SelectMI.getCondReg(), CC, LHS, RHS, MRI&: *MRI);
2058
2059 Register DstReg = SelectMI.getReg(Idx: 0);
2060
2061 unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
2062 if (RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
2063 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
2064 Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
2065 : RISCV::Select_FPR64_Using_CC_GPR;
2066 }
2067
2068 MachineInstr *Result =
2069 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: Opc))
2070 .addDef(RegNo: DstReg)
2071 .addReg(RegNo: LHS)
2072 .addReg(RegNo: RHS)
2073 .addImm(Val: CC)
2074 .addReg(RegNo: SelectMI.getTrueReg())
2075 .addReg(RegNo: SelectMI.getFalseReg());
2076 MI.eraseFromParent();
2077 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
2078 return true;
2079}
2080
2081// Convert an FCMP predicate to one of the supported F or D instructions.
2082static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
2083 assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");
2084 switch (Pred) {
2085 default:
2086 llvm_unreachable("Unsupported predicate");
2087 case CmpInst::FCMP_OLT:
2088 return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
2089 case CmpInst::FCMP_OLE:
2090 return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
2091 case CmpInst::FCMP_OEQ:
2092 return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
2093 }
2094}
2095
2096// Try legalizing an FCMP by swapping or inverting the predicate to one that
2097// is supported.
2098static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,
2099 CmpInst::Predicate &Pred, bool &NeedInvert) {
2100 auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
2101 return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
2102 Pred == CmpInst::FCMP_OEQ;
2103 };
2104
2105 assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
2106
2107 CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(pred: Pred);
2108 if (isLegalFCmpPredicate(InvPred)) {
2109 Pred = InvPred;
2110 std::swap(a&: LHS, b&: RHS);
2111 return true;
2112 }
2113
2114 InvPred = CmpInst::getInversePredicate(pred: Pred);
2115 NeedInvert = true;
2116 if (isLegalFCmpPredicate(InvPred)) {
2117 Pred = InvPred;
2118 return true;
2119 }
2120 InvPred = CmpInst::getSwappedPredicate(pred: InvPred);
2121 if (isLegalFCmpPredicate(InvPred)) {
2122 Pred = InvPred;
2123 std::swap(a&: LHS, b&: RHS);
2124 return true;
2125 }
2126
2127 return false;
2128}
2129
2130// Emit a sequence of instructions to compare LHS and RHS using Pred. Return
2131// the result in DstReg.
2132// FIXME: Maybe we should expand this earlier.
2133bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI) const {
2134 auto &CmpMI = cast<GFCmp>(Val&: MI);
2135 CmpInst::Predicate Pred = CmpMI.getCond();
2136
2137 Register DstReg = CmpMI.getReg(Idx: 0);
2138 Register LHS = CmpMI.getLHSReg();
2139 Register RHS = CmpMI.getRHSReg();
2140
2141 unsigned Size = MRI->getType(Reg: LHS).getSizeInBits();
2142 assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
2143
2144 Register TmpReg = DstReg;
2145
2146 bool NeedInvert = false;
2147 // First try swapping operands or inverting.
2148 if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
2149 if (NeedInvert)
2150 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2151 MachineInstr *Cmp = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2152 MCID: TII.get(Opcode: getFCmpOpcode(Pred, Size)), DestReg: TmpReg)
2153 .addReg(RegNo: LHS)
2154 .addReg(RegNo: RHS);
2155 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
2156 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
2157 // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
2158 NeedInvert = Pred == CmpInst::FCMP_UEQ;
2159 Register Cmp1Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2160 MachineInstr *Cmp1 =
2161 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2162 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size)), DestReg: Cmp1Reg)
2163 .addReg(RegNo: LHS)
2164 .addReg(RegNo: RHS);
2165 constrainSelectedInstRegOperands(I&: *Cmp1, TII, TRI, RBI);
2166 Register Cmp2Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2167 MachineInstr *Cmp2 =
2168 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2169 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size)), DestReg: Cmp2Reg)
2170 .addReg(RegNo: RHS)
2171 .addReg(RegNo: LHS);
2172 constrainSelectedInstRegOperands(I&: *Cmp2, TII, TRI, RBI);
2173 if (NeedInvert)
2174 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2175 MachineInstr *Or = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2176 MCID: TII.get(Opcode: RISCV::OR), DestReg: TmpReg)
2177 .addReg(RegNo: Cmp1Reg)
2178 .addReg(RegNo: Cmp2Reg);
2179 constrainSelectedInstRegOperands(I&: *Or, TII, TRI, RBI);
2180 } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
2181 // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
2182 // If LHS and RHS are the same, a single FEQ suffices.
2183 NeedInvert = Pred == CmpInst::FCMP_UNO;
2184 if (NeedInvert)
2185 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2186 if (LHS == RHS) {
2187 MachineInstr *Cmp =
2188 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2189 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: TmpReg)
2190 .addReg(RegNo: LHS)
2191 .addReg(RegNo: LHS);
2192 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
2193 } else {
2194 Register Cmp1Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2195 MachineInstr *Cmp1 =
2196 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2197 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: Cmp1Reg)
2198 .addReg(RegNo: LHS)
2199 .addReg(RegNo: LHS);
2200 constrainSelectedInstRegOperands(I&: *Cmp1, TII, TRI, RBI);
2201 Register Cmp2Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2202 MachineInstr *Cmp2 =
2203 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2204 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: Cmp2Reg)
2205 .addReg(RegNo: RHS)
2206 .addReg(RegNo: RHS);
2207 constrainSelectedInstRegOperands(I&: *Cmp2, TII, TRI, RBI);
2208 MachineInstr *And = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2209 MCID: TII.get(Opcode: RISCV::AND), DestReg: TmpReg)
2210 .addReg(RegNo: Cmp1Reg)
2211 .addReg(RegNo: Cmp2Reg);
2212 constrainSelectedInstRegOperands(I&: *And, TII, TRI, RBI);
2213 }
2214 } else
2215 llvm_unreachable("Unhandled predicate");
2216
2217 // Emit an XORI to invert the result if needed.
2218 if (NeedInvert) {
2219 MachineInstr *Xor = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2220 MCID: TII.get(Opcode: RISCV::XORI), DestReg: DstReg)
2221 .addReg(RegNo: TmpReg)
2222 .addImm(Val: 1);
2223 constrainSelectedInstRegOperands(I&: *Xor, TII, TRI, RBI);
2224 }
2225
2226 MI.eraseFromParent();
2227 return true;
2228}
2229
2230void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
2231 SyncScope::ID FenceSSID,
2232 MachineInstr &MI) const {
2233 MachineBasicBlock &MBB = *MI.getParent();
2234 DebugLoc DL = MI.getDebugLoc();
2235
2236 if (STI.hasStdExtZtso()) {
2237 // The only fence that needs an instruction is a sequentially-consistent
2238 // cross-thread fence.
2239 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
2240 FenceSSID == SyncScope::System) {
2241 // fence rw, rw
2242 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE))
2243 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W)
2244 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W);
2245 return;
2246 }
2247
2248 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
2249 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER));
2250 return;
2251 }
2252
2253 // singlethread fences only synchronize with signal handlers on the same
2254 // thread and thus only need to preserve instruction order, not actually
2255 // enforce memory ordering.
2256 if (FenceSSID == SyncScope::SingleThread) {
2257 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER));
2258 return;
2259 }
2260
2261 // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
2262 // Manual: Volume I.
2263 unsigned Pred, Succ;
2264 switch (FenceOrdering) {
2265 default:
2266 llvm_unreachable("Unexpected ordering");
2267 case AtomicOrdering::AcquireRelease:
2268 // fence acq_rel -> fence.tso
2269 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE_TSO));
2270 return;
2271 case AtomicOrdering::Acquire:
2272 // fence acquire -> fence r, rw
2273 Pred = RISCVFenceField::R;
2274 Succ = RISCVFenceField::R | RISCVFenceField::W;
2275 break;
2276 case AtomicOrdering::Release:
2277 // fence release -> fence rw, w
2278 Pred = RISCVFenceField::R | RISCVFenceField::W;
2279 Succ = RISCVFenceField::W;
2280 break;
2281 case AtomicOrdering::SequentiallyConsistent:
2282 // fence seq_cst -> fence rw, rw
2283 Pred = RISCVFenceField::R | RISCVFenceField::W;
2284 Succ = RISCVFenceField::R | RISCVFenceField::W;
2285 break;
2286 }
2287 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE)).addImm(Val: Pred).addImm(Val: Succ);
2288}
2289
2290namespace llvm {
2291InstructionSelector *
2292createRISCVInstructionSelector(const RISCVTargetMachine &TM,
2293 const RISCVSubtarget &Subtarget,
2294 const RISCVRegisterBankInfo &RBI) {
2295 return new RISCVInstructionSelector(TM, Subtarget, RBI);
2296}
2297} // end namespace llvm
2298