1//===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// RISC-V.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/RISCVMatInt.h"
15#include "RISCVRegisterBankInfo.h"
16#include "RISCVSubtarget.h"
17#include "RISCVTargetMachine.h"
18#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
19#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
20#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
21#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
22#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/MachineJumpTableInfo.h"
25#include "llvm/IR/IntrinsicsRISCV.h"
26#include "llvm/Support/Debug.h"
27
28#define DEBUG_TYPE "riscv-isel"
29
30using namespace llvm;
31using namespace MIPatternMatch;
32
33#define GET_GLOBALISEL_PREDICATE_BITSET
34#include "RISCVGenGlobalISel.inc"
35#undef GET_GLOBALISEL_PREDICATE_BITSET
36
37namespace {
38
39class RISCVInstructionSelector : public InstructionSelector {
40public:
41 RISCVInstructionSelector(const RISCVTargetMachine &TM,
42 const RISCVSubtarget &STI,
43 const RISCVRegisterBankInfo &RBI);
44
45 bool select(MachineInstr &MI) override;
46
47 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
48 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
49 BlockFrequencyInfo *BFI) override {
50 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
51 MRI = &MF.getRegInfo();
52 }
53
54 static const char *getName() { return DEBUG_TYPE; }
55
56private:
57 static constexpr unsigned MaxRecursionDepth = 6;
58
59 bool hasAllNBitUsers(const MachineInstr &MI, unsigned Bits,
60 const unsigned Depth = 0) const;
61 bool hasAllHUsers(const MachineInstr &MI) const {
62 return hasAllNBitUsers(MI, Bits: 16);
63 }
64 bool hasAllWUsers(const MachineInstr &MI) const {
65 return hasAllNBitUsers(MI, Bits: 32);
66 }
67
68 bool isRegInGprb(Register Reg) const;
69 bool isRegInFprb(Register Reg) const;
70
71 // tblgen-erated 'select' implementation, used as the initial selector for
72 // the patterns that don't require complex C++.
73 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
74
75 // A lowering phase that runs before any selection attempts.
76 // Returns true if the instruction was modified.
77 void preISelLower(MachineInstr &MI);
78
79 bool replacePtrWithInt(MachineOperand &Op);
80
81 // Custom selection methods
82 bool selectCopy(MachineInstr &MI) const;
83 bool selectImplicitDef(MachineInstr &MI) const;
84 bool materializeImm(Register Reg, int64_t Imm, MachineInstr &MI) const;
85 // Emit a constant-materialization instruction sequence.
86 bool materializeInstSeq(Register DstReg, const RISCVMatInt::InstSeq &Seq,
87 MachineInstr &MI) const;
88 bool selectAddr(MachineInstr &MI, bool IsLocal = true,
89 bool IsExternWeak = false) const;
90 bool selectSelect(MachineInstr &MI) const;
91 bool selectFPCompare(MachineInstr &MI) const;
92 void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
93 MachineInstr &MI) const;
94 bool selectUnmergeValues(MachineInstr &MI) const;
95 void addVectorLoadStoreOperands(MachineInstr &I,
96 SmallVectorImpl<Register> &SrcOps,
97 unsigned &CurOp, bool IsMasked,
98 bool IsStridedOrIndexed,
99 LLT *IndexVT = nullptr) const;
100 bool selectIntrinsicWithSideEffects(MachineInstr &I) const;
101 bool selectIntrinsic(MachineInstr &I) const;
102 bool selectExtractSubvector(MachineInstr &MI) const;
103 bool selectInsertSubVector(MachineInstr &I) const;
104 ComplexRendererFns selectShiftMask(MachineOperand &Root,
105 unsigned ShiftWidth) const;
106 ComplexRendererFns selectShiftMaskXLen(MachineOperand &Root) const {
107 return selectShiftMask(Root, ShiftWidth: STI.getXLen());
108 }
109 ComplexRendererFns selectShiftMask32(MachineOperand &Root) const {
110 return selectShiftMask(Root, ShiftWidth: 32);
111 }
112 ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
113 ComplexRendererFns selectAddrRegImmLsb00000(MachineOperand &Root) const;
114
115 // Plan for materializing a constant address as (Hi materialization, Lo12
116 // offset). Lo12 is a simm12 that, for prefetch (IsPrefetch), must be
117 // a multiple of 32.
118 struct ConstAddrPlan {
119 enum { X0, LUI, InstSeq } Kind = X0;
120 int64_t Hi20 = 0;
121 RISCVMatInt::InstSeq Seq;
122 int64_t Lo12 = 0;
123 };
124 ComplexRendererFns computeConstAddr(int64_t CVal, bool IsPrefetch,
125 Register OrigBase) const;
126 // Materialize the high part of Plan into a register. If OrigBase is valid,
127 // ADD it to the materialized high part (for G_PTR_ADD + large constant).
128 Register materializeConstBase(MachineInstrBuilder &MIB,
129 const ConstAddrPlan &Plan,
130 Register OrigBase) const;
131
132 ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;
133 template <unsigned Bits>
134 ComplexRendererFns selectSExtBits(MachineOperand &Root) const {
135 return selectSExtBits(Root, Bits);
136 }
137
138 ComplexRendererFns selectZExtBits(MachineOperand &Root, unsigned Bits) const;
139 template <unsigned Bits>
140 ComplexRendererFns selectZExtBits(MachineOperand &Root) const {
141 return selectZExtBits(Root, Bits);
142 }
143
144 ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
145 template <unsigned ShAmt>
146 ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
147 return selectSHXADDOp(Root, ShAmt);
148 }
149
150 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
151 unsigned ShAmt) const;
152 template <unsigned ShAmt>
153 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
154 return selectSHXADD_UWOp(Root, ShAmt);
155 }
156
157 ComplexRendererFns renderVLOp(MachineOperand &Root) const;
158
159 // Custom renderers for tablegen
160 void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
161 int OpIdx) const;
162 void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
163 int OpIdx) const;
164 void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
165 int OpIdx) const;
166 void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
167 int OpIdx) const;
168
169 void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
170 int OpIdx) const;
171 void renderXLenSubTrailingOnes(MachineInstrBuilder &MIB,
172 const MachineInstr &MI, int OpIdx) const;
173
174 void renderAddiPairImmLarge(MachineInstrBuilder &MIB, const MachineInstr &MI,
175 int OpIdx) const;
176 void renderAddiPairImmSmall(MachineInstrBuilder &MIB, const MachineInstr &MI,
177 int OpIdx) const;
178
179 const RISCVSubtarget &STI;
180 const RISCVInstrInfo &TII;
181 const RISCVRegisterInfo &TRI;
182 const RISCVRegisterBankInfo &RBI;
183 const RISCVTargetMachine &TM;
184
185 MachineRegisterInfo *MRI = nullptr;
186
187 // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
188 // uses "STI." in the code generated by TableGen. We need to unify the name of
189 // Subtarget variable.
190 const RISCVSubtarget *Subtarget = &STI;
191
192#define GET_GLOBALISEL_PREDICATES_DECL
193#include "RISCVGenGlobalISel.inc"
194#undef GET_GLOBALISEL_PREDICATES_DECL
195
196#define GET_GLOBALISEL_TEMPORARIES_DECL
197#include "RISCVGenGlobalISel.inc"
198#undef GET_GLOBALISEL_TEMPORARIES_DECL
199};
200
201} // end anonymous namespace
202
203#define GET_GLOBALISEL_IMPL
204#include "RISCVGenGlobalISel.inc"
205#undef GET_GLOBALISEL_IMPL
206
207RISCVInstructionSelector::RISCVInstructionSelector(
208 const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
209 const RISCVRegisterBankInfo &RBI)
210 : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
211 TM(TM),
212
213#define GET_GLOBALISEL_PREDICATES_INIT
214#include "RISCVGenGlobalISel.inc"
215#undef GET_GLOBALISEL_PREDICATES_INIT
216#define GET_GLOBALISEL_TEMPORARIES_INIT
217#include "RISCVGenGlobalISel.inc"
218#undef GET_GLOBALISEL_TEMPORARIES_INIT
219{
220}
221
222// Mimics optimizations in ISel and RISCVOptWInst Pass
223bool RISCVInstructionSelector::hasAllNBitUsers(const MachineInstr &MI,
224 unsigned Bits,
225 const unsigned Depth) const {
226
227 assert((MI.getOpcode() == TargetOpcode::G_ADD ||
228 MI.getOpcode() == TargetOpcode::G_SUB ||
229 MI.getOpcode() == TargetOpcode::G_MUL ||
230 MI.getOpcode() == TargetOpcode::G_SHL ||
231 MI.getOpcode() == TargetOpcode::G_LSHR ||
232 MI.getOpcode() == TargetOpcode::G_AND ||
233 MI.getOpcode() == TargetOpcode::G_OR ||
234 MI.getOpcode() == TargetOpcode::G_XOR ||
235 MI.getOpcode() == TargetOpcode::G_SEXT_INREG || Depth != 0) &&
236 "Unexpected opcode");
237
238 if (Depth >= RISCVInstructionSelector::MaxRecursionDepth)
239 return false;
240
241 auto DestReg = MI.getOperand(i: 0).getReg();
242 for (auto &UserOp : MRI->use_nodbg_operands(Reg: DestReg)) {
243 assert(UserOp.getParent() && "UserOp must have a parent");
244 const MachineInstr &UserMI = *UserOp.getParent();
245 unsigned OpIdx = UserOp.getOperandNo();
246
247 switch (UserMI.getOpcode()) {
248 default:
249 return false;
250 case RISCV::ADDW:
251 case RISCV::ADDIW:
252 case RISCV::SUBW:
253 case RISCV::FCVT_D_W:
254 case RISCV::FCVT_S_W:
255 if (Bits >= 32)
256 break;
257 return false;
258 case RISCV::SLL:
259 case RISCV::SRA:
260 case RISCV::SRL:
261 // Shift amount operands only use log2(Xlen) bits.
262 if (OpIdx == 2 && Bits >= Log2_32(Value: Subtarget->getXLen()))
263 break;
264 return false;
265 case RISCV::SLLI:
266 // SLLI only uses the lower (XLen - ShAmt) bits.
267 if (Bits >= Subtarget->getXLen() - UserMI.getOperand(i: 2).getImm())
268 break;
269 return false;
270 case RISCV::ANDI:
271 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(
272 Value: (uint64_t)UserMI.getOperand(i: 2).getImm()))
273 break;
274 goto RecCheck;
275 case RISCV::AND:
276 case RISCV::OR:
277 case RISCV::XOR:
278 RecCheck:
279 if (hasAllNBitUsers(MI: UserMI, Bits, Depth: Depth + 1))
280 break;
281 return false;
282 case RISCV::SRLI: {
283 unsigned ShAmt = UserMI.getOperand(i: 2).getImm();
284 // If we are shifting right by less than Bits, and users don't demand any
285 // bits that were shifted into [Bits-1:0], then we can consider this as an
286 // N-Bit user.
287 if (Bits > ShAmt && hasAllNBitUsers(MI: UserMI, Bits: Bits - ShAmt, Depth: Depth + 1))
288 break;
289 return false;
290 }
291 }
292 }
293
294 return true;
295}
296
297InstructionSelector::ComplexRendererFns
298RISCVInstructionSelector::selectShiftMask(MachineOperand &Root,
299 unsigned ShiftWidth) const {
300 if (!Root.isReg())
301 return std::nullopt;
302
303 using namespace llvm::MIPatternMatch;
304
305 Register ShAmtReg = Root.getReg();
306 // Peek through zext.
307 Register ZExtSrcReg;
308 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrcReg))))
309 ShAmtReg = ZExtSrcReg;
310
311 APInt AndMask;
312 Register AndSrcReg;
313 // Try to combine the following pattern (applicable to other shift
314 // instructions as well as 32-bit ones):
315 //
316 // %4:gprb(s64) = G_AND %3, %2
317 // %5:gprb(s64) = G_LSHR %1, %4(s64)
318 //
319 // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
320 // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
321 // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
322 // then it can be eliminated. Given register rs1 or rs2 holding a constant
323 // (the and mask), there are two cases G_AND can be erased:
324 //
325 // 1. the lowest log2(XLEN) bits of the and mask are all set
326 // 2. the bits of the register being masked are already unset (zero set)
327 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: AndSrcReg), R: m_ICst(Cst&: AndMask)))) {
328 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
329 if (ShMask.isSubsetOf(RHS: AndMask)) {
330 ShAmtReg = AndSrcReg;
331 } else {
332 // SimplifyDemandedBits may have optimized the mask so try restoring any
333 // bits that are known zero.
334 KnownBits Known = VT->getKnownBits(R: AndSrcReg);
335 if (ShMask.isSubsetOf(RHS: AndMask | Known.Zero))
336 ShAmtReg = AndSrcReg;
337 }
338 }
339
340 APInt Imm;
341 Register Reg;
342 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAdd(L: m_Reg(R&: Reg), R: m_ICst(Cst&: Imm)))) {
343 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0)
344 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
345 // to avoid the ADD.
346 ShAmtReg = Reg;
347 } else if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GSub(L: m_ICst(Cst&: Imm), R: m_Reg(R&: Reg)))) {
348 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0) {
349 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
350 // to generate a NEG instead of a SUB of a constant.
351 ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
352 unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
353 return {{[=](MachineInstrBuilder &MIB) {
354 MachineIRBuilder(*MIB.getInstr())
355 .buildInstr(Opc: NegOpc, DstOps: {ShAmtReg}, SrcOps: {Register(RISCV::X0), Reg});
356 MIB.addReg(RegNo: ShAmtReg);
357 }}};
358 }
359 if (Imm.urem(RHS: ShiftWidth) == ShiftWidth - 1) {
360 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
361 // to generate a NOT instead of a SUB of a constant.
362 ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
363 return {{[=](MachineInstrBuilder &MIB) {
364 MachineIRBuilder(*MIB.getInstr())
365 .buildInstr(Opc: RISCV::XORI, DstOps: {ShAmtReg}, SrcOps: {Reg})
366 .addImm(Val: -1);
367 MIB.addReg(RegNo: ShAmtReg);
368 }}};
369 }
370 }
371
372 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ShAmtReg); }}};
373}
374
375InstructionSelector::ComplexRendererFns
376RISCVInstructionSelector::selectSExtBits(MachineOperand &Root,
377 unsigned Bits) const {
378 if (!Root.isReg())
379 return std::nullopt;
380 Register RootReg = Root.getReg();
381
382 Register SrcReg;
383 if (mi_match(R: RootReg, MRI: *MRI,
384 P: m_GSExtInReg(Src: m_Reg(R&: SrcReg), Imm: m_SpecificImm(RequestedValue: Bits)))) {
385 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SrcReg); }}};
386 }
387
388 unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits();
389 if ((Size - VT->computeNumSignBits(R: RootReg)) < Bits)
390 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}};
391
392 return std::nullopt;
393}
394
395InstructionSelector::ComplexRendererFns
396RISCVInstructionSelector::selectZExtBits(MachineOperand &Root,
397 unsigned Bits) const {
398 if (!Root.isReg())
399 return std::nullopt;
400 Register RootReg = Root.getReg();
401
402 Register RegX;
403 uint64_t Mask = maskTrailingOnes<uint64_t>(N: Bits);
404 if (mi_match(R: RootReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: RegX), R: m_SpecificICst(RequestedValue: Mask)))) {
405 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}};
406 }
407
408 if (mi_match(R: RootReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: RegX))) &&
409 MRI->getType(Reg: RegX).getScalarSizeInBits() == Bits)
410 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}};
411
412 unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits();
413 if (VT->maskedValueIsZero(Val: RootReg, Mask: APInt::getBitsSetFrom(numBits: Size, loBit: Bits)))
414 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}};
415
416 return std::nullopt;
417}
418
419InstructionSelector::ComplexRendererFns
420RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
421 unsigned ShAmt) const {
422 using namespace llvm::MIPatternMatch;
423
424 if (!Root.isReg())
425 return std::nullopt;
426 Register RootReg = Root.getReg();
427
428 const unsigned XLen = STI.getXLen();
429 APInt Mask, C2;
430 Register RegY;
431 std::optional<bool> LeftShift;
432 // (and (shl y, c2), mask)
433 if (mi_match(R: RootReg, MRI: *MRI,
434 P: m_GAnd(L: m_GShl(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
435 LeftShift = true;
436 // (and (lshr y, c2), mask)
437 else if (mi_match(R: RootReg, MRI: *MRI,
438 P: m_GAnd(L: m_GLShr(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
439 LeftShift = false;
440
441 if (LeftShift.has_value()) {
442 if (*LeftShift)
443 Mask &= maskTrailingZeros<uint64_t>(N: C2.getZExtValue());
444 else
445 Mask &= maskTrailingOnes<uint64_t>(N: XLen - C2.getZExtValue());
446
447 if (Mask.isShiftedMask()) {
448 unsigned Leading = XLen - Mask.getActiveBits();
449 unsigned Trailing = Mask.countr_zero();
450 // Given (and (shl y, c2), mask) in which mask has no leading zeros and
451 // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
452 if (*LeftShift && Leading == 0 && C2.ult(RHS: Trailing) && Trailing == ShAmt) {
453 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
454 return {{[=](MachineInstrBuilder &MIB) {
455 MachineIRBuilder(*MIB.getInstr())
456 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
457 .addImm(Val: Trailing - C2.getZExtValue());
458 MIB.addReg(RegNo: DstReg);
459 }}};
460 }
461
462 // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
463 // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
464 if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
465 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
466 return {{[=](MachineInstrBuilder &MIB) {
467 MachineIRBuilder(*MIB.getInstr())
468 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
469 .addImm(Val: Leading + Trailing);
470 MIB.addReg(RegNo: DstReg);
471 }}};
472 }
473 }
474 }
475
476 LeftShift.reset();
477
478 // (shl (and y, mask), c2)
479 if (mi_match(R: RootReg, MRI: *MRI,
480 P: m_GShl(L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
481 R: m_ICst(Cst&: C2))))
482 LeftShift = true;
483 // (lshr (and y, mask), c2)
484 else if (mi_match(R: RootReg, MRI: *MRI,
485 P: m_GLShr(L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
486 R: m_ICst(Cst&: C2))))
487 LeftShift = false;
488
489 if (LeftShift.has_value() && Mask.isShiftedMask()) {
490 unsigned Leading = XLen - Mask.getActiveBits();
491 unsigned Trailing = Mask.countr_zero();
492
493 // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
494 // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
495 bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&
496 (Trailing + C2.getZExtValue()) == ShAmt;
497 if (!Cond)
498 // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
499 // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
500 Cond = !*LeftShift && Leading == 32 && C2.ult(RHS: Trailing) &&
501 (Trailing - C2.getZExtValue()) == ShAmt;
502
503 if (Cond) {
504 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
505 return {{[=](MachineInstrBuilder &MIB) {
506 MachineIRBuilder(*MIB.getInstr())
507 .buildInstr(Opc: RISCV::SRLIW, DstOps: {DstReg}, SrcOps: {RegY})
508 .addImm(Val: Trailing);
509 MIB.addReg(RegNo: DstReg);
510 }}};
511 }
512 }
513
514 return std::nullopt;
515}
516
517InstructionSelector::ComplexRendererFns
518RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
519 unsigned ShAmt) const {
520 using namespace llvm::MIPatternMatch;
521
522 if (!Root.isReg())
523 return std::nullopt;
524 Register RootReg = Root.getReg();
525
526 // Given (and (shl x, c2), mask) in which mask is a shifted mask with
527 // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
528 // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
529 APInt Mask, C2;
530 Register RegX;
531 if (mi_match(
532 R: RootReg, MRI: *MRI,
533 P: m_OneNonDBGUse(SP: m_GAnd(L: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: RegX), R: m_ICst(Cst&: C2))),
534 R: m_ICst(Cst&: Mask))))) {
535 Mask &= maskTrailingZeros<uint64_t>(N: C2.getZExtValue());
536
537 if (Mask.isShiftedMask()) {
538 unsigned Leading = Mask.countl_zero();
539 unsigned Trailing = Mask.countr_zero();
540 if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
541 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
542 return {{[=](MachineInstrBuilder &MIB) {
543 MachineIRBuilder(*MIB.getInstr())
544 .buildInstr(Opc: RISCV::SLLI, DstOps: {DstReg}, SrcOps: {RegX})
545 .addImm(Val: C2.getZExtValue() - ShAmt);
546 MIB.addReg(RegNo: DstReg);
547 }}};
548 }
549 }
550 }
551
552 return std::nullopt;
553}
554
555InstructionSelector::ComplexRendererFns
556RISCVInstructionSelector::renderVLOp(MachineOperand &Root) const {
557 assert(Root.isReg() && "Expected operand to be a Register");
558 std::optional<ValueAndVReg> C;
559 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GCst(ValReg&: C))) {
560 if (C->Value.isAllOnes())
561 // If the operand is a G_CONSTANT with value of all ones it is larger than
562 // VLMAX. We convert it to an immediate with value VLMaxSentinel. This is
563 // recognized specially by the vsetvli insertion pass.
564 return {{[=](MachineInstrBuilder &MIB) {
565 MIB.addImm(Val: RISCV::VLMaxSentinel);
566 }}};
567
568 if (isUInt<5>(x: C->Value.getZExtValue())) {
569 uint64_t ZExtC = C->Value.getZExtValue();
570 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ZExtC); }}};
571 }
572 }
573 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Root.getReg()); }}};
574}
575
576InstructionSelector::ComplexRendererFns
577RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
578 if (!Root.isReg())
579 return std::nullopt;
580
581 MachineInstr *RootDef = MRI->getVRegDef(Reg: Root.getReg());
582 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
583 return {{
584 [=](MachineInstrBuilder &MIB) { MIB.add(MO: RootDef->getOperand(i: 1)); },
585 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
586 }};
587 }
588
589 if (isBaseWithConstantOffset(Root, MRI: *MRI)) {
590 MachineOperand &LHS = RootDef->getOperand(i: 1);
591 MachineOperand &RHS = RootDef->getOperand(i: 2);
592 MachineInstr *LHSDef = MRI->getVRegDef(Reg: LHS.getReg());
593 MachineInstr *RHSDef = MRI->getVRegDef(Reg: RHS.getReg());
594
595 int64_t RHSC = RHSDef->getOperand(i: 1).getCImm()->getSExtValue();
596 if (isInt<12>(x: RHSC)) {
597 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
598 return {{
599 [=](MachineInstrBuilder &MIB) { MIB.add(MO: LHSDef->getOperand(i: 1)); },
600 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
601 }};
602
603 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: LHS); },
604 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
605 }
606 }
607
608 // TODO: Need to get the immediate from a G_PTR_ADD. Should this be done in
609 // the combiner?
610 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Root.getReg()); },
611 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
612}
613
614InstructionSelector::ComplexRendererFns
615RISCVInstructionSelector::selectAddrRegImmLsb00000(MachineOperand &Root) const {
616 if (!Root.isReg())
617 return std::nullopt;
618
619 MachineInstr *RootDef = MRI->getVRegDef(Reg: Root.getReg());
620 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
621 return {{
622 [=](MachineInstrBuilder &MIB) { MIB.add(MO: RootDef->getOperand(i: 1)); },
623 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
624 }};
625 }
626
627 if (isBaseWithConstantOffset(Root, MRI: *MRI)) {
628 MachineOperand &LHS = RootDef->getOperand(i: 1);
629 MachineOperand &RHS = RootDef->getOperand(i: 2);
630 MachineInstr *LHSDef = MRI->getVRegDef(Reg: LHS.getReg());
631 MachineInstr *RHSDef = MRI->getVRegDef(Reg: RHS.getReg());
632 int64_t RHSC = RHSDef->getOperand(i: 1).getCImm()->getSExtValue();
633
634 if (isInt<12>(x: RHSC)) {
635 // Not a multiple of 32: can't encode, use the address as-is.
636 if ((RHSC & 0b11111) != 0) {
637 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Root.getReg()); },
638 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
639 }
640 // Fold the offset.
641 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
642 return {{
643 [=](MachineInstrBuilder &MIB) { MIB.add(MO: LHSDef->getOperand(i: 1)); },
644 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
645 }};
646 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: LHS); },
647 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
648 }
649
650 // Large constant: fold a -2048/2016 adjustment to save an instruction.
651 if ((-2049 >= RHSC && RHSC >= -4096) || (4063 >= RHSC && RHSC >= 2017)) {
652 int64_t Adj = RHSC < 0 ? -2048 : 2016;
653 int64_t AdjustedOffset = RHSC - Adj;
654 Register BaseReg = LHS.getReg();
655 return {{[=](MachineInstrBuilder &MIB) {
656 Register Tmp = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
657 MachineInstr *Addi =
658 BuildMI(BB&: *MIB->getParent(), I&: *MIB.getInstr(),
659 MIMD: MIB->getDebugLoc(), MCID: TII.get(Opcode: RISCV::ADDI), DestReg: Tmp)
660 .addReg(RegNo: BaseReg)
661 .addImm(Val: AdjustedOffset);
662 constrainSelectedInstRegOperands(I&: *Addi, TII, TRI, RBI);
663 MIB.addReg(RegNo: Tmp);
664 },
665 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Adj); }}};
666 }
667
668 // Otherwise split the constant into Hi (materialized + added to the base)
669 // and Lo12 (folded offset).
670 if (auto Fns = computeConstAddr(CVal: RHSC, /*IsPrefetch=*/true, OrigBase: LHS.getReg()))
671 return Fns;
672 }
673
674 // Bare constant address. IRTranslator emits inttoptr(C) as
675 // G_INTTOPTR(G_CONSTANT); look through the G_INTTOPTR to reach the constant.
676 if (RootDef->getOpcode() == TargetOpcode::G_INTTOPTR) {
677 MachineInstr *SrcDef = MRI->getVRegDef(Reg: RootDef->getOperand(i: 1).getReg());
678 if (SrcDef->getOpcode() == TargetOpcode::G_CONSTANT)
679 RootDef = SrcDef;
680 }
681 if (RootDef->getOpcode() == TargetOpcode::G_CONSTANT) {
682 int64_t CVal = RootDef->getOperand(i: 1).getCImm()->getSExtValue();
683 if (auto Fns = computeConstAddr(CVal, /*IsPrefetch=*/true, OrigBase: Register()))
684 return Fns;
685 }
686
687 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Root.getReg()); },
688 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
689}
690
691/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
692/// CC Must be an ICMP Predicate.
693static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
694 switch (CC) {
695 default:
696 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
697 case CmpInst::Predicate::ICMP_EQ:
698 return RISCVCC::COND_EQ;
699 case CmpInst::Predicate::ICMP_NE:
700 return RISCVCC::COND_NE;
701 case CmpInst::Predicate::ICMP_ULT:
702 return RISCVCC::COND_LTU;
703 case CmpInst::Predicate::ICMP_SLT:
704 return RISCVCC::COND_LT;
705 case CmpInst::Predicate::ICMP_UGE:
706 return RISCVCC::COND_GEU;
707 case CmpInst::Predicate::ICMP_SGE:
708 return RISCVCC::COND_GE;
709 }
710}
711
712static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC,
713 Register &LHS, Register &RHS,
714 MachineRegisterInfo &MRI) {
715 // Try to fold an ICmp. If that fails, use a NE compare with X0.
716 CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
717 if (!mi_match(R: CondReg, MRI, P: m_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: LHS), R: m_Reg(R&: RHS)))) {
718 LHS = CondReg;
719 RHS = RISCV::X0;
720 CC = RISCVCC::COND_NE;
721 return;
722 }
723
724 // We found an ICmp, do some canonicalization.
725
726 // Adjust comparisons to use comparison with 0 if possible.
727 if (auto Constant = getIConstantVRegSExtVal(VReg: RHS, MRI)) {
728 switch (Pred) {
729 case CmpInst::Predicate::ICMP_SGT:
730 // Convert X > -1 to X >= 0
731 if (*Constant == -1) {
732 CC = RISCVCC::COND_GE;
733 RHS = RISCV::X0;
734 return;
735 }
736 break;
737 case CmpInst::Predicate::ICMP_SLT:
738 // Convert X < 1 to 0 >= X
739 if (*Constant == 1) {
740 CC = RISCVCC::COND_GE;
741 RHS = LHS;
742 LHS = RISCV::X0;
743 return;
744 }
745 break;
746 default:
747 break;
748 }
749 }
750
751 switch (Pred) {
752 default:
753 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
754 case CmpInst::Predicate::ICMP_EQ:
755 case CmpInst::Predicate::ICMP_NE:
756 case CmpInst::Predicate::ICMP_ULT:
757 case CmpInst::Predicate::ICMP_SLT:
758 case CmpInst::Predicate::ICMP_UGE:
759 case CmpInst::Predicate::ICMP_SGE:
760 // These CCs are supported directly by RISC-V branches.
761 break;
762 case CmpInst::Predicate::ICMP_SGT:
763 case CmpInst::Predicate::ICMP_SLE:
764 case CmpInst::Predicate::ICMP_UGT:
765 case CmpInst::Predicate::ICMP_ULE:
766 // These CCs are not supported directly by RISC-V branches, but changing the
767 // direction of the CC and swapping LHS and RHS are.
768 Pred = CmpInst::getSwappedPredicate(pred: Pred);
769 std::swap(a&: LHS, b&: RHS);
770 break;
771 }
772
773 CC = getRISCVCCFromICmp(CC: Pred);
774}
775
776/// Select the RISC-V Zalasr opcode for the G_LOAD or G_STORE operation
777/// \p GenericOpc, appropriate for the GPR register bank and of memory access
778/// size \p OpSize.
779static unsigned selectZalasrLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {
780 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
781 switch (OpSize) {
782 default:
783 llvm_unreachable("Unexpected memory size");
784 case 8:
785 return IsStore ? RISCV::SB_RL : RISCV::LB_AQ;
786 case 16:
787 return IsStore ? RISCV::SH_RL : RISCV::LH_AQ;
788 case 32:
789 return IsStore ? RISCV::SW_RL : RISCV::LW_AQ;
790 case 64:
791 return IsStore ? RISCV::SD_RL : RISCV::LD_AQ;
792 }
793}
794
795/// Select the RISC-V regimm opcode for the G_LOAD or G_STORE operation
796/// \p GenericOpc, appropriate for the GPR register bank and of memory access
797/// size \p OpSize. \returns \p GenericOpc if the combination is unsupported.
798static unsigned selectRegImmLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {
799 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
800 switch (OpSize) {
801 case 8:
802 // Prefer unsigned due to no c.lb in Zcb.
803 return IsStore ? RISCV::SB : RISCV::LBU;
804 case 16:
805 return IsStore ? RISCV::SH : RISCV::LH;
806 case 32:
807 return IsStore ? RISCV::SW : RISCV::LW;
808 case 64:
809 return IsStore ? RISCV::SD : RISCV::LD;
810 }
811
812 return GenericOpc;
813}
814
815void RISCVInstructionSelector::addVectorLoadStoreOperands(
816 MachineInstr &I, SmallVectorImpl<Register> &SrcOps, unsigned &CurOp,
817 bool IsMasked, bool IsStridedOrIndexed, LLT *IndexVT) const {
818 // Base Pointer
819 auto PtrReg = I.getOperand(i: CurOp++).getReg();
820 SrcOps.push_back(Elt: PtrReg);
821
822 // Stride or Index
823 if (IsStridedOrIndexed) {
824 auto StrideReg = I.getOperand(i: CurOp++).getReg();
825 SrcOps.push_back(Elt: StrideReg);
826 if (IndexVT)
827 *IndexVT = MRI->getType(Reg: StrideReg);
828 }
829
830 // Mask
831 if (IsMasked) {
832 auto MaskReg = I.getOperand(i: CurOp++).getReg();
833 SrcOps.push_back(Elt: MaskReg);
834 }
835}
836
837bool RISCVInstructionSelector::selectIntrinsicWithSideEffects(
838 MachineInstr &I) const {
839 // Find the intrinsic ID.
840 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
841 // Select the instruction.
842 switch (IntrinID) {
843 default:
844 return false;
845 case Intrinsic::riscv_vlm:
846 case Intrinsic::riscv_vle:
847 case Intrinsic::riscv_vle_mask:
848 case Intrinsic::riscv_vlse:
849 case Intrinsic::riscv_vlse_mask: {
850 bool IsMasked = IntrinID == Intrinsic::riscv_vle_mask ||
851 IntrinID == Intrinsic::riscv_vlse_mask;
852 bool IsStrided = IntrinID == Intrinsic::riscv_vlse ||
853 IntrinID == Intrinsic::riscv_vlse_mask;
854 LLT VT = MRI->getType(Reg: I.getOperand(i: 0).getReg());
855 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
856
857 // Result vector
858 const Register DstReg = I.getOperand(i: 0).getReg();
859
860 // Sources
861 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
862 unsigned CurOp = 2;
863 SmallVector<Register, 4> SrcOps; // Source registers.
864
865 // Passthru
866 if (HasPassthruOperand) {
867 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
868 SrcOps.push_back(Elt: PassthruReg);
869 } else {
870 SrcOps.push_back(Elt: Register(RISCV::NoRegister));
871 }
872
873 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: IsStrided);
874
875 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
876 const RISCV::VLEPseudo *P =
877 RISCV::getVLEPseudo(Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
878 LMUL: static_cast<unsigned>(LMUL));
879
880 MachineInstrBuilder PseudoMI =
881 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo), DestReg: DstReg);
882 for (Register Reg : SrcOps)
883 PseudoMI.addReg(RegNo: Reg);
884
885 // Select VL
886 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
887 for (auto &RenderFn : *VLOpFn)
888 RenderFn(PseudoMI);
889
890 // SEW
891 PseudoMI.addImm(Val: Log2SEW);
892
893 // Policy
894 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
895 if (IsMasked)
896 Policy = I.getOperand(i: CurOp++).getImm();
897 PseudoMI.addImm(Val: Policy);
898
899 // Memref
900 PseudoMI.cloneMemRefs(OtherMI: I);
901
902 I.eraseFromParent();
903 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
904 return true;
905 }
906 case Intrinsic::riscv_vloxei:
907 case Intrinsic::riscv_vloxei_mask:
908 case Intrinsic::riscv_vluxei:
909 case Intrinsic::riscv_vluxei_mask: {
910 bool IsMasked = IntrinID == Intrinsic::riscv_vloxei_mask ||
911 IntrinID == Intrinsic::riscv_vluxei_mask;
912 bool IsOrdered = IntrinID == Intrinsic::riscv_vloxei ||
913 IntrinID == Intrinsic::riscv_vloxei_mask;
914 LLT VT = MRI->getType(Reg: I.getOperand(i: 0).getReg());
915 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
916
917 // Result vector
918 const Register DstReg = I.getOperand(i: 0).getReg();
919
920 // Sources
921 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
922 unsigned CurOp = 2;
923 SmallVector<Register, 4> SrcOps; // Source registers.
924
925 // Passthru
926 if (HasPassthruOperand) {
927 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
928 SrcOps.push_back(Elt: PassthruReg);
929 } else {
930 // Use NoRegister if there is no specified passthru.
931 SrcOps.push_back(Elt: Register());
932 }
933 LLT IndexVT;
934 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: true, IndexVT: &IndexVT);
935
936 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
937 RISCVVType::VLMUL IndexLMUL =
938 RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: IndexVT));
939 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
940 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
941 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
942 "values when XLEN=32");
943 }
944 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
945 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
946 IndexLMUL: static_cast<unsigned>(IndexLMUL));
947
948 MachineInstrBuilder PseudoMI =
949 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo), DestReg: DstReg);
950 for (Register Reg : SrcOps)
951 PseudoMI.addReg(RegNo: Reg);
952
953 // Select VL
954 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
955 for (auto &RenderFn : *VLOpFn)
956 RenderFn(PseudoMI);
957
958 // SEW
959 PseudoMI.addImm(Val: Log2SEW);
960
961 // Policy
962 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
963 if (IsMasked)
964 Policy = I.getOperand(i: CurOp++).getImm();
965 PseudoMI.addImm(Val: Policy);
966
967 // Memref
968 PseudoMI.cloneMemRefs(OtherMI: I);
969
970 I.eraseFromParent();
971 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
972 return true;
973 }
974 case Intrinsic::riscv_vsm:
975 case Intrinsic::riscv_vse:
976 case Intrinsic::riscv_vse_mask:
977 case Intrinsic::riscv_vsse:
978 case Intrinsic::riscv_vsse_mask: {
979 bool IsMasked = IntrinID == Intrinsic::riscv_vse_mask ||
980 IntrinID == Intrinsic::riscv_vsse_mask;
981 bool IsStrided = IntrinID == Intrinsic::riscv_vsse ||
982 IntrinID == Intrinsic::riscv_vsse_mask;
983 LLT VT = MRI->getType(Reg: I.getOperand(i: 1).getReg());
984 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
985
986 // Sources
987 unsigned CurOp = 1;
988 SmallVector<Register, 4> SrcOps; // Source registers.
989
990 // Store value
991 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
992 SrcOps.push_back(Elt: PassthruReg);
993
994 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: IsStrided);
995
996 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
997 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
998 Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
999
1000 MachineInstrBuilder PseudoMI =
1001 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo));
1002 for (Register Reg : SrcOps)
1003 PseudoMI.addReg(RegNo: Reg);
1004
1005 // Select VL
1006 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
1007 for (auto &RenderFn : *VLOpFn)
1008 RenderFn(PseudoMI);
1009
1010 // SEW
1011 PseudoMI.addImm(Val: Log2SEW);
1012
1013 // Memref
1014 PseudoMI.cloneMemRefs(OtherMI: I);
1015
1016 I.eraseFromParent();
1017 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1018 return true;
1019 }
1020 case Intrinsic::riscv_vsoxei:
1021 case Intrinsic::riscv_vsoxei_mask:
1022 case Intrinsic::riscv_vsuxei:
1023 case Intrinsic::riscv_vsuxei_mask: {
1024 bool IsMasked = IntrinID == Intrinsic::riscv_vsoxei_mask ||
1025 IntrinID == Intrinsic::riscv_vsuxei_mask;
1026 bool IsOrdered = IntrinID == Intrinsic::riscv_vsoxei ||
1027 IntrinID == Intrinsic::riscv_vsoxei_mask;
1028 LLT VT = MRI->getType(Reg: I.getOperand(i: 1).getReg());
1029 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
1030
1031 // Sources
1032 unsigned CurOp = 1;
1033 SmallVector<Register, 4> SrcOps; // Source registers.
1034
1035 // Store value
1036 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
1037 SrcOps.push_back(Elt: PassthruReg);
1038
1039 LLT IndexVT;
1040 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: true, IndexVT: &IndexVT);
1041
1042 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
1043 RISCVVType::VLMUL IndexLMUL =
1044 RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: IndexVT));
1045 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
1046 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
1047 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
1048 "values when XLEN=32");
1049 }
1050 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
1051 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
1052 IndexLMUL: static_cast<unsigned>(IndexLMUL));
1053
1054 MachineInstrBuilder PseudoMI =
1055 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo));
1056 for (Register Reg : SrcOps)
1057 PseudoMI.addReg(RegNo: Reg);
1058
1059 // Select VL
1060 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
1061 for (auto &RenderFn : *VLOpFn)
1062 RenderFn(PseudoMI);
1063
1064 // SEW
1065 PseudoMI.addImm(Val: Log2SEW);
1066
1067 // Memref
1068 PseudoMI.cloneMemRefs(OtherMI: I);
1069
1070 I.eraseFromParent();
1071 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1072 return true;
1073 }
1074 }
1075}
1076
1077bool RISCVInstructionSelector::selectIntrinsic(MachineInstr &I) const {
1078 // Find the intrinsic ID.
1079 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
1080 // Select the instruction.
1081 switch (IntrinID) {
1082 default:
1083 return false;
1084 case Intrinsic::riscv_vsetvli:
1085 case Intrinsic::riscv_vsetvlimax: {
1086
1087 bool VLMax = IntrinID == Intrinsic::riscv_vsetvlimax;
1088
1089 unsigned Offset = VLMax ? 2 : 3;
1090 unsigned SEW = RISCVVType::decodeVSEW(VSEW: I.getOperand(i: Offset).getImm() & 0x7);
1091 RISCVVType::VLMUL VLMul =
1092 static_cast<RISCVVType::VLMUL>(I.getOperand(i: Offset + 1).getImm() & 0x7);
1093
1094 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMUL: VLMul, SEW, /*TailAgnostic*/ true,
1095 /*MaskAgnostic*/ true);
1096
1097 Register DstReg = I.getOperand(i: 0).getReg();
1098
1099 Register VLOperand;
1100 unsigned Opcode = RISCV::PseudoVSETVLI;
1101
1102 // Check if AVL is a constant that equals VLMAX.
1103 if (!VLMax) {
1104 Register AVLReg = I.getOperand(i: 2).getReg();
1105 if (auto AVLConst = getIConstantVRegValWithLookThrough(VReg: AVLReg, MRI: *MRI)) {
1106 uint64_t AVL = AVLConst->Value.getZExtValue();
1107 if (auto VLEN = Subtarget->getRealVLen()) {
1108 if (*VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == AVL)
1109 VLMax = true;
1110 }
1111 }
1112
1113 if (mi_match(R: AVLReg, MRI: *MRI, P: m_AllOnes()))
1114 VLMax = true;
1115 }
1116
1117 if (VLMax) {
1118 VLOperand = Register(RISCV::X0);
1119 Opcode = RISCV::PseudoVSETVLIX0;
1120 } else {
1121 Register AVLReg = I.getOperand(i: 2).getReg();
1122 VLOperand = AVLReg;
1123
1124 // Check if AVL is a small constant that can use PseudoVSETIVLI.
1125 if (auto AVLConst = getIConstantVRegValWithLookThrough(VReg: AVLReg, MRI: *MRI)) {
1126 uint64_t AVL = AVLConst->Value.getZExtValue();
1127 if (isUInt<5>(x: AVL)) {
1128 MachineInstr *PseudoMI =
1129 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1130 MCID: TII.get(Opcode: RISCV::PseudoVSETIVLI), DestReg: DstReg)
1131 .addImm(Val: AVL)
1132 .addImm(Val: VTypeI);
1133 I.eraseFromParent();
1134 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1135 return true;
1136 }
1137 }
1138 }
1139
1140 MachineInstr *PseudoMI =
1141 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode), DestReg: DstReg)
1142 .addReg(RegNo: VLOperand)
1143 .addImm(Val: VTypeI);
1144 I.eraseFromParent();
1145 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1146 return true;
1147 }
1148 }
1149}
1150
1151bool RISCVInstructionSelector::selectExtractSubvector(MachineInstr &MI) const {
1152 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_SUBVECTOR);
1153
1154 Register DstReg = MI.getOperand(i: 0).getReg();
1155 Register SrcReg = MI.getOperand(i: 1).getReg();
1156
1157 LLT DstTy = MRI->getType(Reg: DstReg);
1158 LLT SrcTy = MRI->getType(Reg: SrcReg);
1159
1160 unsigned Idx = static_cast<unsigned>(MI.getOperand(i: 2).getImm());
1161
1162 MVT DstMVT = getMVTForLLT(Ty: DstTy);
1163 MVT SrcMVT = getMVTForLLT(Ty: SrcTy);
1164
1165 unsigned SubRegIdx;
1166 std::tie(args&: SubRegIdx, args&: Idx) =
1167 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1168 VecVT: SrcMVT, SubVecVT: DstMVT, InsertExtractIdx: Idx, TRI: &TRI);
1169
1170 if (Idx != 0)
1171 return false;
1172
1173 unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: DstMVT);
1174 const TargetRegisterClass *DstRC = TRI.getRegClass(i: DstRegClassID);
1175 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1176 return false;
1177
1178 unsigned SrcRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: SrcMVT);
1179 const TargetRegisterClass *SrcRC = TRI.getRegClass(i: SrcRegClassID);
1180 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
1181 return false;
1182
1183 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1184 DestReg: DstReg)
1185 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubRegIdx);
1186
1187 MI.eraseFromParent();
1188 return true;
1189}
1190
1191bool RISCVInstructionSelector::selectInsertSubVector(MachineInstr &MI) const {
1192 assert(MI.getOpcode() == TargetOpcode::G_INSERT_SUBVECTOR);
1193
1194 Register DstReg = MI.getOperand(i: 0).getReg();
1195 Register VecReg = MI.getOperand(i: 1).getReg();
1196 Register SubVecReg = MI.getOperand(i: 2).getReg();
1197
1198 LLT VecTy = MRI->getType(Reg: VecReg);
1199 LLT SubVecTy = MRI->getType(Reg: SubVecReg);
1200
1201 MVT VecMVT = getMVTForLLT(Ty: VecTy);
1202 MVT SubVecMVT = getMVTForLLT(Ty: SubVecTy);
1203
1204 unsigned Idx = static_cast<unsigned>(MI.getOperand(i: 3).getImm());
1205
1206 unsigned SubRegIdx;
1207 std::tie(args&: SubRegIdx, args&: Idx) =
1208 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1209 VecVT: VecMVT, SubVecVT: SubVecMVT, InsertExtractIdx: Idx, TRI: &TRI);
1210
1211 // If the Idx hasn't been completely eliminated then this is a subvector
1212 // insert which doesn't naturally align to a vector register. These must
1213 // be handled using instructions to manipulate the vector registers.
1214 if (Idx != 0)
1215 return false;
1216
1217 // Constrain dst
1218 unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: VecMVT);
1219 const TargetRegisterClass *DstRC = TRI.getRegClass(i: DstRegClassID);
1220 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1221 return false;
1222
1223 // If we haven't set a SubRegIdx, then we must be going between
1224 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
1225 if (SubRegIdx == RISCV::NoSubRegister) {
1226 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecMVT) ==
1227 DstRegClassID &&
1228 "Unexpected subvector insert");
1229 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1230 DestReg: DstReg)
1231 .addReg(RegNo: SubVecReg);
1232 MI.eraseFromParent();
1233 return true;
1234 }
1235
1236 // Use INSERT_SUBREG to insert the subvector into the vector at the
1237 // appropriate subregister index.
1238 MachineInstr *Ins = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1239 MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DstReg)
1240 .addReg(RegNo: VecReg)
1241 .addReg(RegNo: SubVecReg)
1242 .addImm(Val: SubRegIdx);
1243
1244 MI.eraseFromParent();
1245 constrainSelectedInstRegOperands(I&: *Ins, TII, TRI, RBI);
1246 return true;
1247}
1248
1249bool RISCVInstructionSelector::select(MachineInstr &MI) {
1250 preISelLower(MI);
1251 const unsigned Opc = MI.getOpcode();
1252
1253 if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
1254 if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
1255 const Register DefReg = MI.getOperand(i: 0).getReg();
1256 const LLT DefTy = MRI->getType(Reg: DefReg);
1257
1258 const RegClassOrRegBank &RegClassOrBank =
1259 MRI->getRegClassOrRegBank(Reg: DefReg);
1260
1261 const TargetRegisterClass *DefRC =
1262 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
1263 if (!DefRC) {
1264 if (!DefTy.isValid()) {
1265 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
1266 return false;
1267 }
1268
1269 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
1270 DefRC = TRI.getRegClassForTypeOnBank(Ty: DefTy, RB, Is64Bit: STI.is64Bit());
1271 if (!DefRC) {
1272 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
1273 return false;
1274 }
1275 }
1276
1277 MI.setDesc(TII.get(Opcode: TargetOpcode::PHI));
1278 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI&: *MRI);
1279 }
1280
1281 // Certain non-generic instructions also need some special handling.
1282 if (MI.isCopy())
1283 return selectCopy(MI);
1284
1285 return true;
1286 }
1287
1288 if (selectImpl(I&: MI, CoverageInfo&: *CoverageInfo))
1289 return true;
1290
1291 switch (Opc) {
1292 case TargetOpcode::G_ANYEXT:
1293 case TargetOpcode::G_PTRTOINT:
1294 case TargetOpcode::G_INTTOPTR:
1295 case TargetOpcode::G_TRUNC:
1296 case TargetOpcode::G_FREEZE:
1297 return selectCopy(MI);
1298 case TargetOpcode::G_CONSTANT: {
1299 Register DstReg = MI.getOperand(i: 0).getReg();
1300 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
1301
1302 if (!materializeImm(Reg: DstReg, Imm, MI))
1303 return false;
1304
1305 MI.eraseFromParent();
1306 return true;
1307 }
1308 case TargetOpcode::G_ZEXT:
1309 case TargetOpcode::G_SEXT: {
1310 bool IsSigned = Opc != TargetOpcode::G_ZEXT;
1311 Register DstReg = MI.getOperand(i: 0).getReg();
1312 Register SrcReg = MI.getOperand(i: 1).getReg();
1313 LLT SrcTy = MRI->getType(Reg: SrcReg);
1314 unsigned SrcSize = SrcTy.getSizeInBits();
1315
1316 if (SrcTy.isVector())
1317 return false; // Should be handled by imported patterns.
1318
1319 assert((*RBI.getRegBank(DstReg, *MRI, TRI)).getID() ==
1320 RISCV::GPRBRegBankID &&
1321 "Unexpected ext regbank");
1322
1323 // Use addiw SrcReg, 0 (sext.w) for i32.
1324 if (IsSigned && SrcSize == 32) {
1325 MI.setDesc(TII.get(Opcode: RISCV::ADDIW));
1326 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
1327 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1328 return true;
1329 }
1330
1331 // Use add.uw SrcReg, X0 (zext.w) for i32 with Zba.
1332 if (!IsSigned && SrcSize == 32 && STI.hasStdExtZba()) {
1333 MI.setDesc(TII.get(Opcode: RISCV::ADD_UW));
1334 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::X0, /*isDef=*/false));
1335 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1336 return true;
1337 }
1338
1339 // Use sext.h/zext.h for i16 with Zbb.
1340 if (SrcSize == 16 &&
1341 (STI.hasStdExtZbb() || (!IsSigned && STI.hasStdExtZbkb()))) {
1342 MI.setDesc(TII.get(Opcode: IsSigned ? RISCV::SEXT_H
1343 : STI.isRV64() ? RISCV::ZEXT_H_RV64
1344 : RISCV::ZEXT_H_RV32));
1345 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1346 return true;
1347 }
1348
1349 // Fall back to shift pair.
1350 Register ShiftLeftReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1351 MachineInstr *ShiftLeft = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1352 MCID: TII.get(Opcode: RISCV::SLLI), DestReg: ShiftLeftReg)
1353 .addReg(RegNo: SrcReg)
1354 .addImm(Val: STI.getXLen() - SrcSize);
1355 constrainSelectedInstRegOperands(I&: *ShiftLeft, TII, TRI, RBI);
1356 MachineInstr *ShiftRight =
1357 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1358 MCID: TII.get(Opcode: IsSigned ? RISCV::SRAI : RISCV::SRLI), DestReg: DstReg)
1359 .addReg(RegNo: ShiftLeftReg)
1360 .addImm(Val: STI.getXLen() - SrcSize);
1361 constrainSelectedInstRegOperands(I&: *ShiftRight, TII, TRI, RBI);
1362 MI.eraseFromParent();
1363 return true;
1364 }
1365 case TargetOpcode::G_FCONSTANT: {
1366 // TODO: Use constant pool for complex constants.
1367 Register DstReg = MI.getOperand(i: 0).getReg();
1368 const APFloat &FPimm = MI.getOperand(i: 1).getFPImm()->getValueAPF();
1369 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
1370 if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
1371 Register GPRReg;
1372 if (FPimm.isPosZero()) {
1373 GPRReg = RISCV::X0;
1374 } else {
1375 GPRReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1376 APInt Imm = FPimm.bitcastToAPInt();
1377 if (!materializeImm(Reg: GPRReg, Imm: Imm.getSExtValue(), MI))
1378 return false;
1379 }
1380
1381 unsigned Opcode = Size == 64 ? RISCV::FMV_D_X
1382 : Size == 32 ? RISCV::FMV_W_X
1383 : RISCV::FMV_H_X;
1384 MachineInstr *FMV = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1385 MCID: TII.get(Opcode), DestReg: DstReg)
1386 .addReg(RegNo: GPRReg);
1387 constrainSelectedInstRegOperands(I&: *FMV, TII, TRI, RBI);
1388 } else {
1389 // s64 on rv32
1390 assert(Size == 64 && !Subtarget->is64Bit() &&
1391 "Unexpected size or subtarget");
1392
1393 if (FPimm.isPosZero()) {
1394 // Optimize +0.0 to use fcvt.d.w
1395 MachineInstr *FCVT = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1396 MCID: TII.get(Opcode: RISCV::FCVT_D_W), DestReg: DstReg)
1397 .addReg(RegNo: RISCV::X0)
1398 .addImm(Val: RISCVFPRndMode::RNE);
1399 constrainSelectedInstRegOperands(I&: *FCVT, TII, TRI, RBI);
1400
1401 MI.eraseFromParent();
1402 return true;
1403 }
1404
1405 // Split into two pieces and build through the stack.
1406 Register GPRRegHigh = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1407 Register GPRRegLow = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1408 APInt Imm = FPimm.bitcastToAPInt();
1409 if (!materializeImm(Reg: GPRRegHigh, Imm: Imm.extractBits(numBits: 32, bitPosition: 32).getSExtValue(),
1410 MI))
1411 return false;
1412 if (!materializeImm(Reg: GPRRegLow, Imm: Imm.trunc(width: 32).getSExtValue(), MI))
1413 return false;
1414 MachineInstr *PairF64 =
1415 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1416 MCID: TII.get(Opcode: RISCV::BuildPairF64Pseudo), DestReg: DstReg)
1417 .addReg(RegNo: GPRRegLow)
1418 .addReg(RegNo: GPRRegHigh);
1419 constrainSelectedInstRegOperands(I&: *PairF64, TII, TRI, RBI);
1420 }
1421
1422 MI.eraseFromParent();
1423 return true;
1424 }
1425 case TargetOpcode::G_GLOBAL_VALUE: {
1426 auto *GV = MI.getOperand(i: 1).getGlobal();
1427 if (GV->isThreadLocal()) {
1428 // TODO: implement this case.
1429 return false;
1430 }
1431
1432 return selectAddr(MI, IsLocal: GV->isDSOLocal(), IsExternWeak: GV->hasExternalWeakLinkage());
1433 }
1434 case TargetOpcode::G_JUMP_TABLE:
1435 case TargetOpcode::G_CONSTANT_POOL:
1436 return selectAddr(MI);
1437 case TargetOpcode::G_BRCOND: {
1438 Register LHS, RHS;
1439 RISCVCC::CondCode CC;
1440 getOperandsForBranch(CondReg: MI.getOperand(i: 0).getReg(), CC, LHS, RHS, MRI&: *MRI);
1441
1442 MachineInstr *Bcc = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1443 MCID: TII.get(Opcode: RISCVCC::getBrCond(CC)))
1444 .addReg(RegNo: LHS)
1445 .addReg(RegNo: RHS)
1446 .addMBB(MBB: MI.getOperand(i: 1).getMBB());
1447 MI.eraseFromParent();
1448 constrainSelectedInstRegOperands(I&: *Bcc, TII, TRI, RBI);
1449 return true;
1450 }
1451 case TargetOpcode::G_BRINDIRECT:
1452 MI.setDesc(TII.get(Opcode: RISCV::PseudoBRIND));
1453 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
1454 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1455 return true;
1456 case TargetOpcode::G_SELECT:
1457 return selectSelect(MI);
1458 case TargetOpcode::G_FCMP:
1459 return selectFPCompare(MI);
1460 case TargetOpcode::G_FENCE: {
1461 AtomicOrdering FenceOrdering =
1462 static_cast<AtomicOrdering>(MI.getOperand(i: 0).getImm());
1463 SyncScope::ID FenceSSID =
1464 static_cast<SyncScope::ID>(MI.getOperand(i: 1).getImm());
1465 emitFence(FenceOrdering, FenceSSID, MI);
1466 MI.eraseFromParent();
1467 return true;
1468 }
1469 case TargetOpcode::G_IMPLICIT_DEF:
1470 return selectImplicitDef(MI);
1471 case TargetOpcode::G_UNMERGE_VALUES:
1472 return selectUnmergeValues(MI);
1473 case TargetOpcode::G_LOAD:
1474 case TargetOpcode::G_STORE: {
1475 GLoadStore &LdSt = cast<GLoadStore>(Val&: MI);
1476 const Register ValReg = LdSt.getReg(Idx: 0);
1477 const Register PtrReg = LdSt.getPointerReg();
1478 LLT PtrTy = MRI->getType(Reg: PtrReg);
1479
1480 const RegisterBank &RB = *RBI.getRegBank(Reg: ValReg, MRI: *MRI, TRI);
1481 if (RB.getID() != RISCV::GPRBRegBankID)
1482 return false;
1483
1484#ifndef NDEBUG
1485 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, *MRI, TRI);
1486 // Check that the pointer register is valid.
1487 assert(PtrRB.getID() == RISCV::GPRBRegBankID &&
1488 "Load/Store pointer operand isn't a GPR");
1489 assert(PtrTy.isPointer() && "Load/Store pointer operand isn't a pointer");
1490#endif
1491
1492 // Can only handle AddressSpace 0.
1493 if (PtrTy.getAddressSpace() != 0)
1494 return false;
1495
1496 unsigned MemSize = LdSt.getMemSizeInBits().getValue();
1497 AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();
1498
1499 if (isStrongerThanMonotonic(AO: Order)) {
1500 MI.setDesc(TII.get(Opcode: selectZalasrLoadStoreOp(GenericOpc: Opc, OpSize: MemSize)));
1501 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1502 return true;
1503 }
1504
1505 const unsigned NewOpc = selectRegImmLoadStoreOp(GenericOpc: MI.getOpcode(), OpSize: MemSize);
1506 if (NewOpc == MI.getOpcode())
1507 return false;
1508
1509 // Check if we can fold anything into the addressing mode.
1510 auto AddrModeFns = selectAddrRegImm(Root&: MI.getOperand(i: 1));
1511 if (!AddrModeFns)
1512 return false;
1513
1514 // Folded something. Create a new instruction and return it.
1515 MachineInstrBuilder NewInst =
1516 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: NewOpc));
1517 NewInst.setMIFlags(MI.getFlags());
1518 if (isa<GStore>(Val: MI))
1519 NewInst.addUse(RegNo: ValReg);
1520 else
1521 NewInst.addDef(RegNo: ValReg);
1522 NewInst.cloneMemRefs(OtherMI: MI);
1523 for (auto &Fn : *AddrModeFns)
1524 Fn(NewInst);
1525 MI.eraseFromParent();
1526
1527 constrainSelectedInstRegOperands(I&: *NewInst, TII, TRI, RBI);
1528 return true;
1529 }
1530 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1531 return selectIntrinsicWithSideEffects(I&: MI);
1532 case TargetOpcode::G_INTRINSIC:
1533 return selectIntrinsic(I&: MI);
1534 case TargetOpcode::G_EXTRACT_SUBVECTOR:
1535 return selectExtractSubvector(MI);
1536 case TargetOpcode::G_INSERT_SUBVECTOR:
1537 return selectInsertSubVector(MI);
1538 default:
1539 return false;
1540 }
1541}
1542
1543bool RISCVInstructionSelector::selectUnmergeValues(MachineInstr &MI) const {
1544 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
1545
1546 if (!Subtarget->hasStdExtZfa())
1547 return false;
1548
1549 // Split F64 Src into two s32 parts
1550 if (MI.getNumOperands() != 3)
1551 return false;
1552 Register Src = MI.getOperand(i: 2).getReg();
1553 Register Lo = MI.getOperand(i: 0).getReg();
1554 Register Hi = MI.getOperand(i: 1).getReg();
1555 if (!isRegInFprb(Reg: Src) || !isRegInGprb(Reg: Lo) || !isRegInGprb(Reg: Hi))
1556 return false;
1557
1558 MachineInstr *ExtractLo = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1559 MCID: TII.get(Opcode: RISCV::FMV_X_W_FPR64), DestReg: Lo)
1560 .addReg(RegNo: Src);
1561 constrainSelectedInstRegOperands(I&: *ExtractLo, TII, TRI, RBI);
1562
1563 MachineInstr *ExtractHi = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1564 MCID: TII.get(Opcode: RISCV::FMVH_X_D), DestReg: Hi)
1565 .addReg(RegNo: Src);
1566 constrainSelectedInstRegOperands(I&: *ExtractHi, TII, TRI, RBI);
1567
1568 MI.eraseFromParent();
1569 return true;
1570}
1571
1572bool RISCVInstructionSelector::replacePtrWithInt(MachineOperand &Op) {
1573 Register PtrReg = Op.getReg();
1574 assert(MRI->getType(PtrReg).isPointer() && "Operand is not a pointer!");
1575
1576 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1577 MachineInstr &ParentMI = *Op.getParent();
1578 Register IntReg = MRI->createGenericVirtualRegister(Ty: sXLen);
1579 MRI->setRegBank(Reg: IntReg, RegBank: RBI.getRegBank(ID: RISCV::GPRBRegBankID));
1580 MachineInstr *PtrToInt =
1581 BuildMI(BB&: *ParentMI.getParent(), I&: ParentMI, MIMD: ParentMI.getDebugLoc(),
1582 MCID: TII.get(Opcode: TargetOpcode::G_PTRTOINT), DestReg: IntReg)
1583 .addReg(RegNo: PtrReg);
1584 Op.setReg(IntReg);
1585 return select(MI&: *PtrToInt);
1586}
1587
1588void RISCVInstructionSelector::preISelLower(MachineInstr &MI) {
1589 switch (MI.getOpcode()) {
1590 case TargetOpcode::G_PTR_ADD: {
1591 Register DstReg = MI.getOperand(i: 0).getReg();
1592 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1593
1594 replacePtrWithInt(Op&: MI.getOperand(i: 1));
1595 MI.setDesc(TII.get(Opcode: TargetOpcode::G_ADD));
1596 MRI->setType(VReg: DstReg, Ty: sXLen);
1597 break;
1598 }
1599 case TargetOpcode::G_PTRMASK: {
1600 Register DstReg = MI.getOperand(i: 0).getReg();
1601 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1602 replacePtrWithInt(Op&: MI.getOperand(i: 1));
1603 MI.setDesc(TII.get(Opcode: TargetOpcode::G_AND));
1604 MRI->setType(VReg: DstReg, Ty: sXLen);
1605 break;
1606 }
1607 }
1608}
1609
1610void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
1611 const MachineInstr &MI,
1612 int OpIdx) const {
1613 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1614 "Expected G_CONSTANT");
1615 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
1616 MIB.addImm(Val: -CstVal);
1617}
1618
1619void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
1620 const MachineInstr &MI,
1621 int OpIdx) const {
1622 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1623 "Expected G_CONSTANT");
1624 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1625 MIB.addImm(Val: STI.getXLen() - CstVal);
1626}
1627
1628void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
1629 const MachineInstr &MI,
1630 int OpIdx) const {
1631 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1632 "Expected G_CONSTANT");
1633 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1634 MIB.addImm(Val: 32 - CstVal);
1635}
1636
1637void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
1638 const MachineInstr &MI,
1639 int OpIdx) const {
1640 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1641 "Expected G_CONSTANT");
1642 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
1643 MIB.addImm(Val: CstVal + 1);
1644}
1645
1646void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
1647 const MachineInstr &MI,
1648 int OpIdx) const {
1649 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1650 "Expected G_CONSTANT");
1651 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
1652 MIB.addImm(Val: llvm::countr_zero(Val: C));
1653}
1654
1655void RISCVInstructionSelector::renderXLenSubTrailingOnes(
1656 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
1657 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1658 "Expected G_CONSTANT");
1659 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
1660 MIB.addImm(Val: Subtarget->getXLen() - llvm::countr_one(Value: C));
1661}
1662
1663void RISCVInstructionSelector::renderAddiPairImmSmall(MachineInstrBuilder &MIB,
1664 const MachineInstr &MI,
1665 int OpIdx) const {
1666 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1667 "Expected G_CONSTANT");
1668 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
1669 int64_t Adj = Imm < 0 ? -2048 : 2047;
1670 MIB.addImm(Val: Imm - Adj);
1671}
1672
1673void RISCVInstructionSelector::renderAddiPairImmLarge(MachineInstrBuilder &MIB,
1674 const MachineInstr &MI,
1675 int OpIdx) const {
1676 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1677 "Expected G_CONSTANT");
1678 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue() < 0 ? -2048 : 2047;
1679 MIB.addImm(Val: Imm);
1680}
1681
1682bool RISCVInstructionSelector::isRegInGprb(Register Reg) const {
1683 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::GPRBRegBankID;
1684}
1685
1686bool RISCVInstructionSelector::isRegInFprb(Register Reg) const {
1687 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID;
1688}
1689
1690bool RISCVInstructionSelector::selectCopy(MachineInstr &MI) const {
1691 Register DstReg = MI.getOperand(i: 0).getReg();
1692
1693 if (DstReg.isPhysical())
1694 return true;
1695
1696 const TargetRegisterClass *DstRC =
1697 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
1698
1699 assert(DstRC &&
1700 "Register class not available for LLT, register bank combination");
1701
1702 // No need to constrain SrcReg. It will get constrained when
1703 // we hit another of its uses or its defs.
1704 // Copies do not have constraints.
1705 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
1706 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1707 << " operand\n");
1708 return false;
1709 }
1710
1711 MI.setDesc(TII.get(Opcode: RISCV::COPY));
1712 return true;
1713}
1714
1715bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI) const {
1716 assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
1717
1718 const Register DstReg = MI.getOperand(i: 0).getReg();
1719 const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(
1720 Ty: MRI->getType(Reg: DstReg), RB: *RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI), Is64Bit: STI.is64Bit());
1721
1722 assert(DstRC &&
1723 "Register class not available for LLT, register bank combination");
1724
1725 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
1726 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1727 << " operand\n");
1728 }
1729 MI.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
1730 return true;
1731}
1732
1733bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
1734 MachineInstr &MI) const {
1735 if (Imm == 0) {
1736 MachineBasicBlock &MBB = *MI.getParent();
1737 DebugLoc DL = MI.getDebugLoc();
1738 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg).addReg(RegNo: RISCV::X0);
1739 RBI.constrainGenericRegister(Reg: DstReg, RC: RISCV::GPRRegClass, MRI&: *MRI);
1740 return true;
1741 }
1742
1743 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: *Subtarget);
1744 return materializeInstSeq(DstReg, Seq, MI);
1745}
1746
1747bool RISCVInstructionSelector::materializeInstSeq(
1748 Register DstReg, const RISCVMatInt::InstSeq &Seq, MachineInstr &MI) const {
1749 assert(!Seq.empty() && "materializeInstSeq requires a non-empty sequence");
1750
1751 MachineBasicBlock &MBB = *MI.getParent();
1752 DebugLoc DL = MI.getDebugLoc();
1753 unsigned NumInsts = Seq.size();
1754 Register SrcReg = RISCV::X0;
1755
1756 for (unsigned i = 0; i < NumInsts; i++) {
1757 Register TmpReg = i < NumInsts - 1
1758 ? MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass)
1759 : DstReg;
1760 const RISCVMatInt::Inst &I = Seq[i];
1761 MachineInstr *Result;
1762
1763 switch (I.getOpndKind()) {
1764 case RISCVMatInt::Imm:
1765 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1766 .addImm(Val: I.getImm());
1767 break;
1768 case RISCVMatInt::RegX0:
1769 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1770 .addReg(RegNo: SrcReg)
1771 .addReg(RegNo: RISCV::X0);
1772 break;
1773 case RISCVMatInt::RegReg:
1774 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1775 .addReg(RegNo: SrcReg)
1776 .addReg(RegNo: SrcReg);
1777 break;
1778 case RISCVMatInt::RegImm:
1779 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1780 .addReg(RegNo: SrcReg)
1781 .addImm(Val: I.getImm());
1782 break;
1783 }
1784
1785 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1786
1787 SrcReg = TmpReg;
1788 }
1789
1790 return true;
1791}
1792
1793InstructionSelector::ComplexRendererFns
1794RISCVInstructionSelector::computeConstAddr(int64_t CVal, bool IsPrefetch,
1795 Register OrigBase) const {
1796 // Split the constant into a materialized high part (the base) and
1797 // a simm12 low part (the offset). For prefetch the low part
1798 // must additionally be a multiple of 32 (simm12_lsb00000).
1799 int64_t Lo12 = SignExtend64<12>(x: CVal);
1800 int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
1801 auto emit = [&](ConstAddrPlan Plan) -> ComplexRendererFns {
1802 return {{[=](MachineInstrBuilder &MIB) {
1803 MIB.addReg(RegNo: materializeConstBase(MIB, Plan, OrigBase));
1804 },
1805 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Plan.Lo12); }}};
1806 };
1807 if (!Subtarget->is64Bit() || isInt<32>(x: Hi)) {
1808 if (IsPrefetch && (Lo12 & 0b11111) != 0)
1809 return std::nullopt;
1810 ConstAddrPlan Plan;
1811 Plan.Lo12 = Lo12;
1812 if (Hi) {
1813 Plan.Kind = ConstAddrPlan::LUI;
1814 Plan.Hi20 = (Hi >> 12) & 0xfffff;
1815 }
1816 return emit(std::move(Plan));
1817 }
1818
1819 // Otherwise ask constant materialization how it would handle the constant
1820 // and fold the trailing ADDI into the offset.
1821 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: CVal, STI: *Subtarget);
1822 if (Seq.back().getOpcode() != RISCV::ADDI)
1823 return std::nullopt;
1824 Lo12 = Seq.back().getImm();
1825 if (IsPrefetch && (Lo12 & 0b11111) != 0)
1826 return std::nullopt;
1827 Seq.pop_back();
1828 if (Seq.empty())
1829 return std::nullopt;
1830 ConstAddrPlan Plan;
1831 Plan.Kind = ConstAddrPlan::InstSeq;
1832 Plan.Seq = std::move(Seq);
1833 Plan.Lo12 = Lo12;
1834 return emit(std::move(Plan));
1835}
1836
1837Register
1838RISCVInstructionSelector::materializeConstBase(MachineInstrBuilder &MIB,
1839 const ConstAddrPlan &Plan,
1840 Register OrigBase) const {
1841 MachineBasicBlock &MBB = *MIB->getParent();
1842 DebugLoc DL = MIB->getDebugLoc();
1843 MachineInstr &InsertPt = *MIB.getInstr();
1844
1845 Register HiReg = RISCV::X0;
1846 switch (Plan.Kind) {
1847 case ConstAddrPlan::X0:
1848 break;
1849 case ConstAddrPlan::LUI: {
1850 HiReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1851 MachineInstr *LUI = BuildMI(BB&: MBB, I&: InsertPt, MIMD: DL, MCID: TII.get(Opcode: RISCV::LUI), DestReg: HiReg)
1852 .addImm(Val: Plan.Hi20);
1853 constrainSelectedInstRegOperands(I&: *LUI, TII, TRI, RBI);
1854 break;
1855 }
1856 case ConstAddrPlan::InstSeq: {
1857 HiReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1858 materializeInstSeq(DstReg: HiReg, Seq: Plan.Seq, MI&: InsertPt);
1859 break;
1860 }
1861 }
1862
1863 // For G_PTR_ADD + large constant, add the original base to the materialized
1864 // high part.
1865 if (OrigBase.isValid() && HiReg != RISCV::X0) {
1866 Register BaseReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1867 MachineInstr *Add = BuildMI(BB&: MBB, I&: InsertPt, MIMD: DL, MCID: TII.get(Opcode: RISCV::ADD), DestReg: BaseReg)
1868 .addReg(RegNo: OrigBase)
1869 .addReg(RegNo: HiReg);
1870 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
1871 return BaseReg;
1872 }
1873 return OrigBase.isValid() ? OrigBase : HiReg;
1874}
1875
1876bool RISCVInstructionSelector::selectAddr(MachineInstr &MI, bool IsLocal,
1877 bool IsExternWeak) const {
1878 assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1879 MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
1880 MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
1881 "Unexpected opcode");
1882
1883 const MachineOperand &DispMO = MI.getOperand(i: 1);
1884
1885 Register DefReg = MI.getOperand(i: 0).getReg();
1886 const LLT DefTy = MRI->getType(Reg: DefReg);
1887
1888 // When HWASAN is used and tagging of global variables is enabled
1889 // they should be accessed via the GOT, since the tagged address of a global
1890 // is incompatible with existing code models. This also applies to non-pic
1891 // mode.
1892 if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
1893 if (IsLocal && !Subtarget->allowTaggedGlobals()) {
1894 // Use PC-relative addressing to access the symbol. This generates the
1895 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
1896 // %pcrel_lo(auipc)).
1897 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
1898 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1899 return true;
1900 }
1901
1902 // Use PC-relative addressing to access the GOT for this symbol, then
1903 // load the address from the GOT. This generates the pattern (PseudoLGA
1904 // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
1905 // %pcrel_lo(auipc))).
1906 MachineFunction &MF = *MI.getParent()->getParent();
1907 MachineMemOperand *MemOp = MF.getMachineMemOperand(
1908 PtrInfo: MachinePointerInfo::getGOT(MF),
1909 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1910 MachineMemOperand::MOInvariant,
1911 MemTy: DefTy, BaseAlignment: Align(DefTy.getSizeInBits() / 8));
1912
1913 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1914 MCID: TII.get(Opcode: RISCV::PseudoLGA), DestReg: DefReg)
1915 .addDisp(Disp: DispMO, off: 0)
1916 .addMemOperand(MMO: MemOp);
1917
1918 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1919
1920 MI.eraseFromParent();
1921 return true;
1922 }
1923
1924 switch (TM.getCodeModel()) {
1925 default: {
1926 reportGISelFailure(MF&: *MF, MORE&: *MORE, PassName: getName(),
1927 Msg: "Unsupported code model for lowering", MI);
1928 return false;
1929 }
1930 case CodeModel::Small: {
1931 // Must lie within a single 2 GiB address range and must lie between
1932 // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
1933 // (lui %hi(sym)) %lo(sym)).
1934 Register AddrHiDest = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1935 MachineInstr *AddrHi = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1936 MCID: TII.get(Opcode: RISCV::LUI), DestReg: AddrHiDest)
1937 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_HI);
1938
1939 constrainSelectedInstRegOperands(I&: *AddrHi, TII, TRI, RBI);
1940
1941 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1942 MCID: TII.get(Opcode: RISCV::ADDI), DestReg: DefReg)
1943 .addReg(RegNo: AddrHiDest)
1944 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_LO);
1945
1946 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1947
1948 MI.eraseFromParent();
1949 return true;
1950 }
1951 case CodeModel::Medium:
1952 // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
1953 // relocation needs to reference a label that points to the auipc
1954 // instruction itself, not the global. This cannot be done inside the
1955 // instruction selector.
1956 if (IsExternWeak) {
1957 // An extern weak symbol may be undefined, i.e. have value 0, which may
1958 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
1959 // symbol. This generates the pattern (PseudoLGA sym), which expands to
1960 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
1961 MachineFunction &MF = *MI.getParent()->getParent();
1962 MachineMemOperand *MemOp = MF.getMachineMemOperand(
1963 PtrInfo: MachinePointerInfo::getGOT(MF),
1964 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1965 MachineMemOperand::MOInvariant,
1966 MemTy: DefTy, BaseAlignment: Align(DefTy.getSizeInBits() / 8));
1967
1968 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1969 MCID: TII.get(Opcode: RISCV::PseudoLGA), DestReg: DefReg)
1970 .addDisp(Disp: DispMO, off: 0)
1971 .addMemOperand(MMO: MemOp);
1972
1973 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1974
1975 MI.eraseFromParent();
1976 return true;
1977 }
1978
1979 // Generate a sequence for accessing addresses within any 2GiB range
1980 // within the address space. This generates the pattern (PseudoLLA sym),
1981 // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
1982 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
1983 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1984 return true;
1985 }
1986
1987 return false;
1988}
1989
1990bool RISCVInstructionSelector::selectSelect(MachineInstr &MI) const {
1991 auto &SelectMI = cast<GSelect>(Val&: MI);
1992
1993 Register LHS, RHS;
1994 RISCVCC::CondCode CC;
1995 getOperandsForBranch(CondReg: SelectMI.getCondReg(), CC, LHS, RHS, MRI&: *MRI);
1996
1997 Register DstReg = SelectMI.getReg(Idx: 0);
1998
1999 unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
2000 if (RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
2001 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
2002 Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
2003 : RISCV::Select_FPR64_Using_CC_GPR;
2004 }
2005
2006 MachineInstr *Result =
2007 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: Opc))
2008 .addDef(RegNo: DstReg)
2009 .addReg(RegNo: LHS)
2010 .addReg(RegNo: RHS)
2011 .addImm(Val: CC)
2012 .addReg(RegNo: SelectMI.getTrueReg())
2013 .addReg(RegNo: SelectMI.getFalseReg());
2014 MI.eraseFromParent();
2015 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
2016 return true;
2017}
2018
2019// Convert an FCMP predicate to one of the supported F or D instructions.
2020static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
2021 assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");
2022 switch (Pred) {
2023 default:
2024 llvm_unreachable("Unsupported predicate");
2025 case CmpInst::FCMP_OLT:
2026 return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
2027 case CmpInst::FCMP_OLE:
2028 return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
2029 case CmpInst::FCMP_OEQ:
2030 return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
2031 }
2032}
2033
2034// Try legalizing an FCMP by swapping or inverting the predicate to one that
2035// is supported.
2036static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,
2037 CmpInst::Predicate &Pred, bool &NeedInvert) {
2038 auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
2039 return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
2040 Pred == CmpInst::FCMP_OEQ;
2041 };
2042
2043 assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
2044
2045 CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(pred: Pred);
2046 if (isLegalFCmpPredicate(InvPred)) {
2047 Pred = InvPred;
2048 std::swap(a&: LHS, b&: RHS);
2049 return true;
2050 }
2051
2052 InvPred = CmpInst::getInversePredicate(pred: Pred);
2053 NeedInvert = true;
2054 if (isLegalFCmpPredicate(InvPred)) {
2055 Pred = InvPred;
2056 return true;
2057 }
2058 InvPred = CmpInst::getSwappedPredicate(pred: InvPred);
2059 if (isLegalFCmpPredicate(InvPred)) {
2060 Pred = InvPred;
2061 std::swap(a&: LHS, b&: RHS);
2062 return true;
2063 }
2064
2065 return false;
2066}
2067
2068// Emit a sequence of instructions to compare LHS and RHS using Pred. Return
2069// the result in DstReg.
2070// FIXME: Maybe we should expand this earlier.
2071bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI) const {
2072 auto &CmpMI = cast<GFCmp>(Val&: MI);
2073 CmpInst::Predicate Pred = CmpMI.getCond();
2074
2075 Register DstReg = CmpMI.getReg(Idx: 0);
2076 Register LHS = CmpMI.getLHSReg();
2077 Register RHS = CmpMI.getRHSReg();
2078
2079 unsigned Size = MRI->getType(Reg: LHS).getSizeInBits();
2080 assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
2081
2082 Register TmpReg = DstReg;
2083
2084 bool NeedInvert = false;
2085 // First try swapping operands or inverting.
2086 if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
2087 if (NeedInvert)
2088 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2089 MachineInstr *Cmp = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2090 MCID: TII.get(Opcode: getFCmpOpcode(Pred, Size)), DestReg: TmpReg)
2091 .addReg(RegNo: LHS)
2092 .addReg(RegNo: RHS);
2093 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
2094 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
2095 // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
2096 NeedInvert = Pred == CmpInst::FCMP_UEQ;
2097 Register Cmp1Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2098 MachineInstr *Cmp1 =
2099 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2100 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size)), DestReg: Cmp1Reg)
2101 .addReg(RegNo: LHS)
2102 .addReg(RegNo: RHS);
2103 constrainSelectedInstRegOperands(I&: *Cmp1, TII, TRI, RBI);
2104 Register Cmp2Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2105 MachineInstr *Cmp2 =
2106 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2107 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size)), DestReg: Cmp2Reg)
2108 .addReg(RegNo: RHS)
2109 .addReg(RegNo: LHS);
2110 constrainSelectedInstRegOperands(I&: *Cmp2, TII, TRI, RBI);
2111 if (NeedInvert)
2112 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2113 MachineInstr *Or = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2114 MCID: TII.get(Opcode: RISCV::OR), DestReg: TmpReg)
2115 .addReg(RegNo: Cmp1Reg)
2116 .addReg(RegNo: Cmp2Reg);
2117 constrainSelectedInstRegOperands(I&: *Or, TII, TRI, RBI);
2118 } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
2119 // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
2120 // If LHS and RHS are the same, a single FEQ suffices.
2121 NeedInvert = Pred == CmpInst::FCMP_UNO;
2122 if (NeedInvert)
2123 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2124 if (LHS == RHS) {
2125 MachineInstr *Cmp =
2126 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2127 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: TmpReg)
2128 .addReg(RegNo: LHS)
2129 .addReg(RegNo: LHS);
2130 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
2131 } else {
2132 Register Cmp1Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2133 MachineInstr *Cmp1 =
2134 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2135 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: Cmp1Reg)
2136 .addReg(RegNo: LHS)
2137 .addReg(RegNo: LHS);
2138 constrainSelectedInstRegOperands(I&: *Cmp1, TII, TRI, RBI);
2139 Register Cmp2Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2140 MachineInstr *Cmp2 =
2141 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2142 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: Cmp2Reg)
2143 .addReg(RegNo: RHS)
2144 .addReg(RegNo: RHS);
2145 constrainSelectedInstRegOperands(I&: *Cmp2, TII, TRI, RBI);
2146 MachineInstr *And = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2147 MCID: TII.get(Opcode: RISCV::AND), DestReg: TmpReg)
2148 .addReg(RegNo: Cmp1Reg)
2149 .addReg(RegNo: Cmp2Reg);
2150 constrainSelectedInstRegOperands(I&: *And, TII, TRI, RBI);
2151 }
2152 } else
2153 llvm_unreachable("Unhandled predicate");
2154
2155 // Emit an XORI to invert the result if needed.
2156 if (NeedInvert) {
2157 MachineInstr *Xor = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2158 MCID: TII.get(Opcode: RISCV::XORI), DestReg: DstReg)
2159 .addReg(RegNo: TmpReg)
2160 .addImm(Val: 1);
2161 constrainSelectedInstRegOperands(I&: *Xor, TII, TRI, RBI);
2162 }
2163
2164 MI.eraseFromParent();
2165 return true;
2166}
2167
2168void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
2169 SyncScope::ID FenceSSID,
2170 MachineInstr &MI) const {
2171 MachineBasicBlock &MBB = *MI.getParent();
2172 DebugLoc DL = MI.getDebugLoc();
2173
2174 if (STI.hasStdExtZtso()) {
2175 // The only fence that needs an instruction is a sequentially-consistent
2176 // cross-thread fence.
2177 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
2178 FenceSSID == SyncScope::System) {
2179 // fence rw, rw
2180 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE))
2181 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W)
2182 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W);
2183 return;
2184 }
2185
2186 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
2187 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER));
2188 return;
2189 }
2190
2191 // singlethread fences only synchronize with signal handlers on the same
2192 // thread and thus only need to preserve instruction order, not actually
2193 // enforce memory ordering.
2194 if (FenceSSID == SyncScope::SingleThread) {
2195 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER));
2196 return;
2197 }
2198
2199 // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
2200 // Manual: Volume I.
2201 unsigned Pred, Succ;
2202 switch (FenceOrdering) {
2203 default:
2204 llvm_unreachable("Unexpected ordering");
2205 case AtomicOrdering::AcquireRelease:
2206 // fence acq_rel -> fence.tso
2207 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE_TSO));
2208 return;
2209 case AtomicOrdering::Acquire:
2210 // fence acquire -> fence r, rw
2211 Pred = RISCVFenceField::R;
2212 Succ = RISCVFenceField::R | RISCVFenceField::W;
2213 break;
2214 case AtomicOrdering::Release:
2215 // fence release -> fence rw, w
2216 Pred = RISCVFenceField::R | RISCVFenceField::W;
2217 Succ = RISCVFenceField::W;
2218 break;
2219 case AtomicOrdering::SequentiallyConsistent:
2220 // fence seq_cst -> fence rw, rw
2221 Pred = RISCVFenceField::R | RISCVFenceField::W;
2222 Succ = RISCVFenceField::R | RISCVFenceField::W;
2223 break;
2224 }
2225 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE)).addImm(Val: Pred).addImm(Val: Succ);
2226}
2227
2228namespace llvm {
2229InstructionSelector *
2230createRISCVInstructionSelector(const RISCVTargetMachine &TM,
2231 const RISCVSubtarget &Subtarget,
2232 const RISCVRegisterBankInfo &RBI) {
2233 return new RISCVInstructionSelector(TM, Subtarget, RBI);
2234}
2235} // end namespace llvm
2236