1//===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// RISC-V.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/RISCVMatInt.h"
15#include "RISCVRegisterBankInfo.h"
16#include "RISCVSubtarget.h"
17#include "RISCVTargetMachine.h"
18#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
19#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
20#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
21#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
22#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/MachineJumpTableInfo.h"
25#include "llvm/IR/IntrinsicsRISCV.h"
26#include "llvm/Support/Debug.h"
27
28#define DEBUG_TYPE "riscv-isel"
29
30using namespace llvm;
31using namespace MIPatternMatch;
32
33#define GET_GLOBALISEL_PREDICATE_BITSET
34#include "RISCVGenGlobalISel.inc"
35#undef GET_GLOBALISEL_PREDICATE_BITSET
36
37namespace {
38
39class RISCVInstructionSelector : public InstructionSelector {
40public:
41 RISCVInstructionSelector(const RISCVTargetMachine &TM,
42 const RISCVSubtarget &STI,
43 const RISCVRegisterBankInfo &RBI);
44
45 bool select(MachineInstr &MI) override;
46
47 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
48 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
49 BlockFrequencyInfo *BFI) override {
50 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
51 MRI = &MF.getRegInfo();
52 }
53
54 static const char *getName() { return DEBUG_TYPE; }
55
56private:
57 static constexpr unsigned MaxRecursionDepth = 6;
58
59 bool hasAllNBitUsers(const MachineInstr &MI, unsigned Bits,
60 const unsigned Depth = 0) const;
61 bool hasAllHUsers(const MachineInstr &MI) const {
62 return hasAllNBitUsers(MI, Bits: 16);
63 }
64 bool hasAllWUsers(const MachineInstr &MI) const {
65 return hasAllNBitUsers(MI, Bits: 32);
66 }
67
68 bool isRegInGprb(Register Reg) const;
69 bool isRegInFprb(Register Reg) const;
70 bool isWorthFoldingAdd(Register AddResult) const;
71
72 // tblgen-erated 'select' implementation, used as the initial selector for
73 // the patterns that don't require complex C++.
74 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
75
76 // A lowering phase that runs before any selection attempts.
77 // Returns true if the instruction was modified.
78 void preISelLower(MachineInstr &MI);
79
80 bool replacePtrWithInt(MachineInstr &MI, unsigned OpIdx);
81
82 // Custom selection methods
83 bool selectCopy(MachineInstr &MI) const;
84 bool selectImplicitDef(MachineInstr &MI) const;
85 bool materializeImm(Register Reg, int64_t Imm, MachineInstr &MI) const;
86 // Emit a constant-materialization instruction sequence.
87 bool materializeInstSeq(Register DstReg, const RISCVMatInt::InstSeq &Seq,
88 MachineInstr &MI) const;
89 bool selectAddr(MachineInstr &MI, bool IsLocal = true,
90 bool IsExternWeak = false) const;
91 bool selectSelect(MachineInstr &MI) const;
92 bool selectFPCompare(MachineInstr &MI) const;
93 void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
94 MachineInstr &MI) const;
95 bool selectUnmergeValues(MachineInstr &MI) const;
96 void addVectorLoadStoreOperands(MachineInstr &I,
97 SmallVectorImpl<Register> &SrcOps,
98 unsigned &CurOp, bool IsMasked,
99 bool IsStridedOrIndexed,
100 LLT *IndexVT = nullptr) const;
101 bool selectIntrinsicWithSideEffects(MachineInstr &I) const;
102 bool selectIntrinsic(MachineInstr &I) const;
103 bool selectExtractSubvector(MachineInstr &MI) const;
104 bool selectInsertSubVector(MachineInstr &I) const;
105 ComplexRendererFns selectShiftMask(MachineOperand &Root,
106 unsigned ShiftWidth) const;
107 ComplexRendererFns selectShiftMaskXLen(MachineOperand &Root) const {
108 return selectShiftMask(Root, ShiftWidth: STI.getXLen());
109 }
110 ComplexRendererFns selectShiftMask32(MachineOperand &Root) const {
111 return selectShiftMask(Root, ShiftWidth: 32);
112 }
113 ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
114 ComplexRendererFns selectBrindRegImm(MachineOperand &Root) const;
115 ComplexRendererFns selectAddrRegImmLsb00000(MachineOperand &Root) const;
116
117 // Plan for materializing a constant address as (Hi materialization, Lo12
118 // offset). Lo12 is a simm12 that, for prefetch (IsPrefetch), must be
119 // a multiple of 32.
120 struct ConstAddrPlan {
121 enum { X0, LUI, InstSeq } Kind = X0;
122 int64_t Hi20 = 0;
123 RISCVMatInt::InstSeq Seq;
124 int64_t Lo12 = 0;
125 };
126 ComplexRendererFns computeConstAddr(int64_t CVal, bool IsPrefetch,
127 Register OrigBase) const;
128 // Materialize the high part of Plan into a register. If OrigBase is valid,
129 // ADD it to the materialized high part (for G_PTR_ADD + large constant).
130 Register materializeConstBase(MachineInstrBuilder &MIB,
131 const ConstAddrPlan &Plan,
132 Register OrigBase) const;
133
134 ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;
135 template <unsigned Bits>
136 ComplexRendererFns selectSExtBits(MachineOperand &Root) const {
137 return selectSExtBits(Root, Bits);
138 }
139
140 ComplexRendererFns selectZExtBits(MachineOperand &Root, unsigned Bits) const;
141 template <unsigned Bits>
142 ComplexRendererFns selectZExtBits(MachineOperand &Root) const {
143 return selectZExtBits(Root, Bits);
144 }
145
146 ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
147 template <unsigned ShAmt>
148 ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
149 return selectSHXADDOp(Root, ShAmt);
150 }
151
152 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
153 unsigned ShAmt) const;
154 template <unsigned ShAmt>
155 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
156 return selectSHXADD_UWOp(Root, ShAmt);
157 }
158
159 ComplexRendererFns renderVLOp(MachineOperand &Root) const;
160 ComplexRendererFns renderAddiPair(Register BaseReg, int64_t AddiImm,
161 int64_t OffsetImm) const;
162 // Custom renderers for tablegen
163 void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
164 int OpIdx) const;
165 void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
166 int OpIdx) const;
167 void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
168 int OpIdx) const;
169 void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
170 int OpIdx) const;
171
172 void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
173 int OpIdx) const;
174 void renderXLenSubTrailingOnes(MachineInstrBuilder &MIB,
175 const MachineInstr &MI, int OpIdx) const;
176
177 void renderAddiPairImmLarge(MachineInstrBuilder &MIB, const MachineInstr &MI,
178 int OpIdx) const;
179 void renderAddiPairImmSmall(MachineInstrBuilder &MIB, const MachineInstr &MI,
180 int OpIdx) const;
181
182 const RISCVSubtarget &STI;
183 const RISCVInstrInfo &TII;
184 const RISCVRegisterInfo &TRI;
185 const RISCVRegisterBankInfo &RBI;
186 const RISCVTargetMachine &TM;
187
188 MachineRegisterInfo *MRI = nullptr;
189
190 // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
191 // uses "STI." in the code generated by TableGen. We need to unify the name of
192 // Subtarget variable.
193 const RISCVSubtarget *Subtarget = &STI;
194
195#define GET_GLOBALISEL_PREDICATES_DECL
196#include "RISCVGenGlobalISel.inc"
197#undef GET_GLOBALISEL_PREDICATES_DECL
198
199#define GET_GLOBALISEL_TEMPORARIES_DECL
200#include "RISCVGenGlobalISel.inc"
201#undef GET_GLOBALISEL_TEMPORARIES_DECL
202};
203
204} // end anonymous namespace
205
206#define GET_GLOBALISEL_IMPL
207#include "RISCVGenGlobalISel.inc"
208#undef GET_GLOBALISEL_IMPL
209
210RISCVInstructionSelector::RISCVInstructionSelector(
211 const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
212 const RISCVRegisterBankInfo &RBI)
213 : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
214 TM(TM),
215
216#define GET_GLOBALISEL_PREDICATES_INIT
217#include "RISCVGenGlobalISel.inc"
218#undef GET_GLOBALISEL_PREDICATES_INIT
219#define GET_GLOBALISEL_TEMPORARIES_INIT
220#include "RISCVGenGlobalISel.inc"
221#undef GET_GLOBALISEL_TEMPORARIES_INIT
222{
223}
224
225// Mimics optimizations in ISel and RISCVOptWInst Pass
226bool RISCVInstructionSelector::hasAllNBitUsers(const MachineInstr &MI,
227 unsigned Bits,
228 const unsigned Depth) const {
229
230 assert((MI.getOpcode() == TargetOpcode::G_ADD ||
231 MI.getOpcode() == TargetOpcode::G_SUB ||
232 MI.getOpcode() == TargetOpcode::G_MUL ||
233 MI.getOpcode() == TargetOpcode::G_SHL ||
234 MI.getOpcode() == TargetOpcode::G_LSHR ||
235 MI.getOpcode() == TargetOpcode::G_AND ||
236 MI.getOpcode() == TargetOpcode::G_OR ||
237 MI.getOpcode() == TargetOpcode::G_XOR ||
238 MI.getOpcode() == TargetOpcode::G_SEXT_INREG || Depth != 0) &&
239 "Unexpected opcode");
240
241 if (Depth >= RISCVInstructionSelector::MaxRecursionDepth)
242 return false;
243
244 auto DestReg = MI.getOperand(i: 0).getReg();
245 for (auto &UserOp : MRI->use_nodbg_operands(Reg: DestReg)) {
246 assert(UserOp.getParent() && "UserOp must have a parent");
247 const MachineInstr &UserMI = *UserOp.getParent();
248 unsigned OpIdx = UserOp.getOperandNo();
249
250 switch (UserMI.getOpcode()) {
251 default:
252 return false;
253 case RISCV::ADDW:
254 case RISCV::ADDIW:
255 case RISCV::SUBW:
256 case RISCV::FCVT_D_W:
257 case RISCV::FCVT_S_W:
258 if (Bits >= 32)
259 break;
260 return false;
261 case RISCV::SLL:
262 case RISCV::SRA:
263 case RISCV::SRL:
264 // Shift amount operands only use log2(Xlen) bits.
265 if (OpIdx == 2 && Bits >= Log2_32(Value: Subtarget->getXLen()))
266 break;
267 return false;
268 case RISCV::SLLI:
269 // SLLI only uses the lower (XLen - ShAmt) bits.
270 if (Bits >= Subtarget->getXLen() - UserMI.getOperand(i: 2).getImm())
271 break;
272 return false;
273 case RISCV::ANDI:
274 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(
275 Value: (uint64_t)UserMI.getOperand(i: 2).getImm()))
276 break;
277 goto RecCheck;
278 case RISCV::AND:
279 case RISCV::OR:
280 case RISCV::XOR:
281 RecCheck:
282 if (hasAllNBitUsers(MI: UserMI, Bits, Depth: Depth + 1))
283 break;
284 return false;
285 case RISCV::SRLI: {
286 unsigned ShAmt = UserMI.getOperand(i: 2).getImm();
287 // If we are shifting right by less than Bits, and users don't demand any
288 // bits that were shifted into [Bits-1:0], then we can consider this as an
289 // N-Bit user.
290 if (Bits > ShAmt && hasAllNBitUsers(MI: UserMI, Bits: Bits - ShAmt, Depth: Depth + 1))
291 break;
292 return false;
293 }
294 }
295 }
296
297 return true;
298}
299
300InstructionSelector::ComplexRendererFns
301RISCVInstructionSelector::selectShiftMask(MachineOperand &Root,
302 unsigned ShiftWidth) const {
303 if (!Root.isReg())
304 return std::nullopt;
305
306 using namespace llvm::MIPatternMatch;
307
308 Register ShAmtReg = Root.getReg();
309 // Peek through zext.
310 Register ZExtSrcReg;
311 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrcReg))))
312 ShAmtReg = ZExtSrcReg;
313
314 APInt AndMask;
315 Register AndSrcReg;
316 // Try to combine the following pattern (applicable to other shift
317 // instructions as well as 32-bit ones):
318 //
319 // %4:gprb(s64) = G_AND %3, %2
320 // %5:gprb(s64) = G_LSHR %1, %4(s64)
321 //
322 // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
323 // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
324 // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
325 // then it can be eliminated. Given register rs1 or rs2 holding a constant
326 // (the and mask), there are two cases G_AND can be erased:
327 //
328 // 1. the lowest log2(XLEN) bits of the and mask are all set
329 // 2. the bits of the register being masked are already unset (zero set)
330 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: AndSrcReg), R: m_ICst(Cst&: AndMask)))) {
331 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
332 if (ShMask.isSubsetOf(RHS: AndMask)) {
333 ShAmtReg = AndSrcReg;
334 } else {
335 // SimplifyDemandedBits may have optimized the mask so try restoring any
336 // bits that are known zero.
337 KnownBits Known = VT->getKnownBits(R: AndSrcReg);
338 if (ShMask.isSubsetOf(RHS: AndMask | Known.Zero))
339 ShAmtReg = AndSrcReg;
340 }
341 }
342
343 APInt Imm;
344 Register Reg;
345 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAdd(L: m_Reg(R&: Reg), R: m_ICst(Cst&: Imm)))) {
346 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0)
347 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
348 // to avoid the ADD.
349 ShAmtReg = Reg;
350 } else if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GSub(L: m_ICst(Cst&: Imm), R: m_Reg(R&: Reg)))) {
351 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0) {
352 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
353 // to generate a NEG instead of a SUB of a constant.
354 ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
355 unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
356 return {{[=](MachineInstrBuilder &MIB) {
357 MachineIRBuilder(*MIB.getInstr())
358 .buildInstr(Opc: NegOpc, DstOps: {ShAmtReg}, SrcOps: {Register(RISCV::X0), Reg});
359 MIB.addReg(RegNo: ShAmtReg);
360 }}};
361 }
362 if (Imm.urem(RHS: ShiftWidth) == ShiftWidth - 1) {
363 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
364 // to generate a NOT instead of a SUB of a constant.
365 ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
366 return {{[=](MachineInstrBuilder &MIB) {
367 MachineIRBuilder(*MIB.getInstr())
368 .buildInstr(Opc: RISCV::XORI, DstOps: {ShAmtReg}, SrcOps: {Reg})
369 .addImm(Val: -1);
370 MIB.addReg(RegNo: ShAmtReg);
371 }}};
372 }
373 }
374
375 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ShAmtReg); }}};
376}
377
378InstructionSelector::ComplexRendererFns
379RISCVInstructionSelector::selectSExtBits(MachineOperand &Root,
380 unsigned Bits) const {
381 if (!Root.isReg())
382 return std::nullopt;
383 Register RootReg = Root.getReg();
384
385 Register SrcReg;
386 if (mi_match(R: RootReg, MRI: *MRI,
387 P: m_GSExtInReg(Src: m_Reg(R&: SrcReg), Imm: m_SpecificImm(RequestedValue: Bits)))) {
388 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SrcReg); }}};
389 }
390
391 unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits();
392 if ((Size - VT->computeNumSignBits(R: RootReg)) < Bits)
393 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}};
394
395 return std::nullopt;
396}
397
398InstructionSelector::ComplexRendererFns
399RISCVInstructionSelector::selectZExtBits(MachineOperand &Root,
400 unsigned Bits) const {
401 if (!Root.isReg())
402 return std::nullopt;
403 Register RootReg = Root.getReg();
404
405 Register RegX;
406 uint64_t Mask = maskTrailingOnes<uint64_t>(N: Bits);
407 if (mi_match(R: RootReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: RegX), R: m_SpecificICst(RequestedValue: Mask)))) {
408 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}};
409 }
410
411 if (mi_match(R: RootReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: RegX))) &&
412 MRI->getType(Reg: RegX).getScalarSizeInBits() == Bits)
413 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}};
414
415 unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits();
416 if (VT->maskedValueIsZero(Val: RootReg, Mask: APInt::getBitsSetFrom(numBits: Size, loBit: Bits)))
417 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}};
418
419 return std::nullopt;
420}
421
422InstructionSelector::ComplexRendererFns
423RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
424 unsigned ShAmt) const {
425 using namespace llvm::MIPatternMatch;
426
427 if (!Root.isReg())
428 return std::nullopt;
429 Register RootReg = Root.getReg();
430
431 const unsigned XLen = STI.getXLen();
432 APInt Mask;
433 int64_t C2;
434 Register RegY;
435 std::optional<bool> LeftShift;
436 // (and (shl y, c2), mask)
437 if (mi_match(R: RootReg, MRI: *MRI,
438 P: m_GAnd(L: m_GShl(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
439 LeftShift = true;
440 // (and (lshr y, c2), mask)
441 else if (mi_match(R: RootReg, MRI: *MRI,
442 P: m_GAnd(L: m_GLShr(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
443 LeftShift = false;
444
445 if (LeftShift.has_value()) {
446 assert(Mask.getBitWidth() == XLen && "Unexpected mask size!");
447 assert(C2 > 0 && C2 < XLen && "Unexpected shift amount!");
448 if (*LeftShift)
449 Mask &= maskTrailingZeros<uint64_t>(N: C2);
450 else
451 Mask &= maskTrailingOnes<uint64_t>(N: XLen - C2);
452
453 if (Mask.isShiftedMask()) {
454 unsigned Leading = XLen - Mask.getActiveBits();
455 unsigned Trailing = Mask.countr_zero();
456 // Given (and (shl y, c2), mask) in which mask has no leading zeros and
457 // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
458 if (*LeftShift && Leading == 0 && C2 < Trailing && Trailing == ShAmt) {
459 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
460 return {{[=](MachineInstrBuilder &MIB) {
461 MachineIRBuilder(*MIB.getInstr())
462 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
463 .addImm(Val: Trailing - C2);
464 MIB.addReg(RegNo: DstReg);
465 }}};
466 }
467
468 // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
469 // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
470 if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
471 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
472 return {{[=](MachineInstrBuilder &MIB) {
473 MachineIRBuilder(*MIB.getInstr())
474 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
475 .addImm(Val: Leading + Trailing);
476 MIB.addReg(RegNo: DstReg);
477 }}};
478 }
479 }
480 }
481
482 LeftShift.reset();
483
484 // (shl (and y, mask), c2)
485 if (mi_match(R: RootReg, MRI: *MRI,
486 P: m_GShl(L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
487 R: m_ICst(Cst&: C2))))
488 LeftShift = true;
489 // (lshr (and y, mask), c2)
490 else if (mi_match(R: RootReg, MRI: *MRI,
491 P: m_GLShr(L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
492 R: m_ICst(Cst&: C2))))
493 LeftShift = false;
494
495 if (LeftShift.has_value() && Mask.isShiftedMask()) {
496 unsigned Leading = XLen - Mask.getActiveBits();
497 unsigned Trailing = Mask.countr_zero();
498
499 // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
500 // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
501 bool Cond =
502 *LeftShift && Leading == 32 && Trailing > 0 && (Trailing + C2) == ShAmt;
503 if (!Cond)
504 // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
505 // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
506 Cond = !*LeftShift && Leading == 32 && C2 < Trailing &&
507 (Trailing - C2) == ShAmt;
508
509 if (Cond) {
510 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
511 return {{[=](MachineInstrBuilder &MIB) {
512 MachineIRBuilder(*MIB.getInstr())
513 .buildInstr(Opc: RISCV::SRLIW, DstOps: {DstReg}, SrcOps: {RegY})
514 .addImm(Val: Trailing);
515 MIB.addReg(RegNo: DstReg);
516 }}};
517 }
518 }
519
520 return std::nullopt;
521}
522
523InstructionSelector::ComplexRendererFns
524RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
525 unsigned ShAmt) const {
526 using namespace llvm::MIPatternMatch;
527
528 if (!Root.isReg())
529 return std::nullopt;
530 Register RootReg = Root.getReg();
531
532 // Given (and (shl x, c2), mask) in which mask is a shifted mask with
533 // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
534 // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
535 APInt Mask;
536 int64_t C2;
537 Register RegX;
538 if (mi_match(
539 R: RootReg, MRI: *MRI,
540 P: m_OneNonDBGUse(SP: m_GAnd(L: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: RegX), R: m_ICst(Cst&: C2))),
541 R: m_ICst(Cst&: Mask))))) {
542 assert(C2 > 0 && C2 < Mask.getBitWidth() && "Unexpected shift amount!");
543 Mask &= maskTrailingZeros<uint64_t>(N: C2);
544
545 if (Mask.isShiftedMask()) {
546 unsigned Leading = Mask.countl_zero();
547 unsigned Trailing = Mask.countr_zero();
548 if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
549 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
550 return {{[=](MachineInstrBuilder &MIB) {
551 MachineIRBuilder(*MIB.getInstr())
552 .buildInstr(Opc: RISCV::SLLI, DstOps: {DstReg}, SrcOps: {RegX})
553 .addImm(Val: C2 - ShAmt);
554 MIB.addReg(RegNo: DstReg);
555 }}};
556 }
557 }
558 }
559
560 return std::nullopt;
561}
562
563InstructionSelector::ComplexRendererFns
564RISCVInstructionSelector::renderVLOp(MachineOperand &Root) const {
565 assert(Root.isReg() && "Expected operand to be a Register");
566 std::optional<ValueAndVReg> C;
567 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GCst(ValReg&: C))) {
568 if (C->Value.isAllOnes())
569 // If the operand is a G_CONSTANT with value of all ones it is larger than
570 // VLMAX. We convert it to an immediate with value VLMaxSentinel. This is
571 // recognized specially by the vsetvli insertion pass.
572 return {{[=](MachineInstrBuilder &MIB) {
573 MIB.addImm(Val: RISCV::VLMaxSentinel);
574 }}};
575
576 if (isUInt<5>(x: C->Value.getZExtValue())) {
577 uint64_t ZExtC = C->Value.getZExtValue();
578 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ZExtC); }}};
579 }
580 }
581 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Root.getReg()); }}};
582}
583
584InstructionSelector::ComplexRendererFns
585RISCVInstructionSelector::renderAddiPair(Register BaseReg, int64_t AddiImm,
586 int64_t OffsetImm) const {
587 return {{[=](MachineInstrBuilder &MIB) {
588 Register Tmp = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
589 MachineInstr *Addi =
590 BuildMI(BB&: *MIB->getParent(), I&: *MIB.getInstr(), MIMD: MIB->getDebugLoc(),
591 MCID: TII.get(Opcode: RISCV::ADDI), DestReg: Tmp)
592 .addReg(RegNo: BaseReg)
593 .addImm(Val: AddiImm);
594 constrainSelectedInstRegOperands(I&: *Addi, TII, TRI, RBI);
595 MIB.addReg(RegNo: Tmp);
596 },
597 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: OffsetImm); }}};
598}
599
600InstructionSelector::ComplexRendererFns
601RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
602 if (!Root.isReg())
603 return std::nullopt;
604
605 Register RootReg = Root.getReg();
606
607 // Frame index.
608 int FI;
609 if (mi_match(R: RootReg, MRI: *MRI, P: m_GFrameIndex(FI))) {
610 return {{
611 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); },
612 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
613 }};
614 }
615
616 // base + constant offset (G_PTR_ADD).
617 Register BaseReg;
618 int64_t RHSC;
619 if (mi_match(R: RootReg, MRI: *MRI, P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_ICst(Cst&: RHSC)))) {
620 if (isInt<12>(x: RHSC)) {
621 int BaseFI;
622 if (mi_match(R: BaseReg, MRI: *MRI, P: m_GFrameIndex(FI&: BaseFI)))
623 return {{
624 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: BaseFI); },
625 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
626 }};
627
628 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: BaseReg); },
629 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
630 }
631
632 // Large constant offset. Fold a -2048/2047 adjustment so the whole
633 // constant can be split across an ADDI and the load/store offset.
634 if (RHSC >= -4096 && RHSC <= 4094) {
635 int64_t Adj = RHSC < 0 ? -2048 : 2047;
636 return renderAddiPair(BaseReg, AddiImm: Adj, OffsetImm: RHSC - Adj);
637 }
638
639 if (isWorthFoldingAdd(AddResult: RootReg))
640 if (auto Fns = computeConstAddr(CVal: RHSC, /*IsPrefetch=*/false, OrigBase: BaseReg))
641 return Fns;
642 }
643
644 // Bare constant address. IRTranslator lowers inttoptr(C) to
645 // G_INTTOPTR(G_CONSTANT); look through it to reach the constant.
646 int64_t CVal;
647 if (mi_match(R: RootReg, MRI: *MRI, P: m_GIntToPtr(Src: m_ICst(Cst&: CVal))) ||
648 mi_match(R: RootReg, MRI: *MRI, P: m_ICst(Cst&: CVal))) {
649 if (auto Fns = computeConstAddr(CVal, /*IsPrefetch=*/false, OrigBase: Register()))
650 return Fns;
651 }
652
653 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); },
654 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
655}
656
657InstructionSelector::ComplexRendererFns
658RISCVInstructionSelector::selectBrindRegImm(MachineOperand &Root) const {
659 if (!Root.isReg())
660 return std::nullopt;
661
662 Register RootReg = Root.getReg();
663
664 // base + constant offset (G_PTR_ADD). Doesn't match a FrameIndex, unlike
665 // selectAddrRegImm, since the callers of this (e.g. indirect branches)
666 // can't take a FrameIndex or global address operand.
667 Register BaseReg;
668 int64_t RHSC;
669 if (mi_match(R: RootReg, MRI: *MRI, P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_ICst(Cst&: RHSC))) &&
670 isInt<12>(x: RHSC)) {
671 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: BaseReg); },
672 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
673 }
674
675 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); },
676 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
677}
678
679InstructionSelector::ComplexRendererFns
680RISCVInstructionSelector::selectAddrRegImmLsb00000(MachineOperand &Root) const {
681 if (!Root.isReg())
682 return std::nullopt;
683
684 Register RootReg = Root.getReg();
685
686 // Frame index.
687 int FI;
688 if (mi_match(R: RootReg, MRI: *MRI, P: m_GFrameIndex(FI))) {
689 return {{
690 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); },
691 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
692 }};
693 }
694
695 // base + constant offset (G_PTR_ADD).
696 Register BaseReg;
697 int64_t RHSC;
698 if (mi_match(R: RootReg, MRI: *MRI, P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_ICst(Cst&: RHSC)))) {
699 if (isInt<12>(x: RHSC)) {
700 // Not a multiple of 32: can't encode, use the address as-is.
701 if ((RHSC & 0b11111) != 0) {
702 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); },
703 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
704 }
705 // Fold the offset.
706 int BaseFI;
707 if (mi_match(R: BaseReg, MRI: *MRI, P: m_GFrameIndex(FI&: BaseFI)))
708 return {{
709 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: BaseFI); },
710 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
711 }};
712 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: BaseReg); },
713 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
714 }
715
716 // Large constant: fold a -2048/2016 adjustment to save an instruction.
717 if ((-2049 >= RHSC && RHSC >= -4096) || (4063 >= RHSC && RHSC >= 2017)) {
718 int64_t Adj = RHSC < 0 ? -2048 : 2016;
719 return renderAddiPair(BaseReg, AddiImm: RHSC - Adj, OffsetImm: Adj);
720 }
721
722 // Otherwise split the constant into Hi (materialized + added to the base)
723 // and Lo12 (folded offset).
724 if (auto Fns = computeConstAddr(CVal: RHSC, /*IsPrefetch=*/true, OrigBase: BaseReg))
725 return Fns;
726 }
727
728 // Bare constant address. IRTranslator emits inttoptr(C) as
729 // G_INTTOPTR(G_CONSTANT); look through the G_INTTOPTR to reach the constant.
730 int64_t CVal;
731 if (mi_match(R: RootReg, MRI: *MRI, P: m_GIntToPtr(Src: m_ICst(Cst&: CVal))) ||
732 mi_match(R: RootReg, MRI: *MRI, P: m_ICst(Cst&: CVal))) {
733 if (auto Fns = computeConstAddr(CVal, /*IsPrefetch=*/true, OrigBase: Register()))
734 return Fns;
735 }
736
737 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); },
738 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
739}
740
741/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
742/// CC Must be an ICMP Predicate.
743static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
744 switch (CC) {
745 default:
746 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
747 case CmpInst::Predicate::ICMP_EQ:
748 return RISCVCC::COND_EQ;
749 case CmpInst::Predicate::ICMP_NE:
750 return RISCVCC::COND_NE;
751 case CmpInst::Predicate::ICMP_ULT:
752 return RISCVCC::COND_LTU;
753 case CmpInst::Predicate::ICMP_SLT:
754 return RISCVCC::COND_LT;
755 case CmpInst::Predicate::ICMP_UGE:
756 return RISCVCC::COND_GEU;
757 case CmpInst::Predicate::ICMP_SGE:
758 return RISCVCC::COND_GE;
759 }
760}
761
762static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC,
763 Register &LHS, Register &RHS,
764 MachineRegisterInfo &MRI) {
765 // Try to fold an ICmp. If that fails, use a NE compare with X0.
766 CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
767 if (!mi_match(R: CondReg, MRI, P: m_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: LHS), R: m_Reg(R&: RHS)))) {
768 LHS = CondReg;
769 RHS = RISCV::X0;
770 CC = RISCVCC::COND_NE;
771 return;
772 }
773
774 // We found an ICmp, do some canonicalization.
775
776 // Adjust comparisons to use comparison with 0 if possible.
777 if (auto Constant = getIConstantVRegSExtVal(VReg: RHS, MRI)) {
778 switch (Pred) {
779 case CmpInst::Predicate::ICMP_SGT:
780 // Convert X > -1 to X >= 0
781 if (*Constant == -1) {
782 CC = RISCVCC::COND_GE;
783 RHS = RISCV::X0;
784 return;
785 }
786 break;
787 case CmpInst::Predicate::ICMP_SLT:
788 // Convert X < 1 to 0 >= X
789 if (*Constant == 1) {
790 CC = RISCVCC::COND_GE;
791 RHS = LHS;
792 LHS = RISCV::X0;
793 return;
794 }
795 break;
796 default:
797 break;
798 }
799 }
800
801 switch (Pred) {
802 default:
803 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
804 case CmpInst::Predicate::ICMP_EQ:
805 case CmpInst::Predicate::ICMP_NE:
806 case CmpInst::Predicate::ICMP_ULT:
807 case CmpInst::Predicate::ICMP_SLT:
808 case CmpInst::Predicate::ICMP_UGE:
809 case CmpInst::Predicate::ICMP_SGE:
810 // These CCs are supported directly by RISC-V branches.
811 break;
812 case CmpInst::Predicate::ICMP_SGT:
813 case CmpInst::Predicate::ICMP_SLE:
814 case CmpInst::Predicate::ICMP_UGT:
815 case CmpInst::Predicate::ICMP_ULE:
816 // These CCs are not supported directly by RISC-V branches, but changing the
817 // direction of the CC and swapping LHS and RHS are.
818 Pred = CmpInst::getSwappedPredicate(pred: Pred);
819 std::swap(a&: LHS, b&: RHS);
820 break;
821 }
822
823 CC = getRISCVCCFromICmp(CC: Pred);
824}
825
826/// Select the RISC-V Zalasr opcode for the G_LOAD or G_STORE operation
827/// \p GenericOpc, appropriate for the GPR register bank and of memory access
828/// size \p OpSize.
829static unsigned selectZalasrLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {
830 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
831 switch (OpSize) {
832 default:
833 llvm_unreachable("Unexpected memory size");
834 case 8:
835 return IsStore ? RISCV::SB_RL : RISCV::LB_AQ;
836 case 16:
837 return IsStore ? RISCV::SH_RL : RISCV::LH_AQ;
838 case 32:
839 return IsStore ? RISCV::SW_RL : RISCV::LW_AQ;
840 case 64:
841 return IsStore ? RISCV::SD_RL : RISCV::LD_AQ;
842 }
843}
844
845/// Select the RISC-V regimm opcode for the G_LOAD or G_STORE operation
846/// \p GenericOpc, appropriate for the GPR register bank and of memory access
847/// size \p OpSize. \returns \p GenericOpc if the combination is unsupported.
848static unsigned selectRegImmLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {
849 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
850 switch (OpSize) {
851 case 8:
852 // Prefer unsigned due to no c.lb in Zcb.
853 return IsStore ? RISCV::SB : RISCV::LBU;
854 case 16:
855 return IsStore ? RISCV::SH : RISCV::LH;
856 case 32:
857 return IsStore ? RISCV::SW : RISCV::LW;
858 case 64:
859 return IsStore ? RISCV::SD : RISCV::LD;
860 }
861
862 return GenericOpc;
863}
864
865void RISCVInstructionSelector::addVectorLoadStoreOperands(
866 MachineInstr &I, SmallVectorImpl<Register> &SrcOps, unsigned &CurOp,
867 bool IsMasked, bool IsStridedOrIndexed, LLT *IndexVT) const {
868 // Base Pointer
869 auto PtrReg = I.getOperand(i: CurOp++).getReg();
870 SrcOps.push_back(Elt: PtrReg);
871
872 // Stride or Index
873 if (IsStridedOrIndexed) {
874 auto StrideReg = I.getOperand(i: CurOp++).getReg();
875 SrcOps.push_back(Elt: StrideReg);
876 if (IndexVT)
877 *IndexVT = MRI->getType(Reg: StrideReg);
878 }
879
880 // Mask
881 if (IsMasked) {
882 auto MaskReg = I.getOperand(i: CurOp++).getReg();
883 SrcOps.push_back(Elt: MaskReg);
884 }
885}
886
887bool RISCVInstructionSelector::selectIntrinsicWithSideEffects(
888 MachineInstr &I) const {
889 // Find the intrinsic ID.
890 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
891 // Select the instruction.
892 switch (IntrinID) {
893 default:
894 return false;
895 case Intrinsic::riscv_vlm:
896 case Intrinsic::riscv_vle:
897 case Intrinsic::riscv_vle_mask:
898 case Intrinsic::riscv_vlse:
899 case Intrinsic::riscv_vlse_mask: {
900 bool IsMasked = IntrinID == Intrinsic::riscv_vle_mask ||
901 IntrinID == Intrinsic::riscv_vlse_mask;
902 bool IsStrided = IntrinID == Intrinsic::riscv_vlse ||
903 IntrinID == Intrinsic::riscv_vlse_mask;
904 LLT VT = MRI->getType(Reg: I.getOperand(i: 0).getReg());
905 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
906
907 // Result vector
908 const Register DstReg = I.getOperand(i: 0).getReg();
909
910 // Sources
911 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
912 unsigned CurOp = 2;
913 SmallVector<Register, 4> SrcOps; // Source registers.
914
915 // Passthru
916 if (HasPassthruOperand) {
917 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
918 SrcOps.push_back(Elt: PassthruReg);
919 } else {
920 SrcOps.push_back(Elt: Register(RISCV::NoRegister));
921 }
922
923 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: IsStrided);
924
925 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
926 const RISCV::VLEPseudo *P =
927 RISCV::getVLEPseudo(Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
928 LMUL: static_cast<unsigned>(LMUL));
929
930 MachineInstrBuilder PseudoMI =
931 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo), DestReg: DstReg);
932 for (Register Reg : SrcOps)
933 PseudoMI.addReg(RegNo: Reg);
934
935 // Select VL
936 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
937 for (auto &RenderFn : *VLOpFn)
938 RenderFn(PseudoMI);
939
940 // SEW
941 PseudoMI.addImm(Val: Log2SEW);
942
943 // Policy
944 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
945 if (IsMasked)
946 Policy = I.getOperand(i: CurOp++).getImm();
947 PseudoMI.addImm(Val: Policy);
948
949 // Memref
950 PseudoMI.cloneMemRefs(OtherMI: I);
951
952 I.eraseFromParent();
953 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
954 return true;
955 }
956 case Intrinsic::riscv_vloxei:
957 case Intrinsic::riscv_vloxei_mask:
958 case Intrinsic::riscv_vluxei:
959 case Intrinsic::riscv_vluxei_mask: {
960 bool IsMasked = IntrinID == Intrinsic::riscv_vloxei_mask ||
961 IntrinID == Intrinsic::riscv_vluxei_mask;
962 bool IsOrdered = IntrinID == Intrinsic::riscv_vloxei ||
963 IntrinID == Intrinsic::riscv_vloxei_mask;
964 LLT VT = MRI->getType(Reg: I.getOperand(i: 0).getReg());
965 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
966
967 // Result vector
968 const Register DstReg = I.getOperand(i: 0).getReg();
969
970 // Sources
971 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
972 unsigned CurOp = 2;
973 SmallVector<Register, 4> SrcOps; // Source registers.
974
975 // Passthru
976 if (HasPassthruOperand) {
977 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
978 SrcOps.push_back(Elt: PassthruReg);
979 } else {
980 // Use NoRegister if there is no specified passthru.
981 SrcOps.push_back(Elt: Register());
982 }
983 LLT IndexVT;
984 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: true, IndexVT: &IndexVT);
985
986 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
987 RISCVVType::VLMUL IndexLMUL =
988 RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: IndexVT));
989 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
990 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
991 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
992 "values when XLEN=32");
993 }
994 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
995 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
996 IndexLMUL: static_cast<unsigned>(IndexLMUL));
997
998 MachineInstrBuilder PseudoMI =
999 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo), DestReg: DstReg);
1000 for (Register Reg : SrcOps)
1001 PseudoMI.addReg(RegNo: Reg);
1002
1003 // Select VL
1004 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
1005 for (auto &RenderFn : *VLOpFn)
1006 RenderFn(PseudoMI);
1007
1008 // SEW
1009 PseudoMI.addImm(Val: Log2SEW);
1010
1011 // Policy
1012 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
1013 if (IsMasked)
1014 Policy = I.getOperand(i: CurOp++).getImm();
1015 PseudoMI.addImm(Val: Policy);
1016
1017 // Memref
1018 PseudoMI.cloneMemRefs(OtherMI: I);
1019
1020 I.eraseFromParent();
1021 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1022 return true;
1023 }
1024 case Intrinsic::riscv_vsm:
1025 case Intrinsic::riscv_vse:
1026 case Intrinsic::riscv_vse_mask:
1027 case Intrinsic::riscv_vsse:
1028 case Intrinsic::riscv_vsse_mask: {
1029 bool IsMasked = IntrinID == Intrinsic::riscv_vse_mask ||
1030 IntrinID == Intrinsic::riscv_vsse_mask;
1031 bool IsStrided = IntrinID == Intrinsic::riscv_vsse ||
1032 IntrinID == Intrinsic::riscv_vsse_mask;
1033 LLT VT = MRI->getType(Reg: I.getOperand(i: 1).getReg());
1034 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
1035
1036 // Sources
1037 unsigned CurOp = 1;
1038 SmallVector<Register, 4> SrcOps; // Source registers.
1039
1040 // Store value
1041 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
1042 SrcOps.push_back(Elt: PassthruReg);
1043
1044 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: IsStrided);
1045
1046 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
1047 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
1048 Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
1049
1050 MachineInstrBuilder PseudoMI =
1051 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo));
1052 for (Register Reg : SrcOps)
1053 PseudoMI.addReg(RegNo: Reg);
1054
1055 // Select VL
1056 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
1057 for (auto &RenderFn : *VLOpFn)
1058 RenderFn(PseudoMI);
1059
1060 // SEW
1061 PseudoMI.addImm(Val: Log2SEW);
1062
1063 // Memref
1064 PseudoMI.cloneMemRefs(OtherMI: I);
1065
1066 I.eraseFromParent();
1067 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1068 return true;
1069 }
1070 case Intrinsic::riscv_vsoxei:
1071 case Intrinsic::riscv_vsoxei_mask:
1072 case Intrinsic::riscv_vsuxei:
1073 case Intrinsic::riscv_vsuxei_mask: {
1074 bool IsMasked = IntrinID == Intrinsic::riscv_vsoxei_mask ||
1075 IntrinID == Intrinsic::riscv_vsuxei_mask;
1076 bool IsOrdered = IntrinID == Intrinsic::riscv_vsoxei ||
1077 IntrinID == Intrinsic::riscv_vsoxei_mask;
1078 LLT VT = MRI->getType(Reg: I.getOperand(i: 1).getReg());
1079 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
1080
1081 // Sources
1082 unsigned CurOp = 1;
1083 SmallVector<Register, 4> SrcOps; // Source registers.
1084
1085 // Store value
1086 auto PassthruReg = I.getOperand(i: CurOp++).getReg();
1087 SrcOps.push_back(Elt: PassthruReg);
1088
1089 LLT IndexVT;
1090 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: true, IndexVT: &IndexVT);
1091
1092 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT));
1093 RISCVVType::VLMUL IndexLMUL =
1094 RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: IndexVT));
1095 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
1096 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
1097 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
1098 "values when XLEN=32");
1099 }
1100 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
1101 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
1102 IndexLMUL: static_cast<unsigned>(IndexLMUL));
1103
1104 MachineInstrBuilder PseudoMI =
1105 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo));
1106 for (Register Reg : SrcOps)
1107 PseudoMI.addReg(RegNo: Reg);
1108
1109 // Select VL
1110 auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++));
1111 for (auto &RenderFn : *VLOpFn)
1112 RenderFn(PseudoMI);
1113
1114 // SEW
1115 PseudoMI.addImm(Val: Log2SEW);
1116
1117 // Memref
1118 PseudoMI.cloneMemRefs(OtherMI: I);
1119
1120 I.eraseFromParent();
1121 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1122 return true;
1123 }
1124 }
1125}
1126
1127bool RISCVInstructionSelector::selectIntrinsic(MachineInstr &I) const {
1128 // Find the intrinsic ID.
1129 unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
1130 // Select the instruction.
1131 switch (IntrinID) {
1132 default:
1133 return false;
1134 case Intrinsic::riscv_vsetvli:
1135 case Intrinsic::riscv_vsetvlimax: {
1136
1137 bool VLMax = IntrinID == Intrinsic::riscv_vsetvlimax;
1138
1139 unsigned Offset = VLMax ? 2 : 3;
1140 unsigned SEW = RISCVVType::decodeVSEW(VSEW: I.getOperand(i: Offset).getImm() & 0x7);
1141 RISCVVType::VLMUL VLMul =
1142 static_cast<RISCVVType::VLMUL>(I.getOperand(i: Offset + 1).getImm() & 0x7);
1143
1144 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMUL: VLMul, SEW, /*TailAgnostic*/ true,
1145 /*MaskAgnostic*/ true);
1146
1147 Register DstReg = I.getOperand(i: 0).getReg();
1148
1149 Register VLOperand;
1150 unsigned Opcode = RISCV::PseudoVSETVLI;
1151
1152 // Check if AVL is a constant that equals VLMAX.
1153 if (!VLMax) {
1154 Register AVLReg = I.getOperand(i: 2).getReg();
1155 if (auto AVLConst = getIConstantVRegValWithLookThrough(VReg: AVLReg, MRI: *MRI)) {
1156 uint64_t AVL = AVLConst->Value.getZExtValue();
1157 if (auto VLEN = Subtarget->getRealVLen()) {
1158 if (*VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == AVL)
1159 VLMax = true;
1160 }
1161 }
1162
1163 if (mi_match(R: AVLReg, MRI: *MRI, P: m_AllOnes()))
1164 VLMax = true;
1165 }
1166
1167 if (VLMax) {
1168 VLOperand = Register(RISCV::X0);
1169 Opcode = RISCV::PseudoVSETVLIX0;
1170 } else {
1171 Register AVLReg = I.getOperand(i: 2).getReg();
1172 VLOperand = AVLReg;
1173
1174 // Check if AVL is a small constant that can use PseudoVSETIVLI.
1175 if (auto AVLConst = getIConstantVRegValWithLookThrough(VReg: AVLReg, MRI: *MRI)) {
1176 uint64_t AVL = AVLConst->Value.getZExtValue();
1177 if (isUInt<5>(x: AVL)) {
1178 MachineInstr *PseudoMI =
1179 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1180 MCID: TII.get(Opcode: RISCV::PseudoVSETIVLI), DestReg: DstReg)
1181 .addImm(Val: AVL)
1182 .addImm(Val: VTypeI);
1183 I.eraseFromParent();
1184 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1185 return true;
1186 }
1187 }
1188 }
1189
1190 MachineInstr *PseudoMI =
1191 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode), DestReg: DstReg)
1192 .addReg(RegNo: VLOperand)
1193 .addImm(Val: VTypeI);
1194 I.eraseFromParent();
1195 constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI);
1196 return true;
1197 }
1198 }
1199}
1200
1201bool RISCVInstructionSelector::selectExtractSubvector(MachineInstr &MI) const {
1202 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_SUBVECTOR);
1203
1204 Register DstReg = MI.getOperand(i: 0).getReg();
1205 Register SrcReg = MI.getOperand(i: 1).getReg();
1206
1207 LLT DstTy = MRI->getType(Reg: DstReg);
1208 LLT SrcTy = MRI->getType(Reg: SrcReg);
1209
1210 unsigned Idx = static_cast<unsigned>(MI.getOperand(i: 2).getImm());
1211
1212 MVT DstMVT = getMVTForLLT(Ty: DstTy);
1213 MVT SrcMVT = getMVTForLLT(Ty: SrcTy);
1214
1215 unsigned SubRegIdx;
1216 std::tie(args&: SubRegIdx, args&: Idx) =
1217 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1218 VecVT: SrcMVT, SubVecVT: DstMVT, InsertExtractIdx: Idx, TRI: &TRI);
1219
1220 if (Idx != 0)
1221 return false;
1222
1223 unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: DstMVT);
1224 const TargetRegisterClass *DstRC = TRI.getRegClass(i: DstRegClassID);
1225 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1226 return false;
1227
1228 unsigned SrcRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: SrcMVT);
1229 const TargetRegisterClass *SrcRC = TRI.getRegClass(i: SrcRegClassID);
1230 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
1231 return false;
1232
1233 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1234 DestReg: DstReg)
1235 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubRegIdx);
1236
1237 MI.eraseFromParent();
1238 return true;
1239}
1240
1241bool RISCVInstructionSelector::selectInsertSubVector(MachineInstr &MI) const {
1242 assert(MI.getOpcode() == TargetOpcode::G_INSERT_SUBVECTOR);
1243
1244 Register DstReg = MI.getOperand(i: 0).getReg();
1245 Register VecReg = MI.getOperand(i: 1).getReg();
1246 Register SubVecReg = MI.getOperand(i: 2).getReg();
1247
1248 LLT VecTy = MRI->getType(Reg: VecReg);
1249 LLT SubVecTy = MRI->getType(Reg: SubVecReg);
1250
1251 MVT VecMVT = getMVTForLLT(Ty: VecTy);
1252 MVT SubVecMVT = getMVTForLLT(Ty: SubVecTy);
1253
1254 unsigned Idx = static_cast<unsigned>(MI.getOperand(i: 3).getImm());
1255
1256 unsigned SubRegIdx;
1257 std::tie(args&: SubRegIdx, args&: Idx) =
1258 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1259 VecVT: VecMVT, SubVecVT: SubVecMVT, InsertExtractIdx: Idx, TRI: &TRI);
1260
1261 // If the Idx hasn't been completely eliminated then this is a subvector
1262 // insert which doesn't naturally align to a vector register. These must
1263 // be handled using instructions to manipulate the vector registers.
1264 if (Idx != 0)
1265 return false;
1266
1267 // Constrain dst
1268 unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: VecMVT);
1269 const TargetRegisterClass *DstRC = TRI.getRegClass(i: DstRegClassID);
1270 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1271 return false;
1272
1273 // If we haven't set a SubRegIdx, then we must be going between
1274 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
1275 if (SubRegIdx == RISCV::NoSubRegister) {
1276 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecMVT) ==
1277 DstRegClassID &&
1278 "Unexpected subvector insert");
1279 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1280 DestReg: DstReg)
1281 .addReg(RegNo: SubVecReg);
1282 MI.eraseFromParent();
1283 return true;
1284 }
1285
1286 // Use INSERT_SUBREG to insert the subvector into the vector at the
1287 // appropriate subregister index.
1288 MachineInstr *Ins = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1289 MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DstReg)
1290 .addReg(RegNo: VecReg)
1291 .addReg(RegNo: SubVecReg)
1292 .addImm(Val: SubRegIdx);
1293
1294 MI.eraseFromParent();
1295 constrainSelectedInstRegOperands(I&: *Ins, TII, TRI, RBI);
1296 return true;
1297}
1298
1299bool RISCVInstructionSelector::select(MachineInstr &MI) {
1300 preISelLower(MI);
1301 const unsigned Opc = MI.getOpcode();
1302
1303 if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
1304 if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
1305 const Register DefReg = MI.getOperand(i: 0).getReg();
1306 const LLT DefTy = MRI->getType(Reg: DefReg);
1307
1308 const RegClassOrRegBank &RegClassOrBank =
1309 MRI->getRegClassOrRegBank(Reg: DefReg);
1310
1311 const TargetRegisterClass *DefRC =
1312 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
1313 if (!DefRC) {
1314 if (!DefTy.isValid()) {
1315 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
1316 return false;
1317 }
1318
1319 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
1320 DefRC = TRI.getRegClassForTypeOnBank(Ty: DefTy, RB, Is64Bit: STI.is64Bit());
1321 if (!DefRC) {
1322 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
1323 return false;
1324 }
1325 }
1326
1327 MI.setDesc(TII.get(Opcode: TargetOpcode::PHI));
1328 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI&: *MRI);
1329 }
1330
1331 // Certain non-generic instructions also need some special handling.
1332 if (MI.isCopy())
1333 return selectCopy(MI);
1334
1335 return true;
1336 }
1337
1338 if (selectImpl(I&: MI, CoverageInfo&: *CoverageInfo))
1339 return true;
1340
1341 switch (Opc) {
1342 case TargetOpcode::G_ANYEXT:
1343 case TargetOpcode::G_PTRTOINT:
1344 case TargetOpcode::G_INTTOPTR:
1345 case TargetOpcode::G_TRUNC:
1346 case TargetOpcode::G_FREEZE:
1347 return selectCopy(MI);
1348 case TargetOpcode::G_CONSTANT: {
1349 Register DstReg = MI.getOperand(i: 0).getReg();
1350 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
1351
1352 if (!materializeImm(Reg: DstReg, Imm, MI))
1353 return false;
1354
1355 MI.eraseFromParent();
1356 return true;
1357 }
1358 case TargetOpcode::G_ZEXT:
1359 case TargetOpcode::G_SEXT: {
1360 bool IsSigned = Opc != TargetOpcode::G_ZEXT;
1361 Register DstReg = MI.getOperand(i: 0).getReg();
1362 Register SrcReg = MI.getOperand(i: 1).getReg();
1363 LLT SrcTy = MRI->getType(Reg: SrcReg);
1364 unsigned SrcSize = SrcTy.getSizeInBits();
1365
1366 if (SrcTy.isVector())
1367 return false; // Should be handled by imported patterns.
1368
1369 assert((*RBI.getRegBank(DstReg, *MRI, TRI)).getID() ==
1370 RISCV::GPRBRegBankID &&
1371 "Unexpected ext regbank");
1372
1373 // Use addiw SrcReg, 0 (sext.w) for i32.
1374 if (IsSigned && SrcSize == 32) {
1375 MI.setDesc(TII.get(Opcode: RISCV::ADDIW));
1376 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
1377 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1378 return true;
1379 }
1380
1381 // Use add.uw SrcReg, X0 (zext.w) for i32 with Zba.
1382 if (!IsSigned && SrcSize == 32 && STI.hasStdExtZba()) {
1383 MI.setDesc(TII.get(Opcode: RISCV::ADD_UW));
1384 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::X0, /*isDef=*/false));
1385 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1386 return true;
1387 }
1388
1389 // Use sext.h/zext.h for i16 with Zbb.
1390 if (SrcSize == 16 &&
1391 (STI.hasStdExtZbb() || (!IsSigned && STI.hasStdExtZbkb()))) {
1392 MI.setDesc(TII.get(Opcode: IsSigned ? RISCV::SEXT_H
1393 : STI.is64Bit() ? RISCV::ZEXT_H_RV64
1394 : RISCV::ZEXT_H_RV32));
1395 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1396 return true;
1397 }
1398
1399 // Fall back to shift pair.
1400 Register ShiftLeftReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1401 MachineInstr *ShiftLeft = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1402 MCID: TII.get(Opcode: RISCV::SLLI), DestReg: ShiftLeftReg)
1403 .addReg(RegNo: SrcReg)
1404 .addImm(Val: STI.getXLen() - SrcSize);
1405 constrainSelectedInstRegOperands(I&: *ShiftLeft, TII, TRI, RBI);
1406 MachineInstr *ShiftRight =
1407 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1408 MCID: TII.get(Opcode: IsSigned ? RISCV::SRAI : RISCV::SRLI), DestReg: DstReg)
1409 .addReg(RegNo: ShiftLeftReg)
1410 .addImm(Val: STI.getXLen() - SrcSize);
1411 constrainSelectedInstRegOperands(I&: *ShiftRight, TII, TRI, RBI);
1412 MI.eraseFromParent();
1413 return true;
1414 }
1415 case TargetOpcode::G_FCONSTANT: {
1416 // TODO: Use constant pool for complex constants.
1417 Register DstReg = MI.getOperand(i: 0).getReg();
1418 const APFloat &FPimm = MI.getOperand(i: 1).getFPImm()->getValueAPF();
1419 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
1420 if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
1421 Register GPRReg;
1422 if (FPimm.isPosZero()) {
1423 GPRReg = RISCV::X0;
1424 } else {
1425 GPRReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1426 APInt Imm = FPimm.bitcastToAPInt();
1427 if (!materializeImm(Reg: GPRReg, Imm: Imm.getSExtValue(), MI))
1428 return false;
1429 }
1430
1431 unsigned Opcode = Size == 64 ? RISCV::FMV_D_X
1432 : Size == 32 ? RISCV::FMV_W_X
1433 : RISCV::FMV_H_X;
1434 MachineInstr *FMV = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1435 MCID: TII.get(Opcode), DestReg: DstReg)
1436 .addReg(RegNo: GPRReg);
1437 constrainSelectedInstRegOperands(I&: *FMV, TII, TRI, RBI);
1438 } else {
1439 // s64 on rv32
1440 assert(Size == 64 && !Subtarget->is64Bit() &&
1441 "Unexpected size or subtarget");
1442
1443 if (FPimm.isPosZero()) {
1444 // Optimize +0.0 to use fcvt.d.w
1445 MachineInstr *FCVT = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1446 MCID: TII.get(Opcode: RISCV::FCVT_D_W), DestReg: DstReg)
1447 .addReg(RegNo: RISCV::X0)
1448 .addImm(Val: RISCVFPRndMode::RNE);
1449 constrainSelectedInstRegOperands(I&: *FCVT, TII, TRI, RBI);
1450
1451 MI.eraseFromParent();
1452 return true;
1453 }
1454
1455 // Split into two pieces and build through the stack.
1456 Register GPRRegHigh = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1457 Register GPRRegLow = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1458 APInt Imm = FPimm.bitcastToAPInt();
1459 if (!materializeImm(Reg: GPRRegHigh, Imm: Imm.extractBits(numBits: 32, bitPosition: 32).getSExtValue(),
1460 MI))
1461 return false;
1462 if (!materializeImm(Reg: GPRRegLow, Imm: Imm.trunc(width: 32).getSExtValue(), MI))
1463 return false;
1464 MachineInstr *PairF64 =
1465 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1466 MCID: TII.get(Opcode: RISCV::BuildPairF64Pseudo), DestReg: DstReg)
1467 .addReg(RegNo: GPRRegLow)
1468 .addReg(RegNo: GPRRegHigh);
1469 constrainSelectedInstRegOperands(I&: *PairF64, TII, TRI, RBI);
1470 }
1471
1472 MI.eraseFromParent();
1473 return true;
1474 }
1475 case TargetOpcode::G_GLOBAL_VALUE: {
1476 auto *GV = MI.getOperand(i: 1).getGlobal();
1477 if (GV->isThreadLocal()) {
1478 // TODO: implement this case.
1479 return false;
1480 }
1481
1482 return selectAddr(MI, IsLocal: GV->isDSOLocal(), IsExternWeak: GV->hasExternalWeakLinkage());
1483 }
1484 case TargetOpcode::G_JUMP_TABLE:
1485 case TargetOpcode::G_CONSTANT_POOL:
1486 return selectAddr(MI);
1487 case TargetOpcode::G_BRCOND: {
1488 Register LHS, RHS;
1489 RISCVCC::CondCode CC;
1490 getOperandsForBranch(CondReg: MI.getOperand(i: 0).getReg(), CC, LHS, RHS, MRI&: *MRI);
1491
1492 MachineInstr *Bcc = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1493 MCID: TII.get(Opcode: RISCVCC::getBrCond(CC)))
1494 .addReg(RegNo: LHS)
1495 .addReg(RegNo: RHS)
1496 .addMBB(MBB: MI.getOperand(i: 1).getMBB());
1497 MI.eraseFromParent();
1498 constrainSelectedInstRegOperands(I&: *Bcc, TII, TRI, RBI);
1499 return true;
1500 }
1501 case TargetOpcode::G_SELECT:
1502 return selectSelect(MI);
1503 case TargetOpcode::G_FCMP:
1504 return selectFPCompare(MI);
1505 case TargetOpcode::G_FENCE: {
1506 AtomicOrdering FenceOrdering =
1507 static_cast<AtomicOrdering>(MI.getOperand(i: 0).getImm());
1508 SyncScope::ID FenceSSID =
1509 static_cast<SyncScope::ID>(MI.getOperand(i: 1).getImm());
1510 emitFence(FenceOrdering, FenceSSID, MI);
1511 MI.eraseFromParent();
1512 return true;
1513 }
1514 case TargetOpcode::G_IMPLICIT_DEF:
1515 return selectImplicitDef(MI);
1516 case TargetOpcode::G_UNMERGE_VALUES:
1517 return selectUnmergeValues(MI);
1518 case TargetOpcode::G_LOAD:
1519 case TargetOpcode::G_STORE: {
1520 GLoadStore &LdSt = cast<GLoadStore>(Val&: MI);
1521 const Register ValReg = LdSt.getReg(Idx: 0);
1522 const Register PtrReg = LdSt.getPointerReg();
1523 LLT PtrTy = MRI->getType(Reg: PtrReg);
1524
1525 const RegisterBank &RB = *RBI.getRegBank(Reg: ValReg, MRI: *MRI, TRI);
1526 if (RB.getID() != RISCV::GPRBRegBankID)
1527 return false;
1528
1529#ifndef NDEBUG
1530 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, *MRI, TRI);
1531 // Check that the pointer register is valid.
1532 assert(PtrRB.getID() == RISCV::GPRBRegBankID &&
1533 "Load/Store pointer operand isn't a GPR");
1534 assert(PtrTy.isPointer() && "Load/Store pointer operand isn't a pointer");
1535#endif
1536
1537 // Can only handle AddressSpace 0.
1538 if (PtrTy.getAddressSpace() != 0)
1539 return false;
1540
1541 unsigned MemSize = LdSt.getMemSizeInBits().getValue();
1542 AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();
1543
1544 if (isStrongerThanMonotonic(AO: Order)) {
1545 MI.setDesc(TII.get(Opcode: selectZalasrLoadStoreOp(GenericOpc: Opc, OpSize: MemSize)));
1546 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1547 return true;
1548 }
1549
1550 const unsigned NewOpc = selectRegImmLoadStoreOp(GenericOpc: MI.getOpcode(), OpSize: MemSize);
1551 if (NewOpc == MI.getOpcode())
1552 return false;
1553
1554 // Check if we can fold anything into the addressing mode.
1555 auto AddrModeFns = selectAddrRegImm(Root&: MI.getOperand(i: 1));
1556 if (!AddrModeFns)
1557 return false;
1558
1559 // Folded something. Create a new instruction and return it.
1560 MachineInstrBuilder NewInst =
1561 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: NewOpc));
1562 NewInst.setMIFlags(MI.getFlags());
1563 if (isa<GStore>(Val: MI))
1564 NewInst.addUse(RegNo: ValReg);
1565 else
1566 NewInst.addDef(RegNo: ValReg);
1567 NewInst.cloneMemRefs(OtherMI: MI);
1568 for (auto &Fn : *AddrModeFns)
1569 Fn(NewInst);
1570 MI.eraseFromParent();
1571
1572 constrainSelectedInstRegOperands(I&: *NewInst, TII, TRI, RBI);
1573 return true;
1574 }
1575 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1576 return selectIntrinsicWithSideEffects(I&: MI);
1577 case TargetOpcode::G_INTRINSIC:
1578 return selectIntrinsic(I&: MI);
1579 case TargetOpcode::G_EXTRACT_SUBVECTOR:
1580 return selectExtractSubvector(MI);
1581 case TargetOpcode::G_INSERT_SUBVECTOR:
1582 return selectInsertSubVector(MI);
1583 default:
1584 return false;
1585 }
1586}
1587
1588bool RISCVInstructionSelector::selectUnmergeValues(MachineInstr &MI) const {
1589 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
1590
1591 if (!Subtarget->hasStdExtZfa())
1592 return false;
1593
1594 // Split F64 Src into two s32 parts
1595 if (MI.getNumOperands() != 3)
1596 return false;
1597 Register Src = MI.getOperand(i: 2).getReg();
1598 Register Lo = MI.getOperand(i: 0).getReg();
1599 Register Hi = MI.getOperand(i: 1).getReg();
1600 if (!isRegInFprb(Reg: Src) || !isRegInGprb(Reg: Lo) || !isRegInGprb(Reg: Hi))
1601 return false;
1602
1603 MachineInstr *ExtractLo = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1604 MCID: TII.get(Opcode: RISCV::FMV_X_W_FPR64), DestReg: Lo)
1605 .addReg(RegNo: Src);
1606 constrainSelectedInstRegOperands(I&: *ExtractLo, TII, TRI, RBI);
1607
1608 MachineInstr *ExtractHi = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1609 MCID: TII.get(Opcode: RISCV::FMVH_X_D), DestReg: Hi)
1610 .addReg(RegNo: Src);
1611 constrainSelectedInstRegOperands(I&: *ExtractHi, TII, TRI, RBI);
1612
1613 MI.eraseFromParent();
1614 return true;
1615}
1616
1617bool RISCVInstructionSelector::replacePtrWithInt(MachineInstr &MI,
1618 unsigned OpIdx) {
1619 MachineOperand &Op = MI.getOperand(i: OpIdx);
1620 Register PtrReg = Op.getReg();
1621 assert(MRI->getType(PtrReg).isPointer() && "Operand is not a pointer!");
1622
1623 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1624 Register IntReg = MRI->createGenericVirtualRegister(Ty: sXLen);
1625 MRI->setRegBank(Reg: IntReg, RegBank: RBI.getRegBank(ID: RISCV::GPRBRegBankID));
1626 MachineInstr *PtrToInt = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1627 MCID: TII.get(Opcode: TargetOpcode::G_PTRTOINT), DestReg: IntReg)
1628 .addReg(RegNo: PtrReg);
1629 Op.setReg(IntReg);
1630 return select(MI&: *PtrToInt);
1631}
1632
1633void RISCVInstructionSelector::preISelLower(MachineInstr &MI) {
1634 switch (MI.getOpcode()) {
1635 case TargetOpcode::G_PTR_ADD: {
1636 Register DstReg = MI.getOperand(i: 0).getReg();
1637 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1638
1639 replacePtrWithInt(MI, OpIdx: 1);
1640 MI.setDesc(TII.get(Opcode: TargetOpcode::G_ADD));
1641 MRI->setType(VReg: DstReg, Ty: sXLen);
1642 break;
1643 }
1644 case TargetOpcode::G_PTRMASK: {
1645 Register DstReg = MI.getOperand(i: 0).getReg();
1646 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
1647 replacePtrWithInt(MI, OpIdx: 1);
1648 MI.setDesc(TII.get(Opcode: TargetOpcode::G_AND));
1649 MRI->setType(VReg: DstReg, Ty: sXLen);
1650 break;
1651 }
1652 }
1653}
1654
1655void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
1656 const MachineInstr &MI,
1657 int OpIdx) const {
1658 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1659 "Expected G_CONSTANT");
1660 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
1661 MIB.addImm(Val: -CstVal);
1662}
1663
1664void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
1665 const MachineInstr &MI,
1666 int OpIdx) const {
1667 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1668 "Expected G_CONSTANT");
1669 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1670 MIB.addImm(Val: STI.getXLen() - CstVal);
1671}
1672
1673void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
1674 const MachineInstr &MI,
1675 int OpIdx) const {
1676 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1677 "Expected G_CONSTANT");
1678 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1679 MIB.addImm(Val: 32 - CstVal);
1680}
1681
1682void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
1683 const MachineInstr &MI,
1684 int OpIdx) const {
1685 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1686 "Expected G_CONSTANT");
1687 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
1688 MIB.addImm(Val: CstVal + 1);
1689}
1690
1691void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
1692 const MachineInstr &MI,
1693 int OpIdx) const {
1694 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1695 "Expected G_CONSTANT");
1696 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
1697 MIB.addImm(Val: llvm::countr_zero(Val: C));
1698}
1699
1700void RISCVInstructionSelector::renderXLenSubTrailingOnes(
1701 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
1702 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1703 "Expected G_CONSTANT");
1704 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
1705 MIB.addImm(Val: Subtarget->getXLen() - llvm::countr_one(Value: C));
1706}
1707
1708void RISCVInstructionSelector::renderAddiPairImmSmall(MachineInstrBuilder &MIB,
1709 const MachineInstr &MI,
1710 int OpIdx) const {
1711 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1712 "Expected G_CONSTANT");
1713 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
1714 int64_t Adj = Imm < 0 ? -2048 : 2047;
1715 MIB.addImm(Val: Imm - Adj);
1716}
1717
1718void RISCVInstructionSelector::renderAddiPairImmLarge(MachineInstrBuilder &MIB,
1719 const MachineInstr &MI,
1720 int OpIdx) const {
1721 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1722 "Expected G_CONSTANT");
1723 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue() < 0 ? -2048 : 2047;
1724 MIB.addImm(Val: Imm);
1725}
1726
1727bool RISCVInstructionSelector::isRegInGprb(Register Reg) const {
1728 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::GPRBRegBankID;
1729}
1730
1731bool RISCVInstructionSelector::isRegInFprb(Register Reg) const {
1732 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID;
1733}
1734
1735// A G_PTR_ADD result is worth splitting into Hi (materialized) +
1736// Lo12 (folded offset) only if every user is a plain scalar load/store
1737// using it as the address. Otherwise the ADD is selected on its own with
1738// the full materialized constant, making the Hi materialization here redundant.
1739bool RISCVInstructionSelector::isWorthFoldingAdd(Register AddResult) const {
1740 for (const MachineInstr &User : MRI->use_nodbg_instructions(Reg: AddResult)) {
1741 auto *LdSt = dyn_cast<GLoadStore>(Val: &User);
1742 if (!LdSt)
1743 return false;
1744 // Must be used as the pointer, not the stored value.
1745 if (LdSt->getPointerReg() != AddResult)
1746 return false;
1747 if (isStrongerThanMonotonic(AO: LdSt->getMMO().getSuccessOrdering()))
1748 return false;
1749 // Only scalar integer/f16/f32/f64 memory (exclude vectors, f128, ...).
1750 LLT Ty = MRI->getType(Reg: User.getOperand(i: 0).getReg());
1751 if (!Ty.isScalar() || Ty.getSizeInBits() > 64)
1752 return false;
1753 }
1754 return true;
1755}
1756
1757bool RISCVInstructionSelector::selectCopy(MachineInstr &MI) const {
1758 Register DstReg = MI.getOperand(i: 0).getReg();
1759
1760 if (DstReg.isPhysical())
1761 return true;
1762
1763 const TargetRegisterClass *DstRC =
1764 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
1765
1766 assert(DstRC &&
1767 "Register class not available for LLT, register bank combination");
1768
1769 // No need to constrain SrcReg. It will get constrained when
1770 // we hit another of its uses or its defs.
1771 // Copies do not have constraints.
1772 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
1773 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1774 << " operand\n");
1775 return false;
1776 }
1777
1778 MI.setDesc(TII.get(Opcode: RISCV::COPY));
1779 return true;
1780}
1781
1782bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI) const {
1783 assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
1784
1785 const Register DstReg = MI.getOperand(i: 0).getReg();
1786 const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(
1787 Ty: MRI->getType(Reg: DstReg), RB: *RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI), Is64Bit: STI.is64Bit());
1788
1789 assert(DstRC &&
1790 "Register class not available for LLT, register bank combination");
1791
1792 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
1793 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1794 << " operand\n");
1795 }
1796 MI.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
1797 return true;
1798}
1799
1800bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
1801 MachineInstr &MI) const {
1802 if (Imm == 0) {
1803 MachineBasicBlock &MBB = *MI.getParent();
1804 DebugLoc DL = MI.getDebugLoc();
1805 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg).addReg(RegNo: RISCV::X0);
1806 RBI.constrainGenericRegister(Reg: DstReg, RC: RISCV::GPRRegClass, MRI&: *MRI);
1807 return true;
1808 }
1809
1810 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: *Subtarget);
1811 return materializeInstSeq(DstReg, Seq, MI);
1812}
1813
1814bool RISCVInstructionSelector::materializeInstSeq(
1815 Register DstReg, const RISCVMatInt::InstSeq &Seq, MachineInstr &MI) const {
1816 assert(!Seq.empty() && "materializeInstSeq requires a non-empty sequence");
1817
1818 MachineBasicBlock &MBB = *MI.getParent();
1819 DebugLoc DL = MI.getDebugLoc();
1820 unsigned NumInsts = Seq.size();
1821 Register SrcReg = RISCV::X0;
1822
1823 for (unsigned i = 0; i < NumInsts; i++) {
1824 Register TmpReg = i < NumInsts - 1
1825 ? MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass)
1826 : DstReg;
1827 const RISCVMatInt::Inst &I = Seq[i];
1828 MachineInstr *Result;
1829
1830 switch (I.getOpndKind()) {
1831 case RISCVMatInt::Imm:
1832 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1833 .addImm(Val: I.getImm());
1834 break;
1835 case RISCVMatInt::RegX0:
1836 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1837 .addReg(RegNo: SrcReg)
1838 .addReg(RegNo: RISCV::X0);
1839 break;
1840 case RISCVMatInt::RegReg:
1841 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1842 .addReg(RegNo: SrcReg)
1843 .addReg(RegNo: SrcReg);
1844 break;
1845 case RISCVMatInt::RegImm:
1846 Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg)
1847 .addReg(RegNo: SrcReg)
1848 .addImm(Val: I.getImm());
1849 break;
1850 }
1851
1852 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1853
1854 SrcReg = TmpReg;
1855 }
1856
1857 return true;
1858}
1859
1860InstructionSelector::ComplexRendererFns
1861RISCVInstructionSelector::computeConstAddr(int64_t CVal, bool IsPrefetch,
1862 Register OrigBase) const {
1863 // Split the constant into a materialized high part (the base) and
1864 // a simm12 low part (the offset). For prefetch the low part
1865 // must additionally be a multiple of 32 (simm12_lsb00000).
1866 int64_t Lo12 = SignExtend64<12>(x: CVal);
1867 int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
1868 auto emit = [&](ConstAddrPlan Plan) -> ComplexRendererFns {
1869 return {{[=](MachineInstrBuilder &MIB) {
1870 MIB.addReg(RegNo: materializeConstBase(MIB, Plan, OrigBase));
1871 },
1872 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Plan.Lo12); }}};
1873 };
1874 if (!Subtarget->is64Bit() || isInt<32>(x: Hi)) {
1875 if (IsPrefetch && (Lo12 & 0b11111) != 0)
1876 return std::nullopt;
1877 ConstAddrPlan Plan;
1878 Plan.Lo12 = Lo12;
1879 if (Hi) {
1880 Plan.Kind = ConstAddrPlan::LUI;
1881 Plan.Hi20 = (Hi >> 12) & 0xfffff;
1882 }
1883 return emit(std::move(Plan));
1884 }
1885
1886 // Otherwise ask constant materialization how it would handle the constant
1887 // and fold the trailing ADDI into the offset.
1888 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: CVal, STI: *Subtarget);
1889 if (Seq.back().getOpcode() != RISCV::ADDI)
1890 return std::nullopt;
1891 Lo12 = Seq.back().getImm();
1892 if (IsPrefetch && (Lo12 & 0b11111) != 0)
1893 return std::nullopt;
1894 Seq.pop_back();
1895 if (Seq.empty())
1896 return std::nullopt;
1897 ConstAddrPlan Plan;
1898 Plan.Kind = ConstAddrPlan::InstSeq;
1899 Plan.Seq = std::move(Seq);
1900 Plan.Lo12 = Lo12;
1901 return emit(std::move(Plan));
1902}
1903
1904Register
1905RISCVInstructionSelector::materializeConstBase(MachineInstrBuilder &MIB,
1906 const ConstAddrPlan &Plan,
1907 Register OrigBase) const {
1908 MachineBasicBlock &MBB = *MIB->getParent();
1909 DebugLoc DL = MIB->getDebugLoc();
1910 MachineInstr &InsertPt = *MIB.getInstr();
1911
1912 Register HiReg = RISCV::X0;
1913 switch (Plan.Kind) {
1914 case ConstAddrPlan::X0:
1915 break;
1916 case ConstAddrPlan::LUI: {
1917 HiReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1918 MachineInstr *LUI = BuildMI(BB&: MBB, I&: InsertPt, MIMD: DL, MCID: TII.get(Opcode: RISCV::LUI), DestReg: HiReg)
1919 .addImm(Val: Plan.Hi20);
1920 constrainSelectedInstRegOperands(I&: *LUI, TII, TRI, RBI);
1921 break;
1922 }
1923 case ConstAddrPlan::InstSeq: {
1924 HiReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1925 materializeInstSeq(DstReg: HiReg, Seq: Plan.Seq, MI&: InsertPt);
1926 break;
1927 }
1928 }
1929
1930 // For G_PTR_ADD + large constant, add the original base to the materialized
1931 // high part.
1932 if (OrigBase.isValid() && HiReg != RISCV::X0) {
1933 Register BaseReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1934 MachineInstr *Add = BuildMI(BB&: MBB, I&: InsertPt, MIMD: DL, MCID: TII.get(Opcode: RISCV::ADD), DestReg: BaseReg)
1935 .addReg(RegNo: OrigBase)
1936 .addReg(RegNo: HiReg);
1937 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
1938 return BaseReg;
1939 }
1940 return OrigBase.isValid() ? OrigBase : HiReg;
1941}
1942
1943bool RISCVInstructionSelector::selectAddr(MachineInstr &MI, bool IsLocal,
1944 bool IsExternWeak) const {
1945 assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1946 MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
1947 MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
1948 "Unexpected opcode");
1949
1950 const MachineOperand &DispMO = MI.getOperand(i: 1);
1951
1952 Register DefReg = MI.getOperand(i: 0).getReg();
1953 const LLT DefTy = MRI->getType(Reg: DefReg);
1954
1955 // When HWASAN is used and tagging of global variables is enabled
1956 // they should be accessed via the GOT, since the tagged address of a global
1957 // is incompatible with existing code models. This also applies to non-pic
1958 // mode.
1959 if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
1960 if (IsLocal && !Subtarget->allowTaggedGlobals()) {
1961 // Use PC-relative addressing to access the symbol. This generates the
1962 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
1963 // %pcrel_lo(auipc)).
1964 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
1965 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1966 return true;
1967 }
1968
1969 // Use PC-relative addressing to access the GOT for this symbol, then
1970 // load the address from the GOT. This generates the pattern (PseudoLGA
1971 // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
1972 // %pcrel_lo(auipc))).
1973 MachineFunction &MF = *MI.getParent()->getParent();
1974 MachineMemOperand *MemOp = MF.getMachineMemOperand(
1975 PtrInfo: MachinePointerInfo::getGOT(MF),
1976 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1977 MachineMemOperand::MOInvariant,
1978 MemTy: DefTy, BaseAlignment: Align(DefTy.getSizeInBits() / 8));
1979
1980 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
1981 MCID: TII.get(Opcode: RISCV::PseudoLGA), DestReg: DefReg)
1982 .addDisp(Disp: DispMO, off: 0)
1983 .addMemOperand(MMO: MemOp);
1984
1985 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1986
1987 MI.eraseFromParent();
1988 return true;
1989 }
1990
1991 switch (TM.getCodeModel()) {
1992 default: {
1993 reportGISelFailure(MF&: *MF, MORE&: *MORE, PassName: getName(),
1994 Msg: "Unsupported code model for lowering", MI);
1995 return false;
1996 }
1997 case CodeModel::Small: {
1998 // Must lie within a single 2 GiB address range and must lie between
1999 // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
2000 // (lui %hi(sym)) %lo(sym)).
2001 Register AddrHiDest = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2002 MachineInstr *AddrHi = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2003 MCID: TII.get(Opcode: RISCV::LUI), DestReg: AddrHiDest)
2004 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_HI);
2005
2006 constrainSelectedInstRegOperands(I&: *AddrHi, TII, TRI, RBI);
2007
2008 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2009 MCID: TII.get(Opcode: RISCV::ADDI), DestReg: DefReg)
2010 .addReg(RegNo: AddrHiDest)
2011 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_LO);
2012
2013 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
2014
2015 MI.eraseFromParent();
2016 return true;
2017 }
2018 case CodeModel::Medium:
2019 // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
2020 // relocation needs to reference a label that points to the auipc
2021 // instruction itself, not the global. This cannot be done inside the
2022 // instruction selector.
2023 if (IsExternWeak) {
2024 // An extern weak symbol may be undefined, i.e. have value 0, which may
2025 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
2026 // symbol. This generates the pattern (PseudoLGA sym), which expands to
2027 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
2028 MachineFunction &MF = *MI.getParent()->getParent();
2029 MachineMemOperand *MemOp = MF.getMachineMemOperand(
2030 PtrInfo: MachinePointerInfo::getGOT(MF),
2031 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
2032 MachineMemOperand::MOInvariant,
2033 MemTy: DefTy, BaseAlignment: Align(DefTy.getSizeInBits() / 8));
2034
2035 MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2036 MCID: TII.get(Opcode: RISCV::PseudoLGA), DestReg: DefReg)
2037 .addDisp(Disp: DispMO, off: 0)
2038 .addMemOperand(MMO: MemOp);
2039
2040 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
2041
2042 MI.eraseFromParent();
2043 return true;
2044 }
2045
2046 // Generate a sequence for accessing addresses within any 2GiB range
2047 // within the address space. This generates the pattern (PseudoLLA sym),
2048 // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
2049 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
2050 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
2051 return true;
2052 }
2053
2054 return false;
2055}
2056
2057bool RISCVInstructionSelector::selectSelect(MachineInstr &MI) const {
2058 auto &SelectMI = cast<GSelect>(Val&: MI);
2059
2060 Register LHS, RHS;
2061 RISCVCC::CondCode CC;
2062 getOperandsForBranch(CondReg: SelectMI.getCondReg(), CC, LHS, RHS, MRI&: *MRI);
2063
2064 Register DstReg = SelectMI.getReg(Idx: 0);
2065
2066 unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
2067 if (RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
2068 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
2069 Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
2070 : RISCV::Select_FPR64_Using_CC_GPR;
2071 }
2072
2073 MachineInstr *Result =
2074 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: Opc))
2075 .addDef(RegNo: DstReg)
2076 .addReg(RegNo: LHS)
2077 .addReg(RegNo: RHS)
2078 .addImm(Val: CC)
2079 .addReg(RegNo: SelectMI.getTrueReg())
2080 .addReg(RegNo: SelectMI.getFalseReg());
2081 MI.eraseFromParent();
2082 constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
2083 return true;
2084}
2085
2086// Convert an FCMP predicate to one of the supported F or D instructions.
2087static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
2088 assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");
2089 switch (Pred) {
2090 default:
2091 llvm_unreachable("Unsupported predicate");
2092 case CmpInst::FCMP_OLT:
2093 return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
2094 case CmpInst::FCMP_OLE:
2095 return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
2096 case CmpInst::FCMP_OEQ:
2097 return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
2098 }
2099}
2100
2101// Try legalizing an FCMP by swapping or inverting the predicate to one that
2102// is supported.
2103static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,
2104 CmpInst::Predicate &Pred, bool &NeedInvert) {
2105 auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
2106 return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
2107 Pred == CmpInst::FCMP_OEQ;
2108 };
2109
2110 assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
2111
2112 CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(pred: Pred);
2113 if (isLegalFCmpPredicate(InvPred)) {
2114 Pred = InvPred;
2115 std::swap(a&: LHS, b&: RHS);
2116 return true;
2117 }
2118
2119 InvPred = CmpInst::getInversePredicate(pred: Pred);
2120 NeedInvert = true;
2121 if (isLegalFCmpPredicate(InvPred)) {
2122 Pred = InvPred;
2123 return true;
2124 }
2125 InvPred = CmpInst::getSwappedPredicate(pred: InvPred);
2126 if (isLegalFCmpPredicate(InvPred)) {
2127 Pred = InvPred;
2128 std::swap(a&: LHS, b&: RHS);
2129 return true;
2130 }
2131
2132 return false;
2133}
2134
2135// Emit a sequence of instructions to compare LHS and RHS using Pred. Return
2136// the result in DstReg.
2137// FIXME: Maybe we should expand this earlier.
2138bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI) const {
2139 auto &CmpMI = cast<GFCmp>(Val&: MI);
2140 CmpInst::Predicate Pred = CmpMI.getCond();
2141
2142 Register DstReg = CmpMI.getReg(Idx: 0);
2143 Register LHS = CmpMI.getLHSReg();
2144 Register RHS = CmpMI.getRHSReg();
2145
2146 unsigned Size = MRI->getType(Reg: LHS).getSizeInBits();
2147 assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
2148
2149 Register TmpReg = DstReg;
2150
2151 bool NeedInvert = false;
2152 // First try swapping operands or inverting.
2153 if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
2154 if (NeedInvert)
2155 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2156 MachineInstr *Cmp = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2157 MCID: TII.get(Opcode: getFCmpOpcode(Pred, Size)), DestReg: TmpReg)
2158 .setMIFlag(MachineInstr::NoFPExcept)
2159 .addReg(RegNo: LHS)
2160 .addReg(RegNo: RHS);
2161 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
2162 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
2163 // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
2164 NeedInvert = Pred == CmpInst::FCMP_UEQ;
2165 Register Cmp1Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2166 MachineInstr *Cmp1 =
2167 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2168 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size)), DestReg: Cmp1Reg)
2169 .setMIFlag(MachineInstr::NoFPExcept)
2170 .addReg(RegNo: LHS)
2171 .addReg(RegNo: RHS);
2172 constrainSelectedInstRegOperands(I&: *Cmp1, TII, TRI, RBI);
2173 Register Cmp2Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2174 MachineInstr *Cmp2 =
2175 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2176 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size)), DestReg: Cmp2Reg)
2177 .setMIFlag(MachineInstr::NoFPExcept)
2178 .addReg(RegNo: RHS)
2179 .addReg(RegNo: LHS);
2180 constrainSelectedInstRegOperands(I&: *Cmp2, TII, TRI, RBI);
2181 if (NeedInvert)
2182 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2183 MachineInstr *Or = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2184 MCID: TII.get(Opcode: RISCV::OR), DestReg: TmpReg)
2185 .addReg(RegNo: Cmp1Reg)
2186 .addReg(RegNo: Cmp2Reg);
2187 constrainSelectedInstRegOperands(I&: *Or, TII, TRI, RBI);
2188 } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
2189 // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
2190 // If LHS and RHS are the same, a single FEQ suffices.
2191 NeedInvert = Pred == CmpInst::FCMP_UNO;
2192 if (NeedInvert)
2193 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2194 if (LHS == RHS) {
2195 MachineInstr *Cmp =
2196 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2197 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: TmpReg)
2198 .setMIFlag(MachineInstr::NoFPExcept)
2199 .addReg(RegNo: LHS)
2200 .addReg(RegNo: LHS);
2201 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
2202 } else {
2203 Register Cmp1Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2204 MachineInstr *Cmp1 =
2205 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2206 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: Cmp1Reg)
2207 .setMIFlag(MachineInstr::NoFPExcept)
2208 .addReg(RegNo: LHS)
2209 .addReg(RegNo: LHS);
2210 constrainSelectedInstRegOperands(I&: *Cmp1, TII, TRI, RBI);
2211 Register Cmp2Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2212 MachineInstr *Cmp2 =
2213 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2214 MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: Cmp2Reg)
2215 .setMIFlag(MachineInstr::NoFPExcept)
2216 .addReg(RegNo: RHS)
2217 .addReg(RegNo: RHS);
2218 constrainSelectedInstRegOperands(I&: *Cmp2, TII, TRI, RBI);
2219 MachineInstr *And = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2220 MCID: TII.get(Opcode: RISCV::AND), DestReg: TmpReg)
2221 .addReg(RegNo: Cmp1Reg)
2222 .addReg(RegNo: Cmp2Reg);
2223 constrainSelectedInstRegOperands(I&: *And, TII, TRI, RBI);
2224 }
2225 } else
2226 llvm_unreachable("Unhandled predicate");
2227
2228 // Emit an XORI to invert the result if needed.
2229 if (NeedInvert) {
2230 MachineInstr *Xor = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2231 MCID: TII.get(Opcode: RISCV::XORI), DestReg: DstReg)
2232 .addReg(RegNo: TmpReg)
2233 .addImm(Val: 1);
2234 constrainSelectedInstRegOperands(I&: *Xor, TII, TRI, RBI);
2235 }
2236
2237 MI.eraseFromParent();
2238 return true;
2239}
2240
2241void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
2242 SyncScope::ID FenceSSID,
2243 MachineInstr &MI) const {
2244 MachineBasicBlock &MBB = *MI.getParent();
2245 DebugLoc DL = MI.getDebugLoc();
2246
2247 if (STI.hasStdExtZtso()) {
2248 // The only fence that needs an instruction is a sequentially-consistent
2249 // cross-thread fence.
2250 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
2251 FenceSSID == SyncScope::System) {
2252 // fence rw, rw
2253 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE))
2254 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W)
2255 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W);
2256 return;
2257 }
2258
2259 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
2260 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER));
2261 return;
2262 }
2263
2264 // singlethread fences only synchronize with signal handlers on the same
2265 // thread and thus only need to preserve instruction order, not actually
2266 // enforce memory ordering.
2267 if (FenceSSID == SyncScope::SingleThread) {
2268 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER));
2269 return;
2270 }
2271
2272 // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
2273 // Manual: Volume I.
2274 unsigned Pred, Succ;
2275 switch (FenceOrdering) {
2276 default:
2277 llvm_unreachable("Unexpected ordering");
2278 case AtomicOrdering::AcquireRelease:
2279 // fence acq_rel -> fence.tso
2280 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE_TSO));
2281 return;
2282 case AtomicOrdering::Acquire:
2283 // fence acquire -> fence r, rw
2284 Pred = RISCVFenceField::R;
2285 Succ = RISCVFenceField::R | RISCVFenceField::W;
2286 break;
2287 case AtomicOrdering::Release:
2288 // fence release -> fence rw, w
2289 Pred = RISCVFenceField::R | RISCVFenceField::W;
2290 Succ = RISCVFenceField::W;
2291 break;
2292 case AtomicOrdering::SequentiallyConsistent:
2293 // fence seq_cst -> fence rw, rw
2294 Pred = RISCVFenceField::R | RISCVFenceField::W;
2295 Succ = RISCVFenceField::R | RISCVFenceField::W;
2296 break;
2297 }
2298 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE)).addImm(Val: Pred).addImm(Val: Succ);
2299}
2300
2301namespace llvm {
2302InstructionSelector *
2303createRISCVInstructionSelector(const RISCVTargetMachine &TM,
2304 const RISCVSubtarget &Subtarget,
2305 const RISCVRegisterBankInfo &RBI) {
2306 return new RISCVInstructionSelector(TM, Subtarget, RBI);
2307}
2308} // end namespace llvm
2309