1//===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// RISC-V.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/RISCVMatInt.h"
15#include "RISCVRegisterBankInfo.h"
16#include "RISCVSubtarget.h"
17#include "RISCVTargetMachine.h"
18#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
19#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
20#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
21#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
22#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/MachineJumpTableInfo.h"
25#include "llvm/IR/IntrinsicsRISCV.h"
26#include "llvm/Support/Debug.h"
27
28#define DEBUG_TYPE "riscv-isel"
29
30using namespace llvm;
31using namespace MIPatternMatch;
32
33#define GET_GLOBALISEL_PREDICATE_BITSET
34#include "RISCVGenGlobalISel.inc"
35#undef GET_GLOBALISEL_PREDICATE_BITSET
36
37namespace {
38
39class RISCVInstructionSelector : public InstructionSelector {
40public:
41 RISCVInstructionSelector(const RISCVTargetMachine &TM,
42 const RISCVSubtarget &STI,
43 const RISCVRegisterBankInfo &RBI);
44
45 bool select(MachineInstr &MI) override;
46
47 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
48 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
49 BlockFrequencyInfo *BFI) override {
50 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
51 MRI = &MF.getRegInfo();
52 }
53
54 static const char *getName() { return DEBUG_TYPE; }
55
56private:
57 const TargetRegisterClass *
58 getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB) const;
59
60 static constexpr unsigned MaxRecursionDepth = 6;
61
62 bool hasAllNBitUsers(const MachineInstr &MI, unsigned Bits,
63 const unsigned Depth = 0) const;
64 bool hasAllHUsers(const MachineInstr &MI) const {
65 return hasAllNBitUsers(MI, Bits: 16);
66 }
67 bool hasAllWUsers(const MachineInstr &MI) const {
68 return hasAllNBitUsers(MI, Bits: 32);
69 }
70
71 bool isRegInGprb(Register Reg) const;
72 bool isRegInFprb(Register Reg) const;
73
74 // tblgen-erated 'select' implementation, used as the initial selector for
75 // the patterns that don't require complex C++.
76 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
77
78 // A lowering phase that runs before any selection attempts.
79 // Returns true if the instruction was modified.
80 void preISelLower(MachineInstr &MI, MachineIRBuilder &MIB);
81
82 bool replacePtrWithInt(MachineOperand &Op, MachineIRBuilder &MIB);
83
84 // Custom selection methods
85 bool selectCopy(MachineInstr &MI) const;
86 bool selectImplicitDef(MachineInstr &MI, MachineIRBuilder &MIB) const;
87 bool materializeImm(Register Reg, int64_t Imm, MachineIRBuilder &MIB) const;
88 bool selectAddr(MachineInstr &MI, MachineIRBuilder &MIB, bool IsLocal = true,
89 bool IsExternWeak = false) const;
90 bool selectSelect(MachineInstr &MI, MachineIRBuilder &MIB) const;
91 bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB) const;
92 void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
93 MachineIRBuilder &MIB) const;
94 bool selectUnmergeValues(MachineInstr &MI, MachineIRBuilder &MIB) const;
95
96 ComplexRendererFns selectShiftMask(MachineOperand &Root,
97 unsigned ShiftWidth) const;
98 ComplexRendererFns selectShiftMaskXLen(MachineOperand &Root) const {
99 return selectShiftMask(Root, ShiftWidth: STI.getXLen());
100 }
101 ComplexRendererFns selectShiftMask32(MachineOperand &Root) const {
102 return selectShiftMask(Root, ShiftWidth: 32);
103 }
104 ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
105
106 ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;
107 template <unsigned Bits>
108 ComplexRendererFns selectSExtBits(MachineOperand &Root) const {
109 return selectSExtBits(Root, Bits);
110 }
111
112 ComplexRendererFns selectZExtBits(MachineOperand &Root, unsigned Bits) const;
113 template <unsigned Bits>
114 ComplexRendererFns selectZExtBits(MachineOperand &Root) const {
115 return selectZExtBits(Root, Bits);
116 }
117
118 ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
119 template <unsigned ShAmt>
120 ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
121 return selectSHXADDOp(Root, ShAmt);
122 }
123
124 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
125 unsigned ShAmt) const;
126 template <unsigned ShAmt>
127 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
128 return selectSHXADD_UWOp(Root, ShAmt);
129 }
130
131 ComplexRendererFns renderVLOp(MachineOperand &Root) const;
132
133 // Custom renderers for tablegen
134 void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
135 int OpIdx) const;
136 void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
137 int OpIdx) const;
138 void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
139 int OpIdx) const;
140 void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
141 int OpIdx) const;
142 void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI,
143 int OpIdx) const;
144
145 void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
146 int OpIdx) const;
147 void renderXLenSubTrailingOnes(MachineInstrBuilder &MIB,
148 const MachineInstr &MI, int OpIdx) const;
149
150 void renderAddiPairImmLarge(MachineInstrBuilder &MIB, const MachineInstr &MI,
151 int OpIdx) const;
152 void renderAddiPairImmSmall(MachineInstrBuilder &MIB, const MachineInstr &MI,
153 int OpIdx) const;
154
155 const RISCVSubtarget &STI;
156 const RISCVInstrInfo &TII;
157 const RISCVRegisterInfo &TRI;
158 const RISCVRegisterBankInfo &RBI;
159 const RISCVTargetMachine &TM;
160
161 MachineRegisterInfo *MRI = nullptr;
162
163 // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
164 // uses "STI." in the code generated by TableGen. We need to unify the name of
165 // Subtarget variable.
166 const RISCVSubtarget *Subtarget = &STI;
167
168#define GET_GLOBALISEL_PREDICATES_DECL
169#include "RISCVGenGlobalISel.inc"
170#undef GET_GLOBALISEL_PREDICATES_DECL
171
172#define GET_GLOBALISEL_TEMPORARIES_DECL
173#include "RISCVGenGlobalISel.inc"
174#undef GET_GLOBALISEL_TEMPORARIES_DECL
175};
176
177} // end anonymous namespace
178
179#define GET_GLOBALISEL_IMPL
180#include "RISCVGenGlobalISel.inc"
181#undef GET_GLOBALISEL_IMPL
182
183RISCVInstructionSelector::RISCVInstructionSelector(
184 const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
185 const RISCVRegisterBankInfo &RBI)
186 : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
187 TM(TM),
188
189#define GET_GLOBALISEL_PREDICATES_INIT
190#include "RISCVGenGlobalISel.inc"
191#undef GET_GLOBALISEL_PREDICATES_INIT
192#define GET_GLOBALISEL_TEMPORARIES_INIT
193#include "RISCVGenGlobalISel.inc"
194#undef GET_GLOBALISEL_TEMPORARIES_INIT
195{
196}
197
198// Mimics optimizations in ISel and RISCVOptWInst Pass
199bool RISCVInstructionSelector::hasAllNBitUsers(const MachineInstr &MI,
200 unsigned Bits,
201 const unsigned Depth) const {
202
203 assert((MI.getOpcode() == TargetOpcode::G_ADD ||
204 MI.getOpcode() == TargetOpcode::G_SUB ||
205 MI.getOpcode() == TargetOpcode::G_MUL ||
206 MI.getOpcode() == TargetOpcode::G_SHL ||
207 MI.getOpcode() == TargetOpcode::G_LSHR ||
208 MI.getOpcode() == TargetOpcode::G_AND ||
209 MI.getOpcode() == TargetOpcode::G_OR ||
210 MI.getOpcode() == TargetOpcode::G_XOR ||
211 MI.getOpcode() == TargetOpcode::G_SEXT_INREG || Depth != 0) &&
212 "Unexpected opcode");
213
214 if (Depth >= RISCVInstructionSelector::MaxRecursionDepth)
215 return false;
216
217 auto DestReg = MI.getOperand(i: 0).getReg();
218 for (auto &UserOp : MRI->use_nodbg_operands(Reg: DestReg)) {
219 assert(UserOp.getParent() && "UserOp must have a parent");
220 const MachineInstr &UserMI = *UserOp.getParent();
221 unsigned OpIdx = UserOp.getOperandNo();
222
223 switch (UserMI.getOpcode()) {
224 default:
225 return false;
226 case RISCV::ADDW:
227 case RISCV::ADDIW:
228 case RISCV::SUBW:
229 if (Bits >= 32)
230 break;
231 return false;
232 case RISCV::SLL:
233 case RISCV::SRA:
234 case RISCV::SRL:
235 // Shift amount operands only use log2(Xlen) bits.
236 if (OpIdx == 2 && Bits >= Log2_32(Value: Subtarget->getXLen()))
237 break;
238 return false;
239 case RISCV::SLLI:
240 // SLLI only uses the lower (XLen - ShAmt) bits.
241 if (Bits >= Subtarget->getXLen() - UserMI.getOperand(i: 2).getImm())
242 break;
243 return false;
244 case RISCV::ANDI:
245 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(
246 Value: (uint64_t)UserMI.getOperand(i: 2).getImm()))
247 break;
248 goto RecCheck;
249 case RISCV::AND:
250 case RISCV::OR:
251 case RISCV::XOR:
252 RecCheck:
253 if (hasAllNBitUsers(MI: UserMI, Bits, Depth: Depth + 1))
254 break;
255 return false;
256 case RISCV::SRLI: {
257 unsigned ShAmt = UserMI.getOperand(i: 2).getImm();
258 // If we are shifting right by less than Bits, and users don't demand any
259 // bits that were shifted into [Bits-1:0], then we can consider this as an
260 // N-Bit user.
261 if (Bits > ShAmt && hasAllNBitUsers(MI: UserMI, Bits: Bits - ShAmt, Depth: Depth + 1))
262 break;
263 return false;
264 }
265 }
266 }
267
268 return true;
269}
270
271InstructionSelector::ComplexRendererFns
272RISCVInstructionSelector::selectShiftMask(MachineOperand &Root,
273 unsigned ShiftWidth) const {
274 if (!Root.isReg())
275 return std::nullopt;
276
277 using namespace llvm::MIPatternMatch;
278
279 Register ShAmtReg = Root.getReg();
280 // Peek through zext.
281 Register ZExtSrcReg;
282 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrcReg))))
283 ShAmtReg = ZExtSrcReg;
284
285 APInt AndMask;
286 Register AndSrcReg;
287 // Try to combine the following pattern (applicable to other shift
288 // instructions as well as 32-bit ones):
289 //
290 // %4:gprb(s64) = G_AND %3, %2
291 // %5:gprb(s64) = G_LSHR %1, %4(s64)
292 //
293 // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
294 // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
295 // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
296 // then it can be eliminated. Given register rs1 or rs2 holding a constant
297 // (the and mask), there are two cases G_AND can be erased:
298 //
299 // 1. the lowest log2(XLEN) bits of the and mask are all set
300 // 2. the bits of the register being masked are already unset (zero set)
301 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: AndSrcReg), R: m_ICst(Cst&: AndMask)))) {
302 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
303 if (ShMask.isSubsetOf(RHS: AndMask)) {
304 ShAmtReg = AndSrcReg;
305 } else {
306 // SimplifyDemandedBits may have optimized the mask so try restoring any
307 // bits that are known zero.
308 KnownBits Known = VT->getKnownBits(R: AndSrcReg);
309 if (ShMask.isSubsetOf(RHS: AndMask | Known.Zero))
310 ShAmtReg = AndSrcReg;
311 }
312 }
313
314 APInt Imm;
315 Register Reg;
316 if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAdd(L: m_Reg(R&: Reg), R: m_ICst(Cst&: Imm)))) {
317 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0)
318 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
319 // to avoid the ADD.
320 ShAmtReg = Reg;
321 } else if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GSub(L: m_ICst(Cst&: Imm), R: m_Reg(R&: Reg)))) {
322 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0) {
323 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
324 // to generate a NEG instead of a SUB of a constant.
325 ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
326 unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
327 return {{[=](MachineInstrBuilder &MIB) {
328 MachineIRBuilder(*MIB.getInstr())
329 .buildInstr(Opc: NegOpc, DstOps: {ShAmtReg}, SrcOps: {Register(RISCV::X0), Reg});
330 MIB.addReg(RegNo: ShAmtReg);
331 }}};
332 }
333 if (Imm.urem(RHS: ShiftWidth) == ShiftWidth - 1) {
334 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
335 // to generate a NOT instead of a SUB of a constant.
336 ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
337 return {{[=](MachineInstrBuilder &MIB) {
338 MachineIRBuilder(*MIB.getInstr())
339 .buildInstr(Opc: RISCV::XORI, DstOps: {ShAmtReg}, SrcOps: {Reg})
340 .addImm(Val: -1);
341 MIB.addReg(RegNo: ShAmtReg);
342 }}};
343 }
344 }
345
346 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ShAmtReg); }}};
347}
348
349InstructionSelector::ComplexRendererFns
350RISCVInstructionSelector::selectSExtBits(MachineOperand &Root,
351 unsigned Bits) const {
352 if (!Root.isReg())
353 return std::nullopt;
354 Register RootReg = Root.getReg();
355 MachineInstr *RootDef = MRI->getVRegDef(Reg: RootReg);
356
357 if (RootDef->getOpcode() == TargetOpcode::G_SEXT_INREG &&
358 RootDef->getOperand(i: 2).getImm() == Bits) {
359 return {
360 {[=](MachineInstrBuilder &MIB) { MIB.add(MO: RootDef->getOperand(i: 1)); }}};
361 }
362
363 unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits();
364 if ((Size - VT->computeNumSignBits(R: RootReg)) < Bits)
365 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}};
366
367 return std::nullopt;
368}
369
370InstructionSelector::ComplexRendererFns
371RISCVInstructionSelector::selectZExtBits(MachineOperand &Root,
372 unsigned Bits) const {
373 if (!Root.isReg())
374 return std::nullopt;
375 Register RootReg = Root.getReg();
376
377 Register RegX;
378 uint64_t Mask = maskTrailingOnes<uint64_t>(N: Bits);
379 if (mi_match(R: RootReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: RegX), R: m_SpecificICst(RequestedValue: Mask)))) {
380 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}};
381 }
382
383 if (mi_match(R: RootReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: RegX))) &&
384 MRI->getType(Reg: RegX).getScalarSizeInBits() == Bits)
385 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}};
386
387 unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits();
388 if (VT->maskedValueIsZero(Val: RootReg, Mask: APInt::getBitsSetFrom(numBits: Size, loBit: Bits)))
389 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}};
390
391 return std::nullopt;
392}
393
394InstructionSelector::ComplexRendererFns
395RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
396 unsigned ShAmt) const {
397 using namespace llvm::MIPatternMatch;
398
399 if (!Root.isReg())
400 return std::nullopt;
401 Register RootReg = Root.getReg();
402
403 const unsigned XLen = STI.getXLen();
404 APInt Mask, C2;
405 Register RegY;
406 std::optional<bool> LeftShift;
407 // (and (shl y, c2), mask)
408 if (mi_match(R: RootReg, MRI: *MRI,
409 P: m_GAnd(L: m_GShl(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
410 LeftShift = true;
411 // (and (lshr y, c2), mask)
412 else if (mi_match(R: RootReg, MRI: *MRI,
413 P: m_GAnd(L: m_GLShr(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
414 LeftShift = false;
415
416 if (LeftShift.has_value()) {
417 if (*LeftShift)
418 Mask &= maskTrailingZeros<uint64_t>(N: C2.getLimitedValue());
419 else
420 Mask &= maskTrailingOnes<uint64_t>(N: XLen - C2.getLimitedValue());
421
422 if (Mask.isShiftedMask()) {
423 unsigned Leading = XLen - Mask.getActiveBits();
424 unsigned Trailing = Mask.countr_zero();
425 // Given (and (shl y, c2), mask) in which mask has no leading zeros and
426 // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
427 if (*LeftShift && Leading == 0 && C2.ult(RHS: Trailing) && Trailing == ShAmt) {
428 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
429 return {{[=](MachineInstrBuilder &MIB) {
430 MachineIRBuilder(*MIB.getInstr())
431 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
432 .addImm(Val: Trailing - C2.getLimitedValue());
433 MIB.addReg(RegNo: DstReg);
434 }}};
435 }
436
437 // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
438 // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
439 if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
440 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
441 return {{[=](MachineInstrBuilder &MIB) {
442 MachineIRBuilder(*MIB.getInstr())
443 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
444 .addImm(Val: Leading + Trailing);
445 MIB.addReg(RegNo: DstReg);
446 }}};
447 }
448 }
449 }
450
451 LeftShift.reset();
452
453 // (shl (and y, mask), c2)
454 if (mi_match(R: RootReg, MRI: *MRI,
455 P: m_GShl(L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
456 R: m_ICst(Cst&: C2))))
457 LeftShift = true;
458 // (lshr (and y, mask), c2)
459 else if (mi_match(R: RootReg, MRI: *MRI,
460 P: m_GLShr(L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
461 R: m_ICst(Cst&: C2))))
462 LeftShift = false;
463
464 if (LeftShift.has_value() && Mask.isShiftedMask()) {
465 unsigned Leading = XLen - Mask.getActiveBits();
466 unsigned Trailing = Mask.countr_zero();
467
468 // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
469 // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
470 bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&
471 (Trailing + C2.getLimitedValue()) == ShAmt;
472 if (!Cond)
473 // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
474 // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
475 Cond = !*LeftShift && Leading == 32 && C2.ult(RHS: Trailing) &&
476 (Trailing - C2.getLimitedValue()) == ShAmt;
477
478 if (Cond) {
479 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
480 return {{[=](MachineInstrBuilder &MIB) {
481 MachineIRBuilder(*MIB.getInstr())
482 .buildInstr(Opc: RISCV::SRLIW, DstOps: {DstReg}, SrcOps: {RegY})
483 .addImm(Val: Trailing);
484 MIB.addReg(RegNo: DstReg);
485 }}};
486 }
487 }
488
489 return std::nullopt;
490}
491
492InstructionSelector::ComplexRendererFns
493RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
494 unsigned ShAmt) const {
495 using namespace llvm::MIPatternMatch;
496
497 if (!Root.isReg())
498 return std::nullopt;
499 Register RootReg = Root.getReg();
500
501 // Given (and (shl x, c2), mask) in which mask is a shifted mask with
502 // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
503 // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
504 APInt Mask, C2;
505 Register RegX;
506 if (mi_match(
507 R: RootReg, MRI: *MRI,
508 P: m_OneNonDBGUse(SP: m_GAnd(L: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: RegX), R: m_ICst(Cst&: C2))),
509 R: m_ICst(Cst&: Mask))))) {
510 Mask &= maskTrailingZeros<uint64_t>(N: C2.getLimitedValue());
511
512 if (Mask.isShiftedMask()) {
513 unsigned Leading = Mask.countl_zero();
514 unsigned Trailing = Mask.countr_zero();
515 if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
516 Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
517 return {{[=](MachineInstrBuilder &MIB) {
518 MachineIRBuilder(*MIB.getInstr())
519 .buildInstr(Opc: RISCV::SLLI, DstOps: {DstReg}, SrcOps: {RegX})
520 .addImm(Val: C2.getLimitedValue() - ShAmt);
521 MIB.addReg(RegNo: DstReg);
522 }}};
523 }
524 }
525 }
526
527 return std::nullopt;
528}
529
530InstructionSelector::ComplexRendererFns
531RISCVInstructionSelector::renderVLOp(MachineOperand &Root) const {
532 assert(Root.isReg() && "Expected operand to be a Register");
533 MachineInstr *RootDef = MRI->getVRegDef(Reg: Root.getReg());
534
535 if (RootDef->getOpcode() == TargetOpcode::G_CONSTANT) {
536 auto C = RootDef->getOperand(i: 1).getCImm();
537 if (C->getValue().isAllOnes())
538 // If the operand is a G_CONSTANT with value of all ones it is larger than
539 // VLMAX. We convert it to an immediate with value VLMaxSentinel. This is
540 // recognized specially by the vsetvli insertion pass.
541 return {{[=](MachineInstrBuilder &MIB) {
542 MIB.addImm(Val: RISCV::VLMaxSentinel);
543 }}};
544
545 if (isUInt<5>(x: C->getZExtValue())) {
546 uint64_t ZExtC = C->getZExtValue();
547 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ZExtC); }}};
548 }
549 }
550 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Root.getReg()); }}};
551}
552
553InstructionSelector::ComplexRendererFns
554RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
555 if (!Root.isReg())
556 return std::nullopt;
557
558 MachineInstr *RootDef = MRI->getVRegDef(Reg: Root.getReg());
559 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
560 return {{
561 [=](MachineInstrBuilder &MIB) { MIB.add(MO: RootDef->getOperand(i: 1)); },
562 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
563 }};
564 }
565
566 if (isBaseWithConstantOffset(Root, MRI: *MRI)) {
567 MachineOperand &LHS = RootDef->getOperand(i: 1);
568 MachineOperand &RHS = RootDef->getOperand(i: 2);
569 MachineInstr *LHSDef = MRI->getVRegDef(Reg: LHS.getReg());
570 MachineInstr *RHSDef = MRI->getVRegDef(Reg: RHS.getReg());
571
572 int64_t RHSC = RHSDef->getOperand(i: 1).getCImm()->getSExtValue();
573 if (isInt<12>(x: RHSC)) {
574 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
575 return {{
576 [=](MachineInstrBuilder &MIB) { MIB.add(MO: LHSDef->getOperand(i: 1)); },
577 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
578 }};
579
580 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: LHS); },
581 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
582 }
583 }
584
585 // TODO: Need to get the immediate from a G_PTR_ADD. Should this be done in
586 // the combiner?
587 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Root.getReg()); },
588 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
589}
590
591/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
592/// CC Must be an ICMP Predicate.
593static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
594 switch (CC) {
595 default:
596 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
597 case CmpInst::Predicate::ICMP_EQ:
598 return RISCVCC::COND_EQ;
599 case CmpInst::Predicate::ICMP_NE:
600 return RISCVCC::COND_NE;
601 case CmpInst::Predicate::ICMP_ULT:
602 return RISCVCC::COND_LTU;
603 case CmpInst::Predicate::ICMP_SLT:
604 return RISCVCC::COND_LT;
605 case CmpInst::Predicate::ICMP_UGE:
606 return RISCVCC::COND_GEU;
607 case CmpInst::Predicate::ICMP_SGE:
608 return RISCVCC::COND_GE;
609 }
610}
611
612static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC,
613 Register &LHS, Register &RHS,
614 MachineRegisterInfo &MRI) {
615 // Try to fold an ICmp. If that fails, use a NE compare with X0.
616 CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
617 if (!mi_match(R: CondReg, MRI, P: m_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: LHS), R: m_Reg(R&: RHS)))) {
618 LHS = CondReg;
619 RHS = RISCV::X0;
620 CC = RISCVCC::COND_NE;
621 return;
622 }
623
624 // We found an ICmp, do some canonicalization.
625
626 // Adjust comparisons to use comparison with 0 if possible.
627 if (auto Constant = getIConstantVRegSExtVal(VReg: RHS, MRI)) {
628 switch (Pred) {
629 case CmpInst::Predicate::ICMP_SGT:
630 // Convert X > -1 to X >= 0
631 if (*Constant == -1) {
632 CC = RISCVCC::COND_GE;
633 RHS = RISCV::X0;
634 return;
635 }
636 break;
637 case CmpInst::Predicate::ICMP_SLT:
638 // Convert X < 1 to 0 >= X
639 if (*Constant == 1) {
640 CC = RISCVCC::COND_GE;
641 RHS = LHS;
642 LHS = RISCV::X0;
643 return;
644 }
645 break;
646 default:
647 break;
648 }
649 }
650
651 switch (Pred) {
652 default:
653 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
654 case CmpInst::Predicate::ICMP_EQ:
655 case CmpInst::Predicate::ICMP_NE:
656 case CmpInst::Predicate::ICMP_ULT:
657 case CmpInst::Predicate::ICMP_SLT:
658 case CmpInst::Predicate::ICMP_UGE:
659 case CmpInst::Predicate::ICMP_SGE:
660 // These CCs are supported directly by RISC-V branches.
661 break;
662 case CmpInst::Predicate::ICMP_SGT:
663 case CmpInst::Predicate::ICMP_SLE:
664 case CmpInst::Predicate::ICMP_UGT:
665 case CmpInst::Predicate::ICMP_ULE:
666 // These CCs are not supported directly by RISC-V branches, but changing the
667 // direction of the CC and swapping LHS and RHS are.
668 Pred = CmpInst::getSwappedPredicate(pred: Pred);
669 std::swap(a&: LHS, b&: RHS);
670 break;
671 }
672
673 CC = getRISCVCCFromICmp(CC: Pred);
674}
675
676bool RISCVInstructionSelector::select(MachineInstr &MI) {
677 MachineIRBuilder MIB(MI);
678
679 preISelLower(MI, MIB);
680 const unsigned Opc = MI.getOpcode();
681
682 if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
683 if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
684 const Register DefReg = MI.getOperand(i: 0).getReg();
685 const LLT DefTy = MRI->getType(Reg: DefReg);
686
687 const RegClassOrRegBank &RegClassOrBank =
688 MRI->getRegClassOrRegBank(Reg: DefReg);
689
690 const TargetRegisterClass *DefRC =
691 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
692 if (!DefRC) {
693 if (!DefTy.isValid()) {
694 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
695 return false;
696 }
697
698 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
699 DefRC = getRegClassForTypeOnBank(Ty: DefTy, RB);
700 if (!DefRC) {
701 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
702 return false;
703 }
704 }
705
706 MI.setDesc(TII.get(Opcode: TargetOpcode::PHI));
707 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI&: *MRI);
708 }
709
710 // Certain non-generic instructions also need some special handling.
711 if (MI.isCopy())
712 return selectCopy(MI);
713
714 return true;
715 }
716
717 if (selectImpl(I&: MI, CoverageInfo&: *CoverageInfo))
718 return true;
719
720 switch (Opc) {
721 case TargetOpcode::G_ANYEXT:
722 case TargetOpcode::G_PTRTOINT:
723 case TargetOpcode::G_INTTOPTR:
724 case TargetOpcode::G_TRUNC:
725 case TargetOpcode::G_FREEZE:
726 return selectCopy(MI);
727 case TargetOpcode::G_CONSTANT: {
728 Register DstReg = MI.getOperand(i: 0).getReg();
729 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
730
731 if (!materializeImm(Reg: DstReg, Imm, MIB))
732 return false;
733
734 MI.eraseFromParent();
735 return true;
736 }
737 case TargetOpcode::G_FCONSTANT: {
738 // TODO: Use constant pool for complex constants.
739 // TODO: Optimize +0.0 to use fcvt.d.w for s64 on rv32.
740 Register DstReg = MI.getOperand(i: 0).getReg();
741 const APFloat &FPimm = MI.getOperand(i: 1).getFPImm()->getValueAPF();
742 APInt Imm = FPimm.bitcastToAPInt();
743 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
744 if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
745 Register GPRReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
746 if (!materializeImm(Reg: GPRReg, Imm: Imm.getSExtValue(), MIB))
747 return false;
748
749 unsigned Opcode = Size == 64 ? RISCV::FMV_D_X
750 : Size == 32 ? RISCV::FMV_W_X
751 : RISCV::FMV_H_X;
752 auto FMV = MIB.buildInstr(Opc: Opcode, DstOps: {DstReg}, SrcOps: {GPRReg});
753 if (!FMV.constrainAllUses(TII, TRI, RBI))
754 return false;
755 } else {
756 assert(Size == 64 && !Subtarget->is64Bit() &&
757 "Unexpected size or subtarget");
758 // Split into two pieces and build through the stack.
759 Register GPRRegHigh = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
760 Register GPRRegLow = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
761 if (!materializeImm(Reg: GPRRegHigh, Imm: Imm.extractBits(numBits: 32, bitPosition: 32).getSExtValue(),
762 MIB))
763 return false;
764 if (!materializeImm(Reg: GPRRegLow, Imm: Imm.trunc(width: 32).getSExtValue(), MIB))
765 return false;
766 MachineInstrBuilder PairF64 = MIB.buildInstr(
767 Opc: RISCV::BuildPairF64Pseudo, DstOps: {DstReg}, SrcOps: {GPRRegLow, GPRRegHigh});
768 if (!PairF64.constrainAllUses(TII, TRI, RBI))
769 return false;
770 }
771
772 MI.eraseFromParent();
773 return true;
774 }
775 case TargetOpcode::G_GLOBAL_VALUE: {
776 auto *GV = MI.getOperand(i: 1).getGlobal();
777 if (GV->isThreadLocal()) {
778 // TODO: implement this case.
779 return false;
780 }
781
782 return selectAddr(MI, MIB, IsLocal: GV->isDSOLocal(), IsExternWeak: GV->hasExternalWeakLinkage());
783 }
784 case TargetOpcode::G_JUMP_TABLE:
785 case TargetOpcode::G_CONSTANT_POOL:
786 return selectAddr(MI, MIB, IsLocal: MRI);
787 case TargetOpcode::G_BRCOND: {
788 Register LHS, RHS;
789 RISCVCC::CondCode CC;
790 getOperandsForBranch(CondReg: MI.getOperand(i: 0).getReg(), CC, LHS, RHS, MRI&: *MRI);
791
792 auto Bcc = MIB.buildInstr(Opc: RISCVCC::getBrCond(CC), DstOps: {}, SrcOps: {LHS, RHS})
793 .addMBB(MBB: MI.getOperand(i: 1).getMBB());
794 MI.eraseFromParent();
795 return constrainSelectedInstRegOperands(I&: *Bcc, TII, TRI, RBI);
796 }
797 case TargetOpcode::G_BRINDIRECT:
798 MI.setDesc(TII.get(Opcode: RISCV::PseudoBRIND));
799 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
800 return constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
801 case TargetOpcode::G_SELECT:
802 return selectSelect(MI, MIB);
803 case TargetOpcode::G_FCMP:
804 return selectFPCompare(MI, MIB);
805 case TargetOpcode::G_FENCE: {
806 AtomicOrdering FenceOrdering =
807 static_cast<AtomicOrdering>(MI.getOperand(i: 0).getImm());
808 SyncScope::ID FenceSSID =
809 static_cast<SyncScope::ID>(MI.getOperand(i: 1).getImm());
810 emitFence(FenceOrdering, FenceSSID, MIB);
811 MI.eraseFromParent();
812 return true;
813 }
814 case TargetOpcode::G_IMPLICIT_DEF:
815 return selectImplicitDef(MI, MIB);
816 case TargetOpcode::G_UNMERGE_VALUES:
817 return selectUnmergeValues(MI, MIB);
818 default:
819 return false;
820 }
821}
822
823bool RISCVInstructionSelector::selectUnmergeValues(
824 MachineInstr &MI, MachineIRBuilder &MIB) const {
825 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
826
827 if (!Subtarget->hasStdExtZfa())
828 return false;
829
830 // Split F64 Src into two s32 parts
831 if (MI.getNumOperands() != 3)
832 return false;
833 Register Src = MI.getOperand(i: 2).getReg();
834 Register Lo = MI.getOperand(i: 0).getReg();
835 Register Hi = MI.getOperand(i: 1).getReg();
836 if (!isRegInFprb(Reg: Src) || !isRegInGprb(Reg: Lo) || !isRegInGprb(Reg: Hi))
837 return false;
838
839 MachineInstr *ExtractLo = MIB.buildInstr(Opc: RISCV::FMV_X_W_FPR64, DstOps: {Lo}, SrcOps: {Src});
840 if (!constrainSelectedInstRegOperands(I&: *ExtractLo, TII, TRI, RBI))
841 return false;
842
843 MachineInstr *ExtractHi = MIB.buildInstr(Opc: RISCV::FMVH_X_D, DstOps: {Hi}, SrcOps: {Src});
844 if (!constrainSelectedInstRegOperands(I&: *ExtractHi, TII, TRI, RBI))
845 return false;
846
847 MI.eraseFromParent();
848 return true;
849}
850
851bool RISCVInstructionSelector::replacePtrWithInt(MachineOperand &Op,
852 MachineIRBuilder &MIB) {
853 Register PtrReg = Op.getReg();
854 assert(MRI->getType(PtrReg).isPointer() && "Operand is not a pointer!");
855
856 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
857 auto PtrToInt = MIB.buildPtrToInt(Dst: sXLen, Src: PtrReg);
858 MRI->setRegBank(Reg: PtrToInt.getReg(Idx: 0), RegBank: RBI.getRegBank(ID: RISCV::GPRBRegBankID));
859 Op.setReg(PtrToInt.getReg(Idx: 0));
860 return select(MI&: *PtrToInt);
861}
862
863void RISCVInstructionSelector::preISelLower(MachineInstr &MI,
864 MachineIRBuilder &MIB) {
865 switch (MI.getOpcode()) {
866 case TargetOpcode::G_PTR_ADD: {
867 Register DstReg = MI.getOperand(i: 0).getReg();
868 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
869
870 replacePtrWithInt(Op&: MI.getOperand(i: 1), MIB);
871 MI.setDesc(TII.get(Opcode: TargetOpcode::G_ADD));
872 MRI->setType(VReg: DstReg, Ty: sXLen);
873 break;
874 }
875 case TargetOpcode::G_PTRMASK: {
876 Register DstReg = MI.getOperand(i: 0).getReg();
877 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
878 replacePtrWithInt(Op&: MI.getOperand(i: 1), MIB);
879 MI.setDesc(TII.get(Opcode: TargetOpcode::G_AND));
880 MRI->setType(VReg: DstReg, Ty: sXLen);
881 break;
882 }
883 }
884}
885
886void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
887 const MachineInstr &MI,
888 int OpIdx) const {
889 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
890 "Expected G_CONSTANT");
891 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
892 MIB.addImm(Val: -CstVal);
893}
894
895void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
896 const MachineInstr &MI,
897 int OpIdx) const {
898 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
899 "Expected G_CONSTANT");
900 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
901 MIB.addImm(Val: STI.getXLen() - CstVal);
902}
903
904void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
905 const MachineInstr &MI,
906 int OpIdx) const {
907 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
908 "Expected G_CONSTANT");
909 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
910 MIB.addImm(Val: 32 - CstVal);
911}
912
913void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
914 const MachineInstr &MI,
915 int OpIdx) const {
916 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
917 "Expected G_CONSTANT");
918 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
919 MIB.addImm(Val: CstVal + 1);
920}
921
922void RISCVInstructionSelector::renderFrameIndex(MachineInstrBuilder &MIB,
923 const MachineInstr &MI,
924 int OpIdx) const {
925 assert(MI.getOpcode() == TargetOpcode::G_FRAME_INDEX && OpIdx == -1 &&
926 "Expected G_FRAME_INDEX");
927 MIB.add(MO: MI.getOperand(i: 1));
928}
929
930void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
931 const MachineInstr &MI,
932 int OpIdx) const {
933 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
934 "Expected G_CONSTANT");
935 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
936 MIB.addImm(Val: llvm::countr_zero(Val: C));
937}
938
939void RISCVInstructionSelector::renderXLenSubTrailingOnes(
940 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
941 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
942 "Expected G_CONSTANT");
943 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
944 MIB.addImm(Val: Subtarget->getXLen() - llvm::countr_one(Value: C));
945}
946
947void RISCVInstructionSelector::renderAddiPairImmSmall(MachineInstrBuilder &MIB,
948 const MachineInstr &MI,
949 int OpIdx) const {
950 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
951 "Expected G_CONSTANT");
952 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
953 int64_t Adj = Imm < 0 ? -2048 : 2047;
954 MIB.addImm(Val: Imm - Adj);
955}
956
957void RISCVInstructionSelector::renderAddiPairImmLarge(MachineInstrBuilder &MIB,
958 const MachineInstr &MI,
959 int OpIdx) const {
960 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
961 "Expected G_CONSTANT");
962 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue() < 0 ? -2048 : 2047;
963 MIB.addImm(Val: Imm);
964}
965
966const TargetRegisterClass *RISCVInstructionSelector::getRegClassForTypeOnBank(
967 LLT Ty, const RegisterBank &RB) const {
968 if (RB.getID() == RISCV::GPRBRegBankID) {
969 if (Ty.getSizeInBits() <= 32 || (STI.is64Bit() && Ty.getSizeInBits() == 64))
970 return &RISCV::GPRRegClass;
971 }
972
973 if (RB.getID() == RISCV::FPRBRegBankID) {
974 if (Ty.getSizeInBits() == 16)
975 return &RISCV::FPR16RegClass;
976 if (Ty.getSizeInBits() == 32)
977 return &RISCV::FPR32RegClass;
978 if (Ty.getSizeInBits() == 64)
979 return &RISCV::FPR64RegClass;
980 }
981
982 if (RB.getID() == RISCV::VRBRegBankID) {
983 if (Ty.getSizeInBits().getKnownMinValue() <= 64)
984 return &RISCV::VRRegClass;
985
986 if (Ty.getSizeInBits().getKnownMinValue() == 128)
987 return &RISCV::VRM2RegClass;
988
989 if (Ty.getSizeInBits().getKnownMinValue() == 256)
990 return &RISCV::VRM4RegClass;
991
992 if (Ty.getSizeInBits().getKnownMinValue() == 512)
993 return &RISCV::VRM8RegClass;
994 }
995
996 return nullptr;
997}
998
999bool RISCVInstructionSelector::isRegInGprb(Register Reg) const {
1000 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::GPRBRegBankID;
1001}
1002
1003bool RISCVInstructionSelector::isRegInFprb(Register Reg) const {
1004 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID;
1005}
1006
1007bool RISCVInstructionSelector::selectCopy(MachineInstr &MI) const {
1008 Register DstReg = MI.getOperand(i: 0).getReg();
1009
1010 if (DstReg.isPhysical())
1011 return true;
1012
1013 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
1014 Ty: MRI->getType(Reg: DstReg), RB: *RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI));
1015 assert(DstRC &&
1016 "Register class not available for LLT, register bank combination");
1017
1018 // No need to constrain SrcReg. It will get constrained when
1019 // we hit another of its uses or its defs.
1020 // Copies do not have constraints.
1021 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
1022 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1023 << " operand\n");
1024 return false;
1025 }
1026
1027 MI.setDesc(TII.get(Opcode: RISCV::COPY));
1028 return true;
1029}
1030
1031bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI,
1032 MachineIRBuilder &MIB) const {
1033 assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
1034
1035 const Register DstReg = MI.getOperand(i: 0).getReg();
1036 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
1037 Ty: MRI->getType(Reg: DstReg), RB: *RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI));
1038
1039 assert(DstRC &&
1040 "Register class not available for LLT, register bank combination");
1041
1042 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
1043 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1044 << " operand\n");
1045 }
1046 MI.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
1047 return true;
1048}
1049
1050bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
1051 MachineIRBuilder &MIB) const {
1052 if (Imm == 0) {
1053 MIB.buildCopy(Res: DstReg, Op: Register(RISCV::X0));
1054 RBI.constrainGenericRegister(Reg: DstReg, RC: RISCV::GPRRegClass, MRI&: *MRI);
1055 return true;
1056 }
1057
1058 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: *Subtarget);
1059 unsigned NumInsts = Seq.size();
1060 Register SrcReg = RISCV::X0;
1061
1062 for (unsigned i = 0; i < NumInsts; i++) {
1063 Register TmpReg = i < NumInsts - 1
1064 ? MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass)
1065 : DstReg;
1066 const RISCVMatInt::Inst &I = Seq[i];
1067 MachineInstr *Result;
1068
1069 switch (I.getOpndKind()) {
1070 case RISCVMatInt::Imm:
1071 // clang-format off
1072 Result = MIB.buildInstr(Opc: I.getOpcode(), DstOps: {TmpReg}, SrcOps: {})
1073 .addImm(Val: I.getImm());
1074 // clang-format on
1075 break;
1076 case RISCVMatInt::RegX0:
1077 Result = MIB.buildInstr(Opc: I.getOpcode(), DstOps: {TmpReg},
1078 SrcOps: {SrcReg, Register(RISCV::X0)});
1079 break;
1080 case RISCVMatInt::RegReg:
1081 Result = MIB.buildInstr(Opc: I.getOpcode(), DstOps: {TmpReg}, SrcOps: {SrcReg, SrcReg});
1082 break;
1083 case RISCVMatInt::RegImm:
1084 Result =
1085 MIB.buildInstr(Opc: I.getOpcode(), DstOps: {TmpReg}, SrcOps: {SrcReg}).addImm(Val: I.getImm());
1086 break;
1087 }
1088
1089 if (!constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI))
1090 return false;
1091
1092 SrcReg = TmpReg;
1093 }
1094
1095 return true;
1096}
1097
1098bool RISCVInstructionSelector::selectAddr(MachineInstr &MI,
1099 MachineIRBuilder &MIB, bool IsLocal,
1100 bool IsExternWeak) const {
1101 assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1102 MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
1103 MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
1104 "Unexpected opcode");
1105
1106 const MachineOperand &DispMO = MI.getOperand(i: 1);
1107
1108 Register DefReg = MI.getOperand(i: 0).getReg();
1109 const LLT DefTy = MRI->getType(Reg: DefReg);
1110
1111 // When HWASAN is used and tagging of global variables is enabled
1112 // they should be accessed via the GOT, since the tagged address of a global
1113 // is incompatible with existing code models. This also applies to non-pic
1114 // mode.
1115 if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
1116 if (IsLocal && !Subtarget->allowTaggedGlobals()) {
1117 // Use PC-relative addressing to access the symbol. This generates the
1118 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
1119 // %pcrel_lo(auipc)).
1120 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
1121 return constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1122 }
1123
1124 // Use PC-relative addressing to access the GOT for this symbol, then
1125 // load the address from the GOT. This generates the pattern (PseudoLGA
1126 // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
1127 // %pcrel_lo(auipc))).
1128 MachineFunction &MF = *MI.getParent()->getParent();
1129 MachineMemOperand *MemOp = MF.getMachineMemOperand(
1130 PtrInfo: MachinePointerInfo::getGOT(MF),
1131 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1132 MachineMemOperand::MOInvariant,
1133 MemTy: DefTy, base_alignment: Align(DefTy.getSizeInBits() / 8));
1134
1135 auto Result = MIB.buildInstr(Opc: RISCV::PseudoLGA, DstOps: {DefReg}, SrcOps: {})
1136 .addDisp(Disp: DispMO, off: 0)
1137 .addMemOperand(MMO: MemOp);
1138
1139 if (!constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI))
1140 return false;
1141
1142 MI.eraseFromParent();
1143 return true;
1144 }
1145
1146 switch (TM.getCodeModel()) {
1147 default: {
1148 reportGISelFailure(MF&: const_cast<MachineFunction &>(*MF), TPC: *TPC, MORE&: *MORE,
1149 PassName: getName(), Msg: "Unsupported code model for lowering", MI);
1150 return false;
1151 }
1152 case CodeModel::Small: {
1153 // Must lie within a single 2 GiB address range and must lie between
1154 // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
1155 // (lui %hi(sym)) %lo(sym)).
1156 Register AddrHiDest = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1157 MachineInstr *AddrHi = MIB.buildInstr(Opc: RISCV::LUI, DstOps: {AddrHiDest}, SrcOps: {})
1158 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_HI);
1159
1160 if (!constrainSelectedInstRegOperands(I&: *AddrHi, TII, TRI, RBI))
1161 return false;
1162
1163 auto Result = MIB.buildInstr(Opc: RISCV::ADDI, DstOps: {DefReg}, SrcOps: {AddrHiDest})
1164 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_LO);
1165
1166 if (!constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI))
1167 return false;
1168
1169 MI.eraseFromParent();
1170 return true;
1171 }
1172 case CodeModel::Medium:
1173 // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
1174 // relocation needs to reference a label that points to the auipc
1175 // instruction itself, not the global. This cannot be done inside the
1176 // instruction selector.
1177 if (IsExternWeak) {
1178 // An extern weak symbol may be undefined, i.e. have value 0, which may
1179 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
1180 // symbol. This generates the pattern (PseudoLGA sym), which expands to
1181 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
1182 MachineFunction &MF = *MI.getParent()->getParent();
1183 MachineMemOperand *MemOp = MF.getMachineMemOperand(
1184 PtrInfo: MachinePointerInfo::getGOT(MF),
1185 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1186 MachineMemOperand::MOInvariant,
1187 MemTy: DefTy, base_alignment: Align(DefTy.getSizeInBits() / 8));
1188
1189 auto Result = MIB.buildInstr(Opc: RISCV::PseudoLGA, DstOps: {DefReg}, SrcOps: {})
1190 .addDisp(Disp: DispMO, off: 0)
1191 .addMemOperand(MMO: MemOp);
1192
1193 if (!constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI))
1194 return false;
1195
1196 MI.eraseFromParent();
1197 return true;
1198 }
1199
1200 // Generate a sequence for accessing addresses within any 2GiB range
1201 // within the address space. This generates the pattern (PseudoLLA sym),
1202 // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
1203 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
1204 return constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1205 }
1206
1207 return false;
1208}
1209
1210bool RISCVInstructionSelector::selectSelect(MachineInstr &MI,
1211 MachineIRBuilder &MIB) const {
1212 auto &SelectMI = cast<GSelect>(Val&: MI);
1213
1214 Register LHS, RHS;
1215 RISCVCC::CondCode CC;
1216 getOperandsForBranch(CondReg: SelectMI.getCondReg(), CC, LHS, RHS, MRI&: *MRI);
1217
1218 Register DstReg = SelectMI.getReg(Idx: 0);
1219
1220 unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
1221 if (RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
1222 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
1223 Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
1224 : RISCV::Select_FPR64_Using_CC_GPR;
1225 }
1226
1227 MachineInstr *Result = MIB.buildInstr(Opcode: Opc)
1228 .addDef(RegNo: DstReg)
1229 .addReg(RegNo: LHS)
1230 .addReg(RegNo: RHS)
1231 .addImm(Val: CC)
1232 .addReg(RegNo: SelectMI.getTrueReg())
1233 .addReg(RegNo: SelectMI.getFalseReg());
1234 MI.eraseFromParent();
1235 return constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1236}
1237
1238// Convert an FCMP predicate to one of the supported F or D instructions.
1239static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
1240 assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");
1241 switch (Pred) {
1242 default:
1243 llvm_unreachable("Unsupported predicate");
1244 case CmpInst::FCMP_OLT:
1245 return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
1246 case CmpInst::FCMP_OLE:
1247 return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
1248 case CmpInst::FCMP_OEQ:
1249 return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
1250 }
1251}
1252
1253// Try legalizing an FCMP by swapping or inverting the predicate to one that
1254// is supported.
1255static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,
1256 CmpInst::Predicate &Pred, bool &NeedInvert) {
1257 auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
1258 return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
1259 Pred == CmpInst::FCMP_OEQ;
1260 };
1261
1262 assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
1263
1264 CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(pred: Pred);
1265 if (isLegalFCmpPredicate(InvPred)) {
1266 Pred = InvPred;
1267 std::swap(a&: LHS, b&: RHS);
1268 return true;
1269 }
1270
1271 InvPred = CmpInst::getInversePredicate(pred: Pred);
1272 NeedInvert = true;
1273 if (isLegalFCmpPredicate(InvPred)) {
1274 Pred = InvPred;
1275 return true;
1276 }
1277 InvPred = CmpInst::getSwappedPredicate(pred: InvPred);
1278 if (isLegalFCmpPredicate(InvPred)) {
1279 Pred = InvPred;
1280 std::swap(a&: LHS, b&: RHS);
1281 return true;
1282 }
1283
1284 return false;
1285}
1286
1287// Emit a sequence of instructions to compare LHS and RHS using Pred. Return
1288// the result in DstReg.
1289// FIXME: Maybe we should expand this earlier.
1290bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
1291 MachineIRBuilder &MIB) const {
1292 auto &CmpMI = cast<GFCmp>(Val&: MI);
1293 CmpInst::Predicate Pred = CmpMI.getCond();
1294
1295 Register DstReg = CmpMI.getReg(Idx: 0);
1296 Register LHS = CmpMI.getLHSReg();
1297 Register RHS = CmpMI.getRHSReg();
1298
1299 unsigned Size = MRI->getType(Reg: LHS).getSizeInBits();
1300 assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
1301
1302 Register TmpReg = DstReg;
1303
1304 bool NeedInvert = false;
1305 // First try swapping operands or inverting.
1306 if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
1307 if (NeedInvert)
1308 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1309 auto Cmp = MIB.buildInstr(Opc: getFCmpOpcode(Pred, Size), DstOps: {TmpReg}, SrcOps: {LHS, RHS});
1310 if (!Cmp.constrainAllUses(TII, TRI, RBI))
1311 return false;
1312 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
1313 // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
1314 NeedInvert = Pred == CmpInst::FCMP_UEQ;
1315 auto Cmp1 = MIB.buildInstr(Opc: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size),
1316 DstOps: {&RISCV::GPRRegClass}, SrcOps: {LHS, RHS});
1317 if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1318 return false;
1319 auto Cmp2 = MIB.buildInstr(Opc: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size),
1320 DstOps: {&RISCV::GPRRegClass}, SrcOps: {RHS, LHS});
1321 if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1322 return false;
1323 if (NeedInvert)
1324 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1325 auto Or =
1326 MIB.buildInstr(Opc: RISCV::OR, DstOps: {TmpReg}, SrcOps: {Cmp1.getReg(Idx: 0), Cmp2.getReg(Idx: 0)});
1327 if (!Or.constrainAllUses(TII, TRI, RBI))
1328 return false;
1329 } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
1330 // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
1331 // FIXME: If LHS and RHS are the same we can use a single FEQ.
1332 NeedInvert = Pred == CmpInst::FCMP_UNO;
1333 auto Cmp1 = MIB.buildInstr(Opc: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size),
1334 DstOps: {&RISCV::GPRRegClass}, SrcOps: {LHS, LHS});
1335 if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1336 return false;
1337 auto Cmp2 = MIB.buildInstr(Opc: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size),
1338 DstOps: {&RISCV::GPRRegClass}, SrcOps: {RHS, RHS});
1339 if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1340 return false;
1341 if (NeedInvert)
1342 TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1343 auto And =
1344 MIB.buildInstr(Opc: RISCV::AND, DstOps: {TmpReg}, SrcOps: {Cmp1.getReg(Idx: 0), Cmp2.getReg(Idx: 0)});
1345 if (!And.constrainAllUses(TII, TRI, RBI))
1346 return false;
1347 } else
1348 llvm_unreachable("Unhandled predicate");
1349
1350 // Emit an XORI to invert the result if needed.
1351 if (NeedInvert) {
1352 auto Xor = MIB.buildInstr(Opc: RISCV::XORI, DstOps: {DstReg}, SrcOps: {TmpReg}).addImm(Val: 1);
1353 if (!Xor.constrainAllUses(TII, TRI, RBI))
1354 return false;
1355 }
1356
1357 MI.eraseFromParent();
1358 return true;
1359}
1360
1361void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
1362 SyncScope::ID FenceSSID,
1363 MachineIRBuilder &MIB) const {
1364 if (STI.hasStdExtZtso()) {
1365 // The only fence that needs an instruction is a sequentially-consistent
1366 // cross-thread fence.
1367 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
1368 FenceSSID == SyncScope::System) {
1369 // fence rw, rw
1370 MIB.buildInstr(Opc: RISCV::FENCE, DstOps: {}, SrcOps: {})
1371 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W)
1372 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W);
1373 return;
1374 }
1375
1376 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
1377 MIB.buildInstr(Opc: TargetOpcode::MEMBARRIER, DstOps: {}, SrcOps: {});
1378 return;
1379 }
1380
1381 // singlethread fences only synchronize with signal handlers on the same
1382 // thread and thus only need to preserve instruction order, not actually
1383 // enforce memory ordering.
1384 if (FenceSSID == SyncScope::SingleThread) {
1385 MIB.buildInstr(Opc: TargetOpcode::MEMBARRIER, DstOps: {}, SrcOps: {});
1386 return;
1387 }
1388
1389 // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
1390 // Manual: Volume I.
1391 unsigned Pred, Succ;
1392 switch (FenceOrdering) {
1393 default:
1394 llvm_unreachable("Unexpected ordering");
1395 case AtomicOrdering::AcquireRelease:
1396 // fence acq_rel -> fence.tso
1397 MIB.buildInstr(Opc: RISCV::FENCE_TSO, DstOps: {}, SrcOps: {});
1398 return;
1399 case AtomicOrdering::Acquire:
1400 // fence acquire -> fence r, rw
1401 Pred = RISCVFenceField::R;
1402 Succ = RISCVFenceField::R | RISCVFenceField::W;
1403 break;
1404 case AtomicOrdering::Release:
1405 // fence release -> fence rw, w
1406 Pred = RISCVFenceField::R | RISCVFenceField::W;
1407 Succ = RISCVFenceField::W;
1408 break;
1409 case AtomicOrdering::SequentiallyConsistent:
1410 // fence seq_cst -> fence rw, rw
1411 Pred = RISCVFenceField::R | RISCVFenceField::W;
1412 Succ = RISCVFenceField::R | RISCVFenceField::W;
1413 break;
1414 }
1415 MIB.buildInstr(Opc: RISCV::FENCE, DstOps: {}, SrcOps: {}).addImm(Val: Pred).addImm(Val: Succ);
1416}
1417
1418namespace llvm {
1419InstructionSelector *
1420createRISCVInstructionSelector(const RISCVTargetMachine &TM,
1421 const RISCVSubtarget &Subtarget,
1422 const RISCVRegisterBankInfo &RBI) {
1423 return new RISCVInstructionSelector(TM, Subtarget, RBI);
1424}
1425} // end namespace llvm
1426