1//===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// RISC-V.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/RISCVMatInt.h"
15#include "RISCVRegisterBankInfo.h"
16#include "RISCVSubtarget.h"
17#include "RISCVTargetMachine.h"
18#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
19#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
20#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
21#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
22#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24#include "llvm/CodeGen/MachineJumpTableInfo.h"
25#include "llvm/IR/IntrinsicsRISCV.h"
26#include "llvm/Support/Debug.h"
27
28#define DEBUG_TYPE "riscv-isel"
29
30using namespace llvm;
31using namespace MIPatternMatch;
32
33#define GET_GLOBALISEL_PREDICATE_BITSET
34#include "RISCVGenGlobalISel.inc"
35#undef GET_GLOBALISEL_PREDICATE_BITSET
36
37namespace {
38
39class RISCVInstructionSelector : public InstructionSelector {
40public:
41 RISCVInstructionSelector(const RISCVTargetMachine &TM,
42 const RISCVSubtarget &STI,
43 const RISCVRegisterBankInfo &RBI);
44
45 bool select(MachineInstr &MI) override;
46 static const char *getName() { return DEBUG_TYPE; }
47
48private:
49 const TargetRegisterClass *
50 getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB) const;
51
52 bool isRegInGprb(Register Reg, MachineRegisterInfo &MRI) const;
53 bool isRegInFprb(Register Reg, MachineRegisterInfo &MRI) const;
54
55 // tblgen-erated 'select' implementation, used as the initial selector for
56 // the patterns that don't require complex C++.
57 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
58
59 // A lowering phase that runs before any selection attempts.
60 // Returns true if the instruction was modified.
61 void preISelLower(MachineInstr &MI, MachineIRBuilder &MIB,
62 MachineRegisterInfo &MRI);
63
64 bool replacePtrWithInt(MachineOperand &Op, MachineIRBuilder &MIB,
65 MachineRegisterInfo &MRI);
66
67 // Custom selection methods
68 bool selectCopy(MachineInstr &MI, MachineRegisterInfo &MRI) const;
69 bool selectImplicitDef(MachineInstr &MI, MachineIRBuilder &MIB,
70 MachineRegisterInfo &MRI) const;
71 bool materializeImm(Register Reg, int64_t Imm, MachineIRBuilder &MIB) const;
72 bool selectAddr(MachineInstr &MI, MachineIRBuilder &MIB,
73 MachineRegisterInfo &MRI, bool IsLocal = true,
74 bool IsExternWeak = false) const;
75 bool selectSExtInreg(MachineInstr &MI, MachineIRBuilder &MIB) const;
76 bool selectSelect(MachineInstr &MI, MachineIRBuilder &MIB,
77 MachineRegisterInfo &MRI) const;
78 bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB,
79 MachineRegisterInfo &MRI) const;
80 void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
81 MachineIRBuilder &MIB) const;
82 bool selectMergeValues(MachineInstr &MI, MachineIRBuilder &MIB,
83 MachineRegisterInfo &MRI) const;
84 bool selectUnmergeValues(MachineInstr &MI, MachineIRBuilder &MIB,
85 MachineRegisterInfo &MRI) const;
86
87 ComplexRendererFns selectShiftMask(MachineOperand &Root) const;
88 ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
89
90 ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
91 template <unsigned ShAmt>
92 ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
93 return selectSHXADDOp(Root, ShAmt);
94 }
95
96 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
97 unsigned ShAmt) const;
98 template <unsigned ShAmt>
99 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
100 return selectSHXADD_UWOp(Root, ShAmt);
101 }
102
103 // Custom renderers for tablegen
104 void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
105 int OpIdx) const;
106 void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
107 int OpIdx) const;
108 void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
109 int OpIdx) const;
110 void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
111 int OpIdx) const;
112 void renderImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
113 int OpIdx) const;
114
115 void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
116 int OpIdx) const;
117
118 const RISCVSubtarget &STI;
119 const RISCVInstrInfo &TII;
120 const RISCVRegisterInfo &TRI;
121 const RISCVRegisterBankInfo &RBI;
122 const RISCVTargetMachine &TM;
123
124 // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
125 // uses "STI." in the code generated by TableGen. We need to unify the name of
126 // Subtarget variable.
127 const RISCVSubtarget *Subtarget = &STI;
128
129#define GET_GLOBALISEL_PREDICATES_DECL
130#include "RISCVGenGlobalISel.inc"
131#undef GET_GLOBALISEL_PREDICATES_DECL
132
133#define GET_GLOBALISEL_TEMPORARIES_DECL
134#include "RISCVGenGlobalISel.inc"
135#undef GET_GLOBALISEL_TEMPORARIES_DECL
136};
137
138} // end anonymous namespace
139
140#define GET_GLOBALISEL_IMPL
141#include "RISCVGenGlobalISel.inc"
142#undef GET_GLOBALISEL_IMPL
143
144RISCVInstructionSelector::RISCVInstructionSelector(
145 const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
146 const RISCVRegisterBankInfo &RBI)
147 : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
148 TM(TM),
149
150#define GET_GLOBALISEL_PREDICATES_INIT
151#include "RISCVGenGlobalISel.inc"
152#undef GET_GLOBALISEL_PREDICATES_INIT
153#define GET_GLOBALISEL_TEMPORARIES_INIT
154#include "RISCVGenGlobalISel.inc"
155#undef GET_GLOBALISEL_TEMPORARIES_INIT
156{
157}
158
159InstructionSelector::ComplexRendererFns
160RISCVInstructionSelector::selectShiftMask(MachineOperand &Root) const {
161 if (!Root.isReg())
162 return std::nullopt;
163
164 using namespace llvm::MIPatternMatch;
165 MachineRegisterInfo &MRI = MF->getRegInfo();
166
167 Register RootReg = Root.getReg();
168 Register ShAmtReg = RootReg;
169 const LLT ShiftLLT = MRI.getType(Reg: RootReg);
170 unsigned ShiftWidth = ShiftLLT.getSizeInBits();
171 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
172 // Peek through zext.
173 Register ZExtSrcReg;
174 if (mi_match(R: ShAmtReg, MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrcReg)))) {
175 ShAmtReg = ZExtSrcReg;
176 }
177
178 APInt AndMask;
179 Register AndSrcReg;
180 // Try to combine the following pattern (applicable to other shift
181 // instructions as well as 32-bit ones):
182 //
183 // %4:gprb(s64) = G_AND %3, %2
184 // %5:gprb(s64) = G_LSHR %1, %4(s64)
185 //
186 // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
187 // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
188 // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
189 // then it can be eliminated. Given register rs1 or rs2 holding a constant
190 // (the and mask), there are two cases G_AND can be erased:
191 //
192 // 1. the lowest log2(XLEN) bits of the and mask are all set
193 // 2. the bits of the register being masked are already unset (zero set)
194 if (mi_match(R: ShAmtReg, MRI, P: m_GAnd(L: m_Reg(R&: AndSrcReg), R: m_ICst(Cst&: AndMask)))) {
195 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
196 if (ShMask.isSubsetOf(RHS: AndMask)) {
197 ShAmtReg = AndSrcReg;
198 } else {
199 // SimplifyDemandedBits may have optimized the mask so try restoring any
200 // bits that are known zero.
201 KnownBits Known = KB->getKnownBits(R: AndSrcReg);
202 if (ShMask.isSubsetOf(RHS: AndMask | Known.Zero))
203 ShAmtReg = AndSrcReg;
204 }
205 }
206
207 APInt Imm;
208 Register Reg;
209 if (mi_match(R: ShAmtReg, MRI, P: m_GAdd(L: m_Reg(R&: Reg), R: m_ICst(Cst&: Imm)))) {
210 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0)
211 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
212 // to avoid the ADD.
213 ShAmtReg = Reg;
214 } else if (mi_match(R: ShAmtReg, MRI, P: m_GSub(L: m_ICst(Cst&: Imm), R: m_Reg(R&: Reg)))) {
215 if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0) {
216 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
217 // to generate a NEG instead of a SUB of a constant.
218 ShAmtReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
219 unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
220 return {{[=](MachineInstrBuilder &MIB) {
221 MachineIRBuilder(*MIB.getInstr())
222 .buildInstr(Opc: NegOpc, DstOps: {ShAmtReg}, SrcOps: {Register(RISCV::X0), Reg});
223 MIB.addReg(RegNo: ShAmtReg);
224 }}};
225 }
226 if (Imm.urem(RHS: ShiftWidth) == ShiftWidth - 1) {
227 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
228 // to generate a NOT instead of a SUB of a constant.
229 ShAmtReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
230 return {{[=](MachineInstrBuilder &MIB) {
231 MachineIRBuilder(*MIB.getInstr())
232 .buildInstr(Opc: RISCV::XORI, DstOps: {ShAmtReg}, SrcOps: {Reg})
233 .addImm(Val: -1);
234 MIB.addReg(RegNo: ShAmtReg);
235 }}};
236 }
237 }
238
239 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ShAmtReg); }}};
240}
241
242InstructionSelector::ComplexRendererFns
243RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
244 unsigned ShAmt) const {
245 using namespace llvm::MIPatternMatch;
246 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
247 MachineRegisterInfo &MRI = MF.getRegInfo();
248
249 if (!Root.isReg())
250 return std::nullopt;
251 Register RootReg = Root.getReg();
252
253 const unsigned XLen = STI.getXLen();
254 APInt Mask, C2;
255 Register RegY;
256 std::optional<bool> LeftShift;
257 // (and (shl y, c2), mask)
258 if (mi_match(R: RootReg, MRI,
259 P: m_GAnd(L: m_GShl(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
260 LeftShift = true;
261 // (and (lshr y, c2), mask)
262 else if (mi_match(R: RootReg, MRI,
263 P: m_GAnd(L: m_GLShr(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask))))
264 LeftShift = false;
265
266 if (LeftShift.has_value()) {
267 if (*LeftShift)
268 Mask &= maskTrailingZeros<uint64_t>(N: C2.getLimitedValue());
269 else
270 Mask &= maskTrailingOnes<uint64_t>(N: XLen - C2.getLimitedValue());
271
272 if (Mask.isShiftedMask()) {
273 unsigned Leading = XLen - Mask.getActiveBits();
274 unsigned Trailing = Mask.countr_zero();
275 // Given (and (shl y, c2), mask) in which mask has no leading zeros and
276 // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
277 if (*LeftShift && Leading == 0 && C2.ult(RHS: Trailing) && Trailing == ShAmt) {
278 Register DstReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
279 return {{[=](MachineInstrBuilder &MIB) {
280 MachineIRBuilder(*MIB.getInstr())
281 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
282 .addImm(Val: Trailing - C2.getLimitedValue());
283 MIB.addReg(RegNo: DstReg);
284 }}};
285 }
286
287 // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
288 // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
289 if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
290 Register DstReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
291 return {{[=](MachineInstrBuilder &MIB) {
292 MachineIRBuilder(*MIB.getInstr())
293 .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY})
294 .addImm(Val: Leading + Trailing);
295 MIB.addReg(RegNo: DstReg);
296 }}};
297 }
298 }
299 }
300
301 LeftShift.reset();
302
303 // (shl (and y, mask), c2)
304 if (mi_match(R: RootReg, MRI,
305 P: m_GShl(L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
306 R: m_ICst(Cst&: C2))))
307 LeftShift = true;
308 // (lshr (and y, mask), c2)
309 else if (mi_match(R: RootReg, MRI,
310 P: m_GLShr(L: m_OneNonDBGUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))),
311 R: m_ICst(Cst&: C2))))
312 LeftShift = false;
313
314 if (LeftShift.has_value() && Mask.isShiftedMask()) {
315 unsigned Leading = XLen - Mask.getActiveBits();
316 unsigned Trailing = Mask.countr_zero();
317
318 // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
319 // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
320 bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&
321 (Trailing + C2.getLimitedValue()) == ShAmt;
322 if (!Cond)
323 // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
324 // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
325 Cond = !*LeftShift && Leading == 32 && C2.ult(RHS: Trailing) &&
326 (Trailing - C2.getLimitedValue()) == ShAmt;
327
328 if (Cond) {
329 Register DstReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
330 return {{[=](MachineInstrBuilder &MIB) {
331 MachineIRBuilder(*MIB.getInstr())
332 .buildInstr(Opc: RISCV::SRLIW, DstOps: {DstReg}, SrcOps: {RegY})
333 .addImm(Val: Trailing);
334 MIB.addReg(RegNo: DstReg);
335 }}};
336 }
337 }
338
339 return std::nullopt;
340}
341
342InstructionSelector::ComplexRendererFns
343RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
344 unsigned ShAmt) const {
345 using namespace llvm::MIPatternMatch;
346 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
347 MachineRegisterInfo &MRI = MF.getRegInfo();
348
349 if (!Root.isReg())
350 return std::nullopt;
351 Register RootReg = Root.getReg();
352
353 // Given (and (shl x, c2), mask) in which mask is a shifted mask with
354 // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
355 // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
356 APInt Mask, C2;
357 Register RegX;
358 if (mi_match(
359 R: RootReg, MRI,
360 P: m_OneNonDBGUse(SP: m_GAnd(L: m_OneNonDBGUse(SP: m_GShl(L: m_Reg(R&: RegX), R: m_ICst(Cst&: C2))),
361 R: m_ICst(Cst&: Mask))))) {
362 Mask &= maskTrailingZeros<uint64_t>(N: C2.getLimitedValue());
363
364 if (Mask.isShiftedMask()) {
365 unsigned Leading = Mask.countl_zero();
366 unsigned Trailing = Mask.countr_zero();
367 if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
368 Register DstReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
369 return {{[=](MachineInstrBuilder &MIB) {
370 MachineIRBuilder(*MIB.getInstr())
371 .buildInstr(Opc: RISCV::SLLI, DstOps: {DstReg}, SrcOps: {RegX})
372 .addImm(Val: C2.getLimitedValue() - ShAmt);
373 MIB.addReg(RegNo: DstReg);
374 }}};
375 }
376 }
377 }
378
379 return std::nullopt;
380}
381
382InstructionSelector::ComplexRendererFns
383RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
384 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
385 MachineRegisterInfo &MRI = MF.getRegInfo();
386
387 if (!Root.isReg())
388 return std::nullopt;
389
390 MachineInstr *RootDef = MRI.getVRegDef(Reg: Root.getReg());
391 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
392 return {{
393 [=](MachineInstrBuilder &MIB) { MIB.add(MO: RootDef->getOperand(i: 1)); },
394 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); },
395 }};
396 }
397
398 if (isBaseWithConstantOffset(Root, MRI)) {
399 MachineOperand &LHS = RootDef->getOperand(i: 1);
400 MachineOperand &RHS = RootDef->getOperand(i: 2);
401 MachineInstr *LHSDef = MRI.getVRegDef(Reg: LHS.getReg());
402 MachineInstr *RHSDef = MRI.getVRegDef(Reg: RHS.getReg());
403
404 int64_t RHSC = RHSDef->getOperand(i: 1).getCImm()->getSExtValue();
405 if (isInt<12>(x: RHSC)) {
406 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
407 return {{
408 [=](MachineInstrBuilder &MIB) { MIB.add(MO: LHSDef->getOperand(i: 1)); },
409 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); },
410 }};
411
412 return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: LHS); },
413 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}};
414 }
415 }
416
417 // TODO: Need to get the immediate from a G_PTR_ADD. Should this be done in
418 // the combiner?
419 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Root.getReg()); },
420 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}};
421}
422
423/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
424/// CC Must be an ICMP Predicate.
425static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
426 switch (CC) {
427 default:
428 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
429 case CmpInst::Predicate::ICMP_EQ:
430 return RISCVCC::COND_EQ;
431 case CmpInst::Predicate::ICMP_NE:
432 return RISCVCC::COND_NE;
433 case CmpInst::Predicate::ICMP_ULT:
434 return RISCVCC::COND_LTU;
435 case CmpInst::Predicate::ICMP_SLT:
436 return RISCVCC::COND_LT;
437 case CmpInst::Predicate::ICMP_UGE:
438 return RISCVCC::COND_GEU;
439 case CmpInst::Predicate::ICMP_SGE:
440 return RISCVCC::COND_GE;
441 }
442}
443
444static void getOperandsForBranch(Register CondReg, MachineRegisterInfo &MRI,
445 RISCVCC::CondCode &CC, Register &LHS,
446 Register &RHS) {
447 // Try to fold an ICmp. If that fails, use a NE compare with X0.
448 CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
449 if (!mi_match(R: CondReg, MRI, P: m_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: LHS), R: m_Reg(R&: RHS)))) {
450 LHS = CondReg;
451 RHS = RISCV::X0;
452 CC = RISCVCC::COND_NE;
453 return;
454 }
455
456 // We found an ICmp, do some canonicalizations.
457
458 // Adjust comparisons to use comparison with 0 if possible.
459 if (auto Constant = getIConstantVRegSExtVal(VReg: RHS, MRI)) {
460 switch (Pred) {
461 case CmpInst::Predicate::ICMP_SGT:
462 // Convert X > -1 to X >= 0
463 if (*Constant == -1) {
464 CC = RISCVCC::COND_GE;
465 RHS = RISCV::X0;
466 return;
467 }
468 break;
469 case CmpInst::Predicate::ICMP_SLT:
470 // Convert X < 1 to 0 >= X
471 if (*Constant == 1) {
472 CC = RISCVCC::COND_GE;
473 RHS = LHS;
474 LHS = RISCV::X0;
475 return;
476 }
477 break;
478 default:
479 break;
480 }
481 }
482
483 switch (Pred) {
484 default:
485 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
486 case CmpInst::Predicate::ICMP_EQ:
487 case CmpInst::Predicate::ICMP_NE:
488 case CmpInst::Predicate::ICMP_ULT:
489 case CmpInst::Predicate::ICMP_SLT:
490 case CmpInst::Predicate::ICMP_UGE:
491 case CmpInst::Predicate::ICMP_SGE:
492 // These CCs are supported directly by RISC-V branches.
493 break;
494 case CmpInst::Predicate::ICMP_SGT:
495 case CmpInst::Predicate::ICMP_SLE:
496 case CmpInst::Predicate::ICMP_UGT:
497 case CmpInst::Predicate::ICMP_ULE:
498 // These CCs are not supported directly by RISC-V branches, but changing the
499 // direction of the CC and swapping LHS and RHS are.
500 Pred = CmpInst::getSwappedPredicate(pred: Pred);
501 std::swap(a&: LHS, b&: RHS);
502 break;
503 }
504
505 CC = getRISCVCCFromICmp(CC: Pred);
506 return;
507}
508
509bool RISCVInstructionSelector::select(MachineInstr &MI) {
510 MachineBasicBlock &MBB = *MI.getParent();
511 MachineFunction &MF = *MBB.getParent();
512 MachineRegisterInfo &MRI = MF.getRegInfo();
513 MachineIRBuilder MIB(MI);
514
515 preISelLower(MI, MIB, MRI);
516 const unsigned Opc = MI.getOpcode();
517
518 if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
519 if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
520 const Register DefReg = MI.getOperand(i: 0).getReg();
521 const LLT DefTy = MRI.getType(Reg: DefReg);
522
523 const RegClassOrRegBank &RegClassOrBank =
524 MRI.getRegClassOrRegBank(Reg: DefReg);
525
526 const TargetRegisterClass *DefRC =
527 RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
528 if (!DefRC) {
529 if (!DefTy.isValid()) {
530 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
531 return false;
532 }
533
534 const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
535 DefRC = getRegClassForTypeOnBank(Ty: DefTy, RB);
536 if (!DefRC) {
537 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
538 return false;
539 }
540 }
541
542 MI.setDesc(TII.get(Opcode: TargetOpcode::PHI));
543 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI);
544 }
545
546 // Certain non-generic instructions also need some special handling.
547 if (MI.isCopy())
548 return selectCopy(MI, MRI);
549
550 return true;
551 }
552
553 if (selectImpl(I&: MI, CoverageInfo&: *CoverageInfo))
554 return true;
555
556 switch (Opc) {
557 case TargetOpcode::G_ANYEXT:
558 case TargetOpcode::G_PTRTOINT:
559 case TargetOpcode::G_INTTOPTR:
560 case TargetOpcode::G_TRUNC:
561 case TargetOpcode::G_FREEZE:
562 return selectCopy(MI, MRI);
563 case TargetOpcode::G_CONSTANT: {
564 Register DstReg = MI.getOperand(i: 0).getReg();
565 int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue();
566
567 if (!materializeImm(Reg: DstReg, Imm, MIB))
568 return false;
569
570 MI.eraseFromParent();
571 return true;
572 }
573 case TargetOpcode::G_FCONSTANT: {
574 // TODO: Use constant pool for complext constants.
575 // TODO: Optimize +0.0 to use fcvt.d.w for s64 on rv32.
576 Register DstReg = MI.getOperand(i: 0).getReg();
577 const APFloat &FPimm = MI.getOperand(i: 1).getFPImm()->getValueAPF();
578 APInt Imm = FPimm.bitcastToAPInt();
579 unsigned Size = MRI.getType(Reg: DstReg).getSizeInBits();
580 if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
581 Register GPRReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
582 if (!materializeImm(Reg: GPRReg, Imm: Imm.getSExtValue(), MIB))
583 return false;
584
585 unsigned Opcode = Size == 64 ? RISCV::FMV_D_X
586 : Size == 32 ? RISCV::FMV_W_X
587 : RISCV::FMV_H_X;
588 auto FMV = MIB.buildInstr(Opc: Opcode, DstOps: {DstReg}, SrcOps: {GPRReg});
589 if (!FMV.constrainAllUses(TII, TRI, RBI))
590 return false;
591 } else {
592 assert(Size == 64 && !Subtarget->is64Bit() &&
593 "Unexpected size or subtarget");
594 // Split into two pieces and build through the stack.
595 Register GPRRegHigh = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
596 Register GPRRegLow = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
597 if (!materializeImm(Reg: GPRRegHigh, Imm: Imm.extractBits(numBits: 32, bitPosition: 32).getSExtValue(),
598 MIB))
599 return false;
600 if (!materializeImm(Reg: GPRRegLow, Imm: Imm.trunc(width: 32).getSExtValue(), MIB))
601 return false;
602 MachineInstrBuilder PairF64 = MIB.buildInstr(
603 Opc: RISCV::BuildPairF64Pseudo, DstOps: {DstReg}, SrcOps: {GPRRegLow, GPRRegHigh});
604 if (!PairF64.constrainAllUses(TII, TRI, RBI))
605 return false;
606 }
607
608 MI.eraseFromParent();
609 return true;
610 }
611 case TargetOpcode::G_GLOBAL_VALUE: {
612 auto *GV = MI.getOperand(i: 1).getGlobal();
613 if (GV->isThreadLocal()) {
614 // TODO: implement this case.
615 return false;
616 }
617
618 return selectAddr(MI, MIB, MRI, IsLocal: GV->isDSOLocal(),
619 IsExternWeak: GV->hasExternalWeakLinkage());
620 }
621 case TargetOpcode::G_JUMP_TABLE:
622 case TargetOpcode::G_CONSTANT_POOL:
623 return selectAddr(MI, MIB, MRI);
624 case TargetOpcode::G_BRCOND: {
625 Register LHS, RHS;
626 RISCVCC::CondCode CC;
627 getOperandsForBranch(CondReg: MI.getOperand(i: 0).getReg(), MRI, CC, LHS, RHS);
628
629 auto Bcc = MIB.buildInstr(Opc: RISCVCC::getBrCond(CC), DstOps: {}, SrcOps: {LHS, RHS})
630 .addMBB(MBB: MI.getOperand(i: 1).getMBB());
631 MI.eraseFromParent();
632 return constrainSelectedInstRegOperands(I&: *Bcc, TII, TRI, RBI);
633 }
634 case TargetOpcode::G_BRJT: {
635 // FIXME: Move to legalization?
636 const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
637 unsigned EntrySize = MJTI->getEntrySize(TD: MF.getDataLayout());
638 assert((EntrySize == 4 || (Subtarget->is64Bit() && EntrySize == 8)) &&
639 "Unsupported jump-table entry size");
640 assert(
641 (MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32 ||
642 MJTI->getEntryKind() == MachineJumpTableInfo::EK_Custom32 ||
643 MJTI->getEntryKind() == MachineJumpTableInfo::EK_BlockAddress) &&
644 "Unexpected jump-table entry kind");
645
646 auto SLL =
647 MIB.buildInstr(Opc: RISCV::SLLI, DstOps: {&RISCV::GPRRegClass}, SrcOps: {MI.getOperand(i: 2)})
648 .addImm(Val: Log2_32(Value: EntrySize));
649 if (!SLL.constrainAllUses(TII, TRI, RBI))
650 return false;
651
652 // TODO: Use SHXADD. Moving to legalization would fix this automatically.
653 auto ADD = MIB.buildInstr(Opc: RISCV::ADD, DstOps: {&RISCV::GPRRegClass},
654 SrcOps: {MI.getOperand(i: 0), SLL.getReg(Idx: 0)});
655 if (!ADD.constrainAllUses(TII, TRI, RBI))
656 return false;
657
658 unsigned LdOpc = EntrySize == 8 ? RISCV::LD : RISCV::LW;
659 auto Dest =
660 MIB.buildInstr(Opc: LdOpc, DstOps: {&RISCV::GPRRegClass}, SrcOps: {ADD.getReg(Idx: 0)})
661 .addImm(Val: 0)
662 .addMemOperand(MMO: MF.getMachineMemOperand(
663 PtrInfo: MachinePointerInfo::getJumpTable(MF), F: MachineMemOperand::MOLoad,
664 Size: EntrySize, BaseAlignment: Align(MJTI->getEntryAlignment(TD: MF.getDataLayout()))));
665 if (!Dest.constrainAllUses(TII, TRI, RBI))
666 return false;
667
668 // If the Kind is EK_LabelDifference32, the table stores an offset from
669 // the location of the table. Add the table address to get an absolute
670 // address.
671 if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32) {
672 Dest = MIB.buildInstr(Opc: RISCV::ADD, DstOps: {&RISCV::GPRRegClass},
673 SrcOps: {Dest.getReg(Idx: 0), MI.getOperand(i: 0)});
674 if (!Dest.constrainAllUses(TII, TRI, RBI))
675 return false;
676 }
677
678 auto Branch =
679 MIB.buildInstr(Opc: RISCV::PseudoBRIND, DstOps: {}, SrcOps: {Dest.getReg(Idx: 0)}).addImm(Val: 0);
680 if (!Branch.constrainAllUses(TII, TRI, RBI))
681 return false;
682
683 MI.eraseFromParent();
684 return true;
685 }
686 case TargetOpcode::G_BRINDIRECT:
687 MI.setDesc(TII.get(Opcode: RISCV::PseudoBRIND));
688 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
689 return constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
690 case TargetOpcode::G_SEXT_INREG:
691 return selectSExtInreg(MI, MIB);
692 case TargetOpcode::G_FRAME_INDEX: {
693 // TODO: We may want to replace this code with the SelectionDAG patterns,
694 // which fail to get imported because it uses FrameAddrRegImm, which is a
695 // ComplexPattern
696 MI.setDesc(TII.get(Opcode: RISCV::ADDI));
697 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
698 return constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
699 }
700 case TargetOpcode::G_SELECT:
701 return selectSelect(MI, MIB, MRI);
702 case TargetOpcode::G_FCMP:
703 return selectFPCompare(MI, MIB, MRI);
704 case TargetOpcode::G_FENCE: {
705 AtomicOrdering FenceOrdering =
706 static_cast<AtomicOrdering>(MI.getOperand(i: 0).getImm());
707 SyncScope::ID FenceSSID =
708 static_cast<SyncScope::ID>(MI.getOperand(i: 1).getImm());
709 emitFence(FenceOrdering, FenceSSID, MIB);
710 MI.eraseFromParent();
711 return true;
712 }
713 case TargetOpcode::G_IMPLICIT_DEF:
714 return selectImplicitDef(MI, MIB, MRI);
715 case TargetOpcode::G_MERGE_VALUES:
716 return selectMergeValues(MI, MIB, MRI);
717 case TargetOpcode::G_UNMERGE_VALUES:
718 return selectUnmergeValues(MI, MIB, MRI);
719 default:
720 return false;
721 }
722}
723
724bool RISCVInstructionSelector::selectMergeValues(
725 MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const {
726 assert(MI.getOpcode() == TargetOpcode::G_MERGE_VALUES);
727
728 // Build a F64 Pair from operands
729 if (MI.getNumOperands() != 3)
730 return false;
731 Register Dst = MI.getOperand(i: 0).getReg();
732 Register Lo = MI.getOperand(i: 1).getReg();
733 Register Hi = MI.getOperand(i: 2).getReg();
734 if (!isRegInFprb(Reg: Dst, MRI) || !isRegInGprb(Reg: Lo, MRI) || !isRegInGprb(Reg: Hi, MRI))
735 return false;
736 MI.setDesc(TII.get(Opcode: RISCV::BuildPairF64Pseudo));
737 return constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
738}
739
740bool RISCVInstructionSelector::selectUnmergeValues(
741 MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const {
742 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
743
744 // Split F64 Src into two s32 parts
745 if (MI.getNumOperands() != 3)
746 return false;
747 Register Src = MI.getOperand(i: 2).getReg();
748 Register Lo = MI.getOperand(i: 0).getReg();
749 Register Hi = MI.getOperand(i: 1).getReg();
750 if (!isRegInFprb(Reg: Src, MRI) || !isRegInGprb(Reg: Lo, MRI) || !isRegInGprb(Reg: Hi, MRI))
751 return false;
752 MI.setDesc(TII.get(Opcode: RISCV::SplitF64Pseudo));
753 return constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
754}
755
756bool RISCVInstructionSelector::replacePtrWithInt(MachineOperand &Op,
757 MachineIRBuilder &MIB,
758 MachineRegisterInfo &MRI) {
759 Register PtrReg = Op.getReg();
760 assert(MRI.getType(PtrReg).isPointer() && "Operand is not a pointer!");
761
762 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
763 auto PtrToInt = MIB.buildPtrToInt(Dst: sXLen, Src: PtrReg);
764 MRI.setRegBank(Reg: PtrToInt.getReg(Idx: 0), RegBank: RBI.getRegBank(ID: RISCV::GPRBRegBankID));
765 Op.setReg(PtrToInt.getReg(Idx: 0));
766 return select(MI&: *PtrToInt);
767}
768
769void RISCVInstructionSelector::preISelLower(MachineInstr &MI,
770 MachineIRBuilder &MIB,
771 MachineRegisterInfo &MRI) {
772 switch (MI.getOpcode()) {
773 case TargetOpcode::G_PTR_ADD: {
774 Register DstReg = MI.getOperand(i: 0).getReg();
775 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
776
777 replacePtrWithInt(Op&: MI.getOperand(i: 1), MIB, MRI);
778 MI.setDesc(TII.get(Opcode: TargetOpcode::G_ADD));
779 MRI.setType(VReg: DstReg, Ty: sXLen);
780 break;
781 }
782 case TargetOpcode::G_PTRMASK: {
783 Register DstReg = MI.getOperand(i: 0).getReg();
784 const LLT sXLen = LLT::scalar(SizeInBits: STI.getXLen());
785 replacePtrWithInt(Op&: MI.getOperand(i: 1), MIB, MRI);
786 MI.setDesc(TII.get(Opcode: TargetOpcode::G_AND));
787 MRI.setType(VReg: DstReg, Ty: sXLen);
788 }
789 }
790}
791
792void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
793 const MachineInstr &MI,
794 int OpIdx) const {
795 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
796 "Expected G_CONSTANT");
797 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
798 MIB.addImm(Val: -CstVal);
799}
800
801void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
802 const MachineInstr &MI,
803 int OpIdx) const {
804 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
805 "Expected G_CONSTANT");
806 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
807 MIB.addImm(Val: STI.getXLen() - CstVal);
808}
809
810void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
811 const MachineInstr &MI,
812 int OpIdx) const {
813 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
814 "Expected G_CONSTANT");
815 uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
816 MIB.addImm(Val: 32 - CstVal);
817}
818
819void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
820 const MachineInstr &MI,
821 int OpIdx) const {
822 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
823 "Expected G_CONSTANT");
824 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
825 MIB.addImm(Val: CstVal + 1);
826}
827
828void RISCVInstructionSelector::renderImm(MachineInstrBuilder &MIB,
829 const MachineInstr &MI,
830 int OpIdx) const {
831 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
832 "Expected G_CONSTANT");
833 int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue();
834 MIB.addImm(Val: CstVal);
835}
836
837void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
838 const MachineInstr &MI,
839 int OpIdx) const {
840 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
841 "Expected G_CONSTANT");
842 uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue();
843 MIB.addImm(Val: llvm::countr_zero(Val: C));
844}
845
846const TargetRegisterClass *RISCVInstructionSelector::getRegClassForTypeOnBank(
847 LLT Ty, const RegisterBank &RB) const {
848 if (RB.getID() == RISCV::GPRBRegBankID) {
849 if (Ty.getSizeInBits() <= 32 || (STI.is64Bit() && Ty.getSizeInBits() == 64))
850 return &RISCV::GPRRegClass;
851 }
852
853 if (RB.getID() == RISCV::FPRBRegBankID) {
854 if (Ty.getSizeInBits() == 16)
855 return &RISCV::FPR16RegClass;
856 if (Ty.getSizeInBits() == 32)
857 return &RISCV::FPR32RegClass;
858 if (Ty.getSizeInBits() == 64)
859 return &RISCV::FPR64RegClass;
860 }
861
862 if (RB.getID() == RISCV::VRBRegBankID) {
863 if (Ty.getSizeInBits().getKnownMinValue() <= 64)
864 return &RISCV::VRRegClass;
865
866 if (Ty.getSizeInBits().getKnownMinValue() == 128)
867 return &RISCV::VRM2RegClass;
868
869 if (Ty.getSizeInBits().getKnownMinValue() == 256)
870 return &RISCV::VRM4RegClass;
871
872 if (Ty.getSizeInBits().getKnownMinValue() == 512)
873 return &RISCV::VRM8RegClass;
874 }
875
876 return nullptr;
877}
878
879bool RISCVInstructionSelector::isRegInGprb(Register Reg,
880 MachineRegisterInfo &MRI) const {
881 return RBI.getRegBank(Reg, MRI, TRI)->getID() == RISCV::GPRBRegBankID;
882}
883
884bool RISCVInstructionSelector::isRegInFprb(Register Reg,
885 MachineRegisterInfo &MRI) const {
886 return RBI.getRegBank(Reg, MRI, TRI)->getID() == RISCV::FPRBRegBankID;
887}
888
889bool RISCVInstructionSelector::selectCopy(MachineInstr &MI,
890 MachineRegisterInfo &MRI) const {
891 Register DstReg = MI.getOperand(i: 0).getReg();
892
893 if (DstReg.isPhysical())
894 return true;
895
896 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
897 Ty: MRI.getType(Reg: DstReg), RB: *RBI.getRegBank(Reg: DstReg, MRI, TRI));
898 assert(DstRC &&
899 "Register class not available for LLT, register bank combination");
900
901 // No need to constrain SrcReg. It will get constrained when
902 // we hit another of its uses or its defs.
903 // Copies do not have constraints.
904 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
905 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
906 << " operand\n");
907 return false;
908 }
909
910 MI.setDesc(TII.get(Opcode: RISCV::COPY));
911 return true;
912}
913
914bool RISCVInstructionSelector::selectImplicitDef(
915 MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const {
916 assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
917
918 const Register DstReg = MI.getOperand(i: 0).getReg();
919 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
920 Ty: MRI.getType(Reg: DstReg), RB: *RBI.getRegBank(Reg: DstReg, MRI, TRI));
921
922 assert(DstRC &&
923 "Register class not available for LLT, register bank combination");
924
925 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
926 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
927 << " operand\n");
928 }
929 MI.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
930 return true;
931}
932
933bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
934 MachineIRBuilder &MIB) const {
935 MachineRegisterInfo &MRI = *MIB.getMRI();
936
937 if (Imm == 0) {
938 MIB.buildCopy(Res: DstReg, Op: Register(RISCV::X0));
939 RBI.constrainGenericRegister(Reg: DstReg, RC: RISCV::GPRRegClass, MRI);
940 return true;
941 }
942
943 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: *Subtarget);
944 unsigned NumInsts = Seq.size();
945 Register SrcReg = RISCV::X0;
946
947 for (unsigned i = 0; i < NumInsts; i++) {
948 Register TmpReg = i < NumInsts - 1
949 ? MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass)
950 : DstReg;
951 const RISCVMatInt::Inst &I = Seq[i];
952 MachineInstr *Result;
953
954 switch (I.getOpndKind()) {
955 case RISCVMatInt::Imm:
956 // clang-format off
957 Result = MIB.buildInstr(Opc: I.getOpcode(), DstOps: {TmpReg}, SrcOps: {})
958 .addImm(Val: I.getImm());
959 // clang-format on
960 break;
961 case RISCVMatInt::RegX0:
962 Result = MIB.buildInstr(Opc: I.getOpcode(), DstOps: {TmpReg},
963 SrcOps: {SrcReg, Register(RISCV::X0)});
964 break;
965 case RISCVMatInt::RegReg:
966 Result = MIB.buildInstr(Opc: I.getOpcode(), DstOps: {TmpReg}, SrcOps: {SrcReg, SrcReg});
967 break;
968 case RISCVMatInt::RegImm:
969 Result =
970 MIB.buildInstr(Opc: I.getOpcode(), DstOps: {TmpReg}, SrcOps: {SrcReg}).addImm(Val: I.getImm());
971 break;
972 }
973
974 if (!constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI))
975 return false;
976
977 SrcReg = TmpReg;
978 }
979
980 return true;
981}
982
983bool RISCVInstructionSelector::selectAddr(MachineInstr &MI,
984 MachineIRBuilder &MIB,
985 MachineRegisterInfo &MRI,
986 bool IsLocal,
987 bool IsExternWeak) const {
988 assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
989 MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
990 MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
991 "Unexpected opcode");
992
993 const MachineOperand &DispMO = MI.getOperand(i: 1);
994
995 Register DefReg = MI.getOperand(i: 0).getReg();
996 const LLT DefTy = MRI.getType(Reg: DefReg);
997
998 // When HWASAN is used and tagging of global variables is enabled
999 // they should be accessed via the GOT, since the tagged address of a global
1000 // is incompatible with existing code models. This also applies to non-pic
1001 // mode.
1002 if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
1003 if (IsLocal && !Subtarget->allowTaggedGlobals()) {
1004 // Use PC-relative addressing to access the symbol. This generates the
1005 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
1006 // %pcrel_lo(auipc)).
1007 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
1008 return constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1009 }
1010
1011 // Use PC-relative addressing to access the GOT for this symbol, then
1012 // load the address from the GOT. This generates the pattern (PseudoLGA
1013 // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
1014 // %pcrel_lo(auipc))).
1015 MachineFunction &MF = *MI.getParent()->getParent();
1016 MachineMemOperand *MemOp = MF.getMachineMemOperand(
1017 PtrInfo: MachinePointerInfo::getGOT(MF),
1018 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1019 MachineMemOperand::MOInvariant,
1020 MemTy: DefTy, base_alignment: Align(DefTy.getSizeInBits() / 8));
1021
1022 auto Result = MIB.buildInstr(Opc: RISCV::PseudoLGA, DstOps: {DefReg}, SrcOps: {})
1023 .addDisp(Disp: DispMO, off: 0)
1024 .addMemOperand(MMO: MemOp);
1025
1026 if (!constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI))
1027 return false;
1028
1029 MI.eraseFromParent();
1030 return true;
1031 }
1032
1033 switch (TM.getCodeModel()) {
1034 default: {
1035 reportGISelFailure(MF&: const_cast<MachineFunction &>(*MF), TPC: *TPC, MORE&: *MORE,
1036 PassName: getName(), Msg: "Unsupported code model for lowering", MI);
1037 return false;
1038 }
1039 case CodeModel::Small: {
1040 // Must lie within a single 2 GiB address range and must lie between
1041 // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
1042 // (lui %hi(sym)) %lo(sym)).
1043 Register AddrHiDest = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1044 MachineInstr *AddrHi = MIB.buildInstr(Opc: RISCV::LUI, DstOps: {AddrHiDest}, SrcOps: {})
1045 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_HI);
1046
1047 if (!constrainSelectedInstRegOperands(I&: *AddrHi, TII, TRI, RBI))
1048 return false;
1049
1050 auto Result = MIB.buildInstr(Opc: RISCV::ADDI, DstOps: {DefReg}, SrcOps: {AddrHiDest})
1051 .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_LO);
1052
1053 if (!constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI))
1054 return false;
1055
1056 MI.eraseFromParent();
1057 return true;
1058 }
1059 case CodeModel::Medium:
1060 // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
1061 // relocation needs to reference a label that points to the auipc
1062 // instruction itself, not the global. This cannot be done inside the
1063 // instruction selector.
1064 if (IsExternWeak) {
1065 // An extern weak symbol may be undefined, i.e. have value 0, which may
1066 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
1067 // symbol. This generates the pattern (PseudoLGA sym), which expands to
1068 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
1069 MachineFunction &MF = *MI.getParent()->getParent();
1070 MachineMemOperand *MemOp = MF.getMachineMemOperand(
1071 PtrInfo: MachinePointerInfo::getGOT(MF),
1072 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1073 MachineMemOperand::MOInvariant,
1074 MemTy: DefTy, base_alignment: Align(DefTy.getSizeInBits() / 8));
1075
1076 auto Result = MIB.buildInstr(Opc: RISCV::PseudoLGA, DstOps: {DefReg}, SrcOps: {})
1077 .addDisp(Disp: DispMO, off: 0)
1078 .addMemOperand(MMO: MemOp);
1079
1080 if (!constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI))
1081 return false;
1082
1083 MI.eraseFromParent();
1084 return true;
1085 }
1086
1087 // Generate a sequence for accessing addresses within any 2GiB range
1088 // within the address space. This generates the pattern (PseudoLLA sym),
1089 // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
1090 MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA));
1091 return constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
1092 }
1093
1094 return false;
1095}
1096
1097bool RISCVInstructionSelector::selectSExtInreg(MachineInstr &MI,
1098 MachineIRBuilder &MIB) const {
1099 if (!STI.isRV64())
1100 return false;
1101
1102 const MachineOperand &Size = MI.getOperand(i: 2);
1103 // Only Size == 32 (i.e. shift by 32 bits) is acceptable at this point.
1104 if (!Size.isImm() || Size.getImm() != 32)
1105 return false;
1106
1107 const MachineOperand &Src = MI.getOperand(i: 1);
1108 const MachineOperand &Dst = MI.getOperand(i: 0);
1109 // addiw rd, rs, 0 (i.e. sext.w rd, rs)
1110 MachineInstr *NewMI =
1111 MIB.buildInstr(Opc: RISCV::ADDIW, DstOps: {Dst.getReg()}, SrcOps: {Src.getReg()}).addImm(Val: 0U);
1112
1113 if (!constrainSelectedInstRegOperands(I&: *NewMI, TII, TRI, RBI))
1114 return false;
1115
1116 MI.eraseFromParent();
1117 return true;
1118}
1119
1120bool RISCVInstructionSelector::selectSelect(MachineInstr &MI,
1121 MachineIRBuilder &MIB,
1122 MachineRegisterInfo &MRI) const {
1123 auto &SelectMI = cast<GSelect>(Val&: MI);
1124
1125 Register LHS, RHS;
1126 RISCVCC::CondCode CC;
1127 getOperandsForBranch(CondReg: SelectMI.getCondReg(), MRI, CC, LHS, RHS);
1128
1129 Register DstReg = SelectMI.getReg(Idx: 0);
1130
1131 unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
1132 if (RBI.getRegBank(Reg: DstReg, MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
1133 unsigned Size = MRI.getType(Reg: DstReg).getSizeInBits();
1134 Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
1135 : RISCV::Select_FPR64_Using_CC_GPR;
1136 }
1137
1138 MachineInstr *Result = MIB.buildInstr(Opcode: Opc)
1139 .addDef(RegNo: DstReg)
1140 .addReg(RegNo: LHS)
1141 .addReg(RegNo: RHS)
1142 .addImm(Val: CC)
1143 .addReg(RegNo: SelectMI.getTrueReg())
1144 .addReg(RegNo: SelectMI.getFalseReg());
1145 MI.eraseFromParent();
1146 return constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI);
1147}
1148
1149// Convert an FCMP predicate to one of the supported F or D instructions.
1150static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
1151 assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");
1152 switch (Pred) {
1153 default:
1154 llvm_unreachable("Unsupported predicate");
1155 case CmpInst::FCMP_OLT:
1156 return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
1157 case CmpInst::FCMP_OLE:
1158 return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
1159 case CmpInst::FCMP_OEQ:
1160 return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
1161 }
1162}
1163
1164// Try legalizing an FCMP by swapping or inverting the predicate to one that
1165// is supported.
1166static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,
1167 CmpInst::Predicate &Pred, bool &NeedInvert) {
1168 auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
1169 return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
1170 Pred == CmpInst::FCMP_OEQ;
1171 };
1172
1173 assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
1174
1175 CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(pred: Pred);
1176 if (isLegalFCmpPredicate(InvPred)) {
1177 Pred = InvPred;
1178 std::swap(a&: LHS, b&: RHS);
1179 return true;
1180 }
1181
1182 InvPred = CmpInst::getInversePredicate(pred: Pred);
1183 NeedInvert = true;
1184 if (isLegalFCmpPredicate(InvPred)) {
1185 Pred = InvPred;
1186 return true;
1187 }
1188 InvPred = CmpInst::getSwappedPredicate(pred: InvPred);
1189 if (isLegalFCmpPredicate(InvPred)) {
1190 Pred = InvPred;
1191 std::swap(a&: LHS, b&: RHS);
1192 return true;
1193 }
1194
1195 return false;
1196}
1197
1198// Emit a sequence of instructions to compare LHS and RHS using Pred. Return
1199// the result in DstReg.
1200// FIXME: Maybe we should expand this earlier.
1201bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
1202 MachineIRBuilder &MIB,
1203 MachineRegisterInfo &MRI) const {
1204 auto &CmpMI = cast<GFCmp>(Val&: MI);
1205 CmpInst::Predicate Pred = CmpMI.getCond();
1206
1207 Register DstReg = CmpMI.getReg(Idx: 0);
1208 Register LHS = CmpMI.getLHSReg();
1209 Register RHS = CmpMI.getRHSReg();
1210
1211 unsigned Size = MRI.getType(Reg: LHS).getSizeInBits();
1212 assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
1213
1214 Register TmpReg = DstReg;
1215
1216 bool NeedInvert = false;
1217 // First try swapping operands or inverting.
1218 if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
1219 if (NeedInvert)
1220 TmpReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1221 auto Cmp = MIB.buildInstr(Opc: getFCmpOpcode(Pred, Size), DstOps: {TmpReg}, SrcOps: {LHS, RHS});
1222 if (!Cmp.constrainAllUses(TII, TRI, RBI))
1223 return false;
1224 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
1225 // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
1226 NeedInvert = Pred == CmpInst::FCMP_UEQ;
1227 auto Cmp1 = MIB.buildInstr(Opc: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size),
1228 DstOps: {&RISCV::GPRRegClass}, SrcOps: {LHS, RHS});
1229 if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1230 return false;
1231 auto Cmp2 = MIB.buildInstr(Opc: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size),
1232 DstOps: {&RISCV::GPRRegClass}, SrcOps: {RHS, LHS});
1233 if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1234 return false;
1235 if (NeedInvert)
1236 TmpReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1237 auto Or =
1238 MIB.buildInstr(Opc: RISCV::OR, DstOps: {TmpReg}, SrcOps: {Cmp1.getReg(Idx: 0), Cmp2.getReg(Idx: 0)});
1239 if (!Or.constrainAllUses(TII, TRI, RBI))
1240 return false;
1241 } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
1242 // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
1243 // FIXME: If LHS and RHS are the same we can use a single FEQ.
1244 NeedInvert = Pred == CmpInst::FCMP_UNO;
1245 auto Cmp1 = MIB.buildInstr(Opc: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size),
1246 DstOps: {&RISCV::GPRRegClass}, SrcOps: {LHS, LHS});
1247 if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1248 return false;
1249 auto Cmp2 = MIB.buildInstr(Opc: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size),
1250 DstOps: {&RISCV::GPRRegClass}, SrcOps: {RHS, RHS});
1251 if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1252 return false;
1253 if (NeedInvert)
1254 TmpReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1255 auto And =
1256 MIB.buildInstr(Opc: RISCV::AND, DstOps: {TmpReg}, SrcOps: {Cmp1.getReg(Idx: 0), Cmp2.getReg(Idx: 0)});
1257 if (!And.constrainAllUses(TII, TRI, RBI))
1258 return false;
1259 } else
1260 llvm_unreachable("Unhandled predicate");
1261
1262 // Emit an XORI to invert the result if needed.
1263 if (NeedInvert) {
1264 auto Xor = MIB.buildInstr(Opc: RISCV::XORI, DstOps: {DstReg}, SrcOps: {TmpReg}).addImm(Val: 1);
1265 if (!Xor.constrainAllUses(TII, TRI, RBI))
1266 return false;
1267 }
1268
1269 MI.eraseFromParent();
1270 return true;
1271}
1272
1273void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
1274 SyncScope::ID FenceSSID,
1275 MachineIRBuilder &MIB) const {
1276 if (STI.hasStdExtZtso()) {
1277 // The only fence that needs an instruction is a sequentially-consistent
1278 // cross-thread fence.
1279 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
1280 FenceSSID == SyncScope::System) {
1281 // fence rw, rw
1282 MIB.buildInstr(Opc: RISCV::FENCE, DstOps: {}, SrcOps: {})
1283 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W)
1284 .addImm(Val: RISCVFenceField::R | RISCVFenceField::W);
1285 return;
1286 }
1287
1288 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
1289 MIB.buildInstr(Opc: TargetOpcode::MEMBARRIER, DstOps: {}, SrcOps: {});
1290 return;
1291 }
1292
1293 // singlethread fences only synchronize with signal handlers on the same
1294 // thread and thus only need to preserve instruction order, not actually
1295 // enforce memory ordering.
1296 if (FenceSSID == SyncScope::SingleThread) {
1297 MIB.buildInstr(Opc: TargetOpcode::MEMBARRIER, DstOps: {}, SrcOps: {});
1298 return;
1299 }
1300
1301 // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
1302 // Manual: Volume I.
1303 unsigned Pred, Succ;
1304 switch (FenceOrdering) {
1305 default:
1306 llvm_unreachable("Unexpected ordering");
1307 case AtomicOrdering::AcquireRelease:
1308 // fence acq_rel -> fence.tso
1309 MIB.buildInstr(Opc: RISCV::FENCE_TSO, DstOps: {}, SrcOps: {});
1310 return;
1311 case AtomicOrdering::Acquire:
1312 // fence acquire -> fence r, rw
1313 Pred = RISCVFenceField::R;
1314 Succ = RISCVFenceField::R | RISCVFenceField::W;
1315 break;
1316 case AtomicOrdering::Release:
1317 // fence release -> fence rw, w
1318 Pred = RISCVFenceField::R | RISCVFenceField::W;
1319 Succ = RISCVFenceField::W;
1320 break;
1321 case AtomicOrdering::SequentiallyConsistent:
1322 // fence seq_cst -> fence rw, rw
1323 Pred = RISCVFenceField::R | RISCVFenceField::W;
1324 Succ = RISCVFenceField::R | RISCVFenceField::W;
1325 break;
1326 }
1327 MIB.buildInstr(Opc: RISCV::FENCE, DstOps: {}, SrcOps: {}).addImm(Val: Pred).addImm(Val: Succ);
1328}
1329
1330namespace llvm {
1331InstructionSelector *
1332createRISCVInstructionSelector(const RISCVTargetMachine &TM,
1333 const RISCVSubtarget &Subtarget,
1334 const RISCVRegisterBankInfo &RBI) {
1335 return new RISCVInstructionSelector(TM, Subtarget, RBI);
1336}
1337} // end namespace llvm
1338