| 1 | //===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// \file |
| 9 | /// This file implements the targeting of the InstructionSelector class for |
| 10 | /// RISC-V. |
| 11 | /// \todo This should be generated by TableGen. |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "MCTargetDesc/RISCVMatInt.h" |
| 15 | #include "RISCVRegisterBankInfo.h" |
| 16 | #include "RISCVSubtarget.h" |
| 17 | #include "RISCVTargetMachine.h" |
| 18 | #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" |
| 19 | #include "llvm/CodeGen/GlobalISel/GISelValueTracking.h" |
| 20 | #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" |
| 21 | #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" |
| 22 | #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" |
| 23 | #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" |
| 24 | #include "llvm/CodeGen/MachineJumpTableInfo.h" |
| 25 | #include "llvm/IR/IntrinsicsRISCV.h" |
| 26 | #include "llvm/Support/Debug.h" |
| 27 | |
| 28 | #define DEBUG_TYPE "riscv-isel" |
| 29 | |
| 30 | using namespace llvm; |
| 31 | using namespace MIPatternMatch; |
| 32 | |
| 33 | #define GET_GLOBALISEL_PREDICATE_BITSET |
| 34 | #include "RISCVGenGlobalISel.inc" |
| 35 | #undef GET_GLOBALISEL_PREDICATE_BITSET |
| 36 | |
| 37 | namespace { |
| 38 | |
| 39 | class RISCVInstructionSelector : public InstructionSelector { |
| 40 | public: |
| 41 | RISCVInstructionSelector(const RISCVTargetMachine &TM, |
| 42 | const RISCVSubtarget &STI, |
| 43 | const RISCVRegisterBankInfo &RBI); |
| 44 | |
| 45 | bool select(MachineInstr &MI) override; |
| 46 | |
| 47 | void setupMF(MachineFunction &MF, GISelValueTracking *VT, |
| 48 | CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, |
| 49 | BlockFrequencyInfo *BFI) override { |
| 50 | InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI); |
| 51 | MRI = &MF.getRegInfo(); |
| 52 | } |
| 53 | |
| 54 | static const char *getName() { return DEBUG_TYPE; } |
| 55 | |
| 56 | private: |
| 57 | static constexpr unsigned MaxRecursionDepth = 6; |
| 58 | |
| 59 | bool hasAllNBitUsers(const MachineInstr &MI, unsigned Bits, |
| 60 | const unsigned Depth = 0) const; |
| 61 | bool hasAllHUsers(const MachineInstr &MI) const { |
| 62 | return hasAllNBitUsers(MI, Bits: 16); |
| 63 | } |
| 64 | bool hasAllWUsers(const MachineInstr &MI) const { |
| 65 | return hasAllNBitUsers(MI, Bits: 32); |
| 66 | } |
| 67 | |
| 68 | bool isRegInGprb(Register Reg) const; |
| 69 | bool isRegInFprb(Register Reg) const; |
| 70 | bool isWorthFoldingAdd(Register AddResult) const; |
| 71 | |
| 72 | // tblgen-erated 'select' implementation, used as the initial selector for |
| 73 | // the patterns that don't require complex C++. |
| 74 | bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; |
| 75 | |
| 76 | // A lowering phase that runs before any selection attempts. |
| 77 | // Returns true if the instruction was modified. |
| 78 | void preISelLower(MachineInstr &MI); |
| 79 | |
| 80 | bool replacePtrWithInt(MachineInstr &MI, unsigned OpIdx); |
| 81 | |
| 82 | // Custom selection methods |
| 83 | bool selectCopy(MachineInstr &MI) const; |
| 84 | bool selectImplicitDef(MachineInstr &MI) const; |
| 85 | bool materializeImm(Register Reg, int64_t Imm, MachineInstr &MI) const; |
| 86 | // Emit a constant-materialization instruction sequence. |
| 87 | bool materializeInstSeq(Register DstReg, const RISCVMatInt::InstSeq &Seq, |
| 88 | MachineInstr &MI) const; |
| 89 | bool selectAddr(MachineInstr &MI, bool IsLocal = true, |
| 90 | bool IsExternWeak = false) const; |
| 91 | bool selectSelect(MachineInstr &MI) const; |
| 92 | bool selectFPCompare(MachineInstr &MI) const; |
| 93 | void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID, |
| 94 | MachineInstr &MI) const; |
| 95 | bool selectUnmergeValues(MachineInstr &MI) const; |
| 96 | bool selectMergeValues(MachineInstr &MI) const; |
| 97 | void addVectorLoadStoreOperands(MachineInstr &I, |
| 98 | SmallVectorImpl<Register> &SrcOps, |
| 99 | unsigned &CurOp, bool IsMasked, |
| 100 | bool IsStridedOrIndexed, |
| 101 | LLT *IndexVT = nullptr) const; |
| 102 | bool selectIntrinsicWithSideEffects(MachineInstr &I) const; |
| 103 | bool selectIntrinsic(MachineInstr &I) const; |
| 104 | bool selectExtractSubvector(MachineInstr &MI) const; |
| 105 | bool selectInsertSubVector(MachineInstr &I) const; |
| 106 | ComplexRendererFns selectShiftMask(MachineOperand &Root, |
| 107 | unsigned ShiftWidth) const; |
| 108 | ComplexRendererFns selectShiftMaskXLen(MachineOperand &Root) const { |
| 109 | return selectShiftMask(Root, ShiftWidth: STI.getXLen()); |
| 110 | } |
| 111 | ComplexRendererFns selectShiftMask32(MachineOperand &Root) const { |
| 112 | return selectShiftMask(Root, ShiftWidth: 32); |
| 113 | } |
| 114 | ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const; |
| 115 | ComplexRendererFns selectBrindRegImm(MachineOperand &Root) const; |
| 116 | ComplexRendererFns selectAddrRegImmLsb00000(MachineOperand &Root) const; |
| 117 | |
| 118 | // Plan for materializing a constant address as (Hi materialization, Lo12 |
| 119 | // offset). Lo12 is a simm12 that, for prefetch (IsPrefetch), must be |
| 120 | // a multiple of 32. |
| 121 | struct ConstAddrPlan { |
| 122 | enum { X0, LUI, InstSeq } Kind = X0; |
| 123 | int64_t Hi20 = 0; |
| 124 | RISCVMatInt::InstSeq Seq; |
| 125 | int64_t Lo12 = 0; |
| 126 | }; |
| 127 | ComplexRendererFns computeConstAddr(int64_t CVal, bool IsPrefetch, |
| 128 | Register OrigBase) const; |
| 129 | // Materialize the high part of Plan into a register. If OrigBase is valid, |
| 130 | // ADD it to the materialized high part (for G_PTR_ADD + large constant). |
| 131 | Register materializeConstBase(MachineInstrBuilder &MIB, |
| 132 | const ConstAddrPlan &Plan, |
| 133 | Register OrigBase) const; |
| 134 | |
| 135 | ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const; |
| 136 | template <unsigned Bits> |
| 137 | ComplexRendererFns selectSExtBits(MachineOperand &Root) const { |
| 138 | return selectSExtBits(Root, Bits); |
| 139 | } |
| 140 | |
| 141 | ComplexRendererFns selectZExtBits(MachineOperand &Root, unsigned Bits) const; |
| 142 | template <unsigned Bits> |
| 143 | ComplexRendererFns selectZExtBits(MachineOperand &Root) const { |
| 144 | return selectZExtBits(Root, Bits); |
| 145 | } |
| 146 | |
| 147 | ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const; |
| 148 | template <unsigned ShAmt> |
| 149 | ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const { |
| 150 | return selectSHXADDOp(Root, ShAmt); |
| 151 | } |
| 152 | |
| 153 | ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root, |
| 154 | unsigned ShAmt) const; |
| 155 | template <unsigned ShAmt> |
| 156 | ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const { |
| 157 | return selectSHXADD_UWOp(Root, ShAmt); |
| 158 | } |
| 159 | |
| 160 | ComplexRendererFns renderVLOp(MachineOperand &Root) const; |
| 161 | ComplexRendererFns renderAddiPair(Register BaseReg, int64_t AddiImm, |
| 162 | int64_t OffsetImm) const; |
| 163 | // Custom renderers for tablegen |
| 164 | void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI, |
| 165 | int OpIdx) const; |
| 166 | void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI, |
| 167 | int OpIdx) const; |
| 168 | void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI, |
| 169 | int OpIdx) const; |
| 170 | void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI, |
| 171 | int OpIdx) const; |
| 172 | |
| 173 | void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI, |
| 174 | int OpIdx) const; |
| 175 | void renderXLenSubTrailingOnes(MachineInstrBuilder &MIB, |
| 176 | const MachineInstr &MI, int OpIdx) const; |
| 177 | |
| 178 | void renderAddiPairImmLarge(MachineInstrBuilder &MIB, const MachineInstr &MI, |
| 179 | int OpIdx) const; |
| 180 | void renderAddiPairImmSmall(MachineInstrBuilder &MIB, const MachineInstr &MI, |
| 181 | int OpIdx) const; |
| 182 | |
| 183 | const RISCVSubtarget &STI; |
| 184 | const RISCVInstrInfo &TII; |
| 185 | const RISCVRegisterInfo &TRI; |
| 186 | const RISCVRegisterBankInfo &RBI; |
| 187 | const RISCVTargetMachine &TM; |
| 188 | |
| 189 | MachineRegisterInfo *MRI = nullptr; |
| 190 | |
| 191 | // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel |
| 192 | // uses "STI." in the code generated by TableGen. We need to unify the name of |
| 193 | // Subtarget variable. |
| 194 | const RISCVSubtarget *Subtarget = &STI; |
| 195 | |
| 196 | #define GET_GLOBALISEL_PREDICATES_DECL |
| 197 | #include "RISCVGenGlobalISel.inc" |
| 198 | #undef GET_GLOBALISEL_PREDICATES_DECL |
| 199 | |
| 200 | #define GET_GLOBALISEL_TEMPORARIES_DECL |
| 201 | #include "RISCVGenGlobalISel.inc" |
| 202 | #undef GET_GLOBALISEL_TEMPORARIES_DECL |
| 203 | }; |
| 204 | |
| 205 | } // end anonymous namespace |
| 206 | |
| 207 | #define GET_GLOBALISEL_IMPL |
| 208 | #include "RISCVGenGlobalISel.inc" |
| 209 | #undef GET_GLOBALISEL_IMPL |
| 210 | |
| 211 | RISCVInstructionSelector::RISCVInstructionSelector( |
| 212 | const RISCVTargetMachine &TM, const RISCVSubtarget &STI, |
| 213 | const RISCVRegisterBankInfo &RBI) |
| 214 | : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI), |
| 215 | TM(TM), |
| 216 | |
| 217 | #define GET_GLOBALISEL_PREDICATES_INIT |
| 218 | #include "RISCVGenGlobalISel.inc" |
| 219 | #undef GET_GLOBALISEL_PREDICATES_INIT |
| 220 | #define GET_GLOBALISEL_TEMPORARIES_INIT |
| 221 | #include "RISCVGenGlobalISel.inc" |
| 222 | #undef GET_GLOBALISEL_TEMPORARIES_INIT |
| 223 | { |
| 224 | } |
| 225 | |
| 226 | // Mimics optimizations in ISel and RISCVOptWInst Pass |
| 227 | bool RISCVInstructionSelector::hasAllNBitUsers(const MachineInstr &MI, |
| 228 | unsigned Bits, |
| 229 | const unsigned Depth) const { |
| 230 | |
| 231 | assert((MI.getOpcode() == TargetOpcode::G_ADD || |
| 232 | MI.getOpcode() == TargetOpcode::G_SUB || |
| 233 | MI.getOpcode() == TargetOpcode::G_MUL || |
| 234 | MI.getOpcode() == TargetOpcode::G_SHL || |
| 235 | MI.getOpcode() == TargetOpcode::G_LSHR || |
| 236 | MI.getOpcode() == TargetOpcode::G_AND || |
| 237 | MI.getOpcode() == TargetOpcode::G_OR || |
| 238 | MI.getOpcode() == TargetOpcode::G_XOR || |
| 239 | MI.getOpcode() == TargetOpcode::G_SEXT_INREG || Depth != 0) && |
| 240 | "Unexpected opcode" ); |
| 241 | |
| 242 | if (Depth >= RISCVInstructionSelector::MaxRecursionDepth) |
| 243 | return false; |
| 244 | |
| 245 | auto DestReg = MI.getOperand(i: 0).getReg(); |
| 246 | for (auto &UserOp : MRI->use_nodbg_operands(Reg: DestReg)) { |
| 247 | assert(UserOp.getParent() && "UserOp must have a parent" ); |
| 248 | const MachineInstr &UserMI = *UserOp.getParent(); |
| 249 | unsigned OpIdx = UserOp.getOperandNo(); |
| 250 | |
| 251 | switch (UserMI.getOpcode()) { |
| 252 | default: |
| 253 | return false; |
| 254 | case RISCV::ADDW: |
| 255 | case RISCV::ADDIW: |
| 256 | case RISCV::SUBW: |
| 257 | case RISCV::FCVT_D_W: |
| 258 | case RISCV::FCVT_S_W: |
| 259 | if (Bits >= 32) |
| 260 | break; |
| 261 | return false; |
| 262 | case RISCV::SLL: |
| 263 | case RISCV::SRA: |
| 264 | case RISCV::SRL: |
| 265 | // Shift amount operands only use log2(Xlen) bits. |
| 266 | if (OpIdx == 2 && Bits >= Log2_32(Value: Subtarget->getXLen())) |
| 267 | break; |
| 268 | return false; |
| 269 | case RISCV::SLLI: |
| 270 | // SLLI only uses the lower (XLen - ShAmt) bits. |
| 271 | if (Bits >= Subtarget->getXLen() - UserMI.getOperand(i: 2).getImm()) |
| 272 | break; |
| 273 | return false; |
| 274 | case RISCV::ANDI: |
| 275 | if (Bits >= (unsigned)llvm::bit_width<uint64_t>( |
| 276 | Value: (uint64_t)UserMI.getOperand(i: 2).getImm())) |
| 277 | break; |
| 278 | goto RecCheck; |
| 279 | case RISCV::AND: |
| 280 | case RISCV::OR: |
| 281 | case RISCV::XOR: |
| 282 | RecCheck: |
| 283 | if (hasAllNBitUsers(MI: UserMI, Bits, Depth: Depth + 1)) |
| 284 | break; |
| 285 | return false; |
| 286 | case RISCV::SRLI: { |
| 287 | unsigned ShAmt = UserMI.getOperand(i: 2).getImm(); |
| 288 | // If we are shifting right by less than Bits, and users don't demand any |
| 289 | // bits that were shifted into [Bits-1:0], then we can consider this as an |
| 290 | // N-Bit user. |
| 291 | if (Bits > ShAmt && hasAllNBitUsers(MI: UserMI, Bits: Bits - ShAmt, Depth: Depth + 1)) |
| 292 | break; |
| 293 | return false; |
| 294 | } |
| 295 | } |
| 296 | } |
| 297 | |
| 298 | return true; |
| 299 | } |
| 300 | |
| 301 | InstructionSelector::ComplexRendererFns |
| 302 | RISCVInstructionSelector::selectShiftMask(MachineOperand &Root, |
| 303 | unsigned ShiftWidth) const { |
| 304 | if (!Root.isReg()) |
| 305 | return std::nullopt; |
| 306 | |
| 307 | using namespace llvm::MIPatternMatch; |
| 308 | |
| 309 | Register ShAmtReg = Root.getReg(); |
| 310 | // Peek through zext. |
| 311 | Register ZExtSrcReg; |
| 312 | if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrcReg)))) |
| 313 | ShAmtReg = ZExtSrcReg; |
| 314 | |
| 315 | APInt AndMask; |
| 316 | Register AndSrcReg; |
| 317 | // Try to combine the following pattern (applicable to other shift |
| 318 | // instructions as well as 32-bit ones): |
| 319 | // |
| 320 | // %4:gprb(s64) = G_AND %3, %2 |
| 321 | // %5:gprb(s64) = G_LSHR %1, %4(s64) |
| 322 | // |
| 323 | // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than |
| 324 | // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if |
| 325 | // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same, |
| 326 | // then it can be eliminated. Given register rs1 or rs2 holding a constant |
| 327 | // (the and mask), there are two cases G_AND can be erased: |
| 328 | // |
| 329 | // 1. the lowest log2(XLEN) bits of the and mask are all set |
| 330 | // 2. the bits of the register being masked are already unset (zero set) |
| 331 | if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: AndSrcReg), R: m_ICst(Cst&: AndMask)))) { |
| 332 | APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1); |
| 333 | if (ShMask.isSubsetOf(RHS: AndMask)) { |
| 334 | ShAmtReg = AndSrcReg; |
| 335 | } else { |
| 336 | // SimplifyDemandedBits may have optimized the mask so try restoring any |
| 337 | // bits that are known zero. |
| 338 | KnownBits Known = VT->getKnownBits(R: AndSrcReg); |
| 339 | if (ShMask.isSubsetOf(RHS: AndMask | Known.Zero)) |
| 340 | ShAmtReg = AndSrcReg; |
| 341 | } |
| 342 | } |
| 343 | |
| 344 | APInt Imm; |
| 345 | Register Reg; |
| 346 | if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GAdd(L: m_Reg(R&: Reg), R: m_ICst(Cst&: Imm)))) { |
| 347 | if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0) |
| 348 | // If we are shifting by X+N where N == 0 mod Size, then just shift by X |
| 349 | // to avoid the ADD. |
| 350 | ShAmtReg = Reg; |
| 351 | } else if (mi_match(R: ShAmtReg, MRI: *MRI, P: m_GSub(L: m_ICst(Cst&: Imm), R: m_Reg(R&: Reg)))) { |
| 352 | if (Imm != 0 && Imm.urem(RHS: ShiftWidth) == 0) { |
| 353 | // If we are shifting by N-X where N == 0 mod Size, then just shift by -X |
| 354 | // to generate a NEG instead of a SUB of a constant. |
| 355 | ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 356 | unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB; |
| 357 | return {{[=](MachineInstrBuilder &MIB) { |
| 358 | MachineIRBuilder(*MIB.getInstr()) |
| 359 | .buildInstr(Opc: NegOpc, DstOps: {ShAmtReg}, SrcOps: {Register(RISCV::X0), Reg}); |
| 360 | MIB.addReg(RegNo: ShAmtReg); |
| 361 | }}}; |
| 362 | } |
| 363 | if (Imm.urem(RHS: ShiftWidth) == ShiftWidth - 1) { |
| 364 | // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X |
| 365 | // to generate a NOT instead of a SUB of a constant. |
| 366 | ShAmtReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 367 | return {{[=](MachineInstrBuilder &MIB) { |
| 368 | MachineIRBuilder(*MIB.getInstr()) |
| 369 | .buildInstr(Opc: RISCV::XORI, DstOps: {ShAmtReg}, SrcOps: {Reg}) |
| 370 | .addImm(Val: -1); |
| 371 | MIB.addReg(RegNo: ShAmtReg); |
| 372 | }}}; |
| 373 | } |
| 374 | } |
| 375 | |
| 376 | return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: ShAmtReg); }}}; |
| 377 | } |
| 378 | |
| 379 | InstructionSelector::ComplexRendererFns |
| 380 | RISCVInstructionSelector::selectSExtBits(MachineOperand &Root, |
| 381 | unsigned Bits) const { |
| 382 | if (!Root.isReg()) |
| 383 | return std::nullopt; |
| 384 | Register RootReg = Root.getReg(); |
| 385 | |
| 386 | Register SrcReg; |
| 387 | if (mi_match(R: RootReg, MRI: *MRI, |
| 388 | P: m_GSExtInReg(Src: m_Reg(R&: SrcReg), Imm: m_SpecificImm(RequestedValue: Bits)))) { |
| 389 | return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SrcReg); }}}; |
| 390 | } |
| 391 | |
| 392 | unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits(); |
| 393 | if ((Size - VT->computeNumSignBits(R: RootReg)) < Bits) |
| 394 | return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}}; |
| 395 | |
| 396 | return std::nullopt; |
| 397 | } |
| 398 | |
| 399 | InstructionSelector::ComplexRendererFns |
| 400 | RISCVInstructionSelector::selectZExtBits(MachineOperand &Root, |
| 401 | unsigned Bits) const { |
| 402 | if (!Root.isReg()) |
| 403 | return std::nullopt; |
| 404 | Register RootReg = Root.getReg(); |
| 405 | |
| 406 | Register RegX; |
| 407 | uint64_t Mask = maskTrailingOnes<uint64_t>(N: Bits); |
| 408 | if (mi_match(R: RootReg, MRI: *MRI, P: m_GAnd(L: m_Reg(R&: RegX), R: m_SpecificICst(RequestedValue: Mask)))) { |
| 409 | return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}}; |
| 410 | } |
| 411 | |
| 412 | if (mi_match(R: RootReg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: RegX))) && |
| 413 | MRI->getType(Reg: RegX).getScalarSizeInBits() == Bits) |
| 414 | return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RegX); }}}; |
| 415 | |
| 416 | unsigned Size = MRI->getType(Reg: RootReg).getScalarSizeInBits(); |
| 417 | if (VT->maskedValueIsZero(Val: RootReg, Mask: APInt::getBitsSetFrom(numBits: Size, loBit: Bits))) |
| 418 | return {{[=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }}}; |
| 419 | |
| 420 | return std::nullopt; |
| 421 | } |
| 422 | |
| 423 | InstructionSelector::ComplexRendererFns |
| 424 | RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root, |
| 425 | unsigned ShAmt) const { |
| 426 | using namespace llvm::MIPatternMatch; |
| 427 | |
| 428 | if (!Root.isReg()) |
| 429 | return std::nullopt; |
| 430 | Register RootReg = Root.getReg(); |
| 431 | |
| 432 | const unsigned XLen = STI.getXLen(); |
| 433 | APInt Mask; |
| 434 | int64_t C2; |
| 435 | Register RegY; |
| 436 | std::optional<bool> LeftShift; |
| 437 | // (and (shl y, c2), mask) |
| 438 | if (mi_match(R: RootReg, MRI: *MRI, |
| 439 | P: m_GAnd(L: m_GShl(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask)))) |
| 440 | LeftShift = true; |
| 441 | // (and (lshr y, c2), mask) |
| 442 | else if (mi_match(R: RootReg, MRI: *MRI, |
| 443 | P: m_GAnd(L: m_GLShr(L: m_Reg(R&: RegY), R: m_ICst(Cst&: C2)), R: m_ICst(Cst&: Mask)))) |
| 444 | LeftShift = false; |
| 445 | |
| 446 | if (LeftShift.has_value()) { |
| 447 | assert(Mask.getBitWidth() == XLen && "Unexpected mask size!" ); |
| 448 | assert(C2 > 0 && C2 < XLen && "Unexpected shift amount!" ); |
| 449 | if (*LeftShift) |
| 450 | Mask &= maskTrailingZeros<uint64_t>(N: C2); |
| 451 | else |
| 452 | Mask &= maskTrailingOnes<uint64_t>(N: XLen - C2); |
| 453 | |
| 454 | if (Mask.isShiftedMask()) { |
| 455 | unsigned Leading = XLen - Mask.getActiveBits(); |
| 456 | unsigned Trailing = Mask.countr_zero(); |
| 457 | // Given (and (shl y, c2), mask) in which mask has no leading zeros and |
| 458 | // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD. |
| 459 | if (*LeftShift && Leading == 0 && C2 < Trailing && Trailing == ShAmt) { |
| 460 | Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 461 | return {{[=](MachineInstrBuilder &MIB) { |
| 462 | MachineIRBuilder(*MIB.getInstr()) |
| 463 | .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY}) |
| 464 | .addImm(Val: Trailing - C2); |
| 465 | MIB.addReg(RegNo: DstReg); |
| 466 | }}}; |
| 467 | } |
| 468 | |
| 469 | // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and |
| 470 | // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD. |
| 471 | if (!*LeftShift && Leading == C2 && Trailing == ShAmt) { |
| 472 | Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 473 | return {{[=](MachineInstrBuilder &MIB) { |
| 474 | MachineIRBuilder(*MIB.getInstr()) |
| 475 | .buildInstr(Opc: RISCV::SRLI, DstOps: {DstReg}, SrcOps: {RegY}) |
| 476 | .addImm(Val: Leading + Trailing); |
| 477 | MIB.addReg(RegNo: DstReg); |
| 478 | }}}; |
| 479 | } |
| 480 | } |
| 481 | } |
| 482 | |
| 483 | LeftShift.reset(); |
| 484 | |
| 485 | // (shl (and y, mask), c2) |
| 486 | if (mi_match(R: RootReg, MRI: *MRI, |
| 487 | P: m_GShl(L: m_OneUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))), R: m_ICst(Cst&: C2)))) |
| 488 | LeftShift = true; |
| 489 | // (lshr (and y, mask), c2) |
| 490 | else if (mi_match(R: RootReg, MRI: *MRI, |
| 491 | P: m_GLShr(L: m_OneUse(SP: m_GAnd(L: m_Reg(R&: RegY), R: m_ICst(Cst&: Mask))), |
| 492 | R: m_ICst(Cst&: C2)))) |
| 493 | LeftShift = false; |
| 494 | |
| 495 | if (LeftShift.has_value() && Mask.isShiftedMask()) { |
| 496 | unsigned Leading = XLen - Mask.getActiveBits(); |
| 497 | unsigned Trailing = Mask.countr_zero(); |
| 498 | |
| 499 | // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and |
| 500 | // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD. |
| 501 | bool Cond = |
| 502 | *LeftShift && Leading == 32 && Trailing > 0 && (Trailing + C2) == ShAmt; |
| 503 | if (!Cond) |
| 504 | // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and |
| 505 | // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD. |
| 506 | Cond = !*LeftShift && Leading == 32 && C2 < Trailing && |
| 507 | (Trailing - C2) == ShAmt; |
| 508 | |
| 509 | if (Cond) { |
| 510 | Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 511 | return {{[=](MachineInstrBuilder &MIB) { |
| 512 | MachineIRBuilder(*MIB.getInstr()) |
| 513 | .buildInstr(Opc: RISCV::SRLIW, DstOps: {DstReg}, SrcOps: {RegY}) |
| 514 | .addImm(Val: Trailing); |
| 515 | MIB.addReg(RegNo: DstReg); |
| 516 | }}}; |
| 517 | } |
| 518 | } |
| 519 | |
| 520 | return std::nullopt; |
| 521 | } |
| 522 | |
| 523 | InstructionSelector::ComplexRendererFns |
| 524 | RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root, |
| 525 | unsigned ShAmt) const { |
| 526 | using namespace llvm::MIPatternMatch; |
| 527 | |
| 528 | if (!Root.isReg()) |
| 529 | return std::nullopt; |
| 530 | Register RootReg = Root.getReg(); |
| 531 | |
| 532 | // Given (and (shl x, c2), mask) in which mask is a shifted mask with |
| 533 | // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by |
| 534 | // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount. |
| 535 | APInt Mask; |
| 536 | int64_t C2; |
| 537 | Register RegX; |
| 538 | if (mi_match(R: RootReg, MRI: *MRI, |
| 539 | P: m_OneUse(SP: m_GAnd(L: m_OneUse(SP: m_GShl(L: m_Reg(R&: RegX), R: m_ICst(Cst&: C2))), |
| 540 | R: m_ICst(Cst&: Mask))))) { |
| 541 | assert(C2 > 0 && C2 < Mask.getBitWidth() && "Unexpected shift amount!" ); |
| 542 | Mask &= maskTrailingZeros<uint64_t>(N: C2); |
| 543 | |
| 544 | if (Mask.isShiftedMask()) { |
| 545 | unsigned Leading = Mask.countl_zero(); |
| 546 | unsigned Trailing = Mask.countr_zero(); |
| 547 | if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) { |
| 548 | Register DstReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 549 | return {{[=](MachineInstrBuilder &MIB) { |
| 550 | MachineIRBuilder(*MIB.getInstr()) |
| 551 | .buildInstr(Opc: RISCV::SLLI, DstOps: {DstReg}, SrcOps: {RegX}) |
| 552 | .addImm(Val: C2 - ShAmt); |
| 553 | MIB.addReg(RegNo: DstReg); |
| 554 | }}}; |
| 555 | } |
| 556 | } |
| 557 | } |
| 558 | |
| 559 | return std::nullopt; |
| 560 | } |
| 561 | |
| 562 | InstructionSelector::ComplexRendererFns |
| 563 | RISCVInstructionSelector::renderVLOp(MachineOperand &Root) const { |
| 564 | assert(Root.isReg() && "Expected operand to be a Register" ); |
| 565 | std::optional<ValueAndVReg> C; |
| 566 | if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GCst(ValReg&: C))) { |
| 567 | if (C->Value.isAllOnes()) |
| 568 | // If the operand is a G_CONSTANT with value of all ones it is larger than |
| 569 | // VLMAX. We convert it to an immediate with value VLMaxSentinel. This is |
| 570 | // recognized specially by the vsetvli insertion pass. |
| 571 | return {{[=](MachineInstrBuilder &MIB) { |
| 572 | MIB.addImm(Val: RISCV::VLMaxSentinel); |
| 573 | }}}; |
| 574 | |
| 575 | if (isUInt<5>(x: C->Value.getZExtValue())) { |
| 576 | uint64_t ZExtC = C->Value.getZExtValue(); |
| 577 | return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ZExtC); }}}; |
| 578 | } |
| 579 | } |
| 580 | return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Root.getReg()); }}}; |
| 581 | } |
| 582 | |
| 583 | InstructionSelector::ComplexRendererFns |
| 584 | RISCVInstructionSelector::renderAddiPair(Register BaseReg, int64_t AddiImm, |
| 585 | int64_t OffsetImm) const { |
| 586 | return {{[=](MachineInstrBuilder &MIB) { |
| 587 | Register Tmp = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 588 | MachineInstr *Addi = |
| 589 | BuildMI(BB&: *MIB->getParent(), I&: *MIB.getInstr(), MIMD: MIB->getDebugLoc(), |
| 590 | MCID: TII.get(Opcode: RISCV::ADDI), DestReg: Tmp) |
| 591 | .addReg(RegNo: BaseReg) |
| 592 | .addImm(Val: AddiImm); |
| 593 | constrainSelectedInstRegOperands(I&: *Addi, TII, TRI, RBI); |
| 594 | MIB.addReg(RegNo: Tmp); |
| 595 | }, |
| 596 | [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: OffsetImm); }}}; |
| 597 | } |
| 598 | |
| 599 | InstructionSelector::ComplexRendererFns |
| 600 | RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const { |
| 601 | if (!Root.isReg()) |
| 602 | return std::nullopt; |
| 603 | |
| 604 | Register RootReg = Root.getReg(); |
| 605 | |
| 606 | // Frame index. |
| 607 | int FI; |
| 608 | if (mi_match(R: RootReg, MRI: *MRI, P: m_GFrameIndex(FI))) { |
| 609 | return {{ |
| 610 | [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); }, |
| 611 | [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, |
| 612 | }}; |
| 613 | } |
| 614 | |
| 615 | // base + constant offset (G_PTR_ADD). |
| 616 | Register BaseReg; |
| 617 | int64_t RHSC; |
| 618 | if (mi_match(R: RootReg, MRI: *MRI, P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_ICst(Cst&: RHSC)))) { |
| 619 | if (isInt<12>(x: RHSC)) { |
| 620 | int BaseFI; |
| 621 | if (mi_match(R: BaseReg, MRI: *MRI, P: m_GFrameIndex(FI&: BaseFI))) |
| 622 | return {{ |
| 623 | [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: BaseFI); }, |
| 624 | [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }, |
| 625 | }}; |
| 626 | |
| 627 | return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: BaseReg); }, |
| 628 | [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}}; |
| 629 | } |
| 630 | |
| 631 | // Large constant offset. Fold a -2048/2047 adjustment so the whole |
| 632 | // constant can be split across an ADDI and the load/store offset. |
| 633 | if (RHSC >= -4096 && RHSC <= 4094) { |
| 634 | int64_t Adj = RHSC < 0 ? -2048 : 2047; |
| 635 | return renderAddiPair(BaseReg, AddiImm: Adj, OffsetImm: RHSC - Adj); |
| 636 | } |
| 637 | |
| 638 | if (isWorthFoldingAdd(AddResult: RootReg)) |
| 639 | if (auto Fns = computeConstAddr(CVal: RHSC, /*IsPrefetch=*/false, OrigBase: BaseReg)) |
| 640 | return Fns; |
| 641 | } |
| 642 | |
| 643 | // Bare constant address. IRTranslator lowers inttoptr(C) to |
| 644 | // G_INTTOPTR(G_CONSTANT); look through it to reach the constant. |
| 645 | int64_t CVal; |
| 646 | if (mi_match(R: RootReg, MRI: *MRI, P: m_GIntToPtr(Src: m_ICst(Cst&: CVal))) || |
| 647 | mi_match(R: RootReg, MRI: *MRI, P: m_ICst(Cst&: CVal))) { |
| 648 | if (auto Fns = computeConstAddr(CVal, /*IsPrefetch=*/false, OrigBase: Register())) |
| 649 | return Fns; |
| 650 | } |
| 651 | |
| 652 | return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); }, |
| 653 | [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}}; |
| 654 | } |
| 655 | |
| 656 | InstructionSelector::ComplexRendererFns |
| 657 | RISCVInstructionSelector::selectBrindRegImm(MachineOperand &Root) const { |
| 658 | if (!Root.isReg()) |
| 659 | return std::nullopt; |
| 660 | |
| 661 | Register RootReg = Root.getReg(); |
| 662 | |
| 663 | // base + constant offset (G_PTR_ADD). Doesn't match a FrameIndex, unlike |
| 664 | // selectAddrRegImm, since the callers of this (e.g. indirect branches) |
| 665 | // can't take a FrameIndex or global address operand. |
| 666 | Register BaseReg; |
| 667 | int64_t RHSC; |
| 668 | if (mi_match(R: RootReg, MRI: *MRI, P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_ICst(Cst&: RHSC))) && |
| 669 | isInt<12>(x: RHSC)) { |
| 670 | return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: BaseReg); }, |
| 671 | [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}}; |
| 672 | } |
| 673 | |
| 674 | return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); }, |
| 675 | [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}}; |
| 676 | } |
| 677 | |
| 678 | InstructionSelector::ComplexRendererFns |
| 679 | RISCVInstructionSelector::selectAddrRegImmLsb00000(MachineOperand &Root) const { |
| 680 | if (!Root.isReg()) |
| 681 | return std::nullopt; |
| 682 | |
| 683 | Register RootReg = Root.getReg(); |
| 684 | |
| 685 | // Frame index. |
| 686 | int FI; |
| 687 | if (mi_match(R: RootReg, MRI: *MRI, P: m_GFrameIndex(FI))) { |
| 688 | return {{ |
| 689 | [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); }, |
| 690 | [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, |
| 691 | }}; |
| 692 | } |
| 693 | |
| 694 | // base + constant offset (G_PTR_ADD). |
| 695 | Register BaseReg; |
| 696 | int64_t RHSC; |
| 697 | if (mi_match(R: RootReg, MRI: *MRI, P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_ICst(Cst&: RHSC)))) { |
| 698 | if (isInt<12>(x: RHSC)) { |
| 699 | // Not a multiple of 32: can't encode, use the address as-is. |
| 700 | if ((RHSC & 0b11111) != 0) { |
| 701 | return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); }, |
| 702 | [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}}; |
| 703 | } |
| 704 | // Fold the offset. |
| 705 | int BaseFI; |
| 706 | if (mi_match(R: BaseReg, MRI: *MRI, P: m_GFrameIndex(FI&: BaseFI))) |
| 707 | return {{ |
| 708 | [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: BaseFI); }, |
| 709 | [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }, |
| 710 | }}; |
| 711 | return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: BaseReg); }, |
| 712 | [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: RHSC); }}}; |
| 713 | } |
| 714 | |
| 715 | // Large constant: fold a -2048/2016 adjustment to save an instruction. |
| 716 | if ((-2049 >= RHSC && RHSC >= -4096) || (4063 >= RHSC && RHSC >= 2017)) { |
| 717 | int64_t Adj = RHSC < 0 ? -2048 : 2016; |
| 718 | return renderAddiPair(BaseReg, AddiImm: RHSC - Adj, OffsetImm: Adj); |
| 719 | } |
| 720 | |
| 721 | // Otherwise split the constant into Hi (materialized + added to the base) |
| 722 | // and Lo12 (folded offset). |
| 723 | if (auto Fns = computeConstAddr(CVal: RHSC, /*IsPrefetch=*/true, OrigBase: BaseReg)) |
| 724 | return Fns; |
| 725 | } |
| 726 | |
| 727 | // Bare constant address. IRTranslator emits inttoptr(C) as |
| 728 | // G_INTTOPTR(G_CONSTANT); look through the G_INTTOPTR to reach the constant. |
| 729 | int64_t CVal; |
| 730 | if (mi_match(R: RootReg, MRI: *MRI, P: m_GIntToPtr(Src: m_ICst(Cst&: CVal))) || |
| 731 | mi_match(R: RootReg, MRI: *MRI, P: m_ICst(Cst&: CVal))) { |
| 732 | if (auto Fns = computeConstAddr(CVal, /*IsPrefetch=*/true, OrigBase: Register())) |
| 733 | return Fns; |
| 734 | } |
| 735 | |
| 736 | return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RootReg); }, |
| 737 | [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }}}; |
| 738 | } |
| 739 | |
| 740 | /// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC. |
| 741 | /// CC Must be an ICMP Predicate. |
| 742 | static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) { |
| 743 | switch (CC) { |
| 744 | default: |
| 745 | llvm_unreachable("Expected ICMP CmpInst::Predicate." ); |
| 746 | case CmpInst::Predicate::ICMP_EQ: |
| 747 | return RISCVCC::COND_EQ; |
| 748 | case CmpInst::Predicate::ICMP_NE: |
| 749 | return RISCVCC::COND_NE; |
| 750 | case CmpInst::Predicate::ICMP_ULT: |
| 751 | return RISCVCC::COND_LTU; |
| 752 | case CmpInst::Predicate::ICMP_SLT: |
| 753 | return RISCVCC::COND_LT; |
| 754 | case CmpInst::Predicate::ICMP_UGE: |
| 755 | return RISCVCC::COND_GEU; |
| 756 | case CmpInst::Predicate::ICMP_SGE: |
| 757 | return RISCVCC::COND_GE; |
| 758 | } |
| 759 | } |
| 760 | |
| 761 | static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC, |
| 762 | Register &LHS, Register &RHS, |
| 763 | MachineRegisterInfo &MRI) { |
| 764 | // Try to fold an ICmp. If that fails, use a NE compare with X0. |
| 765 | CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE; |
| 766 | if (!mi_match(R: CondReg, MRI, P: m_GICmp(P: m_Pred(P&: Pred), L: m_Reg(R&: LHS), R: m_Reg(R&: RHS)))) { |
| 767 | LHS = CondReg; |
| 768 | RHS = RISCV::X0; |
| 769 | CC = RISCVCC::COND_NE; |
| 770 | return; |
| 771 | } |
| 772 | |
| 773 | // We found an ICmp, do some canonicalization. |
| 774 | |
| 775 | // Adjust comparisons to use comparison with 0 if possible. |
| 776 | if (auto Constant = getIConstantVRegSExtVal(VReg: RHS, MRI)) { |
| 777 | switch (Pred) { |
| 778 | case CmpInst::Predicate::ICMP_SGT: |
| 779 | // Convert X > -1 to X >= 0 |
| 780 | if (*Constant == -1) { |
| 781 | CC = RISCVCC::COND_GE; |
| 782 | RHS = RISCV::X0; |
| 783 | return; |
| 784 | } |
| 785 | break; |
| 786 | case CmpInst::Predicate::ICMP_SLT: |
| 787 | // Convert X < 1 to 0 >= X |
| 788 | if (*Constant == 1) { |
| 789 | CC = RISCVCC::COND_GE; |
| 790 | RHS = LHS; |
| 791 | LHS = RISCV::X0; |
| 792 | return; |
| 793 | } |
| 794 | break; |
| 795 | default: |
| 796 | break; |
| 797 | } |
| 798 | } |
| 799 | |
| 800 | switch (Pred) { |
| 801 | default: |
| 802 | llvm_unreachable("Expected ICMP CmpInst::Predicate." ); |
| 803 | case CmpInst::Predicate::ICMP_EQ: |
| 804 | case CmpInst::Predicate::ICMP_NE: |
| 805 | case CmpInst::Predicate::ICMP_ULT: |
| 806 | case CmpInst::Predicate::ICMP_SLT: |
| 807 | case CmpInst::Predicate::ICMP_UGE: |
| 808 | case CmpInst::Predicate::ICMP_SGE: |
| 809 | // These CCs are supported directly by RISC-V branches. |
| 810 | break; |
| 811 | case CmpInst::Predicate::ICMP_SGT: |
| 812 | case CmpInst::Predicate::ICMP_SLE: |
| 813 | case CmpInst::Predicate::ICMP_UGT: |
| 814 | case CmpInst::Predicate::ICMP_ULE: |
| 815 | // These CCs are not supported directly by RISC-V branches, but changing the |
| 816 | // direction of the CC and swapping LHS and RHS are. |
| 817 | Pred = CmpInst::getSwappedPredicate(pred: Pred); |
| 818 | std::swap(a&: LHS, b&: RHS); |
| 819 | break; |
| 820 | } |
| 821 | |
| 822 | CC = getRISCVCCFromICmp(CC: Pred); |
| 823 | } |
| 824 | |
| 825 | /// Select the RISC-V Zalasr opcode for the G_LOAD or G_STORE operation |
| 826 | /// \p GenericOpc, appropriate for the GPR register bank and of memory access |
| 827 | /// size \p OpSize. |
| 828 | static unsigned selectZalasrLoadStoreOp(unsigned GenericOpc, unsigned OpSize) { |
| 829 | const bool IsStore = GenericOpc == TargetOpcode::G_STORE; |
| 830 | switch (OpSize) { |
| 831 | default: |
| 832 | llvm_unreachable("Unexpected memory size" ); |
| 833 | case 8: |
| 834 | return IsStore ? RISCV::SB_RL : RISCV::LB_AQ; |
| 835 | case 16: |
| 836 | return IsStore ? RISCV::SH_RL : RISCV::LH_AQ; |
| 837 | case 32: |
| 838 | return IsStore ? RISCV::SW_RL : RISCV::LW_AQ; |
| 839 | case 64: |
| 840 | return IsStore ? RISCV::SD_RL : RISCV::LD_AQ; |
| 841 | } |
| 842 | } |
| 843 | |
| 844 | /// Select the RISC-V regimm opcode for the G_LOAD or G_STORE operation |
| 845 | /// \p GenericOpc on \p STI, appropriate for the GPR register bank and of |
| 846 | /// memory access size \p OpSize. \returns \p GenericOpc if the combination |
| 847 | /// is unsupported. |
| 848 | static unsigned selectRegImmLoadStoreOp(const RISCVSubtarget &STI, |
| 849 | unsigned GenericOpc, unsigned OpSize) { |
| 850 | const bool IsStore = GenericOpc == TargetOpcode::G_STORE; |
| 851 | switch (OpSize) { |
| 852 | case 8: |
| 853 | // Prefer unsigned due to no c.lb in Zcb. |
| 854 | return IsStore ? RISCV::SB : RISCV::LBU; |
| 855 | case 16: |
| 856 | return IsStore ? RISCV::SH : RISCV::LH; |
| 857 | case 32: |
| 858 | return IsStore ? RISCV::SW : RISCV::LW; |
| 859 | case 64: |
| 860 | // LD/SD are RV64-only. On RV32 a 64-bit access goes through an even/odd |
| 861 | // GPR pair, which Zilsd provides via the LD_RV32/SD_RV32 pseudos. |
| 862 | if (!STI.is64Bit() && STI.hasStdExtZilsd()) |
| 863 | return IsStore ? RISCV::SD_RV32 : RISCV::LD_RV32; |
| 864 | return IsStore ? RISCV::SD : RISCV::LD; |
| 865 | } |
| 866 | |
| 867 | return GenericOpc; |
| 868 | } |
| 869 | |
| 870 | void RISCVInstructionSelector::addVectorLoadStoreOperands( |
| 871 | MachineInstr &I, SmallVectorImpl<Register> &SrcOps, unsigned &CurOp, |
| 872 | bool IsMasked, bool IsStridedOrIndexed, LLT *IndexVT) const { |
| 873 | // Base Pointer |
| 874 | auto PtrReg = I.getOperand(i: CurOp++).getReg(); |
| 875 | SrcOps.push_back(Elt: PtrReg); |
| 876 | |
| 877 | // Stride or Index |
| 878 | if (IsStridedOrIndexed) { |
| 879 | auto StrideReg = I.getOperand(i: CurOp++).getReg(); |
| 880 | SrcOps.push_back(Elt: StrideReg); |
| 881 | if (IndexVT) |
| 882 | *IndexVT = MRI->getType(Reg: StrideReg); |
| 883 | } |
| 884 | |
| 885 | // Mask |
| 886 | if (IsMasked) { |
| 887 | auto MaskReg = I.getOperand(i: CurOp++).getReg(); |
| 888 | SrcOps.push_back(Elt: MaskReg); |
| 889 | } |
| 890 | } |
| 891 | |
| 892 | bool RISCVInstructionSelector::selectIntrinsicWithSideEffects( |
| 893 | MachineInstr &I) const { |
| 894 | // Find the intrinsic ID. |
| 895 | unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID(); |
| 896 | // Select the instruction. |
| 897 | switch (IntrinID) { |
| 898 | default: |
| 899 | return false; |
| 900 | case Intrinsic::riscv_vlm: |
| 901 | case Intrinsic::riscv_vle: |
| 902 | case Intrinsic::riscv_vle_mask: |
| 903 | case Intrinsic::riscv_vlse: |
| 904 | case Intrinsic::riscv_vlse_mask: { |
| 905 | bool IsMasked = IntrinID == Intrinsic::riscv_vle_mask || |
| 906 | IntrinID == Intrinsic::riscv_vlse_mask; |
| 907 | bool IsStrided = IntrinID == Intrinsic::riscv_vlse || |
| 908 | IntrinID == Intrinsic::riscv_vlse_mask; |
| 909 | LLT VT = MRI->getType(Reg: I.getOperand(i: 0).getReg()); |
| 910 | unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits()); |
| 911 | |
| 912 | // Result vector |
| 913 | const Register DstReg = I.getOperand(i: 0).getReg(); |
| 914 | |
| 915 | // Sources |
| 916 | bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm; |
| 917 | unsigned CurOp = 2; |
| 918 | SmallVector<Register, 4> SrcOps; // Source registers. |
| 919 | |
| 920 | // Passthru |
| 921 | if (HasPassthruOperand) { |
| 922 | auto PassthruReg = I.getOperand(i: CurOp++).getReg(); |
| 923 | SrcOps.push_back(Elt: PassthruReg); |
| 924 | } else { |
| 925 | SrcOps.push_back(Elt: Register(RISCV::NoRegister)); |
| 926 | } |
| 927 | |
| 928 | addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: IsStrided); |
| 929 | |
| 930 | RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT)); |
| 931 | const RISCV::VLEPseudo *P = |
| 932 | RISCV::getVLEPseudo(Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW, |
| 933 | LMUL: static_cast<unsigned>(LMUL)); |
| 934 | |
| 935 | MachineInstrBuilder PseudoMI = |
| 936 | BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo), DestReg: DstReg); |
| 937 | for (Register Reg : SrcOps) |
| 938 | PseudoMI.addReg(RegNo: Reg); |
| 939 | |
| 940 | // Select VL |
| 941 | auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++)); |
| 942 | for (auto &RenderFn : *VLOpFn) |
| 943 | RenderFn(PseudoMI); |
| 944 | |
| 945 | // SEW |
| 946 | PseudoMI.addImm(Val: Log2SEW); |
| 947 | |
| 948 | // Policy |
| 949 | uint64_t Policy = RISCVVType::MASK_AGNOSTIC; |
| 950 | if (IsMasked) |
| 951 | Policy = I.getOperand(i: CurOp++).getImm(); |
| 952 | PseudoMI.addImm(Val: Policy); |
| 953 | |
| 954 | // Memref |
| 955 | PseudoMI.cloneMemRefs(OtherMI: I); |
| 956 | |
| 957 | I.eraseFromParent(); |
| 958 | constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI); |
| 959 | return true; |
| 960 | } |
| 961 | case Intrinsic::riscv_vloxei: |
| 962 | case Intrinsic::riscv_vloxei_mask: |
| 963 | case Intrinsic::riscv_vluxei: |
| 964 | case Intrinsic::riscv_vluxei_mask: { |
| 965 | bool IsMasked = IntrinID == Intrinsic::riscv_vloxei_mask || |
| 966 | IntrinID == Intrinsic::riscv_vluxei_mask; |
| 967 | bool IsOrdered = IntrinID == Intrinsic::riscv_vloxei || |
| 968 | IntrinID == Intrinsic::riscv_vloxei_mask; |
| 969 | LLT VT = MRI->getType(Reg: I.getOperand(i: 0).getReg()); |
| 970 | unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits()); |
| 971 | |
| 972 | // Result vector |
| 973 | const Register DstReg = I.getOperand(i: 0).getReg(); |
| 974 | |
| 975 | // Sources |
| 976 | bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm; |
| 977 | unsigned CurOp = 2; |
| 978 | SmallVector<Register, 4> SrcOps; // Source registers. |
| 979 | |
| 980 | // Passthru |
| 981 | if (HasPassthruOperand) { |
| 982 | auto PassthruReg = I.getOperand(i: CurOp++).getReg(); |
| 983 | SrcOps.push_back(Elt: PassthruReg); |
| 984 | } else { |
| 985 | // Use NoRegister if there is no specified passthru. |
| 986 | SrcOps.push_back(Elt: Register()); |
| 987 | } |
| 988 | LLT IndexVT; |
| 989 | addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: true, IndexVT: &IndexVT); |
| 990 | |
| 991 | RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT)); |
| 992 | RISCVVType::VLMUL IndexLMUL = |
| 993 | RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: IndexVT)); |
| 994 | unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits()); |
| 995 | if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { |
| 996 | reportFatalUsageError(reason: "The V extension does not support EEW=64 for index " |
| 997 | "values when XLEN=32" ); |
| 998 | } |
| 999 | const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo( |
| 1000 | Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL), |
| 1001 | IndexLMUL: static_cast<unsigned>(IndexLMUL)); |
| 1002 | |
| 1003 | MachineInstrBuilder PseudoMI = |
| 1004 | BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo), DestReg: DstReg); |
| 1005 | for (Register Reg : SrcOps) |
| 1006 | PseudoMI.addReg(RegNo: Reg); |
| 1007 | |
| 1008 | // Select VL |
| 1009 | auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++)); |
| 1010 | for (auto &RenderFn : *VLOpFn) |
| 1011 | RenderFn(PseudoMI); |
| 1012 | |
| 1013 | // SEW |
| 1014 | PseudoMI.addImm(Val: Log2SEW); |
| 1015 | |
| 1016 | // Policy |
| 1017 | uint64_t Policy = RISCVVType::MASK_AGNOSTIC; |
| 1018 | if (IsMasked) |
| 1019 | Policy = I.getOperand(i: CurOp++).getImm(); |
| 1020 | PseudoMI.addImm(Val: Policy); |
| 1021 | |
| 1022 | // Memref |
| 1023 | PseudoMI.cloneMemRefs(OtherMI: I); |
| 1024 | |
| 1025 | I.eraseFromParent(); |
| 1026 | constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI); |
| 1027 | return true; |
| 1028 | } |
| 1029 | case Intrinsic::riscv_vsm: |
| 1030 | case Intrinsic::riscv_vse: |
| 1031 | case Intrinsic::riscv_vse_mask: |
| 1032 | case Intrinsic::riscv_vsse: |
| 1033 | case Intrinsic::riscv_vsse_mask: { |
| 1034 | bool IsMasked = IntrinID == Intrinsic::riscv_vse_mask || |
| 1035 | IntrinID == Intrinsic::riscv_vsse_mask; |
| 1036 | bool IsStrided = IntrinID == Intrinsic::riscv_vsse || |
| 1037 | IntrinID == Intrinsic::riscv_vsse_mask; |
| 1038 | LLT VT = MRI->getType(Reg: I.getOperand(i: 1).getReg()); |
| 1039 | unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits()); |
| 1040 | |
| 1041 | // Sources |
| 1042 | unsigned CurOp = 1; |
| 1043 | SmallVector<Register, 4> SrcOps; // Source registers. |
| 1044 | |
| 1045 | // Store value |
| 1046 | auto PassthruReg = I.getOperand(i: CurOp++).getReg(); |
| 1047 | SrcOps.push_back(Elt: PassthruReg); |
| 1048 | |
| 1049 | addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: IsStrided); |
| 1050 | |
| 1051 | RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT)); |
| 1052 | const RISCV::VSEPseudo *P = RISCV::getVSEPseudo( |
| 1053 | Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL)); |
| 1054 | |
| 1055 | MachineInstrBuilder PseudoMI = |
| 1056 | BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo)); |
| 1057 | for (Register Reg : SrcOps) |
| 1058 | PseudoMI.addReg(RegNo: Reg); |
| 1059 | |
| 1060 | // Select VL |
| 1061 | auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++)); |
| 1062 | for (auto &RenderFn : *VLOpFn) |
| 1063 | RenderFn(PseudoMI); |
| 1064 | |
| 1065 | // SEW |
| 1066 | PseudoMI.addImm(Val: Log2SEW); |
| 1067 | |
| 1068 | // Memref |
| 1069 | PseudoMI.cloneMemRefs(OtherMI: I); |
| 1070 | |
| 1071 | I.eraseFromParent(); |
| 1072 | constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI); |
| 1073 | return true; |
| 1074 | } |
| 1075 | case Intrinsic::riscv_vsoxei: |
| 1076 | case Intrinsic::riscv_vsoxei_mask: |
| 1077 | case Intrinsic::riscv_vsuxei: |
| 1078 | case Intrinsic::riscv_vsuxei_mask: { |
| 1079 | bool IsMasked = IntrinID == Intrinsic::riscv_vsoxei_mask || |
| 1080 | IntrinID == Intrinsic::riscv_vsuxei_mask; |
| 1081 | bool IsOrdered = IntrinID == Intrinsic::riscv_vsoxei || |
| 1082 | IntrinID == Intrinsic::riscv_vsoxei_mask; |
| 1083 | LLT VT = MRI->getType(Reg: I.getOperand(i: 1).getReg()); |
| 1084 | unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits()); |
| 1085 | |
| 1086 | // Sources |
| 1087 | unsigned CurOp = 1; |
| 1088 | SmallVector<Register, 4> SrcOps; // Source registers. |
| 1089 | |
| 1090 | // Store value |
| 1091 | auto PassthruReg = I.getOperand(i: CurOp++).getReg(); |
| 1092 | SrcOps.push_back(Elt: PassthruReg); |
| 1093 | |
| 1094 | LLT IndexVT; |
| 1095 | addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStridedOrIndexed: true, IndexVT: &IndexVT); |
| 1096 | |
| 1097 | RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: VT)); |
| 1098 | RISCVVType::VLMUL IndexLMUL = |
| 1099 | RISCVTargetLowering::getLMUL(VT: getMVTForLLT(Ty: IndexVT)); |
| 1100 | unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits()); |
| 1101 | if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) { |
| 1102 | reportFatalUsageError(reason: "The V extension does not support EEW=64 for index " |
| 1103 | "values when XLEN=32" ); |
| 1104 | } |
| 1105 | const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo( |
| 1106 | Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL), |
| 1107 | IndexLMUL: static_cast<unsigned>(IndexLMUL)); |
| 1108 | |
| 1109 | MachineInstrBuilder PseudoMI = |
| 1110 | BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: P->Pseudo)); |
| 1111 | for (Register Reg : SrcOps) |
| 1112 | PseudoMI.addReg(RegNo: Reg); |
| 1113 | |
| 1114 | // Select VL |
| 1115 | auto VLOpFn = renderVLOp(Root&: I.getOperand(i: CurOp++)); |
| 1116 | for (auto &RenderFn : *VLOpFn) |
| 1117 | RenderFn(PseudoMI); |
| 1118 | |
| 1119 | // SEW |
| 1120 | PseudoMI.addImm(Val: Log2SEW); |
| 1121 | |
| 1122 | // Memref |
| 1123 | PseudoMI.cloneMemRefs(OtherMI: I); |
| 1124 | |
| 1125 | I.eraseFromParent(); |
| 1126 | constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI); |
| 1127 | return true; |
| 1128 | } |
| 1129 | } |
| 1130 | } |
| 1131 | |
| 1132 | bool RISCVInstructionSelector::selectIntrinsic(MachineInstr &I) const { |
| 1133 | // Find the intrinsic ID. |
| 1134 | unsigned IntrinID = cast<GIntrinsic>(Val&: I).getIntrinsicID(); |
| 1135 | // Select the instruction. |
| 1136 | switch (IntrinID) { |
| 1137 | default: |
| 1138 | return false; |
| 1139 | case Intrinsic::riscv_vsetvli: |
| 1140 | case Intrinsic::riscv_vsetvlimax: { |
| 1141 | |
| 1142 | bool VLMax = IntrinID == Intrinsic::riscv_vsetvlimax; |
| 1143 | |
| 1144 | unsigned Offset = VLMax ? 2 : 3; |
| 1145 | unsigned SEW = RISCVVType::decodeVSEW(VSEW: I.getOperand(i: Offset).getImm() & 0x7); |
| 1146 | RISCVVType::VLMUL VLMul = |
| 1147 | static_cast<RISCVVType::VLMUL>(I.getOperand(i: Offset + 1).getImm() & 0x7); |
| 1148 | |
| 1149 | unsigned VTypeI = RISCVVType::encodeVTYPE(VLMUL: VLMul, SEW, /*TailAgnostic*/ true, |
| 1150 | /*MaskAgnostic*/ true); |
| 1151 | |
| 1152 | Register DstReg = I.getOperand(i: 0).getReg(); |
| 1153 | |
| 1154 | Register VLOperand; |
| 1155 | unsigned Opcode = RISCV::PseudoVSETVLI; |
| 1156 | |
| 1157 | // Check if AVL is a constant that equals VLMAX. |
| 1158 | if (!VLMax) { |
| 1159 | Register AVLReg = I.getOperand(i: 2).getReg(); |
| 1160 | if (auto AVLConst = getIConstantVRegValWithLookThrough(VReg: AVLReg, MRI: *MRI)) { |
| 1161 | uint64_t AVL = AVLConst->Value.getZExtValue(); |
| 1162 | if (auto VLEN = Subtarget->getRealVLen()) { |
| 1163 | if (*VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == AVL) |
| 1164 | VLMax = true; |
| 1165 | } |
| 1166 | } |
| 1167 | |
| 1168 | if (mi_match(R: AVLReg, MRI: *MRI, P: m_AllOnes())) |
| 1169 | VLMax = true; |
| 1170 | } |
| 1171 | |
| 1172 | if (VLMax) { |
| 1173 | VLOperand = Register(RISCV::X0); |
| 1174 | Opcode = RISCV::PseudoVSETVLIX0; |
| 1175 | } else { |
| 1176 | Register AVLReg = I.getOperand(i: 2).getReg(); |
| 1177 | VLOperand = AVLReg; |
| 1178 | |
| 1179 | // Check if AVL is a small constant that can use PseudoVSETIVLI. |
| 1180 | if (auto AVLConst = getIConstantVRegValWithLookThrough(VReg: AVLReg, MRI: *MRI)) { |
| 1181 | uint64_t AVL = AVLConst->Value.getZExtValue(); |
| 1182 | if (isUInt<5>(x: AVL)) { |
| 1183 | MachineInstr *PseudoMI = |
| 1184 | BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), |
| 1185 | MCID: TII.get(Opcode: RISCV::PseudoVSETIVLI), DestReg: DstReg) |
| 1186 | .addImm(Val: AVL) |
| 1187 | .addImm(Val: VTypeI); |
| 1188 | I.eraseFromParent(); |
| 1189 | constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI); |
| 1190 | return true; |
| 1191 | } |
| 1192 | } |
| 1193 | } |
| 1194 | |
| 1195 | MachineInstr *PseudoMI = |
| 1196 | BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode), DestReg: DstReg) |
| 1197 | .addReg(RegNo: VLOperand) |
| 1198 | .addImm(Val: VTypeI); |
| 1199 | I.eraseFromParent(); |
| 1200 | constrainSelectedInstRegOperands(I&: *PseudoMI, TII, TRI, RBI); |
| 1201 | return true; |
| 1202 | } |
| 1203 | } |
| 1204 | } |
| 1205 | |
| 1206 | bool RISCVInstructionSelector::(MachineInstr &MI) const { |
| 1207 | assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_SUBVECTOR); |
| 1208 | |
| 1209 | Register DstReg = MI.getOperand(i: 0).getReg(); |
| 1210 | Register SrcReg = MI.getOperand(i: 1).getReg(); |
| 1211 | |
| 1212 | LLT DstTy = MRI->getType(Reg: DstReg); |
| 1213 | LLT SrcTy = MRI->getType(Reg: SrcReg); |
| 1214 | |
| 1215 | unsigned Idx = static_cast<unsigned>(MI.getOperand(i: 2).getImm()); |
| 1216 | |
| 1217 | MVT DstMVT = getMVTForLLT(Ty: DstTy); |
| 1218 | MVT SrcMVT = getMVTForLLT(Ty: SrcTy); |
| 1219 | |
| 1220 | unsigned SubRegIdx; |
| 1221 | std::tie(args&: SubRegIdx, args&: Idx) = |
| 1222 | RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs( |
| 1223 | VecVT: SrcMVT, SubVecVT: DstMVT, InsertExtractIdx: Idx, TRI: &TRI); |
| 1224 | |
| 1225 | if (Idx != 0) |
| 1226 | return false; |
| 1227 | |
| 1228 | unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: DstMVT); |
| 1229 | const TargetRegisterClass *DstRC = TRI.getRegClass(i: DstRegClassID); |
| 1230 | if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) |
| 1231 | return false; |
| 1232 | |
| 1233 | unsigned SrcRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: SrcMVT); |
| 1234 | const TargetRegisterClass *SrcRC = TRI.getRegClass(i: SrcRegClassID); |
| 1235 | if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI)) |
| 1236 | return false; |
| 1237 | |
| 1238 | BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY), |
| 1239 | DestReg: DstReg) |
| 1240 | .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubRegIdx); |
| 1241 | |
| 1242 | MI.eraseFromParent(); |
| 1243 | return true; |
| 1244 | } |
| 1245 | |
| 1246 | bool RISCVInstructionSelector::selectInsertSubVector(MachineInstr &MI) const { |
| 1247 | assert(MI.getOpcode() == TargetOpcode::G_INSERT_SUBVECTOR); |
| 1248 | |
| 1249 | Register DstReg = MI.getOperand(i: 0).getReg(); |
| 1250 | Register VecReg = MI.getOperand(i: 1).getReg(); |
| 1251 | Register SubVecReg = MI.getOperand(i: 2).getReg(); |
| 1252 | |
| 1253 | LLT VecTy = MRI->getType(Reg: VecReg); |
| 1254 | LLT SubVecTy = MRI->getType(Reg: SubVecReg); |
| 1255 | |
| 1256 | MVT VecMVT = getMVTForLLT(Ty: VecTy); |
| 1257 | MVT SubVecMVT = getMVTForLLT(Ty: SubVecTy); |
| 1258 | |
| 1259 | unsigned Idx = static_cast<unsigned>(MI.getOperand(i: 3).getImm()); |
| 1260 | |
| 1261 | unsigned SubRegIdx; |
| 1262 | std::tie(args&: SubRegIdx, args&: Idx) = |
| 1263 | RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs( |
| 1264 | VecVT: VecMVT, SubVecVT: SubVecMVT, InsertExtractIdx: Idx, TRI: &TRI); |
| 1265 | |
| 1266 | // If the Idx hasn't been completely eliminated then this is a subvector |
| 1267 | // insert which doesn't naturally align to a vector register. These must |
| 1268 | // be handled using instructions to manipulate the vector registers. |
| 1269 | if (Idx != 0) |
| 1270 | return false; |
| 1271 | |
| 1272 | // Constrain dst |
| 1273 | unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: VecMVT); |
| 1274 | const TargetRegisterClass *DstRC = TRI.getRegClass(i: DstRegClassID); |
| 1275 | if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) |
| 1276 | return false; |
| 1277 | |
| 1278 | // If we haven't set a SubRegIdx, then we must be going between |
| 1279 | // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy. |
| 1280 | if (SubRegIdx == RISCV::NoSubRegister) { |
| 1281 | assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecMVT) == |
| 1282 | DstRegClassID && |
| 1283 | "Unexpected subvector insert" ); |
| 1284 | BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY), |
| 1285 | DestReg: DstReg) |
| 1286 | .addReg(RegNo: SubVecReg); |
| 1287 | MI.eraseFromParent(); |
| 1288 | return true; |
| 1289 | } |
| 1290 | |
| 1291 | // Use INSERT_SUBREG to insert the subvector into the vector at the |
| 1292 | // appropriate subregister index. |
| 1293 | MachineInstr *Ins = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 1294 | MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DstReg) |
| 1295 | .addReg(RegNo: VecReg) |
| 1296 | .addReg(RegNo: SubVecReg) |
| 1297 | .addImm(Val: SubRegIdx); |
| 1298 | |
| 1299 | MI.eraseFromParent(); |
| 1300 | constrainSelectedInstRegOperands(I&: *Ins, TII, TRI, RBI); |
| 1301 | return true; |
| 1302 | } |
| 1303 | |
| 1304 | bool RISCVInstructionSelector::select(MachineInstr &MI) { |
| 1305 | preISelLower(MI); |
| 1306 | const unsigned Opc = MI.getOpcode(); |
| 1307 | |
| 1308 | if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) { |
| 1309 | if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) { |
| 1310 | const Register DefReg = MI.getOperand(i: 0).getReg(); |
| 1311 | const LLT DefTy = MRI->getType(Reg: DefReg); |
| 1312 | |
| 1313 | const RegClassOrRegBank &RegClassOrBank = |
| 1314 | MRI->getRegClassOrRegBank(Reg: DefReg); |
| 1315 | |
| 1316 | const TargetRegisterClass *DefRC = |
| 1317 | dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank); |
| 1318 | if (!DefRC) { |
| 1319 | if (!DefTy.isValid()) { |
| 1320 | LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n" ); |
| 1321 | return false; |
| 1322 | } |
| 1323 | |
| 1324 | const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank); |
| 1325 | DefRC = TRI.getRegClassForTypeOnBank(Ty: DefTy, RB, Is64Bit: STI.is64Bit()); |
| 1326 | if (!DefRC) { |
| 1327 | LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n" ); |
| 1328 | return false; |
| 1329 | } |
| 1330 | } |
| 1331 | |
| 1332 | MI.setDesc(TII.get(Opcode: TargetOpcode::PHI)); |
| 1333 | return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI&: *MRI); |
| 1334 | } |
| 1335 | |
| 1336 | // Certain non-generic instructions also need some special handling. |
| 1337 | if (MI.isCopy()) |
| 1338 | return selectCopy(MI); |
| 1339 | |
| 1340 | return true; |
| 1341 | } |
| 1342 | |
| 1343 | if (selectImpl(I&: MI, CoverageInfo&: *CoverageInfo)) |
| 1344 | return true; |
| 1345 | |
| 1346 | switch (Opc) { |
| 1347 | case TargetOpcode::G_BITCAST: |
| 1348 | case TargetOpcode::G_ANYEXT: |
| 1349 | case TargetOpcode::G_PTRTOINT: |
| 1350 | case TargetOpcode::G_INTTOPTR: |
| 1351 | case TargetOpcode::G_TRUNC: |
| 1352 | case TargetOpcode::G_FREEZE: |
| 1353 | return selectCopy(MI); |
| 1354 | case TargetOpcode::G_CONSTANT: { |
| 1355 | Register DstReg = MI.getOperand(i: 0).getReg(); |
| 1356 | int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue(); |
| 1357 | |
| 1358 | if (!materializeImm(Reg: DstReg, Imm, MI)) |
| 1359 | return false; |
| 1360 | |
| 1361 | MI.eraseFromParent(); |
| 1362 | return true; |
| 1363 | } |
| 1364 | case TargetOpcode::G_ZEXT: |
| 1365 | case TargetOpcode::G_SEXT: { |
| 1366 | bool IsSigned = Opc != TargetOpcode::G_ZEXT; |
| 1367 | Register DstReg = MI.getOperand(i: 0).getReg(); |
| 1368 | Register SrcReg = MI.getOperand(i: 1).getReg(); |
| 1369 | LLT SrcTy = MRI->getType(Reg: SrcReg); |
| 1370 | unsigned SrcSize = SrcTy.getSizeInBits(); |
| 1371 | |
| 1372 | if (SrcTy.isVector()) |
| 1373 | return false; // Should be handled by imported patterns. |
| 1374 | |
| 1375 | assert((*RBI.getRegBank(DstReg, *MRI, TRI)).getID() == |
| 1376 | RISCV::GPRBRegBankID && |
| 1377 | "Unexpected ext regbank" ); |
| 1378 | |
| 1379 | // Use addiw SrcReg, 0 (sext.w) for i32. |
| 1380 | if (IsSigned && SrcSize == 32) { |
| 1381 | MI.setDesc(TII.get(Opcode: RISCV::ADDIW)); |
| 1382 | MI.addOperand(Op: MachineOperand::CreateImm(Val: 0)); |
| 1383 | constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI); |
| 1384 | return true; |
| 1385 | } |
| 1386 | |
| 1387 | // Use add.uw SrcReg, X0 (zext.w) for i32 with Zba. |
| 1388 | if (!IsSigned && SrcSize == 32 && STI.hasStdExtZba()) { |
| 1389 | MI.setDesc(TII.get(Opcode: RISCV::ADD_UW)); |
| 1390 | MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::X0, /*isDef=*/false)); |
| 1391 | constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI); |
| 1392 | return true; |
| 1393 | } |
| 1394 | |
| 1395 | // Use sext.h/zext.h for i16 with Zbb. |
| 1396 | if (SrcSize == 16 && |
| 1397 | (STI.hasStdExtZbb() || (!IsSigned && STI.hasStdExtZbkb()))) { |
| 1398 | MI.setDesc(TII.get(Opcode: IsSigned ? RISCV::SEXT_H |
| 1399 | : STI.is64Bit() ? RISCV::ZEXT_H_RV64 |
| 1400 | : RISCV::ZEXT_H_RV32)); |
| 1401 | constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI); |
| 1402 | return true; |
| 1403 | } |
| 1404 | |
| 1405 | // Fall back to shift pair. |
| 1406 | Register ShiftLeftReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 1407 | MachineInstr *ShiftLeft = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 1408 | MCID: TII.get(Opcode: RISCV::SLLI), DestReg: ShiftLeftReg) |
| 1409 | .addReg(RegNo: SrcReg) |
| 1410 | .addImm(Val: STI.getXLen() - SrcSize); |
| 1411 | constrainSelectedInstRegOperands(I&: *ShiftLeft, TII, TRI, RBI); |
| 1412 | MachineInstr *ShiftRight = |
| 1413 | BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 1414 | MCID: TII.get(Opcode: IsSigned ? RISCV::SRAI : RISCV::SRLI), DestReg: DstReg) |
| 1415 | .addReg(RegNo: ShiftLeftReg) |
| 1416 | .addImm(Val: STI.getXLen() - SrcSize); |
| 1417 | constrainSelectedInstRegOperands(I&: *ShiftRight, TII, TRI, RBI); |
| 1418 | MI.eraseFromParent(); |
| 1419 | return true; |
| 1420 | } |
| 1421 | case TargetOpcode::G_FCONSTANT: { |
| 1422 | // TODO: Use constant pool for complex constants. |
| 1423 | Register DstReg = MI.getOperand(i: 0).getReg(); |
| 1424 | const APFloat &FPimm = MI.getOperand(i: 1).getFPImm()->getValueAPF(); |
| 1425 | unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits(); |
| 1426 | if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) { |
| 1427 | Register GPRReg; |
| 1428 | if (FPimm.isPosZero()) { |
| 1429 | GPRReg = RISCV::X0; |
| 1430 | } else { |
| 1431 | GPRReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 1432 | APInt Imm = FPimm.bitcastToAPInt(); |
| 1433 | if (!materializeImm(Reg: GPRReg, Imm: Imm.getSExtValue(), MI)) |
| 1434 | return false; |
| 1435 | } |
| 1436 | |
| 1437 | unsigned Opcode = Size == 64 ? RISCV::FMV_D_X |
| 1438 | : Size == 32 ? RISCV::FMV_W_X |
| 1439 | : RISCV::FMV_H_X; |
| 1440 | MachineInstr *FMV = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 1441 | MCID: TII.get(Opcode), DestReg: DstReg) |
| 1442 | .addReg(RegNo: GPRReg); |
| 1443 | constrainSelectedInstRegOperands(I&: *FMV, TII, TRI, RBI); |
| 1444 | } else { |
| 1445 | // s64 on rv32 |
| 1446 | assert(Size == 64 && !Subtarget->is64Bit() && |
| 1447 | "Unexpected size or subtarget" ); |
| 1448 | |
| 1449 | if (FPimm.isPosZero()) { |
| 1450 | // Optimize +0.0 to use fcvt.d.w |
| 1451 | MachineInstr *FCVT = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 1452 | MCID: TII.get(Opcode: RISCV::FCVT_D_W), DestReg: DstReg) |
| 1453 | .addReg(RegNo: RISCV::X0) |
| 1454 | .addImm(Val: RISCVFPRndMode::RNE); |
| 1455 | constrainSelectedInstRegOperands(I&: *FCVT, TII, TRI, RBI); |
| 1456 | |
| 1457 | MI.eraseFromParent(); |
| 1458 | return true; |
| 1459 | } |
| 1460 | |
| 1461 | // Split into two pieces and build through the stack. |
| 1462 | Register GPRRegHigh = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 1463 | Register GPRRegLow = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 1464 | APInt Imm = FPimm.bitcastToAPInt(); |
| 1465 | if (!materializeImm(Reg: GPRRegHigh, Imm: Imm.extractBits(numBits: 32, bitPosition: 32).getSExtValue(), |
| 1466 | MI)) |
| 1467 | return false; |
| 1468 | if (!materializeImm(Reg: GPRRegLow, Imm: Imm.trunc(width: 32).getSExtValue(), MI)) |
| 1469 | return false; |
| 1470 | MachineInstr *PairF64 = |
| 1471 | BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 1472 | MCID: TII.get(Opcode: RISCV::BuildPairF64Pseudo), DestReg: DstReg) |
| 1473 | .addReg(RegNo: GPRRegLow) |
| 1474 | .addReg(RegNo: GPRRegHigh); |
| 1475 | constrainSelectedInstRegOperands(I&: *PairF64, TII, TRI, RBI); |
| 1476 | } |
| 1477 | |
| 1478 | MI.eraseFromParent(); |
| 1479 | return true; |
| 1480 | } |
| 1481 | case TargetOpcode::G_GLOBAL_VALUE: { |
| 1482 | auto *GV = MI.getOperand(i: 1).getGlobal(); |
| 1483 | if (GV->isThreadLocal()) { |
| 1484 | // TODO: implement this case. |
| 1485 | return false; |
| 1486 | } |
| 1487 | |
| 1488 | return selectAddr(MI, IsLocal: GV->isDSOLocal(), IsExternWeak: GV->hasExternalWeakLinkage()); |
| 1489 | } |
| 1490 | case TargetOpcode::G_JUMP_TABLE: |
| 1491 | case TargetOpcode::G_CONSTANT_POOL: |
| 1492 | return selectAddr(MI); |
| 1493 | case TargetOpcode::G_BRCOND: { |
| 1494 | Register LHS, RHS; |
| 1495 | RISCVCC::CondCode CC; |
| 1496 | getOperandsForBranch(CondReg: MI.getOperand(i: 0).getReg(), CC, LHS, RHS, MRI&: *MRI); |
| 1497 | |
| 1498 | MachineInstr *Bcc = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 1499 | MCID: TII.get(Opcode: RISCVCC::getBrCond(CC))) |
| 1500 | .addReg(RegNo: LHS) |
| 1501 | .addReg(RegNo: RHS) |
| 1502 | .addMBB(MBB: MI.getOperand(i: 1).getMBB()); |
| 1503 | MI.eraseFromParent(); |
| 1504 | constrainSelectedInstRegOperands(I&: *Bcc, TII, TRI, RBI); |
| 1505 | return true; |
| 1506 | } |
| 1507 | case TargetOpcode::G_SELECT: |
| 1508 | return selectSelect(MI); |
| 1509 | case TargetOpcode::G_FCMP: |
| 1510 | return selectFPCompare(MI); |
| 1511 | case TargetOpcode::G_FENCE: { |
| 1512 | AtomicOrdering FenceOrdering = |
| 1513 | static_cast<AtomicOrdering>(MI.getOperand(i: 0).getImm()); |
| 1514 | SyncScope::ID FenceSSID = |
| 1515 | static_cast<SyncScope::ID>(MI.getOperand(i: 1).getImm()); |
| 1516 | emitFence(FenceOrdering, FenceSSID, MI); |
| 1517 | MI.eraseFromParent(); |
| 1518 | return true; |
| 1519 | } |
| 1520 | case TargetOpcode::G_IMPLICIT_DEF: |
| 1521 | return selectImplicitDef(MI); |
| 1522 | case TargetOpcode::G_UNMERGE_VALUES: |
| 1523 | return selectUnmergeValues(MI); |
| 1524 | case TargetOpcode::G_MERGE_VALUES: |
| 1525 | return selectMergeValues(MI); |
| 1526 | case TargetOpcode::G_LOAD: |
| 1527 | case TargetOpcode::G_STORE: { |
| 1528 | GLoadStore &LdSt = cast<GLoadStore>(Val&: MI); |
| 1529 | const Register ValReg = LdSt.getReg(Idx: 0); |
| 1530 | const Register PtrReg = LdSt.getPointerReg(); |
| 1531 | LLT PtrTy = MRI->getType(Reg: PtrReg); |
| 1532 | |
| 1533 | const RegisterBank &RB = *RBI.getRegBank(Reg: ValReg, MRI: *MRI, TRI); |
| 1534 | if (RB.getID() != RISCV::GPRBRegBankID) |
| 1535 | return false; |
| 1536 | |
| 1537 | #ifndef NDEBUG |
| 1538 | const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, *MRI, TRI); |
| 1539 | // Check that the pointer register is valid. |
| 1540 | assert(PtrRB.getID() == RISCV::GPRBRegBankID && |
| 1541 | "Load/Store pointer operand isn't a GPR" ); |
| 1542 | assert(PtrTy.isPointer() && "Load/Store pointer operand isn't a pointer" ); |
| 1543 | #endif |
| 1544 | |
| 1545 | // Can only handle AddressSpace 0. |
| 1546 | if (PtrTy.getAddressSpace() != 0) |
| 1547 | return false; |
| 1548 | |
| 1549 | unsigned MemSize = LdSt.getMemSizeInBits().getValue(); |
| 1550 | AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering(); |
| 1551 | |
| 1552 | if (isStrongerThanMonotonic(AO: Order)) { |
| 1553 | MI.setDesc(TII.get(Opcode: selectZalasrLoadStoreOp(GenericOpc: Opc, OpSize: MemSize))); |
| 1554 | constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI); |
| 1555 | return true; |
| 1556 | } |
| 1557 | |
| 1558 | const unsigned NewOpc = |
| 1559 | selectRegImmLoadStoreOp(STI, GenericOpc: MI.getOpcode(), OpSize: MemSize); |
| 1560 | if (NewOpc == MI.getOpcode()) |
| 1561 | return false; |
| 1562 | |
| 1563 | // Check if we can fold anything into the addressing mode. |
| 1564 | auto AddrModeFns = selectAddrRegImm(Root&: MI.getOperand(i: 1)); |
| 1565 | if (!AddrModeFns) |
| 1566 | return false; |
| 1567 | |
| 1568 | // Folded something. Create a new instruction and return it. |
| 1569 | MachineInstrBuilder NewInst = |
| 1570 | BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: NewOpc)); |
| 1571 | NewInst.setMIFlags(MI.getFlags()); |
| 1572 | if (isa<GStore>(Val: MI)) |
| 1573 | NewInst.addUse(RegNo: ValReg); |
| 1574 | else |
| 1575 | NewInst.addDef(RegNo: ValReg); |
| 1576 | NewInst.cloneMemRefs(OtherMI: MI); |
| 1577 | for (auto &Fn : *AddrModeFns) |
| 1578 | Fn(NewInst); |
| 1579 | MI.eraseFromParent(); |
| 1580 | |
| 1581 | constrainSelectedInstRegOperands(I&: *NewInst, TII, TRI, RBI); |
| 1582 | return true; |
| 1583 | } |
| 1584 | case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: |
| 1585 | return selectIntrinsicWithSideEffects(I&: MI); |
| 1586 | case TargetOpcode::G_INTRINSIC: |
| 1587 | return selectIntrinsic(I&: MI); |
| 1588 | case TargetOpcode::G_EXTRACT_SUBVECTOR: |
| 1589 | return selectExtractSubvector(MI); |
| 1590 | case TargetOpcode::G_INSERT_SUBVECTOR: |
| 1591 | return selectInsertSubVector(MI); |
| 1592 | default: |
| 1593 | return false; |
| 1594 | } |
| 1595 | } |
| 1596 | |
| 1597 | bool RISCVInstructionSelector::selectUnmergeValues(MachineInstr &MI) const { |
| 1598 | assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES); |
| 1599 | |
| 1600 | // Split Src into two s32 parts. |
| 1601 | if (MI.getNumOperands() != 3) |
| 1602 | return false; |
| 1603 | Register Src = MI.getOperand(i: 2).getReg(); |
| 1604 | Register Lo = MI.getOperand(i: 0).getReg(); |
| 1605 | Register Hi = MI.getOperand(i: 1).getReg(); |
| 1606 | |
| 1607 | // RV32+Zilsd: split a GPRPair (i64) into its two s32 GPR halves. |
| 1608 | if (!Subtarget->is64Bit() && Subtarget->hasStdExtZilsd() && |
| 1609 | MRI->getType(Reg: Src) == LLT::scalar(SizeInBits: 64) && isRegInGprb(Reg: Src) && |
| 1610 | isRegInGprb(Reg: Lo) && isRegInGprb(Reg: Hi)) { |
| 1611 | // GPRPairNoX0 rather than GPRPair: a sub-register index is only valid for |
| 1612 | // a class when every register in it supports the index, and X0_Pair has no |
| 1613 | // sub_gpr_odd. SelectionDAG gets the same effect through |
| 1614 | // InstrEmitter::ConstrainForSubReg. |
| 1615 | if (!RBI.constrainGenericRegister(Reg: Src, RC: RISCV::GPRPairNoX0RegClass, MRI&: *MRI)) |
| 1616 | return false; |
| 1617 | // COPY with a sub-register index rather than EXTRACT_SUBREG, which is what |
| 1618 | // SelectionDAG emits too: InstrEmitter lowers an EXTRACT_SUBREG SDNode to |
| 1619 | // %dst = COPY %src:sub and never builds the pseudo itself. A literal |
| 1620 | // EXTRACT_SUBREG would also survive to ExpandPostRAPseudos, which treats |
| 1621 | // one as unreachable. |
| 1622 | auto = [&](Register Dst, unsigned SubIdx) { |
| 1623 | RBI.constrainGenericRegister(Reg: Dst, RC: RISCV::GPRRegClass, MRI&: *MRI); |
| 1624 | MachineInstr &C = *BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 1625 | MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Dst) |
| 1626 | .addReg(RegNo: Src, Flags: {}, SubReg: SubIdx); |
| 1627 | constrainSelectedInstRegOperands(I&: C, TII, TRI, RBI); |
| 1628 | }; |
| 1629 | ExtractHalf(Lo, RISCV::sub_gpr_even); |
| 1630 | ExtractHalf(Hi, RISCV::sub_gpr_odd); |
| 1631 | MI.eraseFromParent(); |
| 1632 | return true; |
| 1633 | } |
| 1634 | |
| 1635 | if (!Subtarget->hasStdExtZfa()) |
| 1636 | return false; |
| 1637 | |
| 1638 | if (!isRegInFprb(Reg: Src) || !isRegInGprb(Reg: Lo) || !isRegInGprb(Reg: Hi)) |
| 1639 | return false; |
| 1640 | |
| 1641 | MachineInstr * = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 1642 | MCID: TII.get(Opcode: RISCV::FMV_X_W_FPR64), DestReg: Lo) |
| 1643 | .addReg(RegNo: Src); |
| 1644 | constrainSelectedInstRegOperands(I&: *ExtractLo, TII, TRI, RBI); |
| 1645 | |
| 1646 | MachineInstr * = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 1647 | MCID: TII.get(Opcode: RISCV::FMVH_X_D), DestReg: Hi) |
| 1648 | .addReg(RegNo: Src); |
| 1649 | constrainSelectedInstRegOperands(I&: *ExtractHi, TII, TRI, RBI); |
| 1650 | |
| 1651 | MI.eraseFromParent(); |
| 1652 | return true; |
| 1653 | } |
| 1654 | |
| 1655 | bool RISCVInstructionSelector::selectMergeValues(MachineInstr &MI) const { |
| 1656 | assert(MI.getOpcode() == TargetOpcode::G_MERGE_VALUES); |
| 1657 | |
| 1658 | if (MI.getNumOperands() != 3) |
| 1659 | return false; |
| 1660 | Register Dst = MI.getOperand(i: 0).getReg(); |
| 1661 | Register Lo = MI.getOperand(i: 1).getReg(); |
| 1662 | Register Hi = MI.getOperand(i: 2).getReg(); |
| 1663 | |
| 1664 | // RV32+Zilsd: build a GPRPair (i64) from two s32 GPR halves. |
| 1665 | if (!Subtarget->is64Bit() && Subtarget->hasStdExtZilsd() && |
| 1666 | MRI->getType(Reg: Dst) == LLT::scalar(SizeInBits: 64) && isRegInGprb(Reg: Dst) && |
| 1667 | isRegInGprb(Reg: Lo) && isRegInGprb(Reg: Hi)) { |
| 1668 | // GPRPairNoX0 rather than GPRPair, because sub_gpr_odd is not valid for |
| 1669 | // X0_Pair. Constrain Dst up front: constrainSelectedInstRegOperands below |
| 1670 | // would otherwise infer GPRPair from the s64 type. |
| 1671 | if (!RBI.constrainGenericRegister(Reg: Dst, RC: RISCV::GPRPairNoX0RegClass, MRI&: *MRI)) |
| 1672 | return false; |
| 1673 | MachineInstr &Seq = *BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 1674 | MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dst) |
| 1675 | .addReg(RegNo: Lo) |
| 1676 | .addImm(Val: RISCV::sub_gpr_even) |
| 1677 | .addReg(RegNo: Hi) |
| 1678 | .addImm(Val: RISCV::sub_gpr_odd); |
| 1679 | MI.eraseFromParent(); |
| 1680 | constrainSelectedInstRegOperands(I&: Seq, TII, TRI, RBI); |
| 1681 | return true; |
| 1682 | } |
| 1683 | return false; |
| 1684 | } |
| 1685 | |
| 1686 | bool RISCVInstructionSelector::replacePtrWithInt(MachineInstr &MI, |
| 1687 | unsigned OpIdx) { |
| 1688 | MachineOperand &Op = MI.getOperand(i: OpIdx); |
| 1689 | Register PtrReg = Op.getReg(); |
| 1690 | assert(MRI->getType(PtrReg).isPointer() && "Operand is not a pointer!" ); |
| 1691 | |
| 1692 | const LLT iXLen = LLT::integer(SizeInBits: STI.getXLen()); |
| 1693 | Register IntReg = MRI->createGenericVirtualRegister(Ty: iXLen); |
| 1694 | MRI->setRegBank(Reg: IntReg, RegBank: RBI.getRegBank(ID: RISCV::GPRBRegBankID)); |
| 1695 | MachineInstr *PtrToInt = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 1696 | MCID: TII.get(Opcode: TargetOpcode::G_PTRTOINT), DestReg: IntReg) |
| 1697 | .addReg(RegNo: PtrReg); |
| 1698 | Op.setReg(IntReg); |
| 1699 | return select(MI&: *PtrToInt); |
| 1700 | } |
| 1701 | |
| 1702 | void RISCVInstructionSelector::preISelLower(MachineInstr &MI) { |
| 1703 | switch (MI.getOpcode()) { |
| 1704 | case TargetOpcode::G_PTR_ADD: { |
| 1705 | Register DstReg = MI.getOperand(i: 0).getReg(); |
| 1706 | const LLT iXLen = LLT::integer(SizeInBits: STI.getXLen()); |
| 1707 | |
| 1708 | replacePtrWithInt(MI, OpIdx: 1); |
| 1709 | MI.setDesc(TII.get(Opcode: TargetOpcode::G_ADD)); |
| 1710 | MRI->setType(VReg: DstReg, Ty: iXLen); |
| 1711 | break; |
| 1712 | } |
| 1713 | case TargetOpcode::G_PTRMASK: { |
| 1714 | Register DstReg = MI.getOperand(i: 0).getReg(); |
| 1715 | const LLT iXLen = LLT::integer(SizeInBits: STI.getXLen()); |
| 1716 | replacePtrWithInt(MI, OpIdx: 1); |
| 1717 | MI.setDesc(TII.get(Opcode: TargetOpcode::G_AND)); |
| 1718 | MRI->setType(VReg: DstReg, Ty: iXLen); |
| 1719 | break; |
| 1720 | } |
| 1721 | } |
| 1722 | } |
| 1723 | |
| 1724 | void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB, |
| 1725 | const MachineInstr &MI, |
| 1726 | int OpIdx) const { |
| 1727 | assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && |
| 1728 | "Expected G_CONSTANT" ); |
| 1729 | int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue(); |
| 1730 | MIB.addImm(Val: -CstVal); |
| 1731 | } |
| 1732 | |
| 1733 | void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB, |
| 1734 | const MachineInstr &MI, |
| 1735 | int OpIdx) const { |
| 1736 | assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && |
| 1737 | "Expected G_CONSTANT" ); |
| 1738 | uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue(); |
| 1739 | MIB.addImm(Val: STI.getXLen() - CstVal); |
| 1740 | } |
| 1741 | |
| 1742 | void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB, |
| 1743 | const MachineInstr &MI, |
| 1744 | int OpIdx) const { |
| 1745 | assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && |
| 1746 | "Expected G_CONSTANT" ); |
| 1747 | uint64_t CstVal = MI.getOperand(i: 1).getCImm()->getZExtValue(); |
| 1748 | MIB.addImm(Val: 32 - CstVal); |
| 1749 | } |
| 1750 | |
| 1751 | void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB, |
| 1752 | const MachineInstr &MI, |
| 1753 | int OpIdx) const { |
| 1754 | assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && |
| 1755 | "Expected G_CONSTANT" ); |
| 1756 | int64_t CstVal = MI.getOperand(i: 1).getCImm()->getSExtValue(); |
| 1757 | MIB.addImm(Val: CstVal + 1); |
| 1758 | } |
| 1759 | |
| 1760 | void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB, |
| 1761 | const MachineInstr &MI, |
| 1762 | int OpIdx) const { |
| 1763 | assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && |
| 1764 | "Expected G_CONSTANT" ); |
| 1765 | uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue(); |
| 1766 | MIB.addImm(Val: llvm::countr_zero(Val: C)); |
| 1767 | } |
| 1768 | |
| 1769 | void RISCVInstructionSelector::renderXLenSubTrailingOnes( |
| 1770 | MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const { |
| 1771 | assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && |
| 1772 | "Expected G_CONSTANT" ); |
| 1773 | uint64_t C = MI.getOperand(i: 1).getCImm()->getZExtValue(); |
| 1774 | MIB.addImm(Val: Subtarget->getXLen() - llvm::countr_one(Value: C)); |
| 1775 | } |
| 1776 | |
| 1777 | void RISCVInstructionSelector::renderAddiPairImmSmall(MachineInstrBuilder &MIB, |
| 1778 | const MachineInstr &MI, |
| 1779 | int OpIdx) const { |
| 1780 | assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && |
| 1781 | "Expected G_CONSTANT" ); |
| 1782 | int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue(); |
| 1783 | int64_t Adj = Imm < 0 ? -2048 : 2047; |
| 1784 | MIB.addImm(Val: Imm - Adj); |
| 1785 | } |
| 1786 | |
| 1787 | void RISCVInstructionSelector::renderAddiPairImmLarge(MachineInstrBuilder &MIB, |
| 1788 | const MachineInstr &MI, |
| 1789 | int OpIdx) const { |
| 1790 | assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && |
| 1791 | "Expected G_CONSTANT" ); |
| 1792 | int64_t Imm = MI.getOperand(i: 1).getCImm()->getSExtValue() < 0 ? -2048 : 2047; |
| 1793 | MIB.addImm(Val: Imm); |
| 1794 | } |
| 1795 | |
| 1796 | bool RISCVInstructionSelector::isRegInGprb(Register Reg) const { |
| 1797 | return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::GPRBRegBankID; |
| 1798 | } |
| 1799 | |
| 1800 | bool RISCVInstructionSelector::isRegInFprb(Register Reg) const { |
| 1801 | return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID; |
| 1802 | } |
| 1803 | |
| 1804 | // A G_PTR_ADD result is worth splitting into Hi (materialized) + |
| 1805 | // Lo12 (folded offset) only if every user is a plain scalar load/store |
| 1806 | // using it as the address. Otherwise the ADD is selected on its own with |
| 1807 | // the full materialized constant, making the Hi materialization here redundant. |
| 1808 | bool RISCVInstructionSelector::isWorthFoldingAdd(Register AddResult) const { |
| 1809 | for (const MachineInstr &User : MRI->use_nodbg_instructions(Reg: AddResult)) { |
| 1810 | auto *LdSt = dyn_cast<GLoadStore>(Val: &User); |
| 1811 | if (!LdSt) |
| 1812 | return false; |
| 1813 | // Must be used as the pointer, not the stored value. |
| 1814 | if (LdSt->getPointerReg() != AddResult) |
| 1815 | return false; |
| 1816 | if (isStrongerThanMonotonic(AO: LdSt->getMMO().getSuccessOrdering())) |
| 1817 | return false; |
| 1818 | // Only scalar integer/f16/f32/f64 memory (exclude vectors, f128, ...). |
| 1819 | LLT Ty = MRI->getType(Reg: User.getOperand(i: 0).getReg()); |
| 1820 | if (!Ty.isScalar() || Ty.getSizeInBits() > 64) |
| 1821 | return false; |
| 1822 | } |
| 1823 | return true; |
| 1824 | } |
| 1825 | |
| 1826 | bool RISCVInstructionSelector::selectCopy(MachineInstr &MI) const { |
| 1827 | Register DstReg = MI.getOperand(i: 0).getReg(); |
| 1828 | |
| 1829 | if (DstReg.isPhysical()) |
| 1830 | return true; |
| 1831 | |
| 1832 | const TargetRegisterClass *DstRC = |
| 1833 | TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI); |
| 1834 | |
| 1835 | assert(DstRC && |
| 1836 | "Register class not available for LLT, register bank combination" ); |
| 1837 | |
| 1838 | // No need to constrain SrcReg. It will get constrained when |
| 1839 | // we hit another of its uses or its defs. |
| 1840 | // Copies do not have constraints. |
| 1841 | if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) { |
| 1842 | LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode()) |
| 1843 | << " operand\n" ); |
| 1844 | return false; |
| 1845 | } |
| 1846 | |
| 1847 | MI.setDesc(TII.get(Opcode: RISCV::COPY)); |
| 1848 | return true; |
| 1849 | } |
| 1850 | |
| 1851 | bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI) const { |
| 1852 | assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF); |
| 1853 | |
| 1854 | const Register DstReg = MI.getOperand(i: 0).getReg(); |
| 1855 | const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank( |
| 1856 | Ty: MRI->getType(Reg: DstReg), RB: *RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI), Is64Bit: STI.is64Bit()); |
| 1857 | |
| 1858 | assert(DstRC && |
| 1859 | "Register class not available for LLT, register bank combination" ); |
| 1860 | |
| 1861 | if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) { |
| 1862 | LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode()) |
| 1863 | << " operand\n" ); |
| 1864 | } |
| 1865 | MI.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF)); |
| 1866 | return true; |
| 1867 | } |
| 1868 | |
| 1869 | bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm, |
| 1870 | MachineInstr &MI) const { |
| 1871 | if (Imm == 0) { |
| 1872 | MachineBasicBlock &MBB = *MI.getParent(); |
| 1873 | DebugLoc DL = MI.getDebugLoc(); |
| 1874 | BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg).addReg(RegNo: RISCV::X0); |
| 1875 | RBI.constrainGenericRegister(Reg: DstReg, RC: RISCV::GPRRegClass, MRI&: *MRI); |
| 1876 | return true; |
| 1877 | } |
| 1878 | |
| 1879 | RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: *Subtarget); |
| 1880 | return materializeInstSeq(DstReg, Seq, MI); |
| 1881 | } |
| 1882 | |
| 1883 | bool RISCVInstructionSelector::materializeInstSeq( |
| 1884 | Register DstReg, const RISCVMatInt::InstSeq &Seq, MachineInstr &MI) const { |
| 1885 | assert(!Seq.empty() && "materializeInstSeq requires a non-empty sequence" ); |
| 1886 | |
| 1887 | MachineBasicBlock &MBB = *MI.getParent(); |
| 1888 | DebugLoc DL = MI.getDebugLoc(); |
| 1889 | unsigned NumInsts = Seq.size(); |
| 1890 | Register SrcReg = RISCV::X0; |
| 1891 | |
| 1892 | for (unsigned i = 0; i < NumInsts; i++) { |
| 1893 | Register TmpReg = i < NumInsts - 1 |
| 1894 | ? MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass) |
| 1895 | : DstReg; |
| 1896 | const RISCVMatInt::Inst &I = Seq[i]; |
| 1897 | MachineInstr *Result; |
| 1898 | |
| 1899 | switch (I.getOpndKind()) { |
| 1900 | case RISCVMatInt::Imm: |
| 1901 | Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg) |
| 1902 | .addImm(Val: I.getImm()); |
| 1903 | break; |
| 1904 | case RISCVMatInt::RegX0: |
| 1905 | Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg) |
| 1906 | .addReg(RegNo: SrcReg) |
| 1907 | .addReg(RegNo: RISCV::X0); |
| 1908 | break; |
| 1909 | case RISCVMatInt::RegReg: |
| 1910 | Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg) |
| 1911 | .addReg(RegNo: SrcReg) |
| 1912 | .addReg(RegNo: SrcReg); |
| 1913 | break; |
| 1914 | case RISCVMatInt::RegImm: |
| 1915 | Result = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: I.getOpcode()), DestReg: TmpReg) |
| 1916 | .addReg(RegNo: SrcReg) |
| 1917 | .addImm(Val: I.getImm()); |
| 1918 | break; |
| 1919 | } |
| 1920 | |
| 1921 | constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI); |
| 1922 | |
| 1923 | SrcReg = TmpReg; |
| 1924 | } |
| 1925 | |
| 1926 | return true; |
| 1927 | } |
| 1928 | |
| 1929 | InstructionSelector::ComplexRendererFns |
| 1930 | RISCVInstructionSelector::computeConstAddr(int64_t CVal, bool IsPrefetch, |
| 1931 | Register OrigBase) const { |
| 1932 | // Split the constant into a materialized high part (the base) and |
| 1933 | // a simm12 low part (the offset). For prefetch the low part |
| 1934 | // must additionally be a multiple of 32 (simm12_lsb00000). |
| 1935 | int64_t Lo12 = SignExtend64<12>(x: CVal); |
| 1936 | int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12; |
| 1937 | auto emit = [&](ConstAddrPlan Plan) -> ComplexRendererFns { |
| 1938 | return {{[=](MachineInstrBuilder &MIB) { |
| 1939 | MIB.addReg(RegNo: materializeConstBase(MIB, Plan, OrigBase)); |
| 1940 | }, |
| 1941 | [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Plan.Lo12); }}}; |
| 1942 | }; |
| 1943 | if (!Subtarget->is64Bit() || isInt<32>(x: Hi)) { |
| 1944 | if (IsPrefetch && (Lo12 & 0b11111) != 0) |
| 1945 | return std::nullopt; |
| 1946 | ConstAddrPlan Plan; |
| 1947 | Plan.Lo12 = Lo12; |
| 1948 | if (Hi) { |
| 1949 | Plan.Kind = ConstAddrPlan::LUI; |
| 1950 | Plan.Hi20 = (Hi >> 12) & 0xfffff; |
| 1951 | } |
| 1952 | return emit(std::move(Plan)); |
| 1953 | } |
| 1954 | |
| 1955 | // Otherwise ask constant materialization how it would handle the constant |
| 1956 | // and fold the trailing ADDI into the offset. |
| 1957 | RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: CVal, STI: *Subtarget); |
| 1958 | if (Seq.back().getOpcode() != RISCV::ADDI) |
| 1959 | return std::nullopt; |
| 1960 | Lo12 = Seq.back().getImm(); |
| 1961 | if (IsPrefetch && (Lo12 & 0b11111) != 0) |
| 1962 | return std::nullopt; |
| 1963 | Seq.pop_back(); |
| 1964 | if (Seq.empty()) |
| 1965 | return std::nullopt; |
| 1966 | ConstAddrPlan Plan; |
| 1967 | Plan.Kind = ConstAddrPlan::InstSeq; |
| 1968 | Plan.Seq = std::move(Seq); |
| 1969 | Plan.Lo12 = Lo12; |
| 1970 | return emit(std::move(Plan)); |
| 1971 | } |
| 1972 | |
| 1973 | Register |
| 1974 | RISCVInstructionSelector::materializeConstBase(MachineInstrBuilder &MIB, |
| 1975 | const ConstAddrPlan &Plan, |
| 1976 | Register OrigBase) const { |
| 1977 | MachineBasicBlock &MBB = *MIB->getParent(); |
| 1978 | DebugLoc DL = MIB->getDebugLoc(); |
| 1979 | MachineInstr &InsertPt = *MIB.getInstr(); |
| 1980 | |
| 1981 | Register HiReg = RISCV::X0; |
| 1982 | switch (Plan.Kind) { |
| 1983 | case ConstAddrPlan::X0: |
| 1984 | break; |
| 1985 | case ConstAddrPlan::LUI: { |
| 1986 | HiReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 1987 | MachineInstr *LUI = BuildMI(BB&: MBB, I&: InsertPt, MIMD: DL, MCID: TII.get(Opcode: RISCV::LUI), DestReg: HiReg) |
| 1988 | .addImm(Val: Plan.Hi20); |
| 1989 | constrainSelectedInstRegOperands(I&: *LUI, TII, TRI, RBI); |
| 1990 | break; |
| 1991 | } |
| 1992 | case ConstAddrPlan::InstSeq: { |
| 1993 | HiReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 1994 | materializeInstSeq(DstReg: HiReg, Seq: Plan.Seq, MI&: InsertPt); |
| 1995 | break; |
| 1996 | } |
| 1997 | } |
| 1998 | |
| 1999 | // For G_PTR_ADD + large constant, add the original base to the materialized |
| 2000 | // high part. |
| 2001 | if (OrigBase.isValid() && HiReg != RISCV::X0) { |
| 2002 | Register BaseReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 2003 | MachineInstr *Add = BuildMI(BB&: MBB, I&: InsertPt, MIMD: DL, MCID: TII.get(Opcode: RISCV::ADD), DestReg: BaseReg) |
| 2004 | .addReg(RegNo: OrigBase) |
| 2005 | .addReg(RegNo: HiReg); |
| 2006 | constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI); |
| 2007 | return BaseReg; |
| 2008 | } |
| 2009 | return OrigBase.isValid() ? OrigBase : HiReg; |
| 2010 | } |
| 2011 | |
| 2012 | bool RISCVInstructionSelector::selectAddr(MachineInstr &MI, bool IsLocal, |
| 2013 | bool IsExternWeak) const { |
| 2014 | assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE || |
| 2015 | MI.getOpcode() == TargetOpcode::G_JUMP_TABLE || |
| 2016 | MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) && |
| 2017 | "Unexpected opcode" ); |
| 2018 | |
| 2019 | const MachineOperand &DispMO = MI.getOperand(i: 1); |
| 2020 | |
| 2021 | Register DefReg = MI.getOperand(i: 0).getReg(); |
| 2022 | const LLT DefTy = MRI->getType(Reg: DefReg); |
| 2023 | |
| 2024 | // When HWASAN is used and tagging of global variables is enabled |
| 2025 | // they should be accessed via the GOT, since the tagged address of a global |
| 2026 | // is incompatible with existing code models. This also applies to non-pic |
| 2027 | // mode. |
| 2028 | if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) { |
| 2029 | if (IsLocal && !Subtarget->allowTaggedGlobals()) { |
| 2030 | // Use PC-relative addressing to access the symbol. This generates the |
| 2031 | // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym)) |
| 2032 | // %pcrel_lo(auipc)). |
| 2033 | MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA)); |
| 2034 | constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI); |
| 2035 | return true; |
| 2036 | } |
| 2037 | |
| 2038 | // Use PC-relative addressing to access the GOT for this symbol, then |
| 2039 | // load the address from the GOT. This generates the pattern (PseudoLGA |
| 2040 | // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym)) |
| 2041 | // %pcrel_lo(auipc))). |
| 2042 | MachineFunction &MF = *MI.getParent()->getParent(); |
| 2043 | MachineMemOperand *MemOp = MF.getMachineMemOperand( |
| 2044 | PtrInfo: MachinePointerInfo::getGOT(MF), |
| 2045 | F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable | |
| 2046 | MachineMemOperand::MOInvariant, |
| 2047 | MemTy: DefTy, BaseAlignment: Align(DefTy.getSizeInBits() / 8)); |
| 2048 | |
| 2049 | MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 2050 | MCID: TII.get(Opcode: RISCV::PseudoLGA), DestReg: DefReg) |
| 2051 | .addDisp(Disp: DispMO, off: 0) |
| 2052 | .addMemOperand(MMO: MemOp); |
| 2053 | |
| 2054 | constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI); |
| 2055 | |
| 2056 | MI.eraseFromParent(); |
| 2057 | return true; |
| 2058 | } |
| 2059 | |
| 2060 | switch (TM.getCodeModel()) { |
| 2061 | default: { |
| 2062 | reportGISelFailure(MF&: *MF, MORE&: *MORE, PassName: getName(), |
| 2063 | Msg: "Unsupported code model for lowering" , MI); |
| 2064 | return false; |
| 2065 | } |
| 2066 | case CodeModel::Small: { |
| 2067 | // Must lie within a single 2 GiB address range and must lie between |
| 2068 | // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi |
| 2069 | // (lui %hi(sym)) %lo(sym)). |
| 2070 | Register AddrHiDest = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 2071 | MachineInstr *AddrHi = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 2072 | MCID: TII.get(Opcode: RISCV::LUI), DestReg: AddrHiDest) |
| 2073 | .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_HI); |
| 2074 | |
| 2075 | constrainSelectedInstRegOperands(I&: *AddrHi, TII, TRI, RBI); |
| 2076 | |
| 2077 | MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 2078 | MCID: TII.get(Opcode: RISCV::ADDI), DestReg: DefReg) |
| 2079 | .addReg(RegNo: AddrHiDest) |
| 2080 | .addDisp(Disp: DispMO, off: 0, TargetFlags: RISCVII::MO_LO); |
| 2081 | |
| 2082 | constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI); |
| 2083 | |
| 2084 | MI.eraseFromParent(); |
| 2085 | return true; |
| 2086 | } |
| 2087 | case CodeModel::Medium: |
| 2088 | // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo |
| 2089 | // relocation needs to reference a label that points to the auipc |
| 2090 | // instruction itself, not the global. This cannot be done inside the |
| 2091 | // instruction selector. |
| 2092 | if (IsExternWeak) { |
| 2093 | // An extern weak symbol may be undefined, i.e. have value 0, which may |
| 2094 | // not be within 2GiB of PC, so use GOT-indirect addressing to access the |
| 2095 | // symbol. This generates the pattern (PseudoLGA sym), which expands to |
| 2096 | // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))). |
| 2097 | MachineFunction &MF = *MI.getParent()->getParent(); |
| 2098 | MachineMemOperand *MemOp = MF.getMachineMemOperand( |
| 2099 | PtrInfo: MachinePointerInfo::getGOT(MF), |
| 2100 | F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable | |
| 2101 | MachineMemOperand::MOInvariant, |
| 2102 | MemTy: DefTy, BaseAlignment: Align(DefTy.getSizeInBits() / 8)); |
| 2103 | |
| 2104 | MachineInstr *Result = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 2105 | MCID: TII.get(Opcode: RISCV::PseudoLGA), DestReg: DefReg) |
| 2106 | .addDisp(Disp: DispMO, off: 0) |
| 2107 | .addMemOperand(MMO: MemOp); |
| 2108 | |
| 2109 | constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI); |
| 2110 | |
| 2111 | MI.eraseFromParent(); |
| 2112 | return true; |
| 2113 | } |
| 2114 | |
| 2115 | // Generate a sequence for accessing addresses within any 2GiB range |
| 2116 | // within the address space. This generates the pattern (PseudoLLA sym), |
| 2117 | // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)). |
| 2118 | MI.setDesc(TII.get(Opcode: RISCV::PseudoLLA)); |
| 2119 | constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI); |
| 2120 | return true; |
| 2121 | } |
| 2122 | |
| 2123 | return false; |
| 2124 | } |
| 2125 | |
| 2126 | bool RISCVInstructionSelector::selectSelect(MachineInstr &MI) const { |
| 2127 | auto &SelectMI = cast<GSelect>(Val&: MI); |
| 2128 | |
| 2129 | Register LHS, RHS; |
| 2130 | RISCVCC::CondCode CC; |
| 2131 | getOperandsForBranch(CondReg: SelectMI.getCondReg(), CC, LHS, RHS, MRI&: *MRI); |
| 2132 | |
| 2133 | Register DstReg = SelectMI.getReg(Idx: 0); |
| 2134 | |
| 2135 | unsigned Opc = RISCV::Select_GPR_Using_CC_GPR; |
| 2136 | if (RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI)->getID() == RISCV::FPRBRegBankID) { |
| 2137 | unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits(); |
| 2138 | Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR |
| 2139 | : RISCV::Select_FPR64_Using_CC_GPR; |
| 2140 | } |
| 2141 | |
| 2142 | MachineInstr *Result = |
| 2143 | BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: Opc)) |
| 2144 | .addDef(RegNo: DstReg) |
| 2145 | .addReg(RegNo: LHS) |
| 2146 | .addReg(RegNo: RHS) |
| 2147 | .addImm(Val: CC) |
| 2148 | .addReg(RegNo: SelectMI.getTrueReg()) |
| 2149 | .addReg(RegNo: SelectMI.getFalseReg()); |
| 2150 | MI.eraseFromParent(); |
| 2151 | constrainSelectedInstRegOperands(I&: *Result, TII, TRI, RBI); |
| 2152 | return true; |
| 2153 | } |
| 2154 | |
| 2155 | // Convert an FCMP predicate to one of the supported F or D instructions. |
| 2156 | static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) { |
| 2157 | assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size" ); |
| 2158 | switch (Pred) { |
| 2159 | default: |
| 2160 | llvm_unreachable("Unsupported predicate" ); |
| 2161 | case CmpInst::FCMP_OLT: |
| 2162 | return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D; |
| 2163 | case CmpInst::FCMP_OLE: |
| 2164 | return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D; |
| 2165 | case CmpInst::FCMP_OEQ: |
| 2166 | return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D; |
| 2167 | } |
| 2168 | } |
| 2169 | |
| 2170 | // Try legalizing an FCMP by swapping or inverting the predicate to one that |
| 2171 | // is supported. |
| 2172 | static bool legalizeFCmpPredicate(Register &LHS, Register &RHS, |
| 2173 | CmpInst::Predicate &Pred, bool &NeedInvert) { |
| 2174 | auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) { |
| 2175 | return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE || |
| 2176 | Pred == CmpInst::FCMP_OEQ; |
| 2177 | }; |
| 2178 | |
| 2179 | assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?" ); |
| 2180 | |
| 2181 | CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(pred: Pred); |
| 2182 | if (isLegalFCmpPredicate(InvPred)) { |
| 2183 | Pred = InvPred; |
| 2184 | std::swap(a&: LHS, b&: RHS); |
| 2185 | return true; |
| 2186 | } |
| 2187 | |
| 2188 | InvPred = CmpInst::getInversePredicate(pred: Pred); |
| 2189 | NeedInvert = true; |
| 2190 | if (isLegalFCmpPredicate(InvPred)) { |
| 2191 | Pred = InvPred; |
| 2192 | return true; |
| 2193 | } |
| 2194 | InvPred = CmpInst::getSwappedPredicate(pred: InvPred); |
| 2195 | if (isLegalFCmpPredicate(InvPred)) { |
| 2196 | Pred = InvPred; |
| 2197 | std::swap(a&: LHS, b&: RHS); |
| 2198 | return true; |
| 2199 | } |
| 2200 | |
| 2201 | return false; |
| 2202 | } |
| 2203 | |
| 2204 | // Emit a sequence of instructions to compare LHS and RHS using Pred. Return |
| 2205 | // the result in DstReg. |
| 2206 | // FIXME: Maybe we should expand this earlier. |
| 2207 | bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI) const { |
| 2208 | auto &CmpMI = cast<GFCmp>(Val&: MI); |
| 2209 | CmpInst::Predicate Pred = CmpMI.getCond(); |
| 2210 | |
| 2211 | Register DstReg = CmpMI.getReg(Idx: 0); |
| 2212 | Register LHS = CmpMI.getLHSReg(); |
| 2213 | Register RHS = CmpMI.getRHSReg(); |
| 2214 | |
| 2215 | unsigned Size = MRI->getType(Reg: LHS).getSizeInBits(); |
| 2216 | assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size" ); |
| 2217 | |
| 2218 | Register TmpReg = DstReg; |
| 2219 | |
| 2220 | bool NeedInvert = false; |
| 2221 | // First try swapping operands or inverting. |
| 2222 | if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) { |
| 2223 | if (NeedInvert) |
| 2224 | TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 2225 | MachineInstr *Cmp = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 2226 | MCID: TII.get(Opcode: getFCmpOpcode(Pred, Size)), DestReg: TmpReg) |
| 2227 | .setMIFlag(MachineInstr::NoFPExcept) |
| 2228 | .addReg(RegNo: LHS) |
| 2229 | .addReg(RegNo: RHS); |
| 2230 | constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI); |
| 2231 | } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) { |
| 2232 | // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS)) |
| 2233 | NeedInvert = Pred == CmpInst::FCMP_UEQ; |
| 2234 | Register Cmp1Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 2235 | MachineInstr *Cmp1 = |
| 2236 | BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 2237 | MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size)), DestReg: Cmp1Reg) |
| 2238 | .setMIFlag(MachineInstr::NoFPExcept) |
| 2239 | .addReg(RegNo: LHS) |
| 2240 | .addReg(RegNo: RHS); |
| 2241 | constrainSelectedInstRegOperands(I&: *Cmp1, TII, TRI, RBI); |
| 2242 | Register Cmp2Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 2243 | MachineInstr *Cmp2 = |
| 2244 | BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 2245 | MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OLT, Size)), DestReg: Cmp2Reg) |
| 2246 | .setMIFlag(MachineInstr::NoFPExcept) |
| 2247 | .addReg(RegNo: RHS) |
| 2248 | .addReg(RegNo: LHS); |
| 2249 | constrainSelectedInstRegOperands(I&: *Cmp2, TII, TRI, RBI); |
| 2250 | if (NeedInvert) |
| 2251 | TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 2252 | MachineInstr *Or = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 2253 | MCID: TII.get(Opcode: RISCV::OR), DestReg: TmpReg) |
| 2254 | .addReg(RegNo: Cmp1Reg) |
| 2255 | .addReg(RegNo: Cmp2Reg); |
| 2256 | constrainSelectedInstRegOperands(I&: *Or, TII, TRI, RBI); |
| 2257 | } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) { |
| 2258 | // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS)) |
| 2259 | // If LHS and RHS are the same, a single FEQ suffices. |
| 2260 | NeedInvert = Pred == CmpInst::FCMP_UNO; |
| 2261 | if (NeedInvert) |
| 2262 | TmpReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 2263 | if (LHS == RHS) { |
| 2264 | MachineInstr *Cmp = |
| 2265 | BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 2266 | MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: TmpReg) |
| 2267 | .setMIFlag(MachineInstr::NoFPExcept) |
| 2268 | .addReg(RegNo: LHS) |
| 2269 | .addReg(RegNo: LHS); |
| 2270 | constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI); |
| 2271 | } else { |
| 2272 | Register Cmp1Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 2273 | MachineInstr *Cmp1 = |
| 2274 | BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 2275 | MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: Cmp1Reg) |
| 2276 | .setMIFlag(MachineInstr::NoFPExcept) |
| 2277 | .addReg(RegNo: LHS) |
| 2278 | .addReg(RegNo: LHS); |
| 2279 | constrainSelectedInstRegOperands(I&: *Cmp1, TII, TRI, RBI); |
| 2280 | Register Cmp2Reg = MRI->createVirtualRegister(RegClass: &RISCV::GPRRegClass); |
| 2281 | MachineInstr *Cmp2 = |
| 2282 | BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 2283 | MCID: TII.get(Opcode: getFCmpOpcode(Pred: CmpInst::FCMP_OEQ, Size)), DestReg: Cmp2Reg) |
| 2284 | .setMIFlag(MachineInstr::NoFPExcept) |
| 2285 | .addReg(RegNo: RHS) |
| 2286 | .addReg(RegNo: RHS); |
| 2287 | constrainSelectedInstRegOperands(I&: *Cmp2, TII, TRI, RBI); |
| 2288 | MachineInstr *And = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 2289 | MCID: TII.get(Opcode: RISCV::AND), DestReg: TmpReg) |
| 2290 | .addReg(RegNo: Cmp1Reg) |
| 2291 | .addReg(RegNo: Cmp2Reg); |
| 2292 | constrainSelectedInstRegOperands(I&: *And, TII, TRI, RBI); |
| 2293 | } |
| 2294 | } else |
| 2295 | llvm_unreachable("Unhandled predicate" ); |
| 2296 | |
| 2297 | // Emit an XORI to invert the result if needed. |
| 2298 | if (NeedInvert) { |
| 2299 | MachineInstr *Xor = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), |
| 2300 | MCID: TII.get(Opcode: RISCV::XORI), DestReg: DstReg) |
| 2301 | .addReg(RegNo: TmpReg) |
| 2302 | .addImm(Val: 1); |
| 2303 | constrainSelectedInstRegOperands(I&: *Xor, TII, TRI, RBI); |
| 2304 | } |
| 2305 | |
| 2306 | MI.eraseFromParent(); |
| 2307 | return true; |
| 2308 | } |
| 2309 | |
| 2310 | void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering, |
| 2311 | SyncScope::ID FenceSSID, |
| 2312 | MachineInstr &MI) const { |
| 2313 | MachineBasicBlock &MBB = *MI.getParent(); |
| 2314 | DebugLoc DL = MI.getDebugLoc(); |
| 2315 | |
| 2316 | if (STI.hasStdExtZtso()) { |
| 2317 | // The only fence that needs an instruction is a sequentially-consistent |
| 2318 | // cross-thread fence. |
| 2319 | if (FenceOrdering == AtomicOrdering::SequentiallyConsistent && |
| 2320 | FenceSSID == SyncScope::System) { |
| 2321 | // fence rw, rw |
| 2322 | BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE)) |
| 2323 | .addImm(Val: RISCVFenceField::R | RISCVFenceField::W) |
| 2324 | .addImm(Val: RISCVFenceField::R | RISCVFenceField::W); |
| 2325 | return; |
| 2326 | } |
| 2327 | |
| 2328 | // MEMBARRIER is a compiler barrier; it codegens to a no-op. |
| 2329 | BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER)); |
| 2330 | return; |
| 2331 | } |
| 2332 | |
| 2333 | // singlethread fences only synchronize with signal handlers on the same |
| 2334 | // thread and thus only need to preserve instruction order, not actually |
| 2335 | // enforce memory ordering. |
| 2336 | if (FenceSSID == SyncScope::SingleThread) { |
| 2337 | BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::MEMBARRIER)); |
| 2338 | return; |
| 2339 | } |
| 2340 | |
| 2341 | // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set |
| 2342 | // Manual: Volume I. |
| 2343 | unsigned Pred, Succ; |
| 2344 | switch (FenceOrdering) { |
| 2345 | default: |
| 2346 | llvm_unreachable("Unexpected ordering" ); |
| 2347 | case AtomicOrdering::AcquireRelease: |
| 2348 | // fence acq_rel -> fence.tso |
| 2349 | BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE_TSO)); |
| 2350 | return; |
| 2351 | case AtomicOrdering::Acquire: |
| 2352 | // fence acquire -> fence r, rw |
| 2353 | Pred = RISCVFenceField::R; |
| 2354 | Succ = RISCVFenceField::R | RISCVFenceField::W; |
| 2355 | break; |
| 2356 | case AtomicOrdering::Release: |
| 2357 | // fence release -> fence rw, w |
| 2358 | Pred = RISCVFenceField::R | RISCVFenceField::W; |
| 2359 | Succ = RISCVFenceField::W; |
| 2360 | break; |
| 2361 | case AtomicOrdering::SequentiallyConsistent: |
| 2362 | // fence seq_cst -> fence rw, rw |
| 2363 | Pred = RISCVFenceField::R | RISCVFenceField::W; |
| 2364 | Succ = RISCVFenceField::R | RISCVFenceField::W; |
| 2365 | break; |
| 2366 | } |
| 2367 | BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::FENCE)).addImm(Val: Pred).addImm(Val: Succ); |
| 2368 | } |
| 2369 | |
| 2370 | namespace llvm { |
| 2371 | InstructionSelector * |
| 2372 | createRISCVInstructionSelector(const RISCVTargetMachine &TM, |
| 2373 | const RISCVSubtarget &Subtarget, |
| 2374 | const RISCVRegisterBankInfo &RBI) { |
| 2375 | return new RISCVInstructionSelector(TM, Subtarget, RBI); |
| 2376 | } |
| 2377 | } // end namespace llvm |
| 2378 | |