| 1 | //===----- RISCVMergeBaseOffset.cpp - Optimise address calculations ------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Merge the offset of address calculation into the offset field |
| 10 | // of instructions in a global address lowering sequence. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "RISCV.h" |
| 15 | #include "RISCVSubtarget.h" |
| 16 | #include "llvm/CodeGen/MachineFunctionAnalysisManager.h" |
| 17 | #include "llvm/CodeGen/MachineFunctionPass.h" |
| 18 | #include "llvm/CodeGen/MachinePassManager.h" |
| 19 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
| 20 | #include "llvm/CodeGen/Passes.h" |
| 21 | #include "llvm/IR/Analysis.h" |
| 22 | #include "llvm/IR/DebugInfoMetadata.h" |
| 23 | #include "llvm/MC/TargetRegistry.h" |
| 24 | #include "llvm/Support/Debug.h" |
| 25 | #include "llvm/Target/TargetOptions.h" |
| 26 | #include <optional> |
| 27 | using namespace llvm; |
| 28 | |
| 29 | #define DEBUG_TYPE "riscv-merge-base-offset" |
| 30 | #define RISCV_MERGE_BASE_OFFSET_NAME "RISC-V Merge Base Offset" |
| 31 | namespace { |
| 32 | |
| 33 | class RISCVMergeBaseOffsetLegacy : public MachineFunctionPass { |
| 34 | public: |
| 35 | static char ID; |
| 36 | bool runOnMachineFunction(MachineFunction &MF) override; |
| 37 | |
| 38 | RISCVMergeBaseOffsetLegacy() : MachineFunctionPass(ID) {} |
| 39 | |
| 40 | MachineFunctionProperties getRequiredProperties() const override { |
| 41 | return MachineFunctionProperties().setIsSSA(); |
| 42 | } |
| 43 | |
| 44 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
| 45 | AU.setPreservesCFG(); |
| 46 | MachineFunctionPass::getAnalysisUsage(AU); |
| 47 | } |
| 48 | |
| 49 | StringRef getPassName() const override { |
| 50 | return RISCV_MERGE_BASE_OFFSET_NAME; |
| 51 | } |
| 52 | }; |
| 53 | } // end anonymous namespace |
| 54 | |
| 55 | char RISCVMergeBaseOffsetLegacy::ID = 0; |
| 56 | INITIALIZE_PASS(RISCVMergeBaseOffsetLegacy, DEBUG_TYPE, |
| 57 | RISCV_MERGE_BASE_OFFSET_NAME, false, false) |
| 58 | |
| 59 | // Detect either of the patterns: |
| 60 | // |
| 61 | // 1. (medlow pattern): |
| 62 | // a. lui vreg1, %hi(s) |
| 63 | // addi vreg2, vreg1, %lo(s) |
| 64 | // |
| 65 | // b. qc.e.li vreg1, s |
| 66 | // |
| 67 | // 2. (medany pattern): |
| 68 | // .Lpcrel_hi1: |
| 69 | // auipc vreg1, %pcrel_hi(s) |
| 70 | // addi vreg2, vreg1, %pcrel_lo(.Lpcrel_hi1) |
| 71 | // |
| 72 | // The pattern is only accepted if: |
| 73 | // 1) The first instruction has only one use, which is the ADDI. |
| 74 | // 2) The address operands have the appropriate type, reflecting the |
| 75 | // lowering of a global address or constant pool using medlow or medany. |
| 76 | // 3) The offset value in the Global Address or Constant Pool is 0. |
| 77 | static bool detectFoldable(MachineInstr &Hi, MachineInstr *&Lo, |
| 78 | MachineRegisterInfo *MRI) { |
| 79 | auto HiOpc = Hi.getOpcode(); |
| 80 | if (HiOpc != RISCV::LUI && HiOpc != RISCV::AUIPC && |
| 81 | HiOpc != RISCV::PseudoMovAddr && HiOpc != RISCV::QC_E_LI) |
| 82 | return false; |
| 83 | |
| 84 | const MachineOperand &HiOp1 = Hi.getOperand(i: 1); |
| 85 | unsigned ExpectedFlags = HiOpc == RISCV::AUIPC ? RISCVII::MO_PCREL_HI |
| 86 | : HiOpc == RISCV::QC_E_LI ? RISCVII::MO_None |
| 87 | : RISCVII::MO_HI; |
| 88 | if (HiOp1.getTargetFlags() != ExpectedFlags) |
| 89 | return false; |
| 90 | |
| 91 | if (!(HiOp1.isGlobal() || HiOp1.isCPI() || HiOp1.isBlockAddress()) || |
| 92 | HiOp1.getOffset() != 0) |
| 93 | return false; |
| 94 | |
| 95 | if (HiOpc == RISCV::PseudoMovAddr || HiOpc == RISCV::QC_E_LI) { |
| 96 | // Most of the code should handle it correctly without modification by |
| 97 | // setting Lo and Hi both point to PseudoMovAddr/QC_E_LI |
| 98 | Lo = &Hi; |
| 99 | } else { |
| 100 | Register HiDestReg = Hi.getOperand(i: 0).getReg(); |
| 101 | if (!MRI->hasOneUse(RegNo: HiDestReg)) |
| 102 | return false; |
| 103 | |
| 104 | Lo = &*MRI->use_instr_begin(RegNo: HiDestReg); |
| 105 | if (Lo->getOpcode() != RISCV::ADDI) |
| 106 | return false; |
| 107 | } |
| 108 | |
| 109 | if (HiOpc != RISCV::QC_E_LI) { |
| 110 | const MachineOperand &LoOp2 = Lo->getOperand(i: 2); |
| 111 | if (HiOpc == RISCV::LUI || HiOpc == RISCV::PseudoMovAddr) { |
| 112 | if (LoOp2.getTargetFlags() != RISCVII::MO_LO || |
| 113 | !(LoOp2.isGlobal() || LoOp2.isCPI() || LoOp2.isBlockAddress()) || |
| 114 | LoOp2.getOffset() != 0) |
| 115 | return false; |
| 116 | } else { |
| 117 | assert(HiOpc == RISCV::AUIPC); |
| 118 | if (LoOp2.getTargetFlags() != RISCVII::MO_PCREL_LO || |
| 119 | LoOp2.getType() != MachineOperand::MO_MCSymbol) |
| 120 | return false; |
| 121 | } |
| 122 | } |
| 123 | |
| 124 | if (HiOp1.isGlobal()) { |
| 125 | LLVM_DEBUG(dbgs() << " Found lowered global address: " |
| 126 | << *HiOp1.getGlobal() << "\n" ); |
| 127 | } else if (HiOp1.isBlockAddress()) { |
| 128 | LLVM_DEBUG(dbgs() << " Found lowered basic address: " |
| 129 | << *HiOp1.getBlockAddress() << "\n" ); |
| 130 | } else if (HiOp1.isCPI()) { |
| 131 | LLVM_DEBUG(dbgs() << " Found lowered constant pool: " << HiOp1.getIndex() |
| 132 | << "\n" ); |
| 133 | } |
| 134 | |
| 135 | return true; |
| 136 | } |
| 137 | |
| 138 | // Update the offset in Hi and Lo instructions. |
| 139 | // Delete the tail instruction and update all the uses to use the |
| 140 | // output from Lo. |
| 141 | static bool foldOffset(MachineInstr &Hi, MachineInstr &Lo, MachineInstr &Tail, |
| 142 | int64_t Offset, MachineRegisterInfo *MRI) { |
| 143 | assert(isInt<32>(Offset) && "Unexpected offset" ); |
| 144 | |
| 145 | // If Hi is an AUIPC, don't fold the offset if it is outside the bounds of |
| 146 | // the global object. The object may be within 2GB of the PC, but addresses |
| 147 | // outside of the object might not be. |
| 148 | auto HiOpc = Hi.getOpcode(); |
| 149 | if (HiOpc == RISCV::AUIPC && Hi.getOperand(i: 1).isGlobal()) { |
| 150 | const GlobalValue *GV = Hi.getOperand(i: 1).getGlobal(); |
| 151 | Type *Ty = GV->getValueType(); |
| 152 | if (!Ty->isSized() || Offset < 0 || |
| 153 | (uint64_t)Offset > GV->getDataLayout().getTypeAllocSize(Ty)) |
| 154 | return false; |
| 155 | } |
| 156 | |
| 157 | // Salvage the debug users of Lo's result by subtracting the offset again. |
| 158 | SmallVector<uint64_t, 3> UndoOffset; |
| 159 | DIExpression::appendOffset(Ops&: UndoOffset, Offset: -Offset); |
| 160 | for (MachineOperand &MO : MRI->use_operands(Reg: Lo.getOperand(i: 0).getReg())) { |
| 161 | MachineInstr &DbgMI = *MO.getParent(); |
| 162 | if (!DbgMI.isDebugValue()) |
| 163 | continue; |
| 164 | DbgMI.getDebugExpressionOp().setMetadata( |
| 165 | DIExpression::appendOpsToArg(Expr: DbgMI.getDebugExpression(), Ops: UndoOffset, |
| 166 | ArgNo: DbgMI.getDebugOperandIndex(Op: &MO))); |
| 167 | } |
| 168 | |
| 169 | // Put the offset back in Hi and the Lo |
| 170 | Hi.getOperand(i: 1).setOffset(Offset); |
| 171 | if (Hi.getOpcode() != RISCV::AUIPC && Hi.getOpcode() != RISCV::QC_E_LI) |
| 172 | Lo.getOperand(i: 2).setOffset(Offset); |
| 173 | // Delete the tail instruction. |
| 174 | Register LoOp0Reg = Lo.getOperand(i: 0).getReg(); |
| 175 | Register TailOp0Reg = Tail.getOperand(i: 0).getReg(); |
| 176 | MRI->constrainRegClass(Reg: LoOp0Reg, RC: MRI->getRegClass(Reg: TailOp0Reg)); |
| 177 | MRI->replaceRegWith(FromReg: TailOp0Reg, ToReg: LoOp0Reg); |
| 178 | Tail.eraseFromParent(); |
| 179 | LLVM_DEBUG(dbgs() << " Merged offset " << Offset << " into base.\n" |
| 180 | << " " << Hi << " " << Lo;); |
| 181 | return true; |
| 182 | } |
| 183 | |
| 184 | // Detect patterns for large offsets that are passed into an ADD instruction. |
| 185 | // If the pattern is found, updates the offset in Hi and Lo instructions |
| 186 | // and deletes TailAdd and the instructions that produced the offset. |
| 187 | // |
| 188 | // Base address lowering is of the form: |
| 189 | // Hi: lui vreg1, %hi(s) |
| 190 | // Lo: addi vreg2, vreg1, %lo(s) |
| 191 | // / \ |
| 192 | // / \ |
| 193 | // / \ |
| 194 | // / The large offset can be of two forms: \ |
| 195 | // 1) Offset that has non zero bits in lower 2) Offset that has non zero |
| 196 | // 12 bits and upper 20 bits bits in upper 20 bits only |
| 197 | // OffseLUI: lui vreg3, 4 |
| 198 | // OffsetTail: addi voff, vreg3, 188 OffsetTail: lui voff, 128 |
| 199 | // \ / |
| 200 | // \ / |
| 201 | // \ / |
| 202 | // \ / |
| 203 | // TailAdd: add vreg4, vreg2, voff |
| 204 | static bool foldLargeOffset(MachineInstr &Hi, MachineInstr &Lo, |
| 205 | MachineInstr &TailAdd, Register GAReg, |
| 206 | MachineRegisterInfo *MRI, |
| 207 | const RISCVSubtarget *ST) { |
| 208 | assert((TailAdd.getOpcode() == RISCV::ADD) && "Expected ADD instruction!" ); |
| 209 | Register Rs = TailAdd.getOperand(i: 1).getReg(); |
| 210 | Register Rt = TailAdd.getOperand(i: 2).getReg(); |
| 211 | Register Reg = Rs == GAReg ? Rt : Rs; |
| 212 | |
| 213 | // Can't fold if the register has more than one use. |
| 214 | if (!Reg.isVirtual() || !MRI->hasOneNonDBGUse(RegNo: Reg)) |
| 215 | return false; |
| 216 | // This can point to an ADDI(W) or a LUI: |
| 217 | MachineInstr &OffsetTail = *MRI->getVRegDef(Reg); |
| 218 | auto OffsetTailOpc = OffsetTail.getOpcode(); |
| 219 | if (OffsetTailOpc == RISCV::ADDI || OffsetTailOpc == RISCV::ADDIW) { |
| 220 | // The offset value has non zero bits in both %hi and %lo parts. |
| 221 | // Detect an ADDI that feeds from a LUI instruction. |
| 222 | MachineOperand &AddiImmOp = OffsetTail.getOperand(i: 2); |
| 223 | if (AddiImmOp.getTargetFlags() != RISCVII::MO_None) |
| 224 | return false; |
| 225 | // Make sure this isn't something weird like a frame index. |
| 226 | if (!OffsetTail.getOperand(i: 1).isReg()) |
| 227 | return false; |
| 228 | Register AddiReg = OffsetTail.getOperand(i: 1).getReg(); |
| 229 | int64_t OffLo = AddiImmOp.getImm(); |
| 230 | |
| 231 | // Handle rs1 of ADDI is X0. |
| 232 | if (AddiReg == RISCV::X0) { |
| 233 | LLVM_DEBUG(dbgs() << " Offset Instrs: " << OffsetTail); |
| 234 | if (!foldOffset(Hi, Lo, Tail&: TailAdd, Offset: OffLo, MRI)) |
| 235 | return false; |
| 236 | OffsetTail.eraseFromParent(); |
| 237 | return true; |
| 238 | } |
| 239 | |
| 240 | MachineInstr &OffsetLui = *MRI->getVRegDef(Reg: AddiReg); |
| 241 | MachineOperand &LuiImmOp = OffsetLui.getOperand(i: 1); |
| 242 | if (OffsetLui.getOpcode() != RISCV::LUI || |
| 243 | LuiImmOp.getTargetFlags() != RISCVII::MO_None || |
| 244 | !MRI->hasOneNonDBGUse(RegNo: OffsetLui.getOperand(i: 0).getReg())) |
| 245 | return false; |
| 246 | int64_t Offset = SignExtend64<32>(x: LuiImmOp.getImm() << 12); |
| 247 | Offset += OffLo; |
| 248 | // RV32 ignores the upper 32 bits. ADDIW sign extends the result. |
| 249 | if (!ST->is64Bit() || OffsetTailOpc == RISCV::ADDIW) |
| 250 | Offset = SignExtend64<32>(x: Offset); |
| 251 | // We can only fold simm32 offsets. |
| 252 | if (!isInt<32>(x: Offset)) |
| 253 | return false; |
| 254 | LLVM_DEBUG(dbgs() << " Offset Instrs: " << OffsetTail |
| 255 | << " " << OffsetLui); |
| 256 | if (!foldOffset(Hi, Lo, Tail&: TailAdd, Offset, MRI)) |
| 257 | return false; |
| 258 | OffsetTail.eraseFromParent(); |
| 259 | OffsetLui.eraseFromParent(); |
| 260 | return true; |
| 261 | } else if (OffsetTailOpc == RISCV::LUI) { |
| 262 | // The offset value has all zero bits in the lower 12 bits. Only LUI |
| 263 | // exists. |
| 264 | LLVM_DEBUG(dbgs() << " Offset Instr: " << OffsetTail); |
| 265 | int64_t Offset = SignExtend64<32>(x: OffsetTail.getOperand(i: 1).getImm() << 12); |
| 266 | if (!foldOffset(Hi, Lo, Tail&: TailAdd, Offset, MRI)) |
| 267 | return false; |
| 268 | OffsetTail.eraseFromParent(); |
| 269 | return true; |
| 270 | } |
| 271 | return false; |
| 272 | } |
| 273 | |
| 274 | // Detect patterns for offsets that are passed into a SHXADD instruction. |
| 275 | // The offset has 1, 2, or 3 trailing zeros and fits in simm13, simm14, simm15. |
| 276 | // The constant is created with addi voff, x0, C, and shXadd is used to |
| 277 | // fill insert the trailing zeros and do the addition. |
| 278 | // If the pattern is found, updates the offset in Hi and Lo instructions |
| 279 | // and deletes TailShXAdd and the instructions that produced the offset. |
| 280 | // |
| 281 | // Hi: lui vreg1, %hi(s) |
| 282 | // Lo: addi vreg2, vreg1, %lo(s) |
| 283 | // OffsetTail: addi voff, x0, C |
| 284 | // TailAdd: shXadd vreg4, voff, vreg2 |
| 285 | static bool foldShiftedOffset(MachineInstr &Hi, MachineInstr &Lo, |
| 286 | MachineInstr &TailShXAdd, Register GAReg, |
| 287 | MachineRegisterInfo *MRI) { |
| 288 | assert((TailShXAdd.getOpcode() == RISCV::SH1ADD || |
| 289 | TailShXAdd.getOpcode() == RISCV::SH2ADD || |
| 290 | TailShXAdd.getOpcode() == RISCV::SH3ADD) && |
| 291 | "Expected SHXADD instruction!" ); |
| 292 | |
| 293 | if (GAReg != TailShXAdd.getOperand(i: 2).getReg()) |
| 294 | return false; |
| 295 | |
| 296 | // The first source is the shifted operand. |
| 297 | Register Rs1 = TailShXAdd.getOperand(i: 1).getReg(); |
| 298 | |
| 299 | // Can't fold if the register has more than one use. |
| 300 | if (!Rs1.isVirtual() || !MRI->hasOneNonDBGUse(RegNo: Rs1)) |
| 301 | return false; |
| 302 | // This can point to an ADDI X0, C. |
| 303 | MachineInstr &OffsetTail = *MRI->getVRegDef(Reg: Rs1); |
| 304 | if (OffsetTail.getOpcode() != RISCV::ADDI) |
| 305 | return false; |
| 306 | if (!OffsetTail.getOperand(i: 1).isReg() || |
| 307 | OffsetTail.getOperand(i: 1).getReg() != RISCV::X0 || |
| 308 | !OffsetTail.getOperand(i: 2).isImm()) |
| 309 | return false; |
| 310 | |
| 311 | int64_t Offset = OffsetTail.getOperand(i: 2).getImm(); |
| 312 | assert(isInt<12>(Offset) && "Unexpected offset" ); |
| 313 | |
| 314 | unsigned ShAmt; |
| 315 | switch (TailShXAdd.getOpcode()) { |
| 316 | default: llvm_unreachable("Unexpected opcode" ); |
| 317 | case RISCV::SH1ADD: ShAmt = 1; break; |
| 318 | case RISCV::SH2ADD: ShAmt = 2; break; |
| 319 | case RISCV::SH3ADD: ShAmt = 3; break; |
| 320 | } |
| 321 | |
| 322 | Offset = (uint64_t)Offset << ShAmt; |
| 323 | |
| 324 | LLVM_DEBUG(dbgs() << " Offset Instr: " << OffsetTail); |
| 325 | if (!foldOffset(Hi, Lo, Tail&: TailShXAdd, Offset, MRI)) |
| 326 | return false; |
| 327 | OffsetTail.eraseFromParent(); |
| 328 | return true; |
| 329 | } |
| 330 | |
| 331 | static bool detectAndFoldOffset(MachineInstr &Hi, MachineInstr &Lo, |
| 332 | MachineRegisterInfo *MRI, |
| 333 | const RISCVSubtarget *ST) { |
| 334 | Register DestReg = Lo.getOperand(i: 0).getReg(); |
| 335 | |
| 336 | // Look for arithmetic instructions we can get an offset from. |
| 337 | // We might be able to remove the arithmetic instructions by folding the |
| 338 | // offset into the LUI+ADDI. |
| 339 | if (!MRI->hasOneNonDBGUse(RegNo: DestReg)) |
| 340 | return false; |
| 341 | |
| 342 | // Lo has only one use. |
| 343 | MachineInstr &Tail = *MRI->use_instr_nodbg_begin(RegNo: DestReg); |
| 344 | switch (Tail.getOpcode()) { |
| 345 | default: |
| 346 | LLVM_DEBUG(dbgs() << "Don't know how to get offset from this instr: " |
| 347 | << Tail); |
| 348 | break; |
| 349 | case RISCV::ADDI: |
| 350 | case RISCV::QC_E_ADDI: |
| 351 | case RISCV::QC_E_ADDAI: { |
| 352 | // Offset is simply an immediate operand. |
| 353 | int64_t Offset = Tail.getOperand(i: 2).getImm(); |
| 354 | if (Tail.getOpcode() == RISCV::ADDI) { |
| 355 | // We might have two ADDIs in a row. |
| 356 | Register TailDestReg = Tail.getOperand(i: 0).getReg(); |
| 357 | if (MRI->hasOneNonDBGUse(RegNo: TailDestReg)) { |
| 358 | MachineInstr &TailTail = *MRI->use_instr_nodbg_begin(RegNo: TailDestReg); |
| 359 | if (TailTail.getOpcode() == RISCV::ADDI) { |
| 360 | Offset += TailTail.getOperand(i: 2).getImm(); |
| 361 | LLVM_DEBUG(dbgs() << " Offset Instrs: " << Tail << TailTail); |
| 362 | if (!foldOffset(Hi, Lo, Tail&: TailTail, Offset, MRI)) |
| 363 | return false; |
| 364 | Tail.eraseFromParent(); |
| 365 | return true; |
| 366 | } |
| 367 | } |
| 368 | } |
| 369 | |
| 370 | LLVM_DEBUG(dbgs() << " Offset Instr: " << Tail); |
| 371 | return foldOffset(Hi, Lo, Tail, Offset, MRI); |
| 372 | } |
| 373 | case RISCV::ADD: |
| 374 | // The offset is too large to fit in the immediate field of ADDI. |
| 375 | // This can be in two forms: |
| 376 | // 1) LUI hi_Offset followed by: |
| 377 | // ADDI lo_offset |
| 378 | // This happens in case the offset has non zero bits in |
| 379 | // both hi 20 and lo 12 bits. |
| 380 | // 2) LUI (offset20) |
| 381 | // This happens in case the lower 12 bits of the offset are zeros. |
| 382 | return foldLargeOffset(Hi, Lo, TailAdd&: Tail, GAReg: DestReg, MRI, ST); |
| 383 | case RISCV::SH1ADD: |
| 384 | case RISCV::SH2ADD: |
| 385 | case RISCV::SH3ADD: |
| 386 | // The offset is too large to fit in the immediate field of ADDI. |
| 387 | // It may be encoded as (SH2ADD (ADDI X0, C), DestReg) or |
| 388 | // (SH3ADD (ADDI X0, C), DestReg). |
| 389 | return foldShiftedOffset(Hi, Lo, TailShXAdd&: Tail, GAReg: DestReg, MRI); |
| 390 | } |
| 391 | |
| 392 | return false; |
| 393 | } |
| 394 | |
| 395 | static void overwriteMachineOperandInPlace(MachineOperand &MO, |
| 396 | const MachineOperand &ImmOp) { |
| 397 | switch (ImmOp.getType()) { |
| 398 | case MachineOperand::MO_ConstantPoolIndex: |
| 399 | MO.ChangeToCPI(Idx: ImmOp.getIndex(), Offset: ImmOp.getOffset(), TargetFlags: ImmOp.getTargetFlags()); |
| 400 | break; |
| 401 | case MachineOperand::MO_GlobalAddress: |
| 402 | MO.ChangeToGA(GV: ImmOp.getGlobal(), Offset: ImmOp.getOffset(), TargetFlags: ImmOp.getTargetFlags()); |
| 403 | break; |
| 404 | case MachineOperand::MO_MCSymbol: |
| 405 | MO.ChangeToMCSymbol(Sym: ImmOp.getMCSymbol(), TargetFlags: ImmOp.getTargetFlags()); |
| 406 | MO.setOffset(ImmOp.getOffset()); |
| 407 | break; |
| 408 | case MachineOperand::MO_BlockAddress: |
| 409 | MO.ChangeToBA(BA: ImmOp.getBlockAddress(), Offset: ImmOp.getOffset(), |
| 410 | TargetFlags: ImmOp.getTargetFlags()); |
| 411 | break; |
| 412 | default: |
| 413 | report_fatal_error(reason: "unsupported machine operand type" ); |
| 414 | break; |
| 415 | } |
| 416 | } |
| 417 | |
| 418 | static bool foldIntoMemoryOps(MachineInstr &Hi, MachineInstr &Lo, |
| 419 | MachineRegisterInfo *MRI, |
| 420 | const RISCVSubtarget *ST) { |
| 421 | Register DestReg = Lo.getOperand(i: 0).getReg(); |
| 422 | |
| 423 | // If all the uses are memory ops with the same offset, we can transform: |
| 424 | // |
| 425 | // 1. (medlow pattern): |
| 426 | // a. Hi: lui vreg1, %hi(foo) ---> lui vreg1, %hi(foo+8) |
| 427 | // Lo: addi vreg2, vreg1, %lo(foo) ---> lw vreg3, lo(foo+8)(vreg1) |
| 428 | // Tail: lw vreg3, 8(vreg2) |
| 429 | // |
| 430 | // b. Hi: qc.e.li vreg1, foo ---> qc.e.li vreg1, foo+8 |
| 431 | // Tail: lw vreg2, 8(vreg1) ---> lw vreg2, 0(vreg1) |
| 432 | // |
| 433 | // 2. (medany pattern): |
| 434 | // Hi: 1:auipc vreg1, %pcrel_hi(s) ---> auipc vreg1, %pcrel_hi(foo+8) |
| 435 | // Lo: addi vreg2, vreg1, %pcrel_lo(1b) ---> lw vreg3, %pcrel_lo(1b)(vreg1) |
| 436 | // Tail: lw vreg3, 8(vreg2) |
| 437 | |
| 438 | std::optional<int64_t> CommonOffset; |
| 439 | DenseMap<const MachineInstr *, SmallVector<unsigned>> |
| 440 | InlineAsmMemoryOpIndexesMap; |
| 441 | for (const MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg: DestReg)) { |
| 442 | switch (UseMI.getOpcode()) { |
| 443 | default: |
| 444 | LLVM_DEBUG(dbgs() << "Not a load or store instruction: " << UseMI); |
| 445 | return false; |
| 446 | case RISCV::LB: |
| 447 | case RISCV::LH: |
| 448 | case RISCV::LH_INX: |
| 449 | case RISCV::LW: |
| 450 | case RISCV::LW_INX: |
| 451 | case RISCV::LBU: |
| 452 | case RISCV::LHU: |
| 453 | case RISCV::LWU: |
| 454 | case RISCV::LD: |
| 455 | case RISCV::LD_RV32: |
| 456 | case RISCV::FLH: |
| 457 | case RISCV::FLW: |
| 458 | case RISCV::FLD: |
| 459 | case RISCV::SB: |
| 460 | case RISCV::SH: |
| 461 | case RISCV::SH_INX: |
| 462 | case RISCV::SW: |
| 463 | case RISCV::SW_INX: |
| 464 | case RISCV::SD: |
| 465 | case RISCV::SD_RV32: |
| 466 | case RISCV::FSH: |
| 467 | case RISCV::FSW: |
| 468 | case RISCV::FSD: { |
| 469 | if (UseMI.getOperand(i: 1).isFI()) |
| 470 | return false; |
| 471 | // Register defined by Lo should not be the value register. |
| 472 | if (DestReg == UseMI.getOperand(i: 0).getReg()) |
| 473 | return false; |
| 474 | assert(DestReg == UseMI.getOperand(1).getReg() && |
| 475 | "Expected base address use" ); |
| 476 | // All load/store instructions must use the same offset. |
| 477 | int64_t Offset = UseMI.getOperand(i: 2).getImm(); |
| 478 | if (CommonOffset && Offset != CommonOffset) |
| 479 | return false; |
| 480 | CommonOffset = Offset; |
| 481 | break; |
| 482 | } |
| 483 | case RISCV::PseudoCCLD: |
| 484 | case RISCV::PseudoCCLW: |
| 485 | case RISCV::PseudoCCLWU: |
| 486 | case RISCV::PseudoCCLH: |
| 487 | case RISCV::PseudoCCLHU: |
| 488 | case RISCV::PseudoCCLB: |
| 489 | case RISCV::PseudoCCLBU: { |
| 490 | // The SFB Pseudos are like their non-SFB counterparts but have more |
| 491 | // operands. |
| 492 | if (UseMI.getOperand(i: 2).isFI()) |
| 493 | return false; |
| 494 | // Register defined by Lo should not be the (tied) false value, or a |
| 495 | // register used in the branch predicate. |
| 496 | if (DestReg == UseMI.getOperand(i: 1).getReg() || |
| 497 | DestReg == UseMI.getOperand(i: 5).getReg()) |
| 498 | return false; |
| 499 | if (UseMI.getOperand(i: 6).isReg() && |
| 500 | DestReg == UseMI.getOperand(i: 6).getReg()) |
| 501 | return false; |
| 502 | assert(DestReg == UseMI.getOperand(2).getReg() && |
| 503 | "Expected base address use" ); |
| 504 | // All load/store instructions must use the same offset. |
| 505 | int64_t Offset = UseMI.getOperand(i: 3).getImm(); |
| 506 | if (CommonOffset && Offset != CommonOffset) |
| 507 | return false; |
| 508 | CommonOffset = Offset; |
| 509 | break; |
| 510 | } |
| 511 | case RISCV::INLINEASM: |
| 512 | case RISCV::INLINEASM_BR: { |
| 513 | SmallVector<unsigned> InlineAsmMemoryOpIndexes; |
| 514 | unsigned NumOps = 0; |
| 515 | for (unsigned I = InlineAsm::MIOp_FirstOperand; |
| 516 | I < UseMI.getNumOperands(); I += 1 + NumOps) { |
| 517 | const MachineOperand &FlagsMO = UseMI.getOperand(i: I); |
| 518 | // Should be an imm. |
| 519 | if (!FlagsMO.isImm()) |
| 520 | continue; |
| 521 | |
| 522 | const InlineAsm::Flag Flags(FlagsMO.getImm()); |
| 523 | NumOps = Flags.getNumOperandRegisters(); |
| 524 | |
| 525 | // Memory constraints have two operands. |
| 526 | if (NumOps != 2 || !Flags.isMemKind()) { |
| 527 | // If the register is used by something other than a memory |
| 528 | // constraint, we should not fold. |
| 529 | for (unsigned J = 0; J < NumOps; ++J) { |
| 530 | const MachineOperand &MO = UseMI.getOperand(i: I + 1 + J); |
| 531 | if (MO.isReg() && MO.getReg() == DestReg) |
| 532 | return false; |
| 533 | } |
| 534 | continue; |
| 535 | } |
| 536 | |
| 537 | // We can't do this for constraint A because AMO instructions don't have |
| 538 | // an immediate offset field. |
| 539 | if (Flags.getMemoryConstraintID() == InlineAsm::ConstraintCode::A) |
| 540 | return false; |
| 541 | |
| 542 | const MachineOperand &AddrMO = UseMI.getOperand(i: I + 1); |
| 543 | if (!AddrMO.isReg() || AddrMO.getReg() != DestReg) |
| 544 | continue; |
| 545 | |
| 546 | const MachineOperand &OffsetMO = UseMI.getOperand(i: I + 2); |
| 547 | if (!OffsetMO.isImm()) |
| 548 | continue; |
| 549 | |
| 550 | // All inline asm memory operands must use the same offset. |
| 551 | int64_t Offset = OffsetMO.getImm(); |
| 552 | if (CommonOffset && Offset != CommonOffset) |
| 553 | return false; |
| 554 | CommonOffset = Offset; |
| 555 | InlineAsmMemoryOpIndexes.push_back(Elt: I + 1); |
| 556 | } |
| 557 | InlineAsmMemoryOpIndexesMap.insert( |
| 558 | KV: std::make_pair(x: &UseMI, y&: InlineAsmMemoryOpIndexes)); |
| 559 | break; |
| 560 | } |
| 561 | } |
| 562 | } |
| 563 | |
| 564 | if (!CommonOffset) |
| 565 | return false; |
| 566 | |
| 567 | // We found a common offset. |
| 568 | // Update the offsets in global address lowering. |
| 569 | // We may have already folded some arithmetic so we need to add to any |
| 570 | // existing offset. |
| 571 | int64_t NewOffset = Hi.getOperand(i: 1).getOffset() + *CommonOffset; |
| 572 | // RV32 ignores the upper 32 bits. |
| 573 | if (!ST->is64Bit()) |
| 574 | NewOffset = SignExtend64<32>(x: NewOffset); |
| 575 | // We can only fold simm32 offsets. |
| 576 | if (!isInt<32>(x: NewOffset)) |
| 577 | return false; |
| 578 | |
| 579 | // Folding changes the value represented by DestReg, so its debug users can |
| 580 | // no longer describe the same address. |
| 581 | MRI->markUsesInDebugValueAsUndef(Reg: DestReg); |
| 582 | |
| 583 | Hi.getOperand(i: 1).setOffset(NewOffset); |
| 584 | MachineOperand &ImmOp = |
| 585 | Hi.getOpcode() == RISCV::QC_E_LI ? Lo.getOperand(i: 1) : Lo.getOperand(i: 2); |
| 586 | auto HiOpc = Hi.getOpcode(); |
| 587 | // Expand PseudoMovAddr into LUI |
| 588 | if (HiOpc == RISCV::PseudoMovAddr) { |
| 589 | auto *TII = ST->getInstrInfo(); |
| 590 | Hi.setDesc(TII->get(Opcode: RISCV::LUI)); |
| 591 | Hi.removeOperand(OpNo: 2); |
| 592 | } |
| 593 | |
| 594 | if (HiOpc != RISCV::AUIPC) |
| 595 | ImmOp.setOffset(NewOffset); |
| 596 | |
| 597 | // Update the immediate in the load/store instructions to add the offset. |
| 598 | for (MachineInstr &UseMI : |
| 599 | llvm::make_early_inc_range(Range: MRI->use_nodbg_instructions(Reg: DestReg))) { |
| 600 | if (UseMI.getOpcode() == RISCV::INLINEASM || |
| 601 | UseMI.getOpcode() == RISCV::INLINEASM_BR) { |
| 602 | auto &InlineAsmMemoryOpIndexes = InlineAsmMemoryOpIndexesMap[&UseMI]; |
| 603 | for (unsigned I : InlineAsmMemoryOpIndexes) { |
| 604 | MachineOperand &MO = UseMI.getOperand(i: I + 1); |
| 605 | overwriteMachineOperandInPlace(MO, ImmOp); |
| 606 | } |
| 607 | } else { |
| 608 | unsigned ImmIdx; |
| 609 | switch (UseMI.getOpcode()) { |
| 610 | case RISCV::INLINEASM: |
| 611 | case RISCV::INLINEASM_BR: |
| 612 | llvm_unreachable("Should have been dealt with before this else" ); |
| 613 | case RISCV::PseudoCCLD: |
| 614 | case RISCV::PseudoCCLW: |
| 615 | case RISCV::PseudoCCLWU: |
| 616 | case RISCV::PseudoCCLH: |
| 617 | case RISCV::PseudoCCLHU: |
| 618 | case RISCV::PseudoCCLB: |
| 619 | case RISCV::PseudoCCLBU: |
| 620 | ImmIdx = 3; |
| 621 | break; |
| 622 | case RISCV::LB: |
| 623 | case RISCV::LH: |
| 624 | case RISCV::LH_INX: |
| 625 | case RISCV::LW: |
| 626 | case RISCV::LW_INX: |
| 627 | case RISCV::LBU: |
| 628 | case RISCV::LHU: |
| 629 | case RISCV::LWU: |
| 630 | case RISCV::LD: |
| 631 | case RISCV::LD_RV32: |
| 632 | case RISCV::FLH: |
| 633 | case RISCV::FLW: |
| 634 | case RISCV::FLD: |
| 635 | case RISCV::SB: |
| 636 | case RISCV::SH: |
| 637 | case RISCV::SH_INX: |
| 638 | case RISCV::SW: |
| 639 | case RISCV::SW_INX: |
| 640 | case RISCV::SD: |
| 641 | case RISCV::SD_RV32: |
| 642 | case RISCV::FSH: |
| 643 | case RISCV::FSW: |
| 644 | case RISCV::FSD: |
| 645 | ImmIdx = 2; |
| 646 | break; |
| 647 | default: |
| 648 | llvm_unreachable("Unknown Instruction" ); |
| 649 | } |
| 650 | |
| 651 | MachineOperand &MO = UseMI.getOperand(i: ImmIdx); |
| 652 | if (Hi.getOpcode() == RISCV::QC_E_LI) { |
| 653 | MO.ChangeToImmediate(ImmVal: 0); |
| 654 | } else { |
| 655 | overwriteMachineOperandInPlace(MO, ImmOp); |
| 656 | } |
| 657 | } |
| 658 | } |
| 659 | |
| 660 | // Prevent Lo (originally PseudoMovAddr, which is also pointed by Hi) from |
| 661 | // being erased |
| 662 | if (&Lo == &Hi) |
| 663 | return true; |
| 664 | |
| 665 | MRI->replaceRegWith(FromReg: Lo.getOperand(i: 0).getReg(), ToReg: Hi.getOperand(i: 0).getReg()); |
| 666 | Lo.eraseFromParent(); |
| 667 | return true; |
| 668 | } |
| 669 | |
| 670 | // Try to fold sequences of the form: |
| 671 | // Hi/lo: qc.e.li vreg1, s -> qc.e.li vreg1, s+imm |
| 672 | // TailAdd: shxadd vreg2, vreg3, vreg1 -> deleted |
| 673 | // Tail: lx vreg4, imm(vreg2) -> qc.lrx vreg4, vreg1, vreg3, (1-7) |
| 674 | static bool foldShxaddIntoScaledMemory(MachineInstr &Hi, MachineInstr &Lo, |
| 675 | MachineRegisterInfo *MRI, |
| 676 | const RISCVSubtarget *ST) { |
| 677 | if (!ST->hasVendorXqcisls() || ST->is64Bit()) |
| 678 | return false; |
| 679 | |
| 680 | if (Hi.getOpcode() != RISCV::QC_E_LI) |
| 681 | return false; |
| 682 | |
| 683 | Register BaseReg = Hi.getOperand(i: 0).getReg(); |
| 684 | if (!BaseReg.isVirtual() || !MRI->hasOneUse(RegNo: BaseReg)) |
| 685 | return false; |
| 686 | |
| 687 | MachineInstr &ShxAdd = *MRI->use_instr_begin(RegNo: BaseReg); |
| 688 | unsigned ShxOpc = ShxAdd.getOpcode(); |
| 689 | unsigned ShAmt = 0; |
| 690 | switch (ShxOpc) { |
| 691 | default: |
| 692 | return false; |
| 693 | case RISCV::SH1ADD: |
| 694 | ShAmt = 1; |
| 695 | break; |
| 696 | case RISCV::SH2ADD: |
| 697 | ShAmt = 2; |
| 698 | break; |
| 699 | case RISCV::SH3ADD: |
| 700 | ShAmt = 3; |
| 701 | break; |
| 702 | case RISCV::QC_SHLADD: |
| 703 | uint8_t ShlImm = ShxAdd.getOperand(i: 3).getImm(); |
| 704 | if (ShlImm > 7) |
| 705 | return false; |
| 706 | ShAmt = ShlImm; |
| 707 | break; |
| 708 | } |
| 709 | |
| 710 | // shxadd Rd, Rs1, Rs2 |
| 711 | Register ScaledReg = ShxAdd.getOperand(i: 0).getReg(); |
| 712 | Register IndexReg = ShxAdd.getOperand(i: 1).getReg(); |
| 713 | |
| 714 | if (!IndexReg.isVirtual()) |
| 715 | return false; |
| 716 | |
| 717 | if (ShxAdd.getOperand(i: 2).getReg() != BaseReg) |
| 718 | return false; |
| 719 | |
| 720 | if (!ScaledReg.isVirtual() || !MRI->hasOneUse(RegNo: ScaledReg)) |
| 721 | return false; |
| 722 | |
| 723 | MachineInstr &TailMem = *MRI->use_instr_begin(RegNo: ScaledReg); |
| 724 | unsigned Opc = TailMem.getOpcode(); |
| 725 | unsigned NewOpc = 0; |
| 726 | |
| 727 | switch (Opc) { |
| 728 | case RISCV::LB: |
| 729 | NewOpc = RISCV::QC_LRB; |
| 730 | break; |
| 731 | case RISCV::LBU: |
| 732 | NewOpc = RISCV::QC_LRBU; |
| 733 | break; |
| 734 | case RISCV::LH: |
| 735 | NewOpc = RISCV::QC_LRH; |
| 736 | break; |
| 737 | case RISCV::LHU: |
| 738 | NewOpc = RISCV::QC_LRHU; |
| 739 | break; |
| 740 | case RISCV::LW: |
| 741 | NewOpc = RISCV::QC_LRW; |
| 742 | break; |
| 743 | case RISCV::SB: |
| 744 | NewOpc = RISCV::QC_SRB; |
| 745 | break; |
| 746 | case RISCV::SH: |
| 747 | NewOpc = RISCV::QC_SRH; |
| 748 | break; |
| 749 | case RISCV::SW: |
| 750 | NewOpc = RISCV::QC_SRW; |
| 751 | break; |
| 752 | default: |
| 753 | return false; |
| 754 | } |
| 755 | |
| 756 | if (!TailMem.getOperand(i: 1).isReg() || |
| 757 | TailMem.getOperand(i: 1).getReg() != ScaledReg) |
| 758 | return false; |
| 759 | if (!TailMem.getOperand(i: 2).isImm()) |
| 760 | return false; |
| 761 | int64_t Imm = TailMem.getOperand(i: 2).getImm(); |
| 762 | |
| 763 | // Update QC_E_LI offset. |
| 764 | int64_t NewOffset = SignExtend64<32>(x: Hi.getOperand(i: 1).getOffset() + Imm); |
| 765 | |
| 766 | Hi.getOperand(i: 1).setOffset(NewOffset); |
| 767 | |
| 768 | // Build scaled load/store. |
| 769 | auto *TII = ST->getInstrInfo(); |
| 770 | auto *MBB = TailMem.getParent(); |
| 771 | |
| 772 | // Ensure index register satisfies GPRNoX0 class required by QC_LR*/QC_SR*. |
| 773 | MRI->constrainRegClass(Reg: IndexReg, RC: &RISCV::GPRNoX0RegClass); |
| 774 | |
| 775 | BuildMI(BB&: *MBB, I&: TailMem, MIMD: TailMem.getDebugLoc(), MCID: TII->get(Opcode: NewOpc)) |
| 776 | .add(MO: TailMem.getOperand(i: 0)) |
| 777 | .addReg(RegNo: BaseReg, Flags: getKillRegState(B: ShxAdd.getOperand(i: 2).isKill())) |
| 778 | .addReg(RegNo: IndexReg, Flags: getKillRegState(B: ShxAdd.getOperand(i: 1).isKill())) |
| 779 | .addImm(Val: ShAmt) |
| 780 | .cloneMemRefs(OtherMI: TailMem); |
| 781 | |
| 782 | TailMem.eraseFromParent(); |
| 783 | ShxAdd.eraseFromParent(); |
| 784 | return true; |
| 785 | } |
| 786 | |
| 787 | static bool runMergeBaseOffset(MachineFunction &Fn) { |
| 788 | const RISCVSubtarget *ST = &Fn.getSubtarget<RISCVSubtarget>(); |
| 789 | |
| 790 | bool MadeChange = false; |
| 791 | MachineRegisterInfo *MRI = &Fn.getRegInfo(); |
| 792 | for (MachineBasicBlock &MBB : Fn) { |
| 793 | LLVM_DEBUG(dbgs() << "MBB: " << MBB.getName() << "\n" ); |
| 794 | for (MachineInstr &Hi : MBB) { |
| 795 | MachineInstr *Lo = nullptr; |
| 796 | if (!detectFoldable(Hi, Lo, MRI)) |
| 797 | continue; |
| 798 | MadeChange |= detectAndFoldOffset(Hi, Lo&: *Lo, MRI, ST); |
| 799 | MadeChange |= foldIntoMemoryOps(Hi, Lo&: *Lo, MRI, ST); |
| 800 | MadeChange |= foldShxaddIntoScaledMemory(Hi, Lo&: *Lo, MRI, ST); |
| 801 | } |
| 802 | } |
| 803 | |
| 804 | return MadeChange; |
| 805 | } |
| 806 | |
| 807 | /// Returns an instance of the Merge Base Offset Optimization pass. |
| 808 | FunctionPass *llvm::createRISCVMergeBaseOffsetLegacyPass() { |
| 809 | return new RISCVMergeBaseOffsetLegacy(); |
| 810 | } |
| 811 | |
| 812 | bool RISCVMergeBaseOffsetLegacy::runOnMachineFunction(MachineFunction &MF) { |
| 813 | if (skipFunction(F: MF.getFunction())) |
| 814 | return false; |
| 815 | return runMergeBaseOffset(Fn&: MF); |
| 816 | } |
| 817 | |
| 818 | PreservedAnalyses |
| 819 | RISCVMergeBaseOffsetPass::run(MachineFunction &MF, |
| 820 | MachineFunctionAnalysisManager &MFAM) { |
| 821 | return runMergeBaseOffset(Fn&: MF) ? getMachineFunctionPassPreservedAnalyses() |
| 822 | .preserveSet<CFGAnalyses>() |
| 823 | : PreservedAnalyses::all(); |
| 824 | } |
| 825 | |