1//===----- RISCVMergeBaseOffset.cpp - Optimise address calculations ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Merge the offset of address calculation into the offset field
10// of instructions in a global address lowering sequence.
11//
12//===----------------------------------------------------------------------===//
13
14#include "RISCV.h"
15#include "RISCVTargetMachine.h"
16#include "llvm/CodeGen/MachineFunctionPass.h"
17#include "llvm/CodeGen/Passes.h"
18#include "llvm/IR/DebugInfoMetadata.h"
19#include "llvm/MC/TargetRegistry.h"
20#include "llvm/Support/Debug.h"
21#include "llvm/Target/TargetOptions.h"
22#include <optional>
23using namespace llvm;
24
25#define DEBUG_TYPE "riscv-merge-base-offset"
26#define RISCV_MERGE_BASE_OFFSET_NAME "RISC-V Merge Base Offset"
27namespace {
28
29class RISCVMergeBaseOffsetOpt : public MachineFunctionPass {
30 const RISCVSubtarget *ST = nullptr;
31 MachineRegisterInfo *MRI;
32
33public:
34 static char ID;
35 bool runOnMachineFunction(MachineFunction &Fn) override;
36 bool detectFoldable(MachineInstr &Hi, MachineInstr *&Lo);
37
38 bool detectAndFoldOffset(MachineInstr &Hi, MachineInstr &Lo);
39 bool foldOffset(MachineInstr &Hi, MachineInstr &Lo, MachineInstr &Tail,
40 int64_t Offset);
41 bool foldLargeOffset(MachineInstr &Hi, MachineInstr &Lo,
42 MachineInstr &TailAdd, Register GSReg);
43 bool foldShiftedOffset(MachineInstr &Hi, MachineInstr &Lo,
44 MachineInstr &TailShXAdd, Register GSReg);
45
46 bool foldIntoMemoryOps(MachineInstr &Hi, MachineInstr &Lo);
47 bool foldShxaddIntoScaledMemory(MachineInstr &Hi, MachineInstr &Lo);
48
49 RISCVMergeBaseOffsetOpt() : MachineFunctionPass(ID) {}
50
51 MachineFunctionProperties getRequiredProperties() const override {
52 return MachineFunctionProperties().setIsSSA();
53 }
54
55 void getAnalysisUsage(AnalysisUsage &AU) const override {
56 AU.setPreservesCFG();
57 MachineFunctionPass::getAnalysisUsage(AU);
58 }
59
60 StringRef getPassName() const override {
61 return RISCV_MERGE_BASE_OFFSET_NAME;
62 }
63};
64} // end anonymous namespace
65
66char RISCVMergeBaseOffsetOpt::ID = 0;
67INITIALIZE_PASS(RISCVMergeBaseOffsetOpt, DEBUG_TYPE,
68 RISCV_MERGE_BASE_OFFSET_NAME, false, false)
69
70// Detect either of the patterns:
71//
72// 1. (medlow pattern):
73// a. lui vreg1, %hi(s)
74// addi vreg2, vreg1, %lo(s)
75//
76// b. qc.e.li vreg1, s
77//
78// 2. (medany pattern):
79// .Lpcrel_hi1:
80// auipc vreg1, %pcrel_hi(s)
81// addi vreg2, vreg1, %pcrel_lo(.Lpcrel_hi1)
82//
83// The pattern is only accepted if:
84// 1) The first instruction has only one use, which is the ADDI.
85// 2) The address operands have the appropriate type, reflecting the
86// lowering of a global address or constant pool using medlow or medany.
87// 3) The offset value in the Global Address or Constant Pool is 0.
88bool RISCVMergeBaseOffsetOpt::detectFoldable(MachineInstr &Hi,
89 MachineInstr *&Lo) {
90 auto HiOpc = Hi.getOpcode();
91 if (HiOpc != RISCV::LUI && HiOpc != RISCV::AUIPC &&
92 HiOpc != RISCV::PseudoMovAddr && HiOpc != RISCV::QC_E_LI)
93 return false;
94
95 const MachineOperand &HiOp1 = Hi.getOperand(i: 1);
96 unsigned ExpectedFlags = HiOpc == RISCV::AUIPC ? RISCVII::MO_PCREL_HI
97 : HiOpc == RISCV::QC_E_LI ? RISCVII::MO_None
98 : RISCVII::MO_HI;
99 if (HiOp1.getTargetFlags() != ExpectedFlags)
100 return false;
101
102 if (!(HiOp1.isGlobal() || HiOp1.isCPI() || HiOp1.isBlockAddress()) ||
103 HiOp1.getOffset() != 0)
104 return false;
105
106 if (HiOpc == RISCV::PseudoMovAddr || HiOpc == RISCV::QC_E_LI) {
107 // Most of the code should handle it correctly without modification by
108 // setting Lo and Hi both point to PseudoMovAddr/QC_E_LI
109 Lo = &Hi;
110 } else {
111 Register HiDestReg = Hi.getOperand(i: 0).getReg();
112 if (!MRI->hasOneUse(RegNo: HiDestReg))
113 return false;
114
115 Lo = &*MRI->use_instr_begin(RegNo: HiDestReg);
116 if (Lo->getOpcode() != RISCV::ADDI)
117 return false;
118 }
119
120 if (HiOpc != RISCV::QC_E_LI) {
121 const MachineOperand &LoOp2 = Lo->getOperand(i: 2);
122 if (HiOpc == RISCV::LUI || HiOpc == RISCV::PseudoMovAddr) {
123 if (LoOp2.getTargetFlags() != RISCVII::MO_LO ||
124 !(LoOp2.isGlobal() || LoOp2.isCPI() || LoOp2.isBlockAddress()) ||
125 LoOp2.getOffset() != 0)
126 return false;
127 } else {
128 assert(HiOpc == RISCV::AUIPC);
129 if (LoOp2.getTargetFlags() != RISCVII::MO_PCREL_LO ||
130 LoOp2.getType() != MachineOperand::MO_MCSymbol)
131 return false;
132 }
133 }
134
135 if (HiOp1.isGlobal()) {
136 LLVM_DEBUG(dbgs() << " Found lowered global address: "
137 << *HiOp1.getGlobal() << "\n");
138 } else if (HiOp1.isBlockAddress()) {
139 LLVM_DEBUG(dbgs() << " Found lowered basic address: "
140 << *HiOp1.getBlockAddress() << "\n");
141 } else if (HiOp1.isCPI()) {
142 LLVM_DEBUG(dbgs() << " Found lowered constant pool: " << HiOp1.getIndex()
143 << "\n");
144 }
145
146 return true;
147}
148
149// Update the offset in Hi and Lo instructions.
150// Delete the tail instruction and update all the uses to use the
151// output from Lo.
152bool RISCVMergeBaseOffsetOpt::foldOffset(MachineInstr &Hi, MachineInstr &Lo,
153 MachineInstr &Tail, int64_t Offset) {
154 assert(isInt<32>(Offset) && "Unexpected offset");
155
156 // If Hi is an AUIPC, don't fold the offset if it is outside the bounds of
157 // the global object. The object may be within 2GB of the PC, but addresses
158 // outside of the object might not be.
159 auto HiOpc = Hi.getOpcode();
160 if (HiOpc == RISCV::AUIPC && Hi.getOperand(i: 1).isGlobal()) {
161 const GlobalValue *GV = Hi.getOperand(i: 1).getGlobal();
162 Type *Ty = GV->getValueType();
163 if (!Ty->isSized() || Offset < 0 ||
164 (uint64_t)Offset > GV->getDataLayout().getTypeAllocSize(Ty))
165 return false;
166 }
167
168 // Salvage the debug users of Lo's result by subtracting the offset again.
169 SmallVector<uint64_t, 3> UndoOffset;
170 DIExpression::appendOffset(Ops&: UndoOffset, Offset: -Offset);
171 for (MachineOperand &MO : MRI->use_operands(Reg: Lo.getOperand(i: 0).getReg())) {
172 MachineInstr &DbgMI = *MO.getParent();
173 if (!DbgMI.isDebugValue())
174 continue;
175 DbgMI.getDebugExpressionOp().setMetadata(
176 DIExpression::appendOpsToArg(Expr: DbgMI.getDebugExpression(), Ops: UndoOffset,
177 ArgNo: DbgMI.getDebugOperandIndex(Op: &MO)));
178 }
179
180 // Put the offset back in Hi and the Lo
181 Hi.getOperand(i: 1).setOffset(Offset);
182 if (Hi.getOpcode() != RISCV::AUIPC && Hi.getOpcode() != RISCV::QC_E_LI)
183 Lo.getOperand(i: 2).setOffset(Offset);
184 // Delete the tail instruction.
185 Register LoOp0Reg = Lo.getOperand(i: 0).getReg();
186 Register TailOp0Reg = Tail.getOperand(i: 0).getReg();
187 MRI->constrainRegClass(Reg: LoOp0Reg, RC: MRI->getRegClass(Reg: TailOp0Reg));
188 MRI->replaceRegWith(FromReg: TailOp0Reg, ToReg: LoOp0Reg);
189 Tail.eraseFromParent();
190 LLVM_DEBUG(dbgs() << " Merged offset " << Offset << " into base.\n"
191 << " " << Hi << " " << Lo;);
192 return true;
193}
194
195// Detect patterns for large offsets that are passed into an ADD instruction.
196// If the pattern is found, updates the offset in Hi and Lo instructions
197// and deletes TailAdd and the instructions that produced the offset.
198//
199// Base address lowering is of the form:
200// Hi: lui vreg1, %hi(s)
201// Lo: addi vreg2, vreg1, %lo(s)
202// / \
203// / \
204// / \
205// / The large offset can be of two forms: \
206// 1) Offset that has non zero bits in lower 2) Offset that has non zero
207// 12 bits and upper 20 bits bits in upper 20 bits only
208// OffseLUI: lui vreg3, 4
209// OffsetTail: addi voff, vreg3, 188 OffsetTail: lui voff, 128
210// \ /
211// \ /
212// \ /
213// \ /
214// TailAdd: add vreg4, vreg2, voff
215bool RISCVMergeBaseOffsetOpt::foldLargeOffset(MachineInstr &Hi,
216 MachineInstr &Lo,
217 MachineInstr &TailAdd,
218 Register GAReg) {
219 assert((TailAdd.getOpcode() == RISCV::ADD) && "Expected ADD instruction!");
220 Register Rs = TailAdd.getOperand(i: 1).getReg();
221 Register Rt = TailAdd.getOperand(i: 2).getReg();
222 Register Reg = Rs == GAReg ? Rt : Rs;
223
224 // Can't fold if the register has more than one use.
225 if (!Reg.isVirtual() || !MRI->hasOneNonDBGUse(RegNo: Reg))
226 return false;
227 // This can point to an ADDI(W) or a LUI:
228 MachineInstr &OffsetTail = *MRI->getVRegDef(Reg);
229 auto OffsetTailOpc = OffsetTail.getOpcode();
230 if (OffsetTailOpc == RISCV::ADDI || OffsetTailOpc == RISCV::ADDIW) {
231 // The offset value has non zero bits in both %hi and %lo parts.
232 // Detect an ADDI that feeds from a LUI instruction.
233 MachineOperand &AddiImmOp = OffsetTail.getOperand(i: 2);
234 if (AddiImmOp.getTargetFlags() != RISCVII::MO_None)
235 return false;
236 // Make sure this isn't something weird like a frame index.
237 if (!OffsetTail.getOperand(i: 1).isReg())
238 return false;
239 Register AddiReg = OffsetTail.getOperand(i: 1).getReg();
240 int64_t OffLo = AddiImmOp.getImm();
241
242 // Handle rs1 of ADDI is X0.
243 if (AddiReg == RISCV::X0) {
244 LLVM_DEBUG(dbgs() << " Offset Instrs: " << OffsetTail);
245 if (!foldOffset(Hi, Lo, Tail&: TailAdd, Offset: OffLo))
246 return false;
247 OffsetTail.eraseFromParent();
248 return true;
249 }
250
251 MachineInstr &OffsetLui = *MRI->getVRegDef(Reg: AddiReg);
252 MachineOperand &LuiImmOp = OffsetLui.getOperand(i: 1);
253 if (OffsetLui.getOpcode() != RISCV::LUI ||
254 LuiImmOp.getTargetFlags() != RISCVII::MO_None ||
255 !MRI->hasOneNonDBGUse(RegNo: OffsetLui.getOperand(i: 0).getReg()))
256 return false;
257 int64_t Offset = SignExtend64<32>(x: LuiImmOp.getImm() << 12);
258 Offset += OffLo;
259 // RV32 ignores the upper 32 bits. ADDIW sign extends the result.
260 if (!ST->is64Bit() || OffsetTailOpc == RISCV::ADDIW)
261 Offset = SignExtend64<32>(x: Offset);
262 // We can only fold simm32 offsets.
263 if (!isInt<32>(x: Offset))
264 return false;
265 LLVM_DEBUG(dbgs() << " Offset Instrs: " << OffsetTail
266 << " " << OffsetLui);
267 if (!foldOffset(Hi, Lo, Tail&: TailAdd, Offset))
268 return false;
269 OffsetTail.eraseFromParent();
270 OffsetLui.eraseFromParent();
271 return true;
272 } else if (OffsetTailOpc == RISCV::LUI) {
273 // The offset value has all zero bits in the lower 12 bits. Only LUI
274 // exists.
275 LLVM_DEBUG(dbgs() << " Offset Instr: " << OffsetTail);
276 int64_t Offset = SignExtend64<32>(x: OffsetTail.getOperand(i: 1).getImm() << 12);
277 if (!foldOffset(Hi, Lo, Tail&: TailAdd, Offset))
278 return false;
279 OffsetTail.eraseFromParent();
280 return true;
281 }
282 return false;
283}
284
285// Detect patterns for offsets that are passed into a SHXADD instruction.
286// The offset has 1, 2, or 3 trailing zeros and fits in simm13, simm14, simm15.
287// The constant is created with addi voff, x0, C, and shXadd is used to
288// fill insert the trailing zeros and do the addition.
289// If the pattern is found, updates the offset in Hi and Lo instructions
290// and deletes TailShXAdd and the instructions that produced the offset.
291//
292// Hi: lui vreg1, %hi(s)
293// Lo: addi vreg2, vreg1, %lo(s)
294// OffsetTail: addi voff, x0, C
295// TailAdd: shXadd vreg4, voff, vreg2
296bool RISCVMergeBaseOffsetOpt::foldShiftedOffset(MachineInstr &Hi,
297 MachineInstr &Lo,
298 MachineInstr &TailShXAdd,
299 Register GAReg) {
300 assert((TailShXAdd.getOpcode() == RISCV::SH1ADD ||
301 TailShXAdd.getOpcode() == RISCV::SH2ADD ||
302 TailShXAdd.getOpcode() == RISCV::SH3ADD) &&
303 "Expected SHXADD instruction!");
304
305 if (GAReg != TailShXAdd.getOperand(i: 2).getReg())
306 return false;
307
308 // The first source is the shifted operand.
309 Register Rs1 = TailShXAdd.getOperand(i: 1).getReg();
310
311 // Can't fold if the register has more than one use.
312 if (!Rs1.isVirtual() || !MRI->hasOneNonDBGUse(RegNo: Rs1))
313 return false;
314 // This can point to an ADDI X0, C.
315 MachineInstr &OffsetTail = *MRI->getVRegDef(Reg: Rs1);
316 if (OffsetTail.getOpcode() != RISCV::ADDI)
317 return false;
318 if (!OffsetTail.getOperand(i: 1).isReg() ||
319 OffsetTail.getOperand(i: 1).getReg() != RISCV::X0 ||
320 !OffsetTail.getOperand(i: 2).isImm())
321 return false;
322
323 int64_t Offset = OffsetTail.getOperand(i: 2).getImm();
324 assert(isInt<12>(Offset) && "Unexpected offset");
325
326 unsigned ShAmt;
327 switch (TailShXAdd.getOpcode()) {
328 default: llvm_unreachable("Unexpected opcode");
329 case RISCV::SH1ADD: ShAmt = 1; break;
330 case RISCV::SH2ADD: ShAmt = 2; break;
331 case RISCV::SH3ADD: ShAmt = 3; break;
332 }
333
334 Offset = (uint64_t)Offset << ShAmt;
335
336 LLVM_DEBUG(dbgs() << " Offset Instr: " << OffsetTail);
337 if (!foldOffset(Hi, Lo, Tail&: TailShXAdd, Offset))
338 return false;
339 OffsetTail.eraseFromParent();
340 return true;
341}
342
343bool RISCVMergeBaseOffsetOpt::detectAndFoldOffset(MachineInstr &Hi,
344 MachineInstr &Lo) {
345 Register DestReg = Lo.getOperand(i: 0).getReg();
346
347 // Look for arithmetic instructions we can get an offset from.
348 // We might be able to remove the arithmetic instructions by folding the
349 // offset into the LUI+ADDI.
350 if (!MRI->hasOneNonDBGUse(RegNo: DestReg))
351 return false;
352
353 // Lo has only one use.
354 MachineInstr &Tail = *MRI->use_instr_nodbg_begin(RegNo: DestReg);
355 switch (Tail.getOpcode()) {
356 default:
357 LLVM_DEBUG(dbgs() << "Don't know how to get offset from this instr: "
358 << Tail);
359 break;
360 case RISCV::ADDI:
361 case RISCV::QC_E_ADDI:
362 case RISCV::QC_E_ADDAI: {
363 // Offset is simply an immediate operand.
364 int64_t Offset = Tail.getOperand(i: 2).getImm();
365 if (Tail.getOpcode() == RISCV::ADDI) {
366 // We might have two ADDIs in a row.
367 Register TailDestReg = Tail.getOperand(i: 0).getReg();
368 if (MRI->hasOneNonDBGUse(RegNo: TailDestReg)) {
369 MachineInstr &TailTail = *MRI->use_instr_nodbg_begin(RegNo: TailDestReg);
370 if (TailTail.getOpcode() == RISCV::ADDI) {
371 Offset += TailTail.getOperand(i: 2).getImm();
372 LLVM_DEBUG(dbgs() << " Offset Instrs: " << Tail << TailTail);
373 if (!foldOffset(Hi, Lo, Tail&: TailTail, Offset))
374 return false;
375 Tail.eraseFromParent();
376 return true;
377 }
378 }
379 }
380
381 LLVM_DEBUG(dbgs() << " Offset Instr: " << Tail);
382 return foldOffset(Hi, Lo, Tail, Offset);
383 }
384 case RISCV::ADD:
385 // The offset is too large to fit in the immediate field of ADDI.
386 // This can be in two forms:
387 // 1) LUI hi_Offset followed by:
388 // ADDI lo_offset
389 // This happens in case the offset has non zero bits in
390 // both hi 20 and lo 12 bits.
391 // 2) LUI (offset20)
392 // This happens in case the lower 12 bits of the offset are zeros.
393 return foldLargeOffset(Hi, Lo, TailAdd&: Tail, GAReg: DestReg);
394 case RISCV::SH1ADD:
395 case RISCV::SH2ADD:
396 case RISCV::SH3ADD:
397 // The offset is too large to fit in the immediate field of ADDI.
398 // It may be encoded as (SH2ADD (ADDI X0, C), DestReg) or
399 // (SH3ADD (ADDI X0, C), DestReg).
400 return foldShiftedOffset(Hi, Lo, TailShXAdd&: Tail, GAReg: DestReg);
401 }
402
403 return false;
404}
405
406static void overwriteMachineOperandInPlace(MachineOperand &MO,
407 const MachineOperand &ImmOp) {
408 switch (ImmOp.getType()) {
409 case MachineOperand::MO_ConstantPoolIndex:
410 MO.ChangeToCPI(Idx: ImmOp.getIndex(), Offset: ImmOp.getOffset(), TargetFlags: ImmOp.getTargetFlags());
411 break;
412 case MachineOperand::MO_GlobalAddress:
413 MO.ChangeToGA(GV: ImmOp.getGlobal(), Offset: ImmOp.getOffset(), TargetFlags: ImmOp.getTargetFlags());
414 break;
415 case MachineOperand::MO_MCSymbol:
416 MO.ChangeToMCSymbol(Sym: ImmOp.getMCSymbol(), TargetFlags: ImmOp.getTargetFlags());
417 MO.setOffset(ImmOp.getOffset());
418 break;
419 case MachineOperand::MO_BlockAddress:
420 MO.ChangeToBA(BA: ImmOp.getBlockAddress(), Offset: ImmOp.getOffset(),
421 TargetFlags: ImmOp.getTargetFlags());
422 break;
423 default:
424 report_fatal_error(reason: "unsupported machine operand type");
425 break;
426 }
427}
428
429bool RISCVMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi,
430 MachineInstr &Lo) {
431 Register DestReg = Lo.getOperand(i: 0).getReg();
432
433 // If all the uses are memory ops with the same offset, we can transform:
434 //
435 // 1. (medlow pattern):
436 // a. Hi: lui vreg1, %hi(foo) ---> lui vreg1, %hi(foo+8)
437 // Lo: addi vreg2, vreg1, %lo(foo) ---> lw vreg3, lo(foo+8)(vreg1)
438 // Tail: lw vreg3, 8(vreg2)
439 //
440 // b. Hi: qc.e.li vreg1, foo ---> qc.e.li vreg1, foo+8
441 // Tail: lw vreg2, 8(vreg1) ---> lw vreg2, 0(vreg1)
442 //
443 // 2. (medany pattern):
444 // Hi: 1:auipc vreg1, %pcrel_hi(s) ---> auipc vreg1, %pcrel_hi(foo+8)
445 // Lo: addi vreg2, vreg1, %pcrel_lo(1b) ---> lw vreg3, %pcrel_lo(1b)(vreg1)
446 // Tail: lw vreg3, 8(vreg2)
447
448 std::optional<int64_t> CommonOffset;
449 DenseMap<const MachineInstr *, SmallVector<unsigned>>
450 InlineAsmMemoryOpIndexesMap;
451 for (const MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg: DestReg)) {
452 switch (UseMI.getOpcode()) {
453 default:
454 LLVM_DEBUG(dbgs() << "Not a load or store instruction: " << UseMI);
455 return false;
456 case RISCV::LB:
457 case RISCV::LH:
458 case RISCV::LH_INX:
459 case RISCV::LW:
460 case RISCV::LW_INX:
461 case RISCV::LBU:
462 case RISCV::LHU:
463 case RISCV::LWU:
464 case RISCV::LD:
465 case RISCV::LD_RV32:
466 case RISCV::FLH:
467 case RISCV::FLW:
468 case RISCV::FLD:
469 case RISCV::SB:
470 case RISCV::SH:
471 case RISCV::SH_INX:
472 case RISCV::SW:
473 case RISCV::SW_INX:
474 case RISCV::SD:
475 case RISCV::SD_RV32:
476 case RISCV::FSH:
477 case RISCV::FSW:
478 case RISCV::FSD: {
479 if (UseMI.getOperand(i: 1).isFI())
480 return false;
481 // Register defined by Lo should not be the value register.
482 if (DestReg == UseMI.getOperand(i: 0).getReg())
483 return false;
484 assert(DestReg == UseMI.getOperand(1).getReg() &&
485 "Expected base address use");
486 // All load/store instructions must use the same offset.
487 int64_t Offset = UseMI.getOperand(i: 2).getImm();
488 if (CommonOffset && Offset != CommonOffset)
489 return false;
490 CommonOffset = Offset;
491 break;
492 }
493 case RISCV::PseudoCCLD:
494 case RISCV::PseudoCCLW:
495 case RISCV::PseudoCCLWU:
496 case RISCV::PseudoCCLH:
497 case RISCV::PseudoCCLHU:
498 case RISCV::PseudoCCLB:
499 case RISCV::PseudoCCLBU: {
500 // The SFB Pseudos are like their non-SFB counterparts but have more
501 // operands.
502 if (UseMI.getOperand(i: 2).isFI())
503 return false;
504 // Register defined by Lo should not be the (tied) false value, or a
505 // register used in the branch predicate.
506 if (DestReg == UseMI.getOperand(i: 1).getReg() ||
507 DestReg == UseMI.getOperand(i: 5).getReg())
508 return false;
509 if (UseMI.getOperand(i: 6).isReg() &&
510 DestReg == UseMI.getOperand(i: 6).getReg())
511 return false;
512 assert(DestReg == UseMI.getOperand(2).getReg() &&
513 "Expected base address use");
514 // All load/store instructions must use the same offset.
515 int64_t Offset = UseMI.getOperand(i: 3).getImm();
516 if (CommonOffset && Offset != CommonOffset)
517 return false;
518 CommonOffset = Offset;
519 break;
520 }
521 case RISCV::INLINEASM:
522 case RISCV::INLINEASM_BR: {
523 SmallVector<unsigned> InlineAsmMemoryOpIndexes;
524 unsigned NumOps = 0;
525 for (unsigned I = InlineAsm::MIOp_FirstOperand;
526 I < UseMI.getNumOperands(); I += 1 + NumOps) {
527 const MachineOperand &FlagsMO = UseMI.getOperand(i: I);
528 // Should be an imm.
529 if (!FlagsMO.isImm())
530 continue;
531
532 const InlineAsm::Flag Flags(FlagsMO.getImm());
533 NumOps = Flags.getNumOperandRegisters();
534
535 // Memory constraints have two operands.
536 if (NumOps != 2 || !Flags.isMemKind()) {
537 // If the register is used by something other than a memory
538 // constraint, we should not fold.
539 for (unsigned J = 0; J < NumOps; ++J) {
540 const MachineOperand &MO = UseMI.getOperand(i: I + 1 + J);
541 if (MO.isReg() && MO.getReg() == DestReg)
542 return false;
543 }
544 continue;
545 }
546
547 // We can't do this for constraint A because AMO instructions don't have
548 // an immediate offset field.
549 if (Flags.getMemoryConstraintID() == InlineAsm::ConstraintCode::A)
550 return false;
551
552 const MachineOperand &AddrMO = UseMI.getOperand(i: I + 1);
553 if (!AddrMO.isReg() || AddrMO.getReg() != DestReg)
554 continue;
555
556 const MachineOperand &OffsetMO = UseMI.getOperand(i: I + 2);
557 if (!OffsetMO.isImm())
558 continue;
559
560 // All inline asm memory operands must use the same offset.
561 int64_t Offset = OffsetMO.getImm();
562 if (CommonOffset && Offset != CommonOffset)
563 return false;
564 CommonOffset = Offset;
565 InlineAsmMemoryOpIndexes.push_back(Elt: I + 1);
566 }
567 InlineAsmMemoryOpIndexesMap.insert(
568 KV: std::make_pair(x: &UseMI, y&: InlineAsmMemoryOpIndexes));
569 break;
570 }
571 }
572 }
573
574 if (!CommonOffset)
575 return false;
576
577 // We found a common offset.
578 // Update the offsets in global address lowering.
579 // We may have already folded some arithmetic so we need to add to any
580 // existing offset.
581 int64_t NewOffset = Hi.getOperand(i: 1).getOffset() + *CommonOffset;
582 // RV32 ignores the upper 32 bits.
583 if (!ST->is64Bit())
584 NewOffset = SignExtend64<32>(x: NewOffset);
585 // We can only fold simm32 offsets.
586 if (!isInt<32>(x: NewOffset))
587 return false;
588
589 // Folding changes the value represented by DestReg, so its debug users can
590 // no longer describe the same address.
591 MRI->markUsesInDebugValueAsUndef(Reg: DestReg);
592
593 Hi.getOperand(i: 1).setOffset(NewOffset);
594 MachineOperand &ImmOp =
595 Hi.getOpcode() == RISCV::QC_E_LI ? Lo.getOperand(i: 1) : Lo.getOperand(i: 2);
596 auto HiOpc = Hi.getOpcode();
597 // Expand PseudoMovAddr into LUI
598 if (HiOpc == RISCV::PseudoMovAddr) {
599 auto *TII = ST->getInstrInfo();
600 Hi.setDesc(TII->get(Opcode: RISCV::LUI));
601 Hi.removeOperand(OpNo: 2);
602 }
603
604 if (HiOpc != RISCV::AUIPC)
605 ImmOp.setOffset(NewOffset);
606
607 // Update the immediate in the load/store instructions to add the offset.
608 for (MachineInstr &UseMI :
609 llvm::make_early_inc_range(Range: MRI->use_nodbg_instructions(Reg: DestReg))) {
610 if (UseMI.getOpcode() == RISCV::INLINEASM ||
611 UseMI.getOpcode() == RISCV::INLINEASM_BR) {
612 auto &InlineAsmMemoryOpIndexes = InlineAsmMemoryOpIndexesMap[&UseMI];
613 for (unsigned I : InlineAsmMemoryOpIndexes) {
614 MachineOperand &MO = UseMI.getOperand(i: I + 1);
615 overwriteMachineOperandInPlace(MO, ImmOp);
616 }
617 } else {
618 unsigned ImmIdx;
619 switch (UseMI.getOpcode()) {
620 case RISCV::INLINEASM:
621 case RISCV::INLINEASM_BR:
622 llvm_unreachable("Should have been dealt with before this else");
623 case RISCV::PseudoCCLD:
624 case RISCV::PseudoCCLW:
625 case RISCV::PseudoCCLWU:
626 case RISCV::PseudoCCLH:
627 case RISCV::PseudoCCLHU:
628 case RISCV::PseudoCCLB:
629 case RISCV::PseudoCCLBU:
630 ImmIdx = 3;
631 break;
632 case RISCV::LB:
633 case RISCV::LH:
634 case RISCV::LH_INX:
635 case RISCV::LW:
636 case RISCV::LW_INX:
637 case RISCV::LBU:
638 case RISCV::LHU:
639 case RISCV::LWU:
640 case RISCV::LD:
641 case RISCV::LD_RV32:
642 case RISCV::FLH:
643 case RISCV::FLW:
644 case RISCV::FLD:
645 case RISCV::SB:
646 case RISCV::SH:
647 case RISCV::SH_INX:
648 case RISCV::SW:
649 case RISCV::SW_INX:
650 case RISCV::SD:
651 case RISCV::SD_RV32:
652 case RISCV::FSH:
653 case RISCV::FSW:
654 case RISCV::FSD:
655 ImmIdx = 2;
656 break;
657 default:
658 llvm_unreachable("Unknown Instruction");
659 }
660
661 MachineOperand &MO = UseMI.getOperand(i: ImmIdx);
662 if (Hi.getOpcode() == RISCV::QC_E_LI) {
663 MO.ChangeToImmediate(ImmVal: 0);
664 } else {
665 overwriteMachineOperandInPlace(MO, ImmOp);
666 }
667 }
668 }
669
670 // Prevent Lo (originally PseudoMovAddr, which is also pointed by Hi) from
671 // being erased
672 if (&Lo == &Hi)
673 return true;
674
675 MRI->replaceRegWith(FromReg: Lo.getOperand(i: 0).getReg(), ToReg: Hi.getOperand(i: 0).getReg());
676 Lo.eraseFromParent();
677 return true;
678}
679
680// Try to fold sequences of the form:
681// Hi/lo: qc.e.li vreg1, s -> qc.e.li vreg1, s+imm
682// TailAdd: shxadd vreg2, vreg3, vreg1 -> deleted
683// Tail: lx vreg4, imm(vreg2) -> qc.lrx vreg4, vreg1, vreg3, (1-7)
684bool RISCVMergeBaseOffsetOpt::foldShxaddIntoScaledMemory(MachineInstr &Hi,
685 MachineInstr &Lo) {
686 if (!ST->hasVendorXqcisls() || ST->is64Bit())
687 return false;
688
689 if (Hi.getOpcode() != RISCV::QC_E_LI)
690 return false;
691
692 Register BaseReg = Hi.getOperand(i: 0).getReg();
693 if (!BaseReg.isVirtual() || !MRI->hasOneUse(RegNo: BaseReg))
694 return false;
695
696 MachineInstr &ShxAdd = *MRI->use_instr_begin(RegNo: BaseReg);
697 unsigned ShxOpc = ShxAdd.getOpcode();
698 unsigned ShAmt = 0;
699 switch (ShxOpc) {
700 default:
701 return false;
702 case RISCV::SH1ADD:
703 ShAmt = 1;
704 break;
705 case RISCV::SH2ADD:
706 ShAmt = 2;
707 break;
708 case RISCV::SH3ADD:
709 ShAmt = 3;
710 break;
711 case RISCV::QC_SHLADD:
712 uint8_t ShlImm = ShxAdd.getOperand(i: 3).getImm();
713 if (ShlImm > 7)
714 return false;
715 ShAmt = ShlImm;
716 break;
717 }
718
719 // shxadd Rd, Rs1, Rs2
720 Register ScaledReg = ShxAdd.getOperand(i: 0).getReg();
721 Register IndexReg = ShxAdd.getOperand(i: 1).getReg();
722
723 if (!IndexReg.isVirtual())
724 return false;
725
726 if (ShxAdd.getOperand(i: 2).getReg() != BaseReg)
727 return false;
728
729 if (!ScaledReg.isVirtual() || !MRI->hasOneUse(RegNo: ScaledReg))
730 return false;
731
732 MachineInstr &TailMem = *MRI->use_instr_begin(RegNo: ScaledReg);
733 unsigned Opc = TailMem.getOpcode();
734 unsigned NewOpc = 0;
735
736 switch (Opc) {
737 case RISCV::LB:
738 NewOpc = RISCV::QC_LRB;
739 break;
740 case RISCV::LBU:
741 NewOpc = RISCV::QC_LRBU;
742 break;
743 case RISCV::LH:
744 NewOpc = RISCV::QC_LRH;
745 break;
746 case RISCV::LHU:
747 NewOpc = RISCV::QC_LRHU;
748 break;
749 case RISCV::LW:
750 NewOpc = RISCV::QC_LRW;
751 break;
752 case RISCV::SB:
753 NewOpc = RISCV::QC_SRB;
754 break;
755 case RISCV::SH:
756 NewOpc = RISCV::QC_SRH;
757 break;
758 case RISCV::SW:
759 NewOpc = RISCV::QC_SRW;
760 break;
761 default:
762 return false;
763 }
764
765 if (!TailMem.getOperand(i: 1).isReg() ||
766 TailMem.getOperand(i: 1).getReg() != ScaledReg)
767 return false;
768 if (!TailMem.getOperand(i: 2).isImm())
769 return false;
770 int64_t Imm = TailMem.getOperand(i: 2).getImm();
771
772 // Update QC_E_LI offset.
773 int64_t NewOffset = SignExtend64<32>(x: Hi.getOperand(i: 1).getOffset() + Imm);
774
775 Hi.getOperand(i: 1).setOffset(NewOffset);
776
777 // Build scaled load/store.
778 auto *TII = ST->getInstrInfo();
779 auto *MBB = TailMem.getParent();
780
781 // Ensure index register satisfies GPRNoX0 class required by QC_LR*/QC_SR*.
782 MRI->constrainRegClass(Reg: IndexReg, RC: &RISCV::GPRNoX0RegClass);
783
784 BuildMI(BB&: *MBB, I&: TailMem, MIMD: TailMem.getDebugLoc(), MCID: TII->get(Opcode: NewOpc))
785 .add(MO: TailMem.getOperand(i: 0))
786 .addReg(RegNo: BaseReg, Flags: getKillRegState(B: ShxAdd.getOperand(i: 2).isKill()))
787 .addReg(RegNo: IndexReg, Flags: getKillRegState(B: ShxAdd.getOperand(i: 1).isKill()))
788 .addImm(Val: ShAmt)
789 .cloneMemRefs(OtherMI: TailMem);
790
791 TailMem.eraseFromParent();
792 ShxAdd.eraseFromParent();
793 return true;
794}
795
796bool RISCVMergeBaseOffsetOpt::runOnMachineFunction(MachineFunction &Fn) {
797 if (skipFunction(F: Fn.getFunction()))
798 return false;
799
800 ST = &Fn.getSubtarget<RISCVSubtarget>();
801
802 bool MadeChange = false;
803 MRI = &Fn.getRegInfo();
804 for (MachineBasicBlock &MBB : Fn) {
805 LLVM_DEBUG(dbgs() << "MBB: " << MBB.getName() << "\n");
806 for (MachineInstr &Hi : MBB) {
807 MachineInstr *Lo = nullptr;
808 if (!detectFoldable(Hi, Lo))
809 continue;
810 MadeChange |= detectAndFoldOffset(Hi, Lo&: *Lo);
811 MadeChange |= foldIntoMemoryOps(Hi, Lo&: *Lo);
812 MadeChange |= foldShxaddIntoScaledMemory(Hi, Lo&: *Lo);
813 }
814 }
815
816 return MadeChange;
817}
818
819/// Returns an instance of the Merge Base Offset Optimization pass.
820FunctionPass *llvm::createRISCVMergeBaseOffsetOptPass() {
821 return new RISCVMergeBaseOffsetOpt();
822}
823