1//===---- LoongArchMergeBaseOffset.cpp - Optimise address calculations ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Merge the offset of address calculation into the offset field
10// of instructions in a global address lowering sequence.
11//
12//===----------------------------------------------------------------------===//
13
14#include "LoongArch.h"
15#include "LoongArchTargetMachine.h"
16#include "llvm/CodeGen/MachineFunctionPass.h"
17#include "llvm/CodeGen/Passes.h"
18#include "llvm/MC/TargetRegistry.h"
19#include "llvm/Support/Debug.h"
20#include "llvm/Target/TargetOptions.h"
21#include <optional>
22
23using namespace llvm;
24
25#define DEBUG_TYPE "loongarch-merge-base-offset"
26#define LoongArch_MERGE_BASE_OFFSET_NAME "LoongArch Merge Base Offset"
27
28namespace {
29
30class LoongArchMergeBaseOffsetOpt : public MachineFunctionPass {
31 const LoongArchSubtarget *ST = nullptr;
32 MachineRegisterInfo *MRI;
33
34public:
35 static char ID;
36 bool runOnMachineFunction(MachineFunction &Fn) override;
37 bool detectFoldable(MachineInstr &Hi20, MachineInstr *&Lo12,
38 MachineInstr *&Lo20, MachineInstr *&Hi12,
39 MachineInstr *&Last);
40 bool detectFoldable(MachineInstr &Hi20, MachineInstr *&Add,
41 MachineInstr *&Lo12);
42
43 bool detectAndFoldOffset(MachineInstr &Hi20, MachineInstr &Lo12,
44 MachineInstr *&Lo20, MachineInstr *&Hi12,
45 MachineInstr *&Last);
46 void foldOffset(MachineInstr &Hi20, MachineInstr &Lo12, MachineInstr *&Lo20,
47 MachineInstr *&Hi12, MachineInstr *&Last, MachineInstr &Tail,
48 int64_t Offset);
49 bool foldLargeOffset(MachineInstr &Hi20, MachineInstr &Lo12,
50 MachineInstr *&Lo20, MachineInstr *&Hi12,
51 MachineInstr *&Last, MachineInstr &TailAdd,
52 Register GAReg);
53
54 bool foldIntoMemoryOps(MachineInstr &Hi20, MachineInstr &Lo12,
55 MachineInstr *&Lo20, MachineInstr *&Hi12,
56 MachineInstr *&Last);
57
58 LoongArchMergeBaseOffsetOpt() : MachineFunctionPass(ID) {}
59
60 MachineFunctionProperties getRequiredProperties() const override {
61 return MachineFunctionProperties().setIsSSA();
62 }
63
64 void getAnalysisUsage(AnalysisUsage &AU) const override {
65 AU.setPreservesCFG();
66 MachineFunctionPass::getAnalysisUsage(AU);
67 }
68
69 StringRef getPassName() const override {
70 return LoongArch_MERGE_BASE_OFFSET_NAME;
71 }
72};
73} // end anonymous namespace
74
75char LoongArchMergeBaseOffsetOpt::ID = 0;
76INITIALIZE_PASS(LoongArchMergeBaseOffsetOpt, DEBUG_TYPE,
77 LoongArch_MERGE_BASE_OFFSET_NAME, false, false)
78
79static inline bool isPCAddLo(unsigned Flags) {
80 switch (Flags) {
81 case LoongArchII::MO_PCADD_LO:
82 case LoongArchII::MO_GOT_PCADD_LO:
83 case LoongArchII::MO_IE_PCADD_LO:
84 case LoongArchII::MO_LD_PCADD_LO:
85 case LoongArchII::MO_GD_PCADD_LO:
86 case LoongArchII::MO_DESC_PCADD_LO:
87 return true;
88 }
89 return false;
90}
91
92// Detect either of the patterns:
93//
94// 1. (small/medium):
95// pcaxxu12i vreg1, %pc_hi20(s)
96// addi.d vreg2, vreg1, %pc_lo12(s)
97//
98// 2. (large):
99// pcalau12i vreg1, %pc_hi20(s)
100// addi.d vreg2, $zero, %pc_lo12(s)
101// lu32i.d vreg3, vreg2, %pc64_lo20(s)
102// lu52i.d vreg4, vreg3, %pc64_hi12(s)
103// add.d vreg5, vreg4, vreg1
104
105// The pattern is only accepted if:
106// 1) For small and medium pattern, the first instruction has only one use,
107// which is the ADDI.
108// 2) For large pattern, the first four instructions each have only one use,
109// and the user of the fourth instruction is ADD.
110// 3) The address operands have the appropriate type, reflecting the
111// lowering of a global address or constant pool using the pattern.
112// 4) The offset value in the Global Address or Constant Pool is 0.
113bool LoongArchMergeBaseOffsetOpt::detectFoldable(MachineInstr &Hi20,
114 MachineInstr *&Lo12,
115 MachineInstr *&Lo20,
116 MachineInstr *&Hi12,
117 MachineInstr *&Last) {
118 if (Hi20.getOpcode() != LoongArch::PCALAU12I &&
119 Hi20.getOpcode() != LoongArch::PCADDU12I)
120 return false;
121
122 const MachineOperand &Hi20Op1 = Hi20.getOperand(i: 1);
123 if (LoongArchII::getDirectFlags(MO: Hi20Op1) != LoongArchII::MO_PCREL_HI &&
124 LoongArchII::getDirectFlags(MO: Hi20Op1) != LoongArchII::MO_PCADD_HI)
125 return false;
126
127 auto isGlobalOrCPIOrBlockAddress = [](const MachineOperand &Op) {
128 return Op.isGlobal() || Op.isCPI() || Op.isBlockAddress();
129 };
130
131 if (!isGlobalOrCPIOrBlockAddress(Hi20Op1) || Hi20Op1.getOffset() != 0)
132 return false;
133
134 Register HiDestReg = Hi20.getOperand(i: 0).getReg();
135 if (!MRI->hasOneUse(RegNo: HiDestReg))
136 return false;
137
138 MachineInstr *UseInst = &*MRI->use_instr_begin(RegNo: HiDestReg);
139 if (UseInst->getOpcode() != LoongArch::ADD_D) {
140 Lo12 = UseInst;
141 if ((ST->is64Bit() && Lo12->getOpcode() != LoongArch::ADDI_D) ||
142 (!ST->is64Bit() && Lo12->getOpcode() != LoongArch::ADDI_W))
143 return false;
144 } else {
145 assert(ST->is64Bit());
146 Last = UseInst;
147
148 Register LastOp1Reg = Last->getOperand(i: 1).getReg();
149 if (!LastOp1Reg.isVirtual())
150 return false;
151 Hi12 = MRI->getVRegDef(Reg: LastOp1Reg);
152 const MachineOperand &Hi12Op2 = Hi12->getOperand(i: 2);
153 if (Hi12Op2.getTargetFlags() != LoongArchII::MO_PCREL64_HI)
154 return false;
155 if (!isGlobalOrCPIOrBlockAddress(Hi12Op2) || Hi12Op2.getOffset() != 0)
156 return false;
157 if (!MRI->hasOneUse(RegNo: Hi12->getOperand(i: 0).getReg()))
158 return false;
159
160 Lo20 = MRI->getVRegDef(Reg: Hi12->getOperand(i: 1).getReg());
161 const MachineOperand &Lo20Op2 = Lo20->getOperand(i: 2);
162 if (Lo20Op2.getTargetFlags() != LoongArchII::MO_PCREL64_LO)
163 return false;
164 if (!isGlobalOrCPIOrBlockAddress(Lo20Op2) || Lo20Op2.getOffset() != 0)
165 return false;
166 if (!MRI->hasOneUse(RegNo: Lo20->getOperand(i: 0).getReg()))
167 return false;
168
169 Lo12 = MRI->getVRegDef(Reg: Lo20->getOperand(i: 1).getReg());
170 if (!MRI->hasOneUse(RegNo: Lo12->getOperand(i: 0).getReg()))
171 return false;
172 }
173
174 const MachineOperand &Lo12Op2 = Lo12->getOperand(i: 2);
175 assert(Hi20.getOpcode() == LoongArch::PCALAU12I ||
176 Hi20.getOpcode() == LoongArch::PCADDU12I);
177 if ((LoongArchII::getDirectFlags(MO: Lo12Op2) != LoongArchII::MO_PCREL_LO &&
178 !isPCAddLo(Flags: LoongArchII::getDirectFlags(MO: Lo12Op2))) ||
179 !(isGlobalOrCPIOrBlockAddress(Lo12Op2) || Lo12Op2.isMCSymbol()) ||
180 Lo12Op2.getOffset() != 0)
181 return false;
182
183 if (Hi20Op1.isGlobal()) {
184 LLVM_DEBUG(dbgs() << " Found lowered global address: "
185 << *Hi20Op1.getGlobal() << "\n");
186 } else if (Hi20Op1.isBlockAddress()) {
187 LLVM_DEBUG(dbgs() << " Found lowered basic address: "
188 << *Hi20Op1.getBlockAddress() << "\n");
189 } else if (Hi20Op1.isCPI()) {
190 LLVM_DEBUG(dbgs() << " Found lowered constant pool: " << Hi20Op1.getIndex()
191 << "\n");
192 }
193
194 return true;
195}
196
197// Detect the pattern:
198//
199// (small/medium):
200// lu12i.w vreg1, %le_hi20_r(s)
201// add.w/d vreg2, vreg1, r2, %le_add_r(s)
202// addi.w/d vreg3, vreg2, %le_lo12_r(s)
203
204// The pattern is only accepted if:
205// 1) The first instruction has only one use, which is the PseudoAddTPRel.
206// The second instruction has only one use, which is the ADDI. The
207// second instruction's last operand is the tp register.
208// 2) The address operands have the appropriate type, reflecting the
209// lowering of a thread_local global address using the pattern.
210// 3) The offset value in the ThreadLocal Global Address is 0.
211bool LoongArchMergeBaseOffsetOpt::detectFoldable(MachineInstr &Hi20,
212 MachineInstr *&Add,
213 MachineInstr *&Lo12) {
214 if (Hi20.getOpcode() != LoongArch::LU12I_W)
215 return false;
216
217 auto isGlobalOrCPI = [](const MachineOperand &Op) {
218 return Op.isGlobal() || Op.isCPI();
219 };
220
221 const MachineOperand &Hi20Op1 = Hi20.getOperand(i: 1);
222 if (LoongArchII::getDirectFlags(MO: Hi20Op1) != LoongArchII::MO_LE_HI_R ||
223 !isGlobalOrCPI(Hi20Op1) || Hi20Op1.getOffset() != 0)
224 return false;
225
226 Register HiDestReg = Hi20.getOperand(i: 0).getReg();
227 if (!MRI->hasOneUse(RegNo: HiDestReg))
228 return false;
229
230 Add = &*MRI->use_instr_begin(RegNo: HiDestReg);
231 if ((ST->is64Bit() && Add->getOpcode() != LoongArch::PseudoAddTPRel_D) ||
232 (!ST->is64Bit() && Add->getOpcode() != LoongArch::PseudoAddTPRel_W))
233 return false;
234
235 if (Add->getOperand(i: 2).getReg() != LoongArch::R2)
236 return false;
237
238 const MachineOperand &AddOp3 = Add->getOperand(i: 3);
239 if (LoongArchII::getDirectFlags(MO: AddOp3) != LoongArchII::MO_LE_ADD_R ||
240 !(isGlobalOrCPI(AddOp3) || AddOp3.isMCSymbol()) ||
241 AddOp3.getOffset() != 0)
242 return false;
243
244 Register AddDestReg = Add->getOperand(i: 0).getReg();
245 if (!MRI->hasOneUse(RegNo: AddDestReg))
246 return false;
247
248 Lo12 = &*MRI->use_instr_begin(RegNo: AddDestReg);
249 if ((ST->is64Bit() && Lo12->getOpcode() != LoongArch::ADDI_D) ||
250 (!ST->is64Bit() && Lo12->getOpcode() != LoongArch::ADDI_W))
251 return false;
252
253 const MachineOperand &Lo12Op2 = Lo12->getOperand(i: 2);
254 if (LoongArchII::getDirectFlags(MO: Lo12Op2) != LoongArchII::MO_LE_LO_R ||
255 !(isGlobalOrCPI(Lo12Op2) || Lo12Op2.isMCSymbol()) ||
256 Lo12Op2.getOffset() != 0)
257 return false;
258
259 if (Hi20Op1.isGlobal()) {
260 LLVM_DEBUG(dbgs() << " Found lowered global address: "
261 << *Hi20Op1.getGlobal() << "\n");
262 } else if (Hi20Op1.isCPI()) {
263 LLVM_DEBUG(dbgs() << " Found lowered constant pool: " << Hi20Op1.getIndex()
264 << "\n");
265 }
266
267 return true;
268}
269
270// Update the offset in Hi20, (Add), Lo12, (Lo20 and Hi12) instructions.
271// Delete the tail instruction and update all the uses to use the
272// output from Last.
273void LoongArchMergeBaseOffsetOpt::foldOffset(
274 MachineInstr &Hi20, MachineInstr &Lo12, MachineInstr *&Lo20,
275 MachineInstr *&Hi12, MachineInstr *&Last, MachineInstr &Tail,
276 int64_t Offset) {
277 // Put the offset back in Hi and the Lo
278 MachineOperand &Lo12Op2 = Lo12.getOperand(i: 2);
279 Hi20.getOperand(i: 1).setOffset(Offset);
280 if (!isPCAddLo(Flags: LoongArchII::getDirectFlags(MO: Lo12Op2)))
281 Lo12Op2.setOffset(Offset);
282 if (Lo20 && Hi12) {
283 Lo20->getOperand(i: 2).setOffset(Offset);
284 Hi12->getOperand(i: 2).setOffset(Offset);
285 }
286
287 // For tls-le, offset of the second PseudoAddTPRel instr should also be
288 // updated.
289 MachineInstr *Add = &*MRI->use_instr_begin(RegNo: Hi20.getOperand(i: 0).getReg());
290 if (Hi20.getOpcode() == LoongArch::LU12I_W)
291 Add->getOperand(i: 3).setOffset(Offset);
292
293 // Delete the tail instruction.
294 MachineInstr *Def = Last ? Last : &Lo12;
295 MRI->constrainRegClass(Reg: Def->getOperand(i: 0).getReg(),
296 RC: MRI->getRegClass(Reg: Tail.getOperand(i: 0).getReg()));
297 MRI->replaceRegWith(FromReg: Tail.getOperand(i: 0).getReg(), ToReg: Def->getOperand(i: 0).getReg());
298 Tail.eraseFromParent();
299
300 LLVM_DEBUG(dbgs() << " Merged offset " << Offset << " into base.\n"
301 << " " << Hi20;);
302 if (Hi20.getOpcode() == LoongArch::LU12I_W) {
303 LLVM_DEBUG(dbgs() << " " << *Add;);
304 }
305 LLVM_DEBUG(dbgs() << " " << Lo12;);
306 if (Lo20 && Hi12) {
307 LLVM_DEBUG(dbgs() << " " << *Lo20 << " " << *Hi12;);
308 }
309}
310
311// Detect patterns for large offsets that are passed into an ADD instruction.
312// If the pattern is found, updates the offset in Hi20, (Add), Lo12,
313// (Lo20 and Hi12) instructions and deletes TailAdd and the instructions that
314// produced the offset.
315//
316// (The instructions marked with "!" are not necessarily present)
317//
318// Base address lowering is of the form:
319// 1) pcala:
320// Hi20: pcaxxu12i vreg1, %pc_hi20(s)
321// +--- Lo12: addi.d vreg2, vreg1, %pc_lo12(s)
322// | Lo20: lu32i.d vreg2, %pc64_lo20(s) !
323// +--- Hi12: lu52i.d vreg2, vreg2, %pc64_hi12(s) !
324// |
325// | 2) tls-le:
326// | Hi20: lu12i.w vreg1, %le_hi20_r(s)
327// | Add: add.w/d vreg1, vreg1, r2, %le_add_r(s)
328// +--- Lo12: addi.w/d vreg2, vreg1, %le_lo12_r(s)
329// |
330// | The large offset can be one of the forms:
331// |
332// +-> 1) Offset that has non zero bits in Hi20 and Lo12 bits:
333// | OffsetHi20: lu12i.w vreg3, 4
334// | OffsetLo12: ori voff, vreg3, 188 ------------------+
335// | |
336// +-> 2) Offset that has non zero bits in Hi20 bits only: |
337// | OffsetHi20: lu12i.w voff, 128 ------------------+
338// | |
339// +-> 3) Offset that has non zero bits in Lo20 bits: |
340// | OffsetHi20: lu12i.w vreg3, 121 ! |
341// | OffsetLo12: ori voff, vreg3, 122 ! |
342// | OffsetLo20: lu32i.d voff, 123 ------------------+
343// +-> 4) Offset that has non zero bits in Hi12 bits: |
344// | OffsetHi20: lu12i.w vreg3, 121 ! |
345// | OffsetLo12: ori voff, vreg3, 122 ! |
346// | OffsetLo20: lu32i.d vreg3, 123 ! |
347// | OffsetHi12: lu52i.d voff, vreg3, 124 -----------------+
348// +-> 5) Offset that is a field shifted into place: |
349// lu12i.w voff, 4 |
350// slli.d voff, voff, 20 ------------------+
351// |
352// TailAdd: add.d vreg4, vreg2, voff <------------------+
353//
354bool LoongArchMergeBaseOffsetOpt::foldLargeOffset(
355 MachineInstr &Hi20, MachineInstr &Lo12, MachineInstr *&Lo20,
356 MachineInstr *&Hi12, MachineInstr *&Last, MachineInstr &TailAdd,
357 Register GAReg) {
358 assert((TailAdd.getOpcode() == LoongArch::ADD_W ||
359 TailAdd.getOpcode() == LoongArch::ADD_D) &&
360 "Expected ADD instruction!");
361 Register Rs = TailAdd.getOperand(i: 1).getReg();
362 Register Rt = TailAdd.getOperand(i: 2).getReg();
363 Register Reg = Rs == GAReg ? Rt : Rs;
364 SmallVector<MachineInstr *, 4> Instrs;
365
366 // Walk backwards through the single-use chain that materializes the offset.
367 // This can point to one of [ORI, LU12I.W, LU32I.D, LU52I.D, SLLI.D]:
368 for (int I = 0; I < 4; I++) {
369 // Handle Reg is R0.
370 if (Reg == LoongArch::R0)
371 break;
372
373 // Can't fold if the register has more than one non-debug use.
374 if (!Reg.isVirtual() || !MRI->hasOneNonDBGUse(RegNo: Reg))
375 return false;
376
377 MachineInstr *Curr = MRI->getVRegDef(Reg);
378 if (!Curr)
379 break;
380
381 switch (Curr->getOpcode()) {
382 default:
383 // Can't fold if the instruction opcode is unexpected.
384 return false;
385 case LoongArch::ORI:
386 if (Curr->getOperand(i: 2).getTargetFlags() != LoongArchII::MO_None)
387 return false;
388 Reg = Curr->getOperand(i: 1).getReg();
389 break;
390 case LoongArch::LU12I_W:
391 if (Curr->getOperand(i: 1).getTargetFlags() != LoongArchII::MO_None)
392 return false;
393 Reg = LoongArch::R0;
394 break;
395 case LoongArch::LU32I_D:
396 if (Curr->getOperand(i: 2).getTargetFlags() != LoongArchII::MO_None || !Lo20)
397 return false;
398 Reg = Curr->getOperand(i: 1).getReg();
399 break;
400 case LoongArch::LU52I_D:
401 if (Curr->getOperand(i: 2).getTargetFlags() != LoongArchII::MO_None || !Hi12)
402 return false;
403 Reg = Curr->getOperand(i: 1).getReg();
404 break;
405 case LoongArch::SLLI_D:
406 if (Curr->getOperand(i: 2).getTargetFlags() != LoongArchII::MO_None ||
407 !(Lo20 && Hi12))
408 return false;
409 Reg = Curr->getOperand(i: 1).getReg();
410 break;
411 }
412 Instrs.push_back(Elt: Curr);
413 }
414
415 // Can't fold if the current register does not reach R0.
416 if (Reg != LoongArch::R0)
417 return false;
418
419 // Compute the offset by evaluating the chain in program order with the exact
420 // semantics of each instruction. The chain is not necessarily in the
421 // canonical form produced by LoongArchMatInt (e.g. when part of the constant
422 // is materialized in another basic block), so the instructions may write
423 // overlapping bits.
424 uint64_t Val = 0;
425 for (MachineInstr *MI : reverse(C&: Instrs)) {
426 switch (MI->getOpcode()) {
427 default:
428 llvm_unreachable("Unexpected opcode");
429 case LoongArch::ORI:
430 Val |= static_cast<uint64_t>(MI->getOperand(i: 2).getImm()) & 0xFFF;
431 break;
432 case LoongArch::LU12I_W:
433 Val = SignExtend64<32>(x: static_cast<uint64_t>(MI->getOperand(i: 1).getImm())
434 << 12);
435 break;
436 case LoongArch::LU32I_D:
437 Val = (Val & 0xFFFFFFFF) |
438 SignExtend64<52>(x: static_cast<uint64_t>(MI->getOperand(i: 2).getImm())
439 << 32);
440 break;
441 case LoongArch::LU52I_D:
442 Val = (Val & 0x000FFFFFFFFFFFFF) |
443 (static_cast<uint64_t>(MI->getOperand(i: 2).getImm()) << 52);
444 break;
445 case LoongArch::SLLI_D:
446 Val <<= MI->getOperand(i: 2).getImm() & 0x3F;
447 break;
448 }
449 }
450 int64_t Offset = Val;
451
452 // Can't fold if the offset is not extracted.
453 if (!Offset)
454 return false;
455
456 foldOffset(Hi20, Lo12, Lo20, Hi12, Last, Tail&: TailAdd, Offset);
457 LLVM_DEBUG(dbgs() << " Offset Instrs:\n");
458 for (auto I : Instrs) {
459 LLVM_DEBUG(dbgs() << " " << *I);
460 // The offset instruction is about to be erased, so its debug users can no
461 // longer be described.
462 MRI->markUsesInDebugValueAsUndef(Reg: I->getOperand(i: 0).getReg());
463 I->eraseFromParent();
464 }
465
466 return true;
467}
468
469bool LoongArchMergeBaseOffsetOpt::detectAndFoldOffset(MachineInstr &Hi20,
470 MachineInstr &Lo12,
471 MachineInstr *&Lo20,
472 MachineInstr *&Hi12,
473 MachineInstr *&Last) {
474 Register DestReg =
475 Last ? Last->getOperand(i: 0).getReg() : Lo12.getOperand(i: 0).getReg();
476
477 // Look for arithmetic instructions we can get an offset from.
478 // We might be able to remove the arithmetic instructions by folding the
479 // offset into the PCAXXU12I+(ADDI/ADDI+LU32I+LU52I) or
480 // LU12I_W+PseudoAddTPRel+ADDI.
481 if (!MRI->hasOneUse(RegNo: DestReg))
482 return false;
483
484 // DestReg has only one use.
485 MachineInstr &Tail = *MRI->use_instr_begin(RegNo: DestReg);
486 switch (Tail.getOpcode()) {
487 default:
488 LLVM_DEBUG(dbgs() << "Don't know how to get offset from this instr:"
489 << Tail);
490 break;
491 case LoongArch::ADDI_W:
492 if (ST->is64Bit())
493 return false;
494 [[fallthrough]];
495 case LoongArch::ADDI_D:
496 case LoongArch::ADDU16I_D: {
497 // Offset is simply an immediate operand.
498 int64_t Offset = Tail.getOperand(i: 2).getImm();
499 if (Tail.getOpcode() == LoongArch::ADDU16I_D)
500 Offset = SignExtend64<32>(x: static_cast<uint64_t>(Offset) << 16);
501
502 // We might have two ADDIs in a row.
503 Register TailDestReg = Tail.getOperand(i: 0).getReg();
504 if (MRI->hasOneUse(RegNo: TailDestReg)) {
505 MachineInstr &TailTail = *MRI->use_instr_begin(RegNo: TailDestReg);
506 if (ST->is64Bit() && TailTail.getOpcode() == LoongArch::ADDI_W)
507 return false;
508 if (TailTail.getOpcode() == LoongArch::ADDI_W ||
509 TailTail.getOpcode() == LoongArch::ADDI_D) {
510 Offset += TailTail.getOperand(i: 2).getImm();
511 LLVM_DEBUG(dbgs() << " Offset Instrs: " << Tail << TailTail);
512 foldOffset(Hi20, Lo12, Lo20, Hi12, Last, Tail&: TailTail, Offset);
513 Tail.eraseFromParent();
514 return true;
515 }
516 }
517
518 LLVM_DEBUG(dbgs() << " Offset Instr: " << Tail);
519 foldOffset(Hi20, Lo12, Lo20, Hi12, Last, Tail, Offset);
520 return true;
521 }
522 case LoongArch::ADD_W:
523 if (ST->is64Bit())
524 return false;
525 [[fallthrough]];
526 case LoongArch::ADD_D:
527 // The offset is too large to fit in the immediate field of ADDI.
528 return foldLargeOffset(Hi20, Lo12, Lo20, Hi12, Last, TailAdd&: Tail, GAReg: DestReg);
529 break;
530 }
531
532 return false;
533}
534
535// Memory access opcode mapping for transforms.
536static unsigned getNewOpc(unsigned Op, bool isLarge) {
537 switch (Op) {
538 case LoongArch::LD_B:
539 return isLarge ? LoongArch::LDX_B : LoongArch::LD_B;
540 case LoongArch::LD_H:
541 return isLarge ? LoongArch::LDX_H : LoongArch::LD_H;
542 case LoongArch::LD_W:
543 case LoongArch::LDPTR_W:
544 return isLarge ? LoongArch::LDX_W : LoongArch::LD_W;
545 case LoongArch::LD_D:
546 case LoongArch::LDPTR_D:
547 return isLarge ? LoongArch::LDX_D : LoongArch::LD_D;
548 case LoongArch::LD_BU:
549 return isLarge ? LoongArch::LDX_BU : LoongArch::LD_BU;
550 case LoongArch::LD_HU:
551 return isLarge ? LoongArch::LDX_HU : LoongArch::LD_HU;
552 case LoongArch::LD_WU:
553 return isLarge ? LoongArch::LDX_WU : LoongArch::LD_WU;
554 case LoongArch::FLD_S:
555 return isLarge ? LoongArch::FLDX_S : LoongArch::FLD_S;
556 case LoongArch::FLD_D:
557 return isLarge ? LoongArch::FLDX_D : LoongArch::FLD_D;
558 case LoongArch::VLD:
559 return isLarge ? LoongArch::VLDX : LoongArch::VLD;
560 case LoongArch::XVLD:
561 return isLarge ? LoongArch::XVLDX : LoongArch::XVLD;
562 case LoongArch::VLDREPL_B:
563 return LoongArch::VLDREPL_B;
564 case LoongArch::XVLDREPL_B:
565 return LoongArch::XVLDREPL_B;
566 case LoongArch::ST_B:
567 return isLarge ? LoongArch::STX_B : LoongArch::ST_B;
568 case LoongArch::ST_H:
569 return isLarge ? LoongArch::STX_H : LoongArch::ST_H;
570 case LoongArch::ST_W:
571 case LoongArch::STPTR_W:
572 return isLarge ? LoongArch::STX_W : LoongArch::ST_W;
573 case LoongArch::ST_D:
574 case LoongArch::STPTR_D:
575 return isLarge ? LoongArch::STX_D : LoongArch::ST_D;
576 case LoongArch::FST_S:
577 return isLarge ? LoongArch::FSTX_S : LoongArch::FST_S;
578 case LoongArch::FST_D:
579 return isLarge ? LoongArch::FSTX_D : LoongArch::FST_D;
580 case LoongArch::VST:
581 return isLarge ? LoongArch::VSTX : LoongArch::VST;
582 case LoongArch::XVST:
583 return isLarge ? LoongArch::XVSTX : LoongArch::XVST;
584 default:
585 llvm_unreachable("Unexpected opcode for replacement");
586 }
587}
588
589bool LoongArchMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi20,
590 MachineInstr &Lo12,
591 MachineInstr *&Lo20,
592 MachineInstr *&Hi12,
593 MachineInstr *&Last) {
594 Register DestReg =
595 Last ? Last->getOperand(i: 0).getReg() : Lo12.getOperand(i: 0).getReg();
596
597 // If all the uses are memory ops with the same offset, we can transform:
598 //
599 // 1. (small/medium):
600 // 1.1. pcala
601 // pcaxxu12i vreg1, %pc_hi20(s)
602 // addi.d vreg2, vreg1, %pc_lo12(s)
603 // ld.w vreg3, 8(vreg2)
604 //
605 // =>
606 //
607 // pcalxx12i vreg1, %pc_hi20(s+8)
608 // ld.w vreg3, vreg1, %pc_lo12(s+8)(vreg1)
609 //
610 // 1.2. tls-le
611 // lu12i.w vreg1, %le_hi20_r(s)
612 // add.w/d vreg2, vreg1, r2, %le_add_r(s)
613 // addi.w/d vreg3, vreg2, %le_lo12_r(s)
614 // ld.w vreg4, 8(vreg3)
615 //
616 // =>
617 //
618 // lu12i.w vreg1, %le_hi20_r(s+8)
619 // add.w/d vreg2, vreg1, r2, %le_add_r(s+8)
620 // ld.w vreg4, vreg2, %le_lo12_r(s+8)(vreg2)
621 //
622 // 2. (large):
623 // pcalau12i vreg1, %pc_hi20(s)
624 // addi.d vreg2, $zero, %pc_lo12(s)
625 // lu32i.d vreg3, vreg2, %pc64_lo20(s)
626 // lu52i.d vreg4, vreg3, %pc64_hi12(s)
627 // add.d vreg5, vreg4, vreg1
628 // ld.w vreg6, 8(vreg5)
629 //
630 // =>
631 //
632 // pcalau12i vreg1, %pc_hi20(s+8)
633 // addi.d vreg2, $zero, %pc_lo12(s+8)
634 // lu32i.d vreg3, vreg2, %pc64_lo20(s+8)
635 // lu52i.d vreg4, vreg3, %pc64_hi12(s+8)
636 // ldx.w vreg6, vreg4, vreg1
637
638 std::optional<int64_t> CommonOffset;
639 DenseMap<const MachineInstr *, SmallVector<unsigned>>
640 InlineAsmMemoryOpIndexesMap;
641 for (const MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg: DestReg)) {
642 switch (UseMI.getOpcode()) {
643 default:
644 LLVM_DEBUG(dbgs() << "Not a load or store instruction: " << UseMI);
645 return false;
646 case LoongArch::VLDREPL_B:
647 case LoongArch::XVLDREPL_B:
648 // We can't do this for large pattern.
649 if (Last)
650 return false;
651 [[fallthrough]];
652 case LoongArch::LD_B:
653 case LoongArch::LD_H:
654 case LoongArch::LD_W:
655 case LoongArch::LD_D:
656 case LoongArch::LD_BU:
657 case LoongArch::LD_HU:
658 case LoongArch::LD_WU:
659 case LoongArch::LDPTR_W:
660 case LoongArch::LDPTR_D:
661 case LoongArch::FLD_S:
662 case LoongArch::FLD_D:
663 case LoongArch::VLD:
664 case LoongArch::XVLD:
665 case LoongArch::ST_B:
666 case LoongArch::ST_H:
667 case LoongArch::ST_W:
668 case LoongArch::ST_D:
669 case LoongArch::STPTR_W:
670 case LoongArch::STPTR_D:
671 case LoongArch::FST_S:
672 case LoongArch::FST_D:
673 case LoongArch::VST:
674 case LoongArch::XVST: {
675 if (UseMI.getOperand(i: 1).isFI())
676 return false;
677 // Register defined by Lo should not be the value register.
678 if (DestReg == UseMI.getOperand(i: 0).getReg())
679 return false;
680 assert(DestReg == UseMI.getOperand(1).getReg() &&
681 "Expected base address use");
682 // All load/store instructions must use the same offset.
683 int64_t Offset = UseMI.getOperand(i: 2).getImm();
684 if (CommonOffset && Offset != CommonOffset)
685 return false;
686 CommonOffset = Offset;
687 break;
688 }
689 case LoongArch::INLINEASM:
690 case LoongArch::INLINEASM_BR: {
691 // We can't do this for large pattern.
692 if (Last)
693 return false;
694 SmallVector<unsigned> InlineAsmMemoryOpIndexes;
695 unsigned NumOps = 0;
696 for (unsigned I = InlineAsm::MIOp_FirstOperand;
697 I < UseMI.getNumOperands(); I += 1 + NumOps) {
698 const MachineOperand &FlagsMO = UseMI.getOperand(i: I);
699 // Should be an imm.
700 if (!FlagsMO.isImm())
701 continue;
702
703 const InlineAsm::Flag Flags(FlagsMO.getImm());
704 NumOps = Flags.getNumOperandRegisters();
705
706 // Memory constraints have two operands.
707 if (NumOps != 2 || !Flags.isMemKind()) {
708 // If the register is used by something other than a memory contraint,
709 // we should not fold.
710 for (unsigned J = 0; J < NumOps; ++J) {
711 const MachineOperand &MO = UseMI.getOperand(i: I + 1 + J);
712 if (MO.isReg() && MO.getReg() == DestReg)
713 return false;
714 }
715 continue;
716 }
717
718 // We can only do this for constraint m.
719 if (Flags.getMemoryConstraintID() != InlineAsm::ConstraintCode::m)
720 return false;
721
722 const MachineOperand &AddrMO = UseMI.getOperand(i: I + 1);
723 if (!AddrMO.isReg() || AddrMO.getReg() != DestReg)
724 continue;
725
726 const MachineOperand &OffsetMO = UseMI.getOperand(i: I + 2);
727 if (!OffsetMO.isImm())
728 continue;
729
730 // All inline asm memory operands must use the same offset.
731 int64_t Offset = OffsetMO.getImm();
732 if (CommonOffset && Offset != CommonOffset)
733 return false;
734 CommonOffset = Offset;
735 InlineAsmMemoryOpIndexes.push_back(Elt: I + 1);
736 }
737 InlineAsmMemoryOpIndexesMap.insert(
738 KV: std::make_pair(x: &UseMI, y&: InlineAsmMemoryOpIndexes));
739 break;
740 }
741 }
742 }
743
744 if (!CommonOffset)
745 return false;
746
747 // We found a common offset.
748 // Update the offsets in global address lowering.
749 // We may have already folded some arithmetic so we need to add to any
750 // existing offset.
751 int64_t NewOffset = Hi20.getOperand(i: 1).getOffset() + *CommonOffset;
752 // LA32 ignores the upper 32 bits.
753 if (!ST->is64Bit())
754 NewOffset = SignExtend64<32>(x: NewOffset);
755 // We can only fold simm32 offsets.
756 if (!isInt<32>(x: NewOffset))
757 return false;
758
759 // Folding changes the value represented by DestReg, so its debug users can
760 // no longer describe the same address.
761 MRI->markUsesInDebugValueAsUndef(Reg: DestReg);
762
763 // If optimized by this pass successfully, MO_RELAX bitmask target-flag should
764 // be removed from the pcala code sequence. Code sequence of tls-le can still
765 // be relaxed after being optimized.
766 //
767 // For example:
768 // pcaxxu12i $a0, %pc_hi20(symbol)
769 // addi.d $a0, $a0, %pc_lo12(symbol)
770 // ld.w $a0, $a0, 0
771 //
772 // =>
773 //
774 // pcaxxu12i $a0, %pc_hi20(symbol)
775 // ld.w $a0, $a0, %pc_lo12(symbol)
776 //
777 // Code sequence optimized before can be relax by linker. But after being
778 // optimized, it cannot be relaxed any more. So MO_RELAX flag should not be
779 // carried by them.
780 Hi20.getOperand(i: 1).setOffset(NewOffset);
781 MachineOperand &ImmOp = Lo12.getOperand(i: 2);
782 if (!isPCAddLo(Flags: LoongArchII::getDirectFlags(MO: ImmOp)))
783 ImmOp.setOffset(NewOffset);
784 if (Lo20 && Hi12) {
785 Lo20->getOperand(i: 2).setOffset(NewOffset);
786 Hi12->getOperand(i: 2).setOffset(NewOffset);
787 }
788 if (Hi20.getOpcode() == LoongArch::PCADDU12I ||
789 Hi20.getOpcode() == LoongArch::PCALAU12I) {
790 Hi20.getOperand(i: 1).setTargetFlags(
791 LoongArchII::getDirectFlags(MO: Hi20.getOperand(i: 1)));
792 ImmOp.setTargetFlags(LoongArchII::getDirectFlags(MO: ImmOp));
793 } else if (Hi20.getOpcode() == LoongArch::LU12I_W) {
794 MachineInstr *Add = &*MRI->use_instr_begin(RegNo: Hi20.getOperand(i: 0).getReg());
795 Add->getOperand(i: 3).setOffset(NewOffset);
796 }
797
798 // Update the immediate in the load/store instructions to add the offset.
799 const LoongArchInstrInfo &TII = *ST->getInstrInfo();
800 for (MachineInstr &UseMI :
801 llvm::make_early_inc_range(Range: MRI->use_nodbg_instructions(Reg: DestReg))) {
802 if (UseMI.getOpcode() == LoongArch::INLINEASM ||
803 UseMI.getOpcode() == LoongArch::INLINEASM_BR) {
804 auto &InlineAsmMemoryOpIndexes = InlineAsmMemoryOpIndexesMap[&UseMI];
805 for (unsigned I : InlineAsmMemoryOpIndexes) {
806 MachineOperand &MO = UseMI.getOperand(i: I + 1);
807 switch (ImmOp.getType()) {
808 case MachineOperand::MO_GlobalAddress:
809 MO.ChangeToGA(GV: ImmOp.getGlobal(), Offset: ImmOp.getOffset(),
810 TargetFlags: LoongArchII::getDirectFlags(MO: ImmOp));
811 break;
812 case MachineOperand::MO_MCSymbol:
813 MO.ChangeToMCSymbol(Sym: ImmOp.getMCSymbol(),
814 TargetFlags: LoongArchII::getDirectFlags(MO: ImmOp));
815 MO.setOffset(ImmOp.getOffset());
816 break;
817 case MachineOperand::MO_BlockAddress:
818 MO.ChangeToBA(BA: ImmOp.getBlockAddress(), Offset: ImmOp.getOffset(),
819 TargetFlags: LoongArchII::getDirectFlags(MO: ImmOp));
820 break;
821 case MachineOperand::MO_ConstantPoolIndex:
822 MO.ChangeToCPI(Idx: ImmOp.getIndex(), Offset: ImmOp.getOffset(),
823 TargetFlags: LoongArchII::getDirectFlags(MO: ImmOp));
824 break;
825 default:
826 report_fatal_error(reason: "unsupported machine operand type");
827 break;
828 }
829 }
830 } else {
831 UseMI.setDesc(TII.get(Opcode: getNewOpc(Op: UseMI.getOpcode(), isLarge: Last)));
832 if (Last) {
833 UseMI.removeOperand(OpNo: 2);
834 UseMI.removeOperand(OpNo: 1);
835 UseMI.addOperand(Op: Last->getOperand(i: 1));
836 UseMI.addOperand(Op: Last->getOperand(i: 2));
837 UseMI.getOperand(i: 1).setIsKill(false);
838 UseMI.getOperand(i: 2).setIsKill(false);
839 } else {
840 UseMI.removeOperand(OpNo: 2);
841 UseMI.addOperand(Op: ImmOp);
842 }
843 }
844 }
845
846 if (Last) {
847 Last->eraseFromParent();
848 return true;
849 }
850
851 if (Hi20.getOpcode() == LoongArch::PCADDU12I ||
852 Hi20.getOpcode() == LoongArch::PCALAU12I) {
853 MRI->replaceRegWith(FromReg: Lo12.getOperand(i: 0).getReg(),
854 ToReg: Hi20.getOperand(i: 0).getReg());
855 } else if (Hi20.getOpcode() == LoongArch::LU12I_W) {
856 MachineInstr *Add = &*MRI->use_instr_begin(RegNo: Hi20.getOperand(i: 0).getReg());
857 MRI->replaceRegWith(FromReg: Lo12.getOperand(i: 0).getReg(),
858 ToReg: Add->getOperand(i: 0).getReg());
859 }
860 Lo12.eraseFromParent();
861 return true;
862}
863
864bool LoongArchMergeBaseOffsetOpt::runOnMachineFunction(MachineFunction &Fn) {
865 if (skipFunction(F: Fn.getFunction()))
866 return false;
867
868 ST = &Fn.getSubtarget<LoongArchSubtarget>();
869
870 bool MadeChange = false;
871 MRI = &Fn.getRegInfo();
872 for (MachineBasicBlock &MBB : Fn) {
873 LLVM_DEBUG(dbgs() << "MBB: " << MBB.getName() << "\n");
874 for (MachineInstr &Hi20 : MBB) {
875 MachineInstr *Lo12 = nullptr;
876 MachineInstr *Lo20 = nullptr;
877 MachineInstr *Hi12 = nullptr;
878 MachineInstr *Last = nullptr;
879 if (Hi20.getOpcode() == LoongArch::PCADDU12I ||
880 Hi20.getOpcode() == LoongArch::PCALAU12I) {
881 // Detect foldable pcala code sequence in small/medium/large code model.
882 if (!detectFoldable(Hi20, Lo12, Lo20, Hi12, Last))
883 continue;
884 } else if (Hi20.getOpcode() == LoongArch::LU12I_W) {
885 MachineInstr *Add = nullptr;
886 // Detect foldable tls-le code sequence in small/medium code model.
887 if (!detectFoldable(Hi20, Add, Lo12))
888 continue;
889 } else {
890 continue;
891 }
892 // For tls-le, we do not pass the second PseudoAddTPRel instr in order to
893 // reuse the existing hooks and the last three paramaters should always be
894 // nullptr.
895 MadeChange |= detectAndFoldOffset(Hi20, Lo12&: *Lo12, Lo20, Hi12, Last);
896 MadeChange |= foldIntoMemoryOps(Hi20, Lo12&: *Lo12, Lo20, Hi12, Last);
897 }
898 }
899
900 return MadeChange;
901}
902
903/// Returns an instance of the Merge Base Offset Optimization pass.
904FunctionPass *llvm::createLoongArchMergeBaseOffsetOptPass() {
905 return new LoongArchMergeBaseOffsetOpt();
906}
907