1//===----- RISCVLoadStoreOptimizer.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Load/Store Pairing: It identifies pairs of load or store instructions
10// operating on consecutive memory locations and merges them into a single
11// paired instruction, leveraging hardware support for paired memory accesses.
12// Much of the pairing logic is adapted from the AArch64LoadStoreOpt pass.
13//
14// Post-allocation Zilsd decomposition: Fixes invalid LD/SD instructions if
15// register allocation didn't provide suitable consecutive registers.
16//
17// NOTE: The AArch64LoadStoreOpt pass performs additional optimizations such as
18// merging zero store instructions, promoting loads that read directly from a
19// preceding store, and merging base register updates with load/store
20// instructions (via pre-/post-indexed addressing). These advanced
21// transformations are not yet implemented in the RISC-V pass but represent
22// potential future enhancements for further optimizing RISC-V memory
23// operations.
24//
25//===----------------------------------------------------------------------===//
26
27#include "RISCV.h"
28#include "RISCVTargetMachine.h"
29#include "llvm/ADT/SmallVector.h"
30#include "llvm/ADT/Statistic.h"
31#include "llvm/Analysis/AliasAnalysis.h"
32#include "llvm/CodeGen/Passes.h"
33#include "llvm/MC/TargetRegistry.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Target/TargetOptions.h"
36
37using namespace llvm;
38
39#define DEBUG_TYPE "riscv-load-store-opt"
40#define RISCV_LOAD_STORE_OPT_NAME "RISC-V Load / Store Optimizer"
41
42STATISTIC(NumLD2LW, "Number of LD instructions split back to LW");
43STATISTIC(NumSD2SW, "Number of SD instructions split back to SW");
44
45namespace {
46
47struct RISCVLoadStoreOpt : public MachineFunctionPass {
48 static char ID;
49 bool runOnMachineFunction(MachineFunction &Fn) override;
50
51 RISCVLoadStoreOpt() : MachineFunctionPass(ID) {}
52
53 MachineFunctionProperties getRequiredProperties() const override {
54 return MachineFunctionProperties().setNoVRegs();
55 }
56
57 void getAnalysisUsage(AnalysisUsage &AU) const override {
58 AU.addRequired<AAResultsWrapperPass>();
59 MachineFunctionPass::getAnalysisUsage(AU);
60 }
61
62 StringRef getPassName() const override { return RISCV_LOAD_STORE_OPT_NAME; }
63
64 // Find and pair load/store instructions.
65 bool tryToPairLdStInst(MachineBasicBlock::iterator &MBBI);
66
67 // Convert load/store pairs to single instructions.
68 bool tryConvertToLdStPair(MachineBasicBlock::iterator First,
69 MachineBasicBlock::iterator Second);
70 bool tryConvertToXqcilsmLdStPair(MachineFunction *MF,
71 MachineBasicBlock::iterator First,
72 MachineBasicBlock::iterator Second);
73 bool tryConvertToXqcilsmMultiLdSt(MachineBasicBlock::iterator &First);
74 bool tryConvertToMIPSLdStPair(MachineFunction *MF,
75 MachineBasicBlock::iterator First,
76 MachineBasicBlock::iterator Second);
77
78 // Scan the instructions looking for a load/store that can be combined
79 // with the current instruction into a load/store pair.
80 // Return the matching instruction if one is found, else MBB->end().
81 MachineBasicBlock::iterator findMatchingInsn(MachineBasicBlock::iterator I,
82 bool &MergeForward);
83
84 MachineBasicBlock::iterator
85 mergePairedInsns(MachineBasicBlock::iterator I,
86 MachineBasicBlock::iterator Paired, bool MergeForward);
87
88 // Post reg-alloc zilsd part
89 bool fixInvalidRegPairOp(MachineBasicBlock &MBB,
90 MachineBasicBlock::iterator &MBBI);
91 bool isValidZilsdRegPair(Register First, Register Second);
92 void splitLdSdIntoTwo(MachineBasicBlock &MBB,
93 MachineBasicBlock::iterator &MBBI, bool IsLoad);
94
95private:
96 AliasAnalysis *AA;
97 MachineRegisterInfo *MRI;
98 const RISCVInstrInfo *TII;
99 const RISCVRegisterInfo *TRI;
100 const RISCVSubtarget *STI = nullptr;
101 LiveRegUnits ModifiedRegUnits, UsedRegUnits;
102};
103} // end anonymous namespace
104
105char RISCVLoadStoreOpt::ID = 0;
106INITIALIZE_PASS(RISCVLoadStoreOpt, DEBUG_TYPE, RISCV_LOAD_STORE_OPT_NAME, false,
107 false)
108
109bool RISCVLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
110 if (skipFunction(F: Fn.getFunction()))
111 return false;
112
113 bool MadeChange = false;
114 STI = &Fn.getSubtarget<RISCVSubtarget>();
115 TII = STI->getInstrInfo();
116 TRI = STI->getRegisterInfo();
117 MRI = &Fn.getRegInfo();
118 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
119 ModifiedRegUnits.init(TRI: *TRI);
120 UsedRegUnits.init(TRI: *TRI);
121
122 if (STI->useMIPSLoadStorePairs() || STI->hasVendorXqcilsm()) {
123 for (MachineBasicBlock &MBB : Fn) {
124 LLVM_DEBUG(dbgs() << "MBB: " << MBB.getName() << "\n");
125
126 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
127 MBBI != E;) {
128 if (TII->isPairableLdStInstOpc(Opc: MBBI->getOpcode()) &&
129 tryToPairLdStInst(MBBI))
130 MadeChange = true;
131 else
132 ++MBBI;
133 }
134 }
135 }
136
137 if (!STI->is64Bit() && STI->hasStdExtZilsd()) {
138 for (auto &MBB : Fn) {
139 for (auto MBBI = MBB.begin(), E = MBB.end(); MBBI != E;) {
140 if (fixInvalidRegPairOp(MBB, MBBI)) {
141 MadeChange = true;
142 // Iterator was updated by fixInvalidRegPairOp
143 } else {
144 ++MBBI;
145 }
146 }
147 }
148 }
149
150 return MadeChange;
151}
152
153// Find loads and stores that can be merged into a single load or store pair
154// instruction.
155bool RISCVLoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
156 MachineInstr &MI = *MBBI;
157
158 // If this is volatile, it is not a candidate.
159 if (MI.hasOrderedMemoryRef())
160 return false;
161
162 if (!TII->isLdStSafeToPair(LdSt: MI, TRI))
163 return false;
164
165 // If Xqcilsm is available, first try to form a multi-instruction group (>2).
166 if (!STI->is64Bit() && STI->hasVendorXqcilsm()) {
167 if (tryConvertToXqcilsmMultiLdSt(First&: MBBI))
168 return true;
169 }
170
171 // Look ahead for a pairable instruction.
172 MachineBasicBlock::iterator E = MI.getParent()->end();
173 bool MergeForward;
174 MachineBasicBlock::iterator Paired = findMatchingInsn(I: MBBI, MergeForward);
175 if (Paired != E) {
176 MBBI = mergePairedInsns(I: MBBI, Paired, MergeForward);
177 return true;
178 }
179 return false;
180}
181
182static bool isMemOpAligned(MachineInstr &MI, Align RequiredAlignment) {
183 const MachineMemOperand *MMO = *MI.memoperands_begin();
184 Align MMOAlign = MMO->getAlign();
185 return MMOAlign >= RequiredAlignment;
186}
187
188// Convert set of 3 or more LW/SW instructions to QC_LWMI/QC_SWMI/QC_SETWMI.
189// For now this only handles consecutive loads and stores traversing the basic
190// block top-down.
191// TODO: Traverse the basic block bottom-up as well.
192bool RISCVLoadStoreOpt::tryConvertToXqcilsmMultiLdSt(
193 MachineBasicBlock::iterator &FirstIt) {
194 MachineInstr &FirstMI = *FirstIt;
195 MachineFunction *MF = FirstMI.getMF();
196
197 if (STI->is64Bit() || !STI->hasVendorXqcilsm())
198 return false;
199
200 unsigned Opc = FirstMI.getOpcode();
201 if (Opc != RISCV::LW && Opc != RISCV::SW)
202 return false;
203
204 if (!FirstMI.hasOneMemOperand())
205 return false;
206
207 if (!isMemOpAligned(MI&: FirstMI, RequiredAlignment: Align(4)))
208 return false;
209
210 // Require simple reg+imm addressing.
211 const MachineOperand &BaseOp = FirstMI.getOperand(i: 1);
212 const MachineOperand &OffOp = FirstMI.getOperand(i: 2);
213 if (!BaseOp.isReg() || !OffOp.isImm())
214 return false;
215
216 Register Base = BaseOp.getReg();
217 int64_t BaseOff = OffOp.getImm();
218
219 if (!isShiftedUInt<5, 2>(x: BaseOff))
220 return false;
221
222 Register StartReg = FirstMI.getOperand(i: 0).getReg();
223 bool IsLoad = (Opc == RISCV::LW);
224
225 // Load rd cannot be x0 and must not clobber the base register.
226 if (IsLoad) {
227 if (StartReg == RISCV::X0)
228 return false;
229 if (StartReg == Base)
230 return false;
231 }
232
233 // Collect a set of consecutive matching instructions.
234 SmallVector<MachineInstr *, 8> Group;
235 Group.push_back(Elt: &FirstMI);
236
237 MachineBasicBlock::iterator E = FirstIt->getParent()->end();
238 MachineBasicBlock::iterator It = next_nodbg(It: FirstIt, End: E);
239 int64_t ExpectedOff = BaseOff + 4;
240 unsigned Index = 1;
241 enum class StoreMode { Unknown, Setwmi, Swmi };
242 StoreMode SMode = StoreMode::Unknown;
243
244 while (It != E) {
245 MachineInstr &MI = *It;
246
247 if (!TII->isPairableLdStInstOpc(Opc: MI.getOpcode()))
248 break;
249 if (MI.getOpcode() != Opc)
250 break;
251 if (!TII->isLdStSafeToPair(LdSt: MI, TRI))
252 break;
253 if (!MI.hasOneMemOperand())
254 break;
255 if (!isMemOpAligned(MI, RequiredAlignment: Align(4)))
256 break;
257
258 const MachineOperand &BaseMIOp = MI.getOperand(i: 1);
259 const MachineOperand &OffsetMIOp = MI.getOperand(i: 2);
260 if (!BaseMIOp.isReg() || !OffsetMIOp.isImm())
261 break;
262 if (BaseMIOp.getReg() != Base)
263 break;
264 int64_t Off = OffsetMIOp.getImm();
265 if (Off != ExpectedOff)
266 break;
267
268 Register Reg = MI.getOperand(i: 0).getReg();
269 if (IsLoad) {
270 // For loads, require consecutive destination registers.
271 if (Reg != StartReg + Index)
272 break;
273 if (Reg == Base)
274 break;
275 } else {
276 // For stores, decide mode based on the second instruction and then
277 // enforce the same for the rest.
278 if (SMode == StoreMode::Unknown) {
279 if (Reg == StartReg)
280 SMode = StoreMode::Setwmi;
281 else if (Reg == StartReg + 1)
282 SMode = StoreMode::Swmi;
283 else
284 break;
285 } else if (SMode == StoreMode::Setwmi) {
286 if (Reg != StartReg)
287 break;
288 } else {
289 if (Reg != StartReg + Index)
290 break;
291 }
292 }
293
294 // Passed checks, extend the group.
295 Group.push_back(Elt: &MI);
296 ++Index;
297 ExpectedOff += 4;
298 It = next_nodbg(It, End: E);
299 }
300
301 // We only handle more than 2 here. Pairs are handled in
302 // tryConvertToXqcilsmLdStPair.
303 unsigned Len = Group.size();
304 if (Len < 3 || Len > 31)
305 return false;
306
307 unsigned NewOpc;
308 RegState StartRegState;
309 bool AddImplicitRegs = true;
310
311 if (IsLoad) {
312 NewOpc = RISCV::QC_LWMI;
313 StartRegState = RegState::Define;
314 } else {
315 assert(SMode != StoreMode::Unknown &&
316 "Group should be large enough to know the store mode");
317 if (SMode == StoreMode::Setwmi) {
318 NewOpc = RISCV::QC_SETWMI;
319 // Kill if any of the individual stores killed the reg.
320 bool StartKill = false;
321 for (MachineInstr *MI : Group)
322 StartKill |= MI->getOperand(i: 0).isKill();
323 StartRegState = getKillRegState(B: StartKill);
324 AddImplicitRegs = false;
325 } else {
326 // SWMI requires consecutive source regs and rd != x0.
327 if (StartReg == RISCV::X0)
328 return false;
329 NewOpc = RISCV::QC_SWMI;
330 StartRegState = getKillRegState(B: Group.front()->getOperand(i: 0).isKill());
331 }
332 }
333
334 // Aggregate kill on base.
335 bool BaseKill = false;
336 for (MachineInstr *MI : Group)
337 BaseKill |= MI->getOperand(i: 1).isKill();
338
339 // Build the new instruction.
340 DebugLoc DL = FirstMI.getDebugLoc();
341 if (!DL)
342 DL = Group.back()->getDebugLoc();
343 MachineInstrBuilder MIB = BuildMI(MF&: *MF, MIMD: DL, MCID: TII->get(Opcode: NewOpc));
344 MIB.addReg(RegNo: StartReg, Flags: StartRegState)
345 .addReg(RegNo: Base, Flags: getKillRegState(B: BaseKill))
346 .addImm(Val: Len)
347 .addImm(Val: BaseOff);
348
349 // Merge memory references.
350 MIB.cloneMergedMemRefs(OtherMIs: Group);
351
352 if (AddImplicitRegs) {
353 // Add implicit operands for the additional registers.
354 for (unsigned i = 1; i < Len; ++i) {
355 Register R = StartReg + i;
356 RegState State;
357 if (IsLoad)
358 State = RegState::ImplicitDefine;
359 else
360 State = RegState::Implicit |
361 getKillRegState(B: Group[i]->getOperand(i: 0).isKill());
362 MIB.addReg(RegNo: R, Flags: State);
363 }
364 }
365
366 // Insert before the first instruction and remove all in the group.
367 MachineBasicBlock *MBB = FirstIt->getParent();
368 MachineBasicBlock::iterator NewIt = MBB->insert(I: FirstIt, MI: MIB);
369 for (MachineInstr *MI : Group)
370 MI->removeFromParent();
371
372 // Advance the cursor to the next non-debug instruction after the group.
373 FirstIt = next_nodbg(It: NewIt, End: MBB->end());
374 return true;
375}
376
377bool RISCVLoadStoreOpt::tryConvertToXqcilsmLdStPair(
378 MachineFunction *MF, MachineBasicBlock::iterator First,
379 MachineBasicBlock::iterator Second) {
380 unsigned Opc = First->getOpcode();
381 if ((Opc != RISCV::LW && Opc != RISCV::SW) || Second->getOpcode() != Opc)
382 return false;
383
384 const auto &FirstOp1 = First->getOperand(i: 1);
385 const auto &SecondOp1 = Second->getOperand(i: 1);
386 const auto &FirstOp2 = First->getOperand(i: 2);
387 const auto &SecondOp2 = Second->getOperand(i: 2);
388
389 // Require simple reg+imm addressing for both.
390 if (!FirstOp1.isReg() || !SecondOp1.isReg() || !FirstOp2.isImm() ||
391 !SecondOp2.isImm())
392 return false;
393
394 Register Base1 = FirstOp1.getReg();
395 Register Base2 = SecondOp1.getReg();
396
397 if (Base1 != Base2)
398 return false;
399
400 if (!First->hasOneMemOperand() || !Second->hasOneMemOperand())
401 return false;
402
403 if (!isMemOpAligned(MI&: *First, RequiredAlignment: Align(4)) || !isMemOpAligned(MI&: *Second, RequiredAlignment: Align(4)))
404 return false;
405
406 auto &FirstOp0 = First->getOperand(i: 0);
407 auto &SecondOp0 = Second->getOperand(i: 0);
408
409 int64_t Off1 = FirstOp2.getImm();
410 int64_t Off2 = SecondOp2.getImm();
411
412 if (Off2 < Off1) {
413 std::swap(a&: FirstOp0, b&: SecondOp0);
414 std::swap(a&: Off1, b&: Off2);
415 }
416
417 if (!isShiftedUInt<5, 2>(x: Off1) || (Off2 - Off1 != 4))
418 return false;
419
420 Register StartReg = FirstOp0.getReg();
421 Register NextReg = SecondOp0.getReg();
422
423 unsigned XqciOpc;
424 RegState StartRegState;
425 RegState NextRegState = {};
426 bool AddNextReg = true;
427
428 if (Opc == RISCV::LW) {
429
430 if (StartReg == RISCV::X0)
431 return false;
432
433 // If the base reg gets overwritten by one of the loads bail out.
434 if (StartReg == Base1 || NextReg == Base1)
435 return false;
436
437 // The registers need to be consecutive.
438 if (NextReg != StartReg + 1)
439 return false;
440
441 XqciOpc = RISCV::QC_LWMI;
442 StartRegState = RegState::Define;
443 NextRegState = RegState::ImplicitDefine;
444 } else {
445 assert(Opc == RISCV::SW && "Expected a SW instruction");
446 if (StartReg == NextReg) {
447 XqciOpc = RISCV::QC_SETWMI;
448 StartRegState = getKillRegState(B: FirstOp0.isKill() || SecondOp0.isKill());
449 AddNextReg = false;
450 } else if (NextReg == StartReg + 1 && StartReg != RISCV::X0) {
451 XqciOpc = RISCV::QC_SWMI;
452 StartRegState = getKillRegState(B: FirstOp0.isKill());
453 NextRegState = RegState::Implicit | getKillRegState(B: SecondOp0.isKill());
454 } else {
455 return false;
456 }
457 }
458
459 DebugLoc DL =
460 First->getDebugLoc() ? First->getDebugLoc() : Second->getDebugLoc();
461 MachineInstrBuilder MIB = BuildMI(MF&: *MF, MIMD: DL, MCID: TII->get(Opcode: XqciOpc));
462 MIB.addReg(RegNo: StartReg, Flags: StartRegState)
463 .addReg(RegNo: Base1, Flags: getKillRegState(B: FirstOp1.isKill() || SecondOp1.isKill()))
464 .addImm(Val: 2)
465 .addImm(Val: Off1)
466 .cloneMergedMemRefs(OtherMIs: {&*First, &*Second});
467
468 if (AddNextReg)
469 MIB.addReg(RegNo: NextReg, Flags: NextRegState);
470
471 First->getParent()->insert(I: First, MI: MIB);
472 First->removeFromParent();
473 Second->removeFromParent();
474
475 return true;
476}
477
478bool RISCVLoadStoreOpt::tryConvertToMIPSLdStPair(
479 MachineFunction *MF, MachineBasicBlock::iterator First,
480 MachineBasicBlock::iterator Second) {
481 // Try converting to SWP/LWP/LDP/SDP.
482 // SWP/LWP requires 8-byte alignment whereas LDP/SDP needs 16-byte alignment.
483 unsigned PairOpc;
484 Align RequiredAlignment;
485 switch (First->getOpcode()) {
486 default:
487 llvm_unreachable("Unsupported load/store instruction for pairing");
488 case RISCV::SW:
489 PairOpc = RISCV::MIPS_SWP;
490 RequiredAlignment = Align(8);
491 break;
492 case RISCV::LW:
493 PairOpc = RISCV::MIPS_LWP;
494 RequiredAlignment = Align(8);
495 break;
496 case RISCV::SD:
497 PairOpc = RISCV::MIPS_SDP;
498 RequiredAlignment = Align(16);
499 break;
500 case RISCV::LD:
501 PairOpc = RISCV::MIPS_LDP;
502 RequiredAlignment = Align(16);
503 break;
504 }
505
506 if (!First->hasOneMemOperand())
507 return false;
508
509 if (!isMemOpAligned(MI&: *First, RequiredAlignment))
510 return false;
511
512 int64_t Offset = First->getOperand(i: 2).getImm();
513 if (!isUInt<7>(x: Offset))
514 return false;
515
516 MachineInstrBuilder MIB = BuildMI(
517 MF&: *MF, MIMD: First->getDebugLoc() ? First->getDebugLoc() : Second->getDebugLoc(),
518 MCID: TII->get(Opcode: PairOpc));
519 MIB.add(MO: First->getOperand(i: 0))
520 .add(MO: Second->getOperand(i: 0))
521 .add(MO: First->getOperand(i: 1))
522 .add(MO: First->getOperand(i: 2))
523 .cloneMergedMemRefs(OtherMIs: {&*First, &*Second});
524
525 First->getParent()->insert(I: First, MI: MIB);
526
527 First->removeFromParent();
528 Second->removeFromParent();
529
530 return true;
531}
532
533// Merge two adjacent load/store instructions into a paired instruction.
534// This function calls the vendor specific implementation that seelects the
535// appropriate paired opcode, verifies that the memory operand is properly
536// aligned, and checks that the offset is valid. If all conditions are met, it
537// builds and inserts the paired instruction.
538bool RISCVLoadStoreOpt::tryConvertToLdStPair(
539 MachineBasicBlock::iterator First, MachineBasicBlock::iterator Second) {
540 MachineFunction *MF = First->getMF();
541
542 // Try converting to QC_LWMI/QC_SWMI if the XQCILSM extension is enabled.
543 if (!STI->is64Bit() && STI->hasVendorXqcilsm())
544 return tryConvertToXqcilsmLdStPair(MF, First, Second);
545
546 // Else try to convert them into MIPS Paired Loads/Stores.
547 return tryConvertToMIPSLdStPair(MF, First, Second);
548}
549
550static bool mayAlias(MachineInstr &MIa,
551 SmallVectorImpl<MachineInstr *> &MemInsns,
552 AliasAnalysis *AA) {
553 for (MachineInstr *MIb : MemInsns)
554 if (MIa.mayAlias(AA, Other: *MIb, /*UseTBAA*/ false))
555 return true;
556
557 return false;
558}
559
560// Scan the instructions looking for a load/store that can be combined with the
561// current instruction into a wider equivalent or a load/store pair.
562// TODO: Extend pairing logic to consider reordering both instructions
563// to a safe "middle" position rather than only merging forward/backward.
564// This requires more sophisticated checks for aliasing, register
565// liveness, and potential scheduling hazards.
566MachineBasicBlock::iterator
567RISCVLoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
568 bool &MergeForward) {
569 MachineBasicBlock::iterator E = I->getParent()->end();
570 MachineBasicBlock::iterator MBBI = I;
571 MachineInstr &FirstMI = *I;
572 MBBI = next_nodbg(It: MBBI, End: E);
573
574 bool MayLoad = FirstMI.mayLoad();
575 Register Reg = FirstMI.getOperand(i: 0).getReg();
576 Register BaseReg = FirstMI.getOperand(i: 1).getReg();
577 int64_t Offset = FirstMI.getOperand(i: 2).getImm();
578 int64_t OffsetStride = (*FirstMI.memoperands_begin())->getSize().getValue();
579
580 MergeForward = false;
581
582 // Track which register units have been modified and used between the first
583 // insn (inclusive) and the second insn.
584 ModifiedRegUnits.clear();
585 UsedRegUnits.clear();
586
587 // Remember any instructions that read/write memory between FirstMI and MI.
588 SmallVector<MachineInstr *, 4> MemInsns;
589
590 for (unsigned Count = 0;
591 MBBI != E && Count < STI->getCLOpts().load_store_scan_limit;
592 MBBI = next_nodbg(It: MBBI, End: E)) {
593 MachineInstr &MI = *MBBI;
594
595 // Don't count transient instructions towards the search limit since there
596 // may be different numbers of them if e.g. debug information is present.
597 if (!MI.isTransient())
598 ++Count;
599
600 if (MI.getOpcode() == FirstMI.getOpcode() &&
601 TII->isLdStSafeToPair(LdSt: MI, TRI)) {
602 Register MIBaseReg = MI.getOperand(i: 1).getReg();
603 int64_t MIOffset = MI.getOperand(i: 2).getImm();
604
605 if (BaseReg == MIBaseReg) {
606 if ((Offset != MIOffset + OffsetStride) &&
607 (Offset + OffsetStride != MIOffset)) {
608 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
609 TRI);
610 MemInsns.push_back(Elt: &MI);
611 continue;
612 }
613
614 // If the destination register of one load is the same register or a
615 // sub/super register of the other load, bail and keep looking.
616 if (MayLoad &&
617 TRI->isSuperOrSubRegisterEq(RegA: Reg, RegB: MI.getOperand(i: 0).getReg())) {
618 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
619 TRI);
620 MemInsns.push_back(Elt: &MI);
621 continue;
622 }
623
624 // If the BaseReg has been modified, then we cannot do the optimization.
625 if (!ModifiedRegUnits.available(Reg: BaseReg))
626 return E;
627
628 // If the Rt of the second instruction was not modified or used between
629 // the two instructions and none of the instructions between the second
630 // and first alias with the second, we can combine the second into the
631 // first.
632 if (ModifiedRegUnits.available(Reg: MI.getOperand(i: 0).getReg()) &&
633 !(MI.mayLoad() &&
634 !UsedRegUnits.available(Reg: MI.getOperand(i: 0).getReg())) &&
635 !mayAlias(MIa&: MI, MemInsns, AA)) {
636
637 MergeForward = false;
638 return MBBI;
639 }
640
641 // Likewise, if the Rt of the first instruction is not modified or used
642 // between the two instructions and none of the instructions between the
643 // first and the second alias with the first, we can combine the first
644 // into the second.
645 if (!(MayLoad &&
646 !UsedRegUnits.available(Reg: FirstMI.getOperand(i: 0).getReg())) &&
647 !mayAlias(MIa&: FirstMI, MemInsns, AA)) {
648
649 if (ModifiedRegUnits.available(Reg: FirstMI.getOperand(i: 0).getReg())) {
650 MergeForward = true;
651 return MBBI;
652 }
653 }
654 // Unable to combine these instructions due to interference in between.
655 // Keep looking.
656 }
657 }
658
659 // If the instruction wasn't a matching load or store. Stop searching if we
660 // encounter a call instruction that might modify memory.
661 if (MI.isCall())
662 return E;
663
664 // Update modified / uses register units.
665 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
666
667 // Otherwise, if the base register is modified, we have no match, so
668 // return early.
669 if (!ModifiedRegUnits.available(Reg: BaseReg))
670 return E;
671
672 // Update list of instructions that read/write memory.
673 if (MI.mayLoadOrStore())
674 MemInsns.push_back(Elt: &MI);
675 }
676 return E;
677}
678
679MachineBasicBlock::iterator
680RISCVLoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
681 MachineBasicBlock::iterator Paired,
682 bool MergeForward) {
683 MachineBasicBlock::iterator E = I->getParent()->end();
684 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
685 // If NextI is the second of the two instructions to be merged, skip one
686 // further for now. For the MIPS load/store, the merge will invalidate the
687 // iterator, and we don't need to scan the new instruction, as it's a pairwise
688 // instruction, which we're not considering for further action anyway. For the
689 // Xqcilsm load/store, we may not want to do this as the second instruction
690 // could possibly be the first in another pair if we do not merge here. This
691 // is handled in the else block after the call to tryConvertToLdStPair below.
692 if (NextI == Paired)
693 NextI = next_nodbg(It: NextI, End: E);
694
695 // Insert our new paired instruction after whichever of the paired
696 // instructions MergeForward indicates.
697 MachineBasicBlock::iterator InsertionPoint = MergeForward ? Paired : I;
698 MachineBasicBlock::iterator DeletionPoint = MergeForward ? I : Paired;
699 int Offset = I->getOperand(i: 2).getImm();
700 int PairedOffset = Paired->getOperand(i: 2).getImm();
701 bool InsertAfter = (Offset < PairedOffset) ^ MergeForward;
702
703 if (!MergeForward)
704 Paired->getOperand(i: 1).setIsKill(false);
705
706 // Kill flags may become invalid when moving stores for pairing.
707 if (I->getOperand(i: 0).isUse()) {
708 if (!MergeForward) {
709 // Check if the Paired store's source register has a kill flag and clear
710 // it only if there are intermediate uses between I and Paired.
711 MachineOperand &PairedRegOp = Paired->getOperand(i: 0);
712 if (PairedRegOp.isKill()) {
713 for (auto It = std::next(x: I); It != Paired; ++It) {
714 if (It->readsRegister(Reg: PairedRegOp.getReg(), TRI)) {
715 PairedRegOp.setIsKill(false);
716 break;
717 }
718 }
719 }
720 } else {
721 // Clear kill flags of the first store's register in the forward
722 // direction.
723 Register Reg = I->getOperand(i: 0).getReg();
724 for (MachineInstr &MI : make_range(x: std::next(x: I), y: std::next(x: Paired)))
725 MI.clearRegisterKills(Reg, RegInfo: TRI);
726 }
727 }
728
729 // Remember the original position of the instruction we're moving so we can
730 // restore it if we fail to form a pair.
731 MachineBasicBlock::iterator OrigNext = std::next(x: DeletionPoint);
732
733 MachineInstr *ToInsert = DeletionPoint->removeFromParent();
734 MachineBasicBlock &MBB = *InsertionPoint->getParent();
735 MachineBasicBlock::iterator First, Second, Moved;
736
737 if (!InsertAfter) {
738 First = MBB.insert(I: InsertionPoint, MI: ToInsert);
739 Second = InsertionPoint;
740 Moved = First;
741 } else {
742 Second = MBB.insertAfter(I: InsertionPoint, MI: ToInsert);
743 First = InsertionPoint;
744 Moved = Second;
745 }
746
747 if (tryConvertToLdStPair(First, Second)) {
748 LLVM_DEBUG(dbgs() << "Pairing load/store:\n ");
749 LLVM_DEBUG(prev_nodbg(NextI, MBB.begin())->print(dbgs()));
750 } else if (!STI->is64Bit() && STI->hasVendorXqcilsm()) {
751 // We were unable to form the pair, so move the instruction back to it's
752 // original place. Point NextI to the next non-debug instruction after the
753 // first instruction we had wanted to merge.
754 MachineInstr *MovedMI = Moved->removeFromParent();
755 MBB.insert(I: OrigNext, MI: MovedMI);
756 NextI = next_nodbg(It: I, End: E);
757 }
758
759 return NextI;
760}
761
762//===----------------------------------------------------------------------===//
763// Post reg-alloc zilsd pass implementation
764//===----------------------------------------------------------------------===//
765
766bool RISCVLoadStoreOpt::isValidZilsdRegPair(Register First, Register Second) {
767 // Special case: First register can not be zero unless both registers are
768 // zeros.
769 // Spec says: LD instructions with destination x0 are processed as any other
770 // load, but the result is discarded entirely and x1 is not written. If using
771 // x0 as src of SD, the entire 64-bit operand is zero — i.e., register x1 is
772 // not accessed.
773 if (First == RISCV::X0)
774 return Second == RISCV::X0;
775
776 // Check if registers form a valid even/odd pair for Zilsd
777 unsigned FirstNum = TRI->getEncodingValue(Reg: First);
778 unsigned SecondNum = TRI->getEncodingValue(Reg: Second);
779
780 // Must be consecutive and first must be even
781 return (FirstNum % 2 == 0) && (SecondNum == FirstNum + 1);
782}
783
784void RISCVLoadStoreOpt::splitLdSdIntoTwo(MachineBasicBlock &MBB,
785 MachineBasicBlock::iterator &MBBI,
786 bool IsLoad) {
787 MachineInstr *MI = &*MBBI;
788 DebugLoc DL = MI->getDebugLoc();
789
790 const MachineOperand &FirstOp = MI->getOperand(i: 0);
791 const MachineOperand &SecondOp = MI->getOperand(i: 1);
792 const MachineOperand &BaseOp = MI->getOperand(i: 2);
793 Register FirstReg = FirstOp.getReg();
794 Register SecondReg = SecondOp.getReg();
795 Register BaseReg = BaseOp.getReg();
796
797 // Handle both immediate and symbolic operands for offset
798 const MachineOperand &OffsetOp = MI->getOperand(i: 3);
799 int BaseOffset;
800 if (OffsetOp.isImm())
801 BaseOffset = OffsetOp.getImm();
802 else
803 // For symbolic operands, extract the embedded offset
804 BaseOffset = OffsetOp.getOffset();
805
806 unsigned Opc = IsLoad ? RISCV::LW : RISCV::SW;
807 MachineInstrBuilder MIB1, MIB2;
808
809 // Create two separate instructions
810 if (IsLoad) {
811 // It's possible that first register is same as base register, when we split
812 // it becomes incorrect because base register is overwritten, e.g.
813 // X10, X13 = PseudoLD_RV32_OPT killed X10, 0
814 // =>
815 // X10 = LW X10, 0
816 // X13 = LW killed X10, 4
817 // we can just switch the order to resolve that:
818 // X13 = LW X10, 4
819 // X10 = LW killed X10, 0
820 if (FirstReg == BaseReg) {
821 MIB2 = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc))
822 .addDef(RegNo: SecondReg, Flags: getDeadRegState(B: SecondOp.isDead()))
823 .addReg(RegNo: BaseReg);
824 MIB1 = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc))
825 .addDef(RegNo: FirstReg, Flags: getDeadRegState(B: FirstOp.isDead()))
826 .addReg(RegNo: BaseReg, Flags: getKillRegState(B: BaseOp.isKill()));
827
828 } else {
829 MIB1 = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc))
830 .addDef(RegNo: FirstReg, Flags: getDeadRegState(B: FirstOp.isDead()))
831 .addReg(RegNo: BaseReg);
832
833 MIB2 = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc))
834 .addDef(RegNo: SecondReg, Flags: getDeadRegState(B: SecondOp.isDead()))
835 .addReg(RegNo: BaseReg, Flags: getKillRegState(B: BaseOp.isKill()));
836 }
837
838 ++NumLD2LW;
839 LLVM_DEBUG(dbgs() << "Split LD back to two LW instructions\n");
840 } else {
841 assert(
842 FirstReg != SecondReg &&
843 "First register and second register is impossible to be same register");
844 MIB1 = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc))
845 .addReg(RegNo: FirstReg, Flags: getKillRegState(B: FirstOp.isKill()))
846 .addReg(RegNo: BaseReg);
847
848 MIB2 = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc))
849 .addReg(RegNo: SecondReg, Flags: getKillRegState(B: SecondOp.isKill()))
850 .addReg(RegNo: BaseReg, Flags: getKillRegState(B: BaseOp.isKill()));
851
852 ++NumSD2SW;
853 LLVM_DEBUG(dbgs() << "Split SD back to two SW instructions\n");
854 }
855
856 // Add offset operands - preserve symbolic references
857 MIB1.add(MO: OffsetOp);
858 if (OffsetOp.isImm())
859 MIB2.addImm(Val: BaseOffset + 4);
860 else if (OffsetOp.isGlobal())
861 MIB2.addGlobalAddress(GV: OffsetOp.getGlobal(), Offset: BaseOffset + 4,
862 TargetFlags: OffsetOp.getTargetFlags());
863 else if (OffsetOp.isCPI())
864 MIB2.addConstantPoolIndex(Idx: OffsetOp.getIndex(), Offset: BaseOffset + 4,
865 TargetFlags: OffsetOp.getTargetFlags());
866 else if (OffsetOp.isBlockAddress())
867 MIB2.addBlockAddress(BA: OffsetOp.getBlockAddress(), Offset: BaseOffset + 4,
868 TargetFlags: OffsetOp.getTargetFlags());
869
870 // Copy memory operands if the original instruction had them
871 // FIXME: This is overly conservative; the new instruction accesses 4 bytes,
872 // not 8.
873 MIB1.cloneMemRefs(OtherMI: *MI);
874 MIB2.cloneMemRefs(OtherMI: *MI);
875
876 // Remove the original paired instruction and update iterator
877 MBBI = MBB.erase(I: MBBI);
878}
879
880bool RISCVLoadStoreOpt::fixInvalidRegPairOp(MachineBasicBlock &MBB,
881 MachineBasicBlock::iterator &MBBI) {
882 MachineInstr *MI = &*MBBI;
883 unsigned Opcode = MI->getOpcode();
884
885 // Check if this is a Zilsd pseudo that needs fixing
886 if (Opcode != RISCV::PseudoLD_RV32_OPT && Opcode != RISCV::PseudoSD_RV32_OPT)
887 return false;
888
889 bool IsLoad = Opcode == RISCV::PseudoLD_RV32_OPT;
890
891 const MachineOperand &FirstOp = MI->getOperand(i: 0);
892 const MachineOperand &SecondOp = MI->getOperand(i: 1);
893 Register FirstReg = FirstOp.getReg();
894 Register SecondReg = SecondOp.getReg();
895
896 if (!isValidZilsdRegPair(First: FirstReg, Second: SecondReg)) {
897 // Need to split back into two instructions
898 splitLdSdIntoTwo(MBB, MBBI, IsLoad);
899 return true;
900 }
901
902 // Registers are valid, convert to real LD/SD instruction
903 const MachineOperand &BaseOp = MI->getOperand(i: 2);
904 Register BaseReg = BaseOp.getReg();
905 DebugLoc DL = MI->getDebugLoc();
906 // Handle both immediate and symbolic operands for offset
907 const MachineOperand &OffsetOp = MI->getOperand(i: 3);
908
909 unsigned RealOpc = IsLoad ? RISCV::LD_RV32 : RISCV::SD_RV32;
910
911 // Create register pair from the two individual registers
912 MCRegister RegPair = TRI->getMatchingSuperReg(Reg: FirstReg, SubIdx: RISCV::sub_gpr_even,
913 RC: &RISCV::GPRPairRegClass);
914 // Create the real LD/SD instruction with register pair
915 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RealOpc));
916
917 if (IsLoad) {
918 // For LD, the register pair is the destination
919 MIB.addDef(RegNo: RegPair, Flags: getDeadRegState(B: FirstOp.isDead() && SecondOp.isDead()));
920 } else {
921 // For SD, the register pair is the source
922 MIB.addReg(RegNo: RegPair, Flags: getKillRegState(B: FirstOp.isKill() && SecondOp.isKill()));
923 }
924
925 MIB.addReg(RegNo: BaseReg, Flags: getKillRegState(B: BaseOp.isKill()))
926 .add(MO: OffsetOp)
927 .cloneMemRefs(OtherMI: *MI);
928
929 LLVM_DEBUG(dbgs() << "Converted pseudo to real instruction: " << *MIB
930 << "\n");
931
932 // Remove the pseudo instruction and update iterator
933 MBBI = MBB.erase(I: MBBI);
934
935 return true;
936}
937
938// Returns an instance of the Load / Store Optimization pass.
939FunctionPass *llvm::createRISCVLoadStoreOptPass() {
940 return new RISCVLoadStoreOpt();
941}
942