1//===----- RISCVLoadStoreOptimizer.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Load/Store Pairing: It identifies pairs of load or store instructions
10// operating on consecutive memory locations and merges them into a single
11// paired instruction, leveraging hardware support for paired memory accesses.
12// Much of the pairing logic is adapted from the AArch64LoadStoreOpt pass.
13//
14// Post-allocation Zilsd decomposition: Fixes invalid LD/SD instructions if
15// register allocation didn't provide suitable consecutive registers.
16//
17// NOTE: The AArch64LoadStoreOpt pass performs additional optimizations such as
18// merging zero store instructions, promoting loads that read directly from a
19// preceding store, and merging base register updates with load/store
20// instructions (via pre-/post-indexed addressing). These advanced
21// transformations are not yet implemented in the RISC-V pass but represent
22// potential future enhancements for further optimizing RISC-V memory
23// operations.
24//
25//===----------------------------------------------------------------------===//
26
27#include "RISCV.h"
28#include "RISCVTargetMachine.h"
29#include "llvm/ADT/SmallVector.h"
30#include "llvm/ADT/Statistic.h"
31#include "llvm/Analysis/AliasAnalysis.h"
32#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
33#include "llvm/CodeGen/MachinePassManager.h"
34#include "llvm/CodeGen/Passes.h"
35#include "llvm/IR/Analysis.h"
36#include "llvm/MC/TargetRegistry.h"
37#include "llvm/Support/Debug.h"
38#include "llvm/Target/TargetOptions.h"
39
40using namespace llvm;
41
42#define DEBUG_TYPE "riscv-load-store-opt"
43#define RISCV_LOAD_STORE_OPT_NAME "RISC-V Load / Store Optimizer"
44
45STATISTIC(NumLD2LW, "Number of LD instructions split back to LW");
46STATISTIC(NumSD2SW, "Number of SD instructions split back to SW");
47
48namespace {
49
50class RISCVLoadStoreOptLegacy : public MachineFunctionPass {
51public:
52 static char ID;
53 bool runOnMachineFunction(MachineFunction &Fn) override;
54
55 RISCVLoadStoreOptLegacy() : MachineFunctionPass(ID) {}
56
57 MachineFunctionProperties getRequiredProperties() const override {
58 return MachineFunctionProperties().setNoVRegs();
59 }
60
61 void getAnalysisUsage(AnalysisUsage &AU) const override {
62 AU.setPreservesCFG();
63 AU.addRequired<AAResultsWrapperPass>();
64 AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
65 MachineFunctionPass::getAnalysisUsage(AU);
66 }
67
68 StringRef getPassName() const override { return RISCV_LOAD_STORE_OPT_NAME; }
69};
70
71class RISCVLoadStoreOptImpl {
72 // Find and pair load/store instructions.
73 bool tryToPairLdStInst(MachineBasicBlock::iterator &MBBI);
74
75 // Convert load/store pairs to single instructions.
76 bool tryConvertToLdStPair(MachineBasicBlock::iterator First,
77 MachineBasicBlock::iterator Second);
78 bool tryConvertToXqcilsmLdStPair(MachineFunction *MF,
79 MachineBasicBlock::iterator First,
80 MachineBasicBlock::iterator Second);
81 bool tryConvertToXqcilsmMultiLdSt(MachineBasicBlock::iterator &First);
82 bool tryConvertToMIPSLdStPair(MachineFunction *MF,
83 MachineBasicBlock::iterator First,
84 MachineBasicBlock::iterator Second);
85
86 // Scan the instructions looking for a load/store that can be combined
87 // with the current instruction into a load/store pair.
88 // Return the matching instruction if one is found, else MBB->end().
89 MachineBasicBlock::iterator findMatchingInsn(MachineBasicBlock::iterator I,
90 bool &MergeForward);
91
92 MachineBasicBlock::iterator
93 mergePairedInsns(MachineBasicBlock::iterator I,
94 MachineBasicBlock::iterator Paired, bool MergeForward);
95
96 // Post reg-alloc zilsd part
97 bool fixInvalidRegPairOp(MachineBasicBlock &MBB,
98 MachineBasicBlock::iterator &MBBI);
99 bool isValidZilsdRegPair(Register First, Register Second);
100 void splitLdSdIntoTwo(MachineBasicBlock &MBB,
101 MachineBasicBlock::iterator &MBBI, bool IsLoad);
102
103 AliasAnalysis *AA;
104 MachineRegisterInfo *MRI;
105 const RISCVInstrInfo *TII;
106 const RISCVRegisterInfo *TRI;
107 const RISCVSubtarget *STI = nullptr;
108 LiveRegUnits ModifiedRegUnits, UsedRegUnits;
109
110public:
111 bool runOnMachineFunction(MachineFunction &MF,
112 function_ref<AliasAnalysis *()> GetAA);
113};
114} // end anonymous namespace
115
116char RISCVLoadStoreOptLegacy::ID = 0;
117INITIALIZE_PASS(RISCVLoadStoreOptLegacy, DEBUG_TYPE, RISCV_LOAD_STORE_OPT_NAME,
118 false, false)
119
120bool RISCVLoadStoreOptImpl::runOnMachineFunction(
121 MachineFunction &MF, function_ref<AliasAnalysis *()> GetAA) {
122
123 bool MadeChange = false;
124 STI = &MF.getSubtarget<RISCVSubtarget>();
125 TII = STI->getInstrInfo();
126 TRI = STI->getRegisterInfo();
127 MRI = &MF.getRegInfo();
128 AA = GetAA();
129 ModifiedRegUnits.init(TRI: *TRI);
130 UsedRegUnits.init(TRI: *TRI);
131
132 if (STI->useMIPSLoadStorePairs() || STI->hasVendorXqcilsm()) {
133 for (MachineBasicBlock &MBB : MF) {
134 LLVM_DEBUG(dbgs() << "MBB: " << MBB.getName() << "\n");
135
136 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
137 MBBI != E;) {
138 if (TII->isPairableLdStInstOpc(Opc: MBBI->getOpcode()) &&
139 tryToPairLdStInst(MBBI))
140 MadeChange = true;
141 else
142 ++MBBI;
143 }
144 }
145 }
146
147 if (!STI->is64Bit() && STI->hasStdExtZilsd()) {
148 for (auto &MBB : MF) {
149 for (auto MBBI = MBB.begin(), E = MBB.end(); MBBI != E;) {
150 if (fixInvalidRegPairOp(MBB, MBBI)) {
151 MadeChange = true;
152 // Iterator was updated by fixInvalidRegPairOp
153 } else {
154 ++MBBI;
155 }
156 }
157 }
158 }
159
160 return MadeChange;
161}
162
163// Find loads and stores that can be merged into a single load or store pair
164// instruction.
165bool RISCVLoadStoreOptImpl::tryToPairLdStInst(
166 MachineBasicBlock::iterator &MBBI) {
167 MachineInstr &MI = *MBBI;
168
169 // If this is volatile, it is not a candidate.
170 if (MI.hasOrderedMemoryRef())
171 return false;
172
173 if (!TII->isLdStSafeToPair(LdSt: MI, TRI))
174 return false;
175
176 // If Xqcilsm is available, first try to form a multi-instruction group (>2).
177 if (!STI->is64Bit() && STI->hasVendorXqcilsm()) {
178 if (tryConvertToXqcilsmMultiLdSt(First&: MBBI))
179 return true;
180 }
181
182 // Look ahead for a pairable instruction.
183 MachineBasicBlock::iterator E = MI.getParent()->end();
184 bool MergeForward;
185 MachineBasicBlock::iterator Paired = findMatchingInsn(I: MBBI, MergeForward);
186 if (Paired != E) {
187 MBBI = mergePairedInsns(I: MBBI, Paired, MergeForward);
188 return true;
189 }
190 return false;
191}
192
193static bool isMemOpAligned(MachineInstr &MI, Align RequiredAlignment) {
194 const MachineMemOperand *MMO = *MI.memoperands_begin();
195 Align MMOAlign = MMO->getAlign();
196 return MMOAlign >= RequiredAlignment;
197}
198
199// Convert set of 3 or more LW/SW instructions to QC_LWMI/QC_SWMI/QC_SETWMI.
200// For now this only handles consecutive loads and stores traversing the basic
201// block top-down.
202// TODO: Traverse the basic block bottom-up as well.
203bool RISCVLoadStoreOptImpl::tryConvertToXqcilsmMultiLdSt(
204 MachineBasicBlock::iterator &FirstIt) {
205 MachineInstr &FirstMI = *FirstIt;
206 MachineFunction *MF = FirstMI.getMF();
207
208 if (STI->is64Bit() || !STI->hasVendorXqcilsm())
209 return false;
210
211 unsigned Opc = FirstMI.getOpcode();
212 if (Opc != RISCV::LW && Opc != RISCV::SW)
213 return false;
214
215 if (!FirstMI.hasOneMemOperand())
216 return false;
217
218 if (!isMemOpAligned(MI&: FirstMI, RequiredAlignment: Align(4)))
219 return false;
220
221 // Require simple reg+imm addressing.
222 const MachineOperand &BaseOp = FirstMI.getOperand(i: 1);
223 const MachineOperand &OffOp = FirstMI.getOperand(i: 2);
224 if (!BaseOp.isReg() || !OffOp.isImm())
225 return false;
226
227 Register Base = BaseOp.getReg();
228 int64_t BaseOff = OffOp.getImm();
229
230 if (!isShiftedUInt<5, 2>(x: BaseOff))
231 return false;
232
233 Register StartReg = FirstMI.getOperand(i: 0).getReg();
234 bool IsLoad = (Opc == RISCV::LW);
235
236 // Load rd cannot be x0 and must not clobber the base register.
237 if (IsLoad) {
238 if (StartReg == RISCV::X0)
239 return false;
240 if (StartReg == Base)
241 return false;
242 }
243
244 // Collect a set of consecutive matching instructions.
245 SmallVector<MachineInstr *, 8> Group;
246 Group.push_back(Elt: &FirstMI);
247
248 MachineBasicBlock::iterator E = FirstIt->getParent()->end();
249 MachineBasicBlock::iterator It = next_nodbg(It: FirstIt, End: E);
250 int64_t ExpectedOff = BaseOff + 4;
251 unsigned Index = 1;
252 enum class StoreMode { Unknown, Setwmi, Swmi };
253 StoreMode SMode = StoreMode::Unknown;
254
255 while (It != E) {
256 MachineInstr &MI = *It;
257
258 if (!TII->isPairableLdStInstOpc(Opc: MI.getOpcode()))
259 break;
260 if (MI.getOpcode() != Opc)
261 break;
262 if (!TII->isLdStSafeToPair(LdSt: MI, TRI))
263 break;
264 if (!MI.hasOneMemOperand())
265 break;
266 if (!isMemOpAligned(MI, RequiredAlignment: Align(4)))
267 break;
268
269 const MachineOperand &BaseMIOp = MI.getOperand(i: 1);
270 const MachineOperand &OffsetMIOp = MI.getOperand(i: 2);
271 if (!BaseMIOp.isReg() || !OffsetMIOp.isImm())
272 break;
273 if (BaseMIOp.getReg() != Base)
274 break;
275 int64_t Off = OffsetMIOp.getImm();
276 if (Off != ExpectedOff)
277 break;
278
279 Register Reg = MI.getOperand(i: 0).getReg();
280 if (IsLoad) {
281 // For loads, require consecutive destination registers.
282 if (Reg != StartReg + Index)
283 break;
284 if (Reg == Base)
285 break;
286 } else {
287 // For stores, decide mode based on the second instruction and then
288 // enforce the same for the rest.
289 if (SMode == StoreMode::Unknown) {
290 if (Reg == StartReg)
291 SMode = StoreMode::Setwmi;
292 else if (Reg == StartReg + 1)
293 SMode = StoreMode::Swmi;
294 else
295 break;
296 } else if (SMode == StoreMode::Setwmi) {
297 if (Reg != StartReg)
298 break;
299 } else {
300 if (Reg != StartReg + Index)
301 break;
302 }
303 }
304
305 // Passed checks, extend the group.
306 Group.push_back(Elt: &MI);
307 ++Index;
308 ExpectedOff += 4;
309 It = next_nodbg(It, End: E);
310 }
311
312 // We only handle more than 2 here. Pairs are handled in
313 // tryConvertToXqcilsmLdStPair.
314 unsigned Len = Group.size();
315 if (Len < 3 || Len > 31)
316 return false;
317
318 unsigned NewOpc;
319 RegState StartRegState;
320 bool AddImplicitRegs = true;
321
322 if (IsLoad) {
323 NewOpc = RISCV::QC_LWMI;
324 StartRegState = RegState::Define;
325 } else {
326 assert(SMode != StoreMode::Unknown &&
327 "Group should be large enough to know the store mode");
328 if (SMode == StoreMode::Setwmi) {
329 NewOpc = RISCV::QC_SETWMI;
330 // Kill if any of the individual stores killed the reg.
331 bool StartKill = false;
332 for (MachineInstr *MI : Group)
333 StartKill |= MI->getOperand(i: 0).isKill();
334 StartRegState = getKillRegState(B: StartKill);
335 AddImplicitRegs = false;
336 } else {
337 // SWMI requires consecutive source regs and rd != x0.
338 if (StartReg == RISCV::X0)
339 return false;
340 NewOpc = RISCV::QC_SWMI;
341 StartRegState = getKillRegState(B: Group.front()->getOperand(i: 0).isKill());
342 }
343 }
344
345 // Aggregate kill on base.
346 bool BaseKill = false;
347 for (MachineInstr *MI : Group)
348 BaseKill |= MI->getOperand(i: 1).isKill();
349
350 // Build the new instruction.
351 DebugLoc DL = FirstMI.getDebugLoc();
352 if (!DL)
353 DL = Group.back()->getDebugLoc();
354 MachineInstrBuilder MIB = BuildMI(MF&: *MF, MIMD: DL, MCID: TII->get(Opcode: NewOpc));
355 MIB.addReg(RegNo: StartReg, Flags: StartRegState)
356 .addReg(RegNo: Base, Flags: getKillRegState(B: BaseKill))
357 .addImm(Val: Len)
358 .addImm(Val: BaseOff);
359
360 // Merge memory references.
361 MIB.cloneMergedMemRefs(OtherMIs: Group);
362
363 if (AddImplicitRegs) {
364 // Add implicit operands for the additional registers.
365 for (unsigned i = 1; i < Len; ++i) {
366 Register R = StartReg + i;
367 RegState State;
368 if (IsLoad)
369 State = RegState::ImplicitDefine;
370 else
371 State = RegState::Implicit |
372 getKillRegState(B: Group[i]->getOperand(i: 0).isKill());
373 MIB.addReg(RegNo: R, Flags: State);
374 }
375 }
376
377 // Insert before the first instruction and remove all in the group.
378 MachineBasicBlock *MBB = FirstIt->getParent();
379 MachineBasicBlock::iterator NewIt = MBB->insert(I: FirstIt, MI: MIB);
380 for (MachineInstr *MI : Group)
381 MI->removeFromParent();
382
383 // Advance the cursor to the next non-debug instruction after the group.
384 FirstIt = next_nodbg(It: NewIt, End: MBB->end());
385 return true;
386}
387
388bool RISCVLoadStoreOptImpl::tryConvertToXqcilsmLdStPair(
389 MachineFunction *MF, MachineBasicBlock::iterator First,
390 MachineBasicBlock::iterator Second) {
391 unsigned Opc = First->getOpcode();
392 if ((Opc != RISCV::LW && Opc != RISCV::SW) || Second->getOpcode() != Opc)
393 return false;
394
395 const auto &FirstOp1 = First->getOperand(i: 1);
396 const auto &SecondOp1 = Second->getOperand(i: 1);
397 const auto &FirstOp2 = First->getOperand(i: 2);
398 const auto &SecondOp2 = Second->getOperand(i: 2);
399
400 // Require simple reg+imm addressing for both.
401 if (!FirstOp1.isReg() || !SecondOp1.isReg() || !FirstOp2.isImm() ||
402 !SecondOp2.isImm())
403 return false;
404
405 Register Base1 = FirstOp1.getReg();
406 Register Base2 = SecondOp1.getReg();
407
408 if (Base1 != Base2)
409 return false;
410
411 if (!First->hasOneMemOperand() || !Second->hasOneMemOperand())
412 return false;
413
414 if (!isMemOpAligned(MI&: *First, RequiredAlignment: Align(4)) || !isMemOpAligned(MI&: *Second, RequiredAlignment: Align(4)))
415 return false;
416
417 const auto &FirstOp0 = First->getOperand(i: 0);
418 const auto &SecondOp0 = Second->getOperand(i: 0);
419
420 int64_t Off1 = FirstOp2.getImm();
421 int64_t Off2 = SecondOp2.getImm();
422 assert(Off1 < Off2 && "Expected instructions to be ordered by offset");
423
424 if (!isShiftedUInt<5, 2>(x: Off1) || (Off2 - Off1 != 4))
425 return false;
426
427 Register StartReg = FirstOp0.getReg();
428 Register NextReg = SecondOp0.getReg();
429
430 unsigned XqciOpc;
431 RegState StartRegState;
432 RegState NextRegState = {};
433 bool AddNextReg = true;
434
435 if (Opc == RISCV::LW) {
436
437 if (StartReg == RISCV::X0)
438 return false;
439
440 // If the base reg gets overwritten by one of the loads bail out.
441 if (StartReg == Base1 || NextReg == Base1)
442 return false;
443
444 // The registers need to be consecutive.
445 if (NextReg != StartReg + 1)
446 return false;
447
448 XqciOpc = RISCV::QC_LWMI;
449 StartRegState = RegState::Define;
450 NextRegState = RegState::ImplicitDefine;
451 } else {
452 assert(Opc == RISCV::SW && "Expected a SW instruction");
453 if (StartReg == NextReg) {
454 XqciOpc = RISCV::QC_SETWMI;
455 StartRegState = getKillRegState(B: FirstOp0.isKill() || SecondOp0.isKill());
456 AddNextReg = false;
457 } else if (NextReg == StartReg + 1 && StartReg != RISCV::X0) {
458 XqciOpc = RISCV::QC_SWMI;
459 StartRegState = getKillRegState(B: FirstOp0.isKill());
460 NextRegState = RegState::Implicit | getKillRegState(B: SecondOp0.isKill());
461 } else {
462 return false;
463 }
464 }
465
466 DebugLoc DL =
467 First->getDebugLoc() ? First->getDebugLoc() : Second->getDebugLoc();
468 MachineInstrBuilder MIB = BuildMI(MF&: *MF, MIMD: DL, MCID: TII->get(Opcode: XqciOpc));
469 MIB.addReg(RegNo: StartReg, Flags: StartRegState)
470 .addReg(RegNo: Base1, Flags: getKillRegState(B: FirstOp1.isKill() || SecondOp1.isKill()))
471 .addImm(Val: 2)
472 .addImm(Val: Off1)
473 .cloneMergedMemRefs(OtherMIs: {&*First, &*Second});
474
475 if (AddNextReg)
476 MIB.addReg(RegNo: NextReg, Flags: NextRegState);
477
478 First->getParent()->insert(I: First, MI: MIB);
479 First->removeFromParent();
480 Second->removeFromParent();
481
482 return true;
483}
484
485bool RISCVLoadStoreOptImpl::tryConvertToMIPSLdStPair(
486 MachineFunction *MF, MachineBasicBlock::iterator First,
487 MachineBasicBlock::iterator Second) {
488 // Try converting to SWP/LWP/LDP/SDP.
489 // SWP/LWP requires 8-byte alignment whereas LDP/SDP needs 16-byte alignment.
490 unsigned PairOpc;
491 Align RequiredAlignment;
492 switch (First->getOpcode()) {
493 default:
494 llvm_unreachable("Unsupported load/store instruction for pairing");
495 case RISCV::SW:
496 PairOpc = RISCV::MIPS_SWP;
497 RequiredAlignment = Align(8);
498 break;
499 case RISCV::LW:
500 PairOpc = RISCV::MIPS_LWP;
501 RequiredAlignment = Align(8);
502 break;
503 case RISCV::SD:
504 PairOpc = RISCV::MIPS_SDP;
505 RequiredAlignment = Align(16);
506 break;
507 case RISCV::LD:
508 PairOpc = RISCV::MIPS_LDP;
509 RequiredAlignment = Align(16);
510 break;
511 }
512
513 if (!First->hasOneMemOperand())
514 return false;
515
516 if (!isMemOpAligned(MI&: *First, RequiredAlignment))
517 return false;
518
519 int64_t Offset = First->getOperand(i: 2).getImm();
520 if (!isUInt<7>(x: Offset))
521 return false;
522
523 MachineInstrBuilder MIB = BuildMI(
524 MF&: *MF, MIMD: First->getDebugLoc() ? First->getDebugLoc() : Second->getDebugLoc(),
525 MCID: TII->get(Opcode: PairOpc));
526 MIB.add(MO: First->getOperand(i: 0))
527 .add(MO: Second->getOperand(i: 0))
528 .add(MO: First->getOperand(i: 1))
529 .add(MO: First->getOperand(i: 2))
530 .cloneMergedMemRefs(OtherMIs: {&*First, &*Second});
531
532 First->getParent()->insert(I: First, MI: MIB);
533
534 First->removeFromParent();
535 Second->removeFromParent();
536
537 return true;
538}
539
540// Merge two adjacent load/store instructions into a paired instruction.
541// This function calls the vendor specific implementation that seelects the
542// appropriate paired opcode, verifies that the memory operand is properly
543// aligned, and checks that the offset is valid. If all conditions are met, it
544// builds and inserts the paired instruction.
545bool RISCVLoadStoreOptImpl::tryConvertToLdStPair(
546 MachineBasicBlock::iterator First, MachineBasicBlock::iterator Second) {
547 MachineFunction *MF = First->getMF();
548
549 // Try converting to QC_LWMI/QC_SWMI if the XQCILSM extension is enabled.
550 if (!STI->is64Bit() && STI->hasVendorXqcilsm())
551 return tryConvertToXqcilsmLdStPair(MF, First, Second);
552
553 // Else try to convert them into MIPS Paired Loads/Stores.
554 return tryConvertToMIPSLdStPair(MF, First, Second);
555}
556
557static bool mayAlias(MachineInstr &MIa,
558 SmallVectorImpl<MachineInstr *> &MemInsns,
559 AliasAnalysis *AA) {
560 for (MachineInstr *MIb : MemInsns)
561 if (MIa.mayAlias(AA, Other: *MIb, /*UseTBAA*/ false))
562 return true;
563
564 return false;
565}
566
567// Scan the instructions looking for a load/store that can be combined with the
568// current instruction into a wider equivalent or a load/store pair.
569// TODO: Extend pairing logic to consider reordering both instructions
570// to a safe "middle" position rather than only merging forward/backward.
571// This requires more sophisticated checks for aliasing, register
572// liveness, and potential scheduling hazards.
573MachineBasicBlock::iterator
574RISCVLoadStoreOptImpl::findMatchingInsn(MachineBasicBlock::iterator I,
575 bool &MergeForward) {
576 MachineBasicBlock::iterator E = I->getParent()->end();
577 MachineBasicBlock::iterator MBBI = I;
578 MachineInstr &FirstMI = *I;
579 MBBI = next_nodbg(It: MBBI, End: E);
580
581 bool MayLoad = FirstMI.mayLoad();
582 Register Reg = FirstMI.getOperand(i: 0).getReg();
583 Register BaseReg = FirstMI.getOperand(i: 1).getReg();
584 int64_t Offset = FirstMI.getOperand(i: 2).getImm();
585 int64_t OffsetStride = (*FirstMI.memoperands_begin())->getSize().getValue();
586
587 MergeForward = false;
588
589 // Track which register units have been modified and used between the first
590 // insn (inclusive) and the second insn.
591 ModifiedRegUnits.clear();
592 UsedRegUnits.clear();
593
594 // Remember any instructions that read/write memory between FirstMI and MI.
595 SmallVector<MachineInstr *, 4> MemInsns;
596
597 for (unsigned Count = 0;
598 MBBI != E && Count < STI->getCLOpts().load_store_scan_limit;
599 MBBI = next_nodbg(It: MBBI, End: E)) {
600 MachineInstr &MI = *MBBI;
601
602 // Don't count transient instructions towards the search limit since there
603 // may be different numbers of them if e.g. debug information is present.
604 if (!MI.isTransient())
605 ++Count;
606
607 if (MI.getOpcode() == FirstMI.getOpcode() &&
608 TII->isLdStSafeToPair(LdSt: MI, TRI)) {
609 Register MIBaseReg = MI.getOperand(i: 1).getReg();
610 int64_t MIOffset = MI.getOperand(i: 2).getImm();
611
612 if (BaseReg == MIBaseReg) {
613 if ((Offset != MIOffset + OffsetStride) &&
614 (Offset + OffsetStride != MIOffset)) {
615 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
616 TRI);
617 MemInsns.push_back(Elt: &MI);
618 continue;
619 }
620
621 // If the destination register of one load is the same register or a
622 // sub/super register of the other load, bail and keep looking.
623 if (MayLoad &&
624 TRI->isSuperOrSubRegisterEq(RegA: Reg, RegB: MI.getOperand(i: 0).getReg())) {
625 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
626 TRI);
627 MemInsns.push_back(Elt: &MI);
628 continue;
629 }
630
631 // If the BaseReg has been modified, then we cannot do the optimization.
632 if (!ModifiedRegUnits.available(Reg: BaseReg))
633 return E;
634
635 // If the Rt of the second instruction was not modified or used between
636 // the two instructions and none of the instructions between the second
637 // and first alias with the second, we can combine the second into the
638 // first.
639 if (ModifiedRegUnits.available(Reg: MI.getOperand(i: 0).getReg()) &&
640 !(MI.mayLoad() &&
641 !UsedRegUnits.available(Reg: MI.getOperand(i: 0).getReg())) &&
642 !mayAlias(MIa&: MI, MemInsns, AA)) {
643
644 MergeForward = false;
645 return MBBI;
646 }
647
648 // Likewise, if the Rt of the first instruction is not modified or used
649 // between the two instructions and none of the instructions between the
650 // first and the second alias with the first, we can combine the first
651 // into the second.
652 if (!(MayLoad &&
653 !UsedRegUnits.available(Reg: FirstMI.getOperand(i: 0).getReg())) &&
654 !mayAlias(MIa&: FirstMI, MemInsns, AA)) {
655
656 if (ModifiedRegUnits.available(Reg: FirstMI.getOperand(i: 0).getReg())) {
657 MergeForward = true;
658 return MBBI;
659 }
660 }
661 // Unable to combine these instructions due to interference in between.
662 // Keep looking.
663 }
664 }
665
666 // If the instruction wasn't a matching load or store. Stop searching if we
667 // encounter a call instruction that might modify memory.
668 if (MI.isCall())
669 return E;
670
671 // Update modified / uses register units.
672 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
673
674 // Otherwise, if the base register is modified, we have no match, so
675 // return early.
676 if (!ModifiedRegUnits.available(Reg: BaseReg))
677 return E;
678
679 // Update list of instructions that read/write memory.
680 if (MI.mayLoadOrStore())
681 MemInsns.push_back(Elt: &MI);
682 }
683 return E;
684}
685
686MachineBasicBlock::iterator
687RISCVLoadStoreOptImpl::mergePairedInsns(MachineBasicBlock::iterator I,
688 MachineBasicBlock::iterator Paired,
689 bool MergeForward) {
690 MachineBasicBlock::iterator E = I->getParent()->end();
691 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
692 // If NextI is the second of the two instructions to be merged, skip one
693 // further for now. For the MIPS load/store, the merge will invalidate the
694 // iterator, and we don't need to scan the new instruction, as it's a pairwise
695 // instruction, which we're not considering for further action anyway. For the
696 // Xqcilsm load/store, we may not want to do this as the second instruction
697 // could possibly be the first in another pair if we do not merge here. This
698 // is handled in the else block after the call to tryConvertToLdStPair below.
699 if (NextI == Paired)
700 NextI = next_nodbg(It: NextI, End: E);
701
702 // Insert our new paired instruction after whichever of the paired
703 // instructions MergeForward indicates.
704 MachineBasicBlock::iterator InsertionPoint = MergeForward ? Paired : I;
705 MachineBasicBlock::iterator DeletionPoint = MergeForward ? I : Paired;
706 int Offset = I->getOperand(i: 2).getImm();
707 int PairedOffset = Paired->getOperand(i: 2).getImm();
708 bool InsertAfter = (Offset < PairedOffset) ^ MergeForward;
709
710 if (!MergeForward)
711 Paired->getOperand(i: 1).setIsKill(false);
712
713 // Kill flags may become invalid when moving stores for pairing.
714 if (I->getOperand(i: 0).isUse()) {
715 if (!MergeForward) {
716 // Check if the Paired store's source register has a kill flag and clear
717 // it only if there are intermediate uses between I and Paired.
718 MachineOperand &PairedRegOp = Paired->getOperand(i: 0);
719 if (PairedRegOp.isKill()) {
720 for (auto It = std::next(x: I); It != Paired; ++It) {
721 if (It->readsRegister(Reg: PairedRegOp.getReg(), TRI)) {
722 PairedRegOp.setIsKill(false);
723 break;
724 }
725 }
726 }
727 } else {
728 // Clear kill flags of the first store's register in the forward
729 // direction.
730 Register Reg = I->getOperand(i: 0).getReg();
731 for (MachineInstr &MI : make_range(x: std::next(x: I), y: std::next(x: Paired)))
732 MI.clearRegisterKills(Reg, RegInfo: TRI);
733 }
734 }
735
736 // Remember the original position of the instruction we're moving so we can
737 // restore it if we fail to form a pair.
738 MachineBasicBlock::iterator OrigNext = std::next(x: DeletionPoint);
739
740 MachineInstr *ToInsert = DeletionPoint->removeFromParent();
741 MachineBasicBlock &MBB = *InsertionPoint->getParent();
742 MachineBasicBlock::iterator First, Second, Moved;
743
744 if (!InsertAfter) {
745 First = MBB.insert(I: InsertionPoint, MI: ToInsert);
746 Second = InsertionPoint;
747 Moved = First;
748 } else {
749 Second = MBB.insertAfter(I: InsertionPoint, MI: ToInsert);
750 First = InsertionPoint;
751 Moved = Second;
752 }
753
754 if (tryConvertToLdStPair(First, Second)) {
755 LLVM_DEBUG(dbgs() << "Pairing load/store:\n ");
756 LLVM_DEBUG(prev_nodbg(NextI, MBB.begin())->print(dbgs()));
757 } else if (!STI->is64Bit() && STI->hasVendorXqcilsm()) {
758 // We were unable to form the pair, so move the instruction back to it's
759 // original place. Point NextI to the next non-debug instruction after the
760 // first instruction we had wanted to merge.
761 MachineInstr *MovedMI = Moved->removeFromParent();
762 MBB.insert(I: OrigNext, MI: MovedMI);
763 NextI = next_nodbg(It: I, End: E);
764 }
765
766 return NextI;
767}
768
769//===----------------------------------------------------------------------===//
770// Post reg-alloc zilsd pass implementation
771//===----------------------------------------------------------------------===//
772
773bool RISCVLoadStoreOptImpl::isValidZilsdRegPair(Register First,
774 Register Second) {
775 // Special case: First register can not be zero unless both registers are
776 // zeros.
777 // Spec says: LD instructions with destination x0 are processed as any other
778 // load, but the result is discarded entirely and x1 is not written. If using
779 // x0 as src of SD, the entire 64-bit operand is zero — i.e., register x1 is
780 // not accessed.
781 if (First == RISCV::X0)
782 return Second == RISCV::X0;
783
784 // Check if registers form a valid even/odd pair for Zilsd
785 unsigned FirstNum = TRI->getEncodingValue(Reg: First);
786 unsigned SecondNum = TRI->getEncodingValue(Reg: Second);
787
788 // Must be consecutive and first must be even
789 return (FirstNum % 2 == 0) && (SecondNum == FirstNum + 1);
790}
791
792void RISCVLoadStoreOptImpl::splitLdSdIntoTwo(MachineBasicBlock &MBB,
793 MachineBasicBlock::iterator &MBBI,
794 bool IsLoad) {
795 MachineInstr *MI = &*MBBI;
796 DebugLoc DL = MI->getDebugLoc();
797
798 const MachineOperand &FirstOp = MI->getOperand(i: 0);
799 const MachineOperand &SecondOp = MI->getOperand(i: 1);
800 const MachineOperand &BaseOp = MI->getOperand(i: 2);
801 Register FirstReg = FirstOp.getReg();
802 Register SecondReg = SecondOp.getReg();
803 Register BaseReg = BaseOp.getReg();
804
805 // Handle both immediate and symbolic operands for offset
806 const MachineOperand &OffsetOp = MI->getOperand(i: 3);
807 int BaseOffset;
808 if (OffsetOp.isImm())
809 BaseOffset = OffsetOp.getImm();
810 else
811 // For symbolic operands, extract the embedded offset
812 BaseOffset = OffsetOp.getOffset();
813
814 unsigned Opc = IsLoad ? RISCV::LW : RISCV::SW;
815 MachineInstrBuilder MIB1, MIB2;
816
817 // Create two separate instructions
818 if (IsLoad) {
819 // It's possible that first register is same as base register, when we split
820 // it becomes incorrect because base register is overwritten, e.g.
821 // X10, X13 = PseudoLD_RV32_OPT killed X10, 0
822 // =>
823 // X10 = LW X10, 0
824 // X13 = LW killed X10, 4
825 // we can just switch the order to resolve that:
826 // X13 = LW X10, 4
827 // X10 = LW killed X10, 0
828 if (FirstReg == BaseReg) {
829 MIB2 = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc))
830 .addDef(RegNo: SecondReg, Flags: getDeadRegState(B: SecondOp.isDead()))
831 .addReg(RegNo: BaseReg);
832 MIB1 = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc))
833 .addDef(RegNo: FirstReg, Flags: getDeadRegState(B: FirstOp.isDead()))
834 .addReg(RegNo: BaseReg, Flags: getKillRegState(B: BaseOp.isKill()));
835
836 } else {
837 MIB1 = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc))
838 .addDef(RegNo: FirstReg, Flags: getDeadRegState(B: FirstOp.isDead()))
839 .addReg(RegNo: BaseReg);
840
841 MIB2 = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc))
842 .addDef(RegNo: SecondReg, Flags: getDeadRegState(B: SecondOp.isDead()))
843 .addReg(RegNo: BaseReg, Flags: getKillRegState(B: BaseOp.isKill()));
844 }
845
846 ++NumLD2LW;
847 LLVM_DEBUG(dbgs() << "Split LD back to two LW instructions\n");
848 } else {
849 assert(
850 FirstReg != SecondReg &&
851 "First register and second register is impossible to be same register");
852 MIB1 = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc))
853 .addReg(RegNo: FirstReg, Flags: getKillRegState(B: FirstOp.isKill()))
854 .addReg(RegNo: BaseReg);
855
856 MIB2 = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc))
857 .addReg(RegNo: SecondReg, Flags: getKillRegState(B: SecondOp.isKill()))
858 .addReg(RegNo: BaseReg, Flags: getKillRegState(B: BaseOp.isKill()));
859
860 ++NumSD2SW;
861 LLVM_DEBUG(dbgs() << "Split SD back to two SW instructions\n");
862 }
863
864 // Add offset operands - preserve symbolic references
865 MIB1.add(MO: OffsetOp);
866 if (OffsetOp.isImm())
867 MIB2.addImm(Val: BaseOffset + 4);
868 else if (OffsetOp.isGlobal())
869 MIB2.addGlobalAddress(GV: OffsetOp.getGlobal(), Offset: BaseOffset + 4,
870 TargetFlags: OffsetOp.getTargetFlags());
871 else if (OffsetOp.isCPI())
872 MIB2.addConstantPoolIndex(Idx: OffsetOp.getIndex(), Offset: BaseOffset + 4,
873 TargetFlags: OffsetOp.getTargetFlags());
874 else if (OffsetOp.isBlockAddress())
875 MIB2.addBlockAddress(BA: OffsetOp.getBlockAddress(), Offset: BaseOffset + 4,
876 TargetFlags: OffsetOp.getTargetFlags());
877
878 // Copy memory operands if the original instruction had them
879 // FIXME: This is overly conservative; the new instruction accesses 4 bytes,
880 // not 8.
881 MIB1.cloneMemRefs(OtherMI: *MI);
882 MIB2.cloneMemRefs(OtherMI: *MI);
883
884 // Remove the original paired instruction and update iterator
885 MBBI = MBB.erase(I: MBBI);
886}
887
888bool RISCVLoadStoreOptImpl::fixInvalidRegPairOp(
889 MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI) {
890 MachineInstr *MI = &*MBBI;
891 unsigned Opcode = MI->getOpcode();
892
893 // Check if this is a Zilsd pseudo that needs fixing
894 if (Opcode != RISCV::PseudoLD_RV32_OPT && Opcode != RISCV::PseudoSD_RV32_OPT)
895 return false;
896
897 bool IsLoad = Opcode == RISCV::PseudoLD_RV32_OPT;
898
899 const MachineOperand &FirstOp = MI->getOperand(i: 0);
900 const MachineOperand &SecondOp = MI->getOperand(i: 1);
901 Register FirstReg = FirstOp.getReg();
902 Register SecondReg = SecondOp.getReg();
903
904 if (!isValidZilsdRegPair(First: FirstReg, Second: SecondReg)) {
905 // Need to split back into two instructions
906 splitLdSdIntoTwo(MBB, MBBI, IsLoad);
907 return true;
908 }
909
910 // Registers are valid, convert to real LD/SD instruction
911 const MachineOperand &BaseOp = MI->getOperand(i: 2);
912 Register BaseReg = BaseOp.getReg();
913 DebugLoc DL = MI->getDebugLoc();
914 // Handle both immediate and symbolic operands for offset
915 const MachineOperand &OffsetOp = MI->getOperand(i: 3);
916
917 unsigned RealOpc = IsLoad ? RISCV::LD_RV32 : RISCV::SD_RV32;
918
919 // Create register pair from the two individual registers
920 MCRegister RegPair = TRI->getMatchingSuperReg(Reg: FirstReg, SubIdx: RISCV::sub_gpr_even,
921 RC: &RISCV::GPRPairRegClass);
922 // Create the real LD/SD instruction with register pair
923 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RealOpc));
924
925 if (IsLoad) {
926 // For LD, the register pair is the destination
927 MIB.addDef(RegNo: RegPair, Flags: getDeadRegState(B: FirstOp.isDead() && SecondOp.isDead()));
928 } else {
929 // For SD, the register pair is the source
930 MIB.addReg(RegNo: RegPair, Flags: getKillRegState(B: FirstOp.isKill() && SecondOp.isKill()));
931 }
932
933 MIB.addReg(RegNo: BaseReg, Flags: getKillRegState(B: BaseOp.isKill()))
934 .add(MO: OffsetOp)
935 .cloneMemRefs(OtherMI: *MI);
936
937 LLVM_DEBUG(dbgs() << "Converted pseudo to real instruction: " << *MIB
938 << "\n");
939
940 // Remove the pseudo instruction and update iterator
941 MBBI = MBB.erase(I: MBBI);
942
943 return true;
944}
945
946// Returns an instance of the Load / Store Optimization pass.
947FunctionPass *llvm::createRISCVLoadStoreOptLegacyPass() {
948 return new RISCVLoadStoreOptLegacy();
949}
950
951bool RISCVLoadStoreOptLegacy::runOnMachineFunction(MachineFunction &MF) {
952 if (skipFunction(F: MF.getFunction()))
953 return false;
954 RISCVLoadStoreOptImpl Impl;
955 return Impl.runOnMachineFunction(MF, GetAA: [&]() {
956 return &getAnalysis<AAResultsWrapperPass>().getAAResults();
957 });
958}
959
960PreservedAnalyses
961RISCVLoadStoreOptPass::run(MachineFunction &MF,
962 MachineFunctionAnalysisManager &MFAM) {
963 RISCVLoadStoreOptImpl Impl;
964 bool Changed = Impl.runOnMachineFunction(MF, GetAA: [&]() {
965 return &MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
966 .getManager()
967 .getResult<AAManager>(IR&: MF.getFunction());
968 });
969 return Changed ? getMachineFunctionPassPreservedAnalyses()
970 .preserveSet<CFGAnalyses>()
971 : PreservedAnalyses::all();
972}
973