1//===-- AMDGPUMarkLastScratchLoad.cpp -------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Mark scratch load/spill instructions which are guaranteed to be the last time
10// this scratch slot is used so it can be evicted from caches.
11//
12// TODO: Handle general stack accesses not just spilling.
13//
14//===----------------------------------------------------------------------===//
15
16#include "AMDGPU.h"
17#include "GCNSubtarget.h"
18#include "llvm/CodeGen/LiveIntervals.h"
19#include "llvm/CodeGen/LiveStacks.h"
20#include "llvm/CodeGen/MachineOperand.h"
21
22using namespace llvm;
23
24#define DEBUG_TYPE "amdgpu-mark-last-scratch-load"
25
26namespace {
27
28class AMDGPUMarkLastScratchLoad {
29private:
30 LiveStacks *LS = nullptr;
31 LiveIntervals *LIS = nullptr;
32 SlotIndexes *SI = nullptr;
33 const SIInstrInfo *SII = nullptr;
34
35public:
36 AMDGPUMarkLastScratchLoad(LiveStacks *LS, LiveIntervals *LIS, SlotIndexes *SI)
37 : LS(LS), LIS(LIS), SI(SI) {}
38 bool run(MachineFunction &MF);
39};
40
41class AMDGPUMarkLastScratchLoadLegacy : public MachineFunctionPass {
42public:
43 static char ID;
44
45 AMDGPUMarkLastScratchLoadLegacy() : MachineFunctionPass(ID) {}
46
47 bool runOnMachineFunction(MachineFunction &MF) override;
48
49 void getAnalysisUsage(AnalysisUsage &AU) const override {
50 AU.addRequired<SlotIndexesWrapperPass>();
51 AU.addRequired<LiveIntervalsWrapperPass>();
52 AU.addRequired<LiveStacksWrapperLegacy>();
53 AU.setPreservesAll();
54 MachineFunctionPass::getAnalysisUsage(AU);
55 }
56
57 StringRef getPassName() const override {
58 return "AMDGPU Mark Last Scratch Load";
59 }
60};
61
62} // end anonymous namespace
63
64bool AMDGPUMarkLastScratchLoadLegacy::runOnMachineFunction(
65 MachineFunction &MF) {
66 if (skipFunction(F: MF.getFunction()))
67 return false;
68
69 auto &LS = getAnalysis<LiveStacksWrapperLegacy>().getLS();
70 auto &LIS = getAnalysis<LiveIntervalsWrapperPass>().getLIS();
71 auto &SI = getAnalysis<SlotIndexesWrapperPass>().getSI();
72
73 return AMDGPUMarkLastScratchLoad(&LS, &LIS, &SI).run(MF);
74}
75
76PreservedAnalyses
77AMDGPUMarkLastScratchLoadPass::run(MachineFunction &MF,
78 MachineFunctionAnalysisManager &MFAM) {
79 auto &LS = MFAM.getResult<LiveStacksAnalysis>(IR&: MF);
80 auto &LIS = MFAM.getResult<LiveIntervalsAnalysis>(IR&: MF);
81 auto &SI = MFAM.getResult<SlotIndexesAnalysis>(IR&: MF);
82
83 AMDGPUMarkLastScratchLoad(&LS, &LIS, &SI).run(MF);
84 return PreservedAnalyses::all();
85}
86
87bool AMDGPUMarkLastScratchLoad::run(MachineFunction &MF) {
88 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
89 if (ST.getGeneration() < AMDGPUSubtarget::GFX12)
90 return false;
91
92 SII = ST.getInstrInfo();
93 SlotIndexes &Slots = *LIS->getSlotIndexes();
94
95 const unsigned NumSlots = LS->getNumIntervals();
96 if (NumSlots == 0) {
97 LLVM_DEBUG(dbgs() << "No live slots, skipping\n");
98 return false;
99 }
100
101 LLVM_DEBUG(dbgs() << LS->getNumIntervals() << " intervals\n");
102
103 bool Changed = false;
104
105 for (auto &[SS, LI] : *LS) {
106 for (const LiveRange::Segment &Segment : LI.segments) {
107
108 // Ignore segments that run to the end of basic block because in this case
109 // slot is still live at the end of it.
110 if (Segment.end.isBlock())
111 continue;
112
113 const int FrameIndex = LI.reg().stackSlotIndex();
114 MachineInstr *LastLoad = nullptr;
115
116 MachineInstr *MISegmentEnd = SI->getInstructionFromIndex(index: Segment.end);
117
118 // If there is no instruction at this slot because it was deleted take the
119 // instruction from the next slot.
120 if (!MISegmentEnd) {
121 SlotIndex NextSlot = Slots.getNextNonNullIndex(Index: Segment.end);
122 MISegmentEnd = SI->getInstructionFromIndex(index: NextSlot);
123 }
124
125 MachineInstr *MISegmentStart = SI->getInstructionFromIndex(index: Segment.start);
126 MachineBasicBlock *BB = MISegmentEnd->getParent();
127
128 // Start iteration backwards from segment end until the start of basic
129 // block or start of segment if it is in the same basic block.
130 auto End = BB->rend();
131 if (MISegmentStart && MISegmentStart->getParent() == BB)
132 End = MISegmentStart->getReverseIterator();
133
134 for (auto MI = MISegmentEnd->getReverseIterator(); MI != End; ++MI) {
135 int LoadFI = 0;
136
137 if (SII->isLoadFromStackSlot(MI: *MI, FrameIndex&: LoadFI) && LoadFI == FrameIndex) {
138 LastLoad = &*MI;
139 break;
140 }
141 }
142
143 if (LastLoad && !LastLoad->memoperands_empty()) {
144 MachineMemOperand *MMO = *LastLoad->memoperands_begin();
145 MMO->setFlags(MOLastUse);
146 Changed = true;
147 LLVM_DEBUG(dbgs() << " Found last load: " << *LastLoad);
148 }
149 }
150 }
151
152 return Changed;
153}
154
155char AMDGPUMarkLastScratchLoadLegacy::ID = 0;
156
157char &llvm::AMDGPUMarkLastScratchLoadID = AMDGPUMarkLastScratchLoadLegacy::ID;
158
159INITIALIZE_PASS_BEGIN(AMDGPUMarkLastScratchLoadLegacy, DEBUG_TYPE,
160 "AMDGPU Mark last scratch load", false, false)
161INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
162INITIALIZE_PASS_DEPENDENCY(LiveStacksWrapperLegacy)
163INITIALIZE_PASS_END(AMDGPUMarkLastScratchLoadLegacy, DEBUG_TYPE,
164 "AMDGPU Mark last scratch load", false, false)
165