1//===-- SlotIndexes.cpp - Slot Indexes Pass ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/CodeGen/SlotIndexes.h"
10#include "llvm/ADT/Statistic.h"
11#include "llvm/CodeGen/MachineFunction.h"
12#include "llvm/Config/llvm-config.h"
13#include "llvm/InitializePasses.h"
14#include "llvm/Support/Debug.h"
15#include "llvm/Support/raw_ostream.h"
16
17using namespace llvm;
18
19#define DEBUG_TYPE "slot-indexes"
20
21AnalysisKey SlotIndexesAnalysis::Key;
22
23SlotIndexesAnalysis::Result
24SlotIndexesAnalysis::run(MachineFunction &MF,
25 MachineFunctionAnalysisManager &) {
26 return Result(MF);
27}
28
29PreservedAnalyses
30SlotIndexesPrinterPass::run(MachineFunction &MF,
31 MachineFunctionAnalysisManager &MFAM) {
32 OS << "Slot indexes in machine function: " << MF.getName() << '\n';
33 MFAM.getResult<SlotIndexesAnalysis>(IR&: MF).print(OS);
34 return PreservedAnalyses::all();
35}
36char SlotIndexesWrapperPass::ID = 0;
37
38SlotIndexesWrapperPass::SlotIndexesWrapperPass() : MachineFunctionPass(ID) {}
39
40SlotIndexes::~SlotIndexes() {
41 // The indexList's nodes are all allocated in the BumpPtrAllocator.
42 indexList.clear();
43}
44
45INITIALIZE_PASS(SlotIndexesWrapperPass, DEBUG_TYPE, "Slot index numbering",
46 false, false)
47
48STATISTIC(NumLocalRenum, "Number of local renumberings");
49
50void SlotIndexesWrapperPass::getAnalysisUsage(AnalysisUsage &au) const {
51 au.setPreservesAll();
52 MachineFunctionPass::getAnalysisUsage(AU&: au);
53}
54
55void SlotIndexes::clear() {
56 mi2iMap.clear();
57 MBBRanges.clear();
58 idx2MBBMap.clear();
59 indexList.clear();
60 ileAllocator.Reset();
61}
62
63void SlotIndexes::analyze(MachineFunction &fn) {
64
65 // Compute numbering as follows:
66 // Grab an iterator to the start of the index list.
67 // Iterate over all MBBs, and within each MBB all MIs, keeping the MI
68 // iterator in lock-step (though skipping it over indexes which have
69 // null pointers in the instruction field).
70 // At each iteration assert that the instruction pointed to in the index
71 // is the same one pointed to by the MI iterator. This
72
73 // FIXME: This can be simplified. The mi2iMap_, Idx2MBBMap, etc. should
74 // only need to be set up once after the first numbering is computed.
75
76 mf = &fn;
77
78 // Check that the list contains only the sentinel.
79 assert(indexList.empty() && "Index list non-empty at initial numbering?");
80 assert(idx2MBBMap.empty() &&
81 "Index -> MBB mapping non-empty at initial numbering?");
82 assert(MBBRanges.empty() &&
83 "MBB -> Index mapping non-empty at initial numbering?");
84 assert(mi2iMap.empty() &&
85 "MachineInstr -> Index mapping non-empty at initial numbering?");
86
87 unsigned index = 0;
88 MBBRanges.resize(N: mf->getMaxAnalysisBlockNumber());
89 idx2MBBMap.reserve(N: mf->size());
90
91 indexList.push_back(Node&: *createEntry(mi: nullptr, index));
92
93 // Iterate over the function.
94 for (MachineBasicBlock &MBB : *mf) {
95 // Insert an index for the MBB start.
96 SlotIndex blockStartIndex(&indexList.back(), SlotIndex::Slot_Block);
97
98 for (MachineInstr &MI : MBB) {
99 if (MI.isDebugOrPseudoInstr())
100 continue;
101
102 // Insert a store index for the instr.
103 indexList.push_back(Node&: *createEntry(mi: &MI, index: index += SlotIndex::InstrDist));
104
105 // Save this base index in the maps.
106 mi2iMap.insert(KV: std::make_pair(
107 x: &MI, y: SlotIndex(&indexList.back(), SlotIndex::Slot_Block)));
108 }
109
110 // We insert one blank instructions between basic blocks.
111 indexList.push_back(Node&: *createEntry(mi: nullptr, index: index += SlotIndex::InstrDist));
112
113 MBBRanges[MBB.getAnalysisNumber()].first = blockStartIndex;
114 MBBRanges[MBB.getAnalysisNumber()].second =
115 SlotIndex(&indexList.back(), SlotIndex::Slot_Block);
116 idx2MBBMap.push_back(Elt: IdxMBBPair(blockStartIndex, &MBB));
117 }
118
119 // Sort the Idx2MBBMap
120 llvm::sort(C&: idx2MBBMap, Comp: less_first());
121
122 LLVM_DEBUG(mf->print(dbgs(), this));
123}
124
125bool SlotIndexes::isBlockBoundaryIndex(SlotIndex Idx) const {
126 if (getInstructionFromIndex(index: Idx))
127 return false;
128
129 // Adjacent blocks share a boundary entry, so a boundary is either the start
130 // index of the block Idx falls in or the end index of the last block. A block
131 // dropped by removeMBBFromMaps() is gone from idx2MBBMap, so its start entry
132 // classifies as stale.
133 assert(!idx2MBBMap.empty() && "Index -> MBB mapping is empty");
134 SlotIndex Base = Idx.getBaseIndex();
135 MBBIndexIterator I = std::prev(x: getMBBUpperBound(Idx: Base));
136 return Base == I->first || Base == getMBBEndIdx(mbb: I->second);
137}
138
139SlotIndex SlotIndexes::canonicalizeIndex(SlotIndex Idx) const {
140 if (!isStaleIndex(Idx))
141 return Idx;
142
143 // The block start is a boundary entry, so it bounds the walk.
144 SlotIndex BlockStart = getMBBStartIdx(mbb: getMBBFromIndex(index: Idx));
145 IndexList::iterator I = Idx.listEntry()->getIterator();
146 while (&*I != BlockStart.listEntry()) {
147 --I;
148 if (I->getInstr())
149 return SlotIndex(&*I, SlotIndex::Slot_Register);
150 }
151 return BlockStart;
152}
153
154void SlotIndexes::removeMachineInstrFromMaps(MachineInstr &MI,
155 bool AllowBundled) {
156 assert((AllowBundled || !MI.isBundledWithPred()) &&
157 "Use removeSingleMachineInstrFromMaps() instead");
158 Mi2IndexMap::iterator mi2iItr = mi2iMap.find(Val: &MI);
159 if (mi2iItr == mi2iMap.end())
160 return;
161
162 SlotIndex MIIndex = mi2iItr->second;
163 IndexListEntry &MIEntry = *MIIndex.listEntry();
164 assert(MIEntry.getInstr() == &MI && "Instruction indexes broken.");
165 mi2iMap.erase(I: mi2iItr);
166 // FIXME: Eventually we want to actually delete these indexes.
167 MIEntry.setInstr(nullptr);
168}
169
170void SlotIndexes::removeSingleMachineInstrFromMaps(MachineInstr &MI) {
171 Mi2IndexMap::iterator mi2iItr = mi2iMap.find(Val: &MI);
172 if (mi2iItr == mi2iMap.end())
173 return;
174
175 SlotIndex MIIndex = mi2iItr->second;
176 IndexListEntry &MIEntry = *MIIndex.listEntry();
177 assert(MIEntry.getInstr() == &MI && "Instruction indexes broken.");
178 mi2iMap.erase(I: mi2iItr);
179
180 // When removing the first instruction of a bundle update mapping to next
181 // instruction.
182 if (MI.isBundledWithSucc()) {
183 // Only the first instruction of a bundle should have an index assigned.
184 assert(!MI.isBundledWithPred() && "Should be first bundle instruction");
185
186 MachineBasicBlock::instr_iterator Next = std::next(x: MI.getIterator());
187 MachineInstr &NextMI = *Next;
188 MIEntry.setInstr(&NextMI);
189 mi2iMap.insert(KV: std::make_pair(x: &NextMI, y&: MIIndex));
190 return;
191 } else {
192 // FIXME: Eventually we want to actually delete these indexes.
193 MIEntry.setInstr(nullptr);
194 }
195}
196
197void SlotIndexes::removeMBBFromMaps(MachineBasicBlock &MBB) {
198 assert(&MBB != &MBB.getParent()->front() &&
199 "Can't remove the first block of a function.");
200
201 unsigned Num = MBB.getAnalysisNumber();
202 SlotIndex StartIdx = MBBRanges[Num].first;
203 SlotIndex EndIdx = MBBRanges[Num].second;
204
205 // Give MBB's slot range to its layout predecessor so blocks stay contiguous.
206 auto PrevMBB = std::prev(x: MBB.getIterator());
207 MBBRanges[PrevMBB->getAnalysisNumber()].second = EndIdx;
208
209 // Drop MBB's index -> MBB entry, which would dangle once MBB is erased.
210 auto It = getMBBLowerBound(Idx: StartIdx);
211 assert(It != MBBIndexEnd() && It->first == StartIdx && It->second == &MBB &&
212 "MBB not found in index -> MBB map");
213 idx2MBBMap.erase(CI: It);
214
215 // Clear the block-start boundary entry. MBBRanges is never renumbered, so
216 // MBB's now-stale slot is simply left in place.
217 StartIdx.listEntry()->setInstr(nullptr);
218}
219
220// Renumber indexes locally after curItr was inserted, but failed to get a new
221// index.
222void SlotIndexes::renumberIndexes(IndexList::iterator curItr) {
223 // Number indexes with half the default spacing so we can catch up quickly.
224 const unsigned Space = SlotIndex::InstrDist/2;
225 static_assert((Space & 3) == 0, "InstrDist must be a multiple of 2*NUM");
226
227 IndexList::iterator startItr = std::prev(x: curItr);
228 unsigned index = startItr->getIndex();
229 unsigned BeginIndex = index;
230 do {
231 curItr->setIndex(index += Space);
232 ++curItr;
233 // If the next index is bigger, we have caught up.
234 } while (curItr != indexList.end() && curItr->getIndex() <= index);
235
236 LLVM_DEBUG(dbgs() << "\n*** Renumbered SlotIndexes " << startItr->getIndex()
237 << '-' << index << " ***\n");
238
239 // If we repack more than 20% of a function, add spacing in between the
240 // instructions so that future renumberings are able to catch up
241 // without also renumbering so much.
242 if (index - BeginIndex >
243 (getLastIndex().getIndex() - getZeroIndex().getIndex()) / 5)
244 packIndexes();
245
246 ++NumLocalRenum;
247}
248
249// Repair indexes after adding and removing instructions.
250void SlotIndexes::repairIndexesInRange(MachineBasicBlock *MBB,
251 MachineBasicBlock::iterator Begin,
252 MachineBasicBlock::iterator End) {
253 bool includeStart = (Begin == MBB->begin());
254 SlotIndex startIdx;
255 if (includeStart)
256 startIdx = getMBBStartIdx(mbb: MBB);
257 else
258 startIdx = getInstructionIndex(MI: *--Begin);
259
260 SlotIndex endIdx;
261 if (End == MBB->end())
262 endIdx = getMBBEndIdx(mbb: MBB);
263 else
264 endIdx = getInstructionIndex(MI: *End);
265
266 // FIXME: Conceptually, this code is implementing an iterator on MBB that
267 // optionally includes an additional position prior to MBB->begin(), indicated
268 // by the includeStart flag. This is done so that we can iterate MIs in a MBB
269 // in parallel with SlotIndexes, but there should be a better way to do this.
270 IndexList::iterator ListB = startIdx.listEntry()->getIterator();
271 IndexList::iterator ListI = endIdx.listEntry()->getIterator();
272 MachineBasicBlock::iterator MBBI = End;
273 bool pastStart = false;
274 bool OldIndexesRemoved = false;
275 while (ListI != ListB || MBBI != Begin || (includeStart && !pastStart)) {
276 assert(ListI->getIndex() >= startIdx.getIndex() &&
277 (includeStart || !pastStart) &&
278 "Decremented past the beginning of region to repair.");
279
280 MachineInstr *SlotMI = ListI->getInstr();
281 MachineInstr *MI = (MBBI != MBB->end() && !pastStart) ? &*MBBI : nullptr;
282 bool MBBIAtBegin = MBBI == Begin && (!includeStart || pastStart);
283 bool MIIndexNotFound = MI && !mi2iMap.contains(Val: MI);
284 bool SlotMIRemoved = false;
285
286 if (SlotMI == MI && !MBBIAtBegin) {
287 --ListI;
288 if (MBBI != Begin)
289 --MBBI;
290 else
291 pastStart = true;
292 } else if (MIIndexNotFound || OldIndexesRemoved) {
293 if (MBBI != Begin)
294 --MBBI;
295 else
296 pastStart = true;
297 } else {
298 // We ran through all the indexes on the interval
299 // -> The only thing left is to go through all the
300 // remaining MBB instructions and update their indexes
301 if (ListI == ListB)
302 OldIndexesRemoved = true;
303 else
304 --ListI;
305 if (SlotMI) {
306 removeMachineInstrFromMaps(MI&: *SlotMI);
307 SlotMIRemoved = true;
308 }
309 }
310
311 MachineInstr *InstrToInsert = SlotMIRemoved ? SlotMI : MI;
312
313 // Insert instruction back into the maps after passing it/removing the index
314 if ((MIIndexNotFound || SlotMIRemoved) && InstrToInsert->getParent() &&
315 !InstrToInsert->isDebugOrPseudoInstr())
316 insertMachineInstrInMaps(MI&: *InstrToInsert);
317 }
318}
319
320void SlotIndexes::packIndexes() {
321 for (auto [Index, Entry] : enumerate(First&: indexList))
322 Entry.setIndex(Index * SlotIndex::InstrDist);
323}
324
325void SlotIndexes::print(raw_ostream &OS) const {
326 for (const IndexListEntry &ILE : indexList) {
327 OS << ILE.getIndex() << ' ';
328
329 if (ILE.getInstr())
330 OS << *ILE.getInstr();
331 else
332 OS << '\n';
333 }
334
335 for (const MachineBasicBlock &MBB : *mf)
336 OS << printMBBReference(MBB) << "\t[" << getMBBStartIdx(mbb: &MBB) << ';'
337 << getMBBEndIdx(mbb: &MBB) << ")\n";
338}
339
340#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
341LLVM_DUMP_METHOD void SlotIndexes::dump() const { print(dbgs()); }
342#endif
343
344// Print a SlotIndex to a raw_ostream.
345void SlotIndex::print(raw_ostream &os) const {
346 if (isValid())
347 os << listEntry()->getIndex() << "Berd"[getSlot()];
348 else
349 os << "invalid";
350}
351
352#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
353// Dump a SlotIndex to stderr.
354LLVM_DUMP_METHOD void SlotIndex::dump() const {
355 print(dbgs());
356 dbgs() << "\n";
357}
358#endif
359