1//===-- BasicBlockPathCloning.cpp ---=========-----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// BasicBlockPathCloning implementation.
11///
12/// The purpose of this pass is to clone basic block paths based on information
13/// provided by the -fbasic-block-sections=list option.
14/// Please refer to BasicBlockSectionsProfileReader.cpp to see a path cloning
15/// example.
16//===----------------------------------------------------------------------===//
17// This pass clones the machine basic blocks alongs the given paths and sets up
18// the CFG. It assigns BBIDs to the cloned blocks so that the
19// `BasicBlockSections` pass can correctly map the cluster information to the
20// blocks. The cloned block's BBID will have the same BaseID as the original
21// block, but will get a unique non-zero CloneID (original blocks all have zero
22// CloneIDs). This pass applies a path cloning if it satisfies the following
23// conditions:
24// 1. All BBIDs in the path should be mapped to existing blocks.
25// 2. Each two consecutive BBIDs in the path must have a successor
26// relationship in the CFG.
27// 3. The path should not include a block with indirect branches, except for
28// the last block.
29// If a path does not satisfy all three conditions, it will be rejected, but the
30// CloneIDs for its (supposed to be cloned) blocks will be bypassed to make sure
31// that the `BasicBlockSections` pass can map cluster info correctly to the
32// actually-cloned blocks.
33//===----------------------------------------------------------------------===//
34
35#include "llvm/ADT/SmallVector.h"
36#include "llvm/ADT/StringRef.h"
37#include "llvm/CodeGen/BasicBlockSectionUtils.h"
38#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
39#include "llvm/CodeGen/MachineFunction.h"
40#include "llvm/CodeGen/MachineFunctionPass.h"
41#include "llvm/CodeGen/Passes.h"
42#include "llvm/CodeGen/TargetInstrInfo.h"
43#include "llvm/InitializePasses.h"
44#include "llvm/Support/WithColor.h"
45#include "llvm/Target/TargetMachine.h"
46
47using namespace llvm;
48
49namespace {
50
51// Clones the given block and assigns the given `CloneID` to its BBID. Copies
52// the instructions into the new block and sets up its successors.
53MachineBasicBlock *CloneMachineBasicBlock(MachineBasicBlock &OrigBB,
54 unsigned CloneID) {
55 auto &MF = *OrigBB.getParent();
56 auto TII = MF.getSubtarget().getInstrInfo();
57 // Create the clone block and set its BBID based on the original block.
58 MachineBasicBlock *CloneBB = MF.CreateMachineBasicBlock(
59 BB: OrigBB.getBasicBlock(), BBID: UniqueBBID{.BaseID: OrigBB.getBBID()->BaseID, .CloneID: CloneID});
60 MF.push_back(MBB: CloneBB);
61
62 // Copy the instructions.
63 for (auto &I : OrigBB.instrs()) {
64 // Bundled instructions are duplicated together.
65 if (I.isBundledWithPred())
66 continue;
67 TII->duplicate(MBB&: *CloneBB, InsertBefore: CloneBB->end(), Orig: I);
68 }
69
70 // Add the successors of the original block as the new block's successors.
71 // We set the predecessor after returning from this call.
72 for (auto SI = OrigBB.succ_begin(), SE = OrigBB.succ_end(); SI != SE; ++SI)
73 CloneBB->copySuccessor(Orig: &OrigBB, I: SI);
74
75 if (auto FT = OrigBB.getFallThrough(/*JumpToFallThrough=*/false)) {
76 // The original block has an implicit fall through.
77 // Insert an explicit unconditional jump from the cloned block to the
78 // fallthrough block. Technically, this is only needed for the last block
79 // of the path, but we do it for all clones for consistency.
80 TII->insertUnconditionalBranch(MBB&: *CloneBB, DestBB: FT, DL: CloneBB->findBranchDebugLoc());
81 }
82 return CloneBB;
83}
84
85// Returns if we can legally apply the cloning represented by `ClonePath`.
86// `BBIDToBlock` contains the original basic blocks in function `MF` keyed by
87// their `BBID::BaseID`.
88bool IsValidCloning(const MachineFunction &MF,
89 const DenseMap<unsigned, MachineBasicBlock *> &BBIDToBlock,
90 const SmallVector<unsigned> &ClonePath) {
91 const MachineBasicBlock *PrevBB = nullptr;
92 for (size_t I = 0; I < ClonePath.size(); ++I) {
93 unsigned BBID = ClonePath[I];
94 const MachineBasicBlock *PathBB = BBIDToBlock.lookup(Val: BBID);
95 if (!PathBB) {
96 WithColor::warning() << "no block with id " << BBID << " in function "
97 << MF.getName() << "\n";
98 return false;
99 }
100
101 if (PrevBB) {
102 if (!PrevBB->isSuccessor(MBB: PathBB)) {
103 WithColor::warning()
104 << "block #" << BBID << " is not a successor of block #"
105 << PrevBB->getBBID()->BaseID << " in function " << MF.getName()
106 << "\n";
107 return false;
108 }
109
110 for (auto &MI : *PathBB) {
111 // Avoid cloning when the block contains non-duplicable instructions.
112 // CFI instructions are marked as non-duplicable only because of Darwin,
113 // so we exclude them from this check.
114 if (MI.isNotDuplicable() && !MI.isCFIInstruction()) {
115 WithColor::warning()
116 << "block #" << BBID
117 << " has non-duplicable instructions in function " << MF.getName()
118 << "\n";
119 return false;
120 }
121 }
122 if (PathBB->isMachineBlockAddressTaken()) {
123 // Avoid cloning blocks which have their address taken since we can't
124 // rewire branches to those blocks as easily.
125 WithColor::warning()
126 << "block #" << BBID
127 << " has its machine block address taken in function "
128 << MF.getName() << "\n";
129 return false;
130 }
131 if (PathBB->isInlineAsmBrIndirectTarget()) {
132 // Similarly for branches to the block within an asm goto.
133 WithColor::warning()
134 << "block #" << BBID
135 << " is a branch target of an 'asm goto' in function "
136 << MF.getName() << "\n";
137 return false;
138 }
139 }
140
141 if (I != ClonePath.size() - 1 && !PathBB->empty() &&
142 PathBB->back().isIndirectBranch()) {
143 WithColor::warning()
144 << "block #" << BBID
145 << " has indirect branch and appears as the non-tail block of a "
146 "path in function "
147 << MF.getName() << "\n";
148 return false;
149 }
150 PrevBB = PathBB;
151 }
152 return true;
153}
154
155// Applies all clonings specified in `ClonePaths` to `MF`. Returns true
156// if any clonings have been applied.
157bool ApplyCloning(MachineFunction &MF,
158 const SmallVector<SmallVector<unsigned>> &ClonePaths) {
159 if (ClonePaths.empty())
160 return false;
161 bool AnyPathsCloned = false;
162 // Map from the final BB IDs to the `MachineBasicBlock`s.
163 DenseMap<unsigned, MachineBasicBlock *> BBIDToBlock;
164 for (auto &BB : MF)
165 BBIDToBlock.try_emplace(Key: BB.getBBID()->BaseID, Args: &BB);
166
167 DenseMap<unsigned, unsigned> NClonesForBBID;
168 auto TII = MF.getSubtarget().getInstrInfo();
169 for (const auto &ClonePath : ClonePaths) {
170 if (!IsValidCloning(MF, BBIDToBlock, ClonePath)) {
171 // We still need to increment the number of clones so we can map
172 // to the cluster info correctly.
173 for (unsigned BBID : ClonePath)
174 ++NClonesForBBID[BBID];
175 continue;
176 }
177 MachineBasicBlock *PrevBB = nullptr;
178 for (unsigned BBID : ClonePath) {
179 MachineBasicBlock *OrigBB = BBIDToBlock.at(Val: BBID);
180 if (PrevBB == nullptr) {
181 // The first block in the path is not cloned. We only need to make it
182 // branch to the next cloned block in the path. Here, we make its
183 // fallthrough explicit so we can change it later.
184 if (auto FT = OrigBB->getFallThrough(/*JumpToFallThrough=*/false)) {
185 TII->insertUnconditionalBranch(MBB&: *OrigBB, DestBB: FT,
186 DL: OrigBB->findBranchDebugLoc());
187 }
188 PrevBB = OrigBB;
189 continue;
190 }
191 MachineBasicBlock *CloneBB =
192 CloneMachineBasicBlock(OrigBB&: *OrigBB, CloneID: ++NClonesForBBID[BBID]);
193
194 // Set up the previous block in the path to jump to the clone. This also
195 // transfers the successor/predecessor relationship of PrevBB and OrigBB
196 // to that of PrevBB and CloneBB.
197 PrevBB->ReplaceUsesOfBlockWith(Old: OrigBB, New: CloneBB);
198
199 // Copy the livein set.
200 for (auto &LiveIn : OrigBB->liveins())
201 CloneBB->addLiveIn(RegMaskPair: LiveIn);
202
203 PrevBB = CloneBB;
204 }
205 AnyPathsCloned = true;
206 }
207 return AnyPathsCloned;
208}
209} // end anonymous namespace
210
211namespace llvm {
212class BasicBlockPathCloning : public MachineFunctionPass {
213public:
214 static char ID;
215
216 BasicBlockSectionsProfileReaderWrapperPass *BBSectionsProfileReader = nullptr;
217
218 BasicBlockPathCloning() : MachineFunctionPass(ID) {
219 initializeBasicBlockPathCloningPass(*PassRegistry::getPassRegistry());
220 }
221
222 StringRef getPassName() const override { return "Basic Block Path Cloning"; }
223
224 void getAnalysisUsage(AnalysisUsage &AU) const override;
225
226 /// Identify basic blocks that need separate sections and prepare to emit them
227 /// accordingly.
228 bool runOnMachineFunction(MachineFunction &MF) override;
229};
230
231} // namespace llvm
232
233char BasicBlockPathCloning::ID = 0;
234INITIALIZE_PASS_BEGIN(
235 BasicBlockPathCloning, "bb-path-cloning",
236 "Applies path clonings for the -basic-block-sections=list option", false,
237 false)
238INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReaderWrapperPass)
239INITIALIZE_PASS_END(
240 BasicBlockPathCloning, "bb-path-cloning",
241 "Applies path clonings for the -basic-block-sections=list option", false,
242 false)
243
244bool BasicBlockPathCloning::runOnMachineFunction(MachineFunction &MF) {
245 assert(MF.getTarget().getBBSectionsType() == BasicBlockSection::List &&
246 "BB Sections list not enabled!");
247 if (hasInstrProfHashMismatch(MF))
248 return false;
249
250 return ApplyCloning(MF,
251 ClonePaths: getAnalysis<BasicBlockSectionsProfileReaderWrapperPass>()
252 .getClonePathsForFunction(FuncName: MF.getName()));
253}
254
255void BasicBlockPathCloning::getAnalysisUsage(AnalysisUsage &AU) const {
256 AU.setPreservesAll();
257 AU.addRequired<BasicBlockSectionsProfileReaderWrapperPass>();
258 MachineFunctionPass::getAnalysisUsage(AU);
259}
260
261MachineFunctionPass *llvm::createBasicBlockPathCloningPass() {
262 return new BasicBlockPathCloning();
263}
264