1//===- SIAnnotateControlFlow.cpp ------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Annotates the control flow with hardware specific intrinsics.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "AMDGPUTargetMachine.h"
16#include "GCNSubtarget.h"
17#include "llvm/Analysis/LoopInfo.h"
18#include "llvm/Analysis/UniformityAnalysis.h"
19#include "llvm/CodeGen/TargetPassConfig.h"
20#include "llvm/IR/BasicBlock.h"
21#include "llvm/IR/Constants.h"
22#include "llvm/IR/Dominators.h"
23#include "llvm/IR/IRBuilder.h"
24#include "llvm/IR/IntrinsicsAMDGPU.h"
25#include "llvm/Target/TargetMachine.h"
26#include "llvm/Transforms/Utils/BasicBlockUtils.h"
27#include "llvm/Transforms/Utils/Local.h"
28
29using namespace llvm;
30
31#define DEBUG_TYPE "si-annotate-control-flow"
32
33namespace {
34
35// Complex types used in this pass
36using StackEntry = std::pair<BasicBlock *, Value *>;
37using StackVector = SmallVector<StackEntry, 16>;
38
39class SIAnnotateControlFlow {
40private:
41 Function *F;
42 UniformityInfo *UA;
43
44 Type *Boolean;
45 Type *Void;
46 Type *IntMask;
47 Type *ReturnStruct;
48
49 ConstantInt *BoolTrue;
50 ConstantInt *BoolFalse;
51 PoisonValue *BoolPoison;
52 Constant *IntMaskZero;
53
54 Function *If = nullptr;
55 Function *Else = nullptr;
56 Function *IfBreak = nullptr;
57 Function *Loop = nullptr;
58 Function *EndCf = nullptr;
59
60 DominatorTree *DT;
61 StackVector Stack;
62
63 LoopInfo *LI;
64
65 void initialize(const GCNSubtarget &ST);
66
67 bool isUniform(BranchInst *T);
68
69 bool isTopOfStack(BasicBlock *BB);
70
71 Value *popSaved();
72
73 void push(BasicBlock *BB, Value *Saved);
74
75 bool isElse(PHINode *Phi);
76
77 bool hasKill(const BasicBlock *BB);
78
79 bool eraseIfUnused(PHINode *Phi);
80
81 bool openIf(BranchInst *Term);
82
83 bool insertElse(BranchInst *Term);
84
85 Value *
86 handleLoopCondition(Value *Cond, PHINode *Broken, llvm::Loop *L,
87 BranchInst *Term);
88
89 bool handleLoop(BranchInst *Term);
90
91 bool closeControlFlow(BasicBlock *BB);
92
93 Function *getDecl(Function *&Cache, Intrinsic::ID ID, ArrayRef<Type *> Tys) {
94 if (!Cache)
95 Cache = Intrinsic::getOrInsertDeclaration(M: F->getParent(), id: ID, Tys);
96 return Cache;
97 }
98
99public:
100 SIAnnotateControlFlow(Function &F, const GCNSubtarget &ST, DominatorTree &DT,
101 LoopInfo &LI, UniformityInfo &UA)
102 : F(&F), UA(&UA), DT(&DT), LI(&LI) {
103 initialize(ST);
104 }
105
106 bool run();
107};
108
109} // end anonymous namespace
110
111/// Initialize all the types and constants used in the pass
112void SIAnnotateControlFlow::initialize(const GCNSubtarget &ST) {
113 LLVMContext &Context = F->getContext();
114
115 Void = Type::getVoidTy(C&: Context);
116 Boolean = Type::getInt1Ty(C&: Context);
117 IntMask = ST.isWave32() ? Type::getInt32Ty(C&: Context)
118 : Type::getInt64Ty(C&: Context);
119 ReturnStruct = StructType::get(elt1: Boolean, elts: IntMask);
120
121 BoolTrue = ConstantInt::getTrue(Context);
122 BoolFalse = ConstantInt::getFalse(Context);
123 BoolPoison = PoisonValue::get(T: Boolean);
124 IntMaskZero = ConstantInt::get(Ty: IntMask, V: 0);
125}
126
127/// Is the branch condition uniform or did the StructurizeCFG pass
128/// consider it as such?
129bool SIAnnotateControlFlow::isUniform(BranchInst *T) {
130 return UA->isUniform(I: T) || T->hasMetadata(Kind: "structurizecfg.uniform");
131}
132
133/// Is BB the last block saved on the stack ?
134bool SIAnnotateControlFlow::isTopOfStack(BasicBlock *BB) {
135 return !Stack.empty() && Stack.back().first == BB;
136}
137
138/// Pop the last saved value from the control flow stack
139Value *SIAnnotateControlFlow::popSaved() {
140 return Stack.pop_back_val().second;
141}
142
143/// Push a BB and saved value to the control flow stack
144void SIAnnotateControlFlow::push(BasicBlock *BB, Value *Saved) {
145 Stack.push_back(Elt: std::pair(BB, Saved));
146}
147
148/// Can the condition represented by this PHI node treated like
149/// an "Else" block?
150bool SIAnnotateControlFlow::isElse(PHINode *Phi) {
151 BasicBlock *IDom = DT->getNode(BB: Phi->getParent())->getIDom()->getBlock();
152 for (unsigned i = 0, e = Phi->getNumIncomingValues(); i != e; ++i) {
153 if (Phi->getIncomingBlock(i) == IDom) {
154
155 if (Phi->getIncomingValue(i) != BoolTrue)
156 return false;
157
158 } else {
159 if (Phi->getIncomingValue(i) != BoolFalse)
160 return false;
161
162 }
163 }
164 return true;
165}
166
167bool SIAnnotateControlFlow::hasKill(const BasicBlock *BB) {
168 for (const Instruction &I : *BB) {
169 if (const CallInst *CI = dyn_cast<CallInst>(Val: &I))
170 if (CI->getIntrinsicID() == Intrinsic::amdgcn_kill)
171 return true;
172 }
173 return false;
174}
175
176// Erase "Phi" if it is not used any more. Return true if any change was made.
177bool SIAnnotateControlFlow::eraseIfUnused(PHINode *Phi) {
178 bool Changed = RecursivelyDeleteDeadPHINode(PN: Phi);
179 if (Changed)
180 LLVM_DEBUG(dbgs() << "Erased unused condition phi\n");
181 return Changed;
182}
183
184/// Open a new "If" block
185bool SIAnnotateControlFlow::openIf(BranchInst *Term) {
186 if (isUniform(T: Term))
187 return false;
188
189 IRBuilder<> IRB(Term);
190 Value *IfCall = IRB.CreateCall(Callee: getDecl(Cache&: If, ID: Intrinsic::amdgcn_if, Tys: IntMask),
191 Args: {Term->getCondition()});
192 Value *Cond = IRB.CreateExtractValue(Agg: IfCall, Idxs: {0});
193 Value *Mask = IRB.CreateExtractValue(Agg: IfCall, Idxs: {1});
194 Term->setCondition(Cond);
195 push(BB: Term->getSuccessor(i: 1), Saved: Mask);
196 return true;
197}
198
199/// Close the last "If" block and open a new "Else" block
200bool SIAnnotateControlFlow::insertElse(BranchInst *Term) {
201 if (isUniform(T: Term)) {
202 return false;
203 }
204
205 IRBuilder<> IRB(Term);
206 Value *ElseCall = IRB.CreateCall(
207 Callee: getDecl(Cache&: Else, ID: Intrinsic::amdgcn_else, Tys: {IntMask, IntMask}), Args: {popSaved()});
208 Value *Cond = IRB.CreateExtractValue(Agg: ElseCall, Idxs: {0});
209 Value *Mask = IRB.CreateExtractValue(Agg: ElseCall, Idxs: {1});
210 Term->setCondition(Cond);
211 push(BB: Term->getSuccessor(i: 1), Saved: Mask);
212 return true;
213}
214
215/// Recursively handle the condition leading to a loop
216Value *SIAnnotateControlFlow::handleLoopCondition(
217 Value *Cond, PHINode *Broken, llvm::Loop *L, BranchInst *Term) {
218
219 auto CreateBreak = [this, Cond, Broken](Instruction *I) -> CallInst * {
220 return IRBuilder<>(I).CreateCall(
221 Callee: getDecl(Cache&: IfBreak, ID: Intrinsic::amdgcn_if_break, Tys: IntMask), Args: {Cond, Broken});
222 };
223
224 if (Instruction *Inst = dyn_cast<Instruction>(Val: Cond)) {
225 BasicBlock *Parent = Inst->getParent();
226 Instruction *Insert;
227 if (LI->getLoopFor(BB: Parent) == L) {
228 // Insert IfBreak in the same BB as Cond, which can help
229 // SILowerControlFlow to know that it does not have to insert an
230 // AND with EXEC.
231 Insert = Parent->getTerminator();
232 } else if (L->contains(Inst)) {
233 Insert = Term;
234 } else {
235 Insert = &*L->getHeader()->getFirstNonPHIOrDbgOrLifetime();
236 }
237
238 return CreateBreak(Insert);
239 }
240
241 // Insert IfBreak in the loop header TERM for constant COND other than true.
242 if (isa<Constant>(Val: Cond)) {
243 Instruction *Insert = Cond == BoolTrue ?
244 Term : L->getHeader()->getTerminator();
245
246 return CreateBreak(Insert);
247 }
248
249 if (isa<Argument>(Val: Cond)) {
250 Instruction *Insert = &*L->getHeader()->getFirstNonPHIOrDbgOrLifetime();
251 return CreateBreak(Insert);
252 }
253
254 llvm_unreachable("Unhandled loop condition!");
255}
256
257/// Handle a back edge (loop)
258bool SIAnnotateControlFlow::handleLoop(BranchInst *Term) {
259 if (isUniform(T: Term))
260 return false;
261
262 BasicBlock *BB = Term->getParent();
263 llvm::Loop *L = LI->getLoopFor(BB);
264 if (!L)
265 return false;
266
267 BasicBlock *Target = Term->getSuccessor(i: 1);
268 PHINode *Broken = PHINode::Create(Ty: IntMask, NumReservedValues: 0, NameStr: "phi.broken");
269 Broken->insertBefore(InsertPos: Target->begin());
270
271 Value *Cond = Term->getCondition();
272 Term->setCondition(BoolTrue);
273 Value *Arg = handleLoopCondition(Cond, Broken, L, Term);
274
275 for (BasicBlock *Pred : predecessors(BB: Target)) {
276 Value *PHIValue = IntMaskZero;
277 if (Pred == BB) // Remember the value of the previous iteration.
278 PHIValue = Arg;
279 // If the backedge from Pred to Target could be executed before the exit
280 // of the loop at BB, it should not reset or change "Broken", which keeps
281 // track of the number of threads exited the loop at BB.
282 else if (L->contains(BB: Pred) && DT->dominates(A: Pred, B: BB))
283 PHIValue = Broken;
284 Broken->addIncoming(V: PHIValue, BB: Pred);
285 }
286
287 CallInst *LoopCall = IRBuilder<>(Term).CreateCall(
288 Callee: getDecl(Cache&: Loop, ID: Intrinsic::amdgcn_loop, Tys: IntMask), Args: {Arg});
289 Term->setCondition(LoopCall);
290
291 push(BB: Term->getSuccessor(i: 0), Saved: Arg);
292
293 return true;
294}
295
296/// Close the last opened control flow
297bool SIAnnotateControlFlow::closeControlFlow(BasicBlock *BB) {
298 llvm::Loop *L = LI->getLoopFor(BB);
299
300 assert(Stack.back().first == BB);
301
302 if (L && L->getHeader() == BB) {
303 // We can't insert an EndCF call into a loop header, because it will
304 // get executed on every iteration of the loop, when it should be
305 // executed only once before the loop.
306 SmallVector <BasicBlock *, 8> Latches;
307 L->getLoopLatches(LoopLatches&: Latches);
308
309 SmallVector<BasicBlock *, 2> Preds;
310 for (BasicBlock *Pred : predecessors(BB)) {
311 if (!is_contained(Range&: Latches, Element: Pred))
312 Preds.push_back(Elt: Pred);
313 }
314
315 BB = SplitBlockPredecessors(BB, Preds, Suffix: "endcf.split", DT, LI, MSSAU: nullptr,
316 PreserveLCSSA: false);
317 }
318
319 Value *Exec = popSaved();
320 BasicBlock::iterator FirstInsertionPt = BB->getFirstInsertionPt();
321 if (!isa<UndefValue>(Val: Exec) && !isa<UnreachableInst>(Val: FirstInsertionPt)) {
322 Instruction *ExecDef = cast<Instruction>(Val: Exec);
323 BasicBlock *DefBB = ExecDef->getParent();
324 if (!DT->dominates(A: DefBB, B: BB)) {
325 // Split edge to make Def dominate Use
326 FirstInsertionPt = SplitEdge(From: DefBB, To: BB, DT, LI)->getFirstInsertionPt();
327 }
328 IRBuilder<> IRB(FirstInsertionPt->getParent(), FirstInsertionPt);
329 // TODO: StructurizeCFG 'Flow' blocks have debug locations from the
330 // condition, for now just avoid copying these DebugLocs so that stepping
331 // out of the then/else block in a debugger doesn't step to the condition.
332 IRB.SetCurrentDebugLocation(DebugLoc());
333 IRB.CreateCall(Callee: getDecl(Cache&: EndCf, ID: Intrinsic::amdgcn_end_cf, Tys: IntMask), Args: {Exec});
334 }
335
336 return true;
337}
338
339/// Annotate the control flow with intrinsics so the backend can
340/// recognize if/then/else and loops.
341bool SIAnnotateControlFlow::run() {
342 bool Changed = false;
343
344 for (df_iterator<BasicBlock *> I = df_begin(G: &F->getEntryBlock()),
345 E = df_end(G: &F->getEntryBlock());
346 I != E; ++I) {
347 BasicBlock *BB = *I;
348 BranchInst *Term = dyn_cast<BranchInst>(Val: BB->getTerminator());
349
350 if (!Term || Term->isUnconditional()) {
351 if (isTopOfStack(BB))
352 Changed |= closeControlFlow(BB);
353
354 continue;
355 }
356
357 if (I.nodeVisited(Node: Term->getSuccessor(i: 1))) {
358 if (isTopOfStack(BB))
359 Changed |= closeControlFlow(BB);
360
361 if (DT->dominates(A: Term->getSuccessor(i: 1), B: BB))
362 Changed |= handleLoop(Term);
363 continue;
364 }
365
366 if (isTopOfStack(BB)) {
367 PHINode *Phi = dyn_cast<PHINode>(Val: Term->getCondition());
368 if (Phi && Phi->getParent() == BB && isElse(Phi) && !hasKill(BB)) {
369 Changed |= insertElse(Term);
370 Changed |= eraseIfUnused(Phi);
371 continue;
372 }
373
374 Changed |= closeControlFlow(BB);
375 }
376
377 Changed |= openIf(Term);
378 }
379
380 if (!Stack.empty()) {
381 // CFG was probably not structured.
382 report_fatal_error(reason: "failed to annotate CFG");
383 }
384
385 return Changed;
386}
387
388PreservedAnalyses SIAnnotateControlFlowPass::run(Function &F,
389 FunctionAnalysisManager &FAM) {
390 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
391
392 DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(IR&: F);
393 UniformityInfo &UI = FAM.getResult<UniformityInfoAnalysis>(IR&: F);
394 LoopInfo &LI = FAM.getResult<LoopAnalysis>(IR&: F);
395
396 SIAnnotateControlFlow Impl(F, ST, DT, LI, UI);
397
398 bool Changed = Impl.run();
399 if (!Changed)
400 return PreservedAnalyses::all();
401
402 // TODO: Is LoopInfo preserved?
403 PreservedAnalyses PA = PreservedAnalyses::none();
404 PA.preserve<DominatorTreeAnalysis>();
405 return PA;
406}
407
408class SIAnnotateControlFlowLegacy : public FunctionPass {
409public:
410 static char ID;
411
412 SIAnnotateControlFlowLegacy() : FunctionPass(ID) {}
413
414 StringRef getPassName() const override { return "SI annotate control flow"; }
415
416 void getAnalysisUsage(AnalysisUsage &AU) const override {
417 AU.addRequired<LoopInfoWrapperPass>();
418 AU.addRequired<DominatorTreeWrapperPass>();
419 AU.addRequired<UniformityInfoWrapperPass>();
420 AU.addPreserved<LoopInfoWrapperPass>();
421 AU.addPreserved<DominatorTreeWrapperPass>();
422 AU.addRequired<TargetPassConfig>();
423 FunctionPass::getAnalysisUsage(AU);
424 }
425
426 bool runOnFunction(Function &F) override {
427 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
428 LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
429 UniformityInfo &UI =
430 getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
431 TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
432 const TargetMachine &TM = TPC.getTM<TargetMachine>();
433 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
434
435 SIAnnotateControlFlow Impl(F, ST, DT, LI, UI);
436 return Impl.run();
437 }
438};
439
440INITIALIZE_PASS_BEGIN(SIAnnotateControlFlowLegacy, DEBUG_TYPE,
441 "Annotate SI Control Flow", false, false)
442INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
443INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
444INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
445INITIALIZE_PASS_END(SIAnnotateControlFlowLegacy, DEBUG_TYPE,
446 "Annotate SI Control Flow", false, false)
447
448char SIAnnotateControlFlowLegacy::ID = 0;
449
450/// Create the annotation pass
451FunctionPass *llvm::createSIAnnotateControlFlowLegacyPass() {
452 return new SIAnnotateControlFlowLegacy();
453}
454