1//===-- SILateBranchLowering.cpp - Final preparation of branches ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass mainly lowers early terminate pseudo instructions.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "AMDGPULaneMaskUtils.h"
16#include "GCNSubtarget.h"
17#include "SIMachineFunctionInfo.h"
18#include "llvm/CodeGen/MachineDominators.h"
19#include "llvm/CodeGen/MachineLoopInfo.h"
20#include "llvm/CodeGen/MachinePassManager.h"
21#include "llvm/InitializePasses.h"
22
23using namespace llvm;
24
25#define DEBUG_TYPE "si-late-branch-lowering"
26
27namespace {
28
29class SILateBranchLowering {
30private:
31 const GCNSubtarget &ST;
32 const SIInstrInfo *TII;
33 const SIRegisterInfo *TRI;
34 MachineDominatorTree *MDT;
35 MachineLoopInfo *MLI;
36 const AMDGPU::LaneMaskConstants &LMC;
37
38 void expandChainCall(MachineInstr &MI, const GCNSubtarget &ST,
39 bool DynamicVGPR);
40 void earlyTerm(MachineInstr &MI, MachineBasicBlock *EarlyExitBlock);
41
42public:
43 SILateBranchLowering(const GCNSubtarget &ST, MachineDominatorTree *MDT,
44 MachineLoopInfo *MLI)
45 : ST(ST), TII(ST.getInstrInfo()), TRI(&TII->getRegisterInfo()), MDT(MDT),
46 MLI(MLI), LMC(AMDGPU::LaneMaskConstants::get(ST)) {}
47
48 bool run(MachineFunction &MF);
49};
50
51class SILateBranchLoweringLegacy : public MachineFunctionPass {
52public:
53 static char ID;
54 SILateBranchLoweringLegacy() : MachineFunctionPass(ID) {}
55
56 bool runOnMachineFunction(MachineFunction &MF) override {
57 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
58 auto *MDT = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
59 auto *MLIWP = getAnalysisIfAvailable<MachineLoopInfoWrapperPass>();
60 MachineLoopInfo *MLI = MLIWP ? &MLIWP->getLI() : nullptr;
61 return SILateBranchLowering(ST, MDT, MLI).run(MF);
62 }
63
64 StringRef getPassName() const override {
65 return "SI Final Branch Preparation";
66 }
67
68 void getAnalysisUsage(AnalysisUsage &AU) const override {
69 AU.addRequired<MachineDominatorTreeWrapperPass>();
70 AU.addPreserved<MachineDominatorTreeWrapperPass>();
71 AU.addPreserved<MachineLoopInfoWrapperPass>();
72 MachineFunctionPass::getAnalysisUsage(AU);
73 }
74};
75
76} // end anonymous namespace
77
78char SILateBranchLoweringLegacy::ID = 0;
79
80INITIALIZE_PASS_BEGIN(SILateBranchLoweringLegacy, DEBUG_TYPE,
81 "SI insert s_cbranch_execz instructions", false, false)
82INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
83INITIALIZE_PASS_END(SILateBranchLoweringLegacy, DEBUG_TYPE,
84 "SI insert s_cbranch_execz instructions", false, false)
85
86char &llvm::SILateBranchLoweringPassID = SILateBranchLoweringLegacy::ID;
87
88static void generateEndPgm(MachineBasicBlock &MBB,
89 MachineBasicBlock::iterator I, DebugLoc DL,
90 const SIInstrInfo *TII, MachineFunction &MF) {
91 const Function &F = MF.getFunction();
92 bool IsPS = F.getCallingConv() == CallingConv::AMDGPU_PS;
93
94 // Check if hardware has been configured to expect color or depth exports.
95 bool HasColorExports = AMDGPU::getHasColorExport(F);
96 bool HasDepthExports = AMDGPU::getHasDepthExport(F);
97 bool HasExports = HasColorExports || HasDepthExports;
98
99 // Prior to GFX10, hardware always expects at least one export for PS.
100 bool MustExport = !AMDGPU::isGFX10Plus(STI: TII->getSubtarget());
101
102 if (IsPS && (HasExports || MustExport)) {
103 // Generate "null export" if hardware is expecting PS to export.
104 const GCNSubtarget &ST = MBB.getParent()->getSubtarget<GCNSubtarget>();
105 int Target =
106 ST.hasNullExportTarget()
107 ? AMDGPU::Exp::ET_NULL
108 : (HasColorExports ? AMDGPU::Exp::ET_MRT0 : AMDGPU::Exp::ET_MRTZ);
109 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::EXP_DONE))
110 .addImm(Val: Target)
111 .addReg(RegNo: AMDGPU::VGPR0, Flags: RegState::Undef)
112 .addReg(RegNo: AMDGPU::VGPR0, Flags: RegState::Undef)
113 .addReg(RegNo: AMDGPU::VGPR0, Flags: RegState::Undef)
114 .addReg(RegNo: AMDGPU::VGPR0, Flags: RegState::Undef)
115 .addImm(Val: 1) // vm
116 .addImm(Val: 0) // compr
117 .addImm(Val: 0); // en
118 }
119
120 // s_endpgm
121 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ENDPGM)).addImm(Val: 0);
122}
123
124static void splitBlock(MachineBasicBlock &MBB, MachineInstr &MI,
125 MachineDominatorTree *MDT, MachineLoopInfo *MLI) {
126 MachineBasicBlock *SplitBB = MBB.splitAt(SplitInst&: MI, /*UpdateLiveIns*/ true);
127
128 // Update dominator tree
129 using DomTreeT = DomTreeBase<MachineBasicBlock>;
130 SmallVector<DomTreeT::UpdateType, 16> DTUpdates;
131 for (MachineBasicBlock *Succ : SplitBB->successors()) {
132 DTUpdates.push_back(Elt: {DomTreeT::Insert, SplitBB, Succ});
133 DTUpdates.push_back(Elt: {DomTreeT::Delete, &MBB, Succ});
134 }
135 DTUpdates.push_back(Elt: {DomTreeT::Insert, &MBB, SplitBB});
136 MDT->applyUpdates(Updates: DTUpdates);
137
138 // Update loop info if available
139 if (MLI) {
140 if (MachineLoop *Loop = MLI->getLoopFor(BB: &MBB))
141 Loop->addBasicBlockToLoop(NewBB: SplitBB, LI&: *MLI);
142 }
143}
144
145static void copyOpWithoutRegFlags(MachineInstrBuilder &MIB,
146 MachineOperand &Op) {
147 if (Op.isReg())
148 MIB.addReg(RegNo: Op.getReg());
149 else
150 MIB.add(MO: Op);
151}
152
153void SILateBranchLowering::expandChainCall(MachineInstr &MI,
154 const GCNSubtarget &ST,
155 bool DynamicVGPR) {
156 // This is a tail call that needs to be expanded into at least
157 // 2 instructions, one for setting EXEC and one for the actual tail call.
158 int ExecIdx =
159 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::exec);
160 assert(ExecIdx != -1 && "Missing EXEC operand");
161 const DebugLoc &DL = MI.getDebugLoc();
162 if (DynamicVGPR) {
163 // We have 3 extra operands and we need to:
164 // * Try to change the VGPR allocation
165 // * Select the callee based on the result of the reallocation attempt
166 // * Select the EXEC mask based on the result of the reallocation attempt
167 // If any of the register operands of the chain pseudo is used in more than
168 // one of these instructions, we need to make sure that the kill flags
169 // aren't copied along.
170 auto AllocMI =
171 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ALLOC_VGPR));
172 copyOpWithoutRegFlags(MIB&: AllocMI,
173 Op&: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::numvgprs));
174
175 auto SelectCallee =
176 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CSELECT_B64))
177 .addDef(RegNo: TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0)->getReg());
178 copyOpWithoutRegFlags(MIB&: SelectCallee,
179 Op&: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0));
180 copyOpWithoutRegFlags(MIB&: SelectCallee,
181 Op&: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::fbcallee));
182
183 auto SelectExec = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: TII->get(Opcode: LMC.CSelectOpc))
184 .addDef(RegNo: LMC.ExecReg);
185
186 copyOpWithoutRegFlags(MIB&: SelectExec,
187 Op&: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::exec));
188 copyOpWithoutRegFlags(MIB&: SelectExec,
189 Op&: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::fbexec));
190 } else {
191 auto SetExec =
192 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg);
193 copyOpWithoutRegFlags(MIB&: SetExec,
194 Op&: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::exec));
195 }
196
197 for (int OpIdx = MI.getNumExplicitOperands() - 1; OpIdx >= ExecIdx; --OpIdx)
198 MI.removeOperand(OpNo: OpIdx);
199
200 MI.setDesc(TII->get(Opcode: AMDGPU::SI_TCRETURN_CHAIN));
201}
202
203void SILateBranchLowering::earlyTerm(MachineInstr &MI,
204 MachineBasicBlock *EarlyExitBlock) {
205 MachineBasicBlock &MBB = *MI.getParent();
206 const DebugLoc &DL = MI.getDebugLoc();
207
208 auto BranchMI = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC0))
209 .addMBB(MBB: EarlyExitBlock);
210 auto Next = std::next(x: MI.getIterator());
211
212 if (Next != MBB.end() && !Next->isTerminator())
213 splitBlock(MBB, MI&: *BranchMI, MDT, MLI);
214
215 MBB.addSuccessor(Succ: EarlyExitBlock);
216 MDT->insertEdge(From: &MBB, To: EarlyExitBlock);
217}
218
219PreservedAnalyses
220llvm::SILateBranchLoweringPass::run(MachineFunction &MF,
221 MachineFunctionAnalysisManager &MFAM) {
222 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
223 auto *MDT = &MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
224 auto *MLI = MFAM.getCachedResult<MachineLoopAnalysis>(IR&: MF);
225 if (!SILateBranchLowering(ST, MDT, MLI).run(MF))
226 return PreservedAnalyses::all();
227
228 auto PA = getMachineFunctionPassPreservedAnalyses();
229 PA.preserve<MachineDominatorTreeAnalysis>();
230 PA.preserve<MachineLoopAnalysis>();
231 return PA;
232}
233
234bool SILateBranchLowering::run(MachineFunction &MF) {
235 SmallVector<MachineInstr *, 4> EarlyTermInstrs;
236 SmallVector<MachineInstr *, 1> EpilogInstrs;
237 bool MadeChange = false;
238
239 for (MachineBasicBlock &MBB : MF) {
240 for (MachineInstr &MI : llvm::make_early_inc_range(Range&: MBB)) {
241 switch (MI.getOpcode()) {
242 case AMDGPU::S_BRANCH:
243 // Optimize out branches to the next block.
244 // This only occurs in -O0 when BranchFolding is not executed.
245 if (MBB.isLayoutSuccessor(MBB: MI.getOperand(i: 0).getMBB())) {
246 assert(&MI == &MBB.back());
247 MI.eraseFromParent();
248 MadeChange = true;
249 }
250 break;
251
252 case AMDGPU::SI_CS_CHAIN_TC_W32:
253 case AMDGPU::SI_CS_CHAIN_TC_W64:
254 expandChainCall(MI, ST, /*DynamicVGPR=*/false);
255 MadeChange = true;
256 break;
257 case AMDGPU::SI_CS_CHAIN_TC_W32_DVGPR:
258 case AMDGPU::SI_CS_CHAIN_TC_W64_DVGPR:
259 expandChainCall(MI, ST, /*DynamicVGPR=*/true);
260 MadeChange = true;
261 break;
262
263 case AMDGPU::SI_EARLY_TERMINATE_SCC0:
264 EarlyTermInstrs.push_back(Elt: &MI);
265 break;
266
267 case AMDGPU::SI_RETURN_TO_EPILOG:
268 EpilogInstrs.push_back(Elt: &MI);
269 break;
270
271 default:
272 break;
273 }
274 }
275 }
276
277 // Lower any early exit branches first
278 if (!EarlyTermInstrs.empty()) {
279 MachineBasicBlock *EarlyExitBlock = MF.CreateMachineBasicBlock();
280 DebugLoc DL;
281
282 MF.insert(MBBI: MF.end(), MBB: EarlyExitBlock);
283 BuildMI(BB&: *EarlyExitBlock, I: EarlyExitBlock->end(), MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc),
284 DestReg: LMC.ExecReg)
285 .addImm(Val: 0);
286 generateEndPgm(MBB&: *EarlyExitBlock, I: EarlyExitBlock->end(), DL, TII, MF);
287
288 for (MachineInstr *Instr : EarlyTermInstrs) {
289 // Early termination in GS does nothing
290 if (MF.getFunction().getCallingConv() != CallingConv::AMDGPU_GS)
291 earlyTerm(MI&: *Instr, EarlyExitBlock);
292 Instr->eraseFromParent();
293 }
294
295 EarlyTermInstrs.clear();
296 MadeChange = true;
297 }
298
299 // Now check return to epilog instructions occur at function end
300 if (!EpilogInstrs.empty()) {
301 MachineBasicBlock *EmptyMBBAtEnd = nullptr;
302 assert(!MF.getInfo<SIMachineFunctionInfo>()->returnsVoid());
303
304 // If there are multiple returns to epilog then all will
305 // become jumps to new empty end block.
306 if (EpilogInstrs.size() > 1) {
307 EmptyMBBAtEnd = MF.CreateMachineBasicBlock();
308 MF.insert(MBBI: MF.end(), MBB: EmptyMBBAtEnd);
309 }
310
311 for (auto *MI : EpilogInstrs) {
312 auto *MBB = MI->getParent();
313 if (MBB == &MF.back() && MI == &MBB->back())
314 continue;
315
316 // SI_RETURN_TO_EPILOG is not the last instruction.
317 // Jump to empty block at function end.
318 if (!EmptyMBBAtEnd) {
319 EmptyMBBAtEnd = MF.CreateMachineBasicBlock();
320 MF.insert(MBBI: MF.end(), MBB: EmptyMBBAtEnd);
321 }
322
323 MBB->addSuccessor(Succ: EmptyMBBAtEnd);
324 MDT->insertEdge(From: MBB, To: EmptyMBBAtEnd);
325 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_BRANCH))
326 .addMBB(MBB: EmptyMBBAtEnd);
327 MI->eraseFromParent();
328 MadeChange = true;
329 }
330
331 EpilogInstrs.clear();
332 }
333
334 return MadeChange;
335}
336