1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Analysis/SPIRVConvergenceRegionAnalysis.h"
10#include "SPIRV.h"
11#include "SPIRVAsmPrinter.h"
12#include "SPIRVModuleAnalysis.h"
13#include "SPIRVTargetMachine.h"
14#include "llvm/CodeGen/AtomicExpand.h"
15#include "llvm/CodeGen/BranchFoldingPass.h"
16#include "llvm/CodeGen/FuncletLayout.h"
17#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
18#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
19#include "llvm/CodeGen/GlobalISel/Legalizer.h"
20#include "llvm/CodeGen/LiveDebugValuesPass.h"
21#include "llvm/CodeGen/MachineBlockPlacement.h"
22#include "llvm/CodeGen/MachineCopyPropagation.h"
23#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
24#include "llvm/CodeGen/PatchableFunction.h"
25#include "llvm/CodeGen/PostRAMachineSink.h"
26#include "llvm/CodeGen/PostRASchedulerList.h"
27#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
28#include "llvm/CodeGen/ShrinkWrap.h"
29#include "llvm/IR/PassInstrumentation.h"
30#include "llvm/Passes/CodeGenPassBuilder.h"
31#include "llvm/Passes/PassBuilder.h"
32#include "llvm/Target/CGPassBuilderOption.h"
33#include "llvm/Transforms/IPO/ExpandVariadics.h"
34#include "llvm/Transforms/Scalar/InferAddressSpaces.h"
35#include "llvm/Transforms/Scalar/Reg2Mem.h"
36#include "llvm/Transforms/Utils/LoopSimplify.h"
37#include "llvm/Transforms/Utils/Mem2Reg.h"
38#include "llvm/Transforms/Utils/StripConvergenceIntrinsics.h"
39
40using namespace llvm;
41
42namespace {
43
44class SPIRVCodeGenPassBuilder : public CodeGenPassBuilder {
45 using Base = CodeGenPassBuilder;
46
47 SPIRVTargetMachine &getTM() const {
48 return static_cast<SPIRVTargetMachine &>(TM);
49 }
50
51public:
52 explicit SPIRVCodeGenPassBuilder(SPIRVTargetMachine &TM,
53 const CGPassBuilderOption &Opts,
54 PassInstrumentationCallbacks *PIC)
55 : CodeGenPassBuilder(TM, Opts, PIC) {
56 // Disable passes that break from assuming no virtual registers exist.
57 disablePass<MachineCopyPropagationPass, PostRAMachineSinkingPass,
58 PostRASchedulerPass, FuncletLayoutPass, StackMapLivenessPass,
59 PatchableFunctionPass, ShrinkWrapPass, LiveDebugValuesPass,
60 MachineLateInstrsCleanupPass, RemoveLoadsIntoFakeUsesPass,
61 BranchFolderPass, MachineBlockPlacementPass>();
62 }
63
64 void addIRPasses(PassManagerWrapper &PMW) override;
65 void addISelPrepare(PassManagerWrapper &PMW) override;
66 Error addIRTranslator(PassManagerWrapper &PMW) override;
67 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
68 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
69 Error addRegBankSelect(PassManagerWrapper &PMW) override;
70 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
71
72 Error addFastRegAlloc(PassManagerWrapper &PMW) override;
73
74 Error addOptimizedRegAlloc(PassManagerWrapper &PMW) override;
75
76 void addPreEmitPass2(PassManagerWrapper &PMW) override;
77
78 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
79 void addAsmPrinter(PassManagerWrapper &PMW) override;
80 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
81};
82
83void SPIRVCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
84 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
85
86 Base::addIRPasses(PMW);
87
88 flushFPMsToMPM(PMW);
89
90 if (getTM().getSubtargetImpl()->isShader()) {
91 if (getOptLevel() != CodeGenOptLevel::None) {
92 addModulePass(Pass: SPIRVFinalizeShaderLinkagePass(getTM()), PMW);
93 }
94 } else {
95 // Variadic function calls aren't supported in shader code.
96 // This needs to come before SPIRVPrepareFunctions because this
97 // may introduce intrinsic calls.
98 addModulePass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Lowering), PMW);
99 }
100
101 addFunctionPass(Pass: SPIRVRegularizerPass(), PMW);
102 flushFPMsToMPM(PMW);
103 addModulePass(Pass: SPIRVCtorDtorLoweringPass(), PMW);
104 addModulePass(Pass: SPIRVPrepareFunctionsPass(getTM()), PMW);
105 addModulePass(Pass: SPIRVPrepareGlobalsPass(), PMW);
106}
107
108void SPIRVCodeGenPassBuilder::addISelPrepare(PassManagerWrapper &PMW) {
109 SPIRVTargetMachine &TM = getTM();
110 if (getTM().getSubtargetImpl()->isShader()) {
111 // Vulkan does not allow address space casts. This pass is run to remove
112 // address space casts that can be removed.
113 // If an address space cast is not removed while targeting Vulkan, lowering
114 // will fail during MIR lowering.
115 addFunctionPass(Pass: InferAddressSpacesPass(), PMW);
116
117 // 1. Simplify loop for subsequent transformations. After this steps, loops
118 // have the following properties:
119 // - loops have a single entry edge (pre-header to loop header).
120 // - all loop exits are dominated by the loop pre-header.
121 // - loops have a single back-edge.
122 addFunctionPass(Pass: LoopSimplifyPass(), PMW);
123
124 // 2. Removes registers whose lifetime spans across basic blocks. Also
125 // removes phi nodes. This will greatly simplify the next steps.
126 addFunctionPass(Pass: RegToMemPass(), PMW);
127
128 // 3. Merge the convergence region exit nodes into one. After this step,
129 // regions are single-entry, single-exit. This will help determine the
130 // correct merge block.
131 addFunctionPass(Pass: SPIRVMergeRegionExitTargetsPass(), PMW);
132
133 // 4. Structurize.
134 addFunctionPass(Pass: SPIRVStructurizerPass(), PMW);
135
136 // 5. Reduce the amount of variables required by pushing some operations
137 // back to virtual registers.
138 addFunctionPass(Pass: PromotePass(), PMW);
139 } else {
140 // Canonicalize loops so they have a single latch and preheader.
141 // This enables OpLoopMerge emission for non-shader targets.
142 addFunctionPass(Pass: LoopSimplifyPass(), PMW);
143 }
144 addFunctionPass(Pass: StripConvergenceIntrinsicsPass(), PMW);
145 flushFPMsToMPM(PMW);
146 addModulePass(Pass: SPIRVLegalizeResourceBindingPass(), PMW);
147 addModulePass(Pass: SPIRVLegalizeZeroSizeArraysPass(getTM()), PMW);
148 addModulePass(Pass: SPIRVCBufferAccessPass(), PMW);
149 addModulePass(Pass: SPIRVPushConstantAccessPass(getTM()), PMW);
150 addModulePass(Pass: SPIRVEmitIntrinsicsPass(getTM()), PMW);
151 if (TM.getSubtargetImpl()->isLogicalSPIRV())
152 addFunctionPass(Pass: SPIRVLegalizePointerCastPass(getTM()), PMW);
153 Base::addISelPrepare(PMW);
154}
155
156Error SPIRVCodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
157 addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW);
158 return Error::success();
159}
160
161void SPIRVCodeGenPassBuilder::addPreLegalizeMachineIR(PassManagerWrapper &PMW) {
162 addMachineFunctionPass(Pass: SPIRVPreLegalizerCombinerPass(), PMW);
163 addMachineFunctionPass(Pass: SPIRVPreLegalizerPass(), PMW);
164}
165
166Error SPIRVCodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
167 addMachineFunctionPass(Pass: LegalizerPass(), PMW);
168 addMachineFunctionPass(Pass: SPIRVPostLegalizerPass(), PMW);
169 return Error::success();
170}
171
172Error SPIRVCodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
173 // We do not add RegBankSelectPass as we only ever need virtual registers.
174 return Error::success();
175}
176
177Error SPIRVCodeGenPassBuilder::addGlobalInstructionSelect(
178 PassManagerWrapper &PMW) {
179 addMachineFunctionPass(
180 Pass: InstructionSelectPass(getOptLevel(), /*RequireRegBankSelection=*/false),
181 PMW);
182 return Error::success();
183}
184
185// We do nothing in register allocation as we keep virtual registers.
186Error SPIRVCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
187 return Error::success();
188}
189
190Error SPIRVCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
191 return Error::success();
192}
193
194void SPIRVCodeGenPassBuilder::addPreEmitPass2(PassManagerWrapper &PMW) {
195 addMachineFunctionPass(Pass: SPIRVPrepareModuleAnalysisPass(), PMW);
196}
197
198void SPIRVCodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
199 addModulePass(Pass: SPIRVAsmPrinterBeginPass(), PMW);
200}
201
202void SPIRVCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
203 // SPIRVModuleAnalysis needs every MachineFunction before printing starts.
204 if (CompletesPipeline) {
205 flushFPMsToMPM(PMW);
206 addModulePass(Pass: RequireAnalysisPass<SPIRVModuleAnalysis, Module>(), PMW,
207 /*Force=*/true);
208 }
209 addMachineFunctionPass(Pass: SPIRVAsmPrinterPass(), PMW);
210}
211
212void SPIRVCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
213 addModulePass(Pass: SPIRVAsmPrinterEndPass(), PMW);
214}
215
216} // namespace
217
218void SPIRVTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB){
219#define GET_PASS_REGISTRY "SPIRVPassRegistry.def"
220#include "llvm/Passes/TargetPassRegistry.inc"
221}
222
223Error SPIRVTargetMachine::buildCodeGenPipeline(
224 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
225 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
226 const CGPassBuilderOption &Opt, MCContext &Ctx,
227 PassInstrumentationCallbacks *PIC) {
228 auto CGPB = SPIRVCodeGenPassBuilder(*this, Opt, PIC);
229 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
230}