1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Analysis/SPIRVConvergenceRegionAnalysis.h"
10#include "SPIRV.h"
11#include "SPIRVAsmPrinter.h"
12#include "SPIRVModuleAnalysis.h"
13#include "SPIRVTargetMachine.h"
14#include "llvm/CodeGen/AtomicExpand.h"
15#include "llvm/CodeGen/BranchFoldingPass.h"
16#include "llvm/CodeGen/FuncletLayout.h"
17#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
18#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
19#include "llvm/CodeGen/GlobalISel/Legalizer.h"
20#include "llvm/CodeGen/LiveDebugValuesPass.h"
21#include "llvm/CodeGen/MachineBlockPlacement.h"
22#include "llvm/CodeGen/MachineCopyPropagation.h"
23#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
24#include "llvm/CodeGen/PatchableFunction.h"
25#include "llvm/CodeGen/PostRAMachineSink.h"
26#include "llvm/CodeGen/PostRASchedulerList.h"
27#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
28#include "llvm/CodeGen/ShrinkWrap.h"
29#include "llvm/IR/PassInstrumentation.h"
30#include "llvm/Passes/CodeGenPassBuilder.h"
31#include "llvm/Passes/PassBuilder.h"
32#include "llvm/Target/CGPassBuilderOption.h"
33#include "llvm/Transforms/IPO/ExpandVariadics.h"
34#include "llvm/Transforms/Scalar/InferAddressSpaces.h"
35#include "llvm/Transforms/Scalar/Reg2Mem.h"
36#include "llvm/Transforms/Utils/LoopSimplify.h"
37#include "llvm/Transforms/Utils/Mem2Reg.h"
38#include "llvm/Transforms/Utils/StripConvergenceIntrinsics.h"
39
40using namespace llvm;
41
42namespace {
43
44class SPIRVCodeGenPassBuilder : public CodeGenPassBuilder {
45 using Base = CodeGenPassBuilder;
46
47 SPIRVTargetMachine &getTM() const {
48 return static_cast<SPIRVTargetMachine &>(TM);
49 }
50
51public:
52 explicit SPIRVCodeGenPassBuilder(SPIRVTargetMachine &TM,
53 const CGPassBuilderOption &Opts,
54 PassInstrumentationCallbacks *PIC)
55 : CodeGenPassBuilder(TM, Opts, PIC) {
56 // Disable passes that break from assuming no virtual registers exist.
57 disablePass<MachineCopyPropagationPass, PostRAMachineSinkingPass,
58 PostRASchedulerPass, FuncletLayoutPass, StackMapLivenessPass,
59 PatchableFunctionPass, ShrinkWrapPass, LiveDebugValuesPass,
60 MachineLateInstrsCleanupPass, RemoveLoadsIntoFakeUsesPass,
61 BranchFolderPass, MachineBlockPlacementPass>();
62 // Logical SPIR-V cannot express the pointer induction variables that loop
63 // strength reduction creates.
64 if (TM.getSubtargetImpl()->isLogicalSPIRV())
65 Opt.DisableLSR = true;
66 }
67
68 void addIRPasses(PassManagerWrapper &PMW) override;
69 void addISelPrepare(PassManagerWrapper &PMW) override;
70 Error addIRTranslator(PassManagerWrapper &PMW) override;
71 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
72 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
73 Error addRegBankSelect(PassManagerWrapper &PMW) override;
74 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
75
76 Error addFastRegAlloc(PassManagerWrapper &PMW) override;
77
78 Error addOptimizedRegAlloc(PassManagerWrapper &PMW) override;
79
80 void addPreEmitPass2(PassManagerWrapper &PMW) override;
81
82 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
83 void addAsmPrinter(PassManagerWrapper &PMW) override;
84 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
85};
86
87void SPIRVCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
88 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
89
90 Base::addIRPasses(PMW);
91
92 flushFPMsToMPM(PMW);
93
94 if (getTM().getSubtargetImpl()->isShader()) {
95 if (getOptLevel() != CodeGenOptLevel::None) {
96 addModulePass(Pass: SPIRVFinalizeShaderLinkagePass(getTM()), PMW);
97 }
98 addModulePass(Pass: SPIRVRemoveUnusedResourcesPass(), PMW);
99 } else {
100 // Variadic function calls aren't supported in shader code.
101 // This needs to come before SPIRVPrepareFunctions because this
102 // may introduce intrinsic calls.
103 addModulePass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Lowering), PMW);
104 }
105
106 addFunctionPass(Pass: SPIRVRegularizerPass(), PMW);
107 flushFPMsToMPM(PMW);
108 addModulePass(Pass: SPIRVCtorDtorLoweringPass(), PMW);
109 addModulePass(Pass: SPIRVPrepareFunctionsPass(getTM()), PMW);
110 addModulePass(Pass: SPIRVPrepareGlobalsPass(), PMW);
111}
112
113void SPIRVCodeGenPassBuilder::addISelPrepare(PassManagerWrapper &PMW) {
114 SPIRVTargetMachine &TM = getTM();
115 if (getTM().getSubtargetImpl()->isShader()) {
116 // Vulkan does not allow address space casts. This pass is run to remove
117 // address space casts that can be removed.
118 // If an address space cast is not removed while targeting Vulkan, lowering
119 // will fail during MIR lowering.
120 addFunctionPass(Pass: InferAddressSpacesPass(), PMW);
121
122 // 1. Simplify loop for subsequent transformations. After this steps, loops
123 // have the following properties:
124 // - loops have a single entry edge (pre-header to loop header).
125 // - all loop exits are dominated by the loop pre-header.
126 // - loops have a single back-edge.
127 addFunctionPass(Pass: LoopSimplifyPass(), PMW);
128
129 // 2. Removes registers whose lifetime spans across basic blocks. Also
130 // removes phi nodes. This will greatly simplify the next steps.
131 addFunctionPass(Pass: RegToMemPass(), PMW);
132
133 // 3. Merge the convergence region exit nodes into one. After this step,
134 // regions are single-entry, single-exit. This will help determine the
135 // correct merge block.
136 addFunctionPass(Pass: SPIRVMergeRegionExitTargetsPass(), PMW);
137
138 // 4. Structurize.
139 addFunctionPass(Pass: SPIRVStructurizerPass(), PMW);
140
141 // 5. Reduce the amount of variables required by pushing some operations
142 // back to virtual registers.
143 addFunctionPass(Pass: PromotePass(), PMW);
144 } else {
145 // Canonicalize loops so they have a single latch and preheader.
146 // This enables OpLoopMerge emission for non-shader targets.
147 addFunctionPass(Pass: LoopSimplifyPass(), PMW);
148 }
149 addFunctionPass(Pass: StripConvergenceIntrinsicsPass(), PMW);
150 flushFPMsToMPM(PMW);
151 addModulePass(Pass: SPIRVLegalizeResourceBindingPass(), PMW);
152 addModulePass(Pass: SPIRVLegalizeZeroSizeArraysPass(getTM()), PMW);
153 addModulePass(Pass: SPIRVCBufferAccessPass(), PMW);
154 addModulePass(Pass: SPIRVPushConstantAccessPass(getTM()), PMW);
155 addModulePass(Pass: SPIRVEmitIntrinsicsPass(getTM()), PMW);
156 if (TM.getSubtargetImpl()->isLogicalSPIRV())
157 addFunctionPass(Pass: SPIRVLegalizePointerCastPass(getTM()), PMW);
158 Base::addISelPrepare(PMW);
159}
160
161Error SPIRVCodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
162 addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW);
163 return Error::success();
164}
165
166void SPIRVCodeGenPassBuilder::addPreLegalizeMachineIR(PassManagerWrapper &PMW) {
167 addMachineFunctionPass(Pass: SPIRVPreLegalizerCombinerPass(), PMW);
168 addMachineFunctionPass(Pass: SPIRVPreLegalizerPass(), PMW);
169}
170
171Error SPIRVCodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
172 addMachineFunctionPass(Pass: LegalizerPass(), PMW);
173 addMachineFunctionPass(Pass: SPIRVPostLegalizerPass(), PMW);
174 return Error::success();
175}
176
177Error SPIRVCodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
178 // We do not add RegBankSelectPass as we only ever need virtual registers.
179 return Error::success();
180}
181
182Error SPIRVCodeGenPassBuilder::addGlobalInstructionSelect(
183 PassManagerWrapper &PMW) {
184 addMachineFunctionPass(Pass: InstructionSelectPass(getOptLevel()), PMW);
185 return Error::success();
186}
187
188// We do nothing in register allocation as we keep virtual registers.
189Error SPIRVCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
190 return Error::success();
191}
192
193Error SPIRVCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
194 return Error::success();
195}
196
197void SPIRVCodeGenPassBuilder::addPreEmitPass2(PassManagerWrapper &PMW) {
198 addMachineFunctionPass(Pass: SPIRVPrepareModuleAnalysisPass(), PMW);
199}
200
201void SPIRVCodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
202 addModulePass(Pass: SPIRVAsmPrinterBeginPass(), PMW, /*Force=*/true);
203}
204
205void SPIRVCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
206 // SPIRVModuleAnalysis needs every MachineFunction before printing starts.
207 if (CompletesPipeline) {
208 flushFPMsToMPM(PMW);
209 addModulePass(Pass: RequireAnalysisPass<SPIRVModuleAnalysis, Module>(), PMW,
210 /*Force=*/true);
211 }
212 addMachineFunctionPass(Pass: SPIRVAsmPrinterPass(), PMW);
213}
214
215void SPIRVCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
216 addModulePass(Pass: SPIRVAsmPrinterEndPass(), PMW, /*Force=*/true);
217}
218
219} // namespace
220
221void SPIRVTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB){
222#define GET_PASS_REGISTRY "SPIRVPassRegistry.def"
223#include "llvm/Passes/TargetPassRegistry.inc"
224}
225
226Error SPIRVTargetMachine::buildCodeGenPipeline(
227 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
228 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
229 const CGPassBuilderOption &Opt, MCContext &Ctx,
230 PassInstrumentationCallbacks *PIC) {
231 auto CGPB = SPIRVCodeGenPassBuilder(*this, Opt, PIC);
232 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
233}
234