1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Analysis/SPIRVConvergenceRegionAnalysis.h"
10#include "SPIRV.h"
11#include "SPIRVAsmPrinter.h"
12#include "SPIRVModuleAnalysis.h"
13#include "SPIRVTargetMachine.h"
14#include "llvm/CodeGen/AtomicExpand.h"
15#include "llvm/CodeGen/BranchFoldingPass.h"
16#include "llvm/CodeGen/FuncletLayout.h"
17#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
18#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
19#include "llvm/CodeGen/GlobalISel/Legalizer.h"
20#include "llvm/CodeGen/LiveDebugValuesPass.h"
21#include "llvm/CodeGen/MachineBlockPlacement.h"
22#include "llvm/CodeGen/MachineCopyPropagation.h"
23#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
24#include "llvm/CodeGen/PatchableFunction.h"
25#include "llvm/CodeGen/PostRAMachineSink.h"
26#include "llvm/CodeGen/PostRASchedulerList.h"
27#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
28#include "llvm/CodeGen/ShrinkWrap.h"
29#include "llvm/IR/PassInstrumentation.h"
30#include "llvm/Passes/CodeGenPassBuilder.h"
31#include "llvm/Passes/PassBuilder.h"
32#include "llvm/Target/CGPassBuilderOption.h"
33#include "llvm/Transforms/IPO/ExpandVariadics.h"
34#include "llvm/Transforms/Scalar/InferAddressSpaces.h"
35#include "llvm/Transforms/Scalar/Reg2Mem.h"
36#include "llvm/Transforms/Utils/LoopSimplify.h"
37#include "llvm/Transforms/Utils/Mem2Reg.h"
38#include "llvm/Transforms/Utils/StripConvergenceIntrinsics.h"
39
40using namespace llvm;
41
42namespace {
43
44class SPIRVCodeGenPassBuilder : public CodeGenPassBuilder {
45 using Base = CodeGenPassBuilder;
46
47 SPIRVTargetMachine &getTM() const {
48 return static_cast<SPIRVTargetMachine &>(TM);
49 }
50
51public:
52 explicit SPIRVCodeGenPassBuilder(SPIRVTargetMachine &TM,
53 const CGPassBuilderOption &Opts,
54 PassInstrumentationCallbacks *PIC)
55 : CodeGenPassBuilder(TM, Opts, PIC) {
56 // Disable passes that break from assuming no virtual registers exist.
57 disablePass<MachineCopyPropagationPass, PostRAMachineSinkingPass,
58 PostRASchedulerPass, FuncletLayoutPass, StackMapLivenessPass,
59 PatchableFunctionPass, ShrinkWrapPass, LiveDebugValuesPass,
60 MachineLateInstrsCleanupPass, RemoveLoadsIntoFakeUsesPass,
61 BranchFolderPass, MachineBlockPlacementPass>();
62 }
63
64 void addIRPasses(PassManagerWrapper &PMW) override;
65 void addISelPrepare(PassManagerWrapper &PMW) override;
66 Error addIRTranslator(PassManagerWrapper &PMW) override;
67 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
68 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
69 Error addRegBankSelect(PassManagerWrapper &PMW) override;
70 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
71
72 Error addFastRegAlloc(PassManagerWrapper &PMW) override;
73
74 Error addOptimizedRegAlloc(PassManagerWrapper &PMW) override;
75
76 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
77 void addAsmPrinter(PassManagerWrapper &PMW) override;
78 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
79};
80
81void SPIRVCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
82 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
83
84 Base::addIRPasses(PMW);
85
86 flushFPMsToMPM(PMW);
87
88 if (getTM().getSubtargetImpl()->isShader()) {
89 if (getOptLevel() != CodeGenOptLevel::None) {
90 addModulePass(Pass: SPIRVFinalizeShaderLinkagePass(getTM()), PMW);
91 }
92 } else {
93 // Variadic function calls aren't supported in shader code.
94 // This needs to come before SPIRVPrepareFunctions because this
95 // may introduce intrinsic calls.
96 addModulePass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Lowering), PMW);
97 }
98
99 addFunctionPass(Pass: SPIRVRegularizerPass(), PMW);
100 flushFPMsToMPM(PMW);
101 addModulePass(Pass: SPIRVCtorDtorLoweringPass(), PMW);
102 addModulePass(Pass: SPIRVPrepareFunctionsPass(getTM()), PMW);
103 addModulePass(Pass: SPIRVPrepareGlobalsPass(), PMW);
104}
105
106void SPIRVCodeGenPassBuilder::addISelPrepare(PassManagerWrapper &PMW) {
107 SPIRVTargetMachine &TM = getTM();
108 if (getTM().getSubtargetImpl()->isShader()) {
109 // Vulkan does not allow address space casts. This pass is run to remove
110 // address space casts that can be removed.
111 // If an address space cast is not removed while targeting Vulkan, lowering
112 // will fail during MIR lowering.
113 addFunctionPass(Pass: InferAddressSpacesPass(), PMW);
114
115 // 1. Simplify loop for subsequent transformations. After this steps, loops
116 // have the following properties:
117 // - loops have a single entry edge (pre-header to loop header).
118 // - all loop exits are dominated by the loop pre-header.
119 // - loops have a single back-edge.
120 addFunctionPass(Pass: LoopSimplifyPass(), PMW);
121
122 // 2. Removes registers whose lifetime spans across basic blocks. Also
123 // removes phi nodes. This will greatly simplify the next steps.
124 addFunctionPass(Pass: RegToMemPass(), PMW);
125
126 // 3. Merge the convergence region exit nodes into one. After this step,
127 // regions are single-entry, single-exit. This will help determine the
128 // correct merge block.
129 addFunctionPass(Pass: SPIRVMergeRegionExitTargetsPass(), PMW);
130
131 // 4. Structurize.
132 addFunctionPass(Pass: SPIRVStructurizerPass(), PMW);
133
134 // 5. Reduce the amount of variables required by pushing some operations
135 // back to virtual registers.
136 addFunctionPass(Pass: PromotePass(), PMW);
137 } else {
138 // Canonicalize loops so they have a single latch and preheader.
139 // This enables OpLoopMerge emission for non-shader targets.
140 addFunctionPass(Pass: LoopSimplifyPass(), PMW);
141 }
142 addFunctionPass(Pass: StripConvergenceIntrinsicsPass(), PMW);
143 flushFPMsToMPM(PMW);
144 addModulePass(Pass: SPIRVLegalizeResourceBindingPass(), PMW);
145 addModulePass(Pass: SPIRVLegalizeZeroSizeArraysPass(getTM()), PMW);
146 addModulePass(Pass: SPIRVCBufferAccessPass(), PMW);
147 addModulePass(Pass: SPIRVPushConstantAccessPass(getTM()), PMW);
148 addModulePass(Pass: SPIRVEmitIntrinsicsPass(getTM()), PMW);
149 if (TM.getSubtargetImpl()->isLogicalSPIRV())
150 addFunctionPass(Pass: SPIRVLegalizePointerCastPass(getTM()), PMW);
151 Base::addISelPrepare(PMW);
152}
153
154Error SPIRVCodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
155 addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW);
156 return Error::success();
157}
158
159void SPIRVCodeGenPassBuilder::addPreLegalizeMachineIR(PassManagerWrapper &PMW) {
160 addMachineFunctionPass(Pass: SPIRVPreLegalizerCombinerPass(), PMW);
161 addMachineFunctionPass(Pass: SPIRVPreLegalizerPass(), PMW);
162}
163
164Error SPIRVCodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
165 addMachineFunctionPass(Pass: LegalizerPass(), PMW);
166 addMachineFunctionPass(Pass: SPIRVPostLegalizerPass(), PMW);
167 return Error::success();
168}
169
170Error SPIRVCodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
171 // We do not add RegBankSelectPass as we only ever need virtual registers.
172 return Error::success();
173}
174
175Error SPIRVCodeGenPassBuilder::addGlobalInstructionSelect(
176 PassManagerWrapper &PMW) {
177 addMachineFunctionPass(
178 Pass: InstructionSelectPass(getOptLevel(), /*RequireRegBankSelection=*/false),
179 PMW);
180 return Error::success();
181}
182
183// We do nothing in register allocation as we keep virtual registers.
184Error SPIRVCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
185 return Error::success();
186}
187
188Error SPIRVCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
189 return Error::success();
190}
191
192void SPIRVCodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
193 addModulePass(Pass: SPIRVAsmPrinterBeginPass(), PMW);
194}
195
196void SPIRVCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
197 // SPIRVModuleAnalysis needs every MachineFunction before printing starts.
198 if (TargetPassConfig::willCompleteCodeGenPipeline()) {
199 flushFPMsToMPM(PMW);
200 addModulePass(Pass: RequireAnalysisPass<SPIRVModuleAnalysis, Module>(), PMW,
201 /*Force=*/true);
202 }
203 addMachineFunctionPass(Pass: SPIRVAsmPrinterPass(), PMW);
204}
205
206void SPIRVCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
207 addModulePass(Pass: SPIRVAsmPrinterEndPass(), PMW);
208}
209
210} // namespace
211
212void SPIRVTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB){
213#define GET_PASS_REGISTRY "SPIRVPassRegistry.def"
214#include "llvm/Passes/TargetPassRegistry.inc"
215}
216
217Error SPIRVTargetMachine::buildCodeGenPipeline(
218 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
219 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
220 const CGPassBuilderOption &Opt, MCContext &Ctx,
221 PassInstrumentationCallbacks *PIC) {
222 auto CGPB = SPIRVCodeGenPassBuilder(*this, Opt, PIC);
223 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
224}