1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Analysis/SPIRVConvergenceRegionAnalysis.h"
10#include "SPIRV.h"
11#include "SPIRVAsmPrinter.h"
12#include "SPIRVTargetMachine.h"
13#include "llvm/CodeGen/AtomicExpand.h"
14#include "llvm/CodeGen/BranchFoldingPass.h"
15#include "llvm/CodeGen/FuncletLayout.h"
16#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
17#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
18#include "llvm/CodeGen/GlobalISel/Legalizer.h"
19#include "llvm/CodeGen/LiveDebugValuesPass.h"
20#include "llvm/CodeGen/MachineBlockPlacement.h"
21#include "llvm/CodeGen/MachineCopyPropagation.h"
22#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
23#include "llvm/CodeGen/PatchableFunction.h"
24#include "llvm/CodeGen/PostRAMachineSink.h"
25#include "llvm/CodeGen/PostRASchedulerList.h"
26#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
27#include "llvm/CodeGen/ShrinkWrap.h"
28#include "llvm/IR/PassInstrumentation.h"
29#include "llvm/Passes/CodeGenPassBuilder.h"
30#include "llvm/Passes/PassBuilder.h"
31#include "llvm/Target/CGPassBuilderOption.h"
32#include "llvm/Transforms/IPO/ExpandVariadics.h"
33#include "llvm/Transforms/Scalar/InferAddressSpaces.h"
34#include "llvm/Transforms/Scalar/Reg2Mem.h"
35#include "llvm/Transforms/Utils/LoopSimplify.h"
36#include "llvm/Transforms/Utils/Mem2Reg.h"
37#include "llvm/Transforms/Utils/StripConvergenceIntrinsics.h"
38
39using namespace llvm;
40
41namespace {
42
43class SPIRVCodeGenPassBuilder : public CodeGenPassBuilder {
44 using Base = CodeGenPassBuilder;
45
46 SPIRVTargetMachine &getTM() const {
47 return static_cast<SPIRVTargetMachine &>(TM);
48 }
49
50public:
51 explicit SPIRVCodeGenPassBuilder(SPIRVTargetMachine &TM,
52 const CGPassBuilderOption &Opts,
53 PassInstrumentationCallbacks *PIC)
54 : CodeGenPassBuilder(TM, Opts, PIC) {
55 // Disable passes that break from assuming no virtual registers exist.
56 disablePass<MachineCopyPropagationPass, PostRAMachineSinkingPass,
57 PostRASchedulerPass, FuncletLayoutPass, StackMapLivenessPass,
58 PatchableFunctionPass, ShrinkWrapPass, LiveDebugValuesPass,
59 MachineLateInstrsCleanupPass, RemoveLoadsIntoFakeUsesPass,
60 BranchFolderPass, MachineBlockPlacementPass>();
61 }
62
63 void addIRPasses(PassManagerWrapper &PMW) override;
64 void addISelPrepare(PassManagerWrapper &PMW) override;
65 Error addIRTranslator(PassManagerWrapper &PMW) override;
66 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
67 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
68 Error addRegBankSelect(PassManagerWrapper &PMW) override;
69 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
70
71 Error addFastRegAlloc(PassManagerWrapper &PMW) override;
72
73 Error addOptimizedRegAlloc(PassManagerWrapper &PMW) override;
74
75 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
76 void addAsmPrinter(PassManagerWrapper &PMW) override;
77 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
78};
79
80void SPIRVCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
81 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
82
83 Base::addIRPasses(PMW);
84
85 flushFPMsToMPM(PMW);
86
87 if (getTM().getSubtargetImpl()->isShader()) {
88 if (getOptLevel() != CodeGenOptLevel::None) {
89 addModulePass(Pass: SPIRVFinalizeShaderLinkagePass(getTM()), PMW);
90 }
91 } else {
92 // Variadic function calls aren't supported in shader code.
93 // This needs to come before SPIRVPrepareFunctions because this
94 // may introduce intrinsic calls.
95 addModulePass(Pass: ExpandVariadicsPass(ExpandVariadicsMode::Lowering), PMW);
96 }
97
98 addFunctionPass(Pass: SPIRVRegularizerPass(), PMW);
99 flushFPMsToMPM(PMW);
100 addModulePass(Pass: SPIRVCtorDtorLoweringPass(), PMW);
101 addModulePass(Pass: SPIRVPrepareFunctionsPass(getTM()), PMW);
102 addModulePass(Pass: SPIRVPrepareGlobalsPass(), PMW);
103}
104
105void SPIRVCodeGenPassBuilder::addISelPrepare(PassManagerWrapper &PMW) {
106 SPIRVTargetMachine &TM = getTM();
107 if (getTM().getSubtargetImpl()->isShader()) {
108 // Vulkan does not allow address space casts. This pass is run to remove
109 // address space casts that can be removed.
110 // If an address space cast is not removed while targeting Vulkan, lowering
111 // will fail during MIR lowering.
112 addFunctionPass(Pass: InferAddressSpacesPass(), PMW);
113
114 // 1. Simplify loop for subsequent transformations. After this steps, loops
115 // have the following properties:
116 // - loops have a single entry edge (pre-header to loop header).
117 // - all loop exits are dominated by the loop pre-header.
118 // - loops have a single back-edge.
119 addFunctionPass(Pass: LoopSimplifyPass(), PMW);
120
121 // 2. Removes registers whose lifetime spans across basic blocks. Also
122 // removes phi nodes. This will greatly simplify the next steps.
123 addFunctionPass(Pass: RegToMemPass(), PMW);
124
125 // 3. Merge the convergence region exit nodes into one. After this step,
126 // regions are single-entry, single-exit. This will help determine the
127 // correct merge block.
128 addFunctionPass(Pass: SPIRVMergeRegionExitTargetsPass(), PMW);
129
130 // 4. Structurize.
131 addFunctionPass(Pass: SPIRVStructurizerPass(), PMW);
132
133 // 5. Reduce the amount of variables required by pushing some operations
134 // back to virtual registers.
135 addFunctionPass(Pass: PromotePass(), PMW);
136 } else {
137 // Canonicalize loops so they have a single latch and preheader.
138 // This enables OpLoopMerge emission for non-shader targets.
139 addFunctionPass(Pass: LoopSimplifyPass(), PMW);
140 }
141 addFunctionPass(Pass: StripConvergenceIntrinsicsPass(), PMW);
142 flushFPMsToMPM(PMW);
143 addModulePass(Pass: SPIRVLegalizeImplicitBindingPass(), PMW);
144 addModulePass(Pass: SPIRVLegalizeZeroSizeArraysPass(getTM()), PMW);
145 addModulePass(Pass: SPIRVCBufferAccessPass(), PMW);
146 addModulePass(Pass: SPIRVPushConstantAccessPass(getTM()), PMW);
147 addModulePass(Pass: SPIRVEmitIntrinsicsPass(getTM()), PMW);
148 if (TM.getSubtargetImpl()->isLogicalSPIRV())
149 addFunctionPass(Pass: SPIRVLegalizePointerCastPass(getTM()), PMW);
150 Base::addISelPrepare(PMW);
151}
152
153Error SPIRVCodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
154 addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW);
155 return Error::success();
156}
157
158void SPIRVCodeGenPassBuilder::addPreLegalizeMachineIR(PassManagerWrapper &PMW) {
159 addMachineFunctionPass(Pass: SPIRVPreLegalizerCombinerPass(), PMW);
160 addMachineFunctionPass(Pass: SPIRVPreLegalizerPass(), PMW);
161}
162
163Error SPIRVCodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
164 addMachineFunctionPass(Pass: LegalizerPass(), PMW);
165 addMachineFunctionPass(Pass: SPIRVPostLegalizerPass(), PMW);
166 return Error::success();
167}
168
169Error SPIRVCodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
170 // We do not add RegBankSelectPass as we only ever need virtual registers.
171 return Error::success();
172}
173
174Error SPIRVCodeGenPassBuilder::addGlobalInstructionSelect(
175 PassManagerWrapper &PMW) {
176 addMachineFunctionPass(Pass: InstructionSelectPass(getOptLevel()), PMW);
177 return Error::success();
178}
179
180// We do nothing in register allocation as we keep virtual registers.
181Error SPIRVCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
182 return Error::success();
183}
184
185Error SPIRVCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
186 return Error::success();
187}
188
189void SPIRVCodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
190 addModulePass(Pass: SPIRVAsmPrinterBeginPass(), PMW);
191}
192
193void SPIRVCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
194 addMachineFunctionPass(Pass: SPIRVAsmPrinterPass(), PMW);
195}
196
197void SPIRVCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
198 addModulePass(Pass: SPIRVAsmPrinterEndPass(), PMW);
199}
200
201} // namespace
202
203void SPIRVTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB){
204#define GET_PASS_REGISTRY "SPIRVPassRegistry.def"
205#include "llvm/Passes/TargetPassRegistry.inc"
206}
207
208Error SPIRVTargetMachine::buildCodeGenPipeline(
209 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
210 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
211 const CGPassBuilderOption &Opt, MCContext &Ctx,
212 PassInstrumentationCallbacks *PIC) {
213 auto CGPB = SPIRVCodeGenPassBuilder(*this, Opt, PIC);
214 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
215}