1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file contains the RISC-V CodeGen pipeline builder.
10//===----------------------------------------------------------------------===//
11
12#include "RISCV.h"
13#include "RISCVAsmPrinter.h"
14#include "RISCVTargetMachine.h"
15#include "llvm/CodeGen/AtomicExpand.h"
16#include "llvm/CodeGen/BranchRelaxation.h"
17#include "llvm/CodeGen/CFIInstrInserter.h"
18#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
19#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
20#include "llvm/CodeGen/GlobalISel/Legalizer.h"
21#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
22#include "llvm/CodeGen/GlobalMerge.h"
23#include "llvm/CodeGen/InitUndef.h"
24#include "llvm/CodeGen/InterleavedAccess.h"
25#include "llvm/CodeGen/KCFI.h"
26#include "llvm/CodeGen/MachineCombiner.h"
27#include "llvm/CodeGen/MachineCopyPropagation.h"
28#include "llvm/CodeGen/MachineInstrBundle.h"
29#include "llvm/CodeGen/MachineLICM.h"
30#include "llvm/CodeGen/RegAllocFast.h"
31#include "llvm/CodeGen/RegAllocGreedyPass.h"
32#include "llvm/CodeGen/SelectOptimize.h"
33#include "llvm/CodeGen/TypePromotion.h"
34#include "llvm/CodeGen/VirtRegMap.h"
35#include "llvm/IR/PassInstrumentation.h"
36#include "llvm/MC/MCStreamer.h"
37#include "llvm/Passes/CodeGenPassBuilder.h"
38#include "llvm/Passes/PassBuilder.h"
39#include "llvm/Support/CodeGen.h"
40#include "llvm/Target/CGPassBuilderOption.h"
41#include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
42#include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h"
43
44using namespace llvm;
45
46namespace {
47
48class RISCVCodeGenPassBuilder : public CodeGenPassBuilder {
49 using Base = CodeGenPassBuilder;
50
51 RISCVTargetMachine &getTM() const {
52 return static_cast<RISCVTargetMachine &>(TM);
53 }
54
55public:
56 explicit RISCVCodeGenPassBuilder(RISCVTargetMachine &TM,
57 const CGPassBuilderOption &Opts,
58 PassInstrumentationCallbacks *PIC)
59 : CodeGenPassBuilder(TM, Opts, PIC) {
60 // TODO: See the FIXME on RISCVPassConfig::setEnableSinkAndFold in the
61 // legacy pass manager pipeline. There is currently no way to plumb
62 // -riscv-enable-sink-fold through to CGPassBuilderOption::EnableSinkAndFold
63 // from a target-local CodeGenPassBuilder, so this NewPM pipeline always
64 // uses the base class default (disabled).
65 }
66
67 void addIRPasses(PassManagerWrapper &PMW) override;
68 void addPreISel(PassManagerWrapper &PMW) override;
69 void addCodeGenPrepare(PassManagerWrapper &PMW) override;
70
71 Error addInstSelector(PassManagerWrapper &PMW) override;
72
73 Error addIRTranslator(PassManagerWrapper &PMW) override;
74 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
75 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
76 void addPreRegBankSelect(PassManagerWrapper &PMW) override;
77 Error addRegBankSelect(PassManagerWrapper &PMW) override;
78 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
79
80 void addMachineSSAOptimization(PassManagerWrapper &PMW) override;
81 void addILPOpts(PassManagerWrapper &PMW) override;
82 void addPreRegAlloc(PassManagerWrapper &PMW) override;
83 Error addFastRegAlloc(PassManagerWrapper &PMW) override;
84 Error addRegAssignAndRewriteFast(PassManagerWrapper &PMW) override;
85 Expected<bool>
86 addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) override;
87 void addPostRegAlloc(PassManagerWrapper &PMW) override;
88 void addPreSched2(PassManagerWrapper &PMW) override;
89 void addPreEmitPass(PassManagerWrapper &PMW) override;
90 void addPreEmitPass2(PassManagerWrapper &PMW) override;
91 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
92 void addAsmPrinter(PassManagerWrapper &PMW) override;
93 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
94};
95
96void RISCVCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
97 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
98 addFunctionPass(Pass: RISCVZacasABIFixPass(&getTM()), PMW);
99
100 if (getOptLevel() != CodeGenOptLevel::None) {
101 if (getTM().getCLOpts().enable_loop_data_prefetch)
102 addFunctionPass(Pass: LoopDataPrefetchPass(), PMW);
103
104 addFunctionPass(Pass: RISCVGatherScatterLoweringPass(&getTM()), PMW);
105 addFunctionPass(Pass: InterleavedAccessPass(TM), PMW);
106 addFunctionPass(Pass: RISCVCodeGenPreparePass(&getTM()), PMW);
107 }
108
109 Base::addIRPasses(PMW);
110
111 if (getOptLevel() == CodeGenOptLevel::Aggressive &&
112 getTM().getCLOpts().select_opt)
113 addFunctionPass(Pass: SelectOptimizePass(getTM()), PMW);
114}
115
116void RISCVCodeGenPassBuilder::addPreISel(PassManagerWrapper &PMW) {
117 if (getOptLevel() != CodeGenOptLevel::None) {
118 flushFPMsToMPM(PMW);
119 addModulePass(Pass: RISCVPromoteConstantPass(&getTM()), PMW);
120 }
121
122 if (valueOr(X: getTM().getCLOpts().enable_global_merge,
123 Default: getOptLevel() != CodeGenOptLevel::None)) {
124 // FIXME: Like AArch64, we disable extern global merging by default due to
125 // concerns it might regress some workloads. Unlike AArch64, we don't
126 // currently support enabling the pass in an "OnlyOptimizeForSize" mode.
127 // Investigating and addressing both items are TODO.
128 flushFPMsToMPM(PMW);
129 GlobalMergeOptions GMOptions;
130 GMOptions.MaxOffset = 2047;
131 GMOptions.MergeExternal = true;
132 GMOptions.SizeOnly = false;
133 addModulePass(Pass: GlobalMergePass(&getTM(), GMOptions), PMW);
134 }
135}
136
137void RISCVCodeGenPassBuilder::addCodeGenPrepare(PassManagerWrapper &PMW) {
138 if (getOptLevel() != CodeGenOptLevel::None)
139 addFunctionPass(Pass: TypePromotionPass(TM), PMW);
140 Base::addCodeGenPrepare(PMW);
141}
142
143Error RISCVCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
144 addMachineFunctionPass(Pass: RISCVISelDAGToDAGPass(getTM(), getOptLevel()), PMW);
145 return Error::success();
146}
147
148Error RISCVCodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
149 addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW);
150 return Error::success();
151}
152
153void RISCVCodeGenPassBuilder::addPreLegalizeMachineIR(PassManagerWrapper &PMW) {
154 if (getOptLevel() == CodeGenOptLevel::None)
155 addMachineFunctionPass(Pass: RISCVO0PreLegalizerCombinerPass(), PMW);
156 else
157 addMachineFunctionPass(Pass: RISCVPreLegalizerCombinerPass(), PMW);
158}
159
160Error RISCVCodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
161 addMachineFunctionPass(Pass: LegalizerPass(), PMW);
162 return Error::success();
163}
164
165void RISCVCodeGenPassBuilder::addPreRegBankSelect(PassManagerWrapper &PMW) {
166 if (getOptLevel() != CodeGenOptLevel::None)
167 addMachineFunctionPass(Pass: RISCVPostLegalizerCombinerPass(), PMW);
168}
169
170Error RISCVCodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
171 addMachineFunctionPass(Pass: RegBankSelectPass(), PMW);
172 return Error::success();
173}
174
175Error RISCVCodeGenPassBuilder::addGlobalInstructionSelect(
176 PassManagerWrapper &PMW) {
177 addMachineFunctionPass(Pass: InstructionSelectPass(getOptLevel()), PMW);
178 return Error::success();
179}
180
181void RISCVCodeGenPassBuilder::addMachineSSAOptimization(
182 PassManagerWrapper &PMW) {
183 // It's beneficial to reduce the VL to enable more
184 // Machine SSA optimizations.
185 if (getOptLevel() != CodeGenOptLevel::None) {
186 // RISCVVLOptimizer can make loop invariant instructions like vmv.v.i
187 // loop variant by propagating a VL defined inside the loop. Run LICM and
188 // hoist them early. Don't do this at -O0 to avoid the compile-time
189 // overhead. Not reducing the VL of loop invariant pseudos results in more
190 // vsetvli toggles, and still requires the MachineLoopInfo analysis to be
191 // run.
192 addMachineFunctionPass(Pass: EarlyMachineLICMPass(), PMW);
193 addMachineFunctionPass(Pass: RISCVVLOptimizerPass(), PMW);
194 }
195
196 addMachineFunctionPass(Pass: RISCVVectorPeepholePass(), PMW);
197 addMachineFunctionPass(Pass: RISCVFoldMemOffsetPass(), PMW);
198
199 Base::addMachineSSAOptimization(PMW);
200
201 if (TM.getTargetTriple().isRISCV64())
202 addMachineFunctionPass(Pass: RISCVOptWInstrsPass(), PMW);
203}
204
205void RISCVCodeGenPassBuilder::addILPOpts(PassManagerWrapper &PMW) {
206 if (getTM().getCLOpts().enable_machine_combiner)
207 addMachineFunctionPass(Pass: MachineCombinerPass(), PMW);
208}
209
210void RISCVCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
211 addMachineFunctionPass(Pass: RISCVExpandPseudoPreRAPass(), PMW);
212 if (getOptLevel() != CodeGenOptLevel::None) {
213 addMachineFunctionPass(Pass: RISCVMergeBaseOffsetPass(), PMW);
214 // Add Zilsd pre-allocation load/store optimization
215 addMachineFunctionPass(Pass: RISCVPreAllocZilsdOptPass(), PMW);
216 }
217
218 addMachineFunctionPass(Pass: RISCVInsertReadWriteCSRPass(), PMW);
219 addMachineFunctionPass(Pass: RISCVInsertWriteVXRMPass(), PMW);
220 addMachineFunctionPass(Pass: RISCVLandingPadSetupPass(), PMW);
221 addMachineFunctionPass(Pass: RISCVVMV0EliminationPass(), PMW);
222}
223
224Error RISCVCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
225 addMachineFunctionPass(Pass: InitUndefPass(), PMW);
226 return Base::addFastRegAlloc(PMW);
227}
228
229static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI,
230 const MachineRegisterInfo &MRI,
231 const Register Reg) {
232 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
233 return RISCVRegisterInfo::isRVVRegClass(RC);
234}
235
236Error RISCVCodeGenPassBuilder::addRegAssignAndRewriteFast(
237 PassManagerWrapper &PMW) {
238 addMachineFunctionPass(
239 Pass: RegAllocFastPass({onlyAllocateRVVReg, "rvv", /*ClearVRegs=*/false}), PMW);
240 addMachineFunctionPass(Pass: RISCVInsertVSETVLIPass(), PMW);
241 if (getOptLevel() != CodeGenOptLevel::None &&
242 getTM().getCLOpts().enable_dead_defs)
243 addMachineFunctionPass(Pass: RISCVDeadRegisterDefinitionsPass(), PMW);
244 return Base::addRegAssignAndRewriteFast(PMW);
245}
246
247Expected<bool> RISCVCodeGenPassBuilder::addRegAssignAndRewriteOptimized(
248 PassManagerWrapper &PMW) {
249 addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateRVVReg, "rvv"}), PMW);
250 addMachineFunctionPass(Pass: VirtRegRewriterPass(false), PMW);
251 addMachineFunctionPass(Pass: RISCVInsertVSETVLIPass(), PMW);
252 if (getOptLevel() != CodeGenOptLevel::None &&
253 getTM().getCLOpts().enable_dead_defs)
254 addMachineFunctionPass(Pass: RISCVDeadRegisterDefinitionsPass(), PMW);
255 return Base::addRegAssignAndRewriteOptimized(PMW);
256}
257
258void RISCVCodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) {
259 if (getOptLevel() != CodeGenOptLevel::None &&
260 getTM().getCLOpts().enable_copyelim)
261 addMachineFunctionPass(Pass: RISCVRedundantCopyEliminationPass(), PMW);
262}
263
264void RISCVCodeGenPassBuilder::addPreSched2(PassManagerWrapper &PMW) {
265 addMachineFunctionPass(Pass: RISCVExpandPseudoPostRAPass(), PMW);
266
267 addMachineFunctionPass(Pass: MachineKCFIPass(), PMW);
268 if (getOptLevel() != CodeGenOptLevel::None)
269 addMachineFunctionPass(Pass: RISCVLoadStoreOptPass(), PMW);
270}
271
272void RISCVCodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
273 // TODO: It would potentially be better to schedule copy propagation after
274 // expanding pseudos (in addPreEmitPass2). However, performing copy
275 // propagation after the machine outliner (which runs after addPreEmitPass)
276 // currently leads to incorrect code-gen, where copies to registers within
277 // outlined functions are removed erroneously.
278 if (getOptLevel() >= CodeGenOptLevel::Default &&
279 getTM().getCLOpts().enable_copy_propagation)
280 addMachineFunctionPass(Pass: MachineCopyPropagationPass(true), PMW);
281 if (getOptLevel() >= CodeGenOptLevel::Default)
282 addMachineFunctionPass(Pass: RISCVLateBranchOptPass(), PMW);
283 // The IndirectBranchTrackingPass inserts lpad and could have changed the
284 // basic block alignment. It must be done before Branch Relaxation to
285 // prevent the adjusted offset exceeding the branch range.
286 addMachineFunctionPass(Pass: RISCVIndirectBranchTrackingPass(), PMW);
287 addMachineFunctionPass(Pass: BranchRelaxationPass(), PMW);
288 addMachineFunctionPass(Pass: RISCVMakeCompressiblePass(), PMW);
289}
290
291void RISCVCodeGenPassBuilder::addPreEmitPass2(PassManagerWrapper &PMW) {
292 if (getOptLevel() != CodeGenOptLevel::None) {
293 addMachineFunctionPass(Pass: RISCVMoveMergePass(), PMW);
294 // Schedule PushPop Optimization before expansion of Pseudo instruction,
295 // ensuring return instruction is detected correctly.
296 addMachineFunctionPass(Pass: RISCVPushPopOptPass(), PMW);
297 }
298 addMachineFunctionPass(Pass: RISCVExpandPseudoPreEmitPass(), PMW);
299
300 // Add QC Relaxation Markers as late as possible, and only for RV32
301 if (getOptLevel() != CodeGenOptLevel::None &&
302 TM.getTargetTriple().isRISCV32()) {
303 addMachineFunctionPass(Pass: RISCVQCRelaxMarkingPass(), PMW);
304 }
305
306 // Schedule the expansion of AMOs at the last possible moment, avoiding the
307 // possibility for other passes to break the requirements for forward
308 // progress in the LR/SC block.
309 addMachineFunctionPass(Pass: RISCVExpandPseudoAtomicsPass(), PMW);
310
311 // KCFI indirect call checks are lowered to a bundle.
312 addMachineFunctionPass(
313 Pass: UnpackMachineBundlesPass([&](const MachineFunction &MF) {
314 return MF.getFunction().getParent()->getModuleFlag(Key: "kcfi");
315 }),
316 PMW);
317
318 // RISCVTargetMachine's constructor sets Options.enable_cfi_instr_inserter to
319 // the inverse of -riscv-enable-cfi-instr-inserter (a flag private to
320 // RISCVTargetMachine.cpp), so checking it here is equivalent to checking
321 // that flag directly -- the two passes solve overlapping problems and
322 // this target picks exactly one.
323 if (getTM().getCLOpts().enable_cfi_instr_inserter)
324 addMachineFunctionPass(Pass: CFIInstrInserterPass(), PMW);
325}
326
327void RISCVCodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
328 addModulePass(Pass: RISCVAsmPrinterBeginPass(), PMW, /*Force=*/true);
329}
330
331void RISCVCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
332 addMachineFunctionPass(Pass: RISCVAsmPrinterPass(), PMW);
333}
334
335void RISCVCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
336 addModulePass(Pass: RISCVAsmPrinterEndPass(), PMW, /*Force=*/true);
337}
338
339} // namespace
340
341void RISCVTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
342#define GET_PASS_REGISTRY "RISCVPassRegistry.def"
343#include "llvm/Passes/TargetPassRegistry.inc"
344
345 PB.registerLateLoopOptimizationsEPCallback(C: [=](LoopPassManager &LPM,
346 OptimizationLevel Level) {
347 if (Level != OptimizationLevel::O0)
348 LPM.addPass(Pass: LoopIdiomVectorizePass(LoopIdiomVectorizeStyle::Predicated));
349 });
350}
351
352Error RISCVTargetMachine::buildCodeGenPipeline(
353 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
354 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
355 const CGPassBuilderOption &Opt, MCContext &Ctx,
356 PassInstrumentationCallbacks *PIC) {
357 auto CGPB = RISCVCodeGenPassBuilder(*this, Opt, PIC);
358 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
359}
360