| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// \file |
| 9 | /// This file contains the RISC-V CodeGen pipeline builder. |
| 10 | //===----------------------------------------------------------------------===// |
| 11 | |
| 12 | #include "RISCV.h" |
| 13 | #include "RISCVAsmPrinter.h" |
| 14 | #include "RISCVTargetMachine.h" |
| 15 | #include "llvm/CodeGen/AtomicExpand.h" |
| 16 | #include "llvm/CodeGen/BranchRelaxation.h" |
| 17 | #include "llvm/CodeGen/CFIInstrInserter.h" |
| 18 | #include "llvm/CodeGen/GlobalISel/IRTranslator.h" |
| 19 | #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" |
| 20 | #include "llvm/CodeGen/GlobalISel/Legalizer.h" |
| 21 | #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" |
| 22 | #include "llvm/CodeGen/GlobalMerge.h" |
| 23 | #include "llvm/CodeGen/InitUndef.h" |
| 24 | #include "llvm/CodeGen/InterleavedAccess.h" |
| 25 | #include "llvm/CodeGen/KCFI.h" |
| 26 | #include "llvm/CodeGen/MachineCombiner.h" |
| 27 | #include "llvm/CodeGen/MachineCopyPropagation.h" |
| 28 | #include "llvm/CodeGen/MachineInstrBundle.h" |
| 29 | #include "llvm/CodeGen/MachineLICM.h" |
| 30 | #include "llvm/CodeGen/RegAllocFast.h" |
| 31 | #include "llvm/CodeGen/RegAllocGreedyPass.h" |
| 32 | #include "llvm/CodeGen/SelectOptimize.h" |
| 33 | #include "llvm/CodeGen/TypePromotion.h" |
| 34 | #include "llvm/CodeGen/VirtRegMap.h" |
| 35 | #include "llvm/IR/PassInstrumentation.h" |
| 36 | #include "llvm/MC/MCStreamer.h" |
| 37 | #include "llvm/Passes/CodeGenPassBuilder.h" |
| 38 | #include "llvm/Passes/PassBuilder.h" |
| 39 | #include "llvm/Support/CodeGen.h" |
| 40 | #include "llvm/Target/CGPassBuilderOption.h" |
| 41 | #include "llvm/Transforms/Scalar/LoopDataPrefetch.h" |
| 42 | #include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h" |
| 43 | |
| 44 | using namespace llvm; |
| 45 | |
| 46 | namespace { |
| 47 | |
| 48 | class RISCVCodeGenPassBuilder : public CodeGenPassBuilder { |
| 49 | using Base = CodeGenPassBuilder; |
| 50 | |
| 51 | RISCVTargetMachine &getTM() const { |
| 52 | return static_cast<RISCVTargetMachine &>(TM); |
| 53 | } |
| 54 | |
| 55 | public: |
| 56 | explicit RISCVCodeGenPassBuilder(RISCVTargetMachine &TM, |
| 57 | const CGPassBuilderOption &Opts, |
| 58 | PassInstrumentationCallbacks *PIC) |
| 59 | : CodeGenPassBuilder(TM, Opts, PIC) { |
| 60 | // TODO: See the FIXME on RISCVPassConfig::setEnableSinkAndFold in the |
| 61 | // legacy pass manager pipeline. There is currently no way to plumb |
| 62 | // -riscv-enable-sink-fold through to CGPassBuilderOption::EnableSinkAndFold |
| 63 | // from a target-local CodeGenPassBuilder, so this NewPM pipeline always |
| 64 | // uses the base class default (disabled). |
| 65 | } |
| 66 | |
| 67 | void addIRPasses(PassManagerWrapper &PMW) override; |
| 68 | void addPreISel(PassManagerWrapper &PMW) override; |
| 69 | void addCodeGenPrepare(PassManagerWrapper &PMW) override; |
| 70 | |
| 71 | Error addInstSelector(PassManagerWrapper &PMW) override; |
| 72 | |
| 73 | Error addIRTranslator(PassManagerWrapper &PMW) override; |
| 74 | void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override; |
| 75 | Error addLegalizeMachineIR(PassManagerWrapper &PMW) override; |
| 76 | void addPreRegBankSelect(PassManagerWrapper &PMW) override; |
| 77 | Error addRegBankSelect(PassManagerWrapper &PMW) override; |
| 78 | Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override; |
| 79 | |
| 80 | void addMachineSSAOptimization(PassManagerWrapper &PMW) override; |
| 81 | void addILPOpts(PassManagerWrapper &PMW) override; |
| 82 | void addPreRegAlloc(PassManagerWrapper &PMW) override; |
| 83 | Error addFastRegAlloc(PassManagerWrapper &PMW) override; |
| 84 | Error addRegAssignAndRewriteFast(PassManagerWrapper &PMW) override; |
| 85 | Expected<bool> |
| 86 | addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) override; |
| 87 | void addPostRegAlloc(PassManagerWrapper &PMW) override; |
| 88 | void addPreSched2(PassManagerWrapper &PMW) override; |
| 89 | void addPreEmitPass(PassManagerWrapper &PMW) override; |
| 90 | void addPreEmitPass2(PassManagerWrapper &PMW) override; |
| 91 | void addAsmPrinterBegin(PassManagerWrapper &PMW) override; |
| 92 | void addAsmPrinter(PassManagerWrapper &PMW) override; |
| 93 | void addAsmPrinterEnd(PassManagerWrapper &PMW) override; |
| 94 | }; |
| 95 | |
| 96 | void RISCVCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) { |
| 97 | addFunctionPass(Pass: AtomicExpandPass(TM), PMW); |
| 98 | addFunctionPass(Pass: RISCVZacasABIFixPass(&getTM()), PMW); |
| 99 | |
| 100 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 101 | if (getTM().getCLOpts().enable_loop_data_prefetch) |
| 102 | addFunctionPass(Pass: LoopDataPrefetchPass(), PMW); |
| 103 | |
| 104 | addFunctionPass(Pass: RISCVGatherScatterLoweringPass(&getTM()), PMW); |
| 105 | addFunctionPass(Pass: InterleavedAccessPass(TM), PMW); |
| 106 | addFunctionPass(Pass: RISCVCodeGenPreparePass(&getTM()), PMW); |
| 107 | } |
| 108 | |
| 109 | Base::addIRPasses(PMW); |
| 110 | |
| 111 | if (getOptLevel() == CodeGenOptLevel::Aggressive && |
| 112 | getTM().getCLOpts().select_opt) |
| 113 | addFunctionPass(Pass: SelectOptimizePass(getTM()), PMW); |
| 114 | } |
| 115 | |
| 116 | void RISCVCodeGenPassBuilder::addPreISel(PassManagerWrapper &PMW) { |
| 117 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 118 | flushFPMsToMPM(PMW); |
| 119 | addModulePass(Pass: RISCVPromoteConstantPass(&getTM()), PMW); |
| 120 | } |
| 121 | |
| 122 | if (valueOr(X: getTM().getCLOpts().enable_global_merge, |
| 123 | Default: getOptLevel() != CodeGenOptLevel::None)) { |
| 124 | // FIXME: Like AArch64, we disable extern global merging by default due to |
| 125 | // concerns it might regress some workloads. Unlike AArch64, we don't |
| 126 | // currently support enabling the pass in an "OnlyOptimizeForSize" mode. |
| 127 | // Investigating and addressing both items are TODO. |
| 128 | flushFPMsToMPM(PMW); |
| 129 | GlobalMergeOptions GMOptions; |
| 130 | GMOptions.MaxOffset = 2047; |
| 131 | GMOptions.MergeExternal = true; |
| 132 | GMOptions.SizeOnly = false; |
| 133 | addModulePass(Pass: GlobalMergePass(&getTM(), GMOptions), PMW); |
| 134 | } |
| 135 | } |
| 136 | |
| 137 | void RISCVCodeGenPassBuilder::addCodeGenPrepare(PassManagerWrapper &PMW) { |
| 138 | if (getOptLevel() != CodeGenOptLevel::None) |
| 139 | addFunctionPass(Pass: TypePromotionPass(TM), PMW); |
| 140 | Base::addCodeGenPrepare(PMW); |
| 141 | } |
| 142 | |
| 143 | Error RISCVCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) { |
| 144 | addMachineFunctionPass(Pass: RISCVISelDAGToDAGPass(getTM(), getOptLevel()), PMW); |
| 145 | return Error::success(); |
| 146 | } |
| 147 | |
| 148 | Error RISCVCodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) { |
| 149 | addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW); |
| 150 | return Error::success(); |
| 151 | } |
| 152 | |
| 153 | void RISCVCodeGenPassBuilder::addPreLegalizeMachineIR(PassManagerWrapper &PMW) { |
| 154 | if (getOptLevel() == CodeGenOptLevel::None) |
| 155 | addMachineFunctionPass(Pass: RISCVO0PreLegalizerCombinerPass(), PMW); |
| 156 | else |
| 157 | addMachineFunctionPass(Pass: RISCVPreLegalizerCombinerPass(), PMW); |
| 158 | } |
| 159 | |
| 160 | Error RISCVCodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) { |
| 161 | addMachineFunctionPass(Pass: LegalizerPass(), PMW); |
| 162 | return Error::success(); |
| 163 | } |
| 164 | |
| 165 | void RISCVCodeGenPassBuilder::addPreRegBankSelect(PassManagerWrapper &PMW) { |
| 166 | if (getOptLevel() != CodeGenOptLevel::None) |
| 167 | addMachineFunctionPass(Pass: RISCVPostLegalizerCombinerPass(), PMW); |
| 168 | } |
| 169 | |
| 170 | Error RISCVCodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) { |
| 171 | addMachineFunctionPass(Pass: RegBankSelectPass(), PMW); |
| 172 | return Error::success(); |
| 173 | } |
| 174 | |
| 175 | Error RISCVCodeGenPassBuilder::addGlobalInstructionSelect( |
| 176 | PassManagerWrapper &PMW) { |
| 177 | addMachineFunctionPass(Pass: InstructionSelectPass(getOptLevel()), PMW); |
| 178 | return Error::success(); |
| 179 | } |
| 180 | |
| 181 | void RISCVCodeGenPassBuilder::addMachineSSAOptimization( |
| 182 | PassManagerWrapper &PMW) { |
| 183 | // It's beneficial to reduce the VL to enable more |
| 184 | // Machine SSA optimizations. |
| 185 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 186 | // RISCVVLOptimizer can make loop invariant instructions like vmv.v.i |
| 187 | // loop variant by propagating a VL defined inside the loop. Run LICM and |
| 188 | // hoist them early. Don't do this at -O0 to avoid the compile-time |
| 189 | // overhead. Not reducing the VL of loop invariant pseudos results in more |
| 190 | // vsetvli toggles, and still requires the MachineLoopInfo analysis to be |
| 191 | // run. |
| 192 | addMachineFunctionPass(Pass: EarlyMachineLICMPass(), PMW); |
| 193 | addMachineFunctionPass(Pass: RISCVVLOptimizerPass(), PMW); |
| 194 | } |
| 195 | |
| 196 | addMachineFunctionPass(Pass: RISCVVectorPeepholePass(), PMW); |
| 197 | addMachineFunctionPass(Pass: RISCVFoldMemOffsetPass(), PMW); |
| 198 | |
| 199 | Base::addMachineSSAOptimization(PMW); |
| 200 | |
| 201 | if (TM.getTargetTriple().isRISCV64()) |
| 202 | addMachineFunctionPass(Pass: RISCVOptWInstrsPass(), PMW); |
| 203 | } |
| 204 | |
| 205 | void RISCVCodeGenPassBuilder::addILPOpts(PassManagerWrapper &PMW) { |
| 206 | if (getTM().getCLOpts().enable_machine_combiner) |
| 207 | addMachineFunctionPass(Pass: MachineCombinerPass(), PMW); |
| 208 | } |
| 209 | |
| 210 | void RISCVCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) { |
| 211 | addMachineFunctionPass(Pass: RISCVExpandPseudoPreRAPass(), PMW); |
| 212 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 213 | addMachineFunctionPass(Pass: RISCVMergeBaseOffsetPass(), PMW); |
| 214 | // Add Zilsd pre-allocation load/store optimization |
| 215 | addMachineFunctionPass(Pass: RISCVPreAllocZilsdOptPass(), PMW); |
| 216 | } |
| 217 | |
| 218 | addMachineFunctionPass(Pass: RISCVInsertReadWriteCSRPass(), PMW); |
| 219 | addMachineFunctionPass(Pass: RISCVInsertWriteVXRMPass(), PMW); |
| 220 | addMachineFunctionPass(Pass: RISCVLandingPadSetupPass(), PMW); |
| 221 | addMachineFunctionPass(Pass: RISCVVMV0EliminationPass(), PMW); |
| 222 | } |
| 223 | |
| 224 | Error RISCVCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) { |
| 225 | addMachineFunctionPass(Pass: InitUndefPass(), PMW); |
| 226 | return Base::addFastRegAlloc(PMW); |
| 227 | } |
| 228 | |
| 229 | static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI, |
| 230 | const MachineRegisterInfo &MRI, |
| 231 | const Register Reg) { |
| 232 | const TargetRegisterClass *RC = MRI.getRegClass(Reg); |
| 233 | return RISCVRegisterInfo::isRVVRegClass(RC); |
| 234 | } |
| 235 | |
| 236 | Error RISCVCodeGenPassBuilder::addRegAssignAndRewriteFast( |
| 237 | PassManagerWrapper &PMW) { |
| 238 | addMachineFunctionPass( |
| 239 | Pass: RegAllocFastPass({onlyAllocateRVVReg, "rvv" , /*ClearVRegs=*/false}), PMW); |
| 240 | addMachineFunctionPass(Pass: RISCVInsertVSETVLIPass(), PMW); |
| 241 | if (getOptLevel() != CodeGenOptLevel::None && |
| 242 | getTM().getCLOpts().enable_dead_defs) |
| 243 | addMachineFunctionPass(Pass: RISCVDeadRegisterDefinitionsPass(), PMW); |
| 244 | return Base::addRegAssignAndRewriteFast(PMW); |
| 245 | } |
| 246 | |
| 247 | Expected<bool> RISCVCodeGenPassBuilder::addRegAssignAndRewriteOptimized( |
| 248 | PassManagerWrapper &PMW) { |
| 249 | addMachineFunctionPass(Pass: RAGreedyPass({onlyAllocateRVVReg, "rvv" }), PMW); |
| 250 | addMachineFunctionPass(Pass: VirtRegRewriterPass(false), PMW); |
| 251 | addMachineFunctionPass(Pass: RISCVInsertVSETVLIPass(), PMW); |
| 252 | if (getOptLevel() != CodeGenOptLevel::None && |
| 253 | getTM().getCLOpts().enable_dead_defs) |
| 254 | addMachineFunctionPass(Pass: RISCVDeadRegisterDefinitionsPass(), PMW); |
| 255 | return Base::addRegAssignAndRewriteOptimized(PMW); |
| 256 | } |
| 257 | |
| 258 | void RISCVCodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) { |
| 259 | if (getOptLevel() != CodeGenOptLevel::None && |
| 260 | getTM().getCLOpts().enable_copyelim) |
| 261 | addMachineFunctionPass(Pass: RISCVRedundantCopyEliminationPass(), PMW); |
| 262 | } |
| 263 | |
| 264 | void RISCVCodeGenPassBuilder::addPreSched2(PassManagerWrapper &PMW) { |
| 265 | addMachineFunctionPass(Pass: RISCVExpandPseudoPostRAPass(), PMW); |
| 266 | |
| 267 | addMachineFunctionPass(Pass: MachineKCFIPass(), PMW); |
| 268 | if (getOptLevel() != CodeGenOptLevel::None) |
| 269 | addMachineFunctionPass(Pass: RISCVLoadStoreOptPass(), PMW); |
| 270 | } |
| 271 | |
| 272 | void RISCVCodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) { |
| 273 | // TODO: It would potentially be better to schedule copy propagation after |
| 274 | // expanding pseudos (in addPreEmitPass2). However, performing copy |
| 275 | // propagation after the machine outliner (which runs after addPreEmitPass) |
| 276 | // currently leads to incorrect code-gen, where copies to registers within |
| 277 | // outlined functions are removed erroneously. |
| 278 | if (getOptLevel() >= CodeGenOptLevel::Default && |
| 279 | getTM().getCLOpts().enable_copy_propagation) |
| 280 | addMachineFunctionPass(Pass: MachineCopyPropagationPass(true), PMW); |
| 281 | if (getOptLevel() >= CodeGenOptLevel::Default) |
| 282 | addMachineFunctionPass(Pass: RISCVLateBranchOptPass(), PMW); |
| 283 | // The IndirectBranchTrackingPass inserts lpad and could have changed the |
| 284 | // basic block alignment. It must be done before Branch Relaxation to |
| 285 | // prevent the adjusted offset exceeding the branch range. |
| 286 | addMachineFunctionPass(Pass: RISCVIndirectBranchTrackingPass(), PMW); |
| 287 | addMachineFunctionPass(Pass: BranchRelaxationPass(), PMW); |
| 288 | addMachineFunctionPass(Pass: RISCVMakeCompressiblePass(), PMW); |
| 289 | } |
| 290 | |
| 291 | void RISCVCodeGenPassBuilder::addPreEmitPass2(PassManagerWrapper &PMW) { |
| 292 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 293 | addMachineFunctionPass(Pass: RISCVMoveMergePass(), PMW); |
| 294 | // Schedule PushPop Optimization before expansion of Pseudo instruction, |
| 295 | // ensuring return instruction is detected correctly. |
| 296 | addMachineFunctionPass(Pass: RISCVPushPopOptPass(), PMW); |
| 297 | } |
| 298 | addMachineFunctionPass(Pass: RISCVExpandPseudoPreEmitPass(), PMW); |
| 299 | |
| 300 | // Add QC Relaxation Markers as late as possible, and only for RV32 |
| 301 | if (getOptLevel() != CodeGenOptLevel::None && |
| 302 | TM.getTargetTriple().isRISCV32()) { |
| 303 | addMachineFunctionPass(Pass: RISCVQCRelaxMarkingPass(), PMW); |
| 304 | } |
| 305 | |
| 306 | // Schedule the expansion of AMOs at the last possible moment, avoiding the |
| 307 | // possibility for other passes to break the requirements for forward |
| 308 | // progress in the LR/SC block. |
| 309 | addMachineFunctionPass(Pass: RISCVExpandPseudoAtomicsPass(), PMW); |
| 310 | |
| 311 | // KCFI indirect call checks are lowered to a bundle. |
| 312 | addMachineFunctionPass( |
| 313 | Pass: UnpackMachineBundlesPass([&](const MachineFunction &MF) { |
| 314 | return MF.getFunction().getParent()->getModuleFlag(Key: "kcfi" ); |
| 315 | }), |
| 316 | PMW); |
| 317 | |
| 318 | // RISCVTargetMachine's constructor sets Options.enable_cfi_instr_inserter to |
| 319 | // the inverse of -riscv-enable-cfi-instr-inserter (a flag private to |
| 320 | // RISCVTargetMachine.cpp), so checking it here is equivalent to checking |
| 321 | // that flag directly -- the two passes solve overlapping problems and |
| 322 | // this target picks exactly one. |
| 323 | if (getTM().getCLOpts().enable_cfi_instr_inserter) |
| 324 | addMachineFunctionPass(Pass: CFIInstrInserterPass(), PMW); |
| 325 | } |
| 326 | |
| 327 | void RISCVCodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) { |
| 328 | addModulePass(Pass: RISCVAsmPrinterBeginPass(), PMW, /*Force=*/true); |
| 329 | } |
| 330 | |
| 331 | void RISCVCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) { |
| 332 | addMachineFunctionPass(Pass: RISCVAsmPrinterPass(), PMW); |
| 333 | } |
| 334 | |
| 335 | void RISCVCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) { |
| 336 | addModulePass(Pass: RISCVAsmPrinterEndPass(), PMW, /*Force=*/true); |
| 337 | } |
| 338 | |
| 339 | } // namespace |
| 340 | |
| 341 | void RISCVTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) { |
| 342 | #define GET_PASS_REGISTRY "RISCVPassRegistry.def" |
| 343 | #include "llvm/Passes/TargetPassRegistry.inc" |
| 344 | |
| 345 | PB.registerLateLoopOptimizationsEPCallback(C: [=](LoopPassManager &LPM, |
| 346 | OptimizationLevel Level) { |
| 347 | if (Level != OptimizationLevel::O0) |
| 348 | LPM.addPass(Pass: LoopIdiomVectorizePass(LoopIdiomVectorizeStyle::Predicated)); |
| 349 | }); |
| 350 | } |
| 351 | |
| 352 | Error RISCVTargetMachine::buildCodeGenPipeline( |
| 353 | ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out, |
| 354 | raw_pwrite_stream *DwoOut, CodeGenFileType FileType, |
| 355 | const CGPassBuilderOption &Opt, MCContext &Ctx, |
| 356 | PassInstrumentationCallbacks *PIC) { |
| 357 | auto CGPB = RISCVCodeGenPassBuilder(*this, Opt, PIC); |
| 358 | return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx); |
| 359 | } |
| 360 | |