| 1 | //===-- RISCVTargetMachine.cpp - Define TargetMachine for RISC-V ----------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Implements the info about RISC-V target spec. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "RISCVTargetMachine.h" |
| 14 | #include "RISCV.h" |
| 15 | #include "RISCVMachineFunctionInfo.h" |
| 16 | #include "RISCVMachineScheduler.h" |
| 17 | #include "RISCVTargetObjectFile.h" |
| 18 | #include "RISCVTargetTransformInfo.h" |
| 19 | #include "TargetInfo/RISCVTargetInfo.h" |
| 20 | #include "llvm/Analysis/TargetTransformInfo.h" |
| 21 | #include "llvm/CodeGen/GlobalISel/CSEInfo.h" |
| 22 | #include "llvm/CodeGen/GlobalISel/IRTranslator.h" |
| 23 | #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" |
| 24 | #include "llvm/CodeGen/GlobalISel/Legalizer.h" |
| 25 | #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" |
| 26 | #include "llvm/CodeGen/MIRParser/MIParser.h" |
| 27 | #include "llvm/CodeGen/MIRYamlMapping.h" |
| 28 | #include "llvm/CodeGen/MachineScheduler.h" |
| 29 | #include "llvm/CodeGen/MacroFusion.h" |
| 30 | #include "llvm/CodeGen/Passes.h" |
| 31 | #include "llvm/CodeGen/RegAllocRegistry.h" |
| 32 | #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" |
| 33 | #include "llvm/CodeGen/TargetPassConfig.h" |
| 34 | #include "llvm/InitializePasses.h" |
| 35 | #include "llvm/MC/TargetRegistry.h" |
| 36 | #include "llvm/Option/LibraryOptions.h" |
| 37 | #include "llvm/Support/Compiler.h" |
| 38 | #include "llvm/Target/TargetOptions.h" |
| 39 | #include "llvm/Transforms/IPO.h" |
| 40 | #include "llvm/Transforms/Scalar.h" |
| 41 | #include <optional> |
| 42 | using namespace llvm; |
| 43 | |
| 44 | extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() { |
| 45 | RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target()); |
| 46 | RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target()); |
| 47 | RegisterTargetMachine<RISCVTargetMachine> A(getTheRISCV32beTarget()); |
| 48 | RegisterTargetMachine<RISCVTargetMachine> B(getTheRISCV64beTarget()); |
| 49 | static opt::RegisterLibraryOptions<RISCVOptions> O; |
| 50 | auto *PR = PassRegistry::getPassRegistry(); |
| 51 | initializeGlobalISel(*PR); |
| 52 | initializeRISCVO0PreLegalizerCombinerLegacyPass(*PR); |
| 53 | initializeRISCVPreLegalizerCombinerLegacyPass(*PR); |
| 54 | initializeRISCVPostLegalizerCombinerLegacyPass(*PR); |
| 55 | initializeMachineKCFILegacyPass(*PR); |
| 56 | initializeRISCVDeadRegisterDefinitionsLegacyPass(*PR); |
| 57 | initializeRISCVLateBranchOptLegacyPass(*PR); |
| 58 | initializeRISCVMakeCompressibleLegacyPass(*PR); |
| 59 | initializeRISCVQCRelaxMarkingLegacyPass(*PR); |
| 60 | initializeRISCVGatherScatterLoweringLegacyPass(*PR); |
| 61 | initializeRISCVCodeGenPrepareLegacyPass(*PR); |
| 62 | initializeRISCVZacasABIFixLegacyPass(*PR); |
| 63 | initializeRISCVExpandPseudoPostRALegacyPass(*PR); |
| 64 | initializeRISCVMergeBaseOffsetLegacyPass(*PR); |
| 65 | initializeRISCVOptWInstrsLegacyPass(*PR); |
| 66 | initializeRISCVFoldMemOffsetLegacyPass(*PR); |
| 67 | initializeRISCVExpandPseudoPreRALegacyPass(*PR); |
| 68 | initializeRISCVExpandPseudoPreEmitLegacyPass(*PR); |
| 69 | initializeRISCVVectorPeepholeLegacyPass(*PR); |
| 70 | initializeRISCVVLOptimizerLegacyPass(*PR); |
| 71 | initializeRISCVVMV0EliminationLegacyPass(*PR); |
| 72 | initializeRISCVInsertVSETVLILegacyPass(*PR); |
| 73 | initializeRISCVInsertReadWriteCSRLegacyPass(*PR); |
| 74 | initializeRISCVInsertWriteVXRMLegacyPass(*PR); |
| 75 | initializeRISCVLandingPadSetupLegacyPass(*PR); |
| 76 | initializeRISCVDAGToDAGISelLegacyPass(*PR); |
| 77 | initializeRISCVMoveMergeLegacyPass(*PR); |
| 78 | initializeRISCVPushPopOptLegacyPass(*PR); |
| 79 | initializeRISCVIndirectBranchTrackingLegacyPass(*PR); |
| 80 | initializeRISCVLoadStoreOptLegacyPass(*PR); |
| 81 | initializeRISCVPreAllocZilsdOptLegacyPass(*PR); |
| 82 | initializeRISCVExpandPseudoAtomicsLegacyPass(*PR); |
| 83 | initializeRISCVRedundantCopyEliminationLegacyPass(*PR); |
| 84 | initializeRISCVAsmPrinterPass(*PR); |
| 85 | initializeRISCVPromoteConstantLegacyPass(*PR); |
| 86 | } |
| 87 | |
| 88 | static Reloc::Model getEffectiveRelocModel(const Triple &TT, |
| 89 | std::optional<Reloc::Model> RM) { |
| 90 | if (TT.isOSBinFormatMachO()) |
| 91 | return RM.value_or(u: Reloc::PIC_); |
| 92 | |
| 93 | return RM.value_or(u: Reloc::Static); |
| 94 | } |
| 95 | |
| 96 | static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { |
| 97 | if (TT.isOSBinFormatMachO()) |
| 98 | return std::make_unique<RISCVMachOTargetObjectFile>(); |
| 99 | return std::make_unique<RISCVELFTargetObjectFile>(); |
| 100 | } |
| 101 | |
| 102 | RISCVTargetMachine::RISCVTargetMachine(const Target &T, const Triple &TT, |
| 103 | StringRef CPU, StringRef FS, |
| 104 | const TargetOptions &Options, |
| 105 | std::optional<Reloc::Model> RM, |
| 106 | std::optional<CodeModel::Model> CM, |
| 107 | CodeGenOptLevel OL, bool JIT) |
| 108 | : CodeGenTargetMachineImpl(T, TT, CPU, FS, Options, |
| 109 | getEffectiveRelocModel(TT, RM), |
| 110 | getEffectiveCodeModel(CM, Default: CodeModel::Small), OL), |
| 111 | CLOpts(cl::LibraryOptionsContext::get<RISCVOptions>( |
| 112 | C: Options.MCOptions.LibraryOptions)), |
| 113 | TLOF(createTLOF(TT)) { |
| 114 | initAsmInfo(); |
| 115 | |
| 116 | LLT::setUseExtended(true); |
| 117 | |
| 118 | // RISC-V supports the MachineOutliner. |
| 119 | setMachineOutliner(true); |
| 120 | setSupportsDefaultOutlining(true); |
| 121 | |
| 122 | // RISC-V supports the debug entry values. |
| 123 | setSupportsDebugEntryValues(true); |
| 124 | |
| 125 | if (TT.isOSFuchsia() && !TT.isArch64Bit()) |
| 126 | report_fatal_error(reason: "Fuchsia is only supported for 64-bit" ); |
| 127 | |
| 128 | setCFIFixup(!CLOpts.enable_cfi_instr_inserter); |
| 129 | } |
| 130 | |
| 131 | const RISCVSubtarget * |
| 132 | RISCVTargetMachine::getSubtargetImpl(const Function &F) const { |
| 133 | AttributeSet FnAttrs = F.getAttributes().getFnAttrs(); |
| 134 | const Module *M = F.getParent(); |
| 135 | if (const RISCVSubtarget *ST = LastSubtarget.lookup(NewFnAttrs: FnAttrs, NewM: M)) |
| 136 | return ST; |
| 137 | |
| 138 | SubtargetAttrs Attrs; |
| 139 | Attrs.CPU = FnAttrs.getAttribute(Kind: "target-cpu" ); |
| 140 | Attrs.TuneCPU = FnAttrs.getAttribute(Kind: "tune-cpu" ); |
| 141 | Attrs.FS = FnAttrs.getAttribute(Kind: "target-features" ); |
| 142 | Attrs.VScaleRange = FnAttrs.getAttribute(Kind: Attribute::VScaleRange); |
| 143 | |
| 144 | if (const RISCVSubtarget *ST = LastSubtarget.lookup(NewFnAttrs: FnAttrs, NewM: M, NewAttrs: Attrs)) |
| 145 | return ST; |
| 146 | |
| 147 | StringRef CPU = |
| 148 | Attrs.CPU.isValid() ? Attrs.CPU.getValueAsString() : TargetCPU; |
| 149 | StringRef TuneCPU = |
| 150 | Attrs.TuneCPU.isValid() ? Attrs.TuneCPU.getValueAsString() : CPU; |
| 151 | StringRef FS = Attrs.FS.isValid() ? Attrs.FS.getValueAsString() : TargetFS; |
| 152 | |
| 153 | unsigned RVVBitsMin = CLOpts.v_vector_bits_min.value_or(u: -1); |
| 154 | unsigned RVVBitsMax = CLOpts.v_vector_bits_max.value_or(u: 0); |
| 155 | |
| 156 | if (Attrs.VScaleRange.isValid()) { |
| 157 | if (!CLOpts.v_vector_bits_min) |
| 158 | RVVBitsMin = |
| 159 | Attrs.VScaleRange.getVScaleRangeMin() * RISCV::RVVBitsPerBlock; |
| 160 | std::optional<unsigned> VScaleMax = Attrs.VScaleRange.getVScaleRangeMax(); |
| 161 | if (VScaleMax.has_value() && !CLOpts.v_vector_bits_max) |
| 162 | RVVBitsMax = *VScaleMax * RISCV::RVVBitsPerBlock; |
| 163 | } |
| 164 | |
| 165 | if (RVVBitsMin != -1U) { |
| 166 | // FIXME: Change to >= 32 when VLEN = 32 is supported. |
| 167 | assert((RVVBitsMin == 0 || (RVVBitsMin >= 64 && RVVBitsMin <= 65536 && |
| 168 | isPowerOf2_32(RVVBitsMin))) && |
| 169 | "V or Zve* extension requires vector length to be in the range of " |
| 170 | "64 to 65536 and a power 2!" ); |
| 171 | assert((RVVBitsMax >= RVVBitsMin || RVVBitsMax == 0) && |
| 172 | "Minimum V extension vector length should not be larger than its " |
| 173 | "maximum!" ); |
| 174 | } |
| 175 | assert((RVVBitsMax == 0 || (RVVBitsMax >= 64 && RVVBitsMax <= 65536 && |
| 176 | isPowerOf2_32(RVVBitsMax))) && |
| 177 | "V or Zve* extension requires vector length to be in the range of " |
| 178 | "64 to 65536 and a power 2!" ); |
| 179 | |
| 180 | if (RVVBitsMin != -1U) { |
| 181 | if (RVVBitsMax != 0) { |
| 182 | RVVBitsMin = std::min(a: RVVBitsMin, b: RVVBitsMax); |
| 183 | RVVBitsMax = std::max(a: RVVBitsMin, b: RVVBitsMax); |
| 184 | } |
| 185 | |
| 186 | RVVBitsMin = llvm::bit_floor( |
| 187 | Value: (RVVBitsMin < 64 || RVVBitsMin > 65536) ? 0 : RVVBitsMin); |
| 188 | } |
| 189 | RVVBitsMax = |
| 190 | llvm::bit_floor(Value: (RVVBitsMax < 64 || RVVBitsMax > 65536) ? 0 : RVVBitsMax); |
| 191 | |
| 192 | SmallString<512> Key; |
| 193 | raw_svector_ostream(Key) << "RVVMin" << RVVBitsMin << "RVVMax" << RVVBitsMax |
| 194 | << CPU << TuneCPU << FS; |
| 195 | auto &I = SubtargetMap[Key]; |
| 196 | if (!I) { |
| 197 | StringRef ABIName = getTargetABIName(M: *F.getParent()); |
| 198 | I = std::make_unique<RISCVSubtarget>( |
| 199 | args: TargetTriple, args&: CPU, args&: TuneCPU, args&: FS, args&: ABIName, args&: RVVBitsMin, args&: RVVBitsMax, args: *this); |
| 200 | } |
| 201 | LastSubtarget.set(NewFnAttrs: FnAttrs, NewM: M, NewAttrs: Attrs, NewSubtarget: I.get()); |
| 202 | return I.get(); |
| 203 | } |
| 204 | |
| 205 | MachineFunctionInfo *RISCVTargetMachine::createMachineFunctionInfo( |
| 206 | BumpPtrAllocator &Allocator, const Function &F, |
| 207 | const TargetSubtargetInfo *STI) const { |
| 208 | return RISCVMachineFunctionInfo::create<RISCVMachineFunctionInfo>( |
| 209 | Allocator, F, STI: static_cast<const RISCVSubtarget *>(STI)); |
| 210 | } |
| 211 | |
| 212 | TargetTransformInfo |
| 213 | RISCVTargetMachine::getTargetTransformInfo(const Function &F) const { |
| 214 | return TargetTransformInfo(std::make_unique<RISCVTTIImpl>(args: this, args: F)); |
| 215 | } |
| 216 | |
| 217 | // A RISC-V hart has a single byte-addressable address space of 2^XLEN bytes |
| 218 | // for all memory accesses, so it is reasonable to assume that an |
| 219 | // implementation has no-op address space casts. If an implementation makes a |
| 220 | // change to this, they can override it here. |
| 221 | bool RISCVTargetMachine::isNoopAddrSpaceCast(const DataLayout &DL, |
| 222 | unsigned SrcAS, |
| 223 | unsigned DstAS) const { |
| 224 | return true; |
| 225 | } |
| 226 | |
| 227 | ScheduleDAGInstrs * |
| 228 | RISCVTargetMachine::createMachineScheduler(MachineSchedContext *C) const { |
| 229 | const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>(); |
| 230 | ScheduleDAGMILive *DAG = createSchedLive<RISCVPreRAMachineSchedStrategy>(C); |
| 231 | |
| 232 | // Add MacroFusion mutation first with a higher priority than later clustering |
| 233 | const auto &MacroFusions = ST.getMacroFusions(); |
| 234 | if (!MacroFusions.empty()) |
| 235 | DAG->addMutation(Mutation: createMacroFusionDAGMutation(Predicates: MacroFusions)); |
| 236 | |
| 237 | if (ST.enableMISchedLoadClustering()) |
| 238 | DAG->addMutation(Mutation: createLoadClusterDAGMutation( |
| 239 | TII: DAG->TII, /*ReorderWhileClustering=*/true)); |
| 240 | |
| 241 | if (ST.enableMISchedStoreClustering()) |
| 242 | DAG->addMutation(Mutation: createStoreClusterDAGMutation( |
| 243 | TII: DAG->TII, /*ReorderWhileClustering=*/true)); |
| 244 | |
| 245 | if (CLOpts.vector_mask_mutation && ST.hasVInstructions()) |
| 246 | DAG->addMutation(Mutation: createRISCVVectorMaskDAGMutation(TRI: DAG->TRI)); |
| 247 | |
| 248 | return DAG; |
| 249 | } |
| 250 | |
| 251 | ScheduleDAGInstrs * |
| 252 | RISCVTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const { |
| 253 | const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>(); |
| 254 | ScheduleDAGMI *DAG = createSchedPostRA(C); |
| 255 | |
| 256 | // Add MacroFusion mutation first with a higher priority than later clustering |
| 257 | const auto &MacroFusions = ST.getMacroFusions(); |
| 258 | if (!MacroFusions.empty()) |
| 259 | DAG->addMutation(Mutation: createMacroFusionDAGMutation(Predicates: MacroFusions)); |
| 260 | |
| 261 | if (ST.enablePostMISchedLoadClustering()) |
| 262 | DAG->addMutation(Mutation: createLoadClusterDAGMutation( |
| 263 | TII: DAG->TII, /*ReorderWhileClustering=*/true)); |
| 264 | |
| 265 | if (ST.enablePostMISchedStoreClustering()) |
| 266 | DAG->addMutation(Mutation: createStoreClusterDAGMutation( |
| 267 | TII: DAG->TII, /*ReorderWhileClustering=*/true)); |
| 268 | |
| 269 | return DAG; |
| 270 | } |
| 271 | |
| 272 | namespace { |
| 273 | |
| 274 | class RVVRegisterRegAlloc : public RegisterRegAllocBase<RVVRegisterRegAlloc> { |
| 275 | public: |
| 276 | RVVRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C) |
| 277 | : RegisterRegAllocBase(N, D, C) {} |
| 278 | }; |
| 279 | |
| 280 | static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI, |
| 281 | const MachineRegisterInfo &MRI, |
| 282 | const Register Reg) { |
| 283 | const TargetRegisterClass *RC = MRI.getRegClass(Reg); |
| 284 | return RISCVRegisterInfo::isRVVRegClass(RC); |
| 285 | } |
| 286 | |
| 287 | static FunctionPass *useDefaultRegisterAllocator() { return nullptr; } |
| 288 | |
| 289 | static llvm::once_flag InitializeDefaultRVVRegisterAllocatorFlag; |
| 290 | |
| 291 | /// -riscv-rvv-regalloc=<fast|basic|greedy> command line option. |
| 292 | /// This option could designate the rvv register allocator only. |
| 293 | /// For example: -riscv-rvv-regalloc=basic |
| 294 | static cl::opt<RVVRegisterRegAlloc::FunctionPassCtor, false, |
| 295 | RegisterPassParser<RVVRegisterRegAlloc>> |
| 296 | RVVRegAlloc("riscv-rvv-regalloc" , cl::Hidden, |
| 297 | cl::init(Val: &useDefaultRegisterAllocator), |
| 298 | cl::desc("Register allocator to use for RVV register." )); |
| 299 | |
| 300 | static void initializeDefaultRVVRegisterAllocatorOnce() { |
| 301 | RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault(); |
| 302 | |
| 303 | if (!Ctor) { |
| 304 | Ctor = RVVRegAlloc; |
| 305 | RVVRegisterRegAlloc::setDefault(RVVRegAlloc); |
| 306 | } |
| 307 | } |
| 308 | |
| 309 | static FunctionPass *createBasicRVVRegisterAllocator() { |
| 310 | return createBasicRegisterAllocator(F: onlyAllocateRVVReg); |
| 311 | } |
| 312 | |
| 313 | static FunctionPass *createGreedyRVVRegisterAllocator() { |
| 314 | return createGreedyRegisterAllocator(F: onlyAllocateRVVReg); |
| 315 | } |
| 316 | |
| 317 | static FunctionPass *createFastRVVRegisterAllocator() { |
| 318 | return createFastRegisterAllocator(F: onlyAllocateRVVReg, ClearVirtRegs: false); |
| 319 | } |
| 320 | |
| 321 | static RVVRegisterRegAlloc basicRegAllocRVVReg("basic" , |
| 322 | "basic register allocator" , |
| 323 | createBasicRVVRegisterAllocator); |
| 324 | static RVVRegisterRegAlloc |
| 325 | greedyRegAllocRVVReg("greedy" , "greedy register allocator" , |
| 326 | createGreedyRVVRegisterAllocator); |
| 327 | |
| 328 | static RVVRegisterRegAlloc fastRegAllocRVVReg("fast" , "fast register allocator" , |
| 329 | createFastRVVRegisterAllocator); |
| 330 | |
| 331 | class RISCVPassConfig : public TargetPassConfig { |
| 332 | const RISCVOptions &CLOpts; |
| 333 | |
| 334 | public: |
| 335 | RISCVPassConfig(RISCVTargetMachine &TM, PassManagerBase &PM) |
| 336 | : TargetPassConfig(TM, PM), CLOpts(TM.getCLOpts()) { |
| 337 | if (TM.getOptLevel() != CodeGenOptLevel::None) |
| 338 | substitutePass(StandardID: &PostRASchedulerID, TargetID: &PostMachineSchedulerID); |
| 339 | setEnableSinkAndFold(CLOpts.enable_sink_fold); |
| 340 | EnableLoopTermFold = true; |
| 341 | } |
| 342 | |
| 343 | RISCVTargetMachine &getRISCVTargetMachine() const { |
| 344 | return getTM<RISCVTargetMachine>(); |
| 345 | } |
| 346 | |
| 347 | void addIRPasses() override; |
| 348 | bool addPreISel() override; |
| 349 | void addCodeGenPrepare() override; |
| 350 | bool addInstSelector() override; |
| 351 | bool addIRTranslator() override; |
| 352 | void addPreLegalizeMachineIR() override; |
| 353 | bool addLegalizeMachineIR() override; |
| 354 | void addPreRegBankSelect() override; |
| 355 | bool addRegBankSelect() override; |
| 356 | bool addGlobalInstructionSelect() override; |
| 357 | void addPreEmitPass() override; |
| 358 | void addPreEmitPass2() override; |
| 359 | void addPreSched2() override; |
| 360 | void addMachineSSAOptimization() override; |
| 361 | FunctionPass *createRVVRegAllocPass(bool Optimized); |
| 362 | bool addRegAssignAndRewriteFast() override; |
| 363 | bool addRegAssignAndRewriteOptimized() override; |
| 364 | void addPreRegAlloc() override; |
| 365 | void addPostRegAlloc() override; |
| 366 | void addFastRegAlloc() override; |
| 367 | bool addILPOpts() override; |
| 368 | |
| 369 | std::unique_ptr<CSEConfigBase> getCSEConfig() const override; |
| 370 | }; |
| 371 | } // namespace |
| 372 | |
| 373 | TargetPassConfig *RISCVTargetMachine::createPassConfig(PassManagerBase &PM) { |
| 374 | return new RISCVPassConfig(*this, PM); |
| 375 | } |
| 376 | |
| 377 | std::unique_ptr<CSEConfigBase> RISCVPassConfig::getCSEConfig() const { |
| 378 | return getStandardCSEConfigForOpt(Level: TM->getOptLevel()); |
| 379 | } |
| 380 | |
| 381 | FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) { |
| 382 | // Initialize the global default. |
| 383 | llvm::call_once(flag&: InitializeDefaultRVVRegisterAllocatorFlag, |
| 384 | F&: initializeDefaultRVVRegisterAllocatorOnce); |
| 385 | |
| 386 | RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault(); |
| 387 | if (Ctor != useDefaultRegisterAllocator) |
| 388 | return Ctor(); |
| 389 | |
| 390 | if (Optimized) |
| 391 | return createGreedyRVVRegisterAllocator(); |
| 392 | |
| 393 | return createFastRVVRegisterAllocator(); |
| 394 | } |
| 395 | |
| 396 | bool RISCVPassConfig::addRegAssignAndRewriteFast() { |
| 397 | addPass(P: createRVVRegAllocPass(Optimized: false)); |
| 398 | addPass(P: createRISCVInsertVSETVLILegacyPass()); |
| 399 | if (TM->getOptLevel() != CodeGenOptLevel::None && CLOpts.enable_dead_defs) |
| 400 | addPass(P: createRISCVDeadRegisterDefinitionsLegacyPass()); |
| 401 | return TargetPassConfig::addRegAssignAndRewriteFast(); |
| 402 | } |
| 403 | |
| 404 | bool RISCVPassConfig::addRegAssignAndRewriteOptimized() { |
| 405 | addPass(P: createRVVRegAllocPass(Optimized: true)); |
| 406 | addPass(P: createVirtRegRewriter(ClearVirtRegs: false)); |
| 407 | addPass(P: createRISCVInsertVSETVLILegacyPass()); |
| 408 | if (TM->getOptLevel() != CodeGenOptLevel::None && CLOpts.enable_dead_defs) |
| 409 | addPass(P: createRISCVDeadRegisterDefinitionsLegacyPass()); |
| 410 | return TargetPassConfig::addRegAssignAndRewriteOptimized(); |
| 411 | } |
| 412 | |
| 413 | void RISCVPassConfig::addIRPasses() { |
| 414 | addPass(P: createAtomicExpandLegacyPass()); |
| 415 | addPass(P: createRISCVZacasABIFixLegacyPass()); |
| 416 | |
| 417 | if (getOptLevel() != CodeGenOptLevel::None) { |
| 418 | if (CLOpts.enable_loop_data_prefetch) |
| 419 | addPass(P: createLoopDataPrefetchPass()); |
| 420 | |
| 421 | addPass(P: createRISCVGatherScatterLoweringLegacyPass()); |
| 422 | addPass(P: createInterleavedAccessPass()); |
| 423 | addPass(P: createRISCVCodeGenPrepareLegacyPass()); |
| 424 | } |
| 425 | |
| 426 | TargetPassConfig::addIRPasses(); |
| 427 | |
| 428 | if (getOptLevel() == CodeGenOptLevel::Aggressive && CLOpts.select_opt) |
| 429 | addPass(P: createSelectOptimizePass()); |
| 430 | } |
| 431 | |
| 432 | bool RISCVPassConfig::addPreISel() { |
| 433 | if (TM->getOptLevel() != CodeGenOptLevel::None) |
| 434 | addPass(P: createRISCVPromoteConstantLegacyPass()); |
| 435 | if (TM->getOptLevel() != CodeGenOptLevel::None) { |
| 436 | // Add a barrier before instruction selection so that we will not get |
| 437 | // deleted block address after enabling default outlining. See D99707 for |
| 438 | // more details. |
| 439 | addPass(P: createBarrierNoopPass()); |
| 440 | } |
| 441 | |
| 442 | if (valueOr(X: CLOpts.enable_global_merge, |
| 443 | Default: TM->getOptLevel() != CodeGenOptLevel::None)) { |
| 444 | // FIXME: Like AArch64, we disable extern global merging by default due to |
| 445 | // concerns it might regress some workloads. Unlike AArch64, we don't |
| 446 | // currently support enabling the pass in an "OnlyOptimizeForSize" mode. |
| 447 | // Investigating and addressing both items are TODO. |
| 448 | addPass(P: createGlobalMergePass(TM, /* MaxOffset */ MaximalOffset: 2047, |
| 449 | /* OnlyOptimizeForSize */ false, |
| 450 | /* MergeExternalByDefault */ true)); |
| 451 | } |
| 452 | |
| 453 | return false; |
| 454 | } |
| 455 | |
| 456 | void RISCVPassConfig::addCodeGenPrepare() { |
| 457 | if (getOptLevel() != CodeGenOptLevel::None) |
| 458 | addPass(P: createTypePromotionLegacyPass()); |
| 459 | TargetPassConfig::addCodeGenPrepare(); |
| 460 | } |
| 461 | |
| 462 | bool RISCVPassConfig::addInstSelector() { |
| 463 | addPass(P: createRISCVISelDagLegacyPass(TM&: getRISCVTargetMachine(), OptLevel: getOptLevel())); |
| 464 | |
| 465 | return false; |
| 466 | } |
| 467 | |
| 468 | bool RISCVPassConfig::addIRTranslator() { |
| 469 | addPass(P: new IRTranslatorLegacy(getOptLevel())); |
| 470 | return false; |
| 471 | } |
| 472 | |
| 473 | void RISCVPassConfig::addPreLegalizeMachineIR() { |
| 474 | if (getOptLevel() == CodeGenOptLevel::None) { |
| 475 | addPass(P: createRISCVO0PreLegalizerCombinerLegacyPass()); |
| 476 | } else { |
| 477 | addPass(P: createRISCVPreLegalizerCombinerLegacyPass()); |
| 478 | } |
| 479 | } |
| 480 | |
| 481 | bool RISCVPassConfig::addLegalizeMachineIR() { |
| 482 | addPass(P: new LegalizerLegacy()); |
| 483 | return false; |
| 484 | } |
| 485 | |
| 486 | void RISCVPassConfig::addPreRegBankSelect() { |
| 487 | if (getOptLevel() != CodeGenOptLevel::None) |
| 488 | addPass(P: createRISCVPostLegalizerCombinerLegacyPass()); |
| 489 | } |
| 490 | |
| 491 | bool RISCVPassConfig::addRegBankSelect() { |
| 492 | addPass(P: new RegBankSelectLegacy()); |
| 493 | return false; |
| 494 | } |
| 495 | |
| 496 | bool RISCVPassConfig::addGlobalInstructionSelect() { |
| 497 | addPass(P: new InstructionSelectLegacy(getOptLevel())); |
| 498 | return false; |
| 499 | } |
| 500 | |
| 501 | void RISCVPassConfig::addPreSched2() { |
| 502 | addPass(P: createRISCVExpandPseudoPostRALegacyPass()); |
| 503 | |
| 504 | // Emit KCFI checks for indirect calls. |
| 505 | addPass(P: createKCFIPass()); |
| 506 | if (TM->getOptLevel() != CodeGenOptLevel::None) |
| 507 | addPass(P: createRISCVLoadStoreOptLegacyPass()); |
| 508 | } |
| 509 | |
| 510 | void RISCVPassConfig::addPreEmitPass() { |
| 511 | // TODO: It would potentially be better to schedule copy propagation after |
| 512 | // expanding pseudos (in addPreEmitPass2). However, performing copy |
| 513 | // propagation after the machine outliner (which runs after addPreEmitPass) |
| 514 | // currently leads to incorrect code-gen, where copies to registers within |
| 515 | // outlined functions are removed erroneously. |
| 516 | if (TM->getOptLevel() >= CodeGenOptLevel::Default && |
| 517 | CLOpts.enable_copy_propagation) |
| 518 | addPass(P: createMachineCopyPropagationPass(UseCopyInstr: true)); |
| 519 | if (TM->getOptLevel() >= CodeGenOptLevel::Default) |
| 520 | addPass(P: createRISCVLateBranchOptLegacyPass()); |
| 521 | // The IndirectBranchTrackingPass inserts lpad and could have changed the |
| 522 | // basic block alignment. It must be done before Branch Relaxation to |
| 523 | // prevent the adjusted offset exceeding the branch range. |
| 524 | addPass(P: createRISCVIndirectBranchTrackingLegacyPass()); |
| 525 | addPass(PassID: &BranchRelaxationPassID); |
| 526 | addPass(P: createRISCVMakeCompressibleLegacyPass()); |
| 527 | } |
| 528 | |
| 529 | void RISCVPassConfig::addPreEmitPass2() { |
| 530 | if (TM->getOptLevel() != CodeGenOptLevel::None) { |
| 531 | addPass(P: createRISCVMoveMergeLegacyPass()); |
| 532 | // Schedule PushPop Optimization before expansion of Pseudo instruction, |
| 533 | // ensuring return instruction is detected correctly. |
| 534 | addPass(P: createRISCVPushPopOptLegacyPass()); |
| 535 | } |
| 536 | addPass(P: createRISCVExpandPseudoPreEmitLegacyPass()); |
| 537 | |
| 538 | // Add QC Relaxation Markers as late as possible, and only for RV32 |
| 539 | if (TM->getOptLevel() != CodeGenOptLevel::None && |
| 540 | TM->getTargetTriple().isRISCV32()) |
| 541 | addPass(P: createRISCVQCRelaxMarkingLegacyPass()); |
| 542 | |
| 543 | // Schedule the expansion of AMOs at the last possible moment, avoiding the |
| 544 | // possibility for other passes to break the requirements for forward |
| 545 | // progress in the LR/SC block. |
| 546 | addPass(P: createRISCVExpandPseudoAtomicsLegacyPass()); |
| 547 | |
| 548 | // KCFI indirect call checks are lowered to a bundle. |
| 549 | addPass(P: createUnpackMachineBundlesLegacy(Ftor: [&](const MachineFunction &MF) { |
| 550 | return MF.getFunction().getParent()->getModuleFlag(Key: "kcfi" ); |
| 551 | })); |
| 552 | |
| 553 | if (CLOpts.enable_cfi_instr_inserter) |
| 554 | addPass(P: createCFIInstrInserterLegacy()); |
| 555 | } |
| 556 | |
| 557 | void RISCVPassConfig::addMachineSSAOptimization() { |
| 558 | // It's beneficial to reduce the VL to enable more |
| 559 | // Machine SSA optimizations. |
| 560 | if (TM->getOptLevel() != CodeGenOptLevel::None) { |
| 561 | // RISCVVLOptimizer can make loop invariant instructions like vmv.v.i |
| 562 | // loop variant by propagating a VL defined inside the loop. Run LICM and |
| 563 | // hoist them early. Don't do this at -O0 to avoid the compile-time |
| 564 | // overhead. Not reducing the VL of loop invariant pseudos results in more |
| 565 | // vsetvli toggles, and still requires the MachineLoopInfo analysis to be |
| 566 | // run. |
| 567 | addPass(PassID: &EarlyMachineLICMID); |
| 568 | addPass(P: createRISCVVLOptimizerLegacyPass()); |
| 569 | } |
| 570 | |
| 571 | addPass(P: createRISCVVectorPeepholeLegacyPass()); |
| 572 | addPass(P: createRISCVFoldMemOffsetLegacyPass()); |
| 573 | |
| 574 | TargetPassConfig::addMachineSSAOptimization(); |
| 575 | |
| 576 | if (TM->getTargetTriple().isRISCV64()) { |
| 577 | addPass(P: createRISCVOptWInstrsLegacyPass()); |
| 578 | } |
| 579 | } |
| 580 | |
| 581 | void RISCVPassConfig::addPreRegAlloc() { |
| 582 | addPass(P: createRISCVExpandPseudoPreRALegacyPass()); |
| 583 | if (TM->getOptLevel() != CodeGenOptLevel::None) { |
| 584 | addPass(P: createRISCVMergeBaseOffsetLegacyPass()); |
| 585 | addPass(P: createRISCVPreAllocZilsdOptLegacyPass()); |
| 586 | } |
| 587 | |
| 588 | addPass(P: createRISCVInsertReadWriteCSRLegacyPass()); |
| 589 | addPass(P: createRISCVInsertWriteVXRMLegacyPass()); |
| 590 | addPass(P: createRISCVLandingPadSetupLegacyPass()); |
| 591 | addPass(P: createRISCVVMV0EliminationLegacyPass()); |
| 592 | } |
| 593 | |
| 594 | void RISCVPassConfig::addFastRegAlloc() { |
| 595 | addPass(PassID: &InitUndefID); |
| 596 | TargetPassConfig::addFastRegAlloc(); |
| 597 | } |
| 598 | |
| 599 | |
| 600 | void RISCVPassConfig::addPostRegAlloc() { |
| 601 | if (TM->getOptLevel() != CodeGenOptLevel::None && CLOpts.enable_copyelim) |
| 602 | addPass(P: createRISCVRedundantCopyEliminationLegacyPass()); |
| 603 | } |
| 604 | |
| 605 | bool RISCVPassConfig::addILPOpts() { |
| 606 | if (CLOpts.enable_machine_combiner) |
| 607 | addPass(PassID: &MachineCombinerID); |
| 608 | |
| 609 | return true; |
| 610 | } |
| 611 | |
| 612 | yaml::MachineFunctionInfo * |
| 613 | RISCVTargetMachine::createDefaultFuncInfoYAML() const { |
| 614 | return new yaml::RISCVMachineFunctionInfo(); |
| 615 | } |
| 616 | |
| 617 | yaml::MachineFunctionInfo * |
| 618 | RISCVTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const { |
| 619 | const auto *MFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
| 620 | return new yaml::RISCVMachineFunctionInfo(*MFI); |
| 621 | } |
| 622 | |
| 623 | bool RISCVTargetMachine::parseMachineFunctionInfo( |
| 624 | const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS, |
| 625 | SMDiagnostic &Error, SMRange &SourceRange) const { |
| 626 | const auto &YamlMFI = |
| 627 | static_cast<const yaml::RISCVMachineFunctionInfo &>(MFI); |
| 628 | PFS.MF.getInfo<RISCVMachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI); |
| 629 | return false; |
| 630 | } |
| 631 | |