1//===-- RISCVTargetMachine.cpp - Define TargetMachine for RISC-V ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Implements the info about RISC-V target spec.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVTargetMachine.h"
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "RISCV.h"
16#include "RISCVMachineFunctionInfo.h"
17#include "RISCVMachineScheduler.h"
18#include "RISCVTargetObjectFile.h"
19#include "RISCVTargetTransformInfo.h"
20#include "TargetInfo/RISCVTargetInfo.h"
21#include "llvm/Analysis/TargetTransformInfo.h"
22#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
23#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
24#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
25#include "llvm/CodeGen/GlobalISel/Legalizer.h"
26#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
27#include "llvm/CodeGen/MIRParser/MIParser.h"
28#include "llvm/CodeGen/MIRYamlMapping.h"
29#include "llvm/CodeGen/MachineScheduler.h"
30#include "llvm/CodeGen/MacroFusion.h"
31#include "llvm/CodeGen/Passes.h"
32#include "llvm/CodeGen/RegAllocRegistry.h"
33#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
34#include "llvm/CodeGen/TargetPassConfig.h"
35#include "llvm/InitializePasses.h"
36#include "llvm/MC/TargetRegistry.h"
37#include "llvm/Passes/PassBuilder.h"
38#include "llvm/Support/Compiler.h"
39#include "llvm/Target/TargetOptions.h"
40#include "llvm/Transforms/IPO.h"
41#include "llvm/Transforms/Scalar.h"
42#include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h"
43#include <optional>
44using namespace llvm;
45
46static cl::opt<bool> EnableRedundantCopyElimination(
47 "riscv-enable-copyelim",
48 cl::desc("Enable the redundant copy elimination pass"), cl::init(Val: true),
49 cl::Hidden);
50
51// FIXME: Unify control over GlobalMerge.
52static cl::opt<cl::boolOrDefault>
53 EnableGlobalMerge("riscv-enable-global-merge", cl::Hidden,
54 cl::desc("Enable the global merge pass"));
55
56static cl::opt<bool>
57 EnableMachineCombiner("riscv-enable-machine-combiner",
58 cl::desc("Enable the machine combiner pass"),
59 cl::init(Val: true), cl::Hidden);
60
61static cl::opt<unsigned> RVVVectorBitsMaxOpt(
62 "riscv-v-vector-bits-max",
63 cl::desc("Assume V extension vector registers are at most this big, "
64 "with zero meaning no maximum size is assumed."),
65 cl::init(Val: 0), cl::Hidden);
66
67static cl::opt<int> RVVVectorBitsMinOpt(
68 "riscv-v-vector-bits-min",
69 cl::desc("Assume V extension vector registers are at least this big, "
70 "with zero meaning no minimum size is assumed. A value of -1 "
71 "means use Zvl*b extension. This is primarily used to enable "
72 "autovectorization with fixed width vectors."),
73 cl::init(Val: -1), cl::Hidden);
74
75static cl::opt<bool> EnableRISCVCopyPropagation(
76 "riscv-enable-copy-propagation",
77 cl::desc("Enable the copy propagation with RISC-V copy instr"),
78 cl::init(Val: true), cl::Hidden);
79
80static cl::opt<bool> EnableRISCVDeadRegisterElimination(
81 "riscv-enable-dead-defs", cl::Hidden,
82 cl::desc("Enable the pass that removes dead"
83 " definitions and replaces stores to"
84 " them with stores to x0"),
85 cl::init(Val: true));
86
87static cl::opt<bool>
88 EnableSinkFold("riscv-enable-sink-fold",
89 cl::desc("Enable sinking and folding of instruction copies"),
90 cl::init(Val: true), cl::Hidden);
91
92static cl::opt<bool>
93 EnableLoopDataPrefetch("riscv-enable-loop-data-prefetch", cl::Hidden,
94 cl::desc("Enable the loop data prefetch pass"),
95 cl::init(Val: true));
96
97static cl::opt<bool> DisableVectorMaskMutation(
98 "riscv-disable-vector-mask-mutation",
99 cl::desc("Disable the vector mask scheduling mutation"), cl::init(Val: false),
100 cl::Hidden);
101
102static cl::opt<bool>
103 EnableMachinePipeliner("riscv-enable-pipeliner",
104 cl::desc("Enable Machine Pipeliner for RISC-V"),
105 cl::init(Val: false), cl::Hidden);
106
107static cl::opt<bool> EnableCFIInstrInserter(
108 "riscv-enable-cfi-instr-inserter",
109 cl::desc("Enable CFI Instruction Inserter for RISC-V"), cl::init(Val: false),
110 cl::Hidden);
111
112static cl::opt<bool>
113 EnableSelectOpt("riscv-select-opt", cl::Hidden,
114 cl::desc("Enable select to branch optimizations"),
115 cl::init(Val: true));
116
117extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
118 RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
119 RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
120 RegisterTargetMachine<RISCVTargetMachine> A(getTheRISCV32beTarget());
121 RegisterTargetMachine<RISCVTargetMachine> B(getTheRISCV64beTarget());
122 auto *PR = PassRegistry::getPassRegistry();
123 initializeGlobalISel(*PR);
124 initializeRISCVO0PreLegalizerCombinerPass(*PR);
125 initializeRISCVPreLegalizerCombinerPass(*PR);
126 initializeRISCVPostLegalizerCombinerPass(*PR);
127 initializeMachineKCFILegacyPass(*PR);
128 initializeRISCVDeadRegisterDefinitionsPass(*PR);
129 initializeRISCVLateBranchOptPass(*PR);
130 initializeRISCVMakeCompressibleOptPass(*PR);
131 initializeRISCVQCRelaxMarkingPass(*PR);
132 initializeRISCVGatherScatterLoweringPass(*PR);
133 initializeRISCVCodeGenPrepareLegacyPassPass(*PR);
134 initializeRISCVPostRAExpandPseudoPass(*PR);
135 initializeRISCVMergeBaseOffsetOptPass(*PR);
136 initializeRISCVOptWInstrsPass(*PR);
137 initializeRISCVFoldMemOffsetPass(*PR);
138 initializeRISCVPreRAExpandPseudoPass(*PR);
139 initializeRISCVExpandPseudoPass(*PR);
140 initializeRISCVVectorPeepholePass(*PR);
141 initializeRISCVVLOptimizerPass(*PR);
142 initializeRISCVVMV0EliminationPass(*PR);
143 initializeRISCVInsertVSETVLIPass(*PR);
144 initializeRISCVInsertReadWriteCSRPass(*PR);
145 initializeRISCVInsertWriteVXRMPass(*PR);
146 initializeRISCVDAGToDAGISelLegacyPass(*PR);
147 initializeRISCVMoveMergePass(*PR);
148 initializeRISCVPushPopOptPass(*PR);
149 initializeRISCVIndirectBranchTrackingPass(*PR);
150 initializeRISCVLoadStoreOptPass(*PR);
151 initializeRISCVPreAllocZilsdOptPass(*PR);
152 initializeRISCVExpandAtomicPseudoPass(*PR);
153 initializeRISCVRedundantCopyEliminationPass(*PR);
154 initializeRISCVAsmPrinterPass(*PR);
155 initializeRISCVPromoteConstantPass(*PR);
156}
157
158static Reloc::Model getEffectiveRelocModel(const Triple &TT,
159 std::optional<Reloc::Model> RM) {
160 if (TT.isOSBinFormatMachO())
161 return RM.value_or(u: Reloc::PIC_);
162
163 return RM.value_or(u: Reloc::Static);
164}
165
166static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
167 if (TT.isOSBinFormatMachO())
168 return std::make_unique<RISCVMachOTargetObjectFile>();
169 return std::make_unique<RISCVELFTargetObjectFile>();
170}
171
172RISCVTargetMachine::RISCVTargetMachine(const Target &T, const Triple &TT,
173 StringRef CPU, StringRef FS,
174 const TargetOptions &Options,
175 std::optional<Reloc::Model> RM,
176 std::optional<CodeModel::Model> CM,
177 CodeGenOptLevel OL, bool JIT)
178 : CodeGenTargetMachineImpl(
179 T, TT.computeDataLayout(ABIName: Options.MCOptions.getABIName()), TT, CPU, FS,
180 Options, getEffectiveRelocModel(TT, RM),
181 getEffectiveCodeModel(CM, Default: CodeModel::Small), OL),
182 TLOF(createTLOF(TT)) {
183 initAsmInfo();
184
185 // RISC-V supports the MachineOutliner.
186 setMachineOutliner(true);
187 setSupportsDefaultOutlining(true);
188
189 // RISC-V supports the debug entry values.
190 setSupportsDebugEntryValues(true);
191
192 if (TT.isOSFuchsia() && !TT.isArch64Bit())
193 report_fatal_error(reason: "Fuchsia is only supported for 64-bit");
194
195 setCFIFixup(!EnableCFIInstrInserter);
196}
197
198const RISCVSubtarget *
199RISCVTargetMachine::getSubtargetImpl(const Function &F) const {
200 Attribute CPUAttr = F.getFnAttribute(Kind: "target-cpu");
201 Attribute TuneAttr = F.getFnAttribute(Kind: "tune-cpu");
202 Attribute FSAttr = F.getFnAttribute(Kind: "target-features");
203
204 std::string CPU =
205 CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU;
206 std::string TuneCPU =
207 TuneAttr.isValid() ? TuneAttr.getValueAsString().str() : CPU;
208 std::string FS =
209 FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS;
210
211 unsigned RVVBitsMin = RVVVectorBitsMinOpt;
212 unsigned RVVBitsMax = RVVVectorBitsMaxOpt;
213
214 Attribute VScaleRangeAttr = F.getFnAttribute(Kind: Attribute::VScaleRange);
215 if (VScaleRangeAttr.isValid()) {
216 if (!RVVVectorBitsMinOpt.getNumOccurrences())
217 RVVBitsMin = VScaleRangeAttr.getVScaleRangeMin() * RISCV::RVVBitsPerBlock;
218 std::optional<unsigned> VScaleMax = VScaleRangeAttr.getVScaleRangeMax();
219 if (VScaleMax.has_value() && !RVVVectorBitsMaxOpt.getNumOccurrences())
220 RVVBitsMax = *VScaleMax * RISCV::RVVBitsPerBlock;
221 }
222
223 if (RVVBitsMin != -1U) {
224 // FIXME: Change to >= 32 when VLEN = 32 is supported.
225 assert((RVVBitsMin == 0 || (RVVBitsMin >= 64 && RVVBitsMin <= 65536 &&
226 isPowerOf2_32(RVVBitsMin))) &&
227 "V or Zve* extension requires vector length to be in the range of "
228 "64 to 65536 and a power 2!");
229 assert((RVVBitsMax >= RVVBitsMin || RVVBitsMax == 0) &&
230 "Minimum V extension vector length should not be larger than its "
231 "maximum!");
232 }
233 assert((RVVBitsMax == 0 || (RVVBitsMax >= 64 && RVVBitsMax <= 65536 &&
234 isPowerOf2_32(RVVBitsMax))) &&
235 "V or Zve* extension requires vector length to be in the range of "
236 "64 to 65536 and a power 2!");
237
238 if (RVVBitsMin != -1U) {
239 if (RVVBitsMax != 0) {
240 RVVBitsMin = std::min(a: RVVBitsMin, b: RVVBitsMax);
241 RVVBitsMax = std::max(a: RVVBitsMin, b: RVVBitsMax);
242 }
243
244 RVVBitsMin = llvm::bit_floor(
245 Value: (RVVBitsMin < 64 || RVVBitsMin > 65536) ? 0 : RVVBitsMin);
246 }
247 RVVBitsMax =
248 llvm::bit_floor(Value: (RVVBitsMax < 64 || RVVBitsMax > 65536) ? 0 : RVVBitsMax);
249
250 SmallString<512> Key;
251 raw_svector_ostream(Key) << "RVVMin" << RVVBitsMin << "RVVMax" << RVVBitsMax
252 << CPU << TuneCPU << FS;
253 auto &I = SubtargetMap[Key];
254 if (!I) {
255 auto ABIName = Options.MCOptions.getABIName();
256 if (const MDString *ModuleTargetABI = dyn_cast_or_null<MDString>(
257 Val: F.getParent()->getModuleFlag(Key: "target-abi"))) {
258 auto TargetABI = RISCVABI::getTargetABI(ABIName);
259 if (TargetABI != RISCVABI::ABI_Unknown &&
260 ModuleTargetABI->getString() != ABIName) {
261 report_fatal_error(reason: "-target-abi option != target-abi module flag");
262 }
263 ABIName = ModuleTargetABI->getString();
264 }
265 I = std::make_unique<RISCVSubtarget>(
266 args: TargetTriple, args&: CPU, args&: TuneCPU, args&: FS, args&: ABIName, args&: RVVBitsMin, args&: RVVBitsMax, args: *this);
267 }
268 return I.get();
269}
270
271MachineFunctionInfo *RISCVTargetMachine::createMachineFunctionInfo(
272 BumpPtrAllocator &Allocator, const Function &F,
273 const TargetSubtargetInfo *STI) const {
274 return RISCVMachineFunctionInfo::create<RISCVMachineFunctionInfo>(
275 Allocator, F, STI: static_cast<const RISCVSubtarget *>(STI));
276}
277
278TargetTransformInfo
279RISCVTargetMachine::getTargetTransformInfo(const Function &F) const {
280 return TargetTransformInfo(std::make_unique<RISCVTTIImpl>(args: this, args: F));
281}
282
283// A RISC-V hart has a single byte-addressable address space of 2^XLEN bytes
284// for all memory accesses, so it is reasonable to assume that an
285// implementation has no-op address space casts. If an implementation makes a
286// change to this, they can override it here.
287bool RISCVTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
288 unsigned DstAS) const {
289 return true;
290}
291
292ScheduleDAGInstrs *
293RISCVTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
294 const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
295 ScheduleDAGMILive *DAG = createSchedLive<RISCVPreRAMachineSchedStrategy>(C);
296
297 // Add MacroFusion mutation first with a higher priority than later clustering
298 const auto &MacroFusions = ST.getMacroFusions();
299 if (!MacroFusions.empty())
300 DAG->addMutation(Mutation: createMacroFusionDAGMutation(Predicates: MacroFusions));
301
302 if (ST.enableMISchedLoadClustering())
303 DAG->addMutation(Mutation: createLoadClusterDAGMutation(
304 TII: DAG->TII, TRI: DAG->TRI, /*ReorderWhileClustering=*/true));
305
306 if (ST.enableMISchedStoreClustering())
307 DAG->addMutation(Mutation: createStoreClusterDAGMutation(
308 TII: DAG->TII, TRI: DAG->TRI, /*ReorderWhileClustering=*/true));
309
310 if (!DisableVectorMaskMutation && ST.hasVInstructions())
311 DAG->addMutation(Mutation: createRISCVVectorMaskDAGMutation(TRI: DAG->TRI));
312
313 return DAG;
314}
315
316ScheduleDAGInstrs *
317RISCVTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
318 const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
319 ScheduleDAGMI *DAG = createSchedPostRA(C);
320
321 // Add MacroFusion mutation first with a higher priority than later clustering
322 const auto &MacroFusions = ST.getMacroFusions();
323 if (!MacroFusions.empty())
324 DAG->addMutation(Mutation: createMacroFusionDAGMutation(Predicates: MacroFusions));
325
326 if (ST.enablePostMISchedLoadClustering())
327 DAG->addMutation(Mutation: createLoadClusterDAGMutation(
328 TII: DAG->TII, TRI: DAG->TRI, /*ReorderWhileClustering=*/true));
329
330 if (ST.enablePostMISchedStoreClustering())
331 DAG->addMutation(Mutation: createStoreClusterDAGMutation(
332 TII: DAG->TII, TRI: DAG->TRI, /*ReorderWhileClustering=*/true));
333
334 return DAG;
335}
336
337namespace {
338
339class RVVRegisterRegAlloc : public RegisterRegAllocBase<RVVRegisterRegAlloc> {
340public:
341 RVVRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
342 : RegisterRegAllocBase(N, D, C) {}
343};
344
345static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI,
346 const MachineRegisterInfo &MRI,
347 const Register Reg) {
348 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
349 return RISCVRegisterInfo::isRVVRegClass(RC);
350}
351
352static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
353
354static llvm::once_flag InitializeDefaultRVVRegisterAllocatorFlag;
355
356/// -riscv-rvv-regalloc=<fast|basic|greedy> command line option.
357/// This option could designate the rvv register allocator only.
358/// For example: -riscv-rvv-regalloc=basic
359static cl::opt<RVVRegisterRegAlloc::FunctionPassCtor, false,
360 RegisterPassParser<RVVRegisterRegAlloc>>
361 RVVRegAlloc("riscv-rvv-regalloc", cl::Hidden,
362 cl::init(Val: &useDefaultRegisterAllocator),
363 cl::desc("Register allocator to use for RVV register."));
364
365static void initializeDefaultRVVRegisterAllocatorOnce() {
366 RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
367
368 if (!Ctor) {
369 Ctor = RVVRegAlloc;
370 RVVRegisterRegAlloc::setDefault(RVVRegAlloc);
371 }
372}
373
374static FunctionPass *createBasicRVVRegisterAllocator() {
375 return createBasicRegisterAllocator(F: onlyAllocateRVVReg);
376}
377
378static FunctionPass *createGreedyRVVRegisterAllocator() {
379 return createGreedyRegisterAllocator(F: onlyAllocateRVVReg);
380}
381
382static FunctionPass *createFastRVVRegisterAllocator() {
383 return createFastRegisterAllocator(F: onlyAllocateRVVReg, ClearVirtRegs: false);
384}
385
386static RVVRegisterRegAlloc basicRegAllocRVVReg("basic",
387 "basic register allocator",
388 createBasicRVVRegisterAllocator);
389static RVVRegisterRegAlloc
390 greedyRegAllocRVVReg("greedy", "greedy register allocator",
391 createGreedyRVVRegisterAllocator);
392
393static RVVRegisterRegAlloc fastRegAllocRVVReg("fast", "fast register allocator",
394 createFastRVVRegisterAllocator);
395
396class RISCVPassConfig : public TargetPassConfig {
397public:
398 RISCVPassConfig(RISCVTargetMachine &TM, PassManagerBase &PM)
399 : TargetPassConfig(TM, PM) {
400 if (TM.getOptLevel() != CodeGenOptLevel::None)
401 substitutePass(StandardID: &PostRASchedulerID, TargetID: &PostMachineSchedulerID);
402 setEnableSinkAndFold(EnableSinkFold);
403 EnableLoopTermFold = true;
404 }
405
406 RISCVTargetMachine &getRISCVTargetMachine() const {
407 return getTM<RISCVTargetMachine>();
408 }
409
410 void addIRPasses() override;
411 bool addPreISel() override;
412 void addCodeGenPrepare() override;
413 bool addInstSelector() override;
414 bool addIRTranslator() override;
415 void addPreLegalizeMachineIR() override;
416 bool addLegalizeMachineIR() override;
417 void addPreRegBankSelect() override;
418 bool addRegBankSelect() override;
419 bool addGlobalInstructionSelect() override;
420 void addPreEmitPass() override;
421 void addPreEmitPass2() override;
422 void addPreSched2() override;
423 void addMachineSSAOptimization() override;
424 FunctionPass *createRVVRegAllocPass(bool Optimized);
425 bool addRegAssignAndRewriteFast() override;
426 bool addRegAssignAndRewriteOptimized() override;
427 void addPreRegAlloc() override;
428 void addPostRegAlloc() override;
429 void addFastRegAlloc() override;
430 bool addILPOpts() override;
431
432 std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
433};
434} // namespace
435
436TargetPassConfig *RISCVTargetMachine::createPassConfig(PassManagerBase &PM) {
437 return new RISCVPassConfig(*this, PM);
438}
439
440std::unique_ptr<CSEConfigBase> RISCVPassConfig::getCSEConfig() const {
441 return getStandardCSEConfigForOpt(Level: TM->getOptLevel());
442}
443
444FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {
445 // Initialize the global default.
446 llvm::call_once(flag&: InitializeDefaultRVVRegisterAllocatorFlag,
447 F&: initializeDefaultRVVRegisterAllocatorOnce);
448
449 RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
450 if (Ctor != useDefaultRegisterAllocator)
451 return Ctor();
452
453 if (Optimized)
454 return createGreedyRVVRegisterAllocator();
455
456 return createFastRVVRegisterAllocator();
457}
458
459bool RISCVPassConfig::addRegAssignAndRewriteFast() {
460 addPass(P: createRVVRegAllocPass(Optimized: false));
461 addPass(P: createRISCVInsertVSETVLIPass());
462 if (TM->getOptLevel() != CodeGenOptLevel::None &&
463 EnableRISCVDeadRegisterElimination)
464 addPass(P: createRISCVDeadRegisterDefinitionsPass());
465 return TargetPassConfig::addRegAssignAndRewriteFast();
466}
467
468bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {
469 addPass(P: createRVVRegAllocPass(Optimized: true));
470 addPass(P: createVirtRegRewriter(ClearVirtRegs: false));
471 addPass(P: createRISCVInsertVSETVLIPass());
472 if (TM->getOptLevel() != CodeGenOptLevel::None &&
473 EnableRISCVDeadRegisterElimination)
474 addPass(P: createRISCVDeadRegisterDefinitionsPass());
475 return TargetPassConfig::addRegAssignAndRewriteOptimized();
476}
477
478void RISCVPassConfig::addIRPasses() {
479 addPass(P: createAtomicExpandLegacyPass());
480 addPass(P: createRISCVZacasABIFixPass());
481
482 if (getOptLevel() != CodeGenOptLevel::None) {
483 if (EnableLoopDataPrefetch)
484 addPass(P: createLoopDataPrefetchPass());
485
486 addPass(P: createRISCVGatherScatterLoweringPass());
487 addPass(P: createInterleavedAccessPass());
488 addPass(P: createRISCVCodeGenPrepareLegacyPass());
489 }
490
491 TargetPassConfig::addIRPasses();
492
493 if (getOptLevel() == CodeGenOptLevel::Aggressive && EnableSelectOpt)
494 addPass(P: createSelectOptimizePass());
495}
496
497bool RISCVPassConfig::addPreISel() {
498 if (TM->getOptLevel() != CodeGenOptLevel::None)
499 addPass(P: createRISCVPromoteConstantPass());
500 if (TM->getOptLevel() != CodeGenOptLevel::None) {
501 // Add a barrier before instruction selection so that we will not get
502 // deleted block address after enabling default outlining. See D99707 for
503 // more details.
504 addPass(P: createBarrierNoopPass());
505 }
506
507 if ((TM->getOptLevel() != CodeGenOptLevel::None &&
508 EnableGlobalMerge == cl::boolOrDefault::BOU_UNSET) ||
509 EnableGlobalMerge == cl::boolOrDefault::BOU_TRUE) {
510 // FIXME: Like AArch64, we disable extern global merging by default due to
511 // concerns it might regress some workloads. Unlike AArch64, we don't
512 // currently support enabling the pass in an "OnlyOptimizeForSize" mode.
513 // Investigating and addressing both items are TODO.
514 addPass(P: createGlobalMergePass(TM, /* MaxOffset */ MaximalOffset: 2047,
515 /* OnlyOptimizeForSize */ false,
516 /* MergeExternalByDefault */ true));
517 }
518
519 return false;
520}
521
522void RISCVPassConfig::addCodeGenPrepare() {
523 if (getOptLevel() != CodeGenOptLevel::None)
524 addPass(P: createTypePromotionLegacyPass());
525 TargetPassConfig::addCodeGenPrepare();
526}
527
528bool RISCVPassConfig::addInstSelector() {
529 addPass(P: createRISCVISelDag(TM&: getRISCVTargetMachine(), OptLevel: getOptLevel()));
530
531 return false;
532}
533
534bool RISCVPassConfig::addIRTranslator() {
535 addPass(P: new IRTranslator(getOptLevel()));
536 return false;
537}
538
539void RISCVPassConfig::addPreLegalizeMachineIR() {
540 if (getOptLevel() == CodeGenOptLevel::None) {
541 addPass(P: createRISCVO0PreLegalizerCombiner());
542 } else {
543 addPass(P: createRISCVPreLegalizerCombiner());
544 }
545}
546
547bool RISCVPassConfig::addLegalizeMachineIR() {
548 addPass(P: new Legalizer());
549 return false;
550}
551
552void RISCVPassConfig::addPreRegBankSelect() {
553 if (getOptLevel() != CodeGenOptLevel::None)
554 addPass(P: createRISCVPostLegalizerCombiner());
555}
556
557bool RISCVPassConfig::addRegBankSelect() {
558 addPass(P: new RegBankSelect());
559 return false;
560}
561
562bool RISCVPassConfig::addGlobalInstructionSelect() {
563 addPass(P: new InstructionSelect(getOptLevel()));
564 return false;
565}
566
567void RISCVPassConfig::addPreSched2() {
568 addPass(P: createRISCVPostRAExpandPseudoPass());
569
570 // Emit KCFI checks for indirect calls.
571 addPass(P: createKCFIPass());
572 if (TM->getOptLevel() != CodeGenOptLevel::None)
573 addPass(P: createRISCVLoadStoreOptPass());
574}
575
576void RISCVPassConfig::addPreEmitPass() {
577 // TODO: It would potentially be better to schedule copy propagation after
578 // expanding pseudos (in addPreEmitPass2). However, performing copy
579 // propagation after the machine outliner (which runs after addPreEmitPass)
580 // currently leads to incorrect code-gen, where copies to registers within
581 // outlined functions are removed erroneously.
582 if (TM->getOptLevel() >= CodeGenOptLevel::Default &&
583 EnableRISCVCopyPropagation)
584 addPass(P: createMachineCopyPropagationPass(UseCopyInstr: true));
585 if (TM->getOptLevel() >= CodeGenOptLevel::Default)
586 addPass(P: createRISCVLateBranchOptPass());
587 // The IndirectBranchTrackingPass inserts lpad and could have changed the
588 // basic block alignment. It must be done before Branch Relaxation to
589 // prevent the adjusted offset exceeding the branch range.
590 addPass(P: createRISCVIndirectBranchTrackingPass());
591 addPass(PassID: &BranchRelaxationPassID);
592 addPass(P: createRISCVMakeCompressibleOptPass());
593}
594
595void RISCVPassConfig::addPreEmitPass2() {
596 if (TM->getOptLevel() != CodeGenOptLevel::None) {
597 addPass(P: createRISCVMoveMergePass());
598 // Schedule PushPop Optimization before expansion of Pseudo instruction,
599 // ensuring return instruction is detected correctly.
600 addPass(P: createRISCVPushPopOptimizationPass());
601 }
602 addPass(P: createRISCVExpandPseudoPass());
603
604 // Add QC Relaxation Markers as late as possible, and only for RV32
605 if (TM->getOptLevel() != CodeGenOptLevel::None &&
606 TM->getTargetTriple().isRISCV32())
607 addPass(P: createRISCVQCRelaxMarkingPass());
608
609 // Schedule the expansion of AMOs at the last possible moment, avoiding the
610 // possibility for other passes to break the requirements for forward
611 // progress in the LR/SC block.
612 addPass(P: createRISCVExpandAtomicPseudoPass());
613
614 // KCFI indirect call checks are lowered to a bundle.
615 addPass(P: createUnpackMachineBundlesLegacy(Ftor: [&](const MachineFunction &MF) {
616 return MF.getFunction().getParent()->getModuleFlag(Key: "kcfi");
617 }));
618
619 if (EnableCFIInstrInserter)
620 addPass(P: createCFIInstrInserter());
621}
622
623void RISCVPassConfig::addMachineSSAOptimization() {
624 // It's beneficial to reduce the VL to enable more
625 // Machine SSA optimizations.
626 if (TM->getOptLevel() != CodeGenOptLevel::None)
627 addPass(P: createRISCVVLOptimizerPass());
628
629 addPass(P: createRISCVVectorPeepholePass());
630 addPass(P: createRISCVFoldMemOffsetPass());
631
632 TargetPassConfig::addMachineSSAOptimization();
633
634 if (TM->getTargetTriple().isRISCV64()) {
635 addPass(P: createRISCVOptWInstrsPass());
636 }
637}
638
639void RISCVPassConfig::addPreRegAlloc() {
640 addPass(P: createRISCVPreRAExpandPseudoPass());
641 if (TM->getOptLevel() != CodeGenOptLevel::None) {
642 addPass(P: createRISCVMergeBaseOffsetOptPass());
643 // Add Zilsd pre-allocation load/store optimization
644 addPass(P: createRISCVPreAllocZilsdOptPass());
645 }
646
647 addPass(P: createRISCVInsertReadWriteCSRPass());
648 addPass(P: createRISCVInsertWriteVXRMPass());
649 addPass(P: createRISCVLandingPadSetupPass());
650
651 if (TM->getOptLevel() != CodeGenOptLevel::None && EnableMachinePipeliner)
652 addPass(PassID: &MachinePipelinerID);
653
654 addPass(P: createRISCVVMV0EliminationPass());
655}
656
657void RISCVPassConfig::addFastRegAlloc() {
658 addPass(PassID: &InitUndefID);
659 TargetPassConfig::addFastRegAlloc();
660}
661
662
663void RISCVPassConfig::addPostRegAlloc() {
664 if (TM->getOptLevel() != CodeGenOptLevel::None &&
665 EnableRedundantCopyElimination)
666 addPass(P: createRISCVRedundantCopyEliminationPass());
667}
668
669bool RISCVPassConfig::addILPOpts() {
670 if (EnableMachineCombiner)
671 addPass(PassID: &MachineCombinerID);
672
673 return true;
674}
675
676void RISCVTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
677#define GET_PASS_REGISTRY "RISCVPassRegistry.def"
678#include "llvm/Passes/TargetPassRegistry.inc"
679
680 PB.registerLateLoopOptimizationsEPCallback(C: [=](LoopPassManager &LPM,
681 OptimizationLevel Level) {
682 if (Level != OptimizationLevel::O0)
683 LPM.addPass(Pass: LoopIdiomVectorizePass(LoopIdiomVectorizeStyle::Predicated));
684 });
685}
686
687yaml::MachineFunctionInfo *
688RISCVTargetMachine::createDefaultFuncInfoYAML() const {
689 return new yaml::RISCVMachineFunctionInfo();
690}
691
692yaml::MachineFunctionInfo *
693RISCVTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
694 const auto *MFI = MF.getInfo<RISCVMachineFunctionInfo>();
695 return new yaml::RISCVMachineFunctionInfo(*MFI);
696}
697
698bool RISCVTargetMachine::parseMachineFunctionInfo(
699 const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
700 SMDiagnostic &Error, SMRange &SourceRange) const {
701 const auto &YamlMFI =
702 static_cast<const yaml::RISCVMachineFunctionInfo &>(MFI);
703 PFS.MF.getInfo<RISCVMachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
704 return false;
705}
706