1//===-- RISCVTargetMachine.cpp - Define TargetMachine for RISC-V ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Implements the info about RISC-V target spec.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVTargetMachine.h"
14#include "RISCV.h"
15#include "RISCVMachineFunctionInfo.h"
16#include "RISCVMachineScheduler.h"
17#include "RISCVTargetObjectFile.h"
18#include "RISCVTargetTransformInfo.h"
19#include "TargetInfo/RISCVTargetInfo.h"
20#include "llvm/Analysis/TargetTransformInfo.h"
21#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
22#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
23#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
24#include "llvm/CodeGen/GlobalISel/Legalizer.h"
25#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
26#include "llvm/CodeGen/MIRParser/MIParser.h"
27#include "llvm/CodeGen/MIRYamlMapping.h"
28#include "llvm/CodeGen/MachineScheduler.h"
29#include "llvm/CodeGen/MacroFusion.h"
30#include "llvm/CodeGen/Passes.h"
31#include "llvm/CodeGen/RegAllocRegistry.h"
32#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
33#include "llvm/CodeGen/TargetPassConfig.h"
34#include "llvm/InitializePasses.h"
35#include "llvm/MC/TargetRegistry.h"
36#include "llvm/Support/Compiler.h"
37#include "llvm/Target/TargetOptions.h"
38#include "llvm/Transforms/IPO.h"
39#include "llvm/Transforms/Scalar.h"
40#include <optional>
41using namespace llvm;
42
43static cl::opt<bool> EnableRedundantCopyElimination(
44 "riscv-enable-copyelim",
45 cl::desc("Enable the redundant copy elimination pass"), cl::init(Val: true),
46 cl::Hidden);
47
48// FIXME: Unify control over GlobalMerge.
49static cl::opt<cl::boolOrDefault>
50 EnableGlobalMerge("riscv-enable-global-merge", cl::Hidden,
51 cl::desc("Enable the global merge pass"));
52
53static cl::opt<bool>
54 EnableMachineCombiner("riscv-enable-machine-combiner",
55 cl::desc("Enable the machine combiner pass"),
56 cl::init(Val: true), cl::Hidden);
57
58static cl::opt<unsigned> RVVVectorBitsMaxOpt(
59 "riscv-v-vector-bits-max",
60 cl::desc("Assume V extension vector registers are at most this big, "
61 "with zero meaning no maximum size is assumed."),
62 cl::init(Val: 0), cl::Hidden);
63
64static cl::opt<int> RVVVectorBitsMinOpt(
65 "riscv-v-vector-bits-min",
66 cl::desc("Assume V extension vector registers are at least this big, "
67 "with zero meaning no minimum size is assumed. A value of -1 "
68 "means use Zvl*b extension. This is primarily used to enable "
69 "autovectorization with fixed width vectors."),
70 cl::init(Val: -1), cl::Hidden);
71
72static cl::opt<bool> EnableRISCVCopyPropagation(
73 "riscv-enable-copy-propagation",
74 cl::desc("Enable the copy propagation with RISC-V copy instr"),
75 cl::init(Val: true), cl::Hidden);
76
77static cl::opt<bool> EnableRISCVDeadRegisterElimination(
78 "riscv-enable-dead-defs", cl::Hidden,
79 cl::desc("Enable the pass that removes dead"
80 " definitions and replaces stores to"
81 " them with stores to x0"),
82 cl::init(Val: true));
83
84static cl::opt<bool>
85 EnableSinkFold("riscv-enable-sink-fold",
86 cl::desc("Enable sinking and folding of instruction copies"),
87 cl::init(Val: true), cl::Hidden);
88
89static cl::opt<bool>
90 EnableLoopDataPrefetch("riscv-enable-loop-data-prefetch", cl::Hidden,
91 cl::desc("Enable the loop data prefetch pass"),
92 cl::init(Val: true));
93
94static cl::opt<bool> DisableVectorMaskMutation(
95 "riscv-disable-vector-mask-mutation",
96 cl::desc("Disable the vector mask scheduling mutation"), cl::init(Val: false),
97 cl::Hidden);
98
99static cl::opt<bool>
100 EnableMachinePipeliner("riscv-enable-pipeliner",
101 cl::desc("Enable Machine Pipeliner for RISC-V"),
102 cl::init(Val: false), cl::Hidden);
103
104static cl::opt<bool> EnableCFIInstrInserter(
105 "riscv-enable-cfi-instr-inserter",
106 cl::desc("Enable CFI Instruction Inserter for RISC-V"), cl::init(Val: false),
107 cl::Hidden);
108
109static cl::opt<bool>
110 EnableSelectOpt("riscv-select-opt", cl::Hidden,
111 cl::desc("Enable select to branch optimizations"),
112 cl::init(Val: true));
113
114extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
115 RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
116 RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
117 RegisterTargetMachine<RISCVTargetMachine> A(getTheRISCV32beTarget());
118 RegisterTargetMachine<RISCVTargetMachine> B(getTheRISCV64beTarget());
119 auto *PR = PassRegistry::getPassRegistry();
120 initializeGlobalISel(*PR);
121 initializeRISCVO0PreLegalizerCombinerPass(*PR);
122 initializeRISCVPreLegalizerCombinerPass(*PR);
123 initializeRISCVPostLegalizerCombinerPass(*PR);
124 initializeMachineKCFILegacyPass(*PR);
125 initializeRISCVDeadRegisterDefinitionsPass(*PR);
126 initializeRISCVLateBranchOptPass(*PR);
127 initializeRISCVMakeCompressibleOptPass(*PR);
128 initializeRISCVQCRelaxMarkingPass(*PR);
129 initializeRISCVGatherScatterLoweringLegacyPass(*PR);
130 initializeRISCVCodeGenPrepareLegacyPass(*PR);
131 initializeRISCVZacasABIFixLegacyPass(*PR);
132 initializeRISCVExpandPseudoPostRALegacyPass(*PR);
133 initializeRISCVMergeBaseOffsetOptPass(*PR);
134 initializeRISCVOptWInstrsLegacyPass(*PR);
135 initializeRISCVFoldMemOffsetLegacyPass(*PR);
136 initializeRISCVExpandPseudoPreRALegacyPass(*PR);
137 initializeRISCVExpandPseudoPreEmitLegacyPass(*PR);
138 initializeRISCVVectorPeepholeLegacyPass(*PR);
139 initializeRISCVVLOptimizerLegacyPass(*PR);
140 initializeRISCVVMV0EliminationPass(*PR);
141 initializeRISCVInsertVSETVLIPass(*PR);
142 initializeRISCVInsertReadWriteCSRPass(*PR);
143 initializeRISCVInsertWriteVXRMPass(*PR);
144 initializeRISCVDAGToDAGISelLegacyPass(*PR);
145 initializeRISCVMoveMergePass(*PR);
146 initializeRISCVPushPopOptPass(*PR);
147 initializeRISCVIndirectBranchTrackingPass(*PR);
148 initializeRISCVLoadStoreOptPass(*PR);
149 initializeRISCVPreAllocZilsdOptPass(*PR);
150 initializeRISCVExpandPseudoAtomicsLegacyPass(*PR);
151 initializeRISCVRedundantCopyEliminationPass(*PR);
152 initializeRISCVAsmPrinterPass(*PR);
153 initializeRISCVPromoteConstantPass(*PR);
154}
155
156static Reloc::Model getEffectiveRelocModel(const Triple &TT,
157 std::optional<Reloc::Model> RM) {
158 if (TT.isOSBinFormatMachO())
159 return RM.value_or(u: Reloc::PIC_);
160
161 return RM.value_or(u: Reloc::Static);
162}
163
164static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
165 if (TT.isOSBinFormatMachO())
166 return std::make_unique<RISCVMachOTargetObjectFile>();
167 return std::make_unique<RISCVELFTargetObjectFile>();
168}
169
170RISCVTargetMachine::RISCVTargetMachine(const Target &T, const Triple &TT,
171 StringRef CPU, StringRef FS,
172 const TargetOptions &Options,
173 std::optional<Reloc::Model> RM,
174 std::optional<CodeModel::Model> CM,
175 CodeGenOptLevel OL, bool JIT)
176 : CodeGenTargetMachineImpl(
177 T, TT.computeDataLayout(ABIName: Options.MCOptions.getABIName()), TT, CPU, FS,
178 Options, getEffectiveRelocModel(TT, RM),
179 getEffectiveCodeModel(CM, Default: CodeModel::Small), OL),
180 TLOF(createTLOF(TT)) {
181 initAsmInfo();
182
183 // RISC-V supports the MachineOutliner.
184 setMachineOutliner(true);
185 setSupportsDefaultOutlining(true);
186
187 // RISC-V supports the debug entry values.
188 setSupportsDebugEntryValues(true);
189
190 if (TT.isOSFuchsia() && !TT.isArch64Bit())
191 report_fatal_error(reason: "Fuchsia is only supported for 64-bit");
192
193 setCFIFixup(!EnableCFIInstrInserter);
194}
195
196const RISCVSubtarget *
197RISCVTargetMachine::getSubtargetImpl(const Function &F) const {
198 Attribute CPUAttr = F.getFnAttribute(Kind: "target-cpu");
199 Attribute TuneAttr = F.getFnAttribute(Kind: "tune-cpu");
200 Attribute FSAttr = F.getFnAttribute(Kind: "target-features");
201
202 std::string CPU =
203 CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU;
204 std::string TuneCPU =
205 TuneAttr.isValid() ? TuneAttr.getValueAsString().str() : CPU;
206 std::string FS =
207 FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS;
208
209 unsigned RVVBitsMin = RVVVectorBitsMinOpt;
210 unsigned RVVBitsMax = RVVVectorBitsMaxOpt;
211
212 Attribute VScaleRangeAttr = F.getFnAttribute(Kind: Attribute::VScaleRange);
213 if (VScaleRangeAttr.isValid()) {
214 if (!RVVVectorBitsMinOpt.getNumOccurrences())
215 RVVBitsMin = VScaleRangeAttr.getVScaleRangeMin() * RISCV::RVVBitsPerBlock;
216 std::optional<unsigned> VScaleMax = VScaleRangeAttr.getVScaleRangeMax();
217 if (VScaleMax.has_value() && !RVVVectorBitsMaxOpt.getNumOccurrences())
218 RVVBitsMax = *VScaleMax * RISCV::RVVBitsPerBlock;
219 }
220
221 if (RVVBitsMin != -1U) {
222 // FIXME: Change to >= 32 when VLEN = 32 is supported.
223 assert((RVVBitsMin == 0 || (RVVBitsMin >= 64 && RVVBitsMin <= 65536 &&
224 isPowerOf2_32(RVVBitsMin))) &&
225 "V or Zve* extension requires vector length to be in the range of "
226 "64 to 65536 and a power 2!");
227 assert((RVVBitsMax >= RVVBitsMin || RVVBitsMax == 0) &&
228 "Minimum V extension vector length should not be larger than its "
229 "maximum!");
230 }
231 assert((RVVBitsMax == 0 || (RVVBitsMax >= 64 && RVVBitsMax <= 65536 &&
232 isPowerOf2_32(RVVBitsMax))) &&
233 "V or Zve* extension requires vector length to be in the range of "
234 "64 to 65536 and a power 2!");
235
236 if (RVVBitsMin != -1U) {
237 if (RVVBitsMax != 0) {
238 RVVBitsMin = std::min(a: RVVBitsMin, b: RVVBitsMax);
239 RVVBitsMax = std::max(a: RVVBitsMin, b: RVVBitsMax);
240 }
241
242 RVVBitsMin = llvm::bit_floor(
243 Value: (RVVBitsMin < 64 || RVVBitsMin > 65536) ? 0 : RVVBitsMin);
244 }
245 RVVBitsMax =
246 llvm::bit_floor(Value: (RVVBitsMax < 64 || RVVBitsMax > 65536) ? 0 : RVVBitsMax);
247
248 SmallString<512> Key;
249 raw_svector_ostream(Key) << "RVVMin" << RVVBitsMin << "RVVMax" << RVVBitsMax
250 << CPU << TuneCPU << FS;
251 auto &I = SubtargetMap[Key];
252 if (!I) {
253 StringRef ABIName = getTargetABIName(M: *F.getParent());
254 I = std::make_unique<RISCVSubtarget>(
255 args: TargetTriple, args&: CPU, args&: TuneCPU, args&: FS, args&: ABIName, args&: RVVBitsMin, args&: RVVBitsMax, args: *this);
256 }
257 return I.get();
258}
259
260MachineFunctionInfo *RISCVTargetMachine::createMachineFunctionInfo(
261 BumpPtrAllocator &Allocator, const Function &F,
262 const TargetSubtargetInfo *STI) const {
263 return RISCVMachineFunctionInfo::create<RISCVMachineFunctionInfo>(
264 Allocator, F, STI: static_cast<const RISCVSubtarget *>(STI));
265}
266
267TargetTransformInfo
268RISCVTargetMachine::getTargetTransformInfo(const Function &F) const {
269 return TargetTransformInfo(std::make_unique<RISCVTTIImpl>(args: this, args: F));
270}
271
272// A RISC-V hart has a single byte-addressable address space of 2^XLEN bytes
273// for all memory accesses, so it is reasonable to assume that an
274// implementation has no-op address space casts. If an implementation makes a
275// change to this, they can override it here.
276bool RISCVTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
277 unsigned DstAS) const {
278 return true;
279}
280
281ScheduleDAGInstrs *
282RISCVTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
283 const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
284 ScheduleDAGMILive *DAG = createSchedLive<RISCVPreRAMachineSchedStrategy>(C);
285
286 // Add MacroFusion mutation first with a higher priority than later clustering
287 const auto &MacroFusions = ST.getMacroFusions();
288 if (!MacroFusions.empty())
289 DAG->addMutation(Mutation: createMacroFusionDAGMutation(Predicates: MacroFusions));
290
291 if (ST.enableMISchedLoadClustering())
292 DAG->addMutation(Mutation: createLoadClusterDAGMutation(
293 TII: DAG->TII, TRI: DAG->TRI, /*ReorderWhileClustering=*/true));
294
295 if (ST.enableMISchedStoreClustering())
296 DAG->addMutation(Mutation: createStoreClusterDAGMutation(
297 TII: DAG->TII, TRI: DAG->TRI, /*ReorderWhileClustering=*/true));
298
299 if (!DisableVectorMaskMutation && ST.hasVInstructions())
300 DAG->addMutation(Mutation: createRISCVVectorMaskDAGMutation(TRI: DAG->TRI));
301
302 return DAG;
303}
304
305ScheduleDAGInstrs *
306RISCVTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
307 const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
308 ScheduleDAGMI *DAG = createSchedPostRA(C);
309
310 // Add MacroFusion mutation first with a higher priority than later clustering
311 const auto &MacroFusions = ST.getMacroFusions();
312 if (!MacroFusions.empty())
313 DAG->addMutation(Mutation: createMacroFusionDAGMutation(Predicates: MacroFusions));
314
315 if (ST.enablePostMISchedLoadClustering())
316 DAG->addMutation(Mutation: createLoadClusterDAGMutation(
317 TII: DAG->TII, TRI: DAG->TRI, /*ReorderWhileClustering=*/true));
318
319 if (ST.enablePostMISchedStoreClustering())
320 DAG->addMutation(Mutation: createStoreClusterDAGMutation(
321 TII: DAG->TII, TRI: DAG->TRI, /*ReorderWhileClustering=*/true));
322
323 return DAG;
324}
325
326namespace {
327
328class RVVRegisterRegAlloc : public RegisterRegAllocBase<RVVRegisterRegAlloc> {
329public:
330 RVVRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
331 : RegisterRegAllocBase(N, D, C) {}
332};
333
334static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI,
335 const MachineRegisterInfo &MRI,
336 const Register Reg) {
337 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
338 return RISCVRegisterInfo::isRVVRegClass(RC);
339}
340
341static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
342
343static llvm::once_flag InitializeDefaultRVVRegisterAllocatorFlag;
344
345/// -riscv-rvv-regalloc=<fast|basic|greedy> command line option.
346/// This option could designate the rvv register allocator only.
347/// For example: -riscv-rvv-regalloc=basic
348static cl::opt<RVVRegisterRegAlloc::FunctionPassCtor, false,
349 RegisterPassParser<RVVRegisterRegAlloc>>
350 RVVRegAlloc("riscv-rvv-regalloc", cl::Hidden,
351 cl::init(Val: &useDefaultRegisterAllocator),
352 cl::desc("Register allocator to use for RVV register."));
353
354static void initializeDefaultRVVRegisterAllocatorOnce() {
355 RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
356
357 if (!Ctor) {
358 Ctor = RVVRegAlloc;
359 RVVRegisterRegAlloc::setDefault(RVVRegAlloc);
360 }
361}
362
363static FunctionPass *createBasicRVVRegisterAllocator() {
364 return createBasicRegisterAllocator(F: onlyAllocateRVVReg);
365}
366
367static FunctionPass *createGreedyRVVRegisterAllocator() {
368 return createGreedyRegisterAllocator(F: onlyAllocateRVVReg);
369}
370
371static FunctionPass *createFastRVVRegisterAllocator() {
372 return createFastRegisterAllocator(F: onlyAllocateRVVReg, ClearVirtRegs: false);
373}
374
375static RVVRegisterRegAlloc basicRegAllocRVVReg("basic",
376 "basic register allocator",
377 createBasicRVVRegisterAllocator);
378static RVVRegisterRegAlloc
379 greedyRegAllocRVVReg("greedy", "greedy register allocator",
380 createGreedyRVVRegisterAllocator);
381
382static RVVRegisterRegAlloc fastRegAllocRVVReg("fast", "fast register allocator",
383 createFastRVVRegisterAllocator);
384
385class RISCVPassConfig : public TargetPassConfig {
386public:
387 RISCVPassConfig(RISCVTargetMachine &TM, PassManagerBase &PM)
388 : TargetPassConfig(TM, PM) {
389 if (TM.getOptLevel() != CodeGenOptLevel::None)
390 substitutePass(StandardID: &PostRASchedulerID, TargetID: &PostMachineSchedulerID);
391 setEnableSinkAndFold(EnableSinkFold);
392 EnableLoopTermFold = true;
393 }
394
395 RISCVTargetMachine &getRISCVTargetMachine() const {
396 return getTM<RISCVTargetMachine>();
397 }
398
399 void addIRPasses() override;
400 bool addPreISel() override;
401 void addCodeGenPrepare() override;
402 bool addInstSelector() override;
403 bool addIRTranslator() override;
404 void addPreLegalizeMachineIR() override;
405 bool addLegalizeMachineIR() override;
406 void addPreRegBankSelect() override;
407 bool addRegBankSelect() override;
408 bool addGlobalInstructionSelect() override;
409 void addPreEmitPass() override;
410 void addPreEmitPass2() override;
411 void addPreSched2() override;
412 void addMachineSSAOptimization() override;
413 FunctionPass *createRVVRegAllocPass(bool Optimized);
414 bool addRegAssignAndRewriteFast() override;
415 bool addRegAssignAndRewriteOptimized() override;
416 void addPreRegAlloc() override;
417 void addPostRegAlloc() override;
418 void addFastRegAlloc() override;
419 bool addILPOpts() override;
420
421 std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
422};
423} // namespace
424
425TargetPassConfig *RISCVTargetMachine::createPassConfig(PassManagerBase &PM) {
426 return new RISCVPassConfig(*this, PM);
427}
428
429std::unique_ptr<CSEConfigBase> RISCVPassConfig::getCSEConfig() const {
430 return getStandardCSEConfigForOpt(Level: TM->getOptLevel());
431}
432
433FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {
434 // Initialize the global default.
435 llvm::call_once(flag&: InitializeDefaultRVVRegisterAllocatorFlag,
436 F&: initializeDefaultRVVRegisterAllocatorOnce);
437
438 RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
439 if (Ctor != useDefaultRegisterAllocator)
440 return Ctor();
441
442 if (Optimized)
443 return createGreedyRVVRegisterAllocator();
444
445 return createFastRVVRegisterAllocator();
446}
447
448bool RISCVPassConfig::addRegAssignAndRewriteFast() {
449 addPass(P: createRVVRegAllocPass(Optimized: false));
450 addPass(P: createRISCVInsertVSETVLIPass());
451 if (TM->getOptLevel() != CodeGenOptLevel::None &&
452 EnableRISCVDeadRegisterElimination)
453 addPass(P: createRISCVDeadRegisterDefinitionsPass());
454 return TargetPassConfig::addRegAssignAndRewriteFast();
455}
456
457bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {
458 addPass(P: createRVVRegAllocPass(Optimized: true));
459 addPass(P: createVirtRegRewriter(ClearVirtRegs: false));
460 addPass(P: createRISCVInsertVSETVLIPass());
461 if (TM->getOptLevel() != CodeGenOptLevel::None &&
462 EnableRISCVDeadRegisterElimination)
463 addPass(P: createRISCVDeadRegisterDefinitionsPass());
464 return TargetPassConfig::addRegAssignAndRewriteOptimized();
465}
466
467void RISCVPassConfig::addIRPasses() {
468 addPass(P: createAtomicExpandLegacyPass());
469 addPass(P: createRISCVZacasABIFixLegacyPass());
470
471 if (getOptLevel() != CodeGenOptLevel::None) {
472 if (EnableLoopDataPrefetch)
473 addPass(P: createLoopDataPrefetchPass());
474
475 addPass(P: createRISCVGatherScatterLoweringLegacyPass());
476 addPass(P: createInterleavedAccessPass());
477 addPass(P: createRISCVCodeGenPrepareLegacyPass());
478 }
479
480 TargetPassConfig::addIRPasses();
481
482 if (getOptLevel() == CodeGenOptLevel::Aggressive && EnableSelectOpt)
483 addPass(P: createSelectOptimizePass());
484}
485
486bool RISCVPassConfig::addPreISel() {
487 if (TM->getOptLevel() != CodeGenOptLevel::None)
488 addPass(P: createRISCVPromoteConstantPass());
489 if (TM->getOptLevel() != CodeGenOptLevel::None) {
490 // Add a barrier before instruction selection so that we will not get
491 // deleted block address after enabling default outlining. See D99707 for
492 // more details.
493 addPass(P: createBarrierNoopPass());
494 }
495
496 if ((TM->getOptLevel() != CodeGenOptLevel::None &&
497 EnableGlobalMerge == cl::boolOrDefault::BOU_UNSET) ||
498 EnableGlobalMerge == cl::boolOrDefault::BOU_TRUE) {
499 // FIXME: Like AArch64, we disable extern global merging by default due to
500 // concerns it might regress some workloads. Unlike AArch64, we don't
501 // currently support enabling the pass in an "OnlyOptimizeForSize" mode.
502 // Investigating and addressing both items are TODO.
503 addPass(P: createGlobalMergePass(TM, /* MaxOffset */ MaximalOffset: 2047,
504 /* OnlyOptimizeForSize */ false,
505 /* MergeExternalByDefault */ true));
506 }
507
508 return false;
509}
510
511void RISCVPassConfig::addCodeGenPrepare() {
512 if (getOptLevel() != CodeGenOptLevel::None)
513 addPass(P: createTypePromotionLegacyPass());
514 TargetPassConfig::addCodeGenPrepare();
515}
516
517bool RISCVPassConfig::addInstSelector() {
518 addPass(P: createRISCVISelDagLegacyPass(TM&: getRISCVTargetMachine(), OptLevel: getOptLevel()));
519
520 return false;
521}
522
523bool RISCVPassConfig::addIRTranslator() {
524 addPass(P: new IRTranslatorLegacy(getOptLevel()));
525 return false;
526}
527
528void RISCVPassConfig::addPreLegalizeMachineIR() {
529 if (getOptLevel() == CodeGenOptLevel::None) {
530 addPass(P: createRISCVO0PreLegalizerCombiner());
531 } else {
532 addPass(P: createRISCVPreLegalizerCombiner());
533 }
534}
535
536bool RISCVPassConfig::addLegalizeMachineIR() {
537 addPass(P: new LegalizerLegacy());
538 return false;
539}
540
541void RISCVPassConfig::addPreRegBankSelect() {
542 if (getOptLevel() != CodeGenOptLevel::None)
543 addPass(P: createRISCVPostLegalizerCombiner());
544}
545
546bool RISCVPassConfig::addRegBankSelect() {
547 addPass(P: new RegBankSelectLegacy());
548 return false;
549}
550
551bool RISCVPassConfig::addGlobalInstructionSelect() {
552 addPass(P: new InstructionSelectLegacy(getOptLevel()));
553 return false;
554}
555
556void RISCVPassConfig::addPreSched2() {
557 addPass(P: createRISCVExpandPseudoPostRALegacyPass());
558
559 // Emit KCFI checks for indirect calls.
560 addPass(P: createKCFIPass());
561 if (TM->getOptLevel() != CodeGenOptLevel::None)
562 addPass(P: createRISCVLoadStoreOptPass());
563}
564
565void RISCVPassConfig::addPreEmitPass() {
566 // TODO: It would potentially be better to schedule copy propagation after
567 // expanding pseudos (in addPreEmitPass2). However, performing copy
568 // propagation after the machine outliner (which runs after addPreEmitPass)
569 // currently leads to incorrect code-gen, where copies to registers within
570 // outlined functions are removed erroneously.
571 if (TM->getOptLevel() >= CodeGenOptLevel::Default &&
572 EnableRISCVCopyPropagation)
573 addPass(P: createMachineCopyPropagationPass(UseCopyInstr: true));
574 if (TM->getOptLevel() >= CodeGenOptLevel::Default)
575 addPass(P: createRISCVLateBranchOptPass());
576 // The IndirectBranchTrackingPass inserts lpad and could have changed the
577 // basic block alignment. It must be done before Branch Relaxation to
578 // prevent the adjusted offset exceeding the branch range.
579 addPass(P: createRISCVIndirectBranchTrackingPass());
580 addPass(PassID: &BranchRelaxationPassID);
581 addPass(P: createRISCVMakeCompressibleOptPass());
582}
583
584void RISCVPassConfig::addPreEmitPass2() {
585 if (TM->getOptLevel() != CodeGenOptLevel::None) {
586 addPass(P: createRISCVMoveMergePass());
587 // Schedule PushPop Optimization before expansion of Pseudo instruction,
588 // ensuring return instruction is detected correctly.
589 addPass(P: createRISCVPushPopOptimizationPass());
590 }
591 addPass(P: createRISCVExpandPseudoPreEmitLegacyPass());
592
593 // Add QC Relaxation Markers as late as possible, and only for RV32
594 if (TM->getOptLevel() != CodeGenOptLevel::None &&
595 TM->getTargetTriple().isRISCV32())
596 addPass(P: createRISCVQCRelaxMarkingPass());
597
598 // Schedule the expansion of AMOs at the last possible moment, avoiding the
599 // possibility for other passes to break the requirements for forward
600 // progress in the LR/SC block.
601 addPass(P: createRISCVExpandPseudoAtomicsLegacyPass());
602
603 // KCFI indirect call checks are lowered to a bundle.
604 addPass(P: createUnpackMachineBundlesLegacy(Ftor: [&](const MachineFunction &MF) {
605 return MF.getFunction().getParent()->getModuleFlag(Key: "kcfi");
606 }));
607
608 if (EnableCFIInstrInserter)
609 addPass(P: createCFIInstrInserterLegacy());
610}
611
612void RISCVPassConfig::addMachineSSAOptimization() {
613 // It's beneficial to reduce the VL to enable more
614 // Machine SSA optimizations.
615 if (TM->getOptLevel() != CodeGenOptLevel::None) {
616 // RISCVVLOptimizer can make loop invariant instructions like vmv.v.i
617 // loop variant by propagating a VL defined inside the loop. Run LICM and
618 // hoist them early. Don't do this at -O0 to avoid the compile-time
619 // overhead. Not reducing the VL of loop invariant pseudos results in more
620 // vsetvli toggles, and still requires the MachineLoopInfo analysis to be
621 // run.
622 addPass(PassID: &EarlyMachineLICMID);
623 addPass(P: createRISCVVLOptimizerLegacyPass());
624 }
625
626 addPass(P: createRISCVVectorPeepholeLegacyPass());
627 addPass(P: createRISCVFoldMemOffsetLegacyPass());
628
629 TargetPassConfig::addMachineSSAOptimization();
630
631 if (TM->getTargetTriple().isRISCV64()) {
632 addPass(P: createRISCVOptWInstrsLegacyPass());
633 }
634}
635
636void RISCVPassConfig::addPreRegAlloc() {
637 addPass(P: createRISCVExpandPseudoPreRALegacyPass());
638 if (TM->getOptLevel() != CodeGenOptLevel::None) {
639 addPass(P: createRISCVMergeBaseOffsetOptPass());
640 // Add Zilsd pre-allocation load/store optimization
641 addPass(P: createRISCVPreAllocZilsdOptPass());
642 }
643
644 addPass(P: createRISCVInsertReadWriteCSRPass());
645 addPass(P: createRISCVInsertWriteVXRMPass());
646 addPass(P: createRISCVLandingPadSetupPass());
647
648 if (TM->getOptLevel() != CodeGenOptLevel::None && EnableMachinePipeliner)
649 addPass(PassID: &MachinePipelinerID);
650
651 addPass(P: createRISCVVMV0EliminationPass());
652}
653
654void RISCVPassConfig::addFastRegAlloc() {
655 addPass(PassID: &InitUndefID);
656 TargetPassConfig::addFastRegAlloc();
657}
658
659
660void RISCVPassConfig::addPostRegAlloc() {
661 if (TM->getOptLevel() != CodeGenOptLevel::None &&
662 EnableRedundantCopyElimination)
663 addPass(P: createRISCVRedundantCopyEliminationPass());
664}
665
666bool RISCVPassConfig::addILPOpts() {
667 if (EnableMachineCombiner)
668 addPass(PassID: &MachineCombinerID);
669
670 return true;
671}
672
673yaml::MachineFunctionInfo *
674RISCVTargetMachine::createDefaultFuncInfoYAML() const {
675 return new yaml::RISCVMachineFunctionInfo();
676}
677
678yaml::MachineFunctionInfo *
679RISCVTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
680 const auto *MFI = MF.getInfo<RISCVMachineFunctionInfo>();
681 return new yaml::RISCVMachineFunctionInfo(*MFI);
682}
683
684bool RISCVTargetMachine::parseMachineFunctionInfo(
685 const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
686 SMDiagnostic &Error, SMRange &SourceRange) const {
687 const auto &YamlMFI =
688 static_cast<const yaml::RISCVMachineFunctionInfo &>(MFI);
689 PFS.MF.getInfo<RISCVMachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
690 return false;
691}
692