1//===-- RISCVTargetMachine.cpp - Define TargetMachine for RISC-V ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Implements the info about RISC-V target spec.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVTargetMachine.h"
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "RISCV.h"
16#include "RISCVMachineFunctionInfo.h"
17#include "RISCVMachineScheduler.h"
18#include "RISCVTargetObjectFile.h"
19#include "RISCVTargetTransformInfo.h"
20#include "TargetInfo/RISCVTargetInfo.h"
21#include "llvm/Analysis/TargetTransformInfo.h"
22#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
23#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
24#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
25#include "llvm/CodeGen/GlobalISel/Legalizer.h"
26#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
27#include "llvm/CodeGen/MIRParser/MIParser.h"
28#include "llvm/CodeGen/MIRYamlMapping.h"
29#include "llvm/CodeGen/MachineScheduler.h"
30#include "llvm/CodeGen/MacroFusion.h"
31#include "llvm/CodeGen/Passes.h"
32#include "llvm/CodeGen/RegAllocRegistry.h"
33#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
34#include "llvm/CodeGen/TargetPassConfig.h"
35#include "llvm/InitializePasses.h"
36#include "llvm/MC/TargetRegistry.h"
37#include "llvm/Support/Compiler.h"
38#include "llvm/Target/TargetOptions.h"
39#include "llvm/Transforms/IPO.h"
40#include "llvm/Transforms/Scalar.h"
41#include <optional>
42using namespace llvm;
43
44static cl::opt<bool> EnableRedundantCopyElimination(
45 "riscv-enable-copyelim",
46 cl::desc("Enable the redundant copy elimination pass"), cl::init(Val: true),
47 cl::Hidden);
48
49// FIXME: Unify control over GlobalMerge.
50static cl::opt<cl::boolOrDefault>
51 EnableGlobalMerge("riscv-enable-global-merge", cl::Hidden,
52 cl::desc("Enable the global merge pass"));
53
54static cl::opt<bool>
55 EnableMachineCombiner("riscv-enable-machine-combiner",
56 cl::desc("Enable the machine combiner pass"),
57 cl::init(Val: true), cl::Hidden);
58
59static cl::opt<unsigned> RVVVectorBitsMaxOpt(
60 "riscv-v-vector-bits-max",
61 cl::desc("Assume V extension vector registers are at most this big, "
62 "with zero meaning no maximum size is assumed."),
63 cl::init(Val: 0), cl::Hidden);
64
65static cl::opt<int> RVVVectorBitsMinOpt(
66 "riscv-v-vector-bits-min",
67 cl::desc("Assume V extension vector registers are at least this big, "
68 "with zero meaning no minimum size is assumed. A value of -1 "
69 "means use Zvl*b extension. This is primarily used to enable "
70 "autovectorization with fixed width vectors."),
71 cl::init(Val: -1), cl::Hidden);
72
73static cl::opt<bool> EnableRISCVCopyPropagation(
74 "riscv-enable-copy-propagation",
75 cl::desc("Enable the copy propagation with RISC-V copy instr"),
76 cl::init(Val: true), cl::Hidden);
77
78static cl::opt<bool> EnableRISCVDeadRegisterElimination(
79 "riscv-enable-dead-defs", cl::Hidden,
80 cl::desc("Enable the pass that removes dead"
81 " definitions and replaces stores to"
82 " them with stores to x0"),
83 cl::init(Val: true));
84
85static cl::opt<bool>
86 EnableSinkFold("riscv-enable-sink-fold",
87 cl::desc("Enable sinking and folding of instruction copies"),
88 cl::init(Val: true), cl::Hidden);
89
90static cl::opt<bool>
91 EnableLoopDataPrefetch("riscv-enable-loop-data-prefetch", cl::Hidden,
92 cl::desc("Enable the loop data prefetch pass"),
93 cl::init(Val: true));
94
95static cl::opt<bool> DisableVectorMaskMutation(
96 "riscv-disable-vector-mask-mutation",
97 cl::desc("Disable the vector mask scheduling mutation"), cl::init(Val: false),
98 cl::Hidden);
99
100static cl::opt<bool>
101 EnableMachinePipeliner("riscv-enable-pipeliner",
102 cl::desc("Enable Machine Pipeliner for RISC-V"),
103 cl::init(Val: false), cl::Hidden);
104
105static cl::opt<bool> EnableCFIInstrInserter(
106 "riscv-enable-cfi-instr-inserter",
107 cl::desc("Enable CFI Instruction Inserter for RISC-V"), cl::init(Val: false),
108 cl::Hidden);
109
110static cl::opt<bool>
111 EnableSelectOpt("riscv-select-opt", cl::Hidden,
112 cl::desc("Enable select to branch optimizations"),
113 cl::init(Val: true));
114
115extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
116 RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
117 RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
118 RegisterTargetMachine<RISCVTargetMachine> A(getTheRISCV32beTarget());
119 RegisterTargetMachine<RISCVTargetMachine> B(getTheRISCV64beTarget());
120 auto *PR = PassRegistry::getPassRegistry();
121 initializeGlobalISel(*PR);
122 initializeRISCVO0PreLegalizerCombinerPass(*PR);
123 initializeRISCVPreLegalizerCombinerPass(*PR);
124 initializeRISCVPostLegalizerCombinerPass(*PR);
125 initializeMachineKCFILegacyPass(*PR);
126 initializeRISCVDeadRegisterDefinitionsPass(*PR);
127 initializeRISCVLateBranchOptPass(*PR);
128 initializeRISCVMakeCompressibleOptPass(*PR);
129 initializeRISCVQCRelaxMarkingPass(*PR);
130 initializeRISCVGatherScatterLoweringLegacyPass(*PR);
131 initializeRISCVCodeGenPrepareLegacyPass(*PR);
132 initializeRISCVZacasABIFixLegacyPass(*PR);
133 initializeRISCVExpandPseudoPostRALegacyPass(*PR);
134 initializeRISCVMergeBaseOffsetOptPass(*PR);
135 initializeRISCVOptWInstrsLegacyPass(*PR);
136 initializeRISCVFoldMemOffsetLegacyPass(*PR);
137 initializeRISCVExpandPseudoPreRALegacyPass(*PR);
138 initializeRISCVExpandPseudoPreEmitLegacyPass(*PR);
139 initializeRISCVVectorPeepholeLegacyPass(*PR);
140 initializeRISCVVLOptimizerLegacyPass(*PR);
141 initializeRISCVVMV0EliminationPass(*PR);
142 initializeRISCVInsertVSETVLIPass(*PR);
143 initializeRISCVInsertReadWriteCSRPass(*PR);
144 initializeRISCVInsertWriteVXRMPass(*PR);
145 initializeRISCVDAGToDAGISelLegacyPass(*PR);
146 initializeRISCVMoveMergePass(*PR);
147 initializeRISCVPushPopOptPass(*PR);
148 initializeRISCVIndirectBranchTrackingPass(*PR);
149 initializeRISCVLoadStoreOptPass(*PR);
150 initializeRISCVPreAllocZilsdOptPass(*PR);
151 initializeRISCVExpandPseudoAtomicsLegacyPass(*PR);
152 initializeRISCVRedundantCopyEliminationPass(*PR);
153 initializeRISCVAsmPrinterPass(*PR);
154 initializeRISCVPromoteConstantPass(*PR);
155}
156
157static Reloc::Model getEffectiveRelocModel(const Triple &TT,
158 std::optional<Reloc::Model> RM) {
159 if (TT.isOSBinFormatMachO())
160 return RM.value_or(u: Reloc::PIC_);
161
162 return RM.value_or(u: Reloc::Static);
163}
164
165static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
166 if (TT.isOSBinFormatMachO())
167 return std::make_unique<RISCVMachOTargetObjectFile>();
168 return std::make_unique<RISCVELFTargetObjectFile>();
169}
170
171RISCVTargetMachine::RISCVTargetMachine(const Target &T, const Triple &TT,
172 StringRef CPU, StringRef FS,
173 const TargetOptions &Options,
174 std::optional<Reloc::Model> RM,
175 std::optional<CodeModel::Model> CM,
176 CodeGenOptLevel OL, bool JIT)
177 : CodeGenTargetMachineImpl(
178 T, TT.computeDataLayout(ABIName: Options.MCOptions.getABIName()), TT, CPU, FS,
179 Options, getEffectiveRelocModel(TT, RM),
180 getEffectiveCodeModel(CM, Default: CodeModel::Small), OL),
181 TLOF(createTLOF(TT)) {
182 initAsmInfo();
183
184 // RISC-V supports the MachineOutliner.
185 setMachineOutliner(true);
186 setSupportsDefaultOutlining(true);
187
188 // RISC-V supports the debug entry values.
189 setSupportsDebugEntryValues(true);
190
191 if (TT.isOSFuchsia() && !TT.isArch64Bit())
192 report_fatal_error(reason: "Fuchsia is only supported for 64-bit");
193
194 setCFIFixup(!EnableCFIInstrInserter);
195}
196
197const RISCVSubtarget *
198RISCVTargetMachine::getSubtargetImpl(const Function &F) const {
199 Attribute CPUAttr = F.getFnAttribute(Kind: "target-cpu");
200 Attribute TuneAttr = F.getFnAttribute(Kind: "tune-cpu");
201 Attribute FSAttr = F.getFnAttribute(Kind: "target-features");
202
203 std::string CPU =
204 CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU;
205 std::string TuneCPU =
206 TuneAttr.isValid() ? TuneAttr.getValueAsString().str() : CPU;
207 std::string FS =
208 FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS;
209
210 unsigned RVVBitsMin = RVVVectorBitsMinOpt;
211 unsigned RVVBitsMax = RVVVectorBitsMaxOpt;
212
213 Attribute VScaleRangeAttr = F.getFnAttribute(Kind: Attribute::VScaleRange);
214 if (VScaleRangeAttr.isValid()) {
215 if (!RVVVectorBitsMinOpt.getNumOccurrences())
216 RVVBitsMin = VScaleRangeAttr.getVScaleRangeMin() * RISCV::RVVBitsPerBlock;
217 std::optional<unsigned> VScaleMax = VScaleRangeAttr.getVScaleRangeMax();
218 if (VScaleMax.has_value() && !RVVVectorBitsMaxOpt.getNumOccurrences())
219 RVVBitsMax = *VScaleMax * RISCV::RVVBitsPerBlock;
220 }
221
222 if (RVVBitsMin != -1U) {
223 // FIXME: Change to >= 32 when VLEN = 32 is supported.
224 assert((RVVBitsMin == 0 || (RVVBitsMin >= 64 && RVVBitsMin <= 65536 &&
225 isPowerOf2_32(RVVBitsMin))) &&
226 "V or Zve* extension requires vector length to be in the range of "
227 "64 to 65536 and a power 2!");
228 assert((RVVBitsMax >= RVVBitsMin || RVVBitsMax == 0) &&
229 "Minimum V extension vector length should not be larger than its "
230 "maximum!");
231 }
232 assert((RVVBitsMax == 0 || (RVVBitsMax >= 64 && RVVBitsMax <= 65536 &&
233 isPowerOf2_32(RVVBitsMax))) &&
234 "V or Zve* extension requires vector length to be in the range of "
235 "64 to 65536 and a power 2!");
236
237 if (RVVBitsMin != -1U) {
238 if (RVVBitsMax != 0) {
239 RVVBitsMin = std::min(a: RVVBitsMin, b: RVVBitsMax);
240 RVVBitsMax = std::max(a: RVVBitsMin, b: RVVBitsMax);
241 }
242
243 RVVBitsMin = llvm::bit_floor(
244 Value: (RVVBitsMin < 64 || RVVBitsMin > 65536) ? 0 : RVVBitsMin);
245 }
246 RVVBitsMax =
247 llvm::bit_floor(Value: (RVVBitsMax < 64 || RVVBitsMax > 65536) ? 0 : RVVBitsMax);
248
249 SmallString<512> Key;
250 raw_svector_ostream(Key) << "RVVMin" << RVVBitsMin << "RVVMax" << RVVBitsMax
251 << CPU << TuneCPU << FS;
252 auto &I = SubtargetMap[Key];
253 if (!I) {
254 StringRef ABIName = getTargetABIName(M: *F.getParent());
255 I = std::make_unique<RISCVSubtarget>(
256 args: TargetTriple, args&: CPU, args&: TuneCPU, args&: FS, args&: ABIName, args&: RVVBitsMin, args&: RVVBitsMax, args: *this);
257 }
258 return I.get();
259}
260
261MachineFunctionInfo *RISCVTargetMachine::createMachineFunctionInfo(
262 BumpPtrAllocator &Allocator, const Function &F,
263 const TargetSubtargetInfo *STI) const {
264 return RISCVMachineFunctionInfo::create<RISCVMachineFunctionInfo>(
265 Allocator, F, STI: static_cast<const RISCVSubtarget *>(STI));
266}
267
268TargetTransformInfo
269RISCVTargetMachine::getTargetTransformInfo(const Function &F) const {
270 return TargetTransformInfo(std::make_unique<RISCVTTIImpl>(args: this, args: F));
271}
272
273// A RISC-V hart has a single byte-addressable address space of 2^XLEN bytes
274// for all memory accesses, so it is reasonable to assume that an
275// implementation has no-op address space casts. If an implementation makes a
276// change to this, they can override it here.
277bool RISCVTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
278 unsigned DstAS) const {
279 return true;
280}
281
282ScheduleDAGInstrs *
283RISCVTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
284 const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
285 ScheduleDAGMILive *DAG = createSchedLive<RISCVPreRAMachineSchedStrategy>(C);
286
287 // Add MacroFusion mutation first with a higher priority than later clustering
288 const auto &MacroFusions = ST.getMacroFusions();
289 if (!MacroFusions.empty())
290 DAG->addMutation(Mutation: createMacroFusionDAGMutation(Predicates: MacroFusions));
291
292 if (ST.enableMISchedLoadClustering())
293 DAG->addMutation(Mutation: createLoadClusterDAGMutation(
294 TII: DAG->TII, TRI: DAG->TRI, /*ReorderWhileClustering=*/true));
295
296 if (ST.enableMISchedStoreClustering())
297 DAG->addMutation(Mutation: createStoreClusterDAGMutation(
298 TII: DAG->TII, TRI: DAG->TRI, /*ReorderWhileClustering=*/true));
299
300 if (!DisableVectorMaskMutation && ST.hasVInstructions())
301 DAG->addMutation(Mutation: createRISCVVectorMaskDAGMutation(TRI: DAG->TRI));
302
303 return DAG;
304}
305
306ScheduleDAGInstrs *
307RISCVTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
308 const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
309 ScheduleDAGMI *DAG = createSchedPostRA(C);
310
311 // Add MacroFusion mutation first with a higher priority than later clustering
312 const auto &MacroFusions = ST.getMacroFusions();
313 if (!MacroFusions.empty())
314 DAG->addMutation(Mutation: createMacroFusionDAGMutation(Predicates: MacroFusions));
315
316 if (ST.enablePostMISchedLoadClustering())
317 DAG->addMutation(Mutation: createLoadClusterDAGMutation(
318 TII: DAG->TII, TRI: DAG->TRI, /*ReorderWhileClustering=*/true));
319
320 if (ST.enablePostMISchedStoreClustering())
321 DAG->addMutation(Mutation: createStoreClusterDAGMutation(
322 TII: DAG->TII, TRI: DAG->TRI, /*ReorderWhileClustering=*/true));
323
324 return DAG;
325}
326
327namespace {
328
329class RVVRegisterRegAlloc : public RegisterRegAllocBase<RVVRegisterRegAlloc> {
330public:
331 RVVRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
332 : RegisterRegAllocBase(N, D, C) {}
333};
334
335static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI,
336 const MachineRegisterInfo &MRI,
337 const Register Reg) {
338 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
339 return RISCVRegisterInfo::isRVVRegClass(RC);
340}
341
342static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
343
344static llvm::once_flag InitializeDefaultRVVRegisterAllocatorFlag;
345
346/// -riscv-rvv-regalloc=<fast|basic|greedy> command line option.
347/// This option could designate the rvv register allocator only.
348/// For example: -riscv-rvv-regalloc=basic
349static cl::opt<RVVRegisterRegAlloc::FunctionPassCtor, false,
350 RegisterPassParser<RVVRegisterRegAlloc>>
351 RVVRegAlloc("riscv-rvv-regalloc", cl::Hidden,
352 cl::init(Val: &useDefaultRegisterAllocator),
353 cl::desc("Register allocator to use for RVV register."));
354
355static void initializeDefaultRVVRegisterAllocatorOnce() {
356 RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
357
358 if (!Ctor) {
359 Ctor = RVVRegAlloc;
360 RVVRegisterRegAlloc::setDefault(RVVRegAlloc);
361 }
362}
363
364static FunctionPass *createBasicRVVRegisterAllocator() {
365 return createBasicRegisterAllocator(F: onlyAllocateRVVReg);
366}
367
368static FunctionPass *createGreedyRVVRegisterAllocator() {
369 return createGreedyRegisterAllocator(F: onlyAllocateRVVReg);
370}
371
372static FunctionPass *createFastRVVRegisterAllocator() {
373 return createFastRegisterAllocator(F: onlyAllocateRVVReg, ClearVirtRegs: false);
374}
375
376static RVVRegisterRegAlloc basicRegAllocRVVReg("basic",
377 "basic register allocator",
378 createBasicRVVRegisterAllocator);
379static RVVRegisterRegAlloc
380 greedyRegAllocRVVReg("greedy", "greedy register allocator",
381 createGreedyRVVRegisterAllocator);
382
383static RVVRegisterRegAlloc fastRegAllocRVVReg("fast", "fast register allocator",
384 createFastRVVRegisterAllocator);
385
386class RISCVPassConfig : public TargetPassConfig {
387public:
388 RISCVPassConfig(RISCVTargetMachine &TM, PassManagerBase &PM)
389 : TargetPassConfig(TM, PM) {
390 if (TM.getOptLevel() != CodeGenOptLevel::None)
391 substitutePass(StandardID: &PostRASchedulerID, TargetID: &PostMachineSchedulerID);
392 setEnableSinkAndFold(EnableSinkFold);
393 EnableLoopTermFold = true;
394 }
395
396 RISCVTargetMachine &getRISCVTargetMachine() const {
397 return getTM<RISCVTargetMachine>();
398 }
399
400 void addIRPasses() override;
401 bool addPreISel() override;
402 void addCodeGenPrepare() override;
403 bool addInstSelector() override;
404 bool addIRTranslator() override;
405 void addPreLegalizeMachineIR() override;
406 bool addLegalizeMachineIR() override;
407 void addPreRegBankSelect() override;
408 bool addRegBankSelect() override;
409 bool addGlobalInstructionSelect() override;
410 void addPreEmitPass() override;
411 void addPreEmitPass2() override;
412 void addPreSched2() override;
413 void addMachineSSAOptimization() override;
414 FunctionPass *createRVVRegAllocPass(bool Optimized);
415 bool addRegAssignAndRewriteFast() override;
416 bool addRegAssignAndRewriteOptimized() override;
417 void addPreRegAlloc() override;
418 void addPostRegAlloc() override;
419 void addFastRegAlloc() override;
420 bool addILPOpts() override;
421
422 std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
423};
424} // namespace
425
426TargetPassConfig *RISCVTargetMachine::createPassConfig(PassManagerBase &PM) {
427 return new RISCVPassConfig(*this, PM);
428}
429
430std::unique_ptr<CSEConfigBase> RISCVPassConfig::getCSEConfig() const {
431 return getStandardCSEConfigForOpt(Level: TM->getOptLevel());
432}
433
434FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {
435 // Initialize the global default.
436 llvm::call_once(flag&: InitializeDefaultRVVRegisterAllocatorFlag,
437 F&: initializeDefaultRVVRegisterAllocatorOnce);
438
439 RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
440 if (Ctor != useDefaultRegisterAllocator)
441 return Ctor();
442
443 if (Optimized)
444 return createGreedyRVVRegisterAllocator();
445
446 return createFastRVVRegisterAllocator();
447}
448
449bool RISCVPassConfig::addRegAssignAndRewriteFast() {
450 addPass(P: createRVVRegAllocPass(Optimized: false));
451 addPass(P: createRISCVInsertVSETVLIPass());
452 if (TM->getOptLevel() != CodeGenOptLevel::None &&
453 EnableRISCVDeadRegisterElimination)
454 addPass(P: createRISCVDeadRegisterDefinitionsPass());
455 return TargetPassConfig::addRegAssignAndRewriteFast();
456}
457
458bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {
459 addPass(P: createRVVRegAllocPass(Optimized: true));
460 addPass(P: createVirtRegRewriter(ClearVirtRegs: false));
461 addPass(P: createRISCVInsertVSETVLIPass());
462 if (TM->getOptLevel() != CodeGenOptLevel::None &&
463 EnableRISCVDeadRegisterElimination)
464 addPass(P: createRISCVDeadRegisterDefinitionsPass());
465 return TargetPassConfig::addRegAssignAndRewriteOptimized();
466}
467
468void RISCVPassConfig::addIRPasses() {
469 addPass(P: createAtomicExpandLegacyPass());
470 addPass(P: createRISCVZacasABIFixLegacyPass());
471
472 if (getOptLevel() != CodeGenOptLevel::None) {
473 if (EnableLoopDataPrefetch)
474 addPass(P: createLoopDataPrefetchPass());
475
476 addPass(P: createRISCVGatherScatterLoweringLegacyPass());
477 addPass(P: createInterleavedAccessPass());
478 addPass(P: createRISCVCodeGenPrepareLegacyPass());
479 }
480
481 TargetPassConfig::addIRPasses();
482
483 if (getOptLevel() == CodeGenOptLevel::Aggressive && EnableSelectOpt)
484 addPass(P: createSelectOptimizePass());
485}
486
487bool RISCVPassConfig::addPreISel() {
488 if (TM->getOptLevel() != CodeGenOptLevel::None)
489 addPass(P: createRISCVPromoteConstantPass());
490 if (TM->getOptLevel() != CodeGenOptLevel::None) {
491 // Add a barrier before instruction selection so that we will not get
492 // deleted block address after enabling default outlining. See D99707 for
493 // more details.
494 addPass(P: createBarrierNoopPass());
495 }
496
497 if ((TM->getOptLevel() != CodeGenOptLevel::None &&
498 EnableGlobalMerge == cl::boolOrDefault::BOU_UNSET) ||
499 EnableGlobalMerge == cl::boolOrDefault::BOU_TRUE) {
500 // FIXME: Like AArch64, we disable extern global merging by default due to
501 // concerns it might regress some workloads. Unlike AArch64, we don't
502 // currently support enabling the pass in an "OnlyOptimizeForSize" mode.
503 // Investigating and addressing both items are TODO.
504 addPass(P: createGlobalMergePass(TM, /* MaxOffset */ MaximalOffset: 2047,
505 /* OnlyOptimizeForSize */ false,
506 /* MergeExternalByDefault */ true));
507 }
508
509 return false;
510}
511
512void RISCVPassConfig::addCodeGenPrepare() {
513 if (getOptLevel() != CodeGenOptLevel::None)
514 addPass(P: createTypePromotionLegacyPass());
515 TargetPassConfig::addCodeGenPrepare();
516}
517
518bool RISCVPassConfig::addInstSelector() {
519 addPass(P: createRISCVISelDagLegacyPass(TM&: getRISCVTargetMachine(), OptLevel: getOptLevel()));
520
521 return false;
522}
523
524bool RISCVPassConfig::addIRTranslator() {
525 addPass(P: new IRTranslatorLegacy(getOptLevel()));
526 return false;
527}
528
529void RISCVPassConfig::addPreLegalizeMachineIR() {
530 if (getOptLevel() == CodeGenOptLevel::None) {
531 addPass(P: createRISCVO0PreLegalizerCombiner());
532 } else {
533 addPass(P: createRISCVPreLegalizerCombiner());
534 }
535}
536
537bool RISCVPassConfig::addLegalizeMachineIR() {
538 addPass(P: new LegalizerLegacy());
539 return false;
540}
541
542void RISCVPassConfig::addPreRegBankSelect() {
543 if (getOptLevel() != CodeGenOptLevel::None)
544 addPass(P: createRISCVPostLegalizerCombiner());
545}
546
547bool RISCVPassConfig::addRegBankSelect() {
548 addPass(P: new RegBankSelectLegacy());
549 return false;
550}
551
552bool RISCVPassConfig::addGlobalInstructionSelect() {
553 addPass(P: new InstructionSelectLegacy(getOptLevel()));
554 return false;
555}
556
557void RISCVPassConfig::addPreSched2() {
558 addPass(P: createRISCVExpandPseudoPostRALegacyPass());
559
560 // Emit KCFI checks for indirect calls.
561 addPass(P: createKCFIPass());
562 if (TM->getOptLevel() != CodeGenOptLevel::None)
563 addPass(P: createRISCVLoadStoreOptPass());
564}
565
566void RISCVPassConfig::addPreEmitPass() {
567 // TODO: It would potentially be better to schedule copy propagation after
568 // expanding pseudos (in addPreEmitPass2). However, performing copy
569 // propagation after the machine outliner (which runs after addPreEmitPass)
570 // currently leads to incorrect code-gen, where copies to registers within
571 // outlined functions are removed erroneously.
572 if (TM->getOptLevel() >= CodeGenOptLevel::Default &&
573 EnableRISCVCopyPropagation)
574 addPass(P: createMachineCopyPropagationPass(UseCopyInstr: true));
575 if (TM->getOptLevel() >= CodeGenOptLevel::Default)
576 addPass(P: createRISCVLateBranchOptPass());
577 // The IndirectBranchTrackingPass inserts lpad and could have changed the
578 // basic block alignment. It must be done before Branch Relaxation to
579 // prevent the adjusted offset exceeding the branch range.
580 addPass(P: createRISCVIndirectBranchTrackingPass());
581 addPass(PassID: &BranchRelaxationPassID);
582 addPass(P: createRISCVMakeCompressibleOptPass());
583}
584
585void RISCVPassConfig::addPreEmitPass2() {
586 if (TM->getOptLevel() != CodeGenOptLevel::None) {
587 addPass(P: createRISCVMoveMergePass());
588 // Schedule PushPop Optimization before expansion of Pseudo instruction,
589 // ensuring return instruction is detected correctly.
590 addPass(P: createRISCVPushPopOptimizationPass());
591 }
592 addPass(P: createRISCVExpandPseudoPreEmitLegacyPass());
593
594 // Add QC Relaxation Markers as late as possible, and only for RV32
595 if (TM->getOptLevel() != CodeGenOptLevel::None &&
596 TM->getTargetTriple().isRISCV32())
597 addPass(P: createRISCVQCRelaxMarkingPass());
598
599 // Schedule the expansion of AMOs at the last possible moment, avoiding the
600 // possibility for other passes to break the requirements for forward
601 // progress in the LR/SC block.
602 addPass(P: createRISCVExpandPseudoAtomicsLegacyPass());
603
604 // KCFI indirect call checks are lowered to a bundle.
605 addPass(P: createUnpackMachineBundlesLegacy(Ftor: [&](const MachineFunction &MF) {
606 return MF.getFunction().getParent()->getModuleFlag(Key: "kcfi");
607 }));
608
609 if (EnableCFIInstrInserter)
610 addPass(P: createCFIInstrInserterLegacy());
611}
612
613void RISCVPassConfig::addMachineSSAOptimization() {
614 // It's beneficial to reduce the VL to enable more
615 // Machine SSA optimizations.
616 if (TM->getOptLevel() != CodeGenOptLevel::None) {
617 // RISCVVLOptimizer can make loop invariant instructions like vmv.v.i
618 // loop variant by propagating a VL defined inside the loop. Run LICM and
619 // hoist them early. Don't do this at -O0 to avoid the compile-time
620 // overhead. Not reducing the VL of loop invariant pseudos results in more
621 // vsetvli toggles, and still requires the MachineLoopInfo analysis to be
622 // run.
623 addPass(PassID: &EarlyMachineLICMID);
624 addPass(P: createRISCVVLOptimizerLegacyPass());
625 }
626
627 addPass(P: createRISCVVectorPeepholeLegacyPass());
628 addPass(P: createRISCVFoldMemOffsetLegacyPass());
629
630 TargetPassConfig::addMachineSSAOptimization();
631
632 if (TM->getTargetTriple().isRISCV64()) {
633 addPass(P: createRISCVOptWInstrsLegacyPass());
634 }
635}
636
637void RISCVPassConfig::addPreRegAlloc() {
638 addPass(P: createRISCVExpandPseudoPreRALegacyPass());
639 if (TM->getOptLevel() != CodeGenOptLevel::None) {
640 addPass(P: createRISCVMergeBaseOffsetOptPass());
641 // Add Zilsd pre-allocation load/store optimization
642 addPass(P: createRISCVPreAllocZilsdOptPass());
643 }
644
645 addPass(P: createRISCVInsertReadWriteCSRPass());
646 addPass(P: createRISCVInsertWriteVXRMPass());
647 addPass(P: createRISCVLandingPadSetupPass());
648
649 if (TM->getOptLevel() != CodeGenOptLevel::None && EnableMachinePipeliner)
650 addPass(PassID: &MachinePipelinerID);
651
652 addPass(P: createRISCVVMV0EliminationPass());
653}
654
655void RISCVPassConfig::addFastRegAlloc() {
656 addPass(PassID: &InitUndefID);
657 TargetPassConfig::addFastRegAlloc();
658}
659
660
661void RISCVPassConfig::addPostRegAlloc() {
662 if (TM->getOptLevel() != CodeGenOptLevel::None &&
663 EnableRedundantCopyElimination)
664 addPass(P: createRISCVRedundantCopyEliminationPass());
665}
666
667bool RISCVPassConfig::addILPOpts() {
668 if (EnableMachineCombiner)
669 addPass(PassID: &MachineCombinerID);
670
671 return true;
672}
673
674yaml::MachineFunctionInfo *
675RISCVTargetMachine::createDefaultFuncInfoYAML() const {
676 return new yaml::RISCVMachineFunctionInfo();
677}
678
679yaml::MachineFunctionInfo *
680RISCVTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
681 const auto *MFI = MF.getInfo<RISCVMachineFunctionInfo>();
682 return new yaml::RISCVMachineFunctionInfo(*MFI);
683}
684
685bool RISCVTargetMachine::parseMachineFunctionInfo(
686 const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
687 SMDiagnostic &Error, SMRange &SourceRange) const {
688 const auto &YamlMFI =
689 static_cast<const yaml::RISCVMachineFunctionInfo &>(MFI);
690 PFS.MF.getInfo<RISCVMachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
691 return false;
692}
693