1//===-- RISCVTargetMachine.cpp - Define TargetMachine for RISC-V ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Implements the info about RISC-V target spec.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVTargetMachine.h"
14#include "RISCV.h"
15#include "RISCVMachineFunctionInfo.h"
16#include "RISCVMachineScheduler.h"
17#include "RISCVTargetObjectFile.h"
18#include "RISCVTargetTransformInfo.h"
19#include "TargetInfo/RISCVTargetInfo.h"
20#include "llvm/Analysis/TargetTransformInfo.h"
21#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
22#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
23#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
24#include "llvm/CodeGen/GlobalISel/Legalizer.h"
25#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
26#include "llvm/CodeGen/MIRParser/MIParser.h"
27#include "llvm/CodeGen/MIRYamlMapping.h"
28#include "llvm/CodeGen/MachineScheduler.h"
29#include "llvm/CodeGen/MacroFusion.h"
30#include "llvm/CodeGen/Passes.h"
31#include "llvm/CodeGen/RegAllocRegistry.h"
32#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
33#include "llvm/CodeGen/TargetPassConfig.h"
34#include "llvm/InitializePasses.h"
35#include "llvm/MC/TargetRegistry.h"
36#include "llvm/Option/LibraryOptions.h"
37#include "llvm/Support/Compiler.h"
38#include "llvm/Target/TargetOptions.h"
39#include "llvm/Transforms/IPO.h"
40#include "llvm/Transforms/Scalar.h"
41#include <optional>
42using namespace llvm;
43
44extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
45 RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
46 RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
47 RegisterTargetMachine<RISCVTargetMachine> A(getTheRISCV32beTarget());
48 RegisterTargetMachine<RISCVTargetMachine> B(getTheRISCV64beTarget());
49 static opt::RegisterLibraryOptions<RISCVOptions> O;
50 auto *PR = PassRegistry::getPassRegistry();
51 initializeGlobalISel(*PR);
52 initializeRISCVO0PreLegalizerCombinerLegacyPass(*PR);
53 initializeRISCVPreLegalizerCombinerLegacyPass(*PR);
54 initializeRISCVPostLegalizerCombinerLegacyPass(*PR);
55 initializeMachineKCFILegacyPass(*PR);
56 initializeRISCVDeadRegisterDefinitionsLegacyPass(*PR);
57 initializeRISCVLateBranchOptLegacyPass(*PR);
58 initializeRISCVMakeCompressibleLegacyPass(*PR);
59 initializeRISCVQCRelaxMarkingLegacyPass(*PR);
60 initializeRISCVGatherScatterLoweringLegacyPass(*PR);
61 initializeRISCVCodeGenPrepareLegacyPass(*PR);
62 initializeRISCVZacasABIFixLegacyPass(*PR);
63 initializeRISCVExpandPseudoPostRALegacyPass(*PR);
64 initializeRISCVMergeBaseOffsetLegacyPass(*PR);
65 initializeRISCVOptWInstrsLegacyPass(*PR);
66 initializeRISCVFoldMemOffsetLegacyPass(*PR);
67 initializeRISCVExpandPseudoPreRALegacyPass(*PR);
68 initializeRISCVExpandPseudoPreEmitLegacyPass(*PR);
69 initializeRISCVVectorPeepholeLegacyPass(*PR);
70 initializeRISCVVLOptimizerLegacyPass(*PR);
71 initializeRISCVVMV0EliminationLegacyPass(*PR);
72 initializeRISCVInsertVSETVLILegacyPass(*PR);
73 initializeRISCVInsertReadWriteCSRLegacyPass(*PR);
74 initializeRISCVInsertWriteVXRMLegacyPass(*PR);
75 initializeRISCVLandingPadSetupLegacyPass(*PR);
76 initializeRISCVDAGToDAGISelLegacyPass(*PR);
77 initializeRISCVMoveMergeLegacyPass(*PR);
78 initializeRISCVPushPopOptLegacyPass(*PR);
79 initializeRISCVIndirectBranchTrackingLegacyPass(*PR);
80 initializeRISCVLoadStoreOptLegacyPass(*PR);
81 initializeRISCVPreAllocZilsdOptLegacyPass(*PR);
82 initializeRISCVExpandPseudoAtomicsLegacyPass(*PR);
83 initializeRISCVRedundantCopyEliminationLegacyPass(*PR);
84 initializeRISCVAsmPrinterPass(*PR);
85 initializeRISCVPromoteConstantLegacyPass(*PR);
86}
87
88static Reloc::Model getEffectiveRelocModel(const Triple &TT,
89 std::optional<Reloc::Model> RM) {
90 if (TT.isOSBinFormatMachO())
91 return RM.value_or(u: Reloc::PIC_);
92
93 return RM.value_or(u: Reloc::Static);
94}
95
96static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
97 if (TT.isOSBinFormatMachO())
98 return std::make_unique<RISCVMachOTargetObjectFile>();
99 return std::make_unique<RISCVELFTargetObjectFile>();
100}
101
102RISCVTargetMachine::RISCVTargetMachine(const Target &T, const Triple &TT,
103 StringRef CPU, StringRef FS,
104 const TargetOptions &Options,
105 std::optional<Reloc::Model> RM,
106 std::optional<CodeModel::Model> CM,
107 CodeGenOptLevel OL, bool JIT)
108 : CodeGenTargetMachineImpl(T, TT, CPU, FS, Options,
109 getEffectiveRelocModel(TT, RM),
110 getEffectiveCodeModel(CM, Default: CodeModel::Small), OL),
111 CLOpts(cl::LibraryOptionsContext::get<RISCVOptions>(
112 C: Options.MCOptions.LibraryOptions)),
113 TLOF(createTLOF(TT)) {
114 initAsmInfo();
115
116 LLT::setUseExtended(true);
117
118 // RISC-V supports the MachineOutliner.
119 setMachineOutliner(true);
120 setSupportsDefaultOutlining(true);
121
122 // RISC-V supports the debug entry values.
123 setSupportsDebugEntryValues(true);
124
125 if (TT.isOSFuchsia() && !TT.isArch64Bit())
126 report_fatal_error(reason: "Fuchsia is only supported for 64-bit");
127
128 setCFIFixup(!CLOpts.enable_cfi_instr_inserter);
129}
130
131const RISCVSubtarget *
132RISCVTargetMachine::getSubtargetImpl(const Function &F) const {
133 AttributeSet FnAttrs = F.getAttributes().getFnAttrs();
134 const Module *M = F.getParent();
135 if (const RISCVSubtarget *ST = LastSubtarget.lookup(NewFnAttrs: FnAttrs, NewM: M))
136 return ST;
137
138 SubtargetAttrs Attrs;
139 Attrs.CPU = FnAttrs.getAttribute(Kind: "target-cpu");
140 Attrs.TuneCPU = FnAttrs.getAttribute(Kind: "tune-cpu");
141 Attrs.FS = FnAttrs.getAttribute(Kind: "target-features");
142 Attrs.VScaleRange = FnAttrs.getAttribute(Kind: Attribute::VScaleRange);
143
144 if (const RISCVSubtarget *ST = LastSubtarget.lookup(NewFnAttrs: FnAttrs, NewM: M, NewAttrs: Attrs))
145 return ST;
146
147 StringRef CPU =
148 Attrs.CPU.isValid() ? Attrs.CPU.getValueAsString() : TargetCPU;
149 StringRef TuneCPU =
150 Attrs.TuneCPU.isValid() ? Attrs.TuneCPU.getValueAsString() : CPU;
151 StringRef FS = Attrs.FS.isValid() ? Attrs.FS.getValueAsString() : TargetFS;
152
153 unsigned RVVBitsMin = CLOpts.v_vector_bits_min.value_or(u: -1);
154 unsigned RVVBitsMax = CLOpts.v_vector_bits_max.value_or(u: 0);
155
156 if (Attrs.VScaleRange.isValid()) {
157 if (!CLOpts.v_vector_bits_min)
158 RVVBitsMin =
159 Attrs.VScaleRange.getVScaleRangeMin() * RISCV::RVVBitsPerBlock;
160 std::optional<unsigned> VScaleMax = Attrs.VScaleRange.getVScaleRangeMax();
161 if (VScaleMax.has_value() && !CLOpts.v_vector_bits_max)
162 RVVBitsMax = *VScaleMax * RISCV::RVVBitsPerBlock;
163 }
164
165 if (RVVBitsMin != -1U) {
166 // FIXME: Change to >= 32 when VLEN = 32 is supported.
167 assert((RVVBitsMin == 0 || (RVVBitsMin >= 64 && RVVBitsMin <= 65536 &&
168 isPowerOf2_32(RVVBitsMin))) &&
169 "V or Zve* extension requires vector length to be in the range of "
170 "64 to 65536 and a power 2!");
171 assert((RVVBitsMax >= RVVBitsMin || RVVBitsMax == 0) &&
172 "Minimum V extension vector length should not be larger than its "
173 "maximum!");
174 }
175 assert((RVVBitsMax == 0 || (RVVBitsMax >= 64 && RVVBitsMax <= 65536 &&
176 isPowerOf2_32(RVVBitsMax))) &&
177 "V or Zve* extension requires vector length to be in the range of "
178 "64 to 65536 and a power 2!");
179
180 if (RVVBitsMin != -1U) {
181 if (RVVBitsMax != 0) {
182 RVVBitsMin = std::min(a: RVVBitsMin, b: RVVBitsMax);
183 RVVBitsMax = std::max(a: RVVBitsMin, b: RVVBitsMax);
184 }
185
186 RVVBitsMin = llvm::bit_floor(
187 Value: (RVVBitsMin < 64 || RVVBitsMin > 65536) ? 0 : RVVBitsMin);
188 }
189 RVVBitsMax =
190 llvm::bit_floor(Value: (RVVBitsMax < 64 || RVVBitsMax > 65536) ? 0 : RVVBitsMax);
191
192 SmallString<512> Key;
193 raw_svector_ostream(Key) << "RVVMin" << RVVBitsMin << "RVVMax" << RVVBitsMax
194 << CPU << TuneCPU << FS;
195 auto &I = SubtargetMap[Key];
196 if (!I) {
197 StringRef ABIName = getTargetABIName(M: *F.getParent());
198 I = std::make_unique<RISCVSubtarget>(
199 args: TargetTriple, args&: CPU, args&: TuneCPU, args&: FS, args&: ABIName, args&: RVVBitsMin, args&: RVVBitsMax, args: *this);
200 }
201 LastSubtarget.set(NewFnAttrs: FnAttrs, NewM: M, NewAttrs: Attrs, NewSubtarget: I.get());
202 return I.get();
203}
204
205MachineFunctionInfo *RISCVTargetMachine::createMachineFunctionInfo(
206 BumpPtrAllocator &Allocator, const Function &F,
207 const TargetSubtargetInfo *STI) const {
208 return RISCVMachineFunctionInfo::create<RISCVMachineFunctionInfo>(
209 Allocator, F, STI: static_cast<const RISCVSubtarget *>(STI));
210}
211
212TargetTransformInfo
213RISCVTargetMachine::getTargetTransformInfo(const Function &F) const {
214 return TargetTransformInfo(std::make_unique<RISCVTTIImpl>(args: this, args: F));
215}
216
217// A RISC-V hart has a single byte-addressable address space of 2^XLEN bytes
218// for all memory accesses, so it is reasonable to assume that an
219// implementation has no-op address space casts. If an implementation makes a
220// change to this, they can override it here.
221bool RISCVTargetMachine::isNoopAddrSpaceCast(const DataLayout &DL,
222 unsigned SrcAS,
223 unsigned DstAS) const {
224 return true;
225}
226
227ScheduleDAGInstrs *
228RISCVTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
229 const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
230 ScheduleDAGMILive *DAG = createSchedLive<RISCVPreRAMachineSchedStrategy>(C);
231
232 // Add MacroFusion mutation first with a higher priority than later clustering
233 const auto &MacroFusions = ST.getMacroFusions();
234 if (!MacroFusions.empty())
235 DAG->addMutation(Mutation: createMacroFusionDAGMutation(Predicates: MacroFusions));
236
237 if (ST.enableMISchedLoadClustering())
238 DAG->addMutation(Mutation: createLoadClusterDAGMutation(
239 TII: DAG->TII, /*ReorderWhileClustering=*/true));
240
241 if (ST.enableMISchedStoreClustering())
242 DAG->addMutation(Mutation: createStoreClusterDAGMutation(
243 TII: DAG->TII, /*ReorderWhileClustering=*/true));
244
245 if (CLOpts.vector_mask_mutation && ST.hasVInstructions())
246 DAG->addMutation(Mutation: createRISCVVectorMaskDAGMutation(TRI: DAG->TRI));
247
248 return DAG;
249}
250
251ScheduleDAGInstrs *
252RISCVTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
253 const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
254 ScheduleDAGMI *DAG = createSchedPostRA(C);
255
256 // Add MacroFusion mutation first with a higher priority than later clustering
257 const auto &MacroFusions = ST.getMacroFusions();
258 if (!MacroFusions.empty())
259 DAG->addMutation(Mutation: createMacroFusionDAGMutation(Predicates: MacroFusions));
260
261 if (ST.enablePostMISchedLoadClustering())
262 DAG->addMutation(Mutation: createLoadClusterDAGMutation(
263 TII: DAG->TII, /*ReorderWhileClustering=*/true));
264
265 if (ST.enablePostMISchedStoreClustering())
266 DAG->addMutation(Mutation: createStoreClusterDAGMutation(
267 TII: DAG->TII, /*ReorderWhileClustering=*/true));
268
269 return DAG;
270}
271
272namespace {
273
274class RVVRegisterRegAlloc : public RegisterRegAllocBase<RVVRegisterRegAlloc> {
275public:
276 RVVRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
277 : RegisterRegAllocBase(N, D, C) {}
278};
279
280static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI,
281 const MachineRegisterInfo &MRI,
282 const Register Reg) {
283 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
284 return RISCVRegisterInfo::isRVVRegClass(RC);
285}
286
287static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
288
289static llvm::once_flag InitializeDefaultRVVRegisterAllocatorFlag;
290
291/// -riscv-rvv-regalloc=<fast|basic|greedy> command line option.
292/// This option could designate the rvv register allocator only.
293/// For example: -riscv-rvv-regalloc=basic
294static cl::opt<RVVRegisterRegAlloc::FunctionPassCtor, false,
295 RegisterPassParser<RVVRegisterRegAlloc>>
296 RVVRegAlloc("riscv-rvv-regalloc", cl::Hidden,
297 cl::init(Val: &useDefaultRegisterAllocator),
298 cl::desc("Register allocator to use for RVV register."));
299
300static void initializeDefaultRVVRegisterAllocatorOnce() {
301 RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
302
303 if (!Ctor) {
304 Ctor = RVVRegAlloc;
305 RVVRegisterRegAlloc::setDefault(RVVRegAlloc);
306 }
307}
308
309static FunctionPass *createBasicRVVRegisterAllocator() {
310 return createBasicRegisterAllocator(F: onlyAllocateRVVReg);
311}
312
313static FunctionPass *createGreedyRVVRegisterAllocator() {
314 return createGreedyRegisterAllocator(F: onlyAllocateRVVReg);
315}
316
317static FunctionPass *createFastRVVRegisterAllocator() {
318 return createFastRegisterAllocator(F: onlyAllocateRVVReg, ClearVirtRegs: false);
319}
320
321static RVVRegisterRegAlloc basicRegAllocRVVReg("basic",
322 "basic register allocator",
323 createBasicRVVRegisterAllocator);
324static RVVRegisterRegAlloc
325 greedyRegAllocRVVReg("greedy", "greedy register allocator",
326 createGreedyRVVRegisterAllocator);
327
328static RVVRegisterRegAlloc fastRegAllocRVVReg("fast", "fast register allocator",
329 createFastRVVRegisterAllocator);
330
331class RISCVPassConfig : public TargetPassConfig {
332 const RISCVOptions &CLOpts;
333
334public:
335 RISCVPassConfig(RISCVTargetMachine &TM, PassManagerBase &PM)
336 : TargetPassConfig(TM, PM), CLOpts(TM.getCLOpts()) {
337 if (TM.getOptLevel() != CodeGenOptLevel::None)
338 substitutePass(StandardID: &PostRASchedulerID, TargetID: &PostMachineSchedulerID);
339 setEnableSinkAndFold(CLOpts.enable_sink_fold);
340 EnableLoopTermFold = true;
341 }
342
343 RISCVTargetMachine &getRISCVTargetMachine() const {
344 return getTM<RISCVTargetMachine>();
345 }
346
347 void addIRPasses() override;
348 bool addPreISel() override;
349 void addCodeGenPrepare() override;
350 bool addInstSelector() override;
351 bool addIRTranslator() override;
352 void addPreLegalizeMachineIR() override;
353 bool addLegalizeMachineIR() override;
354 void addPreRegBankSelect() override;
355 bool addRegBankSelect() override;
356 bool addGlobalInstructionSelect() override;
357 void addPreEmitPass() override;
358 void addPreEmitPass2() override;
359 void addPreSched2() override;
360 void addMachineSSAOptimization() override;
361 FunctionPass *createRVVRegAllocPass(bool Optimized);
362 bool addRegAssignAndRewriteFast() override;
363 bool addRegAssignAndRewriteOptimized() override;
364 void addPreRegAlloc() override;
365 void addPostRegAlloc() override;
366 void addFastRegAlloc() override;
367 bool addILPOpts() override;
368
369 std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
370};
371} // namespace
372
373TargetPassConfig *RISCVTargetMachine::createPassConfig(PassManagerBase &PM) {
374 return new RISCVPassConfig(*this, PM);
375}
376
377std::unique_ptr<CSEConfigBase> RISCVPassConfig::getCSEConfig() const {
378 return getStandardCSEConfigForOpt(Level: TM->getOptLevel());
379}
380
381FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {
382 // Initialize the global default.
383 llvm::call_once(flag&: InitializeDefaultRVVRegisterAllocatorFlag,
384 F&: initializeDefaultRVVRegisterAllocatorOnce);
385
386 RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
387 if (Ctor != useDefaultRegisterAllocator)
388 return Ctor();
389
390 if (Optimized)
391 return createGreedyRVVRegisterAllocator();
392
393 return createFastRVVRegisterAllocator();
394}
395
396bool RISCVPassConfig::addRegAssignAndRewriteFast() {
397 addPass(P: createRVVRegAllocPass(Optimized: false));
398 addPass(P: createRISCVInsertVSETVLILegacyPass());
399 if (TM->getOptLevel() != CodeGenOptLevel::None && CLOpts.enable_dead_defs)
400 addPass(P: createRISCVDeadRegisterDefinitionsLegacyPass());
401 return TargetPassConfig::addRegAssignAndRewriteFast();
402}
403
404bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {
405 addPass(P: createRVVRegAllocPass(Optimized: true));
406 addPass(P: createVirtRegRewriter(ClearVirtRegs: false));
407 addPass(P: createRISCVInsertVSETVLILegacyPass());
408 if (TM->getOptLevel() != CodeGenOptLevel::None && CLOpts.enable_dead_defs)
409 addPass(P: createRISCVDeadRegisterDefinitionsLegacyPass());
410 return TargetPassConfig::addRegAssignAndRewriteOptimized();
411}
412
413void RISCVPassConfig::addIRPasses() {
414 addPass(P: createAtomicExpandLegacyPass());
415 addPass(P: createRISCVZacasABIFixLegacyPass());
416
417 if (getOptLevel() != CodeGenOptLevel::None) {
418 if (CLOpts.enable_loop_data_prefetch)
419 addPass(P: createLoopDataPrefetchPass());
420
421 addPass(P: createRISCVGatherScatterLoweringLegacyPass());
422 addPass(P: createInterleavedAccessPass());
423 addPass(P: createRISCVCodeGenPrepareLegacyPass());
424 }
425
426 TargetPassConfig::addIRPasses();
427
428 if (getOptLevel() == CodeGenOptLevel::Aggressive && CLOpts.select_opt)
429 addPass(P: createSelectOptimizePass());
430}
431
432bool RISCVPassConfig::addPreISel() {
433 if (TM->getOptLevel() != CodeGenOptLevel::None)
434 addPass(P: createRISCVPromoteConstantLegacyPass());
435 if (TM->getOptLevel() != CodeGenOptLevel::None) {
436 // Add a barrier before instruction selection so that we will not get
437 // deleted block address after enabling default outlining. See D99707 for
438 // more details.
439 addPass(P: createBarrierNoopPass());
440 }
441
442 if (valueOr(X: CLOpts.enable_global_merge,
443 Default: TM->getOptLevel() != CodeGenOptLevel::None)) {
444 // FIXME: Like AArch64, we disable extern global merging by default due to
445 // concerns it might regress some workloads. Unlike AArch64, we don't
446 // currently support enabling the pass in an "OnlyOptimizeForSize" mode.
447 // Investigating and addressing both items are TODO.
448 addPass(P: createGlobalMergePass(TM, /* MaxOffset */ MaximalOffset: 2047,
449 /* OnlyOptimizeForSize */ false,
450 /* MergeExternalByDefault */ true));
451 }
452
453 return false;
454}
455
456void RISCVPassConfig::addCodeGenPrepare() {
457 if (getOptLevel() != CodeGenOptLevel::None)
458 addPass(P: createTypePromotionLegacyPass());
459 TargetPassConfig::addCodeGenPrepare();
460}
461
462bool RISCVPassConfig::addInstSelector() {
463 addPass(P: createRISCVISelDagLegacyPass(TM&: getRISCVTargetMachine(), OptLevel: getOptLevel()));
464
465 return false;
466}
467
468bool RISCVPassConfig::addIRTranslator() {
469 addPass(P: new IRTranslatorLegacy(getOptLevel()));
470 return false;
471}
472
473void RISCVPassConfig::addPreLegalizeMachineIR() {
474 if (getOptLevel() == CodeGenOptLevel::None) {
475 addPass(P: createRISCVO0PreLegalizerCombinerLegacyPass());
476 } else {
477 addPass(P: createRISCVPreLegalizerCombinerLegacyPass());
478 }
479}
480
481bool RISCVPassConfig::addLegalizeMachineIR() {
482 addPass(P: new LegalizerLegacy());
483 return false;
484}
485
486void RISCVPassConfig::addPreRegBankSelect() {
487 if (getOptLevel() != CodeGenOptLevel::None)
488 addPass(P: createRISCVPostLegalizerCombinerLegacyPass());
489}
490
491bool RISCVPassConfig::addRegBankSelect() {
492 addPass(P: new RegBankSelectLegacy());
493 return false;
494}
495
496bool RISCVPassConfig::addGlobalInstructionSelect() {
497 addPass(P: new InstructionSelectLegacy(getOptLevel()));
498 return false;
499}
500
501void RISCVPassConfig::addPreSched2() {
502 addPass(P: createRISCVExpandPseudoPostRALegacyPass());
503
504 // Emit KCFI checks for indirect calls.
505 addPass(P: createKCFIPass());
506 if (TM->getOptLevel() != CodeGenOptLevel::None)
507 addPass(P: createRISCVLoadStoreOptLegacyPass());
508}
509
510void RISCVPassConfig::addPreEmitPass() {
511 // TODO: It would potentially be better to schedule copy propagation after
512 // expanding pseudos (in addPreEmitPass2). However, performing copy
513 // propagation after the machine outliner (which runs after addPreEmitPass)
514 // currently leads to incorrect code-gen, where copies to registers within
515 // outlined functions are removed erroneously.
516 if (TM->getOptLevel() >= CodeGenOptLevel::Default &&
517 CLOpts.enable_copy_propagation)
518 addPass(P: createMachineCopyPropagationPass(UseCopyInstr: true));
519 if (TM->getOptLevel() >= CodeGenOptLevel::Default)
520 addPass(P: createRISCVLateBranchOptLegacyPass());
521 // The IndirectBranchTrackingPass inserts lpad and could have changed the
522 // basic block alignment. It must be done before Branch Relaxation to
523 // prevent the adjusted offset exceeding the branch range.
524 addPass(P: createRISCVIndirectBranchTrackingLegacyPass());
525 addPass(PassID: &BranchRelaxationPassID);
526 addPass(P: createRISCVMakeCompressibleLegacyPass());
527}
528
529void RISCVPassConfig::addPreEmitPass2() {
530 if (TM->getOptLevel() != CodeGenOptLevel::None) {
531 addPass(P: createRISCVMoveMergeLegacyPass());
532 // Schedule PushPop Optimization before expansion of Pseudo instruction,
533 // ensuring return instruction is detected correctly.
534 addPass(P: createRISCVPushPopOptLegacyPass());
535 }
536 addPass(P: createRISCVExpandPseudoPreEmitLegacyPass());
537
538 // Add QC Relaxation Markers as late as possible, and only for RV32
539 if (TM->getOptLevel() != CodeGenOptLevel::None &&
540 TM->getTargetTriple().isRISCV32())
541 addPass(P: createRISCVQCRelaxMarkingLegacyPass());
542
543 // Schedule the expansion of AMOs at the last possible moment, avoiding the
544 // possibility for other passes to break the requirements for forward
545 // progress in the LR/SC block.
546 addPass(P: createRISCVExpandPseudoAtomicsLegacyPass());
547
548 // KCFI indirect call checks are lowered to a bundle.
549 addPass(P: createUnpackMachineBundlesLegacy(Ftor: [&](const MachineFunction &MF) {
550 return MF.getFunction().getParent()->getModuleFlag(Key: "kcfi");
551 }));
552
553 if (CLOpts.enable_cfi_instr_inserter)
554 addPass(P: createCFIInstrInserterLegacy());
555}
556
557void RISCVPassConfig::addMachineSSAOptimization() {
558 // It's beneficial to reduce the VL to enable more
559 // Machine SSA optimizations.
560 if (TM->getOptLevel() != CodeGenOptLevel::None) {
561 // RISCVVLOptimizer can make loop invariant instructions like vmv.v.i
562 // loop variant by propagating a VL defined inside the loop. Run LICM and
563 // hoist them early. Don't do this at -O0 to avoid the compile-time
564 // overhead. Not reducing the VL of loop invariant pseudos results in more
565 // vsetvli toggles, and still requires the MachineLoopInfo analysis to be
566 // run.
567 addPass(PassID: &EarlyMachineLICMID);
568 addPass(P: createRISCVVLOptimizerLegacyPass());
569 }
570
571 addPass(P: createRISCVVectorPeepholeLegacyPass());
572 addPass(P: createRISCVFoldMemOffsetLegacyPass());
573
574 TargetPassConfig::addMachineSSAOptimization();
575
576 if (TM->getTargetTriple().isRISCV64()) {
577 addPass(P: createRISCVOptWInstrsLegacyPass());
578 }
579}
580
581void RISCVPassConfig::addPreRegAlloc() {
582 addPass(P: createRISCVExpandPseudoPreRALegacyPass());
583 if (TM->getOptLevel() != CodeGenOptLevel::None) {
584 addPass(P: createRISCVMergeBaseOffsetLegacyPass());
585 addPass(P: createRISCVPreAllocZilsdOptLegacyPass());
586 }
587
588 addPass(P: createRISCVInsertReadWriteCSRLegacyPass());
589 addPass(P: createRISCVInsertWriteVXRMLegacyPass());
590 addPass(P: createRISCVLandingPadSetupLegacyPass());
591 addPass(P: createRISCVVMV0EliminationLegacyPass());
592}
593
594void RISCVPassConfig::addFastRegAlloc() {
595 addPass(PassID: &InitUndefID);
596 TargetPassConfig::addFastRegAlloc();
597}
598
599
600void RISCVPassConfig::addPostRegAlloc() {
601 if (TM->getOptLevel() != CodeGenOptLevel::None && CLOpts.enable_copyelim)
602 addPass(P: createRISCVRedundantCopyEliminationLegacyPass());
603}
604
605bool RISCVPassConfig::addILPOpts() {
606 if (CLOpts.enable_machine_combiner)
607 addPass(PassID: &MachineCombinerID);
608
609 return true;
610}
611
612yaml::MachineFunctionInfo *
613RISCVTargetMachine::createDefaultFuncInfoYAML() const {
614 return new yaml::RISCVMachineFunctionInfo();
615}
616
617yaml::MachineFunctionInfo *
618RISCVTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
619 const auto *MFI = MF.getInfo<RISCVMachineFunctionInfo>();
620 return new yaml::RISCVMachineFunctionInfo(*MFI);
621}
622
623bool RISCVTargetMachine::parseMachineFunctionInfo(
624 const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
625 SMDiagnostic &Error, SMRange &SourceRange) const {
626 const auto &YamlMFI =
627 static_cast<const yaml::RISCVMachineFunctionInfo &>(MFI);
628 PFS.MF.getInfo<RISCVMachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
629 return false;
630}
631