1//===-- RISCVFrameLowering.cpp - RISC-V Frame Information -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the RISC-V implementation of TargetFrameLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVFrameLowering.h"
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "MCTargetDesc/RISCVMCTargetDesc.h"
16#include "RISCVMachineFunctionInfo.h"
17#include "RISCVSubtarget.h"
18#include "llvm/BinaryFormat/Dwarf.h"
19#include "llvm/CodeGen/CFIInstBuilder.h"
20#include "llvm/CodeGen/LivePhysRegs.h"
21#include "llvm/CodeGen/MachineFrameInfo.h"
22#include "llvm/CodeGen/MachineFunction.h"
23#include "llvm/CodeGen/MachineInstrBuilder.h"
24#include "llvm/CodeGen/MachineRegisterInfo.h"
25#include "llvm/CodeGen/RegisterScavenging.h"
26#include "llvm/CodeGen/TargetFrameLowering.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/MC/MCDwarf.h"
29#include "llvm/Support/LEB128.h"
30
31#include <algorithm>
32#include <cstdint>
33
34#define DEBUG_TYPE "riscv-frame"
35
36using namespace llvm;
37
38RISCVFrameLowering::RISCVFrameLowering(const RISCVSubtarget &STI)
39 : TargetFrameLowering(StackGrowsDown,
40 RISCVABI::getStackAlignment(TargetABI: STI.getTargetABI()),
41 /*LocalAreaOffset=*/0,
42 /*TransientStackAlignment=*/
43 RISCVABI::getStackAlignment(TargetABI: STI.getTargetABI())),
44 STI(STI) {}
45
46// The register used to hold the frame pointer.
47static constexpr MCPhysReg FPReg = RISCV::X8;
48
49// The register used to hold the stack pointer.
50static constexpr MCPhysReg SPReg = RISCV::X2;
51
52// The register used to hold the return address.
53static constexpr MCPhysReg RAReg = RISCV::X1;
54
55// LIst of CSRs that are given a fixed location by save/restore libcalls or
56// Zcmp/Xqccmp Push/Pop. The order in this table indicates the order the
57// registers are saved on the stack. Zcmp uses the reverse order of save/restore
58// and Xqccmp on the stack, but this is handled when offsets are calculated.
59static const MCPhysReg FixedCSRFIMap[] = {
60 /*ra*/ RAReg, /*s0*/ FPReg, /*s1*/ RISCV::X9,
61 /*s2*/ RISCV::X18, /*s3*/ RISCV::X19, /*s4*/ RISCV::X20,
62 /*s5*/ RISCV::X21, /*s6*/ RISCV::X22, /*s7*/ RISCV::X23,
63 /*s8*/ RISCV::X24, /*s9*/ RISCV::X25, /*s10*/ RISCV::X26,
64 /*s11*/ RISCV::X27};
65
66// The number of stack bytes allocated by `QC.C.MIENTER(.NEST)` and popped by
67// `QC.C.MILEAVERET`.
68static constexpr uint64_t QCIInterruptPushAmount = 96;
69
70static const std::pair<MCPhysReg, int8_t> FixedCSRFIQCIInterruptMap[] = {
71 /* -1 is a gap for mepc/mnepc */
72 {/*fp*/ FPReg, -2},
73 /* -3 is a gap for qc.mcause */
74 {/*ra*/ RAReg, -4},
75 /* -5 is reserved */
76 {/*t0*/ RISCV::X5, -6},
77 {/*t1*/ RISCV::X6, -7},
78 {/*t2*/ RISCV::X7, -8},
79 {/*a0*/ RISCV::X10, -9},
80 {/*a1*/ RISCV::X11, -10},
81 {/*a2*/ RISCV::X12, -11},
82 {/*a3*/ RISCV::X13, -12},
83 {/*a4*/ RISCV::X14, -13},
84 {/*a5*/ RISCV::X15, -14},
85 {/*a6*/ RISCV::X16, -15},
86 {/*a7*/ RISCV::X17, -16},
87 {/*t3*/ RISCV::X28, -17},
88 {/*t4*/ RISCV::X29, -18},
89 {/*t5*/ RISCV::X30, -19},
90 {/*t6*/ RISCV::X31, -20},
91 /* -21, -22, -23, -24 are reserved */
92};
93
94/// Returns true if DWARF CFI instructions ("frame moves") should be emitted.
95static bool needsDwarfCFI(const MachineFunction &MF) {
96 return MF.needsFrameMoves();
97}
98
99// For now we use x3, a.k.a gp, as pointer to shadow call stack.
100// User should not use x3 in their asm.
101static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
102 MachineBasicBlock::iterator MI,
103 const DebugLoc &DL) {
104 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
105 RISCVMachineFunctionInfo::ShadowStackKind SSK = RVFI->getShadowStackKind(MF);
106 if (SSK == RISCVMachineFunctionInfo::ShadowStackKind::None)
107 return;
108
109 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
110 const llvm::RISCVRegisterInfo *TRI = STI.getRegisterInfo();
111
112 // Do not save RA to the SCS if it's not saved to the regular stack,
113 // i.e. RA is not at risk of being overwritten.
114 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
115 if (llvm::none_of(
116 Range&: CSI, P: [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
117 return;
118
119 const RISCVInstrInfo *TII = STI.getInstrInfo();
120 if (SSK == RISCVMachineFunctionInfo::ShadowStackKind::Hardware) {
121 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SSPUSH))
122 .addReg(RegNo: RAReg)
123 .setMIFlag(MachineInstr::FrameSetup);
124 return;
125 }
126
127 assert(SSK == RISCVMachineFunctionInfo::ShadowStackKind::Software &&
128 "Unexpected Shadow Stack Kind");
129
130 Register SCSPReg = RISCVABI::getSCSPReg();
131
132 bool IsRV64 = STI.is64Bit();
133 int64_t SlotSize = STI.getXLen() / 8;
134 // Store return address to shadow call stack
135 // addi gp, gp, [4|8]
136 // s[w|d] ra, -[4|8](gp)
137 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: SCSPReg)
138 .addReg(RegNo: SCSPReg)
139 .addImm(Val: SlotSize)
140 .setMIFlag(MachineInstr::FrameSetup);
141 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
142 .addReg(RegNo: RAReg)
143 .addReg(RegNo: SCSPReg)
144 .addImm(Val: -SlotSize)
145 .setMIFlag(MachineInstr::FrameSetup);
146
147 if (!needsDwarfCFI(MF))
148 return;
149
150 // Emit a CFI instruction that causes SlotSize to be subtracted from the value
151 // of the shadow stack pointer when unwinding past this frame.
152 char DwarfSCSReg = TRI->getDwarfRegNum(Reg: SCSPReg, /*IsEH*/ isEH: true);
153 assert(DwarfSCSReg < 32 && "SCS Register should be < 32 (X3).");
154
155 char Offset = static_cast<char>(-SlotSize) & 0x7f;
156 const char CFIInst[] = {
157 dwarf::DW_CFA_val_expression,
158 DwarfSCSReg, // register
159 2, // length
160 static_cast<char>(unsigned(dwarf::DW_OP_breg0 + DwarfSCSReg)),
161 Offset, // addend (sleb128)
162 };
163
164 CFIInstBuilder(MBB, MI, MachineInstr::FrameSetup)
165 .buildEscape(Bytes: StringRef(CFIInst, sizeof(CFIInst)));
166}
167
168static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB,
169 MachineBasicBlock::iterator MI,
170 const DebugLoc &DL) {
171 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
172 RISCVMachineFunctionInfo::ShadowStackKind SSK = RVFI->getShadowStackKind(MF);
173 if (SSK == RISCVMachineFunctionInfo::ShadowStackKind::None)
174 return;
175
176 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
177
178 // See emitSCSPrologue() above.
179 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
180 if (llvm::none_of(
181 Range&: CSI, P: [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
182 return;
183
184 // The shadow call stack popchk needs to happen after cm.pop that loads ra.
185 if (MI != MBB.end() &&
186 (MI->getOpcode() == RISCV::CM_POP || MI->getOpcode() == RISCV::QC_CM_POP))
187 ++MI;
188 const RISCVInstrInfo *TII = STI.getInstrInfo();
189 if (SSK == RISCVMachineFunctionInfo::ShadowStackKind::Hardware) {
190 // `sspopchk x5` is the only compressible form, but this would require using
191 // `x5` as the return address everywhere, which would also mean reserving
192 // it. We prefer to just use `x1` to avoid that complexity.
193 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SSPOPCHK))
194 .addReg(RegNo: RAReg)
195 .setMIFlag(MachineInstr::FrameDestroy);
196 return;
197 }
198
199 assert(SSK == RISCVMachineFunctionInfo::ShadowStackKind::Software &&
200 "Unexpected Shadow Stack Kind");
201
202 Register SCSPReg = RISCVABI::getSCSPReg();
203
204 bool IsRV64 = STI.is64Bit();
205 int64_t SlotSize = STI.getXLen() / 8;
206 // Load return address from shadow call stack
207 // l[w|d] ra, -[4|8](gp)
208 // addi gp, gp, -[4|8]
209 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::LD : RISCV::LW), DestReg: RAReg)
210 .addReg(RegNo: SCSPReg)
211 .addImm(Val: -SlotSize)
212 .setMIFlag(MachineInstr::FrameDestroy);
213 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: SCSPReg)
214 .addReg(RegNo: SCSPReg)
215 .addImm(Val: -SlotSize)
216 .setMIFlag(MachineInstr::FrameDestroy);
217 if (needsDwarfCFI(MF)) {
218 // Restore the SCS pointer
219 CFIInstBuilder(MBB, MI, MachineInstr::FrameDestroy).buildRestore(Reg: SCSPReg);
220 }
221}
222
223// Insert instruction to swap mscratchsw with sp
224static void emitSiFiveCLICStackSwap(MachineFunction &MF, MachineBasicBlock &MBB,
225 MachineBasicBlock::iterator MBBI,
226 const DebugLoc &DL,
227 MachineInstr::MIFlag FrameFlag) {
228 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
229
230 if (!RVFI->isSiFiveStackSwapInterrupt(MF))
231 return;
232
233 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
234 const RISCVInstrInfo *TII = STI.getInstrInfo();
235
236 assert(STI.hasVendorXSfmclic() && "Stack Swapping Requires XSfmclic");
237
238 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRW), DestReg: SPReg)
239 .addImm(Val: RISCVSysReg::sf_mscratchcsw)
240 .addReg(RegNo: SPReg, Flags: RegState::Kill)
241 .setMIFlag(FrameFlag);
242
243 // FIXME: CFI Information for this swap.
244}
245
246static void
247createSiFivePreemptibleInterruptFrameEntries(MachineFunction &MF,
248 RISCVMachineFunctionInfo &RVFI) {
249 if (!RVFI.isSiFivePreemptibleInterrupt(MF))
250 return;
251
252 const TargetRegisterClass &RC = RISCV::GPRRegClass;
253 const TargetRegisterInfo &TRI =
254 *MF.getSubtarget<RISCVSubtarget>().getRegisterInfo();
255 MachineFrameInfo &MFI = MF.getFrameInfo();
256
257 // Create two frame objects for saving `mcause` and `mepc`.
258 for (int I = 0; I < 2; ++I) {
259 int FI = MFI.CreateStackObject(Size: TRI.getSpillSize(RC), Alignment: TRI.getSpillAlign(RC),
260 isSpillSlot: true);
261 RVFI.pushInterruptCSRFrameIndex(FI);
262 }
263}
264
265// The scratch register retains an ordinary CSI slot, but its save and restore
266// are emitted explicitly as part of the SiFive CLIC interrupt sequence.
267static int getSiFiveCLICScratchFrameIndex(const MachineFunction &MF) {
268 const auto &CSI = MF.getFrameInfo().getCalleeSavedInfo();
269 auto ScratchCS = llvm::find_if(
270 Range: CSI, P: [](const CalleeSavedInfo &CS) { return CS.getReg() == RISCV::X5; });
271 assert(ScratchCS != CSI.end() && "Missing SiFive CLIC scratch spill slot");
272 return ScratchCS->getFrameIdx();
273}
274
275static void emitSiFiveCLICPreemptibleSaves(MachineFunction &MF,
276 MachineBasicBlock &MBB,
277 MachineBasicBlock::iterator MBBI,
278 const DebugLoc &DL) {
279 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
280
281 if (!RVFI->isSiFivePreemptibleInterrupt(MF))
282 return;
283
284 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
285 const RISCVInstrInfo *TII = STI.getInstrInfo();
286
287 // FIXME: CFI information for `mcause` and `mepc` is missing.
288
289 // Preserve X5 before using it to save the interrupt CSRs. Other GPRs
290 // are saved by the ordinary spill sequence after preemption is enabled.
291 int ScratchFI = getSiFiveCLICScratchFrameIndex(MF);
292 TII->storeRegToStackSlot(MBB, MBBI, SrcReg: RISCV::X5, /*IsKill=*/true, FrameIndex: ScratchFI,
293 RC: &RISCV::GPRRegClass, VReg: Register(),
294 Flags: MachineInstr::FrameSetup);
295 if (needsDwarfCFI(MF))
296 CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup)
297 .buildOffset(Reg: RISCV::X5, Offset: MF.getFrameInfo().getObjectOffset(ObjectIdx: ScratchFI));
298
299 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: RISCV::X5)
300 .addImm(Val: RISCVSysReg::mcause)
301 .addReg(RegNo: RISCV::X0)
302 .setMIFlag(MachineInstr::FrameSetup);
303 TII->storeRegToStackSlot(MBB, MBBI, SrcReg: RISCV::X5, /* IsKill=*/true,
304 FrameIndex: RVFI->getInterruptCSRFrameIndex(Idx: 0),
305 RC: &RISCV::GPRRegClass, VReg: Register(),
306 Flags: MachineInstr::FrameSetup);
307
308 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: RISCV::X5)
309 .addImm(Val: RISCVSysReg::mepc)
310 .addReg(RegNo: RISCV::X0)
311 .setMIFlag(MachineInstr::FrameSetup);
312
313 // Enable interrupts.
314 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRSI), DestReg: RISCV::X0)
315 .addImm(Val: RISCVSysReg::mstatus)
316 .addImm(Val: 8)
317 .setMIFlag(MachineInstr::FrameSetup);
318 TII->storeRegToStackSlot(MBB, MBBI, SrcReg: RISCV::X5, /* IsKill=*/true,
319 FrameIndex: RVFI->getInterruptCSRFrameIndex(Idx: 1),
320 RC: &RISCV::GPRRegClass, VReg: Register(),
321 Flags: MachineInstr::FrameSetup);
322}
323
324static void emitSiFiveCLICPreemptibleRestores(MachineFunction &MF,
325 MachineBasicBlock &MBB,
326 MachineBasicBlock::iterator MBBI,
327 CFIInstBuilder &CFIBuilder,
328 const DebugLoc &DL) {
329 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
330
331 if (!RVFI->isSiFivePreemptibleInterrupt(MF))
332 return;
333
334 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
335 const RISCVInstrInfo *TII = STI.getInstrInfo();
336
337 // FIXME: CFI information for `mcause` and `mepc` is missing.
338
339 // Load mepc while preemption is still enabled. A nested handler preserves
340 // X5. Interrupts only need to be disabled before writing the CSRs back.
341 TII->loadRegFromStackSlot(MBB, MBBI, DstReg: RISCV::X5,
342 FrameIndex: RVFI->getInterruptCSRFrameIndex(Idx: 1),
343 RC: &RISCV::GPRRegClass, VReg: Register(),
344 SubReg: RISCV::NoSubRegister, Flags: MachineInstr::FrameDestroy);
345
346 // Disable interrupts.
347 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRCI), DestReg: RISCV::X0)
348 .addImm(Val: RISCVSysReg::mstatus)
349 .addImm(Val: 8)
350 .setMIFlag(MachineInstr::FrameDestroy);
351
352 // Restore `mepc` and `mcause` through X5, then restore the value X5 held
353 // on entry to the handler.
354 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRW), DestReg: RISCV::X0)
355 .addImm(Val: RISCVSysReg::mepc)
356 .addReg(RegNo: RISCV::X5, Flags: RegState::Kill)
357 .setMIFlag(MachineInstr::FrameDestroy);
358
359 TII->loadRegFromStackSlot(MBB, MBBI, DstReg: RISCV::X5,
360 FrameIndex: RVFI->getInterruptCSRFrameIndex(Idx: 0),
361 RC: &RISCV::GPRRegClass, VReg: Register(),
362 SubReg: RISCV::NoSubRegister, Flags: MachineInstr::FrameDestroy);
363 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRW), DestReg: RISCV::X0)
364 .addImm(Val: RISCVSysReg::mcause)
365 .addReg(RegNo: RISCV::X5, Flags: RegState::Kill)
366 .setMIFlag(MachineInstr::FrameDestroy);
367
368 // The ordinary reloads have finished. Recover the interrupted value of X5
369 // only after it has restored both CSRs.
370 TII->loadRegFromStackSlot(MBB, MBBI, DstReg: RISCV::X5,
371 FrameIndex: getSiFiveCLICScratchFrameIndex(MF),
372 RC: &RISCV::GPRRegClass, VReg: Register(),
373 SubReg: RISCV::NoSubRegister, Flags: MachineInstr::FrameDestroy);
374 if (needsDwarfCFI(MF))
375 CFIBuilder.buildRestore(Reg: RISCV::X5);
376}
377
378// Get the ID of the libcall used for spilling and restoring callee saved
379// registers. The ID is representative of the number of registers saved or
380// restored by the libcall, except it is zero-indexed - ID 0 corresponds to a
381// single register.
382static int getLibCallID(const MachineFunction &MF,
383 const std::vector<CalleeSavedInfo> &CSI) {
384 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
385
386 if (CSI.empty() || !RVFI->useSaveRestoreLibCalls(MF))
387 return -1;
388
389 MCRegister MaxReg;
390 for (auto &CS : CSI)
391 // assignCalleeSavedSpillSlots assigns negative frame indexes to
392 // registers which can be saved by libcall.
393 if (CS.getFrameIdx() < 0)
394 MaxReg = std::max(a: MaxReg.id(), b: CS.getReg().id());
395
396 if (!MaxReg)
397 return -1;
398
399 switch (MaxReg.id()) {
400 default:
401 llvm_unreachable("Something has gone wrong!");
402 // clang-format off
403 case /*s11*/ RISCV::X27: return 12;
404 case /*s10*/ RISCV::X26: return 11;
405 case /*s9*/ RISCV::X25: return 10;
406 case /*s8*/ RISCV::X24: return 9;
407 case /*s7*/ RISCV::X23: return 8;
408 case /*s6*/ RISCV::X22: return 7;
409 case /*s5*/ RISCV::X21: return 6;
410 case /*s4*/ RISCV::X20: return 5;
411 case /*s3*/ RISCV::X19: return 4;
412 case /*s2*/ RISCV::X18: return 3;
413 case /*s1*/ RISCV::X9: return 2;
414 case /*s0*/ FPReg: return 1;
415 case /*ra*/ RAReg: return 0;
416 // clang-format on
417 }
418}
419
420// Get the name of the libcall used for spilling callee saved registers.
421// If this function will not use save/restore libcalls, then return a nullptr.
422static const char *
423getSpillLibCallName(const MachineFunction &MF,
424 const std::vector<CalleeSavedInfo> &CSI) {
425 static const char *const SpillLibCalls[] = {
426 "__riscv_save_0",
427 "__riscv_save_1",
428 "__riscv_save_2",
429 "__riscv_save_3",
430 "__riscv_save_4",
431 "__riscv_save_5",
432 "__riscv_save_6",
433 "__riscv_save_7",
434 "__riscv_save_8",
435 "__riscv_save_9",
436 "__riscv_save_10",
437 "__riscv_save_11",
438 "__riscv_save_12"
439 };
440
441 int LibCallID = getLibCallID(MF, CSI);
442 if (LibCallID == -1)
443 return nullptr;
444 return SpillLibCalls[LibCallID];
445}
446
447// Get the name of the libcall used for restoring callee saved registers.
448// If this function will not use save/restore libcalls, then return a nullptr.
449static const char *
450getRestoreLibCallName(const MachineFunction &MF,
451 const std::vector<CalleeSavedInfo> &CSI) {
452 static const char *const RestoreLibCalls[] = {
453 "__riscv_restore_0",
454 "__riscv_restore_1",
455 "__riscv_restore_2",
456 "__riscv_restore_3",
457 "__riscv_restore_4",
458 "__riscv_restore_5",
459 "__riscv_restore_6",
460 "__riscv_restore_7",
461 "__riscv_restore_8",
462 "__riscv_restore_9",
463 "__riscv_restore_10",
464 "__riscv_restore_11",
465 "__riscv_restore_12"
466 };
467
468 int LibCallID = getLibCallID(MF, CSI);
469 if (LibCallID == -1)
470 return nullptr;
471 return RestoreLibCalls[LibCallID];
472}
473
474// Get the max reg of Push/Pop for restoring callee saved registers.
475static unsigned getNumPushPopRegs(const std::vector<CalleeSavedInfo> &CSI) {
476 unsigned NumPushPopRegs = 0;
477 for (auto &CS : CSI) {
478 auto *FII = llvm::find_if(Range: FixedCSRFIMap,
479 P: [&](MCPhysReg P) { return P == CS.getReg(); });
480 if (FII != std::end(arr: FixedCSRFIMap)) {
481 unsigned RegNum = std::distance(first: std::begin(arr: FixedCSRFIMap), last: FII);
482 NumPushPopRegs = std::max(a: NumPushPopRegs, b: RegNum + 1);
483 }
484 }
485 assert(NumPushPopRegs != 12 && "x26 requires x27 to also be pushed");
486 return NumPushPopRegs;
487}
488
489// Return true if the specified function should have a dedicated frame
490// pointer register. This is true if frame pointer elimination is
491// disabled, if it needs dynamic stack realignment, if the function has
492// variable sized allocas, or if the frame address is taken.
493bool RISCVFrameLowering::hasFPImpl(const MachineFunction &MF) const {
494 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
495
496 const MachineFrameInfo &MFI = MF.getFrameInfo();
497 if (MF.disableFramePointerElim() || RegInfo->hasStackRealignment(MF) ||
498 MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken())
499 return true;
500
501 // With large callframes around we may need to use FP to access the scavenging
502 // emergency spillslot.
503 //
504 // We calculate the MaxCallFrameSize at the end of isel so this value should
505 // be stable for the whole post-isel MIR pipeline.
506 //
507 // NOTE: The idea of forcing a frame pointer is copied from AArch64, but they
508 // conservatively return true when the call frame size hasd not been
509 // computed yet. On RISC-V that caused MachineOutliner tests to fail the
510 // MachineVerifier due to outlined functions not computing max call frame
511 // size thus the frame pointer would always be reserved.
512 if (MFI.isMaxCallFrameSizeComputed() && MFI.getMaxCallFrameSize() > 2047)
513 return true;
514
515 return false;
516}
517
518bool RISCVFrameLowering::hasBP(const MachineFunction &MF) const {
519 const MachineFrameInfo &MFI = MF.getFrameInfo();
520 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
521
522 // If we do not reserve stack space for outgoing arguments in prologue,
523 // we will adjust the stack pointer before call instruction. After the
524 // adjustment, we can not use SP to access the stack objects for the
525 // arguments. Instead, use BP to access these stack objects.
526 return (MFI.hasVarSizedObjects() ||
527 (!hasReservedCallFrame(MF) && (!MFI.isMaxCallFrameSizeComputed() ||
528 MFI.getMaxCallFrameSize() != 0))) &&
529 TRI->hasStackRealignment(MF);
530}
531
532// Determines the size of the frame and maximum call frame size.
533void RISCVFrameLowering::determineFrameLayout(MachineFunction &MF) const {
534 MachineFrameInfo &MFI = MF.getFrameInfo();
535 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
536
537 // Get the number of bytes to allocate from the FrameInfo.
538 uint64_t FrameSize = MFI.getStackSize();
539
540 // QCI Interrupts use at least 96 bytes of stack space
541 if (RVFI->useQCIInterrupt(MF))
542 FrameSize = std::max(a: FrameSize, b: QCIInterruptPushAmount);
543
544 // Get the alignment.
545 Align StackAlign = getStackAlign();
546
547 // Make sure the frame is aligned.
548 FrameSize = alignTo(Size: FrameSize, A: StackAlign);
549
550 // Update frame info.
551 MFI.setStackSize(FrameSize);
552
553 // When using SP or BP to access stack objects, we may require extra padding
554 // to ensure the bottom of the RVV stack is correctly aligned within the main
555 // stack. We calculate this as the amount required to align the scalar local
556 // variable section up to the RVV alignment.
557 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
558 if (RVFI->getRVVStackSize() && (!hasFP(MF) || TRI->hasStackRealignment(MF))) {
559 int ScalarLocalVarSize = FrameSize - RVFI->getCalleeSavedStackSize() -
560 RVFI->getVarArgsSaveSize();
561 if (auto RVVPadding =
562 offsetToAlignment(Value: ScalarLocalVarSize, Alignment: RVFI->getRVVStackAlign()))
563 RVFI->setRVVPadding(RVVPadding);
564 }
565}
566
567// Returns the stack size including RVV padding (when required), rounded back
568// up to the required stack alignment.
569uint64_t RISCVFrameLowering::getStackSizeWithRVVPadding(
570 const MachineFunction &MF) const {
571 const MachineFrameInfo &MFI = MF.getFrameInfo();
572 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
573 return alignTo(Size: MFI.getStackSize() + RVFI->getRVVPadding(), A: getStackAlign());
574}
575
576static SmallVector<CalleeSavedInfo, 8>
577getUnmanagedCSI(const MachineFunction &MF,
578 const std::vector<CalleeSavedInfo> &CSI,
579 bool ReverseOrder = false) {
580 const MachineFrameInfo &MFI = MF.getFrameInfo();
581 SmallVector<CalleeSavedInfo, 8> NonLibcallCSI;
582
583 for (auto &CS : CSI) {
584 int FI = CS.getFrameIdx();
585 // Callee saves with negative frame indices have fixed positions on the
586 // stack, usually relating to libcalls or push/pop.
587 bool IsManagedFI = (FI < 0);
588 if (!IsManagedFI && MFI.getStackID(ObjectIdx: FI) == TargetStackID::Default)
589 NonLibcallCSI.push_back(Elt: CS);
590 }
591
592 // Reverse the order so that load/store operations use ascending addresses,
593 // enabling better load/store clustering and fusion.
594 if (ReverseOrder)
595 std::reverse(first: NonLibcallCSI.begin(), last: NonLibcallCSI.end());
596
597 return NonLibcallCSI;
598}
599
600// Exclude X5 from ordinary spills and restores for SiFive CLIC preemptible
601// handlers, which save and restore it explicitly.
602static SmallVector<CalleeSavedInfo, 8>
603getUnmanagedInterruptCSI(const MachineFunction &MF,
604 const std::vector<CalleeSavedInfo> &CSI,
605 bool ReverseOrder = false) {
606 auto InterruptCSI = getUnmanagedCSI(MF, CSI, ReverseOrder);
607 if (MF.getInfo<RISCVMachineFunctionInfo>()->isSiFivePreemptibleInterrupt(MF))
608 llvm::erase_if(C&: InterruptCSI, P: [](const CalleeSavedInfo &CS) {
609 return CS.getReg() == RISCV::X5;
610 });
611 return InterruptCSI;
612}
613
614static SmallVector<CalleeSavedInfo, 8>
615getRVVCalleeSavedInfo(const MachineFunction &MF,
616 const std::vector<CalleeSavedInfo> &CSI) {
617 const MachineFrameInfo &MFI = MF.getFrameInfo();
618 SmallVector<CalleeSavedInfo, 8> RVVCSI;
619
620 for (auto &CS : CSI) {
621 int FI = CS.getFrameIdx();
622 if (FI >= 0 && MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector)
623 RVVCSI.push_back(Elt: CS);
624 }
625
626 return RVVCSI;
627}
628
629static SmallVector<CalleeSavedInfo, 8>
630getPushOrLibCallsSavedInfo(const MachineFunction &MF,
631 const std::vector<CalleeSavedInfo> &CSI) {
632 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
633
634 SmallVector<CalleeSavedInfo, 8> PushOrLibCallsCSI;
635 if (!RVFI->useSaveRestoreLibCalls(MF) && !RVFI->isPushable(MF))
636 return PushOrLibCallsCSI;
637
638 for (const auto &CS : CSI) {
639 if (RVFI->useQCIInterrupt(MF)) {
640 // Some registers are saved by both `QC.C.MIENTER(.NEST)` and
641 // `QC.CM.PUSH(FP)`. In these cases, prioritise the CFI info that points
642 // to the versions saved by `QC.C.MIENTER(.NEST)` which is what FP
643 // unwinding would use.
644 if (llvm::is_contained(Range: llvm::make_first_range(c: FixedCSRFIQCIInterruptMap),
645 Element: CS.getReg()))
646 continue;
647 }
648
649 if (llvm::is_contained(Range: FixedCSRFIMap, Element: CS.getReg()))
650 PushOrLibCallsCSI.push_back(Elt: CS);
651 }
652
653 return PushOrLibCallsCSI;
654}
655
656static SmallVector<CalleeSavedInfo, 8>
657getQCISavedInfo(const MachineFunction &MF,
658 const std::vector<CalleeSavedInfo> &CSI) {
659 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
660
661 SmallVector<CalleeSavedInfo, 8> QCIInterruptCSI;
662 if (!RVFI->useQCIInterrupt(MF))
663 return QCIInterruptCSI;
664
665 for (const auto &CS : CSI) {
666 if (llvm::is_contained(Range: llvm::make_first_range(c: FixedCSRFIQCIInterruptMap),
667 Element: CS.getReg()))
668 QCIInterruptCSI.push_back(Elt: CS);
669 }
670
671 return QCIInterruptCSI;
672}
673
674static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs,
675 const MachineBasicBlock &MBB) {
676 const MachineFunction *MF = MBB.getParent();
677 LiveRegs.addLiveIns(MBB);
678 const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
679 for (unsigned i = 0; CSRegs[i]; ++i)
680 LiveRegs.addReg(Reg: CSRegs[i]);
681}
682
683Register RISCVFrameLowering::findScratchNonCalleeSaveRegister(
684 MachineBasicBlock *MBB, Register PreferredReg, Register DontUseReg) const {
685 MachineFunction *MF = MBB->getParent();
686
687 // Stack protection code is being inserted at beginning of function, use
688 // register which has been historically used
689 if (&MF->front() == MBB)
690 return PreferredReg;
691
692 const RISCVSubtarget &Subtarget = MF->getSubtarget<RISCVSubtarget>();
693 const TargetRegisterInfo &TRI = *Subtarget.getRegisterInfo();
694 LivePhysRegs LiveRegs(TRI);
695 getLiveRegsForEntryMBB(LiveRegs, MBB: *MBB);
696
697 const MachineRegisterInfo &MRI = MF->getRegInfo();
698 // Prefer the register which has been historically used for stack protector
699 if (LiveRegs.available(MRI, Reg: PreferredReg))
700 return PreferredReg;
701
702 static const MCPhysReg CandidateRegs[] = {
703 RISCV::X5, RISCV::X6, RISCV::X7, RISCV::X28,
704 RISCV::X29, RISCV::X30, RISCV::X31,
705 };
706
707 for (unsigned Reg : CandidateRegs) {
708 if (Reg != DontUseReg && LiveRegs.available(MRI, Reg))
709 return Reg;
710 }
711
712 return Register();
713}
714
715void RISCVFrameLowering::allocateAndProbeStackForRVV(
716 MachineFunction &MF, MachineBasicBlock &MBB,
717 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, int64_t Amount,
718 MachineInstr::MIFlag Flag, bool EmitCFI, bool DynAllocation) const {
719 assert(Amount != 0 && "Did not need to adjust stack pointer for RVV.");
720
721 // Emit a variable-length allocation probing loop.
722
723 // Get VLEN in TargetReg
724 Register TargetReg = findScratchNonCalleeSaveRegister(MBB: &MBB, PreferredReg: RISCV::X6);
725 assert(TargetReg.isValid() &&
726 "No available scratch register for stack probing");
727 const RISCVInstrInfo *TII = STI.getInstrInfo();
728 uint32_t NumOfVReg = Amount / RISCV::RVVBytesPerBlock;
729 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::PseudoReadVLENB), DestReg: TargetReg)
730 .setMIFlag(Flag);
731 TII->mulImm(MF, MBB, II: MBBI, DL, DestReg: TargetReg, Amt: NumOfVReg, Flag);
732
733 CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
734 if (EmitCFI) {
735 // Set the CFA register to TargetReg.
736 CFIBuilder.buildDefCFA(Reg: TargetReg, Offset: -Amount);
737 }
738
739 // It will be expanded to a probe loop in `inlineStackProbe`.
740 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::PROBED_STACKALLOC_RVV))
741 .addReg(RegNo: TargetReg);
742
743 if (EmitCFI) {
744 // Set the CFA register back to SP.
745 CFIBuilder.buildDefCFARegister(Reg: SPReg);
746 }
747
748 // SUB SP, SP, T1
749 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SUB), DestReg: SPReg)
750 .addReg(RegNo: SPReg)
751 .addReg(RegNo: TargetReg)
752 .setMIFlag(Flag);
753
754 // If we have a dynamic allocation later we need to probe any residuals.
755 if (DynAllocation) {
756 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: STI.is64Bit() ? RISCV::SD : RISCV::SW))
757 .addReg(RegNo: RISCV::X0)
758 .addReg(RegNo: SPReg)
759 .addImm(Val: 0)
760 .setMIFlags(MachineInstr::FrameSetup);
761 }
762}
763
764static void appendScalableVectorExpression(const TargetRegisterInfo &TRI,
765 SmallVectorImpl<char> &Expr,
766 StackOffset Offset,
767 llvm::raw_string_ostream &Comment) {
768 int64_t FixedOffset = Offset.getFixed();
769 int64_t ScalableOffset = Offset.getScalable();
770 unsigned DwarfVLenB = TRI.getDwarfRegNum(Reg: RISCV::VLENB, isEH: true);
771 if (FixedOffset) {
772 Expr.push_back(Elt: dwarf::DW_OP_consts);
773 appendLEB128<LEB128Sign::Signed>(Buffer&: Expr, Value: FixedOffset);
774 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_plus);
775 Comment << (FixedOffset < 0 ? " - " : " + ") << std::abs(i: FixedOffset);
776 }
777
778 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_consts);
779 appendLEB128<LEB128Sign::Signed>(Buffer&: Expr, Value: ScalableOffset);
780
781 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_bregx);
782 appendLEB128<LEB128Sign::Unsigned>(Buffer&: Expr, Value: DwarfVLenB);
783 Expr.push_back(Elt: 0);
784
785 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_mul);
786 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_plus);
787
788 Comment << (ScalableOffset < 0 ? " - " : " + ") << std::abs(i: ScalableOffset)
789 << " * vlenb";
790}
791
792static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
793 Register Reg,
794 StackOffset Offset) {
795 assert(Offset.getScalable() != 0 && "Did not need to adjust CFA for RVV");
796 SmallString<64> Expr;
797 std::string CommentBuffer;
798 llvm::raw_string_ostream Comment(CommentBuffer);
799 // Build up the expression (Reg + FixedOffset + ScalableOffset * VLENB).
800 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
801 Expr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
802 Expr.push_back(Elt: 0);
803 if (Reg == SPReg)
804 Comment << "sp";
805 else
806 Comment << printReg(Reg, TRI: &TRI);
807
808 appendScalableVectorExpression(TRI, Expr, Offset, Comment);
809
810 SmallString<64> DefCfaExpr;
811 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
812 appendLEB128<LEB128Sign::Unsigned>(Buffer&: DefCfaExpr, Value: Expr.size());
813 DefCfaExpr.append(RHS: Expr.str());
814
815 return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(),
816 Comment: Comment.str());
817}
818
819static MCCFIInstruction createDefCFAOffset(const TargetRegisterInfo &TRI,
820 Register Reg, StackOffset Offset) {
821 assert(Offset.getScalable() != 0 && "Did not need to adjust CFA for RVV");
822 SmallString<64> Expr;
823 std::string CommentBuffer;
824 llvm::raw_string_ostream Comment(CommentBuffer);
825 Comment << printReg(Reg, TRI: &TRI) << " @ cfa";
826
827 // Build up the expression (FixedOffset + ScalableOffset * VLENB).
828 appendScalableVectorExpression(TRI, Expr, Offset, Comment);
829
830 SmallString<64> DefCfaExpr;
831 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
832 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
833 appendLEB128<LEB128Sign::Unsigned>(Buffer&: DefCfaExpr, Value: DwarfReg);
834 appendLEB128<LEB128Sign::Unsigned>(Buffer&: DefCfaExpr, Value: Expr.size());
835 DefCfaExpr.append(RHS: Expr.str());
836
837 return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(),
838 Comment: Comment.str());
839}
840
841// Allocate stack space and probe it if necessary.
842void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
843 MachineBasicBlock::iterator MBBI,
844 MachineFunction &MF, uint64_t Offset,
845 uint64_t RealStackSize, bool EmitCFI,
846 bool NeedProbe, uint64_t ProbeSize,
847 bool DynAllocation,
848 MachineInstr::MIFlag Flag) const {
849 DebugLoc DL;
850 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
851 const RISCVInstrInfo *TII = STI.getInstrInfo();
852 bool IsRV64 = STI.is64Bit();
853 CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
854
855 // Simply allocate the stack if it's not big enough to require a probe.
856 if (!NeedProbe || Offset <= ProbeSize) {
857 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: -Offset),
858 Flag, RequiredAlign: getStackAlign());
859
860 if (EmitCFI)
861 CFIBuilder.buildDefCFAOffset(Offset: RealStackSize);
862
863 if (NeedProbe && DynAllocation) {
864 // s[d|w] zero, 0(sp)
865 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
866 .addReg(RegNo: RISCV::X0)
867 .addReg(RegNo: SPReg)
868 .addImm(Val: 0)
869 .setMIFlags(Flag);
870 }
871
872 return;
873 }
874
875 // The amount of stack that was already allocated before this allocation.
876 uint64_t CFAAdjust = RealStackSize - Offset;
877
878 // Unroll the probe loop depending on the number of iterations.
879 if (Offset < ProbeSize * 5) {
880 uint64_t CurrentOffset = 0;
881 while (CurrentOffset + ProbeSize <= Offset) {
882 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg,
883 Offset: StackOffset::getFixed(Fixed: -ProbeSize), Flag, RequiredAlign: getStackAlign());
884 // s[d|w] zero, 0(sp)
885 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
886 .addReg(RegNo: RISCV::X0)
887 .addReg(RegNo: SPReg)
888 .addImm(Val: 0)
889 .setMIFlags(Flag);
890
891 CurrentOffset += ProbeSize;
892 if (EmitCFI)
893 CFIBuilder.buildDefCFAOffset(Offset: CurrentOffset + CFAAdjust);
894 }
895
896 uint64_t Residual = Offset - CurrentOffset;
897 if (Residual) {
898 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg,
899 Offset: StackOffset::getFixed(Fixed: -Residual), Flag, RequiredAlign: getStackAlign());
900 if (EmitCFI)
901 CFIBuilder.buildDefCFAOffset(Offset: RealStackSize);
902
903 if (DynAllocation) {
904 // s[d|w] zero, 0(sp)
905 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
906 .addReg(RegNo: RISCV::X0)
907 .addReg(RegNo: SPReg)
908 .addImm(Val: 0)
909 .setMIFlags(Flag);
910 }
911 }
912
913 return;
914 }
915
916 // Emit a variable-length allocation probing loop.
917 uint64_t RoundedSize = alignDown(Value: Offset, Align: ProbeSize);
918 uint64_t Residual = Offset - RoundedSize;
919
920 Register TargetReg = findScratchNonCalleeSaveRegister(MBB: &MBB, PreferredReg: RISCV::X6);
921 assert(TargetReg.isValid() &&
922 "No available scratch register for stack probing");
923 // SUB TargetReg, SP, RoundedSize
924 RI->adjustReg(MBB, II: MBBI, DL, DestReg: TargetReg, SrcReg: SPReg,
925 Offset: StackOffset::getFixed(Fixed: -RoundedSize), Flag, RequiredAlign: getStackAlign());
926
927 if (EmitCFI) {
928 // Set the CFA register to TargetReg.
929 CFIBuilder.buildDefCFA(Reg: TargetReg, Offset: RoundedSize + CFAAdjust);
930 }
931
932 // It will be expanded to a probe loop in `inlineStackProbe`.
933 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::PROBED_STACKALLOC)).addReg(RegNo: TargetReg);
934
935 if (EmitCFI) {
936 // Set the CFA register back to SP.
937 CFIBuilder.buildDefCFARegister(Reg: SPReg);
938 }
939
940 if (Residual) {
941 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: -Residual),
942 Flag, RequiredAlign: getStackAlign());
943 if (DynAllocation) {
944 // s[d|w] zero, 0(sp)
945 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
946 .addReg(RegNo: RISCV::X0)
947 .addReg(RegNo: SPReg)
948 .addImm(Val: 0)
949 .setMIFlags(Flag);
950 }
951 }
952
953 if (EmitCFI)
954 CFIBuilder.buildDefCFAOffset(Offset: RealStackSize);
955}
956
957static bool isPush(unsigned Opcode) {
958 switch (Opcode) {
959 case RISCV::CM_PUSH:
960 case RISCV::QC_CM_PUSH:
961 case RISCV::QC_CM_PUSHFP:
962 return true;
963 default:
964 return false;
965 }
966}
967
968static bool isPop(unsigned Opcode) {
969 // There are other pops but these are the only ones introduced during this
970 // pass.
971 switch (Opcode) {
972 case RISCV::CM_POP:
973 case RISCV::QC_CM_POP:
974 return true;
975 default:
976 return false;
977 }
978}
979
980static unsigned getPushOpcode(RISCVMachineFunctionInfo::PushPopKind Kind,
981 bool UpdateFP) {
982 switch (Kind) {
983 case RISCVMachineFunctionInfo::PushPopKind::StdExtZcmp:
984 return RISCV::CM_PUSH;
985 case RISCVMachineFunctionInfo::PushPopKind::VendorXqccmp:
986 return UpdateFP ? RISCV::QC_CM_PUSHFP : RISCV::QC_CM_PUSH;
987 default:
988 llvm_unreachable("Unhandled PushPopKind");
989 }
990}
991
992static unsigned getPopOpcode(RISCVMachineFunctionInfo::PushPopKind Kind) {
993 // There are other pops but they are introduced later by the Push/Pop
994 // Optimizer.
995 switch (Kind) {
996 case RISCVMachineFunctionInfo::PushPopKind::StdExtZcmp:
997 return RISCV::CM_POP;
998 case RISCVMachineFunctionInfo::PushPopKind::VendorXqccmp:
999 return RISCV::QC_CM_POP;
1000 default:
1001 llvm_unreachable("Unhandled PushPopKind");
1002 }
1003}
1004
1005void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
1006 MachineBasicBlock &MBB) const {
1007 MachineFrameInfo &MFI = MF.getFrameInfo();
1008 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1009 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
1010 MachineBasicBlock::iterator MBBI = MBB.begin();
1011 bool PreferAscendingLS = STI.preferAscendingLoadStore();
1012
1013 Register BPReg = RISCVABI::getBPReg();
1014
1015 // Debug location must be unknown since the first debug location is used
1016 // to determine the end of the prologue.
1017 DebugLoc DL;
1018
1019 // All calls are tail calls in GHC calling conv, and functions have no
1020 // prologue/epilogue.
1021 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
1022 return;
1023
1024 // SiFive CLIC needs to swap `sp` into `sf.mscratchcsw`
1025 emitSiFiveCLICStackSwap(MF, MBB, MBBI, DL, FrameFlag: MachineInstr::FrameSetup);
1026
1027 // Emit prologue for shadow call stack.
1028 emitSCSPrologue(MF, MBB, MI: MBBI, DL);
1029
1030 // We keep track of the first instruction because it might be a
1031 // `(QC.)CM.PUSH(FP)`, and we may need to adjust the immediate rather than
1032 // inserting an `addi sp, sp, -N*16`
1033 auto PossiblePush = MBBI;
1034
1035 // Skip past all callee-saved register spill instructions.
1036 while (MBBI != MBB.end() && MBBI->getFlag(Flag: MachineInstr::FrameSetup))
1037 ++MBBI;
1038
1039 // Determine the correct frame layout
1040 determineFrameLayout(MF);
1041
1042 const auto &CSI = MFI.getCalleeSavedInfo();
1043
1044 // Skip to before the spills of scalar callee-saved registers
1045 // FIXME: assumes exactly one instruction is used to restore each
1046 // callee-saved register.
1047 MBBI = std::prev(
1048 x: MBBI, n: getRVVCalleeSavedInfo(MF, CSI).size() +
1049 getUnmanagedInterruptCSI(MF, CSI, ReverseOrder: PreferAscendingLS).size());
1050 CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
1051 bool NeedsDwarfCFI = needsDwarfCFI(MF);
1052
1053 if (unsigned LibCallFrameSize = RVFI->getLibCallStackSize()) {
1054 if (NeedsDwarfCFI) {
1055 CFIBuilder.buildDefCFAOffset(Offset: LibCallFrameSize);
1056 for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
1057 CFIBuilder.buildOffset(Reg: CS.getReg(),
1058 Offset: MFI.getObjectOffset(ObjectIdx: CS.getFrameIdx()));
1059 }
1060 }
1061
1062 // FIXME (note copied from Lanai): This appears to be overallocating. Needs
1063 // investigation. Get the number of bytes to allocate from the FrameInfo.
1064 uint64_t RealStackSize = getStackSizeWithRVVPadding(MF);
1065 uint64_t StackSize = RealStackSize - RVFI->getReservedSpillsSize();
1066 uint64_t RVVStackSize = RVFI->getRVVStackSize();
1067
1068 // Early exit if there is no need to allocate on the stack
1069 if (RealStackSize == 0 && !MFI.adjustsStack() && RVVStackSize == 0)
1070 return;
1071
1072 // If the stack pointer has been marked as reserved, then produce an error if
1073 // the frame requires stack allocation
1074 if (STI.isRegisterReservedByUser(i: SPReg))
1075 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
1076 MF.getFunction(), "Stack pointer required, but has been reserved."});
1077
1078 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
1079 // Split the SP adjustment to reduce the offsets of callee saved spill.
1080 if (FirstSPAdjustAmount) {
1081 StackSize = FirstSPAdjustAmount;
1082 RealStackSize = FirstSPAdjustAmount;
1083 }
1084
1085 if (RVFI->useQCIInterrupt(MF)) {
1086 // The function starts with `QC.C.MIENTER(.NEST)`, so the `(QC.)CM.PUSH(FP)`
1087 // could only be the next instruction.
1088 ++PossiblePush;
1089
1090 if (NeedsDwarfCFI) {
1091 // Insert the CFI metadata before where we think the `(QC.)CM.PUSH(FP)`
1092 // could be. The PUSH will also get its own CFI metadata for its own
1093 // modifications, which should come after the PUSH.
1094 CFIInstBuilder PushCFIBuilder(MBB, PossiblePush,
1095 MachineInstr::FrameSetup);
1096 PushCFIBuilder.buildDefCFAOffset(Offset: QCIInterruptPushAmount);
1097 for (const CalleeSavedInfo &CS : getQCISavedInfo(MF, CSI))
1098 PushCFIBuilder.buildOffset(Reg: CS.getReg(),
1099 Offset: MFI.getObjectOffset(ObjectIdx: CS.getFrameIdx()));
1100 }
1101 }
1102
1103 if (RVFI->isPushable(MF) && PossiblePush != MBB.end() &&
1104 isPush(Opcode: PossiblePush->getOpcode())) {
1105 // Use available stack adjustment in push instruction to allocate additional
1106 // stack space. Align the stack size down to a multiple of 16. This is
1107 // needed for RVE.
1108 // FIXME: Can we increase the stack size to a multiple of 16 instead?
1109 uint64_t StackAdj =
1110 std::min(a: alignDown(Value: StackSize, Align: 16), b: static_cast<uint64_t>(48));
1111 PossiblePush->getOperand(i: 1).setImm(StackAdj);
1112 StackSize -= StackAdj;
1113
1114 if (NeedsDwarfCFI) {
1115 CFIBuilder.buildDefCFAOffset(Offset: RealStackSize - StackSize);
1116 for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
1117 CFIBuilder.buildOffset(Reg: CS.getReg(),
1118 Offset: MFI.getObjectOffset(ObjectIdx: CS.getFrameIdx()));
1119 }
1120 }
1121
1122 // Allocate space on the stack if necessary.
1123 auto &Subtarget = MF.getSubtarget<RISCVSubtarget>();
1124 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
1125 bool NeedProbe = TLI->hasInlineStackProbe(MF);
1126 uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign: getStackAlign());
1127 bool DynAllocation =
1128 MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
1129 if (StackSize != 0)
1130 allocateStack(MBB, MBBI, MF, Offset: StackSize, RealStackSize, EmitCFI: NeedsDwarfCFI,
1131 NeedProbe, ProbeSize, DynAllocation,
1132 Flag: MachineInstr::FrameSetup);
1133
1134 // Save SiFive CLIC CSRs into Stack
1135 emitSiFiveCLICPreemptibleSaves(MF, MBB, MBBI, DL);
1136
1137 // The frame pointer is callee-saved, and code has been generated for us to
1138 // save it to the stack. We need to skip over the storing of callee-saved
1139 // registers as the frame pointer must be modified after it has been saved
1140 // to the stack, not before.
1141 // FIXME: assumes exactly one instruction is used to save each callee-saved
1142 // register.
1143 std::advance(i&: MBBI,
1144 n: getUnmanagedInterruptCSI(MF, CSI, ReverseOrder: PreferAscendingLS).size());
1145 CFIBuilder.setInsertPoint(MBBI);
1146
1147 // Iterate over list of callee-saved registers and emit .cfi_offset
1148 // directives.
1149 if (NeedsDwarfCFI) {
1150 for (const CalleeSavedInfo &CS :
1151 getUnmanagedInterruptCSI(MF, CSI, ReverseOrder: PreferAscendingLS)) {
1152 MCRegister Reg = CS.getReg();
1153 int64_t Offset = MFI.getObjectOffset(ObjectIdx: CS.getFrameIdx());
1154 // Emit CFI for both sub-registers. The even register is at the base
1155 // offset and odd at base+4.
1156 if (RISCV::GPRPairRegClass.contains(Reg)) {
1157 MCRegister EvenReg = RI->getSubReg(Reg, Idx: RISCV::sub_gpr_even);
1158 MCRegister OddReg = RI->getSubReg(Reg, Idx: RISCV::sub_gpr_odd);
1159 CFIBuilder.buildOffset(Reg: EvenReg, Offset);
1160 CFIBuilder.buildOffset(Reg: OddReg, Offset: Offset + 4);
1161 } else {
1162 CFIBuilder.buildOffset(Reg, Offset);
1163 }
1164 }
1165 }
1166
1167 // Generate new FP.
1168 if (hasFP(MF)) {
1169 if (STI.isRegisterReservedByUser(i: FPReg))
1170 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
1171 MF.getFunction(), "Frame pointer required, but has been reserved."});
1172 // The frame pointer does need to be reserved from register allocation.
1173 assert(MF.getRegInfo().isReserved(FPReg) && "FP not reserved");
1174
1175 // Some stack management variants automatically keep FP updated, so we don't
1176 // need an instruction to do so.
1177 if (!RVFI->hasImplicitFPUpdates(MF)) {
1178 RI->adjustReg(
1179 MBB, II: MBBI, DL, DestReg: FPReg, SrcReg: SPReg,
1180 Offset: StackOffset::getFixed(Fixed: RealStackSize - RVFI->getVarArgsSaveSize()),
1181 Flag: MachineInstr::FrameSetup, RequiredAlign: getStackAlign());
1182 }
1183
1184 if (NeedsDwarfCFI)
1185 CFIBuilder.buildDefCFA(Reg: FPReg, Offset: RVFI->getVarArgsSaveSize());
1186 }
1187
1188 uint64_t SecondSPAdjustAmount = 0;
1189 // Emit the second SP adjustment after saving callee saved registers.
1190 if (FirstSPAdjustAmount) {
1191 SecondSPAdjustAmount = getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
1192 assert(SecondSPAdjustAmount > 0 &&
1193 "SecondSPAdjustAmount should be greater than zero");
1194
1195 allocateStack(MBB, MBBI, MF, Offset: SecondSPAdjustAmount,
1196 RealStackSize: getStackSizeWithRVVPadding(MF), EmitCFI: NeedsDwarfCFI && !hasFP(MF),
1197 NeedProbe, ProbeSize, DynAllocation,
1198 Flag: MachineInstr::FrameSetup);
1199 }
1200
1201 if (RVVStackSize) {
1202 if (NeedProbe) {
1203 allocateAndProbeStackForRVV(MF, MBB, MBBI, DL, Amount: RVVStackSize,
1204 Flag: MachineInstr::FrameSetup,
1205 EmitCFI: NeedsDwarfCFI && !hasFP(MF), DynAllocation);
1206 } else {
1207 // We must keep the stack pointer aligned through any intermediate
1208 // updates.
1209 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg,
1210 Offset: StackOffset::getScalable(Scalable: -RVVStackSize),
1211 Flag: MachineInstr::FrameSetup, RequiredAlign: getStackAlign());
1212 }
1213
1214 if (NeedsDwarfCFI && !hasFP(MF)) {
1215 // Emit .cfi_def_cfa_expression "sp + StackSize + RVVStackSize * vlenb".
1216 CFIBuilder.insertCFIInst(CFIInst: createDefCFAExpression(
1217 TRI: *RI, Reg: SPReg,
1218 Offset: StackOffset::get(Fixed: getStackSizeWithRVVPadding(MF), Scalable: RVVStackSize / 8)));
1219 }
1220
1221 std::advance(i&: MBBI, n: getRVVCalleeSavedInfo(MF, CSI).size());
1222 if (NeedsDwarfCFI)
1223 emitCalleeSavedRVVPrologCFI(MBB, MI: MBBI, HasFP: hasFP(MF));
1224 }
1225
1226 if (hasFP(MF)) {
1227 // Realign Stack
1228 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
1229 if (RI->hasStackRealignment(MF)) {
1230 Align MaxAlignment = MFI.getMaxAlign();
1231
1232 const RISCVInstrInfo *TII = STI.getInstrInfo();
1233 if (isInt<12>(x: -(int64_t)MaxAlignment.value())) {
1234 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::ANDI), DestReg: SPReg)
1235 .addReg(RegNo: SPReg)
1236 .addImm(Val: -(int64_t)MaxAlignment.value())
1237 .setMIFlag(MachineInstr::FrameSetup);
1238 } else {
1239 unsigned ShiftAmount = Log2(A: MaxAlignment);
1240 Register VR =
1241 MF.getRegInfo().createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1242 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SRLI), DestReg: VR)
1243 .addReg(RegNo: SPReg)
1244 .addImm(Val: ShiftAmount)
1245 .setMIFlag(MachineInstr::FrameSetup);
1246 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SLLI), DestReg: SPReg)
1247 .addReg(RegNo: VR)
1248 .addImm(Val: ShiftAmount)
1249 .setMIFlag(MachineInstr::FrameSetup);
1250 }
1251 if (NeedProbe && RVVStackSize == 0) {
1252 // Do a probe if the align + size allocated just passed the probe size
1253 // and was not yet probed.
1254 if (SecondSPAdjustAmount < ProbeSize &&
1255 SecondSPAdjustAmount + MaxAlignment.value() >= ProbeSize) {
1256 bool IsRV64 = STI.is64Bit();
1257 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
1258 .addReg(RegNo: RISCV::X0)
1259 .addReg(RegNo: SPReg)
1260 .addImm(Val: 0)
1261 .setMIFlags(MachineInstr::FrameSetup);
1262 }
1263 }
1264 // FP will be used to restore the frame in the epilogue, so we need
1265 // another base register BP to record SP after re-alignment. SP will
1266 // track the current stack after allocating variable sized objects.
1267 if (hasBP(MF)) {
1268 // move BP, SP
1269 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: BPReg)
1270 .addReg(RegNo: SPReg)
1271 .addImm(Val: 0)
1272 .setMIFlag(MachineInstr::FrameSetup);
1273 }
1274 }
1275 }
1276}
1277
1278void RISCVFrameLowering::deallocateStack(MachineFunction &MF,
1279 MachineBasicBlock &MBB,
1280 MachineBasicBlock::iterator MBBI,
1281 const DebugLoc &DL,
1282 uint64_t &StackSize,
1283 int64_t CFAOffset) const {
1284 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
1285
1286 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: StackSize),
1287 Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign());
1288 StackSize = 0;
1289
1290 if (needsDwarfCFI(MF))
1291 CFIInstBuilder(MBB, MBBI, MachineInstr::FrameDestroy)
1292 .buildDefCFAOffset(Offset: CFAOffset);
1293}
1294
1295void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
1296 MachineBasicBlock &MBB) const {
1297 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
1298 MachineFrameInfo &MFI = MF.getFrameInfo();
1299 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1300 bool PreferAscendingLS = STI.preferAscendingLoadStore();
1301
1302 // All calls are tail calls in GHC calling conv, and functions have no
1303 // prologue/epilogue.
1304 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
1305 return;
1306
1307 // Get the insert location for the epilogue. If there were no terminators in
1308 // the block, get the last instruction.
1309 MachineBasicBlock::iterator MBBI = MBB.end();
1310 DebugLoc DL;
1311 if (!MBB.empty()) {
1312 MBBI = MBB.getLastNonDebugInstr();
1313 if (MBBI != MBB.end())
1314 DL = MBBI->getDebugLoc();
1315
1316 MBBI = MBB.getFirstTerminator();
1317
1318 // Skip to before the restores of all callee-saved registers.
1319 while (MBBI != MBB.begin() &&
1320 std::prev(x: MBBI)->getFlag(Flag: MachineInstr::FrameDestroy))
1321 --MBBI;
1322 }
1323
1324 const auto &CSI = MFI.getCalleeSavedInfo();
1325
1326 // Skip to before the restores of scalar callee-saved registers
1327 // FIXME: assumes exactly one instruction is used to restore each
1328 // callee-saved register.
1329 auto FirstScalarCSRRestoreInsn =
1330 std::next(x: MBBI, n: getRVVCalleeSavedInfo(MF, CSI).size());
1331 CFIInstBuilder CFIBuilder(MBB, FirstScalarCSRRestoreInsn,
1332 MachineInstr::FrameDestroy);
1333 bool NeedsDwarfCFI = needsDwarfCFI(MF);
1334
1335 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
1336 uint64_t RealStackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
1337 : getStackSizeWithRVVPadding(MF);
1338 uint64_t StackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
1339 : getStackSizeWithRVVPadding(MF) -
1340 RVFI->getReservedSpillsSize();
1341 uint64_t FPOffset = RealStackSize - RVFI->getVarArgsSaveSize();
1342 uint64_t RVVStackSize = RVFI->getRVVStackSize();
1343
1344 // An unreserved call frame moves SP only around calls that pass arguments
1345 // on the stack. Otherwise SP is where the prologue left it, and the FP
1346 // restore is worth keeping only where it replaces the multi-instruction
1347 // undo of an RVV area or a split SP adjustment.
1348 bool CallFrameMovesSP =
1349 !hasReservedCallFrame(MF) &&
1350 (!MFI.isMaxCallFrameSizeComputed() || MFI.getMaxCallFrameSize() != 0);
1351 bool RestoreSPFromFP =
1352 RI->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
1353 CallFrameMovesSP ||
1354 (!hasReservedCallFrame(MF) && (RVVStackSize || FirstSPAdjustAmount));
1355 if (RVVStackSize) {
1356 // If RestoreSPFromFP the stack pointer will be restored using the frame
1357 // pointer value.
1358 if (!RestoreSPFromFP)
1359 RI->adjustReg(MBB, II: FirstScalarCSRRestoreInsn, DL, DestReg: SPReg, SrcReg: SPReg,
1360 Offset: StackOffset::getScalable(Scalable: RVVStackSize),
1361 Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign());
1362
1363 if (NeedsDwarfCFI) {
1364 if (!hasFP(MF))
1365 CFIBuilder.buildDefCFA(Reg: SPReg, Offset: RealStackSize);
1366 emitCalleeSavedRVVEpilogCFI(MBB, MI: FirstScalarCSRRestoreInsn);
1367 }
1368 }
1369
1370 if (FirstSPAdjustAmount) {
1371 uint64_t SecondSPAdjustAmount =
1372 getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
1373 assert(SecondSPAdjustAmount > 0 &&
1374 "SecondSPAdjustAmount should be greater than zero");
1375
1376 // If RestoreSPFromFP the stack pointer will be restored using the frame
1377 // pointer value.
1378 if (!RestoreSPFromFP)
1379 RI->adjustReg(MBB, II: FirstScalarCSRRestoreInsn, DL, DestReg: SPReg, SrcReg: SPReg,
1380 Offset: StackOffset::getFixed(Fixed: SecondSPAdjustAmount),
1381 Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign());
1382
1383 if (NeedsDwarfCFI && !hasFP(MF))
1384 CFIBuilder.buildDefCFAOffset(Offset: FirstSPAdjustAmount);
1385 }
1386
1387 // Restore the stack pointer using the value of the frame pointer. Only
1388 // necessary if the stack pointer was modified, meaning the stack size is
1389 // unknown.
1390 //
1391 // In order to make sure the stack point is right through the EH region,
1392 // we also need to restore stack pointer from the frame pointer if we
1393 // don't preserve stack space within prologue/epilogue for outgoing variables,
1394 // normally it's just checking the variable sized object is present or not
1395 // is enough, but we also don't preserve that at prologue/epilogue when
1396 // have vector objects in stack.
1397 if (RestoreSPFromFP) {
1398 assert(hasFP(MF) && "frame pointer should not have been eliminated");
1399 RI->adjustReg(MBB, II: FirstScalarCSRRestoreInsn, DL, DestReg: SPReg, SrcReg: FPReg,
1400 Offset: StackOffset::getFixed(Fixed: -FPOffset), Flag: MachineInstr::FrameDestroy,
1401 RequiredAlign: getStackAlign());
1402 }
1403
1404 if (NeedsDwarfCFI && hasFP(MF))
1405 CFIBuilder.buildDefCFA(Reg: SPReg, Offset: RealStackSize);
1406
1407 // Skip to after the restores of scalar callee-saved registers
1408 // FIXME: assumes exactly one instruction is used to restore each
1409 // callee-saved register.
1410 MBBI = std::next(x: FirstScalarCSRRestoreInsn,
1411 n: getUnmanagedInterruptCSI(MF, CSI, ReverseOrder: PreferAscendingLS).size());
1412 CFIBuilder.setInsertPoint(MBBI);
1413 emitSiFiveCLICPreemptibleRestores(MF, MBB, MBBI, CFIBuilder, DL);
1414
1415 auto emitRestoreCFI = [&](auto CSInfo) {
1416 for (auto &CS : CSInfo) {
1417 MCRegister Reg = CS.getReg();
1418 // Emit CFI for both sub-registers.
1419 if (RISCV::GPRPairRegClass.contains(Reg)) {
1420 MCRegister EvenReg = RI->getSubReg(Reg, Idx: RISCV::sub_gpr_even);
1421 MCRegister OddReg = RI->getSubReg(Reg, Idx: RISCV::sub_gpr_odd);
1422 CFIBuilder.buildRestore(Reg: EvenReg);
1423 CFIBuilder.buildRestore(Reg: OddReg);
1424 } else {
1425 CFIBuilder.buildRestore(Reg);
1426 }
1427 }
1428 };
1429
1430 if (RVFI->useSaveRestoreLibCalls(MF)) {
1431 if (RVFI->hasShadowStack(MF)) {
1432 // We don't use `__riscv_restore_<N>` to restore in this case, so we need
1433 // to make sure we're correctly accounting for the stack deallocation we
1434 // need to do, which now must include the allocation made by
1435 // `__riscv_save_<N>`.
1436 StackSize += RVFI->getLibCallStackSize();
1437
1438 SmallVector<CalleeSavedInfo, 8> LibcallCSI =
1439 getPushOrLibCallsSavedInfo(MF, CSI);
1440 MBBI = std::next(x: MBBI, n: LibcallCSI.size());
1441 CFIBuilder.setInsertPoint(MBBI);
1442
1443 if (NeedsDwarfCFI)
1444 emitRestoreCFI(LibcallCSI);
1445 } else {
1446 // tail __riscv_restore_[0-12] instruction is considered as a terminator,
1447 // therefore it is unnecessary to place any CFI instructions after it.
1448 // Just deallocate stack if needed and return.
1449 if (StackSize != 0)
1450 deallocateStack(MF, MBB, MBBI, DL, StackSize,
1451 CFAOffset: RVFI->getLibCallStackSize());
1452
1453 // Emit epilogue for shadow call stack.
1454 emitSCSEpilogue(MF, MBB, MI: MBBI, DL);
1455 return;
1456 }
1457 }
1458
1459 // Recover callee-saved registers.
1460 if (NeedsDwarfCFI)
1461 emitRestoreCFI(getUnmanagedInterruptCSI(MF, CSI, ReverseOrder: PreferAscendingLS));
1462
1463 if (RVFI->isPushable(MF) && MBBI != MBB.end() && isPop(Opcode: MBBI->getOpcode())) {
1464 // Use available stack adjustment in pop instruction to deallocate stack
1465 // space. Align the stack size down to a multiple of 16. This is needed for
1466 // RVE.
1467 // FIXME: Can we increase the stack size to a multiple of 16 instead?
1468 uint64_t StackAdj =
1469 std::min(a: alignDown(Value: StackSize, Align: 16), b: static_cast<uint64_t>(48));
1470 MBBI->getOperand(i: 1).setImm(StackAdj);
1471 StackSize -= StackAdj;
1472
1473 if (StackSize != 0)
1474 deallocateStack(MF, MBB, MBBI, DL, StackSize,
1475 /*stack_adj of cm.pop instr*/ CFAOffset: RealStackSize - StackSize);
1476
1477 auto NextI = next_nodbg(It: MBBI, End: MBB.end());
1478 if (NextI == MBB.end() || NextI->getOpcode() != RISCV::PseudoRET) {
1479 ++MBBI;
1480 if (NeedsDwarfCFI) {
1481 CFIBuilder.setInsertPoint(MBBI);
1482
1483 for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
1484 CFIBuilder.buildRestore(Reg: CS.getReg());
1485
1486 // Update CFA Offset. If this is a QCI interrupt function, there will
1487 // be a leftover offset which is deallocated by `QC.C.MILEAVERET`,
1488 // otherwise getQCIInterruptStackSize() will be 0.
1489 CFIBuilder.buildDefCFAOffset(Offset: RVFI->getQCIInterruptStackSize());
1490 }
1491 }
1492 }
1493
1494 // Deallocate stack if StackSize isn't a zero yet. If this is a QCI interrupt
1495 // function, there will be a leftover offset which is deallocated by
1496 // `QC.C.MILEAVERET`, otherwise getQCIInterruptStackSize() will be 0.
1497 if (StackSize != 0)
1498 deallocateStack(MF, MBB, MBBI, DL, StackSize,
1499 CFAOffset: RVFI->getQCIInterruptStackSize());
1500
1501 // Emit epilogue for shadow call stack.
1502 emitSCSEpilogue(MF, MBB, MI: MBBI, DL);
1503
1504 // SiFive CLIC needs to swap `sf.mscratchcsw` into `sp`
1505 emitSiFiveCLICStackSwap(MF, MBB, MBBI, DL, FrameFlag: MachineInstr::FrameDestroy);
1506}
1507
1508static MCRegister getPhysicalGPR(const TargetRegisterInfo &TRI,
1509 MCRegister Reg) {
1510 if (RISCV::GPRRegClass.contains(Reg))
1511 return Reg;
1512
1513 std::array<TargetRegisterClass const *, 2> RegisterClasses = {
1514 &RISCV::GPRF16RegClass, &RISCV::GPRF32RegClass};
1515 std::array<unsigned, 2> SubIdx = {RISCV::sub_16, RISCV::sub_32};
1516
1517 for (auto [RegClass, SubReg] : zip(t&: RegisterClasses, u&: SubIdx)) {
1518 if (RegClass->contains(Reg)) {
1519 if (MCRegister Super =
1520 TRI.getMatchingSuperReg(Reg, SubIdx: SubReg, RC: &RISCV::GPRRegClass))
1521 return Super;
1522 }
1523 }
1524
1525 llvm::reportFatalInternalError(
1526 reason: "getPhysicalGPR called with unsupported register");
1527}
1528
1529static MCRegister getLargestFPRegisterOrZero(const RISCVSubtarget &STI,
1530 const TargetRegisterInfo &TRI,
1531 MCRegister Reg) {
1532 if (!STI.hasStdExtF())
1533 return MCRegister();
1534
1535 TargetRegisterClass const *LargestFPRegClass = STI.getLargestFPRegClass();
1536 assert(LargestFPRegClass);
1537
1538 if (LargestFPRegClass->contains(Reg))
1539 return Reg;
1540
1541 std::array<TargetRegisterClass const *, 3> RegisterClasses = {
1542 &RISCV::FPR16RegClass, &RISCV::FPR32RegClass, &RISCV::FPR64RegClass};
1543 std::array<unsigned, 3> SubIdx = {RISCV::sub_16, RISCV::sub_32,
1544 RISCV::sub_64};
1545
1546 for (auto [RegClass, SubReg] : zip(t&: RegisterClasses, u&: SubIdx)) {
1547 if (RegClass->contains(Reg)) {
1548 if (MCRegister Super =
1549 TRI.getMatchingSuperReg(Reg, SubIdx: SubReg, RC: LargestFPRegClass))
1550 return Super;
1551 }
1552 }
1553
1554 // Reg is bigger than what's currently available for the target, we can ignore
1555 // it.
1556 return MCRegister();
1557}
1558
1559void RISCVFrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero,
1560 MachineBasicBlock &MBB,
1561 RegScavenger *RS) const {
1562 // Insertion point.
1563 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
1564
1565 // Fake a debug loc.
1566 DebugLoc DL;
1567 if (MBBI != MBB.end())
1568 DL = MBBI->getDebugLoc();
1569
1570 const MachineFunction &MF = *MBB.getParent();
1571 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
1572 const RISCVInstrInfo &TII = *STI.getInstrInfo();
1573
1574 BitVector FinalRegsToZero(TRI.getNumRegs());
1575
1576 bool HasVRegister = false;
1577
1578 for (MCRegister Reg : RegsToZero.set_bits()) {
1579 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
1580 FinalRegsToZero.set(getPhysicalGPR(TRI, Reg).id());
1581 } else if (RISCV::GPRPairRegClass.contains(Reg)) {
1582 FinalRegsToZero.set(
1583 getPhysicalGPR(TRI, Reg: TRI.getSubReg(Reg, Idx: RISCV::sub_gpr_even)).id());
1584 FinalRegsToZero.set(
1585 getPhysicalGPR(TRI, Reg: TRI.getSubReg(Reg, Idx: RISCV::sub_gpr_odd)).id());
1586 } else if (TRI.isFPRegister(Reg)) {
1587 if (MCRegister MaybeReg = getLargestFPRegisterOrZero(STI, TRI, Reg))
1588 FinalRegsToZero.set(MaybeReg.id());
1589 } else if (RISCVRegisterInfo::isRVVRegClass(
1590 RC: TRI.getMinimalPhysRegClass(Reg))) {
1591 if (!STI.hasVInstructions())
1592 continue;
1593 HasVRegister = true;
1594
1595 for (MCRegister SubReg : TRI.subregs_inclusive(Reg)) {
1596 if (TRI.subregs(Reg: SubReg).empty())
1597 FinalRegsToZero.set(SubReg.id());
1598 }
1599 }
1600 }
1601
1602 if (HasVRegister) {
1603 RISCVVType::VLMUL VLMUL = RISCVVType::encodeLMUL(LMUL: 1, /*Fractional=*/false);
1604 unsigned VTypeImm = RISCVVType::encodeVTYPE(
1605 VLMUL, /*SEW=*/32, /*TailAgnostic=*/true, /*MaskAgnostic=*/true);
1606
1607 MCRegister TemporaryReg = RISCV::NoRegister;
1608 for (MCRegister Reg : FinalRegsToZero.set_bits()) {
1609 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
1610 TemporaryReg = Reg;
1611 break;
1612 }
1613 }
1614
1615 if (TemporaryReg == RISCV::NoRegister) {
1616 RS->enterBasicBlockEnd(MBB);
1617 TemporaryReg = RS->scavengeRegisterBackwards(RC: RISCV::GPRRegClass, To: MBBI,
1618 /*RestoreAfter=*/false,
1619 /*SPAdj=*/0);
1620 }
1621
1622 if (MBB.getParent()
1623 ->getFunction()
1624 .getFnAttribute(Kind: "zero-call-used-regs")
1625 .getValueAsString() == "used")
1626 FinalRegsToZero.set(TemporaryReg.id());
1627
1628 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: RISCV::VSETVLI), DestReg: TemporaryReg)
1629 .addReg(RegNo: RISCV::X0)
1630 .addImm(Val: VTypeImm)
1631 .addReg(RegNo: RISCV::VL, Flags: RegState::ImplicitDefine)
1632 .addReg(RegNo: RISCV::VTYPE, Flags: RegState::ImplicitDefine);
1633 }
1634
1635 for (MCRegister Reg : FinalRegsToZero.set_bits())
1636 TII.buildClearRegister(Reg, MBB, Iter: MBBI, DL);
1637}
1638
1639StackOffset
1640RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
1641 Register &FrameReg) const {
1642 const MachineFrameInfo &MFI = MF.getFrameInfo();
1643 const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
1644 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1645
1646 // Callee-saved registers should be referenced relative to the stack
1647 // pointer (positive offset), otherwise use the frame pointer (negative
1648 // offset).
1649 const auto &CSI = getUnmanagedCSI(MF, CSI: MFI.getCalleeSavedInfo(),
1650 ReverseOrder: STI.preferAscendingLoadStore());
1651 int MinCSFI = 0;
1652 int MaxCSFI = -1;
1653 StackOffset Offset;
1654 auto StackID = MFI.getStackID(ObjectIdx: FI);
1655
1656 assert((StackID == TargetStackID::Default ||
1657 StackID == TargetStackID::ScalableVector) &&
1658 "Unexpected stack ID for the frame object.");
1659 if (StackID == TargetStackID::Default) {
1660 assert(getOffsetOfLocalArea() == 0 && "LocalAreaOffset is not 0!");
1661 Offset = StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FI) +
1662 MFI.getOffsetAdjustment());
1663 } else if (StackID == TargetStackID::ScalableVector) {
1664 Offset = StackOffset::getScalable(Scalable: MFI.getObjectOffset(ObjectIdx: FI));
1665 }
1666
1667 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
1668
1669 if (CSI.size()) {
1670 MinCSFI = std::min(a: CSI.front().getFrameIdx(), b: CSI.back().getFrameIdx());
1671 MaxCSFI = std::max(a: CSI.front().getFrameIdx(), b: CSI.back().getFrameIdx());
1672 }
1673
1674 auto FIIsCalleeSave = [&](const CalleeSavedInfo &CS) -> bool {
1675 return FI == CS.getFrameIdx();
1676 };
1677
1678 bool IsInterruptCSR = RVFI->isSiFivePreemptibleInterrupt(MF) &&
1679 (FI == RVFI->getInterruptCSRFrameIndex(Idx: 0) ||
1680 FI == RVFI->getInterruptCSRFrameIndex(Idx: 1));
1681 bool IsInlineRestoredLibcallCSR = llvm::any_of(
1682 Range: getPushOrLibCallsSavedInfo(MF, CSI: MFI.getCalleeSavedInfo()), P: FIIsCalleeSave);
1683 if ((FI >= MinCSFI && FI <= MaxCSFI) || IsInterruptCSR ||
1684 IsInlineRestoredLibcallCSR) {
1685 FrameReg = SPReg;
1686
1687 if (FirstSPAdjustAmount)
1688 Offset += StackOffset::getFixed(Fixed: FirstSPAdjustAmount);
1689 else
1690 Offset += StackOffset::getFixed(Fixed: getStackSizeWithRVVPadding(MF));
1691 return Offset;
1692 }
1693
1694 if (RI->hasStackRealignment(MF) && !MFI.isFixedObjectIndex(ObjectIdx: FI)) {
1695 // If the stack was realigned, the frame pointer is set in order to allow
1696 // SP to be restored, so we need another base register to record the stack
1697 // after realignment.
1698 // |--------------------------| --
1699 // | callee-allocated save | | <----|
1700 // | area for register varargs| | |
1701 // |--------------------------| <-- FP |
1702 // | callee-saved registers | | |
1703 // |--------------------------| -- |
1704 // | realignment (the size of | | |
1705 // | this area is not counted | | |
1706 // | in MFI.getStackSize()) | | |
1707 // |--------------------------| -- |-- MFI.getStackSize()
1708 // | RVV alignment padding | | |
1709 // | (not counted in | | |
1710 // | MFI.getStackSize() but | | |
1711 // | counted in | | |
1712 // | RVFI.getRVVStackSize()) | | |
1713 // |--------------------------| -- |
1714 // | RVV objects | | |
1715 // | (not counted in | | |
1716 // | MFI.getStackSize()) | | |
1717 // |--------------------------| -- |
1718 // | padding before RVV | | |
1719 // | (not counted in | | |
1720 // | MFI.getStackSize() or in | | |
1721 // | RVFI.getRVVStackSize()) | | |
1722 // |--------------------------| -- |
1723 // | scalar local variables | | <----'
1724 // |--------------------------| -- <-- BP (if var sized objects present)
1725 // | VarSize objects | |
1726 // |--------------------------| -- <-- SP
1727 if (hasBP(MF)) {
1728 FrameReg = RISCVABI::getBPReg();
1729 } else {
1730 // VarSize objects must be empty in this case!
1731 assert(!MFI.hasVarSizedObjects());
1732 FrameReg = SPReg;
1733 }
1734 } else if (!RI->hasStackRealignment(MF)) {
1735 // Note: Keeping the following as multiple 'if' statements rather than
1736 // merging to a single expression for readability.
1737 if (!hasFP(MF)) {
1738 // No FP available, must use SP.
1739 FrameReg = SPReg;
1740 } else {
1741 FrameReg = FPReg;
1742 // SP-relative addressing is only valid when SP is stable throughout
1743 // the function body: no dynamic SP adjustments for outgoing call args,
1744 // no variable-sized objects, and no RVV scalable stack regions.
1745 // hasReservedCallFrame() conservatively encompasses all these checks.
1746 if (hasReservedCallFrame(MF)) {
1747 // Both FP and SP are candidates.
1748 // Prefer SP when the SP-relative offset fits in the compressed
1749 // instruction immediate range.
1750 int64_t SPOff = Offset.getFixed() + MFI.getStackSize();
1751 int64_t CLWSPMaxOffset = 252;
1752 int64_t CLDSPMaxOffset = 504;
1753 int64_t SPThreshold = STI.is64Bit() ? CLDSPMaxOffset : CLWSPMaxOffset;
1754 if (SPOff >= 0 && SPOff <= SPThreshold)
1755 FrameReg = SPReg;
1756 }
1757 }
1758 } else {
1759 assert(RI->hasStackRealignment(MF) && MFI.isFixedObjectIndex(FI) &&
1760 "Expected fixed object with stack realignment");
1761 assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
1762 FrameReg = FPReg;
1763 }
1764
1765 if (FrameReg == FPReg) {
1766 Offset += StackOffset::getFixed(Fixed: RVFI->getVarArgsSaveSize());
1767 // When using FP to access scalable vector objects, we need to minus
1768 // the frame size.
1769 //
1770 // |--------------------------| --
1771 // | callee-allocated save | |
1772 // | area for register varargs| |
1773 // |--------------------------| | -- <-- FP
1774 // | callee-saved registers | |
1775 // |--------------------------| | MFI.getStackSize()
1776 // | scalar local variables | |
1777 // |--------------------------| -- (Offset of RVV objects is from here.)
1778 // | RVV objects |
1779 // |--------------------------|
1780 // | VarSize objects |
1781 // |--------------------------| <-- SP
1782 if (StackID == TargetStackID::ScalableVector) {
1783 assert(!RI->hasStackRealignment(MF) &&
1784 "Can't index across variable sized realign");
1785 // We don't expect any extra RVV alignment padding, as the stack size
1786 // and RVV object sections should be correct aligned in their own
1787 // right.
1788 assert(MFI.getStackSize() == getStackSizeWithRVVPadding(MF) &&
1789 "Inconsistent stack layout");
1790 Offset -= StackOffset::getFixed(Fixed: MFI.getStackSize());
1791 }
1792 return Offset;
1793 }
1794
1795 // This case handles indexing off both SP and BP.
1796 // If indexing off SP, there must not be any var sized objects
1797 assert(FrameReg == RISCVABI::getBPReg() || !MFI.hasVarSizedObjects());
1798
1799 // When using SP to access frame objects, we need to add RVV stack size.
1800 //
1801 // |--------------------------| --
1802 // | callee-allocated save | | <----|
1803 // | area for register varargs| | |
1804 // |--------------------------| | | <-- FP
1805 // | callee-saved registers | | |
1806 // |--------------------------| -- |
1807 // | RVV alignment padding | | |
1808 // | (not counted in | | |
1809 // | MFI.getStackSize() but | | |
1810 // | counted in | | |
1811 // | RVFI.getRVVStackSize()) | | |
1812 // |--------------------------| -- |
1813 // | RVV objects | | |-- MFI.getStackSize()
1814 // | (not counted in | | |
1815 // | MFI.getStackSize()) | | |
1816 // |--------------------------| -- |
1817 // | padding before RVV | | |
1818 // | (not counted in | | |
1819 // | MFI.getStackSize()) | | |
1820 // |--------------------------| -- |
1821 // | scalar local variables | | <----'
1822 // |--------------------------| -- <-- BP (if var sized objects present)
1823 // | VarSize objects | |
1824 // |--------------------------| -- <-- SP
1825 //
1826 // The total amount of padding surrounding RVV objects is described by
1827 // RVV->getRVVPadding() and it can be zero. It allows us to align the RVV
1828 // objects to the required alignment.
1829 if (MFI.getStackID(ObjectIdx: FI) == TargetStackID::Default) {
1830 if (MFI.isFixedObjectIndex(ObjectIdx: FI)) {
1831 assert(!RI->hasStackRealignment(MF) &&
1832 "Can't index across variable sized realign");
1833 Offset += StackOffset::get(Fixed: getStackSizeWithRVVPadding(MF),
1834 Scalable: RVFI->getRVVStackSize());
1835 } else {
1836 Offset += StackOffset::getFixed(Fixed: MFI.getStackSize());
1837 }
1838 } else if (MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector) {
1839 // Ensure the base of the RVV stack is correctly aligned: add on the
1840 // alignment padding.
1841 int64_t ScalarLocalVarSize =
1842 MFI.getStackSize() - RVFI->getCalleeSavedStackSize() -
1843 RVFI->getVarArgsSaveSize() + RVFI->getRVVPadding();
1844 Offset += StackOffset::get(Fixed: ScalarLocalVarSize, Scalable: RVFI->getRVVStackSize());
1845 }
1846 return Offset;
1847}
1848
1849static MCRegister getRVVBaseRegister(const RISCVRegisterInfo &TRI,
1850 const Register &Reg) {
1851 MCRegister BaseReg = TRI.getSubReg(Reg, Idx: RISCV::sub_vrm1_0);
1852 // If it's not a grouped vector register, it doesn't have subregister, so
1853 // the base register is just itself.
1854 if (!BaseReg.isValid())
1855 BaseReg = Reg;
1856 return BaseReg;
1857}
1858
1859void RISCVFrameLowering::determineCalleeSaves(MachineFunction &MF,
1860 BitVector &SavedRegs,
1861 RegScavenger *RS) const {
1862 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
1863
1864 // In TargetFrameLowering::determineCalleeSaves, any vector register is marked
1865 // as saved if any of its subregister is clobbered, this is not correct in
1866 // vector registers. We only want the vector register to be marked as saved
1867 // if all of its subregisters are clobbered.
1868 // For example:
1869 // Original behavior: If v24 is marked, v24m2, v24m4, v24m8 are also marked.
1870 // Correct behavior: v24m2 is marked only if v24 and v25 are marked.
1871 MachineRegisterInfo &MRI = MF.getRegInfo();
1872 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
1873 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
1874 for (unsigned i = 0; CSRegs[i]; ++i) {
1875 unsigned CSReg = CSRegs[i];
1876 // Only vector registers need special care.
1877 if (!RISCV::VRRegClass.contains(Reg: getRVVBaseRegister(TRI, Reg: CSReg)))
1878 continue;
1879
1880 SavedRegs.reset(Idx: CSReg);
1881
1882 auto SubRegs = TRI.subregs(Reg: CSReg);
1883 // Set the register and all its subregisters.
1884 if (!MRI.def_empty(RegNo: CSReg) || MRI.getUsedPhysRegsMask().test(Idx: CSReg)) {
1885 SavedRegs.set(CSReg);
1886 for (unsigned Reg : SubRegs)
1887 SavedRegs.set(Reg);
1888 }
1889
1890 }
1891
1892 // Unconditionally spill RA and FP only if the function uses a frame
1893 // pointer.
1894 if (hasFP(MF)) {
1895 SavedRegs.set(RAReg);
1896 SavedRegs.set(FPReg);
1897 }
1898 // Mark BP as used if function has dedicated base pointer.
1899 if (hasBP(MF))
1900 SavedRegs.set(RISCVABI::getBPReg());
1901
1902 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1903 // X5 is used as a temporary for saving and restoring `mcause` and `mepc`.
1904 if (RVFI->isSiFivePreemptibleInterrupt(MF))
1905 SavedRegs.set(RISCV::X5);
1906
1907 // When using cm.push/pop we must save X27 if we save X26.
1908 if (RVFI->isPushable(MF) && SavedRegs.test(Idx: RISCV::X26))
1909 SavedRegs.set(RISCV::X27);
1910
1911 // For Zilsd on RV32, append GPRPair registers to the CSR list. This prevents
1912 // the need to create register sets for each abi which is a lot more complex.
1913 // Don't use Zilsd for callee-saved coalescing if the required alignment
1914 // exceeds the stack alignment or when Zcmp/Xqccmp or save/restore libcalls
1915 // are enabled.
1916 bool UseZilsd = !STI.is64Bit() && STI.hasStdExtZilsd() &&
1917 STI.getZilsdAlign() <= getStackAlign() &&
1918 !RVFI->isPushable(MF) && !RVFI->useSaveRestoreLibCalls(MF);
1919 if (UseZilsd) {
1920 SmallVector<MCPhysReg, 32> NewCSRs;
1921 SmallSet<MCPhysReg, 16> CSRSet;
1922 for (unsigned i = 0; CSRegs[i]; ++i) {
1923 NewCSRs.push_back(Elt: CSRegs[i]);
1924 CSRSet.insert(V: CSRegs[i]);
1925 }
1926
1927 // Append GPRPair registers for pairs where both sub-registers are in CSR
1928 // list. Iterate through all GPRPairs and check if both sub-regs are CSRs.
1929 for (MCPhysReg Pair : RISCV::GPRPairRegClass) {
1930 // Do not append a pair that's already in the CSR list.
1931 if (CSRSet.contains(V: Pair))
1932 continue;
1933 MCRegister EvenReg = TRI.getSubReg(Reg: Pair, Idx: RISCV::sub_gpr_even);
1934 MCRegister OddReg = TRI.getSubReg(Reg: Pair, Idx: RISCV::sub_gpr_odd);
1935 if (CSRSet.contains(V: EvenReg.id()) && CSRSet.contains(V: OddReg.id())) {
1936 NewCSRs.push_back(Elt: Pair);
1937 CSRSet.insert(V: Pair);
1938 }
1939 }
1940
1941 MRI.setCalleeSavedRegs(NewCSRs);
1942 CSRegs = MRI.getCalleeSavedRegs();
1943 }
1944
1945 // Check if all subregisters are marked for saving. If so, set the super
1946 // register bit. For GPRPair, only check sub_gpr_even and sub_gpr_odd, not
1947 // aliases like X8_W or X8_H which are not set in SavedRegs.
1948 for (unsigned i = 0; CSRegs[i]; ++i) {
1949 MCRegister CSReg = CSRegs[i];
1950 bool CombineToSuperReg;
1951 if (RISCV::GPRPairRegClass.contains(Reg: CSReg)) {
1952 MCRegister EvenReg = TRI.getSubReg(Reg: CSReg, Idx: RISCV::sub_gpr_even);
1953 MCRegister OddReg = TRI.getSubReg(Reg: CSReg, Idx: RISCV::sub_gpr_odd);
1954 CombineToSuperReg =
1955 SavedRegs.test(Idx: EvenReg.id()) && SavedRegs.test(Idx: OddReg.id());
1956 // If s0(x8) is used as FP we can't generate load/store pair because it
1957 // breaks the frame chain.
1958 if (hasFP(MF) && CSReg == RISCV::X8_X9)
1959 CombineToSuperReg = false;
1960 } else {
1961 auto SubRegs = TRI.subregs(Reg: CSReg);
1962 CombineToSuperReg =
1963 !SubRegs.empty() && llvm::all_of(Range&: SubRegs, P: [&](unsigned Reg) {
1964 return SavedRegs.test(Idx: Reg);
1965 });
1966 }
1967
1968 if (CombineToSuperReg)
1969 SavedRegs.set(CSReg);
1970 }
1971
1972 // SiFive Preemptible Interrupt Handlers need additional frame entries
1973 createSiFivePreemptibleInterruptFrameEntries(MF, RVFI&: *RVFI);
1974}
1975
1976std::pair<int64_t, Align>
1977RISCVFrameLowering::assignRVVStackObjectOffsets(MachineFunction &MF) const {
1978 MachineFrameInfo &MFI = MF.getFrameInfo();
1979 // Create a buffer of RVV objects to allocate.
1980 SmallVector<int, 8> ObjectsToAllocate;
1981 auto pushRVVObjects = [&](int FIBegin, int FIEnd) {
1982 for (int I = FIBegin, E = FIEnd; I != E; ++I) {
1983 unsigned StackID = MFI.getStackID(ObjectIdx: I);
1984 if (StackID != TargetStackID::ScalableVector)
1985 continue;
1986 if (MFI.isDeadObjectIndex(ObjectIdx: I))
1987 continue;
1988
1989 ObjectsToAllocate.push_back(Elt: I);
1990 }
1991 };
1992 // First push RVV Callee Saved object, then push RVV stack object
1993 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
1994 const auto &RVVCSI = getRVVCalleeSavedInfo(MF, CSI);
1995 if (!RVVCSI.empty())
1996 pushRVVObjects(RVVCSI[0].getFrameIdx(),
1997 RVVCSI[RVVCSI.size() - 1].getFrameIdx() + 1);
1998 pushRVVObjects(0, MFI.getObjectIndexEnd() - RVVCSI.size());
1999
2000 // The minimum alignment is 16 bytes.
2001 Align RVVStackAlign(16);
2002 const auto &ST = MF.getSubtarget<RISCVSubtarget>();
2003
2004 if (!ST.hasVInstructions()) {
2005 assert(ObjectsToAllocate.empty() &&
2006 "Can't allocate scalable-vector objects without V instructions");
2007 return std::make_pair(x: 0, y&: RVVStackAlign);
2008 }
2009
2010 // Allocate all RVV locals and spills
2011 int64_t Offset = 0;
2012 for (int FI : ObjectsToAllocate) {
2013 // ObjectSize in bytes.
2014 int64_t ObjectSize = MFI.getObjectSize(ObjectIdx: FI);
2015 auto ObjectAlign =
2016 std::max(a: Align(RISCV::RVVBytesPerBlock), b: MFI.getObjectAlign(ObjectIdx: FI));
2017 // If the data type is the fractional vector type, reserve one vector
2018 // register for it.
2019 if (ObjectSize < RISCV::RVVBytesPerBlock)
2020 ObjectSize = RISCV::RVVBytesPerBlock;
2021 Offset = alignTo(Size: Offset + ObjectSize, A: ObjectAlign);
2022 MFI.setObjectOffset(ObjectIdx: FI, SPOffset: -Offset);
2023 // Update the maximum alignment of the RVV stack section
2024 RVVStackAlign = std::max(a: RVVStackAlign, b: ObjectAlign);
2025 }
2026
2027 uint64_t StackSize = Offset;
2028
2029 // Ensure the alignment of the RVV stack. Since we want the most-aligned
2030 // object right at the bottom (i.e., any padding at the top of the frame),
2031 // readjust all RVV objects down by the alignment padding.
2032 // Stack size and offsets are multiples of vscale, stack alignment is in
2033 // bytes, we can divide stack alignment by minimum vscale to get a maximum
2034 // stack alignment multiple of vscale.
2035 auto VScale =
2036 std::max<uint64_t>(a: ST.getRealMinVLen() / RISCV::RVVBitsPerBlock, b: 1);
2037 if (auto RVVStackAlignVScale = RVVStackAlign.value() / VScale) {
2038 if (auto AlignmentPadding =
2039 offsetToAlignment(Value: StackSize, Alignment: Align(RVVStackAlignVScale))) {
2040 StackSize += AlignmentPadding;
2041 for (int FI : ObjectsToAllocate)
2042 MFI.setObjectOffset(ObjectIdx: FI, SPOffset: MFI.getObjectOffset(ObjectIdx: FI) - AlignmentPadding);
2043 }
2044 }
2045
2046 return std::make_pair(x&: StackSize, y&: RVVStackAlign);
2047}
2048
2049static unsigned getScavSlotsNumForRVV(MachineFunction &MF) {
2050 // For RVV spill, scalable stack offsets computing requires up to two scratch
2051 // registers
2052 static constexpr unsigned ScavSlotsNumRVVSpillScalableObject = 2;
2053
2054 // For RVV spill, non-scalable stack offsets computing requires up to one
2055 // scratch register.
2056 static constexpr unsigned ScavSlotsNumRVVSpillNonScalableObject = 1;
2057
2058 // ADDI instruction's destination register can be used for computing
2059 // offsets. So Scalable stack offsets require up to one scratch register.
2060 static constexpr unsigned ScavSlotsADDIScalableObject = 1;
2061
2062 static constexpr unsigned MaxScavSlotsNumKnown =
2063 std::max(l: {ScavSlotsADDIScalableObject, ScavSlotsNumRVVSpillScalableObject,
2064 ScavSlotsNumRVVSpillNonScalableObject});
2065
2066 unsigned MaxScavSlotsNum = 0;
2067 if (!MF.getSubtarget<RISCVSubtarget>().hasVInstructions())
2068 return false;
2069 for (const MachineBasicBlock &MBB : MF)
2070 for (const MachineInstr &MI : MBB) {
2071 bool IsRVVSpill = RISCV::isRVVSpill(MI);
2072 for (auto &MO : MI.operands()) {
2073 if (!MO.isFI())
2074 continue;
2075 bool IsScalableVectorID = MF.getFrameInfo().getStackID(ObjectIdx: MO.getIndex()) ==
2076 TargetStackID::ScalableVector;
2077 if (IsRVVSpill) {
2078 MaxScavSlotsNum = std::max(
2079 a: MaxScavSlotsNum, b: IsScalableVectorID
2080 ? ScavSlotsNumRVVSpillScalableObject
2081 : ScavSlotsNumRVVSpillNonScalableObject);
2082 } else if (MI.getOpcode() == RISCV::ADDI && IsScalableVectorID) {
2083 MaxScavSlotsNum =
2084 std::max(a: MaxScavSlotsNum, b: ScavSlotsADDIScalableObject);
2085 }
2086 }
2087 if (MaxScavSlotsNum == MaxScavSlotsNumKnown)
2088 return MaxScavSlotsNumKnown;
2089 }
2090 return MaxScavSlotsNum;
2091}
2092
2093static bool hasRVVFrameObject(const MachineFunction &MF) {
2094 // Originally, the function will scan all the stack objects to check whether
2095 // if there is any scalable vector object on the stack or not. However, it
2096 // causes errors in the register allocator. In issue 53016, it returns false
2097 // before RA because there is no RVV stack objects. After RA, it returns true
2098 // because there are spilling slots for RVV values during RA. It will not
2099 // reserve BP during register allocation and generate BP access in the PEI
2100 // pass due to the inconsistent behavior of the function.
2101 //
2102 // The function is changed to use hasVInstructions() as the return value. It
2103 // is not precise, but it can make the register allocation correct.
2104 //
2105 // FIXME: Find a better way to make the decision or revisit the solution in
2106 // D103622.
2107 //
2108 // Refer to https://github.com/llvm/llvm-project/issues/53016.
2109 return MF.getSubtarget<RISCVSubtarget>().hasVInstructions();
2110}
2111
2112static unsigned estimateFunctionSizeInBytes(const MachineFunction &MF,
2113 const RISCVInstrInfo &TII) {
2114 unsigned FnSize = 0;
2115 for (auto &MBB : MF) {
2116 for (auto &MI : MBB) {
2117 // Far branches over 20-bit offset will be relaxed in branch relaxation
2118 // pass. In the worst case, conditional branches will be relaxed into
2119 // the following instruction sequence. Unconditional branches are
2120 // relaxed in the same way, with the exception that there is no first
2121 // branch instruction.
2122 //
2123 // foo
2124 // bne t5, t6, .rev_cond # `TII->getInstSizeInBytes(MI)` bytes
2125 // sd s11, 0(sp) # 4 bytes, or 2 bytes with Zca
2126 // jump .restore, s11 # 8 bytes
2127 // .rev_cond
2128 // bar
2129 // j .dest_bb # 4 bytes, or 2 bytes with Zca
2130 // .restore:
2131 // ld s11, 0(sp) # 4 bytes, or 2 bytes with Zca
2132 // .dest:
2133 // baz
2134 if (MI.isConditionalBranch())
2135 FnSize += TII.getInstSizeInBytes(MI);
2136 if (MI.isConditionalBranch() || MI.isUnconditionalBranch()) {
2137 if (MF.getSubtarget<RISCVSubtarget>().hasStdExtZca())
2138 FnSize += 2 + 8 + 2 + 2;
2139 else
2140 FnSize += 4 + 8 + 4 + 4;
2141 continue;
2142 }
2143
2144 FnSize += TII.getInstSizeInBytes(MI);
2145 }
2146 }
2147 return FnSize;
2148}
2149
2150void RISCVFrameLowering::processFunctionBeforeFrameFinalized(
2151 MachineFunction &MF, RegScavenger *RS) const {
2152 const RISCVRegisterInfo *RegInfo =
2153 MF.getSubtarget<RISCVSubtarget>().getRegisterInfo();
2154 const RISCVInstrInfo *TII = MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
2155 MachineFrameInfo &MFI = MF.getFrameInfo();
2156 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
2157 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
2158
2159 int64_t RVVStackSize;
2160 Align RVVStackAlign;
2161 std::tie(args&: RVVStackSize, args&: RVVStackAlign) = assignRVVStackObjectOffsets(MF);
2162
2163 RVFI->setRVVStackSize(RVVStackSize);
2164 RVFI->setRVVStackAlign(RVVStackAlign);
2165
2166 if (hasRVVFrameObject(MF)) {
2167 // Ensure the entire stack is aligned to at least the RVV requirement: some
2168 // scalable-vector object alignments are not considered by the
2169 // target-independent code.
2170 MFI.ensureMaxAlignment(Alignment: RVVStackAlign);
2171 }
2172
2173 unsigned ScavSlotsNum = 0;
2174
2175 // estimateStackSize has been observed to under-estimate the final stack
2176 // size, so give ourselves wiggle-room by checking for stack size
2177 // representable an 11-bit signed field rather than 12-bits.
2178 if (!isInt<11>(x: MFI.estimateStackSize(MF)))
2179 ScavSlotsNum = 1;
2180
2181 // Far branches over 20-bit offset require a spill slot for scratch register.
2182 bool IsLargeFunction = !isInt<20>(x: estimateFunctionSizeInBytes(MF, TII: *TII));
2183 if (IsLargeFunction)
2184 ScavSlotsNum = std::max(a: ScavSlotsNum, b: 1u);
2185
2186 // RVV loads & stores have no capacity to hold the immediate address offsets
2187 // so we must always reserve an emergency spill slot if the MachineFunction
2188 // contains any RVV spills.
2189 ScavSlotsNum = std::max(a: ScavSlotsNum, b: getScavSlotsNumForRVV(MF));
2190
2191 for (unsigned I = 0; I < ScavSlotsNum; I++) {
2192 int FI = MFI.CreateSpillStackObject(Size: RegInfo->getSpillSize(RC: *RC),
2193 Alignment: RegInfo->getSpillAlign(RC: *RC));
2194 RS->addScavengingFrameIndex(FI);
2195
2196 if (IsLargeFunction && RVFI->getBranchRelaxationScratchFrameIndex() == -1)
2197 RVFI->setBranchRelaxationScratchFrameIndex(FI);
2198 }
2199
2200 unsigned Size = RVFI->getReservedSpillsSize();
2201 for (const auto &Info : MFI.getCalleeSavedInfo()) {
2202 int FrameIdx = Info.getFrameIdx();
2203 if (FrameIdx < 0 || MFI.getStackID(ObjectIdx: FrameIdx) != TargetStackID::Default)
2204 continue;
2205
2206 Size += MFI.getObjectSize(ObjectIdx: FrameIdx);
2207 }
2208 RVFI->setCalleeSavedStackSize(Size);
2209}
2210
2211// Not preserve stack space within prologue for outgoing variables when the
2212// function contains variable size objects or there are vector objects accessed
2213// by the frame pointer.
2214// Let eliminateCallFramePseudoInstr preserve stack space for it.
2215bool RISCVFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
2216 return !MF.getFrameInfo().hasVarSizedObjects() &&
2217 !(hasFP(MF) && hasRVVFrameObject(MF));
2218}
2219
2220// Eliminate ADJCALLSTACKDOWN, ADJCALLSTACKUP pseudo instructions.
2221MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr(
2222 MachineFunction &MF, MachineBasicBlock &MBB,
2223 MachineBasicBlock::iterator MI) const {
2224 DebugLoc DL = MI->getDebugLoc();
2225
2226 if (!hasReservedCallFrame(MF)) {
2227 // If space has not been reserved for a call frame, ADJCALLSTACKDOWN and
2228 // ADJCALLSTACKUP must be converted to instructions manipulating the stack
2229 // pointer. This is necessary when there is a variable length stack
2230 // allocation (e.g. alloca), which means it's not possible to allocate
2231 // space for outgoing arguments from within the function prologue.
2232 int64_t Amount = MI->getOperand(i: 0).getImm();
2233
2234 if (Amount != 0) {
2235 // Ensure the stack remains aligned after adjustment.
2236 Amount = alignSPAdjust(SPAdj: Amount);
2237
2238 if (MI->getOpcode() == RISCV::ADJCALLSTACKDOWN)
2239 Amount = -Amount;
2240
2241 const RISCVTargetLowering *TLI =
2242 MF.getSubtarget<RISCVSubtarget>().getTargetLowering();
2243 int64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign: getStackAlign());
2244 if (TLI->hasInlineStackProbe(MF) && -Amount >= ProbeSize) {
2245 // When stack probing is enabled, the decrement of SP may need to be
2246 // probed. We can handle both the decrement and the probing in
2247 // allocateStack.
2248 bool DynAllocation =
2249 MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
2250 allocateStack(MBB, MBBI: MI, MF, Offset: -Amount, RealStackSize: -Amount,
2251 EmitCFI: needsDwarfCFI(MF) && !hasFP(MF),
2252 /*NeedProbe=*/true, ProbeSize, DynAllocation,
2253 Flag: MachineInstr::NoFlags);
2254 inlineStackProbe(MF, PrologueMBB&: MBB);
2255 } else {
2256 const RISCVRegisterInfo &RI = *STI.getRegisterInfo();
2257 RI.adjustReg(MBB, II: MI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: Amount),
2258 Flag: MachineInstr::NoFlags, RequiredAlign: getStackAlign());
2259 }
2260 }
2261 }
2262
2263 return MBB.erase(I: MI);
2264}
2265
2266// We would like to split the SP adjustment to reduce prologue/epilogue
2267// as following instructions. In this way, the offset of the callee saved
2268// register could fit in a single store. Supposed that the first sp adjust
2269// amount is 2032.
2270// add sp,sp,-2032
2271// sw ra,2028(sp)
2272// sw s0,2024(sp)
2273// sw s1,2020(sp)
2274// sw s3,2012(sp)
2275// sw s4,2008(sp)
2276// add sp,sp,-64
2277uint64_t
2278RISCVFrameLowering::getFirstSPAdjustAmount(const MachineFunction &MF) const {
2279 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
2280 const MachineFrameInfo &MFI = MF.getFrameInfo();
2281 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
2282 uint64_t StackSize = getStackSizeWithRVVPadding(MF);
2283
2284 // Disable SplitSPAdjust if save-restore libcall, push/pop or QCI interrupts
2285 // are used. The callee-saved registers will be pushed by the save-restore
2286 // libcalls, so we don't have to split the SP adjustment in this case.
2287 if (RVFI->getReservedSpillsSize())
2288 return 0;
2289
2290 // Return the FirstSPAdjustAmount if the StackSize can not fit in a signed
2291 // 12-bit and there exists a callee-saved register needing to be pushed.
2292 if (!isInt<12>(x: StackSize) && (CSI.size() > 0)) {
2293 // FirstSPAdjustAmount is chosen at most as (2048 - StackAlign) because
2294 // 2048 will cause sp = sp + 2048 in the epilogue to be split into multiple
2295 // instructions. Offsets smaller than 2048 can fit in a single load/store
2296 // instruction, and we have to stick with the stack alignment. 2048 has
2297 // 16-byte alignment. The stack alignment for RV32 and RV64 is 16 and for
2298 // RV32E it is 4. So (2048 - StackAlign) will satisfy the stack alignment.
2299 const uint64_t StackAlign = getStackAlign().value();
2300
2301 // Amount of (2048 - StackAlign) will prevent callee saved and restored
2302 // instructions be compressed, so try to adjust the amount to the largest
2303 // offset that stack compression instructions accept when target supports
2304 // compression instructions.
2305 if (STI.hasStdExtZca()) {
2306 // The compression extensions may support the following instructions:
2307 // riscv32: c.lwsp rd, offset[7:2] => 2^(6 + 2)
2308 // c.swsp rs2, offset[7:2] => 2^(6 + 2)
2309 // c.flwsp rd, offset[7:2] => 2^(6 + 2)
2310 // c.fswsp rs2, offset[7:2] => 2^(6 + 2)
2311 // riscv64: c.ldsp rd, offset[8:3] => 2^(6 + 3)
2312 // c.sdsp rs2, offset[8:3] => 2^(6 + 3)
2313 // c.fldsp rd, offset[8:3] => 2^(6 + 3)
2314 // c.fsdsp rs2, offset[8:3] => 2^(6 + 3)
2315 const uint64_t RVCompressLen = STI.getXLen() * 8;
2316 // Compared with amount (2048 - StackAlign), StackSize needs to
2317 // satisfy the following conditions to avoid using more instructions
2318 // to adjust the sp after adjusting the amount, such as
2319 // StackSize meets the condition (StackSize <= 2048 + RVCompressLen),
2320 // case1: Amount is 2048 - StackAlign: use addi + addi to adjust sp.
2321 // case2: Amount is RVCompressLen: use addi + addi to adjust sp.
2322 auto CanCompress = [&](uint64_t CompressLen) -> bool {
2323 if (StackSize <= 2047 + CompressLen ||
2324 (StackSize > 2048 * 2 - StackAlign &&
2325 StackSize <= 2047 * 2 + CompressLen) ||
2326 StackSize > 2048 * 3 - StackAlign)
2327 return true;
2328
2329 return false;
2330 };
2331 // In the epilogue, addi sp, sp, 496 is used to recover the sp and it
2332 // can be compressed(C.ADDI16SP, offset can be [-512, 496]), but
2333 // addi sp, sp, 512 can not be compressed. So try to use 496 first.
2334 const uint64_t ADDI16SPCompressLen = 496;
2335 if (STI.is64Bit() && CanCompress(ADDI16SPCompressLen))
2336 return ADDI16SPCompressLen;
2337 if (CanCompress(RVCompressLen))
2338 return RVCompressLen;
2339 }
2340 return 2048 - StackAlign;
2341 }
2342 return 0;
2343}
2344
2345bool RISCVFrameLowering::assignCalleeSavedSpillSlots(
2346 MachineFunction &MF, const TargetRegisterInfo *TRI,
2347 std::vector<CalleeSavedInfo> &CSI) const {
2348 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
2349 MachineFrameInfo &MFI = MF.getFrameInfo();
2350 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
2351
2352 // Preemptible Interrupts have two additional Callee-save Frame Indexes,
2353 // not tracked by `CSI`.
2354 if (RVFI->isSiFivePreemptibleInterrupt(MF)) {
2355 for (int I = 0; I < 2; ++I) {
2356 int FI = RVFI->getInterruptCSRFrameIndex(Idx: I);
2357 MFI.setIsCalleeSavedObjectIndex(ObjectIdx: FI, IsCalleeSaved: true);
2358 }
2359 }
2360
2361 // Early exit if no callee saved registers are modified!
2362 if (CSI.empty())
2363 return true;
2364
2365 if (RVFI->useQCIInterrupt(MF)) {
2366 RVFI->setQCIInterruptStackSize(QCIInterruptPushAmount);
2367 }
2368
2369 if (RVFI->isPushable(MF)) {
2370 // Determine how many GPRs we need to push and save it to RVFI.
2371 unsigned PushedRegNum = getNumPushPopRegs(CSI);
2372
2373 // `QC.C.MIENTER(.NEST)` will save `ra` and `s0`, so we should only push if
2374 // we want to push more than 2 registers. Otherwise, we should push if we
2375 // want to push more than 0 registers.
2376 unsigned OnlyPushIfMoreThan = RVFI->useQCIInterrupt(MF) ? 2 : 0;
2377 if (PushedRegNum > OnlyPushIfMoreThan) {
2378 RVFI->setRVPushRegs(PushedRegNum);
2379 RVFI->setRVPushStackSize(alignTo(Value: (STI.getXLen() / 8) * PushedRegNum, Align: 16));
2380 }
2381 }
2382
2383 for (auto &CS : CSI) {
2384 MCRegister Reg = CS.getReg();
2385 const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
2386 unsigned Size = RegInfo->getSpillSize(RC: *RC);
2387
2388 if (RVFI->useQCIInterrupt(MF)) {
2389 const auto *FFI = llvm::find_if(Range: FixedCSRFIQCIInterruptMap, P: [&](auto P) {
2390 return P.first == CS.getReg();
2391 });
2392 if (FFI != std::end(arr: FixedCSRFIQCIInterruptMap)) {
2393 int64_t Offset = FFI->second * (int64_t)Size;
2394
2395 int FrameIdx = MFI.CreateFixedSpillStackObject(Size, SPOffset: Offset);
2396 assert(FrameIdx < 0);
2397 CS.setFrameIdx(FrameIdx);
2398 continue;
2399 }
2400 }
2401
2402 if (RVFI->useSaveRestoreLibCalls(MF) || RVFI->isPushable(MF)) {
2403 const auto *FII = llvm::find_if(
2404 Range: FixedCSRFIMap, P: [&](MCPhysReg P) { return P == CS.getReg(); });
2405 unsigned RegNum = std::distance(first: std::begin(arr: FixedCSRFIMap), last: FII);
2406
2407 if (FII != std::end(arr: FixedCSRFIMap)) {
2408 int64_t Offset;
2409 if (RVFI->getPushPopKind(MF) ==
2410 RISCVMachineFunctionInfo::PushPopKind::StdExtZcmp)
2411 Offset = -int64_t(RVFI->getRVPushRegs() - RegNum) * Size;
2412 else
2413 Offset = -int64_t(RegNum + 1) * Size;
2414
2415 if (RVFI->useQCIInterrupt(MF))
2416 Offset -= QCIInterruptPushAmount;
2417
2418 int FrameIdx = MFI.CreateFixedSpillStackObject(Size, SPOffset: Offset);
2419 assert(FrameIdx < 0);
2420 CS.setFrameIdx(FrameIdx);
2421 continue;
2422 }
2423 }
2424
2425 // For GPRPair registers, use 8-byte slots with required alignment by zilsd.
2426 if (!STI.is64Bit() && STI.hasStdExtZilsd() &&
2427 RISCV::GPRPairRegClass.contains(Reg)) {
2428 Align PairAlign = STI.getZilsdAlign();
2429 int FrameIdx = MFI.CreateStackObject(Size: 8, Alignment: PairAlign, isSpillSlot: true);
2430 MFI.setIsCalleeSavedObjectIndex(ObjectIdx: FrameIdx, IsCalleeSaved: true);
2431 CS.setFrameIdx(FrameIdx);
2432 continue;
2433 }
2434
2435 // Not a fixed slot.
2436 Align Alignment = RegInfo->getSpillAlign(RC: *RC);
2437 // We may not be able to satisfy the desired alignment specification of
2438 // the TargetRegisterClass if the stack alignment is smaller. Use the
2439 // min.
2440 Alignment = std::min(a: Alignment, b: getStackAlign());
2441 int FrameIdx = MFI.CreateStackObject(Size, Alignment, isSpillSlot: true);
2442 MFI.setIsCalleeSavedObjectIndex(ObjectIdx: FrameIdx, IsCalleeSaved: true);
2443 CS.setFrameIdx(FrameIdx);
2444 if (RISCVRegisterInfo::isRVVRegClass(RC))
2445 MFI.setStackID(ObjectIdx: FrameIdx, ID: TargetStackID::ScalableVector);
2446 }
2447
2448 if (RVFI->useQCIInterrupt(MF)) {
2449 // Allocate a fixed object that covers the entire QCI stack allocation,
2450 // because there are gaps which are reserved for future use.
2451 MFI.CreateFixedSpillStackObject(
2452 Size: QCIInterruptPushAmount, SPOffset: -static_cast<int64_t>(QCIInterruptPushAmount));
2453 }
2454
2455 if (RVFI->isPushable(MF)) {
2456 int64_t QCIOffset = RVFI->useQCIInterrupt(MF) ? QCIInterruptPushAmount : 0;
2457 // Allocate a fixed object that covers the full push.
2458 if (int64_t PushSize = RVFI->getRVPushStackSize())
2459 MFI.CreateFixedSpillStackObject(Size: PushSize, SPOffset: -PushSize - QCIOffset);
2460 } else if (int LibCallRegs = getLibCallID(MF, CSI) + 1) {
2461 // If libcalls are used to spill and restore callee-saved registers, the
2462 // frame has two sections; the opaque section managed by the libcalls, and
2463 // the section managed by MachineFrameInfo which can also hold callee saved
2464 // registers in fixed stack slots, both of which have negative frame
2465 // indices. This gets even more complicated when incoming arguments are
2466 // passed via the stack, as these too have negative frame indices. An
2467 // example is detailed below:
2468 //
2469 // | incoming arg | <- FI[-3]
2470 // | libcallspill |
2471 // | calleespill | <- FI[-2]
2472 // | calleespill | <- FI[-1]
2473 // | this_frame | <- FI[0]
2474 //
2475 // For negative frame indices, the offset from the frame pointer will differ
2476 // depending on which of these groups the frame index applies to.
2477 // The following calculates the correct offset knowing the number of callee
2478 // saved registers spilt by the two methods.
2479 unsigned LibCallFrameSize =
2480 alignTo(Size: (STI.getXLen() / 8) * LibCallRegs, A: getStackAlign());
2481 RVFI->setLibCallStackSize(LibCallFrameSize);
2482 // Allocate a fixed object that covers the entire libcall area.
2483 MFI.CreateFixedSpillStackObject(Size: LibCallFrameSize,
2484 SPOffset: -static_cast<int64_t>(LibCallFrameSize));
2485 }
2486
2487 return true;
2488}
2489
2490bool RISCVFrameLowering::spillCalleeSavedRegisters(
2491 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2492 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
2493 if (CSI.empty())
2494 return true;
2495
2496 MachineFunction *MF = MBB.getParent();
2497 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
2498 DebugLoc DL;
2499 if (MI != MBB.end() && !MI->isDebugInstr())
2500 DL = MI->getDebugLoc();
2501
2502 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2503 if (RVFI->useQCIInterrupt(MF: *MF)) {
2504 // Emit QC.C.MIENTER(.NEST)
2505 BuildMI(
2506 BB&: MBB, I: MI, MIMD: DL,
2507 MCID: TII.get(Opcode: RVFI->getInterruptStackKind(MF: *MF) ==
2508 RISCVMachineFunctionInfo::InterruptStackKind::QCINest
2509 ? RISCV::QC_C_MIENTER_NEST
2510 : RISCV::QC_C_MIENTER))
2511 .setMIFlag(MachineInstr::FrameSetup);
2512
2513 for (auto [Reg, _Offset] : FixedCSRFIQCIInterruptMap)
2514 MBB.addLiveIn(PhysReg: Reg);
2515 }
2516
2517 if (RVFI->isPushable(MF: *MF)) {
2518 // Emit CM.PUSH with base StackAdj & evaluate Push stack
2519 unsigned PushedRegNum = RVFI->getRVPushRegs();
2520 if (PushedRegNum > 0) {
2521 // Use encoded number to represent registers to spill.
2522 unsigned Opcode = getPushOpcode(
2523 Kind: RVFI->getPushPopKind(MF: *MF), UpdateFP: hasFP(MF: *MF) && !RVFI->useQCIInterrupt(MF: *MF));
2524 unsigned RegEnc = RISCVZC::encodeRegListNumRegs(NumRegs: PushedRegNum);
2525 MachineInstrBuilder PushBuilder =
2526 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode))
2527 .setMIFlag(MachineInstr::FrameSetup);
2528 PushBuilder.addImm(Val: RegEnc);
2529 PushBuilder.addImm(Val: 0);
2530
2531 for (unsigned i = 0; i < PushedRegNum; i++)
2532 PushBuilder.addUse(RegNo: FixedCSRFIMap[i], Flags: RegState::Implicit);
2533 }
2534 } else if (const char *SpillLibCall = getSpillLibCallName(MF: *MF, CSI)) {
2535 // Add spill libcall via non-callee-saved register t0.
2536 MachineInstrBuilder NewMI =
2537 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::PseudoCALLReg), DestReg: RISCV::X5)
2538 .addExternalSymbol(FnName: SpillLibCall, TargetFlags: RISCVII::MO_CALL)
2539 .setMIFlag(MachineInstr::FrameSetup)
2540 .addUse(RegNo: RISCV::X2, Flags: RegState::Implicit)
2541 .addDef(RegNo: RISCV::X2, Flags: RegState::ImplicitDefine)
2542 // The libcall may use t1 as a scratch register.
2543 .addDef(RegNo: RISCV::X6, Flags: RegState::ImplicitDefine | RegState::Dead)
2544 // libgcc may emit an lui t2, 0 to set a landing pad label.
2545 .addDef(RegNo: RISCV::X7, Flags: RegState::ImplicitDefine | RegState::Dead);
2546
2547 // Add registers spilled as implicit used.
2548 for (auto &CS : CSI)
2549 NewMI.addUse(RegNo: CS.getReg(), Flags: RegState::Implicit);
2550 }
2551
2552 // Manually spill values not spilled by libcall & Push/Pop.
2553 const auto &UnmanagedCSI =
2554 getUnmanagedInterruptCSI(MF: *MF, CSI, ReverseOrder: STI.preferAscendingLoadStore());
2555 const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI);
2556
2557 auto storeRegsToStackSlots = [&](ArrayRef<CalleeSavedInfo> CSInfo) {
2558 for (auto &CS : CSInfo) {
2559 // Insert the spill to the stack frame.
2560 MCRegister Reg = CS.getReg();
2561 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
2562 TII.storeRegToStackSlot(MBB, MI, SrcReg: Reg, isKill: !MBB.isLiveIn(Reg),
2563 FrameIndex: CS.getFrameIdx(), RC, VReg: Register(),
2564 Flags: MachineInstr::FrameSetup);
2565 }
2566 };
2567 storeRegsToStackSlots(UnmanagedCSI);
2568 storeRegsToStackSlots(RVVCSI);
2569
2570 return true;
2571}
2572
2573static unsigned getCalleeSavedRVVNumRegs(const Register &BaseReg) {
2574 return RISCV::VRRegClass.contains(Reg: BaseReg) ? 1
2575 : RISCV::VRM2RegClass.contains(Reg: BaseReg) ? 2
2576 : RISCV::VRM4RegClass.contains(Reg: BaseReg) ? 4
2577 : 8;
2578}
2579
2580void RISCVFrameLowering::emitCalleeSavedRVVPrologCFI(
2581 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, bool HasFP) const {
2582 MachineFunction *MF = MBB.getParent();
2583 const MachineFrameInfo &MFI = MF->getFrameInfo();
2584 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2585 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
2586
2587 const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI: MFI.getCalleeSavedInfo());
2588 if (RVVCSI.empty())
2589 return;
2590
2591 uint64_t FixedSize = getStackSizeWithRVVPadding(MF: *MF);
2592 if (!HasFP) {
2593 uint64_t ScalarLocalVarSize =
2594 MFI.getStackSize() - RVFI->getCalleeSavedStackSize() -
2595 RVFI->getVarArgsSaveSize() + RVFI->getRVVPadding();
2596 FixedSize -= ScalarLocalVarSize;
2597 }
2598
2599 CFIInstBuilder CFIBuilder(MBB, MI, MachineInstr::FrameSetup);
2600 for (auto &CS : RVVCSI) {
2601 // Insert the spill to the stack frame.
2602 int FI = CS.getFrameIdx();
2603 MCRegister BaseReg = getRVVBaseRegister(TRI, Reg: CS.getReg());
2604 unsigned NumRegs = getCalleeSavedRVVNumRegs(BaseReg: CS.getReg());
2605 for (unsigned i = 0; i < NumRegs; ++i) {
2606 CFIBuilder.insertCFIInst(CFIInst: createDefCFAOffset(
2607 TRI, Reg: BaseReg + i,
2608 Offset: StackOffset::get(Fixed: -FixedSize, Scalable: MFI.getObjectOffset(ObjectIdx: FI) / 8 + i)));
2609 }
2610 }
2611}
2612
2613void RISCVFrameLowering::emitCalleeSavedRVVEpilogCFI(
2614 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const {
2615 MachineFunction *MF = MBB.getParent();
2616 const MachineFrameInfo &MFI = MF->getFrameInfo();
2617 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
2618
2619 CFIInstBuilder CFIHelper(MBB, MI, MachineInstr::FrameDestroy);
2620 const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI: MFI.getCalleeSavedInfo());
2621 for (auto &CS : RVVCSI) {
2622 MCRegister BaseReg = getRVVBaseRegister(TRI, Reg: CS.getReg());
2623 unsigned NumRegs = getCalleeSavedRVVNumRegs(BaseReg: CS.getReg());
2624 for (unsigned i = 0; i < NumRegs; ++i)
2625 CFIHelper.buildRestore(Reg: BaseReg + i);
2626 }
2627}
2628
2629bool RISCVFrameLowering::restoreCalleeSavedRegisters(
2630 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2631 MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
2632 if (CSI.empty())
2633 return true;
2634
2635 MachineFunction *MF = MBB.getParent();
2636 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
2637 DebugLoc DL;
2638 if (MI != MBB.end() && !MI->isDebugInstr())
2639 DL = MI->getDebugLoc();
2640
2641 // Manually restore values not restored by libcall & Push/Pop.
2642 // Reverse the restore order in epilog. In addition, the return
2643 // address will be restored first in the epilogue. It increases
2644 // the opportunity to avoid the load-to-use data hazard between
2645 // loading RA and return by RA. loadRegFromStackSlot can insert
2646 // multiple instructions.
2647 const auto &UnmanagedCSI =
2648 getUnmanagedInterruptCSI(MF: *MF, CSI, ReverseOrder: STI.preferAscendingLoadStore());
2649 const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI);
2650
2651 auto loadRegFromStackSlot = [&](ArrayRef<CalleeSavedInfo> CSInfo) {
2652 for (auto &CS : CSInfo) {
2653 MCRegister Reg = CS.getReg();
2654 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
2655 TII.loadRegFromStackSlot(MBB, MI, DestReg: Reg, FrameIndex: CS.getFrameIdx(), RC, VReg: Register(),
2656 SubReg: RISCV::NoSubRegister,
2657 Flags: MachineInstr::FrameDestroy);
2658 assert(MI != MBB.begin() &&
2659 "loadRegFromStackSlot didn't insert any code!");
2660 }
2661 };
2662 loadRegFromStackSlot(RVVCSI);
2663 loadRegFromStackSlot(UnmanagedCSI);
2664
2665 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2666 if (RVFI->useQCIInterrupt(MF: *MF)) {
2667 // Don't emit anything here because restoration is handled by
2668 // QC.C.MILEAVERET which we already inserted to return.
2669 assert(MI->getOpcode() == RISCV::QC_C_MILEAVERET &&
2670 "Unexpected QCI Interrupt Return Instruction");
2671 }
2672
2673 if (RVFI->isPushable(MF: *MF)) {
2674 unsigned PushedRegNum = RVFI->getRVPushRegs();
2675 if (PushedRegNum > 0) {
2676 unsigned Opcode = getPopOpcode(Kind: RVFI->getPushPopKind(MF: *MF));
2677 unsigned RegEnc = RISCVZC::encodeRegListNumRegs(NumRegs: PushedRegNum);
2678 MachineInstrBuilder PopBuilder =
2679 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode))
2680 .setMIFlag(MachineInstr::FrameDestroy);
2681 // Use encoded number to represent registers to restore.
2682 PopBuilder.addImm(Val: RegEnc);
2683 PopBuilder.addImm(Val: 0);
2684
2685 for (unsigned i = 0; i < RVFI->getRVPushRegs(); i++)
2686 PopBuilder.addDef(RegNo: FixedCSRFIMap[i], Flags: RegState::ImplicitDefine);
2687 }
2688 } else if (const char *RestoreLibCall = getRestoreLibCallName(MF: *MF, CSI)) {
2689 assert(RVFI->useSaveRestoreLibCalls(*MF) &&
2690 "Expected restore with libcall to use save restore libcalls");
2691
2692 // Restore is not compatible with shadow stacks, so do the restore manually.
2693 // This ensures we still get the code size saving of `__riscv_save_<N>`.
2694 if (RVFI->hasShadowStack(MF: *MF)) {
2695 loadRegFromStackSlot(getPushOrLibCallsSavedInfo(MF: *MF, CSI));
2696 return true;
2697 }
2698
2699 // Add restore libcall via tail call. Use the software guarded variant when
2700 // cf-protection-branch is enabled because the restore function does not
2701 // contain an lpad.
2702 unsigned TailOpc =
2703 RVFI->hasCFProtectionBranch() ? RISCV::PseudoTAILX7 : RISCV::PseudoTAIL;
2704 MachineInstrBuilder NewMI =
2705 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: TailOpc))
2706 .addExternalSymbol(FnName: RestoreLibCall, TargetFlags: RISCVII::MO_CALL)
2707 .setMIFlag(MachineInstr::FrameDestroy)
2708 .addDef(RegNo: RISCV::X2, Flags: RegState::ImplicitDefine);
2709
2710 // Add registers restored as implicit defined.
2711 for (auto &CS : CSI)
2712 NewMI.addDef(RegNo: CS.getReg(), Flags: RegState::ImplicitDefine);
2713
2714 // Remove trailing returns, since the terminator is now a tail call to the
2715 // restore function.
2716 if (MI != MBB.end() && MI->getOpcode() == RISCV::PseudoRET) {
2717 NewMI.getInstr()->copyImplicitOps(MF&: *MF, MI: *MI);
2718 MI->eraseFromParent();
2719 }
2720 }
2721 return true;
2722}
2723
2724bool RISCVFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
2725 // Keep the conventional code flow when not optimizing.
2726 if (MF.getFunction().hasOptNone())
2727 return false;
2728
2729 // QCI and SiFive CLIC interrupt entry sequences must precede all handler
2730 // code.
2731 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
2732 if (RVFI->useQCIInterrupt(MF) || RVFI->useSiFiveInterrupt(MF))
2733 return false;
2734
2735 return true;
2736}
2737
2738bool RISCVFrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const {
2739 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
2740 const MachineFunction *MF = MBB.getParent();
2741 const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2742
2743 // Make sure VTYPE and VL are not live-in since we will use vsetvli in the
2744 // prologue to get the VLEN, and that will clobber these registers.
2745 //
2746 // We may do also check the stack contains objects with scalable vector type,
2747 // but this will require iterating over all the stack objects, but this may
2748 // not worth since the situation is rare, we could do further check in future
2749 // if we find it is necessary.
2750 if (STI.preferVsetvliOverReadVLENB() &&
2751 (MBB.isLiveIn(Reg: RISCV::VTYPE) || MBB.isLiveIn(Reg: RISCV::VL)))
2752 return false;
2753
2754 if (!RVFI->useSaveRestoreLibCalls(MF: *MF))
2755 return true;
2756
2757 // Inserting a call to a __riscv_save libcall requires the use of the register
2758 // t0 (X5) to hold the return address. Therefore if this register is already
2759 // used we can't insert the call.
2760
2761 RegScavenger RS;
2762 RS.enterBasicBlock(MBB&: *TmpMBB);
2763 return !RS.isRegUsed(Reg: RISCV::X5);
2764}
2765
2766bool RISCVFrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
2767 const MachineFunction *MF = MBB.getParent();
2768 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
2769 const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2770
2771 // If we have a shadow stack, we don't use `__riscv_restore` libcalls.
2772 if (RVFI->hasShadowStack(MF: *MF))
2773 return true;
2774
2775 if (!RVFI->useSaveRestoreLibCalls(MF: *MF))
2776 return true;
2777
2778 // Using the __riscv_restore libcalls to restore CSRs requires a tail call.
2779 // This means if we still need to continue executing code within this function
2780 // the restore cannot take place in this basic block.
2781
2782 if (MBB.succ_size() > 1)
2783 return false;
2784
2785 MachineBasicBlock *SuccMBB =
2786 MBB.succ_empty() ? TmpMBB->getFallThrough() : *MBB.succ_begin();
2787
2788 // Doing a tail call should be safe if there are no successors, because either
2789 // we have a returning block or the end of the block is unreachable, so the
2790 // restore will be eliminated regardless.
2791 if (!SuccMBB)
2792 return true;
2793
2794 // The successor can only contain a return and debug instructions, since we
2795 // would effectively replace it with our own tail return at the end of this
2796 // block. The debug instructions would not execute on the tail-return path.
2797 return SuccMBB->isReturnBlock() &&
2798 llvm::count_if(Range: SuccMBB->instrs(), P: [](const MachineInstr &MI) {
2799 return !MI.isDebugInstr();
2800 }) == 1;
2801}
2802
2803bool RISCVFrameLowering::isSupportedStackID(TargetStackID::Value ID) const {
2804 return ID == TargetStackID::Default || ID == TargetStackID::ScalableVector;
2805}
2806
2807TargetStackID::Value RISCVFrameLowering::getStackIDForScalableVectors() const {
2808 return TargetStackID::ScalableVector;
2809}
2810
2811// Synthesize the probe loop.
2812static void emitStackProbeInline(MachineBasicBlock::iterator MBBI, DebugLoc DL,
2813 Register TargetReg, Register ScratchReg,
2814 bool IsRVV) {
2815 assert(TargetReg != RISCV::X2 && "New top of stack cannot already be in SP");
2816 assert(ScratchReg != RISCV::X2 && "Scratch register cannot be SP");
2817 assert(TargetReg != ScratchReg && "Target and scratch must be different");
2818
2819 MachineBasicBlock &MBB = *MBBI->getParent();
2820 MachineFunction &MF = *MBB.getParent();
2821
2822 auto &Subtarget = MF.getSubtarget<RISCVSubtarget>();
2823 const RISCVInstrInfo *TII = Subtarget.getInstrInfo();
2824 bool IsRV64 = Subtarget.is64Bit();
2825 Align StackAlign = Subtarget.getFrameLowering()->getStackAlign();
2826 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
2827 uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
2828
2829 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB.getIterator());
2830 MachineBasicBlock *LoopTestMBB =
2831 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
2832 MF.insert(MBBI: MBBInsertPoint, MBB: LoopTestMBB);
2833 MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
2834 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
2835 MachineInstr::MIFlag Flags = MachineInstr::FrameSetup;
2836
2837 // ScratchReg = ProbeSize
2838 TII->movImm(MBB, MBBI, DL, DstReg: ScratchReg, Val: ProbeSize, Flag: Flags);
2839
2840 // LoopTest:
2841 // SUB SP, SP, ProbeSize
2842 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::SUB), DestReg: SPReg)
2843 .addReg(RegNo: SPReg)
2844 .addReg(RegNo: ScratchReg)
2845 .setMIFlags(Flags);
2846
2847 // s[d|w] zero, 0(sp)
2848 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL,
2849 MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
2850 .addReg(RegNo: RISCV::X0)
2851 .addReg(RegNo: SPReg)
2852 .addImm(Val: 0)
2853 .setMIFlags(Flags);
2854
2855 if (IsRVV) {
2856 // SUB TargetReg, TargetReg, ProbeSize
2857 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::SUB),
2858 DestReg: TargetReg)
2859 .addReg(RegNo: TargetReg)
2860 .addReg(RegNo: ScratchReg)
2861 .setMIFlags(Flags);
2862
2863 // BGE TargetReg, ProbeSize, LoopTest
2864 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::BGE))
2865 .addReg(RegNo: TargetReg)
2866 .addReg(RegNo: ScratchReg)
2867 .addMBB(MBB: LoopTestMBB)
2868 .setMIFlags(Flags);
2869
2870 } else {
2871 // BNE SP, TargetReg, LoopTest
2872 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
2873 .addReg(RegNo: SPReg)
2874 .addReg(RegNo: TargetReg)
2875 .addMBB(MBB: LoopTestMBB)
2876 .setMIFlags(Flags);
2877 }
2878
2879 ExitMBB->splice(Where: ExitMBB->end(), Other: &MBB, From: std::next(x: MBBI), To: MBB.end());
2880 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
2881
2882 LoopTestMBB->addSuccessor(Succ: ExitMBB);
2883 LoopTestMBB->addSuccessor(Succ: LoopTestMBB);
2884 MBB.addSuccessor(Succ: LoopTestMBB);
2885 // Update liveins.
2886 fullyRecomputeLiveIns(MBBs: {ExitMBB, LoopTestMBB});
2887}
2888
2889void RISCVFrameLowering::inlineStackProbe(MachineFunction &MF,
2890 MachineBasicBlock &MBB) const {
2891 // Get the instructions that need to be replaced. We emit at most two of
2892 // these. Remember them in order to avoid complications coming from the need
2893 // to traverse the block while potentially creating more blocks.
2894 SmallVector<MachineInstr *, 4> ToReplace;
2895 for (MachineInstr &MI : MBB) {
2896 unsigned Opc = MI.getOpcode();
2897 if (Opc == RISCV::PROBED_STACKALLOC ||
2898 Opc == RISCV::PROBED_STACKALLOC_RVV) {
2899 ToReplace.push_back(Elt: &MI);
2900 }
2901 }
2902
2903 for (MachineInstr *MI : ToReplace) {
2904 if (MI->getOpcode() == RISCV::PROBED_STACKALLOC ||
2905 MI->getOpcode() == RISCV::PROBED_STACKALLOC_RVV) {
2906 MachineBasicBlock::iterator MBBI = MI->getIterator();
2907 DebugLoc DL = MBB.findDebugLoc(MBBI);
2908 Register TargetReg = MI->getOperand(i: 0).getReg();
2909
2910 Register ScratchReg =
2911 findScratchNonCalleeSaveRegister(MBB: &MBB, PreferredReg: RISCV::X7, DontUseReg: TargetReg);
2912
2913 assert(ScratchReg.isValid() &&
2914 "No available scratch register for stack probe loop");
2915
2916 emitStackProbeInline(MBBI, DL, TargetReg, ScratchReg,
2917 IsRVV: (MI->getOpcode() == RISCV::PROBED_STACKALLOC_RVV));
2918 MBBI->eraseFromParent();
2919 }
2920 }
2921}
2922
2923int RISCVFrameLowering::getInitialCFAOffset(const MachineFunction &MF) const {
2924 return 0;
2925}
2926
2927Register
2928RISCVFrameLowering::getInitialCFARegister(const MachineFunction &MF) const {
2929 return RISCV::X2;
2930}
2931
2932// On 64-bit systems the fixed stack can hold INT64_MAX bytes, since
2933// stack-offset calculation is done in 2s-complement.
2934// NOTE: In theory a register can hold any 64-bit number, so this constraint
2935// might be relaxed to UINT64_MAX in the future, if anyone actually needs
2936// that.
2937uint64_t RISCVFrameLowering::getStackThreshold() const {
2938 return STI.is64Bit() ? INT64_MAX : UINT32_MAX;
2939}
2940