1//===-- RISCVFrameLowering.cpp - RISC-V Frame Information -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the RISC-V implementation of TargetFrameLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVFrameLowering.h"
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "MCTargetDesc/RISCVMCTargetDesc.h"
16#include "RISCVMachineFunctionInfo.h"
17#include "RISCVSubtarget.h"
18#include "llvm/BinaryFormat/Dwarf.h"
19#include "llvm/CodeGen/CFIInstBuilder.h"
20#include "llvm/CodeGen/LivePhysRegs.h"
21#include "llvm/CodeGen/MachineFrameInfo.h"
22#include "llvm/CodeGen/MachineFunction.h"
23#include "llvm/CodeGen/MachineInstrBuilder.h"
24#include "llvm/CodeGen/MachineRegisterInfo.h"
25#include "llvm/CodeGen/RegisterScavenging.h"
26#include "llvm/CodeGen/TargetFrameLowering.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/MC/MCDwarf.h"
29#include "llvm/Support/LEB128.h"
30
31#include <algorithm>
32#include <cstdint>
33
34#define DEBUG_TYPE "riscv-frame"
35
36using namespace llvm;
37
38static Align getABIStackAlignment(RISCVABI::ABI ABI) {
39 if (ABI == RISCVABI::ABI_ILP32E)
40 return Align(4);
41 if (ABI == RISCVABI::ABI_LP64E)
42 return Align(8);
43 return Align(16);
44}
45
46RISCVFrameLowering::RISCVFrameLowering(const RISCVSubtarget &STI)
47 : TargetFrameLowering(
48 StackGrowsDown, getABIStackAlignment(ABI: STI.getTargetABI()),
49 /*LocalAreaOffset=*/0,
50 /*TransientStackAlignment=*/getABIStackAlignment(ABI: STI.getTargetABI())),
51 STI(STI) {}
52
53// The register used to hold the frame pointer.
54static constexpr MCPhysReg FPReg = RISCV::X8;
55
56// The register used to hold the stack pointer.
57static constexpr MCPhysReg SPReg = RISCV::X2;
58
59// The register used to hold the return address.
60static constexpr MCPhysReg RAReg = RISCV::X1;
61
62// LIst of CSRs that are given a fixed location by save/restore libcalls or
63// Zcmp/Xqccmp Push/Pop. The order in this table indicates the order the
64// registers are saved on the stack. Zcmp uses the reverse order of save/restore
65// and Xqccmp on the stack, but this is handled when offsets are calculated.
66static const MCPhysReg FixedCSRFIMap[] = {
67 /*ra*/ RAReg, /*s0*/ FPReg, /*s1*/ RISCV::X9,
68 /*s2*/ RISCV::X18, /*s3*/ RISCV::X19, /*s4*/ RISCV::X20,
69 /*s5*/ RISCV::X21, /*s6*/ RISCV::X22, /*s7*/ RISCV::X23,
70 /*s8*/ RISCV::X24, /*s9*/ RISCV::X25, /*s10*/ RISCV::X26,
71 /*s11*/ RISCV::X27};
72
73// The number of stack bytes allocated by `QC.C.MIENTER(.NEST)` and popped by
74// `QC.C.MILEAVERET`.
75static constexpr uint64_t QCIInterruptPushAmount = 96;
76
77static const std::pair<MCPhysReg, int8_t> FixedCSRFIQCIInterruptMap[] = {
78 /* -1 is a gap for mepc/mnepc */
79 {/*fp*/ FPReg, -2},
80 /* -3 is a gap for qc.mcause */
81 {/*ra*/ RAReg, -4},
82 /* -5 is reserved */
83 {/*t0*/ RISCV::X5, -6},
84 {/*t1*/ RISCV::X6, -7},
85 {/*t2*/ RISCV::X7, -8},
86 {/*a0*/ RISCV::X10, -9},
87 {/*a1*/ RISCV::X11, -10},
88 {/*a2*/ RISCV::X12, -11},
89 {/*a3*/ RISCV::X13, -12},
90 {/*a4*/ RISCV::X14, -13},
91 {/*a5*/ RISCV::X15, -14},
92 {/*a6*/ RISCV::X16, -15},
93 {/*a7*/ RISCV::X17, -16},
94 {/*t3*/ RISCV::X28, -17},
95 {/*t4*/ RISCV::X29, -18},
96 {/*t5*/ RISCV::X30, -19},
97 {/*t6*/ RISCV::X31, -20},
98 /* -21, -22, -23, -24 are reserved */
99};
100
101/// Returns true if DWARF CFI instructions ("frame moves") should be emitted.
102static bool needsDwarfCFI(const MachineFunction &MF) {
103 return MF.needsFrameMoves();
104}
105
106// For now we use x3, a.k.a gp, as pointer to shadow call stack.
107// User should not use x3 in their asm.
108static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
109 MachineBasicBlock::iterator MI,
110 const DebugLoc &DL) {
111 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
112 // We check Zimop instead of (Zimop || Zcmop) to determine whether HW shadow
113 // stack is available despite the fact that sspush/sspopchk both have a
114 // compressed form, because if only Zcmop is available, we would need to
115 // reserve X5 due to c.sspopchk only takes X5 and we currently do not support
116 // using X5 as the return address register.
117 // However, we can still aggressively use c.sspush x1 if zcmop is available.
118 bool HasHWShadowStack = MF.getFunction().hasFnAttribute(Kind: "hw-shadow-stack") &&
119 STI.hasStdExtZimop();
120 bool HasSWShadowStack =
121 MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack);
122 if (!HasHWShadowStack && !HasSWShadowStack)
123 return;
124
125 const llvm::RISCVRegisterInfo *TRI = STI.getRegisterInfo();
126
127 // Do not save RA to the SCS if it's not saved to the regular stack,
128 // i.e. RA is not at risk of being overwritten.
129 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
130 if (llvm::none_of(
131 Range&: CSI, P: [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
132 return;
133
134 const RISCVInstrInfo *TII = STI.getInstrInfo();
135 if (HasHWShadowStack) {
136 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SSPUSH))
137 .addReg(RegNo: RAReg)
138 .setMIFlag(MachineInstr::FrameSetup);
139 return;
140 }
141
142 Register SCSPReg = RISCVABI::getSCSPReg();
143
144 bool IsRV64 = STI.is64Bit();
145 int64_t SlotSize = STI.getXLen() / 8;
146 // Store return address to shadow call stack
147 // addi gp, gp, [4|8]
148 // s[w|d] ra, -[4|8](gp)
149 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI))
150 .addReg(RegNo: SCSPReg, Flags: RegState::Define)
151 .addReg(RegNo: SCSPReg)
152 .addImm(Val: SlotSize)
153 .setMIFlag(MachineInstr::FrameSetup);
154 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
155 .addReg(RegNo: RAReg)
156 .addReg(RegNo: SCSPReg)
157 .addImm(Val: -SlotSize)
158 .setMIFlag(MachineInstr::FrameSetup);
159
160 if (!needsDwarfCFI(MF))
161 return;
162
163 // Emit a CFI instruction that causes SlotSize to be subtracted from the value
164 // of the shadow stack pointer when unwinding past this frame.
165 char DwarfSCSReg = TRI->getDwarfRegNum(Reg: SCSPReg, /*IsEH*/ isEH: true);
166 assert(DwarfSCSReg < 32 && "SCS Register should be < 32 (X3).");
167
168 char Offset = static_cast<char>(-SlotSize) & 0x7f;
169 const char CFIInst[] = {
170 dwarf::DW_CFA_val_expression,
171 DwarfSCSReg, // register
172 2, // length
173 static_cast<char>(unsigned(dwarf::DW_OP_breg0 + DwarfSCSReg)),
174 Offset, // addend (sleb128)
175 };
176
177 CFIInstBuilder(MBB, MI, MachineInstr::FrameSetup)
178 .buildEscape(Bytes: StringRef(CFIInst, sizeof(CFIInst)));
179}
180
181static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB,
182 MachineBasicBlock::iterator MI,
183 const DebugLoc &DL) {
184 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
185 bool HasHWShadowStack = MF.getFunction().hasFnAttribute(Kind: "hw-shadow-stack") &&
186 STI.hasStdExtZimop();
187 bool HasSWShadowStack =
188 MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack);
189 if (!HasHWShadowStack && !HasSWShadowStack)
190 return;
191
192 // See emitSCSPrologue() above.
193 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
194 if (llvm::none_of(
195 Range&: CSI, P: [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
196 return;
197
198 // The shadow call stack popchk needs to happen after cm.pop that loads ra.
199 if (MI != MBB.end() &&
200 (MI->getOpcode() == RISCV::CM_POP || MI->getOpcode() == RISCV::QC_CM_POP))
201 ++MI;
202 const RISCVInstrInfo *TII = STI.getInstrInfo();
203 if (HasHWShadowStack) {
204 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SSPOPCHK))
205 .addReg(RegNo: RAReg)
206 .setMIFlag(MachineInstr::FrameDestroy);
207 return;
208 }
209
210 Register SCSPReg = RISCVABI::getSCSPReg();
211
212 bool IsRV64 = STI.is64Bit();
213 int64_t SlotSize = STI.getXLen() / 8;
214 // Load return address from shadow call stack
215 // l[w|d] ra, -[4|8](gp)
216 // addi gp, gp, -[4|8]
217 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::LD : RISCV::LW))
218 .addReg(RegNo: RAReg, Flags: RegState::Define)
219 .addReg(RegNo: SCSPReg)
220 .addImm(Val: -SlotSize)
221 .setMIFlag(MachineInstr::FrameDestroy);
222 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI))
223 .addReg(RegNo: SCSPReg, Flags: RegState::Define)
224 .addReg(RegNo: SCSPReg)
225 .addImm(Val: -SlotSize)
226 .setMIFlag(MachineInstr::FrameDestroy);
227 if (needsDwarfCFI(MF)) {
228 // Restore the SCS pointer
229 CFIInstBuilder(MBB, MI, MachineInstr::FrameDestroy).buildRestore(Reg: SCSPReg);
230 }
231}
232
233// Insert instruction to swap mscratchsw with sp
234static void emitSiFiveCLICStackSwap(MachineFunction &MF, MachineBasicBlock &MBB,
235 MachineBasicBlock::iterator MBBI,
236 const DebugLoc &DL,
237 MachineInstr::MIFlag FrameFlag) {
238 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
239
240 if (!RVFI->isSiFiveStackSwapInterrupt(MF))
241 return;
242
243 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
244 const RISCVInstrInfo *TII = STI.getInstrInfo();
245
246 assert(STI.hasVendorXSfmclic() && "Stack Swapping Requires XSfmclic");
247
248 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRW))
249 .addReg(RegNo: SPReg, Flags: RegState::Define)
250 .addImm(Val: RISCVSysReg::sf_mscratchcsw)
251 .addReg(RegNo: SPReg, Flags: RegState::Kill)
252 .setMIFlag(FrameFlag);
253
254 // FIXME: CFI Information for this swap.
255}
256
257static void
258createSiFivePreemptibleInterruptFrameEntries(MachineFunction &MF,
259 RISCVMachineFunctionInfo &RVFI) {
260 if (!RVFI.isSiFivePreemptibleInterrupt(MF))
261 return;
262
263 const TargetRegisterClass &RC = RISCV::GPRRegClass;
264 const TargetRegisterInfo &TRI =
265 *MF.getSubtarget<RISCVSubtarget>().getRegisterInfo();
266 MachineFrameInfo &MFI = MF.getFrameInfo();
267
268 // Create two frame objects for spilling X8 and X9, which will be done in
269 // `emitSiFiveCLICPreemptibleSaves`. This is in addition to any other stack
270 // objects we might have for X8 and X9, as they might be saved twice.
271 for (int I = 0; I < 2; ++I) {
272 int FI = MFI.CreateStackObject(Size: TRI.getSpillSize(RC), Alignment: TRI.getSpillAlign(RC),
273 isSpillSlot: true);
274 RVFI.pushInterruptCSRFrameIndex(FI);
275 }
276}
277
278static void emitSiFiveCLICPreemptibleSaves(MachineFunction &MF,
279 MachineBasicBlock &MBB,
280 MachineBasicBlock::iterator MBBI,
281 const DebugLoc &DL) {
282 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
283
284 if (!RVFI->isSiFivePreemptibleInterrupt(MF))
285 return;
286
287 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
288 const RISCVInstrInfo *TII = STI.getInstrInfo();
289
290 // FIXME: CFI Information here is nonexistent/wrong.
291
292 // X8 and X9 might be stored into the stack twice, initially into the
293 // `interruptCSRFrameIndex` here, and then maybe again into their CSI frame
294 // index.
295 //
296 // This is done instead of telling the register allocator that we need two
297 // VRegs to store the value of `mcause` and `mepc` through the instruction,
298 // which affects other passes.
299 TII->storeRegToStackSlot(MBB, MBBI, SrcReg: RISCV::X8, /* IsKill=*/true,
300 FrameIndex: RVFI->getInterruptCSRFrameIndex(Idx: 0),
301 RC: &RISCV::GPRRegClass, VReg: Register(),
302 Flags: MachineInstr::FrameSetup);
303 TII->storeRegToStackSlot(MBB, MBBI, SrcReg: RISCV::X9, /* IsKill=*/true,
304 FrameIndex: RVFI->getInterruptCSRFrameIndex(Idx: 1),
305 RC: &RISCV::GPRRegClass, VReg: Register(),
306 Flags: MachineInstr::FrameSetup);
307
308 // Put `mcause` into X8 (s0), and `mepc` into X9 (s1). If either of these are
309 // used in the function, then they will appear in `getUnmanagedCSI` and will
310 // be saved again.
311 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS))
312 .addReg(RegNo: RISCV::X8, Flags: RegState::Define)
313 .addImm(Val: RISCVSysReg::mcause)
314 .addReg(RegNo: RISCV::X0)
315 .setMIFlag(MachineInstr::FrameSetup);
316 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS))
317 .addReg(RegNo: RISCV::X9, Flags: RegState::Define)
318 .addImm(Val: RISCVSysReg::mepc)
319 .addReg(RegNo: RISCV::X0)
320 .setMIFlag(MachineInstr::FrameSetup);
321
322 // Enable interrupts.
323 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRSI))
324 .addReg(RegNo: RISCV::X0, Flags: RegState::Define)
325 .addImm(Val: RISCVSysReg::mstatus)
326 .addImm(Val: 8)
327 .setMIFlag(MachineInstr::FrameSetup);
328}
329
330static void emitSiFiveCLICPreemptibleRestores(MachineFunction &MF,
331 MachineBasicBlock &MBB,
332 MachineBasicBlock::iterator MBBI,
333 const DebugLoc &DL) {
334 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
335
336 if (!RVFI->isSiFivePreemptibleInterrupt(MF))
337 return;
338
339 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
340 const RISCVInstrInfo *TII = STI.getInstrInfo();
341
342 // FIXME: CFI Information here is nonexistent/wrong.
343
344 // Disable interrupts.
345 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRCI))
346 .addReg(RegNo: RISCV::X0, Flags: RegState::Define)
347 .addImm(Val: RISCVSysReg::mstatus)
348 .addImm(Val: 8)
349 .setMIFlag(MachineInstr::FrameDestroy);
350
351 // Restore `mepc` from x9 (s1), and `mcause` from x8 (s0). If either were used
352 // in the function, they have already been restored once, so now have the
353 // value stored in `emitSiFiveCLICPreemptibleSaves`.
354 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRW))
355 .addReg(RegNo: RISCV::X0, Flags: RegState::Define)
356 .addImm(Val: RISCVSysReg::mepc)
357 .addReg(RegNo: RISCV::X9, Flags: RegState::Kill)
358 .setMIFlag(MachineInstr::FrameDestroy);
359 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRW))
360 .addReg(RegNo: RISCV::X0, Flags: RegState::Define)
361 .addImm(Val: RISCVSysReg::mcause)
362 .addReg(RegNo: RISCV::X8, Flags: RegState::Kill)
363 .setMIFlag(MachineInstr::FrameDestroy);
364
365 // X8 and X9 need to be restored to their values on function entry, which we
366 // saved onto the stack in `emitSiFiveCLICPreemptibleSaves`.
367 TII->loadRegFromStackSlot(MBB, MBBI, DstReg: RISCV::X9,
368 FrameIndex: RVFI->getInterruptCSRFrameIndex(Idx: 1),
369 RC: &RISCV::GPRRegClass, VReg: Register(),
370 SubReg: RISCV::NoSubRegister, Flags: MachineInstr::FrameDestroy);
371 TII->loadRegFromStackSlot(MBB, MBBI, DstReg: RISCV::X8,
372 FrameIndex: RVFI->getInterruptCSRFrameIndex(Idx: 0),
373 RC: &RISCV::GPRRegClass, VReg: Register(),
374 SubReg: RISCV::NoSubRegister, Flags: MachineInstr::FrameDestroy);
375}
376
377// Get the ID of the libcall used for spilling and restoring callee saved
378// registers. The ID is representative of the number of registers saved or
379// restored by the libcall, except it is zero-indexed - ID 0 corresponds to a
380// single register.
381static int getLibCallID(const MachineFunction &MF,
382 const std::vector<CalleeSavedInfo> &CSI) {
383 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
384
385 if (CSI.empty() || !RVFI->useSaveRestoreLibCalls(MF))
386 return -1;
387
388 MCRegister MaxReg;
389 for (auto &CS : CSI)
390 // assignCalleeSavedSpillSlots assigns negative frame indexes to
391 // registers which can be saved by libcall.
392 if (CS.getFrameIdx() < 0)
393 MaxReg = std::max(a: MaxReg.id(), b: CS.getReg().id());
394
395 if (!MaxReg)
396 return -1;
397
398 switch (MaxReg.id()) {
399 default:
400 llvm_unreachable("Something has gone wrong!");
401 // clang-format off
402 case /*s11*/ RISCV::X27: return 12;
403 case /*s10*/ RISCV::X26: return 11;
404 case /*s9*/ RISCV::X25: return 10;
405 case /*s8*/ RISCV::X24: return 9;
406 case /*s7*/ RISCV::X23: return 8;
407 case /*s6*/ RISCV::X22: return 7;
408 case /*s5*/ RISCV::X21: return 6;
409 case /*s4*/ RISCV::X20: return 5;
410 case /*s3*/ RISCV::X19: return 4;
411 case /*s2*/ RISCV::X18: return 3;
412 case /*s1*/ RISCV::X9: return 2;
413 case /*s0*/ FPReg: return 1;
414 case /*ra*/ RAReg: return 0;
415 // clang-format on
416 }
417}
418
419// Get the name of the libcall used for spilling callee saved registers.
420// If this function will not use save/restore libcalls, then return a nullptr.
421static const char *
422getSpillLibCallName(const MachineFunction &MF,
423 const std::vector<CalleeSavedInfo> &CSI) {
424 static const char *const SpillLibCalls[] = {
425 "__riscv_save_0",
426 "__riscv_save_1",
427 "__riscv_save_2",
428 "__riscv_save_3",
429 "__riscv_save_4",
430 "__riscv_save_5",
431 "__riscv_save_6",
432 "__riscv_save_7",
433 "__riscv_save_8",
434 "__riscv_save_9",
435 "__riscv_save_10",
436 "__riscv_save_11",
437 "__riscv_save_12"
438 };
439
440 int LibCallID = getLibCallID(MF, CSI);
441 if (LibCallID == -1)
442 return nullptr;
443 return SpillLibCalls[LibCallID];
444}
445
446// Get the name of the libcall used for restoring callee saved registers.
447// If this function will not use save/restore libcalls, then return a nullptr.
448static const char *
449getRestoreLibCallName(const MachineFunction &MF,
450 const std::vector<CalleeSavedInfo> &CSI) {
451 static const char *const RestoreLibCalls[] = {
452 "__riscv_restore_0",
453 "__riscv_restore_1",
454 "__riscv_restore_2",
455 "__riscv_restore_3",
456 "__riscv_restore_4",
457 "__riscv_restore_5",
458 "__riscv_restore_6",
459 "__riscv_restore_7",
460 "__riscv_restore_8",
461 "__riscv_restore_9",
462 "__riscv_restore_10",
463 "__riscv_restore_11",
464 "__riscv_restore_12"
465 };
466
467 int LibCallID = getLibCallID(MF, CSI);
468 if (LibCallID == -1)
469 return nullptr;
470 return RestoreLibCalls[LibCallID];
471}
472
473// Get the max reg of Push/Pop for restoring callee saved registers.
474static unsigned getNumPushPopRegs(const std::vector<CalleeSavedInfo> &CSI) {
475 unsigned NumPushPopRegs = 0;
476 for (auto &CS : CSI) {
477 auto *FII = llvm::find_if(Range: FixedCSRFIMap,
478 P: [&](MCPhysReg P) { return P == CS.getReg(); });
479 if (FII != std::end(arr: FixedCSRFIMap)) {
480 unsigned RegNum = std::distance(first: std::begin(arr: FixedCSRFIMap), last: FII);
481 NumPushPopRegs = std::max(a: NumPushPopRegs, b: RegNum + 1);
482 }
483 }
484 assert(NumPushPopRegs != 12 && "x26 requires x27 to also be pushed");
485 return NumPushPopRegs;
486}
487
488// Return true if the specified function should have a dedicated frame
489// pointer register. This is true if frame pointer elimination is
490// disabled, if it needs dynamic stack realignment, if the function has
491// variable sized allocas, or if the frame address is taken.
492bool RISCVFrameLowering::hasFPImpl(const MachineFunction &MF) const {
493 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
494
495 const MachineFrameInfo &MFI = MF.getFrameInfo();
496 if (MF.disableFramePointerElim() || RegInfo->hasStackRealignment(MF) ||
497 MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken())
498 return true;
499
500 // With large callframes around we may need to use FP to access the scavenging
501 // emergency spillslot.
502 //
503 // We calculate the MaxCallFrameSize at the end of isel so this value should
504 // be stable for the whole post-isel MIR pipeline.
505 //
506 // NOTE: The idea of forcing a frame pointer is copied from AArch64, but they
507 // conservatively return true when the call frame size hasd not been
508 // computed yet. On RISC-V that caused MachineOutliner tests to fail the
509 // MachineVerifier due to outlined functions not computing max call frame
510 // size thus the frame pointer would always be reserved.
511 if (MFI.isMaxCallFrameSizeComputed() && MFI.getMaxCallFrameSize() > 2047)
512 return true;
513
514 return false;
515}
516
517bool RISCVFrameLowering::hasBP(const MachineFunction &MF) const {
518 const MachineFrameInfo &MFI = MF.getFrameInfo();
519 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
520
521 // If we do not reserve stack space for outgoing arguments in prologue,
522 // we will adjust the stack pointer before call instruction. After the
523 // adjustment, we can not use SP to access the stack objects for the
524 // arguments. Instead, use BP to access these stack objects.
525 return (MFI.hasVarSizedObjects() ||
526 (!hasReservedCallFrame(MF) && (!MFI.isMaxCallFrameSizeComputed() ||
527 MFI.getMaxCallFrameSize() != 0))) &&
528 TRI->hasStackRealignment(MF);
529}
530
531// Determines the size of the frame and maximum call frame size.
532void RISCVFrameLowering::determineFrameLayout(MachineFunction &MF) const {
533 MachineFrameInfo &MFI = MF.getFrameInfo();
534 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
535
536 // Get the number of bytes to allocate from the FrameInfo.
537 uint64_t FrameSize = MFI.getStackSize();
538
539 // QCI Interrupts use at least 96 bytes of stack space
540 if (RVFI->useQCIInterrupt(MF))
541 FrameSize = std::max(a: FrameSize, b: QCIInterruptPushAmount);
542
543 // Get the alignment.
544 Align StackAlign = getStackAlign();
545
546 // Make sure the frame is aligned.
547 FrameSize = alignTo(Size: FrameSize, A: StackAlign);
548
549 // Update frame info.
550 MFI.setStackSize(FrameSize);
551
552 // When using SP or BP to access stack objects, we may require extra padding
553 // to ensure the bottom of the RVV stack is correctly aligned within the main
554 // stack. We calculate this as the amount required to align the scalar local
555 // variable section up to the RVV alignment.
556 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
557 if (RVFI->getRVVStackSize() && (!hasFP(MF) || TRI->hasStackRealignment(MF))) {
558 int ScalarLocalVarSize = FrameSize - RVFI->getCalleeSavedStackSize() -
559 RVFI->getVarArgsSaveSize();
560 if (auto RVVPadding =
561 offsetToAlignment(Value: ScalarLocalVarSize, Alignment: RVFI->getRVVStackAlign()))
562 RVFI->setRVVPadding(RVVPadding);
563 }
564}
565
566// Returns the stack size including RVV padding (when required), rounded back
567// up to the required stack alignment.
568uint64_t RISCVFrameLowering::getStackSizeWithRVVPadding(
569 const MachineFunction &MF) const {
570 const MachineFrameInfo &MFI = MF.getFrameInfo();
571 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
572 return alignTo(Size: MFI.getStackSize() + RVFI->getRVVPadding(), A: getStackAlign());
573}
574
575static SmallVector<CalleeSavedInfo, 8>
576getUnmanagedCSI(const MachineFunction &MF,
577 const std::vector<CalleeSavedInfo> &CSI,
578 bool ReverseOrder = false) {
579 const MachineFrameInfo &MFI = MF.getFrameInfo();
580 SmallVector<CalleeSavedInfo, 8> NonLibcallCSI;
581
582 for (auto &CS : CSI) {
583 int FI = CS.getFrameIdx();
584 if (FI >= 0 && MFI.getStackID(ObjectIdx: FI) == TargetStackID::Default)
585 NonLibcallCSI.push_back(Elt: CS);
586 }
587
588 // Reverse the order so that load/store operations use ascending addresses,
589 // enabling better load/store clustering and fusion.
590 if (ReverseOrder)
591 std::reverse(first: NonLibcallCSI.begin(), last: NonLibcallCSI.end());
592
593 return NonLibcallCSI;
594}
595
596static SmallVector<CalleeSavedInfo, 8>
597getRVVCalleeSavedInfo(const MachineFunction &MF,
598 const std::vector<CalleeSavedInfo> &CSI) {
599 const MachineFrameInfo &MFI = MF.getFrameInfo();
600 SmallVector<CalleeSavedInfo, 8> RVVCSI;
601
602 for (auto &CS : CSI) {
603 int FI = CS.getFrameIdx();
604 if (FI >= 0 && MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector)
605 RVVCSI.push_back(Elt: CS);
606 }
607
608 return RVVCSI;
609}
610
611static SmallVector<CalleeSavedInfo, 8>
612getPushOrLibCallsSavedInfo(const MachineFunction &MF,
613 const std::vector<CalleeSavedInfo> &CSI) {
614 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
615
616 SmallVector<CalleeSavedInfo, 8> PushOrLibCallsCSI;
617 if (!RVFI->useSaveRestoreLibCalls(MF) && !RVFI->isPushable(MF))
618 return PushOrLibCallsCSI;
619
620 for (const auto &CS : CSI) {
621 if (RVFI->useQCIInterrupt(MF)) {
622 // Some registers are saved by both `QC.C.MIENTER(.NEST)` and
623 // `QC.CM.PUSH(FP)`. In these cases, prioritise the CFI info that points
624 // to the versions saved by `QC.C.MIENTER(.NEST)` which is what FP
625 // unwinding would use.
626 if (llvm::is_contained(Range: llvm::make_first_range(c: FixedCSRFIQCIInterruptMap),
627 Element: CS.getReg()))
628 continue;
629 }
630
631 if (llvm::is_contained(Range: FixedCSRFIMap, Element: CS.getReg()))
632 PushOrLibCallsCSI.push_back(Elt: CS);
633 }
634
635 return PushOrLibCallsCSI;
636}
637
638static SmallVector<CalleeSavedInfo, 8>
639getQCISavedInfo(const MachineFunction &MF,
640 const std::vector<CalleeSavedInfo> &CSI) {
641 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
642
643 SmallVector<CalleeSavedInfo, 8> QCIInterruptCSI;
644 if (!RVFI->useQCIInterrupt(MF))
645 return QCIInterruptCSI;
646
647 for (const auto &CS : CSI) {
648 if (llvm::is_contained(Range: llvm::make_first_range(c: FixedCSRFIQCIInterruptMap),
649 Element: CS.getReg()))
650 QCIInterruptCSI.push_back(Elt: CS);
651 }
652
653 return QCIInterruptCSI;
654}
655
656static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs,
657 const MachineBasicBlock &MBB) {
658 const MachineFunction *MF = MBB.getParent();
659 LiveRegs.addLiveIns(MBB);
660 const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
661 for (unsigned i = 0; CSRegs[i]; ++i)
662 LiveRegs.addReg(Reg: CSRegs[i]);
663}
664
665Register RISCVFrameLowering::findScratchNonCalleeSaveRegister(
666 MachineBasicBlock *MBB, Register PreferredReg, Register DontUseReg) const {
667 MachineFunction *MF = MBB->getParent();
668
669 // Stack protection code is being inserted at beginning of function, use
670 // register which has been historically used
671 if (&MF->front() == MBB)
672 return PreferredReg;
673
674 const RISCVSubtarget &Subtarget = MF->getSubtarget<RISCVSubtarget>();
675 const TargetRegisterInfo &TRI = *Subtarget.getRegisterInfo();
676 LivePhysRegs LiveRegs(TRI);
677 getLiveRegsForEntryMBB(LiveRegs, MBB: *MBB);
678
679 const MachineRegisterInfo &MRI = MF->getRegInfo();
680 // Prefer the register which has been historically used for stack protector
681 if (LiveRegs.available(MRI, Reg: PreferredReg))
682 return PreferredReg;
683
684 static const MCPhysReg CandidateRegs[] = {
685 RISCV::X5, RISCV::X6, RISCV::X7, RISCV::X28,
686 RISCV::X29, RISCV::X30, RISCV::X31,
687 };
688
689 for (unsigned Reg : CandidateRegs) {
690 if (Reg != DontUseReg && LiveRegs.available(MRI, Reg))
691 return Reg;
692 }
693
694 return Register();
695}
696
697void RISCVFrameLowering::allocateAndProbeStackForRVV(
698 MachineFunction &MF, MachineBasicBlock &MBB,
699 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, int64_t Amount,
700 MachineInstr::MIFlag Flag, bool EmitCFI, bool DynAllocation) const {
701 assert(Amount != 0 && "Did not need to adjust stack pointer for RVV.");
702
703 // Emit a variable-length allocation probing loop.
704
705 // Get VLEN in TargetReg
706 Register TargetReg = findScratchNonCalleeSaveRegister(MBB: &MBB, PreferredReg: RISCV::X6);
707 assert(TargetReg.isValid() &&
708 "No available scratch register for stack probing");
709 const RISCVInstrInfo *TII = STI.getInstrInfo();
710 uint32_t NumOfVReg = Amount / RISCV::RVVBytesPerBlock;
711 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::PseudoReadVLENB), DestReg: TargetReg)
712 .setMIFlag(Flag);
713 TII->mulImm(MF, MBB, II: MBBI, DL, DestReg: TargetReg, Amt: NumOfVReg, Flag);
714
715 CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
716 if (EmitCFI) {
717 // Set the CFA register to TargetReg.
718 CFIBuilder.buildDefCFA(Reg: TargetReg, Offset: -Amount);
719 }
720
721 // It will be expanded to a probe loop in `inlineStackProbe`.
722 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::PROBED_STACKALLOC_RVV))
723 .addReg(RegNo: TargetReg);
724
725 if (EmitCFI) {
726 // Set the CFA register back to SP.
727 CFIBuilder.buildDefCFARegister(Reg: SPReg);
728 }
729
730 // SUB SP, SP, T1
731 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SUB), DestReg: SPReg)
732 .addReg(RegNo: SPReg)
733 .addReg(RegNo: TargetReg)
734 .setMIFlag(Flag);
735
736 // If we have a dynamic allocation later we need to probe any residuals.
737 if (DynAllocation) {
738 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: STI.is64Bit() ? RISCV::SD : RISCV::SW))
739 .addReg(RegNo: RISCV::X0)
740 .addReg(RegNo: SPReg)
741 .addImm(Val: 0)
742 .setMIFlags(MachineInstr::FrameSetup);
743 }
744}
745
746static void appendScalableVectorExpression(const TargetRegisterInfo &TRI,
747 SmallVectorImpl<char> &Expr,
748 StackOffset Offset,
749 llvm::raw_string_ostream &Comment) {
750 int64_t FixedOffset = Offset.getFixed();
751 int64_t ScalableOffset = Offset.getScalable();
752 unsigned DwarfVLenB = TRI.getDwarfRegNum(Reg: RISCV::VLENB, isEH: true);
753 if (FixedOffset) {
754 Expr.push_back(Elt: dwarf::DW_OP_consts);
755 appendLEB128<LEB128Sign::Signed>(Buffer&: Expr, Value: FixedOffset);
756 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_plus);
757 Comment << (FixedOffset < 0 ? " - " : " + ") << std::abs(i: FixedOffset);
758 }
759
760 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_consts);
761 appendLEB128<LEB128Sign::Signed>(Buffer&: Expr, Value: ScalableOffset);
762
763 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_bregx);
764 appendLEB128<LEB128Sign::Unsigned>(Buffer&: Expr, Value: DwarfVLenB);
765 Expr.push_back(Elt: 0);
766
767 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_mul);
768 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_plus);
769
770 Comment << (ScalableOffset < 0 ? " - " : " + ") << std::abs(i: ScalableOffset)
771 << " * vlenb";
772}
773
774static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
775 Register Reg,
776 StackOffset Offset) {
777 assert(Offset.getScalable() != 0 && "Did not need to adjust CFA for RVV");
778 SmallString<64> Expr;
779 std::string CommentBuffer;
780 llvm::raw_string_ostream Comment(CommentBuffer);
781 // Build up the expression (Reg + FixedOffset + ScalableOffset * VLENB).
782 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
783 Expr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
784 Expr.push_back(Elt: 0);
785 if (Reg == SPReg)
786 Comment << "sp";
787 else
788 Comment << printReg(Reg, TRI: &TRI);
789
790 appendScalableVectorExpression(TRI, Expr, Offset, Comment);
791
792 SmallString<64> DefCfaExpr;
793 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
794 appendLEB128<LEB128Sign::Unsigned>(Buffer&: DefCfaExpr, Value: Expr.size());
795 DefCfaExpr.append(RHS: Expr.str());
796
797 return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(),
798 Comment: Comment.str());
799}
800
801static MCCFIInstruction createDefCFAOffset(const TargetRegisterInfo &TRI,
802 Register Reg, StackOffset Offset) {
803 assert(Offset.getScalable() != 0 && "Did not need to adjust CFA for RVV");
804 SmallString<64> Expr;
805 std::string CommentBuffer;
806 llvm::raw_string_ostream Comment(CommentBuffer);
807 Comment << printReg(Reg, TRI: &TRI) << " @ cfa";
808
809 // Build up the expression (FixedOffset + ScalableOffset * VLENB).
810 appendScalableVectorExpression(TRI, Expr, Offset, Comment);
811
812 SmallString<64> DefCfaExpr;
813 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
814 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
815 appendLEB128<LEB128Sign::Unsigned>(Buffer&: DefCfaExpr, Value: DwarfReg);
816 appendLEB128<LEB128Sign::Unsigned>(Buffer&: DefCfaExpr, Value: Expr.size());
817 DefCfaExpr.append(RHS: Expr.str());
818
819 return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(),
820 Comment: Comment.str());
821}
822
823// Allocate stack space and probe it if necessary.
824void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
825 MachineBasicBlock::iterator MBBI,
826 MachineFunction &MF, uint64_t Offset,
827 uint64_t RealStackSize, bool EmitCFI,
828 bool NeedProbe, uint64_t ProbeSize,
829 bool DynAllocation,
830 MachineInstr::MIFlag Flag) const {
831 DebugLoc DL;
832 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
833 const RISCVInstrInfo *TII = STI.getInstrInfo();
834 bool IsRV64 = STI.is64Bit();
835 CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
836
837 // Simply allocate the stack if it's not big enough to require a probe.
838 if (!NeedProbe || Offset <= ProbeSize) {
839 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: -Offset),
840 Flag, RequiredAlign: getStackAlign());
841
842 if (EmitCFI)
843 CFIBuilder.buildDefCFAOffset(Offset: RealStackSize);
844
845 if (NeedProbe && DynAllocation) {
846 // s[d|w] zero, 0(sp)
847 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
848 .addReg(RegNo: RISCV::X0)
849 .addReg(RegNo: SPReg)
850 .addImm(Val: 0)
851 .setMIFlags(Flag);
852 }
853
854 return;
855 }
856
857 // Unroll the probe loop depending on the number of iterations.
858 if (Offset < ProbeSize * 5) {
859 uint64_t CFAAdjust = RealStackSize - Offset;
860
861 uint64_t CurrentOffset = 0;
862 while (CurrentOffset + ProbeSize <= Offset) {
863 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg,
864 Offset: StackOffset::getFixed(Fixed: -ProbeSize), Flag, RequiredAlign: getStackAlign());
865 // s[d|w] zero, 0(sp)
866 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
867 .addReg(RegNo: RISCV::X0)
868 .addReg(RegNo: SPReg)
869 .addImm(Val: 0)
870 .setMIFlags(Flag);
871
872 CurrentOffset += ProbeSize;
873 if (EmitCFI)
874 CFIBuilder.buildDefCFAOffset(Offset: CurrentOffset + CFAAdjust);
875 }
876
877 uint64_t Residual = Offset - CurrentOffset;
878 if (Residual) {
879 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg,
880 Offset: StackOffset::getFixed(Fixed: -Residual), Flag, RequiredAlign: getStackAlign());
881 if (EmitCFI)
882 CFIBuilder.buildDefCFAOffset(Offset: RealStackSize);
883
884 if (DynAllocation) {
885 // s[d|w] zero, 0(sp)
886 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
887 .addReg(RegNo: RISCV::X0)
888 .addReg(RegNo: SPReg)
889 .addImm(Val: 0)
890 .setMIFlags(Flag);
891 }
892 }
893
894 return;
895 }
896
897 // Emit a variable-length allocation probing loop.
898 uint64_t RoundedSize = alignDown(Value: Offset, Align: ProbeSize);
899 uint64_t Residual = Offset - RoundedSize;
900
901 Register TargetReg = findScratchNonCalleeSaveRegister(MBB: &MBB, PreferredReg: RISCV::X6);
902 assert(TargetReg.isValid() &&
903 "No available scratch register for stack probing");
904 // SUB TargetReg, SP, RoundedSize
905 RI->adjustReg(MBB, II: MBBI, DL, DestReg: TargetReg, SrcReg: SPReg,
906 Offset: StackOffset::getFixed(Fixed: -RoundedSize), Flag, RequiredAlign: getStackAlign());
907
908 if (EmitCFI) {
909 // Set the CFA register to TargetReg.
910 CFIBuilder.buildDefCFA(Reg: TargetReg, Offset: RoundedSize);
911 }
912
913 // It will be expanded to a probe loop in `inlineStackProbe`.
914 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::PROBED_STACKALLOC)).addReg(RegNo: TargetReg);
915
916 if (EmitCFI) {
917 // Set the CFA register back to SP.
918 CFIBuilder.buildDefCFARegister(Reg: SPReg);
919 }
920
921 if (Residual) {
922 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: -Residual),
923 Flag, RequiredAlign: getStackAlign());
924 if (DynAllocation) {
925 // s[d|w] zero, 0(sp)
926 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
927 .addReg(RegNo: RISCV::X0)
928 .addReg(RegNo: SPReg)
929 .addImm(Val: 0)
930 .setMIFlags(Flag);
931 }
932 }
933
934 if (EmitCFI)
935 CFIBuilder.buildDefCFAOffset(Offset);
936}
937
938static bool isPush(unsigned Opcode) {
939 switch (Opcode) {
940 case RISCV::CM_PUSH:
941 case RISCV::QC_CM_PUSH:
942 case RISCV::QC_CM_PUSHFP:
943 return true;
944 default:
945 return false;
946 }
947}
948
949static bool isPop(unsigned Opcode) {
950 // There are other pops but these are the only ones introduced during this
951 // pass.
952 switch (Opcode) {
953 case RISCV::CM_POP:
954 case RISCV::QC_CM_POP:
955 return true;
956 default:
957 return false;
958 }
959}
960
961static unsigned getPushOpcode(RISCVMachineFunctionInfo::PushPopKind Kind,
962 bool UpdateFP) {
963 switch (Kind) {
964 case RISCVMachineFunctionInfo::PushPopKind::StdExtZcmp:
965 return RISCV::CM_PUSH;
966 case RISCVMachineFunctionInfo::PushPopKind::VendorXqccmp:
967 return UpdateFP ? RISCV::QC_CM_PUSHFP : RISCV::QC_CM_PUSH;
968 default:
969 llvm_unreachable("Unhandled PushPopKind");
970 }
971}
972
973static unsigned getPopOpcode(RISCVMachineFunctionInfo::PushPopKind Kind) {
974 // There are other pops but they are introduced later by the Push/Pop
975 // Optimizer.
976 switch (Kind) {
977 case RISCVMachineFunctionInfo::PushPopKind::StdExtZcmp:
978 return RISCV::CM_POP;
979 case RISCVMachineFunctionInfo::PushPopKind::VendorXqccmp:
980 return RISCV::QC_CM_POP;
981 default:
982 llvm_unreachable("Unhandled PushPopKind");
983 }
984}
985
986void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
987 MachineBasicBlock &MBB) const {
988 MachineFrameInfo &MFI = MF.getFrameInfo();
989 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
990 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
991 MachineBasicBlock::iterator MBBI = MBB.begin();
992 bool PreferAscendingLS = STI.preferAscendingLoadStore();
993
994 Register BPReg = RISCVABI::getBPReg();
995
996 // Debug location must be unknown since the first debug location is used
997 // to determine the end of the prologue.
998 DebugLoc DL;
999
1000 // All calls are tail calls in GHC calling conv, and functions have no
1001 // prologue/epilogue.
1002 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
1003 return;
1004
1005 // SiFive CLIC needs to swap `sp` into `sf.mscratchcsw`
1006 emitSiFiveCLICStackSwap(MF, MBB, MBBI, DL, FrameFlag: MachineInstr::FrameSetup);
1007
1008 // Emit prologue for shadow call stack.
1009 emitSCSPrologue(MF, MBB, MI: MBBI, DL);
1010
1011 // We keep track of the first instruction because it might be a
1012 // `(QC.)CM.PUSH(FP)`, and we may need to adjust the immediate rather than
1013 // inserting an `addi sp, sp, -N*16`
1014 auto PossiblePush = MBBI;
1015
1016 // Skip past all callee-saved register spill instructions.
1017 while (MBBI != MBB.end() && MBBI->getFlag(Flag: MachineInstr::FrameSetup))
1018 ++MBBI;
1019
1020 // Determine the correct frame layout
1021 determineFrameLayout(MF);
1022
1023 const auto &CSI = MFI.getCalleeSavedInfo();
1024
1025 // Skip to before the spills of scalar callee-saved registers
1026 // FIXME: assumes exactly one instruction is used to restore each
1027 // callee-saved register.
1028 MBBI =
1029 std::prev(x: MBBI, n: getRVVCalleeSavedInfo(MF, CSI).size() +
1030 getUnmanagedCSI(MF, CSI, ReverseOrder: PreferAscendingLS).size());
1031 CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
1032 bool NeedsDwarfCFI = needsDwarfCFI(MF);
1033
1034 // If libcalls are used to spill and restore callee-saved registers, the frame
1035 // has two sections; the opaque section managed by the libcalls, and the
1036 // section managed by MachineFrameInfo which can also hold callee saved
1037 // registers in fixed stack slots, both of which have negative frame indices.
1038 // This gets even more complicated when incoming arguments are passed via the
1039 // stack, as these too have negative frame indices. An example is detailed
1040 // below:
1041 //
1042 // | incoming arg | <- FI[-3]
1043 // | libcallspill |
1044 // | calleespill | <- FI[-2]
1045 // | calleespill | <- FI[-1]
1046 // | this_frame | <- FI[0]
1047 //
1048 // For negative frame indices, the offset from the frame pointer will differ
1049 // depending on which of these groups the frame index applies to.
1050 // The following calculates the correct offset knowing the number of callee
1051 // saved registers spilt by the two methods.
1052 if (int LibCallRegs = getLibCallID(MF, CSI: MFI.getCalleeSavedInfo()) + 1) {
1053 // Calculate the size of the frame managed by the libcall. The stack
1054 // alignment of these libcalls should be the same as how we set it in
1055 // getABIStackAlignment.
1056 unsigned LibCallFrameSize =
1057 alignTo(Size: (STI.getXLen() / 8) * LibCallRegs, A: getStackAlign());
1058 RVFI->setLibCallStackSize(LibCallFrameSize);
1059
1060 if (NeedsDwarfCFI) {
1061 CFIBuilder.buildDefCFAOffset(Offset: LibCallFrameSize);
1062 for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
1063 CFIBuilder.buildOffset(Reg: CS.getReg(),
1064 Offset: MFI.getObjectOffset(ObjectIdx: CS.getFrameIdx()));
1065 }
1066 }
1067
1068 // FIXME (note copied from Lanai): This appears to be overallocating. Needs
1069 // investigation. Get the number of bytes to allocate from the FrameInfo.
1070 uint64_t RealStackSize = getStackSizeWithRVVPadding(MF);
1071 uint64_t StackSize = RealStackSize - RVFI->getReservedSpillsSize();
1072 uint64_t RVVStackSize = RVFI->getRVVStackSize();
1073
1074 // Early exit if there is no need to allocate on the stack
1075 if (RealStackSize == 0 && !MFI.adjustsStack() && RVVStackSize == 0)
1076 return;
1077
1078 // If the stack pointer has been marked as reserved, then produce an error if
1079 // the frame requires stack allocation
1080 if (STI.isRegisterReservedByUser(i: SPReg))
1081 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
1082 MF.getFunction(), "Stack pointer required, but has been reserved."});
1083
1084 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
1085 // Split the SP adjustment to reduce the offsets of callee saved spill.
1086 if (FirstSPAdjustAmount) {
1087 StackSize = FirstSPAdjustAmount;
1088 RealStackSize = FirstSPAdjustAmount;
1089 }
1090
1091 if (RVFI->useQCIInterrupt(MF)) {
1092 // The function starts with `QC.C.MIENTER(.NEST)`, so the `(QC.)CM.PUSH(FP)`
1093 // could only be the next instruction.
1094 ++PossiblePush;
1095
1096 if (NeedsDwarfCFI) {
1097 // Insert the CFI metadata before where we think the `(QC.)CM.PUSH(FP)`
1098 // could be. The PUSH will also get its own CFI metadata for its own
1099 // modifications, which should come after the PUSH.
1100 CFIInstBuilder PushCFIBuilder(MBB, PossiblePush,
1101 MachineInstr::FrameSetup);
1102 PushCFIBuilder.buildDefCFAOffset(Offset: QCIInterruptPushAmount);
1103 for (const CalleeSavedInfo &CS : getQCISavedInfo(MF, CSI))
1104 PushCFIBuilder.buildOffset(Reg: CS.getReg(),
1105 Offset: MFI.getObjectOffset(ObjectIdx: CS.getFrameIdx()));
1106 }
1107 }
1108
1109 if (RVFI->isPushable(MF) && PossiblePush != MBB.end() &&
1110 isPush(Opcode: PossiblePush->getOpcode())) {
1111 // Use available stack adjustment in push instruction to allocate additional
1112 // stack space. Align the stack size down to a multiple of 16. This is
1113 // needed for RVE.
1114 // FIXME: Can we increase the stack size to a multiple of 16 instead?
1115 uint64_t StackAdj =
1116 std::min(a: alignDown(Value: StackSize, Align: 16), b: static_cast<uint64_t>(48));
1117 PossiblePush->getOperand(i: 1).setImm(StackAdj);
1118 StackSize -= StackAdj;
1119
1120 if (NeedsDwarfCFI) {
1121 CFIBuilder.buildDefCFAOffset(Offset: RealStackSize - StackSize);
1122 for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
1123 CFIBuilder.buildOffset(Reg: CS.getReg(),
1124 Offset: MFI.getObjectOffset(ObjectIdx: CS.getFrameIdx()));
1125 }
1126 }
1127
1128 // Allocate space on the stack if necessary.
1129 auto &Subtarget = MF.getSubtarget<RISCVSubtarget>();
1130 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
1131 bool NeedProbe = TLI->hasInlineStackProbe(MF);
1132 uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign: getStackAlign());
1133 bool DynAllocation =
1134 MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
1135 if (StackSize != 0)
1136 allocateStack(MBB, MBBI, MF, Offset: StackSize, RealStackSize, EmitCFI: NeedsDwarfCFI,
1137 NeedProbe, ProbeSize, DynAllocation,
1138 Flag: MachineInstr::FrameSetup);
1139
1140 // Save SiFive CLIC CSRs into Stack
1141 emitSiFiveCLICPreemptibleSaves(MF, MBB, MBBI, DL);
1142
1143 // The frame pointer is callee-saved, and code has been generated for us to
1144 // save it to the stack. We need to skip over the storing of callee-saved
1145 // registers as the frame pointer must be modified after it has been saved
1146 // to the stack, not before.
1147 // FIXME: assumes exactly one instruction is used to save each callee-saved
1148 // register.
1149 std::advance(i&: MBBI, n: getUnmanagedCSI(MF, CSI, ReverseOrder: PreferAscendingLS).size());
1150 CFIBuilder.setInsertPoint(MBBI);
1151
1152 // Iterate over list of callee-saved registers and emit .cfi_offset
1153 // directives.
1154 if (NeedsDwarfCFI) {
1155 for (const CalleeSavedInfo &CS :
1156 getUnmanagedCSI(MF, CSI, ReverseOrder: PreferAscendingLS)) {
1157 MCRegister Reg = CS.getReg();
1158 int64_t Offset = MFI.getObjectOffset(ObjectIdx: CS.getFrameIdx());
1159 // Emit CFI for both sub-registers. The even register is at the base
1160 // offset and odd at base+4.
1161 if (RISCV::GPRPairRegClass.contains(Reg)) {
1162 MCRegister EvenReg = RI->getSubReg(Reg, Idx: RISCV::sub_gpr_even);
1163 MCRegister OddReg = RI->getSubReg(Reg, Idx: RISCV::sub_gpr_odd);
1164 CFIBuilder.buildOffset(Reg: EvenReg, Offset);
1165 CFIBuilder.buildOffset(Reg: OddReg, Offset: Offset + 4);
1166 } else {
1167 CFIBuilder.buildOffset(Reg, Offset);
1168 }
1169 }
1170 }
1171
1172 // Generate new FP.
1173 if (hasFP(MF)) {
1174 if (STI.isRegisterReservedByUser(i: FPReg))
1175 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
1176 MF.getFunction(), "Frame pointer required, but has been reserved."});
1177 // The frame pointer does need to be reserved from register allocation.
1178 assert(MF.getRegInfo().isReserved(FPReg) && "FP not reserved");
1179
1180 // Some stack management variants automatically keep FP updated, so we don't
1181 // need an instruction to do so.
1182 if (!RVFI->hasImplicitFPUpdates(MF)) {
1183 RI->adjustReg(
1184 MBB, II: MBBI, DL, DestReg: FPReg, SrcReg: SPReg,
1185 Offset: StackOffset::getFixed(Fixed: RealStackSize - RVFI->getVarArgsSaveSize()),
1186 Flag: MachineInstr::FrameSetup, RequiredAlign: getStackAlign());
1187 }
1188
1189 if (NeedsDwarfCFI)
1190 CFIBuilder.buildDefCFA(Reg: FPReg, Offset: RVFI->getVarArgsSaveSize());
1191 }
1192
1193 uint64_t SecondSPAdjustAmount = 0;
1194 // Emit the second SP adjustment after saving callee saved registers.
1195 if (FirstSPAdjustAmount) {
1196 SecondSPAdjustAmount = getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
1197 assert(SecondSPAdjustAmount > 0 &&
1198 "SecondSPAdjustAmount should be greater than zero");
1199
1200 allocateStack(MBB, MBBI, MF, Offset: SecondSPAdjustAmount,
1201 RealStackSize: getStackSizeWithRVVPadding(MF), EmitCFI: NeedsDwarfCFI && !hasFP(MF),
1202 NeedProbe, ProbeSize, DynAllocation,
1203 Flag: MachineInstr::FrameSetup);
1204 }
1205
1206 if (RVVStackSize) {
1207 if (NeedProbe) {
1208 allocateAndProbeStackForRVV(MF, MBB, MBBI, DL, Amount: RVVStackSize,
1209 Flag: MachineInstr::FrameSetup,
1210 EmitCFI: NeedsDwarfCFI && !hasFP(MF), DynAllocation);
1211 } else {
1212 // We must keep the stack pointer aligned through any intermediate
1213 // updates.
1214 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg,
1215 Offset: StackOffset::getScalable(Scalable: -RVVStackSize),
1216 Flag: MachineInstr::FrameSetup, RequiredAlign: getStackAlign());
1217 }
1218
1219 if (NeedsDwarfCFI && !hasFP(MF)) {
1220 // Emit .cfi_def_cfa_expression "sp + StackSize + RVVStackSize * vlenb".
1221 CFIBuilder.insertCFIInst(CFIInst: createDefCFAExpression(
1222 TRI: *RI, Reg: SPReg,
1223 Offset: StackOffset::get(Fixed: getStackSizeWithRVVPadding(MF), Scalable: RVVStackSize / 8)));
1224 }
1225
1226 std::advance(i&: MBBI, n: getRVVCalleeSavedInfo(MF, CSI).size());
1227 if (NeedsDwarfCFI)
1228 emitCalleeSavedRVVPrologCFI(MBB, MI: MBBI, HasFP: hasFP(MF));
1229 }
1230
1231 if (hasFP(MF)) {
1232 // Realign Stack
1233 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
1234 if (RI->hasStackRealignment(MF)) {
1235 Align MaxAlignment = MFI.getMaxAlign();
1236
1237 const RISCVInstrInfo *TII = STI.getInstrInfo();
1238 if (isInt<12>(x: -(int)MaxAlignment.value())) {
1239 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::ANDI), DestReg: SPReg)
1240 .addReg(RegNo: SPReg)
1241 .addImm(Val: -(int)MaxAlignment.value())
1242 .setMIFlag(MachineInstr::FrameSetup);
1243 } else {
1244 unsigned ShiftAmount = Log2(A: MaxAlignment);
1245 Register VR =
1246 MF.getRegInfo().createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1247 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SRLI), DestReg: VR)
1248 .addReg(RegNo: SPReg)
1249 .addImm(Val: ShiftAmount)
1250 .setMIFlag(MachineInstr::FrameSetup);
1251 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SLLI), DestReg: SPReg)
1252 .addReg(RegNo: VR)
1253 .addImm(Val: ShiftAmount)
1254 .setMIFlag(MachineInstr::FrameSetup);
1255 }
1256 if (NeedProbe && RVVStackSize == 0) {
1257 // Do a probe if the align + size allocated just passed the probe size
1258 // and was not yet probed.
1259 if (SecondSPAdjustAmount < ProbeSize &&
1260 SecondSPAdjustAmount + MaxAlignment.value() >= ProbeSize) {
1261 bool IsRV64 = STI.is64Bit();
1262 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
1263 .addReg(RegNo: RISCV::X0)
1264 .addReg(RegNo: SPReg)
1265 .addImm(Val: 0)
1266 .setMIFlags(MachineInstr::FrameSetup);
1267 }
1268 }
1269 // FP will be used to restore the frame in the epilogue, so we need
1270 // another base register BP to record SP after re-alignment. SP will
1271 // track the current stack after allocating variable sized objects.
1272 if (hasBP(MF)) {
1273 // move BP, SP
1274 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: BPReg)
1275 .addReg(RegNo: SPReg)
1276 .addImm(Val: 0)
1277 .setMIFlag(MachineInstr::FrameSetup);
1278 }
1279 }
1280 }
1281}
1282
1283void RISCVFrameLowering::deallocateStack(MachineFunction &MF,
1284 MachineBasicBlock &MBB,
1285 MachineBasicBlock::iterator MBBI,
1286 const DebugLoc &DL,
1287 uint64_t &StackSize,
1288 int64_t CFAOffset) const {
1289 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
1290
1291 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: StackSize),
1292 Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign());
1293 StackSize = 0;
1294
1295 if (needsDwarfCFI(MF))
1296 CFIInstBuilder(MBB, MBBI, MachineInstr::FrameDestroy)
1297 .buildDefCFAOffset(Offset: CFAOffset);
1298}
1299
1300void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
1301 MachineBasicBlock &MBB) const {
1302 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
1303 MachineFrameInfo &MFI = MF.getFrameInfo();
1304 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1305 bool PreferAscendingLS = STI.preferAscendingLoadStore();
1306
1307 // All calls are tail calls in GHC calling conv, and functions have no
1308 // prologue/epilogue.
1309 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
1310 return;
1311
1312 // Get the insert location for the epilogue. If there were no terminators in
1313 // the block, get the last instruction.
1314 MachineBasicBlock::iterator MBBI = MBB.end();
1315 DebugLoc DL;
1316 if (!MBB.empty()) {
1317 MBBI = MBB.getLastNonDebugInstr();
1318 if (MBBI != MBB.end())
1319 DL = MBBI->getDebugLoc();
1320
1321 MBBI = MBB.getFirstTerminator();
1322
1323 // Skip to before the restores of all callee-saved registers.
1324 while (MBBI != MBB.begin() &&
1325 std::prev(x: MBBI)->getFlag(Flag: MachineInstr::FrameDestroy))
1326 --MBBI;
1327 }
1328
1329 const auto &CSI = MFI.getCalleeSavedInfo();
1330
1331 // Skip to before the restores of scalar callee-saved registers
1332 // FIXME: assumes exactly one instruction is used to restore each
1333 // callee-saved register.
1334 auto FirstScalarCSRRestoreInsn =
1335 std::next(x: MBBI, n: getRVVCalleeSavedInfo(MF, CSI).size());
1336 CFIInstBuilder CFIBuilder(MBB, FirstScalarCSRRestoreInsn,
1337 MachineInstr::FrameDestroy);
1338 bool NeedsDwarfCFI = needsDwarfCFI(MF);
1339
1340 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
1341 uint64_t RealStackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
1342 : getStackSizeWithRVVPadding(MF);
1343 uint64_t StackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
1344 : getStackSizeWithRVVPadding(MF) -
1345 RVFI->getReservedSpillsSize();
1346 uint64_t FPOffset = RealStackSize - RVFI->getVarArgsSaveSize();
1347 uint64_t RVVStackSize = RVFI->getRVVStackSize();
1348
1349 bool RestoreSPFromFP = RI->hasStackRealignment(MF) ||
1350 MFI.hasVarSizedObjects() || !hasReservedCallFrame(MF);
1351 if (RVVStackSize) {
1352 // If RestoreSPFromFP the stack pointer will be restored using the frame
1353 // pointer value.
1354 if (!RestoreSPFromFP)
1355 RI->adjustReg(MBB, II: FirstScalarCSRRestoreInsn, DL, DestReg: SPReg, SrcReg: SPReg,
1356 Offset: StackOffset::getScalable(Scalable: RVVStackSize),
1357 Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign());
1358
1359 if (NeedsDwarfCFI) {
1360 if (!hasFP(MF))
1361 CFIBuilder.buildDefCFA(Reg: SPReg, Offset: RealStackSize);
1362 emitCalleeSavedRVVEpilogCFI(MBB, MI: FirstScalarCSRRestoreInsn);
1363 }
1364 }
1365
1366 if (FirstSPAdjustAmount) {
1367 uint64_t SecondSPAdjustAmount =
1368 getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
1369 assert(SecondSPAdjustAmount > 0 &&
1370 "SecondSPAdjustAmount should be greater than zero");
1371
1372 // If RestoreSPFromFP the stack pointer will be restored using the frame
1373 // pointer value.
1374 if (!RestoreSPFromFP)
1375 RI->adjustReg(MBB, II: FirstScalarCSRRestoreInsn, DL, DestReg: SPReg, SrcReg: SPReg,
1376 Offset: StackOffset::getFixed(Fixed: SecondSPAdjustAmount),
1377 Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign());
1378
1379 if (NeedsDwarfCFI && !hasFP(MF))
1380 CFIBuilder.buildDefCFAOffset(Offset: FirstSPAdjustAmount);
1381 }
1382
1383 // Restore the stack pointer using the value of the frame pointer. Only
1384 // necessary if the stack pointer was modified, meaning the stack size is
1385 // unknown.
1386 //
1387 // In order to make sure the stack point is right through the EH region,
1388 // we also need to restore stack pointer from the frame pointer if we
1389 // don't preserve stack space within prologue/epilogue for outgoing variables,
1390 // normally it's just checking the variable sized object is present or not
1391 // is enough, but we also don't preserve that at prologue/epilogue when
1392 // have vector objects in stack.
1393 if (RestoreSPFromFP) {
1394 assert(hasFP(MF) && "frame pointer should not have been eliminated");
1395 RI->adjustReg(MBB, II: FirstScalarCSRRestoreInsn, DL, DestReg: SPReg, SrcReg: FPReg,
1396 Offset: StackOffset::getFixed(Fixed: -FPOffset), Flag: MachineInstr::FrameDestroy,
1397 RequiredAlign: getStackAlign());
1398 }
1399
1400 if (NeedsDwarfCFI && hasFP(MF))
1401 CFIBuilder.buildDefCFA(Reg: SPReg, Offset: RealStackSize);
1402
1403 // Skip to after the restores of scalar callee-saved registers
1404 // FIXME: assumes exactly one instruction is used to restore each
1405 // callee-saved register.
1406 MBBI = std::next(x: FirstScalarCSRRestoreInsn,
1407 n: getUnmanagedCSI(MF, CSI, ReverseOrder: PreferAscendingLS).size());
1408 CFIBuilder.setInsertPoint(MBBI);
1409
1410 if (getLibCallID(MF, CSI) != -1) {
1411 // tail __riscv_restore_[0-12] instruction is considered as a terminator,
1412 // therefore it is unnecessary to place any CFI instructions after it. Just
1413 // deallocate stack if needed and return.
1414 if (StackSize != 0)
1415 deallocateStack(MF, MBB, MBBI, DL, StackSize,
1416 CFAOffset: RVFI->getLibCallStackSize());
1417
1418 // Emit epilogue for shadow call stack.
1419 emitSCSEpilogue(MF, MBB, MI: MBBI, DL);
1420 return;
1421 }
1422
1423 // Recover callee-saved registers.
1424 if (NeedsDwarfCFI) {
1425 for (const CalleeSavedInfo &CS :
1426 getUnmanagedCSI(MF, CSI, ReverseOrder: PreferAscendingLS)) {
1427 MCRegister Reg = CS.getReg();
1428 // Emit CFI for both sub-registers.
1429 if (RISCV::GPRPairRegClass.contains(Reg)) {
1430 MCRegister EvenReg = RI->getSubReg(Reg, Idx: RISCV::sub_gpr_even);
1431 MCRegister OddReg = RI->getSubReg(Reg, Idx: RISCV::sub_gpr_odd);
1432 CFIBuilder.buildRestore(Reg: EvenReg);
1433 CFIBuilder.buildRestore(Reg: OddReg);
1434 } else {
1435 CFIBuilder.buildRestore(Reg);
1436 }
1437 }
1438 }
1439
1440 if (RVFI->isPushable(MF) && MBBI != MBB.end() && isPop(Opcode: MBBI->getOpcode())) {
1441 // Use available stack adjustment in pop instruction to deallocate stack
1442 // space. Align the stack size down to a multiple of 16. This is needed for
1443 // RVE.
1444 // FIXME: Can we increase the stack size to a multiple of 16 instead?
1445 uint64_t StackAdj =
1446 std::min(a: alignDown(Value: StackSize, Align: 16), b: static_cast<uint64_t>(48));
1447 MBBI->getOperand(i: 1).setImm(StackAdj);
1448 StackSize -= StackAdj;
1449
1450 if (StackSize != 0)
1451 deallocateStack(MF, MBB, MBBI, DL, StackSize,
1452 /*stack_adj of cm.pop instr*/ CFAOffset: RealStackSize - StackSize);
1453
1454 auto NextI = next_nodbg(It: MBBI, End: MBB.end());
1455 if (NextI == MBB.end() || NextI->getOpcode() != RISCV::PseudoRET) {
1456 ++MBBI;
1457 if (NeedsDwarfCFI) {
1458 CFIBuilder.setInsertPoint(MBBI);
1459
1460 for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
1461 CFIBuilder.buildRestore(Reg: CS.getReg());
1462
1463 // Update CFA Offset. If this is a QCI interrupt function, there will
1464 // be a leftover offset which is deallocated by `QC.C.MILEAVERET`,
1465 // otherwise getQCIInterruptStackSize() will be 0.
1466 CFIBuilder.buildDefCFAOffset(Offset: RVFI->getQCIInterruptStackSize());
1467 }
1468 }
1469 }
1470
1471 emitSiFiveCLICPreemptibleRestores(MF, MBB, MBBI, DL);
1472
1473 // Deallocate stack if StackSize isn't a zero yet. If this is a QCI interrupt
1474 // function, there will be a leftover offset which is deallocated by
1475 // `QC.C.MILEAVERET`, otherwise getQCIInterruptStackSize() will be 0.
1476 if (StackSize != 0)
1477 deallocateStack(MF, MBB, MBBI, DL, StackSize,
1478 CFAOffset: RVFI->getQCIInterruptStackSize());
1479
1480 // Emit epilogue for shadow call stack.
1481 emitSCSEpilogue(MF, MBB, MI: MBBI, DL);
1482
1483 // SiFive CLIC needs to swap `sf.mscratchcsw` into `sp`
1484 emitSiFiveCLICStackSwap(MF, MBB, MBBI, DL, FrameFlag: MachineInstr::FrameDestroy);
1485}
1486
1487static MCRegister getPhysicalGPR(const TargetRegisterInfo &TRI,
1488 MCRegister Reg) {
1489 if (RISCV::GPRRegClass.contains(Reg))
1490 return Reg;
1491
1492 std::array<TargetRegisterClass const *, 2> RegisterClasses = {
1493 &RISCV::GPRF16RegClass, &RISCV::GPRF32RegClass};
1494 std::array<unsigned, 2> SubIdx = {RISCV::sub_16, RISCV::sub_32};
1495
1496 for (auto [RegClass, SubReg] : zip(t&: RegisterClasses, u&: SubIdx)) {
1497 if (RegClass->contains(Reg)) {
1498 if (MCRegister Super =
1499 TRI.getMatchingSuperReg(Reg, SubIdx: SubReg, RC: &RISCV::GPRRegClass))
1500 return Super;
1501 }
1502 }
1503
1504 llvm::reportFatalInternalError(
1505 reason: "getPhysicalGPR called with unsupported register");
1506}
1507
1508static MCRegister getLargestFPRegisterOrZero(const RISCVSubtarget &STI,
1509 const TargetRegisterInfo &TRI,
1510 MCRegister Reg) {
1511 if (!STI.hasStdExtF())
1512 return MCRegister();
1513
1514 TargetRegisterClass const *LargestFPRegClass = STI.getLargestFPRegClass();
1515 assert(LargestFPRegClass);
1516
1517 if (LargestFPRegClass->contains(Reg))
1518 return Reg;
1519
1520 std::array<TargetRegisterClass const *, 3> RegisterClasses = {
1521 &RISCV::FPR16RegClass, &RISCV::FPR32RegClass, &RISCV::FPR64RegClass};
1522 std::array<unsigned, 3> SubIdx = {RISCV::sub_16, RISCV::sub_32,
1523 RISCV::sub_64};
1524
1525 for (auto [RegClass, SubReg] : zip(t&: RegisterClasses, u&: SubIdx)) {
1526 if (RegClass->contains(Reg)) {
1527 if (MCRegister Super =
1528 TRI.getMatchingSuperReg(Reg, SubIdx: SubReg, RC: LargestFPRegClass))
1529 return Super;
1530 }
1531 }
1532
1533 // Reg is bigger than what's currently available for the target, we can ignore
1534 // it.
1535 return MCRegister();
1536}
1537
1538void RISCVFrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero,
1539 MachineBasicBlock &MBB,
1540 RegScavenger *RS) const {
1541 // Insertion point.
1542 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
1543
1544 // Fake a debug loc.
1545 DebugLoc DL;
1546 if (MBBI != MBB.end())
1547 DL = MBBI->getDebugLoc();
1548
1549 const MachineFunction &MF = *MBB.getParent();
1550 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
1551 const RISCVInstrInfo &TII = *STI.getInstrInfo();
1552
1553 BitVector FinalRegsToZero(TRI.getNumRegs());
1554
1555 bool HasVRegister = false;
1556
1557 for (MCRegister Reg : RegsToZero.set_bits()) {
1558 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
1559 FinalRegsToZero.set(getPhysicalGPR(TRI, Reg).id());
1560 } else if (RISCV::GPRPairRegClass.contains(Reg)) {
1561 FinalRegsToZero.set(
1562 getPhysicalGPR(TRI, Reg: TRI.getSubReg(Reg, Idx: RISCV::sub_gpr_even)).id());
1563 FinalRegsToZero.set(
1564 getPhysicalGPR(TRI, Reg: TRI.getSubReg(Reg, Idx: RISCV::sub_gpr_odd)).id());
1565 } else if (TRI.isFPRegister(Reg)) {
1566 if (MCRegister MaybeReg = getLargestFPRegisterOrZero(STI, TRI, Reg))
1567 FinalRegsToZero.set(MaybeReg.id());
1568 } else if (RISCVRegisterInfo::isRVVRegClass(
1569 RC: TRI.getMinimalPhysRegClass(Reg))) {
1570 if (!STI.hasVInstructions())
1571 continue;
1572 HasVRegister = true;
1573
1574 for (MCRegister SubReg : TRI.subregs_inclusive(Reg)) {
1575 if (TRI.subregs(Reg: SubReg).empty())
1576 FinalRegsToZero.set(SubReg.id());
1577 }
1578 }
1579 }
1580
1581 if (HasVRegister) {
1582 RISCVVType::VLMUL VLMUL = RISCVVType::encodeLMUL(LMUL: 1, /*Fractional=*/false);
1583 unsigned VTypeImm = RISCVVType::encodeVTYPE(
1584 VLMUL, /*SEW=*/32, /*TailAgnostic=*/true, /*MaskAgnostic=*/true);
1585
1586 MCRegister TemporaryReg = RISCV::NoRegister;
1587 for (MCRegister Reg : FinalRegsToZero.set_bits()) {
1588 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
1589 TemporaryReg = Reg;
1590 break;
1591 }
1592 }
1593
1594 if (TemporaryReg == RISCV::NoRegister) {
1595 RS->enterBasicBlockEnd(MBB);
1596 TemporaryReg = RS->scavengeRegisterBackwards(RC: RISCV::GPRRegClass, To: MBBI,
1597 /*RestoreAfter=*/false,
1598 /*SPAdj=*/0);
1599 }
1600
1601 if (MBB.getParent()
1602 ->getFunction()
1603 .getFnAttribute(Kind: "zero-call-used-regs")
1604 .getValueAsString() == "used")
1605 FinalRegsToZero.set(TemporaryReg.id());
1606
1607 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: RISCV::VSETVLI), DestReg: TemporaryReg)
1608 .addReg(RegNo: RISCV::X0)
1609 .addImm(Val: VTypeImm)
1610 .addReg(RegNo: RISCV::VL, Flags: RegState::ImplicitDefine)
1611 .addReg(RegNo: RISCV::VTYPE, Flags: RegState::ImplicitDefine);
1612 }
1613
1614 for (MCRegister Reg : FinalRegsToZero.set_bits())
1615 TII.buildClearRegister(Reg, MBB, Iter: MBBI, DL);
1616}
1617
1618StackOffset
1619RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
1620 Register &FrameReg) const {
1621 const MachineFrameInfo &MFI = MF.getFrameInfo();
1622 const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
1623 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1624
1625 // Callee-saved registers should be referenced relative to the stack
1626 // pointer (positive offset), otherwise use the frame pointer (negative
1627 // offset).
1628 const auto &CSI = getUnmanagedCSI(MF, CSI: MFI.getCalleeSavedInfo(),
1629 ReverseOrder: STI.preferAscendingLoadStore());
1630 int MinCSFI = 0;
1631 int MaxCSFI = -1;
1632 StackOffset Offset;
1633 auto StackID = MFI.getStackID(ObjectIdx: FI);
1634
1635 assert((StackID == TargetStackID::Default ||
1636 StackID == TargetStackID::ScalableVector) &&
1637 "Unexpected stack ID for the frame object.");
1638 if (StackID == TargetStackID::Default) {
1639 assert(getOffsetOfLocalArea() == 0 && "LocalAreaOffset is not 0!");
1640 Offset = StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FI) +
1641 MFI.getOffsetAdjustment());
1642 } else if (StackID == TargetStackID::ScalableVector) {
1643 Offset = StackOffset::getScalable(Scalable: MFI.getObjectOffset(ObjectIdx: FI));
1644 }
1645
1646 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
1647
1648 if (CSI.size()) {
1649 MinCSFI = std::min(a: CSI.front().getFrameIdx(), b: CSI.back().getFrameIdx());
1650 MaxCSFI = std::max(a: CSI.front().getFrameIdx(), b: CSI.back().getFrameIdx());
1651 }
1652
1653 if (FI >= MinCSFI && FI <= MaxCSFI) {
1654 FrameReg = SPReg;
1655
1656 if (FirstSPAdjustAmount)
1657 Offset += StackOffset::getFixed(Fixed: FirstSPAdjustAmount);
1658 else
1659 Offset += StackOffset::getFixed(Fixed: getStackSizeWithRVVPadding(MF));
1660 return Offset;
1661 }
1662
1663 if (RI->hasStackRealignment(MF) && !MFI.isFixedObjectIndex(ObjectIdx: FI)) {
1664 // If the stack was realigned, the frame pointer is set in order to allow
1665 // SP to be restored, so we need another base register to record the stack
1666 // after realignment.
1667 // |--------------------------| --
1668 // | callee-allocated save | | <----|
1669 // | area for register varargs| | |
1670 // |--------------------------| <-- FP |
1671 // | callee-saved registers | | |
1672 // |--------------------------| -- |
1673 // | realignment (the size of | | |
1674 // | this area is not counted | | |
1675 // | in MFI.getStackSize()) | | |
1676 // |--------------------------| -- |-- MFI.getStackSize()
1677 // | RVV alignment padding | | |
1678 // | (not counted in | | |
1679 // | MFI.getStackSize() but | | |
1680 // | counted in | | |
1681 // | RVFI.getRVVStackSize()) | | |
1682 // |--------------------------| -- |
1683 // | RVV objects | | |
1684 // | (not counted in | | |
1685 // | MFI.getStackSize()) | | |
1686 // |--------------------------| -- |
1687 // | padding before RVV | | |
1688 // | (not counted in | | |
1689 // | MFI.getStackSize() or in | | |
1690 // | RVFI.getRVVStackSize()) | | |
1691 // |--------------------------| -- |
1692 // | scalar local variables | | <----'
1693 // |--------------------------| -- <-- BP (if var sized objects present)
1694 // | VarSize objects | |
1695 // |--------------------------| -- <-- SP
1696 if (hasBP(MF)) {
1697 FrameReg = RISCVABI::getBPReg();
1698 } else {
1699 // VarSize objects must be empty in this case!
1700 assert(!MFI.hasVarSizedObjects());
1701 FrameReg = SPReg;
1702 }
1703 } else if (!RI->hasStackRealignment(MF)) {
1704 // Note: Keeping the following as multiple 'if' statements rather than
1705 // merging to a single expression for readability.
1706 if (!hasFP(MF)) {
1707 // No FP available, must use SP.
1708 FrameReg = SPReg;
1709 } else {
1710 FrameReg = FPReg;
1711 // SP-relative addressing is only valid when SP is stable throughout
1712 // the function body: no dynamic SP adjustments for outgoing call args,
1713 // no variable-sized objects, and no RVV scalable stack regions.
1714 // hasReservedCallFrame() conservatively encompasses all these checks.
1715 if (hasReservedCallFrame(MF)) {
1716 // Both FP and SP are candidates.
1717 // Prefer SP when the SP-relative offset fits in the compressed
1718 // instruction immediate range.
1719 int64_t SPOff = Offset.getFixed() + MFI.getStackSize();
1720 int64_t CLWSPMaxOffset = 252;
1721 int64_t CLDSPMaxOffset = 504;
1722 int64_t SPThreshold = STI.is64Bit() ? CLDSPMaxOffset : CLWSPMaxOffset;
1723 if (SPOff >= 0 && SPOff <= SPThreshold)
1724 FrameReg = SPReg;
1725 }
1726 }
1727 } else {
1728 assert(RI->hasStackRealignment(MF) && MFI.isFixedObjectIndex(FI) &&
1729 "Expected fixed object with stack realignment");
1730 assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
1731 FrameReg = FPReg;
1732 }
1733
1734 if (FrameReg == FPReg) {
1735 Offset += StackOffset::getFixed(Fixed: RVFI->getVarArgsSaveSize());
1736 // When using FP to access scalable vector objects, we need to minus
1737 // the frame size.
1738 //
1739 // |--------------------------| --
1740 // | callee-allocated save | |
1741 // | area for register varargs| |
1742 // |--------------------------| | -- <-- FP
1743 // | callee-saved registers | |
1744 // |--------------------------| | MFI.getStackSize()
1745 // | scalar local variables | |
1746 // |--------------------------| -- (Offset of RVV objects is from here.)
1747 // | RVV objects |
1748 // |--------------------------|
1749 // | VarSize objects |
1750 // |--------------------------| <-- SP
1751 if (StackID == TargetStackID::ScalableVector) {
1752 assert(!RI->hasStackRealignment(MF) &&
1753 "Can't index across variable sized realign");
1754 // We don't expect any extra RVV alignment padding, as the stack size
1755 // and RVV object sections should be correct aligned in their own
1756 // right.
1757 assert(MFI.getStackSize() == getStackSizeWithRVVPadding(MF) &&
1758 "Inconsistent stack layout");
1759 Offset -= StackOffset::getFixed(Fixed: MFI.getStackSize());
1760 }
1761 return Offset;
1762 }
1763
1764 // This case handles indexing off both SP and BP.
1765 // If indexing off SP, there must not be any var sized objects
1766 assert(FrameReg == RISCVABI::getBPReg() || !MFI.hasVarSizedObjects());
1767
1768 // When using SP to access frame objects, we need to add RVV stack size.
1769 //
1770 // |--------------------------| --
1771 // | callee-allocated save | | <----|
1772 // | area for register varargs| | |
1773 // |--------------------------| | | <-- FP
1774 // | callee-saved registers | | |
1775 // |--------------------------| -- |
1776 // | RVV alignment padding | | |
1777 // | (not counted in | | |
1778 // | MFI.getStackSize() but | | |
1779 // | counted in | | |
1780 // | RVFI.getRVVStackSize()) | | |
1781 // |--------------------------| -- |
1782 // | RVV objects | | |-- MFI.getStackSize()
1783 // | (not counted in | | |
1784 // | MFI.getStackSize()) | | |
1785 // |--------------------------| -- |
1786 // | padding before RVV | | |
1787 // | (not counted in | | |
1788 // | MFI.getStackSize()) | | |
1789 // |--------------------------| -- |
1790 // | scalar local variables | | <----'
1791 // |--------------------------| -- <-- BP (if var sized objects present)
1792 // | VarSize objects | |
1793 // |--------------------------| -- <-- SP
1794 //
1795 // The total amount of padding surrounding RVV objects is described by
1796 // RVV->getRVVPadding() and it can be zero. It allows us to align the RVV
1797 // objects to the required alignment.
1798 if (MFI.getStackID(ObjectIdx: FI) == TargetStackID::Default) {
1799 if (MFI.isFixedObjectIndex(ObjectIdx: FI)) {
1800 assert(!RI->hasStackRealignment(MF) &&
1801 "Can't index across variable sized realign");
1802 Offset += StackOffset::get(Fixed: getStackSizeWithRVVPadding(MF),
1803 Scalable: RVFI->getRVVStackSize());
1804 } else {
1805 Offset += StackOffset::getFixed(Fixed: MFI.getStackSize());
1806 }
1807 } else if (MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector) {
1808 // Ensure the base of the RVV stack is correctly aligned: add on the
1809 // alignment padding.
1810 int64_t ScalarLocalVarSize =
1811 MFI.getStackSize() - RVFI->getCalleeSavedStackSize() -
1812 RVFI->getVarArgsSaveSize() + RVFI->getRVVPadding();
1813 Offset += StackOffset::get(Fixed: ScalarLocalVarSize, Scalable: RVFI->getRVVStackSize());
1814 }
1815 return Offset;
1816}
1817
1818static MCRegister getRVVBaseRegister(const RISCVRegisterInfo &TRI,
1819 const Register &Reg) {
1820 MCRegister BaseReg = TRI.getSubReg(Reg, Idx: RISCV::sub_vrm1_0);
1821 // If it's not a grouped vector register, it doesn't have subregister, so
1822 // the base register is just itself.
1823 if (!BaseReg.isValid())
1824 BaseReg = Reg;
1825 return BaseReg;
1826}
1827
1828void RISCVFrameLowering::determineCalleeSaves(MachineFunction &MF,
1829 BitVector &SavedRegs,
1830 RegScavenger *RS) const {
1831 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
1832
1833 // In TargetFrameLowering::determineCalleeSaves, any vector register is marked
1834 // as saved if any of its subregister is clobbered, this is not correct in
1835 // vector registers. We only want the vector register to be marked as saved
1836 // if all of its subregisters are clobbered.
1837 // For example:
1838 // Original behavior: If v24 is marked, v24m2, v24m4, v24m8 are also marked.
1839 // Correct behavior: v24m2 is marked only if v24 and v25 are marked.
1840 MachineRegisterInfo &MRI = MF.getRegInfo();
1841 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
1842 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
1843 for (unsigned i = 0; CSRegs[i]; ++i) {
1844 unsigned CSReg = CSRegs[i];
1845 // Only vector registers need special care.
1846 if (!RISCV::VRRegClass.contains(Reg: getRVVBaseRegister(TRI, Reg: CSReg)))
1847 continue;
1848
1849 SavedRegs.reset(Idx: CSReg);
1850
1851 auto SubRegs = TRI.subregs(Reg: CSReg);
1852 // Set the register and all its subregisters.
1853 if (!MRI.def_empty(RegNo: CSReg) || MRI.getUsedPhysRegsMask().test(Idx: CSReg)) {
1854 SavedRegs.set(CSReg);
1855 for (unsigned Reg : SubRegs)
1856 SavedRegs.set(Reg);
1857 }
1858
1859 }
1860
1861 // Unconditionally spill RA and FP only if the function uses a frame
1862 // pointer.
1863 if (hasFP(MF)) {
1864 SavedRegs.set(RAReg);
1865 SavedRegs.set(FPReg);
1866 }
1867 // Mark BP as used if function has dedicated base pointer.
1868 if (hasBP(MF))
1869 SavedRegs.set(RISCVABI::getBPReg());
1870
1871 // When using cm.push/pop we must save X27 if we save X26.
1872 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1873 if (RVFI->isPushable(MF) && SavedRegs.test(Idx: RISCV::X26))
1874 SavedRegs.set(RISCV::X27);
1875
1876 // For Zilsd on RV32, append GPRPair registers to the CSR list. This prevents
1877 // the need to create register sets for each abi which is a lot more complex.
1878 // Don't use Zilsd for callee-saved coalescing if the required alignment
1879 // exceeds the stack alignment or when Zcmp/Xqccmp or save/restore libcalls
1880 // are enabled.
1881 bool UseZilsd = !STI.is64Bit() && STI.hasStdExtZilsd() &&
1882 STI.getZilsdAlign() <= getStackAlign() &&
1883 !RVFI->isPushable(MF) && !RVFI->useSaveRestoreLibCalls(MF);
1884 if (UseZilsd) {
1885 SmallVector<MCPhysReg, 32> NewCSRs;
1886 SmallSet<MCPhysReg, 16> CSRSet;
1887 for (unsigned i = 0; CSRegs[i]; ++i) {
1888 NewCSRs.push_back(Elt: CSRegs[i]);
1889 CSRSet.insert(V: CSRegs[i]);
1890 }
1891
1892 // Append GPRPair registers for pairs where both sub-registers are in CSR
1893 // list. Iterate through all GPRPairs and check if both sub-regs are CSRs.
1894 for (MCPhysReg Pair : RISCV::GPRPairRegClass) {
1895 // Do not append a pair that's already in the CSR list.
1896 if (CSRSet.contains(V: Pair))
1897 continue;
1898 MCRegister EvenReg = TRI.getSubReg(Reg: Pair, Idx: RISCV::sub_gpr_even);
1899 MCRegister OddReg = TRI.getSubReg(Reg: Pair, Idx: RISCV::sub_gpr_odd);
1900 if (CSRSet.contains(V: EvenReg.id()) && CSRSet.contains(V: OddReg.id())) {
1901 NewCSRs.push_back(Elt: Pair);
1902 CSRSet.insert(V: Pair);
1903 }
1904 }
1905
1906 MRI.setCalleeSavedRegs(NewCSRs);
1907 CSRegs = MRI.getCalleeSavedRegs();
1908 }
1909
1910 // Check if all subregisters are marked for saving. If so, set the super
1911 // register bit. For GPRPair, only check sub_gpr_even and sub_gpr_odd, not
1912 // aliases like X8_W or X8_H which are not set in SavedRegs.
1913 for (unsigned i = 0; CSRegs[i]; ++i) {
1914 MCRegister CSReg = CSRegs[i];
1915 bool CombineToSuperReg;
1916 if (RISCV::GPRPairRegClass.contains(Reg: CSReg)) {
1917 MCRegister EvenReg = TRI.getSubReg(Reg: CSReg, Idx: RISCV::sub_gpr_even);
1918 MCRegister OddReg = TRI.getSubReg(Reg: CSReg, Idx: RISCV::sub_gpr_odd);
1919 CombineToSuperReg =
1920 SavedRegs.test(Idx: EvenReg.id()) && SavedRegs.test(Idx: OddReg.id());
1921 // If s0(x8) is used as FP we can't generate load/store pair because it
1922 // breaks the frame chain.
1923 if (hasFP(MF) && CSReg == RISCV::X8_X9)
1924 CombineToSuperReg = false;
1925 } else {
1926 auto SubRegs = TRI.subregs(Reg: CSReg);
1927 CombineToSuperReg =
1928 !SubRegs.empty() && llvm::all_of(Range&: SubRegs, P: [&](unsigned Reg) {
1929 return SavedRegs.test(Idx: Reg);
1930 });
1931 }
1932
1933 if (CombineToSuperReg)
1934 SavedRegs.set(CSReg);
1935 }
1936
1937 // SiFive Preemptible Interrupt Handlers need additional frame entries
1938 createSiFivePreemptibleInterruptFrameEntries(MF, RVFI&: *RVFI);
1939}
1940
1941std::pair<int64_t, Align>
1942RISCVFrameLowering::assignRVVStackObjectOffsets(MachineFunction &MF) const {
1943 MachineFrameInfo &MFI = MF.getFrameInfo();
1944 // Create a buffer of RVV objects to allocate.
1945 SmallVector<int, 8> ObjectsToAllocate;
1946 auto pushRVVObjects = [&](int FIBegin, int FIEnd) {
1947 for (int I = FIBegin, E = FIEnd; I != E; ++I) {
1948 unsigned StackID = MFI.getStackID(ObjectIdx: I);
1949 if (StackID != TargetStackID::ScalableVector)
1950 continue;
1951 if (MFI.isDeadObjectIndex(ObjectIdx: I))
1952 continue;
1953
1954 ObjectsToAllocate.push_back(Elt: I);
1955 }
1956 };
1957 // First push RVV Callee Saved object, then push RVV stack object
1958 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
1959 const auto &RVVCSI = getRVVCalleeSavedInfo(MF, CSI);
1960 if (!RVVCSI.empty())
1961 pushRVVObjects(RVVCSI[0].getFrameIdx(),
1962 RVVCSI[RVVCSI.size() - 1].getFrameIdx() + 1);
1963 pushRVVObjects(0, MFI.getObjectIndexEnd() - RVVCSI.size());
1964
1965 // The minimum alignment is 16 bytes.
1966 Align RVVStackAlign(16);
1967 const auto &ST = MF.getSubtarget<RISCVSubtarget>();
1968
1969 if (!ST.hasVInstructions()) {
1970 assert(ObjectsToAllocate.empty() &&
1971 "Can't allocate scalable-vector objects without V instructions");
1972 return std::make_pair(x: 0, y&: RVVStackAlign);
1973 }
1974
1975 // Allocate all RVV locals and spills
1976 int64_t Offset = 0;
1977 for (int FI : ObjectsToAllocate) {
1978 // ObjectSize in bytes.
1979 int64_t ObjectSize = MFI.getObjectSize(ObjectIdx: FI);
1980 auto ObjectAlign =
1981 std::max(a: Align(RISCV::RVVBytesPerBlock), b: MFI.getObjectAlign(ObjectIdx: FI));
1982 // If the data type is the fractional vector type, reserve one vector
1983 // register for it.
1984 if (ObjectSize < RISCV::RVVBytesPerBlock)
1985 ObjectSize = RISCV::RVVBytesPerBlock;
1986 Offset = alignTo(Size: Offset + ObjectSize, A: ObjectAlign);
1987 MFI.setObjectOffset(ObjectIdx: FI, SPOffset: -Offset);
1988 // Update the maximum alignment of the RVV stack section
1989 RVVStackAlign = std::max(a: RVVStackAlign, b: ObjectAlign);
1990 }
1991
1992 uint64_t StackSize = Offset;
1993
1994 // Ensure the alignment of the RVV stack. Since we want the most-aligned
1995 // object right at the bottom (i.e., any padding at the top of the frame),
1996 // readjust all RVV objects down by the alignment padding.
1997 // Stack size and offsets are multiples of vscale, stack alignment is in
1998 // bytes, we can divide stack alignment by minimum vscale to get a maximum
1999 // stack alignment multiple of vscale.
2000 auto VScale =
2001 std::max<uint64_t>(a: ST.getRealMinVLen() / RISCV::RVVBitsPerBlock, b: 1);
2002 if (auto RVVStackAlignVScale = RVVStackAlign.value() / VScale) {
2003 if (auto AlignmentPadding =
2004 offsetToAlignment(Value: StackSize, Alignment: Align(RVVStackAlignVScale))) {
2005 StackSize += AlignmentPadding;
2006 for (int FI : ObjectsToAllocate)
2007 MFI.setObjectOffset(ObjectIdx: FI, SPOffset: MFI.getObjectOffset(ObjectIdx: FI) - AlignmentPadding);
2008 }
2009 }
2010
2011 return std::make_pair(x&: StackSize, y&: RVVStackAlign);
2012}
2013
2014static unsigned getScavSlotsNumForRVV(MachineFunction &MF) {
2015 // For RVV spill, scalable stack offsets computing requires up to two scratch
2016 // registers
2017 static constexpr unsigned ScavSlotsNumRVVSpillScalableObject = 2;
2018
2019 // For RVV spill, non-scalable stack offsets computing requires up to one
2020 // scratch register.
2021 static constexpr unsigned ScavSlotsNumRVVSpillNonScalableObject = 1;
2022
2023 // ADDI instruction's destination register can be used for computing
2024 // offsets. So Scalable stack offsets require up to one scratch register.
2025 static constexpr unsigned ScavSlotsADDIScalableObject = 1;
2026
2027 static constexpr unsigned MaxScavSlotsNumKnown =
2028 std::max(l: {ScavSlotsADDIScalableObject, ScavSlotsNumRVVSpillScalableObject,
2029 ScavSlotsNumRVVSpillNonScalableObject});
2030
2031 unsigned MaxScavSlotsNum = 0;
2032 if (!MF.getSubtarget<RISCVSubtarget>().hasVInstructions())
2033 return false;
2034 for (const MachineBasicBlock &MBB : MF)
2035 for (const MachineInstr &MI : MBB) {
2036 bool IsRVVSpill = RISCV::isRVVSpill(MI);
2037 for (auto &MO : MI.operands()) {
2038 if (!MO.isFI())
2039 continue;
2040 bool IsScalableVectorID = MF.getFrameInfo().getStackID(ObjectIdx: MO.getIndex()) ==
2041 TargetStackID::ScalableVector;
2042 if (IsRVVSpill) {
2043 MaxScavSlotsNum = std::max(
2044 a: MaxScavSlotsNum, b: IsScalableVectorID
2045 ? ScavSlotsNumRVVSpillScalableObject
2046 : ScavSlotsNumRVVSpillNonScalableObject);
2047 } else if (MI.getOpcode() == RISCV::ADDI && IsScalableVectorID) {
2048 MaxScavSlotsNum =
2049 std::max(a: MaxScavSlotsNum, b: ScavSlotsADDIScalableObject);
2050 }
2051 }
2052 if (MaxScavSlotsNum == MaxScavSlotsNumKnown)
2053 return MaxScavSlotsNumKnown;
2054 }
2055 return MaxScavSlotsNum;
2056}
2057
2058static bool hasRVVFrameObject(const MachineFunction &MF) {
2059 // Originally, the function will scan all the stack objects to check whether
2060 // if there is any scalable vector object on the stack or not. However, it
2061 // causes errors in the register allocator. In issue 53016, it returns false
2062 // before RA because there is no RVV stack objects. After RA, it returns true
2063 // because there are spilling slots for RVV values during RA. It will not
2064 // reserve BP during register allocation and generate BP access in the PEI
2065 // pass due to the inconsistent behavior of the function.
2066 //
2067 // The function is changed to use hasVInstructions() as the return value. It
2068 // is not precise, but it can make the register allocation correct.
2069 //
2070 // FIXME: Find a better way to make the decision or revisit the solution in
2071 // D103622.
2072 //
2073 // Refer to https://github.com/llvm/llvm-project/issues/53016.
2074 return MF.getSubtarget<RISCVSubtarget>().hasVInstructions();
2075}
2076
2077static unsigned estimateFunctionSizeInBytes(const MachineFunction &MF,
2078 const RISCVInstrInfo &TII) {
2079 unsigned FnSize = 0;
2080 for (auto &MBB : MF) {
2081 for (auto &MI : MBB) {
2082 // Far branches over 20-bit offset will be relaxed in branch relaxation
2083 // pass. In the worst case, conditional branches will be relaxed into
2084 // the following instruction sequence. Unconditional branches are
2085 // relaxed in the same way, with the exception that there is no first
2086 // branch instruction.
2087 //
2088 // foo
2089 // bne t5, t6, .rev_cond # `TII->getInstSizeInBytes(MI)` bytes
2090 // sd s11, 0(sp) # 4 bytes, or 2 bytes with Zca
2091 // jump .restore, s11 # 8 bytes
2092 // .rev_cond
2093 // bar
2094 // j .dest_bb # 4 bytes, or 2 bytes with Zca
2095 // .restore:
2096 // ld s11, 0(sp) # 4 bytes, or 2 bytes with Zca
2097 // .dest:
2098 // baz
2099 if (MI.isConditionalBranch())
2100 FnSize += TII.getInstSizeInBytes(MI);
2101 if (MI.isConditionalBranch() || MI.isUnconditionalBranch()) {
2102 if (MF.getSubtarget<RISCVSubtarget>().hasStdExtZca())
2103 FnSize += 2 + 8 + 2 + 2;
2104 else
2105 FnSize += 4 + 8 + 4 + 4;
2106 continue;
2107 }
2108
2109 FnSize += TII.getInstSizeInBytes(MI);
2110 }
2111 }
2112 return FnSize;
2113}
2114
2115void RISCVFrameLowering::processFunctionBeforeFrameFinalized(
2116 MachineFunction &MF, RegScavenger *RS) const {
2117 const RISCVRegisterInfo *RegInfo =
2118 MF.getSubtarget<RISCVSubtarget>().getRegisterInfo();
2119 const RISCVInstrInfo *TII = MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
2120 MachineFrameInfo &MFI = MF.getFrameInfo();
2121 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
2122 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
2123
2124 int64_t RVVStackSize;
2125 Align RVVStackAlign;
2126 std::tie(args&: RVVStackSize, args&: RVVStackAlign) = assignRVVStackObjectOffsets(MF);
2127
2128 RVFI->setRVVStackSize(RVVStackSize);
2129 RVFI->setRVVStackAlign(RVVStackAlign);
2130
2131 if (hasRVVFrameObject(MF)) {
2132 // Ensure the entire stack is aligned to at least the RVV requirement: some
2133 // scalable-vector object alignments are not considered by the
2134 // target-independent code.
2135 MFI.ensureMaxAlignment(Alignment: RVVStackAlign);
2136 }
2137
2138 unsigned ScavSlotsNum = 0;
2139
2140 // estimateStackSize has been observed to under-estimate the final stack
2141 // size, so give ourselves wiggle-room by checking for stack size
2142 // representable an 11-bit signed field rather than 12-bits.
2143 if (!isInt<11>(x: MFI.estimateStackSize(MF)))
2144 ScavSlotsNum = 1;
2145
2146 // Far branches over 20-bit offset require a spill slot for scratch register.
2147 bool IsLargeFunction = !isInt<20>(x: estimateFunctionSizeInBytes(MF, TII: *TII));
2148 if (IsLargeFunction)
2149 ScavSlotsNum = std::max(a: ScavSlotsNum, b: 1u);
2150
2151 // RVV loads & stores have no capacity to hold the immediate address offsets
2152 // so we must always reserve an emergency spill slot if the MachineFunction
2153 // contains any RVV spills.
2154 ScavSlotsNum = std::max(a: ScavSlotsNum, b: getScavSlotsNumForRVV(MF));
2155
2156 for (unsigned I = 0; I < ScavSlotsNum; I++) {
2157 int FI = MFI.CreateSpillStackObject(Size: RegInfo->getSpillSize(RC: *RC),
2158 Alignment: RegInfo->getSpillAlign(RC: *RC));
2159 RS->addScavengingFrameIndex(FI);
2160
2161 if (IsLargeFunction && RVFI->getBranchRelaxationScratchFrameIndex() == -1)
2162 RVFI->setBranchRelaxationScratchFrameIndex(FI);
2163 }
2164
2165 unsigned Size = RVFI->getReservedSpillsSize();
2166 for (const auto &Info : MFI.getCalleeSavedInfo()) {
2167 int FrameIdx = Info.getFrameIdx();
2168 if (FrameIdx < 0 || MFI.getStackID(ObjectIdx: FrameIdx) != TargetStackID::Default)
2169 continue;
2170
2171 Size += MFI.getObjectSize(ObjectIdx: FrameIdx);
2172 }
2173 RVFI->setCalleeSavedStackSize(Size);
2174}
2175
2176// Not preserve stack space within prologue for outgoing variables when the
2177// function contains variable size objects or there are vector objects accessed
2178// by the frame pointer.
2179// Let eliminateCallFramePseudoInstr preserve stack space for it.
2180bool RISCVFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
2181 return !MF.getFrameInfo().hasVarSizedObjects() &&
2182 !(hasFP(MF) && hasRVVFrameObject(MF));
2183}
2184
2185// Eliminate ADJCALLSTACKDOWN, ADJCALLSTACKUP pseudo instructions.
2186MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr(
2187 MachineFunction &MF, MachineBasicBlock &MBB,
2188 MachineBasicBlock::iterator MI) const {
2189 DebugLoc DL = MI->getDebugLoc();
2190
2191 if (!hasReservedCallFrame(MF)) {
2192 // If space has not been reserved for a call frame, ADJCALLSTACKDOWN and
2193 // ADJCALLSTACKUP must be converted to instructions manipulating the stack
2194 // pointer. This is necessary when there is a variable length stack
2195 // allocation (e.g. alloca), which means it's not possible to allocate
2196 // space for outgoing arguments from within the function prologue.
2197 int64_t Amount = MI->getOperand(i: 0).getImm();
2198
2199 if (Amount != 0) {
2200 // Ensure the stack remains aligned after adjustment.
2201 Amount = alignSPAdjust(SPAdj: Amount);
2202
2203 if (MI->getOpcode() == RISCV::ADJCALLSTACKDOWN)
2204 Amount = -Amount;
2205
2206 const RISCVTargetLowering *TLI =
2207 MF.getSubtarget<RISCVSubtarget>().getTargetLowering();
2208 int64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign: getStackAlign());
2209 if (TLI->hasInlineStackProbe(MF) && -Amount >= ProbeSize) {
2210 // When stack probing is enabled, the decrement of SP may need to be
2211 // probed. We can handle both the decrement and the probing in
2212 // allocateStack.
2213 bool DynAllocation =
2214 MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
2215 allocateStack(MBB, MBBI: MI, MF, Offset: -Amount, RealStackSize: -Amount,
2216 EmitCFI: needsDwarfCFI(MF) && !hasFP(MF),
2217 /*NeedProbe=*/true, ProbeSize, DynAllocation,
2218 Flag: MachineInstr::NoFlags);
2219 inlineStackProbe(MF, PrologueMBB&: MBB);
2220 } else {
2221 const RISCVRegisterInfo &RI = *STI.getRegisterInfo();
2222 RI.adjustReg(MBB, II: MI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: Amount),
2223 Flag: MachineInstr::NoFlags, RequiredAlign: getStackAlign());
2224 }
2225 }
2226 }
2227
2228 return MBB.erase(I: MI);
2229}
2230
2231// We would like to split the SP adjustment to reduce prologue/epilogue
2232// as following instructions. In this way, the offset of the callee saved
2233// register could fit in a single store. Supposed that the first sp adjust
2234// amount is 2032.
2235// add sp,sp,-2032
2236// sw ra,2028(sp)
2237// sw s0,2024(sp)
2238// sw s1,2020(sp)
2239// sw s3,2012(sp)
2240// sw s4,2008(sp)
2241// add sp,sp,-64
2242uint64_t
2243RISCVFrameLowering::getFirstSPAdjustAmount(const MachineFunction &MF) const {
2244 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
2245 const MachineFrameInfo &MFI = MF.getFrameInfo();
2246 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
2247 uint64_t StackSize = getStackSizeWithRVVPadding(MF);
2248
2249 // Disable SplitSPAdjust if save-restore libcall, push/pop or QCI interrupts
2250 // are used. The callee-saved registers will be pushed by the save-restore
2251 // libcalls, so we don't have to split the SP adjustment in this case.
2252 if (RVFI->getReservedSpillsSize())
2253 return 0;
2254
2255 // Return the FirstSPAdjustAmount if the StackSize can not fit in a signed
2256 // 12-bit and there exists a callee-saved register needing to be pushed.
2257 if (!isInt<12>(x: StackSize) && (CSI.size() > 0)) {
2258 // FirstSPAdjustAmount is chosen at most as (2048 - StackAlign) because
2259 // 2048 will cause sp = sp + 2048 in the epilogue to be split into multiple
2260 // instructions. Offsets smaller than 2048 can fit in a single load/store
2261 // instruction, and we have to stick with the stack alignment. 2048 has
2262 // 16-byte alignment. The stack alignment for RV32 and RV64 is 16 and for
2263 // RV32E it is 4. So (2048 - StackAlign) will satisfy the stack alignment.
2264 const uint64_t StackAlign = getStackAlign().value();
2265
2266 // Amount of (2048 - StackAlign) will prevent callee saved and restored
2267 // instructions be compressed, so try to adjust the amount to the largest
2268 // offset that stack compression instructions accept when target supports
2269 // compression instructions.
2270 if (STI.hasStdExtZca()) {
2271 // The compression extensions may support the following instructions:
2272 // riscv32: c.lwsp rd, offset[7:2] => 2^(6 + 2)
2273 // c.swsp rs2, offset[7:2] => 2^(6 + 2)
2274 // c.flwsp rd, offset[7:2] => 2^(6 + 2)
2275 // c.fswsp rs2, offset[7:2] => 2^(6 + 2)
2276 // riscv64: c.ldsp rd, offset[8:3] => 2^(6 + 3)
2277 // c.sdsp rs2, offset[8:3] => 2^(6 + 3)
2278 // c.fldsp rd, offset[8:3] => 2^(6 + 3)
2279 // c.fsdsp rs2, offset[8:3] => 2^(6 + 3)
2280 const uint64_t RVCompressLen = STI.getXLen() * 8;
2281 // Compared with amount (2048 - StackAlign), StackSize needs to
2282 // satisfy the following conditions to avoid using more instructions
2283 // to adjust the sp after adjusting the amount, such as
2284 // StackSize meets the condition (StackSize <= 2048 + RVCompressLen),
2285 // case1: Amount is 2048 - StackAlign: use addi + addi to adjust sp.
2286 // case2: Amount is RVCompressLen: use addi + addi to adjust sp.
2287 auto CanCompress = [&](uint64_t CompressLen) -> bool {
2288 if (StackSize <= 2047 + CompressLen ||
2289 (StackSize > 2048 * 2 - StackAlign &&
2290 StackSize <= 2047 * 2 + CompressLen) ||
2291 StackSize > 2048 * 3 - StackAlign)
2292 return true;
2293
2294 return false;
2295 };
2296 // In the epilogue, addi sp, sp, 496 is used to recover the sp and it
2297 // can be compressed(C.ADDI16SP, offset can be [-512, 496]), but
2298 // addi sp, sp, 512 can not be compressed. So try to use 496 first.
2299 const uint64_t ADDI16SPCompressLen = 496;
2300 if (STI.is64Bit() && CanCompress(ADDI16SPCompressLen))
2301 return ADDI16SPCompressLen;
2302 if (CanCompress(RVCompressLen))
2303 return RVCompressLen;
2304 }
2305 return 2048 - StackAlign;
2306 }
2307 return 0;
2308}
2309
2310bool RISCVFrameLowering::assignCalleeSavedSpillSlots(
2311 MachineFunction &MF, const TargetRegisterInfo *TRI,
2312 std::vector<CalleeSavedInfo> &CSI) const {
2313 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
2314 MachineFrameInfo &MFI = MF.getFrameInfo();
2315 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
2316
2317 // Preemptible Interrupts have two additional Callee-save Frame Indexes,
2318 // not tracked by `CSI`.
2319 if (RVFI->isSiFivePreemptibleInterrupt(MF)) {
2320 for (int I = 0; I < 2; ++I) {
2321 int FI = RVFI->getInterruptCSRFrameIndex(Idx: I);
2322 MFI.setIsCalleeSavedObjectIndex(ObjectIdx: FI, IsCalleeSaved: true);
2323 }
2324 }
2325
2326 // Early exit if no callee saved registers are modified!
2327 if (CSI.empty())
2328 return true;
2329
2330 if (RVFI->useQCIInterrupt(MF)) {
2331 RVFI->setQCIInterruptStackSize(QCIInterruptPushAmount);
2332 }
2333
2334 if (RVFI->isPushable(MF)) {
2335 // Determine how many GPRs we need to push and save it to RVFI.
2336 unsigned PushedRegNum = getNumPushPopRegs(CSI);
2337
2338 // `QC.C.MIENTER(.NEST)` will save `ra` and `s0`, so we should only push if
2339 // we want to push more than 2 registers. Otherwise, we should push if we
2340 // want to push more than 0 registers.
2341 unsigned OnlyPushIfMoreThan = RVFI->useQCIInterrupt(MF) ? 2 : 0;
2342 if (PushedRegNum > OnlyPushIfMoreThan) {
2343 RVFI->setRVPushRegs(PushedRegNum);
2344 RVFI->setRVPushStackSize(alignTo(Value: (STI.getXLen() / 8) * PushedRegNum, Align: 16));
2345 }
2346 }
2347
2348 for (auto &CS : CSI) {
2349 MCRegister Reg = CS.getReg();
2350 const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
2351 unsigned Size = RegInfo->getSpillSize(RC: *RC);
2352
2353 if (RVFI->useQCIInterrupt(MF)) {
2354 const auto *FFI = llvm::find_if(Range: FixedCSRFIQCIInterruptMap, P: [&](auto P) {
2355 return P.first == CS.getReg();
2356 });
2357 if (FFI != std::end(arr: FixedCSRFIQCIInterruptMap)) {
2358 int64_t Offset = FFI->second * (int64_t)Size;
2359
2360 int FrameIdx = MFI.CreateFixedSpillStackObject(Size, SPOffset: Offset);
2361 assert(FrameIdx < 0);
2362 CS.setFrameIdx(FrameIdx);
2363 continue;
2364 }
2365 }
2366
2367 if (RVFI->useSaveRestoreLibCalls(MF) || RVFI->isPushable(MF)) {
2368 const auto *FII = llvm::find_if(
2369 Range: FixedCSRFIMap, P: [&](MCPhysReg P) { return P == CS.getReg(); });
2370 unsigned RegNum = std::distance(first: std::begin(arr: FixedCSRFIMap), last: FII);
2371
2372 if (FII != std::end(arr: FixedCSRFIMap)) {
2373 int64_t Offset;
2374 if (RVFI->getPushPopKind(MF) ==
2375 RISCVMachineFunctionInfo::PushPopKind::StdExtZcmp)
2376 Offset = -int64_t(RVFI->getRVPushRegs() - RegNum) * Size;
2377 else
2378 Offset = -int64_t(RegNum + 1) * Size;
2379
2380 if (RVFI->useQCIInterrupt(MF))
2381 Offset -= QCIInterruptPushAmount;
2382
2383 int FrameIdx = MFI.CreateFixedSpillStackObject(Size, SPOffset: Offset);
2384 assert(FrameIdx < 0);
2385 CS.setFrameIdx(FrameIdx);
2386 continue;
2387 }
2388 }
2389
2390 // For GPRPair registers, use 8-byte slots with required alignment by zilsd.
2391 if (!STI.is64Bit() && STI.hasStdExtZilsd() &&
2392 RISCV::GPRPairRegClass.contains(Reg)) {
2393 Align PairAlign = STI.getZilsdAlign();
2394 int FrameIdx = MFI.CreateStackObject(Size: 8, Alignment: PairAlign, isSpillSlot: true);
2395 MFI.setIsCalleeSavedObjectIndex(ObjectIdx: FrameIdx, IsCalleeSaved: true);
2396 CS.setFrameIdx(FrameIdx);
2397 continue;
2398 }
2399
2400 // Not a fixed slot.
2401 Align Alignment = RegInfo->getSpillAlign(RC: *RC);
2402 // We may not be able to satisfy the desired alignment specification of
2403 // the TargetRegisterClass if the stack alignment is smaller. Use the
2404 // min.
2405 Alignment = std::min(a: Alignment, b: getStackAlign());
2406 int FrameIdx = MFI.CreateStackObject(Size, Alignment, isSpillSlot: true);
2407 MFI.setIsCalleeSavedObjectIndex(ObjectIdx: FrameIdx, IsCalleeSaved: true);
2408 CS.setFrameIdx(FrameIdx);
2409 if (RISCVRegisterInfo::isRVVRegClass(RC))
2410 MFI.setStackID(ObjectIdx: FrameIdx, ID: TargetStackID::ScalableVector);
2411 }
2412
2413 if (RVFI->useQCIInterrupt(MF)) {
2414 // Allocate a fixed object that covers the entire QCI stack allocation,
2415 // because there are gaps which are reserved for future use.
2416 MFI.CreateFixedSpillStackObject(
2417 Size: QCIInterruptPushAmount, SPOffset: -static_cast<int64_t>(QCIInterruptPushAmount));
2418 }
2419
2420 if (RVFI->isPushable(MF)) {
2421 int64_t QCIOffset = RVFI->useQCIInterrupt(MF) ? QCIInterruptPushAmount : 0;
2422 // Allocate a fixed object that covers the full push.
2423 if (int64_t PushSize = RVFI->getRVPushStackSize())
2424 MFI.CreateFixedSpillStackObject(Size: PushSize, SPOffset: -PushSize - QCIOffset);
2425 } else if (int LibCallRegs = getLibCallID(MF, CSI) + 1) {
2426 int64_t LibCallFrameSize =
2427 alignTo(Size: (STI.getXLen() / 8) * LibCallRegs, A: getStackAlign());
2428 MFI.CreateFixedSpillStackObject(Size: LibCallFrameSize, SPOffset: -LibCallFrameSize);
2429 }
2430
2431 return true;
2432}
2433
2434bool RISCVFrameLowering::spillCalleeSavedRegisters(
2435 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2436 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
2437 if (CSI.empty())
2438 return true;
2439
2440 MachineFunction *MF = MBB.getParent();
2441 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
2442 DebugLoc DL;
2443 if (MI != MBB.end() && !MI->isDebugInstr())
2444 DL = MI->getDebugLoc();
2445
2446 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2447 if (RVFI->useQCIInterrupt(MF: *MF)) {
2448 // Emit QC.C.MIENTER(.NEST)
2449 BuildMI(
2450 BB&: MBB, I: MI, MIMD: DL,
2451 MCID: TII.get(Opcode: RVFI->getInterruptStackKind(MF: *MF) ==
2452 RISCVMachineFunctionInfo::InterruptStackKind::QCINest
2453 ? RISCV::QC_C_MIENTER_NEST
2454 : RISCV::QC_C_MIENTER))
2455 .setMIFlag(MachineInstr::FrameSetup);
2456
2457 for (auto [Reg, _Offset] : FixedCSRFIQCIInterruptMap)
2458 MBB.addLiveIn(PhysReg: Reg);
2459 }
2460
2461 if (RVFI->isPushable(MF: *MF)) {
2462 // Emit CM.PUSH with base StackAdj & evaluate Push stack
2463 unsigned PushedRegNum = RVFI->getRVPushRegs();
2464 if (PushedRegNum > 0) {
2465 // Use encoded number to represent registers to spill.
2466 unsigned Opcode = getPushOpcode(
2467 Kind: RVFI->getPushPopKind(MF: *MF), UpdateFP: hasFP(MF: *MF) && !RVFI->useQCIInterrupt(MF: *MF));
2468 unsigned RegEnc = RISCVZC::encodeRegListNumRegs(NumRegs: PushedRegNum);
2469 MachineInstrBuilder PushBuilder =
2470 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode))
2471 .setMIFlag(MachineInstr::FrameSetup);
2472 PushBuilder.addImm(Val: RegEnc);
2473 PushBuilder.addImm(Val: 0);
2474
2475 for (unsigned i = 0; i < PushedRegNum; i++)
2476 PushBuilder.addUse(RegNo: FixedCSRFIMap[i], Flags: RegState::Implicit);
2477 }
2478 } else if (const char *SpillLibCall = getSpillLibCallName(MF: *MF, CSI)) {
2479 // Add spill libcall via non-callee-saved register t0.
2480 MachineInstrBuilder NewMI =
2481 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::PseudoCALLReg), DestReg: RISCV::X5)
2482 .addExternalSymbol(FnName: SpillLibCall, TargetFlags: RISCVII::MO_CALL)
2483 .setMIFlag(MachineInstr::FrameSetup)
2484 .addUse(RegNo: RISCV::X2, Flags: RegState::Implicit)
2485 .addDef(RegNo: RISCV::X2, Flags: RegState::ImplicitDefine);
2486
2487 // Add registers spilled as implicit used.
2488 for (auto &CS : CSI)
2489 NewMI.addUse(RegNo: CS.getReg(), Flags: RegState::Implicit);
2490 }
2491
2492 // Manually spill values not spilled by libcall & Push/Pop.
2493 const auto &UnmanagedCSI =
2494 getUnmanagedCSI(MF: *MF, CSI, ReverseOrder: STI.preferAscendingLoadStore());
2495 const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI);
2496
2497 auto storeRegsToStackSlots = [&](decltype(UnmanagedCSI) CSInfo) {
2498 for (auto &CS : CSInfo) {
2499 // Insert the spill to the stack frame.
2500 MCRegister Reg = CS.getReg();
2501 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
2502 TII.storeRegToStackSlot(MBB, MI, SrcReg: Reg, isKill: !MBB.isLiveIn(Reg),
2503 FrameIndex: CS.getFrameIdx(), RC, VReg: Register(),
2504 Flags: MachineInstr::FrameSetup);
2505 }
2506 };
2507 storeRegsToStackSlots(UnmanagedCSI);
2508 storeRegsToStackSlots(RVVCSI);
2509
2510 return true;
2511}
2512
2513static unsigned getCalleeSavedRVVNumRegs(const Register &BaseReg) {
2514 return RISCV::VRRegClass.contains(Reg: BaseReg) ? 1
2515 : RISCV::VRM2RegClass.contains(Reg: BaseReg) ? 2
2516 : RISCV::VRM4RegClass.contains(Reg: BaseReg) ? 4
2517 : 8;
2518}
2519
2520void RISCVFrameLowering::emitCalleeSavedRVVPrologCFI(
2521 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, bool HasFP) const {
2522 MachineFunction *MF = MBB.getParent();
2523 const MachineFrameInfo &MFI = MF->getFrameInfo();
2524 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2525 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
2526
2527 const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI: MFI.getCalleeSavedInfo());
2528 if (RVVCSI.empty())
2529 return;
2530
2531 uint64_t FixedSize = getStackSizeWithRVVPadding(MF: *MF);
2532 if (!HasFP) {
2533 uint64_t ScalarLocalVarSize =
2534 MFI.getStackSize() - RVFI->getCalleeSavedStackSize() -
2535 RVFI->getVarArgsSaveSize() + RVFI->getRVVPadding();
2536 FixedSize -= ScalarLocalVarSize;
2537 }
2538
2539 CFIInstBuilder CFIBuilder(MBB, MI, MachineInstr::FrameSetup);
2540 for (auto &CS : RVVCSI) {
2541 // Insert the spill to the stack frame.
2542 int FI = CS.getFrameIdx();
2543 MCRegister BaseReg = getRVVBaseRegister(TRI, Reg: CS.getReg());
2544 unsigned NumRegs = getCalleeSavedRVVNumRegs(BaseReg: CS.getReg());
2545 for (unsigned i = 0; i < NumRegs; ++i) {
2546 CFIBuilder.insertCFIInst(CFIInst: createDefCFAOffset(
2547 TRI, Reg: BaseReg + i,
2548 Offset: StackOffset::get(Fixed: -FixedSize, Scalable: MFI.getObjectOffset(ObjectIdx: FI) / 8 + i)));
2549 }
2550 }
2551}
2552
2553void RISCVFrameLowering::emitCalleeSavedRVVEpilogCFI(
2554 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const {
2555 MachineFunction *MF = MBB.getParent();
2556 const MachineFrameInfo &MFI = MF->getFrameInfo();
2557 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
2558
2559 CFIInstBuilder CFIHelper(MBB, MI, MachineInstr::FrameDestroy);
2560 const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI: MFI.getCalleeSavedInfo());
2561 for (auto &CS : RVVCSI) {
2562 MCRegister BaseReg = getRVVBaseRegister(TRI, Reg: CS.getReg());
2563 unsigned NumRegs = getCalleeSavedRVVNumRegs(BaseReg: CS.getReg());
2564 for (unsigned i = 0; i < NumRegs; ++i)
2565 CFIHelper.buildRestore(Reg: BaseReg + i);
2566 }
2567}
2568
2569bool RISCVFrameLowering::restoreCalleeSavedRegisters(
2570 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2571 MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
2572 if (CSI.empty())
2573 return true;
2574
2575 MachineFunction *MF = MBB.getParent();
2576 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
2577 DebugLoc DL;
2578 if (MI != MBB.end() && !MI->isDebugInstr())
2579 DL = MI->getDebugLoc();
2580
2581 // Manually restore values not restored by libcall & Push/Pop.
2582 // Reverse the restore order in epilog. In addition, the return
2583 // address will be restored first in the epilogue. It increases
2584 // the opportunity to avoid the load-to-use data hazard between
2585 // loading RA and return by RA. loadRegFromStackSlot can insert
2586 // multiple instructions.
2587 const auto &UnmanagedCSI =
2588 getUnmanagedCSI(MF: *MF, CSI, ReverseOrder: STI.preferAscendingLoadStore());
2589 const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI);
2590
2591 auto loadRegFromStackSlot = [&](decltype(UnmanagedCSI) CSInfo) {
2592 for (auto &CS : CSInfo) {
2593 MCRegister Reg = CS.getReg();
2594 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
2595 TII.loadRegFromStackSlot(MBB, MI, DestReg: Reg, FrameIndex: CS.getFrameIdx(), RC, VReg: Register(),
2596 SubReg: RISCV::NoSubRegister,
2597 Flags: MachineInstr::FrameDestroy);
2598 assert(MI != MBB.begin() &&
2599 "loadRegFromStackSlot didn't insert any code!");
2600 }
2601 };
2602 loadRegFromStackSlot(RVVCSI);
2603 loadRegFromStackSlot(UnmanagedCSI);
2604
2605 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2606 if (RVFI->useQCIInterrupt(MF: *MF)) {
2607 // Don't emit anything here because restoration is handled by
2608 // QC.C.MILEAVERET which we already inserted to return.
2609 assert(MI->getOpcode() == RISCV::QC_C_MILEAVERET &&
2610 "Unexpected QCI Interrupt Return Instruction");
2611 }
2612
2613 if (RVFI->isPushable(MF: *MF)) {
2614 unsigned PushedRegNum = RVFI->getRVPushRegs();
2615 if (PushedRegNum > 0) {
2616 unsigned Opcode = getPopOpcode(Kind: RVFI->getPushPopKind(MF: *MF));
2617 unsigned RegEnc = RISCVZC::encodeRegListNumRegs(NumRegs: PushedRegNum);
2618 MachineInstrBuilder PopBuilder =
2619 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode))
2620 .setMIFlag(MachineInstr::FrameDestroy);
2621 // Use encoded number to represent registers to restore.
2622 PopBuilder.addImm(Val: RegEnc);
2623 PopBuilder.addImm(Val: 0);
2624
2625 for (unsigned i = 0; i < RVFI->getRVPushRegs(); i++)
2626 PopBuilder.addDef(RegNo: FixedCSRFIMap[i], Flags: RegState::ImplicitDefine);
2627 }
2628 } else if (const char *RestoreLibCall = getRestoreLibCallName(MF: *MF, CSI)) {
2629 // Add restore libcall via tail call.
2630 MachineInstrBuilder NewMI =
2631 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::PseudoTAIL))
2632 .addExternalSymbol(FnName: RestoreLibCall, TargetFlags: RISCVII::MO_CALL)
2633 .setMIFlag(MachineInstr::FrameDestroy)
2634 .addDef(RegNo: RISCV::X2, Flags: RegState::ImplicitDefine);
2635
2636 // Add registers restored as implicit defined.
2637 for (auto &CS : CSI)
2638 NewMI.addDef(RegNo: CS.getReg(), Flags: RegState::ImplicitDefine);
2639
2640 // Remove trailing returns, since the terminator is now a tail call to the
2641 // restore function.
2642 if (MI != MBB.end() && MI->getOpcode() == RISCV::PseudoRET) {
2643 NewMI.getInstr()->copyImplicitOps(MF&: *MF, MI: *MI);
2644 MI->eraseFromParent();
2645 }
2646 }
2647 return true;
2648}
2649
2650bool RISCVFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
2651 // Keep the conventional code flow when not optimizing.
2652 if (MF.getFunction().hasOptNone())
2653 return false;
2654
2655 // QCI and SiFive CLIC interrupt entry sequences must precede all handler
2656 // code.
2657 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
2658 if (RVFI->useQCIInterrupt(MF) || RVFI->useSiFiveInterrupt(MF))
2659 return false;
2660
2661 return true;
2662}
2663
2664bool RISCVFrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const {
2665 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
2666 const MachineFunction *MF = MBB.getParent();
2667 const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2668
2669 // Make sure VTYPE and VL are not live-in since we will use vsetvli in the
2670 // prologue to get the VLEN, and that will clobber these registers.
2671 //
2672 // We may do also check the stack contains objects with scalable vector type,
2673 // but this will require iterating over all the stack objects, but this may
2674 // not worth since the situation is rare, we could do further check in future
2675 // if we find it is necessary.
2676 if (STI.preferVsetvliOverReadVLENB() &&
2677 (MBB.isLiveIn(Reg: RISCV::VTYPE) || MBB.isLiveIn(Reg: RISCV::VL)))
2678 return false;
2679
2680 if (!RVFI->useSaveRestoreLibCalls(MF: *MF))
2681 return true;
2682
2683 // Inserting a call to a __riscv_save libcall requires the use of the register
2684 // t0 (X5) to hold the return address. Therefore if this register is already
2685 // used we can't insert the call.
2686
2687 RegScavenger RS;
2688 RS.enterBasicBlock(MBB&: *TmpMBB);
2689 return !RS.isRegUsed(Reg: RISCV::X5);
2690}
2691
2692bool RISCVFrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
2693 const MachineFunction *MF = MBB.getParent();
2694 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
2695 const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2696
2697 if (!RVFI->useSaveRestoreLibCalls(MF: *MF))
2698 return true;
2699
2700 // Using the __riscv_restore libcalls to restore CSRs requires a tail call.
2701 // This means if we still need to continue executing code within this function
2702 // the restore cannot take place in this basic block.
2703
2704 if (MBB.succ_size() > 1)
2705 return false;
2706
2707 MachineBasicBlock *SuccMBB =
2708 MBB.succ_empty() ? TmpMBB->getFallThrough() : *MBB.succ_begin();
2709
2710 // Doing a tail call should be safe if there are no successors, because either
2711 // we have a returning block or the end of the block is unreachable, so the
2712 // restore will be eliminated regardless.
2713 if (!SuccMBB)
2714 return true;
2715
2716 // The successor can only contain a return and debug instructions, since we
2717 // would effectively replace it with our own tail return at the end of this
2718 // block. The debug instructions would not execute on the tail-return path.
2719 return SuccMBB->isReturnBlock() &&
2720 llvm::count_if(Range: SuccMBB->instrs(), P: [](const MachineInstr &MI) {
2721 return !MI.isDebugInstr();
2722 }) == 1;
2723}
2724
2725bool RISCVFrameLowering::isSupportedStackID(TargetStackID::Value ID) const {
2726 switch (ID) {
2727 case TargetStackID::Default:
2728 case TargetStackID::ScalableVector:
2729 return true;
2730 case TargetStackID::NoAlloc:
2731 case TargetStackID::SGPRSpill:
2732 case TargetStackID::WasmLocal:
2733 case TargetStackID::ScalablePredicateVector:
2734 case TargetStackID::AvrAlign:
2735 return false;
2736 }
2737 llvm_unreachable("Invalid TargetStackID::Value");
2738}
2739
2740TargetStackID::Value RISCVFrameLowering::getStackIDForScalableVectors() const {
2741 return TargetStackID::ScalableVector;
2742}
2743
2744// Synthesize the probe loop.
2745static void emitStackProbeInline(MachineBasicBlock::iterator MBBI, DebugLoc DL,
2746 Register TargetReg, Register ScratchReg,
2747 bool IsRVV) {
2748 assert(TargetReg != RISCV::X2 && "New top of stack cannot already be in SP");
2749 assert(ScratchReg != RISCV::X2 && "Scratch register cannot be SP");
2750 assert(TargetReg != ScratchReg && "Target and scratch must be different");
2751
2752 MachineBasicBlock &MBB = *MBBI->getParent();
2753 MachineFunction &MF = *MBB.getParent();
2754
2755 auto &Subtarget = MF.getSubtarget<RISCVSubtarget>();
2756 const RISCVInstrInfo *TII = Subtarget.getInstrInfo();
2757 bool IsRV64 = Subtarget.is64Bit();
2758 Align StackAlign = Subtarget.getFrameLowering()->getStackAlign();
2759 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
2760 uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
2761
2762 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB.getIterator());
2763 MachineBasicBlock *LoopTestMBB =
2764 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
2765 MF.insert(MBBI: MBBInsertPoint, MBB: LoopTestMBB);
2766 MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
2767 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
2768 MachineInstr::MIFlag Flags = MachineInstr::FrameSetup;
2769
2770 // ScratchReg = ProbeSize
2771 TII->movImm(MBB, MBBI, DL, DstReg: ScratchReg, Val: ProbeSize, Flag: Flags);
2772
2773 // LoopTest:
2774 // SUB SP, SP, ProbeSize
2775 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::SUB), DestReg: SPReg)
2776 .addReg(RegNo: SPReg)
2777 .addReg(RegNo: ScratchReg)
2778 .setMIFlags(Flags);
2779
2780 // s[d|w] zero, 0(sp)
2781 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL,
2782 MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
2783 .addReg(RegNo: RISCV::X0)
2784 .addReg(RegNo: SPReg)
2785 .addImm(Val: 0)
2786 .setMIFlags(Flags);
2787
2788 if (IsRVV) {
2789 // SUB TargetReg, TargetReg, ProbeSize
2790 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::SUB),
2791 DestReg: TargetReg)
2792 .addReg(RegNo: TargetReg)
2793 .addReg(RegNo: ScratchReg)
2794 .setMIFlags(Flags);
2795
2796 // BGE TargetReg, ProbeSize, LoopTest
2797 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::BGE))
2798 .addReg(RegNo: TargetReg)
2799 .addReg(RegNo: ScratchReg)
2800 .addMBB(MBB: LoopTestMBB)
2801 .setMIFlags(Flags);
2802
2803 } else {
2804 // BNE SP, TargetReg, LoopTest
2805 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
2806 .addReg(RegNo: SPReg)
2807 .addReg(RegNo: TargetReg)
2808 .addMBB(MBB: LoopTestMBB)
2809 .setMIFlags(Flags);
2810 }
2811
2812 ExitMBB->splice(Where: ExitMBB->end(), Other: &MBB, From: std::next(x: MBBI), To: MBB.end());
2813 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
2814
2815 LoopTestMBB->addSuccessor(Succ: ExitMBB);
2816 LoopTestMBB->addSuccessor(Succ: LoopTestMBB);
2817 MBB.addSuccessor(Succ: LoopTestMBB);
2818 // Update liveins.
2819 fullyRecomputeLiveIns(MBBs: {ExitMBB, LoopTestMBB});
2820}
2821
2822void RISCVFrameLowering::inlineStackProbe(MachineFunction &MF,
2823 MachineBasicBlock &MBB) const {
2824 // Get the instructions that need to be replaced. We emit at most two of
2825 // these. Remember them in order to avoid complications coming from the need
2826 // to traverse the block while potentially creating more blocks.
2827 SmallVector<MachineInstr *, 4> ToReplace;
2828 for (MachineInstr &MI : MBB) {
2829 unsigned Opc = MI.getOpcode();
2830 if (Opc == RISCV::PROBED_STACKALLOC ||
2831 Opc == RISCV::PROBED_STACKALLOC_RVV) {
2832 ToReplace.push_back(Elt: &MI);
2833 }
2834 }
2835
2836 for (MachineInstr *MI : ToReplace) {
2837 if (MI->getOpcode() == RISCV::PROBED_STACKALLOC ||
2838 MI->getOpcode() == RISCV::PROBED_STACKALLOC_RVV) {
2839 MachineBasicBlock::iterator MBBI = MI->getIterator();
2840 DebugLoc DL = MBB.findDebugLoc(MBBI);
2841 Register TargetReg = MI->getOperand(i: 0).getReg();
2842
2843 Register ScratchReg =
2844 findScratchNonCalleeSaveRegister(MBB: &MBB, PreferredReg: RISCV::X7, DontUseReg: TargetReg);
2845
2846 assert(ScratchReg.isValid() &&
2847 "No available scratch register for stack probe loop");
2848
2849 emitStackProbeInline(MBBI, DL, TargetReg, ScratchReg,
2850 IsRVV: (MI->getOpcode() == RISCV::PROBED_STACKALLOC_RVV));
2851 MBBI->eraseFromParent();
2852 }
2853 }
2854}
2855
2856int RISCVFrameLowering::getInitialCFAOffset(const MachineFunction &MF) const {
2857 return 0;
2858}
2859
2860Register
2861RISCVFrameLowering::getInitialCFARegister(const MachineFunction &MF) const {
2862 return RISCV::X2;
2863}
2864
2865// On 64-bit systems the fixed stack can hold INT64_MAX bytes, since
2866// stack-offset calculation is done in 2s-complement.
2867// NOTE: In theory a register can hold any 64-bit number, so this constraint
2868// might be relaxed to UINT64_MAX in the future, if anyone actually needs
2869// that.
2870uint64_t RISCVFrameLowering::getStackThreshold() const {
2871 return STI.is64Bit() ? INT64_MAX : UINT32_MAX;
2872}
2873