1//===-- RISCVFrameLowering.cpp - RISC-V Frame Information -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the RISC-V implementation of TargetFrameLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVFrameLowering.h"
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "RISCVMachineFunctionInfo.h"
16#include "RISCVSubtarget.h"
17#include "llvm/BinaryFormat/Dwarf.h"
18#include "llvm/CodeGen/CFIInstBuilder.h"
19#include "llvm/CodeGen/LivePhysRegs.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/MachineRegisterInfo.h"
24#include "llvm/CodeGen/RegisterScavenging.h"
25#include "llvm/IR/DiagnosticInfo.h"
26#include "llvm/MC/MCDwarf.h"
27#include "llvm/Support/LEB128.h"
28
29#include <algorithm>
30
31#define DEBUG_TYPE "riscv-frame"
32
33using namespace llvm;
34
35static Align getABIStackAlignment(RISCVABI::ABI ABI) {
36 if (ABI == RISCVABI::ABI_ILP32E)
37 return Align(4);
38 if (ABI == RISCVABI::ABI_LP64E)
39 return Align(8);
40 return Align(16);
41}
42
43RISCVFrameLowering::RISCVFrameLowering(const RISCVSubtarget &STI)
44 : TargetFrameLowering(
45 StackGrowsDown, getABIStackAlignment(ABI: STI.getTargetABI()),
46 /*LocalAreaOffset=*/0,
47 /*TransientStackAlignment=*/getABIStackAlignment(ABI: STI.getTargetABI())),
48 STI(STI) {}
49
50// The register used to hold the frame pointer.
51static constexpr MCPhysReg FPReg = RISCV::X8;
52
53// The register used to hold the stack pointer.
54static constexpr MCPhysReg SPReg = RISCV::X2;
55
56// The register used to hold the return address.
57static constexpr MCPhysReg RAReg = RISCV::X1;
58
59// LIst of CSRs that are given a fixed location by save/restore libcalls or
60// Zcmp/Xqccmp Push/Pop. The order in this table indicates the order the
61// registers are saved on the stack. Zcmp uses the reverse order of save/restore
62// and Xqccmp on the stack, but this is handled when offsets are calculated.
63static const MCPhysReg FixedCSRFIMap[] = {
64 /*ra*/ RAReg, /*s0*/ FPReg, /*s1*/ RISCV::X9,
65 /*s2*/ RISCV::X18, /*s3*/ RISCV::X19, /*s4*/ RISCV::X20,
66 /*s5*/ RISCV::X21, /*s6*/ RISCV::X22, /*s7*/ RISCV::X23,
67 /*s8*/ RISCV::X24, /*s9*/ RISCV::X25, /*s10*/ RISCV::X26,
68 /*s11*/ RISCV::X27};
69
70// The number of stack bytes allocated by `QC.C.MIENTER(.NEST)` and popped by
71// `QC.C.MILEAVERET`.
72static constexpr uint64_t QCIInterruptPushAmount = 96;
73
74static const std::pair<MCPhysReg, int8_t> FixedCSRFIQCIInterruptMap[] = {
75 /* -1 is a gap for mepc/mnepc */
76 {/*fp*/ FPReg, -2},
77 /* -3 is a gap for qc.mcause */
78 {/*ra*/ RAReg, -4},
79 /* -5 is reserved */
80 {/*t0*/ RISCV::X5, -6},
81 {/*t1*/ RISCV::X6, -7},
82 {/*t2*/ RISCV::X7, -8},
83 {/*a0*/ RISCV::X10, -9},
84 {/*a1*/ RISCV::X11, -10},
85 {/*a2*/ RISCV::X12, -11},
86 {/*a3*/ RISCV::X13, -12},
87 {/*a4*/ RISCV::X14, -13},
88 {/*a5*/ RISCV::X15, -14},
89 {/*a6*/ RISCV::X16, -15},
90 {/*a7*/ RISCV::X17, -16},
91 {/*t3*/ RISCV::X28, -17},
92 {/*t4*/ RISCV::X29, -18},
93 {/*t5*/ RISCV::X30, -19},
94 {/*t6*/ RISCV::X31, -20},
95 /* -21, -22, -23, -24 are reserved */
96};
97
98// For now we use x3, a.k.a gp, as pointer to shadow call stack.
99// User should not use x3 in their asm.
100static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
101 MachineBasicBlock::iterator MI,
102 const DebugLoc &DL) {
103 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
104 bool HasHWShadowStack = MF.getFunction().hasFnAttribute(Kind: "hw-shadow-stack") &&
105 STI.hasStdExtZicfiss();
106 bool HasSWShadowStack =
107 MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack);
108 if (!HasHWShadowStack && !HasSWShadowStack)
109 return;
110
111 const llvm::RISCVRegisterInfo *TRI = STI.getRegisterInfo();
112
113 // Do not save RA to the SCS if it's not saved to the regular stack,
114 // i.e. RA is not at risk of being overwritten.
115 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
116 if (llvm::none_of(
117 Range&: CSI, P: [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
118 return;
119
120 const RISCVInstrInfo *TII = STI.getInstrInfo();
121 if (HasHWShadowStack) {
122 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SSPUSH)).addReg(RegNo: RAReg);
123 return;
124 }
125
126 Register SCSPReg = RISCVABI::getSCSPReg();
127
128 bool IsRV64 = STI.is64Bit();
129 int64_t SlotSize = STI.getXLen() / 8;
130 // Store return address to shadow call stack
131 // addi gp, gp, [4|8]
132 // s[w|d] ra, -[4|8](gp)
133 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI))
134 .addReg(RegNo: SCSPReg, flags: RegState::Define)
135 .addReg(RegNo: SCSPReg)
136 .addImm(Val: SlotSize)
137 .setMIFlag(MachineInstr::FrameSetup);
138 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
139 .addReg(RegNo: RAReg)
140 .addReg(RegNo: SCSPReg)
141 .addImm(Val: -SlotSize)
142 .setMIFlag(MachineInstr::FrameSetup);
143
144 // Emit a CFI instruction that causes SlotSize to be subtracted from the value
145 // of the shadow stack pointer when unwinding past this frame.
146 char DwarfSCSReg = TRI->getDwarfRegNum(RegNum: SCSPReg, /*IsEH*/ isEH: true);
147 assert(DwarfSCSReg < 32 && "SCS Register should be < 32 (X3).");
148
149 char Offset = static_cast<char>(-SlotSize) & 0x7f;
150 const char CFIInst[] = {
151 dwarf::DW_CFA_val_expression,
152 DwarfSCSReg, // register
153 2, // length
154 static_cast<char>(unsigned(dwarf::DW_OP_breg0 + DwarfSCSReg)),
155 Offset, // addend (sleb128)
156 };
157
158 CFIInstBuilder(MBB, MI, MachineInstr::FrameSetup)
159 .buildEscape(Bytes: StringRef(CFIInst, sizeof(CFIInst)));
160}
161
162static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB,
163 MachineBasicBlock::iterator MI,
164 const DebugLoc &DL) {
165 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
166 bool HasHWShadowStack = MF.getFunction().hasFnAttribute(Kind: "hw-shadow-stack") &&
167 STI.hasStdExtZicfiss();
168 bool HasSWShadowStack =
169 MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack);
170 if (!HasHWShadowStack && !HasSWShadowStack)
171 return;
172
173 // See emitSCSPrologue() above.
174 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
175 if (llvm::none_of(
176 Range&: CSI, P: [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
177 return;
178
179 const RISCVInstrInfo *TII = STI.getInstrInfo();
180 if (HasHWShadowStack) {
181 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SSPOPCHK)).addReg(RegNo: RAReg);
182 return;
183 }
184
185 Register SCSPReg = RISCVABI::getSCSPReg();
186
187 bool IsRV64 = STI.is64Bit();
188 int64_t SlotSize = STI.getXLen() / 8;
189 // Load return address from shadow call stack
190 // l[w|d] ra, -[4|8](gp)
191 // addi gp, gp, -[4|8]
192 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::LD : RISCV::LW))
193 .addReg(RegNo: RAReg, flags: RegState::Define)
194 .addReg(RegNo: SCSPReg)
195 .addImm(Val: -SlotSize)
196 .setMIFlag(MachineInstr::FrameDestroy);
197 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI))
198 .addReg(RegNo: SCSPReg, flags: RegState::Define)
199 .addReg(RegNo: SCSPReg)
200 .addImm(Val: -SlotSize)
201 .setMIFlag(MachineInstr::FrameDestroy);
202 // Restore the SCS pointer
203 CFIInstBuilder(MBB, MI, MachineInstr::FrameDestroy).buildRestore(Reg: SCSPReg);
204}
205
206// Insert instruction to swap mscratchsw with sp
207static void emitSiFiveCLICStackSwap(MachineFunction &MF, MachineBasicBlock &MBB,
208 MachineBasicBlock::iterator MBBI,
209 const DebugLoc &DL) {
210 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
211
212 if (!RVFI->isSiFiveStackSwapInterrupt(MF))
213 return;
214
215 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
216 const RISCVInstrInfo *TII = STI.getInstrInfo();
217
218 assert(STI.hasVendorXSfmclic() && "Stack Swapping Requires XSfmclic");
219
220 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRW))
221 .addReg(RegNo: SPReg, flags: RegState::Define)
222 .addImm(Val: RISCVSysReg::sf_mscratchcsw)
223 .addReg(RegNo: SPReg, flags: RegState::Kill)
224 .setMIFlag(MachineInstr::FrameSetup);
225
226 // FIXME: CFI Information for this swap.
227}
228
229static void
230createSiFivePreemptibleInterruptFrameEntries(MachineFunction &MF,
231 RISCVMachineFunctionInfo &RVFI) {
232 if (!RVFI.isSiFivePreemptibleInterrupt(MF))
233 return;
234
235 const TargetRegisterClass &RC = RISCV::GPRRegClass;
236 const TargetRegisterInfo &TRI =
237 *MF.getSubtarget<RISCVSubtarget>().getRegisterInfo();
238 MachineFrameInfo &MFI = MF.getFrameInfo();
239
240 // Create two frame objects for spilling X8 and X9, which will be done in
241 // `emitSiFiveCLICPreemptibleSaves`. This is in addition to any other stack
242 // objects we might have for X8 and X9, as they might be saved twice.
243 for (int I = 0; I < 2; ++I) {
244 int FI = MFI.CreateStackObject(Size: TRI.getSpillSize(RC), Alignment: TRI.getSpillAlign(RC),
245 isSpillSlot: true);
246 RVFI.pushInterruptCSRFrameIndex(FI);
247 }
248}
249
250static void emitSiFiveCLICPreemptibleSaves(MachineFunction &MF,
251 MachineBasicBlock &MBB,
252 MachineBasicBlock::iterator MBBI,
253 const DebugLoc &DL) {
254 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
255
256 if (!RVFI->isSiFivePreemptibleInterrupt(MF))
257 return;
258
259 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
260 const RISCVInstrInfo *TII = STI.getInstrInfo();
261
262 // FIXME: CFI Information here is nonexistent/wrong.
263
264 // X8 and X9 might be stored into the stack twice, initially into the
265 // `interruptCSRFrameIndex` here, and then maybe again into their CSI frame
266 // index.
267 //
268 // This is done instead of telling the register allocator that we need two
269 // VRegs to store the value of `mcause` and `mepc` through the instruction,
270 // which affects other passes.
271 TII->storeRegToStackSlot(MBB, MBBI, SrcReg: RISCV::X8, /* IsKill=*/true,
272 FrameIndex: RVFI->getInterruptCSRFrameIndex(Idx: 0),
273 RC: &RISCV::GPRRegClass, TRI: STI.getRegisterInfo(),
274 VReg: Register(), Flags: MachineInstr::FrameSetup);
275 TII->storeRegToStackSlot(MBB, MBBI, SrcReg: RISCV::X9, /* IsKill=*/true,
276 FrameIndex: RVFI->getInterruptCSRFrameIndex(Idx: 1),
277 RC: &RISCV::GPRRegClass, TRI: STI.getRegisterInfo(),
278 VReg: Register(), Flags: MachineInstr::FrameSetup);
279
280 // Put `mcause` into X8 (s0), and `mepc` into X9 (s1). If either of these are
281 // used in the function, then they will appear in `getUnmanagedCSI` and will
282 // be saved again.
283 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS))
284 .addReg(RegNo: RISCV::X8, flags: RegState::Define)
285 .addImm(Val: RISCVSysReg::mcause)
286 .addReg(RegNo: RISCV::X0)
287 .setMIFlag(MachineInstr::FrameSetup);
288 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS))
289 .addReg(RegNo: RISCV::X9, flags: RegState::Define)
290 .addImm(Val: RISCVSysReg::mepc)
291 .addReg(RegNo: RISCV::X0)
292 .setMIFlag(MachineInstr::FrameSetup);
293
294 // Enable interrupts.
295 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRSI))
296 .addReg(RegNo: RISCV::X0, flags: RegState::Define)
297 .addImm(Val: RISCVSysReg::mstatus)
298 .addImm(Val: 8)
299 .setMIFlag(MachineInstr::FrameSetup);
300}
301
302static void emitSiFiveCLICPreemptibleRestores(MachineFunction &MF,
303 MachineBasicBlock &MBB,
304 MachineBasicBlock::iterator MBBI,
305 const DebugLoc &DL) {
306 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
307
308 if (!RVFI->isSiFivePreemptibleInterrupt(MF))
309 return;
310
311 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
312 const RISCVInstrInfo *TII = STI.getInstrInfo();
313
314 // FIXME: CFI Information here is nonexistent/wrong.
315
316 // Disable interrupts.
317 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRCI))
318 .addReg(RegNo: RISCV::X0, flags: RegState::Define)
319 .addImm(Val: RISCVSysReg::mstatus)
320 .addImm(Val: 8)
321 .setMIFlag(MachineInstr::FrameSetup);
322
323 // Restore `mepc` from x9 (s1), and `mcause` from x8 (s0). If either were used
324 // in the function, they have already been restored once, so now have the
325 // value stored in `emitSiFiveCLICPreemptibleSaves`.
326 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRW))
327 .addReg(RegNo: RISCV::X0, flags: RegState::Define)
328 .addImm(Val: RISCVSysReg::mepc)
329 .addReg(RegNo: RISCV::X9, flags: RegState::Kill)
330 .setMIFlag(MachineInstr::FrameSetup);
331 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRW))
332 .addReg(RegNo: RISCV::X0, flags: RegState::Define)
333 .addImm(Val: RISCVSysReg::mcause)
334 .addReg(RegNo: RISCV::X8, flags: RegState::Kill)
335 .setMIFlag(MachineInstr::FrameSetup);
336
337 // X8 and X9 need to be restored to their values on function entry, which we
338 // saved onto the stack in `emitSiFiveCLICPreemptibleSaves`.
339 TII->loadRegFromStackSlot(MBB, MBBI, DstReg: RISCV::X9,
340 FrameIndex: RVFI->getInterruptCSRFrameIndex(Idx: 1),
341 RC: &RISCV::GPRRegClass, TRI: STI.getRegisterInfo(),
342 VReg: Register(), Flags: MachineInstr::FrameSetup);
343 TII->loadRegFromStackSlot(MBB, MBBI, DstReg: RISCV::X8,
344 FrameIndex: RVFI->getInterruptCSRFrameIndex(Idx: 0),
345 RC: &RISCV::GPRRegClass, TRI: STI.getRegisterInfo(),
346 VReg: Register(), Flags: MachineInstr::FrameSetup);
347}
348
349// Get the ID of the libcall used for spilling and restoring callee saved
350// registers. The ID is representative of the number of registers saved or
351// restored by the libcall, except it is zero-indexed - ID 0 corresponds to a
352// single register.
353static int getLibCallID(const MachineFunction &MF,
354 const std::vector<CalleeSavedInfo> &CSI) {
355 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
356
357 if (CSI.empty() || !RVFI->useSaveRestoreLibCalls(MF))
358 return -1;
359
360 MCRegister MaxReg;
361 for (auto &CS : CSI)
362 // assignCalleeSavedSpillSlots assigns negative frame indexes to
363 // registers which can be saved by libcall.
364 if (CS.getFrameIdx() < 0)
365 MaxReg = std::max(a: MaxReg.id(), b: CS.getReg().id());
366
367 if (!MaxReg)
368 return -1;
369
370 switch (MaxReg.id()) {
371 default:
372 llvm_unreachable("Something has gone wrong!");
373 // clang-format off
374 case /*s11*/ RISCV::X27: return 12;
375 case /*s10*/ RISCV::X26: return 11;
376 case /*s9*/ RISCV::X25: return 10;
377 case /*s8*/ RISCV::X24: return 9;
378 case /*s7*/ RISCV::X23: return 8;
379 case /*s6*/ RISCV::X22: return 7;
380 case /*s5*/ RISCV::X21: return 6;
381 case /*s4*/ RISCV::X20: return 5;
382 case /*s3*/ RISCV::X19: return 4;
383 case /*s2*/ RISCV::X18: return 3;
384 case /*s1*/ RISCV::X9: return 2;
385 case /*s0*/ FPReg: return 1;
386 case /*ra*/ RAReg: return 0;
387 // clang-format on
388 }
389}
390
391// Get the name of the libcall used for spilling callee saved registers.
392// If this function will not use save/restore libcalls, then return a nullptr.
393static const char *
394getSpillLibCallName(const MachineFunction &MF,
395 const std::vector<CalleeSavedInfo> &CSI) {
396 static const char *const SpillLibCalls[] = {
397 "__riscv_save_0",
398 "__riscv_save_1",
399 "__riscv_save_2",
400 "__riscv_save_3",
401 "__riscv_save_4",
402 "__riscv_save_5",
403 "__riscv_save_6",
404 "__riscv_save_7",
405 "__riscv_save_8",
406 "__riscv_save_9",
407 "__riscv_save_10",
408 "__riscv_save_11",
409 "__riscv_save_12"
410 };
411
412 int LibCallID = getLibCallID(MF, CSI);
413 if (LibCallID == -1)
414 return nullptr;
415 return SpillLibCalls[LibCallID];
416}
417
418// Get the name of the libcall used for restoring callee saved registers.
419// If this function will not use save/restore libcalls, then return a nullptr.
420static const char *
421getRestoreLibCallName(const MachineFunction &MF,
422 const std::vector<CalleeSavedInfo> &CSI) {
423 static const char *const RestoreLibCalls[] = {
424 "__riscv_restore_0",
425 "__riscv_restore_1",
426 "__riscv_restore_2",
427 "__riscv_restore_3",
428 "__riscv_restore_4",
429 "__riscv_restore_5",
430 "__riscv_restore_6",
431 "__riscv_restore_7",
432 "__riscv_restore_8",
433 "__riscv_restore_9",
434 "__riscv_restore_10",
435 "__riscv_restore_11",
436 "__riscv_restore_12"
437 };
438
439 int LibCallID = getLibCallID(MF, CSI);
440 if (LibCallID == -1)
441 return nullptr;
442 return RestoreLibCalls[LibCallID];
443}
444
445// Get the max reg of Push/Pop for restoring callee saved registers.
446static unsigned getNumPushPopRegs(const std::vector<CalleeSavedInfo> &CSI) {
447 unsigned NumPushPopRegs = 0;
448 for (auto &CS : CSI) {
449 auto *FII = llvm::find_if(Range: FixedCSRFIMap,
450 P: [&](MCPhysReg P) { return P == CS.getReg(); });
451 if (FII != std::end(arr: FixedCSRFIMap)) {
452 unsigned RegNum = std::distance(first: std::begin(arr: FixedCSRFIMap), last: FII);
453 NumPushPopRegs = std::max(a: NumPushPopRegs, b: RegNum + 1);
454 }
455 }
456 assert(NumPushPopRegs != 12 && "x26 requires x27 to also be pushed");
457 return NumPushPopRegs;
458}
459
460// Return true if the specified function should have a dedicated frame
461// pointer register. This is true if frame pointer elimination is
462// disabled, if it needs dynamic stack realignment, if the function has
463// variable sized allocas, or if the frame address is taken.
464bool RISCVFrameLowering::hasFPImpl(const MachineFunction &MF) const {
465 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
466
467 const MachineFrameInfo &MFI = MF.getFrameInfo();
468 return MF.getTarget().Options.DisableFramePointerElim(MF) ||
469 RegInfo->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
470 MFI.isFrameAddressTaken();
471}
472
473bool RISCVFrameLowering::hasBP(const MachineFunction &MF) const {
474 const MachineFrameInfo &MFI = MF.getFrameInfo();
475 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
476
477 // If we do not reserve stack space for outgoing arguments in prologue,
478 // we will adjust the stack pointer before call instruction. After the
479 // adjustment, we can not use SP to access the stack objects for the
480 // arguments. Instead, use BP to access these stack objects.
481 return (MFI.hasVarSizedObjects() ||
482 (!hasReservedCallFrame(MF) && (!MFI.isMaxCallFrameSizeComputed() ||
483 MFI.getMaxCallFrameSize() != 0))) &&
484 TRI->hasStackRealignment(MF);
485}
486
487// Determines the size of the frame and maximum call frame size.
488void RISCVFrameLowering::determineFrameLayout(MachineFunction &MF) const {
489 MachineFrameInfo &MFI = MF.getFrameInfo();
490 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
491
492 // Get the number of bytes to allocate from the FrameInfo.
493 uint64_t FrameSize = MFI.getStackSize();
494
495 // QCI Interrupts use at least 96 bytes of stack space
496 if (RVFI->useQCIInterrupt(MF))
497 FrameSize = std::max(a: FrameSize, b: QCIInterruptPushAmount);
498
499 // Get the alignment.
500 Align StackAlign = getStackAlign();
501
502 // Make sure the frame is aligned.
503 FrameSize = alignTo(Size: FrameSize, A: StackAlign);
504
505 // Update frame info.
506 MFI.setStackSize(FrameSize);
507
508 // When using SP or BP to access stack objects, we may require extra padding
509 // to ensure the bottom of the RVV stack is correctly aligned within the main
510 // stack. We calculate this as the amount required to align the scalar local
511 // variable section up to the RVV alignment.
512 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
513 if (RVFI->getRVVStackSize() && (!hasFP(MF) || TRI->hasStackRealignment(MF))) {
514 int ScalarLocalVarSize = FrameSize - RVFI->getCalleeSavedStackSize() -
515 RVFI->getVarArgsSaveSize();
516 if (auto RVVPadding =
517 offsetToAlignment(Value: ScalarLocalVarSize, Alignment: RVFI->getRVVStackAlign()))
518 RVFI->setRVVPadding(RVVPadding);
519 }
520}
521
522// Returns the stack size including RVV padding (when required), rounded back
523// up to the required stack alignment.
524uint64_t RISCVFrameLowering::getStackSizeWithRVVPadding(
525 const MachineFunction &MF) const {
526 const MachineFrameInfo &MFI = MF.getFrameInfo();
527 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
528 return alignTo(Size: MFI.getStackSize() + RVFI->getRVVPadding(), A: getStackAlign());
529}
530
531static SmallVector<CalleeSavedInfo, 8>
532getUnmanagedCSI(const MachineFunction &MF,
533 const std::vector<CalleeSavedInfo> &CSI) {
534 const MachineFrameInfo &MFI = MF.getFrameInfo();
535 SmallVector<CalleeSavedInfo, 8> NonLibcallCSI;
536
537 for (auto &CS : CSI) {
538 int FI = CS.getFrameIdx();
539 if (FI >= 0 && MFI.getStackID(ObjectIdx: FI) == TargetStackID::Default)
540 NonLibcallCSI.push_back(Elt: CS);
541 }
542
543 return NonLibcallCSI;
544}
545
546static SmallVector<CalleeSavedInfo, 8>
547getRVVCalleeSavedInfo(const MachineFunction &MF,
548 const std::vector<CalleeSavedInfo> &CSI) {
549 const MachineFrameInfo &MFI = MF.getFrameInfo();
550 SmallVector<CalleeSavedInfo, 8> RVVCSI;
551
552 for (auto &CS : CSI) {
553 int FI = CS.getFrameIdx();
554 if (FI >= 0 && MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector)
555 RVVCSI.push_back(Elt: CS);
556 }
557
558 return RVVCSI;
559}
560
561static SmallVector<CalleeSavedInfo, 8>
562getPushOrLibCallsSavedInfo(const MachineFunction &MF,
563 const std::vector<CalleeSavedInfo> &CSI) {
564 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
565
566 SmallVector<CalleeSavedInfo, 8> PushOrLibCallsCSI;
567 if (!RVFI->useSaveRestoreLibCalls(MF) && !RVFI->isPushable(MF))
568 return PushOrLibCallsCSI;
569
570 for (const auto &CS : CSI) {
571 if (RVFI->useQCIInterrupt(MF)) {
572 // Some registers are saved by both `QC.C.MIENTER(.NEST)` and
573 // `QC.CM.PUSH(FP)`. In these cases, prioritise the CFI info that points
574 // to the versions saved by `QC.C.MIENTER(.NEST)` which is what FP
575 // unwinding would use.
576 if (llvm::is_contained(Range: llvm::make_first_range(c: FixedCSRFIQCIInterruptMap),
577 Element: CS.getReg()))
578 continue;
579 }
580
581 if (llvm::is_contained(Range: FixedCSRFIMap, Element: CS.getReg()))
582 PushOrLibCallsCSI.push_back(Elt: CS);
583 }
584
585 return PushOrLibCallsCSI;
586}
587
588static SmallVector<CalleeSavedInfo, 8>
589getQCISavedInfo(const MachineFunction &MF,
590 const std::vector<CalleeSavedInfo> &CSI) {
591 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
592
593 SmallVector<CalleeSavedInfo, 8> QCIInterruptCSI;
594 if (!RVFI->useQCIInterrupt(MF))
595 return QCIInterruptCSI;
596
597 for (const auto &CS : CSI) {
598 if (llvm::is_contained(Range: llvm::make_first_range(c: FixedCSRFIQCIInterruptMap),
599 Element: CS.getReg()))
600 QCIInterruptCSI.push_back(Elt: CS);
601 }
602
603 return QCIInterruptCSI;
604}
605
606void RISCVFrameLowering::allocateAndProbeStackForRVV(
607 MachineFunction &MF, MachineBasicBlock &MBB,
608 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, int64_t Amount,
609 MachineInstr::MIFlag Flag, bool EmitCFI, bool DynAllocation) const {
610 assert(Amount != 0 && "Did not need to adjust stack pointer for RVV.");
611
612 // Emit a variable-length allocation probing loop.
613
614 // Get VLEN in TargetReg
615 const RISCVInstrInfo *TII = STI.getInstrInfo();
616 Register TargetReg = RISCV::X6;
617 uint32_t NumOfVReg = Amount / RISCV::RVVBytesPerBlock;
618 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::PseudoReadVLENB), DestReg: TargetReg)
619 .setMIFlag(Flag);
620 TII->mulImm(MF, MBB, II: MBBI, DL, DestReg: TargetReg, Amt: NumOfVReg, Flag);
621
622 CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
623 if (EmitCFI) {
624 // Set the CFA register to TargetReg.
625 CFIBuilder.buildDefCFA(Reg: TargetReg, Offset: -Amount);
626 }
627
628 // It will be expanded to a probe loop in `inlineStackProbe`.
629 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::PROBED_STACKALLOC_RVV))
630 .addReg(RegNo: TargetReg);
631
632 if (EmitCFI) {
633 // Set the CFA register back to SP.
634 CFIBuilder.buildDefCFARegister(Reg: SPReg);
635 }
636
637 // SUB SP, SP, T1
638 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SUB), DestReg: SPReg)
639 .addReg(RegNo: SPReg)
640 .addReg(RegNo: TargetReg)
641 .setMIFlag(Flag);
642
643 // If we have a dynamic allocation later we need to probe any residuals.
644 if (DynAllocation) {
645 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: STI.is64Bit() ? RISCV::SD : RISCV::SW))
646 .addReg(RegNo: RISCV::X0)
647 .addReg(RegNo: SPReg)
648 .addImm(Val: 0)
649 .setMIFlags(MachineInstr::FrameSetup);
650 }
651}
652
653static void appendScalableVectorExpression(const TargetRegisterInfo &TRI,
654 SmallVectorImpl<char> &Expr,
655 int FixedOffset, int ScalableOffset,
656 llvm::raw_string_ostream &Comment) {
657 unsigned DwarfVLenB = TRI.getDwarfRegNum(RegNum: RISCV::VLENB, isEH: true);
658 uint8_t Buffer[16];
659 if (FixedOffset) {
660 Expr.push_back(Elt: dwarf::DW_OP_consts);
661 Expr.append(in_start: Buffer, in_end: Buffer + encodeSLEB128(Value: FixedOffset, p: Buffer));
662 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_plus);
663 Comment << (FixedOffset < 0 ? " - " : " + ") << std::abs(x: FixedOffset);
664 }
665
666 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_consts);
667 Expr.append(in_start: Buffer, in_end: Buffer + encodeSLEB128(Value: ScalableOffset, p: Buffer));
668
669 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_bregx);
670 Expr.append(in_start: Buffer, in_end: Buffer + encodeULEB128(Value: DwarfVLenB, p: Buffer));
671 Expr.push_back(Elt: 0);
672
673 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_mul);
674 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_plus);
675
676 Comment << (ScalableOffset < 0 ? " - " : " + ") << std::abs(x: ScalableOffset)
677 << " * vlenb";
678}
679
680static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
681 Register Reg,
682 uint64_t FixedOffset,
683 uint64_t ScalableOffset) {
684 assert(ScalableOffset != 0 && "Did not need to adjust CFA for RVV");
685 SmallString<64> Expr;
686 std::string CommentBuffer;
687 llvm::raw_string_ostream Comment(CommentBuffer);
688 // Build up the expression (Reg + FixedOffset + ScalableOffset * VLENB).
689 unsigned DwarfReg = TRI.getDwarfRegNum(RegNum: Reg, isEH: true);
690 Expr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
691 Expr.push_back(Elt: 0);
692 if (Reg == SPReg)
693 Comment << "sp";
694 else
695 Comment << printReg(Reg, TRI: &TRI);
696
697 appendScalableVectorExpression(TRI, Expr, FixedOffset, ScalableOffset,
698 Comment);
699
700 SmallString<64> DefCfaExpr;
701 uint8_t Buffer[16];
702 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
703 DefCfaExpr.append(in_start: Buffer, in_end: Buffer + encodeULEB128(Value: Expr.size(), p: Buffer));
704 DefCfaExpr.append(RHS: Expr.str());
705
706 return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(),
707 Comment: Comment.str());
708}
709
710static MCCFIInstruction createDefCFAOffset(const TargetRegisterInfo &TRI,
711 Register Reg, uint64_t FixedOffset,
712 uint64_t ScalableOffset) {
713 assert(ScalableOffset != 0 && "Did not need to adjust CFA for RVV");
714 SmallString<64> Expr;
715 std::string CommentBuffer;
716 llvm::raw_string_ostream Comment(CommentBuffer);
717 Comment << printReg(Reg, TRI: &TRI) << " @ cfa";
718
719 // Build up the expression (FixedOffset + ScalableOffset * VLENB).
720 appendScalableVectorExpression(TRI, Expr, FixedOffset, ScalableOffset,
721 Comment);
722
723 SmallString<64> DefCfaExpr;
724 uint8_t Buffer[16];
725 unsigned DwarfReg = TRI.getDwarfRegNum(RegNum: Reg, isEH: true);
726 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
727 DefCfaExpr.append(in_start: Buffer, in_end: Buffer + encodeULEB128(Value: DwarfReg, p: Buffer));
728 DefCfaExpr.append(in_start: Buffer, in_end: Buffer + encodeULEB128(Value: Expr.size(), p: Buffer));
729 DefCfaExpr.append(RHS: Expr.str());
730
731 return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(),
732 Comment: Comment.str());
733}
734
735// Allocate stack space and probe it if necessary.
736void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
737 MachineBasicBlock::iterator MBBI,
738 MachineFunction &MF, uint64_t Offset,
739 uint64_t RealStackSize, bool EmitCFI,
740 bool NeedProbe, uint64_t ProbeSize,
741 bool DynAllocation) const {
742 DebugLoc DL;
743 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
744 const RISCVInstrInfo *TII = STI.getInstrInfo();
745 bool IsRV64 = STI.is64Bit();
746 CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
747
748 // Simply allocate the stack if it's not big enough to require a probe.
749 if (!NeedProbe || Offset <= ProbeSize) {
750 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: -Offset),
751 Flag: MachineInstr::FrameSetup, RequiredAlign: getStackAlign());
752
753 if (EmitCFI)
754 CFIBuilder.buildDefCFAOffset(Offset: RealStackSize);
755
756 if (NeedProbe && DynAllocation) {
757 // s[d|w] zero, 0(sp)
758 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
759 .addReg(RegNo: RISCV::X0)
760 .addReg(RegNo: SPReg)
761 .addImm(Val: 0)
762 .setMIFlags(MachineInstr::FrameSetup);
763 }
764
765 return;
766 }
767
768 // Unroll the probe loop depending on the number of iterations.
769 if (Offset < ProbeSize * 5) {
770 uint64_t CurrentOffset = 0;
771 while (CurrentOffset + ProbeSize <= Offset) {
772 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg,
773 Offset: StackOffset::getFixed(Fixed: -ProbeSize), Flag: MachineInstr::FrameSetup,
774 RequiredAlign: getStackAlign());
775 // s[d|w] zero, 0(sp)
776 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
777 .addReg(RegNo: RISCV::X0)
778 .addReg(RegNo: SPReg)
779 .addImm(Val: 0)
780 .setMIFlags(MachineInstr::FrameSetup);
781
782 CurrentOffset += ProbeSize;
783 if (EmitCFI)
784 CFIBuilder.buildDefCFAOffset(Offset: CurrentOffset);
785 }
786
787 uint64_t Residual = Offset - CurrentOffset;
788 if (Residual) {
789 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg,
790 Offset: StackOffset::getFixed(Fixed: -Residual), Flag: MachineInstr::FrameSetup,
791 RequiredAlign: getStackAlign());
792 if (EmitCFI)
793 CFIBuilder.buildDefCFAOffset(Offset);
794
795 if (DynAllocation) {
796 // s[d|w] zero, 0(sp)
797 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
798 .addReg(RegNo: RISCV::X0)
799 .addReg(RegNo: SPReg)
800 .addImm(Val: 0)
801 .setMIFlags(MachineInstr::FrameSetup);
802 }
803 }
804
805 return;
806 }
807
808 // Emit a variable-length allocation probing loop.
809 uint64_t RoundedSize = alignDown(Value: Offset, Align: ProbeSize);
810 uint64_t Residual = Offset - RoundedSize;
811
812 Register TargetReg = RISCV::X6;
813 // SUB TargetReg, SP, RoundedSize
814 RI->adjustReg(MBB, II: MBBI, DL, DestReg: TargetReg, SrcReg: SPReg,
815 Offset: StackOffset::getFixed(Fixed: -RoundedSize), Flag: MachineInstr::FrameSetup,
816 RequiredAlign: getStackAlign());
817
818 if (EmitCFI) {
819 // Set the CFA register to TargetReg.
820 CFIBuilder.buildDefCFA(Reg: TargetReg, Offset: RoundedSize);
821 }
822
823 // It will be expanded to a probe loop in `inlineStackProbe`.
824 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::PROBED_STACKALLOC)).addReg(RegNo: TargetReg);
825
826 if (EmitCFI) {
827 // Set the CFA register back to SP.
828 CFIBuilder.buildDefCFARegister(Reg: SPReg);
829 }
830
831 if (Residual) {
832 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: -Residual),
833 Flag: MachineInstr::FrameSetup, RequiredAlign: getStackAlign());
834 if (DynAllocation) {
835 // s[d|w] zero, 0(sp)
836 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
837 .addReg(RegNo: RISCV::X0)
838 .addReg(RegNo: SPReg)
839 .addImm(Val: 0)
840 .setMIFlags(MachineInstr::FrameSetup);
841 }
842 }
843
844 if (EmitCFI)
845 CFIBuilder.buildDefCFAOffset(Offset);
846}
847
848static bool isPush(unsigned Opcode) {
849 switch (Opcode) {
850 case RISCV::CM_PUSH:
851 case RISCV::QC_CM_PUSH:
852 case RISCV::QC_CM_PUSHFP:
853 return true;
854 default:
855 return false;
856 }
857}
858
859static bool isPop(unsigned Opcode) {
860 // There are other pops but these are the only ones introduced during this
861 // pass.
862 switch (Opcode) {
863 case RISCV::CM_POP:
864 case RISCV::QC_CM_POP:
865 return true;
866 default:
867 return false;
868 }
869}
870
871static unsigned getPushOpcode(RISCVMachineFunctionInfo::PushPopKind Kind,
872 bool UpdateFP) {
873 switch (Kind) {
874 case RISCVMachineFunctionInfo::PushPopKind::StdExtZcmp:
875 return RISCV::CM_PUSH;
876 case RISCVMachineFunctionInfo::PushPopKind::VendorXqccmp:
877 return UpdateFP ? RISCV::QC_CM_PUSHFP : RISCV::QC_CM_PUSH;
878 default:
879 llvm_unreachable("Unhandled PushPopKind");
880 }
881}
882
883static unsigned getPopOpcode(RISCVMachineFunctionInfo::PushPopKind Kind) {
884 // There are other pops but they are introduced later by the Push/Pop
885 // Optimizer.
886 switch (Kind) {
887 case RISCVMachineFunctionInfo::PushPopKind::StdExtZcmp:
888 return RISCV::CM_POP;
889 case RISCVMachineFunctionInfo::PushPopKind::VendorXqccmp:
890 return RISCV::QC_CM_POP;
891 default:
892 llvm_unreachable("Unhandled PushPopKind");
893 }
894}
895
896void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
897 MachineBasicBlock &MBB) const {
898 MachineFrameInfo &MFI = MF.getFrameInfo();
899 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
900 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
901 MachineBasicBlock::iterator MBBI = MBB.begin();
902
903 Register BPReg = RISCVABI::getBPReg();
904
905 // Debug location must be unknown since the first debug location is used
906 // to determine the end of the prologue.
907 DebugLoc DL;
908
909 // All calls are tail calls in GHC calling conv, and functions have no
910 // prologue/epilogue.
911 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
912 return;
913
914 // SiFive CLIC needs to swap `sp` into `sf.mscratchcsw`
915 emitSiFiveCLICStackSwap(MF, MBB, MBBI, DL);
916
917 // Emit prologue for shadow call stack.
918 emitSCSPrologue(MF, MBB, MI: MBBI, DL);
919
920 // We keep track of the first instruction because it might be a
921 // `(QC.)CM.PUSH(FP)`, and we may need to adjust the immediate rather than
922 // inserting an `addi sp, sp, -N*16`
923 auto PossiblePush = MBBI;
924
925 // Skip past all callee-saved register spill instructions.
926 while (MBBI != MBB.end() && MBBI->getFlag(Flag: MachineInstr::FrameSetup))
927 ++MBBI;
928
929 // Determine the correct frame layout
930 determineFrameLayout(MF);
931
932 const auto &CSI = MFI.getCalleeSavedInfo();
933
934 // Skip to before the spills of scalar callee-saved registers
935 // FIXME: assumes exactly one instruction is used to restore each
936 // callee-saved register.
937 MBBI = std::prev(x: MBBI, n: getRVVCalleeSavedInfo(MF, CSI).size() +
938 getUnmanagedCSI(MF, CSI).size());
939 CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
940
941 // If libcalls are used to spill and restore callee-saved registers, the frame
942 // has two sections; the opaque section managed by the libcalls, and the
943 // section managed by MachineFrameInfo which can also hold callee saved
944 // registers in fixed stack slots, both of which have negative frame indices.
945 // This gets even more complicated when incoming arguments are passed via the
946 // stack, as these too have negative frame indices. An example is detailed
947 // below:
948 //
949 // | incoming arg | <- FI[-3]
950 // | libcallspill |
951 // | calleespill | <- FI[-2]
952 // | calleespill | <- FI[-1]
953 // | this_frame | <- FI[0]
954 //
955 // For negative frame indices, the offset from the frame pointer will differ
956 // depending on which of these groups the frame index applies to.
957 // The following calculates the correct offset knowing the number of callee
958 // saved registers spilt by the two methods.
959 if (int LibCallRegs = getLibCallID(MF, CSI: MFI.getCalleeSavedInfo()) + 1) {
960 // Calculate the size of the frame managed by the libcall. The stack
961 // alignment of these libcalls should be the same as how we set it in
962 // getABIStackAlignment.
963 unsigned LibCallFrameSize =
964 alignTo(Size: (STI.getXLen() / 8) * LibCallRegs, A: getStackAlign());
965 RVFI->setLibCallStackSize(LibCallFrameSize);
966
967 CFIBuilder.buildDefCFAOffset(Offset: LibCallFrameSize);
968 for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
969 CFIBuilder.buildOffset(Reg: CS.getReg(),
970 Offset: MFI.getObjectOffset(ObjectIdx: CS.getFrameIdx()));
971 }
972
973 // FIXME (note copied from Lanai): This appears to be overallocating. Needs
974 // investigation. Get the number of bytes to allocate from the FrameInfo.
975 uint64_t RealStackSize = getStackSizeWithRVVPadding(MF);
976 uint64_t StackSize = RealStackSize - RVFI->getReservedSpillsSize();
977 uint64_t RVVStackSize = RVFI->getRVVStackSize();
978
979 // Early exit if there is no need to allocate on the stack
980 if (RealStackSize == 0 && !MFI.adjustsStack() && RVVStackSize == 0)
981 return;
982
983 // If the stack pointer has been marked as reserved, then produce an error if
984 // the frame requires stack allocation
985 if (STI.isRegisterReservedByUser(i: SPReg))
986 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
987 MF.getFunction(), "Stack pointer required, but has been reserved."});
988
989 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
990 // Split the SP adjustment to reduce the offsets of callee saved spill.
991 if (FirstSPAdjustAmount) {
992 StackSize = FirstSPAdjustAmount;
993 RealStackSize = FirstSPAdjustAmount;
994 }
995
996 if (RVFI->useQCIInterrupt(MF)) {
997 // The function starts with `QC.C.MIENTER(.NEST)`, so the `(QC.)CM.PUSH(FP)`
998 // could only be the next instruction.
999 ++PossiblePush;
1000
1001 // Insert the CFI metadata before where we think the `(QC.)CM.PUSH(FP)`
1002 // could be. The PUSH will also get its own CFI metadata for its own
1003 // modifications, which should come after the PUSH.
1004 CFIInstBuilder PushCFIBuilder(MBB, PossiblePush, MachineInstr::FrameSetup);
1005 PushCFIBuilder.buildDefCFAOffset(Offset: QCIInterruptPushAmount);
1006 for (const CalleeSavedInfo &CS : getQCISavedInfo(MF, CSI))
1007 PushCFIBuilder.buildOffset(Reg: CS.getReg(),
1008 Offset: MFI.getObjectOffset(ObjectIdx: CS.getFrameIdx()));
1009 }
1010
1011 if (RVFI->isPushable(MF) && PossiblePush != MBB.end() &&
1012 isPush(Opcode: PossiblePush->getOpcode())) {
1013 // Use available stack adjustment in push instruction to allocate additional
1014 // stack space. Align the stack size down to a multiple of 16. This is
1015 // needed for RVE.
1016 // FIXME: Can we increase the stack size to a multiple of 16 instead?
1017 uint64_t StackAdj =
1018 std::min(a: alignDown(Value: StackSize, Align: 16), b: static_cast<uint64_t>(48));
1019 PossiblePush->getOperand(i: 1).setImm(StackAdj);
1020 StackSize -= StackAdj;
1021
1022 CFIBuilder.buildDefCFAOffset(Offset: RealStackSize - StackSize);
1023 for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
1024 CFIBuilder.buildOffset(Reg: CS.getReg(),
1025 Offset: MFI.getObjectOffset(ObjectIdx: CS.getFrameIdx()));
1026 }
1027
1028 // Allocate space on the stack if necessary.
1029 auto &Subtarget = MF.getSubtarget<RISCVSubtarget>();
1030 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
1031 bool NeedProbe = TLI->hasInlineStackProbe(MF);
1032 uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign: getStackAlign());
1033 bool DynAllocation =
1034 MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
1035 if (StackSize != 0)
1036 allocateStack(MBB, MBBI, MF, Offset: StackSize, RealStackSize, /*EmitCFI=*/true,
1037 NeedProbe, ProbeSize, DynAllocation);
1038
1039 // Save SiFive CLIC CSRs into Stack
1040 emitSiFiveCLICPreemptibleSaves(MF, MBB, MBBI, DL);
1041
1042 // The frame pointer is callee-saved, and code has been generated for us to
1043 // save it to the stack. We need to skip over the storing of callee-saved
1044 // registers as the frame pointer must be modified after it has been saved
1045 // to the stack, not before.
1046 // FIXME: assumes exactly one instruction is used to save each callee-saved
1047 // register.
1048 std::advance(i&: MBBI, n: getUnmanagedCSI(MF, CSI).size());
1049 CFIBuilder.setInsertPoint(MBBI);
1050
1051 // Iterate over list of callee-saved registers and emit .cfi_offset
1052 // directives.
1053 for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
1054 CFIBuilder.buildOffset(Reg: CS.getReg(), Offset: MFI.getObjectOffset(ObjectIdx: CS.getFrameIdx()));
1055
1056 // Generate new FP.
1057 if (hasFP(MF)) {
1058 if (STI.isRegisterReservedByUser(i: FPReg))
1059 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
1060 MF.getFunction(), "Frame pointer required, but has been reserved."});
1061 // The frame pointer does need to be reserved from register allocation.
1062 assert(MF.getRegInfo().isReserved(FPReg) && "FP not reserved");
1063
1064 // Some stack management variants automatically keep FP updated, so we don't
1065 // need an instruction to do so.
1066 if (!RVFI->hasImplicitFPUpdates(MF)) {
1067 RI->adjustReg(
1068 MBB, II: MBBI, DL, DestReg: FPReg, SrcReg: SPReg,
1069 Offset: StackOffset::getFixed(Fixed: RealStackSize - RVFI->getVarArgsSaveSize()),
1070 Flag: MachineInstr::FrameSetup, RequiredAlign: getStackAlign());
1071 }
1072
1073 CFIBuilder.buildDefCFA(Reg: FPReg, Offset: RVFI->getVarArgsSaveSize());
1074 }
1075
1076 uint64_t SecondSPAdjustAmount = 0;
1077 // Emit the second SP adjustment after saving callee saved registers.
1078 if (FirstSPAdjustAmount) {
1079 SecondSPAdjustAmount = getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
1080 assert(SecondSPAdjustAmount > 0 &&
1081 "SecondSPAdjustAmount should be greater than zero");
1082
1083 allocateStack(MBB, MBBI, MF, Offset: SecondSPAdjustAmount,
1084 RealStackSize: getStackSizeWithRVVPadding(MF), EmitCFI: !hasFP(MF), NeedProbe,
1085 ProbeSize, DynAllocation);
1086 }
1087
1088 if (RVVStackSize) {
1089 if (NeedProbe) {
1090 allocateAndProbeStackForRVV(MF, MBB, MBBI, DL, Amount: RVVStackSize,
1091 Flag: MachineInstr::FrameSetup, EmitCFI: !hasFP(MF),
1092 DynAllocation);
1093 } else {
1094 // We must keep the stack pointer aligned through any intermediate
1095 // updates.
1096 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg,
1097 Offset: StackOffset::getScalable(Scalable: -RVVStackSize),
1098 Flag: MachineInstr::FrameSetup, RequiredAlign: getStackAlign());
1099 }
1100
1101 if (!hasFP(MF)) {
1102 // Emit .cfi_def_cfa_expression "sp + StackSize + RVVStackSize * vlenb".
1103 CFIBuilder.insertCFIInst(CFIInst: createDefCFAExpression(
1104 TRI: *RI, Reg: SPReg, FixedOffset: getStackSizeWithRVVPadding(MF), ScalableOffset: RVVStackSize / 8));
1105 }
1106
1107 std::advance(i&: MBBI, n: getRVVCalleeSavedInfo(MF, CSI).size());
1108 emitCalleeSavedRVVPrologCFI(MBB, MI: MBBI, HasFP: hasFP(MF));
1109 }
1110
1111 if (hasFP(MF)) {
1112 // Realign Stack
1113 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
1114 if (RI->hasStackRealignment(MF)) {
1115 Align MaxAlignment = MFI.getMaxAlign();
1116
1117 const RISCVInstrInfo *TII = STI.getInstrInfo();
1118 if (isInt<12>(x: -(int)MaxAlignment.value())) {
1119 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::ANDI), DestReg: SPReg)
1120 .addReg(RegNo: SPReg)
1121 .addImm(Val: -(int)MaxAlignment.value())
1122 .setMIFlag(MachineInstr::FrameSetup);
1123 } else {
1124 unsigned ShiftAmount = Log2(A: MaxAlignment);
1125 Register VR =
1126 MF.getRegInfo().createVirtualRegister(RegClass: &RISCV::GPRRegClass);
1127 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SRLI), DestReg: VR)
1128 .addReg(RegNo: SPReg)
1129 .addImm(Val: ShiftAmount)
1130 .setMIFlag(MachineInstr::FrameSetup);
1131 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SLLI), DestReg: SPReg)
1132 .addReg(RegNo: VR)
1133 .addImm(Val: ShiftAmount)
1134 .setMIFlag(MachineInstr::FrameSetup);
1135 }
1136 if (NeedProbe && RVVStackSize == 0) {
1137 // Do a probe if the align + size allocated just passed the probe size
1138 // and was not yet probed.
1139 if (SecondSPAdjustAmount < ProbeSize &&
1140 SecondSPAdjustAmount + MaxAlignment.value() >= ProbeSize) {
1141 bool IsRV64 = STI.is64Bit();
1142 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
1143 .addReg(RegNo: RISCV::X0)
1144 .addReg(RegNo: SPReg)
1145 .addImm(Val: 0)
1146 .setMIFlags(MachineInstr::FrameSetup);
1147 }
1148 }
1149 // FP will be used to restore the frame in the epilogue, so we need
1150 // another base register BP to record SP after re-alignment. SP will
1151 // track the current stack after allocating variable sized objects.
1152 if (hasBP(MF)) {
1153 // move BP, SP
1154 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: BPReg)
1155 .addReg(RegNo: SPReg)
1156 .addImm(Val: 0)
1157 .setMIFlag(MachineInstr::FrameSetup);
1158 }
1159 }
1160 }
1161}
1162
1163void RISCVFrameLowering::deallocateStack(MachineFunction &MF,
1164 MachineBasicBlock &MBB,
1165 MachineBasicBlock::iterator MBBI,
1166 const DebugLoc &DL,
1167 uint64_t &StackSize,
1168 int64_t CFAOffset) const {
1169 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
1170
1171 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: StackSize),
1172 Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign());
1173 StackSize = 0;
1174
1175 CFIInstBuilder(MBB, MBBI, MachineInstr::FrameDestroy)
1176 .buildDefCFAOffset(Offset: CFAOffset);
1177}
1178
1179void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
1180 MachineBasicBlock &MBB) const {
1181 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
1182 MachineFrameInfo &MFI = MF.getFrameInfo();
1183 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1184
1185 // All calls are tail calls in GHC calling conv, and functions have no
1186 // prologue/epilogue.
1187 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
1188 return;
1189
1190 // Get the insert location for the epilogue. If there were no terminators in
1191 // the block, get the last instruction.
1192 MachineBasicBlock::iterator MBBI = MBB.end();
1193 DebugLoc DL;
1194 if (!MBB.empty()) {
1195 MBBI = MBB.getLastNonDebugInstr();
1196 if (MBBI != MBB.end())
1197 DL = MBBI->getDebugLoc();
1198
1199 MBBI = MBB.getFirstTerminator();
1200
1201 // Skip to before the restores of all callee-saved registers.
1202 while (MBBI != MBB.begin() &&
1203 std::prev(x: MBBI)->getFlag(Flag: MachineInstr::FrameDestroy))
1204 --MBBI;
1205 }
1206
1207 const auto &CSI = MFI.getCalleeSavedInfo();
1208
1209 // Skip to before the restores of scalar callee-saved registers
1210 // FIXME: assumes exactly one instruction is used to restore each
1211 // callee-saved register.
1212 auto FirstScalarCSRRestoreInsn =
1213 std::next(x: MBBI, n: getRVVCalleeSavedInfo(MF, CSI).size());
1214 CFIInstBuilder CFIBuilder(MBB, FirstScalarCSRRestoreInsn,
1215 MachineInstr::FrameDestroy);
1216
1217 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
1218 uint64_t RealStackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
1219 : getStackSizeWithRVVPadding(MF);
1220 uint64_t StackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
1221 : getStackSizeWithRVVPadding(MF) -
1222 RVFI->getReservedSpillsSize();
1223 uint64_t FPOffset = RealStackSize - RVFI->getVarArgsSaveSize();
1224 uint64_t RVVStackSize = RVFI->getRVVStackSize();
1225
1226 bool RestoreSPFromFP = RI->hasStackRealignment(MF) ||
1227 MFI.hasVarSizedObjects() || !hasReservedCallFrame(MF);
1228 if (RVVStackSize) {
1229 // If RestoreSPFromFP the stack pointer will be restored using the frame
1230 // pointer value.
1231 if (!RestoreSPFromFP)
1232 RI->adjustReg(MBB, II: FirstScalarCSRRestoreInsn, DL, DestReg: SPReg, SrcReg: SPReg,
1233 Offset: StackOffset::getScalable(Scalable: RVVStackSize),
1234 Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign());
1235
1236 if (!hasFP(MF))
1237 CFIBuilder.buildDefCFA(Reg: SPReg, Offset: RealStackSize);
1238
1239 emitCalleeSavedRVVEpilogCFI(MBB, MI: FirstScalarCSRRestoreInsn);
1240 }
1241
1242 if (FirstSPAdjustAmount) {
1243 uint64_t SecondSPAdjustAmount =
1244 getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
1245 assert(SecondSPAdjustAmount > 0 &&
1246 "SecondSPAdjustAmount should be greater than zero");
1247
1248 // If RestoreSPFromFP the stack pointer will be restored using the frame
1249 // pointer value.
1250 if (!RestoreSPFromFP)
1251 RI->adjustReg(MBB, II: FirstScalarCSRRestoreInsn, DL, DestReg: SPReg, SrcReg: SPReg,
1252 Offset: StackOffset::getFixed(Fixed: SecondSPAdjustAmount),
1253 Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign());
1254
1255 if (!hasFP(MF))
1256 CFIBuilder.buildDefCFAOffset(Offset: FirstSPAdjustAmount);
1257 }
1258
1259 // Restore the stack pointer using the value of the frame pointer. Only
1260 // necessary if the stack pointer was modified, meaning the stack size is
1261 // unknown.
1262 //
1263 // In order to make sure the stack point is right through the EH region,
1264 // we also need to restore stack pointer from the frame pointer if we
1265 // don't preserve stack space within prologue/epilogue for outgoing variables,
1266 // normally it's just checking the variable sized object is present or not
1267 // is enough, but we also don't preserve that at prologue/epilogue when
1268 // have vector objects in stack.
1269 if (RestoreSPFromFP) {
1270 assert(hasFP(MF) && "frame pointer should not have been eliminated");
1271 RI->adjustReg(MBB, II: FirstScalarCSRRestoreInsn, DL, DestReg: SPReg, SrcReg: FPReg,
1272 Offset: StackOffset::getFixed(Fixed: -FPOffset), Flag: MachineInstr::FrameDestroy,
1273 RequiredAlign: getStackAlign());
1274 }
1275
1276 if (hasFP(MF))
1277 CFIBuilder.buildDefCFA(Reg: SPReg, Offset: RealStackSize);
1278
1279 // Skip to after the restores of scalar callee-saved registers
1280 // FIXME: assumes exactly one instruction is used to restore each
1281 // callee-saved register.
1282 MBBI = std::next(x: FirstScalarCSRRestoreInsn, n: getUnmanagedCSI(MF, CSI).size());
1283 CFIBuilder.setInsertPoint(MBBI);
1284
1285 if (getLibCallID(MF, CSI) != -1) {
1286 // tail __riscv_restore_[0-12] instruction is considered as a terminator,
1287 // therefore it is unnecessary to place any CFI instructions after it. Just
1288 // deallocate stack if needed and return.
1289 if (StackSize != 0)
1290 deallocateStack(MF, MBB, MBBI, DL, StackSize,
1291 CFAOffset: RVFI->getLibCallStackSize());
1292
1293 // Emit epilogue for shadow call stack.
1294 emitSCSEpilogue(MF, MBB, MI: MBBI, DL);
1295 return;
1296 }
1297
1298 // Recover callee-saved registers.
1299 for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
1300 CFIBuilder.buildRestore(Reg: CS.getReg());
1301
1302 if (RVFI->isPushable(MF) && MBBI != MBB.end() && isPop(Opcode: MBBI->getOpcode())) {
1303 // Use available stack adjustment in pop instruction to deallocate stack
1304 // space. Align the stack size down to a multiple of 16. This is needed for
1305 // RVE.
1306 // FIXME: Can we increase the stack size to a multiple of 16 instead?
1307 uint64_t StackAdj =
1308 std::min(a: alignDown(Value: StackSize, Align: 16), b: static_cast<uint64_t>(48));
1309 MBBI->getOperand(i: 1).setImm(StackAdj);
1310 StackSize -= StackAdj;
1311
1312 if (StackSize != 0)
1313 deallocateStack(MF, MBB, MBBI, DL, StackSize,
1314 /*stack_adj of cm.pop instr*/ CFAOffset: RealStackSize - StackSize);
1315
1316 auto NextI = next_nodbg(It: MBBI, End: MBB.end());
1317 if (NextI == MBB.end() || NextI->getOpcode() != RISCV::PseudoRET) {
1318 ++MBBI;
1319 CFIBuilder.setInsertPoint(MBBI);
1320
1321 for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
1322 CFIBuilder.buildRestore(Reg: CS.getReg());
1323
1324 // Update CFA Offset. If this is a QCI interrupt function, there will be a
1325 // leftover offset which is deallocated by `QC.C.MILEAVERET`, otherwise
1326 // getQCIInterruptStackSize() will be 0.
1327 CFIBuilder.buildDefCFAOffset(Offset: RVFI->getQCIInterruptStackSize());
1328 }
1329 }
1330
1331 emitSiFiveCLICPreemptibleRestores(MF, MBB, MBBI, DL);
1332
1333 // Deallocate stack if StackSize isn't a zero yet. If this is a QCI interrupt
1334 // function, there will be a leftover offset which is deallocated by
1335 // `QC.C.MILEAVERET`, otherwise getQCIInterruptStackSize() will be 0.
1336 if (StackSize != 0)
1337 deallocateStack(MF, MBB, MBBI, DL, StackSize,
1338 CFAOffset: RVFI->getQCIInterruptStackSize());
1339
1340 // Emit epilogue for shadow call stack.
1341 emitSCSEpilogue(MF, MBB, MI: MBBI, DL);
1342
1343 // SiFive CLIC needs to swap `sf.mscratchcsw` into `sp`
1344 emitSiFiveCLICStackSwap(MF, MBB, MBBI, DL);
1345}
1346
1347StackOffset
1348RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
1349 Register &FrameReg) const {
1350 const MachineFrameInfo &MFI = MF.getFrameInfo();
1351 const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
1352 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1353
1354 // Callee-saved registers should be referenced relative to the stack
1355 // pointer (positive offset), otherwise use the frame pointer (negative
1356 // offset).
1357 const auto &CSI = getUnmanagedCSI(MF, CSI: MFI.getCalleeSavedInfo());
1358 int MinCSFI = 0;
1359 int MaxCSFI = -1;
1360 StackOffset Offset;
1361 auto StackID = MFI.getStackID(ObjectIdx: FI);
1362
1363 assert((StackID == TargetStackID::Default ||
1364 StackID == TargetStackID::ScalableVector) &&
1365 "Unexpected stack ID for the frame object.");
1366 if (StackID == TargetStackID::Default) {
1367 assert(getOffsetOfLocalArea() == 0 && "LocalAreaOffset is not 0!");
1368 Offset = StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FI) +
1369 MFI.getOffsetAdjustment());
1370 } else if (StackID == TargetStackID::ScalableVector) {
1371 Offset = StackOffset::getScalable(Scalable: MFI.getObjectOffset(ObjectIdx: FI));
1372 }
1373
1374 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
1375
1376 if (CSI.size()) {
1377 MinCSFI = CSI[0].getFrameIdx();
1378 MaxCSFI = CSI[CSI.size() - 1].getFrameIdx();
1379 }
1380
1381 if (FI >= MinCSFI && FI <= MaxCSFI) {
1382 FrameReg = SPReg;
1383
1384 if (FirstSPAdjustAmount)
1385 Offset += StackOffset::getFixed(Fixed: FirstSPAdjustAmount);
1386 else
1387 Offset += StackOffset::getFixed(Fixed: getStackSizeWithRVVPadding(MF));
1388 return Offset;
1389 }
1390
1391 if (RI->hasStackRealignment(MF) && !MFI.isFixedObjectIndex(ObjectIdx: FI)) {
1392 // If the stack was realigned, the frame pointer is set in order to allow
1393 // SP to be restored, so we need another base register to record the stack
1394 // after realignment.
1395 // |--------------------------| -- <-- FP
1396 // | callee-allocated save | | <----|
1397 // | area for register varargs| | |
1398 // |--------------------------| | |
1399 // | callee-saved registers | | |
1400 // |--------------------------| -- |
1401 // | realignment (the size of | | |
1402 // | this area is not counted | | |
1403 // | in MFI.getStackSize()) | | |
1404 // |--------------------------| -- |-- MFI.getStackSize()
1405 // | RVV alignment padding | | |
1406 // | (not counted in | | |
1407 // | MFI.getStackSize() but | | |
1408 // | counted in | | |
1409 // | RVFI.getRVVStackSize()) | | |
1410 // |--------------------------| -- |
1411 // | RVV objects | | |
1412 // | (not counted in | | |
1413 // | MFI.getStackSize()) | | |
1414 // |--------------------------| -- |
1415 // | padding before RVV | | |
1416 // | (not counted in | | |
1417 // | MFI.getStackSize() or in | | |
1418 // | RVFI.getRVVStackSize()) | | |
1419 // |--------------------------| -- |
1420 // | scalar local variables | | <----'
1421 // |--------------------------| -- <-- BP (if var sized objects present)
1422 // | VarSize objects | |
1423 // |--------------------------| -- <-- SP
1424 if (hasBP(MF)) {
1425 FrameReg = RISCVABI::getBPReg();
1426 } else {
1427 // VarSize objects must be empty in this case!
1428 assert(!MFI.hasVarSizedObjects());
1429 FrameReg = SPReg;
1430 }
1431 } else {
1432 FrameReg = RI->getFrameRegister(MF);
1433 }
1434
1435 if (FrameReg == FPReg) {
1436 Offset += StackOffset::getFixed(Fixed: RVFI->getVarArgsSaveSize());
1437 // When using FP to access scalable vector objects, we need to minus
1438 // the frame size.
1439 //
1440 // |--------------------------| -- <-- FP
1441 // | callee-allocated save | |
1442 // | area for register varargs| |
1443 // |--------------------------| |
1444 // | callee-saved registers | |
1445 // |--------------------------| | MFI.getStackSize()
1446 // | scalar local variables | |
1447 // |--------------------------| -- (Offset of RVV objects is from here.)
1448 // | RVV objects |
1449 // |--------------------------|
1450 // | VarSize objects |
1451 // |--------------------------| <-- SP
1452 if (StackID == TargetStackID::ScalableVector) {
1453 assert(!RI->hasStackRealignment(MF) &&
1454 "Can't index across variable sized realign");
1455 // We don't expect any extra RVV alignment padding, as the stack size
1456 // and RVV object sections should be correct aligned in their own
1457 // right.
1458 assert(MFI.getStackSize() == getStackSizeWithRVVPadding(MF) &&
1459 "Inconsistent stack layout");
1460 Offset -= StackOffset::getFixed(Fixed: MFI.getStackSize());
1461 }
1462 return Offset;
1463 }
1464
1465 // This case handles indexing off both SP and BP.
1466 // If indexing off SP, there must not be any var sized objects
1467 assert(FrameReg == RISCVABI::getBPReg() || !MFI.hasVarSizedObjects());
1468
1469 // When using SP to access frame objects, we need to add RVV stack size.
1470 //
1471 // |--------------------------| -- <-- FP
1472 // | callee-allocated save | | <----|
1473 // | area for register varargs| | |
1474 // |--------------------------| | |
1475 // | callee-saved registers | | |
1476 // |--------------------------| -- |
1477 // | RVV alignment padding | | |
1478 // | (not counted in | | |
1479 // | MFI.getStackSize() but | | |
1480 // | counted in | | |
1481 // | RVFI.getRVVStackSize()) | | |
1482 // |--------------------------| -- |
1483 // | RVV objects | | |-- MFI.getStackSize()
1484 // | (not counted in | | |
1485 // | MFI.getStackSize()) | | |
1486 // |--------------------------| -- |
1487 // | padding before RVV | | |
1488 // | (not counted in | | |
1489 // | MFI.getStackSize()) | | |
1490 // |--------------------------| -- |
1491 // | scalar local variables | | <----'
1492 // |--------------------------| -- <-- BP (if var sized objects present)
1493 // | VarSize objects | |
1494 // |--------------------------| -- <-- SP
1495 //
1496 // The total amount of padding surrounding RVV objects is described by
1497 // RVV->getRVVPadding() and it can be zero. It allows us to align the RVV
1498 // objects to the required alignment.
1499 if (MFI.getStackID(ObjectIdx: FI) == TargetStackID::Default) {
1500 if (MFI.isFixedObjectIndex(ObjectIdx: FI)) {
1501 assert(!RI->hasStackRealignment(MF) &&
1502 "Can't index across variable sized realign");
1503 Offset += StackOffset::get(Fixed: getStackSizeWithRVVPadding(MF),
1504 Scalable: RVFI->getRVVStackSize());
1505 } else {
1506 Offset += StackOffset::getFixed(Fixed: MFI.getStackSize());
1507 }
1508 } else if (MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector) {
1509 // Ensure the base of the RVV stack is correctly aligned: add on the
1510 // alignment padding.
1511 int ScalarLocalVarSize = MFI.getStackSize() -
1512 RVFI->getCalleeSavedStackSize() -
1513 RVFI->getVarArgsSaveSize() + RVFI->getRVVPadding();
1514 Offset += StackOffset::get(Fixed: ScalarLocalVarSize, Scalable: RVFI->getRVVStackSize());
1515 }
1516 return Offset;
1517}
1518
1519void RISCVFrameLowering::determineCalleeSaves(MachineFunction &MF,
1520 BitVector &SavedRegs,
1521 RegScavenger *RS) const {
1522 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
1523 // Unconditionally spill RA and FP only if the function uses a frame
1524 // pointer.
1525 if (hasFP(MF)) {
1526 SavedRegs.set(RAReg);
1527 SavedRegs.set(FPReg);
1528 }
1529 // Mark BP as used if function has dedicated base pointer.
1530 if (hasBP(MF))
1531 SavedRegs.set(RISCVABI::getBPReg());
1532
1533 // When using cm.push/pop we must save X27 if we save X26.
1534 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1535 if (RVFI->isPushable(MF) && SavedRegs.test(Idx: RISCV::X26))
1536 SavedRegs.set(RISCV::X27);
1537
1538 // SiFive Preemptible Interrupt Handlers need additional frame entries
1539 createSiFivePreemptibleInterruptFrameEntries(MF, RVFI&: *RVFI);
1540}
1541
1542std::pair<int64_t, Align>
1543RISCVFrameLowering::assignRVVStackObjectOffsets(MachineFunction &MF) const {
1544 MachineFrameInfo &MFI = MF.getFrameInfo();
1545 // Create a buffer of RVV objects to allocate.
1546 SmallVector<int, 8> ObjectsToAllocate;
1547 auto pushRVVObjects = [&](int FIBegin, int FIEnd) {
1548 for (int I = FIBegin, E = FIEnd; I != E; ++I) {
1549 unsigned StackID = MFI.getStackID(ObjectIdx: I);
1550 if (StackID != TargetStackID::ScalableVector)
1551 continue;
1552 if (MFI.isDeadObjectIndex(ObjectIdx: I))
1553 continue;
1554
1555 ObjectsToAllocate.push_back(Elt: I);
1556 }
1557 };
1558 // First push RVV Callee Saved object, then push RVV stack object
1559 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
1560 const auto &RVVCSI = getRVVCalleeSavedInfo(MF, CSI);
1561 if (!RVVCSI.empty())
1562 pushRVVObjects(RVVCSI[0].getFrameIdx(),
1563 RVVCSI[RVVCSI.size() - 1].getFrameIdx() + 1);
1564 pushRVVObjects(0, MFI.getObjectIndexEnd() - RVVCSI.size());
1565
1566 // The minimum alignment is 16 bytes.
1567 Align RVVStackAlign(16);
1568 const auto &ST = MF.getSubtarget<RISCVSubtarget>();
1569
1570 if (!ST.hasVInstructions()) {
1571 assert(ObjectsToAllocate.empty() &&
1572 "Can't allocate scalable-vector objects without V instructions");
1573 return std::make_pair(x: 0, y&: RVVStackAlign);
1574 }
1575
1576 // Allocate all RVV locals and spills
1577 int64_t Offset = 0;
1578 for (int FI : ObjectsToAllocate) {
1579 // ObjectSize in bytes.
1580 int64_t ObjectSize = MFI.getObjectSize(ObjectIdx: FI);
1581 auto ObjectAlign =
1582 std::max(a: Align(RISCV::RVVBytesPerBlock), b: MFI.getObjectAlign(ObjectIdx: FI));
1583 // If the data type is the fractional vector type, reserve one vector
1584 // register for it.
1585 if (ObjectSize < RISCV::RVVBytesPerBlock)
1586 ObjectSize = RISCV::RVVBytesPerBlock;
1587 Offset = alignTo(Size: Offset + ObjectSize, A: ObjectAlign);
1588 MFI.setObjectOffset(ObjectIdx: FI, SPOffset: -Offset);
1589 // Update the maximum alignment of the RVV stack section
1590 RVVStackAlign = std::max(a: RVVStackAlign, b: ObjectAlign);
1591 }
1592
1593 uint64_t StackSize = Offset;
1594
1595 // Ensure the alignment of the RVV stack. Since we want the most-aligned
1596 // object right at the bottom (i.e., any padding at the top of the frame),
1597 // readjust all RVV objects down by the alignment padding.
1598 // Stack size and offsets are multiples of vscale, stack alignment is in
1599 // bytes, we can divide stack alignment by minimum vscale to get a maximum
1600 // stack alignment multiple of vscale.
1601 auto VScale =
1602 std::max<uint64_t>(a: ST.getRealMinVLen() / RISCV::RVVBitsPerBlock, b: 1);
1603 if (auto RVVStackAlignVScale = RVVStackAlign.value() / VScale) {
1604 if (auto AlignmentPadding =
1605 offsetToAlignment(Value: StackSize, Alignment: Align(RVVStackAlignVScale))) {
1606 StackSize += AlignmentPadding;
1607 for (int FI : ObjectsToAllocate)
1608 MFI.setObjectOffset(ObjectIdx: FI, SPOffset: MFI.getObjectOffset(ObjectIdx: FI) - AlignmentPadding);
1609 }
1610 }
1611
1612 return std::make_pair(x&: StackSize, y&: RVVStackAlign);
1613}
1614
1615static unsigned getScavSlotsNumForRVV(MachineFunction &MF) {
1616 // For RVV spill, scalable stack offsets computing requires up to two scratch
1617 // registers
1618 static constexpr unsigned ScavSlotsNumRVVSpillScalableObject = 2;
1619
1620 // For RVV spill, non-scalable stack offsets computing requires up to one
1621 // scratch register.
1622 static constexpr unsigned ScavSlotsNumRVVSpillNonScalableObject = 1;
1623
1624 // ADDI instruction's destination register can be used for computing
1625 // offsets. So Scalable stack offsets require up to one scratch register.
1626 static constexpr unsigned ScavSlotsADDIScalableObject = 1;
1627
1628 static constexpr unsigned MaxScavSlotsNumKnown =
1629 std::max(l: {ScavSlotsADDIScalableObject, ScavSlotsNumRVVSpillScalableObject,
1630 ScavSlotsNumRVVSpillNonScalableObject});
1631
1632 unsigned MaxScavSlotsNum = 0;
1633 if (!MF.getSubtarget<RISCVSubtarget>().hasVInstructions())
1634 return false;
1635 for (const MachineBasicBlock &MBB : MF)
1636 for (const MachineInstr &MI : MBB) {
1637 bool IsRVVSpill = RISCV::isRVVSpill(MI);
1638 for (auto &MO : MI.operands()) {
1639 if (!MO.isFI())
1640 continue;
1641 bool IsScalableVectorID = MF.getFrameInfo().getStackID(ObjectIdx: MO.getIndex()) ==
1642 TargetStackID::ScalableVector;
1643 if (IsRVVSpill) {
1644 MaxScavSlotsNum = std::max(
1645 a: MaxScavSlotsNum, b: IsScalableVectorID
1646 ? ScavSlotsNumRVVSpillScalableObject
1647 : ScavSlotsNumRVVSpillNonScalableObject);
1648 } else if (MI.getOpcode() == RISCV::ADDI && IsScalableVectorID) {
1649 MaxScavSlotsNum =
1650 std::max(a: MaxScavSlotsNum, b: ScavSlotsADDIScalableObject);
1651 }
1652 }
1653 if (MaxScavSlotsNum == MaxScavSlotsNumKnown)
1654 return MaxScavSlotsNumKnown;
1655 }
1656 return MaxScavSlotsNum;
1657}
1658
1659static bool hasRVVFrameObject(const MachineFunction &MF) {
1660 // Originally, the function will scan all the stack objects to check whether
1661 // if there is any scalable vector object on the stack or not. However, it
1662 // causes errors in the register allocator. In issue 53016, it returns false
1663 // before RA because there is no RVV stack objects. After RA, it returns true
1664 // because there are spilling slots for RVV values during RA. It will not
1665 // reserve BP during register allocation and generate BP access in the PEI
1666 // pass due to the inconsistent behavior of the function.
1667 //
1668 // The function is changed to use hasVInstructions() as the return value. It
1669 // is not precise, but it can make the register allocation correct.
1670 //
1671 // FIXME: Find a better way to make the decision or revisit the solution in
1672 // D103622.
1673 //
1674 // Refer to https://github.com/llvm/llvm-project/issues/53016.
1675 return MF.getSubtarget<RISCVSubtarget>().hasVInstructions();
1676}
1677
1678static unsigned estimateFunctionSizeInBytes(const MachineFunction &MF,
1679 const RISCVInstrInfo &TII) {
1680 unsigned FnSize = 0;
1681 for (auto &MBB : MF) {
1682 for (auto &MI : MBB) {
1683 // Far branches over 20-bit offset will be relaxed in branch relaxation
1684 // pass. In the worst case, conditional branches will be relaxed into
1685 // the following instruction sequence. Unconditional branches are
1686 // relaxed in the same way, with the exception that there is no first
1687 // branch instruction.
1688 //
1689 // foo
1690 // bne t5, t6, .rev_cond # `TII->getInstSizeInBytes(MI)` bytes
1691 // sd s11, 0(sp) # 4 bytes, or 2 bytes with Zca
1692 // jump .restore, s11 # 8 bytes
1693 // .rev_cond
1694 // bar
1695 // j .dest_bb # 4 bytes, or 2 bytes with Zca
1696 // .restore:
1697 // ld s11, 0(sp) # 4 bytes, or 2 bytes with Zca
1698 // .dest:
1699 // baz
1700 if (MI.isConditionalBranch())
1701 FnSize += TII.getInstSizeInBytes(MI);
1702 if (MI.isConditionalBranch() || MI.isUnconditionalBranch()) {
1703 if (MF.getSubtarget<RISCVSubtarget>().hasStdExtZca())
1704 FnSize += 2 + 8 + 2 + 2;
1705 else
1706 FnSize += 4 + 8 + 4 + 4;
1707 continue;
1708 }
1709
1710 FnSize += TII.getInstSizeInBytes(MI);
1711 }
1712 }
1713 return FnSize;
1714}
1715
1716void RISCVFrameLowering::processFunctionBeforeFrameFinalized(
1717 MachineFunction &MF, RegScavenger *RS) const {
1718 const RISCVRegisterInfo *RegInfo =
1719 MF.getSubtarget<RISCVSubtarget>().getRegisterInfo();
1720 const RISCVInstrInfo *TII = MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
1721 MachineFrameInfo &MFI = MF.getFrameInfo();
1722 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1723 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1724
1725 int64_t RVVStackSize;
1726 Align RVVStackAlign;
1727 std::tie(args&: RVVStackSize, args&: RVVStackAlign) = assignRVVStackObjectOffsets(MF);
1728
1729 RVFI->setRVVStackSize(RVVStackSize);
1730 RVFI->setRVVStackAlign(RVVStackAlign);
1731
1732 if (hasRVVFrameObject(MF)) {
1733 // Ensure the entire stack is aligned to at least the RVV requirement: some
1734 // scalable-vector object alignments are not considered by the
1735 // target-independent code.
1736 MFI.ensureMaxAlignment(Alignment: RVVStackAlign);
1737 }
1738
1739 unsigned ScavSlotsNum = 0;
1740
1741 // estimateStackSize has been observed to under-estimate the final stack
1742 // size, so give ourselves wiggle-room by checking for stack size
1743 // representable an 11-bit signed field rather than 12-bits.
1744 if (!isInt<11>(x: MFI.estimateStackSize(MF)))
1745 ScavSlotsNum = 1;
1746
1747 // Far branches over 20-bit offset require a spill slot for scratch register.
1748 bool IsLargeFunction = !isInt<20>(x: estimateFunctionSizeInBytes(MF, TII: *TII));
1749 if (IsLargeFunction)
1750 ScavSlotsNum = std::max(a: ScavSlotsNum, b: 1u);
1751
1752 // RVV loads & stores have no capacity to hold the immediate address offsets
1753 // so we must always reserve an emergency spill slot if the MachineFunction
1754 // contains any RVV spills.
1755 ScavSlotsNum = std::max(a: ScavSlotsNum, b: getScavSlotsNumForRVV(MF));
1756
1757 for (unsigned I = 0; I < ScavSlotsNum; I++) {
1758 int FI = MFI.CreateSpillStackObject(Size: RegInfo->getSpillSize(RC: *RC),
1759 Alignment: RegInfo->getSpillAlign(RC: *RC));
1760 RS->addScavengingFrameIndex(FI);
1761
1762 if (IsLargeFunction && RVFI->getBranchRelaxationScratchFrameIndex() == -1)
1763 RVFI->setBranchRelaxationScratchFrameIndex(FI);
1764 }
1765
1766 unsigned Size = RVFI->getReservedSpillsSize();
1767 for (const auto &Info : MFI.getCalleeSavedInfo()) {
1768 int FrameIdx = Info.getFrameIdx();
1769 if (FrameIdx < 0 || MFI.getStackID(ObjectIdx: FrameIdx) != TargetStackID::Default)
1770 continue;
1771
1772 Size += MFI.getObjectSize(ObjectIdx: FrameIdx);
1773 }
1774 RVFI->setCalleeSavedStackSize(Size);
1775}
1776
1777// Not preserve stack space within prologue for outgoing variables when the
1778// function contains variable size objects or there are vector objects accessed
1779// by the frame pointer.
1780// Let eliminateCallFramePseudoInstr preserve stack space for it.
1781bool RISCVFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
1782 return !MF.getFrameInfo().hasVarSizedObjects() &&
1783 !(hasFP(MF) && hasRVVFrameObject(MF));
1784}
1785
1786// Eliminate ADJCALLSTACKDOWN, ADJCALLSTACKUP pseudo instructions.
1787MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr(
1788 MachineFunction &MF, MachineBasicBlock &MBB,
1789 MachineBasicBlock::iterator MI) const {
1790 DebugLoc DL = MI->getDebugLoc();
1791
1792 if (!hasReservedCallFrame(MF)) {
1793 // If space has not been reserved for a call frame, ADJCALLSTACKDOWN and
1794 // ADJCALLSTACKUP must be converted to instructions manipulating the stack
1795 // pointer. This is necessary when there is a variable length stack
1796 // allocation (e.g. alloca), which means it's not possible to allocate
1797 // space for outgoing arguments from within the function prologue.
1798 int64_t Amount = MI->getOperand(i: 0).getImm();
1799
1800 if (Amount != 0) {
1801 // Ensure the stack remains aligned after adjustment.
1802 Amount = alignSPAdjust(SPAdj: Amount);
1803
1804 if (MI->getOpcode() == RISCV::ADJCALLSTACKDOWN)
1805 Amount = -Amount;
1806
1807 const RISCVTargetLowering *TLI =
1808 MF.getSubtarget<RISCVSubtarget>().getTargetLowering();
1809 int64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign: getStackAlign());
1810 if (TLI->hasInlineStackProbe(MF) && -Amount >= ProbeSize) {
1811 // When stack probing is enabled, the decrement of SP may need to be
1812 // probed. We can handle both the decrement and the probing in
1813 // allocateStack.
1814 bool DynAllocation =
1815 MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
1816 allocateStack(MBB, MBBI: MI, MF, Offset: -Amount, RealStackSize: -Amount, EmitCFI: !hasFP(MF),
1817 /*NeedProbe=*/true, ProbeSize, DynAllocation);
1818 } else {
1819 const RISCVRegisterInfo &RI = *STI.getRegisterInfo();
1820 RI.adjustReg(MBB, II: MI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: Amount),
1821 Flag: MachineInstr::NoFlags, RequiredAlign: getStackAlign());
1822 }
1823 }
1824 }
1825
1826 return MBB.erase(I: MI);
1827}
1828
1829// We would like to split the SP adjustment to reduce prologue/epilogue
1830// as following instructions. In this way, the offset of the callee saved
1831// register could fit in a single store. Supposed that the first sp adjust
1832// amount is 2032.
1833// add sp,sp,-2032
1834// sw ra,2028(sp)
1835// sw s0,2024(sp)
1836// sw s1,2020(sp)
1837// sw s3,2012(sp)
1838// sw s4,2008(sp)
1839// add sp,sp,-64
1840uint64_t
1841RISCVFrameLowering::getFirstSPAdjustAmount(const MachineFunction &MF) const {
1842 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1843 const MachineFrameInfo &MFI = MF.getFrameInfo();
1844 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
1845 uint64_t StackSize = getStackSizeWithRVVPadding(MF);
1846
1847 // Disable SplitSPAdjust if save-restore libcall, push/pop or QCI interrupts
1848 // are used. The callee-saved registers will be pushed by the save-restore
1849 // libcalls, so we don't have to split the SP adjustment in this case.
1850 if (RVFI->getReservedSpillsSize())
1851 return 0;
1852
1853 // Return the FirstSPAdjustAmount if the StackSize can not fit in a signed
1854 // 12-bit and there exists a callee-saved register needing to be pushed.
1855 if (!isInt<12>(x: StackSize) && (CSI.size() > 0)) {
1856 // FirstSPAdjustAmount is chosen at most as (2048 - StackAlign) because
1857 // 2048 will cause sp = sp + 2048 in the epilogue to be split into multiple
1858 // instructions. Offsets smaller than 2048 can fit in a single load/store
1859 // instruction, and we have to stick with the stack alignment. 2048 has
1860 // 16-byte alignment. The stack alignment for RV32 and RV64 is 16 and for
1861 // RV32E it is 4. So (2048 - StackAlign) will satisfy the stack alignment.
1862 const uint64_t StackAlign = getStackAlign().value();
1863
1864 // Amount of (2048 - StackAlign) will prevent callee saved and restored
1865 // instructions be compressed, so try to adjust the amount to the largest
1866 // offset that stack compression instructions accept when target supports
1867 // compression instructions.
1868 if (STI.hasStdExtZca()) {
1869 // The compression extensions may support the following instructions:
1870 // riscv32: c.lwsp rd, offset[7:2] => 2^(6 + 2)
1871 // c.swsp rs2, offset[7:2] => 2^(6 + 2)
1872 // c.flwsp rd, offset[7:2] => 2^(6 + 2)
1873 // c.fswsp rs2, offset[7:2] => 2^(6 + 2)
1874 // riscv64: c.ldsp rd, offset[8:3] => 2^(6 + 3)
1875 // c.sdsp rs2, offset[8:3] => 2^(6 + 3)
1876 // c.fldsp rd, offset[8:3] => 2^(6 + 3)
1877 // c.fsdsp rs2, offset[8:3] => 2^(6 + 3)
1878 const uint64_t RVCompressLen = STI.getXLen() * 8;
1879 // Compared with amount (2048 - StackAlign), StackSize needs to
1880 // satisfy the following conditions to avoid using more instructions
1881 // to adjust the sp after adjusting the amount, such as
1882 // StackSize meets the condition (StackSize <= 2048 + RVCompressLen),
1883 // case1: Amount is 2048 - StackAlign: use addi + addi to adjust sp.
1884 // case2: Amount is RVCompressLen: use addi + addi to adjust sp.
1885 auto CanCompress = [&](uint64_t CompressLen) -> bool {
1886 if (StackSize <= 2047 + CompressLen ||
1887 (StackSize > 2048 * 2 - StackAlign &&
1888 StackSize <= 2047 * 2 + CompressLen) ||
1889 StackSize > 2048 * 3 - StackAlign)
1890 return true;
1891
1892 return false;
1893 };
1894 // In the epilogue, addi sp, sp, 496 is used to recover the sp and it
1895 // can be compressed(C.ADDI16SP, offset can be [-512, 496]), but
1896 // addi sp, sp, 512 can not be compressed. So try to use 496 first.
1897 const uint64_t ADDI16SPCompressLen = 496;
1898 if (STI.is64Bit() && CanCompress(ADDI16SPCompressLen))
1899 return ADDI16SPCompressLen;
1900 if (CanCompress(RVCompressLen))
1901 return RVCompressLen;
1902 }
1903 return 2048 - StackAlign;
1904 }
1905 return 0;
1906}
1907
1908bool RISCVFrameLowering::assignCalleeSavedSpillSlots(
1909 MachineFunction &MF, const TargetRegisterInfo *TRI,
1910 std::vector<CalleeSavedInfo> &CSI, unsigned &MinCSFrameIndex,
1911 unsigned &MaxCSFrameIndex) const {
1912 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1913
1914 // Preemptible Interrupts have two additional Callee-save Frame Indexes,
1915 // not tracked by `CSI`.
1916 if (RVFI->isSiFivePreemptibleInterrupt(MF)) {
1917 for (int I = 0; I < 2; ++I) {
1918 int FI = RVFI->getInterruptCSRFrameIndex(Idx: I);
1919 MinCSFrameIndex = std::min<unsigned>(a: MinCSFrameIndex, b: FI);
1920 MaxCSFrameIndex = std::max<unsigned>(a: MaxCSFrameIndex, b: FI);
1921 }
1922 }
1923
1924 // Early exit if no callee saved registers are modified!
1925 if (CSI.empty())
1926 return true;
1927
1928 if (RVFI->useQCIInterrupt(MF)) {
1929 RVFI->setQCIInterruptStackSize(QCIInterruptPushAmount);
1930 }
1931
1932 if (RVFI->isPushable(MF)) {
1933 // Determine how many GPRs we need to push and save it to RVFI.
1934 unsigned PushedRegNum = getNumPushPopRegs(CSI);
1935
1936 // `QC.C.MIENTER(.NEST)` will save `ra` and `s0`, so we should only push if
1937 // we want to push more than 2 registers. Otherwise, we should push if we
1938 // want to push more than 0 registers.
1939 unsigned OnlyPushIfMoreThan = RVFI->useQCIInterrupt(MF) ? 2 : 0;
1940 if (PushedRegNum > OnlyPushIfMoreThan) {
1941 RVFI->setRVPushRegs(PushedRegNum);
1942 RVFI->setRVPushStackSize(alignTo(Value: (STI.getXLen() / 8) * PushedRegNum, Align: 16));
1943 }
1944 }
1945
1946 MachineFrameInfo &MFI = MF.getFrameInfo();
1947 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
1948
1949 for (auto &CS : CSI) {
1950 MCRegister Reg = CS.getReg();
1951 const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
1952 unsigned Size = RegInfo->getSpillSize(RC: *RC);
1953
1954 if (RVFI->useQCIInterrupt(MF)) {
1955 const auto *FFI = llvm::find_if(Range: FixedCSRFIQCIInterruptMap, P: [&](auto P) {
1956 return P.first == CS.getReg();
1957 });
1958 if (FFI != std::end(arr: FixedCSRFIQCIInterruptMap)) {
1959 int64_t Offset = FFI->second * (int64_t)Size;
1960
1961 int FrameIdx = MFI.CreateFixedSpillStackObject(Size, SPOffset: Offset);
1962 assert(FrameIdx < 0);
1963 CS.setFrameIdx(FrameIdx);
1964 continue;
1965 }
1966 }
1967
1968 if (RVFI->useSaveRestoreLibCalls(MF) || RVFI->isPushable(MF)) {
1969 const auto *FII = llvm::find_if(
1970 Range: FixedCSRFIMap, P: [&](MCPhysReg P) { return P == CS.getReg(); });
1971 unsigned RegNum = std::distance(first: std::begin(arr: FixedCSRFIMap), last: FII);
1972
1973 if (FII != std::end(arr: FixedCSRFIMap)) {
1974 int64_t Offset;
1975 if (RVFI->getPushPopKind(MF) ==
1976 RISCVMachineFunctionInfo::PushPopKind::StdExtZcmp)
1977 Offset = -int64_t(RVFI->getRVPushRegs() - RegNum) * Size;
1978 else
1979 Offset = -int64_t(RegNum + 1) * Size;
1980
1981 if (RVFI->useQCIInterrupt(MF))
1982 Offset -= QCIInterruptPushAmount;
1983
1984 int FrameIdx = MFI.CreateFixedSpillStackObject(Size, SPOffset: Offset);
1985 assert(FrameIdx < 0);
1986 CS.setFrameIdx(FrameIdx);
1987 continue;
1988 }
1989 }
1990
1991 // Not a fixed slot.
1992 Align Alignment = RegInfo->getSpillAlign(RC: *RC);
1993 // We may not be able to satisfy the desired alignment specification of
1994 // the TargetRegisterClass if the stack alignment is smaller. Use the
1995 // min.
1996 Alignment = std::min(a: Alignment, b: getStackAlign());
1997 int FrameIdx = MFI.CreateStackObject(Size, Alignment, isSpillSlot: true);
1998 if ((unsigned)FrameIdx < MinCSFrameIndex)
1999 MinCSFrameIndex = FrameIdx;
2000 if ((unsigned)FrameIdx > MaxCSFrameIndex)
2001 MaxCSFrameIndex = FrameIdx;
2002 CS.setFrameIdx(FrameIdx);
2003 if (RISCVRegisterInfo::isRVVRegClass(RC))
2004 MFI.setStackID(ObjectIdx: FrameIdx, ID: TargetStackID::ScalableVector);
2005 }
2006
2007 if (RVFI->useQCIInterrupt(MF)) {
2008 // Allocate a fixed object that covers the entire QCI stack allocation,
2009 // because there are gaps which are reserved for future use.
2010 MFI.CreateFixedSpillStackObject(
2011 Size: QCIInterruptPushAmount, SPOffset: -static_cast<int64_t>(QCIInterruptPushAmount));
2012 }
2013
2014 if (RVFI->isPushable(MF)) {
2015 int64_t QCIOffset = RVFI->useQCIInterrupt(MF) ? QCIInterruptPushAmount : 0;
2016 // Allocate a fixed object that covers the full push.
2017 if (int64_t PushSize = RVFI->getRVPushStackSize())
2018 MFI.CreateFixedSpillStackObject(Size: PushSize, SPOffset: -PushSize - QCIOffset);
2019 } else if (int LibCallRegs = getLibCallID(MF, CSI) + 1) {
2020 int64_t LibCallFrameSize =
2021 alignTo(Size: (STI.getXLen() / 8) * LibCallRegs, A: getStackAlign());
2022 MFI.CreateFixedSpillStackObject(Size: LibCallFrameSize, SPOffset: -LibCallFrameSize);
2023 }
2024
2025 return true;
2026}
2027
2028bool RISCVFrameLowering::spillCalleeSavedRegisters(
2029 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2030 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
2031 if (CSI.empty())
2032 return true;
2033
2034 MachineFunction *MF = MBB.getParent();
2035 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
2036 DebugLoc DL;
2037 if (MI != MBB.end() && !MI->isDebugInstr())
2038 DL = MI->getDebugLoc();
2039
2040 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2041 if (RVFI->useQCIInterrupt(MF: *MF)) {
2042 // Emit QC.C.MIENTER(.NEST)
2043 BuildMI(
2044 BB&: MBB, I: MI, MIMD: DL,
2045 MCID: TII.get(Opcode: RVFI->getInterruptStackKind(MF: *MF) ==
2046 RISCVMachineFunctionInfo::InterruptStackKind::QCINest
2047 ? RISCV::QC_C_MIENTER_NEST
2048 : RISCV::QC_C_MIENTER))
2049 .setMIFlag(MachineInstr::FrameSetup);
2050
2051 for (auto [Reg, _Offset] : FixedCSRFIQCIInterruptMap)
2052 MBB.addLiveIn(PhysReg: Reg);
2053 }
2054
2055 if (RVFI->isPushable(MF: *MF)) {
2056 // Emit CM.PUSH with base StackAdj & evaluate Push stack
2057 unsigned PushedRegNum = RVFI->getRVPushRegs();
2058 if (PushedRegNum > 0) {
2059 // Use encoded number to represent registers to spill.
2060 unsigned Opcode = getPushOpcode(
2061 Kind: RVFI->getPushPopKind(MF: *MF), UpdateFP: hasFP(MF: *MF) && !RVFI->useQCIInterrupt(MF: *MF));
2062 unsigned RegEnc = RISCVZC::encodeRegListNumRegs(NumRegs: PushedRegNum);
2063 MachineInstrBuilder PushBuilder =
2064 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode))
2065 .setMIFlag(MachineInstr::FrameSetup);
2066 PushBuilder.addImm(Val: RegEnc);
2067 PushBuilder.addImm(Val: 0);
2068
2069 for (unsigned i = 0; i < PushedRegNum; i++)
2070 PushBuilder.addUse(RegNo: FixedCSRFIMap[i], Flags: RegState::Implicit);
2071 }
2072 } else if (const char *SpillLibCall = getSpillLibCallName(MF: *MF, CSI)) {
2073 // Add spill libcall via non-callee-saved register t0.
2074 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::PseudoCALLReg), DestReg: RISCV::X5)
2075 .addExternalSymbol(FnName: SpillLibCall, TargetFlags: RISCVII::MO_CALL)
2076 .setMIFlag(MachineInstr::FrameSetup);
2077
2078 // Add registers spilled in libcall as liveins.
2079 for (auto &CS : CSI)
2080 MBB.addLiveIn(PhysReg: CS.getReg());
2081 }
2082
2083 // Manually spill values not spilled by libcall & Push/Pop.
2084 const auto &UnmanagedCSI = getUnmanagedCSI(MF: *MF, CSI);
2085 const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI);
2086
2087 auto storeRegsToStackSlots = [&](decltype(UnmanagedCSI) CSInfo) {
2088 for (auto &CS : CSInfo) {
2089 // Insert the spill to the stack frame.
2090 MCRegister Reg = CS.getReg();
2091 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
2092 TII.storeRegToStackSlot(MBB, MI, SrcReg: Reg, isKill: !MBB.isLiveIn(Reg),
2093 FrameIndex: CS.getFrameIdx(), RC, TRI, VReg: Register(),
2094 Flags: MachineInstr::FrameSetup);
2095 }
2096 };
2097 storeRegsToStackSlots(UnmanagedCSI);
2098 storeRegsToStackSlots(RVVCSI);
2099
2100 return true;
2101}
2102
2103static unsigned getCalleeSavedRVVNumRegs(const Register &BaseReg) {
2104 return RISCV::VRRegClass.contains(Reg: BaseReg) ? 1
2105 : RISCV::VRM2RegClass.contains(Reg: BaseReg) ? 2
2106 : RISCV::VRM4RegClass.contains(Reg: BaseReg) ? 4
2107 : 8;
2108}
2109
2110static MCRegister getRVVBaseRegister(const RISCVRegisterInfo &TRI,
2111 const Register &Reg) {
2112 MCRegister BaseReg = TRI.getSubReg(Reg, Idx: RISCV::sub_vrm1_0);
2113 // If it's not a grouped vector register, it doesn't have subregister, so
2114 // the base register is just itself.
2115 if (BaseReg == RISCV::NoRegister)
2116 BaseReg = Reg;
2117 return BaseReg;
2118}
2119
2120void RISCVFrameLowering::emitCalleeSavedRVVPrologCFI(
2121 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, bool HasFP) const {
2122 MachineFunction *MF = MBB.getParent();
2123 const MachineFrameInfo &MFI = MF->getFrameInfo();
2124 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2125 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
2126
2127 const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI: MFI.getCalleeSavedInfo());
2128 if (RVVCSI.empty())
2129 return;
2130
2131 uint64_t FixedSize = getStackSizeWithRVVPadding(MF: *MF);
2132 if (!HasFP) {
2133 uint64_t ScalarLocalVarSize =
2134 MFI.getStackSize() - RVFI->getCalleeSavedStackSize() -
2135 RVFI->getVarArgsSaveSize() + RVFI->getRVVPadding();
2136 FixedSize -= ScalarLocalVarSize;
2137 }
2138
2139 CFIInstBuilder CFIBuilder(MBB, MI, MachineInstr::FrameSetup);
2140 for (auto &CS : RVVCSI) {
2141 // Insert the spill to the stack frame.
2142 int FI = CS.getFrameIdx();
2143 MCRegister BaseReg = getRVVBaseRegister(TRI, Reg: CS.getReg());
2144 unsigned NumRegs = getCalleeSavedRVVNumRegs(BaseReg: CS.getReg());
2145 for (unsigned i = 0; i < NumRegs; ++i) {
2146 CFIBuilder.insertCFIInst(CFIInst: createDefCFAOffset(
2147 TRI, Reg: BaseReg + i, FixedOffset: -FixedSize, ScalableOffset: MFI.getObjectOffset(ObjectIdx: FI) / 8 + i));
2148 }
2149 }
2150}
2151
2152void RISCVFrameLowering::emitCalleeSavedRVVEpilogCFI(
2153 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const {
2154 MachineFunction *MF = MBB.getParent();
2155 const MachineFrameInfo &MFI = MF->getFrameInfo();
2156 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
2157
2158 CFIInstBuilder CFIHelper(MBB, MI, MachineInstr::FrameDestroy);
2159 const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI: MFI.getCalleeSavedInfo());
2160 for (auto &CS : RVVCSI) {
2161 MCRegister BaseReg = getRVVBaseRegister(TRI, Reg: CS.getReg());
2162 unsigned NumRegs = getCalleeSavedRVVNumRegs(BaseReg: CS.getReg());
2163 for (unsigned i = 0; i < NumRegs; ++i)
2164 CFIHelper.buildRestore(Reg: BaseReg + i);
2165 }
2166}
2167
2168bool RISCVFrameLowering::restoreCalleeSavedRegisters(
2169 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2170 MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
2171 if (CSI.empty())
2172 return true;
2173
2174 MachineFunction *MF = MBB.getParent();
2175 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
2176 DebugLoc DL;
2177 if (MI != MBB.end() && !MI->isDebugInstr())
2178 DL = MI->getDebugLoc();
2179
2180 // Manually restore values not restored by libcall & Push/Pop.
2181 // Reverse the restore order in epilog. In addition, the return
2182 // address will be restored first in the epilogue. It increases
2183 // the opportunity to avoid the load-to-use data hazard between
2184 // loading RA and return by RA. loadRegFromStackSlot can insert
2185 // multiple instructions.
2186 const auto &UnmanagedCSI = getUnmanagedCSI(MF: *MF, CSI);
2187 const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI);
2188
2189 auto loadRegFromStackSlot = [&](decltype(UnmanagedCSI) CSInfo) {
2190 for (auto &CS : CSInfo) {
2191 MCRegister Reg = CS.getReg();
2192 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
2193 TII.loadRegFromStackSlot(MBB, MI, DestReg: Reg, FrameIndex: CS.getFrameIdx(), RC, TRI,
2194 VReg: Register(), Flags: MachineInstr::FrameDestroy);
2195 assert(MI != MBB.begin() &&
2196 "loadRegFromStackSlot didn't insert any code!");
2197 }
2198 };
2199 loadRegFromStackSlot(RVVCSI);
2200 loadRegFromStackSlot(UnmanagedCSI);
2201
2202 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2203 if (RVFI->useQCIInterrupt(MF: *MF)) {
2204 // Don't emit anything here because restoration is handled by
2205 // QC.C.MILEAVERET which we already inserted to return.
2206 assert(MI->getOpcode() == RISCV::QC_C_MILEAVERET &&
2207 "Unexpected QCI Interrupt Return Instruction");
2208 }
2209
2210 if (RVFI->isPushable(MF: *MF)) {
2211 unsigned PushedRegNum = RVFI->getRVPushRegs();
2212 if (PushedRegNum > 0) {
2213 unsigned Opcode = getPopOpcode(Kind: RVFI->getPushPopKind(MF: *MF));
2214 unsigned RegEnc = RISCVZC::encodeRegListNumRegs(NumRegs: PushedRegNum);
2215 MachineInstrBuilder PopBuilder =
2216 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode))
2217 .setMIFlag(MachineInstr::FrameDestroy);
2218 // Use encoded number to represent registers to restore.
2219 PopBuilder.addImm(Val: RegEnc);
2220 PopBuilder.addImm(Val: 0);
2221
2222 for (unsigned i = 0; i < RVFI->getRVPushRegs(); i++)
2223 PopBuilder.addDef(RegNo: FixedCSRFIMap[i], Flags: RegState::ImplicitDefine);
2224 }
2225 } else {
2226 const char *RestoreLibCall = getRestoreLibCallName(MF: *MF, CSI);
2227 if (RestoreLibCall) {
2228 // Add restore libcall via tail call.
2229 MachineBasicBlock::iterator NewMI =
2230 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::PseudoTAIL))
2231 .addExternalSymbol(FnName: RestoreLibCall, TargetFlags: RISCVII::MO_CALL)
2232 .setMIFlag(MachineInstr::FrameDestroy);
2233
2234 // Remove trailing returns, since the terminator is now a tail call to the
2235 // restore function.
2236 if (MI != MBB.end() && MI->getOpcode() == RISCV::PseudoRET) {
2237 NewMI->copyImplicitOps(MF&: *MF, MI: *MI);
2238 MI->eraseFromParent();
2239 }
2240 }
2241 }
2242 return true;
2243}
2244
2245bool RISCVFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
2246 // Keep the conventional code flow when not optimizing.
2247 if (MF.getFunction().hasOptNone())
2248 return false;
2249
2250 return true;
2251}
2252
2253bool RISCVFrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const {
2254 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
2255 const MachineFunction *MF = MBB.getParent();
2256 const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2257
2258 // Make sure VTYPE and VL are not live-in since we will use vsetvli in the
2259 // prologue to get the VLEN, and that will clobber these registers.
2260 //
2261 // We may do also check the stack contains objects with scalable vector type,
2262 // but this will require iterating over all the stack objects, but this may
2263 // not worth since the situation is rare, we could do further check in future
2264 // if we find it is necessary.
2265 if (STI.preferVsetvliOverReadVLENB() &&
2266 (MBB.isLiveIn(Reg: RISCV::VTYPE) || MBB.isLiveIn(Reg: RISCV::VL)))
2267 return false;
2268
2269 if (!RVFI->useSaveRestoreLibCalls(MF: *MF))
2270 return true;
2271
2272 // Inserting a call to a __riscv_save libcall requires the use of the register
2273 // t0 (X5) to hold the return address. Therefore if this register is already
2274 // used we can't insert the call.
2275
2276 RegScavenger RS;
2277 RS.enterBasicBlock(MBB&: *TmpMBB);
2278 return !RS.isRegUsed(Reg: RISCV::X5);
2279}
2280
2281bool RISCVFrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
2282 const MachineFunction *MF = MBB.getParent();
2283 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
2284 const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2285
2286 // We do not want QC.C.MILEAVERET to be subject to shrink-wrapping - it must
2287 // come in the final block of its function as it both pops and returns.
2288 if (RVFI->useQCIInterrupt(MF: *MF))
2289 return MBB.succ_empty();
2290
2291 if (!RVFI->useSaveRestoreLibCalls(MF: *MF))
2292 return true;
2293
2294 // Using the __riscv_restore libcalls to restore CSRs requires a tail call.
2295 // This means if we still need to continue executing code within this function
2296 // the restore cannot take place in this basic block.
2297
2298 if (MBB.succ_size() > 1)
2299 return false;
2300
2301 MachineBasicBlock *SuccMBB =
2302 MBB.succ_empty() ? TmpMBB->getFallThrough() : *MBB.succ_begin();
2303
2304 // Doing a tail call should be safe if there are no successors, because either
2305 // we have a returning block or the end of the block is unreachable, so the
2306 // restore will be eliminated regardless.
2307 if (!SuccMBB)
2308 return true;
2309
2310 // The successor can only contain a return, since we would effectively be
2311 // replacing the successor with our own tail return at the end of our block.
2312 return SuccMBB->isReturnBlock() && SuccMBB->size() == 1;
2313}
2314
2315bool RISCVFrameLowering::isSupportedStackID(TargetStackID::Value ID) const {
2316 switch (ID) {
2317 case TargetStackID::Default:
2318 case TargetStackID::ScalableVector:
2319 return true;
2320 case TargetStackID::NoAlloc:
2321 case TargetStackID::SGPRSpill:
2322 case TargetStackID::WasmLocal:
2323 return false;
2324 }
2325 llvm_unreachable("Invalid TargetStackID::Value");
2326}
2327
2328TargetStackID::Value RISCVFrameLowering::getStackIDForScalableVectors() const {
2329 return TargetStackID::ScalableVector;
2330}
2331
2332// Synthesize the probe loop.
2333static void emitStackProbeInline(MachineBasicBlock::iterator MBBI, DebugLoc DL,
2334 Register TargetReg, bool IsRVV) {
2335 assert(TargetReg != RISCV::X2 && "New top of stack cannot already be in SP");
2336
2337 MachineBasicBlock &MBB = *MBBI->getParent();
2338 MachineFunction &MF = *MBB.getParent();
2339
2340 auto &Subtarget = MF.getSubtarget<RISCVSubtarget>();
2341 const RISCVInstrInfo *TII = Subtarget.getInstrInfo();
2342 bool IsRV64 = Subtarget.is64Bit();
2343 Align StackAlign = Subtarget.getFrameLowering()->getStackAlign();
2344 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
2345 uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
2346
2347 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB.getIterator());
2348 MachineBasicBlock *LoopTestMBB =
2349 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
2350 MF.insert(MBBI: MBBInsertPoint, MBB: LoopTestMBB);
2351 MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
2352 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
2353 MachineInstr::MIFlag Flags = MachineInstr::FrameSetup;
2354 Register ScratchReg = RISCV::X7;
2355
2356 // ScratchReg = ProbeSize
2357 TII->movImm(MBB, MBBI, DL, DstReg: ScratchReg, Val: ProbeSize, Flag: Flags);
2358
2359 // LoopTest:
2360 // SUB SP, SP, ProbeSize
2361 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::SUB), DestReg: SPReg)
2362 .addReg(RegNo: SPReg)
2363 .addReg(RegNo: ScratchReg)
2364 .setMIFlags(Flags);
2365
2366 // s[d|w] zero, 0(sp)
2367 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL,
2368 MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
2369 .addReg(RegNo: RISCV::X0)
2370 .addReg(RegNo: SPReg)
2371 .addImm(Val: 0)
2372 .setMIFlags(Flags);
2373
2374 if (IsRVV) {
2375 // SUB TargetReg, TargetReg, ProbeSize
2376 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::SUB),
2377 DestReg: TargetReg)
2378 .addReg(RegNo: TargetReg)
2379 .addReg(RegNo: ScratchReg)
2380 .setMIFlags(Flags);
2381
2382 // BGE TargetReg, ProbeSize, LoopTest
2383 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::BGE))
2384 .addReg(RegNo: TargetReg)
2385 .addReg(RegNo: ScratchReg)
2386 .addMBB(MBB: LoopTestMBB)
2387 .setMIFlags(Flags);
2388
2389 } else {
2390 // BNE SP, TargetReg, LoopTest
2391 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
2392 .addReg(RegNo: SPReg)
2393 .addReg(RegNo: TargetReg)
2394 .addMBB(MBB: LoopTestMBB)
2395 .setMIFlags(Flags);
2396 }
2397
2398 ExitMBB->splice(Where: ExitMBB->end(), Other: &MBB, From: std::next(x: MBBI), To: MBB.end());
2399 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
2400
2401 LoopTestMBB->addSuccessor(Succ: ExitMBB);
2402 LoopTestMBB->addSuccessor(Succ: LoopTestMBB);
2403 MBB.addSuccessor(Succ: LoopTestMBB);
2404 // Update liveins.
2405 fullyRecomputeLiveIns(MBBs: {ExitMBB, LoopTestMBB});
2406}
2407
2408void RISCVFrameLowering::inlineStackProbe(MachineFunction &MF,
2409 MachineBasicBlock &MBB) const {
2410 // Get the instructions that need to be replaced. We emit at most two of
2411 // these. Remember them in order to avoid complications coming from the need
2412 // to traverse the block while potentially creating more blocks.
2413 SmallVector<MachineInstr *, 4> ToReplace;
2414 for (MachineInstr &MI : MBB) {
2415 unsigned Opc = MI.getOpcode();
2416 if (Opc == RISCV::PROBED_STACKALLOC ||
2417 Opc == RISCV::PROBED_STACKALLOC_RVV) {
2418 ToReplace.push_back(Elt: &MI);
2419 }
2420 }
2421
2422 for (MachineInstr *MI : ToReplace) {
2423 if (MI->getOpcode() == RISCV::PROBED_STACKALLOC ||
2424 MI->getOpcode() == RISCV::PROBED_STACKALLOC_RVV) {
2425 MachineBasicBlock::iterator MBBI = MI->getIterator();
2426 DebugLoc DL = MBB.findDebugLoc(MBBI);
2427 Register TargetReg = MI->getOperand(i: 0).getReg();
2428 emitStackProbeInline(MBBI, DL, TargetReg,
2429 IsRVV: (MI->getOpcode() == RISCV::PROBED_STACKALLOC_RVV));
2430 MBBI->eraseFromParent();
2431 }
2432 }
2433}
2434