1//===-- RISCVFrameLowering.cpp - RISC-V Frame Information -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the RISC-V implementation of TargetFrameLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVFrameLowering.h"
14#include "RISCVMachineFunctionInfo.h"
15#include "RISCVSubtarget.h"
16#include "llvm/BinaryFormat/Dwarf.h"
17#include "llvm/CodeGen/MachineFrameInfo.h"
18#include "llvm/CodeGen/MachineFunction.h"
19#include "llvm/CodeGen/MachineInstrBuilder.h"
20#include "llvm/CodeGen/MachineRegisterInfo.h"
21#include "llvm/CodeGen/RegisterScavenging.h"
22#include "llvm/IR/DiagnosticInfo.h"
23#include "llvm/MC/MCDwarf.h"
24#include "llvm/Support/LEB128.h"
25
26#include <algorithm>
27
28using namespace llvm;
29
30static Align getABIStackAlignment(RISCVABI::ABI ABI) {
31 if (ABI == RISCVABI::ABI_ILP32E)
32 return Align(4);
33 if (ABI == RISCVABI::ABI_LP64E)
34 return Align(8);
35 return Align(16);
36}
37
38RISCVFrameLowering::RISCVFrameLowering(const RISCVSubtarget &STI)
39 : TargetFrameLowering(
40 StackGrowsDown, getABIStackAlignment(ABI: STI.getTargetABI()),
41 /*LocalAreaOffset=*/0,
42 /*TransientStackAlignment=*/getABIStackAlignment(ABI: STI.getTargetABI())),
43 STI(STI) {}
44
45// Offsets which need to be scale by XLen representing locations of CSRs which
46// are given a fixed location by save/restore libcalls or Zcmp Push/Pop.
47static const std::pair<MCPhysReg, int8_t> FixedCSRFIMap[] = {
48 {/*ra*/ RISCV::X1, -1}, {/*s0*/ RISCV::X8, -2},
49 {/*s1*/ RISCV::X9, -3}, {/*s2*/ RISCV::X18, -4},
50 {/*s3*/ RISCV::X19, -5}, {/*s4*/ RISCV::X20, -6},
51 {/*s5*/ RISCV::X21, -7}, {/*s6*/ RISCV::X22, -8},
52 {/*s7*/ RISCV::X23, -9}, {/*s8*/ RISCV::X24, -10},
53 {/*s9*/ RISCV::X25, -11}, {/*s10*/ RISCV::X26, -12},
54 {/*s11*/ RISCV::X27, -13}};
55
56// For now we use x3, a.k.a gp, as pointer to shadow call stack.
57// User should not use x3 in their asm.
58static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
59 MachineBasicBlock::iterator MI,
60 const DebugLoc &DL) {
61 if (!MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack))
62 return;
63
64 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
65 const llvm::RISCVRegisterInfo *TRI = STI.getRegisterInfo();
66 Register RAReg = TRI->getRARegister();
67
68 // Do not save RA to the SCS if it's not saved to the regular stack,
69 // i.e. RA is not at risk of being overwritten.
70 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
71 if (llvm::none_of(
72 Range&: CSI, P: [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
73 return;
74
75 const RISCVInstrInfo *TII = STI.getInstrInfo();
76 if (!STI.hasForcedSWShadowStack() && STI.hasStdExtZicfiss()) {
77 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SSPUSH)).addReg(RegNo: RAReg);
78 return;
79 }
80
81 Register SCSPReg = RISCVABI::getSCSPReg();
82
83 bool IsRV64 = STI.hasFeature(Feature: RISCV::Feature64Bit);
84 int64_t SlotSize = STI.getXLen() / 8;
85 // Store return address to shadow call stack
86 // addi gp, gp, [4|8]
87 // s[w|d] ra, -[4|8](gp)
88 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI))
89 .addReg(RegNo: SCSPReg, flags: RegState::Define)
90 .addReg(RegNo: SCSPReg)
91 .addImm(Val: SlotSize)
92 .setMIFlag(MachineInstr::FrameSetup);
93 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
94 .addReg(RegNo: RAReg)
95 .addReg(RegNo: SCSPReg)
96 .addImm(Val: -SlotSize)
97 .setMIFlag(MachineInstr::FrameSetup);
98
99 // Emit a CFI instruction that causes SlotSize to be subtracted from the value
100 // of the shadow stack pointer when unwinding past this frame.
101 char DwarfSCSReg = TRI->getDwarfRegNum(RegNum: SCSPReg, /*IsEH*/ isEH: true);
102 assert(DwarfSCSReg < 32 && "SCS Register should be < 32 (X3).");
103
104 char Offset = static_cast<char>(-SlotSize) & 0x7f;
105 const char CFIInst[] = {
106 dwarf::DW_CFA_val_expression,
107 DwarfSCSReg, // register
108 2, // length
109 static_cast<char>(unsigned(dwarf::DW_OP_breg0 + DwarfSCSReg)),
110 Offset, // addend (sleb128)
111 };
112
113 unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createEscape(
114 L: nullptr, Vals: StringRef(CFIInst, sizeof(CFIInst))));
115 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
116 .addCFIIndex(CFIIndex)
117 .setMIFlag(MachineInstr::FrameSetup);
118}
119
120static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB,
121 MachineBasicBlock::iterator MI,
122 const DebugLoc &DL) {
123 if (!MF.getFunction().hasFnAttribute(Kind: Attribute::ShadowCallStack))
124 return;
125
126 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
127 Register RAReg = STI.getRegisterInfo()->getRARegister();
128
129 // See emitSCSPrologue() above.
130 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
131 if (llvm::none_of(
132 Range&: CSI, P: [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
133 return;
134
135 const RISCVInstrInfo *TII = STI.getInstrInfo();
136 if (!STI.hasForcedSWShadowStack() && STI.hasStdExtZicfiss()) {
137 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SSPOPCHK)).addReg(RegNo: RAReg);
138 return;
139 }
140
141 Register SCSPReg = RISCVABI::getSCSPReg();
142
143 bool IsRV64 = STI.hasFeature(Feature: RISCV::Feature64Bit);
144 int64_t SlotSize = STI.getXLen() / 8;
145 // Load return address from shadow call stack
146 // l[w|d] ra, -[4|8](gp)
147 // addi gp, gp, -[4|8]
148 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: IsRV64 ? RISCV::LD : RISCV::LW))
149 .addReg(RegNo: RAReg, flags: RegState::Define)
150 .addReg(RegNo: SCSPReg)
151 .addImm(Val: -SlotSize)
152 .setMIFlag(MachineInstr::FrameDestroy);
153 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI))
154 .addReg(RegNo: SCSPReg, flags: RegState::Define)
155 .addReg(RegNo: SCSPReg)
156 .addImm(Val: -SlotSize)
157 .setMIFlag(MachineInstr::FrameDestroy);
158 // Restore the SCS pointer
159 unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createRestore(
160 L: nullptr, Register: STI.getRegisterInfo()->getDwarfRegNum(RegNum: SCSPReg, /*IsEH*/ isEH: true)));
161 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
162 .addCFIIndex(CFIIndex)
163 .setMIFlags(MachineInstr::FrameDestroy);
164}
165
166// Get the ID of the libcall used for spilling and restoring callee saved
167// registers. The ID is representative of the number of registers saved or
168// restored by the libcall, except it is zero-indexed - ID 0 corresponds to a
169// single register.
170static int getLibCallID(const MachineFunction &MF,
171 const std::vector<CalleeSavedInfo> &CSI) {
172 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
173
174 if (CSI.empty() || !RVFI->useSaveRestoreLibCalls(MF))
175 return -1;
176
177 Register MaxReg = RISCV::NoRegister;
178 for (auto &CS : CSI)
179 // assignCalleeSavedSpillSlots assigns negative frame indexes to
180 // registers which can be saved by libcall.
181 if (CS.getFrameIdx() < 0)
182 MaxReg = std::max(a: MaxReg.id(), b: CS.getReg().id());
183
184 if (MaxReg == RISCV::NoRegister)
185 return -1;
186
187 switch (MaxReg) {
188 default:
189 llvm_unreachable("Something has gone wrong!");
190 case /*s11*/ RISCV::X27: return 12;
191 case /*s10*/ RISCV::X26: return 11;
192 case /*s9*/ RISCV::X25: return 10;
193 case /*s8*/ RISCV::X24: return 9;
194 case /*s7*/ RISCV::X23: return 8;
195 case /*s6*/ RISCV::X22: return 7;
196 case /*s5*/ RISCV::X21: return 6;
197 case /*s4*/ RISCV::X20: return 5;
198 case /*s3*/ RISCV::X19: return 4;
199 case /*s2*/ RISCV::X18: return 3;
200 case /*s1*/ RISCV::X9: return 2;
201 case /*s0*/ RISCV::X8: return 1;
202 case /*ra*/ RISCV::X1: return 0;
203 }
204}
205
206// Get the name of the libcall used for spilling callee saved registers.
207// If this function will not use save/restore libcalls, then return a nullptr.
208static const char *
209getSpillLibCallName(const MachineFunction &MF,
210 const std::vector<CalleeSavedInfo> &CSI) {
211 static const char *const SpillLibCalls[] = {
212 "__riscv_save_0",
213 "__riscv_save_1",
214 "__riscv_save_2",
215 "__riscv_save_3",
216 "__riscv_save_4",
217 "__riscv_save_5",
218 "__riscv_save_6",
219 "__riscv_save_7",
220 "__riscv_save_8",
221 "__riscv_save_9",
222 "__riscv_save_10",
223 "__riscv_save_11",
224 "__riscv_save_12"
225 };
226
227 int LibCallID = getLibCallID(MF, CSI);
228 if (LibCallID == -1)
229 return nullptr;
230 return SpillLibCalls[LibCallID];
231}
232
233// Get the name of the libcall used for restoring callee saved registers.
234// If this function will not use save/restore libcalls, then return a nullptr.
235static const char *
236getRestoreLibCallName(const MachineFunction &MF,
237 const std::vector<CalleeSavedInfo> &CSI) {
238 static const char *const RestoreLibCalls[] = {
239 "__riscv_restore_0",
240 "__riscv_restore_1",
241 "__riscv_restore_2",
242 "__riscv_restore_3",
243 "__riscv_restore_4",
244 "__riscv_restore_5",
245 "__riscv_restore_6",
246 "__riscv_restore_7",
247 "__riscv_restore_8",
248 "__riscv_restore_9",
249 "__riscv_restore_10",
250 "__riscv_restore_11",
251 "__riscv_restore_12"
252 };
253
254 int LibCallID = getLibCallID(MF, CSI);
255 if (LibCallID == -1)
256 return nullptr;
257 return RestoreLibCalls[LibCallID];
258}
259
260// Return encoded value and register count for PUSH/POP instruction,
261// representing registers to store/load.
262static std::pair<unsigned, unsigned>
263getPushPopEncodingAndNum(const Register MaxReg) {
264 switch (MaxReg) {
265 default:
266 llvm_unreachable("Unexpected Reg for Push/Pop Inst");
267 case RISCV::X27: /*s11*/
268 return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S11, y: 13);
269 case RISCV::X25: /*s9*/
270 return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S9, y: 11);
271 case RISCV::X24: /*s8*/
272 return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S8, y: 10);
273 case RISCV::X23: /*s7*/
274 return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S7, y: 9);
275 case RISCV::X22: /*s6*/
276 return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S6, y: 8);
277 case RISCV::X21: /*s5*/
278 return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S5, y: 7);
279 case RISCV::X20: /*s4*/
280 return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S4, y: 6);
281 case RISCV::X19: /*s3*/
282 return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S3, y: 5);
283 case RISCV::X18: /*s2*/
284 return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S2, y: 4);
285 case RISCV::X9: /*s1*/
286 return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0_S1, y: 3);
287 case RISCV::X8: /*s0*/
288 return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA_S0, y: 2);
289 case RISCV::X1: /*ra*/
290 return std::make_pair(x: llvm::RISCVZC::RLISTENCODE::RA, y: 1);
291 }
292}
293
294// Get the max reg of Push/Pop for restoring callee saved registers.
295static Register getMaxPushPopReg(const MachineFunction &MF,
296 const std::vector<CalleeSavedInfo> &CSI) {
297 Register MaxPushPopReg = RISCV::NoRegister;
298 for (auto &CS : CSI) {
299 if (llvm::find_if(Range: FixedCSRFIMap, P: [&](auto P) {
300 return P.first == CS.getReg();
301 }) != std::end(arr: FixedCSRFIMap))
302 MaxPushPopReg = std::max(a: MaxPushPopReg.id(), b: CS.getReg().id());
303 }
304 assert(MaxPushPopReg != RISCV::X26 && "x26 requires x27 to also be pushed");
305 return MaxPushPopReg;
306}
307
308// Return true if the specified function should have a dedicated frame
309// pointer register. This is true if frame pointer elimination is
310// disabled, if it needs dynamic stack realignment, if the function has
311// variable sized allocas, or if the frame address is taken.
312bool RISCVFrameLowering::hasFP(const MachineFunction &MF) const {
313 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
314
315 const MachineFrameInfo &MFI = MF.getFrameInfo();
316 return MF.getTarget().Options.DisableFramePointerElim(MF) ||
317 RegInfo->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
318 MFI.isFrameAddressTaken();
319}
320
321bool RISCVFrameLowering::hasBP(const MachineFunction &MF) const {
322 const MachineFrameInfo &MFI = MF.getFrameInfo();
323 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
324
325 // If we do not reserve stack space for outgoing arguments in prologue,
326 // we will adjust the stack pointer before call instruction. After the
327 // adjustment, we can not use SP to access the stack objects for the
328 // arguments. Instead, use BP to access these stack objects.
329 return (MFI.hasVarSizedObjects() ||
330 (!hasReservedCallFrame(MF) && (!MFI.isMaxCallFrameSizeComputed() ||
331 MFI.getMaxCallFrameSize() != 0))) &&
332 TRI->hasStackRealignment(MF);
333}
334
335// Determines the size of the frame and maximum call frame size.
336void RISCVFrameLowering::determineFrameLayout(MachineFunction &MF) const {
337 MachineFrameInfo &MFI = MF.getFrameInfo();
338 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
339
340 // Get the number of bytes to allocate from the FrameInfo.
341 uint64_t FrameSize = MFI.getStackSize();
342
343 // Get the alignment.
344 Align StackAlign = getStackAlign();
345
346 // Make sure the frame is aligned.
347 FrameSize = alignTo(Size: FrameSize, A: StackAlign);
348
349 // Update frame info.
350 MFI.setStackSize(FrameSize);
351
352 // When using SP or BP to access stack objects, we may require extra padding
353 // to ensure the bottom of the RVV stack is correctly aligned within the main
354 // stack. We calculate this as the amount required to align the scalar local
355 // variable section up to the RVV alignment.
356 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
357 if (RVFI->getRVVStackSize() && (!hasFP(MF) || TRI->hasStackRealignment(MF))) {
358 int ScalarLocalVarSize = FrameSize - RVFI->getCalleeSavedStackSize() -
359 RVFI->getVarArgsSaveSize();
360 if (auto RVVPadding =
361 offsetToAlignment(Value: ScalarLocalVarSize, Alignment: RVFI->getRVVStackAlign()))
362 RVFI->setRVVPadding(RVVPadding);
363 }
364}
365
366// Returns the stack size including RVV padding (when required), rounded back
367// up to the required stack alignment.
368uint64_t RISCVFrameLowering::getStackSizeWithRVVPadding(
369 const MachineFunction &MF) const {
370 const MachineFrameInfo &MFI = MF.getFrameInfo();
371 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
372 return alignTo(Size: MFI.getStackSize() + RVFI->getRVVPadding(), A: getStackAlign());
373}
374
375// Returns the register used to hold the frame pointer.
376static Register getFPReg(const RISCVSubtarget &STI) { return RISCV::X8; }
377
378// Returns the register used to hold the stack pointer.
379static Register getSPReg(const RISCVSubtarget &STI) { return RISCV::X2; }
380
381static SmallVector<CalleeSavedInfo, 8>
382getUnmanagedCSI(const MachineFunction &MF,
383 const std::vector<CalleeSavedInfo> &CSI) {
384 const MachineFrameInfo &MFI = MF.getFrameInfo();
385 SmallVector<CalleeSavedInfo, 8> NonLibcallCSI;
386
387 for (auto &CS : CSI) {
388 int FI = CS.getFrameIdx();
389 if (FI >= 0 && MFI.getStackID(ObjectIdx: FI) == TargetStackID::Default)
390 NonLibcallCSI.push_back(Elt: CS);
391 }
392
393 return NonLibcallCSI;
394}
395
396static SmallVector<CalleeSavedInfo, 8>
397getRVVCalleeSavedInfo(const MachineFunction &MF,
398 const std::vector<CalleeSavedInfo> &CSI) {
399 const MachineFrameInfo &MFI = MF.getFrameInfo();
400 SmallVector<CalleeSavedInfo, 8> RVVCSI;
401
402 for (auto &CS : CSI) {
403 int FI = CS.getFrameIdx();
404 if (FI >= 0 && MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector)
405 RVVCSI.push_back(Elt: CS);
406 }
407
408 return RVVCSI;
409}
410
411void RISCVFrameLowering::adjustStackForRVV(MachineFunction &MF,
412 MachineBasicBlock &MBB,
413 MachineBasicBlock::iterator MBBI,
414 const DebugLoc &DL, int64_t Amount,
415 MachineInstr::MIFlag Flag) const {
416 assert(Amount != 0 && "Did not need to adjust stack pointer for RVV.");
417
418 const Register SPReg = getSPReg(STI);
419
420 // Optimize compile time offset case
421 StackOffset Offset = StackOffset::getScalable(Scalable: Amount);
422 if (auto VLEN = STI.getRealVLen()) {
423 // 1. Multiply the number of v-slots by the (constant) length of register
424 const int64_t VLENB = *VLEN / 8;
425 assert(Amount % 8 == 0 &&
426 "Reserve the stack by the multiple of one vector size.");
427 const int64_t NumOfVReg = Amount / 8;
428 const int64_t FixedOffset = NumOfVReg * VLENB;
429 if (!isInt<32>(x: FixedOffset)) {
430 report_fatal_error(
431 reason: "Frame size outside of the signed 32-bit range not supported");
432 }
433 Offset = StackOffset::getFixed(Fixed: FixedOffset);
434 }
435
436 const RISCVRegisterInfo &RI = *STI.getRegisterInfo();
437 // We must keep the stack pointer aligned through any intermediate
438 // updates.
439 RI.adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, Offset,
440 Flag, RequiredAlign: getStackAlign());
441}
442
443static void appendScalableVectorExpression(const TargetRegisterInfo &TRI,
444 SmallVectorImpl<char> &Expr,
445 int FixedOffset, int ScalableOffset,
446 llvm::raw_string_ostream &Comment) {
447 unsigned DwarfVLenB = TRI.getDwarfRegNum(RegNum: RISCV::VLENB, isEH: true);
448 uint8_t Buffer[16];
449 if (FixedOffset) {
450 Expr.push_back(Elt: dwarf::DW_OP_consts);
451 Expr.append(in_start: Buffer, in_end: Buffer + encodeSLEB128(Value: FixedOffset, p: Buffer));
452 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_plus);
453 Comment << (FixedOffset < 0 ? " - " : " + ") << std::abs(x: FixedOffset);
454 }
455
456 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_consts);
457 Expr.append(in_start: Buffer, in_end: Buffer + encodeSLEB128(Value: ScalableOffset, p: Buffer));
458
459 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_bregx);
460 Expr.append(in_start: Buffer, in_end: Buffer + encodeULEB128(Value: DwarfVLenB, p: Buffer));
461 Expr.push_back(Elt: 0);
462
463 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_mul);
464 Expr.push_back(Elt: (uint8_t)dwarf::DW_OP_plus);
465
466 Comment << (ScalableOffset < 0 ? " - " : " + ") << std::abs(x: ScalableOffset)
467 << " * vlenb";
468}
469
470static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
471 Register Reg,
472 uint64_t FixedOffset,
473 uint64_t ScalableOffset) {
474 assert(ScalableOffset != 0 && "Did not need to adjust CFA for RVV");
475 SmallString<64> Expr;
476 std::string CommentBuffer;
477 llvm::raw_string_ostream Comment(CommentBuffer);
478 // Build up the expression (Reg + FixedOffset + ScalableOffset * VLENB).
479 unsigned DwarfReg = TRI.getDwarfRegNum(RegNum: Reg, isEH: true);
480 Expr.push_back(Elt: (uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
481 Expr.push_back(Elt: 0);
482 if (Reg == RISCV::X2)
483 Comment << "sp";
484 else
485 Comment << printReg(Reg, TRI: &TRI);
486
487 appendScalableVectorExpression(TRI, Expr, FixedOffset, ScalableOffset,
488 Comment);
489
490 SmallString<64> DefCfaExpr;
491 uint8_t Buffer[16];
492 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
493 DefCfaExpr.append(in_start: Buffer, in_end: Buffer + encodeULEB128(Value: Expr.size(), p: Buffer));
494 DefCfaExpr.append(RHS: Expr.str());
495
496 return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(),
497 Comment: Comment.str());
498}
499
500static MCCFIInstruction createDefCFAOffset(const TargetRegisterInfo &TRI,
501 Register Reg, uint64_t FixedOffset,
502 uint64_t ScalableOffset) {
503 assert(ScalableOffset != 0 && "Did not need to adjust CFA for RVV");
504 SmallString<64> Expr;
505 std::string CommentBuffer;
506 llvm::raw_string_ostream Comment(CommentBuffer);
507 Comment << printReg(Reg, TRI: &TRI) << " @ cfa";
508
509 // Build up the expression (FixedOffset + ScalableOffset * VLENB).
510 appendScalableVectorExpression(TRI, Expr, FixedOffset, ScalableOffset,
511 Comment);
512
513 SmallString<64> DefCfaExpr;
514 uint8_t Buffer[16];
515 unsigned DwarfReg = TRI.getDwarfRegNum(RegNum: Reg, isEH: true);
516 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
517 DefCfaExpr.append(in_start: Buffer, in_end: Buffer + encodeULEB128(Value: DwarfReg, p: Buffer));
518 DefCfaExpr.append(in_start: Buffer, in_end: Buffer + encodeULEB128(Value: Expr.size(), p: Buffer));
519 DefCfaExpr.append(RHS: Expr.str());
520
521 return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(),
522 Comment: Comment.str());
523}
524
525void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
526 MachineBasicBlock &MBB) const {
527 MachineFrameInfo &MFI = MF.getFrameInfo();
528 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
529 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
530 const RISCVInstrInfo *TII = STI.getInstrInfo();
531 MachineBasicBlock::iterator MBBI = MBB.begin();
532
533 Register FPReg = getFPReg(STI);
534 Register SPReg = getSPReg(STI);
535 Register BPReg = RISCVABI::getBPReg();
536
537 // Debug location must be unknown since the first debug location is used
538 // to determine the end of the prologue.
539 DebugLoc DL;
540
541 // All calls are tail calls in GHC calling conv, and functions have no
542 // prologue/epilogue.
543 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
544 return;
545
546 // Emit prologue for shadow call stack.
547 emitSCSPrologue(MF, MBB, MI: MBBI, DL);
548
549 auto FirstFrameSetup = MBBI;
550
551 // Since spillCalleeSavedRegisters may have inserted a libcall, skip past
552 // any instructions marked as FrameSetup
553 while (MBBI != MBB.end() && MBBI->getFlag(Flag: MachineInstr::FrameSetup))
554 ++MBBI;
555
556 // Determine the correct frame layout
557 determineFrameLayout(MF);
558
559 // If libcalls are used to spill and restore callee-saved registers, the frame
560 // has two sections; the opaque section managed by the libcalls, and the
561 // section managed by MachineFrameInfo which can also hold callee saved
562 // registers in fixed stack slots, both of which have negative frame indices.
563 // This gets even more complicated when incoming arguments are passed via the
564 // stack, as these too have negative frame indices. An example is detailed
565 // below:
566 //
567 // | incoming arg | <- FI[-3]
568 // | libcallspill |
569 // | calleespill | <- FI[-2]
570 // | calleespill | <- FI[-1]
571 // | this_frame | <- FI[0]
572 //
573 // For negative frame indices, the offset from the frame pointer will differ
574 // depending on which of these groups the frame index applies to.
575 // The following calculates the correct offset knowing the number of callee
576 // saved registers spilt by the two methods.
577 if (int LibCallRegs = getLibCallID(MF, CSI: MFI.getCalleeSavedInfo()) + 1) {
578 // Calculate the size of the frame managed by the libcall. The stack
579 // alignment of these libcalls should be the same as how we set it in
580 // getABIStackAlignment.
581 unsigned LibCallFrameSize =
582 alignTo(Size: (STI.getXLen() / 8) * LibCallRegs, A: getStackAlign());
583 RVFI->setLibCallStackSize(LibCallFrameSize);
584 }
585
586 // FIXME (note copied from Lanai): This appears to be overallocating. Needs
587 // investigation. Get the number of bytes to allocate from the FrameInfo.
588 uint64_t RealStackSize = getStackSizeWithRVVPadding(MF);
589 uint64_t StackSize = RealStackSize - RVFI->getReservedSpillsSize();
590 uint64_t RVVStackSize = RVFI->getRVVStackSize();
591
592 // Early exit if there is no need to allocate on the stack
593 if (RealStackSize == 0 && !MFI.adjustsStack() && RVVStackSize == 0)
594 return;
595
596 // If the stack pointer has been marked as reserved, then produce an error if
597 // the frame requires stack allocation
598 if (STI.isRegisterReservedByUser(i: SPReg))
599 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
600 MF.getFunction(), "Stack pointer required, but has been reserved."});
601
602 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
603 // Split the SP adjustment to reduce the offsets of callee saved spill.
604 if (FirstSPAdjustAmount) {
605 StackSize = FirstSPAdjustAmount;
606 RealStackSize = FirstSPAdjustAmount;
607 }
608
609 if (RVFI->isPushable(MF) && FirstFrameSetup != MBB.end() &&
610 FirstFrameSetup->getOpcode() == RISCV::CM_PUSH) {
611 // Use available stack adjustment in push instruction to allocate additional
612 // stack space. Align the stack size down to a multiple of 16. This is
613 // needed for RVE.
614 // FIXME: Can we increase the stack size to a multiple of 16 instead?
615 uint64_t Spimm = std::min(a: alignDown(Value: StackSize, Align: 16), b: (uint64_t)48);
616 FirstFrameSetup->getOperand(i: 1).setImm(Spimm);
617 StackSize -= Spimm;
618 }
619
620 if (StackSize != 0) {
621 // Allocate space on the stack if necessary.
622 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg,
623 Offset: StackOffset::getFixed(Fixed: -StackSize), Flag: MachineInstr::FrameSetup,
624 RequiredAlign: getStackAlign());
625 }
626
627 // Emit ".cfi_def_cfa_offset RealStackSize"
628 unsigned CFIIndex = MF.addFrameInst(
629 Inst: MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: RealStackSize));
630 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
631 .addCFIIndex(CFIIndex)
632 .setMIFlag(MachineInstr::FrameSetup);
633
634 const auto &CSI = MFI.getCalleeSavedInfo();
635
636 // The frame pointer is callee-saved, and code has been generated for us to
637 // save it to the stack. We need to skip over the storing of callee-saved
638 // registers as the frame pointer must be modified after it has been saved
639 // to the stack, not before.
640 // FIXME: assumes exactly one instruction is used to save each callee-saved
641 // register.
642 std::advance(i&: MBBI, n: getUnmanagedCSI(MF, CSI).size());
643
644 // Iterate over list of callee-saved registers and emit .cfi_offset
645 // directives.
646 for (const auto &Entry : CSI) {
647 int FrameIdx = Entry.getFrameIdx();
648 if (FrameIdx >= 0 &&
649 MFI.getStackID(ObjectIdx: FrameIdx) == TargetStackID::ScalableVector)
650 continue;
651
652 int64_t Offset = MFI.getObjectOffset(ObjectIdx: FrameIdx);
653 Register Reg = Entry.getReg();
654 unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::createOffset(
655 L: nullptr, Register: RI->getDwarfRegNum(RegNum: Reg, isEH: true), Offset));
656 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
657 .addCFIIndex(CFIIndex)
658 .setMIFlag(MachineInstr::FrameSetup);
659 }
660
661 // Generate new FP.
662 if (hasFP(MF)) {
663 if (STI.isRegisterReservedByUser(i: FPReg))
664 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
665 MF.getFunction(), "Frame pointer required, but has been reserved."});
666 // The frame pointer does need to be reserved from register allocation.
667 assert(MF.getRegInfo().isReserved(FPReg) && "FP not reserved");
668
669 RI->adjustReg(MBB, II: MBBI, DL, DestReg: FPReg, SrcReg: SPReg,
670 Offset: StackOffset::getFixed(Fixed: RealStackSize - RVFI->getVarArgsSaveSize()),
671 Flag: MachineInstr::FrameSetup, RequiredAlign: getStackAlign());
672
673 // Emit ".cfi_def_cfa $fp, RVFI->getVarArgsSaveSize()"
674 unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::cfiDefCfa(
675 L: nullptr, Register: RI->getDwarfRegNum(RegNum: FPReg, isEH: true), Offset: RVFI->getVarArgsSaveSize()));
676 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
677 .addCFIIndex(CFIIndex)
678 .setMIFlag(MachineInstr::FrameSetup);
679 }
680
681 // Emit the second SP adjustment after saving callee saved registers.
682 if (FirstSPAdjustAmount) {
683 uint64_t SecondSPAdjustAmount =
684 getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
685 assert(SecondSPAdjustAmount > 0 &&
686 "SecondSPAdjustAmount should be greater than zero");
687 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg,
688 Offset: StackOffset::getFixed(Fixed: -SecondSPAdjustAmount),
689 Flag: MachineInstr::FrameSetup, RequiredAlign: getStackAlign());
690
691 // If we are using a frame-pointer, and thus emitted ".cfi_def_cfa fp, 0",
692 // don't emit an sp-based .cfi_def_cfa_offset
693 if (!hasFP(MF)) {
694 // Emit ".cfi_def_cfa_offset StackSize"
695 unsigned CFIIndex = MF.addFrameInst(Inst: MCCFIInstruction::cfiDefCfaOffset(
696 L: nullptr, Offset: getStackSizeWithRVVPadding(MF)));
697 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
698 .addCFIIndex(CFIIndex)
699 .setMIFlag(MachineInstr::FrameSetup);
700 }
701 }
702
703 if (RVVStackSize) {
704 adjustStackForRVV(MF, MBB, MBBI, DL, Amount: -RVVStackSize,
705 Flag: MachineInstr::FrameSetup);
706 if (!hasFP(MF)) {
707 // Emit .cfi_def_cfa_expression "sp + StackSize + RVVStackSize * vlenb".
708 unsigned CFIIndex = MF.addFrameInst(Inst: createDefCFAExpression(
709 TRI: *RI, Reg: SPReg, FixedOffset: getStackSizeWithRVVPadding(MF), ScalableOffset: RVVStackSize / 8));
710 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
711 .addCFIIndex(CFIIndex)
712 .setMIFlag(MachineInstr::FrameSetup);
713 }
714
715 std::advance(i&: MBBI, n: getRVVCalleeSavedInfo(MF, CSI).size());
716 emitCalleeSavedRVVPrologCFI(MBB, MI: MBBI, HasFP: hasFP(MF));
717 }
718
719 if (hasFP(MF)) {
720 // Realign Stack
721 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
722 if (RI->hasStackRealignment(MF)) {
723 Align MaxAlignment = MFI.getMaxAlign();
724
725 const RISCVInstrInfo *TII = STI.getInstrInfo();
726 if (isInt<12>(x: -(int)MaxAlignment.value())) {
727 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::ANDI), DestReg: SPReg)
728 .addReg(RegNo: SPReg)
729 .addImm(Val: -(int)MaxAlignment.value())
730 .setMIFlag(MachineInstr::FrameSetup);
731 } else {
732 unsigned ShiftAmount = Log2(A: MaxAlignment);
733 Register VR =
734 MF.getRegInfo().createVirtualRegister(RegClass: &RISCV::GPRRegClass);
735 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SRLI), DestReg: VR)
736 .addReg(RegNo: SPReg)
737 .addImm(Val: ShiftAmount)
738 .setMIFlag(MachineInstr::FrameSetup);
739 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::SLLI), DestReg: SPReg)
740 .addReg(RegNo: VR)
741 .addImm(Val: ShiftAmount)
742 .setMIFlag(MachineInstr::FrameSetup);
743 }
744 // FP will be used to restore the frame in the epilogue, so we need
745 // another base register BP to record SP after re-alignment. SP will
746 // track the current stack after allocating variable sized objects.
747 if (hasBP(MF)) {
748 // move BP, SP
749 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: BPReg)
750 .addReg(RegNo: SPReg)
751 .addImm(Val: 0)
752 .setMIFlag(MachineInstr::FrameSetup);
753 }
754 }
755 }
756}
757
758void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
759 MachineBasicBlock &MBB) const {
760 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
761 MachineFrameInfo &MFI = MF.getFrameInfo();
762 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
763 Register FPReg = getFPReg(STI);
764 Register SPReg = getSPReg(STI);
765
766 // All calls are tail calls in GHC calling conv, and functions have no
767 // prologue/epilogue.
768 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
769 return;
770
771 // Get the insert location for the epilogue. If there were no terminators in
772 // the block, get the last instruction.
773 MachineBasicBlock::iterator MBBI = MBB.end();
774 DebugLoc DL;
775 if (!MBB.empty()) {
776 MBBI = MBB.getLastNonDebugInstr();
777 if (MBBI != MBB.end())
778 DL = MBBI->getDebugLoc();
779
780 MBBI = MBB.getFirstTerminator();
781
782 // If callee-saved registers are saved via libcall, place stack adjustment
783 // before this call.
784 while (MBBI != MBB.begin() &&
785 std::prev(x: MBBI)->getFlag(Flag: MachineInstr::FrameDestroy))
786 --MBBI;
787 }
788
789 const auto &CSI = getUnmanagedCSI(MF, CSI: MFI.getCalleeSavedInfo());
790
791 // Skip to before the restores of scalar callee-saved registers
792 // FIXME: assumes exactly one instruction is used to restore each
793 // callee-saved register.
794 auto LastFrameDestroy = MBBI;
795 if (!CSI.empty())
796 LastFrameDestroy = std::prev(x: MBBI, n: CSI.size());
797
798 uint64_t RealStackSize = getStackSizeWithRVVPadding(MF);
799 uint64_t StackSize = RealStackSize - RVFI->getReservedSpillsSize();
800 uint64_t FPOffset = RealStackSize - RVFI->getVarArgsSaveSize();
801 uint64_t RVVStackSize = RVFI->getRVVStackSize();
802
803 // Restore the stack pointer using the value of the frame pointer. Only
804 // necessary if the stack pointer was modified, meaning the stack size is
805 // unknown.
806 //
807 // In order to make sure the stack point is right through the EH region,
808 // we also need to restore stack pointer from the frame pointer if we
809 // don't preserve stack space within prologue/epilogue for outgoing variables,
810 // normally it's just checking the variable sized object is present or not
811 // is enough, but we also don't preserve that at prologue/epilogue when
812 // have vector objects in stack.
813 if (RI->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
814 !hasReservedCallFrame(MF)) {
815 assert(hasFP(MF) && "frame pointer should not have been eliminated");
816 RI->adjustReg(MBB, II: LastFrameDestroy, DL, DestReg: SPReg, SrcReg: FPReg,
817 Offset: StackOffset::getFixed(Fixed: -FPOffset),
818 Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign());
819 } else {
820 if (RVVStackSize)
821 adjustStackForRVV(MF, MBB, MBBI: LastFrameDestroy, DL, Amount: RVVStackSize,
822 Flag: MachineInstr::FrameDestroy);
823 }
824
825 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
826 if (FirstSPAdjustAmount) {
827 uint64_t SecondSPAdjustAmount =
828 getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
829 assert(SecondSPAdjustAmount > 0 &&
830 "SecondSPAdjustAmount should be greater than zero");
831
832 RI->adjustReg(MBB, II: LastFrameDestroy, DL, DestReg: SPReg, SrcReg: SPReg,
833 Offset: StackOffset::getFixed(Fixed: SecondSPAdjustAmount),
834 Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign());
835 }
836
837 if (FirstSPAdjustAmount)
838 StackSize = FirstSPAdjustAmount;
839
840 if (RVFI->isPushable(MF) && MBBI != MBB.end() &&
841 MBBI->getOpcode() == RISCV::CM_POP) {
842 // Use available stack adjustment in pop instruction to deallocate stack
843 // space. Align the stack size down to a multiple of 16. This is needed for
844 // RVE.
845 // FIXME: Can we increase the stack size to a multiple of 16 instead?
846 uint64_t Spimm = std::min(a: alignDown(Value: StackSize, Align: 16), b: (uint64_t)48);
847 MBBI->getOperand(i: 1).setImm(Spimm);
848 StackSize -= Spimm;
849 }
850
851 // Deallocate stack
852 if (StackSize != 0) {
853 RI->adjustReg(MBB, II: MBBI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: StackSize),
854 Flag: MachineInstr::FrameDestroy, RequiredAlign: getStackAlign());
855 }
856
857 // Emit epilogue for shadow call stack.
858 emitSCSEpilogue(MF, MBB, MI: MBBI, DL);
859}
860
861StackOffset
862RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
863 Register &FrameReg) const {
864 const MachineFrameInfo &MFI = MF.getFrameInfo();
865 const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
866 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
867
868 // Callee-saved registers should be referenced relative to the stack
869 // pointer (positive offset), otherwise use the frame pointer (negative
870 // offset).
871 const auto &CSI = getUnmanagedCSI(MF, CSI: MFI.getCalleeSavedInfo());
872 int MinCSFI = 0;
873 int MaxCSFI = -1;
874 StackOffset Offset;
875 auto StackID = MFI.getStackID(ObjectIdx: FI);
876
877 assert((StackID == TargetStackID::Default ||
878 StackID == TargetStackID::ScalableVector) &&
879 "Unexpected stack ID for the frame object.");
880 if (StackID == TargetStackID::Default) {
881 assert(getOffsetOfLocalArea() == 0 && "LocalAreaOffset is not 0!");
882 Offset = StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FI) +
883 MFI.getOffsetAdjustment());
884 } else if (StackID == TargetStackID::ScalableVector) {
885 Offset = StackOffset::getScalable(Scalable: MFI.getObjectOffset(ObjectIdx: FI));
886 }
887
888 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
889
890 if (CSI.size()) {
891 MinCSFI = CSI[0].getFrameIdx();
892 MaxCSFI = CSI[CSI.size() - 1].getFrameIdx();
893 }
894
895 if (FI >= MinCSFI && FI <= MaxCSFI) {
896 FrameReg = RISCV::X2;
897
898 if (FirstSPAdjustAmount)
899 Offset += StackOffset::getFixed(Fixed: FirstSPAdjustAmount);
900 else
901 Offset += StackOffset::getFixed(Fixed: getStackSizeWithRVVPadding(MF));
902 return Offset;
903 }
904
905 if (RI->hasStackRealignment(MF) && !MFI.isFixedObjectIndex(ObjectIdx: FI)) {
906 // If the stack was realigned, the frame pointer is set in order to allow
907 // SP to be restored, so we need another base register to record the stack
908 // after realignment.
909 // |--------------------------| -- <-- FP
910 // | callee-allocated save | | <----|
911 // | area for register varargs| | |
912 // |--------------------------| | |
913 // | callee-saved registers | | |
914 // |--------------------------| -- |
915 // | realignment (the size of | | |
916 // | this area is not counted | | |
917 // | in MFI.getStackSize()) | | |
918 // |--------------------------| -- |-- MFI.getStackSize()
919 // | RVV alignment padding | | |
920 // | (not counted in | | |
921 // | MFI.getStackSize() but | | |
922 // | counted in | | |
923 // | RVFI.getRVVStackSize()) | | |
924 // |--------------------------| -- |
925 // | RVV objects | | |
926 // | (not counted in | | |
927 // | MFI.getStackSize()) | | |
928 // |--------------------------| -- |
929 // | padding before RVV | | |
930 // | (not counted in | | |
931 // | MFI.getStackSize() or in | | |
932 // | RVFI.getRVVStackSize()) | | |
933 // |--------------------------| -- |
934 // | scalar local variables | | <----'
935 // |--------------------------| -- <-- BP (if var sized objects present)
936 // | VarSize objects | |
937 // |--------------------------| -- <-- SP
938 if (hasBP(MF)) {
939 FrameReg = RISCVABI::getBPReg();
940 } else {
941 // VarSize objects must be empty in this case!
942 assert(!MFI.hasVarSizedObjects());
943 FrameReg = RISCV::X2;
944 }
945 } else {
946 FrameReg = RI->getFrameRegister(MF);
947 }
948
949 if (FrameReg == getFPReg(STI)) {
950 Offset += StackOffset::getFixed(Fixed: RVFI->getVarArgsSaveSize());
951 // When using FP to access scalable vector objects, we need to minus
952 // the frame size.
953 //
954 // |--------------------------| -- <-- FP
955 // | callee-allocated save | |
956 // | area for register varargs| |
957 // |--------------------------| |
958 // | callee-saved registers | |
959 // |--------------------------| | MFI.getStackSize()
960 // | scalar local variables | |
961 // |--------------------------| -- (Offset of RVV objects is from here.)
962 // | RVV objects |
963 // |--------------------------|
964 // | VarSize objects |
965 // |--------------------------| <-- SP
966 if (MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector) {
967 assert(!RI->hasStackRealignment(MF) &&
968 "Can't index across variable sized realign");
969 // We don't expect any extra RVV alignment padding, as the stack size
970 // and RVV object sections should be correct aligned in their own
971 // right.
972 assert(MFI.getStackSize() == getStackSizeWithRVVPadding(MF) &&
973 "Inconsistent stack layout");
974 Offset -= StackOffset::getFixed(Fixed: MFI.getStackSize());
975 }
976 return Offset;
977 }
978
979 // This case handles indexing off both SP and BP.
980 // If indexing off SP, there must not be any var sized objects
981 assert(FrameReg == RISCVABI::getBPReg() || !MFI.hasVarSizedObjects());
982
983 // When using SP to access frame objects, we need to add RVV stack size.
984 //
985 // |--------------------------| -- <-- FP
986 // | callee-allocated save | | <----|
987 // | area for register varargs| | |
988 // |--------------------------| | |
989 // | callee-saved registers | | |
990 // |--------------------------| -- |
991 // | RVV alignment padding | | |
992 // | (not counted in | | |
993 // | MFI.getStackSize() but | | |
994 // | counted in | | |
995 // | RVFI.getRVVStackSize()) | | |
996 // |--------------------------| -- |
997 // | RVV objects | | |-- MFI.getStackSize()
998 // | (not counted in | | |
999 // | MFI.getStackSize()) | | |
1000 // |--------------------------| -- |
1001 // | padding before RVV | | |
1002 // | (not counted in | | |
1003 // | MFI.getStackSize()) | | |
1004 // |--------------------------| -- |
1005 // | scalar local variables | | <----'
1006 // |--------------------------| -- <-- BP (if var sized objects present)
1007 // | VarSize objects | |
1008 // |--------------------------| -- <-- SP
1009 //
1010 // The total amount of padding surrounding RVV objects is described by
1011 // RVV->getRVVPadding() and it can be zero. It allows us to align the RVV
1012 // objects to the required alignment.
1013 if (MFI.getStackID(ObjectIdx: FI) == TargetStackID::Default) {
1014 if (MFI.isFixedObjectIndex(ObjectIdx: FI)) {
1015 assert(!RI->hasStackRealignment(MF) &&
1016 "Can't index across variable sized realign");
1017 Offset += StackOffset::get(Fixed: getStackSizeWithRVVPadding(MF),
1018 Scalable: RVFI->getRVVStackSize());
1019 } else {
1020 Offset += StackOffset::getFixed(Fixed: MFI.getStackSize());
1021 }
1022 } else if (MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector) {
1023 // Ensure the base of the RVV stack is correctly aligned: add on the
1024 // alignment padding.
1025 int ScalarLocalVarSize = MFI.getStackSize() -
1026 RVFI->getCalleeSavedStackSize() -
1027 RVFI->getRVPushStackSize() -
1028 RVFI->getVarArgsSaveSize() + RVFI->getRVVPadding();
1029 Offset += StackOffset::get(Fixed: ScalarLocalVarSize, Scalable: RVFI->getRVVStackSize());
1030 }
1031 return Offset;
1032}
1033
1034void RISCVFrameLowering::determineCalleeSaves(MachineFunction &MF,
1035 BitVector &SavedRegs,
1036 RegScavenger *RS) const {
1037 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
1038 // Unconditionally spill RA and FP only if the function uses a frame
1039 // pointer.
1040 if (hasFP(MF)) {
1041 SavedRegs.set(RISCV::X1);
1042 SavedRegs.set(RISCV::X8);
1043 }
1044 // Mark BP as used if function has dedicated base pointer.
1045 if (hasBP(MF))
1046 SavedRegs.set(RISCVABI::getBPReg());
1047
1048 // When using cm.push/pop we must save X27 if we save X26.
1049 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1050 if (RVFI->isPushable(MF) && SavedRegs.test(Idx: RISCV::X26))
1051 SavedRegs.set(RISCV::X27);
1052}
1053
1054std::pair<int64_t, Align>
1055RISCVFrameLowering::assignRVVStackObjectOffsets(MachineFunction &MF) const {
1056 MachineFrameInfo &MFI = MF.getFrameInfo();
1057 // Create a buffer of RVV objects to allocate.
1058 SmallVector<int, 8> ObjectsToAllocate;
1059 auto pushRVVObjects = [&](int FIBegin, int FIEnd) {
1060 for (int I = FIBegin, E = FIEnd; I != E; ++I) {
1061 unsigned StackID = MFI.getStackID(ObjectIdx: I);
1062 if (StackID != TargetStackID::ScalableVector)
1063 continue;
1064 if (MFI.isDeadObjectIndex(ObjectIdx: I))
1065 continue;
1066
1067 ObjectsToAllocate.push_back(Elt: I);
1068 }
1069 };
1070 // First push RVV Callee Saved object, then push RVV stack object
1071 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
1072 const auto &RVVCSI = getRVVCalleeSavedInfo(MF, CSI);
1073 if (!RVVCSI.empty())
1074 pushRVVObjects(RVVCSI[0].getFrameIdx(),
1075 RVVCSI[RVVCSI.size() - 1].getFrameIdx() + 1);
1076 pushRVVObjects(0, MFI.getObjectIndexEnd() - RVVCSI.size());
1077
1078 // The minimum alignment is 16 bytes.
1079 Align RVVStackAlign(16);
1080 const auto &ST = MF.getSubtarget<RISCVSubtarget>();
1081
1082 if (!ST.hasVInstructions()) {
1083 assert(ObjectsToAllocate.empty() &&
1084 "Can't allocate scalable-vector objects without V instructions");
1085 return std::make_pair(x: 0, y&: RVVStackAlign);
1086 }
1087
1088 // Allocate all RVV locals and spills
1089 int64_t Offset = 0;
1090 for (int FI : ObjectsToAllocate) {
1091 // ObjectSize in bytes.
1092 int64_t ObjectSize = MFI.getObjectSize(ObjectIdx: FI);
1093 auto ObjectAlign = std::max(a: Align(8), b: MFI.getObjectAlign(ObjectIdx: FI));
1094 // If the data type is the fractional vector type, reserve one vector
1095 // register for it.
1096 if (ObjectSize < 8)
1097 ObjectSize = 8;
1098 Offset = alignTo(Size: Offset + ObjectSize, A: ObjectAlign);
1099 MFI.setObjectOffset(ObjectIdx: FI, SPOffset: -Offset);
1100 // Update the maximum alignment of the RVV stack section
1101 RVVStackAlign = std::max(a: RVVStackAlign, b: ObjectAlign);
1102 }
1103
1104 // Ensure the alignment of the RVV stack. Since we want the most-aligned
1105 // object right at the bottom (i.e., any padding at the top of the frame),
1106 // readjust all RVV objects down by the alignment padding.
1107 uint64_t StackSize = Offset;
1108 if (auto AlignmentPadding = offsetToAlignment(Value: StackSize, Alignment: RVVStackAlign)) {
1109 StackSize += AlignmentPadding;
1110 for (int FI : ObjectsToAllocate)
1111 MFI.setObjectOffset(ObjectIdx: FI, SPOffset: MFI.getObjectOffset(ObjectIdx: FI) - AlignmentPadding);
1112 }
1113
1114 return std::make_pair(x&: StackSize, y&: RVVStackAlign);
1115}
1116
1117static unsigned getScavSlotsNumForRVV(MachineFunction &MF) {
1118 // For RVV spill, scalable stack offsets computing requires up to two scratch
1119 // registers
1120 static constexpr unsigned ScavSlotsNumRVVSpillScalableObject = 2;
1121
1122 // For RVV spill, non-scalable stack offsets computing requires up to one
1123 // scratch register.
1124 static constexpr unsigned ScavSlotsNumRVVSpillNonScalableObject = 1;
1125
1126 // ADDI instruction's destination register can be used for computing
1127 // offsets. So Scalable stack offsets require up to one scratch register.
1128 static constexpr unsigned ScavSlotsADDIScalableObject = 1;
1129
1130 static constexpr unsigned MaxScavSlotsNumKnown =
1131 std::max(l: {ScavSlotsADDIScalableObject, ScavSlotsNumRVVSpillScalableObject,
1132 ScavSlotsNumRVVSpillNonScalableObject});
1133
1134 unsigned MaxScavSlotsNum = 0;
1135 if (!MF.getSubtarget<RISCVSubtarget>().hasVInstructions())
1136 return false;
1137 for (const MachineBasicBlock &MBB : MF)
1138 for (const MachineInstr &MI : MBB) {
1139 bool IsRVVSpill = RISCV::isRVVSpill(MI);
1140 for (auto &MO : MI.operands()) {
1141 if (!MO.isFI())
1142 continue;
1143 bool IsScalableVectorID = MF.getFrameInfo().getStackID(ObjectIdx: MO.getIndex()) ==
1144 TargetStackID::ScalableVector;
1145 if (IsRVVSpill) {
1146 MaxScavSlotsNum = std::max(
1147 a: MaxScavSlotsNum, b: IsScalableVectorID
1148 ? ScavSlotsNumRVVSpillScalableObject
1149 : ScavSlotsNumRVVSpillNonScalableObject);
1150 } else if (MI.getOpcode() == RISCV::ADDI && IsScalableVectorID) {
1151 MaxScavSlotsNum =
1152 std::max(a: MaxScavSlotsNum, b: ScavSlotsADDIScalableObject);
1153 }
1154 }
1155 if (MaxScavSlotsNum == MaxScavSlotsNumKnown)
1156 return MaxScavSlotsNumKnown;
1157 }
1158 return MaxScavSlotsNum;
1159}
1160
1161static bool hasRVVFrameObject(const MachineFunction &MF) {
1162 // Originally, the function will scan all the stack objects to check whether
1163 // if there is any scalable vector object on the stack or not. However, it
1164 // causes errors in the register allocator. In issue 53016, it returns false
1165 // before RA because there is no RVV stack objects. After RA, it returns true
1166 // because there are spilling slots for RVV values during RA. It will not
1167 // reserve BP during register allocation and generate BP access in the PEI
1168 // pass due to the inconsistent behavior of the function.
1169 //
1170 // The function is changed to use hasVInstructions() as the return value. It
1171 // is not precise, but it can make the register allocation correct.
1172 //
1173 // FIXME: Find a better way to make the decision or revisit the solution in
1174 // D103622.
1175 //
1176 // Refer to https://github.com/llvm/llvm-project/issues/53016.
1177 return MF.getSubtarget<RISCVSubtarget>().hasVInstructions();
1178}
1179
1180static unsigned estimateFunctionSizeInBytes(const MachineFunction &MF,
1181 const RISCVInstrInfo &TII) {
1182 unsigned FnSize = 0;
1183 for (auto &MBB : MF) {
1184 for (auto &MI : MBB) {
1185 // Far branches over 20-bit offset will be relaxed in branch relaxation
1186 // pass. In the worst case, conditional branches will be relaxed into
1187 // the following instruction sequence. Unconditional branches are
1188 // relaxed in the same way, with the exception that there is no first
1189 // branch instruction.
1190 //
1191 // foo
1192 // bne t5, t6, .rev_cond # `TII->getInstSizeInBytes(MI)` bytes
1193 // sd s11, 0(sp) # 4 bytes, or 2 bytes in RVC
1194 // jump .restore, s11 # 8 bytes
1195 // .rev_cond
1196 // bar
1197 // j .dest_bb # 4 bytes, or 2 bytes in RVC
1198 // .restore:
1199 // ld s11, 0(sp) # 4 bytes, or 2 bytes in RVC
1200 // .dest:
1201 // baz
1202 if (MI.isConditionalBranch())
1203 FnSize += TII.getInstSizeInBytes(MI);
1204 if (MI.isConditionalBranch() || MI.isUnconditionalBranch()) {
1205 if (MF.getSubtarget<RISCVSubtarget>().hasStdExtCOrZca())
1206 FnSize += 2 + 8 + 2 + 2;
1207 else
1208 FnSize += 4 + 8 + 4 + 4;
1209 continue;
1210 }
1211
1212 FnSize += TII.getInstSizeInBytes(MI);
1213 }
1214 }
1215 return FnSize;
1216}
1217
1218void RISCVFrameLowering::processFunctionBeforeFrameFinalized(
1219 MachineFunction &MF, RegScavenger *RS) const {
1220 const RISCVRegisterInfo *RegInfo =
1221 MF.getSubtarget<RISCVSubtarget>().getRegisterInfo();
1222 const RISCVInstrInfo *TII = MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
1223 MachineFrameInfo &MFI = MF.getFrameInfo();
1224 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1225 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1226
1227 int64_t RVVStackSize;
1228 Align RVVStackAlign;
1229 std::tie(args&: RVVStackSize, args&: RVVStackAlign) = assignRVVStackObjectOffsets(MF);
1230
1231 RVFI->setRVVStackSize(RVVStackSize);
1232 RVFI->setRVVStackAlign(RVVStackAlign);
1233
1234 if (hasRVVFrameObject(MF)) {
1235 // Ensure the entire stack is aligned to at least the RVV requirement: some
1236 // scalable-vector object alignments are not considered by the
1237 // target-independent code.
1238 MFI.ensureMaxAlignment(Alignment: RVVStackAlign);
1239 }
1240
1241 unsigned ScavSlotsNum = 0;
1242
1243 // estimateStackSize has been observed to under-estimate the final stack
1244 // size, so give ourselves wiggle-room by checking for stack size
1245 // representable an 11-bit signed field rather than 12-bits.
1246 if (!isInt<11>(x: MFI.estimateStackSize(MF)))
1247 ScavSlotsNum = 1;
1248
1249 // Far branches over 20-bit offset require a spill slot for scratch register.
1250 bool IsLargeFunction = !isInt<20>(x: estimateFunctionSizeInBytes(MF, TII: *TII));
1251 if (IsLargeFunction)
1252 ScavSlotsNum = std::max(a: ScavSlotsNum, b: 1u);
1253
1254 // RVV loads & stores have no capacity to hold the immediate address offsets
1255 // so we must always reserve an emergency spill slot if the MachineFunction
1256 // contains any RVV spills.
1257 ScavSlotsNum = std::max(a: ScavSlotsNum, b: getScavSlotsNumForRVV(MF));
1258
1259 for (unsigned I = 0; I < ScavSlotsNum; I++) {
1260 int FI = MFI.CreateStackObject(Size: RegInfo->getSpillSize(RC: *RC),
1261 Alignment: RegInfo->getSpillAlign(RC: *RC), isSpillSlot: false);
1262 RS->addScavengingFrameIndex(FI);
1263
1264 if (IsLargeFunction && RVFI->getBranchRelaxationScratchFrameIndex() == -1)
1265 RVFI->setBranchRelaxationScratchFrameIndex(FI);
1266 }
1267
1268 unsigned Size = RVFI->getReservedSpillsSize();
1269 for (const auto &Info : MFI.getCalleeSavedInfo()) {
1270 int FrameIdx = Info.getFrameIdx();
1271 if (FrameIdx < 0 || MFI.getStackID(ObjectIdx: FrameIdx) != TargetStackID::Default)
1272 continue;
1273
1274 Size += MFI.getObjectSize(ObjectIdx: FrameIdx);
1275 }
1276 RVFI->setCalleeSavedStackSize(Size);
1277}
1278
1279// Not preserve stack space within prologue for outgoing variables when the
1280// function contains variable size objects or there are vector objects accessed
1281// by the frame pointer.
1282// Let eliminateCallFramePseudoInstr preserve stack space for it.
1283bool RISCVFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
1284 return !MF.getFrameInfo().hasVarSizedObjects() &&
1285 !(hasFP(MF) && hasRVVFrameObject(MF));
1286}
1287
1288// Eliminate ADJCALLSTACKDOWN, ADJCALLSTACKUP pseudo instructions.
1289MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr(
1290 MachineFunction &MF, MachineBasicBlock &MBB,
1291 MachineBasicBlock::iterator MI) const {
1292 Register SPReg = RISCV::X2;
1293 DebugLoc DL = MI->getDebugLoc();
1294
1295 if (!hasReservedCallFrame(MF)) {
1296 // If space has not been reserved for a call frame, ADJCALLSTACKDOWN and
1297 // ADJCALLSTACKUP must be converted to instructions manipulating the stack
1298 // pointer. This is necessary when there is a variable length stack
1299 // allocation (e.g. alloca), which means it's not possible to allocate
1300 // space for outgoing arguments from within the function prologue.
1301 int64_t Amount = MI->getOperand(i: 0).getImm();
1302
1303 if (Amount != 0) {
1304 // Ensure the stack remains aligned after adjustment.
1305 Amount = alignSPAdjust(SPAdj: Amount);
1306
1307 if (MI->getOpcode() == RISCV::ADJCALLSTACKDOWN)
1308 Amount = -Amount;
1309
1310 const RISCVRegisterInfo &RI = *STI.getRegisterInfo();
1311 RI.adjustReg(MBB, II: MI, DL, DestReg: SPReg, SrcReg: SPReg, Offset: StackOffset::getFixed(Fixed: Amount),
1312 Flag: MachineInstr::NoFlags, RequiredAlign: getStackAlign());
1313 }
1314 }
1315
1316 return MBB.erase(I: MI);
1317}
1318
1319// We would like to split the SP adjustment to reduce prologue/epilogue
1320// as following instructions. In this way, the offset of the callee saved
1321// register could fit in a single store. Supposed that the first sp adjust
1322// amount is 2032.
1323// add sp,sp,-2032
1324// sw ra,2028(sp)
1325// sw s0,2024(sp)
1326// sw s1,2020(sp)
1327// sw s3,2012(sp)
1328// sw s4,2008(sp)
1329// add sp,sp,-64
1330uint64_t
1331RISCVFrameLowering::getFirstSPAdjustAmount(const MachineFunction &MF) const {
1332 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1333 const MachineFrameInfo &MFI = MF.getFrameInfo();
1334 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
1335 uint64_t StackSize = getStackSizeWithRVVPadding(MF);
1336
1337 // Disable SplitSPAdjust if save-restore libcall is used. The callee-saved
1338 // registers will be pushed by the save-restore libcalls, so we don't have to
1339 // split the SP adjustment in this case.
1340 if (RVFI->getReservedSpillsSize())
1341 return 0;
1342
1343 // Return the FirstSPAdjustAmount if the StackSize can not fit in a signed
1344 // 12-bit and there exists a callee-saved register needing to be pushed.
1345 if (!isInt<12>(x: StackSize) && (CSI.size() > 0)) {
1346 // FirstSPAdjustAmount is chosen at most as (2048 - StackAlign) because
1347 // 2048 will cause sp = sp + 2048 in the epilogue to be split into multiple
1348 // instructions. Offsets smaller than 2048 can fit in a single load/store
1349 // instruction, and we have to stick with the stack alignment. 2048 has
1350 // 16-byte alignment. The stack alignment for RV32 and RV64 is 16 and for
1351 // RV32E it is 4. So (2048 - StackAlign) will satisfy the stack alignment.
1352 const uint64_t StackAlign = getStackAlign().value();
1353
1354 // Amount of (2048 - StackAlign) will prevent callee saved and restored
1355 // instructions be compressed, so try to adjust the amount to the largest
1356 // offset that stack compression instructions accept when target supports
1357 // compression instructions.
1358 if (STI.hasStdExtCOrZca()) {
1359 // The compression extensions may support the following instructions:
1360 // riscv32: c.lwsp rd, offset[7:2] => 2^(6 + 2)
1361 // c.swsp rs2, offset[7:2] => 2^(6 + 2)
1362 // c.flwsp rd, offset[7:2] => 2^(6 + 2)
1363 // c.fswsp rs2, offset[7:2] => 2^(6 + 2)
1364 // riscv64: c.ldsp rd, offset[8:3] => 2^(6 + 3)
1365 // c.sdsp rs2, offset[8:3] => 2^(6 + 3)
1366 // c.fldsp rd, offset[8:3] => 2^(6 + 3)
1367 // c.fsdsp rs2, offset[8:3] => 2^(6 + 3)
1368 const uint64_t RVCompressLen = STI.getXLen() * 8;
1369 // Compared with amount (2048 - StackAlign), StackSize needs to
1370 // satisfy the following conditions to avoid using more instructions
1371 // to adjust the sp after adjusting the amount, such as
1372 // StackSize meets the condition (StackSize <= 2048 + RVCompressLen),
1373 // case1: Amount is 2048 - StackAlign: use addi + addi to adjust sp.
1374 // case2: Amount is RVCompressLen: use addi + addi to adjust sp.
1375 auto CanCompress = [&](uint64_t CompressLen) -> bool {
1376 if (StackSize <= 2047 + CompressLen ||
1377 (StackSize > 2048 * 2 - StackAlign &&
1378 StackSize <= 2047 * 2 + CompressLen) ||
1379 StackSize > 2048 * 3 - StackAlign)
1380 return true;
1381
1382 return false;
1383 };
1384 // In the epilogue, addi sp, sp, 496 is used to recover the sp and it
1385 // can be compressed(C.ADDI16SP, offset can be [-512, 496]), but
1386 // addi sp, sp, 512 can not be compressed. So try to use 496 first.
1387 const uint64_t ADDI16SPCompressLen = 496;
1388 if (STI.is64Bit() && CanCompress(ADDI16SPCompressLen))
1389 return ADDI16SPCompressLen;
1390 if (CanCompress(RVCompressLen))
1391 return RVCompressLen;
1392 }
1393 return 2048 - StackAlign;
1394 }
1395 return 0;
1396}
1397
1398bool RISCVFrameLowering::assignCalleeSavedSpillSlots(
1399 MachineFunction &MF, const TargetRegisterInfo *TRI,
1400 std::vector<CalleeSavedInfo> &CSI, unsigned &MinCSFrameIndex,
1401 unsigned &MaxCSFrameIndex) const {
1402 // Early exit if no callee saved registers are modified!
1403 if (CSI.empty())
1404 return true;
1405
1406 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1407
1408 if (RVFI->isPushable(MF)) {
1409 // Determine how many GPRs we need to push and save it to RVFI.
1410 Register MaxReg = getMaxPushPopReg(MF, CSI);
1411 if (MaxReg != RISCV::NoRegister) {
1412 auto [RegEnc, PushedRegNum] = getPushPopEncodingAndNum(MaxReg);
1413 RVFI->setRVPushRegs(PushedRegNum);
1414 RVFI->setRVPushStackSize(alignTo(Value: (STI.getXLen() / 8) * PushedRegNum, Align: 16));
1415
1416 // Use encoded number to represent registers to spill.
1417 RVFI->setRVPushRlist(RegEnc);
1418 }
1419 }
1420
1421 MachineFrameInfo &MFI = MF.getFrameInfo();
1422 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
1423
1424 for (auto &CS : CSI) {
1425 unsigned Reg = CS.getReg();
1426 const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
1427 unsigned Size = RegInfo->getSpillSize(RC: *RC);
1428
1429 // This might need a fixed stack slot.
1430 if (RVFI->useSaveRestoreLibCalls(MF) || RVFI->isPushable(MF)) {
1431 const auto *FII = llvm::find_if(
1432 Range: FixedCSRFIMap, P: [&](auto P) { return P.first == CS.getReg(); });
1433 if (FII != std::end(arr: FixedCSRFIMap)) {
1434 int64_t Offset;
1435 if (RVFI->isPushable(MF))
1436 Offset = -((FII->second + RVFI->getRVPushRegs() + 1) * (int64_t)Size);
1437 else
1438 Offset = FII->second * (int64_t)Size;
1439
1440 int FrameIdx = MFI.CreateFixedSpillStackObject(Size, SPOffset: Offset);
1441 assert(FrameIdx < 0);
1442 CS.setFrameIdx(FrameIdx);
1443 continue;
1444 }
1445 }
1446
1447 // Not a fixed slot.
1448 Align Alignment = RegInfo->getSpillAlign(RC: *RC);
1449 // We may not be able to satisfy the desired alignment specification of
1450 // the TargetRegisterClass if the stack alignment is smaller. Use the
1451 // min.
1452 Alignment = std::min(a: Alignment, b: getStackAlign());
1453 int FrameIdx = MFI.CreateStackObject(Size, Alignment, isSpillSlot: true);
1454 if ((unsigned)FrameIdx < MinCSFrameIndex)
1455 MinCSFrameIndex = FrameIdx;
1456 if ((unsigned)FrameIdx > MaxCSFrameIndex)
1457 MaxCSFrameIndex = FrameIdx;
1458 CS.setFrameIdx(FrameIdx);
1459 }
1460
1461 // Allocate a fixed object that covers the full push or libcall size.
1462 if (RVFI->isPushable(MF)) {
1463 if (int64_t PushSize = RVFI->getRVPushStackSize())
1464 MFI.CreateFixedSpillStackObject(Size: PushSize, SPOffset: -PushSize);
1465 } else if (int LibCallRegs = getLibCallID(MF, CSI) + 1) {
1466 int64_t LibCallFrameSize =
1467 alignTo(Size: (STI.getXLen() / 8) * LibCallRegs, A: getStackAlign());
1468 MFI.CreateFixedSpillStackObject(Size: LibCallFrameSize, SPOffset: -LibCallFrameSize);
1469 }
1470
1471 return true;
1472}
1473
1474bool RISCVFrameLowering::spillCalleeSavedRegisters(
1475 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1476 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
1477 if (CSI.empty())
1478 return true;
1479
1480 MachineFunction *MF = MBB.getParent();
1481 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
1482 DebugLoc DL;
1483 if (MI != MBB.end() && !MI->isDebugInstr())
1484 DL = MI->getDebugLoc();
1485
1486 // Emit CM.PUSH with base SPimm & evaluate Push stack
1487 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1488 if (RVFI->isPushable(MF: *MF)) {
1489 unsigned PushedRegNum = RVFI->getRVPushRegs();
1490 if (PushedRegNum > 0) {
1491 // Use encoded number to represent registers to spill.
1492 int RegEnc = RVFI->getRVPushRlist();
1493 MachineInstrBuilder PushBuilder =
1494 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::CM_PUSH))
1495 .setMIFlag(MachineInstr::FrameSetup);
1496 PushBuilder.addImm(Val: (int64_t)RegEnc);
1497 PushBuilder.addImm(Val: 0);
1498
1499 for (unsigned i = 0; i < PushedRegNum; i++)
1500 PushBuilder.addUse(RegNo: FixedCSRFIMap[i].first, Flags: RegState::Implicit);
1501 }
1502 } else if (const char *SpillLibCall = getSpillLibCallName(MF: *MF, CSI)) {
1503 // Add spill libcall via non-callee-saved register t0.
1504 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::PseudoCALLReg), DestReg: RISCV::X5)
1505 .addExternalSymbol(FnName: SpillLibCall, TargetFlags: RISCVII::MO_CALL)
1506 .setMIFlag(MachineInstr::FrameSetup);
1507
1508 // Add registers spilled in libcall as liveins.
1509 for (auto &CS : CSI)
1510 MBB.addLiveIn(PhysReg: CS.getReg());
1511 }
1512
1513 // Manually spill values not spilled by libcall & Push/Pop.
1514 const auto &UnmanagedCSI = getUnmanagedCSI(MF: *MF, CSI);
1515 const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI);
1516
1517 auto storeRegToStackSlot = [&](decltype(UnmanagedCSI) CSInfo) {
1518 for (auto &CS : CSInfo) {
1519 // Insert the spill to the stack frame.
1520 Register Reg = CS.getReg();
1521 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
1522 TII.storeRegToStackSlot(MBB, MI, SrcReg: Reg, isKill: !MBB.isLiveIn(Reg),
1523 FrameIndex: CS.getFrameIdx(), RC, TRI, VReg: Register());
1524 }
1525 };
1526 storeRegToStackSlot(UnmanagedCSI);
1527 storeRegToStackSlot(RVVCSI);
1528
1529 return true;
1530}
1531
1532void RISCVFrameLowering::emitCalleeSavedRVVPrologCFI(
1533 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, bool HasFP) const {
1534 MachineFunction *MF = MBB.getParent();
1535 const MachineFrameInfo &MFI = MF->getFrameInfo();
1536 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1537 const TargetInstrInfo &TII = *STI.getInstrInfo();
1538 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
1539
1540 const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI: MFI.getCalleeSavedInfo());
1541 if (RVVCSI.empty())
1542 return;
1543
1544 uint64_t FixedSize = getStackSizeWithRVVPadding(MF: *MF);
1545 if (!HasFP) {
1546 uint64_t ScalarLocalVarSize =
1547 MFI.getStackSize() - RVFI->getCalleeSavedStackSize() -
1548 RVFI->getRVPushStackSize() - RVFI->getVarArgsSaveSize() +
1549 RVFI->getRVVPadding();
1550 FixedSize -= ScalarLocalVarSize;
1551 }
1552
1553 for (auto &CS : RVVCSI) {
1554 // Insert the spill to the stack frame.
1555 int FI = CS.getFrameIdx();
1556 if (FI >= 0 && MFI.getStackID(ObjectIdx: FI) == TargetStackID::ScalableVector) {
1557 unsigned CFIIndex = MF->addFrameInst(
1558 Inst: createDefCFAOffset(TRI: *STI.getRegisterInfo(), Reg: CS.getReg(), FixedOffset: -FixedSize,
1559 ScalableOffset: MFI.getObjectOffset(ObjectIdx: FI) / 8));
1560 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::CFI_INSTRUCTION))
1561 .addCFIIndex(CFIIndex)
1562 .setMIFlag(MachineInstr::FrameSetup);
1563 }
1564 }
1565}
1566
1567bool RISCVFrameLowering::restoreCalleeSavedRegisters(
1568 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1569 MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
1570 if (CSI.empty())
1571 return true;
1572
1573 MachineFunction *MF = MBB.getParent();
1574 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
1575 DebugLoc DL;
1576 if (MI != MBB.end() && !MI->isDebugInstr())
1577 DL = MI->getDebugLoc();
1578
1579 // Manually restore values not restored by libcall & Push/Pop.
1580 // Reverse the restore order in epilog. In addition, the return
1581 // address will be restored first in the epilogue. It increases
1582 // the opportunity to avoid the load-to-use data hazard between
1583 // loading RA and return by RA. loadRegFromStackSlot can insert
1584 // multiple instructions.
1585 const auto &UnmanagedCSI = getUnmanagedCSI(MF: *MF, CSI);
1586 const auto &RVVCSI = getRVVCalleeSavedInfo(MF: *MF, CSI);
1587
1588 auto loadRegFromStackSlot = [&](decltype(UnmanagedCSI) CSInfo) {
1589 for (auto &CS : CSInfo) {
1590 Register Reg = CS.getReg();
1591 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
1592 TII.loadRegFromStackSlot(MBB, MI, DestReg: Reg, FrameIndex: CS.getFrameIdx(), RC, TRI,
1593 VReg: Register());
1594 assert(MI != MBB.begin() &&
1595 "loadRegFromStackSlot didn't insert any code!");
1596 }
1597 };
1598 loadRegFromStackSlot(RVVCSI);
1599 loadRegFromStackSlot(UnmanagedCSI);
1600
1601 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1602 if (RVFI->isPushable(MF: *MF)) {
1603 int RegEnc = RVFI->getRVPushRlist();
1604 if (RegEnc != llvm::RISCVZC::RLISTENCODE::INVALID_RLIST) {
1605 MachineInstrBuilder PopBuilder =
1606 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::CM_POP))
1607 .setMIFlag(MachineInstr::FrameDestroy);
1608 // Use encoded number to represent registers to restore.
1609 PopBuilder.addImm(Val: RegEnc);
1610 PopBuilder.addImm(Val: 0);
1611
1612 for (unsigned i = 0; i < RVFI->getRVPushRegs(); i++)
1613 PopBuilder.addDef(RegNo: FixedCSRFIMap[i].first, Flags: RegState::ImplicitDefine);
1614 }
1615 } else {
1616 const char *RestoreLibCall = getRestoreLibCallName(MF: *MF, CSI);
1617 if (RestoreLibCall) {
1618 // Add restore libcall via tail call.
1619 MachineBasicBlock::iterator NewMI =
1620 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::PseudoTAIL))
1621 .addExternalSymbol(FnName: RestoreLibCall, TargetFlags: RISCVII::MO_CALL)
1622 .setMIFlag(MachineInstr::FrameDestroy);
1623
1624 // Remove trailing returns, since the terminator is now a tail call to the
1625 // restore function.
1626 if (MI != MBB.end() && MI->getOpcode() == RISCV::PseudoRET) {
1627 NewMI->copyImplicitOps(MF&: *MF, MI: *MI);
1628 MI->eraseFromParent();
1629 }
1630 }
1631 }
1632 return true;
1633}
1634
1635bool RISCVFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
1636 // Keep the conventional code flow when not optimizing.
1637 if (MF.getFunction().hasOptNone())
1638 return false;
1639
1640 return true;
1641}
1642
1643bool RISCVFrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const {
1644 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
1645 const MachineFunction *MF = MBB.getParent();
1646 const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1647
1648 if (!RVFI->useSaveRestoreLibCalls(MF: *MF))
1649 return true;
1650
1651 // Inserting a call to a __riscv_save libcall requires the use of the register
1652 // t0 (X5) to hold the return address. Therefore if this register is already
1653 // used we can't insert the call.
1654
1655 RegScavenger RS;
1656 RS.enterBasicBlock(MBB&: *TmpMBB);
1657 return !RS.isRegUsed(Reg: RISCV::X5);
1658}
1659
1660bool RISCVFrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
1661 const MachineFunction *MF = MBB.getParent();
1662 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
1663 const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1664
1665 if (!RVFI->useSaveRestoreLibCalls(MF: *MF))
1666 return true;
1667
1668 // Using the __riscv_restore libcalls to restore CSRs requires a tail call.
1669 // This means if we still need to continue executing code within this function
1670 // the restore cannot take place in this basic block.
1671
1672 if (MBB.succ_size() > 1)
1673 return false;
1674
1675 MachineBasicBlock *SuccMBB =
1676 MBB.succ_empty() ? TmpMBB->getFallThrough() : *MBB.succ_begin();
1677
1678 // Doing a tail call should be safe if there are no successors, because either
1679 // we have a returning block or the end of the block is unreachable, so the
1680 // restore will be eliminated regardless.
1681 if (!SuccMBB)
1682 return true;
1683
1684 // The successor can only contain a return, since we would effectively be
1685 // replacing the successor with our own tail return at the end of our block.
1686 return SuccMBB->isReturnBlock() && SuccMBB->size() == 1;
1687}
1688
1689bool RISCVFrameLowering::isSupportedStackID(TargetStackID::Value ID) const {
1690 switch (ID) {
1691 case TargetStackID::Default:
1692 case TargetStackID::ScalableVector:
1693 return true;
1694 case TargetStackID::NoAlloc:
1695 case TargetStackID::SGPRSpill:
1696 case TargetStackID::WasmLocal:
1697 return false;
1698 }
1699 llvm_unreachable("Invalid TargetStackID::Value");
1700}
1701
1702TargetStackID::Value RISCVFrameLowering::getStackIDForScalableVectors() const {
1703 return TargetStackID::ScalableVector;
1704}
1705