1//=== ARMCallingConv.cpp - ARM Custom CC Routines ---------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the custom routines for the ARM Calling Convention that
10// aren't done by tablegen, and includes the table generated implementations.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARMCallingConv.h"
15#include "ARMSubtarget.h"
16using namespace llvm;
17
18// APCS f64 is in register pairs, possibly split to stack
19static bool f64AssignAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
20 CCValAssign::LocInfo LocInfo,
21 CCState &State, bool CanFail) {
22 static const MCPhysReg RegList[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };
23
24 // Try to get the first register.
25 if (MCRegister Reg = State.AllocateReg(Regs: RegList))
26 State.addLoc(V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
27 else {
28 // For the 2nd half of a v2f64, do not fail.
29 if (CanFail)
30 return false;
31
32 // Put the whole thing on the stack.
33 State.addLoc(V: CCValAssign::getCustomMem(
34 ValNo, ValVT, Offset: State.AllocateStack(Size: 8, Alignment: Align(4)), LocVT, HTP: LocInfo));
35 return true;
36 }
37
38 // Try to get the second register.
39 if (MCRegister Reg = State.AllocateReg(Regs: RegList))
40 State.addLoc(V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
41 else
42 State.addLoc(V: CCValAssign::getCustomMem(
43 ValNo, ValVT, Offset: State.AllocateStack(Size: 4, Alignment: Align(4)), LocVT, HTP: LocInfo));
44 return true;
45}
46
47static bool CC_ARM_APCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,
48 CCValAssign::LocInfo LocInfo,
49 ISD::ArgFlagsTy ArgFlags,
50 CCState &State) {
51 if (!f64AssignAPCS(ValNo, ValVT, LocVT, LocInfo, State, CanFail: true))
52 return false;
53 if (LocVT == MVT::v2f64 &&
54 !f64AssignAPCS(ValNo, ValVT, LocVT, LocInfo, State, CanFail: false))
55 return false;
56 return true; // we handled it
57}
58
59// AAPCS f64 is in aligned register pairs
60static bool f64AssignAAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
61 CCValAssign::LocInfo LocInfo,
62 CCState &State, bool CanFail) {
63 static const MCPhysReg HiRegList[] = { ARM::R0, ARM::R2 };
64 static const MCPhysReg LoRegList[] = { ARM::R1, ARM::R3 };
65 static const MCPhysReg ShadowRegList[] = { ARM::R0, ARM::R1 };
66 static const MCPhysReg GPRArgRegs[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };
67
68 MCRegister Reg = State.AllocateReg(Regs: HiRegList, ShadowRegs: ShadowRegList);
69 if (!Reg) {
70
71 // If we had R3 unallocated only, now we still must to waste it.
72 Reg = State.AllocateReg(Regs: GPRArgRegs);
73 assert((!Reg || Reg == ARM::R3) && "Wrong GPRs usage for f64");
74
75 // For the 2nd half of a v2f64, do not just fail.
76 if (CanFail)
77 return false;
78
79 // Put the whole thing on the stack.
80 State.addLoc(V: CCValAssign::getCustomMem(
81 ValNo, ValVT, Offset: State.AllocateStack(Size: 8, Alignment: Align(8)), LocVT, HTP: LocInfo));
82 return true;
83 }
84
85 unsigned i;
86 for (i = 0; i < 2; ++i)
87 if (HiRegList[i] == Reg)
88 break;
89
90 MCRegister T = State.AllocateReg(Reg: LoRegList[i]);
91 (void)T;
92 assert(T == LoRegList[i] && "Could not allocate register");
93
94 State.addLoc(V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
95 State.addLoc(V: CCValAssign::getCustomReg(ValNo, ValVT, Reg: LoRegList[i],
96 LocVT, HTP: LocInfo));
97 return true;
98}
99
100static bool CC_ARM_AAPCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,
101 CCValAssign::LocInfo LocInfo,
102 ISD::ArgFlagsTy ArgFlags,
103 CCState &State) {
104 if (!f64AssignAAPCS(ValNo, ValVT, LocVT, LocInfo, State, CanFail: true))
105 return false;
106 if (LocVT == MVT::v2f64 &&
107 !f64AssignAAPCS(ValNo, ValVT, LocVT, LocInfo, State, CanFail: false))
108 return false;
109 return true; // we handled it
110}
111
112static bool f64RetAssign(unsigned ValNo, MVT ValVT, MVT LocVT,
113 CCValAssign::LocInfo LocInfo, CCState &State) {
114 static const MCPhysReg HiRegList[] = { ARM::R0, ARM::R2 };
115 static const MCPhysReg LoRegList[] = { ARM::R1, ARM::R3 };
116
117 MCRegister Reg = State.AllocateReg(Regs: HiRegList, ShadowRegs: LoRegList);
118 if (!Reg)
119 return false; // we didn't handle it
120
121 unsigned i;
122 for (i = 0; i < 2; ++i)
123 if (HiRegList[i] == Reg)
124 break;
125
126 State.addLoc(V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
127 State.addLoc(V: CCValAssign::getCustomReg(ValNo, ValVT, Reg: LoRegList[i],
128 LocVT, HTP: LocInfo));
129 return true;
130}
131
132static bool RetCC_ARM_APCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,
133 CCValAssign::LocInfo LocInfo,
134 ISD::ArgFlagsTy ArgFlags,
135 CCState &State) {
136 if (!f64RetAssign(ValNo, ValVT, LocVT, LocInfo, State))
137 return false;
138 if (LocVT == MVT::v2f64 && !f64RetAssign(ValNo, ValVT, LocVT, LocInfo, State))
139 return false;
140 return true; // we handled it
141}
142
143static bool RetCC_ARM_AAPCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,
144 CCValAssign::LocInfo LocInfo,
145 ISD::ArgFlagsTy ArgFlags,
146 CCState &State) {
147 return RetCC_ARM_APCS_Custom_f64(ValNo, ValVT, LocVT, LocInfo, ArgFlags,
148 State);
149}
150
151static const MCPhysReg RRegList[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };
152
153static const MCPhysReg SRegList[] = { ARM::S0, ARM::S1, ARM::S2, ARM::S3,
154 ARM::S4, ARM::S5, ARM::S6, ARM::S7,
155 ARM::S8, ARM::S9, ARM::S10, ARM::S11,
156 ARM::S12, ARM::S13, ARM::S14, ARM::S15 };
157static const MCPhysReg DRegList[] = { ARM::D0, ARM::D1, ARM::D2, ARM::D3,
158 ARM::D4, ARM::D5, ARM::D6, ARM::D7 };
159static const MCPhysReg QRegList[] = { ARM::Q0, ARM::Q1, ARM::Q2, ARM::Q3 };
160
161
162// Allocate part of an AAPCS HFA or HVA. We assume that each member of the HA
163// has InConsecutiveRegs set, and that the last member also has
164// InConsecutiveRegsLast set. We must process all members of the HA before
165// we can allocate it, as we need to know the total number of registers that
166// will be needed in order to (attempt to) allocate a contiguous block.
167static bool CC_ARM_AAPCS_Custom_Aggregate(unsigned ValNo, MVT ValVT,
168 MVT LocVT,
169 CCValAssign::LocInfo LocInfo,
170 ISD::ArgFlagsTy ArgFlags,
171 CCState &State) {
172 SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs();
173
174 // AAPCS HFAs must have 1-4 elements, all of the same type
175 if (PendingMembers.size() > 0)
176 assert(PendingMembers[0].getLocVT() == LocVT);
177
178 // Add the argument to the list to be allocated once we know the size of the
179 // aggregate. Store the type's required alignment as extra info for later: in
180 // the [N x i64] case all trace has been removed by the time we actually get
181 // to do allocation.
182 PendingMembers.push_back(Elt: CCValAssign::getPending(
183 ValNo, ValVT, LocVT, HTP: LocInfo, ExtraInfo: ArgFlags.getNonZeroOrigAlign().value()));
184
185 if (!ArgFlags.isInConsecutiveRegsLast())
186 return true;
187
188 const MachineFunction &MF = State.getMachineFunction();
189 // Try to allocate a contiguous block of registers, each of the correct
190 // size to hold one member.
191 auto &DL = MF.getDataLayout();
192 const MaybeAlign StackAlign = DL.getStackAlignment();
193 assert(StackAlign && "data layout string is missing stack alignment");
194 const Align FirstMemberAlign(PendingMembers[0].getExtraInfo());
195 Align Alignment = std::min(a: FirstMemberAlign, b: *StackAlign);
196
197 ArrayRef<MCPhysReg> RegList;
198 switch (LocVT.SimpleTy) {
199 case MVT::i32: {
200 RegList = RRegList;
201 unsigned RegIdx = State.getFirstUnallocated(Regs: RegList);
202
203 // First consume all registers that would give an unaligned object. Whether
204 // we go on stack or in regs, no-one will be using them in future.
205 unsigned RegAlign = alignTo(Value: Alignment.value(), Align: 4) / 4;
206 while (RegIdx % RegAlign != 0 && RegIdx < RegList.size())
207 State.AllocateReg(Reg: RegList[RegIdx++]);
208
209 break;
210 }
211 case MVT::f16:
212 case MVT::bf16:
213 case MVT::f32:
214 RegList = SRegList;
215 break;
216 case MVT::v4f16:
217 case MVT::v4bf16:
218 case MVT::f64:
219 RegList = DRegList;
220 break;
221 case MVT::v8f16:
222 case MVT::v8bf16:
223 case MVT::v2f64:
224 RegList = QRegList;
225 break;
226 default:
227 llvm_unreachable("Unexpected member type for block aggregate");
228 break;
229 }
230
231 ArrayRef<MCPhysReg> RegResult =
232 State.AllocateRegBlock(Regs: RegList, RegsRequired: PendingMembers.size());
233 if (!RegResult.empty()) {
234 for (const auto &[PendingMember, Reg] : zip(t&: PendingMembers, u&: RegResult)) {
235 PendingMember.convertToReg(Reg);
236 State.addLoc(V: PendingMember);
237 }
238 PendingMembers.clear();
239 return true;
240 }
241
242 // Register allocation failed, we'll be needing the stack
243 unsigned Size = LocVT.getSizeInBits() / 8;
244 if (LocVT == MVT::i32 && State.getStackSize() == 0) {
245 // If nothing else has used the stack until this point, a non-HFA aggregate
246 // can be split between regs and stack.
247 unsigned RegIdx = State.getFirstUnallocated(Regs: RegList);
248 for (auto &It : PendingMembers) {
249 if (RegIdx >= RegList.size())
250 It.convertToMem(Offset: State.AllocateStack(Size, Alignment: Align(Size)));
251 else
252 It.convertToReg(Reg: State.AllocateReg(Reg: RegList[RegIdx++]));
253
254 State.addLoc(V: It);
255 }
256 PendingMembers.clear();
257 return true;
258 }
259
260 if (LocVT != MVT::i32)
261 RegList = SRegList;
262
263 // Mark all regs as unavailable (AAPCS rule C.2.vfp for VFP, C.6 for core)
264 for (auto Reg : RegList)
265 State.AllocateReg(Reg);
266
267 // Clamp the alignment between 4 and 8.
268 if (MF.getTarget().getTargetTriple().isTargetAEABI())
269 Alignment = ArgFlags.getNonZeroMemAlign() <= 4 ? Align(4) : Align(8);
270
271 // After the first item has been allocated, the rest are packed as tightly as
272 // possible. (E.g. an incoming i64 would have starting Align of 8, but we'll
273 // be allocating a bunch of i32 slots).
274 for (auto &It : PendingMembers) {
275 It.convertToMem(Offset: State.AllocateStack(Size, Alignment));
276 State.addLoc(V: It);
277 Alignment = Align(1);
278 }
279
280 // All pending members have now been allocated
281 PendingMembers.clear();
282
283 // This will be allocated by the last member of the aggregate
284 return true;
285}
286
287static bool CustomAssignInRegList(unsigned ValNo, MVT ValVT, MVT LocVT,
288 CCValAssign::LocInfo LocInfo, CCState &State,
289 ArrayRef<MCPhysReg> RegList) {
290 MCRegister Reg = State.AllocateReg(Regs: RegList);
291 if (Reg) {
292 State.addLoc(V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, HTP: LocInfo));
293 return true;
294 }
295 return false;
296}
297
298static bool CC_ARM_AAPCS_Custom_f16(unsigned ValNo, MVT ValVT, MVT LocVT,
299 CCValAssign::LocInfo LocInfo,
300 ISD::ArgFlagsTy ArgFlags, CCState &State) {
301 // f16 and bf16 arguments are extended to i32 and assigned to a register in
302 // [r0, r3].
303 return CustomAssignInRegList(ValNo, ValVT, LocVT: MVT::i32, LocInfo, State,
304 RegList: RRegList);
305}
306
307static bool CC_ARM_AAPCS_VFP_Custom_f16(unsigned ValNo, MVT ValVT, MVT LocVT,
308 CCValAssign::LocInfo LocInfo,
309 ISD::ArgFlagsTy ArgFlags,
310 CCState &State) {
311 // f16 and bf16 arguments are extended to f32 and assigned to a register in
312 // [s0, s15].
313 return CustomAssignInRegList(ValNo, ValVT, LocVT: MVT::f32, LocInfo, State,
314 RegList: SRegList);
315}
316
317static bool CC_ARM_AAPCS_Common_Custom_f16_Stack(unsigned ValNo, MVT ValVT,
318 MVT LocVT,
319 CCValAssign::LocInfo LocInfo,
320 ISD::ArgFlagsTy ArgFlags,
321 CCState &State) {
322 // f16 and bf16 (if not passed in a register) are assigned to a 32-bit stack
323 // slot, with the most-significant 16 bits unspecified. The 32-bit slot is
324 // important to make sure that the byte ordering is correct for big endian
325 // targets.
326 State.addLoc(V: CCValAssign::getCustomMem(
327 ValNo, ValVT, Offset: State.AllocateStack(Size: 4, Alignment: Align(4)), LocVT: MVT::i32, HTP: LocInfo));
328 return true;
329}
330
331// Include the table generated calling convention implementations.
332#define GET_CALLING_CONV_IMPL
333#include "ARMGenCallingConv.inc"
334