1//===--- AArch64CallLowering.cpp - Call lowering --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the lowering of LLVM calls to machine code calls for
11/// GlobalISel.
12///
13//===----------------------------------------------------------------------===//
14
15#include "AArch64CallLowering.h"
16#include "AArch64GlobalISelUtils.h"
17#include "AArch64ISelLowering.h"
18#include "AArch64MachineFunctionInfo.h"
19#include "AArch64RegisterInfo.h"
20#include "AArch64SMEAttributes.h"
21#include "AArch64Subtarget.h"
22#include "AArch64TargetMachine.h"
23#include "Utils/AArch64BaseInfo.h"
24#include "llvm/ADT/ArrayRef.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/Analysis/ObjCARCUtil.h"
27#include "llvm/CodeGen/Analysis.h"
28#include "llvm/CodeGen/CallingConvLower.h"
29#include "llvm/CodeGen/FunctionLoweringInfo.h"
30#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
31#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
32#include "llvm/CodeGen/GlobalISel/Utils.h"
33#include "llvm/CodeGen/LowLevelTypeUtils.h"
34#include "llvm/CodeGen/MachineBasicBlock.h"
35#include "llvm/CodeGen/MachineFrameInfo.h"
36#include "llvm/CodeGen/MachineFunction.h"
37#include "llvm/CodeGen/MachineInstrBuilder.h"
38#include "llvm/CodeGen/MachineMemOperand.h"
39#include "llvm/CodeGen/MachineOperand.h"
40#include "llvm/CodeGen/MachineRegisterInfo.h"
41#include "llvm/CodeGen/TargetOpcodes.h"
42#include "llvm/CodeGen/TargetRegisterInfo.h"
43#include "llvm/CodeGen/TargetSubtargetInfo.h"
44#include "llvm/CodeGen/ValueTypes.h"
45#include "llvm/CodeGenTypes/MachineValueType.h"
46#include "llvm/IR/Argument.h"
47#include "llvm/IR/Attributes.h"
48#include "llvm/IR/Function.h"
49#include "llvm/IR/Type.h"
50#include "llvm/IR/Value.h"
51#include <algorithm>
52#include <cassert>
53#include <cstdint>
54
55#define DEBUG_TYPE "aarch64-call-lowering"
56
57using namespace llvm;
58using namespace AArch64GISelUtils;
59
60extern cl::opt<bool> EnableSVEGISel;
61
62static bool isSimpleGPRCallValue(const CallLowering::ArgInfo &Arg) {
63 if (Arg.Regs.size() != 1 || any_of(Range: Arg.Flags, P: [](ISD::ArgFlagsTy Flags) {
64 auto FlagVals = Flags.getFlags();
65 return FlagVals != ISD::ArgFlagsTy::NoFlags &&
66 FlagVals != ISD::ArgFlagsTy::Pointer;
67 }))
68 return false;
69
70 Type *Ty = Arg.Ty;
71 return Ty->isPointerTy() || Ty->isIntegerTy(BitWidth: 32) || Ty->isIntegerTy(BitWidth: 64);
72}
73
74// Avoid the generic assignment machinery when every argument maps directly to
75// w0-w7/x0-x7. Fast path for compile-time.
76static bool tryAssignSimpleGPRCallArgs(MachineIRBuilder &MIRBuilder,
77 MachineInstrBuilder MIB,
78 ArrayRef<CallLowering::ArgInfo> Args) {
79 if (Args.size() > 8)
80 return false;
81
82 for (const CallLowering::ArgInfo &Arg : Args)
83 if (!isSimpleGPRCallValue(Arg))
84 return false;
85
86 for (unsigned I = 0, E = Args.size(); I != E; ++I) {
87 const CallLowering::ArgInfo &Arg = Args[I];
88 MCRegister XReg = AArch64::getGPRArgRegs()[I];
89 Register PhysReg = Arg.Ty->isIntegerTy(BitWidth: 32) ? getWRegFromXReg(Reg: XReg) : XReg;
90 MIB.addUse(RegNo: PhysReg, Flags: RegState::Implicit);
91 MIRBuilder.buildCopy(Res: PhysReg, Op: Arg.Regs[0]);
92 }
93 return true;
94}
95
96// Avoid the generic assignment machinery when the return value maps directly
97// to w0/x0. Fast path for compile-time.
98static bool tryAssignSimpleGPRCallReturn(MachineIRBuilder &MIRBuilder,
99 MachineInstrBuilder MIB,
100 ArrayRef<CallLowering::ArgInfo> Rets) {
101 if (Rets.size() != 1)
102 return false;
103
104 const CallLowering::ArgInfo &Ret = Rets[0];
105 if (!isSimpleGPRCallValue(Arg: Ret))
106 return false;
107
108 Register PhysReg = Ret.Ty->isIntegerTy(BitWidth: 32) ? AArch64::W0 : AArch64::X0;
109 MIB.addDef(RegNo: PhysReg, Flags: RegState::Implicit);
110 MIRBuilder.buildCopy(Res: Ret.Regs[0], Op: PhysReg);
111 return true;
112}
113
114AArch64CallLowering::AArch64CallLowering(const AArch64TargetLowering &TLI)
115 : CallLowering(&TLI) {}
116
117static void applyStackPassedSmallTypeDAGHack(EVT OrigVT, MVT &ValVT,
118 MVT &LocVT) {
119 // If ValVT is i1/i8/i16, we should set LocVT to i8/i8/i16. This is a legacy
120 // hack because the DAG calls the assignment function with pre-legalized
121 // register typed values, not the raw type.
122 //
123 // This hack is not applied to return values which are not passed on the
124 // stack.
125 if (OrigVT == MVT::i1 || OrigVT == MVT::i8)
126 ValVT = LocVT = MVT::i8;
127 else if (OrigVT == MVT::i16)
128 ValVT = LocVT = MVT::i16;
129}
130
131// Account for i1/i8/i16 stack passed value hack
132static LLT getStackValueStoreTypeHack(const CCValAssign &VA) {
133 const MVT ValVT = VA.getValVT();
134 return (ValVT == MVT::i8 || ValVT == MVT::i16) ? LLT(ValVT)
135 : LLT(VA.getLocVT());
136}
137
138namespace {
139
140struct AArch64IncomingValueAssigner
141 : public CallLowering::IncomingValueAssigner {
142 AArch64IncomingValueAssigner(CCAssignFn *AssignFn_,
143 CCAssignFn *AssignFnVarArg_)
144 : IncomingValueAssigner(AssignFn_, AssignFnVarArg_) {}
145
146 bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
147 CCValAssign::LocInfo LocInfo,
148 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags,
149 CCState &State) override {
150 applyStackPassedSmallTypeDAGHack(OrigVT, ValVT, LocVT);
151 return IncomingValueAssigner::assignArg(ValNo, OrigVT, ValVT, LocVT,
152 LocInfo, Info, Flags, State);
153 }
154};
155
156struct AArch64OutgoingValueAssigner
157 : public CallLowering::OutgoingValueAssigner {
158 const AArch64Subtarget &Subtarget;
159
160 /// Track if this is used for a return instead of function argument
161 /// passing. We apply a hack to i1/i8/i16 stack passed values, but do not use
162 /// stack passed returns for them and cannot apply the type adjustment.
163 bool IsReturn;
164
165 AArch64OutgoingValueAssigner(CCAssignFn *AssignFn_,
166 CCAssignFn *AssignFnVarArg_,
167 const AArch64Subtarget &Subtarget_,
168 bool IsReturn)
169 : OutgoingValueAssigner(AssignFn_, AssignFnVarArg_),
170 Subtarget(Subtarget_), IsReturn(IsReturn) {}
171
172 bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
173 CCValAssign::LocInfo LocInfo,
174 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags,
175 CCState &State) override {
176 const Function &F = State.getMachineFunction().getFunction();
177 bool IsCalleeWin =
178 Subtarget.isCallingConvWin64(CC: State.getCallingConv(), IsVarArg: F.isVarArg());
179 bool UseVarArgsCCForFixed = IsCalleeWin && State.isVarArg();
180
181 bool Res;
182 if (!Flags.isVarArg() && !UseVarArgsCCForFixed) {
183 if (!IsReturn)
184 applyStackPassedSmallTypeDAGHack(OrigVT, ValVT, LocVT);
185 Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, Info.Ty, State);
186 } else
187 Res = AssignFnVarArg(ValNo, ValVT, LocVT, LocInfo, Flags, Info.Ty, State);
188
189 StackSize = State.getStackSize();
190 return Res;
191 }
192};
193
194struct IncomingArgHandler : public CallLowering::IncomingValueHandler {
195 IncomingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
196 : IncomingValueHandler(MIRBuilder, MRI) {}
197
198 Register getStackAddress(uint64_t Size, int64_t Offset,
199 MachinePointerInfo &MPO,
200 ISD::ArgFlagsTy Flags) override {
201 auto &MFI = MIRBuilder.getMF().getFrameInfo();
202
203 // Byval is assumed to be writable memory, but other stack passed arguments
204 // are not.
205 const bool IsImmutable = !Flags.isByVal();
206
207 int FI = MFI.CreateFixedObject(Size, SPOffset: Offset, IsImmutable);
208 MPO = MachinePointerInfo::getFixedStack(MF&: MIRBuilder.getMF(), FI);
209 auto AddrReg = MIRBuilder.buildFrameIndex(Res: LLT::pointer(AddressSpace: 0, SizeInBits: 64), Idx: FI);
210 return AddrReg.getReg(Idx: 0);
211 }
212
213 LLT getStackValueStoreType(const DataLayout &DL, const CCValAssign &VA,
214 ISD::ArgFlagsTy Flags) const override {
215 // For pointers, we just need to fixup the integer types reported in the
216 // CCValAssign.
217 if (Flags.isPointer())
218 return CallLowering::ValueHandler::getStackValueStoreType(DL, VA, Flags);
219 return getStackValueStoreTypeHack(VA);
220 }
221
222 void assignValueToReg(Register ValVReg, Register PhysReg,
223 const CCValAssign &VA,
224 ISD::ArgFlagsTy Flags = {}) override {
225 markRegUsed(Reg: PhysReg);
226 IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA);
227 }
228
229 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
230 const MachinePointerInfo &MPO,
231 const CCValAssign &VA) override {
232 MachineFunction &MF = MIRBuilder.getMF();
233
234 LLT ValTy(VA.getValVT());
235 LLT LocTy(VA.getLocVT());
236
237 // Fixup the types for the DAG compatibility hack.
238 if (VA.getValVT() == MVT::i8 || VA.getValVT() == MVT::i16)
239 std::swap(a&: ValTy, b&: LocTy);
240 else {
241 // The calling code knows if this is a pointer or not, we're only touching
242 // the LocTy for the i8/i16 hack.
243 assert(LocTy.getSizeInBits() == MemTy.getSizeInBits());
244 LocTy = MemTy;
245 }
246
247 auto MMO = MF.getMachineMemOperand(
248 PtrInfo: MPO, F: MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, MemTy: LocTy,
249 BaseAlignment: inferAlignFromPtrInfo(MF, MPO));
250
251 switch (VA.getLocInfo()) {
252 case CCValAssign::LocInfo::ZExt:
253 MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_ZEXTLOAD, Res: ValVReg, Addr, MMO&: *MMO);
254 return;
255 case CCValAssign::LocInfo::SExt:
256 MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_SEXTLOAD, Res: ValVReg, Addr, MMO&: *MMO);
257 return;
258 default:
259 MIRBuilder.buildLoad(Res: ValVReg, Addr, MMO&: *MMO);
260 return;
261 }
262 }
263
264 /// How the physical register gets marked varies between formal
265 /// parameters (it's a basic-block live-in), and a call instruction
266 /// (it's an implicit-def of the BL).
267 virtual void markRegUsed(Register Reg) = 0;
268};
269
270struct FormalArgHandler : public IncomingArgHandler {
271 FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
272 : IncomingArgHandler(MIRBuilder, MRI) {}
273
274 void markRegUsed(Register Reg) override {
275 MIRBuilder.getMRI()->addLiveIn(Reg: Reg.asMCReg());
276 MIRBuilder.getMBB().addLiveIn(PhysReg: Reg.asMCReg());
277 }
278};
279
280struct CallReturnHandler : public IncomingArgHandler {
281 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
282 MachineInstrBuilder MIB)
283 : IncomingArgHandler(MIRBuilder, MRI), MIB(MIB) {}
284
285 void markRegUsed(Register Reg) override {
286 MIB.addDef(RegNo: Reg, Flags: RegState::Implicit);
287 }
288
289 MachineInstrBuilder MIB;
290};
291
292/// A special return arg handler for "returned" attribute arg calls.
293struct ReturnedArgCallReturnHandler : public CallReturnHandler {
294 ReturnedArgCallReturnHandler(MachineIRBuilder &MIRBuilder,
295 MachineRegisterInfo &MRI,
296 MachineInstrBuilder MIB)
297 : CallReturnHandler(MIRBuilder, MRI, MIB) {}
298
299 void markRegUsed(Register Reg) override {}
300};
301
302struct OutgoingArgHandler : public CallLowering::OutgoingValueHandler {
303 OutgoingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
304 MachineInstrBuilder MIB, bool IsTailCall = false,
305 int FPDiff = 0)
306 : OutgoingValueHandler(MIRBuilder, MRI), MIB(MIB), IsTailCall(IsTailCall),
307 FPDiff(FPDiff),
308 Subtarget(MIRBuilder.getMF().getSubtarget<AArch64Subtarget>()) {}
309
310 Register getStackAddress(uint64_t Size, int64_t Offset,
311 MachinePointerInfo &MPO,
312 ISD::ArgFlagsTy Flags) override {
313 MachineFunction &MF = MIRBuilder.getMF();
314 LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: 64);
315 LLT s64 = LLT::integer(SizeInBits: 64);
316
317 if (IsTailCall) {
318 assert(!Flags.isByVal() && "byval unhandled with tail calls");
319
320 Offset += FPDiff;
321 int FI = MF.getFrameInfo().CreateFixedObject(Size, SPOffset: Offset, IsImmutable: true);
322 auto FIReg = MIRBuilder.buildFrameIndex(Res: p0, Idx: FI);
323 MPO = MachinePointerInfo::getFixedStack(MF, FI);
324 return FIReg.getReg(Idx: 0);
325 }
326
327 if (!SPReg)
328 SPReg = MIRBuilder.buildCopy(Res: p0, Op: Register(AArch64::SP)).getReg(Idx: 0);
329
330 auto OffsetReg = MIRBuilder.buildConstant(Res: s64, Val: Offset);
331
332 auto AddrReg = MIRBuilder.buildPtrAdd(Res: p0, Op0: SPReg, Op1: OffsetReg);
333
334 MPO = MachinePointerInfo::getStack(MF, Offset);
335 return AddrReg.getReg(Idx: 0);
336 }
337
338 /// We need to fixup the reported store size for certain value types because
339 /// we invert the interpretation of ValVT and LocVT in certain cases. This is
340 /// for compatibility with the DAG call lowering implementation, which we're
341 /// currently building on top of.
342 LLT getStackValueStoreType(const DataLayout &DL, const CCValAssign &VA,
343 ISD::ArgFlagsTy Flags) const override {
344 if (Flags.isPointer())
345 return CallLowering::ValueHandler::getStackValueStoreType(DL, VA, Flags);
346 return getStackValueStoreTypeHack(VA);
347 }
348
349 void assignValueToReg(Register ValVReg, Register PhysReg,
350 const CCValAssign &VA, ISD::ArgFlagsTy Flags) override {
351 MIB.addUse(RegNo: PhysReg, Flags: RegState::Implicit);
352 Register ExtReg = extendRegister(ValReg: ValVReg, VA);
353 MIRBuilder.buildCopy(Res: PhysReg, Op: ExtReg);
354 }
355
356 /// Check whether a stack argument requires lowering in a tail call.
357 static bool shouldLowerTailCallStackArg(const MachineFunction &MF,
358 const CCValAssign &VA,
359 Register ValVReg,
360 Register StoreAddr) {
361 const MachineRegisterInfo &MRI = MF.getRegInfo();
362 // Print the defining instruction for the value.
363 auto *DefMI = MRI.getVRegDef(Reg: ValVReg);
364 assert(DefMI && "No defining instruction");
365 for (;;) {
366 // Look through nodes that don't alter the bits of the incoming value.
367 unsigned Op = DefMI->getOpcode();
368 if (Op == TargetOpcode::G_ZEXT || Op == TargetOpcode::G_ANYEXT ||
369 Op == TargetOpcode::G_BITCAST || isAssertMI(MI: *DefMI)) {
370 DefMI = MRI.getVRegDef(Reg: DefMI->getOperand(i: 1).getReg());
371 continue;
372 }
373 break;
374 }
375
376 auto *Load = dyn_cast<GLoad>(Val: DefMI);
377 if (!Load)
378 return true;
379 Register LoadReg = Load->getPointerReg();
380 auto *LoadAddrDef = MRI.getVRegDef(Reg: LoadReg);
381 if (LoadAddrDef->getOpcode() != TargetOpcode::G_FRAME_INDEX)
382 return true;
383 const MachineFrameInfo &MFI = MF.getFrameInfo();
384 int LoadFI = LoadAddrDef->getOperand(i: 1).getIndex();
385
386 auto *StoreAddrDef = MRI.getVRegDef(Reg: StoreAddr);
387 if (StoreAddrDef->getOpcode() != TargetOpcode::G_FRAME_INDEX)
388 return true;
389 int StoreFI = StoreAddrDef->getOperand(i: 1).getIndex();
390
391 if (!MFI.isImmutableObjectIndex(ObjectIdx: LoadFI))
392 return true;
393 if (MFI.getObjectOffset(ObjectIdx: LoadFI) != MFI.getObjectOffset(ObjectIdx: StoreFI))
394 return true;
395 if (Load->getMemSize() != MFI.getObjectSize(ObjectIdx: StoreFI))
396 return true;
397
398 return false;
399 }
400
401 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
402 const MachinePointerInfo &MPO,
403 const CCValAssign &VA) override {
404 MachineFunction &MF = MIRBuilder.getMF();
405 if (!FPDiff && !shouldLowerTailCallStackArg(MF, VA, ValVReg, StoreAddr: Addr))
406 return;
407 auto MMO = MF.getMachineMemOperand(PtrInfo: MPO, F: MachineMemOperand::MOStore, MemTy,
408 BaseAlignment: inferAlignFromPtrInfo(MF, MPO));
409 MIRBuilder.buildStore(Val: ValVReg, Addr, MMO&: *MMO);
410 }
411
412 void assignValueToAddress(const CallLowering::ArgInfo &Arg, unsigned RegIndex,
413 Register Addr, LLT MemTy,
414 const MachinePointerInfo &MPO,
415 const CCValAssign &VA) override {
416 unsigned MaxSize = MemTy.getSizeInBytes() * 8;
417 // For varargs, we always want to extend them to 8 bytes, in which case
418 // we disable setting a max.
419 if (Arg.Flags[0].isVarArg())
420 MaxSize = 0;
421
422 Register ValVReg = Arg.Regs[RegIndex];
423 if (VA.getLocInfo() != CCValAssign::LocInfo::FPExt) {
424 MVT LocVT = VA.getLocVT();
425 MVT ValVT = VA.getValVT();
426
427 if (VA.getValVT() == MVT::i8 || VA.getValVT() == MVT::i16) {
428 std::swap(a&: ValVT, b&: LocVT);
429 MemTy = LLT(VA.getValVT());
430 }
431
432 ValVReg = extendRegister(ValReg: ValVReg, VA, MaxSizeBits: MaxSize);
433 } else {
434 // The store does not cover the full allocated stack slot.
435 MemTy = LLT(VA.getValVT());
436 }
437
438 assignValueToAddress(ValVReg, Addr, MemTy, MPO, VA);
439 }
440
441 MachineInstrBuilder MIB;
442
443 bool IsTailCall;
444
445 /// For tail calls, the byte offset of the call's argument area from the
446 /// callee's. Unused elsewhere.
447 int FPDiff;
448
449 // Cache the SP register vreg if we need it more than once in this call site.
450 Register SPReg;
451
452 const AArch64Subtarget &Subtarget;
453};
454} // namespace
455
456static bool doesCalleeRestoreStack(CallingConv::ID CallConv, bool TailCallOpt) {
457 return (CallConv == CallingConv::Fast && TailCallOpt) ||
458 CallConv == CallingConv::Tail || CallConv == CallingConv::SwiftTail;
459}
460
461bool AArch64CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
462 const Value *Val,
463 ArrayRef<Register> VRegs,
464 FunctionLoweringInfo &FLI,
465 Register SwiftErrorVReg) const {
466 auto MIB = MIRBuilder.buildInstrNoInsert(Opcode: AArch64::RET_ReallyLR);
467 assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) &&
468 "Return value without a vreg");
469
470 bool Success = true;
471 if (!FLI.CanLowerReturn) {
472 insertSRetStores(MIRBuilder, RetTy: Val->getType(), VRegs, DemoteReg: FLI.DemoteRegister);
473 } else if (!VRegs.empty()) {
474 MachineFunction &MF = MIRBuilder.getMF();
475 const Function &F = MF.getFunction();
476 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
477
478 MachineRegisterInfo &MRI = MF.getRegInfo();
479 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
480 CCAssignFn *AssignFn = TLI.CCAssignFnForReturn(CC: F.getCallingConv());
481 auto &DL = F.getDataLayout();
482 LLVMContext &Ctx = Val->getType()->getContext();
483
484 SmallVector<EVT, 4> SplitEVTs;
485 ComputeValueVTs(TLI, DL, Ty: Val->getType(), ValueVTs&: SplitEVTs);
486 assert(VRegs.size() == SplitEVTs.size() &&
487 "For each split Type there should be exactly one VReg.");
488
489 SmallVector<ArgInfo, 8> SplitArgs;
490 CallingConv::ID CC = F.getCallingConv();
491
492 for (unsigned i = 0; i < SplitEVTs.size(); ++i) {
493 Register CurVReg = VRegs[i];
494 ArgInfo CurArgInfo = ArgInfo{CurVReg, SplitEVTs[i].getTypeForEVT(Context&: Ctx), 0};
495 setArgFlags(Arg&: CurArgInfo, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: F);
496
497 // i1 is a special case because SDAG i1 true is naturally zero extended
498 // when widened using ANYEXT. We need to do it explicitly here.
499 auto &Flags = CurArgInfo.Flags[0];
500 if (MRI.getType(Reg: CurVReg).getSizeInBits() == TypeSize::getFixed(ExactSize: 1) &&
501 !Flags.isSExt() && !Flags.isZExt()) {
502 CurVReg = MIRBuilder.buildZExt(Res: LLT::integer(SizeInBits: 8), Op: CurVReg).getReg(Idx: 0);
503 } else if (TLI.getNumRegistersForCallingConv(Context&: Ctx, CC, VT: SplitEVTs[i]) ==
504 1) {
505 // Some types will need extending as specified by the CC.
506 MVT NewVT = TLI.getRegisterTypeForCallingConv(Context&: Ctx, CC, VT: SplitEVTs[i]);
507 if (EVT(NewVT) != SplitEVTs[i]) {
508 unsigned ExtendOp = TargetOpcode::G_ANYEXT;
509 if (F.getAttributes().hasRetAttr(Kind: Attribute::SExt))
510 ExtendOp = TargetOpcode::G_SEXT;
511 else if (F.getAttributes().hasRetAttr(Kind: Attribute::ZExt))
512 ExtendOp = TargetOpcode::G_ZEXT;
513
514 LLT NewLLT(NewVT);
515 LLT OldLLT = getLLTForType(Ty&: *CurArgInfo.Ty, DL);
516 CurArgInfo.Ty = EVT(NewVT).getTypeForEVT(Context&: Ctx);
517 // Instead of an extend, we might have a vector type which needs
518 // padding with more elements, e.g. <2 x half> -> <4 x half>.
519 if (NewVT.isVector()) {
520 if (OldLLT.isVector()) {
521 if (NewLLT.getNumElements() > OldLLT.getNumElements()) {
522 CurVReg =
523 MIRBuilder.buildPadVectorWithUndefElements(Res: NewLLT, Op0: CurVReg)
524 .getReg(Idx: 0);
525 } else {
526 // Just do a vector extend.
527 CurVReg = MIRBuilder.buildInstr(Opc: ExtendOp, DstOps: {NewLLT}, SrcOps: {CurVReg})
528 .getReg(Idx: 0);
529 }
530 } else if (NewLLT.getNumElements() >= 2 &&
531 NewLLT.getNumElements() <= 8) {
532 // We need to pad a <1 x S> type to <2/4/8 x S>. Since we don't
533 // have <1 x S> vector types in GISel we use a build_vector
534 // instead of a vector merge/concat.
535 CurVReg =
536 MIRBuilder.buildPadVectorWithUndefElements(Res: NewLLT, Op0: CurVReg)
537 .getReg(Idx: 0);
538 } else {
539 LLVM_DEBUG(dbgs() << "Could not handle ret ty\n");
540 return false;
541 }
542 } else {
543 // If the split EVT was a <1 x T> vector, and NewVT is T, then we
544 // don't have to do anything since we don't distinguish between the
545 // two.
546 if (NewLLT.getScalarSizeInBits() !=
547 MRI.getType(Reg: CurVReg).getScalarSizeInBits()) {
548 // A scalar extend.
549 CurVReg = MIRBuilder.buildInstr(Opc: ExtendOp, DstOps: {NewLLT}, SrcOps: {CurVReg})
550 .getReg(Idx: 0);
551 }
552 }
553 }
554 }
555 if (CurVReg != CurArgInfo.Regs[0]) {
556 CurArgInfo.Regs[0] = CurVReg;
557 // Reset the arg flags after modifying CurVReg.
558 setArgFlags(Arg&: CurArgInfo, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: F);
559 }
560 splitToValueTypes(OrigArgInfo: CurArgInfo, SplitArgs, DL, CallConv: CC);
561 }
562
563 AArch64OutgoingValueAssigner Assigner(AssignFn, AssignFn, Subtarget,
564 /*IsReturn*/ true);
565 OutgoingArgHandler Handler(MIRBuilder, MRI, MIB);
566 Success = determineAndHandleAssignments(Handler, Assigner, Args&: SplitArgs,
567 MIRBuilder, CallConv: CC, IsVarArg: F.isVarArg());
568 }
569
570 if (SwiftErrorVReg) {
571 MIB.addUse(RegNo: AArch64::X21, Flags: RegState::Implicit);
572 MIRBuilder.buildCopy(Res: AArch64::X21, Op: SwiftErrorVReg);
573 }
574
575 MIRBuilder.insertInstr(MIB);
576 return Success;
577}
578
579bool AArch64CallLowering::canLowerReturn(MachineFunction &MF,
580 CallingConv::ID CallConv,
581 SmallVectorImpl<BaseArgInfo> &Outs,
582 bool IsVarArg) const {
583 SmallVector<CCValAssign, 16> ArgLocs;
584 const auto &TLI = *getTLI<AArch64TargetLowering>();
585 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs,
586 MF.getFunction().getContext());
587
588 return checkReturn(CCInfo, Outs, Fn: TLI.CCAssignFnForReturn(CC: CallConv));
589}
590
591/// Helper function to compute forwarded registers for musttail calls. Computes
592/// the forwarded registers, sets MBB liveness, and emits COPY instructions that
593/// can be used to save + restore registers later.
594static void handleMustTailForwardedRegisters(MachineIRBuilder &MIRBuilder,
595 CCAssignFn *AssignFn) {
596 MachineBasicBlock &MBB = MIRBuilder.getMBB();
597 MachineFunction &MF = MIRBuilder.getMF();
598 MachineFrameInfo &MFI = MF.getFrameInfo();
599
600 if (!MFI.hasMustTailInVarArgFunc())
601 return;
602
603 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
604 const Function &F = MF.getFunction();
605 assert(F.isVarArg() && "Expected F to be vararg?");
606
607 // Compute the set of forwarded registers. The rest are scratch.
608 SmallVector<CCValAssign, 16> ArgLocs;
609 CCState CCInfo(F.getCallingConv(), /*IsVarArg=*/true, MF, ArgLocs,
610 F.getContext());
611 SmallVector<MVT, 2> RegParmTypes;
612 RegParmTypes.push_back(Elt: MVT::i64);
613 RegParmTypes.push_back(Elt: MVT::f128);
614
615 // Later on, we can use this vector to restore the registers if necessary.
616 SmallVectorImpl<ForwardedRegister> &Forwards =
617 FuncInfo->getForwardedMustTailRegParms();
618 CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, Fn: AssignFn);
619
620 // Conservatively forward X8, since it might be used for an aggregate
621 // return.
622 if (!CCInfo.isAllocated(Reg: AArch64::X8)) {
623 Register X8VReg = MF.addLiveIn(PReg: AArch64::X8, RC: &AArch64::GPR64RegClass);
624 Forwards.push_back(Elt: ForwardedRegister(X8VReg, AArch64::X8, MVT::i64));
625 }
626
627 // Add the forwards to the MachineBasicBlock and MachineFunction.
628 for (const auto &F : Forwards) {
629 MBB.addLiveIn(PhysReg: F.PReg);
630 MIRBuilder.buildCopy(Res: Register(F.VReg), Op: Register(F.PReg));
631 }
632}
633
634bool AArch64CallLowering::fallBackToDAGISel(const MachineFunction &MF) const {
635 auto &F = MF.getFunction();
636 const auto &TM = static_cast<const AArch64TargetMachine &>(MF.getTarget());
637
638 if (!EnableSVEGISel && (F.getReturnType()->isScalableTy() ||
639 llvm::any_of(Range: F.args(), P: [](const Argument &A) {
640 return A.getType()->isScalableTy();
641 })))
642 return true;
643 const auto &ST = MF.getSubtarget<AArch64Subtarget>();
644 if (!ST.hasNEON() || !ST.hasFPARMv8()) {
645 LLVM_DEBUG(dbgs() << "Falling back to SDAG because we don't support no-NEON\n");
646 return true;
647 }
648
649 SMEAttrs Attrs = MF.getInfo<AArch64FunctionInfo>()->getSMEFnAttrs();
650 if (Attrs.hasZAState() || Attrs.hasZT0State() ||
651 Attrs.hasStreamingInterfaceOrBody() ||
652 Attrs.hasStreamingCompatibleInterface())
653 return true;
654
655 auto OptLevel = MF.getTarget().getOptLevel();
656 bool IsGlobalISelPreferred =
657 getCGPassBuilderOption().EnableGlobalISelOption ==
658 cl::boolOrDefault::BOU_TRUE ||
659 static_cast<unsigned>(OptLevel) <= TM.getEnableGlobalISelAtO() ||
660 F.hasOptNone();
661 return !IsGlobalISelPreferred;
662}
663
664void AArch64CallLowering::saveVarArgRegisters(
665 MachineIRBuilder &MIRBuilder, CallLowering::IncomingValueHandler &Handler,
666 CCState &CCInfo) const {
667 auto GPRArgRegs = AArch64::getGPRArgRegs();
668 auto FPRArgRegs = AArch64::getFPRArgRegs();
669
670 MachineFunction &MF = MIRBuilder.getMF();
671 MachineRegisterInfo &MRI = MF.getRegInfo();
672 MachineFrameInfo &MFI = MF.getFrameInfo();
673 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
674 auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
675 bool IsWin64CC = Subtarget.isCallingConvWin64(CC: CCInfo.getCallingConv(),
676 IsVarArg: MF.getFunction().isVarArg());
677 const LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: 64);
678 const LLT s64 = LLT::integer(SizeInBits: 64);
679
680 unsigned FirstVariadicGPR = CCInfo.getFirstUnallocated(Regs: GPRArgRegs);
681 unsigned NumVariadicGPRArgRegs = GPRArgRegs.size() - FirstVariadicGPR + 1;
682
683 unsigned GPRSaveSize = 8 * (GPRArgRegs.size() - FirstVariadicGPR);
684 int GPRIdx = 0;
685 if (GPRSaveSize != 0) {
686 if (IsWin64CC) {
687 GPRIdx = MFI.CreateFixedObject(Size: GPRSaveSize,
688 SPOffset: -static_cast<int>(GPRSaveSize), IsImmutable: false);
689 if (GPRSaveSize & 15)
690 // The extra size here, if triggered, will always be 8.
691 MFI.CreateFixedObject(Size: 16 - (GPRSaveSize & 15),
692 SPOffset: -static_cast<int>(alignTo(Value: GPRSaveSize, Align: 16)),
693 IsImmutable: false);
694 } else
695 GPRIdx = MFI.CreateStackObject(Size: GPRSaveSize, Alignment: Align(8), isSpillSlot: false);
696
697 auto FIN = MIRBuilder.buildFrameIndex(Res: p0, Idx: GPRIdx);
698 auto Offset =
699 MIRBuilder.buildConstant(Res: MRI.createGenericVirtualRegister(Ty: s64), Val: 8);
700
701 for (unsigned i = FirstVariadicGPR; i < GPRArgRegs.size(); ++i) {
702 Register Val = MRI.createGenericVirtualRegister(Ty: s64);
703 Handler.assignValueToReg(
704 ValVReg: Val, PhysReg: GPRArgRegs[i],
705 VA: CCValAssign::getReg(ValNo: i + MF.getFunction().getNumOperands(), ValVT: MVT::i64,
706 Reg: GPRArgRegs[i], LocVT: MVT::i64, HTP: CCValAssign::Full));
707 auto MPO = IsWin64CC ? MachinePointerInfo::getFixedStack(
708 MF, FI: GPRIdx, Offset: (i - FirstVariadicGPR) * 8)
709 : MachinePointerInfo::getStack(MF, Offset: i * 8);
710 MIRBuilder.buildStore(Val, Addr: FIN, PtrInfo: MPO, Alignment: inferAlignFromPtrInfo(MF, MPO));
711
712 FIN = MIRBuilder.buildPtrAdd(Res: MRI.createGenericVirtualRegister(Ty: p0),
713 Op0: FIN.getReg(Idx: 0), Op1: Offset);
714 }
715 }
716 FuncInfo->setVarArgsGPRIndex(GPRIdx);
717 FuncInfo->setVarArgsGPRSize(GPRSaveSize);
718
719 if (Subtarget.hasFPARMv8() && !IsWin64CC) {
720 unsigned FirstVariadicFPR = CCInfo.getFirstUnallocated(Regs: FPRArgRegs);
721
722 unsigned FPRSaveSize = 16 * (FPRArgRegs.size() - FirstVariadicFPR);
723 int FPRIdx = 0;
724 if (FPRSaveSize != 0) {
725 FPRIdx = MFI.CreateStackObject(Size: FPRSaveSize, Alignment: Align(16), isSpillSlot: false);
726
727 auto FIN = MIRBuilder.buildFrameIndex(Res: p0, Idx: FPRIdx);
728 auto Offset =
729 MIRBuilder.buildConstant(Res: MRI.createGenericVirtualRegister(Ty: s64), Val: 16);
730
731 for (unsigned i = FirstVariadicFPR; i < FPRArgRegs.size(); ++i) {
732 Register Val = MRI.createGenericVirtualRegister(Ty: LLT::float128());
733 Handler.assignValueToReg(
734 ValVReg: Val, PhysReg: FPRArgRegs[i],
735 VA: CCValAssign::getReg(
736 ValNo: i + MF.getFunction().getNumOperands() + NumVariadicGPRArgRegs,
737 ValVT: MVT::f128, Reg: FPRArgRegs[i], LocVT: MVT::f128, HTP: CCValAssign::Full));
738
739 auto MPO = MachinePointerInfo::getStack(MF, Offset: i * 16);
740 MIRBuilder.buildStore(Val, Addr: FIN, PtrInfo: MPO, Alignment: inferAlignFromPtrInfo(MF, MPO));
741
742 FIN = MIRBuilder.buildPtrAdd(Res: MRI.createGenericVirtualRegister(Ty: p0),
743 Op0: FIN.getReg(Idx: 0), Op1: Offset);
744 }
745 }
746 FuncInfo->setVarArgsFPRIndex(FPRIdx);
747 FuncInfo->setVarArgsFPRSize(FPRSaveSize);
748 }
749}
750
751bool AArch64CallLowering::lowerFormalArguments(
752 MachineIRBuilder &MIRBuilder, const Function &F,
753 ArrayRef<ArrayRef<Register>> VRegs, FunctionLoweringInfo &FLI) const {
754 MachineFunction &MF = MIRBuilder.getMF();
755 MachineBasicBlock &MBB = MIRBuilder.getMBB();
756 MachineRegisterInfo &MRI = MF.getRegInfo();
757 auto &DL = F.getDataLayout();
758 auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
759
760 // Arm64EC has extra requirements for varargs calls which are only implemented
761 // in SelectionDAG; bail out for now.
762 if (F.isVarArg() && Subtarget.isWindowsArm64EC())
763 return false;
764
765 // Arm64EC thunks have a special calling convention which is only implemented
766 // in SelectionDAG; bail out for now.
767 if (F.getCallingConv() == CallingConv::ARM64EC_Thunk_Native ||
768 F.getCallingConv() == CallingConv::ARM64EC_Thunk_X64)
769 return false;
770
771 bool IsWin64 =
772 Subtarget.isCallingConvWin64(CC: F.getCallingConv(), IsVarArg: F.isVarArg()) &&
773 !Subtarget.isWindowsArm64EC();
774
775 SmallVector<ArgInfo, 8> SplitArgs;
776 SmallVector<std::pair<Register, Register>> BoolArgs;
777
778 // Insert the hidden sret parameter if the return value won't fit in the
779 // return registers.
780 if (!FLI.CanLowerReturn)
781 insertSRetIncomingArgument(F, SplitArgs, DemoteReg&: FLI.DemoteRegister, MRI, DL);
782
783 unsigned i = 0;
784 for (auto &Arg : F.args()) {
785 if (DL.getTypeStoreSize(Ty: Arg.getType()).isZero())
786 continue;
787
788 ArgInfo OrigArg{VRegs[i], Arg, i};
789 setArgFlags(Arg&: OrigArg, OpIdx: i + AttributeList::FirstArgIndex, DL, FuncInfo: F);
790
791 // i1 arguments are zero-extended to i8 by the caller. Emit a
792 // hint to reflect this.
793 if (OrigArg.Ty->isIntegerTy(BitWidth: 1)) {
794 assert(OrigArg.Regs.size() == 1 &&
795 MRI.getType(OrigArg.Regs[0]).getSizeInBits() == 1 &&
796 "Unexpected registers used for i1 arg");
797
798 auto &Flags = OrigArg.Flags[0];
799 if (!Flags.isZExt() && !Flags.isSExt()) {
800 // Lower i1 argument as i8, and insert AssertZExt + Trunc later.
801 Register OrigReg = OrigArg.Regs[0];
802 Register WideReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 8));
803 OrigArg.Regs[0] = WideReg;
804 BoolArgs.push_back(Elt: {OrigReg, WideReg});
805 }
806 }
807
808 if (Arg.hasAttribute(Kind: Attribute::SwiftAsync))
809 MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
810
811 splitToValueTypes(OrigArgInfo: OrigArg, SplitArgs, DL, CallConv: F.getCallingConv());
812 ++i;
813 }
814
815 if (!MBB.empty())
816 MIRBuilder.setInstr(*MBB.begin());
817
818 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
819 CCAssignFn *AssignFn = TLI.CCAssignFnForCall(CC: F.getCallingConv(), IsVarArg: IsWin64 && F.isVarArg());
820
821 AArch64IncomingValueAssigner Assigner(AssignFn, AssignFn);
822 FormalArgHandler Handler(MIRBuilder, MRI);
823 SmallVector<CCValAssign, 16> ArgLocs;
824 CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());
825 if (!determineAssignments(Assigner, Args&: SplitArgs, CCInfo) ||
826 !handleAssignments(Handler, Args&: SplitArgs, CCState&: CCInfo, ArgLocs, MIRBuilder))
827 return false;
828
829 if (!BoolArgs.empty()) {
830 for (auto &KV : BoolArgs) {
831 Register OrigReg = KV.first;
832 Register WideReg = KV.second;
833 LLT WideTy = MRI.getType(Reg: WideReg);
834 assert(MRI.getType(OrigReg).getScalarSizeInBits() == 1 &&
835 "Unexpected bit size of a bool arg");
836 MIRBuilder.buildTrunc(
837 Res: OrigReg, Op: MIRBuilder.buildAssertZExt(Res: WideTy, Op: WideReg, Size: 1).getReg(Idx: 0));
838 }
839 }
840
841 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
842 uint64_t StackSize = Assigner.StackSize;
843 if (F.isVarArg()) {
844 if ((!Subtarget.isTargetDarwin() && !Subtarget.isWindowsArm64EC()) || IsWin64) {
845 // The AAPCS variadic function ABI is identical to the non-variadic
846 // one. As a result there may be more arguments in registers and we should
847 // save them for future reference.
848 // Win64 variadic functions also pass arguments in registers, but all
849 // float arguments are passed in integer registers.
850 saveVarArgRegisters(MIRBuilder, Handler, CCInfo);
851 } else if (Subtarget.isWindowsArm64EC()) {
852 return false;
853 }
854
855 // We currently pass all varargs at 8-byte alignment, or 4 in ILP32.
856 StackSize = alignTo(Value: Assigner.StackSize, Align: Subtarget.isTargetILP32() ? 4 : 8);
857
858 auto &MFI = MIRBuilder.getMF().getFrameInfo();
859 FuncInfo->setVarArgsStackIndex(MFI.CreateFixedObject(Size: 4, SPOffset: StackSize, IsImmutable: true));
860 }
861
862 if (doesCalleeRestoreStack(CallConv: F.getCallingConv(),
863 TailCallOpt: MF.getTarget().Options.GuaranteedTailCallOpt)) {
864 // We have a non-standard ABI, so why not make full use of the stack that
865 // we're going to pop? It must be aligned to 16 B in any case.
866 StackSize = alignTo(Value: StackSize, Align: 16);
867
868 // If we're expected to restore the stack (e.g. fastcc), then we'll be
869 // adding a multiple of 16.
870 FuncInfo->setArgumentStackToRestore(StackSize);
871
872 // Our own callers will guarantee that the space is free by giving an
873 // aligned value to CALLSEQ_START.
874 }
875
876 // When we tail call, we need to check if the callee's arguments
877 // will fit on the caller's stack. So, whenever we lower formal arguments,
878 // we should keep track of this information, since we might lower a tail call
879 // in this function later.
880 FuncInfo->setBytesInStackArgArea(StackSize);
881
882 if (Subtarget.hasCustomCallingConv())
883 Subtarget.getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF);
884
885 handleMustTailForwardedRegisters(MIRBuilder, AssignFn);
886
887 // Move back to the end of the basic block.
888 MIRBuilder.setMBB(MBB);
889
890 return true;
891}
892
893/// Return true if the calling convention is one that we can guarantee TCO for.
894static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls) {
895 return (CC == CallingConv::Fast && GuaranteeTailCalls) ||
896 CC == CallingConv::Tail || CC == CallingConv::SwiftTail;
897}
898
899/// Return true if we might ever do TCO for calls with this calling convention.
900static bool mayTailCallThisCC(CallingConv::ID CC) {
901 switch (CC) {
902 case CallingConv::C:
903 case CallingConv::PreserveMost:
904 case CallingConv::PreserveAll:
905 case CallingConv::PreserveNone:
906 case CallingConv::Swift:
907 case CallingConv::SwiftTail:
908 case CallingConv::Tail:
909 case CallingConv::Fast:
910 return true;
911 default:
912 return false;
913 }
914}
915
916/// Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for
917/// CC.
918static std::pair<CCAssignFn *, CCAssignFn *>
919getAssignFnsForCC(CallingConv::ID CC, const AArch64TargetLowering &TLI) {
920 return {TLI.CCAssignFnForCall(CC, IsVarArg: false), TLI.CCAssignFnForCall(CC, IsVarArg: true)};
921}
922
923bool AArch64CallLowering::doCallerAndCalleePassArgsTheSameWay(
924 CallLoweringInfo &Info, MachineFunction &MF,
925 SmallVectorImpl<ArgInfo> &InArgs) const {
926 const Function &CallerF = MF.getFunction();
927 CallingConv::ID CalleeCC = Info.CallConv;
928 CallingConv::ID CallerCC = CallerF.getCallingConv();
929
930 // If the calling conventions match, then everything must be the same.
931 if (CalleeCC == CallerCC)
932 return true;
933
934 // Check if the caller and callee will handle arguments in the same way.
935 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
936 CCAssignFn *CalleeAssignFnFixed;
937 CCAssignFn *CalleeAssignFnVarArg;
938 std::tie(args&: CalleeAssignFnFixed, args&: CalleeAssignFnVarArg) =
939 getAssignFnsForCC(CC: CalleeCC, TLI);
940
941 CCAssignFn *CallerAssignFnFixed;
942 CCAssignFn *CallerAssignFnVarArg;
943 std::tie(args&: CallerAssignFnFixed, args&: CallerAssignFnVarArg) =
944 getAssignFnsForCC(CC: CallerCC, TLI);
945
946 AArch64IncomingValueAssigner CalleeAssigner(CalleeAssignFnFixed,
947 CalleeAssignFnVarArg);
948 AArch64IncomingValueAssigner CallerAssigner(CallerAssignFnFixed,
949 CallerAssignFnVarArg);
950
951 if (!resultsCompatible(Info, MF, InArgs, CalleeAssigner, CallerAssigner))
952 return false;
953
954 // Make sure that the caller and callee preserve all of the same registers.
955 auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
956 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
957 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
958 if (MF.getSubtarget<AArch64Subtarget>().hasCustomCallingConv()) {
959 TRI->UpdateCustomCallPreservedMask(MF, Mask: &CallerPreserved);
960 TRI->UpdateCustomCallPreservedMask(MF, Mask: &CalleePreserved);
961 }
962
963 return TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved);
964}
965
966bool AArch64CallLowering::areCalleeOutgoingArgsTailCallable(
967 CallLoweringInfo &Info, MachineFunction &MF,
968 SmallVectorImpl<ArgInfo> &OrigOutArgs) const {
969 // If there are no outgoing arguments, then we are done.
970 if (OrigOutArgs.empty())
971 return true;
972
973 const Function &CallerF = MF.getFunction();
974 LLVMContext &Ctx = CallerF.getContext();
975 CallingConv::ID CalleeCC = Info.CallConv;
976 CallingConv::ID CallerCC = CallerF.getCallingConv();
977 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
978 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
979
980 CCAssignFn *AssignFnFixed;
981 CCAssignFn *AssignFnVarArg;
982 std::tie(args&: AssignFnFixed, args&: AssignFnVarArg) = getAssignFnsForCC(CC: CalleeCC, TLI);
983
984 // We have outgoing arguments. Make sure that we can tail call with them.
985 SmallVector<CCValAssign, 16> OutLocs;
986 CCState OutInfo(CalleeCC, false, MF, OutLocs, Ctx);
987
988 AArch64OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg,
989 Subtarget, /*IsReturn*/ false);
990 // determineAssignments() may modify argument flags, so make a copy.
991 SmallVector<ArgInfo, 8> OutArgs;
992 append_range(C&: OutArgs, R&: OrigOutArgs);
993 if (!determineAssignments(Assigner&: CalleeAssigner, Args&: OutArgs, CCInfo&: OutInfo)) {
994 LLVM_DEBUG(dbgs() << "... Could not analyze call operands.\n");
995 return false;
996 }
997
998 // Make sure that they can fit on the caller's stack.
999 const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1000 if (OutInfo.getStackSize() > FuncInfo->getBytesInStackArgArea()) {
1001 LLVM_DEBUG(dbgs() << "... Cannot fit call operands on caller's stack.\n");
1002 return false;
1003 }
1004
1005 // Verify that the parameters in callee-saved registers match.
1006 // TODO: Port this over to CallLowering as general code once swiftself is
1007 // supported.
1008 auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
1009 const uint32_t *CallerPreservedMask = TRI->getCallPreservedMask(MF, CallerCC);
1010 MachineRegisterInfo &MRI = MF.getRegInfo();
1011
1012 if (Info.IsVarArg) {
1013 // Be conservative and disallow variadic memory operands to match SDAG's
1014 // behaviour.
1015 // FIXME: If the caller's calling convention is C, then we can
1016 // potentially use its argument area. However, for cases like fastcc,
1017 // we can't do anything.
1018 for (unsigned i = 0; i < OutLocs.size(); ++i) {
1019 auto &ArgLoc = OutLocs[i];
1020 if (ArgLoc.isRegLoc())
1021 continue;
1022
1023 LLVM_DEBUG(
1024 dbgs()
1025 << "... Cannot tail call vararg function with stack arguments\n");
1026 return false;
1027 }
1028 }
1029
1030 return parametersInCSRMatch(MRI, CallerPreservedMask, ArgLocs: OutLocs, OutVals: OutArgs);
1031}
1032
1033bool AArch64CallLowering::isEligibleForTailCallOptimization(
1034 MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
1035 SmallVectorImpl<ArgInfo> &InArgs,
1036 SmallVectorImpl<ArgInfo> &OutArgs) const {
1037
1038 // Must pass all target-independent checks in order to tail call optimize.
1039 if (!Info.IsTailCall)
1040 return false;
1041
1042 CallingConv::ID CalleeCC = Info.CallConv;
1043 MachineFunction &MF = MIRBuilder.getMF();
1044 const Function &CallerF = MF.getFunction();
1045
1046 LLVM_DEBUG(dbgs() << "Attempting to lower call as tail call\n");
1047
1048 if (Info.SwiftErrorVReg) {
1049 // TODO: We should handle this.
1050 // Note that this is also handled by the check for no outgoing arguments.
1051 // Proactively disabling this though, because the swifterror handling in
1052 // lowerCall inserts a COPY *after* the location of the call.
1053 LLVM_DEBUG(dbgs() << "... Cannot handle tail calls with swifterror yet.\n");
1054 return false;
1055 }
1056
1057 if (!mayTailCallThisCC(CC: CalleeCC)) {
1058 LLVM_DEBUG(dbgs() << "... Calling convention cannot be tail called.\n");
1059 return false;
1060 }
1061
1062 // Byval parameters hand the function a pointer directly into the stack area
1063 // we want to reuse during a tail call. Working around this *is* possible (see
1064 // X86).
1065 //
1066 // FIXME: In AArch64ISelLowering, this isn't worked around. Can/should we try
1067 // it?
1068 //
1069 // On Windows, "inreg" attributes signify non-aggregate indirect returns.
1070 // In this case, it is necessary to save/restore X0 in the callee. Tail
1071 // call opt interferes with this. So we disable tail call opt when the
1072 // caller has an argument with "inreg" attribute.
1073 //
1074 // FIXME: Check whether the callee also has an "inreg" argument.
1075 //
1076 // When the caller has a swifterror argument, we don't want to tail call
1077 // because would have to move into the swifterror register before the
1078 // tail call.
1079 if (any_of(Range: CallerF.args(), P: [](const Argument &A) {
1080 return A.hasByValAttr() || A.hasInRegAttr() || A.hasSwiftErrorAttr();
1081 })) {
1082 LLVM_DEBUG(dbgs() << "... Cannot tail call from callers with byval, "
1083 "inreg, or swifterror arguments\n");
1084 return false;
1085 }
1086
1087 // Externally-defined functions with weak linkage should not be
1088 // tail-called on AArch64 when the OS does not support dynamic
1089 // pre-emption of symbols, as the AAELF spec requires normal calls
1090 // to undefined weak functions to be replaced with a NOP or jump to the
1091 // next instruction. The behaviour of branch instructions in this
1092 // situation (as used for tail calls) is implementation-defined, so we
1093 // cannot rely on the linker replacing the tail call with a return.
1094 if (Info.Callee.isGlobal()) {
1095 const GlobalValue *GV = Info.Callee.getGlobal();
1096 const Triple &TT = MF.getTarget().getTargetTriple();
1097 if (GV->hasExternalWeakLinkage() &&
1098 (!TT.isOSWindows() || TT.isOSBinFormatELF() ||
1099 TT.isOSBinFormatMachO())) {
1100 LLVM_DEBUG(dbgs() << "... Cannot tail call externally-defined function "
1101 "with weak linkage for this OS.\n");
1102 return false;
1103 }
1104 }
1105
1106 // If we have -tailcallopt, then we're done.
1107 if (canGuaranteeTCO(CC: CalleeCC, GuaranteeTailCalls: MF.getTarget().Options.GuaranteedTailCallOpt))
1108 return CalleeCC == CallerF.getCallingConv();
1109
1110 // We don't have -tailcallopt, so we're allowed to change the ABI (sibcall).
1111 // Try to find cases where we can do that.
1112
1113 // I want anyone implementing a new calling convention to think long and hard
1114 // about this assert.
1115 assert((!Info.IsVarArg || CalleeCC == CallingConv::C) &&
1116 "Unexpected variadic calling convention");
1117
1118 // Verify that the incoming and outgoing arguments from the callee are
1119 // safe to tail call.
1120 if (!doCallerAndCalleePassArgsTheSameWay(Info, MF, InArgs)) {
1121 LLVM_DEBUG(
1122 dbgs()
1123 << "... Caller and callee have incompatible calling conventions.\n");
1124 return false;
1125 }
1126
1127 if (!areCalleeOutgoingArgsTailCallable(Info, MF, OrigOutArgs&: OutArgs))
1128 return false;
1129
1130 LLVM_DEBUG(
1131 dbgs() << "... Call is eligible for tail call optimization.\n");
1132 return true;
1133}
1134
1135static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect,
1136 bool IsTailCall,
1137 std::optional<CallLowering::PtrAuthInfo> &PAI,
1138 MachineRegisterInfo &MRI) {
1139 const AArch64FunctionInfo *FuncInfo = CallerF.getInfo<AArch64FunctionInfo>();
1140
1141 if (!IsTailCall) {
1142 if (!PAI)
1143 return IsIndirect ? getBLRCallOpcode(MF: CallerF) : (unsigned)AArch64::BL;
1144
1145 assert(IsIndirect && "Direct call should not be authenticated");
1146 assert((PAI->Key == AArch64PACKey::IA || PAI->Key == AArch64PACKey::IB) &&
1147 "Invalid auth call key");
1148 return AArch64::BLRA;
1149 }
1150
1151 if (!IsIndirect)
1152 return AArch64::TCRETURNdi;
1153
1154 // When BTI or PAuthLR are enabled, there are restrictions on using x16 and
1155 // x17 to hold the function pointer.
1156 if (FuncInfo->branchTargetEnforcement()) {
1157 if (FuncInfo->branchProtectionPAuthLR()) {
1158 assert(!PAI && "ptrauth tail-calls not yet supported with PAuthLR");
1159 return AArch64::TCRETURNrix17;
1160 }
1161 if (PAI)
1162 return AArch64::AUTH_TCRETURN_BTI;
1163 return AArch64::TCRETURNrix16x17;
1164 }
1165
1166 if (FuncInfo->branchProtectionPAuthLR()) {
1167 assert(!PAI && "ptrauth tail-calls not yet supported with PAuthLR");
1168 return AArch64::TCRETURNrinotx16;
1169 }
1170
1171 if (PAI)
1172 return AArch64::AUTH_TCRETURN;
1173 return AArch64::TCRETURNri;
1174}
1175
1176static const uint32_t *
1177getMaskForArgs(SmallVectorImpl<AArch64CallLowering::ArgInfo> &OutArgs,
1178 AArch64CallLowering::CallLoweringInfo &Info,
1179 const AArch64RegisterInfo &TRI, MachineFunction &MF) {
1180 const uint32_t *Mask;
1181 if (!OutArgs.empty() && OutArgs[0].Flags[0].isReturned()) {
1182 // For 'this' returns, use the X0-preserving mask if applicable
1183 Mask = TRI.getThisReturnPreservedMask(MF, Info.CallConv);
1184 if (!Mask) {
1185 OutArgs[0].Flags[0].setReturned(false);
1186 Mask = TRI.getCallPreservedMask(MF, Info.CallConv);
1187 }
1188 } else {
1189 Mask = TRI.getCallPreservedMask(MF, Info.CallConv);
1190 }
1191 return Mask;
1192}
1193
1194bool AArch64CallLowering::lowerTailCall(
1195 MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
1196 SmallVectorImpl<ArgInfo> &OutArgs) const {
1197 MachineFunction &MF = MIRBuilder.getMF();
1198 const Function &F = MF.getFunction();
1199 MachineRegisterInfo &MRI = MF.getRegInfo();
1200 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
1201 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1202
1203 // True when we're tail calling, but without -tailcallopt.
1204 bool IsSibCall = !MF.getTarget().Options.GuaranteedTailCallOpt &&
1205 Info.CallConv != CallingConv::Tail &&
1206 Info.CallConv != CallingConv::SwiftTail;
1207
1208 // Find out which ABI gets to decide where things go.
1209 CallingConv::ID CalleeCC = Info.CallConv;
1210 CCAssignFn *AssignFnFixed;
1211 CCAssignFn *AssignFnVarArg;
1212 std::tie(args&: AssignFnFixed, args&: AssignFnVarArg) = getAssignFnsForCC(CC: CalleeCC, TLI);
1213
1214 MachineInstrBuilder CallSeqStart;
1215 if (!IsSibCall)
1216 CallSeqStart = MIRBuilder.buildInstr(Opcode: AArch64::ADJCALLSTACKDOWN);
1217
1218 unsigned Opc = getCallOpcode(CallerF: MF, IsIndirect: Info.Callee.isReg(), IsTailCall: true, PAI&: Info.PAI, MRI);
1219 auto MIB = MIRBuilder.buildInstrNoInsert(Opcode: Opc);
1220 MIB.add(MO: Info.Callee);
1221
1222 // Tell the call which registers are clobbered.
1223 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1224 auto TRI = Subtarget.getRegisterInfo();
1225
1226 // Byte offset for the tail call. When we are sibcalling, this will always
1227 // be 0.
1228 MIB.addImm(Val: 0);
1229
1230 // Authenticated tail calls always take key/discriminator arguments.
1231 if (Opc == AArch64::AUTH_TCRETURN || Opc == AArch64::AUTH_TCRETURN_BTI) {
1232 assert((Info.PAI->Key == AArch64PACKey::IA ||
1233 Info.PAI->Key == AArch64PACKey::IB) &&
1234 "Invalid auth call key");
1235 MIB.addImm(Val: Info.PAI->Key);
1236
1237 Register AddrDisc = 0;
1238 uint16_t IntDisc = 0;
1239 std::tie(args&: IntDisc, args&: AddrDisc) =
1240 extractPtrauthBlendDiscriminators(Disc: Info.PAI->Discriminator, MRI);
1241
1242 MIB.addImm(Val: IntDisc);
1243 MIB.addUse(RegNo: AddrDisc);
1244 if (AddrDisc.isValid()) {
1245 MIB->getOperand(i: 4).setReg(constrainOperandRegClass(
1246 MF, TRI: *TRI, MRI, TII: *MF.getSubtarget().getInstrInfo(),
1247 RBI: *MF.getSubtarget().getRegBankInfo(), InsertPt&: *MIB, II: MIB->getDesc(),
1248 RegMO&: MIB->getOperand(i: 4), OpIdx: 4));
1249 }
1250 }
1251
1252 // Tell the call which registers are clobbered.
1253 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CalleeCC);
1254 if (Subtarget.hasCustomCallingConv())
1255 TRI->UpdateCustomCallPreservedMask(MF, Mask: &Mask);
1256 MIB.addRegMask(Mask);
1257
1258 if (Info.CFIType)
1259 MIB->setCFIType(MF, Type: Info.CFIType->getZExtValue());
1260
1261 if (TRI->isAnyArgRegReserved(MF))
1262 TRI->emitReservedArgRegCallError(MF);
1263
1264 // FPDiff is the byte offset of the call's argument area from the callee's.
1265 // Stores to callee stack arguments will be placed in FixedStackSlots offset
1266 // by this amount for a tail call. In a sibling call it must be 0 because the
1267 // caller will deallocate the entire stack and the callee still expects its
1268 // arguments to begin at SP+0.
1269 int FPDiff = 0;
1270
1271 // This will be 0 for sibcalls, potentially nonzero for tail calls produced
1272 // by -tailcallopt. For sibcalls, the memory operands for the call are
1273 // already available in the caller's incoming argument space.
1274 unsigned NumBytes = 0;
1275 if (!IsSibCall) {
1276 // We aren't sibcalling, so we need to compute FPDiff. We need to do this
1277 // before handling assignments, because FPDiff must be known for memory
1278 // arguments.
1279 unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
1280 SmallVector<CCValAssign, 16> OutLocs;
1281 CCState OutInfo(CalleeCC, false, MF, OutLocs, F.getContext());
1282
1283 AArch64OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg,
1284 Subtarget, /*IsReturn*/ false);
1285 if (!determineAssignments(Assigner&: CalleeAssigner, Args&: OutArgs, CCInfo&: OutInfo))
1286 return false;
1287
1288 // The callee will pop the argument stack as a tail call. Thus, we must
1289 // keep it 16-byte aligned.
1290 NumBytes = alignTo(Value: OutInfo.getStackSize(), Align: 16);
1291
1292 // FPDiff will be negative if this tail call requires more space than we
1293 // would automatically have in our incoming argument space. Positive if we
1294 // actually shrink the stack.
1295 FPDiff = NumReusableBytes - NumBytes;
1296
1297 // Update the required reserved area if this is the tail call requiring the
1298 // most argument stack space.
1299 if (FPDiff < 0 && FuncInfo->getTailCallReservedStack() < (unsigned)-FPDiff)
1300 FuncInfo->setTailCallReservedStack(-FPDiff);
1301
1302 // The stack pointer must be 16-byte aligned at all times it's used for a
1303 // memory operation, which in practice means at *all* times and in
1304 // particular across call boundaries. Therefore our own arguments started at
1305 // a 16-byte aligned SP and the delta applied for the tail call should
1306 // satisfy the same constraint.
1307 assert(FPDiff % 16 == 0 && "unaligned stack on tail call");
1308 }
1309
1310 const auto &Forwards = FuncInfo->getForwardedMustTailRegParms();
1311
1312 AArch64OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg,
1313 Subtarget, /*IsReturn*/ false);
1314
1315 // Do the actual argument marshalling.
1316 OutgoingArgHandler Handler(MIRBuilder, MRI, MIB,
1317 /*IsTailCall*/ true, FPDiff);
1318 if (!determineAndHandleAssignments(Handler, Assigner, Args&: OutArgs, MIRBuilder,
1319 CallConv: CalleeCC, IsVarArg: Info.IsVarArg))
1320 return false;
1321
1322 Mask = getMaskForArgs(OutArgs, Info, TRI: *TRI, MF);
1323
1324 if (Info.IsVarArg && Info.IsMustTailCall) {
1325 // Now we know what's being passed to the function. Add uses to the call for
1326 // the forwarded registers that we *aren't* passing as parameters. This will
1327 // preserve the copies we build earlier.
1328 for (const auto &F : Forwards) {
1329 Register ForwardedReg = F.PReg;
1330 // If the register is already passed, or aliases a register which is
1331 // already being passed, then skip it.
1332 if (any_of(Range: MIB->uses(), P: [&ForwardedReg, &TRI](const MachineOperand &Use) {
1333 if (!Use.isReg())
1334 return false;
1335 return TRI->regsOverlap(RegA: Use.getReg(), RegB: ForwardedReg);
1336 }))
1337 continue;
1338
1339 // We aren't passing it already, so we should add it to the call.
1340 MIRBuilder.buildCopy(Res: ForwardedReg, Op: Register(F.VReg));
1341 MIB.addReg(RegNo: ForwardedReg, Flags: RegState::Implicit);
1342 }
1343 }
1344
1345 // If we have -tailcallopt, we need to adjust the stack. We'll do the call
1346 // sequence start and end here.
1347 if (!IsSibCall) {
1348 MIB->getOperand(i: 1).setImm(FPDiff);
1349 CallSeqStart.addImm(Val: 0).addImm(Val: 0);
1350 // End the call sequence *before* emitting the call. Normally, we would
1351 // tidy the frame up after the call. However, here, we've laid out the
1352 // parameters so that when SP is reset, they will be in the correct
1353 // location.
1354 MIRBuilder.buildInstr(Opcode: AArch64::ADJCALLSTACKUP).addImm(Val: 0).addImm(Val: 0);
1355 }
1356
1357 // Now we can add the actual call instruction to the correct basic block.
1358 MIRBuilder.insertInstr(MIB);
1359
1360 // If Callee is a reg, since it is used by a target specific instruction,
1361 // it must have a register class matching the constraint of that instruction.
1362 if (MIB->getOperand(i: 0).isReg())
1363 constrainOperandRegClass(MF, TRI: *TRI, MRI, TII: *MF.getSubtarget().getInstrInfo(),
1364 RBI: *MF.getSubtarget().getRegBankInfo(), InsertPt&: *MIB,
1365 II: MIB->getDesc(), RegMO&: MIB->getOperand(i: 0), OpIdx: 0);
1366
1367 MF.getFrameInfo().setHasTailCall();
1368 Info.LoweredTailCall = true;
1369 return true;
1370}
1371
1372bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
1373 CallLoweringInfo &Info) const {
1374 MachineFunction &MF = MIRBuilder.getMF();
1375 const Function &F = MF.getFunction();
1376 MachineRegisterInfo &MRI = MF.getRegInfo();
1377 auto &DL = F.getDataLayout();
1378 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
1379 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1380
1381 // Arm64EC has extra requirements for varargs calls; bail out for now.
1382 //
1383 // Arm64EC has special mangling rules for calls; bail out on all calls for
1384 // now.
1385 if (Subtarget.isWindowsArm64EC())
1386 return false;
1387
1388 // Arm64EC thunks have a special calling convention which is only implemented
1389 // in SelectionDAG; bail out for now.
1390 if (Info.CallConv == CallingConv::ARM64EC_Thunk_Native ||
1391 Info.CallConv == CallingConv::ARM64EC_Thunk_X64)
1392 return false;
1393
1394 SmallVector<ArgInfo, 8> OutArgs;
1395 for (auto &OrigArg : Info.OrigArgs) {
1396 splitToValueTypes(OrigArgInfo: OrigArg, SplitArgs&: OutArgs, DL, CallConv: Info.CallConv);
1397 // AAPCS requires that we zero-extend i1 to 8 bits by the caller.
1398 auto &Flags = OrigArg.Flags[0];
1399 if (OrigArg.Ty->isIntegerTy(BitWidth: 1) && !Flags.isSExt() && !Flags.isZExt()) {
1400 ArgInfo &OutArg = OutArgs.back();
1401 assert(OutArg.Regs.size() == 1 &&
1402 MRI.getType(OutArg.Regs[0]).getSizeInBits() == 1 &&
1403 "Unexpected registers used for i1 arg");
1404
1405 // We cannot use a ZExt ArgInfo flag here, because it will
1406 // zero-extend the argument to i32 instead of just i8.
1407 OutArg.Regs[0] =
1408 MIRBuilder.buildZExt(Res: LLT::integer(SizeInBits: 8), Op: OutArg.Regs[0]).getReg(Idx: 0);
1409 LLVMContext &Ctx = MF.getFunction().getContext();
1410 OutArg.Ty = Type::getInt8Ty(C&: Ctx);
1411 }
1412 }
1413
1414 SmallVector<ArgInfo, 8> InArgs;
1415 if (!Info.OrigRet.Ty->isVoidTy())
1416 splitToValueTypes(OrigArgInfo: Info.OrigRet, SplitArgs&: InArgs, DL, CallConv: Info.CallConv);
1417
1418 // If we can lower as a tail call, do that instead.
1419 bool CanTailCallOpt =
1420 isEligibleForTailCallOptimization(MIRBuilder, Info, InArgs, OutArgs);
1421
1422 // We must emit a tail call if we have musttail.
1423 if (Info.IsMustTailCall && !CanTailCallOpt) {
1424 // There are types of incoming/outgoing arguments we can't handle yet, so
1425 // it doesn't make sense to actually die here like in ISelLowering. Instead,
1426 // fall back to SelectionDAG and let it try to handle this.
1427 LLVM_DEBUG(dbgs() << "Failed to lower musttail call as tail call\n");
1428 return false;
1429 }
1430
1431 Info.IsTailCall = CanTailCallOpt;
1432 if (CanTailCallOpt)
1433 return lowerTailCall(MIRBuilder, Info, OutArgs);
1434
1435 // Find out which ABI gets to decide where things go.
1436 CCAssignFn *AssignFnFixed;
1437 CCAssignFn *AssignFnVarArg;
1438 std::tie(args&: AssignFnFixed, args&: AssignFnVarArg) =
1439 getAssignFnsForCC(CC: Info.CallConv, TLI);
1440
1441 MachineInstrBuilder CallSeqStart;
1442 CallSeqStart = MIRBuilder.buildInstr(Opcode: AArch64::ADJCALLSTACKDOWN);
1443
1444 // Create a temporarily-floating call instruction so we can add the implicit
1445 // uses of arg registers.
1446
1447 unsigned Opc = 0;
1448 // Calls with operand bundle "clang.arc.attachedcall" are special. They should
1449 // be expanded to the call, directly followed by a special marker sequence and
1450 // a call to an ObjC library function.
1451 if (Info.CB && objcarc::hasAttachedCallOpBundle(CB: Info.CB))
1452 Opc = Info.PAI ? AArch64::BLRA_RVMARKER : AArch64::BLR_RVMARKER;
1453 // A call to a returns twice function like setjmp must be followed by a bti
1454 // instruction.
1455 else if (Info.CB && Info.CB->hasFnAttr(Kind: Attribute::ReturnsTwice) &&
1456 !Subtarget.noBTIAtReturnTwice() &&
1457 MF.getInfo<AArch64FunctionInfo>()->branchTargetEnforcement())
1458 Opc = AArch64::BLR_BTI;
1459 else {
1460 // For an intrinsic call (e.g. memset), use GOT if "RtLibUseGOT" (-fno-plt)
1461 // is set.
1462 if (Info.Callee.isSymbol() && F.getParent()->getRtLibUseGOT()) {
1463 auto MIB = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_GLOBAL_VALUE);
1464 DstOp(getLLTForType(Ty&: *F.getType(), DL)).addDefToMIB(MRI, MIB);
1465 MIB.addExternalSymbol(FnName: Info.Callee.getSymbolName(), TargetFlags: AArch64II::MO_GOT);
1466 Info.Callee = MachineOperand::CreateReg(Reg: MIB.getReg(Idx: 0), isDef: false);
1467 }
1468 Opc = getCallOpcode(CallerF: MF, IsIndirect: Info.Callee.isReg(), IsTailCall: false, PAI&: Info.PAI, MRI);
1469 }
1470
1471 auto MIB = MIRBuilder.buildInstrNoInsert(Opcode: Opc);
1472 unsigned CalleeOpNo = 0;
1473
1474 if (Opc == AArch64::BLR_RVMARKER || Opc == AArch64::BLRA_RVMARKER) {
1475 // Add a target global address for the retainRV/claimRV runtime function
1476 // just before the call target.
1477 Function *ARCFn = *objcarc::getAttachedARCFunction(CB: Info.CB);
1478 MIB.addGlobalAddress(GV: ARCFn);
1479 ++CalleeOpNo;
1480
1481 // We may or may not need to emit both the marker and the retain/claim call.
1482 // Tell the pseudo expansion using an additional boolean op.
1483 MIB.addImm(Val: objcarc::attachedCallOpBundleNeedsMarker(CB: Info.CB));
1484 ++CalleeOpNo;
1485 } else if (Info.CFIType) {
1486 MIB->setCFIType(MF, Type: Info.CFIType->getZExtValue());
1487 }
1488 MIB->setDeactivationSymbol(MF, DS: Info.DeactivationSymbol);
1489
1490 MIB.add(MO: Info.Callee);
1491
1492 // Tell the call which registers are clobbered.
1493 const uint32_t *Mask;
1494 const auto *TRI = Subtarget.getRegisterInfo();
1495
1496 AArch64OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg,
1497 Subtarget, /*IsReturn*/ false);
1498 // Do the actual argument marshalling.
1499 OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, /*IsReturn*/ false);
1500 bool AssignedCallArgs = Info.CallConv == CallingConv::C &&
1501 tryAssignSimpleGPRCallArgs(MIRBuilder, MIB, Args: OutArgs);
1502 if (!AssignedCallArgs &&
1503 !determineAndHandleAssignments(Handler, Assigner, Args&: OutArgs, MIRBuilder,
1504 CallConv: Info.CallConv, IsVarArg: Info.IsVarArg))
1505 return false;
1506
1507 Mask = getMaskForArgs(OutArgs, Info, TRI: *TRI, MF);
1508
1509 if (Opc == AArch64::BLRA || Opc == AArch64::BLRA_RVMARKER) {
1510 assert((Info.PAI->Key == AArch64PACKey::IA ||
1511 Info.PAI->Key == AArch64PACKey::IB) &&
1512 "Invalid auth call key");
1513 MIB.addImm(Val: Info.PAI->Key);
1514
1515 Register AddrDisc = 0;
1516 uint16_t IntDisc = 0;
1517 std::tie(args&: IntDisc, args&: AddrDisc) =
1518 extractPtrauthBlendDiscriminators(Disc: Info.PAI->Discriminator, MRI);
1519
1520 MIB.addImm(Val: IntDisc);
1521 MIB.addUse(RegNo: AddrDisc);
1522 if (AddrDisc.isValid()) {
1523 constrainOperandRegClass(MF, TRI: *TRI, MRI, TII: *MF.getSubtarget().getInstrInfo(),
1524 RBI: *MF.getSubtarget().getRegBankInfo(), InsertPt&: *MIB,
1525 II: MIB->getDesc(), RegMO&: MIB->getOperand(i: CalleeOpNo + 3),
1526 OpIdx: CalleeOpNo + 3);
1527 }
1528 }
1529
1530 // Tell the call which registers are clobbered.
1531 if (MF.getSubtarget<AArch64Subtarget>().hasCustomCallingConv())
1532 TRI->UpdateCustomCallPreservedMask(MF, Mask: &Mask);
1533 MIB.addRegMask(Mask);
1534
1535 if (TRI->isAnyArgRegReserved(MF))
1536 TRI->emitReservedArgRegCallError(MF);
1537
1538 // Now we can add the actual call instruction to the correct basic block.
1539 MIRBuilder.insertInstr(MIB);
1540
1541 uint64_t CalleePopBytes =
1542 doesCalleeRestoreStack(CallConv: Info.CallConv,
1543 TailCallOpt: MF.getTarget().Options.GuaranteedTailCallOpt)
1544 ? alignTo(Value: Assigner.StackSize, Align: 16)
1545 : 0;
1546
1547 CallSeqStart.addImm(Val: Assigner.StackSize).addImm(Val: 0);
1548 MIRBuilder.buildInstr(Opcode: AArch64::ADJCALLSTACKUP)
1549 .addImm(Val: Assigner.StackSize)
1550 .addImm(Val: CalleePopBytes);
1551
1552 // If Callee is a reg, since it is used by a target specific
1553 // instruction, it must have a register class matching the
1554 // constraint of that instruction.
1555 if (MIB->getOperand(i: CalleeOpNo).isReg())
1556 constrainOperandRegClass(MF, TRI: *TRI, MRI, TII: *Subtarget.getInstrInfo(),
1557 RBI: *Subtarget.getRegBankInfo(), InsertPt&: *MIB, II: MIB->getDesc(),
1558 RegMO&: MIB->getOperand(i: CalleeOpNo), OpIdx: CalleeOpNo);
1559
1560 // Finally we can copy the returned value back into its virtual-register. In
1561 // symmetry with the arguments, the physical register must be an
1562 // implicit-define of the call instruction.
1563 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy()) {
1564 CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(CC: Info.CallConv);
1565 CallReturnHandler Handler(MIRBuilder, MRI, MIB);
1566 bool UsingReturnedArg =
1567 !OutArgs.empty() && OutArgs[0].Flags[0].isReturned();
1568
1569 AArch64OutgoingValueAssigner Assigner(RetAssignFn, RetAssignFn, Subtarget,
1570 /*IsReturn*/ false);
1571 ReturnedArgCallReturnHandler ReturnedArgHandler(MIRBuilder, MRI, MIB);
1572 bool AssignedCallReturn =
1573 Info.CallConv == CallingConv::C && !UsingReturnedArg &&
1574 tryAssignSimpleGPRCallReturn(MIRBuilder, MIB, Rets: InArgs);
1575 if (!AssignedCallReturn &&
1576 !determineAndHandleAssignments(
1577 Handler&: UsingReturnedArg ? ReturnedArgHandler : Handler, Assigner, Args&: InArgs,
1578 MIRBuilder, CallConv: Info.CallConv, IsVarArg: Info.IsVarArg,
1579 ThisReturnRegs: UsingReturnedArg ? ArrayRef(OutArgs[0].Regs)
1580 : ArrayRef<Register>()))
1581 return false;
1582 }
1583
1584 if (Info.SwiftErrorVReg) {
1585 MIB.addDef(RegNo: AArch64::X21, Flags: RegState::Implicit);
1586 MIRBuilder.buildCopy(Res: Info.SwiftErrorVReg, Op: Register(AArch64::X21));
1587 }
1588
1589 if (!Info.CanLowerReturn) {
1590 insertSRetLoads(MIRBuilder, RetTy: Info.OrigRet.Ty, VRegs: Info.OrigRet.Regs,
1591 DemoteReg: Info.DemoteRegister, FI: Info.DemoteStackIndex);
1592 }
1593 return true;
1594}
1595
1596bool AArch64CallLowering::isTypeIsValidForThisReturn(EVT Ty) const {
1597 return Ty.getSizeInBits() == 64;
1598}
1599