1//===--- AArch64CallLowering.cpp - Call lowering --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the lowering of LLVM calls to machine code calls for
11/// GlobalISel.
12///
13//===----------------------------------------------------------------------===//
14
15#include "AArch64CallLowering.h"
16#include "AArch64GlobalISelUtils.h"
17#include "AArch64ISelLowering.h"
18#include "AArch64MachineFunctionInfo.h"
19#include "AArch64RegisterInfo.h"
20#include "AArch64SMEAttributes.h"
21#include "AArch64Subtarget.h"
22#include "AArch64TargetMachine.h"
23#include "Utils/AArch64BaseInfo.h"
24#include "llvm/ADT/ArrayRef.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/Analysis/ObjCARCUtil.h"
27#include "llvm/CodeGen/Analysis.h"
28#include "llvm/CodeGen/CallingConvLower.h"
29#include "llvm/CodeGen/FunctionLoweringInfo.h"
30#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
31#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
32#include "llvm/CodeGen/GlobalISel/Utils.h"
33#include "llvm/CodeGen/LowLevelTypeUtils.h"
34#include "llvm/CodeGen/MachineBasicBlock.h"
35#include "llvm/CodeGen/MachineFrameInfo.h"
36#include "llvm/CodeGen/MachineFunction.h"
37#include "llvm/CodeGen/MachineInstrBuilder.h"
38#include "llvm/CodeGen/MachineMemOperand.h"
39#include "llvm/CodeGen/MachineOperand.h"
40#include "llvm/CodeGen/MachineRegisterInfo.h"
41#include "llvm/CodeGen/TargetOpcodes.h"
42#include "llvm/CodeGen/TargetRegisterInfo.h"
43#include "llvm/CodeGen/TargetSubtargetInfo.h"
44#include "llvm/CodeGen/ValueTypes.h"
45#include "llvm/CodeGenTypes/MachineValueType.h"
46#include "llvm/IR/Argument.h"
47#include "llvm/IR/Attributes.h"
48#include "llvm/IR/Function.h"
49#include "llvm/IR/Type.h"
50#include "llvm/IR/Value.h"
51#include <algorithm>
52#include <cassert>
53#include <cstdint>
54
55#define DEBUG_TYPE "aarch64-call-lowering"
56
57using namespace llvm;
58using namespace AArch64GISelUtils;
59
60static bool isSimpleGPRCallValue(const CallLowering::ArgInfo &Arg) {
61 if (Arg.Regs.size() != 1 || any_of(Range: Arg.Flags, P: [](ISD::ArgFlagsTy Flags) {
62 auto FlagVals = Flags.getFlags();
63 return FlagVals != ISD::ArgFlagsTy::NoFlags &&
64 FlagVals != ISD::ArgFlagsTy::Pointer;
65 }))
66 return false;
67
68 Type *Ty = Arg.Ty;
69 return Ty->isPointerTy() || Ty->isIntegerTy(BitWidth: 32) || Ty->isIntegerTy(BitWidth: 64);
70}
71
72// Avoid the generic assignment machinery when every argument maps directly to
73// w0-w7/x0-x7. Fast path for compile-time.
74static bool tryAssignSimpleGPRCallArgs(MachineIRBuilder &MIRBuilder,
75 MachineInstrBuilder MIB,
76 ArrayRef<CallLowering::ArgInfo> Args) {
77 if (Args.size() > 8)
78 return false;
79
80 for (const CallLowering::ArgInfo &Arg : Args)
81 if (!isSimpleGPRCallValue(Arg))
82 return false;
83
84 for (unsigned I = 0, E = Args.size(); I != E; ++I) {
85 const CallLowering::ArgInfo &Arg = Args[I];
86 MCRegister XReg = AArch64::getGPRArgRegs()[I];
87 Register PhysReg = Arg.Ty->isIntegerTy(BitWidth: 32) ? getWRegFromXReg(Reg: XReg) : XReg;
88 MIB.addUse(RegNo: PhysReg, Flags: RegState::Implicit);
89 MIRBuilder.buildCopy(Res: PhysReg, Op: Arg.Regs[0]);
90 }
91 return true;
92}
93
94// Avoid the generic assignment machinery when the return value maps directly
95// to w0/x0. Fast path for compile-time.
96static bool tryAssignSimpleGPRCallReturn(MachineIRBuilder &MIRBuilder,
97 MachineInstrBuilder MIB,
98 ArrayRef<CallLowering::ArgInfo> Rets) {
99 if (Rets.size() != 1)
100 return false;
101
102 const CallLowering::ArgInfo &Ret = Rets[0];
103 if (!isSimpleGPRCallValue(Arg: Ret))
104 return false;
105
106 Register PhysReg = Ret.Ty->isIntegerTy(BitWidth: 32) ? AArch64::W0 : AArch64::X0;
107 MIB.addDef(RegNo: PhysReg, Flags: RegState::Implicit);
108 MIRBuilder.buildCopy(Res: Ret.Regs[0], Op: PhysReg);
109 return true;
110}
111
112AArch64CallLowering::AArch64CallLowering(const AArch64TargetLowering &TLI)
113 : CallLowering(&TLI) {}
114
115static void applyStackPassedSmallTypeDAGHack(EVT OrigVT, MVT &ValVT,
116 MVT &LocVT) {
117 // If ValVT is i1/i8/i16, we should set LocVT to i8/i8/i16. This is a legacy
118 // hack because the DAG calls the assignment function with pre-legalized
119 // register typed values, not the raw type.
120 //
121 // This hack is not applied to return values which are not passed on the
122 // stack.
123 if (OrigVT == MVT::i1 || OrigVT == MVT::i8)
124 ValVT = LocVT = MVT::i8;
125 else if (OrigVT == MVT::i16)
126 ValVT = LocVT = MVT::i16;
127}
128
129// Account for i1/i8/i16 stack passed value hack
130static LLT getStackValueStoreTypeHack(const CCValAssign &VA) {
131 const MVT ValVT = VA.getValVT();
132 return (ValVT == MVT::i8 || ValVT == MVT::i16) ? LLT(ValVT)
133 : LLT(VA.getLocVT());
134}
135
136namespace {
137
138struct AArch64IncomingValueAssigner
139 : public CallLowering::IncomingValueAssigner {
140 AArch64IncomingValueAssigner(CCAssignFn *AssignFn_,
141 CCAssignFn *AssignFnVarArg_)
142 : IncomingValueAssigner(AssignFn_, AssignFnVarArg_) {}
143
144 bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
145 CCValAssign::LocInfo LocInfo,
146 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags,
147 CCState &State) override {
148 applyStackPassedSmallTypeDAGHack(OrigVT, ValVT, LocVT);
149 return IncomingValueAssigner::assignArg(ValNo, OrigVT, ValVT, LocVT,
150 LocInfo, Info, Flags, State);
151 }
152};
153
154struct AArch64OutgoingValueAssigner
155 : public CallLowering::OutgoingValueAssigner {
156 const AArch64Subtarget &Subtarget;
157
158 /// Track if this is used for a return instead of function argument
159 /// passing. We apply a hack to i1/i8/i16 stack passed values, but do not use
160 /// stack passed returns for them and cannot apply the type adjustment.
161 bool IsReturn;
162
163 AArch64OutgoingValueAssigner(CCAssignFn *AssignFn_,
164 CCAssignFn *AssignFnVarArg_,
165 const AArch64Subtarget &Subtarget_,
166 bool IsReturn)
167 : OutgoingValueAssigner(AssignFn_, AssignFnVarArg_),
168 Subtarget(Subtarget_), IsReturn(IsReturn) {}
169
170 bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
171 CCValAssign::LocInfo LocInfo,
172 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags,
173 CCState &State) override {
174 const Function &F = State.getMachineFunction().getFunction();
175 bool IsCalleeWin =
176 Subtarget.isCallingConvWin64(CC: State.getCallingConv(), IsVarArg: F.isVarArg());
177 bool UseVarArgsCCForFixed = IsCalleeWin && State.isVarArg();
178
179 bool Res;
180 if (!Flags.isVarArg() && !UseVarArgsCCForFixed) {
181 if (!IsReturn)
182 applyStackPassedSmallTypeDAGHack(OrigVT, ValVT, LocVT);
183 Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, Info.Ty, State);
184 } else
185 Res = AssignFnVarArg(ValNo, ValVT, LocVT, LocInfo, Flags, Info.Ty, State);
186
187 StackSize = State.getStackSize();
188 return Res;
189 }
190};
191
192struct IncomingArgHandler : public CallLowering::IncomingValueHandler {
193 IncomingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
194 : IncomingValueHandler(MIRBuilder, MRI) {}
195
196 Register getStackAddress(uint64_t Size, int64_t Offset,
197 MachinePointerInfo &MPO,
198 ISD::ArgFlagsTy Flags) override {
199 auto &MFI = MIRBuilder.getMF().getFrameInfo();
200
201 // Byval is assumed to be writable memory, but other stack passed arguments
202 // are not.
203 const bool IsImmutable = !Flags.isByVal();
204
205 int FI = MFI.CreateFixedObject(Size, SPOffset: Offset, IsImmutable);
206 MPO = MachinePointerInfo::getFixedStack(MF&: MIRBuilder.getMF(), FI);
207 auto AddrReg = MIRBuilder.buildFrameIndex(Res: LLT::pointer(AddressSpace: 0, SizeInBits: 64), Idx: FI);
208 return AddrReg.getReg(Idx: 0);
209 }
210
211 LLT getStackValueStoreType(const DataLayout &DL, const CCValAssign &VA,
212 ISD::ArgFlagsTy Flags) const override {
213 // For pointers, we just need to fixup the integer types reported in the
214 // CCValAssign.
215 if (Flags.isPointer())
216 return CallLowering::ValueHandler::getStackValueStoreType(DL, VA, Flags);
217 return getStackValueStoreTypeHack(VA);
218 }
219
220 void assignValueToReg(Register ValVReg, Register PhysReg,
221 const CCValAssign &VA,
222 ISD::ArgFlagsTy Flags = {}) override {
223 markRegUsed(Reg: PhysReg);
224 IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA);
225 }
226
227 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
228 const MachinePointerInfo &MPO,
229 const CCValAssign &VA) override {
230 MachineFunction &MF = MIRBuilder.getMF();
231
232 LLT ValTy(VA.getValVT());
233 LLT LocTy(VA.getLocVT());
234
235 // Fixup the types for the DAG compatibility hack.
236 if (VA.getValVT() == MVT::i8 || VA.getValVT() == MVT::i16)
237 std::swap(a&: ValTy, b&: LocTy);
238 else {
239 // The calling code knows if this is a pointer or not, we're only touching
240 // the LocTy for the i8/i16 hack.
241 assert(LocTy.getSizeInBits() == MemTy.getSizeInBits());
242 LocTy = MemTy;
243 }
244
245 auto MMO = MF.getMachineMemOperand(
246 PtrInfo: MPO, F: MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, MemTy: LocTy,
247 BaseAlignment: inferAlignFromPtrInfo(MF, MPO));
248
249 switch (VA.getLocInfo()) {
250 case CCValAssign::LocInfo::ZExt:
251 MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_ZEXTLOAD, Res: ValVReg, Addr, MMO&: *MMO);
252 return;
253 case CCValAssign::LocInfo::SExt:
254 MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_SEXTLOAD, Res: ValVReg, Addr, MMO&: *MMO);
255 return;
256 default:
257 MIRBuilder.buildLoad(Res: ValVReg, Addr, MMO&: *MMO);
258 return;
259 }
260 }
261
262 /// How the physical register gets marked varies between formal
263 /// parameters (it's a basic-block live-in), and a call instruction
264 /// (it's an implicit-def of the BL).
265 virtual void markRegUsed(Register Reg) = 0;
266};
267
268struct FormalArgHandler : public IncomingArgHandler {
269 FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
270 : IncomingArgHandler(MIRBuilder, MRI) {}
271
272 void markRegUsed(Register Reg) override {
273 MIRBuilder.getMRI()->addLiveIn(Reg: Reg.asMCReg());
274 MIRBuilder.getMBB().addLiveIn(PhysReg: Reg.asMCReg());
275 }
276};
277
278struct CallReturnHandler : public IncomingArgHandler {
279 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
280 MachineInstrBuilder MIB)
281 : IncomingArgHandler(MIRBuilder, MRI), MIB(MIB) {}
282
283 void markRegUsed(Register Reg) override {
284 MIB.addDef(RegNo: Reg, Flags: RegState::Implicit);
285 }
286
287 MachineInstrBuilder MIB;
288};
289
290/// A special return arg handler for "returned" attribute arg calls.
291struct ReturnedArgCallReturnHandler : public CallReturnHandler {
292 ReturnedArgCallReturnHandler(MachineIRBuilder &MIRBuilder,
293 MachineRegisterInfo &MRI,
294 MachineInstrBuilder MIB)
295 : CallReturnHandler(MIRBuilder, MRI, MIB) {}
296
297 void markRegUsed(Register Reg) override {}
298};
299
300struct OutgoingArgHandler : public CallLowering::OutgoingValueHandler {
301 OutgoingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
302 MachineInstrBuilder MIB, bool IsTailCall = false,
303 int FPDiff = 0)
304 : OutgoingValueHandler(MIRBuilder, MRI), MIB(MIB), IsTailCall(IsTailCall),
305 FPDiff(FPDiff),
306 Subtarget(MIRBuilder.getMF().getSubtarget<AArch64Subtarget>()) {}
307
308 Register getStackAddress(uint64_t Size, int64_t Offset,
309 MachinePointerInfo &MPO,
310 ISD::ArgFlagsTy Flags) override {
311 MachineFunction &MF = MIRBuilder.getMF();
312 LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: 64);
313 LLT s64 = LLT::integer(SizeInBits: 64);
314
315 if (IsTailCall) {
316 assert(!Flags.isByVal() && "byval unhandled with tail calls");
317
318 Offset += FPDiff;
319 int FI = MF.getFrameInfo().CreateFixedObject(Size, SPOffset: Offset, IsImmutable: true);
320 auto FIReg = MIRBuilder.buildFrameIndex(Res: p0, Idx: FI);
321 MPO = MachinePointerInfo::getFixedStack(MF, FI);
322 return FIReg.getReg(Idx: 0);
323 }
324
325 if (!SPReg)
326 SPReg = MIRBuilder.buildCopy(Res: p0, Op: Register(AArch64::SP)).getReg(Idx: 0);
327
328 auto OffsetReg = MIRBuilder.buildConstant(Res: s64, Val: Offset);
329
330 auto AddrReg = MIRBuilder.buildPtrAdd(Res: p0, Op0: SPReg, Op1: OffsetReg);
331
332 MPO = MachinePointerInfo::getStack(MF, Offset);
333 return AddrReg.getReg(Idx: 0);
334 }
335
336 /// We need to fixup the reported store size for certain value types because
337 /// we invert the interpretation of ValVT and LocVT in certain cases. This is
338 /// for compatibility with the DAG call lowering implementation, which we're
339 /// currently building on top of.
340 LLT getStackValueStoreType(const DataLayout &DL, const CCValAssign &VA,
341 ISD::ArgFlagsTy Flags) const override {
342 if (Flags.isPointer())
343 return CallLowering::ValueHandler::getStackValueStoreType(DL, VA, Flags);
344 return getStackValueStoreTypeHack(VA);
345 }
346
347 void assignValueToReg(Register ValVReg, Register PhysReg,
348 const CCValAssign &VA, ISD::ArgFlagsTy Flags) override {
349 MIB.addUse(RegNo: PhysReg, Flags: RegState::Implicit);
350 Register ExtReg = extendRegister(ValReg: ValVReg, VA);
351 MIRBuilder.buildCopy(Res: PhysReg, Op: ExtReg);
352 }
353
354 /// Check whether a stack argument requires lowering in a tail call.
355 static bool shouldLowerTailCallStackArg(const MachineFunction &MF,
356 const CCValAssign &VA,
357 Register ValVReg,
358 Register StoreAddr) {
359 const MachineRegisterInfo &MRI = MF.getRegInfo();
360 // Print the defining instruction for the value.
361 auto *DefMI = MRI.getVRegDef(Reg: ValVReg);
362 assert(DefMI && "No defining instruction");
363 for (;;) {
364 // Look through nodes that don't alter the bits of the incoming value.
365 unsigned Op = DefMI->getOpcode();
366 if (Op == TargetOpcode::G_ZEXT || Op == TargetOpcode::G_ANYEXT ||
367 Op == TargetOpcode::G_BITCAST || isAssertMI(MI: *DefMI)) {
368 DefMI = MRI.getVRegDef(Reg: DefMI->getOperand(i: 1).getReg());
369 continue;
370 }
371 break;
372 }
373
374 auto *Load = dyn_cast<GLoad>(Val: DefMI);
375 if (!Load)
376 return true;
377 Register LoadReg = Load->getPointerReg();
378 auto *LoadAddrDef = MRI.getVRegDef(Reg: LoadReg);
379 if (LoadAddrDef->getOpcode() != TargetOpcode::G_FRAME_INDEX)
380 return true;
381 const MachineFrameInfo &MFI = MF.getFrameInfo();
382 int LoadFI = LoadAddrDef->getOperand(i: 1).getIndex();
383
384 auto *StoreAddrDef = MRI.getVRegDef(Reg: StoreAddr);
385 if (StoreAddrDef->getOpcode() != TargetOpcode::G_FRAME_INDEX)
386 return true;
387 int StoreFI = StoreAddrDef->getOperand(i: 1).getIndex();
388
389 if (!MFI.isImmutableObjectIndex(ObjectIdx: LoadFI))
390 return true;
391 if (MFI.getObjectOffset(ObjectIdx: LoadFI) != MFI.getObjectOffset(ObjectIdx: StoreFI))
392 return true;
393 if (Load->getMemSize() != MFI.getObjectSize(ObjectIdx: StoreFI))
394 return true;
395
396 return false;
397 }
398
399 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
400 const MachinePointerInfo &MPO,
401 const CCValAssign &VA) override {
402 MachineFunction &MF = MIRBuilder.getMF();
403 if (!FPDiff && !shouldLowerTailCallStackArg(MF, VA, ValVReg, StoreAddr: Addr))
404 return;
405 auto MMO = MF.getMachineMemOperand(PtrInfo: MPO, F: MachineMemOperand::MOStore, MemTy,
406 BaseAlignment: inferAlignFromPtrInfo(MF, MPO));
407 MIRBuilder.buildStore(Val: ValVReg, Addr, MMO&: *MMO);
408 }
409
410 void assignValueToAddress(const CallLowering::ArgInfo &Arg, unsigned RegIndex,
411 Register Addr, LLT MemTy,
412 const MachinePointerInfo &MPO,
413 const CCValAssign &VA) override {
414 unsigned MaxSize = MemTy.getSizeInBytes() * 8;
415 // For varargs, we always want to extend them to 8 bytes, in which case
416 // we disable setting a max.
417 if (Arg.Flags[0].isVarArg())
418 MaxSize = 0;
419
420 Register ValVReg = Arg.Regs[RegIndex];
421 if (VA.getLocInfo() != CCValAssign::LocInfo::FPExt) {
422 MVT LocVT = VA.getLocVT();
423 MVT ValVT = VA.getValVT();
424
425 if (VA.getValVT() == MVT::i8 || VA.getValVT() == MVT::i16) {
426 std::swap(a&: ValVT, b&: LocVT);
427 MemTy = LLT(VA.getValVT());
428 }
429
430 ValVReg = extendRegister(ValReg: ValVReg, VA, MaxSizeBits: MaxSize);
431 } else {
432 // The store does not cover the full allocated stack slot.
433 MemTy = LLT(VA.getValVT());
434 }
435
436 assignValueToAddress(ValVReg, Addr, MemTy, MPO, VA);
437 }
438
439 MachineInstrBuilder MIB;
440
441 bool IsTailCall;
442
443 /// For tail calls, the byte offset of the call's argument area from the
444 /// callee's. Unused elsewhere.
445 int FPDiff;
446
447 // Cache the SP register vreg if we need it more than once in this call site.
448 Register SPReg;
449
450 const AArch64Subtarget &Subtarget;
451};
452} // namespace
453
454static bool doesCalleeRestoreStack(CallingConv::ID CallConv, bool TailCallOpt) {
455 return (CallConv == CallingConv::Fast && TailCallOpt) ||
456 CallConv == CallingConv::Tail || CallConv == CallingConv::SwiftTail;
457}
458
459bool AArch64CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
460 const Value *Val,
461 ArrayRef<Register> VRegs,
462 FunctionLoweringInfo &FLI,
463 Register SwiftErrorVReg) const {
464 auto MIB = MIRBuilder.buildInstrNoInsert(Opcode: AArch64::RET_ReallyLR);
465 assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) &&
466 "Return value without a vreg");
467
468 bool Success = true;
469 if (!FLI.CanLowerReturn) {
470 insertSRetStores(MIRBuilder, RetTy: Val->getType(), VRegs, DemoteReg: FLI.DemoteRegister);
471 } else if (!VRegs.empty()) {
472 MachineFunction &MF = MIRBuilder.getMF();
473 const Function &F = MF.getFunction();
474 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
475
476 MachineRegisterInfo &MRI = MF.getRegInfo();
477 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
478 CCAssignFn *AssignFn = TLI.CCAssignFnForReturn(CC: F.getCallingConv());
479 auto &DL = F.getDataLayout();
480 LLVMContext &Ctx = Val->getType()->getContext();
481
482 SmallVector<EVT, 4> SplitEVTs;
483 ComputeValueVTs(TLI, DL, Ty: Val->getType(), ValueVTs&: SplitEVTs);
484 assert(VRegs.size() == SplitEVTs.size() &&
485 "For each split Type there should be exactly one VReg.");
486
487 SmallVector<ArgInfo, 8> SplitArgs;
488 CallingConv::ID CC = F.getCallingConv();
489
490 for (unsigned i = 0; i < SplitEVTs.size(); ++i) {
491 Register CurVReg = VRegs[i];
492 ArgInfo CurArgInfo = ArgInfo{CurVReg, SplitEVTs[i].getTypeForEVT(Context&: Ctx), 0};
493 setArgFlags(Arg&: CurArgInfo, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: F);
494
495 // i1 is a special case because SDAG i1 true is naturally zero extended
496 // when widened using ANYEXT. We need to do it explicitly here.
497 auto &Flags = CurArgInfo.Flags[0];
498 if (MRI.getType(Reg: CurVReg).getSizeInBits() == TypeSize::getFixed(ExactSize: 1) &&
499 !Flags.isSExt() && !Flags.isZExt()) {
500 CurVReg = MIRBuilder.buildZExt(Res: LLT::integer(SizeInBits: 8), Op: CurVReg).getReg(Idx: 0);
501 } else if (TLI.getNumRegistersForCallingConv(Context&: Ctx, CC, VT: SplitEVTs[i]) ==
502 1) {
503 // Some types will need extending as specified by the CC.
504 MVT NewVT = TLI.getRegisterTypeForCallingConv(Context&: Ctx, CC, VT: SplitEVTs[i]);
505 if (EVT(NewVT) != SplitEVTs[i]) {
506 unsigned ExtendOp = TargetOpcode::G_ANYEXT;
507 if (F.getAttributes().hasRetAttr(Kind: Attribute::SExt))
508 ExtendOp = TargetOpcode::G_SEXT;
509 else if (F.getAttributes().hasRetAttr(Kind: Attribute::ZExt))
510 ExtendOp = TargetOpcode::G_ZEXT;
511
512 LLT NewLLT(NewVT);
513 LLT OldLLT = getLLTForType(Ty&: *CurArgInfo.Ty, DL);
514 CurArgInfo.Ty = EVT(NewVT).getTypeForEVT(Context&: Ctx);
515 // Instead of an extend, we might have a vector type which needs
516 // padding with more elements, e.g. <2 x half> -> <4 x half>.
517 if (NewVT.isVector()) {
518 if (OldLLT.isVector()) {
519 if (NewLLT.getNumElements() > OldLLT.getNumElements()) {
520 CurVReg =
521 MIRBuilder.buildPadVectorWithUndefElements(Res: NewLLT, Op0: CurVReg)
522 .getReg(Idx: 0);
523 } else {
524 // Just do a vector extend.
525 CurVReg = MIRBuilder.buildInstr(Opc: ExtendOp, DstOps: {NewLLT}, SrcOps: {CurVReg})
526 .getReg(Idx: 0);
527 }
528 } else if (NewLLT.getNumElements() >= 2 &&
529 NewLLT.getNumElements() <= 8) {
530 // We need to pad a <1 x S> type to <2/4/8 x S>. Since we don't
531 // have <1 x S> vector types in GISel we use a build_vector
532 // instead of a vector merge/concat.
533 CurVReg =
534 MIRBuilder.buildPadVectorWithUndefElements(Res: NewLLT, Op0: CurVReg)
535 .getReg(Idx: 0);
536 } else {
537 LLVM_DEBUG(dbgs() << "Could not handle ret ty\n");
538 return false;
539 }
540 } else {
541 // If the split EVT was a <1 x T> vector, and NewVT is T, then we
542 // don't have to do anything since we don't distinguish between the
543 // two.
544 if (NewLLT.getScalarSizeInBits() !=
545 MRI.getType(Reg: CurVReg).getScalarSizeInBits()) {
546 // A scalar extend.
547 CurVReg = MIRBuilder.buildInstr(Opc: ExtendOp, DstOps: {NewLLT}, SrcOps: {CurVReg})
548 .getReg(Idx: 0);
549 }
550 }
551 }
552 }
553 if (CurVReg != CurArgInfo.Regs[0]) {
554 CurArgInfo.Regs[0] = CurVReg;
555 // Reset the arg flags after modifying CurVReg.
556 setArgFlags(Arg&: CurArgInfo, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: F);
557 }
558 splitToValueTypes(OrigArgInfo: CurArgInfo, SplitArgs, DL, CallConv: CC);
559 }
560
561 AArch64OutgoingValueAssigner Assigner(AssignFn, AssignFn, Subtarget,
562 /*IsReturn*/ true);
563 OutgoingArgHandler Handler(MIRBuilder, MRI, MIB);
564 Success = determineAndHandleAssignments(Handler, Assigner, Args&: SplitArgs,
565 MIRBuilder, CallConv: CC, IsVarArg: F.isVarArg());
566 }
567
568 if (SwiftErrorVReg) {
569 MIB.addUse(RegNo: AArch64::X21, Flags: RegState::Implicit);
570 MIRBuilder.buildCopy(Res: AArch64::X21, Op: SwiftErrorVReg);
571 }
572
573 MIRBuilder.insertInstr(MIB);
574 return Success;
575}
576
577bool AArch64CallLowering::canLowerReturn(MachineFunction &MF,
578 CallingConv::ID CallConv,
579 SmallVectorImpl<BaseArgInfo> &Outs,
580 bool IsVarArg) const {
581 SmallVector<CCValAssign, 16> ArgLocs;
582 const auto &TLI = *getTLI<AArch64TargetLowering>();
583 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs,
584 MF.getFunction().getContext());
585
586 return checkReturn(CCInfo, Outs, Fn: TLI.CCAssignFnForReturn(CC: CallConv));
587}
588
589/// Helper function to compute forwarded registers for musttail calls. Computes
590/// the forwarded registers, sets MBB liveness, and emits COPY instructions that
591/// can be used to save + restore registers later.
592static void handleMustTailForwardedRegisters(MachineIRBuilder &MIRBuilder,
593 CCAssignFn *AssignFn) {
594 MachineBasicBlock &MBB = MIRBuilder.getMBB();
595 MachineFunction &MF = MIRBuilder.getMF();
596 MachineFrameInfo &MFI = MF.getFrameInfo();
597
598 if (!MFI.hasMustTailInVarArgFunc())
599 return;
600
601 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
602 const Function &F = MF.getFunction();
603 assert(F.isVarArg() && "Expected F to be vararg?");
604
605 // Compute the set of forwarded registers. The rest are scratch.
606 SmallVector<CCValAssign, 16> ArgLocs;
607 CCState CCInfo(F.getCallingConv(), /*IsVarArg=*/true, MF, ArgLocs,
608 F.getContext());
609 SmallVector<MVT, 2> RegParmTypes;
610 RegParmTypes.push_back(Elt: MVT::i64);
611 RegParmTypes.push_back(Elt: MVT::f128);
612
613 // Later on, we can use this vector to restore the registers if necessary.
614 SmallVectorImpl<ForwardedRegister> &Forwards =
615 FuncInfo->getForwardedMustTailRegParms();
616 CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, Fn: AssignFn);
617
618 // Conservatively forward X8, since it might be used for an aggregate
619 // return.
620 if (!CCInfo.isAllocated(Reg: AArch64::X8)) {
621 Register X8VReg = MF.addLiveIn(PReg: AArch64::X8, RC: &AArch64::GPR64RegClass);
622 Forwards.push_back(Elt: ForwardedRegister(X8VReg, AArch64::X8, MVT::i64));
623 }
624
625 // Add the forwards to the MachineBasicBlock and MachineFunction.
626 for (const auto &F : Forwards) {
627 MBB.addLiveIn(PhysReg: F.PReg);
628 MIRBuilder.buildCopy(Res: Register(F.VReg), Op: Register(F.PReg));
629 }
630}
631
632bool AArch64CallLowering::fallBackToDAGISel(const MachineFunction &MF) const {
633 auto &F = MF.getFunction();
634 const auto &TM = static_cast<const AArch64TargetMachine &>(MF.getTarget());
635
636 if (!TM.getCLOpts().enable_gisel_sve &&
637 (F.getReturnType()->isScalableTy() ||
638 llvm::any_of(Range: F.args(), P: [](const Argument &A) {
639 return A.getType()->isScalableTy();
640 })))
641 return true;
642 const auto &ST = MF.getSubtarget<AArch64Subtarget>();
643 if (!ST.hasNEON() || !ST.hasFPARMv8()) {
644 LLVM_DEBUG(dbgs() << "Falling back to SDAG because we don't support no-NEON\n");
645 return true;
646 }
647
648 SMEAttrs Attrs = MF.getInfo<AArch64FunctionInfo>()->getSMEFnAttrs();
649 if (Attrs.hasZAState() || Attrs.hasZT0State() ||
650 Attrs.hasStreamingInterfaceOrBody() ||
651 Attrs.hasStreamingCompatibleInterface())
652 return true;
653
654 auto OptLevel = MF.getTarget().getOptLevel();
655 bool IsGlobalISelPreferred =
656 getCGPassBuilderOption().EnableGlobalISelOption ==
657 cl::boolOrDefault::BOU_TRUE ||
658 static_cast<unsigned>(OptLevel) <= TM.getEnableGlobalISelAtO() ||
659 F.hasOptNone();
660 return !IsGlobalISelPreferred;
661}
662
663void AArch64CallLowering::saveVarArgRegisters(
664 MachineIRBuilder &MIRBuilder, CallLowering::IncomingValueHandler &Handler,
665 CCState &CCInfo) const {
666 auto GPRArgRegs = AArch64::getGPRArgRegs();
667 auto FPRArgRegs = AArch64::getFPRArgRegs();
668
669 MachineFunction &MF = MIRBuilder.getMF();
670 MachineRegisterInfo &MRI = MF.getRegInfo();
671 MachineFrameInfo &MFI = MF.getFrameInfo();
672 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
673 auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
674 bool IsWin64CC = Subtarget.isCallingConvWin64(CC: CCInfo.getCallingConv(),
675 IsVarArg: MF.getFunction().isVarArg());
676 const LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: 64);
677 const LLT s64 = LLT::integer(SizeInBits: 64);
678
679 unsigned FirstVariadicGPR = CCInfo.getFirstUnallocated(Regs: GPRArgRegs);
680 unsigned NumVariadicGPRArgRegs = GPRArgRegs.size() - FirstVariadicGPR + 1;
681
682 unsigned GPRSaveSize = 8 * (GPRArgRegs.size() - FirstVariadicGPR);
683 int GPRIdx = 0;
684 if (GPRSaveSize != 0) {
685 if (IsWin64CC) {
686 GPRIdx = MFI.CreateFixedObject(Size: GPRSaveSize,
687 SPOffset: -static_cast<int>(GPRSaveSize), IsImmutable: false);
688 if (GPRSaveSize & 15)
689 // The extra size here, if triggered, will always be 8.
690 MFI.CreateFixedObject(Size: 16 - (GPRSaveSize & 15),
691 SPOffset: -static_cast<int>(alignTo(Value: GPRSaveSize, Align: 16)),
692 IsImmutable: false);
693 } else
694 GPRIdx = MFI.CreateStackObject(Size: GPRSaveSize, Alignment: Align(8), isSpillSlot: false);
695
696 auto FIN = MIRBuilder.buildFrameIndex(Res: p0, Idx: GPRIdx);
697 auto Offset =
698 MIRBuilder.buildConstant(Res: MRI.createGenericVirtualRegister(Ty: s64), Val: 8);
699
700 for (unsigned i = FirstVariadicGPR; i < GPRArgRegs.size(); ++i) {
701 Register Val = MRI.createGenericVirtualRegister(Ty: s64);
702 Handler.assignValueToReg(
703 ValVReg: Val, PhysReg: GPRArgRegs[i],
704 VA: CCValAssign::getReg(ValNo: i + MF.getFunction().getNumOperands(), ValVT: MVT::i64,
705 Reg: GPRArgRegs[i], LocVT: MVT::i64, HTP: CCValAssign::Full));
706 auto MPO = IsWin64CC ? MachinePointerInfo::getFixedStack(
707 MF, FI: GPRIdx, Offset: (i - FirstVariadicGPR) * 8)
708 : MachinePointerInfo::getStack(MF, Offset: i * 8);
709 MIRBuilder.buildStore(Val, Addr: FIN, PtrInfo: MPO, Alignment: inferAlignFromPtrInfo(MF, MPO));
710
711 FIN = MIRBuilder.buildPtrAdd(Res: MRI.createGenericVirtualRegister(Ty: p0),
712 Op0: FIN.getReg(Idx: 0), Op1: Offset);
713 }
714 }
715 FuncInfo->setVarArgsGPRIndex(GPRIdx);
716 FuncInfo->setVarArgsGPRSize(GPRSaveSize);
717
718 if (Subtarget.hasFPARMv8() && !IsWin64CC) {
719 unsigned FirstVariadicFPR = CCInfo.getFirstUnallocated(Regs: FPRArgRegs);
720
721 unsigned FPRSaveSize = 16 * (FPRArgRegs.size() - FirstVariadicFPR);
722 int FPRIdx = 0;
723 if (FPRSaveSize != 0) {
724 FPRIdx = MFI.CreateStackObject(Size: FPRSaveSize, Alignment: Align(16), isSpillSlot: false);
725
726 auto FIN = MIRBuilder.buildFrameIndex(Res: p0, Idx: FPRIdx);
727 auto Offset =
728 MIRBuilder.buildConstant(Res: MRI.createGenericVirtualRegister(Ty: s64), Val: 16);
729
730 for (unsigned i = FirstVariadicFPR; i < FPRArgRegs.size(); ++i) {
731 Register Val = MRI.createGenericVirtualRegister(Ty: LLT::float128());
732 Handler.assignValueToReg(
733 ValVReg: Val, PhysReg: FPRArgRegs[i],
734 VA: CCValAssign::getReg(
735 ValNo: i + MF.getFunction().getNumOperands() + NumVariadicGPRArgRegs,
736 ValVT: MVT::f128, Reg: FPRArgRegs[i], LocVT: MVT::f128, HTP: CCValAssign::Full));
737
738 auto MPO = MachinePointerInfo::getStack(MF, Offset: i * 16);
739 MIRBuilder.buildStore(Val, Addr: FIN, PtrInfo: MPO, Alignment: inferAlignFromPtrInfo(MF, MPO));
740
741 FIN = MIRBuilder.buildPtrAdd(Res: MRI.createGenericVirtualRegister(Ty: p0),
742 Op0: FIN.getReg(Idx: 0), Op1: Offset);
743 }
744 }
745 FuncInfo->setVarArgsFPRIndex(FPRIdx);
746 FuncInfo->setVarArgsFPRSize(FPRSaveSize);
747 }
748}
749
750bool AArch64CallLowering::lowerFormalArguments(
751 MachineIRBuilder &MIRBuilder, const Function &F,
752 ArrayRef<ArrayRef<Register>> VRegs, FunctionLoweringInfo &FLI) const {
753 MachineFunction &MF = MIRBuilder.getMF();
754 MachineBasicBlock &MBB = MIRBuilder.getMBB();
755 MachineRegisterInfo &MRI = MF.getRegInfo();
756 auto &DL = F.getDataLayout();
757 auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
758
759 // Arm64EC has extra requirements for varargs calls which are only implemented
760 // in SelectionDAG; bail out for now.
761 if (F.isVarArg() && Subtarget.isWindowsArm64EC())
762 return false;
763
764 // Arm64EC thunks have a special calling convention which is only implemented
765 // in SelectionDAG; bail out for now.
766 if (F.getCallingConv() == CallingConv::ARM64EC_Thunk_Native ||
767 F.getCallingConv() == CallingConv::ARM64EC_Thunk_X64)
768 return false;
769
770 bool IsWin64 = Subtarget.isCallingConvWin64(CC: F.getCallingConv(), IsVarArg: F.isVarArg());
771
772 // If an argument is marked "sret" and "inreg", it must be returned in x0.
773 // Bail for now.
774 if (IsWin64 && any_of(Range: F.args(), P: [](const Argument &A) {
775 return A.hasStructRetAttr() && A.hasInRegAttr();
776 }))
777 return false;
778
779 SmallVector<ArgInfo, 8> SplitArgs;
780 SmallVector<std::pair<Register, Register>> BoolArgs;
781
782 // Insert the hidden sret parameter if the return value won't fit in the
783 // return registers.
784 if (!FLI.CanLowerReturn)
785 insertSRetIncomingArgument(F, SplitArgs, DemoteReg&: FLI.DemoteRegister, MRI, DL);
786
787 unsigned i = 0;
788 for (auto &Arg : F.args()) {
789 if (DL.getTypeStoreSize(Ty: Arg.getType()).isZero())
790 continue;
791
792 ArgInfo OrigArg{VRegs[i], Arg, i};
793 setArgFlags(Arg&: OrigArg, OpIdx: i + AttributeList::FirstArgIndex, DL, FuncInfo: F);
794
795 // i1 arguments are zero-extended to i8 by the caller. Emit a
796 // hint to reflect this.
797 if (OrigArg.Ty->isIntegerTy(BitWidth: 1)) {
798 assert(OrigArg.Regs.size() == 1 &&
799 MRI.getType(OrigArg.Regs[0]).getSizeInBits() == 1 &&
800 "Unexpected registers used for i1 arg");
801
802 auto &Flags = OrigArg.Flags[0];
803 if (!Flags.isZExt() && !Flags.isSExt()) {
804 // Lower i1 argument as i8, and insert AssertZExt + Trunc later.
805 Register OrigReg = OrigArg.Regs[0];
806 Register WideReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: 8));
807 OrigArg.Regs[0] = WideReg;
808 BoolArgs.push_back(Elt: {OrigReg, WideReg});
809 }
810 }
811
812 if (Arg.hasAttribute(Kind: Attribute::SwiftAsync))
813 MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
814
815 splitToValueTypes(OrigArgInfo: OrigArg, SplitArgs, DL, CallConv: F.getCallingConv());
816 ++i;
817 }
818
819 if (!MBB.empty())
820 MIRBuilder.setInstr(*MBB.begin());
821
822 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
823 CCAssignFn *AssignFn = TLI.CCAssignFnForCall(CC: F.getCallingConv(), IsVarArg: IsWin64 && F.isVarArg());
824
825 AArch64IncomingValueAssigner Assigner(AssignFn, AssignFn);
826 FormalArgHandler Handler(MIRBuilder, MRI);
827 SmallVector<CCValAssign, 16> ArgLocs;
828 CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());
829 if (!determineAssignments(Assigner, Args&: SplitArgs, CCInfo) ||
830 !handleAssignments(Handler, Args&: SplitArgs, CCState&: CCInfo, ArgLocs, MIRBuilder))
831 return false;
832
833 if (!BoolArgs.empty()) {
834 for (auto &KV : BoolArgs) {
835 Register OrigReg = KV.first;
836 Register WideReg = KV.second;
837 LLT WideTy = MRI.getType(Reg: WideReg);
838 assert(MRI.getType(OrigReg).getScalarSizeInBits() == 1 &&
839 "Unexpected bit size of a bool arg");
840 MIRBuilder.buildTrunc(
841 Res: OrigReg, Op: MIRBuilder.buildAssertZExt(Res: WideTy, Op: WideReg, Size: 1).getReg(Idx: 0));
842 }
843 }
844
845 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
846 uint64_t StackSize = Assigner.StackSize;
847 if (F.isVarArg()) {
848 if ((!Subtarget.isTargetDarwin() && !Subtarget.isWindowsArm64EC()) || IsWin64) {
849 // The AAPCS variadic function ABI is identical to the non-variadic
850 // one. As a result there may be more arguments in registers and we should
851 // save them for future reference.
852 // Win64 variadic functions also pass arguments in registers, but all
853 // float arguments are passed in integer registers.
854 saveVarArgRegisters(MIRBuilder, Handler, CCInfo);
855 } else if (Subtarget.isWindowsArm64EC()) {
856 return false;
857 }
858
859 // We currently pass all varargs at 8-byte alignment, or 4 in ILP32.
860 StackSize = alignTo(Value: Assigner.StackSize, Align: Subtarget.isTargetILP32() ? 4 : 8);
861
862 auto &MFI = MIRBuilder.getMF().getFrameInfo();
863 FuncInfo->setVarArgsStackIndex(MFI.CreateFixedObject(Size: 4, SPOffset: StackSize, IsImmutable: true));
864 }
865
866 if (doesCalleeRestoreStack(CallConv: F.getCallingConv(),
867 TailCallOpt: MF.getTarget().Options.GuaranteedTailCallOpt)) {
868 // We have a non-standard ABI, so why not make full use of the stack that
869 // we're going to pop? It must be aligned to 16 B in any case.
870 StackSize = alignTo(Value: StackSize, Align: 16);
871
872 // If we're expected to restore the stack (e.g. fastcc), then we'll be
873 // adding a multiple of 16.
874 FuncInfo->setArgumentStackToRestore(StackSize);
875
876 // Our own callers will guarantee that the space is free by giving an
877 // aligned value to CALLSEQ_START.
878 }
879
880 // When we tail call, we need to check if the callee's arguments
881 // will fit on the caller's stack. So, whenever we lower formal arguments,
882 // we should keep track of this information, since we might lower a tail call
883 // in this function later.
884 FuncInfo->setBytesInStackArgArea(StackSize);
885
886 if (Subtarget.hasCustomCallingConv())
887 Subtarget.getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF);
888
889 handleMustTailForwardedRegisters(MIRBuilder, AssignFn);
890
891 // Move back to the end of the basic block.
892 MIRBuilder.setMBB(MBB);
893
894 return true;
895}
896
897/// Return true if the calling convention is one that we can guarantee TCO for.
898static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls) {
899 return (CC == CallingConv::Fast && GuaranteeTailCalls) ||
900 CC == CallingConv::Tail || CC == CallingConv::SwiftTail;
901}
902
903/// Return true if we might ever do TCO for calls with this calling convention.
904static bool mayTailCallThisCC(CallingConv::ID CC) {
905 switch (CC) {
906 case CallingConv::C:
907 case CallingConv::PreserveMost:
908 case CallingConv::PreserveAll:
909 case CallingConv::PreserveNone:
910 case CallingConv::Swift:
911 case CallingConv::SwiftTail:
912 case CallingConv::Tail:
913 case CallingConv::Fast:
914 return true;
915 default:
916 return false;
917 }
918}
919
920/// Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for
921/// CC.
922static std::pair<CCAssignFn *, CCAssignFn *>
923getAssignFnsForCC(CallingConv::ID CC, const AArch64TargetLowering &TLI) {
924 return {TLI.CCAssignFnForCall(CC, IsVarArg: false), TLI.CCAssignFnForCall(CC, IsVarArg: true)};
925}
926
927bool AArch64CallLowering::doCallerAndCalleePassArgsTheSameWay(
928 CallLoweringInfo &Info, MachineFunction &MF,
929 SmallVectorImpl<ArgInfo> &InArgs) const {
930 const Function &CallerF = MF.getFunction();
931 CallingConv::ID CalleeCC = Info.CallConv;
932 CallingConv::ID CallerCC = CallerF.getCallingConv();
933
934 // If the calling conventions match, then everything must be the same.
935 if (CalleeCC == CallerCC)
936 return true;
937
938 // Check if the caller and callee will handle arguments in the same way.
939 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
940 CCAssignFn *CalleeAssignFnFixed;
941 CCAssignFn *CalleeAssignFnVarArg;
942 std::tie(args&: CalleeAssignFnFixed, args&: CalleeAssignFnVarArg) =
943 getAssignFnsForCC(CC: CalleeCC, TLI);
944
945 CCAssignFn *CallerAssignFnFixed;
946 CCAssignFn *CallerAssignFnVarArg;
947 std::tie(args&: CallerAssignFnFixed, args&: CallerAssignFnVarArg) =
948 getAssignFnsForCC(CC: CallerCC, TLI);
949
950 AArch64IncomingValueAssigner CalleeAssigner(CalleeAssignFnFixed,
951 CalleeAssignFnVarArg);
952 AArch64IncomingValueAssigner CallerAssigner(CallerAssignFnFixed,
953 CallerAssignFnVarArg);
954
955 if (!resultsCompatible(Info, MF, InArgs, CalleeAssigner, CallerAssigner))
956 return false;
957
958 // Make sure that the caller and callee preserve all of the same registers.
959 auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
960 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
961 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
962 if (MF.getSubtarget<AArch64Subtarget>().hasCustomCallingConv()) {
963 TRI->UpdateCustomCallPreservedMask(MF, Mask: &CallerPreserved);
964 TRI->UpdateCustomCallPreservedMask(MF, Mask: &CalleePreserved);
965 }
966
967 return TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved);
968}
969
970bool AArch64CallLowering::areCalleeOutgoingArgsTailCallable(
971 CallLoweringInfo &Info, MachineFunction &MF,
972 SmallVectorImpl<ArgInfo> &OrigOutArgs) const {
973 // If there are no outgoing arguments, then we are done.
974 if (OrigOutArgs.empty())
975 return true;
976
977 const Function &CallerF = MF.getFunction();
978 LLVMContext &Ctx = CallerF.getContext();
979 CallingConv::ID CalleeCC = Info.CallConv;
980 CallingConv::ID CallerCC = CallerF.getCallingConv();
981 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
982 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
983
984 CCAssignFn *AssignFnFixed;
985 CCAssignFn *AssignFnVarArg;
986 std::tie(args&: AssignFnFixed, args&: AssignFnVarArg) = getAssignFnsForCC(CC: CalleeCC, TLI);
987
988 // We have outgoing arguments. Make sure that we can tail call with them.
989 SmallVector<CCValAssign, 16> OutLocs;
990 CCState OutInfo(CalleeCC, false, MF, OutLocs, Ctx);
991
992 AArch64OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg,
993 Subtarget, /*IsReturn*/ false);
994 // determineAssignments() may modify argument flags, so make a copy.
995 SmallVector<ArgInfo, 8> OutArgs;
996 append_range(C&: OutArgs, R&: OrigOutArgs);
997 if (!determineAssignments(Assigner&: CalleeAssigner, Args&: OutArgs, CCInfo&: OutInfo)) {
998 LLVM_DEBUG(dbgs() << "... Could not analyze call operands.\n");
999 return false;
1000 }
1001
1002 // Make sure that they can fit on the caller's stack.
1003 const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1004 if (OutInfo.getStackSize() > FuncInfo->getBytesInStackArgArea()) {
1005 LLVM_DEBUG(dbgs() << "... Cannot fit call operands on caller's stack.\n");
1006 return false;
1007 }
1008
1009 // Verify that the parameters in callee-saved registers match.
1010 // TODO: Port this over to CallLowering as general code once swiftself is
1011 // supported.
1012 auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
1013 const uint32_t *CallerPreservedMask = TRI->getCallPreservedMask(MF, CallerCC);
1014 MachineRegisterInfo &MRI = MF.getRegInfo();
1015
1016 if (Info.IsVarArg) {
1017 // Be conservative and disallow variadic memory operands to match SDAG's
1018 // behaviour.
1019 // FIXME: If the caller's calling convention is C, then we can
1020 // potentially use its argument area. However, for cases like fastcc,
1021 // we can't do anything.
1022 for (unsigned i = 0; i < OutLocs.size(); ++i) {
1023 auto &ArgLoc = OutLocs[i];
1024 if (ArgLoc.isRegLoc())
1025 continue;
1026
1027 LLVM_DEBUG(
1028 dbgs()
1029 << "... Cannot tail call vararg function with stack arguments\n");
1030 return false;
1031 }
1032 }
1033
1034 return parametersInCSRMatch(MRI, CallerPreservedMask, ArgLocs: OutLocs, OutVals: OutArgs);
1035}
1036
1037bool AArch64CallLowering::isEligibleForTailCallOptimization(
1038 MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
1039 SmallVectorImpl<ArgInfo> &InArgs,
1040 SmallVectorImpl<ArgInfo> &OutArgs) const {
1041
1042 // Must pass all target-independent checks in order to tail call optimize.
1043 if (!Info.IsTailCall)
1044 return false;
1045
1046 CallingConv::ID CalleeCC = Info.CallConv;
1047 MachineFunction &MF = MIRBuilder.getMF();
1048 const Function &CallerF = MF.getFunction();
1049
1050 LLVM_DEBUG(dbgs() << "Attempting to lower call as tail call\n");
1051
1052 if (Info.SwiftErrorVReg) {
1053 // TODO: We should handle this.
1054 // Note that this is also handled by the check for no outgoing arguments.
1055 // Proactively disabling this though, because the swifterror handling in
1056 // lowerCall inserts a COPY *after* the location of the call.
1057 LLVM_DEBUG(dbgs() << "... Cannot handle tail calls with swifterror yet.\n");
1058 return false;
1059 }
1060
1061 if (!mayTailCallThisCC(CC: CalleeCC)) {
1062 LLVM_DEBUG(dbgs() << "... Calling convention cannot be tail called.\n");
1063 return false;
1064 }
1065
1066 // Byval parameters hand the function a pointer directly into the stack area
1067 // we want to reuse during a tail call. Working around this *is* possible (see
1068 // X86).
1069 //
1070 // FIXME: In AArch64ISelLowering, this isn't worked around. Can/should we try
1071 // it?
1072 //
1073 // On Windows, "inreg" attributes signify non-aggregate indirect returns.
1074 // In this case, it is necessary to save/restore X0 in the callee. Tail
1075 // call opt interferes with this. So we disable tail call opt when the
1076 // caller has an argument with "inreg" attribute.
1077 //
1078 // FIXME: Check whether the callee also has an "inreg" argument.
1079 //
1080 // When the caller has a swifterror argument, we don't want to tail call
1081 // because would have to move into the swifterror register before the
1082 // tail call.
1083 if (any_of(Range: CallerF.args(), P: [](const Argument &A) {
1084 return A.hasByValAttr() || A.hasInRegAttr() || A.hasSwiftErrorAttr();
1085 })) {
1086 LLVM_DEBUG(dbgs() << "... Cannot tail call from callers with byval, "
1087 "inreg, or swifterror arguments\n");
1088 return false;
1089 }
1090
1091 // Externally-defined functions with weak linkage should not be
1092 // tail-called on AArch64 when the OS does not support dynamic
1093 // pre-emption of symbols, as the AAELF spec requires normal calls
1094 // to undefined weak functions to be replaced with a NOP or jump to the
1095 // next instruction. The behaviour of branch instructions in this
1096 // situation (as used for tail calls) is implementation-defined, so we
1097 // cannot rely on the linker replacing the tail call with a return.
1098 if (Info.Callee.isGlobal()) {
1099 const GlobalValue *GV = Info.Callee.getGlobal();
1100 const Triple &TT = GV->getParent()->getTargetTriple();
1101 if (GV->hasExternalWeakLinkage() &&
1102 (!TT.isOSWindows() || TT.isOSBinFormatELF() ||
1103 TT.isOSBinFormatMachO())) {
1104 LLVM_DEBUG(dbgs() << "... Cannot tail call externally-defined function "
1105 "with weak linkage for this OS.\n");
1106 return false;
1107 }
1108 }
1109
1110 // If we have -tailcallopt, then we're done.
1111 if (canGuaranteeTCO(CC: CalleeCC, GuaranteeTailCalls: MF.getTarget().Options.GuaranteedTailCallOpt))
1112 return CalleeCC == CallerF.getCallingConv();
1113
1114 // We don't have -tailcallopt, so we're allowed to change the ABI (sibcall).
1115 // Try to find cases where we can do that.
1116
1117 // I want anyone implementing a new calling convention to think long and hard
1118 // about this assert.
1119 assert((!Info.IsVarArg || CalleeCC == CallingConv::C) &&
1120 "Unexpected variadic calling convention");
1121
1122 // Verify that the incoming and outgoing arguments from the callee are
1123 // safe to tail call.
1124 if (!doCallerAndCalleePassArgsTheSameWay(Info, MF, InArgs)) {
1125 LLVM_DEBUG(
1126 dbgs()
1127 << "... Caller and callee have incompatible calling conventions.\n");
1128 return false;
1129 }
1130
1131 if (!areCalleeOutgoingArgsTailCallable(Info, MF, OrigOutArgs&: OutArgs))
1132 return false;
1133
1134 LLVM_DEBUG(
1135 dbgs() << "... Call is eligible for tail call optimization.\n");
1136 return true;
1137}
1138
1139static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect,
1140 bool IsTailCall,
1141 std::optional<CallLowering::PtrAuthInfo> &PAI,
1142 MachineRegisterInfo &MRI) {
1143 const AArch64FunctionInfo *FuncInfo = CallerF.getInfo<AArch64FunctionInfo>();
1144
1145 if (!IsTailCall) {
1146 if (!PAI)
1147 return IsIndirect ? getBLRCallOpcode(MF: CallerF) : (unsigned)AArch64::BL;
1148
1149 assert(IsIndirect && "Direct call should not be authenticated");
1150 assert((PAI->Key == AArch64PACKey::IA || PAI->Key == AArch64PACKey::IB) &&
1151 "Invalid auth call key");
1152 return AArch64::BLRA;
1153 }
1154
1155 if (!IsIndirect)
1156 return AArch64::TCRETURNdi;
1157
1158 // When BTI or PAuthLR are enabled, there are restrictions on using x16 and
1159 // x17 to hold the function pointer.
1160 if (FuncInfo->branchTargetEnforcement()) {
1161 if (FuncInfo->branchProtectionPAuthLR()) {
1162 assert(!PAI && "ptrauth tail-calls not yet supported with PAuthLR");
1163 return AArch64::TCRETURNrix17;
1164 }
1165 if (PAI)
1166 return AArch64::AUTH_TCRETURN_BTI;
1167 return AArch64::TCRETURNrix16x17;
1168 }
1169
1170 if (FuncInfo->branchProtectionPAuthLR()) {
1171 assert(!PAI && "ptrauth tail-calls not yet supported with PAuthLR");
1172 return AArch64::TCRETURNrinotx16;
1173 }
1174
1175 if (PAI)
1176 return AArch64::AUTH_TCRETURN;
1177 return AArch64::TCRETURNri;
1178}
1179
1180static const uint32_t *
1181getMaskForArgs(SmallVectorImpl<AArch64CallLowering::ArgInfo> &OutArgs,
1182 AArch64CallLowering::CallLoweringInfo &Info,
1183 const AArch64RegisterInfo &TRI, MachineFunction &MF) {
1184 const uint32_t *Mask;
1185 if (!OutArgs.empty() && OutArgs[0].Flags[0].isReturned()) {
1186 // For 'this' returns, use the X0-preserving mask if applicable
1187 Mask = TRI.getThisReturnPreservedMask(MF, Info.CallConv);
1188 if (!Mask) {
1189 OutArgs[0].Flags[0].setReturned(false);
1190 Mask = TRI.getCallPreservedMask(MF, Info.CallConv);
1191 }
1192 } else {
1193 Mask = TRI.getCallPreservedMask(MF, Info.CallConv);
1194 }
1195 return Mask;
1196}
1197
1198bool AArch64CallLowering::lowerTailCall(
1199 MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
1200 SmallVectorImpl<ArgInfo> &OutArgs) const {
1201 MachineFunction &MF = MIRBuilder.getMF();
1202 const Function &F = MF.getFunction();
1203 MachineRegisterInfo &MRI = MF.getRegInfo();
1204 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
1205 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1206
1207 // True when we're tail calling, but without -tailcallopt.
1208 bool IsSibCall = !MF.getTarget().Options.GuaranteedTailCallOpt &&
1209 Info.CallConv != CallingConv::Tail &&
1210 Info.CallConv != CallingConv::SwiftTail;
1211
1212 // Find out which ABI gets to decide where things go.
1213 CallingConv::ID CalleeCC = Info.CallConv;
1214 CCAssignFn *AssignFnFixed;
1215 CCAssignFn *AssignFnVarArg;
1216 std::tie(args&: AssignFnFixed, args&: AssignFnVarArg) = getAssignFnsForCC(CC: CalleeCC, TLI);
1217
1218 MachineInstrBuilder CallSeqStart;
1219 if (!IsSibCall)
1220 CallSeqStart = MIRBuilder.buildInstr(Opcode: AArch64::ADJCALLSTACKDOWN);
1221
1222 unsigned Opc = getCallOpcode(CallerF: MF, IsIndirect: Info.Callee.isReg(), IsTailCall: true, PAI&: Info.PAI, MRI);
1223 auto MIB = MIRBuilder.buildInstrNoInsert(Opcode: Opc);
1224 MIB.add(MO: Info.Callee);
1225
1226 // Tell the call which registers are clobbered.
1227 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1228 auto TRI = Subtarget.getRegisterInfo();
1229
1230 // Byte offset for the tail call. When we are sibcalling, this will always
1231 // be 0.
1232 MIB.addImm(Val: 0);
1233
1234 // Authenticated tail calls always take key/discriminator arguments.
1235 if (Opc == AArch64::AUTH_TCRETURN || Opc == AArch64::AUTH_TCRETURN_BTI) {
1236 assert((Info.PAI->Key == AArch64PACKey::IA ||
1237 Info.PAI->Key == AArch64PACKey::IB) &&
1238 "Invalid auth call key");
1239 MIB.addImm(Val: Info.PAI->Key);
1240
1241 Register AddrDisc = 0;
1242 uint16_t IntDisc = 0;
1243 std::tie(args&: IntDisc, args&: AddrDisc) =
1244 extractPtrauthBlendDiscriminators(Disc: Info.PAI->Discriminator, MRI);
1245
1246 MIB.addImm(Val: IntDisc);
1247 MIB.addUse(RegNo: AddrDisc);
1248 if (AddrDisc.isValid()) {
1249 MIB->getOperand(i: 4).setReg(constrainOperandRegClass(
1250 MF, TRI: *TRI, MRI, TII: *MF.getSubtarget().getInstrInfo(),
1251 RBI: *MF.getSubtarget().getRegBankInfo(), InsertPt&: *MIB, II: MIB->getDesc(),
1252 RegMO&: MIB->getOperand(i: 4), OpIdx: 4));
1253 }
1254 }
1255
1256 // Tell the call which registers are clobbered.
1257 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CalleeCC);
1258 if (Subtarget.hasCustomCallingConv())
1259 TRI->UpdateCustomCallPreservedMask(MF, Mask: &Mask);
1260 MIB.addRegMask(Mask);
1261
1262 if (Info.CFIType)
1263 MIB->setCFIType(MF, Type: Info.CFIType->getZExtValue());
1264
1265 if (TRI->isAnyArgRegReserved(MF))
1266 TRI->emitReservedArgRegCallError(MF);
1267
1268 // FPDiff is the byte offset of the call's argument area from the callee's.
1269 // Stores to callee stack arguments will be placed in FixedStackSlots offset
1270 // by this amount for a tail call. In a sibling call it must be 0 because the
1271 // caller will deallocate the entire stack and the callee still expects its
1272 // arguments to begin at SP+0.
1273 int FPDiff = 0;
1274
1275 // This will be 0 for sibcalls, potentially nonzero for tail calls produced
1276 // by -tailcallopt. For sibcalls, the memory operands for the call are
1277 // already available in the caller's incoming argument space.
1278 unsigned NumBytes = 0;
1279 if (!IsSibCall) {
1280 // We aren't sibcalling, so we need to compute FPDiff. We need to do this
1281 // before handling assignments, because FPDiff must be known for memory
1282 // arguments.
1283 unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
1284 SmallVector<CCValAssign, 16> OutLocs;
1285 CCState OutInfo(CalleeCC, false, MF, OutLocs, F.getContext());
1286
1287 AArch64OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg,
1288 Subtarget, /*IsReturn*/ false);
1289 if (!determineAssignments(Assigner&: CalleeAssigner, Args&: OutArgs, CCInfo&: OutInfo))
1290 return false;
1291
1292 // The callee will pop the argument stack as a tail call. Thus, we must
1293 // keep it 16-byte aligned.
1294 NumBytes = alignTo(Value: OutInfo.getStackSize(), Align: 16);
1295
1296 // FPDiff will be negative if this tail call requires more space than we
1297 // would automatically have in our incoming argument space. Positive if we
1298 // actually shrink the stack.
1299 FPDiff = NumReusableBytes - NumBytes;
1300
1301 // Update the required reserved area if this is the tail call requiring the
1302 // most argument stack space.
1303 if (FPDiff < 0 && FuncInfo->getTailCallReservedStack() < (unsigned)-FPDiff)
1304 FuncInfo->setTailCallReservedStack(-FPDiff);
1305
1306 // The stack pointer must be 16-byte aligned at all times it's used for a
1307 // memory operation, which in practice means at *all* times and in
1308 // particular across call boundaries. Therefore our own arguments started at
1309 // a 16-byte aligned SP and the delta applied for the tail call should
1310 // satisfy the same constraint.
1311 assert(FPDiff % 16 == 0 && "unaligned stack on tail call");
1312 }
1313
1314 const auto &Forwards = FuncInfo->getForwardedMustTailRegParms();
1315
1316 AArch64OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg,
1317 Subtarget, /*IsReturn*/ false);
1318
1319 // Do the actual argument marshalling.
1320 OutgoingArgHandler Handler(MIRBuilder, MRI, MIB,
1321 /*IsTailCall*/ true, FPDiff);
1322 if (!determineAndHandleAssignments(Handler, Assigner, Args&: OutArgs, MIRBuilder,
1323 CallConv: CalleeCC, IsVarArg: Info.IsVarArg))
1324 return false;
1325
1326 Mask = getMaskForArgs(OutArgs, Info, TRI: *TRI, MF);
1327
1328 if (Info.IsVarArg && Info.IsMustTailCall) {
1329 // Now we know what's being passed to the function. Add uses to the call for
1330 // the forwarded registers that we *aren't* passing as parameters. This will
1331 // preserve the copies we build earlier.
1332 for (const auto &F : Forwards) {
1333 Register ForwardedReg = F.PReg;
1334 // If the register is already passed, or aliases a register which is
1335 // already being passed, then skip it.
1336 if (any_of(Range: MIB->uses(), P: [&ForwardedReg, &TRI](const MachineOperand &Use) {
1337 if (!Use.isReg())
1338 return false;
1339 return TRI->regsOverlap(RegA: Use.getReg(), RegB: ForwardedReg);
1340 }))
1341 continue;
1342
1343 // We aren't passing it already, so we should add it to the call.
1344 MIRBuilder.buildCopy(Res: ForwardedReg, Op: Register(F.VReg));
1345 MIB.addReg(RegNo: ForwardedReg, Flags: RegState::Implicit);
1346 }
1347 }
1348
1349 // If we have -tailcallopt, we need to adjust the stack. We'll do the call
1350 // sequence start and end here.
1351 if (!IsSibCall) {
1352 MIB->getOperand(i: 1).setImm(FPDiff);
1353 CallSeqStart.addImm(Val: 0).addImm(Val: 0);
1354 // End the call sequence *before* emitting the call. Normally, we would
1355 // tidy the frame up after the call. However, here, we've laid out the
1356 // parameters so that when SP is reset, they will be in the correct
1357 // location.
1358 MIRBuilder.buildInstr(Opcode: AArch64::ADJCALLSTACKUP).addImm(Val: 0).addImm(Val: 0);
1359 }
1360
1361 // Now we can add the actual call instruction to the correct basic block.
1362 MIRBuilder.insertInstr(MIB);
1363
1364 // If Callee is a reg, since it is used by a target specific instruction,
1365 // it must have a register class matching the constraint of that instruction.
1366 if (MIB->getOperand(i: 0).isReg())
1367 constrainOperandRegClass(MF, TRI: *TRI, MRI, TII: *MF.getSubtarget().getInstrInfo(),
1368 RBI: *MF.getSubtarget().getRegBankInfo(), InsertPt&: *MIB,
1369 II: MIB->getDesc(), RegMO&: MIB->getOperand(i: 0), OpIdx: 0);
1370
1371 MF.getFrameInfo().setHasTailCall();
1372 Info.LoweredTailCall = true;
1373 return true;
1374}
1375
1376bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
1377 CallLoweringInfo &Info) const {
1378 MachineFunction &MF = MIRBuilder.getMF();
1379 const Function &F = MF.getFunction();
1380 MachineRegisterInfo &MRI = MF.getRegInfo();
1381 auto &DL = F.getDataLayout();
1382 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
1383 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1384
1385 // Arm64EC has extra requirements for varargs calls; bail out for now.
1386 //
1387 // Arm64EC has special mangling rules for calls; bail out on all calls for
1388 // now.
1389 if (Subtarget.isWindowsArm64EC())
1390 return false;
1391
1392 // Arm64EC thunks have a special calling convention which is only implemented
1393 // in SelectionDAG; bail out for now.
1394 if (Info.CallConv == CallingConv::ARM64EC_Thunk_Native ||
1395 Info.CallConv == CallingConv::ARM64EC_Thunk_X64)
1396 return false;
1397
1398 SmallVector<ArgInfo, 8> OutArgs;
1399 for (auto &OrigArg : Info.OrigArgs) {
1400 splitToValueTypes(OrigArgInfo: OrigArg, SplitArgs&: OutArgs, DL, CallConv: Info.CallConv);
1401 // AAPCS requires that we zero-extend i1 to 8 bits by the caller.
1402 auto &Flags = OrigArg.Flags[0];
1403 if (OrigArg.Ty->isIntegerTy(BitWidth: 1) && !Flags.isSExt() && !Flags.isZExt()) {
1404 ArgInfo &OutArg = OutArgs.back();
1405 assert(OutArg.Regs.size() == 1 &&
1406 MRI.getType(OutArg.Regs[0]).getSizeInBits() == 1 &&
1407 "Unexpected registers used for i1 arg");
1408
1409 // We cannot use a ZExt ArgInfo flag here, because it will
1410 // zero-extend the argument to i32 instead of just i8.
1411 OutArg.Regs[0] =
1412 MIRBuilder.buildZExt(Res: LLT::integer(SizeInBits: 8), Op: OutArg.Regs[0]).getReg(Idx: 0);
1413 LLVMContext &Ctx = MF.getFunction().getContext();
1414 OutArg.Ty = Type::getInt8Ty(C&: Ctx);
1415 }
1416 }
1417
1418 SmallVector<ArgInfo, 8> InArgs;
1419 if (!Info.OrigRet.Ty->isVoidTy())
1420 splitToValueTypes(OrigArgInfo: Info.OrigRet, SplitArgs&: InArgs, DL, CallConv: Info.CallConv);
1421
1422 // If we can lower as a tail call, do that instead.
1423 bool CanTailCallOpt =
1424 isEligibleForTailCallOptimization(MIRBuilder, Info, InArgs, OutArgs);
1425
1426 // We must emit a tail call if we have musttail.
1427 if (Info.IsMustTailCall && !CanTailCallOpt) {
1428 // There are types of incoming/outgoing arguments we can't handle yet, so
1429 // it doesn't make sense to actually die here like in ISelLowering. Instead,
1430 // fall back to SelectionDAG and let it try to handle this.
1431 LLVM_DEBUG(dbgs() << "Failed to lower musttail call as tail call\n");
1432 return false;
1433 }
1434
1435 Info.IsTailCall = CanTailCallOpt;
1436 if (CanTailCallOpt)
1437 return lowerTailCall(MIRBuilder, Info, OutArgs);
1438
1439 // Find out which ABI gets to decide where things go.
1440 CCAssignFn *AssignFnFixed;
1441 CCAssignFn *AssignFnVarArg;
1442 std::tie(args&: AssignFnFixed, args&: AssignFnVarArg) =
1443 getAssignFnsForCC(CC: Info.CallConv, TLI);
1444
1445 MachineInstrBuilder CallSeqStart;
1446 CallSeqStart = MIRBuilder.buildInstr(Opcode: AArch64::ADJCALLSTACKDOWN);
1447
1448 // Create a temporarily-floating call instruction so we can add the implicit
1449 // uses of arg registers.
1450
1451 unsigned Opc = 0;
1452 // Calls with operand bundle "clang.arc.attachedcall" are special. They should
1453 // be expanded to the call, directly followed by a special marker sequence and
1454 // a call to an ObjC library function.
1455 if (Info.CB && objcarc::hasAttachedCallOpBundle(CB: Info.CB))
1456 Opc = Info.PAI ? AArch64::BLRA_RVMARKER : AArch64::BLR_RVMARKER;
1457 // A call to a returns twice function like setjmp must be followed by a bti
1458 // instruction.
1459 else if (Info.CB && Info.CB->hasFnAttr(Kind: Attribute::ReturnsTwice) &&
1460 !Subtarget.noBTIAtReturnTwice() &&
1461 MF.getInfo<AArch64FunctionInfo>()->branchTargetEnforcement())
1462 Opc = AArch64::BLR_BTI;
1463 else {
1464 // For an intrinsic call (e.g. memset), use GOT if "RtLibUseGOT" (-fno-plt)
1465 // is set.
1466 if (Info.Callee.isSymbol() && F.getParent()->getRtLibUseGOT()) {
1467 auto MIB = MIRBuilder.buildInstr(Opcode: TargetOpcode::G_GLOBAL_VALUE);
1468 DstOp(getLLTForType(Ty&: *F.getType(), DL)).addDefToMIB(MRI, MIB);
1469 MIB.addExternalSymbol(FnName: Info.Callee.getSymbolName(), TargetFlags: AArch64II::MO_GOT);
1470 Info.Callee = MachineOperand::CreateReg(Reg: MIB.getReg(Idx: 0), isDef: false);
1471 }
1472 Opc = getCallOpcode(CallerF: MF, IsIndirect: Info.Callee.isReg(), IsTailCall: false, PAI&: Info.PAI, MRI);
1473 }
1474
1475 auto MIB = MIRBuilder.buildInstrNoInsert(Opcode: Opc);
1476 unsigned CalleeOpNo = 0;
1477
1478 if (Opc == AArch64::BLR_RVMARKER || Opc == AArch64::BLRA_RVMARKER) {
1479 // Add a target global address for the retainRV/claimRV runtime function
1480 // just before the call target.
1481 Function *ARCFn = *objcarc::getAttachedARCFunction(CB: Info.CB);
1482 MIB.addGlobalAddress(GV: ARCFn);
1483 ++CalleeOpNo;
1484
1485 // We may or may not need to emit both the marker and the retain/claim call.
1486 // Tell the pseudo expansion using an additional boolean op.
1487 MIB.addImm(Val: objcarc::attachedCallOpBundleNeedsMarker(CB: Info.CB));
1488 ++CalleeOpNo;
1489 } else if (Info.CFIType) {
1490 MIB->setCFIType(MF, Type: Info.CFIType->getZExtValue());
1491 }
1492 MIB->setDeactivationSymbol(MF, DS: Info.DeactivationSymbol);
1493
1494 MIB.add(MO: Info.Callee);
1495
1496 // Tell the call which registers are clobbered.
1497 const uint32_t *Mask;
1498 const auto *TRI = Subtarget.getRegisterInfo();
1499
1500 AArch64OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg,
1501 Subtarget, /*IsReturn*/ false);
1502 // Do the actual argument marshalling.
1503 OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, /*IsReturn*/ false);
1504 bool AssignedCallArgs = Info.CallConv == CallingConv::C &&
1505 tryAssignSimpleGPRCallArgs(MIRBuilder, MIB, Args: OutArgs);
1506 if (!AssignedCallArgs &&
1507 !determineAndHandleAssignments(Handler, Assigner, Args&: OutArgs, MIRBuilder,
1508 CallConv: Info.CallConv, IsVarArg: Info.IsVarArg))
1509 return false;
1510
1511 Mask = getMaskForArgs(OutArgs, Info, TRI: *TRI, MF);
1512
1513 if (Opc == AArch64::BLRA || Opc == AArch64::BLRA_RVMARKER) {
1514 assert((Info.PAI->Key == AArch64PACKey::IA ||
1515 Info.PAI->Key == AArch64PACKey::IB) &&
1516 "Invalid auth call key");
1517 MIB.addImm(Val: Info.PAI->Key);
1518
1519 Register AddrDisc = 0;
1520 uint16_t IntDisc = 0;
1521 std::tie(args&: IntDisc, args&: AddrDisc) =
1522 extractPtrauthBlendDiscriminators(Disc: Info.PAI->Discriminator, MRI);
1523
1524 MIB.addImm(Val: IntDisc);
1525 MIB.addUse(RegNo: AddrDisc);
1526 if (AddrDisc.isValid()) {
1527 constrainOperandRegClass(MF, TRI: *TRI, MRI, TII: *MF.getSubtarget().getInstrInfo(),
1528 RBI: *MF.getSubtarget().getRegBankInfo(), InsertPt&: *MIB,
1529 II: MIB->getDesc(), RegMO&: MIB->getOperand(i: CalleeOpNo + 3),
1530 OpIdx: CalleeOpNo + 3);
1531 }
1532 }
1533
1534 // Tell the call which registers are clobbered.
1535 if (MF.getSubtarget<AArch64Subtarget>().hasCustomCallingConv())
1536 TRI->UpdateCustomCallPreservedMask(MF, Mask: &Mask);
1537 MIB.addRegMask(Mask);
1538
1539 if (TRI->isAnyArgRegReserved(MF))
1540 TRI->emitReservedArgRegCallError(MF);
1541
1542 // Now we can add the actual call instruction to the correct basic block.
1543 MIRBuilder.insertInstr(MIB);
1544
1545 // Add dead flag to already inserted implicit-def.
1546 MIB->addRegisterDead(Reg: AArch64::LR, RegInfo: TRI);
1547
1548 uint64_t CalleePopBytes =
1549 doesCalleeRestoreStack(CallConv: Info.CallConv,
1550 TailCallOpt: MF.getTarget().Options.GuaranteedTailCallOpt)
1551 ? alignTo(Value: Assigner.StackSize, Align: 16)
1552 : 0;
1553
1554 CallSeqStart.addImm(Val: Assigner.StackSize).addImm(Val: 0);
1555 MIRBuilder.buildInstr(Opcode: AArch64::ADJCALLSTACKUP)
1556 .addImm(Val: Assigner.StackSize)
1557 .addImm(Val: CalleePopBytes);
1558
1559 // If Callee is a reg, since it is used by a target specific
1560 // instruction, it must have a register class matching the
1561 // constraint of that instruction.
1562 if (MIB->getOperand(i: CalleeOpNo).isReg())
1563 constrainOperandRegClass(MF, TRI: *TRI, MRI, TII: *Subtarget.getInstrInfo(),
1564 RBI: *Subtarget.getRegBankInfo(), InsertPt&: *MIB, II: MIB->getDesc(),
1565 RegMO&: MIB->getOperand(i: CalleeOpNo), OpIdx: CalleeOpNo);
1566
1567 // Finally we can copy the returned value back into its virtual-register. In
1568 // symmetry with the arguments, the physical register must be an
1569 // implicit-define of the call instruction.
1570 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy()) {
1571 CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(CC: Info.CallConv);
1572 CallReturnHandler Handler(MIRBuilder, MRI, MIB);
1573 bool UsingReturnedArg =
1574 !OutArgs.empty() && OutArgs[0].Flags[0].isReturned();
1575
1576 AArch64OutgoingValueAssigner Assigner(RetAssignFn, RetAssignFn, Subtarget,
1577 /*IsReturn*/ false);
1578 ReturnedArgCallReturnHandler ReturnedArgHandler(MIRBuilder, MRI, MIB);
1579 bool AssignedCallReturn =
1580 Info.CallConv == CallingConv::C && !UsingReturnedArg &&
1581 tryAssignSimpleGPRCallReturn(MIRBuilder, MIB, Rets: InArgs);
1582 if (!AssignedCallReturn &&
1583 !determineAndHandleAssignments(
1584 Handler&: UsingReturnedArg ? ReturnedArgHandler : Handler, Assigner, Args&: InArgs,
1585 MIRBuilder, CallConv: Info.CallConv, IsVarArg: Info.IsVarArg,
1586 ThisReturnRegs: UsingReturnedArg ? ArrayRef(OutArgs[0].Regs)
1587 : ArrayRef<Register>()))
1588 return false;
1589 }
1590
1591 if (Info.SwiftErrorVReg) {
1592 MIB.addDef(RegNo: AArch64::X21, Flags: RegState::Implicit);
1593 MIRBuilder.buildCopy(Res: Info.SwiftErrorVReg, Op: Register(AArch64::X21));
1594 }
1595
1596 if (!Info.CanLowerReturn) {
1597 insertSRetLoads(MIRBuilder, RetTy: Info.OrigRet.Ty, VRegs: Info.OrigRet.Regs,
1598 DemoteReg: Info.DemoteRegister, FI: Info.DemoteStackIndex);
1599 }
1600 return true;
1601}
1602
1603bool AArch64CallLowering::isTypeIsValidForThisReturn(EVT Ty) const {
1604 return Ty.getSizeInBits() == 64;
1605}
1606