1//===-- llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp - Call lowering -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the lowering of LLVM calls to machine code calls for
11/// GlobalISel.
12///
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPUCallLowering.h"
16#include "AMDGPU.h"
17#include "AMDGPULegalizerInfo.h"
18#include "SIMachineFunctionInfo.h"
19#include "SIRegisterInfo.h"
20#include "llvm/CodeGen/Analysis.h"
21#include "llvm/CodeGen/FunctionLoweringInfo.h"
22#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
23#include "llvm/CodeGen/MachineFrameInfo.h"
24#include "llvm/CodeGen/PseudoSourceValueManager.h"
25#include "llvm/IR/IntrinsicsAMDGPU.h"
26
27#define DEBUG_TYPE "amdgpu-call-lowering"
28
29using namespace llvm;
30
31namespace {
32
33/// Wrapper around extendRegister to ensure we extend to a full 32-bit register.
34static Register extendRegisterMin32(CallLowering::ValueHandler &Handler,
35 Register ValVReg, const CCValAssign &VA) {
36 if (VA.getLocVT().getSizeInBits() < 32) {
37 // 16-bit types are reported as legal for 32-bit registers. We need to
38 // extend and do a 32-bit copy to avoid the verifier complaining about it.
39 return Handler.MIRBuilder.buildAnyExt(Res: LLT::integer(SizeInBits: 32), Op: ValVReg).getReg(Idx: 0);
40 }
41
42 return Handler.extendRegister(ValReg: ValVReg, VA);
43}
44
45struct AMDGPUOutgoingValueHandler : public CallLowering::OutgoingValueHandler {
46 AMDGPUOutgoingValueHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI,
47 MachineInstrBuilder MIB)
48 : OutgoingValueHandler(B, MRI), MIB(MIB) {}
49
50 MachineInstrBuilder MIB;
51
52 Register getStackAddress(uint64_t Size, int64_t Offset,
53 MachinePointerInfo &MPO,
54 ISD::ArgFlagsTy Flags) override {
55 llvm_unreachable("not implemented");
56 }
57
58 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
59 const MachinePointerInfo &MPO,
60 const CCValAssign &VA) override {
61 llvm_unreachable("not implemented");
62 }
63
64 void assignValueToReg(Register ValVReg, Register PhysReg,
65 const CCValAssign &VA,
66 ISD::ArgFlagsTy Flags = {}) override {
67 Register ExtReg = extendRegisterMin32(Handler&: *this, ValVReg, VA);
68
69 // If this is a scalar return, insert a readfirstlane just in case the value
70 // ends up in a VGPR.
71 // FIXME: Assert this is a shader return.
72 const SIRegisterInfo *TRI
73 = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
74 if (TRI->isSGPRReg(MRI, Reg: PhysReg)) {
75 LLT Ty = MRI.getType(Reg: ExtReg);
76 LLT I32 = LLT::integer(SizeInBits: 32);
77 if (Ty != I32 && Ty != LLT::float32()) {
78 // FIXME: We should probably support readfirstlane intrinsics with all
79 // legal 32-bit types.
80 assert(Ty.getSizeInBits() == 32);
81 if (Ty.isPointer())
82 ExtReg = MIRBuilder.buildPtrToInt(Dst: I32, Src: ExtReg).getReg(Idx: 0);
83 else
84 ExtReg = MIRBuilder.buildBitcast(Dst: I32, Src: ExtReg).getReg(Idx: 0);
85 }
86
87 auto ToSGPR = MIRBuilder
88 .buildIntrinsic(ID: Intrinsic::amdgcn_readfirstlane,
89 Res: {MRI.getType(Reg: ExtReg)})
90 .addReg(RegNo: ExtReg);
91 ExtReg = ToSGPR.getReg(Idx: 0);
92 }
93
94 MIRBuilder.buildCopy(Res: PhysReg, Op: ExtReg);
95 MIB.addUse(RegNo: PhysReg, Flags: RegState::Implicit);
96 }
97};
98
99struct AMDGPUIncomingArgHandler : public CallLowering::IncomingValueHandler {
100 uint64_t StackUsed = 0;
101
102 AMDGPUIncomingArgHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI)
103 : IncomingValueHandler(B, MRI) {}
104
105 Register getStackAddress(uint64_t Size, int64_t Offset,
106 MachinePointerInfo &MPO,
107 ISD::ArgFlagsTy Flags) override {
108 auto &MFI = MIRBuilder.getMF().getFrameInfo();
109
110 // Byval is assumed to be writable memory, but other stack passed arguments
111 // are not.
112 const bool IsImmutable = !Flags.isByVal();
113 int FI = MFI.CreateFixedObject(Size, SPOffset: Offset, IsImmutable);
114 MPO = MachinePointerInfo::getFixedStack(MF&: MIRBuilder.getMF(), FI);
115 auto AddrReg = MIRBuilder.buildFrameIndex(
116 Res: LLT::pointer(AddressSpace: AMDGPUAS::PRIVATE_ADDRESS, SizeInBits: 32), Idx: FI);
117 StackUsed = std::max(a: StackUsed, b: Size + Offset);
118 return AddrReg.getReg(Idx: 0);
119 }
120
121 void copyToReg(Register ValVReg, Register PhysReg, const CCValAssign &VA) {
122 if (VA.getLocVT().getSizeInBits() < 32) {
123 // 16-bit types are reported as legal for 32-bit registers. We need to
124 // do a 32-bit copy, and truncate to avoid the verifier complaining
125 // about it.
126 auto Copy = MIRBuilder.buildCopy(Res: LLT::integer(SizeInBits: 32), Op: PhysReg);
127
128 // If we have signext/zeroext, it applies to the whole 32-bit register
129 // before truncation.
130 auto Extended =
131 buildExtensionHint(VA, SrcReg: Copy.getReg(Idx: 0), NarrowTy: LLT(VA.getLocVT()));
132 LLT ValTy = MRI.getType(Reg: ValVReg);
133 if (ValTy.isInteger()) {
134 MIRBuilder.buildTrunc(Res: ValVReg, Op: Extended);
135 } else {
136 auto Trunc = MIRBuilder.buildTrunc(Res: LLT::integer(SizeInBits: ValTy.getSizeInBits()),
137 Op: Extended);
138 MIRBuilder.buildBitcast(Dst: ValVReg, Src: Trunc);
139 }
140 return;
141 }
142
143 IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA);
144 }
145
146 void readLaneToSGPR(Register ValVReg, Register PhysReg,
147 const CCValAssign &VA) {
148 // Handle inreg parameters passed through VGPRs due to SGPR exhaustion.
149 // When SGPRs are exhausted, the calling convention may allocate inreg
150 // parameters to VGPRs. We insert readfirstlane to move the value from
151 // VGPR to SGPR, as required by the inreg ABI.
152 //
153 // FIXME: This may increase instruction count in some cases. If the
154 // readfirstlane result is subsequently copied back to a VGPR, we cannot
155 // optimize away the unnecessary VGPR->SGPR->VGPR sequence in later passes
156 // because the inreg attribute information is not preserved in MIR. We could
157 // use WWM_COPY (or similar instructions) and mark it as foldable to enable
158 // later optimization passes to eliminate the redundant readfirstlane.
159 if (VA.getLocVT().getSizeInBits() < 32) {
160 auto Copy = MIRBuilder.buildCopy(Res: LLT::integer(SizeInBits: 32), Op: PhysReg);
161 auto ToSGPR = MIRBuilder
162 .buildIntrinsic(ID: Intrinsic::amdgcn_readfirstlane,
163 Res: {MRI.getType(Reg: Copy.getReg(Idx: 0))})
164 .addReg(RegNo: Copy.getReg(Idx: 0));
165 auto Extended =
166 buildExtensionHint(VA, SrcReg: ToSGPR.getReg(Idx: 0), NarrowTy: LLT(VA.getLocVT()));
167 LLT ValTy = MRI.getType(Reg: ValVReg);
168 if (ValTy.isInteger()) {
169 MIRBuilder.buildTrunc(Res: ValVReg, Op: Extended);
170 } else {
171 auto Trunc = MIRBuilder.buildTrunc(Res: LLT::integer(SizeInBits: ValTy.getSizeInBits()),
172 Op: Extended);
173 MIRBuilder.buildBitcast(Dst: ValVReg, Src: Trunc);
174 }
175 return;
176 }
177
178 auto Copy = MIRBuilder.buildCopy(Res: MRI.getType(Reg: ValVReg), Op: PhysReg);
179 MIRBuilder.buildIntrinsic(ID: Intrinsic::amdgcn_readfirstlane, Res: ValVReg)
180 .addReg(RegNo: Copy.getReg(Idx: 0));
181 }
182
183 void assignValueToReg(Register ValVReg, Register PhysReg,
184 const CCValAssign &VA,
185 ISD::ArgFlagsTy Flags = {}) override {
186 markPhysRegUsed(PhysReg);
187
188 const SIRegisterInfo *TRI =
189 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
190
191 // Inreg flag should be the same across SplitArg[i]
192 if (Flags.isInReg() && TRI->isVGPR(MRI, Reg: PhysReg))
193 readLaneToSGPR(ValVReg, PhysReg, VA);
194 else
195 copyToReg(ValVReg, PhysReg, VA);
196 }
197
198 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
199 const MachinePointerInfo &MPO,
200 const CCValAssign &VA) override {
201 MachineFunction &MF = MIRBuilder.getMF();
202
203 auto *MMO = MF.getMachineMemOperand(
204 PtrInfo: MPO, f: MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, MemTy,
205 base_alignment: inferAlignFromPtrInfo(MF, MPO));
206 MIRBuilder.buildLoad(Res: ValVReg, Addr, MMO&: *MMO);
207 }
208
209 /// How the physical register gets marked varies between formal
210 /// parameters (it's a basic-block live-in), and a call instruction
211 /// (it's an implicit-def of the BL).
212 virtual void markPhysRegUsed(unsigned PhysReg) = 0;
213};
214
215struct FormalArgHandler : public AMDGPUIncomingArgHandler {
216 FormalArgHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI)
217 : AMDGPUIncomingArgHandler(B, MRI) {}
218
219 void markPhysRegUsed(unsigned PhysReg) override {
220 MIRBuilder.getMBB().addLiveIn(PhysReg);
221 }
222};
223
224struct CallReturnHandler : public AMDGPUIncomingArgHandler {
225 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
226 MachineInstrBuilder MIB)
227 : AMDGPUIncomingArgHandler(MIRBuilder, MRI), MIB(MIB) {}
228
229 void markPhysRegUsed(unsigned PhysReg) override {
230 MIB.addDef(RegNo: PhysReg, Flags: RegState::Implicit);
231 }
232
233 MachineInstrBuilder MIB;
234};
235
236struct AMDGPUOutgoingArgHandler : public AMDGPUOutgoingValueHandler {
237 /// For tail calls, the byte offset of the call's argument area from the
238 /// callee's. Unused elsewhere.
239 int FPDiff;
240
241 // Cache the SP register vreg if we need it more than once in this call site.
242 Register SPReg;
243
244 bool IsTailCall;
245
246 AMDGPUOutgoingArgHandler(MachineIRBuilder &MIRBuilder,
247 MachineRegisterInfo &MRI, MachineInstrBuilder MIB,
248 bool IsTailCall = false, int FPDiff = 0)
249 : AMDGPUOutgoingValueHandler(MIRBuilder, MRI, MIB), FPDiff(FPDiff),
250 IsTailCall(IsTailCall) {}
251
252 Register getStackAddress(uint64_t Size, int64_t Offset,
253 MachinePointerInfo &MPO,
254 ISD::ArgFlagsTy Flags) override {
255 MachineFunction &MF = MIRBuilder.getMF();
256 const LLT PtrTy = LLT::pointer(AddressSpace: AMDGPUAS::PRIVATE_ADDRESS, SizeInBits: 32);
257 const LLT I32 = LLT::integer(SizeInBits: 32);
258
259 if (IsTailCall) {
260 Offset += FPDiff;
261 int FI = MF.getFrameInfo().CreateFixedObject(Size, SPOffset: Offset, IsImmutable: true);
262 auto FIReg = MIRBuilder.buildFrameIndex(Res: PtrTy, Idx: FI);
263 MPO = MachinePointerInfo::getFixedStack(MF, FI);
264 return FIReg.getReg(Idx: 0);
265 }
266
267 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
268
269 if (!SPReg) {
270 const GCNSubtarget &ST = MIRBuilder.getMF().getSubtarget<GCNSubtarget>();
271 if (ST.hasFlatScratchEnabled()) {
272 // The stack is accessed unswizzled, so we can use a regular copy.
273 SPReg = MIRBuilder.buildCopy(Res: PtrTy,
274 Op: MFI->getStackPtrOffsetReg()).getReg(Idx: 0);
275 } else {
276 // The address we produce here, without knowing the use context, is going
277 // to be interpreted as a vector address, so we need to convert to a
278 // swizzled address.
279 SPReg = MIRBuilder.buildInstr(Opc: AMDGPU::G_AMDGPU_WAVE_ADDRESS, DstOps: {PtrTy},
280 SrcOps: {MFI->getStackPtrOffsetReg()}).getReg(Idx: 0);
281 }
282 }
283
284 auto OffsetReg = MIRBuilder.buildConstant(Res: I32, Val: Offset);
285
286 auto AddrReg = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: SPReg, Op1: OffsetReg);
287 MPO = MachinePointerInfo::getStack(MF, Offset);
288 return AddrReg.getReg(Idx: 0);
289 }
290
291 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
292 const MachinePointerInfo &MPO,
293 const CCValAssign &VA) override {
294 MachineFunction &MF = MIRBuilder.getMF();
295 uint64_t LocMemOffset = VA.getLocMemOffset();
296 const auto &ST = MF.getSubtarget<GCNSubtarget>();
297
298 auto *MMO = MF.getMachineMemOperand(
299 PtrInfo: MPO, f: MachineMemOperand::MOStore, MemTy,
300 base_alignment: commonAlignment(A: ST.getStackAlignment(), Offset: LocMemOffset));
301 MIRBuilder.buildStore(Val: ValVReg, Addr, MMO&: *MMO);
302 }
303
304 void assignValueToAddress(const CallLowering::ArgInfo &Arg,
305 unsigned ValRegIndex, Register Addr, LLT MemTy,
306 const MachinePointerInfo &MPO,
307 const CCValAssign &VA) override {
308 Register ValVReg = VA.getLocInfo() != CCValAssign::LocInfo::FPExt
309 ? extendRegister(ValReg: Arg.Regs[ValRegIndex], VA)
310 : Arg.Regs[ValRegIndex];
311 assignValueToAddress(ValVReg, Addr, MemTy, MPO, VA);
312 }
313};
314} // anonymous namespace
315
316AMDGPUCallLowering::AMDGPUCallLowering(const TargetLowering &TLI)
317 : CallLowering(&TLI) {}
318
319// FIXME: Compatibility shim
320static ISD::NodeType extOpcodeToISDExtOpcode(unsigned MIOpc) {
321 switch (MIOpc) {
322 case TargetOpcode::G_SEXT:
323 return ISD::SIGN_EXTEND;
324 case TargetOpcode::G_ZEXT:
325 return ISD::ZERO_EXTEND;
326 case TargetOpcode::G_ANYEXT:
327 return ISD::ANY_EXTEND;
328 default:
329 llvm_unreachable("not an extend opcode");
330 }
331}
332
333bool AMDGPUCallLowering::canLowerReturn(MachineFunction &MF,
334 CallingConv::ID CallConv,
335 SmallVectorImpl<BaseArgInfo> &Outs,
336 bool IsVarArg) const {
337 // For shaders. Vector types should be explicitly handled by CC.
338 if (AMDGPU::isEntryFunctionCC(CC: CallConv))
339 return true;
340
341 SmallVector<CCValAssign, 16> ArgLocs;
342 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
343 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs,
344 MF.getFunction().getContext());
345
346 return checkReturn(CCInfo, Outs, Fn: TLI.CCAssignFnForReturn(CC: CallConv, IsVarArg));
347}
348
349/// Lower the return value for the already existing \p Ret. This assumes that
350/// \p B's insertion point is correct.
351bool AMDGPUCallLowering::lowerReturnVal(MachineIRBuilder &B,
352 const Value *Val, ArrayRef<Register> VRegs,
353 MachineInstrBuilder &Ret) const {
354 if (!Val)
355 return true;
356
357 auto &MF = B.getMF();
358 const auto &F = MF.getFunction();
359 const DataLayout &DL = MF.getDataLayout();
360 MachineRegisterInfo *MRI = B.getMRI();
361 LLVMContext &Ctx = F.getContext();
362
363 CallingConv::ID CC = F.getCallingConv();
364 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
365
366 SmallVector<EVT, 8> SplitEVTs;
367 ComputeValueVTs(TLI, DL, Ty: Val->getType(), ValueVTs&: SplitEVTs);
368 assert(VRegs.size() == SplitEVTs.size() &&
369 "For each split Type there should be exactly one VReg.");
370
371 SmallVector<ArgInfo, 8> SplitRetInfos;
372
373 for (unsigned i = 0; i < SplitEVTs.size(); ++i) {
374 EVT VT = SplitEVTs[i];
375 Register Reg = VRegs[i];
376 ArgInfo RetInfo(Reg, VT.getTypeForEVT(Context&: Ctx), 0);
377 setArgFlags(Arg&: RetInfo, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: F);
378
379 if (VT.isScalarInteger()) {
380 unsigned ExtendOp = TargetOpcode::G_ANYEXT;
381 if (RetInfo.Flags[0].isSExt()) {
382 assert(RetInfo.Regs.size() == 1 && "expect only simple return values");
383 ExtendOp = TargetOpcode::G_SEXT;
384 } else if (RetInfo.Flags[0].isZExt()) {
385 assert(RetInfo.Regs.size() == 1 && "expect only simple return values");
386 ExtendOp = TargetOpcode::G_ZEXT;
387 }
388
389 EVT ExtVT = TLI.getTypeForExtReturn(Context&: Ctx, VT,
390 ExtendKind: extOpcodeToISDExtOpcode(MIOpc: ExtendOp));
391 if (ExtVT != VT) {
392 RetInfo.Ty = ExtVT.getTypeForEVT(Context&: Ctx);
393 LLT ExtTy = getLLTForType(Ty&: *RetInfo.Ty, DL);
394 Reg = B.buildInstr(Opc: ExtendOp, DstOps: {ExtTy}, SrcOps: {Reg}).getReg(Idx: 0);
395 }
396 }
397
398 if (Reg != RetInfo.Regs[0]) {
399 RetInfo.Regs[0] = Reg;
400 // Reset the arg flags after modifying Reg.
401 setArgFlags(Arg&: RetInfo, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: F);
402 }
403
404 splitToValueTypes(OrigArgInfo: RetInfo, SplitArgs&: SplitRetInfos, DL, CallConv: CC);
405 }
406
407 CCAssignFn *AssignFn = TLI.CCAssignFnForReturn(CC, IsVarArg: F.isVarArg());
408
409 OutgoingValueAssigner Assigner(AssignFn);
410 AMDGPUOutgoingValueHandler RetHandler(B, *MRI, Ret);
411 return determineAndHandleAssignments(Handler&: RetHandler, Assigner, Args&: SplitRetInfos, MIRBuilder&: B,
412 CallConv: CC, IsVarArg: F.isVarArg());
413}
414
415bool AMDGPUCallLowering::lowerReturn(MachineIRBuilder &B, const Value *Val,
416 ArrayRef<Register> VRegs,
417 FunctionLoweringInfo &FLI) const {
418
419 MachineFunction &MF = B.getMF();
420 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
421 MFI->setIfReturnsVoid(!Val);
422
423 assert(!Val == VRegs.empty() && "Return value without a vreg");
424
425 CallingConv::ID CC = B.getMF().getFunction().getCallingConv();
426 const bool IsShader = AMDGPU::isShader(CC);
427 const bool IsWaveEnd =
428 (IsShader && MFI->returnsVoid()) || AMDGPU::isKernel(CC);
429 if (IsWaveEnd) {
430 B.buildInstr(Opcode: AMDGPU::S_ENDPGM)
431 .addImm(Val: 0);
432 return true;
433 }
434
435 const bool IsWholeWave = MFI->isWholeWaveFunction();
436 unsigned ReturnOpc = IsWholeWave ? AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_RETURN
437 : IsShader ? AMDGPU::SI_RETURN_TO_EPILOG
438 : AMDGPU::SI_RETURN;
439 auto Ret = B.buildInstrNoInsert(Opcode: ReturnOpc);
440
441 if (!FLI.CanLowerReturn)
442 insertSRetStores(MIRBuilder&: B, RetTy: Val->getType(), VRegs, DemoteReg: FLI.DemoteRegister);
443 else if (!lowerReturnVal(B, Val, VRegs, Ret))
444 return false;
445
446 if (IsWholeWave)
447 addOriginalExecToReturn(MF&: B.getMF(), Ret);
448
449 // TODO: Handle CalleeSavedRegsViaCopy.
450
451 B.insertInstr(MIB: Ret);
452 return true;
453}
454
455void AMDGPUCallLowering::lowerParameterPtr(Register DstReg, MachineIRBuilder &B,
456 uint64_t Offset) const {
457 MachineFunction &MF = B.getMF();
458 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
459 MachineRegisterInfo &MRI = MF.getRegInfo();
460 Register KernArgSegmentPtr =
461 MFI->getPreloadedReg(Value: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
462 Register KernArgSegmentVReg = MRI.getLiveInVirtReg(PReg: KernArgSegmentPtr);
463
464 auto OffsetReg = B.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: Offset);
465
466 B.buildPtrAdd(Res: DstReg, Op0: KernArgSegmentVReg, Op1: OffsetReg);
467}
468
469void AMDGPUCallLowering::lowerParameter(MachineIRBuilder &B, ArgInfo &OrigArg,
470 uint64_t Offset,
471 Align Alignment) const {
472 MachineFunction &MF = B.getMF();
473 const Function &F = MF.getFunction();
474 const DataLayout &DL = F.getDataLayout();
475 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
476 MachinePointerInfo PtrInfo = TLI.getKernargSegmentPtrInfo(MF);
477
478 LLT PtrTy = LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64);
479
480 SmallVector<ArgInfo, 32> SplitArgs;
481 SmallVector<TypeSize> FieldOffsets;
482 splitToValueTypes(OrigArgInfo: OrigArg, SplitArgs, DL, CallConv: F.getCallingConv(), Offsets: &FieldOffsets);
483
484 unsigned Idx = 0;
485 for (ArgInfo &SplitArg : SplitArgs) {
486 Register PtrReg = B.getMRI()->createGenericVirtualRegister(Ty: PtrTy);
487 lowerParameterPtr(DstReg: PtrReg, B, Offset: Offset + FieldOffsets[Idx]);
488
489 LLT ArgTy = getLLTForType(Ty&: *SplitArg.Ty, DL);
490 if (SplitArg.Flags[0].isPointer()) {
491 // Compensate for losing pointeriness in splitValueTypes.
492 LLT PtrTy = LLT::pointer(AddressSpace: SplitArg.Flags[0].getPointerAddrSpace(),
493 SizeInBits: ArgTy.getScalarSizeInBits());
494 ArgTy = ArgTy.isVector() ? LLT::vector(EC: ArgTy.getElementCount(), ScalarTy: PtrTy)
495 : PtrTy;
496 }
497
498 MachineMemOperand *MMO = MF.getMachineMemOperand(
499 PtrInfo,
500 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
501 MachineMemOperand::MOInvariant,
502 MemTy: ArgTy, base_alignment: commonAlignment(A: Alignment, Offset: FieldOffsets[Idx]));
503
504 assert(SplitArg.Regs.size() == 1);
505
506 B.buildLoad(Res: SplitArg.Regs[0], Addr: PtrReg, MMO&: *MMO);
507 ++Idx;
508 }
509}
510
511// Allocate special inputs passed in user SGPRs.
512static void allocateHSAUserSGPRs(CCState &CCInfo,
513 MachineIRBuilder &B,
514 MachineFunction &MF,
515 const SIRegisterInfo &TRI,
516 SIMachineFunctionInfo &Info) {
517 // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
518 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
519 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
520 Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
521 MF.addLiveIn(PReg: PrivateSegmentBufferReg, RC: &AMDGPU::SGPR_128RegClass);
522 CCInfo.AllocateReg(Reg: PrivateSegmentBufferReg);
523 }
524
525 if (UserSGPRInfo.hasDispatchPtr()) {
526 Register DispatchPtrReg = Info.addDispatchPtr(TRI);
527 MF.addLiveIn(PReg: DispatchPtrReg, RC: &AMDGPU::SGPR_64RegClass);
528 CCInfo.AllocateReg(Reg: DispatchPtrReg);
529 }
530
531 if (UserSGPRInfo.hasQueuePtr()) {
532 Register QueuePtrReg = Info.addQueuePtr(TRI);
533 MF.addLiveIn(PReg: QueuePtrReg, RC: &AMDGPU::SGPR_64RegClass);
534 CCInfo.AllocateReg(Reg: QueuePtrReg);
535 }
536
537 if (UserSGPRInfo.hasKernargSegmentPtr()) {
538 MachineRegisterInfo &MRI = MF.getRegInfo();
539 Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
540 const LLT P4 = LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64);
541 Register VReg = MRI.createGenericVirtualRegister(Ty: P4);
542 MRI.addLiveIn(Reg: InputPtrReg, vreg: VReg);
543 B.getMBB().addLiveIn(PhysReg: InputPtrReg);
544 B.buildCopy(Res: VReg, Op: InputPtrReg);
545 CCInfo.AllocateReg(Reg: InputPtrReg);
546 }
547
548 if (UserSGPRInfo.hasDispatchID()) {
549 Register DispatchIDReg = Info.addDispatchID(TRI);
550 MF.addLiveIn(PReg: DispatchIDReg, RC: &AMDGPU::SGPR_64RegClass);
551 CCInfo.AllocateReg(Reg: DispatchIDReg);
552 }
553
554 if (UserSGPRInfo.hasFlatScratchInit()) {
555 Register FlatScratchInitReg = Info.addFlatScratchInit(TRI);
556 MF.addLiveIn(PReg: FlatScratchInitReg, RC: &AMDGPU::SGPR_64RegClass);
557 CCInfo.AllocateReg(Reg: FlatScratchInitReg);
558 }
559
560 if (UserSGPRInfo.hasPrivateSegmentSize()) {
561 Register PrivateSegmentSizeReg = Info.addPrivateSegmentSize(TRI);
562 MF.addLiveIn(PReg: PrivateSegmentSizeReg, RC: &AMDGPU::SGPR_32RegClass);
563 CCInfo.AllocateReg(Reg: PrivateSegmentSizeReg);
564 }
565
566 // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
567 // these from the dispatch pointer.
568}
569
570bool AMDGPUCallLowering::lowerFormalArgumentsKernel(
571 MachineIRBuilder &B, const Function &F,
572 ArrayRef<ArrayRef<Register>> VRegs) const {
573 MachineFunction &MF = B.getMF();
574 const GCNSubtarget *Subtarget = &MF.getSubtarget<GCNSubtarget>();
575 MachineRegisterInfo &MRI = MF.getRegInfo();
576 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
577 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
578 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
579 const DataLayout &DL = F.getDataLayout();
580
581 SmallVector<CCValAssign, 16> ArgLocs;
582 CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());
583
584 allocateHSAUserSGPRs(CCInfo, B, MF, TRI: *TRI, Info&: *Info);
585
586 unsigned i = 0;
587 const Align KernArgBaseAlign(16);
588 const unsigned BaseOffset = Subtarget->getExplicitKernelArgOffset();
589 uint64_t ExplicitArgOffset = 0;
590
591 // TODO: Align down to dword alignment and extract bits for extending loads.
592 for (auto &Arg : F.args()) {
593 // TODO: Add support for kernarg preload.
594 if (Arg.hasAttribute(Kind: "amdgpu-hidden-argument")) {
595 LLVM_DEBUG(dbgs() << "Preloading hidden arguments is not supported\n");
596 return false;
597 }
598
599 const bool IsByRef = Arg.hasByRefAttr();
600 Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();
601 unsigned AllocSize = DL.getTypeAllocSize(Ty: ArgTy);
602 if (AllocSize == 0)
603 continue;
604
605 MaybeAlign ParamAlign = IsByRef ? Arg.getParamAlign() : std::nullopt;
606 Align ABIAlign = DL.getValueOrABITypeAlignment(Alignment: ParamAlign, Ty: ArgTy);
607
608 uint64_t ArgOffset = alignTo(Size: ExplicitArgOffset, A: ABIAlign) + BaseOffset;
609 ExplicitArgOffset = alignTo(Size: ExplicitArgOffset, A: ABIAlign) + AllocSize;
610
611 if (Arg.use_empty()) {
612 ++i;
613 continue;
614 }
615
616 Align Alignment = commonAlignment(A: KernArgBaseAlign, Offset: ArgOffset);
617
618 if (IsByRef) {
619 unsigned ByRefAS = cast<PointerType>(Val: Arg.getType())->getAddressSpace();
620
621 assert(VRegs[i].size() == 1 &&
622 "expected only one register for byval pointers");
623 if (ByRefAS == AMDGPUAS::CONSTANT_ADDRESS) {
624 lowerParameterPtr(DstReg: VRegs[i][0], B, Offset: ArgOffset);
625 } else {
626 const LLT ConstPtrTy = LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64);
627 Register PtrReg = MRI.createGenericVirtualRegister(Ty: ConstPtrTy);
628 lowerParameterPtr(DstReg: PtrReg, B, Offset: ArgOffset);
629
630 B.buildAddrSpaceCast(Dst: VRegs[i][0], Src: PtrReg);
631 }
632 } else {
633 ArgInfo OrigArg(VRegs[i], Arg, i);
634 const unsigned OrigArgIdx = i + AttributeList::FirstArgIndex;
635 setArgFlags(Arg&: OrigArg, OpIdx: OrigArgIdx, DL, FuncInfo: F);
636 lowerParameter(B, OrigArg, Offset: ArgOffset, Alignment);
637 }
638
639 ++i;
640 }
641
642 if (Info->getNumKernargPreloadedSGPRs())
643 Info->setNumWaveDispatchSGPRs(Info->getNumUserSGPRs());
644
645 TLI.allocateSpecialEntryInputVGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
646 TLI.allocateSystemSGPRs(CCInfo, MF, Info&: *Info, CallConv: F.getCallingConv(), IsShader: false);
647 return true;
648}
649
650bool AMDGPUCallLowering::lowerFormalArguments(
651 MachineIRBuilder &B, const Function &F, ArrayRef<ArrayRef<Register>> VRegs,
652 FunctionLoweringInfo &FLI) const {
653 CallingConv::ID CC = F.getCallingConv();
654
655 // The infrastructure for normal calling convention lowering is essentially
656 // useless for kernels. We want to avoid any kind of legalization or argument
657 // splitting.
658 if (CC == CallingConv::AMDGPU_KERNEL)
659 return lowerFormalArgumentsKernel(B, F, VRegs);
660
661 const bool IsGraphics = AMDGPU::isGraphics(CC);
662 const bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CC);
663
664 MachineFunction &MF = B.getMF();
665 MachineBasicBlock &MBB = B.getMBB();
666 MachineRegisterInfo &MRI = MF.getRegInfo();
667 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
668 const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();
669 const SIRegisterInfo *TRI = Subtarget.getRegisterInfo();
670 const DataLayout &DL = F.getDataLayout();
671
672 SmallVector<CCValAssign, 16> ArgLocs;
673 CCState CCInfo(CC, F.isVarArg(), MF, ArgLocs, F.getContext());
674 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info->getUserSGPRInfo();
675
676 if (UserSGPRInfo.hasImplicitBufferPtr()) {
677 Register ImplicitBufferPtrReg = Info->addImplicitBufferPtr(TRI: *TRI);
678 MF.addLiveIn(PReg: ImplicitBufferPtrReg, RC: &AMDGPU::SGPR_64RegClass);
679 CCInfo.AllocateReg(Reg: ImplicitBufferPtrReg);
680 }
681
682 // FIXME: This probably isn't defined for mesa
683 if (UserSGPRInfo.hasFlatScratchInit() && !Subtarget.isAmdPalOS()) {
684 Register FlatScratchInitReg = Info->addFlatScratchInit(TRI: *TRI);
685 MF.addLiveIn(PReg: FlatScratchInitReg, RC: &AMDGPU::SGPR_64RegClass);
686 CCInfo.AllocateReg(Reg: FlatScratchInitReg);
687 }
688
689 SmallVector<ArgInfo, 32> SplitArgs;
690 unsigned Idx = 0;
691 unsigned PSInputNum = 0;
692
693 // Insert the hidden sret parameter if the return value won't fit in the
694 // return registers.
695 if (!FLI.CanLowerReturn)
696 insertSRetIncomingArgument(F, SplitArgs, DemoteReg&: FLI.DemoteRegister, MRI, DL);
697
698 for (auto &Arg : F.args()) {
699 if (DL.getTypeStoreSize(Ty: Arg.getType()) == 0)
700 continue;
701
702 if (Info->isWholeWaveFunction() && Idx == 0) {
703 assert(VRegs[Idx].size() == 1 && "Expected only one register");
704
705 // The first argument for whole wave functions is the original EXEC value.
706 B.buildInstr(Opcode: AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)
707 .addDef(RegNo: VRegs[Idx][0]);
708
709 ++Idx;
710 continue;
711 }
712
713 const bool InReg = Arg.hasAttribute(Kind: Attribute::InReg);
714
715 if (Arg.hasAttribute(Kind: Attribute::SwiftSelf) ||
716 Arg.hasAttribute(Kind: Attribute::SwiftError) ||
717 Arg.hasAttribute(Kind: Attribute::Nest))
718 return false;
719
720 if (CC == CallingConv::AMDGPU_PS && !InReg && PSInputNum <= 15) {
721 const bool ArgUsed = !Arg.use_empty();
722 bool SkipArg = !ArgUsed && !Info->isPSInputAllocated(Index: PSInputNum);
723
724 if (!SkipArg) {
725 Info->markPSInputAllocated(Index: PSInputNum);
726 if (ArgUsed)
727 Info->markPSInputEnabled(Index: PSInputNum);
728 }
729
730 ++PSInputNum;
731
732 if (SkipArg) {
733 for (Register R : VRegs[Idx])
734 B.buildUndef(Res: R);
735
736 ++Idx;
737 continue;
738 }
739 }
740
741 ArgInfo OrigArg(VRegs[Idx], Arg, Idx);
742 const unsigned OrigArgIdx = Idx + AttributeList::FirstArgIndex;
743 setArgFlags(Arg&: OrigArg, OpIdx: OrigArgIdx, DL, FuncInfo: F);
744
745 splitToValueTypes(OrigArgInfo: OrigArg, SplitArgs, DL, CallConv: CC);
746 ++Idx;
747 }
748
749 // At least one interpolation mode must be enabled or else the GPU will
750 // hang.
751 //
752 // Check PSInputAddr instead of PSInputEnable. The idea is that if the user
753 // set PSInputAddr, the user wants to enable some bits after the compilation
754 // based on run-time states. Since we can't know what the final PSInputEna
755 // will look like, so we shouldn't do anything here and the user should take
756 // responsibility for the correct programming.
757 //
758 // Otherwise, the following restrictions apply:
759 // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
760 // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
761 // enabled too.
762 if (CC == CallingConv::AMDGPU_PS) {
763 if ((Info->getPSInputAddr() & 0x7F) == 0 ||
764 ((Info->getPSInputAddr() & 0xF) == 0 &&
765 Info->isPSInputAllocated(Index: 11))) {
766 CCInfo.AllocateReg(Reg: AMDGPU::VGPR0);
767 CCInfo.AllocateReg(Reg: AMDGPU::VGPR1);
768 Info->markPSInputAllocated(Index: 0);
769 Info->markPSInputEnabled(Index: 0);
770 }
771
772 if (Subtarget.isAmdPalOS()) {
773 // For isAmdPalOS, the user does not enable some bits after compilation
774 // based on run-time states; the register values being generated here are
775 // the final ones set in hardware. Therefore we need to apply the
776 // workaround to PSInputAddr and PSInputEnable together. (The case where
777 // a bit is set in PSInputAddr but not PSInputEnable is where the frontend
778 // set up an input arg for a particular interpolation mode, but nothing
779 // uses that input arg. Really we should have an earlier pass that removes
780 // such an arg.)
781 unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
782 if ((PsInputBits & 0x7F) == 0 ||
783 ((PsInputBits & 0xF) == 0 &&
784 (PsInputBits >> 11 & 1)))
785 Info->markPSInputEnabled(Index: llvm::countr_zero(Val: Info->getPSInputAddr()));
786 }
787 }
788
789 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
790 CCAssignFn *AssignFn = TLI.CCAssignFnForCall(CC, IsVarArg: F.isVarArg());
791
792 if (!MBB.empty())
793 B.setInstr(*MBB.begin());
794
795 if (!IsEntryFunc && !IsGraphics) {
796 // For the fixed ABI, pass workitem IDs in the last argument register.
797 TLI.allocateSpecialInputVGPRsFixed(CCInfo, MF, TRI: *TRI, Info&: *Info);
798
799 if (!Subtarget.hasFlatScratchEnabled())
800 CCInfo.AllocateReg(Reg: Info->getScratchRSrcReg());
801 TLI.allocateSpecialInputSGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
802 }
803
804 IncomingValueAssigner Assigner(AssignFn);
805 if (!determineAssignments(Assigner, Args&: SplitArgs, CCInfo))
806 return false;
807
808 if (IsEntryFunc) {
809 // This assumes the registers are allocated by CCInfo in ascending order
810 // with no gaps.
811 Info->setNumWaveDispatchSGPRs(
812 CCInfo.getFirstUnallocated(Regs: AMDGPU::SGPR_32RegClass.getRegisters()));
813 Info->setNumWaveDispatchVGPRs(
814 CCInfo.getFirstUnallocated(Regs: AMDGPU::VGPR_32RegClass.getRegisters()));
815 }
816
817 FormalArgHandler Handler(B, MRI);
818 if (!handleAssignments(Handler, Args&: SplitArgs, CCState&: CCInfo, ArgLocs, MIRBuilder&: B))
819 return false;
820
821 uint64_t StackSize = Assigner.StackSize;
822
823 // Start adding system SGPRs.
824 if (IsEntryFunc)
825 TLI.allocateSystemSGPRs(CCInfo, MF, Info&: *Info, CallConv: CC, IsShader: IsGraphics);
826
827 // When we tail call, we need to check if the callee's arguments will fit on
828 // the caller's stack. So, whenever we lower formal arguments, we should keep
829 // track of this information, since we might lower a tail call in this
830 // function later.
831 Info->setBytesInStackArgArea(StackSize);
832
833 // Move back to the end of the basic block.
834 B.setMBB(MBB);
835
836 return true;
837}
838
839bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder,
840 CCState &CCInfo,
841 SmallVectorImpl<std::pair<MCRegister, Register>> &ArgRegs,
842 CallLoweringInfo &Info) const {
843 MachineFunction &MF = MIRBuilder.getMF();
844
845 // If there's no call site, this doesn't correspond to a call from the IR and
846 // doesn't need implicit inputs.
847 if (!Info.CB)
848 return true;
849
850 const AMDGPUFunctionArgInfo &CalleeArgInfo =
851 AMDGPUFunctionArgInfo::FixedABIFunctionInfo;
852
853 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
854 const AMDGPUFunctionArgInfo &CallerArgInfo = MFI->getArgInfo();
855
856
857 // TODO: Unify with private memory register handling. This is complicated by
858 // the fact that at least in kernels, the input argument is not necessarily
859 // in the same location as the input.
860 AMDGPUFunctionArgInfo::PreloadedValue InputRegs[] = {
861 AMDGPUFunctionArgInfo::DISPATCH_PTR,
862 AMDGPUFunctionArgInfo::QUEUE_PTR,
863 AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR,
864 AMDGPUFunctionArgInfo::DISPATCH_ID,
865 AMDGPUFunctionArgInfo::WORKGROUP_ID_X,
866 AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,
867 AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,
868 AMDGPUFunctionArgInfo::LDS_KERNEL_ID,
869 };
870
871 static constexpr StringLiteral ImplicitAttrNames[][2] = {
872 {"amdgpu-no-dispatch-ptr", ""},
873 {"amdgpu-no-queue-ptr", ""},
874 {"amdgpu-no-implicitarg-ptr", ""},
875 {"amdgpu-no-dispatch-id", ""},
876 {"amdgpu-no-workgroup-id-x", "amdgpu-no-cluster-id-x"},
877 {"amdgpu-no-workgroup-id-y", "amdgpu-no-cluster-id-y"},
878 {"amdgpu-no-workgroup-id-z", "amdgpu-no-cluster-id-z"},
879 {"amdgpu-no-lds-kernel-id", ""},
880 };
881
882 MachineRegisterInfo &MRI = MF.getRegInfo();
883
884 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
885 const AMDGPULegalizerInfo *LI
886 = static_cast<const AMDGPULegalizerInfo*>(ST.getLegalizerInfo());
887
888 unsigned I = 0;
889 for (auto InputID : InputRegs) {
890 const ArgDescriptor *OutgoingArg;
891 const TargetRegisterClass *ArgRC;
892 LLT ArgTy;
893
894 // If the callee does not use the attribute value, skip copying the value.
895 if (all_of(Range: ImplicitAttrNames[I++], P: [&](StringRef AttrName) {
896 return AttrName.empty() || Info.CB->hasFnAttr(Kind: AttrName);
897 }))
898 continue;
899
900 std::tie(args&: OutgoingArg, args&: ArgRC, args&: ArgTy) =
901 CalleeArgInfo.getPreloadedValue(Value: InputID);
902 if (!OutgoingArg)
903 continue;
904
905 const ArgDescriptor *IncomingArg;
906 const TargetRegisterClass *IncomingArgRC;
907 std::tie(args&: IncomingArg, args&: IncomingArgRC, args&: ArgTy) =
908 CallerArgInfo.getPreloadedValue(Value: InputID);
909 assert(IncomingArgRC == ArgRC);
910
911 Register InputReg = MRI.createGenericVirtualRegister(Ty: ArgTy);
912
913 if (IncomingArg) {
914 LI->buildLoadInputValue(DstReg: InputReg, B&: MIRBuilder, Arg: IncomingArg, ArgRC, ArgTy);
915 } else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) {
916 LI->getImplicitArgPtr(DstReg: InputReg, MRI, B&: MIRBuilder);
917 } else if (InputID == AMDGPUFunctionArgInfo::LDS_KERNEL_ID) {
918 std::optional<uint32_t> Id =
919 AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F: MF.getFunction());
920 if (Id) {
921 MIRBuilder.buildConstant(Res: InputReg, Val: *Id);
922 } else {
923 MIRBuilder.buildUndef(Res: InputReg);
924 }
925 } else {
926 // We may have proven the input wasn't needed, although the ABI is
927 // requiring it. We just need to allocate the register appropriately.
928 MIRBuilder.buildUndef(Res: InputReg);
929 }
930
931 if (OutgoingArg->isRegister()) {
932 ArgRegs.emplace_back(Args: OutgoingArg->getRegister(), Args&: InputReg);
933 if (!CCInfo.AllocateReg(Reg: OutgoingArg->getRegister()))
934 report_fatal_error(reason: "failed to allocate implicit input argument");
935 } else {
936 LLVM_DEBUG(dbgs() << "Unhandled stack passed implicit input argument\n");
937 return false;
938 }
939 }
940
941 // Pack workitem IDs into a single register or pass it as is if already
942 // packed.
943 const ArgDescriptor *OutgoingArg;
944 const TargetRegisterClass *ArgRC;
945 LLT ArgTy;
946
947 std::tie(args&: OutgoingArg, args&: ArgRC, args&: ArgTy) =
948 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_X);
949 if (!OutgoingArg)
950 std::tie(args&: OutgoingArg, args&: ArgRC, args&: ArgTy) =
951 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
952 if (!OutgoingArg)
953 std::tie(args&: OutgoingArg, args&: ArgRC, args&: ArgTy) =
954 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
955 if (!OutgoingArg)
956 return false;
957
958 auto WorkitemIDX =
959 CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_X);
960 auto WorkitemIDY =
961 CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
962 auto WorkitemIDZ =
963 CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
964
965 const ArgDescriptor *IncomingArgX = std::get<0>(t&: WorkitemIDX);
966 const ArgDescriptor *IncomingArgY = std::get<0>(t&: WorkitemIDY);
967 const ArgDescriptor *IncomingArgZ = std::get<0>(t&: WorkitemIDZ);
968 const LLT I32 = LLT::integer(SizeInBits: 32);
969
970 const bool NeedWorkItemIDX = !Info.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-x");
971 const bool NeedWorkItemIDY = !Info.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-y");
972 const bool NeedWorkItemIDZ = !Info.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-z");
973
974 // If incoming ids are not packed we need to pack them.
975 // FIXME: Should consider known workgroup size to eliminate known 0 cases.
976 Register InputReg;
977 if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo.WorkItemIDX &&
978 NeedWorkItemIDX) {
979 if (ST.getMaxWorkitemID(Kernel: MF.getFunction(), Dimension: 0) != 0) {
980 InputReg = MRI.createGenericVirtualRegister(Ty: I32);
981 LI->buildLoadInputValue(DstReg: InputReg, B&: MIRBuilder, Arg: IncomingArgX,
982 ArgRC: std::get<1>(t&: WorkitemIDX),
983 ArgTy: std::get<2>(t&: WorkitemIDX));
984 } else {
985 InputReg = MIRBuilder.buildConstant(Res: I32, Val: 0).getReg(Idx: 0);
986 }
987 }
988
989 if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo.WorkItemIDY &&
990 NeedWorkItemIDY && ST.getMaxWorkitemID(Kernel: MF.getFunction(), Dimension: 1) != 0) {
991 Register Y = MRI.createGenericVirtualRegister(Ty: I32);
992 LI->buildLoadInputValue(DstReg: Y, B&: MIRBuilder, Arg: IncomingArgY,
993 ArgRC: std::get<1>(t&: WorkitemIDY), ArgTy: std::get<2>(t&: WorkitemIDY));
994
995 Y = MIRBuilder.buildShl(Dst: I32, Src0: Y, Src1: MIRBuilder.buildConstant(Res: I32, Val: 10))
996 .getReg(Idx: 0);
997 InputReg = InputReg ? MIRBuilder.buildOr(Dst: I32, Src0: InputReg, Src1: Y).getReg(Idx: 0) : Y;
998 }
999
1000 if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo.WorkItemIDZ &&
1001 NeedWorkItemIDZ && ST.getMaxWorkitemID(Kernel: MF.getFunction(), Dimension: 2) != 0) {
1002 Register Z = MRI.createGenericVirtualRegister(Ty: I32);
1003 LI->buildLoadInputValue(DstReg: Z, B&: MIRBuilder, Arg: IncomingArgZ,
1004 ArgRC: std::get<1>(t&: WorkitemIDZ), ArgTy: std::get<2>(t&: WorkitemIDZ));
1005
1006 Z = MIRBuilder.buildShl(Dst: I32, Src0: Z, Src1: MIRBuilder.buildConstant(Res: I32, Val: 20))
1007 .getReg(Idx: 0);
1008 InputReg = InputReg ? MIRBuilder.buildOr(Dst: I32, Src0: InputReg, Src1: Z).getReg(Idx: 0) : Z;
1009 }
1010
1011 if (!InputReg &&
1012 (NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ)) {
1013 InputReg = MRI.createGenericVirtualRegister(Ty: I32);
1014 if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {
1015 // We're in a situation where the outgoing function requires the workitem
1016 // ID, but the calling function does not have it (e.g a graphics function
1017 // calling a C calling convention function). This is illegal, but we need
1018 // to produce something.
1019 MIRBuilder.buildUndef(Res: InputReg);
1020 } else {
1021 // Workitem ids are already packed, any of present incoming arguments will
1022 // carry all required fields.
1023 ArgDescriptor IncomingArg = ArgDescriptor::createArg(
1024 Arg: IncomingArgX ? *IncomingArgX :
1025 IncomingArgY ? *IncomingArgY : *IncomingArgZ, Mask: ~0u);
1026 LI->buildLoadInputValue(DstReg: InputReg, B&: MIRBuilder, Arg: &IncomingArg,
1027 ArgRC: &AMDGPU::VGPR_32RegClass, ArgTy: I32);
1028 }
1029 }
1030
1031 if (OutgoingArg->isRegister()) {
1032 if (InputReg)
1033 ArgRegs.emplace_back(Args: OutgoingArg->getRegister(), Args&: InputReg);
1034
1035 if (!CCInfo.AllocateReg(Reg: OutgoingArg->getRegister()))
1036 report_fatal_error(reason: "failed to allocate implicit input argument");
1037 } else {
1038 LLVM_DEBUG(dbgs() << "Unhandled stack passed implicit input argument\n");
1039 return false;
1040 }
1041
1042 return true;
1043}
1044
1045/// Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for
1046/// CC.
1047static std::pair<CCAssignFn *, CCAssignFn *>
1048getAssignFnsForCC(CallingConv::ID CC, const SITargetLowering &TLI) {
1049 return {TLI.CCAssignFnForCall(CC, IsVarArg: false), TLI.CCAssignFnForCall(CC, IsVarArg: true)};
1050}
1051
1052static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect,
1053 bool IsTailCall, bool IsWave32,
1054 CallingConv::ID CC,
1055 bool IsDynamicVGPRChainCall = false) {
1056 // For calls to amdgpu_cs_chain functions, the address is known to be uniform.
1057 assert((AMDGPU::isChainCC(CC) || !IsIndirect || !IsTailCall) &&
1058 "Indirect calls can't be tail calls, "
1059 "because the address can be divergent");
1060 if (!IsTailCall)
1061 return AMDGPU::G_SI_CALL;
1062
1063 if (AMDGPU::isChainCC(CC)) {
1064 if (IsDynamicVGPRChainCall)
1065 return IsWave32 ? AMDGPU::SI_CS_CHAIN_TC_W32_DVGPR
1066 : AMDGPU::SI_CS_CHAIN_TC_W64_DVGPR;
1067 return IsWave32 ? AMDGPU::SI_CS_CHAIN_TC_W32 : AMDGPU::SI_CS_CHAIN_TC_W64;
1068 }
1069
1070 if (CallerF.getFunction().getCallingConv() ==
1071 CallingConv::AMDGPU_Gfx_WholeWave)
1072 return AMDGPU::SI_TCRETURN_GFX_WholeWave;
1073
1074 if (CC == CallingConv::AMDGPU_Gfx || CC == CallingConv::AMDGPU_Gfx_WholeWave)
1075 return AMDGPU::SI_TCRETURN_GFX;
1076
1077 return AMDGPU::SI_TCRETURN;
1078}
1079
1080// Add operands to call instruction to track the callee.
1081static bool addCallTargetOperands(MachineInstrBuilder &CallInst,
1082 MachineIRBuilder &MIRBuilder,
1083 AMDGPUCallLowering::CallLoweringInfo &Info,
1084 bool IsDynamicVGPRChainCall = false) {
1085 if (Info.Callee.isReg()) {
1086 CallInst.addReg(RegNo: Info.Callee.getReg());
1087 CallInst.addImm(Val: 0);
1088 } else if (Info.Callee.isGlobal() && Info.Callee.getOffset() == 0) {
1089 // The call lowering lightly assumed we can directly encode a call target in
1090 // the instruction, which is not the case. Materialize the address here.
1091 const GlobalValue *GV = Info.Callee.getGlobal();
1092 auto Ptr = MIRBuilder.buildGlobalValue(
1093 Res: LLT::pointer(AddressSpace: GV->getAddressSpace(), SizeInBits: 64), GV);
1094 CallInst.addReg(RegNo: Ptr.getReg(Idx: 0));
1095
1096 if (IsDynamicVGPRChainCall) {
1097 // DynamicVGPR chain calls are always indirect.
1098 CallInst.addImm(Val: 0);
1099 } else
1100 CallInst.add(MO: Info.Callee);
1101 } else
1102 return false;
1103
1104 return true;
1105}
1106
1107bool AMDGPUCallLowering::doCallerAndCalleePassArgsTheSameWay(
1108 CallLoweringInfo &Info, MachineFunction &MF,
1109 SmallVectorImpl<ArgInfo> &InArgs) const {
1110 const Function &CallerF = MF.getFunction();
1111 CallingConv::ID CalleeCC = Info.CallConv;
1112 CallingConv::ID CallerCC = CallerF.getCallingConv();
1113
1114 // If the calling conventions match, then everything must be the same.
1115 if (CalleeCC == CallerCC)
1116 return true;
1117
1118 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1119
1120 // Make sure that the caller and callee preserve all of the same registers.
1121 const auto *TRI = ST.getRegisterInfo();
1122
1123 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
1124 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
1125 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved))
1126 return false;
1127
1128 // Check if the caller and callee will handle arguments in the same way.
1129 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
1130 CCAssignFn *CalleeAssignFnFixed;
1131 CCAssignFn *CalleeAssignFnVarArg;
1132 std::tie(args&: CalleeAssignFnFixed, args&: CalleeAssignFnVarArg) =
1133 getAssignFnsForCC(CC: CalleeCC, TLI);
1134
1135 CCAssignFn *CallerAssignFnFixed;
1136 CCAssignFn *CallerAssignFnVarArg;
1137 std::tie(args&: CallerAssignFnFixed, args&: CallerAssignFnVarArg) =
1138 getAssignFnsForCC(CC: CallerCC, TLI);
1139
1140 // FIXME: We are not accounting for potential differences in implicitly passed
1141 // inputs, but only the fixed ABI is supported now anyway.
1142 IncomingValueAssigner CalleeAssigner(CalleeAssignFnFixed,
1143 CalleeAssignFnVarArg);
1144 IncomingValueAssigner CallerAssigner(CallerAssignFnFixed,
1145 CallerAssignFnVarArg);
1146 return resultsCompatible(Info, MF, InArgs, CalleeAssigner, CallerAssigner);
1147}
1148
1149bool AMDGPUCallLowering::areCalleeOutgoingArgsTailCallable(
1150 CallLoweringInfo &Info, MachineFunction &MF,
1151 SmallVectorImpl<ArgInfo> &OutArgs) const {
1152 // If there are no outgoing arguments, then we are done.
1153 if (OutArgs.empty())
1154 return true;
1155
1156 const Function &CallerF = MF.getFunction();
1157 CallingConv::ID CalleeCC = Info.CallConv;
1158 CallingConv::ID CallerCC = CallerF.getCallingConv();
1159 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
1160
1161 CCAssignFn *AssignFnFixed;
1162 CCAssignFn *AssignFnVarArg;
1163 std::tie(args&: AssignFnFixed, args&: AssignFnVarArg) = getAssignFnsForCC(CC: CalleeCC, TLI);
1164
1165 // We have outgoing arguments. Make sure that we can tail call with them.
1166 SmallVector<CCValAssign, 16> OutLocs;
1167 CCState OutInfo(CalleeCC, false, MF, OutLocs, CallerF.getContext());
1168 OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg);
1169
1170 if (!determineAssignments(Assigner, Args&: OutArgs, CCInfo&: OutInfo)) {
1171 LLVM_DEBUG(dbgs() << "... Could not analyze call operands.\n");
1172 return false;
1173 }
1174
1175 // Make sure that they can fit on the caller's stack.
1176 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1177 if (OutInfo.getStackSize() > FuncInfo->getBytesInStackArgArea()) {
1178 LLVM_DEBUG(dbgs() << "... Cannot fit call operands on caller's stack.\n");
1179 return false;
1180 }
1181
1182 // Verify that the parameters in callee-saved registers match.
1183 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1184 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1185 const uint32_t *CallerPreservedMask = TRI->getCallPreservedMask(MF, CallerCC);
1186 MachineRegisterInfo &MRI = MF.getRegInfo();
1187 return parametersInCSRMatch(MRI, CallerPreservedMask, ArgLocs: OutLocs, OutVals: OutArgs);
1188}
1189
1190bool AMDGPUCallLowering::isEligibleForTailCallOptimization(
1191 MachineIRBuilder &B, CallLoweringInfo &Info,
1192 SmallVectorImpl<ArgInfo> &InArgs, SmallVectorImpl<ArgInfo> &OutArgs) const {
1193 // Must pass all target-independent checks in order to tail call optimize.
1194 if (!Info.IsTailCall)
1195 return false;
1196
1197 // Indirect calls can't be tail calls, because the address can be divergent.
1198 // TODO Check divergence info if the call really is divergent.
1199 if (Info.Callee.isReg())
1200 return false;
1201
1202 MachineFunction &MF = B.getMF();
1203 const Function &CallerF = MF.getFunction();
1204 CallingConv::ID CalleeCC = Info.CallConv;
1205 CallingConv::ID CallerCC = CallerF.getCallingConv();
1206
1207 const SIRegisterInfo *TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
1208 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
1209 // Kernels aren't callable, and don't have a live in return address so it
1210 // doesn't make sense to do a tail call with entry functions.
1211 if (!CallerPreserved)
1212 return false;
1213
1214 if (!AMDGPU::mayTailCallThisCC(CC: CalleeCC)) {
1215 LLVM_DEBUG(dbgs() << "... Calling convention cannot be tail called.\n");
1216 return false;
1217 }
1218
1219 if (any_of(Range: CallerF.args(), P: [](const Argument &A) {
1220 return A.hasByValAttr() || A.hasSwiftErrorAttr();
1221 })) {
1222 LLVM_DEBUG(dbgs() << "... Cannot tail call from callers with byval "
1223 "or swifterror arguments\n");
1224 return false;
1225 }
1226
1227 // If we have -tailcallopt, then we're done.
1228 if (MF.getTarget().Options.GuaranteedTailCallOpt) {
1229 return AMDGPU::canGuaranteeTCO(CC: CalleeCC) &&
1230 CalleeCC == CallerF.getCallingConv();
1231 }
1232
1233 // Verify that the incoming and outgoing arguments from the callee are
1234 // safe to tail call.
1235 if (!doCallerAndCalleePassArgsTheSameWay(Info, MF, InArgs)) {
1236 LLVM_DEBUG(
1237 dbgs()
1238 << "... Caller and callee have incompatible calling conventions.\n");
1239 return false;
1240 }
1241
1242 // FIXME: We need to check if any arguments passed in SGPR are uniform. If
1243 // they are not, this cannot be a tail call. If they are uniform, but may be
1244 // VGPR, we need to insert readfirstlanes.
1245 if (!areCalleeOutgoingArgsTailCallable(Info, MF, OutArgs))
1246 return false;
1247
1248 LLVM_DEBUG(dbgs() << "... Call is eligible for tail call optimization.\n");
1249 return true;
1250}
1251
1252// Insert outgoing implicit arguments for a call, by inserting copies to the
1253// implicit argument registers and adding the necessary implicit uses to the
1254// call instruction.
1255void AMDGPUCallLowering::handleImplicitCallArguments(
1256 MachineIRBuilder &MIRBuilder, MachineInstrBuilder &CallInst,
1257 const GCNSubtarget &ST, const SIMachineFunctionInfo &FuncInfo,
1258 CallingConv::ID CalleeCC,
1259 ArrayRef<std::pair<MCRegister, Register>> ImplicitArgRegs) const {
1260 if (!ST.hasFlatScratchEnabled()) {
1261 // Insert copies for the SRD. In the HSA case, this should be an identity
1262 // copy.
1263 auto ScratchRSrcReg = MIRBuilder.buildCopy(Res: LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32),
1264 Op: FuncInfo.getScratchRSrcReg());
1265
1266 auto CalleeRSrcReg = AMDGPU::isChainCC(CC: CalleeCC)
1267 ? AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51
1268 : AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
1269
1270 MIRBuilder.buildCopy(Res: CalleeRSrcReg, Op: ScratchRSrcReg);
1271 CallInst.addReg(RegNo: CalleeRSrcReg, Flags: RegState::Implicit);
1272 }
1273
1274 for (std::pair<MCRegister, Register> ArgReg : ImplicitArgRegs) {
1275 MIRBuilder.buildCopy(Res: (Register)ArgReg.first, Op: ArgReg.second);
1276 CallInst.addReg(RegNo: ArgReg.first, Flags: RegState::Implicit);
1277 }
1278}
1279
1280namespace {
1281// Chain calls have special arguments that we need to handle. These have the
1282// same index as they do in the llvm.amdgcn.cs.chain intrinsic.
1283enum ChainCallArgIdx {
1284 Exec = 1,
1285 Flags = 4,
1286 NumVGPRs = 5,
1287 FallbackExec = 6,
1288 FallbackCallee = 7,
1289};
1290} // anonymous namespace
1291
1292bool AMDGPUCallLowering::lowerTailCall(
1293 MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
1294 SmallVectorImpl<ArgInfo> &OutArgs) const {
1295 MachineFunction &MF = MIRBuilder.getMF();
1296 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1297 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1298 const Function &F = MF.getFunction();
1299 MachineRegisterInfo &MRI = MF.getRegInfo();
1300 const SIInstrInfo *TII = ST.getInstrInfo();
1301 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1302 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
1303
1304 // True when we're tail calling, but without -tailcallopt.
1305 bool IsSibCall = !MF.getTarget().Options.GuaranteedTailCallOpt;
1306
1307 // Find out which ABI gets to decide where things go.
1308 CallingConv::ID CalleeCC = Info.CallConv;
1309 CCAssignFn *AssignFnFixed;
1310 CCAssignFn *AssignFnVarArg;
1311 std::tie(args&: AssignFnFixed, args&: AssignFnVarArg) = getAssignFnsForCC(CC: CalleeCC, TLI);
1312
1313 MachineInstrBuilder CallSeqStart;
1314 if (!IsSibCall)
1315 CallSeqStart = MIRBuilder.buildInstr(Opcode: AMDGPU::ADJCALLSTACKUP);
1316
1317 bool IsChainCall = AMDGPU::isChainCC(CC: Info.CallConv);
1318 bool IsDynamicVGPRChainCall = false;
1319
1320 if (IsChainCall) {
1321 ArgInfo FlagsArg = Info.OrigArgs[ChainCallArgIdx::Flags];
1322 const APInt &FlagsValue = cast<ConstantInt>(Val: FlagsArg.OrigValue)->getValue();
1323 if (FlagsValue.isZero()) {
1324 if (Info.OrigArgs.size() != 5) {
1325 LLVM_DEBUG(dbgs() << "No additional args allowed if flags == 0\n");
1326 return false;
1327 }
1328 } else if (FlagsValue.isOneBitSet(BitNo: 0)) {
1329 IsDynamicVGPRChainCall = true;
1330
1331 if (Info.OrigArgs.size() != 8) {
1332 LLVM_DEBUG(dbgs() << "Expected 3 additional args\n");
1333 return false;
1334 }
1335
1336 // On GFX12, we can only change the VGPR allocation for wave32.
1337 if (!ST.isWave32()) {
1338 F.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1339 F, "dynamic VGPR mode is only supported for wave32"));
1340 return false;
1341 }
1342
1343 ArgInfo FallbackExecArg = Info.OrigArgs[ChainCallArgIdx::FallbackExec];
1344 assert(FallbackExecArg.Regs.size() == 1 &&
1345 "Expected single register for fallback EXEC");
1346 if (!FallbackExecArg.Ty->isIntegerTy(BitWidth: ST.getWavefrontSize())) {
1347 LLVM_DEBUG(dbgs() << "Bad type for fallback EXEC\n");
1348 return false;
1349 }
1350 }
1351 }
1352
1353 unsigned Opc = getCallOpcode(CallerF: MF, IsIndirect: Info.Callee.isReg(), /*IsTailCall*/ true,
1354 IsWave32: ST.isWave32(), CC: CalleeCC, IsDynamicVGPRChainCall);
1355 auto MIB = MIRBuilder.buildInstrNoInsert(Opcode: Opc);
1356
1357 if (FuncInfo->isWholeWaveFunction())
1358 addOriginalExecToReturn(MF, Ret&: MIB);
1359
1360 // Keep track of the index of the next operand to be added to the call
1361 unsigned CalleeIdx = MIB->getNumOperands();
1362
1363 if (!addCallTargetOperands(CallInst&: MIB, MIRBuilder, Info, IsDynamicVGPRChainCall))
1364 return false;
1365
1366 // Byte offset for the tail call. When we are sibcalling, this will always
1367 // be 0.
1368 MIB.addImm(Val: 0);
1369
1370 // If this is a chain call, we need to pass in the EXEC mask as well as any
1371 // other special args.
1372 if (IsChainCall) {
1373 auto AddRegOrImm = [&](const ArgInfo &Arg) {
1374 if (auto CI = dyn_cast<ConstantInt>(Val: Arg.OrigValue)) {
1375 MIB.addImm(Val: CI->getSExtValue());
1376 } else {
1377 MIB.addReg(RegNo: Arg.Regs[0]);
1378 unsigned Idx = MIB->getNumOperands() - 1;
1379 MIB->getOperand(i: Idx).setReg(constrainOperandRegClass(
1380 MF, TRI: *TRI, MRI, TII: *TII, RBI: *ST.getRegBankInfo(), InsertPt&: *MIB, II: MIB->getDesc(),
1381 RegMO&: MIB->getOperand(i: Idx), OpIdx: Idx));
1382 }
1383 };
1384
1385 ArgInfo ExecArg = Info.OrigArgs[ChainCallArgIdx::Exec];
1386 assert(ExecArg.Regs.size() == 1 && "Too many regs for EXEC");
1387
1388 if (!ExecArg.Ty->isIntegerTy(BitWidth: ST.getWavefrontSize())) {
1389 LLVM_DEBUG(dbgs() << "Bad type for EXEC");
1390 return false;
1391 }
1392
1393 AddRegOrImm(ExecArg);
1394 if (IsDynamicVGPRChainCall)
1395 std::for_each(first: Info.OrigArgs.begin() + ChainCallArgIdx::NumVGPRs,
1396 last: Info.OrigArgs.end(), f: AddRegOrImm);
1397 }
1398
1399 // Tell the call which registers are clobbered.
1400 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CalleeCC);
1401 MIB.addRegMask(Mask);
1402
1403 // FPDiff is the byte offset of the call's argument area from the callee's.
1404 // Stores to callee stack arguments will be placed in FixedStackSlots offset
1405 // by this amount for a tail call. In a sibling call it must be 0 because the
1406 // caller will deallocate the entire stack and the callee still expects its
1407 // arguments to begin at SP+0.
1408 int FPDiff = 0;
1409
1410 // This will be 0 for sibcalls, potentially nonzero for tail calls produced
1411 // by -tailcallopt. For sibcalls, the memory operands for the call are
1412 // already available in the caller's incoming argument space.
1413 unsigned NumBytes = 0;
1414 if (!IsSibCall) {
1415 // We aren't sibcalling, so we need to compute FPDiff. We need to do this
1416 // before handling assignments, because FPDiff must be known for memory
1417 // arguments.
1418 unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
1419 SmallVector<CCValAssign, 16> OutLocs;
1420 CCState OutInfo(CalleeCC, false, MF, OutLocs, F.getContext());
1421
1422 // FIXME: Not accounting for callee implicit inputs
1423 OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg);
1424 if (!determineAssignments(Assigner&: CalleeAssigner, Args&: OutArgs, CCInfo&: OutInfo))
1425 return false;
1426
1427 // The callee will pop the argument stack as a tail call. Thus, we must
1428 // keep it 16-byte aligned.
1429 NumBytes = alignTo(Size: OutInfo.getStackSize(), A: ST.getStackAlignment());
1430
1431 // FPDiff will be negative if this tail call requires more space than we
1432 // would automatically have in our incoming argument space. Positive if we
1433 // actually shrink the stack.
1434 FPDiff = NumReusableBytes - NumBytes;
1435
1436 // The stack pointer must be 16-byte aligned at all times it's used for a
1437 // memory operation, which in practice means at *all* times and in
1438 // particular across call boundaries. Therefore our own arguments started at
1439 // a 16-byte aligned SP and the delta applied for the tail call should
1440 // satisfy the same constraint.
1441 assert(isAligned(ST.getStackAlignment(), FPDiff) &&
1442 "unaligned stack on tail call");
1443 }
1444
1445 SmallVector<CCValAssign, 16> ArgLocs;
1446 CCState CCInfo(Info.CallConv, Info.IsVarArg, MF, ArgLocs, F.getContext());
1447
1448 // We could pass MIB and directly add the implicit uses to the call
1449 // now. However, as an aesthetic choice, place implicit argument operands
1450 // after the ordinary user argument registers.
1451 SmallVector<std::pair<MCRegister, Register>, 12> ImplicitArgRegs;
1452
1453 if (Info.CallConv != CallingConv::AMDGPU_Gfx &&
1454 Info.CallConv != CallingConv::AMDGPU_Gfx_WholeWave &&
1455 !AMDGPU::isChainCC(CC: Info.CallConv)) {
1456 // With a fixed ABI, allocate fixed registers before user arguments.
1457 if (!passSpecialInputs(MIRBuilder, CCInfo, ArgRegs&: ImplicitArgRegs, Info))
1458 return false;
1459 }
1460
1461 // Mark the scratch resource descriptor as allocated so the CC analysis
1462 // does not assign user arguments to these registers, matching the callee.
1463 if (!ST.hasFlatScratchEnabled())
1464 CCInfo.AllocateReg(Reg: FuncInfo->getScratchRSrcReg());
1465
1466 OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg);
1467
1468 if (!determineAssignments(Assigner, Args&: OutArgs, CCInfo))
1469 return false;
1470
1471 // Do the actual argument marshalling.
1472 AMDGPUOutgoingArgHandler Handler(MIRBuilder, MRI, MIB, true, FPDiff);
1473 if (!handleAssignments(Handler, Args&: OutArgs, CCState&: CCInfo, ArgLocs, MIRBuilder))
1474 return false;
1475
1476 if (Info.ConvergenceCtrlToken) {
1477 MIB.addUse(RegNo: Info.ConvergenceCtrlToken, Flags: RegState::Implicit);
1478 }
1479 handleImplicitCallArguments(MIRBuilder, CallInst&: MIB, ST, FuncInfo: *FuncInfo, CalleeCC,
1480 ImplicitArgRegs);
1481
1482 // If we have -tailcallopt, we need to adjust the stack. We'll do the call
1483 // sequence start and end here.
1484 if (!IsSibCall) {
1485 MIB->getOperand(i: CalleeIdx + 1).setImm(FPDiff);
1486 CallSeqStart.addImm(Val: NumBytes).addImm(Val: 0);
1487 // End the call sequence *before* emitting the call. Normally, we would
1488 // tidy the frame up after the call. However, here, we've laid out the
1489 // parameters so that when SP is reset, they will be in the correct
1490 // location.
1491 MIRBuilder.buildInstr(Opcode: AMDGPU::ADJCALLSTACKDOWN).addImm(Val: NumBytes).addImm(Val: 0);
1492 }
1493
1494 // Now we can add the actual call instruction to the correct basic block.
1495 MIRBuilder.insertInstr(MIB);
1496
1497 // If this is a whole wave tail call, we need to constrain the register for
1498 // the original EXEC.
1499 if (MIB->getOpcode() == AMDGPU::SI_TCRETURN_GFX_WholeWave) {
1500 MIB->getOperand(i: 0).setReg(
1501 constrainOperandRegClass(MF, TRI: *TRI, MRI, TII: *TII, RBI: *ST.getRegBankInfo(),
1502 InsertPt&: *MIB, II: MIB->getDesc(), RegMO&: MIB->getOperand(i: 0), OpIdx: 0));
1503 }
1504
1505 // If Callee is a reg, since it is used by a target specific
1506 // instruction, it must have a register class matching the
1507 // constraint of that instruction.
1508
1509 // FIXME: We should define regbankselectable call instructions to handle
1510 // divergent call targets.
1511 if (MIB->getOperand(i: CalleeIdx).isReg()) {
1512 MIB->getOperand(i: CalleeIdx).setReg(constrainOperandRegClass(
1513 MF, TRI: *TRI, MRI, TII: *TII, RBI: *ST.getRegBankInfo(), InsertPt&: *MIB, II: MIB->getDesc(),
1514 RegMO&: MIB->getOperand(i: CalleeIdx), OpIdx: CalleeIdx));
1515 }
1516
1517 MF.getFrameInfo().setHasTailCall();
1518 Info.LoweredTailCall = true;
1519 return true;
1520}
1521
1522/// Lower a call to the @llvm.amdgcn.cs.chain intrinsic.
1523bool AMDGPUCallLowering::lowerChainCall(MachineIRBuilder &MIRBuilder,
1524 CallLoweringInfo &Info) const {
1525 ArgInfo Callee = Info.OrigArgs[0];
1526 ArgInfo SGPRArgs = Info.OrigArgs[2];
1527 ArgInfo VGPRArgs = Info.OrigArgs[3];
1528
1529 MachineFunction &MF = MIRBuilder.getMF();
1530 const Function &F = MF.getFunction();
1531 const DataLayout &DL = F.getDataLayout();
1532
1533 // The function to jump to is actually the first argument, so we'll change the
1534 // Callee and other info to match that before using our existing helper.
1535 const Value *CalleeV = Callee.OrigValue->stripPointerCasts();
1536 if (const Function *F = dyn_cast<Function>(Val: CalleeV)) {
1537 Info.Callee = MachineOperand::CreateGA(GV: F, Offset: 0);
1538 Info.CallConv = F->getCallingConv();
1539 } else {
1540 assert(Callee.Regs.size() == 1 && "Too many regs for the callee");
1541 Register CalleeReg = Callee.Regs[0];
1542 LLT CalleeTy = MIRBuilder.getMRI()->getType(Reg: CalleeReg);
1543 Register UniformCallee =
1544 MIRBuilder.buildIntrinsic(ID: Intrinsic::amdgcn_readfirstlane, Res: {CalleeTy})
1545 .addReg(RegNo: CalleeReg)
1546 .getReg(Idx: 0);
1547 Info.Callee = MachineOperand::CreateReg(Reg: UniformCallee, isDef: false);
1548 Info.CallConv = CallingConv::AMDGPU_CS_Chain; // amdgpu_cs_chain_preserve
1549 // behaves the same here.
1550 }
1551
1552 // The function that we're calling cannot be vararg (only the intrinsic is).
1553 Info.IsVarArg = false;
1554
1555 assert(
1556 all_of(SGPRArgs.Flags, [](ISD::ArgFlagsTy F) { return F.isInReg(); }) &&
1557 "SGPR arguments should be marked inreg");
1558 assert(
1559 none_of(VGPRArgs.Flags, [](ISD::ArgFlagsTy F) { return F.isInReg(); }) &&
1560 "VGPR arguments should not be marked inreg");
1561
1562 SmallVector<ArgInfo, 8> OutArgs;
1563 splitToValueTypes(OrigArgInfo: SGPRArgs, SplitArgs&: OutArgs, DL, CallConv: Info.CallConv);
1564 splitToValueTypes(OrigArgInfo: VGPRArgs, SplitArgs&: OutArgs, DL, CallConv: Info.CallConv);
1565
1566 Info.IsMustTailCall = true;
1567 return lowerTailCall(MIRBuilder, Info, OutArgs);
1568}
1569
1570bool AMDGPUCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
1571 CallLoweringInfo &Info) const {
1572 if (Function *F = Info.CB->getCalledFunction())
1573 if (F->isIntrinsic()) {
1574 switch (F->getIntrinsicID()) {
1575 case Intrinsic::amdgcn_cs_chain:
1576 return lowerChainCall(MIRBuilder, Info);
1577 case Intrinsic::amdgcn_call_whole_wave:
1578 Info.CallConv = CallingConv::AMDGPU_Gfx_WholeWave;
1579
1580 // Get the callee from the original instruction, so it doesn't look like
1581 // this is an indirect call.
1582 Info.Callee = MachineOperand::CreateGA(
1583 GV: cast<GlobalValue>(Val: Info.CB->getOperand(i_nocapture: 0)), /*Offset=*/0);
1584 Info.OrigArgs.erase(CI: Info.OrigArgs.begin());
1585 Info.IsVarArg = false;
1586 break;
1587 default:
1588 llvm_unreachable("Unexpected intrinsic call");
1589 }
1590 }
1591
1592 if (Info.IsVarArg) {
1593 LLVM_DEBUG(dbgs() << "Variadic functions not implemented\n");
1594 return false;
1595 }
1596
1597 MachineFunction &MF = MIRBuilder.getMF();
1598 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1599 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1600
1601 const Function &F = MF.getFunction();
1602 MachineRegisterInfo &MRI = MF.getRegInfo();
1603 const SITargetLowering &TLI = *getTLI<SITargetLowering>();
1604 const DataLayout &DL = F.getDataLayout();
1605
1606 SmallVector<ArgInfo, 8> OutArgs;
1607 for (auto &OrigArg : Info.OrigArgs)
1608 splitToValueTypes(OrigArgInfo: OrigArg, SplitArgs&: OutArgs, DL, CallConv: Info.CallConv);
1609
1610 SmallVector<ArgInfo, 8> InArgs;
1611 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy())
1612 splitToValueTypes(OrigArgInfo: Info.OrigRet, SplitArgs&: InArgs, DL, CallConv: Info.CallConv);
1613
1614 // If we can lower as a tail call, do that instead.
1615 bool CanTailCallOpt =
1616 isEligibleForTailCallOptimization(B&: MIRBuilder, Info, InArgs, OutArgs);
1617
1618 // We must emit a tail call if we have musttail.
1619 if (Info.IsMustTailCall && !CanTailCallOpt) {
1620 LLVM_DEBUG(dbgs() << "Failed to lower musttail call as tail call\n");
1621 return false;
1622 }
1623
1624 Info.IsTailCall = CanTailCallOpt;
1625 if (CanTailCallOpt)
1626 return lowerTailCall(MIRBuilder, Info, OutArgs);
1627
1628 // Find out which ABI gets to decide where things go.
1629 CCAssignFn *AssignFnFixed;
1630 CCAssignFn *AssignFnVarArg;
1631 std::tie(args&: AssignFnFixed, args&: AssignFnVarArg) =
1632 getAssignFnsForCC(CC: Info.CallConv, TLI);
1633
1634 MIRBuilder.buildInstr(Opcode: AMDGPU::ADJCALLSTACKUP)
1635 .addImm(Val: 0)
1636 .addImm(Val: 0);
1637
1638 // Create a temporarily-floating call instruction so we can add the implicit
1639 // uses of arg registers.
1640 unsigned Opc = getCallOpcode(CallerF: MF, IsIndirect: Info.Callee.isReg(), IsTailCall: false, IsWave32: ST.isWave32(),
1641 CC: Info.CallConv);
1642
1643 auto MIB = MIRBuilder.buildInstrNoInsert(Opcode: Opc);
1644 MIB.addDef(RegNo: TRI->getReturnAddressReg(MF));
1645
1646 if (!Info.IsConvergent)
1647 MIB.setMIFlag(MachineInstr::NoConvergent);
1648
1649 if (!addCallTargetOperands(CallInst&: MIB, MIRBuilder, Info))
1650 return false;
1651
1652 // Tell the call which registers are clobbered.
1653 const uint32_t *Mask = TRI->getCallPreservedMask(MF, Info.CallConv);
1654 MIB.addRegMask(Mask);
1655
1656 SmallVector<CCValAssign, 16> ArgLocs;
1657 CCState CCInfo(Info.CallConv, Info.IsVarArg, MF, ArgLocs, F.getContext());
1658
1659 // We could pass MIB and directly add the implicit uses to the call
1660 // now. However, as an aesthetic choice, place implicit argument operands
1661 // after the ordinary user argument registers.
1662 SmallVector<std::pair<MCRegister, Register>, 12> ImplicitArgRegs;
1663
1664 if (Info.CallConv != CallingConv::AMDGPU_Gfx &&
1665 Info.CallConv != CallingConv::AMDGPU_Gfx_WholeWave) {
1666 // With a fixed ABI, allocate fixed registers before user arguments.
1667 if (!passSpecialInputs(MIRBuilder, CCInfo, ArgRegs&: ImplicitArgRegs, Info))
1668 return false;
1669 }
1670
1671 // Mark the scratch resource descriptor as allocated so the CC analysis
1672 // does not assign user arguments to these registers, matching the callee.
1673 if (!ST.hasFlatScratchEnabled()) {
1674 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1675 CCInfo.AllocateReg(Reg: FuncInfo->getScratchRSrcReg());
1676 }
1677
1678 // Do the actual argument marshalling.
1679 OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg);
1680 if (!determineAssignments(Assigner, Args&: OutArgs, CCInfo))
1681 return false;
1682
1683 AMDGPUOutgoingArgHandler Handler(MIRBuilder, MRI, MIB, false);
1684 if (!handleAssignments(Handler, Args&: OutArgs, CCState&: CCInfo, ArgLocs, MIRBuilder))
1685 return false;
1686
1687 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1688
1689 if (Info.ConvergenceCtrlToken) {
1690 MIB.addUse(RegNo: Info.ConvergenceCtrlToken, Flags: RegState::Implicit);
1691 }
1692 handleImplicitCallArguments(MIRBuilder, CallInst&: MIB, ST, FuncInfo: *MFI, CalleeCC: Info.CallConv,
1693 ImplicitArgRegs);
1694
1695 // Get a count of how many bytes are to be pushed on the stack.
1696 unsigned NumBytes = CCInfo.getStackSize();
1697
1698 // If Callee is a reg, since it is used by a target specific
1699 // instruction, it must have a register class matching the
1700 // constraint of that instruction.
1701
1702 // FIXME: We should define regbankselectable call instructions to handle
1703 // divergent call targets.
1704 if (MIB->getOperand(i: 1).isReg()) {
1705 MIB->getOperand(i: 1).setReg(constrainOperandRegClass(
1706 MF, TRI: *TRI, MRI, TII: *ST.getInstrInfo(),
1707 RBI: *ST.getRegBankInfo(), InsertPt&: *MIB, II: MIB->getDesc(), RegMO&: MIB->getOperand(i: 1),
1708 OpIdx: 1));
1709 }
1710
1711 // Now we can add the actual call instruction to the correct position.
1712 MIRBuilder.insertInstr(MIB);
1713
1714 // Finally we can copy the returned value back into its virtual-register. In
1715 // symmetry with the arguments, the physical register must be an
1716 // implicit-define of the call instruction.
1717 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy()) {
1718 CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(CC: Info.CallConv,
1719 IsVarArg: Info.IsVarArg);
1720 IncomingValueAssigner Assigner(RetAssignFn);
1721 CallReturnHandler Handler(MIRBuilder, MRI, MIB);
1722 if (!determineAndHandleAssignments(Handler, Assigner, Args&: InArgs, MIRBuilder,
1723 CallConv: Info.CallConv, IsVarArg: Info.IsVarArg))
1724 return false;
1725 }
1726
1727 uint64_t CalleePopBytes = NumBytes;
1728
1729 MIRBuilder.buildInstr(Opcode: AMDGPU::ADJCALLSTACKDOWN)
1730 .addImm(Val: 0)
1731 .addImm(Val: CalleePopBytes);
1732
1733 if (!Info.CanLowerReturn) {
1734 insertSRetLoads(MIRBuilder, RetTy: Info.OrigRet.Ty, VRegs: Info.OrigRet.Regs,
1735 DemoteReg: Info.DemoteRegister, FI: Info.DemoteStackIndex);
1736 }
1737
1738 return true;
1739}
1740
1741void AMDGPUCallLowering::addOriginalExecToReturn(
1742 MachineFunction &MF, MachineInstrBuilder &Ret) const {
1743 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1744 const SIInstrInfo *TII = ST.getInstrInfo();
1745 const MachineInstr *Setup = TII->getWholeWaveFunctionSetup(MF);
1746 Ret.addReg(RegNo: Setup->getOperand(i: 0).getReg());
1747}
1748