1//===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements some simple delegations needed for call lowering.
11///
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/GlobalISel/CallLowering.h"
15#include "llvm/CodeGen/Analysis.h"
16#include "llvm/CodeGen/CallingConvLower.h"
17#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18#include "llvm/CodeGen/GlobalISel/Utils.h"
19#include "llvm/CodeGen/MachineFrameInfo.h"
20#include "llvm/CodeGen/MachineOperand.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/CodeGen/TargetLowering.h"
23#include "llvm/IR/DataLayout.h"
24#include "llvm/IR/LLVMContext.h"
25#include "llvm/IR/Module.h"
26#include "llvm/Target/TargetMachine.h"
27
28#define DEBUG_TYPE "call-lowering"
29
30using namespace llvm;
31
32void CallLowering::anchor() {}
33
34/// Helper function which updates \p Flags based on the contents of \p Attrs.
35static void addFlagsFromAttrSet(ISD::ArgFlagsTy &Flags, AttributeSet Attrs) {
36 if (!Attrs.hasAttributes())
37 return;
38
39 // TODO: There are missing flags. Add them here.
40 if (Attrs.hasAttribute(Kind: Attribute::SExt))
41 Flags.setSExt();
42 if (Attrs.hasAttribute(Kind: Attribute::ZExt))
43 Flags.setZExt();
44 if (Attrs.hasAttribute(Kind: Attribute::InReg))
45 Flags.setInReg();
46 if (Attrs.hasAttribute(Kind: Attribute::StructRet))
47 Flags.setSRet();
48 if (Attrs.hasAttribute(Kind: Attribute::Nest))
49 Flags.setNest();
50 if (Attrs.hasAttribute(Kind: Attribute::ByVal))
51 Flags.setByVal();
52 if (Attrs.hasAttribute(Kind: Attribute::ByRef))
53 Flags.setByRef();
54 if (Attrs.hasAttribute(Kind: Attribute::InAlloca)) {
55 Flags.setInAlloca();
56 // Set the byval flag for CCAssignFn callbacks that don't know about
57 // inalloca. This way we can know how many bytes we should've allocated
58 // and how many bytes a callee cleanup function will pop. If we port
59 // inalloca to more targets, we'll have to add custom inalloca handling
60 // in the various CC lowering callbacks.
61 Flags.setByVal();
62 }
63 if (Attrs.hasAttribute(Kind: Attribute::Preallocated)) {
64 Flags.setPreallocated();
65 // Set the byval flag for CCAssignFn callbacks that don't know about
66 // preallocated. This way we can know how many bytes we should've
67 // allocated and how many bytes a callee cleanup function will pop. If
68 // we port preallocated to more targets, we'll have to add custom
69 // preallocated handling in the various CC lowering callbacks.
70 Flags.setByVal();
71 }
72 if (Attrs.hasAttribute(Kind: Attribute::Returned))
73 Flags.setReturned();
74 if (Attrs.hasAttribute(Kind: Attribute::SwiftSelf))
75 Flags.setSwiftSelf();
76 if (Attrs.hasAttribute(Kind: Attribute::SwiftAsync))
77 Flags.setSwiftAsync();
78 if (Attrs.hasAttribute(Kind: Attribute::SwiftError))
79 Flags.setSwiftError();
80}
81
82ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call,
83 unsigned ArgIdx) const {
84 ISD::ArgFlagsTy Flags;
85 const AttributeList &Attrs = Call.getAttributes();
86 addFlagsFromAttrSet(Flags, Attrs: Attrs.getParamAttrs(ArgNo: ArgIdx));
87 if (const Function *F = Call.getCalledFunction())
88 addFlagsFromAttrSet(Flags, Attrs: F->getAttributes().getParamAttrs(ArgNo: ArgIdx));
89 return Flags;
90}
91
92ISD::ArgFlagsTy
93CallLowering::getAttributesForReturn(const CallBase &Call) const {
94 ISD::ArgFlagsTy Flags;
95 addFlagsFromAttrSet(Flags, Attrs: Call.getAttributes().getRetAttrs());
96 if (const Function *F = Call.getCalledFunction())
97 addFlagsFromAttrSet(Flags, Attrs: F->getAttributes().getRetAttrs());
98 return Flags;
99}
100
101void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags,
102 const AttributeList &Attrs,
103 unsigned OpIdx) const {
104 addFlagsFromAttrSet(Flags, Attrs: Attrs.getAttributes(Index: OpIdx));
105}
106
107bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
108 ArrayRef<Register> ResRegs,
109 ArrayRef<ArrayRef<Register>> ArgRegs,
110 Register SwiftErrorVReg,
111 std::optional<PtrAuthInfo> PAI,
112 Register ConvergenceCtrlToken,
113 std::function<Register()> GetCalleeReg) const {
114 CallLoweringInfo Info;
115 const DataLayout &DL = MIRBuilder.getDataLayout();
116 MachineFunction &MF = MIRBuilder.getMF();
117 MachineRegisterInfo &MRI = MF.getRegInfo();
118 bool CanBeTailCalled = CB.isTailCall() &&
119 isInTailCallPosition(Call: CB, TM: MF.getTarget()) &&
120 (MF.getFunction()
121 .getFnAttribute(Kind: "disable-tail-calls")
122 .getValueAsString() != "true");
123
124 CallingConv::ID CallConv = CB.getCallingConv();
125 Type *RetTy = CB.getType();
126 bool IsVarArg = CB.getFunctionType()->isVarArg();
127
128 SmallVector<BaseArgInfo, 4> SplitArgs;
129 getReturnInfo(CallConv, RetTy, Attrs: CB.getAttributes(), Outs&: SplitArgs, DL);
130 Info.CanLowerReturn = canLowerReturn(MF, CallConv, Outs&: SplitArgs, IsVarArg);
131
132 Info.IsConvergent = CB.isConvergent();
133
134 if (!Info.CanLowerReturn) {
135 // Callee requires sret demotion.
136 insertSRetOutgoingArgument(MIRBuilder, CB, Info);
137
138 // The sret demotion isn't compatible with tail-calls, since the sret
139 // argument points into the caller's stack frame.
140 CanBeTailCalled = false;
141 }
142
143 // First step is to marshall all the function's parameters into the correct
144 // physregs and memory locations. Gather the sequence of argument types that
145 // we'll pass to the assigner function.
146 unsigned i = 0;
147 unsigned NumFixedArgs = CB.getFunctionType()->getNumParams();
148 for (const auto &Arg : CB.args()) {
149 ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(Call: CB, ArgIdx: i)};
150 setArgFlags(Arg&: OrigArg, OpIdx: i + AttributeList::FirstArgIndex, DL, FuncInfo: CB);
151 if (i >= NumFixedArgs)
152 OrigArg.Flags[0].setVarArg();
153
154 // If we have an explicit sret argument that is an Instruction, (i.e., it
155 // might point to function-local memory), we can't meaningfully tail-call.
156 if (OrigArg.Flags[0].isSRet() && isa<Instruction>(Val: &Arg))
157 CanBeTailCalled = false;
158
159 Info.OrigArgs.push_back(Elt: OrigArg);
160 ++i;
161 }
162
163 // Try looking through a bitcast from one function type to another.
164 // Commonly happens with calls to objc_msgSend().
165 const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts();
166
167 // If IRTranslator chose to drop the ptrauth info, we can turn this into
168 // a direct call.
169 if (!PAI && CB.countOperandBundlesOfType(ID: LLVMContext::OB_ptrauth)) {
170 CalleeV = cast<ConstantPtrAuth>(Val: CalleeV)->getPointer();
171 assert(isa<Function>(CalleeV));
172 }
173
174 if (const Function *F = dyn_cast<Function>(Val: CalleeV)) {
175 if (F->hasFnAttribute(Kind: Attribute::NonLazyBind)) {
176 LLT Ty = getLLTForType(Ty&: *F->getType(), DL);
177 Register Reg = MIRBuilder.buildGlobalValue(Res: Ty, GV: F).getReg(Idx: 0);
178 Info.Callee = MachineOperand::CreateReg(Reg, isDef: false);
179 } else {
180 Info.Callee = MachineOperand::CreateGA(GV: F, Offset: 0);
181 }
182 } else if (isa<GlobalIFunc>(Val: CalleeV) || isa<GlobalAlias>(Val: CalleeV)) {
183 // IR IFuncs and Aliases can't be forward declared (only defined), so the
184 // callee must be in the same TU and therefore we can direct-call it without
185 // worrying about it being out of range.
186 Info.Callee = MachineOperand::CreateGA(GV: cast<GlobalValue>(Val: CalleeV), Offset: 0);
187 } else
188 Info.Callee = MachineOperand::CreateReg(Reg: GetCalleeReg(), isDef: false);
189
190 Register ReturnHintAlignReg;
191 Align ReturnHintAlign;
192
193 Info.OrigRet = ArgInfo{ResRegs, RetTy, 0, getAttributesForReturn(Call: CB)};
194
195 if (!Info.OrigRet.Ty->isVoidTy()) {
196 setArgFlags(Arg&: Info.OrigRet, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: CB);
197
198 if (MaybeAlign Alignment = CB.getRetAlign()) {
199 if (*Alignment > Align(1)) {
200 ReturnHintAlignReg = MRI.cloneVirtualRegister(VReg: ResRegs[0]);
201 Info.OrigRet.Regs[0] = ReturnHintAlignReg;
202 ReturnHintAlign = *Alignment;
203 }
204 }
205 }
206
207 auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_kcfi);
208 if (Bundle && CB.isIndirectCall()) {
209 Info.CFIType = cast<ConstantInt>(Val: Bundle->Inputs[0]);
210 assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
211 }
212
213 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_deactivation_symbol)) {
214 Info.DeactivationSymbol = cast<GlobalValue>(Val: Bundle->Inputs[0]);
215 }
216
217 Info.CB = &CB;
218 Info.KnownCallees = CB.getMetadata(KindID: LLVMContext::MD_callees);
219 Info.CallConv = CallConv;
220 Info.SwiftErrorVReg = SwiftErrorVReg;
221 Info.PAI = PAI;
222 Info.ConvergenceCtrlToken = ConvergenceCtrlToken;
223 Info.IsMustTailCall = CB.isMustTailCall();
224 Info.IsTailCall = CanBeTailCalled;
225 Info.IsVarArg = IsVarArg;
226 if (!lowerCall(MIRBuilder, Info))
227 return false;
228
229 if (ReturnHintAlignReg && !Info.LoweredTailCall) {
230 MIRBuilder.buildAssertAlign(Res: ResRegs[0], Op: ReturnHintAlignReg,
231 AlignVal: ReturnHintAlign);
232 }
233
234 return true;
235}
236
237template <typename FuncInfoTy>
238void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
239 const DataLayout &DL,
240 const FuncInfoTy &FuncInfo) const {
241 auto &Flags = Arg.Flags[0];
242 const AttributeList &Attrs = FuncInfo.getAttributes();
243 addArgFlagsFromAttributes(Flags, Attrs, OpIdx);
244
245 PointerType *PtrTy = dyn_cast<PointerType>(Val: Arg.Ty->getScalarType());
246 if (PtrTy) {
247 Flags.setPointer();
248 Flags.setPointerAddrSpace(PtrTy->getPointerAddressSpace());
249 }
250
251 Align MemAlign = DL.getABITypeAlign(Ty: Arg.Ty);
252 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
253 Flags.isByRef()) {
254 assert(OpIdx >= AttributeList::FirstArgIndex);
255 unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex;
256
257 Type *ElementTy = FuncInfo.getParamByValType(ParamIdx);
258 if (!ElementTy)
259 ElementTy = FuncInfo.getParamByRefType(ParamIdx);
260 if (!ElementTy)
261 ElementTy = FuncInfo.getParamInAllocaType(ParamIdx);
262 if (!ElementTy)
263 ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx);
264
265 assert(ElementTy && "Must have byval, inalloca or preallocated type");
266
267 uint64_t MemSize = DL.getTypeAllocSize(Ty: ElementTy);
268 if (Flags.isByRef())
269 Flags.setByRefSize(MemSize);
270 else
271 Flags.setByValSize(MemSize);
272
273 // For ByVal, alignment should be passed from FE. BE will guess if
274 // this info is not there but there are cases it cannot get right.
275 if (auto ParamAlign = FuncInfo.getParamStackAlign(ParamIdx))
276 MemAlign = *ParamAlign;
277 else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx)))
278 MemAlign = *ParamAlign;
279 else
280 MemAlign = getTLI()->getByValTypeAlignment(Ty: ElementTy, DL);
281 } else if (OpIdx >= AttributeList::FirstArgIndex) {
282 if (auto ParamAlign =
283 FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex))
284 MemAlign = *ParamAlign;
285 }
286 Flags.setMemAlign(MemAlign);
287 Flags.setOrigAlign(DL.getABITypeAlign(Ty: Arg.Ty));
288
289 // Don't try to use the returned attribute if the argument is marked as
290 // swiftself, since it won't be passed in x0.
291 if (Flags.isSwiftSelf())
292 Flags.setReturned(false);
293}
294
295template void
296CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
297 const DataLayout &DL,
298 const Function &FuncInfo) const;
299
300template void
301CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
302 const DataLayout &DL,
303 const CallBase &FuncInfo) const;
304
305void CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
306 SmallVectorImpl<ArgInfo> &SplitArgs,
307 const DataLayout &DL,
308 CallingConv::ID CallConv,
309 SmallVectorImpl<TypeSize> *Offsets) const {
310 SmallVector<Type *, 4> SplitTys;
311 ComputeValueTypes(DL, Ty: OrigArg.Ty, Types&: SplitTys, Offsets);
312
313 if (SplitTys.size() == 0)
314 return;
315
316 if (SplitTys.size() == 1) {
317 // No splitting to do, but we want to replace the original type (e.g. [1 x
318 // double] -> double).
319 SplitArgs.emplace_back(Args: OrigArg.Regs[0], Args&: SplitTys[0], Args: OrigArg.OrigArgIndex,
320 Args: OrigArg.Flags[0], Args: OrigArg.OrigValue);
321 return;
322 }
323
324 // Create one ArgInfo for each virtual register in the original ArgInfo.
325 assert(OrigArg.Regs.size() == SplitTys.size() && "Regs / types mismatch");
326
327 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
328 Ty: OrigArg.Ty, CallConv, isVarArg: false, DL);
329 for (unsigned i = 0, e = SplitTys.size(); i < e; ++i) {
330 SplitArgs.emplace_back(Args: OrigArg.Regs[i], Args&: SplitTys[i], Args: OrigArg.OrigArgIndex,
331 Args: OrigArg.Flags[0]);
332 if (NeedsRegBlock)
333 SplitArgs.back().Flags[0].setInConsecutiveRegs();
334 }
335
336 SplitArgs.back().Flags[0].setInConsecutiveRegsLast();
337}
338
339/// Pack values \p SrcRegs to cover the vector type result \p DstRegs.
340static MachineInstrBuilder
341mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
342 ArrayRef<Register> SrcRegs) {
343 MachineRegisterInfo &MRI = *B.getMRI();
344 LLT LLTy = MRI.getType(Reg: DstRegs[0]);
345 LLT PartLLT = MRI.getType(Reg: SrcRegs[0]);
346
347 // Deal with v3s16 split into v2s16
348 LLT LCMTy = getCoverTy(OrigTy: LLTy, TargetTy: PartLLT);
349 if (LCMTy == LLTy) {
350 // Common case where no padding is needed.
351 assert(DstRegs.size() == 1);
352
353 SmallVector<Register, 8> ConcatRegs(SrcRegs.size());
354 llvm::copy(Range&: SrcRegs, Out: ConcatRegs.begin());
355
356 if (LLTy.getScalarType() != PartLLT.getScalarType())
357 for (size_t I = 0, E = SrcRegs.size(); I != E; ++I) {
358 auto BitcastDst =
359 MRI.getType(Reg: SrcRegs[I]).changeElementType(NewEltTy: LLTy.getScalarType());
360 ConcatRegs[I] = B.buildBitcast(Dst: BitcastDst, Src: SrcRegs[I]).getReg(Idx: 0);
361 }
362
363 return B.buildConcatVectors(Res: DstRegs[0], Ops: ConcatRegs);
364 }
365
366 // We need to create an unmerge to the result registers, which may require
367 // widening the original value.
368 Register UnmergeSrcReg;
369 if (LCMTy.getSizeInBits() != PartLLT.getSizeInBits()) {
370 assert(DstRegs.size() == 1);
371 return B.buildDeleteTrailingVectorElements(
372 Res: DstRegs[0], Op0: B.buildMergeLikeInstr(Res: LCMTy, Ops: SrcRegs));
373 } else {
374 // We don't need to widen anything if we're extracting a scalar which was
375 // promoted to a vector e.g. s8 -> v4s8 -> s8
376 assert(SrcRegs.size() == 1);
377 UnmergeSrcReg = SrcRegs[0];
378 }
379
380 size_t NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits();
381
382 SmallVector<Register, 8> PadDstRegs(NumDst);
383 llvm::copy(Range&: DstRegs, Out: PadDstRegs.begin());
384
385 // Create the excess dead defs for the unmerge.
386 for (size_t I = DstRegs.size(); I != NumDst; ++I)
387 PadDstRegs[I] = MRI.createGenericVirtualRegister(Ty: LLTy);
388
389 if (PartLLT != LCMTy)
390 UnmergeSrcReg = B.buildBitcast(Dst: LCMTy, Src: UnmergeSrcReg).getReg(Idx: 0);
391
392 if (PadDstRegs.size() == 1)
393 return B.buildDeleteTrailingVectorElements(Res: DstRegs[0], Op0: UnmergeSrcReg);
394 return B.buildUnmerge(Res: PadDstRegs, Op: UnmergeSrcReg);
395}
396
397void CallLowering::buildCopyFromRegs(MachineIRBuilder &B,
398 ArrayRef<Register> OrigRegs,
399 ArrayRef<Register> Regs, LLT LLTy,
400 LLT PartLLT, const ISD::ArgFlagsTy Flags) {
401 MachineRegisterInfo &MRI = *B.getMRI();
402
403 if (PartLLT == LLTy) {
404 // We should have avoided introducing a new virtual register, and just
405 // directly assigned here.
406 assert(OrigRegs[0] == Regs[0]);
407 return;
408 }
409
410 if (PartLLT.getSizeInBits() == LLTy.getSizeInBits() && OrigRegs.size() == 1 &&
411 Regs.size() == 1) {
412 B.buildBitcast(Dst: OrigRegs[0], Src: Regs[0]);
413 return;
414 }
415
416 // A vector PartLLT needs extending to LLTy's element size.
417 // E.g. <2 x s64> = G_SEXT <2 x s32>.
418 if (PartLLT.isVector() == LLTy.isVector() &&
419 PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits() &&
420 (!PartLLT.isVector() ||
421 PartLLT.getElementCount() == LLTy.getElementCount()) &&
422 OrigRegs.size() == 1 && Regs.size() == 1) {
423 Register SrcReg = Regs[0];
424
425 LLT LocTy = MRI.getType(Reg: SrcReg);
426
427 if (Flags.isSExt()) {
428 SrcReg = B.buildAssertSExt(Res: LocTy, Op: SrcReg, Size: LLTy.getScalarSizeInBits())
429 .getReg(Idx: 0);
430 } else if (Flags.isZExt()) {
431 SrcReg = B.buildAssertZExt(Res: LocTy, Op: SrcReg, Size: LLTy.getScalarSizeInBits())
432 .getReg(Idx: 0);
433 }
434
435 // Sometimes pointers are passed zero extended.
436 LLT OrigTy = MRI.getType(Reg: OrigRegs[0]);
437 if (OrigTy.isPointer()) {
438 LLT IntPtrTy = LLT::scalar(SizeInBits: OrigTy.getSizeInBits());
439 B.buildIntToPtr(Dst: OrigRegs[0], Src: B.buildTrunc(Res: IntPtrTy, Op: SrcReg));
440 return;
441 }
442
443 B.buildTrunc(Res: OrigRegs[0], Op: SrcReg);
444 return;
445 }
446
447 if (!LLTy.isVector() && !PartLLT.isVector()) {
448 assert(OrigRegs.size() == 1);
449 LLT OrigTy = MRI.getType(Reg: OrigRegs[0]);
450
451 unsigned SrcSize = PartLLT.getSizeInBits().getFixedValue() * Regs.size();
452 if (SrcSize == OrigTy.getSizeInBits())
453 B.buildMergeValues(Res: OrigRegs[0], Ops: Regs);
454 else {
455 auto Widened = B.buildMergeLikeInstr(Res: LLT::scalar(SizeInBits: SrcSize), Ops: Regs);
456 B.buildTrunc(Res: OrigRegs[0], Op: Widened);
457 }
458
459 return;
460 }
461
462 if (PartLLT.isVector()) {
463 assert(OrigRegs.size() == 1);
464 SmallVector<Register> CastRegs(Regs);
465
466 // If PartLLT is a mismatched vector in both number of elements and element
467 // size, e.g. PartLLT == v2s64 and LLTy is v3s32, then first coerce it to
468 // have the same elt type, i.e. v4s32.
469 // TODO: Extend this coersion to element multiples other than just 2.
470 if (TypeSize::isKnownGT(LHS: PartLLT.getSizeInBits(), RHS: LLTy.getSizeInBits()) &&
471 PartLLT.getScalarSizeInBits() == LLTy.getScalarSizeInBits() * 2 &&
472 Regs.size() == 1) {
473 LLT NewTy = PartLLT.changeElementType(NewEltTy: LLTy.getElementType())
474 .changeElementCount(EC: PartLLT.getElementCount() * 2);
475 CastRegs[0] = B.buildBitcast(Dst: NewTy, Src: Regs[0]).getReg(Idx: 0);
476 PartLLT = NewTy;
477 }
478
479 if (LLTy.getScalarSizeInBits() == PartLLT.getScalarSizeInBits()) {
480 mergeVectorRegsToResultRegs(B, DstRegs: OrigRegs, SrcRegs: CastRegs);
481 } else {
482 unsigned I = 0;
483 LLT GCDTy = getGCDType(OrigTy: LLTy, TargetTy: PartLLT);
484
485 // We are both splitting a vector, and bitcasting its element types. Cast
486 // the source pieces into the appropriate number of pieces with the result
487 // element type.
488 for (Register SrcReg : CastRegs)
489 CastRegs[I++] = B.buildBitcast(Dst: GCDTy, Src: SrcReg).getReg(Idx: 0);
490 mergeVectorRegsToResultRegs(B, DstRegs: OrigRegs, SrcRegs: CastRegs);
491 }
492
493 return;
494 }
495
496 assert(LLTy.isVector() && !PartLLT.isVector());
497
498 LLT DstEltTy = LLTy.getElementType();
499
500 // Pointer information was discarded. We'll need to coerce some register types
501 // to avoid violating type constraints.
502 LLT RealDstEltTy = MRI.getType(Reg: OrigRegs[0]).getElementType();
503
504 assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits());
505
506 if (DstEltTy == PartLLT) {
507 // Vector was trivially scalarized.
508
509 if (RealDstEltTy.isPointer()) {
510 for (Register Reg : Regs)
511 MRI.setType(VReg: Reg, Ty: RealDstEltTy);
512 }
513
514 B.buildBuildVector(Res: OrigRegs[0], Ops: Regs);
515 } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) {
516 // Deal with vector with 64-bit elements decomposed to 32-bit
517 // registers. Need to create intermediate 64-bit elements.
518 SmallVector<Register, 8> EltMerges;
519 int PartsPerElt =
520 divideCeil(Numerator: DstEltTy.getSizeInBits(), Denominator: PartLLT.getSizeInBits());
521 LLT ExtendedPartTy = LLT::scalar(SizeInBits: PartLLT.getSizeInBits() * PartsPerElt);
522
523 for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) {
524 auto Merge =
525 B.buildMergeLikeInstr(Res: ExtendedPartTy, Ops: Regs.take_front(N: PartsPerElt));
526 if (ExtendedPartTy.getSizeInBits() > RealDstEltTy.getSizeInBits())
527 Merge = B.buildTrunc(Res: RealDstEltTy, Op: Merge);
528 // Fix the type in case this is really a vector of pointers.
529 MRI.setType(VReg: Merge.getReg(Idx: 0), Ty: RealDstEltTy);
530 EltMerges.push_back(Elt: Merge.getReg(Idx: 0));
531 Regs = Regs.drop_front(N: PartsPerElt);
532 }
533
534 B.buildBuildVector(Res: OrigRegs[0], Ops: EltMerges);
535 } else {
536 // Vector was split, and elements promoted to a wider type.
537 // FIXME: Should handle floating point promotions.
538 unsigned NumElts = LLTy.getNumElements();
539 LLT BVType = LLT::fixed_vector(NumElements: NumElts, ScalarTy: PartLLT);
540
541 Register BuildVec;
542 if (NumElts == Regs.size())
543 BuildVec = B.buildBuildVector(Res: BVType, Ops: Regs).getReg(Idx: 0);
544 else {
545 // Vector elements are packed in the inputs.
546 // e.g. we have a <4 x s16> but 2 x s32 in regs.
547 assert(NumElts > Regs.size());
548 LLT SrcEltTy = MRI.getType(Reg: Regs[0]);
549
550 LLT OriginalEltTy = MRI.getType(Reg: OrigRegs[0]).getElementType();
551
552 // Input registers contain packed elements.
553 // Determine how many elements per reg.
554 assert((SrcEltTy.getSizeInBits() % OriginalEltTy.getSizeInBits()) == 0);
555 unsigned EltPerReg =
556 (SrcEltTy.getSizeInBits() / OriginalEltTy.getSizeInBits());
557
558 SmallVector<Register, 0> BVRegs;
559 BVRegs.reserve(N: Regs.size() * EltPerReg);
560 for (Register R : Regs) {
561 auto Unmerge = B.buildUnmerge(Res: OriginalEltTy, Op: R);
562 for (unsigned K = 0; K < EltPerReg; ++K)
563 BVRegs.push_back(Elt: B.buildAnyExt(Res: PartLLT, Op: Unmerge.getReg(Idx: K)).getReg(Idx: 0));
564 }
565
566 // We may have some more elements in BVRegs, e.g. if we have 2 s32 pieces
567 // for a <3 x s16> vector. We should have less than EltPerReg extra items.
568 if (BVRegs.size() > NumElts) {
569 assert((BVRegs.size() - NumElts) < EltPerReg);
570 BVRegs.truncate(N: NumElts);
571 }
572 BuildVec = B.buildBuildVector(Res: BVType, Ops: BVRegs).getReg(Idx: 0);
573 }
574 B.buildTrunc(Res: OrigRegs[0], Op: BuildVec);
575 }
576}
577
578void CallLowering::buildCopyToRegs(MachineIRBuilder &B,
579 ArrayRef<Register> DstRegs, Register SrcReg,
580 LLT SrcTy, LLT PartTy, unsigned ExtendOp) {
581 // We could just insert a regular copy, but this is unreachable at the moment.
582 assert(SrcTy != PartTy && "identical part types shouldn't reach here");
583
584 const TypeSize PartSize = PartTy.getSizeInBits();
585
586 if (PartSize == SrcTy.getSizeInBits() && DstRegs.size() == 1) {
587 // TODO: Handle int<->ptr casts. It just happens the ABI lowering
588 // assignments are not pointer aware.
589 B.buildBitcast(Dst: DstRegs[0], Src: SrcReg);
590 return;
591 }
592
593 if (PartTy.isVector() == SrcTy.isVector() &&
594 PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) {
595 assert(DstRegs.size() == 1);
596 B.buildInstr(Opc: ExtendOp, DstOps: {DstRegs[0]}, SrcOps: {SrcReg});
597 return;
598 }
599
600 if (SrcTy.isVector() && !PartTy.isVector() &&
601 TypeSize::isKnownGT(LHS: PartSize, RHS: SrcTy.getElementType().getSizeInBits()) &&
602 SrcTy.getElementCount() == ElementCount::getFixed(MinVal: DstRegs.size())) {
603 // Vector was scalarized, and the elements extended.
604 auto UnmergeToEltTy = B.buildUnmerge(Res: SrcTy.getElementType(), Op: SrcReg);
605 for (int i = 0, e = DstRegs.size(); i != e; ++i)
606 B.buildAnyExt(Res: DstRegs[i], Op: UnmergeToEltTy.getReg(Idx: i));
607 return;
608 }
609
610 if (SrcTy.isVector() && PartTy.isVector() &&
611 PartTy.getSizeInBits() == SrcTy.getSizeInBits() &&
612 ElementCount::isKnownLT(LHS: SrcTy.getElementCount(),
613 RHS: PartTy.getElementCount())) {
614 // A coercion like: v2f32 -> v4f32 or nxv2f32 -> nxv4f32
615 Register DstReg = DstRegs.front();
616 B.buildPadVectorWithUndefElements(Res: DstReg, Op0: SrcReg);
617 return;
618 }
619
620 LLT GCDTy = getGCDType(OrigTy: SrcTy, TargetTy: PartTy);
621 if (GCDTy == PartTy) {
622 // If this already evenly divisible, we can create a simple unmerge.
623 B.buildUnmerge(Res: DstRegs, Op: SrcReg);
624 return;
625 }
626
627 if (SrcTy.isVector() && !PartTy.isVector() &&
628 SrcTy.getScalarSizeInBits() > PartTy.getSizeInBits()) {
629 LLT ExtTy =
630 LLT::vector(EC: SrcTy.getElementCount(),
631 ScalarTy: LLT::scalar(SizeInBits: PartTy.getScalarSizeInBits() * DstRegs.size() /
632 SrcTy.getNumElements()));
633 auto Ext = B.buildAnyExt(Res: ExtTy, Op: SrcReg);
634 B.buildUnmerge(Res: DstRegs, Op: Ext);
635 return;
636 }
637
638 MachineRegisterInfo &MRI = *B.getMRI();
639 LLT DstTy = MRI.getType(Reg: DstRegs[0]);
640 LLT CoverTy = getCoverTy(OrigTy: SrcTy, TargetTy: PartTy);
641 if (SrcTy.isVector() && DstRegs.size() > 1) {
642 TypeSize FullCoverSize =
643 DstTy.getSizeInBits().multiplyCoefficientBy(RHS: DstRegs.size());
644
645 LLT EltTy = SrcTy.getElementType();
646 TypeSize EltSize = EltTy.getSizeInBits();
647 if (FullCoverSize.isKnownMultipleOf(RHS: EltSize)) {
648 TypeSize VecSize = FullCoverSize.divideCoefficientBy(RHS: EltSize);
649 CoverTy =
650 LLT::vector(EC: ElementCount::get(MinVal: VecSize, Scalable: VecSize.isScalable()), ScalarTy: EltTy);
651 }
652 }
653
654 if (PartTy.isVector() && CoverTy == PartTy) {
655 assert(DstRegs.size() == 1);
656 B.buildPadVectorWithUndefElements(Res: DstRegs[0], Op0: SrcReg);
657 return;
658 }
659
660 const unsigned DstSize = DstTy.getSizeInBits();
661 const unsigned SrcSize = SrcTy.getSizeInBits();
662 unsigned CoveringSize = CoverTy.getSizeInBits();
663
664 Register UnmergeSrc = SrcReg;
665
666 if (!CoverTy.isVector() && CoveringSize != SrcSize) {
667 // For scalars, it's common to be able to use a simple extension.
668 if (SrcTy.isScalar() && DstTy.isScalar()) {
669 CoveringSize = alignTo(Value: SrcSize, Align: DstSize);
670 LLT CoverTy = LLT::scalar(SizeInBits: CoveringSize);
671 UnmergeSrc = B.buildInstr(Opc: ExtendOp, DstOps: {CoverTy}, SrcOps: {SrcReg}).getReg(Idx: 0);
672 } else {
673 // Widen to the common type.
674 // FIXME: This should respect the extend type
675 Register Undef = B.buildUndef(Res: SrcTy).getReg(Idx: 0);
676 SmallVector<Register, 8> MergeParts(1, SrcReg);
677 for (unsigned Size = SrcSize; Size != CoveringSize; Size += SrcSize)
678 MergeParts.push_back(Elt: Undef);
679 UnmergeSrc = B.buildMergeLikeInstr(Res: CoverTy, Ops: MergeParts).getReg(Idx: 0);
680 }
681 }
682
683 if (CoverTy.isVector() && CoveringSize != SrcSize)
684 UnmergeSrc = B.buildPadVectorWithUndefElements(Res: CoverTy, Op0: SrcReg).getReg(Idx: 0);
685
686 B.buildUnmerge(Res: DstRegs, Op: UnmergeSrc);
687}
688
689bool CallLowering::determineAndHandleAssignments(
690 ValueHandler &Handler, ValueAssigner &Assigner,
691 SmallVectorImpl<ArgInfo> &Args, MachineIRBuilder &MIRBuilder,
692 CallingConv::ID CallConv, bool IsVarArg,
693 ArrayRef<Register> ThisReturnRegs) const {
694 MachineFunction &MF = MIRBuilder.getMF();
695 const Function &F = MF.getFunction();
696 SmallVector<CCValAssign, 16> ArgLocs;
697
698 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext());
699 if (!determineAssignments(Assigner, Args, CCInfo))
700 return false;
701
702 return handleAssignments(Handler, Args, CCState&: CCInfo, ArgLocs, MIRBuilder,
703 ThisReturnRegs);
704}
705
706static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) {
707 if (Flags.isSExt())
708 return TargetOpcode::G_SEXT;
709 if (Flags.isZExt())
710 return TargetOpcode::G_ZEXT;
711 return TargetOpcode::G_ANYEXT;
712}
713
714bool CallLowering::determineAssignments(ValueAssigner &Assigner,
715 SmallVectorImpl<ArgInfo> &Args,
716 CCState &CCInfo) const {
717 LLVMContext &Ctx = CCInfo.getContext();
718 const DataLayout &DL = CCInfo.getMachineFunction().getDataLayout();
719 const CallingConv::ID CallConv = CCInfo.getCallingConv();
720
721 unsigned NumArgs = Args.size();
722 for (unsigned i = 0; i != NumArgs; ++i) {
723 EVT CurVT = TLI->getValueType(DL, Ty: Args[i].Ty);
724
725 MVT NewVT = TLI->getRegisterTypeForCallingConv(Context&: Ctx, CC: CallConv, VT: CurVT);
726
727 // If we need to split the type over multiple regs, check it's a scenario
728 // we currently support.
729 unsigned NumParts =
730 TLI->getNumRegistersForCallingConv(Context&: Ctx, CC: CallConv, VT: CurVT);
731
732 if (NumParts == 1) {
733 // Try to use the register type if we couldn't assign the VT.
734 if (Assigner.assignArg(ValNo: i, OrigVT: CurVT, ValVT: NewVT, LocVT: NewVT, LocInfo: CCValAssign::Full, Info: Args[i],
735 Flags: Args[i].Flags[0], State&: CCInfo))
736 return false;
737 continue;
738 }
739
740 // For incoming arguments (physregs to vregs), we could have values in
741 // physregs (or memlocs) which we want to extract and copy to vregs.
742 // During this, we might have to deal with the LLT being split across
743 // multiple regs, so we have to record this information for later.
744 //
745 // If we have outgoing args, then we have the opposite case. We have a
746 // vreg with an LLT which we want to assign to a physical location, and
747 // we might have to record that the value has to be split later.
748
749 // We're handling an incoming arg which is split over multiple regs.
750 // E.g. passing an s128 on AArch64.
751 ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
752 Args[i].Flags.clear();
753
754 for (unsigned Part = 0; Part < NumParts; ++Part) {
755 ISD::ArgFlagsTy Flags = OrigFlags;
756 if (Part == 0) {
757 Flags.setSplit();
758 } else {
759 Flags.setOrigAlign(Align(1));
760 if (Part == NumParts - 1)
761 Flags.setSplitEnd();
762 }
763
764 Args[i].Flags.push_back(Elt: Flags);
765 if (Assigner.assignArg(ValNo: i, OrigVT: CurVT, ValVT: NewVT, LocVT: NewVT, LocInfo: CCValAssign::Full, Info: Args[i],
766 Flags: Args[i].Flags[Part], State&: CCInfo)) {
767 // Still couldn't assign this smaller part type for some reason.
768 return false;
769 }
770 }
771 }
772
773 return true;
774}
775
776bool CallLowering::handleAssignments(ValueHandler &Handler,
777 SmallVectorImpl<ArgInfo> &Args,
778 CCState &CCInfo,
779 SmallVectorImpl<CCValAssign> &ArgLocs,
780 MachineIRBuilder &MIRBuilder,
781 ArrayRef<Register> ThisReturnRegs) const {
782 MachineFunction &MF = MIRBuilder.getMF();
783 MachineRegisterInfo &MRI = MF.getRegInfo();
784 const Function &F = MF.getFunction();
785 const DataLayout &DL = F.getDataLayout();
786
787 const unsigned NumArgs = Args.size();
788
789 // Stores thunks for outgoing register assignments. This is used so we delay
790 // generating register copies until mem loc assignments are done. We do this
791 // so that if the target is using the delayed stack protector feature, we can
792 // find the split point of the block accurately. E.g. if we have:
793 // G_STORE %val, %memloc
794 // $x0 = COPY %foo
795 // $x1 = COPY %bar
796 // CALL func
797 // ... then the split point for the block will correctly be at, and including,
798 // the copy to $x0. If instead the G_STORE instruction immediately precedes
799 // the CALL, then we'd prematurely choose the CALL as the split point, thus
800 // generating a split block with a CALL that uses undefined physregs.
801 SmallVector<std::function<void()>> DelayedOutgoingRegAssignments;
802
803 for (unsigned i = 0, j = 0; i != NumArgs; ++i, ++j) {
804 assert(j < ArgLocs.size() && "Skipped too many arg locs");
805 CCValAssign &VA = ArgLocs[j];
806 assert(VA.getValNo() == i && "Location doesn't correspond to current arg");
807
808 if (VA.needsCustom()) {
809 std::function<void()> Thunk;
810 unsigned NumArgRegs = Handler.assignCustomValue(
811 Arg&: Args[i], VAs: ArrayRef(ArgLocs).slice(N: j), Thunk: &Thunk);
812 if (Thunk)
813 DelayedOutgoingRegAssignments.emplace_back(Args&: Thunk);
814 if (!NumArgRegs)
815 return false;
816 j += (NumArgRegs - 1);
817 continue;
818 }
819
820 auto AllocaAddressSpace = MF.getDataLayout().getAllocaAddrSpace();
821
822 const MVT ValVT = VA.getValVT();
823 const MVT LocVT = VA.getLocVT();
824
825 const LLT LocTy = getLLTForMVT(Ty: LocVT);
826 const LLT ValTy = getLLTForMVT(Ty: ValVT);
827 const LLT NewLLT = Handler.isIncomingArgumentHandler() ? LocTy : ValTy;
828 const EVT OrigVT = TLI->getValueType(DL, Ty: Args[i].Ty);
829 // Use the EVT here to strip pointerness.
830 const LLT OrigTy = getLLTForType(Ty&: *OrigVT.getTypeForEVT(Context&: F.getContext()), DL);
831 const LLT PointerTy = LLT::pointer(
832 AddressSpace: AllocaAddressSpace, SizeInBits: DL.getPointerSizeInBits(AS: AllocaAddressSpace));
833
834 // Expected to be multiple regs for a single incoming arg.
835 // There should be Regs.size() ArgLocs per argument.
836 // This should be the same as getNumRegistersForCallingConv
837 const unsigned NumParts = Args[i].Flags.size();
838
839 // Now split the registers into the assigned types.
840 Args[i].OrigRegs.assign(in_start: Args[i].Regs.begin(), in_end: Args[i].Regs.end());
841
842 if (NumParts != 1 || NewLLT != OrigTy) {
843 // If we can't directly assign the register, we need one or more
844 // intermediate values.
845 Args[i].Regs.resize(N: NumParts);
846
847 // When we have indirect parameter passing we are receiving a pointer,
848 // that points to the actual value, so we need one "temporary" pointer.
849 if (VA.getLocInfo() == CCValAssign::Indirect) {
850 if (Handler.isIncomingArgumentHandler())
851 Args[i].Regs[0] = MRI.createGenericVirtualRegister(Ty: PointerTy);
852 } else {
853 // For each split register, create and assign a vreg that will store
854 // the incoming component of the larger value. These will later be
855 // merged to form the final vreg.
856 for (unsigned Part = 0; Part < NumParts; ++Part)
857 Args[i].Regs[Part] = MRI.createGenericVirtualRegister(Ty: NewLLT);
858 }
859 }
860
861 assert((j + (NumParts - 1)) < ArgLocs.size() &&
862 "Too many regs for number of args");
863
864 // Coerce into outgoing value types before register assignment.
865 if (!Handler.isIncomingArgumentHandler() && OrigTy != ValTy &&
866 VA.getLocInfo() != CCValAssign::Indirect) {
867 assert(Args[i].OrigRegs.size() == 1);
868 buildCopyToRegs(B&: MIRBuilder, DstRegs: Args[i].Regs, SrcReg: Args[i].OrigRegs[0], SrcTy: OrigTy,
869 PartTy: ValTy, ExtendOp: extendOpFromFlags(Flags: Args[i].Flags[0]));
870 }
871
872 bool IndirectParameterPassingHandled = false;
873 bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(VT: OrigVT, DL);
874 for (unsigned Part = 0; Part < NumParts; ++Part) {
875 assert((VA.getLocInfo() != CCValAssign::Indirect || Part == 0) &&
876 "Only the first parameter should be processed when "
877 "handling indirect passing!");
878 Register ArgReg = Args[i].Regs[Part];
879 // There should be Regs.size() ArgLocs per argument.
880 unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part;
881 CCValAssign &VA = ArgLocs[j + Idx];
882 const ISD::ArgFlagsTy Flags = Args[i].Flags[Part];
883
884 // We found an indirect parameter passing, and we have an
885 // OutgoingValueHandler as our handler (so we are at the call site or the
886 // return value). In this case, start the construction of the following
887 // GMIR, that is responsible for the preparation of indirect parameter
888 // passing:
889 //
890 // %1(indirectly passed type) = The value to pass
891 // %3(pointer) = G_FRAME_INDEX %stack.0
892 // G_STORE %1, %3 :: (store (s128), align 8)
893 //
894 // After this GMIR, the remaining part of the loop body will decide how
895 // to get the value to the caller and we break out of the loop.
896 if (VA.getLocInfo() == CCValAssign::Indirect &&
897 !Handler.isIncomingArgumentHandler()) {
898 Align AlignmentForStored = DL.getPrefTypeAlign(Ty: Args[i].Ty);
899 MachineFrameInfo &MFI = MF.getFrameInfo();
900 // Get some space on the stack for the value, so later we can pass it
901 // as a reference.
902 int FrameIdx = MFI.CreateStackObject(Size: OrigTy.getScalarSizeInBits(),
903 Alignment: AlignmentForStored, isSpillSlot: false);
904 Register PointerToStackReg =
905 MIRBuilder.buildFrameIndex(Res: PointerTy, Idx: FrameIdx).getReg(Idx: 0);
906 MachinePointerInfo StackPointerMPO =
907 MachinePointerInfo::getFixedStack(MF, FI: FrameIdx);
908 // Store the value in the previously created stack space.
909 MIRBuilder.buildStore(Val: Args[i].OrigRegs[Part], Addr: PointerToStackReg,
910 PtrInfo: StackPointerMPO,
911 Alignment: inferAlignFromPtrInfo(MF, MPO: StackPointerMPO));
912
913 ArgReg = PointerToStackReg;
914 IndirectParameterPassingHandled = true;
915 }
916
917 if (VA.isMemLoc() && !Flags.isByVal()) {
918 // Individual pieces may have been spilled to the stack and others
919 // passed in registers.
920
921 // TODO: The memory size may be larger than the value we need to
922 // store. We may need to adjust the offset for big endian targets.
923 LLT MemTy = Handler.getStackValueStoreType(DL, VA, Flags);
924
925 MachinePointerInfo MPO;
926 Register StackAddr =
927 Handler.getStackAddress(MemSize: VA.getLocInfo() == CCValAssign::Indirect
928 ? PointerTy.getSizeInBytes()
929 : MemTy.getSizeInBytes(),
930 Offset: VA.getLocMemOffset(), MPO, Flags);
931
932 // Finish the handling of indirect passing from the passers
933 // (OutgoingParameterHandler) side.
934 // This branch is needed, so the pointer to the value is loaded onto the
935 // stack.
936 if (VA.getLocInfo() == CCValAssign::Indirect)
937 Handler.assignValueToAddress(ValVReg: ArgReg, Addr: StackAddr, MemTy: PointerTy, MPO, VA);
938 else
939 Handler.assignValueToAddress(Arg: Args[i], ValRegIndex: Part, Addr: StackAddr, MemTy, MPO,
940 VA);
941 } else if (VA.isMemLoc() && Flags.isByVal()) {
942 assert(Args[i].Regs.size() == 1 && "didn't expect split byval pointer");
943
944 if (Handler.isIncomingArgumentHandler()) {
945 // We just need to copy the frame index value to the pointer.
946 MachinePointerInfo MPO;
947 Register StackAddr = Handler.getStackAddress(
948 MemSize: Flags.getByValSize(), Offset: VA.getLocMemOffset(), MPO, Flags);
949 MIRBuilder.buildCopy(Res: Args[i].Regs[0], Op: StackAddr);
950 } else {
951 // For outgoing byval arguments, insert the implicit copy byval
952 // implies, such that writes in the callee do not modify the caller's
953 // value.
954 uint64_t MemSize = Flags.getByValSize();
955 int64_t Offset = VA.getLocMemOffset();
956
957 MachinePointerInfo DstMPO;
958 Register StackAddr =
959 Handler.getStackAddress(MemSize, Offset, MPO&: DstMPO, Flags);
960
961 MachinePointerInfo SrcMPO(Args[i].OrigValue);
962 if (!Args[i].OrigValue) {
963 // We still need to accurately track the stack address space if we
964 // don't know the underlying value.
965 const LLT PtrTy = MRI.getType(Reg: StackAddr);
966 SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace());
967 }
968
969 Align DstAlign = std::max(a: Flags.getNonZeroByValAlign(),
970 b: inferAlignFromPtrInfo(MF, MPO: DstMPO));
971
972 Align SrcAlign = std::max(a: Flags.getNonZeroByValAlign(),
973 b: inferAlignFromPtrInfo(MF, MPO: SrcMPO));
974
975 Handler.copyArgumentMemory(Arg: Args[i], DstPtr: StackAddr, SrcPtr: Args[i].Regs[0],
976 DstPtrInfo: DstMPO, DstAlign, SrcPtrInfo: SrcMPO, SrcAlign,
977 MemSize, VA);
978 }
979 } else if (i == 0 && !ThisReturnRegs.empty() &&
980 Handler.isIncomingArgumentHandler() &&
981 isTypeIsValidForThisReturn(Ty: ValVT)) {
982 Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: ThisReturnRegs[Part], VA, Flags);
983 } else if (Handler.isIncomingArgumentHandler()) {
984 Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: VA.getLocReg(), VA, Flags);
985 } else {
986 DelayedOutgoingRegAssignments.emplace_back(Args: [=, &Handler]() {
987 Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: VA.getLocReg(), VA, Flags);
988 });
989 }
990
991 // Finish the handling of indirect parameter passing when receiving
992 // the value (we are in the called function or the caller when receiving
993 // the return value).
994 if (VA.getLocInfo() == CCValAssign::Indirect &&
995 Handler.isIncomingArgumentHandler()) {
996 Align Alignment = DL.getABITypeAlign(Ty: Args[i].Ty);
997 MachinePointerInfo MPO = MachinePointerInfo::getUnknownStack(MF);
998
999 // Since we are doing indirect parameter passing, we know that the value
1000 // in the temporary register is not the value passed to the function,
1001 // but rather a pointer to that value. Let's load that value into the
1002 // virtual register where the parameter should go.
1003 MIRBuilder.buildLoad(Res: Args[i].OrigRegs[0], Addr: Args[i].Regs[0], PtrInfo: MPO,
1004 Alignment);
1005
1006 IndirectParameterPassingHandled = true;
1007 }
1008
1009 if (IndirectParameterPassingHandled)
1010 break;
1011 }
1012
1013 // Now that all pieces have been assigned, re-pack the register typed values
1014 // into the original value typed registers. This is only necessary, when
1015 // the value was passed in multiple registers, not indirectly.
1016 if (Handler.isIncomingArgumentHandler() && OrigVT != LocVT &&
1017 !IndirectParameterPassingHandled) {
1018 // Merge the split registers into the expected larger result vregs of
1019 // the original call.
1020 buildCopyFromRegs(B&: MIRBuilder, OrigRegs: Args[i].OrigRegs, Regs: Args[i].Regs, LLTy: OrigTy,
1021 PartLLT: LocTy, Flags: Args[i].Flags[0]);
1022 }
1023
1024 j += NumParts - 1;
1025 }
1026 for (auto &Fn : DelayedOutgoingRegAssignments)
1027 Fn();
1028
1029 return true;
1030}
1031
1032void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy,
1033 ArrayRef<Register> VRegs, Register DemoteReg,
1034 int FI) const {
1035 MachineFunction &MF = MIRBuilder.getMF();
1036 MachineRegisterInfo &MRI = MF.getRegInfo();
1037 const DataLayout &DL = MF.getDataLayout();
1038
1039 SmallVector<EVT, 4> SplitVTs;
1040 SmallVector<uint64_t, 4> Offsets;
1041 ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs, /*MemVTs=*/nullptr, FixedOffsets: &Offsets, StartingOffset: 0);
1042
1043 assert(VRegs.size() == SplitVTs.size());
1044
1045 unsigned NumValues = SplitVTs.size();
1046 Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy);
1047 Type *RetPtrTy =
1048 PointerType::get(C&: RetTy->getContext(), AddressSpace: DL.getAllocaAddrSpace());
1049 LLT OffsetLLTy = getLLTForType(Ty&: *DL.getIndexType(PtrTy: RetPtrTy), DL);
1050
1051 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
1052
1053 for (unsigned I = 0; I < NumValues; ++I) {
1054 Register Addr;
1055 MIRBuilder.materializeObjectPtrOffset(Res&: Addr, Op0: DemoteReg, ValueTy: OffsetLLTy,
1056 Value: Offsets[I]);
1057 auto *MMO = MF.getMachineMemOperand(PtrInfo, f: MachineMemOperand::MOLoad,
1058 MemTy: MRI.getType(Reg: VRegs[I]),
1059 base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[I]));
1060 MIRBuilder.buildLoad(Res: VRegs[I], Addr, MMO&: *MMO);
1061 }
1062}
1063
1064void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy,
1065 ArrayRef<Register> VRegs,
1066 Register DemoteReg) const {
1067 MachineFunction &MF = MIRBuilder.getMF();
1068 MachineRegisterInfo &MRI = MF.getRegInfo();
1069 const DataLayout &DL = MF.getDataLayout();
1070
1071 SmallVector<EVT, 4> SplitVTs;
1072 SmallVector<uint64_t, 4> Offsets;
1073 ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs, /*MemVTs=*/nullptr, FixedOffsets: &Offsets, StartingOffset: 0);
1074
1075 assert(VRegs.size() == SplitVTs.size());
1076
1077 unsigned NumValues = SplitVTs.size();
1078 Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy);
1079 unsigned AS = DL.getAllocaAddrSpace();
1080 LLT OffsetLLTy = getLLTForType(Ty&: *DL.getIndexType(C&: RetTy->getContext(), AddressSpace: AS), DL);
1081
1082 MachinePointerInfo PtrInfo(AS);
1083
1084 for (unsigned I = 0; I < NumValues; ++I) {
1085 Register Addr;
1086 MIRBuilder.materializeObjectPtrOffset(Res&: Addr, Op0: DemoteReg, ValueTy: OffsetLLTy,
1087 Value: Offsets[I]);
1088 auto *MMO = MF.getMachineMemOperand(PtrInfo, f: MachineMemOperand::MOStore,
1089 MemTy: MRI.getType(Reg: VRegs[I]),
1090 base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[I]));
1091 MIRBuilder.buildStore(Val: VRegs[I], Addr, MMO&: *MMO);
1092 }
1093}
1094
1095void CallLowering::insertSRetIncomingArgument(
1096 const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg,
1097 MachineRegisterInfo &MRI, const DataLayout &DL) const {
1098 unsigned AS = DL.getAllocaAddrSpace();
1099 DemoteReg = MRI.createGenericVirtualRegister(
1100 Ty: LLT::pointer(AddressSpace: AS, SizeInBits: DL.getPointerSizeInBits(AS)));
1101
1102 Type *PtrTy = PointerType::get(C&: F.getContext(), AddressSpace: AS);
1103
1104 SmallVector<EVT, 1> ValueVTs;
1105 ComputeValueVTs(TLI: *TLI, DL, Ty: PtrTy, ValueVTs);
1106
1107 // NOTE: Assume that a pointer won't get split into more than one VT.
1108 assert(ValueVTs.size() == 1);
1109
1110 ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(Context&: PtrTy->getContext()),
1111 ArgInfo::NoArgIndex);
1112 setArgFlags(Arg&: DemoteArg, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: F);
1113 DemoteArg.Flags[0].setSRet();
1114 SplitArgs.insert(I: SplitArgs.begin(), Elt: DemoteArg);
1115}
1116
1117void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder,
1118 const CallBase &CB,
1119 CallLoweringInfo &Info) const {
1120 const DataLayout &DL = MIRBuilder.getDataLayout();
1121 Type *RetTy = CB.getType();
1122 unsigned AS = DL.getAllocaAddrSpace();
1123 LLT FramePtrTy = LLT::pointer(AddressSpace: AS, SizeInBits: DL.getPointerSizeInBits(AS));
1124
1125 int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject(
1126 Size: DL.getTypeAllocSize(Ty: RetTy), Alignment: DL.getPrefTypeAlign(Ty: RetTy), isSpillSlot: false);
1127
1128 Register DemoteReg = MIRBuilder.buildFrameIndex(Res: FramePtrTy, Idx: FI).getReg(Idx: 0);
1129 ArgInfo DemoteArg(DemoteReg, PointerType::get(C&: RetTy->getContext(), AddressSpace: AS),
1130 ArgInfo::NoArgIndex);
1131 setArgFlags(Arg&: DemoteArg, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: CB);
1132 DemoteArg.Flags[0].setSRet();
1133
1134 Info.OrigArgs.insert(I: Info.OrigArgs.begin(), Elt: DemoteArg);
1135 Info.DemoteStackIndex = FI;
1136 Info.DemoteRegister = DemoteReg;
1137}
1138
1139bool CallLowering::checkReturn(CCState &CCInfo,
1140 SmallVectorImpl<BaseArgInfo> &Outs,
1141 CCAssignFn *Fn) const {
1142 for (unsigned I = 0, E = Outs.size(); I < E; ++I) {
1143 MVT VT = MVT::getVT(Ty: Outs[I].Ty);
1144 if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], Outs[I].Ty, CCInfo))
1145 return false;
1146 }
1147 return true;
1148}
1149
1150void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy,
1151 AttributeList Attrs,
1152 SmallVectorImpl<BaseArgInfo> &Outs,
1153 const DataLayout &DL) const {
1154 LLVMContext &Context = RetTy->getContext();
1155 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1156
1157 SmallVector<EVT, 4> SplitVTs;
1158 ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs);
1159 addArgFlagsFromAttributes(Flags, Attrs, OpIdx: AttributeList::ReturnIndex);
1160
1161 for (EVT VT : SplitVTs) {
1162 unsigned NumParts =
1163 TLI->getNumRegistersForCallingConv(Context, CC: CallConv, VT);
1164 MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CC: CallConv, VT);
1165 Type *PartTy = EVT(RegVT).getTypeForEVT(Context);
1166
1167 for (unsigned I = 0; I < NumParts; ++I) {
1168 Outs.emplace_back(Args&: PartTy, Args&: Flags);
1169 }
1170 }
1171}
1172
1173bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const {
1174 const auto &F = MF.getFunction();
1175 Type *ReturnType = F.getReturnType();
1176 CallingConv::ID CallConv = F.getCallingConv();
1177
1178 SmallVector<BaseArgInfo, 4> SplitArgs;
1179 getReturnInfo(CallConv, RetTy: ReturnType, Attrs: F.getAttributes(), Outs&: SplitArgs,
1180 DL: MF.getDataLayout());
1181 return canLowerReturn(MF, CallConv, Outs&: SplitArgs, IsVarArg: F.isVarArg());
1182}
1183
1184bool CallLowering::parametersInCSRMatch(
1185 const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask,
1186 const SmallVectorImpl<CCValAssign> &OutLocs,
1187 const SmallVectorImpl<ArgInfo> &OutArgs) const {
1188 for (unsigned i = 0; i < OutLocs.size(); ++i) {
1189 const auto &ArgLoc = OutLocs[i];
1190 // If it's not a register, it's fine.
1191 if (!ArgLoc.isRegLoc())
1192 continue;
1193
1194 MCRegister PhysReg = ArgLoc.getLocReg();
1195
1196 // Only look at callee-saved registers.
1197 if (MachineOperand::clobbersPhysReg(RegMask: CallerPreservedMask, PhysReg))
1198 continue;
1199
1200 LLVM_DEBUG(
1201 dbgs()
1202 << "... Call has an argument passed in a callee-saved register.\n");
1203
1204 // Check if it was copied from.
1205 const ArgInfo &OutInfo = OutArgs[i];
1206
1207 if (OutInfo.Regs.size() > 1) {
1208 LLVM_DEBUG(
1209 dbgs() << "... Cannot handle arguments in multiple registers.\n");
1210 return false;
1211 }
1212
1213 // Check if we copy the register, walking through copies from virtual
1214 // registers. Note that getDefIgnoringCopies does not ignore copies from
1215 // physical registers.
1216 MachineInstr *RegDef = getDefIgnoringCopies(Reg: OutInfo.Regs[0], MRI);
1217 if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) {
1218 LLVM_DEBUG(
1219 dbgs()
1220 << "... Parameter was not copied into a VReg, cannot tail call.\n");
1221 return false;
1222 }
1223
1224 // Got a copy. Verify that it's the same as the register we want.
1225 Register CopyRHS = RegDef->getOperand(i: 1).getReg();
1226 if (CopyRHS != PhysReg) {
1227 LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into "
1228 "VReg, cannot tail call.\n");
1229 return false;
1230 }
1231 }
1232
1233 return true;
1234}
1235
1236bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
1237 MachineFunction &MF,
1238 SmallVectorImpl<ArgInfo> &InArgs,
1239 ValueAssigner &CalleeAssigner,
1240 ValueAssigner &CallerAssigner) const {
1241 const Function &F = MF.getFunction();
1242 CallingConv::ID CalleeCC = Info.CallConv;
1243 CallingConv::ID CallerCC = F.getCallingConv();
1244
1245 if (CallerCC == CalleeCC)
1246 return true;
1247
1248 SmallVector<CCValAssign, 16> ArgLocs1;
1249 CCState CCInfo1(CalleeCC, Info.IsVarArg, MF, ArgLocs1, F.getContext());
1250 if (!determineAssignments(Assigner&: CalleeAssigner, Args&: InArgs, CCInfo&: CCInfo1))
1251 return false;
1252
1253 SmallVector<CCValAssign, 16> ArgLocs2;
1254 CCState CCInfo2(CallerCC, F.isVarArg(), MF, ArgLocs2, F.getContext());
1255 if (!determineAssignments(Assigner&: CallerAssigner, Args&: InArgs, CCInfo&: CCInfo2))
1256 return false;
1257
1258 // We need the argument locations to match up exactly. If there's more in
1259 // one than the other, then we are done.
1260 if (ArgLocs1.size() != ArgLocs2.size())
1261 return false;
1262
1263 // Make sure that each location is passed in exactly the same way.
1264 for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {
1265 const CCValAssign &Loc1 = ArgLocs1[i];
1266 const CCValAssign &Loc2 = ArgLocs2[i];
1267
1268 // We need both of them to be the same. So if one is a register and one
1269 // isn't, we're done.
1270 if (Loc1.isRegLoc() != Loc2.isRegLoc())
1271 return false;
1272
1273 if (Loc1.isRegLoc()) {
1274 // If they don't have the same register location, we're done.
1275 if (Loc1.getLocReg() != Loc2.getLocReg())
1276 return false;
1277
1278 // They matched, so we can move to the next ArgLoc.
1279 continue;
1280 }
1281
1282 // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.
1283 if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())
1284 return false;
1285 }
1286
1287 return true;
1288}
1289
1290LLT CallLowering::ValueHandler::getStackValueStoreType(
1291 const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const {
1292 const MVT ValVT = VA.getValVT();
1293 if (ValVT != MVT::iPTR) {
1294 LLT ValTy(ValVT);
1295
1296 // We lost the pointeriness going through CCValAssign, so try to restore it
1297 // based on the flags.
1298 if (Flags.isPointer()) {
1299 LLT PtrTy = LLT::pointer(AddressSpace: Flags.getPointerAddrSpace(),
1300 SizeInBits: ValTy.getScalarSizeInBits());
1301 if (ValVT.isVector() && ValVT.getVectorNumElements() != 1)
1302 return LLT::vector(EC: ValTy.getElementCount(), ScalarTy: PtrTy);
1303 return PtrTy;
1304 }
1305
1306 return ValTy;
1307 }
1308
1309 unsigned AddrSpace = Flags.getPointerAddrSpace();
1310 return LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSize(AS: AddrSpace));
1311}
1312
1313void CallLowering::ValueHandler::copyArgumentMemory(
1314 const ArgInfo &Arg, Register DstPtr, Register SrcPtr,
1315 const MachinePointerInfo &DstPtrInfo, Align DstAlign,
1316 const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize,
1317 CCValAssign &VA) const {
1318 MachineFunction &MF = MIRBuilder.getMF();
1319 MachineMemOperand *SrcMMO = MF.getMachineMemOperand(
1320 PtrInfo: SrcPtrInfo,
1321 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, Size: MemSize,
1322 BaseAlignment: SrcAlign);
1323
1324 MachineMemOperand *DstMMO = MF.getMachineMemOperand(
1325 PtrInfo: DstPtrInfo,
1326 F: MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable,
1327 Size: MemSize, BaseAlignment: DstAlign);
1328
1329 const LLT PtrTy = MRI.getType(Reg: DstPtr);
1330 const LLT SizeTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
1331
1332 auto SizeConst = MIRBuilder.buildConstant(Res: SizeTy, Val: MemSize);
1333 MIRBuilder.buildMemCpy(DstPtr, SrcPtr, Size: SizeConst, DstMMO&: *DstMMO, SrcMMO&: *SrcMMO);
1334}
1335
1336Register CallLowering::ValueHandler::extendRegister(Register ValReg,
1337 const CCValAssign &VA,
1338 unsigned MaxSizeBits) {
1339 LLT LocTy{VA.getLocVT()};
1340 LLT ValTy{VA.getValVT()};
1341
1342 if (LocTy.getSizeInBits() == ValTy.getSizeInBits())
1343 return ValReg;
1344
1345 if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) {
1346 if (MaxSizeBits <= ValTy.getSizeInBits())
1347 return ValReg;
1348 LocTy = LLT::scalar(SizeInBits: MaxSizeBits);
1349 }
1350
1351 const LLT ValRegTy = MRI.getType(Reg: ValReg);
1352 if (ValRegTy.isPointer()) {
1353 // The x32 ABI wants to zero extend 32-bit pointers to 64-bit registers, so
1354 // we have to cast to do the extension.
1355 LLT IntPtrTy = LLT::scalar(SizeInBits: ValRegTy.getSizeInBits());
1356 ValReg = MIRBuilder.buildPtrToInt(Dst: IntPtrTy, Src: ValReg).getReg(Idx: 0);
1357 }
1358
1359 switch (VA.getLocInfo()) {
1360 default:
1361 break;
1362 case CCValAssign::Full:
1363 case CCValAssign::BCvt:
1364 case CCValAssign::Indirect:
1365 // FIXME: bitconverting between vector types may or may not be a
1366 // nop in big-endian situations.
1367 return ValReg;
1368 case CCValAssign::AExt: {
1369 auto MIB = MIRBuilder.buildAnyExt(Res: LocTy, Op: ValReg);
1370 return MIB.getReg(Idx: 0);
1371 }
1372 case CCValAssign::SExt: {
1373 Register NewReg = MRI.createGenericVirtualRegister(Ty: LocTy);
1374 MIRBuilder.buildSExt(Res: NewReg, Op: ValReg);
1375 return NewReg;
1376 }
1377 case CCValAssign::ZExt: {
1378 Register NewReg = MRI.createGenericVirtualRegister(Ty: LocTy);
1379 MIRBuilder.buildZExt(Res: NewReg, Op: ValReg);
1380 return NewReg;
1381 }
1382 }
1383 llvm_unreachable("unable to extend register");
1384}
1385
1386void CallLowering::ValueAssigner::anchor() {}
1387
1388Register CallLowering::IncomingValueHandler::buildExtensionHint(
1389 const CCValAssign &VA, Register SrcReg, LLT NarrowTy) {
1390 switch (VA.getLocInfo()) {
1391 case CCValAssign::LocInfo::ZExt: {
1392 return MIRBuilder
1393 .buildAssertZExt(Res: MRI.cloneVirtualRegister(VReg: SrcReg), Op: SrcReg,
1394 Size: NarrowTy.getScalarSizeInBits())
1395 .getReg(Idx: 0);
1396 }
1397 case CCValAssign::LocInfo::SExt: {
1398 return MIRBuilder
1399 .buildAssertSExt(Res: MRI.cloneVirtualRegister(VReg: SrcReg), Op: SrcReg,
1400 Size: NarrowTy.getScalarSizeInBits())
1401 .getReg(Idx: 0);
1402 break;
1403 }
1404 default:
1405 return SrcReg;
1406 }
1407}
1408
1409/// Check if we can use a basic COPY instruction between the two types.
1410///
1411/// We're currently building on top of the infrastructure using MVT, which loses
1412/// pointer information in the CCValAssign. We accept copies from physical
1413/// registers that have been reported as integers if it's to an equivalent sized
1414/// pointer LLT.
1415static bool isCopyCompatibleType(LLT SrcTy, LLT DstTy) {
1416 if (SrcTy == DstTy)
1417 return true;
1418
1419 if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1420 return false;
1421
1422 SrcTy = SrcTy.getScalarType();
1423 DstTy = DstTy.getScalarType();
1424
1425 return (SrcTy.isPointer() && DstTy.isScalar()) ||
1426 (DstTy.isPointer() && SrcTy.isScalar());
1427}
1428
1429void CallLowering::IncomingValueHandler::assignValueToReg(
1430 Register ValVReg, Register PhysReg, const CCValAssign &VA,
1431 ISD::ArgFlagsTy Flags) {
1432 const MVT LocVT = VA.getLocVT();
1433 const LLT LocTy = getLLTForMVT(Ty: LocVT);
1434 const LLT RegTy = MRI.getType(Reg: ValVReg);
1435
1436 if (isCopyCompatibleType(SrcTy: RegTy, DstTy: LocTy)) {
1437 MIRBuilder.buildCopy(Res: ValVReg, Op: PhysReg);
1438 return;
1439 }
1440
1441 auto Copy = MIRBuilder.buildCopy(Res: LocTy, Op: PhysReg);
1442 auto Hint = buildExtensionHint(VA, SrcReg: Copy.getReg(Idx: 0), NarrowTy: RegTy);
1443 MIRBuilder.buildTrunc(Res: ValVReg, Op: Hint);
1444}
1445