1//===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements some simple delegations needed for call lowering.
11///
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/GlobalISel/CallLowering.h"
15#include "llvm/CodeGen/Analysis.h"
16#include "llvm/CodeGen/CallingConvLower.h"
17#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18#include "llvm/CodeGen/GlobalISel/Utils.h"
19#include "llvm/CodeGen/MachineFrameInfo.h"
20#include "llvm/CodeGen/MachineOperand.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/CodeGen/TargetLowering.h"
23#include "llvm/IR/DataLayout.h"
24#include "llvm/IR/LLVMContext.h"
25#include "llvm/IR/Module.h"
26#include "llvm/Target/TargetMachine.h"
27
28#define DEBUG_TYPE "call-lowering"
29
30using namespace llvm;
31
32void CallLowering::anchor() {}
33
34/// Helper function which updates \p Flags when \p AttrFn returns true.
35static void
36addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags,
37 const std::function<bool(Attribute::AttrKind)> &AttrFn) {
38 // TODO: There are missing flags. Add them here.
39 if (AttrFn(Attribute::SExt))
40 Flags.setSExt();
41 if (AttrFn(Attribute::ZExt))
42 Flags.setZExt();
43 if (AttrFn(Attribute::InReg))
44 Flags.setInReg();
45 if (AttrFn(Attribute::StructRet))
46 Flags.setSRet();
47 if (AttrFn(Attribute::Nest))
48 Flags.setNest();
49 if (AttrFn(Attribute::ByVal))
50 Flags.setByVal();
51 if (AttrFn(Attribute::ByRef))
52 Flags.setByRef();
53 if (AttrFn(Attribute::Preallocated))
54 Flags.setPreallocated();
55 if (AttrFn(Attribute::InAlloca))
56 Flags.setInAlloca();
57 if (AttrFn(Attribute::Returned))
58 Flags.setReturned();
59 if (AttrFn(Attribute::SwiftSelf))
60 Flags.setSwiftSelf();
61 if (AttrFn(Attribute::SwiftAsync))
62 Flags.setSwiftAsync();
63 if (AttrFn(Attribute::SwiftError))
64 Flags.setSwiftError();
65}
66
67ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call,
68 unsigned ArgIdx) const {
69 ISD::ArgFlagsTy Flags;
70 addFlagsUsingAttrFn(Flags, AttrFn: [&Call, &ArgIdx](Attribute::AttrKind Attr) {
71 return Call.paramHasAttr(ArgNo: ArgIdx, Kind: Attr);
72 });
73 return Flags;
74}
75
76ISD::ArgFlagsTy
77CallLowering::getAttributesForReturn(const CallBase &Call) const {
78 ISD::ArgFlagsTy Flags;
79 addFlagsUsingAttrFn(Flags, AttrFn: [&Call](Attribute::AttrKind Attr) {
80 return Call.hasRetAttr(Kind: Attr);
81 });
82 return Flags;
83}
84
85void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags,
86 const AttributeList &Attrs,
87 unsigned OpIdx) const {
88 addFlagsUsingAttrFn(Flags, AttrFn: [&Attrs, &OpIdx](Attribute::AttrKind Attr) {
89 return Attrs.hasAttributeAtIndex(Index: OpIdx, Kind: Attr);
90 });
91}
92
93bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
94 ArrayRef<Register> ResRegs,
95 ArrayRef<ArrayRef<Register>> ArgRegs,
96 Register SwiftErrorVReg,
97 std::optional<PtrAuthInfo> PAI,
98 Register ConvergenceCtrlToken,
99 std::function<Register()> GetCalleeReg) const {
100 CallLoweringInfo Info;
101 const DataLayout &DL = MIRBuilder.getDataLayout();
102 MachineFunction &MF = MIRBuilder.getMF();
103 MachineRegisterInfo &MRI = MF.getRegInfo();
104 bool CanBeTailCalled = CB.isTailCall() &&
105 isInTailCallPosition(Call: CB, TM: MF.getTarget()) &&
106 (MF.getFunction()
107 .getFnAttribute(Kind: "disable-tail-calls")
108 .getValueAsString() != "true");
109
110 CallingConv::ID CallConv = CB.getCallingConv();
111 Type *RetTy = CB.getType();
112 bool IsVarArg = CB.getFunctionType()->isVarArg();
113
114 SmallVector<BaseArgInfo, 4> SplitArgs;
115 getReturnInfo(CallConv, RetTy, Attrs: CB.getAttributes(), Outs&: SplitArgs, DL);
116 Info.CanLowerReturn = canLowerReturn(MF, CallConv, Outs&: SplitArgs, IsVarArg);
117
118 Info.IsConvergent = CB.isConvergent();
119
120 if (!Info.CanLowerReturn) {
121 // Callee requires sret demotion.
122 insertSRetOutgoingArgument(MIRBuilder, CB, Info);
123
124 // The sret demotion isn't compatible with tail-calls, since the sret
125 // argument points into the caller's stack frame.
126 CanBeTailCalled = false;
127 }
128
129 // First step is to marshall all the function's parameters into the correct
130 // physregs and memory locations. Gather the sequence of argument types that
131 // we'll pass to the assigner function.
132 unsigned i = 0;
133 unsigned NumFixedArgs = CB.getFunctionType()->getNumParams();
134 for (const auto &Arg : CB.args()) {
135 ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(Call: CB, ArgIdx: i),
136 i < NumFixedArgs};
137 setArgFlags(Arg&: OrigArg, OpIdx: i + AttributeList::FirstArgIndex, DL, FuncInfo: CB);
138
139 // If we have an explicit sret argument that is an Instruction, (i.e., it
140 // might point to function-local memory), we can't meaningfully tail-call.
141 if (OrigArg.Flags[0].isSRet() && isa<Instruction>(Val: &Arg))
142 CanBeTailCalled = false;
143
144 Info.OrigArgs.push_back(Elt: OrigArg);
145 ++i;
146 }
147
148 // Try looking through a bitcast from one function type to another.
149 // Commonly happens with calls to objc_msgSend().
150 const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts();
151
152 // If IRTranslator chose to drop the ptrauth info, we can turn this into
153 // a direct call.
154 if (!PAI && CB.countOperandBundlesOfType(ID: LLVMContext::OB_ptrauth)) {
155 CalleeV = cast<ConstantPtrAuth>(Val: CalleeV)->getPointer();
156 assert(isa<Function>(CalleeV));
157 }
158
159 if (const Function *F = dyn_cast<Function>(Val: CalleeV)) {
160 if (F->hasFnAttribute(Kind: Attribute::NonLazyBind)) {
161 LLT Ty = getLLTForType(Ty&: *F->getType(), DL);
162 Register Reg = MIRBuilder.buildGlobalValue(Res: Ty, GV: F).getReg(Idx: 0);
163 Info.Callee = MachineOperand::CreateReg(Reg, isDef: false);
164 } else {
165 Info.Callee = MachineOperand::CreateGA(GV: F, Offset: 0);
166 }
167 } else if (isa<GlobalIFunc>(Val: CalleeV) || isa<GlobalAlias>(Val: CalleeV)) {
168 // IR IFuncs and Aliases can't be forward declared (only defined), so the
169 // callee must be in the same TU and therefore we can direct-call it without
170 // worrying about it being out of range.
171 Info.Callee = MachineOperand::CreateGA(GV: cast<GlobalValue>(Val: CalleeV), Offset: 0);
172 } else
173 Info.Callee = MachineOperand::CreateReg(Reg: GetCalleeReg(), isDef: false);
174
175 Register ReturnHintAlignReg;
176 Align ReturnHintAlign;
177
178 Info.OrigRet = ArgInfo{ResRegs, RetTy, 0, getAttributesForReturn(Call: CB)};
179
180 if (!Info.OrigRet.Ty->isVoidTy()) {
181 setArgFlags(Arg&: Info.OrigRet, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: CB);
182
183 if (MaybeAlign Alignment = CB.getRetAlign()) {
184 if (*Alignment > Align(1)) {
185 ReturnHintAlignReg = MRI.cloneVirtualRegister(VReg: ResRegs[0]);
186 Info.OrigRet.Regs[0] = ReturnHintAlignReg;
187 ReturnHintAlign = *Alignment;
188 }
189 }
190 }
191
192 auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_kcfi);
193 if (Bundle && CB.isIndirectCall()) {
194 Info.CFIType = cast<ConstantInt>(Val: Bundle->Inputs[0]);
195 assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
196 }
197
198 Info.CB = &CB;
199 Info.KnownCallees = CB.getMetadata(KindID: LLVMContext::MD_callees);
200 Info.CallConv = CallConv;
201 Info.SwiftErrorVReg = SwiftErrorVReg;
202 Info.PAI = PAI;
203 Info.ConvergenceCtrlToken = ConvergenceCtrlToken;
204 Info.IsMustTailCall = CB.isMustTailCall();
205 Info.IsTailCall = CanBeTailCalled;
206 Info.IsVarArg = IsVarArg;
207 if (!lowerCall(MIRBuilder, Info))
208 return false;
209
210 if (ReturnHintAlignReg && !Info.LoweredTailCall) {
211 MIRBuilder.buildAssertAlign(Res: ResRegs[0], Op: ReturnHintAlignReg,
212 AlignVal: ReturnHintAlign);
213 }
214
215 return true;
216}
217
218template <typename FuncInfoTy>
219void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
220 const DataLayout &DL,
221 const FuncInfoTy &FuncInfo) const {
222 auto &Flags = Arg.Flags[0];
223 const AttributeList &Attrs = FuncInfo.getAttributes();
224 addArgFlagsFromAttributes(Flags, Attrs, OpIdx);
225
226 PointerType *PtrTy = dyn_cast<PointerType>(Val: Arg.Ty->getScalarType());
227 if (PtrTy) {
228 Flags.setPointer();
229 Flags.setPointerAddrSpace(PtrTy->getPointerAddressSpace());
230 }
231
232 Align MemAlign = DL.getABITypeAlign(Ty: Arg.Ty);
233 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
234 Flags.isByRef()) {
235 assert(OpIdx >= AttributeList::FirstArgIndex);
236 unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex;
237
238 Type *ElementTy = FuncInfo.getParamByValType(ParamIdx);
239 if (!ElementTy)
240 ElementTy = FuncInfo.getParamByRefType(ParamIdx);
241 if (!ElementTy)
242 ElementTy = FuncInfo.getParamInAllocaType(ParamIdx);
243 if (!ElementTy)
244 ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx);
245
246 assert(ElementTy && "Must have byval, inalloca or preallocated type");
247
248 uint64_t MemSize = DL.getTypeAllocSize(Ty: ElementTy);
249 if (Flags.isByRef())
250 Flags.setByRefSize(MemSize);
251 else
252 Flags.setByValSize(MemSize);
253
254 // For ByVal, alignment should be passed from FE. BE will guess if
255 // this info is not there but there are cases it cannot get right.
256 if (auto ParamAlign = FuncInfo.getParamStackAlign(ParamIdx))
257 MemAlign = *ParamAlign;
258 else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx)))
259 MemAlign = *ParamAlign;
260 else
261 MemAlign = getTLI()->getByValTypeAlignment(Ty: ElementTy, DL);
262 } else if (OpIdx >= AttributeList::FirstArgIndex) {
263 if (auto ParamAlign =
264 FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex))
265 MemAlign = *ParamAlign;
266 }
267 Flags.setMemAlign(MemAlign);
268 Flags.setOrigAlign(DL.getABITypeAlign(Ty: Arg.Ty));
269
270 // Don't try to use the returned attribute if the argument is marked as
271 // swiftself, since it won't be passed in x0.
272 if (Flags.isSwiftSelf())
273 Flags.setReturned(false);
274}
275
276template void
277CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
278 const DataLayout &DL,
279 const Function &FuncInfo) const;
280
281template void
282CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
283 const DataLayout &DL,
284 const CallBase &FuncInfo) const;
285
286void CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
287 SmallVectorImpl<ArgInfo> &SplitArgs,
288 const DataLayout &DL,
289 CallingConv::ID CallConv,
290 SmallVectorImpl<uint64_t> *Offsets) const {
291 LLVMContext &Ctx = OrigArg.Ty->getContext();
292
293 SmallVector<EVT, 4> SplitVTs;
294 ComputeValueVTs(TLI: *TLI, DL, Ty: OrigArg.Ty, ValueVTs&: SplitVTs, FixedOffsets: Offsets, StartingOffset: 0);
295
296 if (SplitVTs.size() == 0)
297 return;
298
299 if (SplitVTs.size() == 1) {
300 // No splitting to do, but we want to replace the original type (e.g. [1 x
301 // double] -> double).
302 SplitArgs.emplace_back(Args: OrigArg.Regs[0], Args: SplitVTs[0].getTypeForEVT(Context&: Ctx),
303 Args: OrigArg.OrigArgIndex, Args: OrigArg.Flags[0],
304 Args: OrigArg.IsFixed, Args: OrigArg.OrigValue);
305 return;
306 }
307
308 // Create one ArgInfo for each virtual register in the original ArgInfo.
309 assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch");
310
311 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
312 Ty: OrigArg.Ty, CallConv, isVarArg: false, DL);
313 for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) {
314 Type *SplitTy = SplitVTs[i].getTypeForEVT(Context&: Ctx);
315 SplitArgs.emplace_back(Args: OrigArg.Regs[i], Args&: SplitTy, Args: OrigArg.OrigArgIndex,
316 Args: OrigArg.Flags[0], Args: OrigArg.IsFixed);
317 if (NeedsRegBlock)
318 SplitArgs.back().Flags[0].setInConsecutiveRegs();
319 }
320
321 SplitArgs.back().Flags[0].setInConsecutiveRegsLast();
322}
323
324/// Pack values \p SrcRegs to cover the vector type result \p DstRegs.
325static MachineInstrBuilder
326mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
327 ArrayRef<Register> SrcRegs) {
328 MachineRegisterInfo &MRI = *B.getMRI();
329 LLT LLTy = MRI.getType(Reg: DstRegs[0]);
330 LLT PartLLT = MRI.getType(Reg: SrcRegs[0]);
331
332 // Deal with v3s16 split into v2s16
333 LLT LCMTy = getCoverTy(OrigTy: LLTy, TargetTy: PartLLT);
334 if (LCMTy == LLTy) {
335 // Common case where no padding is needed.
336 assert(DstRegs.size() == 1);
337 return B.buildConcatVectors(Res: DstRegs[0], Ops: SrcRegs);
338 }
339
340 // We need to create an unmerge to the result registers, which may require
341 // widening the original value.
342 Register UnmergeSrcReg;
343 if (LCMTy != PartLLT) {
344 assert(DstRegs.size() == 1);
345 return B.buildDeleteTrailingVectorElements(
346 Res: DstRegs[0], Op0: B.buildMergeLikeInstr(Res: LCMTy, Ops: SrcRegs));
347 } else {
348 // We don't need to widen anything if we're extracting a scalar which was
349 // promoted to a vector e.g. s8 -> v4s8 -> s8
350 assert(SrcRegs.size() == 1);
351 UnmergeSrcReg = SrcRegs[0];
352 }
353
354 int NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits();
355
356 SmallVector<Register, 8> PadDstRegs(NumDst);
357 llvm::copy(Range&: DstRegs, Out: PadDstRegs.begin());
358
359 // Create the excess dead defs for the unmerge.
360 for (int I = DstRegs.size(); I != NumDst; ++I)
361 PadDstRegs[I] = MRI.createGenericVirtualRegister(Ty: LLTy);
362
363 if (PadDstRegs.size() == 1)
364 return B.buildDeleteTrailingVectorElements(Res: DstRegs[0], Op0: UnmergeSrcReg);
365 return B.buildUnmerge(Res: PadDstRegs, Op: UnmergeSrcReg);
366}
367
368/// Create a sequence of instructions to combine pieces split into register
369/// typed values to the original IR value. \p OrigRegs contains the destination
370/// value registers of type \p LLTy, and \p Regs contains the legalized pieces
371/// with type \p PartLLT. This is used for incoming values (physregs to vregs).
372static void buildCopyFromRegs(MachineIRBuilder &B, ArrayRef<Register> OrigRegs,
373 ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT,
374 const ISD::ArgFlagsTy Flags) {
375 MachineRegisterInfo &MRI = *B.getMRI();
376
377 if (PartLLT == LLTy) {
378 // We should have avoided introducing a new virtual register, and just
379 // directly assigned here.
380 assert(OrigRegs[0] == Regs[0]);
381 return;
382 }
383
384 if (PartLLT.getSizeInBits() == LLTy.getSizeInBits() && OrigRegs.size() == 1 &&
385 Regs.size() == 1) {
386 B.buildBitcast(Dst: OrigRegs[0], Src: Regs[0]);
387 return;
388 }
389
390 // A vector PartLLT needs extending to LLTy's element size.
391 // E.g. <2 x s64> = G_SEXT <2 x s32>.
392 if (PartLLT.isVector() == LLTy.isVector() &&
393 PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits() &&
394 (!PartLLT.isVector() ||
395 PartLLT.getElementCount() == LLTy.getElementCount()) &&
396 OrigRegs.size() == 1 && Regs.size() == 1) {
397 Register SrcReg = Regs[0];
398
399 LLT LocTy = MRI.getType(Reg: SrcReg);
400
401 if (Flags.isSExt()) {
402 SrcReg = B.buildAssertSExt(Res: LocTy, Op: SrcReg, Size: LLTy.getScalarSizeInBits())
403 .getReg(Idx: 0);
404 } else if (Flags.isZExt()) {
405 SrcReg = B.buildAssertZExt(Res: LocTy, Op: SrcReg, Size: LLTy.getScalarSizeInBits())
406 .getReg(Idx: 0);
407 }
408
409 // Sometimes pointers are passed zero extended.
410 LLT OrigTy = MRI.getType(Reg: OrigRegs[0]);
411 if (OrigTy.isPointer()) {
412 LLT IntPtrTy = LLT::scalar(SizeInBits: OrigTy.getSizeInBits());
413 B.buildIntToPtr(Dst: OrigRegs[0], Src: B.buildTrunc(Res: IntPtrTy, Op: SrcReg));
414 return;
415 }
416
417 B.buildTrunc(Res: OrigRegs[0], Op: SrcReg);
418 return;
419 }
420
421 if (!LLTy.isVector() && !PartLLT.isVector()) {
422 assert(OrigRegs.size() == 1);
423 LLT OrigTy = MRI.getType(Reg: OrigRegs[0]);
424
425 unsigned SrcSize = PartLLT.getSizeInBits().getFixedValue() * Regs.size();
426 if (SrcSize == OrigTy.getSizeInBits())
427 B.buildMergeValues(Res: OrigRegs[0], Ops: Regs);
428 else {
429 auto Widened = B.buildMergeLikeInstr(Res: LLT::scalar(SizeInBits: SrcSize), Ops: Regs);
430 B.buildTrunc(Res: OrigRegs[0], Op: Widened);
431 }
432
433 return;
434 }
435
436 if (PartLLT.isVector()) {
437 assert(OrigRegs.size() == 1);
438 SmallVector<Register> CastRegs(Regs);
439
440 // If PartLLT is a mismatched vector in both number of elements and element
441 // size, e.g. PartLLT == v2s64 and LLTy is v3s32, then first coerce it to
442 // have the same elt type, i.e. v4s32.
443 // TODO: Extend this coersion to element multiples other than just 2.
444 if (TypeSize::isKnownGT(LHS: PartLLT.getSizeInBits(), RHS: LLTy.getSizeInBits()) &&
445 PartLLT.getScalarSizeInBits() == LLTy.getScalarSizeInBits() * 2 &&
446 Regs.size() == 1) {
447 LLT NewTy = PartLLT.changeElementType(NewEltTy: LLTy.getElementType())
448 .changeElementCount(EC: PartLLT.getElementCount() * 2);
449 CastRegs[0] = B.buildBitcast(Dst: NewTy, Src: Regs[0]).getReg(Idx: 0);
450 PartLLT = NewTy;
451 }
452
453 if (LLTy.getScalarType() == PartLLT.getElementType()) {
454 mergeVectorRegsToResultRegs(B, DstRegs: OrigRegs, SrcRegs: CastRegs);
455 } else {
456 unsigned I = 0;
457 LLT GCDTy = getGCDType(OrigTy: LLTy, TargetTy: PartLLT);
458
459 // We are both splitting a vector, and bitcasting its element types. Cast
460 // the source pieces into the appropriate number of pieces with the result
461 // element type.
462 for (Register SrcReg : CastRegs)
463 CastRegs[I++] = B.buildBitcast(Dst: GCDTy, Src: SrcReg).getReg(Idx: 0);
464 mergeVectorRegsToResultRegs(B, DstRegs: OrigRegs, SrcRegs: CastRegs);
465 }
466
467 return;
468 }
469
470 assert(LLTy.isVector() && !PartLLT.isVector());
471
472 LLT DstEltTy = LLTy.getElementType();
473
474 // Pointer information was discarded. We'll need to coerce some register types
475 // to avoid violating type constraints.
476 LLT RealDstEltTy = MRI.getType(Reg: OrigRegs[0]).getElementType();
477
478 assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits());
479
480 if (DstEltTy == PartLLT) {
481 // Vector was trivially scalarized.
482
483 if (RealDstEltTy.isPointer()) {
484 for (Register Reg : Regs)
485 MRI.setType(VReg: Reg, Ty: RealDstEltTy);
486 }
487
488 B.buildBuildVector(Res: OrigRegs[0], Ops: Regs);
489 } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) {
490 // Deal with vector with 64-bit elements decomposed to 32-bit
491 // registers. Need to create intermediate 64-bit elements.
492 SmallVector<Register, 8> EltMerges;
493 int PartsPerElt =
494 divideCeil(Numerator: DstEltTy.getSizeInBits(), Denominator: PartLLT.getSizeInBits());
495 LLT ExtendedPartTy = LLT::scalar(SizeInBits: PartLLT.getSizeInBits() * PartsPerElt);
496
497 for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) {
498 auto Merge =
499 B.buildMergeLikeInstr(Res: ExtendedPartTy, Ops: Regs.take_front(N: PartsPerElt));
500 if (ExtendedPartTy.getSizeInBits() > RealDstEltTy.getSizeInBits())
501 Merge = B.buildTrunc(Res: RealDstEltTy, Op: Merge);
502 // Fix the type in case this is really a vector of pointers.
503 MRI.setType(VReg: Merge.getReg(Idx: 0), Ty: RealDstEltTy);
504 EltMerges.push_back(Elt: Merge.getReg(Idx: 0));
505 Regs = Regs.drop_front(N: PartsPerElt);
506 }
507
508 B.buildBuildVector(Res: OrigRegs[0], Ops: EltMerges);
509 } else {
510 // Vector was split, and elements promoted to a wider type.
511 // FIXME: Should handle floating point promotions.
512 unsigned NumElts = LLTy.getNumElements();
513 LLT BVType = LLT::fixed_vector(NumElements: NumElts, ScalarTy: PartLLT);
514
515 Register BuildVec;
516 if (NumElts == Regs.size())
517 BuildVec = B.buildBuildVector(Res: BVType, Ops: Regs).getReg(Idx: 0);
518 else {
519 // Vector elements are packed in the inputs.
520 // e.g. we have a <4 x s16> but 2 x s32 in regs.
521 assert(NumElts > Regs.size());
522 LLT SrcEltTy = MRI.getType(Reg: Regs[0]);
523
524 LLT OriginalEltTy = MRI.getType(Reg: OrigRegs[0]).getElementType();
525
526 // Input registers contain packed elements.
527 // Determine how many elements per reg.
528 assert((SrcEltTy.getSizeInBits() % OriginalEltTy.getSizeInBits()) == 0);
529 unsigned EltPerReg =
530 (SrcEltTy.getSizeInBits() / OriginalEltTy.getSizeInBits());
531
532 SmallVector<Register, 0> BVRegs;
533 BVRegs.reserve(N: Regs.size() * EltPerReg);
534 for (Register R : Regs) {
535 auto Unmerge = B.buildUnmerge(Res: OriginalEltTy, Op: R);
536 for (unsigned K = 0; K < EltPerReg; ++K)
537 BVRegs.push_back(Elt: B.buildAnyExt(Res: PartLLT, Op: Unmerge.getReg(Idx: K)).getReg(Idx: 0));
538 }
539
540 // We may have some more elements in BVRegs, e.g. if we have 2 s32 pieces
541 // for a <3 x s16> vector. We should have less than EltPerReg extra items.
542 if (BVRegs.size() > NumElts) {
543 assert((BVRegs.size() - NumElts) < EltPerReg);
544 BVRegs.truncate(N: NumElts);
545 }
546 BuildVec = B.buildBuildVector(Res: BVType, Ops: BVRegs).getReg(Idx: 0);
547 }
548 B.buildTrunc(Res: OrigRegs[0], Op: BuildVec);
549 }
550}
551
552/// Create a sequence of instructions to expand the value in \p SrcReg (of type
553/// \p SrcTy) to the types in \p DstRegs (of type \p PartTy). \p ExtendOp should
554/// contain the type of scalar value extension if necessary.
555///
556/// This is used for outgoing values (vregs to physregs)
557static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
558 Register SrcReg, LLT SrcTy, LLT PartTy,
559 unsigned ExtendOp = TargetOpcode::G_ANYEXT) {
560 // We could just insert a regular copy, but this is unreachable at the moment.
561 assert(SrcTy != PartTy && "identical part types shouldn't reach here");
562
563 const TypeSize PartSize = PartTy.getSizeInBits();
564
565 if (PartTy.isVector() == SrcTy.isVector() &&
566 PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) {
567 assert(DstRegs.size() == 1);
568 B.buildInstr(Opc: ExtendOp, DstOps: {DstRegs[0]}, SrcOps: {SrcReg});
569 return;
570 }
571
572 if (SrcTy.isVector() && !PartTy.isVector() &&
573 TypeSize::isKnownGT(LHS: PartSize, RHS: SrcTy.getElementType().getSizeInBits())) {
574 // Vector was scalarized, and the elements extended.
575 auto UnmergeToEltTy = B.buildUnmerge(Res: SrcTy.getElementType(), Op: SrcReg);
576 for (int i = 0, e = DstRegs.size(); i != e; ++i)
577 B.buildAnyExt(Res: DstRegs[i], Op: UnmergeToEltTy.getReg(Idx: i));
578 return;
579 }
580
581 if (SrcTy.isVector() && PartTy.isVector() &&
582 PartTy.getSizeInBits() == SrcTy.getSizeInBits() &&
583 ElementCount::isKnownLT(LHS: SrcTy.getElementCount(),
584 RHS: PartTy.getElementCount())) {
585 // A coercion like: v2f32 -> v4f32 or nxv2f32 -> nxv4f32
586 Register DstReg = DstRegs.front();
587 B.buildPadVectorWithUndefElements(Res: DstReg, Op0: SrcReg);
588 return;
589 }
590
591 LLT GCDTy = getGCDType(OrigTy: SrcTy, TargetTy: PartTy);
592 if (GCDTy == PartTy) {
593 // If this already evenly divisible, we can create a simple unmerge.
594 B.buildUnmerge(Res: DstRegs, Op: SrcReg);
595 return;
596 }
597
598 if (SrcTy.isVector() && !PartTy.isVector() &&
599 SrcTy.getScalarSizeInBits() > PartTy.getSizeInBits()) {
600 LLT ExtTy =
601 LLT::vector(EC: SrcTy.getElementCount(),
602 ScalarTy: LLT::scalar(SizeInBits: PartTy.getScalarSizeInBits() * DstRegs.size() /
603 SrcTy.getNumElements()));
604 auto Ext = B.buildAnyExt(Res: ExtTy, Op: SrcReg);
605 B.buildUnmerge(Res: DstRegs, Op: Ext);
606 return;
607 }
608
609 MachineRegisterInfo &MRI = *B.getMRI();
610 LLT DstTy = MRI.getType(Reg: DstRegs[0]);
611 LLT LCMTy = getCoverTy(OrigTy: SrcTy, TargetTy: PartTy);
612
613 if (PartTy.isVector() && LCMTy == PartTy) {
614 assert(DstRegs.size() == 1);
615 B.buildPadVectorWithUndefElements(Res: DstRegs[0], Op0: SrcReg);
616 return;
617 }
618
619 const unsigned DstSize = DstTy.getSizeInBits();
620 const unsigned SrcSize = SrcTy.getSizeInBits();
621 unsigned CoveringSize = LCMTy.getSizeInBits();
622
623 Register UnmergeSrc = SrcReg;
624
625 if (!LCMTy.isVector() && CoveringSize != SrcSize) {
626 // For scalars, it's common to be able to use a simple extension.
627 if (SrcTy.isScalar() && DstTy.isScalar()) {
628 CoveringSize = alignTo(Value: SrcSize, Align: DstSize);
629 LLT CoverTy = LLT::scalar(SizeInBits: CoveringSize);
630 UnmergeSrc = B.buildInstr(Opc: ExtendOp, DstOps: {CoverTy}, SrcOps: {SrcReg}).getReg(Idx: 0);
631 } else {
632 // Widen to the common type.
633 // FIXME: This should respect the extend type
634 Register Undef = B.buildUndef(Res: SrcTy).getReg(Idx: 0);
635 SmallVector<Register, 8> MergeParts(1, SrcReg);
636 for (unsigned Size = SrcSize; Size != CoveringSize; Size += SrcSize)
637 MergeParts.push_back(Elt: Undef);
638 UnmergeSrc = B.buildMergeLikeInstr(Res: LCMTy, Ops: MergeParts).getReg(Idx: 0);
639 }
640 }
641
642 if (LCMTy.isVector() && CoveringSize != SrcSize)
643 UnmergeSrc = B.buildPadVectorWithUndefElements(Res: LCMTy, Op0: SrcReg).getReg(Idx: 0);
644
645 B.buildUnmerge(Res: DstRegs, Op: UnmergeSrc);
646}
647
648bool CallLowering::determineAndHandleAssignments(
649 ValueHandler &Handler, ValueAssigner &Assigner,
650 SmallVectorImpl<ArgInfo> &Args, MachineIRBuilder &MIRBuilder,
651 CallingConv::ID CallConv, bool IsVarArg,
652 ArrayRef<Register> ThisReturnRegs) const {
653 MachineFunction &MF = MIRBuilder.getMF();
654 const Function &F = MF.getFunction();
655 SmallVector<CCValAssign, 16> ArgLocs;
656
657 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext());
658 if (!determineAssignments(Assigner, Args, CCInfo))
659 return false;
660
661 return handleAssignments(Handler, Args, CCState&: CCInfo, ArgLocs, MIRBuilder,
662 ThisReturnRegs);
663}
664
665static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) {
666 if (Flags.isSExt())
667 return TargetOpcode::G_SEXT;
668 if (Flags.isZExt())
669 return TargetOpcode::G_ZEXT;
670 return TargetOpcode::G_ANYEXT;
671}
672
673bool CallLowering::determineAssignments(ValueAssigner &Assigner,
674 SmallVectorImpl<ArgInfo> &Args,
675 CCState &CCInfo) const {
676 LLVMContext &Ctx = CCInfo.getContext();
677 const CallingConv::ID CallConv = CCInfo.getCallingConv();
678
679 unsigned NumArgs = Args.size();
680 for (unsigned i = 0; i != NumArgs; ++i) {
681 EVT CurVT = EVT::getEVT(Ty: Args[i].Ty);
682
683 MVT NewVT = TLI->getRegisterTypeForCallingConv(Context&: Ctx, CC: CallConv, VT: CurVT);
684
685 // If we need to split the type over multiple regs, check it's a scenario
686 // we currently support.
687 unsigned NumParts =
688 TLI->getNumRegistersForCallingConv(Context&: Ctx, CC: CallConv, VT: CurVT);
689
690 if (NumParts == 1) {
691 // Try to use the register type if we couldn't assign the VT.
692 if (Assigner.assignArg(ValNo: i, OrigVT: CurVT, ValVT: NewVT, LocVT: NewVT, LocInfo: CCValAssign::Full, Info: Args[i],
693 Flags: Args[i].Flags[0], State&: CCInfo))
694 return false;
695 continue;
696 }
697
698 // For incoming arguments (physregs to vregs), we could have values in
699 // physregs (or memlocs) which we want to extract and copy to vregs.
700 // During this, we might have to deal with the LLT being split across
701 // multiple regs, so we have to record this information for later.
702 //
703 // If we have outgoing args, then we have the opposite case. We have a
704 // vreg with an LLT which we want to assign to a physical location, and
705 // we might have to record that the value has to be split later.
706
707 // We're handling an incoming arg which is split over multiple regs.
708 // E.g. passing an s128 on AArch64.
709 ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
710 Args[i].Flags.clear();
711
712 for (unsigned Part = 0; Part < NumParts; ++Part) {
713 ISD::ArgFlagsTy Flags = OrigFlags;
714 if (Part == 0) {
715 Flags.setSplit();
716 } else {
717 Flags.setOrigAlign(Align(1));
718 if (Part == NumParts - 1)
719 Flags.setSplitEnd();
720 }
721
722 Args[i].Flags.push_back(Elt: Flags);
723 if (Assigner.assignArg(ValNo: i, OrigVT: CurVT, ValVT: NewVT, LocVT: NewVT, LocInfo: CCValAssign::Full, Info: Args[i],
724 Flags: Args[i].Flags[Part], State&: CCInfo)) {
725 // Still couldn't assign this smaller part type for some reason.
726 return false;
727 }
728 }
729 }
730
731 return true;
732}
733
734bool CallLowering::handleAssignments(ValueHandler &Handler,
735 SmallVectorImpl<ArgInfo> &Args,
736 CCState &CCInfo,
737 SmallVectorImpl<CCValAssign> &ArgLocs,
738 MachineIRBuilder &MIRBuilder,
739 ArrayRef<Register> ThisReturnRegs) const {
740 MachineFunction &MF = MIRBuilder.getMF();
741 MachineRegisterInfo &MRI = MF.getRegInfo();
742 const Function &F = MF.getFunction();
743 const DataLayout &DL = F.getDataLayout();
744
745 const unsigned NumArgs = Args.size();
746
747 // Stores thunks for outgoing register assignments. This is used so we delay
748 // generating register copies until mem loc assignments are done. We do this
749 // so that if the target is using the delayed stack protector feature, we can
750 // find the split point of the block accurately. E.g. if we have:
751 // G_STORE %val, %memloc
752 // $x0 = COPY %foo
753 // $x1 = COPY %bar
754 // CALL func
755 // ... then the split point for the block will correctly be at, and including,
756 // the copy to $x0. If instead the G_STORE instruction immediately precedes
757 // the CALL, then we'd prematurely choose the CALL as the split point, thus
758 // generating a split block with a CALL that uses undefined physregs.
759 SmallVector<std::function<void()>> DelayedOutgoingRegAssignments;
760
761 for (unsigned i = 0, j = 0; i != NumArgs; ++i, ++j) {
762 assert(j < ArgLocs.size() && "Skipped too many arg locs");
763 CCValAssign &VA = ArgLocs[j];
764 assert(VA.getValNo() == i && "Location doesn't correspond to current arg");
765
766 if (VA.needsCustom()) {
767 std::function<void()> Thunk;
768 unsigned NumArgRegs = Handler.assignCustomValue(
769 Arg&: Args[i], VAs: ArrayRef(ArgLocs).slice(N: j), Thunk: &Thunk);
770 if (Thunk)
771 DelayedOutgoingRegAssignments.emplace_back(Args&: Thunk);
772 if (!NumArgRegs)
773 return false;
774 j += (NumArgRegs - 1);
775 continue;
776 }
777
778 auto AllocaAddressSpace = MF.getDataLayout().getAllocaAddrSpace();
779
780 const MVT ValVT = VA.getValVT();
781 const MVT LocVT = VA.getLocVT();
782
783 const LLT LocTy(LocVT);
784 const LLT ValTy(ValVT);
785 const LLT NewLLT = Handler.isIncomingArgumentHandler() ? LocTy : ValTy;
786 const EVT OrigVT = EVT::getEVT(Ty: Args[i].Ty);
787 const LLT OrigTy = getLLTForType(Ty&: *Args[i].Ty, DL);
788 const LLT PointerTy = LLT::pointer(
789 AddressSpace: AllocaAddressSpace, SizeInBits: DL.getPointerSizeInBits(AS: AllocaAddressSpace));
790
791 // Expected to be multiple regs for a single incoming arg.
792 // There should be Regs.size() ArgLocs per argument.
793 // This should be the same as getNumRegistersForCallingConv
794 const unsigned NumParts = Args[i].Flags.size();
795
796 // Now split the registers into the assigned types.
797 Args[i].OrigRegs.assign(in_start: Args[i].Regs.begin(), in_end: Args[i].Regs.end());
798
799 if (NumParts != 1 || NewLLT != OrigTy) {
800 // If we can't directly assign the register, we need one or more
801 // intermediate values.
802 Args[i].Regs.resize(N: NumParts);
803
804 // When we have indirect parameter passing we are receiving a pointer,
805 // that points to the actual value, so we need one "temporary" pointer.
806 if (VA.getLocInfo() == CCValAssign::Indirect) {
807 if (Handler.isIncomingArgumentHandler())
808 Args[i].Regs[0] = MRI.createGenericVirtualRegister(Ty: PointerTy);
809 } else {
810 // For each split register, create and assign a vreg that will store
811 // the incoming component of the larger value. These will later be
812 // merged to form the final vreg.
813 for (unsigned Part = 0; Part < NumParts; ++Part)
814 Args[i].Regs[Part] = MRI.createGenericVirtualRegister(Ty: NewLLT);
815 }
816 }
817
818 assert((j + (NumParts - 1)) < ArgLocs.size() &&
819 "Too many regs for number of args");
820
821 // Coerce into outgoing value types before register assignment.
822 if (!Handler.isIncomingArgumentHandler() && OrigTy != ValTy &&
823 VA.getLocInfo() != CCValAssign::Indirect) {
824 assert(Args[i].OrigRegs.size() == 1);
825 buildCopyToRegs(B&: MIRBuilder, DstRegs: Args[i].Regs, SrcReg: Args[i].OrigRegs[0], SrcTy: OrigTy,
826 PartTy: ValTy, ExtendOp: extendOpFromFlags(Flags: Args[i].Flags[0]));
827 }
828
829 bool IndirectParameterPassingHandled = false;
830 bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(VT: OrigVT, DL);
831 for (unsigned Part = 0; Part < NumParts; ++Part) {
832 assert((VA.getLocInfo() != CCValAssign::Indirect || Part == 0) &&
833 "Only the first parameter should be processed when "
834 "handling indirect passing!");
835 Register ArgReg = Args[i].Regs[Part];
836 // There should be Regs.size() ArgLocs per argument.
837 unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part;
838 CCValAssign &VA = ArgLocs[j + Idx];
839 const ISD::ArgFlagsTy Flags = Args[i].Flags[Part];
840
841 // We found an indirect parameter passing, and we have an
842 // OutgoingValueHandler as our handler (so we are at the call site or the
843 // return value). In this case, start the construction of the following
844 // GMIR, that is responsible for the preparation of indirect parameter
845 // passing:
846 //
847 // %1(indirectly passed type) = The value to pass
848 // %3(pointer) = G_FRAME_INDEX %stack.0
849 // G_STORE %1, %3 :: (store (s128), align 8)
850 //
851 // After this GMIR, the remaining part of the loop body will decide how
852 // to get the value to the caller and we break out of the loop.
853 if (VA.getLocInfo() == CCValAssign::Indirect &&
854 !Handler.isIncomingArgumentHandler()) {
855 Align AlignmentForStored = DL.getPrefTypeAlign(Ty: Args[i].Ty);
856 MachineFrameInfo &MFI = MF.getFrameInfo();
857 // Get some space on the stack for the value, so later we can pass it
858 // as a reference.
859 int FrameIdx = MFI.CreateStackObject(Size: OrigTy.getScalarSizeInBits(),
860 Alignment: AlignmentForStored, isSpillSlot: false);
861 Register PointerToStackReg =
862 MIRBuilder.buildFrameIndex(Res: PointerTy, Idx: FrameIdx).getReg(Idx: 0);
863 MachinePointerInfo StackPointerMPO =
864 MachinePointerInfo::getFixedStack(MF, FI: FrameIdx);
865 // Store the value in the previously created stack space.
866 MIRBuilder.buildStore(Val: Args[i].OrigRegs[Part], Addr: PointerToStackReg,
867 PtrInfo: StackPointerMPO,
868 Alignment: inferAlignFromPtrInfo(MF, MPO: StackPointerMPO));
869
870 ArgReg = PointerToStackReg;
871 IndirectParameterPassingHandled = true;
872 }
873
874 if (VA.isMemLoc() && !Flags.isByVal()) {
875 // Individual pieces may have been spilled to the stack and others
876 // passed in registers.
877
878 // TODO: The memory size may be larger than the value we need to
879 // store. We may need to adjust the offset for big endian targets.
880 LLT MemTy = Handler.getStackValueStoreType(DL, VA, Flags);
881
882 MachinePointerInfo MPO;
883 Register StackAddr =
884 Handler.getStackAddress(MemSize: VA.getLocInfo() == CCValAssign::Indirect
885 ? PointerTy.getSizeInBytes()
886 : MemTy.getSizeInBytes(),
887 Offset: VA.getLocMemOffset(), MPO, Flags);
888
889 // Finish the handling of indirect passing from the passers
890 // (OutgoingParameterHandler) side.
891 // This branch is needed, so the pointer to the value is loaded onto the
892 // stack.
893 if (VA.getLocInfo() == CCValAssign::Indirect)
894 Handler.assignValueToAddress(ValVReg: ArgReg, Addr: StackAddr, MemTy: PointerTy, MPO, VA);
895 else
896 Handler.assignValueToAddress(Arg: Args[i], ValRegIndex: Part, Addr: StackAddr, MemTy, MPO,
897 VA);
898 } else if (VA.isMemLoc() && Flags.isByVal()) {
899 assert(Args[i].Regs.size() == 1 && "didn't expect split byval pointer");
900
901 if (Handler.isIncomingArgumentHandler()) {
902 // We just need to copy the frame index value to the pointer.
903 MachinePointerInfo MPO;
904 Register StackAddr = Handler.getStackAddress(
905 MemSize: Flags.getByValSize(), Offset: VA.getLocMemOffset(), MPO, Flags);
906 MIRBuilder.buildCopy(Res: Args[i].Regs[0], Op: StackAddr);
907 } else {
908 // For outgoing byval arguments, insert the implicit copy byval
909 // implies, such that writes in the callee do not modify the caller's
910 // value.
911 uint64_t MemSize = Flags.getByValSize();
912 int64_t Offset = VA.getLocMemOffset();
913
914 MachinePointerInfo DstMPO;
915 Register StackAddr =
916 Handler.getStackAddress(MemSize, Offset, MPO&: DstMPO, Flags);
917
918 MachinePointerInfo SrcMPO(Args[i].OrigValue);
919 if (!Args[i].OrigValue) {
920 // We still need to accurately track the stack address space if we
921 // don't know the underlying value.
922 const LLT PtrTy = MRI.getType(Reg: StackAddr);
923 SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace());
924 }
925
926 Align DstAlign = std::max(a: Flags.getNonZeroByValAlign(),
927 b: inferAlignFromPtrInfo(MF, MPO: DstMPO));
928
929 Align SrcAlign = std::max(a: Flags.getNonZeroByValAlign(),
930 b: inferAlignFromPtrInfo(MF, MPO: SrcMPO));
931
932 Handler.copyArgumentMemory(Arg: Args[i], DstPtr: StackAddr, SrcPtr: Args[i].Regs[0],
933 DstPtrInfo: DstMPO, DstAlign, SrcPtrInfo: SrcMPO, SrcAlign,
934 MemSize, VA);
935 }
936 } else if (i == 0 && !ThisReturnRegs.empty() &&
937 Handler.isIncomingArgumentHandler() &&
938 isTypeIsValidForThisReturn(Ty: ValVT)) {
939 Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: ThisReturnRegs[Part], VA);
940 } else if (Handler.isIncomingArgumentHandler()) {
941 Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: VA.getLocReg(), VA);
942 } else {
943 DelayedOutgoingRegAssignments.emplace_back(Args: [=, &Handler]() {
944 Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: VA.getLocReg(), VA);
945 });
946 }
947
948 // Finish the handling of indirect parameter passing when receiving
949 // the value (we are in the called function or the caller when receiving
950 // the return value).
951 if (VA.getLocInfo() == CCValAssign::Indirect &&
952 Handler.isIncomingArgumentHandler()) {
953 Align Alignment = DL.getABITypeAlign(Ty: Args[i].Ty);
954 MachinePointerInfo MPO = MachinePointerInfo::getUnknownStack(MF);
955
956 // Since we are doing indirect parameter passing, we know that the value
957 // in the temporary register is not the value passed to the function,
958 // but rather a pointer to that value. Let's load that value into the
959 // virtual register where the parameter should go.
960 MIRBuilder.buildLoad(Res: Args[i].OrigRegs[0], Addr: Args[i].Regs[0], PtrInfo: MPO,
961 Alignment);
962
963 IndirectParameterPassingHandled = true;
964 }
965
966 if (IndirectParameterPassingHandled)
967 break;
968 }
969
970 // Now that all pieces have been assigned, re-pack the register typed values
971 // into the original value typed registers. This is only necessary, when
972 // the value was passed in multiple registers, not indirectly.
973 if (Handler.isIncomingArgumentHandler() && OrigVT != LocVT &&
974 !IndirectParameterPassingHandled) {
975 // Merge the split registers into the expected larger result vregs of
976 // the original call.
977 buildCopyFromRegs(B&: MIRBuilder, OrigRegs: Args[i].OrigRegs, Regs: Args[i].Regs, LLTy: OrigTy,
978 PartLLT: LocTy, Flags: Args[i].Flags[0]);
979 }
980
981 j += NumParts - 1;
982 }
983 for (auto &Fn : DelayedOutgoingRegAssignments)
984 Fn();
985
986 return true;
987}
988
989void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy,
990 ArrayRef<Register> VRegs, Register DemoteReg,
991 int FI) const {
992 MachineFunction &MF = MIRBuilder.getMF();
993 MachineRegisterInfo &MRI = MF.getRegInfo();
994 const DataLayout &DL = MF.getDataLayout();
995
996 SmallVector<EVT, 4> SplitVTs;
997 SmallVector<uint64_t, 4> Offsets;
998 ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs, FixedOffsets: &Offsets, StartingOffset: 0);
999
1000 assert(VRegs.size() == SplitVTs.size());
1001
1002 unsigned NumValues = SplitVTs.size();
1003 Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy);
1004 Type *RetPtrTy =
1005 PointerType::get(C&: RetTy->getContext(), AddressSpace: DL.getAllocaAddrSpace());
1006 LLT OffsetLLTy = getLLTForType(Ty&: *DL.getIndexType(PtrTy: RetPtrTy), DL);
1007
1008 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
1009
1010 for (unsigned I = 0; I < NumValues; ++I) {
1011 Register Addr;
1012 MIRBuilder.materializePtrAdd(Res&: Addr, Op0: DemoteReg, ValueTy: OffsetLLTy, Value: Offsets[I]);
1013 auto *MMO = MF.getMachineMemOperand(PtrInfo, f: MachineMemOperand::MOLoad,
1014 MemTy: MRI.getType(Reg: VRegs[I]),
1015 base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[I]));
1016 MIRBuilder.buildLoad(Res: VRegs[I], Addr, MMO&: *MMO);
1017 }
1018}
1019
1020void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy,
1021 ArrayRef<Register> VRegs,
1022 Register DemoteReg) const {
1023 MachineFunction &MF = MIRBuilder.getMF();
1024 MachineRegisterInfo &MRI = MF.getRegInfo();
1025 const DataLayout &DL = MF.getDataLayout();
1026
1027 SmallVector<EVT, 4> SplitVTs;
1028 SmallVector<uint64_t, 4> Offsets;
1029 ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs, FixedOffsets: &Offsets, StartingOffset: 0);
1030
1031 assert(VRegs.size() == SplitVTs.size());
1032
1033 unsigned NumValues = SplitVTs.size();
1034 Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy);
1035 unsigned AS = DL.getAllocaAddrSpace();
1036 LLT OffsetLLTy = getLLTForType(Ty&: *DL.getIndexType(C&: RetTy->getContext(), AddressSpace: AS), DL);
1037
1038 MachinePointerInfo PtrInfo(AS);
1039
1040 for (unsigned I = 0; I < NumValues; ++I) {
1041 Register Addr;
1042 MIRBuilder.materializePtrAdd(Res&: Addr, Op0: DemoteReg, ValueTy: OffsetLLTy, Value: Offsets[I]);
1043 auto *MMO = MF.getMachineMemOperand(PtrInfo, f: MachineMemOperand::MOStore,
1044 MemTy: MRI.getType(Reg: VRegs[I]),
1045 base_alignment: commonAlignment(A: BaseAlign, Offset: Offsets[I]));
1046 MIRBuilder.buildStore(Val: VRegs[I], Addr, MMO&: *MMO);
1047 }
1048}
1049
1050void CallLowering::insertSRetIncomingArgument(
1051 const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg,
1052 MachineRegisterInfo &MRI, const DataLayout &DL) const {
1053 unsigned AS = DL.getAllocaAddrSpace();
1054 DemoteReg = MRI.createGenericVirtualRegister(
1055 Ty: LLT::pointer(AddressSpace: AS, SizeInBits: DL.getPointerSizeInBits(AS)));
1056
1057 Type *PtrTy = PointerType::get(C&: F.getContext(), AddressSpace: AS);
1058
1059 SmallVector<EVT, 1> ValueVTs;
1060 ComputeValueVTs(TLI: *TLI, DL, Ty: PtrTy, ValueVTs);
1061
1062 // NOTE: Assume that a pointer won't get split into more than one VT.
1063 assert(ValueVTs.size() == 1);
1064
1065 ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(Context&: PtrTy->getContext()),
1066 ArgInfo::NoArgIndex);
1067 setArgFlags(Arg&: DemoteArg, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: F);
1068 DemoteArg.Flags[0].setSRet();
1069 SplitArgs.insert(I: SplitArgs.begin(), Elt: DemoteArg);
1070}
1071
1072void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder,
1073 const CallBase &CB,
1074 CallLoweringInfo &Info) const {
1075 const DataLayout &DL = MIRBuilder.getDataLayout();
1076 Type *RetTy = CB.getType();
1077 unsigned AS = DL.getAllocaAddrSpace();
1078 LLT FramePtrTy = LLT::pointer(AddressSpace: AS, SizeInBits: DL.getPointerSizeInBits(AS));
1079
1080 int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject(
1081 Size: DL.getTypeAllocSize(Ty: RetTy), Alignment: DL.getPrefTypeAlign(Ty: RetTy), isSpillSlot: false);
1082
1083 Register DemoteReg = MIRBuilder.buildFrameIndex(Res: FramePtrTy, Idx: FI).getReg(Idx: 0);
1084 ArgInfo DemoteArg(DemoteReg, PointerType::get(C&: RetTy->getContext(), AddressSpace: AS),
1085 ArgInfo::NoArgIndex);
1086 setArgFlags(Arg&: DemoteArg, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: CB);
1087 DemoteArg.Flags[0].setSRet();
1088
1089 Info.OrigArgs.insert(I: Info.OrigArgs.begin(), Elt: DemoteArg);
1090 Info.DemoteStackIndex = FI;
1091 Info.DemoteRegister = DemoteReg;
1092}
1093
1094bool CallLowering::checkReturn(CCState &CCInfo,
1095 SmallVectorImpl<BaseArgInfo> &Outs,
1096 CCAssignFn *Fn) const {
1097 for (unsigned I = 0, E = Outs.size(); I < E; ++I) {
1098 MVT VT = MVT::getVT(Ty: Outs[I].Ty);
1099 if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo))
1100 return false;
1101 }
1102 return true;
1103}
1104
1105void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy,
1106 AttributeList Attrs,
1107 SmallVectorImpl<BaseArgInfo> &Outs,
1108 const DataLayout &DL) const {
1109 LLVMContext &Context = RetTy->getContext();
1110 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1111
1112 SmallVector<EVT, 4> SplitVTs;
1113 ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs);
1114 addArgFlagsFromAttributes(Flags, Attrs, OpIdx: AttributeList::ReturnIndex);
1115
1116 for (EVT VT : SplitVTs) {
1117 unsigned NumParts =
1118 TLI->getNumRegistersForCallingConv(Context, CC: CallConv, VT);
1119 MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CC: CallConv, VT);
1120 Type *PartTy = EVT(RegVT).getTypeForEVT(Context);
1121
1122 for (unsigned I = 0; I < NumParts; ++I) {
1123 Outs.emplace_back(Args&: PartTy, Args&: Flags);
1124 }
1125 }
1126}
1127
1128bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const {
1129 const auto &F = MF.getFunction();
1130 Type *ReturnType = F.getReturnType();
1131 CallingConv::ID CallConv = F.getCallingConv();
1132
1133 SmallVector<BaseArgInfo, 4> SplitArgs;
1134 getReturnInfo(CallConv, RetTy: ReturnType, Attrs: F.getAttributes(), Outs&: SplitArgs,
1135 DL: MF.getDataLayout());
1136 return canLowerReturn(MF, CallConv, Outs&: SplitArgs, IsVarArg: F.isVarArg());
1137}
1138
1139bool CallLowering::parametersInCSRMatch(
1140 const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask,
1141 const SmallVectorImpl<CCValAssign> &OutLocs,
1142 const SmallVectorImpl<ArgInfo> &OutArgs) const {
1143 for (unsigned i = 0; i < OutLocs.size(); ++i) {
1144 const auto &ArgLoc = OutLocs[i];
1145 // If it's not a register, it's fine.
1146 if (!ArgLoc.isRegLoc())
1147 continue;
1148
1149 MCRegister PhysReg = ArgLoc.getLocReg();
1150
1151 // Only look at callee-saved registers.
1152 if (MachineOperand::clobbersPhysReg(RegMask: CallerPreservedMask, PhysReg))
1153 continue;
1154
1155 LLVM_DEBUG(
1156 dbgs()
1157 << "... Call has an argument passed in a callee-saved register.\n");
1158
1159 // Check if it was copied from.
1160 const ArgInfo &OutInfo = OutArgs[i];
1161
1162 if (OutInfo.Regs.size() > 1) {
1163 LLVM_DEBUG(
1164 dbgs() << "... Cannot handle arguments in multiple registers.\n");
1165 return false;
1166 }
1167
1168 // Check if we copy the register, walking through copies from virtual
1169 // registers. Note that getDefIgnoringCopies does not ignore copies from
1170 // physical registers.
1171 MachineInstr *RegDef = getDefIgnoringCopies(Reg: OutInfo.Regs[0], MRI);
1172 if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) {
1173 LLVM_DEBUG(
1174 dbgs()
1175 << "... Parameter was not copied into a VReg, cannot tail call.\n");
1176 return false;
1177 }
1178
1179 // Got a copy. Verify that it's the same as the register we want.
1180 Register CopyRHS = RegDef->getOperand(i: 1).getReg();
1181 if (CopyRHS != PhysReg) {
1182 LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into "
1183 "VReg, cannot tail call.\n");
1184 return false;
1185 }
1186 }
1187
1188 return true;
1189}
1190
1191bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
1192 MachineFunction &MF,
1193 SmallVectorImpl<ArgInfo> &InArgs,
1194 ValueAssigner &CalleeAssigner,
1195 ValueAssigner &CallerAssigner) const {
1196 const Function &F = MF.getFunction();
1197 CallingConv::ID CalleeCC = Info.CallConv;
1198 CallingConv::ID CallerCC = F.getCallingConv();
1199
1200 if (CallerCC == CalleeCC)
1201 return true;
1202
1203 SmallVector<CCValAssign, 16> ArgLocs1;
1204 CCState CCInfo1(CalleeCC, Info.IsVarArg, MF, ArgLocs1, F.getContext());
1205 if (!determineAssignments(Assigner&: CalleeAssigner, Args&: InArgs, CCInfo&: CCInfo1))
1206 return false;
1207
1208 SmallVector<CCValAssign, 16> ArgLocs2;
1209 CCState CCInfo2(CallerCC, F.isVarArg(), MF, ArgLocs2, F.getContext());
1210 if (!determineAssignments(Assigner&: CallerAssigner, Args&: InArgs, CCInfo&: CCInfo2))
1211 return false;
1212
1213 // We need the argument locations to match up exactly. If there's more in
1214 // one than the other, then we are done.
1215 if (ArgLocs1.size() != ArgLocs2.size())
1216 return false;
1217
1218 // Make sure that each location is passed in exactly the same way.
1219 for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {
1220 const CCValAssign &Loc1 = ArgLocs1[i];
1221 const CCValAssign &Loc2 = ArgLocs2[i];
1222
1223 // We need both of them to be the same. So if one is a register and one
1224 // isn't, we're done.
1225 if (Loc1.isRegLoc() != Loc2.isRegLoc())
1226 return false;
1227
1228 if (Loc1.isRegLoc()) {
1229 // If they don't have the same register location, we're done.
1230 if (Loc1.getLocReg() != Loc2.getLocReg())
1231 return false;
1232
1233 // They matched, so we can move to the next ArgLoc.
1234 continue;
1235 }
1236
1237 // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.
1238 if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())
1239 return false;
1240 }
1241
1242 return true;
1243}
1244
1245LLT CallLowering::ValueHandler::getStackValueStoreType(
1246 const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const {
1247 const MVT ValVT = VA.getValVT();
1248 if (ValVT != MVT::iPTR) {
1249 LLT ValTy(ValVT);
1250
1251 // We lost the pointeriness going through CCValAssign, so try to restore it
1252 // based on the flags.
1253 if (Flags.isPointer()) {
1254 LLT PtrTy = LLT::pointer(AddressSpace: Flags.getPointerAddrSpace(),
1255 SizeInBits: ValTy.getScalarSizeInBits());
1256 if (ValVT.isVector())
1257 return LLT::vector(EC: ValTy.getElementCount(), ScalarTy: PtrTy);
1258 return PtrTy;
1259 }
1260
1261 return ValTy;
1262 }
1263
1264 unsigned AddrSpace = Flags.getPointerAddrSpace();
1265 return LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSize(AS: AddrSpace));
1266}
1267
1268void CallLowering::ValueHandler::copyArgumentMemory(
1269 const ArgInfo &Arg, Register DstPtr, Register SrcPtr,
1270 const MachinePointerInfo &DstPtrInfo, Align DstAlign,
1271 const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize,
1272 CCValAssign &VA) const {
1273 MachineFunction &MF = MIRBuilder.getMF();
1274 MachineMemOperand *SrcMMO = MF.getMachineMemOperand(
1275 PtrInfo: SrcPtrInfo,
1276 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, Size: MemSize,
1277 BaseAlignment: SrcAlign);
1278
1279 MachineMemOperand *DstMMO = MF.getMachineMemOperand(
1280 PtrInfo: DstPtrInfo,
1281 F: MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable,
1282 Size: MemSize, BaseAlignment: DstAlign);
1283
1284 const LLT PtrTy = MRI.getType(Reg: DstPtr);
1285 const LLT SizeTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
1286
1287 auto SizeConst = MIRBuilder.buildConstant(Res: SizeTy, Val: MemSize);
1288 MIRBuilder.buildMemCpy(DstPtr, SrcPtr, Size: SizeConst, DstMMO&: *DstMMO, SrcMMO&: *SrcMMO);
1289}
1290
1291Register CallLowering::ValueHandler::extendRegister(Register ValReg,
1292 const CCValAssign &VA,
1293 unsigned MaxSizeBits) {
1294 LLT LocTy{VA.getLocVT()};
1295 LLT ValTy{VA.getValVT()};
1296
1297 if (LocTy.getSizeInBits() == ValTy.getSizeInBits())
1298 return ValReg;
1299
1300 if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) {
1301 if (MaxSizeBits <= ValTy.getSizeInBits())
1302 return ValReg;
1303 LocTy = LLT::scalar(SizeInBits: MaxSizeBits);
1304 }
1305
1306 const LLT ValRegTy = MRI.getType(Reg: ValReg);
1307 if (ValRegTy.isPointer()) {
1308 // The x32 ABI wants to zero extend 32-bit pointers to 64-bit registers, so
1309 // we have to cast to do the extension.
1310 LLT IntPtrTy = LLT::scalar(SizeInBits: ValRegTy.getSizeInBits());
1311 ValReg = MIRBuilder.buildPtrToInt(Dst: IntPtrTy, Src: ValReg).getReg(Idx: 0);
1312 }
1313
1314 switch (VA.getLocInfo()) {
1315 default:
1316 break;
1317 case CCValAssign::Full:
1318 case CCValAssign::BCvt:
1319 // FIXME: bitconverting between vector types may or may not be a
1320 // nop in big-endian situations.
1321 return ValReg;
1322 case CCValAssign::AExt: {
1323 auto MIB = MIRBuilder.buildAnyExt(Res: LocTy, Op: ValReg);
1324 return MIB.getReg(Idx: 0);
1325 }
1326 case CCValAssign::SExt: {
1327 Register NewReg = MRI.createGenericVirtualRegister(Ty: LocTy);
1328 MIRBuilder.buildSExt(Res: NewReg, Op: ValReg);
1329 return NewReg;
1330 }
1331 case CCValAssign::ZExt: {
1332 Register NewReg = MRI.createGenericVirtualRegister(Ty: LocTy);
1333 MIRBuilder.buildZExt(Res: NewReg, Op: ValReg);
1334 return NewReg;
1335 }
1336 }
1337 llvm_unreachable("unable to extend register");
1338}
1339
1340void CallLowering::ValueAssigner::anchor() {}
1341
1342Register CallLowering::IncomingValueHandler::buildExtensionHint(
1343 const CCValAssign &VA, Register SrcReg, LLT NarrowTy) {
1344 switch (VA.getLocInfo()) {
1345 case CCValAssign::LocInfo::ZExt: {
1346 return MIRBuilder
1347 .buildAssertZExt(Res: MRI.cloneVirtualRegister(VReg: SrcReg), Op: SrcReg,
1348 Size: NarrowTy.getScalarSizeInBits())
1349 .getReg(Idx: 0);
1350 }
1351 case CCValAssign::LocInfo::SExt: {
1352 return MIRBuilder
1353 .buildAssertSExt(Res: MRI.cloneVirtualRegister(VReg: SrcReg), Op: SrcReg,
1354 Size: NarrowTy.getScalarSizeInBits())
1355 .getReg(Idx: 0);
1356 break;
1357 }
1358 default:
1359 return SrcReg;
1360 }
1361}
1362
1363/// Check if we can use a basic COPY instruction between the two types.
1364///
1365/// We're currently building on top of the infrastructure using MVT, which loses
1366/// pointer information in the CCValAssign. We accept copies from physical
1367/// registers that have been reported as integers if it's to an equivalent sized
1368/// pointer LLT.
1369static bool isCopyCompatibleType(LLT SrcTy, LLT DstTy) {
1370 if (SrcTy == DstTy)
1371 return true;
1372
1373 if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1374 return false;
1375
1376 SrcTy = SrcTy.getScalarType();
1377 DstTy = DstTy.getScalarType();
1378
1379 return (SrcTy.isPointer() && DstTy.isScalar()) ||
1380 (DstTy.isPointer() && SrcTy.isScalar());
1381}
1382
1383void CallLowering::IncomingValueHandler::assignValueToReg(
1384 Register ValVReg, Register PhysReg, const CCValAssign &VA) {
1385 const MVT LocVT = VA.getLocVT();
1386 const LLT LocTy(LocVT);
1387 const LLT RegTy = MRI.getType(Reg: ValVReg);
1388
1389 if (isCopyCompatibleType(SrcTy: RegTy, DstTy: LocTy)) {
1390 MIRBuilder.buildCopy(Res: ValVReg, Op: PhysReg);
1391 return;
1392 }
1393
1394 auto Copy = MIRBuilder.buildCopy(Res: LocTy, Op: PhysReg);
1395 auto Hint = buildExtensionHint(VA, SrcReg: Copy.getReg(Idx: 0), NarrowTy: RegTy);
1396 MIRBuilder.buildTrunc(Res: ValVReg, Op: Hint);
1397}
1398