1//===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements some simple delegations needed for call lowering.
11///
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/GlobalISel/CallLowering.h"
15#include "llvm/CodeGen/Analysis.h"
16#include "llvm/CodeGen/CallingConvLower.h"
17#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18#include "llvm/CodeGen/GlobalISel/Utils.h"
19#include "llvm/CodeGen/MachineFrameInfo.h"
20#include "llvm/CodeGen/MachineOperand.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/CodeGen/TargetLowering.h"
23#include "llvm/IR/DataLayout.h"
24#include "llvm/IR/LLVMContext.h"
25#include "llvm/IR/Module.h"
26#include "llvm/Target/TargetMachine.h"
27
28#define DEBUG_TYPE "call-lowering"
29
30using namespace llvm;
31
32void CallLowering::anchor() {}
33
34/// Helper function which updates \p Flags based on the contents of \p Attrs.
35static void addFlagsFromAttrSet(ISD::ArgFlagsTy &Flags, AttributeSet Attrs) {
36 if (!Attrs.hasAttributes())
37 return;
38
39 // TODO: There are missing flags. Add them here.
40 for (Attribute Attr : Attrs) {
41 if (Attr.isStringAttribute())
42 continue;
43
44 switch (Attr.getKindAsEnum()) {
45 case Attribute::SExt:
46 Flags.setSExt();
47 break;
48 case Attribute::ZExt:
49 Flags.setZExt();
50 break;
51 case Attribute::InReg:
52 Flags.setInReg();
53 break;
54 case Attribute::StructRet:
55 Flags.setSRet();
56 break;
57 case Attribute::Nest:
58 Flags.setNest();
59 break;
60 case Attribute::ByVal:
61 Flags.setByVal();
62 break;
63 case Attribute::ByRef:
64 Flags.setByRef();
65 break;
66 case Attribute::InAlloca:
67 Flags.setInAlloca();
68 // Set the byval flag for CCAssignFn callbacks that don't know about
69 // inalloca. This way we can know how many bytes we should've allocated
70 // and how many bytes a callee cleanup function will pop. If we port
71 // inalloca to more targets, we'll have to add custom inalloca handling
72 // in the various CC lowering callbacks.
73 Flags.setByVal();
74 break;
75 case Attribute::Preallocated:
76 Flags.setPreallocated();
77 // Set the byval flag for CCAssignFn callbacks that don't know about
78 // preallocated. This way we can know how many bytes we should've
79 // allocated and how many bytes a callee cleanup function will pop. If
80 // we port preallocated to more targets, we'll have to add custom
81 // preallocated handling in the various CC lowering callbacks.
82 Flags.setByVal();
83 break;
84 case Attribute::Returned:
85 Flags.setReturned();
86 break;
87 case Attribute::SwiftSelf:
88 Flags.setSwiftSelf();
89 break;
90 case Attribute::SwiftAsync:
91 Flags.setSwiftAsync();
92 break;
93 case Attribute::SwiftError:
94 Flags.setSwiftError();
95 break;
96 default:
97 break;
98 }
99 }
100}
101
102ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call,
103 unsigned ArgIdx) const {
104 ISD::ArgFlagsTy Flags;
105 const AttributeList &Attrs = Call.getAttributes();
106 addFlagsFromAttrSet(Flags, Attrs: Attrs.getParamAttrs(ArgNo: ArgIdx));
107 if (const Function *F = Call.getCalledFunction())
108 addFlagsFromAttrSet(Flags, Attrs: F->getAttributes().getParamAttrs(ArgNo: ArgIdx));
109 return Flags;
110}
111
112ISD::ArgFlagsTy
113CallLowering::getAttributesForReturn(const CallBase &Call) const {
114 ISD::ArgFlagsTy Flags;
115 addFlagsFromAttrSet(Flags, Attrs: Call.getAttributes().getRetAttrs());
116 if (const Function *F = Call.getCalledFunction())
117 addFlagsFromAttrSet(Flags, Attrs: F->getAttributes().getRetAttrs());
118 return Flags;
119}
120
121void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags,
122 const AttributeList &Attrs,
123 unsigned OpIdx) const {
124 addFlagsFromAttrSet(Flags, Attrs: Attrs.getAttributes(Index: OpIdx));
125}
126
127bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
128 ArrayRef<Register> ResRegs,
129 ArrayRef<ArrayRef<Register>> ArgRegs,
130 Register SwiftErrorVReg,
131 std::optional<PtrAuthInfo> PAI,
132 Register ConvergenceCtrlToken,
133 std::function<Register()> GetCalleeReg) const {
134 CallLoweringInfo Info;
135 const DataLayout &DL = MIRBuilder.getDataLayout();
136 MachineFunction &MF = MIRBuilder.getMF();
137 MachineRegisterInfo &MRI = MF.getRegInfo();
138 bool CanBeTailCalled = CB.isTailCall() &&
139 isInTailCallPosition(Call: CB, TM: MF.getTarget()) &&
140 (MF.getFunction()
141 .getFnAttribute(Kind: "disable-tail-calls")
142 .getValueAsString() != "true");
143
144 CallingConv::ID CallConv = CB.getCallingConv();
145 Type *RetTy = CB.getType();
146 bool IsVarArg = CB.getFunctionType()->isVarArg();
147
148 SmallVector<BaseArgInfo, 4> SplitArgs;
149 getReturnInfo(CallConv, RetTy, Attrs: CB.getAttributes(), Outs&: SplitArgs, DL);
150 Info.CanLowerReturn = canLowerReturn(MF, CallConv, Outs&: SplitArgs, IsVarArg);
151
152 Info.IsConvergent = CB.isConvergent();
153 Info.NoMerge = CB.hasFnAttr(Kind: Attribute::NoMerge);
154
155 if (!Info.CanLowerReturn) {
156 // Callee requires sret demotion.
157 insertSRetOutgoingArgument(MIRBuilder, CB, Info);
158
159 // The sret demotion isn't compatible with tail-calls, since the sret
160 // argument points into the caller's stack frame.
161 CanBeTailCalled = false;
162 }
163
164 // First step is to marshall all the function's parameters into the correct
165 // physregs and memory locations. Gather the sequence of argument types that
166 // we'll pass to the assigner function.
167 unsigned i = 0;
168 unsigned NumFixedArgs = CB.getFunctionType()->getNumParams();
169 for (const auto &Arg : CB.args()) {
170 ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(Call: CB, ArgIdx: i)};
171 setArgFlags(Arg&: OrigArg, OpIdx: i + AttributeList::FirstArgIndex, DL, FuncInfo: CB);
172 if (i >= NumFixedArgs)
173 OrigArg.Flags[0].setVarArg();
174
175 // If we have an explicit sret argument that is an Instruction, (i.e., it
176 // might point to function-local memory), we can't meaningfully tail-call.
177 if (OrigArg.Flags[0].isSRet() && isa<Instruction>(Val: &Arg))
178 CanBeTailCalled = false;
179
180 Info.OrigArgs.push_back(Elt: OrigArg);
181 ++i;
182 }
183
184 // Try looking through a bitcast from one function type to another.
185 // Commonly happens with calls to objc_msgSend().
186 const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts();
187
188 // If IRTranslator chose to drop the ptrauth info, we can turn this into
189 // a direct call.
190 if (!PAI && CB.countOperandBundlesOfType(ID: LLVMContext::OB_ptrauth)) {
191 CalleeV = cast<ConstantPtrAuth>(Val: CalleeV)->getPointer();
192 assert(isa<Function>(CalleeV));
193 }
194
195 if (const Function *F = dyn_cast<Function>(Val: CalleeV)) {
196 if (F->hasFnAttribute(Kind: Attribute::NonLazyBind)) {
197 LLT Ty = getLLTForType(Ty&: *F->getType(), DL);
198 Register Reg = MIRBuilder.buildGlobalValue(Res: Ty, GV: F).getReg(Idx: 0);
199 Info.Callee = MachineOperand::CreateReg(Reg, isDef: false);
200 } else {
201 Info.Callee = MachineOperand::CreateGA(GV: F, Offset: 0);
202 }
203 } else if (isa<GlobalIFunc>(Val: CalleeV) || isa<GlobalAlias>(Val: CalleeV)) {
204 // IR IFuncs and Aliases can't be forward declared (only defined), so the
205 // callee must be in the same TU and therefore we can direct-call it without
206 // worrying about it being out of range.
207 Info.Callee = MachineOperand::CreateGA(GV: cast<GlobalValue>(Val: CalleeV), Offset: 0);
208 } else
209 Info.Callee = MachineOperand::CreateReg(Reg: GetCalleeReg(), isDef: false);
210
211 Register ReturnHintAlignReg;
212 Align ReturnHintAlign;
213
214 Info.OrigRet = ArgInfo{ResRegs, RetTy, 0, getAttributesForReturn(Call: CB)};
215
216 if (!Info.OrigRet.Ty->isVoidTy()) {
217 setArgFlags(Arg&: Info.OrigRet, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: CB);
218
219 if (MaybeAlign Alignment = CB.getRetAlign()) {
220 if (*Alignment > Align(1)) {
221 ReturnHintAlignReg = MRI.cloneVirtualRegister(VReg: ResRegs[0]);
222 Info.OrigRet.Regs[0] = ReturnHintAlignReg;
223 ReturnHintAlign = *Alignment;
224 }
225 }
226 }
227
228 auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_kcfi);
229 if (Bundle && CB.isIndirectCall()) {
230 Info.CFIType = cast<ConstantInt>(Val: Bundle->Inputs[0]);
231 assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
232 }
233
234 if (auto Bundle = CB.getOperandBundle(ID: LLVMContext::OB_deactivation_symbol)) {
235 Info.DeactivationSymbol = cast<GlobalValue>(Val: Bundle->Inputs[0]);
236 }
237
238 Info.CB = &CB;
239 Info.KnownCallees = CB.getMetadata(KindID: LLVMContext::MD_callees);
240 Info.CallConv = CallConv;
241 Info.SwiftErrorVReg = SwiftErrorVReg;
242 Info.PAI = PAI;
243 Info.ConvergenceCtrlToken = ConvergenceCtrlToken;
244 Info.IsMustTailCall = CB.isMustTailCall();
245 Info.IsTailCall = CanBeTailCalled;
246 Info.IsVarArg = IsVarArg;
247 if (!lowerCall(MIRBuilder, Info))
248 return false;
249
250 if (ReturnHintAlignReg && !Info.LoweredTailCall) {
251 MIRBuilder.buildAssertAlign(Res: ResRegs[0], Op: ReturnHintAlignReg,
252 AlignVal: ReturnHintAlign);
253 }
254
255 return true;
256}
257
258template <typename FuncInfoTy>
259void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
260 const DataLayout &DL,
261 const FuncInfoTy &FuncInfo) const {
262 auto &Flags = Arg.Flags[0];
263 const AttributeList &Attrs = FuncInfo.getAttributes();
264 addArgFlagsFromAttributes(Flags, Attrs, OpIdx);
265
266 PointerType *PtrTy = dyn_cast<PointerType>(Val: Arg.Ty->getScalarType());
267 if (PtrTy) {
268 Flags.setPointer();
269 Flags.setPointerAddrSpace(PtrTy->getPointerAddressSpace());
270 }
271
272 Align MemAlign = DL.getABITypeAlign(Ty: Arg.Ty);
273 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
274 Flags.isByRef()) {
275 assert(OpIdx >= AttributeList::FirstArgIndex);
276 unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex;
277
278 Type *ElementTy = FuncInfo.getParamByValType(ParamIdx);
279 if (!ElementTy)
280 ElementTy = FuncInfo.getParamByRefType(ParamIdx);
281 if (!ElementTy)
282 ElementTy = FuncInfo.getParamInAllocaType(ParamIdx);
283 if (!ElementTy)
284 ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx);
285
286 assert(ElementTy && "Must have byval, inalloca or preallocated type");
287
288 uint64_t MemSize = DL.getTypeAllocSize(Ty: ElementTy);
289 if (Flags.isByRef())
290 Flags.setByRefSize(MemSize);
291 else
292 Flags.setByValSize(MemSize);
293
294 // For ByVal, alignment should be passed from FE. BE will guess if
295 // this info is not there but there are cases it cannot get right.
296 if (auto ParamAlign = FuncInfo.getParamStackAlign(ParamIdx))
297 MemAlign = *ParamAlign;
298 else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx)))
299 MemAlign = *ParamAlign;
300 else
301 MemAlign = getTLI()->getByValTypeAlignment(Ty: ElementTy, DL);
302 } else if (OpIdx >= AttributeList::FirstArgIndex) {
303 if (auto ParamAlign =
304 FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex))
305 MemAlign = *ParamAlign;
306 }
307 Flags.setMemAlign(MemAlign);
308 Flags.setOrigAlign(DL.getABITypeAlign(Ty: Arg.Ty));
309
310 // Don't try to use the returned attribute if the argument is marked as
311 // swiftself, since it won't be passed in x0.
312 if (Flags.isSwiftSelf())
313 Flags.setReturned(false);
314}
315
316template void
317CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
318 const DataLayout &DL,
319 const Function &FuncInfo) const;
320
321template void
322CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
323 const DataLayout &DL,
324 const CallBase &FuncInfo) const;
325
326void CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
327 SmallVectorImpl<ArgInfo> &SplitArgs,
328 const DataLayout &DL,
329 CallingConv::ID CallConv,
330 SmallVectorImpl<TypeSize> *Offsets) const {
331 SmallVector<Type *, 4> SplitTys;
332 ComputeValueTypes(DL, Ty: OrigArg.Ty, Types&: SplitTys, Offsets);
333
334 if (SplitTys.size() == 0)
335 return;
336
337 if (SplitTys.size() == 1) {
338 // No splitting to do, but we want to replace the original type (e.g. [1 x
339 // double] -> double).
340 SplitArgs.emplace_back(Args: OrigArg.Regs[0], Args&: SplitTys[0], Args: OrigArg.OrigArgIndex,
341 Args: OrigArg.Flags[0], Args: OrigArg.OrigValue);
342 return;
343 }
344
345 // Create one ArgInfo for each virtual register in the original ArgInfo.
346 assert(OrigArg.Regs.size() == SplitTys.size() && "Regs / types mismatch");
347
348 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
349 Ty: OrigArg.Ty, CallConv, isVarArg: false, DL);
350 for (unsigned i = 0, e = SplitTys.size(); i < e; ++i) {
351 SplitArgs.emplace_back(Args: OrigArg.Regs[i], Args&: SplitTys[i], Args: OrigArg.OrigArgIndex,
352 Args: OrigArg.Flags[0]);
353 if (NeedsRegBlock)
354 SplitArgs.back().Flags[0].setInConsecutiveRegs();
355 }
356
357 SplitArgs.back().Flags[0].setInConsecutiveRegsLast();
358}
359
360/// Pack values \p SrcRegs to cover the vector type result \p DstRegs.
361static MachineInstrBuilder
362mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
363 ArrayRef<Register> SrcRegs) {
364 MachineRegisterInfo &MRI = *B.getMRI();
365 LLT LLTy = MRI.getType(Reg: DstRegs[0]);
366 LLT PartLLT = MRI.getType(Reg: SrcRegs[0]);
367
368 // Deal with v3s16 split into v2s16
369 LLT LCMTy = getCoverTy(OrigTy: LLTy, TargetTy: PartLLT);
370 if (LCMTy == LLTy) {
371 // Common case where no padding is needed.
372 assert(DstRegs.size() == 1);
373
374 SmallVector<Register, 8> ConcatRegs(SrcRegs.size());
375 llvm::copy(Range&: SrcRegs, Out: ConcatRegs.begin());
376
377 if (LLTy.getScalarType() != PartLLT.getScalarType())
378 for (size_t I = 0, E = SrcRegs.size(); I != E; ++I) {
379 auto BitcastDst =
380 MRI.getType(Reg: SrcRegs[I]).changeElementType(NewEltTy: LLTy.getScalarType());
381 ConcatRegs[I] = B.buildBitcast(Dst: BitcastDst, Src: SrcRegs[I]).getReg(Idx: 0);
382 }
383
384 return B.buildConcatVectors(Res: DstRegs[0], Ops: ConcatRegs);
385 }
386
387 // We need to create an unmerge to the result registers, which may require
388 // widening the original value.
389 Register UnmergeSrcReg;
390 if (LCMTy.getSizeInBits() != PartLLT.getSizeInBits()) {
391 assert(DstRegs.size() == 1);
392 return B.buildDeleteTrailingVectorElements(
393 Res: DstRegs[0], Op0: B.buildMergeLikeInstr(Res: LCMTy, Ops: SrcRegs));
394 } else {
395 // We don't need to widen anything if we're extracting a scalar which was
396 // promoted to a vector e.g. s8 -> v4s8 -> s8
397 assert(SrcRegs.size() == 1);
398 UnmergeSrcReg = SrcRegs[0];
399 }
400
401 size_t NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits();
402
403 SmallVector<Register, 8> PadDstRegs(NumDst);
404 llvm::copy(Range&: DstRegs, Out: PadDstRegs.begin());
405
406 // Create the excess dead defs for the unmerge.
407 for (size_t I = DstRegs.size(); I != NumDst; ++I)
408 PadDstRegs[I] = MRI.createGenericVirtualRegister(Ty: LLTy);
409
410 if (PartLLT != LCMTy)
411 UnmergeSrcReg = B.buildBitcast(Dst: LCMTy, Src: UnmergeSrcReg).getReg(Idx: 0);
412
413 if (PadDstRegs.size() == 1)
414 return B.buildDeleteTrailingVectorElements(Res: DstRegs[0], Op0: UnmergeSrcReg);
415 return B.buildUnmerge(Res: PadDstRegs, Op: UnmergeSrcReg);
416}
417
418void CallLowering::buildCopyFromRegs(MachineIRBuilder &B,
419 ArrayRef<Register> OrigRegs,
420 ArrayRef<Register> Regs, LLT LLTy,
421 LLT PartLLT, const ISD::ArgFlagsTy Flags) {
422 MachineRegisterInfo &MRI = *B.getMRI();
423
424 if (PartLLT == LLTy) {
425 // We should have avoided introducing a new virtual register, and just
426 // directly assigned here.
427 assert(OrigRegs[0] == Regs[0]);
428 return;
429 }
430
431 if (PartLLT.getSizeInBits() == LLTy.getSizeInBits() && OrigRegs.size() == 1 &&
432 Regs.size() == 1) {
433 B.buildBitcast(Dst: OrigRegs[0], Src: Regs[0]);
434 return;
435 }
436
437 // A vector PartLLT needs extending to LLTy's element size.
438 // E.g. <2 x s64> = G_SEXT <2 x s32>.
439 if (PartLLT.isVector() == LLTy.isVector() &&
440 PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits() &&
441 (!PartLLT.isVector() ||
442 PartLLT.getElementCount() == LLTy.getElementCount()) &&
443 OrigRegs.size() == 1 && Regs.size() == 1) {
444 Register SrcReg = Regs[0];
445
446 LLT LocTy = MRI.getType(Reg: SrcReg);
447
448 if (Flags.isSExt()) {
449 SrcReg = B.buildAssertSExt(Res: LocTy, Op: SrcReg, Size: LLTy.getScalarSizeInBits())
450 .getReg(Idx: 0);
451 } else if (Flags.isZExt()) {
452 SrcReg = B.buildAssertZExt(Res: LocTy, Op: SrcReg, Size: LLTy.getScalarSizeInBits())
453 .getReg(Idx: 0);
454 }
455
456 // Sometimes pointers are passed zero extended.
457 LLT OrigTy = MRI.getType(Reg: OrigRegs[0]);
458 if (OrigTy.isPointer()) {
459 LLT IntPtrTy = LLT::scalar(SizeInBits: OrigTy.getSizeInBits());
460 B.buildIntToPtr(Dst: OrigRegs[0], Src: B.buildTrunc(Res: IntPtrTy, Op: SrcReg));
461 return;
462 }
463
464 LLT OrigITy = OrigTy.changeToInteger();
465 if (OrigTy == OrigITy) {
466 B.buildTrunc(Res: OrigRegs[0], Op: SrcReg);
467 } else {
468 auto Trunc = B.buildTrunc(Res: OrigITy, Op: SrcReg);
469 B.buildBitcast(Dst: OrigRegs[0], Src: Trunc);
470 }
471 return;
472 }
473
474 if (!LLTy.isVector() && !PartLLT.isVector()) {
475 assert(OrigRegs.size() == 1);
476 LLT OrigTy = MRI.getType(Reg: OrigRegs[0]);
477
478 unsigned SrcSize = PartLLT.getSizeInBits().getFixedValue() * Regs.size();
479 if (SrcSize == OrigTy.getSizeInBits())
480 B.buildMergeValues(Res: OrigRegs[0], Ops: Regs);
481 else {
482 auto Widened = B.buildMergeLikeInstr(Res: LLT::integer(SizeInBits: SrcSize), Ops: Regs);
483 B.buildTrunc(Res: OrigRegs[0], Op: Widened);
484 }
485
486 return;
487 }
488
489 if (PartLLT.isVector()) {
490 assert(OrigRegs.size() == 1);
491 SmallVector<Register> CastRegs(Regs);
492
493 // If PartLLT is a mismatched vector in both number of elements and element
494 // size, e.g. PartLLT == v2s64 and LLTy is v3s32, then first coerce it to
495 // have the same elt type, i.e. v4s32.
496 // TODO: Extend this coersion to element multiples other than just 2.
497 if (TypeSize::isKnownGT(LHS: PartLLT.getSizeInBits(), RHS: LLTy.getSizeInBits()) &&
498 PartLLT.getScalarSizeInBits() == LLTy.getScalarSizeInBits() * 2 &&
499 Regs.size() == 1) {
500 LLT NewTy = PartLLT.changeElementType(NewEltTy: LLTy.getElementType())
501 .changeElementCount(EC: PartLLT.getElementCount() * 2);
502 CastRegs[0] = B.buildBitcast(Dst: NewTy, Src: Regs[0]).getReg(Idx: 0);
503 PartLLT = NewTy;
504 }
505
506 if (LLTy.getScalarSizeInBits() == PartLLT.getScalarSizeInBits()) {
507 mergeVectorRegsToResultRegs(B, DstRegs: OrigRegs, SrcRegs: CastRegs);
508 } else {
509 unsigned I = 0;
510 LLT GCDTy = getGCDType(OrigTy: LLTy, TargetTy: PartLLT);
511
512 // We are both splitting a vector, and bitcasting its element types. Cast
513 // the source pieces into the appropriate number of pieces with the result
514 // element type.
515 for (Register SrcReg : CastRegs)
516 CastRegs[I++] = B.buildBitcast(Dst: GCDTy, Src: SrcReg).getReg(Idx: 0);
517 mergeVectorRegsToResultRegs(B, DstRegs: OrigRegs, SrcRegs: CastRegs);
518 }
519
520 return;
521 }
522
523 assert(LLTy.isVector() && !PartLLT.isVector());
524
525 LLT DstEltTy = LLTy.getElementType();
526
527 // Pointer information was discarded. We'll need to coerce some register types
528 // to avoid violating type constraints.
529 LLT RealDstEltTy = MRI.getType(Reg: OrigRegs[0]).getElementType();
530
531 assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits());
532
533 if (DstEltTy == PartLLT) {
534 // Vector was trivially scalarized.
535
536 if (RealDstEltTy.isPointer()) {
537 for (Register Reg : Regs)
538 MRI.setType(VReg: Reg, Ty: RealDstEltTy);
539 }
540
541 B.buildBuildVector(Res: OrigRegs[0], Ops: Regs);
542 } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) {
543 // Deal with vector with 64-bit elements decomposed to 32-bit
544 // registers. Need to create intermediate 64-bit elements.
545 SmallVector<Register, 8> EltMerges;
546 int PartsPerElt =
547 divideCeil(Numerator: DstEltTy.getSizeInBits(), Denominator: PartLLT.getSizeInBits());
548 LLT ExtendedPartTy = LLT::integer(SizeInBits: PartLLT.getSizeInBits() * PartsPerElt);
549
550 for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) {
551 auto Merge =
552 B.buildMergeLikeInstr(Res: ExtendedPartTy, Ops: Regs.take_front(N: PartsPerElt));
553 if (ExtendedPartTy.getSizeInBits() > RealDstEltTy.getSizeInBits())
554 Merge = B.buildTrunc(Res: RealDstEltTy, Op: Merge);
555 // Fix the type in case this is really a vector of pointers.
556 MRI.setType(VReg: Merge.getReg(Idx: 0), Ty: RealDstEltTy);
557 EltMerges.push_back(Elt: Merge.getReg(Idx: 0));
558 Regs = Regs.drop_front(N: PartsPerElt);
559 }
560
561 B.buildBuildVector(Res: OrigRegs[0], Ops: EltMerges);
562 } else {
563 // Vector was split, and elements promoted to a wider type.
564 // FIXME: Should handle floating point promotions.
565 unsigned NumElts = LLTy.getNumElements();
566 LLT BVType = LLT::fixed_vector(NumElements: NumElts, ScalarTy: PartLLT);
567
568 Register BuildVec;
569 if (NumElts == Regs.size())
570 BuildVec = B.buildBuildVector(Res: BVType, Ops: Regs).getReg(Idx: 0);
571 else {
572 // Vector elements are packed in the inputs.
573 // e.g. we have a <4 x s16> but 2 x s32 in regs.
574 assert(NumElts > Regs.size());
575 LLT SrcEltTy = MRI.getType(Reg: Regs[0]);
576
577 LLT OriginalEltTy = MRI.getType(Reg: OrigRegs[0]).getElementType();
578
579 // Input registers contain packed elements.
580 // Determine how many elements per reg.
581 assert((SrcEltTy.getSizeInBits() % OriginalEltTy.getSizeInBits()) == 0);
582 unsigned EltPerReg =
583 (SrcEltTy.getSizeInBits() / OriginalEltTy.getSizeInBits());
584
585 SmallVector<Register, 0> BVRegs;
586 BVRegs.reserve(N: Regs.size() * EltPerReg);
587 for (Register R : Regs) {
588 auto Unmerge = B.buildUnmerge(Res: OriginalEltTy, Op: R);
589 for (unsigned K = 0; K < EltPerReg; ++K)
590 BVRegs.push_back(Elt: B.buildAnyExt(Res: PartLLT, Op: Unmerge.getReg(Idx: K)).getReg(Idx: 0));
591 }
592
593 // We may have some more elements in BVRegs, e.g. if we have 2 s32 pieces
594 // for a <3 x s16> vector. We should have less than EltPerReg extra items.
595 if (BVRegs.size() > NumElts) {
596 assert((BVRegs.size() - NumElts) < EltPerReg);
597 BVRegs.truncate(N: NumElts);
598 }
599 BuildVec = B.buildBuildVector(Res: BVType, Ops: BVRegs).getReg(Idx: 0);
600 }
601 B.buildTrunc(Res: OrigRegs[0], Op: BuildVec);
602 }
603}
604
605void CallLowering::buildCopyToRegs(MachineIRBuilder &B,
606 ArrayRef<Register> DstRegs, Register SrcReg,
607 LLT SrcTy, LLT PartTy, unsigned ExtendOp) {
608 // We could just insert a regular copy, but this is unreachable at the moment.
609 assert(SrcTy != PartTy && "identical part types shouldn't reach here");
610
611 const TypeSize PartSize = PartTy.getSizeInBits();
612
613 if (PartSize == SrcTy.getSizeInBits() && DstRegs.size() == 1) {
614 // TODO: Handle int<->ptr casts. It just happens the ABI lowering
615 // assignments are not pointer aware.
616 B.buildBitcast(Dst: DstRegs[0], Src: SrcReg);
617 return;
618 }
619
620 if (PartTy.isVector() == SrcTy.isVector() &&
621 PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) {
622 assert(DstRegs.size() == 1);
623 // Convert float to integer if needed
624 LLT SrcITy = SrcTy.changeToInteger();
625 LLT PartITy = PartTy.changeToInteger();
626 if (SrcTy != SrcITy)
627 SrcReg = B.buildBitcast(Dst: SrcITy, Src: SrcReg).getReg(Idx: 0);
628
629 // Emit the sext/zext/anyext
630 Register DstIReg =
631 B.buildInstr(Opc: ExtendOp,
632 DstOps: PartITy == PartTy ? DstOp(DstRegs[0]) : DstOp(PartITy),
633 SrcOps: {SrcReg})
634 .getReg(Idx: 0);
635 // Convert back to the original type if needed
636 if (PartITy != PartTy)
637 B.buildBitcast(Dst: DstRegs[0], Src: DstIReg);
638 return;
639 }
640
641 if (SrcTy.isVector() && !PartTy.isVector() &&
642 TypeSize::isKnownGT(LHS: PartSize, RHS: SrcTy.getElementType().getSizeInBits()) &&
643 SrcTy.getElementCount() == ElementCount::getFixed(MinVal: DstRegs.size())) {
644 // Vector was scalarized, and the elements extended.
645 auto UnmergeToEltTy = B.buildUnmerge(Res: SrcTy.getElementType(), Op: SrcReg);
646 for (int i = 0, e = DstRegs.size(); i != e; ++i)
647 B.buildAnyExt(Res: DstRegs[i], Op: UnmergeToEltTy.getReg(Idx: i));
648 return;
649 }
650
651 if (SrcTy.isVector() && PartTy.isVector() &&
652 PartTy.getSizeInBits() == SrcTy.getSizeInBits() &&
653 ElementCount::isKnownLT(LHS: SrcTy.getElementCount(),
654 RHS: PartTy.getElementCount())) {
655 // A coercion like: v2f32 -> v4f32 or nxv2f32 -> nxv4f32
656 Register DstReg = DstRegs.front();
657 B.buildPadVectorWithUndefElements(Res: DstReg, Op0: SrcReg);
658 return;
659 }
660
661 LLT GCDTy = getGCDType(OrigTy: SrcTy, TargetTy: PartTy);
662 if (GCDTy == PartTy) {
663 // If this already evenly divisible, we can create a simple unmerge.
664 B.buildUnmerge(Res: DstRegs, Op: SrcReg);
665 return;
666 }
667
668 if (SrcTy.isVector() && !PartTy.isVector() &&
669 SrcTy.getScalarSizeInBits() > PartTy.getSizeInBits()) {
670 LLT ExtTy =
671 LLT::vector(EC: SrcTy.getElementCount(),
672 ScalarTy: LLT::integer(SizeInBits: PartTy.getScalarSizeInBits() * DstRegs.size() /
673 SrcTy.getNumElements()));
674 auto Ext = B.buildAnyExt(Res: ExtTy, Op: SrcReg);
675 B.buildUnmerge(Res: DstRegs, Op: Ext);
676 return;
677 }
678
679 MachineRegisterInfo &MRI = *B.getMRI();
680 LLT DstTy = MRI.getType(Reg: DstRegs[0]);
681 LLT CoverTy = getCoverTy(OrigTy: SrcTy, TargetTy: PartTy);
682 if (SrcTy.isVector() && DstRegs.size() > 1) {
683 TypeSize FullCoverSize = DstTy.getSizeInBits() * DstRegs.size();
684
685 LLT EltTy = SrcTy.getElementType();
686 TypeSize EltSize = EltTy.getSizeInBits();
687 if (FullCoverSize.isKnownMultipleOf(RHS: EltSize)) {
688 TypeSize VecSize = FullCoverSize.divideCoefficientBy(RHS: EltSize);
689 CoverTy =
690 LLT::vector(EC: ElementCount::get(MinVal: VecSize, Scalable: VecSize.isScalable()), ScalarTy: EltTy);
691 }
692 }
693
694 if (PartTy.isVector() && CoverTy == PartTy) {
695 assert(DstRegs.size() == 1);
696 B.buildPadVectorWithUndefElements(Res: DstRegs[0], Op0: SrcReg);
697 return;
698 }
699
700 const unsigned DstSize = DstTy.getSizeInBits();
701 const unsigned SrcSize = SrcTy.getSizeInBits();
702 unsigned CoveringSize = CoverTy.getSizeInBits();
703
704 Register UnmergeSrc = SrcReg;
705
706 if (!CoverTy.isVector() && CoveringSize != SrcSize) {
707 // For scalars, it's common to be able to use a simple extension.
708 if (SrcTy.isScalar() && DstTy.isScalar()) {
709 CoveringSize = alignTo(Value: SrcSize, Align: DstSize);
710 LLT CoverTy = LLT::integer(SizeInBits: CoveringSize);
711 UnmergeSrc = B.buildInstr(Opc: ExtendOp, DstOps: {CoverTy}, SrcOps: {SrcReg}).getReg(Idx: 0);
712 } else {
713 // Widen to the common type.
714 // FIXME: This should respect the extend type
715 Register Undef = B.buildUndef(Res: SrcTy).getReg(Idx: 0);
716 SmallVector<Register, 8> MergeParts(1, SrcReg);
717 for (unsigned Size = SrcSize; Size != CoveringSize; Size += SrcSize)
718 MergeParts.push_back(Elt: Undef);
719 UnmergeSrc = B.buildMergeLikeInstr(Res: CoverTy, Ops: MergeParts).getReg(Idx: 0);
720 }
721 }
722
723 if (CoverTy.isVector() && CoveringSize != SrcSize)
724 UnmergeSrc = B.buildPadVectorWithUndefElements(Res: CoverTy, Op0: SrcReg).getReg(Idx: 0);
725
726 B.buildUnmerge(Res: DstRegs, Op: UnmergeSrc);
727}
728
729bool CallLowering::determineAndHandleAssignments(
730 ValueHandler &Handler, ValueAssigner &Assigner,
731 SmallVectorImpl<ArgInfo> &Args, MachineIRBuilder &MIRBuilder,
732 CallingConv::ID CallConv, bool IsVarArg,
733 ArrayRef<Register> ThisReturnRegs) const {
734 MachineFunction &MF = MIRBuilder.getMF();
735 const Function &F = MF.getFunction();
736 SmallVector<CCValAssign, 16> ArgLocs;
737
738 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext());
739 if (!determineAssignments(Assigner, Args, CCInfo))
740 return false;
741
742 return handleAssignments(Handler, Args, CCState&: CCInfo, ArgLocs, MIRBuilder,
743 ThisReturnRegs);
744}
745
746static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) {
747 if (Flags.isSExt())
748 return TargetOpcode::G_SEXT;
749 if (Flags.isZExt())
750 return TargetOpcode::G_ZEXT;
751 return TargetOpcode::G_ANYEXT;
752}
753
754bool CallLowering::determineAssignments(ValueAssigner &Assigner,
755 SmallVectorImpl<ArgInfo> &Args,
756 CCState &CCInfo) const {
757 LLVMContext &Ctx = CCInfo.getContext();
758 const DataLayout &DL = CCInfo.getMachineFunction().getDataLayout();
759 const CallingConv::ID CallConv = CCInfo.getCallingConv();
760
761 unsigned NumArgs = Args.size();
762 for (unsigned i = 0; i != NumArgs; ++i) {
763 EVT CurVT = TLI->getValueType(DL, Ty: Args[i].Ty);
764
765 MVT NewVT = TLI->getRegisterTypeForCallingConv(Context&: Ctx, CC: CallConv, VT: CurVT);
766
767 // If we need to split the type over multiple regs, check it's a scenario
768 // we currently support.
769 unsigned NumParts =
770 TLI->getNumRegistersForCallingConv(Context&: Ctx, CC: CallConv, VT: CurVT);
771
772 if (NumParts == 1) {
773 // Try to use the register type if we couldn't assign the VT.
774 if (Assigner.assignArg(ValNo: i, OrigVT: CurVT, ValVT: NewVT, LocVT: NewVT, LocInfo: CCValAssign::Full, Info: Args[i],
775 Flags: Args[i].Flags[0], State&: CCInfo))
776 return false;
777 continue;
778 }
779
780 // For incoming arguments (physregs to vregs), we could have values in
781 // physregs (or memlocs) which we want to extract and copy to vregs.
782 // During this, we might have to deal with the LLT being split across
783 // multiple regs, so we have to record this information for later.
784 //
785 // If we have outgoing args, then we have the opposite case. We have a
786 // vreg with an LLT which we want to assign to a physical location, and
787 // we might have to record that the value has to be split later.
788
789 // We're handling an incoming arg which is split over multiple regs.
790 // E.g. passing an s128 on AArch64.
791 ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
792 Args[i].Flags.clear();
793
794 for (unsigned Part = 0; Part < NumParts; ++Part) {
795 ISD::ArgFlagsTy Flags = OrigFlags;
796 if (Part == 0) {
797 Flags.setSplit();
798 } else {
799 Flags.setOrigAlign(Align(1));
800 if (Part == NumParts - 1)
801 Flags.setSplitEnd();
802 }
803
804 Args[i].Flags.push_back(Elt: Flags);
805 if (Assigner.assignArg(ValNo: i, OrigVT: CurVT, ValVT: NewVT, LocVT: NewVT, LocInfo: CCValAssign::Full, Info: Args[i],
806 Flags: Args[i].Flags[Part], State&: CCInfo)) {
807 // Still couldn't assign this smaller part type for some reason.
808 return false;
809 }
810 }
811 }
812
813 return true;
814}
815
816bool CallLowering::handleAssignments(ValueHandler &Handler,
817 SmallVectorImpl<ArgInfo> &Args,
818 CCState &CCInfo,
819 SmallVectorImpl<CCValAssign> &ArgLocs,
820 MachineIRBuilder &MIRBuilder,
821 ArrayRef<Register> ThisReturnRegs) const {
822 MachineFunction &MF = MIRBuilder.getMF();
823 MachineRegisterInfo &MRI = MF.getRegInfo();
824 const Function &F = MF.getFunction();
825 const DataLayout &DL = F.getDataLayout();
826
827 const unsigned NumArgs = Args.size();
828
829 // Stores thunks for outgoing register assignments. This is used so we delay
830 // generating register copies until mem loc assignments are done. We do this
831 // so that if the target is using the delayed stack protector feature, we can
832 // find the split point of the block accurately. E.g. if we have:
833 // G_STORE %val, %memloc
834 // $x0 = COPY %foo
835 // $x1 = COPY %bar
836 // CALL func
837 // ... then the split point for the block will correctly be at, and including,
838 // the copy to $x0. If instead the G_STORE instruction immediately precedes
839 // the CALL, then we'd prematurely choose the CALL as the split point, thus
840 // generating a split block with a CALL that uses undefined physregs.
841 SmallVector<std::function<void()>> DelayedOutgoingRegAssignments;
842
843 for (unsigned i = 0, j = 0; i != NumArgs; ++i, ++j) {
844 assert(j < ArgLocs.size() && "Skipped too many arg locs");
845 CCValAssign &VA = ArgLocs[j];
846 assert(VA.getValNo() == i && "Location doesn't correspond to current arg");
847
848 if (VA.needsCustom()) {
849 std::function<void()> Thunk;
850 unsigned NumArgRegs = Handler.assignCustomValue(
851 Arg&: Args[i], VAs: ArrayRef(ArgLocs).slice(N: j), Thunk: &Thunk);
852 if (Thunk)
853 DelayedOutgoingRegAssignments.emplace_back(Args&: Thunk);
854 if (!NumArgRegs)
855 return false;
856 j += (NumArgRegs - 1);
857 continue;
858 }
859
860 auto AllocaAddressSpace = MF.getDataLayout().getAllocaAddrSpace();
861
862 const MVT ValVT = VA.getValVT();
863 const MVT LocVT = VA.getLocVT();
864
865 const LLT LocTy = getLLTForMVT(Ty: LocVT);
866 const LLT ValTy = getLLTForMVT(Ty: ValVT);
867 const LLT NewLLT = Handler.isIncomingArgumentHandler() ? LocTy : ValTy;
868 const EVT OrigVT = TLI->getValueType(DL, Ty: Args[i].Ty);
869 // Use the EVT here to strip pointerness.
870 const LLT OrigTy = getLLTForType(Ty&: *OrigVT.getTypeForEVT(Context&: F.getContext()), DL);
871 const LLT PointerTy = LLT::pointer(
872 AddressSpace: AllocaAddressSpace, SizeInBits: DL.getPointerSizeInBits(AS: AllocaAddressSpace));
873
874 // Expected to be multiple regs for a single incoming arg.
875 // There should be Regs.size() ArgLocs per argument.
876 // This should be the same as getNumRegistersForCallingConv
877 const unsigned NumParts = Args[i].Flags.size();
878
879 // Now split the registers into the assigned types.
880 Args[i].OrigRegs.assign(in_start: Args[i].Regs.begin(), in_end: Args[i].Regs.end());
881
882 if (NumParts != 1 || NewLLT != OrigTy) {
883 // If we can't directly assign the register, we need one or more
884 // intermediate values.
885 Args[i].Regs.resize(N: NumParts);
886
887 // When we have indirect parameter passing we are receiving a pointer,
888 // that points to the actual value, so we need one "temporary" pointer.
889 if (VA.getLocInfo() == CCValAssign::Indirect) {
890 if (Handler.isIncomingArgumentHandler())
891 Args[i].Regs[0] = MRI.createGenericVirtualRegister(Ty: PointerTy);
892 } else {
893 // For each split register, create and assign a vreg that will store
894 // the incoming component of the larger value. These will later be
895 // merged to form the final vreg.
896 for (unsigned Part = 0; Part < NumParts; ++Part)
897 Args[i].Regs[Part] = MRI.createGenericVirtualRegister(Ty: NewLLT);
898 }
899 }
900
901 assert((j + (NumParts - 1)) < ArgLocs.size() &&
902 "Too many regs for number of args");
903
904 // Coerce into outgoing value types before register assignment.
905 if (!Handler.isIncomingArgumentHandler() && OrigTy != ValTy &&
906 VA.getLocInfo() != CCValAssign::Indirect) {
907 assert(Args[i].OrigRegs.size() == 1);
908 buildCopyToRegs(B&: MIRBuilder, DstRegs: Args[i].Regs, SrcReg: Args[i].OrigRegs[0], SrcTy: OrigTy,
909 PartTy: ValTy, ExtendOp: extendOpFromFlags(Flags: Args[i].Flags[0]));
910 }
911
912 bool IndirectParameterPassingHandled = false;
913 bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(VT: OrigVT, DL);
914 for (unsigned Part = 0; Part < NumParts; ++Part) {
915 assert((VA.getLocInfo() != CCValAssign::Indirect || Part == 0) &&
916 "Only the first parameter should be processed when "
917 "handling indirect passing!");
918 Register ArgReg = Args[i].Regs[Part];
919 // There should be Regs.size() ArgLocs per argument.
920 unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part;
921 CCValAssign &VA = ArgLocs[j + Idx];
922 const ISD::ArgFlagsTy Flags = Args[i].Flags[Part];
923
924 // We found an indirect parameter passing, and we have an
925 // OutgoingValueHandler as our handler (so we are at the call site or the
926 // return value). In this case, start the construction of the following
927 // GMIR, that is responsible for the preparation of indirect parameter
928 // passing:
929 //
930 // %1(indirectly passed type) = The value to pass
931 // %3(pointer) = G_FRAME_INDEX %stack.0
932 // G_STORE %1, %3 :: (store (s128), align 8)
933 //
934 // After this GMIR, the remaining part of the loop body will decide how
935 // to get the value to the caller and we break out of the loop.
936 if (VA.getLocInfo() == CCValAssign::Indirect &&
937 !Handler.isIncomingArgumentHandler()) {
938 Align AlignmentForStored = DL.getPrefTypeAlign(Ty: Args[i].Ty);
939 MachineFrameInfo &MFI = MF.getFrameInfo();
940 // Get some space on the stack for the value, so later we can pass it
941 // as a reference.
942 int FrameIdx = MFI.CreateStackObject(Size: OrigTy.getScalarSizeInBits(),
943 Alignment: AlignmentForStored, isSpillSlot: false);
944 Register PointerToStackReg =
945 MIRBuilder.buildFrameIndex(Res: PointerTy, Idx: FrameIdx).getReg(Idx: 0);
946 MachinePointerInfo StackPointerMPO =
947 MachinePointerInfo::getFixedStack(MF, FI: FrameIdx);
948 // Store the value in the previously created stack space.
949 MIRBuilder.buildStore(Val: Args[i].OrigRegs[Part], Addr: PointerToStackReg,
950 PtrInfo: StackPointerMPO,
951 Alignment: inferAlignFromPtrInfo(MF, MPO: StackPointerMPO));
952
953 ArgReg = PointerToStackReg;
954 IndirectParameterPassingHandled = true;
955 }
956
957 if (VA.isMemLoc() && !Flags.isByVal()) {
958 // Individual pieces may have been spilled to the stack and others
959 // passed in registers.
960
961 // TODO: The memory size may be larger than the value we need to
962 // store. We may need to adjust the offset for big endian targets.
963 LLT MemTy = Handler.getStackValueStoreType(DL, VA, Flags);
964
965 MachinePointerInfo MPO;
966 Register StackAddr =
967 Handler.getStackAddress(MemSize: VA.getLocInfo() == CCValAssign::Indirect
968 ? PointerTy.getSizeInBytes()
969 : MemTy.getSizeInBytes(),
970 Offset: VA.getLocMemOffset(), MPO, Flags);
971
972 // Finish the handling of indirect passing from the passers
973 // (OutgoingParameterHandler) side.
974 // This branch is needed, so the pointer to the value is loaded onto the
975 // stack.
976 if (VA.getLocInfo() == CCValAssign::Indirect)
977 Handler.assignValueToAddress(ValVReg: ArgReg, Addr: StackAddr, MemTy: PointerTy, MPO, VA);
978 else
979 Handler.assignValueToAddress(Arg: Args[i], ValRegIndex: Part, Addr: StackAddr, MemTy, MPO,
980 VA);
981 } else if (VA.isMemLoc() && Flags.isByVal()) {
982 assert(Args[i].Regs.size() == 1 && "didn't expect split byval pointer");
983
984 if (Handler.isIncomingArgumentHandler()) {
985 // We just need to copy the frame index value to the pointer.
986 MachinePointerInfo MPO;
987 Register StackAddr = Handler.getStackAddress(
988 MemSize: Flags.getByValSize(), Offset: VA.getLocMemOffset(), MPO, Flags);
989 MIRBuilder.buildCopy(Res: Args[i].Regs[0], Op: StackAddr);
990 } else {
991 // For outgoing byval arguments, insert the implicit copy byval
992 // implies, such that writes in the callee do not modify the caller's
993 // value.
994 uint64_t MemSize = Flags.getByValSize();
995 int64_t Offset = VA.getLocMemOffset();
996
997 MachinePointerInfo DstMPO;
998 Register StackAddr =
999 Handler.getStackAddress(MemSize, Offset, MPO&: DstMPO, Flags);
1000
1001 MachinePointerInfo SrcMPO(Args[i].OrigValue);
1002 if (!Args[i].OrigValue) {
1003 // We still need to accurately track the stack address space if we
1004 // don't know the underlying value.
1005 const LLT PtrTy = MRI.getType(Reg: StackAddr);
1006 SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace());
1007 }
1008
1009 Align DstAlign = std::max(a: Flags.getNonZeroByValAlign(),
1010 b: inferAlignFromPtrInfo(MF, MPO: DstMPO));
1011
1012 Align SrcAlign = std::max(a: Flags.getNonZeroByValAlign(),
1013 b: inferAlignFromPtrInfo(MF, MPO: SrcMPO));
1014
1015 Handler.copyArgumentMemory(Arg: Args[i], DstPtr: StackAddr, SrcPtr: Args[i].Regs[0],
1016 DstPtrInfo: DstMPO, DstAlign, SrcPtrInfo: SrcMPO, SrcAlign,
1017 MemSize, VA);
1018 }
1019 } else if (i == 0 && !ThisReturnRegs.empty() &&
1020 Handler.isIncomingArgumentHandler() &&
1021 isTypeIsValidForThisReturn(Ty: ValVT)) {
1022 Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: ThisReturnRegs[Part], VA, Flags);
1023 } else if (Handler.isIncomingArgumentHandler()) {
1024 Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: VA.getLocReg(), VA, Flags);
1025 } else {
1026 DelayedOutgoingRegAssignments.emplace_back(Args: [=, &Handler]() {
1027 Handler.assignValueToReg(ValVReg: ArgReg, PhysReg: VA.getLocReg(), VA, Flags);
1028 });
1029 }
1030
1031 // Finish the handling of indirect parameter passing when receiving
1032 // the value (we are in the called function or the caller when receiving
1033 // the return value).
1034 if (VA.getLocInfo() == CCValAssign::Indirect &&
1035 Handler.isIncomingArgumentHandler()) {
1036 Align Alignment = DL.getABITypeAlign(Ty: Args[i].Ty);
1037 MachinePointerInfo MPO = MachinePointerInfo::getUnknownStack(MF);
1038
1039 // Since we are doing indirect parameter passing, we know that the value
1040 // in the temporary register is not the value passed to the function,
1041 // but rather a pointer to that value. Let's load that value into the
1042 // virtual register where the parameter should go.
1043 MIRBuilder.buildLoad(Res: Args[i].OrigRegs[0], Addr: Args[i].Regs[0], PtrInfo: MPO,
1044 Alignment);
1045
1046 IndirectParameterPassingHandled = true;
1047 }
1048
1049 if (IndirectParameterPassingHandled)
1050 break;
1051 }
1052
1053 // Now that all pieces have been assigned, re-pack the register typed values
1054 // into the original value typed registers. This is only necessary, when
1055 // the value was passed in multiple registers, not indirectly.
1056 if (Handler.isIncomingArgumentHandler() && OrigVT != LocVT &&
1057 !IndirectParameterPassingHandled) {
1058 // Merge the split registers into the expected larger result vregs of
1059 // the original call.
1060 buildCopyFromRegs(B&: MIRBuilder, OrigRegs: Args[i].OrigRegs, Regs: Args[i].Regs, LLTy: OrigTy,
1061 PartLLT: LocTy, Flags: Args[i].Flags[0]);
1062 }
1063
1064 j += NumParts - 1;
1065 }
1066 for (auto &Fn : DelayedOutgoingRegAssignments)
1067 Fn();
1068
1069 return true;
1070}
1071
1072void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy,
1073 ArrayRef<Register> VRegs, Register DemoteReg,
1074 int FI) const {
1075 MachineFunction &MF = MIRBuilder.getMF();
1076 MachineRegisterInfo &MRI = MF.getRegInfo();
1077 const DataLayout &DL = MF.getDataLayout();
1078
1079 SmallVector<EVT, 4> SplitVTs;
1080 SmallVector<uint64_t, 4> Offsets;
1081 ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs, /*MemVTs=*/nullptr, FixedOffsets: &Offsets, StartingOffset: 0);
1082
1083 assert(VRegs.size() == SplitVTs.size());
1084
1085 unsigned NumValues = SplitVTs.size();
1086 Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy);
1087 Type *RetPtrTy =
1088 PointerType::get(C&: RetTy->getContext(), AddressSpace: DL.getAllocaAddrSpace());
1089 LLT OffsetLLTy = getLLTForType(Ty&: *DL.getIndexType(PtrTy: RetPtrTy), DL);
1090
1091 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
1092
1093 for (unsigned I = 0; I < NumValues; ++I) {
1094 Register Addr;
1095 MIRBuilder.materializeObjectPtrOffset(Res&: Addr, Op0: DemoteReg, ValueTy: OffsetLLTy,
1096 Value: Offsets[I]);
1097 auto *MMO = MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad,
1098 MemTy: MRI.getType(Reg: VRegs[I]),
1099 BaseAlignment: commonAlignment(A: BaseAlign, Offset: Offsets[I]));
1100 MIRBuilder.buildLoad(Res: VRegs[I], Addr, MMO&: *MMO);
1101 }
1102}
1103
1104void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy,
1105 ArrayRef<Register> VRegs,
1106 Register DemoteReg) const {
1107 MachineFunction &MF = MIRBuilder.getMF();
1108 MachineRegisterInfo &MRI = MF.getRegInfo();
1109 const DataLayout &DL = MF.getDataLayout();
1110
1111 SmallVector<EVT, 4> SplitVTs;
1112 SmallVector<uint64_t, 4> Offsets;
1113 ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs, /*MemVTs=*/nullptr, FixedOffsets: &Offsets, StartingOffset: 0);
1114
1115 assert(VRegs.size() == SplitVTs.size());
1116
1117 unsigned NumValues = SplitVTs.size();
1118 Align BaseAlign = DL.getPrefTypeAlign(Ty: RetTy);
1119 unsigned AS = DL.getAllocaAddrSpace();
1120 LLT OffsetLLTy = getLLTForType(Ty&: *DL.getIndexType(C&: RetTy->getContext(), AddressSpace: AS), DL);
1121
1122 MachinePointerInfo PtrInfo(AS);
1123
1124 for (unsigned I = 0; I < NumValues; ++I) {
1125 Register Addr;
1126 MIRBuilder.materializeObjectPtrOffset(Res&: Addr, Op0: DemoteReg, ValueTy: OffsetLLTy,
1127 Value: Offsets[I]);
1128 auto *MMO = MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
1129 MemTy: MRI.getType(Reg: VRegs[I]),
1130 BaseAlignment: commonAlignment(A: BaseAlign, Offset: Offsets[I]));
1131 MIRBuilder.buildStore(Val: VRegs[I], Addr, MMO&: *MMO);
1132 }
1133}
1134
1135void CallLowering::insertSRetIncomingArgument(
1136 const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg,
1137 MachineRegisterInfo &MRI, const DataLayout &DL) const {
1138 unsigned AS = DL.getAllocaAddrSpace();
1139 DemoteReg = MRI.createGenericVirtualRegister(
1140 Ty: LLT::pointer(AddressSpace: AS, SizeInBits: DL.getPointerSizeInBits(AS)));
1141
1142 Type *PtrTy = PointerType::get(C&: F.getContext(), AddressSpace: AS);
1143
1144 SmallVector<EVT, 1> ValueVTs;
1145 ComputeValueVTs(TLI: *TLI, DL, Ty: PtrTy, ValueVTs);
1146
1147 // NOTE: Assume that a pointer won't get split into more than one VT.
1148 assert(ValueVTs.size() == 1);
1149
1150 ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(Context&: PtrTy->getContext()),
1151 ArgInfo::NoArgIndex);
1152 setArgFlags(Arg&: DemoteArg, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: F);
1153 DemoteArg.Flags[0].setSRet();
1154 SplitArgs.insert(I: SplitArgs.begin(), Elt: DemoteArg);
1155}
1156
1157void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder,
1158 const CallBase &CB,
1159 CallLoweringInfo &Info) const {
1160 const DataLayout &DL = MIRBuilder.getDataLayout();
1161 Type *RetTy = CB.getType();
1162 unsigned AS = DL.getAllocaAddrSpace();
1163 LLT FramePtrTy = LLT::pointer(AddressSpace: AS, SizeInBits: DL.getPointerSizeInBits(AS));
1164
1165 int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject(
1166 Size: DL.getTypeAllocSize(Ty: RetTy), Alignment: DL.getPrefTypeAlign(Ty: RetTy), isSpillSlot: false);
1167
1168 Register DemoteReg = MIRBuilder.buildFrameIndex(Res: FramePtrTy, Idx: FI).getReg(Idx: 0);
1169 ArgInfo DemoteArg(DemoteReg, PointerType::get(C&: RetTy->getContext(), AddressSpace: AS),
1170 ArgInfo::NoArgIndex);
1171 setArgFlags(Arg&: DemoteArg, OpIdx: AttributeList::ReturnIndex, DL, FuncInfo: CB);
1172 DemoteArg.Flags[0].setSRet();
1173
1174 Info.OrigArgs.insert(I: Info.OrigArgs.begin(), Elt: DemoteArg);
1175 Info.DemoteStackIndex = FI;
1176 Info.DemoteRegister = DemoteReg;
1177}
1178
1179bool CallLowering::checkReturn(CCState &CCInfo,
1180 SmallVectorImpl<BaseArgInfo> &Outs,
1181 CCAssignFn *Fn) const {
1182 for (unsigned I = 0, E = Outs.size(); I < E; ++I) {
1183 MVT VT = MVT::getVT(Ty: Outs[I].Ty);
1184 if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], Outs[I].Ty, CCInfo))
1185 return false;
1186 }
1187 return true;
1188}
1189
1190void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy,
1191 AttributeList Attrs,
1192 SmallVectorImpl<BaseArgInfo> &Outs,
1193 const DataLayout &DL) const {
1194 LLVMContext &Context = RetTy->getContext();
1195 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1196
1197 SmallVector<EVT, 4> SplitVTs;
1198 ComputeValueVTs(TLI: *TLI, DL, Ty: RetTy, ValueVTs&: SplitVTs);
1199 addArgFlagsFromAttributes(Flags, Attrs, OpIdx: AttributeList::ReturnIndex);
1200
1201 for (EVT VT : SplitVTs) {
1202 unsigned NumParts =
1203 TLI->getNumRegistersForCallingConv(Context, CC: CallConv, VT);
1204 MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CC: CallConv, VT);
1205 Type *PartTy = EVT(RegVT).getTypeForEVT(Context);
1206
1207 for (unsigned I = 0; I < NumParts; ++I) {
1208 Outs.emplace_back(Args&: PartTy, Args&: Flags);
1209 }
1210 }
1211}
1212
1213bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const {
1214 const auto &F = MF.getFunction();
1215 Type *ReturnType = F.getReturnType();
1216 CallingConv::ID CallConv = F.getCallingConv();
1217
1218 SmallVector<BaseArgInfo, 4> SplitArgs;
1219 getReturnInfo(CallConv, RetTy: ReturnType, Attrs: F.getAttributes(), Outs&: SplitArgs,
1220 DL: MF.getDataLayout());
1221 return canLowerReturn(MF, CallConv, Outs&: SplitArgs, IsVarArg: F.isVarArg());
1222}
1223
1224bool CallLowering::parametersInCSRMatch(
1225 const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask,
1226 const SmallVectorImpl<CCValAssign> &OutLocs,
1227 const SmallVectorImpl<ArgInfo> &OutArgs) const {
1228 for (unsigned i = 0; i < OutLocs.size(); ++i) {
1229 const auto &ArgLoc = OutLocs[i];
1230 // If it's not a register, it's fine.
1231 if (!ArgLoc.isRegLoc())
1232 continue;
1233
1234 MCRegister PhysReg = ArgLoc.getLocReg();
1235
1236 // Only look at callee-saved registers.
1237 if (MachineOperand::clobbersPhysReg(RegMask: CallerPreservedMask, PhysReg))
1238 continue;
1239
1240 LLVM_DEBUG(
1241 dbgs()
1242 << "... Call has an argument passed in a callee-saved register.\n");
1243
1244 // Check if it was copied from.
1245 const ArgInfo &OutInfo = OutArgs[i];
1246
1247 if (OutInfo.Regs.size() > 1) {
1248 LLVM_DEBUG(
1249 dbgs() << "... Cannot handle arguments in multiple registers.\n");
1250 return false;
1251 }
1252
1253 // Check if we copy the register, walking through copies from virtual
1254 // registers. Note that getDefIgnoringCopies does not ignore copies from
1255 // physical registers.
1256 MachineInstr *RegDef = getDefIgnoringCopies(Reg: OutInfo.Regs[0], MRI);
1257 if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) {
1258 LLVM_DEBUG(
1259 dbgs()
1260 << "... Parameter was not copied into a VReg, cannot tail call.\n");
1261 return false;
1262 }
1263
1264 // Got a copy. Verify that it's the same as the register we want.
1265 Register CopyRHS = RegDef->getOperand(i: 1).getReg();
1266 if (CopyRHS != PhysReg) {
1267 LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into "
1268 "VReg, cannot tail call.\n");
1269 return false;
1270 }
1271 }
1272
1273 return true;
1274}
1275
1276bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
1277 MachineFunction &MF,
1278 SmallVectorImpl<ArgInfo> &InArgs,
1279 ValueAssigner &CalleeAssigner,
1280 ValueAssigner &CallerAssigner) const {
1281 const Function &F = MF.getFunction();
1282 CallingConv::ID CalleeCC = Info.CallConv;
1283 CallingConv::ID CallerCC = F.getCallingConv();
1284
1285 if (CallerCC == CalleeCC)
1286 return true;
1287
1288 SmallVector<CCValAssign, 16> ArgLocs1;
1289 CCState CCInfo1(CalleeCC, Info.IsVarArg, MF, ArgLocs1, F.getContext());
1290 if (!determineAssignments(Assigner&: CalleeAssigner, Args&: InArgs, CCInfo&: CCInfo1))
1291 return false;
1292
1293 SmallVector<CCValAssign, 16> ArgLocs2;
1294 CCState CCInfo2(CallerCC, F.isVarArg(), MF, ArgLocs2, F.getContext());
1295 if (!determineAssignments(Assigner&: CallerAssigner, Args&: InArgs, CCInfo&: CCInfo2))
1296 return false;
1297
1298 // We need the argument locations to match up exactly. If there's more in
1299 // one than the other, then we are done.
1300 if (ArgLocs1.size() != ArgLocs2.size())
1301 return false;
1302
1303 // Make sure that each location is passed in exactly the same way.
1304 for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {
1305 const CCValAssign &Loc1 = ArgLocs1[i];
1306 const CCValAssign &Loc2 = ArgLocs2[i];
1307
1308 // We need both of them to be the same. So if one is a register and one
1309 // isn't, we're done.
1310 if (Loc1.isRegLoc() != Loc2.isRegLoc())
1311 return false;
1312
1313 if (Loc1.isRegLoc()) {
1314 // If they don't have the same register location, we're done.
1315 if (Loc1.getLocReg() != Loc2.getLocReg())
1316 return false;
1317
1318 // They matched, so we can move to the next ArgLoc.
1319 continue;
1320 }
1321
1322 // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.
1323 if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())
1324 return false;
1325 }
1326
1327 return true;
1328}
1329
1330LLT CallLowering::ValueHandler::getStackValueStoreType(
1331 const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const {
1332 const MVT ValVT = VA.getValVT();
1333 if (ValVT != MVT::iPTR) {
1334 LLT ValTy(ValVT);
1335
1336 // We lost the pointeriness going through CCValAssign, so try to restore it
1337 // based on the flags.
1338 if (Flags.isPointer()) {
1339 LLT PtrTy = LLT::pointer(AddressSpace: Flags.getPointerAddrSpace(),
1340 SizeInBits: ValTy.getScalarSizeInBits());
1341 if (ValVT.isVector() && ValVT.getVectorNumElements() != 1)
1342 return LLT::vector(EC: ValTy.getElementCount(), ScalarTy: PtrTy);
1343 return PtrTy;
1344 }
1345
1346 return ValTy;
1347 }
1348
1349 unsigned AddrSpace = Flags.getPointerAddrSpace();
1350 return LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSize(AS: AddrSpace));
1351}
1352
1353void CallLowering::ValueHandler::copyArgumentMemory(
1354 const ArgInfo &Arg, Register DstPtr, Register SrcPtr,
1355 const MachinePointerInfo &DstPtrInfo, Align DstAlign,
1356 const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize,
1357 CCValAssign &VA) const {
1358 MachineFunction &MF = MIRBuilder.getMF();
1359 MachineMemOperand *SrcMMO = MF.getMachineMemOperand(
1360 PtrInfo: SrcPtrInfo,
1361 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, Size: MemSize,
1362 BaseAlignment: SrcAlign);
1363
1364 MachineMemOperand *DstMMO = MF.getMachineMemOperand(
1365 PtrInfo: DstPtrInfo,
1366 F: MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable,
1367 Size: MemSize, BaseAlignment: DstAlign);
1368
1369 const LLT PtrTy = MRI.getType(Reg: DstPtr);
1370 const LLT SizeTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
1371
1372 auto SizeConst = MIRBuilder.buildConstant(Res: SizeTy, Val: MemSize);
1373 MIRBuilder.buildMemCpy(DstPtr, SrcPtr, Size: SizeConst, DstMMO&: *DstMMO, SrcMMO&: *SrcMMO);
1374}
1375
1376Register CallLowering::ValueHandler::extendRegister(Register ValReg,
1377 const CCValAssign &VA,
1378 unsigned MaxSizeBits) {
1379 LLT LocTy{VA.getLocVT()};
1380 LLT ValTy{VA.getValVT()};
1381
1382 if (LocTy.getSizeInBits() == ValTy.getSizeInBits())
1383 return ValReg;
1384
1385 if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) {
1386 if (MaxSizeBits <= ValTy.getSizeInBits())
1387 return ValReg;
1388 LocTy = LLT::scalar(SizeInBits: MaxSizeBits);
1389 }
1390
1391 const LLT ValRegTy = MRI.getType(Reg: ValReg);
1392 if (ValRegTy.isPointer()) {
1393 // The x32 ABI wants to zero extend 32-bit pointers to 64-bit registers, so
1394 // we have to cast to do the extension.
1395 LLT IntPtrTy = LLT::scalar(SizeInBits: ValRegTy.getSizeInBits());
1396 ValReg = MIRBuilder.buildPtrToInt(Dst: IntPtrTy, Src: ValReg).getReg(Idx: 0);
1397 }
1398
1399 switch (VA.getLocInfo()) {
1400 default:
1401 break;
1402 case CCValAssign::Full:
1403 case CCValAssign::BCvt:
1404 case CCValAssign::Indirect:
1405 // FIXME: bitconverting between vector types may or may not be a
1406 // nop in big-endian situations.
1407 return ValReg;
1408 case CCValAssign::AExt: {
1409 auto MIB = MIRBuilder.buildAnyExt(Res: LocTy, Op: ValReg);
1410 return MIB.getReg(Idx: 0);
1411 }
1412 case CCValAssign::SExt: {
1413 Register NewReg = MRI.createGenericVirtualRegister(Ty: LocTy);
1414 MIRBuilder.buildSExt(Res: NewReg, Op: ValReg);
1415 return NewReg;
1416 }
1417 case CCValAssign::ZExt: {
1418 Register NewReg = MRI.createGenericVirtualRegister(Ty: LocTy);
1419 MIRBuilder.buildZExt(Res: NewReg, Op: ValReg);
1420 return NewReg;
1421 }
1422 }
1423 llvm_unreachable("unable to extend register");
1424}
1425
1426void CallLowering::ValueAssigner::anchor() {}
1427
1428Register CallLowering::IncomingValueHandler::buildExtensionHint(
1429 const CCValAssign &VA, Register SrcReg, LLT NarrowTy) {
1430 switch (VA.getLocInfo()) {
1431 case CCValAssign::LocInfo::ZExt: {
1432 return MIRBuilder
1433 .buildAssertZExt(Res: MRI.cloneVirtualRegister(VReg: SrcReg), Op: SrcReg,
1434 Size: NarrowTy.getScalarSizeInBits())
1435 .getReg(Idx: 0);
1436 }
1437 case CCValAssign::LocInfo::SExt: {
1438 return MIRBuilder
1439 .buildAssertSExt(Res: MRI.cloneVirtualRegister(VReg: SrcReg), Op: SrcReg,
1440 Size: NarrowTy.getScalarSizeInBits())
1441 .getReg(Idx: 0);
1442 break;
1443 }
1444 default:
1445 return SrcReg;
1446 }
1447}
1448
1449/// Check if we can use a basic COPY instruction between the two types.
1450///
1451/// We're currently building on top of the infrastructure using MVT, which loses
1452/// pointer information in the CCValAssign. We accept copies from physical
1453/// registers that have been reported as integers if it's to an equivalent sized
1454/// pointer LLT.
1455static bool isCopyCompatibleType(LLT SrcTy, LLT DstTy) {
1456 if (SrcTy == DstTy)
1457 return true;
1458
1459 if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1460 return false;
1461
1462 SrcTy = SrcTy.getScalarType();
1463 DstTy = DstTy.getScalarType();
1464
1465 return (SrcTy.isPointer() && DstTy.isScalar()) ||
1466 (DstTy.isPointer() && SrcTy.isScalar());
1467}
1468
1469void CallLowering::IncomingValueHandler::assignValueToReg(
1470 Register ValVReg, Register PhysReg, const CCValAssign &VA,
1471 ISD::ArgFlagsTy Flags) {
1472 const MVT LocVT = VA.getLocVT();
1473 const LLT LocTy = getLLTForMVT(Ty: LocVT);
1474 const LLT RegTy = MRI.getType(Reg: ValVReg);
1475
1476 if (isCopyCompatibleType(SrcTy: RegTy, DstTy: LocTy)) {
1477 MIRBuilder.buildCopy(Res: ValVReg, Op: PhysReg);
1478 return;
1479 }
1480
1481 auto Copy = MIRBuilder.buildCopy(Res: LocTy, Op: PhysReg);
1482 auto Hint = buildExtensionHint(VA, SrcReg: Copy.getReg(Idx: 0), NarrowTy: RegTy);
1483 MIRBuilder.buildTrunc(Res: ValVReg, Op: Hint);
1484}
1485