1//===- llvm/lib/Target/X86/X86ISelCallLowering.cpp - Call lowering --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file implements the lowering of LLVM calls to DAG nodes.
11//
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/X86MCAsmInfo.h"
15#include "X86.h"
16#include "X86CallingConv.h"
17#include "X86FrameLowering.h"
18#include "X86ISelLowering.h"
19#include "X86InstrBuilder.h"
20#include "X86MachineFunctionInfo.h"
21#include "X86TargetMachine.h"
22#include "llvm/ADT/Statistic.h"
23#include "llvm/Analysis/ObjCARCUtil.h"
24#include "llvm/CodeGen/MachineJumpTableInfo.h"
25#include "llvm/CodeGen/MachineModuleInfo.h"
26#include "llvm/CodeGen/WinEHFuncInfo.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/Module.h"
30#include "llvm/Transforms/CFGuard.h"
31
32#define DEBUG_TYPE "x86-isel"
33
34using namespace llvm;
35
36STATISTIC(NumTailCalls, "Number of tail calls");
37
38/// Call this when the user attempts to do something unsupported, like
39/// returning a double without SSE2 enabled on x86_64. This is not fatal, unlike
40/// report_fatal_error, so calling code should attempt to recover without
41/// crashing.
42static void errorUnsupported(SelectionDAG &DAG, const SDLoc &dl,
43 const char *Msg) {
44 MachineFunction &MF = DAG.getMachineFunction();
45 DAG.getContext()->diagnose(
46 DI: DiagnosticInfoUnsupported(MF.getFunction(), Msg, dl.getDebugLoc()));
47}
48
49/// Returns true if a CC can dynamically exclude a register from the list of
50/// callee-saved-registers (TargetRegistryInfo::getCalleeSavedRegs()) based on
51/// the return registers.
52static bool shouldDisableRetRegFromCSR(CallingConv::ID CC) {
53 switch (CC) {
54 default:
55 return false;
56 case CallingConv::X86_RegCall:
57 case CallingConv::PreserveMost:
58 case CallingConv::PreserveAll:
59 return true;
60 }
61}
62
63/// Returns true if a CC can dynamically exclude a register from the list of
64/// callee-saved-registers (TargetRegistryInfo::getCalleeSavedRegs()) based on
65/// the parameters.
66static bool shouldDisableArgRegFromCSR(CallingConv::ID CC) {
67 return CC == CallingConv::X86_RegCall;
68}
69
70static std::pair<MVT, unsigned>
71handleMaskRegisterForCallingConv(unsigned NumElts, CallingConv::ID CC,
72 const X86Subtarget &Subtarget) {
73 // v2i1/v4i1/v8i1/v16i1 all pass in xmm registers unless the calling
74 // convention is one that uses k registers.
75 if (NumElts == 2)
76 return {MVT::v2i64, 1};
77 if (NumElts == 4)
78 return {MVT::v4i32, 1};
79 if (NumElts == 8 && CC != CallingConv::X86_RegCall &&
80 CC != CallingConv::Intel_OCL_BI)
81 return {MVT::v8i16, 1};
82 if (NumElts == 16 && CC != CallingConv::X86_RegCall &&
83 CC != CallingConv::Intel_OCL_BI)
84 return {MVT::v16i8, 1};
85 // v32i1 passes in ymm unless we have BWI and the calling convention is
86 // regcall.
87 if (NumElts == 32 && (!Subtarget.hasBWI() || CC != CallingConv::X86_RegCall))
88 return {MVT::v32i8, 1};
89 // Split v64i1 vectors if we don't have v64i8 available.
90 if (NumElts == 64 && Subtarget.hasBWI() && CC != CallingConv::X86_RegCall) {
91 if (Subtarget.useAVX512Regs())
92 return {MVT::v64i8, 1};
93 return {MVT::v32i8, 2};
94 }
95
96 // Break wide or odd vXi1 vectors into scalars to match avx2 behavior.
97 if (!isPowerOf2_32(Value: NumElts) || (NumElts == 64 && !Subtarget.hasBWI()) ||
98 NumElts > 64)
99 return {MVT::i8, NumElts};
100
101 return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};
102}
103
104MVT X86TargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
105 CallingConv::ID CC,
106 EVT VT) const {
107 if (VT.isVector()) {
108 if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512()) {
109 unsigned NumElts = VT.getVectorNumElements();
110
111 MVT RegisterVT;
112 unsigned NumRegisters;
113 std::tie(args&: RegisterVT, args&: NumRegisters) =
114 handleMaskRegisterForCallingConv(NumElts, CC, Subtarget);
115 if (RegisterVT != MVT::INVALID_SIMPLE_VALUE_TYPE)
116 return RegisterVT;
117 }
118
119 if (VT.getVectorElementType() == MVT::f16 && VT.getVectorNumElements() < 8)
120 return MVT::v8f16;
121 }
122
123 // We will use more GPRs for f64 and f80 on 32 bits when x87 is disabled.
124 if ((VT == MVT::f64 || VT == MVT::f80) && !Subtarget.is64Bit() &&
125 !Subtarget.hasX87())
126 return MVT::i32;
127
128 if (isTypeLegal(VT: MVT::f16)) {
129 if (VT.isVectorOf(EltVT: MVT::bf16))
130 return getRegisterTypeForCallingConv(
131 Context, CC, VT: VT.changeVectorElementType(Context, EltVT: MVT::f16));
132
133 if (VT == MVT::bf16)
134 return MVT::f16;
135 }
136
137 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
138}
139
140unsigned X86TargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
141 CallingConv::ID CC,
142 EVT VT) const {
143 if (VT.isVector()) {
144 if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512()) {
145 unsigned NumElts = VT.getVectorNumElements();
146
147 MVT RegisterVT;
148 unsigned NumRegisters;
149 std::tie(args&: RegisterVT, args&: NumRegisters) =
150 handleMaskRegisterForCallingConv(NumElts, CC, Subtarget);
151 if (RegisterVT != MVT::INVALID_SIMPLE_VALUE_TYPE)
152 return NumRegisters;
153 }
154
155 if (VT.getVectorElementType() == MVT::f16 && VT.getVectorNumElements() < 8)
156 return 1;
157 }
158
159 // We have to split f64 to 2 registers and f80 to 3 registers on 32 bits if
160 // x87 is disabled.
161 if (!Subtarget.is64Bit() && !Subtarget.hasX87()) {
162 if (VT == MVT::f64)
163 return 2;
164 if (VT == MVT::f80)
165 return 3;
166 }
167
168 if (VT.isVectorOf(EltVT: MVT::bf16) && isTypeLegal(VT: MVT::f16))
169 return getNumRegistersForCallingConv(
170 Context, CC, VT: VT.changeVectorElementType(Context, EltVT: MVT::f16));
171
172 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
173}
174
175unsigned X86TargetLowering::getVectorTypeBreakdownForCallingConv(
176 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
177 unsigned &NumIntermediates, MVT &RegisterVT) const {
178 // Break wide or odd vXi1 vectors into scalars to match avx2 behavior.
179 if (VT.isVectorOf(EltVT: MVT::i1) && Subtarget.hasAVX512() &&
180 (!isPowerOf2_32(Value: VT.getVectorNumElements()) ||
181 (VT.getVectorNumElements() == 64 && !Subtarget.hasBWI()) ||
182 VT.getVectorNumElements() > 64)) {
183 RegisterVT = MVT::i8;
184 IntermediateVT = MVT::i1;
185 NumIntermediates = VT.getVectorNumElements();
186 return NumIntermediates;
187 }
188
189 // Split v64i1 vectors if we don't have v64i8 available.
190 if (VT == MVT::v64i1 && Subtarget.hasBWI() && !Subtarget.useAVX512Regs() &&
191 CC != CallingConv::X86_RegCall) {
192 RegisterVT = MVT::v32i8;
193 IntermediateVT = MVT::v32i1;
194 NumIntermediates = 2;
195 return 2;
196 }
197
198 // Split vNbf16 vectors according to vNf16.
199 if (VT.isVectorOf(EltVT: MVT::bf16) && isTypeLegal(VT: MVT::f16))
200 VT = VT.changeVectorElementType(Context, EltVT: MVT::f16);
201
202 return TargetLowering::getVectorTypeBreakdownForCallingConv(Context, CC, VT, IntermediateVT,
203 NumIntermediates, RegisterVT);
204}
205
206EVT X86TargetLowering::getSetCCResultType(const DataLayout &DL,
207 LLVMContext& Context,
208 EVT VT) const {
209 if (!VT.isVector())
210 return MVT::i8;
211
212 if (Subtarget.hasAVX512()) {
213 // Figure out what this type will be legalized to.
214 EVT LegalVT = VT;
215 while (getTypeAction(Context, VT: LegalVT) != TypeLegal)
216 LegalVT = getTypeToTransformTo(Context, VT: LegalVT);
217
218 // If we got a 512-bit vector then we'll definitely have a vXi1 compare.
219 if (LegalVT.getSimpleVT().is512BitVector())
220 return EVT::getVectorVT(Context, VT: MVT::i1, EC: VT.getVectorElementCount());
221
222 if (LegalVT.getSimpleVT().isVector() && Subtarget.hasVLX()) {
223 // If we legalized to less than a 512-bit vector, then we will use a vXi1
224 // compare for vXi32/vXi64 for sure. If we have BWI we will also support
225 // vXi16/vXi8.
226 MVT EltVT = LegalVT.getSimpleVT().getVectorElementType();
227 if (Subtarget.hasBWI() || EltVT.getSizeInBits() >= 32)
228 return EVT::getVectorVT(Context, VT: MVT::i1, EC: VT.getVectorElementCount());
229 }
230 }
231
232 return VT.changeVectorElementTypeToInteger();
233}
234
235bool X86TargetLowering::functionArgumentNeedsConsecutiveRegisters(
236 Type *Ty, CallingConv::ID CallConv, bool isVarArg,
237 const DataLayout &DL) const {
238 // On x86-64 i128 is split into two i64s and needs to be allocated to two
239 // consecutive registers, or spilled to the stack as a whole. On x86-32 i128
240 // is split to four i32s and never actually passed in registers, but we use
241 // the consecutive register mark to match it in TableGen.
242 if (Ty->isIntegerTy(BitWidth: 128))
243 return true;
244
245 // On x86-32, fp128 acts the same as i128.
246 if (Subtarget.is32Bit() && Ty->isFP128Ty())
247 return true;
248
249 return false;
250}
251
252/// Helper for getByValTypeAlignment to determine
253/// the desired ByVal argument alignment.
254static void getMaxByValAlign(Type *Ty, Align &MaxAlign) {
255 if (MaxAlign == 16)
256 return;
257 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty)) {
258 if (VTy->getPrimitiveSizeInBits().getFixedValue() == 128)
259 MaxAlign = Align(16);
260 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Val: Ty)) {
261 Align EltAlign;
262 getMaxByValAlign(Ty: ATy->getElementType(), MaxAlign&: EltAlign);
263 if (EltAlign > MaxAlign)
264 MaxAlign = EltAlign;
265 } else if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
266 for (auto *EltTy : STy->elements()) {
267 Align EltAlign;
268 getMaxByValAlign(Ty: EltTy, MaxAlign&: EltAlign);
269 if (EltAlign > MaxAlign)
270 MaxAlign = EltAlign;
271 if (MaxAlign == 16)
272 break;
273 }
274 }
275}
276
277/// Return the desired alignment for ByVal aggregate
278/// function arguments in the caller parameter area. For X86, aggregates
279/// that contain SSE vectors are placed at 16-byte boundaries while the rest
280/// are at 4-byte boundaries.
281Align X86TargetLowering::getByValTypeAlignment(Type *Ty,
282 const DataLayout &DL) const {
283 if (Subtarget.is64Bit())
284 return std::max(a: DL.getABITypeAlign(Ty), b: Align::Constant<8>());
285
286 Align Alignment(4);
287 if (Subtarget.hasSSE1())
288 getMaxByValAlign(Ty, MaxAlign&: Alignment);
289 return Alignment;
290}
291
292/// It returns EVT::Other if the type should be determined using generic
293/// target-independent logic.
294/// For vector ops we check that the overall size isn't larger than our
295/// preferred vector width.
296EVT X86TargetLowering::getOptimalMemOpType(
297 LLVMContext &Context, const MemOp &Op,
298 const AttributeList &FuncAttributes) const {
299 if (!FuncAttributes.hasFnAttr(Kind: Attribute::NoImplicitFloat)) {
300 if (Op.size() >= 16 &&
301 (!Subtarget.isUnalignedMem16Slow() || Op.isAligned(AlignCheck: Align(16)))) {
302 // FIXME: Check if unaligned 64-byte accesses are slow.
303 if (Op.size() >= 64 && Subtarget.hasAVX512() &&
304 (Subtarget.getPreferVectorWidth() >= 512)) {
305 return Subtarget.hasBWI() ? MVT::v64i8 : MVT::v16i32;
306 }
307 // FIXME: Check if unaligned 32-byte accesses are slow.
308 if (Op.size() >= 32 && Subtarget.hasAVX() &&
309 Subtarget.useLight256BitInstructions()) {
310 // Although this isn't a well-supported type for AVX1, we'll let
311 // legalization and shuffle lowering produce the optimal codegen. If we
312 // choose an optimal type with a vector element larger than a byte,
313 // getMemsetStores() may create an intermediate splat (using an integer
314 // multiply) before we splat as a vector.
315 return MVT::v32i8;
316 }
317 if (Subtarget.hasSSE2() && (Subtarget.getPreferVectorWidth() >= 128))
318 return MVT::v16i8;
319 // TODO: Can SSE1 handle a byte vector?
320 // If we have SSE1 registers we should be able to use them.
321 if (Subtarget.hasSSE1() && (Subtarget.is64Bit() || Subtarget.hasX87()) &&
322 (Subtarget.getPreferVectorWidth() >= 128))
323 return MVT::v4f32;
324 } else if (((Op.isMemcpyOrMemmove() && !Op.isMemcpyStrSrc()) ||
325 Op.isZeroMemset()) &&
326 Op.size() >= 8 && !Subtarget.is64Bit() && Subtarget.hasSSE2()) {
327 // Do not use f64 to lower memcpy if source is string constant. It's
328 // better to use i32 to avoid the loads.
329 // Also, do not use f64 to lower memset unless this is a memset of zeros.
330 // The gymnastics of splatting a byte value into an XMM register and then
331 // only using 8-byte stores (because this is a CPU with slow unaligned
332 // 16-byte accesses) makes that a loser.
333 return MVT::f64;
334 }
335 }
336 // This is a compromise. If we reach here, unaligned accesses may be slow on
337 // this target. However, creating smaller, aligned accesses could be even
338 // slower and would certainly be a lot more code.
339 if (Subtarget.is64Bit() && Op.size() >= 8)
340 return MVT::i64;
341 return MVT::i32;
342}
343
344bool X86TargetLowering::isSafeMemOpType(MVT VT) const {
345 if (VT == MVT::f32)
346 return Subtarget.hasSSE1();
347 if (VT == MVT::f64)
348 return Subtarget.hasSSE2();
349 return true;
350}
351
352static bool isBitAligned(Align Alignment, uint64_t SizeInBits) {
353 return (8 * Alignment.value()) % SizeInBits == 0;
354}
355
356bool X86TargetLowering::isMemoryAccessFast(EVT VT, Align Alignment) const {
357 if (isBitAligned(Alignment, SizeInBits: VT.getSizeInBits()))
358 return true;
359 switch (VT.getSizeInBits()) {
360 default:
361 // 8-byte and under are always assumed to be fast.
362 return true;
363 case 128:
364 return !Subtarget.isUnalignedMem16Slow();
365 case 256:
366 return !Subtarget.isUnalignedMem32Slow();
367 // TODO: What about AVX-512 (512-bit) accesses?
368 }
369}
370
371bool X86TargetLowering::allowsMisalignedMemoryAccesses(
372 EVT VT, unsigned, Align Alignment, MachineMemOperand::Flags Flags,
373 unsigned *Fast) const {
374 if (Fast)
375 *Fast = isMemoryAccessFast(VT, Alignment);
376 // NonTemporal vector memory ops must be aligned.
377 if (!!(Flags & MachineMemOperand::MONonTemporal) && VT.isVector()) {
378 // NT loads can only be vector aligned, so if its less aligned than the
379 // minimum vector size (which we can split the vector down to), we might as
380 // well use a regular unaligned vector load.
381 // We don't have any NT loads pre-SSE41.
382 if (!!(Flags & MachineMemOperand::MOLoad))
383 return (Alignment < 16 || !Subtarget.hasSSE41());
384 return false;
385 }
386 // Misaligned accesses of any size are always allowed.
387 return true;
388}
389
390bool X86TargetLowering::allowsMemoryAccess(LLVMContext &Context,
391 const DataLayout &DL, EVT VT,
392 unsigned AddrSpace, Align Alignment,
393 MachineMemOperand::Flags Flags,
394 unsigned *Fast) const {
395 if (Fast)
396 *Fast = isMemoryAccessFast(VT, Alignment);
397 if (!!(Flags & MachineMemOperand::MONonTemporal) && VT.isVector()) {
398 if (allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Flags,
399 /*Fast=*/nullptr))
400 return true;
401 // NonTemporal vector memory ops are special, and must be aligned.
402 if (!isBitAligned(Alignment, SizeInBits: VT.getSizeInBits()))
403 return false;
404 switch (VT.getSizeInBits()) {
405 case 128:
406 if (!!(Flags & MachineMemOperand::MOLoad) && Subtarget.hasSSE41())
407 return true;
408 if (!!(Flags & MachineMemOperand::MOStore) && Subtarget.hasSSE2())
409 return true;
410 return false;
411 case 256:
412 if (!!(Flags & MachineMemOperand::MOLoad) && Subtarget.hasAVX2())
413 return true;
414 if (!!(Flags & MachineMemOperand::MOStore) && Subtarget.hasAVX())
415 return true;
416 return false;
417 case 512:
418 if (Subtarget.hasAVX512())
419 return true;
420 return false;
421 default:
422 return false; // Don't have NonTemporal vector memory ops of this size.
423 }
424 }
425 return true;
426}
427
428/// Return the entry encoding for a jump table in the
429/// current function. The returned value is a member of the
430/// MachineJumpTableInfo::JTEntryKind enum.
431unsigned X86TargetLowering::getJumpTableEncoding() const {
432 // In GOT pic mode, each entry in the jump table is emitted as a @GOTOFF
433 // symbol.
434 if (isPositionIndependent() && Subtarget.isPICStyleGOT())
435 return MachineJumpTableInfo::EK_Custom32;
436 if (isPositionIndependent() &&
437 getTargetMachine().getCodeModel() == CodeModel::Large &&
438 !Subtarget.isTargetCOFF())
439 return MachineJumpTableInfo::EK_LabelDifference64;
440
441 // Otherwise, use the normal jump table encoding heuristics.
442 return TargetLowering::getJumpTableEncoding();
443}
444
445bool X86TargetLowering::useSoftFloat() const {
446 return Subtarget.useSoftFloat();
447}
448
449void X86TargetLowering::markLibCallAttributes(MachineFunction *MF, unsigned CC,
450 ArgListTy &Args) const {
451
452 // Only relabel X86-32 for C / Stdcall CCs.
453 if (Subtarget.is64Bit())
454 return;
455 if (CC != CallingConv::C && CC != CallingConv::X86_StdCall)
456 return;
457 unsigned ParamRegs = 0;
458 if (auto *M = MF->getFunction().getParent())
459 ParamRegs = M->getNumberRegisterParameters();
460
461 // Mark the first N int arguments as having reg
462 for (auto &Arg : Args) {
463 Type *T = Arg.Ty;
464 if (T->isIntOrPtrTy())
465 if (MF->getDataLayout().getTypeAllocSize(Ty: T) <= 8) {
466 unsigned numRegs = 1;
467 if (MF->getDataLayout().getTypeAllocSize(Ty: T) > 4)
468 numRegs = 2;
469 if (ParamRegs < numRegs)
470 return;
471 ParamRegs -= numRegs;
472 Arg.IsInReg = true;
473 }
474 }
475}
476
477const MCExpr *
478X86TargetLowering::LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
479 const MachineBasicBlock *MBB,
480 unsigned uid,MCContext &Ctx) const{
481 assert(isPositionIndependent() && Subtarget.isPICStyleGOT());
482 // In 32-bit ELF systems, our jump table entries are formed with @GOTOFF
483 // entries.
484 return MCSymbolRefExpr::create(Symbol: MBB->getSymbol(), specifier: X86::S_GOTOFF, Ctx);
485}
486
487/// Returns relocation base for the given PIC jumptable.
488SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table,
489 SelectionDAG &DAG) const {
490 if (!Subtarget.is64Bit())
491 // This doesn't have SDLoc associated with it, but is not really the
492 // same as a Register.
493 return DAG.getNode(Opcode: X86ISD::GlobalBaseReg, DL: SDLoc(),
494 VT: getPointerTy(DL: DAG.getDataLayout()));
495 return Table;
496}
497
498/// This returns the relocation base for the given PIC jumptable,
499/// the same as getPICJumpTableRelocBase, but as an MCExpr.
500const MCExpr *X86TargetLowering::
501getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI,
502 MCContext &Ctx) const {
503 // X86-64 uses RIP relative addressing based on the jump table label.
504 if (Subtarget.isPICStyleRIPRel() ||
505 (Subtarget.is64Bit() &&
506 getTargetMachine().getCodeModel() == CodeModel::Large))
507 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
508
509 // Otherwise, the reference is relative to the PIC base.
510 return MCSymbolRefExpr::create(Symbol: MF->getPICBaseSymbol(), Ctx);
511}
512
513std::pair<const TargetRegisterClass *, uint8_t>
514X86TargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
515 MVT VT) const {
516 const TargetRegisterClass *RRC = nullptr;
517 uint8_t Cost = 1;
518 switch (VT.SimpleTy) {
519 default:
520 return TargetLowering::findRepresentativeClass(TRI, VT);
521 case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64:
522 RRC = Subtarget.is64Bit() ? &X86::GR64RegClass : &X86::GR32RegClass;
523 break;
524 case MVT::x86mmx:
525 RRC = &X86::VR64RegClass;
526 break;
527 case MVT::f32: case MVT::f64:
528 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:
529 case MVT::v4f32: case MVT::v2f64:
530 case MVT::v32i8: case MVT::v16i16: case MVT::v8i32: case MVT::v4i64:
531 case MVT::v8f32: case MVT::v4f64:
532 case MVT::v64i8: case MVT::v32i16: case MVT::v16i32: case MVT::v8i64:
533 case MVT::v16f32: case MVT::v8f64:
534 RRC = &X86::VR128XRegClass;
535 break;
536 }
537 return std::make_pair(x&: RRC, y&: Cost);
538}
539
540unsigned X86TargetLowering::getAddressSpace() const {
541 if (Subtarget.is64Bit())
542 return (getTargetMachine().getCodeModel() == CodeModel::Kernel) ? X86AS::GS
543 : X86AS::FS;
544 return X86AS::GS;
545}
546
547static bool hasStackGuardSlotTLS(const Triple &TargetTriple) {
548 return TargetTriple.isOSGlibc() || TargetTriple.isMusl() ||
549 TargetTriple.isOSFuchsia() || TargetTriple.isAndroid();
550}
551
552static Constant* SegmentOffset(IRBuilderBase &IRB,
553 int Offset, unsigned AddressSpace) {
554 return ConstantExpr::getIntToPtr(
555 C: ConstantInt::getSigned(Ty: Type::getInt32Ty(C&: IRB.getContext()), V: Offset),
556 Ty: IRB.getPtrTy(AddrSpace: AddressSpace));
557}
558
559Value *
560X86TargetLowering::getIRStackGuard(IRBuilderBase &IRB,
561 const LibcallLoweringInfo &Libcalls) const {
562 // glibc, bionic, and Fuchsia have a special slot for the stack guard in
563 // tcbhead_t; use it instead of the usual global variable (see
564 // sysdeps/{i386,x86_64}/nptl/tls.h)
565 if (hasStackGuardSlotTLS(TargetTriple: Subtarget.getTargetTriple())) {
566 unsigned AddressSpace = getAddressSpace();
567
568 // <zircon/tls.h> defines ZX_TLS_STACK_GUARD_OFFSET with this value.
569 if (Subtarget.isTargetFuchsia())
570 return SegmentOffset(IRB, Offset: 0x10, AddressSpace);
571
572 Module *M = IRB.getModule();
573 // Specially, some users may customize the base reg and offset.
574 int Offset = M->getStackProtectorGuardOffset();
575 // If we don't set -stack-protector-guard-offset value:
576 // %fs:0x28, unless we're using a Kernel code model, in which case
577 // it's %gs:0x28. gs:0x14 on i386.
578 if (Offset == INT_MAX)
579 Offset = (Subtarget.is64Bit()) ? 0x28 : 0x14;
580
581 StringRef GuardReg = M->getStackProtectorGuardReg();
582 if (GuardReg == "fs")
583 AddressSpace = X86AS::FS;
584 else if (GuardReg == "gs")
585 AddressSpace = X86AS::GS;
586
587 // Use symbol guard if user specify.
588 StringRef GuardSymb = M->getStackProtectorGuardSymbol();
589 if (!GuardSymb.empty()) {
590 GlobalVariable *GV = M->getGlobalVariable(Name: GuardSymb);
591 if (!GV) {
592 Type *Ty = Subtarget.is64Bit() ? Type::getInt64Ty(C&: M->getContext())
593 : Type::getInt32Ty(C&: M->getContext());
594 GV = new GlobalVariable(*M, Ty, false, GlobalValue::ExternalLinkage,
595 nullptr, GuardSymb, nullptr,
596 GlobalValue::NotThreadLocal, AddressSpace);
597 if (!Subtarget.isTargetDarwin())
598 GV->setDSOLocal(M->getDirectAccessExternalData());
599 }
600 return GV;
601 }
602
603 return SegmentOffset(IRB, Offset, AddressSpace);
604 }
605 return TargetLowering::getIRStackGuard(IRB, Libcalls);
606}
607
608void X86TargetLowering::insertSSPDeclarations(
609 Module &M, const LibcallLoweringInfo &Libcalls) const {
610 // MSVC CRT provides functionalities for stack protection.
611 RTLIB::LibcallImpl SecurityCheckCookieLibcall =
612 Libcalls.getLibcallImpl(Call: RTLIB::SECURITY_CHECK_COOKIE);
613
614 RTLIB::LibcallImpl SecurityCookieVar =
615 Libcalls.getLibcallImpl(Call: RTLIB::STACK_CHECK_GUARD);
616 if (SecurityCheckCookieLibcall != RTLIB::Unsupported &&
617 SecurityCookieVar != RTLIB::Unsupported) {
618 // MSVC CRT provides functionalities for stack protection.
619 // MSVC CRT has a global variable holding security cookie.
620 const RTLIB::RuntimeLibcallsInfo &RTLCI = Libcalls.getRuntimeLibcallsInfo();
621 RTLCI.getOrInsertImplGlobal(M, LibcallImpl: SecurityCookieVar);
622
623 // MSVC CRT has a function to validate security cookie.
624 FunctionCallee SecurityCheckCookie =
625 RTLCI.getOrInsertLibcallDecl(M, LibcallImpl: SecurityCheckCookieLibcall);
626
627 if (Function *F = dyn_cast<Function>(Val: SecurityCheckCookie.getCallee())) {
628 F->setCallingConv(CallingConv::X86_FastCall);
629 F->addParamAttr(ArgNo: 0, Kind: Attribute::AttrKind::InReg);
630 }
631 return;
632 }
633
634 StringRef GuardMode = M.getStackProtectorGuard();
635
636 // glibc, bionic, and Fuchsia have a special slot for the stack guard.
637 if ((GuardMode == "tls" || GuardMode.empty()) &&
638 hasStackGuardSlotTLS(TargetTriple: Subtarget.getTargetTriple()))
639 return;
640 TargetLowering::insertSSPDeclarations(M, Libcalls);
641}
642
643Value *X86TargetLowering::getSafeStackPointerLocation(
644 IRBuilderBase &IRB, const LibcallLoweringInfo &Libcalls) const {
645 // Android provides a fixed TLS slot for the SafeStack pointer. See the
646 // definition of TLS_SLOT_SAFESTACK in
647 // https://android.googlesource.com/platform/bionic/+/master/libc/private/bionic_tls.h
648 if (Subtarget.isTargetAndroid()) {
649 // %fs:0x48, unless we're using a Kernel code model, in which case it's %gs:
650 // %gs:0x24 on i386
651 int Offset = (Subtarget.is64Bit()) ? 0x48 : 0x24;
652 return SegmentOffset(IRB, Offset, AddressSpace: getAddressSpace());
653 }
654
655 // Fuchsia is similar.
656 if (Subtarget.isTargetFuchsia()) {
657 // <zircon/tls.h> defines ZX_TLS_UNSAFE_SP_OFFSET with this value.
658 return SegmentOffset(IRB, Offset: 0x18, AddressSpace: getAddressSpace());
659 }
660
661 return TargetLowering::getSafeStackPointerLocation(IRB, Libcalls);
662}
663
664//===----------------------------------------------------------------------===//
665// Return Value Calling Convention Implementation
666//===----------------------------------------------------------------------===//
667
668bool X86TargetLowering::CanLowerReturn(
669 CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg,
670 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
671 const Type *RetTy) const {
672 // Mingw64 GCC returns f128 via sret, and LLVM matches it for compatibility.
673 // This logic exists for libcalls, a frontend should explicitly use sret
674 // rather than rely on the sret demotion here.
675 //
676 // Using sret is a reasonable implementation of the Windows x64 calling
677 // convention:
678 //
679 // https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention?view=msvc-170#return-values
680 //
681 // > Otherwise, the caller must allocate memory for the return value and pass
682 // > a pointer to it as the first argument.
683 //
684 // Although it is not the only reasonable interpretation:
685 //
686 // > Nonscalar types including floats, doubles, and vector types such as
687 // > __m128, __m128i, __m128d are returned in XMM0.
688 //
689 // For now, we prefer compatibility with GCC. If official guidelines are ever
690 // published, this can be revisited.
691 //
692 // Return false, which will perform sret demotion.
693 auto IsWin64F128StackCC = [this](CallingConv::ID CC) -> bool {
694 switch (CC) {
695 case CallingConv::Win64:
696 return true;
697 case CallingConv::C:
698 return Subtarget.isOSWindowsOrUEFI();
699 default:
700 return false;
701 }
702 };
703
704 if (IsWin64F128StackCC(CallConv) &&
705 llvm::any_of(
706 Range: Outs, P: [](const ISD::OutputArg &Out) { return Out.VT == MVT::f128; }))
707 return false;
708
709 SmallVector<CCValAssign, 16> RVLocs;
710 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
711 return CCInfo.CheckReturn(Outs, Fn: RetCC_X86);
712}
713
714const MCPhysReg *X86TargetLowering::getScratchRegisters(CallingConv::ID) const {
715 static const MCPhysReg ScratchRegs[] = { X86::R11, 0 };
716 return ScratchRegs;
717}
718
719ArrayRef<MCPhysReg> X86TargetLowering::getRoundingControlRegisters() const {
720 static const MCPhysReg RCRegs[] = {X86::FPCW, X86::MXCSR};
721 return RCRegs;
722}
723
724/// Lowers masks values (v*i1) to the local register values
725/// \returns DAG node after lowering to register type
726static SDValue lowerMasksToReg(const SDValue &ValArg, const EVT &ValLoc,
727 const SDLoc &DL, SelectionDAG &DAG) {
728 EVT ValVT = ValArg.getValueType();
729
730 if (ValVT == MVT::v1i1)
731 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ValLoc, N1: ValArg,
732 N2: DAG.getIntPtrConstant(Val: 0, DL));
733
734 if ((ValVT == MVT::v8i1 && (ValLoc == MVT::i8 || ValLoc == MVT::i32)) ||
735 (ValVT == MVT::v16i1 && (ValLoc == MVT::i16 || ValLoc == MVT::i32))) {
736 // Two stage lowering might be required
737 // bitcast: v8i1 -> i8 / v16i1 -> i16
738 // anyextend: i8 -> i32 / i16 -> i32
739 EVT TempValLoc = ValVT == MVT::v8i1 ? MVT::i8 : MVT::i16;
740 SDValue ValToCopy = DAG.getBitcast(VT: TempValLoc, V: ValArg);
741 if (ValLoc == MVT::i32)
742 ValToCopy = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ValLoc, Operand: ValToCopy);
743 return ValToCopy;
744 }
745
746 if ((ValVT == MVT::v32i1 && ValLoc == MVT::i32) ||
747 (ValVT == MVT::v64i1 && ValLoc == MVT::i64)) {
748 // One stage lowering is required
749 // bitcast: v32i1 -> i32 / v64i1 -> i64
750 return DAG.getBitcast(VT: ValLoc, V: ValArg);
751 }
752
753 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ValLoc, Operand: ValArg);
754}
755
756/// Breaks v64i1 value into two registers and adds the new node to the DAG
757static void Passv64i1ArgInRegs(
758 const SDLoc &DL, SelectionDAG &DAG, SDValue &Arg,
759 SmallVectorImpl<std::pair<Register, SDValue>> &RegsToPass, CCValAssign &VA,
760 CCValAssign &NextVA, const X86Subtarget &Subtarget) {
761 assert(Subtarget.hasBWI() && "Expected AVX512BW target!");
762 assert(Subtarget.is32Bit() && "Expecting 32 bit target");
763 assert(Arg.getValueType() == MVT::i64 && "Expecting 64 bit value");
764 assert(VA.isRegLoc() && NextVA.isRegLoc() &&
765 "The value should reside in two registers");
766
767 // Before splitting the value we cast it to i64
768 Arg = DAG.getBitcast(VT: MVT::i64, V: Arg);
769
770 // Splitting the value into two i32 types
771 SDValue Lo, Hi;
772 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Arg, DL, LoVT: MVT::i32, HiVT: MVT::i32);
773
774 // Attach the two i32 types into corresponding registers
775 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Lo));
776 RegsToPass.push_back(Elt: std::make_pair(x: NextVA.getLocReg(), y&: Hi));
777}
778
779SDValue
780X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
781 bool isVarArg,
782 const SmallVectorImpl<ISD::OutputArg> &Outs,
783 const SmallVectorImpl<SDValue> &OutVals,
784 const SDLoc &dl, SelectionDAG &DAG) const {
785 MachineFunction &MF = DAG.getMachineFunction();
786 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
787
788 // In some cases we need to disable registers from the default CSR list.
789 // For example, when they are used as return registers (preserve_* and X86's
790 // regcall) or for argument passing (X86's regcall).
791 bool ShouldDisableCalleeSavedRegister =
792 shouldDisableRetRegFromCSR(CC: CallConv) ||
793 MF.getFunction().hasFnAttribute(Kind: "no_caller_saved_registers");
794
795 if (CallConv == CallingConv::X86_INTR && !Outs.empty())
796 report_fatal_error(reason: "X86 interrupts may not return any value");
797
798 SmallVector<CCValAssign, 16> RVLocs;
799 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, *DAG.getContext());
800 CCInfo.AnalyzeReturn(Outs, Fn: RetCC_X86);
801
802 SmallVector<std::pair<Register, SDValue>, 4> RetVals;
803 for (unsigned I = 0, OutsIndex = 0, E = RVLocs.size(); I != E;
804 ++I, ++OutsIndex) {
805 CCValAssign &VA = RVLocs[I];
806 assert(VA.isRegLoc() && "Can only return in registers!");
807
808 // Add the register to the CalleeSaveDisableRegs list.
809 if (ShouldDisableCalleeSavedRegister)
810 MF.getRegInfo().disableCalleeSavedRegister(Reg: VA.getLocReg());
811
812 SDValue ValToCopy = OutVals[OutsIndex];
813 EVT ValVT = ValToCopy.getValueType();
814
815 // Promote values to the appropriate types.
816 if (VA.getLocInfo() == CCValAssign::SExt)
817 ValToCopy = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: ValToCopy);
818 else if (VA.getLocInfo() == CCValAssign::ZExt)
819 ValToCopy = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: ValToCopy);
820 else if (VA.getLocInfo() == CCValAssign::AExt) {
821 if (ValVT.isVectorOf(EltVT: MVT::i1))
822 ValToCopy = lowerMasksToReg(ValArg: ValToCopy, ValLoc: VA.getLocVT(), DL: dl, DAG);
823 else
824 ValToCopy = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: ValToCopy);
825 }
826 else if (VA.getLocInfo() == CCValAssign::BCvt)
827 ValToCopy = DAG.getBitcast(VT: VA.getLocVT(), V: ValToCopy);
828
829 assert(VA.getLocInfo() != CCValAssign::FPExt &&
830 "Unexpected FP-extend for return value.");
831
832 // Report an error if we have attempted to return a value via an XMM
833 // register and SSE was disabled.
834 if (!Subtarget.hasSSE1() && X86::FR32XRegClass.contains(Reg: VA.getLocReg())) {
835 errorUnsupported(DAG, dl, Msg: "SSE register return with SSE disabled");
836 VA.convertToReg(Reg: X86::FP0); // Set reg to FP0, avoid hitting asserts.
837 } else if (!Subtarget.hasSSE2() &&
838 X86::FR64XRegClass.contains(Reg: VA.getLocReg()) &&
839 ValVT == MVT::f64) {
840 // When returning a double via an XMM register, report an error if SSE2 is
841 // not enabled.
842 errorUnsupported(DAG, dl, Msg: "SSE2 register return with SSE2 disabled");
843 VA.convertToReg(Reg: X86::FP0); // Set reg to FP0, avoid hitting asserts.
844 }
845
846 // Returns in ST0/ST1 are handled specially: these are pushed as operands to
847 // the RET instruction and handled by the FP Stackifier.
848 if (VA.getLocReg() == X86::FP0 ||
849 VA.getLocReg() == X86::FP1) {
850 // If this is a copy from an xmm register to ST(0), use an FPExtend to
851 // change the value to the FP stack register class.
852 if (isScalarFPTypeInSSEReg(VT: VA.getValVT()))
853 ValToCopy = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f80, Operand: ValToCopy);
854 RetVals.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ValToCopy));
855 // Don't emit a copytoreg.
856 continue;
857 }
858
859 // 64-bit vector (MMX) values are returned in XMM0 / XMM1 except for v1i64
860 // which is returned in RAX / RDX.
861 if (Subtarget.is64Bit()) {
862 if (ValVT == MVT::x86mmx) {
863 if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) {
864 ValToCopy = DAG.getBitcast(VT: MVT::i64, V: ValToCopy);
865 ValToCopy = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: MVT::v2i64,
866 Operand: ValToCopy);
867 // If we don't have SSE2 available, convert to v4f32 so the generated
868 // register is legal.
869 if (!Subtarget.hasSSE2())
870 ValToCopy = DAG.getBitcast(VT: MVT::v4f32, V: ValToCopy);
871 }
872 }
873 }
874
875 if (VA.needsCustom()) {
876 assert(VA.getValVT() == MVT::v64i1 &&
877 "Currently the only custom case is when we split v64i1 to 2 regs");
878
879 Passv64i1ArgInRegs(DL: dl, DAG, Arg&: ValToCopy, RegsToPass&: RetVals, VA, NextVA&: RVLocs[++I],
880 Subtarget);
881
882 // Add the second register to the CalleeSaveDisableRegs list.
883 if (ShouldDisableCalleeSavedRegister)
884 MF.getRegInfo().disableCalleeSavedRegister(Reg: RVLocs[I].getLocReg());
885 } else {
886 RetVals.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ValToCopy));
887 }
888 }
889
890 SDValue Glue;
891 SmallVector<SDValue, 6> RetOps;
892 RetOps.push_back(Elt: Chain); // Operand #0 = Chain (updated below)
893 // Operand #1 = Bytes To Pop
894 RetOps.push_back(Elt: DAG.getTargetConstant(Val: FuncInfo->getBytesToPopOnReturn(), DL: dl,
895 VT: MVT::i32));
896
897 // Copy the result values into the output registers.
898 for (auto &RetVal : RetVals) {
899 if (RetVal.first == X86::FP0 || RetVal.first == X86::FP1) {
900 RetOps.push_back(Elt: RetVal.second);
901 continue; // Don't emit a copytoreg.
902 }
903
904 Chain = DAG.getCopyToReg(Chain, dl, Reg: RetVal.first, N: RetVal.second, Glue);
905 Glue = Chain.getValue(R: 1);
906 RetOps.push_back(
907 Elt: DAG.getRegister(Reg: RetVal.first, VT: RetVal.second.getValueType()));
908 }
909
910 // Swift calling convention does not require we copy the sret argument
911 // into %rax/%eax for the return, and SRetReturnReg is not set for Swift.
912
913 // All x86 ABIs require that for returning structs by value we copy
914 // the sret argument into %rax/%eax (depending on ABI) for the return.
915 // We saved the argument into a virtual register in the entry block,
916 // so now we copy the value out and into %rax/%eax.
917 //
918 // Checking Function.hasStructRetAttr() here is insufficient because the IR
919 // may not have an explicit sret argument. If FuncInfo.CanLowerReturn is
920 // false, then an sret argument may be implicitly inserted in the SelDAG. In
921 // either case FuncInfo->setSRetReturnReg() will have been called.
922 if (Register SRetReg = FuncInfo->getSRetReturnReg()) {
923 // When we have both sret and another return value, we should use the
924 // original Chain stored in RetOps[0], instead of the current Chain updated
925 // in the above loop. If we only have sret, RetOps[0] equals to Chain.
926
927 // For the case of sret and another return value, we have
928 // Chain_0 at the function entry
929 // Chain_1 = getCopyToReg(Chain_0) in the above loop
930 // If we use Chain_1 in getCopyFromReg, we will have
931 // Val = getCopyFromReg(Chain_1)
932 // Chain_2 = getCopyToReg(Chain_1, Val) from below
933
934 // getCopyToReg(Chain_0) will be glued together with
935 // getCopyToReg(Chain_1, Val) into Unit A, getCopyFromReg(Chain_1) will be
936 // in Unit B, and we will have cyclic dependency between Unit A and Unit B:
937 // Data dependency from Unit B to Unit A due to usage of Val in
938 // getCopyToReg(Chain_1, Val)
939 // Chain dependency from Unit A to Unit B
940
941 // So here, we use RetOps[0] (i.e Chain_0) for getCopyFromReg.
942 SDValue Val = DAG.getCopyFromReg(Chain: RetOps[0], dl, Reg: SRetReg,
943 VT: getPointerTy(DL: MF.getDataLayout()));
944
945 Register RetValReg
946 = (Subtarget.is64Bit() && !Subtarget.isTarget64BitILP32()) ?
947 X86::RAX : X86::EAX;
948 Chain = DAG.getCopyToReg(Chain, dl, Reg: RetValReg, N: Val, Glue);
949 Glue = Chain.getValue(R: 1);
950
951 // RAX/EAX now acts like a return value.
952 RetOps.push_back(
953 Elt: DAG.getRegister(Reg: RetValReg, VT: getPointerTy(DL: DAG.getDataLayout())));
954
955 // Add the returned register to the CalleeSaveDisableRegs list. Don't do
956 // this however for preserve_most/preserve_all to minimize the number of
957 // callee-saved registers for these CCs.
958 if (ShouldDisableCalleeSavedRegister &&
959 CallConv != CallingConv::PreserveAll &&
960 CallConv != CallingConv::PreserveMost)
961 MF.getRegInfo().disableCalleeSavedRegister(Reg: RetValReg);
962 }
963
964 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
965 const MCPhysReg *I =
966 TRI->getCalleeSavedRegsViaCopy(MF: &DAG.getMachineFunction());
967 if (I) {
968 for (; *I; ++I) {
969 if (X86::GR64RegClass.contains(Reg: *I))
970 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::i64));
971 else
972 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
973 }
974 }
975
976 RetOps[0] = Chain; // Update chain.
977
978 // Add the glue if we have it.
979 if (Glue.getNode())
980 RetOps.push_back(Elt: Glue);
981
982 unsigned RetOpcode = X86ISD::RET_GLUE;
983 if (CallConv == CallingConv::X86_INTR)
984 RetOpcode = X86ISD::IRET;
985 return DAG.getNode(Opcode: RetOpcode, DL: dl, VT: MVT::Other, Ops: RetOps);
986}
987
988bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
989 if (N->getNumValues() != 1 || !N->hasNUsesOfValue(NUses: 1, Value: 0))
990 return false;
991
992 SDValue TCChain = Chain;
993 SDNode *Copy = *N->user_begin();
994 if (Copy->getOpcode() == ISD::CopyToReg) {
995 // If the copy has a glue operand, we conservatively assume it isn't safe to
996 // perform a tail call.
997 if (Copy->getOperand(Num: Copy->getNumOperands()-1).getValueType() == MVT::Glue)
998 return false;
999 TCChain = Copy->getOperand(Num: 0);
1000 } else if (Copy->getOpcode() != ISD::FP_EXTEND)
1001 return false;
1002
1003 bool HasRet = false;
1004 for (const SDNode *U : Copy->users()) {
1005 if (U->getOpcode() != X86ISD::RET_GLUE)
1006 return false;
1007 // If we are returning more than one value, we can definitely
1008 // not make a tail call see PR19530
1009 if (U->getNumOperands() > 4)
1010 return false;
1011 if (U->getNumOperands() == 4 &&
1012 U->getOperand(Num: U->getNumOperands() - 1).getValueType() != MVT::Glue)
1013 return false;
1014 HasRet = true;
1015 }
1016
1017 if (!HasRet)
1018 return false;
1019
1020 Chain = TCChain;
1021 return true;
1022}
1023
1024EVT X86TargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT,
1025 ISD::NodeType ExtendKind) const {
1026 MVT ReturnMVT = MVT::i32;
1027
1028 bool Darwin = Subtarget.getTargetTriple().isOSDarwin();
1029 if (VT == MVT::i1 || (!Darwin && (VT == MVT::i8 || VT == MVT::i16))) {
1030 // The ABI does not require i1, i8 or i16 to be extended.
1031 //
1032 // On Darwin, there is code in the wild relying on Clang's old behaviour of
1033 // always extending i8/i16 return values, so keep doing that for now.
1034 // (PR26665).
1035 ReturnMVT = MVT::i8;
1036 }
1037
1038 EVT MinVT = getRegisterType(Context, VT: ReturnMVT);
1039 return VT.bitsLT(VT: MinVT) ? MinVT : VT;
1040}
1041
1042/// Reads two 32 bit registers and creates a 64 bit mask value.
1043/// \param VA The current 32 bit value that need to be assigned.
1044/// \param NextVA The next 32 bit value that need to be assigned.
1045/// \param Root The parent DAG node.
1046/// \param [in,out] InGlue Represents SDvalue in the parent DAG node for
1047/// glue purposes. In the case the DAG is already using
1048/// physical register instead of virtual, we should glue
1049/// our new SDValue to InGlue SDvalue.
1050/// \return a new SDvalue of size 64bit.
1051static SDValue getv64i1Argument(CCValAssign &VA, CCValAssign &NextVA,
1052 SDValue &Root, SelectionDAG &DAG,
1053 const SDLoc &DL, const X86Subtarget &Subtarget,
1054 SDValue *InGlue = nullptr) {
1055 assert((Subtarget.hasBWI()) && "Expected AVX512BW target!");
1056 assert(Subtarget.is32Bit() && "Expecting 32 bit target");
1057 assert(VA.getValVT() == MVT::v64i1 &&
1058 "Expecting first location of 64 bit width type");
1059 assert(NextVA.getValVT() == VA.getValVT() &&
1060 "The locations should have the same type");
1061 assert(VA.isRegLoc() && NextVA.isRegLoc() &&
1062 "The values should reside in two registers");
1063
1064 SDValue Lo, Hi;
1065 SDValue ArgValueLo, ArgValueHi;
1066
1067 MachineFunction &MF = DAG.getMachineFunction();
1068 const TargetRegisterClass *RC = &X86::GR32RegClass;
1069
1070 // Read a 32 bit value from the registers.
1071 if (nullptr == InGlue) {
1072 // When no physical register is present,
1073 // create an intermediate virtual register.
1074 Register Reg = MF.addLiveIn(PReg: VA.getLocReg(), RC);
1075 ArgValueLo = DAG.getCopyFromReg(Chain: Root, dl: DL, Reg, VT: MVT::i32);
1076 Reg = MF.addLiveIn(PReg: NextVA.getLocReg(), RC);
1077 ArgValueHi = DAG.getCopyFromReg(Chain: Root, dl: DL, Reg, VT: MVT::i32);
1078 } else {
1079 // When a physical register is available read the value from it and glue
1080 // the reads together.
1081 ArgValueLo =
1082 DAG.getCopyFromReg(Chain: Root, dl: DL, Reg: VA.getLocReg(), VT: MVT::i32, Glue: *InGlue);
1083 *InGlue = ArgValueLo.getValue(R: 2);
1084 ArgValueHi =
1085 DAG.getCopyFromReg(Chain: Root, dl: DL, Reg: NextVA.getLocReg(), VT: MVT::i32, Glue: *InGlue);
1086 *InGlue = ArgValueHi.getValue(R: 2);
1087 }
1088
1089 // Convert the i32 type into v32i1 type.
1090 Lo = DAG.getBitcast(VT: MVT::v32i1, V: ArgValueLo);
1091
1092 // Convert the i32 type into v32i1 type.
1093 Hi = DAG.getBitcast(VT: MVT::v32i1, V: ArgValueHi);
1094
1095 // Concatenate the two values together.
1096 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v64i1, N1: Lo, N2: Hi);
1097}
1098
1099/// The function will lower a register of various sizes (8/16/32/64)
1100/// to a mask value of the expected size (v8i1/v16i1/v32i1/v64i1)
1101/// \returns a DAG node contains the operand after lowering to mask type.
1102static SDValue lowerRegToMasks(const SDValue &ValArg, const EVT &ValVT,
1103 const EVT &ValLoc, const SDLoc &DL,
1104 SelectionDAG &DAG) {
1105 SDValue ValReturned = ValArg;
1106
1107 if (ValVT == MVT::v1i1)
1108 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v1i1, Operand: ValReturned);
1109
1110 if (ValVT == MVT::v64i1) {
1111 // In 32 bit machine, this case is handled by getv64i1Argument
1112 assert(ValLoc == MVT::i64 && "Expecting only i64 locations");
1113 // In 64 bit machine, There is no need to truncate the value only bitcast
1114 } else {
1115 MVT MaskLenVT;
1116 switch (ValVT.getSimpleVT().SimpleTy) {
1117 case MVT::v8i1:
1118 MaskLenVT = MVT::i8;
1119 break;
1120 case MVT::v16i1:
1121 MaskLenVT = MVT::i16;
1122 break;
1123 case MVT::v32i1:
1124 MaskLenVT = MVT::i32;
1125 break;
1126 default:
1127 llvm_unreachable("Expecting a vector of i1 types");
1128 }
1129
1130 ValReturned = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MaskLenVT, Operand: ValReturned);
1131 }
1132 return DAG.getBitcast(VT: ValVT, V: ValReturned);
1133}
1134
1135static SDValue getPopFromX87Reg(SelectionDAG &DAG, SDValue Chain,
1136 const SDLoc &dl, Register Reg, EVT VT,
1137 SDValue Glue) {
1138 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::Other, VT3: MVT::Glue);
1139 SDValue Ops[] = {Chain, DAG.getRegister(Reg, VT), Glue};
1140 return DAG.getNode(Opcode: X86ISD::POP_FROM_X87_REG, DL: dl, VTList: VTs,
1141 Ops: ArrayRef(Ops, Glue.getNode() ? 3 : 2));
1142}
1143
1144/// Lower the result values of a call into the
1145/// appropriate copies out of appropriate physical registers.
1146///
1147SDValue X86TargetLowering::LowerCallResult(
1148 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg,
1149 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
1150 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
1151 uint32_t *RegMask) const {
1152
1153 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
1154 // Assign locations to each value returned by this call.
1155 SmallVector<CCValAssign, 16> RVLocs;
1156 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
1157 *DAG.getContext());
1158 CCInfo.AnalyzeCallResult(Ins, Fn: RetCC_X86);
1159
1160 // Copy all of the result registers out of their specified physreg.
1161 for (unsigned I = 0, E = RVLocs.size(); I != E; ++I) {
1162 CCValAssign &VA = RVLocs[I];
1163 EVT CopyVT = VA.getLocVT();
1164
1165 // In some calling conventions we need to remove the used registers
1166 // from the register mask.
1167 if (RegMask) {
1168 for (MCPhysReg SubReg : TRI->subregs_inclusive(Reg: VA.getLocReg()))
1169 RegMask[SubReg / 32] &= ~(1u << (SubReg % 32));
1170 }
1171
1172 // Report an error if there was an attempt to return FP values via XMM
1173 // registers.
1174 if (!Subtarget.hasSSE1() && X86::FR32XRegClass.contains(Reg: VA.getLocReg())) {
1175 errorUnsupported(DAG, dl, Msg: "SSE register return with SSE disabled");
1176 if (VA.getLocReg() == X86::XMM1)
1177 VA.convertToReg(Reg: X86::FP1); // Set reg to FP1, avoid hitting asserts.
1178 else
1179 VA.convertToReg(Reg: X86::FP0); // Set reg to FP0, avoid hitting asserts.
1180 } else if (!Subtarget.hasSSE2() &&
1181 X86::FR64XRegClass.contains(Reg: VA.getLocReg()) &&
1182 CopyVT == MVT::f64) {
1183 errorUnsupported(DAG, dl, Msg: "SSE2 register return with SSE2 disabled");
1184 if (VA.getLocReg() == X86::XMM1)
1185 VA.convertToReg(Reg: X86::FP1); // Set reg to FP1, avoid hitting asserts.
1186 else
1187 VA.convertToReg(Reg: X86::FP0); // Set reg to FP0, avoid hitting asserts.
1188 }
1189
1190 // If we prefer to use the value in xmm registers, copy it out as f80 and
1191 // use a truncate to move it from fp stack reg to xmm reg.
1192 bool RoundAfterCopy = false;
1193 bool X87Result = VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1;
1194 if (X87Result && isScalarFPTypeInSSEReg(VT: VA.getValVT())) {
1195 if (!Subtarget.hasX87())
1196 report_fatal_error(reason: "X87 register return with X87 disabled");
1197 CopyVT = MVT::f80;
1198 RoundAfterCopy = (CopyVT != VA.getLocVT());
1199 }
1200
1201 SDValue Val;
1202 if (VA.needsCustom()) {
1203 assert(VA.getValVT() == MVT::v64i1 &&
1204 "Currently the only custom case is when we split v64i1 to 2 regs");
1205 Val =
1206 getv64i1Argument(VA, NextVA&: RVLocs[++I], Root&: Chain, DAG, DL: dl, Subtarget, InGlue: &InGlue);
1207 } else {
1208 Chain =
1209 X87Result
1210 ? getPopFromX87Reg(DAG, Chain, dl, Reg: VA.getLocReg(), VT: CopyVT, Glue: InGlue)
1211 .getValue(R: 1)
1212 : DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: CopyVT, Glue: InGlue)
1213 .getValue(R: 1);
1214 Val = Chain.getValue(R: 0);
1215 InGlue = Chain.getValue(R: 2);
1216 }
1217
1218 if (RoundAfterCopy)
1219 Val = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: VA.getValVT(), N1: Val,
1220 // This truncation won't change the value.
1221 N2: DAG.getIntPtrConstant(Val: 1, DL: dl, /*isTarget=*/true));
1222
1223 if (VA.isExtInLoc()) {
1224 if (VA.getValVT().isVector() &&
1225 VA.getValVT().getScalarType() == MVT::i1 &&
1226 ((VA.getLocVT() == MVT::i64) || (VA.getLocVT() == MVT::i32) ||
1227 (VA.getLocVT() == MVT::i16) || (VA.getLocVT() == MVT::i8))) {
1228 // promoting a mask type (v*i1) into a register of type i64/i32/i16/i8
1229 Val = lowerRegToMasks(ValArg: Val, ValVT: VA.getValVT(), ValLoc: VA.getLocVT(), DL: dl, DAG);
1230 } else
1231 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VA.getValVT(), Operand: Val);
1232 }
1233
1234 if (VA.getLocInfo() == CCValAssign::BCvt)
1235 Val = DAG.getBitcast(VT: VA.getValVT(), V: Val);
1236
1237 InVals.push_back(Elt: Val);
1238 }
1239
1240 return Chain;
1241}
1242
1243/// Determines whether Args, either a set of outgoing arguments to a call, or a
1244/// set of incoming args of a call, contains an sret pointer that the callee
1245/// pops. This happens on most x86-32, System V platforms, unless register
1246/// parameters are in use (-mregparm=1+, regcallcc, etc).
1247template <typename T>
1248static bool hasCalleePopSRet(const SmallVectorImpl<T> &Args,
1249 const SmallVectorImpl<CCValAssign> &ArgLocs,
1250 const X86Subtarget &Subtarget) {
1251 // Not C++20 (yet), so no concepts available.
1252 static_assert(std::is_same_v<T, ISD::OutputArg> ||
1253 std::is_same_v<T, ISD::InputArg>,
1254 "requires ISD::OutputArg or ISD::InputArg");
1255
1256 // Popping the sret pointer only happens on x86-32 System V ABI platforms
1257 // (Linux, Cygwin, BSDs, Mac, etc). That excludes Windows-minus-Cygwin and
1258 // MCU.
1259 const Triple &TT = Subtarget.getTargetTriple();
1260 if (!TT.isX86_32() || TT.isOSMSVCRT() || TT.isOSIAMCU())
1261 return false;
1262
1263 // Check if the first argument is marked sret and if it is passed in memory.
1264 bool IsSRetInMem = false;
1265 if (!Args.empty())
1266 IsSRetInMem = Args.front().Flags.isSRet() && ArgLocs.front().isMemLoc();
1267 return IsSRetInMem;
1268}
1269
1270/// Make a copy of an aggregate at address specified by "Src" to address
1271/// "Dst" with size and alignment information specified by the specific
1272/// parameter attribute. The copy will be passed as a byval function parameter.
1273static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
1274 SDValue Chain, ISD::ArgFlagsTy Flags,
1275 SelectionDAG &DAG, const SDLoc &dl) {
1276 SDValue SizeNode = DAG.getIntPtrConstant(Val: Flags.getByValSize(), DL: dl);
1277 Align Alignment = Flags.getNonZeroByValAlign();
1278 return DAG.getMemcpy(Chain, dl, Dst, Src, Size: SizeNode, DstAlign: Alignment, SrcAlign: Alignment,
1279 /*isVolatile*/ isVol: false, /*AlwaysInline=*/true,
1280 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(),
1281 SrcPtrInfo: MachinePointerInfo());
1282}
1283
1284/// Return true if the calling convention is one that we can guarantee TCO for.
1285static bool canGuaranteeTCO(CallingConv::ID CC) {
1286 return (CC == CallingConv::Fast || CC == CallingConv::GHC ||
1287 CC == CallingConv::X86_RegCall || CC == CallingConv::HiPE ||
1288 CC == CallingConv::Tail || CC == CallingConv::SwiftTail);
1289}
1290
1291/// Return true if we might ever do TCO for calls with this calling convention.
1292static bool mayTailCallThisCC(CallingConv::ID CC) {
1293 switch (CC) {
1294 // C calling conventions:
1295 case CallingConv::C:
1296 case CallingConv::Win64:
1297 case CallingConv::X86_64_SysV:
1298 case CallingConv::PreserveNone:
1299 // Callee pop conventions:
1300 case CallingConv::X86_ThisCall:
1301 case CallingConv::X86_StdCall:
1302 case CallingConv::X86_VectorCall:
1303 case CallingConv::X86_FastCall:
1304 // Swift:
1305 case CallingConv::Swift:
1306 return true;
1307 default:
1308 return canGuaranteeTCO(CC);
1309 }
1310}
1311
1312/// Return true if the function is being made into a tailcall target by
1313/// changing its ABI.
1314static bool shouldGuaranteeTCO(CallingConv::ID CC, bool GuaranteedTailCallOpt) {
1315 return (GuaranteedTailCallOpt && canGuaranteeTCO(CC)) ||
1316 CC == CallingConv::Tail || CC == CallingConv::SwiftTail;
1317}
1318
1319bool X86TargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
1320 if (!CI->isTailCall())
1321 return false;
1322
1323 CallingConv::ID CalleeCC = CI->getCallingConv();
1324 if (!mayTailCallThisCC(CC: CalleeCC))
1325 return false;
1326
1327 return true;
1328}
1329
1330SDValue
1331X86TargetLowering::LowerMemArgument(SDValue Chain, CallingConv::ID CallConv,
1332 const SmallVectorImpl<ISD::InputArg> &Ins,
1333 const SDLoc &dl, SelectionDAG &DAG,
1334 const CCValAssign &VA,
1335 MachineFrameInfo &MFI, unsigned i) const {
1336 // Create the nodes corresponding to a load from this parameter slot.
1337 ISD::ArgFlagsTy Flags = Ins[i].Flags;
1338 bool AlwaysUseMutable = shouldGuaranteeTCO(
1339 CC: CallConv, GuaranteedTailCallOpt: DAG.getTarget().Options.GuaranteedTailCallOpt);
1340 bool isImmutable = !AlwaysUseMutable && !Flags.isByVal();
1341 EVT ValVT;
1342 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
1343
1344 // If value is passed by pointer we have address passed instead of the value
1345 // itself. No need to extend if the mask value and location share the same
1346 // absolute size.
1347 bool ExtendedInMem =
1348 VA.isExtInLoc() && VA.getValVT().getScalarType() == MVT::i1 &&
1349 VA.getValVT().getSizeInBits() != VA.getLocVT().getSizeInBits();
1350
1351 if (VA.getLocInfo() == CCValAssign::Indirect || ExtendedInMem)
1352 ValVT = VA.getLocVT();
1353 else
1354 ValVT = VA.getValVT();
1355
1356 // FIXME: For now, all byval parameter objects are marked mutable. This can be
1357 // changed with more analysis.
1358 // In case of tail call optimization mark all arguments mutable. Since they
1359 // could be overwritten by lowering of arguments in case of a tail call.
1360 if (Flags.isByVal()) {
1361 unsigned Bytes = Flags.getByValSize();
1362 if (Bytes == 0) Bytes = 1; // Don't create zero-sized stack objects.
1363
1364 // FIXME: For now, all byval parameter objects are marked as aliasing. This
1365 // can be improved with deeper analysis.
1366 int FI = MFI.CreateFixedObject(Size: Bytes, SPOffset: VA.getLocMemOffset(), IsImmutable: isImmutable,
1367 /*isAliased=*/true);
1368 return DAG.getFrameIndex(FI, VT: PtrVT);
1369 }
1370
1371 EVT ArgVT = Ins[i].ArgVT;
1372
1373 // If this is a vector that has been split into multiple parts, don't elide
1374 // the copy. The layout on the stack may not match the packed in-memory
1375 // layout.
1376 bool ScalarizedVector = ArgVT.isVector() && !VA.getLocVT().isVector();
1377
1378 // This is an argument in memory. We might be able to perform copy elision.
1379 // If the argument is passed directly in memory without any extension, then we
1380 // can perform copy elision. Large vector types, for example, may be passed
1381 // indirectly by pointer.
1382 if (Flags.isCopyElisionCandidate() &&
1383 VA.getLocInfo() != CCValAssign::Indirect && !ExtendedInMem &&
1384 !ScalarizedVector) {
1385 SDValue PartAddr;
1386 if (Ins[i].PartOffset == 0) {
1387 // If this is a one-part value or the first part of a multi-part value,
1388 // create a stack object for the entire argument value type and return a
1389 // load from our portion of it. This assumes that if the first part of an
1390 // argument is in memory, the rest will also be in memory.
1391 int FI = MFI.CreateFixedObject(Size: ArgVT.getStoreSize(), SPOffset: VA.getLocMemOffset(),
1392 /*IsImmutable=*/false);
1393 PartAddr = DAG.getFrameIndex(FI, VT: PtrVT);
1394 return DAG.getLoad(
1395 VT: ValVT, dl, Chain, Ptr: PartAddr,
1396 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI));
1397 }
1398
1399 // This is not the first piece of an argument in memory. See if there is
1400 // already a fixed stack object including this offset. If so, assume it
1401 // was created by the PartOffset == 0 branch above and create a load from
1402 // the appropriate offset into it.
1403 int64_t PartBegin = VA.getLocMemOffset();
1404 int64_t PartEnd = PartBegin + ValVT.getSizeInBits() / 8;
1405 int FI = MFI.getObjectIndexBegin();
1406 for (; MFI.isFixedObjectIndex(ObjectIdx: FI); ++FI) {
1407 int64_t ObjBegin = MFI.getObjectOffset(ObjectIdx: FI);
1408 int64_t ObjEnd = ObjBegin + MFI.getObjectSize(ObjectIdx: FI);
1409 if (ObjBegin <= PartBegin && PartEnd <= ObjEnd)
1410 break;
1411 }
1412 if (MFI.isFixedObjectIndex(ObjectIdx: FI)) {
1413 SDValue Addr =
1414 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: DAG.getFrameIndex(FI, VT: PtrVT),
1415 N2: DAG.getIntPtrConstant(Val: Ins[i].PartOffset, DL: dl));
1416 return DAG.getLoad(VT: ValVT, dl, Chain, Ptr: Addr,
1417 PtrInfo: MachinePointerInfo::getFixedStack(
1418 MF&: DAG.getMachineFunction(), FI, Offset: Ins[i].PartOffset));
1419 }
1420 }
1421
1422 int FI = MFI.CreateFixedObject(Size: ValVT.getSizeInBits() / 8,
1423 SPOffset: VA.getLocMemOffset(), IsImmutable: isImmutable);
1424
1425 // Set SExt or ZExt flag.
1426 if (VA.getLocInfo() == CCValAssign::ZExt) {
1427 MFI.setObjectZExt(ObjectIdx: FI, IsZExt: true);
1428 } else if (VA.getLocInfo() == CCValAssign::SExt) {
1429 MFI.setObjectSExt(ObjectIdx: FI, IsSExt: true);
1430 }
1431
1432 MaybeAlign Alignment;
1433 if (Subtarget.isTargetWindowsMSVC() && !Subtarget.is64Bit() &&
1434 ValVT != MVT::f80)
1435 Alignment = MaybeAlign(4);
1436 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
1437 SDValue Val = DAG.getLoad(
1438 VT: ValVT, dl, Chain, Ptr: FIN,
1439 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI),
1440 Alignment);
1441 return ExtendedInMem
1442 ? (VA.getValVT().isVector()
1443 ? DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: VA.getValVT(), Operand: Val)
1444 : DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VA.getValVT(), Operand: Val))
1445 : Val;
1446}
1447
1448// FIXME: Get this from tablegen.
1449static ArrayRef<MCPhysReg> get64BitArgumentGPRs(CallingConv::ID CallConv,
1450 const X86Subtarget &Subtarget) {
1451 assert(Subtarget.is64Bit());
1452
1453 if (Subtarget.isCallingConvWin64(CC: CallConv)) {
1454 static const MCPhysReg GPR64ArgRegsWin64[] = {
1455 X86::RCX, X86::RDX, X86::R8, X86::R9
1456 };
1457 return GPR64ArgRegsWin64;
1458 }
1459
1460 static const MCPhysReg GPR64ArgRegs64Bit[] = {
1461 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9
1462 };
1463 return GPR64ArgRegs64Bit;
1464}
1465
1466// FIXME: Get this from tablegen.
1467static ArrayRef<MCPhysReg> get64BitArgumentXMMs(MachineFunction &MF,
1468 CallingConv::ID CallConv,
1469 const X86Subtarget &Subtarget) {
1470 assert(Subtarget.is64Bit());
1471 if (Subtarget.isCallingConvWin64(CC: CallConv)) {
1472 // The XMM registers which might contain var arg parameters are shadowed
1473 // in their paired GPR. So we only need to save the GPR to their home
1474 // slots.
1475 // TODO: __vectorcall will change this.
1476 return {};
1477 }
1478
1479 bool isSoftFloat = Subtarget.useSoftFloat();
1480 if (isSoftFloat || !Subtarget.hasSSE1())
1481 // Kernel mode asks for SSE to be disabled, so there are no XMM argument
1482 // registers.
1483 return {};
1484
1485 static const MCPhysReg XMMArgRegs64Bit[] = {
1486 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
1487 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
1488 };
1489 return XMMArgRegs64Bit;
1490}
1491
1492#ifndef NDEBUG
1493static bool isSortedByValueNo(ArrayRef<CCValAssign> ArgLocs) {
1494 return llvm::is_sorted(
1495 ArgLocs, [](const CCValAssign &A, const CCValAssign &B) -> bool {
1496 return A.getValNo() < B.getValNo();
1497 });
1498}
1499#endif
1500
1501namespace {
1502/// This is a helper class for lowering variable arguments parameters.
1503class VarArgsLoweringHelper {
1504public:
1505 VarArgsLoweringHelper(X86MachineFunctionInfo *FuncInfo, const SDLoc &Loc,
1506 SelectionDAG &DAG, const X86Subtarget &Subtarget,
1507 CallingConv::ID CallConv, CCState &CCInfo)
1508 : FuncInfo(FuncInfo), DL(Loc), DAG(DAG), Subtarget(Subtarget),
1509 TheMachineFunction(DAG.getMachineFunction()),
1510 TheFunction(TheMachineFunction.getFunction()),
1511 FrameInfo(TheMachineFunction.getFrameInfo()),
1512 FrameLowering(*Subtarget.getFrameLowering()),
1513 TargLowering(DAG.getTargetLoweringInfo()), CallConv(CallConv),
1514 CCInfo(CCInfo) {}
1515
1516 // Lower variable arguments parameters.
1517 void lowerVarArgsParameters(SDValue &Chain, unsigned StackSize);
1518
1519private:
1520 void createVarArgAreaAndStoreRegisters(SDValue &Chain, unsigned StackSize);
1521
1522 void forwardMustTailParameters(SDValue &Chain);
1523
1524 bool is64Bit() const { return Subtarget.is64Bit(); }
1525 bool isWin64() const { return Subtarget.isCallingConvWin64(CC: CallConv); }
1526
1527 X86MachineFunctionInfo *FuncInfo;
1528 const SDLoc &DL;
1529 SelectionDAG &DAG;
1530 const X86Subtarget &Subtarget;
1531 MachineFunction &TheMachineFunction;
1532 const Function &TheFunction;
1533 MachineFrameInfo &FrameInfo;
1534 const TargetFrameLowering &FrameLowering;
1535 const TargetLowering &TargLowering;
1536 CallingConv::ID CallConv;
1537 CCState &CCInfo;
1538};
1539} // namespace
1540
1541void VarArgsLoweringHelper::createVarArgAreaAndStoreRegisters(
1542 SDValue &Chain, unsigned StackSize) {
1543 // If the function takes variable number of arguments, make a frame index for
1544 // the start of the first vararg value... for expansion of llvm.va_start. We
1545 // can skip this if there are no va_start calls.
1546 if (is64Bit() || (CallConv != CallingConv::X86_FastCall &&
1547 CallConv != CallingConv::X86_ThisCall)) {
1548 FuncInfo->setVarArgsFrameIndex(
1549 FrameInfo.CreateFixedObject(Size: 1, SPOffset: StackSize, IsImmutable: true));
1550 }
1551
1552 // 64-bit calling conventions support varargs and register parameters, so we
1553 // have to do extra work to spill them in the prologue.
1554 if (is64Bit()) {
1555 // Find the first unallocated argument registers.
1556 ArrayRef<MCPhysReg> ArgGPRs = get64BitArgumentGPRs(CallConv, Subtarget);
1557 ArrayRef<MCPhysReg> ArgXMMs =
1558 get64BitArgumentXMMs(MF&: TheMachineFunction, CallConv, Subtarget);
1559 unsigned NumIntRegs = CCInfo.getFirstUnallocated(Regs: ArgGPRs);
1560 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(Regs: ArgXMMs);
1561
1562 assert(!(NumXMMRegs && !Subtarget.hasSSE1()) &&
1563 "SSE register cannot be used when SSE is disabled!");
1564
1565 if (isWin64()) {
1566 // Get to the caller-allocated home save location. Add 8 to account
1567 // for the return address.
1568 int HomeOffset = FrameLowering.getOffsetOfLocalArea() + 8;
1569 FuncInfo->setRegSaveFrameIndex(
1570 FrameInfo.CreateFixedObject(Size: 1, SPOffset: NumIntRegs * 8 + HomeOffset, IsImmutable: false));
1571 // Fixup to set vararg frame on shadow area (4 x i64).
1572 if (NumIntRegs < 4)
1573 FuncInfo->setVarArgsFrameIndex(FuncInfo->getRegSaveFrameIndex());
1574 } else {
1575 // For X86-64, if there are vararg parameters that are passed via
1576 // registers, then we must store them to their spots on the stack so
1577 // they may be loaded by dereferencing the result of va_next.
1578 FuncInfo->setVarArgsGPOffset(NumIntRegs * 8);
1579 FuncInfo->setVarArgsFPOffset(ArgGPRs.size() * 8 + NumXMMRegs * 16);
1580 FuncInfo->setRegSaveFrameIndex(FrameInfo.CreateStackObject(
1581 Size: ArgGPRs.size() * 8 + ArgXMMs.size() * 16, Alignment: Align(16), isSpillSlot: false));
1582 }
1583
1584 SmallVector<SDValue, 6>
1585 LiveGPRs; // list of SDValue for GPR registers keeping live input value
1586 SmallVector<SDValue, 8> LiveXMMRegs; // list of SDValue for XMM registers
1587 // keeping live input value
1588 SDValue ALVal; // if applicable keeps SDValue for %al register
1589
1590 // Gather all the live in physical registers.
1591 for (MCPhysReg Reg : ArgGPRs.slice(N: NumIntRegs)) {
1592 Register GPR = TheMachineFunction.addLiveIn(PReg: Reg, RC: &X86::GR64RegClass);
1593 LiveGPRs.push_back(Elt: DAG.getCopyFromReg(Chain, dl: DL, Reg: GPR, VT: MVT::i64));
1594 }
1595 const auto &AvailableXmms = ArgXMMs.slice(N: NumXMMRegs);
1596 if (!AvailableXmms.empty()) {
1597 Register AL = TheMachineFunction.addLiveIn(PReg: X86::AL, RC: &X86::GR8RegClass);
1598 ALVal = DAG.getCopyFromReg(Chain, dl: DL, Reg: AL, VT: MVT::i8);
1599 for (MCPhysReg Reg : AvailableXmms) {
1600 // FastRegisterAllocator spills virtual registers at basic
1601 // block boundary. That leads to usages of xmm registers
1602 // outside of check for %al. Pass physical registers to
1603 // VASTART_SAVE_XMM_REGS to avoid unneccessary spilling.
1604 TheMachineFunction.getRegInfo().addLiveIn(Reg);
1605 LiveXMMRegs.push_back(Elt: DAG.getRegister(Reg, VT: MVT::v4f32));
1606 }
1607 }
1608
1609 // Store the integer parameter registers.
1610 SmallVector<SDValue, 8> MemOps;
1611 SDValue RSFIN =
1612 DAG.getFrameIndex(FI: FuncInfo->getRegSaveFrameIndex(),
1613 VT: TargLowering.getPointerTy(DL: DAG.getDataLayout()));
1614 unsigned Offset = FuncInfo->getVarArgsGPOffset();
1615 for (SDValue Val : LiveGPRs) {
1616 SDValue FIN = DAG.getNode(Opcode: ISD::ADD, DL,
1617 VT: TargLowering.getPointerTy(DL: DAG.getDataLayout()),
1618 N1: RSFIN, N2: DAG.getIntPtrConstant(Val: Offset, DL));
1619 SDValue Store =
1620 DAG.getStore(Chain: Val.getValue(R: 1), dl: DL, Val, Ptr: FIN,
1621 PtrInfo: MachinePointerInfo::getFixedStack(
1622 MF&: DAG.getMachineFunction(),
1623 FI: FuncInfo->getRegSaveFrameIndex(), Offset));
1624 MemOps.push_back(Elt: Store);
1625 Offset += 8;
1626 }
1627
1628 // Now store the XMM (fp + vector) parameter registers.
1629 if (!LiveXMMRegs.empty()) {
1630 SmallVector<SDValue, 12> SaveXMMOps;
1631 SaveXMMOps.push_back(Elt: Chain);
1632 SaveXMMOps.push_back(Elt: ALVal);
1633 SaveXMMOps.push_back(Elt: RSFIN);
1634 SaveXMMOps.push_back(
1635 Elt: DAG.getTargetConstant(Val: FuncInfo->getVarArgsFPOffset(), DL, VT: MVT::i32));
1636 llvm::append_range(C&: SaveXMMOps, R&: LiveXMMRegs);
1637 MachineMemOperand *StoreMMO =
1638 DAG.getMachineFunction().getMachineMemOperand(
1639 PtrInfo: MachinePointerInfo::getFixedStack(
1640 MF&: DAG.getMachineFunction(), FI: FuncInfo->getRegSaveFrameIndex(),
1641 Offset),
1642 F: MachineMemOperand::MOStore, Size: 128, BaseAlignment: Align(16));
1643 MemOps.push_back(Elt: DAG.getMemIntrinsicNode(Opcode: X86ISD::VASTART_SAVE_XMM_REGS,
1644 dl: DL, VTList: DAG.getVTList(VT: MVT::Other),
1645 Ops: SaveXMMOps, MemVT: MVT::i8, MMO: StoreMMO));
1646 }
1647
1648 if (!MemOps.empty())
1649 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOps);
1650 }
1651}
1652
1653void VarArgsLoweringHelper::forwardMustTailParameters(SDValue &Chain) {
1654 // Find the largest legal vector type.
1655 MVT VecVT = MVT::Other;
1656 // FIXME: Only some x86_32 calling conventions support AVX512.
1657 if (Subtarget.useAVX512Regs() &&
1658 (is64Bit() || (CallConv == CallingConv::X86_VectorCall ||
1659 CallConv == CallingConv::Intel_OCL_BI)))
1660 VecVT = MVT::v16f32;
1661 else if (Subtarget.hasAVX())
1662 VecVT = MVT::v8f32;
1663 else if (Subtarget.hasSSE2())
1664 VecVT = MVT::v4f32;
1665
1666 // We forward some GPRs and some vector types.
1667 SmallVector<MVT, 2> RegParmTypes;
1668 MVT IntVT = is64Bit() ? MVT::i64 : MVT::i32;
1669 RegParmTypes.push_back(Elt: IntVT);
1670 if (VecVT != MVT::Other)
1671 RegParmTypes.push_back(Elt: VecVT);
1672
1673 // Compute the set of forwarded registers. The rest are scratch.
1674 SmallVectorImpl<ForwardedRegister> &Forwards =
1675 FuncInfo->getForwardedMustTailRegParms();
1676 CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, Fn: CC_X86);
1677
1678 // Forward AL for SysV x86_64 targets, since it is used for varargs.
1679 if (is64Bit() && !isWin64() && !CCInfo.isAllocated(Reg: X86::AL)) {
1680 Register ALVReg = TheMachineFunction.addLiveIn(PReg: X86::AL, RC: &X86::GR8RegClass);
1681 Forwards.push_back(Elt: ForwardedRegister(ALVReg, X86::AL, MVT::i8));
1682 }
1683
1684 // Copy all forwards from physical to virtual registers.
1685 for (ForwardedRegister &FR : Forwards) {
1686 // FIXME: Can we use a less constrained schedule?
1687 SDValue RegVal = DAG.getCopyFromReg(Chain, dl: DL, Reg: FR.VReg, VT: FR.VT);
1688 FR.VReg = TheMachineFunction.getRegInfo().createVirtualRegister(
1689 RegClass: TargLowering.getRegClassFor(VT: FR.VT));
1690 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: FR.VReg, N: RegVal);
1691 }
1692}
1693
1694void VarArgsLoweringHelper::lowerVarArgsParameters(SDValue &Chain,
1695 unsigned StackSize) {
1696 // Set FrameIndex to the 0xAAAAAAA value to mark unset state.
1697 // If necessary, it would be set into the correct value later.
1698 FuncInfo->setVarArgsFrameIndex(0xAAAAAAA);
1699 FuncInfo->setRegSaveFrameIndex(0xAAAAAAA);
1700
1701 if (FrameInfo.hasVAStart())
1702 createVarArgAreaAndStoreRegisters(Chain, StackSize);
1703
1704 if (FrameInfo.hasMustTailInVarArgFunc())
1705 forwardMustTailParameters(Chain);
1706}
1707
1708SDValue X86TargetLowering::LowerFormalArguments(
1709 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1710 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
1711 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1712 MachineFunction &MF = DAG.getMachineFunction();
1713 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
1714
1715 const Function &F = MF.getFunction();
1716 if (F.hasExternalLinkage() && Subtarget.isTargetCygMing() &&
1717 F.getName() == "main")
1718 FuncInfo->setForceFramePointer(true);
1719
1720 MachineFrameInfo &MFI = MF.getFrameInfo();
1721 bool Is64Bit = Subtarget.is64Bit();
1722 bool IsWin64 = Subtarget.isCallingConvWin64(CC: CallConv);
1723
1724 // On x86_64 with x87 disabled, x86_fp80 cannot be handled: the type would
1725 // need to be returned/passed in x87 registers (FP0/FP1) which are
1726 // unavailable. Emit a clear diagnostic instead of crashing later with
1727 // "Cannot select: build_pair".
1728 if (Is64Bit && !Subtarget.hasX87()) {
1729 if (F.getReturnType()->isX86_FP80Ty() ||
1730 any_of(Range: F.args(), P: [](const Argument &Arg) {
1731 return Arg.getType()->isX86_FP80Ty();
1732 }))
1733 reportFatalUsageError(
1734 reason: "cannot use x86_fp80 type with x87 disabled on x86_64 target");
1735 }
1736
1737 assert(
1738 !(IsVarArg && canGuaranteeTCO(CallConv)) &&
1739 "Var args not supported with calling conv' regcall, fastcc, ghc or hipe");
1740
1741 // Assign locations to all of the incoming arguments.
1742 SmallVector<CCValAssign, 16> ArgLocs;
1743 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1744
1745 // Allocate shadow area for Win64.
1746 if (IsWin64)
1747 CCInfo.AllocateStack(Size: 32, Alignment: Align(8));
1748
1749 CCInfo.AnalyzeArguments(Ins, Fn: CC_X86);
1750
1751 // In vectorcall calling convention a second pass is required for the HVA
1752 // types.
1753 if (CallingConv::X86_VectorCall == CallConv) {
1754 CCInfo.AnalyzeArgumentsSecondPass(Args: Ins, Fn: CC_X86);
1755 }
1756
1757 // The next loop assumes that the locations are in the same order of the
1758 // input arguments.
1759 assert(isSortedByValueNo(ArgLocs) &&
1760 "Argument Location list must be sorted before lowering");
1761
1762 SDValue ArgValue;
1763 for (unsigned I = 0, InsIndex = 0, E = ArgLocs.size(); I != E;
1764 ++I, ++InsIndex) {
1765 assert(InsIndex < Ins.size() && "Invalid Ins index");
1766 CCValAssign &VA = ArgLocs[I];
1767
1768 if (VA.isRegLoc()) {
1769 EVT RegVT = VA.getLocVT();
1770 if (VA.needsCustom()) {
1771 assert(
1772 VA.getValVT() == MVT::v64i1 &&
1773 "Currently the only custom case is when we split v64i1 to 2 regs");
1774
1775 // v64i1 values, in regcall calling convention, that are
1776 // compiled to 32 bit arch, are split up into two registers.
1777 ArgValue =
1778 getv64i1Argument(VA, NextVA&: ArgLocs[++I], Root&: Chain, DAG, DL: dl, Subtarget);
1779 } else {
1780 const TargetRegisterClass *RC;
1781 if (RegVT == MVT::i8)
1782 RC = &X86::GR8RegClass;
1783 else if (RegVT == MVT::i16)
1784 RC = &X86::GR16RegClass;
1785 else if (RegVT == MVT::i32)
1786 RC = &X86::GR32RegClass;
1787 else if (Is64Bit && RegVT == MVT::i64)
1788 RC = &X86::GR64RegClass;
1789 else if (RegVT == MVT::f16)
1790 RC = Subtarget.hasAVX512() ? &X86::FR16XRegClass : &X86::FR16RegClass;
1791 else if (RegVT == MVT::f32)
1792 RC = Subtarget.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
1793 else if (RegVT == MVT::f64)
1794 RC = Subtarget.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
1795 else if (RegVT == MVT::f80)
1796 RC = &X86::RFP80RegClass;
1797 else if (RegVT == MVT::f128)
1798 RC = &X86::VR128RegClass;
1799 else if (RegVT.is512BitVector())
1800 RC = &X86::VR512RegClass;
1801 else if (RegVT.is256BitVector())
1802 RC = Subtarget.hasVLX() ? &X86::VR256XRegClass : &X86::VR256RegClass;
1803 else if (RegVT.is128BitVector())
1804 RC = Subtarget.hasVLX() ? &X86::VR128XRegClass : &X86::VR128RegClass;
1805 else if (RegVT == MVT::x86mmx)
1806 RC = &X86::VR64RegClass;
1807 else if (RegVT == MVT::v1i1)
1808 RC = &X86::VK1RegClass;
1809 else if (RegVT == MVT::v8i1)
1810 RC = &X86::VK8RegClass;
1811 else if (RegVT == MVT::v16i1)
1812 RC = &X86::VK16RegClass;
1813 else if (RegVT == MVT::v32i1)
1814 RC = &X86::VK32RegClass;
1815 else if (RegVT == MVT::v64i1)
1816 RC = &X86::VK64RegClass;
1817 else
1818 llvm_unreachable("Unknown argument type!");
1819
1820 Register Reg = MF.addLiveIn(PReg: VA.getLocReg(), RC);
1821 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, VT: RegVT);
1822 }
1823
1824 // If this is an 8 or 16-bit value, it is really passed promoted to 32
1825 // bits. Insert an assert[sz]ext to capture this, then truncate to the
1826 // right size.
1827 if (VA.getLocInfo() == CCValAssign::SExt)
1828 ArgValue = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: RegVT, N1: ArgValue,
1829 N2: DAG.getValueType(VA.getValVT()));
1830 else if (VA.getLocInfo() == CCValAssign::ZExt)
1831 ArgValue = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: RegVT, N1: ArgValue,
1832 N2: DAG.getValueType(VA.getValVT()));
1833 else if (VA.getLocInfo() == CCValAssign::BCvt)
1834 ArgValue = DAG.getBitcast(VT: VA.getValVT(), V: ArgValue);
1835
1836 if (VA.isExtInLoc()) {
1837 // Handle MMX values passed in XMM regs.
1838 if (RegVT.isVector() && VA.getValVT().getScalarType() != MVT::i1)
1839 ArgValue = DAG.getNode(Opcode: X86ISD::MOVDQ2Q, DL: dl, VT: VA.getValVT(), Operand: ArgValue);
1840 else if (VA.getValVT().isVector() &&
1841 VA.getValVT().getScalarType() == MVT::i1 &&
1842 ((VA.getLocVT() == MVT::i64) || (VA.getLocVT() == MVT::i32) ||
1843 (VA.getLocVT() == MVT::i16) || (VA.getLocVT() == MVT::i8))) {
1844 // Promoting a mask type (v*i1) into a register of type i64/i32/i16/i8
1845 ArgValue = lowerRegToMasks(ValArg: ArgValue, ValVT: VA.getValVT(), ValLoc: RegVT, DL: dl, DAG);
1846 } else
1847 ArgValue = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VA.getValVT(), Operand: ArgValue);
1848 }
1849 } else {
1850 assert(VA.isMemLoc());
1851 ArgValue =
1852 LowerMemArgument(Chain, CallConv, Ins, dl, DAG, VA, MFI, i: InsIndex);
1853 }
1854
1855 // If value is passed via pointer - do a load.
1856 if (VA.getLocInfo() == CCValAssign::Indirect &&
1857 !(Ins[I].Flags.isByVal() && VA.isRegLoc())) {
1858 ArgValue =
1859 DAG.getLoad(VT: VA.getValVT(), dl, Chain, Ptr: ArgValue, PtrInfo: MachinePointerInfo());
1860 }
1861
1862 InVals.push_back(Elt: ArgValue);
1863 }
1864
1865 for (unsigned I = 0, E = Ins.size(); I != E; ++I) {
1866 if (Ins[I].Flags.isSwiftAsync()) {
1867 auto X86FI = MF.getInfo<X86MachineFunctionInfo>();
1868 if (X86::isExtendedSwiftAsyncFrameSupported(Subtarget, MF))
1869 X86FI->setHasSwiftAsyncContext(true);
1870 else {
1871 int PtrSize = Subtarget.is64Bit() ? 8 : 4;
1872 int FI =
1873 MF.getFrameInfo().CreateStackObject(Size: PtrSize, Alignment: Align(PtrSize), isSpillSlot: false);
1874 X86FI->setSwiftAsyncContextFrameIdx(FI);
1875 SDValue St = DAG.getStore(
1876 Chain: DAG.getEntryNode(), dl, Val: InVals[I],
1877 Ptr: DAG.getFrameIndex(FI, VT: PtrSize == 8 ? MVT::i64 : MVT::i32),
1878 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
1879 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: St, N2: Chain);
1880 }
1881 }
1882
1883 // Swift calling convention does not require we copy the sret argument
1884 // into %rax/%eax for the return. We don't set SRetReturnReg for Swift.
1885 if (CallConv == CallingConv::Swift || CallConv == CallingConv::SwiftTail)
1886 continue;
1887
1888 // All x86 ABIs require that for returning structs by value we copy the
1889 // sret argument into %rax/%eax (depending on ABI) for the return. Save
1890 // the argument into a virtual register so that we can access it from the
1891 // return points.
1892 if (Ins[I].Flags.isSRet()) {
1893 assert(!FuncInfo->getSRetReturnReg() &&
1894 "SRet return has already been set");
1895 MVT PtrTy = getPointerTy(DL: DAG.getDataLayout());
1896 Register Reg =
1897 MF.getRegInfo().createVirtualRegister(RegClass: getRegClassFor(VT: PtrTy));
1898 FuncInfo->setSRetReturnReg(Reg);
1899 SDValue Copy = DAG.getCopyToReg(Chain: DAG.getEntryNode(), dl, Reg, N: InVals[I]);
1900 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Copy, N2: Chain);
1901 break;
1902 }
1903 }
1904
1905 unsigned StackSize = CCInfo.getStackSize();
1906 // Align stack specially for tail calls.
1907 if (shouldGuaranteeTCO(CC: CallConv,
1908 GuaranteedTailCallOpt: MF.getTarget().Options.GuaranteedTailCallOpt))
1909 StackSize = GetAlignedArgumentStackSize(StackSize, DAG);
1910
1911 if (IsVarArg)
1912 VarArgsLoweringHelper(FuncInfo, dl, DAG, Subtarget, CallConv, CCInfo)
1913 .lowerVarArgsParameters(Chain, StackSize);
1914
1915 // Some CCs need callee pop.
1916 if (X86::isCalleePop(CallingConv: CallConv, is64Bit: Is64Bit, IsVarArg,
1917 GuaranteeTCO: MF.getTarget().Options.GuaranteedTailCallOpt)) {
1918 FuncInfo->setBytesToPopOnReturn(StackSize); // Callee pops everything.
1919 } else if (CallConv == CallingConv::X86_INTR && Ins.size() == 2) {
1920 // X86 interrupts must pop the error code (and the alignment padding) if
1921 // present.
1922 FuncInfo->setBytesToPopOnReturn(Is64Bit ? 16 : 4);
1923 } else {
1924 FuncInfo->setBytesToPopOnReturn(0); // Callee pops nothing.
1925 // If this is an sret function, the return should pop the hidden pointer.
1926 if (hasCalleePopSRet(Args: Ins, ArgLocs, Subtarget))
1927 FuncInfo->setBytesToPopOnReturn(4);
1928 }
1929
1930 if (!Is64Bit) {
1931 // RegSaveFrameIndex is X86-64 only.
1932 FuncInfo->setRegSaveFrameIndex(0xAAAAAAA);
1933 }
1934
1935 FuncInfo->setArgumentStackSize(StackSize);
1936
1937 if (WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo()) {
1938 EHPersonality Personality = classifyEHPersonality(Pers: F.getPersonalityFn());
1939 if (Personality == EHPersonality::CoreCLR) {
1940 assert(Is64Bit);
1941 // TODO: Add a mechanism to frame lowering that will allow us to indicate
1942 // that we'd prefer this slot be allocated towards the bottom of the frame
1943 // (i.e. near the stack pointer after allocating the frame). Every
1944 // funclet needs a copy of this slot in its (mostly empty) frame, and the
1945 // offset from the bottom of this and each funclet's frame must be the
1946 // same, so the size of funclets' (mostly empty) frames is dictated by
1947 // how far this slot is from the bottom (since they allocate just enough
1948 // space to accommodate holding this slot at the correct offset).
1949 int PSPSymFI = MFI.CreateStackObject(Size: 8, Alignment: Align(8), /*isSpillSlot=*/false);
1950 EHInfo->PSPSymFrameIdx = PSPSymFI;
1951 }
1952 }
1953
1954 if (shouldDisableArgRegFromCSR(CC: CallConv) ||
1955 F.hasFnAttribute(Kind: "no_caller_saved_registers")) {
1956 MachineRegisterInfo &MRI = MF.getRegInfo();
1957 for (std::pair<MCRegister, Register> Pair : MRI.liveins())
1958 MRI.disableCalleeSavedRegister(Reg: Pair.first);
1959 }
1960
1961 if (CallingConv::PreserveNone == CallConv)
1962 for (const ISD::InputArg &In : Ins) {
1963 if (In.Flags.isSwiftSelf() || In.Flags.isSwiftAsync() ||
1964 In.Flags.isSwiftError()) {
1965 errorUnsupported(DAG, dl,
1966 Msg: "Swift attributes can't be used with preserve_none");
1967 break;
1968 }
1969 }
1970
1971 return Chain;
1972}
1973
1974SDValue X86TargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,
1975 SDValue Arg, const SDLoc &dl,
1976 SelectionDAG &DAG,
1977 const CCValAssign &VA,
1978 ISD::ArgFlagsTy Flags,
1979 bool isByVal) const {
1980 unsigned LocMemOffset = VA.getLocMemOffset();
1981 SDValue PtrOff = DAG.getIntPtrConstant(Val: LocMemOffset, DL: dl);
1982 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout()),
1983 N1: StackPtr, N2: PtrOff);
1984 if (isByVal)
1985 return CreateCopyOfByValArgument(Src: Arg, Dst: PtrOff, Chain, Flags, DAG, dl);
1986
1987 MaybeAlign Alignment;
1988 if (Subtarget.isTargetWindowsMSVC() && !Subtarget.is64Bit() &&
1989 Arg.getSimpleValueType() != MVT::f80)
1990 Alignment = MaybeAlign(4);
1991 return DAG.getStore(
1992 Chain, dl, Val: Arg, Ptr: PtrOff,
1993 PtrInfo: MachinePointerInfo::getStack(MF&: DAG.getMachineFunction(), Offset: LocMemOffset),
1994 Alignment);
1995}
1996
1997/// Emit a load of return address if tail call
1998/// optimization is performed and it is required.
1999SDValue X86TargetLowering::EmitTailCallLoadRetAddr(
2000 SelectionDAG &DAG, SDValue &OutRetAddr, SDValue Chain, bool IsTailCall,
2001 bool Is64Bit, int FPDiff, const SDLoc &dl) const {
2002 // Adjust the Return address stack slot.
2003 EVT VT = getPointerTy(DL: DAG.getDataLayout());
2004 OutRetAddr = getReturnAddressFrameIndex(DAG);
2005
2006 // Load the "old" Return address.
2007 OutRetAddr = DAG.getLoad(VT, dl, Chain, Ptr: OutRetAddr, PtrInfo: MachinePointerInfo());
2008 return SDValue(OutRetAddr.getNode(), 1);
2009}
2010
2011/// Emit a store of the return address if tail call
2012/// optimization is performed and it is required (FPDiff!=0).
2013static SDValue EmitTailCallStoreRetAddr(SelectionDAG &DAG, MachineFunction &MF,
2014 SDValue Chain, SDValue RetAddrFrIdx,
2015 EVT PtrVT, unsigned SlotSize,
2016 int FPDiff, const SDLoc &dl) {
2017 // Store the return address to the appropriate stack slot.
2018 if (!FPDiff) return Chain;
2019 // Calculate the new stack slot for the return address.
2020 int NewReturnAddrFI =
2021 MF.getFrameInfo().CreateFixedObject(Size: SlotSize, SPOffset: (int64_t)FPDiff - SlotSize,
2022 IsImmutable: false);
2023 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(FI: NewReturnAddrFI, VT: PtrVT);
2024 Chain = DAG.getStore(Chain, dl, Val: RetAddrFrIdx, Ptr: NewRetAddrFrIdx,
2025 PtrInfo: MachinePointerInfo::getFixedStack(
2026 MF&: DAG.getMachineFunction(), FI: NewReturnAddrFI));
2027 return Chain;
2028}
2029
2030/// Returns a vector_shuffle mask for an movs{s|d}, movd
2031/// operation of specified width.
2032SDValue X86TargetLowering::getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT,
2033 SDValue V1, SDValue V2) const {
2034 unsigned NumElems = VT.getVectorNumElements();
2035 SmallVector<int, 8> Mask;
2036 Mask.push_back(Elt: NumElems);
2037 for (unsigned i = 1; i != NumElems; ++i)
2038 Mask.push_back(Elt: i);
2039 return DAG.getVectorShuffle(VT, dl, N1: V1, N2: V2, Mask);
2040}
2041
2042// Returns the type of copying which is required to set up a byval argument to
2043// a tail-called function. This isn't needed for non-tail calls, because they
2044// always need the equivalent of CopyOnce, but tail-calls sometimes need two to
2045// avoid clobbering another argument (CopyViaTemp), and sometimes can be
2046// optimised to zero copies when forwarding an argument from the caller's
2047// caller (NoCopy).
2048X86TargetLowering::ByValCopyKind X86TargetLowering::ByValNeedsCopyForTailCall(
2049 SelectionDAG &DAG, SDValue Src, SDValue Dst, ISD::ArgFlagsTy Flags) const {
2050 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2051
2052 // Globals are always safe to copy from.
2053 if (isa<GlobalAddressSDNode>(Val: Src) || isa<ExternalSymbolSDNode>(Val: Src))
2054 return CopyOnce;
2055
2056 // Can only analyse frame index nodes, conservatively assume we need a
2057 // temporary.
2058 auto *SrcFrameIdxNode = dyn_cast<FrameIndexSDNode>(Val&: Src);
2059 auto *DstFrameIdxNode = dyn_cast<FrameIndexSDNode>(Val&: Dst);
2060 if (!SrcFrameIdxNode || !DstFrameIdxNode)
2061 return CopyViaTemp;
2062
2063 int SrcFI = SrcFrameIdxNode->getIndex();
2064 int DstFI = DstFrameIdxNode->getIndex();
2065 assert(MFI.isFixedObjectIndex(DstFI) &&
2066 "byval passed in non-fixed stack slot");
2067
2068 int64_t SrcOffset = MFI.getObjectOffset(ObjectIdx: SrcFI);
2069 int64_t DstOffset = MFI.getObjectOffset(ObjectIdx: DstFI);
2070
2071 // If the source is in the local frame, then the copy to the argument
2072 // memory is always valid.
2073 bool FixedSrc = MFI.isFixedObjectIndex(ObjectIdx: SrcFI);
2074 if (!FixedSrc || (FixedSrc && SrcOffset < 0))
2075 return CopyOnce;
2076
2077 // If the value is already in the correct location, then no copying is
2078 // needed. If not, then we need to copy via a temporary.
2079 if (SrcOffset == DstOffset)
2080 return NoCopy;
2081 else
2082 return CopyViaTemp;
2083}
2084
2085SDValue
2086X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
2087 SmallVectorImpl<SDValue> &InVals) const {
2088 SelectionDAG &DAG = CLI.DAG;
2089 SDLoc &dl = CLI.DL;
2090 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2091 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2092 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2093 SDValue Chain = CLI.Chain;
2094 SDValue Callee = CLI.Callee;
2095 CallingConv::ID CallConv = CLI.CallConv;
2096 bool &isTailCall = CLI.IsTailCall;
2097 bool isVarArg = CLI.IsVarArg;
2098 const auto *CB = CLI.CB;
2099
2100 MachineFunction &MF = DAG.getMachineFunction();
2101 bool Is64Bit = Subtarget.is64Bit();
2102 bool IsWin64 = Subtarget.isCallingConvWin64(CC: CallConv);
2103 bool ShouldGuaranteeTCO = shouldGuaranteeTCO(
2104 CC: CallConv, GuaranteedTailCallOpt: MF.getTarget().Options.GuaranteedTailCallOpt);
2105 X86MachineFunctionInfo *X86Info = MF.getInfo<X86MachineFunctionInfo>();
2106 bool HasNCSR = (CB && isa<CallInst>(Val: CB) &&
2107 CB->hasFnAttr(Kind: "no_caller_saved_registers"));
2108 bool IsIndirectCall = (CB && isa<CallInst>(Val: CB) && CB->isIndirectCall());
2109 bool IsCFICall = CB && CB->isIndirectCall() && CLI.CFIType;
2110 const Module *M = MF.getFunction().getParent();
2111
2112 // If the indirect call target has the nocf_check attribute, the call needs
2113 // the NOTRACK prefix. For simplicity just disable tail calls as there are
2114 // so many variants.
2115 // FIXME: This will cause backend errors if the user forces the issue.
2116 bool IsNoTrackIndirectCall = IsIndirectCall && CB->doesNoCfCheck() &&
2117 M->getModuleFlag(Key: "cf-protection-branch");
2118 if (IsNoTrackIndirectCall)
2119 isTailCall = false;
2120
2121 MachineFunction::CallSiteInfo CSInfo;
2122 if (CallConv == CallingConv::X86_INTR)
2123 report_fatal_error(reason: "X86 interrupts may not be called directly");
2124
2125 // Set type id for call site info.
2126 setTypeIdForCallsiteInfo(CB, MF, CSInfo);
2127
2128 if (IsIndirectCall && !IsWin64 &&
2129 M->getModuleFlag(Key: "import-call-optimization"))
2130 errorUnsupported(DAG, dl,
2131 Msg: "Indirect calls must have a normal calling convention if "
2132 "Import Call Optimization is enabled");
2133
2134 // Analyze operands of the call, assigning locations to each operand.
2135 SmallVector<CCValAssign, 16> ArgLocs;
2136 CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
2137
2138 // Allocate shadow area for Win64.
2139 if (IsWin64)
2140 CCInfo.AllocateStack(Size: 32, Alignment: Align(8));
2141
2142 CCInfo.AnalyzeArguments(Outs, Fn: CC_X86);
2143
2144 // In vectorcall calling convention a second pass is required for the HVA
2145 // types.
2146 if (CallingConv::X86_VectorCall == CallConv) {
2147 CCInfo.AnalyzeArgumentsSecondPass(Args: Outs, Fn: CC_X86);
2148 }
2149
2150 // We cannot guarantee TCO for mismatched calling conventions.
2151 if (isTailCall && ShouldGuaranteeTCO) {
2152 CallingConv::ID CallerCC = MF.getFunction().getCallingConv();
2153 isTailCall = (CallConv == CallerCC);
2154 }
2155
2156 // Check if this tail call is a "sibling" call, which is loosely defined to
2157 // be a tail call that doesn't require heroics like moving the return
2158 // address or swapping byval arguments. We treat some musttail calls as
2159 // sibling calls to avoid unnecessary argument copies.
2160 bool IsMustTail = CLI.CB && CLI.CB->isMustTailCall();
2161 bool IsSibcall = false;
2162 if (isTailCall) {
2163 IsSibcall = isEligibleForSiblingCallOpt(CLI, CCInfo, ArgLocs);
2164 isTailCall = IsSibcall || IsMustTail || ShouldGuaranteeTCO;
2165 }
2166
2167 if (isTailCall)
2168 ++NumTailCalls;
2169
2170 if (IsMustTail && !isTailCall)
2171 report_fatal_error(reason: "failed to perform tail call elimination on a call "
2172 "site marked musttail");
2173
2174 assert(!(isVarArg && canGuaranteeTCO(CallConv)) &&
2175 "Var args not supported with calling convention fastcc, ghc or hipe");
2176
2177 // Get a count of how many bytes are to be pushed on the stack.
2178 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
2179 if (IsSibcall)
2180 // This is a sibcall. The memory operands are available in caller's
2181 // own caller's stack.
2182 NumBytes = 0;
2183 else if (ShouldGuaranteeTCO && canGuaranteeTCO(CC: CallConv))
2184 // The callee pops this amount, so compute it as LowerFormalArguments does.
2185 NumBytes = GetAlignedArgumentStackSize(StackSize: CCInfo.getStackSize(), DAG);
2186
2187 // A sibcall is ABI-compatible and does not need to adjust the stack pointer.
2188 int FPDiff = 0;
2189 if (isTailCall && ShouldGuaranteeTCO && !IsSibcall) {
2190 // Lower arguments at fp - stackoffset + fpdiff.
2191 unsigned NumBytesCallerPushed = X86Info->getBytesToPopOnReturn();
2192
2193 FPDiff = NumBytesCallerPushed - NumBytes;
2194
2195 // Set the delta of movement of the returnaddr stackslot.
2196 // But only set if delta is greater than previous delta.
2197 if (FPDiff < X86Info->getTCReturnAddrDelta())
2198 X86Info->setTCReturnAddrDelta(FPDiff);
2199 }
2200
2201 unsigned NumBytesToPush = NumBytes;
2202 unsigned NumBytesToPop = NumBytes;
2203
2204 SDValue StackPtr;
2205 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
2206
2207 // If we are doing a tail-call, any byval arguments will be written to stack
2208 // space which was used for incoming arguments. If any the values being used
2209 // are incoming byval arguments to this function, then they might be
2210 // overwritten by the stores of the outgoing arguments. To avoid this, we
2211 // need to make a temporary copy of them in local stack space, then copy back
2212 // to the argument area.
2213 // FIXME: There's potential to improve the code by using virtual registers for
2214 // temporary storage, and letting the register allocator spill if needed.
2215 SmallVector<SDValue, 8> ByValTemporaries;
2216 SDValue ByValTempChain;
2217 if (isTailCall) {
2218 // Use null SDValue to mean "no temporary recorded for this arg index".
2219 ByValTemporaries.assign(NumElts: OutVals.size(), Elt: SDValue());
2220
2221 SmallVector<SDValue, 8> ByValCopyChains;
2222 for (const CCValAssign &VA : ArgLocs) {
2223 unsigned ArgIdx = VA.getValNo();
2224 SDValue Src = OutVals[ArgIdx];
2225 ISD::ArgFlagsTy Flags = Outs[ArgIdx].Flags;
2226
2227 if (!Flags.isByVal())
2228 continue;
2229
2230 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
2231
2232 if (!StackPtr.getNode())
2233 StackPtr =
2234 DAG.getCopyFromReg(Chain, dl, Reg: RegInfo->getStackRegister(), VT: PtrVT);
2235
2236 // Destination: where this byval should live in the callee’s frame
2237 // after the tail call.
2238 int64_t Offset = VA.getLocMemOffset() + FPDiff;
2239 uint64_t Size = VA.getLocVT().getFixedSizeInBits() / 8;
2240 int FI = MF.getFrameInfo().CreateFixedObject(Size, SPOffset: Offset,
2241 /*IsImmutable=*/true);
2242 SDValue Dst = DAG.getFrameIndex(FI, VT: PtrVT);
2243
2244 ByValCopyKind Copy = ByValNeedsCopyForTailCall(DAG, Src, Dst, Flags);
2245
2246 if (Copy == NoCopy) {
2247 // If the argument is already at the correct offset on the stack
2248 // (because we are forwarding a byval argument from our caller), we
2249 // don't need any copying.
2250 continue;
2251 } else if (Copy == CopyOnce) {
2252 // If the argument is in our local stack frame, no other argument
2253 // preparation can clobber it, so we can copy it to the final location
2254 // later.
2255 ByValTemporaries[ArgIdx] = Src;
2256 } else {
2257 assert(Copy == CopyViaTemp && "unexpected enum value");
2258 // If we might be copying this argument from the outgoing argument
2259 // stack area, we need to copy via a temporary in the local stack
2260 // frame.
2261 MachineFrameInfo &MFI = MF.getFrameInfo();
2262 int TempFrameIdx = MFI.CreateStackObject(Size: Flags.getByValSize(),
2263 Alignment: Flags.getNonZeroByValAlign(),
2264 /*isSS=*/isSpillSlot: false);
2265 SDValue Temp =
2266 DAG.getFrameIndex(FI: TempFrameIdx, VT: getPointerTy(DL: DAG.getDataLayout()));
2267
2268 SDValue CopyChain =
2269 CreateCopyOfByValArgument(Src, Dst: Temp, Chain, Flags, DAG, dl);
2270 ByValCopyChains.push_back(Elt: CopyChain);
2271 ByValTemporaries[ArgIdx] = Temp;
2272 }
2273 }
2274 if (!ByValCopyChains.empty())
2275 ByValTempChain =
2276 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: ByValCopyChains);
2277 }
2278
2279 // If we have an inalloca argument, all stack space has already been allocated
2280 // for us and be right at the top of the stack. We don't support multiple
2281 // arguments passed in memory when using inalloca.
2282 if (!Outs.empty() && Outs.back().Flags.isInAlloca()) {
2283 NumBytesToPush = 0;
2284 if (!ArgLocs.back().isMemLoc())
2285 report_fatal_error(reason: "cannot use inalloca attribute on a register "
2286 "parameter");
2287 if (ArgLocs.back().getLocMemOffset() != 0)
2288 report_fatal_error(reason: "any parameter with the inalloca attribute must be "
2289 "the only memory argument");
2290 } else if (CLI.IsPreallocated) {
2291 assert(ArgLocs.back().isMemLoc() &&
2292 "cannot use preallocated attribute on a register "
2293 "parameter");
2294 SmallVector<size_t, 4> PreallocatedOffsets;
2295 for (size_t i = 0; i < CLI.OutVals.size(); ++i) {
2296 if (CLI.CB->paramHasAttr(ArgNo: i, Kind: Attribute::Preallocated)) {
2297 PreallocatedOffsets.push_back(Elt: ArgLocs[i].getLocMemOffset());
2298 }
2299 }
2300 auto *MFI = DAG.getMachineFunction().getInfo<X86MachineFunctionInfo>();
2301 size_t PreallocatedId = MFI->getPreallocatedIdForCallSite(CS: CLI.CB);
2302 MFI->setPreallocatedStackSize(Id: PreallocatedId, StackSize: NumBytes);
2303 MFI->setPreallocatedArgOffsets(Id: PreallocatedId, AO: PreallocatedOffsets);
2304 NumBytesToPush = 0;
2305 }
2306
2307 if (!IsSibcall && !IsMustTail)
2308 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytesToPush,
2309 OutSize: NumBytes - NumBytesToPush, DL: dl);
2310
2311 SDValue RetAddrFrIdx;
2312 // Load return address for tail calls.
2313 if (isTailCall && FPDiff)
2314 Chain = EmitTailCallLoadRetAddr(DAG, OutRetAddr&: RetAddrFrIdx, Chain, IsTailCall: isTailCall,
2315 Is64Bit, FPDiff, dl);
2316
2317 SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
2318 SmallVector<SDValue, 8> MemOpChains;
2319
2320 // The next loop assumes that the locations are in the same order of the
2321 // input arguments.
2322 assert(isSortedByValueNo(ArgLocs) &&
2323 "Argument Location list must be sorted before lowering");
2324
2325 // Walk the register/memloc assignments, inserting copies/loads. In the case
2326 // of tail call optimization arguments are handle later.
2327 for (unsigned I = 0, OutIndex = 0, E = ArgLocs.size(); I != E;
2328 ++I, ++OutIndex) {
2329 assert(OutIndex < Outs.size() && "Invalid Out index");
2330 // Skip inalloca/preallocated arguments, they have already been written.
2331 ISD::ArgFlagsTy Flags = Outs[OutIndex].Flags;
2332 if (Flags.isInAlloca() || Flags.isPreallocated())
2333 continue;
2334
2335 CCValAssign &VA = ArgLocs[I];
2336 EVT RegVT = VA.getLocVT();
2337 SDValue Arg = OutVals[OutIndex];
2338 bool isByVal = Flags.isByVal();
2339
2340 // Promote the value if needed.
2341 switch (VA.getLocInfo()) {
2342 default: llvm_unreachable("Unknown loc info!");
2343 case CCValAssign::Full: break;
2344 case CCValAssign::SExt:
2345 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: RegVT, Operand: Arg);
2346 break;
2347 case CCValAssign::ZExt:
2348 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: RegVT, Operand: Arg);
2349 break;
2350 case CCValAssign::AExt:
2351 if (Arg.getValueType().isVector() &&
2352 Arg.getValueType().getVectorElementType() == MVT::i1)
2353 Arg = lowerMasksToReg(ValArg: Arg, ValLoc: RegVT, DL: dl, DAG);
2354 else if (RegVT.is128BitVector()) {
2355 // Special case: passing MMX values in XMM registers.
2356 Arg = DAG.getBitcast(VT: MVT::i64, V: Arg);
2357 Arg = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: MVT::v2i64, Operand: Arg);
2358 Arg = getMOVL(DAG, dl, VT: MVT::v2i64, V1: DAG.getUNDEF(VT: MVT::v2i64), V2: Arg);
2359 } else
2360 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: RegVT, Operand: Arg);
2361 break;
2362 case CCValAssign::BCvt:
2363 Arg = DAG.getBitcast(VT: RegVT, V: Arg);
2364 break;
2365 case CCValAssign::Indirect: {
2366 if (isByVal) {
2367 // Memcpy the argument to a temporary stack slot to prevent
2368 // the caller from seeing any modifications the callee may make
2369 // as guaranteed by the `byval` attribute.
2370 int FrameIdx = MF.getFrameInfo().CreateStackObject(
2371 Size: Flags.getByValSize(),
2372 Alignment: std::max(a: Align(16), b: Flags.getNonZeroByValAlign()), isSpillSlot: false);
2373 SDValue StackSlot =
2374 DAG.getFrameIndex(FI: FrameIdx, VT: getPointerTy(DL: DAG.getDataLayout()));
2375 Chain =
2376 CreateCopyOfByValArgument(Src: Arg, Dst: StackSlot, Chain, Flags, DAG, dl);
2377 // From now on treat this as a regular pointer
2378 Arg = StackSlot;
2379 isByVal = false;
2380 } else {
2381 // Store the argument.
2382 SDValue SpillSlot = DAG.CreateStackTemporary(VT: VA.getValVT());
2383 int FI = cast<FrameIndexSDNode>(Val&: SpillSlot)->getIndex();
2384 Chain = DAG.getStore(
2385 Chain, dl, Val: Arg, Ptr: SpillSlot,
2386 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI));
2387 Arg = SpillSlot;
2388 }
2389 break;
2390 }
2391 }
2392
2393 if (VA.needsCustom()) {
2394 assert(VA.getValVT() == MVT::v64i1 &&
2395 "Currently the only custom case is when we split v64i1 to 2 regs");
2396 // Split v64i1 value into two registers
2397 Passv64i1ArgInRegs(DL: dl, DAG, Arg, RegsToPass, VA, NextVA&: ArgLocs[++I], Subtarget);
2398 } else if (VA.isRegLoc()) {
2399 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
2400 const TargetOptions &Options = DAG.getTarget().Options;
2401 if (Options.EmitCallSiteInfo)
2402 CSInfo.ArgRegPairs.emplace_back(Args: VA.getLocReg(), Args&: I);
2403 if (isVarArg && IsWin64) {
2404 // Win64 ABI requires argument XMM reg to be copied to the corresponding
2405 // shadow reg if callee is a varargs function.
2406 Register ShadowReg;
2407 switch (VA.getLocReg()) {
2408 case X86::XMM0: ShadowReg = X86::RCX; break;
2409 case X86::XMM1: ShadowReg = X86::RDX; break;
2410 case X86::XMM2: ShadowReg = X86::R8; break;
2411 case X86::XMM3: ShadowReg = X86::R9; break;
2412 }
2413 if (ShadowReg)
2414 RegsToPass.push_back(Elt: std::make_pair(x&: ShadowReg, y&: Arg));
2415 }
2416 } else if (!IsSibcall && (!isTailCall || (isByVal && !IsMustTail))) {
2417 assert(VA.isMemLoc());
2418 if (!StackPtr.getNode())
2419 StackPtr = DAG.getCopyFromReg(Chain, dl, Reg: RegInfo->getStackRegister(),
2420 VT: getPointerTy(DL: DAG.getDataLayout()));
2421 MemOpChains.push_back(Elt: LowerMemOpCallTo(Chain, StackPtr, Arg,
2422 dl, DAG, VA, Flags, isByVal));
2423 }
2424 }
2425
2426 if (!MemOpChains.empty())
2427 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
2428
2429 if (Subtarget.isPICStyleGOT()) {
2430 // ELF / PIC requires GOT in the EBX register before function calls via PLT
2431 // GOT pointer.
2432 if (!isTailCall) {
2433 // Only PLT calls (GlobalAddress or ExternalSymbol) require the GOT in
2434 // EBX. Indirect calls through a register or an absolute address do not
2435 // go through the PLT and do not need EBX to hold the GOT base.
2436 if ((Callee->getOpcode() == ISD::GlobalAddress ||
2437 Callee->getOpcode() == ISD::ExternalSymbol))
2438 RegsToPass.push_back(Elt: std::make_pair(
2439 x: Register(X86::EBX), y: DAG.getNode(Opcode: X86ISD::GlobalBaseReg, DL: SDLoc(),
2440 VT: getPointerTy(DL: DAG.getDataLayout()))));
2441 } else {
2442 // If we are tail calling and generating PIC/GOT style code load the
2443 // address of the callee into ECX. The value in ecx is used as target of
2444 // the tail jump. This is done to circumvent the ebx/callee-saved problem
2445 // for tail calls on PIC/GOT architectures. Normally we would just put the
2446 // address of GOT into ebx and then call target@PLT. But for tail calls
2447 // ebx would be restored (since ebx is callee saved) before jumping to the
2448 // target@PLT.
2449
2450 // Note: The actual moving to ECX is done further down.
2451 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee);
2452 if (G && !G->getGlobal()->hasLocalLinkage() &&
2453 G->getGlobal()->hasDefaultVisibility())
2454 Callee = LowerGlobalAddress(Op: Callee, DAG);
2455 else if (isa<ExternalSymbolSDNode>(Val: Callee))
2456 Callee = LowerExternalSymbol(Op: Callee, DAG);
2457 }
2458 }
2459
2460 if (Is64Bit && isVarArg && !IsWin64 && !IsMustTail &&
2461 (Subtarget.hasSSE1() || !M->getModuleFlag(Key: "SkipRaxSetup"))) {
2462 // From AMD64 ABI document:
2463 // For calls that may call functions that use varargs or stdargs
2464 // (prototype-less calls or calls to functions containing ellipsis (...) in
2465 // the declaration) %al is used as hidden argument to specify the number
2466 // of SSE registers used. The contents of %al do not need to match exactly
2467 // the number of registers, but must be an ubound on the number of SSE
2468 // registers used and is in the range 0 - 8 inclusive.
2469
2470 // Count the number of XMM registers allocated.
2471 static const MCPhysReg XMMArgRegs[] = {
2472 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
2473 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
2474 };
2475 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(Regs: XMMArgRegs);
2476 assert((Subtarget.hasSSE1() || !NumXMMRegs)
2477 && "SSE registers cannot be used when SSE is disabled");
2478 RegsToPass.push_back(Elt: std::make_pair(x: Register(X86::AL),
2479 y: DAG.getConstant(Val: NumXMMRegs, DL: dl,
2480 VT: MVT::i8)));
2481 }
2482
2483 if (isVarArg && IsMustTail) {
2484 const auto &Forwards = X86Info->getForwardedMustTailRegParms();
2485 for (const auto &F : Forwards) {
2486 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: F.VReg, VT: F.VT);
2487 RegsToPass.push_back(Elt: std::make_pair(x: F.PReg, y&: Val));
2488 }
2489 }
2490
2491 // For tail calls lower the arguments to the 'real' stack slots. Sibcalls
2492 // don't need this because the eligibility check rejects calls that require
2493 // shuffling arguments passed in memory.
2494 if (isTailCall && !IsSibcall) {
2495 // Force all the incoming stack arguments to be loaded from the stack
2496 // before any new outgoing arguments or the return address are stored to the
2497 // stack, because the outgoing stack slots may alias the incoming argument
2498 // stack slots, and the alias isn't otherwise explicit. This is slightly
2499 // more conservative than necessary, because it means that each store
2500 // effectively depends on every argument instead of just those arguments it
2501 // would clobber.
2502 Chain = DAG.getStackArgumentTokenFactor(Chain);
2503
2504 if (ByValTempChain)
2505 Chain =
2506 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Chain, N2: ByValTempChain);
2507
2508 SmallVector<SDValue, 8> MemOpChains2;
2509 SDValue FIN;
2510 int FI = 0;
2511 for (unsigned I = 0, OutsIndex = 0, E = ArgLocs.size(); I != E;
2512 ++I, ++OutsIndex) {
2513 CCValAssign &VA = ArgLocs[I];
2514
2515 if (VA.isRegLoc()) {
2516 if (VA.needsCustom()) {
2517 assert((CallConv == CallingConv::X86_RegCall) &&
2518 "Expecting custom case only in regcall calling convention");
2519 // This means that we are in special case where one argument was
2520 // passed through two register locations - Skip the next location
2521 ++I;
2522 }
2523
2524 continue;
2525 }
2526
2527 assert(VA.isMemLoc());
2528 SDValue Arg = OutVals[OutsIndex];
2529 ISD::ArgFlagsTy Flags = Outs[OutsIndex].Flags;
2530 // Skip inalloca/preallocated arguments. They don't require any work.
2531 if (Flags.isInAlloca() || Flags.isPreallocated())
2532 continue;
2533 // Create frame index.
2534 int32_t Offset = VA.getLocMemOffset()+FPDiff;
2535 uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8;
2536 FI = MF.getFrameInfo().CreateFixedObject(Size: OpSize, SPOffset: Offset, IsImmutable: true);
2537 FIN = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
2538
2539 if (Flags.isByVal()) {
2540 if (SDValue ByValSrc = ByValTemporaries[OutsIndex]) {
2541 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
2542 SDValue DstAddr = DAG.getFrameIndex(FI, VT: PtrVT);
2543
2544 MemOpChains2.push_back(Elt: CreateCopyOfByValArgument(
2545 Src: ByValSrc, Dst: DstAddr, Chain, Flags, DAG, dl));
2546 }
2547 } else {
2548 // Store relative to framepointer.
2549 MemOpChains2.push_back(Elt: DAG.getStore(
2550 Chain, dl, Val: Arg, Ptr: FIN,
2551 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI)));
2552 }
2553 }
2554
2555 if (!MemOpChains2.empty())
2556 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains2);
2557
2558 // Store the return address to the appropriate stack slot.
2559 Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx,
2560 PtrVT: getPointerTy(DL: DAG.getDataLayout()),
2561 SlotSize: RegInfo->getSlotSize(), FPDiff, dl);
2562 }
2563
2564 // Build a sequence of copy-to-reg nodes chained together with token chain
2565 // and glue operands which copy the outgoing args into registers.
2566 SDValue InGlue;
2567 for (const auto &[Reg, N] : RegsToPass) {
2568 Chain = DAG.getCopyToReg(Chain, dl, Reg, N, Glue: InGlue);
2569 InGlue = Chain.getValue(R: 1);
2570 }
2571
2572 bool IsImpCall = false;
2573 bool IsCFGuardCall = false;
2574 if (DAG.getTarget().getCodeModel() == CodeModel::Large) {
2575 assert(Is64Bit && "Large code model is only legal in 64-bit mode.");
2576 // In the 64-bit large code model, we have to make all calls
2577 // through a register, since the call instruction's 32-bit
2578 // pc-relative offset may not be large enough to hold the whole
2579 // address.
2580 } else if (Callee->getOpcode() == ISD::GlobalAddress ||
2581 Callee->getOpcode() == ISD::ExternalSymbol) {
2582 // Lower direct calls to global addresses and external symbols. Setting
2583 // ForCall to true here has the effect of removing WrapperRIP when possible
2584 // to allow direct calls to be selected without first materializing the
2585 // address into a register.
2586 Callee = LowerGlobalOrExternal(Op: Callee, DAG, /*ForCall=*/true, IsImpCall: &IsImpCall);
2587 } else if (Subtarget.isTarget64BitILP32() &&
2588 Callee.getValueType() == MVT::i32) {
2589 // Zero-extend the 32-bit Callee address into a 64-bit according to x32 ABI
2590 Callee = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i64, Operand: Callee);
2591 } else if (Is64Bit && CB && isCFGuardCall(CB)) {
2592 // We'll use a specific psuedo instruction for tail calls to control flow
2593 // guard functions to guarantee the instruction used for the call. To do
2594 // this we need to unwrap the load now and use the CFG Func GV as the
2595 // callee.
2596 IsCFGuardCall = true;
2597 auto *LoadNode = cast<LoadSDNode>(Val&: Callee);
2598 GlobalAddressSDNode *GA =
2599 cast<GlobalAddressSDNode>(Val: unwrapAddress(N: LoadNode->getBasePtr()));
2600 assert(isCFGuardFunction(GA->getGlobal()) &&
2601 "CFG Call should be to a guard function");
2602 assert(LoadNode->getOffset()->isUndef() &&
2603 "CFG Function load should not have an offset");
2604 Callee = DAG.getTargetGlobalAddress(
2605 GV: GA->getGlobal(), DL: dl, VT: GA->getValueType(ResNo: 0), offset: 0, TargetFlags: X86II::MO_NO_FLAG);
2606 }
2607
2608 if (Is64Bit && !Subtarget.isTarget64BitILP32() &&
2609 Callee.getValueType() == MVT::i32) {
2610 // On 64-bit targets, extend 32-bit MS __ptr32 callees before the call.
2611 // Use sign-extension for __sptr and zero-extension for __uptr.
2612 assert(CB && "CallBase expected for 32-bit callee");
2613 ISD::NodeType ExtOpc = ISD::SIGN_EXTEND;
2614 unsigned AS = CB->getCalledOperand()->getType()->getPointerAddressSpace();
2615 if (AS == X86AS::PTR32_UPTR)
2616 ExtOpc = ISD::ZERO_EXTEND;
2617 Callee = DAG.getNode(Opcode: ExtOpc, DL: dl, VT: MVT::i64, Operand: Callee);
2618 }
2619
2620 SmallVector<SDValue, 8> Ops;
2621
2622 if (!IsSibcall && isTailCall && !IsMustTail) {
2623 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytesToPop, Size2: 0, Glue: InGlue, DL: dl);
2624 InGlue = Chain.getValue(R: 1);
2625 }
2626
2627 Ops.push_back(Elt: Chain);
2628 Ops.push_back(Elt: Callee);
2629
2630 if (isTailCall)
2631 Ops.push_back(Elt: DAG.getSignedTargetConstant(Val: FPDiff, DL: dl, VT: MVT::i32));
2632
2633 // Add argument registers to the end of the list so that they are known live
2634 // into the call.
2635 for (const auto &[Reg, N] : RegsToPass)
2636 Ops.push_back(Elt: DAG.getRegister(Reg, VT: N.getValueType()));
2637
2638 // Add a register mask operand representing the call-preserved registers.
2639 const uint32_t *Mask = [&]() {
2640 auto AdaptedCC = CallConv;
2641 // If HasNCSR is asserted (attribute NoCallerSavedRegisters exists),
2642 // use X86_INTR calling convention because it has the same CSR mask
2643 // (same preserved registers).
2644 if (HasNCSR)
2645 AdaptedCC = (CallingConv::ID)CallingConv::X86_INTR;
2646 // If NoCalleeSavedRegisters is requested, than use GHC since it happens
2647 // to use the CSR_NoRegs_RegMask.
2648 if (CB && CB->hasFnAttr(Kind: "no_callee_saved_registers"))
2649 AdaptedCC = (CallingConv::ID)CallingConv::GHC;
2650 return RegInfo->getCallPreservedMask(MF, AdaptedCC);
2651 }();
2652 assert(Mask && "Missing call preserved mask for calling convention");
2653
2654 if (MachineOperand::clobbersPhysReg(RegMask: Mask, PhysReg: RegInfo->getFramePtr())) {
2655 X86Info->setFPClobberedByCall(true);
2656 if (CLI.CB && isa<InvokeInst>(Val: CLI.CB))
2657 X86Info->setFPClobberedByInvoke(true);
2658 }
2659 if (MachineOperand::clobbersPhysReg(RegMask: Mask, PhysReg: RegInfo->getBaseRegister())) {
2660 X86Info->setBPClobberedByCall(true);
2661 if (CLI.CB && isa<InvokeInst>(Val: CLI.CB))
2662 X86Info->setBPClobberedByInvoke(true);
2663 }
2664
2665 // If this is an invoke in a 32-bit function using a funclet-based
2666 // personality, assume the function clobbers all registers. If an exception
2667 // is thrown, the runtime will not restore CSRs.
2668 // FIXME: Model this more precisely so that we can register allocate across
2669 // the normal edge and spill and fill across the exceptional edge.
2670 if (!Is64Bit && CLI.CB && isa<InvokeInst>(Val: CLI.CB)) {
2671 const Function &CallerFn = MF.getFunction();
2672 EHPersonality Pers =
2673 CallerFn.hasPersonalityFn()
2674 ? classifyEHPersonality(Pers: CallerFn.getPersonalityFn())
2675 : EHPersonality::Unknown;
2676 if (isFuncletEHPersonality(Pers))
2677 Mask = RegInfo->getNoPreservedMask();
2678 }
2679
2680 // Define a new register mask from the existing mask.
2681 uint32_t *RegMask = nullptr;
2682
2683 // In some calling conventions we need to remove the used physical registers
2684 // from the reg mask. Create a new RegMask for such calling conventions.
2685 // RegMask for calling conventions that disable only return registers (e.g.
2686 // preserve_most) will be modified later in LowerCallResult.
2687 bool ShouldDisableArgRegs = shouldDisableArgRegFromCSR(CC: CallConv) || HasNCSR;
2688 if (ShouldDisableArgRegs || shouldDisableRetRegFromCSR(CC: CallConv)) {
2689 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2690
2691 // Allocate a new Reg Mask and copy Mask.
2692 RegMask = MF.allocateRegMask();
2693 unsigned RegMaskSize = MachineOperand::getRegMaskSize(NumRegs: TRI->getNumRegs());
2694 memcpy(dest: RegMask, src: Mask, n: sizeof(RegMask[0]) * RegMaskSize);
2695
2696 // Make sure all sub registers of the argument registers are reset
2697 // in the RegMask.
2698 if (ShouldDisableArgRegs) {
2699 for (auto const &RegPair : RegsToPass)
2700 for (MCPhysReg SubReg : TRI->subregs_inclusive(Reg: RegPair.first))
2701 RegMask[SubReg / 32] &= ~(1u << (SubReg % 32));
2702 }
2703
2704 // Create the RegMask Operand according to our updated mask.
2705 Ops.push_back(Elt: DAG.getRegisterMask(RegMask));
2706 } else {
2707 // Create the RegMask Operand according to the static mask.
2708 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
2709 }
2710
2711 if (InGlue.getNode())
2712 Ops.push_back(Elt: InGlue);
2713
2714 if (isTailCall) {
2715 // We used to do:
2716 //// If this is the first return lowered for this function, add the regs
2717 //// to the liveout set for the function.
2718 // This isn't right, although it's probably harmless on x86; liveouts
2719 // should be computed from returns not tail calls. Consider a void
2720 // function making a tail call to a function returning int.
2721 MF.getFrameInfo().setHasTailCall();
2722 auto Opcode =
2723 IsCFGuardCall ? X86ISD::TC_RETURN_GLOBALADDR : X86ISD::TC_RETURN;
2724 SDValue Ret = DAG.getNode(Opcode, DL: dl, VT: MVT::Other, Ops);
2725
2726 if (IsCFICall)
2727 Ret.getNode()->setCFIType(CLI.CFIType->getZExtValue());
2728
2729 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
2730 DAG.addCallSiteInfo(Node: Ret.getNode(), CallInfo: std::move(CSInfo));
2731 return Ret;
2732 }
2733
2734 // Returns a chain & a glue for retval copy to use.
2735 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
2736 if (IsImpCall) {
2737 Chain = DAG.getNode(Opcode: X86ISD::IMP_CALL, DL: dl, VTList: NodeTys, Ops);
2738 } else if (IsNoTrackIndirectCall) {
2739 Chain = DAG.getNode(Opcode: X86ISD::NT_CALL, DL: dl, VTList: NodeTys, Ops);
2740 } else if (IsCFGuardCall) {
2741 Chain = DAG.getNode(Opcode: X86ISD::CALL_GLOBALADDR, DL: dl, VTList: NodeTys, Ops);
2742 } else if (CLI.CB && objcarc::hasAttachedCallOpBundle(CB: CLI.CB)) {
2743 // Calls with a "clang.arc.attachedcall" bundle are special. They should be
2744 // expanded to the call, directly followed by a special marker sequence and
2745 // a call to a ObjC library function. Use the CALL_RVMARKER to do that.
2746 assert(!isTailCall &&
2747 "tail calls cannot be marked with clang.arc.attachedcall");
2748 assert(Is64Bit && "clang.arc.attachedcall is only supported in 64bit mode");
2749
2750 // Add a target global address for the retainRV/claimRV runtime function
2751 // just before the call target.
2752 Function *ARCFn = *objcarc::getAttachedARCFunction(CB: CLI.CB);
2753 auto PtrVT = getPointerTy(DL: DAG.getDataLayout());
2754 auto GA = DAG.getTargetGlobalAddress(GV: ARCFn, DL: dl, VT: PtrVT);
2755 Ops.insert(I: Ops.begin() + 1, Elt: GA);
2756 Chain = DAG.getNode(Opcode: X86ISD::CALL_RVMARKER, DL: dl, VTList: NodeTys, Ops);
2757 } else {
2758 Chain = DAG.getNode(Opcode: X86ISD::CALL, DL: dl, VTList: NodeTys, Ops);
2759 }
2760
2761 if (IsCFICall)
2762 Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
2763
2764 InGlue = Chain.getValue(R: 1);
2765 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
2766 DAG.addCallSiteInfo(Node: Chain.getNode(), CallInfo: std::move(CSInfo));
2767
2768 // Save heapallocsite metadata.
2769 if (CLI.CB)
2770 if (MDNode *HeapAlloc = CLI.CB->getMetadata(Kind: "heapallocsite"))
2771 DAG.addHeapAllocSite(Node: Chain.getNode(), MD: HeapAlloc);
2772
2773 // Create the CALLSEQ_END node.
2774 unsigned NumBytesForCalleeToPop = 0; // Callee pops nothing.
2775 if (X86::isCalleePop(CallingConv: CallConv, is64Bit: Is64Bit, IsVarArg: isVarArg,
2776 GuaranteeTCO: DAG.getTarget().Options.GuaranteedTailCallOpt)) {
2777 NumBytesForCalleeToPop = NumBytes; // Callee pops everything
2778 } else if (hasCalleePopSRet(Args: Outs, ArgLocs, Subtarget)) {
2779 // If this call passes a struct-return pointer, the callee
2780 // pops that struct pointer.
2781 NumBytesForCalleeToPop = 4;
2782 }
2783
2784 // Returns a glue for retval copy to use.
2785 if (!IsSibcall) {
2786 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytesToPop, Size2: NumBytesForCalleeToPop,
2787 Glue: InGlue, DL: dl);
2788 InGlue = Chain.getValue(R: 1);
2789 }
2790
2791 if (CallingConv::PreserveNone == CallConv)
2792 for (const ISD::OutputArg &Out : Outs) {
2793 if (Out.Flags.isSwiftSelf() || Out.Flags.isSwiftAsync() ||
2794 Out.Flags.isSwiftError()) {
2795 errorUnsupported(DAG, dl,
2796 Msg: "Swift attributes can't be used with preserve_none");
2797 break;
2798 }
2799 }
2800
2801 // Handle result values, copying them out of physregs into vregs that we
2802 // return.
2803 return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
2804 InVals, RegMask);
2805}
2806
2807//===----------------------------------------------------------------------===//
2808// Fast Calling Convention (tail call) implementation
2809//===----------------------------------------------------------------------===//
2810
2811// Like std call, callee cleans arguments, convention except that ECX is
2812// reserved for storing the tail called function address. Only 2 registers are
2813// free for argument passing (inreg). Tail call optimization is performed
2814// provided:
2815// * tailcallopt is enabled
2816// * caller/callee are fastcc
2817// On X86_64 architecture with GOT-style position independent code only local
2818// (within module) calls are supported at the moment.
2819// To keep the stack aligned according to platform abi the function
2820// GetAlignedArgumentStackSize ensures that argument delta is always multiples
2821// of stack alignment. (Dynamic linkers need this - Darwin's dyld for example)
2822// If a tail called function callee has more arguments than the caller the
2823// caller needs to make sure that there is room to move the RETADDR to. This is
2824// achieved by reserving an area the size of the argument delta right after the
2825// original RETADDR, but before the saved framepointer or the spilled registers
2826// e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4)
2827// stack layout:
2828// arg1
2829// arg2
2830// RETADDR
2831// [ new RETADDR
2832// move area ]
2833// (possible EBP)
2834// ESI
2835// EDI
2836// local1 ..
2837
2838/// Make the stack size align e.g 16n + 12 aligned for a 16-byte align
2839/// requirement.
2840unsigned
2841X86TargetLowering::GetAlignedArgumentStackSize(const unsigned StackSize,
2842 SelectionDAG &DAG) const {
2843 const Align StackAlignment = Subtarget.getFrameLowering()->getStackAlign();
2844 const uint64_t SlotSize = Subtarget.getRegisterInfo()->getSlotSize();
2845 assert(StackSize % SlotSize == 0 &&
2846 "StackSize must be a multiple of SlotSize");
2847 return alignTo(Size: StackSize + SlotSize, A: StackAlignment) - SlotSize;
2848}
2849
2850/// Return true if the given stack call argument is already available in the
2851/// same position (relatively) of the caller's incoming argument stack.
2852static
2853bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
2854 MachineFrameInfo &MFI, const MachineRegisterInfo *MRI,
2855 const X86InstrInfo *TII, const CCValAssign &VA) {
2856 unsigned Bytes = Arg.getValueSizeInBits() / 8;
2857
2858 for (;;) {
2859 // Look through nodes that don't alter the bits of the incoming value.
2860 unsigned Op = Arg.getOpcode();
2861 if (Op == ISD::ZERO_EXTEND || Op == ISD::ANY_EXTEND || Op == ISD::BITCAST ||
2862 Op == ISD::AssertZext) {
2863 Arg = Arg.getOperand(i: 0);
2864 continue;
2865 }
2866 if (Op == ISD::TRUNCATE) {
2867 const SDValue &TruncInput = Arg.getOperand(i: 0);
2868 if (TruncInput.getOpcode() == ISD::AssertZext &&
2869 cast<VTSDNode>(Val: TruncInput.getOperand(i: 1))->getVT() ==
2870 Arg.getValueType()) {
2871 Arg = TruncInput.getOperand(i: 0);
2872 continue;
2873 }
2874 }
2875 break;
2876 }
2877
2878 int FI = INT_MAX;
2879 if (Arg.getOpcode() == ISD::CopyFromReg) {
2880 Register VR = cast<RegisterSDNode>(Val: Arg.getOperand(i: 1))->getReg();
2881 if (!VR.isVirtual())
2882 return false;
2883 MachineInstr *Def = MRI->getVRegDef(Reg: VR);
2884 if (!Def)
2885 return false;
2886 if (!Flags.isByVal()) {
2887 if (!TII->isLoadFromStackSlot(MI: *Def, FrameIndex&: FI))
2888 return false;
2889 } else {
2890 unsigned Opcode = Def->getOpcode();
2891 if ((Opcode == X86::LEA32r || Opcode == X86::LEA64r ||
2892 Opcode == X86::LEA64_32r) &&
2893 Def->getOperand(i: 1).isFI()) {
2894 FI = Def->getOperand(i: 1).getIndex();
2895 Bytes = Flags.getByValSize();
2896 } else
2897 return false;
2898 }
2899 } else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val&: Arg)) {
2900 if (Flags.isByVal())
2901 // ByVal argument is passed in as a pointer but it's now being
2902 // dereferenced. e.g.
2903 // define @foo(%struct.X* %A) {
2904 // tail call @bar(%struct.X* byval %A)
2905 // }
2906 return false;
2907 SDValue Ptr = Ld->getBasePtr();
2908 FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Val&: Ptr);
2909 if (!FINode)
2910 return false;
2911 FI = FINode->getIndex();
2912 } else if (Arg.getOpcode() == ISD::FrameIndex && Flags.isByVal()) {
2913 FrameIndexSDNode *FINode = cast<FrameIndexSDNode>(Val&: Arg);
2914 FI = FINode->getIndex();
2915 Bytes = Flags.getByValSize();
2916 } else
2917 return false;
2918
2919 assert(FI != INT_MAX);
2920 if (!MFI.isFixedObjectIndex(ObjectIdx: FI))
2921 return false;
2922
2923 if (Offset != MFI.getObjectOffset(ObjectIdx: FI))
2924 return false;
2925
2926 // If this is not byval, check that the argument stack object is immutable.
2927 // inalloca and argument copy elision can create mutable argument stack
2928 // objects. Byval objects can be mutated, but a byval call intends to pass the
2929 // mutated memory.
2930 if (!Flags.isByVal() && !MFI.isImmutableObjectIndex(ObjectIdx: FI))
2931 return false;
2932
2933 if (VA.getLocVT().getFixedSizeInBits() >
2934 Arg.getValueSizeInBits().getFixedValue()) {
2935 // If the argument location is wider than the argument type, check that any
2936 // extension flags match.
2937 if (Flags.isZExt() != MFI.isObjectZExt(ObjectIdx: FI) ||
2938 Flags.isSExt() != MFI.isObjectSExt(ObjectIdx: FI)) {
2939 return false;
2940 }
2941 }
2942
2943 return Bytes == MFI.getObjectSize(ObjectIdx: FI);
2944}
2945
2946/// Return true if the call forwards the caller's own sret pointer to the callee
2947/// as its sret argument. We prove this in IR, since SelectionDAG is built per
2948/// basic block and the caller's sret pointer is only visible as a function
2949/// argument in the entry block.
2950static bool isCallerSRetForwarded(const CallBase *CB, const Function &CallerF) {
2951 if (!CB)
2952 return false;
2953
2954 // The verifier allows at most one sret parameter. A caller with a sret return
2955 // register but no sret Argument has an implicit sret from a demoted return,
2956 // which cannot be forwarded from IR.
2957 const Argument *CallerSRet =
2958 llvm::find_if(Range: CallerF.args(),
2959 P: [](const Argument &Arg) { return Arg.hasStructRetAttr(); });
2960 if (CallerSRet == CallerF.arg_end())
2961 return false;
2962
2963 // Check that the callee receives it as its sret argument.
2964 for (const Use &U : CB->args())
2965 if (U.get() == CallerSRet &&
2966 CB->paramHasAttr(ArgNo: CB->getArgOperandNo(U: &U), Kind: Attribute::StructRet))
2967 return true;
2968 return false;
2969}
2970
2971/// Check whether the call is eligible for sibling call optimization. Sibling
2972/// calls are loosely defined to be simple, profitable tail calls that only
2973/// require adjusting register parameters. We do not speculatively to optimize
2974/// complex calls that require lots of argument memory operations that may
2975/// alias.
2976///
2977/// Note that LLVM supports multiple ways, such as musttail, to force tail call
2978/// emission. Returning false from this function will not prevent tail call
2979/// emission in all cases.
2980bool X86TargetLowering::isEligibleForSiblingCallOpt(
2981 TargetLowering::CallLoweringInfo &CLI, CCState &CCInfo,
2982 SmallVectorImpl<CCValAssign> &ArgLocs) const {
2983 SelectionDAG &DAG = CLI.DAG;
2984 const SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2985 const SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2986 const SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2987 SDValue Callee = CLI.Callee;
2988 CallingConv::ID CalleeCC = CLI.CallConv;
2989 bool isVarArg = CLI.IsVarArg;
2990
2991 if (!mayTailCallThisCC(CC: CalleeCC))
2992 return false;
2993
2994 // If -tailcallopt is specified, make fastcc functions tail-callable.
2995 MachineFunction &MF = DAG.getMachineFunction();
2996 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
2997 const Function &CallerF = MF.getFunction();
2998
2999 // If the function return type is x86_fp80 and the callee return type is not,
3000 // then the FP_EXTEND of the call result is not a nop. It's not safe to
3001 // perform a tailcall optimization here.
3002 if (CallerF.getReturnType()->isX86_FP80Ty() && !CLI.RetTy->isX86_FP80Ty())
3003 return false;
3004
3005 // Win64 functions have extra shadow space for argument homing. Don't do the
3006 // sibcall if the caller and callee have mismatched expectations for this
3007 // space.
3008 CallingConv::ID CallerCC = CallerF.getCallingConv();
3009 bool IsCalleeWin64 = Subtarget.isCallingConvWin64(CC: CalleeCC);
3010 bool IsCallerWin64 = Subtarget.isCallingConvWin64(CC: CallerCC);
3011 if (IsCalleeWin64 != IsCallerWin64)
3012 return false;
3013
3014 // Do not optimize vararg calls with 6 arguments for LFI since LFI reserves
3015 // %r11, meaning there will not be enough registers available.
3016 if (Subtarget.isLFI() && ArgLocs.size() > 5)
3017 return false;
3018
3019 // If we are using a GOT, don't generate sibling calls to non-local,
3020 // default-visibility symbols. Tail calling such a symbol requires using a GOT
3021 // relocation, which forces early binding of the symbol. This breaks code that
3022 // require lazy function symbol resolution. Using musttail or
3023 // GuaranteedTailCallOpt will override this.
3024 if (Subtarget.isPICStyleGOT()) {
3025 if (isa<ExternalSymbolSDNode>(Val: Callee))
3026 return false;
3027 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
3028 if (!G->getGlobal()->hasLocalLinkage() &&
3029 G->getGlobal()->hasDefaultVisibility())
3030 return false;
3031 }
3032 }
3033
3034 // Look for obvious safe cases to perform tail call optimization that do not
3035 // require ABI changes. This is what gcc calls sibcall.
3036
3037 // Can't do sibcall if stack needs to be dynamically re-aligned. PEI needs to
3038 // emit a special epilogue.
3039 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
3040 if (RegInfo->hasStackRealignment(MF))
3041 return false;
3042
3043 // Avoid sibcall optimization if we are an sret return function and the callee
3044 // is incompatible, unless such premises are proven wrong. See comment in
3045 // LowerReturn about why hasStructRetAttr is insufficient.
3046 if (FuncInfo->getSRetReturnReg()) {
3047 // For a compatible tail call the callee must return our sret pointer. So it
3048 // needs to be (a) an sret function itself and (b) we pass our sret as its
3049 // sret.
3050 if (!isCallerSRetForwarded(CB: CLI.CB, CallerF))
3051 return false;
3052 } else if (hasCalleePopSRet(Args: Outs, ArgLocs, Subtarget))
3053 // The callee pops an sret, so we cannot tail-call, as our caller doesn't
3054 // expect that.
3055 return false;
3056
3057 // Do not sibcall optimize vararg calls unless all arguments are passed via
3058 // registers.
3059 LLVMContext &C = *DAG.getContext();
3060 if (isVarArg && !Outs.empty()) {
3061 // Optimizing for varargs on Win64 is unlikely to be safe without
3062 // additional testing.
3063 if (IsCalleeWin64 || IsCallerWin64)
3064 return false;
3065
3066 for (const auto &VA : ArgLocs)
3067 if (!VA.isRegLoc())
3068 return false;
3069 }
3070
3071 // If the call result is in ST0 / ST1, it needs to be popped off the x87
3072 // stack. Therefore, if it's not used by the call it is not safe to optimize
3073 // this into a sibcall.
3074 bool Unused = false;
3075 for (const auto &In : Ins) {
3076 if (!In.Used) {
3077 Unused = true;
3078 break;
3079 }
3080 }
3081 if (Unused) {
3082 SmallVector<CCValAssign, 16> RVLocs;
3083 CCState RVCCInfo(CalleeCC, false, MF, RVLocs, C);
3084 RVCCInfo.AnalyzeCallResult(Ins, Fn: RetCC_X86);
3085 for (const auto &VA : RVLocs) {
3086 if (VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1)
3087 return false;
3088 }
3089 }
3090
3091 // Check that the call results are passed in the same way.
3092 if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C, Ins,
3093 CalleeFn: RetCC_X86, CallerFn: RetCC_X86))
3094 return false;
3095 // The callee has to preserve all registers the caller needs to preserve.
3096 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
3097 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
3098 if (CallerCC != CalleeCC) {
3099 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
3100 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved))
3101 return false;
3102 }
3103
3104 // The stack frame of the caller cannot be replaced by the tail-callee one's
3105 // if the function is required to preserve all the registers. Conservatively
3106 // prevent tail optimization even if hypothetically all the registers are used
3107 // for passing formal parameters or returning values.
3108 if (CallerF.hasFnAttribute(Kind: "no_caller_saved_registers"))
3109 return false;
3110
3111 unsigned StackArgsSize = CCInfo.getStackSize();
3112
3113 // If the callee takes no arguments then go on to check the results of the
3114 // call.
3115 if (!Outs.empty()) {
3116 if (StackArgsSize > 0) {
3117 // Check if the arguments are already laid out in the right way as
3118 // the caller's fixed stack objects.
3119 MachineFrameInfo &MFI = MF.getFrameInfo();
3120 const MachineRegisterInfo *MRI = &MF.getRegInfo();
3121 const X86InstrInfo *TII = Subtarget.getInstrInfo();
3122 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
3123 const CCValAssign &VA = ArgLocs[I];
3124 SDValue Arg = OutVals[I];
3125 ISD::ArgFlagsTy Flags = Outs[I].Flags;
3126 if (VA.getLocInfo() == CCValAssign::Indirect)
3127 return false;
3128 if (!VA.isRegLoc()) {
3129 if (!MatchingStackOffset(Arg, Offset: VA.getLocMemOffset(), Flags, MFI, MRI,
3130 TII, VA))
3131 return false;
3132 }
3133 }
3134 }
3135
3136 bool PositionIndependent = isPositionIndependent();
3137 // If the tailcall address may be in a register, then make sure it's
3138 // possible to register allocate for it. In 32-bit, the call address can
3139 // only target EAX, EDX, or ECX since the tail call must be scheduled after
3140 // callee-saved registers are restored. These happen to be the same
3141 // registers used to pass 'inreg' arguments so watch out for those.
3142 if (!Subtarget.is64Bit() && ((!isa<GlobalAddressSDNode>(Val: Callee) &&
3143 !isa<ExternalSymbolSDNode>(Val: Callee)) ||
3144 PositionIndependent)) {
3145 unsigned NumInRegs = 0;
3146 // In PIC we need an extra register to formulate the address computation
3147 // for the callee.
3148 unsigned MaxInRegs = PositionIndependent ? 2 : 3;
3149
3150 for (const auto &VA : ArgLocs) {
3151 if (!VA.isRegLoc())
3152 continue;
3153 Register Reg = VA.getLocReg();
3154 switch (Reg) {
3155 default: break;
3156 case X86::EAX: case X86::EDX: case X86::ECX:
3157 if (++NumInRegs == MaxInRegs)
3158 return false;
3159 break;
3160 }
3161 }
3162 }
3163
3164 const MachineRegisterInfo &MRI = MF.getRegInfo();
3165 if (!parametersInCSRMatch(MRI, CallerPreservedMask: CallerPreserved, ArgLocs, OutVals))
3166 return false;
3167 }
3168
3169 bool CalleeWillPop =
3170 X86::isCalleePop(CallingConv: CalleeCC, is64Bit: Subtarget.is64Bit(), IsVarArg: isVarArg,
3171 GuaranteeTCO: MF.getTarget().Options.GuaranteedTailCallOpt);
3172
3173 if (unsigned BytesToPop = FuncInfo->getBytesToPopOnReturn()) {
3174 // If we have bytes to pop, the callee must pop them.
3175 bool CalleePopMatches = CalleeWillPop && BytesToPop == StackArgsSize;
3176 if (!CalleePopMatches)
3177 return false;
3178 } else if (CalleeWillPop && StackArgsSize > 0) {
3179 // If we don't have bytes to pop, make sure the callee doesn't pop any.
3180 return false;
3181 }
3182
3183 return true;
3184}
3185
3186/// Determines whether the callee is required to pop its own arguments.
3187/// Callee pop is necessary to support tail calls.
3188bool X86::isCalleePop(CallingConv::ID CallingConv,
3189 bool is64Bit, bool IsVarArg, bool GuaranteeTCO) {
3190 // If GuaranteeTCO is true, we force some calls to be callee pop so that we
3191 // can guarantee TCO.
3192 if (!IsVarArg && shouldGuaranteeTCO(CC: CallingConv, GuaranteedTailCallOpt: GuaranteeTCO))
3193 return true;
3194
3195 switch (CallingConv) {
3196 default:
3197 return false;
3198 case CallingConv::X86_StdCall:
3199 case CallingConv::X86_FastCall:
3200 case CallingConv::X86_ThisCall:
3201 case CallingConv::X86_VectorCall:
3202 return !is64Bit;
3203 }
3204}
3205