1//===- X86.cpp ------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ABIInfoImpl.h"
10#include "TargetInfo.h"
11#include "clang/Basic/DiagnosticFrontend.h"
12#include "clang/Basic/SourceLocation.h"
13#include "llvm/ADT/SmallBitVector.h"
14
15using namespace clang;
16using namespace clang::CodeGen;
17
18namespace {
19
20/// IsX86_MMXType - Return true if this is an MMX type.
21bool IsX86_MMXType(llvm::Type *IRType) {
22 // Return true if the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>.
23 return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&
24 cast<llvm::VectorType>(Val: IRType)->getElementType()->isIntegerTy() &&
25 IRType->getScalarSizeInBits() != 64;
26}
27
28static llvm::Type *X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
29 StringRef Constraint,
30 llvm::Type *Ty) {
31 bool IsMMXCons = llvm::StringSwitch<bool>(Constraint)
32 .Cases(CaseStrings: {"y", "&y", "^Ym"}, Value: true)
33 .Default(Value: false);
34 if (IsMMXCons && Ty->isVectorTy() &&
35 cast<llvm::VectorType>(Val: Ty)->getPrimitiveSizeInBits().getFixedValue() !=
36 64)
37 return nullptr; // Invalid MMX constraint
38
39 if (Constraint == "k") {
40 llvm::Type *Int1Ty = llvm::Type::getInt1Ty(C&: CGF.getLLVMContext());
41 return llvm::FixedVectorType::get(ElementType: Int1Ty, NumElts: Ty->getScalarSizeInBits());
42 }
43
44 // No operation needed
45 return Ty;
46}
47
48/// Returns true if this type can be passed in SSE registers with the
49/// X86_VectorCall calling convention. Shared between x86_32 and x86_64.
50static bool isX86VectorTypeForVectorCall(ASTContext &Context, QualType Ty) {
51 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
52 if (BT->isFloatingPoint() && BT->getKind() != BuiltinType::Half) {
53 if (BT->getKind() == BuiltinType::LongDouble) {
54 if (&Context.getTargetInfo().getLongDoubleFormat() ==
55 &llvm::APFloat::x87DoubleExtended())
56 return false;
57 }
58 return true;
59 }
60 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
61 // vectorcall can pass XMM, YMM, and ZMM vectors. We don't pass SSE1 MMX
62 // registers specially.
63 unsigned VecSize = Context.getTypeSize(T: VT);
64 if (VecSize == 128 || VecSize == 256 || VecSize == 512)
65 return true;
66 }
67 return false;
68}
69
70/// Returns true if this aggregate is small enough to be passed in SSE registers
71/// in the X86_VectorCall calling convention. Shared between x86_32 and x86_64.
72static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) {
73 return NumMembers <= 4;
74}
75
76/// Returns a Homogeneous Vector Aggregate ABIArgInfo, used in X86.
77static ABIArgInfo getDirectX86Hva(llvm::Type* T = nullptr) {
78 auto AI = ABIArgInfo::getDirect(T);
79 AI.setInReg(true);
80 AI.setCanBeFlattened(false);
81 return AI;
82}
83
84//===----------------------------------------------------------------------===//
85// X86-32 ABI Implementation
86//===----------------------------------------------------------------------===//
87
88/// Similar to llvm::CCState, but for Clang.
89struct CCState {
90 CCState(CGFunctionInfo &FI)
91 : IsPreassigned(FI.arg_size()), CC(FI.getCallingConvention()),
92 Required(FI.getRequiredArgs()), IsDelegateCall(FI.isDelegateCall()) {}
93
94 llvm::SmallBitVector IsPreassigned;
95 unsigned CC = CallingConv::CC_C;
96 unsigned FreeRegs = 0;
97 unsigned FreeSSERegs = 0;
98 RequiredArgs Required;
99 bool IsDelegateCall = false;
100};
101
102/// X86_32ABIInfo - The X86-32 ABI information.
103class X86_32ABIInfo : public ABIInfo {
104 enum Class { Integer, Float, AlwaysStack };
105
106 static const unsigned MinABIStackAlignInBytes = 4;
107
108 bool IsDarwinVectorABI;
109 bool IsRetSmallStructInRegABI;
110 bool IsWin32StructABI;
111 bool IsSoftFloatABI;
112 bool IsMCUABI;
113 bool IsLinuxABI;
114 unsigned DefaultNumRegisterParameters;
115
116 static bool isRegisterSize(unsigned Size) {
117 return (Size == 8 || Size == 16 || Size == 32 || Size == 64);
118 }
119
120 bool isHomogeneousAggregateBaseType(QualType Ty) const override {
121 // FIXME: Assumes vectorcall is in use.
122 return isX86VectorTypeForVectorCall(Context&: getContext(), Ty);
123 }
124
125 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
126 uint64_t NumMembers) const override {
127 // FIXME: Assumes vectorcall is in use.
128 return isX86VectorCallAggregateSmallEnough(NumMembers);
129 }
130
131 bool shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const;
132
133 /// getIndirectResult - Give a source type \arg Ty, return a suitable result
134 /// such that the argument will be passed in memory.
135 ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
136
137 ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const;
138
139 /// Return the alignment to use for the given type on the stack.
140 unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
141
142 Class classify(QualType Ty) const;
143 ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const;
144 ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State,
145 unsigned ArgIndex) const;
146
147 /// Updates the number of available free registers, returns
148 /// true if any registers were allocated.
149 bool updateFreeRegs(QualType Ty, CCState &State) const;
150
151 bool shouldAggregateUseDirect(QualType Ty, CCState &State, bool &InReg,
152 bool &NeedsPadding) const;
153 bool shouldPrimitiveUseInReg(QualType Ty, CCState &State) const;
154
155 bool canExpandIndirectArgument(QualType Ty) const;
156
157 /// Rewrite the function info so that all memory arguments use
158 /// inalloca.
159 void rewriteWithInAlloca(CGFunctionInfo &FI) const;
160
161 void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
162 CharUnits &StackOffset, ABIArgInfo &Info,
163 QualType Type) const;
164 void runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const;
165
166public:
167
168 void computeInfo(CGFunctionInfo &FI) const override;
169 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
170 AggValueSlot Slot) const override;
171
172 X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
173 bool RetSmallStructInRegABI, bool Win32StructABI,
174 unsigned NumRegisterParameters, bool SoftFloatABI)
175 : ABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI),
176 IsRetSmallStructInRegABI(RetSmallStructInRegABI),
177 IsWin32StructABI(Win32StructABI), IsSoftFloatABI(SoftFloatABI),
178 IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()),
179 IsLinuxABI(CGT.getTarget().getTriple().isOSLinux() ||
180 CGT.getTarget().getTriple().isOSCygMing()),
181 DefaultNumRegisterParameters(NumRegisterParameters) {}
182};
183
184class X86_32SwiftABIInfo : public SwiftABIInfo {
185public:
186 explicit X86_32SwiftABIInfo(CodeGenTypes &CGT)
187 : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/false) {}
188
189 bool shouldPassIndirectly(ArrayRef<llvm::Type *> ComponentTys,
190 bool AsReturnValue) const override {
191 // LLVM's x86-32 lowering currently only assigns up to three
192 // integer registers and three fp registers. Oddly, it'll use up to
193 // four vector registers for vectors, but those can overlap with the
194 // scalar registers.
195 return occupiesMoreThan(scalarTypes: ComponentTys, /*total=*/maxAllRegisters: 3);
196 }
197};
198
199class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
200public:
201 X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
202 bool RetSmallStructInRegABI, bool Win32StructABI,
203 unsigned NumRegisterParameters, bool SoftFloatABI)
204 : TargetCodeGenInfo(std::make_unique<X86_32ABIInfo>(
205 args&: CGT, args&: DarwinVectorABI, args&: RetSmallStructInRegABI, args&: Win32StructABI,
206 args&: NumRegisterParameters, args&: SoftFloatABI)) {
207 SwiftInfo = std::make_unique<X86_32SwiftABIInfo>(args&: CGT);
208 }
209
210 static bool isStructReturnInRegABI(
211 const llvm::Triple &Triple, const CodeGenOptions &Opts);
212
213 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
214 CodeGen::CodeGenModule &CGM) const override;
215
216 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
217 // Darwin uses different dwarf register numbers for EH.
218 if (CGM.getTarget().getTriple().isOSDarwin()) return 5;
219 return 4;
220 }
221
222 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
223 llvm::Value *Address) const override;
224
225 llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
226 StringRef Constraint,
227 llvm::Type* Ty) const override {
228 return X86AdjustInlineAsmType(CGF, Constraint, Ty);
229 }
230
231 void addReturnRegisterOutputs(CodeGenFunction &CGF, LValue ReturnValue,
232 std::string &Constraints,
233 std::vector<llvm::Type *> &ResultRegTypes,
234 std::vector<llvm::Type *> &ResultTruncRegTypes,
235 std::vector<LValue> &ResultRegDests,
236 std::string &AsmString,
237 unsigned NumOutputs) const override;
238
239 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
240 return "movl\t%ebp, %ebp"
241 "\t\t// marker for objc_retainAutoreleaseReturnValue";
242 }
243};
244
245}
246
247/// Rewrite input constraint references after adding some output constraints.
248/// In the case where there is one output and one input and we add one output,
249/// we need to replace all operand references greater than or equal to 1:
250/// mov $0, $1
251/// mov eax, $1
252/// The result will be:
253/// mov $0, $2
254/// mov eax, $2
255static void rewriteInputConstraintReferences(unsigned FirstIn,
256 unsigned NumNewOuts,
257 std::string &AsmString) {
258 std::string Buf;
259 llvm::raw_string_ostream OS(Buf);
260 size_t Pos = 0;
261 while (Pos < AsmString.size()) {
262 size_t DollarStart = AsmString.find(c: '$', pos: Pos);
263 if (DollarStart == std::string::npos)
264 DollarStart = AsmString.size();
265 size_t DollarEnd = AsmString.find_first_not_of(c: '$', pos: DollarStart);
266 if (DollarEnd == std::string::npos)
267 DollarEnd = AsmString.size();
268 OS << StringRef(&AsmString[Pos], DollarEnd - Pos);
269 Pos = DollarEnd;
270 size_t NumDollars = DollarEnd - DollarStart;
271 if (NumDollars % 2 != 0 && Pos < AsmString.size()) {
272 // We have an operand reference.
273 size_t DigitStart = Pos;
274 if (AsmString[DigitStart] == '{') {
275 OS << '{';
276 ++DigitStart;
277 }
278 size_t DigitEnd = AsmString.find_first_not_of(s: "0123456789", pos: DigitStart);
279 if (DigitEnd == std::string::npos)
280 DigitEnd = AsmString.size();
281 StringRef OperandStr(&AsmString[DigitStart], DigitEnd - DigitStart);
282 unsigned OperandIndex;
283 if (!OperandStr.getAsInteger(Radix: 10, Result&: OperandIndex)) {
284 if (OperandIndex >= FirstIn)
285 OperandIndex += NumNewOuts;
286 OS << OperandIndex;
287 } else {
288 OS << OperandStr;
289 }
290 Pos = DigitEnd;
291 }
292 }
293 AsmString = std::move(Buf);
294}
295
296/// Add output constraints for EAX:EDX because they are return registers.
297void X86_32TargetCodeGenInfo::addReturnRegisterOutputs(
298 CodeGenFunction &CGF, LValue ReturnSlot, std::string &Constraints,
299 std::vector<llvm::Type *> &ResultRegTypes,
300 std::vector<llvm::Type *> &ResultTruncRegTypes,
301 std::vector<LValue> &ResultRegDests, std::string &AsmString,
302 unsigned NumOutputs) const {
303 uint64_t RetWidth = CGF.getContext().getTypeSize(T: ReturnSlot.getType());
304
305 // Use the EAX constraint if the width is 32 or smaller and EAX:EDX if it is
306 // larger.
307 if (!Constraints.empty())
308 Constraints += ',';
309 if (RetWidth <= 32) {
310 Constraints += "={eax}";
311 ResultRegTypes.push_back(x: CGF.Int32Ty);
312 } else {
313 // Use the 'A' constraint for EAX:EDX.
314 Constraints += "=A";
315 ResultRegTypes.push_back(x: CGF.Int64Ty);
316 }
317
318 // Truncate EAX or EAX:EDX to an integer of the appropriate size.
319 llvm::Type *CoerceTy = llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: RetWidth);
320 ResultTruncRegTypes.push_back(x: CoerceTy);
321
322 // Coerce the integer by bitcasting the return slot pointer.
323 ReturnSlot.setAddress(ReturnSlot.getAddress().withElementType(ElemTy: CoerceTy));
324 ResultRegDests.push_back(x: ReturnSlot);
325
326 rewriteInputConstraintReferences(FirstIn: NumOutputs, NumNewOuts: 1, AsmString);
327}
328
329/// shouldReturnTypeInRegister - Determine if the given type should be
330/// returned in a register (for the Darwin and MCU ABI).
331bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
332 ASTContext &Context) const {
333 uint64_t Size = Context.getTypeSize(T: Ty);
334
335 // For i386, type must be register sized.
336 // For the MCU ABI, it only needs to be <= 8-byte
337 if ((IsMCUABI && Size > 64) || (!IsMCUABI && !isRegisterSize(Size)))
338 return false;
339
340 if (Ty->isVectorType()) {
341 // 64- and 128- bit vectors inside structures are not returned in
342 // registers.
343 if (Size == 64 || Size == 128)
344 return false;
345
346 return true;
347 }
348
349 // If this is a builtin, pointer, enum, complex type, member pointer, or
350 // member function pointer it is ok.
351 if (Ty->getAs<BuiltinType>() || Ty->hasPointerRepresentation() ||
352 Ty->isAnyComplexType() || Ty->isEnumeralType() ||
353 Ty->isBlockPointerType() || Ty->isMemberPointerType())
354 return true;
355
356 // Arrays are treated like records.
357 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty))
358 return shouldReturnTypeInRegister(Ty: AT->getElementType(), Context);
359
360 // Otherwise, it must be a record type.
361 const auto *RD = Ty->getAsRecordDecl();
362 if (!RD)
363 return false;
364
365 // FIXME: Traverse bases here too.
366
367 // Structure types are passed in register if all fields would be
368 // passed in a register.
369 for (const auto *FD : RD->fields()) {
370 // Empty fields are ignored.
371 if (isEmptyField(Context, FD, AllowArrays: true))
372 continue;
373
374 // Check fields recursively.
375 if (!shouldReturnTypeInRegister(Ty: FD->getType(), Context))
376 return false;
377 }
378 return true;
379}
380
381static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {
382 // Treat complex types as the element type.
383 if (const ComplexType *CTy = Ty->getAs<ComplexType>())
384 Ty = CTy->getElementType();
385
386 // Check for a type which we know has a simple scalar argument-passing
387 // convention without any padding. (We're specifically looking for 32
388 // and 64-bit integer and integer-equivalents, float, and double.)
389 if (!Ty->getAs<BuiltinType>() && !Ty->hasPointerRepresentation() &&
390 !Ty->isEnumeralType() && !Ty->isBlockPointerType())
391 return false;
392
393 uint64_t Size = Context.getTypeSize(T: Ty);
394 return Size == 32 || Size == 64;
395}
396
397static bool addFieldSizes(ASTContext &Context, const RecordDecl *RD,
398 uint64_t &Size) {
399 for (const auto *FD : RD->fields()) {
400 // Scalar arguments on the stack get 4 byte alignment on x86. If the
401 // argument is smaller than 32-bits, expanding the struct will create
402 // alignment padding.
403 if (!is32Or64BitBasicType(Ty: FD->getType(), Context))
404 return false;
405
406 // FIXME: Reject bit-fields wholesale; there are two problems, we don't know
407 // how to expand them yet, and the predicate for telling if a bitfield still
408 // counts as "basic" is more complicated than what we were doing previously.
409 if (FD->isBitField())
410 return false;
411
412 Size += Context.getTypeSize(T: FD->getType());
413 }
414 return true;
415}
416
417static bool addBaseAndFieldSizes(ASTContext &Context, const CXXRecordDecl *RD,
418 uint64_t &Size) {
419 // Don't do this if there are any non-empty bases.
420 for (const CXXBaseSpecifier &Base : RD->bases()) {
421 if (!addBaseAndFieldSizes(Context, RD: Base.getType()->getAsCXXRecordDecl(),
422 Size))
423 return false;
424 }
425 if (!addFieldSizes(Context, RD, Size))
426 return false;
427 return true;
428}
429
430/// Test whether an argument type which is to be passed indirectly (on the
431/// stack) would have the equivalent layout if it was expanded into separate
432/// arguments. If so, we prefer to do the latter to avoid inhibiting
433/// optimizations.
434bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const {
435 // We can only expand structure types.
436 const RecordDecl *RD = Ty->getAsRecordDecl();
437 if (!RD)
438 return false;
439 uint64_t Size = 0;
440 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
441 if (!IsWin32StructABI) {
442 // On non-Windows, we have to conservatively match our old bitcode
443 // prototypes in order to be ABI-compatible at the bitcode level.
444 if (!CXXRD->isCLike())
445 return false;
446 } else {
447 // Don't do this for dynamic classes.
448 if (CXXRD->isDynamicClass())
449 return false;
450 }
451 if (!addBaseAndFieldSizes(Context&: getContext(), RD: CXXRD, Size))
452 return false;
453 } else {
454 if (!addFieldSizes(Context&: getContext(), RD, Size))
455 return false;
456 }
457
458 // We can do this if there was no alignment padding.
459 return Size == getContext().getTypeSize(T: Ty);
460}
461
462ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const {
463 // If the return value is indirect, then the hidden argument is consuming one
464 // integer register.
465 if (State.CC != llvm::CallingConv::X86_FastCall &&
466 State.CC != llvm::CallingConv::X86_VectorCall && State.FreeRegs) {
467 --State.FreeRegs;
468 if (!IsMCUABI)
469 return getNaturalAlignIndirectInReg(Ty: RetTy);
470 }
471 return getNaturalAlignIndirect(
472 Ty: RetTy, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
473 /*ByVal=*/false);
474}
475
476ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
477 CCState &State) const {
478 if (RetTy->isVoidType())
479 return ABIArgInfo::getIgnore();
480
481 const Type *Base = nullptr;
482 uint64_t NumElts = 0;
483 if ((State.CC == llvm::CallingConv::X86_VectorCall ||
484 State.CC == llvm::CallingConv::X86_RegCall) &&
485 isHomogeneousAggregate(Ty: RetTy, Base, Members&: NumElts)) {
486 // The LLVM struct type for such an aggregate should lower properly.
487 return ABIArgInfo::getDirect();
488 }
489
490 if (const VectorType *VT = RetTy->getAs<VectorType>()) {
491 // On Darwin, some vectors are returned in registers.
492 if (IsDarwinVectorABI) {
493 uint64_t Size = getContext().getTypeSize(T: RetTy);
494
495 // 128-bit vectors are a special case; they are returned in
496 // registers and we need to make sure to pick a type the LLVM
497 // backend will like.
498 if (Size == 128)
499 return ABIArgInfo::getDirect(T: llvm::FixedVectorType::get(
500 ElementType: llvm::Type::getInt64Ty(C&: getVMContext()), NumElts: 2));
501
502 // Always return in register if it fits in a general purpose
503 // register, or if it is 64 bits and has a single element.
504 if ((Size == 8 || Size == 16 || Size == 32) ||
505 (Size == 64 && VT->getNumElements() == 1))
506 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(),
507 NumBits: Size));
508
509 return getIndirectReturnResult(RetTy, State);
510 }
511
512 return ABIArgInfo::getDirect();
513 }
514
515 if (isAggregateTypeForABI(T: RetTy)) {
516 if (const auto *RD = RetTy->getAsRecordDecl();
517 RD && RD->hasFlexibleArrayMember())
518 // Structures with flexible arrays are always indirect.
519 return getIndirectReturnResult(RetTy, State);
520
521 // If specified, structs and unions are always indirect.
522 if (!IsRetSmallStructInRegABI && !RetTy->isAnyComplexType())
523 return getIndirectReturnResult(RetTy, State);
524
525 // Ignore empty structs/unions.
526 if (isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true))
527 return ABIArgInfo::getIgnore();
528
529 // Return complex of _Float16 as <2 x half> so the backend will use xmm0.
530 if (const ComplexType *CT = RetTy->getAs<ComplexType>()) {
531 QualType ET = getContext().getCanonicalType(T: CT->getElementType());
532 if (ET->isFloat16Type())
533 return ABIArgInfo::getDirect(T: llvm::FixedVectorType::get(
534 ElementType: llvm::Type::getHalfTy(C&: getVMContext()), NumElts: 2));
535 }
536
537 // Small structures which are register sized are generally returned
538 // in a register.
539 if (shouldReturnTypeInRegister(Ty: RetTy, Context&: getContext())) {
540 uint64_t Size = getContext().getTypeSize(T: RetTy);
541
542 // As a special-case, if the struct is a "single-element" struct, and
543 // the field is of type "float" or "double", return it in a
544 // floating-point register. (MSVC does not apply this special case.)
545 // We apply a similar transformation for pointer types to improve the
546 // quality of the generated IR.
547 if (const Type *SeltTy = isSingleElementStruct(T: RetTy, Context&: getContext()))
548 if ((!IsWin32StructABI && SeltTy->isRealFloatingType())
549 || SeltTy->hasPointerRepresentation())
550 return ABIArgInfo::getDirect(T: CGT.ConvertType(T: QualType(SeltTy, 0)));
551
552 // FIXME: We should be able to narrow this integer in cases with dead
553 // padding.
554 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(),NumBits: Size));
555 }
556
557 return getIndirectReturnResult(RetTy, State);
558 }
559
560 // Treat an enum type as its underlying type.
561 if (const auto *ED = RetTy->getAsEnumDecl())
562 RetTy = ED->getIntegerType();
563
564 if (const auto *EIT = RetTy->getAs<BitIntType>())
565 if (EIT->getNumBits() > 64)
566 return getIndirectReturnResult(RetTy, State);
567
568 return (isPromotableIntegerTypeForABI(Ty: RetTy) ? ABIArgInfo::getExtend(Ty: RetTy)
569 : ABIArgInfo::getDirect());
570}
571
572unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,
573 unsigned Align) const {
574 // Otherwise, if the alignment is less than or equal to the minimum ABI
575 // alignment, just use the default; the backend will handle this.
576 if (Align <= MinABIStackAlignInBytes)
577 return 0; // Use default alignment.
578
579 if (Ty->isFloat128Type())
580 return 16;
581
582 if (IsLinuxABI) {
583 // Exclude other System V OS (e.g Darwin, PS4 and FreeBSD) since we don't
584 // want to spend any effort dealing with the ramifications of ABI breaks.
585 //
586 // If the vector type is __m128/__m256/__m512, return the default alignment.
587 if (Ty->isVectorType() && (Align == 16 || Align == 32 || Align == 64))
588 return Align;
589 }
590 // On non-Darwin, the stack type alignment is always 4.
591 if (!IsDarwinVectorABI) {
592 // Set explicit alignment, since we may need to realign the top.
593 return MinABIStackAlignInBytes;
594 }
595
596 // Otherwise, if the type contains an SSE vector type, the alignment is 16.
597 if (Align >= 16 && (isSIMDVectorType(Context&: getContext(), Ty) ||
598 isRecordWithSIMDVectorType(Context&: getContext(), Ty)))
599 return 16;
600
601 return MinABIStackAlignInBytes;
602}
603
604ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal,
605 CCState &State) const {
606 if (!ByVal) {
607 if (State.FreeRegs) {
608 --State.FreeRegs; // Non-byval indirects just use one pointer.
609 if (!IsMCUABI)
610 return getNaturalAlignIndirectInReg(Ty);
611 }
612 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
613 ByVal: false);
614 }
615
616 // Compute the byval alignment.
617 unsigned TypeAlign = getContext().getTypeAlign(T: Ty) / 8;
618 unsigned StackAlign = getTypeStackAlignInBytes(Ty, Align: TypeAlign);
619 if (StackAlign == 0)
620 return ABIArgInfo::getIndirect(
621 Alignment: CharUnits::fromQuantity(Quantity: 4),
622 /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
623 /*ByVal=*/true);
624
625 // If the stack alignment is less than the type alignment, realign the
626 // argument.
627 bool Realign = TypeAlign > StackAlign;
628 return ABIArgInfo::getIndirect(
629 Alignment: CharUnits::fromQuantity(Quantity: StackAlign),
630 /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(), /*ByVal=*/true,
631 Realign);
632}
633
634/// Like isSingleElementStruct, but only looks through one level of records.
635/// This mirrors how GCC gives a record the mode of its only field.
636static const Type *getSingleFieldType(QualType T, ASTContext &Context) {
637 const auto *RD = T->getAsRecordDecl();
638 if (!RD)
639 return nullptr;
640
641 if (RD->hasFlexibleArrayMember())
642 return nullptr;
643
644 const Type *Found = nullptr;
645
646 // If this is a C++ record, check the bases first.
647 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
648 for (const auto &I : CXXRD->bases()) {
649 // Ignore empty records.
650 if (isEmptyRecord(Context, T: I.getType(), AllowArrays: true))
651 continue;
652
653 // If we already found an element then this isn't a single-element struct.
654 if (Found)
655 return nullptr;
656
657 Found = I.getType().getTypePtr();
658 }
659 }
660
661 // Check for single element.
662 for (const auto *FD : RD->fields()) {
663 QualType FT = FD->getType();
664
665 // Ignore empty fields.
666 if (isEmptyField(Context, FD, AllowArrays: true))
667 continue;
668
669 // If we already found an element then this isn't a single-element
670 // struct.
671 if (Found)
672 return nullptr;
673
674 // Treat single element arrays as the element.
675 while (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: FT)) {
676 if (AT->getZExtSize() != 1)
677 break;
678 FT = AT->getElementType();
679 }
680
681 Found = FT.getTypePtr();
682 }
683
684 // We don't consider a struct a single-element struct if it has
685 // padding beyond the element type.
686 if (Found && Context.getTypeSize(T: Found) != Context.getTypeSize(T))
687 return nullptr;
688
689 return Found;
690}
691
692X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {
693 if (getContext().getLangOpts().isCompatibleWith(
694 Version: LangOptions::ClangABI::Ver23)) {
695 const Type *T = isSingleElementStruct(T: Ty, Context&: getContext());
696 if (!T)
697 T = Ty.getTypePtr();
698
699 if (const BuiltinType *BT = T->getAs<BuiltinType>()) {
700 BuiltinType::Kind K = BT->getKind();
701 if (K == BuiltinType::Float || K == BuiltinType::Double)
702 return Float;
703 }
704 return Integer;
705 }
706
707 while (const RecordDecl *RD = Ty->getAsRecordDecl()) {
708 // Unions are always passed as integers.
709 if (RD->isUnion())
710 return Integer;
711
712 const Type *FT = getSingleFieldType(T: Ty, Context&: getContext());
713
714 // Pick integer for zero-sized types. They will be ignored down the line.
715 if (!FT)
716 return Integer;
717
718 Ty = QualType(FT, 0);
719 }
720
721 // All complex values are passed via the stack.
722 if (Ty->isAnyComplexType())
723 return AlwaysStack;
724
725 // Types like bfloat16 or __float128 are all passed like floats.
726 if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
727 if (BT->isFloatingPoint())
728 return Float;
729
730 return Integer;
731}
732
733bool X86_32ABIInfo::updateFreeRegs(QualType Ty, CCState &State) const {
734 Class C = classify(Ty);
735 if (C == AlwaysStack && !IsMCUABI)
736 return false;
737 if (C == Float && !IsSoftFloatABI)
738 return false;
739
740 unsigned Size = getContext().getTypeSize(T: Ty);
741 unsigned SizeInRegs = (Size + 31) / 32;
742
743 if (SizeInRegs == 0)
744 return false;
745
746 if (!IsMCUABI) {
747 if (SizeInRegs > State.FreeRegs) {
748 State.FreeRegs = 0;
749 return false;
750 }
751 } else {
752 // The MCU psABI allows passing parameters in-reg even if there are
753 // earlier parameters that are passed on the stack. Also,
754 // it does not allow passing >8-byte structs in-register,
755 // even if there are 3 free registers available.
756 if (SizeInRegs > State.FreeRegs || SizeInRegs > 2)
757 return false;
758 }
759
760 State.FreeRegs -= SizeInRegs;
761 return true;
762}
763
764bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State,
765 bool &InReg,
766 bool &NeedsPadding) const {
767 // On Windows, aggregates other than HFAs are never passed in registers, and
768 // they do not consume register slots. Homogenous floating-point aggregates
769 // (HFAs) have already been dealt with at this point.
770 if (IsWin32StructABI && isAggregateTypeForABI(T: Ty))
771 return false;
772
773 NeedsPadding = false;
774 InReg = !IsMCUABI;
775
776 if (!updateFreeRegs(Ty, State))
777 return false;
778
779 if (IsMCUABI)
780 return true;
781
782 if (State.CC == llvm::CallingConv::X86_FastCall ||
783 State.CC == llvm::CallingConv::X86_VectorCall ||
784 State.CC == llvm::CallingConv::X86_RegCall) {
785 if (getContext().getTypeSize(T: Ty) <= 32 && State.FreeRegs)
786 NeedsPadding = true;
787
788 return false;
789 }
790
791 return true;
792}
793
794bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {
795 bool IsPtrOrInt = (getContext().getTypeSize(T: Ty) <= 32) &&
796 (Ty->isIntegralOrEnumerationType() || Ty->isPointerType() ||
797 Ty->isReferenceType());
798
799 if (!IsPtrOrInt && (State.CC == llvm::CallingConv::X86_FastCall ||
800 State.CC == llvm::CallingConv::X86_VectorCall))
801 return false;
802
803 if (!updateFreeRegs(Ty, State))
804 return false;
805
806 if (!IsPtrOrInt && State.CC == llvm::CallingConv::X86_RegCall)
807 return false;
808
809 // Return true to apply inreg to all legal parameters except for MCU targets.
810 return !IsMCUABI;
811}
812
813void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {
814 // Vectorcall x86 works subtly different than in x64, so the format is
815 // a bit different than the x64 version. First, all vector types (not HVAs)
816 // are assigned, with the first 6 ending up in the [XYZ]MM0-5 registers.
817 // This differs from the x64 implementation, where the first 6 by INDEX get
818 // registers.
819 // In the second pass over the arguments, HVAs are passed in the remaining
820 // vector registers if possible, or indirectly by address. The address will be
821 // passed in ECX/EDX if available. Any other arguments are passed according to
822 // the usual fastcall rules.
823 MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
824 for (int I = 0, E = Args.size(); I < E; ++I) {
825 const Type *Base = nullptr;
826 uint64_t NumElts = 0;
827 const QualType &Ty = Args[I].type;
828 if ((Ty->isVectorType() || Ty->isBuiltinType()) &&
829 isHomogeneousAggregate(Ty, Base, Members&: NumElts)) {
830 if (State.FreeSSERegs >= NumElts) {
831 State.FreeSSERegs -= NumElts;
832 Args[I].info = ABIArgInfo::getDirectInReg();
833 State.IsPreassigned.set(I);
834 }
835 }
836 }
837}
838
839ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, CCState &State,
840 unsigned ArgIndex) const {
841 // FIXME: Set alignment on indirect arguments.
842 bool IsFastCall = State.CC == llvm::CallingConv::X86_FastCall;
843 bool IsRegCall = State.CC == llvm::CallingConv::X86_RegCall;
844 bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall;
845
846 Ty = useFirstFieldIfTransparentUnion(Ty);
847 TypeInfo TI = getContext().getTypeInfo(T: Ty);
848
849 // Check with the C++ ABI first.
850 const RecordType *RT = Ty->getAsCanonical<RecordType>();
851 if (RT) {
852 CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, CXXABI&: getCXXABI());
853 if (RAA == CGCXXABI::RAA_Indirect) {
854 return getIndirectResult(Ty, ByVal: false, State);
855 } else if (State.IsDelegateCall) {
856 // Avoid having different alignments on delegate call args by always
857 // setting the alignment to 4, which is what we do for inallocas.
858 ABIArgInfo Res = getIndirectResult(Ty, ByVal: false, State);
859 Res.setIndirectAlign(CharUnits::fromQuantity(Quantity: 4));
860 return Res;
861 } else if (RAA == CGCXXABI::RAA_DirectInMemory) {
862 // The field index doesn't matter, we'll fix it up later.
863 return ABIArgInfo::getInAlloca(/*FieldIndex=*/0);
864 }
865 }
866
867 // Regcall uses the concept of a homogenous vector aggregate, similar
868 // to other targets.
869 const Type *Base = nullptr;
870 uint64_t NumElts = 0;
871 if ((IsRegCall || IsVectorCall) &&
872 isHomogeneousAggregate(Ty, Base, Members&: NumElts)) {
873 if (State.FreeSSERegs >= NumElts) {
874 State.FreeSSERegs -= NumElts;
875
876 // Vectorcall passes HVAs directly and does not flatten them, but regcall
877 // does.
878 if (IsVectorCall)
879 return getDirectX86Hva();
880
881 if (Ty->isBuiltinType() || Ty->isVectorType())
882 return ABIArgInfo::getDirect();
883 return ABIArgInfo::getExpand();
884 }
885 if (IsVectorCall && Ty->isBuiltinType())
886 return ABIArgInfo::getDirect();
887 return getIndirectResult(Ty, /*ByVal=*/false, State);
888 }
889
890 if (isAggregateTypeForABI(T: Ty)) {
891 // Structures with flexible arrays are always indirect.
892 // FIXME: This should not be byval!
893 if (RT && RT->getDecl()->getDefinitionOrSelf()->hasFlexibleArrayMember())
894 return getIndirectResult(Ty, ByVal: true, State);
895
896 // Ignore empty structs/unions on non-Windows.
897 if (!IsWin32StructABI && isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
898 return ABIArgInfo::getIgnore();
899
900 // Ignore 0 sized structs.
901 if (TI.Width == 0)
902 return ABIArgInfo::getIgnore();
903
904 llvm::LLVMContext &LLVMContext = getVMContext();
905 llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(C&: LLVMContext);
906 bool NeedsPadding = false;
907 bool InReg;
908 if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) {
909 unsigned SizeInRegs = (TI.Width + 31) / 32;
910 SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32);
911 llvm::Type *Result = llvm::StructType::get(Context&: LLVMContext, Elements);
912 if (InReg)
913 return ABIArgInfo::getDirectInReg(T: Result);
914 else
915 return ABIArgInfo::getDirect(T: Result);
916 }
917 llvm::IntegerType *PaddingType = NeedsPadding ? Int32 : nullptr;
918
919 // Pass over-aligned aggregates to non-variadic functions on Windows
920 // indirectly. This behavior was added in MSVC 2015. Use the required
921 // alignment from the record layout, since that may be less than the
922 // regular type alignment, and types with required alignment of less than 4
923 // bytes are not passed indirectly.
924 if (IsWin32StructABI && State.Required.isRequiredArg(argIdx: ArgIndex)) {
925 unsigned AlignInBits = 0;
926 if (RT) {
927 const ASTRecordLayout &Layout =
928 getContext().getASTRecordLayout(D: RT->getDecl());
929 AlignInBits = getContext().toBits(CharSize: Layout.getRequiredAlignment());
930 } else if (TI.isAlignRequired()) {
931 AlignInBits = TI.Align;
932 }
933 if (AlignInBits > 32)
934 return getIndirectResult(Ty, /*ByVal=*/false, State);
935 }
936
937 // Expand small (<= 128-bit) record types when we know that the stack layout
938 // of those arguments will match the struct. This is important because the
939 // LLVM backend isn't smart enough to remove byval, which inhibits many
940 // optimizations.
941 // Don't do this for the MCU if there are still free integer registers
942 // (see X86_64 ABI for full explanation).
943 if (TI.Width <= 4 * 32 && (!IsMCUABI || State.FreeRegs == 0) &&
944 canExpandIndirectArgument(Ty))
945 return ABIArgInfo::getExpandWithPadding(
946 PaddingInReg: IsFastCall || IsVectorCall || IsRegCall, Padding: PaddingType);
947
948 return getIndirectResult(Ty, ByVal: true, State);
949 }
950
951 if (const VectorType *VT = Ty->getAs<VectorType>()) {
952 // On Windows, vectors are passed directly if registers are available, or
953 // indirectly if not. This avoids the need to align argument memory. Pass
954 // user-defined vector types larger than 512 bits indirectly for simplicity.
955 if (IsWin32StructABI) {
956 if (TI.Width <= 512 && State.FreeSSERegs > 0) {
957 --State.FreeSSERegs;
958 return ABIArgInfo::getDirectInReg();
959 }
960 return getIndirectResult(Ty, /*ByVal=*/false, State);
961 }
962
963 // On Darwin, some vectors are passed in memory, we handle this by passing
964 // it as an i8/i16/i32/i64.
965 if (IsDarwinVectorABI) {
966 if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) ||
967 (TI.Width == 64 && VT->getNumElements() == 1))
968 return ABIArgInfo::getDirect(
969 T: llvm::IntegerType::get(C&: getVMContext(), NumBits: TI.Width));
970 }
971
972 if (IsX86_MMXType(IRType: CGT.ConvertType(T: Ty)))
973 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(), NumBits: 64));
974
975 return ABIArgInfo::getDirect();
976 }
977
978 if (const auto *ED = Ty->getAsEnumDecl())
979 Ty = ED->getIntegerType();
980
981 bool InReg = shouldPrimitiveUseInReg(Ty, State);
982
983 if (isPromotableIntegerTypeForABI(Ty)) {
984 if (InReg)
985 return ABIArgInfo::getExtendInReg(Ty, T: CGT.ConvertType(T: Ty));
986 return ABIArgInfo::getExtend(Ty, T: CGT.ConvertType(T: Ty));
987 }
988
989 if (const auto *EIT = Ty->getAs<BitIntType>()) {
990 if (EIT->getNumBits() <= 64) {
991 if (InReg)
992 return ABIArgInfo::getDirectInReg();
993 return ABIArgInfo::getDirect();
994 }
995 return getIndirectResult(Ty, /*ByVal=*/false, State);
996 }
997
998 if (InReg)
999 return ABIArgInfo::getDirectInReg();
1000 return ABIArgInfo::getDirect();
1001}
1002
1003void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
1004 CCState State(FI);
1005 if (IsMCUABI)
1006 State.FreeRegs = 3;
1007 else if (State.CC == llvm::CallingConv::X86_FastCall) {
1008 State.FreeRegs = 2;
1009 State.FreeSSERegs = 3;
1010 } else if (State.CC == llvm::CallingConv::X86_VectorCall) {
1011 State.FreeRegs = 2;
1012 State.FreeSSERegs = 6;
1013 } else if (FI.getHasRegParm())
1014 State.FreeRegs = FI.getRegParm();
1015 else if (State.CC == llvm::CallingConv::X86_RegCall) {
1016 State.FreeRegs = 5;
1017 State.FreeSSERegs = 8;
1018 } else if (IsWin32StructABI) {
1019 // Since MSVC 2015, the first three SSE vectors have been passed in
1020 // registers. The rest are passed indirectly.
1021 State.FreeRegs = DefaultNumRegisterParameters;
1022 State.FreeSSERegs = 3;
1023 } else
1024 State.FreeRegs = DefaultNumRegisterParameters;
1025
1026 if (!::classifyReturnType(CXXABI: getCXXABI(), FI, Info: *this)) {
1027 FI.getReturnInfo() = classifyReturnType(RetTy: FI.getReturnType(), State);
1028 } else if (FI.getReturnInfo().isIndirect()) {
1029 // The C++ ABI is not aware of register usage, so we have to check if the
1030 // return value was sret and put it in a register ourselves if appropriate.
1031 if (State.FreeRegs) {
1032 --State.FreeRegs; // The sret parameter consumes a register.
1033 if (!IsMCUABI)
1034 FI.getReturnInfo().setInReg(true);
1035 }
1036 }
1037
1038 // The chain argument effectively gives us another free register.
1039 if (FI.isChainCall())
1040 ++State.FreeRegs;
1041
1042 // For vectorcall, do a first pass over the arguments, assigning FP and vector
1043 // arguments to XMM registers as available.
1044 if (State.CC == llvm::CallingConv::X86_VectorCall)
1045 runVectorCallFirstPass(FI, State);
1046
1047 bool UsedInAlloca = false;
1048 MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
1049 for (unsigned I = 0, E = Args.size(); I < E; ++I) {
1050 // Skip arguments that have already been assigned.
1051 if (State.IsPreassigned.test(Idx: I))
1052 continue;
1053
1054 Args[I].info =
1055 classifyArgumentType(Ty: Args[I].type, State, ArgIndex: I);
1056 UsedInAlloca |= (Args[I].info.getKind() == ABIArgInfo::InAlloca);
1057 }
1058
1059 // If we needed to use inalloca for any argument, do a second pass and rewrite
1060 // all the memory arguments to use inalloca.
1061 if (UsedInAlloca)
1062 rewriteWithInAlloca(FI);
1063}
1064
1065void
1066X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
1067 CharUnits &StackOffset, ABIArgInfo &Info,
1068 QualType Type) const {
1069 // Arguments are always 4-byte-aligned.
1070 CharUnits WordSize = CharUnits::fromQuantity(Quantity: 4);
1071 assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct");
1072
1073 // sret pointers and indirect things will require an extra pointer
1074 // indirection, unless they are byval. Most things are byval, and will not
1075 // require this indirection.
1076 bool IsIndirect = false;
1077 if (Info.isIndirect() && !Info.getIndirectByVal())
1078 IsIndirect = true;
1079 Info = ABIArgInfo::getInAlloca(FieldIndex: FrameFields.size(), Indirect: IsIndirect);
1080 llvm::Type *LLTy = CGT.ConvertTypeForMem(T: Type);
1081 if (IsIndirect)
1082 LLTy = llvm::PointerType::getUnqual(C&: getVMContext());
1083 FrameFields.push_back(Elt: LLTy);
1084 StackOffset += IsIndirect ? WordSize : getContext().getTypeSizeInChars(T: Type);
1085
1086 // Insert padding bytes to respect alignment.
1087 CharUnits FieldEnd = StackOffset;
1088 StackOffset = FieldEnd.alignTo(Align: WordSize);
1089 if (StackOffset != FieldEnd) {
1090 CharUnits NumBytes = StackOffset - FieldEnd;
1091 llvm::Type *Ty = llvm::Type::getInt8Ty(C&: getVMContext());
1092 Ty = llvm::ArrayType::get(ElementType: Ty, NumElements: NumBytes.getQuantity());
1093 FrameFields.push_back(Elt: Ty);
1094 }
1095}
1096
1097static bool isArgInAlloca(const ABIArgInfo &Info) {
1098 // Leave ignored and inreg arguments alone.
1099 switch (Info.getKind()) {
1100 case ABIArgInfo::InAlloca:
1101 return true;
1102 case ABIArgInfo::Ignore:
1103 case ABIArgInfo::IndirectAliased:
1104 case ABIArgInfo::TargetSpecific:
1105 return false;
1106 case ABIArgInfo::Indirect:
1107 case ABIArgInfo::Direct:
1108 case ABIArgInfo::Extend:
1109 return !Info.getInReg();
1110 case ABIArgInfo::Expand:
1111 case ABIArgInfo::CoerceAndExpand:
1112 // These are aggregate types which are never passed in registers when
1113 // inalloca is involved.
1114 return true;
1115 }
1116 llvm_unreachable("invalid enum");
1117}
1118
1119void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const {
1120 assert(IsWin32StructABI && "inalloca only supported on win32");
1121
1122 // Build a packed struct type for all of the arguments in memory.
1123 SmallVector<llvm::Type *, 6> FrameFields;
1124
1125 // The stack alignment is always 4.
1126 CharUnits StackAlign = CharUnits::fromQuantity(Quantity: 4);
1127
1128 CharUnits StackOffset;
1129 CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end();
1130
1131 // Put 'this' into the struct before 'sret', if necessary.
1132 bool IsThisCall =
1133 FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall;
1134 ABIArgInfo &Ret = FI.getReturnInfo();
1135 if (Ret.isIndirect() && Ret.isSRetAfterThis() && !IsThisCall &&
1136 isArgInAlloca(Info: I->info)) {
1137 addFieldToArgStruct(FrameFields, StackOffset, Info&: I->info, Type: I->type);
1138 ++I;
1139 }
1140
1141 // Put the sret parameter into the inalloca struct if it's in memory.
1142 if (Ret.isIndirect() && !Ret.getInReg()) {
1143 addFieldToArgStruct(FrameFields, StackOffset, Info&: Ret, Type: FI.getReturnType());
1144 // On Windows, the hidden sret parameter is always returned in eax.
1145 Ret.setInAllocaSRet(IsWin32StructABI);
1146 }
1147
1148 // Skip the 'this' parameter in ecx.
1149 if (IsThisCall)
1150 ++I;
1151
1152 // Put arguments passed in memory into the struct.
1153 for (; I != E; ++I) {
1154 if (isArgInAlloca(Info: I->info))
1155 addFieldToArgStruct(FrameFields, StackOffset, Info&: I->info, Type: I->type);
1156 }
1157
1158 FI.setArgStruct(Ty: llvm::StructType::get(Context&: getVMContext(), Elements: FrameFields,
1159 /*isPacked=*/true),
1160 Align: StackAlign);
1161}
1162
1163RValue X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1164 QualType Ty, AggValueSlot Slot) const {
1165
1166 auto TypeInfo = getContext().getTypeInfoInChars(T: Ty);
1167
1168 CCState State(*const_cast<CGFunctionInfo *>(CGF.CurFnInfo));
1169 ABIArgInfo AI = classifyArgumentType(Ty, State, /*ArgIndex*/ 0);
1170 // Empty records are ignored for parameter passing purposes.
1171 if (AI.isIgnore())
1172 return Slot.asRValue();
1173
1174 // x86-32 changes the alignment of certain arguments on the stack.
1175 //
1176 // Just messing with TypeInfo like this works because we never pass
1177 // anything indirectly.
1178 TypeInfo.Align = CharUnits::fromQuantity(
1179 Quantity: getTypeStackAlignInBytes(Ty, Align: TypeInfo.Align.getQuantity()));
1180
1181 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, /*Indirect*/ IsIndirect: false, ValueInfo: TypeInfo,
1182 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 4),
1183 /*AllowHigherAlign*/ true, Slot);
1184}
1185
1186bool X86_32TargetCodeGenInfo::isStructReturnInRegABI(
1187 const llvm::Triple &Triple, const CodeGenOptions &Opts) {
1188 assert(Triple.getArch() == llvm::Triple::x86);
1189
1190 switch (Opts.getStructReturnConvention()) {
1191 case CodeGenOptions::SRCK_Default:
1192 break;
1193 case CodeGenOptions::SRCK_OnStack: // -fpcc-struct-return
1194 return false;
1195 case CodeGenOptions::SRCK_InRegs: // -freg-struct-return
1196 return true;
1197 }
1198
1199 if (Triple.isOSDarwin() || Triple.isOSIAMCU())
1200 return true;
1201
1202 switch (Triple.getOS()) {
1203 case llvm::Triple::DragonFly:
1204 case llvm::Triple::FreeBSD:
1205 case llvm::Triple::OpenBSD:
1206 case llvm::Triple::Win32:
1207 return true;
1208 default:
1209 return false;
1210 }
1211}
1212
1213static void addX86InterruptAttrs(const FunctionDecl *FD, llvm::GlobalValue *GV,
1214 CodeGen::CodeGenModule &CGM) {
1215 if (!FD->hasAttr<AnyX86InterruptAttr>())
1216 return;
1217
1218 llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1219 Fn->setCallingConv(llvm::CallingConv::X86_INTR);
1220 if (FD->getNumParams() == 0)
1221 return;
1222
1223 auto PtrTy = cast<PointerType>(Val: FD->getParamDecl(i: 0)->getType());
1224 llvm::Type *ByValTy = CGM.getTypes().ConvertType(T: PtrTy->getPointeeType());
1225 llvm::Attribute NewAttr = llvm::Attribute::getWithByValType(
1226 Context&: Fn->getContext(), Ty: ByValTy);
1227 Fn->addParamAttr(ArgNo: 0, Attr: NewAttr);
1228}
1229
1230void X86_32TargetCodeGenInfo::setTargetAttributes(
1231 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
1232 if (GV->isDeclaration())
1233 return;
1234 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D)) {
1235 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
1236 llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1237 Fn->addFnAttr(Kind: "stackrealign");
1238 }
1239
1240 addX86InterruptAttrs(FD, GV, CGM);
1241 }
1242}
1243
1244bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
1245 CodeGen::CodeGenFunction &CGF,
1246 llvm::Value *Address) const {
1247 CodeGen::CGBuilderTy &Builder = CGF.Builder;
1248
1249 llvm::Value *Four8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 4);
1250
1251 // 0-7 are the eight integer registers; the order is different
1252 // on Darwin (for EH), but the range is the same.
1253 // 8 is %eip.
1254 AssignToArrayRange(Builder, Array: Address, Value: Four8, FirstIndex: 0, LastIndex: 8);
1255
1256 if (CGF.CGM.getTarget().getTriple().isOSDarwin()) {
1257 // 12-16 are st(0..4). Not sure why we stop at 4.
1258 // These have size 16, which is sizeof(long double) on
1259 // platforms with 8-byte alignment for that type.
1260 llvm::Value *Sixteen8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 16);
1261 AssignToArrayRange(Builder, Array: Address, Value: Sixteen8, FirstIndex: 12, LastIndex: 16);
1262
1263 } else {
1264 // 9 is %eflags, which doesn't get a size on Darwin for some
1265 // reason.
1266 Builder.CreateAlignedStore(
1267 Val: Four8, Addr: Builder.CreateConstInBoundsGEP1_32(Ty: CGF.Int8Ty, Ptr: Address, Idx0: 9),
1268 Align: CharUnits::One());
1269
1270 // 11-16 are st(0..5). Not sure why we stop at 5.
1271 // These have size 12, which is sizeof(long double) on
1272 // platforms with 4-byte alignment for that type.
1273 llvm::Value *Twelve8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 12);
1274 AssignToArrayRange(Builder, Array: Address, Value: Twelve8, FirstIndex: 11, LastIndex: 16);
1275 }
1276
1277 return false;
1278}
1279
1280//===----------------------------------------------------------------------===//
1281// X86-64 ABI Implementation
1282//===----------------------------------------------------------------------===//
1283
1284
1285namespace {
1286
1287/// \p returns the size in bits of the largest (native) vector for \p AVXLevel.
1288static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
1289 switch (AVXLevel) {
1290 case X86AVXABILevel::AVX512:
1291 return 512;
1292 case X86AVXABILevel::AVX:
1293 return 256;
1294 case X86AVXABILevel::None:
1295 return 128;
1296 }
1297 llvm_unreachable("Unknown AVXLevel");
1298}
1299
1300/// X86_64ABIInfo - The X86_64 ABI information.
1301class X86_64ABIInfo : public ABIInfo {
1302 enum Class {
1303 Integer = 0,
1304 SSE,
1305 SSEUp,
1306 X87,
1307 X87Up,
1308 ComplexX87,
1309 NoClass,
1310 Memory
1311 };
1312
1313 /// merge - Implement the X86_64 ABI merging algorithm.
1314 ///
1315 /// Merge an accumulating classification \arg Accum with a field
1316 /// classification \arg Field.
1317 ///
1318 /// \param Accum - The accumulating classification. This should
1319 /// always be either NoClass or the result of a previous merge
1320 /// call. In addition, this should never be Memory (the caller
1321 /// should just return Memory for the aggregate).
1322 static Class merge(Class Accum, Class Field);
1323
1324 /// postMerge - Implement the X86_64 ABI post merging algorithm.
1325 ///
1326 /// Post merger cleanup, reduces a malformed Hi and Lo pair to
1327 /// final MEMORY or SSE classes when necessary.
1328 ///
1329 /// \param AggregateSize - The size of the current aggregate in
1330 /// the classification process.
1331 ///
1332 /// \param Lo - The classification for the parts of the type
1333 /// residing in the low word of the containing object.
1334 ///
1335 /// \param Hi - The classification for the parts of the type
1336 /// residing in the higher words of the containing object.
1337 ///
1338 void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
1339
1340 /// classify - Determine the x86_64 register classes in which the
1341 /// given type T should be passed.
1342 ///
1343 /// \param Lo - The classification for the parts of the type
1344 /// residing in the low word of the containing object.
1345 ///
1346 /// \param Hi - The classification for the parts of the type
1347 /// residing in the high word of the containing object.
1348 ///
1349 /// \param OffsetBase - The bit offset of this type in the
1350 /// containing object. Some parameters are classified different
1351 /// depending on whether they straddle an eightbyte boundary.
1352 ///
1353 /// \param isNamedArg - Whether the argument in question is a "named"
1354 /// argument, as used in AMD64-ABI 3.5.7.
1355 ///
1356 /// \param IsRegCall - Whether the calling conversion is regcall.
1357 ///
1358 /// If a word is unused its result will be NoClass; if a type should
1359 /// be passed in Memory then at least the classification of \arg Lo
1360 /// will be Memory.
1361 ///
1362 /// The \arg Lo class will be NoClass iff the argument is ignored.
1363 ///
1364 /// If the \arg Lo class is ComplexX87, then the \arg Hi class will
1365 /// also be ComplexX87.
1366 void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,
1367 bool isNamedArg, bool IsRegCall = false) const;
1368
1369 llvm::Type *GetByteVectorType(QualType Ty) const;
1370 llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
1371 unsigned IROffset, QualType SourceTy,
1372 unsigned SourceOffset) const;
1373 llvm::Type *GetINTEGERTypeAtOffset(llvm::Type *IRType,
1374 unsigned IROffset, QualType SourceTy,
1375 unsigned SourceOffset) const;
1376
1377 /// getIndirectResult - Give a source type \arg Ty, return a suitable result
1378 /// such that the argument will be returned in memory.
1379 ABIArgInfo getIndirectReturnResult(QualType Ty) const;
1380
1381 /// getIndirectResult - Give a source type \arg Ty, return a suitable result
1382 /// such that the argument will be passed in memory.
1383 ///
1384 /// \param freeIntRegs - The number of free integer registers remaining
1385 /// available.
1386 ABIArgInfo getIndirectResult(QualType Ty, unsigned freeIntRegs) const;
1387
1388 ABIArgInfo classifyReturnType(QualType RetTy) const;
1389
1390 ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs,
1391 unsigned &neededInt, unsigned &neededSSE,
1392 bool isNamedArg,
1393 bool IsRegCall = false) const;
1394
1395 ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
1396 unsigned &NeededSSE,
1397 unsigned &MaxVectorWidth) const;
1398
1399 bool passRegCallStructTypeDirectly(QualType Ty,
1400 SmallVectorImpl<llvm::Type *> &CoerceElts,
1401 unsigned &NeededInt, unsigned &NeededSSE,
1402 unsigned &MaxVectorWidth) const;
1403
1404 bool IsIllegalVectorType(QualType Ty) const;
1405
1406 /// The 0.98 ABI revision clarified a lot of ambiguities,
1407 /// unfortunately in ways that were not always consistent with
1408 /// certain previous compilers. In particular, platforms which
1409 /// required strict binary compatibility with older versions of GCC
1410 /// may need to exempt themselves.
1411 bool honorsRevision0_98() const {
1412 return !getTarget().getTriple().isOSDarwin();
1413 }
1414
1415 /// GCC classifies <1 x long long> as SSE but some platform ABIs choose to
1416 /// classify it as INTEGER (for compatibility with older clang compilers).
1417 bool classifyIntegerMMXAsSSE() const {
1418 // Clang <= 3.8 did not do this.
1419 if (getContext().getLangOpts().isCompatibleWith(
1420 Version: LangOptions::ClangABI::Ver3_8))
1421 return false;
1422
1423 const llvm::Triple &Triple = getTarget().getTriple();
1424 if (Triple.isOSDarwin() || Triple.isPS() || Triple.isOSFreeBSD())
1425 return false;
1426 return true;
1427 }
1428
1429 // GCC classifies vectors of __int128 as memory.
1430 bool passInt128VectorsInMem() const {
1431 // Clang <= 9.0 did not do this.
1432 if (getContext().getLangOpts().isCompatibleWith(
1433 Version: LangOptions::ClangABI::Ver9))
1434 return false;
1435
1436 const llvm::Triple &T = getTarget().getTriple();
1437 return T.isOSLinux() || T.isOSNetBSD();
1438 }
1439
1440 bool returnCXXRecordGreaterThan128InMem() const {
1441 // Clang <= 20.0 did not do this, and PlayStation does not do this.
1442 if (getContext().getLangOpts().isCompatibleWith(
1443 Version: LangOptions::ClangABI::Ver20) ||
1444 getTarget().getTriple().isPS())
1445 return false;
1446
1447 return true;
1448 }
1449
1450 X86AVXABILevel AVXLevel;
1451 // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
1452 // 64-bit hardware.
1453 bool Has64BitPointers;
1454
1455public:
1456 X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
1457 : ABIInfo(CGT), AVXLevel(AVXLevel),
1458 Has64BitPointers(CGT.getDataLayout().getPointerSize(AS: 0) == 8) {}
1459
1460 bool isPassedUsingAVXType(QualType type) const {
1461 unsigned neededInt, neededSSE;
1462 // The freeIntRegs argument doesn't matter here.
1463 ABIArgInfo info = classifyArgumentType(Ty: type, freeIntRegs: 0, neededInt, neededSSE,
1464 /*isNamedArg*/true);
1465 if (info.isDirect()) {
1466 llvm::Type *ty = info.getCoerceToType();
1467 if (llvm::VectorType *vectorTy = dyn_cast_or_null<llvm::VectorType>(Val: ty))
1468 return vectorTy->getPrimitiveSizeInBits().getFixedValue() > 128;
1469 }
1470 return false;
1471 }
1472
1473 void computeInfo(CGFunctionInfo &FI) const override;
1474 unsigned getX86ABIAVXLevel(const FunctionDecl *FD,
1475 const FunctionType::ExtInfo &Info) const override;
1476
1477 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
1478 AggValueSlot Slot) const override;
1479 RValue EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
1480 AggValueSlot Slot) const override;
1481
1482 bool has64BitPointers() const {
1483 return Has64BitPointers;
1484 }
1485};
1486
1487/// WinX86_64ABIInfo - The Windows X86_64 ABI information.
1488class WinX86_64ABIInfo : public ABIInfo {
1489public:
1490 WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
1491 : ABIInfo(CGT), AVXLevel(AVXLevel),
1492 IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {}
1493
1494 void computeInfo(CGFunctionInfo &FI) const override;
1495 unsigned getX86ABIAVXLevel(const FunctionDecl *FD,
1496 const FunctionType::ExtInfo &Info) const override;
1497
1498 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
1499 AggValueSlot Slot) const override;
1500
1501 bool isHomogeneousAggregateBaseType(QualType Ty) const override {
1502 // FIXME: Assumes vectorcall is in use.
1503 return isX86VectorTypeForVectorCall(Context&: getContext(), Ty);
1504 }
1505
1506 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
1507 uint64_t NumMembers) const override {
1508 // FIXME: Assumes vectorcall is in use.
1509 return isX86VectorCallAggregateSmallEnough(NumMembers);
1510 }
1511
1512 ABIArgInfo classifyArgForArm64ECVarArg(QualType Ty,
1513 bool IsNamedArg) const override {
1514 unsigned FreeSSERegs = 0;
1515 ClassifyKind Kind =
1516 IsNamedArg ? ClassifyKind::FixedArgument : ClassifyKind::VarArg;
1517 return classify(Ty, FreeSSERegs, Kind, CC: llvm::CallingConv::C);
1518 }
1519
1520private:
1521 enum class ClassifyKind { Return, FixedArgument, VarArg };
1522
1523 ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, ClassifyKind Kind,
1524 unsigned CC) const;
1525 ABIArgInfo reclassifyHvaArgForVectorCall(QualType Ty, unsigned &FreeSSERegs,
1526 const ABIArgInfo &current) const;
1527
1528 X86AVXABILevel AVXLevel;
1529
1530 bool IsMingw64;
1531};
1532
1533class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
1534public:
1535 X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
1536 : TargetCodeGenInfo(std::make_unique<X86_64ABIInfo>(args&: CGT, args&: AVXLevel)) {
1537 SwiftInfo =
1538 std::make_unique<SwiftABIInfo>(args&: CGT, /*SwiftErrorInRegister=*/args: true);
1539 }
1540
1541 /// Disable tail call on x86-64. The epilogue code before the tail jump blocks
1542 /// autoreleaseRV/retainRV and autoreleaseRV/unsafeClaimRV optimizations.
1543 bool markARCOptimizedReturnCallsAsNoTail() const override { return true; }
1544
1545 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1546 return 7;
1547 }
1548
1549 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1550 llvm::Value *Address) const override {
1551 llvm::Value *Eight8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 8);
1552
1553 // 0-15 are the 16 integer registers.
1554 // 16 is %rip.
1555 AssignToArrayRange(Builder&: CGF.Builder, Array: Address, Value: Eight8, FirstIndex: 0, LastIndex: 16);
1556 return false;
1557 }
1558
1559 llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1560 StringRef Constraint,
1561 llvm::Type* Ty) const override {
1562 return X86AdjustInlineAsmType(CGF, Constraint, Ty);
1563 }
1564
1565 bool isNoProtoCallVariadic(const CallArgList &args,
1566 const FunctionNoProtoType *fnType) const override {
1567 // The default CC on x86-64 sets %al to the number of SSA
1568 // registers used, and GCC sets this when calling an unprototyped
1569 // function, so we override the default behavior. However, don't do
1570 // that when AVX types are involved: the ABI explicitly states it is
1571 // undefined, and it doesn't work in practice because of how the ABI
1572 // defines varargs anyway.
1573 if (fnType->getCallConv() == CC_C) {
1574 bool HasAVXType = false;
1575 for (const CallArg &arg : args) {
1576 if (getABIInfo<X86_64ABIInfo>().isPassedUsingAVXType(type: arg.Ty)) {
1577 HasAVXType = true;
1578 break;
1579 }
1580 }
1581
1582 if (!HasAVXType)
1583 return true;
1584 }
1585
1586 return TargetCodeGenInfo::isNoProtoCallVariadic(args, fnType);
1587 }
1588
1589 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1590 CodeGen::CodeGenModule &CGM) const override {
1591 if (GV->isDeclaration())
1592 return;
1593 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D)) {
1594 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
1595 llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1596 Fn->addFnAttr(Kind: "stackrealign");
1597 }
1598
1599 addX86InterruptAttrs(FD, GV, CGM);
1600 }
1601 }
1602
1603 void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
1604 const FunctionDecl *Caller,
1605 const FunctionDecl *Callee, const CallArgList &Args,
1606 QualType ReturnType) const override;
1607
1608 void checkFunctionABI(CodeGenModule &CGM,
1609 const FunctionDecl *FD) const override;
1610};
1611} // namespace
1612
1613static void initFeatureMaps(const ASTContext &Ctx,
1614 llvm::StringMap<bool> &CallerMap,
1615 const FunctionDecl *Caller,
1616 llvm::StringMap<bool> &CalleeMap,
1617 const FunctionDecl *Callee) {
1618 if (CalleeMap.empty() && CallerMap.empty()) {
1619 // The caller is potentially nullptr in the case where the call isn't in a
1620 // function. In this case, the getFunctionFeatureMap ensures we just get
1621 // the TU level setting (since it cannot be modified by 'target'..
1622 Ctx.getFunctionFeatureMap(FeatureMap&: CallerMap, Caller);
1623 Ctx.getFunctionFeatureMap(FeatureMap&: CalleeMap, Callee);
1624 }
1625}
1626
1627static bool checkAVXParamFeature(DiagnosticsEngine &Diag,
1628 SourceLocation CallLoc,
1629 const FunctionDecl &Callee,
1630 const llvm::StringMap<bool> &CallerMap,
1631 const llvm::StringMap<bool> &CalleeMap,
1632 QualType Ty, StringRef Feature,
1633 bool IsArgument) {
1634 bool CallerHasFeat = CallerMap.lookup(Key: Feature);
1635 bool CalleeHasFeat = CalleeMap.lookup(Key: Feature);
1636 // No explicit features and the function is internal, be permissive.
1637 if (!CallerHasFeat && !CalleeHasFeat &&
1638 (!Callee.isExternallyVisible() || Callee.hasAttr<AlwaysInlineAttr>()))
1639 return false;
1640
1641 if (!CallerHasFeat && !CalleeHasFeat)
1642 return Diag.Report(Loc: CallLoc, DiagID: diag::warn_avx_calling_convention)
1643 << IsArgument << Ty << Feature;
1644
1645 // Mixing calling conventions here is very clearly an error.
1646 if (!CallerHasFeat || !CalleeHasFeat)
1647 return Diag.Report(Loc: CallLoc, DiagID: diag::err_avx_calling_convention)
1648 << IsArgument << Ty << Feature;
1649
1650 // Else, both caller and callee have the required feature, so there is no need
1651 // to diagnose.
1652 return false;
1653}
1654
1655static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,
1656 SourceLocation CallLoc, const FunctionDecl &Callee,
1657 const llvm::StringMap<bool> &CallerMap,
1658 const llvm::StringMap<bool> &CalleeMap, QualType Ty,
1659 bool IsArgument) {
1660 uint64_t Size = Ctx.getTypeSize(T: Ty);
1661 if (Size > 256)
1662 return checkAVXParamFeature(Diag, CallLoc, Callee, CallerMap, CalleeMap, Ty,
1663 Feature: "avx512f", IsArgument);
1664
1665 if (Size > 128)
1666 return checkAVXParamFeature(Diag, CallLoc, Callee, CallerMap, CalleeMap, Ty,
1667 Feature: "avx", IsArgument);
1668
1669 return false;
1670}
1671
1672void X86_64TargetCodeGenInfo::checkFunctionABI(CodeGenModule &CGM,
1673 const FunctionDecl *FD) const {
1674 auto GetReturnTypeLoc = [](const FunctionDecl *FD) {
1675 if (const TypeSourceInfo *TSI = FD->getTypeSourceInfo()) {
1676 TypeLoc TL = TSI->getTypeLoc();
1677
1678 if (auto FTL = TL.IgnoreParens().getAs<FunctionTypeLoc>()) {
1679 SourceLocation Loc = FTL.getReturnLoc().getBeginLoc();
1680 if (Loc.isValid())
1681 return Loc;
1682 }
1683 }
1684
1685 SourceLocation Loc = FD->getLocation();
1686 if (Loc.isValid())
1687 return Loc;
1688
1689 return FD->getBeginLoc();
1690 };
1691
1692 auto Check = [&](QualType Ty, SourceLocation Loc, bool IsReturn) {
1693 if (!Ty->isVectorType())
1694 return false;
1695 if (CGM.getContext().getTypeSize(T: Ty) <= 128)
1696 return false;
1697
1698 StringRef Feature =
1699 CGM.getContext().getTypeSize(T: Ty) > 256 ? "avx512f" : "avx";
1700
1701 llvm::StringMap<bool> FeatureMap;
1702 CGM.getContext().getFunctionFeatureMap(FeatureMap, FD);
1703 if (!FeatureMap.lookup(Key: Feature)) {
1704 CGM.getDiags().Report(Loc, DiagID: diag::warn_avx_calling_convention)
1705 << !IsReturn << Ty << Feature;
1706 return true;
1707 }
1708
1709 return false;
1710 };
1711
1712 // psABI warnings & errors for function definitions that are only visible
1713 // in this translation unit are handled at call site by checkFunctionCallABI.
1714 if (!FD->isExternallyVisible())
1715 return;
1716
1717 // First check the return type and emit diagnostic if required.
1718 Check(FD->getReturnType(), GetReturnTypeLoc(FD), true);
1719
1720 // Go through the parameters and emit a warning for the first vector found
1721 // without the matching function AVX level attribute.
1722 for (const ParmVarDecl *P : FD->parameters()) {
1723 SourceLocation Loc = P->getLocation();
1724 if (Loc.isInvalid())
1725 Loc = P->getBeginLoc();
1726 if (Check(P->getType(), Loc, false))
1727 return;
1728 }
1729}
1730
1731void X86_64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM,
1732 SourceLocation CallLoc,
1733 const FunctionDecl *Caller,
1734 const FunctionDecl *Callee,
1735 const CallArgList &Args,
1736 QualType ReturnType) const {
1737 if (!Callee)
1738 return;
1739
1740 llvm::StringMap<bool> CallerMap;
1741 llvm::StringMap<bool> CalleeMap;
1742 unsigned ArgIndex = 0;
1743
1744 // We need to loop through the actual call arguments rather than the
1745 // function's parameters, in case this variadic.
1746 for (const CallArg &Arg : Args) {
1747 // The "avx" feature changes how vectors >128 in size are passed. "avx512f"
1748 // additionally changes how vectors >256 in size are passed. Like GCC, we
1749 // warn when a function is called with an argument where this will change.
1750 // Unlike GCC, we also error when it is an obvious ABI mismatch, that is,
1751 // the caller and callee features are mismatched.
1752 // Unfortunately, we cannot do this diagnostic in SEMA, since the callee can
1753 // change its ABI with attribute-target after this call.
1754 if (Arg.getType()->isVectorType() &&
1755 CGM.getContext().getTypeSize(T: Arg.getType()) > 128) {
1756 initFeatureMaps(Ctx: CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
1757 QualType Ty = Arg.getType();
1758 // The CallArg seems to have desugared the type already, so for clearer
1759 // diagnostics, replace it with the type in the FunctionDecl if possible.
1760 if (ArgIndex < Callee->getNumParams())
1761 Ty = Callee->getParamDecl(i: ArgIndex)->getType();
1762
1763 if (checkAVXParam(Diag&: CGM.getDiags(), Ctx&: CGM.getContext(), CallLoc, Callee: *Callee,
1764 CallerMap, CalleeMap, Ty, /*IsArgument*/ true))
1765 return;
1766 }
1767 ++ArgIndex;
1768 }
1769
1770 // Check return always, as we don't have a good way of knowing in codegen
1771 // whether this value is used, tail-called, etc.
1772 if (Callee->getReturnType()->isVectorType() &&
1773 CGM.getContext().getTypeSize(T: Callee->getReturnType()) > 128) {
1774 initFeatureMaps(Ctx: CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
1775 checkAVXParam(Diag&: CGM.getDiags(), Ctx&: CGM.getContext(), CallLoc, Callee: *Callee, CallerMap,
1776 CalleeMap, Ty: Callee->getReturnType(),
1777 /*IsArgument*/ false);
1778 }
1779}
1780
1781std::string TargetCodeGenInfo::qualifyWindowsLibrary(StringRef Lib) {
1782 // If the argument does not end in .lib, automatically add the suffix.
1783 // If the argument contains a space, enclose it in quotes.
1784 // This matches the behavior of MSVC.
1785 bool Quote = Lib.contains(C: ' ');
1786 std::string ArgStr = Quote ? "\"" : "";
1787 ArgStr += Lib;
1788 if (!Lib.ends_with_insensitive(Suffix: ".lib") && !Lib.ends_with_insensitive(Suffix: ".a"))
1789 ArgStr += ".lib";
1790 ArgStr += Quote ? "\"" : "";
1791 return ArgStr;
1792}
1793
1794namespace {
1795class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo {
1796public:
1797 WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
1798 bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI,
1799 unsigned NumRegisterParameters)
1800 : X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI,
1801 Win32StructABI, NumRegisterParameters, false) {}
1802
1803 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1804 CodeGen::CodeGenModule &CGM) const override;
1805
1806 void getDependentLibraryOption(llvm::StringRef Lib,
1807 llvm::SmallString<24> &Opt) const override {
1808 Opt = "/DEFAULTLIB:";
1809 Opt += qualifyWindowsLibrary(Lib);
1810 }
1811
1812 void getDetectMismatchOption(llvm::StringRef Name,
1813 llvm::StringRef Value,
1814 llvm::SmallString<32> &Opt) const override {
1815 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
1816 }
1817};
1818} // namespace
1819
1820void WinX86_32TargetCodeGenInfo::setTargetAttributes(
1821 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
1822 X86_32TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
1823 if (GV->isDeclaration())
1824 return;
1825 addStackProbeTargetAttributes(D, GV, CGM);
1826}
1827
1828namespace {
1829class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
1830public:
1831 WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
1832 X86AVXABILevel AVXLevel)
1833 : TargetCodeGenInfo(std::make_unique<WinX86_64ABIInfo>(args&: CGT, args&: AVXLevel)) {
1834 SwiftInfo =
1835 std::make_unique<SwiftABIInfo>(args&: CGT, /*SwiftErrorInRegister=*/args: true);
1836 }
1837
1838 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1839 CodeGen::CodeGenModule &CGM) const override;
1840
1841 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1842 return 7;
1843 }
1844
1845 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1846 llvm::Value *Address) const override {
1847 llvm::Value *Eight8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 8);
1848
1849 // 0-15 are the 16 integer registers.
1850 // 16 is %rip.
1851 AssignToArrayRange(Builder&: CGF.Builder, Array: Address, Value: Eight8, FirstIndex: 0, LastIndex: 16);
1852 return false;
1853 }
1854
1855 void getDependentLibraryOption(llvm::StringRef Lib,
1856 llvm::SmallString<24> &Opt) const override {
1857 Opt = "/DEFAULTLIB:";
1858 Opt += qualifyWindowsLibrary(Lib);
1859 }
1860
1861 void getDetectMismatchOption(llvm::StringRef Name,
1862 llvm::StringRef Value,
1863 llvm::SmallString<32> &Opt) const override {
1864 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
1865 }
1866};
1867} // namespace
1868
1869void WinX86_64TargetCodeGenInfo::setTargetAttributes(
1870 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
1871 TargetCodeGenInfo::setTargetAttributes(D, GV, M&: CGM);
1872 if (GV->isDeclaration())
1873 return;
1874 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D)) {
1875 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
1876 llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1877 Fn->addFnAttr(Kind: "stackrealign");
1878 }
1879
1880 addX86InterruptAttrs(FD, GV, CGM);
1881 }
1882
1883 addStackProbeTargetAttributes(D, GV, CGM);
1884}
1885
1886void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo,
1887 Class &Hi) const {
1888 // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
1889 //
1890 // (a) If one of the classes is Memory, the whole argument is passed in
1891 // memory.
1892 //
1893 // (b) If X87UP is not preceded by X87, the whole argument is passed in
1894 // memory.
1895 //
1896 // (c) If the size of the aggregate exceeds two eightbytes and the first
1897 // eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
1898 // argument is passed in memory. NOTE: This is necessary to keep the
1899 // ABI working for processors that don't support the __m256 type.
1900 //
1901 // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
1902 //
1903 // Some of these are enforced by the merging logic. Others can arise
1904 // only with unions; for example:
1905 // union { _Complex double; unsigned; }
1906 //
1907 // Note that clauses (b) and (c) were added in 0.98.
1908 //
1909 if (Hi == Memory)
1910 Lo = Memory;
1911 if (Hi == X87Up && Lo != X87 && honorsRevision0_98())
1912 Lo = Memory;
1913 if (AggregateSize > 128 && (Lo != SSE || Hi != SSEUp))
1914 Lo = Memory;
1915 if (Hi == SSEUp && Lo != SSE)
1916 Hi = SSE;
1917}
1918
1919static X86AVXABILevel getEffectiveX86AVXABILevel(CodeGenTypes &CGT,
1920 X86AVXABILevel GlobalAVXLevel,
1921 const FunctionDecl *FD) {
1922 // Always return global AVX level on PlayStation.
1923 if (CGT.getTarget().getTriple().isPS() ||
1924 CGT.getContext().getLangOpts().getClangABICompat() <=
1925 LangOptions::ClangABI::Ver23) {
1926 return GlobalAVXLevel;
1927 }
1928
1929 X86AVXABILevel Level = GlobalAVXLevel;
1930 // TargetVersionAttr does not apply to x86.
1931 // FIXME: Handling TargetClonesAttr and CPUSpecificAttr is intentionally
1932 // deferred to a follow-up.
1933 if (!FD || !FD->hasAttr<TargetAttr>())
1934 return Level;
1935
1936 llvm::StringMap<bool> FeatureMap;
1937 CGT.getCGM().getContext().getFunctionFeatureMap(FeatureMap, FD);
1938 if (FeatureMap.lookup(Key: "avx512f"))
1939 return std::max(a: Level, b: X86AVXABILevel::AVX512);
1940 if (FeatureMap.lookup(Key: "avx"))
1941 return std::max(a: Level, b: X86AVXABILevel::AVX);
1942 return Level;
1943}
1944
1945X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
1946 // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
1947 // classified recursively so that always two fields are
1948 // considered. The resulting class is calculated according to
1949 // the classes of the fields in the eightbyte:
1950 //
1951 // (a) If both classes are equal, this is the resulting class.
1952 //
1953 // (b) If one of the classes is NO_CLASS, the resulting class is
1954 // the other class.
1955 //
1956 // (c) If one of the classes is MEMORY, the result is the MEMORY
1957 // class.
1958 //
1959 // (d) If one of the classes is INTEGER, the result is the
1960 // INTEGER.
1961 //
1962 // (e) If one of the classes is X87, X87UP, COMPLEX_X87 class,
1963 // MEMORY is used as class.
1964 //
1965 // (f) Otherwise class SSE is used.
1966
1967 // Accum should never be memory (we should have returned) or
1968 // ComplexX87 (because this cannot be passed in a structure).
1969 assert((Accum != Memory && Accum != ComplexX87) &&
1970 "Invalid accumulated classification during merge.");
1971 if (Accum == Field || Field == NoClass)
1972 return Accum;
1973 if (Field == Memory)
1974 return Memory;
1975 if (Accum == NoClass)
1976 return Field;
1977 if (Accum == Integer || Field == Integer)
1978 return Integer;
1979 if (Field == X87 || Field == X87Up || Field == ComplexX87 ||
1980 Accum == X87 || Accum == X87Up)
1981 return Memory;
1982 return SSE;
1983}
1984
1985void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
1986 Class &Hi, bool isNamedArg, bool IsRegCall) const {
1987 // FIXME: This code can be simplified by introducing a simple value class for
1988 // Class pairs with appropriate constructor methods for the various
1989 // situations.
1990
1991 // FIXME: Some of the split computations are wrong; unaligned vectors
1992 // shouldn't be passed in registers for example, so there is no chance they
1993 // can straddle an eightbyte. Verify & simplify.
1994
1995 Lo = Hi = NoClass;
1996
1997 Class &Current = OffsetBase < 64 ? Lo : Hi;
1998 Current = Memory;
1999
2000 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
2001 BuiltinType::Kind k = BT->getKind();
2002
2003 if (k == BuiltinType::Void) {
2004 Current = NoClass;
2005 } else if (k == BuiltinType::Int128 || k == BuiltinType::UInt128) {
2006 Lo = Integer;
2007 Hi = Integer;
2008 } else if (k >= BuiltinType::Bool && k <= BuiltinType::LongLong) {
2009 Current = Integer;
2010 } else if (k == BuiltinType::Float || k == BuiltinType::Double ||
2011 k == BuiltinType::Float16 || k == BuiltinType::BFloat16) {
2012 Current = SSE;
2013 } else if (k == BuiltinType::Float128) {
2014 Lo = SSE;
2015 Hi = SSEUp;
2016 } else if (k == BuiltinType::LongDouble) {
2017 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2018 if (LDF == &llvm::APFloat::IEEEquad()) {
2019 Lo = SSE;
2020 Hi = SSEUp;
2021 } else if (LDF == &llvm::APFloat::x87DoubleExtended()) {
2022 Lo = X87;
2023 Hi = X87Up;
2024 } else if (LDF == &llvm::APFloat::IEEEdouble()) {
2025 Current = SSE;
2026 } else
2027 llvm_unreachable("unexpected long double representation!");
2028 }
2029 // FIXME: _Decimal32 and _Decimal64 are SSE.
2030 // FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
2031 return;
2032 }
2033
2034 if (const auto *ED = Ty->getAsEnumDecl()) {
2035 // Classify the underlying integer type.
2036 classify(Ty: ED->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);
2037 return;
2038 }
2039
2040 if (Ty->hasPointerRepresentation()) {
2041 Current = Integer;
2042 return;
2043 }
2044
2045 if (Ty->isMemberPointerType()) {
2046 if (Ty->isMemberFunctionPointerType()) {
2047 if (Has64BitPointers) {
2048 // If Has64BitPointers, this is an {i64, i64}, so classify both
2049 // Lo and Hi now.
2050 Lo = Hi = Integer;
2051 } else {
2052 // Otherwise, with 32-bit pointers, this is an {i32, i32}. If that
2053 // straddles an eightbyte boundary, Hi should be classified as well.
2054 uint64_t EB_FuncPtr = (OffsetBase) / 64;
2055 uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64;
2056 if (EB_FuncPtr != EB_ThisAdj) {
2057 Lo = Hi = Integer;
2058 } else {
2059 Current = Integer;
2060 }
2061 }
2062 } else {
2063 Current = Integer;
2064 }
2065 return;
2066 }
2067
2068 if (const VectorType *VT = Ty->getAs<VectorType>()) {
2069 uint64_t Size = getContext().getTypeSize(T: VT);
2070 if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
2071 // gcc passes the following as integer:
2072 // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
2073 // 2 bytes - <2 x char>, <1 x short>
2074 // 1 byte - <1 x char>
2075 Current = Integer;
2076
2077 // If this type crosses an eightbyte boundary, it should be
2078 // split.
2079 uint64_t EB_Lo = (OffsetBase) / 64;
2080 uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
2081 if (EB_Lo != EB_Hi)
2082 Hi = Lo;
2083 } else if (Size == 64) {
2084 QualType ElementType = VT->getElementType();
2085
2086 // gcc passes <1 x double> in memory. :(
2087 if (ElementType->isSpecificBuiltinType(K: BuiltinType::Double))
2088 return;
2089
2090 // gcc passes <1 x long long> as SSE but clang used to unconditionally
2091 // pass them as integer. For platforms where clang is the de facto
2092 // platform compiler, we must continue to use integer.
2093 if (!classifyIntegerMMXAsSSE() &&
2094 (ElementType->isSpecificBuiltinType(K: BuiltinType::LongLong) ||
2095 ElementType->isSpecificBuiltinType(K: BuiltinType::ULongLong) ||
2096 ElementType->isSpecificBuiltinType(K: BuiltinType::Long) ||
2097 ElementType->isSpecificBuiltinType(K: BuiltinType::ULong)))
2098 Current = Integer;
2099 else
2100 Current = SSE;
2101
2102 // If this type crosses an eightbyte boundary, it should be
2103 // split.
2104 if (OffsetBase && OffsetBase != 64)
2105 Hi = Lo;
2106 } else if (Size == 128 ||
2107 (isNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
2108 QualType ElementType = VT->getElementType();
2109
2110 // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
2111 if (passInt128VectorsInMem() && Size != 128 &&
2112 (ElementType->isSpecificBuiltinType(K: BuiltinType::Int128) ||
2113 ElementType->isSpecificBuiltinType(K: BuiltinType::UInt128)))
2114 return;
2115
2116 // Arguments of 256-bits are split into four eightbyte chunks. The
2117 // least significant one belongs to class SSE and all the others to class
2118 // SSEUP. The original Lo and Hi design considers that types can't be
2119 // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
2120 // This design isn't correct for 256-bits, but since there're no cases
2121 // where the upper parts would need to be inspected, avoid adding
2122 // complexity and just consider Hi to match the 64-256 part.
2123 //
2124 // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
2125 // registers if they are "named", i.e. not part of the "..." of a
2126 // variadic function.
2127 //
2128 // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
2129 // split into eight eightbyte chunks, one SSE and seven SSEUP.
2130 Lo = SSE;
2131 Hi = SSEUp;
2132 }
2133 return;
2134 }
2135
2136 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
2137 QualType ET = getContext().getCanonicalType(T: CT->getElementType());
2138
2139 uint64_t Size = getContext().getTypeSize(T: Ty);
2140 if (ET->isIntegralOrEnumerationType()) {
2141 if (Size <= 64)
2142 Current = Integer;
2143 else if (Size <= 128)
2144 Lo = Hi = Integer;
2145 } else if (ET->isFloat16Type() || ET == getContext().FloatTy ||
2146 ET->isBFloat16Type()) {
2147 Current = SSE;
2148 } else if (ET == getContext().DoubleTy) {
2149 Lo = Hi = SSE;
2150 } else if (ET == getContext().LongDoubleTy) {
2151 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2152 if (LDF == &llvm::APFloat::IEEEquad())
2153 Current = Memory;
2154 else if (LDF == &llvm::APFloat::x87DoubleExtended())
2155 Current = ComplexX87;
2156 else if (LDF == &llvm::APFloat::IEEEdouble())
2157 Lo = Hi = SSE;
2158 else
2159 llvm_unreachable("unexpected long double representation!");
2160 }
2161
2162 // If this complex type crosses an eightbyte boundary then it
2163 // should be split.
2164 uint64_t EB_Real = (OffsetBase) / 64;
2165 uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(T: ET)) / 64;
2166 if (Hi == NoClass && EB_Real != EB_Imag)
2167 Hi = Lo;
2168
2169 return;
2170 }
2171
2172 if (const auto *EITy = Ty->getAs<BitIntType>()) {
2173 if (EITy->getNumBits() <= 64)
2174 Current = Integer;
2175 else if (EITy->getNumBits() <= 128)
2176 Lo = Hi = Integer;
2177 // Larger values need to get passed in memory.
2178 return;
2179 }
2180
2181 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(T: Ty)) {
2182 // Arrays are treated like structures.
2183
2184 uint64_t Size = getContext().getTypeSize(T: Ty);
2185
2186 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
2187 // than eight eightbytes, ..., it has class MEMORY.
2188 // regcall ABI doesn't have limitation to an object. The only limitation
2189 // is the free registers, which will be checked in computeInfo.
2190 if (!IsRegCall && Size > 512)
2191 return;
2192
2193 // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
2194 // fields, it has class MEMORY.
2195 //
2196 // Only need to check alignment of array base.
2197 if (OffsetBase % getContext().getTypeAlign(T: AT->getElementType()))
2198 return;
2199
2200 // Otherwise implement simplified merge. We could be smarter about
2201 // this, but it isn't worth it and would be harder to verify.
2202 Current = NoClass;
2203 uint64_t EltSize = getContext().getTypeSize(T: AT->getElementType());
2204 uint64_t ArraySize = AT->getZExtSize();
2205
2206 // The only case a 256-bit wide vector could be used is when the array
2207 // contains a single 256-bit element. Since Lo and Hi logic isn't extended
2208 // to work for sizes wider than 128, early check and fallback to memory.
2209 //
2210 if (Size > 128 &&
2211 (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))
2212 return;
2213
2214 for (uint64_t i=0, Offset=OffsetBase; i<ArraySize; ++i, Offset += EltSize) {
2215 Class FieldLo, FieldHi;
2216 classify(Ty: AT->getElementType(), OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, isNamedArg);
2217 Lo = merge(Accum: Lo, Field: FieldLo);
2218 Hi = merge(Accum: Hi, Field: FieldHi);
2219 if (Lo == Memory || Hi == Memory)
2220 break;
2221 }
2222
2223 postMerge(AggregateSize: Size, Lo, Hi);
2224 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
2225 return;
2226 }
2227
2228 if (const RecordType *RT = Ty->getAsCanonical<RecordType>()) {
2229 uint64_t Size = getContext().getTypeSize(T: Ty);
2230
2231 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
2232 // than eight eightbytes, ..., it has class MEMORY.
2233 if (Size > 512)
2234 return;
2235
2236 // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
2237 // copy constructor or a non-trivial destructor, it is passed by invisible
2238 // reference.
2239 if (getRecordArgABI(RT, CXXABI&: getCXXABI()))
2240 return;
2241
2242 const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf();
2243
2244 // Assume variable sized types are passed in memory.
2245 if (RD->hasFlexibleArrayMember())
2246 return;
2247
2248 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: RD);
2249
2250 // Reset Lo class, this will be recomputed.
2251 Current = NoClass;
2252
2253 // If this is a C++ record, classify the bases first.
2254 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
2255 for (const auto &I : CXXRD->bases()) {
2256 assert(!I.isVirtual() && !I.getType()->isDependentType() &&
2257 "Unexpected base class!");
2258 const auto *Base = I.getType()->castAsCXXRecordDecl();
2259 // Classify this field.
2260 //
2261 // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
2262 // single eightbyte, each is classified separately. Each eightbyte gets
2263 // initialized to class NO_CLASS.
2264 Class FieldLo, FieldHi;
2265 uint64_t Offset =
2266 OffsetBase + getContext().toBits(CharSize: Layout.getBaseClassOffset(Base));
2267 classify(Ty: I.getType(), OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, isNamedArg);
2268 Lo = merge(Accum: Lo, Field: FieldLo);
2269 Hi = merge(Accum: Hi, Field: FieldHi);
2270 if (returnCXXRecordGreaterThan128InMem() &&
2271 !isEmptyRecord(Context&: getContext(), T: I.getType(), AllowArrays: true) &&
2272 (Size > 128 && (Size != getContext().getTypeSize(T: I.getType()) ||
2273 Size > getNativeVectorSizeForAVXABI(AVXLevel)))) {
2274 // The only case a 256(or 512)-bit wide vector could be used to return
2275 // is when CXX record contains a single 256(or 512)-bit element.
2276 Lo = Memory;
2277 }
2278 if (Lo == Memory || Hi == Memory) {
2279 postMerge(AggregateSize: Size, Lo, Hi);
2280 return;
2281 }
2282 }
2283 }
2284
2285 // Classify the fields one at a time, merging the results.
2286 unsigned idx = 0;
2287 bool UseClang11Compat = getContext().getLangOpts().isCompatibleWith(
2288 Version: LangOptions::ClangABI::Ver11) ||
2289 getContext().getTargetInfo().getTriple().isPS();
2290 bool ClassifyUnnamedBitFields =
2291 getContext().getLangOpts().getClangABICompat() >
2292 LangOptions::ClangABI::Ver23 &&
2293 !getContext().getTargetInfo().getTriple().isPS();
2294 bool IsUnion = RT->isUnionType() && !UseClang11Compat;
2295
2296 for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
2297 i != e; ++i, ++idx) {
2298 uint64_t Offset = OffsetBase + Layout.getFieldOffset(FieldNo: idx);
2299 bool BitField = i->isBitField();
2300
2301 // Ignore padding bit-fields. Normally only zero-length bit-fields are
2302 // padding, but under -fclang-abi-compat=23 every unnamed bit-field is,
2303 // faithfully reproducing Clang 23 -- including its crash on aggregates
2304 // where skipping one leaves part of a wider access unit (e.g. an
2305 // __int128 bit-field run) unclassified.
2306 if (BitField && (ClassifyUnnamedBitFields ? i->isZeroLengthBitField()
2307 : i->isUnnamedBitField()))
2308 continue;
2309
2310 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than
2311 // eight eightbytes, or it contains unaligned fields, it has class MEMORY.
2312 //
2313 // The only case a 256-bit or a 512-bit wide vector could be used is when
2314 // the struct contains a single 256-bit or 512-bit element. Early check
2315 // and fallback to memory.
2316 //
2317 // FIXME: Extended the Lo and Hi logic properly to work for size wider
2318 // than 128.
2319 if (Size > 128 &&
2320 ((!IsUnion && Size != getContext().getTypeSize(T: i->getType())) ||
2321 Size > getNativeVectorSizeForAVXABI(AVXLevel))) {
2322 Lo = Memory;
2323 postMerge(AggregateSize: Size, Lo, Hi);
2324 return;
2325 }
2326
2327 bool IsInMemory =
2328 Offset % getContext().getTypeAlign(T: i->getType().getCanonicalType());
2329 // Note, skip this test for bit-fields, see below.
2330 if (!BitField && IsInMemory) {
2331 Lo = Memory;
2332 postMerge(AggregateSize: Size, Lo, Hi);
2333 return;
2334 }
2335
2336 // Classify this field.
2337 //
2338 // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
2339 // exceeds a single eightbyte, each is classified
2340 // separately. Each eightbyte gets initialized to class
2341 // NO_CLASS.
2342 Class FieldLo, FieldHi;
2343
2344 // Bit-fields require special handling, they do not force the
2345 // structure to be passed in memory even if unaligned, and
2346 // therefore they can straddle an eightbyte.
2347 if (BitField) {
2348 assert(ClassifyUnnamedBitFields ? !i->isZeroLengthBitField()
2349 : !i->isUnnamedBitField());
2350 uint64_t Offset = OffsetBase + Layout.getFieldOffset(FieldNo: idx);
2351 uint64_t Size = i->getBitWidthValue();
2352
2353 uint64_t EB_Lo = Offset / 64;
2354 uint64_t EB_Hi = (Offset + Size - 1) / 64;
2355
2356 if (EB_Lo) {
2357 assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
2358 FieldLo = NoClass;
2359 FieldHi = Integer;
2360 } else {
2361 FieldLo = Integer;
2362 FieldHi = EB_Hi ? Integer : NoClass;
2363 }
2364 } else
2365 classify(Ty: i->getType(), OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, isNamedArg);
2366 Lo = merge(Accum: Lo, Field: FieldLo);
2367 Hi = merge(Accum: Hi, Field: FieldHi);
2368 if (Lo == Memory || Hi == Memory)
2369 break;
2370 }
2371
2372 postMerge(AggregateSize: Size, Lo, Hi);
2373 }
2374}
2375
2376ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
2377 // If this is a scalar LLVM value then assume LLVM will pass it in the right
2378 // place naturally.
2379 if (!isAggregateTypeForABI(T: Ty)) {
2380 // Treat an enum type as its underlying type.
2381 if (const auto *ED = Ty->getAsEnumDecl())
2382 Ty = ED->getIntegerType();
2383
2384 if (Ty->isBitIntType())
2385 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace());
2386
2387 llvm::Type *IRTy = CGT.ConvertType(T: Ty);
2388 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty, T: IRTy)
2389 : ABIArgInfo::getDirect(T: IRTy));
2390 }
2391
2392 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace());
2393}
2394
2395bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {
2396 if (const VectorType *VecTy = Ty->getAs<VectorType>()) {
2397 uint64_t Size = getContext().getTypeSize(T: VecTy);
2398 unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
2399 if (Size <= 64 || Size > LargestVector)
2400 return true;
2401 QualType EltTy = VecTy->getElementType();
2402 if (passInt128VectorsInMem() &&
2403 (EltTy->isSpecificBuiltinType(K: BuiltinType::Int128) ||
2404 EltTy->isSpecificBuiltinType(K: BuiltinType::UInt128)))
2405 return true;
2406 }
2407
2408 return false;
2409}
2410
2411ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
2412 unsigned freeIntRegs) const {
2413 // If this is a scalar LLVM value then assume LLVM will pass it in the right
2414 // place naturally.
2415 //
2416 // This assumption is optimistic, as there could be free registers available
2417 // when we need to pass this argument in memory, and LLVM could try to pass
2418 // the argument in the free register. This does not seem to happen currently,
2419 // but this code would be much safer if we could mark the argument with
2420 // 'onstack'. See PR12193.
2421 if (!isAggregateTypeForABI(T: Ty) && !IsIllegalVectorType(Ty) &&
2422 !Ty->isBitIntType()) {
2423 // Treat an enum type as its underlying type.
2424 if (const auto *ED = Ty->getAsEnumDecl())
2425 Ty = ED->getIntegerType();
2426
2427 llvm::Type *IRTy = CGT.ConvertType(T: Ty);
2428 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty, T: IRTy)
2429 : ABIArgInfo::getDirect(T: IRTy));
2430 }
2431
2432 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(T: Ty, CXXABI&: getCXXABI()))
2433 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
2434 ByVal: RAA == CGCXXABI::RAA_DirectInMemory);
2435
2436 // Compute the byval alignment. We specify the alignment of the byval in all
2437 // cases so that the mid-level optimizer knows the alignment of the byval.
2438 unsigned Align = std::max(a: getContext().getTypeAlign(T: Ty) / 8, b: 8U);
2439
2440 // Attempt to avoid passing indirect results using byval when possible. This
2441 // is important for good codegen.
2442 //
2443 // We do this by coercing the value into a scalar type which the backend can
2444 // handle naturally (i.e., without using byval).
2445 //
2446 // For simplicity, we currently only do this when we have exhausted all of the
2447 // free integer registers. Doing this when there are free integer registers
2448 // would require more care, as we would have to ensure that the coerced value
2449 // did not claim the unused register. That would require either reording the
2450 // arguments to the function (so that any subsequent inreg values came first),
2451 // or only doing this optimization when there were no following arguments that
2452 // might be inreg.
2453 //
2454 // We currently expect it to be rare (particularly in well written code) for
2455 // arguments to be passed on the stack when there are still free integer
2456 // registers available (this would typically imply large structs being passed
2457 // by value), so this seems like a fair tradeoff for now.
2458 //
2459 // We can revisit this if the backend grows support for 'onstack' parameter
2460 // attributes. See PR12193.
2461 if (freeIntRegs == 0) {
2462 uint64_t Size = getContext().getTypeSize(T: Ty);
2463
2464 // If this type fits in an eightbyte, coerce it into the matching integral
2465 // type, which will end up on the stack (with alignment 8).
2466 if (Align == 8 && Size <= 64)
2467 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(),
2468 NumBits: Size));
2469 }
2470
2471 return ABIArgInfo::getIndirect(Alignment: CharUnits::fromQuantity(Quantity: Align),
2472 AddrSpace: getDataLayout().getAllocaAddrSpace());
2473}
2474
2475/// The ABI specifies that a value should be passed in a full vector XMM/YMM
2476/// register. Pick an LLVM IR type that will be passed as a vector register.
2477llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {
2478 // Wrapper structs/arrays that only contain vectors are passed just like
2479 // vectors; strip them off if present.
2480 if (const Type *InnerTy = isSingleElementStruct(T: Ty, Context&: getContext()))
2481 Ty = QualType(InnerTy, 0);
2482
2483 llvm::Type *IRType = CGT.ConvertType(T: Ty);
2484 if (isa<llvm::VectorType>(Val: IRType)) {
2485 // Don't pass vXi128 vectors in their native type, the backend can't
2486 // legalize them.
2487 if (passInt128VectorsInMem() &&
2488 cast<llvm::VectorType>(Val: IRType)->getElementType()->isIntegerTy(BitWidth: 128)) {
2489 // Use a vXi64 vector.
2490 uint64_t Size = getContext().getTypeSize(T: Ty);
2491 return llvm::FixedVectorType::get(ElementType: llvm::Type::getInt64Ty(C&: getVMContext()),
2492 NumElts: Size / 64);
2493 }
2494
2495 return IRType;
2496 }
2497
2498 if (IRType->getTypeID() == llvm::Type::FP128TyID)
2499 return IRType;
2500
2501 // We couldn't find the preferred IR vector type for 'Ty'.
2502 uint64_t Size = getContext().getTypeSize(T: Ty);
2503 assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!");
2504
2505
2506 // Return a LLVM IR vector type based on the size of 'Ty'.
2507 return llvm::FixedVectorType::get(ElementType: llvm::Type::getDoubleTy(C&: getVMContext()),
2508 NumElts: Size / 64);
2509}
2510
2511/// BitsContainNoUserData - Return true if the specified [start,end) bit range
2512/// is known to either be off the end of the specified type or being in
2513/// alignment padding. The user type specified is known to be at most 128 bits
2514/// in size, and have passed through X86_64ABIInfo::classify with a successful
2515/// classification that put one of the two halves in the INTEGER class.
2516///
2517/// It is conservatively correct to return false.
2518static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
2519 unsigned EndBit, ASTContext &Context) {
2520 // If the bytes being queried are off the end of the type, there is no user
2521 // data hiding here. This handles analysis of builtins, vectors and other
2522 // types that don't contain interesting padding.
2523 unsigned TySize = (unsigned)Context.getTypeSize(T: Ty);
2524 if (TySize <= StartBit)
2525 return true;
2526
2527 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty)) {
2528 unsigned EltSize = (unsigned)Context.getTypeSize(T: AT->getElementType());
2529 unsigned NumElts = (unsigned)AT->getZExtSize();
2530
2531 // Check each element to see if the element overlaps with the queried range.
2532 for (unsigned i = 0; i != NumElts; ++i) {
2533 // If the element is after the span we care about, then we're done..
2534 unsigned EltOffset = i*EltSize;
2535 if (EltOffset >= EndBit) break;
2536
2537 unsigned EltStart = EltOffset < StartBit ? StartBit-EltOffset :0;
2538 if (!BitsContainNoUserData(Ty: AT->getElementType(), StartBit: EltStart,
2539 EndBit: EndBit-EltOffset, Context))
2540 return false;
2541 }
2542 // If it overlaps no elements, then it is safe to process as padding.
2543 return true;
2544 }
2545
2546 if (const auto *RD = Ty->getAsRecordDecl()) {
2547 const ASTRecordLayout &Layout = Context.getASTRecordLayout(D: RD);
2548
2549 // If this is a C++ record, check the bases first.
2550 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
2551 for (const auto &I : CXXRD->bases()) {
2552 assert(!I.isVirtual() && !I.getType()->isDependentType() &&
2553 "Unexpected base class!");
2554 const auto *Base = I.getType()->castAsCXXRecordDecl();
2555
2556 // If the base is after the span we care about, ignore it.
2557 unsigned BaseOffset = Context.toBits(CharSize: Layout.getBaseClassOffset(Base));
2558 if (BaseOffset >= EndBit) continue;
2559
2560 unsigned BaseStart = BaseOffset < StartBit ? StartBit-BaseOffset :0;
2561 if (!BitsContainNoUserData(Ty: I.getType(), StartBit: BaseStart,
2562 EndBit: EndBit-BaseOffset, Context))
2563 return false;
2564 }
2565 }
2566
2567 // Verify that no field has data that overlaps the region of interest. Yes
2568 // this could be sped up a lot by being smarter about queried fields,
2569 // however we're only looking at structs up to 16 bytes, so we don't care
2570 // much.
2571 unsigned idx = 0;
2572 for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
2573 i != e; ++i, ++idx) {
2574 unsigned FieldOffset = (unsigned)Layout.getFieldOffset(FieldNo: idx);
2575
2576 // If we found a field after the region we care about, then we're done.
2577 if (FieldOffset >= EndBit) break;
2578
2579 unsigned FieldStart = FieldOffset < StartBit ? StartBit-FieldOffset :0;
2580 if (!BitsContainNoUserData(Ty: i->getType(), StartBit: FieldStart, EndBit: EndBit-FieldOffset,
2581 Context))
2582 return false;
2583 }
2584
2585 // If nothing in this record overlapped the area of interest, then we're
2586 // clean.
2587 return true;
2588 }
2589
2590 return false;
2591}
2592
2593/// getFPTypeAtOffset - Return a floating point type at the specified offset.
2594static llvm::Type *getFPTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
2595 const llvm::DataLayout &TD) {
2596 if (IROffset == 0 && IRType->isFloatingPointTy())
2597 return IRType;
2598
2599 // If this is a struct, recurse into the field at the specified offset.
2600 if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: IRType)) {
2601 if (!STy->getNumContainedTypes())
2602 return nullptr;
2603
2604 const llvm::StructLayout *SL = TD.getStructLayout(Ty: STy);
2605 unsigned Elt = SL->getElementContainingOffset(FixedOffset: IROffset);
2606 IROffset -= SL->getElementOffset(Idx: Elt);
2607 return getFPTypeAtOffset(IRType: STy->getElementType(N: Elt), IROffset, TD);
2608 }
2609
2610 // If this is an array, recurse into the field at the specified offset.
2611 if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(Val: IRType)) {
2612 llvm::Type *EltTy = ATy->getElementType();
2613 unsigned EltSize = TD.getTypeAllocSize(Ty: EltTy);
2614 if (EltSize == 0)
2615 return nullptr;
2616 IROffset -= IROffset / EltSize * EltSize;
2617 return getFPTypeAtOffset(IRType: EltTy, IROffset, TD);
2618 }
2619
2620 return nullptr;
2621}
2622
2623/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
2624/// low 8 bytes of an XMM register, corresponding to the SSE class.
2625llvm::Type *X86_64ABIInfo::
2626GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
2627 QualType SourceTy, unsigned SourceOffset) const {
2628 const llvm::DataLayout &TD = getDataLayout();
2629 unsigned SourceSize =
2630 (unsigned)getContext().getTypeSize(T: SourceTy) / 8 - SourceOffset;
2631 llvm::Type *T0 = getFPTypeAtOffset(IRType, IROffset, TD);
2632 if (!T0 || T0->isDoubleTy())
2633 return llvm::Type::getDoubleTy(C&: getVMContext());
2634
2635 // Get the adjacent FP type.
2636 llvm::Type *T1 = nullptr;
2637 unsigned T0Size = TD.getTypeAllocSize(Ty: T0);
2638 if (SourceSize > T0Size)
2639 T1 = getFPTypeAtOffset(IRType, IROffset: IROffset + T0Size, TD);
2640 if (T1 == nullptr) {
2641 // Check if IRType is a half/bfloat + float. float type will be in IROffset+4 due
2642 // to its alignment.
2643 if (T0->is16bitFPTy() && SourceSize > 4)
2644 T1 = getFPTypeAtOffset(IRType, IROffset: IROffset + 4, TD);
2645 // If we can't get a second FP type, return a simple half or float.
2646 // avx512fp16-abi.c:pr51813_2 shows it works to return float for
2647 // {float, i8} too.
2648 if (T1 == nullptr)
2649 return T0;
2650 }
2651
2652 if (T0->isFloatTy() && T1->isFloatTy())
2653 return llvm::FixedVectorType::get(ElementType: T0, NumElts: 2);
2654
2655 if (T0->is16bitFPTy() && T1->is16bitFPTy()) {
2656 llvm::Type *T2 = nullptr;
2657 if (SourceSize > 4)
2658 T2 = getFPTypeAtOffset(IRType, IROffset: IROffset + 4, TD);
2659 if (T2 == nullptr)
2660 return llvm::FixedVectorType::get(ElementType: T0, NumElts: 2);
2661 return llvm::FixedVectorType::get(ElementType: T0, NumElts: 4);
2662 }
2663
2664 if (T0->is16bitFPTy() || T1->is16bitFPTy())
2665 return llvm::FixedVectorType::get(ElementType: llvm::Type::getHalfTy(C&: getVMContext()), NumElts: 4);
2666
2667 return llvm::Type::getDoubleTy(C&: getVMContext());
2668}
2669
2670/// GetINTEGERTypeAtOffset - The ABI specifies that a value should be passed in
2671/// one or more 8-byte GPRs. This means that we either have a scalar or we are
2672/// talking about the high and/or low part of an up-to-16-byte struct. This
2673/// routine picks the best LLVM IR type to represent this, which may be i64 or
2674/// may be anything else that the backend will pass in GPRs that works better
2675/// (e.g. i8, %foo*, etc).
2676///
2677/// PrefType is an LLVM IR type that corresponds to (part of) the IR type for
2678/// the source type. IROffset is an offset in bytes into the LLVM IR type that
2679/// the 8-byte value references. PrefType may be null.
2680///
2681/// SourceTy is the source-level type for the entire argument. SourceOffset is
2682/// an offset into this that we're processing (which is always either 0 or 8).
2683///
2684llvm::Type *X86_64ABIInfo::
2685GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
2686 QualType SourceTy, unsigned SourceOffset) const {
2687 // If we're dealing with an un-offset LLVM IR type, then it means that we're
2688 // returning an 8-byte unit starting with it. See if we can safely use it.
2689 if (IROffset == 0) {
2690 // Pointers and int64's always fill the 8-byte unit.
2691 if ((isa<llvm::PointerType>(Val: IRType) && Has64BitPointers) ||
2692 IRType->isIntegerTy(BitWidth: 64))
2693 return IRType;
2694
2695 // If we have a 1/2/4-byte integer, we can use it only if the rest of the
2696 // goodness in the source type is just tail padding. This is allowed to
2697 // kick in for struct {double,int} on the int, but not on
2698 // struct{double,int,int} because we wouldn't return the second int. We
2699 // have to do this analysis on the source type because we can't depend on
2700 // unions being lowered a specific way etc.
2701 if (IRType->isIntegerTy(BitWidth: 8) || IRType->isIntegerTy(BitWidth: 16) ||
2702 IRType->isIntegerTy(BitWidth: 32) ||
2703 (isa<llvm::PointerType>(Val: IRType) && !Has64BitPointers)) {
2704 unsigned BitWidth = isa<llvm::PointerType>(Val: IRType) ? 32 :
2705 cast<llvm::IntegerType>(Val: IRType)->getBitWidth();
2706
2707 if (BitsContainNoUserData(Ty: SourceTy, StartBit: SourceOffset*8+BitWidth,
2708 EndBit: SourceOffset*8+64, Context&: getContext()))
2709 return IRType;
2710 }
2711 }
2712
2713 if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: IRType)) {
2714 // If this is a struct, recurse into the field at the specified offset.
2715 const llvm::StructLayout *SL = getDataLayout().getStructLayout(Ty: STy);
2716 if (IROffset < SL->getSizeInBytes()) {
2717 unsigned FieldIdx = SL->getElementContainingOffset(FixedOffset: IROffset);
2718 IROffset -= SL->getElementOffset(Idx: FieldIdx);
2719
2720 return GetINTEGERTypeAtOffset(IRType: STy->getElementType(N: FieldIdx), IROffset,
2721 SourceTy, SourceOffset);
2722 }
2723 }
2724
2725 if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(Val: IRType)) {
2726 llvm::Type *EltTy = ATy->getElementType();
2727 unsigned EltSize = getDataLayout().getTypeAllocSize(Ty: EltTy);
2728 unsigned EltOffset = IROffset/EltSize*EltSize;
2729 return GetINTEGERTypeAtOffset(IRType: EltTy, IROffset: IROffset-EltOffset, SourceTy,
2730 SourceOffset);
2731 }
2732
2733 // if we have a 128-bit integer, we can pass it safely using an i128
2734 // so we return that
2735 if (IRType->isIntegerTy(BitWidth: 128)) {
2736 assert(IROffset == 0);
2737 return IRType;
2738 }
2739
2740 // Okay, we don't have any better idea of what to pass, so we pass this in an
2741 // integer register that isn't too big to fit the rest of the struct.
2742 unsigned TySizeInBytes =
2743 (unsigned)getContext().getTypeSizeInChars(T: SourceTy).getQuantity();
2744
2745 assert(TySizeInBytes != SourceOffset && "Empty field?");
2746
2747 // It is always safe to classify this as an integer type up to i64 that
2748 // isn't larger than the structure.
2749 return llvm::IntegerType::get(C&: getVMContext(),
2750 NumBits: std::min(a: TySizeInBytes-SourceOffset, b: 8U)*8);
2751}
2752
2753
2754/// GetX86_64ByValArgumentPair - Given a high and low type that can ideally
2755/// be used as elements of a two register pair to pass or return, return a
2756/// first class aggregate to represent them. For example, if the low part of
2757/// a by-value argument should be passed as i32* and the high part as float,
2758/// return {i32*, float}.
2759static llvm::Type *
2760GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,
2761 const llvm::DataLayout &TD) {
2762 // In order to correctly satisfy the ABI, we need to the high part to start
2763 // at offset 8. If the high and low parts we inferred are both 4-byte types
2764 // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
2765 // the second element at offset 8. Check for this:
2766 unsigned LoSize = (unsigned)TD.getTypeAllocSize(Ty: Lo);
2767 llvm::Align HiAlign = TD.getABITypeAlign(Ty: Hi);
2768 unsigned HiStart = llvm::alignTo(Size: LoSize, A: HiAlign);
2769 assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
2770
2771 // To handle this, we have to increase the size of the low part so that the
2772 // second element will start at an 8 byte offset. We can't increase the size
2773 // of the second element because it might make us access off the end of the
2774 // struct.
2775 if (HiStart != 8) {
2776 // There are usually two sorts of types the ABI generation code can produce
2777 // for the low part of a pair that aren't 8 bytes in size: half, float or
2778 // i8/i16/i32. This can also include pointers when they are 32-bit (X32).
2779 // Promote these to a larger type.
2780 if (Lo->isHalfTy() || Lo->isFloatTy())
2781 Lo = llvm::Type::getDoubleTy(C&: Lo->getContext());
2782 else {
2783 assert((Lo->isIntegerTy() || Lo->isPointerTy())
2784 && "Invalid/unknown lo type");
2785 Lo = llvm::Type::getInt64Ty(C&: Lo->getContext());
2786 }
2787 }
2788
2789 llvm::StructType *Result = llvm::StructType::get(elt1: Lo, elts: Hi);
2790
2791 // Verify that the second element is at an 8-byte offset.
2792 assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&
2793 "Invalid x86-64 argument pair!");
2794 return Result;
2795}
2796
2797ABIArgInfo X86_64ABIInfo::classifyReturnType(QualType RetTy) const {
2798 // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
2799 // classification algorithm.
2800 X86_64ABIInfo::Class Lo, Hi;
2801 classify(Ty: RetTy, OffsetBase: 0, Lo, Hi, /*isNamedArg*/ true);
2802
2803 // Check some invariants.
2804 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
2805 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
2806
2807 llvm::Type *ResType = nullptr;
2808 switch (Lo) {
2809 case NoClass:
2810 if (Hi == NoClass)
2811 return ABIArgInfo::getIgnore();
2812 // If the low part is just padding, it takes no register, leave ResType
2813 // null.
2814 assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
2815 "Unknown missing lo part");
2816 break;
2817
2818 case SSEUp:
2819 case X87Up:
2820 llvm_unreachable("Invalid classification for lo word.");
2821
2822 // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
2823 // hidden argument.
2824 case Memory:
2825 return getIndirectReturnResult(Ty: RetTy);
2826
2827 // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
2828 // available register of the sequence %rax, %rdx is used.
2829 case Integer:
2830 ResType = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 0, SourceTy: RetTy, SourceOffset: 0);
2831
2832 // If we have a sign or zero extended integer, make sure to return Extend
2833 // so that the parameter gets the right LLVM IR attributes.
2834 if (Hi == NoClass && isa<llvm::IntegerType>(Val: ResType)) {
2835 // Treat an enum type as its underlying type.
2836 if (const auto *ED = RetTy->getAsEnumDecl())
2837 RetTy = ED->getIntegerType();
2838
2839 if (RetTy->isIntegralOrEnumerationType() &&
2840 isPromotableIntegerTypeForABI(Ty: RetTy))
2841 return ABIArgInfo::getExtend(Ty: RetTy);
2842 }
2843
2844 if (ResType->isIntegerTy(BitWidth: 128)) {
2845 // i128 are passed directly
2846 assert(Hi == Integer);
2847 return ABIArgInfo::getDirect(T: ResType);
2848 }
2849 break;
2850
2851 // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
2852 // available SSE register of the sequence %xmm0, %xmm1 is used.
2853 case SSE:
2854 ResType = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 0, SourceTy: RetTy, SourceOffset: 0);
2855 break;
2856
2857 // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
2858 // returned on the X87 stack in %st0 as 80-bit x87 number.
2859 case X87:
2860 ResType = llvm::Type::getX86_FP80Ty(C&: getVMContext());
2861 break;
2862
2863 // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
2864 // part of the value is returned in %st0 and the imaginary part in
2865 // %st1.
2866 case ComplexX87:
2867 assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
2868 ResType = llvm::StructType::get(elt1: llvm::Type::getX86_FP80Ty(C&: getVMContext()),
2869 elts: llvm::Type::getX86_FP80Ty(C&: getVMContext()));
2870 break;
2871 }
2872
2873 llvm::Type *HighPart = nullptr;
2874 switch (Hi) {
2875 // Memory was handled previously and X87 should
2876 // never occur as a hi class.
2877 case Memory:
2878 case X87:
2879 llvm_unreachable("Invalid classification for hi word.");
2880
2881 case ComplexX87: // Previously handled.
2882 case NoClass:
2883 break;
2884
2885 case Integer:
2886 HighPart = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
2887 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
2888 return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2889 break;
2890 case SSE:
2891 HighPart = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
2892 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
2893 return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2894 break;
2895
2896 // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
2897 // is passed in the next available eightbyte chunk if the last used
2898 // vector register.
2899 //
2900 // SSEUP should always be preceded by SSE, just widen.
2901 case SSEUp:
2902 assert(Lo == SSE && "Unexpected SSEUp classification.");
2903 ResType = GetByteVectorType(Ty: RetTy);
2904 break;
2905
2906 // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is
2907 // returned together with the previous X87 value in %st0.
2908 case X87Up:
2909 // If X87Up is preceded by X87, we don't need to do
2910 // anything. However, in some cases with unions it may not be
2911 // preceded by X87. In such situations we follow gcc and pass the
2912 // extra bits in an SSE reg.
2913 if (Lo != X87) {
2914 HighPart = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
2915 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
2916 return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2917 }
2918 break;
2919 }
2920
2921 // If a high part was specified, merge it together with the low part. It is
2922 // known to pass in the high eightbyte of the result. We do this by forming a
2923 // first class struct aggregate with the high and low part: {low, high}
2924 if (HighPart)
2925 ResType = GetX86_64ByValArgumentPair(Lo: ResType, Hi: HighPart, TD: getDataLayout());
2926
2927 return ABIArgInfo::getDirect(T: ResType);
2928}
2929
2930ABIArgInfo
2931X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned freeIntRegs,
2932 unsigned &neededInt, unsigned &neededSSE,
2933 bool isNamedArg, bool IsRegCall) const {
2934 Ty = useFirstFieldIfTransparentUnion(Ty);
2935
2936 X86_64ABIInfo::Class Lo, Hi;
2937 classify(Ty, OffsetBase: 0, Lo, Hi, isNamedArg, IsRegCall);
2938
2939 // Check some invariants.
2940 // FIXME: Enforce these by construction.
2941 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
2942 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
2943
2944 neededInt = 0;
2945 neededSSE = 0;
2946 llvm::Type *ResType = nullptr;
2947 switch (Lo) {
2948 case NoClass:
2949 if (Hi == NoClass)
2950 return ABIArgInfo::getIgnore();
2951 // If the low part is just padding, it takes no register, leave ResType
2952 // null.
2953 assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
2954 "Unknown missing lo part");
2955 break;
2956
2957 // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
2958 // on the stack.
2959 case Memory:
2960
2961 // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87UP or
2962 // COMPLEX_X87, it is passed in memory.
2963 case X87:
2964 case ComplexX87:
2965 if (getRecordArgABI(T: Ty, CXXABI&: getCXXABI()) == CGCXXABI::RAA_Indirect)
2966 ++neededInt;
2967 return getIndirectResult(Ty, freeIntRegs);
2968
2969 case SSEUp:
2970 case X87Up:
2971 llvm_unreachable("Invalid classification for lo word.");
2972
2973 // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
2974 // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
2975 // and %r9 is used.
2976 case Integer:
2977 ++neededInt;
2978
2979 // Pick an 8-byte type based on the preferred type.
2980 ResType = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: Ty), IROffset: 0, SourceTy: Ty, SourceOffset: 0);
2981
2982 // If we have a sign or zero extended integer, make sure to return Extend
2983 // so that the parameter gets the right LLVM IR attributes.
2984 if (Hi == NoClass && isa<llvm::IntegerType>(Val: ResType)) {
2985 // Treat an enum type as its underlying type.
2986 if (const auto *ED = Ty->getAsEnumDecl())
2987 Ty = ED->getIntegerType();
2988
2989 if (Ty->isIntegralOrEnumerationType() &&
2990 isPromotableIntegerTypeForABI(Ty))
2991 return ABIArgInfo::getExtend(Ty, T: CGT.ConvertType(T: Ty));
2992 }
2993
2994 if (ResType->isIntegerTy(BitWidth: 128)) {
2995 assert(Hi == Integer);
2996 ++neededInt;
2997 return ABIArgInfo::getDirect(T: ResType);
2998 }
2999 break;
3000
3001 // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
3002 // available SSE register is used, the registers are taken in the
3003 // order from %xmm0 to %xmm7.
3004 case SSE: {
3005 llvm::Type *IRType = CGT.ConvertType(T: Ty);
3006 ResType = GetSSETypeAtOffset(IRType, IROffset: 0, SourceTy: Ty, SourceOffset: 0);
3007 ++neededSSE;
3008 break;
3009 }
3010 }
3011
3012 llvm::Type *HighPart = nullptr;
3013 switch (Hi) {
3014 // Memory was handled previously, ComplexX87 and X87 should
3015 // never occur as hi classes, and X87Up must be preceded by X87,
3016 // which is passed in memory.
3017 case Memory:
3018 case X87:
3019 case ComplexX87:
3020 llvm_unreachable("Invalid classification for hi word.");
3021
3022 case NoClass: break;
3023
3024 case Integer:
3025 ++neededInt;
3026 // Pick an 8-byte type based on the preferred type.
3027 HighPart = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: Ty), IROffset: 8, SourceTy: Ty, SourceOffset: 8);
3028
3029 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
3030 return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
3031 break;
3032
3033 // X87Up generally doesn't occur here (long double is passed in
3034 // memory), except in situations involving unions.
3035 case X87Up:
3036 case SSE:
3037 ++neededSSE;
3038 HighPart = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: Ty), IROffset: 8, SourceTy: Ty, SourceOffset: 8);
3039
3040 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
3041 return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
3042 break;
3043
3044 // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
3045 // eightbyte is passed in the upper half of the last used SSE
3046 // register. This only happens when 128-bit vectors are passed.
3047 case SSEUp:
3048 assert(Lo == SSE && "Unexpected SSEUp classification");
3049 ResType = GetByteVectorType(Ty);
3050 break;
3051 }
3052
3053 // If a high part was specified, merge it together with the low part. It is
3054 // known to pass in the high eightbyte of the result. We do this by forming a
3055 // first class struct aggregate with the high and low part: {low, high}
3056 if (HighPart)
3057 ResType = GetX86_64ByValArgumentPair(Lo: ResType, Hi: HighPart, TD: getDataLayout());
3058
3059 return ABIArgInfo::getDirect(T: ResType);
3060}
3061
3062// Returns true if the struct can be passed directly in registers. If so, the
3063// number of registers required will be returned in `NeededInt` and `NeededSSE`,
3064// and `CoerceElts` will contain an expanded sequence of LLVM IR types that each
3065// field should coerce to.
3066bool X86_64ABIInfo::passRegCallStructTypeDirectly(
3067 QualType Ty, SmallVectorImpl<llvm::Type *> &CoerceElts, unsigned &NeededInt,
3068 unsigned &NeededSSE, unsigned &MaxVectorWidth) const {
3069
3070 auto *RD =
3071 cast<RecordType>(Val: Ty.getCanonicalType())->getDecl()->getDefinitionOrSelf();
3072 if (RD->hasFlexibleArrayMember())
3073 return false;
3074
3075 // Classify the bases.
3076 if (auto CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
3077 if (CXXRD->isDynamicClass())
3078 return false;
3079
3080 for (const auto &I : CXXRD->bases()) {
3081 QualType BaseTy = I.getType();
3082 if (isEmptyRecord(Context&: getContext(), T: BaseTy, AllowArrays: true))
3083 continue;
3084 if (!passRegCallStructTypeDirectly(Ty: BaseTy, CoerceElts, NeededInt,
3085 NeededSSE, MaxVectorWidth))
3086 return false;
3087 }
3088 }
3089
3090 // Classify the members.
3091 for (const auto *FD : RD->fields()) {
3092 QualType MTy = FD->getType();
3093 if (MTy->isRecordType() && !MTy->isUnionType()) {
3094 if (isEmptyRecord(Context&: getContext(), T: MTy, AllowArrays: true))
3095 continue;
3096 if (!passRegCallStructTypeDirectly(Ty: MTy, CoerceElts, NeededInt, NeededSSE,
3097 MaxVectorWidth))
3098 return false;
3099 continue;
3100 }
3101
3102 const auto *AT = getContext().getAsConstantArrayType(T: MTy);
3103 if (AT)
3104 MTy = AT->getElementType();
3105
3106 unsigned LocalNeededInt, LocalNeededSSE;
3107 ABIArgInfo AI = classifyArgumentType(Ty: MTy, UINT_MAX, neededInt&: LocalNeededInt,
3108 neededSSE&: LocalNeededSSE, isNamedArg: true, IsRegCall: true);
3109 if (AI.isIgnore())
3110 continue;
3111 if (AI.isIndirect())
3112 return false;
3113
3114 llvm::Type *CoerceTy = AI.getCoerceToType();
3115 assert(CoerceTy && "ABI info for struct member has no coerce type");
3116 if (AT) {
3117 uint64_t NumElts = AT->getZExtSize();
3118 LocalNeededInt *= NumElts;
3119 LocalNeededSSE *= NumElts;
3120 CoerceElts.push_back(Elt: llvm::ArrayType::get(ElementType: CoerceTy, NumElements: NumElts));
3121 } else {
3122 CoerceElts.push_back(Elt: CoerceTy);
3123 }
3124
3125 if (const auto *VT = MTy->getAs<VectorType>())
3126 if (getContext().getTypeSize(T: VT) > MaxVectorWidth)
3127 MaxVectorWidth = getContext().getTypeSize(T: VT);
3128
3129 NeededInt += LocalNeededInt;
3130 NeededSSE += LocalNeededSSE;
3131 }
3132
3133 return true;
3134}
3135
3136ABIArgInfo
3137X86_64ABIInfo::classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
3138 unsigned &NeededSSE,
3139 unsigned &MaxVectorWidth) const {
3140 NeededInt = 0;
3141 NeededSSE = 0;
3142 MaxVectorWidth = 0;
3143
3144 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
3145 return ABIArgInfo::getIgnore();
3146
3147 SmallVector<llvm::Type *, 16> CoerceElts;
3148 if (!passRegCallStructTypeDirectly(Ty, CoerceElts, NeededInt, NeededSSE,
3149 MaxVectorWidth)) {
3150 NeededInt = NeededSSE = 0;
3151 return getIndirectReturnResult(Ty);
3152 }
3153
3154 assert(!CoerceElts.empty() && "Non-empty struct produced no element types");
3155 return ABIArgInfo::getDirect(
3156 T: llvm::StructType::get(Context&: getVMContext(), Elements: CoerceElts));
3157}
3158
3159unsigned
3160X86_64ABIInfo::getX86ABIAVXLevel(const FunctionDecl *FD,
3161 const FunctionType::ExtInfo &Info) const {
3162 return static_cast<unsigned>(getEffectiveX86AVXABILevel(CGT, GlobalAVXLevel: AVXLevel, FD));
3163}
3164
3165void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3166 const unsigned CallingConv = FI.getCallingConvention();
3167 // It is possible to force Win64 calling convention on any x86_64 target by
3168 // using __attribute__((ms_abi)). In such case to correctly emit Win64
3169 // compatible code delegate this call to WinX86_64ABIInfo::computeInfo.
3170 if (CallingConv == llvm::CallingConv::Win64) {
3171 WinX86_64ABIInfo Win64ABIInfo(CGT, AVXLevel);
3172 Win64ABIInfo.computeInfo(FI);
3173 return;
3174 }
3175
3176 assert(FI.getX86ABIAVXLevel() <=
3177 static_cast<unsigned>(X86AVXABILevel::AVX512) &&
3178 "Unexpected X86 AVX ABI level");
3179 X86AVXABILevel EffectiveAVXLevel =
3180 static_cast<X86AVXABILevel>(FI.getX86ABIAVXLevel());
3181 if (EffectiveAVXLevel != AVXLevel) {
3182 X86_64ABIInfo EffectiveABIInfo(CGT, EffectiveAVXLevel);
3183 EffectiveABIInfo.computeInfo(FI);
3184 return;
3185 }
3186
3187 bool IsRegCall = CallingConv == llvm::CallingConv::X86_RegCall;
3188
3189 // Keep track of the number of assigned registers.
3190 unsigned FreeIntRegs = IsRegCall ? 11 : 6;
3191 unsigned FreeSSERegs = IsRegCall ? 16 : 8;
3192 unsigned NeededInt = 0, NeededSSE = 0, MaxVectorWidth = 0;
3193
3194 if (!::classifyReturnType(CXXABI: getCXXABI(), FI, Info: *this)) {
3195 if (IsRegCall && FI.getReturnType()->getTypePtr()->isRecordType() &&
3196 !FI.getReturnType()->getTypePtr()->isUnionType()) {
3197 FI.getReturnInfo() = classifyRegCallStructType(
3198 Ty: FI.getReturnType(), NeededInt, NeededSSE, MaxVectorWidth);
3199 if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3200 FreeIntRegs -= NeededInt;
3201 FreeSSERegs -= NeededSSE;
3202 } else {
3203 FI.getReturnInfo() = getIndirectReturnResult(Ty: FI.getReturnType());
3204 }
3205 } else if (IsRegCall && FI.getReturnType()->getAs<ComplexType>() &&
3206 getContext().getCanonicalType(T: FI.getReturnType()
3207 ->getAs<ComplexType>()
3208 ->getElementType()) ==
3209 getContext().LongDoubleTy)
3210 // Complex Long Double Type is passed in Memory when Regcall
3211 // calling convention is used.
3212 FI.getReturnInfo() = getIndirectReturnResult(Ty: FI.getReturnType());
3213 else
3214 FI.getReturnInfo() = classifyReturnType(RetTy: FI.getReturnType());
3215 }
3216
3217 // If the return value is indirect, then the hidden argument is consuming one
3218 // integer register.
3219 if (FI.getReturnInfo().isIndirect())
3220 --FreeIntRegs;
3221 else if (NeededSSE && MaxVectorWidth > 0)
3222 FI.setMaxVectorWidth(MaxVectorWidth);
3223
3224 // The chain argument effectively gives us another free register.
3225 if (FI.isChainCall())
3226 ++FreeIntRegs;
3227
3228 // RegCall lets us reuse the return registers.
3229 if (IsRegCall)
3230 FreeSSERegs = 16;
3231
3232 unsigned NumRequiredArgs = FI.getNumRequiredArgs();
3233 // AMD64-ABI 3.2.3p3: Once arguments are classified, the registers
3234 // get assigned (in left-to-right order) for passing as follows...
3235 unsigned ArgNo = 0;
3236 for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
3237 it != ie; ++it, ++ArgNo) {
3238 bool IsNamedArg = ArgNo < NumRequiredArgs;
3239
3240 if (IsRegCall && it->type->isStructureOrClassType())
3241 it->info = classifyRegCallStructType(Ty: it->type, NeededInt, NeededSSE,
3242 MaxVectorWidth);
3243 else
3244 it->info = classifyArgumentType(Ty: it->type, freeIntRegs: FreeIntRegs, neededInt&: NeededInt,
3245 neededSSE&: NeededSSE, isNamedArg: IsNamedArg);
3246
3247 // AMD64-ABI 3.2.3p3: If there are no registers available for any
3248 // eightbyte of an argument, the whole argument is passed on the
3249 // stack. If registers have already been assigned for some
3250 // eightbytes of such an argument, the assignments get reverted.
3251 if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3252 FreeIntRegs -= NeededInt;
3253 FreeSSERegs -= NeededSSE;
3254 if (MaxVectorWidth > FI.getMaxVectorWidth())
3255 FI.setMaxVectorWidth(MaxVectorWidth);
3256 } else {
3257 it->info = getIndirectResult(Ty: it->type, freeIntRegs: FreeIntRegs);
3258 }
3259 }
3260}
3261
3262static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF,
3263 Address VAListAddr, QualType Ty) {
3264 Address overflow_arg_area_p =
3265 CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 2, Name: "overflow_arg_area_p");
3266 llvm::Value *overflow_arg_area =
3267 CGF.Builder.CreateLoad(Addr: overflow_arg_area_p, Name: "overflow_arg_area");
3268
3269 // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
3270 // byte boundary if alignment needed by type exceeds 8 byte boundary.
3271 // It isn't stated explicitly in the standard, but in practice we use
3272 // alignment greater than 16 where necessary.
3273 CharUnits Align = CGF.getContext().getTypeAlignInChars(T: Ty);
3274 if (Align > CharUnits::fromQuantity(Quantity: 8)) {
3275 overflow_arg_area = emitRoundPointerUpToAlignment(CGF, Ptr: overflow_arg_area,
3276 Align);
3277 }
3278
3279 // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
3280 llvm::Type *LTy = CGF.ConvertTypeForMem(T: Ty);
3281 llvm::Value *Res = overflow_arg_area;
3282
3283 // AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
3284 // l->overflow_arg_area + sizeof(type).
3285 // AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
3286 // an 8 byte boundary.
3287
3288 uint64_t SizeInBytes = (CGF.getContext().getTypeSize(T: Ty) + 7) / 8;
3289 llvm::Value *Offset =
3290 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: (SizeInBytes + 7) & ~7);
3291 overflow_arg_area = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: overflow_arg_area,
3292 IdxList: Offset, Name: "overflow_arg_area.next");
3293 CGF.Builder.CreateStore(Val: overflow_arg_area, Addr: overflow_arg_area_p);
3294
3295 // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
3296 return Address(Res, LTy, Align);
3297}
3298
3299RValue X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
3300 QualType Ty, AggValueSlot Slot) const {
3301 // Assume that va_list type is correct; should be pointer to LLVM type:
3302 // struct {
3303 // i32 gp_offset;
3304 // i32 fp_offset;
3305 // i8* overflow_arg_area;
3306 // i8* reg_save_area;
3307 // };
3308 unsigned neededInt, neededSSE;
3309
3310 Ty = getContext().getCanonicalType(T: Ty);
3311 ABIArgInfo AI = classifyArgumentType(Ty, freeIntRegs: 0, neededInt, neededSSE,
3312 /*isNamedArg*/false);
3313
3314 // Empty records are ignored for parameter passing purposes.
3315 if (AI.isIgnore())
3316 return Slot.asRValue();
3317
3318 // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
3319 // in the registers. If not go to step 7.
3320 if (!neededInt && !neededSSE)
3321 return CGF.EmitLoadOfAnyValue(
3322 V: CGF.MakeAddrLValue(Addr: EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty), T: Ty),
3323 Slot);
3324
3325 // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
3326 // general purpose registers needed to pass type and num_fp to hold
3327 // the number of floating point registers needed.
3328
3329 // AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
3330 // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
3331 // l->fp_offset > 304 - num_fp * 16 go to step 7.
3332 //
3333 // NOTE: 304 is a typo, there are (6 * 8 + 8 * 16) = 176 bytes of
3334 // register save space).
3335
3336 llvm::Value *InRegs = nullptr;
3337 Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid();
3338 llvm::Value *gp_offset = nullptr, *fp_offset = nullptr;
3339 if (neededInt) {
3340 gp_offset_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 0, Name: "gp_offset_p");
3341 gp_offset = CGF.Builder.CreateLoad(Addr: gp_offset_p, Name: "gp_offset");
3342 InRegs = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 48 - neededInt * 8);
3343 InRegs = CGF.Builder.CreateICmpULE(LHS: gp_offset, RHS: InRegs, Name: "fits_in_gp");
3344 }
3345
3346 if (neededSSE) {
3347 fp_offset_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 1, Name: "fp_offset_p");
3348 fp_offset = CGF.Builder.CreateLoad(Addr: fp_offset_p, Name: "fp_offset");
3349 llvm::Value *FitsInFP =
3350 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 176 - neededSSE * 16);
3351 FitsInFP = CGF.Builder.CreateICmpULE(LHS: fp_offset, RHS: FitsInFP, Name: "fits_in_fp");
3352 InRegs = InRegs ? CGF.Builder.CreateAnd(LHS: InRegs, RHS: FitsInFP) : FitsInFP;
3353 }
3354
3355 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock(name: "vaarg.in_reg");
3356 llvm::BasicBlock *InMemBlock = CGF.createBasicBlock(name: "vaarg.in_mem");
3357 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "vaarg.end");
3358 CGF.Builder.CreateCondBr(Cond: InRegs, True: InRegBlock, False: InMemBlock);
3359
3360 // Emit code to load the value if it was passed in registers.
3361
3362 CGF.EmitBlock(BB: InRegBlock);
3363
3364 // AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
3365 // an offset of l->gp_offset and/or l->fp_offset. This may require
3366 // copying to a temporary location in case the parameter is passed
3367 // in different register classes or requires an alignment greater
3368 // than 8 for general purpose registers and 16 for XMM registers.
3369 //
3370 // FIXME: This really results in shameful code when we end up needing to
3371 // collect arguments from different places; often what should result in a
3372 // simple assembling of a structure from scattered addresses has many more
3373 // loads than necessary. Can we clean this up?
3374 llvm::Type *LTy = CGF.ConvertTypeForMem(T: Ty);
3375 llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(
3376 Addr: CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 3), Name: "reg_save_area");
3377
3378 Address RegAddr = Address::invalid();
3379 if (neededInt && neededSSE) {
3380 // FIXME: Cleanup.
3381 assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
3382 llvm::StructType *ST = cast<llvm::StructType>(Val: AI.getCoerceToType());
3383 Address Tmp = CGF.CreateMemTempWithoutCast(T: Ty);
3384 Tmp = Tmp.withElementType(ElemTy: ST);
3385 assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
3386 llvm::Type *TyLo = ST->getElementType(N: 0);
3387 llvm::Type *TyHi = ST->getElementType(N: 1);
3388 assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&
3389 "Unexpected ABI info for mixed regs");
3390 llvm::Value *GPAddr =
3391 CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: gp_offset);
3392 llvm::Value *FPAddr =
3393 CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: fp_offset);
3394 llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? FPAddr : GPAddr;
3395 llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? GPAddr : FPAddr;
3396
3397 // Copy the first element.
3398 // FIXME: Our choice of alignment here and below is probably pessimistic.
3399 llvm::Value *V = CGF.Builder.CreateAlignedLoad(
3400 Ty: TyLo, Addr: RegLoAddr,
3401 Align: CharUnits::fromQuantity(Quantity: getDataLayout().getABITypeAlign(Ty: TyLo)));
3402 CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 0));
3403
3404 // Copy the second element.
3405 V = CGF.Builder.CreateAlignedLoad(
3406 Ty: TyHi, Addr: RegHiAddr,
3407 Align: CharUnits::fromQuantity(Quantity: getDataLayout().getABITypeAlign(Ty: TyHi)));
3408 CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 1));
3409
3410 RegAddr = Tmp.withElementType(ElemTy: LTy);
3411 } else if (neededInt || neededSSE == 1) {
3412 // Copy to a temporary if necessary to ensure the appropriate alignment.
3413 auto TInfo = getContext().getTypeInfoInChars(T: Ty);
3414 uint64_t TySize = TInfo.Width.getQuantity();
3415 CharUnits TyAlign = TInfo.Align;
3416 llvm::Type *CoTy = nullptr;
3417 if (AI.isDirect())
3418 CoTy = AI.getCoerceToType();
3419
3420 llvm::Value *GpOrFpOffset = neededInt ? gp_offset : fp_offset;
3421 uint64_t Alignment = neededInt ? 8 : 16;
3422 uint64_t RegSize = neededInt ? neededInt * 8 : 16;
3423 // There are two cases require special handling:
3424 // 1)
3425 // ```
3426 // struct {
3427 // struct {} a[8];
3428 // int b;
3429 // };
3430 // ```
3431 // The lower 8 bytes of the structure are not stored,
3432 // so an 8-byte offset is needed when accessing the structure.
3433 // 2)
3434 // ```
3435 // struct {
3436 // long long a;
3437 // struct {} b;
3438 // };
3439 // ```
3440 // The stored size of this structure is smaller than its actual size,
3441 // which may lead to reading past the end of the register save area.
3442 if (CoTy && (AI.getDirectOffset() == 8 || RegSize < TySize)) {
3443 Address Tmp = CGF.CreateMemTempWithoutCast(T: Ty);
3444 llvm::Value *Addr =
3445 CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: GpOrFpOffset);
3446 llvm::Value *Src = CGF.Builder.CreateAlignedLoad(Ty: CoTy, Addr, Align: TyAlign);
3447 llvm::Value *PtrOffset =
3448 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: AI.getDirectOffset());
3449 Address Dst = Address(
3450 CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: Tmp.getBasePointer(), IdxList: PtrOffset),
3451 LTy, TyAlign);
3452 CGF.Builder.CreateStore(Val: Src, Addr: Dst);
3453 RegAddr = Tmp.withElementType(ElemTy: LTy);
3454 } else {
3455 RegAddr =
3456 Address(CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: GpOrFpOffset),
3457 LTy, CharUnits::fromQuantity(Quantity: Alignment));
3458
3459 // Copy into a temporary if the type is more aligned than the
3460 // register save area.
3461 if (neededInt && TyAlign.getQuantity() > 8) {
3462 Address Tmp = CGF.CreateMemTempWithoutCast(T: Ty);
3463 CGF.Builder.CreateMemCpy(Dest: Tmp, Src: RegAddr, Size: TySize, IsVolatile: false);
3464 RegAddr = Tmp;
3465 }
3466 }
3467
3468 } else {
3469 assert(neededSSE == 2 && "Invalid number of needed registers!");
3470 // SSE registers are spaced 16 bytes apart in the register save
3471 // area, we need to collect the two eightbytes together.
3472 // The ABI isn't explicit about this, but it seems reasonable
3473 // to assume that the slots are 16-byte aligned, since the stack is
3474 // naturally 16-byte aligned and the prologue is expected to store
3475 // all the SSE registers to the RSA.
3476 Address RegAddrLo = Address(CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea,
3477 IdxList: fp_offset),
3478 CGF.Int8Ty, CharUnits::fromQuantity(Quantity: 16));
3479 Address RegAddrHi =
3480 CGF.Builder.CreateConstInBoundsByteGEP(Addr: RegAddrLo,
3481 Offset: CharUnits::fromQuantity(Quantity: 16));
3482 llvm::Type *ST = AI.canHaveCoerceToType()
3483 ? AI.getCoerceToType()
3484 : llvm::StructType::get(elt1: CGF.DoubleTy, elts: CGF.DoubleTy);
3485 llvm::Value *V;
3486 Address Tmp = CGF.CreateMemTempWithoutCast(T: Ty);
3487 Tmp = Tmp.withElementType(ElemTy: ST);
3488 V = CGF.Builder.CreateLoad(
3489 Addr: RegAddrLo.withElementType(ElemTy: ST->getStructElementType(N: 0)));
3490 CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 0));
3491 V = CGF.Builder.CreateLoad(
3492 Addr: RegAddrHi.withElementType(ElemTy: ST->getStructElementType(N: 1)));
3493 CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 1));
3494
3495 RegAddr = Tmp.withElementType(ElemTy: LTy);
3496 }
3497
3498 // AMD64-ABI 3.5.7p5: Step 5. Set:
3499 // l->gp_offset = l->gp_offset + num_gp * 8
3500 // l->fp_offset = l->fp_offset + num_fp * 16.
3501 if (neededInt) {
3502 llvm::Value *Offset = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: neededInt * 8);
3503 CGF.Builder.CreateStore(Val: CGF.Builder.CreateAdd(LHS: gp_offset, RHS: Offset),
3504 Addr: gp_offset_p);
3505 }
3506 if (neededSSE) {
3507 llvm::Value *Offset = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: neededSSE * 16);
3508 CGF.Builder.CreateStore(Val: CGF.Builder.CreateAdd(LHS: fp_offset, RHS: Offset),
3509 Addr: fp_offset_p);
3510 }
3511 CGF.EmitBranch(Block: ContBlock);
3512
3513 // Emit code to load the value if it was passed in memory.
3514
3515 CGF.EmitBlock(BB: InMemBlock);
3516 Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
3517
3518 // Return the appropriate result.
3519
3520 CGF.EmitBlock(BB: ContBlock);
3521 Address ResAddr = emitMergePHI(CGF, Addr1: RegAddr, Block1: InRegBlock, Addr2: MemAddr, Block2: InMemBlock,
3522 Name: "vaarg.addr");
3523 return CGF.EmitLoadOfAnyValue(V: CGF.MakeAddrLValue(Addr: ResAddr, T: Ty), Slot);
3524}
3525
3526RValue X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
3527 QualType Ty, AggValueSlot Slot) const {
3528 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3529 // not 1, 2, 4, or 8 bytes, must be passed by reference."
3530 uint64_t Width = getContext().getTypeSize(T: Ty);
3531 bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Value: Width);
3532
3533 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
3534 ValueInfo: CGF.getContext().getTypeInfoInChars(T: Ty),
3535 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 8),
3536 /*allowHigherAlign*/ AllowHigherAlign: false, Slot);
3537}
3538
3539ABIArgInfo WinX86_64ABIInfo::reclassifyHvaArgForVectorCall(
3540 QualType Ty, unsigned &FreeSSERegs, const ABIArgInfo &current) const {
3541 const Type *Base = nullptr;
3542 uint64_t NumElts = 0;
3543
3544 if (!Ty->isBuiltinType() && !Ty->isVectorType() &&
3545 isHomogeneousAggregate(Ty, Base, Members&: NumElts) && FreeSSERegs >= NumElts) {
3546 FreeSSERegs -= NumElts;
3547 return getDirectX86Hva();
3548 }
3549 return current;
3550}
3551
3552ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
3553 ClassifyKind Kind, unsigned CC) const {
3554 bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
3555 bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
3556
3557 if (Ty->isVoidType())
3558 return ABIArgInfo::getIgnore();
3559
3560 bool PromoteScopedEnum = false;
3561 if (const auto *ED = Ty->getAsEnumDecl()) {
3562 Ty = ED->getIntegerType();
3563 PromoteScopedEnum = Kind == ClassifyKind::VarArg && ED->isScoped() &&
3564 getContext().isPromotableIntegerType(T: Ty);
3565 }
3566
3567 // MSVC extends scoped enums with a sub-int underlying type when they are
3568 // passed through an ellipsis. Unlike unscoped enums, scoped enums are not
3569 // subject to the language's default argument promotions, so handle the
3570 // extension as part of the ABI classification.
3571 if (PromoteScopedEnum)
3572 return ABIArgInfo::getExtend(Ty);
3573
3574 TypeInfo Info = getContext().getTypeInfo(T: Ty);
3575 uint64_t Width = Info.Width;
3576 CharUnits Align = getContext().toCharUnitsFromBits(BitSize: Info.Align);
3577
3578 const RecordType *RT = Ty->getAsCanonical<RecordType>();
3579 if (RT) {
3580 if (Kind != ClassifyKind::Return) {
3581 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, CXXABI&: getCXXABI()))
3582 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
3583 ByVal: RAA == CGCXXABI::RAA_DirectInMemory);
3584 }
3585
3586 if (RT->getDecl()->getDefinitionOrSelf()->hasFlexibleArrayMember())
3587 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
3588 /*ByVal=*/false);
3589 }
3590
3591 const Type *Base = nullptr;
3592 uint64_t NumElts = 0;
3593 // vectorcall adds the concept of a homogenous vector aggregate, similar to
3594 // other targets.
3595 if ((IsVectorCall || IsRegCall) &&
3596 isHomogeneousAggregate(Ty, Base, Members&: NumElts)) {
3597 if (IsRegCall) {
3598 if (FreeSSERegs >= NumElts) {
3599 FreeSSERegs -= NumElts;
3600 if (Kind == ClassifyKind::Return || Ty->isBuiltinType() ||
3601 Ty->isVectorType())
3602 return ABIArgInfo::getDirect();
3603 return ABIArgInfo::getExpand();
3604 }
3605 return ABIArgInfo::getIndirect(
3606 Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3607 /*ByVal=*/false);
3608 } else if (IsVectorCall) {
3609 if (FreeSSERegs >= NumElts &&
3610 (Kind == ClassifyKind::Return || Ty->isBuiltinType() ||
3611 Ty->isVectorType())) {
3612 FreeSSERegs -= NumElts;
3613 return ABIArgInfo::getDirect();
3614 } else if (Kind == ClassifyKind::Return) {
3615 return ABIArgInfo::getExpand();
3616 } else if (!Ty->isBuiltinType() && !Ty->isVectorType()) {
3617 // HVAs are delayed and reclassified in the 2nd step.
3618 return ABIArgInfo::getIndirect(
3619 Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3620 /*ByVal=*/false);
3621 }
3622 }
3623 }
3624
3625 if (Ty->isMemberPointerType()) {
3626 // If the member pointer is represented by an LLVM int or ptr, pass it
3627 // directly.
3628 llvm::Type *LLTy = CGT.ConvertType(T: Ty);
3629 if (LLTy->isPointerTy() || LLTy->isIntegerTy())
3630 return ABIArgInfo::getDirect();
3631 }
3632
3633 if (RT || Ty->isAnyComplexType() || Ty->isMemberPointerType()) {
3634 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3635 // not 1, 2, 4, or 8 bytes, must be passed by reference."
3636 if (Width > 64 || !llvm::isPowerOf2_64(Value: Width))
3637 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
3638 /*ByVal=*/false);
3639
3640 // Otherwise, coerce it to a small integer.
3641 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Width));
3642 }
3643
3644 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
3645 switch (BT->getKind()) {
3646 case BuiltinType::Bool:
3647 // Bool type is always extended to the ABI, other builtin types are not
3648 // extended.
3649 return ABIArgInfo::getExtend(Ty);
3650
3651 case BuiltinType::LongDouble:
3652 // Mingw64 GCC uses the old 80 bit extended precision floating point
3653 // unit. It passes them indirectly through memory.
3654 if (IsMingw64) {
3655 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
3656 if (LDF == &llvm::APFloat::x87DoubleExtended())
3657 return ABIArgInfo::getIndirect(
3658 Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3659 /*ByVal=*/false);
3660 }
3661 break;
3662
3663 case BuiltinType::Int128:
3664 case BuiltinType::UInt128:
3665 case BuiltinType::Float128:
3666 // If it's a parameter type, the normal ABI rule is that arguments larger
3667 // than 8 bytes are passed indirectly. GCC follows it. We follow it too,
3668 // even though it isn't particularly efficient.
3669 if (Kind != ClassifyKind::Return)
3670 return ABIArgInfo::getIndirect(
3671 Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3672 /*ByVal=*/false);
3673
3674 // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
3675 // Clang matches them for compatibility.
3676 if (BT->getKind() == BuiltinType::Int128 ||
3677 BT->getKind() == BuiltinType::UInt128)
3678 return ABIArgInfo::getDirect(T: llvm::FixedVectorType::get(
3679 ElementType: llvm::Type::getInt64Ty(C&: getVMContext()), NumElts: 2));
3680
3681 // Mingw64 GCC returns f128 via sret, and Clang matches that for
3682 // compatibility. This mirrors the X86 backend's CanLowerReturn logic.
3683 if (BT->getKind() == BuiltinType::Float128) {
3684 auto IsWin64F128StackCC = [this](unsigned CC) -> bool {
3685 switch (CC) {
3686 case llvm::CallingConv::Win64:
3687 return true;
3688 case llvm::CallingConv::C:
3689 return getTarget().getTriple().isOSWindowsOrUEFI();
3690 default:
3691 return false;
3692 }
3693 };
3694
3695 if (IsWin64F128StackCC(CC))
3696 return getNaturalAlignIndirect(
3697 Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(), /*ByVal=*/false);
3698 }
3699 break;
3700
3701 default:
3702 break;
3703 }
3704 }
3705
3706 if (Ty->isBitIntType()) {
3707 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3708 // not 1, 2, 4, or 8 bytes, must be passed by reference."
3709 // However, non-power-of-two bit-precise integers will be passed as 1, 2, 4,
3710 // or 8 bytes anyway as long is it fits in them, so we don't have to check
3711 // the power of 2.
3712 if (Width <= 64)
3713 return ABIArgInfo::getDirect();
3714 return ABIArgInfo::getIndirect(
3715 Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3716 /*ByVal=*/false);
3717 }
3718
3719 return ABIArgInfo::getDirect();
3720}
3721
3722unsigned
3723WinX86_64ABIInfo::getX86ABIAVXLevel(const FunctionDecl *FD,
3724 const FunctionType::ExtInfo &Info) const {
3725 if (Info.getCC() == CC_X86_64SysV) {
3726 return static_cast<unsigned>(getEffectiveX86AVXABILevel(CGT, GlobalAVXLevel: AVXLevel, FD));
3727 }
3728
3729 return static_cast<unsigned>(AVXLevel);
3730}
3731
3732void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3733 const unsigned CC = FI.getCallingConvention();
3734 bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
3735 bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
3736
3737 // If __attribute__((sysv_abi)) is in use, use the SysV argument
3738 // classification rules.
3739 if (CC == llvm::CallingConv::X86_64_SysV) {
3740 X86_64ABIInfo SysVABIInfo(CGT, AVXLevel);
3741 SysVABIInfo.computeInfo(FI);
3742 return;
3743 }
3744
3745 unsigned FreeSSERegs = 0;
3746 if (IsVectorCall) {
3747 // We can use up to 4 SSE return registers with vectorcall.
3748 FreeSSERegs = 4;
3749 } else if (IsRegCall) {
3750 // RegCall gives us 16 SSE registers.
3751 FreeSSERegs = 16;
3752 }
3753
3754 if (!getCXXABI().classifyReturnType(FI))
3755 FI.getReturnInfo() =
3756 classify(Ty: FI.getReturnType(), FreeSSERegs, Kind: ClassifyKind::Return, CC);
3757
3758 if (IsVectorCall) {
3759 // We can use up to 6 SSE register parameters with vectorcall.
3760 FreeSSERegs = 6;
3761 } else if (IsRegCall) {
3762 // RegCall gives us 16 SSE registers, we can reuse the return registers.
3763 FreeSSERegs = 16;
3764 }
3765
3766 unsigned ArgNum = 0;
3767 unsigned ZeroSSERegs = 0;
3768 for (auto &I : FI.arguments()) {
3769 // Vectorcall in x64 only permits the first 6 arguments to be passed as
3770 // XMM/YMM registers. After the sixth argument, pretend no vector
3771 // registers are left.
3772 unsigned *MaybeFreeSSERegs =
3773 (IsVectorCall && ArgNum >= 6) ? &ZeroSSERegs : &FreeSSERegs;
3774 ClassifyKind Kind = ArgNum >= FI.getNumRequiredArgs()
3775 ? ClassifyKind::VarArg
3776 : ClassifyKind::FixedArgument;
3777 I.info = classify(Ty: I.type, FreeSSERegs&: *MaybeFreeSSERegs, Kind, CC);
3778 ++ArgNum;
3779 }
3780
3781 if (IsVectorCall) {
3782 // For vectorcall, assign aggregate HVAs to any free vector registers in a
3783 // second pass.
3784 for (auto &I : FI.arguments())
3785 I.info = reclassifyHvaArgForVectorCall(Ty: I.type, FreeSSERegs, current: I.info);
3786 }
3787}
3788
3789RValue WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
3790 QualType Ty, AggValueSlot Slot) const {
3791 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3792 // not 1, 2, 4, or 8 bytes, must be passed by reference."
3793 uint64_t Width = getContext().getTypeSize(T: Ty);
3794 bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Value: Width);
3795
3796 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
3797 ValueInfo: CGF.getContext().getTypeInfoInChars(T: Ty),
3798 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 8),
3799 /*allowHigherAlign*/ AllowHigherAlign: false, Slot);
3800}
3801
3802std::unique_ptr<TargetCodeGenInfo> CodeGen::createX86_32TargetCodeGenInfo(
3803 CodeGenModule &CGM, bool DarwinVectorABI, bool Win32StructABI,
3804 unsigned NumRegisterParameters, bool SoftFloatABI) {
3805 bool RetSmallStructInRegABI = X86_32TargetCodeGenInfo::isStructReturnInRegABI(
3806 Triple: CGM.getTriple(), Opts: CGM.getCodeGenOpts());
3807 return std::make_unique<X86_32TargetCodeGenInfo>(
3808 args&: CGM.getTypes(), args&: DarwinVectorABI, args&: RetSmallStructInRegABI, args&: Win32StructABI,
3809 args&: NumRegisterParameters, args&: SoftFloatABI);
3810}
3811
3812std::unique_ptr<TargetCodeGenInfo> CodeGen::createWinX86_32TargetCodeGenInfo(
3813 CodeGenModule &CGM, bool DarwinVectorABI, bool Win32StructABI,
3814 unsigned NumRegisterParameters) {
3815 bool RetSmallStructInRegABI = X86_32TargetCodeGenInfo::isStructReturnInRegABI(
3816 Triple: CGM.getTriple(), Opts: CGM.getCodeGenOpts());
3817 return std::make_unique<WinX86_32TargetCodeGenInfo>(
3818 args&: CGM.getTypes(), args&: DarwinVectorABI, args&: RetSmallStructInRegABI, args&: Win32StructABI,
3819 args&: NumRegisterParameters);
3820}
3821
3822std::unique_ptr<TargetCodeGenInfo>
3823CodeGen::createX86_64TargetCodeGenInfo(CodeGenModule &CGM,
3824 X86AVXABILevel AVXLevel) {
3825 return std::make_unique<X86_64TargetCodeGenInfo>(args&: CGM.getTypes(), args&: AVXLevel);
3826}
3827
3828std::unique_ptr<TargetCodeGenInfo>
3829CodeGen::createWinX86_64TargetCodeGenInfo(CodeGenModule &CGM,
3830 X86AVXABILevel AVXLevel) {
3831 return std::make_unique<WinX86_64TargetCodeGenInfo>(args&: CGM.getTypes(), args&: AVXLevel);
3832}
3833