1//===- X86.cpp ------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ABIInfoImpl.h"
10#include "TargetInfo.h"
11#include "clang/Basic/DiagnosticFrontend.h"
12#include "clang/Basic/SourceLocation.h"
13#include "llvm/ADT/SmallBitVector.h"
14
15using namespace clang;
16using namespace clang::CodeGen;
17
18namespace {
19
20/// IsX86_MMXType - Return true if this is an MMX type.
21bool IsX86_MMXType(llvm::Type *IRType) {
22 // Return true if the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>.
23 return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&
24 cast<llvm::VectorType>(Val: IRType)->getElementType()->isIntegerTy() &&
25 IRType->getScalarSizeInBits() != 64;
26}
27
28static llvm::Type *X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
29 StringRef Constraint,
30 llvm::Type *Ty) {
31 bool IsMMXCons = llvm::StringSwitch<bool>(Constraint)
32 .Cases(CaseStrings: {"y", "&y", "^Ym"}, Value: true)
33 .Default(Value: false);
34 if (IsMMXCons && Ty->isVectorTy() &&
35 cast<llvm::VectorType>(Val: Ty)->getPrimitiveSizeInBits().getFixedValue() !=
36 64)
37 return nullptr; // Invalid MMX constraint
38
39 if (Constraint == "k") {
40 llvm::Type *Int1Ty = llvm::Type::getInt1Ty(C&: CGF.getLLVMContext());
41 return llvm::FixedVectorType::get(ElementType: Int1Ty, NumElts: Ty->getScalarSizeInBits());
42 }
43
44 // No operation needed
45 return Ty;
46}
47
48/// Returns true if this type can be passed in SSE registers with the
49/// X86_VectorCall calling convention. Shared between x86_32 and x86_64.
50static bool isX86VectorTypeForVectorCall(ASTContext &Context, QualType Ty) {
51 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
52 if (BT->isFloatingPoint() && BT->getKind() != BuiltinType::Half) {
53 if (BT->getKind() == BuiltinType::LongDouble) {
54 if (&Context.getTargetInfo().getLongDoubleFormat() ==
55 &llvm::APFloat::x87DoubleExtended())
56 return false;
57 }
58 return true;
59 }
60 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
61 // vectorcall can pass XMM, YMM, and ZMM vectors. We don't pass SSE1 MMX
62 // registers specially.
63 unsigned VecSize = Context.getTypeSize(T: VT);
64 if (VecSize == 128 || VecSize == 256 || VecSize == 512)
65 return true;
66 }
67 return false;
68}
69
70/// Returns true if this aggregate is small enough to be passed in SSE registers
71/// in the X86_VectorCall calling convention. Shared between x86_32 and x86_64.
72static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) {
73 return NumMembers <= 4;
74}
75
76/// Returns a Homogeneous Vector Aggregate ABIArgInfo, used in X86.
77static ABIArgInfo getDirectX86Hva(llvm::Type* T = nullptr) {
78 auto AI = ABIArgInfo::getDirect(T);
79 AI.setInReg(true);
80 AI.setCanBeFlattened(false);
81 return AI;
82}
83
84//===----------------------------------------------------------------------===//
85// X86-32 ABI Implementation
86//===----------------------------------------------------------------------===//
87
88/// Similar to llvm::CCState, but for Clang.
89struct CCState {
90 CCState(CGFunctionInfo &FI)
91 : IsPreassigned(FI.arg_size()), CC(FI.getCallingConvention()),
92 Required(FI.getRequiredArgs()), IsDelegateCall(FI.isDelegateCall()) {}
93
94 llvm::SmallBitVector IsPreassigned;
95 unsigned CC = CallingConv::CC_C;
96 unsigned FreeRegs = 0;
97 unsigned FreeSSERegs = 0;
98 RequiredArgs Required;
99 bool IsDelegateCall = false;
100};
101
102/// X86_32ABIInfo - The X86-32 ABI information.
103class X86_32ABIInfo : public ABIInfo {
104 enum Class {
105 Integer,
106 Float
107 };
108
109 static const unsigned MinABIStackAlignInBytes = 4;
110
111 bool IsDarwinVectorABI;
112 bool IsRetSmallStructInRegABI;
113 bool IsWin32StructABI;
114 bool IsSoftFloatABI;
115 bool IsMCUABI;
116 bool IsLinuxABI;
117 unsigned DefaultNumRegisterParameters;
118
119 static bool isRegisterSize(unsigned Size) {
120 return (Size == 8 || Size == 16 || Size == 32 || Size == 64);
121 }
122
123 bool isHomogeneousAggregateBaseType(QualType Ty) const override {
124 // FIXME: Assumes vectorcall is in use.
125 return isX86VectorTypeForVectorCall(Context&: getContext(), Ty);
126 }
127
128 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
129 uint64_t NumMembers) const override {
130 // FIXME: Assumes vectorcall is in use.
131 return isX86VectorCallAggregateSmallEnough(NumMembers);
132 }
133
134 bool shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const;
135
136 /// getIndirectResult - Give a source type \arg Ty, return a suitable result
137 /// such that the argument will be passed in memory.
138 ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
139
140 ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const;
141
142 /// Return the alignment to use for the given type on the stack.
143 unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
144
145 Class classify(QualType Ty) const;
146 ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const;
147 ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State,
148 unsigned ArgIndex) const;
149
150 /// Updates the number of available free registers, returns
151 /// true if any registers were allocated.
152 bool updateFreeRegs(QualType Ty, CCState &State) const;
153
154 bool shouldAggregateUseDirect(QualType Ty, CCState &State, bool &InReg,
155 bool &NeedsPadding) const;
156 bool shouldPrimitiveUseInReg(QualType Ty, CCState &State) const;
157
158 bool canExpandIndirectArgument(QualType Ty) const;
159
160 /// Rewrite the function info so that all memory arguments use
161 /// inalloca.
162 void rewriteWithInAlloca(CGFunctionInfo &FI) const;
163
164 void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
165 CharUnits &StackOffset, ABIArgInfo &Info,
166 QualType Type) const;
167 void runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const;
168
169public:
170
171 void computeInfo(CGFunctionInfo &FI) const override;
172 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
173 AggValueSlot Slot) const override;
174
175 X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
176 bool RetSmallStructInRegABI, bool Win32StructABI,
177 unsigned NumRegisterParameters, bool SoftFloatABI)
178 : ABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI),
179 IsRetSmallStructInRegABI(RetSmallStructInRegABI),
180 IsWin32StructABI(Win32StructABI), IsSoftFloatABI(SoftFloatABI),
181 IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()),
182 IsLinuxABI(CGT.getTarget().getTriple().isOSLinux() ||
183 CGT.getTarget().getTriple().isOSCygMing()),
184 DefaultNumRegisterParameters(NumRegisterParameters) {}
185};
186
187class X86_32SwiftABIInfo : public SwiftABIInfo {
188public:
189 explicit X86_32SwiftABIInfo(CodeGenTypes &CGT)
190 : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/false) {}
191
192 bool shouldPassIndirectly(ArrayRef<llvm::Type *> ComponentTys,
193 bool AsReturnValue) const override {
194 // LLVM's x86-32 lowering currently only assigns up to three
195 // integer registers and three fp registers. Oddly, it'll use up to
196 // four vector registers for vectors, but those can overlap with the
197 // scalar registers.
198 return occupiesMoreThan(scalarTypes: ComponentTys, /*total=*/maxAllRegisters: 3);
199 }
200};
201
202class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
203public:
204 X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
205 bool RetSmallStructInRegABI, bool Win32StructABI,
206 unsigned NumRegisterParameters, bool SoftFloatABI)
207 : TargetCodeGenInfo(std::make_unique<X86_32ABIInfo>(
208 args&: CGT, args&: DarwinVectorABI, args&: RetSmallStructInRegABI, args&: Win32StructABI,
209 args&: NumRegisterParameters, args&: SoftFloatABI)) {
210 SwiftInfo = std::make_unique<X86_32SwiftABIInfo>(args&: CGT);
211 }
212
213 static bool isStructReturnInRegABI(
214 const llvm::Triple &Triple, const CodeGenOptions &Opts);
215
216 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
217 CodeGen::CodeGenModule &CGM) const override;
218
219 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
220 // Darwin uses different dwarf register numbers for EH.
221 if (CGM.getTarget().getTriple().isOSDarwin()) return 5;
222 return 4;
223 }
224
225 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
226 llvm::Value *Address) const override;
227
228 llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
229 StringRef Constraint,
230 llvm::Type* Ty) const override {
231 return X86AdjustInlineAsmType(CGF, Constraint, Ty);
232 }
233
234 void addReturnRegisterOutputs(CodeGenFunction &CGF, LValue ReturnValue,
235 std::string &Constraints,
236 std::vector<llvm::Type *> &ResultRegTypes,
237 std::vector<llvm::Type *> &ResultTruncRegTypes,
238 std::vector<LValue> &ResultRegDests,
239 std::string &AsmString,
240 unsigned NumOutputs) const override;
241
242 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
243 return "movl\t%ebp, %ebp"
244 "\t\t// marker for objc_retainAutoreleaseReturnValue";
245 }
246};
247
248}
249
250/// Rewrite input constraint references after adding some output constraints.
251/// In the case where there is one output and one input and we add one output,
252/// we need to replace all operand references greater than or equal to 1:
253/// mov $0, $1
254/// mov eax, $1
255/// The result will be:
256/// mov $0, $2
257/// mov eax, $2
258static void rewriteInputConstraintReferences(unsigned FirstIn,
259 unsigned NumNewOuts,
260 std::string &AsmString) {
261 std::string Buf;
262 llvm::raw_string_ostream OS(Buf);
263 size_t Pos = 0;
264 while (Pos < AsmString.size()) {
265 size_t DollarStart = AsmString.find(c: '$', pos: Pos);
266 if (DollarStart == std::string::npos)
267 DollarStart = AsmString.size();
268 size_t DollarEnd = AsmString.find_first_not_of(c: '$', pos: DollarStart);
269 if (DollarEnd == std::string::npos)
270 DollarEnd = AsmString.size();
271 OS << StringRef(&AsmString[Pos], DollarEnd - Pos);
272 Pos = DollarEnd;
273 size_t NumDollars = DollarEnd - DollarStart;
274 if (NumDollars % 2 != 0 && Pos < AsmString.size()) {
275 // We have an operand reference.
276 size_t DigitStart = Pos;
277 if (AsmString[DigitStart] == '{') {
278 OS << '{';
279 ++DigitStart;
280 }
281 size_t DigitEnd = AsmString.find_first_not_of(s: "0123456789", pos: DigitStart);
282 if (DigitEnd == std::string::npos)
283 DigitEnd = AsmString.size();
284 StringRef OperandStr(&AsmString[DigitStart], DigitEnd - DigitStart);
285 unsigned OperandIndex;
286 if (!OperandStr.getAsInteger(Radix: 10, Result&: OperandIndex)) {
287 if (OperandIndex >= FirstIn)
288 OperandIndex += NumNewOuts;
289 OS << OperandIndex;
290 } else {
291 OS << OperandStr;
292 }
293 Pos = DigitEnd;
294 }
295 }
296 AsmString = std::move(Buf);
297}
298
299/// Add output constraints for EAX:EDX because they are return registers.
300void X86_32TargetCodeGenInfo::addReturnRegisterOutputs(
301 CodeGenFunction &CGF, LValue ReturnSlot, std::string &Constraints,
302 std::vector<llvm::Type *> &ResultRegTypes,
303 std::vector<llvm::Type *> &ResultTruncRegTypes,
304 std::vector<LValue> &ResultRegDests, std::string &AsmString,
305 unsigned NumOutputs) const {
306 uint64_t RetWidth = CGF.getContext().getTypeSize(T: ReturnSlot.getType());
307
308 // Use the EAX constraint if the width is 32 or smaller and EAX:EDX if it is
309 // larger.
310 if (!Constraints.empty())
311 Constraints += ',';
312 if (RetWidth <= 32) {
313 Constraints += "={eax}";
314 ResultRegTypes.push_back(x: CGF.Int32Ty);
315 } else {
316 // Use the 'A' constraint for EAX:EDX.
317 Constraints += "=A";
318 ResultRegTypes.push_back(x: CGF.Int64Ty);
319 }
320
321 // Truncate EAX or EAX:EDX to an integer of the appropriate size.
322 llvm::Type *CoerceTy = llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: RetWidth);
323 ResultTruncRegTypes.push_back(x: CoerceTy);
324
325 // Coerce the integer by bitcasting the return slot pointer.
326 ReturnSlot.setAddress(ReturnSlot.getAddress().withElementType(ElemTy: CoerceTy));
327 ResultRegDests.push_back(x: ReturnSlot);
328
329 rewriteInputConstraintReferences(FirstIn: NumOutputs, NumNewOuts: 1, AsmString);
330}
331
332/// shouldReturnTypeInRegister - Determine if the given type should be
333/// returned in a register (for the Darwin and MCU ABI).
334bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
335 ASTContext &Context) const {
336 uint64_t Size = Context.getTypeSize(T: Ty);
337
338 // For i386, type must be register sized.
339 // For the MCU ABI, it only needs to be <= 8-byte
340 if ((IsMCUABI && Size > 64) || (!IsMCUABI && !isRegisterSize(Size)))
341 return false;
342
343 if (Ty->isVectorType()) {
344 // 64- and 128- bit vectors inside structures are not returned in
345 // registers.
346 if (Size == 64 || Size == 128)
347 return false;
348
349 return true;
350 }
351
352 // If this is a builtin, pointer, enum, complex type, member pointer, or
353 // member function pointer it is ok.
354 if (Ty->getAs<BuiltinType>() || Ty->hasPointerRepresentation() ||
355 Ty->isAnyComplexType() || Ty->isEnumeralType() ||
356 Ty->isBlockPointerType() || Ty->isMemberPointerType())
357 return true;
358
359 // Arrays are treated like records.
360 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty))
361 return shouldReturnTypeInRegister(Ty: AT->getElementType(), Context);
362
363 // Otherwise, it must be a record type.
364 const auto *RD = Ty->getAsRecordDecl();
365 if (!RD)
366 return false;
367
368 // FIXME: Traverse bases here too.
369
370 // Structure types are passed in register if all fields would be
371 // passed in a register.
372 for (const auto *FD : RD->fields()) {
373 // Empty fields are ignored.
374 if (isEmptyField(Context, FD, AllowArrays: true))
375 continue;
376
377 // Check fields recursively.
378 if (!shouldReturnTypeInRegister(Ty: FD->getType(), Context))
379 return false;
380 }
381 return true;
382}
383
384static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {
385 // Treat complex types as the element type.
386 if (const ComplexType *CTy = Ty->getAs<ComplexType>())
387 Ty = CTy->getElementType();
388
389 // Check for a type which we know has a simple scalar argument-passing
390 // convention without any padding. (We're specifically looking for 32
391 // and 64-bit integer and integer-equivalents, float, and double.)
392 if (!Ty->getAs<BuiltinType>() && !Ty->hasPointerRepresentation() &&
393 !Ty->isEnumeralType() && !Ty->isBlockPointerType())
394 return false;
395
396 uint64_t Size = Context.getTypeSize(T: Ty);
397 return Size == 32 || Size == 64;
398}
399
400static bool addFieldSizes(ASTContext &Context, const RecordDecl *RD,
401 uint64_t &Size) {
402 for (const auto *FD : RD->fields()) {
403 // Scalar arguments on the stack get 4 byte alignment on x86. If the
404 // argument is smaller than 32-bits, expanding the struct will create
405 // alignment padding.
406 if (!is32Or64BitBasicType(Ty: FD->getType(), Context))
407 return false;
408
409 // FIXME: Reject bit-fields wholesale; there are two problems, we don't know
410 // how to expand them yet, and the predicate for telling if a bitfield still
411 // counts as "basic" is more complicated than what we were doing previously.
412 if (FD->isBitField())
413 return false;
414
415 Size += Context.getTypeSize(T: FD->getType());
416 }
417 return true;
418}
419
420static bool addBaseAndFieldSizes(ASTContext &Context, const CXXRecordDecl *RD,
421 uint64_t &Size) {
422 // Don't do this if there are any non-empty bases.
423 for (const CXXBaseSpecifier &Base : RD->bases()) {
424 if (!addBaseAndFieldSizes(Context, RD: Base.getType()->getAsCXXRecordDecl(),
425 Size))
426 return false;
427 }
428 if (!addFieldSizes(Context, RD, Size))
429 return false;
430 return true;
431}
432
433/// Test whether an argument type which is to be passed indirectly (on the
434/// stack) would have the equivalent layout if it was expanded into separate
435/// arguments. If so, we prefer to do the latter to avoid inhibiting
436/// optimizations.
437bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const {
438 // We can only expand structure types.
439 const RecordDecl *RD = Ty->getAsRecordDecl();
440 if (!RD)
441 return false;
442 uint64_t Size = 0;
443 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
444 if (!IsWin32StructABI) {
445 // On non-Windows, we have to conservatively match our old bitcode
446 // prototypes in order to be ABI-compatible at the bitcode level.
447 if (!CXXRD->isCLike())
448 return false;
449 } else {
450 // Don't do this for dynamic classes.
451 if (CXXRD->isDynamicClass())
452 return false;
453 }
454 if (!addBaseAndFieldSizes(Context&: getContext(), RD: CXXRD, Size))
455 return false;
456 } else {
457 if (!addFieldSizes(Context&: getContext(), RD, Size))
458 return false;
459 }
460
461 // We can do this if there was no alignment padding.
462 return Size == getContext().getTypeSize(T: Ty);
463}
464
465ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const {
466 // If the return value is indirect, then the hidden argument is consuming one
467 // integer register.
468 if (State.CC != llvm::CallingConv::X86_FastCall &&
469 State.CC != llvm::CallingConv::X86_VectorCall && State.FreeRegs) {
470 --State.FreeRegs;
471 if (!IsMCUABI)
472 return getNaturalAlignIndirectInReg(Ty: RetTy);
473 }
474 return getNaturalAlignIndirect(
475 Ty: RetTy, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
476 /*ByVal=*/false);
477}
478
479ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
480 CCState &State) const {
481 if (RetTy->isVoidType())
482 return ABIArgInfo::getIgnore();
483
484 const Type *Base = nullptr;
485 uint64_t NumElts = 0;
486 if ((State.CC == llvm::CallingConv::X86_VectorCall ||
487 State.CC == llvm::CallingConv::X86_RegCall) &&
488 isHomogeneousAggregate(Ty: RetTy, Base, Members&: NumElts)) {
489 // The LLVM struct type for such an aggregate should lower properly.
490 return ABIArgInfo::getDirect();
491 }
492
493 if (const VectorType *VT = RetTy->getAs<VectorType>()) {
494 // On Darwin, some vectors are returned in registers.
495 if (IsDarwinVectorABI) {
496 uint64_t Size = getContext().getTypeSize(T: RetTy);
497
498 // 128-bit vectors are a special case; they are returned in
499 // registers and we need to make sure to pick a type the LLVM
500 // backend will like.
501 if (Size == 128)
502 return ABIArgInfo::getDirect(T: llvm::FixedVectorType::get(
503 ElementType: llvm::Type::getInt64Ty(C&: getVMContext()), NumElts: 2));
504
505 // Always return in register if it fits in a general purpose
506 // register, or if it is 64 bits and has a single element.
507 if ((Size == 8 || Size == 16 || Size == 32) ||
508 (Size == 64 && VT->getNumElements() == 1))
509 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(),
510 NumBits: Size));
511
512 return getIndirectReturnResult(RetTy, State);
513 }
514
515 return ABIArgInfo::getDirect();
516 }
517
518 if (isAggregateTypeForABI(T: RetTy)) {
519 if (const auto *RD = RetTy->getAsRecordDecl();
520 RD && RD->hasFlexibleArrayMember())
521 // Structures with flexible arrays are always indirect.
522 return getIndirectReturnResult(RetTy, State);
523
524 // If specified, structs and unions are always indirect.
525 if (!IsRetSmallStructInRegABI && !RetTy->isAnyComplexType())
526 return getIndirectReturnResult(RetTy, State);
527
528 // Ignore empty structs/unions.
529 if (isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true))
530 return ABIArgInfo::getIgnore();
531
532 // Return complex of _Float16 as <2 x half> so the backend will use xmm0.
533 if (const ComplexType *CT = RetTy->getAs<ComplexType>()) {
534 QualType ET = getContext().getCanonicalType(T: CT->getElementType());
535 if (ET->isFloat16Type())
536 return ABIArgInfo::getDirect(T: llvm::FixedVectorType::get(
537 ElementType: llvm::Type::getHalfTy(C&: getVMContext()), NumElts: 2));
538 }
539
540 // Small structures which are register sized are generally returned
541 // in a register.
542 if (shouldReturnTypeInRegister(Ty: RetTy, Context&: getContext())) {
543 uint64_t Size = getContext().getTypeSize(T: RetTy);
544
545 // As a special-case, if the struct is a "single-element" struct, and
546 // the field is of type "float" or "double", return it in a
547 // floating-point register. (MSVC does not apply this special case.)
548 // We apply a similar transformation for pointer types to improve the
549 // quality of the generated IR.
550 if (const Type *SeltTy = isSingleElementStruct(T: RetTy, Context&: getContext()))
551 if ((!IsWin32StructABI && SeltTy->isRealFloatingType())
552 || SeltTy->hasPointerRepresentation())
553 return ABIArgInfo::getDirect(T: CGT.ConvertType(T: QualType(SeltTy, 0)));
554
555 // FIXME: We should be able to narrow this integer in cases with dead
556 // padding.
557 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(),NumBits: Size));
558 }
559
560 return getIndirectReturnResult(RetTy, State);
561 }
562
563 // Treat an enum type as its underlying type.
564 if (const auto *ED = RetTy->getAsEnumDecl())
565 RetTy = ED->getIntegerType();
566
567 if (const auto *EIT = RetTy->getAs<BitIntType>())
568 if (EIT->getNumBits() > 64)
569 return getIndirectReturnResult(RetTy, State);
570
571 return (isPromotableIntegerTypeForABI(Ty: RetTy) ? ABIArgInfo::getExtend(Ty: RetTy)
572 : ABIArgInfo::getDirect());
573}
574
575unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,
576 unsigned Align) const {
577 // Otherwise, if the alignment is less than or equal to the minimum ABI
578 // alignment, just use the default; the backend will handle this.
579 if (Align <= MinABIStackAlignInBytes)
580 return 0; // Use default alignment.
581
582 if (IsLinuxABI) {
583 // Exclude other System V OS (e.g Darwin, PS4 and FreeBSD) since we don't
584 // want to spend any effort dealing with the ramifications of ABI breaks.
585 //
586 // If the vector type is __m128/__m256/__m512, return the default alignment.
587 if (Ty->isVectorType() && (Align == 16 || Align == 32 || Align == 64))
588 return Align;
589 }
590 // On non-Darwin, the stack type alignment is always 4.
591 if (!IsDarwinVectorABI) {
592 // Set explicit alignment, since we may need to realign the top.
593 return MinABIStackAlignInBytes;
594 }
595
596 // Otherwise, if the type contains an SSE vector type, the alignment is 16.
597 if (Align >= 16 && (isSIMDVectorType(Context&: getContext(), Ty) ||
598 isRecordWithSIMDVectorType(Context&: getContext(), Ty)))
599 return 16;
600
601 return MinABIStackAlignInBytes;
602}
603
604ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal,
605 CCState &State) const {
606 if (!ByVal) {
607 if (State.FreeRegs) {
608 --State.FreeRegs; // Non-byval indirects just use one pointer.
609 if (!IsMCUABI)
610 return getNaturalAlignIndirectInReg(Ty);
611 }
612 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
613 ByVal: false);
614 }
615
616 // Compute the byval alignment.
617 unsigned TypeAlign = getContext().getTypeAlign(T: Ty) / 8;
618 unsigned StackAlign = getTypeStackAlignInBytes(Ty, Align: TypeAlign);
619 if (StackAlign == 0)
620 return ABIArgInfo::getIndirect(
621 Alignment: CharUnits::fromQuantity(Quantity: 4),
622 /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
623 /*ByVal=*/true);
624
625 // If the stack alignment is less than the type alignment, realign the
626 // argument.
627 bool Realign = TypeAlign > StackAlign;
628 return ABIArgInfo::getIndirect(
629 Alignment: CharUnits::fromQuantity(Quantity: StackAlign),
630 /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(), /*ByVal=*/true,
631 Realign);
632}
633
634X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {
635 const Type *T = isSingleElementStruct(T: Ty, Context&: getContext());
636 if (!T)
637 T = Ty.getTypePtr();
638
639 if (const BuiltinType *BT = T->getAs<BuiltinType>()) {
640 BuiltinType::Kind K = BT->getKind();
641 if (K == BuiltinType::Float || K == BuiltinType::Double)
642 return Float;
643 }
644 return Integer;
645}
646
647bool X86_32ABIInfo::updateFreeRegs(QualType Ty, CCState &State) const {
648 if (!IsSoftFloatABI) {
649 Class C = classify(Ty);
650 if (C == Float)
651 return false;
652 }
653
654 unsigned Size = getContext().getTypeSize(T: Ty);
655 unsigned SizeInRegs = (Size + 31) / 32;
656
657 if (SizeInRegs == 0)
658 return false;
659
660 if (!IsMCUABI) {
661 if (SizeInRegs > State.FreeRegs) {
662 State.FreeRegs = 0;
663 return false;
664 }
665 } else {
666 // The MCU psABI allows passing parameters in-reg even if there are
667 // earlier parameters that are passed on the stack. Also,
668 // it does not allow passing >8-byte structs in-register,
669 // even if there are 3 free registers available.
670 if (SizeInRegs > State.FreeRegs || SizeInRegs > 2)
671 return false;
672 }
673
674 State.FreeRegs -= SizeInRegs;
675 return true;
676}
677
678bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State,
679 bool &InReg,
680 bool &NeedsPadding) const {
681 // On Windows, aggregates other than HFAs are never passed in registers, and
682 // they do not consume register slots. Homogenous floating-point aggregates
683 // (HFAs) have already been dealt with at this point.
684 if (IsWin32StructABI && isAggregateTypeForABI(T: Ty))
685 return false;
686
687 NeedsPadding = false;
688 InReg = !IsMCUABI;
689
690 if (!updateFreeRegs(Ty, State))
691 return false;
692
693 if (IsMCUABI)
694 return true;
695
696 if (State.CC == llvm::CallingConv::X86_FastCall ||
697 State.CC == llvm::CallingConv::X86_VectorCall ||
698 State.CC == llvm::CallingConv::X86_RegCall) {
699 if (getContext().getTypeSize(T: Ty) <= 32 && State.FreeRegs)
700 NeedsPadding = true;
701
702 return false;
703 }
704
705 return true;
706}
707
708bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {
709 bool IsPtrOrInt = (getContext().getTypeSize(T: Ty) <= 32) &&
710 (Ty->isIntegralOrEnumerationType() || Ty->isPointerType() ||
711 Ty->isReferenceType());
712
713 if (!IsPtrOrInt && (State.CC == llvm::CallingConv::X86_FastCall ||
714 State.CC == llvm::CallingConv::X86_VectorCall))
715 return false;
716
717 if (!updateFreeRegs(Ty, State))
718 return false;
719
720 if (!IsPtrOrInt && State.CC == llvm::CallingConv::X86_RegCall)
721 return false;
722
723 // Return true to apply inreg to all legal parameters except for MCU targets.
724 return !IsMCUABI;
725}
726
727void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {
728 // Vectorcall x86 works subtly different than in x64, so the format is
729 // a bit different than the x64 version. First, all vector types (not HVAs)
730 // are assigned, with the first 6 ending up in the [XYZ]MM0-5 registers.
731 // This differs from the x64 implementation, where the first 6 by INDEX get
732 // registers.
733 // In the second pass over the arguments, HVAs are passed in the remaining
734 // vector registers if possible, or indirectly by address. The address will be
735 // passed in ECX/EDX if available. Any other arguments are passed according to
736 // the usual fastcall rules.
737 MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
738 for (int I = 0, E = Args.size(); I < E; ++I) {
739 const Type *Base = nullptr;
740 uint64_t NumElts = 0;
741 const QualType &Ty = Args[I].type;
742 if ((Ty->isVectorType() || Ty->isBuiltinType()) &&
743 isHomogeneousAggregate(Ty, Base, Members&: NumElts)) {
744 if (State.FreeSSERegs >= NumElts) {
745 State.FreeSSERegs -= NumElts;
746 Args[I].info = ABIArgInfo::getDirectInReg();
747 State.IsPreassigned.set(I);
748 }
749 }
750 }
751}
752
753ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, CCState &State,
754 unsigned ArgIndex) const {
755 // FIXME: Set alignment on indirect arguments.
756 bool IsFastCall = State.CC == llvm::CallingConv::X86_FastCall;
757 bool IsRegCall = State.CC == llvm::CallingConv::X86_RegCall;
758 bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall;
759
760 Ty = useFirstFieldIfTransparentUnion(Ty);
761 TypeInfo TI = getContext().getTypeInfo(T: Ty);
762
763 // Check with the C++ ABI first.
764 const RecordType *RT = Ty->getAsCanonical<RecordType>();
765 if (RT) {
766 CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, CXXABI&: getCXXABI());
767 if (RAA == CGCXXABI::RAA_Indirect) {
768 return getIndirectResult(Ty, ByVal: false, State);
769 } else if (State.IsDelegateCall) {
770 // Avoid having different alignments on delegate call args by always
771 // setting the alignment to 4, which is what we do for inallocas.
772 ABIArgInfo Res = getIndirectResult(Ty, ByVal: false, State);
773 Res.setIndirectAlign(CharUnits::fromQuantity(Quantity: 4));
774 return Res;
775 } else if (RAA == CGCXXABI::RAA_DirectInMemory) {
776 // The field index doesn't matter, we'll fix it up later.
777 return ABIArgInfo::getInAlloca(/*FieldIndex=*/0);
778 }
779 }
780
781 // Regcall uses the concept of a homogenous vector aggregate, similar
782 // to other targets.
783 const Type *Base = nullptr;
784 uint64_t NumElts = 0;
785 if ((IsRegCall || IsVectorCall) &&
786 isHomogeneousAggregate(Ty, Base, Members&: NumElts)) {
787 if (State.FreeSSERegs >= NumElts) {
788 State.FreeSSERegs -= NumElts;
789
790 // Vectorcall passes HVAs directly and does not flatten them, but regcall
791 // does.
792 if (IsVectorCall)
793 return getDirectX86Hva();
794
795 if (Ty->isBuiltinType() || Ty->isVectorType())
796 return ABIArgInfo::getDirect();
797 return ABIArgInfo::getExpand();
798 }
799 if (IsVectorCall && Ty->isBuiltinType())
800 return ABIArgInfo::getDirect();
801 return getIndirectResult(Ty, /*ByVal=*/false, State);
802 }
803
804 if (isAggregateTypeForABI(T: Ty)) {
805 // Structures with flexible arrays are always indirect.
806 // FIXME: This should not be byval!
807 if (RT && RT->getDecl()->getDefinitionOrSelf()->hasFlexibleArrayMember())
808 return getIndirectResult(Ty, ByVal: true, State);
809
810 // Ignore empty structs/unions on non-Windows.
811 if (!IsWin32StructABI && isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
812 return ABIArgInfo::getIgnore();
813
814 // Ignore 0 sized structs.
815 if (TI.Width == 0)
816 return ABIArgInfo::getIgnore();
817
818 llvm::LLVMContext &LLVMContext = getVMContext();
819 llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(C&: LLVMContext);
820 bool NeedsPadding = false;
821 bool InReg;
822 if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) {
823 unsigned SizeInRegs = (TI.Width + 31) / 32;
824 SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32);
825 llvm::Type *Result = llvm::StructType::get(Context&: LLVMContext, Elements);
826 if (InReg)
827 return ABIArgInfo::getDirectInReg(T: Result);
828 else
829 return ABIArgInfo::getDirect(T: Result);
830 }
831 llvm::IntegerType *PaddingType = NeedsPadding ? Int32 : nullptr;
832
833 // Pass over-aligned aggregates to non-variadic functions on Windows
834 // indirectly. This behavior was added in MSVC 2015. Use the required
835 // alignment from the record layout, since that may be less than the
836 // regular type alignment, and types with required alignment of less than 4
837 // bytes are not passed indirectly.
838 if (IsWin32StructABI && State.Required.isRequiredArg(argIdx: ArgIndex)) {
839 unsigned AlignInBits = 0;
840 if (RT) {
841 const ASTRecordLayout &Layout =
842 getContext().getASTRecordLayout(D: RT->getDecl());
843 AlignInBits = getContext().toBits(CharSize: Layout.getRequiredAlignment());
844 } else if (TI.isAlignRequired()) {
845 AlignInBits = TI.Align;
846 }
847 if (AlignInBits > 32)
848 return getIndirectResult(Ty, /*ByVal=*/false, State);
849 }
850
851 // Expand small (<= 128-bit) record types when we know that the stack layout
852 // of those arguments will match the struct. This is important because the
853 // LLVM backend isn't smart enough to remove byval, which inhibits many
854 // optimizations.
855 // Don't do this for the MCU if there are still free integer registers
856 // (see X86_64 ABI for full explanation).
857 if (TI.Width <= 4 * 32 && (!IsMCUABI || State.FreeRegs == 0) &&
858 canExpandIndirectArgument(Ty))
859 return ABIArgInfo::getExpandWithPadding(
860 PaddingInReg: IsFastCall || IsVectorCall || IsRegCall, Padding: PaddingType);
861
862 return getIndirectResult(Ty, ByVal: true, State);
863 }
864
865 if (const VectorType *VT = Ty->getAs<VectorType>()) {
866 // On Windows, vectors are passed directly if registers are available, or
867 // indirectly if not. This avoids the need to align argument memory. Pass
868 // user-defined vector types larger than 512 bits indirectly for simplicity.
869 if (IsWin32StructABI) {
870 if (TI.Width <= 512 && State.FreeSSERegs > 0) {
871 --State.FreeSSERegs;
872 return ABIArgInfo::getDirectInReg();
873 }
874 return getIndirectResult(Ty, /*ByVal=*/false, State);
875 }
876
877 // On Darwin, some vectors are passed in memory, we handle this by passing
878 // it as an i8/i16/i32/i64.
879 if (IsDarwinVectorABI) {
880 if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) ||
881 (TI.Width == 64 && VT->getNumElements() == 1))
882 return ABIArgInfo::getDirect(
883 T: llvm::IntegerType::get(C&: getVMContext(), NumBits: TI.Width));
884 }
885
886 if (IsX86_MMXType(IRType: CGT.ConvertType(T: Ty)))
887 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(), NumBits: 64));
888
889 return ABIArgInfo::getDirect();
890 }
891
892 if (const auto *ED = Ty->getAsEnumDecl())
893 Ty = ED->getIntegerType();
894
895 bool InReg = shouldPrimitiveUseInReg(Ty, State);
896
897 if (isPromotableIntegerTypeForABI(Ty)) {
898 if (InReg)
899 return ABIArgInfo::getExtendInReg(Ty, T: CGT.ConvertType(T: Ty));
900 return ABIArgInfo::getExtend(Ty, T: CGT.ConvertType(T: Ty));
901 }
902
903 if (const auto *EIT = Ty->getAs<BitIntType>()) {
904 if (EIT->getNumBits() <= 64) {
905 if (InReg)
906 return ABIArgInfo::getDirectInReg();
907 return ABIArgInfo::getDirect();
908 }
909 return getIndirectResult(Ty, /*ByVal=*/false, State);
910 }
911
912 if (InReg)
913 return ABIArgInfo::getDirectInReg();
914 return ABIArgInfo::getDirect();
915}
916
917void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
918 CCState State(FI);
919 if (IsMCUABI)
920 State.FreeRegs = 3;
921 else if (State.CC == llvm::CallingConv::X86_FastCall) {
922 State.FreeRegs = 2;
923 State.FreeSSERegs = 3;
924 } else if (State.CC == llvm::CallingConv::X86_VectorCall) {
925 State.FreeRegs = 2;
926 State.FreeSSERegs = 6;
927 } else if (FI.getHasRegParm())
928 State.FreeRegs = FI.getRegParm();
929 else if (State.CC == llvm::CallingConv::X86_RegCall) {
930 State.FreeRegs = 5;
931 State.FreeSSERegs = 8;
932 } else if (IsWin32StructABI) {
933 // Since MSVC 2015, the first three SSE vectors have been passed in
934 // registers. The rest are passed indirectly.
935 State.FreeRegs = DefaultNumRegisterParameters;
936 State.FreeSSERegs = 3;
937 } else
938 State.FreeRegs = DefaultNumRegisterParameters;
939
940 if (!::classifyReturnType(CXXABI: getCXXABI(), FI, Info: *this)) {
941 FI.getReturnInfo() = classifyReturnType(RetTy: FI.getReturnType(), State);
942 } else if (FI.getReturnInfo().isIndirect()) {
943 // The C++ ABI is not aware of register usage, so we have to check if the
944 // return value was sret and put it in a register ourselves if appropriate.
945 if (State.FreeRegs) {
946 --State.FreeRegs; // The sret parameter consumes a register.
947 if (!IsMCUABI)
948 FI.getReturnInfo().setInReg(true);
949 }
950 }
951
952 // The chain argument effectively gives us another free register.
953 if (FI.isChainCall())
954 ++State.FreeRegs;
955
956 // For vectorcall, do a first pass over the arguments, assigning FP and vector
957 // arguments to XMM registers as available.
958 if (State.CC == llvm::CallingConv::X86_VectorCall)
959 runVectorCallFirstPass(FI, State);
960
961 bool UsedInAlloca = false;
962 MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
963 for (unsigned I = 0, E = Args.size(); I < E; ++I) {
964 // Skip arguments that have already been assigned.
965 if (State.IsPreassigned.test(Idx: I))
966 continue;
967
968 Args[I].info =
969 classifyArgumentType(Ty: Args[I].type, State, ArgIndex: I);
970 UsedInAlloca |= (Args[I].info.getKind() == ABIArgInfo::InAlloca);
971 }
972
973 // If we needed to use inalloca for any argument, do a second pass and rewrite
974 // all the memory arguments to use inalloca.
975 if (UsedInAlloca)
976 rewriteWithInAlloca(FI);
977}
978
979void
980X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
981 CharUnits &StackOffset, ABIArgInfo &Info,
982 QualType Type) const {
983 // Arguments are always 4-byte-aligned.
984 CharUnits WordSize = CharUnits::fromQuantity(Quantity: 4);
985 assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct");
986
987 // sret pointers and indirect things will require an extra pointer
988 // indirection, unless they are byval. Most things are byval, and will not
989 // require this indirection.
990 bool IsIndirect = false;
991 if (Info.isIndirect() && !Info.getIndirectByVal())
992 IsIndirect = true;
993 Info = ABIArgInfo::getInAlloca(FieldIndex: FrameFields.size(), Indirect: IsIndirect);
994 llvm::Type *LLTy = CGT.ConvertTypeForMem(T: Type);
995 if (IsIndirect)
996 LLTy = llvm::PointerType::getUnqual(C&: getVMContext());
997 FrameFields.push_back(Elt: LLTy);
998 StackOffset += IsIndirect ? WordSize : getContext().getTypeSizeInChars(T: Type);
999
1000 // Insert padding bytes to respect alignment.
1001 CharUnits FieldEnd = StackOffset;
1002 StackOffset = FieldEnd.alignTo(Align: WordSize);
1003 if (StackOffset != FieldEnd) {
1004 CharUnits NumBytes = StackOffset - FieldEnd;
1005 llvm::Type *Ty = llvm::Type::getInt8Ty(C&: getVMContext());
1006 Ty = llvm::ArrayType::get(ElementType: Ty, NumElements: NumBytes.getQuantity());
1007 FrameFields.push_back(Elt: Ty);
1008 }
1009}
1010
1011static bool isArgInAlloca(const ABIArgInfo &Info) {
1012 // Leave ignored and inreg arguments alone.
1013 switch (Info.getKind()) {
1014 case ABIArgInfo::InAlloca:
1015 return true;
1016 case ABIArgInfo::Ignore:
1017 case ABIArgInfo::IndirectAliased:
1018 case ABIArgInfo::TargetSpecific:
1019 return false;
1020 case ABIArgInfo::Indirect:
1021 case ABIArgInfo::Direct:
1022 case ABIArgInfo::Extend:
1023 return !Info.getInReg();
1024 case ABIArgInfo::Expand:
1025 case ABIArgInfo::CoerceAndExpand:
1026 // These are aggregate types which are never passed in registers when
1027 // inalloca is involved.
1028 return true;
1029 }
1030 llvm_unreachable("invalid enum");
1031}
1032
1033void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const {
1034 assert(IsWin32StructABI && "inalloca only supported on win32");
1035
1036 // Build a packed struct type for all of the arguments in memory.
1037 SmallVector<llvm::Type *, 6> FrameFields;
1038
1039 // The stack alignment is always 4.
1040 CharUnits StackAlign = CharUnits::fromQuantity(Quantity: 4);
1041
1042 CharUnits StackOffset;
1043 CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end();
1044
1045 // Put 'this' into the struct before 'sret', if necessary.
1046 bool IsThisCall =
1047 FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall;
1048 ABIArgInfo &Ret = FI.getReturnInfo();
1049 if (Ret.isIndirect() && Ret.isSRetAfterThis() && !IsThisCall &&
1050 isArgInAlloca(Info: I->info)) {
1051 addFieldToArgStruct(FrameFields, StackOffset, Info&: I->info, Type: I->type);
1052 ++I;
1053 }
1054
1055 // Put the sret parameter into the inalloca struct if it's in memory.
1056 if (Ret.isIndirect() && !Ret.getInReg()) {
1057 addFieldToArgStruct(FrameFields, StackOffset, Info&: Ret, Type: FI.getReturnType());
1058 // On Windows, the hidden sret parameter is always returned in eax.
1059 Ret.setInAllocaSRet(IsWin32StructABI);
1060 }
1061
1062 // Skip the 'this' parameter in ecx.
1063 if (IsThisCall)
1064 ++I;
1065
1066 // Put arguments passed in memory into the struct.
1067 for (; I != E; ++I) {
1068 if (isArgInAlloca(Info: I->info))
1069 addFieldToArgStruct(FrameFields, StackOffset, Info&: I->info, Type: I->type);
1070 }
1071
1072 FI.setArgStruct(Ty: llvm::StructType::get(Context&: getVMContext(), Elements: FrameFields,
1073 /*isPacked=*/true),
1074 Align: StackAlign);
1075}
1076
1077RValue X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1078 QualType Ty, AggValueSlot Slot) const {
1079
1080 auto TypeInfo = getContext().getTypeInfoInChars(T: Ty);
1081
1082 CCState State(*const_cast<CGFunctionInfo *>(CGF.CurFnInfo));
1083 ABIArgInfo AI = classifyArgumentType(Ty, State, /*ArgIndex*/ 0);
1084 // Empty records are ignored for parameter passing purposes.
1085 if (AI.isIgnore())
1086 return Slot.asRValue();
1087
1088 // x86-32 changes the alignment of certain arguments on the stack.
1089 //
1090 // Just messing with TypeInfo like this works because we never pass
1091 // anything indirectly.
1092 TypeInfo.Align = CharUnits::fromQuantity(
1093 Quantity: getTypeStackAlignInBytes(Ty, Align: TypeInfo.Align.getQuantity()));
1094
1095 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, /*Indirect*/ IsIndirect: false, ValueInfo: TypeInfo,
1096 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 4),
1097 /*AllowHigherAlign*/ true, Slot);
1098}
1099
1100bool X86_32TargetCodeGenInfo::isStructReturnInRegABI(
1101 const llvm::Triple &Triple, const CodeGenOptions &Opts) {
1102 assert(Triple.getArch() == llvm::Triple::x86);
1103
1104 switch (Opts.getStructReturnConvention()) {
1105 case CodeGenOptions::SRCK_Default:
1106 break;
1107 case CodeGenOptions::SRCK_OnStack: // -fpcc-struct-return
1108 return false;
1109 case CodeGenOptions::SRCK_InRegs: // -freg-struct-return
1110 return true;
1111 }
1112
1113 if (Triple.isOSDarwin() || Triple.isOSIAMCU())
1114 return true;
1115
1116 switch (Triple.getOS()) {
1117 case llvm::Triple::DragonFly:
1118 case llvm::Triple::FreeBSD:
1119 case llvm::Triple::OpenBSD:
1120 case llvm::Triple::Win32:
1121 return true;
1122 default:
1123 return false;
1124 }
1125}
1126
1127static void addX86InterruptAttrs(const FunctionDecl *FD, llvm::GlobalValue *GV,
1128 CodeGen::CodeGenModule &CGM) {
1129 if (!FD->hasAttr<AnyX86InterruptAttr>())
1130 return;
1131
1132 llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1133 Fn->setCallingConv(llvm::CallingConv::X86_INTR);
1134 if (FD->getNumParams() == 0)
1135 return;
1136
1137 auto PtrTy = cast<PointerType>(Val: FD->getParamDecl(i: 0)->getType());
1138 llvm::Type *ByValTy = CGM.getTypes().ConvertType(T: PtrTy->getPointeeType());
1139 llvm::Attribute NewAttr = llvm::Attribute::getWithByValType(
1140 Context&: Fn->getContext(), Ty: ByValTy);
1141 Fn->addParamAttr(ArgNo: 0, Attr: NewAttr);
1142}
1143
1144void X86_32TargetCodeGenInfo::setTargetAttributes(
1145 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
1146 if (GV->isDeclaration())
1147 return;
1148 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D)) {
1149 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
1150 llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1151 Fn->addFnAttr(Kind: "stackrealign");
1152 }
1153
1154 addX86InterruptAttrs(FD, GV, CGM);
1155 }
1156}
1157
1158bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
1159 CodeGen::CodeGenFunction &CGF,
1160 llvm::Value *Address) const {
1161 CodeGen::CGBuilderTy &Builder = CGF.Builder;
1162
1163 llvm::Value *Four8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 4);
1164
1165 // 0-7 are the eight integer registers; the order is different
1166 // on Darwin (for EH), but the range is the same.
1167 // 8 is %eip.
1168 AssignToArrayRange(Builder, Array: Address, Value: Four8, FirstIndex: 0, LastIndex: 8);
1169
1170 if (CGF.CGM.getTarget().getTriple().isOSDarwin()) {
1171 // 12-16 are st(0..4). Not sure why we stop at 4.
1172 // These have size 16, which is sizeof(long double) on
1173 // platforms with 8-byte alignment for that type.
1174 llvm::Value *Sixteen8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 16);
1175 AssignToArrayRange(Builder, Array: Address, Value: Sixteen8, FirstIndex: 12, LastIndex: 16);
1176
1177 } else {
1178 // 9 is %eflags, which doesn't get a size on Darwin for some
1179 // reason.
1180 Builder.CreateAlignedStore(
1181 Val: Four8, Addr: Builder.CreateConstInBoundsGEP1_32(Ty: CGF.Int8Ty, Ptr: Address, Idx0: 9),
1182 Align: CharUnits::One());
1183
1184 // 11-16 are st(0..5). Not sure why we stop at 5.
1185 // These have size 12, which is sizeof(long double) on
1186 // platforms with 4-byte alignment for that type.
1187 llvm::Value *Twelve8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 12);
1188 AssignToArrayRange(Builder, Array: Address, Value: Twelve8, FirstIndex: 11, LastIndex: 16);
1189 }
1190
1191 return false;
1192}
1193
1194//===----------------------------------------------------------------------===//
1195// X86-64 ABI Implementation
1196//===----------------------------------------------------------------------===//
1197
1198
1199namespace {
1200
1201/// \p returns the size in bits of the largest (native) vector for \p AVXLevel.
1202static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
1203 switch (AVXLevel) {
1204 case X86AVXABILevel::AVX512:
1205 return 512;
1206 case X86AVXABILevel::AVX:
1207 return 256;
1208 case X86AVXABILevel::None:
1209 return 128;
1210 }
1211 llvm_unreachable("Unknown AVXLevel");
1212}
1213
1214/// X86_64ABIInfo - The X86_64 ABI information.
1215class X86_64ABIInfo : public ABIInfo {
1216 enum Class {
1217 Integer = 0,
1218 SSE,
1219 SSEUp,
1220 X87,
1221 X87Up,
1222 ComplexX87,
1223 NoClass,
1224 Memory
1225 };
1226
1227 /// merge - Implement the X86_64 ABI merging algorithm.
1228 ///
1229 /// Merge an accumulating classification \arg Accum with a field
1230 /// classification \arg Field.
1231 ///
1232 /// \param Accum - The accumulating classification. This should
1233 /// always be either NoClass or the result of a previous merge
1234 /// call. In addition, this should never be Memory (the caller
1235 /// should just return Memory for the aggregate).
1236 static Class merge(Class Accum, Class Field);
1237
1238 /// postMerge - Implement the X86_64 ABI post merging algorithm.
1239 ///
1240 /// Post merger cleanup, reduces a malformed Hi and Lo pair to
1241 /// final MEMORY or SSE classes when necessary.
1242 ///
1243 /// \param AggregateSize - The size of the current aggregate in
1244 /// the classification process.
1245 ///
1246 /// \param Lo - The classification for the parts of the type
1247 /// residing in the low word of the containing object.
1248 ///
1249 /// \param Hi - The classification for the parts of the type
1250 /// residing in the higher words of the containing object.
1251 ///
1252 void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
1253
1254 /// classify - Determine the x86_64 register classes in which the
1255 /// given type T should be passed.
1256 ///
1257 /// \param Lo - The classification for the parts of the type
1258 /// residing in the low word of the containing object.
1259 ///
1260 /// \param Hi - The classification for the parts of the type
1261 /// residing in the high word of the containing object.
1262 ///
1263 /// \param OffsetBase - The bit offset of this type in the
1264 /// containing object. Some parameters are classified different
1265 /// depending on whether they straddle an eightbyte boundary.
1266 ///
1267 /// \param isNamedArg - Whether the argument in question is a "named"
1268 /// argument, as used in AMD64-ABI 3.5.7.
1269 ///
1270 /// \param IsRegCall - Whether the calling conversion is regcall.
1271 ///
1272 /// If a word is unused its result will be NoClass; if a type should
1273 /// be passed in Memory then at least the classification of \arg Lo
1274 /// will be Memory.
1275 ///
1276 /// The \arg Lo class will be NoClass iff the argument is ignored.
1277 ///
1278 /// If the \arg Lo class is ComplexX87, then the \arg Hi class will
1279 /// also be ComplexX87.
1280 void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,
1281 bool isNamedArg, bool IsRegCall = false) const;
1282
1283 llvm::Type *GetByteVectorType(QualType Ty) const;
1284 llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
1285 unsigned IROffset, QualType SourceTy,
1286 unsigned SourceOffset) const;
1287 llvm::Type *GetINTEGERTypeAtOffset(llvm::Type *IRType,
1288 unsigned IROffset, QualType SourceTy,
1289 unsigned SourceOffset) const;
1290
1291 /// getIndirectResult - Give a source type \arg Ty, return a suitable result
1292 /// such that the argument will be returned in memory.
1293 ABIArgInfo getIndirectReturnResult(QualType Ty) const;
1294
1295 /// getIndirectResult - Give a source type \arg Ty, return a suitable result
1296 /// such that the argument will be passed in memory.
1297 ///
1298 /// \param freeIntRegs - The number of free integer registers remaining
1299 /// available.
1300 ABIArgInfo getIndirectResult(QualType Ty, unsigned freeIntRegs) const;
1301
1302 ABIArgInfo classifyReturnType(QualType RetTy) const;
1303
1304 ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs,
1305 unsigned &neededInt, unsigned &neededSSE,
1306 bool isNamedArg,
1307 bool IsRegCall = false) const;
1308
1309 ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
1310 unsigned &NeededSSE,
1311 unsigned &MaxVectorWidth) const;
1312
1313 bool passRegCallStructTypeDirectly(QualType Ty,
1314 SmallVectorImpl<llvm::Type *> &CoerceElts,
1315 unsigned &NeededInt, unsigned &NeededSSE,
1316 unsigned &MaxVectorWidth) const;
1317
1318 bool IsIllegalVectorType(QualType Ty) const;
1319
1320 /// The 0.98 ABI revision clarified a lot of ambiguities,
1321 /// unfortunately in ways that were not always consistent with
1322 /// certain previous compilers. In particular, platforms which
1323 /// required strict binary compatibility with older versions of GCC
1324 /// may need to exempt themselves.
1325 bool honorsRevision0_98() const {
1326 return !getTarget().getTriple().isOSDarwin();
1327 }
1328
1329 /// GCC classifies <1 x long long> as SSE but some platform ABIs choose to
1330 /// classify it as INTEGER (for compatibility with older clang compilers).
1331 bool classifyIntegerMMXAsSSE() const {
1332 // Clang <= 3.8 did not do this.
1333 if (getContext().getLangOpts().isCompatibleWith(
1334 Version: LangOptions::ClangABI::Ver3_8))
1335 return false;
1336
1337 const llvm::Triple &Triple = getTarget().getTriple();
1338 if (Triple.isOSDarwin() || Triple.isPS() || Triple.isOSFreeBSD())
1339 return false;
1340 return true;
1341 }
1342
1343 // GCC classifies vectors of __int128 as memory.
1344 bool passInt128VectorsInMem() const {
1345 // Clang <= 9.0 did not do this.
1346 if (getContext().getLangOpts().isCompatibleWith(
1347 Version: LangOptions::ClangABI::Ver9))
1348 return false;
1349
1350 const llvm::Triple &T = getTarget().getTriple();
1351 return T.isOSLinux() || T.isOSNetBSD();
1352 }
1353
1354 bool returnCXXRecordGreaterThan128InMem() const {
1355 // Clang <= 20.0 did not do this, and PlayStation does not do this.
1356 if (getContext().getLangOpts().isCompatibleWith(
1357 Version: LangOptions::ClangABI::Ver20) ||
1358 getTarget().getTriple().isPS())
1359 return false;
1360
1361 return true;
1362 }
1363
1364 X86AVXABILevel AVXLevel;
1365 // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
1366 // 64-bit hardware.
1367 bool Has64BitPointers;
1368
1369public:
1370 X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
1371 : ABIInfo(CGT), AVXLevel(AVXLevel),
1372 Has64BitPointers(CGT.getDataLayout().getPointerSize(AS: 0) == 8) {}
1373
1374 bool isPassedUsingAVXType(QualType type) const {
1375 unsigned neededInt, neededSSE;
1376 // The freeIntRegs argument doesn't matter here.
1377 ABIArgInfo info = classifyArgumentType(Ty: type, freeIntRegs: 0, neededInt, neededSSE,
1378 /*isNamedArg*/true);
1379 if (info.isDirect()) {
1380 llvm::Type *ty = info.getCoerceToType();
1381 if (llvm::VectorType *vectorTy = dyn_cast_or_null<llvm::VectorType>(Val: ty))
1382 return vectorTy->getPrimitiveSizeInBits().getFixedValue() > 128;
1383 }
1384 return false;
1385 }
1386
1387 void computeInfo(CGFunctionInfo &FI) const override;
1388 unsigned getX86ABIAVXLevel(const FunctionDecl *FD,
1389 const FunctionType::ExtInfo &Info) const override;
1390
1391 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
1392 AggValueSlot Slot) const override;
1393 RValue EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
1394 AggValueSlot Slot) const override;
1395
1396 bool has64BitPointers() const {
1397 return Has64BitPointers;
1398 }
1399};
1400
1401/// WinX86_64ABIInfo - The Windows X86_64 ABI information.
1402class WinX86_64ABIInfo : public ABIInfo {
1403public:
1404 WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
1405 : ABIInfo(CGT), AVXLevel(AVXLevel),
1406 IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {}
1407
1408 void computeInfo(CGFunctionInfo &FI) const override;
1409 unsigned getX86ABIAVXLevel(const FunctionDecl *FD,
1410 const FunctionType::ExtInfo &Info) const override;
1411
1412 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
1413 AggValueSlot Slot) const override;
1414
1415 bool isHomogeneousAggregateBaseType(QualType Ty) const override {
1416 // FIXME: Assumes vectorcall is in use.
1417 return isX86VectorTypeForVectorCall(Context&: getContext(), Ty);
1418 }
1419
1420 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
1421 uint64_t NumMembers) const override {
1422 // FIXME: Assumes vectorcall is in use.
1423 return isX86VectorCallAggregateSmallEnough(NumMembers);
1424 }
1425
1426 ABIArgInfo classifyArgForArm64ECVarArg(QualType Ty) const override {
1427 unsigned FreeSSERegs = 0;
1428 return classify(Ty, FreeSSERegs, /*IsReturnType=*/false,
1429 CC: llvm::CallingConv::C);
1430 }
1431
1432private:
1433 ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, bool IsReturnType,
1434 unsigned CC) const;
1435 ABIArgInfo reclassifyHvaArgForVectorCall(QualType Ty, unsigned &FreeSSERegs,
1436 const ABIArgInfo &current) const;
1437
1438 X86AVXABILevel AVXLevel;
1439
1440 bool IsMingw64;
1441};
1442
1443class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
1444public:
1445 X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
1446 : TargetCodeGenInfo(std::make_unique<X86_64ABIInfo>(args&: CGT, args&: AVXLevel)) {
1447 SwiftInfo =
1448 std::make_unique<SwiftABIInfo>(args&: CGT, /*SwiftErrorInRegister=*/args: true);
1449 }
1450
1451 /// Disable tail call on x86-64. The epilogue code before the tail jump blocks
1452 /// autoreleaseRV/retainRV and autoreleaseRV/unsafeClaimRV optimizations.
1453 bool markARCOptimizedReturnCallsAsNoTail() const override { return true; }
1454
1455 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1456 return 7;
1457 }
1458
1459 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1460 llvm::Value *Address) const override {
1461 llvm::Value *Eight8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 8);
1462
1463 // 0-15 are the 16 integer registers.
1464 // 16 is %rip.
1465 AssignToArrayRange(Builder&: CGF.Builder, Array: Address, Value: Eight8, FirstIndex: 0, LastIndex: 16);
1466 return false;
1467 }
1468
1469 llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1470 StringRef Constraint,
1471 llvm::Type* Ty) const override {
1472 return X86AdjustInlineAsmType(CGF, Constraint, Ty);
1473 }
1474
1475 bool isNoProtoCallVariadic(const CallArgList &args,
1476 const FunctionNoProtoType *fnType) const override {
1477 // The default CC on x86-64 sets %al to the number of SSA
1478 // registers used, and GCC sets this when calling an unprototyped
1479 // function, so we override the default behavior. However, don't do
1480 // that when AVX types are involved: the ABI explicitly states it is
1481 // undefined, and it doesn't work in practice because of how the ABI
1482 // defines varargs anyway.
1483 if (fnType->getCallConv() == CC_C) {
1484 bool HasAVXType = false;
1485 for (const CallArg &arg : args) {
1486 if (getABIInfo<X86_64ABIInfo>().isPassedUsingAVXType(type: arg.Ty)) {
1487 HasAVXType = true;
1488 break;
1489 }
1490 }
1491
1492 if (!HasAVXType)
1493 return true;
1494 }
1495
1496 return TargetCodeGenInfo::isNoProtoCallVariadic(args, fnType);
1497 }
1498
1499 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1500 CodeGen::CodeGenModule &CGM) const override {
1501 if (GV->isDeclaration())
1502 return;
1503 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D)) {
1504 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
1505 llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1506 Fn->addFnAttr(Kind: "stackrealign");
1507 }
1508
1509 addX86InterruptAttrs(FD, GV, CGM);
1510 }
1511 }
1512
1513 void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
1514 const FunctionDecl *Caller,
1515 const FunctionDecl *Callee, const CallArgList &Args,
1516 QualType ReturnType) const override;
1517
1518 void checkFunctionABI(CodeGenModule &CGM,
1519 const FunctionDecl *FD) const override;
1520};
1521} // namespace
1522
1523static void initFeatureMaps(const ASTContext &Ctx,
1524 llvm::StringMap<bool> &CallerMap,
1525 const FunctionDecl *Caller,
1526 llvm::StringMap<bool> &CalleeMap,
1527 const FunctionDecl *Callee) {
1528 if (CalleeMap.empty() && CallerMap.empty()) {
1529 // The caller is potentially nullptr in the case where the call isn't in a
1530 // function. In this case, the getFunctionFeatureMap ensures we just get
1531 // the TU level setting (since it cannot be modified by 'target'..
1532 Ctx.getFunctionFeatureMap(FeatureMap&: CallerMap, Caller);
1533 Ctx.getFunctionFeatureMap(FeatureMap&: CalleeMap, Callee);
1534 }
1535}
1536
1537static bool checkAVXParamFeature(DiagnosticsEngine &Diag,
1538 SourceLocation CallLoc,
1539 const FunctionDecl &Callee,
1540 const llvm::StringMap<bool> &CallerMap,
1541 const llvm::StringMap<bool> &CalleeMap,
1542 QualType Ty, StringRef Feature,
1543 bool IsArgument) {
1544 bool CallerHasFeat = CallerMap.lookup(Key: Feature);
1545 bool CalleeHasFeat = CalleeMap.lookup(Key: Feature);
1546 // No explicit features and the function is internal, be permissive.
1547 if (!CallerHasFeat && !CalleeHasFeat &&
1548 (!Callee.isExternallyVisible() || Callee.hasAttr<AlwaysInlineAttr>()))
1549 return false;
1550
1551 if (!CallerHasFeat && !CalleeHasFeat)
1552 return Diag.Report(Loc: CallLoc, DiagID: diag::warn_avx_calling_convention)
1553 << IsArgument << Ty << Feature;
1554
1555 // Mixing calling conventions here is very clearly an error.
1556 if (!CallerHasFeat || !CalleeHasFeat)
1557 return Diag.Report(Loc: CallLoc, DiagID: diag::err_avx_calling_convention)
1558 << IsArgument << Ty << Feature;
1559
1560 // Else, both caller and callee have the required feature, so there is no need
1561 // to diagnose.
1562 return false;
1563}
1564
1565static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,
1566 SourceLocation CallLoc, const FunctionDecl &Callee,
1567 const llvm::StringMap<bool> &CallerMap,
1568 const llvm::StringMap<bool> &CalleeMap, QualType Ty,
1569 bool IsArgument) {
1570 uint64_t Size = Ctx.getTypeSize(T: Ty);
1571 if (Size > 256)
1572 return checkAVXParamFeature(Diag, CallLoc, Callee, CallerMap, CalleeMap, Ty,
1573 Feature: "avx512f", IsArgument);
1574
1575 if (Size > 128)
1576 return checkAVXParamFeature(Diag, CallLoc, Callee, CallerMap, CalleeMap, Ty,
1577 Feature: "avx", IsArgument);
1578
1579 return false;
1580}
1581
1582void X86_64TargetCodeGenInfo::checkFunctionABI(CodeGenModule &CGM,
1583 const FunctionDecl *FD) const {
1584 auto GetReturnTypeLoc = [](const FunctionDecl *FD) {
1585 if (const TypeSourceInfo *TSI = FD->getTypeSourceInfo()) {
1586 TypeLoc TL = TSI->getTypeLoc();
1587
1588 if (auto FTL = TL.IgnoreParens().getAs<FunctionTypeLoc>()) {
1589 SourceLocation Loc = FTL.getReturnLoc().getBeginLoc();
1590 if (Loc.isValid())
1591 return Loc;
1592 }
1593 }
1594
1595 SourceLocation Loc = FD->getLocation();
1596 if (Loc.isValid())
1597 return Loc;
1598
1599 return FD->getBeginLoc();
1600 };
1601
1602 auto Check = [&](QualType Ty, SourceLocation Loc, bool IsReturn) {
1603 if (!Ty->isVectorType())
1604 return false;
1605 if (CGM.getContext().getTypeSize(T: Ty) <= 128)
1606 return false;
1607
1608 StringRef Feature =
1609 CGM.getContext().getTypeSize(T: Ty) > 256 ? "avx512f" : "avx";
1610
1611 llvm::StringMap<bool> FeatureMap;
1612 CGM.getContext().getFunctionFeatureMap(FeatureMap, FD);
1613 if (!FeatureMap.lookup(Key: Feature)) {
1614 CGM.getDiags().Report(Loc, DiagID: diag::warn_avx_calling_convention)
1615 << !IsReturn << Ty << Feature;
1616 return true;
1617 }
1618
1619 return false;
1620 };
1621
1622 // psABI warnings & errors for function definitions that are only visible
1623 // in this translation unit are handled at call site by checkFunctionCallABI.
1624 if (!FD->isExternallyVisible())
1625 return;
1626
1627 // First check the return type and emit diagnostic if required.
1628 Check(FD->getReturnType(), GetReturnTypeLoc(FD), true);
1629
1630 // Go through the parameters and emit a warning for the first vector found
1631 // without the matching function AVX level attribute.
1632 for (const ParmVarDecl *P : FD->parameters()) {
1633 SourceLocation Loc = P->getLocation();
1634 if (Loc.isInvalid())
1635 Loc = P->getBeginLoc();
1636 if (Check(P->getType(), Loc, false))
1637 return;
1638 }
1639}
1640
1641void X86_64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM,
1642 SourceLocation CallLoc,
1643 const FunctionDecl *Caller,
1644 const FunctionDecl *Callee,
1645 const CallArgList &Args,
1646 QualType ReturnType) const {
1647 if (!Callee)
1648 return;
1649
1650 llvm::StringMap<bool> CallerMap;
1651 llvm::StringMap<bool> CalleeMap;
1652 unsigned ArgIndex = 0;
1653
1654 // We need to loop through the actual call arguments rather than the
1655 // function's parameters, in case this variadic.
1656 for (const CallArg &Arg : Args) {
1657 // The "avx" feature changes how vectors >128 in size are passed. "avx512f"
1658 // additionally changes how vectors >256 in size are passed. Like GCC, we
1659 // warn when a function is called with an argument where this will change.
1660 // Unlike GCC, we also error when it is an obvious ABI mismatch, that is,
1661 // the caller and callee features are mismatched.
1662 // Unfortunately, we cannot do this diagnostic in SEMA, since the callee can
1663 // change its ABI with attribute-target after this call.
1664 if (Arg.getType()->isVectorType() &&
1665 CGM.getContext().getTypeSize(T: Arg.getType()) > 128) {
1666 initFeatureMaps(Ctx: CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
1667 QualType Ty = Arg.getType();
1668 // The CallArg seems to have desugared the type already, so for clearer
1669 // diagnostics, replace it with the type in the FunctionDecl if possible.
1670 if (ArgIndex < Callee->getNumParams())
1671 Ty = Callee->getParamDecl(i: ArgIndex)->getType();
1672
1673 if (checkAVXParam(Diag&: CGM.getDiags(), Ctx&: CGM.getContext(), CallLoc, Callee: *Callee,
1674 CallerMap, CalleeMap, Ty, /*IsArgument*/ true))
1675 return;
1676 }
1677 ++ArgIndex;
1678 }
1679
1680 // Check return always, as we don't have a good way of knowing in codegen
1681 // whether this value is used, tail-called, etc.
1682 if (Callee->getReturnType()->isVectorType() &&
1683 CGM.getContext().getTypeSize(T: Callee->getReturnType()) > 128) {
1684 initFeatureMaps(Ctx: CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
1685 checkAVXParam(Diag&: CGM.getDiags(), Ctx&: CGM.getContext(), CallLoc, Callee: *Callee, CallerMap,
1686 CalleeMap, Ty: Callee->getReturnType(),
1687 /*IsArgument*/ false);
1688 }
1689}
1690
1691std::string TargetCodeGenInfo::qualifyWindowsLibrary(StringRef Lib) {
1692 // If the argument does not end in .lib, automatically add the suffix.
1693 // If the argument contains a space, enclose it in quotes.
1694 // This matches the behavior of MSVC.
1695 bool Quote = Lib.contains(C: ' ');
1696 std::string ArgStr = Quote ? "\"" : "";
1697 ArgStr += Lib;
1698 if (!Lib.ends_with_insensitive(Suffix: ".lib") && !Lib.ends_with_insensitive(Suffix: ".a"))
1699 ArgStr += ".lib";
1700 ArgStr += Quote ? "\"" : "";
1701 return ArgStr;
1702}
1703
1704namespace {
1705class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo {
1706public:
1707 WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
1708 bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI,
1709 unsigned NumRegisterParameters)
1710 : X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI,
1711 Win32StructABI, NumRegisterParameters, false) {}
1712
1713 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1714 CodeGen::CodeGenModule &CGM) const override;
1715
1716 void getDependentLibraryOption(llvm::StringRef Lib,
1717 llvm::SmallString<24> &Opt) const override {
1718 Opt = "/DEFAULTLIB:";
1719 Opt += qualifyWindowsLibrary(Lib);
1720 }
1721
1722 void getDetectMismatchOption(llvm::StringRef Name,
1723 llvm::StringRef Value,
1724 llvm::SmallString<32> &Opt) const override {
1725 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
1726 }
1727};
1728} // namespace
1729
1730void WinX86_32TargetCodeGenInfo::setTargetAttributes(
1731 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
1732 X86_32TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
1733 if (GV->isDeclaration())
1734 return;
1735 addStackProbeTargetAttributes(D, GV, CGM);
1736}
1737
1738namespace {
1739class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
1740public:
1741 WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
1742 X86AVXABILevel AVXLevel)
1743 : TargetCodeGenInfo(std::make_unique<WinX86_64ABIInfo>(args&: CGT, args&: AVXLevel)) {
1744 SwiftInfo =
1745 std::make_unique<SwiftABIInfo>(args&: CGT, /*SwiftErrorInRegister=*/args: true);
1746 }
1747
1748 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1749 CodeGen::CodeGenModule &CGM) const override;
1750
1751 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1752 return 7;
1753 }
1754
1755 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1756 llvm::Value *Address) const override {
1757 llvm::Value *Eight8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 8);
1758
1759 // 0-15 are the 16 integer registers.
1760 // 16 is %rip.
1761 AssignToArrayRange(Builder&: CGF.Builder, Array: Address, Value: Eight8, FirstIndex: 0, LastIndex: 16);
1762 return false;
1763 }
1764
1765 void getDependentLibraryOption(llvm::StringRef Lib,
1766 llvm::SmallString<24> &Opt) const override {
1767 Opt = "/DEFAULTLIB:";
1768 Opt += qualifyWindowsLibrary(Lib);
1769 }
1770
1771 void getDetectMismatchOption(llvm::StringRef Name,
1772 llvm::StringRef Value,
1773 llvm::SmallString<32> &Opt) const override {
1774 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
1775 }
1776};
1777} // namespace
1778
1779void WinX86_64TargetCodeGenInfo::setTargetAttributes(
1780 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
1781 TargetCodeGenInfo::setTargetAttributes(D, GV, M&: CGM);
1782 if (GV->isDeclaration())
1783 return;
1784 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D)) {
1785 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
1786 llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1787 Fn->addFnAttr(Kind: "stackrealign");
1788 }
1789
1790 addX86InterruptAttrs(FD, GV, CGM);
1791 }
1792
1793 addStackProbeTargetAttributes(D, GV, CGM);
1794}
1795
1796void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo,
1797 Class &Hi) const {
1798 // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
1799 //
1800 // (a) If one of the classes is Memory, the whole argument is passed in
1801 // memory.
1802 //
1803 // (b) If X87UP is not preceded by X87, the whole argument is passed in
1804 // memory.
1805 //
1806 // (c) If the size of the aggregate exceeds two eightbytes and the first
1807 // eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
1808 // argument is passed in memory. NOTE: This is necessary to keep the
1809 // ABI working for processors that don't support the __m256 type.
1810 //
1811 // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
1812 //
1813 // Some of these are enforced by the merging logic. Others can arise
1814 // only with unions; for example:
1815 // union { _Complex double; unsigned; }
1816 //
1817 // Note that clauses (b) and (c) were added in 0.98.
1818 //
1819 if (Hi == Memory)
1820 Lo = Memory;
1821 if (Hi == X87Up && Lo != X87 && honorsRevision0_98())
1822 Lo = Memory;
1823 if (AggregateSize > 128 && (Lo != SSE || Hi != SSEUp))
1824 Lo = Memory;
1825 if (Hi == SSEUp && Lo != SSE)
1826 Hi = SSE;
1827}
1828
1829static X86AVXABILevel getEffectiveX86AVXABILevel(CodeGenTypes &CGT,
1830 X86AVXABILevel GlobalAVXLevel,
1831 const FunctionDecl *FD) {
1832 // Always return global AVX level on PlayStation.
1833 if (CGT.getTarget().getTriple().isPS() ||
1834 CGT.getContext().getLangOpts().getClangABICompat() <=
1835 LangOptions::ClangABI::Ver23) {
1836 return GlobalAVXLevel;
1837 }
1838
1839 X86AVXABILevel Level = GlobalAVXLevel;
1840 // TargetVersionAttr does not apply to x86.
1841 // FIXME: Handling TargetClonesAttr and CPUSpecificAttr is intentionally
1842 // deferred to a follow-up.
1843 if (!FD || !FD->hasAttr<TargetAttr>())
1844 return Level;
1845
1846 llvm::StringMap<bool> FeatureMap;
1847 CGT.getCGM().getContext().getFunctionFeatureMap(FeatureMap, FD);
1848 if (FeatureMap.lookup(Key: "avx512f"))
1849 return std::max(a: Level, b: X86AVXABILevel::AVX512);
1850 if (FeatureMap.lookup(Key: "avx"))
1851 return std::max(a: Level, b: X86AVXABILevel::AVX);
1852 return Level;
1853}
1854
1855X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
1856 // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
1857 // classified recursively so that always two fields are
1858 // considered. The resulting class is calculated according to
1859 // the classes of the fields in the eightbyte:
1860 //
1861 // (a) If both classes are equal, this is the resulting class.
1862 //
1863 // (b) If one of the classes is NO_CLASS, the resulting class is
1864 // the other class.
1865 //
1866 // (c) If one of the classes is MEMORY, the result is the MEMORY
1867 // class.
1868 //
1869 // (d) If one of the classes is INTEGER, the result is the
1870 // INTEGER.
1871 //
1872 // (e) If one of the classes is X87, X87UP, COMPLEX_X87 class,
1873 // MEMORY is used as class.
1874 //
1875 // (f) Otherwise class SSE is used.
1876
1877 // Accum should never be memory (we should have returned) or
1878 // ComplexX87 (because this cannot be passed in a structure).
1879 assert((Accum != Memory && Accum != ComplexX87) &&
1880 "Invalid accumulated classification during merge.");
1881 if (Accum == Field || Field == NoClass)
1882 return Accum;
1883 if (Field == Memory)
1884 return Memory;
1885 if (Accum == NoClass)
1886 return Field;
1887 if (Accum == Integer || Field == Integer)
1888 return Integer;
1889 if (Field == X87 || Field == X87Up || Field == ComplexX87 ||
1890 Accum == X87 || Accum == X87Up)
1891 return Memory;
1892 return SSE;
1893}
1894
1895void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
1896 Class &Hi, bool isNamedArg, bool IsRegCall) const {
1897 // FIXME: This code can be simplified by introducing a simple value class for
1898 // Class pairs with appropriate constructor methods for the various
1899 // situations.
1900
1901 // FIXME: Some of the split computations are wrong; unaligned vectors
1902 // shouldn't be passed in registers for example, so there is no chance they
1903 // can straddle an eightbyte. Verify & simplify.
1904
1905 Lo = Hi = NoClass;
1906
1907 Class &Current = OffsetBase < 64 ? Lo : Hi;
1908 Current = Memory;
1909
1910 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
1911 BuiltinType::Kind k = BT->getKind();
1912
1913 if (k == BuiltinType::Void) {
1914 Current = NoClass;
1915 } else if (k == BuiltinType::Int128 || k == BuiltinType::UInt128) {
1916 Lo = Integer;
1917 Hi = Integer;
1918 } else if (k >= BuiltinType::Bool && k <= BuiltinType::LongLong) {
1919 Current = Integer;
1920 } else if (k == BuiltinType::Float || k == BuiltinType::Double ||
1921 k == BuiltinType::Float16 || k == BuiltinType::BFloat16) {
1922 Current = SSE;
1923 } else if (k == BuiltinType::Float128) {
1924 Lo = SSE;
1925 Hi = SSEUp;
1926 } else if (k == BuiltinType::LongDouble) {
1927 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
1928 if (LDF == &llvm::APFloat::IEEEquad()) {
1929 Lo = SSE;
1930 Hi = SSEUp;
1931 } else if (LDF == &llvm::APFloat::x87DoubleExtended()) {
1932 Lo = X87;
1933 Hi = X87Up;
1934 } else if (LDF == &llvm::APFloat::IEEEdouble()) {
1935 Current = SSE;
1936 } else
1937 llvm_unreachable("unexpected long double representation!");
1938 }
1939 // FIXME: _Decimal32 and _Decimal64 are SSE.
1940 // FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
1941 return;
1942 }
1943
1944 if (const auto *ED = Ty->getAsEnumDecl()) {
1945 // Classify the underlying integer type.
1946 classify(Ty: ED->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);
1947 return;
1948 }
1949
1950 if (Ty->hasPointerRepresentation()) {
1951 Current = Integer;
1952 return;
1953 }
1954
1955 if (Ty->isMemberPointerType()) {
1956 if (Ty->isMemberFunctionPointerType()) {
1957 if (Has64BitPointers) {
1958 // If Has64BitPointers, this is an {i64, i64}, so classify both
1959 // Lo and Hi now.
1960 Lo = Hi = Integer;
1961 } else {
1962 // Otherwise, with 32-bit pointers, this is an {i32, i32}. If that
1963 // straddles an eightbyte boundary, Hi should be classified as well.
1964 uint64_t EB_FuncPtr = (OffsetBase) / 64;
1965 uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64;
1966 if (EB_FuncPtr != EB_ThisAdj) {
1967 Lo = Hi = Integer;
1968 } else {
1969 Current = Integer;
1970 }
1971 }
1972 } else {
1973 Current = Integer;
1974 }
1975 return;
1976 }
1977
1978 if (const VectorType *VT = Ty->getAs<VectorType>()) {
1979 uint64_t Size = getContext().getTypeSize(T: VT);
1980 if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
1981 // gcc passes the following as integer:
1982 // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
1983 // 2 bytes - <2 x char>, <1 x short>
1984 // 1 byte - <1 x char>
1985 Current = Integer;
1986
1987 // If this type crosses an eightbyte boundary, it should be
1988 // split.
1989 uint64_t EB_Lo = (OffsetBase) / 64;
1990 uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
1991 if (EB_Lo != EB_Hi)
1992 Hi = Lo;
1993 } else if (Size == 64) {
1994 QualType ElementType = VT->getElementType();
1995
1996 // gcc passes <1 x double> in memory. :(
1997 if (ElementType->isSpecificBuiltinType(K: BuiltinType::Double))
1998 return;
1999
2000 // gcc passes <1 x long long> as SSE but clang used to unconditionally
2001 // pass them as integer. For platforms where clang is the de facto
2002 // platform compiler, we must continue to use integer.
2003 if (!classifyIntegerMMXAsSSE() &&
2004 (ElementType->isSpecificBuiltinType(K: BuiltinType::LongLong) ||
2005 ElementType->isSpecificBuiltinType(K: BuiltinType::ULongLong) ||
2006 ElementType->isSpecificBuiltinType(K: BuiltinType::Long) ||
2007 ElementType->isSpecificBuiltinType(K: BuiltinType::ULong)))
2008 Current = Integer;
2009 else
2010 Current = SSE;
2011
2012 // If this type crosses an eightbyte boundary, it should be
2013 // split.
2014 if (OffsetBase && OffsetBase != 64)
2015 Hi = Lo;
2016 } else if (Size == 128 ||
2017 (isNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
2018 QualType ElementType = VT->getElementType();
2019
2020 // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
2021 if (passInt128VectorsInMem() && Size != 128 &&
2022 (ElementType->isSpecificBuiltinType(K: BuiltinType::Int128) ||
2023 ElementType->isSpecificBuiltinType(K: BuiltinType::UInt128)))
2024 return;
2025
2026 // Arguments of 256-bits are split into four eightbyte chunks. The
2027 // least significant one belongs to class SSE and all the others to class
2028 // SSEUP. The original Lo and Hi design considers that types can't be
2029 // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
2030 // This design isn't correct for 256-bits, but since there're no cases
2031 // where the upper parts would need to be inspected, avoid adding
2032 // complexity and just consider Hi to match the 64-256 part.
2033 //
2034 // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
2035 // registers if they are "named", i.e. not part of the "..." of a
2036 // variadic function.
2037 //
2038 // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
2039 // split into eight eightbyte chunks, one SSE and seven SSEUP.
2040 Lo = SSE;
2041 Hi = SSEUp;
2042 }
2043 return;
2044 }
2045
2046 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
2047 QualType ET = getContext().getCanonicalType(T: CT->getElementType());
2048
2049 uint64_t Size = getContext().getTypeSize(T: Ty);
2050 if (ET->isIntegralOrEnumerationType()) {
2051 if (Size <= 64)
2052 Current = Integer;
2053 else if (Size <= 128)
2054 Lo = Hi = Integer;
2055 } else if (ET->isFloat16Type() || ET == getContext().FloatTy ||
2056 ET->isBFloat16Type()) {
2057 Current = SSE;
2058 } else if (ET == getContext().DoubleTy) {
2059 Lo = Hi = SSE;
2060 } else if (ET == getContext().LongDoubleTy) {
2061 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2062 if (LDF == &llvm::APFloat::IEEEquad())
2063 Current = Memory;
2064 else if (LDF == &llvm::APFloat::x87DoubleExtended())
2065 Current = ComplexX87;
2066 else if (LDF == &llvm::APFloat::IEEEdouble())
2067 Lo = Hi = SSE;
2068 else
2069 llvm_unreachable("unexpected long double representation!");
2070 }
2071
2072 // If this complex type crosses an eightbyte boundary then it
2073 // should be split.
2074 uint64_t EB_Real = (OffsetBase) / 64;
2075 uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(T: ET)) / 64;
2076 if (Hi == NoClass && EB_Real != EB_Imag)
2077 Hi = Lo;
2078
2079 return;
2080 }
2081
2082 if (const auto *EITy = Ty->getAs<BitIntType>()) {
2083 if (EITy->getNumBits() <= 64)
2084 Current = Integer;
2085 else if (EITy->getNumBits() <= 128)
2086 Lo = Hi = Integer;
2087 // Larger values need to get passed in memory.
2088 return;
2089 }
2090
2091 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(T: Ty)) {
2092 // Arrays are treated like structures.
2093
2094 uint64_t Size = getContext().getTypeSize(T: Ty);
2095
2096 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
2097 // than eight eightbytes, ..., it has class MEMORY.
2098 // regcall ABI doesn't have limitation to an object. The only limitation
2099 // is the free registers, which will be checked in computeInfo.
2100 if (!IsRegCall && Size > 512)
2101 return;
2102
2103 // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
2104 // fields, it has class MEMORY.
2105 //
2106 // Only need to check alignment of array base.
2107 if (OffsetBase % getContext().getTypeAlign(T: AT->getElementType()))
2108 return;
2109
2110 // Otherwise implement simplified merge. We could be smarter about
2111 // this, but it isn't worth it and would be harder to verify.
2112 Current = NoClass;
2113 uint64_t EltSize = getContext().getTypeSize(T: AT->getElementType());
2114 uint64_t ArraySize = AT->getZExtSize();
2115
2116 // The only case a 256-bit wide vector could be used is when the array
2117 // contains a single 256-bit element. Since Lo and Hi logic isn't extended
2118 // to work for sizes wider than 128, early check and fallback to memory.
2119 //
2120 if (Size > 128 &&
2121 (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))
2122 return;
2123
2124 for (uint64_t i=0, Offset=OffsetBase; i<ArraySize; ++i, Offset += EltSize) {
2125 Class FieldLo, FieldHi;
2126 classify(Ty: AT->getElementType(), OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, isNamedArg);
2127 Lo = merge(Accum: Lo, Field: FieldLo);
2128 Hi = merge(Accum: Hi, Field: FieldHi);
2129 if (Lo == Memory || Hi == Memory)
2130 break;
2131 }
2132
2133 postMerge(AggregateSize: Size, Lo, Hi);
2134 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
2135 return;
2136 }
2137
2138 if (const RecordType *RT = Ty->getAsCanonical<RecordType>()) {
2139 uint64_t Size = getContext().getTypeSize(T: Ty);
2140
2141 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
2142 // than eight eightbytes, ..., it has class MEMORY.
2143 if (Size > 512)
2144 return;
2145
2146 // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
2147 // copy constructor or a non-trivial destructor, it is passed by invisible
2148 // reference.
2149 if (getRecordArgABI(RT, CXXABI&: getCXXABI()))
2150 return;
2151
2152 const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf();
2153
2154 // Assume variable sized types are passed in memory.
2155 if (RD->hasFlexibleArrayMember())
2156 return;
2157
2158 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: RD);
2159
2160 // Reset Lo class, this will be recomputed.
2161 Current = NoClass;
2162
2163 // If this is a C++ record, classify the bases first.
2164 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
2165 for (const auto &I : CXXRD->bases()) {
2166 assert(!I.isVirtual() && !I.getType()->isDependentType() &&
2167 "Unexpected base class!");
2168 const auto *Base = I.getType()->castAsCXXRecordDecl();
2169 // Classify this field.
2170 //
2171 // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
2172 // single eightbyte, each is classified separately. Each eightbyte gets
2173 // initialized to class NO_CLASS.
2174 Class FieldLo, FieldHi;
2175 uint64_t Offset =
2176 OffsetBase + getContext().toBits(CharSize: Layout.getBaseClassOffset(Base));
2177 classify(Ty: I.getType(), OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, isNamedArg);
2178 Lo = merge(Accum: Lo, Field: FieldLo);
2179 Hi = merge(Accum: Hi, Field: FieldHi);
2180 if (returnCXXRecordGreaterThan128InMem() &&
2181 !isEmptyRecord(Context&: getContext(), T: I.getType(), AllowArrays: true) &&
2182 (Size > 128 && (Size != getContext().getTypeSize(T: I.getType()) ||
2183 Size > getNativeVectorSizeForAVXABI(AVXLevel)))) {
2184 // The only case a 256(or 512)-bit wide vector could be used to return
2185 // is when CXX record contains a single 256(or 512)-bit element.
2186 Lo = Memory;
2187 }
2188 if (Lo == Memory || Hi == Memory) {
2189 postMerge(AggregateSize: Size, Lo, Hi);
2190 return;
2191 }
2192 }
2193 }
2194
2195 // Classify the fields one at a time, merging the results.
2196 unsigned idx = 0;
2197 bool UseClang11Compat = getContext().getLangOpts().isCompatibleWith(
2198 Version: LangOptions::ClangABI::Ver11) ||
2199 getContext().getTargetInfo().getTriple().isPS();
2200 bool IsUnion = RT->isUnionType() && !UseClang11Compat;
2201
2202 for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
2203 i != e; ++i, ++idx) {
2204 uint64_t Offset = OffsetBase + Layout.getFieldOffset(FieldNo: idx);
2205 bool BitField = i->isBitField();
2206
2207 // Ignore padding bit-fields.
2208 if (BitField && i->isUnnamedBitField())
2209 continue;
2210
2211 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than
2212 // eight eightbytes, or it contains unaligned fields, it has class MEMORY.
2213 //
2214 // The only case a 256-bit or a 512-bit wide vector could be used is when
2215 // the struct contains a single 256-bit or 512-bit element. Early check
2216 // and fallback to memory.
2217 //
2218 // FIXME: Extended the Lo and Hi logic properly to work for size wider
2219 // than 128.
2220 if (Size > 128 &&
2221 ((!IsUnion && Size != getContext().getTypeSize(T: i->getType())) ||
2222 Size > getNativeVectorSizeForAVXABI(AVXLevel))) {
2223 Lo = Memory;
2224 postMerge(AggregateSize: Size, Lo, Hi);
2225 return;
2226 }
2227
2228 bool IsInMemory =
2229 Offset % getContext().getTypeAlign(T: i->getType().getCanonicalType());
2230 // Note, skip this test for bit-fields, see below.
2231 if (!BitField && IsInMemory) {
2232 Lo = Memory;
2233 postMerge(AggregateSize: Size, Lo, Hi);
2234 return;
2235 }
2236
2237 // Classify this field.
2238 //
2239 // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
2240 // exceeds a single eightbyte, each is classified
2241 // separately. Each eightbyte gets initialized to class
2242 // NO_CLASS.
2243 Class FieldLo, FieldHi;
2244
2245 // Bit-fields require special handling, they do not force the
2246 // structure to be passed in memory even if unaligned, and
2247 // therefore they can straddle an eightbyte.
2248 if (BitField) {
2249 assert(!i->isUnnamedBitField());
2250 uint64_t Offset = OffsetBase + Layout.getFieldOffset(FieldNo: idx);
2251 uint64_t Size = i->getBitWidthValue();
2252
2253 uint64_t EB_Lo = Offset / 64;
2254 uint64_t EB_Hi = (Offset + Size - 1) / 64;
2255
2256 if (EB_Lo) {
2257 assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
2258 FieldLo = NoClass;
2259 FieldHi = Integer;
2260 } else {
2261 FieldLo = Integer;
2262 FieldHi = EB_Hi ? Integer : NoClass;
2263 }
2264 } else
2265 classify(Ty: i->getType(), OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, isNamedArg);
2266 Lo = merge(Accum: Lo, Field: FieldLo);
2267 Hi = merge(Accum: Hi, Field: FieldHi);
2268 if (Lo == Memory || Hi == Memory)
2269 break;
2270 }
2271
2272 postMerge(AggregateSize: Size, Lo, Hi);
2273 }
2274}
2275
2276ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
2277 // If this is a scalar LLVM value then assume LLVM will pass it in the right
2278 // place naturally.
2279 if (!isAggregateTypeForABI(T: Ty)) {
2280 // Treat an enum type as its underlying type.
2281 if (const auto *ED = Ty->getAsEnumDecl())
2282 Ty = ED->getIntegerType();
2283
2284 if (Ty->isBitIntType())
2285 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace());
2286
2287 llvm::Type *IRTy = CGT.ConvertType(T: Ty);
2288 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty, T: IRTy)
2289 : ABIArgInfo::getDirect(T: IRTy));
2290 }
2291
2292 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace());
2293}
2294
2295bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {
2296 if (const VectorType *VecTy = Ty->getAs<VectorType>()) {
2297 uint64_t Size = getContext().getTypeSize(T: VecTy);
2298 unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
2299 if (Size <= 64 || Size > LargestVector)
2300 return true;
2301 QualType EltTy = VecTy->getElementType();
2302 if (passInt128VectorsInMem() &&
2303 (EltTy->isSpecificBuiltinType(K: BuiltinType::Int128) ||
2304 EltTy->isSpecificBuiltinType(K: BuiltinType::UInt128)))
2305 return true;
2306 }
2307
2308 return false;
2309}
2310
2311ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
2312 unsigned freeIntRegs) const {
2313 // If this is a scalar LLVM value then assume LLVM will pass it in the right
2314 // place naturally.
2315 //
2316 // This assumption is optimistic, as there could be free registers available
2317 // when we need to pass this argument in memory, and LLVM could try to pass
2318 // the argument in the free register. This does not seem to happen currently,
2319 // but this code would be much safer if we could mark the argument with
2320 // 'onstack'. See PR12193.
2321 if (!isAggregateTypeForABI(T: Ty) && !IsIllegalVectorType(Ty) &&
2322 !Ty->isBitIntType()) {
2323 // Treat an enum type as its underlying type.
2324 if (const auto *ED = Ty->getAsEnumDecl())
2325 Ty = ED->getIntegerType();
2326
2327 llvm::Type *IRTy = CGT.ConvertType(T: Ty);
2328 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty, T: IRTy)
2329 : ABIArgInfo::getDirect(T: IRTy));
2330 }
2331
2332 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(T: Ty, CXXABI&: getCXXABI()))
2333 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
2334 ByVal: RAA == CGCXXABI::RAA_DirectInMemory);
2335
2336 // Compute the byval alignment. We specify the alignment of the byval in all
2337 // cases so that the mid-level optimizer knows the alignment of the byval.
2338 unsigned Align = std::max(a: getContext().getTypeAlign(T: Ty) / 8, b: 8U);
2339
2340 // Attempt to avoid passing indirect results using byval when possible. This
2341 // is important for good codegen.
2342 //
2343 // We do this by coercing the value into a scalar type which the backend can
2344 // handle naturally (i.e., without using byval).
2345 //
2346 // For simplicity, we currently only do this when we have exhausted all of the
2347 // free integer registers. Doing this when there are free integer registers
2348 // would require more care, as we would have to ensure that the coerced value
2349 // did not claim the unused register. That would require either reording the
2350 // arguments to the function (so that any subsequent inreg values came first),
2351 // or only doing this optimization when there were no following arguments that
2352 // might be inreg.
2353 //
2354 // We currently expect it to be rare (particularly in well written code) for
2355 // arguments to be passed on the stack when there are still free integer
2356 // registers available (this would typically imply large structs being passed
2357 // by value), so this seems like a fair tradeoff for now.
2358 //
2359 // We can revisit this if the backend grows support for 'onstack' parameter
2360 // attributes. See PR12193.
2361 if (freeIntRegs == 0) {
2362 uint64_t Size = getContext().getTypeSize(T: Ty);
2363
2364 // If this type fits in an eightbyte, coerce it into the matching integral
2365 // type, which will end up on the stack (with alignment 8).
2366 if (Align == 8 && Size <= 64)
2367 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(),
2368 NumBits: Size));
2369 }
2370
2371 return ABIArgInfo::getIndirect(Alignment: CharUnits::fromQuantity(Quantity: Align),
2372 AddrSpace: getDataLayout().getAllocaAddrSpace());
2373}
2374
2375/// The ABI specifies that a value should be passed in a full vector XMM/YMM
2376/// register. Pick an LLVM IR type that will be passed as a vector register.
2377llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {
2378 // Wrapper structs/arrays that only contain vectors are passed just like
2379 // vectors; strip them off if present.
2380 if (const Type *InnerTy = isSingleElementStruct(T: Ty, Context&: getContext()))
2381 Ty = QualType(InnerTy, 0);
2382
2383 llvm::Type *IRType = CGT.ConvertType(T: Ty);
2384 if (isa<llvm::VectorType>(Val: IRType)) {
2385 // Don't pass vXi128 vectors in their native type, the backend can't
2386 // legalize them.
2387 if (passInt128VectorsInMem() &&
2388 cast<llvm::VectorType>(Val: IRType)->getElementType()->isIntegerTy(BitWidth: 128)) {
2389 // Use a vXi64 vector.
2390 uint64_t Size = getContext().getTypeSize(T: Ty);
2391 return llvm::FixedVectorType::get(ElementType: llvm::Type::getInt64Ty(C&: getVMContext()),
2392 NumElts: Size / 64);
2393 }
2394
2395 return IRType;
2396 }
2397
2398 if (IRType->getTypeID() == llvm::Type::FP128TyID)
2399 return IRType;
2400
2401 // We couldn't find the preferred IR vector type for 'Ty'.
2402 uint64_t Size = getContext().getTypeSize(T: Ty);
2403 assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!");
2404
2405
2406 // Return a LLVM IR vector type based on the size of 'Ty'.
2407 return llvm::FixedVectorType::get(ElementType: llvm::Type::getDoubleTy(C&: getVMContext()),
2408 NumElts: Size / 64);
2409}
2410
2411/// BitsContainNoUserData - Return true if the specified [start,end) bit range
2412/// is known to either be off the end of the specified type or being in
2413/// alignment padding. The user type specified is known to be at most 128 bits
2414/// in size, and have passed through X86_64ABIInfo::classify with a successful
2415/// classification that put one of the two halves in the INTEGER class.
2416///
2417/// It is conservatively correct to return false.
2418static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
2419 unsigned EndBit, ASTContext &Context) {
2420 // If the bytes being queried are off the end of the type, there is no user
2421 // data hiding here. This handles analysis of builtins, vectors and other
2422 // types that don't contain interesting padding.
2423 unsigned TySize = (unsigned)Context.getTypeSize(T: Ty);
2424 if (TySize <= StartBit)
2425 return true;
2426
2427 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty)) {
2428 unsigned EltSize = (unsigned)Context.getTypeSize(T: AT->getElementType());
2429 unsigned NumElts = (unsigned)AT->getZExtSize();
2430
2431 // Check each element to see if the element overlaps with the queried range.
2432 for (unsigned i = 0; i != NumElts; ++i) {
2433 // If the element is after the span we care about, then we're done..
2434 unsigned EltOffset = i*EltSize;
2435 if (EltOffset >= EndBit) break;
2436
2437 unsigned EltStart = EltOffset < StartBit ? StartBit-EltOffset :0;
2438 if (!BitsContainNoUserData(Ty: AT->getElementType(), StartBit: EltStart,
2439 EndBit: EndBit-EltOffset, Context))
2440 return false;
2441 }
2442 // If it overlaps no elements, then it is safe to process as padding.
2443 return true;
2444 }
2445
2446 if (const auto *RD = Ty->getAsRecordDecl()) {
2447 const ASTRecordLayout &Layout = Context.getASTRecordLayout(D: RD);
2448
2449 // If this is a C++ record, check the bases first.
2450 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
2451 for (const auto &I : CXXRD->bases()) {
2452 assert(!I.isVirtual() && !I.getType()->isDependentType() &&
2453 "Unexpected base class!");
2454 const auto *Base = I.getType()->castAsCXXRecordDecl();
2455
2456 // If the base is after the span we care about, ignore it.
2457 unsigned BaseOffset = Context.toBits(CharSize: Layout.getBaseClassOffset(Base));
2458 if (BaseOffset >= EndBit) continue;
2459
2460 unsigned BaseStart = BaseOffset < StartBit ? StartBit-BaseOffset :0;
2461 if (!BitsContainNoUserData(Ty: I.getType(), StartBit: BaseStart,
2462 EndBit: EndBit-BaseOffset, Context))
2463 return false;
2464 }
2465 }
2466
2467 // Verify that no field has data that overlaps the region of interest. Yes
2468 // this could be sped up a lot by being smarter about queried fields,
2469 // however we're only looking at structs up to 16 bytes, so we don't care
2470 // much.
2471 unsigned idx = 0;
2472 for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
2473 i != e; ++i, ++idx) {
2474 unsigned FieldOffset = (unsigned)Layout.getFieldOffset(FieldNo: idx);
2475
2476 // If we found a field after the region we care about, then we're done.
2477 if (FieldOffset >= EndBit) break;
2478
2479 unsigned FieldStart = FieldOffset < StartBit ? StartBit-FieldOffset :0;
2480 if (!BitsContainNoUserData(Ty: i->getType(), StartBit: FieldStart, EndBit: EndBit-FieldOffset,
2481 Context))
2482 return false;
2483 }
2484
2485 // If nothing in this record overlapped the area of interest, then we're
2486 // clean.
2487 return true;
2488 }
2489
2490 return false;
2491}
2492
2493/// getFPTypeAtOffset - Return a floating point type at the specified offset.
2494static llvm::Type *getFPTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
2495 const llvm::DataLayout &TD) {
2496 if (IROffset == 0 && IRType->isFloatingPointTy())
2497 return IRType;
2498
2499 // If this is a struct, recurse into the field at the specified offset.
2500 if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: IRType)) {
2501 if (!STy->getNumContainedTypes())
2502 return nullptr;
2503
2504 const llvm::StructLayout *SL = TD.getStructLayout(Ty: STy);
2505 unsigned Elt = SL->getElementContainingOffset(FixedOffset: IROffset);
2506 IROffset -= SL->getElementOffset(Idx: Elt);
2507 return getFPTypeAtOffset(IRType: STy->getElementType(N: Elt), IROffset, TD);
2508 }
2509
2510 // If this is an array, recurse into the field at the specified offset.
2511 if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(Val: IRType)) {
2512 llvm::Type *EltTy = ATy->getElementType();
2513 unsigned EltSize = TD.getTypeAllocSize(Ty: EltTy);
2514 IROffset -= IROffset / EltSize * EltSize;
2515 return getFPTypeAtOffset(IRType: EltTy, IROffset, TD);
2516 }
2517
2518 return nullptr;
2519}
2520
2521/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
2522/// low 8 bytes of an XMM register, corresponding to the SSE class.
2523llvm::Type *X86_64ABIInfo::
2524GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
2525 QualType SourceTy, unsigned SourceOffset) const {
2526 const llvm::DataLayout &TD = getDataLayout();
2527 unsigned SourceSize =
2528 (unsigned)getContext().getTypeSize(T: SourceTy) / 8 - SourceOffset;
2529 llvm::Type *T0 = getFPTypeAtOffset(IRType, IROffset, TD);
2530 if (!T0 || T0->isDoubleTy())
2531 return llvm::Type::getDoubleTy(C&: getVMContext());
2532
2533 // Get the adjacent FP type.
2534 llvm::Type *T1 = nullptr;
2535 unsigned T0Size = TD.getTypeAllocSize(Ty: T0);
2536 if (SourceSize > T0Size)
2537 T1 = getFPTypeAtOffset(IRType, IROffset: IROffset + T0Size, TD);
2538 if (T1 == nullptr) {
2539 // Check if IRType is a half/bfloat + float. float type will be in IROffset+4 due
2540 // to its alignment.
2541 if (T0->is16bitFPTy() && SourceSize > 4)
2542 T1 = getFPTypeAtOffset(IRType, IROffset: IROffset + 4, TD);
2543 // If we can't get a second FP type, return a simple half or float.
2544 // avx512fp16-abi.c:pr51813_2 shows it works to return float for
2545 // {float, i8} too.
2546 if (T1 == nullptr)
2547 return T0;
2548 }
2549
2550 if (T0->isFloatTy() && T1->isFloatTy())
2551 return llvm::FixedVectorType::get(ElementType: T0, NumElts: 2);
2552
2553 if (T0->is16bitFPTy() && T1->is16bitFPTy()) {
2554 llvm::Type *T2 = nullptr;
2555 if (SourceSize > 4)
2556 T2 = getFPTypeAtOffset(IRType, IROffset: IROffset + 4, TD);
2557 if (T2 == nullptr)
2558 return llvm::FixedVectorType::get(ElementType: T0, NumElts: 2);
2559 return llvm::FixedVectorType::get(ElementType: T0, NumElts: 4);
2560 }
2561
2562 if (T0->is16bitFPTy() || T1->is16bitFPTy())
2563 return llvm::FixedVectorType::get(ElementType: llvm::Type::getHalfTy(C&: getVMContext()), NumElts: 4);
2564
2565 return llvm::Type::getDoubleTy(C&: getVMContext());
2566}
2567
2568/// GetINTEGERTypeAtOffset - The ABI specifies that a value should be passed in
2569/// one or more 8-byte GPRs. This means that we either have a scalar or we are
2570/// talking about the high and/or low part of an up-to-16-byte struct. This
2571/// routine picks the best LLVM IR type to represent this, which may be i64 or
2572/// may be anything else that the backend will pass in GPRs that works better
2573/// (e.g. i8, %foo*, etc).
2574///
2575/// PrefType is an LLVM IR type that corresponds to (part of) the IR type for
2576/// the source type. IROffset is an offset in bytes into the LLVM IR type that
2577/// the 8-byte value references. PrefType may be null.
2578///
2579/// SourceTy is the source-level type for the entire argument. SourceOffset is
2580/// an offset into this that we're processing (which is always either 0 or 8).
2581///
2582llvm::Type *X86_64ABIInfo::
2583GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
2584 QualType SourceTy, unsigned SourceOffset) const {
2585 // If we're dealing with an un-offset LLVM IR type, then it means that we're
2586 // returning an 8-byte unit starting with it. See if we can safely use it.
2587 if (IROffset == 0) {
2588 // Pointers and int64's always fill the 8-byte unit.
2589 if ((isa<llvm::PointerType>(Val: IRType) && Has64BitPointers) ||
2590 IRType->isIntegerTy(BitWidth: 64))
2591 return IRType;
2592
2593 // If we have a 1/2/4-byte integer, we can use it only if the rest of the
2594 // goodness in the source type is just tail padding. This is allowed to
2595 // kick in for struct {double,int} on the int, but not on
2596 // struct{double,int,int} because we wouldn't return the second int. We
2597 // have to do this analysis on the source type because we can't depend on
2598 // unions being lowered a specific way etc.
2599 if (IRType->isIntegerTy(BitWidth: 8) || IRType->isIntegerTy(BitWidth: 16) ||
2600 IRType->isIntegerTy(BitWidth: 32) ||
2601 (isa<llvm::PointerType>(Val: IRType) && !Has64BitPointers)) {
2602 unsigned BitWidth = isa<llvm::PointerType>(Val: IRType) ? 32 :
2603 cast<llvm::IntegerType>(Val: IRType)->getBitWidth();
2604
2605 if (BitsContainNoUserData(Ty: SourceTy, StartBit: SourceOffset*8+BitWidth,
2606 EndBit: SourceOffset*8+64, Context&: getContext()))
2607 return IRType;
2608 }
2609 }
2610
2611 if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: IRType)) {
2612 // If this is a struct, recurse into the field at the specified offset.
2613 const llvm::StructLayout *SL = getDataLayout().getStructLayout(Ty: STy);
2614 if (IROffset < SL->getSizeInBytes()) {
2615 unsigned FieldIdx = SL->getElementContainingOffset(FixedOffset: IROffset);
2616 IROffset -= SL->getElementOffset(Idx: FieldIdx);
2617
2618 return GetINTEGERTypeAtOffset(IRType: STy->getElementType(N: FieldIdx), IROffset,
2619 SourceTy, SourceOffset);
2620 }
2621 }
2622
2623 if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(Val: IRType)) {
2624 llvm::Type *EltTy = ATy->getElementType();
2625 unsigned EltSize = getDataLayout().getTypeAllocSize(Ty: EltTy);
2626 unsigned EltOffset = IROffset/EltSize*EltSize;
2627 return GetINTEGERTypeAtOffset(IRType: EltTy, IROffset: IROffset-EltOffset, SourceTy,
2628 SourceOffset);
2629 }
2630
2631 // if we have a 128-bit integer, we can pass it safely using an i128
2632 // so we return that
2633 if (IRType->isIntegerTy(BitWidth: 128)) {
2634 assert(IROffset == 0);
2635 return IRType;
2636 }
2637
2638 // Okay, we don't have any better idea of what to pass, so we pass this in an
2639 // integer register that isn't too big to fit the rest of the struct.
2640 unsigned TySizeInBytes =
2641 (unsigned)getContext().getTypeSizeInChars(T: SourceTy).getQuantity();
2642
2643 assert(TySizeInBytes != SourceOffset && "Empty field?");
2644
2645 // It is always safe to classify this as an integer type up to i64 that
2646 // isn't larger than the structure.
2647 return llvm::IntegerType::get(C&: getVMContext(),
2648 NumBits: std::min(a: TySizeInBytes-SourceOffset, b: 8U)*8);
2649}
2650
2651
2652/// GetX86_64ByValArgumentPair - Given a high and low type that can ideally
2653/// be used as elements of a two register pair to pass or return, return a
2654/// first class aggregate to represent them. For example, if the low part of
2655/// a by-value argument should be passed as i32* and the high part as float,
2656/// return {i32*, float}.
2657static llvm::Type *
2658GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,
2659 const llvm::DataLayout &TD) {
2660 // In order to correctly satisfy the ABI, we need to the high part to start
2661 // at offset 8. If the high and low parts we inferred are both 4-byte types
2662 // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
2663 // the second element at offset 8. Check for this:
2664 unsigned LoSize = (unsigned)TD.getTypeAllocSize(Ty: Lo);
2665 llvm::Align HiAlign = TD.getABITypeAlign(Ty: Hi);
2666 unsigned HiStart = llvm::alignTo(Size: LoSize, A: HiAlign);
2667 assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
2668
2669 // To handle this, we have to increase the size of the low part so that the
2670 // second element will start at an 8 byte offset. We can't increase the size
2671 // of the second element because it might make us access off the end of the
2672 // struct.
2673 if (HiStart != 8) {
2674 // There are usually two sorts of types the ABI generation code can produce
2675 // for the low part of a pair that aren't 8 bytes in size: half, float or
2676 // i8/i16/i32. This can also include pointers when they are 32-bit (X32).
2677 // Promote these to a larger type.
2678 if (Lo->isHalfTy() || Lo->isFloatTy())
2679 Lo = llvm::Type::getDoubleTy(C&: Lo->getContext());
2680 else {
2681 assert((Lo->isIntegerTy() || Lo->isPointerTy())
2682 && "Invalid/unknown lo type");
2683 Lo = llvm::Type::getInt64Ty(C&: Lo->getContext());
2684 }
2685 }
2686
2687 llvm::StructType *Result = llvm::StructType::get(elt1: Lo, elts: Hi);
2688
2689 // Verify that the second element is at an 8-byte offset.
2690 assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&
2691 "Invalid x86-64 argument pair!");
2692 return Result;
2693}
2694
2695ABIArgInfo X86_64ABIInfo::classifyReturnType(QualType RetTy) const {
2696 // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
2697 // classification algorithm.
2698 X86_64ABIInfo::Class Lo, Hi;
2699 classify(Ty: RetTy, OffsetBase: 0, Lo, Hi, /*isNamedArg*/ true);
2700
2701 // Check some invariants.
2702 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
2703 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
2704
2705 llvm::Type *ResType = nullptr;
2706 switch (Lo) {
2707 case NoClass:
2708 if (Hi == NoClass)
2709 return ABIArgInfo::getIgnore();
2710 // If the low part is just padding, it takes no register, leave ResType
2711 // null.
2712 assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
2713 "Unknown missing lo part");
2714 break;
2715
2716 case SSEUp:
2717 case X87Up:
2718 llvm_unreachable("Invalid classification for lo word.");
2719
2720 // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
2721 // hidden argument.
2722 case Memory:
2723 return getIndirectReturnResult(Ty: RetTy);
2724
2725 // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
2726 // available register of the sequence %rax, %rdx is used.
2727 case Integer:
2728 ResType = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 0, SourceTy: RetTy, SourceOffset: 0);
2729
2730 // If we have a sign or zero extended integer, make sure to return Extend
2731 // so that the parameter gets the right LLVM IR attributes.
2732 if (Hi == NoClass && isa<llvm::IntegerType>(Val: ResType)) {
2733 // Treat an enum type as its underlying type.
2734 if (const auto *ED = RetTy->getAsEnumDecl())
2735 RetTy = ED->getIntegerType();
2736
2737 if (RetTy->isIntegralOrEnumerationType() &&
2738 isPromotableIntegerTypeForABI(Ty: RetTy))
2739 return ABIArgInfo::getExtend(Ty: RetTy);
2740 }
2741
2742 if (ResType->isIntegerTy(BitWidth: 128)) {
2743 // i128 are passed directly
2744 assert(Hi == Integer);
2745 return ABIArgInfo::getDirect(T: ResType);
2746 }
2747 break;
2748
2749 // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
2750 // available SSE register of the sequence %xmm0, %xmm1 is used.
2751 case SSE:
2752 ResType = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 0, SourceTy: RetTy, SourceOffset: 0);
2753 break;
2754
2755 // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
2756 // returned on the X87 stack in %st0 as 80-bit x87 number.
2757 case X87:
2758 ResType = llvm::Type::getX86_FP80Ty(C&: getVMContext());
2759 break;
2760
2761 // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
2762 // part of the value is returned in %st0 and the imaginary part in
2763 // %st1.
2764 case ComplexX87:
2765 assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
2766 ResType = llvm::StructType::get(elt1: llvm::Type::getX86_FP80Ty(C&: getVMContext()),
2767 elts: llvm::Type::getX86_FP80Ty(C&: getVMContext()));
2768 break;
2769 }
2770
2771 llvm::Type *HighPart = nullptr;
2772 switch (Hi) {
2773 // Memory was handled previously and X87 should
2774 // never occur as a hi class.
2775 case Memory:
2776 case X87:
2777 llvm_unreachable("Invalid classification for hi word.");
2778
2779 case ComplexX87: // Previously handled.
2780 case NoClass:
2781 break;
2782
2783 case Integer:
2784 HighPart = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
2785 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
2786 return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2787 break;
2788 case SSE:
2789 HighPart = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
2790 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
2791 return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2792 break;
2793
2794 // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
2795 // is passed in the next available eightbyte chunk if the last used
2796 // vector register.
2797 //
2798 // SSEUP should always be preceded by SSE, just widen.
2799 case SSEUp:
2800 assert(Lo == SSE && "Unexpected SSEUp classification.");
2801 ResType = GetByteVectorType(Ty: RetTy);
2802 break;
2803
2804 // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is
2805 // returned together with the previous X87 value in %st0.
2806 case X87Up:
2807 // If X87Up is preceded by X87, we don't need to do
2808 // anything. However, in some cases with unions it may not be
2809 // preceded by X87. In such situations we follow gcc and pass the
2810 // extra bits in an SSE reg.
2811 if (Lo != X87) {
2812 HighPart = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
2813 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
2814 return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2815 }
2816 break;
2817 }
2818
2819 // If a high part was specified, merge it together with the low part. It is
2820 // known to pass in the high eightbyte of the result. We do this by forming a
2821 // first class struct aggregate with the high and low part: {low, high}
2822 if (HighPart)
2823 ResType = GetX86_64ByValArgumentPair(Lo: ResType, Hi: HighPart, TD: getDataLayout());
2824
2825 return ABIArgInfo::getDirect(T: ResType);
2826}
2827
2828ABIArgInfo
2829X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned freeIntRegs,
2830 unsigned &neededInt, unsigned &neededSSE,
2831 bool isNamedArg, bool IsRegCall) const {
2832 Ty = useFirstFieldIfTransparentUnion(Ty);
2833
2834 X86_64ABIInfo::Class Lo, Hi;
2835 classify(Ty, OffsetBase: 0, Lo, Hi, isNamedArg, IsRegCall);
2836
2837 // Check some invariants.
2838 // FIXME: Enforce these by construction.
2839 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
2840 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
2841
2842 neededInt = 0;
2843 neededSSE = 0;
2844 llvm::Type *ResType = nullptr;
2845 switch (Lo) {
2846 case NoClass:
2847 if (Hi == NoClass)
2848 return ABIArgInfo::getIgnore();
2849 // If the low part is just padding, it takes no register, leave ResType
2850 // null.
2851 assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
2852 "Unknown missing lo part");
2853 break;
2854
2855 // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
2856 // on the stack.
2857 case Memory:
2858
2859 // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87UP or
2860 // COMPLEX_X87, it is passed in memory.
2861 case X87:
2862 case ComplexX87:
2863 if (getRecordArgABI(T: Ty, CXXABI&: getCXXABI()) == CGCXXABI::RAA_Indirect)
2864 ++neededInt;
2865 return getIndirectResult(Ty, freeIntRegs);
2866
2867 case SSEUp:
2868 case X87Up:
2869 llvm_unreachable("Invalid classification for lo word.");
2870
2871 // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
2872 // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
2873 // and %r9 is used.
2874 case Integer:
2875 ++neededInt;
2876
2877 // Pick an 8-byte type based on the preferred type.
2878 ResType = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: Ty), IROffset: 0, SourceTy: Ty, SourceOffset: 0);
2879
2880 // If we have a sign or zero extended integer, make sure to return Extend
2881 // so that the parameter gets the right LLVM IR attributes.
2882 if (Hi == NoClass && isa<llvm::IntegerType>(Val: ResType)) {
2883 // Treat an enum type as its underlying type.
2884 if (const auto *ED = Ty->getAsEnumDecl())
2885 Ty = ED->getIntegerType();
2886
2887 if (Ty->isIntegralOrEnumerationType() &&
2888 isPromotableIntegerTypeForABI(Ty))
2889 return ABIArgInfo::getExtend(Ty, T: CGT.ConvertType(T: Ty));
2890 }
2891
2892 if (ResType->isIntegerTy(BitWidth: 128)) {
2893 assert(Hi == Integer);
2894 ++neededInt;
2895 return ABIArgInfo::getDirect(T: ResType);
2896 }
2897 break;
2898
2899 // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
2900 // available SSE register is used, the registers are taken in the
2901 // order from %xmm0 to %xmm7.
2902 case SSE: {
2903 llvm::Type *IRType = CGT.ConvertType(T: Ty);
2904 ResType = GetSSETypeAtOffset(IRType, IROffset: 0, SourceTy: Ty, SourceOffset: 0);
2905 ++neededSSE;
2906 break;
2907 }
2908 }
2909
2910 llvm::Type *HighPart = nullptr;
2911 switch (Hi) {
2912 // Memory was handled previously, ComplexX87 and X87 should
2913 // never occur as hi classes, and X87Up must be preceded by X87,
2914 // which is passed in memory.
2915 case Memory:
2916 case X87:
2917 case ComplexX87:
2918 llvm_unreachable("Invalid classification for hi word.");
2919
2920 case NoClass: break;
2921
2922 case Integer:
2923 ++neededInt;
2924 // Pick an 8-byte type based on the preferred type.
2925 HighPart = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: Ty), IROffset: 8, SourceTy: Ty, SourceOffset: 8);
2926
2927 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
2928 return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2929 break;
2930
2931 // X87Up generally doesn't occur here (long double is passed in
2932 // memory), except in situations involving unions.
2933 case X87Up:
2934 case SSE:
2935 ++neededSSE;
2936 HighPart = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: Ty), IROffset: 8, SourceTy: Ty, SourceOffset: 8);
2937
2938 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
2939 return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2940 break;
2941
2942 // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
2943 // eightbyte is passed in the upper half of the last used SSE
2944 // register. This only happens when 128-bit vectors are passed.
2945 case SSEUp:
2946 assert(Lo == SSE && "Unexpected SSEUp classification");
2947 ResType = GetByteVectorType(Ty);
2948 break;
2949 }
2950
2951 // If a high part was specified, merge it together with the low part. It is
2952 // known to pass in the high eightbyte of the result. We do this by forming a
2953 // first class struct aggregate with the high and low part: {low, high}
2954 if (HighPart)
2955 ResType = GetX86_64ByValArgumentPair(Lo: ResType, Hi: HighPart, TD: getDataLayout());
2956
2957 return ABIArgInfo::getDirect(T: ResType);
2958}
2959
2960// Returns true if the struct can be passed directly in registers. If so, the
2961// number of registers required will be returned in `NeededInt` and `NeededSSE`,
2962// and `CoerceElts` will contain an expanded sequence of LLVM IR types that each
2963// field should coerce to.
2964bool X86_64ABIInfo::passRegCallStructTypeDirectly(
2965 QualType Ty, SmallVectorImpl<llvm::Type *> &CoerceElts, unsigned &NeededInt,
2966 unsigned &NeededSSE, unsigned &MaxVectorWidth) const {
2967
2968 auto *RD =
2969 cast<RecordType>(Val: Ty.getCanonicalType())->getDecl()->getDefinitionOrSelf();
2970 if (RD->hasFlexibleArrayMember())
2971 return false;
2972
2973 // Classify the bases.
2974 if (auto CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
2975 if (CXXRD->isDynamicClass())
2976 return false;
2977
2978 for (const auto &I : CXXRD->bases()) {
2979 QualType BaseTy = I.getType();
2980 if (isEmptyRecord(Context&: getContext(), T: BaseTy, AllowArrays: true))
2981 continue;
2982 if (!passRegCallStructTypeDirectly(Ty: BaseTy, CoerceElts, NeededInt,
2983 NeededSSE, MaxVectorWidth))
2984 return false;
2985 }
2986 }
2987
2988 // Classify the members.
2989 for (const auto *FD : RD->fields()) {
2990 QualType MTy = FD->getType();
2991 if (MTy->isRecordType() && !MTy->isUnionType()) {
2992 if (isEmptyRecord(Context&: getContext(), T: MTy, AllowArrays: true))
2993 continue;
2994 if (!passRegCallStructTypeDirectly(Ty: MTy, CoerceElts, NeededInt, NeededSSE,
2995 MaxVectorWidth))
2996 return false;
2997 continue;
2998 }
2999
3000 const auto *AT = getContext().getAsConstantArrayType(T: MTy);
3001 if (AT)
3002 MTy = AT->getElementType();
3003
3004 unsigned LocalNeededInt, LocalNeededSSE;
3005 ABIArgInfo AI = classifyArgumentType(Ty: MTy, UINT_MAX, neededInt&: LocalNeededInt,
3006 neededSSE&: LocalNeededSSE, isNamedArg: true, IsRegCall: true);
3007 if (AI.isIgnore())
3008 continue;
3009 if (AI.isIndirect())
3010 return false;
3011
3012 llvm::Type *CoerceTy = AI.getCoerceToType();
3013 assert(CoerceTy && "ABI info for struct member has no coerce type");
3014 if (AT) {
3015 uint64_t NumElts = AT->getZExtSize();
3016 LocalNeededInt *= NumElts;
3017 LocalNeededSSE *= NumElts;
3018 CoerceElts.push_back(Elt: llvm::ArrayType::get(ElementType: CoerceTy, NumElements: NumElts));
3019 } else {
3020 CoerceElts.push_back(Elt: CoerceTy);
3021 }
3022
3023 if (const auto *VT = MTy->getAs<VectorType>())
3024 if (getContext().getTypeSize(T: VT) > MaxVectorWidth)
3025 MaxVectorWidth = getContext().getTypeSize(T: VT);
3026
3027 NeededInt += LocalNeededInt;
3028 NeededSSE += LocalNeededSSE;
3029 }
3030
3031 return true;
3032}
3033
3034ABIArgInfo
3035X86_64ABIInfo::classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
3036 unsigned &NeededSSE,
3037 unsigned &MaxVectorWidth) const {
3038 NeededInt = 0;
3039 NeededSSE = 0;
3040 MaxVectorWidth = 0;
3041
3042 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
3043 return ABIArgInfo::getIgnore();
3044
3045 SmallVector<llvm::Type *, 16> CoerceElts;
3046 if (!passRegCallStructTypeDirectly(Ty, CoerceElts, NeededInt, NeededSSE,
3047 MaxVectorWidth)) {
3048 NeededInt = NeededSSE = 0;
3049 return getIndirectReturnResult(Ty);
3050 }
3051
3052 assert(!CoerceElts.empty() && "Non-empty struct produced no element types");
3053 return ABIArgInfo::getDirect(
3054 T: llvm::StructType::get(Context&: getVMContext(), Elements: CoerceElts));
3055}
3056
3057unsigned
3058X86_64ABIInfo::getX86ABIAVXLevel(const FunctionDecl *FD,
3059 const FunctionType::ExtInfo &Info) const {
3060 return static_cast<unsigned>(getEffectiveX86AVXABILevel(CGT, GlobalAVXLevel: AVXLevel, FD));
3061}
3062
3063void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3064 const unsigned CallingConv = FI.getCallingConvention();
3065 // It is possible to force Win64 calling convention on any x86_64 target by
3066 // using __attribute__((ms_abi)). In such case to correctly emit Win64
3067 // compatible code delegate this call to WinX86_64ABIInfo::computeInfo.
3068 if (CallingConv == llvm::CallingConv::Win64) {
3069 WinX86_64ABIInfo Win64ABIInfo(CGT, AVXLevel);
3070 Win64ABIInfo.computeInfo(FI);
3071 return;
3072 }
3073
3074 assert(FI.getX86ABIAVXLevel() <=
3075 static_cast<unsigned>(X86AVXABILevel::AVX512) &&
3076 "Unexpected X86 AVX ABI level");
3077 X86AVXABILevel EffectiveAVXLevel =
3078 static_cast<X86AVXABILevel>(FI.getX86ABIAVXLevel());
3079 if (EffectiveAVXLevel != AVXLevel) {
3080 X86_64ABIInfo EffectiveABIInfo(CGT, EffectiveAVXLevel);
3081 EffectiveABIInfo.computeInfo(FI);
3082 return;
3083 }
3084
3085 bool IsRegCall = CallingConv == llvm::CallingConv::X86_RegCall;
3086
3087 // Keep track of the number of assigned registers.
3088 unsigned FreeIntRegs = IsRegCall ? 11 : 6;
3089 unsigned FreeSSERegs = IsRegCall ? 16 : 8;
3090 unsigned NeededInt = 0, NeededSSE = 0, MaxVectorWidth = 0;
3091
3092 if (!::classifyReturnType(CXXABI: getCXXABI(), FI, Info: *this)) {
3093 if (IsRegCall && FI.getReturnType()->getTypePtr()->isRecordType() &&
3094 !FI.getReturnType()->getTypePtr()->isUnionType()) {
3095 FI.getReturnInfo() = classifyRegCallStructType(
3096 Ty: FI.getReturnType(), NeededInt, NeededSSE, MaxVectorWidth);
3097 if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3098 FreeIntRegs -= NeededInt;
3099 FreeSSERegs -= NeededSSE;
3100 } else {
3101 FI.getReturnInfo() = getIndirectReturnResult(Ty: FI.getReturnType());
3102 }
3103 } else if (IsRegCall && FI.getReturnType()->getAs<ComplexType>() &&
3104 getContext().getCanonicalType(T: FI.getReturnType()
3105 ->getAs<ComplexType>()
3106 ->getElementType()) ==
3107 getContext().LongDoubleTy)
3108 // Complex Long Double Type is passed in Memory when Regcall
3109 // calling convention is used.
3110 FI.getReturnInfo() = getIndirectReturnResult(Ty: FI.getReturnType());
3111 else
3112 FI.getReturnInfo() = classifyReturnType(RetTy: FI.getReturnType());
3113 }
3114
3115 // If the return value is indirect, then the hidden argument is consuming one
3116 // integer register.
3117 if (FI.getReturnInfo().isIndirect())
3118 --FreeIntRegs;
3119 else if (NeededSSE && MaxVectorWidth > 0)
3120 FI.setMaxVectorWidth(MaxVectorWidth);
3121
3122 // The chain argument effectively gives us another free register.
3123 if (FI.isChainCall())
3124 ++FreeIntRegs;
3125
3126 // RegCall lets us reuse the return registers.
3127 if (IsRegCall)
3128 FreeSSERegs = 16;
3129
3130 unsigned NumRequiredArgs = FI.getNumRequiredArgs();
3131 // AMD64-ABI 3.2.3p3: Once arguments are classified, the registers
3132 // get assigned (in left-to-right order) for passing as follows...
3133 unsigned ArgNo = 0;
3134 for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
3135 it != ie; ++it, ++ArgNo) {
3136 bool IsNamedArg = ArgNo < NumRequiredArgs;
3137
3138 if (IsRegCall && it->type->isStructureOrClassType())
3139 it->info = classifyRegCallStructType(Ty: it->type, NeededInt, NeededSSE,
3140 MaxVectorWidth);
3141 else
3142 it->info = classifyArgumentType(Ty: it->type, freeIntRegs: FreeIntRegs, neededInt&: NeededInt,
3143 neededSSE&: NeededSSE, isNamedArg: IsNamedArg);
3144
3145 // AMD64-ABI 3.2.3p3: If there are no registers available for any
3146 // eightbyte of an argument, the whole argument is passed on the
3147 // stack. If registers have already been assigned for some
3148 // eightbytes of such an argument, the assignments get reverted.
3149 if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3150 FreeIntRegs -= NeededInt;
3151 FreeSSERegs -= NeededSSE;
3152 if (MaxVectorWidth > FI.getMaxVectorWidth())
3153 FI.setMaxVectorWidth(MaxVectorWidth);
3154 } else {
3155 it->info = getIndirectResult(Ty: it->type, freeIntRegs: FreeIntRegs);
3156 }
3157 }
3158}
3159
3160static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF,
3161 Address VAListAddr, QualType Ty) {
3162 Address overflow_arg_area_p =
3163 CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 2, Name: "overflow_arg_area_p");
3164 llvm::Value *overflow_arg_area =
3165 CGF.Builder.CreateLoad(Addr: overflow_arg_area_p, Name: "overflow_arg_area");
3166
3167 // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
3168 // byte boundary if alignment needed by type exceeds 8 byte boundary.
3169 // It isn't stated explicitly in the standard, but in practice we use
3170 // alignment greater than 16 where necessary.
3171 CharUnits Align = CGF.getContext().getTypeAlignInChars(T: Ty);
3172 if (Align > CharUnits::fromQuantity(Quantity: 8)) {
3173 overflow_arg_area = emitRoundPointerUpToAlignment(CGF, Ptr: overflow_arg_area,
3174 Align);
3175 }
3176
3177 // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
3178 llvm::Type *LTy = CGF.ConvertTypeForMem(T: Ty);
3179 llvm::Value *Res = overflow_arg_area;
3180
3181 // AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
3182 // l->overflow_arg_area + sizeof(type).
3183 // AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
3184 // an 8 byte boundary.
3185
3186 uint64_t SizeInBytes = (CGF.getContext().getTypeSize(T: Ty) + 7) / 8;
3187 llvm::Value *Offset =
3188 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: (SizeInBytes + 7) & ~7);
3189 overflow_arg_area = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: overflow_arg_area,
3190 IdxList: Offset, Name: "overflow_arg_area.next");
3191 CGF.Builder.CreateStore(Val: overflow_arg_area, Addr: overflow_arg_area_p);
3192
3193 // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
3194 return Address(Res, LTy, Align);
3195}
3196
3197RValue X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
3198 QualType Ty, AggValueSlot Slot) const {
3199 // Assume that va_list type is correct; should be pointer to LLVM type:
3200 // struct {
3201 // i32 gp_offset;
3202 // i32 fp_offset;
3203 // i8* overflow_arg_area;
3204 // i8* reg_save_area;
3205 // };
3206 unsigned neededInt, neededSSE;
3207
3208 Ty = getContext().getCanonicalType(T: Ty);
3209 ABIArgInfo AI = classifyArgumentType(Ty, freeIntRegs: 0, neededInt, neededSSE,
3210 /*isNamedArg*/false);
3211
3212 // Empty records are ignored for parameter passing purposes.
3213 if (AI.isIgnore())
3214 return Slot.asRValue();
3215
3216 // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
3217 // in the registers. If not go to step 7.
3218 if (!neededInt && !neededSSE)
3219 return CGF.EmitLoadOfAnyValue(
3220 V: CGF.MakeAddrLValue(Addr: EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty), T: Ty),
3221 Slot);
3222
3223 // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
3224 // general purpose registers needed to pass type and num_fp to hold
3225 // the number of floating point registers needed.
3226
3227 // AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
3228 // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
3229 // l->fp_offset > 304 - num_fp * 16 go to step 7.
3230 //
3231 // NOTE: 304 is a typo, there are (6 * 8 + 8 * 16) = 176 bytes of
3232 // register save space).
3233
3234 llvm::Value *InRegs = nullptr;
3235 Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid();
3236 llvm::Value *gp_offset = nullptr, *fp_offset = nullptr;
3237 if (neededInt) {
3238 gp_offset_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 0, Name: "gp_offset_p");
3239 gp_offset = CGF.Builder.CreateLoad(Addr: gp_offset_p, Name: "gp_offset");
3240 InRegs = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 48 - neededInt * 8);
3241 InRegs = CGF.Builder.CreateICmpULE(LHS: gp_offset, RHS: InRegs, Name: "fits_in_gp");
3242 }
3243
3244 if (neededSSE) {
3245 fp_offset_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 1, Name: "fp_offset_p");
3246 fp_offset = CGF.Builder.CreateLoad(Addr: fp_offset_p, Name: "fp_offset");
3247 llvm::Value *FitsInFP =
3248 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 176 - neededSSE * 16);
3249 FitsInFP = CGF.Builder.CreateICmpULE(LHS: fp_offset, RHS: FitsInFP, Name: "fits_in_fp");
3250 InRegs = InRegs ? CGF.Builder.CreateAnd(LHS: InRegs, RHS: FitsInFP) : FitsInFP;
3251 }
3252
3253 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock(name: "vaarg.in_reg");
3254 llvm::BasicBlock *InMemBlock = CGF.createBasicBlock(name: "vaarg.in_mem");
3255 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "vaarg.end");
3256 CGF.Builder.CreateCondBr(Cond: InRegs, True: InRegBlock, False: InMemBlock);
3257
3258 // Emit code to load the value if it was passed in registers.
3259
3260 CGF.EmitBlock(BB: InRegBlock);
3261
3262 // AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
3263 // an offset of l->gp_offset and/or l->fp_offset. This may require
3264 // copying to a temporary location in case the parameter is passed
3265 // in different register classes or requires an alignment greater
3266 // than 8 for general purpose registers and 16 for XMM registers.
3267 //
3268 // FIXME: This really results in shameful code when we end up needing to
3269 // collect arguments from different places; often what should result in a
3270 // simple assembling of a structure from scattered addresses has many more
3271 // loads than necessary. Can we clean this up?
3272 llvm::Type *LTy = CGF.ConvertTypeForMem(T: Ty);
3273 llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(
3274 Addr: CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 3), Name: "reg_save_area");
3275
3276 Address RegAddr = Address::invalid();
3277 if (neededInt && neededSSE) {
3278 // FIXME: Cleanup.
3279 assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
3280 llvm::StructType *ST = cast<llvm::StructType>(Val: AI.getCoerceToType());
3281 Address Tmp = CGF.CreateMemTempWithoutCast(T: Ty);
3282 Tmp = Tmp.withElementType(ElemTy: ST);
3283 assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
3284 llvm::Type *TyLo = ST->getElementType(N: 0);
3285 llvm::Type *TyHi = ST->getElementType(N: 1);
3286 assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&
3287 "Unexpected ABI info for mixed regs");
3288 llvm::Value *GPAddr =
3289 CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: gp_offset);
3290 llvm::Value *FPAddr =
3291 CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: fp_offset);
3292 llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? FPAddr : GPAddr;
3293 llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? GPAddr : FPAddr;
3294
3295 // Copy the first element.
3296 // FIXME: Our choice of alignment here and below is probably pessimistic.
3297 llvm::Value *V = CGF.Builder.CreateAlignedLoad(
3298 Ty: TyLo, Addr: RegLoAddr,
3299 Align: CharUnits::fromQuantity(Quantity: getDataLayout().getABITypeAlign(Ty: TyLo)));
3300 CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 0));
3301
3302 // Copy the second element.
3303 V = CGF.Builder.CreateAlignedLoad(
3304 Ty: TyHi, Addr: RegHiAddr,
3305 Align: CharUnits::fromQuantity(Quantity: getDataLayout().getABITypeAlign(Ty: TyHi)));
3306 CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 1));
3307
3308 RegAddr = Tmp.withElementType(ElemTy: LTy);
3309 } else if (neededInt || neededSSE == 1) {
3310 // Copy to a temporary if necessary to ensure the appropriate alignment.
3311 auto TInfo = getContext().getTypeInfoInChars(T: Ty);
3312 uint64_t TySize = TInfo.Width.getQuantity();
3313 CharUnits TyAlign = TInfo.Align;
3314 llvm::Type *CoTy = nullptr;
3315 if (AI.isDirect())
3316 CoTy = AI.getCoerceToType();
3317
3318 llvm::Value *GpOrFpOffset = neededInt ? gp_offset : fp_offset;
3319 uint64_t Alignment = neededInt ? 8 : 16;
3320 uint64_t RegSize = neededInt ? neededInt * 8 : 16;
3321 // There are two cases require special handling:
3322 // 1)
3323 // ```
3324 // struct {
3325 // struct {} a[8];
3326 // int b;
3327 // };
3328 // ```
3329 // The lower 8 bytes of the structure are not stored,
3330 // so an 8-byte offset is needed when accessing the structure.
3331 // 2)
3332 // ```
3333 // struct {
3334 // long long a;
3335 // struct {} b;
3336 // };
3337 // ```
3338 // The stored size of this structure is smaller than its actual size,
3339 // which may lead to reading past the end of the register save area.
3340 if (CoTy && (AI.getDirectOffset() == 8 || RegSize < TySize)) {
3341 Address Tmp = CGF.CreateMemTempWithoutCast(T: Ty);
3342 llvm::Value *Addr =
3343 CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: GpOrFpOffset);
3344 llvm::Value *Src = CGF.Builder.CreateAlignedLoad(Ty: CoTy, Addr, Align: TyAlign);
3345 llvm::Value *PtrOffset =
3346 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: AI.getDirectOffset());
3347 Address Dst = Address(
3348 CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: Tmp.getBasePointer(), IdxList: PtrOffset),
3349 LTy, TyAlign);
3350 CGF.Builder.CreateStore(Val: Src, Addr: Dst);
3351 RegAddr = Tmp.withElementType(ElemTy: LTy);
3352 } else {
3353 RegAddr =
3354 Address(CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: GpOrFpOffset),
3355 LTy, CharUnits::fromQuantity(Quantity: Alignment));
3356
3357 // Copy into a temporary if the type is more aligned than the
3358 // register save area.
3359 if (neededInt && TyAlign.getQuantity() > 8) {
3360 Address Tmp = CGF.CreateMemTempWithoutCast(T: Ty);
3361 CGF.Builder.CreateMemCpy(Dest: Tmp, Src: RegAddr, Size: TySize, IsVolatile: false);
3362 RegAddr = Tmp;
3363 }
3364 }
3365
3366 } else {
3367 assert(neededSSE == 2 && "Invalid number of needed registers!");
3368 // SSE registers are spaced 16 bytes apart in the register save
3369 // area, we need to collect the two eightbytes together.
3370 // The ABI isn't explicit about this, but it seems reasonable
3371 // to assume that the slots are 16-byte aligned, since the stack is
3372 // naturally 16-byte aligned and the prologue is expected to store
3373 // all the SSE registers to the RSA.
3374 Address RegAddrLo = Address(CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea,
3375 IdxList: fp_offset),
3376 CGF.Int8Ty, CharUnits::fromQuantity(Quantity: 16));
3377 Address RegAddrHi =
3378 CGF.Builder.CreateConstInBoundsByteGEP(Addr: RegAddrLo,
3379 Offset: CharUnits::fromQuantity(Quantity: 16));
3380 llvm::Type *ST = AI.canHaveCoerceToType()
3381 ? AI.getCoerceToType()
3382 : llvm::StructType::get(elt1: CGF.DoubleTy, elts: CGF.DoubleTy);
3383 llvm::Value *V;
3384 Address Tmp = CGF.CreateMemTempWithoutCast(T: Ty);
3385 Tmp = Tmp.withElementType(ElemTy: ST);
3386 V = CGF.Builder.CreateLoad(
3387 Addr: RegAddrLo.withElementType(ElemTy: ST->getStructElementType(N: 0)));
3388 CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 0));
3389 V = CGF.Builder.CreateLoad(
3390 Addr: RegAddrHi.withElementType(ElemTy: ST->getStructElementType(N: 1)));
3391 CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 1));
3392
3393 RegAddr = Tmp.withElementType(ElemTy: LTy);
3394 }
3395
3396 // AMD64-ABI 3.5.7p5: Step 5. Set:
3397 // l->gp_offset = l->gp_offset + num_gp * 8
3398 // l->fp_offset = l->fp_offset + num_fp * 16.
3399 if (neededInt) {
3400 llvm::Value *Offset = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: neededInt * 8);
3401 CGF.Builder.CreateStore(Val: CGF.Builder.CreateAdd(LHS: gp_offset, RHS: Offset),
3402 Addr: gp_offset_p);
3403 }
3404 if (neededSSE) {
3405 llvm::Value *Offset = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: neededSSE * 16);
3406 CGF.Builder.CreateStore(Val: CGF.Builder.CreateAdd(LHS: fp_offset, RHS: Offset),
3407 Addr: fp_offset_p);
3408 }
3409 CGF.EmitBranch(Block: ContBlock);
3410
3411 // Emit code to load the value if it was passed in memory.
3412
3413 CGF.EmitBlock(BB: InMemBlock);
3414 Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
3415
3416 // Return the appropriate result.
3417
3418 CGF.EmitBlock(BB: ContBlock);
3419 Address ResAddr = emitMergePHI(CGF, Addr1: RegAddr, Block1: InRegBlock, Addr2: MemAddr, Block2: InMemBlock,
3420 Name: "vaarg.addr");
3421 return CGF.EmitLoadOfAnyValue(V: CGF.MakeAddrLValue(Addr: ResAddr, T: Ty), Slot);
3422}
3423
3424RValue X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
3425 QualType Ty, AggValueSlot Slot) const {
3426 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3427 // not 1, 2, 4, or 8 bytes, must be passed by reference."
3428 uint64_t Width = getContext().getTypeSize(T: Ty);
3429 bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Value: Width);
3430
3431 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
3432 ValueInfo: CGF.getContext().getTypeInfoInChars(T: Ty),
3433 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 8),
3434 /*allowHigherAlign*/ AllowHigherAlign: false, Slot);
3435}
3436
3437ABIArgInfo WinX86_64ABIInfo::reclassifyHvaArgForVectorCall(
3438 QualType Ty, unsigned &FreeSSERegs, const ABIArgInfo &current) const {
3439 const Type *Base = nullptr;
3440 uint64_t NumElts = 0;
3441
3442 if (!Ty->isBuiltinType() && !Ty->isVectorType() &&
3443 isHomogeneousAggregate(Ty, Base, Members&: NumElts) && FreeSSERegs >= NumElts) {
3444 FreeSSERegs -= NumElts;
3445 return getDirectX86Hva();
3446 }
3447 return current;
3448}
3449
3450ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
3451 bool IsReturnType, unsigned CC) const {
3452 bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
3453 bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
3454
3455 if (Ty->isVoidType())
3456 return ABIArgInfo::getIgnore();
3457
3458 if (const auto *ED = Ty->getAsEnumDecl())
3459 Ty = ED->getIntegerType();
3460
3461 TypeInfo Info = getContext().getTypeInfo(T: Ty);
3462 uint64_t Width = Info.Width;
3463 CharUnits Align = getContext().toCharUnitsFromBits(BitSize: Info.Align);
3464
3465 const RecordType *RT = Ty->getAsCanonical<RecordType>();
3466 if (RT) {
3467 if (!IsReturnType) {
3468 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, CXXABI&: getCXXABI()))
3469 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
3470 ByVal: RAA == CGCXXABI::RAA_DirectInMemory);
3471 }
3472
3473 if (RT->getDecl()->getDefinitionOrSelf()->hasFlexibleArrayMember())
3474 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
3475 /*ByVal=*/false);
3476 }
3477
3478 const Type *Base = nullptr;
3479 uint64_t NumElts = 0;
3480 // vectorcall adds the concept of a homogenous vector aggregate, similar to
3481 // other targets.
3482 if ((IsVectorCall || IsRegCall) &&
3483 isHomogeneousAggregate(Ty, Base, Members&: NumElts)) {
3484 if (IsRegCall) {
3485 if (FreeSSERegs >= NumElts) {
3486 FreeSSERegs -= NumElts;
3487 if (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())
3488 return ABIArgInfo::getDirect();
3489 return ABIArgInfo::getExpand();
3490 }
3491 return ABIArgInfo::getIndirect(
3492 Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3493 /*ByVal=*/false);
3494 } else if (IsVectorCall) {
3495 if (FreeSSERegs >= NumElts &&
3496 (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())) {
3497 FreeSSERegs -= NumElts;
3498 return ABIArgInfo::getDirect();
3499 } else if (IsReturnType) {
3500 return ABIArgInfo::getExpand();
3501 } else if (!Ty->isBuiltinType() && !Ty->isVectorType()) {
3502 // HVAs are delayed and reclassified in the 2nd step.
3503 return ABIArgInfo::getIndirect(
3504 Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3505 /*ByVal=*/false);
3506 }
3507 }
3508 }
3509
3510 if (Ty->isMemberPointerType()) {
3511 // If the member pointer is represented by an LLVM int or ptr, pass it
3512 // directly.
3513 llvm::Type *LLTy = CGT.ConvertType(T: Ty);
3514 if (LLTy->isPointerTy() || LLTy->isIntegerTy())
3515 return ABIArgInfo::getDirect();
3516 }
3517
3518 if (RT || Ty->isAnyComplexType() || Ty->isMemberPointerType()) {
3519 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3520 // not 1, 2, 4, or 8 bytes, must be passed by reference."
3521 if (Width > 64 || !llvm::isPowerOf2_64(Value: Width))
3522 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
3523 /*ByVal=*/false);
3524
3525 // Otherwise, coerce it to a small integer.
3526 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Width));
3527 }
3528
3529 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
3530 switch (BT->getKind()) {
3531 case BuiltinType::Bool:
3532 // Bool type is always extended to the ABI, other builtin types are not
3533 // extended.
3534 return ABIArgInfo::getExtend(Ty);
3535
3536 case BuiltinType::LongDouble:
3537 // Mingw64 GCC uses the old 80 bit extended precision floating point
3538 // unit. It passes them indirectly through memory.
3539 if (IsMingw64) {
3540 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
3541 if (LDF == &llvm::APFloat::x87DoubleExtended())
3542 return ABIArgInfo::getIndirect(
3543 Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3544 /*ByVal=*/false);
3545 }
3546 break;
3547
3548 case BuiltinType::Int128:
3549 case BuiltinType::UInt128:
3550 case BuiltinType::Float128:
3551 // If it's a parameter type, the normal ABI rule is that arguments larger
3552 // than 8 bytes are passed indirectly. GCC follows it. We follow it too,
3553 // even though it isn't particularly efficient.
3554 if (!IsReturnType)
3555 return ABIArgInfo::getIndirect(
3556 Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3557 /*ByVal=*/false);
3558
3559 // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
3560 // Clang matches them for compatibility.
3561 if (BT->getKind() == BuiltinType::Int128 ||
3562 BT->getKind() == BuiltinType::UInt128)
3563 return ABIArgInfo::getDirect(T: llvm::FixedVectorType::get(
3564 ElementType: llvm::Type::getInt64Ty(C&: getVMContext()), NumElts: 2));
3565
3566 // Mingw64 GCC returns f128 via sret, and Clang matches that for
3567 // compatibility. This mirrors the X86 backend's CanLowerReturn logic.
3568 if (BT->getKind() == BuiltinType::Float128) {
3569 auto IsWin64F128StackCC = [this](unsigned CC) -> bool {
3570 switch (CC) {
3571 case llvm::CallingConv::Win64:
3572 return true;
3573 case llvm::CallingConv::C:
3574 return getTarget().getTriple().isOSWindowsOrUEFI();
3575 default:
3576 return false;
3577 }
3578 };
3579
3580 if (IsWin64F128StackCC(CC))
3581 return getNaturalAlignIndirect(
3582 Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(), /*ByVal=*/false);
3583 }
3584 break;
3585
3586 default:
3587 break;
3588 }
3589 }
3590
3591 if (Ty->isBitIntType()) {
3592 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3593 // not 1, 2, 4, or 8 bytes, must be passed by reference."
3594 // However, non-power-of-two bit-precise integers will be passed as 1, 2, 4,
3595 // or 8 bytes anyway as long is it fits in them, so we don't have to check
3596 // the power of 2.
3597 if (Width <= 64)
3598 return ABIArgInfo::getDirect();
3599 return ABIArgInfo::getIndirect(
3600 Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3601 /*ByVal=*/false);
3602 }
3603
3604 return ABIArgInfo::getDirect();
3605}
3606
3607unsigned
3608WinX86_64ABIInfo::getX86ABIAVXLevel(const FunctionDecl *FD,
3609 const FunctionType::ExtInfo &Info) const {
3610 if (Info.getCC() == CC_X86_64SysV) {
3611 return static_cast<unsigned>(getEffectiveX86AVXABILevel(CGT, GlobalAVXLevel: AVXLevel, FD));
3612 }
3613
3614 return static_cast<unsigned>(AVXLevel);
3615}
3616
3617void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3618 const unsigned CC = FI.getCallingConvention();
3619 bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
3620 bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
3621
3622 // If __attribute__((sysv_abi)) is in use, use the SysV argument
3623 // classification rules.
3624 if (CC == llvm::CallingConv::X86_64_SysV) {
3625 X86_64ABIInfo SysVABIInfo(CGT, AVXLevel);
3626 SysVABIInfo.computeInfo(FI);
3627 return;
3628 }
3629
3630 unsigned FreeSSERegs = 0;
3631 if (IsVectorCall) {
3632 // We can use up to 4 SSE return registers with vectorcall.
3633 FreeSSERegs = 4;
3634 } else if (IsRegCall) {
3635 // RegCall gives us 16 SSE registers.
3636 FreeSSERegs = 16;
3637 }
3638
3639 if (!getCXXABI().classifyReturnType(FI))
3640 FI.getReturnInfo() = classify(Ty: FI.getReturnType(), FreeSSERegs, IsReturnType: true, CC);
3641
3642 if (IsVectorCall) {
3643 // We can use up to 6 SSE register parameters with vectorcall.
3644 FreeSSERegs = 6;
3645 } else if (IsRegCall) {
3646 // RegCall gives us 16 SSE registers, we can reuse the return registers.
3647 FreeSSERegs = 16;
3648 }
3649
3650 unsigned ArgNum = 0;
3651 unsigned ZeroSSERegs = 0;
3652 for (auto &I : FI.arguments()) {
3653 // Vectorcall in x64 only permits the first 6 arguments to be passed as
3654 // XMM/YMM registers. After the sixth argument, pretend no vector
3655 // registers are left.
3656 unsigned *MaybeFreeSSERegs =
3657 (IsVectorCall && ArgNum >= 6) ? &ZeroSSERegs : &FreeSSERegs;
3658 I.info = classify(Ty: I.type, FreeSSERegs&: *MaybeFreeSSERegs, IsReturnType: false, CC);
3659 ++ArgNum;
3660 }
3661
3662 if (IsVectorCall) {
3663 // For vectorcall, assign aggregate HVAs to any free vector registers in a
3664 // second pass.
3665 for (auto &I : FI.arguments())
3666 I.info = reclassifyHvaArgForVectorCall(Ty: I.type, FreeSSERegs, current: I.info);
3667 }
3668}
3669
3670RValue WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
3671 QualType Ty, AggValueSlot Slot) const {
3672 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3673 // not 1, 2, 4, or 8 bytes, must be passed by reference."
3674 uint64_t Width = getContext().getTypeSize(T: Ty);
3675 bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Value: Width);
3676
3677 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
3678 ValueInfo: CGF.getContext().getTypeInfoInChars(T: Ty),
3679 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 8),
3680 /*allowHigherAlign*/ AllowHigherAlign: false, Slot);
3681}
3682
3683std::unique_ptr<TargetCodeGenInfo> CodeGen::createX86_32TargetCodeGenInfo(
3684 CodeGenModule &CGM, bool DarwinVectorABI, bool Win32StructABI,
3685 unsigned NumRegisterParameters, bool SoftFloatABI) {
3686 bool RetSmallStructInRegABI = X86_32TargetCodeGenInfo::isStructReturnInRegABI(
3687 Triple: CGM.getTriple(), Opts: CGM.getCodeGenOpts());
3688 return std::make_unique<X86_32TargetCodeGenInfo>(
3689 args&: CGM.getTypes(), args&: DarwinVectorABI, args&: RetSmallStructInRegABI, args&: Win32StructABI,
3690 args&: NumRegisterParameters, args&: SoftFloatABI);
3691}
3692
3693std::unique_ptr<TargetCodeGenInfo> CodeGen::createWinX86_32TargetCodeGenInfo(
3694 CodeGenModule &CGM, bool DarwinVectorABI, bool Win32StructABI,
3695 unsigned NumRegisterParameters) {
3696 bool RetSmallStructInRegABI = X86_32TargetCodeGenInfo::isStructReturnInRegABI(
3697 Triple: CGM.getTriple(), Opts: CGM.getCodeGenOpts());
3698 return std::make_unique<WinX86_32TargetCodeGenInfo>(
3699 args&: CGM.getTypes(), args&: DarwinVectorABI, args&: RetSmallStructInRegABI, args&: Win32StructABI,
3700 args&: NumRegisterParameters);
3701}
3702
3703std::unique_ptr<TargetCodeGenInfo>
3704CodeGen::createX86_64TargetCodeGenInfo(CodeGenModule &CGM,
3705 X86AVXABILevel AVXLevel) {
3706 return std::make_unique<X86_64TargetCodeGenInfo>(args&: CGM.getTypes(), args&: AVXLevel);
3707}
3708
3709std::unique_ptr<TargetCodeGenInfo>
3710CodeGen::createWinX86_64TargetCodeGenInfo(CodeGenModule &CGM,
3711 X86AVXABILevel AVXLevel) {
3712 return std::make_unique<WinX86_64TargetCodeGenInfo>(args&: CGM.getTypes(), args&: AVXLevel);
3713}
3714