1//===- X86.cpp ------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ABIInfoImpl.h"
10#include "TargetInfo.h"
11#include "clang/Basic/DiagnosticFrontend.h"
12#include "clang/Basic/SourceLocation.h"
13#include "llvm/ADT/SmallBitVector.h"
14
15using namespace clang;
16using namespace clang::CodeGen;
17
18namespace {
19
20/// IsX86_MMXType - Return true if this is an MMX type.
21bool IsX86_MMXType(llvm::Type *IRType) {
22 // Return true if the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>.
23 return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&
24 cast<llvm::VectorType>(Val: IRType)->getElementType()->isIntegerTy() &&
25 IRType->getScalarSizeInBits() != 64;
26}
27
28static llvm::Type *X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
29 StringRef Constraint,
30 llvm::Type *Ty) {
31 bool IsMMXCons = llvm::StringSwitch<bool>(Constraint)
32 .Cases(CaseStrings: {"y", "&y", "^Ym"}, Value: true)
33 .Default(Value: false);
34 if (IsMMXCons && Ty->isVectorTy() &&
35 cast<llvm::VectorType>(Val: Ty)->getPrimitiveSizeInBits().getFixedValue() !=
36 64)
37 return nullptr; // Invalid MMX constraint
38
39 if (Constraint == "k") {
40 llvm::Type *Int1Ty = llvm::Type::getInt1Ty(C&: CGF.getLLVMContext());
41 return llvm::FixedVectorType::get(ElementType: Int1Ty, NumElts: Ty->getScalarSizeInBits());
42 }
43
44 // No operation needed
45 return Ty;
46}
47
48/// Returns true if this type can be passed in SSE registers with the
49/// X86_VectorCall calling convention. Shared between x86_32 and x86_64.
50static bool isX86VectorTypeForVectorCall(ASTContext &Context, QualType Ty) {
51 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
52 if (BT->isFloatingPoint() && BT->getKind() != BuiltinType::Half) {
53 if (BT->getKind() == BuiltinType::LongDouble) {
54 if (&Context.getTargetInfo().getLongDoubleFormat() ==
55 &llvm::APFloat::x87DoubleExtended())
56 return false;
57 }
58 return true;
59 }
60 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
61 // vectorcall can pass XMM, YMM, and ZMM vectors. We don't pass SSE1 MMX
62 // registers specially.
63 unsigned VecSize = Context.getTypeSize(T: VT);
64 if (VecSize == 128 || VecSize == 256 || VecSize == 512)
65 return true;
66 }
67 return false;
68}
69
70/// Returns true if this aggregate is small enough to be passed in SSE registers
71/// in the X86_VectorCall calling convention. Shared between x86_32 and x86_64.
72static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) {
73 return NumMembers <= 4;
74}
75
76/// Returns a Homogeneous Vector Aggregate ABIArgInfo, used in X86.
77static ABIArgInfo getDirectX86Hva(llvm::Type* T = nullptr) {
78 auto AI = ABIArgInfo::getDirect(T);
79 AI.setInReg(true);
80 AI.setCanBeFlattened(false);
81 return AI;
82}
83
84//===----------------------------------------------------------------------===//
85// X86-32 ABI Implementation
86//===----------------------------------------------------------------------===//
87
88/// Similar to llvm::CCState, but for Clang.
89struct CCState {
90 CCState(CGFunctionInfo &FI)
91 : IsPreassigned(FI.arg_size()), CC(FI.getCallingConvention()),
92 Required(FI.getRequiredArgs()), IsDelegateCall(FI.isDelegateCall()) {}
93
94 llvm::SmallBitVector IsPreassigned;
95 unsigned CC = CallingConv::CC_C;
96 unsigned FreeRegs = 0;
97 unsigned FreeSSERegs = 0;
98 RequiredArgs Required;
99 bool IsDelegateCall = false;
100};
101
102/// X86_32ABIInfo - The X86-32 ABI information.
103class X86_32ABIInfo : public ABIInfo {
104 enum Class { Integer, Float, AlwaysStack };
105
106 static const unsigned MinABIStackAlignInBytes = 4;
107
108 bool IsDarwinVectorABI;
109 bool IsRetSmallStructInRegABI;
110 bool IsWin32StructABI;
111 bool IsSoftFloatABI;
112 bool IsMCUABI;
113 bool IsLinuxABI;
114 unsigned DefaultNumRegisterParameters;
115
116 static bool isRegisterSize(unsigned Size) {
117 return (Size == 8 || Size == 16 || Size == 32 || Size == 64);
118 }
119
120 bool isHomogeneousAggregateBaseType(QualType Ty) const override {
121 // FIXME: Assumes vectorcall is in use.
122 return isX86VectorTypeForVectorCall(Context&: getContext(), Ty);
123 }
124
125 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
126 uint64_t NumMembers) const override {
127 // FIXME: Assumes vectorcall is in use.
128 return isX86VectorCallAggregateSmallEnough(NumMembers);
129 }
130
131 bool shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const;
132
133 /// getIndirectResult - Give a source type \arg Ty, return a suitable result
134 /// such that the argument will be passed in memory.
135 ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
136
137 ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const;
138
139 /// Return the alignment to use for the given type on the stack.
140 unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
141
142 Class classify(QualType Ty) const;
143 ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const;
144 ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State,
145 unsigned ArgIndex) const;
146
147 /// Updates the number of available free registers, returns
148 /// true if any registers were allocated.
149 bool updateFreeRegs(QualType Ty, CCState &State) const;
150
151 bool shouldAggregateUseDirect(QualType Ty, CCState &State, bool &InReg,
152 bool &NeedsPadding) const;
153 bool shouldPrimitiveUseInReg(QualType Ty, CCState &State) const;
154
155 bool canExpandIndirectArgument(QualType Ty) const;
156
157 /// Rewrite the function info so that all memory arguments use
158 /// inalloca.
159 void rewriteWithInAlloca(CGFunctionInfo &FI) const;
160
161 void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
162 CharUnits &StackOffset, ABIArgInfo &Info,
163 QualType Type) const;
164 void runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const;
165
166public:
167
168 void computeInfo(CGFunctionInfo &FI) const override;
169 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
170 AggValueSlot Slot) const override;
171
172 X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
173 bool RetSmallStructInRegABI, bool Win32StructABI,
174 unsigned NumRegisterParameters, bool SoftFloatABI)
175 : ABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI),
176 IsRetSmallStructInRegABI(RetSmallStructInRegABI),
177 IsWin32StructABI(Win32StructABI), IsSoftFloatABI(SoftFloatABI),
178 IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()),
179 IsLinuxABI(CGT.getTarget().getTriple().isOSLinux() ||
180 CGT.getTarget().getTriple().isOSCygMing()),
181 DefaultNumRegisterParameters(NumRegisterParameters) {}
182};
183
184class X86_32SwiftABIInfo : public SwiftABIInfo {
185public:
186 explicit X86_32SwiftABIInfo(CodeGenTypes &CGT)
187 : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/false) {}
188
189 bool shouldPassIndirectly(ArrayRef<llvm::Type *> ComponentTys,
190 bool AsReturnValue) const override {
191 // LLVM's x86-32 lowering currently only assigns up to three
192 // integer registers and three fp registers. Oddly, it'll use up to
193 // four vector registers for vectors, but those can overlap with the
194 // scalar registers.
195 return occupiesMoreThan(scalarTypes: ComponentTys, /*total=*/maxAllRegisters: 3);
196 }
197};
198
199class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
200public:
201 X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
202 bool RetSmallStructInRegABI, bool Win32StructABI,
203 unsigned NumRegisterParameters, bool SoftFloatABI)
204 : TargetCodeGenInfo(std::make_unique<X86_32ABIInfo>(
205 args&: CGT, args&: DarwinVectorABI, args&: RetSmallStructInRegABI, args&: Win32StructABI,
206 args&: NumRegisterParameters, args&: SoftFloatABI)) {
207 SwiftInfo = std::make_unique<X86_32SwiftABIInfo>(args&: CGT);
208 }
209
210 static bool isStructReturnInRegABI(
211 const llvm::Triple &Triple, const CodeGenOptions &Opts);
212
213 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
214 CodeGen::CodeGenModule &CGM) const override;
215
216 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
217 // Darwin uses different dwarf register numbers for EH.
218 if (CGM.getTarget().getTriple().isOSDarwin()) return 5;
219 return 4;
220 }
221
222 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
223 llvm::Value *Address) const override;
224
225 llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
226 StringRef Constraint,
227 llvm::Type* Ty) const override {
228 return X86AdjustInlineAsmType(CGF, Constraint, Ty);
229 }
230
231 void addReturnRegisterOutputs(CodeGenFunction &CGF, LValue ReturnValue,
232 std::string &Constraints,
233 std::vector<llvm::Type *> &ResultRegTypes,
234 std::vector<llvm::Type *> &ResultTruncRegTypes,
235 std::vector<LValue> &ResultRegDests,
236 std::string &AsmString,
237 unsigned NumOutputs) const override;
238
239 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
240 return "movl\t%ebp, %ebp"
241 "\t\t// marker for objc_retainAutoreleaseReturnValue";
242 }
243};
244
245}
246
247/// Rewrite input constraint references after adding some output constraints.
248/// In the case where there is one output and one input and we add one output,
249/// we need to replace all operand references greater than or equal to 1:
250/// mov $0, $1
251/// mov eax, $1
252/// The result will be:
253/// mov $0, $2
254/// mov eax, $2
255static void rewriteInputConstraintReferences(unsigned FirstIn,
256 unsigned NumNewOuts,
257 std::string &AsmString) {
258 std::string Buf;
259 llvm::raw_string_ostream OS(Buf);
260 size_t Pos = 0;
261 while (Pos < AsmString.size()) {
262 size_t DollarStart = AsmString.find(c: '$', pos: Pos);
263 if (DollarStart == std::string::npos)
264 DollarStart = AsmString.size();
265 size_t DollarEnd = AsmString.find_first_not_of(c: '$', pos: DollarStart);
266 if (DollarEnd == std::string::npos)
267 DollarEnd = AsmString.size();
268 OS << StringRef(&AsmString[Pos], DollarEnd - Pos);
269 Pos = DollarEnd;
270 size_t NumDollars = DollarEnd - DollarStart;
271 if (NumDollars % 2 != 0 && Pos < AsmString.size()) {
272 // We have an operand reference.
273 size_t DigitStart = Pos;
274 if (AsmString[DigitStart] == '{') {
275 OS << '{';
276 ++DigitStart;
277 }
278 size_t DigitEnd = AsmString.find_first_not_of(s: "0123456789", pos: DigitStart);
279 if (DigitEnd == std::string::npos)
280 DigitEnd = AsmString.size();
281 StringRef OperandStr(&AsmString[DigitStart], DigitEnd - DigitStart);
282 unsigned OperandIndex;
283 if (!OperandStr.getAsInteger(Radix: 10, Result&: OperandIndex)) {
284 if (OperandIndex >= FirstIn)
285 OperandIndex += NumNewOuts;
286 OS << OperandIndex;
287 } else {
288 OS << OperandStr;
289 }
290 Pos = DigitEnd;
291 }
292 }
293 AsmString = std::move(Buf);
294}
295
296/// Add output constraints for EAX:EDX because they are return registers.
297void X86_32TargetCodeGenInfo::addReturnRegisterOutputs(
298 CodeGenFunction &CGF, LValue ReturnSlot, std::string &Constraints,
299 std::vector<llvm::Type *> &ResultRegTypes,
300 std::vector<llvm::Type *> &ResultTruncRegTypes,
301 std::vector<LValue> &ResultRegDests, std::string &AsmString,
302 unsigned NumOutputs) const {
303 uint64_t RetWidth = CGF.getContext().getTypeSize(T: ReturnSlot.getType());
304
305 // Use the EAX constraint if the width is 32 or smaller and EAX:EDX if it is
306 // larger.
307 if (!Constraints.empty())
308 Constraints += ',';
309 if (RetWidth <= 32) {
310 Constraints += "={eax}";
311 ResultRegTypes.push_back(x: CGF.Int32Ty);
312 } else {
313 // Use the 'A' constraint for EAX:EDX.
314 Constraints += "=A";
315 ResultRegTypes.push_back(x: CGF.Int64Ty);
316 }
317
318 // Truncate EAX or EAX:EDX to an integer of the appropriate size.
319 llvm::Type *CoerceTy = llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: RetWidth);
320 ResultTruncRegTypes.push_back(x: CoerceTy);
321
322 // Coerce the integer by bitcasting the return slot pointer.
323 ReturnSlot.setAddress(ReturnSlot.getAddress().withElementType(ElemTy: CoerceTy));
324 ResultRegDests.push_back(x: ReturnSlot);
325
326 rewriteInputConstraintReferences(FirstIn: NumOutputs, NumNewOuts: 1, AsmString);
327}
328
329/// shouldReturnTypeInRegister - Determine if the given type should be
330/// returned in a register (for the Darwin and MCU ABI).
331bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
332 ASTContext &Context) const {
333 uint64_t Size = Context.getTypeSize(T: Ty);
334
335 // For i386, type must be register sized.
336 // For the MCU ABI, it only needs to be <= 8-byte
337 if ((IsMCUABI && Size > 64) || (!IsMCUABI && !isRegisterSize(Size)))
338 return false;
339
340 if (Ty->isVectorType()) {
341 // 64- and 128- bit vectors inside structures are not returned in
342 // registers.
343 if (Size == 64 || Size == 128)
344 return false;
345
346 return true;
347 }
348
349 // If this is a builtin, pointer, enum, complex type, member pointer, or
350 // member function pointer it is ok.
351 if (Ty->getAs<BuiltinType>() || Ty->hasPointerRepresentation() ||
352 Ty->isAnyComplexType() || Ty->isEnumeralType() ||
353 Ty->isBlockPointerType() || Ty->isMemberPointerType())
354 return true;
355
356 // Arrays are treated like records.
357 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty))
358 return shouldReturnTypeInRegister(Ty: AT->getElementType(), Context);
359
360 // Otherwise, it must be a record type.
361 const auto *RD = Ty->getAsRecordDecl();
362 if (!RD)
363 return false;
364
365 // FIXME: Traverse bases here too.
366
367 // Structure types are passed in register if all fields would be
368 // passed in a register.
369 for (const auto *FD : RD->fields()) {
370 // Empty fields are ignored.
371 if (isEmptyField(Context, FD, AllowArrays: true))
372 continue;
373
374 // Check fields recursively.
375 if (!shouldReturnTypeInRegister(Ty: FD->getType(), Context))
376 return false;
377 }
378 return true;
379}
380
381static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {
382 // Treat complex types as the element type.
383 if (const ComplexType *CTy = Ty->getAs<ComplexType>())
384 Ty = CTy->getElementType();
385
386 // Check for a type which we know has a simple scalar argument-passing
387 // convention without any padding. (We're specifically looking for 32
388 // and 64-bit integer and integer-equivalents, float, and double.)
389 if (!Ty->getAs<BuiltinType>() && !Ty->hasPointerRepresentation() &&
390 !Ty->isEnumeralType() && !Ty->isBlockPointerType())
391 return false;
392
393 uint64_t Size = Context.getTypeSize(T: Ty);
394 return Size == 32 || Size == 64;
395}
396
397static bool addFieldSizes(ASTContext &Context, const RecordDecl *RD,
398 uint64_t &Size) {
399 for (const auto *FD : RD->fields()) {
400 // Scalar arguments on the stack get 4 byte alignment on x86. If the
401 // argument is smaller than 32-bits, expanding the struct will create
402 // alignment padding.
403 if (!is32Or64BitBasicType(Ty: FD->getType(), Context))
404 return false;
405
406 // FIXME: Reject bit-fields wholesale; there are two problems, we don't know
407 // how to expand them yet, and the predicate for telling if a bitfield still
408 // counts as "basic" is more complicated than what we were doing previously.
409 if (FD->isBitField())
410 return false;
411
412 Size += Context.getTypeSize(T: FD->getType());
413 }
414 return true;
415}
416
417static bool addBaseAndFieldSizes(ASTContext &Context, const CXXRecordDecl *RD,
418 uint64_t &Size) {
419 // Don't do this if there are any non-empty bases.
420 for (const CXXBaseSpecifier &Base : RD->bases()) {
421 if (!addBaseAndFieldSizes(Context, RD: Base.getType()->getAsCXXRecordDecl(),
422 Size))
423 return false;
424 }
425 if (!addFieldSizes(Context, RD, Size))
426 return false;
427 return true;
428}
429
430/// Test whether an argument type which is to be passed indirectly (on the
431/// stack) would have the equivalent layout if it was expanded into separate
432/// arguments. If so, we prefer to do the latter to avoid inhibiting
433/// optimizations.
434bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const {
435 // We can only expand structure types.
436 const RecordDecl *RD = Ty->getAsRecordDecl();
437 if (!RD)
438 return false;
439 uint64_t Size = 0;
440 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
441 if (!IsWin32StructABI) {
442 // On non-Windows, we have to conservatively match our old bitcode
443 // prototypes in order to be ABI-compatible at the bitcode level.
444 if (!CXXRD->isCLike())
445 return false;
446 } else {
447 // Don't do this for dynamic classes.
448 if (CXXRD->isDynamicClass())
449 return false;
450 }
451 if (!addBaseAndFieldSizes(Context&: getContext(), RD: CXXRD, Size))
452 return false;
453 } else {
454 if (!addFieldSizes(Context&: getContext(), RD, Size))
455 return false;
456 }
457
458 // We can do this if there was no alignment padding.
459 return Size == getContext().getTypeSize(T: Ty);
460}
461
462static bool shouldExtendReturnType(const ABIInfo &Info, QualType RetTy) {
463 // Bools need to be zero-extended to 8 bits for all ABIs.
464 if (RetTy->isBooleanType())
465 return true;
466
467 // Promotable integer types get extended only for Darwin.
468 if (!Info.getTarget().getTriple().isOSDarwin())
469 return false;
470
471 return Info.isPromotableIntegerTypeForABI(Ty: RetTy);
472}
473
474static CodeGenOptions::SmallIntABICompatKind
475getEffectiveSmallIntABICompat(const CodeGenOptions &Opts,
476 const llvm::Triple &T) {
477 if (T.isOSDarwin() || T.isPS())
478 return CodeGenOptions::SmallIntABICompatKind::Clang23;
479 return Opts.getSmallIntABICompat();
480}
481
482ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const {
483 // If the return value is indirect, then the hidden argument is consuming one
484 // integer register.
485 if (State.CC != llvm::CallingConv::X86_FastCall &&
486 State.CC != llvm::CallingConv::X86_VectorCall && State.FreeRegs) {
487 --State.FreeRegs;
488 if (!IsMCUABI)
489 return getNaturalAlignIndirectInReg(Ty: RetTy);
490 }
491 return getNaturalAlignIndirect(
492 Ty: RetTy, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
493 /*ByVal=*/false);
494}
495
496ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
497 CCState &State) const {
498 if (RetTy->isVoidType())
499 return ABIArgInfo::getIgnore();
500
501 const Type *Base = nullptr;
502 uint64_t NumElts = 0;
503 if ((State.CC == llvm::CallingConv::X86_VectorCall ||
504 State.CC == llvm::CallingConv::X86_RegCall) &&
505 isHomogeneousAggregate(Ty: RetTy, Base, Members&: NumElts)) {
506 // The LLVM struct type for such an aggregate should lower properly.
507 return ABIArgInfo::getDirect();
508 }
509
510 if (const VectorType *VT = RetTy->getAs<VectorType>()) {
511 // On Darwin, some vectors are returned in registers.
512 if (IsDarwinVectorABI) {
513 uint64_t Size = getContext().getTypeSize(T: RetTy);
514
515 // 128-bit vectors are a special case; they are returned in
516 // registers and we need to make sure to pick a type the LLVM
517 // backend will like.
518 if (Size == 128)
519 return ABIArgInfo::getDirect(T: llvm::FixedVectorType::get(
520 ElementType: llvm::Type::getInt64Ty(C&: getVMContext()), NumElts: 2));
521
522 // Always return in register if it fits in a general purpose
523 // register, or if it is 64 bits and has a single element.
524 if ((Size == 8 || Size == 16 || Size == 32) ||
525 (Size == 64 && VT->getNumElements() == 1))
526 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(),
527 NumBits: Size));
528
529 return getIndirectReturnResult(RetTy, State);
530 }
531
532 return ABIArgInfo::getDirect();
533 }
534
535 if (isAggregateTypeForABI(T: RetTy)) {
536 if (const auto *RD = RetTy->getAsRecordDecl();
537 RD && RD->hasFlexibleArrayMember())
538 // Structures with flexible arrays are always indirect.
539 return getIndirectReturnResult(RetTy, State);
540
541 // If specified, structs and unions are always indirect.
542 if (!IsRetSmallStructInRegABI && !RetTy->isAnyComplexType())
543 return getIndirectReturnResult(RetTy, State);
544
545 // Ignore empty structs/unions.
546 if (isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true))
547 return ABIArgInfo::getIgnore();
548
549 // Return complex of _Float16 as <2 x half> so the backend will use xmm0.
550 if (const ComplexType *CT = RetTy->getAs<ComplexType>()) {
551 QualType ET = getContext().getCanonicalType(T: CT->getElementType());
552 if (ET->isFloat16Type())
553 return ABIArgInfo::getDirect(T: llvm::FixedVectorType::get(
554 ElementType: llvm::Type::getHalfTy(C&: getVMContext()), NumElts: 2));
555 }
556
557 // Small structures which are register sized are generally returned
558 // in a register.
559 if (shouldReturnTypeInRegister(Ty: RetTy, Context&: getContext())) {
560 uint64_t Size = getContext().getTypeSize(T: RetTy);
561
562 // As a special-case, if the struct is a "single-element" struct, and
563 // the field is of type "float" or "double", return it in a
564 // floating-point register. (MSVC does not apply this special case.)
565 // We apply a similar transformation for pointer types to improve the
566 // quality of the generated IR.
567 if (const Type *SeltTy = isSingleElementRecord(T: RetTy, Context&: getContext()))
568 if ((!IsWin32StructABI && SeltTy->isRealFloatingType())
569 || SeltTy->hasPointerRepresentation())
570 return ABIArgInfo::getDirect(T: CGT.ConvertType(T: QualType(SeltTy, 0)));
571
572 // FIXME: We should be able to narrow this integer in cases with dead
573 // padding.
574 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(),NumBits: Size));
575 }
576
577 return getIndirectReturnResult(RetTy, State);
578 }
579
580 // Treat an enum type as its underlying type.
581 if (const auto *ED = RetTy->getAsEnumDecl())
582 RetTy = ED->getIntegerType();
583
584 if (const auto *EIT = RetTy->getAs<BitIntType>())
585 if (EIT->getNumBits() > 64)
586 return getIndirectReturnResult(RetTy, State);
587
588 return shouldExtendReturnType(Info: *this, RetTy) ? ABIArgInfo::getExtend(Ty: RetTy)
589 : ABIArgInfo::getDirect();
590}
591
592unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,
593 unsigned Align) const {
594 // Otherwise, if the alignment is less than or equal to the minimum ABI
595 // alignment, just use the default; the backend will handle this.
596 if (Align <= MinABIStackAlignInBytes)
597 return 0; // Use default alignment.
598
599 if (Ty->isFloat128Type())
600 return 16;
601
602 if (IsLinuxABI) {
603 // Exclude other System V OS (e.g Darwin, PS4 and FreeBSD) since we don't
604 // want to spend any effort dealing with the ramifications of ABI breaks.
605 //
606 // If the vector type is __m128/__m256/__m512, return the default alignment.
607 if (Ty->isVectorType() && (Align == 16 || Align == 32 || Align == 64))
608 return Align;
609 }
610 // On non-Darwin, the stack type alignment is always 4.
611 if (!IsDarwinVectorABI) {
612 // Set explicit alignment, since we may need to realign the top.
613 return MinABIStackAlignInBytes;
614 }
615
616 // Otherwise, if the type contains an SSE vector type, the alignment is 16.
617 if (Align >= 16 && (isSIMDVectorType(Context&: getContext(), Ty) ||
618 isRecordWithSIMDVectorType(Context&: getContext(), Ty)))
619 return 16;
620
621 return MinABIStackAlignInBytes;
622}
623
624ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal,
625 CCState &State) const {
626 if (!ByVal) {
627 if (State.FreeRegs) {
628 --State.FreeRegs; // Non-byval indirects just use one pointer.
629 if (!IsMCUABI)
630 return getNaturalAlignIndirectInReg(Ty);
631 }
632 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
633 ByVal: false);
634 }
635
636 // Compute the byval alignment.
637 unsigned TypeAlign = getContext().getTypeAlign(T: Ty) / 8;
638 unsigned StackAlign = getTypeStackAlignInBytes(Ty, Align: TypeAlign);
639 if (StackAlign == 0)
640 return ABIArgInfo::getIndirect(
641 Alignment: CharUnits::fromQuantity(Quantity: 4),
642 /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
643 /*ByVal=*/true);
644
645 // If the stack alignment is less than the type alignment, realign the
646 // argument.
647 bool Realign = TypeAlign > StackAlign;
648 return ABIArgInfo::getIndirect(
649 Alignment: CharUnits::fromQuantity(Quantity: StackAlign),
650 /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(), /*ByVal=*/true,
651 Realign);
652}
653
654/// Like isSingleElementRecord, but only looks through one level of records.
655/// This mirrors how GCC gives a record the mode of its only field.
656static const Type *getSingleFieldType(QualType T, ASTContext &Context) {
657 const auto *RD = T->getAsRecordDecl();
658 if (!RD)
659 return nullptr;
660
661 if (RD->hasFlexibleArrayMember())
662 return nullptr;
663
664 const Type *Found = nullptr;
665
666 // If this is a C++ record, check the bases first.
667 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
668 for (const auto &I : CXXRD->bases()) {
669 // Ignore empty records.
670 if (isEmptyRecord(Context, T: I.getType(), AllowArrays: true))
671 continue;
672
673 // If we already found an element then this isn't a single-element struct.
674 if (Found)
675 return nullptr;
676
677 Found = I.getType().getTypePtr();
678 }
679 }
680
681 // Check for single element.
682 for (const auto *FD : RD->fields()) {
683 QualType FT = FD->getType();
684
685 // Ignore empty fields.
686 if (isEmptyField(Context, FD, AllowArrays: true))
687 continue;
688
689 // If we already found an element then this isn't a single-element
690 // struct.
691 if (Found)
692 return nullptr;
693
694 // Treat single element arrays as the element.
695 while (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: FT)) {
696 if (AT->getZExtSize() != 1)
697 break;
698 FT = AT->getElementType();
699 }
700
701 Found = FT.getTypePtr();
702 }
703
704 // We don't consider a struct a single-element struct if it has
705 // padding beyond the element type.
706 if (Found && Context.getTypeSize(T: Found) != Context.getTypeSize(T))
707 return nullptr;
708
709 return Found;
710}
711
712X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {
713 if (getContext().getLangOpts().isCompatibleWith(
714 Version: LangOptions::ClangABI::Ver23)) {
715 const Type *T = isSingleElementRecord(T: Ty, Context&: getContext());
716 if (!T)
717 T = Ty.getTypePtr();
718
719 if (const BuiltinType *BT = T->getAs<BuiltinType>()) {
720 BuiltinType::Kind K = BT->getKind();
721 if (K == BuiltinType::Float || K == BuiltinType::Double)
722 return Float;
723 }
724 return Integer;
725 }
726
727 while (const RecordDecl *RD = Ty->getAsRecordDecl()) {
728 // Unions are always passed as integers.
729 if (RD->isUnion())
730 return Integer;
731
732 const Type *FT = getSingleFieldType(T: Ty, Context&: getContext());
733
734 // Pick integer for zero-sized types. They will be ignored down the line.
735 if (!FT)
736 return Integer;
737
738 Ty = QualType(FT, 0);
739 }
740
741 // All complex values are passed via the stack.
742 if (Ty->isAnyComplexType())
743 return AlwaysStack;
744
745 // Types like bfloat16 or __float128 are all passed like floats.
746 if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
747 if (BT->isFloatingPoint())
748 return Float;
749
750 return Integer;
751}
752
753bool X86_32ABIInfo::updateFreeRegs(QualType Ty, CCState &State) const {
754 Class C = classify(Ty);
755 if (C == AlwaysStack && !IsMCUABI)
756 return false;
757 if (C == Float && !IsSoftFloatABI)
758 return false;
759
760 unsigned Size = getContext().getTypeSize(T: Ty);
761 unsigned SizeInRegs = (Size + 31) / 32;
762
763 if (SizeInRegs == 0)
764 return false;
765
766 if (!IsMCUABI) {
767 if (SizeInRegs > State.FreeRegs) {
768 State.FreeRegs = 0;
769 return false;
770 }
771 } else {
772 // The MCU psABI allows passing parameters in-reg even if there are
773 // earlier parameters that are passed on the stack. Also,
774 // it does not allow passing >8-byte structs in-register,
775 // even if there are 3 free registers available.
776 if (SizeInRegs > State.FreeRegs || SizeInRegs > 2)
777 return false;
778 }
779
780 State.FreeRegs -= SizeInRegs;
781 return true;
782}
783
784bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State,
785 bool &InReg,
786 bool &NeedsPadding) const {
787 // On Windows, aggregates other than HFAs are never passed in registers, and
788 // they do not consume register slots. Homogenous floating-point aggregates
789 // (HFAs) have already been dealt with at this point.
790 if (IsWin32StructABI && isAggregateTypeForABI(T: Ty))
791 return false;
792
793 NeedsPadding = false;
794 InReg = !IsMCUABI;
795
796 if (!updateFreeRegs(Ty, State))
797 return false;
798
799 if (IsMCUABI)
800 return true;
801
802 if (State.CC == llvm::CallingConv::X86_FastCall ||
803 State.CC == llvm::CallingConv::X86_VectorCall ||
804 State.CC == llvm::CallingConv::X86_RegCall) {
805 if (getContext().getTypeSize(T: Ty) <= 32 && State.FreeRegs)
806 NeedsPadding = true;
807
808 return false;
809 }
810
811 return true;
812}
813
814bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {
815 bool IsPtrOrInt = (getContext().getTypeSize(T: Ty) <= 32) &&
816 (Ty->isIntegralOrEnumerationType() || Ty->isPointerType() ||
817 Ty->isReferenceType());
818
819 if (!IsPtrOrInt && (State.CC == llvm::CallingConv::X86_FastCall ||
820 State.CC == llvm::CallingConv::X86_VectorCall))
821 return false;
822
823 if (!updateFreeRegs(Ty, State))
824 return false;
825
826 if (!IsPtrOrInt && State.CC == llvm::CallingConv::X86_RegCall)
827 return false;
828
829 // Return true to apply inreg to all legal parameters except for MCU targets.
830 return !IsMCUABI;
831}
832
833void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {
834 // Vectorcall x86 works subtly different than in x64, so the format is
835 // a bit different than the x64 version. First, all vector types (not HVAs)
836 // are assigned, with the first 6 ending up in the [XYZ]MM0-5 registers.
837 // This differs from the x64 implementation, where the first 6 by INDEX get
838 // registers.
839 // In the second pass over the arguments, HVAs are passed in the remaining
840 // vector registers if possible, or indirectly by address. The address will be
841 // passed in ECX/EDX if available. Any other arguments are passed according to
842 // the usual fastcall rules.
843 MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
844 for (int I = 0, E = Args.size(); I < E; ++I) {
845 const Type *Base = nullptr;
846 uint64_t NumElts = 0;
847 const QualType &Ty = Args[I].type;
848 if ((Ty->isVectorType() || Ty->isBuiltinType()) &&
849 isHomogeneousAggregate(Ty, Base, Members&: NumElts)) {
850 if (State.FreeSSERegs >= NumElts) {
851 State.FreeSSERegs -= NumElts;
852 Args[I].info = ABIArgInfo::getDirectInReg();
853 State.IsPreassigned.set(I);
854 }
855 }
856 }
857}
858
859ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, CCState &State,
860 unsigned ArgIndex) const {
861 // FIXME: Set alignment on indirect arguments.
862 bool IsFastCall = State.CC == llvm::CallingConv::X86_FastCall;
863 bool IsRegCall = State.CC == llvm::CallingConv::X86_RegCall;
864 bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall;
865
866 Ty = useFirstFieldIfTransparentUnion(Ty);
867 TypeInfo TI = getContext().getTypeInfo(T: Ty);
868
869 // Check with the C++ ABI first.
870 const RecordType *RT = Ty->getAsCanonical<RecordType>();
871 if (RT) {
872 CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, CXXABI&: getCXXABI());
873 if (RAA == CGCXXABI::RAA_Indirect) {
874 return getIndirectResult(Ty, ByVal: false, State);
875 } else if (State.IsDelegateCall) {
876 // Avoid having different alignments on delegate call args by always
877 // setting the alignment to 4, which is what we do for inallocas.
878 ABIArgInfo Res = getIndirectResult(Ty, ByVal: false, State);
879 Res.setIndirectAlign(CharUnits::fromQuantity(Quantity: 4));
880 return Res;
881 } else if (RAA == CGCXXABI::RAA_DirectInMemory) {
882 // The field index doesn't matter, we'll fix it up later.
883 return ABIArgInfo::getInAlloca(/*FieldIndex=*/0);
884 }
885 }
886
887 // Regcall uses the concept of a homogenous vector aggregate, similar
888 // to other targets.
889 const Type *Base = nullptr;
890 uint64_t NumElts = 0;
891 if ((IsRegCall || IsVectorCall) &&
892 isHomogeneousAggregate(Ty, Base, Members&: NumElts)) {
893 if (State.FreeSSERegs >= NumElts) {
894 State.FreeSSERegs -= NumElts;
895
896 // Vectorcall passes HVAs directly and does not flatten them, but regcall
897 // does.
898 if (IsVectorCall)
899 return getDirectX86Hva();
900
901 if (Ty->isBuiltinType() || Ty->isVectorType())
902 return ABIArgInfo::getDirect();
903 return ABIArgInfo::getExpand();
904 }
905 if (IsVectorCall && Ty->isBuiltinType())
906 return ABIArgInfo::getDirect();
907 return getIndirectResult(Ty, /*ByVal=*/false, State);
908 }
909
910 if (isAggregateTypeForABI(T: Ty)) {
911 // Structures with flexible arrays are always indirect.
912 // FIXME: This should not be byval!
913 if (RT && RT->getDecl()->getDefinitionOrSelf()->hasFlexibleArrayMember())
914 return getIndirectResult(Ty, ByVal: true, State);
915
916 // Ignore empty structs/unions on non-Windows.
917 if (!IsWin32StructABI && isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
918 return ABIArgInfo::getIgnore();
919
920 // Ignore 0 sized structs.
921 if (TI.Width == 0)
922 return ABIArgInfo::getIgnore();
923
924 llvm::LLVMContext &LLVMContext = getVMContext();
925 llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(C&: LLVMContext);
926 bool NeedsPadding = false;
927 bool InReg;
928 if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) {
929 unsigned SizeInRegs = (TI.Width + 31) / 32;
930 SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32);
931 llvm::Type *Result = llvm::StructType::get(Context&: LLVMContext, Elements);
932 if (InReg)
933 return ABIArgInfo::getDirectInReg(T: Result);
934 else
935 return ABIArgInfo::getDirect(T: Result);
936 }
937 llvm::IntegerType *PaddingType = NeedsPadding ? Int32 : nullptr;
938
939 // Pass over-aligned aggregates to non-variadic functions on Windows
940 // indirectly. This behavior was added in MSVC 2015. Use the required
941 // alignment from the record layout, since that may be less than the
942 // regular type alignment, and types with required alignment of less than 4
943 // bytes are not passed indirectly.
944 if (IsWin32StructABI && State.Required.isRequiredArg(argIdx: ArgIndex)) {
945 unsigned AlignInBits = 0;
946 if (RT) {
947 const ASTRecordLayout &Layout =
948 getContext().getASTRecordLayout(D: RT->getDecl());
949 AlignInBits = getContext().toBits(CharSize: Layout.getRequiredAlignment());
950 } else if (TI.isAlignRequired()) {
951 AlignInBits = TI.Align;
952 }
953 if (AlignInBits > 32)
954 return getIndirectResult(Ty, /*ByVal=*/false, State);
955 }
956
957 // Expand small (<= 128-bit) record types when we know that the stack layout
958 // of those arguments will match the struct. This is important because the
959 // LLVM backend isn't smart enough to remove byval, which inhibits many
960 // optimizations.
961 // Don't do this for the MCU if there are still free integer registers
962 // (see X86_64 ABI for full explanation).
963 if (TI.Width <= 4 * 32 && (!IsMCUABI || State.FreeRegs == 0) &&
964 canExpandIndirectArgument(Ty))
965 return ABIArgInfo::getExpandWithPadding(
966 PaddingInReg: IsFastCall || IsVectorCall || IsRegCall, Padding: PaddingType);
967
968 return getIndirectResult(Ty, ByVal: true, State);
969 }
970
971 if (const VectorType *VT = Ty->getAs<VectorType>()) {
972 // On Windows, vectors are passed directly if registers are available, or
973 // indirectly if not. This avoids the need to align argument memory. Pass
974 // user-defined vector types larger than 512 bits indirectly for simplicity.
975 if (IsWin32StructABI) {
976 if (TI.Width <= 512 && State.FreeSSERegs > 0) {
977 --State.FreeSSERegs;
978 return ABIArgInfo::getDirectInReg();
979 }
980 return getIndirectResult(Ty, /*ByVal=*/false, State);
981 }
982
983 // On Darwin, some vectors are passed in memory, we handle this by passing
984 // it as an i8/i16/i32/i64.
985 if (IsDarwinVectorABI) {
986 if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) ||
987 (TI.Width == 64 && VT->getNumElements() == 1))
988 return ABIArgInfo::getDirect(
989 T: llvm::IntegerType::get(C&: getVMContext(), NumBits: TI.Width));
990 }
991
992 if (IsX86_MMXType(IRType: CGT.ConvertType(T: Ty)))
993 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(), NumBits: 64));
994
995 return ABIArgInfo::getDirect();
996 }
997
998 if (const auto *ED = Ty->getAsEnumDecl())
999 Ty = ED->getIntegerType();
1000
1001 bool InReg = shouldPrimitiveUseInReg(Ty, State);
1002
1003 if (isPromotableIntegerTypeForABI(Ty)) {
1004 switch (getEffectiveSmallIntABICompat(Opts: getCodeGenOpts(),
1005 T: getTarget().getTriple())) {
1006 case CodeGenOptions::SmallIntABICompatKind::Platform:
1007 if (Ty->isBooleanType()) {
1008 // Bool should be extended to 8-bits. We can't express this right now,
1009 // so use conservative extension.
1010 return InReg
1011 ? ABIArgInfo::getConservativeExtendInReg(Ty,
1012 T: CGT.ConvertType(T: Ty))
1013 : ABIArgInfo::getConservativeExtend(Ty, T: CGT.ConvertType(T: Ty));
1014 }
1015 break;
1016 case CodeGenOptions::SmallIntABICompatKind::Clang23:
1017 return InReg ? ABIArgInfo::getExtendInReg(Ty, T: CGT.ConvertType(T: Ty))
1018 : ABIArgInfo::getExtend(Ty, T: CGT.ConvertType(T: Ty));
1019 case CodeGenOptions::SmallIntABICompatKind::Both:
1020 return InReg ? ABIArgInfo::getConservativeExtendInReg(Ty,
1021 T: CGT.ConvertType(T: Ty))
1022 : ABIArgInfo::getConservativeExtend(Ty, T: CGT.ConvertType(T: Ty));
1023 }
1024 }
1025
1026 if (const auto *EIT = Ty->getAs<BitIntType>()) {
1027 if (EIT->getNumBits() <= 64) {
1028 if (InReg)
1029 return ABIArgInfo::getDirectInReg();
1030 return ABIArgInfo::getDirect();
1031 }
1032 return getIndirectResult(Ty, /*ByVal=*/false, State);
1033 }
1034
1035 if (InReg)
1036 return ABIArgInfo::getDirectInReg();
1037 return ABIArgInfo::getDirect();
1038}
1039
1040void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
1041 CCState State(FI);
1042 if (IsMCUABI)
1043 State.FreeRegs = 3;
1044 else if (State.CC == llvm::CallingConv::X86_FastCall) {
1045 State.FreeRegs = 2;
1046 State.FreeSSERegs = 3;
1047 } else if (State.CC == llvm::CallingConv::X86_VectorCall) {
1048 State.FreeRegs = 2;
1049 State.FreeSSERegs = 6;
1050 } else if (FI.getHasRegParm())
1051 State.FreeRegs = FI.getRegParm();
1052 else if (State.CC == llvm::CallingConv::X86_RegCall) {
1053 State.FreeRegs = 5;
1054 State.FreeSSERegs = 8;
1055 } else if (IsWin32StructABI) {
1056 // Since MSVC 2015, the first three SSE vectors have been passed in
1057 // registers. The rest are passed indirectly.
1058 State.FreeRegs = DefaultNumRegisterParameters;
1059 State.FreeSSERegs = 3;
1060 } else
1061 State.FreeRegs = DefaultNumRegisterParameters;
1062
1063 if (!::classifyReturnType(CXXABI: getCXXABI(), FI, Info: *this)) {
1064 FI.getReturnInfo() = classifyReturnType(RetTy: FI.getReturnType(), State);
1065 } else if (FI.getReturnInfo().isIndirect()) {
1066 // The C++ ABI is not aware of register usage, so we have to check if the
1067 // return value was sret and put it in a register ourselves if appropriate.
1068 if (State.FreeRegs) {
1069 --State.FreeRegs; // The sret parameter consumes a register.
1070 if (!IsMCUABI)
1071 FI.getReturnInfo().setInReg(true);
1072 }
1073 }
1074
1075 // The chain argument effectively gives us another free register.
1076 if (FI.isChainCall())
1077 ++State.FreeRegs;
1078
1079 // For vectorcall, do a first pass over the arguments, assigning FP and vector
1080 // arguments to XMM registers as available.
1081 if (State.CC == llvm::CallingConv::X86_VectorCall)
1082 runVectorCallFirstPass(FI, State);
1083
1084 bool UsedInAlloca = false;
1085 MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
1086 for (unsigned I = 0, E = Args.size(); I < E; ++I) {
1087 // Skip arguments that have already been assigned.
1088 if (State.IsPreassigned.test(Idx: I))
1089 continue;
1090
1091 Args[I].info =
1092 classifyArgumentType(Ty: Args[I].type, State, ArgIndex: I);
1093 UsedInAlloca |= (Args[I].info.getKind() == ABIArgInfo::InAlloca);
1094 }
1095
1096 // If we needed to use inalloca for any argument, do a second pass and rewrite
1097 // all the memory arguments to use inalloca.
1098 if (UsedInAlloca)
1099 rewriteWithInAlloca(FI);
1100}
1101
1102void
1103X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
1104 CharUnits &StackOffset, ABIArgInfo &Info,
1105 QualType Type) const {
1106 // Arguments are always 4-byte-aligned.
1107 CharUnits WordSize = CharUnits::fromQuantity(Quantity: 4);
1108 assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct");
1109
1110 // sret pointers and indirect things will require an extra pointer
1111 // indirection, unless they are byval. Most things are byval, and will not
1112 // require this indirection.
1113 bool IsIndirect = false;
1114 if (Info.isIndirect() && !Info.getIndirectByVal())
1115 IsIndirect = true;
1116 Info = ABIArgInfo::getInAlloca(FieldIndex: FrameFields.size(), Indirect: IsIndirect);
1117 llvm::Type *LLTy = CGT.ConvertTypeForMem(T: Type);
1118 if (IsIndirect)
1119 LLTy = llvm::PointerType::getUnqual(C&: getVMContext());
1120 FrameFields.push_back(Elt: LLTy);
1121 StackOffset += IsIndirect ? WordSize : getContext().getTypeSizeInChars(T: Type);
1122
1123 // Insert padding bytes to respect alignment.
1124 CharUnits FieldEnd = StackOffset;
1125 StackOffset = FieldEnd.alignTo(Align: WordSize);
1126 if (StackOffset != FieldEnd) {
1127 CharUnits NumBytes = StackOffset - FieldEnd;
1128 llvm::Type *Ty = llvm::Type::getInt8Ty(C&: getVMContext());
1129 Ty = llvm::ArrayType::get(ElementType: Ty, NumElements: NumBytes.getQuantity());
1130 FrameFields.push_back(Elt: Ty);
1131 }
1132}
1133
1134static bool isArgInAlloca(const ABIArgInfo &Info) {
1135 // Leave ignored and inreg arguments alone.
1136 switch (Info.getKind()) {
1137 case ABIArgInfo::InAlloca:
1138 return true;
1139 case ABIArgInfo::Ignore:
1140 case ABIArgInfo::IndirectAliased:
1141 case ABIArgInfo::TargetSpecific:
1142 return false;
1143 case ABIArgInfo::Indirect:
1144 case ABIArgInfo::Direct:
1145 case ABIArgInfo::Extend:
1146 return !Info.getInReg();
1147 case ABIArgInfo::Expand:
1148 case ABIArgInfo::CoerceAndExpand:
1149 // These are aggregate types which are never passed in registers when
1150 // inalloca is involved.
1151 return true;
1152 }
1153 llvm_unreachable("invalid enum");
1154}
1155
1156void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const {
1157 assert(IsWin32StructABI && "inalloca only supported on win32");
1158
1159 // Build a packed struct type for all of the arguments in memory.
1160 SmallVector<llvm::Type *, 6> FrameFields;
1161
1162 // The stack alignment is always 4.
1163 CharUnits StackAlign = CharUnits::fromQuantity(Quantity: 4);
1164
1165 CharUnits StackOffset;
1166 CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end();
1167
1168 // Put 'this' into the struct before 'sret', if necessary.
1169 bool IsThisCall =
1170 FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall;
1171 ABIArgInfo &Ret = FI.getReturnInfo();
1172 if (Ret.isIndirect() && Ret.isSRetAfterThis() && !IsThisCall &&
1173 isArgInAlloca(Info: I->info)) {
1174 addFieldToArgStruct(FrameFields, StackOffset, Info&: I->info, Type: I->type);
1175 ++I;
1176 }
1177
1178 // Put the sret parameter into the inalloca struct if it's in memory.
1179 if (Ret.isIndirect() && !Ret.getInReg()) {
1180 addFieldToArgStruct(FrameFields, StackOffset, Info&: Ret, Type: FI.getReturnType());
1181 // On Windows, the hidden sret parameter is always returned in eax.
1182 Ret.setInAllocaSRet(IsWin32StructABI);
1183 }
1184
1185 // Skip the 'this' parameter in ecx.
1186 if (IsThisCall)
1187 ++I;
1188
1189 // Put arguments passed in memory into the struct.
1190 for (; I != E; ++I) {
1191 if (isArgInAlloca(Info: I->info))
1192 addFieldToArgStruct(FrameFields, StackOffset, Info&: I->info, Type: I->type);
1193 }
1194
1195 FI.setArgStruct(Ty: llvm::StructType::get(Context&: getVMContext(), Elements: FrameFields,
1196 /*isPacked=*/true),
1197 Align: StackAlign);
1198}
1199
1200RValue X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1201 QualType Ty, AggValueSlot Slot) const {
1202
1203 auto TypeInfo = getContext().getTypeInfoInChars(T: Ty);
1204
1205 CCState State(*const_cast<CGFunctionInfo *>(CGF.CurFnInfo));
1206 ABIArgInfo AI = classifyArgumentType(Ty, State, /*ArgIndex*/ 0);
1207 // Empty records are ignored for parameter passing purposes.
1208 if (AI.isIgnore())
1209 return Slot.asRValue();
1210
1211 // x86-32 changes the alignment of certain arguments on the stack.
1212 //
1213 // Just messing with TypeInfo like this works because we never pass
1214 // anything indirectly.
1215 TypeInfo.Align = CharUnits::fromQuantity(
1216 Quantity: getTypeStackAlignInBytes(Ty, Align: TypeInfo.Align.getQuantity()));
1217
1218 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, /*Indirect*/ IsIndirect: false, ValueInfo: TypeInfo,
1219 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 4),
1220 /*AllowHigherAlign*/ true, Slot);
1221}
1222
1223bool X86_32TargetCodeGenInfo::isStructReturnInRegABI(
1224 const llvm::Triple &Triple, const CodeGenOptions &Opts) {
1225 assert(Triple.getArch() == llvm::Triple::x86);
1226
1227 switch (Opts.getStructReturnConvention()) {
1228 case CodeGenOptions::SRCK_Default:
1229 break;
1230 case CodeGenOptions::SRCK_OnStack: // -fpcc-struct-return
1231 return false;
1232 case CodeGenOptions::SRCK_InRegs: // -freg-struct-return
1233 return true;
1234 }
1235
1236 if (Triple.isOSDarwin() || Triple.isOSIAMCU())
1237 return true;
1238
1239 switch (Triple.getOS()) {
1240 case llvm::Triple::DragonFly:
1241 case llvm::Triple::FreeBSD:
1242 case llvm::Triple::OpenBSD:
1243 case llvm::Triple::Win32:
1244 return true;
1245 default:
1246 return false;
1247 }
1248}
1249
1250static void addX86InterruptAttrs(const FunctionDecl *FD, llvm::GlobalValue *GV,
1251 CodeGen::CodeGenModule &CGM) {
1252 if (!FD->hasAttr<AnyX86InterruptAttr>())
1253 return;
1254
1255 llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1256 Fn->setCallingConv(llvm::CallingConv::X86_INTR);
1257 if (FD->getNumParams() == 0)
1258 return;
1259
1260 const auto *PtrTy = FD->getParamDecl(i: 0)->getType()->castAs<PointerType>();
1261 llvm::Type *ByValTy = CGM.getTypes().ConvertType(T: PtrTy->getPointeeType());
1262 llvm::Attribute NewAttr = llvm::Attribute::getWithByValType(
1263 Context&: Fn->getContext(), Ty: ByValTy);
1264 Fn->addParamAttr(ArgNo: 0, Attr: NewAttr);
1265}
1266
1267void X86_32TargetCodeGenInfo::setTargetAttributes(
1268 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
1269 if (GV->isDeclaration())
1270 return;
1271 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D)) {
1272 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
1273 llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1274 Fn->addFnAttr(Kind: "stackrealign");
1275 }
1276
1277 addX86InterruptAttrs(FD, GV, CGM);
1278 }
1279}
1280
1281bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
1282 CodeGen::CodeGenFunction &CGF,
1283 llvm::Value *Address) const {
1284 CodeGen::CGBuilderTy &Builder = CGF.Builder;
1285
1286 llvm::Value *Four8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 4);
1287
1288 // 0-7 are the eight integer registers; the order is different
1289 // on Darwin (for EH), but the range is the same.
1290 // 8 is %eip.
1291 AssignToArrayRange(Builder, Array: Address, Value: Four8, FirstIndex: 0, LastIndex: 8);
1292
1293 if (CGF.CGM.getTarget().getTriple().isOSDarwin()) {
1294 // 12-16 are st(0..4). Not sure why we stop at 4.
1295 // These have size 16, which is sizeof(long double) on
1296 // platforms with 8-byte alignment for that type.
1297 llvm::Value *Sixteen8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 16);
1298 AssignToArrayRange(Builder, Array: Address, Value: Sixteen8, FirstIndex: 12, LastIndex: 16);
1299
1300 } else {
1301 // 9 is %eflags, which doesn't get a size on Darwin for some
1302 // reason.
1303 Builder.CreateAlignedStore(
1304 Val: Four8, Addr: Builder.CreateConstInBoundsGEP1_32(Ty: CGF.Int8Ty, Ptr: Address, Idx0: 9),
1305 Align: CharUnits::One());
1306
1307 // 11-16 are st(0..5). Not sure why we stop at 5.
1308 // These have size 12, which is sizeof(long double) on
1309 // platforms with 4-byte alignment for that type.
1310 llvm::Value *Twelve8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 12);
1311 AssignToArrayRange(Builder, Array: Address, Value: Twelve8, FirstIndex: 11, LastIndex: 16);
1312 }
1313
1314 return false;
1315}
1316
1317//===----------------------------------------------------------------------===//
1318// X86-64 ABI Implementation
1319//===----------------------------------------------------------------------===//
1320
1321
1322namespace {
1323
1324/// \p returns the size in bits of the largest (native) vector for \p AVXLevel.
1325static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
1326 switch (AVXLevel) {
1327 case X86AVXABILevel::AVX512:
1328 return 512;
1329 case X86AVXABILevel::AVX:
1330 return 256;
1331 case X86AVXABILevel::None:
1332 return 128;
1333 }
1334 llvm_unreachable("Unknown AVXLevel");
1335}
1336
1337/// X86_64ABIInfo - The X86_64 ABI information.
1338class X86_64ABIInfo : public ABIInfo {
1339 enum Class {
1340 Integer = 0,
1341 SSE,
1342 SSEUp,
1343 X87,
1344 X87Up,
1345 ComplexX87,
1346 NoClass,
1347 Memory
1348 };
1349
1350 /// merge - Implement the X86_64 ABI merging algorithm.
1351 ///
1352 /// Merge an accumulating classification \arg Accum with a field
1353 /// classification \arg Field.
1354 ///
1355 /// \param Accum - The accumulating classification. This should
1356 /// always be either NoClass or the result of a previous merge
1357 /// call. In addition, this should never be Memory (the caller
1358 /// should just return Memory for the aggregate).
1359 static Class merge(Class Accum, Class Field);
1360
1361 /// postMerge - Implement the X86_64 ABI post merging algorithm.
1362 ///
1363 /// Post merger cleanup, reduces a malformed Hi and Lo pair to
1364 /// final MEMORY or SSE classes when necessary.
1365 ///
1366 /// \param AggregateSize - The size of the current aggregate in
1367 /// the classification process.
1368 ///
1369 /// \param Lo - The classification for the parts of the type
1370 /// residing in the low word of the containing object.
1371 ///
1372 /// \param Hi - The classification for the parts of the type
1373 /// residing in the higher words of the containing object.
1374 ///
1375 void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
1376
1377 /// classify - Determine the x86_64 register classes in which the
1378 /// given type T should be passed.
1379 ///
1380 /// \param Lo - The classification for the parts of the type
1381 /// residing in the low word of the containing object.
1382 ///
1383 /// \param Hi - The classification for the parts of the type
1384 /// residing in the high word of the containing object.
1385 ///
1386 /// \param OffsetBase - The bit offset of this type in the
1387 /// containing object. Some parameters are classified different
1388 /// depending on whether they straddle an eightbyte boundary.
1389 ///
1390 /// \param isNamedArg - Whether the argument in question is a "named"
1391 /// argument, as used in AMD64-ABI 3.5.7.
1392 ///
1393 /// \param IsRegCall - Whether the calling conversion is regcall.
1394 ///
1395 /// If a word is unused its result will be NoClass; if a type should
1396 /// be passed in Memory then at least the classification of \arg Lo
1397 /// will be Memory.
1398 ///
1399 /// The \arg Lo class will be NoClass iff the argument is ignored.
1400 ///
1401 /// If the \arg Lo class is ComplexX87, then the \arg Hi class will
1402 /// also be ComplexX87.
1403 void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,
1404 bool isNamedArg, bool IsRegCall = false) const;
1405
1406 llvm::Type *GetByteVectorType(QualType Ty) const;
1407 llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
1408 unsigned IROffset, QualType SourceTy,
1409 unsigned SourceOffset) const;
1410 llvm::Type *GetINTEGERTypeAtOffset(llvm::Type *IRType,
1411 unsigned IROffset, QualType SourceTy,
1412 unsigned SourceOffset) const;
1413
1414 /// getIndirectResult - Give a source type \arg Ty, return a suitable result
1415 /// such that the argument will be returned in memory.
1416 ABIArgInfo getIndirectReturnResult(QualType Ty) const;
1417
1418 /// getIndirectResult - Give a source type \arg Ty, return a suitable result
1419 /// such that the argument will be passed in memory.
1420 ///
1421 /// \param freeIntRegs - The number of free integer registers remaining
1422 /// available.
1423 ABIArgInfo getIndirectResult(QualType Ty, unsigned freeIntRegs) const;
1424
1425 ABIArgInfo classifyReturnType(QualType RetTy) const;
1426
1427 ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs,
1428 unsigned &neededInt, unsigned &neededSSE,
1429 bool isNamedArg,
1430 bool IsRegCall = false) const;
1431
1432 ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
1433 unsigned &NeededSSE,
1434 unsigned &MaxVectorWidth) const;
1435
1436 bool passRegCallStructTypeDirectly(QualType Ty,
1437 SmallVectorImpl<llvm::Type *> &CoerceElts,
1438 unsigned &NeededInt, unsigned &NeededSSE,
1439 unsigned &MaxVectorWidth) const;
1440
1441 bool IsIllegalVectorType(QualType Ty) const;
1442
1443 /// The 0.98 ABI revision clarified a lot of ambiguities,
1444 /// unfortunately in ways that were not always consistent with
1445 /// certain previous compilers. In particular, platforms which
1446 /// required strict binary compatibility with older versions of GCC
1447 /// may need to exempt themselves.
1448 bool honorsRevision0_98() const {
1449 return !getTarget().getTriple().isOSDarwin();
1450 }
1451
1452 /// GCC classifies <1 x long long> as SSE but some platform ABIs choose to
1453 /// classify it as INTEGER (for compatibility with older clang compilers).
1454 bool classifyIntegerMMXAsSSE() const {
1455 // Clang <= 3.8 did not do this.
1456 if (getContext().getLangOpts().isCompatibleWith(
1457 Version: LangOptions::ClangABI::Ver3_8))
1458 return false;
1459
1460 const llvm::Triple &Triple = getTarget().getTriple();
1461 if (Triple.isOSDarwin() || Triple.isPS() || Triple.isOSFreeBSD())
1462 return false;
1463 return true;
1464 }
1465
1466 // GCC classifies vectors of __int128 as memory.
1467 bool passInt128VectorsInMem() const {
1468 // Clang <= 9.0 did not do this.
1469 if (getContext().getLangOpts().isCompatibleWith(
1470 Version: LangOptions::ClangABI::Ver9))
1471 return false;
1472
1473 const llvm::Triple &T = getTarget().getTriple();
1474 return T.isOSLinux() || T.isOSNetBSD();
1475 }
1476
1477 bool returnCXXRecordGreaterThan128InMem() const {
1478 // Clang <= 20.0 did not do this, and PlayStation does not do this.
1479 if (getContext().getLangOpts().isCompatibleWith(
1480 Version: LangOptions::ClangABI::Ver20) ||
1481 getTarget().getTriple().isPS())
1482 return false;
1483
1484 return true;
1485 }
1486
1487 X86AVXABILevel AVXLevel;
1488 // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
1489 // 64-bit hardware.
1490 bool Has64BitPointers;
1491
1492public:
1493 X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
1494 : ABIInfo(CGT), AVXLevel(AVXLevel),
1495 Has64BitPointers(CGT.getDataLayout().getPointerSize(AS: 0) == 8) {}
1496
1497 bool isPassedUsingAVXType(QualType type) const {
1498 unsigned neededInt, neededSSE;
1499 // The freeIntRegs argument doesn't matter here.
1500 ABIArgInfo info = classifyArgumentType(Ty: type, freeIntRegs: 0, neededInt, neededSSE,
1501 /*isNamedArg*/true);
1502 if (info.isDirect()) {
1503 llvm::Type *ty = info.getCoerceToType();
1504 if (llvm::VectorType *vectorTy = dyn_cast_or_null<llvm::VectorType>(Val: ty))
1505 return vectorTy->getPrimitiveSizeInBits().getFixedValue() > 128;
1506 }
1507 return false;
1508 }
1509
1510 void computeInfo(CGFunctionInfo &FI) const override;
1511 unsigned getX86ABIAVXLevel(const FunctionDecl *FD,
1512 const FunctionType::ExtInfo &Info) const override;
1513
1514 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
1515 AggValueSlot Slot) const override;
1516 RValue EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
1517 AggValueSlot Slot) const override;
1518
1519 bool has64BitPointers() const {
1520 return Has64BitPointers;
1521 }
1522};
1523
1524/// WinX86_64ABIInfo - The Windows X86_64 ABI information.
1525class WinX86_64ABIInfo : public ABIInfo {
1526public:
1527 WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
1528 : ABIInfo(CGT), AVXLevel(AVXLevel),
1529 IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {}
1530
1531 void computeInfo(CGFunctionInfo &FI) const override;
1532 unsigned getX86ABIAVXLevel(const FunctionDecl *FD,
1533 const FunctionType::ExtInfo &Info) const override;
1534
1535 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
1536 AggValueSlot Slot) const override;
1537
1538 bool isHomogeneousAggregateBaseType(QualType Ty) const override {
1539 // FIXME: Assumes vectorcall is in use.
1540 return isX86VectorTypeForVectorCall(Context&: getContext(), Ty);
1541 }
1542
1543 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
1544 uint64_t NumMembers) const override {
1545 // FIXME: Assumes vectorcall is in use.
1546 return isX86VectorCallAggregateSmallEnough(NumMembers);
1547 }
1548
1549 ABIArgInfo classifyArgForArm64ECVarArg(QualType Ty,
1550 bool IsNamedArg) const override {
1551 unsigned FreeSSERegs = 0;
1552 ClassifyKind Kind =
1553 IsNamedArg ? ClassifyKind::FixedArgument : ClassifyKind::VarArg;
1554 return classify(Ty, FreeSSERegs, Kind, CC: llvm::CallingConv::C);
1555 }
1556
1557private:
1558 enum class ClassifyKind { Return, FixedArgument, VarArg };
1559
1560 ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, ClassifyKind Kind,
1561 unsigned CC) const;
1562 ABIArgInfo reclassifyHvaArgForVectorCall(QualType Ty, unsigned &FreeSSERegs,
1563 const ABIArgInfo &current) const;
1564
1565 X86AVXABILevel AVXLevel;
1566
1567 bool IsMingw64;
1568};
1569
1570class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
1571public:
1572 X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
1573 : TargetCodeGenInfo(std::make_unique<X86_64ABIInfo>(args&: CGT, args&: AVXLevel)) {
1574 SwiftInfo =
1575 std::make_unique<SwiftABIInfo>(args&: CGT, /*SwiftErrorInRegister=*/args: true);
1576 }
1577
1578 /// Disable tail call on x86-64. The epilogue code before the tail jump blocks
1579 /// autoreleaseRV/retainRV and autoreleaseRV/unsafeClaimRV optimizations.
1580 bool markARCOptimizedReturnCallsAsNoTail() const override { return true; }
1581
1582 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1583 return 7;
1584 }
1585
1586 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1587 llvm::Value *Address) const override {
1588 llvm::Value *Eight8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 8);
1589
1590 // 0-15 are the 16 integer registers.
1591 // 16 is %rip.
1592 AssignToArrayRange(Builder&: CGF.Builder, Array: Address, Value: Eight8, FirstIndex: 0, LastIndex: 16);
1593 return false;
1594 }
1595
1596 llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1597 StringRef Constraint,
1598 llvm::Type* Ty) const override {
1599 return X86AdjustInlineAsmType(CGF, Constraint, Ty);
1600 }
1601
1602 bool isNoProtoCallVariadic(const CallArgList &args,
1603 const FunctionNoProtoType *fnType) const override {
1604 // The default CC on x86-64 sets %al to the number of SSA
1605 // registers used, and GCC sets this when calling an unprototyped
1606 // function, so we override the default behavior. However, don't do
1607 // that when AVX types are involved: the ABI explicitly states it is
1608 // undefined, and it doesn't work in practice because of how the ABI
1609 // defines varargs anyway.
1610 if (fnType->getCallConv() == CC_C) {
1611 bool HasAVXType = false;
1612 for (const CallArg &arg : args) {
1613 if (getABIInfo<X86_64ABIInfo>().isPassedUsingAVXType(type: arg.Ty)) {
1614 HasAVXType = true;
1615 break;
1616 }
1617 }
1618
1619 if (!HasAVXType)
1620 return true;
1621 }
1622
1623 return TargetCodeGenInfo::isNoProtoCallVariadic(args, fnType);
1624 }
1625
1626 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1627 CodeGen::CodeGenModule &CGM) const override {
1628 if (GV->isDeclaration())
1629 return;
1630 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D)) {
1631 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
1632 llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1633 Fn->addFnAttr(Kind: "stackrealign");
1634 }
1635
1636 addX86InterruptAttrs(FD, GV, CGM);
1637 }
1638 }
1639
1640 void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
1641 const FunctionDecl *Caller,
1642 const FunctionDecl *Callee, const CallArgList &Args,
1643 QualType ReturnType) const override;
1644
1645 void checkFunctionABI(CodeGenModule &CGM,
1646 const FunctionDecl *FD) const override;
1647};
1648} // namespace
1649
1650static void initFeatureMaps(const ASTContext &Ctx,
1651 llvm::StringMap<bool> &CallerMap,
1652 const FunctionDecl *Caller,
1653 llvm::StringMap<bool> &CalleeMap,
1654 const FunctionDecl *Callee) {
1655 if (CalleeMap.empty() && CallerMap.empty()) {
1656 // The caller is potentially nullptr in the case where the call isn't in a
1657 // function. In this case, the getFunctionFeatureMap ensures we just get
1658 // the TU level setting (since it cannot be modified by 'target'..
1659 Ctx.getFunctionFeatureMap(FeatureMap&: CallerMap, Caller);
1660 Ctx.getFunctionFeatureMap(FeatureMap&: CalleeMap, Callee);
1661 }
1662}
1663
1664static bool checkAVXParamFeature(DiagnosticsEngine &Diag,
1665 SourceLocation CallLoc,
1666 const FunctionDecl &Callee,
1667 const llvm::StringMap<bool> &CallerMap,
1668 const llvm::StringMap<bool> &CalleeMap,
1669 QualType Ty, StringRef Feature,
1670 bool IsArgument) {
1671 bool CallerHasFeat = CallerMap.lookup(Key: Feature);
1672 bool CalleeHasFeat = CalleeMap.lookup(Key: Feature);
1673 // No explicit features and the function is internal, be permissive.
1674 if (!CallerHasFeat && !CalleeHasFeat &&
1675 (!Callee.isExternallyVisible() || Callee.hasAttr<AlwaysInlineAttr>()))
1676 return false;
1677
1678 if (!CallerHasFeat && !CalleeHasFeat)
1679 return Diag.Report(Loc: CallLoc, DiagID: diag::warn_avx_calling_convention)
1680 << IsArgument << Ty << Feature;
1681
1682 // Mixing calling conventions here is very clearly an error.
1683 if (!CallerHasFeat || !CalleeHasFeat)
1684 return Diag.Report(Loc: CallLoc, DiagID: diag::err_avx_calling_convention)
1685 << IsArgument << Ty << Feature;
1686
1687 // Else, both caller and callee have the required feature, so there is no need
1688 // to diagnose.
1689 return false;
1690}
1691
1692static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,
1693 SourceLocation CallLoc, const FunctionDecl &Callee,
1694 const llvm::StringMap<bool> &CallerMap,
1695 const llvm::StringMap<bool> &CalleeMap, QualType Ty,
1696 bool IsArgument) {
1697 uint64_t Size = Ctx.getTypeSize(T: Ty);
1698 if (Size > 256)
1699 return checkAVXParamFeature(Diag, CallLoc, Callee, CallerMap, CalleeMap, Ty,
1700 Feature: "avx512f", IsArgument);
1701
1702 if (Size > 128)
1703 return checkAVXParamFeature(Diag, CallLoc, Callee, CallerMap, CalleeMap, Ty,
1704 Feature: "avx", IsArgument);
1705
1706 return false;
1707}
1708
1709void X86_64TargetCodeGenInfo::checkFunctionABI(CodeGenModule &CGM,
1710 const FunctionDecl *FD) const {
1711 auto GetReturnTypeLoc = [](const FunctionDecl *FD) {
1712 if (const TypeSourceInfo *TSI = FD->getTypeSourceInfo()) {
1713 TypeLoc TL = TSI->getTypeLoc();
1714
1715 if (auto FTL = TL.IgnoreParens().getAs<FunctionTypeLoc>()) {
1716 SourceLocation Loc = FTL.getReturnLoc().getBeginLoc();
1717 if (Loc.isValid())
1718 return Loc;
1719 }
1720 }
1721
1722 SourceLocation Loc = FD->getLocation();
1723 if (Loc.isValid())
1724 return Loc;
1725
1726 return FD->getBeginLoc();
1727 };
1728
1729 auto Check = [&](QualType Ty, SourceLocation Loc, bool IsReturn) {
1730 if (!Ty->isVectorType())
1731 return false;
1732 if (CGM.getContext().getTypeSize(T: Ty) <= 128)
1733 return false;
1734
1735 StringRef Feature =
1736 CGM.getContext().getTypeSize(T: Ty) > 256 ? "avx512f" : "avx";
1737
1738 llvm::StringMap<bool> FeatureMap;
1739 CGM.getContext().getFunctionFeatureMap(FeatureMap, FD);
1740 if (!FeatureMap.lookup(Key: Feature)) {
1741 CGM.getDiags().Report(Loc, DiagID: diag::warn_avx_calling_convention)
1742 << !IsReturn << Ty << Feature;
1743 return true;
1744 }
1745
1746 return false;
1747 };
1748
1749 // psABI warnings & errors for function definitions that are only visible
1750 // in this translation unit are handled at call site by checkFunctionCallABI.
1751 if (!FD->isExternallyVisible())
1752 return;
1753
1754 // First check the return type and emit diagnostic if required.
1755 Check(FD->getReturnType(), GetReturnTypeLoc(FD), true);
1756
1757 // Go through the parameters and emit a warning for the first vector found
1758 // without the matching function AVX level attribute.
1759 for (const ParmVarDecl *P : FD->parameters()) {
1760 SourceLocation Loc = P->getLocation();
1761 if (Loc.isInvalid())
1762 Loc = P->getBeginLoc();
1763 if (Check(P->getType(), Loc, false))
1764 return;
1765 }
1766}
1767
1768void X86_64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM,
1769 SourceLocation CallLoc,
1770 const FunctionDecl *Caller,
1771 const FunctionDecl *Callee,
1772 const CallArgList &Args,
1773 QualType ReturnType) const {
1774 if (!Callee)
1775 return;
1776
1777 llvm::StringMap<bool> CallerMap;
1778 llvm::StringMap<bool> CalleeMap;
1779 unsigned ArgIndex = 0;
1780
1781 // We need to loop through the actual call arguments rather than the
1782 // function's parameters, in case this variadic.
1783 for (const CallArg &Arg : Args) {
1784 // The "avx" feature changes how vectors >128 in size are passed. "avx512f"
1785 // additionally changes how vectors >256 in size are passed. Like GCC, we
1786 // warn when a function is called with an argument where this will change.
1787 // Unlike GCC, we also error when it is an obvious ABI mismatch, that is,
1788 // the caller and callee features are mismatched.
1789 // Unfortunately, we cannot do this diagnostic in SEMA, since the callee can
1790 // change its ABI with attribute-target after this call.
1791 if (Arg.getType()->isVectorType() &&
1792 CGM.getContext().getTypeSize(T: Arg.getType()) > 128) {
1793 initFeatureMaps(Ctx: CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
1794 QualType Ty = Arg.getType();
1795 // The CallArg seems to have desugared the type already, so for clearer
1796 // diagnostics, replace it with the type in the FunctionDecl if possible.
1797 if (ArgIndex < Callee->getNumParams())
1798 Ty = Callee->getParamDecl(i: ArgIndex)->getType();
1799
1800 if (checkAVXParam(Diag&: CGM.getDiags(), Ctx&: CGM.getContext(), CallLoc, Callee: *Callee,
1801 CallerMap, CalleeMap, Ty, /*IsArgument*/ true))
1802 return;
1803 }
1804 ++ArgIndex;
1805 }
1806
1807 // Check return always, as we don't have a good way of knowing in codegen
1808 // whether this value is used, tail-called, etc.
1809 if (Callee->getReturnType()->isVectorType() &&
1810 CGM.getContext().getTypeSize(T: Callee->getReturnType()) > 128) {
1811 initFeatureMaps(Ctx: CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
1812 checkAVXParam(Diag&: CGM.getDiags(), Ctx&: CGM.getContext(), CallLoc, Callee: *Callee, CallerMap,
1813 CalleeMap, Ty: Callee->getReturnType(),
1814 /*IsArgument*/ false);
1815 }
1816}
1817
1818std::string TargetCodeGenInfo::qualifyWindowsLibrary(StringRef Lib) {
1819 // If the argument does not end in .lib, automatically add the suffix.
1820 // If the argument contains a space, enclose it in quotes.
1821 // This matches the behavior of MSVC.
1822 bool Quote = Lib.contains(C: ' ');
1823 std::string ArgStr = Quote ? "\"" : "";
1824 ArgStr += Lib;
1825 if (!Lib.ends_with_insensitive(Suffix: ".lib") && !Lib.ends_with_insensitive(Suffix: ".a"))
1826 ArgStr += ".lib";
1827 ArgStr += Quote ? "\"" : "";
1828 return ArgStr;
1829}
1830
1831namespace {
1832class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo {
1833public:
1834 WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
1835 bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI,
1836 unsigned NumRegisterParameters)
1837 : X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI,
1838 Win32StructABI, NumRegisterParameters, false) {}
1839
1840 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1841 CodeGen::CodeGenModule &CGM) const override;
1842
1843 void getDependentLibraryOption(llvm::StringRef Lib,
1844 llvm::SmallString<24> &Opt) const override {
1845 Opt = "/DEFAULTLIB:";
1846 Opt += qualifyWindowsLibrary(Lib);
1847 }
1848
1849 void getDetectMismatchOption(llvm::StringRef Name,
1850 llvm::StringRef Value,
1851 llvm::SmallString<32> &Opt) const override {
1852 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
1853 }
1854};
1855} // namespace
1856
1857void WinX86_32TargetCodeGenInfo::setTargetAttributes(
1858 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
1859 X86_32TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
1860 if (GV->isDeclaration())
1861 return;
1862 addStackProbeTargetAttributes(D, GV, CGM);
1863}
1864
1865namespace {
1866class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
1867public:
1868 WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
1869 X86AVXABILevel AVXLevel)
1870 : TargetCodeGenInfo(std::make_unique<WinX86_64ABIInfo>(args&: CGT, args&: AVXLevel)) {
1871 SwiftInfo =
1872 std::make_unique<SwiftABIInfo>(args&: CGT, /*SwiftErrorInRegister=*/args: true);
1873 }
1874
1875 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1876 CodeGen::CodeGenModule &CGM) const override;
1877
1878 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1879 return 7;
1880 }
1881
1882 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1883 llvm::Value *Address) const override {
1884 llvm::Value *Eight8 = llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 8);
1885
1886 // 0-15 are the 16 integer registers.
1887 // 16 is %rip.
1888 AssignToArrayRange(Builder&: CGF.Builder, Array: Address, Value: Eight8, FirstIndex: 0, LastIndex: 16);
1889 return false;
1890 }
1891
1892 void getDependentLibraryOption(llvm::StringRef Lib,
1893 llvm::SmallString<24> &Opt) const override {
1894 Opt = "/DEFAULTLIB:";
1895 Opt += qualifyWindowsLibrary(Lib);
1896 }
1897
1898 void getDetectMismatchOption(llvm::StringRef Name,
1899 llvm::StringRef Value,
1900 llvm::SmallString<32> &Opt) const override {
1901 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
1902 }
1903};
1904} // namespace
1905
1906void WinX86_64TargetCodeGenInfo::setTargetAttributes(
1907 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
1908 TargetCodeGenInfo::setTargetAttributes(D, GV, M&: CGM);
1909 if (GV->isDeclaration())
1910 return;
1911 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D)) {
1912 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
1913 llvm::Function *Fn = cast<llvm::Function>(Val: GV);
1914 Fn->addFnAttr(Kind: "stackrealign");
1915 }
1916
1917 addX86InterruptAttrs(FD, GV, CGM);
1918 }
1919
1920 addStackProbeTargetAttributes(D, GV, CGM);
1921}
1922
1923void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo,
1924 Class &Hi) const {
1925 // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
1926 //
1927 // (a) If one of the classes is Memory, the whole argument is passed in
1928 // memory.
1929 //
1930 // (b) If X87UP is not preceded by X87, the whole argument is passed in
1931 // memory.
1932 //
1933 // (c) If the size of the aggregate exceeds two eightbytes and the first
1934 // eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
1935 // argument is passed in memory. NOTE: This is necessary to keep the
1936 // ABI working for processors that don't support the __m256 type.
1937 //
1938 // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
1939 //
1940 // Some of these are enforced by the merging logic. Others can arise
1941 // only with unions; for example:
1942 // union { _Complex double; unsigned; }
1943 //
1944 // Note that clauses (b) and (c) were added in 0.98.
1945 //
1946 if (Hi == Memory)
1947 Lo = Memory;
1948 if (Hi == X87Up && Lo != X87 && honorsRevision0_98())
1949 Lo = Memory;
1950 if (AggregateSize > 128 && (Lo != SSE || Hi != SSEUp))
1951 Lo = Memory;
1952 if (Hi == SSEUp && Lo != SSE)
1953 Hi = SSE;
1954}
1955
1956static X86AVXABILevel getEffectiveX86AVXABILevel(CodeGenTypes &CGT,
1957 X86AVXABILevel GlobalAVXLevel,
1958 const FunctionDecl *FD) {
1959 // Always return global AVX level on PlayStation.
1960 if (CGT.getTarget().getTriple().isPS() ||
1961 CGT.getContext().getLangOpts().getClangABICompat() <=
1962 LangOptions::ClangABI::Ver23) {
1963 return GlobalAVXLevel;
1964 }
1965
1966 X86AVXABILevel Level = GlobalAVXLevel;
1967 // TargetVersionAttr does not apply to x86.
1968 // FIXME: Handling TargetClonesAttr and CPUSpecificAttr is intentionally
1969 // deferred to a follow-up.
1970 if (!FD || !FD->hasAttr<TargetAttr>())
1971 return Level;
1972
1973 llvm::StringMap<bool> FeatureMap;
1974 CGT.getCGM().getContext().getFunctionFeatureMap(FeatureMap, FD);
1975 if (FeatureMap.lookup(Key: "avx512f"))
1976 return std::max(a: Level, b: X86AVXABILevel::AVX512);
1977 if (FeatureMap.lookup(Key: "avx"))
1978 return std::max(a: Level, b: X86AVXABILevel::AVX);
1979 return Level;
1980}
1981
1982X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
1983 // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
1984 // classified recursively so that always two fields are
1985 // considered. The resulting class is calculated according to
1986 // the classes of the fields in the eightbyte:
1987 //
1988 // (a) If both classes are equal, this is the resulting class.
1989 //
1990 // (b) If one of the classes is NO_CLASS, the resulting class is
1991 // the other class.
1992 //
1993 // (c) If one of the classes is MEMORY, the result is the MEMORY
1994 // class.
1995 //
1996 // (d) If one of the classes is INTEGER, the result is the
1997 // INTEGER.
1998 //
1999 // (e) If one of the classes is X87, X87UP, COMPLEX_X87 class,
2000 // MEMORY is used as class.
2001 //
2002 // (f) Otherwise class SSE is used.
2003
2004 // Accum should never be memory (we should have returned) or
2005 // ComplexX87 (because this cannot be passed in a structure).
2006 assert((Accum != Memory && Accum != ComplexX87) &&
2007 "Invalid accumulated classification during merge.");
2008 if (Accum == Field || Field == NoClass)
2009 return Accum;
2010 if (Field == Memory)
2011 return Memory;
2012 if (Accum == NoClass)
2013 return Field;
2014 if (Accum == Integer || Field == Integer)
2015 return Integer;
2016 if (Field == X87 || Field == X87Up || Field == ComplexX87 ||
2017 Accum == X87 || Accum == X87Up)
2018 return Memory;
2019 return SSE;
2020}
2021
2022void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
2023 Class &Hi, bool isNamedArg, bool IsRegCall) const {
2024 // FIXME: This code can be simplified by introducing a simple value class for
2025 // Class pairs with appropriate constructor methods for the various
2026 // situations.
2027
2028 // FIXME: Some of the split computations are wrong; unaligned vectors
2029 // shouldn't be passed in registers for example, so there is no chance they
2030 // can straddle an eightbyte. Verify & simplify.
2031
2032 Lo = Hi = NoClass;
2033
2034 Class &Current = OffsetBase < 64 ? Lo : Hi;
2035 Current = Memory;
2036
2037 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
2038 BuiltinType::Kind k = BT->getKind();
2039
2040 if (k == BuiltinType::Void) {
2041 Current = NoClass;
2042 } else if (k == BuiltinType::Int128 || k == BuiltinType::UInt128) {
2043 Lo = Integer;
2044 Hi = Integer;
2045 } else if (k >= BuiltinType::Bool && k <= BuiltinType::LongLong) {
2046 Current = Integer;
2047 } else if (k == BuiltinType::Float || k == BuiltinType::Double ||
2048 k == BuiltinType::Float16 || k == BuiltinType::BFloat16) {
2049 Current = SSE;
2050 } else if (k == BuiltinType::Float128) {
2051 Lo = SSE;
2052 Hi = SSEUp;
2053 } else if (k == BuiltinType::LongDouble) {
2054 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2055 if (LDF == &llvm::APFloat::IEEEquad()) {
2056 Lo = SSE;
2057 Hi = SSEUp;
2058 } else if (LDF == &llvm::APFloat::x87DoubleExtended()) {
2059 Lo = X87;
2060 Hi = X87Up;
2061 } else if (LDF == &llvm::APFloat::IEEEdouble()) {
2062 Current = SSE;
2063 } else
2064 llvm_unreachable("unexpected long double representation!");
2065 }
2066 // FIXME: _Decimal32 and _Decimal64 are SSE.
2067 // FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
2068 return;
2069 }
2070
2071 if (const auto *ED = Ty->getAsEnumDecl()) {
2072 // Classify the underlying integer type.
2073 classify(Ty: ED->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);
2074 return;
2075 }
2076
2077 if (Ty->hasPointerRepresentation()) {
2078 Current = Integer;
2079 return;
2080 }
2081
2082 if (Ty->isMemberPointerType()) {
2083 if (Ty->isMemberFunctionPointerType()) {
2084 if (Has64BitPointers) {
2085 // If Has64BitPointers, this is an {i64, i64}, so classify both
2086 // Lo and Hi now.
2087 Lo = Hi = Integer;
2088 } else {
2089 // Otherwise, with 32-bit pointers, this is an {i32, i32}. If that
2090 // straddles an eightbyte boundary, Hi should be classified as well.
2091 uint64_t EB_FuncPtr = (OffsetBase) / 64;
2092 uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64;
2093 if (EB_FuncPtr != EB_ThisAdj) {
2094 Lo = Hi = Integer;
2095 } else {
2096 Current = Integer;
2097 }
2098 }
2099 } else {
2100 Current = Integer;
2101 }
2102 return;
2103 }
2104
2105 if (const VectorType *VT = Ty->getAs<VectorType>()) {
2106 uint64_t Size = getContext().getTypeSize(T: VT);
2107 if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
2108 // gcc passes the following as integer:
2109 // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
2110 // 2 bytes - <2 x char>, <1 x short>
2111 // 1 byte - <1 x char>
2112 Current = Integer;
2113
2114 // If this type crosses an eightbyte boundary, it should be
2115 // split.
2116 uint64_t EB_Lo = (OffsetBase) / 64;
2117 uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
2118 if (EB_Lo != EB_Hi)
2119 Hi = Lo;
2120 } else if (Size == 64) {
2121 QualType ElementType = VT->getElementType();
2122
2123 // gcc passes <1 x double> in memory. :(
2124 if (ElementType->isSpecificBuiltinType(K: BuiltinType::Double))
2125 return;
2126
2127 // gcc passes <1 x long long> as SSE but clang used to unconditionally
2128 // pass them as integer. For platforms where clang is the de facto
2129 // platform compiler, we must continue to use integer.
2130 if (!classifyIntegerMMXAsSSE() &&
2131 (ElementType->isSpecificBuiltinType(K: BuiltinType::LongLong) ||
2132 ElementType->isSpecificBuiltinType(K: BuiltinType::ULongLong) ||
2133 ElementType->isSpecificBuiltinType(K: BuiltinType::Long) ||
2134 ElementType->isSpecificBuiltinType(K: BuiltinType::ULong)))
2135 Current = Integer;
2136 else
2137 Current = SSE;
2138
2139 // If this type crosses an eightbyte boundary, it should be
2140 // split.
2141 if (OffsetBase && OffsetBase != 64)
2142 Hi = Lo;
2143 } else if (Size == 128 ||
2144 (isNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
2145 QualType ElementType = VT->getElementType();
2146
2147 // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
2148 if (passInt128VectorsInMem() && Size != 128 &&
2149 (ElementType->isSpecificBuiltinType(K: BuiltinType::Int128) ||
2150 ElementType->isSpecificBuiltinType(K: BuiltinType::UInt128)))
2151 return;
2152
2153 // Arguments of 256-bits are split into four eightbyte chunks. The
2154 // least significant one belongs to class SSE and all the others to class
2155 // SSEUP. The original Lo and Hi design considers that types can't be
2156 // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
2157 // This design isn't correct for 256-bits, but since there're no cases
2158 // where the upper parts would need to be inspected, avoid adding
2159 // complexity and just consider Hi to match the 64-256 part.
2160 //
2161 // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
2162 // registers if they are "named", i.e. not part of the "..." of a
2163 // variadic function.
2164 //
2165 // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
2166 // split into eight eightbyte chunks, one SSE and seven SSEUP.
2167 Lo = SSE;
2168 Hi = SSEUp;
2169 }
2170 return;
2171 }
2172
2173 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
2174 QualType ET = getContext().getCanonicalType(T: CT->getElementType());
2175
2176 uint64_t Size = getContext().getTypeSize(T: Ty);
2177 if (ET->isIntegralOrEnumerationType()) {
2178 if (Size <= 64)
2179 Current = Integer;
2180 else if (Size <= 128)
2181 Lo = Hi = Integer;
2182 } else if (ET->isFloat16Type() || ET == getContext().FloatTy ||
2183 ET->isBFloat16Type()) {
2184 Current = SSE;
2185 } else if (ET == getContext().DoubleTy) {
2186 Lo = Hi = SSE;
2187 } else if (ET == getContext().LongDoubleTy) {
2188 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2189 if (LDF == &llvm::APFloat::IEEEquad())
2190 Current = Memory;
2191 else if (LDF == &llvm::APFloat::x87DoubleExtended())
2192 Current = ComplexX87;
2193 else if (LDF == &llvm::APFloat::IEEEdouble())
2194 Lo = Hi = SSE;
2195 else
2196 llvm_unreachable("unexpected long double representation!");
2197 }
2198
2199 // If this complex type crosses an eightbyte boundary then it
2200 // should be split.
2201 uint64_t EB_Real = (OffsetBase) / 64;
2202 uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(T: ET)) / 64;
2203 if (Hi == NoClass && EB_Real != EB_Imag)
2204 Hi = Lo;
2205
2206 return;
2207 }
2208
2209 if (const auto *EITy = Ty->getAs<BitIntType>()) {
2210 if (EITy->getNumBits() <= 64)
2211 Current = Integer;
2212 else if (EITy->getNumBits() <= 128)
2213 Lo = Hi = Integer;
2214 // Larger values need to get passed in memory.
2215 return;
2216 }
2217
2218 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(T: Ty)) {
2219 // Arrays are treated like structures.
2220
2221 uint64_t Size = getContext().getTypeSize(T: Ty);
2222
2223 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
2224 // than eight eightbytes, ..., it has class MEMORY.
2225 // regcall ABI doesn't have limitation to an object. The only limitation
2226 // is the free registers, which will be checked in computeInfo.
2227 if (!IsRegCall && Size > 512)
2228 return;
2229
2230 // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
2231 // fields, it has class MEMORY.
2232 //
2233 // Only need to check alignment of array base.
2234 if (OffsetBase % getContext().getTypeAlign(T: AT->getElementType()))
2235 return;
2236
2237 // Otherwise implement simplified merge. We could be smarter about
2238 // this, but it isn't worth it and would be harder to verify.
2239 Current = NoClass;
2240 uint64_t EltSize = getContext().getTypeSize(T: AT->getElementType());
2241 uint64_t ArraySize = AT->getZExtSize();
2242
2243 // The only case a 256-bit wide vector could be used is when the array
2244 // contains a single 256-bit element. Since Lo and Hi logic isn't extended
2245 // to work for sizes wider than 128, early check and fallback to memory.
2246 //
2247 if (Size > 128 &&
2248 (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))
2249 return;
2250
2251 for (uint64_t i=0, Offset=OffsetBase; i<ArraySize; ++i, Offset += EltSize) {
2252 Class FieldLo, FieldHi;
2253 classify(Ty: AT->getElementType(), OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, isNamedArg);
2254 Lo = merge(Accum: Lo, Field: FieldLo);
2255 Hi = merge(Accum: Hi, Field: FieldHi);
2256 if (Lo == Memory || Hi == Memory)
2257 break;
2258 }
2259
2260 postMerge(AggregateSize: Size, Lo, Hi);
2261 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
2262 return;
2263 }
2264
2265 if (const RecordType *RT = Ty->getAsCanonical<RecordType>()) {
2266 uint64_t Size = getContext().getTypeSize(T: Ty);
2267
2268 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
2269 // than eight eightbytes, ..., it has class MEMORY.
2270 if (Size > 512)
2271 return;
2272
2273 // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
2274 // copy constructor or a non-trivial destructor, it is passed by invisible
2275 // reference.
2276 if (getRecordArgABI(RT, CXXABI&: getCXXABI()))
2277 return;
2278
2279 const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf();
2280
2281 // Assume variable sized types are passed in memory.
2282 if (RD->hasFlexibleArrayMember())
2283 return;
2284
2285 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: RD);
2286
2287 // Reset Lo class, this will be recomputed.
2288 Current = NoClass;
2289
2290 // If this is a C++ record, classify the bases first.
2291 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
2292 for (const auto &I : CXXRD->bases()) {
2293 assert(!I.isVirtual() && !I.getType()->isDependentType() &&
2294 "Unexpected base class!");
2295 const auto *Base = I.getType()->castAsCXXRecordDecl();
2296 // Classify this field.
2297 //
2298 // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
2299 // single eightbyte, each is classified separately. Each eightbyte gets
2300 // initialized to class NO_CLASS.
2301 Class FieldLo, FieldHi;
2302 uint64_t Offset =
2303 OffsetBase + getContext().toBits(CharSize: Layout.getBaseClassOffset(Base));
2304 classify(Ty: I.getType(), OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, isNamedArg);
2305 Lo = merge(Accum: Lo, Field: FieldLo);
2306 Hi = merge(Accum: Hi, Field: FieldHi);
2307 if (returnCXXRecordGreaterThan128InMem() &&
2308 !isEmptyRecord(Context&: getContext(), T: I.getType(), AllowArrays: true) &&
2309 (Size > 128 && (Size != getContext().getTypeSize(T: I.getType()) ||
2310 Size > getNativeVectorSizeForAVXABI(AVXLevel)))) {
2311 // The only case a 256(or 512)-bit wide vector could be used to return
2312 // is when CXX record contains a single 256(or 512)-bit element.
2313 Lo = Memory;
2314 }
2315 if (Lo == Memory || Hi == Memory) {
2316 postMerge(AggregateSize: Size, Lo, Hi);
2317 return;
2318 }
2319 }
2320 }
2321
2322 // Classify the fields one at a time, merging the results.
2323 unsigned idx = 0;
2324 bool UseClang11Compat = getContext().getLangOpts().isCompatibleWith(
2325 Version: LangOptions::ClangABI::Ver11) ||
2326 getContext().getTargetInfo().getTriple().isPS();
2327 bool ClassifyUnnamedBitFields =
2328 getContext().getLangOpts().getClangABICompat() >
2329 LangOptions::ClangABI::Ver23 &&
2330 !getContext().getTargetInfo().getTriple().isPS();
2331 bool IsUnion = RT->isUnionType() && !UseClang11Compat;
2332
2333 for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
2334 i != e; ++i, ++idx) {
2335 uint64_t Offset = OffsetBase + Layout.getFieldOffset(FieldNo: idx);
2336 bool BitField = i->isBitField();
2337
2338 // Ignore padding bit-fields. Normally only zero-length bit-fields are
2339 // padding, but under -fclang-abi-compat=23 every unnamed bit-field is,
2340 // faithfully reproducing Clang 23 -- including its crash on aggregates
2341 // where skipping one leaves part of a wider access unit (e.g. an
2342 // __int128 bit-field run) unclassified.
2343 if (BitField && (ClassifyUnnamedBitFields ? i->isZeroLengthBitField()
2344 : i->isUnnamedBitField()))
2345 continue;
2346
2347 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than
2348 // eight eightbytes, or it contains unaligned fields, it has class MEMORY.
2349 //
2350 // The only case a 256-bit or a 512-bit wide vector could be used is when
2351 // the struct contains a single 256-bit or 512-bit element. Early check
2352 // and fallback to memory.
2353 //
2354 // FIXME: Extended the Lo and Hi logic properly to work for size wider
2355 // than 128.
2356 if (Size > 128 &&
2357 ((!IsUnion && Size != getContext().getTypeSize(T: i->getType())) ||
2358 Size > getNativeVectorSizeForAVXABI(AVXLevel))) {
2359 Lo = Memory;
2360 postMerge(AggregateSize: Size, Lo, Hi);
2361 return;
2362 }
2363
2364 bool IsInMemory =
2365 Offset % getContext().getTypeAlign(T: i->getType().getCanonicalType());
2366 // Note, skip this test for bit-fields, see below.
2367 if (!BitField && IsInMemory) {
2368 Lo = Memory;
2369 postMerge(AggregateSize: Size, Lo, Hi);
2370 return;
2371 }
2372
2373 // Classify this field.
2374 //
2375 // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
2376 // exceeds a single eightbyte, each is classified
2377 // separately. Each eightbyte gets initialized to class
2378 // NO_CLASS.
2379 Class FieldLo, FieldHi;
2380
2381 // Bit-fields require special handling, they do not force the
2382 // structure to be passed in memory even if unaligned, and
2383 // therefore they can straddle an eightbyte.
2384 if (BitField) {
2385 assert(ClassifyUnnamedBitFields ? !i->isZeroLengthBitField()
2386 : !i->isUnnamedBitField());
2387 uint64_t Offset = OffsetBase + Layout.getFieldOffset(FieldNo: idx);
2388 uint64_t Size = i->getBitWidthValue();
2389
2390 uint64_t EB_Lo = Offset / 64;
2391 uint64_t EB_Hi = (Offset + Size - 1) / 64;
2392
2393 if (EB_Lo) {
2394 assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
2395 FieldLo = NoClass;
2396 FieldHi = Integer;
2397 } else {
2398 FieldLo = Integer;
2399 FieldHi = EB_Hi ? Integer : NoClass;
2400 }
2401 } else
2402 classify(Ty: i->getType(), OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, isNamedArg);
2403 Lo = merge(Accum: Lo, Field: FieldLo);
2404 Hi = merge(Accum: Hi, Field: FieldHi);
2405 if (Lo == Memory || Hi == Memory)
2406 break;
2407 }
2408
2409 postMerge(AggregateSize: Size, Lo, Hi);
2410 }
2411}
2412
2413ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
2414 // If this is a scalar LLVM value then assume LLVM will pass it in the right
2415 // place naturally.
2416 if (!isAggregateTypeForABI(T: Ty)) {
2417 // Treat an enum type as its underlying type.
2418 if (const auto *ED = Ty->getAsEnumDecl())
2419 Ty = ED->getIntegerType();
2420
2421 if (Ty->isBitIntType())
2422 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace());
2423
2424 llvm::Type *IRTy = CGT.ConvertType(T: Ty);
2425 return shouldExtendReturnType(Info: *this, RetTy: Ty) ? ABIArgInfo::getExtend(Ty, T: IRTy)
2426 : ABIArgInfo::getDirect(T: IRTy);
2427 }
2428
2429 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace());
2430}
2431
2432bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {
2433 if (const VectorType *VecTy = Ty->getAs<VectorType>()) {
2434 uint64_t Size = getContext().getTypeSize(T: VecTy);
2435 unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
2436 if (Size <= 64 || Size > LargestVector)
2437 return true;
2438 QualType EltTy = VecTy->getElementType();
2439 if (passInt128VectorsInMem() &&
2440 (EltTy->isSpecificBuiltinType(K: BuiltinType::Int128) ||
2441 EltTy->isSpecificBuiltinType(K: BuiltinType::UInt128)))
2442 return true;
2443 }
2444
2445 return false;
2446}
2447
2448ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
2449 unsigned freeIntRegs) const {
2450 // If this is a scalar LLVM value then assume LLVM will pass it in the right
2451 // place naturally.
2452 //
2453 // This assumption is optimistic, as there could be free registers available
2454 // when we need to pass this argument in memory, and LLVM could try to pass
2455 // the argument in the free register. This does not seem to happen currently,
2456 // but this code would be much safer if we could mark the argument with
2457 // 'onstack'. See PR12193.
2458 if (!isAggregateTypeForABI(T: Ty) && !IsIllegalVectorType(Ty) &&
2459 !Ty->isBitIntType()) {
2460 // Treat an enum type as its underlying type.
2461 if (const auto *ED = Ty->getAsEnumDecl())
2462 Ty = ED->getIntegerType();
2463
2464 llvm::Type *IRTy = CGT.ConvertType(T: Ty);
2465 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty, T: IRTy)
2466 : ABIArgInfo::getDirect(T: IRTy));
2467 }
2468
2469 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(T: Ty, CXXABI&: getCXXABI()))
2470 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
2471 ByVal: RAA == CGCXXABI::RAA_DirectInMemory);
2472
2473 // Compute the byval alignment. We specify the alignment of the byval in all
2474 // cases so that the mid-level optimizer knows the alignment of the byval.
2475 unsigned Align = std::max(a: getContext().getTypeAlign(T: Ty) / 8, b: 8U);
2476
2477 // Attempt to avoid passing indirect results using byval when possible. This
2478 // is important for good codegen.
2479 //
2480 // We do this by coercing the value into a scalar type which the backend can
2481 // handle naturally (i.e., without using byval).
2482 //
2483 // For simplicity, we currently only do this when we have exhausted all of the
2484 // free integer registers. Doing this when there are free integer registers
2485 // would require more care, as we would have to ensure that the coerced value
2486 // did not claim the unused register. That would require either reording the
2487 // arguments to the function (so that any subsequent inreg values came first),
2488 // or only doing this optimization when there were no following arguments that
2489 // might be inreg.
2490 //
2491 // We currently expect it to be rare (particularly in well written code) for
2492 // arguments to be passed on the stack when there are still free integer
2493 // registers available (this would typically imply large structs being passed
2494 // by value), so this seems like a fair tradeoff for now.
2495 //
2496 // We can revisit this if the backend grows support for 'onstack' parameter
2497 // attributes. See PR12193.
2498 if (freeIntRegs == 0) {
2499 uint64_t Size = getContext().getTypeSize(T: Ty);
2500
2501 // If this type fits in an eightbyte, coerce it into the matching integral
2502 // type, which will end up on the stack (with alignment 8).
2503 if (Align == 8 && Size <= 64)
2504 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(),
2505 NumBits: Size));
2506 }
2507
2508 return ABIArgInfo::getIndirect(Alignment: CharUnits::fromQuantity(Quantity: Align),
2509 AddrSpace: getDataLayout().getAllocaAddrSpace());
2510}
2511
2512/// The ABI specifies that a value should be passed in a full vector XMM/YMM
2513/// register. Pick an LLVM IR type that will be passed as a vector register.
2514llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {
2515 // Wrapper structs/arrays that only contain vectors are passed just like
2516 // vectors; strip them off if present.
2517 if (const Type *InnerTy = isSingleElementRecord(T: Ty, Context&: getContext()))
2518 Ty = QualType(InnerTy, 0);
2519
2520 llvm::Type *IRType = CGT.ConvertType(T: Ty);
2521 if (isa<llvm::VectorType>(Val: IRType)) {
2522 // Don't pass vXi128 vectors in their native type, the backend can't
2523 // legalize them.
2524 if (passInt128VectorsInMem() &&
2525 cast<llvm::VectorType>(Val: IRType)->getElementType()->isIntegerTy(BitWidth: 128)) {
2526 // Use a vXi64 vector.
2527 uint64_t Size = getContext().getTypeSize(T: Ty);
2528 return llvm::FixedVectorType::get(ElementType: llvm::Type::getInt64Ty(C&: getVMContext()),
2529 NumElts: Size / 64);
2530 }
2531
2532 return IRType;
2533 }
2534
2535 if (IRType->getTypeID() == llvm::Type::FP128TyID)
2536 return IRType;
2537
2538 // We couldn't find the preferred IR vector type for 'Ty'.
2539 uint64_t Size = getContext().getTypeSize(T: Ty);
2540 assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!");
2541
2542
2543 // Return a LLVM IR vector type based on the size of 'Ty'.
2544 return llvm::FixedVectorType::get(ElementType: llvm::Type::getDoubleTy(C&: getVMContext()),
2545 NumElts: Size / 64);
2546}
2547
2548/// BitsContainNoUserData - Return true if the specified [start,end) bit range
2549/// is known to either be off the end of the specified type or being in
2550/// alignment padding. The user type specified is known to be at most 128 bits
2551/// in size, and have passed through X86_64ABIInfo::classify with a successful
2552/// classification that put one of the two halves in the INTEGER class.
2553///
2554/// It is conservatively correct to return false.
2555static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
2556 unsigned EndBit, ASTContext &Context) {
2557 // If the bytes being queried are off the end of the type, there is no user
2558 // data hiding here. This handles analysis of builtins, vectors and other
2559 // types that don't contain interesting padding.
2560 unsigned TySize = (unsigned)Context.getTypeSize(T: Ty);
2561 if (TySize <= StartBit)
2562 return true;
2563
2564 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty)) {
2565 unsigned EltSize = (unsigned)Context.getTypeSize(T: AT->getElementType());
2566 unsigned NumElts = (unsigned)AT->getZExtSize();
2567
2568 // Check each element to see if the element overlaps with the queried range.
2569 for (unsigned i = 0; i != NumElts; ++i) {
2570 // If the element is after the span we care about, then we're done..
2571 unsigned EltOffset = i*EltSize;
2572 if (EltOffset >= EndBit) break;
2573
2574 unsigned EltStart = EltOffset < StartBit ? StartBit-EltOffset :0;
2575 if (!BitsContainNoUserData(Ty: AT->getElementType(), StartBit: EltStart,
2576 EndBit: EndBit-EltOffset, Context))
2577 return false;
2578 }
2579 // If it overlaps no elements, then it is safe to process as padding.
2580 return true;
2581 }
2582
2583 if (const auto *RD = Ty->getAsRecordDecl()) {
2584 const ASTRecordLayout &Layout = Context.getASTRecordLayout(D: RD);
2585
2586 // If this is a C++ record, check the bases first.
2587 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
2588 for (const auto &I : CXXRD->bases()) {
2589 assert(!I.isVirtual() && !I.getType()->isDependentType() &&
2590 "Unexpected base class!");
2591 const auto *Base = I.getType()->castAsCXXRecordDecl();
2592
2593 // If the base is after the span we care about, ignore it.
2594 unsigned BaseOffset = Context.toBits(CharSize: Layout.getBaseClassOffset(Base));
2595 if (BaseOffset >= EndBit) continue;
2596
2597 unsigned BaseStart = BaseOffset < StartBit ? StartBit-BaseOffset :0;
2598 if (!BitsContainNoUserData(Ty: I.getType(), StartBit: BaseStart,
2599 EndBit: EndBit-BaseOffset, Context))
2600 return false;
2601 }
2602 }
2603
2604 // Verify that no field has data that overlaps the region of interest. Yes
2605 // this could be sped up a lot by being smarter about queried fields,
2606 // however we're only looking at structs up to 16 bytes, so we don't care
2607 // much.
2608 unsigned idx = 0;
2609 for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
2610 i != e; ++i, ++idx) {
2611 unsigned FieldOffset = (unsigned)Layout.getFieldOffset(FieldNo: idx);
2612
2613 // If we found a field after the region we care about, then we're done.
2614 if (FieldOffset >= EndBit) break;
2615
2616 unsigned FieldStart = FieldOffset < StartBit ? StartBit-FieldOffset :0;
2617 if (!BitsContainNoUserData(Ty: i->getType(), StartBit: FieldStart, EndBit: EndBit-FieldOffset,
2618 Context))
2619 return false;
2620 }
2621
2622 // If nothing in this record overlapped the area of interest, then we're
2623 // clean.
2624 return true;
2625 }
2626
2627 return false;
2628}
2629
2630/// getFPTypeAtOffset - Return a floating point type at the specified offset.
2631static llvm::Type *getFPTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
2632 const llvm::DataLayout &TD) {
2633 if (IROffset == 0 && IRType->isFloatingPointTy())
2634 return IRType;
2635
2636 // If this is a struct, recurse into the field at the specified offset.
2637 if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: IRType)) {
2638 if (!STy->getNumContainedTypes())
2639 return nullptr;
2640
2641 const llvm::StructLayout *SL = TD.getStructLayout(Ty: STy);
2642 unsigned Elt = SL->getElementContainingOffset(FixedOffset: IROffset);
2643 IROffset -= SL->getElementOffset(Idx: Elt);
2644 return getFPTypeAtOffset(IRType: STy->getElementType(N: Elt), IROffset, TD);
2645 }
2646
2647 // If this is an array, recurse into the field at the specified offset.
2648 if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(Val: IRType)) {
2649 llvm::Type *EltTy = ATy->getElementType();
2650 unsigned EltSize = TD.getTypeAllocSize(Ty: EltTy);
2651 if (EltSize == 0)
2652 return nullptr;
2653 IROffset -= IROffset / EltSize * EltSize;
2654 return getFPTypeAtOffset(IRType: EltTy, IROffset, TD);
2655 }
2656
2657 return nullptr;
2658}
2659
2660/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
2661/// low 8 bytes of an XMM register, corresponding to the SSE class.
2662llvm::Type *X86_64ABIInfo::
2663GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
2664 QualType SourceTy, unsigned SourceOffset) const {
2665 const llvm::DataLayout &TD = getDataLayout();
2666 unsigned SourceSize =
2667 (unsigned)getContext().getTypeSize(T: SourceTy) / 8 - SourceOffset;
2668 llvm::Type *T0 = getFPTypeAtOffset(IRType, IROffset, TD);
2669 if (!T0 || T0->isDoubleTy())
2670 return llvm::Type::getDoubleTy(C&: getVMContext());
2671
2672 // Get the adjacent FP type.
2673 llvm::Type *T1 = nullptr;
2674 unsigned T0Size = TD.getTypeAllocSize(Ty: T0);
2675 if (SourceSize > T0Size)
2676 T1 = getFPTypeAtOffset(IRType, IROffset: IROffset + T0Size, TD);
2677 if (T1 == nullptr) {
2678 // Check if IRType is a half/bfloat + float. float type will be in IROffset+4 due
2679 // to its alignment.
2680 if (T0->is16bitFPTy() && SourceSize > 4)
2681 T1 = getFPTypeAtOffset(IRType, IROffset: IROffset + 4, TD);
2682 // If we can't get a second FP type, return a simple half or float.
2683 // avx512fp16-abi.c:pr51813_2 shows it works to return float for
2684 // {float, i8} too.
2685 if (T1 == nullptr)
2686 return T0;
2687 }
2688
2689 if (T0->isFloatTy() && T1->isFloatTy())
2690 return llvm::FixedVectorType::get(ElementType: T0, NumElts: 2);
2691
2692 if (T0->is16bitFPTy() && T1->is16bitFPTy()) {
2693 llvm::Type *T2 = nullptr;
2694 if (SourceSize > 4)
2695 T2 = getFPTypeAtOffset(IRType, IROffset: IROffset + 4, TD);
2696 if (T2 == nullptr)
2697 return llvm::FixedVectorType::get(ElementType: T0, NumElts: 2);
2698 return llvm::FixedVectorType::get(ElementType: T0, NumElts: 4);
2699 }
2700
2701 if (T0->is16bitFPTy() || T1->is16bitFPTy())
2702 return llvm::FixedVectorType::get(ElementType: llvm::Type::getHalfTy(C&: getVMContext()), NumElts: 4);
2703
2704 return llvm::Type::getDoubleTy(C&: getVMContext());
2705}
2706
2707/// GetINTEGERTypeAtOffset - The ABI specifies that a value should be passed in
2708/// one or more 8-byte GPRs. This means that we either have a scalar or we are
2709/// talking about the high and/or low part of an up-to-16-byte struct. This
2710/// routine picks the best LLVM IR type to represent this, which may be i64 or
2711/// may be anything else that the backend will pass in GPRs that works better
2712/// (e.g. i8, %foo*, etc).
2713///
2714/// PrefType is an LLVM IR type that corresponds to (part of) the IR type for
2715/// the source type. IROffset is an offset in bytes into the LLVM IR type that
2716/// the 8-byte value references. PrefType may be null.
2717///
2718/// SourceTy is the source-level type for the entire argument. SourceOffset is
2719/// an offset into this that we're processing (which is always either 0 or 8).
2720///
2721llvm::Type *X86_64ABIInfo::
2722GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
2723 QualType SourceTy, unsigned SourceOffset) const {
2724 // If we're dealing with an un-offset LLVM IR type, then it means that we're
2725 // returning an 8-byte unit starting with it. See if we can safely use it.
2726 if (IROffset == 0) {
2727 // Pointers and int64's always fill the 8-byte unit.
2728 if ((isa<llvm::PointerType>(Val: IRType) && Has64BitPointers) ||
2729 IRType->isIntegerTy(BitWidth: 64))
2730 return IRType;
2731
2732 // If we have a 1/2/4-byte integer, we can use it only if the rest of the
2733 // goodness in the source type is just tail padding. This is allowed to
2734 // kick in for struct {double,int} on the int, but not on
2735 // struct{double,int,int} because we wouldn't return the second int. We
2736 // have to do this analysis on the source type because we can't depend on
2737 // unions being lowered a specific way etc.
2738 if (IRType->isIntegerTy(BitWidth: 8) || IRType->isIntegerTy(BitWidth: 16) ||
2739 IRType->isIntegerTy(BitWidth: 32) ||
2740 (isa<llvm::PointerType>(Val: IRType) && !Has64BitPointers)) {
2741 unsigned BitWidth = isa<llvm::PointerType>(Val: IRType) ? 32 :
2742 cast<llvm::IntegerType>(Val: IRType)->getBitWidth();
2743
2744 if (BitsContainNoUserData(Ty: SourceTy, StartBit: SourceOffset*8+BitWidth,
2745 EndBit: SourceOffset*8+64, Context&: getContext()))
2746 return IRType;
2747 }
2748 }
2749
2750 if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: IRType)) {
2751 // If this is a struct, recurse into the field at the specified offset.
2752 const llvm::StructLayout *SL = getDataLayout().getStructLayout(Ty: STy);
2753 if (IROffset < SL->getSizeInBytes()) {
2754 unsigned FieldIdx = SL->getElementContainingOffset(FixedOffset: IROffset);
2755 IROffset -= SL->getElementOffset(Idx: FieldIdx);
2756
2757 return GetINTEGERTypeAtOffset(IRType: STy->getElementType(N: FieldIdx), IROffset,
2758 SourceTy, SourceOffset);
2759 }
2760 }
2761
2762 if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(Val: IRType)) {
2763 llvm::Type *EltTy = ATy->getElementType();
2764 unsigned EltSize = getDataLayout().getTypeAllocSize(Ty: EltTy);
2765 unsigned EltOffset = IROffset/EltSize*EltSize;
2766 return GetINTEGERTypeAtOffset(IRType: EltTy, IROffset: IROffset-EltOffset, SourceTy,
2767 SourceOffset);
2768 }
2769
2770 // if we have a 128-bit integer, we can pass it safely using an i128
2771 // so we return that
2772 if (IRType->isIntegerTy(BitWidth: 128)) {
2773 assert(IROffset == 0);
2774 return IRType;
2775 }
2776
2777 // Okay, we don't have any better idea of what to pass, so we pass this in an
2778 // integer register that isn't too big to fit the rest of the struct.
2779 unsigned TySizeInBytes =
2780 (unsigned)getContext().getTypeSizeInChars(T: SourceTy).getQuantity();
2781
2782 assert(TySizeInBytes != SourceOffset && "Empty field?");
2783
2784 // It is always safe to classify this as an integer type up to i64 that
2785 // isn't larger than the structure.
2786 return llvm::IntegerType::get(C&: getVMContext(),
2787 NumBits: std::min(a: TySizeInBytes-SourceOffset, b: 8U)*8);
2788}
2789
2790
2791/// GetX86_64ByValArgumentPair - Given a high and low type that can ideally
2792/// be used as elements of a two register pair to pass or return, return a
2793/// first class aggregate to represent them. For example, if the low part of
2794/// a by-value argument should be passed as i32* and the high part as float,
2795/// return {i32*, float}.
2796static llvm::Type *
2797GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,
2798 const llvm::DataLayout &TD) {
2799 // In order to correctly satisfy the ABI, we need to the high part to start
2800 // at offset 8. If the high and low parts we inferred are both 4-byte types
2801 // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
2802 // the second element at offset 8. Check for this:
2803 unsigned LoSize = (unsigned)TD.getTypeAllocSize(Ty: Lo);
2804 llvm::Align HiAlign = TD.getABITypeAlign(Ty: Hi);
2805 unsigned HiStart = llvm::alignTo(Size: LoSize, A: HiAlign);
2806 assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
2807
2808 // To handle this, we have to increase the size of the low part so that the
2809 // second element will start at an 8 byte offset. We can't increase the size
2810 // of the second element because it might make us access off the end of the
2811 // struct.
2812 if (HiStart != 8) {
2813 // There are usually two sorts of types the ABI generation code can produce
2814 // for the low part of a pair that aren't 8 bytes in size: half, float or
2815 // i8/i16/i32. This can also include pointers when they are 32-bit (X32).
2816 // Promote these to a larger type.
2817 if (Lo->isHalfTy() || Lo->isFloatTy())
2818 Lo = llvm::Type::getDoubleTy(C&: Lo->getContext());
2819 else {
2820 assert((Lo->isIntegerTy() || Lo->isPointerTy())
2821 && "Invalid/unknown lo type");
2822 Lo = llvm::Type::getInt64Ty(C&: Lo->getContext());
2823 }
2824 }
2825
2826 llvm::StructType *Result = llvm::StructType::get(elt1: Lo, elts: Hi);
2827
2828 // Verify that the second element is at an 8-byte offset.
2829 assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&
2830 "Invalid x86-64 argument pair!");
2831 return Result;
2832}
2833
2834ABIArgInfo X86_64ABIInfo::classifyReturnType(QualType RetTy) const {
2835 // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
2836 // classification algorithm.
2837 X86_64ABIInfo::Class Lo, Hi;
2838 classify(Ty: RetTy, OffsetBase: 0, Lo, Hi, /*isNamedArg*/ true);
2839
2840 // Check some invariants.
2841 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
2842 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
2843
2844 llvm::Type *ResType = nullptr;
2845 switch (Lo) {
2846 case NoClass:
2847 if (Hi == NoClass)
2848 return ABIArgInfo::getIgnore();
2849 // If the low part is just padding, it takes no register, leave ResType
2850 // null.
2851 assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
2852 "Unknown missing lo part");
2853 break;
2854
2855 case SSEUp:
2856 case X87Up:
2857 llvm_unreachable("Invalid classification for lo word.");
2858
2859 // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
2860 // hidden argument.
2861 case Memory:
2862 return getIndirectReturnResult(Ty: RetTy);
2863
2864 // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
2865 // available register of the sequence %rax, %rdx is used.
2866 case Integer:
2867 ResType = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 0, SourceTy: RetTy, SourceOffset: 0);
2868
2869 // Treat an enum type as its underlying type.
2870 if (const auto *ED = RetTy->getAsEnumDecl())
2871 RetTy = ED->getIntegerType();
2872
2873 // Always returns integral types as plain integers, without padding to a
2874 // larger type or splitting into a struct. For some targets, small integers
2875 // are additionally sign/zero extended.
2876 if (RetTy->isIntegralOrEnumerationType())
2877 return shouldExtendReturnType(Info: *this, RetTy) ? ABIArgInfo::getExtend(Ty: RetTy)
2878 : ABIArgInfo::getDirect();
2879
2880 if (ResType->isIntegerTy(BitWidth: 128)) {
2881 // i128 are passed directly
2882 assert(Hi == Integer);
2883 return ABIArgInfo::getDirect(T: ResType);
2884 }
2885 break;
2886
2887 // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
2888 // available SSE register of the sequence %xmm0, %xmm1 is used.
2889 case SSE:
2890 ResType = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 0, SourceTy: RetTy, SourceOffset: 0);
2891 break;
2892
2893 // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
2894 // returned on the X87 stack in %st0 as 80-bit x87 number.
2895 case X87:
2896 ResType = llvm::Type::getX86_FP80Ty(C&: getVMContext());
2897 break;
2898
2899 // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
2900 // part of the value is returned in %st0 and the imaginary part in
2901 // %st1.
2902 case ComplexX87:
2903 assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
2904 ResType = llvm::StructType::get(elt1: llvm::Type::getX86_FP80Ty(C&: getVMContext()),
2905 elts: llvm::Type::getX86_FP80Ty(C&: getVMContext()));
2906 break;
2907 }
2908
2909 llvm::Type *HighPart = nullptr;
2910 switch (Hi) {
2911 // Memory was handled previously and X87 should
2912 // never occur as a hi class.
2913 case Memory:
2914 case X87:
2915 llvm_unreachable("Invalid classification for hi word.");
2916
2917 case ComplexX87: // Previously handled.
2918 case NoClass:
2919 break;
2920
2921 case Integer:
2922 HighPart = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
2923 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
2924 return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2925 break;
2926 case SSE:
2927 HighPart = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
2928 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
2929 return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2930 break;
2931
2932 // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
2933 // is passed in the next available eightbyte chunk if the last used
2934 // vector register.
2935 //
2936 // SSEUP should always be preceded by SSE, just widen.
2937 case SSEUp:
2938 assert(Lo == SSE && "Unexpected SSEUp classification.");
2939 ResType = GetByteVectorType(Ty: RetTy);
2940 break;
2941
2942 // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is
2943 // returned together with the previous X87 value in %st0.
2944 case X87Up:
2945 // If X87Up is preceded by X87, we don't need to do
2946 // anything. However, in some cases with unions it may not be
2947 // preceded by X87. In such situations we follow gcc and pass the
2948 // extra bits in an SSE reg.
2949 if (Lo != X87) {
2950 HighPart = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: RetTy), IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
2951 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
2952 return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
2953 }
2954 break;
2955 }
2956
2957 // If a high part was specified, merge it together with the low part. It is
2958 // known to pass in the high eightbyte of the result. We do this by forming a
2959 // first class struct aggregate with the high and low part: {low, high}
2960 if (HighPart)
2961 ResType = GetX86_64ByValArgumentPair(Lo: ResType, Hi: HighPart, TD: getDataLayout());
2962
2963 return ABIArgInfo::getDirect(T: ResType);
2964}
2965
2966ABIArgInfo
2967X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned freeIntRegs,
2968 unsigned &neededInt, unsigned &neededSSE,
2969 bool isNamedArg, bool IsRegCall) const {
2970 Ty = useFirstFieldIfTransparentUnion(Ty);
2971
2972 X86_64ABIInfo::Class Lo, Hi;
2973 classify(Ty, OffsetBase: 0, Lo, Hi, isNamedArg, IsRegCall);
2974
2975 // Check some invariants.
2976 // FIXME: Enforce these by construction.
2977 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
2978 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
2979
2980 neededInt = 0;
2981 neededSSE = 0;
2982 llvm::Type *ResType = nullptr;
2983 switch (Lo) {
2984 case NoClass:
2985 if (Hi == NoClass)
2986 return ABIArgInfo::getIgnore();
2987 // If the low part is just padding, it takes no register, leave ResType
2988 // null.
2989 assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
2990 "Unknown missing lo part");
2991 break;
2992
2993 // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
2994 // on the stack.
2995 case Memory:
2996
2997 // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87UP or
2998 // COMPLEX_X87, it is passed in memory.
2999 case X87:
3000 case ComplexX87:
3001 if (getRecordArgABI(T: Ty, CXXABI&: getCXXABI()) == CGCXXABI::RAA_Indirect)
3002 ++neededInt;
3003 return getIndirectResult(Ty, freeIntRegs);
3004
3005 case SSEUp:
3006 case X87Up:
3007 llvm_unreachable("Invalid classification for lo word.");
3008
3009 // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
3010 // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
3011 // and %r9 is used.
3012 case Integer:
3013 ++neededInt;
3014
3015 // Pick an 8-byte type based on the preferred type.
3016 ResType = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: Ty), IROffset: 0, SourceTy: Ty, SourceOffset: 0);
3017
3018 // If we have a sign or zero extended integer, make sure to return Extend
3019 // so that the parameter gets the right LLVM IR attributes.
3020 if (Hi == NoClass && isa<llvm::IntegerType>(Val: ResType)) {
3021 // Treat an enum type as its underlying type.
3022 if (const auto *ED = Ty->getAsEnumDecl())
3023 Ty = ED->getIntegerType();
3024 if (Ty->isIntegralOrEnumerationType() &&
3025 isPromotableIntegerTypeForABI(Ty)) {
3026 switch (getEffectiveSmallIntABICompat(Opts: getCodeGenOpts(),
3027 T: getTarget().getTriple())) {
3028 case CodeGenOptions::SmallIntABICompatKind::Platform:
3029 if (Ty->isBooleanType()) {
3030 // Bool should be extended to 8-bits. We can't express this right
3031 // now, so use conservative extension.
3032 return ABIArgInfo::getConservativeExtend(Ty, T: CGT.ConvertType(T: Ty));
3033 }
3034 break;
3035 case CodeGenOptions::SmallIntABICompatKind::Clang23:
3036 return ABIArgInfo::getExtend(Ty, T: CGT.ConvertType(T: Ty));
3037 case CodeGenOptions::SmallIntABICompatKind::Both:
3038 return ABIArgInfo::getConservativeExtend(Ty, T: CGT.ConvertType(T: Ty));
3039 }
3040 }
3041 }
3042
3043 if (ResType->isIntegerTy(BitWidth: 128)) {
3044 assert(Hi == Integer);
3045 ++neededInt;
3046 return ABIArgInfo::getDirect(T: ResType);
3047 }
3048 break;
3049
3050 // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
3051 // available SSE register is used, the registers are taken in the
3052 // order from %xmm0 to %xmm7.
3053 case SSE: {
3054 llvm::Type *IRType = CGT.ConvertType(T: Ty);
3055 ResType = GetSSETypeAtOffset(IRType, IROffset: 0, SourceTy: Ty, SourceOffset: 0);
3056 ++neededSSE;
3057 break;
3058 }
3059 }
3060
3061 llvm::Type *HighPart = nullptr;
3062 switch (Hi) {
3063 // Memory was handled previously, ComplexX87 and X87 should
3064 // never occur as hi classes, and X87Up must be preceded by X87,
3065 // which is passed in memory.
3066 case Memory:
3067 case X87:
3068 case ComplexX87:
3069 llvm_unreachable("Invalid classification for hi word.");
3070
3071 case NoClass: break;
3072
3073 case Integer:
3074 ++neededInt;
3075 // Pick an 8-byte type based on the preferred type.
3076 HighPart = GetINTEGERTypeAtOffset(IRType: CGT.ConvertType(T: Ty), IROffset: 8, SourceTy: Ty, SourceOffset: 8);
3077
3078 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
3079 return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
3080 break;
3081
3082 // X87Up generally doesn't occur here (long double is passed in
3083 // memory), except in situations involving unions.
3084 case X87Up:
3085 case SSE:
3086 ++neededSSE;
3087 HighPart = GetSSETypeAtOffset(IRType: CGT.ConvertType(T: Ty), IROffset: 8, SourceTy: Ty, SourceOffset: 8);
3088
3089 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
3090 return ABIArgInfo::getDirect(T: HighPart, Offset: 8);
3091 break;
3092
3093 // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
3094 // eightbyte is passed in the upper half of the last used SSE
3095 // register. This only happens when 128-bit vectors are passed.
3096 case SSEUp:
3097 assert(Lo == SSE && "Unexpected SSEUp classification");
3098 ResType = GetByteVectorType(Ty);
3099 break;
3100 }
3101
3102 // If a high part was specified, merge it together with the low part. It is
3103 // known to pass in the high eightbyte of the result. We do this by forming a
3104 // first class struct aggregate with the high and low part: {low, high}
3105 if (HighPart)
3106 ResType = GetX86_64ByValArgumentPair(Lo: ResType, Hi: HighPart, TD: getDataLayout());
3107
3108 return ABIArgInfo::getDirect(T: ResType);
3109}
3110
3111// Returns true if the struct can be passed directly in registers. If so, the
3112// number of registers required will be returned in `NeededInt` and `NeededSSE`,
3113// and `CoerceElts` will contain an expanded sequence of LLVM IR types that each
3114// field should coerce to.
3115bool X86_64ABIInfo::passRegCallStructTypeDirectly(
3116 QualType Ty, SmallVectorImpl<llvm::Type *> &CoerceElts, unsigned &NeededInt,
3117 unsigned &NeededSSE, unsigned &MaxVectorWidth) const {
3118
3119 auto *RD =
3120 cast<RecordType>(Val: Ty.getCanonicalType())->getDecl()->getDefinitionOrSelf();
3121 if (RD->hasFlexibleArrayMember())
3122 return false;
3123
3124 // Classify the bases.
3125 if (auto CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
3126 if (CXXRD->isDynamicClass())
3127 return false;
3128
3129 for (const auto &I : CXXRD->bases()) {
3130 QualType BaseTy = I.getType();
3131 if (isEmptyRecord(Context&: getContext(), T: BaseTy, AllowArrays: true))
3132 continue;
3133 if (!passRegCallStructTypeDirectly(Ty: BaseTy, CoerceElts, NeededInt,
3134 NeededSSE, MaxVectorWidth))
3135 return false;
3136 }
3137 }
3138
3139 // Classify the members.
3140 for (const auto *FD : RD->fields()) {
3141 QualType MTy = FD->getType();
3142 if (MTy->isRecordType() && !MTy->isUnionType()) {
3143 if (isEmptyRecord(Context&: getContext(), T: MTy, AllowArrays: true))
3144 continue;
3145 if (!passRegCallStructTypeDirectly(Ty: MTy, CoerceElts, NeededInt, NeededSSE,
3146 MaxVectorWidth))
3147 return false;
3148 continue;
3149 }
3150
3151 const auto *AT = getContext().getAsConstantArrayType(T: MTy);
3152 if (AT)
3153 MTy = AT->getElementType();
3154
3155 unsigned LocalNeededInt, LocalNeededSSE;
3156 ABIArgInfo AI = classifyArgumentType(Ty: MTy, UINT_MAX, neededInt&: LocalNeededInt,
3157 neededSSE&: LocalNeededSSE, isNamedArg: true, IsRegCall: true);
3158 if (AI.isIgnore())
3159 continue;
3160 if (AI.isIndirect())
3161 return false;
3162
3163 llvm::Type *CoerceTy = AI.getCoerceToType();
3164 assert(CoerceTy && "ABI info for struct member has no coerce type");
3165 if (AT) {
3166 uint64_t NumElts = AT->getZExtSize();
3167 LocalNeededInt *= NumElts;
3168 LocalNeededSSE *= NumElts;
3169 CoerceElts.push_back(Elt: llvm::ArrayType::get(ElementType: CoerceTy, NumElements: NumElts));
3170 } else {
3171 CoerceElts.push_back(Elt: CoerceTy);
3172 }
3173
3174 if (const auto *VT = MTy->getAs<VectorType>())
3175 if (getContext().getTypeSize(T: VT) > MaxVectorWidth)
3176 MaxVectorWidth = getContext().getTypeSize(T: VT);
3177
3178 NeededInt += LocalNeededInt;
3179 NeededSSE += LocalNeededSSE;
3180 }
3181
3182 return true;
3183}
3184
3185ABIArgInfo
3186X86_64ABIInfo::classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
3187 unsigned &NeededSSE,
3188 unsigned &MaxVectorWidth) const {
3189 NeededInt = 0;
3190 NeededSSE = 0;
3191 MaxVectorWidth = 0;
3192
3193 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
3194 return ABIArgInfo::getIgnore();
3195
3196 SmallVector<llvm::Type *, 16> CoerceElts;
3197 if (!passRegCallStructTypeDirectly(Ty, CoerceElts, NeededInt, NeededSSE,
3198 MaxVectorWidth)) {
3199 NeededInt = NeededSSE = 0;
3200 return getIndirectReturnResult(Ty);
3201 }
3202
3203 assert(!CoerceElts.empty() && "Non-empty struct produced no element types");
3204 return ABIArgInfo::getDirect(
3205 T: llvm::StructType::get(Context&: getVMContext(), Elements: CoerceElts));
3206}
3207
3208unsigned
3209X86_64ABIInfo::getX86ABIAVXLevel(const FunctionDecl *FD,
3210 const FunctionType::ExtInfo &Info) const {
3211 return static_cast<unsigned>(getEffectiveX86AVXABILevel(CGT, GlobalAVXLevel: AVXLevel, FD));
3212}
3213
3214void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3215 const unsigned CallingConv = FI.getCallingConvention();
3216 // It is possible to force Win64 calling convention on any x86_64 target by
3217 // using __attribute__((ms_abi)). In such case to correctly emit Win64
3218 // compatible code delegate this call to WinX86_64ABIInfo::computeInfo.
3219 if (CallingConv == llvm::CallingConv::Win64) {
3220 WinX86_64ABIInfo Win64ABIInfo(CGT, AVXLevel);
3221 Win64ABIInfo.computeInfo(FI);
3222 return;
3223 }
3224
3225 assert(FI.getX86ABIAVXLevel() <=
3226 static_cast<unsigned>(X86AVXABILevel::AVX512) &&
3227 "Unexpected X86 AVX ABI level");
3228 X86AVXABILevel EffectiveAVXLevel =
3229 static_cast<X86AVXABILevel>(FI.getX86ABIAVXLevel());
3230 if (EffectiveAVXLevel != AVXLevel) {
3231 X86_64ABIInfo EffectiveABIInfo(CGT, EffectiveAVXLevel);
3232 EffectiveABIInfo.computeInfo(FI);
3233 return;
3234 }
3235
3236 bool IsRegCall = CallingConv == llvm::CallingConv::X86_RegCall;
3237
3238 // Keep track of the number of assigned registers.
3239 unsigned FreeIntRegs = IsRegCall ? 11 : 6;
3240 unsigned FreeSSERegs = IsRegCall ? 16 : 8;
3241 unsigned NeededInt = 0, NeededSSE = 0, MaxVectorWidth = 0;
3242
3243 if (!::classifyReturnType(CXXABI: getCXXABI(), FI, Info: *this)) {
3244 if (IsRegCall && FI.getReturnType()->getTypePtr()->isRecordType() &&
3245 !FI.getReturnType()->getTypePtr()->isUnionType()) {
3246 FI.getReturnInfo() = classifyRegCallStructType(
3247 Ty: FI.getReturnType(), NeededInt, NeededSSE, MaxVectorWidth);
3248 if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3249 FreeIntRegs -= NeededInt;
3250 FreeSSERegs -= NeededSSE;
3251 } else {
3252 FI.getReturnInfo() = getIndirectReturnResult(Ty: FI.getReturnType());
3253 }
3254 } else if (IsRegCall && FI.getReturnType()->getAs<ComplexType>() &&
3255 getContext().getCanonicalType(T: FI.getReturnType()
3256 ->getAs<ComplexType>()
3257 ->getElementType()) ==
3258 getContext().LongDoubleTy)
3259 // Complex Long Double Type is passed in Memory when Regcall
3260 // calling convention is used.
3261 FI.getReturnInfo() = getIndirectReturnResult(Ty: FI.getReturnType());
3262 else
3263 FI.getReturnInfo() = classifyReturnType(RetTy: FI.getReturnType());
3264 }
3265
3266 // If the return value is indirect, then the hidden argument is consuming one
3267 // integer register.
3268 if (FI.getReturnInfo().isIndirect())
3269 --FreeIntRegs;
3270 else if (NeededSSE && MaxVectorWidth > 0)
3271 FI.setMaxVectorWidth(MaxVectorWidth);
3272
3273 // The chain argument effectively gives us another free register.
3274 if (FI.isChainCall())
3275 ++FreeIntRegs;
3276
3277 // RegCall lets us reuse the return registers.
3278 if (IsRegCall)
3279 FreeSSERegs = 16;
3280
3281 unsigned NumRequiredArgs = FI.getNumRequiredArgs();
3282 // AMD64-ABI 3.2.3p3: Once arguments are classified, the registers
3283 // get assigned (in left-to-right order) for passing as follows...
3284 unsigned ArgNo = 0;
3285 for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
3286 it != ie; ++it, ++ArgNo) {
3287 bool IsNamedArg = ArgNo < NumRequiredArgs;
3288
3289 if (IsRegCall && it->type->isStructureOrClassType())
3290 it->info = classifyRegCallStructType(Ty: it->type, NeededInt, NeededSSE,
3291 MaxVectorWidth);
3292 else
3293 it->info = classifyArgumentType(Ty: it->type, freeIntRegs: FreeIntRegs, neededInt&: NeededInt,
3294 neededSSE&: NeededSSE, isNamedArg: IsNamedArg);
3295
3296 // AMD64-ABI 3.2.3p3: If there are no registers available for any
3297 // eightbyte of an argument, the whole argument is passed on the
3298 // stack. If registers have already been assigned for some
3299 // eightbytes of such an argument, the assignments get reverted.
3300 if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3301 FreeIntRegs -= NeededInt;
3302 FreeSSERegs -= NeededSSE;
3303 if (MaxVectorWidth > FI.getMaxVectorWidth())
3304 FI.setMaxVectorWidth(MaxVectorWidth);
3305 } else {
3306 it->info = getIndirectResult(Ty: it->type, freeIntRegs: FreeIntRegs);
3307 }
3308 }
3309}
3310
3311static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF,
3312 Address VAListAddr, QualType Ty) {
3313 Address overflow_arg_area_p =
3314 CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 2, Name: "overflow_arg_area_p");
3315 llvm::Value *overflow_arg_area =
3316 CGF.Builder.CreateLoad(Addr: overflow_arg_area_p, Name: "overflow_arg_area");
3317
3318 // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
3319 // byte boundary if alignment needed by type exceeds 8 byte boundary.
3320 // It isn't stated explicitly in the standard, but in practice we use
3321 // alignment greater than 16 where necessary.
3322 CharUnits Align = CGF.getContext().getTypeAlignInChars(T: Ty);
3323 if (Align > CharUnits::fromQuantity(Quantity: 8)) {
3324 overflow_arg_area = emitRoundPointerUpToAlignment(CGF, Ptr: overflow_arg_area,
3325 Align);
3326 }
3327
3328 // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
3329 llvm::Type *LTy = CGF.ConvertTypeForMem(T: Ty);
3330 llvm::Value *Res = overflow_arg_area;
3331
3332 // AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
3333 // l->overflow_arg_area + sizeof(type).
3334 // AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
3335 // an 8 byte boundary.
3336
3337 uint64_t SizeInBytes = (CGF.getContext().getTypeSize(T: Ty) + 7) / 8;
3338 llvm::Value *Offset =
3339 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: (SizeInBytes + 7) & ~7);
3340 overflow_arg_area = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: overflow_arg_area,
3341 IdxList: Offset, Name: "overflow_arg_area.next");
3342 CGF.Builder.CreateStore(Val: overflow_arg_area, Addr: overflow_arg_area_p);
3343
3344 // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
3345 return Address(Res, LTy, Align);
3346}
3347
3348RValue X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
3349 QualType Ty, AggValueSlot Slot) const {
3350 // Assume that va_list type is correct; should be pointer to LLVM type:
3351 // struct {
3352 // i32 gp_offset;
3353 // i32 fp_offset;
3354 // i8* overflow_arg_area;
3355 // i8* reg_save_area;
3356 // };
3357 unsigned neededInt, neededSSE;
3358
3359 Ty = getContext().getCanonicalType(T: Ty);
3360 ABIArgInfo AI = classifyArgumentType(Ty, freeIntRegs: 0, neededInt, neededSSE,
3361 /*isNamedArg*/false);
3362
3363 // Empty records are ignored for parameter passing purposes.
3364 if (AI.isIgnore())
3365 return Slot.asRValue();
3366
3367 // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
3368 // in the registers. If not go to step 7.
3369 if (!neededInt && !neededSSE)
3370 return CGF.EmitLoadOfAnyValue(
3371 V: CGF.MakeAddrLValue(Addr: EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty), T: Ty),
3372 Slot);
3373
3374 // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
3375 // general purpose registers needed to pass type and num_fp to hold
3376 // the number of floating point registers needed.
3377
3378 // AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
3379 // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
3380 // l->fp_offset > 304 - num_fp * 16 go to step 7.
3381 //
3382 // NOTE: 304 is a typo, there are (6 * 8 + 8 * 16) = 176 bytes of
3383 // register save space).
3384
3385 llvm::Value *InRegs = nullptr;
3386 Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid();
3387 llvm::Value *gp_offset = nullptr, *fp_offset = nullptr;
3388 if (neededInt) {
3389 gp_offset_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 0, Name: "gp_offset_p");
3390 gp_offset = CGF.Builder.CreateLoad(Addr: gp_offset_p, Name: "gp_offset");
3391 InRegs = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 48 - neededInt * 8);
3392 InRegs = CGF.Builder.CreateICmpULE(LHS: gp_offset, RHS: InRegs, Name: "fits_in_gp");
3393 }
3394
3395 if (neededSSE) {
3396 fp_offset_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 1, Name: "fp_offset_p");
3397 fp_offset = CGF.Builder.CreateLoad(Addr: fp_offset_p, Name: "fp_offset");
3398 llvm::Value *FitsInFP =
3399 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 176 - neededSSE * 16);
3400 FitsInFP = CGF.Builder.CreateICmpULE(LHS: fp_offset, RHS: FitsInFP, Name: "fits_in_fp");
3401 InRegs = InRegs ? CGF.Builder.CreateAnd(LHS: InRegs, RHS: FitsInFP) : FitsInFP;
3402 }
3403
3404 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock(name: "vaarg.in_reg");
3405 llvm::BasicBlock *InMemBlock = CGF.createBasicBlock(name: "vaarg.in_mem");
3406 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "vaarg.end");
3407 CGF.Builder.CreateCondBr(Cond: InRegs, True: InRegBlock, False: InMemBlock);
3408
3409 // Emit code to load the value if it was passed in registers.
3410
3411 CGF.EmitBlock(BB: InRegBlock);
3412
3413 // AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
3414 // an offset of l->gp_offset and/or l->fp_offset. This may require
3415 // copying to a temporary location in case the parameter is passed
3416 // in different register classes or requires an alignment greater
3417 // than 8 for general purpose registers and 16 for XMM registers.
3418 //
3419 // FIXME: This really results in shameful code when we end up needing to
3420 // collect arguments from different places; often what should result in a
3421 // simple assembling of a structure from scattered addresses has many more
3422 // loads than necessary. Can we clean this up?
3423 llvm::Type *LTy = CGF.ConvertTypeForMem(T: Ty);
3424 llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(
3425 Addr: CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 3), Name: "reg_save_area");
3426
3427 Address RegAddr = Address::invalid();
3428 if (neededInt && neededSSE) {
3429 // FIXME: Cleanup.
3430 assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
3431 llvm::StructType *ST = cast<llvm::StructType>(Val: AI.getCoerceToType());
3432 Address Tmp = CGF.CreateMemTempWithoutCast(T: Ty);
3433 Tmp = Tmp.withElementType(ElemTy: ST);
3434 assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
3435 llvm::Type *TyLo = ST->getElementType(N: 0);
3436 llvm::Type *TyHi = ST->getElementType(N: 1);
3437 assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&
3438 "Unexpected ABI info for mixed regs");
3439 llvm::Value *GPAddr =
3440 CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: gp_offset);
3441 llvm::Value *FPAddr =
3442 CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: fp_offset);
3443 llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? FPAddr : GPAddr;
3444 llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? GPAddr : FPAddr;
3445
3446 // Copy the first element.
3447 // FIXME: Our choice of alignment here and below is probably pessimistic.
3448 llvm::Value *V = CGF.Builder.CreateAlignedLoad(
3449 Ty: TyLo, Addr: RegLoAddr,
3450 Align: CharUnits::fromQuantity(Quantity: getDataLayout().getABITypeAlign(Ty: TyLo)));
3451 CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 0));
3452
3453 // Copy the second element.
3454 V = CGF.Builder.CreateAlignedLoad(
3455 Ty: TyHi, Addr: RegHiAddr,
3456 Align: CharUnits::fromQuantity(Quantity: getDataLayout().getABITypeAlign(Ty: TyHi)));
3457 CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 1));
3458
3459 RegAddr = Tmp.withElementType(ElemTy: LTy);
3460 } else if (neededInt || neededSSE == 1) {
3461 // Copy to a temporary if necessary to ensure the appropriate alignment.
3462 auto TInfo = getContext().getTypeInfoInChars(T: Ty);
3463 uint64_t TySize = TInfo.Width.getQuantity();
3464 CharUnits TyAlign = TInfo.Align;
3465 llvm::Type *CoTy = nullptr;
3466 if (AI.isDirect())
3467 CoTy = AI.getCoerceToType();
3468
3469 llvm::Value *GpOrFpOffset = neededInt ? gp_offset : fp_offset;
3470 uint64_t Alignment = neededInt ? 8 : 16;
3471 uint64_t RegSize = neededInt ? neededInt * 8 : 16;
3472 // There are two cases require special handling:
3473 // 1)
3474 // ```
3475 // struct {
3476 // struct {} a[8];
3477 // int b;
3478 // };
3479 // ```
3480 // The lower 8 bytes of the structure are not stored,
3481 // so an 8-byte offset is needed when accessing the structure.
3482 // 2)
3483 // ```
3484 // struct {
3485 // long long a;
3486 // struct {} b;
3487 // };
3488 // ```
3489 // The stored size of this structure is smaller than its actual size,
3490 // which may lead to reading past the end of the register save area.
3491 if (CoTy && (AI.getDirectOffset() == 8 || RegSize < TySize)) {
3492 Address Tmp = CGF.CreateMemTempWithoutCast(T: Ty);
3493 llvm::Value *Addr =
3494 CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: GpOrFpOffset);
3495 llvm::Value *Src = CGF.Builder.CreateAlignedLoad(Ty: CoTy, Addr, Align: TyAlign);
3496 llvm::Value *PtrOffset =
3497 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: AI.getDirectOffset());
3498 Address Dst = Address(
3499 CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: Tmp.getBasePointer(), IdxList: PtrOffset),
3500 LTy, TyAlign);
3501 CGF.Builder.CreateStore(Val: Src, Addr: Dst);
3502 RegAddr = Tmp.withElementType(ElemTy: LTy);
3503 } else {
3504 RegAddr =
3505 Address(CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea, IdxList: GpOrFpOffset),
3506 LTy, CharUnits::fromQuantity(Quantity: Alignment));
3507
3508 // Copy into a temporary if the type is more aligned than the
3509 // register save area.
3510 if (neededInt && TyAlign.getQuantity() > 8) {
3511 Address Tmp = CGF.CreateMemTempWithoutCast(T: Ty);
3512 CGF.Builder.CreateMemCpy(Dest: Tmp, Src: RegAddr, Size: TySize, IsVolatile: false);
3513 RegAddr = Tmp;
3514 }
3515 }
3516
3517 } else {
3518 assert(neededSSE == 2 && "Invalid number of needed registers!");
3519 // SSE registers are spaced 16 bytes apart in the register save
3520 // area, we need to collect the two eightbytes together.
3521 // The ABI isn't explicit about this, but it seems reasonable
3522 // to assume that the slots are 16-byte aligned, since the stack is
3523 // naturally 16-byte aligned and the prologue is expected to store
3524 // all the SSE registers to the RSA.
3525 Address RegAddrLo = Address(CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: RegSaveArea,
3526 IdxList: fp_offset),
3527 CGF.Int8Ty, CharUnits::fromQuantity(Quantity: 16));
3528 Address RegAddrHi =
3529 CGF.Builder.CreateConstInBoundsByteGEP(Addr: RegAddrLo,
3530 Offset: CharUnits::fromQuantity(Quantity: 16));
3531 llvm::Type *ST = AI.canHaveCoerceToType()
3532 ? AI.getCoerceToType()
3533 : llvm::StructType::get(elt1: CGF.DoubleTy, elts: CGF.DoubleTy);
3534 llvm::Value *V;
3535 Address Tmp = CGF.CreateMemTempWithoutCast(T: Ty);
3536 Tmp = Tmp.withElementType(ElemTy: ST);
3537 V = CGF.Builder.CreateLoad(
3538 Addr: RegAddrLo.withElementType(ElemTy: ST->getStructElementType(N: 0)));
3539 CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 0));
3540 V = CGF.Builder.CreateLoad(
3541 Addr: RegAddrHi.withElementType(ElemTy: ST->getStructElementType(N: 1)));
3542 CGF.Builder.CreateStore(Val: V, Addr: CGF.Builder.CreateStructGEP(Addr: Tmp, Index: 1));
3543
3544 RegAddr = Tmp.withElementType(ElemTy: LTy);
3545 }
3546
3547 // AMD64-ABI 3.5.7p5: Step 5. Set:
3548 // l->gp_offset = l->gp_offset + num_gp * 8
3549 // l->fp_offset = l->fp_offset + num_fp * 16.
3550 if (neededInt) {
3551 llvm::Value *Offset = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: neededInt * 8);
3552 CGF.Builder.CreateStore(Val: CGF.Builder.CreateAdd(LHS: gp_offset, RHS: Offset),
3553 Addr: gp_offset_p);
3554 }
3555 if (neededSSE) {
3556 llvm::Value *Offset = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: neededSSE * 16);
3557 CGF.Builder.CreateStore(Val: CGF.Builder.CreateAdd(LHS: fp_offset, RHS: Offset),
3558 Addr: fp_offset_p);
3559 }
3560 CGF.EmitBranch(Block: ContBlock);
3561
3562 // Emit code to load the value if it was passed in memory.
3563
3564 CGF.EmitBlock(BB: InMemBlock);
3565 Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
3566
3567 // Return the appropriate result.
3568
3569 CGF.EmitBlock(BB: ContBlock);
3570 Address ResAddr = emitMergePHI(CGF, Addr1: RegAddr, Block1: InRegBlock, Addr2: MemAddr, Block2: InMemBlock,
3571 Name: "vaarg.addr");
3572 return CGF.EmitLoadOfAnyValue(V: CGF.MakeAddrLValue(Addr: ResAddr, T: Ty), Slot);
3573}
3574
3575RValue X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
3576 QualType Ty, AggValueSlot Slot) const {
3577 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3578 // not 1, 2, 4, or 8 bytes, must be passed by reference."
3579 uint64_t Width = getContext().getTypeSize(T: Ty);
3580 bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Value: Width);
3581
3582 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
3583 ValueInfo: CGF.getContext().getTypeInfoInChars(T: Ty),
3584 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 8),
3585 /*allowHigherAlign*/ AllowHigherAlign: false, Slot);
3586}
3587
3588ABIArgInfo WinX86_64ABIInfo::reclassifyHvaArgForVectorCall(
3589 QualType Ty, unsigned &FreeSSERegs, const ABIArgInfo &current) const {
3590 const Type *Base = nullptr;
3591 uint64_t NumElts = 0;
3592
3593 if (!Ty->isBuiltinType() && !Ty->isVectorType() &&
3594 isHomogeneousAggregate(Ty, Base, Members&: NumElts) && FreeSSERegs >= NumElts) {
3595 FreeSSERegs -= NumElts;
3596 return getDirectX86Hva();
3597 }
3598 return current;
3599}
3600
3601ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
3602 ClassifyKind Kind, unsigned CC) const {
3603 bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
3604 bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
3605
3606 if (Ty->isVoidType())
3607 return ABIArgInfo::getIgnore();
3608
3609 bool PromoteScopedEnum = false;
3610 if (const auto *ED = Ty->getAsEnumDecl()) {
3611 Ty = ED->getIntegerType();
3612 PromoteScopedEnum = Kind == ClassifyKind::VarArg && ED->isScoped() &&
3613 getContext().isPromotableIntegerType(T: Ty);
3614 }
3615
3616 // MSVC extends scoped enums with a sub-int underlying type when they are
3617 // passed through an ellipsis. Unlike unscoped enums, scoped enums are not
3618 // subject to the language's default argument promotions, so handle the
3619 // extension as part of the ABI classification.
3620 if (PromoteScopedEnum)
3621 return ABIArgInfo::getExtend(Ty);
3622
3623 TypeInfo Info = getContext().getTypeInfo(T: Ty);
3624 uint64_t Width = Info.Width;
3625 CharUnits Align = getContext().toCharUnitsFromBits(BitSize: Info.Align);
3626
3627 const RecordType *RT = Ty->getAsCanonical<RecordType>();
3628 if (RT) {
3629 if (Kind != ClassifyKind::Return) {
3630 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, CXXABI&: getCXXABI()))
3631 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
3632 ByVal: RAA == CGCXXABI::RAA_DirectInMemory);
3633 }
3634
3635 if (RT->getDecl()->getDefinitionOrSelf()->hasFlexibleArrayMember())
3636 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
3637 /*ByVal=*/false);
3638 }
3639
3640 const Type *Base = nullptr;
3641 uint64_t NumElts = 0;
3642 // vectorcall adds the concept of a homogenous vector aggregate, similar to
3643 // other targets.
3644 if ((IsVectorCall || IsRegCall) &&
3645 isHomogeneousAggregate(Ty, Base, Members&: NumElts)) {
3646 if (IsRegCall) {
3647 if (FreeSSERegs >= NumElts) {
3648 FreeSSERegs -= NumElts;
3649 if (Kind == ClassifyKind::Return || Ty->isBuiltinType() ||
3650 Ty->isVectorType())
3651 return ABIArgInfo::getDirect();
3652 return ABIArgInfo::getExpand();
3653 }
3654 return ABIArgInfo::getIndirect(
3655 Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3656 /*ByVal=*/false);
3657 } else if (IsVectorCall) {
3658 if (FreeSSERegs >= NumElts &&
3659 (Kind == ClassifyKind::Return || Ty->isBuiltinType() ||
3660 Ty->isVectorType())) {
3661 FreeSSERegs -= NumElts;
3662 return ABIArgInfo::getDirect();
3663 } else if (Kind == ClassifyKind::Return) {
3664 return ABIArgInfo::getExpand();
3665 } else if (!Ty->isBuiltinType() && !Ty->isVectorType()) {
3666 // HVAs are delayed and reclassified in the 2nd step.
3667 return ABIArgInfo::getIndirect(
3668 Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3669 /*ByVal=*/false);
3670 }
3671 }
3672 }
3673
3674 if (Ty->isMemberPointerType()) {
3675 // If the member pointer is represented by an LLVM int or ptr, pass it
3676 // directly.
3677 llvm::Type *LLTy = CGT.ConvertType(T: Ty);
3678 if (LLTy->isPointerTy() || LLTy->isIntegerTy())
3679 return ABIArgInfo::getDirect();
3680 }
3681
3682 if (RT || Ty->isAnyComplexType() || Ty->isMemberPointerType()) {
3683 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3684 // not 1, 2, 4, or 8 bytes, must be passed by reference."
3685 if (Width > 64 || !llvm::isPowerOf2_64(Value: Width))
3686 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
3687 /*ByVal=*/false);
3688
3689 // Otherwise, coerce it to a small integer.
3690 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Width));
3691 }
3692
3693 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
3694 switch (BT->getKind()) {
3695 case BuiltinType::Bool:
3696 // Bool type is always extended to the ABI, other builtin types are not
3697 // extended.
3698 return ABIArgInfo::getExtend(Ty);
3699
3700 case BuiltinType::LongDouble:
3701 // Mingw64 GCC uses the old 80 bit extended precision floating point
3702 // unit. It passes them indirectly through memory.
3703 if (IsMingw64) {
3704 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
3705 if (LDF == &llvm::APFloat::x87DoubleExtended())
3706 return ABIArgInfo::getIndirect(
3707 Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3708 /*ByVal=*/false);
3709 }
3710 break;
3711
3712 case BuiltinType::Int128:
3713 case BuiltinType::UInt128:
3714 case BuiltinType::Float128:
3715 // If it's a parameter type, the normal ABI rule is that arguments larger
3716 // than 8 bytes are passed indirectly. GCC follows it. We follow it too,
3717 // even though it isn't particularly efficient.
3718 if (Kind != ClassifyKind::Return)
3719 return ABIArgInfo::getIndirect(
3720 Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3721 /*ByVal=*/false);
3722
3723 // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
3724 // Clang matches them for compatibility.
3725 if (BT->getKind() == BuiltinType::Int128 ||
3726 BT->getKind() == BuiltinType::UInt128)
3727 return ABIArgInfo::getDirect(T: llvm::FixedVectorType::get(
3728 ElementType: llvm::Type::getInt64Ty(C&: getVMContext()), NumElts: 2));
3729
3730 // Mingw64 GCC returns f128 via sret, and Clang matches that for
3731 // compatibility. This mirrors the X86 backend's CanLowerReturn logic.
3732 if (BT->getKind() == BuiltinType::Float128) {
3733 auto IsWin64F128StackCC = [this](unsigned CC) -> bool {
3734 switch (CC) {
3735 case llvm::CallingConv::Win64:
3736 return true;
3737 case llvm::CallingConv::C:
3738 return getTarget().getTriple().isOSWindowsOrUEFI();
3739 default:
3740 return false;
3741 }
3742 };
3743
3744 if (IsWin64F128StackCC(CC))
3745 return getNaturalAlignIndirect(
3746 Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(), /*ByVal=*/false);
3747 }
3748 break;
3749
3750 default:
3751 break;
3752 }
3753 }
3754
3755 if (Ty->isBitIntType()) {
3756 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3757 // not 1, 2, 4, or 8 bytes, must be passed by reference."
3758 // However, non-power-of-two bit-precise integers will be passed as 1, 2, 4,
3759 // or 8 bytes anyway as long is it fits in them, so we don't have to check
3760 // the power of 2.
3761 if (Width <= 64)
3762 return ABIArgInfo::getDirect();
3763 return ABIArgInfo::getIndirect(
3764 Alignment: Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
3765 /*ByVal=*/false);
3766 }
3767
3768 return ABIArgInfo::getDirect();
3769}
3770
3771unsigned
3772WinX86_64ABIInfo::getX86ABIAVXLevel(const FunctionDecl *FD,
3773 const FunctionType::ExtInfo &Info) const {
3774 if (Info.getCC() == CC_X86_64SysV) {
3775 return static_cast<unsigned>(getEffectiveX86AVXABILevel(CGT, GlobalAVXLevel: AVXLevel, FD));
3776 }
3777
3778 return static_cast<unsigned>(AVXLevel);
3779}
3780
3781void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3782 const unsigned CC = FI.getCallingConvention();
3783 bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
3784 bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
3785
3786 // If __attribute__((sysv_abi)) is in use, use the SysV argument
3787 // classification rules.
3788 if (CC == llvm::CallingConv::X86_64_SysV) {
3789 X86_64ABIInfo SysVABIInfo(CGT, AVXLevel);
3790 SysVABIInfo.computeInfo(FI);
3791 return;
3792 }
3793
3794 unsigned FreeSSERegs = 0;
3795 if (IsVectorCall) {
3796 // We can use up to 4 SSE return registers with vectorcall.
3797 FreeSSERegs = 4;
3798 } else if (IsRegCall) {
3799 // RegCall gives us 16 SSE registers.
3800 FreeSSERegs = 16;
3801 }
3802
3803 if (!getCXXABI().classifyReturnType(FI))
3804 FI.getReturnInfo() =
3805 classify(Ty: FI.getReturnType(), FreeSSERegs, Kind: ClassifyKind::Return, CC);
3806
3807 if (IsVectorCall) {
3808 // We can use up to 6 SSE register parameters with vectorcall.
3809 FreeSSERegs = 6;
3810 } else if (IsRegCall) {
3811 // RegCall gives us 16 SSE registers, we can reuse the return registers.
3812 FreeSSERegs = 16;
3813 }
3814
3815 unsigned ArgNum = 0;
3816 unsigned ZeroSSERegs = 0;
3817 for (auto &I : FI.arguments()) {
3818 // Vectorcall in x64 only permits the first 6 arguments to be passed as
3819 // XMM/YMM registers. After the sixth argument, pretend no vector
3820 // registers are left.
3821 unsigned *MaybeFreeSSERegs =
3822 (IsVectorCall && ArgNum >= 6) ? &ZeroSSERegs : &FreeSSERegs;
3823 ClassifyKind Kind = ArgNum >= FI.getNumRequiredArgs()
3824 ? ClassifyKind::VarArg
3825 : ClassifyKind::FixedArgument;
3826 I.info = classify(Ty: I.type, FreeSSERegs&: *MaybeFreeSSERegs, Kind, CC);
3827 ++ArgNum;
3828 }
3829
3830 if (IsVectorCall) {
3831 // For vectorcall, assign aggregate HVAs to any free vector registers in a
3832 // second pass.
3833 for (auto &I : FI.arguments())
3834 I.info = reclassifyHvaArgForVectorCall(Ty: I.type, FreeSSERegs, current: I.info);
3835 }
3836}
3837
3838RValue WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
3839 QualType Ty, AggValueSlot Slot) const {
3840 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
3841 // not 1, 2, 4, or 8 bytes, must be passed by reference."
3842 uint64_t Width = getContext().getTypeSize(T: Ty);
3843 bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Value: Width);
3844
3845 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
3846 ValueInfo: CGF.getContext().getTypeInfoInChars(T: Ty),
3847 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 8),
3848 /*allowHigherAlign*/ AllowHigherAlign: false, Slot);
3849}
3850
3851std::unique_ptr<TargetCodeGenInfo> CodeGen::createX86_32TargetCodeGenInfo(
3852 CodeGenModule &CGM, bool DarwinVectorABI, bool Win32StructABI,
3853 unsigned NumRegisterParameters, bool SoftFloatABI) {
3854 bool RetSmallStructInRegABI = X86_32TargetCodeGenInfo::isStructReturnInRegABI(
3855 Triple: CGM.getTriple(), Opts: CGM.getCodeGenOpts());
3856 return std::make_unique<X86_32TargetCodeGenInfo>(
3857 args&: CGM.getTypes(), args&: DarwinVectorABI, args&: RetSmallStructInRegABI, args&: Win32StructABI,
3858 args&: NumRegisterParameters, args&: SoftFloatABI);
3859}
3860
3861std::unique_ptr<TargetCodeGenInfo> CodeGen::createWinX86_32TargetCodeGenInfo(
3862 CodeGenModule &CGM, bool DarwinVectorABI, bool Win32StructABI,
3863 unsigned NumRegisterParameters) {
3864 bool RetSmallStructInRegABI = X86_32TargetCodeGenInfo::isStructReturnInRegABI(
3865 Triple: CGM.getTriple(), Opts: CGM.getCodeGenOpts());
3866 return std::make_unique<WinX86_32TargetCodeGenInfo>(
3867 args&: CGM.getTypes(), args&: DarwinVectorABI, args&: RetSmallStructInRegABI, args&: Win32StructABI,
3868 args&: NumRegisterParameters);
3869}
3870
3871std::unique_ptr<TargetCodeGenInfo>
3872CodeGen::createX86_64TargetCodeGenInfo(CodeGenModule &CGM,
3873 X86AVXABILevel AVXLevel) {
3874 return std::make_unique<X86_64TargetCodeGenInfo>(args&: CGM.getTypes(), args&: AVXLevel);
3875}
3876
3877std::unique_ptr<TargetCodeGenInfo>
3878CodeGen::createWinX86_64TargetCodeGenInfo(CodeGenModule &CGM,
3879 X86AVXABILevel AVXLevel) {
3880 return std::make_unique<WinX86_64TargetCodeGenInfo>(args&: CGM.getTypes(), args&: AVXLevel);
3881}
3882