1//===- X86.cpp ------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ABI/FunctionInfo.h"
10#include "llvm/ABI/TargetInfo.h"
11#include "llvm/ABI/Types.h"
12#include "llvm/Support/Alignment.h"
13#include "llvm/Support/Casting.h"
14#include "llvm/Support/ErrorHandling.h"
15#include "llvm/Support/MathExtras.h"
16#include "llvm/Support/TypeSize.h"
17#include <algorithm>
18#include <cassert>
19#include <cstdint>
20
21namespace llvm {
22namespace abi {
23
24static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
25 switch (AVXLevel) {
26 case X86AVXABILevel::AVX512:
27 return 512;
28 case X86AVXABILevel::AVX:
29 return 256;
30 case X86AVXABILevel::None:
31 return 128;
32 }
33 llvm_unreachable("Unknown AVXLevel");
34}
35
36// The width of an integer's storage container, mirroring Clang's
37// ASTContext::getTypeSize. For a plain integer this is its bit width; for a
38// _BitInt(N) it is N rounded up to the type's alignment. The x86-64 _BitInt
39// max alignment is 64, so this clamp is target-specific and kept file-local.
40static uint64_t getClangIntegerWidthInBits(const IntegerType *IT) {
41 uint64_t NumBits = IT->getSizeInBits().getFixedValue();
42 if (!IT->isBitInt())
43 return NumBits;
44 uint64_t BitAlign =
45 std::max<uint64_t>(a: 8, b: std::min<uint64_t>(a: 64, b: llvm::bit_ceil(Value: NumBits)));
46 return llvm::alignTo(Value: NumBits, Align: BitAlign);
47}
48
49static uint64_t getClangVectorWidthInBits(const VectorType *VT) {
50 const Type *EltTy = VT->getElementType();
51 uint64_t EltWidth = EltTy->getSizeInBits().getFixedValue();
52 if (const auto *IT = dyn_cast<IntegerType>(Val: EltTy))
53 EltWidth = getClangIntegerWidthInBits(IT);
54 uint64_t Width =
55 std::max<uint64_t>(a: 8, b: EltWidth * VT->getNumElements().getKnownMinValue());
56 return llvm::bit_ceil(Value: Width);
57}
58
59// The storage-container width of a type, mirroring Clang's getTypeSize. Used on
60// the stack path so a _BitInt or illegal vector coerces to the integer covering
61// its storage, not its raw iN width.
62static uint64_t getClangTypeWidthInBits(const Type *Ty) {
63 if (const auto *VT = dyn_cast<VectorType>(Val: Ty))
64 return getClangVectorWidthInBits(VT);
65 if (const auto *IT = dyn_cast<IntegerType>(Val: Ty))
66 return getClangIntegerWidthInBits(IT);
67 return Ty->getSizeInBits().getFixedValue();
68}
69
70class X86_64TargetInfo : public TargetInfo {
71public:
72 enum Class { Integer, Sse, SseUp, X87, X87Up, ComplexX87, NoClass, Memory };
73
74private:
75 X86AVXABILevel AVXLevel;
76 bool Has64BitPointers;
77 X86ABICompatInfo X86CompatInfo;
78
79 static Class merge(Class Accum, Class Field);
80
81 void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
82
83 void classify(const Type *T, uint64_t OffsetBase, Class &Lo, Class &Hi,
84 bool IsNamedArg, bool IsRegCall = false) const;
85
86 const Type *getIntegerTypeAtOffset(const Type *IRType, unsigned IROffset,
87 const Type *SourceTy,
88 unsigned SourceOffset,
89 bool InMemory = false) const;
90
91 const Type *getSSETypeAtOffset(const Type *ABIType, unsigned ABIOffset,
92 const Type *SourceTy,
93 unsigned SourceOffset) const;
94 bool isIllegalVectorType(const Type *Ty) const;
95 bool containsMatrixField(const RecordType *RT) const;
96
97 void computeInfo(FunctionInfo &FI) const override;
98 ArgInfo getIndirectReturnResult(const Type *Ty) const;
99 const Type *getFPTypeAtOffset(const Type *Ty, unsigned Offset) const;
100
101 const Type *isSingleElementStruct(const Type *Ty) const;
102 const Type *getByteVectorType(const Type *Ty) const;
103
104 const Type *createPairType(const Type *Lo, const Type *Hi) const;
105 ArgInfo getIndirectResult(const Type *Ty, unsigned FreeIntRegs) const;
106
107 ArgInfo classifyReturnType(const Type *RetTy) const;
108
109 ArgInfo classifyArgumentType(const Type *Ty, unsigned FreeIntRegs,
110 unsigned &NeededInt, unsigned &NeededSse,
111 bool IsNamedArg, bool IsRegCall = false) const;
112
113public:
114 X86_64TargetInfo(TypeBuilder &TypeBuilder, X86AVXABILevel AVXABILevel,
115 bool Has64BitPtrs, const X86ABICompatInfo &Compat)
116 : TargetInfo(TypeBuilder), AVXLevel(AVXABILevel),
117 Has64BitPointers(Has64BitPtrs), X86CompatInfo(Compat) {}
118
119 bool has64BitPointers() const { return Has64BitPointers; }
120
121 const ABICompatInfo &getABICompatInfo() const override {
122 return X86CompatInfo;
123 }
124
125 const X86ABICompatInfo &getX86ABICompatInfo() const { return X86CompatInfo; }
126};
127
128static bool bitsContainNoUserData(const Type *Ty, unsigned StartBit,
129 unsigned EndBit);
130
131// Gets the "best" type to represent the union.
132static const Type *reduceUnionForX8664(const RecordType *UnionType,
133 TypeBuilder &TB) {
134 assert(UnionType->isUnion() && "Expected union type");
135
136 ArrayRef<FieldInfo> Fields = UnionType->getFields();
137 if (Fields.empty()) {
138 return nullptr;
139 }
140
141 const Type *StorageType = nullptr;
142
143 for (const auto &Field : Fields) {
144 if (Field.IsBitField && Field.IsUnnamedBitfield &&
145 Field.BitFieldWidth == 0) {
146 continue;
147 }
148
149 const Type *FieldType = Field.FieldType;
150
151 if (UnionType->isTransparentUnion() && !StorageType) {
152 StorageType = FieldType;
153 break;
154 }
155
156 // A member that holds no user data supplies no bytes for a coercion to
157 // read, so it must not become the storage type however wide or aligned it
158 // is declared. Clang compares lowered types instead, where an empty class
159 // is a byte array whose i8 leaf lets getIntegerTypeAtOffset narrow the
160 // coercion. A record mapped here holds no fields, so there is no such
161 // leaf and the eightbyte would be sized from the union.
162 if (bitsContainNoUserData(Ty: FieldType, StartBit: 0,
163 EndBit: FieldType->getSizeInBits().getFixedValue()))
164 continue;
165
166 if (!StorageType ||
167 FieldType->getAlignment() > StorageType->getAlignment() ||
168 (FieldType->getAlignment() == StorageType->getAlignment() &&
169 TypeSize::isKnownGT(LHS: FieldType->getSizeInBits(),
170 RHS: StorageType->getSizeInBits()))) {
171 StorageType = FieldType;
172 }
173 }
174 return StorageType;
175}
176
177void X86_64TargetInfo::postMerge(unsigned AggregateSize, Class &Lo,
178 Class &Hi) const {
179 // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
180 //
181 // (a) If one of the classes is Memory, the whole argument is passed in
182 // memory.
183 //
184 // (b) If X87Up is not preceded by X87, the whole argument is passed in
185 // memory.
186 //
187 // (c) If the size of the aggregate exceeds two eightbytes and the first
188 // eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
189 // argument is passed in memory. NOTE: This is necessary to keep the
190 // ABI working for processors that don't support the __m256 type.
191 //
192 // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
193 //
194 // Some of these are enforced by the merging logic. Others can arise
195 // only with unions; for example:
196 // union { _Complex double; unsigned; }
197 //
198 // Note that clauses (b) and (c) were added in 0.98.
199
200 if (Hi == Memory)
201 Lo = Memory;
202 if (Hi == X87Up && Lo != X87 && getX86ABICompatInfo().HonorsRevision98)
203 Lo = Memory;
204 if (AggregateSize > 128 && (Lo != Sse || Hi != SseUp))
205 Lo = Memory;
206 if (Hi == SseUp && Lo != Sse)
207 Hi = Sse;
208}
209X86_64TargetInfo::Class X86_64TargetInfo::merge(Class Accum, Class Field) {
210 // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
211 // classified recursively so that always two fields are
212 // considered. The resulting class is calculated according to
213 // the classes of the fields in the eightbyte:
214 //
215 // (a) If both classes are equal, this is the resulting class.
216 //
217 // (b) If one of the classes is NO_CLASS, the resulting class is
218 // the other class.
219 //
220 // (c) If one of the classes is MEMORY, the result is the MEMORY
221 // class.
222 //
223 // (d) If one of the classes is INTEGER, the result is the
224 // INTEGER.
225 //
226 // (e) If one of the classes is X87, X87Up, COMPLEX_X87 class,
227 // MEMORY is used as class.
228 //
229 // (f) Otherwise class SSE is used.
230
231 // Accum should never be memory (we should have returned) or
232 // ComplexX87 (because this cannot be passed in a structure).
233 assert((Accum != Memory && Accum != ComplexX87) &&
234 "Invalid accumulated classification during merge.");
235
236 if (Accum == Field || Field == NoClass)
237 return Accum;
238 if (Field == Memory)
239 return Memory;
240 if (Accum == NoClass)
241 return Field;
242 if (Accum == Integer || Field == Integer)
243 return Integer;
244 if (Field == X87 || Field == X87Up || Field == ComplexX87 || Accum == X87 ||
245 Accum == X87Up)
246 return Memory;
247
248 return Sse;
249}
250
251// A record with a matrix-extension field is passed in memory. clang has no
252// matrix-specific ABI code: a matrix falls through X86_64ABIInfo::classify to
253// the default MEMORY class. We model matrices as arrays, so this check
254// reproduces that record-with-matrix -> MEMORY result.
255bool X86_64TargetInfo::containsMatrixField(const RecordType *RT) const {
256 for (const auto &Field : RT->getFields()) {
257 const Type *FieldType = Field.FieldType;
258
259 if (const auto *AT = dyn_cast<ArrayType>(Val: FieldType)) {
260 if (AT->isMatrixType())
261 return true;
262 continue;
263 }
264
265 if (const auto *NestedRT = dyn_cast<RecordType>(Val: FieldType))
266 if (containsMatrixField(RT: NestedRT))
267 return true;
268 }
269 return false;
270}
271
272void X86_64TargetInfo::classify(const Type *T, uint64_t OffsetBase, Class &Lo,
273 Class &Hi, bool IsNamedArg,
274 bool IsRegCall) const {
275 Lo = Hi = NoClass;
276 Class &Current = OffsetBase < 64 ? Lo : Hi;
277 Current = Memory;
278
279 if (T->isVoid()) {
280 Current = NoClass;
281 return;
282 }
283
284 if (const auto *IT = dyn_cast<IntegerType>(Val: T)) {
285 auto BitWidth = IT->getSizeInBits().getFixedValue();
286
287 if (BitWidth == 128 ||
288 (IT->isBitInt() && BitWidth > 64 && BitWidth <= 128)) {
289 Lo = Integer;
290 Hi = Integer;
291 } else if (BitWidth <= 64) {
292 Current = Integer;
293 }
294
295 return;
296 }
297
298 if (const auto *FT = dyn_cast<FloatType>(Val: T)) {
299 const auto *FltSem = FT->getSemantics();
300
301 if (FltSem == &llvm::APFloat::IEEEsingle() ||
302 FltSem == &llvm::APFloat::IEEEdouble() ||
303 FltSem == &llvm::APFloat::IEEEhalf() ||
304 FltSem == &llvm::APFloat::BFloat()) {
305 Current = Sse;
306 } else if (FltSem == &llvm::APFloat::IEEEquad()) {
307 Lo = Sse;
308 Hi = SseUp;
309 } else if (FltSem == &llvm::APFloat::x87DoubleExtended()) {
310 Lo = X87;
311 Hi = X87Up;
312 } else {
313 Current = Sse;
314 }
315 return;
316 }
317 if (T->isPointer()) {
318 Current = Integer;
319 return;
320 }
321
322 if (const auto *MPT = dyn_cast<MemberPointerType>(Val: T)) {
323 if (MPT->isFunctionPointer()) {
324 if (Has64BitPointers) {
325 Lo = Hi = Integer;
326 } else {
327 uint64_t EbFuncPtr = OffsetBase / 64;
328 uint64_t EbThisAdj = (OffsetBase + 64 - 1) / 64;
329 if (EbFuncPtr != EbThisAdj) {
330 Lo = Hi = Integer;
331 } else {
332 Current = Integer;
333 }
334 }
335 } else {
336 Current = Integer;
337 }
338 return;
339 }
340
341 if (const auto *VT = dyn_cast<VectorType>(Val: T)) {
342 auto Size = VT->getSizeInBits().getFixedValue();
343 const Type *ElementType = VT->getElementType();
344
345 if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
346 // gcc passes the following as integer:
347 // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
348 // 2 bytes - <2 x char>, <1 x short>
349 // 1 byte - <1 x char>
350 Current = Integer;
351 // If this type crosses an eightbyte boundary, it should be
352 // split.
353 uint64_t EbLo = (OffsetBase) / 64;
354 uint64_t EbHi = (OffsetBase + Size - 1) / 64;
355 if (EbLo != EbHi)
356 Hi = Lo;
357 } else if (Size == 64) {
358 if (const auto *FT = dyn_cast<FloatType>(Val: ElementType)) {
359 // gcc passes <1 x double> in memory. :(
360 if (FT->getSemantics() == &llvm::APFloat::IEEEdouble())
361 return;
362 }
363
364 // gcc passes <1 x long long> as SSE but clang used to unconditionally
365 // pass them as integer. For platforms where clang is the de facto
366 // platform compiler, we must continue to use integer.
367 if (const auto *IT = dyn_cast<IntegerType>(Val: ElementType)) {
368 uint64_t ElemBits = IT->getSizeInBits().getFixedValue();
369 if (!getX86ABICompatInfo().ClassifyIntegerMMXAsSSE && ElemBits == 64 &&
370 !IT->isBitInt()) {
371 Current = Integer;
372 } else {
373 Current = Sse;
374 }
375 } else {
376 Current = Sse;
377 }
378 // If this type crosses an eightbyte boundary, it should be
379 // split.
380 if (OffsetBase && OffsetBase != 64)
381 Hi = Lo;
382 } else if (Size == 128 ||
383 (IsNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
384 if (const auto *IT = dyn_cast<IntegerType>(Val: ElementType)) {
385 uint64_t ElemBits = IT->getSizeInBits().getFixedValue();
386 // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
387 if (getX86ABICompatInfo().PassInt128VectorsInMem && Size != 128 &&
388 ElemBits == 128 && !IT->isBitInt())
389 return;
390 }
391
392 // Arguments of 256-bits are split into four eightbyte chunks. The
393 // least significant one belongs to class SSE and all the others to class
394 // SSEUP. The original Lo and Hi design considers that types can't be
395 // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
396 // This design isn't correct for 256-bits, but since there're no cases
397 // where the upper parts would need to be inspected, avoid adding
398 // complexity and just consider Hi to match the 64-256 part.
399 //
400 // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
401 // registers if they are "named", i.e. not part of the "..." of a
402 // variadic function.
403 //
404 // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
405 // split into eight eightbyte chunks, one SSE and seven SSEUP.
406 Lo = Sse;
407 Hi = SseUp;
408 }
409 return;
410 }
411
412 if (const auto *CT = dyn_cast<ComplexType>(Val: T)) {
413 const Type *ElementType = CT->getElementType();
414 uint64_t Size = T->getSizeInBits().getFixedValue();
415
416 if (isa<IntegerType>(Val: ElementType)) {
417 if (Size <= 64)
418 Current = Integer;
419 else if (Size <= 128)
420 Lo = Hi = Integer;
421 } else if (const auto *EFT = dyn_cast<FloatType>(Val: ElementType)) {
422 const auto *FltSem = EFT->getSemantics();
423 if (FltSem == &llvm::APFloat::IEEEhalf() ||
424 FltSem == &llvm::APFloat::IEEEsingle() ||
425 FltSem == &llvm::APFloat::BFloat())
426 Current = Sse;
427 else if (FltSem == &llvm::APFloat::IEEEquad())
428 Current = Memory;
429 else if (FltSem == &llvm::APFloat::x87DoubleExtended())
430 Current = ComplexX87;
431 else if (FltSem == &llvm::APFloat::IEEEdouble())
432 Lo = Hi = Sse;
433 else
434 llvm_unreachable("Unexpected long double representation!");
435 }
436
437 uint64_t ElementSize = ElementType->getSizeInBits().getFixedValue();
438 // If this complex type crosses an eightbyte boundary then it
439 // should be split.
440 uint64_t EbReal = OffsetBase / 64;
441 uint64_t EbImag = (OffsetBase + ElementSize) / 64;
442 if (Hi == NoClass && EbReal != EbImag)
443 Hi = Lo;
444
445 return;
446 }
447
448 if (const auto *AT = dyn_cast<ArrayType>(Val: T)) {
449 // A matrix type is modeled as an array but, like Clang, is treated as a
450 // non-aggregate scalar: it matches no class here and stays in the Memory
451 // class, so classify*Type later returns it Direct (coerced to its
452 // flattened vector) rather than classifying it field-by-field.
453 if (AT->isMatrixType())
454 return;
455
456 // Arrays are treated like structures.
457 uint64_t Size = AT->getSizeInBits().getFixedValue();
458
459 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
460 // than eight eightbytes, ..., it has class MEMORY.
461 // regcall ABI doesn't have limitation to an object. The only limitation
462 // is the free registers, which will be checked in computeInfo.
463 if (!IsRegCall && Size > 512)
464 return;
465
466 // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
467 // fields, it has class MEMORY.
468 //
469 // Only need to check alignment of array base.
470 const Type *ElementType = AT->getElementType();
471 uint64_t ElemAlign = ElementType->getAlignment().value() * 8;
472 if (OffsetBase % ElemAlign)
473 return;
474
475 // Otherwise implement simplified merge. We could be smarter about
476 // this, but it isn't worth it and would be harder to verify.
477 Current = NoClass;
478 uint64_t EltSize = ElementType->getSizeInBits().getFixedValue();
479 uint64_t ArraySize = AT->getNumElements();
480
481 // The only case a 256-bit wide vector could be used is when the array
482 // contains a single 256-bit element. Since Lo and Hi logic isn't extended
483 // to work for sizes wider than 128, early check and fallback to memory.
484 //
485 if (Size > 128 &&
486 (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))
487 return;
488
489 for (uint64_t I = 0, Offset = OffsetBase; I < ArraySize;
490 ++I, Offset += EltSize) {
491 Class FieldLo, FieldHi;
492 classify(T: ElementType, OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, IsNamedArg);
493 Lo = merge(Accum: Lo, Field: FieldLo);
494 Hi = merge(Accum: Hi, Field: FieldHi);
495 if (Lo == Memory || Hi == Memory)
496 break;
497 }
498 postMerge(AggregateSize: Size, Lo, Hi);
499 assert((Hi != SseUp || Lo == Sse) && "Invalid SseUp array classification.");
500 return;
501 }
502
503 if (const auto *RT = dyn_cast<RecordType>(Val: T)) {
504 uint64_t Size = RT->getSizeInBits().getFixedValue();
505
506 if (containsMatrixField(RT)) {
507 Lo = Memory;
508 return;
509 }
510
511 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
512 // than eight eightbytes, ..., it has class MEMORY.
513 if (Size > 512)
514 return;
515
516 // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
517 // copy constructor or a non-trivial destructor, it is passed by invisible
518 // reference.
519 if (getRecordArgABI(RT))
520 return;
521
522 // Assume variable sized types are passed in memory.
523 if (RT->hasFlexibleArrayMember())
524 return;
525
526 // Reset Lo class, this will be recomputed.
527 Current = NoClass;
528
529 // If this is a C++ record, classify the bases first.
530 if (RT->isCXXRecord()) {
531 for (const auto &Base : RT->getBaseClasses()) {
532
533 // Classify this field.
534 //
535 // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
536 // single eightbyte, each is classified separately. Each eightbyte gets
537 // initialized to class NO_CLASS.
538 Class FieldLo, FieldHi;
539 uint64_t Offset = OffsetBase + Base.OffsetInBits;
540 classify(T: Base.FieldType, OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, IsNamedArg);
541 Lo = merge(Accum: Lo, Field: FieldLo);
542 Hi = merge(Accum: Hi, Field: FieldHi);
543
544 if (getX86ABICompatInfo().ReturnCXXRecordGreaterThan128InMem &&
545 (Size > 128 &&
546 (Size != Base.FieldType->getSizeInBits().getFixedValue() ||
547 Size > getNativeVectorSizeForAVXABI(AVXLevel))))
548 Lo = Memory;
549
550 if (Lo == Memory || Hi == Memory) {
551 postMerge(AggregateSize: Size, Lo, Hi);
552 return;
553 }
554 }
555 }
556
557 // Classify the fields one at a time, merging the results.
558
559 bool IsUnion = RT->isUnion() && !getX86ABICompatInfo().Clang11Compat;
560 for (const auto &Field : RT->getFields()) {
561 uint64_t Offset = OffsetBase + Field.OffsetInBits;
562 bool BitField = Field.IsBitField;
563
564 // Ignore padding bit-fields. Normally only zero-length bit-fields are
565 // padding, but under Clang 23 compatibility every unnamed bit-field is,
566 // faithfully reproducing Clang 23.
567 if (BitField && (getX86ABICompatInfo().ClassifyUnnamedBitFields
568 ? Field.BitFieldWidth == 0
569 : Field.IsUnnamedBitfield))
570 continue;
571
572 if (Size > 128 &&
573 ((!IsUnion &&
574 Size != Field.FieldType->getSizeInBits().getFixedValue()) ||
575 Size > getNativeVectorSizeForAVXABI(AVXLevel))) {
576 Lo = Memory;
577 postMerge(AggregateSize: Size, Lo, Hi);
578 return;
579 }
580
581 bool IsInMemory = Offset % (Field.FieldType->getAlignment().value() * 8);
582 if (!BitField && IsInMemory) {
583 Lo = Memory;
584 postMerge(AggregateSize: Size, Lo, Hi);
585 return;
586 }
587
588 Class FieldLo, FieldHi;
589
590 if (BitField) {
591 uint64_t BitFieldSize = Field.BitFieldWidth;
592 uint64_t EbLo = Offset / 64;
593 uint64_t EbHi = (Offset + BitFieldSize - 1) / 64;
594
595 if (EbLo) {
596 assert(EbHi == EbLo && "Invalid classification, type > 16 bytes.");
597 FieldLo = NoClass;
598 FieldHi = Integer;
599 } else {
600 FieldLo = Integer;
601 FieldHi = EbHi ? Integer : NoClass;
602 }
603 } else {
604 classify(T: Field.FieldType, OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, IsNamedArg);
605 }
606
607 Lo = merge(Accum: Lo, Field: FieldLo);
608 Hi = merge(Accum: Hi, Field: FieldHi);
609 if (Lo == Memory || Hi == Memory)
610 break;
611 }
612 postMerge(AggregateSize: Size, Lo, Hi);
613 return;
614 }
615
616 Lo = Memory;
617 Hi = NoClass;
618}
619
620ArgInfo
621X86_64TargetInfo::classifyArgumentType(const Type *Ty, unsigned FreeIntRegs,
622 unsigned &NeededInt, unsigned &NeededSSE,
623 bool IsNamedArg, bool IsRegCall) const {
624
625 Ty = useFirstFieldIfTransparentUnion(Ty);
626
627 X86_64TargetInfo::Class Lo, Hi;
628 classify(T: Ty, OffsetBase: 0, Lo, Hi, IsNamedArg, IsRegCall);
629
630 // Check some invariants
631 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
632 assert((Hi != SseUp || Lo == Sse) && "Invalid SseUp classification.");
633
634 NeededInt = 0;
635 NeededSSE = 0;
636 const Type *ResType = nullptr;
637
638 switch (Lo) {
639 case NoClass:
640 if (Hi == NoClass)
641 return ArgInfo::getIgnore();
642 // If the low part is just padding, it takes no register, leave ResType
643 // null.
644 assert((Hi == Sse || Hi == Integer || Hi == X87Up) &&
645 "Unknown missing lo part");
646 break;
647
648 // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
649 // on the stack.
650 case Memory:
651 // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87Up or
652 // COMPLEX_X87, it is passed in memory.
653 case X87:
654 case ComplexX87:
655 if (getRecordArgABI(Ty) == RAA_Indirect)
656 ++NeededInt;
657 return getIndirectResult(Ty, FreeIntRegs);
658
659 case SseUp:
660 case X87Up:
661 llvm_unreachable("Invalid classification for lo word.");
662
663 // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
664 // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
665 // and %r9 is used.
666 case Integer:
667 ++NeededInt;
668
669 // Pick an 8-byte type based on the preferred type.
670 ResType = getIntegerTypeAtOffset(IRType: Ty, IROffset: 0, SourceTy: Ty, SourceOffset: 0);
671
672 // If we have a sign or zero extended integer, make sure to return Extend
673 // so that the parameter gets the right LLVM IR attributes.
674 if (Hi == NoClass && ResType->isInteger()) {
675 if (Ty->isInteger() && isPromotableInteger(IT: cast<IntegerType>(Val: Ty)))
676 return ArgInfo::getExtend(T: Ty);
677 }
678
679 if (ResType->isInteger() && ResType->getSizeInBits() == 128) {
680 assert(Hi == Integer);
681 ++NeededInt;
682 return ArgInfo::getDirect(T: ResType);
683 }
684 break;
685
686 // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
687 // available SSE register is used, the registers are taken in the
688 // order from %xmm0 to %xmm7.
689 case Sse:
690 ResType = getSSETypeAtOffset(ABIType: Ty, ABIOffset: 0, SourceTy: Ty, SourceOffset: 0);
691 ++NeededSSE;
692 break;
693 }
694
695 const Type *HighPart = nullptr;
696 switch (Hi) {
697 // Memory was handled previously, ComplexX87 and X87 should
698 // never occur as hi classes, and X87Up must be preceded by X87,
699 // which is passed in memory.
700 case Memory:
701 case X87:
702 case ComplexX87:
703 llvm_unreachable("Invalid classification for hi word.");
704
705 case NoClass:
706 break;
707
708 case Integer:
709 ++NeededInt;
710 // Pick an 8-byte type based on the preferred type.
711 HighPart = getIntegerTypeAtOffset(IRType: Ty, IROffset: 8, SourceTy: Ty, SourceOffset: 8);
712
713 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
714 return ArgInfo::getDirect(T: HighPart, Offset: 8);
715 break;
716
717 // X87Up generally doesn't occur here (long double is passed in
718 // memory), except in situations involving unions.
719 case X87Up:
720 case Sse:
721 ++NeededSSE;
722 HighPart = getSSETypeAtOffset(ABIType: Ty, ABIOffset: 8, SourceTy: Ty, SourceOffset: 8);
723
724 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
725 return ArgInfo::getDirect(T: HighPart, Offset: 8);
726 break;
727
728 // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
729 // eightbyte is passed in the upper half of the last used SSE
730 // register. This only happens when 128-bit vectors are passed.
731 case SseUp:
732 assert(Lo == Sse && "Unexpected SseUp classification");
733 ResType = getByteVectorType(Ty);
734 break;
735 }
736
737 // If a high part was specified, merge it together with the low part. It is
738 // known to pass in the high eightbyte of the result. We do this by forming a
739 // first class struct aggregate with the high and low part: {low, high}
740 if (HighPart)
741 ResType = createPairType(Lo: ResType, Hi: HighPart);
742
743 return ArgInfo::getDirect(T: ResType);
744}
745
746ArgInfo X86_64TargetInfo::classifyReturnType(const Type *RetTy) const {
747 // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
748 // classification algorithm.
749
750 X86_64TargetInfo::Class Lo, Hi;
751 classify(T: RetTy, OffsetBase: 0, Lo, Hi, /*isNamedArg*/ IsNamedArg: true);
752
753 // Check some invariants
754 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
755 assert((Hi != SseUp || Lo == Sse) && "Invalid SseUp classification.");
756
757 const Type *ResType = nullptr;
758 switch (Lo) {
759 case NoClass:
760 if (Hi == NoClass)
761 return ArgInfo::getIgnore();
762 // If the low part is just padding, it takes no register, leave ResType
763 // null.
764 assert((Hi == Sse || Hi == Integer || Hi == X87Up) &&
765 "Unknown missing lo part");
766 break;
767 case SseUp:
768 case X87Up:
769 llvm_unreachable("Invalid classification for lo word.");
770
771 // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
772 // hidden argument.
773 case Memory:
774 return getIndirectReturnResult(Ty: RetTy);
775
776 // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
777 // available register of the sequence %rax, %rdx is used.
778 case Integer:
779 ResType = getIntegerTypeAtOffset(IRType: RetTy, IROffset: 0, SourceTy: RetTy, SourceOffset: 0);
780 // If we have a sign or zero extended integer, make sure to return Extend
781 // so that the parameter gets the right LLVM IR attributes.
782 if (Hi == NoClass && ResType->isInteger()) {
783 if (const IntegerType *IntTy = dyn_cast<IntegerType>(Val: RetTy)) {
784 if (isPromotableInteger(IT: IntTy))
785 return ArgInfo::getExtend(T: RetTy);
786 }
787 }
788 if (ResType->isInteger() && ResType->getSizeInBits() == 128) {
789 assert(Hi == Integer);
790 return ArgInfo::getDirect(T: ResType);
791 }
792 break;
793
794 // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
795 // available SSE register of the sequence %xmm0, %xmm1 is used.
796 case Sse:
797 ResType = getSSETypeAtOffset(ABIType: RetTy, ABIOffset: 0, SourceTy: RetTy, SourceOffset: 0);
798 break;
799
800 // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
801 // returned on the X87 stack in %st0 as 80-bit x87 number.
802 case X87:
803 ResType = TB.getFloatType(Semantics: APFloat::x87DoubleExtended(), Align: Align(16));
804 break;
805
806 // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
807 // part of the value is returned in %st0 and the imaginary part in
808 // %st1.
809 case ComplexX87:
810 assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
811 {
812 const Type *X87Type =
813 TB.getFloatType(Semantics: APFloat::x87DoubleExtended(), Align: Align(16));
814 FieldInfo Fields[] = {FieldInfo(X87Type, 0), FieldInfo(X87Type, 80)};
815 ResType = TB.getRecordType(Fields, Size: TypeSize::getFixed(ExactSize: 160), ABIAlign: Align(16),
816 /*UnadjustedAlign=*/Align(16));
817 }
818 break;
819 }
820
821 const Type *HighPart = nullptr;
822 switch (Hi) {
823 // Memory was handled previously and X87 should
824 // never occur as a hi class.
825 case Memory:
826 case X87:
827 llvm_unreachable("Invalid classification for hi word.");
828
829 case ComplexX87:
830 case NoClass:
831 break;
832
833 case Integer:
834 HighPart = getIntegerTypeAtOffset(IRType: RetTy, IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
835 if (Lo == NoClass)
836 return ArgInfo::getDirect(T: HighPart, Offset: 8);
837 break;
838
839 case Sse:
840 HighPart = getSSETypeAtOffset(ABIType: RetTy, ABIOffset: 8, SourceTy: RetTy, SourceOffset: 8);
841 if (Lo == NoClass)
842 return ArgInfo::getDirect(T: HighPart, Offset: 8);
843 break;
844
845 // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
846 // is passed in the next available eightbyte chunk if the last used
847 // vector register.
848 //
849 // SSEUP should always be preceded by SSE, just widen.
850 case SseUp:
851 assert(Lo == Sse && "Unexpected SseUp classification.");
852 ResType = getByteVectorType(Ty: RetTy);
853 break;
854
855 // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87Up, the value is
856 // returned together with the previous X87 value in %st0.
857 case X87Up:
858 // If X87Up is preceded by X87, we don't need to do
859 // anything. However, in some cases with unions it may not be
860 // preceded by X87. In such situations we follow gcc and pass the
861 // extra bits in an SSE reg.
862 if (Lo != X87) {
863 HighPart = getSSETypeAtOffset(ABIType: RetTy, ABIOffset: 8, SourceTy: RetTy, SourceOffset: 8);
864 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
865 return ArgInfo::getDirect(T: HighPart, Offset: 8);
866 }
867 break;
868 }
869
870 // If a high part was specified, merge it together with the low part. It is
871 // known to pass in the high eightbyte of the result. We do this by forming a
872 // first class struct aggregate with the high and low part: {low, high}
873 if (HighPart)
874 ResType = createPairType(Lo: ResType, Hi: HighPart);
875
876 return ArgInfo::getDirect(T: ResType);
877}
878
879/// Given a high and low type that can ideally
880/// be used as elements of a two register pair to pass or return, return a
881/// first class aggregate to represent them. For example, if the low part of
882/// a by-value argument should be passed as i32* and the high part as float,
883/// return {i32*, float}.
884const Type *X86_64TargetInfo::createPairType(const Type *Lo,
885 const Type *Hi) const {
886 // In order to correctly satisfy the ABI, we need to the high part to start
887 // at offset 8. If the high and low parts we inferred are both 4-byte types
888 // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
889 // the second element at offset 8. Check for this:
890 unsigned LoSize = (unsigned)Lo->getTypeAllocSize();
891 llvm::Align HiAlign = Hi->getAlignment();
892 unsigned HiStart = alignTo(Size: LoSize, A: HiAlign);
893
894 assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
895
896 // To handle this, we have to increase the size of the low part so that the
897 // second element will start at an 8 byte offset. We can't increase the size
898 // of the second element because it might make us access off the end of the
899 // struct.
900 const Type *AdjustedLo = Lo;
901 if (HiStart != 8) {
902 // There are usually two sorts of types the ABI generation code can produce
903 // for the low part of a pair that aren't 8 bytes in size: half, float or
904 // i8/i16/i32. This can also include pointers when they are 32-bit (X32 and
905 // NaCl).
906 // Promote these to a larger type.
907 if (Lo->isFloat()) {
908 const FloatType *FT = cast<FloatType>(Val: Lo);
909 if (FT->getSemantics() == &APFloat::IEEEhalf() ||
910 FT->getSemantics() == &APFloat::IEEEsingle() ||
911 FT->getSemantics() == &APFloat::BFloat())
912 AdjustedLo = TB.getFloatType(Semantics: APFloat::IEEEdouble(), Align: Align(8));
913 }
914 // Promote integers and pointers to i64
915 else if (Lo->isInteger() || Lo->isPointer())
916 AdjustedLo = TB.getIntegerType(BitWidth: 64, Align: Align(8), /*Signed=*/false);
917 else
918 assert((Lo->isInteger() || Lo->isPointer()) &&
919 "Invalid/unknown low type in pair");
920 unsigned AdjustedLoSize = AdjustedLo->getSizeInBits().getFixedValue() / 8;
921 HiStart = alignTo(Size: AdjustedLoSize, A: HiAlign);
922 }
923
924 // Create the pair struct
925 FieldInfo Fields[] = {FieldInfo(AdjustedLo, 0), FieldInfo(Hi, HiStart * 8)};
926
927 // Verify the high part is at offset 8
928 assert((8 * 8) == Fields[1].OffsetInBits &&
929 "High part must be at offset 8 bytes");
930
931 uint64_t PairSizeInBits =
932 Fields[1].OffsetInBits + Hi->getSizeInBits().getFixedValue();
933 return TB.getRecordType(Fields, Size: TypeSize::getFixed(ExactSize: PairSizeInBits), ABIAlign: Align(8),
934 /*UnadjustedAlign=*/Align(8), Pack: StructPacking::Default);
935}
936
937static bool bitsContainNoUserData(const Type *Ty, unsigned StartBit,
938 unsigned EndBit) {
939 // If range is completely beyond type size, it's definitely padding
940 unsigned TySize = Ty->getSizeInBits().getFixedValue();
941 if (TySize <= StartBit)
942 return true;
943
944 // Handle arrays - check each element
945 if (const ArrayType *AT = dyn_cast<ArrayType>(Val: Ty)) {
946 const Type *EltTy = AT->getElementType();
947 unsigned EltSize = EltTy->getSizeInBits().getFixedValue();
948
949 for (unsigned I = 0; I < AT->getNumElements(); ++I) {
950 unsigned EltOffset = I * EltSize;
951 if (EltOffset >= EndBit)
952 break;
953
954 unsigned EltStart = (EltOffset < StartBit) ? StartBit - EltOffset : 0;
955 if (!bitsContainNoUserData(Ty: EltTy, StartBit: EltStart, EndBit: EndBit - EltOffset))
956 return false;
957 }
958 return true;
959 }
960
961 // Handle records - check all fields and base classes. getUnionType places a
962 // union's members at offset zero, so the field loop covers a union too.
963 if (const RecordType *RT = dyn_cast<RecordType>(Val: Ty)) {
964 // Check base classes first (for C++ records)
965 if (RT->isCXXRecord()) {
966 for (unsigned I = 0; I < RT->getNumBaseClasses(); ++I) {
967 const FieldInfo &Base = RT->getBaseClasses()[I];
968 if (Base.OffsetInBits >= EndBit)
969 continue;
970
971 unsigned BaseStart =
972 (Base.OffsetInBits < StartBit) ? StartBit - Base.OffsetInBits : 0;
973 if (!bitsContainNoUserData(Ty: Base.FieldType, StartBit: BaseStart,
974 EndBit: EndBit - Base.OffsetInBits))
975 return false;
976 }
977 }
978
979 for (unsigned I = 0; I < RT->getNumFields(); ++I) {
980 const FieldInfo &Field = RT->getFields()[I];
981 if (Field.OffsetInBits >= EndBit)
982 break;
983
984 unsigned FieldStart =
985 (Field.OffsetInBits < StartBit) ? StartBit - Field.OffsetInBits : 0;
986 if (!bitsContainNoUserData(Ty: Field.FieldType, StartBit: FieldStart,
987 EndBit: EndBit - Field.OffsetInBits))
988 return false;
989 }
990 return true;
991 }
992
993 // For any other type - assume all bits are user data
994 return false;
995}
996
997const Type *X86_64TargetInfo::getIntegerTypeAtOffset(const Type *ABIType,
998 unsigned ABIOffset,
999 const Type *SourceTy,
1000 unsigned SourceOffset,
1001 bool InMemory) const {
1002
1003 const Type *WorkingType = ABIType;
1004 if (InMemory && ABIType->isInteger()) {
1005 const auto *IT = cast<IntegerType>(Val: ABIType);
1006 unsigned OriginalBitWidth = IT->getSizeInBits().getFixedValue();
1007
1008 unsigned WidenedBitWidth = OriginalBitWidth;
1009 if (OriginalBitWidth <= 8) {
1010 WidenedBitWidth = 8;
1011 } else {
1012 WidenedBitWidth = llvm::bit_ceil(Value: OriginalBitWidth);
1013 }
1014
1015 if (WidenedBitWidth != OriginalBitWidth) {
1016 WorkingType = TB.getIntegerType(BitWidth: WidenedBitWidth, Align: ABIType->getAlignment(),
1017 Signed: IT->isSigned());
1018 }
1019 }
1020 // If we're dealing with an un-offset ABI type, then it means that we're
1021 // returning an 8-byte unit starting with it. See if we can safely use it.
1022 if (ABIOffset == 0) {
1023 // Pointers and int64's always fill the 8-byte unit. Return WorkingType,
1024 // which is the in-memory-widened type (e.g. a _BitInt(37) field widened to
1025 // i64): returning the raw ABIType here would coerce the eightbyte to the
1026 // narrow iN instead of the storage integer clang uses.
1027 if ((WorkingType->isPointer() && Has64BitPointers) ||
1028 (WorkingType->isInteger() &&
1029 cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 64))
1030 return WorkingType;
1031
1032 // If we have a 1/2/4-byte integer, we can use it only if the rest of the
1033 // goodness in the source type is just tail padding. This is allowed to
1034 // kick in for struct {double,int} on the int, but not on
1035 // struct{double,int,int} because we wouldn't return the second int. We
1036 // have to do this analysis on the source type because we can't depend on
1037 // unions being lowered a specific way etc.
1038 if ((WorkingType->isInteger() &&
1039 (cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 1 ||
1040 cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 8 ||
1041 cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 16 ||
1042 cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 32)) ||
1043 (WorkingType->isPointer() && !Has64BitPointers)) {
1044
1045 unsigned BitWidth = WorkingType->isPointer()
1046 ? 32
1047 : cast<IntegerType>(Val: WorkingType)->getSizeInBits();
1048
1049 if (bitsContainNoUserData(Ty: SourceTy, StartBit: SourceOffset * 8 + BitWidth,
1050 EndBit: SourceOffset * 8 + 64))
1051 return WorkingType;
1052 }
1053 }
1054
1055 if (const auto *RTy = dyn_cast<RecordType>(Val: ABIType)) {
1056 if (RTy->isUnion()) {
1057 const Type *ReducedType = reduceUnionForX8664(UnionType: RTy, TB);
1058 if (ReducedType) {
1059 if (ABIOffset * 8 < ReducedType->getSizeInBits().getFixedValue())
1060 return getIntegerTypeAtOffset(ABIType: ReducedType, ABIOffset, SourceTy,
1061 SourceOffset, InMemory: true);
1062 // The storage type stops before this offset, so size the coercion
1063 // from the union itself: a byte when the rest of this eightbyte
1064 // holds no data, and the union's remaining bytes otherwise.
1065 if (bitsContainNoUserData(Ty: SourceTy, StartBit: SourceOffset * 8 + 8,
1066 EndBit: SourceOffset * 8 + 64))
1067 return TB.getIntegerType(BitWidth: 8, Align: Align(1), /*Signed=*/false);
1068 unsigned RemainingBytes =
1069 llvm::divideCeil(Numerator: SourceTy->getSizeInBits().getFixedValue(), Denominator: 8) -
1070 SourceOffset;
1071 return TB.getIntegerType(BitWidth: std::min(a: RemainingBytes, b: 8U) * 8, Align: Align(1),
1072 /*Signed=*/false);
1073 }
1074 }
1075 if (const FieldInfo *Element =
1076 RTy->getElementContainingOffset(OffsetInBits: ABIOffset * 8)) {
1077
1078 unsigned ElementOffsetBytes = Element->OffsetInBits / 8;
1079 return getIntegerTypeAtOffset(ABIType: Element->FieldType,
1080 ABIOffset: ABIOffset - ElementOffsetBytes, SourceTy,
1081 SourceOffset, InMemory: true);
1082 }
1083 }
1084
1085 if (const auto *ATy = dyn_cast<ArrayType>(Val: ABIType)) {
1086 const Type *EltTy = ATy->getElementType();
1087 unsigned EltSize = EltTy->getSizeInBits() / 8;
1088 if (EltSize > 0) {
1089 unsigned EltOffset = (ABIOffset / EltSize) * EltSize;
1090 return getIntegerTypeAtOffset(ABIType: EltTy, ABIOffset: ABIOffset - EltOffset, SourceTy,
1091 SourceOffset, InMemory: true);
1092 }
1093 }
1094
1095 // If we have a 128-bit integer, we can pass it safely using an i128
1096 // so we return that
1097 if (ABIType->isInteger() && ABIType->getSizeInBits() == 128) {
1098 assert(ABIOffset == 0);
1099 return ABIType;
1100 }
1101
1102 unsigned TySizeInBytes =
1103 llvm::divideCeil(Numerator: SourceTy->getSizeInBits().getFixedValue(), Denominator: 8);
1104 if (auto *IT = dyn_cast<IntegerType>(Val: SourceTy)) {
1105 if (IT->isBitInt())
1106 TySizeInBytes =
1107 alignTo(Value: SourceTy->getSizeInBits().getFixedValue(), Align: 64) / 8;
1108 }
1109 assert(TySizeInBytes != SourceOffset && "Empty field?");
1110 unsigned AvailableSize = TySizeInBytes - SourceOffset;
1111 return TB.getIntegerType(BitWidth: std::min(a: AvailableSize, b: 8U) * 8, Align: Align(1), Signed: false);
1112}
1113/// Returns the floating point type at the specified offset within a type, or
1114/// nullptr if no floating point type is found at that offset.
1115const Type *X86_64TargetInfo::getFPTypeAtOffset(const Type *Ty,
1116 unsigned Offset) const {
1117 // Check for direct match at offset 0
1118 if (Offset == 0 && Ty->isFloat())
1119 return Ty;
1120
1121 if (const ComplexType *CT = dyn_cast<ComplexType>(Val: Ty)) {
1122 const Type *ElementType = CT->getElementType();
1123 unsigned ElementSize = ElementType->getSizeInBits().getFixedValue() / 8;
1124
1125 if (Offset == 0 || Offset == ElementSize)
1126 return ElementType;
1127 return nullptr;
1128 }
1129
1130 // Handle struct types by checking each field
1131 if (const RecordType *RT = dyn_cast<RecordType>(Val: Ty)) {
1132 if (const FieldInfo *Element = RT->getElementContainingOffset(OffsetInBits: Offset * 8)) {
1133 unsigned ElementOffsetBytes = Element->OffsetInBits / 8;
1134 return getFPTypeAtOffset(Ty: Element->FieldType, Offset: Offset - ElementOffsetBytes);
1135 }
1136 }
1137
1138 // Handle array types
1139 if (const ArrayType *AT = dyn_cast<ArrayType>(Val: Ty)) {
1140 const Type *EltTy = AT->getElementType();
1141 unsigned EltSize = EltTy->getSizeInBits() / 8;
1142 unsigned EltIndex = Offset / EltSize;
1143
1144 return getFPTypeAtOffset(Ty: EltTy, Offset: Offset - (EltIndex * EltSize));
1145 }
1146
1147 // No floating point type found at this offset
1148 return nullptr;
1149}
1150
1151/// Helper to check if a floating point type matches specific semantics
1152static bool isFloatTypeWithSemantics(const Type *Ty,
1153 const fltSemantics &Semantics) {
1154 if (!Ty->isFloat())
1155 return false;
1156 const FloatType *FT = cast<FloatType>(Val: Ty);
1157 return FT->getSemantics() == &Semantics;
1158}
1159
1160/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
1161/// low 8 bytes of an XMM register, corresponding to the SSE class.
1162const Type *X86_64TargetInfo::getSSETypeAtOffset(const Type *ABIType,
1163 unsigned ABIOffset,
1164 const Type *SourceTy,
1165 unsigned SourceOffset) const {
1166
1167 if (const auto *RTy = dyn_cast<RecordType>(Val: ABIType)) {
1168 if (RTy->isUnion()) {
1169 const Type *ReducedType = reduceUnionForX8664(UnionType: RTy, TB);
1170 if (ReducedType) {
1171 return getSSETypeAtOffset(ABIType: ReducedType, ABIOffset, SourceTy,
1172 SourceOffset);
1173 }
1174 }
1175 }
1176
1177 auto Is16bitFpTy = [](const Type *T) {
1178 return isFloatTypeWithSemantics(Ty: T, Semantics: APFloat::IEEEhalf()) ||
1179 isFloatTypeWithSemantics(Ty: T, Semantics: APFloat::BFloat());
1180 };
1181
1182 // Get the floating point type at the requested offset
1183 const Type *T0 = getFPTypeAtOffset(Ty: ABIType, Offset: ABIOffset);
1184 if (!T0 || isFloatTypeWithSemantics(Ty: T0, Semantics: APFloat::IEEEdouble()))
1185 return TB.getFloatType(Semantics: APFloat::IEEEdouble(), Align: Align(8));
1186
1187 // Calculate remaining source size in bytes
1188 unsigned SourceSize =
1189 (SourceTy->getSizeInBits().getFixedValue() / 8) - SourceOffset;
1190
1191 // Try to get adjacent FP type
1192 const Type *T1 = nullptr;
1193 unsigned T0Size =
1194 alignTo(Value: T0->getSizeInBits().getFixedValue(), Align: T0->getAlignment().value()) /
1195 8;
1196 if (SourceSize > T0Size)
1197 T1 = getFPTypeAtOffset(Ty: ABIType, Offset: ABIOffset + T0Size);
1198
1199 if (T1 == nullptr) {
1200 if (Is16bitFpTy(T0) && SourceSize > 4)
1201 T1 = getFPTypeAtOffset(Ty: ABIType, Offset: ABIOffset + 4);
1202
1203 if (T1 == nullptr)
1204 return T0;
1205 }
1206 // Handle vector cases
1207 if (isFloatTypeWithSemantics(Ty: T0, Semantics: APFloat::IEEEsingle()) &&
1208 isFloatTypeWithSemantics(Ty: T1, Semantics: APFloat::IEEEsingle()))
1209 return TB.getVectorType(ElementType: T0, NumElements: ElementCount::getFixed(MinVal: 2), Align: Align(8));
1210
1211 if (Is16bitFpTy(T0) && Is16bitFpTy(T1)) {
1212 const Type *T2 = nullptr;
1213 if (SourceSize > 4)
1214 T2 = getFPTypeAtOffset(Ty: ABIType, Offset: ABIOffset + 4);
1215 if (!T2)
1216 return TB.getVectorType(ElementType: T0, NumElements: ElementCount::getFixed(MinVal: 2), Align: Align(8));
1217 return TB.getVectorType(ElementType: T0, NumElements: ElementCount::getFixed(MinVal: 4), Align: Align(8));
1218 }
1219
1220 // Mixed half-float cases
1221 if (Is16bitFpTy(T0) || Is16bitFpTy(T1))
1222 return TB.getVectorType(ElementType: TB.getFloatType(Semantics: APFloat::IEEEhalf(), Align: Align(2)),
1223 NumElements: ElementCount::getFixed(MinVal: 4), Align: Align(8));
1224
1225 // Default to double
1226 return TB.getFloatType(Semantics: APFloat::IEEEdouble(), Align: Align(8));
1227}
1228
1229/// The ABI specifies that a value should be passed in a full vector XMM/YMM
1230/// register. Pick an LLVM IR type that will be passed as a vector register.
1231const Type *X86_64TargetInfo::getByteVectorType(const Type *Ty) const {
1232 // Wrapper structs/arrays that only contain vectors are passed just like
1233 // vectors; strip them off if present.
1234 if (const Type *InnerTy = isSingleElementStruct(Ty))
1235 Ty = InnerTy;
1236
1237 // Handle vector types
1238 if (const VectorType *VT = dyn_cast<VectorType>(Val: Ty)) {
1239 // Don't pass vXi128 vectors in their native type, the backend can't
1240 // legalize them.
1241 if (getX86ABICompatInfo().PassInt128VectorsInMem &&
1242 VT->getElementType()->isInteger() &&
1243 cast<IntegerType>(Val: VT->getElementType())->getSizeInBits() == 128) {
1244 unsigned Size = VT->getSizeInBits().getFixedValue();
1245 return TB.getVectorType(ElementType: TB.getIntegerType(BitWidth: 64, Align: Align(8), /*Signed=*/false),
1246 NumElements: ElementCount::getFixed(MinVal: Size / 64),
1247 Align: Align(Size / 8));
1248 }
1249 return VT;
1250 }
1251
1252 // Handle fp128
1253 if (isFloatTypeWithSemantics(Ty, Semantics: APFloat::IEEEquad()))
1254 return Ty;
1255
1256 // We couldn't find the preferred IR vector type for 'Ty'.
1257 unsigned Size = Ty->getSizeInBits().getFixedValue();
1258 assert((Size == 128 || Size == 256 || Size == 512) && "Invalid vector size");
1259
1260 return TB.getVectorType(ElementType: TB.getFloatType(Semantics: APFloat::IEEEdouble(), Align: Align(8)),
1261 NumElements: ElementCount::getFixed(MinVal: Size / 64), Align: Align(Size / 8));
1262}
1263
1264// Returns the single element if this is a single-element struct wrapper
1265const Type *X86_64TargetInfo::isSingleElementStruct(const Type *Ty) const {
1266 const auto *RT = dyn_cast<RecordType>(Val: Ty);
1267 if (!RT)
1268 return nullptr;
1269
1270 if (RT->hasFlexibleArrayMember())
1271 return nullptr;
1272
1273 const Type *Found = nullptr;
1274
1275 for (const auto &Base : RT->getBaseClasses()) {
1276 const Type *BaseTy = Base.FieldType;
1277 auto *BaseRT = dyn_cast<RecordType>(Val: BaseTy);
1278
1279 if (!BaseRT || BaseRT->isEmpty())
1280 continue;
1281
1282 const Type *Elem = isSingleElementStruct(Ty: BaseTy);
1283 if (!Elem || Found)
1284 return nullptr;
1285 Found = Elem;
1286 }
1287
1288 for (const auto &FI : RT->getFields()) {
1289 if (FI.isEmpty())
1290 continue;
1291
1292 const Type *FTy = FI.FieldType;
1293
1294 while (auto *AT = dyn_cast<ArrayType>(Val: FTy)) {
1295 if (AT->getNumElements() != 1)
1296 break;
1297 FTy = AT->getElementType();
1298 }
1299
1300 const Type *Elem;
1301 if (auto *InnerRT = dyn_cast<RecordType>(Val: FTy))
1302 Elem = isSingleElementStruct(Ty: InnerRT);
1303 else
1304 Elem = FTy;
1305 if (!Elem || Found)
1306 return nullptr;
1307 Found = Elem;
1308 }
1309
1310 if (!Found)
1311 return nullptr;
1312 if (Found->getSizeInBits() != Ty->getSizeInBits())
1313 return nullptr;
1314
1315 return Found;
1316}
1317
1318bool X86_64TargetInfo::isIllegalVectorType(const Type *Ty) const {
1319 if (const auto *VecTy = dyn_cast<VectorType>(Val: Ty)) {
1320 uint64_t Size = VecTy->getSizeInBits().getFixedValue();
1321 unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
1322
1323 // Vectors <= 64 bits or > largest supported vector size are illegal
1324 if (Size <= 64 || Size > LargestVector)
1325 return true;
1326
1327 // Check for 128-bit integer element vectors that should be passed in memory
1328 const Type *EltTy = VecTy->getElementType();
1329 if (getX86ABICompatInfo().PassInt128VectorsInMem && EltTy->isInteger()) {
1330 const auto *IntTy = cast<IntegerType>(Val: EltTy);
1331 if (IntTy->getSizeInBits().getFixedValue() == 128)
1332 return true;
1333 }
1334 }
1335 return false;
1336}
1337
1338ArgInfo X86_64TargetInfo::getIndirectResult(const Type *Ty,
1339 unsigned FreeIntRegs) const {
1340 // If this is a scalar LLVM value then assume LLVM will pass it in the right
1341 // place naturally.
1342 //
1343 // This assumption is optimistic, as there could be free registers available
1344 // when we need to pass this argument in memory, and LLVM could try to pass
1345 // the argument in the free register. This does not seem to happen currently,
1346 // but this code would be much safer if we could mark the argument with
1347 // 'onstack'. See PR12193.
1348 if (!isAggregateTypeForABI(Ty) && !isIllegalVectorType(Ty) &&
1349 !(Ty->isInteger() && cast<IntegerType>(Val: Ty)->isBitInt())) {
1350 return (Ty->isInteger() && isPromotableInteger(IT: cast<IntegerType>(Val: Ty))
1351 ? ArgInfo::getExtend(T: Ty)
1352 : ArgInfo::getDirect());
1353 }
1354
1355 // Check if this is a record type that needs special handling
1356 if (auto RecordRAA = getRecordArgABI(Ty))
1357 return getNaturalAlignIndirect(Ty, ByVal: RecordRAA ==
1358 RecordArgABI::RAA_DirectInMemory);
1359
1360 // Compute the byval alignment. We specify the alignment of the byval in all
1361 // cases so that the mid-level optimizer knows the alignment of the byval.
1362 uint64_t AlignVal = std::max<uint64_t>(a: Ty->getAlignment().value(), b: 8u);
1363
1364 // Attempt to avoid passing indirect results using byval when possible. This
1365 // is important for good codegen.
1366 //
1367 // We do this by coercing the value into a scalar type which the backend can
1368 // handle naturally (i.e., without using byval).
1369 //
1370 // For simplicity, we currently only do this when we have exhausted all of the
1371 // free integer registers. Doing this when there are free integer registers
1372 // would require more care, as we would have to ensure that the coerced value
1373 // did not claim the unused register. That would require either reording the
1374 // arguments to the function (so that any subsequent inreg values came first),
1375 // or only doing this optimization when there were no following arguments that
1376 // might be inreg.
1377 //
1378 // We currently expect it to be rare (particularly in well written code) for
1379 // arguments to be passed on the stack when there are still free integer
1380 // registers available (this would typically imply large structs being passed
1381 // by value), so this seems like a fair tradeoff for now.
1382 //
1383 // We can revisit this if the backend grows support for 'onstack' parameter
1384 // attributes. See PR12193.
1385 if (FreeIntRegs == 0) {
1386 // Use the storage-container width (like Clang's getTypeSize) so a stack
1387 // _BitInt or illegal vector coerces to the integer covering its storage,
1388 // not its raw iN width.
1389 uint64_t Size = getClangTypeWidthInBits(Ty);
1390
1391 // If this type fits in an eightbyte, coerce it into the matching integral
1392 // type, which will end up on the stack (with alignment 8).
1393 if (AlignVal == 8 && Size <= 64) {
1394 const Type *IntTy =
1395 TB.getIntegerType(BitWidth: Size, Align: llvm::Align(8), /*Signed=*/false);
1396 return ArgInfo::getDirect(T: IntTy);
1397 }
1398 }
1399
1400 return ArgInfo::getIndirect(Align: llvm::Align(AlignVal), /*ByVal=*/true);
1401}
1402
1403ArgInfo X86_64TargetInfo::getIndirectReturnResult(const Type *Ty) const {
1404 if (!isAggregateTypeForABI(Ty)) {
1405 // Bit-precise integers are returned indirectly regardless of size.
1406 if (const auto *IntTy = dyn_cast<IntegerType>(Val: Ty)) {
1407 if (IntTy->isBitInt())
1408 return getNaturalAlignIndirect(Ty: IntTy, /*ByVal=*/true);
1409 if (isPromotableInteger(IT: IntTy))
1410 return ArgInfo::getExtend(T: Ty);
1411 }
1412 return ArgInfo::getDirect();
1413 }
1414
1415 return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
1416}
1417
1418void X86_64TargetInfo::computeInfo(FunctionInfo &FI) const {
1419 CallingConv::ID CallingConv = FI.getCallingConvention();
1420
1421 // Only the standard SysV (C) calling convention is classified here. Any other
1422 // convention must be added explicitly once it has been verified against this
1423 // classifier rather than silently taking the SysV path.
1424 switch (CallingConv) {
1425 case CallingConv::C:
1426 break;
1427 default:
1428 llvm_unreachable(
1429 "calling convention not supported by the LLVMABI X86_64 classifier");
1430 }
1431
1432 unsigned FreeIntRegs = 6;
1433 unsigned FreeSSERegs = 8;
1434 unsigned NeededInt = 0, NeededSSE = 0;
1435
1436 if (!maybeCommonClassifyReturnType(FI)) {
1437 const Type *RetTy = FI.getReturnType();
1438 FI.getReturnInfo() = classifyReturnType(RetTy);
1439 }
1440
1441 if (FI.getReturnInfo().isIndirect())
1442 --FreeIntRegs;
1443
1444 unsigned NumRequiredArgs = FI.getNumRequiredArgs();
1445
1446 unsigned ArgNo = 0;
1447 for (auto IT = FI.arg_begin(), IE = FI.arg_end(); IT != IE; ++IT, ++ArgNo) {
1448 bool IsNamedArg = ArgNo < NumRequiredArgs;
1449 const Type *ArgTy = IT->ABIType;
1450 NeededInt = 0;
1451 NeededSSE = 0;
1452
1453 ArgInfo AI = classifyArgumentType(Ty: ArgTy, FreeIntRegs, NeededInt, NeededSSE,
1454 IsNamedArg);
1455
1456 // AMD64-ABI 3.2.3p3: If there are no registers available for any
1457 // eightbyte of an argument, the whole argument is passed on the
1458 // stack. If registers have already been assigned for some
1459 // eightbytes of such an argument, the assignments get reverted.
1460 if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
1461 FreeIntRegs -= NeededInt;
1462 FreeSSERegs -= NeededSSE;
1463 IT->Info = AI;
1464 } else {
1465 // Not enough registers, pass on stack
1466 IT->Info = getIndirectResult(Ty: ArgTy, FreeIntRegs);
1467 }
1468 }
1469}
1470
1471std::unique_ptr<TargetInfo>
1472createX86_64TargetInfo(TypeBuilder &TB, X86AVXABILevel AVXLevel,
1473 bool Has64BitPointers, const X86ABICompatInfo &Compat) {
1474 return std::make_unique<X86_64TargetInfo>(args&: TB, args&: AVXLevel, args&: Has64BitPointers,
1475 args: Compat);
1476}
1477
1478} // namespace abi
1479} // namespace llvm
1480