1//===- X86.cpp ------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ABI/FunctionInfo.h"
10#include "llvm/ABI/TargetInfo.h"
11#include "llvm/ABI/Types.h"
12#include "llvm/Support/Alignment.h"
13#include "llvm/Support/Casting.h"
14#include "llvm/Support/ErrorHandling.h"
15#include "llvm/Support/MathExtras.h"
16#include "llvm/Support/TypeSize.h"
17#include <algorithm>
18#include <cassert>
19#include <cstdint>
20
21namespace llvm {
22namespace abi {
23
24static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
25 switch (AVXLevel) {
26 case X86AVXABILevel::AVX512:
27 return 512;
28 case X86AVXABILevel::AVX:
29 return 256;
30 case X86AVXABILevel::None:
31 return 128;
32 }
33 llvm_unreachable("Unknown AVXLevel");
34}
35
36class X86_64TargetInfo : public TargetInfo {
37public:
38 enum Class { Integer, Sse, SseUp, X87, X87Up, ComplexX87, NoClass, Memory };
39
40private:
41 X86AVXABILevel AVXLevel;
42 bool Has64BitPointers;
43 X86ABICompatInfo X86CompatInfo;
44
45 static Class merge(Class Accum, Class Field);
46
47 void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
48
49 void classify(const Type *T, uint64_t OffsetBase, Class &Lo, Class &Hi,
50 bool IsNamedArg, bool IsRegCall = false) const;
51
52 const Type *getIntegerTypeAtOffset(const Type *IRType, unsigned IROffset,
53 const Type *SourceTy,
54 unsigned SourceOffset,
55 bool InMemory = false) const;
56
57 const Type *getSSETypeAtOffset(const Type *ABIType, unsigned ABIOffset,
58 const Type *SourceTy,
59 unsigned SourceOffset) const;
60 bool isIllegalVectorType(const Type *Ty) const;
61 bool containsMatrixField(const RecordType *RT) const;
62
63 void computeInfo(FunctionInfo &FI) const override;
64 ArgInfo getIndirectReturnResult(const Type *Ty) const;
65 const Type *getFPTypeAtOffset(const Type *Ty, unsigned Offset) const;
66
67 const Type *getByteVectorType(const Type *Ty) const;
68
69 const Type *createPairType(const Type *Lo, const Type *Hi) const;
70 ArgInfo getIndirectResult(const Type *Ty, unsigned FreeIntRegs) const;
71
72 ArgInfo classifyReturnType(const Type *RetTy) const;
73
74 ArgInfo classifyArgumentType(const Type *Ty, unsigned FreeIntRegs,
75 unsigned &NeededInt, unsigned &NeededSse,
76 bool IsNamedArg, bool IsRegCall = false) const;
77
78 bool shouldExtendReturnType(const IntegerType *RetTy) const;
79
80public:
81 X86_64TargetInfo(TypeBuilder &TypeBuilder, X86AVXABILevel AVXABILevel,
82 bool Has64BitPtrs, const X86ABICompatInfo &Compat)
83 : TargetInfo(TypeBuilder), AVXLevel(AVXABILevel),
84 Has64BitPointers(Has64BitPtrs), X86CompatInfo(Compat) {}
85
86 bool has64BitPointers() const { return Has64BitPointers; }
87
88 const ABICompatInfo &getABICompatInfo() const override {
89 return X86CompatInfo;
90 }
91
92 const X86ABICompatInfo &getX86ABICompatInfo() const { return X86CompatInfo; }
93};
94
95static bool bitsContainNoUserData(const Type *Ty, unsigned StartBit,
96 unsigned EndBit);
97
98// Gets the "best" type to represent the union.
99static const Type *reduceUnionForX8664(const RecordType *UnionType,
100 TypeBuilder &TB) {
101 assert(UnionType->isUnion() && "Expected union type");
102
103 ArrayRef<FieldInfo> Fields = UnionType->getFields();
104 if (Fields.empty()) {
105 return nullptr;
106 }
107
108 const Type *StorageType = nullptr;
109
110 for (const auto &Field : Fields) {
111 if (Field.IsBitField && Field.IsUnnamedBitfield &&
112 Field.BitFieldWidth == 0) {
113 continue;
114 }
115
116 const Type *FieldType = Field.FieldType;
117
118 if (UnionType->isTransparentUnion() && !StorageType) {
119 StorageType = FieldType;
120 break;
121 }
122
123 // A member that holds no user data supplies no bytes for a coercion to
124 // read, so it must not become the storage type however wide or aligned it
125 // is declared. Clang compares lowered types instead, where an empty class
126 // is a byte array whose i8 leaf lets getIntegerTypeAtOffset narrow the
127 // coercion. A record mapped here holds no fields, so there is no such
128 // leaf and the eightbyte would be sized from the union.
129 if (bitsContainNoUserData(Ty: FieldType, StartBit: 0, EndBit: FieldType->getABISizeInBits()))
130 continue;
131
132 if (!StorageType ||
133 FieldType->getAlignment() > StorageType->getAlignment() ||
134 (FieldType->getAlignment() == StorageType->getAlignment() &&
135 TypeSize::isKnownGT(LHS: FieldType->getSizeInBits(),
136 RHS: StorageType->getSizeInBits()))) {
137 StorageType = FieldType;
138 }
139 }
140 return StorageType;
141}
142
143bool X86_64TargetInfo::shouldExtendReturnType(const IntegerType *RetTy) const {
144 // Bools need to be zero-extended to 8 bits for all ABIs.
145 if (RetTy->isBool())
146 return true;
147
148 // Promotable integer types get extended only for Darwin.
149 bool IsDarwin = false; // TODO: Currently not supported by ABI library.
150 if (!IsDarwin)
151 return false;
152
153 return isPromotableInteger(IT: RetTy);
154}
155
156void X86_64TargetInfo::postMerge(unsigned AggregateSize, Class &Lo,
157 Class &Hi) const {
158 // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
159 //
160 // (a) If one of the classes is Memory, the whole argument is passed in
161 // memory.
162 //
163 // (b) If X87Up is not preceded by X87, the whole argument is passed in
164 // memory.
165 //
166 // (c) If the size of the aggregate exceeds two eightbytes and the first
167 // eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
168 // argument is passed in memory. NOTE: This is necessary to keep the
169 // ABI working for processors that don't support the __m256 type.
170 //
171 // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
172 //
173 // Some of these are enforced by the merging logic. Others can arise
174 // only with unions; for example:
175 // union { _Complex double; unsigned; }
176 //
177 // Note that clauses (b) and (c) were added in 0.98.
178
179 if (Hi == Memory)
180 Lo = Memory;
181 if (Hi == X87Up && Lo != X87 && getX86ABICompatInfo().HonorsRevision98)
182 Lo = Memory;
183 if (AggregateSize > 128 && (Lo != Sse || Hi != SseUp))
184 Lo = Memory;
185 if (Hi == SseUp && Lo != Sse)
186 Hi = Sse;
187}
188X86_64TargetInfo::Class X86_64TargetInfo::merge(Class Accum, Class Field) {
189 // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
190 // classified recursively so that always two fields are
191 // considered. The resulting class is calculated according to
192 // the classes of the fields in the eightbyte:
193 //
194 // (a) If both classes are equal, this is the resulting class.
195 //
196 // (b) If one of the classes is NO_CLASS, the resulting class is
197 // the other class.
198 //
199 // (c) If one of the classes is MEMORY, the result is the MEMORY
200 // class.
201 //
202 // (d) If one of the classes is INTEGER, the result is the
203 // INTEGER.
204 //
205 // (e) If one of the classes is X87, X87Up, COMPLEX_X87 class,
206 // MEMORY is used as class.
207 //
208 // (f) Otherwise class SSE is used.
209
210 // Accum should never be memory (we should have returned) or
211 // ComplexX87 (because this cannot be passed in a structure).
212 assert((Accum != Memory && Accum != ComplexX87) &&
213 "Invalid accumulated classification during merge.");
214
215 if (Accum == Field || Field == NoClass)
216 return Accum;
217 if (Field == Memory)
218 return Memory;
219 if (Accum == NoClass)
220 return Field;
221 if (Accum == Integer || Field == Integer)
222 return Integer;
223 if (Field == X87 || Field == X87Up || Field == ComplexX87 || Accum == X87 ||
224 Accum == X87Up)
225 return Memory;
226
227 return Sse;
228}
229
230// A record with a matrix-extension field is passed in memory. clang has no
231// matrix-specific ABI code: a matrix falls through X86_64ABIInfo::classify to
232// the default MEMORY class. We model matrices as arrays, so this check
233// reproduces that record-with-matrix -> MEMORY result.
234bool X86_64TargetInfo::containsMatrixField(const RecordType *RT) const {
235 for (const auto &Field : RT->getFields()) {
236 const Type *FieldType = Field.FieldType;
237
238 if (const auto *AT = dyn_cast<ArrayType>(Val: FieldType)) {
239 if (AT->isMatrixType())
240 return true;
241 continue;
242 }
243
244 if (const auto *NestedRT = dyn_cast<RecordType>(Val: FieldType))
245 if (containsMatrixField(RT: NestedRT))
246 return true;
247 }
248 return false;
249}
250
251void X86_64TargetInfo::classify(const Type *T, uint64_t OffsetBase, Class &Lo,
252 Class &Hi, bool IsNamedArg,
253 bool IsRegCall) const {
254 Lo = Hi = NoClass;
255 Class &Current = OffsetBase < 64 ? Lo : Hi;
256 Current = Memory;
257
258 if (T->isVoid()) {
259 Current = NoClass;
260 return;
261 }
262
263 if (const auto *IT = dyn_cast<IntegerType>(Val: T)) {
264 auto BitWidth = IT->getSizeInBits().getFixedValue();
265
266 if (BitWidth == 128 ||
267 (IT->isBitInt() && BitWidth > 64 && BitWidth <= 128)) {
268 Lo = Integer;
269 Hi = Integer;
270 } else if (BitWidth <= 64) {
271 Current = Integer;
272 }
273
274 return;
275 }
276
277 if (const auto *FT = dyn_cast<FloatType>(Val: T)) {
278 const auto *FltSem = FT->getSemantics();
279
280 if (FltSem == &llvm::APFloat::IEEEsingle() ||
281 FltSem == &llvm::APFloat::IEEEdouble() ||
282 FltSem == &llvm::APFloat::IEEEhalf() ||
283 FltSem == &llvm::APFloat::BFloat()) {
284 Current = Sse;
285 } else if (FltSem == &llvm::APFloat::IEEEquad()) {
286 Lo = Sse;
287 Hi = SseUp;
288 } else if (FltSem == &llvm::APFloat::x87DoubleExtended()) {
289 Lo = X87;
290 Hi = X87Up;
291 } else {
292 Current = Sse;
293 }
294 return;
295 }
296 if (T->isPointer()) {
297 Current = Integer;
298 return;
299 }
300
301 if (const auto *MPT = dyn_cast<MemberPointerType>(Val: T)) {
302 if (MPT->isFunctionPointer()) {
303 if (Has64BitPointers) {
304 Lo = Hi = Integer;
305 } else {
306 uint64_t EbFuncPtr = OffsetBase / 64;
307 uint64_t EbThisAdj = (OffsetBase + 64 - 1) / 64;
308 if (EbFuncPtr != EbThisAdj) {
309 Lo = Hi = Integer;
310 } else {
311 Current = Integer;
312 }
313 }
314 } else {
315 Current = Integer;
316 }
317 return;
318 }
319
320 if (const auto *VT = dyn_cast<VectorType>(Val: T)) {
321 assert(VT->isFixedLength() && "x86-64 has no scalable vectors");
322 uint64_t Size = VT->getABISizeInBits();
323 const Type *ElementType = VT->getElementType();
324
325 if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
326 // gcc passes the following as integer:
327 // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
328 // 2 bytes - <2 x char>, <1 x short>
329 // 1 byte - <1 x char>
330 Current = Integer;
331 // If this type crosses an eightbyte boundary, it should be
332 // split.
333 uint64_t EbLo = (OffsetBase) / 64;
334 uint64_t EbHi = (OffsetBase + Size - 1) / 64;
335 if (EbLo != EbHi)
336 Hi = Lo;
337 } else if (Size == 64) {
338 if (const auto *FT = dyn_cast<FloatType>(Val: ElementType)) {
339 // gcc passes <1 x double> in memory. :(
340 if (FT->getSemantics() == &llvm::APFloat::IEEEdouble())
341 return;
342 }
343
344 // gcc passes <1 x long long> as SSE but clang used to unconditionally
345 // pass them as integer. For platforms where clang is the de facto
346 // platform compiler, we must continue to use integer.
347 if (const auto *IT = dyn_cast<IntegerType>(Val: ElementType)) {
348 uint64_t ElemBits = IT->getSizeInBits().getFixedValue();
349 if (!getX86ABICompatInfo().ClassifyIntegerMMXAsSSE && ElemBits == 64 &&
350 !IT->isBitInt()) {
351 Current = Integer;
352 } else {
353 Current = Sse;
354 }
355 } else {
356 Current = Sse;
357 }
358 // If this type crosses an eightbyte boundary, it should be
359 // split.
360 if (OffsetBase && OffsetBase != 64)
361 Hi = Lo;
362 } else if (Size == 128 ||
363 (IsNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
364 if (const auto *IT = dyn_cast<IntegerType>(Val: ElementType)) {
365 uint64_t ElemBits = IT->getSizeInBits().getFixedValue();
366 // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
367 if (getX86ABICompatInfo().PassInt128VectorsInMem && Size != 128 &&
368 ElemBits == 128 && !IT->isBitInt())
369 return;
370 }
371
372 // Arguments of 256-bits are split into four eightbyte chunks. The
373 // least significant one belongs to class SSE and all the others to class
374 // SSEUP. The original Lo and Hi design considers that types can't be
375 // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
376 // This design isn't correct for 256-bits, but since there're no cases
377 // where the upper parts would need to be inspected, avoid adding
378 // complexity and just consider Hi to match the 64-256 part.
379 //
380 // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
381 // registers if they are "named", i.e. not part of the "..." of a
382 // variadic function.
383 //
384 // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
385 // split into eight eightbyte chunks, one SSE and seven SSEUP.
386 Lo = Sse;
387 Hi = SseUp;
388 }
389 return;
390 }
391
392 if (const auto *CT = dyn_cast<ComplexType>(Val: T)) {
393 const Type *ElementType = CT->getElementType();
394 uint64_t Size = T->getABISizeInBits();
395
396 if (isa<IntegerType>(Val: ElementType)) {
397 if (Size <= 64)
398 Current = Integer;
399 else if (Size <= 128)
400 Lo = Hi = Integer;
401 } else if (const auto *EFT = dyn_cast<FloatType>(Val: ElementType)) {
402 const auto *FltSem = EFT->getSemantics();
403 if (FltSem == &llvm::APFloat::IEEEhalf() ||
404 FltSem == &llvm::APFloat::IEEEsingle() ||
405 FltSem == &llvm::APFloat::BFloat())
406 Current = Sse;
407 else if (FltSem == &llvm::APFloat::IEEEquad())
408 Current = Memory;
409 else if (FltSem == &llvm::APFloat::x87DoubleExtended())
410 Current = ComplexX87;
411 else if (FltSem == &llvm::APFloat::IEEEdouble())
412 Lo = Hi = Sse;
413 else
414 llvm_unreachable("Unexpected long double representation!");
415 }
416
417 uint64_t ElementSize = ElementType->getABISizeInBits();
418 // If this complex type crosses an eightbyte boundary then it
419 // should be split.
420 uint64_t EbReal = OffsetBase / 64;
421 uint64_t EbImag = (OffsetBase + ElementSize) / 64;
422 if (Hi == NoClass && EbReal != EbImag)
423 Hi = Lo;
424
425 return;
426 }
427
428 if (const auto *AT = dyn_cast<ArrayType>(Val: T)) {
429 // A matrix type is modeled as an array but, like Clang, is treated as a
430 // non-aggregate scalar: it matches no class here and stays in the Memory
431 // class, so classify*Type later returns it Direct (coerced to its
432 // flattened vector) rather than classifying it field-by-field.
433 if (AT->isMatrixType())
434 return;
435
436 // Arrays are treated like structures.
437 uint64_t Size = AT->getABISizeInBits();
438
439 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
440 // than eight eightbytes, ..., it has class MEMORY.
441 // regcall ABI doesn't have limitation to an object. The only limitation
442 // is the free registers, which will be checked in computeInfo.
443 if (!IsRegCall && Size > 512)
444 return;
445
446 // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
447 // fields, it has class MEMORY.
448 //
449 // Only need to check alignment of array base.
450 const Type *ElementType = AT->getElementType();
451 uint64_t ElemAlign = ElementType->getAlignment().value() * 8;
452 if (OffsetBase % ElemAlign)
453 return;
454
455 // Otherwise implement simplified merge. We could be smarter about
456 // this, but it isn't worth it and would be harder to verify.
457 Current = NoClass;
458 uint64_t EltSize = ElementType->getABISizeInBits();
459 uint64_t ArraySize = AT->getNumElements();
460
461 // The only case a 256-bit wide vector could be used is when the array
462 // contains a single 256-bit element. Since Lo and Hi logic isn't extended
463 // to work for sizes wider than 128, early check and fallback to memory.
464 //
465 if (Size > 128 &&
466 (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))
467 return;
468
469 for (uint64_t I = 0, Offset = OffsetBase; I < ArraySize;
470 ++I, Offset += EltSize) {
471 Class FieldLo, FieldHi;
472 classify(T: ElementType, OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, IsNamedArg);
473 Lo = merge(Accum: Lo, Field: FieldLo);
474 Hi = merge(Accum: Hi, Field: FieldHi);
475 if (Lo == Memory || Hi == Memory)
476 break;
477 }
478 postMerge(AggregateSize: Size, Lo, Hi);
479 assert((Hi != SseUp || Lo == Sse) && "Invalid SseUp array classification.");
480 return;
481 }
482
483 if (const auto *RT = dyn_cast<RecordType>(Val: T)) {
484 uint64_t Size = RT->getABISizeInBits();
485
486 if (containsMatrixField(RT)) {
487 Lo = Memory;
488 return;
489 }
490
491 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
492 // than eight eightbytes, ..., it has class MEMORY.
493 if (Size > 512)
494 return;
495
496 // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
497 // copy constructor or a non-trivial destructor, it is passed by invisible
498 // reference.
499 if (getRecordArgABI(RT))
500 return;
501
502 // Assume variable sized types are passed in memory.
503 if (RT->hasFlexibleArrayMember())
504 return;
505
506 // Reset Lo class, this will be recomputed.
507 Current = NoClass;
508
509 // If this is a C++ record, classify the bases first.
510 if (RT->isCXXRecord()) {
511 for (const auto &Base : RT->getBaseClasses()) {
512
513 // Classify this field.
514 //
515 // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
516 // single eightbyte, each is classified separately. Each eightbyte gets
517 // initialized to class NO_CLASS.
518 Class FieldLo, FieldHi;
519 uint64_t Offset = OffsetBase + Base.OffsetInBits;
520 classify(T: Base.FieldType, OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, IsNamedArg);
521 Lo = merge(Accum: Lo, Field: FieldLo);
522 Hi = merge(Accum: Hi, Field: FieldHi);
523
524 if (getX86ABICompatInfo().ReturnCXXRecordGreaterThan128InMem &&
525 (Size > 128 && (Size != Base.FieldType->getABISizeInBits() ||
526 Size > getNativeVectorSizeForAVXABI(AVXLevel))))
527 Lo = Memory;
528
529 if (Lo == Memory || Hi == Memory) {
530 postMerge(AggregateSize: Size, Lo, Hi);
531 return;
532 }
533 }
534 }
535
536 // Classify the fields one at a time, merging the results.
537
538 bool IsUnion = RT->isUnion() && !getX86ABICompatInfo().Clang11Compat;
539 for (const auto &Field : RT->getFields()) {
540 uint64_t Offset = OffsetBase + Field.OffsetInBits;
541 bool BitField = Field.IsBitField;
542
543 // Ignore padding bit-fields. Normally only zero-length bit-fields are
544 // padding, but under Clang 23 compatibility every unnamed bit-field is,
545 // faithfully reproducing Clang 23.
546 if (BitField && (getX86ABICompatInfo().ClassifyUnnamedBitFields
547 ? Field.BitFieldWidth == 0
548 : Field.IsUnnamedBitfield))
549 continue;
550
551 uint64_t FieldSize = Field.FieldType->getABISizeInBits();
552 if (Size > 128 && ((!IsUnion && Size != FieldSize) ||
553 Size > getNativeVectorSizeForAVXABI(AVXLevel))) {
554 Lo = Memory;
555 postMerge(AggregateSize: Size, Lo, Hi);
556 return;
557 }
558
559 bool IsInMemory = Offset % (Field.FieldType->getAlignment().value() * 8);
560 if (!BitField && IsInMemory) {
561 Lo = Memory;
562 postMerge(AggregateSize: Size, Lo, Hi);
563 return;
564 }
565
566 Class FieldLo, FieldHi;
567
568 if (BitField) {
569 uint64_t BitFieldSize = Field.BitFieldWidth;
570 uint64_t EbLo = Offset / 64;
571 uint64_t EbHi = (Offset + BitFieldSize - 1) / 64;
572
573 if (EbLo) {
574 assert(EbHi == EbLo && "Invalid classification, type > 16 bytes.");
575 FieldLo = NoClass;
576 FieldHi = Integer;
577 } else {
578 FieldLo = Integer;
579 FieldHi = EbHi ? Integer : NoClass;
580 }
581 } else {
582 classify(T: Field.FieldType, OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, IsNamedArg);
583 }
584
585 Lo = merge(Accum: Lo, Field: FieldLo);
586 Hi = merge(Accum: Hi, Field: FieldHi);
587 if (Lo == Memory || Hi == Memory)
588 break;
589 }
590 postMerge(AggregateSize: Size, Lo, Hi);
591 return;
592 }
593
594 Lo = Memory;
595 Hi = NoClass;
596}
597
598ArgInfo
599X86_64TargetInfo::classifyArgumentType(const Type *Ty, unsigned FreeIntRegs,
600 unsigned &NeededInt, unsigned &NeededSSE,
601 bool IsNamedArg, bool IsRegCall) const {
602
603 Ty = useFirstFieldIfTransparentUnion(Ty);
604
605 X86_64TargetInfo::Class Lo, Hi;
606 classify(T: Ty, OffsetBase: 0, Lo, Hi, IsNamedArg, IsRegCall);
607
608 // Check some invariants
609 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
610 assert((Hi != SseUp || Lo == Sse) && "Invalid SseUp classification.");
611
612 NeededInt = 0;
613 NeededSSE = 0;
614 const Type *ResType = nullptr;
615
616 switch (Lo) {
617 case NoClass:
618 if (Hi == NoClass)
619 return ArgInfo::getIgnore();
620 // If the low part is just padding, it takes no register, leave ResType
621 // null.
622 assert((Hi == Sse || Hi == Integer || Hi == X87Up) &&
623 "Unknown missing lo part");
624 break;
625
626 // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
627 // on the stack.
628 case Memory:
629 // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87Up or
630 // COMPLEX_X87, it is passed in memory.
631 case X87:
632 case ComplexX87:
633 if (getRecordArgABI(Ty) == RAA_Indirect)
634 ++NeededInt;
635 return getIndirectResult(Ty, FreeIntRegs);
636
637 case SseUp:
638 case X87Up:
639 llvm_unreachable("Invalid classification for lo word.");
640
641 // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
642 // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
643 // and %r9 is used.
644 case Integer:
645 ++NeededInt;
646
647 // Pick an 8-byte type based on the preferred type.
648 ResType = getIntegerTypeAtOffset(IRType: Ty, IROffset: 0, SourceTy: Ty, SourceOffset: 0);
649
650 // If we have a sign or zero extended integer, make sure to return Extend
651 // so that the parameter gets the right LLVM IR attributes.
652 if (Hi == NoClass && ResType->isInteger()) {
653 if (Ty->isInteger() && isPromotableInteger(IT: cast<IntegerType>(Val: Ty))) {
654 switch (getX86ABICompatInfo().SmallIntABICompat) {
655 case SmallIntABICompatKind::Platform:
656 if (cast<IntegerType>(Val: Ty)->isBool()) {
657 // Bool should be extended to 8-bits. We can't express this right
658 // now, so use conservative extension.
659 return ArgInfo::getConservativeExtend(T: Ty);
660 }
661 break;
662 case SmallIntABICompatKind::Clang23:
663 return ArgInfo::getExtend(T: Ty);
664 case SmallIntABICompatKind::Both:
665 return ArgInfo::getConservativeExtend(T: Ty);
666 }
667 }
668 }
669
670 if (ResType->isInteger() && ResType->getSizeInBits() == 128) {
671 assert(Hi == Integer);
672 ++NeededInt;
673 return ArgInfo::getDirect(T: ResType);
674 }
675 break;
676
677 // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
678 // available SSE register is used, the registers are taken in the
679 // order from %xmm0 to %xmm7.
680 case Sse:
681 ResType = getSSETypeAtOffset(ABIType: Ty, ABIOffset: 0, SourceTy: Ty, SourceOffset: 0);
682 ++NeededSSE;
683 break;
684 }
685
686 const Type *HighPart = nullptr;
687 switch (Hi) {
688 // Memory was handled previously, ComplexX87 and X87 should
689 // never occur as hi classes, and X87Up must be preceded by X87,
690 // which is passed in memory.
691 case Memory:
692 case X87:
693 case ComplexX87:
694 llvm_unreachable("Invalid classification for hi word.");
695
696 case NoClass:
697 break;
698
699 case Integer:
700 ++NeededInt;
701 // Pick an 8-byte type based on the preferred type.
702 HighPart = getIntegerTypeAtOffset(IRType: Ty, IROffset: 8, SourceTy: Ty, SourceOffset: 8);
703
704 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
705 return ArgInfo::getDirect(T: HighPart, Offset: 8);
706 break;
707
708 // X87Up generally doesn't occur here (long double is passed in
709 // memory), except in situations involving unions.
710 case X87Up:
711 case Sse:
712 ++NeededSSE;
713 HighPart = getSSETypeAtOffset(ABIType: Ty, ABIOffset: 8, SourceTy: Ty, SourceOffset: 8);
714
715 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
716 return ArgInfo::getDirect(T: HighPart, Offset: 8);
717 break;
718
719 // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
720 // eightbyte is passed in the upper half of the last used SSE
721 // register. This only happens when 128-bit vectors are passed.
722 case SseUp:
723 assert(Lo == Sse && "Unexpected SseUp classification");
724 ResType = getByteVectorType(Ty);
725 break;
726 }
727
728 // If a high part was specified, merge it together with the low part. It is
729 // known to pass in the high eightbyte of the result. We do this by forming a
730 // first class struct aggregate with the high and low part: {low, high}
731 if (HighPart)
732 ResType = createPairType(Lo: ResType, Hi: HighPart);
733
734 return ArgInfo::getDirect(T: ResType);
735}
736
737ArgInfo X86_64TargetInfo::classifyReturnType(const Type *RetTy) const {
738 // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
739 // classification algorithm.
740
741 X86_64TargetInfo::Class Lo, Hi;
742 classify(T: RetTy, OffsetBase: 0, Lo, Hi, /*isNamedArg*/ IsNamedArg: true);
743
744 // Check some invariants
745 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
746 assert((Hi != SseUp || Lo == Sse) && "Invalid SseUp classification.");
747
748 const Type *ResType = nullptr;
749 switch (Lo) {
750 case NoClass:
751 if (Hi == NoClass)
752 return ArgInfo::getIgnore();
753 // If the low part is just padding, it takes no register, leave ResType
754 // null.
755 assert((Hi == Sse || Hi == Integer || Hi == X87Up) &&
756 "Unknown missing lo part");
757 break;
758 case SseUp:
759 case X87Up:
760 llvm_unreachable("Invalid classification for lo word.");
761
762 // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
763 // hidden argument.
764 case Memory:
765 return getIndirectReturnResult(Ty: RetTy);
766
767 // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
768 // available register of the sequence %rax, %rdx is used.
769 case Integer:
770 ResType = getIntegerTypeAtOffset(IRType: RetTy, IROffset: 0, SourceTy: RetTy, SourceOffset: 0);
771
772 // Always returns integral types as plain integers, without padding to a
773 // larger type or splitting into a struct. For some targets, small integers
774 // are additionally sign/zero extended.
775 if (const IntegerType *IntTy = dyn_cast<IntegerType>(Val: RetTy))
776 return shouldExtendReturnType(RetTy: IntTy) ? ArgInfo::getExtend(T: RetTy)
777 : ArgInfo::getDirect(T: RetTy);
778
779 if (ResType->isInteger() && ResType->getSizeInBits() == 128) {
780 assert(Hi == Integer);
781 return ArgInfo::getDirect(T: ResType);
782 }
783 break;
784
785 // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
786 // available SSE register of the sequence %xmm0, %xmm1 is used.
787 case Sse:
788 ResType = getSSETypeAtOffset(ABIType: RetTy, ABIOffset: 0, SourceTy: RetTy, SourceOffset: 0);
789 break;
790
791 // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
792 // returned on the X87 stack in %st0 as 80-bit x87 number.
793 case X87:
794 ResType = TB.getFloatType(Semantics: APFloat::x87DoubleExtended(), Align: Align(16));
795 break;
796
797 // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
798 // part of the value is returned in %st0 and the imaginary part in
799 // %st1.
800 case ComplexX87:
801 assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
802 {
803 const Type *X87Type =
804 TB.getFloatType(Semantics: APFloat::x87DoubleExtended(), Align: Align(16));
805 FieldInfo Fields[] = {FieldInfo(X87Type, 0), FieldInfo(X87Type, 80)};
806 ResType = TB.getRecordType(Fields, Size: TypeSize::getFixed(ExactSize: 160), ABIAlign: Align(16),
807 /*UnadjustedAlign=*/Align(16));
808 }
809 break;
810 }
811
812 const Type *HighPart = nullptr;
813 switch (Hi) {
814 // Memory was handled previously and X87 should
815 // never occur as a hi class.
816 case Memory:
817 case X87:
818 llvm_unreachable("Invalid classification for hi word.");
819
820 case ComplexX87:
821 case NoClass:
822 break;
823
824 case Integer:
825 HighPart = getIntegerTypeAtOffset(IRType: RetTy, IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
826 if (Lo == NoClass)
827 return ArgInfo::getDirect(T: HighPart, Offset: 8);
828 break;
829
830 case Sse:
831 HighPart = getSSETypeAtOffset(ABIType: RetTy, ABIOffset: 8, SourceTy: RetTy, SourceOffset: 8);
832 if (Lo == NoClass)
833 return ArgInfo::getDirect(T: HighPart, Offset: 8);
834 break;
835
836 // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
837 // is passed in the next available eightbyte chunk if the last used
838 // vector register.
839 //
840 // SSEUP should always be preceded by SSE, just widen.
841 case SseUp:
842 assert(Lo == Sse && "Unexpected SseUp classification.");
843 ResType = getByteVectorType(Ty: RetTy);
844 break;
845
846 // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87Up, the value is
847 // returned together with the previous X87 value in %st0.
848 case X87Up:
849 // If X87Up is preceded by X87, we don't need to do
850 // anything. However, in some cases with unions it may not be
851 // preceded by X87. In such situations we follow gcc and pass the
852 // extra bits in an SSE reg.
853 if (Lo != X87) {
854 HighPart = getSSETypeAtOffset(ABIType: RetTy, ABIOffset: 8, SourceTy: RetTy, SourceOffset: 8);
855 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
856 return ArgInfo::getDirect(T: HighPart, Offset: 8);
857 }
858 break;
859 }
860
861 // If a high part was specified, merge it together with the low part. It is
862 // known to pass in the high eightbyte of the result. We do this by forming a
863 // first class struct aggregate with the high and low part: {low, high}
864 if (HighPart)
865 ResType = createPairType(Lo: ResType, Hi: HighPart);
866
867 return ArgInfo::getDirect(T: ResType);
868}
869
870/// Given a high and low type that can ideally
871/// be used as elements of a two register pair to pass or return, return a
872/// first class aggregate to represent them. For example, if the low part of
873/// a by-value argument should be passed as i32* and the high part as float,
874/// return {i32*, float}.
875const Type *X86_64TargetInfo::createPairType(const Type *Lo,
876 const Type *Hi) const {
877 // In order to correctly satisfy the ABI, we need to the high part to start
878 // at offset 8. If the high and low parts we inferred are both 4-byte types
879 // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
880 // the second element at offset 8. Check for this:
881 unsigned LoSize = (unsigned)Lo->getTypeAllocSize();
882 llvm::Align HiAlign = Hi->getAlignment();
883 unsigned HiStart = alignTo(Size: LoSize, A: HiAlign);
884
885 assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
886
887 // To handle this, we have to increase the size of the low part so that the
888 // second element will start at an 8 byte offset. We can't increase the size
889 // of the second element because it might make us access off the end of the
890 // struct.
891 const Type *AdjustedLo = Lo;
892 if (HiStart != 8) {
893 // There are usually two sorts of types the ABI generation code can produce
894 // for the low part of a pair that aren't 8 bytes in size: half, float or
895 // i8/i16/i32. This can also include pointers when they are 32-bit (X32 and
896 // NaCl).
897 // Promote these to a larger type.
898 if (Lo->isFloat()) {
899 const FloatType *FT = cast<FloatType>(Val: Lo);
900 if (FT->getSemantics() == &APFloat::IEEEhalf() ||
901 FT->getSemantics() == &APFloat::IEEEsingle() ||
902 FT->getSemantics() == &APFloat::BFloat())
903 AdjustedLo = TB.getFloatType(Semantics: APFloat::IEEEdouble(), Align: Align(8));
904 }
905 // Promote integers and pointers to i64
906 else if (Lo->isInteger() || Lo->isPointer())
907 AdjustedLo = TB.getIntegerType(BitWidth: 64, Align: Align(8), /*Signed=*/false);
908 else
909 assert((Lo->isInteger() || Lo->isPointer()) &&
910 "Invalid/unknown low type in pair");
911 unsigned AdjustedLoSize = AdjustedLo->getSizeInBits().getFixedValue() / 8;
912 HiStart = alignTo(Size: AdjustedLoSize, A: HiAlign);
913 }
914
915 // Create the pair struct
916 FieldInfo Fields[] = {FieldInfo(AdjustedLo, 0), FieldInfo(Hi, HiStart * 8)};
917
918 // Verify the high part is at offset 8
919 assert((8 * 8) == Fields[1].OffsetInBits &&
920 "High part must be at offset 8 bytes");
921
922 uint64_t PairSizeInBits =
923 Fields[1].OffsetInBits + Hi->getSizeInBits().getFixedValue();
924 return TB.getRecordType(Fields, Size: TypeSize::getFixed(ExactSize: PairSizeInBits), ABIAlign: Align(8),
925 /*UnadjustedAlign=*/Align(8), Pack: StructPacking::Default);
926}
927
928static bool bitsContainNoUserData(const Type *Ty, unsigned StartBit,
929 unsigned EndBit) {
930 // If range is completely beyond type size, it's definitely padding
931 unsigned TySize = Ty->getABISizeInBits();
932 if (TySize <= StartBit)
933 return true;
934
935 // Handle arrays - check each element
936 if (const ArrayType *AT = dyn_cast<ArrayType>(Val: Ty)) {
937 const Type *EltTy = AT->getElementType();
938 unsigned EltSize = EltTy->getABISizeInBits();
939
940 for (unsigned I = 0; I < AT->getNumElements(); ++I) {
941 unsigned EltOffset = I * EltSize;
942 if (EltOffset >= EndBit)
943 break;
944
945 unsigned EltStart = (EltOffset < StartBit) ? StartBit - EltOffset : 0;
946 if (!bitsContainNoUserData(Ty: EltTy, StartBit: EltStart, EndBit: EndBit - EltOffset))
947 return false;
948 }
949 return true;
950 }
951
952 // Handle records - check all fields and base classes. getUnionType places a
953 // union's members at offset zero, so the field loop covers a union too.
954 if (const RecordType *RT = dyn_cast<RecordType>(Val: Ty)) {
955 // Check base classes first (for C++ records)
956 if (RT->isCXXRecord()) {
957 for (unsigned I = 0; I < RT->getNumBaseClasses(); ++I) {
958 const FieldInfo &Base = RT->getBaseClasses()[I];
959 if (Base.OffsetInBits >= EndBit)
960 continue;
961
962 unsigned BaseStart =
963 (Base.OffsetInBits < StartBit) ? StartBit - Base.OffsetInBits : 0;
964 if (!bitsContainNoUserData(Ty: Base.FieldType, StartBit: BaseStart,
965 EndBit: EndBit - Base.OffsetInBits))
966 return false;
967 }
968 }
969
970 for (unsigned I = 0; I < RT->getNumFields(); ++I) {
971 const FieldInfo &Field = RT->getFields()[I];
972 if (Field.OffsetInBits >= EndBit)
973 break;
974
975 unsigned FieldStart =
976 (Field.OffsetInBits < StartBit) ? StartBit - Field.OffsetInBits : 0;
977 if (!bitsContainNoUserData(Ty: Field.FieldType, StartBit: FieldStart,
978 EndBit: EndBit - Field.OffsetInBits))
979 return false;
980 }
981 return true;
982 }
983
984 // For any other type - assume all bits are user data
985 return false;
986}
987
988const Type *X86_64TargetInfo::getIntegerTypeAtOffset(const Type *ABIType,
989 unsigned ABIOffset,
990 const Type *SourceTy,
991 unsigned SourceOffset,
992 bool InMemory) const {
993
994 const Type *WorkingType = ABIType;
995 if (InMemory && ABIType->isInteger()) {
996 const auto *IT = cast<IntegerType>(Val: ABIType);
997 unsigned OriginalBitWidth = IT->getSizeInBits().getFixedValue();
998
999 unsigned WidenedBitWidth = OriginalBitWidth;
1000 if (OriginalBitWidth <= 8) {
1001 WidenedBitWidth = 8;
1002 } else {
1003 WidenedBitWidth = llvm::bit_ceil(Value: OriginalBitWidth);
1004 }
1005
1006 if (WidenedBitWidth != OriginalBitWidth) {
1007 WorkingType = TB.getIntegerType(BitWidth: WidenedBitWidth, Align: ABIType->getAlignment(),
1008 Signed: IT->isSigned());
1009 }
1010 }
1011 // A bool vector is stored as an integer with one bit per element, at least a
1012 // byte wide.
1013 if (InMemory && ABIType->isVector()) {
1014 const auto *VT = cast<VectorType>(Val: ABIType);
1015 const auto *IT = dyn_cast<IntegerType>(Val: VT->getElementType());
1016 if (IT && IT->isBool())
1017 WorkingType = TB.getIntegerType(
1018 BitWidth: std::max<uint64_t>(a: VT->getNumElements().getFixedValue(), b: 8),
1019 Align: ABIType->getAlignment(), /*Signed=*/false);
1020 }
1021 // If we're dealing with an un-offset ABI type, then it means that we're
1022 // returning an 8-byte unit starting with it. See if we can safely use it.
1023 if (ABIOffset == 0) {
1024 // Pointers and int64's always fill the 8-byte unit. Return WorkingType,
1025 // which is the in-memory-widened type (e.g. a _BitInt(37) field widened to
1026 // i64): returning the raw ABIType here would coerce the eightbyte to the
1027 // narrow iN instead of the storage integer clang uses.
1028 if ((WorkingType->isPointer() && Has64BitPointers) ||
1029 (WorkingType->isInteger() &&
1030 cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 64))
1031 return WorkingType;
1032
1033 // If we have a 1/2/4-byte integer, we can use it only if the rest of the
1034 // goodness in the source type is just tail padding. This is allowed to
1035 // kick in for struct {double,int} on the int, but not on
1036 // struct{double,int,int} because we wouldn't return the second int. We
1037 // have to do this analysis on the source type because we can't depend on
1038 // unions being lowered a specific way etc.
1039 if ((WorkingType->isInteger() &&
1040 (cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 1 ||
1041 cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 8 ||
1042 cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 16 ||
1043 cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 32)) ||
1044 (WorkingType->isPointer() && !Has64BitPointers)) {
1045
1046 unsigned BitWidth = WorkingType->isPointer()
1047 ? 32
1048 : cast<IntegerType>(Val: WorkingType)->getSizeInBits();
1049
1050 if (bitsContainNoUserData(Ty: SourceTy, StartBit: SourceOffset * 8 + BitWidth,
1051 EndBit: SourceOffset * 8 + 64))
1052 return WorkingType;
1053 }
1054 }
1055
1056 if (const auto *RTy = dyn_cast<RecordType>(Val: ABIType)) {
1057 if (RTy->isUnion()) {
1058 const Type *ReducedType = reduceUnionForX8664(UnionType: RTy, TB);
1059 if (ReducedType) {
1060 if (ABIOffset * 8 < ReducedType->getABISizeInBits())
1061 return getIntegerTypeAtOffset(ABIType: ReducedType, ABIOffset, SourceTy,
1062 SourceOffset, InMemory: true);
1063 // The storage type stops before this offset, so size the coercion
1064 // from the union itself: a byte when the rest of this eightbyte
1065 // holds no data, and the union's remaining bytes otherwise.
1066 if (bitsContainNoUserData(Ty: SourceTy, StartBit: SourceOffset * 8 + 8,
1067 EndBit: SourceOffset * 8 + 64))
1068 return TB.getIntegerType(BitWidth: 8, Align: Align(1), /*Signed=*/false);
1069 unsigned RemainingBytes =
1070 llvm::divideCeil(Numerator: SourceTy->getABISizeInBits(), Denominator: 8) - SourceOffset;
1071 return TB.getIntegerType(BitWidth: std::min(a: RemainingBytes, b: 8U) * 8, Align: Align(1),
1072 /*Signed=*/false);
1073 }
1074 }
1075 if (const FieldInfo *Element =
1076 RTy->getElementContainingOffset(OffsetInBits: ABIOffset * 8)) {
1077
1078 unsigned ElementOffsetBytes = Element->OffsetInBits / 8;
1079 return getIntegerTypeAtOffset(ABIType: Element->FieldType,
1080 ABIOffset: ABIOffset - ElementOffsetBytes, SourceTy,
1081 SourceOffset, InMemory: true);
1082 }
1083 }
1084
1085 if (const auto *ATy = dyn_cast<ArrayType>(Val: ABIType)) {
1086 const Type *EltTy = ATy->getElementType();
1087 unsigned EltSize = EltTy->getABISizeInBits() / 8;
1088 if (EltSize > 0) {
1089 unsigned EltOffset = (ABIOffset / EltSize) * EltSize;
1090 return getIntegerTypeAtOffset(ABIType: EltTy, ABIOffset: ABIOffset - EltOffset, SourceTy,
1091 SourceOffset, InMemory: true);
1092 }
1093 }
1094
1095 // If we have a 128-bit integer, we can pass it safely using an i128
1096 // so we return that
1097 if (ABIType->isInteger() && ABIType->getSizeInBits() == 128) {
1098 assert(ABIOffset == 0);
1099 return ABIType;
1100 }
1101
1102 unsigned TySizeInBytes = llvm::divideCeil(Numerator: SourceTy->getABISizeInBits(), Denominator: 8);
1103 assert(TySizeInBytes != SourceOffset && "Empty field?");
1104 unsigned AvailableSize = TySizeInBytes - SourceOffset;
1105 return TB.getIntegerType(BitWidth: std::min(a: AvailableSize, b: 8U) * 8, Align: Align(1), Signed: false);
1106}
1107/// Returns the floating point type at the specified offset within a type, or
1108/// nullptr if no floating point type is found at that offset.
1109const Type *X86_64TargetInfo::getFPTypeAtOffset(const Type *Ty,
1110 unsigned Offset) const {
1111 // Check for direct match at offset 0
1112 if (Offset == 0 && Ty->isFloat())
1113 return Ty;
1114
1115 if (const ComplexType *CT = dyn_cast<ComplexType>(Val: Ty)) {
1116 const Type *ElementType = CT->getElementType();
1117 unsigned ElementSize = ElementType->getABISizeInBits() / 8;
1118
1119 if (Offset == 0 || Offset == ElementSize)
1120 return ElementType;
1121 return nullptr;
1122 }
1123
1124 // Handle struct types by checking each field
1125 if (const RecordType *RT = dyn_cast<RecordType>(Val: Ty)) {
1126 if (const FieldInfo *Element = RT->getElementContainingOffset(OffsetInBits: Offset * 8)) {
1127 unsigned ElementOffsetBytes = Element->OffsetInBits / 8;
1128 return getFPTypeAtOffset(Ty: Element->FieldType, Offset: Offset - ElementOffsetBytes);
1129 }
1130 }
1131
1132 // Handle array types
1133 if (const ArrayType *AT = dyn_cast<ArrayType>(Val: Ty)) {
1134 const Type *EltTy = AT->getElementType();
1135 unsigned EltSize = EltTy->getABISizeInBits() / 8;
1136 if (EltSize == 0)
1137 return nullptr;
1138 unsigned EltIndex = Offset / EltSize;
1139
1140 return getFPTypeAtOffset(Ty: EltTy, Offset: Offset - (EltIndex * EltSize));
1141 }
1142
1143 // No floating point type found at this offset
1144 return nullptr;
1145}
1146
1147/// Helper to check if a floating point type matches specific semantics
1148static bool isFloatTypeWithSemantics(const Type *Ty,
1149 const fltSemantics &Semantics) {
1150 if (!Ty->isFloat())
1151 return false;
1152 const FloatType *FT = cast<FloatType>(Val: Ty);
1153 return FT->getSemantics() == &Semantics;
1154}
1155
1156/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
1157/// low 8 bytes of an XMM register, corresponding to the SSE class.
1158const Type *X86_64TargetInfo::getSSETypeAtOffset(const Type *ABIType,
1159 unsigned ABIOffset,
1160 const Type *SourceTy,
1161 unsigned SourceOffset) const {
1162
1163 if (const auto *RTy = dyn_cast<RecordType>(Val: ABIType)) {
1164 if (RTy->isUnion()) {
1165 const Type *ReducedType = reduceUnionForX8664(UnionType: RTy, TB);
1166 if (ReducedType) {
1167 return getSSETypeAtOffset(ABIType: ReducedType, ABIOffset, SourceTy,
1168 SourceOffset);
1169 }
1170 }
1171 }
1172
1173 auto Is16bitFpTy = [](const Type *T) {
1174 return isFloatTypeWithSemantics(Ty: T, Semantics: APFloat::IEEEhalf()) ||
1175 isFloatTypeWithSemantics(Ty: T, Semantics: APFloat::BFloat());
1176 };
1177
1178 // Get the floating point type at the requested offset
1179 const Type *T0 = getFPTypeAtOffset(Ty: ABIType, Offset: ABIOffset);
1180 if (!T0 || isFloatTypeWithSemantics(Ty: T0, Semantics: APFloat::IEEEdouble()))
1181 return TB.getFloatType(Semantics: APFloat::IEEEdouble(), Align: Align(8));
1182
1183 // Calculate remaining source size in bytes
1184 unsigned SourceSize = (SourceTy->getABISizeInBits() / 8) - SourceOffset;
1185
1186 // Try to get adjacent FP type
1187 const Type *T1 = nullptr;
1188 unsigned T0Size = T0->getABISizeInBits() / 8;
1189 if (SourceSize > T0Size)
1190 T1 = getFPTypeAtOffset(Ty: ABIType, Offset: ABIOffset + T0Size);
1191
1192 if (T1 == nullptr) {
1193 if (Is16bitFpTy(T0) && SourceSize > 4)
1194 T1 = getFPTypeAtOffset(Ty: ABIType, Offset: ABIOffset + 4);
1195
1196 if (T1 == nullptr)
1197 return T0;
1198 }
1199 // Handle vector cases
1200 if (isFloatTypeWithSemantics(Ty: T0, Semantics: APFloat::IEEEsingle()) &&
1201 isFloatTypeWithSemantics(Ty: T1, Semantics: APFloat::IEEEsingle()))
1202 return TB.getVectorType(ElementType: T0, NumElements: ElementCount::getFixed(MinVal: 2), Align: Align(8));
1203
1204 if (Is16bitFpTy(T0) && Is16bitFpTy(T1)) {
1205 const Type *T2 = nullptr;
1206 if (SourceSize > 4)
1207 T2 = getFPTypeAtOffset(Ty: ABIType, Offset: ABIOffset + 4);
1208 if (!T2)
1209 return TB.getVectorType(ElementType: T0, NumElements: ElementCount::getFixed(MinVal: 2), Align: Align(8));
1210 return TB.getVectorType(ElementType: T0, NumElements: ElementCount::getFixed(MinVal: 4), Align: Align(8));
1211 }
1212
1213 // Mixed half-float cases
1214 if (Is16bitFpTy(T0) || Is16bitFpTy(T1))
1215 return TB.getVectorType(ElementType: TB.getFloatType(Semantics: APFloat::IEEEhalf(), Align: Align(2)),
1216 NumElements: ElementCount::getFixed(MinVal: 4), Align: Align(8));
1217
1218 // Default to double
1219 return TB.getFloatType(Semantics: APFloat::IEEEdouble(), Align: Align(8));
1220}
1221
1222/// The ABI specifies that a value should be passed in a full vector XMM/YMM
1223/// register. Pick an LLVM IR type that will be passed as a vector register.
1224const Type *X86_64TargetInfo::getByteVectorType(const Type *Ty) const {
1225 // Wrapper structs/arrays that only contain vectors are passed just like
1226 // vectors; strip them off if present.
1227 if (const Type *InnerTy = isSingleElementStruct(Ty))
1228 Ty = InnerTy;
1229
1230 // Handle vector types
1231 if (const VectorType *VT = dyn_cast<VectorType>(Val: Ty)) {
1232 // Don't pass vXi128 vectors in their native type, the backend can't
1233 // legalize them.
1234 if (getX86ABICompatInfo().PassInt128VectorsInMem &&
1235 VT->getElementType()->isInteger() &&
1236 cast<IntegerType>(Val: VT->getElementType())->getSizeInBits() == 128) {
1237 unsigned Size = VT->getABISizeInBits();
1238 return TB.getVectorType(ElementType: TB.getIntegerType(BitWidth: 64, Align: Align(8), /*Signed=*/false),
1239 NumElements: ElementCount::getFixed(MinVal: Size / 64),
1240 Align: Align(Size / 8));
1241 }
1242 return VT;
1243 }
1244
1245 // Handle fp128
1246 if (isFloatTypeWithSemantics(Ty, Semantics: APFloat::IEEEquad()))
1247 return Ty;
1248
1249 // We couldn't find the preferred IR vector type for 'Ty'.
1250 unsigned Size = Ty->getABISizeInBits();
1251 assert((Size == 128 || Size == 256 || Size == 512) && "Invalid vector size");
1252
1253 return TB.getVectorType(ElementType: TB.getFloatType(Semantics: APFloat::IEEEdouble(), Align: Align(8)),
1254 NumElements: ElementCount::getFixed(MinVal: Size / 64), Align: Align(Size / 8));
1255}
1256
1257bool X86_64TargetInfo::isIllegalVectorType(const Type *Ty) const {
1258 if (const auto *VecTy = dyn_cast<VectorType>(Val: Ty)) {
1259 uint64_t Size = VecTy->getABISizeInBits();
1260 unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
1261
1262 // Vectors <= 64 bits or > largest supported vector size are illegal
1263 if (Size <= 64 || Size > LargestVector)
1264 return true;
1265
1266 // Check for 128-bit integer element vectors that should be passed in memory
1267 const Type *EltTy = VecTy->getElementType();
1268 if (getX86ABICompatInfo().PassInt128VectorsInMem && EltTy->isInteger()) {
1269 const auto *IntTy = cast<IntegerType>(Val: EltTy);
1270 if (IntTy->getSizeInBits().getFixedValue() == 128 && !IntTy->isBitInt())
1271 return true;
1272 }
1273 }
1274 return false;
1275}
1276
1277ArgInfo X86_64TargetInfo::getIndirectResult(const Type *Ty,
1278 unsigned FreeIntRegs) const {
1279 // If this is a scalar LLVM value then assume LLVM will pass it in the right
1280 // place naturally.
1281 //
1282 // This assumption is optimistic, as there could be free registers available
1283 // when we need to pass this argument in memory, and LLVM could try to pass
1284 // the argument in the free register. This does not seem to happen currently,
1285 // but this code would be much safer if we could mark the argument with
1286 // 'onstack'. See PR12193.
1287 if (!isAggregateTypeForABI(Ty) && !isIllegalVectorType(Ty) &&
1288 !(Ty->isInteger() && cast<IntegerType>(Val: Ty)->isBitInt())) {
1289 return (Ty->isInteger() && isPromotableInteger(IT: cast<IntegerType>(Val: Ty))
1290 ? ArgInfo::getExtend(T: Ty)
1291 : ArgInfo::getDirect());
1292 }
1293
1294 // Check if this is a record type that needs special handling
1295 if (auto RecordRAA = getRecordArgABI(Ty))
1296 return getNaturalAlignIndirect(Ty, AddrSpace: getAllocaAddrSpace(),
1297 /*ByVal=*/RecordRAA ==
1298 RecordArgABI::RAA_DirectInMemory);
1299
1300 // Compute the byval alignment. We specify the alignment of the byval in all
1301 // cases so that the mid-level optimizer knows the alignment of the byval.
1302 uint64_t AlignVal = std::max<uint64_t>(a: Ty->getAlignment().value(), b: 8u);
1303
1304 // Attempt to avoid passing indirect results using byval when possible. This
1305 // is important for good codegen.
1306 //
1307 // We do this by coercing the value into a scalar type which the backend can
1308 // handle naturally (i.e., without using byval).
1309 //
1310 // For simplicity, we currently only do this when we have exhausted all of the
1311 // free integer registers. Doing this when there are free integer registers
1312 // would require more care, as we would have to ensure that the coerced value
1313 // did not claim the unused register. That would require either reording the
1314 // arguments to the function (so that any subsequent inreg values came first),
1315 // or only doing this optimization when there were no following arguments that
1316 // might be inreg.
1317 //
1318 // We currently expect it to be rare (particularly in well written code) for
1319 // arguments to be passed on the stack when there are still free integer
1320 // registers available (this would typically imply large structs being passed
1321 // by value), so this seems like a fair tradeoff for now.
1322 //
1323 // We can revisit this if the backend grows support for 'onstack' parameter
1324 // attributes. See PR12193.
1325 if (FreeIntRegs == 0) {
1326 uint64_t Size = Ty->getABISizeInBits();
1327
1328 // If this type fits in an eightbyte, coerce it into the matching integral
1329 // type, which will end up on the stack (with alignment 8).
1330 if (AlignVal == 8 && Size <= 64) {
1331 const Type *IntTy =
1332 TB.getIntegerType(BitWidth: Size, Align: llvm::Align(8), /*Signed=*/false);
1333 return ArgInfo::getDirect(T: IntTy);
1334 }
1335 }
1336
1337 return ArgInfo::getIndirect(Align: llvm::Align(AlignVal), /*ByVal=*/true);
1338}
1339
1340ArgInfo X86_64TargetInfo::getIndirectReturnResult(const Type *Ty) const {
1341 if (!isAggregateTypeForABI(Ty)) {
1342 // Bit-precise integers are returned indirectly regardless of size.
1343 if (const auto *IntTy = dyn_cast<IntegerType>(Val: Ty)) {
1344 if (IntTy->isBitInt())
1345 return getNaturalAlignIndirect(Ty: IntTy, AddrSpace: getAllocaAddrSpace());
1346 if (shouldExtendReturnType(RetTy: IntTy))
1347 return ArgInfo::getExtend(T: Ty);
1348 }
1349 return ArgInfo::getDirect();
1350 }
1351
1352 return getNaturalAlignIndirect(Ty, AddrSpace: getAllocaAddrSpace());
1353}
1354
1355void X86_64TargetInfo::computeInfo(FunctionInfo &FI) const {
1356 CallingConv::ID CallingConv = FI.getCallingConvention();
1357
1358 // Only the standard SysV (C) calling convention is classified here. Any other
1359 // convention must be added explicitly once it has been verified against this
1360 // classifier rather than silently taking the SysV path.
1361 switch (CallingConv) {
1362 case CallingConv::C:
1363 break;
1364 default:
1365 llvm_unreachable(
1366 "calling convention not supported by the LLVMABI X86_64 classifier");
1367 }
1368
1369 unsigned FreeIntRegs = 6;
1370 unsigned FreeSSERegs = 8;
1371 unsigned NeededInt = 0, NeededSSE = 0;
1372
1373 if (!maybeCommonClassifyReturnType(FI)) {
1374 const Type *RetTy = FI.getReturnType();
1375 FI.getReturnInfo() = classifyReturnType(RetTy);
1376 }
1377
1378 if (FI.getReturnInfo().isIndirect())
1379 --FreeIntRegs;
1380
1381 unsigned NumRequiredArgs = FI.getNumRequiredArgs();
1382
1383 unsigned ArgNo = 0;
1384 for (auto IT = FI.arg_begin(), IE = FI.arg_end(); IT != IE; ++IT, ++ArgNo) {
1385 bool IsNamedArg = ArgNo < NumRequiredArgs;
1386 const Type *ArgTy = IT->ABIType;
1387 NeededInt = 0;
1388 NeededSSE = 0;
1389
1390 ArgInfo AI = classifyArgumentType(Ty: ArgTy, FreeIntRegs, NeededInt, NeededSSE,
1391 IsNamedArg);
1392
1393 // AMD64-ABI 3.2.3p3: If there are no registers available for any
1394 // eightbyte of an argument, the whole argument is passed on the
1395 // stack. If registers have already been assigned for some
1396 // eightbytes of such an argument, the assignments get reverted.
1397 if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
1398 FreeIntRegs -= NeededInt;
1399 FreeSSERegs -= NeededSSE;
1400 AI.setNeededRegs(IntRegs: NeededInt, SseRegs: NeededSSE);
1401 IT->Info = AI;
1402 } else {
1403 // Not enough registers, pass on stack. The demand the classification
1404 // reports is what the argument ends up occupying, which is nothing.
1405 IT->Info = getIndirectResult(Ty: ArgTy, FreeIntRegs);
1406 }
1407 }
1408}
1409
1410std::unique_ptr<TargetInfo>
1411createX86_64TargetInfo(TypeBuilder &TB, X86AVXABILevel AVXLevel,
1412 bool Has64BitPointers, const X86ABICompatInfo &Compat) {
1413 return std::make_unique<X86_64TargetInfo>(args&: TB, args&: AVXLevel, args&: Has64BitPointers,
1414 args: Compat);
1415}
1416
1417} // namespace abi
1418} // namespace llvm
1419