1//===- X86.cpp ------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ABI/FunctionInfo.h"
10#include "llvm/ABI/TargetInfo.h"
11#include "llvm/ABI/Types.h"
12#include "llvm/Support/Alignment.h"
13#include "llvm/Support/Casting.h"
14#include "llvm/Support/ErrorHandling.h"
15#include "llvm/Support/MathExtras.h"
16#include "llvm/Support/TypeSize.h"
17#include <algorithm>
18#include <cassert>
19#include <cstdint>
20
21namespace llvm {
22namespace abi {
23
24static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
25 switch (AVXLevel) {
26 case X86AVXABILevel::AVX512:
27 return 512;
28 case X86AVXABILevel::AVX:
29 return 256;
30 case X86AVXABILevel::None:
31 return 128;
32 }
33 llvm_unreachable("Unknown AVXLevel");
34}
35
36class X86_64TargetInfo : public TargetInfo {
37public:
38 enum Class { Integer, Sse, SseUp, X87, X87Up, ComplexX87, NoClass, Memory };
39
40private:
41 X86AVXABILevel AVXLevel;
42 bool Has64BitPointers;
43 X86ABICompatInfo X86CompatInfo;
44
45 static Class merge(Class Accum, Class Field);
46
47 void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
48
49 void classify(const Type *T, uint64_t OffsetBase, Class &Lo, Class &Hi,
50 bool IsNamedArg, bool IsRegCall = false) const;
51
52 const Type *getIntegerTypeAtOffset(const Type *IRType, unsigned IROffset,
53 const Type *SourceTy,
54 unsigned SourceOffset,
55 bool InMemory = false) const;
56
57 const Type *getSSETypeAtOffset(const Type *ABIType, unsigned ABIOffset,
58 const Type *SourceTy,
59 unsigned SourceOffset) const;
60 bool isIllegalVectorType(const Type *Ty) const;
61 bool containsMatrixField(const RecordType *RT) const;
62
63 void computeInfo(FunctionInfo &FI) const override;
64 ArgInfo getIndirectReturnResult(const Type *Ty) const;
65 const Type *getFPTypeAtOffset(const Type *Ty, unsigned Offset) const;
66
67 const Type *getByteVectorType(const Type *Ty) const;
68
69 const Type *createPairType(const Type *Lo, const Type *Hi) const;
70 ArgInfo getIndirectResult(const Type *Ty, unsigned FreeIntRegs) const;
71
72 ArgInfo classifyReturnType(const Type *RetTy) const;
73
74 ArgInfo classifyArgumentType(const Type *Ty, unsigned FreeIntRegs,
75 unsigned &NeededInt, unsigned &NeededSse,
76 bool IsNamedArg, bool IsRegCall = false) const;
77
78public:
79 X86_64TargetInfo(TypeBuilder &TypeBuilder, X86AVXABILevel AVXABILevel,
80 bool Has64BitPtrs, const X86ABICompatInfo &Compat)
81 : TargetInfo(TypeBuilder), AVXLevel(AVXABILevel),
82 Has64BitPointers(Has64BitPtrs), X86CompatInfo(Compat) {}
83
84 bool has64BitPointers() const { return Has64BitPointers; }
85
86 const ABICompatInfo &getABICompatInfo() const override {
87 return X86CompatInfo;
88 }
89
90 const X86ABICompatInfo &getX86ABICompatInfo() const { return X86CompatInfo; }
91};
92
93static bool bitsContainNoUserData(const Type *Ty, unsigned StartBit,
94 unsigned EndBit);
95
96// Gets the "best" type to represent the union.
97static const Type *reduceUnionForX8664(const RecordType *UnionType,
98 TypeBuilder &TB) {
99 assert(UnionType->isUnion() && "Expected union type");
100
101 ArrayRef<FieldInfo> Fields = UnionType->getFields();
102 if (Fields.empty()) {
103 return nullptr;
104 }
105
106 const Type *StorageType = nullptr;
107
108 for (const auto &Field : Fields) {
109 if (Field.IsBitField && Field.IsUnnamedBitfield &&
110 Field.BitFieldWidth == 0) {
111 continue;
112 }
113
114 const Type *FieldType = Field.FieldType;
115
116 if (UnionType->isTransparentUnion() && !StorageType) {
117 StorageType = FieldType;
118 break;
119 }
120
121 // A member that holds no user data supplies no bytes for a coercion to
122 // read, so it must not become the storage type however wide or aligned it
123 // is declared. Clang compares lowered types instead, where an empty class
124 // is a byte array whose i8 leaf lets getIntegerTypeAtOffset narrow the
125 // coercion. A record mapped here holds no fields, so there is no such
126 // leaf and the eightbyte would be sized from the union.
127 if (bitsContainNoUserData(Ty: FieldType, StartBit: 0, EndBit: FieldType->getABISizeInBits()))
128 continue;
129
130 if (!StorageType ||
131 FieldType->getAlignment() > StorageType->getAlignment() ||
132 (FieldType->getAlignment() == StorageType->getAlignment() &&
133 TypeSize::isKnownGT(LHS: FieldType->getSizeInBits(),
134 RHS: StorageType->getSizeInBits()))) {
135 StorageType = FieldType;
136 }
137 }
138 return StorageType;
139}
140
141void X86_64TargetInfo::postMerge(unsigned AggregateSize, Class &Lo,
142 Class &Hi) const {
143 // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
144 //
145 // (a) If one of the classes is Memory, the whole argument is passed in
146 // memory.
147 //
148 // (b) If X87Up is not preceded by X87, the whole argument is passed in
149 // memory.
150 //
151 // (c) If the size of the aggregate exceeds two eightbytes and the first
152 // eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
153 // argument is passed in memory. NOTE: This is necessary to keep the
154 // ABI working for processors that don't support the __m256 type.
155 //
156 // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
157 //
158 // Some of these are enforced by the merging logic. Others can arise
159 // only with unions; for example:
160 // union { _Complex double; unsigned; }
161 //
162 // Note that clauses (b) and (c) were added in 0.98.
163
164 if (Hi == Memory)
165 Lo = Memory;
166 if (Hi == X87Up && Lo != X87 && getX86ABICompatInfo().HonorsRevision98)
167 Lo = Memory;
168 if (AggregateSize > 128 && (Lo != Sse || Hi != SseUp))
169 Lo = Memory;
170 if (Hi == SseUp && Lo != Sse)
171 Hi = Sse;
172}
173X86_64TargetInfo::Class X86_64TargetInfo::merge(Class Accum, Class Field) {
174 // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
175 // classified recursively so that always two fields are
176 // considered. The resulting class is calculated according to
177 // the classes of the fields in the eightbyte:
178 //
179 // (a) If both classes are equal, this is the resulting class.
180 //
181 // (b) If one of the classes is NO_CLASS, the resulting class is
182 // the other class.
183 //
184 // (c) If one of the classes is MEMORY, the result is the MEMORY
185 // class.
186 //
187 // (d) If one of the classes is INTEGER, the result is the
188 // INTEGER.
189 //
190 // (e) If one of the classes is X87, X87Up, COMPLEX_X87 class,
191 // MEMORY is used as class.
192 //
193 // (f) Otherwise class SSE is used.
194
195 // Accum should never be memory (we should have returned) or
196 // ComplexX87 (because this cannot be passed in a structure).
197 assert((Accum != Memory && Accum != ComplexX87) &&
198 "Invalid accumulated classification during merge.");
199
200 if (Accum == Field || Field == NoClass)
201 return Accum;
202 if (Field == Memory)
203 return Memory;
204 if (Accum == NoClass)
205 return Field;
206 if (Accum == Integer || Field == Integer)
207 return Integer;
208 if (Field == X87 || Field == X87Up || Field == ComplexX87 || Accum == X87 ||
209 Accum == X87Up)
210 return Memory;
211
212 return Sse;
213}
214
215// A record with a matrix-extension field is passed in memory. clang has no
216// matrix-specific ABI code: a matrix falls through X86_64ABIInfo::classify to
217// the default MEMORY class. We model matrices as arrays, so this check
218// reproduces that record-with-matrix -> MEMORY result.
219bool X86_64TargetInfo::containsMatrixField(const RecordType *RT) const {
220 for (const auto &Field : RT->getFields()) {
221 const Type *FieldType = Field.FieldType;
222
223 if (const auto *AT = dyn_cast<ArrayType>(Val: FieldType)) {
224 if (AT->isMatrixType())
225 return true;
226 continue;
227 }
228
229 if (const auto *NestedRT = dyn_cast<RecordType>(Val: FieldType))
230 if (containsMatrixField(RT: NestedRT))
231 return true;
232 }
233 return false;
234}
235
236void X86_64TargetInfo::classify(const Type *T, uint64_t OffsetBase, Class &Lo,
237 Class &Hi, bool IsNamedArg,
238 bool IsRegCall) const {
239 Lo = Hi = NoClass;
240 Class &Current = OffsetBase < 64 ? Lo : Hi;
241 Current = Memory;
242
243 if (T->isVoid()) {
244 Current = NoClass;
245 return;
246 }
247
248 if (const auto *IT = dyn_cast<IntegerType>(Val: T)) {
249 auto BitWidth = IT->getSizeInBits().getFixedValue();
250
251 if (BitWidth == 128 ||
252 (IT->isBitInt() && BitWidth > 64 && BitWidth <= 128)) {
253 Lo = Integer;
254 Hi = Integer;
255 } else if (BitWidth <= 64) {
256 Current = Integer;
257 }
258
259 return;
260 }
261
262 if (const auto *FT = dyn_cast<FloatType>(Val: T)) {
263 const auto *FltSem = FT->getSemantics();
264
265 if (FltSem == &llvm::APFloat::IEEEsingle() ||
266 FltSem == &llvm::APFloat::IEEEdouble() ||
267 FltSem == &llvm::APFloat::IEEEhalf() ||
268 FltSem == &llvm::APFloat::BFloat()) {
269 Current = Sse;
270 } else if (FltSem == &llvm::APFloat::IEEEquad()) {
271 Lo = Sse;
272 Hi = SseUp;
273 } else if (FltSem == &llvm::APFloat::x87DoubleExtended()) {
274 Lo = X87;
275 Hi = X87Up;
276 } else {
277 Current = Sse;
278 }
279 return;
280 }
281 if (T->isPointer()) {
282 Current = Integer;
283 return;
284 }
285
286 if (const auto *MPT = dyn_cast<MemberPointerType>(Val: T)) {
287 if (MPT->isFunctionPointer()) {
288 if (Has64BitPointers) {
289 Lo = Hi = Integer;
290 } else {
291 uint64_t EbFuncPtr = OffsetBase / 64;
292 uint64_t EbThisAdj = (OffsetBase + 64 - 1) / 64;
293 if (EbFuncPtr != EbThisAdj) {
294 Lo = Hi = Integer;
295 } else {
296 Current = Integer;
297 }
298 }
299 } else {
300 Current = Integer;
301 }
302 return;
303 }
304
305 if (const auto *VT = dyn_cast<VectorType>(Val: T)) {
306 assert(VT->isFixedLength() && "x86-64 has no scalable vectors");
307 uint64_t Size = VT->getABISizeInBits();
308 const Type *ElementType = VT->getElementType();
309
310 if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
311 // gcc passes the following as integer:
312 // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
313 // 2 bytes - <2 x char>, <1 x short>
314 // 1 byte - <1 x char>
315 Current = Integer;
316 // If this type crosses an eightbyte boundary, it should be
317 // split.
318 uint64_t EbLo = (OffsetBase) / 64;
319 uint64_t EbHi = (OffsetBase + Size - 1) / 64;
320 if (EbLo != EbHi)
321 Hi = Lo;
322 } else if (Size == 64) {
323 if (const auto *FT = dyn_cast<FloatType>(Val: ElementType)) {
324 // gcc passes <1 x double> in memory. :(
325 if (FT->getSemantics() == &llvm::APFloat::IEEEdouble())
326 return;
327 }
328
329 // gcc passes <1 x long long> as SSE but clang used to unconditionally
330 // pass them as integer. For platforms where clang is the de facto
331 // platform compiler, we must continue to use integer.
332 if (const auto *IT = dyn_cast<IntegerType>(Val: ElementType)) {
333 uint64_t ElemBits = IT->getSizeInBits().getFixedValue();
334 if (!getX86ABICompatInfo().ClassifyIntegerMMXAsSSE && ElemBits == 64 &&
335 !IT->isBitInt()) {
336 Current = Integer;
337 } else {
338 Current = Sse;
339 }
340 } else {
341 Current = Sse;
342 }
343 // If this type crosses an eightbyte boundary, it should be
344 // split.
345 if (OffsetBase && OffsetBase != 64)
346 Hi = Lo;
347 } else if (Size == 128 ||
348 (IsNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
349 if (const auto *IT = dyn_cast<IntegerType>(Val: ElementType)) {
350 uint64_t ElemBits = IT->getSizeInBits().getFixedValue();
351 // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
352 if (getX86ABICompatInfo().PassInt128VectorsInMem && Size != 128 &&
353 ElemBits == 128 && !IT->isBitInt())
354 return;
355 }
356
357 // Arguments of 256-bits are split into four eightbyte chunks. The
358 // least significant one belongs to class SSE and all the others to class
359 // SSEUP. The original Lo and Hi design considers that types can't be
360 // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
361 // This design isn't correct for 256-bits, but since there're no cases
362 // where the upper parts would need to be inspected, avoid adding
363 // complexity and just consider Hi to match the 64-256 part.
364 //
365 // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
366 // registers if they are "named", i.e. not part of the "..." of a
367 // variadic function.
368 //
369 // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
370 // split into eight eightbyte chunks, one SSE and seven SSEUP.
371 Lo = Sse;
372 Hi = SseUp;
373 }
374 return;
375 }
376
377 if (const auto *CT = dyn_cast<ComplexType>(Val: T)) {
378 const Type *ElementType = CT->getElementType();
379 uint64_t Size = T->getABISizeInBits();
380
381 if (isa<IntegerType>(Val: ElementType)) {
382 if (Size <= 64)
383 Current = Integer;
384 else if (Size <= 128)
385 Lo = Hi = Integer;
386 } else if (const auto *EFT = dyn_cast<FloatType>(Val: ElementType)) {
387 const auto *FltSem = EFT->getSemantics();
388 if (FltSem == &llvm::APFloat::IEEEhalf() ||
389 FltSem == &llvm::APFloat::IEEEsingle() ||
390 FltSem == &llvm::APFloat::BFloat())
391 Current = Sse;
392 else if (FltSem == &llvm::APFloat::IEEEquad())
393 Current = Memory;
394 else if (FltSem == &llvm::APFloat::x87DoubleExtended())
395 Current = ComplexX87;
396 else if (FltSem == &llvm::APFloat::IEEEdouble())
397 Lo = Hi = Sse;
398 else
399 llvm_unreachable("Unexpected long double representation!");
400 }
401
402 uint64_t ElementSize = ElementType->getABISizeInBits();
403 // If this complex type crosses an eightbyte boundary then it
404 // should be split.
405 uint64_t EbReal = OffsetBase / 64;
406 uint64_t EbImag = (OffsetBase + ElementSize) / 64;
407 if (Hi == NoClass && EbReal != EbImag)
408 Hi = Lo;
409
410 return;
411 }
412
413 if (const auto *AT = dyn_cast<ArrayType>(Val: T)) {
414 // A matrix type is modeled as an array but, like Clang, is treated as a
415 // non-aggregate scalar: it matches no class here and stays in the Memory
416 // class, so classify*Type later returns it Direct (coerced to its
417 // flattened vector) rather than classifying it field-by-field.
418 if (AT->isMatrixType())
419 return;
420
421 // Arrays are treated like structures.
422 uint64_t Size = AT->getABISizeInBits();
423
424 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
425 // than eight eightbytes, ..., it has class MEMORY.
426 // regcall ABI doesn't have limitation to an object. The only limitation
427 // is the free registers, which will be checked in computeInfo.
428 if (!IsRegCall && Size > 512)
429 return;
430
431 // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
432 // fields, it has class MEMORY.
433 //
434 // Only need to check alignment of array base.
435 const Type *ElementType = AT->getElementType();
436 uint64_t ElemAlign = ElementType->getAlignment().value() * 8;
437 if (OffsetBase % ElemAlign)
438 return;
439
440 // Otherwise implement simplified merge. We could be smarter about
441 // this, but it isn't worth it and would be harder to verify.
442 Current = NoClass;
443 uint64_t EltSize = ElementType->getABISizeInBits();
444 uint64_t ArraySize = AT->getNumElements();
445
446 // The only case a 256-bit wide vector could be used is when the array
447 // contains a single 256-bit element. Since Lo and Hi logic isn't extended
448 // to work for sizes wider than 128, early check and fallback to memory.
449 //
450 if (Size > 128 &&
451 (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))
452 return;
453
454 for (uint64_t I = 0, Offset = OffsetBase; I < ArraySize;
455 ++I, Offset += EltSize) {
456 Class FieldLo, FieldHi;
457 classify(T: ElementType, OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, IsNamedArg);
458 Lo = merge(Accum: Lo, Field: FieldLo);
459 Hi = merge(Accum: Hi, Field: FieldHi);
460 if (Lo == Memory || Hi == Memory)
461 break;
462 }
463 postMerge(AggregateSize: Size, Lo, Hi);
464 assert((Hi != SseUp || Lo == Sse) && "Invalid SseUp array classification.");
465 return;
466 }
467
468 if (const auto *RT = dyn_cast<RecordType>(Val: T)) {
469 uint64_t Size = RT->getABISizeInBits();
470
471 if (containsMatrixField(RT)) {
472 Lo = Memory;
473 return;
474 }
475
476 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
477 // than eight eightbytes, ..., it has class MEMORY.
478 if (Size > 512)
479 return;
480
481 // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
482 // copy constructor or a non-trivial destructor, it is passed by invisible
483 // reference.
484 if (getRecordArgABI(RT))
485 return;
486
487 // Assume variable sized types are passed in memory.
488 if (RT->hasFlexibleArrayMember())
489 return;
490
491 // Reset Lo class, this will be recomputed.
492 Current = NoClass;
493
494 // If this is a C++ record, classify the bases first.
495 if (RT->isCXXRecord()) {
496 for (const auto &Base : RT->getBaseClasses()) {
497
498 // Classify this field.
499 //
500 // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
501 // single eightbyte, each is classified separately. Each eightbyte gets
502 // initialized to class NO_CLASS.
503 Class FieldLo, FieldHi;
504 uint64_t Offset = OffsetBase + Base.OffsetInBits;
505 classify(T: Base.FieldType, OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, IsNamedArg);
506 Lo = merge(Accum: Lo, Field: FieldLo);
507 Hi = merge(Accum: Hi, Field: FieldHi);
508
509 if (getX86ABICompatInfo().ReturnCXXRecordGreaterThan128InMem &&
510 (Size > 128 && (Size != Base.FieldType->getABISizeInBits() ||
511 Size > getNativeVectorSizeForAVXABI(AVXLevel))))
512 Lo = Memory;
513
514 if (Lo == Memory || Hi == Memory) {
515 postMerge(AggregateSize: Size, Lo, Hi);
516 return;
517 }
518 }
519 }
520
521 // Classify the fields one at a time, merging the results.
522
523 bool IsUnion = RT->isUnion() && !getX86ABICompatInfo().Clang11Compat;
524 for (const auto &Field : RT->getFields()) {
525 uint64_t Offset = OffsetBase + Field.OffsetInBits;
526 bool BitField = Field.IsBitField;
527
528 // Ignore padding bit-fields. Normally only zero-length bit-fields are
529 // padding, but under Clang 23 compatibility every unnamed bit-field is,
530 // faithfully reproducing Clang 23.
531 if (BitField && (getX86ABICompatInfo().ClassifyUnnamedBitFields
532 ? Field.BitFieldWidth == 0
533 : Field.IsUnnamedBitfield))
534 continue;
535
536 uint64_t FieldSize = Field.FieldType->getABISizeInBits();
537 if (Size > 128 && ((!IsUnion && Size != FieldSize) ||
538 Size > getNativeVectorSizeForAVXABI(AVXLevel))) {
539 Lo = Memory;
540 postMerge(AggregateSize: Size, Lo, Hi);
541 return;
542 }
543
544 bool IsInMemory = Offset % (Field.FieldType->getAlignment().value() * 8);
545 if (!BitField && IsInMemory) {
546 Lo = Memory;
547 postMerge(AggregateSize: Size, Lo, Hi);
548 return;
549 }
550
551 Class FieldLo, FieldHi;
552
553 if (BitField) {
554 uint64_t BitFieldSize = Field.BitFieldWidth;
555 uint64_t EbLo = Offset / 64;
556 uint64_t EbHi = (Offset + BitFieldSize - 1) / 64;
557
558 if (EbLo) {
559 assert(EbHi == EbLo && "Invalid classification, type > 16 bytes.");
560 FieldLo = NoClass;
561 FieldHi = Integer;
562 } else {
563 FieldLo = Integer;
564 FieldHi = EbHi ? Integer : NoClass;
565 }
566 } else {
567 classify(T: Field.FieldType, OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, IsNamedArg);
568 }
569
570 Lo = merge(Accum: Lo, Field: FieldLo);
571 Hi = merge(Accum: Hi, Field: FieldHi);
572 if (Lo == Memory || Hi == Memory)
573 break;
574 }
575 postMerge(AggregateSize: Size, Lo, Hi);
576 return;
577 }
578
579 Lo = Memory;
580 Hi = NoClass;
581}
582
583ArgInfo
584X86_64TargetInfo::classifyArgumentType(const Type *Ty, unsigned FreeIntRegs,
585 unsigned &NeededInt, unsigned &NeededSSE,
586 bool IsNamedArg, bool IsRegCall) const {
587
588 Ty = useFirstFieldIfTransparentUnion(Ty);
589
590 X86_64TargetInfo::Class Lo, Hi;
591 classify(T: Ty, OffsetBase: 0, Lo, Hi, IsNamedArg, IsRegCall);
592
593 // Check some invariants
594 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
595 assert((Hi != SseUp || Lo == Sse) && "Invalid SseUp classification.");
596
597 NeededInt = 0;
598 NeededSSE = 0;
599 const Type *ResType = nullptr;
600
601 switch (Lo) {
602 case NoClass:
603 if (Hi == NoClass)
604 return ArgInfo::getIgnore();
605 // If the low part is just padding, it takes no register, leave ResType
606 // null.
607 assert((Hi == Sse || Hi == Integer || Hi == X87Up) &&
608 "Unknown missing lo part");
609 break;
610
611 // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
612 // on the stack.
613 case Memory:
614 // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87Up or
615 // COMPLEX_X87, it is passed in memory.
616 case X87:
617 case ComplexX87:
618 if (getRecordArgABI(Ty) == RAA_Indirect)
619 ++NeededInt;
620 return getIndirectResult(Ty, FreeIntRegs);
621
622 case SseUp:
623 case X87Up:
624 llvm_unreachable("Invalid classification for lo word.");
625
626 // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
627 // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
628 // and %r9 is used.
629 case Integer:
630 ++NeededInt;
631
632 // Pick an 8-byte type based on the preferred type.
633 ResType = getIntegerTypeAtOffset(IRType: Ty, IROffset: 0, SourceTy: Ty, SourceOffset: 0);
634
635 // If we have a sign or zero extended integer, make sure to return Extend
636 // so that the parameter gets the right LLVM IR attributes.
637 if (Hi == NoClass && ResType->isInteger()) {
638 if (Ty->isInteger() && isPromotableInteger(IT: cast<IntegerType>(Val: Ty)))
639 return ArgInfo::getExtend(T: Ty);
640 }
641
642 if (ResType->isInteger() && ResType->getSizeInBits() == 128) {
643 assert(Hi == Integer);
644 ++NeededInt;
645 return ArgInfo::getDirect(T: ResType);
646 }
647 break;
648
649 // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
650 // available SSE register is used, the registers are taken in the
651 // order from %xmm0 to %xmm7.
652 case Sse:
653 ResType = getSSETypeAtOffset(ABIType: Ty, ABIOffset: 0, SourceTy: Ty, SourceOffset: 0);
654 ++NeededSSE;
655 break;
656 }
657
658 const Type *HighPart = nullptr;
659 switch (Hi) {
660 // Memory was handled previously, ComplexX87 and X87 should
661 // never occur as hi classes, and X87Up must be preceded by X87,
662 // which is passed in memory.
663 case Memory:
664 case X87:
665 case ComplexX87:
666 llvm_unreachable("Invalid classification for hi word.");
667
668 case NoClass:
669 break;
670
671 case Integer:
672 ++NeededInt;
673 // Pick an 8-byte type based on the preferred type.
674 HighPart = getIntegerTypeAtOffset(IRType: Ty, IROffset: 8, SourceTy: Ty, SourceOffset: 8);
675
676 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
677 return ArgInfo::getDirect(T: HighPart, Offset: 8);
678 break;
679
680 // X87Up generally doesn't occur here (long double is passed in
681 // memory), except in situations involving unions.
682 case X87Up:
683 case Sse:
684 ++NeededSSE;
685 HighPart = getSSETypeAtOffset(ABIType: Ty, ABIOffset: 8, SourceTy: Ty, SourceOffset: 8);
686
687 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
688 return ArgInfo::getDirect(T: HighPart, Offset: 8);
689 break;
690
691 // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
692 // eightbyte is passed in the upper half of the last used SSE
693 // register. This only happens when 128-bit vectors are passed.
694 case SseUp:
695 assert(Lo == Sse && "Unexpected SseUp classification");
696 ResType = getByteVectorType(Ty);
697 break;
698 }
699
700 // If a high part was specified, merge it together with the low part. It is
701 // known to pass in the high eightbyte of the result. We do this by forming a
702 // first class struct aggregate with the high and low part: {low, high}
703 if (HighPart)
704 ResType = createPairType(Lo: ResType, Hi: HighPart);
705
706 return ArgInfo::getDirect(T: ResType);
707}
708
709ArgInfo X86_64TargetInfo::classifyReturnType(const Type *RetTy) const {
710 // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
711 // classification algorithm.
712
713 X86_64TargetInfo::Class Lo, Hi;
714 classify(T: RetTy, OffsetBase: 0, Lo, Hi, /*isNamedArg*/ IsNamedArg: true);
715
716 // Check some invariants
717 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
718 assert((Hi != SseUp || Lo == Sse) && "Invalid SseUp classification.");
719
720 const Type *ResType = nullptr;
721 switch (Lo) {
722 case NoClass:
723 if (Hi == NoClass)
724 return ArgInfo::getIgnore();
725 // If the low part is just padding, it takes no register, leave ResType
726 // null.
727 assert((Hi == Sse || Hi == Integer || Hi == X87Up) &&
728 "Unknown missing lo part");
729 break;
730 case SseUp:
731 case X87Up:
732 llvm_unreachable("Invalid classification for lo word.");
733
734 // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
735 // hidden argument.
736 case Memory:
737 return getIndirectReturnResult(Ty: RetTy);
738
739 // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
740 // available register of the sequence %rax, %rdx is used.
741 case Integer:
742 ResType = getIntegerTypeAtOffset(IRType: RetTy, IROffset: 0, SourceTy: RetTy, SourceOffset: 0);
743 // If we have a sign or zero extended integer, make sure to return Extend
744 // so that the parameter gets the right LLVM IR attributes.
745 if (Hi == NoClass && ResType->isInteger()) {
746 if (const IntegerType *IntTy = dyn_cast<IntegerType>(Val: RetTy)) {
747 if (isPromotableInteger(IT: IntTy))
748 return ArgInfo::getExtend(T: RetTy);
749 }
750 }
751 if (ResType->isInteger() && ResType->getSizeInBits() == 128) {
752 assert(Hi == Integer);
753 return ArgInfo::getDirect(T: ResType);
754 }
755 break;
756
757 // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
758 // available SSE register of the sequence %xmm0, %xmm1 is used.
759 case Sse:
760 ResType = getSSETypeAtOffset(ABIType: RetTy, ABIOffset: 0, SourceTy: RetTy, SourceOffset: 0);
761 break;
762
763 // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
764 // returned on the X87 stack in %st0 as 80-bit x87 number.
765 case X87:
766 ResType = TB.getFloatType(Semantics: APFloat::x87DoubleExtended(), Align: Align(16));
767 break;
768
769 // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
770 // part of the value is returned in %st0 and the imaginary part in
771 // %st1.
772 case ComplexX87:
773 assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
774 {
775 const Type *X87Type =
776 TB.getFloatType(Semantics: APFloat::x87DoubleExtended(), Align: Align(16));
777 FieldInfo Fields[] = {FieldInfo(X87Type, 0), FieldInfo(X87Type, 80)};
778 ResType = TB.getRecordType(Fields, Size: TypeSize::getFixed(ExactSize: 160), ABIAlign: Align(16),
779 /*UnadjustedAlign=*/Align(16));
780 }
781 break;
782 }
783
784 const Type *HighPart = nullptr;
785 switch (Hi) {
786 // Memory was handled previously and X87 should
787 // never occur as a hi class.
788 case Memory:
789 case X87:
790 llvm_unreachable("Invalid classification for hi word.");
791
792 case ComplexX87:
793 case NoClass:
794 break;
795
796 case Integer:
797 HighPart = getIntegerTypeAtOffset(IRType: RetTy, IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
798 if (Lo == NoClass)
799 return ArgInfo::getDirect(T: HighPart, Offset: 8);
800 break;
801
802 case Sse:
803 HighPart = getSSETypeAtOffset(ABIType: RetTy, ABIOffset: 8, SourceTy: RetTy, SourceOffset: 8);
804 if (Lo == NoClass)
805 return ArgInfo::getDirect(T: HighPart, Offset: 8);
806 break;
807
808 // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
809 // is passed in the next available eightbyte chunk if the last used
810 // vector register.
811 //
812 // SSEUP should always be preceded by SSE, just widen.
813 case SseUp:
814 assert(Lo == Sse && "Unexpected SseUp classification.");
815 ResType = getByteVectorType(Ty: RetTy);
816 break;
817
818 // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87Up, the value is
819 // returned together with the previous X87 value in %st0.
820 case X87Up:
821 // If X87Up is preceded by X87, we don't need to do
822 // anything. However, in some cases with unions it may not be
823 // preceded by X87. In such situations we follow gcc and pass the
824 // extra bits in an SSE reg.
825 if (Lo != X87) {
826 HighPart = getSSETypeAtOffset(ABIType: RetTy, ABIOffset: 8, SourceTy: RetTy, SourceOffset: 8);
827 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
828 return ArgInfo::getDirect(T: HighPart, Offset: 8);
829 }
830 break;
831 }
832
833 // If a high part was specified, merge it together with the low part. It is
834 // known to pass in the high eightbyte of the result. We do this by forming a
835 // first class struct aggregate with the high and low part: {low, high}
836 if (HighPart)
837 ResType = createPairType(Lo: ResType, Hi: HighPart);
838
839 return ArgInfo::getDirect(T: ResType);
840}
841
842/// Given a high and low type that can ideally
843/// be used as elements of a two register pair to pass or return, return a
844/// first class aggregate to represent them. For example, if the low part of
845/// a by-value argument should be passed as i32* and the high part as float,
846/// return {i32*, float}.
847const Type *X86_64TargetInfo::createPairType(const Type *Lo,
848 const Type *Hi) const {
849 // In order to correctly satisfy the ABI, we need to the high part to start
850 // at offset 8. If the high and low parts we inferred are both 4-byte types
851 // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
852 // the second element at offset 8. Check for this:
853 unsigned LoSize = (unsigned)Lo->getTypeAllocSize();
854 llvm::Align HiAlign = Hi->getAlignment();
855 unsigned HiStart = alignTo(Size: LoSize, A: HiAlign);
856
857 assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
858
859 // To handle this, we have to increase the size of the low part so that the
860 // second element will start at an 8 byte offset. We can't increase the size
861 // of the second element because it might make us access off the end of the
862 // struct.
863 const Type *AdjustedLo = Lo;
864 if (HiStart != 8) {
865 // There are usually two sorts of types the ABI generation code can produce
866 // for the low part of a pair that aren't 8 bytes in size: half, float or
867 // i8/i16/i32. This can also include pointers when they are 32-bit (X32 and
868 // NaCl).
869 // Promote these to a larger type.
870 if (Lo->isFloat()) {
871 const FloatType *FT = cast<FloatType>(Val: Lo);
872 if (FT->getSemantics() == &APFloat::IEEEhalf() ||
873 FT->getSemantics() == &APFloat::IEEEsingle() ||
874 FT->getSemantics() == &APFloat::BFloat())
875 AdjustedLo = TB.getFloatType(Semantics: APFloat::IEEEdouble(), Align: Align(8));
876 }
877 // Promote integers and pointers to i64
878 else if (Lo->isInteger() || Lo->isPointer())
879 AdjustedLo = TB.getIntegerType(BitWidth: 64, Align: Align(8), /*Signed=*/false);
880 else
881 assert((Lo->isInteger() || Lo->isPointer()) &&
882 "Invalid/unknown low type in pair");
883 unsigned AdjustedLoSize = AdjustedLo->getSizeInBits().getFixedValue() / 8;
884 HiStart = alignTo(Size: AdjustedLoSize, A: HiAlign);
885 }
886
887 // Create the pair struct
888 FieldInfo Fields[] = {FieldInfo(AdjustedLo, 0), FieldInfo(Hi, HiStart * 8)};
889
890 // Verify the high part is at offset 8
891 assert((8 * 8) == Fields[1].OffsetInBits &&
892 "High part must be at offset 8 bytes");
893
894 uint64_t PairSizeInBits =
895 Fields[1].OffsetInBits + Hi->getSizeInBits().getFixedValue();
896 return TB.getRecordType(Fields, Size: TypeSize::getFixed(ExactSize: PairSizeInBits), ABIAlign: Align(8),
897 /*UnadjustedAlign=*/Align(8), Pack: StructPacking::Default);
898}
899
900static bool bitsContainNoUserData(const Type *Ty, unsigned StartBit,
901 unsigned EndBit) {
902 // If range is completely beyond type size, it's definitely padding
903 unsigned TySize = Ty->getABISizeInBits();
904 if (TySize <= StartBit)
905 return true;
906
907 // Handle arrays - check each element
908 if (const ArrayType *AT = dyn_cast<ArrayType>(Val: Ty)) {
909 const Type *EltTy = AT->getElementType();
910 unsigned EltSize = EltTy->getABISizeInBits();
911
912 for (unsigned I = 0; I < AT->getNumElements(); ++I) {
913 unsigned EltOffset = I * EltSize;
914 if (EltOffset >= EndBit)
915 break;
916
917 unsigned EltStart = (EltOffset < StartBit) ? StartBit - EltOffset : 0;
918 if (!bitsContainNoUserData(Ty: EltTy, StartBit: EltStart, EndBit: EndBit - EltOffset))
919 return false;
920 }
921 return true;
922 }
923
924 // Handle records - check all fields and base classes. getUnionType places a
925 // union's members at offset zero, so the field loop covers a union too.
926 if (const RecordType *RT = dyn_cast<RecordType>(Val: Ty)) {
927 // Check base classes first (for C++ records)
928 if (RT->isCXXRecord()) {
929 for (unsigned I = 0; I < RT->getNumBaseClasses(); ++I) {
930 const FieldInfo &Base = RT->getBaseClasses()[I];
931 if (Base.OffsetInBits >= EndBit)
932 continue;
933
934 unsigned BaseStart =
935 (Base.OffsetInBits < StartBit) ? StartBit - Base.OffsetInBits : 0;
936 if (!bitsContainNoUserData(Ty: Base.FieldType, StartBit: BaseStart,
937 EndBit: EndBit - Base.OffsetInBits))
938 return false;
939 }
940 }
941
942 for (unsigned I = 0; I < RT->getNumFields(); ++I) {
943 const FieldInfo &Field = RT->getFields()[I];
944 if (Field.OffsetInBits >= EndBit)
945 break;
946
947 unsigned FieldStart =
948 (Field.OffsetInBits < StartBit) ? StartBit - Field.OffsetInBits : 0;
949 if (!bitsContainNoUserData(Ty: Field.FieldType, StartBit: FieldStart,
950 EndBit: EndBit - Field.OffsetInBits))
951 return false;
952 }
953 return true;
954 }
955
956 // For any other type - assume all bits are user data
957 return false;
958}
959
960const Type *X86_64TargetInfo::getIntegerTypeAtOffset(const Type *ABIType,
961 unsigned ABIOffset,
962 const Type *SourceTy,
963 unsigned SourceOffset,
964 bool InMemory) const {
965
966 const Type *WorkingType = ABIType;
967 if (InMemory && ABIType->isInteger()) {
968 const auto *IT = cast<IntegerType>(Val: ABIType);
969 unsigned OriginalBitWidth = IT->getSizeInBits().getFixedValue();
970
971 unsigned WidenedBitWidth = OriginalBitWidth;
972 if (OriginalBitWidth <= 8) {
973 WidenedBitWidth = 8;
974 } else {
975 WidenedBitWidth = llvm::bit_ceil(Value: OriginalBitWidth);
976 }
977
978 if (WidenedBitWidth != OriginalBitWidth) {
979 WorkingType = TB.getIntegerType(BitWidth: WidenedBitWidth, Align: ABIType->getAlignment(),
980 Signed: IT->isSigned());
981 }
982 }
983 // A bool vector is stored as an integer with one bit per element, at least a
984 // byte wide.
985 if (InMemory && ABIType->isVector()) {
986 const auto *VT = cast<VectorType>(Val: ABIType);
987 const auto *IT = dyn_cast<IntegerType>(Val: VT->getElementType());
988 if (IT && IT->isBool())
989 WorkingType = TB.getIntegerType(
990 BitWidth: std::max<uint64_t>(a: VT->getNumElements().getFixedValue(), b: 8),
991 Align: ABIType->getAlignment(), /*Signed=*/false);
992 }
993 // If we're dealing with an un-offset ABI type, then it means that we're
994 // returning an 8-byte unit starting with it. See if we can safely use it.
995 if (ABIOffset == 0) {
996 // Pointers and int64's always fill the 8-byte unit. Return WorkingType,
997 // which is the in-memory-widened type (e.g. a _BitInt(37) field widened to
998 // i64): returning the raw ABIType here would coerce the eightbyte to the
999 // narrow iN instead of the storage integer clang uses.
1000 if ((WorkingType->isPointer() && Has64BitPointers) ||
1001 (WorkingType->isInteger() &&
1002 cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 64))
1003 return WorkingType;
1004
1005 // If we have a 1/2/4-byte integer, we can use it only if the rest of the
1006 // goodness in the source type is just tail padding. This is allowed to
1007 // kick in for struct {double,int} on the int, but not on
1008 // struct{double,int,int} because we wouldn't return the second int. We
1009 // have to do this analysis on the source type because we can't depend on
1010 // unions being lowered a specific way etc.
1011 if ((WorkingType->isInteger() &&
1012 (cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 1 ||
1013 cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 8 ||
1014 cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 16 ||
1015 cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 32)) ||
1016 (WorkingType->isPointer() && !Has64BitPointers)) {
1017
1018 unsigned BitWidth = WorkingType->isPointer()
1019 ? 32
1020 : cast<IntegerType>(Val: WorkingType)->getSizeInBits();
1021
1022 if (bitsContainNoUserData(Ty: SourceTy, StartBit: SourceOffset * 8 + BitWidth,
1023 EndBit: SourceOffset * 8 + 64))
1024 return WorkingType;
1025 }
1026 }
1027
1028 if (const auto *RTy = dyn_cast<RecordType>(Val: ABIType)) {
1029 if (RTy->isUnion()) {
1030 const Type *ReducedType = reduceUnionForX8664(UnionType: RTy, TB);
1031 if (ReducedType) {
1032 if (ABIOffset * 8 < ReducedType->getABISizeInBits())
1033 return getIntegerTypeAtOffset(ABIType: ReducedType, ABIOffset, SourceTy,
1034 SourceOffset, InMemory: true);
1035 // The storage type stops before this offset, so size the coercion
1036 // from the union itself: a byte when the rest of this eightbyte
1037 // holds no data, and the union's remaining bytes otherwise.
1038 if (bitsContainNoUserData(Ty: SourceTy, StartBit: SourceOffset * 8 + 8,
1039 EndBit: SourceOffset * 8 + 64))
1040 return TB.getIntegerType(BitWidth: 8, Align: Align(1), /*Signed=*/false);
1041 unsigned RemainingBytes =
1042 llvm::divideCeil(Numerator: SourceTy->getABISizeInBits(), Denominator: 8) - SourceOffset;
1043 return TB.getIntegerType(BitWidth: std::min(a: RemainingBytes, b: 8U) * 8, Align: Align(1),
1044 /*Signed=*/false);
1045 }
1046 }
1047 if (const FieldInfo *Element =
1048 RTy->getElementContainingOffset(OffsetInBits: ABIOffset * 8)) {
1049
1050 unsigned ElementOffsetBytes = Element->OffsetInBits / 8;
1051 return getIntegerTypeAtOffset(ABIType: Element->FieldType,
1052 ABIOffset: ABIOffset - ElementOffsetBytes, SourceTy,
1053 SourceOffset, InMemory: true);
1054 }
1055 }
1056
1057 if (const auto *ATy = dyn_cast<ArrayType>(Val: ABIType)) {
1058 const Type *EltTy = ATy->getElementType();
1059 unsigned EltSize = EltTy->getABISizeInBits() / 8;
1060 if (EltSize > 0) {
1061 unsigned EltOffset = (ABIOffset / EltSize) * EltSize;
1062 return getIntegerTypeAtOffset(ABIType: EltTy, ABIOffset: ABIOffset - EltOffset, SourceTy,
1063 SourceOffset, InMemory: true);
1064 }
1065 }
1066
1067 // If we have a 128-bit integer, we can pass it safely using an i128
1068 // so we return that
1069 if (ABIType->isInteger() && ABIType->getSizeInBits() == 128) {
1070 assert(ABIOffset == 0);
1071 return ABIType;
1072 }
1073
1074 unsigned TySizeInBytes = llvm::divideCeil(Numerator: SourceTy->getABISizeInBits(), Denominator: 8);
1075 assert(TySizeInBytes != SourceOffset && "Empty field?");
1076 unsigned AvailableSize = TySizeInBytes - SourceOffset;
1077 return TB.getIntegerType(BitWidth: std::min(a: AvailableSize, b: 8U) * 8, Align: Align(1), Signed: false);
1078}
1079/// Returns the floating point type at the specified offset within a type, or
1080/// nullptr if no floating point type is found at that offset.
1081const Type *X86_64TargetInfo::getFPTypeAtOffset(const Type *Ty,
1082 unsigned Offset) const {
1083 // Check for direct match at offset 0
1084 if (Offset == 0 && Ty->isFloat())
1085 return Ty;
1086
1087 if (const ComplexType *CT = dyn_cast<ComplexType>(Val: Ty)) {
1088 const Type *ElementType = CT->getElementType();
1089 unsigned ElementSize = ElementType->getABISizeInBits() / 8;
1090
1091 if (Offset == 0 || Offset == ElementSize)
1092 return ElementType;
1093 return nullptr;
1094 }
1095
1096 // Handle struct types by checking each field
1097 if (const RecordType *RT = dyn_cast<RecordType>(Val: Ty)) {
1098 if (const FieldInfo *Element = RT->getElementContainingOffset(OffsetInBits: Offset * 8)) {
1099 unsigned ElementOffsetBytes = Element->OffsetInBits / 8;
1100 return getFPTypeAtOffset(Ty: Element->FieldType, Offset: Offset - ElementOffsetBytes);
1101 }
1102 }
1103
1104 // Handle array types
1105 if (const ArrayType *AT = dyn_cast<ArrayType>(Val: Ty)) {
1106 const Type *EltTy = AT->getElementType();
1107 unsigned EltSize = EltTy->getABISizeInBits() / 8;
1108 if (EltSize == 0)
1109 return nullptr;
1110 unsigned EltIndex = Offset / EltSize;
1111
1112 return getFPTypeAtOffset(Ty: EltTy, Offset: Offset - (EltIndex * EltSize));
1113 }
1114
1115 // No floating point type found at this offset
1116 return nullptr;
1117}
1118
1119/// Helper to check if a floating point type matches specific semantics
1120static bool isFloatTypeWithSemantics(const Type *Ty,
1121 const fltSemantics &Semantics) {
1122 if (!Ty->isFloat())
1123 return false;
1124 const FloatType *FT = cast<FloatType>(Val: Ty);
1125 return FT->getSemantics() == &Semantics;
1126}
1127
1128/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
1129/// low 8 bytes of an XMM register, corresponding to the SSE class.
1130const Type *X86_64TargetInfo::getSSETypeAtOffset(const Type *ABIType,
1131 unsigned ABIOffset,
1132 const Type *SourceTy,
1133 unsigned SourceOffset) const {
1134
1135 if (const auto *RTy = dyn_cast<RecordType>(Val: ABIType)) {
1136 if (RTy->isUnion()) {
1137 const Type *ReducedType = reduceUnionForX8664(UnionType: RTy, TB);
1138 if (ReducedType) {
1139 return getSSETypeAtOffset(ABIType: ReducedType, ABIOffset, SourceTy,
1140 SourceOffset);
1141 }
1142 }
1143 }
1144
1145 auto Is16bitFpTy = [](const Type *T) {
1146 return isFloatTypeWithSemantics(Ty: T, Semantics: APFloat::IEEEhalf()) ||
1147 isFloatTypeWithSemantics(Ty: T, Semantics: APFloat::BFloat());
1148 };
1149
1150 // Get the floating point type at the requested offset
1151 const Type *T0 = getFPTypeAtOffset(Ty: ABIType, Offset: ABIOffset);
1152 if (!T0 || isFloatTypeWithSemantics(Ty: T0, Semantics: APFloat::IEEEdouble()))
1153 return TB.getFloatType(Semantics: APFloat::IEEEdouble(), Align: Align(8));
1154
1155 // Calculate remaining source size in bytes
1156 unsigned SourceSize = (SourceTy->getABISizeInBits() / 8) - SourceOffset;
1157
1158 // Try to get adjacent FP type
1159 const Type *T1 = nullptr;
1160 unsigned T0Size = T0->getABISizeInBits() / 8;
1161 if (SourceSize > T0Size)
1162 T1 = getFPTypeAtOffset(Ty: ABIType, Offset: ABIOffset + T0Size);
1163
1164 if (T1 == nullptr) {
1165 if (Is16bitFpTy(T0) && SourceSize > 4)
1166 T1 = getFPTypeAtOffset(Ty: ABIType, Offset: ABIOffset + 4);
1167
1168 if (T1 == nullptr)
1169 return T0;
1170 }
1171 // Handle vector cases
1172 if (isFloatTypeWithSemantics(Ty: T0, Semantics: APFloat::IEEEsingle()) &&
1173 isFloatTypeWithSemantics(Ty: T1, Semantics: APFloat::IEEEsingle()))
1174 return TB.getVectorType(ElementType: T0, NumElements: ElementCount::getFixed(MinVal: 2), Align: Align(8));
1175
1176 if (Is16bitFpTy(T0) && Is16bitFpTy(T1)) {
1177 const Type *T2 = nullptr;
1178 if (SourceSize > 4)
1179 T2 = getFPTypeAtOffset(Ty: ABIType, Offset: ABIOffset + 4);
1180 if (!T2)
1181 return TB.getVectorType(ElementType: T0, NumElements: ElementCount::getFixed(MinVal: 2), Align: Align(8));
1182 return TB.getVectorType(ElementType: T0, NumElements: ElementCount::getFixed(MinVal: 4), Align: Align(8));
1183 }
1184
1185 // Mixed half-float cases
1186 if (Is16bitFpTy(T0) || Is16bitFpTy(T1))
1187 return TB.getVectorType(ElementType: TB.getFloatType(Semantics: APFloat::IEEEhalf(), Align: Align(2)),
1188 NumElements: ElementCount::getFixed(MinVal: 4), Align: Align(8));
1189
1190 // Default to double
1191 return TB.getFloatType(Semantics: APFloat::IEEEdouble(), Align: Align(8));
1192}
1193
1194/// The ABI specifies that a value should be passed in a full vector XMM/YMM
1195/// register. Pick an LLVM IR type that will be passed as a vector register.
1196const Type *X86_64TargetInfo::getByteVectorType(const Type *Ty) const {
1197 // Wrapper structs/arrays that only contain vectors are passed just like
1198 // vectors; strip them off if present.
1199 if (const Type *InnerTy = isSingleElementStruct(Ty))
1200 Ty = InnerTy;
1201
1202 // Handle vector types
1203 if (const VectorType *VT = dyn_cast<VectorType>(Val: Ty)) {
1204 // Don't pass vXi128 vectors in their native type, the backend can't
1205 // legalize them.
1206 if (getX86ABICompatInfo().PassInt128VectorsInMem &&
1207 VT->getElementType()->isInteger() &&
1208 cast<IntegerType>(Val: VT->getElementType())->getSizeInBits() == 128) {
1209 unsigned Size = VT->getABISizeInBits();
1210 return TB.getVectorType(ElementType: TB.getIntegerType(BitWidth: 64, Align: Align(8), /*Signed=*/false),
1211 NumElements: ElementCount::getFixed(MinVal: Size / 64),
1212 Align: Align(Size / 8));
1213 }
1214 return VT;
1215 }
1216
1217 // Handle fp128
1218 if (isFloatTypeWithSemantics(Ty, Semantics: APFloat::IEEEquad()))
1219 return Ty;
1220
1221 // We couldn't find the preferred IR vector type for 'Ty'.
1222 unsigned Size = Ty->getABISizeInBits();
1223 assert((Size == 128 || Size == 256 || Size == 512) && "Invalid vector size");
1224
1225 return TB.getVectorType(ElementType: TB.getFloatType(Semantics: APFloat::IEEEdouble(), Align: Align(8)),
1226 NumElements: ElementCount::getFixed(MinVal: Size / 64), Align: Align(Size / 8));
1227}
1228
1229bool X86_64TargetInfo::isIllegalVectorType(const Type *Ty) const {
1230 if (const auto *VecTy = dyn_cast<VectorType>(Val: Ty)) {
1231 uint64_t Size = VecTy->getABISizeInBits();
1232 unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
1233
1234 // Vectors <= 64 bits or > largest supported vector size are illegal
1235 if (Size <= 64 || Size > LargestVector)
1236 return true;
1237
1238 // Check for 128-bit integer element vectors that should be passed in memory
1239 const Type *EltTy = VecTy->getElementType();
1240 if (getX86ABICompatInfo().PassInt128VectorsInMem && EltTy->isInteger()) {
1241 const auto *IntTy = cast<IntegerType>(Val: EltTy);
1242 if (IntTy->getSizeInBits().getFixedValue() == 128 && !IntTy->isBitInt())
1243 return true;
1244 }
1245 }
1246 return false;
1247}
1248
1249ArgInfo X86_64TargetInfo::getIndirectResult(const Type *Ty,
1250 unsigned FreeIntRegs) const {
1251 // If this is a scalar LLVM value then assume LLVM will pass it in the right
1252 // place naturally.
1253 //
1254 // This assumption is optimistic, as there could be free registers available
1255 // when we need to pass this argument in memory, and LLVM could try to pass
1256 // the argument in the free register. This does not seem to happen currently,
1257 // but this code would be much safer if we could mark the argument with
1258 // 'onstack'. See PR12193.
1259 if (!isAggregateTypeForABI(Ty) && !isIllegalVectorType(Ty) &&
1260 !(Ty->isInteger() && cast<IntegerType>(Val: Ty)->isBitInt())) {
1261 return (Ty->isInteger() && isPromotableInteger(IT: cast<IntegerType>(Val: Ty))
1262 ? ArgInfo::getExtend(T: Ty)
1263 : ArgInfo::getDirect());
1264 }
1265
1266 // Check if this is a record type that needs special handling
1267 if (auto RecordRAA = getRecordArgABI(Ty))
1268 return getNaturalAlignIndirect(Ty, AddrSpace: getAllocaAddrSpace(),
1269 /*ByVal=*/RecordRAA ==
1270 RecordArgABI::RAA_DirectInMemory);
1271
1272 // Compute the byval alignment. We specify the alignment of the byval in all
1273 // cases so that the mid-level optimizer knows the alignment of the byval.
1274 uint64_t AlignVal = std::max<uint64_t>(a: Ty->getAlignment().value(), b: 8u);
1275
1276 // Attempt to avoid passing indirect results using byval when possible. This
1277 // is important for good codegen.
1278 //
1279 // We do this by coercing the value into a scalar type which the backend can
1280 // handle naturally (i.e., without using byval).
1281 //
1282 // For simplicity, we currently only do this when we have exhausted all of the
1283 // free integer registers. Doing this when there are free integer registers
1284 // would require more care, as we would have to ensure that the coerced value
1285 // did not claim the unused register. That would require either reording the
1286 // arguments to the function (so that any subsequent inreg values came first),
1287 // or only doing this optimization when there were no following arguments that
1288 // might be inreg.
1289 //
1290 // We currently expect it to be rare (particularly in well written code) for
1291 // arguments to be passed on the stack when there are still free integer
1292 // registers available (this would typically imply large structs being passed
1293 // by value), so this seems like a fair tradeoff for now.
1294 //
1295 // We can revisit this if the backend grows support for 'onstack' parameter
1296 // attributes. See PR12193.
1297 if (FreeIntRegs == 0) {
1298 uint64_t Size = Ty->getABISizeInBits();
1299
1300 // If this type fits in an eightbyte, coerce it into the matching integral
1301 // type, which will end up on the stack (with alignment 8).
1302 if (AlignVal == 8 && Size <= 64) {
1303 const Type *IntTy =
1304 TB.getIntegerType(BitWidth: Size, Align: llvm::Align(8), /*Signed=*/false);
1305 return ArgInfo::getDirect(T: IntTy);
1306 }
1307 }
1308
1309 return ArgInfo::getIndirect(Align: llvm::Align(AlignVal), /*ByVal=*/true);
1310}
1311
1312ArgInfo X86_64TargetInfo::getIndirectReturnResult(const Type *Ty) const {
1313 if (!isAggregateTypeForABI(Ty)) {
1314 // Bit-precise integers are returned indirectly regardless of size.
1315 if (const auto *IntTy = dyn_cast<IntegerType>(Val: Ty)) {
1316 if (IntTy->isBitInt())
1317 return getNaturalAlignIndirect(Ty: IntTy, AddrSpace: getAllocaAddrSpace());
1318 if (isPromotableInteger(IT: IntTy))
1319 return ArgInfo::getExtend(T: Ty);
1320 }
1321 return ArgInfo::getDirect();
1322 }
1323
1324 return getNaturalAlignIndirect(Ty, AddrSpace: getAllocaAddrSpace());
1325}
1326
1327void X86_64TargetInfo::computeInfo(FunctionInfo &FI) const {
1328 CallingConv::ID CallingConv = FI.getCallingConvention();
1329
1330 // Only the standard SysV (C) calling convention is classified here. Any other
1331 // convention must be added explicitly once it has been verified against this
1332 // classifier rather than silently taking the SysV path.
1333 switch (CallingConv) {
1334 case CallingConv::C:
1335 break;
1336 default:
1337 llvm_unreachable(
1338 "calling convention not supported by the LLVMABI X86_64 classifier");
1339 }
1340
1341 unsigned FreeIntRegs = 6;
1342 unsigned FreeSSERegs = 8;
1343 unsigned NeededInt = 0, NeededSSE = 0;
1344
1345 if (!maybeCommonClassifyReturnType(FI)) {
1346 const Type *RetTy = FI.getReturnType();
1347 FI.getReturnInfo() = classifyReturnType(RetTy);
1348 }
1349
1350 if (FI.getReturnInfo().isIndirect())
1351 --FreeIntRegs;
1352
1353 unsigned NumRequiredArgs = FI.getNumRequiredArgs();
1354
1355 unsigned ArgNo = 0;
1356 for (auto IT = FI.arg_begin(), IE = FI.arg_end(); IT != IE; ++IT, ++ArgNo) {
1357 bool IsNamedArg = ArgNo < NumRequiredArgs;
1358 const Type *ArgTy = IT->ABIType;
1359 NeededInt = 0;
1360 NeededSSE = 0;
1361
1362 ArgInfo AI = classifyArgumentType(Ty: ArgTy, FreeIntRegs, NeededInt, NeededSSE,
1363 IsNamedArg);
1364
1365 // AMD64-ABI 3.2.3p3: If there are no registers available for any
1366 // eightbyte of an argument, the whole argument is passed on the
1367 // stack. If registers have already been assigned for some
1368 // eightbytes of such an argument, the assignments get reverted.
1369 if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
1370 FreeIntRegs -= NeededInt;
1371 FreeSSERegs -= NeededSSE;
1372 AI.setNeededRegs(IntRegs: NeededInt, SseRegs: NeededSSE);
1373 IT->Info = AI;
1374 } else {
1375 // Not enough registers, pass on stack. The demand the classification
1376 // reports is what the argument ends up occupying, which is nothing.
1377 IT->Info = getIndirectResult(Ty: ArgTy, FreeIntRegs);
1378 }
1379 }
1380}
1381
1382std::unique_ptr<TargetInfo>
1383createX86_64TargetInfo(TypeBuilder &TB, X86AVXABILevel AVXLevel,
1384 bool Has64BitPointers, const X86ABICompatInfo &Compat) {
1385 return std::make_unique<X86_64TargetInfo>(args&: TB, args&: AVXLevel, args&: Has64BitPointers,
1386 args: Compat);
1387}
1388
1389} // namespace abi
1390} // namespace llvm
1391