1//===- X86.cpp ------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ABI/FunctionInfo.h"
10#include "llvm/ABI/TargetInfo.h"
11#include "llvm/ABI/Types.h"
12#include "llvm/Support/Alignment.h"
13#include "llvm/Support/Casting.h"
14#include "llvm/Support/ErrorHandling.h"
15#include "llvm/Support/MathExtras.h"
16#include "llvm/Support/TypeSize.h"
17#include <algorithm>
18#include <cassert>
19#include <cstdint>
20
21namespace llvm {
22namespace abi {
23
24static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
25 switch (AVXLevel) {
26 case X86AVXABILevel::AVX512:
27 return 512;
28 case X86AVXABILevel::AVX:
29 return 256;
30 case X86AVXABILevel::None:
31 return 128;
32 }
33 llvm_unreachable("Unknown AVXLevel");
34}
35
36// The width of an integer's storage container, mirroring Clang's
37// ASTContext::getTypeSize. For a plain integer this is its bit width; for a
38// _BitInt(N) it is N rounded up to the type's alignment. The x86-64 _BitInt
39// max alignment is 64, so this clamp is target-specific and kept file-local.
40static uint64_t getClangIntegerWidthInBits(const IntegerType *IT) {
41 uint64_t NumBits = IT->getSizeInBits().getFixedValue();
42 if (!IT->isBitInt())
43 return NumBits;
44 uint64_t BitAlign =
45 std::max<uint64_t>(a: 8, b: std::min<uint64_t>(a: 64, b: llvm::bit_ceil(Value: NumBits)));
46 return llvm::alignTo(Value: NumBits, Align: BitAlign);
47}
48
49static uint64_t getClangVectorWidthInBits(const VectorType *VT) {
50 const Type *EltTy = VT->getElementType();
51 uint64_t EltWidth = EltTy->getSizeInBits().getFixedValue();
52 if (const auto *IT = dyn_cast<IntegerType>(Val: EltTy))
53 EltWidth = getClangIntegerWidthInBits(IT);
54 uint64_t Width =
55 std::max<uint64_t>(a: 8, b: EltWidth * VT->getNumElements().getKnownMinValue());
56 if (Width & (Width - 1))
57 Width = llvm::alignTo(Value: Width, Align: llvm::bit_ceil(Value: Width));
58 return Width;
59}
60
61// The storage-container width of a type, mirroring Clang's getTypeSize. Used on
62// the stack path so a _BitInt or illegal vector coerces to the integer covering
63// its storage, not its raw iN width.
64static uint64_t getClangTypeWidthInBits(const Type *Ty) {
65 if (const auto *VT = dyn_cast<VectorType>(Val: Ty))
66 return getClangVectorWidthInBits(VT);
67 if (const auto *IT = dyn_cast<IntegerType>(Val: Ty))
68 return getClangIntegerWidthInBits(IT);
69 return Ty->getSizeInBits().getFixedValue();
70}
71
72class X86_64TargetInfo : public TargetInfo {
73public:
74 enum Class { Integer, Sse, SseUp, X87, X87Up, ComplexX87, NoClass, Memory };
75
76private:
77 TypeBuilder &TB;
78 X86AVXABILevel AVXLevel;
79 bool Has64BitPointers;
80
81 static Class merge(Class Accum, Class Field);
82
83 void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
84
85 void classify(const Type *T, uint64_t OffsetBase, Class &Lo, Class &Hi,
86 bool IsNamedArg, bool IsRegCall = false) const;
87
88 const Type *getIntegerTypeAtOffset(const Type *IRType, unsigned IROffset,
89 const Type *SourceTy,
90 unsigned SourceOffset,
91 bool InMemory = false) const;
92
93 const Type *getSSETypeAtOffset(const Type *ABIType, unsigned ABIOffset,
94 const Type *SourceTy,
95 unsigned SourceOffset) const;
96 bool isIllegalVectorType(const Type *Ty) const;
97 bool containsMatrixField(const RecordType *RT) const;
98
99 void computeInfo(FunctionInfo &FI) const override;
100 ArgInfo getIndirectReturnResult(const Type *Ty) const;
101 const Type *getFPTypeAtOffset(const Type *Ty, unsigned Offset) const;
102
103 const Type *isSingleElementStruct(const Type *Ty) const;
104 const Type *getByteVectorType(const Type *Ty) const;
105
106 const Type *createPairType(const Type *Lo, const Type *Hi) const;
107 ArgInfo getIndirectResult(const Type *Ty, unsigned FreeIntRegs) const;
108
109 ArgInfo classifyReturnType(const Type *RetTy) const;
110
111 ArgInfo classifyArgumentType(const Type *Ty, unsigned FreeIntRegs,
112 unsigned &NeededInt, unsigned &NeededSse,
113 bool IsNamedArg, bool IsRegCall = false) const;
114 const Type *useFirstFieldIfTransparentUnion(const Type *Ty) const;
115
116public:
117 X86_64TargetInfo(TypeBuilder &TypeBuilder, X86AVXABILevel AVXABILevel,
118 bool Has64BitPtrs, const ABICompatInfo &Compat)
119 : TargetInfo(Compat), TB(TypeBuilder), AVXLevel(AVXABILevel),
120 Has64BitPointers(Has64BitPtrs) {}
121
122 bool has64BitPointers() const { return Has64BitPointers; }
123};
124
125// Gets the "best" type to represent the union.
126static const Type *reduceUnionForX8664(const RecordType *UnionType,
127 TypeBuilder &TB) {
128 assert(UnionType->isUnion() && "Expected union type");
129
130 ArrayRef<FieldInfo> Fields = UnionType->getFields();
131 if (Fields.empty()) {
132 return nullptr;
133 }
134
135 const Type *StorageType = nullptr;
136
137 for (const auto &Field : Fields) {
138 if (Field.IsBitField && Field.IsUnnamedBitfield &&
139 Field.BitFieldWidth == 0) {
140 continue;
141 }
142
143 const Type *FieldType = Field.FieldType;
144
145 if (UnionType->isTransparentUnion() && !StorageType) {
146 StorageType = FieldType;
147 break;
148 }
149
150 if (!StorageType ||
151 FieldType->getAlignment() > StorageType->getAlignment() ||
152 (FieldType->getAlignment() == StorageType->getAlignment() &&
153 TypeSize::isKnownGT(LHS: FieldType->getSizeInBits(),
154 RHS: StorageType->getSizeInBits()))) {
155 StorageType = FieldType;
156 }
157 }
158 return StorageType;
159}
160
161void X86_64TargetInfo::postMerge(unsigned AggregateSize, Class &Lo,
162 Class &Hi) const {
163 // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
164 //
165 // (a) If one of the classes is Memory, the whole argument is passed in
166 // memory.
167 //
168 // (b) If X87Up is not preceded by X87, the whole argument is passed in
169 // memory.
170 //
171 // (c) If the size of the aggregate exceeds two eightbytes and the first
172 // eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
173 // argument is passed in memory. NOTE: This is necessary to keep the
174 // ABI working for processors that don't support the __m256 type.
175 //
176 // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
177 //
178 // Some of these are enforced by the merging logic. Others can arise
179 // only with unions; for example:
180 // union { _Complex double; unsigned; }
181 //
182 // Note that clauses (b) and (c) were added in 0.98.
183
184 if (Hi == Memory)
185 Lo = Memory;
186 if (Hi == X87Up && Lo != X87 && getABICompatInfo().HonorsRevision98)
187 Lo = Memory;
188 if (AggregateSize > 128 && (Lo != Sse || Hi != SseUp))
189 Lo = Memory;
190 if (Hi == SseUp && Lo != Sse)
191 Hi = Sse;
192}
193X86_64TargetInfo::Class X86_64TargetInfo::merge(Class Accum, Class Field) {
194 // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
195 // classified recursively so that always two fields are
196 // considered. The resulting class is calculated according to
197 // the classes of the fields in the eightbyte:
198 //
199 // (a) If both classes are equal, this is the resulting class.
200 //
201 // (b) If one of the classes is NO_CLASS, the resulting class is
202 // the other class.
203 //
204 // (c) If one of the classes is MEMORY, the result is the MEMORY
205 // class.
206 //
207 // (d) If one of the classes is INTEGER, the result is the
208 // INTEGER.
209 //
210 // (e) If one of the classes is X87, X87Up, COMPLEX_X87 class,
211 // MEMORY is used as class.
212 //
213 // (f) Otherwise class SSE is used.
214
215 // Accum should never be memory (we should have returned) or
216 // ComplexX87 (because this cannot be passed in a structure).
217 assert((Accum != Memory && Accum != ComplexX87) &&
218 "Invalid accumulated classification during merge.");
219
220 if (Accum == Field || Field == NoClass)
221 return Accum;
222 if (Field == Memory)
223 return Memory;
224 if (Accum == NoClass)
225 return Field;
226 if (Accum == Integer || Field == Integer)
227 return Integer;
228 if (Field == X87 || Field == X87Up || Field == ComplexX87 || Accum == X87 ||
229 Accum == X87Up)
230 return Memory;
231
232 return Sse;
233}
234
235// A record with a matrix-extension field is passed in memory. clang has no
236// matrix-specific ABI code: a matrix falls through X86_64ABIInfo::classify to
237// the default MEMORY class. We model matrices as arrays, so this check
238// reproduces that record-with-matrix -> MEMORY result.
239bool X86_64TargetInfo::containsMatrixField(const RecordType *RT) const {
240 for (const auto &Field : RT->getFields()) {
241 const Type *FieldType = Field.FieldType;
242
243 if (const auto *AT = dyn_cast<ArrayType>(Val: FieldType)) {
244 if (AT->isMatrixType())
245 return true;
246 continue;
247 }
248
249 if (const auto *NestedRT = dyn_cast<RecordType>(Val: FieldType))
250 if (containsMatrixField(RT: NestedRT))
251 return true;
252 }
253 return false;
254}
255
256void X86_64TargetInfo::classify(const Type *T, uint64_t OffsetBase, Class &Lo,
257 Class &Hi, bool IsNamedArg,
258 bool IsRegCall) const {
259 Lo = Hi = NoClass;
260 Class &Current = OffsetBase < 64 ? Lo : Hi;
261 Current = Memory;
262
263 if (T->isVoid()) {
264 Current = NoClass;
265 return;
266 }
267
268 if (const auto *IT = dyn_cast<IntegerType>(Val: T)) {
269 auto BitWidth = IT->getSizeInBits().getFixedValue();
270
271 if (BitWidth == 128 ||
272 (IT->isBitInt() && BitWidth > 64 && BitWidth <= 128)) {
273 Lo = Integer;
274 Hi = Integer;
275 } else if (BitWidth <= 64) {
276 Current = Integer;
277 }
278
279 return;
280 }
281
282 if (const auto *FT = dyn_cast<FloatType>(Val: T)) {
283 const auto *FltSem = FT->getSemantics();
284
285 if (FltSem == &llvm::APFloat::IEEEsingle() ||
286 FltSem == &llvm::APFloat::IEEEdouble() ||
287 FltSem == &llvm::APFloat::IEEEhalf() ||
288 FltSem == &llvm::APFloat::BFloat()) {
289 Current = Sse;
290 } else if (FltSem == &llvm::APFloat::IEEEquad()) {
291 Lo = Sse;
292 Hi = SseUp;
293 } else if (FltSem == &llvm::APFloat::x87DoubleExtended()) {
294 Lo = X87;
295 Hi = X87Up;
296 } else {
297 Current = Sse;
298 }
299 return;
300 }
301 if (T->isPointer()) {
302 Current = Integer;
303 return;
304 }
305
306 if (const auto *MPT = dyn_cast<MemberPointerType>(Val: T)) {
307 if (MPT->isFunctionPointer()) {
308 if (Has64BitPointers) {
309 Lo = Hi = Integer;
310 } else {
311 uint64_t EbFuncPtr = OffsetBase / 64;
312 uint64_t EbThisAdj = (OffsetBase + 64 - 1) / 64;
313 if (EbFuncPtr != EbThisAdj) {
314 Lo = Hi = Integer;
315 } else {
316 Current = Integer;
317 }
318 }
319 } else {
320 Current = Integer;
321 }
322 return;
323 }
324
325 if (const auto *VT = dyn_cast<VectorType>(Val: T)) {
326 auto Size = VT->getSizeInBits().getFixedValue();
327 const Type *ElementType = VT->getElementType();
328
329 if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
330 // gcc passes the following as integer:
331 // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
332 // 2 bytes - <2 x char>, <1 x short>
333 // 1 byte - <1 x char>
334 Current = Integer;
335 // If this type crosses an eightbyte boundary, it should be
336 // split.
337 uint64_t EbLo = (OffsetBase) / 64;
338 uint64_t EbHi = (OffsetBase + Size - 1) / 64;
339 if (EbLo != EbHi)
340 Hi = Lo;
341 } else if (Size == 64) {
342 if (const auto *FT = dyn_cast<FloatType>(Val: ElementType)) {
343 // gcc passes <1 x double> in memory. :(
344 if (FT->getSemantics() == &llvm::APFloat::IEEEdouble())
345 return;
346 }
347
348 // gcc passes <1 x long long> as SSE but clang used to unconditionally
349 // pass them as integer. For platforms where clang is the de facto
350 // platform compiler, we must continue to use integer.
351 if (const auto *IT = dyn_cast<IntegerType>(Val: ElementType)) {
352 uint64_t ElemBits = IT->getSizeInBits().getFixedValue();
353 if (!getABICompatInfo().ClassifyIntegerMMXAsSSE && ElemBits == 64 &&
354 !IT->isBitInt()) {
355 Current = Integer;
356 } else {
357 Current = Sse;
358 }
359 } else {
360 Current = Sse;
361 }
362 // If this type crosses an eightbyte boundary, it should be
363 // split.
364 if (OffsetBase && OffsetBase != 64)
365 Hi = Lo;
366 } else if (Size == 128 ||
367 (IsNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
368 if (const auto *IT = dyn_cast<IntegerType>(Val: ElementType)) {
369 uint64_t ElemBits = IT->getSizeInBits().getFixedValue();
370 // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
371 if (getABICompatInfo().PassInt128VectorsInMem && Size != 128 &&
372 ElemBits == 128 && !IT->isBitInt())
373 return;
374 }
375
376 // Arguments of 256-bits are split into four eightbyte chunks. The
377 // least significant one belongs to class SSE and all the others to class
378 // SSEUP. The original Lo and Hi design considers that types can't be
379 // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
380 // This design isn't correct for 256-bits, but since there're no cases
381 // where the upper parts would need to be inspected, avoid adding
382 // complexity and just consider Hi to match the 64-256 part.
383 //
384 // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
385 // registers if they are "named", i.e. not part of the "..." of a
386 // variadic function.
387 //
388 // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
389 // split into eight eightbyte chunks, one SSE and seven SSEUP.
390 Lo = Sse;
391 Hi = SseUp;
392 }
393 return;
394 }
395
396 if (const auto *CT = dyn_cast<ComplexType>(Val: T)) {
397 const Type *ElementType = CT->getElementType();
398 uint64_t Size = T->getSizeInBits().getFixedValue();
399
400 if (isa<IntegerType>(Val: ElementType)) {
401 if (Size <= 64)
402 Current = Integer;
403 else if (Size <= 128)
404 Lo = Hi = Integer;
405 } else if (const auto *EFT = dyn_cast<FloatType>(Val: ElementType)) {
406 const auto *FltSem = EFT->getSemantics();
407 if (FltSem == &llvm::APFloat::IEEEhalf() ||
408 FltSem == &llvm::APFloat::IEEEsingle() ||
409 FltSem == &llvm::APFloat::BFloat())
410 Current = Sse;
411 else if (FltSem == &llvm::APFloat::IEEEquad())
412 Current = Memory;
413 else if (FltSem == &llvm::APFloat::x87DoubleExtended())
414 Current = ComplexX87;
415 else if (FltSem == &llvm::APFloat::IEEEdouble())
416 Lo = Hi = Sse;
417 else
418 llvm_unreachable("Unexpected long double representation!");
419 }
420
421 uint64_t ElementSize = ElementType->getSizeInBits().getFixedValue();
422 // If this complex type crosses an eightbyte boundary then it
423 // should be split.
424 uint64_t EbReal = OffsetBase / 64;
425 uint64_t EbImag = (OffsetBase + ElementSize) / 64;
426 if (Hi == NoClass && EbReal != EbImag)
427 Hi = Lo;
428
429 return;
430 }
431
432 if (const auto *AT = dyn_cast<ArrayType>(Val: T)) {
433 // A matrix type is modeled as an array but, like Clang, is treated as a
434 // non-aggregate scalar: it matches no class here and stays in the Memory
435 // class, so classify*Type later returns it Direct (coerced to its
436 // flattened vector) rather than classifying it field-by-field.
437 if (AT->isMatrixType())
438 return;
439
440 // Arrays are treated like structures.
441 uint64_t Size = AT->getSizeInBits().getFixedValue();
442
443 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
444 // than eight eightbytes, ..., it has class MEMORY.
445 // regcall ABI doesn't have limitation to an object. The only limitation
446 // is the free registers, which will be checked in computeInfo.
447 if (!IsRegCall && Size > 512)
448 return;
449
450 // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
451 // fields, it has class MEMORY.
452 //
453 // Only need to check alignment of array base.
454 const Type *ElementType = AT->getElementType();
455 uint64_t ElemAlign = ElementType->getAlignment().value() * 8;
456 if (OffsetBase % ElemAlign)
457 return;
458
459 // Otherwise implement simplified merge. We could be smarter about
460 // this, but it isn't worth it and would be harder to verify.
461 Current = NoClass;
462 uint64_t EltSize = ElementType->getSizeInBits().getFixedValue();
463 uint64_t ArraySize = AT->getNumElements();
464
465 // The only case a 256-bit wide vector could be used is when the array
466 // contains a single 256-bit element. Since Lo and Hi logic isn't extended
467 // to work for sizes wider than 128, early check and fallback to memory.
468 //
469 if (Size > 128 &&
470 (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))
471 return;
472
473 for (uint64_t I = 0, Offset = OffsetBase; I < ArraySize;
474 ++I, Offset += EltSize) {
475 Class FieldLo, FieldHi;
476 classify(T: ElementType, OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, IsNamedArg);
477 Lo = merge(Accum: Lo, Field: FieldLo);
478 Hi = merge(Accum: Hi, Field: FieldHi);
479 if (Lo == Memory || Hi == Memory)
480 break;
481 }
482 postMerge(AggregateSize: Size, Lo, Hi);
483 assert((Hi != SseUp || Lo == Sse) && "Invalid SseUp array classification.");
484 return;
485 }
486
487 if (const auto *RT = dyn_cast<RecordType>(Val: T)) {
488 uint64_t Size = RT->getSizeInBits().getFixedValue();
489
490 if (containsMatrixField(RT)) {
491 Lo = Memory;
492 return;
493 }
494
495 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
496 // than eight eightbytes, ..., it has class MEMORY.
497 if (Size > 512)
498 return;
499
500 // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
501 // copy constructor or a non-trivial destructor, it is passed by invisible
502 // reference.
503 if (getRecordArgABI(RT))
504 return;
505
506 // Assume variable sized types are passed in memory.
507 if (RT->hasFlexibleArrayMember())
508 return;
509
510 // Reset Lo class, this will be recomputed.
511 Current = NoClass;
512
513 // If this is a C++ record, classify the bases first.
514 if (RT->isCXXRecord()) {
515 for (const auto &Base : RT->getBaseClasses()) {
516
517 // Classify this field.
518 //
519 // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
520 // single eightbyte, each is classified separately. Each eightbyte gets
521 // initialized to class NO_CLASS.
522 Class FieldLo, FieldHi;
523 uint64_t Offset = OffsetBase + Base.OffsetInBits;
524 classify(T: Base.FieldType, OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, IsNamedArg);
525 Lo = merge(Accum: Lo, Field: FieldLo);
526 Hi = merge(Accum: Hi, Field: FieldHi);
527
528 if (getABICompatInfo().ReturnCXXRecordGreaterThan128InMem &&
529 (Size > 128 &&
530 (Size != Base.FieldType->getSizeInBits().getFixedValue() ||
531 Size > getNativeVectorSizeForAVXABI(AVXLevel))))
532 Lo = Memory;
533
534 if (Lo == Memory || Hi == Memory) {
535 postMerge(AggregateSize: Size, Lo, Hi);
536 return;
537 }
538 }
539 }
540
541 // Classify the fields one at a time, merging the results.
542
543 bool IsUnion = RT->isUnion() && !getABICompatInfo().Clang11Compat;
544 for (const auto &Field : RT->getFields()) {
545 uint64_t Offset = OffsetBase + Field.OffsetInBits;
546 bool BitField = Field.IsBitField;
547
548 if (BitField && Field.IsUnnamedBitfield)
549 continue;
550
551 if (Size > 128 &&
552 ((!IsUnion &&
553 Size != Field.FieldType->getSizeInBits().getFixedValue()) ||
554 Size > getNativeVectorSizeForAVXABI(AVXLevel))) {
555 Lo = Memory;
556 postMerge(AggregateSize: Size, Lo, Hi);
557 return;
558 }
559
560 bool IsInMemory = Offset % (Field.FieldType->getAlignment().value() * 8);
561 if (!BitField && IsInMemory) {
562 Lo = Memory;
563 postMerge(AggregateSize: Size, Lo, Hi);
564 return;
565 }
566
567 Class FieldLo, FieldHi;
568
569 if (BitField) {
570 uint64_t BitFieldSize = Field.BitFieldWidth;
571 uint64_t EbLo = Offset / 64;
572 uint64_t EbHi = (Offset + BitFieldSize - 1) / 64;
573
574 if (EbLo) {
575 assert(EbHi == EbLo && "Invalid classification, type > 16 bytes.");
576 FieldLo = NoClass;
577 FieldHi = Integer;
578 } else {
579 FieldLo = Integer;
580 FieldHi = EbHi ? Integer : NoClass;
581 }
582 } else {
583 classify(T: Field.FieldType, OffsetBase: Offset, Lo&: FieldLo, Hi&: FieldHi, IsNamedArg);
584 }
585
586 Lo = merge(Accum: Lo, Field: FieldLo);
587 Hi = merge(Accum: Hi, Field: FieldHi);
588 if (Lo == Memory || Hi == Memory)
589 break;
590 }
591 postMerge(AggregateSize: Size, Lo, Hi);
592 return;
593 }
594
595 Lo = Memory;
596 Hi = NoClass;
597}
598
599const Type *
600X86_64TargetInfo::useFirstFieldIfTransparentUnion(const Type *Ty) const {
601 if (const auto *RT = dyn_cast<RecordType>(Val: Ty)) {
602 if (RT->isUnion() && RT->isTransparentUnion()) {
603 auto Fields = RT->getFields();
604 assert(!Fields.empty() && "transparent union cannot be empty");
605 return Fields.front().FieldType;
606 }
607 }
608 return Ty;
609}
610
611ArgInfo
612X86_64TargetInfo::classifyArgumentType(const Type *Ty, unsigned FreeIntRegs,
613 unsigned &NeededInt, unsigned &NeededSSE,
614 bool IsNamedArg, bool IsRegCall) const {
615
616 Ty = useFirstFieldIfTransparentUnion(Ty);
617
618 X86_64TargetInfo::Class Lo, Hi;
619 classify(T: Ty, OffsetBase: 0, Lo, Hi, IsNamedArg, IsRegCall);
620
621 // Check some invariants
622 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
623 assert((Hi != SseUp || Lo == Sse) && "Invalid SseUp classification.");
624
625 NeededInt = 0;
626 NeededSSE = 0;
627 const Type *ResType = nullptr;
628
629 switch (Lo) {
630 case NoClass:
631 if (Hi == NoClass)
632 return ArgInfo::getIgnore();
633 // If the low part is just padding, it takes no register, leave ResType
634 // null.
635 assert((Hi == Sse || Hi == Integer || Hi == X87Up) &&
636 "Unknown missing lo part");
637 break;
638
639 // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
640 // on the stack.
641 case Memory:
642 // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87Up or
643 // COMPLEX_X87, it is passed in memory.
644 case X87:
645 case ComplexX87:
646 if (getRecordArgABI(Ty) == RAA_Indirect)
647 ++NeededInt;
648 return getIndirectResult(Ty, FreeIntRegs);
649
650 case SseUp:
651 case X87Up:
652 llvm_unreachable("Invalid classification for lo word.");
653
654 // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
655 // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
656 // and %r9 is used.
657 case Integer:
658 ++NeededInt;
659
660 // Pick an 8-byte type based on the preferred type.
661 ResType = getIntegerTypeAtOffset(IRType: Ty, IROffset: 0, SourceTy: Ty, SourceOffset: 0);
662
663 // If we have a sign or zero extended integer, make sure to return Extend
664 // so that the parameter gets the right LLVM IR attributes.
665 if (Hi == NoClass && ResType->isInteger()) {
666 if (Ty->isInteger() && isPromotableInteger(IT: cast<IntegerType>(Val: Ty)))
667 return ArgInfo::getExtend(T: Ty);
668 }
669
670 if (ResType->isInteger() && ResType->getSizeInBits() == 128) {
671 assert(Hi == Integer);
672 ++NeededInt;
673 return ArgInfo::getDirect(T: ResType);
674 }
675 break;
676
677 // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
678 // available SSE register is used, the registers are taken in the
679 // order from %xmm0 to %xmm7.
680 case Sse:
681 ResType = getSSETypeAtOffset(ABIType: Ty, ABIOffset: 0, SourceTy: Ty, SourceOffset: 0);
682 ++NeededSSE;
683 break;
684 }
685
686 const Type *HighPart = nullptr;
687 switch (Hi) {
688 // Memory was handled previously, ComplexX87 and X87 should
689 // never occur as hi classes, and X87Up must be preceded by X87,
690 // which is passed in memory.
691 case Memory:
692 case X87:
693 case ComplexX87:
694 llvm_unreachable("Invalid classification for hi word.");
695
696 case NoClass:
697 break;
698
699 case Integer:
700 ++NeededInt;
701 // Pick an 8-byte type based on the preferred type.
702 HighPart = getIntegerTypeAtOffset(IRType: Ty, IROffset: 8, SourceTy: Ty, SourceOffset: 8);
703
704 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
705 return ArgInfo::getDirect(T: HighPart, Offset: 8);
706 break;
707
708 // X87Up generally doesn't occur here (long double is passed in
709 // memory), except in situations involving unions.
710 case X87Up:
711 case Sse:
712 ++NeededSSE;
713 HighPart = getSSETypeAtOffset(ABIType: Ty, ABIOffset: 8, SourceTy: Ty, SourceOffset: 8);
714
715 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.
716 return ArgInfo::getDirect(T: HighPart, Offset: 8);
717 break;
718
719 // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
720 // eightbyte is passed in the upper half of the last used SSE
721 // register. This only happens when 128-bit vectors are passed.
722 case SseUp:
723 assert(Lo == Sse && "Unexpected SseUp classification");
724 ResType = getByteVectorType(Ty);
725 break;
726 }
727
728 // If a high part was specified, merge it together with the low part. It is
729 // known to pass in the high eightbyte of the result. We do this by forming a
730 // first class struct aggregate with the high and low part: {low, high}
731 if (HighPart)
732 ResType = createPairType(Lo: ResType, Hi: HighPart);
733
734 return ArgInfo::getDirect(T: ResType);
735}
736
737ArgInfo X86_64TargetInfo::classifyReturnType(const Type *RetTy) const {
738 // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
739 // classification algorithm.
740
741 X86_64TargetInfo::Class Lo, Hi;
742 classify(T: RetTy, OffsetBase: 0, Lo, Hi, /*isNamedArg*/ IsNamedArg: true);
743
744 // Check some invariants
745 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
746 assert((Hi != SseUp || Lo == Sse) && "Invalid SseUp classification.");
747
748 const Type *ResType = nullptr;
749 switch (Lo) {
750 case NoClass:
751 if (Hi == NoClass)
752 return ArgInfo::getIgnore();
753 // If the low part is just padding, it takes no register, leave ResType
754 // null.
755 assert((Hi == Sse || Hi == Integer || Hi == X87Up) &&
756 "Unknown missing lo part");
757 break;
758 case SseUp:
759 case X87Up:
760 llvm_unreachable("Invalid classification for lo word.");
761
762 // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
763 // hidden argument.
764 case Memory:
765 return getIndirectReturnResult(Ty: RetTy);
766
767 // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
768 // available register of the sequence %rax, %rdx is used.
769 case Integer:
770 ResType = getIntegerTypeAtOffset(IRType: RetTy, IROffset: 0, SourceTy: RetTy, SourceOffset: 0);
771 // If we have a sign or zero extended integer, make sure to return Extend
772 // so that the parameter gets the right LLVM IR attributes.
773 if (Hi == NoClass && ResType->isInteger()) {
774 if (const IntegerType *IntTy = dyn_cast<IntegerType>(Val: RetTy)) {
775 if (isPromotableInteger(IT: IntTy))
776 return ArgInfo::getExtend(T: RetTy);
777 }
778 }
779 if (ResType->isInteger() && ResType->getSizeInBits() == 128) {
780 assert(Hi == Integer);
781 return ArgInfo::getDirect(T: ResType);
782 }
783 break;
784
785 // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
786 // available SSE register of the sequence %xmm0, %xmm1 is used.
787 case Sse:
788 ResType = getSSETypeAtOffset(ABIType: RetTy, ABIOffset: 0, SourceTy: RetTy, SourceOffset: 0);
789 break;
790
791 // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
792 // returned on the X87 stack in %st0 as 80-bit x87 number.
793 case X87:
794 ResType = TB.getFloatType(Semantics: APFloat::x87DoubleExtended(), Align: Align(16));
795 break;
796
797 // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
798 // part of the value is returned in %st0 and the imaginary part in
799 // %st1.
800 case ComplexX87:
801 assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
802 {
803 const Type *X87Type =
804 TB.getFloatType(Semantics: APFloat::x87DoubleExtended(), Align: Align(16));
805 FieldInfo Fields[] = {FieldInfo(X87Type, 0), FieldInfo(X87Type, 80)};
806 ResType = TB.getRecordType(Fields, Size: TypeSize::getFixed(ExactSize: 160), Align: Align(16));
807 }
808 break;
809 }
810
811 const Type *HighPart = nullptr;
812 switch (Hi) {
813 // Memory was handled previously and X87 should
814 // never occur as a hi class.
815 case Memory:
816 case X87:
817 llvm_unreachable("Invalid classification for hi word.");
818
819 case ComplexX87:
820 case NoClass:
821 break;
822
823 case Integer:
824 HighPart = getIntegerTypeAtOffset(IRType: RetTy, IROffset: 8, SourceTy: RetTy, SourceOffset: 8);
825 if (Lo == NoClass)
826 return ArgInfo::getDirect(T: HighPart, Offset: 8);
827 break;
828
829 case Sse:
830 HighPart = getSSETypeAtOffset(ABIType: RetTy, ABIOffset: 8, SourceTy: RetTy, SourceOffset: 8);
831 if (Lo == NoClass)
832 return ArgInfo::getDirect(T: HighPart, Offset: 8);
833 break;
834
835 // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
836 // is passed in the next available eightbyte chunk if the last used
837 // vector register.
838 //
839 // SSEUP should always be preceded by SSE, just widen.
840 case SseUp:
841 assert(Lo == Sse && "Unexpected SseUp classification.");
842 ResType = getByteVectorType(Ty: RetTy);
843 break;
844
845 // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87Up, the value is
846 // returned together with the previous X87 value in %st0.
847 case X87Up:
848 // If X87Up is preceded by X87, we don't need to do
849 // anything. However, in some cases with unions it may not be
850 // preceded by X87. In such situations we follow gcc and pass the
851 // extra bits in an SSE reg.
852 if (Lo != X87) {
853 HighPart = getSSETypeAtOffset(ABIType: RetTy, ABIOffset: 8, SourceTy: RetTy, SourceOffset: 8);
854 if (Lo == NoClass) // Return HighPart at offset 8 in memory.
855 return ArgInfo::getDirect(T: HighPart, Offset: 8);
856 }
857 break;
858 }
859
860 // If a high part was specified, merge it together with the low part. It is
861 // known to pass in the high eightbyte of the result. We do this by forming a
862 // first class struct aggregate with the high and low part: {low, high}
863 if (HighPart)
864 ResType = createPairType(Lo: ResType, Hi: HighPart);
865
866 return ArgInfo::getDirect(T: ResType);
867}
868
869/// Given a high and low type that can ideally
870/// be used as elements of a two register pair to pass or return, return a
871/// first class aggregate to represent them. For example, if the low part of
872/// a by-value argument should be passed as i32* and the high part as float,
873/// return {i32*, float}.
874const Type *X86_64TargetInfo::createPairType(const Type *Lo,
875 const Type *Hi) const {
876 // In order to correctly satisfy the ABI, we need to the high part to start
877 // at offset 8. If the high and low parts we inferred are both 4-byte types
878 // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
879 // the second element at offset 8. Check for this:
880 unsigned LoSize = (unsigned)Lo->getTypeAllocSize();
881 llvm::Align HiAlign = Hi->getAlignment();
882 unsigned HiStart = alignTo(Size: LoSize, A: HiAlign);
883
884 assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
885
886 // To handle this, we have to increase the size of the low part so that the
887 // second element will start at an 8 byte offset. We can't increase the size
888 // of the second element because it might make us access off the end of the
889 // struct.
890 const Type *AdjustedLo = Lo;
891 if (HiStart != 8) {
892 // There are usually two sorts of types the ABI generation code can produce
893 // for the low part of a pair that aren't 8 bytes in size: half, float or
894 // i8/i16/i32. This can also include pointers when they are 32-bit (X32 and
895 // NaCl).
896 // Promote these to a larger type.
897 if (Lo->isFloat()) {
898 const FloatType *FT = cast<FloatType>(Val: Lo);
899 if (FT->getSemantics() == &APFloat::IEEEhalf() ||
900 FT->getSemantics() == &APFloat::IEEEsingle() ||
901 FT->getSemantics() == &APFloat::BFloat())
902 AdjustedLo = TB.getFloatType(Semantics: APFloat::IEEEdouble(), Align: Align(8));
903 }
904 // Promote integers and pointers to i64
905 else if (Lo->isInteger() || Lo->isPointer())
906 AdjustedLo = TB.getIntegerType(BitWidth: 64, Align: Align(8), /*Signed=*/false);
907 else
908 assert((Lo->isInteger() || Lo->isPointer()) &&
909 "Invalid/unknown low type in pair");
910 unsigned AdjustedLoSize = AdjustedLo->getSizeInBits().getFixedValue() / 8;
911 HiStart = alignTo(Size: AdjustedLoSize, A: HiAlign);
912 }
913
914 // Create the pair struct
915 FieldInfo Fields[] = {FieldInfo(AdjustedLo, 0), FieldInfo(Hi, HiStart * 8)};
916
917 // Verify the high part is at offset 8
918 assert((8 * 8) == Fields[1].OffsetInBits &&
919 "High part must be at offset 8 bytes");
920
921 uint64_t PairSizeInBits =
922 Fields[1].OffsetInBits + Hi->getSizeInBits().getFixedValue();
923 return TB.getRecordType(Fields, Size: TypeSize::getFixed(ExactSize: PairSizeInBits), Align: Align(8),
924 Pack: StructPacking::Default);
925}
926
927static bool bitsContainNoUserData(const Type *Ty, unsigned StartBit,
928 unsigned EndBit) {
929 // If range is completely beyond type size, it's definitely padding
930 unsigned TySize = Ty->getSizeInBits().getFixedValue();
931 if (TySize <= StartBit)
932 return true;
933
934 // Handle arrays - check each element
935 if (const ArrayType *AT = dyn_cast<ArrayType>(Val: Ty)) {
936 const Type *EltTy = AT->getElementType();
937 unsigned EltSize = EltTy->getSizeInBits().getFixedValue();
938
939 for (unsigned I = 0; I < AT->getNumElements(); ++I) {
940 unsigned EltOffset = I * EltSize;
941 if (EltOffset >= EndBit)
942 break;
943
944 unsigned EltStart = (EltOffset < StartBit) ? StartBit - EltOffset : 0;
945 if (!bitsContainNoUserData(Ty: EltTy, StartBit: EltStart, EndBit: EndBit - EltOffset))
946 return false;
947 }
948 return true;
949 }
950
951 // Handle structs - check all fields and base classes
952 if (const RecordType *RT = dyn_cast<RecordType>(Val: Ty)) {
953 if (RT->isUnion()) {
954 for (const auto &Field : RT->getFields()) {
955 if (Field.IsUnnamedBitfield)
956 continue;
957
958 unsigned FieldStart =
959 (Field.OffsetInBits < StartBit) ? StartBit - Field.OffsetInBits : 0;
960 unsigned FieldEnd =
961 FieldStart + Field.FieldType->getSizeInBits().getFixedValue();
962
963 // Check if field overlaps with the queried range
964 if (FieldStart < EndBit && FieldEnd > StartBit) {
965 // There's an overlap, so there is user data
966 unsigned RelativeStart =
967 (StartBit > FieldStart) ? StartBit - FieldStart : 0;
968 unsigned RelativeEnd =
969 (EndBit < FieldEnd)
970 ? EndBit - FieldStart
971 : Field.FieldType->getSizeInBits().getFixedValue();
972
973 if (!bitsContainNoUserData(Ty: Field.FieldType, StartBit: RelativeStart,
974 EndBit: RelativeEnd)) {
975 return false;
976 }
977 }
978 }
979 return true;
980 }
981 // Check base classes first (for C++ records)
982 if (RT->isCXXRecord()) {
983 for (unsigned I = 0; I < RT->getNumBaseClasses(); ++I) {
984 const FieldInfo &Base = RT->getBaseClasses()[I];
985 if (Base.OffsetInBits >= EndBit)
986 continue;
987
988 unsigned BaseStart =
989 (Base.OffsetInBits < StartBit) ? StartBit - Base.OffsetInBits : 0;
990 if (!bitsContainNoUserData(Ty: Base.FieldType, StartBit: BaseStart,
991 EndBit: EndBit - Base.OffsetInBits))
992 return false;
993 }
994 }
995
996 for (unsigned I = 0; I < RT->getNumFields(); ++I) {
997 const FieldInfo &Field = RT->getFields()[I];
998 if (Field.OffsetInBits >= EndBit)
999 break;
1000
1001 unsigned FieldStart =
1002 (Field.OffsetInBits < StartBit) ? StartBit - Field.OffsetInBits : 0;
1003 if (!bitsContainNoUserData(Ty: Field.FieldType, StartBit: FieldStart,
1004 EndBit: EndBit - Field.OffsetInBits))
1005 return false;
1006 }
1007 return true;
1008 }
1009
1010 // For unions, vectors, and primitives - assume all bits are user data
1011 return false;
1012}
1013
1014const Type *X86_64TargetInfo::getIntegerTypeAtOffset(const Type *ABIType,
1015 unsigned ABIOffset,
1016 const Type *SourceTy,
1017 unsigned SourceOffset,
1018 bool InMemory) const {
1019
1020 const Type *WorkingType = ABIType;
1021 if (InMemory && ABIType->isInteger()) {
1022 const auto *IT = cast<IntegerType>(Val: ABIType);
1023 unsigned OriginalBitWidth = IT->getSizeInBits().getFixedValue();
1024
1025 unsigned WidenedBitWidth = OriginalBitWidth;
1026 if (OriginalBitWidth <= 8) {
1027 WidenedBitWidth = 8;
1028 } else {
1029 WidenedBitWidth = llvm::bit_ceil(Value: OriginalBitWidth);
1030 }
1031
1032 if (WidenedBitWidth != OriginalBitWidth) {
1033 WorkingType = TB.getIntegerType(BitWidth: WidenedBitWidth, Align: ABIType->getAlignment(),
1034 Signed: IT->isSigned());
1035 }
1036 }
1037 // If we're dealing with an un-offset ABI type, then it means that we're
1038 // returning an 8-byte unit starting with it. See if we can safely use it.
1039 if (ABIOffset == 0) {
1040 // Pointers and int64's always fill the 8-byte unit. Return WorkingType,
1041 // which is the in-memory-widened type (e.g. a _BitInt(37) field widened to
1042 // i64): returning the raw ABIType here would coerce the eightbyte to the
1043 // narrow iN instead of the storage integer clang uses.
1044 if ((WorkingType->isPointer() && Has64BitPointers) ||
1045 (WorkingType->isInteger() &&
1046 cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 64))
1047 return WorkingType;
1048
1049 // If we have a 1/2/4-byte integer, we can use it only if the rest of the
1050 // goodness in the source type is just tail padding. This is allowed to
1051 // kick in for struct {double,int} on the int, but not on
1052 // struct{double,int,int} because we wouldn't return the second int. We
1053 // have to do this analysis on the source type because we can't depend on
1054 // unions being lowered a specific way etc.
1055 if ((WorkingType->isInteger() &&
1056 (cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 1 ||
1057 cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 8 ||
1058 cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 16 ||
1059 cast<IntegerType>(Val: WorkingType)->getSizeInBits() == 32)) ||
1060 (WorkingType->isPointer() && !Has64BitPointers)) {
1061
1062 unsigned BitWidth = WorkingType->isPointer()
1063 ? 32
1064 : cast<IntegerType>(Val: WorkingType)->getSizeInBits();
1065
1066 if (bitsContainNoUserData(Ty: SourceTy, StartBit: SourceOffset * 8 + BitWidth,
1067 EndBit: SourceOffset * 8 + 64))
1068 return WorkingType;
1069 }
1070 }
1071
1072 if (const auto *RTy = dyn_cast<RecordType>(Val: ABIType)) {
1073 if (RTy->isUnion()) {
1074 const Type *ReducedType = reduceUnionForX8664(UnionType: RTy, TB);
1075 if (ReducedType)
1076 return getIntegerTypeAtOffset(ABIType: ReducedType, ABIOffset, SourceTy,
1077 SourceOffset, InMemory: true);
1078 }
1079 if (const FieldInfo *Element =
1080 RTy->getElementContainingOffset(OffsetInBits: ABIOffset * 8)) {
1081
1082 unsigned ElementOffsetBytes = Element->OffsetInBits / 8;
1083 return getIntegerTypeAtOffset(ABIType: Element->FieldType,
1084 ABIOffset: ABIOffset - ElementOffsetBytes, SourceTy,
1085 SourceOffset, InMemory: true);
1086 }
1087 }
1088
1089 if (const auto *ATy = dyn_cast<ArrayType>(Val: ABIType)) {
1090 const Type *EltTy = ATy->getElementType();
1091 unsigned EltSize = EltTy->getSizeInBits() / 8;
1092 if (EltSize > 0) {
1093 unsigned EltOffset = (ABIOffset / EltSize) * EltSize;
1094 return getIntegerTypeAtOffset(ABIType: EltTy, ABIOffset: ABIOffset - EltOffset, SourceTy,
1095 SourceOffset, InMemory: true);
1096 }
1097 }
1098
1099 // If we have a 128-bit integer, we can pass it safely using an i128
1100 // so we return that
1101 if (ABIType->isInteger() && ABIType->getSizeInBits() == 128) {
1102 assert(ABIOffset == 0);
1103 return ABIType;
1104 }
1105
1106 unsigned TySizeInBytes =
1107 llvm::divideCeil(Numerator: SourceTy->getSizeInBits().getFixedValue(), Denominator: 8);
1108 if (auto *IT = dyn_cast<IntegerType>(Val: SourceTy)) {
1109 if (IT->isBitInt())
1110 TySizeInBytes =
1111 alignTo(Value: SourceTy->getSizeInBits().getFixedValue(), Align: 64) / 8;
1112 }
1113 assert(TySizeInBytes != SourceOffset && "Empty field?");
1114 unsigned AvailableSize = TySizeInBytes - SourceOffset;
1115 return TB.getIntegerType(BitWidth: std::min(a: AvailableSize, b: 8U) * 8, Align: Align(1), Signed: false);
1116}
1117/// Returns the floating point type at the specified offset within a type, or
1118/// nullptr if no floating point type is found at that offset.
1119const Type *X86_64TargetInfo::getFPTypeAtOffset(const Type *Ty,
1120 unsigned Offset) const {
1121 // Check for direct match at offset 0
1122 if (Offset == 0 && Ty->isFloat())
1123 return Ty;
1124
1125 if (const ComplexType *CT = dyn_cast<ComplexType>(Val: Ty)) {
1126 const Type *ElementType = CT->getElementType();
1127 unsigned ElementSize = ElementType->getSizeInBits().getFixedValue() / 8;
1128
1129 if (Offset == 0 || Offset == ElementSize)
1130 return ElementType;
1131 return nullptr;
1132 }
1133
1134 // Handle struct types by checking each field
1135 if (const RecordType *RT = dyn_cast<RecordType>(Val: Ty)) {
1136 if (const FieldInfo *Element = RT->getElementContainingOffset(OffsetInBits: Offset * 8)) {
1137 unsigned ElementOffsetBytes = Element->OffsetInBits / 8;
1138 return getFPTypeAtOffset(Ty: Element->FieldType, Offset: Offset - ElementOffsetBytes);
1139 }
1140 }
1141
1142 // Handle array types
1143 if (const ArrayType *AT = dyn_cast<ArrayType>(Val: Ty)) {
1144 const Type *EltTy = AT->getElementType();
1145 unsigned EltSize = EltTy->getSizeInBits() / 8;
1146 unsigned EltIndex = Offset / EltSize;
1147
1148 return getFPTypeAtOffset(Ty: EltTy, Offset: Offset - (EltIndex * EltSize));
1149 }
1150
1151 // No floating point type found at this offset
1152 return nullptr;
1153}
1154
1155/// Helper to check if a floating point type matches specific semantics
1156static bool isFloatTypeWithSemantics(const Type *Ty,
1157 const fltSemantics &Semantics) {
1158 if (!Ty->isFloat())
1159 return false;
1160 const FloatType *FT = cast<FloatType>(Val: Ty);
1161 return FT->getSemantics() == &Semantics;
1162}
1163
1164/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
1165/// low 8 bytes of an XMM register, corresponding to the SSE class.
1166const Type *X86_64TargetInfo::getSSETypeAtOffset(const Type *ABIType,
1167 unsigned ABIOffset,
1168 const Type *SourceTy,
1169 unsigned SourceOffset) const {
1170
1171 if (const auto *RTy = dyn_cast<RecordType>(Val: ABIType)) {
1172 if (RTy->isUnion()) {
1173 const Type *ReducedType = reduceUnionForX8664(UnionType: RTy, TB);
1174 if (ReducedType) {
1175 return getSSETypeAtOffset(ABIType: ReducedType, ABIOffset, SourceTy,
1176 SourceOffset);
1177 }
1178 }
1179 }
1180
1181 auto Is16bitFpTy = [](const Type *T) {
1182 return isFloatTypeWithSemantics(Ty: T, Semantics: APFloat::IEEEhalf()) ||
1183 isFloatTypeWithSemantics(Ty: T, Semantics: APFloat::BFloat());
1184 };
1185
1186 // Get the floating point type at the requested offset
1187 const Type *T0 = getFPTypeAtOffset(Ty: ABIType, Offset: ABIOffset);
1188 if (!T0 || isFloatTypeWithSemantics(Ty: T0, Semantics: APFloat::IEEEdouble()))
1189 return TB.getFloatType(Semantics: APFloat::IEEEdouble(), Align: Align(8));
1190
1191 // Calculate remaining source size in bytes
1192 unsigned SourceSize =
1193 (SourceTy->getSizeInBits().getFixedValue() / 8) - SourceOffset;
1194
1195 // Try to get adjacent FP type
1196 const Type *T1 = nullptr;
1197 unsigned T0Size =
1198 alignTo(Value: T0->getSizeInBits().getFixedValue(), Align: T0->getAlignment().value()) /
1199 8;
1200 if (SourceSize > T0Size)
1201 T1 = getFPTypeAtOffset(Ty: ABIType, Offset: ABIOffset + T0Size);
1202
1203 if (T1 == nullptr) {
1204 if (Is16bitFpTy(T0) && SourceSize > 4)
1205 T1 = getFPTypeAtOffset(Ty: ABIType, Offset: ABIOffset + 4);
1206
1207 if (T1 == nullptr)
1208 return T0;
1209 }
1210 // Handle vector cases
1211 if (isFloatTypeWithSemantics(Ty: T0, Semantics: APFloat::IEEEsingle()) &&
1212 isFloatTypeWithSemantics(Ty: T1, Semantics: APFloat::IEEEsingle()))
1213 return TB.getVectorType(ElementType: T0, NumElements: ElementCount::getFixed(MinVal: 2), Align: Align(8));
1214
1215 if (Is16bitFpTy(T0) && Is16bitFpTy(T1)) {
1216 const Type *T2 = nullptr;
1217 if (SourceSize > 4)
1218 T2 = getFPTypeAtOffset(Ty: ABIType, Offset: ABIOffset + 4);
1219 if (!T2)
1220 return TB.getVectorType(ElementType: T0, NumElements: ElementCount::getFixed(MinVal: 2), Align: Align(8));
1221 return TB.getVectorType(ElementType: T0, NumElements: ElementCount::getFixed(MinVal: 4), Align: Align(8));
1222 }
1223
1224 // Mixed half-float cases
1225 if (Is16bitFpTy(T0) || Is16bitFpTy(T1))
1226 return TB.getVectorType(ElementType: TB.getFloatType(Semantics: APFloat::IEEEhalf(), Align: Align(2)),
1227 NumElements: ElementCount::getFixed(MinVal: 4), Align: Align(8));
1228
1229 // Default to double
1230 return TB.getFloatType(Semantics: APFloat::IEEEdouble(), Align: Align(8));
1231}
1232
1233/// The ABI specifies that a value should be passed in a full vector XMM/YMM
1234/// register. Pick an LLVM IR type that will be passed as a vector register.
1235const Type *X86_64TargetInfo::getByteVectorType(const Type *Ty) const {
1236 // Wrapper structs/arrays that only contain vectors are passed just like
1237 // vectors; strip them off if present.
1238 if (const Type *InnerTy = isSingleElementStruct(Ty))
1239 Ty = InnerTy;
1240
1241 // Handle vector types
1242 if (const VectorType *VT = dyn_cast<VectorType>(Val: Ty)) {
1243 // Don't pass vXi128 vectors in their native type, the backend can't
1244 // legalize them.
1245 if (getABICompatInfo().PassInt128VectorsInMem &&
1246 VT->getElementType()->isInteger() &&
1247 cast<IntegerType>(Val: VT->getElementType())->getSizeInBits() == 128) {
1248 unsigned Size = VT->getSizeInBits().getFixedValue();
1249 return TB.getVectorType(ElementType: TB.getIntegerType(BitWidth: 64, Align: Align(8), /*Signed=*/false),
1250 NumElements: ElementCount::getFixed(MinVal: Size / 64),
1251 Align: Align(Size / 8));
1252 }
1253 return VT;
1254 }
1255
1256 // Handle fp128
1257 if (isFloatTypeWithSemantics(Ty, Semantics: APFloat::IEEEquad()))
1258 return Ty;
1259
1260 // We couldn't find the preferred IR vector type for 'Ty'.
1261 unsigned Size = Ty->getSizeInBits().getFixedValue();
1262 assert((Size == 128 || Size == 256 || Size == 512) && "Invalid vector size");
1263
1264 return TB.getVectorType(ElementType: TB.getFloatType(Semantics: APFloat::IEEEdouble(), Align: Align(8)),
1265 NumElements: ElementCount::getFixed(MinVal: Size / 64), Align: Align(Size / 8));
1266}
1267
1268// Returns the single element if this is a single-element struct wrapper
1269const Type *X86_64TargetInfo::isSingleElementStruct(const Type *Ty) const {
1270 const auto *RT = dyn_cast<RecordType>(Val: Ty);
1271 if (!RT)
1272 return nullptr;
1273
1274 if (RT->hasFlexibleArrayMember())
1275 return nullptr;
1276
1277 const Type *Found = nullptr;
1278
1279 for (const auto &Base : RT->getBaseClasses()) {
1280 const Type *BaseTy = Base.FieldType;
1281 auto *BaseRT = dyn_cast<RecordType>(Val: BaseTy);
1282
1283 if (!BaseRT || BaseRT->isEmpty())
1284 continue;
1285
1286 const Type *Elem = isSingleElementStruct(Ty: BaseTy);
1287 if (!Elem || Found)
1288 return nullptr;
1289 Found = Elem;
1290 }
1291
1292 for (const auto &FI : RT->getFields()) {
1293 if (FI.isEmpty())
1294 continue;
1295
1296 const Type *FTy = FI.FieldType;
1297
1298 while (auto *AT = dyn_cast<ArrayType>(Val: FTy)) {
1299 if (AT->getNumElements() != 1)
1300 break;
1301 FTy = AT->getElementType();
1302 }
1303
1304 const Type *Elem;
1305 if (auto *InnerRT = dyn_cast<RecordType>(Val: FTy))
1306 Elem = isSingleElementStruct(Ty: InnerRT);
1307 else
1308 Elem = FTy;
1309 if (!Elem || Found)
1310 return nullptr;
1311 Found = Elem;
1312 }
1313
1314 if (!Found)
1315 return nullptr;
1316 if (Found->getSizeInBits() != Ty->getSizeInBits())
1317 return nullptr;
1318
1319 return Found;
1320}
1321
1322bool X86_64TargetInfo::isIllegalVectorType(const Type *Ty) const {
1323 if (const auto *VecTy = dyn_cast<VectorType>(Val: Ty)) {
1324 uint64_t Size = VecTy->getSizeInBits().getFixedValue();
1325 unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
1326
1327 // Vectors <= 64 bits or > largest supported vector size are illegal
1328 if (Size <= 64 || Size > LargestVector)
1329 return true;
1330
1331 // Check for 128-bit integer element vectors that should be passed in memory
1332 const Type *EltTy = VecTy->getElementType();
1333 if (getABICompatInfo().PassInt128VectorsInMem && EltTy->isInteger()) {
1334 const auto *IntTy = cast<IntegerType>(Val: EltTy);
1335 if (IntTy->getSizeInBits().getFixedValue() == 128)
1336 return true;
1337 }
1338 }
1339 return false;
1340}
1341
1342ArgInfo X86_64TargetInfo::getIndirectResult(const Type *Ty,
1343 unsigned FreeIntRegs) const {
1344 // If this is a scalar LLVM value then assume LLVM will pass it in the right
1345 // place naturally.
1346 //
1347 // This assumption is optimistic, as there could be free registers available
1348 // when we need to pass this argument in memory, and LLVM could try to pass
1349 // the argument in the free register. This does not seem to happen currently,
1350 // but this code would be much safer if we could mark the argument with
1351 // 'onstack'. See PR12193.
1352 if (!isAggregateTypeForABI(Ty) && !isIllegalVectorType(Ty) &&
1353 !(Ty->isInteger() && cast<IntegerType>(Val: Ty)->isBitInt())) {
1354 return (Ty->isInteger() && isPromotableInteger(IT: cast<IntegerType>(Val: Ty))
1355 ? ArgInfo::getExtend(T: Ty)
1356 : ArgInfo::getDirect());
1357 }
1358
1359 // Check if this is a record type that needs special handling
1360 if (auto RecordRAA = getRecordArgABI(Ty))
1361 return getNaturalAlignIndirect(Ty, ByVal: RecordRAA ==
1362 RecordArgABI::RAA_DirectInMemory);
1363
1364 // Compute the byval alignment. We specify the alignment of the byval in all
1365 // cases so that the mid-level optimizer knows the alignment of the byval.
1366 uint64_t AlignVal = std::max<uint64_t>(a: Ty->getAlignment().value(), b: 8u);
1367
1368 // Attempt to avoid passing indirect results using byval when possible. This
1369 // is important for good codegen.
1370 //
1371 // We do this by coercing the value into a scalar type which the backend can
1372 // handle naturally (i.e., without using byval).
1373 //
1374 // For simplicity, we currently only do this when we have exhausted all of the
1375 // free integer registers. Doing this when there are free integer registers
1376 // would require more care, as we would have to ensure that the coerced value
1377 // did not claim the unused register. That would require either reording the
1378 // arguments to the function (so that any subsequent inreg values came first),
1379 // or only doing this optimization when there were no following arguments that
1380 // might be inreg.
1381 //
1382 // We currently expect it to be rare (particularly in well written code) for
1383 // arguments to be passed on the stack when there are still free integer
1384 // registers available (this would typically imply large structs being passed
1385 // by value), so this seems like a fair tradeoff for now.
1386 //
1387 // We can revisit this if the backend grows support for 'onstack' parameter
1388 // attributes. See PR12193.
1389 if (FreeIntRegs == 0) {
1390 // Use the storage-container width (like Clang's getTypeSize) so a stack
1391 // _BitInt or illegal vector coerces to the integer covering its storage,
1392 // not its raw iN width.
1393 uint64_t Size = getClangTypeWidthInBits(Ty);
1394
1395 // If this type fits in an eightbyte, coerce it into the matching integral
1396 // type, which will end up on the stack (with alignment 8).
1397 if (AlignVal == 8 && Size <= 64) {
1398 const Type *IntTy =
1399 TB.getIntegerType(BitWidth: Size, Align: llvm::Align(8), /*Signed=*/false);
1400 return ArgInfo::getDirect(T: IntTy);
1401 }
1402 }
1403
1404 return ArgInfo::getIndirect(Align: llvm::Align(AlignVal), /*ByVal=*/true);
1405}
1406
1407ArgInfo X86_64TargetInfo::getIndirectReturnResult(const Type *Ty) const {
1408 if (!isAggregateTypeForABI(Ty)) {
1409 // Bit-precise integers are returned indirectly regardless of size.
1410 if (const auto *IntTy = dyn_cast<IntegerType>(Val: Ty)) {
1411 if (IntTy->isBitInt())
1412 return getNaturalAlignIndirect(Ty: IntTy, /*ByVal=*/true);
1413 if (isPromotableInteger(IT: IntTy))
1414 return ArgInfo::getExtend(T: Ty);
1415 }
1416 return ArgInfo::getDirect();
1417 }
1418
1419 return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
1420}
1421
1422static bool classifyCXXReturnType(FunctionInfo &FI) {
1423 const abi::Type *Ty = FI.getReturnType();
1424
1425 if (const auto *RT = llvm::dyn_cast<abi::RecordType>(Val: Ty)) {
1426 if (!RT->canPassInRegisters()) {
1427 // A C++ record that cannot pass in registers (non-trivial copy/dtor)
1428 // is returned indirectly with ByVal=false, matching
1429 // ItaniumCXXABI::classifyReturnType. This is the RAA path and is distinct
1430 // from getIndirectReturnResult (plain aggregates), which uses ByVal=true.
1431 FI.getReturnInfo() =
1432 ArgInfo::getIndirect(Align: RT->getAlignment(), /*ByVal=*/false);
1433 return true;
1434 }
1435 }
1436
1437 return false;
1438}
1439
1440void X86_64TargetInfo::computeInfo(FunctionInfo &FI) const {
1441 CallingConv::ID CallingConv = FI.getCallingConvention();
1442
1443 // Only the standard SysV (C) calling convention is classified here. Any other
1444 // convention must be added explicitly once it has been verified against this
1445 // classifier rather than silently taking the SysV path.
1446 switch (CallingConv) {
1447 case CallingConv::C:
1448 break;
1449 default:
1450 llvm_unreachable(
1451 "calling convention not supported by the LLVMABI X86_64 classifier");
1452 }
1453
1454 unsigned FreeIntRegs = 6;
1455 unsigned FreeSSERegs = 8;
1456 unsigned NeededInt = 0, NeededSSE = 0;
1457
1458 if (!classifyCXXReturnType(FI)) {
1459 const Type *RetTy = FI.getReturnType();
1460 FI.getReturnInfo() = classifyReturnType(RetTy);
1461 }
1462
1463 if (FI.getReturnInfo().isIndirect())
1464 --FreeIntRegs;
1465
1466 unsigned NumRequiredArgs = FI.getNumRequiredArgs();
1467
1468 unsigned ArgNo = 0;
1469 for (auto IT = FI.arg_begin(), IE = FI.arg_end(); IT != IE; ++IT, ++ArgNo) {
1470 bool IsNamedArg = ArgNo < NumRequiredArgs;
1471 const Type *ArgTy = IT->ABIType;
1472 NeededInt = 0;
1473 NeededSSE = 0;
1474
1475 ArgInfo AI = classifyArgumentType(Ty: ArgTy, FreeIntRegs, NeededInt, NeededSSE,
1476 IsNamedArg);
1477
1478 // AMD64-ABI 3.2.3p3: If there are no registers available for any
1479 // eightbyte of an argument, the whole argument is passed on the
1480 // stack. If registers have already been assigned for some
1481 // eightbytes of such an argument, the assignments get reverted.
1482 if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
1483 FreeIntRegs -= NeededInt;
1484 FreeSSERegs -= NeededSSE;
1485 IT->Info = AI;
1486 } else {
1487 // Not enough registers, pass on stack
1488 IT->Info = getIndirectResult(Ty: ArgTy, FreeIntRegs);
1489 }
1490 }
1491}
1492
1493std::unique_ptr<TargetInfo>
1494createX86_64TargetInfo(TypeBuilder &TB, X86AVXABILevel AVXLevel,
1495 bool Has64BitPointers, const ABICompatInfo &Compat) {
1496 return std::make_unique<X86_64TargetInfo>(args&: TB, args&: AVXLevel, args&: Has64BitPointers,
1497 args: Compat);
1498}
1499
1500} // namespace abi
1501} // namespace llvm
1502