1//===- AArch64.cpp - AArch64 ABI Implementation ---------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ABI/FunctionInfo.h"
10#include "llvm/ABI/TargetInfo.h"
11#include "llvm/ABI/Types.h"
12#include "llvm/ADT/STLExtras.h"
13#include "llvm/ADT/SmallVector.h"
14#include "llvm/Support/Casting.h"
15#include "llvm/Support/ErrorHandling.h"
16#include "llvm/Support/MathExtras.h"
17#include "llvm/Support/TypeSize.h"
18#include "llvm/Support/WithColor.h"
19#include <algorithm>
20#include <cstdint>
21
22namespace llvm {
23namespace abi {
24
25class AArch64TargetInfo : public TargetInfo {
26public:
27 AArch64TargetInfo(TypeBuilder &TB, const AArch64ABIOptions &Opts)
28 : TargetInfo(TB), Opts(Opts) {}
29
30 const ABICompatInfo &getABICompatInfo() const override {
31 return Opts.CompatInfo;
32 }
33
34 void computeInfo(FunctionInfo &FI) const override {
35 if (!maybeCommonClassifyReturnType(FI))
36 FI.getReturnInfo() =
37 classifyReturnType(RetTy: FI.getReturnType(), IsVariadicFn: FI.isVariadic());
38
39 unsigned ArgNo = 0;
40 unsigned NSRN = 0, NPRN = 0;
41 for (auto &I : FI.arguments()) {
42 const bool IsNamedArg =
43 !FI.isVariadic() || ArgNo < FI.getNumRequiredArgs();
44 ++ArgNo;
45 I.Info = classifyArgumentType(Ty: I.ABIType, IsVariadicFn: FI.isVariadic(), IsNamedArg,
46 CallingConvention: FI.getCallingConvention(), NSRN, NPRN);
47 }
48 }
49
50private:
51 AArch64ABIOptions Opts;
52
53 ArgInfo classifyReturnType(const Type *RetTy, bool IsVariadicFn) const;
54 ArgInfo classifyArgumentType(const Type *Ty, bool IsVariadicFn,
55 bool IsNamedArg, unsigned CallingConvention,
56 unsigned &NSRN, unsigned &NPRN) const;
57
58 bool isDarwinPCS() const { return Opts.Kind == AArch64ABIKind::DarwinPCS; }
59 bool isSoftFloat() const { return Opts.Kind == AArch64ABIKind::AAPCSSoft; }
60
61 const VectorType *
62 convertFixedToScalableVectorType(const VectorType *VT) const;
63
64 ArgInfo coerceIllegalVector(const VectorType *VT, unsigned &NSRN,
65 unsigned &NPRN) const;
66 ArgInfo coerceAndExpandPureScalableAggregate(
67 const Type *Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
68 const SmallVectorImpl<const Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
69 unsigned &NPRN) const;
70
71 bool isIllegalVectorType(const Type *Ty) const;
72
73 bool passAsAggregateType(const Type *Ty) const;
74 bool passAsPureScalableType(const Type *Ty, unsigned &NV, unsigned &NP,
75 SmallVectorImpl<const Type *> &CoerceToSeq) const;
76
77 void flattenType(const Type *Ty,
78 SmallVectorImpl<const Type *> &Flattened) const;
79 bool containsOnlyPointers(const Type *Ty) const;
80
81 bool isHomogeneousAggregateBaseType(const Type *Ty) const override;
82 bool isHomogeneousAggregateSmallEnough(const Type *Base,
83 uint64_t Members) const override;
84 bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const override;
85 bool isPermittedToBeHomogeneousAggregate(const RecordType *RT) const override;
86};
87
88std::unique_ptr<TargetInfo>
89createAArch64TargetInfo(TypeBuilder &TB, const AArch64ABIOptions &Opts) {
90 return std::make_unique<AArch64TargetInfo>(args&: TB, args: Opts);
91}
92
93static void reportNYI(StringRef Feature) {
94 WithColor::warning()
95 << Feature
96 << " is not yet implemented for AArch64 in the LLVM ABI library.\n";
97}
98
99ArgInfo AArch64TargetInfo::classifyReturnType(const Type *RetTy,
100 bool IsVariadicFn) const {
101 if (RetTy->isVoid())
102 return ArgInfo::getIgnore();
103
104 if (const auto *VT = dyn_cast<VectorType>(Val: RetTy)) {
105 if (VT->isFixedLengthSVEData() || VT->isFixedLengthSVEPredicate()) {
106 unsigned NSRN = 0, NPRN = 0;
107 return coerceIllegalVector(VT, NSRN, NPRN);
108 }
109
110 // Large vector types should be returned via memory.
111 if (VT->getABISizeInBits() > 128)
112 return getNaturalAlignIndirect(Ty: RetTy, AddrSpace: getAllocaAddrSpace());
113 }
114
115 if (!passAsAggregateType(Ty: RetTy)) {
116 if (const auto *IntTy = dyn_cast<IntegerType>(Val: RetTy)) {
117 if (IntTy->isBitInt())
118 if (RetTy->getSizeInBits().getFixedValue() > 128)
119 return getNaturalAlignIndirect(Ty: RetTy, AddrSpace: getAllocaAddrSpace());
120
121 if (isPromotableInteger(IT: IntTy) && isDarwinPCS())
122 return ArgInfo::getExtend(T: IntTy);
123 }
124
125 // Everything not handled above is returned directly.
126 return ArgInfo::getDirect();
127 }
128
129 uint64_t Size = RetTy->getFixedSizeInBitsOrZero();
130 if (!RetTy->isSVESizelessType() && (RetTy->isEmptyRecord() || Size == 0))
131 return ArgInfo::getIgnore();
132
133 const Type *Base = nullptr;
134 uint64_t Members = 0;
135 if (isHomogeneousAggregate(Ty: RetTy, Base, Members) &&
136 !(Opts.IsILP32 && IsVariadicFn)) {
137 // Homogeneous Floating-point Aggregates (HFAs) are returned directly.
138 return ArgInfo::getDirect();
139 }
140
141 // In AAPCS return values of a Pure Scalable type are treated as a single
142 // named argument and passed expanded in registers, or indirectly if there are
143 // not enough registers.
144 if (Opts.Kind == AArch64ABIKind::AAPCS) {
145 unsigned NSRN = 0, NPRN = 0;
146 unsigned NVec = 0, NPred = 0;
147 SmallVector<const Type *> UnpaddedCoerceToSeq;
148 if (passAsPureScalableType(Ty: RetTy, NV&: NVec, NP&: NPred, CoerceToSeq&: UnpaddedCoerceToSeq) &&
149 (NVec + NPred) > 0)
150 return coerceAndExpandPureScalableAggregate(
151 Ty: RetTy, /*IsNamedArg=*/true, NVec, NPred, UnpaddedCoerceToSeq, NSRN,
152 NPRN);
153 }
154
155 // Aggregates of at most 16 bytes are returned directly in registers or on
156 // the stack.
157 if (Size <= 128) {
158 if (Size <= 64 && !Opts.IsBigEndian) {
159 // Composite types are returned in lower bits of a 64-bit register for LE,
160 // and in higher bits for BE. However, integer types are always returned
161 // in lower bits for both LE and BE, and they are not rounded up to
162 // 64-bits. We can skip rounding up of composite types for LE, but not for
163 // BE, otherwise composite types will be indistinguishable from integer
164 // types.
165 return ArgInfo::getDirect(
166 T: TB.getIntegerType(BitWidth: Size, Align: llvm::Align(8), /*Signed=*/false));
167 }
168
169 uint64_t AlignBits = RetTy->getAlignment().value() * 8;
170 Size = llvm::alignTo(Value: Size, Align: 64); // round up to multiple of 8 bytes
171
172 // A 16-byte value whose ABI alignment is below 16 bytes is a pair of
173 // i64. A 16-byte-aligned value is an i128.
174 if (AlignBits < 128 && Size == 128) {
175 const Type *I64 = TB.getIntegerType(BitWidth: 64, Align: llvm::Align(8), /*Signed=*/false);
176 return ArgInfo::getDirect(T: TB.getArrayType(ElementType: I64, NumElements: Size / 64, SizeInBits: Size));
177 }
178 return ArgInfo::getDirect(
179 T: TB.getIntegerType(BitWidth: Size, Align: llvm::Align(8), /*Signed=*/false));
180 }
181
182 return getNaturalAlignIndirect(Ty: RetTy, AddrSpace: getAllocaAddrSpace());
183}
184
185ArgInfo AArch64TargetInfo::classifyArgumentType(
186 const Type *Ty, bool IsVariadicFn, bool IsNamedArg,
187 unsigned CallingConvention, unsigned &NSRN, unsigned &NPRN) const {
188 Ty = useFirstFieldIfTransparentUnion(Ty);
189
190 // Arm64EC variadic functions classify their arguments with the x86-64
191 // rules rather than the AArch64 ones.
192 if (IsVariadicFn && Opts.IsWindowsArm64EC) {
193 reportNYI(Feature: "Arm64EC variadic argument handling");
194 return ArgInfo::getIgnore();
195 }
196
197 // Handle illegal vector types here.
198 if (isIllegalVectorType(Ty))
199 return coerceIllegalVector(VT: cast<VectorType>(Val: Ty), NSRN, NPRN);
200
201 if (!passAsAggregateType(Ty)) {
202 if (const auto *IntTy = dyn_cast<IntegerType>(Val: Ty)) {
203 if (IntTy->isBitInt())
204 if (Ty->getSizeInBits().getFixedValue() > 128)
205 return getNaturalAlignIndirect(Ty, AddrSpace: getAllocaAddrSpace(),
206 /*ByVal=*/false);
207
208 if (isPromotableInteger(IT: IntTy) && isDarwinPCS())
209 return ArgInfo::getExtend(T: IntTy);
210 }
211
212 // Predicates and svcount_t are passed in a predicate register. Legal
213 // vectors, SVE data vectors, floating-point types, and opaque 8-bit
214 // floats are passed in a SIMD and floating-point register. A tuple
215 // occupies one register of the appropriate kind per vector it contains.
216 if (const auto *VT = dyn_cast<VectorType>(Val: Ty)) {
217 if (VT->isSVEPredicate() || VT->isSVECount())
218 NPRN = std::min(a: NPRN + 1, b: 4u);
219 else
220 NSRN = std::min(a: NSRN + 1, b: 8u);
221 } else if (const auto *TT = dyn_cast<TupleType>(Val: Ty)) {
222 if (TT->getVectorType()->isSVEPredicate())
223 NPRN = std::min(a: NPRN + TT->getNumVectors(), b: 4u);
224 else
225 NSRN = std::min(a: NSRN + TT->getNumVectors(), b: 8u);
226 } else if (Ty->isFloat() || Ty->isOpaqueFloat8()) {
227 NSRN = std::min(a: NSRN + 1, b: 8u);
228 }
229
230 // Everything not handled above is returned directly.
231 return ArgInfo::getDirect();
232 }
233
234 // Structures with either a non-trivial destructor or a non-trivial
235 // copy constructor are always indirect.
236 if (auto RecordRAA = getRecordArgABI(Ty)) {
237 return getNaturalAlignIndirect(Ty, AddrSpace: getAllocaAddrSpace(),
238 /*ByVal=*/RecordRAA ==
239 RecordArgABI::RAA_DirectInMemory);
240 }
241
242 // AAPCS64 does not say that empty C records are ignored as arguments,
243 // but other compilers do so in certain situations, and we copy that behavior.
244 uint64_t Size = Ty->getFixedSizeInBitsOrZero();
245 if (!Ty->isSVESizelessType() && (Ty->isEmptyRecord() || Size == 0)) {
246 // Darwin overrides the psABI here to ignore all empty records in all modes.
247 // The ABI explicitly says that an empty class shall be treated as if its
248 // type were an aggregate with a single member of type unsigned byte.
249 if (!Opts.IsCXX || isDarwinPCS())
250 return ArgInfo::getIgnore();
251
252 // In C++ mode, arguments which have sizeof() == 0 (which are non-standard
253 // C++) are ignored. This isn't defined by any standard, so we copy GCC's
254 // behaviour here.
255 if (Size == 0)
256 return ArgInfo::getIgnore();
257 }
258
259 // Homogeneous Floating-point Aggregates (HFAs) need to be expanded.
260 const Type *Base = nullptr;
261 uint64_t Members = 0;
262 bool IsWin64 = Opts.Kind == AArch64ABIKind::Win64 ||
263 CallingConvention == llvm::CallingConv::Win64;
264 bool IsWinVariadic = IsWin64 && IsVariadicFn;
265 // In variadic functions on Windows, all composite types are treated alike,
266 // no special handling of HFAs/HVAs.
267 if (!IsWinVariadic && isHomogeneousAggregate(Ty, Base, Members)) {
268 NSRN = std::min(a: NSRN + Members, b: uint64_t(8));
269 uint64_t BaseAllocSizeInBits = Base->getTypeAllocSize().getFixedValue() * 8;
270 const Type *CoerceTy =
271 TB.getArrayType(ElementType: Base, NumElements: Members, SizeInBits: Members * BaseAllocSizeInBits);
272 if (Opts.Kind != AArch64ABIKind::AAPCS)
273 return ArgInfo::getDirect(T: CoerceTy);
274
275 // For HFAs/HVAs, cap the argument alignment to 16, otherwise
276 // set it to 8 according to the AAPCS64 document.
277 unsigned TyAlign = Ty->getUnadjustedAlignment().value();
278 TyAlign = (TyAlign >= 16) ? 16 : 8;
279 return ArgInfo::getDirect(T: CoerceTy, /*Offset=*/0, Align: llvm::Align(TyAlign));
280 }
281
282 // In AAPCS, named arguments of a Pure Scalable Type are passed expanded
283 // in registers, or indirectly if there are not enough registers.
284 if (Opts.Kind == AArch64ABIKind::AAPCS) {
285 unsigned NVec = 0, NPred = 0;
286 SmallVector<const Type *> UnpaddedCoerceToSeq;
287 if (passAsPureScalableType(Ty, NV&: NVec, NP&: NPred, CoerceToSeq&: UnpaddedCoerceToSeq) &&
288 (NVec + NPred) > 0)
289 return coerceAndExpandPureScalableAggregate(
290 Ty, IsNamedArg, NVec, NPred, UnpaddedCoerceToSeq, NSRN, NPRN);
291 }
292
293 // Aggregates of at most 16 bytes are passed directly in registers or on
294 // the stack.
295 if (Size <= 128) {
296 if (Size < 64 && !Opts.IsBigEndian &&
297 Opts.Kind != AArch64ABIKind::DarwinPCS)
298 return ArgInfo::getDirect(
299 T: TB.getIntegerType(BitWidth: Size, Align: llvm::Align(8), /*Signed=*/false));
300
301 uint64_t AlignBits;
302 if (Opts.Kind == AArch64ABIKind::AAPCS) {
303 // An 8-byte slot when the unadjusted alignment is below 16 bytes, and
304 // a 16-byte slot otherwise.
305 AlignBits = Ty->getUnadjustedAlignment().value() * 8;
306 AlignBits = AlignBits < 128 ? 64 : 128;
307 } else {
308 // The larger of the ABI alignment and the pointer width.
309 uint64_t PointerWidth = Opts.IsILP32 ? 32 : 64;
310 AlignBits = std::max(a: Ty->getAlignment().value() * 8, b: PointerWidth);
311 }
312 Size = llvm::alignTo(Value: Size, Align: AlignBits);
313
314 const Type *BaseTy;
315 if ((Size == 64 || Size == 128) && AlignBits == 64 &&
316 containsOnlyPointers(Ty)) {
317 // A record of 64-bit pointers in an 8-byte slot is coerced to a pointer
318 // or an array of pointers.
319 BaseTy = TB.getPointerType(Size: 64, Align: llvm::Align(8));
320 } else {
321 // Every other aggregate is coerced to an integer of the slot width, or
322 // an array of those integers.
323 BaseTy = TB.getIntegerType(BitWidth: AlignBits, Align: llvm::Align(AlignBits / 8),
324 /*Signed=*/false);
325 }
326
327 if (Size == AlignBits)
328 return ArgInfo::getDirect(T: BaseTy);
329 return ArgInfo::getDirect(T: TB.getArrayType(ElementType: BaseTy, NumElements: Size / AlignBits, SizeInBits: Size));
330 }
331
332 return getNaturalAlignIndirect(Ty, AddrSpace: getAllocaAddrSpace(), /*ByVal=*/false);
333}
334
335// True for a record whose bases and fields are plain pointers, or records of
336// such pointers. A plain pointer is a 64-bit pointer with no address-space
337// qualifier. An array field is inspected through its base element. False for an
338// empty record and for a type that is not a record.
339bool AArch64TargetInfo::containsOnlyPointers(const Type *Ty) const {
340 if (Ty->isEmptyRecord())
341 return false;
342
343 const auto *RT = dyn_cast<RecordType>(Val: Ty);
344 if (!RT)
345 return false;
346
347 for (const FieldInfo &Base : RT->getBaseClasses())
348 if (!containsOnlyPointers(Ty: Base.FieldType))
349 return false;
350
351 for (const FieldInfo &Field : RT->getFields()) {
352 const Type *FT = Field.FieldType;
353 while (const auto *AT = dyn_cast<ArrayType>(Val: FT)) {
354 if (AT->isMatrixType())
355 break;
356 FT = AT->getElementType();
357 }
358
359 bool IsPlainPointer = false;
360 if (const auto *PT = dyn_cast<PointerType>(Val: FT))
361 IsPlainPointer = PT->getSizeInBits().getFixedValue() == 64 &&
362 PT->getAddrSpace() == 0 &&
363 !PT->hasAddressSpaceQualifier();
364 if (!IsPlainPointer && !containsOnlyPointers(Ty: FT))
365 return false;
366 }
367 return true;
368}
369
370bool AArch64TargetInfo::passAsAggregateType(const Type *Ty) const {
371 if (Opts.Kind == AArch64ABIKind::AAPCS && Ty->isSVESizelessType()) {
372 // svcount_t and the single-vector types occupy a register of their own,
373 // so only the data and predicate tuples are passed as aggregates.
374 const auto *TupleTy = dyn_cast<TupleType>(Val: Ty);
375 assert((!TupleTy || TupleTy->getNumVectors() > 1) &&
376 "unexpected single vector tuple");
377 return TupleTy && !TupleTy->getVectorType()->isSVECount();
378 }
379 return isAggregateTypeForABI(Ty);
380}
381
382/// Returns the scalable vector type that \p VT, a fixed-length SVE vector,
383/// is passed as. A scalable SVE vector holds 128 bits per granule, so the
384/// scalable element count is 128 divided by the element size, regardless of
385/// how many elements the fixed-length type has.
386const VectorType *AArch64TargetInfo::convertFixedToScalableVectorType(
387 const VectorType *VT) const {
388 if (VT->isFixedLengthSVEPredicate())
389 return TB.getScalablePredicateOrCountVectorType(ABIAlign: Align(2),
390 Kind: VectorKind::SVEPredicate);
391
392 assert(VT->isFixedLengthSVEData() && "expected a fixed-length SVE vector!");
393
394 const Type *EltTy = VT->getElementType();
395 // A vector of opaque 8-bit floats should be mapped as a vector of i8.
396 assert(!EltTy->isOpaqueFloat8() &&
397 "opaque 8-bit float is not a valid vector element type");
398
399 uint64_t EltBits = EltTy->getSizeInBits().getFixedValue();
400 assert(EltBits >= 8 && EltBits <= 64 && isPowerOf2_64(EltBits) &&
401 "unexpected element type for SVE data vector!");
402
403 return TB.getVectorType(ElementType: EltTy, NumElements: ElementCount::getScalable(MinVal: 128 / EltBits),
404 Align: llvm::Align(16), VecKind: VectorKind::SVEData);
405}
406
407ArgInfo AArch64TargetInfo::coerceIllegalVector(const VectorType *VT,
408 unsigned &NSRN,
409 unsigned &NPRN) const {
410 if (VT->isFixedLengthSVEPredicate()) {
411 // Fixed-length predicates are described with 8-bit elements, but they are
412 // passed in a predicate register as a scalable vector of 16 one-bit
413 // elements.
414 assert(isa<IntegerType>(VT->getElementType()) &&
415 VT->getElementType()->getSizeInBits().getFixedValue() == 8 &&
416 "unexpected element type for SVE predicate!");
417 NPRN = std::min(a: NPRN + 1, b: 4u);
418 return ArgInfo::getDirect(T: TB.getScalablePredicateOrCountVectorType(
419 ABIAlign: Align(2), Kind: VectorKind::SVEPredicate));
420 }
421
422 if (VT->isFixedLengthSVEData()) {
423 NSRN = std::min(a: NSRN + 1, b: 8u);
424 return ArgInfo::getDirect(T: convertFixedToScalableVectorType(VT));
425 }
426
427 uint64_t Size = VT->getABISizeInBits();
428 // Android promotes <2 x i8> to i16, not i32
429 if (Opts.IsAndroidOrOHOS && (Size <= 16)) {
430 auto *ResType = TB.getIntegerType(BitWidth: 16, Align: llvm::Align(2), /*Signed=*/false);
431 return ArgInfo::getDirect(T: ResType);
432 }
433 const Type *I32 = TB.getIntegerType(BitWidth: 32, Align: llvm::Align(4), /*Signed=*/false);
434 if (Size <= 32)
435 return ArgInfo::getDirect(T: I32);
436 if (Size == 64) {
437 NSRN = std::min(a: NSRN + 1, b: 8u);
438 return ArgInfo::getDirect(
439 T: TB.getVectorType(ElementType: I32, NumElements: ElementCount::getFixed(MinVal: 2), Align: llvm::Align(8)));
440 }
441 if (Size == 128) {
442 NSRN = std::min(a: NSRN + 1, b: 8u);
443 return ArgInfo::getDirect(
444 T: TB.getVectorType(ElementType: I32, NumElements: ElementCount::getFixed(MinVal: 4), Align: llvm::Align(16)));
445 }
446
447 return getNaturalAlignIndirect(Ty: VT, AddrSpace: getAllocaAddrSpace(), /*ByVal=*/false);
448}
449
450bool AArch64TargetInfo::isIllegalVectorType(const Type *Ty) const {
451 if (const auto *VT = dyn_cast<VectorType>(Val: Ty)) {
452 // Check whether VT is a fixed-length SVE vector. These types are
453 // represented as scalable vectors in function args/return and must be
454 // coerced from fixed vectors.
455 if (VT->isFixedLengthSVEData() || VT->isFixedLengthSVEPredicate())
456 return true;
457
458 // Scalable SVE types are legal.
459 if (VT->isScalable())
460 return false;
461
462 // Check whether VT is legal.
463 unsigned NumElements = VT->getNumElements().getFixedValue();
464 uint64_t Size = VT->getABISizeInBits();
465 // NumElements should be power of 2.
466 if (!isPowerOf2_32(Value: NumElements))
467 return true;
468
469 // arm64_32 has to be compatible with the ARM logic here, which allows huge
470 // vectors for some reason.
471 if (Opts.IsILP32 && Opts.IsMachO)
472 return Size <= 32;
473
474 return Size != 64 && (Size != 128 || NumElements == 1);
475 }
476 return false;
477}
478
479// Expand a memory type into a sequence with an element for each non-record,
480// non-array member of the type, with the exception of the padding types, which
481// are retained.
482void AArch64TargetInfo::flattenType(
483 const Type *Ty, SmallVectorImpl<const Type *> &Flattened) const {
484 if (ArgInfo::isPaddingForCoerceAndExpand(Ty)) {
485 Flattened.push_back(Elt: Ty);
486 return;
487 }
488
489 if (const auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
490 uint64_t NElt = AT->getNumElements();
491 if (NElt == 0)
492 return;
493
494 SmallVector<const Type *, 4> EltFlattened;
495 flattenType(Ty: AT->getElementType(), Flattened&: EltFlattened);
496
497 for (uint64_t I = 0; I < NElt; ++I)
498 llvm::append_range(C&: Flattened, R&: EltFlattened);
499 return;
500 }
501
502 if (const auto *RT = dyn_cast<RecordType>(Val: Ty)) {
503 for (const FieldInfo &Field : RT->getFields())
504 flattenType(Ty: Field.FieldType, Flattened);
505 return;
506 }
507
508 Flattened.push_back(Elt: Ty);
509}
510
511ArgInfo AArch64TargetInfo::coerceAndExpandPureScalableAggregate(
512 const Type *Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
513 const SmallVectorImpl<const Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
514 unsigned &NPRN) const {
515 // An unnamed argument, or one that does not fit in the remaining Z or P
516 // registers, is passed indirectly and does not consume those registers.
517 if (!IsNamedArg || NSRN + NVec > 8 || NPRN + NPred > 4)
518 return getNaturalAlignIndirect(Ty, AddrSpace: getAllocaAddrSpace(), /*ByVal=*/false);
519
520 NSRN += NVec;
521 NPRN += NPred;
522
523 // A sizeless SVE tuple is already one register per member.
524 if (Ty->isSVESizelessType())
525 return ArgInfo::getDirect();
526
527 assert(!UnpaddedCoerceToSeq.empty() && "pure scalable type has no members");
528 const Type *UnpaddedCoerceToType =
529 UnpaddedCoerceToSeq.size() == 1
530 ? UnpaddedCoerceToSeq[0]
531 : getStructOfTypes(Elems: UnpaddedCoerceToSeq, /*Packed=*/true);
532
533 SmallVector<const Type *, 8> CoerceToSeq;
534 flattenType(Ty: convertTypeForMem(Ty), Flattened&: CoerceToSeq);
535 return ArgInfo::getCoerceAndExpand(
536 CoerceToType: getStructOfTypes(Elems: CoerceToSeq, /*Packed=*/false), UnpaddedCoerceToType);
537}
538
539// A Pure Scalable Type (AAPCS64) is passed in Z and P registers. On success
540// NVec and NPred are how many of each it needs, and CoerceToSeq has one
541// scalable vector per register. A sequence longer than 12 is rejected so the
542// caller treats the type as a large composite.
543bool AArch64TargetInfo::passAsPureScalableType(
544 const Type *Ty, unsigned &NVec, unsigned &NPred,
545 SmallVectorImpl<const Type *> &CoerceToSeq) const {
546 if (const auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
547 if (AT->isMatrixType())
548 return false;
549
550 uint64_t NElt = AT->getNumElements();
551 if (NElt == 0)
552 return false;
553
554 unsigned NV = 0, NP = 0;
555 SmallVector<const Type *, 4> EltCoerceToSeq;
556 if (!passAsPureScalableType(Ty: AT->getElementType(), NVec&: NV, NPred&: NP, CoerceToSeq&: EltCoerceToSeq))
557 return false;
558
559 if (CoerceToSeq.size() + NElt * EltCoerceToSeq.size() > 12)
560 return false;
561
562 for (uint64_t I = 0; I < NElt; ++I)
563 llvm::append_range(C&: CoerceToSeq, R&: EltCoerceToSeq);
564
565 NVec += NElt * NV;
566 NPred += NElt * NP;
567 return true;
568 }
569
570 if (const auto *RT = dyn_cast<RecordType>(Val: Ty)) {
571 if (getRecordArgABI(RT) != RAA_Default)
572 return false;
573 // Pure scalable types are never unions and never contain unions.
574 if (RT->isUnion())
575 return false;
576
577 // A flexible array member is lowered as a zero-length array, which the
578 // field walk below skips. The member disqualifies a pure scalable type.
579 if (RT->hasFlexibleArrayMember())
580 return false;
581
582 // Direct virtual bases are not in getBaseClasses(). A record that has
583 // one cannot be passed in registers, and getRecordArgABI rejected it
584 // above.
585 for (const FieldInfo &Base : RT->getBaseClasses()) {
586 if (Base.FieldType->isEmptyRecord())
587 continue;
588 if (!passAsPureScalableType(Ty: Base.FieldType, NVec, NPred, CoerceToSeq))
589 return false;
590 }
591 for (const FieldInfo &Field : RT->getFields()) {
592 if (Field.isEmpty())
593 continue;
594 if (!passAsPureScalableType(Ty: Field.FieldType, NVec, NPred, CoerceToSeq))
595 return false;
596 }
597 return true;
598 }
599
600 if (const auto *TT = dyn_cast<TupleType>(Val: Ty)) {
601 const VectorType *VT = TT->getVectorType();
602 if (!VT->isScalable() || VT->isSVECount())
603 return false;
604
605 unsigned N = TT->getNumVectors();
606 if (CoerceToSeq.size() + N > 12)
607 return false;
608
609 bool IsPred = VT->isSVEPredicate();
610 if (!IsPred && !VT->isSVEData())
611 return false;
612 if (IsPred)
613 NPred += N;
614 else
615 NVec += N;
616
617 for (unsigned I = 0; I < N; ++I)
618 CoerceToSeq.push_back(Elt: VT);
619 return true;
620 }
621
622 if (const auto *VT = dyn_cast<VectorType>(Val: Ty)) {
623 const Type *Coerced = nullptr;
624 bool IsPred = false;
625 if (VT->isFixedLengthSVEPredicate()) {
626 IsPred = true;
627 Coerced = convertFixedToScalableVectorType(VT);
628 } else if (VT->isFixedLengthSVEData()) {
629 Coerced = convertFixedToScalableVectorType(VT);
630 } else if (VT->isScalable() && VT->isSVEPredicate()) {
631 IsPred = true;
632 Coerced = VT;
633 } else if (VT->isScalable() && VT->isSVEData()) {
634 Coerced = VT;
635 } else {
636 return false;
637 }
638
639 if (CoerceToSeq.size() + 1 > 12)
640 return false;
641
642 if (IsPred)
643 ++NPred;
644 else
645 ++NVec;
646 CoerceToSeq.push_back(Elt: Coerced);
647 return true;
648 }
649
650 return false;
651}
652
653bool AArch64TargetInfo::isHomogeneousAggregateBaseType(const Type *Ty) const {
654 // Soft-float ABI: no types are homogeneous aggregates.
655 if (isSoftFloat())
656 return false;
657
658 // Homogeneous aggregates for AAPCS64 must have base types of a floating
659 // point type or a short-vector type.
660 if (Ty->isFloat())
661 return true;
662
663 if (const auto *VT = dyn_cast<VectorType>(Val: Ty)) {
664 if (VT->isScalable() || VT->isSVEData() || VT->isSVEPredicate())
665 return false;
666
667 uint64_t VecSize = VT->getABISizeInBits();
668 if (VecSize == 64 || VecSize == 128)
669 return true;
670 }
671 return false;
672}
673
674bool AArch64TargetInfo::isHomogeneousAggregateSmallEnough(
675 const Type * /*Base*/, uint64_t Members) const {
676 return Members <= 4;
677}
678
679bool AArch64TargetInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate()
680 const {
681 // AAPCS64 applies homogeneity to the output of the data layout decision, so
682 // zero-length bitfields do not affect homogeneity.
683 return true;
684}
685
686// MSVC on Windows on Arm64 excludes some C++ records from being HFAs that
687// AAPCS64 would otherwise permit. These are the rules observed from MSVC.
688static bool isMSVCPermittedHomogeneousAggregate(const RecordType *RT) {
689 if (RT->isCXXEmpty() || RT->isPolymorphic() ||
690 RT->hasNonTrivialCopyAssignment() || RT->hasNonTrivialDestructor() ||
691 RT->hasNonTrivialDefaultConstructor())
692 return false;
693
694 auto IsPermittedBase = [](const FieldInfo &Base) {
695 const auto *BaseRT = dyn_cast<RecordType>(Val: Base.FieldType);
696 return !BaseRT || isMSVCPermittedHomogeneousAggregate(RT: BaseRT);
697 };
698 return all_of(Range: RT->getBaseClasses(), P: IsPermittedBase) &&
699 all_of(Range: RT->getVirtualBaseClasses(), P: IsPermittedBase);
700}
701
702bool AArch64TargetInfo::isPermittedToBeHomogeneousAggregate(
703 const RecordType *RT) const {
704 if (Opts.IsMicrosoftCXXABI && RT->isCXXRecord())
705 return isMSVCPermittedHomogeneousAggregate(RT);
706 return true;
707}
708
709} // namespace abi
710} // namespace llvm
711