1//===- AArch64.cpp --------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ABIInfoImpl.h"
10#include "TargetInfo.h"
11#include "clang/AST/Decl.h"
12#include "clang/Basic/DiagnosticFrontend.h"
13#include "llvm/TargetParser/AArch64TargetParser.h"
14
15using namespace clang;
16using namespace clang::CodeGen;
17
18//===----------------------------------------------------------------------===//
19// AArch64 ABI Implementation
20//===----------------------------------------------------------------------===//
21
22namespace {
23
24class AArch64ABIInfo : public ABIInfo {
25 AArch64ABIKind Kind;
26
27 std::unique_ptr<TargetCodeGenInfo> WinX86_64CodegenInfo;
28
29public:
30 AArch64ABIInfo(CodeGenModule &CGM, AArch64ABIKind Kind)
31 : ABIInfo(CGM.getTypes()), Kind(Kind) {
32 if (getTarget().getTriple().isWindowsArm64EC()) {
33 WinX86_64CodegenInfo =
34 createWinX86_64TargetCodeGenInfo(CGM, AVXLevel: X86AVXABILevel::None);
35 }
36 }
37
38 bool isSoftFloat() const { return Kind == AArch64ABIKind::AAPCSSoft; }
39
40private:
41 AArch64ABIKind getABIKind() const { return Kind; }
42 bool isDarwinPCS() const { return Kind == AArch64ABIKind::DarwinPCS; }
43
44 ABIArgInfo classifyReturnType(QualType RetTy, bool IsVariadicFn) const;
45 ABIArgInfo classifyArgumentType(QualType RetTy, bool IsVariadicFn,
46 bool IsNamedArg, unsigned CallingConvention,
47 unsigned &NSRN, unsigned &NPRN) const;
48 llvm::Type *convertFixedToScalableVectorType(const VectorType *VT) const;
49 ABIArgInfo coerceIllegalVector(QualType Ty, unsigned &NSRN,
50 unsigned &NPRN) const;
51 ABIArgInfo coerceAndExpandPureScalableAggregate(
52 QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
53 const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
54 unsigned &NPRN) const;
55 bool isHomogeneousAggregateBaseType(QualType Ty) const override;
56 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
57 uint64_t Members) const override;
58 bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const override;
59
60 bool isIllegalVectorType(QualType Ty) const;
61
62 bool passAsAggregateType(QualType Ty) const;
63 bool passAsPureScalableType(QualType Ty, unsigned &NV, unsigned &NP,
64 SmallVectorImpl<llvm::Type *> &CoerceToSeq) const;
65
66 void flattenType(llvm::Type *Ty,
67 SmallVectorImpl<llvm::Type *> &Flattened) const;
68
69 void computeInfo(CGFunctionInfo &FI) const override {
70 if (!::classifyReturnType(CXXABI: getCXXABI(), FI, Info: *this))
71 FI.getReturnInfo() =
72 classifyReturnType(RetTy: FI.getReturnType(), IsVariadicFn: FI.isVariadic());
73
74 unsigned ArgNo = 0;
75 unsigned NSRN = 0, NPRN = 0;
76 for (auto &it : FI.arguments()) {
77 const bool IsNamedArg =
78 !FI.isVariadic() || ArgNo < FI.getRequiredArgs().getNumRequiredArgs();
79 ++ArgNo;
80 it.info = classifyArgumentType(RetTy: it.type, IsVariadicFn: FI.isVariadic(), IsNamedArg,
81 CallingConvention: FI.getCallingConvention(), NSRN, NPRN);
82 }
83 }
84
85 RValue EmitDarwinVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,
86 AggValueSlot Slot) const;
87
88 RValue EmitAAPCSVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,
89 AArch64ABIKind Kind, AggValueSlot Slot) const;
90
91 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
92 AggValueSlot Slot) const override {
93 llvm::Type *BaseTy = CGF.ConvertType(T: Ty);
94 if (isa<llvm::ScalableVectorType>(Val: BaseTy))
95 llvm::report_fatal_error(reason: "Passing SVE types to variadic functions is "
96 "currently not supported");
97
98 return Kind == AArch64ABIKind::Win64
99 ? EmitMSVAArg(CGF, VAListAddr, Ty, Slot)
100 : isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF, Slot)
101 : EmitAAPCSVAArg(VAListAddr, Ty, CGF, Kind, Slot);
102 }
103
104 RValue EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
105 AggValueSlot Slot) const override;
106
107 bool allowBFloatArgsAndRet() const override {
108 return getTarget().hasBFloat16Type();
109 }
110
111 using ABIInfo::appendAttributeMangling;
112 void appendAttributeMangling(TargetClonesAttr *Attr, unsigned Index,
113 raw_ostream &Out) const override;
114 void appendAttributeMangling(StringRef AttrStr,
115 raw_ostream &Out) const override;
116};
117
118class AArch64SwiftABIInfo : public SwiftABIInfo {
119public:
120 explicit AArch64SwiftABIInfo(CodeGenTypes &CGT)
121 : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/true) {}
122
123 bool isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy,
124 unsigned NumElts) const override;
125};
126
127class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
128public:
129 AArch64TargetCodeGenInfo(CodeGenModule &CGM, AArch64ABIKind Kind)
130 : TargetCodeGenInfo(std::make_unique<AArch64ABIInfo>(args&: CGM, args&: Kind)) {
131 SwiftInfo = std::make_unique<AArch64SwiftABIInfo>(args&: CGM.getTypes());
132 }
133
134 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
135 return "mov\tfp, fp\t\t// marker for objc_retainAutoreleaseReturnValue";
136 }
137
138 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
139 return 31;
140 }
141
142 bool doesReturnSlotInterfereWithArgs() const override { return false; }
143
144 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
145 CodeGen::CodeGenModule &CGM) const override {
146 auto *Fn = dyn_cast<llvm::Function>(Val: GV);
147 if (!Fn)
148 return;
149
150 const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
151 TargetInfo::BranchProtectionInfo BPI(CGM.getLangOpts());
152
153 if (FD && FD->hasAttr<TargetAttr>()) {
154 const auto *TA = FD->getAttr<TargetAttr>();
155 ParsedTargetAttr Attr =
156 CGM.getTarget().parseTargetAttr(Str: TA->getFeaturesStr());
157 if (!Attr.BranchProtection.empty()) {
158 StringRef Error;
159 (void)CGM.getTarget().validateBranchProtection(
160 Spec: Attr.BranchProtection, Arch: Attr.CPU, BPI, LO: CGM.getLangOpts(), Err&: Error);
161 assert(Error.empty());
162 }
163 }
164 setBranchProtectionFnAttributes(BPI, F&: *Fn);
165 setPointerAuthFnAttributes(Opts: CGM.getCodeGenOpts().PointerAuth, F&: *Fn);
166 }
167
168 bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF,
169 llvm::Type *Ty) const override {
170 if (CGF.getTarget().hasFeature(Feature: "ls64")) {
171 auto *ST = dyn_cast<llvm::StructType>(Val: Ty);
172 if (ST && ST->getNumElements() == 1) {
173 auto *AT = dyn_cast<llvm::ArrayType>(Val: ST->getElementType(N: 0));
174 if (AT && AT->getNumElements() == 8 &&
175 AT->getElementType()->isIntegerTy(Bitwidth: 64))
176 return true;
177 }
178 }
179 return TargetCodeGenInfo::isScalarizableAsmOperand(CGF, Ty);
180 }
181
182 void checkFunctionABI(CodeGenModule &CGM,
183 const FunctionDecl *Decl) const override;
184
185 void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
186 const FunctionDecl *Caller,
187 const FunctionDecl *Callee, const CallArgList &Args,
188 QualType ReturnType) const override;
189
190 bool wouldInliningViolateFunctionCallABI(
191 const FunctionDecl *Caller, const FunctionDecl *Callee) const override;
192
193private:
194 // Diagnose calls between functions with incompatible Streaming SVE
195 // attributes.
196 void checkFunctionCallABIStreaming(CodeGenModule &CGM, SourceLocation CallLoc,
197 const FunctionDecl *Caller,
198 const FunctionDecl *Callee) const;
199 // Diagnose calls which must pass arguments in floating-point registers when
200 // the selected target does not have floating-point registers.
201 void checkFunctionCallABISoftFloat(CodeGenModule &CGM, SourceLocation CallLoc,
202 const FunctionDecl *Caller,
203 const FunctionDecl *Callee,
204 const CallArgList &Args,
205 QualType ReturnType) const;
206};
207
208class WindowsAArch64TargetCodeGenInfo : public AArch64TargetCodeGenInfo {
209public:
210 WindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM, AArch64ABIKind K)
211 : AArch64TargetCodeGenInfo(CGM, K) {}
212
213 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
214 CodeGen::CodeGenModule &CGM) const override;
215
216 void getDependentLibraryOption(llvm::StringRef Lib,
217 llvm::SmallString<24> &Opt) const override {
218 Opt = "/DEFAULTLIB:" + qualifyWindowsLibrary(Lib);
219 }
220
221 void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value,
222 llvm::SmallString<32> &Opt) const override {
223 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
224 }
225};
226
227void WindowsAArch64TargetCodeGenInfo::setTargetAttributes(
228 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
229 AArch64TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
230 if (GV->isDeclaration())
231 return;
232 addStackProbeTargetAttributes(D, GV, CGM);
233}
234}
235
236llvm::Type *
237AArch64ABIInfo::convertFixedToScalableVectorType(const VectorType *VT) const {
238 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
239
240 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
241 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
242 BuiltinType::UChar &&
243 "unexpected builtin type for SVE predicate!");
244 return llvm::ScalableVectorType::get(ElementType: llvm::Type::getInt1Ty(C&: getVMContext()),
245 MinNumElts: 16);
246 }
247
248 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
249 const auto *BT = VT->getElementType()->castAs<BuiltinType>();
250 switch (BT->getKind()) {
251 default:
252 llvm_unreachable("unexpected builtin type for SVE vector!");
253
254 case BuiltinType::SChar:
255 case BuiltinType::UChar:
256 case BuiltinType::MFloat8:
257 return llvm::ScalableVectorType::get(
258 ElementType: llvm::Type::getInt8Ty(C&: getVMContext()), MinNumElts: 16);
259
260 case BuiltinType::Short:
261 case BuiltinType::UShort:
262 return llvm::ScalableVectorType::get(
263 ElementType: llvm::Type::getInt16Ty(C&: getVMContext()), MinNumElts: 8);
264
265 case BuiltinType::Int:
266 case BuiltinType::UInt:
267 return llvm::ScalableVectorType::get(
268 ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), MinNumElts: 4);
269
270 case BuiltinType::Long:
271 case BuiltinType::ULong:
272 return llvm::ScalableVectorType::get(
273 ElementType: llvm::Type::getInt64Ty(C&: getVMContext()), MinNumElts: 2);
274
275 case BuiltinType::Half:
276 return llvm::ScalableVectorType::get(
277 ElementType: llvm::Type::getHalfTy(C&: getVMContext()), MinNumElts: 8);
278
279 case BuiltinType::Float:
280 return llvm::ScalableVectorType::get(
281 ElementType: llvm::Type::getFloatTy(C&: getVMContext()), MinNumElts: 4);
282
283 case BuiltinType::Double:
284 return llvm::ScalableVectorType::get(
285 ElementType: llvm::Type::getDoubleTy(C&: getVMContext()), MinNumElts: 2);
286
287 case BuiltinType::BFloat16:
288 return llvm::ScalableVectorType::get(
289 ElementType: llvm::Type::getBFloatTy(C&: getVMContext()), MinNumElts: 8);
290 }
291 }
292
293 llvm_unreachable("expected fixed-length SVE vector");
294}
295
296ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty, unsigned &NSRN,
297 unsigned &NPRN) const {
298 assert(Ty->isVectorType() && "expected vector type!");
299
300 const auto *VT = Ty->castAs<VectorType>();
301 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
302 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
303 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
304 BuiltinType::UChar &&
305 "unexpected builtin type for SVE predicate!");
306 NPRN = std::min(a: NPRN + 1, b: 4u);
307 return ABIArgInfo::getDirect(T: llvm::ScalableVectorType::get(
308 ElementType: llvm::Type::getInt1Ty(C&: getVMContext()), MinNumElts: 16));
309 }
310
311 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
312 NSRN = std::min(a: NSRN + 1, b: 8u);
313 return ABIArgInfo::getDirect(T: convertFixedToScalableVectorType(VT));
314 }
315
316 uint64_t Size = getContext().getTypeSize(T: Ty);
317 // Android promotes <2 x i8> to i16, not i32
318 if ((isAndroid() || isOHOSFamily()) && (Size <= 16)) {
319 llvm::Type *ResType = llvm::Type::getInt16Ty(C&: getVMContext());
320 return ABIArgInfo::getDirect(T: ResType);
321 }
322 if (Size <= 32) {
323 llvm::Type *ResType = llvm::Type::getInt32Ty(C&: getVMContext());
324 return ABIArgInfo::getDirect(T: ResType);
325 }
326 if (Size == 64) {
327 NSRN = std::min(a: NSRN + 1, b: 8u);
328 auto *ResType =
329 llvm::FixedVectorType::get(ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), NumElts: 2);
330 return ABIArgInfo::getDirect(T: ResType);
331 }
332 if (Size == 128) {
333 NSRN = std::min(a: NSRN + 1, b: 8u);
334 auto *ResType =
335 llvm::FixedVectorType::get(ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), NumElts: 4);
336 return ABIArgInfo::getDirect(T: ResType);
337 }
338
339 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
340 /*ByVal=*/false);
341}
342
343ABIArgInfo AArch64ABIInfo::coerceAndExpandPureScalableAggregate(
344 QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
345 const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
346 unsigned &NPRN) const {
347 if (!IsNamedArg || NSRN + NVec > 8 || NPRN + NPred > 4)
348 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
349 /*ByVal=*/false);
350 NSRN += NVec;
351 NPRN += NPred;
352
353 // Handle SVE vector tuples.
354 if (Ty->isSVESizelessBuiltinType())
355 return ABIArgInfo::getDirect();
356
357 llvm::Type *UnpaddedCoerceToType =
358 UnpaddedCoerceToSeq.size() == 1
359 ? UnpaddedCoerceToSeq[0]
360 : llvm::StructType::get(Context&: CGT.getLLVMContext(), Elements: UnpaddedCoerceToSeq,
361 isPacked: true);
362
363 SmallVector<llvm::Type *> CoerceToSeq;
364 flattenType(Ty: CGT.ConvertType(T: Ty), Flattened&: CoerceToSeq);
365 auto *CoerceToType =
366 llvm::StructType::get(Context&: CGT.getLLVMContext(), Elements: CoerceToSeq, isPacked: false);
367
368 return ABIArgInfo::getCoerceAndExpand(coerceToType: CoerceToType, unpaddedCoerceToType: UnpaddedCoerceToType);
369}
370
371ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadicFn,
372 bool IsNamedArg,
373 unsigned CallingConvention,
374 unsigned &NSRN,
375 unsigned &NPRN) const {
376 Ty = useFirstFieldIfTransparentUnion(Ty);
377
378 if (IsVariadicFn && getTarget().getTriple().isWindowsArm64EC()) {
379 // Arm64EC varargs functions use the x86_64 classification rules,
380 // not the AArch64 ABI rules.
381 return WinX86_64CodegenInfo->getABIInfo().classifyArgForArm64ECVarArg(Ty);
382 }
383
384 // Handle illegal vector types here.
385 if (isIllegalVectorType(Ty))
386 return coerceIllegalVector(Ty, NSRN, NPRN);
387
388 if (!passAsAggregateType(Ty)) {
389 // Treat an enum type as its underlying type.
390 if (const auto *ED = Ty->getAsEnumDecl())
391 Ty = ED->getIntegerType();
392
393 if (const auto *EIT = Ty->getAs<BitIntType>())
394 if (EIT->getNumBits() > 128)
395 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
396 ByVal: false);
397
398 if (Ty->isVectorType())
399 NSRN = std::min(a: NSRN + 1, b: 8u);
400 else if (const auto *BT = Ty->getAs<BuiltinType>()) {
401 if (BT->isFloatingPoint())
402 NSRN = std::min(a: NSRN + 1, b: 8u);
403 else {
404 switch (BT->getKind()) {
405 case BuiltinType::SveBool:
406 case BuiltinType::SveCount:
407 NPRN = std::min(a: NPRN + 1, b: 4u);
408 break;
409 case BuiltinType::SveBoolx2:
410 NPRN = std::min(a: NPRN + 2, b: 4u);
411 break;
412 case BuiltinType::SveBoolx4:
413 NPRN = std::min(a: NPRN + 4, b: 4u);
414 break;
415 default:
416 if (BT->isSVESizelessBuiltinType())
417 NSRN = std::min(
418 a: NSRN + getContext().getBuiltinVectorTypeInfo(VecTy: BT).NumVectors,
419 b: 8u);
420 }
421 }
422 }
423
424 return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS()
425 ? ABIArgInfo::getExtend(Ty, T: CGT.ConvertType(T: Ty))
426 : ABIArgInfo::getDirect());
427 }
428
429 // Structures with either a non-trivial destructor or a non-trivial
430 // copy constructor are always indirect.
431 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(T: Ty, CXXABI&: getCXXABI())) {
432 return getNaturalAlignIndirect(
433 Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
434 /*ByVal=*/RAA == CGCXXABI::RAA_DirectInMemory);
435 }
436
437 // Empty records:
438 // AAPCS64 does not say that empty records are ignored as arguments,
439 // but other compilers do so in certain situations, and we copy that behavior.
440 // Those situations are in fact language-mode-specific, which seems really
441 // unfortunate, but it's something we just have to accept. If this doesn't
442 // apply, just fall through to the standard argument-handling path.
443 // Darwin overrides the psABI here to ignore all empty records in all modes.
444 uint64_t Size = getContext().getTypeSize(T: Ty);
445 bool IsEmpty = isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true);
446 if (!Ty->isSVESizelessBuiltinType() && (IsEmpty || Size == 0)) {
447 // Empty records are ignored in C mode, and in C++ on Darwin.
448 if (!getContext().getLangOpts().CPlusPlus || isDarwinPCS())
449 return ABIArgInfo::getIgnore();
450
451 // In C++ mode, arguments which have sizeof() == 0 (which are non-standard
452 // C++) are ignored. This isn't defined by any standard, so we copy GCC's
453 // behaviour here.
454 if (Size == 0)
455 return ABIArgInfo::getIgnore();
456 }
457
458 // Homogeneous Floating-point Aggregates (HFAs) need to be expanded.
459 const Type *Base = nullptr;
460 uint64_t Members = 0;
461 bool IsWin64 = Kind == AArch64ABIKind::Win64 ||
462 CallingConvention == llvm::CallingConv::Win64;
463 bool IsWinVariadic = IsWin64 && IsVariadicFn;
464 // In variadic functions on Windows, all composite types are treated alike,
465 // no special handling of HFAs/HVAs.
466 if (!IsWinVariadic && isHomogeneousAggregate(Ty, Base, Members)) {
467 NSRN = std::min(a: NSRN + Members, b: uint64_t(8));
468 if (Kind != AArch64ABIKind::AAPCS)
469 return ABIArgInfo::getDirect(
470 T: llvm::ArrayType::get(ElementType: CGT.ConvertType(T: QualType(Base, 0)), NumElements: Members));
471
472 // For HFAs/HVAs, cap the argument alignment to 16, otherwise
473 // set it to 8 according to the AAPCS64 document.
474 unsigned Align =
475 getContext().getTypeUnadjustedAlignInChars(T: Ty).getQuantity();
476 Align = (Align >= 16) ? 16 : 8;
477 return ABIArgInfo::getDirect(
478 T: llvm::ArrayType::get(ElementType: CGT.ConvertType(T: QualType(Base, 0)), NumElements: Members), Offset: 0,
479 Padding: nullptr, CanBeFlattened: true, Align);
480 }
481
482 // In AAPCS named arguments of a Pure Scalable Type are passed expanded in
483 // registers, or indirectly if there are not enough registers.
484 if (Kind == AArch64ABIKind::AAPCS) {
485 unsigned NVec = 0, NPred = 0;
486 SmallVector<llvm::Type *> UnpaddedCoerceToSeq;
487 if (passAsPureScalableType(Ty, NV&: NVec, NP&: NPred, CoerceToSeq&: UnpaddedCoerceToSeq) &&
488 (NVec + NPred) > 0)
489 return coerceAndExpandPureScalableAggregate(
490 Ty, IsNamedArg, NVec, NPred, UnpaddedCoerceToSeq, NSRN, NPRN);
491 }
492
493 // Aggregates <= 16 bytes are passed directly in registers or on the stack.
494 if (Size <= 128) {
495 unsigned Alignment;
496 if (Kind == AArch64ABIKind::AAPCS) {
497 Alignment = getContext().getTypeUnadjustedAlign(T: Ty);
498 Alignment = Alignment < 128 ? 64 : 128;
499 } else {
500 Alignment =
501 std::max(a: getContext().getTypeAlign(T: Ty),
502 b: (unsigned)getTarget().getPointerWidth(AddrSpace: LangAS::Default));
503 }
504 Size = llvm::alignTo(Value: Size, Align: Alignment);
505
506 // If the Aggregate is made up of pointers, use an array of pointers for the
507 // coerced type. This prevents having to convert ptr2int->int2ptr through
508 // the call, allowing alias analysis to produce better code.
509 auto ContainsOnlyPointers = [&](const auto &Self, QualType Ty) {
510 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
511 return false;
512 const auto *RD = Ty->getAsRecordDecl();
513 if (!RD)
514 return false;
515 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
516 for (const auto &I : CXXRD->bases())
517 if (!Self(Self, I.getType()))
518 return false;
519 }
520 return all_of(RD->fields(), [&](FieldDecl *FD) {
521 QualType FDTy = FD->getType();
522 if (FDTy->isArrayType())
523 FDTy = getContext().getBaseElementType(QT: FDTy);
524 return (FDTy->isPointerOrReferenceType() &&
525 getContext().getTypeSize(T: FDTy) == 64 &&
526 !FDTy->getPointeeType().hasAddressSpace()) ||
527 Self(Self, FDTy);
528 });
529 };
530
531 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
532 // For aggregates with 16-byte alignment, we use i128.
533 llvm::Type *BaseTy = llvm::Type::getIntNTy(C&: getVMContext(), N: Alignment);
534 if ((Size == 64 || Size == 128) && Alignment == 64 &&
535 ContainsOnlyPointers(ContainsOnlyPointers, Ty))
536 BaseTy = llvm::PointerType::getUnqual(C&: getVMContext());
537 return ABIArgInfo::getDirect(
538 T: Size == Alignment ? BaseTy
539 : llvm::ArrayType::get(ElementType: BaseTy, NumElements: Size / Alignment));
540 }
541
542 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
543 /*ByVal=*/false);
544}
545
546ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,
547 bool IsVariadicFn) const {
548 if (RetTy->isVoidType())
549 return ABIArgInfo::getIgnore();
550
551 if (const auto *VT = RetTy->getAs<VectorType>()) {
552 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
553 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
554 unsigned NSRN = 0, NPRN = 0;
555 return coerceIllegalVector(Ty: RetTy, NSRN, NPRN);
556 }
557 }
558
559 // Large vector types should be returned via memory.
560 if (RetTy->isVectorType() && getContext().getTypeSize(T: RetTy) > 128)
561 return getNaturalAlignIndirect(Ty: RetTy, AddrSpace: getDataLayout().getAllocaAddrSpace());
562
563 if (!passAsAggregateType(Ty: RetTy)) {
564 // Treat an enum type as its underlying type.
565 if (const auto *ED = RetTy->getAsEnumDecl())
566 RetTy = ED->getIntegerType();
567
568 if (const auto *EIT = RetTy->getAs<BitIntType>())
569 if (EIT->getNumBits() > 128)
570 return getNaturalAlignIndirect(Ty: RetTy,
571 AddrSpace: getDataLayout().getAllocaAddrSpace());
572
573 return (isPromotableIntegerTypeForABI(Ty: RetTy) && isDarwinPCS()
574 ? ABIArgInfo::getExtend(Ty: RetTy)
575 : ABIArgInfo::getDirect());
576 }
577
578 uint64_t Size = getContext().getTypeSize(T: RetTy);
579 if (!RetTy->isSVESizelessBuiltinType() &&
580 (isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true) || Size == 0))
581 return ABIArgInfo::getIgnore();
582
583 const Type *Base = nullptr;
584 uint64_t Members = 0;
585 if (isHomogeneousAggregate(Ty: RetTy, Base, Members) &&
586 !(getTarget().getTriple().getArch() == llvm::Triple::aarch64_32 &&
587 IsVariadicFn))
588 // Homogeneous Floating-point Aggregates (HFAs) are returned directly.
589 return ABIArgInfo::getDirect();
590
591 // In AAPCS return values of a Pure Scalable type are treated as a single
592 // named argument and passed expanded in registers, or indirectly if there are
593 // not enough registers.
594 if (Kind == AArch64ABIKind::AAPCS) {
595 unsigned NSRN = 0, NPRN = 0;
596 unsigned NVec = 0, NPred = 0;
597 SmallVector<llvm::Type *> UnpaddedCoerceToSeq;
598 if (passAsPureScalableType(Ty: RetTy, NV&: NVec, NP&: NPred, CoerceToSeq&: UnpaddedCoerceToSeq) &&
599 (NVec + NPred) > 0)
600 return coerceAndExpandPureScalableAggregate(
601 Ty: RetTy, /* IsNamedArg */ true, NVec, NPred, UnpaddedCoerceToSeq, NSRN,
602 NPRN);
603 }
604
605 // Aggregates <= 16 bytes are returned directly in registers or on the stack.
606 if (Size <= 128) {
607 if (Size <= 64 && getDataLayout().isLittleEndian()) {
608 // Composite types are returned in lower bits of a 64-bit register for LE,
609 // and in higher bits for BE. However, integer types are always returned
610 // in lower bits for both LE and BE, and they are not rounded up to
611 // 64-bits. We can skip rounding up of composite types for LE, but not for
612 // BE, otherwise composite types will be indistinguishable from integer
613 // types.
614 return ABIArgInfo::getDirect(
615 T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Size));
616 }
617
618 unsigned Alignment = getContext().getTypeAlign(T: RetTy);
619 Size = llvm::alignTo(Value: Size, Align: 64); // round up to multiple of 8 bytes
620
621 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
622 // For aggregates with 16-byte alignment, we use i128.
623 if (Alignment < 128 && Size == 128) {
624 llvm::Type *BaseTy = llvm::Type::getInt64Ty(C&: getVMContext());
625 return ABIArgInfo::getDirect(T: llvm::ArrayType::get(ElementType: BaseTy, NumElements: Size / 64));
626 }
627 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Size));
628 }
629
630 return getNaturalAlignIndirect(Ty: RetTy, AddrSpace: getDataLayout().getAllocaAddrSpace());
631}
632
633/// isIllegalVectorType - check whether the vector type is legal for AArch64.
634bool AArch64ABIInfo::isIllegalVectorType(QualType Ty) const {
635 if (const VectorType *VT = Ty->getAs<VectorType>()) {
636 // Check whether VT is a fixed-length SVE vector. These types are
637 // represented as scalable vectors in function args/return and must be
638 // coerced from fixed vectors.
639 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
640 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate)
641 return true;
642
643 // Check whether VT is legal.
644 unsigned NumElements = VT->getNumElements();
645 uint64_t Size = getContext().getTypeSize(T: VT);
646 // NumElements should be power of 2.
647 if (!llvm::isPowerOf2_32(Value: NumElements))
648 return true;
649
650 // arm64_32 has to be compatible with the ARM logic here, which allows huge
651 // vectors for some reason.
652 llvm::Triple Triple = getTarget().getTriple();
653 if (Triple.getArch() == llvm::Triple::aarch64_32 &&
654 Triple.isOSBinFormatMachO())
655 return Size <= 32;
656
657 return Size != 64 && (Size != 128 || NumElements == 1);
658 }
659 return false;
660}
661
662bool AArch64SwiftABIInfo::isLegalVectorType(CharUnits VectorSize,
663 llvm::Type *EltTy,
664 unsigned NumElts) const {
665 if (!llvm::isPowerOf2_32(Value: NumElts))
666 return false;
667 if (VectorSize.getQuantity() != 8 &&
668 (VectorSize.getQuantity() != 16 || NumElts == 1))
669 return false;
670 return true;
671}
672
673bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
674 // For the soft-float ABI variant, no types are considered to be homogeneous
675 // aggregates.
676 if (isSoftFloat())
677 return false;
678
679 // Homogeneous aggregates for AAPCS64 must have base types of a floating
680 // point type or a short-vector type. This is the same as the 32-bit ABI,
681 // but with the difference that any floating-point type is allowed,
682 // including __fp16.
683 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
684 if (BT->isFloatingPoint())
685 return true;
686 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
687 if (auto Kind = VT->getVectorKind();
688 Kind == VectorKind::SveFixedLengthData ||
689 Kind == VectorKind::SveFixedLengthPredicate)
690 return false;
691
692 unsigned VecSize = getContext().getTypeSize(T: VT);
693 if (VecSize == 64 || VecSize == 128)
694 return true;
695 }
696 return false;
697}
698
699bool AArch64ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
700 uint64_t Members) const {
701 return Members <= 4;
702}
703
704bool AArch64ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate()
705 const {
706 // AAPCS64 says that the rule for whether something is a homogeneous
707 // aggregate is applied to the output of the data layout decision. So
708 // anything that doesn't affect the data layout also does not affect
709 // homogeneity. In particular, zero-length bitfields don't stop a struct
710 // being homogeneous.
711 return true;
712}
713
714bool AArch64ABIInfo::passAsAggregateType(QualType Ty) const {
715 if (Kind == AArch64ABIKind::AAPCS && Ty->isSVESizelessBuiltinType()) {
716 const auto *BT = Ty->castAs<BuiltinType>();
717 return !BT->isSVECount() &&
718 getContext().getBuiltinVectorTypeInfo(VecTy: BT).NumVectors > 1;
719 }
720 return isAggregateTypeForABI(T: Ty);
721}
722
723// Check if a type needs to be passed in registers as a Pure Scalable Type (as
724// defined by AAPCS64). Return the number of data vectors and the number of
725// predicate vectors in the type, into `NVec` and `NPred`, respectively. Upon
726// return `CoerceToSeq` contains an expanded sequence of LLVM IR types, one
727// element for each non-composite member. For practical purposes, limit the
728// length of `CoerceToSeq` to about 12 (the maximum that could possibly fit
729// in registers) and return false, the effect of which will be to pass the
730// argument under the rules for a large (> 128 bytes) composite.
731bool AArch64ABIInfo::passAsPureScalableType(
732 QualType Ty, unsigned &NVec, unsigned &NPred,
733 SmallVectorImpl<llvm::Type *> &CoerceToSeq) const {
734 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(T: Ty)) {
735 uint64_t NElt = AT->getZExtSize();
736 if (NElt == 0)
737 return false;
738
739 unsigned NV = 0, NP = 0;
740 SmallVector<llvm::Type *> EltCoerceToSeq;
741 if (!passAsPureScalableType(Ty: AT->getElementType(), NVec&: NV, NPred&: NP, CoerceToSeq&: EltCoerceToSeq))
742 return false;
743
744 if (CoerceToSeq.size() + NElt * EltCoerceToSeq.size() > 12)
745 return false;
746
747 for (uint64_t I = 0; I < NElt; ++I)
748 llvm::append_range(C&: CoerceToSeq, R&: EltCoerceToSeq);
749
750 NVec += NElt * NV;
751 NPred += NElt * NP;
752 return true;
753 }
754
755 if (const RecordType *RT = Ty->getAsCanonical<RecordType>()) {
756 // If the record cannot be passed in registers, then it's not a PST.
757 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, CXXABI&: getCXXABI());
758 RAA != CGCXXABI::RAA_Default)
759 return false;
760
761 // Pure scalable types are never unions and never contain unions.
762 const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf();
763 if (RD->isUnion())
764 return false;
765
766 // If this is a C++ record, check the bases.
767 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
768 for (const auto &I : CXXRD->bases()) {
769 if (isEmptyRecord(Context&: getContext(), T: I.getType(), AllowArrays: true))
770 continue;
771 if (!passAsPureScalableType(Ty: I.getType(), NVec, NPred, CoerceToSeq))
772 return false;
773 }
774 }
775
776 // Check members.
777 for (const auto *FD : RD->fields()) {
778 QualType FT = FD->getType();
779 if (isEmptyField(Context&: getContext(), FD, /* AllowArrays */ true))
780 continue;
781 if (!passAsPureScalableType(Ty: FT, NVec, NPred, CoerceToSeq))
782 return false;
783 }
784
785 return true;
786 }
787
788 if (const auto *VT = Ty->getAs<VectorType>()) {
789 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
790 ++NPred;
791 if (CoerceToSeq.size() + 1 > 12)
792 return false;
793 CoerceToSeq.push_back(Elt: convertFixedToScalableVectorType(VT));
794 return true;
795 }
796
797 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
798 ++NVec;
799 if (CoerceToSeq.size() + 1 > 12)
800 return false;
801 CoerceToSeq.push_back(Elt: convertFixedToScalableVectorType(VT));
802 return true;
803 }
804
805 return false;
806 }
807
808 if (!Ty->isBuiltinType())
809 return false;
810
811 bool isPredicate;
812 switch (Ty->castAs<BuiltinType>()->getKind()) {
813#define SVE_VECTOR_TYPE(Name, MangledName, Id, SingletonId) \
814 case BuiltinType::Id: \
815 isPredicate = false; \
816 break;
817#define SVE_PREDICATE_TYPE(Name, MangledName, Id, SingletonId) \
818 case BuiltinType::Id: \
819 isPredicate = true; \
820 break;
821#include "clang/Basic/AArch64ACLETypes.def"
822 default:
823 return false;
824 }
825
826 ASTContext::BuiltinVectorTypeInfo Info =
827 getContext().getBuiltinVectorTypeInfo(VecTy: cast<BuiltinType>(Val&: Ty));
828 assert(Info.NumVectors > 0 && Info.NumVectors <= 4 &&
829 "Expected 1, 2, 3 or 4 vectors!");
830 if (isPredicate)
831 NPred += Info.NumVectors;
832 else
833 NVec += Info.NumVectors;
834 llvm::Type *EltTy = Info.ElementType->isMFloat8Type()
835 ? llvm::Type::getInt8Ty(C&: getVMContext())
836 : CGT.ConvertType(T: Info.ElementType);
837 auto *VTy = llvm::ScalableVectorType::get(ElementType: EltTy, MinNumElts: Info.EC.getKnownMinValue());
838
839 if (CoerceToSeq.size() + Info.NumVectors > 12)
840 return false;
841 std::fill_n(first: std::back_inserter(x&: CoerceToSeq), n: Info.NumVectors, value: VTy);
842
843 return true;
844}
845
846// Expand an LLVM IR type into a sequence with a element for each non-struct,
847// non-array member of the type, with the exception of the padding types, which
848// are retained.
849void AArch64ABIInfo::flattenType(
850 llvm::Type *Ty, SmallVectorImpl<llvm::Type *> &Flattened) const {
851
852 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType: Ty)) {
853 Flattened.push_back(Elt: Ty);
854 return;
855 }
856
857 if (const auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty)) {
858 uint64_t NElt = AT->getNumElements();
859 if (NElt == 0)
860 return;
861
862 SmallVector<llvm::Type *> EltFlattened;
863 flattenType(Ty: AT->getElementType(), Flattened&: EltFlattened);
864
865 for (uint64_t I = 0; I < NElt; ++I)
866 llvm::append_range(C&: Flattened, R&: EltFlattened);
867 return;
868 }
869
870 if (const auto *ST = dyn_cast<llvm::StructType>(Val: Ty)) {
871 for (auto *ET : ST->elements())
872 flattenType(Ty: ET, Flattened);
873 return;
874 }
875
876 Flattened.push_back(Elt: Ty);
877}
878
879RValue AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
880 CodeGenFunction &CGF, AArch64ABIKind Kind,
881 AggValueSlot Slot) const {
882 // These numbers are not used for variadic arguments, hence it doesn't matter
883 // they don't retain their values across multiple calls to
884 // `classifyArgumentType` here.
885 unsigned NSRN = 0, NPRN = 0;
886 ABIArgInfo AI =
887 classifyArgumentType(Ty, /*IsVariadicFn=*/true, /* IsNamedArg */ false,
888 CallingConvention: CGF.CurFnInfo->getCallingConvention(), NSRN, NPRN);
889 // Empty records are ignored for parameter passing purposes.
890 if (AI.isIgnore())
891 return Slot.asRValue();
892
893 bool IsIndirect = AI.isIndirect();
894
895 llvm::Type *BaseTy = CGF.ConvertType(T: Ty);
896 if (IsIndirect)
897 BaseTy = llvm::PointerType::getUnqual(C&: BaseTy->getContext());
898 else if (AI.getCoerceToType())
899 BaseTy = AI.getCoerceToType();
900
901 unsigned NumRegs = 1;
902 if (llvm::ArrayType *ArrTy = dyn_cast<llvm::ArrayType>(Val: BaseTy)) {
903 BaseTy = ArrTy->getElementType();
904 NumRegs = ArrTy->getNumElements();
905 }
906 bool IsFPR =
907 !isSoftFloat() && (BaseTy->isFloatingPointTy() || BaseTy->isVectorTy());
908
909 // The AArch64 va_list type and handling is specified in the Procedure Call
910 // Standard, section B.4:
911 //
912 // struct {
913 // void *__stack;
914 // void *__gr_top;
915 // void *__vr_top;
916 // int __gr_offs;
917 // int __vr_offs;
918 // };
919
920 llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock(name: "vaarg.maybe_reg");
921 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock(name: "vaarg.in_reg");
922 llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock(name: "vaarg.on_stack");
923 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "vaarg.end");
924
925 CharUnits TySize = getContext().getTypeSizeInChars(T: Ty);
926 CharUnits TyAlign = getContext().getTypeUnadjustedAlignInChars(T: Ty);
927
928 Address reg_offs_p = Address::invalid();
929 llvm::Value *reg_offs = nullptr;
930 int reg_top_index;
931 int RegSize = IsIndirect ? 8 : TySize.getQuantity();
932 if (!IsFPR) {
933 // 3 is the field number of __gr_offs
934 reg_offs_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 3, Name: "gr_offs_p");
935 reg_offs = CGF.Builder.CreateLoad(Addr: reg_offs_p, Name: "gr_offs");
936 reg_top_index = 1; // field number for __gr_top
937 RegSize = llvm::alignTo(Value: RegSize, Align: 8);
938 } else {
939 // 4 is the field number of __vr_offs.
940 reg_offs_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 4, Name: "vr_offs_p");
941 reg_offs = CGF.Builder.CreateLoad(Addr: reg_offs_p, Name: "vr_offs");
942 reg_top_index = 2; // field number for __vr_top
943 RegSize = 16 * NumRegs;
944 }
945
946 //=======================================
947 // Find out where argument was passed
948 //=======================================
949
950 // If reg_offs >= 0 we're already using the stack for this type of
951 // argument. We don't want to keep updating reg_offs (in case it overflows,
952 // though anyone passing 2GB of arguments, each at most 16 bytes, deserves
953 // whatever they get).
954 llvm::Value *UsingStack = nullptr;
955 UsingStack = CGF.Builder.CreateICmpSGE(
956 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
957
958 CGF.Builder.CreateCondBr(Cond: UsingStack, True: OnStackBlock, False: MaybeRegBlock);
959
960 // Otherwise, at least some kind of argument could go in these registers, the
961 // question is whether this particular type is too big.
962 CGF.EmitBlock(BB: MaybeRegBlock);
963
964 // Integer arguments may need to correct register alignment (for example a
965 // "struct { __int128 a; };" gets passed in x_2N, x_{2N+1}). In this case we
966 // align __gr_offs to calculate the potential address.
967 if (!IsFPR && !IsIndirect && TyAlign.getQuantity() > 8) {
968 int Align = TyAlign.getQuantity();
969
970 reg_offs = CGF.Builder.CreateAdd(
971 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: Align - 1),
972 Name: "align_regoffs");
973 reg_offs = CGF.Builder.CreateAnd(
974 LHS: reg_offs, RHS: llvm::ConstantInt::getSigned(Ty: CGF.Int32Ty, V: -Align),
975 Name: "aligned_regoffs");
976 }
977
978 // Update the gr_offs/vr_offs pointer for next call to va_arg on this va_list.
979 // The fact that this is done unconditionally reflects the fact that
980 // allocating an argument to the stack also uses up all the remaining
981 // registers of the appropriate kind.
982 llvm::Value *NewOffset = nullptr;
983 NewOffset = CGF.Builder.CreateAdd(
984 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: RegSize), Name: "new_reg_offs");
985 CGF.Builder.CreateStore(Val: NewOffset, Addr: reg_offs_p);
986
987 // Now we're in a position to decide whether this argument really was in
988 // registers or not.
989 llvm::Value *InRegs = nullptr;
990 InRegs = CGF.Builder.CreateICmpSLE(
991 LHS: NewOffset, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0), Name: "inreg");
992
993 CGF.Builder.CreateCondBr(Cond: InRegs, True: InRegBlock, False: OnStackBlock);
994
995 //=======================================
996 // Argument was in registers
997 //=======================================
998
999 // Now we emit the code for if the argument was originally passed in
1000 // registers. First start the appropriate block:
1001 CGF.EmitBlock(BB: InRegBlock);
1002
1003 llvm::Value *reg_top = nullptr;
1004 Address reg_top_p =
1005 CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: reg_top_index, Name: "reg_top_p");
1006 reg_top = CGF.Builder.CreateLoad(Addr: reg_top_p, Name: "reg_top");
1007 Address BaseAddr(CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: reg_top, IdxList: reg_offs),
1008 CGF.Int8Ty, CharUnits::fromQuantity(Quantity: IsFPR ? 16 : 8));
1009 Address RegAddr = Address::invalid();
1010 llvm::Type *MemTy = CGF.ConvertTypeForMem(T: Ty), *ElementTy = MemTy;
1011
1012 if (IsIndirect) {
1013 // If it's been passed indirectly (actually a struct), whatever we find from
1014 // stored registers or on the stack will actually be a struct **.
1015 MemTy = llvm::PointerType::getUnqual(C&: MemTy->getContext());
1016 }
1017
1018 const Type *Base = nullptr;
1019 uint64_t NumMembers = 0;
1020 bool IsHFA = isHomogeneousAggregate(Ty, Base, Members&: NumMembers);
1021 if (IsHFA && NumMembers > 1) {
1022 // Homogeneous aggregates passed in registers will have their elements split
1023 // and stored 16-bytes apart regardless of size (they're notionally in qN,
1024 // qN+1, ...). We reload and store into a temporary local variable
1025 // contiguously.
1026 assert(!IsIndirect && "Homogeneous aggregates should be passed directly");
1027 auto BaseTyInfo = getContext().getTypeInfoInChars(T: QualType(Base, 0));
1028 llvm::Type *BaseTy = CGF.ConvertType(T: QualType(Base, 0));
1029 llvm::Type *HFATy = llvm::ArrayType::get(ElementType: BaseTy, NumElements: NumMembers);
1030 Address Tmp = CGF.CreateTempAlloca(Ty: HFATy,
1031 align: std::max(a: TyAlign, b: BaseTyInfo.Align));
1032
1033 // On big-endian platforms, the value will be right-aligned in its slot.
1034 int Offset = 0;
1035 if (CGF.CGM.getDataLayout().isBigEndian() &&
1036 BaseTyInfo.Width.getQuantity() < 16)
1037 Offset = 16 - BaseTyInfo.Width.getQuantity();
1038
1039 for (unsigned i = 0; i < NumMembers; ++i) {
1040 CharUnits BaseOffset = CharUnits::fromQuantity(Quantity: 16 * i + Offset);
1041 Address LoadAddr =
1042 CGF.Builder.CreateConstInBoundsByteGEP(Addr: BaseAddr, Offset: BaseOffset);
1043 LoadAddr = LoadAddr.withElementType(ElemTy: BaseTy);
1044
1045 Address StoreAddr = CGF.Builder.CreateConstArrayGEP(Addr: Tmp, Index: i);
1046
1047 llvm::Value *Elem = CGF.Builder.CreateLoad(Addr: LoadAddr);
1048 CGF.Builder.CreateStore(Val: Elem, Addr: StoreAddr);
1049 }
1050
1051 RegAddr = Tmp.withElementType(ElemTy: MemTy);
1052 } else {
1053 // Otherwise the object is contiguous in memory.
1054
1055 // It might be right-aligned in its slot.
1056 CharUnits SlotSize = BaseAddr.getAlignment();
1057 if (CGF.CGM.getDataLayout().isBigEndian() && !IsIndirect &&
1058 (IsHFA || !isAggregateTypeForABI(T: Ty)) &&
1059 TySize < SlotSize) {
1060 CharUnits Offset = SlotSize - TySize;
1061 BaseAddr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: BaseAddr, Offset);
1062 }
1063
1064 RegAddr = BaseAddr.withElementType(ElemTy: MemTy);
1065 }
1066
1067 CGF.EmitBranch(Block: ContBlock);
1068
1069 //=======================================
1070 // Argument was on the stack
1071 //=======================================
1072 CGF.EmitBlock(BB: OnStackBlock);
1073
1074 Address stack_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 0, Name: "stack_p");
1075 llvm::Value *OnStackPtr = CGF.Builder.CreateLoad(Addr: stack_p, Name: "stack");
1076
1077 // Again, stack arguments may need realignment. In this case both integer and
1078 // floating-point ones might be affected.
1079 if (!IsIndirect && TyAlign.getQuantity() > 8) {
1080 OnStackPtr = emitRoundPointerUpToAlignment(CGF, Ptr: OnStackPtr, Align: TyAlign);
1081 }
1082 Address OnStackAddr = Address(OnStackPtr, CGF.Int8Ty,
1083 std::max(a: CharUnits::fromQuantity(Quantity: 8), b: TyAlign));
1084
1085 // All stack slots are multiples of 8 bytes.
1086 CharUnits StackSlotSize = CharUnits::fromQuantity(Quantity: 8);
1087 CharUnits StackSize;
1088 if (IsIndirect)
1089 StackSize = StackSlotSize;
1090 else
1091 StackSize = TySize.alignTo(Align: StackSlotSize);
1092
1093 llvm::Value *StackSizeC = CGF.Builder.getSize(N: StackSize);
1094 llvm::Value *NewStack = CGF.Builder.CreateInBoundsGEP(
1095 Ty: CGF.Int8Ty, Ptr: OnStackPtr, IdxList: StackSizeC, Name: "new_stack");
1096
1097 // Write the new value of __stack for the next call to va_arg
1098 CGF.Builder.CreateStore(Val: NewStack, Addr: stack_p);
1099
1100 if (CGF.CGM.getDataLayout().isBigEndian() && !isAggregateTypeForABI(T: Ty) &&
1101 TySize < StackSlotSize) {
1102 CharUnits Offset = StackSlotSize - TySize;
1103 OnStackAddr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: OnStackAddr, Offset);
1104 }
1105
1106 OnStackAddr = OnStackAddr.withElementType(ElemTy: MemTy);
1107
1108 CGF.EmitBranch(Block: ContBlock);
1109
1110 //=======================================
1111 // Tidy up
1112 //=======================================
1113 CGF.EmitBlock(BB: ContBlock);
1114
1115 Address ResAddr = emitMergePHI(CGF, Addr1: RegAddr, Block1: InRegBlock, Addr2: OnStackAddr,
1116 Block2: OnStackBlock, Name: "vaargs.addr");
1117
1118 if (IsIndirect)
1119 return CGF.EmitLoadOfAnyValue(
1120 V: CGF.MakeAddrLValue(
1121 Addr: Address(CGF.Builder.CreateLoad(Addr: ResAddr, Name: "vaarg.addr"), ElementTy,
1122 TyAlign),
1123 T: Ty),
1124 Slot);
1125
1126 return CGF.EmitLoadOfAnyValue(V: CGF.MakeAddrLValue(Addr: ResAddr, T: Ty), Slot);
1127}
1128
1129RValue AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty,
1130 CodeGenFunction &CGF,
1131 AggValueSlot Slot) const {
1132 // The backend's lowering doesn't support va_arg for aggregates or
1133 // illegal vector types. Lower VAArg here for these cases and use
1134 // the LLVM va_arg instruction for everything else.
1135 if (!isAggregateTypeForABI(T: Ty) && !isIllegalVectorType(Ty))
1136 return CGF.EmitLoadOfAnyValue(
1137 V: CGF.MakeAddrLValue(
1138 Addr: EmitVAArgInstr(CGF, VAListAddr, Ty, AI: ABIArgInfo::getDirect()), T: Ty),
1139 Slot);
1140
1141 uint64_t PointerSize = getTarget().getPointerWidth(AddrSpace: LangAS::Default) / 8;
1142 CharUnits SlotSize = CharUnits::fromQuantity(Quantity: PointerSize);
1143
1144 // Empty records are ignored for parameter passing purposes.
1145 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
1146 return Slot.asRValue();
1147
1148 // The size of the actual thing passed, which might end up just
1149 // being a pointer for indirect types.
1150 auto TyInfo = getContext().getTypeInfoInChars(T: Ty);
1151
1152 // Arguments bigger than 16 bytes which aren't homogeneous
1153 // aggregates should be passed indirectly.
1154 bool IsIndirect = false;
1155 if (TyInfo.Width.getQuantity() > 16) {
1156 const Type *Base = nullptr;
1157 uint64_t Members = 0;
1158 IsIndirect = !isHomogeneousAggregate(Ty, Base, Members);
1159 }
1160
1161 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect, ValueInfo: TyInfo, SlotSizeAndAlign: SlotSize,
1162 /*AllowHigherAlign*/ true, Slot);
1163}
1164
1165RValue AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
1166 QualType Ty, AggValueSlot Slot) const {
1167 bool IsIndirect = false;
1168
1169 if (getTarget().getTriple().isWindowsArm64EC()) {
1170 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
1171 // not 1, 2, 4, or 8 bytes, must be passed by reference."
1172 uint64_t Width = getContext().getTypeSize(T: Ty);
1173 IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Value: Width);
1174 } else {
1175 // Composites larger than 16 bytes are passed by reference.
1176 if (isAggregateTypeForABI(T: Ty) && getContext().getTypeSize(T: Ty) > 128)
1177 IsIndirect = true;
1178 }
1179
1180 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
1181 ValueInfo: CGF.getContext().getTypeInfoInChars(T: Ty),
1182 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 8),
1183 /*allowHigherAlign*/ AllowHigherAlign: false, Slot);
1184}
1185
1186static bool isStreamingCompatible(const FunctionDecl *F) {
1187 if (const auto *T = F->getType()->getAs<FunctionProtoType>())
1188 return T->getAArch64SMEAttributes() &
1189 FunctionType::SME_PStateSMCompatibleMask;
1190 return false;
1191}
1192
1193// Report an error if an argument or return value of type Ty would need to be
1194// passed in a floating-point register.
1195static void diagnoseIfNeedsFPReg(DiagnosticsEngine &Diags,
1196 const StringRef ABIName,
1197 const AArch64ABIInfo &ABIInfo,
1198 const QualType &Ty, const NamedDecl *D,
1199 SourceLocation loc) {
1200 const Type *HABase = nullptr;
1201 uint64_t HAMembers = 0;
1202 if (Ty->isFloatingType() || Ty->isVectorType() ||
1203 ABIInfo.isHomogeneousAggregate(Ty, Base&: HABase, Members&: HAMembers)) {
1204 Diags.Report(Loc: loc, DiagID: diag::err_target_unsupported_type_for_abi)
1205 << D->getDeclName() << Ty << ABIName;
1206 }
1207}
1208
1209// If we are using a hard-float ABI, but do not have floating point registers,
1210// then report an error for any function arguments or returns which would be
1211// passed in floating-pint registers.
1212void AArch64TargetCodeGenInfo::checkFunctionABI(
1213 CodeGenModule &CGM, const FunctionDecl *FuncDecl) const {
1214 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
1215 const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();
1216
1217 if (!TI.hasFeature(Feature: "fp") && !ABIInfo.isSoftFloat()) {
1218 diagnoseIfNeedsFPReg(Diags&: CGM.getDiags(), ABIName: TI.getABI(), ABIInfo,
1219 Ty: FuncDecl->getReturnType(), D: FuncDecl,
1220 loc: FuncDecl->getLocation());
1221 for (ParmVarDecl *PVD : FuncDecl->parameters()) {
1222 diagnoseIfNeedsFPReg(Diags&: CGM.getDiags(), ABIName: TI.getABI(), ABIInfo, Ty: PVD->getType(),
1223 D: PVD, loc: FuncDecl->getLocation());
1224 }
1225 }
1226}
1227
1228enum class ArmSMEInlinability : uint8_t {
1229 Ok = 0,
1230 ErrorCalleeRequiresNewZA = 1 << 0,
1231 ErrorCalleeRequiresNewZT0 = 1 << 1,
1232 WarnIncompatibleStreamingModes = 1 << 2,
1233 ErrorIncompatibleStreamingModes = 1 << 3,
1234
1235 IncompatibleStreamingModes =
1236 WarnIncompatibleStreamingModes | ErrorIncompatibleStreamingModes,
1237
1238 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/ErrorIncompatibleStreamingModes),
1239};
1240
1241/// Determines if there are any Arm SME ABI issues with inlining \p Callee into
1242/// \p Caller. Returns the issue (if any) in the ArmSMEInlinability bit enum.
1243static ArmSMEInlinability GetArmSMEInlinability(const FunctionDecl *Caller,
1244 const FunctionDecl *Callee) {
1245 bool CallerIsStreaming =
1246 IsArmStreamingFunction(FD: Caller, /*IncludeLocallyStreaming=*/true);
1247 bool CalleeIsStreaming =
1248 IsArmStreamingFunction(FD: Callee, /*IncludeLocallyStreaming=*/true);
1249 bool CallerIsStreamingCompatible = isStreamingCompatible(F: Caller);
1250 bool CalleeIsStreamingCompatible = isStreamingCompatible(F: Callee);
1251
1252 ArmSMEInlinability Inlinability = ArmSMEInlinability::Ok;
1253
1254 if (!CalleeIsStreamingCompatible &&
1255 (CallerIsStreaming != CalleeIsStreaming || CallerIsStreamingCompatible)) {
1256 if (CalleeIsStreaming)
1257 Inlinability |= ArmSMEInlinability::ErrorIncompatibleStreamingModes;
1258 else
1259 Inlinability |= ArmSMEInlinability::WarnIncompatibleStreamingModes;
1260 }
1261 if (auto *NewAttr = Callee->getAttr<ArmNewAttr>()) {
1262 if (NewAttr->isNewZA())
1263 Inlinability |= ArmSMEInlinability::ErrorCalleeRequiresNewZA;
1264 if (NewAttr->isNewZT0())
1265 Inlinability |= ArmSMEInlinability::ErrorCalleeRequiresNewZT0;
1266 }
1267
1268 return Inlinability;
1269}
1270
1271void AArch64TargetCodeGenInfo::checkFunctionCallABIStreaming(
1272 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
1273 const FunctionDecl *Callee) const {
1274 if (!Caller || !Callee || !Callee->hasAttr<AlwaysInlineAttr>())
1275 return;
1276
1277 ArmSMEInlinability Inlinability = GetArmSMEInlinability(Caller, Callee);
1278
1279 if ((Inlinability & ArmSMEInlinability::IncompatibleStreamingModes) !=
1280 ArmSMEInlinability::Ok)
1281 CGM.getDiags().Report(
1282 Loc: CallLoc,
1283 DiagID: (Inlinability & ArmSMEInlinability::ErrorIncompatibleStreamingModes) ==
1284 ArmSMEInlinability::ErrorIncompatibleStreamingModes
1285 ? diag::err_function_always_inline_attribute_mismatch
1286 : diag::warn_function_always_inline_attribute_mismatch)
1287 << Caller->getDeclName() << Callee->getDeclName() << "streaming";
1288
1289 if ((Inlinability & ArmSMEInlinability::ErrorCalleeRequiresNewZA) ==
1290 ArmSMEInlinability::ErrorCalleeRequiresNewZA)
1291 CGM.getDiags().Report(Loc: CallLoc, DiagID: diag::err_function_always_inline_new_za)
1292 << Callee->getDeclName();
1293
1294 if ((Inlinability & ArmSMEInlinability::ErrorCalleeRequiresNewZT0) ==
1295 ArmSMEInlinability::ErrorCalleeRequiresNewZT0)
1296 CGM.getDiags().Report(Loc: CallLoc, DiagID: diag::err_function_always_inline_new_zt0)
1297 << Callee->getDeclName();
1298}
1299
1300// If the target does not have floating-point registers, but we are using a
1301// hard-float ABI, there is no way to pass floating-point, vector or HFA values
1302// to functions, so we report an error.
1303void AArch64TargetCodeGenInfo::checkFunctionCallABISoftFloat(
1304 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
1305 const FunctionDecl *Callee, const CallArgList &Args,
1306 QualType ReturnType) const {
1307 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
1308 const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();
1309
1310 if (!Caller || TI.hasFeature(Feature: "fp") || ABIInfo.isSoftFloat())
1311 return;
1312
1313 diagnoseIfNeedsFPReg(Diags&: CGM.getDiags(), ABIName: TI.getABI(), ABIInfo, Ty: ReturnType,
1314 D: Callee ? Callee : Caller, loc: CallLoc);
1315
1316 for (const CallArg &Arg : Args)
1317 diagnoseIfNeedsFPReg(Diags&: CGM.getDiags(), ABIName: TI.getABI(), ABIInfo, Ty: Arg.getType(),
1318 D: Callee ? Callee : Caller, loc: CallLoc);
1319}
1320
1321void AArch64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM,
1322 SourceLocation CallLoc,
1323 const FunctionDecl *Caller,
1324 const FunctionDecl *Callee,
1325 const CallArgList &Args,
1326 QualType ReturnType) const {
1327 checkFunctionCallABIStreaming(CGM, CallLoc, Caller, Callee);
1328 checkFunctionCallABISoftFloat(CGM, CallLoc, Caller, Callee, Args, ReturnType);
1329}
1330
1331bool AArch64TargetCodeGenInfo::wouldInliningViolateFunctionCallABI(
1332 const FunctionDecl *Caller, const FunctionDecl *Callee) const {
1333 return Caller && Callee &&
1334 GetArmSMEInlinability(Caller, Callee) != ArmSMEInlinability::Ok;
1335}
1336
1337void AArch64ABIInfo::appendAttributeMangling(TargetClonesAttr *Attr,
1338 unsigned Index,
1339 raw_ostream &Out) const {
1340 appendAttributeMangling(AttrStr: Attr->getFeatureStr(Index), Out);
1341}
1342
1343void AArch64ABIInfo::appendAttributeMangling(StringRef AttrStr,
1344 raw_ostream &Out) const {
1345 if (AttrStr == "default") {
1346 Out << ".default";
1347 return;
1348 }
1349
1350 Out << "._";
1351 SmallVector<StringRef, 8> Features;
1352 AttrStr.split(A&: Features, Separator: "+");
1353 for (auto &Feat : Features)
1354 Feat = Feat.trim();
1355
1356 llvm::sort(C&: Features, Comp: [](const StringRef LHS, const StringRef RHS) {
1357 return LHS.compare(RHS) < 0;
1358 });
1359
1360 llvm::SmallDenseSet<StringRef, 8> UniqueFeats;
1361 for (auto &Feat : Features)
1362 if (getTarget().doesFeatureAffectCodeGen(Feature: Feat))
1363 if (auto Ext = llvm::AArch64::parseFMVExtension(Extension: Feat))
1364 if (UniqueFeats.insert(V: Ext->Name).second)
1365 Out << 'M' << Ext->Name;
1366}
1367
1368std::unique_ptr<TargetCodeGenInfo>
1369CodeGen::createAArch64TargetCodeGenInfo(CodeGenModule &CGM,
1370 AArch64ABIKind Kind) {
1371 return std::make_unique<AArch64TargetCodeGenInfo>(args&: CGM, args&: Kind);
1372}
1373
1374std::unique_ptr<TargetCodeGenInfo>
1375CodeGen::createWindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM,
1376 AArch64ABIKind K) {
1377 return std::make_unique<WindowsAArch64TargetCodeGenInfo>(args&: CGM, args&: K);
1378}
1379