1//===- AArch64.cpp --------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ABIInfoImpl.h"
10#include "TargetInfo.h"
11#include "clang/AST/Decl.h"
12#include "clang/Basic/DiagnosticFrontend.h"
13#include "clang/CodeGenUtils/TargetUtils.h"
14#include "llvm/TargetParser/AArch64TargetParser.h"
15
16using namespace clang;
17using namespace clang::CodeGen;
18
19//===----------------------------------------------------------------------===//
20// AArch64 ABI Implementation
21//===----------------------------------------------------------------------===//
22
23namespace {
24
25class AArch64ABIInfo : public ABIInfo {
26 AArch64ABIKind Kind;
27
28 std::unique_ptr<TargetCodeGenInfo> WinX86_64CodegenInfo;
29
30public:
31 AArch64ABIInfo(CodeGenModule &CGM, AArch64ABIKind Kind)
32 : ABIInfo(CGM.getTypes()), Kind(Kind) {
33 if (getTarget().getTriple().isWindowsArm64EC()) {
34 WinX86_64CodegenInfo =
35 createWinX86_64TargetCodeGenInfo(CGM, AVXLevel: X86AVXABILevel::None);
36 }
37 }
38
39 bool isSoftFloat() const { return Kind == AArch64ABIKind::AAPCSSoft; }
40
41private:
42 AArch64ABIKind getABIKind() const { return Kind; }
43 bool isDarwinPCS() const { return Kind == AArch64ABIKind::DarwinPCS; }
44
45 ABIArgInfo classifyReturnType(QualType RetTy, bool IsVariadicFn) const;
46 ABIArgInfo classifyArgumentType(QualType RetTy, bool IsVariadicFn,
47 bool IsNamedArg, unsigned CallingConvention,
48 unsigned &NSRN, unsigned &NPRN) const;
49 llvm::Type *convertFixedToScalableVectorType(const VectorType *VT) const;
50 ABIArgInfo coerceIllegalVector(QualType Ty, unsigned &NSRN,
51 unsigned &NPRN) const;
52 ABIArgInfo coerceAndExpandPureScalableAggregate(
53 QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
54 const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
55 unsigned &NPRN) const;
56 bool isHomogeneousAggregateBaseType(QualType Ty) const override;
57 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
58 uint64_t Members) const override;
59 bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const override;
60
61 bool isIllegalVectorType(QualType Ty) const;
62
63 bool passAsAggregateType(QualType Ty) const;
64 bool passAsPureScalableType(QualType Ty, unsigned &NV, unsigned &NP,
65 SmallVectorImpl<llvm::Type *> &CoerceToSeq) const;
66
67 void flattenType(llvm::Type *Ty,
68 SmallVectorImpl<llvm::Type *> &Flattened) const;
69
70 void computeInfo(CGFunctionInfo &FI) const override {
71 if (!::classifyReturnType(CXXABI: getCXXABI(), FI, Info: *this))
72 FI.getReturnInfo() =
73 classifyReturnType(RetTy: FI.getReturnType(), IsVariadicFn: FI.isVariadic());
74
75 unsigned ArgNo = 0;
76 unsigned NSRN = 0, NPRN = 0;
77 for (auto &it : FI.arguments()) {
78 const bool IsNamedArg =
79 !FI.isVariadic() || ArgNo < FI.getRequiredArgs().getNumRequiredArgs();
80 ++ArgNo;
81 it.info = classifyArgumentType(RetTy: it.type, IsVariadicFn: FI.isVariadic(), IsNamedArg,
82 CallingConvention: FI.getCallingConvention(), NSRN, NPRN);
83 }
84 }
85
86 RValue EmitDarwinVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,
87 AggValueSlot Slot) const;
88
89 RValue EmitAAPCSVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,
90 AArch64ABIKind Kind, AggValueSlot Slot) const;
91
92 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
93 AggValueSlot Slot) const override {
94 llvm::Type *BaseTy = CGF.ConvertType(T: Ty);
95 if (isa<llvm::ScalableVectorType>(Val: BaseTy))
96 llvm::report_fatal_error(reason: "Passing SVE types to variadic functions is "
97 "currently not supported");
98
99 return Kind == AArch64ABIKind::Win64
100 ? EmitMSVAArg(CGF, VAListAddr, Ty, Slot)
101 : isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF, Slot)
102 : EmitAAPCSVAArg(VAListAddr, Ty, CGF, Kind, Slot);
103 }
104
105 RValue EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
106 AggValueSlot Slot) const override;
107
108 bool allowBFloatArgsAndRet() const override {
109 return getTarget().hasBFloat16Type();
110 }
111
112 using ABIInfo::appendAttributeMangling;
113 void appendAttributeMangling(TargetClonesAttr *Attr, unsigned Index,
114 raw_ostream &Out) const override;
115 void appendAttributeMangling(StringRef AttrStr,
116 raw_ostream &Out) const override;
117};
118
119class AArch64SwiftABIInfo : public SwiftABIInfo {
120public:
121 explicit AArch64SwiftABIInfo(CodeGenTypes &CGT)
122 : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/true) {}
123
124 bool isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy,
125 unsigned NumElts) const override;
126};
127
128class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
129public:
130 AArch64TargetCodeGenInfo(CodeGenModule &CGM, AArch64ABIKind Kind)
131 : TargetCodeGenInfo(std::make_unique<AArch64ABIInfo>(args&: CGM, args&: Kind)) {
132 SwiftInfo = std::make_unique<AArch64SwiftABIInfo>(args&: CGM.getTypes());
133 }
134
135 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
136 return "mov\tfp, fp\t\t// marker for objc_retainAutoreleaseReturnValue";
137 }
138
139 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
140 return 31;
141 }
142
143 bool doesReturnSlotInterfereWithArgs() const override { return false; }
144
145 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
146 CodeGen::CodeGenModule &CGM) const override {
147 auto *Fn = dyn_cast<llvm::Function>(Val: GV);
148 if (!Fn)
149 return;
150
151 const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
152 TargetInfo::BranchProtectionInfo BPI(CGM.getLangOpts());
153
154 if (FD && FD->hasAttr<TargetAttr>()) {
155 const auto *TA = FD->getAttr<TargetAttr>();
156 ParsedTargetAttr Attr =
157 CGM.getTarget().parseTargetAttr(Str: TA->getFeaturesStr());
158 if (!Attr.BranchProtection.empty()) {
159 StringRef Error;
160 (void)CGM.getTarget().validateBranchProtection(
161 Spec: Attr.BranchProtection, Arch: Attr.CPU, BPI, LO: CGM.getLangOpts(), Err&: Error);
162 assert(Error.empty());
163 }
164 }
165 setBranchProtectionFnAttributes(BPI, F&: *Fn);
166 setPointerAuthFnAttributes(Opts: CGM.getCodeGenOpts().PointerAuth, F&: *Fn);
167 }
168
169 bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF,
170 llvm::Type *Ty) const override {
171 if (CGF.getTarget().hasFeature(Feature: "ls64")) {
172 auto *ST = dyn_cast<llvm::StructType>(Val: Ty);
173 if (ST && ST->getNumElements() == 1) {
174 auto *AT = dyn_cast<llvm::ArrayType>(Val: ST->getElementType(N: 0));
175 if (AT && AT->getNumElements() == 8 &&
176 AT->getElementType()->isIntegerTy(BitWidth: 64))
177 return true;
178 }
179 }
180 return TargetCodeGenInfo::isScalarizableAsmOperand(CGF, Ty);
181 }
182
183 void checkFunctionABI(CodeGenModule &CGM,
184 const FunctionDecl *Decl) const override;
185
186 void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
187 const FunctionDecl *Caller,
188 const FunctionDecl *Callee, const CallArgList &Args,
189 QualType ReturnType) const override;
190
191 bool wouldInliningViolateFunctionCallABI(
192 const FunctionDecl *Caller, const FunctionDecl *Callee) const override;
193
194private:
195 // Diagnose calls between functions with incompatible Streaming SVE
196 // attributes.
197 void checkFunctionCallABIStreaming(CodeGenModule &CGM, SourceLocation CallLoc,
198 const FunctionDecl *Caller,
199 const FunctionDecl *Callee) const;
200 // Diagnose calls which must pass arguments in floating-point registers when
201 // the selected target does not have floating-point registers.
202 void checkFunctionCallABISoftFloat(CodeGenModule &CGM, SourceLocation CallLoc,
203 const FunctionDecl *Caller,
204 const FunctionDecl *Callee,
205 const CallArgList &Args,
206 QualType ReturnType) const;
207};
208
209class WindowsAArch64TargetCodeGenInfo : public AArch64TargetCodeGenInfo {
210public:
211 WindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM, AArch64ABIKind K)
212 : AArch64TargetCodeGenInfo(CGM, K) {}
213
214 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
215 CodeGen::CodeGenModule &CGM) const override;
216
217 void getDependentLibraryOption(llvm::StringRef Lib,
218 llvm::SmallString<24> &Opt) const override {
219 Opt = "/DEFAULTLIB:" + qualifyWindowsLibrary(Lib);
220 }
221
222 void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value,
223 llvm::SmallString<32> &Opt) const override {
224 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
225 }
226};
227
228void WindowsAArch64TargetCodeGenInfo::setTargetAttributes(
229 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
230 AArch64TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
231 if (GV->isDeclaration())
232 return;
233 addStackProbeTargetAttributes(D, GV, CGM);
234}
235}
236
237llvm::Type *
238AArch64ABIInfo::convertFixedToScalableVectorType(const VectorType *VT) const {
239 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
240
241 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
242 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
243 BuiltinType::UChar &&
244 "unexpected builtin type for SVE predicate!");
245 return llvm::ScalableVectorType::get(ElementType: llvm::Type::getInt1Ty(C&: getVMContext()),
246 MinNumElts: 16);
247 }
248
249 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
250 const auto *BT = VT->getElementType()->castAs<BuiltinType>();
251 switch (BT->getKind()) {
252 default:
253 llvm_unreachable("unexpected builtin type for SVE vector!");
254
255 case BuiltinType::SChar:
256 case BuiltinType::UChar:
257 case BuiltinType::MFloat8:
258 return llvm::ScalableVectorType::get(
259 ElementType: llvm::Type::getInt8Ty(C&: getVMContext()), MinNumElts: 16);
260
261 case BuiltinType::Short:
262 case BuiltinType::UShort:
263 return llvm::ScalableVectorType::get(
264 ElementType: llvm::Type::getInt16Ty(C&: getVMContext()), MinNumElts: 8);
265
266 case BuiltinType::Int:
267 case BuiltinType::UInt:
268 return llvm::ScalableVectorType::get(
269 ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), MinNumElts: 4);
270
271 case BuiltinType::Long:
272 case BuiltinType::ULong:
273 return llvm::ScalableVectorType::get(
274 ElementType: llvm::Type::getInt64Ty(C&: getVMContext()), MinNumElts: 2);
275
276 case BuiltinType::Half:
277 return llvm::ScalableVectorType::get(
278 ElementType: llvm::Type::getHalfTy(C&: getVMContext()), MinNumElts: 8);
279
280 case BuiltinType::Float:
281 return llvm::ScalableVectorType::get(
282 ElementType: llvm::Type::getFloatTy(C&: getVMContext()), MinNumElts: 4);
283
284 case BuiltinType::Double:
285 return llvm::ScalableVectorType::get(
286 ElementType: llvm::Type::getDoubleTy(C&: getVMContext()), MinNumElts: 2);
287
288 case BuiltinType::BFloat16:
289 return llvm::ScalableVectorType::get(
290 ElementType: llvm::Type::getBFloatTy(C&: getVMContext()), MinNumElts: 8);
291 }
292 }
293
294 llvm_unreachable("expected fixed-length SVE vector");
295}
296
297ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty, unsigned &NSRN,
298 unsigned &NPRN) const {
299 assert(Ty->isVectorType() && "expected vector type!");
300
301 const auto *VT = Ty->castAs<VectorType>();
302 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
303 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
304 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
305 BuiltinType::UChar &&
306 "unexpected builtin type for SVE predicate!");
307 NPRN = std::min(a: NPRN + 1, b: 4u);
308 return ABIArgInfo::getDirect(T: llvm::ScalableVectorType::get(
309 ElementType: llvm::Type::getInt1Ty(C&: getVMContext()), MinNumElts: 16));
310 }
311
312 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
313 NSRN = std::min(a: NSRN + 1, b: 8u);
314 return ABIArgInfo::getDirect(T: convertFixedToScalableVectorType(VT));
315 }
316
317 uint64_t Size = getContext().getTypeSize(T: Ty);
318 // Android promotes <2 x i8> to i16, not i32
319 if ((isAndroid() || isOHOSFamily()) && (Size <= 16)) {
320 llvm::Type *ResType = llvm::Type::getInt16Ty(C&: getVMContext());
321 return ABIArgInfo::getDirect(T: ResType);
322 }
323 if (Size <= 32) {
324 llvm::Type *ResType = llvm::Type::getInt32Ty(C&: getVMContext());
325 return ABIArgInfo::getDirect(T: ResType);
326 }
327 if (Size == 64) {
328 NSRN = std::min(a: NSRN + 1, b: 8u);
329 auto *ResType =
330 llvm::FixedVectorType::get(ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), NumElts: 2);
331 return ABIArgInfo::getDirect(T: ResType);
332 }
333 if (Size == 128) {
334 NSRN = std::min(a: NSRN + 1, b: 8u);
335 auto *ResType =
336 llvm::FixedVectorType::get(ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), NumElts: 4);
337 return ABIArgInfo::getDirect(T: ResType);
338 }
339
340 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
341 /*ByVal=*/false);
342}
343
344ABIArgInfo AArch64ABIInfo::coerceAndExpandPureScalableAggregate(
345 QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
346 const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
347 unsigned &NPRN) const {
348 if (!IsNamedArg || NSRN + NVec > 8 || NPRN + NPred > 4)
349 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
350 /*ByVal=*/false);
351 NSRN += NVec;
352 NPRN += NPred;
353
354 // Handle SVE vector tuples.
355 if (Ty->isSVESizelessBuiltinType())
356 return ABIArgInfo::getDirect();
357
358 llvm::Type *UnpaddedCoerceToType =
359 UnpaddedCoerceToSeq.size() == 1
360 ? UnpaddedCoerceToSeq[0]
361 : llvm::StructType::get(Context&: CGT.getLLVMContext(), Elements: UnpaddedCoerceToSeq,
362 isPacked: true);
363
364 SmallVector<llvm::Type *> CoerceToSeq;
365 flattenType(Ty: CGT.ConvertType(T: Ty), Flattened&: CoerceToSeq);
366 auto *CoerceToType =
367 llvm::StructType::get(Context&: CGT.getLLVMContext(), Elements: CoerceToSeq, isPacked: false);
368
369 return ABIArgInfo::getCoerceAndExpand(coerceToType: CoerceToType, unpaddedCoerceToType: UnpaddedCoerceToType);
370}
371
372ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadicFn,
373 bool IsNamedArg,
374 unsigned CallingConvention,
375 unsigned &NSRN,
376 unsigned &NPRN) const {
377 Ty = useFirstFieldIfTransparentUnion(Ty);
378
379 if (IsVariadicFn && getTarget().getTriple().isWindowsArm64EC()) {
380 // Arm64EC varargs functions use the x86_64 classification rules,
381 // not the AArch64 ABI rules.
382 return WinX86_64CodegenInfo->getABIInfo().classifyArgForArm64ECVarArg(
383 Ty, IsNamedArg);
384 }
385
386 // Handle illegal vector types here.
387 if (isIllegalVectorType(Ty))
388 return coerceIllegalVector(Ty, NSRN, NPRN);
389
390 if (!passAsAggregateType(Ty)) {
391 // Treat an enum type as its underlying type.
392 if (const auto *ED = Ty->getAsEnumDecl())
393 Ty = ED->getIntegerType();
394
395 if (const auto *EIT = Ty->getAs<BitIntType>())
396 if (EIT->getNumBits() > 128)
397 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
398 ByVal: false);
399
400 if (Ty->isVectorType())
401 NSRN = std::min(a: NSRN + 1, b: 8u);
402 else if (const auto *BT = Ty->getAs<BuiltinType>()) {
403 if (BT->isFloatingPoint())
404 NSRN = std::min(a: NSRN + 1, b: 8u);
405 else {
406 switch (BT->getKind()) {
407 case BuiltinType::SveBool:
408 case BuiltinType::SveCount:
409 NPRN = std::min(a: NPRN + 1, b: 4u);
410 break;
411 case BuiltinType::SveBoolx2:
412 NPRN = std::min(a: NPRN + 2, b: 4u);
413 break;
414 case BuiltinType::SveBoolx4:
415 NPRN = std::min(a: NPRN + 4, b: 4u);
416 break;
417 case BuiltinType::MFloat8:
418 NSRN = std::min(a: NSRN + 1, b: 8u);
419 break;
420 default:
421 if (BT->isSVESizelessBuiltinType())
422 NSRN = std::min(
423 a: NSRN + getContext().getBuiltinVectorTypeInfo(VecTy: BT).NumVectors,
424 b: 8u);
425 }
426 }
427 }
428
429 return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS()
430 ? ABIArgInfo::getExtend(Ty, T: CGT.ConvertType(T: Ty))
431 : ABIArgInfo::getDirect());
432 }
433
434 // Structures with either a non-trivial destructor or a non-trivial
435 // copy constructor are always indirect.
436 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(T: Ty, CXXABI&: getCXXABI())) {
437 return getNaturalAlignIndirect(
438 Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
439 /*ByVal=*/RAA == CGCXXABI::RAA_DirectInMemory);
440 }
441
442 // Empty records:
443 // AAPCS64 does not say that empty records are ignored as arguments,
444 // but other compilers do so in certain situations, and we copy that behavior.
445 // Those situations are in fact language-mode-specific, which seems really
446 // unfortunate, but it's something we just have to accept. If this doesn't
447 // apply, just fall through to the standard argument-handling path.
448 // Darwin overrides the psABI here to ignore all empty records in all modes.
449 uint64_t Size = getContext().getTypeSize(T: Ty);
450 bool IsEmpty = isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true);
451 if (!Ty->isSVESizelessBuiltinType() && (IsEmpty || Size == 0)) {
452 // Empty records are ignored in C mode, and in C++ on Darwin.
453 if (!getContext().getLangOpts().CPlusPlus || isDarwinPCS())
454 return ABIArgInfo::getIgnore();
455
456 // In C++ mode, arguments which have sizeof() == 0 (which are non-standard
457 // C++) are ignored. This isn't defined by any standard, so we copy GCC's
458 // behaviour here.
459 if (Size == 0)
460 return ABIArgInfo::getIgnore();
461 }
462
463 // Homogeneous Floating-point Aggregates (HFAs) need to be expanded.
464 const Type *Base = nullptr;
465 uint64_t Members = 0;
466 bool IsWin64 = Kind == AArch64ABIKind::Win64 ||
467 CallingConvention == llvm::CallingConv::Win64;
468 bool IsWinVariadic = IsWin64 && IsVariadicFn;
469 // In variadic functions on Windows, all composite types are treated alike,
470 // no special handling of HFAs/HVAs.
471 if (!IsWinVariadic && isHomogeneousAggregate(Ty, Base, Members)) {
472 NSRN = std::min(a: NSRN + Members, b: uint64_t(8));
473 if (Kind != AArch64ABIKind::AAPCS)
474 return ABIArgInfo::getDirect(
475 T: llvm::ArrayType::get(ElementType: CGT.ConvertType(T: QualType(Base, 0)), NumElements: Members));
476
477 // For HFAs/HVAs, cap the argument alignment to 16, otherwise
478 // set it to 8 according to the AAPCS64 document.
479 unsigned Align =
480 getContext().getTypeUnadjustedAlignInChars(T: Ty).getQuantity();
481 Align = (Align >= 16) ? 16 : 8;
482 return ABIArgInfo::getDirect(
483 T: llvm::ArrayType::get(ElementType: CGT.ConvertType(T: QualType(Base, 0)), NumElements: Members), Offset: 0,
484 Padding: nullptr, CanBeFlattened: true, Align);
485 }
486
487 // In AAPCS named arguments of a Pure Scalable Type are passed expanded in
488 // registers, or indirectly if there are not enough registers.
489 if (Kind == AArch64ABIKind::AAPCS) {
490 unsigned NVec = 0, NPred = 0;
491 SmallVector<llvm::Type *> UnpaddedCoerceToSeq;
492 if (passAsPureScalableType(Ty, NV&: NVec, NP&: NPred, CoerceToSeq&: UnpaddedCoerceToSeq) &&
493 (NVec + NPred) > 0)
494 return coerceAndExpandPureScalableAggregate(
495 Ty, IsNamedArg, NVec, NPred, UnpaddedCoerceToSeq, NSRN, NPRN);
496 }
497
498 // Aggregates <= 16 bytes are passed directly in registers or on the stack.
499 if (Size <= 128) {
500 unsigned Alignment;
501 if (Kind == AArch64ABIKind::AAPCS) {
502 Alignment = getContext().getTypeUnadjustedAlign(T: Ty);
503 Alignment = Alignment < 128 ? 64 : 128;
504 } else {
505 Alignment =
506 std::max(a: getContext().getTypeAlign(T: Ty),
507 b: (unsigned)getTarget().getPointerWidth(AddrSpace: LangAS::Default));
508 }
509 Size = llvm::alignTo(Value: Size, Align: Alignment);
510
511 // If the Aggregate is made up of pointers, use an array of pointers for the
512 // coerced type. This prevents having to convert ptr2int->int2ptr through
513 // the call, allowing alias analysis to produce better code.
514 auto ContainsOnlyPointers = [&](const auto &Self, QualType Ty) {
515 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
516 return false;
517 const auto *RD = Ty->getAsRecordDecl();
518 if (!RD)
519 return false;
520 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
521 for (const auto &I : CXXRD->bases())
522 if (!Self(Self, I.getType()))
523 return false;
524 }
525 return all_of(RD->fields(), [&](FieldDecl *FD) {
526 QualType FDTy = FD->getType();
527 if (FDTy->isArrayType())
528 FDTy = getContext().getBaseElementType(QT: FDTy);
529 return (FDTy->isPointerOrReferenceType() &&
530 getContext().getTypeSize(T: FDTy) == 64 &&
531 !FDTy->getPointeeType().hasAddressSpace()) ||
532 Self(Self, FDTy);
533 });
534 };
535
536 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
537 // For aggregates with 16-byte alignment, we use i128.
538 llvm::Type *BaseTy = llvm::Type::getIntNTy(C&: getVMContext(), N: Alignment);
539 if ((Size == 64 || Size == 128) && Alignment == 64 &&
540 ContainsOnlyPointers(ContainsOnlyPointers, Ty))
541 BaseTy = llvm::PointerType::getUnqual(C&: getVMContext());
542 return ABIArgInfo::getDirect(
543 T: Size == Alignment ? BaseTy
544 : llvm::ArrayType::get(ElementType: BaseTy, NumElements: Size / Alignment));
545 }
546
547 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
548 /*ByVal=*/false);
549}
550
551ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,
552 bool IsVariadicFn) const {
553 if (RetTy->isVoidType())
554 return ABIArgInfo::getIgnore();
555
556 if (const auto *VT = RetTy->getAs<VectorType>()) {
557 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
558 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
559 unsigned NSRN = 0, NPRN = 0;
560 return coerceIllegalVector(Ty: RetTy, NSRN, NPRN);
561 }
562 }
563
564 // Large vector types should be returned via memory.
565 if (RetTy->isVectorType() && getContext().getTypeSize(T: RetTy) > 128)
566 return getNaturalAlignIndirect(Ty: RetTy, AddrSpace: getDataLayout().getAllocaAddrSpace());
567
568 if (!passAsAggregateType(Ty: RetTy)) {
569 // Treat an enum type as its underlying type.
570 if (const auto *ED = RetTy->getAsEnumDecl())
571 RetTy = ED->getIntegerType();
572
573 if (const auto *EIT = RetTy->getAs<BitIntType>())
574 if (EIT->getNumBits() > 128)
575 return getNaturalAlignIndirect(Ty: RetTy,
576 AddrSpace: getDataLayout().getAllocaAddrSpace());
577
578 return (isPromotableIntegerTypeForABI(Ty: RetTy) && isDarwinPCS()
579 ? ABIArgInfo::getExtend(Ty: RetTy)
580 : ABIArgInfo::getDirect());
581 }
582
583 uint64_t Size = getContext().getTypeSize(T: RetTy);
584 if (!RetTy->isSVESizelessBuiltinType() &&
585 (isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true) || Size == 0))
586 return ABIArgInfo::getIgnore();
587
588 const Type *Base = nullptr;
589 uint64_t Members = 0;
590 if (isHomogeneousAggregate(Ty: RetTy, Base, Members) &&
591 !(getTarget().getTriple().getArch() == llvm::Triple::aarch64_32 &&
592 IsVariadicFn))
593 // Homogeneous Floating-point Aggregates (HFAs) are returned directly.
594 return ABIArgInfo::getDirect();
595
596 // In AAPCS return values of a Pure Scalable type are treated as a single
597 // named argument and passed expanded in registers, or indirectly if there are
598 // not enough registers.
599 if (Kind == AArch64ABIKind::AAPCS) {
600 unsigned NSRN = 0, NPRN = 0;
601 unsigned NVec = 0, NPred = 0;
602 SmallVector<llvm::Type *> UnpaddedCoerceToSeq;
603 if (passAsPureScalableType(Ty: RetTy, NV&: NVec, NP&: NPred, CoerceToSeq&: UnpaddedCoerceToSeq) &&
604 (NVec + NPred) > 0)
605 return coerceAndExpandPureScalableAggregate(
606 Ty: RetTy, /* IsNamedArg */ true, NVec, NPred, UnpaddedCoerceToSeq, NSRN,
607 NPRN);
608 }
609
610 // Aggregates <= 16 bytes are returned directly in registers or on the stack.
611 if (Size <= 128) {
612 if (Size <= 64 && getDataLayout().isLittleEndian()) {
613 // Composite types are returned in lower bits of a 64-bit register for LE,
614 // and in higher bits for BE. However, integer types are always returned
615 // in lower bits for both LE and BE, and they are not rounded up to
616 // 64-bits. We can skip rounding up of composite types for LE, but not for
617 // BE, otherwise composite types will be indistinguishable from integer
618 // types.
619 return ABIArgInfo::getDirect(
620 T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Size));
621 }
622
623 unsigned Alignment = getContext().getTypeAlign(T: RetTy);
624 Size = llvm::alignTo(Value: Size, Align: 64); // round up to multiple of 8 bytes
625
626 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
627 // For aggregates with 16-byte alignment, we use i128.
628 if (Alignment < 128 && Size == 128) {
629 llvm::Type *BaseTy = llvm::Type::getInt64Ty(C&: getVMContext());
630 return ABIArgInfo::getDirect(T: llvm::ArrayType::get(ElementType: BaseTy, NumElements: Size / 64));
631 }
632 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Size));
633 }
634
635 return getNaturalAlignIndirect(Ty: RetTy, AddrSpace: getDataLayout().getAllocaAddrSpace());
636}
637
638/// isIllegalVectorType - check whether the vector type is legal for AArch64.
639bool AArch64ABIInfo::isIllegalVectorType(QualType Ty) const {
640 if (const VectorType *VT = Ty->getAs<VectorType>()) {
641 // Check whether VT is a fixed-length SVE vector. These types are
642 // represented as scalable vectors in function args/return and must be
643 // coerced from fixed vectors.
644 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
645 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate)
646 return true;
647
648 // Check whether VT is legal.
649 unsigned NumElements = VT->getNumElements();
650 uint64_t Size = getContext().getTypeSize(T: VT);
651 // NumElements should be power of 2.
652 if (!llvm::isPowerOf2_32(Value: NumElements))
653 return true;
654
655 // arm64_32 has to be compatible with the ARM logic here, which allows huge
656 // vectors for some reason.
657 llvm::Triple Triple = getTarget().getTriple();
658 if (Triple.getArch() == llvm::Triple::aarch64_32 &&
659 Triple.isOSBinFormatMachO())
660 return Size <= 32;
661
662 return Size != 64 && (Size != 128 || NumElements == 1);
663 }
664 return false;
665}
666
667bool AArch64SwiftABIInfo::isLegalVectorType(CharUnits VectorSize,
668 llvm::Type *EltTy,
669 unsigned NumElts) const {
670 if (!llvm::isPowerOf2_32(Value: NumElts))
671 return false;
672 if (VectorSize.getQuantity() != 8 &&
673 (VectorSize.getQuantity() != 16 || NumElts == 1))
674 return false;
675 return true;
676}
677
678bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
679 // For the soft-float ABI variant, no types are considered to be homogeneous
680 // aggregates.
681 if (isSoftFloat())
682 return false;
683
684 // Homogeneous aggregates for AAPCS64 must have base types of a floating
685 // point type or a short-vector type. This is the same as the 32-bit ABI,
686 // but with the difference that any floating-point type is allowed,
687 // including __fp16.
688 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
689 if (BT->isFloatingPoint())
690 return true;
691 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
692 if (auto Kind = VT->getVectorKind();
693 Kind == VectorKind::SveFixedLengthData ||
694 Kind == VectorKind::SveFixedLengthPredicate)
695 return false;
696
697 unsigned VecSize = getContext().getTypeSize(T: VT);
698 if (VecSize == 64 || VecSize == 128)
699 return true;
700 }
701 return false;
702}
703
704bool AArch64ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
705 uint64_t Members) const {
706 return Members <= 4;
707}
708
709bool AArch64ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate()
710 const {
711 // AAPCS64 says that the rule for whether something is a homogeneous
712 // aggregate is applied to the output of the data layout decision. So
713 // anything that doesn't affect the data layout also does not affect
714 // homogeneity. In particular, zero-length bitfields don't stop a struct
715 // being homogeneous.
716 return true;
717}
718
719bool AArch64ABIInfo::passAsAggregateType(QualType Ty) const {
720 if (Kind == AArch64ABIKind::AAPCS && Ty->isSVESizelessBuiltinType()) {
721 const auto *BT = Ty->castAs<BuiltinType>();
722 return !BT->isSVECount() &&
723 getContext().getBuiltinVectorTypeInfo(VecTy: BT).NumVectors > 1;
724 }
725 return isAggregateTypeForABI(T: Ty);
726}
727
728// Check if a type needs to be passed in registers as a Pure Scalable Type (as
729// defined by AAPCS64). Return the number of data vectors and the number of
730// predicate vectors in the type, into `NVec` and `NPred`, respectively. Upon
731// return `CoerceToSeq` contains an expanded sequence of LLVM IR types, one
732// element for each non-composite member. For practical purposes, limit the
733// length of `CoerceToSeq` to about 12 (the maximum that could possibly fit
734// in registers) and return false, the effect of which will be to pass the
735// argument under the rules for a large (> 128 bytes) composite.
736bool AArch64ABIInfo::passAsPureScalableType(
737 QualType Ty, unsigned &NVec, unsigned &NPred,
738 SmallVectorImpl<llvm::Type *> &CoerceToSeq) const {
739 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(T: Ty)) {
740 uint64_t NElt = AT->getZExtSize();
741 if (NElt == 0)
742 return false;
743
744 unsigned NV = 0, NP = 0;
745 SmallVector<llvm::Type *> EltCoerceToSeq;
746 if (!passAsPureScalableType(Ty: AT->getElementType(), NVec&: NV, NPred&: NP, CoerceToSeq&: EltCoerceToSeq))
747 return false;
748
749 if (CoerceToSeq.size() + NElt * EltCoerceToSeq.size() > 12)
750 return false;
751
752 for (uint64_t I = 0; I < NElt; ++I)
753 llvm::append_range(C&: CoerceToSeq, R&: EltCoerceToSeq);
754
755 NVec += NElt * NV;
756 NPred += NElt * NP;
757 return true;
758 }
759
760 if (const RecordType *RT = Ty->getAsCanonical<RecordType>()) {
761 // If the record cannot be passed in registers, then it's not a PST.
762 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, CXXABI&: getCXXABI());
763 RAA != CGCXXABI::RAA_Default)
764 return false;
765
766 // Pure scalable types are never unions and never contain unions.
767 const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf();
768 if (RD->isUnion())
769 return false;
770
771 // If this is a C++ record, check the bases.
772 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
773 for (const auto &I : CXXRD->bases()) {
774 if (isEmptyRecord(Context&: getContext(), T: I.getType(), AllowArrays: true))
775 continue;
776 if (!passAsPureScalableType(Ty: I.getType(), NVec, NPred, CoerceToSeq))
777 return false;
778 }
779 }
780
781 // Check members.
782 for (const auto *FD : RD->fields()) {
783 QualType FT = FD->getType();
784 if (isEmptyField(Context&: getContext(), FD, /* AllowArrays */ true))
785 continue;
786 if (!passAsPureScalableType(Ty: FT, NVec, NPred, CoerceToSeq))
787 return false;
788 }
789
790 return true;
791 }
792
793 if (const auto *VT = Ty->getAs<VectorType>()) {
794 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
795 ++NPred;
796 if (CoerceToSeq.size() + 1 > 12)
797 return false;
798 CoerceToSeq.push_back(Elt: convertFixedToScalableVectorType(VT));
799 return true;
800 }
801
802 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
803 ++NVec;
804 if (CoerceToSeq.size() + 1 > 12)
805 return false;
806 CoerceToSeq.push_back(Elt: convertFixedToScalableVectorType(VT));
807 return true;
808 }
809
810 return false;
811 }
812
813 if (!Ty->isBuiltinType())
814 return false;
815
816 bool isPredicate;
817 switch (Ty->castAs<BuiltinType>()->getKind()) {
818#define SVE_VECTOR_TYPE(Name, MangledName, Id, SingletonId) \
819 case BuiltinType::Id: \
820 isPredicate = false; \
821 break;
822#define SVE_PREDICATE_TYPE(Name, MangledName, Id, SingletonId) \
823 case BuiltinType::Id: \
824 isPredicate = true; \
825 break;
826#include "clang/Basic/AArch64ACLETypes.def"
827 default:
828 return false;
829 }
830
831 ASTContext::BuiltinVectorTypeInfo Info =
832 getContext().getBuiltinVectorTypeInfo(VecTy: cast<BuiltinType>(Val&: Ty));
833 assert(Info.NumVectors > 0 && Info.NumVectors <= 4 &&
834 "Expected 1, 2, 3 or 4 vectors!");
835 if (isPredicate)
836 NPred += Info.NumVectors;
837 else
838 NVec += Info.NumVectors;
839 llvm::Type *EltTy = Info.ElementType->isMFloat8Type()
840 ? llvm::Type::getInt8Ty(C&: getVMContext())
841 : CGT.ConvertType(T: Info.ElementType);
842 auto *VTy = llvm::ScalableVectorType::get(ElementType: EltTy, MinNumElts: Info.EC.getKnownMinValue());
843
844 if (CoerceToSeq.size() + Info.NumVectors > 12)
845 return false;
846 std::fill_n(first: std::back_inserter(x&: CoerceToSeq), n: Info.NumVectors, value: VTy);
847
848 return true;
849}
850
851// Expand an LLVM IR type into a sequence with a element for each non-struct,
852// non-array member of the type, with the exception of the padding types, which
853// are retained.
854void AArch64ABIInfo::flattenType(
855 llvm::Type *Ty, SmallVectorImpl<llvm::Type *> &Flattened) const {
856
857 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType: Ty)) {
858 Flattened.push_back(Elt: Ty);
859 return;
860 }
861
862 if (const auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty)) {
863 uint64_t NElt = AT->getNumElements();
864 if (NElt == 0)
865 return;
866
867 SmallVector<llvm::Type *> EltFlattened;
868 flattenType(Ty: AT->getElementType(), Flattened&: EltFlattened);
869
870 for (uint64_t I = 0; I < NElt; ++I)
871 llvm::append_range(C&: Flattened, R&: EltFlattened);
872 return;
873 }
874
875 if (const auto *ST = dyn_cast<llvm::StructType>(Val: Ty)) {
876 for (auto *ET : ST->elements())
877 flattenType(Ty: ET, Flattened);
878 return;
879 }
880
881 Flattened.push_back(Elt: Ty);
882}
883
884RValue AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
885 CodeGenFunction &CGF, AArch64ABIKind Kind,
886 AggValueSlot Slot) const {
887 // These numbers are not used for variadic arguments, hence it doesn't matter
888 // they don't retain their values across multiple calls to
889 // `classifyArgumentType` here.
890 unsigned NSRN = 0, NPRN = 0;
891 ABIArgInfo AI =
892 classifyArgumentType(Ty, /*IsVariadicFn=*/true, /* IsNamedArg */ false,
893 CallingConvention: CGF.CurFnInfo->getCallingConvention(), NSRN, NPRN);
894 // Empty records are ignored for parameter passing purposes.
895 if (AI.isIgnore())
896 return Slot.asRValue();
897
898 bool IsIndirect = AI.isIndirect();
899
900 llvm::Type *BaseTy = CGF.ConvertType(T: Ty);
901 if (IsIndirect)
902 BaseTy = llvm::PointerType::getUnqual(C&: BaseTy->getContext());
903 else if (AI.getCoerceToType())
904 BaseTy = AI.getCoerceToType();
905
906 unsigned NumRegs = 1;
907 if (llvm::ArrayType *ArrTy = dyn_cast<llvm::ArrayType>(Val: BaseTy)) {
908 BaseTy = ArrTy->getElementType();
909 NumRegs = ArrTy->getNumElements();
910 }
911 bool IsFPR =
912 !isSoftFloat() && (BaseTy->isFloatingPointTy() || BaseTy->isVectorTy());
913
914 // The AArch64 va_list type and handling is specified in the Procedure Call
915 // Standard, section B.4:
916 //
917 // struct {
918 // void *__stack;
919 // void *__gr_top;
920 // void *__vr_top;
921 // int __gr_offs;
922 // int __vr_offs;
923 // };
924
925 llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock(name: "vaarg.maybe_reg");
926 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock(name: "vaarg.in_reg");
927 llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock(name: "vaarg.on_stack");
928 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "vaarg.end");
929
930 CharUnits TySize = getContext().getTypeSizeInChars(T: Ty);
931 CharUnits TyAlign = getContext().getTypeUnadjustedAlignInChars(T: Ty);
932
933 Address reg_offs_p = Address::invalid();
934 llvm::Value *reg_offs = nullptr;
935 int reg_top_index;
936 int RegSize = IsIndirect ? 8 : TySize.getQuantity();
937 if (!IsFPR) {
938 // 3 is the field number of __gr_offs
939 reg_offs_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 3, Name: "gr_offs_p");
940 reg_offs = CGF.Builder.CreateLoad(Addr: reg_offs_p, Name: "gr_offs");
941 reg_top_index = 1; // field number for __gr_top
942 RegSize = llvm::alignTo(Value: RegSize, Align: 8);
943 } else {
944 // 4 is the field number of __vr_offs.
945 reg_offs_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 4, Name: "vr_offs_p");
946 reg_offs = CGF.Builder.CreateLoad(Addr: reg_offs_p, Name: "vr_offs");
947 reg_top_index = 2; // field number for __vr_top
948 RegSize = 16 * NumRegs;
949 }
950
951 //=======================================
952 // Find out where argument was passed
953 //=======================================
954
955 // If reg_offs >= 0 we're already using the stack for this type of
956 // argument. We don't want to keep updating reg_offs (in case it overflows,
957 // though anyone passing 2GB of arguments, each at most 16 bytes, deserves
958 // whatever they get).
959 llvm::Value *UsingStack = nullptr;
960 UsingStack = CGF.Builder.CreateICmpSGE(
961 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
962
963 CGF.Builder.CreateCondBr(Cond: UsingStack, True: OnStackBlock, False: MaybeRegBlock);
964
965 // Otherwise, at least some kind of argument could go in these registers, the
966 // question is whether this particular type is too big.
967 CGF.EmitBlock(BB: MaybeRegBlock);
968
969 // Integer arguments may need to correct register alignment (for example a
970 // "struct { __int128 a; };" gets passed in x_2N, x_{2N+1}). In this case we
971 // align __gr_offs to calculate the potential address.
972 if (!IsFPR && !IsIndirect && TyAlign.getQuantity() > 8) {
973 int Align = TyAlign.getQuantity();
974
975 reg_offs = CGF.Builder.CreateAdd(
976 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: Align - 1),
977 Name: "align_regoffs");
978 reg_offs = CGF.Builder.CreateAnd(
979 LHS: reg_offs, RHS: llvm::ConstantInt::getSigned(Ty: CGF.Int32Ty, V: -Align),
980 Name: "aligned_regoffs");
981 }
982
983 // Update the gr_offs/vr_offs pointer for next call to va_arg on this va_list.
984 // The fact that this is done unconditionally reflects the fact that
985 // allocating an argument to the stack also uses up all the remaining
986 // registers of the appropriate kind.
987 llvm::Value *NewOffset = nullptr;
988 NewOffset = CGF.Builder.CreateAdd(
989 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: RegSize), Name: "new_reg_offs");
990 CGF.Builder.CreateStore(Val: NewOffset, Addr: reg_offs_p);
991
992 // Now we're in a position to decide whether this argument really was in
993 // registers or not.
994 llvm::Value *InRegs = nullptr;
995 InRegs = CGF.Builder.CreateICmpSLE(
996 LHS: NewOffset, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0), Name: "inreg");
997
998 CGF.Builder.CreateCondBr(Cond: InRegs, True: InRegBlock, False: OnStackBlock);
999
1000 //=======================================
1001 // Argument was in registers
1002 //=======================================
1003
1004 // Now we emit the code for if the argument was originally passed in
1005 // registers. First start the appropriate block:
1006 CGF.EmitBlock(BB: InRegBlock);
1007
1008 llvm::Value *reg_top = nullptr;
1009 Address reg_top_p =
1010 CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: reg_top_index, Name: "reg_top_p");
1011 reg_top = CGF.Builder.CreateLoad(Addr: reg_top_p, Name: "reg_top");
1012 Address BaseAddr(CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: reg_top, IdxList: reg_offs),
1013 CGF.Int8Ty, CharUnits::fromQuantity(Quantity: IsFPR ? 16 : 8));
1014 Address RegAddr = Address::invalid();
1015 llvm::Type *MemTy = CGF.ConvertTypeForMem(T: Ty), *ElementTy = MemTy;
1016
1017 if (IsIndirect) {
1018 // If it's been passed indirectly (actually a struct), whatever we find from
1019 // stored registers or on the stack will actually be a struct **.
1020 MemTy = llvm::PointerType::getUnqual(C&: MemTy->getContext());
1021 }
1022
1023 const Type *Base = nullptr;
1024 uint64_t NumMembers = 0;
1025 bool IsHFA = isHomogeneousAggregate(Ty, Base, Members&: NumMembers);
1026 if (IsHFA && NumMembers > 1) {
1027 // Homogeneous aggregates passed in registers will have their elements split
1028 // and stored 16-bytes apart regardless of size (they're notionally in qN,
1029 // qN+1, ...). We reload and store into a temporary local variable
1030 // contiguously.
1031 assert(!IsIndirect && "Homogeneous aggregates should be passed directly");
1032 auto BaseTyInfo = getContext().getTypeInfoInChars(T: QualType(Base, 0));
1033 llvm::Type *BaseTy = CGF.ConvertType(T: QualType(Base, 0));
1034 llvm::Type *HFATy = llvm::ArrayType::get(ElementType: BaseTy, NumElements: NumMembers);
1035 Address Tmp = CGF.CreateTempAlloca(Ty: HFATy,
1036 align: std::max(a: TyAlign, b: BaseTyInfo.Align));
1037
1038 // On big-endian platforms, the value will be right-aligned in its slot.
1039 int Offset = 0;
1040 if (CGF.CGM.getDataLayout().isBigEndian() &&
1041 BaseTyInfo.Width.getQuantity() < 16)
1042 Offset = 16 - BaseTyInfo.Width.getQuantity();
1043
1044 for (unsigned i = 0; i < NumMembers; ++i) {
1045 CharUnits BaseOffset = CharUnits::fromQuantity(Quantity: 16 * i + Offset);
1046 Address LoadAddr =
1047 CGF.Builder.CreateConstInBoundsByteGEP(Addr: BaseAddr, Offset: BaseOffset);
1048 LoadAddr = LoadAddr.withElementType(ElemTy: BaseTy);
1049
1050 Address StoreAddr = CGF.Builder.CreateConstArrayGEP(Addr: Tmp, Index: i);
1051
1052 llvm::Value *Elem = CGF.Builder.CreateLoad(Addr: LoadAddr);
1053 CGF.Builder.CreateStore(Val: Elem, Addr: StoreAddr);
1054 }
1055
1056 RegAddr = Tmp.withElementType(ElemTy: MemTy);
1057 } else {
1058 // Otherwise the object is contiguous in memory.
1059
1060 // It might be right-aligned in its slot.
1061 CharUnits SlotSize = BaseAddr.getAlignment();
1062 if (CGF.CGM.getDataLayout().isBigEndian() && !IsIndirect &&
1063 (IsHFA || !isAggregateTypeForABI(T: Ty)) &&
1064 TySize < SlotSize) {
1065 CharUnits Offset = SlotSize - TySize;
1066 BaseAddr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: BaseAddr, Offset);
1067 }
1068
1069 RegAddr = BaseAddr.withElementType(ElemTy: MemTy);
1070 }
1071
1072 CGF.EmitBranch(Block: ContBlock);
1073
1074 //=======================================
1075 // Argument was on the stack
1076 //=======================================
1077 CGF.EmitBlock(BB: OnStackBlock);
1078
1079 Address stack_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 0, Name: "stack_p");
1080 llvm::Value *OnStackPtr = CGF.Builder.CreateLoad(Addr: stack_p, Name: "stack");
1081
1082 // Again, stack arguments may need realignment. In this case both integer and
1083 // floating-point ones might be affected.
1084 if (!IsIndirect && TyAlign.getQuantity() > 8) {
1085 OnStackPtr = emitRoundPointerUpToAlignment(CGF, Ptr: OnStackPtr, Align: TyAlign);
1086 }
1087 Address OnStackAddr = Address(OnStackPtr, CGF.Int8Ty,
1088 std::max(a: CharUnits::fromQuantity(Quantity: 8), b: TyAlign));
1089
1090 // All stack slots are multiples of 8 bytes.
1091 CharUnits StackSlotSize = CharUnits::fromQuantity(Quantity: 8);
1092 CharUnits StackSize;
1093 if (IsIndirect)
1094 StackSize = StackSlotSize;
1095 else
1096 StackSize = TySize.alignTo(Align: StackSlotSize);
1097
1098 llvm::Value *StackSizeC = CGF.Builder.getSize(N: StackSize);
1099 llvm::Value *NewStack = CGF.Builder.CreateInBoundsGEP(
1100 Ty: CGF.Int8Ty, Ptr: OnStackPtr, IdxList: StackSizeC, Name: "new_stack");
1101
1102 // Write the new value of __stack for the next call to va_arg
1103 CGF.Builder.CreateStore(Val: NewStack, Addr: stack_p);
1104
1105 if (CGF.CGM.getDataLayout().isBigEndian() && !isAggregateTypeForABI(T: Ty) &&
1106 TySize < StackSlotSize) {
1107 CharUnits Offset = StackSlotSize - TySize;
1108 OnStackAddr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: OnStackAddr, Offset);
1109 }
1110
1111 OnStackAddr = OnStackAddr.withElementType(ElemTy: MemTy);
1112
1113 CGF.EmitBranch(Block: ContBlock);
1114
1115 //=======================================
1116 // Tidy up
1117 //=======================================
1118 CGF.EmitBlock(BB: ContBlock);
1119
1120 Address ResAddr = emitMergePHI(CGF, Addr1: RegAddr, Block1: InRegBlock, Addr2: OnStackAddr,
1121 Block2: OnStackBlock, Name: "vaargs.addr");
1122
1123 if (IsIndirect)
1124 return CGF.EmitLoadOfAnyValue(
1125 V: CGF.MakeAddrLValue(
1126 Addr: Address(CGF.Builder.CreateLoad(Addr: ResAddr, Name: "vaarg.addr"), ElementTy,
1127 TyAlign),
1128 T: Ty),
1129 Slot);
1130
1131 return CGF.EmitLoadOfAnyValue(V: CGF.MakeAddrLValue(Addr: ResAddr, T: Ty), Slot);
1132}
1133
1134RValue AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty,
1135 CodeGenFunction &CGF,
1136 AggValueSlot Slot) const {
1137 // The backend's lowering doesn't support va_arg for aggregates or
1138 // illegal vector types. Lower VAArg here for these cases and use
1139 // the LLVM va_arg instruction for everything else.
1140 if (!isAggregateTypeForABI(T: Ty) && !isIllegalVectorType(Ty))
1141 return CGF.EmitLoadOfAnyValue(
1142 V: CGF.MakeAddrLValue(
1143 Addr: EmitVAArgInstr(CGF, VAListAddr, Ty, AI: ABIArgInfo::getDirect()), T: Ty),
1144 Slot);
1145
1146 uint64_t PointerSize = getTarget().getPointerWidth(AddrSpace: LangAS::Default) / 8;
1147 CharUnits SlotSize = CharUnits::fromQuantity(Quantity: PointerSize);
1148
1149 // Empty records are ignored for parameter passing purposes.
1150 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
1151 return Slot.asRValue();
1152
1153 // The size of the actual thing passed, which might end up just
1154 // being a pointer for indirect types.
1155 auto TyInfo = getContext().getTypeInfoInChars(T: Ty);
1156
1157 // Arguments bigger than 16 bytes which aren't homogeneous
1158 // aggregates should be passed indirectly.
1159 bool IsIndirect = false;
1160 if (TyInfo.Width.getQuantity() > 16) {
1161 const Type *Base = nullptr;
1162 uint64_t Members = 0;
1163 IsIndirect = !isHomogeneousAggregate(Ty, Base, Members);
1164 }
1165
1166 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect, ValueInfo: TyInfo, SlotSizeAndAlign: SlotSize,
1167 /*AllowHigherAlign*/ true, Slot);
1168}
1169
1170RValue AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
1171 QualType Ty, AggValueSlot Slot) const {
1172 bool AllowHigherAlign = false;
1173 bool IsIndirect = false;
1174
1175 if (getTarget().getTriple().isWindowsArm64EC()) {
1176 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
1177 // not 1, 2, 4, or 8 bytes, must be passed by reference."
1178 uint64_t Width = getContext().getTypeSize(T: Ty);
1179 IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Value: Width);
1180 } else {
1181 // E.g. __int128 when passed is aligned to 16 bytes, so it must be read
1182 // with the same alignment.
1183 AllowHigherAlign = true;
1184
1185 // Composites larger than 16 bytes are passed by reference.
1186 if (isAggregateTypeForABI(T: Ty) && getContext().getTypeSize(T: Ty) > 128)
1187 IsIndirect = true;
1188 }
1189
1190 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
1191 ValueInfo: CGF.getContext().getTypeInfoInChars(T: Ty),
1192 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 8), AllowHigherAlign, Slot);
1193}
1194
1195// Report an error if an argument or return value of type Ty would need to be
1196// passed in a floating-point register.
1197static void diagnoseIfNeedsFPReg(DiagnosticsEngine &Diags,
1198 const StringRef ABIName,
1199 const AArch64ABIInfo &ABIInfo,
1200 const QualType &Ty, const NamedDecl *D,
1201 SourceLocation loc) {
1202 const Type *HABase = nullptr;
1203 uint64_t HAMembers = 0;
1204 if (Ty->isFloatingType() || Ty->isVectorType() ||
1205 ABIInfo.isHomogeneousAggregate(Ty, Base&: HABase, Members&: HAMembers)) {
1206 Diags.Report(Loc: loc, DiagID: diag::err_target_unsupported_type_for_abi)
1207 << D->getDeclName() << Ty << ABIName;
1208 }
1209}
1210
1211// If we are using a hard-float ABI, but do not have floating point registers,
1212// then report an error for any function arguments or returns which would be
1213// passed in floating-pint registers.
1214void AArch64TargetCodeGenInfo::checkFunctionABI(
1215 CodeGenModule &CGM, const FunctionDecl *FuncDecl) const {
1216 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
1217 const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();
1218
1219 if (!TI.hasFeature(Feature: "fp") && !ABIInfo.isSoftFloat()) {
1220 diagnoseIfNeedsFPReg(Diags&: CGM.getDiags(), ABIName: TI.getABI(), ABIInfo,
1221 Ty: FuncDecl->getReturnType(), D: FuncDecl,
1222 loc: FuncDecl->getLocation());
1223 for (ParmVarDecl *PVD : FuncDecl->parameters()) {
1224 diagnoseIfNeedsFPReg(Diags&: CGM.getDiags(), ABIName: TI.getABI(), ABIInfo, Ty: PVD->getType(),
1225 D: PVD, loc: FuncDecl->getLocation());
1226 }
1227 }
1228}
1229
1230void AArch64TargetCodeGenInfo::checkFunctionCallABIStreaming(
1231 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
1232 const FunctionDecl *Callee) const {
1233 if (!Caller || !Callee || !Callee->hasAttr<AlwaysInlineAttr>())
1234 return;
1235
1236 CodeGenUtils::ArmSMEInlinability Inlinability =
1237 CodeGenUtils::getArmSMEInlinability(Caller, Callee);
1238
1239 if ((Inlinability &
1240 CodeGenUtils::ArmSMEInlinability::IncompatibleStreamingModes) !=
1241 CodeGenUtils::ArmSMEInlinability::Ok)
1242 CGM.getDiags().Report(
1243 Loc: CallLoc,
1244 DiagID: (Inlinability &
1245 CodeGenUtils::ArmSMEInlinability::ErrorIncompatibleStreamingModes) ==
1246 CodeGenUtils::ArmSMEInlinability::
1247 ErrorIncompatibleStreamingModes
1248 ? diag::err_function_always_inline_attribute_mismatch
1249 : diag::warn_function_always_inline_attribute_mismatch)
1250 << Caller->getDeclName() << Callee->getDeclName() << "streaming";
1251
1252 if ((Inlinability &
1253 CodeGenUtils::ArmSMEInlinability::ErrorCalleeRequiresNewZA) ==
1254 CodeGenUtils::ArmSMEInlinability::ErrorCalleeRequiresNewZA)
1255 CGM.getDiags().Report(Loc: CallLoc, DiagID: diag::err_function_always_inline_new_za)
1256 << Callee->getDeclName();
1257
1258 if ((Inlinability &
1259 CodeGenUtils::ArmSMEInlinability::ErrorCalleeRequiresNewZT0) ==
1260 CodeGenUtils::ArmSMEInlinability::ErrorCalleeRequiresNewZT0)
1261 CGM.getDiags().Report(Loc: CallLoc, DiagID: diag::err_function_always_inline_new_zt0)
1262 << Callee->getDeclName();
1263}
1264
1265// If the target does not have floating-point registers, but we are using a
1266// hard-float ABI, there is no way to pass floating-point, vector or HFA values
1267// to functions, so we report an error.
1268void AArch64TargetCodeGenInfo::checkFunctionCallABISoftFloat(
1269 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
1270 const FunctionDecl *Callee, const CallArgList &Args,
1271 QualType ReturnType) const {
1272 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
1273 const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();
1274
1275 if (!Caller || TI.hasFeature(Feature: "fp") || ABIInfo.isSoftFloat())
1276 return;
1277
1278 diagnoseIfNeedsFPReg(Diags&: CGM.getDiags(), ABIName: TI.getABI(), ABIInfo, Ty: ReturnType,
1279 D: Callee ? Callee : Caller, loc: CallLoc);
1280
1281 for (const CallArg &Arg : Args)
1282 diagnoseIfNeedsFPReg(Diags&: CGM.getDiags(), ABIName: TI.getABI(), ABIInfo, Ty: Arg.getType(),
1283 D: Callee ? Callee : Caller, loc: CallLoc);
1284}
1285
1286void AArch64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM,
1287 SourceLocation CallLoc,
1288 const FunctionDecl *Caller,
1289 const FunctionDecl *Callee,
1290 const CallArgList &Args,
1291 QualType ReturnType) const {
1292 checkFunctionCallABIStreaming(CGM, CallLoc, Caller, Callee);
1293 checkFunctionCallABISoftFloat(CGM, CallLoc, Caller, Callee, Args, ReturnType);
1294}
1295
1296bool AArch64TargetCodeGenInfo::wouldInliningViolateFunctionCallABI(
1297 const FunctionDecl *Caller, const FunctionDecl *Callee) const {
1298 return Caller && Callee &&
1299 CodeGenUtils::getArmSMEInlinability(Caller, Callee) !=
1300 CodeGenUtils::ArmSMEInlinability::Ok;
1301}
1302
1303void AArch64ABIInfo::appendAttributeMangling(TargetClonesAttr *Attr,
1304 unsigned Index,
1305 raw_ostream &Out) const {
1306 appendAttributeMangling(AttrStr: Attr->getFeatureStr(Index), Out);
1307}
1308
1309void AArch64ABIInfo::appendAttributeMangling(StringRef AttrStr,
1310 raw_ostream &Out) const {
1311 if (AttrStr == "default") {
1312 Out << ".default";
1313 return;
1314 }
1315
1316 Out << "._";
1317 SmallVector<StringRef, 8> Features;
1318 AttrStr.split(A&: Features, Separator: "+");
1319 for (auto &Feat : Features)
1320 Feat = Feat.trim();
1321
1322 llvm::sort(C&: Features, Comp: [](const StringRef LHS, const StringRef RHS) {
1323 return LHS.compare(RHS) < 0;
1324 });
1325
1326 llvm::SmallDenseSet<StringRef, 8> UniqueFeats;
1327 for (auto &Feat : Features)
1328 if (getTarget().doesFeatureAffectCodeGen(Feature: Feat))
1329 if (auto Ext = llvm::AArch64::parseFMVExtension(Extension: Feat))
1330 if (UniqueFeats.insert(V: Ext->Name).second)
1331 Out << 'M' << Ext->Name;
1332}
1333
1334std::unique_ptr<TargetCodeGenInfo>
1335CodeGen::createAArch64TargetCodeGenInfo(CodeGenModule &CGM,
1336 AArch64ABIKind Kind) {
1337 return std::make_unique<AArch64TargetCodeGenInfo>(args&: CGM, args&: Kind);
1338}
1339
1340std::unique_ptr<TargetCodeGenInfo>
1341CodeGen::createWindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM,
1342 AArch64ABIKind K) {
1343 return std::make_unique<WindowsAArch64TargetCodeGenInfo>(args&: CGM, args&: K);
1344}
1345