1//===- AArch64.cpp --------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ABIInfoImpl.h"
10#include "TargetInfo.h"
11#include "clang/AST/Decl.h"
12#include "clang/Basic/DiagnosticFrontend.h"
13#include "llvm/TargetParser/AArch64TargetParser.h"
14
15using namespace clang;
16using namespace clang::CodeGen;
17
18//===----------------------------------------------------------------------===//
19// AArch64 ABI Implementation
20//===----------------------------------------------------------------------===//
21
22namespace {
23
24class AArch64ABIInfo : public ABIInfo {
25 AArch64ABIKind Kind;
26
27 std::unique_ptr<TargetCodeGenInfo> WinX86_64CodegenInfo;
28
29public:
30 AArch64ABIInfo(CodeGenModule &CGM, AArch64ABIKind Kind)
31 : ABIInfo(CGM.getTypes()), Kind(Kind) {
32 if (getTarget().getTriple().isWindowsArm64EC()) {
33 WinX86_64CodegenInfo =
34 createWinX86_64TargetCodeGenInfo(CGM, AVXLevel: X86AVXABILevel::None);
35 }
36 }
37
38 bool isSoftFloat() const { return Kind == AArch64ABIKind::AAPCSSoft; }
39
40private:
41 AArch64ABIKind getABIKind() const { return Kind; }
42 bool isDarwinPCS() const { return Kind == AArch64ABIKind::DarwinPCS; }
43
44 ABIArgInfo classifyReturnType(QualType RetTy, bool IsVariadicFn) const;
45 ABIArgInfo classifyArgumentType(QualType RetTy, bool IsVariadicFn,
46 bool IsNamedArg, unsigned CallingConvention,
47 unsigned &NSRN, unsigned &NPRN) const;
48 llvm::Type *convertFixedToScalableVectorType(const VectorType *VT) const;
49 ABIArgInfo coerceIllegalVector(QualType Ty, unsigned &NSRN,
50 unsigned &NPRN) const;
51 ABIArgInfo coerceAndExpandPureScalableAggregate(
52 QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
53 const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
54 unsigned &NPRN) const;
55 bool isHomogeneousAggregateBaseType(QualType Ty) const override;
56 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
57 uint64_t Members) const override;
58 bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const override;
59
60 bool isIllegalVectorType(QualType Ty) const;
61
62 bool passAsAggregateType(QualType Ty) const;
63 bool passAsPureScalableType(QualType Ty, unsigned &NV, unsigned &NP,
64 SmallVectorImpl<llvm::Type *> &CoerceToSeq) const;
65
66 void flattenType(llvm::Type *Ty,
67 SmallVectorImpl<llvm::Type *> &Flattened) const;
68
69 void computeInfo(CGFunctionInfo &FI) const override {
70 if (!::classifyReturnType(CXXABI: getCXXABI(), FI, Info: *this))
71 FI.getReturnInfo() =
72 classifyReturnType(RetTy: FI.getReturnType(), IsVariadicFn: FI.isVariadic());
73
74 unsigned ArgNo = 0;
75 unsigned NSRN = 0, NPRN = 0;
76 for (auto &it : FI.arguments()) {
77 const bool IsNamedArg =
78 !FI.isVariadic() || ArgNo < FI.getRequiredArgs().getNumRequiredArgs();
79 ++ArgNo;
80 it.info = classifyArgumentType(RetTy: it.type, IsVariadicFn: FI.isVariadic(), IsNamedArg,
81 CallingConvention: FI.getCallingConvention(), NSRN, NPRN);
82 }
83 }
84
85 RValue EmitDarwinVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,
86 AggValueSlot Slot) const;
87
88 RValue EmitAAPCSVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,
89 AArch64ABIKind Kind, AggValueSlot Slot) const;
90
91 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
92 AggValueSlot Slot) const override {
93 llvm::Type *BaseTy = CGF.ConvertType(T: Ty);
94 if (isa<llvm::ScalableVectorType>(Val: BaseTy))
95 llvm::report_fatal_error(reason: "Passing SVE types to variadic functions is "
96 "currently not supported");
97
98 return Kind == AArch64ABIKind::Win64
99 ? EmitMSVAArg(CGF, VAListAddr, Ty, Slot)
100 : isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF, Slot)
101 : EmitAAPCSVAArg(VAListAddr, Ty, CGF, Kind, Slot);
102 }
103
104 RValue EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
105 AggValueSlot Slot) const override;
106
107 bool allowBFloatArgsAndRet() const override {
108 return getTarget().hasBFloat16Type();
109 }
110
111 using ABIInfo::appendAttributeMangling;
112 void appendAttributeMangling(TargetClonesAttr *Attr, unsigned Index,
113 raw_ostream &Out) const override;
114 void appendAttributeMangling(StringRef AttrStr,
115 raw_ostream &Out) const override;
116};
117
118class AArch64SwiftABIInfo : public SwiftABIInfo {
119public:
120 explicit AArch64SwiftABIInfo(CodeGenTypes &CGT)
121 : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/true) {}
122
123 bool isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy,
124 unsigned NumElts) const override;
125};
126
127class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
128public:
129 AArch64TargetCodeGenInfo(CodeGenModule &CGM, AArch64ABIKind Kind)
130 : TargetCodeGenInfo(std::make_unique<AArch64ABIInfo>(args&: CGM, args&: Kind)) {
131 SwiftInfo = std::make_unique<AArch64SwiftABIInfo>(args&: CGM.getTypes());
132 }
133
134 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
135 return "mov\tfp, fp\t\t// marker for objc_retainAutoreleaseReturnValue";
136 }
137
138 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
139 return 31;
140 }
141
142 bool doesReturnSlotInterfereWithArgs() const override { return false; }
143
144 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
145 CodeGen::CodeGenModule &CGM) const override {
146 auto *Fn = dyn_cast<llvm::Function>(Val: GV);
147 if (!Fn)
148 return;
149
150 const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
151 TargetInfo::BranchProtectionInfo BPI(CGM.getLangOpts());
152
153 if (FD && FD->hasAttr<TargetAttr>()) {
154 const auto *TA = FD->getAttr<TargetAttr>();
155 ParsedTargetAttr Attr =
156 CGM.getTarget().parseTargetAttr(Str: TA->getFeaturesStr());
157 if (!Attr.BranchProtection.empty()) {
158 StringRef Error;
159 (void)CGM.getTarget().validateBranchProtection(
160 Spec: Attr.BranchProtection, Arch: Attr.CPU, BPI, LO: CGM.getLangOpts(), Err&: Error);
161 assert(Error.empty());
162 }
163 }
164 setBranchProtectionFnAttributes(BPI, F&: *Fn);
165 setPointerAuthFnAttributes(Opts: CGM.getCodeGenOpts().PointerAuth, F&: *Fn);
166 }
167
168 bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF,
169 llvm::Type *Ty) const override {
170 if (CGF.getTarget().hasFeature(Feature: "ls64")) {
171 auto *ST = dyn_cast<llvm::StructType>(Val: Ty);
172 if (ST && ST->getNumElements() == 1) {
173 auto *AT = dyn_cast<llvm::ArrayType>(Val: ST->getElementType(N: 0));
174 if (AT && AT->getNumElements() == 8 &&
175 AT->getElementType()->isIntegerTy(BitWidth: 64))
176 return true;
177 }
178 }
179 return TargetCodeGenInfo::isScalarizableAsmOperand(CGF, Ty);
180 }
181
182 void checkFunctionABI(CodeGenModule &CGM,
183 const FunctionDecl *Decl) const override;
184
185 void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
186 const FunctionDecl *Caller,
187 const FunctionDecl *Callee, const CallArgList &Args,
188 QualType ReturnType) const override;
189
190 bool wouldInliningViolateFunctionCallABI(
191 const FunctionDecl *Caller, const FunctionDecl *Callee) const override;
192
193private:
194 // Diagnose calls between functions with incompatible Streaming SVE
195 // attributes.
196 void checkFunctionCallABIStreaming(CodeGenModule &CGM, SourceLocation CallLoc,
197 const FunctionDecl *Caller,
198 const FunctionDecl *Callee) const;
199 // Diagnose calls which must pass arguments in floating-point registers when
200 // the selected target does not have floating-point registers.
201 void checkFunctionCallABISoftFloat(CodeGenModule &CGM, SourceLocation CallLoc,
202 const FunctionDecl *Caller,
203 const FunctionDecl *Callee,
204 const CallArgList &Args,
205 QualType ReturnType) const;
206};
207
208class WindowsAArch64TargetCodeGenInfo : public AArch64TargetCodeGenInfo {
209public:
210 WindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM, AArch64ABIKind K)
211 : AArch64TargetCodeGenInfo(CGM, K) {}
212
213 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
214 CodeGen::CodeGenModule &CGM) const override;
215
216 void getDependentLibraryOption(llvm::StringRef Lib,
217 llvm::SmallString<24> &Opt) const override {
218 Opt = "/DEFAULTLIB:" + qualifyWindowsLibrary(Lib);
219 }
220
221 void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value,
222 llvm::SmallString<32> &Opt) const override {
223 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
224 }
225};
226
227void WindowsAArch64TargetCodeGenInfo::setTargetAttributes(
228 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
229 AArch64TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
230 if (GV->isDeclaration())
231 return;
232 addStackProbeTargetAttributes(D, GV, CGM);
233}
234}
235
236llvm::Type *
237AArch64ABIInfo::convertFixedToScalableVectorType(const VectorType *VT) const {
238 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
239
240 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
241 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
242 BuiltinType::UChar &&
243 "unexpected builtin type for SVE predicate!");
244 return llvm::ScalableVectorType::get(ElementType: llvm::Type::getInt1Ty(C&: getVMContext()),
245 MinNumElts: 16);
246 }
247
248 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
249 const auto *BT = VT->getElementType()->castAs<BuiltinType>();
250 switch (BT->getKind()) {
251 default:
252 llvm_unreachable("unexpected builtin type for SVE vector!");
253
254 case BuiltinType::SChar:
255 case BuiltinType::UChar:
256 case BuiltinType::MFloat8:
257 return llvm::ScalableVectorType::get(
258 ElementType: llvm::Type::getInt8Ty(C&: getVMContext()), MinNumElts: 16);
259
260 case BuiltinType::Short:
261 case BuiltinType::UShort:
262 return llvm::ScalableVectorType::get(
263 ElementType: llvm::Type::getInt16Ty(C&: getVMContext()), MinNumElts: 8);
264
265 case BuiltinType::Int:
266 case BuiltinType::UInt:
267 return llvm::ScalableVectorType::get(
268 ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), MinNumElts: 4);
269
270 case BuiltinType::Long:
271 case BuiltinType::ULong:
272 return llvm::ScalableVectorType::get(
273 ElementType: llvm::Type::getInt64Ty(C&: getVMContext()), MinNumElts: 2);
274
275 case BuiltinType::Half:
276 return llvm::ScalableVectorType::get(
277 ElementType: llvm::Type::getHalfTy(C&: getVMContext()), MinNumElts: 8);
278
279 case BuiltinType::Float:
280 return llvm::ScalableVectorType::get(
281 ElementType: llvm::Type::getFloatTy(C&: getVMContext()), MinNumElts: 4);
282
283 case BuiltinType::Double:
284 return llvm::ScalableVectorType::get(
285 ElementType: llvm::Type::getDoubleTy(C&: getVMContext()), MinNumElts: 2);
286
287 case BuiltinType::BFloat16:
288 return llvm::ScalableVectorType::get(
289 ElementType: llvm::Type::getBFloatTy(C&: getVMContext()), MinNumElts: 8);
290 }
291 }
292
293 llvm_unreachable("expected fixed-length SVE vector");
294}
295
296ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty, unsigned &NSRN,
297 unsigned &NPRN) const {
298 assert(Ty->isVectorType() && "expected vector type!");
299
300 const auto *VT = Ty->castAs<VectorType>();
301 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
302 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
303 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
304 BuiltinType::UChar &&
305 "unexpected builtin type for SVE predicate!");
306 NPRN = std::min(a: NPRN + 1, b: 4u);
307 return ABIArgInfo::getDirect(T: llvm::ScalableVectorType::get(
308 ElementType: llvm::Type::getInt1Ty(C&: getVMContext()), MinNumElts: 16));
309 }
310
311 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
312 NSRN = std::min(a: NSRN + 1, b: 8u);
313 return ABIArgInfo::getDirect(T: convertFixedToScalableVectorType(VT));
314 }
315
316 uint64_t Size = getContext().getTypeSize(T: Ty);
317 // Android promotes <2 x i8> to i16, not i32
318 if ((isAndroid() || isOHOSFamily()) && (Size <= 16)) {
319 llvm::Type *ResType = llvm::Type::getInt16Ty(C&: getVMContext());
320 return ABIArgInfo::getDirect(T: ResType);
321 }
322 if (Size <= 32) {
323 llvm::Type *ResType = llvm::Type::getInt32Ty(C&: getVMContext());
324 return ABIArgInfo::getDirect(T: ResType);
325 }
326 if (Size == 64) {
327 NSRN = std::min(a: NSRN + 1, b: 8u);
328 auto *ResType =
329 llvm::FixedVectorType::get(ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), NumElts: 2);
330 return ABIArgInfo::getDirect(T: ResType);
331 }
332 if (Size == 128) {
333 NSRN = std::min(a: NSRN + 1, b: 8u);
334 auto *ResType =
335 llvm::FixedVectorType::get(ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), NumElts: 4);
336 return ABIArgInfo::getDirect(T: ResType);
337 }
338
339 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
340 /*ByVal=*/false);
341}
342
343ABIArgInfo AArch64ABIInfo::coerceAndExpandPureScalableAggregate(
344 QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
345 const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
346 unsigned &NPRN) const {
347 if (!IsNamedArg || NSRN + NVec > 8 || NPRN + NPred > 4)
348 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
349 /*ByVal=*/false);
350 NSRN += NVec;
351 NPRN += NPred;
352
353 // Handle SVE vector tuples.
354 if (Ty->isSVESizelessBuiltinType())
355 return ABIArgInfo::getDirect();
356
357 llvm::Type *UnpaddedCoerceToType =
358 UnpaddedCoerceToSeq.size() == 1
359 ? UnpaddedCoerceToSeq[0]
360 : llvm::StructType::get(Context&: CGT.getLLVMContext(), Elements: UnpaddedCoerceToSeq,
361 isPacked: true);
362
363 SmallVector<llvm::Type *> CoerceToSeq;
364 flattenType(Ty: CGT.ConvertType(T: Ty), Flattened&: CoerceToSeq);
365 auto *CoerceToType =
366 llvm::StructType::get(Context&: CGT.getLLVMContext(), Elements: CoerceToSeq, isPacked: false);
367
368 return ABIArgInfo::getCoerceAndExpand(coerceToType: CoerceToType, unpaddedCoerceToType: UnpaddedCoerceToType);
369}
370
371ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadicFn,
372 bool IsNamedArg,
373 unsigned CallingConvention,
374 unsigned &NSRN,
375 unsigned &NPRN) const {
376 Ty = useFirstFieldIfTransparentUnion(Ty);
377
378 if (IsVariadicFn && getTarget().getTriple().isWindowsArm64EC()) {
379 // Arm64EC varargs functions use the x86_64 classification rules,
380 // not the AArch64 ABI rules.
381 return WinX86_64CodegenInfo->getABIInfo().classifyArgForArm64ECVarArg(
382 Ty, IsNamedArg);
383 }
384
385 // Handle illegal vector types here.
386 if (isIllegalVectorType(Ty))
387 return coerceIllegalVector(Ty, NSRN, NPRN);
388
389 if (!passAsAggregateType(Ty)) {
390 // Treat an enum type as its underlying type.
391 if (const auto *ED = Ty->getAsEnumDecl())
392 Ty = ED->getIntegerType();
393
394 if (const auto *EIT = Ty->getAs<BitIntType>())
395 if (EIT->getNumBits() > 128)
396 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
397 ByVal: false);
398
399 if (Ty->isVectorType())
400 NSRN = std::min(a: NSRN + 1, b: 8u);
401 else if (const auto *BT = Ty->getAs<BuiltinType>()) {
402 if (BT->isFloatingPoint())
403 NSRN = std::min(a: NSRN + 1, b: 8u);
404 else {
405 switch (BT->getKind()) {
406 case BuiltinType::SveBool:
407 case BuiltinType::SveCount:
408 NPRN = std::min(a: NPRN + 1, b: 4u);
409 break;
410 case BuiltinType::SveBoolx2:
411 NPRN = std::min(a: NPRN + 2, b: 4u);
412 break;
413 case BuiltinType::SveBoolx4:
414 NPRN = std::min(a: NPRN + 4, b: 4u);
415 break;
416 case BuiltinType::MFloat8:
417 NSRN = std::min(a: NSRN + 1, b: 8u);
418 break;
419 default:
420 if (BT->isSVESizelessBuiltinType())
421 NSRN = std::min(
422 a: NSRN + getContext().getBuiltinVectorTypeInfo(VecTy: BT).NumVectors,
423 b: 8u);
424 }
425 }
426 }
427
428 return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS()
429 ? ABIArgInfo::getExtend(Ty, T: CGT.ConvertType(T: Ty))
430 : ABIArgInfo::getDirect());
431 }
432
433 // Structures with either a non-trivial destructor or a non-trivial
434 // copy constructor are always indirect.
435 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(T: Ty, CXXABI&: getCXXABI())) {
436 return getNaturalAlignIndirect(
437 Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
438 /*ByVal=*/RAA == CGCXXABI::RAA_DirectInMemory);
439 }
440
441 // Empty records:
442 // AAPCS64 does not say that empty records are ignored as arguments,
443 // but other compilers do so in certain situations, and we copy that behavior.
444 // Those situations are in fact language-mode-specific, which seems really
445 // unfortunate, but it's something we just have to accept. If this doesn't
446 // apply, just fall through to the standard argument-handling path.
447 // Darwin overrides the psABI here to ignore all empty records in all modes.
448 uint64_t Size = getContext().getTypeSize(T: Ty);
449 bool IsEmpty = isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true);
450 if (!Ty->isSVESizelessBuiltinType() && (IsEmpty || Size == 0)) {
451 // Empty records are ignored in C mode, and in C++ on Darwin.
452 if (!getContext().getLangOpts().CPlusPlus || isDarwinPCS())
453 return ABIArgInfo::getIgnore();
454
455 // In C++ mode, arguments which have sizeof() == 0 (which are non-standard
456 // C++) are ignored. This isn't defined by any standard, so we copy GCC's
457 // behaviour here.
458 if (Size == 0)
459 return ABIArgInfo::getIgnore();
460 }
461
462 // Homogeneous Floating-point Aggregates (HFAs) need to be expanded.
463 const Type *Base = nullptr;
464 uint64_t Members = 0;
465 bool IsWin64 = Kind == AArch64ABIKind::Win64 ||
466 CallingConvention == llvm::CallingConv::Win64;
467 bool IsWinVariadic = IsWin64 && IsVariadicFn;
468 // In variadic functions on Windows, all composite types are treated alike,
469 // no special handling of HFAs/HVAs.
470 if (!IsWinVariadic && isHomogeneousAggregate(Ty, Base, Members)) {
471 NSRN = std::min(a: NSRN + Members, b: uint64_t(8));
472 if (Kind != AArch64ABIKind::AAPCS)
473 return ABIArgInfo::getDirect(
474 T: llvm::ArrayType::get(ElementType: CGT.ConvertType(T: QualType(Base, 0)), NumElements: Members));
475
476 // For HFAs/HVAs, cap the argument alignment to 16, otherwise
477 // set it to 8 according to the AAPCS64 document.
478 unsigned Align =
479 getContext().getTypeUnadjustedAlignInChars(T: Ty).getQuantity();
480 Align = (Align >= 16) ? 16 : 8;
481 return ABIArgInfo::getDirect(
482 T: llvm::ArrayType::get(ElementType: CGT.ConvertType(T: QualType(Base, 0)), NumElements: Members), Offset: 0,
483 Padding: nullptr, CanBeFlattened: true, Align);
484 }
485
486 // In AAPCS named arguments of a Pure Scalable Type are passed expanded in
487 // registers, or indirectly if there are not enough registers.
488 if (Kind == AArch64ABIKind::AAPCS) {
489 unsigned NVec = 0, NPred = 0;
490 SmallVector<llvm::Type *> UnpaddedCoerceToSeq;
491 if (passAsPureScalableType(Ty, NV&: NVec, NP&: NPred, CoerceToSeq&: UnpaddedCoerceToSeq) &&
492 (NVec + NPred) > 0)
493 return coerceAndExpandPureScalableAggregate(
494 Ty, IsNamedArg, NVec, NPred, UnpaddedCoerceToSeq, NSRN, NPRN);
495 }
496
497 // Aggregates <= 16 bytes are passed directly in registers or on the stack.
498 if (Size <= 128) {
499 unsigned Alignment;
500 if (Kind == AArch64ABIKind::AAPCS) {
501 Alignment = getContext().getTypeUnadjustedAlign(T: Ty);
502 Alignment = Alignment < 128 ? 64 : 128;
503 } else {
504 Alignment =
505 std::max(a: getContext().getTypeAlign(T: Ty),
506 b: (unsigned)getTarget().getPointerWidth(AddrSpace: LangAS::Default));
507 }
508 Size = llvm::alignTo(Value: Size, Align: Alignment);
509
510 // If the Aggregate is made up of pointers, use an array of pointers for the
511 // coerced type. This prevents having to convert ptr2int->int2ptr through
512 // the call, allowing alias analysis to produce better code.
513 auto ContainsOnlyPointers = [&](const auto &Self, QualType Ty) {
514 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
515 return false;
516 const auto *RD = Ty->getAsRecordDecl();
517 if (!RD)
518 return false;
519 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
520 for (const auto &I : CXXRD->bases())
521 if (!Self(Self, I.getType()))
522 return false;
523 }
524 return all_of(RD->fields(), [&](FieldDecl *FD) {
525 QualType FDTy = FD->getType();
526 if (FDTy->isArrayType())
527 FDTy = getContext().getBaseElementType(QT: FDTy);
528 return (FDTy->isPointerOrReferenceType() &&
529 getContext().getTypeSize(T: FDTy) == 64 &&
530 !FDTy->getPointeeType().hasAddressSpace()) ||
531 Self(Self, FDTy);
532 });
533 };
534
535 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
536 // For aggregates with 16-byte alignment, we use i128.
537 llvm::Type *BaseTy = llvm::Type::getIntNTy(C&: getVMContext(), N: Alignment);
538 if ((Size == 64 || Size == 128) && Alignment == 64 &&
539 ContainsOnlyPointers(ContainsOnlyPointers, Ty))
540 BaseTy = llvm::PointerType::getUnqual(C&: getVMContext());
541 return ABIArgInfo::getDirect(
542 T: Size == Alignment ? BaseTy
543 : llvm::ArrayType::get(ElementType: BaseTy, NumElements: Size / Alignment));
544 }
545
546 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
547 /*ByVal=*/false);
548}
549
550ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,
551 bool IsVariadicFn) const {
552 if (RetTy->isVoidType())
553 return ABIArgInfo::getIgnore();
554
555 if (const auto *VT = RetTy->getAs<VectorType>()) {
556 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
557 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
558 unsigned NSRN = 0, NPRN = 0;
559 return coerceIllegalVector(Ty: RetTy, NSRN, NPRN);
560 }
561 }
562
563 // Large vector types should be returned via memory.
564 if (RetTy->isVectorType() && getContext().getTypeSize(T: RetTy) > 128)
565 return getNaturalAlignIndirect(Ty: RetTy, AddrSpace: getDataLayout().getAllocaAddrSpace());
566
567 if (!passAsAggregateType(Ty: RetTy)) {
568 // Treat an enum type as its underlying type.
569 if (const auto *ED = RetTy->getAsEnumDecl())
570 RetTy = ED->getIntegerType();
571
572 if (const auto *EIT = RetTy->getAs<BitIntType>())
573 if (EIT->getNumBits() > 128)
574 return getNaturalAlignIndirect(Ty: RetTy,
575 AddrSpace: getDataLayout().getAllocaAddrSpace());
576
577 return (isPromotableIntegerTypeForABI(Ty: RetTy) && isDarwinPCS()
578 ? ABIArgInfo::getExtend(Ty: RetTy)
579 : ABIArgInfo::getDirect());
580 }
581
582 uint64_t Size = getContext().getTypeSize(T: RetTy);
583 if (!RetTy->isSVESizelessBuiltinType() &&
584 (isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true) || Size == 0))
585 return ABIArgInfo::getIgnore();
586
587 const Type *Base = nullptr;
588 uint64_t Members = 0;
589 if (isHomogeneousAggregate(Ty: RetTy, Base, Members) &&
590 !(getTarget().getTriple().getArch() == llvm::Triple::aarch64_32 &&
591 IsVariadicFn))
592 // Homogeneous Floating-point Aggregates (HFAs) are returned directly.
593 return ABIArgInfo::getDirect();
594
595 // In AAPCS return values of a Pure Scalable type are treated as a single
596 // named argument and passed expanded in registers, or indirectly if there are
597 // not enough registers.
598 if (Kind == AArch64ABIKind::AAPCS) {
599 unsigned NSRN = 0, NPRN = 0;
600 unsigned NVec = 0, NPred = 0;
601 SmallVector<llvm::Type *> UnpaddedCoerceToSeq;
602 if (passAsPureScalableType(Ty: RetTy, NV&: NVec, NP&: NPred, CoerceToSeq&: UnpaddedCoerceToSeq) &&
603 (NVec + NPred) > 0)
604 return coerceAndExpandPureScalableAggregate(
605 Ty: RetTy, /* IsNamedArg */ true, NVec, NPred, UnpaddedCoerceToSeq, NSRN,
606 NPRN);
607 }
608
609 // Aggregates <= 16 bytes are returned directly in registers or on the stack.
610 if (Size <= 128) {
611 if (Size <= 64 && getDataLayout().isLittleEndian()) {
612 // Composite types are returned in lower bits of a 64-bit register for LE,
613 // and in higher bits for BE. However, integer types are always returned
614 // in lower bits for both LE and BE, and they are not rounded up to
615 // 64-bits. We can skip rounding up of composite types for LE, but not for
616 // BE, otherwise composite types will be indistinguishable from integer
617 // types.
618 return ABIArgInfo::getDirect(
619 T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Size));
620 }
621
622 unsigned Alignment = getContext().getTypeAlign(T: RetTy);
623 Size = llvm::alignTo(Value: Size, Align: 64); // round up to multiple of 8 bytes
624
625 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
626 // For aggregates with 16-byte alignment, we use i128.
627 if (Alignment < 128 && Size == 128) {
628 llvm::Type *BaseTy = llvm::Type::getInt64Ty(C&: getVMContext());
629 return ABIArgInfo::getDirect(T: llvm::ArrayType::get(ElementType: BaseTy, NumElements: Size / 64));
630 }
631 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Size));
632 }
633
634 return getNaturalAlignIndirect(Ty: RetTy, AddrSpace: getDataLayout().getAllocaAddrSpace());
635}
636
637/// isIllegalVectorType - check whether the vector type is legal for AArch64.
638bool AArch64ABIInfo::isIllegalVectorType(QualType Ty) const {
639 if (const VectorType *VT = Ty->getAs<VectorType>()) {
640 // Check whether VT is a fixed-length SVE vector. These types are
641 // represented as scalable vectors in function args/return and must be
642 // coerced from fixed vectors.
643 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
644 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate)
645 return true;
646
647 // Check whether VT is legal.
648 unsigned NumElements = VT->getNumElements();
649 uint64_t Size = getContext().getTypeSize(T: VT);
650 // NumElements should be power of 2.
651 if (!llvm::isPowerOf2_32(Value: NumElements))
652 return true;
653
654 // arm64_32 has to be compatible with the ARM logic here, which allows huge
655 // vectors for some reason.
656 llvm::Triple Triple = getTarget().getTriple();
657 if (Triple.getArch() == llvm::Triple::aarch64_32 &&
658 Triple.isOSBinFormatMachO())
659 return Size <= 32;
660
661 return Size != 64 && (Size != 128 || NumElements == 1);
662 }
663 return false;
664}
665
666bool AArch64SwiftABIInfo::isLegalVectorType(CharUnits VectorSize,
667 llvm::Type *EltTy,
668 unsigned NumElts) const {
669 if (!llvm::isPowerOf2_32(Value: NumElts))
670 return false;
671 if (VectorSize.getQuantity() != 8 &&
672 (VectorSize.getQuantity() != 16 || NumElts == 1))
673 return false;
674 return true;
675}
676
677bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
678 // For the soft-float ABI variant, no types are considered to be homogeneous
679 // aggregates.
680 if (isSoftFloat())
681 return false;
682
683 // Homogeneous aggregates for AAPCS64 must have base types of a floating
684 // point type or a short-vector type. This is the same as the 32-bit ABI,
685 // but with the difference that any floating-point type is allowed,
686 // including __fp16.
687 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
688 if (BT->isFloatingPoint())
689 return true;
690 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
691 if (auto Kind = VT->getVectorKind();
692 Kind == VectorKind::SveFixedLengthData ||
693 Kind == VectorKind::SveFixedLengthPredicate)
694 return false;
695
696 unsigned VecSize = getContext().getTypeSize(T: VT);
697 if (VecSize == 64 || VecSize == 128)
698 return true;
699 }
700 return false;
701}
702
703bool AArch64ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
704 uint64_t Members) const {
705 return Members <= 4;
706}
707
708bool AArch64ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate()
709 const {
710 // AAPCS64 says that the rule for whether something is a homogeneous
711 // aggregate is applied to the output of the data layout decision. So
712 // anything that doesn't affect the data layout also does not affect
713 // homogeneity. In particular, zero-length bitfields don't stop a struct
714 // being homogeneous.
715 return true;
716}
717
718bool AArch64ABIInfo::passAsAggregateType(QualType Ty) const {
719 if (Kind == AArch64ABIKind::AAPCS && Ty->isSVESizelessBuiltinType()) {
720 const auto *BT = Ty->castAs<BuiltinType>();
721 return !BT->isSVECount() &&
722 getContext().getBuiltinVectorTypeInfo(VecTy: BT).NumVectors > 1;
723 }
724 return isAggregateTypeForABI(T: Ty);
725}
726
727// Check if a type needs to be passed in registers as a Pure Scalable Type (as
728// defined by AAPCS64). Return the number of data vectors and the number of
729// predicate vectors in the type, into `NVec` and `NPred`, respectively. Upon
730// return `CoerceToSeq` contains an expanded sequence of LLVM IR types, one
731// element for each non-composite member. For practical purposes, limit the
732// length of `CoerceToSeq` to about 12 (the maximum that could possibly fit
733// in registers) and return false, the effect of which will be to pass the
734// argument under the rules for a large (> 128 bytes) composite.
735bool AArch64ABIInfo::passAsPureScalableType(
736 QualType Ty, unsigned &NVec, unsigned &NPred,
737 SmallVectorImpl<llvm::Type *> &CoerceToSeq) const {
738 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(T: Ty)) {
739 uint64_t NElt = AT->getZExtSize();
740 if (NElt == 0)
741 return false;
742
743 unsigned NV = 0, NP = 0;
744 SmallVector<llvm::Type *> EltCoerceToSeq;
745 if (!passAsPureScalableType(Ty: AT->getElementType(), NVec&: NV, NPred&: NP, CoerceToSeq&: EltCoerceToSeq))
746 return false;
747
748 if (CoerceToSeq.size() + NElt * EltCoerceToSeq.size() > 12)
749 return false;
750
751 for (uint64_t I = 0; I < NElt; ++I)
752 llvm::append_range(C&: CoerceToSeq, R&: EltCoerceToSeq);
753
754 NVec += NElt * NV;
755 NPred += NElt * NP;
756 return true;
757 }
758
759 if (const RecordType *RT = Ty->getAsCanonical<RecordType>()) {
760 // If the record cannot be passed in registers, then it's not a PST.
761 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, CXXABI&: getCXXABI());
762 RAA != CGCXXABI::RAA_Default)
763 return false;
764
765 // Pure scalable types are never unions and never contain unions.
766 const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf();
767 if (RD->isUnion())
768 return false;
769
770 // If this is a C++ record, check the bases.
771 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
772 for (const auto &I : CXXRD->bases()) {
773 if (isEmptyRecord(Context&: getContext(), T: I.getType(), AllowArrays: true))
774 continue;
775 if (!passAsPureScalableType(Ty: I.getType(), NVec, NPred, CoerceToSeq))
776 return false;
777 }
778 }
779
780 // Check members.
781 for (const auto *FD : RD->fields()) {
782 QualType FT = FD->getType();
783 if (isEmptyField(Context&: getContext(), FD, /* AllowArrays */ true))
784 continue;
785 if (!passAsPureScalableType(Ty: FT, NVec, NPred, CoerceToSeq))
786 return false;
787 }
788
789 return true;
790 }
791
792 if (const auto *VT = Ty->getAs<VectorType>()) {
793 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
794 ++NPred;
795 if (CoerceToSeq.size() + 1 > 12)
796 return false;
797 CoerceToSeq.push_back(Elt: convertFixedToScalableVectorType(VT));
798 return true;
799 }
800
801 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
802 ++NVec;
803 if (CoerceToSeq.size() + 1 > 12)
804 return false;
805 CoerceToSeq.push_back(Elt: convertFixedToScalableVectorType(VT));
806 return true;
807 }
808
809 return false;
810 }
811
812 if (!Ty->isBuiltinType())
813 return false;
814
815 bool isPredicate;
816 switch (Ty->castAs<BuiltinType>()->getKind()) {
817#define SVE_VECTOR_TYPE(Name, MangledName, Id, SingletonId) \
818 case BuiltinType::Id: \
819 isPredicate = false; \
820 break;
821#define SVE_PREDICATE_TYPE(Name, MangledName, Id, SingletonId) \
822 case BuiltinType::Id: \
823 isPredicate = true; \
824 break;
825#include "clang/Basic/AArch64ACLETypes.def"
826 default:
827 return false;
828 }
829
830 ASTContext::BuiltinVectorTypeInfo Info =
831 getContext().getBuiltinVectorTypeInfo(VecTy: cast<BuiltinType>(Val&: Ty));
832 assert(Info.NumVectors > 0 && Info.NumVectors <= 4 &&
833 "Expected 1, 2, 3 or 4 vectors!");
834 if (isPredicate)
835 NPred += Info.NumVectors;
836 else
837 NVec += Info.NumVectors;
838 llvm::Type *EltTy = Info.ElementType->isMFloat8Type()
839 ? llvm::Type::getInt8Ty(C&: getVMContext())
840 : CGT.ConvertType(T: Info.ElementType);
841 auto *VTy = llvm::ScalableVectorType::get(ElementType: EltTy, MinNumElts: Info.EC.getKnownMinValue());
842
843 if (CoerceToSeq.size() + Info.NumVectors > 12)
844 return false;
845 std::fill_n(first: std::back_inserter(x&: CoerceToSeq), n: Info.NumVectors, value: VTy);
846
847 return true;
848}
849
850// Expand an LLVM IR type into a sequence with a element for each non-struct,
851// non-array member of the type, with the exception of the padding types, which
852// are retained.
853void AArch64ABIInfo::flattenType(
854 llvm::Type *Ty, SmallVectorImpl<llvm::Type *> &Flattened) const {
855
856 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType: Ty)) {
857 Flattened.push_back(Elt: Ty);
858 return;
859 }
860
861 if (const auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty)) {
862 uint64_t NElt = AT->getNumElements();
863 if (NElt == 0)
864 return;
865
866 SmallVector<llvm::Type *> EltFlattened;
867 flattenType(Ty: AT->getElementType(), Flattened&: EltFlattened);
868
869 for (uint64_t I = 0; I < NElt; ++I)
870 llvm::append_range(C&: Flattened, R&: EltFlattened);
871 return;
872 }
873
874 if (const auto *ST = dyn_cast<llvm::StructType>(Val: Ty)) {
875 for (auto *ET : ST->elements())
876 flattenType(Ty: ET, Flattened);
877 return;
878 }
879
880 Flattened.push_back(Elt: Ty);
881}
882
883RValue AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
884 CodeGenFunction &CGF, AArch64ABIKind Kind,
885 AggValueSlot Slot) const {
886 // These numbers are not used for variadic arguments, hence it doesn't matter
887 // they don't retain their values across multiple calls to
888 // `classifyArgumentType` here.
889 unsigned NSRN = 0, NPRN = 0;
890 ABIArgInfo AI =
891 classifyArgumentType(Ty, /*IsVariadicFn=*/true, /* IsNamedArg */ false,
892 CallingConvention: CGF.CurFnInfo->getCallingConvention(), NSRN, NPRN);
893 // Empty records are ignored for parameter passing purposes.
894 if (AI.isIgnore())
895 return Slot.asRValue();
896
897 bool IsIndirect = AI.isIndirect();
898
899 llvm::Type *BaseTy = CGF.ConvertType(T: Ty);
900 if (IsIndirect)
901 BaseTy = llvm::PointerType::getUnqual(C&: BaseTy->getContext());
902 else if (AI.getCoerceToType())
903 BaseTy = AI.getCoerceToType();
904
905 unsigned NumRegs = 1;
906 if (llvm::ArrayType *ArrTy = dyn_cast<llvm::ArrayType>(Val: BaseTy)) {
907 BaseTy = ArrTy->getElementType();
908 NumRegs = ArrTy->getNumElements();
909 }
910 bool IsFPR =
911 !isSoftFloat() && (BaseTy->isFloatingPointTy() || BaseTy->isVectorTy());
912
913 // The AArch64 va_list type and handling is specified in the Procedure Call
914 // Standard, section B.4:
915 //
916 // struct {
917 // void *__stack;
918 // void *__gr_top;
919 // void *__vr_top;
920 // int __gr_offs;
921 // int __vr_offs;
922 // };
923
924 llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock(name: "vaarg.maybe_reg");
925 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock(name: "vaarg.in_reg");
926 llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock(name: "vaarg.on_stack");
927 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "vaarg.end");
928
929 CharUnits TySize = getContext().getTypeSizeInChars(T: Ty);
930 CharUnits TyAlign = getContext().getTypeUnadjustedAlignInChars(T: Ty);
931
932 Address reg_offs_p = Address::invalid();
933 llvm::Value *reg_offs = nullptr;
934 int reg_top_index;
935 int RegSize = IsIndirect ? 8 : TySize.getQuantity();
936 if (!IsFPR) {
937 // 3 is the field number of __gr_offs
938 reg_offs_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 3, Name: "gr_offs_p");
939 reg_offs = CGF.Builder.CreateLoad(Addr: reg_offs_p, Name: "gr_offs");
940 reg_top_index = 1; // field number for __gr_top
941 RegSize = llvm::alignTo(Value: RegSize, Align: 8);
942 } else {
943 // 4 is the field number of __vr_offs.
944 reg_offs_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 4, Name: "vr_offs_p");
945 reg_offs = CGF.Builder.CreateLoad(Addr: reg_offs_p, Name: "vr_offs");
946 reg_top_index = 2; // field number for __vr_top
947 RegSize = 16 * NumRegs;
948 }
949
950 //=======================================
951 // Find out where argument was passed
952 //=======================================
953
954 // If reg_offs >= 0 we're already using the stack for this type of
955 // argument. We don't want to keep updating reg_offs (in case it overflows,
956 // though anyone passing 2GB of arguments, each at most 16 bytes, deserves
957 // whatever they get).
958 llvm::Value *UsingStack = nullptr;
959 UsingStack = CGF.Builder.CreateICmpSGE(
960 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
961
962 CGF.Builder.CreateCondBr(Cond: UsingStack, True: OnStackBlock, False: MaybeRegBlock);
963
964 // Otherwise, at least some kind of argument could go in these registers, the
965 // question is whether this particular type is too big.
966 CGF.EmitBlock(BB: MaybeRegBlock);
967
968 // Integer arguments may need to correct register alignment (for example a
969 // "struct { __int128 a; };" gets passed in x_2N, x_{2N+1}). In this case we
970 // align __gr_offs to calculate the potential address.
971 if (!IsFPR && !IsIndirect && TyAlign.getQuantity() > 8) {
972 int Align = TyAlign.getQuantity();
973
974 reg_offs = CGF.Builder.CreateAdd(
975 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: Align - 1),
976 Name: "align_regoffs");
977 reg_offs = CGF.Builder.CreateAnd(
978 LHS: reg_offs, RHS: llvm::ConstantInt::getSigned(Ty: CGF.Int32Ty, V: -Align),
979 Name: "aligned_regoffs");
980 }
981
982 // Update the gr_offs/vr_offs pointer for next call to va_arg on this va_list.
983 // The fact that this is done unconditionally reflects the fact that
984 // allocating an argument to the stack also uses up all the remaining
985 // registers of the appropriate kind.
986 llvm::Value *NewOffset = nullptr;
987 NewOffset = CGF.Builder.CreateAdd(
988 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: RegSize), Name: "new_reg_offs");
989 CGF.Builder.CreateStore(Val: NewOffset, Addr: reg_offs_p);
990
991 // Now we're in a position to decide whether this argument really was in
992 // registers or not.
993 llvm::Value *InRegs = nullptr;
994 InRegs = CGF.Builder.CreateICmpSLE(
995 LHS: NewOffset, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0), Name: "inreg");
996
997 CGF.Builder.CreateCondBr(Cond: InRegs, True: InRegBlock, False: OnStackBlock);
998
999 //=======================================
1000 // Argument was in registers
1001 //=======================================
1002
1003 // Now we emit the code for if the argument was originally passed in
1004 // registers. First start the appropriate block:
1005 CGF.EmitBlock(BB: InRegBlock);
1006
1007 llvm::Value *reg_top = nullptr;
1008 Address reg_top_p =
1009 CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: reg_top_index, Name: "reg_top_p");
1010 reg_top = CGF.Builder.CreateLoad(Addr: reg_top_p, Name: "reg_top");
1011 Address BaseAddr(CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: reg_top, IdxList: reg_offs),
1012 CGF.Int8Ty, CharUnits::fromQuantity(Quantity: IsFPR ? 16 : 8));
1013 Address RegAddr = Address::invalid();
1014 llvm::Type *MemTy = CGF.ConvertTypeForMem(T: Ty), *ElementTy = MemTy;
1015
1016 if (IsIndirect) {
1017 // If it's been passed indirectly (actually a struct), whatever we find from
1018 // stored registers or on the stack will actually be a struct **.
1019 MemTy = llvm::PointerType::getUnqual(C&: MemTy->getContext());
1020 }
1021
1022 const Type *Base = nullptr;
1023 uint64_t NumMembers = 0;
1024 bool IsHFA = isHomogeneousAggregate(Ty, Base, Members&: NumMembers);
1025 if (IsHFA && NumMembers > 1) {
1026 // Homogeneous aggregates passed in registers will have their elements split
1027 // and stored 16-bytes apart regardless of size (they're notionally in qN,
1028 // qN+1, ...). We reload and store into a temporary local variable
1029 // contiguously.
1030 assert(!IsIndirect && "Homogeneous aggregates should be passed directly");
1031 auto BaseTyInfo = getContext().getTypeInfoInChars(T: QualType(Base, 0));
1032 llvm::Type *BaseTy = CGF.ConvertType(T: QualType(Base, 0));
1033 llvm::Type *HFATy = llvm::ArrayType::get(ElementType: BaseTy, NumElements: NumMembers);
1034 Address Tmp = CGF.CreateTempAlloca(Ty: HFATy,
1035 align: std::max(a: TyAlign, b: BaseTyInfo.Align));
1036
1037 // On big-endian platforms, the value will be right-aligned in its slot.
1038 int Offset = 0;
1039 if (CGF.CGM.getDataLayout().isBigEndian() &&
1040 BaseTyInfo.Width.getQuantity() < 16)
1041 Offset = 16 - BaseTyInfo.Width.getQuantity();
1042
1043 for (unsigned i = 0; i < NumMembers; ++i) {
1044 CharUnits BaseOffset = CharUnits::fromQuantity(Quantity: 16 * i + Offset);
1045 Address LoadAddr =
1046 CGF.Builder.CreateConstInBoundsByteGEP(Addr: BaseAddr, Offset: BaseOffset);
1047 LoadAddr = LoadAddr.withElementType(ElemTy: BaseTy);
1048
1049 Address StoreAddr = CGF.Builder.CreateConstArrayGEP(Addr: Tmp, Index: i);
1050
1051 llvm::Value *Elem = CGF.Builder.CreateLoad(Addr: LoadAddr);
1052 CGF.Builder.CreateStore(Val: Elem, Addr: StoreAddr);
1053 }
1054
1055 RegAddr = Tmp.withElementType(ElemTy: MemTy);
1056 } else {
1057 // Otherwise the object is contiguous in memory.
1058
1059 // It might be right-aligned in its slot.
1060 CharUnits SlotSize = BaseAddr.getAlignment();
1061 if (CGF.CGM.getDataLayout().isBigEndian() && !IsIndirect &&
1062 (IsHFA || !isAggregateTypeForABI(T: Ty)) &&
1063 TySize < SlotSize) {
1064 CharUnits Offset = SlotSize - TySize;
1065 BaseAddr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: BaseAddr, Offset);
1066 }
1067
1068 RegAddr = BaseAddr.withElementType(ElemTy: MemTy);
1069 }
1070
1071 CGF.EmitBranch(Block: ContBlock);
1072
1073 //=======================================
1074 // Argument was on the stack
1075 //=======================================
1076 CGF.EmitBlock(BB: OnStackBlock);
1077
1078 Address stack_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 0, Name: "stack_p");
1079 llvm::Value *OnStackPtr = CGF.Builder.CreateLoad(Addr: stack_p, Name: "stack");
1080
1081 // Again, stack arguments may need realignment. In this case both integer and
1082 // floating-point ones might be affected.
1083 if (!IsIndirect && TyAlign.getQuantity() > 8) {
1084 OnStackPtr = emitRoundPointerUpToAlignment(CGF, Ptr: OnStackPtr, Align: TyAlign);
1085 }
1086 Address OnStackAddr = Address(OnStackPtr, CGF.Int8Ty,
1087 std::max(a: CharUnits::fromQuantity(Quantity: 8), b: TyAlign));
1088
1089 // All stack slots are multiples of 8 bytes.
1090 CharUnits StackSlotSize = CharUnits::fromQuantity(Quantity: 8);
1091 CharUnits StackSize;
1092 if (IsIndirect)
1093 StackSize = StackSlotSize;
1094 else
1095 StackSize = TySize.alignTo(Align: StackSlotSize);
1096
1097 llvm::Value *StackSizeC = CGF.Builder.getSize(N: StackSize);
1098 llvm::Value *NewStack = CGF.Builder.CreateInBoundsGEP(
1099 Ty: CGF.Int8Ty, Ptr: OnStackPtr, IdxList: StackSizeC, Name: "new_stack");
1100
1101 // Write the new value of __stack for the next call to va_arg
1102 CGF.Builder.CreateStore(Val: NewStack, Addr: stack_p);
1103
1104 if (CGF.CGM.getDataLayout().isBigEndian() && !isAggregateTypeForABI(T: Ty) &&
1105 TySize < StackSlotSize) {
1106 CharUnits Offset = StackSlotSize - TySize;
1107 OnStackAddr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: OnStackAddr, Offset);
1108 }
1109
1110 OnStackAddr = OnStackAddr.withElementType(ElemTy: MemTy);
1111
1112 CGF.EmitBranch(Block: ContBlock);
1113
1114 //=======================================
1115 // Tidy up
1116 //=======================================
1117 CGF.EmitBlock(BB: ContBlock);
1118
1119 Address ResAddr = emitMergePHI(CGF, Addr1: RegAddr, Block1: InRegBlock, Addr2: OnStackAddr,
1120 Block2: OnStackBlock, Name: "vaargs.addr");
1121
1122 if (IsIndirect)
1123 return CGF.EmitLoadOfAnyValue(
1124 V: CGF.MakeAddrLValue(
1125 Addr: Address(CGF.Builder.CreateLoad(Addr: ResAddr, Name: "vaarg.addr"), ElementTy,
1126 TyAlign),
1127 T: Ty),
1128 Slot);
1129
1130 return CGF.EmitLoadOfAnyValue(V: CGF.MakeAddrLValue(Addr: ResAddr, T: Ty), Slot);
1131}
1132
1133RValue AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty,
1134 CodeGenFunction &CGF,
1135 AggValueSlot Slot) const {
1136 // The backend's lowering doesn't support va_arg for aggregates or
1137 // illegal vector types. Lower VAArg here for these cases and use
1138 // the LLVM va_arg instruction for everything else.
1139 if (!isAggregateTypeForABI(T: Ty) && !isIllegalVectorType(Ty))
1140 return CGF.EmitLoadOfAnyValue(
1141 V: CGF.MakeAddrLValue(
1142 Addr: EmitVAArgInstr(CGF, VAListAddr, Ty, AI: ABIArgInfo::getDirect()), T: Ty),
1143 Slot);
1144
1145 uint64_t PointerSize = getTarget().getPointerWidth(AddrSpace: LangAS::Default) / 8;
1146 CharUnits SlotSize = CharUnits::fromQuantity(Quantity: PointerSize);
1147
1148 // Empty records are ignored for parameter passing purposes.
1149 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
1150 return Slot.asRValue();
1151
1152 // The size of the actual thing passed, which might end up just
1153 // being a pointer for indirect types.
1154 auto TyInfo = getContext().getTypeInfoInChars(T: Ty);
1155
1156 // Arguments bigger than 16 bytes which aren't homogeneous
1157 // aggregates should be passed indirectly.
1158 bool IsIndirect = false;
1159 if (TyInfo.Width.getQuantity() > 16) {
1160 const Type *Base = nullptr;
1161 uint64_t Members = 0;
1162 IsIndirect = !isHomogeneousAggregate(Ty, Base, Members);
1163 }
1164
1165 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect, ValueInfo: TyInfo, SlotSizeAndAlign: SlotSize,
1166 /*AllowHigherAlign*/ true, Slot);
1167}
1168
1169RValue AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
1170 QualType Ty, AggValueSlot Slot) const {
1171 bool AllowHigherAlign = false;
1172 bool IsIndirect = false;
1173
1174 if (getTarget().getTriple().isWindowsArm64EC()) {
1175 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
1176 // not 1, 2, 4, or 8 bytes, must be passed by reference."
1177 uint64_t Width = getContext().getTypeSize(T: Ty);
1178 IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Value: Width);
1179 } else {
1180 // E.g. __int128 when passed is aligned to 16 bytes, so it must be read
1181 // with the same alignment.
1182 AllowHigherAlign = true;
1183
1184 // Composites larger than 16 bytes are passed by reference.
1185 if (isAggregateTypeForABI(T: Ty) && getContext().getTypeSize(T: Ty) > 128)
1186 IsIndirect = true;
1187 }
1188
1189 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
1190 ValueInfo: CGF.getContext().getTypeInfoInChars(T: Ty),
1191 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 8), AllowHigherAlign, Slot);
1192}
1193
1194static bool isStreamingCompatible(const FunctionDecl *F) {
1195 if (const auto *T = F->getType()->getAs<FunctionProtoType>())
1196 return T->getAArch64SMEAttributes() &
1197 FunctionType::SME_PStateSMCompatibleMask;
1198 return false;
1199}
1200
1201// Report an error if an argument or return value of type Ty would need to be
1202// passed in a floating-point register.
1203static void diagnoseIfNeedsFPReg(DiagnosticsEngine &Diags,
1204 const StringRef ABIName,
1205 const AArch64ABIInfo &ABIInfo,
1206 const QualType &Ty, const NamedDecl *D,
1207 SourceLocation loc) {
1208 const Type *HABase = nullptr;
1209 uint64_t HAMembers = 0;
1210 if (Ty->isFloatingType() || Ty->isVectorType() ||
1211 ABIInfo.isHomogeneousAggregate(Ty, Base&: HABase, Members&: HAMembers)) {
1212 Diags.Report(Loc: loc, DiagID: diag::err_target_unsupported_type_for_abi)
1213 << D->getDeclName() << Ty << ABIName;
1214 }
1215}
1216
1217// If we are using a hard-float ABI, but do not have floating point registers,
1218// then report an error for any function arguments or returns which would be
1219// passed in floating-pint registers.
1220void AArch64TargetCodeGenInfo::checkFunctionABI(
1221 CodeGenModule &CGM, const FunctionDecl *FuncDecl) const {
1222 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
1223 const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();
1224
1225 if (!TI.hasFeature(Feature: "fp") && !ABIInfo.isSoftFloat()) {
1226 diagnoseIfNeedsFPReg(Diags&: CGM.getDiags(), ABIName: TI.getABI(), ABIInfo,
1227 Ty: FuncDecl->getReturnType(), D: FuncDecl,
1228 loc: FuncDecl->getLocation());
1229 for (ParmVarDecl *PVD : FuncDecl->parameters()) {
1230 diagnoseIfNeedsFPReg(Diags&: CGM.getDiags(), ABIName: TI.getABI(), ABIInfo, Ty: PVD->getType(),
1231 D: PVD, loc: FuncDecl->getLocation());
1232 }
1233 }
1234}
1235
1236enum class ArmSMEInlinability : uint8_t {
1237 Ok = 0,
1238 ErrorCalleeRequiresNewZA = 1 << 0,
1239 ErrorCalleeRequiresNewZT0 = 1 << 1,
1240 WarnIncompatibleStreamingModes = 1 << 2,
1241 ErrorIncompatibleStreamingModes = 1 << 3,
1242
1243 IncompatibleStreamingModes =
1244 WarnIncompatibleStreamingModes | ErrorIncompatibleStreamingModes,
1245
1246 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/ErrorIncompatibleStreamingModes),
1247};
1248
1249/// Determines if there are any Arm SME ABI issues with inlining \p Callee into
1250/// \p Caller. Returns the issue (if any) in the ArmSMEInlinability bit enum.
1251static ArmSMEInlinability GetArmSMEInlinability(const FunctionDecl *Caller,
1252 const FunctionDecl *Callee) {
1253 bool CallerIsStreaming =
1254 IsArmStreamingFunction(FD: Caller, /*IncludeLocallyStreaming=*/true);
1255 bool CalleeIsStreaming =
1256 IsArmStreamingFunction(FD: Callee, /*IncludeLocallyStreaming=*/true);
1257 bool CallerIsStreamingCompatible = isStreamingCompatible(F: Caller);
1258 bool CalleeIsStreamingCompatible = isStreamingCompatible(F: Callee);
1259
1260 ArmSMEInlinability Inlinability = ArmSMEInlinability::Ok;
1261
1262 if (!CalleeIsStreamingCompatible &&
1263 (CallerIsStreaming != CalleeIsStreaming || CallerIsStreamingCompatible)) {
1264 if (CalleeIsStreaming)
1265 Inlinability |= ArmSMEInlinability::ErrorIncompatibleStreamingModes;
1266 else
1267 Inlinability |= ArmSMEInlinability::WarnIncompatibleStreamingModes;
1268 }
1269 if (auto *NewAttr = Callee->getAttr<ArmNewAttr>()) {
1270 if (NewAttr->isNewZA())
1271 Inlinability |= ArmSMEInlinability::ErrorCalleeRequiresNewZA;
1272 if (NewAttr->isNewZT0())
1273 Inlinability |= ArmSMEInlinability::ErrorCalleeRequiresNewZT0;
1274 }
1275
1276 return Inlinability;
1277}
1278
1279void AArch64TargetCodeGenInfo::checkFunctionCallABIStreaming(
1280 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
1281 const FunctionDecl *Callee) const {
1282 if (!Caller || !Callee || !Callee->hasAttr<AlwaysInlineAttr>())
1283 return;
1284
1285 ArmSMEInlinability Inlinability = GetArmSMEInlinability(Caller, Callee);
1286
1287 if ((Inlinability & ArmSMEInlinability::IncompatibleStreamingModes) !=
1288 ArmSMEInlinability::Ok)
1289 CGM.getDiags().Report(
1290 Loc: CallLoc,
1291 DiagID: (Inlinability & ArmSMEInlinability::ErrorIncompatibleStreamingModes) ==
1292 ArmSMEInlinability::ErrorIncompatibleStreamingModes
1293 ? diag::err_function_always_inline_attribute_mismatch
1294 : diag::warn_function_always_inline_attribute_mismatch)
1295 << Caller->getDeclName() << Callee->getDeclName() << "streaming";
1296
1297 if ((Inlinability & ArmSMEInlinability::ErrorCalleeRequiresNewZA) ==
1298 ArmSMEInlinability::ErrorCalleeRequiresNewZA)
1299 CGM.getDiags().Report(Loc: CallLoc, DiagID: diag::err_function_always_inline_new_za)
1300 << Callee->getDeclName();
1301
1302 if ((Inlinability & ArmSMEInlinability::ErrorCalleeRequiresNewZT0) ==
1303 ArmSMEInlinability::ErrorCalleeRequiresNewZT0)
1304 CGM.getDiags().Report(Loc: CallLoc, DiagID: diag::err_function_always_inline_new_zt0)
1305 << Callee->getDeclName();
1306}
1307
1308// If the target does not have floating-point registers, but we are using a
1309// hard-float ABI, there is no way to pass floating-point, vector or HFA values
1310// to functions, so we report an error.
1311void AArch64TargetCodeGenInfo::checkFunctionCallABISoftFloat(
1312 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
1313 const FunctionDecl *Callee, const CallArgList &Args,
1314 QualType ReturnType) const {
1315 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
1316 const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();
1317
1318 if (!Caller || TI.hasFeature(Feature: "fp") || ABIInfo.isSoftFloat())
1319 return;
1320
1321 diagnoseIfNeedsFPReg(Diags&: CGM.getDiags(), ABIName: TI.getABI(), ABIInfo, Ty: ReturnType,
1322 D: Callee ? Callee : Caller, loc: CallLoc);
1323
1324 for (const CallArg &Arg : Args)
1325 diagnoseIfNeedsFPReg(Diags&: CGM.getDiags(), ABIName: TI.getABI(), ABIInfo, Ty: Arg.getType(),
1326 D: Callee ? Callee : Caller, loc: CallLoc);
1327}
1328
1329void AArch64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM,
1330 SourceLocation CallLoc,
1331 const FunctionDecl *Caller,
1332 const FunctionDecl *Callee,
1333 const CallArgList &Args,
1334 QualType ReturnType) const {
1335 checkFunctionCallABIStreaming(CGM, CallLoc, Caller, Callee);
1336 checkFunctionCallABISoftFloat(CGM, CallLoc, Caller, Callee, Args, ReturnType);
1337}
1338
1339bool AArch64TargetCodeGenInfo::wouldInliningViolateFunctionCallABI(
1340 const FunctionDecl *Caller, const FunctionDecl *Callee) const {
1341 return Caller && Callee &&
1342 GetArmSMEInlinability(Caller, Callee) != ArmSMEInlinability::Ok;
1343}
1344
1345void AArch64ABIInfo::appendAttributeMangling(TargetClonesAttr *Attr,
1346 unsigned Index,
1347 raw_ostream &Out) const {
1348 appendAttributeMangling(AttrStr: Attr->getFeatureStr(Index), Out);
1349}
1350
1351void AArch64ABIInfo::appendAttributeMangling(StringRef AttrStr,
1352 raw_ostream &Out) const {
1353 if (AttrStr == "default") {
1354 Out << ".default";
1355 return;
1356 }
1357
1358 Out << "._";
1359 SmallVector<StringRef, 8> Features;
1360 AttrStr.split(A&: Features, Separator: "+");
1361 for (auto &Feat : Features)
1362 Feat = Feat.trim();
1363
1364 llvm::sort(C&: Features, Comp: [](const StringRef LHS, const StringRef RHS) {
1365 return LHS.compare(RHS) < 0;
1366 });
1367
1368 llvm::SmallDenseSet<StringRef, 8> UniqueFeats;
1369 for (auto &Feat : Features)
1370 if (getTarget().doesFeatureAffectCodeGen(Feature: Feat))
1371 if (auto Ext = llvm::AArch64::parseFMVExtension(Extension: Feat))
1372 if (UniqueFeats.insert(V: Ext->Name).second)
1373 Out << 'M' << Ext->Name;
1374}
1375
1376std::unique_ptr<TargetCodeGenInfo>
1377CodeGen::createAArch64TargetCodeGenInfo(CodeGenModule &CGM,
1378 AArch64ABIKind Kind) {
1379 return std::make_unique<AArch64TargetCodeGenInfo>(args&: CGM, args&: Kind);
1380}
1381
1382std::unique_ptr<TargetCodeGenInfo>
1383CodeGen::createWindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM,
1384 AArch64ABIKind K) {
1385 return std::make_unique<WindowsAArch64TargetCodeGenInfo>(args&: CGM, args&: K);
1386}
1387