1//===- AArch64.cpp --------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ABIInfoImpl.h"
10#include "TargetInfo.h"
11#include "clang/AST/Decl.h"
12#include "clang/Basic/DiagnosticFrontend.h"
13#include "clang/CodeGenUtils/TargetUtils.h"
14#include "llvm/TargetParser/AArch64TargetParser.h"
15
16using namespace clang;
17using namespace clang::CodeGen;
18
19//===----------------------------------------------------------------------===//
20// AArch64 ABI Implementation
21//===----------------------------------------------------------------------===//
22
23namespace {
24
25class AArch64ABIInfo : public ABIInfo {
26 AArch64ABIKind Kind;
27
28 std::unique_ptr<TargetCodeGenInfo> WinX86_64CodegenInfo;
29
30public:
31 AArch64ABIInfo(CodeGenModule &CGM, AArch64ABIKind Kind)
32 : ABIInfo(CGM.getTypes()), Kind(Kind) {
33 if (getTarget().getTriple().isWindowsArm64EC()) {
34 WinX86_64CodegenInfo =
35 createWinX86_64TargetCodeGenInfo(CGM, AVXLevel: X86AVXABILevel::None);
36 }
37 }
38
39 bool isSoftFloat() const { return Kind == AArch64ABIKind::AAPCSSoft; }
40
41private:
42 AArch64ABIKind getABIKind() const { return Kind; }
43 bool isDarwinPCS() const { return Kind == AArch64ABIKind::DarwinPCS; }
44
45 ABIArgInfo classifyReturnType(QualType RetTy, bool IsVariadicFn) const;
46 ABIArgInfo classifyArgumentType(QualType RetTy, bool IsVariadicFn,
47 bool IsNamedArg, unsigned CallingConvention,
48 unsigned &NSRN, unsigned &NPRN) const;
49 llvm::Type *convertFixedToScalableVectorType(const VectorType *VT) const;
50 ABIArgInfo coerceIllegalVector(QualType Ty, unsigned &NSRN,
51 unsigned &NPRN) const;
52 ABIArgInfo coerceAndExpandPureScalableAggregate(
53 QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
54 const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
55 unsigned &NPRN) const;
56 bool isHomogeneousAggregateBaseType(QualType Ty) const override;
57 bool isHomogeneousAggregateSmallEnough(const Type *Ty,
58 uint64_t Members) const override;
59 bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const override;
60
61 bool isIllegalVectorType(QualType Ty) const;
62
63 bool passAsAggregateType(QualType Ty) const;
64 bool passAsPureScalableType(QualType Ty, unsigned &NV, unsigned &NP,
65 SmallVectorImpl<llvm::Type *> &CoerceToSeq) const;
66
67 void flattenType(llvm::Type *Ty,
68 SmallVectorImpl<llvm::Type *> &Flattened) const;
69
70 void computeInfo(CGFunctionInfo &FI) const override {
71 if (!::classifyReturnType(CXXABI: getCXXABI(), FI, Info: *this))
72 FI.getReturnInfo() =
73 classifyReturnType(RetTy: FI.getReturnType(), IsVariadicFn: FI.isVariadic());
74
75 unsigned ArgNo = 0;
76 unsigned NSRN = 0, NPRN = 0;
77 for (auto &it : FI.arguments()) {
78 const bool IsNamedArg =
79 !FI.isVariadic() || ArgNo < FI.getRequiredArgs().getNumRequiredArgs();
80 ++ArgNo;
81 it.info = classifyArgumentType(RetTy: it.type, IsVariadicFn: FI.isVariadic(), IsNamedArg,
82 CallingConvention: FI.getCallingConvention(), NSRN, NPRN);
83 }
84 }
85
86 RValue EmitDarwinVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,
87 AggValueSlot Slot) const;
88
89 RValue EmitAAPCSVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,
90 AArch64ABIKind Kind, AggValueSlot Slot) const;
91
92 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
93 AggValueSlot Slot) const override {
94 llvm::Type *BaseTy = CGF.ConvertType(T: Ty);
95 if (isa<llvm::ScalableVectorType>(Val: BaseTy))
96 llvm::report_fatal_error(reason: "Passing SVE types to variadic functions is "
97 "currently not supported");
98
99 return Kind == AArch64ABIKind::Win64
100 ? EmitMSVAArg(CGF, VAListAddr, Ty, Slot)
101 : isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF, Slot)
102 : EmitAAPCSVAArg(VAListAddr, Ty, CGF, Kind, Slot);
103 }
104
105 RValue EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
106 AggValueSlot Slot) const override;
107
108 bool allowBFloatArgsAndRet() const override {
109 return getTarget().hasBFloat16Type();
110 }
111
112 using ABIInfo::appendAttributeMangling;
113 void appendAttributeMangling(TargetClonesAttr *Attr, unsigned Index,
114 raw_ostream &Out) const override;
115 void appendAttributeMangling(StringRef AttrStr,
116 raw_ostream &Out) const override;
117};
118
119class AArch64SwiftABIInfo : public SwiftABIInfo {
120public:
121 explicit AArch64SwiftABIInfo(CodeGenTypes &CGT)
122 : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/true) {}
123
124 bool isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy,
125 unsigned NumElts) const override;
126};
127
128class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
129public:
130 AArch64TargetCodeGenInfo(CodeGenModule &CGM, AArch64ABIKind Kind)
131 : TargetCodeGenInfo(std::make_unique<AArch64ABIInfo>(args&: CGM, args&: Kind)) {
132 SwiftInfo = std::make_unique<AArch64SwiftABIInfo>(args&: CGM.getTypes());
133 }
134
135 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
136 return "mov\tfp, fp\t\t// marker for objc_retainAutoreleaseReturnValue";
137 }
138
139 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
140 return 31;
141 }
142
143 bool doesReturnSlotInterfereWithArgs() const override { return false; }
144
145 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
146 CodeGen::CodeGenModule &CGM) const override {
147 auto *Fn = dyn_cast<llvm::Function>(Val: GV);
148 if (!Fn)
149 return;
150
151 const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
152 TargetInfo::BranchProtectionInfo BPI(CGM.getLangOpts());
153
154 if (FD && FD->hasAttr<TargetAttr>()) {
155 const auto *TA = FD->getAttr<TargetAttr>();
156 ParsedTargetAttr Attr =
157 CGM.getTarget().parseTargetAttr(Str: TA->getFeaturesStr());
158 if (!Attr.BranchProtection.empty()) {
159 StringRef Error;
160 (void)CGM.getTarget().validateBranchProtection(
161 Attr, BPI, LO: CGM.getLangOpts(), Err&: Error);
162 assert(Error.empty());
163 }
164 }
165 setBranchProtectionFnAttributes(BPI, F&: *Fn);
166 setPointerAuthFnAttributes(Opts: CGM.getCodeGenOpts().PointerAuth, F&: *Fn);
167 }
168
169 bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF,
170 llvm::Type *Ty) const override {
171 if (CGF.getTarget().hasFeature(Feature: "ls64")) {
172 auto *ST = dyn_cast<llvm::StructType>(Val: Ty);
173 if (ST && ST->getNumElements() == 1) {
174 auto *AT = dyn_cast<llvm::ArrayType>(Val: ST->getElementType(N: 0));
175 if (AT && AT->getNumElements() == 8 &&
176 AT->getElementType()->isIntegerTy(BitWidth: 64))
177 return true;
178 }
179 }
180 return TargetCodeGenInfo::isScalarizableAsmOperand(CGF, Ty);
181 }
182
183 void checkFunctionABI(CodeGenModule &CGM,
184 const FunctionDecl *Decl) const override;
185
186 void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
187 const FunctionDecl *Caller,
188 const FunctionDecl *Callee, const CallArgList &Args,
189 QualType ReturnType) const override;
190
191 bool wouldInliningViolateFunctionCallABI(
192 const FunctionDecl *Caller, const FunctionDecl *Callee) const override;
193
194private:
195 // Diagnose calls between functions with incompatible Streaming SVE
196 // attributes.
197 void checkFunctionCallABIStreaming(CodeGenModule &CGM, SourceLocation CallLoc,
198 const FunctionDecl *Caller,
199 const FunctionDecl *Callee) const;
200 // Diagnose calls which must pass arguments in floating-point registers when
201 // the selected target does not have floating-point registers.
202 void checkFunctionCallABISoftFloat(CodeGenModule &CGM, SourceLocation CallLoc,
203 const FunctionDecl *Caller,
204 const FunctionDecl *Callee,
205 const CallArgList &Args,
206 QualType ReturnType) const;
207};
208
209class WindowsAArch64TargetCodeGenInfo : public AArch64TargetCodeGenInfo {
210public:
211 WindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM, AArch64ABIKind K)
212 : AArch64TargetCodeGenInfo(CGM, K) {}
213
214 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
215 CodeGen::CodeGenModule &CGM) const override;
216
217 void getDependentLibraryOption(llvm::StringRef Lib,
218 llvm::SmallString<24> &Opt) const override {
219 Opt = "/DEFAULTLIB:" + qualifyWindowsLibrary(Lib);
220 }
221
222 void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value,
223 llvm::SmallString<32> &Opt) const override {
224 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
225 }
226};
227
228void WindowsAArch64TargetCodeGenInfo::setTargetAttributes(
229 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
230 AArch64TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
231 if (GV->isDeclaration())
232 return;
233 addStackProbeTargetAttributes(D, GV, CGM);
234}
235}
236
237llvm::Type *
238AArch64ABIInfo::convertFixedToScalableVectorType(const VectorType *VT) const {
239 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
240
241 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
242 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
243 BuiltinType::UChar &&
244 "unexpected builtin type for SVE predicate!");
245 return llvm::ScalableVectorType::get(ElementType: llvm::Type::getInt1Ty(C&: getVMContext()),
246 MinNumElts: 16);
247 }
248
249 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
250 const auto *BT = VT->getElementType()->castAs<BuiltinType>();
251 switch (BT->getKind()) {
252 default:
253 llvm_unreachable("unexpected builtin type for SVE vector!");
254
255 case BuiltinType::SChar:
256 case BuiltinType::UChar:
257 case BuiltinType::MFloat8:
258 return llvm::ScalableVectorType::get(
259 ElementType: llvm::Type::getInt8Ty(C&: getVMContext()), MinNumElts: 16);
260
261 case BuiltinType::Short:
262 case BuiltinType::UShort:
263 return llvm::ScalableVectorType::get(
264 ElementType: llvm::Type::getInt16Ty(C&: getVMContext()), MinNumElts: 8);
265
266 case BuiltinType::Int:
267 case BuiltinType::UInt:
268 return llvm::ScalableVectorType::get(
269 ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), MinNumElts: 4);
270
271 case BuiltinType::Long:
272 case BuiltinType::ULong:
273 return llvm::ScalableVectorType::get(
274 ElementType: llvm::Type::getInt64Ty(C&: getVMContext()), MinNumElts: 2);
275
276 case BuiltinType::Half:
277 return llvm::ScalableVectorType::get(
278 ElementType: llvm::Type::getHalfTy(C&: getVMContext()), MinNumElts: 8);
279
280 case BuiltinType::Float:
281 return llvm::ScalableVectorType::get(
282 ElementType: llvm::Type::getFloatTy(C&: getVMContext()), MinNumElts: 4);
283
284 case BuiltinType::Double:
285 return llvm::ScalableVectorType::get(
286 ElementType: llvm::Type::getDoubleTy(C&: getVMContext()), MinNumElts: 2);
287
288 case BuiltinType::BFloat16:
289 return llvm::ScalableVectorType::get(
290 ElementType: llvm::Type::getBFloatTy(C&: getVMContext()), MinNumElts: 8);
291 }
292 }
293
294 llvm_unreachable("expected fixed-length SVE vector");
295}
296
297ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty, unsigned &NSRN,
298 unsigned &NPRN) const {
299 assert(Ty->isVectorType() && "expected vector type!");
300
301 const auto *VT = Ty->castAs<VectorType>();
302 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
303 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
304 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==
305 BuiltinType::UChar &&
306 "unexpected builtin type for SVE predicate!");
307 NPRN = std::min(a: NPRN + 1, b: 4u);
308 return ABIArgInfo::getDirect(T: llvm::ScalableVectorType::get(
309 ElementType: llvm::Type::getInt1Ty(C&: getVMContext()), MinNumElts: 16));
310 }
311
312 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
313 NSRN = std::min(a: NSRN + 1, b: 8u);
314 return ABIArgInfo::getDirect(T: convertFixedToScalableVectorType(VT));
315 }
316
317 uint64_t Size = getContext().getTypeSize(T: Ty);
318 // Android promotes <2 x i8> to i16, not i32
319 if ((isAndroid() || isOHOSFamily()) && (Size <= 16)) {
320 llvm::Type *ResType = llvm::Type::getInt16Ty(C&: getVMContext());
321 return ABIArgInfo::getDirect(T: ResType);
322 }
323 if (Size <= 32) {
324 llvm::Type *ResType = llvm::Type::getInt32Ty(C&: getVMContext());
325 return ABIArgInfo::getDirect(T: ResType);
326 }
327 if (Size == 64) {
328 NSRN = std::min(a: NSRN + 1, b: 8u);
329 auto *ResType =
330 llvm::FixedVectorType::get(ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), NumElts: 2);
331 return ABIArgInfo::getDirect(T: ResType);
332 }
333 if (Size == 128) {
334 NSRN = std::min(a: NSRN + 1, b: 8u);
335 auto *ResType =
336 llvm::FixedVectorType::get(ElementType: llvm::Type::getInt32Ty(C&: getVMContext()), NumElts: 4);
337 return ABIArgInfo::getDirect(T: ResType);
338 }
339
340 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
341 /*ByVal=*/false);
342}
343
344ABIArgInfo AArch64ABIInfo::coerceAndExpandPureScalableAggregate(
345 QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,
346 const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,
347 unsigned &NPRN) const {
348 if (!IsNamedArg || NSRN + NVec > 8 || NPRN + NPred > 4)
349 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
350 /*ByVal=*/false);
351 NSRN += NVec;
352 NPRN += NPred;
353
354 // Handle SVE vector tuples.
355 if (Ty->isSVESizelessBuiltinType())
356 return ABIArgInfo::getDirect();
357
358 llvm::Type *UnpaddedCoerceToType =
359 UnpaddedCoerceToSeq.size() == 1
360 ? UnpaddedCoerceToSeq[0]
361 : llvm::StructType::get(Context&: CGT.getLLVMContext(), Elements: UnpaddedCoerceToSeq,
362 isPacked: true);
363
364 SmallVector<llvm::Type *> CoerceToSeq;
365 flattenType(Ty: CGT.ConvertType(T: Ty), Flattened&: CoerceToSeq);
366 auto *CoerceToType =
367 llvm::StructType::get(Context&: CGT.getLLVMContext(), Elements: CoerceToSeq, isPacked: false);
368
369 return ABIArgInfo::getCoerceAndExpand(coerceToType: CoerceToType, unpaddedCoerceToType: UnpaddedCoerceToType);
370}
371
372ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadicFn,
373 bool IsNamedArg,
374 unsigned CallingConvention,
375 unsigned &NSRN,
376 unsigned &NPRN) const {
377 Ty = useFirstFieldIfTransparentUnion(Ty);
378
379 if (IsVariadicFn && getTarget().getTriple().isWindowsArm64EC()) {
380 // Arm64EC varargs functions use the x86_64 classification rules,
381 // not the AArch64 ABI rules.
382 return WinX86_64CodegenInfo->getABIInfo().classifyArgForArm64ECVarArg(
383 Ty, IsNamedArg);
384 }
385
386 // Handle illegal vector types here.
387 if (isIllegalVectorType(Ty))
388 return coerceIllegalVector(Ty, NSRN, NPRN);
389
390 if (!passAsAggregateType(Ty)) {
391 // Treat an enum type as its underlying type.
392 if (const auto *ED = Ty->getAsEnumDecl())
393 Ty = ED->getIntegerType();
394
395 if (const auto *EIT = Ty->getAs<BitIntType>())
396 if (EIT->getNumBits() > 128)
397 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
398 ByVal: false);
399
400 if (Ty->isVectorType())
401 NSRN = std::min(a: NSRN + 1, b: 8u);
402 else if (const auto *BT = Ty->getAs<BuiltinType>()) {
403 if (BT->isFloatingPoint())
404 NSRN = std::min(a: NSRN + 1, b: 8u);
405 else {
406 switch (BT->getKind()) {
407 case BuiltinType::SveBool:
408 case BuiltinType::SveCount:
409 NPRN = std::min(a: NPRN + 1, b: 4u);
410 break;
411 case BuiltinType::SveBoolx2:
412 NPRN = std::min(a: NPRN + 2, b: 4u);
413 break;
414 case BuiltinType::SveBoolx4:
415 NPRN = std::min(a: NPRN + 4, b: 4u);
416 break;
417 case BuiltinType::MFloat8:
418 NSRN = std::min(a: NSRN + 1, b: 8u);
419 break;
420 default:
421 if (BT->isSVESizelessBuiltinType())
422 NSRN = std::min(
423 a: NSRN + getContext().getBuiltinVectorTypeInfo(VecTy: BT).NumVectors,
424 b: 8u);
425 }
426 }
427 }
428
429 return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS()
430 ? ABIArgInfo::getExtend(Ty, T: CGT.ConvertType(T: Ty))
431 : ABIArgInfo::getDirect());
432 }
433
434 // Structures with either a non-trivial destructor or a non-trivial
435 // copy constructor are always indirect.
436 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(T: Ty, CXXABI&: getCXXABI())) {
437 return getNaturalAlignIndirect(
438 Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),
439 /*ByVal=*/RAA == CGCXXABI::RAA_DirectInMemory);
440 }
441
442 // Empty records:
443 // AAPCS64 does not say that empty records are ignored as arguments,
444 // but other compilers do so in certain situations, and we copy that behavior.
445 // Those situations are in fact language-mode-specific, which seems really
446 // unfortunate, but it's something we just have to accept. If this doesn't
447 // apply, just fall through to the standard argument-handling path.
448 // Darwin overrides the psABI here to ignore all empty records in all modes.
449 uint64_t Size = getContext().getTypeSize(T: Ty);
450 bool IsEmpty = isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true);
451 if (!Ty->isSVESizelessBuiltinType() && (IsEmpty || Size == 0)) {
452 // Empty records are ignored in C mode, and in C++ on Darwin.
453 if (!getContext().getLangOpts().CPlusPlus || isDarwinPCS())
454 return ABIArgInfo::getIgnore();
455
456 // In C++ mode, arguments which have sizeof() == 0 (which are non-standard
457 // C++) are ignored. This isn't defined by any standard, so we copy GCC's
458 // behaviour here.
459 if (Size == 0)
460 return ABIArgInfo::getIgnore();
461 }
462
463 // Homogeneous Floating-point Aggregates (HFAs) need to be expanded.
464 const Type *Base = nullptr;
465 uint64_t Members = 0;
466 bool IsWin64 = Kind == AArch64ABIKind::Win64 ||
467 CallingConvention == llvm::CallingConv::Win64;
468 bool IsWinVariadic = IsWin64 && IsVariadicFn;
469 // In variadic functions on Windows, all composite types are treated alike,
470 // no special handling of HFAs/HVAs.
471 if (!IsWinVariadic && isHomogeneousAggregate(Ty, Base, Members)) {
472 NSRN = std::min(a: NSRN + Members, b: uint64_t(8));
473 if (Kind != AArch64ABIKind::AAPCS)
474 return ABIArgInfo::getDirect(
475 T: llvm::ArrayType::get(ElementType: CGT.ConvertType(T: QualType(Base, 0)), NumElements: Members));
476
477 // For HFAs/HVAs, cap the argument alignment to 16, otherwise
478 // set it to 8 according to the AAPCS64 document.
479 unsigned Align =
480 getContext().getTypeUnadjustedAlignInChars(T: Ty).getQuantity();
481 Align = (Align >= 16) ? 16 : 8;
482 return ABIArgInfo::getDirect(
483 T: llvm::ArrayType::get(ElementType: CGT.ConvertType(T: QualType(Base, 0)), NumElements: Members), Offset: 0,
484 Padding: nullptr, CanBeFlattened: true, Align);
485 }
486
487 // In AAPCS named arguments of a Pure Scalable Type are passed expanded in
488 // registers, or indirectly if there are not enough registers.
489 if (Kind == AArch64ABIKind::AAPCS) {
490 unsigned NVec = 0, NPred = 0;
491 SmallVector<llvm::Type *> UnpaddedCoerceToSeq;
492 if (passAsPureScalableType(Ty, NV&: NVec, NP&: NPred, CoerceToSeq&: UnpaddedCoerceToSeq) &&
493 (NVec + NPred) > 0)
494 return coerceAndExpandPureScalableAggregate(
495 Ty, IsNamedArg, NVec, NPred, UnpaddedCoerceToSeq, NSRN, NPRN);
496 }
497
498 // Aggregates <= 16 bytes are passed directly in registers or on the stack.
499 if (Size <= 128) {
500 unsigned Alignment;
501 if (Kind == AArch64ABIKind::AAPCS) {
502 Alignment = getContext().getTypeUnadjustedAlign(T: Ty);
503 Alignment = Alignment < 128 ? 64 : 128;
504 } else {
505 Alignment =
506 std::max(a: getContext().getTypeAlign(T: Ty),
507 b: (unsigned)getTarget().getPointerWidth(AddrSpace: LangAS::Default));
508 }
509 if (Size < 64 && getDataLayout().isLittleEndian() && !isDarwinPCS())
510 return ABIArgInfo::getDirect(
511 T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Size));
512
513 Size = llvm::alignTo(Value: Size, Align: Alignment);
514
515 // If the Aggregate is made up of pointers, use an array of pointers for the
516 // coerced type. This prevents having to convert ptr2int->int2ptr through
517 // the call, allowing alias analysis to produce better code.
518 auto ContainsOnlyPointers = [&](const auto &Self, QualType Ty) {
519 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
520 return false;
521 const auto *RD = Ty->getAsRecordDecl();
522 if (!RD)
523 return false;
524 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
525 for (const auto &I : CXXRD->bases())
526 if (!Self(Self, I.getType()))
527 return false;
528 }
529 return all_of(RD->fields(), [&](FieldDecl *FD) {
530 QualType FDTy = FD->getType();
531 if (FDTy->isArrayType())
532 FDTy = getContext().getBaseElementType(QT: FDTy);
533 return (FDTy->isPointerOrReferenceType() &&
534 getContext().getTypeSize(T: FDTy) == 64 &&
535 !FDTy->getPointeeType().hasAddressSpace()) ||
536 Self(Self, FDTy);
537 });
538 };
539
540 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
541 // For aggregates with 16-byte alignment, we use i128.
542 llvm::Type *BaseTy = llvm::Type::getIntNTy(C&: getVMContext(), N: Alignment);
543 if ((Size == 64 || Size == 128) && Alignment == 64 &&
544 ContainsOnlyPointers(ContainsOnlyPointers, Ty))
545 BaseTy = llvm::PointerType::getUnqual(C&: getVMContext());
546 return ABIArgInfo::getDirect(
547 T: Size == Alignment ? BaseTy
548 : llvm::ArrayType::get(ElementType: BaseTy, NumElements: Size / Alignment));
549 }
550
551 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
552 /*ByVal=*/false);
553}
554
555ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,
556 bool IsVariadicFn) const {
557 if (RetTy->isVoidType())
558 return ABIArgInfo::getIgnore();
559
560 if (const auto *VT = RetTy->getAs<VectorType>()) {
561 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
562 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
563 unsigned NSRN = 0, NPRN = 0;
564 return coerceIllegalVector(Ty: RetTy, NSRN, NPRN);
565 }
566 }
567
568 // Large vector types should be returned via memory.
569 if (RetTy->isVectorType() && getContext().getTypeSize(T: RetTy) > 128)
570 return getNaturalAlignIndirect(Ty: RetTy, AddrSpace: getDataLayout().getAllocaAddrSpace());
571
572 if (!passAsAggregateType(Ty: RetTy)) {
573 // Treat an enum type as its underlying type.
574 if (const auto *ED = RetTy->getAsEnumDecl())
575 RetTy = ED->getIntegerType();
576
577 if (const auto *EIT = RetTy->getAs<BitIntType>())
578 if (EIT->getNumBits() > 128)
579 return getNaturalAlignIndirect(Ty: RetTy,
580 AddrSpace: getDataLayout().getAllocaAddrSpace());
581
582 return (isPromotableIntegerTypeForABI(Ty: RetTy) && isDarwinPCS()
583 ? ABIArgInfo::getExtend(Ty: RetTy)
584 : ABIArgInfo::getDirect());
585 }
586
587 uint64_t Size = getContext().getTypeSize(T: RetTy);
588 if (!RetTy->isSVESizelessBuiltinType() &&
589 (isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true) || Size == 0))
590 return ABIArgInfo::getIgnore();
591
592 const Type *Base = nullptr;
593 uint64_t Members = 0;
594 if (isHomogeneousAggregate(Ty: RetTy, Base, Members) &&
595 !(getTarget().getTriple().getArch() == llvm::Triple::aarch64_32 &&
596 IsVariadicFn))
597 // Homogeneous Floating-point Aggregates (HFAs) are returned directly.
598 return ABIArgInfo::getDirect();
599
600 // In AAPCS return values of a Pure Scalable type are treated as a single
601 // named argument and passed expanded in registers, or indirectly if there are
602 // not enough registers.
603 if (Kind == AArch64ABIKind::AAPCS) {
604 unsigned NSRN = 0, NPRN = 0;
605 unsigned NVec = 0, NPred = 0;
606 SmallVector<llvm::Type *> UnpaddedCoerceToSeq;
607 if (passAsPureScalableType(Ty: RetTy, NV&: NVec, NP&: NPred, CoerceToSeq&: UnpaddedCoerceToSeq) &&
608 (NVec + NPred) > 0)
609 return coerceAndExpandPureScalableAggregate(
610 Ty: RetTy, /* IsNamedArg */ true, NVec, NPred, UnpaddedCoerceToSeq, NSRN,
611 NPRN);
612 }
613
614 // Aggregates <= 16 bytes are returned directly in registers or on the stack.
615 if (Size <= 128) {
616 if (Size <= 64 && getDataLayout().isLittleEndian()) {
617 // Composite types are returned in lower bits of a 64-bit register for LE,
618 // and in higher bits for BE. However, integer types are always returned
619 // in lower bits for both LE and BE, and they are not rounded up to
620 // 64-bits. We can skip rounding up of composite types for LE, but not for
621 // BE, otherwise composite types will be indistinguishable from integer
622 // types.
623 return ABIArgInfo::getDirect(
624 T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Size));
625 }
626
627 unsigned Alignment = getContext().getTypeAlign(T: RetTy);
628 Size = llvm::alignTo(Value: Size, Align: 64); // round up to multiple of 8 bytes
629
630 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.
631 // For aggregates with 16-byte alignment, we use i128.
632 if (Alignment < 128 && Size == 128) {
633 llvm::Type *BaseTy = llvm::Type::getInt64Ty(C&: getVMContext());
634 return ABIArgInfo::getDirect(T: llvm::ArrayType::get(ElementType: BaseTy, NumElements: Size / 64));
635 }
636 return ABIArgInfo::getDirect(T: llvm::IntegerType::get(C&: getVMContext(), NumBits: Size));
637 }
638
639 return getNaturalAlignIndirect(Ty: RetTy, AddrSpace: getDataLayout().getAllocaAddrSpace());
640}
641
642/// isIllegalVectorType - check whether the vector type is legal for AArch64.
643bool AArch64ABIInfo::isIllegalVectorType(QualType Ty) const {
644 if (const VectorType *VT = Ty->getAs<VectorType>()) {
645 // Check whether VT is a fixed-length SVE vector. These types are
646 // represented as scalable vectors in function args/return and must be
647 // coerced from fixed vectors.
648 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
649 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate)
650 return true;
651
652 // Check whether VT is legal.
653 unsigned NumElements = VT->getNumElements();
654 uint64_t Size = getContext().getTypeSize(T: VT);
655 // NumElements should be power of 2.
656 if (!llvm::isPowerOf2_32(Value: NumElements))
657 return true;
658
659 // arm64_32 has to be compatible with the ARM logic here, which allows huge
660 // vectors for some reason.
661 llvm::Triple Triple = getTarget().getTriple();
662 if (Triple.getArch() == llvm::Triple::aarch64_32 &&
663 Triple.isOSBinFormatMachO())
664 return Size <= 32;
665
666 return Size != 64 && (Size != 128 || NumElements == 1);
667 }
668 return false;
669}
670
671bool AArch64SwiftABIInfo::isLegalVectorType(CharUnits VectorSize,
672 llvm::Type *EltTy,
673 unsigned NumElts) const {
674 if (!llvm::isPowerOf2_32(Value: NumElts))
675 return false;
676 if (VectorSize.getQuantity() != 8 &&
677 (VectorSize.getQuantity() != 16 || NumElts == 1))
678 return false;
679 return true;
680}
681
682bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
683 // For the soft-float ABI variant, no types are considered to be homogeneous
684 // aggregates.
685 if (isSoftFloat())
686 return false;
687
688 // Homogeneous aggregates for AAPCS64 must have base types of a floating
689 // point type or a short-vector type. This is the same as the 32-bit ABI,
690 // but with the difference that any floating-point type is allowed,
691 // including __fp16.
692 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
693 if (BT->isFloatingPoint())
694 return true;
695 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
696 if (auto Kind = VT->getVectorKind();
697 Kind == VectorKind::SveFixedLengthData ||
698 Kind == VectorKind::SveFixedLengthPredicate)
699 return false;
700
701 unsigned VecSize = getContext().getTypeSize(T: VT);
702 if (VecSize == 64 || VecSize == 128)
703 return true;
704 }
705 return false;
706}
707
708bool AArch64ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
709 uint64_t Members) const {
710 return Members <= 4;
711}
712
713bool AArch64ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate()
714 const {
715 // AAPCS64 says that the rule for whether something is a homogeneous
716 // aggregate is applied to the output of the data layout decision. So
717 // anything that doesn't affect the data layout also does not affect
718 // homogeneity. In particular, zero-length bitfields don't stop a struct
719 // being homogeneous.
720 return true;
721}
722
723bool AArch64ABIInfo::passAsAggregateType(QualType Ty) const {
724 if (Kind == AArch64ABIKind::AAPCS && Ty->isSVESizelessBuiltinType()) {
725 const auto *BT = Ty->castAs<BuiltinType>();
726 return !BT->isSVECount() &&
727 getContext().getBuiltinVectorTypeInfo(VecTy: BT).NumVectors > 1;
728 }
729 return isAggregateTypeForABI(T: Ty);
730}
731
732// Check if a type needs to be passed in registers as a Pure Scalable Type (as
733// defined by AAPCS64). Return the number of data vectors and the number of
734// predicate vectors in the type, into `NVec` and `NPred`, respectively. Upon
735// return `CoerceToSeq` contains an expanded sequence of LLVM IR types, one
736// element for each non-composite member. For practical purposes, limit the
737// length of `CoerceToSeq` to about 12 (the maximum that could possibly fit
738// in registers) and return false, the effect of which will be to pass the
739// argument under the rules for a large (> 128 bytes) composite.
740bool AArch64ABIInfo::passAsPureScalableType(
741 QualType Ty, unsigned &NVec, unsigned &NPred,
742 SmallVectorImpl<llvm::Type *> &CoerceToSeq) const {
743 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(T: Ty)) {
744 uint64_t NElt = AT->getZExtSize();
745 if (NElt == 0)
746 return false;
747
748 unsigned NV = 0, NP = 0;
749 SmallVector<llvm::Type *> EltCoerceToSeq;
750 if (!passAsPureScalableType(Ty: AT->getElementType(), NVec&: NV, NPred&: NP, CoerceToSeq&: EltCoerceToSeq))
751 return false;
752
753 if (CoerceToSeq.size() + NElt * EltCoerceToSeq.size() > 12)
754 return false;
755
756 for (uint64_t I = 0; I < NElt; ++I)
757 llvm::append_range(C&: CoerceToSeq, R&: EltCoerceToSeq);
758
759 NVec += NElt * NV;
760 NPred += NElt * NP;
761 return true;
762 }
763
764 if (const RecordType *RT = Ty->getAsCanonical<RecordType>()) {
765 // If the record cannot be passed in registers, then it's not a PST.
766 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, CXXABI&: getCXXABI());
767 RAA != CGCXXABI::RAA_Default)
768 return false;
769
770 // Pure scalable types are never unions and never contain unions.
771 const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf();
772 if (RD->isUnion())
773 return false;
774
775 // If this is a C++ record, check the bases.
776 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
777 for (const auto &I : CXXRD->bases()) {
778 if (isEmptyRecord(Context&: getContext(), T: I.getType(), AllowArrays: true))
779 continue;
780 if (!passAsPureScalableType(Ty: I.getType(), NVec, NPred, CoerceToSeq))
781 return false;
782 }
783 }
784
785 // Check members.
786 for (const auto *FD : RD->fields()) {
787 QualType FT = FD->getType();
788 if (isEmptyField(Context&: getContext(), FD, /* AllowArrays */ true))
789 continue;
790 if (!passAsPureScalableType(Ty: FT, NVec, NPred, CoerceToSeq))
791 return false;
792 }
793
794 return true;
795 }
796
797 if (const auto *VT = Ty->getAs<VectorType>()) {
798 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {
799 ++NPred;
800 if (CoerceToSeq.size() + 1 > 12)
801 return false;
802 CoerceToSeq.push_back(Elt: convertFixedToScalableVectorType(VT));
803 return true;
804 }
805
806 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {
807 ++NVec;
808 if (CoerceToSeq.size() + 1 > 12)
809 return false;
810 CoerceToSeq.push_back(Elt: convertFixedToScalableVectorType(VT));
811 return true;
812 }
813
814 return false;
815 }
816
817 if (!Ty->isBuiltinType())
818 return false;
819
820 bool isPredicate;
821 switch (Ty->castAs<BuiltinType>()->getKind()) {
822#define SVE_VECTOR_TYPE(Name, MangledName, Id, SingletonId) \
823 case BuiltinType::Id: \
824 isPredicate = false; \
825 break;
826#define SVE_PREDICATE_TYPE(Name, MangledName, Id, SingletonId) \
827 case BuiltinType::Id: \
828 isPredicate = true; \
829 break;
830#include "clang/Basic/AArch64ACLETypes.def"
831 default:
832 return false;
833 }
834
835 ASTContext::BuiltinVectorTypeInfo Info =
836 getContext().getBuiltinVectorTypeInfo(VecTy: cast<BuiltinType>(Val&: Ty));
837 assert(Info.NumVectors > 0 && Info.NumVectors <= 4 &&
838 "Expected 1, 2, 3 or 4 vectors!");
839 if (isPredicate)
840 NPred += Info.NumVectors;
841 else
842 NVec += Info.NumVectors;
843 llvm::Type *EltTy = Info.ElementType->isMFloat8Type()
844 ? llvm::Type::getInt8Ty(C&: getVMContext())
845 : CGT.ConvertType(T: Info.ElementType);
846 auto *VTy = llvm::ScalableVectorType::get(ElementType: EltTy, MinNumElts: Info.EC.getKnownMinValue());
847
848 if (CoerceToSeq.size() + Info.NumVectors > 12)
849 return false;
850 std::fill_n(first: std::back_inserter(x&: CoerceToSeq), n: Info.NumVectors, value: VTy);
851
852 return true;
853}
854
855// Expand an LLVM IR type into a sequence with a element for each non-struct,
856// non-array member of the type, with the exception of the padding types, which
857// are retained.
858void AArch64ABIInfo::flattenType(
859 llvm::Type *Ty, SmallVectorImpl<llvm::Type *> &Flattened) const {
860
861 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType: Ty)) {
862 Flattened.push_back(Elt: Ty);
863 return;
864 }
865
866 if (const auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty)) {
867 uint64_t NElt = AT->getNumElements();
868 if (NElt == 0)
869 return;
870
871 SmallVector<llvm::Type *> EltFlattened;
872 flattenType(Ty: AT->getElementType(), Flattened&: EltFlattened);
873
874 for (uint64_t I = 0; I < NElt; ++I)
875 llvm::append_range(C&: Flattened, R&: EltFlattened);
876 return;
877 }
878
879 if (const auto *ST = dyn_cast<llvm::StructType>(Val: Ty)) {
880 for (auto *ET : ST->elements())
881 flattenType(Ty: ET, Flattened);
882 return;
883 }
884
885 Flattened.push_back(Elt: Ty);
886}
887
888RValue AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,
889 CodeGenFunction &CGF, AArch64ABIKind Kind,
890 AggValueSlot Slot) const {
891 // These numbers are not used for variadic arguments, hence it doesn't matter
892 // they don't retain their values across multiple calls to
893 // `classifyArgumentType` here.
894 unsigned NSRN = 0, NPRN = 0;
895 ABIArgInfo AI =
896 classifyArgumentType(Ty, /*IsVariadicFn=*/true, /* IsNamedArg */ false,
897 CallingConvention: CGF.CurFnInfo->getCallingConvention(), NSRN, NPRN);
898 // Empty records are ignored for parameter passing purposes.
899 if (AI.isIgnore())
900 return Slot.asRValue();
901
902 bool IsIndirect = AI.isIndirect();
903
904 llvm::Type *BaseTy = CGF.ConvertType(T: Ty);
905 if (IsIndirect)
906 BaseTy = llvm::PointerType::getUnqual(C&: BaseTy->getContext());
907 else if (AI.getCoerceToType())
908 BaseTy = AI.getCoerceToType();
909
910 unsigned NumRegs = 1;
911 if (llvm::ArrayType *ArrTy = dyn_cast<llvm::ArrayType>(Val: BaseTy)) {
912 BaseTy = ArrTy->getElementType();
913 NumRegs = ArrTy->getNumElements();
914 }
915 bool IsFPR =
916 !isSoftFloat() && (BaseTy->isFloatingPointTy() || BaseTy->isVectorTy());
917
918 // The AArch64 va_list type and handling is specified in the Procedure Call
919 // Standard, section B.4:
920 //
921 // struct {
922 // void *__stack;
923 // void *__gr_top;
924 // void *__vr_top;
925 // int __gr_offs;
926 // int __vr_offs;
927 // };
928
929 llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock(name: "vaarg.maybe_reg");
930 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock(name: "vaarg.in_reg");
931 llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock(name: "vaarg.on_stack");
932 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "vaarg.end");
933
934 CharUnits TySize = getContext().getTypeSizeInChars(T: Ty);
935 CharUnits TyAlign = getContext().getTypeUnadjustedAlignInChars(T: Ty);
936
937 Address reg_offs_p = Address::invalid();
938 llvm::Value *reg_offs = nullptr;
939 int reg_top_index;
940 int RegSize = IsIndirect ? 8 : TySize.getQuantity();
941 if (!IsFPR) {
942 // 3 is the field number of __gr_offs
943 reg_offs_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 3, Name: "gr_offs_p");
944 reg_offs = CGF.Builder.CreateLoad(Addr: reg_offs_p, Name: "gr_offs");
945 reg_top_index = 1; // field number for __gr_top
946 RegSize = llvm::alignTo(Value: RegSize, Align: 8);
947 } else {
948 // 4 is the field number of __vr_offs.
949 reg_offs_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 4, Name: "vr_offs_p");
950 reg_offs = CGF.Builder.CreateLoad(Addr: reg_offs_p, Name: "vr_offs");
951 reg_top_index = 2; // field number for __vr_top
952 RegSize = 16 * NumRegs;
953 }
954
955 //=======================================
956 // Find out where argument was passed
957 //=======================================
958
959 // If reg_offs >= 0 we're already using the stack for this type of
960 // argument. We don't want to keep updating reg_offs (in case it overflows,
961 // though anyone passing 2GB of arguments, each at most 16 bytes, deserves
962 // whatever they get).
963 llvm::Value *UsingStack = nullptr;
964 UsingStack = CGF.Builder.CreateICmpSGE(
965 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
966
967 CGF.Builder.CreateCondBr(Cond: UsingStack, True: OnStackBlock, False: MaybeRegBlock);
968
969 // Otherwise, at least some kind of argument could go in these registers, the
970 // question is whether this particular type is too big.
971 CGF.EmitBlock(BB: MaybeRegBlock);
972
973 // Integer arguments may need to correct register alignment (for example a
974 // "struct { __int128 a; };" gets passed in x_2N, x_{2N+1}). In this case we
975 // align __gr_offs to calculate the potential address.
976 if (!IsFPR && !IsIndirect && TyAlign.getQuantity() > 8) {
977 int Align = TyAlign.getQuantity();
978
979 reg_offs = CGF.Builder.CreateAdd(
980 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: Align - 1),
981 Name: "align_regoffs");
982 reg_offs = CGF.Builder.CreateAnd(
983 LHS: reg_offs, RHS: llvm::ConstantInt::getSigned(Ty: CGF.Int32Ty, V: -Align),
984 Name: "aligned_regoffs");
985 }
986
987 // Update the gr_offs/vr_offs pointer for next call to va_arg on this va_list.
988 // The fact that this is done unconditionally reflects the fact that
989 // allocating an argument to the stack also uses up all the remaining
990 // registers of the appropriate kind.
991 llvm::Value *NewOffset = nullptr;
992 NewOffset = CGF.Builder.CreateAdd(
993 LHS: reg_offs, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: RegSize), Name: "new_reg_offs");
994 CGF.Builder.CreateStore(Val: NewOffset, Addr: reg_offs_p);
995
996 // Now we're in a position to decide whether this argument really was in
997 // registers or not.
998 llvm::Value *InRegs = nullptr;
999 InRegs = CGF.Builder.CreateICmpSLE(
1000 LHS: NewOffset, RHS: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0), Name: "inreg");
1001
1002 CGF.Builder.CreateCondBr(Cond: InRegs, True: InRegBlock, False: OnStackBlock);
1003
1004 //=======================================
1005 // Argument was in registers
1006 //=======================================
1007
1008 // Now we emit the code for if the argument was originally passed in
1009 // registers. First start the appropriate block:
1010 CGF.EmitBlock(BB: InRegBlock);
1011
1012 llvm::Value *reg_top = nullptr;
1013 Address reg_top_p =
1014 CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: reg_top_index, Name: "reg_top_p");
1015 reg_top = CGF.Builder.CreateLoad(Addr: reg_top_p, Name: "reg_top");
1016 Address BaseAddr(CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: reg_top, IdxList: reg_offs),
1017 CGF.Int8Ty, CharUnits::fromQuantity(Quantity: IsFPR ? 16 : 8));
1018 Address RegAddr = Address::invalid();
1019 llvm::Type *MemTy = CGF.ConvertTypeForMem(T: Ty), *ElementTy = MemTy;
1020
1021 if (IsIndirect) {
1022 // If it's been passed indirectly (actually a struct), whatever we find from
1023 // stored registers or on the stack will actually be a struct **.
1024 MemTy = llvm::PointerType::getUnqual(C&: MemTy->getContext());
1025 }
1026
1027 const Type *Base = nullptr;
1028 uint64_t NumMembers = 0;
1029 bool IsHFA = isHomogeneousAggregate(Ty, Base, Members&: NumMembers);
1030 if (IsHFA && NumMembers > 1) {
1031 // Homogeneous aggregates passed in registers will have their elements split
1032 // and stored 16-bytes apart regardless of size (they're notionally in qN,
1033 // qN+1, ...). We reload and store into a temporary local variable
1034 // contiguously.
1035 assert(!IsIndirect && "Homogeneous aggregates should be passed directly");
1036 auto BaseTyInfo = getContext().getTypeInfoInChars(T: QualType(Base, 0));
1037 llvm::Type *BaseTy = CGF.ConvertType(T: QualType(Base, 0));
1038 llvm::Type *HFATy = llvm::ArrayType::get(ElementType: BaseTy, NumElements: NumMembers);
1039 Address Tmp = CGF.CreateTempAlloca(Ty: HFATy,
1040 align: std::max(a: TyAlign, b: BaseTyInfo.Align));
1041
1042 // On big-endian platforms, the value will be right-aligned in its slot.
1043 int Offset = 0;
1044 if (CGF.CGM.getDataLayout().isBigEndian() &&
1045 BaseTyInfo.Width.getQuantity() < 16)
1046 Offset = 16 - BaseTyInfo.Width.getQuantity();
1047
1048 for (unsigned i = 0; i < NumMembers; ++i) {
1049 CharUnits BaseOffset = CharUnits::fromQuantity(Quantity: 16 * i + Offset);
1050 Address LoadAddr =
1051 CGF.Builder.CreateConstInBoundsByteGEP(Addr: BaseAddr, Offset: BaseOffset);
1052 LoadAddr = LoadAddr.withElementType(ElemTy: BaseTy);
1053
1054 Address StoreAddr = CGF.Builder.CreateConstArrayGEP(Addr: Tmp, Index: i);
1055
1056 llvm::Value *Elem = CGF.Builder.CreateLoad(Addr: LoadAddr);
1057 CGF.Builder.CreateStore(Val: Elem, Addr: StoreAddr);
1058 }
1059
1060 RegAddr = Tmp.withElementType(ElemTy: MemTy);
1061 } else {
1062 // Otherwise the object is contiguous in memory.
1063
1064 // It might be right-aligned in its slot.
1065 CharUnits SlotSize = BaseAddr.getAlignment();
1066 if (CGF.CGM.getDataLayout().isBigEndian() && !IsIndirect &&
1067 (IsHFA || !isAggregateTypeForABI(T: Ty)) &&
1068 TySize < SlotSize) {
1069 CharUnits Offset = SlotSize - TySize;
1070 BaseAddr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: BaseAddr, Offset);
1071 }
1072
1073 RegAddr = BaseAddr.withElementType(ElemTy: MemTy);
1074 }
1075
1076 CGF.EmitBranch(Block: ContBlock);
1077
1078 //=======================================
1079 // Argument was on the stack
1080 //=======================================
1081 CGF.EmitBlock(BB: OnStackBlock);
1082
1083 Address stack_p = CGF.Builder.CreateStructGEP(Addr: VAListAddr, Index: 0, Name: "stack_p");
1084 llvm::Value *OnStackPtr = CGF.Builder.CreateLoad(Addr: stack_p, Name: "stack");
1085
1086 // Again, stack arguments may need realignment. In this case both integer and
1087 // floating-point ones might be affected.
1088 if (!IsIndirect && TyAlign.getQuantity() > 8) {
1089 OnStackPtr = emitRoundPointerUpToAlignment(CGF, Ptr: OnStackPtr, Align: TyAlign);
1090 }
1091 Address OnStackAddr = Address(OnStackPtr, CGF.Int8Ty,
1092 std::max(a: CharUnits::fromQuantity(Quantity: 8), b: TyAlign));
1093
1094 // All stack slots are multiples of 8 bytes.
1095 CharUnits StackSlotSize = CharUnits::fromQuantity(Quantity: 8);
1096 CharUnits StackSize;
1097 if (IsIndirect)
1098 StackSize = StackSlotSize;
1099 else
1100 StackSize = TySize.alignTo(Align: StackSlotSize);
1101
1102 llvm::Value *StackSizeC = CGF.Builder.getSize(N: StackSize);
1103 llvm::Value *NewStack = CGF.Builder.CreateInBoundsGEP(
1104 Ty: CGF.Int8Ty, Ptr: OnStackPtr, IdxList: StackSizeC, Name: "new_stack");
1105
1106 // Write the new value of __stack for the next call to va_arg
1107 CGF.Builder.CreateStore(Val: NewStack, Addr: stack_p);
1108
1109 if (CGF.CGM.getDataLayout().isBigEndian() && !isAggregateTypeForABI(T: Ty) &&
1110 TySize < StackSlotSize) {
1111 CharUnits Offset = StackSlotSize - TySize;
1112 OnStackAddr = CGF.Builder.CreateConstInBoundsByteGEP(Addr: OnStackAddr, Offset);
1113 }
1114
1115 OnStackAddr = OnStackAddr.withElementType(ElemTy: MemTy);
1116
1117 CGF.EmitBranch(Block: ContBlock);
1118
1119 //=======================================
1120 // Tidy up
1121 //=======================================
1122 CGF.EmitBlock(BB: ContBlock);
1123
1124 Address ResAddr = emitMergePHI(CGF, Addr1: RegAddr, Block1: InRegBlock, Addr2: OnStackAddr,
1125 Block2: OnStackBlock, Name: "vaargs.addr");
1126
1127 if (IsIndirect)
1128 return CGF.EmitLoadOfAnyValue(
1129 V: CGF.MakeAddrLValue(
1130 Addr: Address(CGF.Builder.CreateLoad(Addr: ResAddr, Name: "vaarg.addr"), ElementTy,
1131 TyAlign),
1132 T: Ty),
1133 Slot);
1134
1135 return CGF.EmitLoadOfAnyValue(V: CGF.MakeAddrLValue(Addr: ResAddr, T: Ty), Slot);
1136}
1137
1138RValue AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty,
1139 CodeGenFunction &CGF,
1140 AggValueSlot Slot) const {
1141 // The backend's lowering doesn't support va_arg for aggregates or
1142 // illegal vector types. Lower VAArg here for these cases and use
1143 // the LLVM va_arg instruction for everything else.
1144 if (!isAggregateTypeForABI(T: Ty) && !isIllegalVectorType(Ty))
1145 return CGF.EmitLoadOfAnyValue(
1146 V: CGF.MakeAddrLValue(
1147 Addr: EmitVAArgInstr(CGF, VAListAddr, Ty, AI: ABIArgInfo::getDirect()), T: Ty),
1148 Slot);
1149
1150 uint64_t PointerSize = getTarget().getPointerWidth(AddrSpace: LangAS::Default) / 8;
1151 CharUnits SlotSize = CharUnits::fromQuantity(Quantity: PointerSize);
1152
1153 // Empty records are ignored for parameter passing purposes.
1154 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
1155 return Slot.asRValue();
1156
1157 // The size of the actual thing passed, which might end up just
1158 // being a pointer for indirect types.
1159 auto TyInfo = getContext().getTypeInfoInChars(T: Ty);
1160
1161 // Arguments bigger than 16 bytes which aren't homogeneous
1162 // aggregates should be passed indirectly.
1163 bool IsIndirect = false;
1164 if (TyInfo.Width.getQuantity() > 16) {
1165 const Type *Base = nullptr;
1166 uint64_t Members = 0;
1167 IsIndirect = !isHomogeneousAggregate(Ty, Base, Members);
1168 }
1169
1170 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect, ValueInfo: TyInfo, SlotSizeAndAlign: SlotSize,
1171 /*AllowHigherAlign*/ true, Slot);
1172}
1173
1174RValue AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
1175 QualType Ty, AggValueSlot Slot) const {
1176 bool AllowHigherAlign = false;
1177 bool IsIndirect = false;
1178
1179 if (getTarget().getTriple().isWindowsArm64EC()) {
1180 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
1181 // not 1, 2, 4, or 8 bytes, must be passed by reference."
1182 uint64_t Width = getContext().getTypeSize(T: Ty);
1183 IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Value: Width);
1184 } else {
1185 // E.g. __int128 when passed is aligned to 16 bytes, so it must be read
1186 // with the same alignment.
1187 AllowHigherAlign = true;
1188
1189 // Composites larger than 16 bytes are passed by reference.
1190 if (isAggregateTypeForABI(T: Ty) && getContext().getTypeSize(T: Ty) > 128)
1191 IsIndirect = true;
1192 }
1193
1194 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, IsIndirect,
1195 ValueInfo: CGF.getContext().getTypeInfoInChars(T: Ty),
1196 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 8), AllowHigherAlign, Slot);
1197}
1198
1199// Report an error if an argument or return value of type Ty would need to be
1200// passed in a floating-point register.
1201static void diagnoseIfNeedsFPReg(DiagnosticsEngine &Diags,
1202 const StringRef ABIName,
1203 const AArch64ABIInfo &ABIInfo,
1204 const QualType &Ty, const NamedDecl *D,
1205 SourceLocation loc) {
1206 const Type *HABase = nullptr;
1207 uint64_t HAMembers = 0;
1208 if (Ty->isFloatingType() || Ty->isVectorType() ||
1209 ABIInfo.isHomogeneousAggregate(Ty, Base&: HABase, Members&: HAMembers)) {
1210 Diags.Report(Loc: loc, DiagID: diag::err_target_unsupported_type_for_abi)
1211 << D->getDeclName() << Ty << ABIName;
1212 }
1213}
1214
1215// If we are using a hard-float ABI, but do not have floating point registers,
1216// then report an error for any function arguments or returns which would be
1217// passed in floating-pint registers.
1218void AArch64TargetCodeGenInfo::checkFunctionABI(
1219 CodeGenModule &CGM, const FunctionDecl *FuncDecl) const {
1220 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
1221 const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();
1222
1223 if (!TI.hasFeature(Feature: "fp") && !ABIInfo.isSoftFloat()) {
1224 diagnoseIfNeedsFPReg(Diags&: CGM.getDiags(), ABIName: TI.getABI(), ABIInfo,
1225 Ty: FuncDecl->getReturnType(), D: FuncDecl,
1226 loc: FuncDecl->getLocation());
1227 for (ParmVarDecl *PVD : FuncDecl->parameters()) {
1228 diagnoseIfNeedsFPReg(Diags&: CGM.getDiags(), ABIName: TI.getABI(), ABIInfo, Ty: PVD->getType(),
1229 D: PVD, loc: FuncDecl->getLocation());
1230 }
1231 }
1232}
1233
1234void AArch64TargetCodeGenInfo::checkFunctionCallABIStreaming(
1235 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
1236 const FunctionDecl *Callee) const {
1237 if (!Caller || !Callee || !Callee->hasAttr<AlwaysInlineAttr>())
1238 return;
1239
1240 CodeGenUtils::ArmSMEInlinability Inlinability =
1241 CodeGenUtils::getArmSMEInlinability(Caller, Callee);
1242
1243 if ((Inlinability &
1244 CodeGenUtils::ArmSMEInlinability::IncompatibleStreamingModes) !=
1245 CodeGenUtils::ArmSMEInlinability::Ok)
1246 CGM.getDiags().Report(
1247 Loc: CallLoc,
1248 DiagID: (Inlinability &
1249 CodeGenUtils::ArmSMEInlinability::ErrorIncompatibleStreamingModes) ==
1250 CodeGenUtils::ArmSMEInlinability::
1251 ErrorIncompatibleStreamingModes
1252 ? diag::err_function_always_inline_attribute_mismatch
1253 : diag::warn_function_always_inline_attribute_mismatch)
1254 << Caller->getDeclName() << Callee->getDeclName() << "streaming";
1255
1256 if ((Inlinability &
1257 CodeGenUtils::ArmSMEInlinability::ErrorCalleeRequiresNewZA) ==
1258 CodeGenUtils::ArmSMEInlinability::ErrorCalleeRequiresNewZA)
1259 CGM.getDiags().Report(Loc: CallLoc, DiagID: diag::err_function_always_inline_new_za)
1260 << Callee->getDeclName();
1261
1262 if ((Inlinability &
1263 CodeGenUtils::ArmSMEInlinability::ErrorCalleeRequiresNewZT0) ==
1264 CodeGenUtils::ArmSMEInlinability::ErrorCalleeRequiresNewZT0)
1265 CGM.getDiags().Report(Loc: CallLoc, DiagID: diag::err_function_always_inline_new_zt0)
1266 << Callee->getDeclName();
1267}
1268
1269// If the target does not have floating-point registers, but we are using a
1270// hard-float ABI, there is no way to pass floating-point, vector or HFA values
1271// to functions, so we report an error.
1272void AArch64TargetCodeGenInfo::checkFunctionCallABISoftFloat(
1273 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
1274 const FunctionDecl *Callee, const CallArgList &Args,
1275 QualType ReturnType) const {
1276 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();
1277 const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();
1278
1279 if (!Caller || TI.hasFeature(Feature: "fp") || ABIInfo.isSoftFloat())
1280 return;
1281
1282 diagnoseIfNeedsFPReg(Diags&: CGM.getDiags(), ABIName: TI.getABI(), ABIInfo, Ty: ReturnType,
1283 D: Callee ? Callee : Caller, loc: CallLoc);
1284
1285 for (const CallArg &Arg : Args)
1286 diagnoseIfNeedsFPReg(Diags&: CGM.getDiags(), ABIName: TI.getABI(), ABIInfo, Ty: Arg.getType(),
1287 D: Callee ? Callee : Caller, loc: CallLoc);
1288}
1289
1290void AArch64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM,
1291 SourceLocation CallLoc,
1292 const FunctionDecl *Caller,
1293 const FunctionDecl *Callee,
1294 const CallArgList &Args,
1295 QualType ReturnType) const {
1296 checkFunctionCallABIStreaming(CGM, CallLoc, Caller, Callee);
1297 checkFunctionCallABISoftFloat(CGM, CallLoc, Caller, Callee, Args, ReturnType);
1298}
1299
1300bool AArch64TargetCodeGenInfo::wouldInliningViolateFunctionCallABI(
1301 const FunctionDecl *Caller, const FunctionDecl *Callee) const {
1302 return Caller && Callee &&
1303 CodeGenUtils::getArmSMEInlinability(Caller, Callee) !=
1304 CodeGenUtils::ArmSMEInlinability::Ok;
1305}
1306
1307void AArch64ABIInfo::appendAttributeMangling(TargetClonesAttr *Attr,
1308 unsigned Index,
1309 raw_ostream &Out) const {
1310 appendAttributeMangling(AttrStr: Attr->getFeatureStr(Index), Out);
1311}
1312
1313void AArch64ABIInfo::appendAttributeMangling(StringRef AttrStr,
1314 raw_ostream &Out) const {
1315 if (AttrStr == "default") {
1316 Out << ".default";
1317 return;
1318 }
1319
1320 Out << "._";
1321 SmallVector<StringRef, 8> Features;
1322 AttrStr.split(A&: Features, Separator: "+");
1323 for (auto &Feat : Features)
1324 Feat = Feat.trim();
1325
1326 llvm::sort(C&: Features, Comp: [](const StringRef LHS, const StringRef RHS) {
1327 return LHS.compare(RHS) < 0;
1328 });
1329
1330 llvm::SmallDenseSet<StringRef, 8> UniqueFeats;
1331 for (auto &Feat : Features)
1332 if (getTarget().doesFeatureAffectCodeGen(Feature: Feat))
1333 if (auto Ext = llvm::AArch64::parseFMVExtension(Extension: Feat))
1334 if (UniqueFeats.insert(V: Ext->Name).second)
1335 Out << 'M' << Ext->Name;
1336}
1337
1338std::unique_ptr<TargetCodeGenInfo>
1339CodeGen::createAArch64TargetCodeGenInfo(CodeGenModule &CGM,
1340 AArch64ABIKind Kind) {
1341 return std::make_unique<AArch64TargetCodeGenInfo>(args&: CGM, args&: Kind);
1342}
1343
1344std::unique_ptr<TargetCodeGenInfo>
1345CodeGen::createWindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM,
1346 AArch64ABIKind K) {
1347 return std::make_unique<WindowsAArch64TargetCodeGenInfo>(args&: CGM, args&: K);
1348}
1349