1//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// These classes wrap the information about a call or function
10// definition used to handle ABI compliancy.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CGCall.h"
15#include "ABIInfo.h"
16#include "ABIInfoImpl.h"
17#include "CGBlocks.h"
18#include "CGCXXABI.h"
19#include "CGCleanup.h"
20#include "CGDebugInfo.h"
21#include "CGRecordLayout.h"
22#include "CodeGenFunction.h"
23#include "CodeGenModule.h"
24#include "CodeGenPGO.h"
25#include "QualTypeMapper.h"
26#include "TargetInfo.h"
27#include "clang/AST/Attr.h"
28#include "clang/AST/Decl.h"
29#include "clang/AST/DeclCXX.h"
30#include "clang/AST/DeclObjC.h"
31#include "clang/AST/RecordLayout.h"
32#include "clang/Basic/CodeGenOptions.h"
33#include "clang/Basic/TargetInfo.h"
34#include "clang/CodeGen/CGFunctionInfo.h"
35#include "clang/CodeGen/SwiftCallingConv.h"
36#include "llvm/ABI/FunctionInfo.h"
37#include "llvm/ABI/IRTypeMapper.h"
38#include "llvm/ABI/TargetInfo.h"
39#include "llvm/ABI/Types.h"
40#include "llvm/ADT/STLExtras.h"
41#include "llvm/ADT/StringExtras.h"
42#include "llvm/Analysis/ValueTracking.h"
43#include "llvm/IR/Assumptions.h"
44#include "llvm/IR/AttributeMask.h"
45#include "llvm/IR/Attributes.h"
46#include "llvm/IR/CallingConv.h"
47#include "llvm/IR/DataLayout.h"
48#include "llvm/IR/DebugInfoMetadata.h"
49#include "llvm/IR/InlineAsm.h"
50#include "llvm/IR/IntrinsicInst.h"
51#include "llvm/IR/Intrinsics.h"
52#include "llvm/IR/Type.h"
53#include "llvm/Transforms/Utils/Local.h"
54#include <optional>
55using namespace clang;
56using namespace CodeGen;
57
58/***/
59
60unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
61 switch (CC) {
62 default:
63 return llvm::CallingConv::C;
64 case CC_X86StdCall:
65 return llvm::CallingConv::X86_StdCall;
66 case CC_X86FastCall:
67 return llvm::CallingConv::X86_FastCall;
68 case CC_X86RegCall:
69 return llvm::CallingConv::X86_RegCall;
70 case CC_X86ThisCall:
71 return llvm::CallingConv::X86_ThisCall;
72 case CC_Win64:
73 return llvm::CallingConv::Win64;
74 case CC_X86_64SysV:
75 return llvm::CallingConv::X86_64_SysV;
76 case CC_AAPCS:
77 return llvm::CallingConv::ARM_AAPCS;
78 case CC_AAPCS_VFP:
79 return llvm::CallingConv::ARM_AAPCS_VFP;
80 case CC_IntelOclBicc:
81 return llvm::CallingConv::Intel_OCL_BI;
82 // TODO: Add support for __pascal to LLVM.
83 case CC_X86Pascal:
84 return llvm::CallingConv::C;
85 // TODO: Add support for __vectorcall to LLVM.
86 case CC_X86VectorCall:
87 return llvm::CallingConv::X86_VectorCall;
88 case CC_AArch64VectorCall:
89 return llvm::CallingConv::AArch64_VectorCall;
90 case CC_AArch64SVEPCS:
91 return llvm::CallingConv::AArch64_SVE_VectorCall;
92 case CC_SpirFunction:
93 return llvm::CallingConv::SPIR_FUNC;
94 case CC_DeviceKernel:
95 return CGM.getTargetCodeGenInfo().getDeviceKernelCallingConv();
96 case CC_PreserveMost:
97 return llvm::CallingConv::PreserveMost;
98 case CC_PreserveAll:
99 return llvm::CallingConv::PreserveAll;
100 case CC_Swift:
101 return llvm::CallingConv::Swift;
102 case CC_SwiftAsync:
103 return llvm::CallingConv::SwiftTail;
104 case CC_M68kRTD:
105 return llvm::CallingConv::M68k_RTD;
106 case CC_PreserveNone:
107 return llvm::CallingConv::PreserveNone;
108 // clang-format off
109 case CC_RISCVVectorCall: return llvm::CallingConv::RISCV_VectorCall;
110 // clang-format on
111#define CC_VLS_CASE(ABI_VLEN) \
112 case CC_RISCVVLSCall_##ABI_VLEN: \
113 return llvm::CallingConv::RISCV_VLSCall_##ABI_VLEN;
114 CC_VLS_CASE(32)
115 CC_VLS_CASE(64)
116 CC_VLS_CASE(128)
117 CC_VLS_CASE(256)
118 CC_VLS_CASE(512)
119 CC_VLS_CASE(1024)
120 CC_VLS_CASE(2048)
121 CC_VLS_CASE(4096)
122 CC_VLS_CASE(8192)
123 CC_VLS_CASE(16384)
124 CC_VLS_CASE(32768)
125 CC_VLS_CASE(65536)
126#undef CC_VLS_CASE
127 }
128}
129
130/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
131/// qualification. Either or both of RD and MD may be null. A null RD indicates
132/// that there is no meaningful 'this' type, and a null MD can occur when
133/// calling a method pointer.
134CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
135 const CXXMethodDecl *MD) {
136 CanQualType RecTy;
137 if (RD)
138 RecTy = Context.getCanonicalTagType(TD: RD);
139 else
140 RecTy = Context.VoidTy;
141
142 if (MD)
143 RecTy = CanQualType::CreateUnsafe(Other: Context.getAddrSpaceQualType(
144 T: RecTy, AddressSpace: MD->getMethodQualifiers().getAddressSpace()));
145 return Context.getPointerType(T: RecTy);
146}
147
148/// Returns the canonical formal type of the given C++ method.
149static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
150 return MD->getType()
151 ->getCanonicalTypeUnqualified()
152 .getAs<FunctionProtoType>();
153}
154
155/// Returns the "extra-canonicalized" return type, which discards
156/// qualifiers on the return type. Codegen doesn't care about them,
157/// and it makes ABI code a little easier to be able to assume that
158/// all parameter and return types are top-level unqualified.
159static CanQualType GetReturnType(QualType RetTy) {
160 return RetTy->getCanonicalTypeUnqualified();
161}
162
163/// Arrange the argument and result information for a value of the given
164/// unprototyped freestanding function type.
165const CGFunctionInfo &
166CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
167 // When translating an unprototyped function type, always use a
168 // variadic type.
169 return arrangeLLVMFunctionInfo(returnType: FTNP->getReturnType().getUnqualifiedType(),
170 opts: FnInfoOpts::None, argTypes: {}, info: FTNP->getExtInfo(), paramInfos: {},
171 args: RequiredArgs(0));
172}
173
174static void addExtParameterInfosForCall(
175 llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
176 const FunctionProtoType *proto, unsigned prefixArgs, unsigned totalArgs) {
177 assert(proto->hasExtParameterInfos());
178 assert(paramInfos.size() <= prefixArgs);
179 assert(proto->getNumParams() + prefixArgs <= totalArgs);
180
181 paramInfos.reserve(N: totalArgs);
182
183 // Add default infos for any prefix args that don't already have infos.
184 paramInfos.resize(N: prefixArgs);
185
186 // Add infos for the prototype.
187 for (const auto &ParamInfo : proto->getExtParameterInfos()) {
188 paramInfos.push_back(Elt: ParamInfo);
189 // pass_object_size params have no parameter info.
190 if (ParamInfo.hasPassObjectSize())
191 paramInfos.emplace_back();
192 }
193
194 assert(paramInfos.size() <= totalArgs &&
195 "Did we forget to insert pass_object_size args?");
196 // Add default infos for the variadic and/or suffix arguments.
197 paramInfos.resize(N: totalArgs);
198}
199
200/// Adds the formal parameters in FPT to the given prefix. If any parameter in
201/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
202static void appendParameterTypes(
203 const CodeGenTypes &CGT, SmallVectorImpl<CanQualType> &prefix,
204 SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
205 CanQual<FunctionProtoType> FPT) {
206 // Fast path: don't touch param info if we don't need to.
207 if (!FPT->hasExtParameterInfos()) {
208 assert(paramInfos.empty() &&
209 "We have paramInfos, but the prototype doesn't?");
210 prefix.append(in_start: FPT->param_type_begin(), in_end: FPT->param_type_end());
211 return;
212 }
213
214 unsigned PrefixSize = prefix.size();
215 // In the vast majority of cases, we'll have precisely FPT->getNumParams()
216 // parameters; the only thing that can change this is the presence of
217 // pass_object_size. So, we preallocate for the common case.
218 prefix.reserve(N: prefix.size() + FPT->getNumParams());
219
220 auto ExtInfos = FPT->getExtParameterInfos();
221 assert(ExtInfos.size() == FPT->getNumParams());
222 for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
223 prefix.push_back(Elt: FPT->getParamType(i: I));
224 if (ExtInfos[I].hasPassObjectSize())
225 prefix.push_back(Elt: CGT.getContext().getCanonicalSizeType());
226 }
227
228 addExtParameterInfosForCall(paramInfos, proto: FPT.getTypePtr(), prefixArgs: PrefixSize,
229 totalArgs: prefix.size());
230}
231
232using ExtParameterInfoList =
233 SmallVector<FunctionProtoType::ExtParameterInfo, 16>;
234
235/// Arrange the LLVM function layout for a value of the given function
236/// type, on top of any implicit parameters already stored.
237static const CGFunctionInfo &
238arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
239 SmallVectorImpl<CanQualType> &prefix,
240 CanQual<FunctionProtoType> FTP) {
241 ExtParameterInfoList paramInfos;
242 RequiredArgs Required = RequiredArgs::forPrototypePlus(prototype: FTP, additional: prefix.size());
243 appendParameterTypes(CGT, prefix, paramInfos, FPT: FTP);
244 CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
245
246 FnInfoOpts opts =
247 instanceMethod ? FnInfoOpts::IsInstanceMethod : FnInfoOpts::None;
248 return CGT.arrangeLLVMFunctionInfo(returnType: resultType, opts, argTypes: prefix,
249 info: FTP->getExtInfo(), paramInfos, args: Required);
250}
251
252using CanQualTypeList = SmallVector<CanQualType, 16>;
253
254/// Arrange the argument and result information for a value of the
255/// given freestanding function type.
256const CGFunctionInfo &
257CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
258 CanQualTypeList argTypes;
259 return ::arrangeLLVMFunctionInfo(CGT&: *this, /*instanceMethod=*/false, prefix&: argTypes,
260 FTP);
261}
262
263static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
264 bool IsTargetDefaultMSABI) {
265 // Set the appropriate calling convention for the Function.
266 if (D->hasAttr<StdCallAttr>())
267 return CC_X86StdCall;
268
269 if (D->hasAttr<FastCallAttr>())
270 return CC_X86FastCall;
271
272 if (D->hasAttr<RegCallAttr>())
273 return CC_X86RegCall;
274
275 if (D->hasAttr<ThisCallAttr>())
276 return CC_X86ThisCall;
277
278 if (D->hasAttr<VectorCallAttr>())
279 return CC_X86VectorCall;
280
281 if (D->hasAttr<PascalAttr>())
282 return CC_X86Pascal;
283
284 if (PcsAttr *PCS = D->getAttr<PcsAttr>())
285 return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
286
287 if (D->hasAttr<AArch64VectorPcsAttr>())
288 return CC_AArch64VectorCall;
289
290 if (D->hasAttr<AArch64SVEPcsAttr>())
291 return CC_AArch64SVEPCS;
292
293 if (D->hasAttr<DeviceKernelAttr>())
294 return CC_DeviceKernel;
295
296 if (D->hasAttr<IntelOclBiccAttr>())
297 return CC_IntelOclBicc;
298
299 if (D->hasAttr<MSABIAttr>())
300 return IsTargetDefaultMSABI ? CC_C : CC_Win64;
301
302 if (D->hasAttr<SysVABIAttr>())
303 return IsTargetDefaultMSABI ? CC_X86_64SysV : CC_C;
304
305 if (D->hasAttr<PreserveMostAttr>())
306 return CC_PreserveMost;
307
308 if (D->hasAttr<PreserveAllAttr>())
309 return CC_PreserveAll;
310
311 if (D->hasAttr<M68kRTDAttr>())
312 return CC_M68kRTD;
313
314 if (D->hasAttr<PreserveNoneAttr>())
315 return CC_PreserveNone;
316
317 if (D->hasAttr<RISCVVectorCCAttr>())
318 return CC_RISCVVectorCall;
319
320 if (RISCVVLSCCAttr *PCS = D->getAttr<RISCVVLSCCAttr>()) {
321 switch (PCS->getVectorWidth()) {
322 default:
323 llvm_unreachable("Invalid RISC-V VLS ABI VLEN");
324#define CC_VLS_CASE(ABI_VLEN) \
325 case ABI_VLEN: \
326 return CC_RISCVVLSCall_##ABI_VLEN;
327 CC_VLS_CASE(32)
328 CC_VLS_CASE(64)
329 CC_VLS_CASE(128)
330 CC_VLS_CASE(256)
331 CC_VLS_CASE(512)
332 CC_VLS_CASE(1024)
333 CC_VLS_CASE(2048)
334 CC_VLS_CASE(4096)
335 CC_VLS_CASE(8192)
336 CC_VLS_CASE(16384)
337 CC_VLS_CASE(32768)
338 CC_VLS_CASE(65536)
339#undef CC_VLS_CASE
340 }
341 }
342
343 return CC_C;
344}
345
346/// Arrange the argument and result information for a call to an
347/// unknown C++ non-static member function of the given abstract type.
348/// (A null RD means we don't have any meaningful "this" argument type,
349/// so fall back to a generic pointer type).
350/// The member function must be an ordinary function, i.e. not a
351/// constructor or destructor.
352const CGFunctionInfo &
353CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
354 const FunctionProtoType *FTP,
355 const CXXMethodDecl *MD) {
356 CanQualTypeList argTypes;
357
358 // Add the 'this' pointer.
359 argTypes.push_back(Elt: DeriveThisType(RD, MD));
360
361 return ::arrangeLLVMFunctionInfo(
362 CGT&: *this, /*instanceMethod=*/true, prefix&: argTypes,
363 FTP: FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
364}
365
366/// Set calling convention for CUDA/HIP kernel.
367static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
368 const FunctionDecl *FD) {
369 if (FD->hasAttr<CUDAGlobalAttr>()) {
370 const FunctionType *FT = FTy->getAs<FunctionType>();
371 CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
372 FTy = FT->getCanonicalTypeUnqualified();
373 }
374}
375
376/// Arrange the argument and result information for a declaration or
377/// definition of the given C++ non-static member function. The
378/// member function must be an ordinary function, i.e. not a
379/// constructor or destructor.
380const CGFunctionInfo &
381CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
382 assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
383 assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
384
385 CanQualType FT = GetFormalType(MD).getAs<Type>();
386 setCUDAKernelCallingConvention(FTy&: FT, CGM, FD: MD);
387 auto prototype = FT.getAs<FunctionProtoType>();
388
389 if (MD->isImplicitObjectMemberFunction()) {
390 // The abstract case is perfectly fine.
391 const CXXRecordDecl *ThisType =
392 getCXXABI().getThisArgumentTypeForMethod(GD: MD);
393 return arrangeCXXMethodType(RD: ThisType, FTP: prototype.getTypePtr(), MD);
394 }
395
396 return arrangeFreeFunctionType(FTP: prototype);
397}
398
399bool CodeGenTypes::inheritingCtorHasParams(
400 const InheritedConstructor &Inherited, CXXCtorType Type) {
401 // Parameters are unnecessary if we're constructing a base class subobject
402 // and the inherited constructor lives in a virtual base.
403 return Type == Ctor_Complete ||
404 !Inherited.getShadowDecl()->constructsVirtualBase() ||
405 !Target.getCXXABI().hasConstructorVariants();
406}
407
408const CGFunctionInfo &
409CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
410 auto *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
411
412 CanQualTypeList argTypes;
413 ExtParameterInfoList paramInfos;
414
415 const CXXRecordDecl *ThisType = getCXXABI().getThisArgumentTypeForMethod(GD);
416 argTypes.push_back(Elt: DeriveThisType(RD: ThisType, MD));
417
418 bool PassParams = true;
419
420 if (auto *CD = dyn_cast<CXXConstructorDecl>(Val: MD)) {
421 // A base class inheriting constructor doesn't get forwarded arguments
422 // needed to construct a virtual base (or base class thereof).
423 if (auto Inherited = CD->getInheritedConstructor())
424 PassParams = inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
425 }
426
427 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
428
429 // Add the formal parameters.
430 if (PassParams)
431 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: FTP);
432
433 CGCXXABI::AddedStructorArgCounts AddedArgs =
434 getCXXABI().buildStructorSignature(GD, ArgTys&: argTypes);
435 if (!paramInfos.empty()) {
436 // Note: prefix implies after the first param.
437 if (AddedArgs.Prefix)
438 paramInfos.insert(I: paramInfos.begin() + 1, NumToInsert: AddedArgs.Prefix,
439 Elt: FunctionProtoType::ExtParameterInfo{});
440 if (AddedArgs.Suffix)
441 paramInfos.append(NumInputs: AddedArgs.Suffix,
442 Elt: FunctionProtoType::ExtParameterInfo{});
443 }
444
445 RequiredArgs required =
446 (PassParams && MD->isVariadic() ? RequiredArgs(argTypes.size())
447 : RequiredArgs::All);
448
449 FunctionType::ExtInfo extInfo = FTP->getExtInfo();
450 CanQualType resultType = getCXXABI().HasThisReturn(GD) ? argTypes.front()
451 : getCXXABI().hasMostDerivedReturn(GD)
452 ? CGM.getContext().VoidPtrTy
453 : Context.VoidTy;
454 return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::IsInstanceMethod,
455 argTypes, info: extInfo, paramInfos, args: required);
456}
457
458static CanQualTypeList getArgTypesForCall(ASTContext &ctx,
459 const CallArgList &args) {
460 CanQualTypeList argTypes;
461 for (auto &arg : args)
462 argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg.Ty));
463 return argTypes;
464}
465
466static CanQualTypeList getArgTypesForDeclaration(ASTContext &ctx,
467 const FunctionArgList &args) {
468 CanQualTypeList argTypes;
469 for (auto &arg : args)
470 argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg->getType()));
471 return argTypes;
472}
473
474static ExtParameterInfoList
475getExtParameterInfosForCall(const FunctionProtoType *proto, unsigned prefixArgs,
476 unsigned totalArgs) {
477 ExtParameterInfoList result;
478 if (proto->hasExtParameterInfos()) {
479 addExtParameterInfosForCall(paramInfos&: result, proto, prefixArgs, totalArgs);
480 }
481 return result;
482}
483
484/// Arrange a call to a C++ method, passing the given arguments.
485///
486/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
487/// parameter.
488/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
489/// args.
490/// PassProtoArgs indicates whether `args` has args for the parameters in the
491/// given CXXConstructorDecl.
492const CGFunctionInfo &CodeGenTypes::arrangeCXXConstructorCall(
493 const CallArgList &args, const CXXConstructorDecl *D, CXXCtorType CtorKind,
494 unsigned ExtraPrefixArgs, unsigned ExtraSuffixArgs, bool PassProtoArgs) {
495 CanQualTypeList ArgTypes;
496 for (const auto &Arg : args)
497 ArgTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
498
499 // +1 for implicit this, which should always be args[0].
500 unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
501
502 CanQual<FunctionProtoType> FPT = GetFormalType(MD: D);
503 RequiredArgs Required = PassProtoArgs
504 ? RequiredArgs::forPrototypePlus(
505 prototype: FPT, additional: TotalPrefixArgs + ExtraSuffixArgs)
506 : RequiredArgs::All;
507
508 GlobalDecl GD(D, CtorKind);
509 CanQualType ResultType = getCXXABI().HasThisReturn(GD) ? ArgTypes.front()
510 : getCXXABI().hasMostDerivedReturn(GD)
511 ? CGM.getContext().VoidPtrTy
512 : Context.VoidTy;
513
514 FunctionType::ExtInfo Info = FPT->getExtInfo();
515 ExtParameterInfoList ParamInfos;
516 // If the prototype args are elided, we should only have ABI-specific args,
517 // which never have param info.
518 if (PassProtoArgs && FPT->hasExtParameterInfos()) {
519 // ABI-specific suffix arguments are treated the same as variadic arguments.
520 addExtParameterInfosForCall(paramInfos&: ParamInfos, proto: FPT.getTypePtr(), prefixArgs: TotalPrefixArgs,
521 totalArgs: ArgTypes.size());
522 }
523
524 return arrangeLLVMFunctionInfo(returnType: ResultType, opts: FnInfoOpts::IsInstanceMethod,
525 argTypes: ArgTypes, info: Info, paramInfos: ParamInfos, args: Required);
526}
527
528/// Arrange the argument and result information for the declaration or
529/// definition of the given function.
530const CGFunctionInfo &
531CodeGenTypes::arrangeFunctionDeclaration(const GlobalDecl GD) {
532 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
533 if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD))
534 if (MD->isImplicitObjectMemberFunction())
535 return arrangeCXXMethodDeclaration(MD);
536
537 CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
538
539 assert(isa<FunctionType>(FTy));
540 setCUDAKernelCallingConvention(FTy, CGM, FD);
541
542 if (DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()) &&
543 GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
544 const FunctionType *FT = FTy->getAs<FunctionType>();
545 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
546 FTy = FT->getCanonicalTypeUnqualified();
547 }
548
549 // When declaring a function without a prototype, always use a
550 // non-variadic type.
551 if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
552 return arrangeLLVMFunctionInfo(returnType: noProto->getReturnType(), opts: FnInfoOpts::None,
553 argTypes: {}, info: noProto->getExtInfo(), paramInfos: {},
554 args: RequiredArgs::All);
555 }
556
557 return arrangeFreeFunctionType(FTP: FTy.castAs<FunctionProtoType>());
558}
559
560/// Arrange the argument and result information for the declaration or
561/// definition of an Objective-C method.
562const CGFunctionInfo &
563CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
564 // It happens that this is the same as a call with no optional
565 // arguments, except also using the formal 'self' type.
566 return arrangeObjCMessageSendSignature(MD, receiverType: MD->getSelfDecl()->getType());
567}
568
569/// Arrange the argument and result information for the function type
570/// through which to perform a send to the given Objective-C method,
571/// using the given receiver type. The receiver type is not always
572/// the 'self' type of the method or even an Objective-C pointer type.
573/// This is *not* the right method for actually performing such a
574/// message send, due to the possibility of optional arguments.
575const CGFunctionInfo &
576CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
577 QualType receiverType) {
578 CanQualTypeList argTys;
579 ExtParameterInfoList extParamInfos(MD->isDirectMethod() ? 1 : 2);
580 argTys.push_back(Elt: Context.getCanonicalParamType(T: receiverType));
581 if (!MD->isDirectMethod())
582 argTys.push_back(Elt: Context.getCanonicalParamType(T: Context.getObjCSelType()));
583 for (const auto *I : MD->parameters()) {
584 argTys.push_back(Elt: Context.getCanonicalParamType(T: I->getType()));
585 auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
586 NoEscape: I->hasAttr<NoEscapeAttr>());
587 extParamInfos.push_back(Elt: extParamInfo);
588 }
589
590 FunctionType::ExtInfo einfo;
591 bool IsTargetDefaultMSABI =
592 getContext().getTargetInfo().getTriple().isOSWindows() ||
593 getContext().getTargetInfo().getTriple().isUEFI();
594 einfo = einfo.withCallingConv(
595 cc: getCallingConventionForDecl(D: MD, IsTargetDefaultMSABI));
596
597 if (getContext().getLangOpts().ObjCAutoRefCount &&
598 MD->hasAttr<NSReturnsRetainedAttr>())
599 einfo = einfo.withProducesResult(producesResult: true);
600
601 RequiredArgs required =
602 (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);
603
604 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: MD->getReturnType()),
605 opts: FnInfoOpts::None, argTypes: argTys, info: einfo, paramInfos: extParamInfos,
606 args: required);
607}
608
609const CGFunctionInfo &
610CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
611 const CallArgList &args) {
612 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
613 FunctionType::ExtInfo einfo;
614
615 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: returnType), opts: FnInfoOpts::None,
616 argTypes, info: einfo, paramInfos: {}, args: RequiredArgs::All);
617}
618
619const CGFunctionInfo &CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
620 // FIXME: Do we need to handle ObjCMethodDecl?
621 if (isa<CXXConstructorDecl>(Val: GD.getDecl()) ||
622 isa<CXXDestructorDecl>(Val: GD.getDecl()))
623 return arrangeCXXStructorDeclaration(GD);
624
625 return arrangeFunctionDeclaration(GD);
626}
627
628/// Arrange a thunk that takes 'this' as the first parameter followed by
629/// varargs. Return a void pointer, regardless of the actual return type.
630/// The body of the thunk will end in a musttail call to a function of the
631/// correct type, and the caller will bitcast the function to the correct
632/// prototype.
633const CGFunctionInfo &
634CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
635 assert(MD->isVirtual() && "only methods have thunks");
636 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
637 CanQualType ArgTys[] = {DeriveThisType(RD: MD->getParent(), MD)};
638 return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::None, argTypes: ArgTys,
639 info: FTP->getExtInfo(), paramInfos: {}, args: RequiredArgs(1));
640}
641
642const CGFunctionInfo &
643CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
644 CXXCtorType CT) {
645 assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
646
647 CanQual<FunctionProtoType> FTP = GetFormalType(MD: CD);
648 SmallVector<CanQualType, 2> ArgTys;
649 const CXXRecordDecl *RD = CD->getParent();
650 ArgTys.push_back(Elt: DeriveThisType(RD, MD: CD));
651 if (CT == Ctor_CopyingClosure)
652 ArgTys.push_back(Elt: *FTP->param_type_begin());
653 if (RD->getNumVBases() > 0)
654 ArgTys.push_back(Elt: Context.IntTy);
655 CallingConv CC = Context.getDefaultCallingConvention(
656 /*IsVariadic=*/false, /*IsCXXMethod=*/true);
657 return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::IsInstanceMethod,
658 argTypes: ArgTys, info: FunctionType::ExtInfo(CC), paramInfos: {},
659 args: RequiredArgs::All);
660}
661
662/// Arrange a call as unto a free function, except possibly with an
663/// additional number of formal parameters considered required.
664static const CGFunctionInfo &
665arrangeFreeFunctionLikeCall(CodeGenTypes &CGT, CodeGenModule &CGM,
666 const CallArgList &args, const FunctionType *fnType,
667 unsigned numExtraRequiredArgs, bool chainCall) {
668 assert(args.size() >= numExtraRequiredArgs);
669
670 ExtParameterInfoList paramInfos;
671
672 // In most cases, there are no optional arguments.
673 RequiredArgs required = RequiredArgs::All;
674
675 // If we have a variadic prototype, the required arguments are the
676 // extra prefix plus the arguments in the prototype.
677 if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(Val: fnType)) {
678 if (proto->isVariadic())
679 required = RequiredArgs::forPrototypePlus(prototype: proto, additional: numExtraRequiredArgs);
680
681 if (proto->hasExtParameterInfos())
682 addExtParameterInfosForCall(paramInfos, proto, prefixArgs: numExtraRequiredArgs,
683 totalArgs: args.size());
684
685 // If we don't have a prototype at all, but we're supposed to
686 // explicitly use the variadic convention for unprototyped calls,
687 // treat all of the arguments as required but preserve the nominal
688 // possibility of variadics.
689 } else if (CGM.getTargetCodeGenInfo().isNoProtoCallVariadic(
690 args, fnType: cast<FunctionNoProtoType>(Val: fnType))) {
691 required = RequiredArgs(args.size());
692 }
693
694 CanQualTypeList argTypes;
695 for (const auto &arg : args)
696 argTypes.push_back(Elt: CGT.getContext().getCanonicalParamType(T: arg.Ty));
697 FnInfoOpts opts = chainCall ? FnInfoOpts::IsChainCall : FnInfoOpts::None;
698 return CGT.arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: fnType->getReturnType()),
699 opts, argTypes, info: fnType->getExtInfo(),
700 paramInfos, args: required);
701}
702
703/// Figure out the rules for calling a function with the given formal
704/// type using the given arguments. The arguments are necessary
705/// because the function might be unprototyped, in which case it's
706/// target-dependent in crazy ways.
707const CGFunctionInfo &CodeGenTypes::arrangeFreeFunctionCall(
708 const CallArgList &args, const FunctionType *fnType, bool chainCall) {
709 return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType,
710 numExtraRequiredArgs: chainCall ? 1 : 0, chainCall);
711}
712
713/// A block function is essentially a free function with an
714/// extra implicit argument.
715const CGFunctionInfo &
716CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
717 const FunctionType *fnType) {
718 return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType, numExtraRequiredArgs: 1,
719 /*chainCall=*/false);
720}
721
722const CGFunctionInfo &
723CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
724 const FunctionArgList &params) {
725 ExtParameterInfoList paramInfos =
726 getExtParameterInfosForCall(proto, prefixArgs: 1, totalArgs: params.size());
727 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args: params);
728
729 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: proto->getReturnType()),
730 opts: FnInfoOpts::None, argTypes,
731 info: proto->getExtInfo(), paramInfos,
732 args: RequiredArgs::forPrototypePlus(prototype: proto, additional: 1));
733}
734
735const CGFunctionInfo &
736CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
737 const CallArgList &args) {
738 CanQualTypeList argTypes;
739 for (const auto &Arg : args)
740 argTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
741 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
742 argTypes, info: FunctionType::ExtInfo(),
743 /*paramInfos=*/{}, args: RequiredArgs::All);
744}
745
746const CGFunctionInfo &
747CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
748 const FunctionArgList &args) {
749 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
750
751 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
752 argTypes, info: FunctionType::ExtInfo(), paramInfos: {},
753 args: RequiredArgs::All);
754}
755
756const CGFunctionInfo &CodeGenTypes::arrangeBuiltinFunctionDeclaration(
757 CanQualType resultType, ArrayRef<CanQualType> argTypes) {
758 return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::None, argTypes,
759 info: FunctionType::ExtInfo(), paramInfos: {},
760 args: RequiredArgs::All);
761}
762
763const CGFunctionInfo &CodeGenTypes::arrangeDeviceKernelCallerDeclaration(
764 QualType resultType, const FunctionArgList &args) {
765 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
766
767 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
768 argTypes,
769 info: FunctionType::ExtInfo(CC_DeviceKernel),
770 /*paramInfos=*/{}, args: RequiredArgs::All);
771}
772
773/// Arrange a call to a C++ method, passing the given arguments.
774///
775/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
776/// does not count `this`.
777const CGFunctionInfo &CodeGenTypes::arrangeCXXMethodCall(
778 const CallArgList &args, const FunctionProtoType *proto,
779 RequiredArgs required, unsigned numPrefixArgs) {
780 assert(numPrefixArgs + 1 <= args.size() &&
781 "Emitting a call with less args than the required prefix?");
782 // Add one to account for `this`. It's a bit awkward here, but we don't count
783 // `this` in similar places elsewhere.
784 ExtParameterInfoList paramInfos =
785 getExtParameterInfosForCall(proto, prefixArgs: numPrefixArgs + 1, totalArgs: args.size());
786
787 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
788
789 FunctionType::ExtInfo info = proto->getExtInfo();
790 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: proto->getReturnType()),
791 opts: FnInfoOpts::IsInstanceMethod, argTypes, info,
792 paramInfos, args: required);
793}
794
795const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
796 return arrangeLLVMFunctionInfo(returnType: getContext().VoidTy, opts: FnInfoOpts::None, argTypes: {},
797 info: FunctionType::ExtInfo(), paramInfos: {},
798 args: RequiredArgs::All);
799}
800
801const CGFunctionInfo &CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
802 const CallArgList &args) {
803 assert(signature.arg_size() <= args.size());
804 if (signature.arg_size() == args.size())
805 return signature;
806
807 ExtParameterInfoList paramInfos;
808 auto sigParamInfos = signature.getExtParameterInfos();
809 if (!sigParamInfos.empty()) {
810 paramInfos.append(in_start: sigParamInfos.begin(), in_end: sigParamInfos.end());
811 paramInfos.resize(N: args.size());
812 }
813
814 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
815
816 assert(signature.getRequiredArgs().allowsOptionalArgs());
817 FnInfoOpts opts = FnInfoOpts::None;
818 if (signature.isInstanceMethod())
819 opts |= FnInfoOpts::IsInstanceMethod;
820 if (signature.isChainCall())
821 opts |= FnInfoOpts::IsChainCall;
822 if (signature.isDelegateCall())
823 opts |= FnInfoOpts::IsDelegateCall;
824 return arrangeLLVMFunctionInfo(returnType: signature.getReturnType(), opts, argTypes,
825 info: signature.getExtInfo(), paramInfos,
826 args: signature.getRequiredArgs());
827}
828
829namespace clang {
830namespace CodeGen {
831void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
832} // namespace CodeGen
833} // namespace clang
834
835#ifndef NDEBUG
836static const char *abiKindToString(ABIArgInfo::Kind K) {
837 switch (K) {
838 case ABIArgInfo::Direct:
839 return "Direct";
840 case ABIArgInfo::Extend:
841 return "Extend";
842 case ABIArgInfo::Indirect:
843 return "Indirect";
844 case ABIArgInfo::IndirectAliased:
845 return "IndirectAliased";
846 case ABIArgInfo::Ignore:
847 return "Ignore";
848 case ABIArgInfo::Expand:
849 return "Expand";
850 case ABIArgInfo::CoerceAndExpand:
851 return "CoerceAndExpand";
852 case ABIArgInfo::TargetSpecific:
853 return "TargetSpecific";
854 case ABIArgInfo::InAlloca:
855 return "InAlloca";
856 }
857 llvm_unreachable("Unknown kind");
858}
859#endif
860
861void CodeGenModule::computeABIInfoUsingLib(CGFunctionInfo &FI) {
862 SmallVector<const llvm::abi::Type *> MappedArgTypes;
863 MappedArgTypes.reserve(N: FI.arg_size());
864 for (const auto &Arg : FI.arguments())
865 MappedArgTypes.push_back(Elt: AbiMapper->convertType(QT: Arg.type));
866
867 std::optional<unsigned> NumRequired;
868 RequiredArgs Required = FI.getRequiredArgs();
869 if (Required.allowsOptionalArgs())
870 NumRequired = Required.getNumRequiredArgs();
871
872 auto AbiFI = llvm::abi::FunctionInfo::create(
873 CC: FI.getCallingConvention(), ReturnType: AbiMapper->convertType(QT: FI.getReturnType()),
874 ArgTypes: MappedArgTypes, NumRequired);
875
876 getLLVMABITargetInfo(TB&: AbiMapper->getTypeBuilder()).computeInfo(FI&: *AbiFI);
877
878#ifndef NDEBUG
879 // With assertions enabled, also compute info using Clang ABI logic,
880 // so we can ensure the results are consistent.
881 getABIInfo().computeInfo(FI);
882
883 auto ConvertABIArgInfo = [&](ABIArgInfo &Target,
884 const llvm::abi::ArgInfo &AbiInfo, QualType Type,
885 int ArgNo) {
886 auto Check = [&](bool Cond, llvm::function_ref<void()> MessageFn) {
887 if (Cond)
888 return;
889 if (ArgNo == -1)
890 llvm::dbgs() << "For return value of type ";
891 else
892 llvm::dbgs() << "For argument " << ArgNo << " of type ";
893 llvm::dbgs() << Type << ": ";
894 MessageFn();
895 llvm::dbgs() << "\n";
896 abort();
897 };
898 auto CheckSimple = [&](auto TargetVal, auto ResVal, StringRef What) {
899 Check(TargetVal == ResVal, [&]() {
900 llvm::dbgs() << What << " mismatch (expected: " << TargetVal
901 << ", given: " << ResVal << ")";
902 });
903 };
904
905 ABIArgInfo Res = convertABIArgInfo(AbiInfo, Type);
906 Check(Target.getKind() == Res.getKind(), [&]() {
907 llvm::dbgs() << "Kind mismatch (expected: "
908 << abiKindToString(Target.getKind())
909 << ", given: " << abiKindToString(Res.getKind()) << ")";
910 });
911
912 if (Res.canHaveCoerceToType()) {
913 // Normalize nullptr types.
914 llvm::Type *TargetType = Target.getCoerceToType();
915 llvm::Type *ResType = Res.getCoerceToType();
916 if (!TargetType)
917 TargetType = getTypes().ConvertType(Type);
918 if (!ResType)
919 ResType = getTypes().ConvertType(Type);
920
921 Check(TargetType == ResType, [&]() {
922 llvm::dbgs() << "CoerceToType mismatch (expected: " << *TargetType
923 << ", given: " << *ResType << ")";
924 });
925 }
926
927 switch (Res.getKind()) {
928 case ABIArgInfo::Extend:
929 CheckSimple(Target.isSignExt(), Res.isSignExt(), "SignExt");
930 CheckSimple(Target.isZeroExt(), Res.isZeroExt(), "ZeroExt");
931 [[fallthrough]];
932 case ABIArgInfo::Direct:
933 CheckSimple(Target.getDirectAlign(), Res.getDirectAlign(), "DirectAlign");
934 CheckSimple(Target.getDirectOffset(), Res.getDirectOffset(),
935 "DirectOffset");
936 break;
937 case ABIArgInfo::Indirect:
938 CheckSimple(Target.getIndirectByVal(), Res.getIndirectByVal(),
939 "IndirectByVal");
940 [[fallthrough]];
941 case ABIArgInfo::IndirectAliased:
942 CheckSimple(Target.getIndirectAddrSpace(), Res.getIndirectAddrSpace(),
943 "IndirectAddrSpace");
944 CheckSimple(Target.getIndirectRealign(), Res.getIndirectRealign(),
945 "IndirectRealign");
946 Check(Target.getIndirectAlign() == Res.getIndirectAlign(), [&]() {
947 llvm::dbgs() << "IndirectAlign mismatch (expected: "
948 << Target.getIndirectAlign().getQuantity()
949 << ", given: " << Res.getIndirectAlign().getQuantity()
950 << ")";
951 });
952 break;
953 default:
954 break;
955 }
956
957 Target = Res;
958 };
959#else
960 auto ConvertABIArgInfo =
961 [&](ABIArgInfo &Target, const llvm::abi::ArgInfo &AbiInfo, QualType Type,
962 int ArgNo) { Target = convertABIArgInfo(AbiInfo, Type); };
963#endif
964
965 ConvertABIArgInfo(FI.getReturnInfo(), AbiFI->getReturnInfo(),
966 FI.getReturnType(), -1);
967
968 int ArgNo = 0;
969 for (auto [CGArg, AbiArg] :
970 llvm::zip_equal(t: FI.arguments(), u: AbiFI->arguments()))
971 ConvertABIArgInfo(CGArg.info, AbiArg.Info, CGArg.type, ArgNo++);
972}
973
974ABIArgInfo CodeGenModule::convertABIArgInfo(const llvm::abi::ArgInfo &AbiInfo,
975 QualType Type) {
976 switch (AbiInfo.getKind()) {
977 case llvm::abi::ArgInfo::Direct: {
978 llvm::Type *CoercedType = nullptr;
979 if (AbiInfo.getCoerceToType())
980 CoercedType = AbiReverseMapper->convertType(ABIType: AbiInfo.getCoerceToType());
981 if (!CoercedType)
982 CoercedType = getTypes().ConvertType(T: Type);
983 return ABIArgInfo::getDirect(T: CoercedType, Offset: AbiInfo.getDirectOffset());
984 }
985 case llvm::abi::ArgInfo::Extend: {
986 llvm::Type *CoercedType = nullptr;
987 if (AbiInfo.getCoerceToType())
988 CoercedType = AbiReverseMapper->convertType(ABIType: AbiInfo.getCoerceToType());
989 if (!CoercedType)
990 CoercedType = getTypes().ConvertType(T: Type);
991 if (AbiInfo.isSignExt())
992 return ABIArgInfo::getSignExtend(Ty: Type, T: CoercedType);
993 if (AbiInfo.isZeroExt())
994 return ABIArgInfo::getZeroExtend(Ty: Type, T: CoercedType);
995 return ABIArgInfo::getExtend(Ty: Type, T: CoercedType);
996 }
997 case llvm::abi::ArgInfo::Indirect: {
998 CharUnits Alignment =
999 CharUnits::fromQuantity(Quantity: AbiInfo.getIndirectAlign().value());
1000 return ABIArgInfo::getIndirect(Alignment, AddrSpace: AbiInfo.getIndirectAddrSpace(),
1001 ByVal: AbiInfo.getIndirectByVal(),
1002 Realign: AbiInfo.getIndirectRealign());
1003 }
1004 case llvm::abi::ArgInfo::Ignore:
1005 return ABIArgInfo::getIgnore();
1006 }
1007 llvm_unreachable("Unexpected llvm::abi::ArgInfo kind");
1008}
1009
1010/// Arrange the argument and result information for an abstract value
1011/// of a given function type. This is the method which all of the
1012/// above functions ultimately defer to.
1013const CGFunctionInfo &CodeGenTypes::arrangeLLVMFunctionInfo(
1014 CanQualType resultType, FnInfoOpts opts, ArrayRef<CanQualType> argTypes,
1015 FunctionType::ExtInfo info,
1016 ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
1017 RequiredArgs required) {
1018 assert(llvm::all_of(argTypes,
1019 [](CanQualType T) { return T.isCanonicalAsParam(); }));
1020
1021 // Lookup or create unique function info.
1022 llvm::FoldingSetNodeID ID;
1023 bool isInstanceMethod =
1024 (opts & FnInfoOpts::IsInstanceMethod) == FnInfoOpts::IsInstanceMethod;
1025 bool isChainCall =
1026 (opts & FnInfoOpts::IsChainCall) == FnInfoOpts::IsChainCall;
1027 bool isDelegateCall =
1028 (opts & FnInfoOpts::IsDelegateCall) == FnInfoOpts::IsDelegateCall;
1029 CGFunctionInfo::Profile(ID, InstanceMethod: isInstanceMethod, ChainCall: isChainCall, IsDelegateCall: isDelegateCall,
1030 info, paramInfos, required, resultType, argTypes);
1031
1032 void *insertPos = nullptr;
1033 CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, InsertPos&: insertPos);
1034 if (FI)
1035 return *FI;
1036
1037 unsigned CC = ClangCallConvToLLVMCallConv(CC: info.getCC());
1038
1039 // Construct the function info. We co-allocate the ArgInfos.
1040 FI = CGFunctionInfo::create(llvmCC: CC, instanceMethod: isInstanceMethod, chainCall: isChainCall, delegateCall: isDelegateCall,
1041 extInfo: info, paramInfos, resultType, argTypes, required);
1042 FunctionInfos.InsertNode(N: FI, InsertPos: insertPos);
1043
1044 bool inserted = FunctionsBeingProcessed.insert(Ptr: FI).second;
1045 (void)inserted;
1046 assert(inserted && "Recursively being processed?");
1047
1048 // Compute ABI information.
1049 if (info.getCC() == CC_DeviceKernel &&
1050 (CC == llvm::CallingConv::SPIR_KERNEL || CC == llvm::CallingConv::C)) {
1051 // Force target independent argument handling for the host visible
1052 // kernel functions.
1053 //
1054 // For CPU targets, this currently only works for OpenCL.
1055 assert(CC != llvm::CallingConv::C || getContext().getLangOpts().OpenCL);
1056 computeSPIRKernelABIInfo(CGM, FI&: *FI);
1057 } else if (info.getCC() == CC_Swift || info.getCC() == CC_SwiftAsync) {
1058 swiftcall::computeABIInfo(CGM, FI&: *FI);
1059 } else if (CGM.shouldUseLLVMABILowering()) {
1060 CGM.computeABIInfoUsingLib(FI&: *FI);
1061 } else {
1062 CGM.getABIInfo().computeInfo(FI&: *FI);
1063 }
1064
1065 // Loop over all of the computed argument and return value info. If any of
1066 // them are direct or extend without a specified coerce type, specify the
1067 // default now.
1068 ABIArgInfo &retInfo = FI->getReturnInfo();
1069 if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
1070 retInfo.setCoerceToType(ConvertType(T: FI->getReturnType()));
1071
1072 for (auto &I : FI->arguments())
1073 if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
1074 I.info.setCoerceToType(ConvertType(T: I.type));
1075
1076 bool erased = FunctionsBeingProcessed.erase(Ptr: FI);
1077 (void)erased;
1078 assert(erased && "Not in set?");
1079
1080 return *FI;
1081}
1082
1083CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC, bool instanceMethod,
1084 bool chainCall, bool delegateCall,
1085 const FunctionType::ExtInfo &info,
1086 ArrayRef<ExtParameterInfo> paramInfos,
1087 CanQualType resultType,
1088 ArrayRef<CanQualType> argTypes,
1089 RequiredArgs required) {
1090 assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
1091 assert(!required.allowsOptionalArgs() ||
1092 required.getNumRequiredArgs() <= argTypes.size());
1093
1094 void *buffer = operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(
1095 Counts: argTypes.size() + 1, Counts: paramInfos.size()));
1096
1097 CGFunctionInfo *FI = new (buffer) CGFunctionInfo();
1098 FI->CallingConvention = llvmCC;
1099 FI->EffectiveCallingConvention = llvmCC;
1100 FI->ASTCallingConvention = info.getCC();
1101 FI->InstanceMethod = instanceMethod;
1102 FI->ChainCall = chainCall;
1103 FI->DelegateCall = delegateCall;
1104 FI->CmseNSCall = info.getCmseNSCall();
1105 FI->NoReturn = info.getNoReturn();
1106 FI->ReturnsRetained = info.getProducesResult();
1107 FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
1108 FI->NoCfCheck = info.getNoCfCheck();
1109 FI->Required = required;
1110 FI->HasRegParm = info.getHasRegParm();
1111 FI->RegParm = info.getRegParm();
1112 FI->ArgStruct = nullptr;
1113 FI->ArgStructAlign = 0;
1114 FI->NumArgs = argTypes.size();
1115 FI->HasExtParameterInfos = !paramInfos.empty();
1116 FI->getArgsBuffer()[0].type = resultType;
1117 FI->MaxVectorWidth = 0;
1118 for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
1119 FI->getArgsBuffer()[i + 1].type = argTypes[i];
1120 for (unsigned i = 0, e = paramInfos.size(); i != e; ++i)
1121 FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
1122 return FI;
1123}
1124
1125/***/
1126
1127namespace {
1128// ABIArgInfo::Expand implementation.
1129
1130// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
1131struct TypeExpansion {
1132 enum TypeExpansionKind {
1133 // Elements of constant arrays are expanded recursively.
1134 TEK_ConstantArray,
1135 // Record fields are expanded recursively (but if record is a union, only
1136 // the field with the largest size is expanded).
1137 TEK_Record,
1138 // For complex types, real and imaginary parts are expanded recursively.
1139 TEK_Complex,
1140 // All other types are not expandable.
1141 TEK_None
1142 };
1143
1144 const TypeExpansionKind Kind;
1145
1146 TypeExpansion(TypeExpansionKind K) : Kind(K) {}
1147 virtual ~TypeExpansion() {}
1148};
1149
1150struct ConstantArrayExpansion : TypeExpansion {
1151 QualType EltTy;
1152 uint64_t NumElts;
1153
1154 ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
1155 : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
1156 static bool classof(const TypeExpansion *TE) {
1157 return TE->Kind == TEK_ConstantArray;
1158 }
1159};
1160
1161struct RecordExpansion : TypeExpansion {
1162 SmallVector<const CXXBaseSpecifier *, 1> Bases;
1163
1164 SmallVector<const FieldDecl *, 1> Fields;
1165
1166 RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
1167 SmallVector<const FieldDecl *, 1> &&Fields)
1168 : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
1169 Fields(std::move(Fields)) {}
1170 static bool classof(const TypeExpansion *TE) {
1171 return TE->Kind == TEK_Record;
1172 }
1173};
1174
1175struct ComplexExpansion : TypeExpansion {
1176 QualType EltTy;
1177
1178 ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
1179 static bool classof(const TypeExpansion *TE) {
1180 return TE->Kind == TEK_Complex;
1181 }
1182};
1183
1184struct NoExpansion : TypeExpansion {
1185 NoExpansion() : TypeExpansion(TEK_None) {}
1186 static bool classof(const TypeExpansion *TE) { return TE->Kind == TEK_None; }
1187};
1188} // namespace
1189
1190static std::unique_ptr<TypeExpansion>
1191getTypeExpansion(QualType Ty, const ASTContext &Context) {
1192 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty)) {
1193 return std::make_unique<ConstantArrayExpansion>(args: AT->getElementType(),
1194 args: AT->getZExtSize());
1195 }
1196 if (const auto *RD = Ty->getAsRecordDecl()) {
1197 SmallVector<const CXXBaseSpecifier *, 1> Bases;
1198 SmallVector<const FieldDecl *, 1> Fields;
1199 assert(!RD->hasFlexibleArrayMember() &&
1200 "Cannot expand structure with flexible array.");
1201 if (RD->isUnion()) {
1202 // Unions can be here only in degenerative cases - all the fields are same
1203 // after flattening. Thus we have to use the "largest" field.
1204 const FieldDecl *LargestFD = nullptr;
1205 CharUnits UnionSize = CharUnits::Zero();
1206
1207 for (const auto *FD : RD->fields()) {
1208 if (FD->isZeroLengthBitField())
1209 continue;
1210 assert(!FD->isBitField() &&
1211 "Cannot expand structure with bit-field members.");
1212 CharUnits FieldSize = Context.getTypeSizeInChars(T: FD->getType());
1213 if (UnionSize < FieldSize) {
1214 UnionSize = FieldSize;
1215 LargestFD = FD;
1216 }
1217 }
1218 if (LargestFD)
1219 Fields.push_back(Elt: LargestFD);
1220 } else {
1221 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
1222 assert(!CXXRD->isDynamicClass() &&
1223 "cannot expand vtable pointers in dynamic classes");
1224 llvm::append_range(C&: Bases, R: llvm::make_pointer_range(Range: CXXRD->bases()));
1225 }
1226
1227 for (const auto *FD : RD->fields()) {
1228 if (FD->isZeroLengthBitField())
1229 continue;
1230 assert(!FD->isBitField() &&
1231 "Cannot expand structure with bit-field members.");
1232 Fields.push_back(Elt: FD);
1233 }
1234 }
1235 return std::make_unique<RecordExpansion>(args: std::move(Bases),
1236 args: std::move(Fields));
1237 }
1238 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
1239 return std::make_unique<ComplexExpansion>(args: CT->getElementType());
1240 }
1241 return std::make_unique<NoExpansion>();
1242}
1243
1244static int getExpansionSize(QualType Ty, const ASTContext &Context) {
1245 auto Exp = getTypeExpansion(Ty, Context);
1246 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1247 return CAExp->NumElts * getExpansionSize(Ty: CAExp->EltTy, Context);
1248 }
1249 if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1250 int Res = 0;
1251 for (auto BS : RExp->Bases)
1252 Res += getExpansionSize(Ty: BS->getType(), Context);
1253 for (auto FD : RExp->Fields)
1254 Res += getExpansionSize(Ty: FD->getType(), Context);
1255 return Res;
1256 }
1257 if (isa<ComplexExpansion>(Val: Exp.get()))
1258 return 2;
1259 assert(isa<NoExpansion>(Exp.get()));
1260 return 1;
1261}
1262
1263void CodeGenTypes::getExpandedTypes(
1264 QualType Ty, SmallVectorImpl<llvm::Type *>::iterator &TI) {
1265 auto Exp = getTypeExpansion(Ty, Context);
1266 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1267 for (int i = 0, n = CAExp->NumElts; i < n; i++) {
1268 getExpandedTypes(Ty: CAExp->EltTy, TI);
1269 }
1270 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1271 for (auto BS : RExp->Bases)
1272 getExpandedTypes(Ty: BS->getType(), TI);
1273 for (auto FD : RExp->Fields)
1274 getExpandedTypes(Ty: FD->getType(), TI);
1275 } else if (auto CExp = dyn_cast<ComplexExpansion>(Val: Exp.get())) {
1276 llvm::Type *EltTy = ConvertType(T: CExp->EltTy);
1277 *TI++ = EltTy;
1278 *TI++ = EltTy;
1279 } else {
1280 assert(isa<NoExpansion>(Exp.get()));
1281 *TI++ = ConvertType(T: Ty);
1282 }
1283}
1284
1285static void forConstantArrayExpansion(CodeGenFunction &CGF,
1286 ConstantArrayExpansion *CAE,
1287 Address BaseAddr,
1288 llvm::function_ref<void(Address)> Fn) {
1289 for (int i = 0, n = CAE->NumElts; i < n; i++) {
1290 Address EltAddr = CGF.Builder.CreateConstGEP2_32(Addr: BaseAddr, Idx0: 0, Idx1: i);
1291 Fn(EltAddr);
1292 }
1293}
1294
1295void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1296 llvm::Function::arg_iterator &AI) {
1297 assert(LV.isSimple() &&
1298 "Unexpected non-simple lvalue during struct expansion.");
1299
1300 auto Exp = getTypeExpansion(Ty, Context: getContext());
1301 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1302 forConstantArrayExpansion(
1303 CGF&: *this, CAE: CAExp, BaseAddr: LV.getAddress(), Fn: [&](Address EltAddr) {
1304 LValue LV = MakeAddrLValue(Addr: EltAddr, T: CAExp->EltTy);
1305 ExpandTypeFromArgs(Ty: CAExp->EltTy, LV, AI);
1306 });
1307 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1308 Address This = LV.getAddress();
1309 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1310 // Perform a single step derived-to-base conversion.
1311 Address Base =
1312 GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1313 /*NullCheckValue=*/false, Loc: SourceLocation());
1314 LValue SubLV = MakeAddrLValue(Addr: Base, T: BS->getType());
1315
1316 // Recurse onto bases.
1317 ExpandTypeFromArgs(Ty: BS->getType(), LV: SubLV, AI);
1318 }
1319 for (auto FD : RExp->Fields) {
1320 // FIXME: What are the right qualifiers here?
1321 LValue SubLV = EmitLValueForFieldInitialization(Base: LV, Field: FD);
1322 ExpandTypeFromArgs(Ty: FD->getType(), LV: SubLV, AI);
1323 }
1324 } else if (isa<ComplexExpansion>(Val: Exp.get())) {
1325 auto realValue = &*AI++;
1326 auto imagValue = &*AI++;
1327 EmitStoreOfComplex(V: ComplexPairTy(realValue, imagValue), dest: LV, /*init*/ isInit: true);
1328 } else {
1329 // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1330 // primitive store.
1331 assert(isa<NoExpansion>(Exp.get()));
1332 llvm::Value *Arg = &*AI++;
1333 if (LV.isBitField()) {
1334 EmitStoreThroughLValue(Src: RValue::get(V: Arg), Dst: LV);
1335 } else {
1336 // TODO: currently there are some places are inconsistent in what LLVM
1337 // pointer type they use (see D118744). Once clang uses opaque pointers
1338 // all LLVM pointer types will be the same and we can remove this check.
1339 if (Arg->getType()->isPointerTy()) {
1340 Address Addr = LV.getAddress();
1341 Arg = Builder.CreateBitCast(V: Arg, DestTy: Addr.getElementType());
1342 }
1343 EmitStoreOfScalar(value: Arg, lvalue: LV);
1344 }
1345 }
1346}
1347
1348void CodeGenFunction::ExpandTypeToArgs(
1349 QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1350 SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1351 auto Exp = getTypeExpansion(Ty, Context: getContext());
1352 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1353 Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1354 : Arg.getKnownRValue().getAggregateAddress();
1355 forConstantArrayExpansion(CGF&: *this, CAE: CAExp, BaseAddr: Addr, Fn: [&](Address EltAddr) {
1356 CallArg EltArg =
1357 CallArg(convertTempToRValue(addr: EltAddr, type: CAExp->EltTy, Loc: SourceLocation()),
1358 CAExp->EltTy);
1359 ExpandTypeToArgs(Ty: CAExp->EltTy, Arg: EltArg, IRFuncTy, IRCallArgs,
1360 IRCallArgPos);
1361 });
1362 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1363 Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1364 : Arg.getKnownRValue().getAggregateAddress();
1365 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1366 // Perform a single step derived-to-base conversion.
1367 Address Base =
1368 GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1369 /*NullCheckValue=*/false, Loc: SourceLocation());
1370 CallArg BaseArg = CallArg(RValue::getAggregate(addr: Base), BS->getType());
1371
1372 // Recurse onto bases.
1373 ExpandTypeToArgs(Ty: BS->getType(), Arg: BaseArg, IRFuncTy, IRCallArgs,
1374 IRCallArgPos);
1375 }
1376
1377 LValue LV = MakeAddrLValue(Addr: This, T: Ty);
1378 for (auto FD : RExp->Fields) {
1379 CallArg FldArg =
1380 CallArg(EmitRValueForField(LV, FD, Loc: SourceLocation()), FD->getType());
1381 ExpandTypeToArgs(Ty: FD->getType(), Arg: FldArg, IRFuncTy, IRCallArgs,
1382 IRCallArgPos);
1383 }
1384 } else if (isa<ComplexExpansion>(Val: Exp.get())) {
1385 ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1386 IRCallArgs[IRCallArgPos++] = CV.first;
1387 IRCallArgs[IRCallArgPos++] = CV.second;
1388 } else {
1389 assert(isa<NoExpansion>(Exp.get()));
1390 auto RV = Arg.getKnownRValue();
1391 assert(RV.isScalar() &&
1392 "Unexpected non-scalar rvalue during struct expansion.");
1393
1394 // Insert a bitcast as needed.
1395 llvm::Value *V = RV.getScalarVal();
1396 if (IRCallArgPos < IRFuncTy->getNumParams() &&
1397 V->getType() != IRFuncTy->getParamType(i: IRCallArgPos))
1398 V = Builder.CreateBitCast(V, DestTy: IRFuncTy->getParamType(i: IRCallArgPos));
1399
1400 IRCallArgs[IRCallArgPos++] = V;
1401 }
1402}
1403
1404/// Create a temporary allocation for the purposes of coercion.
1405static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF,
1406 llvm::Type *Ty,
1407 CharUnits MinAlign,
1408 const Twine &Name = "tmp") {
1409 // Don't use an alignment that's worse than what LLVM would prefer.
1410 auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty);
1411 CharUnits Align = std::max(a: MinAlign, b: CharUnits::fromQuantity(Quantity: PrefAlign));
1412
1413 return CGF.CreateTempAlloca(Ty, align: Align, Name: Name + ".coerce");
1414}
1415
1416/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1417/// accessing some number of bytes out of it, try to gep into the struct to get
1418/// at its inner goodness. Dive as deep as possible without entering an element
1419/// with an in-memory size smaller than DstSize.
1420static Address EnterStructPointerForCoercedAccess(Address SrcPtr,
1421 llvm::StructType *SrcSTy,
1422 uint64_t DstSize,
1423 CodeGenFunction &CGF) {
1424 // We can't dive into a zero-element struct.
1425 if (SrcSTy->getNumElements() == 0)
1426 return SrcPtr;
1427
1428 llvm::Type *FirstElt = SrcSTy->getElementType(N: 0);
1429
1430 // If the first elt is at least as large as what we're looking for, or if the
1431 // first element is the same size as the whole struct, we can enter it. The
1432 // comparison must be made on the store size and not the alloca size. Using
1433 // the alloca size may overstate the size of the load.
1434 uint64_t FirstEltSize = CGF.CGM.getDataLayout().getTypeStoreSize(Ty: FirstElt);
1435 if (FirstEltSize < DstSize &&
1436 FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(Ty: SrcSTy))
1437 return SrcPtr;
1438
1439 // GEP into the first element.
1440 SrcPtr = CGF.Builder.CreateStructGEP(Addr: SrcPtr, Index: 0, Name: "coerce.dive");
1441
1442 // If the first element is a struct, recurse.
1443 llvm::Type *SrcTy = SrcPtr.getElementType();
1444 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy))
1445 return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1446
1447 return SrcPtr;
1448}
1449
1450/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1451/// are either integers or pointers. This does a truncation of the value if it
1452/// is too large or a zero extension if it is too small.
1453///
1454/// This behaves as if the value were coerced through memory, so on big-endian
1455/// targets the high bits are preserved in a truncation, while little-endian
1456/// targets preserve the low bits.
1457static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, llvm::Type *Ty,
1458 CodeGenFunction &CGF) {
1459 if (Val->getType() == Ty)
1460 return Val;
1461
1462 if (isa<llvm::PointerType>(Val: Val->getType())) {
1463 // If this is Pointer->Pointer avoid conversion to and from int.
1464 if (isa<llvm::PointerType>(Val: Ty))
1465 return CGF.Builder.CreateBitCast(V: Val, DestTy: Ty, Name: "coerce.val");
1466
1467 // Convert the pointer to an integer so we can play with its width.
1468 Val = CGF.Builder.CreatePtrToInt(V: Val, DestTy: CGF.IntPtrTy, Name: "coerce.val.pi");
1469 }
1470
1471 llvm::Type *DestIntTy = Ty;
1472 if (isa<llvm::PointerType>(Val: DestIntTy))
1473 DestIntTy = CGF.IntPtrTy;
1474
1475 if (Val->getType() != DestIntTy) {
1476 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1477 if (DL.isBigEndian()) {
1478 // Preserve the high bits on big-endian targets.
1479 // That is what memory coercion does.
1480 uint64_t SrcSize = DL.getTypeSizeInBits(Ty: Val->getType());
1481 uint64_t DstSize = DL.getTypeSizeInBits(Ty: DestIntTy);
1482
1483 if (SrcSize > DstSize) {
1484 Val = CGF.Builder.CreateLShr(LHS: Val, RHS: SrcSize - DstSize, Name: "coerce.highbits");
1485 Val = CGF.Builder.CreateTrunc(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1486 } else {
1487 Val = CGF.Builder.CreateZExt(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1488 Val = CGF.Builder.CreateShl(LHS: Val, RHS: DstSize - SrcSize, Name: "coerce.highbits");
1489 }
1490 } else {
1491 // Little-endian targets preserve the low bits. No shifts required.
1492 Val = CGF.Builder.CreateIntCast(V: Val, DestTy: DestIntTy, isSigned: false, Name: "coerce.val.ii");
1493 }
1494 }
1495
1496 if (isa<llvm::PointerType>(Val: Ty))
1497 Val = CGF.Builder.CreateIntToPtr(V: Val, DestTy: Ty, Name: "coerce.val.ip");
1498 return Val;
1499}
1500
1501static llvm::Value *CreatePFPCoercedLoad(Address Src, QualType SrcFETy,
1502 llvm::Type *Ty, CodeGenFunction &CGF) {
1503 std::vector<PFPField> PFPFields = CGF.getContext().findPFPFields(Ty: SrcFETy);
1504 if (PFPFields.empty())
1505 return nullptr;
1506
1507 auto LoadCoercedField = [&](CharUnits Offset,
1508 llvm::Type *FieldType) -> llvm::Value * {
1509 // Check whether the field at Offset is a PFP field. This function is called
1510 // in ascending order of offset, and PFPFields is sorted by offset. This
1511 // means that we only need to check the first element (and remove it from
1512 // PFPFields if matching).
1513 if (!PFPFields.empty() && PFPFields[0].Offset == Offset) {
1514 auto FieldAddr = CGF.EmitAddressOfPFPField(RecordPtr: Src, Field: PFPFields[0]);
1515 llvm::Value *FieldVal = CGF.Builder.CreateLoad(Addr: FieldAddr);
1516 if (isa<llvm::IntegerType>(Val: FieldType))
1517 FieldVal = CGF.Builder.CreatePtrToInt(V: FieldVal, DestTy: FieldType);
1518 PFPFields.erase(position: PFPFields.begin());
1519 return FieldVal;
1520 }
1521 auto FieldAddr =
1522 CGF.Builder
1523 .CreateConstInBoundsByteGEP(Addr: Src.withElementType(ElemTy: CGF.Int8Ty), Offset)
1524 .withElementType(ElemTy: FieldType);
1525 return CGF.Builder.CreateLoad(Addr: FieldAddr);
1526 };
1527
1528 // The types handled by this function are the only ones that may be generated
1529 // by AArch64ABIInfo::classify{Argument,Return}Type for struct types with
1530 // pointers. PFP is only supported on AArch64.
1531 if (isa<llvm::IntegerType>(Val: Ty) || isa<llvm::PointerType>(Val: Ty)) {
1532 auto Addr = CGF.EmitAddressOfPFPField(RecordPtr: Src, Field: PFPFields[0]);
1533 llvm::Value *Val = CGF.Builder.CreateLoad(Addr);
1534 if (isa<llvm::IntegerType>(Val: Ty))
1535 Val = CGF.Builder.CreatePtrToInt(V: Val, DestTy: Ty);
1536 return Val;
1537 }
1538 auto *AT = cast<llvm::ArrayType>(Val: Ty);
1539 auto *ET = AT->getElementType();
1540 CharUnits WordSize = CGF.getContext().toCharUnitsFromBits(
1541 BitSize: CGF.CGM.getDataLayout().getTypeSizeInBits(Ty: ET));
1542 CharUnits Offset = CharUnits::Zero();
1543 llvm::Value *Val = llvm::PoisonValue::get(T: AT);
1544 for (unsigned Idx = 0; Idx != AT->getNumElements(); ++Idx, Offset += WordSize)
1545 Val = CGF.Builder.CreateInsertValue(Agg: Val, Val: LoadCoercedField(Offset, ET), Idxs: Idx);
1546 return Val;
1547}
1548
1549/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1550/// a pointer to an object of type \arg Ty, known to be aligned to
1551/// \arg SrcAlign bytes.
1552///
1553/// This safely handles the case when the src type is smaller than the
1554/// destination type; in this situation the values of bits which not
1555/// present in the src are undefined.
1556static llvm::Value *CreateCoercedLoad(Address Src, QualType SrcFETy,
1557 llvm::Type *Ty, CodeGenFunction &CGF) {
1558 llvm::Type *SrcTy = Src.getElementType();
1559
1560 // If SrcTy and Ty are the same, just do a load.
1561 if (SrcTy == Ty)
1562 return CGF.Builder.CreateLoad(Addr: Src);
1563
1564 if (llvm::Value *V = CreatePFPCoercedLoad(Src, SrcFETy, Ty, CGF))
1565 return V;
1566
1567 llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1568
1569 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy)) {
1570 Src = EnterStructPointerForCoercedAccess(SrcPtr: Src, SrcSTy,
1571 DstSize: DstSize.getFixedValue(), CGF);
1572 SrcTy = Src.getElementType();
1573 }
1574
1575 llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1576
1577 // If the source and destination are integer or pointer types, just do an
1578 // extension or truncation to the desired type.
1579 if ((isa<llvm::IntegerType>(Val: Ty) || isa<llvm::PointerType>(Val: Ty)) &&
1580 (isa<llvm::IntegerType>(Val: SrcTy) || isa<llvm::PointerType>(Val: SrcTy))) {
1581 llvm::Value *Load = CGF.Builder.CreateLoad(Addr: Src);
1582 return CoerceIntOrPtrToIntOrPtr(Val: Load, Ty, CGF);
1583 }
1584
1585 // If load is legal, just bitcast the src pointer.
1586 if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1587 SrcSize.getFixedValue() >= DstSize.getFixedValue()) {
1588 // Generally SrcSize is never greater than DstSize, since this means we are
1589 // losing bits. However, this can happen in cases where the structure has
1590 // additional padding, for example due to a user specified alignment.
1591 //
1592 // FIXME: Assert that we aren't truncating non-padding bits when have access
1593 // to that information.
1594 Src = Src.withElementType(ElemTy: Ty);
1595 return CGF.Builder.CreateLoad(Addr: Src);
1596 }
1597
1598 // If coercing a fixed vector to a scalable vector for ABI compatibility, and
1599 // the types match, use the llvm.vector.insert intrinsic to perform the
1600 // conversion.
1601 if (auto *ScalableDstTy = dyn_cast<llvm::ScalableVectorType>(Val: Ty)) {
1602 if (auto *FixedSrcTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy)) {
1603 // If we are casting a fixed i8 vector to a scalable i1 predicate
1604 // vector, use a vector insert and bitcast the result.
1605 if (ScalableDstTy->getElementType()->isIntegerTy(BitWidth: 1) &&
1606 FixedSrcTy->getElementType()->isIntegerTy(BitWidth: 8)) {
1607 ScalableDstTy = llvm::ScalableVectorType::get(
1608 ElementType: FixedSrcTy->getElementType(),
1609 MinNumElts: llvm::divideCeil(
1610 Numerator: ScalableDstTy->getElementCount().getKnownMinValue(), Denominator: 8));
1611 }
1612 if (ScalableDstTy->getElementType() == FixedSrcTy->getElementType()) {
1613 auto *Load = CGF.Builder.CreateLoad(Addr: Src);
1614 auto *PoisonVec = llvm::PoisonValue::get(T: ScalableDstTy);
1615 llvm::Value *Result = CGF.Builder.CreateInsertVector(
1616 DstType: ScalableDstTy, SrcVec: PoisonVec, SubVec: Load, Idx: uint64_t(0), Name: "cast.scalable");
1617 ScalableDstTy = cast<llvm::ScalableVectorType>(
1618 Val: llvm::VectorType::getWithSizeAndScalar(SizeTy: ScalableDstTy, EltTy: Ty));
1619 if (Result->getType() != ScalableDstTy)
1620 Result = CGF.Builder.CreateBitCast(V: Result, DestTy: ScalableDstTy);
1621 if (Result->getType() != Ty)
1622 Result = CGF.Builder.CreateExtractVector(DstType: Ty, SrcVec: Result, Idx: uint64_t(0));
1623 return Result;
1624 }
1625 }
1626 }
1627
1628 // Otherwise do coercion through memory. This is stupid, but simple.
1629 RawAddress Tmp =
1630 CreateTempAllocaForCoercion(CGF, Ty, MinAlign: Src.getAlignment(), Name: Src.getName());
1631 CGF.Builder.CreateMemCpy(
1632 Dst: Tmp.getPointer(), DstAlign: Tmp.getAlignment().getAsAlign(),
1633 Src: Src.emitRawPointer(CGF), SrcAlign: Src.getAlignment().getAsAlign(),
1634 Size: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: SrcSize.getKnownMinValue()));
1635 return CGF.Builder.CreateLoad(Addr: Tmp);
1636}
1637
1638static bool CreatePFPCoercedStore(llvm::Value *Src, QualType SrcFETy,
1639 Address Dst, CodeGenFunction &CGF) {
1640 std::vector<PFPField> PFPFields = CGF.getContext().findPFPFields(Ty: SrcFETy);
1641 if (PFPFields.empty())
1642 return false;
1643
1644 llvm::Type *SrcTy = Src->getType();
1645 auto StoreCoercedField = [&](CharUnits Offset, llvm::Value *FieldVal) {
1646 if (!PFPFields.empty() && PFPFields[0].Offset == Offset) {
1647 auto FieldAddr = CGF.EmitAddressOfPFPField(RecordPtr: Dst, Field: PFPFields[0]);
1648 if (isa<llvm::IntegerType>(Val: FieldVal->getType()))
1649 FieldVal = CGF.Builder.CreateIntToPtr(V: FieldVal, DestTy: CGF.VoidPtrTy);
1650 CGF.Builder.CreateStore(Val: FieldVal, Addr: FieldAddr);
1651 PFPFields.erase(position: PFPFields.begin());
1652 } else {
1653 auto FieldAddr = CGF.Builder
1654 .CreateConstInBoundsByteGEP(
1655 Addr: Dst.withElementType(ElemTy: CGF.Int8Ty), Offset)
1656 .withElementType(ElemTy: FieldVal->getType());
1657 CGF.Builder.CreateStore(Val: FieldVal, Addr: FieldAddr);
1658 }
1659 };
1660
1661 // The types handled by this function are the only ones that may be generated
1662 // by AArch64ABIInfo::classify{Argument,Return}Type for struct types with
1663 // pointers. PFP is only supported on AArch64.
1664 if (isa<llvm::IntegerType>(Val: SrcTy) || isa<llvm::PointerType>(Val: SrcTy)) {
1665 if (isa<llvm::IntegerType>(Val: SrcTy))
1666 Src = CGF.Builder.CreateIntToPtr(V: Src, DestTy: CGF.VoidPtrTy);
1667 auto Addr = CGF.EmitAddressOfPFPField(RecordPtr: Dst, Field: PFPFields[0]);
1668 CGF.Builder.CreateStore(Val: Src, Addr);
1669 } else {
1670 auto *AT = cast<llvm::ArrayType>(Val: SrcTy);
1671 auto *ET = AT->getElementType();
1672 CharUnits WordSize = CGF.getContext().toCharUnitsFromBits(
1673 BitSize: CGF.CGM.getDataLayout().getTypeSizeInBits(Ty: ET));
1674 CharUnits Offset = CharUnits::Zero();
1675 for (unsigned i = 0; i != AT->getNumElements(); ++i, Offset += WordSize)
1676 StoreCoercedField(Offset, CGF.Builder.CreateExtractValue(Agg: Src, Idxs: i));
1677 }
1678 return true;
1679}
1680
1681void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, QualType SrcFETy,
1682 Address Dst, llvm::TypeSize DstSize,
1683 bool DstIsVolatile) {
1684 if (!DstSize)
1685 return;
1686
1687 llvm::Type *SrcTy = Src->getType();
1688 llvm::TypeSize SrcSize = CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1689
1690 // GEP into structs to try to make types match.
1691 // FIXME: This isn't really that useful with opaque types, but it impacts a
1692 // lot of regression tests.
1693 if (SrcTy != Dst.getElementType()) {
1694 if (llvm::StructType *DstSTy =
1695 dyn_cast<llvm::StructType>(Val: Dst.getElementType())) {
1696 assert(!SrcSize.isScalable());
1697 Dst = EnterStructPointerForCoercedAccess(SrcPtr: Dst, SrcSTy: DstSTy,
1698 DstSize: SrcSize.getFixedValue(), CGF&: *this);
1699 }
1700 }
1701
1702 if (CreatePFPCoercedStore(Src, SrcFETy, Dst, CGF&: *this))
1703 return;
1704
1705 if (SrcSize.isScalable() || SrcSize <= DstSize) {
1706 if (SrcTy->isIntegerTy() && Dst.getElementType()->isPointerTy() &&
1707 SrcSize == CGM.getDataLayout().getTypeAllocSize(Ty: Dst.getElementType())) {
1708 // If the value is supposed to be a pointer, convert it before storing it.
1709 Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: Dst.getElementType(), CGF&: *this);
1710 auto *I = Builder.CreateStore(Val: Src, Addr: Dst, IsVolatile: DstIsVolatile);
1711 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1712 } else if (llvm::StructType *STy =
1713 dyn_cast<llvm::StructType>(Val: Src->getType())) {
1714 // Prefer scalar stores to first-class aggregate stores.
1715 Dst = Dst.withElementType(ElemTy: SrcTy);
1716 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1717 Address EltPtr = Builder.CreateStructGEP(Addr: Dst, Index: i);
1718 llvm::Value *Elt = Builder.CreateExtractValue(Agg: Src, Idxs: i);
1719 auto *I = Builder.CreateStore(Val: Elt, Addr: EltPtr, IsVolatile: DstIsVolatile);
1720 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Elt);
1721 }
1722 } else {
1723 auto *I =
1724 Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: SrcTy), IsVolatile: DstIsVolatile);
1725 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1726 }
1727 } else if (SrcTy->isIntegerTy()) {
1728 // If the source is a simple integer, coerce it directly.
1729 llvm::Type *DstIntTy = Builder.getIntNTy(N: DstSize.getFixedValue() * 8);
1730 Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: DstIntTy, CGF&: *this);
1731 auto *I =
1732 Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: DstIntTy), IsVolatile: DstIsVolatile);
1733 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1734 } else {
1735 // Otherwise do coercion through memory. This is stupid, but
1736 // simple.
1737
1738 // Generally SrcSize is never greater than DstSize, since this means we are
1739 // losing bits. However, this can happen in cases where the structure has
1740 // additional padding, for example due to a user specified alignment.
1741 //
1742 // FIXME: Assert that we aren't truncating non-padding bits when have access
1743 // to that information.
1744 RawAddress Tmp =
1745 CreateTempAllocaForCoercion(CGF&: *this, Ty: SrcTy, MinAlign: Dst.getAlignment());
1746 Builder.CreateStore(Val: Src, Addr: Tmp);
1747 auto *I = Builder.CreateMemCpy(
1748 Dst: Dst.emitRawPointer(CGF&: *this), DstAlign: Dst.getAlignment().getAsAlign(),
1749 Src: Tmp.getPointer(), SrcAlign: Tmp.getAlignment().getAsAlign(),
1750 Size: Builder.CreateTypeSize(Ty: IntPtrTy, Size: DstSize));
1751 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1752 }
1753}
1754
1755static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1756 const ABIArgInfo &info) {
1757 if (unsigned offset = info.getDirectOffset()) {
1758 addr = addr.withElementType(ElemTy: CGF.Int8Ty);
1759 addr = CGF.Builder.CreateConstInBoundsByteGEP(
1760 Addr: addr, Offset: CharUnits::fromQuantity(Quantity: offset));
1761 addr = addr.withElementType(ElemTy: info.getCoerceToType());
1762 }
1763 return addr;
1764}
1765
1766static std::pair<llvm::Value *, bool>
1767CoerceScalableToFixed(CodeGenFunction &CGF, llvm::FixedVectorType *ToTy,
1768 llvm::ScalableVectorType *FromTy, llvm::Value *V,
1769 StringRef Name = "") {
1770 // If we are casting a scalable i1 predicate vector to a fixed i8
1771 // vector, first bitcast the source.
1772 if (FromTy->getElementType()->isIntegerTy(BitWidth: 1) &&
1773 ToTy->getElementType() == CGF.Builder.getInt8Ty()) {
1774 if (!FromTy->getElementCount().isKnownMultipleOf(RHS: 8)) {
1775 FromTy = llvm::ScalableVectorType::get(
1776 ElementType: FromTy->getElementType(),
1777 MinNumElts: llvm::alignTo<8>(Value: FromTy->getElementCount().getKnownMinValue()));
1778 llvm::Value *ZeroVec = llvm::Constant::getNullValue(Ty: FromTy);
1779 V = CGF.Builder.CreateInsertVector(DstType: FromTy, SrcVec: ZeroVec, SubVec: V, Idx: uint64_t(0));
1780 }
1781 FromTy = llvm::ScalableVectorType::get(
1782 ElementType: ToTy->getElementType(),
1783 MinNumElts: FromTy->getElementCount().getKnownMinValue() / 8);
1784 V = CGF.Builder.CreateBitCast(V, DestTy: FromTy);
1785 }
1786 if (FromTy->getElementType() == ToTy->getElementType()) {
1787 V->setName(Name + ".coerce");
1788 V = CGF.Builder.CreateExtractVector(DstType: ToTy, SrcVec: V, Idx: uint64_t(0), Name: "cast.fixed");
1789 return {V, true};
1790 }
1791 return {V, false};
1792}
1793
1794namespace {
1795
1796/// Encapsulates information about the way function arguments from
1797/// CGFunctionInfo should be passed to actual LLVM IR function.
1798class ClangToLLVMArgMapping {
1799 static const unsigned InvalidIndex = ~0U;
1800 unsigned InallocaArgNo;
1801 unsigned SRetArgNo;
1802 unsigned TotalIRArgs;
1803
1804 /// Arguments of LLVM IR function corresponding to single Clang argument.
1805 struct IRArgs {
1806 unsigned PaddingArgIndex;
1807 // Argument is expanded to IR arguments at positions
1808 // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1809 unsigned FirstArgIndex;
1810 unsigned NumberOfArgs;
1811
1812 IRArgs()
1813 : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1814 NumberOfArgs(0) {}
1815 };
1816
1817 SmallVector<IRArgs, 8> ArgInfo;
1818
1819public:
1820 ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1821 bool OnlyRequiredArgs = false)
1822 : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1823 ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1824 construct(Context, FI, OnlyRequiredArgs);
1825 }
1826
1827 bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1828 unsigned getInallocaArgNo() const {
1829 assert(hasInallocaArg());
1830 return InallocaArgNo;
1831 }
1832
1833 bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1834 unsigned getSRetArgNo() const {
1835 assert(hasSRetArg());
1836 return SRetArgNo;
1837 }
1838
1839 unsigned totalIRArgs() const { return TotalIRArgs; }
1840
1841 bool hasPaddingArg(unsigned ArgNo) const {
1842 assert(ArgNo < ArgInfo.size());
1843 return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1844 }
1845 unsigned getPaddingArgNo(unsigned ArgNo) const {
1846 assert(hasPaddingArg(ArgNo));
1847 return ArgInfo[ArgNo].PaddingArgIndex;
1848 }
1849
1850 /// Returns index of first IR argument corresponding to ArgNo, and their
1851 /// quantity.
1852 std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1853 assert(ArgNo < ArgInfo.size());
1854 return std::make_pair(x: ArgInfo[ArgNo].FirstArgIndex,
1855 y: ArgInfo[ArgNo].NumberOfArgs);
1856 }
1857
1858private:
1859 void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1860 bool OnlyRequiredArgs);
1861};
1862
1863void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1864 const CGFunctionInfo &FI,
1865 bool OnlyRequiredArgs) {
1866 unsigned IRArgNo = 0;
1867 bool SwapThisWithSRet = false;
1868 const ABIArgInfo &RetAI = FI.getReturnInfo();
1869
1870 if (RetAI.getKind() == ABIArgInfo::Indirect) {
1871 SwapThisWithSRet = RetAI.isSRetAfterThis();
1872 SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1873 }
1874
1875 unsigned ArgNo = 0;
1876 unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();
1877 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1878 ++I, ++ArgNo) {
1879 assert(I != FI.arg_end());
1880 QualType ArgType = I->type;
1881 const ABIArgInfo &AI = I->info;
1882 // Collect data about IR arguments corresponding to Clang argument ArgNo.
1883 auto &IRArgs = ArgInfo[ArgNo];
1884
1885 if (AI.getPaddingType())
1886 IRArgs.PaddingArgIndex = IRArgNo++;
1887
1888 switch (AI.getKind()) {
1889 case ABIArgInfo::TargetSpecific:
1890 case ABIArgInfo::Extend:
1891 case ABIArgInfo::Direct: {
1892 // FIXME: handle sseregparm someday...
1893 llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: AI.getCoerceToType());
1894 if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
1895 IRArgs.NumberOfArgs = STy->getNumElements();
1896 } else {
1897 IRArgs.NumberOfArgs = 1;
1898 }
1899 break;
1900 }
1901 case ABIArgInfo::Indirect:
1902 case ABIArgInfo::IndirectAliased:
1903 IRArgs.NumberOfArgs = 1;
1904 break;
1905 case ABIArgInfo::Ignore:
1906 case ABIArgInfo::InAlloca:
1907 // ignore and inalloca doesn't have matching LLVM parameters.
1908 IRArgs.NumberOfArgs = 0;
1909 break;
1910 case ABIArgInfo::CoerceAndExpand:
1911 IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1912 break;
1913 case ABIArgInfo::Expand:
1914 IRArgs.NumberOfArgs = getExpansionSize(Ty: ArgType, Context);
1915 break;
1916 }
1917
1918 if (IRArgs.NumberOfArgs > 0) {
1919 IRArgs.FirstArgIndex = IRArgNo;
1920 IRArgNo += IRArgs.NumberOfArgs;
1921 }
1922
1923 // Skip over the sret parameter when it comes second. We already handled it
1924 // above.
1925 if (IRArgNo == 1 && SwapThisWithSRet)
1926 IRArgNo++;
1927 }
1928 assert(ArgNo == ArgInfo.size());
1929
1930 if (FI.usesInAlloca())
1931 InallocaArgNo = IRArgNo++;
1932
1933 TotalIRArgs = IRArgNo;
1934}
1935} // namespace
1936
1937/***/
1938
1939bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1940 const auto &RI = FI.getReturnInfo();
1941 return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());
1942}
1943
1944bool CodeGenModule::ReturnTypeHasInReg(const CGFunctionInfo &FI) {
1945 const auto &RI = FI.getReturnInfo();
1946 return RI.getInReg();
1947}
1948
1949bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1950 return ReturnTypeUsesSRet(FI) &&
1951 getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
1952}
1953
1954bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1955 if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1956 switch (BT->getKind()) {
1957 default:
1958 return false;
1959 case BuiltinType::Float:
1960 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Float);
1961 case BuiltinType::Double:
1962 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Double);
1963 case BuiltinType::LongDouble:
1964 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::LongDouble);
1965 }
1966 }
1967
1968 return false;
1969}
1970
1971bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1972 if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1973 if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1974 if (BT->getKind() == BuiltinType::LongDouble)
1975 return getTarget().useObjCFP2RetForComplexLongDouble();
1976 }
1977 }
1978
1979 return false;
1980}
1981
1982llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1983 const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1984 return GetFunctionType(Info: FI);
1985}
1986
1987llvm::FunctionType *CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1988
1989 bool Inserted = FunctionsBeingProcessed.insert(Ptr: &FI).second;
1990 (void)Inserted;
1991 assert(Inserted && "Recursively being processed?");
1992
1993 llvm::Type *resultType = nullptr;
1994 const ABIArgInfo &retAI = FI.getReturnInfo();
1995 switch (retAI.getKind()) {
1996 case ABIArgInfo::Expand:
1997 case ABIArgInfo::IndirectAliased:
1998 llvm_unreachable("Invalid ABI kind for return argument");
1999
2000 case ABIArgInfo::TargetSpecific:
2001 case ABIArgInfo::Extend:
2002 case ABIArgInfo::Direct:
2003 resultType = retAI.getCoerceToType();
2004 break;
2005
2006 case ABIArgInfo::InAlloca:
2007 if (retAI.getInAllocaSRet()) {
2008 // sret things on win32 aren't void, they return the sret pointer.
2009 QualType ret = FI.getReturnType();
2010 unsigned addressSpace = CGM.getTypes().getTargetAddressSpace(T: ret);
2011 resultType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: addressSpace);
2012 } else {
2013 resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
2014 }
2015 break;
2016
2017 case ABIArgInfo::Indirect:
2018 case ABIArgInfo::Ignore:
2019 resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
2020 break;
2021
2022 case ABIArgInfo::CoerceAndExpand:
2023 resultType = retAI.getUnpaddedCoerceAndExpandType();
2024 break;
2025 }
2026
2027 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
2028 SmallVector<llvm::Type *, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
2029
2030 // Add type for sret argument.
2031 if (IRFunctionArgs.hasSRetArg()) {
2032 ArgTypes[IRFunctionArgs.getSRetArgNo()] = llvm::PointerType::get(
2033 C&: getLLVMContext(), AddressSpace: FI.getReturnInfo().getIndirectAddrSpace());
2034 }
2035
2036 // Add type for inalloca argument.
2037 if (IRFunctionArgs.hasInallocaArg())
2038 ArgTypes[IRFunctionArgs.getInallocaArgNo()] =
2039 llvm::PointerType::getUnqual(C&: getLLVMContext());
2040
2041 // Add in all of the required arguments.
2042 unsigned ArgNo = 0;
2043 CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
2044 ie = it + FI.getNumRequiredArgs();
2045 for (; it != ie; ++it, ++ArgNo) {
2046 const ABIArgInfo &ArgInfo = it->info;
2047
2048 // Insert a padding type to ensure proper alignment.
2049 if (IRFunctionArgs.hasPaddingArg(ArgNo))
2050 ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2051 ArgInfo.getPaddingType();
2052
2053 unsigned FirstIRArg, NumIRArgs;
2054 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2055
2056 switch (ArgInfo.getKind()) {
2057 case ABIArgInfo::Ignore:
2058 case ABIArgInfo::InAlloca:
2059 assert(NumIRArgs == 0);
2060 break;
2061
2062 case ABIArgInfo::Indirect:
2063 assert(NumIRArgs == 1);
2064 // indirect arguments are always on the stack, which is alloca addr space.
2065 ArgTypes[FirstIRArg] = llvm::PointerType::get(
2066 C&: getLLVMContext(), AddressSpace: CGM.getDataLayout().getAllocaAddrSpace());
2067 break;
2068 case ABIArgInfo::IndirectAliased:
2069 assert(NumIRArgs == 1);
2070 ArgTypes[FirstIRArg] = llvm::PointerType::get(
2071 C&: getLLVMContext(), AddressSpace: ArgInfo.getIndirectAddrSpace());
2072 break;
2073 case ABIArgInfo::TargetSpecific:
2074 case ABIArgInfo::Extend:
2075 case ABIArgInfo::Direct: {
2076 // Fast-isel and the optimizer generally like scalar values better than
2077 // FCAs, so we flatten them if this is safe to do for this argument.
2078 llvm::Type *argType = ArgInfo.getCoerceToType();
2079 llvm::StructType *st = dyn_cast<llvm::StructType>(Val: argType);
2080 if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
2081 assert(NumIRArgs == st->getNumElements());
2082 for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
2083 ArgTypes[FirstIRArg + i] = st->getElementType(N: i);
2084 } else {
2085 assert(NumIRArgs == 1);
2086 ArgTypes[FirstIRArg] = argType;
2087 }
2088 break;
2089 }
2090
2091 case ABIArgInfo::CoerceAndExpand: {
2092 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
2093 for (auto *EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
2094 *ArgTypesIter++ = EltTy;
2095 }
2096 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
2097 break;
2098 }
2099
2100 case ABIArgInfo::Expand:
2101 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
2102 getExpandedTypes(Ty: it->type, TI&: ArgTypesIter);
2103 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
2104 break;
2105 }
2106 }
2107
2108 bool Erased = FunctionsBeingProcessed.erase(Ptr: &FI);
2109 (void)Erased;
2110 assert(Erased && "Not in set?");
2111
2112 return llvm::FunctionType::get(Result: resultType, Params: ArgTypes, isVarArg: FI.isVariadic());
2113}
2114
2115llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
2116 const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
2117 const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
2118
2119 if (!isFuncTypeConvertible(FT: FPT))
2120 return llvm::StructType::get(Context&: getLLVMContext());
2121
2122 return GetFunctionType(GD);
2123}
2124
2125static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
2126 llvm::AttrBuilder &FuncAttrs,
2127 const FunctionProtoType *FPT) {
2128 if (!FPT)
2129 return;
2130
2131 if (!isUnresolvedExceptionSpec(ESpecType: FPT->getExceptionSpecType()) &&
2132 FPT->isNothrow())
2133 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2134
2135 unsigned SMEBits = FPT->getAArch64SMEAttributes();
2136 if (SMEBits & FunctionType::SME_PStateSMEnabledMask)
2137 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_enabled");
2138 if (SMEBits & FunctionType::SME_PStateSMCompatibleMask)
2139 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_compatible");
2140 if (SMEBits & FunctionType::SME_AgnosticZAStateMask)
2141 FuncAttrs.addAttribute(A: "aarch64_za_state_agnostic");
2142
2143 // ZA
2144 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
2145 FuncAttrs.addAttribute(A: "aarch64_preserves_za");
2146 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_In)
2147 FuncAttrs.addAttribute(A: "aarch64_in_za");
2148 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Out)
2149 FuncAttrs.addAttribute(A: "aarch64_out_za");
2150 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_InOut)
2151 FuncAttrs.addAttribute(A: "aarch64_inout_za");
2152
2153 // ZT0
2154 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
2155 FuncAttrs.addAttribute(A: "aarch64_preserves_zt0");
2156 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_In)
2157 FuncAttrs.addAttribute(A: "aarch64_in_zt0");
2158 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Out)
2159 FuncAttrs.addAttribute(A: "aarch64_out_zt0");
2160 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_InOut)
2161 FuncAttrs.addAttribute(A: "aarch64_inout_zt0");
2162}
2163
2164static void AddAttributesFromOMPAssumes(llvm::AttrBuilder &FuncAttrs,
2165 const Decl *Callee) {
2166 if (!Callee)
2167 return;
2168
2169 SmallVector<StringRef, 4> Attrs;
2170
2171 for (const OMPAssumeAttr *AA : Callee->specific_attrs<OMPAssumeAttr>())
2172 AA->getAssumption().split(A&: Attrs, Separator: ",");
2173
2174 if (!Attrs.empty())
2175 FuncAttrs.addAttribute(A: llvm::AssumptionAttrKey,
2176 V: llvm::join(Begin: Attrs.begin(), End: Attrs.end(), Separator: ","));
2177}
2178
2179bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,
2180 QualType ReturnType) const {
2181 // We can't just discard the return value for a record type with a
2182 // complex destructor or a non-trivially copyable type.
2183 if (const RecordType *RT =
2184 ReturnType.getCanonicalType()->getAsCanonical<RecordType>()) {
2185 if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(Val: RT->getDecl()))
2186 return ClassDecl->hasTrivialDestructor();
2187 }
2188 return ReturnType.isTriviallyCopyableType(Context);
2189}
2190
2191static bool HasStrictReturn(const CodeGenModule &Module, QualType RetTy,
2192 const Decl *TargetDecl) {
2193 // As-is msan can not tolerate noundef mismatch between caller and
2194 // implementation. Mismatch is possible for e.g. indirect calls from C-caller
2195 // into C++. Such mismatches lead to confusing false reports. To avoid
2196 // expensive workaround on msan we enforce initialization event in uncommon
2197 // cases where it's allowed.
2198 if (Module.getLangOpts().Sanitize.has(K: SanitizerKind::Memory))
2199 return true;
2200 // C++ explicitly makes returning undefined values UB. C's rule only applies
2201 // to used values, so we never mark them noundef for now.
2202 if (!Module.getLangOpts().CPlusPlus)
2203 return false;
2204 if (TargetDecl) {
2205 if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2206 if (FDecl->isExternC())
2207 return false;
2208 } else if (const VarDecl *VDecl = dyn_cast<VarDecl>(Val: TargetDecl)) {
2209 // Function pointer.
2210 if (VDecl->isExternC())
2211 return false;
2212 }
2213 }
2214
2215 // We don't want to be too aggressive with the return checking, unless
2216 // it's explicit in the code opts or we're using an appropriate sanitizer.
2217 // Try to respect what the programmer intended.
2218 return Module.getCodeGenOpts().StrictReturn ||
2219 !Module.MayDropFunctionReturn(Context: Module.getContext(), ReturnType: RetTy) ||
2220 Module.getLangOpts().Sanitize.has(K: SanitizerKind::Return);
2221}
2222
2223/// Add denormal-fp-math and denormal-fp-math-f32 as appropriate for the
2224/// requested denormal behavior, accounting for the overriding behavior of the
2225/// -f32 case.
2226static void addDenormalModeAttrs(llvm::DenormalMode FPDenormalMode,
2227 llvm::DenormalMode FP32DenormalMode,
2228 llvm::AttrBuilder &FuncAttrs) {
2229 llvm::DenormalFPEnv FPEnv(FPDenormalMode, FP32DenormalMode);
2230 if (FPEnv != llvm::DenormalFPEnv::getDefault())
2231 FuncAttrs.addDenormalFPEnvAttr(Mode: FPEnv);
2232}
2233
2234/// Add default attributes to a function, which have merge semantics under
2235/// -mlink-builtin-bitcode and should not simply overwrite any existing
2236/// attributes in the linked library.
2237static void
2238addMergableDefaultFunctionAttributes(const CodeGenOptions &CodeGenOpts,
2239 llvm::AttrBuilder &FuncAttrs) {
2240 addDenormalModeAttrs(FPDenormalMode: CodeGenOpts.FPDenormalMode, FP32DenormalMode: CodeGenOpts.FP32DenormalMode,
2241 FuncAttrs);
2242}
2243
2244static void getTrivialDefaultFunctionAttributes(
2245 StringRef Name, bool HasOptnone, const CodeGenOptions &CodeGenOpts,
2246 const LangOptions &LangOpts, bool AttrOnCallSite,
2247 llvm::AttrBuilder &FuncAttrs) {
2248 // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
2249 if (!HasOptnone) {
2250 if (CodeGenOpts.OptimizeSize)
2251 FuncAttrs.addAttribute(Val: llvm::Attribute::OptimizeForSize);
2252 if (CodeGenOpts.OptimizeSize == 2)
2253 FuncAttrs.addAttribute(Val: llvm::Attribute::MinSize);
2254 }
2255
2256 if (CodeGenOpts.DisableRedZone)
2257 FuncAttrs.addAttribute(Val: llvm::Attribute::NoRedZone);
2258 if (CodeGenOpts.IndirectTlsSegRefs)
2259 FuncAttrs.addAttribute(A: "indirect-tls-seg-refs");
2260 if (CodeGenOpts.NoImplicitFloat)
2261 FuncAttrs.addAttribute(Val: llvm::Attribute::NoImplicitFloat);
2262
2263 if (AttrOnCallSite) {
2264 // Attributes that should go on the call site only.
2265 // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
2266 // the -fno-builtin-foo list.
2267 if (!CodeGenOpts.SimplifyLibCalls || LangOpts.isNoBuiltinFunc(Name))
2268 FuncAttrs.addAttribute(Val: llvm::Attribute::NoBuiltin);
2269 if (!CodeGenOpts.TrapFuncName.empty())
2270 FuncAttrs.addAttribute(A: "trap-func-name", V: CodeGenOpts.TrapFuncName);
2271 } else {
2272 switch (CodeGenOpts.getFramePointer()) {
2273 case CodeGenOptions::FramePointerKind::None:
2274 // This is the default behavior.
2275 break;
2276 case CodeGenOptions::FramePointerKind::Reserved:
2277 case CodeGenOptions::FramePointerKind::NonLeafNoReserve:
2278 case CodeGenOptions::FramePointerKind::NonLeaf:
2279 case CodeGenOptions::FramePointerKind::All:
2280 FuncAttrs.addAttribute(A: "frame-pointer",
2281 V: CodeGenOptions::getFramePointerKindName(
2282 Kind: CodeGenOpts.getFramePointer()));
2283 }
2284
2285 if (CodeGenOpts.LessPreciseFPMAD)
2286 FuncAttrs.addAttribute(A: "less-precise-fpmad", V: "true");
2287
2288 if (CodeGenOpts.NullPointerIsValid)
2289 FuncAttrs.addAttribute(Val: llvm::Attribute::NullPointerIsValid);
2290
2291 if (LangOpts.getDefaultExceptionMode() == LangOptions::FPE_Ignore)
2292 FuncAttrs.addAttribute(A: "no-trapping-math", V: "true");
2293
2294 // TODO: Are these all needed?
2295 // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
2296 if (CodeGenOpts.SoftFloat)
2297 FuncAttrs.addAttribute(A: "use-soft-float", V: "true");
2298 FuncAttrs.addAttribute(A: "stack-protector-buffer-size",
2299 V: llvm::utostr(X: CodeGenOpts.SSPBufferSize));
2300 if (LangOpts.NoSignedZero)
2301 FuncAttrs.addAttribute(A: "no-signed-zeros-fp-math", V: "true");
2302
2303 // TODO: Reciprocal estimate codegen options should apply to instructions?
2304 const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
2305 if (!Recips.empty())
2306 FuncAttrs.addAttribute(A: "reciprocal-estimates", V: llvm::join(R: Recips, Separator: ","));
2307
2308 if (!CodeGenOpts.PreferVectorWidth.empty() &&
2309 CodeGenOpts.PreferVectorWidth != "none")
2310 FuncAttrs.addAttribute(A: "prefer-vector-width",
2311 V: CodeGenOpts.PreferVectorWidth);
2312
2313 if (CodeGenOpts.StackRealignment)
2314 FuncAttrs.addAttribute(A: "stackrealign");
2315 if (CodeGenOpts.Backchain)
2316 FuncAttrs.addAttribute(A: "backchain");
2317 if (CodeGenOpts.EnableSegmentedStacks)
2318 FuncAttrs.addAttribute(A: "split-stack");
2319
2320 if (CodeGenOpts.SpeculativeLoadHardening)
2321 FuncAttrs.addAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2322
2323 // Add zero-call-used-regs attribute.
2324 switch (CodeGenOpts.getZeroCallUsedRegs()) {
2325 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
2326 FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2327 break;
2328 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
2329 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr-arg");
2330 break;
2331 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
2332 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr");
2333 break;
2334 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
2335 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-arg");
2336 break;
2337 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
2338 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used");
2339 break;
2340 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
2341 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr-arg");
2342 break;
2343 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
2344 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr");
2345 break;
2346 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
2347 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-arg");
2348 break;
2349 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
2350 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all");
2351 break;
2352 }
2353 }
2354
2355 if (LangOpts.assumeFunctionsAreConvergent()) {
2356 // Conservatively, mark all functions and calls in CUDA and OpenCL as
2357 // convergent (meaning, they may call an intrinsically convergent op, such
2358 // as __syncthreads() / barrier(), and so can't have certain optimizations
2359 // applied around them). LLVM will remove this attribute where it safely
2360 // can.
2361 FuncAttrs.addAttribute(Val: llvm::Attribute::Convergent);
2362 }
2363
2364 // TODO: NoUnwind attribute should be added for other GPU modes HIP,
2365 // OpenMP offload. AFAIK, neither of them support exceptions in device code.
2366 if ((LangOpts.CUDA && LangOpts.CUDAIsDevice) || LangOpts.OpenCL ||
2367 LangOpts.SYCLIsDevice) {
2368 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2369 }
2370
2371 if (CodeGenOpts.SaveRegParams && !AttrOnCallSite)
2372 FuncAttrs.addAttribute(A: "save-reg-params");
2373
2374 for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
2375 StringRef Var, Value;
2376 std::tie(args&: Var, args&: Value) = Attr.split(Separator: '=');
2377 FuncAttrs.addAttribute(A: Var, V: Value);
2378 }
2379
2380 TargetInfo::BranchProtectionInfo BPI(LangOpts);
2381 TargetCodeGenInfo::initBranchProtectionFnAttributes(BPI, FuncAttrs);
2382}
2383
2384/// Merges `target-features` from \TargetOpts and \F, and sets the result in
2385/// \FuncAttr
2386/// * features from \F are always kept
2387/// * a feature from \TargetOpts is kept if itself and its opposite are absent
2388/// from \F
2389static void
2390overrideFunctionFeaturesWithTargetFeatures(llvm::AttrBuilder &FuncAttr,
2391 const llvm::Function &F,
2392 const TargetOptions &TargetOpts) {
2393 auto FFeatures = F.getFnAttribute(Kind: "target-features");
2394
2395 llvm::StringSet<> MergedNames;
2396 SmallVector<StringRef> MergedFeatures;
2397 MergedFeatures.reserve(N: TargetOpts.Features.size());
2398
2399 auto AddUnmergedFeatures = [&](auto &&FeatureRange) {
2400 for (StringRef Feature : FeatureRange) {
2401 if (Feature.empty())
2402 continue;
2403 assert(Feature[0] == '+' || Feature[0] == '-');
2404 StringRef Name = Feature.drop_front(N: 1);
2405 bool Merged = !MergedNames.insert(key: Name).second;
2406 if (!Merged)
2407 MergedFeatures.push_back(Elt: Feature);
2408 }
2409 };
2410
2411 if (FFeatures.isValid())
2412 AddUnmergedFeatures(llvm::split(Str: FFeatures.getValueAsString(), Separator: ','));
2413 AddUnmergedFeatures(TargetOpts.Features);
2414
2415 if (!MergedFeatures.empty()) {
2416 llvm::sort(C&: MergedFeatures);
2417 FuncAttr.addAttribute(A: "target-features", V: llvm::join(R&: MergedFeatures, Separator: ","));
2418 }
2419}
2420
2421void CodeGen::mergeDefaultFunctionDefinitionAttributes(
2422 llvm::Function &F, const CodeGenOptions &CodeGenOpts,
2423 const LangOptions &LangOpts, const TargetOptions &TargetOpts,
2424 bool WillInternalize) {
2425
2426 llvm::AttrBuilder FuncAttrs(F.getContext());
2427 // Here we only extract the options that are relevant compared to the version
2428 // from GetCPUAndFeaturesAttributes.
2429 if (!TargetOpts.CPU.empty())
2430 FuncAttrs.addAttribute(A: "target-cpu", V: TargetOpts.CPU);
2431 if (!TargetOpts.TuneCPU.empty())
2432 FuncAttrs.addAttribute(A: "tune-cpu", V: TargetOpts.TuneCPU);
2433
2434 ::getTrivialDefaultFunctionAttributes(Name: F.getName(), HasOptnone: F.hasOptNone(),
2435 CodeGenOpts, LangOpts,
2436 /*AttrOnCallSite=*/false, FuncAttrs);
2437
2438 if (!WillInternalize && F.isInterposable()) {
2439 // Do not promote "dynamic" denormal-fp-math to this translation unit's
2440 // setting for weak functions that won't be internalized. The user has no
2441 // real control for how builtin bitcode is linked, so we shouldn't assume
2442 // later copies will use a consistent mode.
2443 F.addFnAttrs(Attrs: FuncAttrs);
2444 return;
2445 }
2446
2447 llvm::AttributeMask AttrsToRemove;
2448
2449 llvm::DenormalFPEnv OptsFPEnv(CodeGenOpts.FPDenormalMode,
2450 CodeGenOpts.FP32DenormalMode);
2451 llvm::DenormalFPEnv MergedFPEnv =
2452 OptsFPEnv.mergeCalleeMode(Callee: F.getDenormalFPEnv());
2453
2454 if (MergedFPEnv == llvm::DenormalFPEnv::getDefault()) {
2455 AttrsToRemove.addAttribute(Val: llvm::Attribute::DenormalFPEnv);
2456 } else {
2457 // Overwrite existing attribute
2458 FuncAttrs.addDenormalFPEnvAttr(Mode: MergedFPEnv);
2459 }
2460
2461 F.removeFnAttrs(Attrs: AttrsToRemove);
2462
2463 overrideFunctionFeaturesWithTargetFeatures(FuncAttr&: FuncAttrs, F, TargetOpts);
2464
2465 F.addFnAttrs(Attrs: FuncAttrs);
2466}
2467
2468void CodeGenModule::getTrivialDefaultFunctionAttributes(
2469 StringRef Name, bool HasOptnone, bool AttrOnCallSite,
2470 llvm::AttrBuilder &FuncAttrs) {
2471 ::getTrivialDefaultFunctionAttributes(Name, HasOptnone, CodeGenOpts: getCodeGenOpts(),
2472 LangOpts: getLangOpts(), AttrOnCallSite,
2473 FuncAttrs);
2474}
2475
2476void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
2477 bool HasOptnone,
2478 bool AttrOnCallSite,
2479 llvm::AttrBuilder &FuncAttrs) {
2480 getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,
2481 FuncAttrs);
2482
2483 if (!AttrOnCallSite)
2484 TargetCodeGenInfo::initPointerAuthFnAttributes(Opts: CodeGenOpts.PointerAuth,
2485 FuncAttrs);
2486
2487 // If we're just getting the default, get the default values for mergeable
2488 // attributes.
2489 if (!AttrOnCallSite)
2490 addMergableDefaultFunctionAttributes(CodeGenOpts, FuncAttrs);
2491}
2492
2493void CodeGenModule::addDefaultFunctionDefinitionAttributes(
2494 llvm::AttrBuilder &attrs) {
2495 getDefaultFunctionAttributes(/*function name*/ Name: "", /*optnone*/ HasOptnone: false,
2496 /*for call*/ AttrOnCallSite: false, FuncAttrs&: attrs);
2497 GetCPUAndFeaturesAttributes(GD: GlobalDecl(), AttrBuilder&: attrs);
2498}
2499
2500static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
2501 const LangOptions &LangOpts,
2502 const NoBuiltinAttr *NBA = nullptr) {
2503 auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
2504 SmallString<32> AttributeName;
2505 AttributeName += "no-builtin-";
2506 AttributeName += BuiltinName;
2507 FuncAttrs.addAttribute(A: AttributeName);
2508 };
2509
2510 // First, handle the language options passed through -fno-builtin.
2511 if (LangOpts.NoBuiltin) {
2512 // -fno-builtin disables them all.
2513 FuncAttrs.addAttribute(A: "no-builtins");
2514 return;
2515 }
2516
2517 // Then, add attributes for builtins specified through -fno-builtin-<name>.
2518 llvm::for_each(Range: LangOpts.NoBuiltinFuncs, F: AddNoBuiltinAttr);
2519
2520 // Now, let's check the __attribute__((no_builtin("...")) attribute added to
2521 // the source.
2522 if (!NBA)
2523 return;
2524
2525 // If there is a wildcard in the builtin names specified through the
2526 // attribute, disable them all.
2527 if (llvm::is_contained(Range: NBA->builtinNames(), Element: "*")) {
2528 FuncAttrs.addAttribute(A: "no-builtins");
2529 return;
2530 }
2531
2532 // And last, add the rest of the builtin names.
2533 llvm::for_each(Range: NBA->builtinNames(), F: AddNoBuiltinAttr);
2534}
2535
2536static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
2537 const llvm::DataLayout &DL, const ABIArgInfo &AI,
2538 bool CheckCoerce = true) {
2539 llvm::Type *Ty = Types.ConvertTypeForMem(T: QTy);
2540 if (AI.getKind() == ABIArgInfo::Indirect ||
2541 AI.getKind() == ABIArgInfo::IndirectAliased)
2542 return true;
2543 if (AI.getKind() == ABIArgInfo::Extend && !AI.isNoExt())
2544 return true;
2545 if (!DL.typeSizeEqualsStoreSize(Ty))
2546 // TODO: This will result in a modest amount of values not marked noundef
2547 // when they could be. We care about values that *invisibly* contain undef
2548 // bits from the perspective of LLVM IR.
2549 return false;
2550 if (CheckCoerce && AI.canHaveCoerceToType()) {
2551 llvm::Type *CoerceTy = AI.getCoerceToType();
2552 if (llvm::TypeSize::isKnownGT(LHS: DL.getTypeSizeInBits(Ty: CoerceTy),
2553 RHS: DL.getTypeSizeInBits(Ty)))
2554 // If we're coercing to a type with a greater size than the canonical one,
2555 // we're introducing new undef bits.
2556 // Coercing to a type of smaller or equal size is ok, as we know that
2557 // there's no internal padding (typeSizeEqualsStoreSize).
2558 return false;
2559 }
2560 if (QTy->isBitIntType())
2561 return true;
2562 if (QTy->isReferenceType())
2563 return true;
2564 if (QTy->isNullPtrType())
2565 return false;
2566 if (QTy->isMemberPointerType())
2567 // TODO: Some member pointers are `noundef`, but it depends on the ABI. For
2568 // now, never mark them.
2569 return false;
2570 if (QTy->isScalarType()) {
2571 if (const ComplexType *Complex = dyn_cast<ComplexType>(Val&: QTy))
2572 return DetermineNoUndef(QTy: Complex->getElementType(), Types, DL, AI, CheckCoerce: false);
2573 return true;
2574 }
2575 if (const VectorType *Vector = dyn_cast<VectorType>(Val&: QTy))
2576 return DetermineNoUndef(QTy: Vector->getElementType(), Types, DL, AI, CheckCoerce: false);
2577 if (const MatrixType *Matrix = dyn_cast<MatrixType>(Val&: QTy))
2578 return DetermineNoUndef(QTy: Matrix->getElementType(), Types, DL, AI, CheckCoerce: false);
2579 if (const ArrayType *Array = dyn_cast<ArrayType>(Val&: QTy))
2580 return DetermineNoUndef(QTy: Array->getElementType(), Types, DL, AI, CheckCoerce: false);
2581
2582 // TODO: Some structs may be `noundef`, in specific situations.
2583 return false;
2584}
2585
2586/// Check if the argument of a function has maybe_undef attribute.
2587static bool IsArgumentMaybeUndef(const Decl *TargetDecl,
2588 unsigned NumRequiredArgs, unsigned ArgNo) {
2589 const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
2590 if (!FD)
2591 return false;
2592
2593 // Assume variadic arguments do not have maybe_undef attribute.
2594 if (ArgNo >= NumRequiredArgs)
2595 return false;
2596
2597 // Check if argument has maybe_undef attribute.
2598 if (ArgNo < FD->getNumParams()) {
2599 const ParmVarDecl *Param = FD->getParamDecl(i: ArgNo);
2600 if (Param && Param->hasAttr<MaybeUndefAttr>())
2601 return true;
2602 }
2603
2604 return false;
2605}
2606
2607/// Test if it's legal to apply nofpclass for the given parameter type and it's
2608/// lowered IR type.
2609static bool canApplyNoFPClass(const ABIArgInfo &AI, QualType ParamType,
2610 bool IsReturn) {
2611 // Should only apply to FP types in the source, not ABI promoted.
2612 if (!ParamType->hasFloatingRepresentation())
2613 return false;
2614
2615 // The promoted-to IR type also needs to support nofpclass.
2616 llvm::Type *IRTy = AI.getCoerceToType();
2617 if (llvm::AttributeFuncs::isNoFPClassCompatibleType(Ty: IRTy))
2618 return true;
2619
2620 if (llvm::StructType *ST = dyn_cast<llvm::StructType>(Val: IRTy)) {
2621 return !IsReturn && AI.getCanBeFlattened() &&
2622 llvm::all_of(Range: ST->elements(),
2623 P: llvm::AttributeFuncs::isNoFPClassCompatibleType);
2624 }
2625
2626 return false;
2627}
2628
2629/// Return the nofpclass mask that can be applied to floating-point parameters.
2630static llvm::FPClassTest getNoFPClassTestMask(const LangOptions &LangOpts) {
2631 llvm::FPClassTest Mask = llvm::fcNone;
2632 if (LangOpts.NoHonorInfs)
2633 Mask |= llvm::fcInf;
2634 if (LangOpts.NoHonorNaNs)
2635 Mask |= llvm::fcNan;
2636 return Mask;
2637}
2638
2639void CodeGenModule::AdjustMemoryAttribute(StringRef Name,
2640 CGCalleeInfo CalleeInfo,
2641 llvm::AttributeList &Attrs) {
2642 if (Attrs.getMemoryEffects().getModRef() == llvm::ModRefInfo::NoModRef) {
2643 Attrs = Attrs.removeFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::Memory);
2644 llvm::Attribute MemoryAttr = llvm::Attribute::getWithMemoryEffects(
2645 Context&: getLLVMContext(), ME: llvm::MemoryEffects::writeOnly());
2646 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Attr: MemoryAttr);
2647 }
2648}
2649
2650/// Construct the IR attribute list of a function or call.
2651///
2652/// When adding an attribute, please consider where it should be handled:
2653///
2654/// - getDefaultFunctionAttributes is for attributes that are essentially
2655/// part of the global target configuration (but perhaps can be
2656/// overridden on a per-function basis). Adding attributes there
2657/// will cause them to also be set in frontends that build on Clang's
2658/// target-configuration logic, as well as for code defined in library
2659/// modules such as CUDA's libdevice.
2660///
2661/// - ConstructAttributeList builds on top of getDefaultFunctionAttributes
2662/// and adds declaration-specific, convention-specific, and
2663/// frontend-specific logic. The last is of particular importance:
2664/// attributes that restrict how the frontend generates code must be
2665/// added here rather than getDefaultFunctionAttributes.
2666///
2667void CodeGenModule::ConstructAttributeList(StringRef Name,
2668 const CGFunctionInfo &FI,
2669 CGCalleeInfo CalleeInfo,
2670 llvm::AttributeList &AttrList,
2671 unsigned &CallingConv,
2672 bool AttrOnCallSite, bool IsThunk) {
2673 llvm::AttrBuilder FuncAttrs(getLLVMContext());
2674 llvm::AttrBuilder RetAttrs(getLLVMContext());
2675
2676 // Collect function IR attributes from the CC lowering.
2677 // We'll collect the paramete and result attributes later.
2678 CallingConv = FI.getEffectiveCallingConvention();
2679 if (FI.isNoReturn())
2680 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2681 if (FI.isCmseNSCall())
2682 FuncAttrs.addAttribute(A: "cmse_nonsecure_call");
2683
2684 // Collect function IR attributes from the callee prototype if we have one.
2685 AddAttributesFromFunctionProtoType(Ctx&: getContext(), FuncAttrs,
2686 FPT: CalleeInfo.getCalleeFunctionProtoType());
2687 const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2688
2689 // Attach assumption attributes to the declaration. If this is a call
2690 // site, attach assumptions from the caller to the call as well.
2691 AddAttributesFromOMPAssumes(FuncAttrs, Callee: TargetDecl);
2692
2693 bool HasOptnone = false;
2694 // The NoBuiltinAttr attached to the target FunctionDecl.
2695 const NoBuiltinAttr *NBA = nullptr;
2696
2697 // Some ABIs may result in additional accesses to arguments that may
2698 // otherwise not be present.
2699 std::optional<llvm::Attribute::AttrKind> MemAttrForPtrArgs;
2700 bool AddedPotentialArgAccess = false;
2701 auto AddPotentialArgAccess = [&]() {
2702 AddedPotentialArgAccess = true;
2703 llvm::Attribute A = FuncAttrs.getAttribute(Kind: llvm::Attribute::Memory);
2704 if (A.isValid())
2705 FuncAttrs.addMemoryAttr(ME: A.getMemoryEffects() |
2706 llvm::MemoryEffects::argMemOnly());
2707 };
2708
2709 // Collect function IR attributes based on declaration-specific
2710 // information.
2711 // FIXME: handle sseregparm someday...
2712 if (TargetDecl) {
2713 if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2714 FuncAttrs.addAttribute(Val: llvm::Attribute::ReturnsTwice);
2715 if (TargetDecl->hasAttr<NoThrowAttr>())
2716 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2717 if (TargetDecl->hasAttr<NoReturnAttr>())
2718 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2719 if (TargetDecl->hasAttr<ColdAttr>())
2720 FuncAttrs.addAttribute(Val: llvm::Attribute::Cold);
2721 if (TargetDecl->hasAttr<HotAttr>())
2722 FuncAttrs.addAttribute(Val: llvm::Attribute::Hot);
2723 if (TargetDecl->hasAttr<NoDuplicateAttr>())
2724 FuncAttrs.addAttribute(Val: llvm::Attribute::NoDuplicate);
2725 if (TargetDecl->hasAttr<ConvergentAttr>())
2726 FuncAttrs.addAttribute(Val: llvm::Attribute::Convergent);
2727
2728 if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2729 AddAttributesFromFunctionProtoType(
2730 Ctx&: getContext(), FuncAttrs, FPT: Fn->getType()->getAs<FunctionProtoType>());
2731 if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
2732 // A sane operator new returns a non-aliasing pointer.
2733 auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2734 if (getCodeGenOpts().AssumeSaneOperatorNew &&
2735 (Kind == OO_New || Kind == OO_Array_New))
2736 RetAttrs.addAttribute(Val: llvm::Attribute::NoAlias);
2737 }
2738 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: Fn);
2739 const bool IsVirtualCall = MD && MD->isVirtual();
2740 // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2741 // virtual function. These attributes are not inherited by overloads.
2742 if (!(AttrOnCallSite && IsVirtualCall)) {
2743 if (Fn->isNoReturn())
2744 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2745 NBA = Fn->getAttr<NoBuiltinAttr>();
2746 }
2747 }
2748
2749 if (isa<FunctionDecl>(Val: TargetDecl) || isa<VarDecl>(Val: TargetDecl)) {
2750 // Only place nomerge attribute on call sites, never functions. This
2751 // allows it to work on indirect virtual function calls.
2752 if (AttrOnCallSite && TargetDecl->hasAttr<NoMergeAttr>())
2753 FuncAttrs.addAttribute(Val: llvm::Attribute::NoMerge);
2754 }
2755
2756 // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2757 if (TargetDecl->hasAttr<ConstAttr>()) {
2758 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::none());
2759 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2760 // gcc specifies that 'const' functions have greater restrictions than
2761 // 'pure' functions, so they also cannot have infinite loops.
2762 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
2763 MemAttrForPtrArgs = llvm::Attribute::ReadNone;
2764 } else if (TargetDecl->hasAttr<PureAttr>()) {
2765 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::readOnly());
2766 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2767 // gcc specifies that 'pure' functions cannot have infinite loops.
2768 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
2769 MemAttrForPtrArgs = llvm::Attribute::ReadOnly;
2770 } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2771 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::inaccessibleOrArgMemOnly());
2772 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2773 }
2774 if (const auto *RA = TargetDecl->getAttr<RestrictAttr>();
2775 RA && RA->getDeallocator() == nullptr)
2776 RetAttrs.addAttribute(Val: llvm::Attribute::NoAlias);
2777 if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2778 !CodeGenOpts.NullPointerIsValid)
2779 RetAttrs.addAttribute(Val: llvm::Attribute::NonNull);
2780 if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2781 FuncAttrs.addAttribute(A: "no_caller_saved_registers");
2782 if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2783 FuncAttrs.addAttribute(Val: llvm::Attribute::NoCfCheck);
2784 if (TargetDecl->hasAttr<LeafAttr>())
2785 FuncAttrs.addAttribute(Val: llvm::Attribute::NoCallback);
2786 if (TargetDecl->hasAttr<BPFFastCallAttr>())
2787 FuncAttrs.addAttribute(A: "bpf_fastcall");
2788
2789 HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2790 if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2791 std::optional<unsigned> NumElemsParam;
2792 if (AllocSize->getNumElemsParam().isValid())
2793 NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2794 FuncAttrs.addAllocSizeAttr(ElemSizeArg: AllocSize->getElemSizeParam().getLLVMIndex(),
2795 NumElemsArg: NumElemsParam);
2796 }
2797
2798 // OpenCL v2.0 Work groups may be whether uniform or not.
2799 // '-cl-uniform-work-group-size' compile option gets a hint
2800 // to the compiler that the global work-size be a multiple of
2801 // the work-group size specified to clEnqueueNDRangeKernel
2802 // (i.e. work groups are uniform).
2803 if (getLangOpts().OffloadUniformBlock)
2804 FuncAttrs.addAttribute(A: "uniform-work-group-size");
2805
2806 if (TargetDecl->hasAttr<ArmLocallyStreamingAttr>())
2807 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_body");
2808
2809 if (auto *ModularFormat = TargetDecl->getAttr<ModularFormatAttr>()) {
2810 FormatAttr *Format = TargetDecl->getAttr<FormatAttr>();
2811 StringRef Type = Format->getType()->getName();
2812 std::string FormatIdx = std::to_string(val: Format->getFormatIdx());
2813 std::string FirstArg = std::to_string(val: Format->getFirstArg());
2814 SmallVector<StringRef> Args = {
2815 Type, FormatIdx, FirstArg,
2816 ModularFormat->getModularImplFn()->getName(),
2817 ModularFormat->getImplName()};
2818 llvm::append_range(C&: Args, R: ModularFormat->aspects());
2819 FuncAttrs.addAttribute(A: "modular-format", V: llvm::join(R&: Args, Separator: ","));
2820 }
2821 }
2822
2823 // Attach "no-builtins" attributes to:
2824 // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2825 // * definitions: "no-builtins" or "no-builtin-<name>" only.
2826 // The attributes can come from:
2827 // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2828 // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2829 addNoBuiltinAttributes(FuncAttrs, LangOpts: getLangOpts(), NBA);
2830
2831 // Collect function IR attributes based on global settiings.
2832 getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2833
2834 // Override some default IR attributes based on declaration-specific
2835 // information.
2836 if (TargetDecl) {
2837 if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2838 FuncAttrs.removeAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2839 if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2840 FuncAttrs.addAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2841 if (TargetDecl->hasAttr<NoSplitStackAttr>())
2842 FuncAttrs.removeAttribute(A: "split-stack");
2843 if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
2844 // A function "__attribute__((...))" overrides the command-line flag.
2845 auto Kind =
2846 TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2847 FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2848 FuncAttrs.addAttribute(
2849 A: "zero-call-used-regs",
2850 V: ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Val: Kind));
2851 }
2852
2853 // Add NonLazyBind attribute to function declarations when -fno-plt
2854 // is used.
2855 // FIXME: what if we just haven't processed the function definition
2856 // yet, or if it's an external definition like C99 inline?
2857 if (CodeGenOpts.NoPLT) {
2858 if (auto *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2859 if (!Fn->isDefined() && !AttrOnCallSite) {
2860 FuncAttrs.addAttribute(Val: llvm::Attribute::NonLazyBind);
2861 }
2862 }
2863 }
2864 // Remove 'convergent' if requested.
2865 if (TargetDecl->hasAttr<NoConvergentAttr>())
2866 FuncAttrs.removeAttribute(Val: llvm::Attribute::Convergent);
2867 }
2868
2869 // Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2870 // functions with -funique-internal-linkage-names.
2871 if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames) {
2872 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
2873 if (!FD->isExternallyVisible())
2874 FuncAttrs.addAttribute(A: "sample-profile-suffix-elision-policy",
2875 V: "selected");
2876 }
2877 }
2878
2879 // Collect non-call-site function IR attributes from declaration-specific
2880 // information.
2881 if (!AttrOnCallSite) {
2882 if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>())
2883 FuncAttrs.addAttribute(A: "cmse_nonsecure_entry");
2884
2885 // Whether tail calls are enabled.
2886 auto shouldDisableTailCalls = [&] {
2887 // Should this be honored in getDefaultFunctionAttributes?
2888 if (CodeGenOpts.DisableTailCalls)
2889 return true;
2890
2891 if (!TargetDecl)
2892 return false;
2893
2894 if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2895 TargetDecl->hasAttr<AnyX86InterruptAttr>())
2896 return true;
2897
2898 if (CodeGenOpts.NoEscapingBlockTailCalls) {
2899 if (const auto *BD = dyn_cast<BlockDecl>(Val: TargetDecl))
2900 if (!BD->doesNotEscape())
2901 return true;
2902 }
2903
2904 return false;
2905 };
2906 if (shouldDisableTailCalls())
2907 FuncAttrs.addAttribute(A: "disable-tail-calls", V: "true");
2908
2909 // These functions require the returns_twice attribute for correct codegen,
2910 // but the attribute may not be added if -fno-builtin is specified. We
2911 // explicitly add that attribute here.
2912 static const llvm::StringSet<> ReturnsTwiceFn{
2913 "_setjmpex", "setjmp", "_setjmp", "vfork",
2914 "sigsetjmp", "__sigsetjmp", "savectx", "getcontext"};
2915 if (ReturnsTwiceFn.contains(key: Name))
2916 FuncAttrs.addAttribute(Val: llvm::Attribute::ReturnsTwice);
2917
2918 // CPU/feature overrides. addDefaultFunctionDefinitionAttributes
2919 // handles these separately to set them based on the global defaults.
2920 GetCPUAndFeaturesAttributes(GD: CalleeInfo.getCalleeDecl(), AttrBuilder&: FuncAttrs);
2921
2922 // Windows hotpatching support
2923 if (!MSHotPatchFunctions.empty()) {
2924 bool IsHotPatched = llvm::binary_search(Range&: MSHotPatchFunctions, Value&: Name);
2925 if (IsHotPatched)
2926 FuncAttrs.addAttribute(A: "marked_for_windows_hot_patching");
2927 }
2928 }
2929
2930 // Mark functions that are replaceable by the loader.
2931 if (CodeGenOpts.isLoaderReplaceableFunctionName(FuncName: Name))
2932 FuncAttrs.addAttribute(A: "loader-replaceable");
2933
2934 // Collect attributes from arguments and return values.
2935 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2936
2937 QualType RetTy = FI.getReturnType();
2938 const ABIArgInfo &RetAI = FI.getReturnInfo();
2939 const llvm::DataLayout &DL = getDataLayout();
2940
2941 // Determine if the return type could be partially undef
2942 if (CodeGenOpts.EnableNoundefAttrs &&
2943 HasStrictReturn(Module: *this, RetTy, TargetDecl)) {
2944 if (!RetTy->isVoidType() && RetAI.getKind() != ABIArgInfo::Indirect &&
2945 DetermineNoUndef(QTy: RetTy, Types&: getTypes(), DL, AI: RetAI))
2946 RetAttrs.addAttribute(Val: llvm::Attribute::NoUndef);
2947 }
2948
2949 switch (RetAI.getKind()) {
2950 case ABIArgInfo::Extend:
2951 if (RetAI.isSignExt())
2952 RetAttrs.addAttribute(Val: llvm::Attribute::SExt);
2953 else if (RetAI.isZeroExt())
2954 RetAttrs.addAttribute(Val: llvm::Attribute::ZExt);
2955 else
2956 RetAttrs.addAttribute(Val: llvm::Attribute::NoExt);
2957 [[fallthrough]];
2958 case ABIArgInfo::TargetSpecific:
2959 case ABIArgInfo::Direct:
2960 if (RetAI.getInReg())
2961 RetAttrs.addAttribute(Val: llvm::Attribute::InReg);
2962
2963 if (canApplyNoFPClass(AI: RetAI, ParamType: RetTy, IsReturn: true))
2964 RetAttrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
2965
2966 break;
2967 case ABIArgInfo::Ignore:
2968 break;
2969
2970 case ABIArgInfo::InAlloca:
2971 case ABIArgInfo::Indirect: {
2972 // inalloca and sret disable readnone and readonly
2973 AddPotentialArgAccess();
2974 break;
2975 }
2976
2977 case ABIArgInfo::CoerceAndExpand:
2978 break;
2979
2980 case ABIArgInfo::Expand:
2981 case ABIArgInfo::IndirectAliased:
2982 llvm_unreachable("Invalid ABI kind for return argument");
2983 }
2984
2985 if (!IsThunk) {
2986 // FIXME: fix this properly, https://reviews.llvm.org/D100388
2987 if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2988 QualType PTy = RefTy->getPointeeType();
2989 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2990 RetAttrs.addDereferenceableAttr(
2991 Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
2992 if (getTypes().getTargetAddressSpace(T: PTy) == 0 &&
2993 !CodeGenOpts.NullPointerIsValid)
2994 RetAttrs.addAttribute(Val: llvm::Attribute::NonNull);
2995 if (PTy->isObjectType()) {
2996 llvm::Align Alignment =
2997 getNaturalPointeeTypeAlignment(T: RetTy).getAsAlign();
2998 RetAttrs.addAlignmentAttr(Align: Alignment);
2999 }
3000 }
3001 }
3002
3003 bool hasUsedSRet = false;
3004 SmallVector<llvm::AttrBuilder, 4> ArgAttrs;
3005 for (unsigned I = 0; I < IRFunctionArgs.totalIRArgs(); ++I)
3006 ArgAttrs.emplace_back(Args&: getLLVMContext());
3007
3008 // Attach attributes to sret.
3009 if (IRFunctionArgs.hasSRetArg()) {
3010 llvm::AttrBuilder &SRETAttrs = ArgAttrs[IRFunctionArgs.getSRetArgNo()];
3011 SRETAttrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: RetTy));
3012 SRETAttrs.addAttribute(Val: llvm::Attribute::Writable);
3013 SRETAttrs.addAttribute(Val: llvm::Attribute::DeadOnUnwind);
3014 hasUsedSRet = true;
3015 if (RetAI.getInReg())
3016 SRETAttrs.addAttribute(Val: llvm::Attribute::InReg);
3017 SRETAttrs.addAlignmentAttr(Align: RetAI.getIndirectAlign().getQuantity());
3018 }
3019
3020 // Attach attributes to inalloca argument.
3021 if (IRFunctionArgs.hasInallocaArg()) {
3022 ArgAttrs[IRFunctionArgs.getInallocaArgNo()].addInAllocaAttr(
3023 Ty: FI.getArgStruct());
3024 }
3025
3026 // Apply `nonnull`, `dereferenceable(N)` and `align N` to the `this` argument,
3027 // unless this is a thunk function. Add dead_on_return to the `this` argument
3028 // in base class destructors to aid in DSE.
3029 // FIXME: fix this properly, https://reviews.llvm.org/D100388
3030 if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() &&
3031 !FI.arg_begin()->type->isVoidPointerType() && !IsThunk) {
3032 auto IRArgs = IRFunctionArgs.getIRArgs(ArgNo: 0);
3033
3034 assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
3035
3036 llvm::AttrBuilder &Attrs = ArgAttrs[IRArgs.first];
3037
3038 QualType ThisTy = FI.arg_begin()->type.getTypePtr()->getPointeeType();
3039 int64_t ThisSz = getMinimumObjectSize(Ty: ThisTy).getQuantity();
3040
3041 if (!CodeGenOpts.NullPointerIsValid &&
3042 getTypes().getTargetAddressSpace(T: FI.arg_begin()->type) == 0) {
3043 Attrs.addAttribute(Val: llvm::Attribute::NonNull);
3044 Attrs.addDereferenceableAttr(Bytes: ThisSz);
3045 } else {
3046 // FIXME dereferenceable should be correct here, regardless of
3047 // NullPointerIsValid. However, dereferenceable currently does not always
3048 // respect NullPointerIsValid and may imply nonnull and break the program.
3049 // See https://reviews.llvm.org/D66618 for discussions.
3050 Attrs.addDereferenceableOrNullAttr(Bytes: ThisSz);
3051 }
3052
3053 llvm::Align Alignment =
3054 getNaturalTypeAlignment(T: ThisTy, /*BaseInfo=*/nullptr,
3055 /*TBAAInfo=*/nullptr, /*forPointeeType=*/true)
3056 .getAsAlign();
3057 Attrs.addAlignmentAttr(Align: Alignment);
3058
3059 const auto *DD = dyn_cast_if_present<CXXDestructorDecl>(
3060 Val: CalleeInfo.getCalleeDecl().getDecl());
3061 // Do not annotate vector deleting destructors with dead_on_return as the
3062 // this pointer in that case points to an array which we cannot
3063 // statically know the size of. Also do not mark deleting destructors
3064 // dead_on_return as then we might delete stores inside of a user-defined
3065 // operator delete implementation if it gets inlined, which would be
3066 // incorrect as the object's lifetime has already ended and the operator
3067 // delete implementation is allowed to manipulate the underlying storage.
3068 if (DD &&
3069 CalleeInfo.getCalleeDecl().getDtorType() !=
3070 CXXDtorType::Dtor_VectorDeleting &&
3071 CalleeInfo.getCalleeDecl().getDtorType() !=
3072 CXXDtorType::Dtor_Deleting &&
3073 CodeGenOpts.StrictLifetimes) {
3074 const CXXRecordDecl *ClassDecl =
3075 dyn_cast<CXXRecordDecl>(Val: DD->getDeclContext());
3076 // We cannot add dead_on_return if we have virtual base classes because
3077 // they will generally still be live after the base object destructor.
3078 if (ClassDecl->getNumVBases() == 0)
3079 Attrs.addDeadOnReturnAttr(Info: llvm::DeadOnReturnInfo(
3080 Context.getASTRecordLayout(D: ClassDecl).getDataSize().getQuantity()));
3081 }
3082 }
3083
3084 unsigned ArgNo = 0;
3085 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(), E = FI.arg_end();
3086 I != E; ++I, ++ArgNo) {
3087 QualType ParamType = I->type;
3088 const ABIArgInfo &AI = I->info;
3089 llvm::AttrBuilder Attrs(getLLVMContext());
3090
3091 // Add attribute for padding argument, if necessary.
3092 if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
3093 if (AI.getPaddingInReg()) {
3094 ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)].addAttribute(
3095 Val: llvm::Attribute::InReg);
3096 }
3097 }
3098
3099 // Decide whether the argument we're handling could be partially undef
3100 if (CodeGenOpts.EnableNoundefAttrs &&
3101 DetermineNoUndef(QTy: ParamType, Types&: getTypes(), DL, AI)) {
3102 Attrs.addAttribute(Val: llvm::Attribute::NoUndef);
3103 }
3104
3105 // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
3106 // have the corresponding parameter variable. It doesn't make
3107 // sense to do it here because parameters are so messed up.
3108 switch (AI.getKind()) {
3109 case ABIArgInfo::Extend:
3110 if (AI.isSignExt())
3111 Attrs.addAttribute(Val: llvm::Attribute::SExt);
3112 else if (AI.isZeroExt())
3113 Attrs.addAttribute(Val: llvm::Attribute::ZExt);
3114 else
3115 Attrs.addAttribute(Val: llvm::Attribute::NoExt);
3116 [[fallthrough]];
3117 case ABIArgInfo::TargetSpecific:
3118 case ABIArgInfo::Direct:
3119 if (ArgNo == 0 && FI.isChainCall())
3120 Attrs.addAttribute(Val: llvm::Attribute::Nest);
3121 else if (AI.getInReg())
3122 Attrs.addAttribute(Val: llvm::Attribute::InReg);
3123 Attrs.addStackAlignmentAttr(Align: llvm::MaybeAlign(AI.getDirectAlign()));
3124
3125 if (canApplyNoFPClass(AI, ParamType, IsReturn: false))
3126 Attrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
3127 break;
3128 case ABIArgInfo::Indirect: {
3129 if (AI.getInReg())
3130 Attrs.addAttribute(Val: llvm::Attribute::InReg);
3131
3132 // HLSL out and inout parameters must not be marked with ByVal or
3133 // DeadOnReturn attributes because stores to these parameters by the
3134 // callee are visible to the caller.
3135 if (auto ParamABI = FI.getExtParameterInfo(argIndex: ArgNo).getABI();
3136 ParamABI != ParameterABI::HLSLOut &&
3137 ParamABI != ParameterABI::HLSLInOut) {
3138
3139 // Depending on the ABI, this may be either a byval or a dead_on_return
3140 // argument.
3141 if (AI.getIndirectByVal()) {
3142 Attrs.addByValAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
3143 } else {
3144 // Add dead_on_return when the object's lifetime ends in the callee.
3145 // This includes trivially-destructible objects, as well as objects
3146 // whose destruction / clean-up is carried out within the callee
3147 // (e.g., Obj-C ARC-managed structs, MSVC callee-destroyed objects).
3148 if (!ParamType.isDestructedType() || !ParamType->isRecordType() ||
3149 ParamType->castAsRecordDecl()->isParamDestroyedInCallee())
3150 Attrs.addDeadOnReturnAttr(Info: llvm::DeadOnReturnInfo());
3151 }
3152 }
3153
3154 auto *Decl = ParamType->getAsRecordDecl();
3155 if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
3156 Decl->getArgPassingRestrictions() ==
3157 RecordArgPassingKind::CanPassInRegs)
3158 // When calling the function, the pointer passed in will be the only
3159 // reference to the underlying object. Mark it accordingly.
3160 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
3161
3162 // TODO: We could add the byref attribute if not byval, but it would
3163 // require updating many testcases.
3164
3165 CharUnits Align = AI.getIndirectAlign();
3166
3167 // In a byval argument, it is important that the required
3168 // alignment of the type is honored, as LLVM might be creating a
3169 // *new* stack object, and needs to know what alignment to give
3170 // it. (Sometimes it can deduce a sensible alignment on its own,
3171 // but not if clang decides it must emit a packed struct, or the
3172 // user specifies increased alignment requirements.)
3173 //
3174 // This is different from indirect *not* byval, where the object
3175 // exists already, and the align attribute is purely
3176 // informative.
3177 assert(!Align.isZero());
3178
3179 // For now, only add this when we have a byval argument.
3180 // TODO: be less lazy about updating test cases.
3181 if (AI.getIndirectByVal())
3182 Attrs.addAlignmentAttr(Align: Align.getQuantity());
3183
3184 // byval disables readnone and readonly.
3185 AddPotentialArgAccess();
3186 break;
3187 }
3188 case ABIArgInfo::IndirectAliased: {
3189 CharUnits Align = AI.getIndirectAlign();
3190 Attrs.addByRefAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
3191 Attrs.addAlignmentAttr(Align: Align.getQuantity());
3192 break;
3193 }
3194 case ABIArgInfo::Ignore:
3195 case ABIArgInfo::Expand:
3196 case ABIArgInfo::CoerceAndExpand:
3197 break;
3198
3199 case ABIArgInfo::InAlloca:
3200 // inalloca disables readnone and readonly.
3201 AddPotentialArgAccess();
3202 continue;
3203 }
3204
3205 if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
3206 QualType PTy = RefTy->getPointeeType();
3207 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
3208 Attrs.addDereferenceableAttr(Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
3209 if (getTypes().getTargetAddressSpace(T: PTy) == 0 &&
3210 !CodeGenOpts.NullPointerIsValid)
3211 Attrs.addAttribute(Val: llvm::Attribute::NonNull);
3212 if (PTy->isObjectType()) {
3213 llvm::Align Alignment =
3214 getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
3215 Attrs.addAlignmentAttr(Align: Alignment);
3216 }
3217 }
3218
3219 // From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:
3220 // > For arguments to a __kernel function declared to be a pointer to a
3221 // > data type, the OpenCL compiler can assume that the pointee is always
3222 // > appropriately aligned as required by the data type.
3223 if (TargetDecl &&
3224 DeviceKernelAttr::isOpenCLSpelling(
3225 A: TargetDecl->getAttr<DeviceKernelAttr>()) &&
3226 ParamType->isPointerType()) {
3227 QualType PTy = ParamType->getPointeeType();
3228 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
3229 llvm::Align Alignment =
3230 getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
3231 Attrs.addAlignmentAttr(Align: Alignment);
3232 }
3233 }
3234
3235 switch (FI.getExtParameterInfo(argIndex: ArgNo).getABI()) {
3236 case ParameterABI::HLSLOut:
3237 case ParameterABI::HLSLInOut:
3238 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
3239 break;
3240 case ParameterABI::Ordinary:
3241 break;
3242
3243 case ParameterABI::SwiftIndirectResult: {
3244 // Add 'sret' if we haven't already used it for something, but
3245 // only if the result is void.
3246 if (!hasUsedSRet && RetTy->isVoidType()) {
3247 Attrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
3248 hasUsedSRet = true;
3249 }
3250
3251 // Add 'noalias' in either case.
3252 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
3253
3254 // Add 'dereferenceable' and 'alignment'.
3255 auto PTy = ParamType->getPointeeType();
3256 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
3257 auto info = getContext().getTypeInfoInChars(T: PTy);
3258 Attrs.addDereferenceableAttr(Bytes: info.Width.getQuantity());
3259 Attrs.addAlignmentAttr(Align: info.Align.getAsAlign());
3260 }
3261 break;
3262 }
3263
3264 case ParameterABI::SwiftErrorResult:
3265 Attrs.addAttribute(Val: llvm::Attribute::SwiftError);
3266 break;
3267
3268 case ParameterABI::SwiftContext:
3269 Attrs.addAttribute(Val: llvm::Attribute::SwiftSelf);
3270 break;
3271
3272 case ParameterABI::SwiftAsyncContext:
3273 Attrs.addAttribute(Val: llvm::Attribute::SwiftAsync);
3274 break;
3275 }
3276
3277 if (FI.getExtParameterInfo(argIndex: ArgNo).isNoEscape())
3278 Attrs.addCapturesAttr(
3279 CI: llvm::CaptureInfo(llvm::CaptureComponents::Address));
3280
3281 if (Attrs.hasAttributes()) {
3282 unsigned FirstIRArg, NumIRArgs;
3283 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3284 for (unsigned i = 0; i < NumIRArgs; i++)
3285 ArgAttrs[FirstIRArg + i].merge(B: Attrs);
3286 }
3287 }
3288 assert(ArgNo == FI.arg_size());
3289
3290 // We can't see all potential arguments in a varargs declaration; treat them
3291 // as if they can access memory.
3292 if (!AttrOnCallSite && FI.isVariadic())
3293 AddPotentialArgAccess();
3294
3295 ArgNo = 0;
3296 if (AddedPotentialArgAccess && MemAttrForPtrArgs) {
3297 llvm::FunctionType *FunctionType = getTypes().GetFunctionType(FI);
3298 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
3299 E = FI.arg_end();
3300 I != E; ++I, ++ArgNo) {
3301 if (I->info.isDirect() || I->info.isExpand() ||
3302 I->info.isCoerceAndExpand()) {
3303 unsigned FirstIRArg, NumIRArgs;
3304 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3305 for (unsigned i = FirstIRArg; i < FirstIRArg + NumIRArgs; ++i) {
3306 // The index may be out-of-bounds if the callee is a varargs
3307 // function.
3308 //
3309 // FIXME: We can compute the types of varargs arguments without going
3310 // through the function type, but the relevant code isn't exposed
3311 // in a way that can be called from here.
3312 if (i < FunctionType->getNumParams() &&
3313 FunctionType->getParamType(i)->isPointerTy()) {
3314 ArgAttrs[i].addAttribute(Val: *MemAttrForPtrArgs);
3315 }
3316 }
3317 }
3318 }
3319 }
3320
3321 SmallVector<llvm::AttributeSet, 4> ArgAttrSets;
3322 for (const llvm::AttrBuilder &Attrs : ArgAttrs)
3323 ArgAttrSets.push_back(Elt: llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs));
3324
3325 AttrList = llvm::AttributeList::get(
3326 C&: getLLVMContext(), FnAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: FuncAttrs),
3327 RetAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: RetAttrs), ArgAttrs: ArgAttrSets);
3328}
3329
3330/// An argument came in as a promoted argument; demote it back to its
3331/// declared type.
3332static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
3333 const VarDecl *var,
3334 llvm::Value *value) {
3335 llvm::Type *varType = CGF.ConvertType(T: var->getType());
3336
3337 // This can happen with promotions that actually don't change the
3338 // underlying type, like the enum promotions.
3339 if (value->getType() == varType)
3340 return value;
3341
3342 assert((varType->isIntegerTy() || varType->isFloatingPointTy()) &&
3343 "unexpected promotion type");
3344
3345 if (isa<llvm::IntegerType>(Val: varType))
3346 return CGF.Builder.CreateTrunc(V: value, DestTy: varType, Name: "arg.unpromote");
3347
3348 return CGF.Builder.CreateFPCast(V: value, DestTy: varType, Name: "arg.unpromote");
3349}
3350
3351/// Returns the attribute (either parameter attribute, or function
3352/// attribute), which declares argument ArgNo to be non-null.
3353static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
3354 QualType ArgType, unsigned ArgNo) {
3355 // FIXME: __attribute__((nonnull)) can also be applied to:
3356 // - references to pointers, where the pointee is known to be
3357 // nonnull (apparently a Clang extension)
3358 // - transparent unions containing pointers
3359 // In the former case, LLVM IR cannot represent the constraint. In
3360 // the latter case, we have no guarantee that the transparent union
3361 // is in fact passed as a pointer.
3362 if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType())
3363 return nullptr;
3364 // First, check attribute on parameter itself.
3365 if (PVD) {
3366 if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
3367 return ParmNNAttr;
3368 }
3369 // Check function attributes.
3370 if (!FD)
3371 return nullptr;
3372 for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
3373 if (NNAttr->isNonNull(IdxAST: ArgNo))
3374 return NNAttr;
3375 }
3376 return nullptr;
3377}
3378
3379namespace {
3380struct CopyBackSwiftError final : EHScopeStack::Cleanup {
3381 Address Temp;
3382 Address Arg;
3383 CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
3384 void Emit(CodeGenFunction &CGF, Flags flags) override {
3385 llvm::Value *errorValue = CGF.Builder.CreateLoad(Addr: Temp);
3386 CGF.Builder.CreateStore(Val: errorValue, Addr: Arg);
3387 }
3388};
3389} // namespace
3390
3391void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
3392 llvm::Function *Fn,
3393 const FunctionArgList &Args) {
3394 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())
3395 // Naked functions don't have prologues.
3396 return;
3397
3398 // If this is an implicit-return-zero function, go ahead and
3399 // initialize the return value. TODO: it might be nice to have
3400 // a more general mechanism for this that didn't require synthesized
3401 // return statements.
3402 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl)) {
3403 if (FD->hasImplicitReturnZero()) {
3404 QualType RetTy = FD->getReturnType().getUnqualifiedType();
3405 llvm::Type *LLVMTy = CGM.getTypes().ConvertType(T: RetTy);
3406 llvm::Constant *Zero = llvm::Constant::getNullValue(Ty: LLVMTy);
3407 Builder.CreateStore(Val: Zero, Addr: ReturnValue);
3408 }
3409 }
3410
3411 // FIXME: We no longer need the types from FunctionArgList; lift up and
3412 // simplify.
3413
3414 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
3415 assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
3416
3417 // If we're using inalloca, all the memory arguments are GEPs off of the last
3418 // parameter, which is a pointer to the complete memory area.
3419 Address ArgStruct = Address::invalid();
3420 if (IRFunctionArgs.hasInallocaArg())
3421 ArgStruct = Address(Fn->getArg(i: IRFunctionArgs.getInallocaArgNo()),
3422 FI.getArgStruct(), FI.getArgStructAlignment());
3423
3424 // Name the struct return parameter.
3425 if (IRFunctionArgs.hasSRetArg()) {
3426 auto AI = Fn->getArg(i: IRFunctionArgs.getSRetArgNo());
3427 AI->setName("agg.result");
3428 AI->addAttr(Kind: llvm::Attribute::NoAlias);
3429 }
3430
3431 // Track if we received the parameter as a pointer (indirect, byval, or
3432 // inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it
3433 // into a local alloca for us.
3434 SmallVector<ParamValue, 16> ArgVals;
3435 ArgVals.reserve(N: Args.size());
3436
3437 // Create a pointer value for every parameter declaration. This usually
3438 // entails copying one or more LLVM IR arguments into an alloca. Don't push
3439 // any cleanups or do anything that might unwind. We do that separately, so
3440 // we can push the cleanups in the correct order for the ABI.
3441 assert(FI.arg_size() == Args.size() &&
3442 "Mismatch between function signature & arguments.");
3443 unsigned ArgNo = 0;
3444 CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
3445 for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end(); i != e;
3446 ++i, ++info_it, ++ArgNo) {
3447 const VarDecl *Arg = *i;
3448 const ABIArgInfo &ArgI = info_it->info;
3449
3450 bool isPromoted =
3451 isa<ParmVarDecl>(Val: Arg) && cast<ParmVarDecl>(Val: Arg)->isKNRPromoted();
3452 // We are converting from ABIArgInfo type to VarDecl type directly, unless
3453 // the parameter is promoted. In this case we convert to
3454 // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
3455 QualType Ty = isPromoted ? info_it->type : Arg->getType();
3456 assert(hasScalarEvaluationKind(Ty) ==
3457 hasScalarEvaluationKind(Arg->getType()));
3458
3459 unsigned FirstIRArg, NumIRArgs;
3460 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3461
3462 switch (ArgI.getKind()) {
3463 case ABIArgInfo::InAlloca: {
3464 assert(NumIRArgs == 0);
3465 auto FieldIndex = ArgI.getInAllocaFieldIndex();
3466 Address V =
3467 Builder.CreateStructGEP(Addr: ArgStruct, Index: FieldIndex, Name: Arg->getName());
3468 if (ArgI.getInAllocaIndirect())
3469 V = Address(Builder.CreateLoad(Addr: V), ConvertTypeForMem(T: Ty),
3470 getContext().getTypeAlignInChars(T: Ty));
3471 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: V));
3472 break;
3473 }
3474
3475 case ABIArgInfo::Indirect:
3476 case ABIArgInfo::IndirectAliased: {
3477 assert(NumIRArgs == 1);
3478 Address ParamAddr = makeNaturalAddressForPointer(
3479 Ptr: Fn->getArg(i: FirstIRArg), T: Ty, Alignment: ArgI.getIndirectAlign(), ForPointeeType: false, BaseInfo: nullptr,
3480 TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
3481
3482 if (!hasScalarEvaluationKind(T: Ty)) {
3483 // Aggregates and complex variables are accessed by reference. All we
3484 // need to do is realign the value, if requested. Also, if the address
3485 // may be aliased, copy it to ensure that the parameter variable is
3486 // mutable and has a unique adress, as C requires.
3487 if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) {
3488 RawAddress AlignedTemp = CreateMemTempWithoutCast(T: Ty, Name: "coerce");
3489
3490 // Copy from the incoming argument pointer to the temporary with the
3491 // appropriate alignment.
3492 //
3493 // FIXME: We should have a common utility for generating an aggregate
3494 // copy.
3495 CharUnits Size = getContext().getTypeSizeInChars(T: Ty);
3496 Builder.CreateMemCpy(
3497 Dst: AlignedTemp.getPointer(), DstAlign: AlignedTemp.getAlignment().getAsAlign(),
3498 Src: ParamAddr.emitRawPointer(CGF&: *this),
3499 SrcAlign: ParamAddr.getAlignment().getAsAlign(),
3500 Size: llvm::ConstantInt::get(Ty: IntPtrTy, V: Size.getQuantity()));
3501 ParamAddr = AlignedTemp;
3502 }
3503 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: ParamAddr));
3504 } else {
3505 // Load scalar value from indirect argument.
3506 llvm::Value *V =
3507 EmitLoadOfScalar(Addr: ParamAddr, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3508
3509 if (isPromoted)
3510 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3511 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3512 }
3513 break;
3514 }
3515
3516 case ABIArgInfo::Extend:
3517 case ABIArgInfo::Direct: {
3518 auto AI = Fn->getArg(i: FirstIRArg);
3519 llvm::Type *LTy = ConvertType(T: Arg->getType());
3520
3521 // Prepare parameter attributes. So far, only attributes for pointer
3522 // parameters are prepared. See
3523 // http://llvm.org/docs/LangRef.html#paramattrs.
3524 if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy() &&
3525 ArgI.getCoerceToType()->isPointerTy()) {
3526 assert(NumIRArgs == 1);
3527
3528 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: Arg)) {
3529 // Set `nonnull` attribute if any.
3530 if (getNonNullAttr(FD: CurCodeDecl, PVD, ArgType: PVD->getType(),
3531 ArgNo: PVD->getFunctionScopeIndex()) &&
3532 !CGM.getCodeGenOpts().NullPointerIsValid)
3533 AI->addAttr(Kind: llvm::Attribute::NonNull);
3534
3535 QualType OTy = PVD->getOriginalType();
3536 if (const auto *ArrTy = getContext().getAsConstantArrayType(T: OTy)) {
3537 // A C99 array parameter declaration with the static keyword also
3538 // indicates dereferenceability, and if the size is constant we can
3539 // use the dereferenceable attribute (which requires the size in
3540 // bytes).
3541 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3542 QualType ETy = ArrTy->getElementType();
3543 llvm::Align Alignment =
3544 CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3545 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3546 .addAlignmentAttr(Align: Alignment));
3547 uint64_t ArrSize = ArrTy->getZExtSize();
3548 if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
3549 ArrSize) {
3550 llvm::AttrBuilder Attrs(getLLVMContext());
3551 Attrs.addDereferenceableAttr(
3552 Bytes: getContext().getTypeSizeInChars(T: ETy).getQuantity() *
3553 ArrSize);
3554 AI->addAttrs(B&: Attrs);
3555 } else if (getContext().getTargetInfo().getNullPointerValue(
3556 AddrSpace: ETy.getAddressSpace()) == 0 &&
3557 !CGM.getCodeGenOpts().NullPointerIsValid) {
3558 AI->addAttr(Kind: llvm::Attribute::NonNull);
3559 }
3560 }
3561 } else if (const auto *ArrTy =
3562 getContext().getAsVariableArrayType(T: OTy)) {
3563 // For C99 VLAs with the static keyword, we don't know the size so
3564 // we can't use the dereferenceable attribute, but in addrspace(0)
3565 // we know that it must be nonnull.
3566 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3567 QualType ETy = ArrTy->getElementType();
3568 llvm::Align Alignment =
3569 CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3570 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3571 .addAlignmentAttr(Align: Alignment));
3572 if (!getTypes().getTargetAddressSpace(T: ETy) &&
3573 !CGM.getCodeGenOpts().NullPointerIsValid)
3574 AI->addAttr(Kind: llvm::Attribute::NonNull);
3575 }
3576 }
3577
3578 // Set `align` attribute if any.
3579 const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
3580 if (!AVAttr)
3581 if (const auto *TOTy = OTy->getAs<TypedefType>())
3582 AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
3583 if (AVAttr && !SanOpts.has(K: SanitizerKind::Alignment)) {
3584 // If alignment-assumption sanitizer is enabled, we do *not* add
3585 // alignment attribute here, but emit normal alignment assumption,
3586 // so the UBSAN check could function.
3587 llvm::ConstantInt *AlignmentCI =
3588 cast<llvm::ConstantInt>(Val: EmitScalarExpr(E: AVAttr->getAlignment()));
3589 uint64_t AlignmentInt =
3590 AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment);
3591 if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
3592 AI->removeAttr(Kind: llvm::Attribute::AttrKind::Alignment);
3593 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3594 .addAlignmentAttr(Align: llvm::Align(AlignmentInt)));
3595 }
3596 }
3597 }
3598
3599 // Set 'noalias' if an argument type has the `restrict` qualifier.
3600 if (Arg->getType().isRestrictQualified())
3601 AI->addAttr(Kind: llvm::Attribute::NoAlias);
3602 }
3603
3604 // Prepare the argument value. If we have the trivial case, handle it
3605 // with no muss and fuss.
3606 if (!isa<llvm::StructType>(Val: ArgI.getCoerceToType()) &&
3607 ArgI.getCoerceToType() == ConvertType(T: Ty) &&
3608 ArgI.getDirectOffset() == 0) {
3609 assert(NumIRArgs == 1);
3610
3611 // LLVM expects swifterror parameters to be used in very restricted
3612 // ways. Copy the value into a less-restricted temporary.
3613 llvm::Value *V = AI;
3614 if (FI.getExtParameterInfo(argIndex: ArgNo).getABI() ==
3615 ParameterABI::SwiftErrorResult) {
3616 QualType pointeeTy = Ty->getPointeeType();
3617 assert(pointeeTy->isPointerType());
3618 RawAddress temp = CreateMemTempWithoutCast(
3619 T: pointeeTy, Align: getPointerAlign(), Name: "swifterror.temp");
3620 Address arg = makeNaturalAddressForPointer(
3621 Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
3622 llvm::Value *incomingErrorValue = Builder.CreateLoad(Addr: arg);
3623 Builder.CreateStore(Val: incomingErrorValue, Addr: temp);
3624 V = temp.getPointer();
3625
3626 // Push a cleanup to copy the value back at the end of the function.
3627 // The convention does not guarantee that the value will be written
3628 // back if the function exits with an unwind exception.
3629 EHStack.pushCleanup<CopyBackSwiftError>(Kind: NormalCleanup, A: temp, A: arg);
3630 }
3631
3632 // Ensure the argument is the correct type.
3633 if (V->getType() != ArgI.getCoerceToType())
3634 V = Builder.CreateBitCast(V, DestTy: ArgI.getCoerceToType());
3635
3636 if (isPromoted)
3637 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3638
3639 // Because of merging of function types from multiple decls it is
3640 // possible for the type of an argument to not match the corresponding
3641 // type in the function type. Since we are codegening the callee
3642 // in here, add a cast to the argument type.
3643 llvm::Type *LTy = ConvertType(T: Arg->getType());
3644 if (V->getType() != LTy)
3645 V = Builder.CreateBitCast(V, DestTy: LTy);
3646
3647 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3648 break;
3649 }
3650
3651 // VLST arguments are coerced to VLATs at the function boundary for
3652 // ABI consistency. If this is a VLST that was coerced to
3653 // a VLAT at the function boundary and the types match up, use
3654 // llvm.vector.extract to convert back to the original VLST.
3655 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: ConvertType(T: Ty))) {
3656 llvm::Value *ArgVal = Fn->getArg(i: FirstIRArg);
3657 if (auto *VecTyFrom =
3658 dyn_cast<llvm::ScalableVectorType>(Val: ArgVal->getType())) {
3659 auto [Coerced, Extracted] = CoerceScalableToFixed(
3660 CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: ArgVal, Name: Arg->getName());
3661 if (Extracted) {
3662 assert(NumIRArgs == 1);
3663 ArgVals.push_back(Elt: ParamValue::forDirect(value: Coerced));
3664 break;
3665 }
3666 }
3667 }
3668
3669 llvm::StructType *STy =
3670 dyn_cast<llvm::StructType>(Val: ArgI.getCoerceToType());
3671 Address Alloca = CreateMemTempWithoutCast(
3672 T: Ty, Align: getContext().getDeclAlign(D: Arg), Name: Arg->getName());
3673
3674 // Pointer to store into.
3675 Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3676
3677 // Fast-isel and the optimizer generally like scalar values better than
3678 // FCAs, so we flatten them if this is safe to do for this argument.
3679 if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
3680 STy->getNumElements() > 1) {
3681 llvm::TypeSize StructSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
3682 llvm::TypeSize PtrElementSize =
3683 CGM.getDataLayout().getTypeAllocSize(Ty: Ptr.getElementType());
3684 if (StructSize.isScalable()) {
3685 assert(STy->containsHomogeneousScalableVectorTypes() &&
3686 "ABI only supports structure with homogeneous scalable vector "
3687 "type");
3688 assert(StructSize == PtrElementSize &&
3689 "Only allow non-fractional movement of structure with"
3690 "homogeneous scalable vector type");
3691 assert(STy->getNumElements() == NumIRArgs);
3692
3693 llvm::Value *LoadedStructValue = llvm::PoisonValue::get(T: STy);
3694 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3695 auto *AI = Fn->getArg(i: FirstIRArg + i);
3696 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3697 LoadedStructValue =
3698 Builder.CreateInsertValue(Agg: LoadedStructValue, Val: AI, Idxs: i);
3699 }
3700
3701 Builder.CreateStore(Val: LoadedStructValue, Addr: Ptr);
3702 } else {
3703 uint64_t SrcSize = StructSize.getFixedValue();
3704 uint64_t DstSize = PtrElementSize.getFixedValue();
3705
3706 Address AddrToStoreInto = Address::invalid();
3707 if (SrcSize <= DstSize) {
3708 AddrToStoreInto = Ptr.withElementType(ElemTy: STy);
3709 } else {
3710 AddrToStoreInto =
3711 CreateTempAlloca(Ty: STy, align: Alloca.getAlignment(), Name: "coerce");
3712 }
3713
3714 assert(STy->getNumElements() == NumIRArgs);
3715 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3716 auto AI = Fn->getArg(i: FirstIRArg + i);
3717 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3718 Address EltPtr = Builder.CreateStructGEP(Addr: AddrToStoreInto, Index: i);
3719 Builder.CreateStore(Val: AI, Addr: EltPtr);
3720 }
3721
3722 if (SrcSize > DstSize) {
3723 Builder.CreateMemCpy(Dest: Ptr, Src: AddrToStoreInto, Size: DstSize);
3724 }
3725
3726 // Structures with PFP fields require a coerced store to add any
3727 // pointer signatures.
3728 if (getContext().hasPFPFields(Ty)) {
3729 llvm::Value *Struct = Builder.CreateLoad(Addr: Ptr);
3730 CreatePFPCoercedStore(Src: Struct, SrcFETy: Ty, Dst: Ptr, CGF&: *this);
3731 }
3732 }
3733 } else {
3734 // Simple case, just do a coerced store of the argument into the alloca.
3735 assert(NumIRArgs == 1);
3736 auto AI = Fn->getArg(i: FirstIRArg);
3737 AI->setName(Arg->getName() + ".coerce");
3738 CreateCoercedStore(
3739 Src: AI, SrcFETy: Ty, Dst: Ptr,
3740 DstSize: llvm::TypeSize::getFixed(
3741 ExactSize: getContext().getTypeSizeInChars(T: Ty).getQuantity() -
3742 ArgI.getDirectOffset()),
3743 /*DstIsVolatile=*/false);
3744 }
3745
3746 // Match to what EmitParmDecl is expecting for this type.
3747 if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3748 llvm::Value *V =
3749 EmitLoadOfScalar(Addr: Alloca, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3750 if (isPromoted)
3751 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3752 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3753 } else {
3754 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3755 }
3756 break;
3757 }
3758
3759 case ABIArgInfo::CoerceAndExpand: {
3760 // Reconstruct into a temporary.
3761 Address alloca =
3762 CreateMemTempWithoutCast(T: Ty, Align: getContext().getDeclAlign(D: Arg));
3763 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: alloca));
3764
3765 auto coercionType = ArgI.getCoerceAndExpandType();
3766 auto unpaddedCoercionType = ArgI.getUnpaddedCoerceAndExpandType();
3767 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
3768
3769 alloca = alloca.withElementType(ElemTy: coercionType);
3770
3771 unsigned argIndex = FirstIRArg;
3772 unsigned unpaddedIndex = 0;
3773 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3774 llvm::Type *eltType = coercionType->getElementType(N: i);
3775 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
3776 continue;
3777
3778 auto eltAddr = Builder.CreateStructGEP(Addr: alloca, Index: i);
3779 llvm::Value *elt = Fn->getArg(i: argIndex++);
3780
3781 auto paramType = unpaddedStruct
3782 ? unpaddedStruct->getElementType(N: unpaddedIndex++)
3783 : unpaddedCoercionType;
3784
3785 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: eltType)) {
3786 if (auto *VecTyFrom = dyn_cast<llvm::ScalableVectorType>(Val: paramType)) {
3787 bool Extracted;
3788 std::tie(args&: elt, args&: Extracted) = CoerceScalableToFixed(
3789 CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: elt, Name: elt->getName());
3790 assert(Extracted && "Unexpected scalable to fixed vector coercion");
3791 }
3792 }
3793 Builder.CreateStore(Val: elt, Addr: eltAddr);
3794 }
3795 assert(argIndex == FirstIRArg + NumIRArgs);
3796 break;
3797 }
3798
3799 case ABIArgInfo::Expand: {
3800 // If this structure was expanded into multiple arguments then
3801 // we need to create a temporary and reconstruct it from the
3802 // arguments.
3803 Address Alloca =
3804 CreateMemTempWithoutCast(T: Ty, Align: getContext().getDeclAlign(D: Arg));
3805 LValue LV = MakeAddrLValue(Addr: Alloca, T: Ty);
3806 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3807
3808 auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3809 ExpandTypeFromArgs(Ty, LV, AI&: FnArgIter);
3810 assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3811 for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
3812 auto AI = Fn->getArg(i: FirstIRArg + i);
3813 AI->setName(Arg->getName() + "." + Twine(i));
3814 }
3815 break;
3816 }
3817
3818 case ABIArgInfo::TargetSpecific: {
3819 auto *AI = Fn->getArg(i: FirstIRArg);
3820 AI->setName(Arg->getName() + ".target_coerce");
3821 Address Alloca = CreateMemTempWithoutCast(
3822 T: Ty, Align: getContext().getDeclAlign(D: Arg), Name: Arg->getName());
3823 Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3824 CGM.getABIInfo().createCoercedStore(Val: AI, DstAddr: Ptr, AI: ArgI, DestIsVolatile: false, CGF&: *this);
3825 if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3826 llvm::Value *V =
3827 EmitLoadOfScalar(Addr: Alloca, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3828 if (isPromoted) {
3829 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3830 }
3831 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3832 } else {
3833 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3834 }
3835 break;
3836 }
3837 case ABIArgInfo::Ignore:
3838 assert(NumIRArgs == 0);
3839 // Initialize the local variable appropriately.
3840 if (!hasScalarEvaluationKind(T: Ty)) {
3841 ArgVals.push_back(
3842 Elt: ParamValue::forIndirect(addr: CreateMemTempWithoutCast(T: Ty)));
3843 } else {
3844 llvm::Value *U = llvm::UndefValue::get(T: ConvertType(T: Arg->getType()));
3845 ArgVals.push_back(Elt: ParamValue::forDirect(value: U));
3846 }
3847 break;
3848 }
3849 }
3850
3851 if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3852 for (int I = Args.size() - 1; I >= 0; --I)
3853 EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3854 } else {
3855 for (unsigned I = 0, E = Args.size(); I != E; ++I)
3856 EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3857 }
3858}
3859
3860static void eraseUnusedBitCasts(llvm::Instruction *insn) {
3861 while (insn->use_empty()) {
3862 llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: insn);
3863 if (!bitcast)
3864 return;
3865
3866 // This is "safe" because we would have used a ConstantExpr otherwise.
3867 insn = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3868 bitcast->eraseFromParent();
3869 }
3870}
3871
3872/// Try to emit a fused autorelease of a return result.
3873static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
3874 llvm::Value *result) {
3875 // We must be immediately followed the cast.
3876 llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3877 if (BB->empty())
3878 return nullptr;
3879 if (&BB->back() != result)
3880 return nullptr;
3881
3882 llvm::Type *resultType = result->getType();
3883
3884 // result is in a BasicBlock and is therefore an Instruction.
3885 llvm::Instruction *generator = cast<llvm::Instruction>(Val: result);
3886
3887 SmallVector<llvm::Instruction *, 4> InstsToKill;
3888
3889 // Look for:
3890 // %generator = bitcast %type1* %generator2 to %type2*
3891 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: generator)) {
3892 // We would have emitted this as a constant if the operand weren't
3893 // an Instruction.
3894 generator = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3895
3896 // Require the generator to be immediately followed by the cast.
3897 if (generator->getNextNode() != bitcast)
3898 return nullptr;
3899
3900 InstsToKill.push_back(Elt: bitcast);
3901 }
3902
3903 // Look for:
3904 // %generator = call i8* @objc_retain(i8* %originalResult)
3905 // or
3906 // %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
3907 llvm::CallInst *call = dyn_cast<llvm::CallInst>(Val: generator);
3908 if (!call)
3909 return nullptr;
3910
3911 bool doRetainAutorelease;
3912
3913 if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
3914 doRetainAutorelease = true;
3915 } else if (call->getCalledOperand() ==
3916 CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
3917 doRetainAutorelease = false;
3918
3919 // If we emitted an assembly marker for this call (and the
3920 // ARCEntrypoints field should have been set if so), go looking
3921 // for that call. If we can't find it, we can't do this
3922 // optimization. But it should always be the immediately previous
3923 // instruction, unless we needed bitcasts around the call.
3924 if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
3925 llvm::Instruction *prev = call->getPrevNode();
3926 assert(prev);
3927 if (isa<llvm::BitCastInst>(Val: prev)) {
3928 prev = prev->getPrevNode();
3929 assert(prev);
3930 }
3931 assert(isa<llvm::CallInst>(prev));
3932 assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3933 CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
3934 InstsToKill.push_back(Elt: prev);
3935 }
3936 } else {
3937 return nullptr;
3938 }
3939
3940 result = call->getArgOperand(i: 0);
3941 InstsToKill.push_back(Elt: call);
3942
3943 // Keep killing bitcasts, for sanity. Note that we no longer care
3944 // about precise ordering as long as there's exactly one use.
3945 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: result)) {
3946 if (!bitcast->hasOneUse())
3947 break;
3948 InstsToKill.push_back(Elt: bitcast);
3949 result = bitcast->getOperand(i_nocapture: 0);
3950 }
3951
3952 // Delete all the unnecessary instructions, from latest to earliest.
3953 for (auto *I : InstsToKill)
3954 I->eraseFromParent();
3955
3956 // Do the fused retain/autorelease if we were asked to.
3957 if (doRetainAutorelease)
3958 result = CGF.EmitARCRetainAutoreleaseReturnValue(value: result);
3959
3960 // Cast back to the result type.
3961 return CGF.Builder.CreateBitCast(V: result, DestTy: resultType);
3962}
3963
3964/// If this is a +1 of the value of an immutable 'self', remove it.
3965static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
3966 llvm::Value *result) {
3967 // This is only applicable to a method with an immutable 'self'.
3968 const ObjCMethodDecl *method =
3969 dyn_cast_or_null<ObjCMethodDecl>(Val: CGF.CurCodeDecl);
3970 if (!method)
3971 return nullptr;
3972 const VarDecl *self = method->getSelfDecl();
3973 if (!self->getType().isConstQualified())
3974 return nullptr;
3975
3976 // Look for a retain call. Note: stripPointerCasts looks through returned arg
3977 // functions, which would cause us to miss the retain.
3978 llvm::CallInst *retainCall = dyn_cast<llvm::CallInst>(Val: result);
3979 if (!retainCall || retainCall->getCalledOperand() !=
3980 CGF.CGM.getObjCEntrypoints().objc_retain)
3981 return nullptr;
3982
3983 // Look for an ordinary load of 'self'.
3984 llvm::Value *retainedValue = retainCall->getArgOperand(i: 0);
3985 llvm::LoadInst *load =
3986 dyn_cast<llvm::LoadInst>(Val: retainedValue->stripPointerCasts());
3987 if (!load || load->isAtomic() || load->isVolatile() ||
3988 load->getPointerOperand() != CGF.GetAddrOfLocalVar(VD: self).getBasePointer())
3989 return nullptr;
3990
3991 // Okay! Burn it all down. This relies for correctness on the
3992 // assumption that the retain is emitted as part of the return and
3993 // that thereafter everything is used "linearly".
3994 llvm::Type *resultType = result->getType();
3995 eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: result));
3996 assert(retainCall->use_empty());
3997 retainCall->eraseFromParent();
3998 eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: retainedValue));
3999
4000 return CGF.Builder.CreateBitCast(V: load, DestTy: resultType);
4001}
4002
4003/// Emit an ARC autorelease of the result of a function.
4004///
4005/// \return the value to actually return from the function
4006static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
4007 llvm::Value *result) {
4008 // If we're returning 'self', kill the initial retain. This is a
4009 // heuristic attempt to "encourage correctness" in the really unfortunate
4010 // case where we have a return of self during a dealloc and we desperately
4011 // need to avoid the possible autorelease.
4012 if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
4013 return self;
4014
4015 // At -O0, try to emit a fused retain/autorelease.
4016 if (CGF.shouldUseFusedARCCalls())
4017 if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
4018 return fused;
4019
4020 return CGF.EmitARCAutoreleaseReturnValue(value: result);
4021}
4022
4023/// Heuristically search for a dominating store to the return-value slot.
4024static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
4025 llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer();
4026
4027 // Check if a User is a store which pointerOperand is the ReturnValue.
4028 // We are looking for stores to the ReturnValue, not for stores of the
4029 // ReturnValue to some other location.
4030 auto GetStoreIfValid = [&CGF,
4031 ReturnValuePtr](llvm::User *U) -> llvm::StoreInst * {
4032 auto *SI = dyn_cast<llvm::StoreInst>(Val: U);
4033 if (!SI || SI->getPointerOperand() != ReturnValuePtr ||
4034 SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())
4035 return nullptr;
4036 // These aren't actually possible for non-coerced returns, and we
4037 // only care about non-coerced returns on this code path.
4038 // All memory instructions inside __try block are volatile.
4039 assert(!SI->isAtomic() &&
4040 (!SI->isVolatile() || CGF.currentFunctionUsesSEHTry()));
4041 return SI;
4042 };
4043 // If there are multiple uses of the return-value slot, just check
4044 // for something immediately preceding the IP. Sometimes this can
4045 // happen with how we generate implicit-returns; it can also happen
4046 // with noreturn cleanups.
4047 if (!ReturnValuePtr->hasOneUse()) {
4048 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
4049 if (IP->empty())
4050 return nullptr;
4051
4052 // Look at directly preceding instruction, skipping bitcasts, lifetime
4053 // markers, and fake uses and their operands.
4054 const llvm::Instruction *LoadIntoFakeUse = nullptr;
4055 for (llvm::Instruction &I : llvm::reverse(C&: *IP)) {
4056 // Ignore instructions that are just loads for fake uses; the load should
4057 // immediately precede the fake use, so we only need to remember the
4058 // operand for the last fake use seen.
4059 if (LoadIntoFakeUse == &I)
4060 continue;
4061 if (isa<llvm::BitCastInst>(Val: &I))
4062 continue;
4063 if (auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &I)) {
4064 if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
4065 continue;
4066
4067 if (II->getIntrinsicID() == llvm::Intrinsic::fake_use) {
4068 LoadIntoFakeUse = dyn_cast<llvm::Instruction>(Val: II->getArgOperand(i: 0));
4069 continue;
4070 }
4071 }
4072 return GetStoreIfValid(&I);
4073 }
4074 return nullptr;
4075 }
4076
4077 llvm::StoreInst *store = GetStoreIfValid(ReturnValuePtr->user_back());
4078 if (!store)
4079 return nullptr;
4080
4081 // Now do a first-and-dirty dominance check: just walk up the
4082 // single-predecessors chain from the current insertion point.
4083 llvm::BasicBlock *StoreBB = store->getParent();
4084 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
4085 llvm::SmallPtrSet<llvm::BasicBlock *, 4> SeenBBs;
4086 while (IP != StoreBB) {
4087 if (!SeenBBs.insert(Ptr: IP).second || !(IP = IP->getSinglePredecessor()))
4088 return nullptr;
4089 }
4090
4091 // Okay, the store's basic block dominates the insertion point; we
4092 // can do our thing.
4093 return store;
4094}
4095
4096// Helper functions for EmitCMSEClearRecord
4097
4098// Set the bits corresponding to a field having width `BitWidth` and located at
4099// offset `BitOffset` (from the least significant bit) within a storage unit of
4100// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
4101// Use little-endian layout, i.e.`Bits[0]` is the LSB.
4102static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
4103 int BitWidth, int CharWidth) {
4104 assert(CharWidth <= 64);
4105 assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
4106
4107 int Pos = 0;
4108 if (BitOffset >= CharWidth) {
4109 Pos += BitOffset / CharWidth;
4110 BitOffset = BitOffset % CharWidth;
4111 }
4112
4113 const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
4114 if (BitOffset + BitWidth >= CharWidth) {
4115 Bits[Pos++] |= (Used << BitOffset) & Used;
4116 BitWidth -= CharWidth - BitOffset;
4117 BitOffset = 0;
4118 }
4119
4120 while (BitWidth >= CharWidth) {
4121 Bits[Pos++] = Used;
4122 BitWidth -= CharWidth;
4123 }
4124
4125 if (BitWidth > 0)
4126 Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
4127}
4128
4129// Set the bits corresponding to a field having width `BitWidth` and located at
4130// offset `BitOffset` (from the least significant bit) within a storage unit of
4131// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
4132// `Bits` corresponds to one target byte. Use target endian layout.
4133static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
4134 int StorageSize, int BitOffset, int BitWidth,
4135 int CharWidth, bool BigEndian) {
4136
4137 SmallVector<uint64_t, 8> TmpBits(StorageSize);
4138 setBitRange(Bits&: TmpBits, BitOffset, BitWidth, CharWidth);
4139
4140 if (BigEndian)
4141 std::reverse(first: TmpBits.begin(), last: TmpBits.end());
4142
4143 for (uint64_t V : TmpBits)
4144 Bits[StorageOffset++] |= V;
4145}
4146
4147static void setUsedBits(CodeGenModule &, QualType, int,
4148 SmallVectorImpl<uint64_t> &);
4149
4150// Set the bits in `Bits`, which correspond to the value representations of
4151// the actual members of the record type `RTy`. Note that this function does
4152// not handle base classes, virtual tables, etc, since they cannot happen in
4153// CMSE function arguments or return. The bit mask corresponds to the target
4154// memory layout, i.e. it's endian dependent.
4155static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
4156 SmallVectorImpl<uint64_t> &Bits) {
4157 ASTContext &Context = CGM.getContext();
4158 int CharWidth = Context.getCharWidth();
4159 const RecordDecl *RD = RTy->getDecl()->getDefinition();
4160 const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(D: RD);
4161 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
4162
4163 int Idx = 0;
4164 for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) {
4165 const FieldDecl *F = *I;
4166
4167 if (F->isUnnamedBitField() || F->isZeroLengthBitField() ||
4168 F->getType()->isIncompleteArrayType())
4169 continue;
4170
4171 if (F->isBitField()) {
4172 const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(FD: F);
4173 setBitRange(Bits, StorageOffset: Offset + BFI.StorageOffset.getQuantity(),
4174 StorageSize: BFI.StorageSize / CharWidth, BitOffset: BFI.Offset, BitWidth: BFI.Size, CharWidth,
4175 BigEndian: CGM.getDataLayout().isBigEndian());
4176 continue;
4177 }
4178
4179 setUsedBits(CGM, F->getType(),
4180 Offset + ASTLayout.getFieldOffset(FieldNo: Idx) / CharWidth, Bits);
4181 }
4182}
4183
4184// Set the bits in `Bits`, which correspond to the value representations of
4185// the elements of an array type `ATy`.
4186static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
4187 int Offset, SmallVectorImpl<uint64_t> &Bits) {
4188 const ASTContext &Context = CGM.getContext();
4189
4190 QualType ETy = Context.getBaseElementType(VAT: ATy);
4191 int Size = Context.getTypeSizeInChars(T: ETy).getQuantity();
4192 SmallVector<uint64_t, 4> TmpBits(Size);
4193 setUsedBits(CGM, ETy, 0, TmpBits);
4194
4195 for (int I = 0, N = Context.getConstantArrayElementCount(CA: ATy); I < N; ++I) {
4196 auto Src = TmpBits.begin();
4197 auto Dst = Bits.begin() + Offset + I * Size;
4198 for (int J = 0; J < Size; ++J)
4199 *Dst++ |= *Src++;
4200 }
4201}
4202
4203// Set the bits in `Bits`, which correspond to the value representations of
4204// the type `QTy`.
4205static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
4206 SmallVectorImpl<uint64_t> &Bits) {
4207 if (const auto *RTy = QTy->getAsCanonical<RecordType>())
4208 return setUsedBits(CGM, RTy, Offset, Bits);
4209
4210 ASTContext &Context = CGM.getContext();
4211 if (const auto *ATy = Context.getAsConstantArrayType(T: QTy))
4212 return setUsedBits(CGM, ATy, Offset, Bits);
4213
4214 int Size = Context.getTypeSizeInChars(T: QTy).getQuantity();
4215 if (Size <= 0)
4216 return;
4217
4218 std::fill_n(first: Bits.begin() + Offset, n: Size,
4219 value: (uint64_t(1) << Context.getCharWidth()) - 1);
4220}
4221
4222static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
4223 int Pos, int Size, int CharWidth,
4224 bool BigEndian) {
4225 assert(Size > 0);
4226 uint64_t Mask = 0;
4227 if (BigEndian) {
4228 for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
4229 ++P)
4230 Mask = (Mask << CharWidth) | *P;
4231 } else {
4232 auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
4233 do
4234 Mask = (Mask << CharWidth) | *--P;
4235 while (P != End);
4236 }
4237 return Mask;
4238}
4239
4240// Emit code to clear the bits in a record, which aren't a part of any user
4241// declared member, when the record is a function return.
4242llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
4243 llvm::IntegerType *ITy,
4244 QualType QTy) {
4245 assert(Src->getType() == ITy);
4246 assert(ITy->getScalarSizeInBits() <= 64);
4247
4248 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
4249 int Size = DataLayout.getTypeStoreSize(Ty: ITy);
4250 SmallVector<uint64_t, 4> Bits(Size);
4251 setUsedBits(CGM, RTy: QTy->castAsCanonical<RecordType>(), Offset: 0, Bits);
4252
4253 int CharWidth = CGM.getContext().getCharWidth();
4254 uint64_t Mask =
4255 buildMultiCharMask(Bits, Pos: 0, Size, CharWidth, BigEndian: DataLayout.isBigEndian());
4256
4257 return Builder.CreateAnd(LHS: Src, RHS: Mask, Name: "cmse.clear");
4258}
4259
4260// Emit code to clear the bits in a record, which aren't a part of any user
4261// declared member, when the record is a function argument.
4262llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
4263 llvm::ArrayType *ATy,
4264 QualType QTy) {
4265 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
4266 int Size = DataLayout.getTypeStoreSize(Ty: ATy);
4267 SmallVector<uint64_t, 16> Bits(Size);
4268 setUsedBits(CGM, RTy: QTy->castAsCanonical<RecordType>(), Offset: 0, Bits);
4269
4270 // Clear each element of the LLVM array.
4271 int CharWidth = CGM.getContext().getCharWidth();
4272 int CharsPerElt =
4273 ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
4274 int MaskIndex = 0;
4275 llvm::Value *R = llvm::PoisonValue::get(T: ATy);
4276 for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
4277 uint64_t Mask = buildMultiCharMask(Bits, Pos: MaskIndex, Size: CharsPerElt, CharWidth,
4278 BigEndian: DataLayout.isBigEndian());
4279 MaskIndex += CharsPerElt;
4280 llvm::Value *T0 = Builder.CreateExtractValue(Agg: Src, Idxs: I);
4281 llvm::Value *T1 = Builder.CreateAnd(LHS: T0, RHS: Mask, Name: "cmse.clear");
4282 R = Builder.CreateInsertValue(Agg: R, Val: T1, Idxs: I);
4283 }
4284
4285 return R;
4286}
4287
4288void CodeGenFunction::EmitFunctionEpilog(
4289 const CGFunctionInfo &FI, bool EmitRetDbgLoc, SourceLocation EndLoc,
4290 uint64_t RetKeyInstructionsSourceAtom) {
4291 if (FI.isNoReturn()) {
4292 // Noreturn functions don't return.
4293 EmitUnreachable(Loc: EndLoc);
4294 return;
4295 }
4296
4297 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {
4298 // Naked functions don't have epilogues.
4299 Builder.CreateUnreachable();
4300 return;
4301 }
4302
4303 // Functions with no result always return void.
4304 if (!ReturnValue.isValid()) {
4305 auto *I = Builder.CreateRetVoid();
4306 if (RetKeyInstructionsSourceAtom)
4307 addInstToSpecificSourceAtom(KeyInstruction: I, Backup: nullptr, Atom: RetKeyInstructionsSourceAtom);
4308 else
4309 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4310 return;
4311 }
4312
4313 llvm::DebugLoc RetDbgLoc;
4314 llvm::Value *RV = nullptr;
4315 QualType RetTy = FI.getReturnType();
4316 const ABIArgInfo &RetAI = FI.getReturnInfo();
4317
4318 switch (RetAI.getKind()) {
4319 case ABIArgInfo::InAlloca:
4320 // Aggregates get evaluated directly into the destination. Sometimes we
4321 // need to return the sret value in a register, though.
4322 assert(hasAggregateEvaluationKind(RetTy));
4323 if (RetAI.getInAllocaSRet()) {
4324 llvm::Function::arg_iterator EI = CurFn->arg_end();
4325 --EI;
4326 llvm::Value *ArgStruct = &*EI;
4327 llvm::Value *SRet = Builder.CreateStructGEP(
4328 Ty: FI.getArgStruct(), Ptr: ArgStruct, Idx: RetAI.getInAllocaFieldIndex());
4329 llvm::Type *Ty =
4330 cast<llvm::GetElementPtrInst>(Val: SRet)->getResultElementType();
4331 RV = Builder.CreateAlignedLoad(Ty, Addr: SRet, Align: getPointerAlign(), Name: "sret");
4332 }
4333 break;
4334
4335 case ABIArgInfo::Indirect: {
4336 auto AI = CurFn->arg_begin();
4337 if (RetAI.isSRetAfterThis())
4338 ++AI;
4339 switch (getEvaluationKind(T: RetTy)) {
4340 case TEK_Complex: {
4341 ComplexPairTy RT =
4342 EmitLoadOfComplex(src: MakeAddrLValue(Addr: ReturnValue, T: RetTy), loc: EndLoc);
4343 EmitStoreOfComplex(V: RT, dest: MakeNaturalAlignAddrLValue(V: &*AI, T: RetTy),
4344 /*isInit*/ true);
4345 break;
4346 }
4347 case TEK_Aggregate:
4348 // Do nothing; aggregates get evaluated directly into the destination.
4349 break;
4350 case TEK_Scalar: {
4351 LValueBaseInfo BaseInfo;
4352 TBAAAccessInfo TBAAInfo;
4353 CharUnits Alignment =
4354 CGM.getNaturalTypeAlignment(T: RetTy, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
4355 Address ArgAddr(&*AI, ConvertType(T: RetTy), Alignment);
4356 LValue ArgVal =
4357 LValue::MakeAddr(Addr: ArgAddr, type: RetTy, Context&: getContext(), BaseInfo, TBAAInfo);
4358 EmitStoreOfScalar(
4359 value: EmitLoadOfScalar(lvalue: MakeAddrLValue(Addr: ReturnValue, T: RetTy), Loc: EndLoc), lvalue: ArgVal,
4360 /*isInit*/ true);
4361 break;
4362 }
4363 }
4364 break;
4365 }
4366
4367 case ABIArgInfo::Extend:
4368 case ABIArgInfo::Direct:
4369 if (RetAI.getCoerceToType() == ConvertType(T: RetTy) &&
4370 RetAI.getDirectOffset() == 0) {
4371 // The internal return value temp always will have pointer-to-return-type
4372 // type, just do a load.
4373
4374 // If there is a dominating store to ReturnValue, we can elide
4375 // the load, zap the store, and usually zap the alloca.
4376 if (llvm::StoreInst *SI = findDominatingStoreToReturnValue(CGF&: *this)) {
4377 // Reuse the debug location from the store unless there is
4378 // cleanup code to be emitted between the store and return
4379 // instruction.
4380 if (EmitRetDbgLoc && !AutoreleaseResult)
4381 RetDbgLoc = SI->getDebugLoc();
4382 // Get the stored value and nuke the now-dead store.
4383 RV = SI->getValueOperand();
4384 SI->eraseFromParent();
4385
4386 // Otherwise, we have to do a simple load.
4387 } else {
4388 RV = Builder.CreateLoad(Addr: ReturnValue);
4389 }
4390 } else {
4391 // If the value is offset in memory, apply the offset now.
4392 Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
4393
4394 RV = CreateCoercedLoad(Src: V, SrcFETy: RetTy, Ty: RetAI.getCoerceToType(), CGF&: *this);
4395 }
4396
4397 // In ARC, end functions that return a retainable type with a call
4398 // to objc_autoreleaseReturnValue.
4399 if (AutoreleaseResult) {
4400#ifndef NDEBUG
4401 // Type::isObjCRetainabletype has to be called on a QualType that hasn't
4402 // been stripped of the typedefs, so we cannot use RetTy here. Get the
4403 // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
4404 // CurCodeDecl or BlockInfo.
4405 QualType RT;
4406
4407 if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
4408 RT = FD->getReturnType();
4409 else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
4410 RT = MD->getReturnType();
4411 else if (isa<BlockDecl>(CurCodeDecl))
4412 RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
4413 else
4414 llvm_unreachable("Unexpected function/method type");
4415
4416 assert(getLangOpts().ObjCAutoRefCount && !FI.isReturnsRetained() &&
4417 RT->isObjCRetainableType());
4418#endif
4419 RV = emitAutoreleaseOfResult(CGF&: *this, result: RV);
4420 }
4421
4422 break;
4423
4424 case ABIArgInfo::Ignore:
4425 break;
4426
4427 case ABIArgInfo::CoerceAndExpand: {
4428 auto coercionType = RetAI.getCoerceAndExpandType();
4429 auto unpaddedCoercionType = RetAI.getUnpaddedCoerceAndExpandType();
4430 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
4431
4432 // Load all of the coerced elements out into results.
4433 llvm::SmallVector<llvm::Value *, 4> results;
4434 Address addr = ReturnValue.withElementType(ElemTy: coercionType);
4435 unsigned unpaddedIndex = 0;
4436 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
4437 auto coercedEltType = coercionType->getElementType(N: i);
4438 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType: coercedEltType))
4439 continue;
4440
4441 auto eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
4442 llvm::Value *elt = CreateCoercedLoad(
4443 Src: eltAddr, SrcFETy: RetTy,
4444 Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
4445 : unpaddedCoercionType,
4446 CGF&: *this);
4447 results.push_back(Elt: elt);
4448 }
4449
4450 // If we have one result, it's the single direct result type.
4451 if (results.size() == 1) {
4452 RV = results[0];
4453
4454 // Otherwise, we need to make a first-class aggregate.
4455 } else {
4456 // Construct a return type that lacks padding elements.
4457 llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
4458
4459 RV = llvm::PoisonValue::get(T: returnType);
4460 for (unsigned i = 0, e = results.size(); i != e; ++i) {
4461 RV = Builder.CreateInsertValue(Agg: RV, Val: results[i], Idxs: i);
4462 }
4463 }
4464 break;
4465 }
4466 case ABIArgInfo::TargetSpecific: {
4467 Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
4468 RV = CGM.getABIInfo().createCoercedLoad(SrcAddr: V, AI: RetAI, CGF&: *this);
4469 break;
4470 }
4471 case ABIArgInfo::Expand:
4472 case ABIArgInfo::IndirectAliased:
4473 llvm_unreachable("Invalid ABI kind for return argument");
4474 }
4475
4476 llvm::Instruction *Ret;
4477 if (RV) {
4478 if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) {
4479 // For certain return types, clear padding bits, as they may reveal
4480 // sensitive information.
4481 // Small struct/union types are passed as integers.
4482 auto *ITy = dyn_cast<llvm::IntegerType>(Val: RV->getType());
4483 if (ITy != nullptr && isa<RecordType>(Val: RetTy.getCanonicalType()))
4484 RV = EmitCMSEClearRecord(Src: RV, ITy, QTy: RetTy);
4485 }
4486 EmitReturnValueCheck(RV);
4487 Ret = Builder.CreateRet(V: RV);
4488 } else {
4489 Ret = Builder.CreateRetVoid();
4490 }
4491
4492 if (RetDbgLoc)
4493 Ret->setDebugLoc(std::move(RetDbgLoc));
4494
4495 llvm::Value *Backup = RV ? Ret->getOperand(i: 0) : nullptr;
4496 if (RetKeyInstructionsSourceAtom)
4497 addInstToSpecificSourceAtom(KeyInstruction: Ret, Backup, Atom: RetKeyInstructionsSourceAtom);
4498 else
4499 addInstToNewSourceAtom(KeyInstruction: Ret, Backup);
4500}
4501
4502void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
4503 // A current decl may not be available when emitting vtable thunks.
4504 if (!CurCodeDecl)
4505 return;
4506
4507 // If the return block isn't reachable, neither is this check, so don't emit
4508 // it.
4509 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty())
4510 return;
4511
4512 ReturnsNonNullAttr *RetNNAttr = nullptr;
4513 if (SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute))
4514 RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
4515
4516 if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
4517 return;
4518
4519 // Prefer the returns_nonnull attribute if it's present.
4520 SourceLocation AttrLoc;
4521 SanitizerKind::SanitizerOrdinal CheckKind;
4522 SanitizerHandler Handler;
4523 if (RetNNAttr) {
4524 assert(!requiresReturnValueNullabilityCheck() &&
4525 "Cannot check nullability and the nonnull attribute");
4526 AttrLoc = RetNNAttr->getLocation();
4527 CheckKind = SanitizerKind::SO_ReturnsNonnullAttribute;
4528 Handler = SanitizerHandler::NonnullReturn;
4529 } else {
4530 if (auto *DD = dyn_cast<DeclaratorDecl>(Val: CurCodeDecl))
4531 if (auto *TSI = DD->getTypeSourceInfo())
4532 if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
4533 AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
4534 CheckKind = SanitizerKind::SO_NullabilityReturn;
4535 Handler = SanitizerHandler::NullabilityReturn;
4536 }
4537
4538 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4539
4540 // Make sure the "return" source location is valid. If we're checking a
4541 // nullability annotation, make sure the preconditions for the check are met.
4542 llvm::BasicBlock *Check = createBasicBlock(name: "nullcheck");
4543 llvm::BasicBlock *NoCheck = createBasicBlock(name: "no.nullcheck");
4544 llvm::Value *SLocPtr = Builder.CreateLoad(Addr: ReturnLocation, Name: "return.sloc.load");
4545 llvm::Value *CanNullCheck = Builder.CreateIsNotNull(Arg: SLocPtr);
4546 if (requiresReturnValueNullabilityCheck())
4547 CanNullCheck =
4548 Builder.CreateAnd(LHS: CanNullCheck, RHS: RetValNullabilityPrecondition);
4549 Builder.CreateCondBr(Cond: CanNullCheck, True: Check, False: NoCheck);
4550 EmitBlock(BB: Check);
4551
4552 // Now do the null check.
4553 llvm::Value *Cond = Builder.CreateIsNotNull(Arg: RV);
4554 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc: AttrLoc)};
4555 llvm::Value *DynamicData[] = {SLocPtr};
4556 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: DynamicData);
4557
4558 EmitBlock(BB: NoCheck);
4559
4560#ifndef NDEBUG
4561 // The return location should not be used after the check has been emitted.
4562 ReturnLocation = Address::invalid();
4563#endif
4564}
4565
4566static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
4567 const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
4568 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
4569}
4570
4571static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, QualType Ty) {
4572 // FIXME: Generate IR in one pass, rather than going back and fixing up these
4573 // placeholders.
4574 llvm::Type *IRTy = CGF.ConvertTypeForMem(T: Ty);
4575 llvm::Type *IRPtrTy = llvm::PointerType::getUnqual(C&: CGF.getLLVMContext());
4576 llvm::Value *Placeholder = llvm::PoisonValue::get(T: IRPtrTy);
4577
4578 // FIXME: When we generate this IR in one pass, we shouldn't need
4579 // this win32-specific alignment hack.
4580 CharUnits Align = CharUnits::fromQuantity(Quantity: 4);
4581 Placeholder = CGF.Builder.CreateAlignedLoad(Ty: IRPtrTy, Addr: Placeholder, Align);
4582
4583 return AggValueSlot::forAddr(
4584 addr: Address(Placeholder, IRTy, Align), quals: Ty.getQualifiers(),
4585 isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
4586 isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
4587}
4588
4589void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
4590 const VarDecl *param,
4591 SourceLocation loc) {
4592 // StartFunction converted the ABI-lowered parameter(s) into a
4593 // local alloca. We need to turn that into an r-value suitable
4594 // for EmitCall.
4595 Address local = GetAddrOfLocalVar(VD: param);
4596
4597 QualType type = param->getType();
4598
4599 // GetAddrOfLocalVar returns a pointer-to-pointer for references,
4600 // but the argument needs to be the original pointer.
4601 if (type->isReferenceType()) {
4602 args.add(rvalue: RValue::get(V: Builder.CreateLoad(Addr: local)), type);
4603
4604 // In ARC, move out of consumed arguments so that the release cleanup
4605 // entered by StartFunction doesn't cause an over-release. This isn't
4606 // optimal -O0 code generation, but it should get cleaned up when
4607 // optimization is enabled. This also assumes that delegate calls are
4608 // performed exactly once for a set of arguments, but that should be safe.
4609 } else if (getLangOpts().ObjCAutoRefCount &&
4610 param->hasAttr<NSConsumedAttr>() && type->isObjCRetainableType()) {
4611 llvm::Value *ptr = Builder.CreateLoad(Addr: local);
4612 auto null =
4613 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: ptr->getType()));
4614 Builder.CreateStore(Val: null, Addr: local);
4615 args.add(rvalue: RValue::get(V: ptr), type);
4616
4617 // For the most part, we just need to load the alloca, except that
4618 // aggregate r-values are actually pointers to temporaries.
4619 } else {
4620 args.add(rvalue: convertTempToRValue(addr: local, type, Loc: loc), type);
4621 }
4622
4623 // Deactivate the cleanup for the callee-destructed param that was pushed.
4624 if (type->isRecordType() && !CurFuncIsThunk &&
4625 type->castAsRecordDecl()->isParamDestroyedInCallee() &&
4626 param->needsDestruction(Ctx: getContext())) {
4627 EHScopeStack::stable_iterator cleanup =
4628 CalleeDestructedParamCleanups.lookup(Val: cast<ParmVarDecl>(Val: param));
4629 assert(cleanup.isValid() &&
4630 "cleanup for callee-destructed param not recorded");
4631 // This unreachable is a temporary marker which will be removed later.
4632 llvm::Instruction *isActive = Builder.CreateUnreachable();
4633 args.addArgCleanupDeactivation(Cleanup: cleanup, IsActiveIP: isActive);
4634 }
4635}
4636
4637static bool isProvablyNull(llvm::Value *addr) {
4638 return llvm::isa_and_nonnull<llvm::ConstantPointerNull>(Val: addr);
4639}
4640
4641static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) {
4642 return llvm::isKnownNonZero(V: Addr.getBasePointer(), Q: CGF.CGM.getDataLayout());
4643}
4644
4645/// Emit the actual writing-back of a writeback.
4646static void emitWriteback(CodeGenFunction &CGF,
4647 const CallArgList::Writeback &writeback) {
4648 const LValue &srcLV = writeback.Source;
4649 Address srcAddr = srcLV.getAddress();
4650 assert(!isProvablyNull(srcAddr.getBasePointer()) &&
4651 "shouldn't have writeback for provably null argument");
4652
4653 if (writeback.WritebackExpr) {
4654 CGF.EmitIgnoredExpr(E: writeback.WritebackExpr);
4655 CGF.EmitLifetimeEnd(Addr: writeback.Temporary.getBasePointer());
4656 return;
4657 }
4658
4659 llvm::BasicBlock *contBB = nullptr;
4660
4661 // If the argument wasn't provably non-null, we need to null check
4662 // before doing the store.
4663 bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4664
4665 if (!provablyNonNull) {
4666 llvm::BasicBlock *writebackBB = CGF.createBasicBlock(name: "icr.writeback");
4667 contBB = CGF.createBasicBlock(name: "icr.done");
4668
4669 llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4670 CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: writebackBB);
4671 CGF.EmitBlock(BB: writebackBB);
4672 }
4673
4674 // Load the value to writeback.
4675 llvm::Value *value = CGF.Builder.CreateLoad(Addr: writeback.Temporary);
4676
4677 // Cast it back, in case we're writing an id to a Foo* or something.
4678 value = CGF.Builder.CreateBitCast(V: value, DestTy: srcAddr.getElementType(),
4679 Name: "icr.writeback-cast");
4680
4681 // Perform the writeback.
4682
4683 // If we have a "to use" value, it's something we need to emit a use
4684 // of. This has to be carefully threaded in: if it's done after the
4685 // release it's potentially undefined behavior (and the optimizer
4686 // will ignore it), and if it happens before the retain then the
4687 // optimizer could move the release there.
4688 if (writeback.ToUse) {
4689 assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
4690
4691 // Retain the new value. No need to block-copy here: the block's
4692 // being passed up the stack.
4693 value = CGF.EmitARCRetainNonBlock(value);
4694
4695 // Emit the intrinsic use here.
4696 CGF.EmitARCIntrinsicUse(values: writeback.ToUse);
4697
4698 // Load the old value (primitively).
4699 llvm::Value *oldValue = CGF.EmitLoadOfScalar(lvalue: srcLV, Loc: SourceLocation());
4700
4701 // Put the new value in place (primitively).
4702 CGF.EmitStoreOfScalar(value, lvalue: srcLV, /*init*/ isInit: false);
4703
4704 // Release the old value.
4705 CGF.EmitARCRelease(value: oldValue, precise: srcLV.isARCPreciseLifetime());
4706
4707 // Otherwise, we can just do a normal lvalue store.
4708 } else {
4709 CGF.EmitStoreThroughLValue(Src: RValue::get(V: value), Dst: srcLV);
4710 }
4711
4712 // Jump to the continuation block.
4713 if (!provablyNonNull)
4714 CGF.EmitBlock(BB: contBB);
4715}
4716
4717static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
4718 const CallArgList &CallArgs) {
4719 ArrayRef<CallArgList::CallArgCleanup> Cleanups =
4720 CallArgs.getCleanupsToDeactivate();
4721 // Iterate in reverse to increase the likelihood of popping the cleanup.
4722 for (const auto &I : llvm::reverse(C&: Cleanups)) {
4723 CGF.DeactivateCleanupBlock(Cleanup: I.Cleanup, DominatingIP: I.IsActiveIP);
4724 I.IsActiveIP->eraseFromParent();
4725 }
4726}
4727
4728static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
4729 if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(Val: E->IgnoreParens()))
4730 if (uop->getOpcode() == UO_AddrOf)
4731 return uop->getSubExpr();
4732 return nullptr;
4733}
4734
4735/// Emit an argument that's being passed call-by-writeback. That is,
4736/// we are passing the address of an __autoreleased temporary; it
4737/// might be copy-initialized with the current value of the given
4738/// address, but it will definitely be copied out of after the call.
4739static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
4740 const ObjCIndirectCopyRestoreExpr *CRE) {
4741 LValue srcLV;
4742
4743 // Make an optimistic effort to emit the address as an l-value.
4744 // This can fail if the argument expression is more complicated.
4745 if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(E: CRE->getSubExpr())) {
4746 srcLV = CGF.EmitLValue(E: lvExpr);
4747
4748 // Otherwise, just emit it as a scalar.
4749 } else {
4750 Address srcAddr = CGF.EmitPointerWithAlignment(Addr: CRE->getSubExpr());
4751
4752 QualType srcAddrType =
4753 CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
4754 srcLV = CGF.MakeAddrLValue(Addr: srcAddr, T: srcAddrType);
4755 }
4756 Address srcAddr = srcLV.getAddress();
4757
4758 // The dest and src types don't necessarily match in LLVM terms
4759 // because of the crazy ObjC compatibility rules.
4760
4761 llvm::PointerType *destType =
4762 cast<llvm::PointerType>(Val: CGF.ConvertType(T: CRE->getType()));
4763 llvm::Type *destElemType =
4764 CGF.ConvertTypeForMem(T: CRE->getType()->getPointeeType());
4765
4766 // If the address is a constant null, just pass the appropriate null.
4767 if (isProvablyNull(addr: srcAddr.getBasePointer())) {
4768 args.add(rvalue: RValue::get(V: llvm::ConstantPointerNull::get(T: destType)),
4769 type: CRE->getType());
4770 return;
4771 }
4772
4773 // Create the temporary.
4774 Address temp =
4775 CGF.CreateTempAlloca(Ty: destElemType, align: CGF.getPointerAlign(), Name: "icr.temp");
4776 // Loading an l-value can introduce a cleanup if the l-value is __weak,
4777 // and that cleanup will be conditional if we can't prove that the l-value
4778 // isn't null, so we need to register a dominating point so that the cleanups
4779 // system will make valid IR.
4780 CodeGenFunction::ConditionalEvaluation condEval(CGF);
4781
4782 // Zero-initialize it if we're not doing a copy-initialization.
4783 bool shouldCopy = CRE->shouldCopy();
4784 if (!shouldCopy) {
4785 llvm::Value *null =
4786 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: destElemType));
4787 CGF.Builder.CreateStore(Val: null, Addr: temp);
4788 }
4789
4790 llvm::BasicBlock *contBB = nullptr;
4791 llvm::BasicBlock *originBB = nullptr;
4792
4793 // If the address is *not* known to be non-null, we need to switch.
4794 llvm::Value *finalArgument;
4795
4796 bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4797
4798 if (provablyNonNull) {
4799 finalArgument = temp.emitRawPointer(CGF);
4800 } else {
4801 llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4802
4803 finalArgument = CGF.Builder.CreateSelect(
4804 C: isNull, True: llvm::ConstantPointerNull::get(T: destType),
4805 False: temp.emitRawPointer(CGF), Name: "icr.argument");
4806
4807 // If we need to copy, then the load has to be conditional, which
4808 // means we need control flow.
4809 if (shouldCopy) {
4810 originBB = CGF.Builder.GetInsertBlock();
4811 contBB = CGF.createBasicBlock(name: "icr.cont");
4812 llvm::BasicBlock *copyBB = CGF.createBasicBlock(name: "icr.copy");
4813 CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: copyBB);
4814 CGF.EmitBlock(BB: copyBB);
4815 condEval.begin(CGF);
4816 }
4817 }
4818
4819 llvm::Value *valueToUse = nullptr;
4820
4821 // Perform a copy if necessary.
4822 if (shouldCopy) {
4823 RValue srcRV = CGF.EmitLoadOfLValue(V: srcLV, Loc: SourceLocation());
4824 assert(srcRV.isScalar());
4825
4826 llvm::Value *src = srcRV.getScalarVal();
4827 src = CGF.Builder.CreateBitCast(V: src, DestTy: destElemType, Name: "icr.cast");
4828
4829 // Use an ordinary store, not a store-to-lvalue.
4830 CGF.Builder.CreateStore(Val: src, Addr: temp);
4831
4832 // If optimization is enabled, and the value was held in a
4833 // __strong variable, we need to tell the optimizer that this
4834 // value has to stay alive until we're doing the store back.
4835 // This is because the temporary is effectively unretained,
4836 // and so otherwise we can violate the high-level semantics.
4837 if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4838 srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {
4839 valueToUse = src;
4840 }
4841 }
4842
4843 // Finish the control flow if we needed it.
4844 if (shouldCopy && !provablyNonNull) {
4845 llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
4846 CGF.EmitBlock(BB: contBB);
4847
4848 // Make a phi for the value to intrinsically use.
4849 if (valueToUse) {
4850 llvm::PHINode *phiToUse =
4851 CGF.Builder.CreatePHI(Ty: valueToUse->getType(), NumReservedValues: 2, Name: "icr.to-use");
4852 phiToUse->addIncoming(V: valueToUse, BB: copyBB);
4853 phiToUse->addIncoming(V: llvm::PoisonValue::get(T: valueToUse->getType()),
4854 BB: originBB);
4855 valueToUse = phiToUse;
4856 }
4857
4858 condEval.end(CGF);
4859 }
4860
4861 args.addWriteback(srcLV, temporary: temp, toUse: valueToUse);
4862 args.add(rvalue: RValue::get(V: finalArgument), type: CRE->getType());
4863}
4864
4865void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
4866 assert(!StackBase);
4867
4868 // Save the stack.
4869 StackBase = CGF.Builder.CreateStackSave(Name: "inalloca.save");
4870}
4871
4872void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
4873 if (StackBase) {
4874 // Restore the stack after the call.
4875 CGF.Builder.CreateStackRestore(Ptr: StackBase);
4876 }
4877}
4878
4879void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
4880 SourceLocation ArgLoc,
4881 AbstractCallee AC, unsigned ParmNum) {
4882 if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
4883 SanOpts.has(K: SanitizerKind::NullabilityArg)))
4884 return;
4885
4886 // The param decl may be missing in a variadic function.
4887 auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(I: ParmNum) : nullptr;
4888 unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
4889
4890 // Prefer the nonnull attribute if it's present.
4891 const NonNullAttr *NNAttr = nullptr;
4892 if (SanOpts.has(K: SanitizerKind::NonnullAttribute))
4893 NNAttr = getNonNullAttr(FD: AC.getDecl(), PVD, ArgType, ArgNo);
4894
4895 bool CanCheckNullability = false;
4896 if (SanOpts.has(K: SanitizerKind::NullabilityArg) && !NNAttr && PVD &&
4897 !PVD->getType()->isRecordType()) {
4898 auto Nullability = PVD->getType()->getNullability();
4899 CanCheckNullability = Nullability &&
4900 *Nullability == NullabilityKind::NonNull &&
4901 PVD->getTypeSourceInfo();
4902 }
4903
4904 if (!NNAttr && !CanCheckNullability)
4905 return;
4906
4907 SourceLocation AttrLoc;
4908 SanitizerKind::SanitizerOrdinal CheckKind;
4909 SanitizerHandler Handler;
4910 if (NNAttr) {
4911 AttrLoc = NNAttr->getLocation();
4912 CheckKind = SanitizerKind::SO_NonnullAttribute;
4913 Handler = SanitizerHandler::NonnullArg;
4914 } else {
4915 AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4916 CheckKind = SanitizerKind::SO_NullabilityArg;
4917 Handler = SanitizerHandler::NullabilityArg;
4918 }
4919
4920 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4921 llvm::Value *Cond = EmitNonNullRValueCheck(RV, T: ArgType);
4922 llvm::Constant *StaticData[] = {
4923 EmitCheckSourceLocation(Loc: ArgLoc),
4924 EmitCheckSourceLocation(Loc: AttrLoc),
4925 llvm::ConstantInt::get(Ty: Int32Ty, V: ArgNo + 1),
4926 };
4927 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: {});
4928}
4929
4930void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType,
4931 SourceLocation ArgLoc,
4932 AbstractCallee AC, unsigned ParmNum) {
4933 if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
4934 SanOpts.has(K: SanitizerKind::NullabilityArg)))
4935 return;
4936
4937 EmitNonNullArgCheck(RV: RValue::get(Addr, CGF&: *this), ArgType, ArgLoc, AC, ParmNum);
4938}
4939
4940// Check if the call is going to use the inalloca convention. This needs to
4941// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
4942// later, so we can't check it directly.
4943static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
4944 ArrayRef<QualType> ArgTypes) {
4945 // The Swift calling conventions don't go through the target-specific
4946 // argument classification, they never use inalloca.
4947 // TODO: Consider limiting inalloca use to only calling conventions supported
4948 // by MSVC.
4949 if (ExplicitCC == CC_Swift || ExplicitCC == CC_SwiftAsync)
4950 return false;
4951 if (!CGM.getTarget().getCXXABI().isMicrosoft())
4952 return false;
4953 return llvm::any_of(Range&: ArgTypes, P: [&](QualType Ty) {
4954 return isInAllocaArgument(ABI&: CGM.getCXXABI(), type: Ty);
4955 });
4956}
4957
4958#ifndef NDEBUG
4959// Determine whether the given argument is an Objective-C method
4960// that may have type parameters in its signature.
4961static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
4962 const DeclContext *dc = method->getDeclContext();
4963 if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {
4964 return classDecl->getTypeParamListAsWritten();
4965 }
4966
4967 if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
4968 return catDecl->getTypeParamList();
4969 }
4970
4971 return false;
4972}
4973#endif
4974
4975/// EmitCallArgs - Emit call arguments for a function.
4976void CodeGenFunction::EmitCallArgs(
4977 CallArgList &Args, PrototypeWrapper Prototype,
4978 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4979 AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
4980 SmallVector<QualType, 16> ArgTypes;
4981
4982 assert((ParamsToSkip == 0 || Prototype.P) &&
4983 "Can't skip parameters if type info is not provided");
4984
4985 // This variable only captures *explicitly* written conventions, not those
4986 // applied by default via command line flags or target defaults, such as
4987 // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
4988 // require knowing if this is a C++ instance method or being able to see
4989 // unprototyped FunctionTypes.
4990 CallingConv ExplicitCC = CC_C;
4991
4992 // First, if a prototype was provided, use those argument types.
4993 bool IsVariadic = false;
4994 if (Prototype.P) {
4995 const auto *MD = dyn_cast<const ObjCMethodDecl *>(Val&: Prototype.P);
4996 if (MD) {
4997 IsVariadic = MD->isVariadic();
4998 ExplicitCC = getCallingConventionForDecl(
4999 D: MD, IsTargetDefaultMSABI: CGM.getTarget().getTriple().isOSWindows());
5000 ArgTypes.assign(in_start: MD->param_type_begin() + ParamsToSkip,
5001 in_end: MD->param_type_end());
5002 } else {
5003 const auto *FPT = cast<const FunctionProtoType *>(Val&: Prototype.P);
5004 IsVariadic = FPT->isVariadic();
5005 ExplicitCC = FPT->getExtInfo().getCC();
5006 ArgTypes.assign(in_start: FPT->param_type_begin() + ParamsToSkip,
5007 in_end: FPT->param_type_end());
5008 }
5009
5010#ifndef NDEBUG
5011 // Check that the prototyped types match the argument expression types.
5012 bool isGenericMethod = MD && isObjCMethodWithTypeParams(MD);
5013 CallExpr::const_arg_iterator Arg = ArgRange.begin();
5014 for (QualType Ty : ArgTypes) {
5015 assert(Arg != ArgRange.end() && "Running over edge of argument list!");
5016 QualType ParamTy = Ty.getNonReferenceType();
5017 QualType ArgTy = (*Arg)->getType();
5018 if (const auto *OBT = ParamTy->getAs<OverflowBehaviorType>())
5019 ParamTy = OBT->getUnderlyingType();
5020 if (const auto *OBT = ArgTy->getAs<OverflowBehaviorType>())
5021 ArgTy = OBT->getUnderlyingType();
5022 assert((isGenericMethod || Ty->isVariablyModifiedType() ||
5023 ParamTy->isObjCRetainableType() ||
5024 getContext().getCanonicalType(ParamTy).getTypePtr() ==
5025 getContext().getCanonicalType(ArgTy).getTypePtr()) &&
5026 "type mismatch in call argument!");
5027 ++Arg;
5028 }
5029
5030 // Either we've emitted all the call args, or we have a call to variadic
5031 // function.
5032 assert((Arg == ArgRange.end() || IsVariadic) &&
5033 "Extra arguments in non-variadic function!");
5034#endif
5035 }
5036
5037 // If we still have any arguments, emit them using the type of the argument.
5038 for (auto *A : llvm::drop_begin(RangeOrContainer&: ArgRange, N: ArgTypes.size()))
5039 ArgTypes.push_back(Elt: IsVariadic ? getVarArgType(Arg: A) : A->getType());
5040 assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
5041
5042 // We must evaluate arguments from right to left in the MS C++ ABI,
5043 // because arguments are destroyed left to right in the callee. As a special
5044 // case, there are certain language constructs that require left-to-right
5045 // evaluation, and in those cases we consider the evaluation order requirement
5046 // to trump the "destruction order is reverse construction order" guarantee.
5047 bool LeftToRight =
5048 CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
5049 ? Order == EvaluationOrder::ForceLeftToRight
5050 : Order != EvaluationOrder::ForceRightToLeft;
5051
5052 auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
5053 RValue EmittedArg) {
5054 if (!AC.hasFunctionDecl() || I >= AC.getNumParams())
5055 return;
5056 auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
5057 if (PS == nullptr)
5058 return;
5059
5060 const auto &Context = getContext();
5061 auto SizeTy = Context.getSizeType();
5062 auto T = Builder.getIntNTy(N: Context.getTypeSize(T: SizeTy));
5063 assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
5064 llvm::Value *V = evaluateOrEmitBuiltinObjectSize(
5065 E: Arg, Type: PS->getType(), ResType: T, EmittedE: EmittedArg.getScalarVal(), IsDynamic: PS->isDynamic());
5066 Args.add(rvalue: RValue::get(V), type: SizeTy);
5067 // If we're emitting args in reverse, be sure to do so with
5068 // pass_object_size, as well.
5069 if (!LeftToRight)
5070 std::swap(a&: Args.back(), b&: *(&Args.back() - 1));
5071 };
5072
5073 // Insert a stack save if we're going to need any inalloca args.
5074 if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
5075 assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
5076 "inalloca only supported on x86");
5077 Args.allocateArgumentMemory(CGF&: *this);
5078 }
5079
5080 // Evaluate each argument in the appropriate order.
5081 size_t CallArgsStart = Args.size();
5082 for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
5083 unsigned Idx = LeftToRight ? I : E - I - 1;
5084 CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
5085 unsigned InitialArgSize = Args.size();
5086 // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
5087 // the argument and parameter match or the objc method is parameterized.
5088 assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
5089 getContext().hasSameUnqualifiedType((*Arg)->getType(),
5090 ArgTypes[Idx]) ||
5091 (isa<ObjCMethodDecl>(AC.getDecl()) &&
5092 isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
5093 "Argument and parameter types don't match");
5094 EmitCallArg(args&: Args, E: *Arg, ArgType: ArgTypes[Idx]);
5095 // In particular, we depend on it being the last arg in Args, and the
5096 // objectsize bits depend on there only being one arg if !LeftToRight.
5097 assert(InitialArgSize + 1 == Args.size() &&
5098 "The code below depends on only adding one arg per EmitCallArg");
5099 (void)InitialArgSize;
5100 // Since pointer argument are never emitted as LValue, it is safe to emit
5101 // non-null argument check for r-value only.
5102 if (!Args.back().hasLValue()) {
5103 RValue RVArg = Args.back().getKnownRValue();
5104 EmitNonNullArgCheck(RV: RVArg, ArgType: ArgTypes[Idx], ArgLoc: (*Arg)->getExprLoc(), AC,
5105 ParmNum: ParamsToSkip + Idx);
5106 // @llvm.objectsize should never have side-effects and shouldn't need
5107 // destruction/cleanups, so we can safely "emit" it after its arg,
5108 // regardless of right-to-leftness
5109 MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
5110 }
5111 }
5112
5113 if (!LeftToRight) {
5114 // Un-reverse the arguments we just evaluated so they match up with the LLVM
5115 // IR function.
5116 std::reverse(first: Args.begin() + CallArgsStart, last: Args.end());
5117
5118 // Reverse the writebacks to match the MSVC ABI.
5119 Args.reverseWritebacks();
5120 }
5121}
5122
5123namespace {
5124
5125struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
5126 DestroyUnpassedArg(Address Addr, QualType Ty) : Addr(Addr), Ty(Ty) {}
5127
5128 Address Addr;
5129 QualType Ty;
5130
5131 void Emit(CodeGenFunction &CGF, Flags flags) override {
5132 QualType::DestructionKind DtorKind = Ty.isDestructedType();
5133 if (DtorKind == QualType::DK_cxx_destructor) {
5134 const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
5135 assert(!Dtor->isTrivial());
5136 CGF.EmitCXXDestructorCall(D: Dtor, Type: Dtor_Complete, /*for vbase*/ ForVirtualBase: false,
5137 /*Delegating=*/false, This: Addr, ThisTy: Ty);
5138 } else {
5139 CGF.callCStructDestructor(Dst: CGF.MakeAddrLValue(Addr, T: Ty));
5140 }
5141 }
5142};
5143
5144} // end anonymous namespace
5145
5146RValue CallArg::getRValue(CodeGenFunction &CGF) const {
5147 if (!HasLV)
5148 return RV;
5149 LValue Copy = CGF.MakeAddrLValue(Addr: CGF.CreateMemTempWithoutCast(T: Ty), T: Ty);
5150 CGF.EmitAggregateCopy(Dest: Copy, Src: LV, EltTy: Ty, MayOverlap: AggValueSlot::DoesNotOverlap,
5151 isVolatile: LV.isVolatile());
5152 IsUsed = true;
5153 return RValue::getAggregate(addr: Copy.getAddress());
5154}
5155
5156void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
5157 LValue Dst = CGF.MakeAddrLValue(Addr, T: Ty);
5158 if (!HasLV && RV.isScalar())
5159 CGF.EmitStoreOfScalar(value: RV.getScalarVal(), lvalue: Dst, /*isInit=*/true);
5160 else if (!HasLV && RV.isComplex())
5161 CGF.EmitStoreOfComplex(V: RV.getComplexVal(), dest: Dst, /*init=*/isInit: true);
5162 else {
5163 auto Addr = HasLV ? LV.getAddress() : RV.getAggregateAddress();
5164 LValue SrcLV = CGF.MakeAddrLValue(Addr, T: Ty);
5165 // We assume that call args are never copied into subobjects.
5166 CGF.EmitAggregateCopy(Dest: Dst, Src: SrcLV, EltTy: Ty, MayOverlap: AggValueSlot::DoesNotOverlap,
5167 isVolatile: HasLV ? LV.isVolatileQualified()
5168 : RV.isVolatileQualified());
5169 }
5170 IsUsed = true;
5171}
5172
5173void CodeGenFunction::EmitWritebacks(const CallArgList &args) {
5174 for (const auto &I : args.writebacks())
5175 emitWriteback(CGF&: *this, writeback: I);
5176}
5177
5178void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
5179 QualType type) {
5180 std::optional<DisableDebugLocationUpdates> Dis;
5181 if (isa<CXXDefaultArgExpr>(Val: E))
5182 Dis.emplace(args&: *this);
5183 if (const ObjCIndirectCopyRestoreExpr *CRE =
5184 dyn_cast<ObjCIndirectCopyRestoreExpr>(Val: E)) {
5185 assert(getLangOpts().ObjCAutoRefCount);
5186 return emitWritebackArg(CGF&: *this, args, CRE);
5187 }
5188
5189 // Add writeback for HLSLOutParamExpr.
5190 // Needs to be before the assert below because HLSLOutArgExpr is an LValue
5191 // and is not a reference.
5192 if (const HLSLOutArgExpr *OE = dyn_cast<HLSLOutArgExpr>(Val: E)) {
5193 EmitHLSLOutArgExpr(E: OE, Args&: args, Ty: type);
5194 return;
5195 }
5196
5197 assert(type->isReferenceType() == E->isGLValue() &&
5198 "reference binding to unmaterialized r-value!");
5199
5200 if (E->isGLValue()) {
5201 assert(E->getObjectKind() == OK_Ordinary);
5202 return args.add(rvalue: EmitReferenceBindingToExpr(E), type);
5203 }
5204
5205 bool HasAggregateEvalKind = hasAggregateEvaluationKind(T: type);
5206
5207 // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
5208 // However, we still have to push an EH-only cleanup in case we unwind before
5209 // we make it to the call.
5210 if (type->isRecordType() &&
5211 type->castAsRecordDecl()->isParamDestroyedInCallee()) {
5212 // If we're using inalloca, use the argument memory. Otherwise, use a
5213 // temporary.
5214 AggValueSlot Slot = args.isUsingInAlloca()
5215 ? createPlaceholderSlot(CGF&: *this, Ty: type)
5216 : CreateAggTemp(T: type, Name: "agg.tmp");
5217
5218 bool DestroyedInCallee = true, NeedsCleanup = true;
5219 if (const auto *RD = type->getAsCXXRecordDecl())
5220 DestroyedInCallee = RD->hasNonTrivialDestructor();
5221 else
5222 NeedsCleanup = type.isDestructedType();
5223
5224 if (DestroyedInCallee)
5225 Slot.setExternallyDestructed();
5226
5227 EmitAggExpr(E, AS: Slot);
5228 RValue RV = Slot.asRValue();
5229 args.add(rvalue: RV, type);
5230
5231 if (DestroyedInCallee && NeedsCleanup) {
5232 // Create a no-op GEP between the placeholder and the cleanup so we can
5233 // RAUW it successfully. It also serves as a marker of the first
5234 // instruction where the cleanup is active.
5235 pushFullExprCleanup<DestroyUnpassedArg>(kind: NormalAndEHCleanup,
5236 A: Slot.getAddress(), A: type);
5237 // This unreachable is a temporary marker which will be removed later.
5238 llvm::Instruction *IsActive =
5239 Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
5240 args.addArgCleanupDeactivation(Cleanup: EHStack.stable_begin(), IsActiveIP: IsActive);
5241 }
5242 return;
5243 }
5244
5245 if (HasAggregateEvalKind) {
5246 auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E);
5247 if (ICE && ICE->getCastKind() == CK_LValueToRValue &&
5248 ICE->getSubExpr()->getType().getAddressSpace() !=
5249 LangAS::hlsl_constant &&
5250 !type->isArrayParameterType() && !type.isNonTrivialToPrimitiveCopy()) {
5251 LValue L = EmitLValue(E: cast<CastExpr>(Val: E)->getSubExpr());
5252 assert(L.isSimple());
5253 args.addUncopiedAggregate(LV: L, type);
5254 return;
5255 }
5256 }
5257
5258 args.add(rvalue: EmitAnyExprToTemp(E), type);
5259}
5260
5261QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
5262 // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
5263 // implicitly widens null pointer constants that are arguments to varargs
5264 // functions to pointer-sized ints.
5265 if (!getTarget().getTriple().isOSWindows())
5266 return Arg->getType();
5267
5268 if (Arg->getType()->isIntegerType() &&
5269 getContext().getTypeSize(T: Arg->getType()) <
5270 getContext().getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default) &&
5271 Arg->isNullPointerConstant(Ctx&: getContext(),
5272 NPC: Expr::NPC_ValueDependentIsNotNull)) {
5273 return getContext().getIntPtrType();
5274 }
5275
5276 return Arg->getType();
5277}
5278
5279// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5280// optimizer it can aggressively ignore unwind edges.
5281void CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
5282 if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
5283 !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
5284 Inst->setMetadata(Kind: "clang.arc.no_objc_arc_exceptions",
5285 Node: CGM.getNoObjCARCExceptionsMetadata());
5286}
5287
5288/// Emits a call to the given no-arguments nounwind runtime function.
5289llvm::CallInst *
5290CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5291 const llvm::Twine &name) {
5292 return EmitNounwindRuntimeCall(callee, args: ArrayRef<llvm::Value *>(), name);
5293}
5294
5295/// Emits a call to the given nounwind runtime function.
5296llvm::CallInst *
5297CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5298 ArrayRef<Address> args,
5299 const llvm::Twine &name) {
5300 SmallVector<llvm::Value *, 3> values;
5301 for (auto arg : args)
5302 values.push_back(Elt: arg.emitRawPointer(CGF&: *this));
5303 return EmitNounwindRuntimeCall(callee, args: values, name);
5304}
5305
5306llvm::CallInst *
5307CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5308 ArrayRef<llvm::Value *> args,
5309 const llvm::Twine &name) {
5310 llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
5311 call->setDoesNotThrow();
5312 return call;
5313}
5314
5315/// Emits a simple call (never an invoke) to the given no-arguments
5316/// runtime function.
5317llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5318 const llvm::Twine &name) {
5319 return EmitRuntimeCall(callee, args: {}, name);
5320}
5321
5322// Calls which may throw must have operand bundles indicating which funclet
5323// they are nested within.
5324SmallVector<llvm::OperandBundleDef, 1>
5325CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
5326 // There is no need for a funclet operand bundle if we aren't inside a
5327 // funclet.
5328 if (!CurrentFuncletPad)
5329 return (SmallVector<llvm::OperandBundleDef, 1>());
5330
5331 // Skip intrinsics which cannot throw (as long as they don't lower into
5332 // regular function calls in the course of IR transformations).
5333 if (auto *CalleeFn = dyn_cast<llvm::Function>(Val: Callee->stripPointerCasts())) {
5334 if (CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) {
5335 auto IID = CalleeFn->getIntrinsicID();
5336 if (!llvm::IntrinsicInst::mayLowerToFunctionCall(IID))
5337 return (SmallVector<llvm::OperandBundleDef, 1>());
5338 }
5339 }
5340
5341 SmallVector<llvm::OperandBundleDef, 1> BundleList;
5342 BundleList.emplace_back(Args: "funclet", Args&: CurrentFuncletPad);
5343 return BundleList;
5344}
5345
5346/// Emits a simple call (never an invoke) to the given runtime function.
5347llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5348 ArrayRef<llvm::Value *> args,
5349 const llvm::Twine &name) {
5350 llvm::CallInst *call = Builder.CreateCall(
5351 Callee: callee, Args: args, OpBundles: getBundlesForFunclet(Callee: callee.getCallee()), Name: name);
5352 call->setCallingConv(getRuntimeCC());
5353
5354 if (CGM.shouldEmitConvergenceTokens() && call->isConvergent())
5355 return cast<llvm::CallInst>(Val: addConvergenceControlToken(Input: call));
5356 return call;
5357}
5358
5359llvm::CallInst *CodeGenFunction::EmitIntrinsicCall(llvm::Intrinsic::ID ID,
5360 const llvm::Twine &Name) {
5361 return EmitIntrinsicCall(ID, Types: {}, Args: {}, Name);
5362}
5363
5364llvm::CallInst *CodeGenFunction::EmitIntrinsicCall(llvm::Intrinsic::ID ID,
5365 ArrayRef<llvm::Value *> Args,
5366 const llvm::Twine &Name) {
5367 return EmitIntrinsicCall(ID, Types: {}, Args, Name);
5368}
5369
5370llvm::CallInst *CodeGenFunction::EmitIntrinsicCall(llvm::Intrinsic::ID ID,
5371 ArrayRef<llvm::Type *> Types,
5372 ArrayRef<llvm::Value *> Args,
5373 const llvm::Twine &Name) {
5374 llvm::Function *F =
5375 llvm::Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID, OverloadTys: Types);
5376 llvm::CallInst *Call =
5377 Builder.CreateCall(Callee: F, Args, OpBundles: getBundlesForFunclet(Callee: F), Name);
5378 if (CGM.shouldEmitConvergenceTokens() && Call->isConvergent())
5379 return cast<llvm::CallInst>(Val: addConvergenceControlToken(Input: Call));
5380 return Call;
5381}
5382
5383/// Emits a call or invoke to the given noreturn runtime function.
5384void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
5385 llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
5386 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5387 getBundlesForFunclet(Callee: callee.getCallee());
5388
5389 if (getInvokeDest()) {
5390 llvm::InvokeInst *invoke = Builder.CreateInvoke(
5391 Callee: callee, NormalDest: getUnreachableBlock(), UnwindDest: getInvokeDest(), Args: args, OpBundles: BundleList);
5392 invoke->setDoesNotReturn();
5393 invoke->setCallingConv(getRuntimeCC());
5394 } else {
5395 llvm::CallInst *call = Builder.CreateCall(Callee: callee, Args: args, OpBundles: BundleList);
5396 call->setDoesNotReturn();
5397 call->setCallingConv(getRuntimeCC());
5398 Builder.CreateUnreachable();
5399 }
5400}
5401
5402/// Emits a call or invoke instruction to the given nullary runtime function.
5403llvm::CallBase *
5404CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5405 const Twine &name) {
5406 return EmitRuntimeCallOrInvoke(callee, args: {}, name);
5407}
5408
5409/// Emits a call or invoke instruction to the given runtime function.
5410llvm::CallBase *
5411CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5412 ArrayRef<llvm::Value *> args,
5413 const Twine &name) {
5414 llvm::CallBase *call = EmitCallOrInvoke(Callee: callee, Args: args, Name: name);
5415 call->setCallingConv(getRuntimeCC());
5416 return call;
5417}
5418
5419/// Emits a call or invoke instruction to the given function, depending
5420/// on the current state of the EH stack.
5421llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
5422 ArrayRef<llvm::Value *> Args,
5423 const Twine &Name) {
5424 llvm::BasicBlock *InvokeDest = getInvokeDest();
5425 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5426 getBundlesForFunclet(Callee: Callee.getCallee());
5427
5428 llvm::CallBase *Inst;
5429 if (!InvokeDest)
5430 Inst = Builder.CreateCall(Callee, Args, OpBundles: BundleList, Name);
5431 else {
5432 llvm::BasicBlock *ContBB = createBasicBlock(name: "invoke.cont");
5433 Inst = Builder.CreateInvoke(Callee, NormalDest: ContBB, UnwindDest: InvokeDest, Args, OpBundles: BundleList,
5434 Name);
5435 EmitBlock(BB: ContBB);
5436 }
5437
5438 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5439 // optimizer it can aggressively ignore unwind edges.
5440 if (CGM.getLangOpts().ObjCAutoRefCount)
5441 AddObjCARCExceptionMetadata(Inst);
5442
5443 return Inst;
5444}
5445
5446void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
5447 llvm::Value *New) {
5448 DeferredReplacements.push_back(
5449 Elt: std::make_pair(x: llvm::WeakTrackingVH(Old), y&: New));
5450}
5451
5452namespace {
5453
5454/// Specify given \p NewAlign as the alignment of return value attribute. If
5455/// such attribute already exists, re-set it to the maximal one of two options.
5456[[nodiscard]] llvm::AttributeList
5457maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
5458 const llvm::AttributeList &Attrs,
5459 llvm::Align NewAlign) {
5460 llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
5461 if (CurAlign >= NewAlign)
5462 return Attrs;
5463 llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Context&: Ctx, Alignment: NewAlign);
5464 return Attrs.removeRetAttribute(C&: Ctx, Kind: llvm::Attribute::AttrKind::Alignment)
5465 .addRetAttribute(C&: Ctx, Attr: AlignAttr);
5466}
5467
5468template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
5469protected:
5470 CodeGenFunction &CGF;
5471
5472 /// We do nothing if this is, or becomes, nullptr.
5473 const AlignedAttrTy *AA = nullptr;
5474
5475 llvm::Value *Alignment = nullptr; // May or may not be a constant.
5476 llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
5477
5478 AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5479 : CGF(CGF_) {
5480 if (!FuncDecl)
5481 return;
5482 AA = FuncDecl->getAttr<AlignedAttrTy>();
5483 }
5484
5485public:
5486 /// If we can, materialize the alignment as an attribute on return value.
5487 [[nodiscard]] llvm::AttributeList
5488 TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
5489 if (!AA || OffsetCI || CGF.SanOpts.has(K: SanitizerKind::Alignment))
5490 return Attrs;
5491 const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Val: Alignment);
5492 if (!AlignmentCI)
5493 return Attrs;
5494 // We may legitimately have non-power-of-2 alignment here.
5495 // If so, this is UB land, emit it via `@llvm.assume` instead.
5496 if (!AlignmentCI->getValue().isPowerOf2())
5497 return Attrs;
5498 llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
5499 Ctx&: CGF.getLLVMContext(), Attrs,
5500 NewAlign: llvm::Align(
5501 AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment)));
5502 AA = nullptr; // We're done. Disallow doing anything else.
5503 return NewAttrs;
5504 }
5505
5506 /// Emit alignment assumption.
5507 /// This is a general fallback that we take if either there is an offset,
5508 /// or the alignment is variable or we are sanitizing for alignment.
5509 void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
5510 if (!AA)
5511 return;
5512 CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
5513 AA->getLocation(), Alignment, OffsetCI);
5514 AA = nullptr; // We're done. Disallow doing anything else.
5515 }
5516};
5517
5518/// Helper data structure to emit `AssumeAlignedAttr`.
5519class AssumeAlignedAttrEmitter final
5520 : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
5521public:
5522 AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5523 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5524 if (!AA)
5525 return;
5526 // It is guaranteed that the alignment/offset are constants.
5527 Alignment = cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: AA->getAlignment()));
5528 if (Expr *Offset = AA->getOffset()) {
5529 OffsetCI = cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: Offset));
5530 if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
5531 OffsetCI = nullptr;
5532 }
5533 }
5534};
5535
5536/// Helper data structure to emit `AllocAlignAttr`.
5537class AllocAlignAttrEmitter final
5538 : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
5539public:
5540 AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
5541 const CallArgList &CallArgs)
5542 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5543 if (!AA)
5544 return;
5545 // Alignment may or may not be a constant, and that is okay.
5546 Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
5547 .getRValue(CGF)
5548 .getScalarVal();
5549 }
5550};
5551
5552} // namespace
5553
5554static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
5555 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Ty))
5556 return VT->getPrimitiveSizeInBits().getKnownMinValue();
5557 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty))
5558 return getMaxVectorWidth(Ty: AT->getElementType());
5559
5560 unsigned MaxVectorWidth = 0;
5561 if (auto *ST = dyn_cast<llvm::StructType>(Val: Ty))
5562 for (auto *I : ST->elements())
5563 MaxVectorWidth = std::max(a: MaxVectorWidth, b: getMaxVectorWidth(Ty: I));
5564 return MaxVectorWidth;
5565}
5566
5567RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
5568 const CGCallee &Callee,
5569 ReturnValueSlot ReturnValue,
5570 const CallArgList &CallArgs,
5571 llvm::CallBase **callOrInvoke, bool IsMustTail,
5572 SourceLocation Loc,
5573 bool IsVirtualFunctionPointerThunk) {
5574 // FIXME: We no longer need the types from CallArgs; lift up and simplify.
5575
5576 assert(Callee.isOrdinary() || Callee.isVirtual());
5577
5578 // Handle struct-return functions by passing a pointer to the
5579 // location that we would like to return into.
5580 QualType RetTy = CallInfo.getReturnType();
5581 const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
5582
5583 llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(FI: CallInfo);
5584
5585 const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
5586 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
5587 // We can only guarantee that a function is called from the correct
5588 // context/function based on the appropriate target attributes,
5589 // so only check in the case where we have both always_inline and target
5590 // since otherwise we could be making a conditional call after a check for
5591 // the proper cpu features (and it won't cause code generation issues due to
5592 // function based code generation).
5593 if ((TargetDecl->hasAttr<AlwaysInlineAttr>() &&
5594 (TargetDecl->hasAttr<TargetAttr>() ||
5595 (CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>()))) ||
5596 (CurFuncDecl && CurFuncDecl->hasAttr<FlattenAttr>() &&
5597 (CurFuncDecl->hasAttr<TargetAttr>() ||
5598 TargetDecl->hasAttr<TargetAttr>())))
5599 checkTargetFeatures(Loc, TargetDecl: FD);
5600 }
5601
5602 // Some architectures (such as x86-64) have the ABI changed based on
5603 // attribute-target/features. Give them a chance to diagnose.
5604 const FunctionDecl *CallerDecl = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
5605 const FunctionDecl *CalleeDecl = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
5606 CGM.getTargetCodeGenInfo().checkFunctionCallABI(CGM, CallLoc: Loc, Caller: CallerDecl,
5607 Callee: CalleeDecl, Args: CallArgs, ReturnType: RetTy);
5608
5609 // 1. Set up the arguments.
5610
5611 // If we're using inalloca, insert the allocation after the stack save.
5612 // FIXME: Do this earlier rather than hacking it in here!
5613 RawAddress ArgMemory = RawAddress::invalid();
5614 if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
5615 const llvm::DataLayout &DL = CGM.getDataLayout();
5616 llvm::Instruction *IP = CallArgs.getStackBase();
5617 llvm::AllocaInst *AI;
5618 if (IP) {
5619 IP = IP->getNextNode();
5620 AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(), "argmem",
5621 IP->getIterator());
5622 } else {
5623 AI = CreateTempAlloca(Ty: ArgStruct, Name: "argmem");
5624 }
5625 auto Align = CallInfo.getArgStructAlignment();
5626 AI->setAlignment(Align.getAsAlign());
5627 AI->setUsedWithInAlloca(true);
5628 assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
5629 ArgMemory = RawAddress(AI, ArgStruct, Align);
5630 }
5631
5632 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
5633 SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
5634
5635 // If the call returns a temporary with struct return, create a temporary
5636 // alloca to hold the result, unless one is given to us.
5637 Address SRetPtr = Address::invalid();
5638 // Original alloca for lifetime markers
5639 Address SRetAlloca = Address::invalid();
5640 bool NeedSRetLifetimeEnd = false;
5641 if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
5642 // For virtual function pointer thunks and musttail calls, we must always
5643 // forward an incoming SRet pointer to the callee, because a local alloca
5644 // would be de-allocated before the call. These cases both guarantee that
5645 // there will be an incoming SRet argument of the correct type.
5646 if ((IsVirtualFunctionPointerThunk || IsMustTail) && RetAI.isIndirect()) {
5647 SRetPtr = makeNaturalAddressForPointer(Ptr: CurFn->arg_begin() +
5648 IRFunctionArgs.getSRetArgNo(),
5649 T: RetTy, Alignment: CharUnits::fromQuantity(Quantity: 1));
5650 } else if (!ReturnValue.isNull()) {
5651 SRetPtr = ReturnValue.getAddress();
5652 } else {
5653 SRetPtr = CreateMemTempWithoutCast(T: RetTy, Name: "tmp");
5654 if (HaveInsertPoint() && ReturnValue.isUnused()) {
5655 NeedSRetLifetimeEnd = EmitLifetimeStart(Addr: SRetPtr.getBasePointer());
5656 if (NeedSRetLifetimeEnd)
5657 SRetAlloca = SRetPtr;
5658 }
5659 }
5660 if (IRFunctionArgs.hasSRetArg()) {
5661 // A mismatch between the allocated return value's AS and the target's
5662 // chosen IndirectAS can happen e.g. when passing the this pointer through
5663 // a chain involving stores to / loads from the DefaultAS; we address this
5664 // here, symmetrically with the handling we have for normal pointer args.
5665 if (SRetPtr.getAddressSpace() != RetAI.getIndirectAddrSpace()) {
5666 llvm::Value *V = SRetPtr.getBasePointer();
5667 llvm::Type *Ty = llvm::PointerType::get(C&: getLLVMContext(),
5668 AddressSpace: RetAI.getIndirectAddrSpace());
5669
5670 SRetPtr = SRetPtr.withPointer(NewPointer: performAddrSpaceCast(Src: V, DestTy: Ty),
5671 IsKnownNonNull: SRetPtr.isKnownNonNull());
5672 }
5673 IRCallArgs[IRFunctionArgs.getSRetArgNo()] =
5674 getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy);
5675 } else if (RetAI.isInAlloca()) {
5676 Address Addr =
5677 Builder.CreateStructGEP(Addr: ArgMemory, Index: RetAI.getInAllocaFieldIndex());
5678 Builder.CreateStore(Val: getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy), Addr);
5679 }
5680 }
5681
5682 RawAddress swiftErrorTemp = RawAddress::invalid();
5683 Address swiftErrorArg = Address::invalid();
5684
5685 // When passing arguments using temporary allocas, we need to add the
5686 // appropriate lifetime markers. This vector keeps track of all the lifetime
5687 // markers that need to be ended right after the call.
5688 SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
5689
5690 // Translate all of the arguments as necessary to match the IR lowering.
5691 assert(CallInfo.arg_size() == CallArgs.size() &&
5692 "Mismatch between function signature & arguments.");
5693 unsigned ArgNo = 0;
5694 CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
5695 for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
5696 I != E; ++I, ++info_it, ++ArgNo) {
5697 const ABIArgInfo &ArgInfo = info_it->info;
5698
5699 // Insert a padding argument to ensure proper alignment.
5700 if (IRFunctionArgs.hasPaddingArg(ArgNo))
5701 IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
5702 llvm::UndefValue::get(T: ArgInfo.getPaddingType());
5703
5704 unsigned FirstIRArg, NumIRArgs;
5705 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
5706
5707 bool ArgHasMaybeUndefAttr =
5708 IsArgumentMaybeUndef(TargetDecl, NumRequiredArgs: CallInfo.getNumRequiredArgs(), ArgNo);
5709
5710 switch (ArgInfo.getKind()) {
5711 case ABIArgInfo::InAlloca: {
5712 assert(NumIRArgs == 0);
5713 assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
5714 if (I->isAggregate()) {
5715 RawAddress Addr = I->hasLValue()
5716 ? I->getKnownLValue().getAddress()
5717 : I->getKnownRValue().getAggregateAddress();
5718 llvm::Instruction *Placeholder =
5719 cast<llvm::Instruction>(Val: Addr.getPointer());
5720
5721 if (!ArgInfo.getInAllocaIndirect()) {
5722 // Replace the placeholder with the appropriate argument slot GEP.
5723 CGBuilderTy::InsertPoint IP = Builder.saveIP();
5724 Builder.SetInsertPoint(Placeholder);
5725 Addr = Builder.CreateStructGEP(Addr: ArgMemory,
5726 Index: ArgInfo.getInAllocaFieldIndex());
5727 Builder.restoreIP(IP);
5728 } else {
5729 // For indirect things such as overaligned structs, replace the
5730 // placeholder with a regular aggregate temporary alloca. Store the
5731 // address of this alloca into the struct.
5732 Addr =
5733 CreateMemTempWithoutCast(T: info_it->type, Name: "inalloca.indirect.tmp");
5734 Address ArgSlot = Builder.CreateStructGEP(
5735 Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5736 Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5737 }
5738 deferPlaceholderReplacement(Old: Placeholder, New: Addr.getPointer());
5739 } else if (ArgInfo.getInAllocaIndirect()) {
5740 // Make a temporary alloca and store the address of it into the argument
5741 // struct.
5742 RawAddress Addr = CreateMemTempWithoutCast(
5743 T: I->Ty, Align: getContext().getTypeAlignInChars(T: I->Ty),
5744 Name: "indirect-arg-temp");
5745 I->copyInto(CGF&: *this, Addr);
5746 Address ArgSlot =
5747 Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5748 Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5749 } else {
5750 // Store the RValue into the argument struct.
5751 Address Addr =
5752 Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5753 Addr = Addr.withElementType(ElemTy: ConvertTypeForMem(T: I->Ty));
5754 I->copyInto(CGF&: *this, Addr);
5755 }
5756 break;
5757 }
5758
5759 case ABIArgInfo::Indirect:
5760 case ABIArgInfo::IndirectAliased: {
5761 assert(NumIRArgs == 1);
5762 if (I->isAggregate()) {
5763 // We want to avoid creating an unnecessary temporary+copy here;
5764 // however, we need one in three cases:
5765 // 1. If the argument is not byval, and we are required to copy the
5766 // source. (This case doesn't occur on any common architecture.)
5767 // 2. If the argument is byval, RV is not sufficiently aligned, and
5768 // we cannot force it to be sufficiently aligned.
5769 // 3. If the argument is byval, but RV is not located in default
5770 // or alloca address space.
5771 Address Addr = I->hasLValue()
5772 ? I->getKnownLValue().getAddress()
5773 : I->getKnownRValue().getAggregateAddress();
5774 CharUnits Align = ArgInfo.getIndirectAlign();
5775 const llvm::DataLayout *TD = &CGM.getDataLayout();
5776
5777 assert((FirstIRArg >= IRFuncTy->getNumParams() ||
5778 IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
5779 TD->getAllocaAddrSpace()) &&
5780 "indirect argument must be in alloca address space");
5781
5782 bool NeedCopy = false;
5783 if (Addr.getAlignment() < Align &&
5784 llvm::getOrEnforceKnownAlignment(V: Addr.emitRawPointer(CGF&: *this),
5785 PrefAlign: Align.getAsAlign(),
5786 DL: *TD) < Align.getAsAlign()) {
5787 NeedCopy = true;
5788 } else if (I->hasLValue()) {
5789 auto LV = I->getKnownLValue();
5790
5791 bool isByValOrRef =
5792 ArgInfo.isIndirectAliased() || ArgInfo.getIndirectByVal();
5793
5794 if (!isByValOrRef ||
5795 (LV.getAlignment() < getContext().getTypeAlignInChars(T: I->Ty))) {
5796 NeedCopy = true;
5797 }
5798
5799 if (isByValOrRef && Addr.getType()->getAddressSpace() !=
5800 ArgInfo.getIndirectAddrSpace()) {
5801 NeedCopy = true;
5802 }
5803 }
5804
5805 if (!NeedCopy) {
5806 // Skip the extra memcpy call.
5807 llvm::Value *V = getAsNaturalPointerTo(Addr, PointeeType: I->Ty);
5808 auto *T = llvm::PointerType::get(C&: CGM.getLLVMContext(),
5809 AddressSpace: ArgInfo.getIndirectAddrSpace());
5810
5811 // FIXME: This should not depend on the language address spaces, and
5812 // only the contextual values. If the address space mismatches, see if
5813 // we can look through a cast to a compatible address space value,
5814 // otherwise emit a copy.
5815 llvm::Value *Val = performAddrSpaceCast(Src: V, DestTy: T);
5816 if (ArgHasMaybeUndefAttr)
5817 Val = Builder.CreateFreeze(V: Val);
5818 IRCallArgs[FirstIRArg] = Val;
5819 break;
5820 }
5821 } else if (I->getType()->isArrayParameterType()) {
5822 // Don't produce a temporary for ArrayParameterType arguments.
5823 // ArrayParameterType arguments are only created from
5824 // HLSL_ArrayRValue casts and HLSLOutArgExpr expressions, both
5825 // of which create temporaries already. This allows us to just use the
5826 // scalar for the decayed array pointer as the argument directly.
5827 IRCallArgs[FirstIRArg] = I->getKnownRValue().getScalarVal();
5828 break;
5829 }
5830
5831 // For non-aggregate args and aggregate args meeting conditions above
5832 // we need to create an aligned temporary, and copy to it.
5833 RawAddress AI = CreateMemTempWithoutCast(
5834 T: I->Ty, Align: ArgInfo.getIndirectAlign(), Name: "byval-temp");
5835 llvm::Value *Val = getAsNaturalPointerTo(Addr: AI, PointeeType: I->Ty);
5836 if (ArgHasMaybeUndefAttr)
5837 Val = Builder.CreateFreeze(V: Val);
5838 IRCallArgs[FirstIRArg] = Val;
5839
5840 // Emit lifetime markers for the temporary alloca and add cleanup code to
5841 // emit the end lifetime marker after the call.
5842 if (EmitLifetimeStart(Addr: AI.getPointer()))
5843 CallLifetimeEndAfterCall.emplace_back(Args&: AI);
5844
5845 // Generate the copy.
5846 I->copyInto(CGF&: *this, Addr: AI);
5847 break;
5848 }
5849
5850 case ABIArgInfo::Ignore:
5851 assert(NumIRArgs == 0);
5852 break;
5853
5854 case ABIArgInfo::Extend:
5855 case ABIArgInfo::Direct: {
5856 if (!isa<llvm::StructType>(Val: ArgInfo.getCoerceToType()) &&
5857 ArgInfo.getCoerceToType() == ConvertType(T: info_it->type) &&
5858 ArgInfo.getDirectOffset() == 0) {
5859 assert(NumIRArgs == 1);
5860 llvm::Value *V;
5861 if (!I->isAggregate())
5862 V = I->getKnownRValue().getScalarVal();
5863 else
5864 V = Builder.CreateLoad(
5865 Addr: I->hasLValue() ? I->getKnownLValue().getAddress()
5866 : I->getKnownRValue().getAggregateAddress());
5867
5868 // Implement swifterror by copying into a new swifterror argument.
5869 // We'll write back in the normal path out of the call.
5870 if (CallInfo.getExtParameterInfo(argIndex: ArgNo).getABI() ==
5871 ParameterABI::SwiftErrorResult) {
5872 assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
5873
5874 QualType pointeeTy = I->Ty->getPointeeType();
5875 swiftErrorArg = makeNaturalAddressForPointer(
5876 Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
5877
5878 swiftErrorTemp = CreateMemTempWithoutCast(
5879 T: pointeeTy, Align: getPointerAlign(), Name: "swifterror.temp");
5880 V = swiftErrorTemp.getPointer();
5881 cast<llvm::AllocaInst>(Val: V)->setSwiftError(true);
5882
5883 llvm::Value *errorValue = Builder.CreateLoad(Addr: swiftErrorArg);
5884 Builder.CreateStore(Val: errorValue, Addr: swiftErrorTemp);
5885 }
5886
5887 // We might have to widen integers, but we should never truncate.
5888 if (ArgInfo.getCoerceToType() != V->getType() &&
5889 V->getType()->isIntegerTy())
5890 V = Builder.CreateZExt(V, DestTy: ArgInfo.getCoerceToType());
5891
5892 // The only plausible mismatch here would be for pointer address spaces.
5893 // We assume that the target has a reasonable mapping for the DefaultAS
5894 // (it can be casted to from incoming specific ASes), and insert an AS
5895 // cast to address the mismatch.
5896 if (FirstIRArg < IRFuncTy->getNumParams() &&
5897 V->getType() != IRFuncTy->getParamType(i: FirstIRArg)) {
5898 assert(V->getType()->isPointerTy() && "Only pointers can mismatch!");
5899 V = performAddrSpaceCast(Src: V, DestTy: IRFuncTy->getParamType(i: FirstIRArg));
5900 }
5901
5902 if (ArgHasMaybeUndefAttr)
5903 V = Builder.CreateFreeze(V);
5904 IRCallArgs[FirstIRArg] = V;
5905 break;
5906 }
5907
5908 llvm::StructType *STy =
5909 dyn_cast<llvm::StructType>(Val: ArgInfo.getCoerceToType());
5910
5911 // FIXME: Avoid the conversion through memory if possible.
5912 Address Src = Address::invalid();
5913 if (!I->isAggregate()) {
5914 Src = CreateMemTempWithoutCast(T: I->Ty, Name: "coerce");
5915 I->copyInto(CGF&: *this, Addr: Src);
5916 } else {
5917 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
5918 : I->getKnownRValue().getAggregateAddress();
5919 }
5920
5921 // If the value is offset in memory, apply the offset now.
5922 Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
5923
5924 // Fast-isel and the optimizer generally like scalar values better than
5925 // FCAs, so we flatten them if this is safe to do for this argument.
5926 if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
5927 llvm::Type *SrcTy = Src.getElementType();
5928 llvm::TypeSize SrcTypeSize =
5929 CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
5930 llvm::TypeSize DstTypeSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
5931 if (SrcTypeSize.isScalable()) {
5932 assert(STy->containsHomogeneousScalableVectorTypes() &&
5933 "ABI only supports structure with homogeneous scalable vector "
5934 "type");
5935 assert(SrcTypeSize == DstTypeSize &&
5936 "Only allow non-fractional movement of structure with "
5937 "homogeneous scalable vector type");
5938 assert(NumIRArgs == STy->getNumElements());
5939
5940 llvm::Value *StoredStructValue =
5941 Builder.CreateLoad(Addr: Src, Name: Src.getName() + ".tuple");
5942 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5943 llvm::Value *Extract = Builder.CreateExtractValue(
5944 Agg: StoredStructValue, Idxs: i, Name: Src.getName() + ".extract" + Twine(i));
5945 IRCallArgs[FirstIRArg + i] = Extract;
5946 }
5947 } else {
5948 uint64_t SrcSize = SrcTypeSize.getFixedValue();
5949 uint64_t DstSize = DstTypeSize.getFixedValue();
5950 bool HasPFPFields = getContext().hasPFPFields(Ty: I->Ty);
5951
5952 // If the source type is smaller than the destination type of the
5953 // coerce-to logic, copy the source value into a temp alloca the size
5954 // of the destination type to allow loading all of it. The bits past
5955 // the source value are left undef.
5956 if (HasPFPFields || SrcSize < DstSize) {
5957 Address TempAlloca = CreateTempAlloca(Ty: STy, align: Src.getAlignment(),
5958 Name: Src.getName() + ".coerce");
5959 if (HasPFPFields) {
5960 // Structures with PFP fields require a coerced load to remove any
5961 // pointer signatures.
5962 Builder.CreateStore(
5963 Val: CreatePFPCoercedLoad(Src, SrcFETy: I->Ty, Ty: ArgInfo.getCoerceToType(),
5964 CGF&: *this),
5965 Addr: TempAlloca);
5966 } else
5967 Builder.CreateMemCpy(Dest: TempAlloca, Src, Size: SrcSize);
5968 Src = TempAlloca;
5969 } else {
5970 Src = Src.withElementType(ElemTy: STy);
5971 }
5972
5973 assert(NumIRArgs == STy->getNumElements());
5974 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5975 Address EltPtr = Builder.CreateStructGEP(Addr: Src, Index: i);
5976 llvm::Value *LI = Builder.CreateLoad(Addr: EltPtr);
5977 if (ArgHasMaybeUndefAttr)
5978 LI = Builder.CreateFreeze(V: LI);
5979 IRCallArgs[FirstIRArg + i] = LI;
5980 }
5981 }
5982 } else {
5983 // In the simple case, just pass the coerced loaded value.
5984 assert(NumIRArgs == 1);
5985 llvm::Value *Load =
5986 CreateCoercedLoad(Src, SrcFETy: I->Ty, Ty: ArgInfo.getCoerceToType(), CGF&: *this);
5987
5988 if (CallInfo.isCmseNSCall()) {
5989 // For certain parameter types, clear padding bits, as they may reveal
5990 // sensitive information.
5991 // Small struct/union types are passed as integer arrays.
5992 auto *ATy = dyn_cast<llvm::ArrayType>(Val: Load->getType());
5993 if (ATy != nullptr && isa<RecordType>(Val: I->Ty.getCanonicalType()))
5994 Load = EmitCMSEClearRecord(Src: Load, ATy, QTy: I->Ty);
5995 }
5996
5997 if (ArgHasMaybeUndefAttr)
5998 Load = Builder.CreateFreeze(V: Load);
5999 IRCallArgs[FirstIRArg] = Load;
6000 }
6001
6002 break;
6003 }
6004
6005 case ABIArgInfo::CoerceAndExpand: {
6006 auto coercionType = ArgInfo.getCoerceAndExpandType();
6007 auto layout = CGM.getDataLayout().getStructLayout(Ty: coercionType);
6008 auto unpaddedCoercionType = ArgInfo.getUnpaddedCoerceAndExpandType();
6009 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
6010
6011 Address addr = Address::invalid();
6012 RawAddress AllocaAddr = RawAddress::invalid();
6013 bool NeedLifetimeEnd = false;
6014 if (I->isAggregate()) {
6015 addr = I->hasLValue() ? I->getKnownLValue().getAddress()
6016 : I->getKnownRValue().getAggregateAddress();
6017
6018 } else {
6019 RValue RV = I->getKnownRValue();
6020 assert(RV.isScalar()); // complex should always just be direct
6021
6022 llvm::Type *scalarType = RV.getScalarVal()->getType();
6023 auto scalarAlign = CGM.getDataLayout().getPrefTypeAlign(Ty: scalarType);
6024
6025 // Materialize to a temporary.
6026 addr = CreateTempAlloca(Ty: RV.getScalarVal()->getType(),
6027 align: CharUnits::fromQuantity(Quantity: std::max(
6028 a: layout->getAlignment(), b: scalarAlign)),
6029 Name: "tmp",
6030 /*ArraySize=*/nullptr, Alloca: &AllocaAddr);
6031 NeedLifetimeEnd = EmitLifetimeStart(Addr: AllocaAddr.getPointer());
6032
6033 Builder.CreateStore(Val: RV.getScalarVal(), Addr: addr);
6034 }
6035
6036 addr = addr.withElementType(ElemTy: coercionType);
6037
6038 unsigned IRArgPos = FirstIRArg;
6039 unsigned unpaddedIndex = 0;
6040 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
6041 llvm::Type *eltType = coercionType->getElementType(N: i);
6042 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
6043 continue;
6044 Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
6045 llvm::Value *elt = CreateCoercedLoad(
6046 Src: eltAddr, SrcFETy: I->Ty,
6047 Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
6048 : unpaddedCoercionType,
6049 CGF&: *this);
6050 if (ArgHasMaybeUndefAttr)
6051 elt = Builder.CreateFreeze(V: elt);
6052 IRCallArgs[IRArgPos++] = elt;
6053 }
6054 assert(IRArgPos == FirstIRArg + NumIRArgs);
6055
6056 if (NeedLifetimeEnd)
6057 EmitLifetimeEnd(Addr: AllocaAddr.getPointer());
6058 break;
6059 }
6060
6061 case ABIArgInfo::Expand: {
6062 unsigned IRArgPos = FirstIRArg;
6063 ExpandTypeToArgs(Ty: I->Ty, Arg: *I, IRFuncTy, IRCallArgs, IRCallArgPos&: IRArgPos);
6064 assert(IRArgPos == FirstIRArg + NumIRArgs);
6065 break;
6066 }
6067
6068 case ABIArgInfo::TargetSpecific: {
6069 Address Src = Address::invalid();
6070 if (!I->isAggregate()) {
6071 Src = CreateMemTempWithoutCast(T: I->Ty, Name: "target_coerce");
6072 I->copyInto(CGF&: *this, Addr: Src);
6073 } else {
6074 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
6075 : I->getKnownRValue().getAggregateAddress();
6076 }
6077
6078 // If the value is offset in memory, apply the offset now.
6079 Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
6080 llvm::Value *Load =
6081 CGM.getABIInfo().createCoercedLoad(SrcAddr: Src, AI: ArgInfo, CGF&: *this);
6082 IRCallArgs[FirstIRArg] = Load;
6083 break;
6084 }
6085 }
6086 }
6087
6088 const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(CGF&: *this);
6089 llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
6090
6091 // If we're using inalloca, set up that argument.
6092 if (ArgMemory.isValid()) {
6093 llvm::Value *Arg = ArgMemory.getPointer();
6094 assert(IRFunctionArgs.hasInallocaArg());
6095 IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
6096 }
6097
6098 // 2. Prepare the function pointer.
6099
6100 // If the callee is a bitcast of a non-variadic function to have a
6101 // variadic function pointer type, check to see if we can remove the
6102 // bitcast. This comes up with unprototyped functions.
6103 //
6104 // This makes the IR nicer, but more importantly it ensures that we
6105 // can inline the function at -O0 if it is marked always_inline.
6106 auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
6107 llvm::Value *Ptr) -> llvm::Function * {
6108 if (!CalleeFT->isVarArg())
6109 return nullptr;
6110
6111 // Get underlying value if it's a bitcast
6112 if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Val: Ptr)) {
6113 if (CE->getOpcode() == llvm::Instruction::BitCast)
6114 Ptr = CE->getOperand(i_nocapture: 0);
6115 }
6116
6117 llvm::Function *OrigFn = dyn_cast<llvm::Function>(Val: Ptr);
6118 if (!OrigFn)
6119 return nullptr;
6120
6121 llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
6122
6123 // If the original type is variadic, or if any of the component types
6124 // disagree, we cannot remove the cast.
6125 if (OrigFT->isVarArg() ||
6126 OrigFT->getNumParams() != CalleeFT->getNumParams() ||
6127 OrigFT->getReturnType() != CalleeFT->getReturnType())
6128 return nullptr;
6129
6130 for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i)
6131 if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
6132 return nullptr;
6133
6134 return OrigFn;
6135 };
6136
6137 if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {
6138 CalleePtr = OrigFn;
6139 IRFuncTy = OrigFn->getFunctionType();
6140 }
6141
6142 // 3. Perform the actual call.
6143
6144 // Deactivate any cleanups that we're supposed to do immediately before
6145 // the call.
6146 if (!CallArgs.getCleanupsToDeactivate().empty())
6147 deactivateArgCleanupsBeforeCall(CGF&: *this, CallArgs);
6148
6149 // Update the largest vector width if any arguments have vector types.
6150 for (unsigned i = 0; i < IRCallArgs.size(); ++i)
6151 LargestVectorWidth = std::max(a: LargestVectorWidth,
6152 b: getMaxVectorWidth(Ty: IRCallArgs[i]->getType()));
6153
6154 // Compute the calling convention and attributes.
6155 unsigned CallingConv;
6156 llvm::AttributeList Attrs;
6157 CGM.ConstructAttributeList(Name: CalleePtr->getName(), FI: CallInfo,
6158 CalleeInfo: Callee.getAbstractInfo(), AttrList&: Attrs, CallingConv,
6159 /*AttrOnCallSite=*/true,
6160 /*IsThunk=*/false);
6161
6162 if (CallingConv == llvm::CallingConv::X86_VectorCall &&
6163 getTarget().getTriple().isWindowsArm64EC()) {
6164 CGM.Error(loc: Loc, error: "__vectorcall calling convention is not currently "
6165 "supported");
6166 }
6167
6168 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
6169 if (FD->hasAttr<StrictFPAttr>())
6170 // All calls within a strictfp function are marked strictfp
6171 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::StrictFP);
6172
6173 // If -ffast-math is enabled and the function is guarded by an
6174 // '__attribute__((optnone)) adjust the memory attribute so the BE emits the
6175 // library call instead of the intrinsic.
6176 if (FD->hasAttr<OptimizeNoneAttr>() && getLangOpts().FastMath)
6177 CGM.AdjustMemoryAttribute(Name: CalleePtr->getName(), CalleeInfo: Callee.getAbstractInfo(),
6178 Attrs);
6179 }
6180 // Add call-site nomerge attribute if exists.
6181 if (InNoMergeAttributedStmt)
6182 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoMerge);
6183
6184 // Add call-site noinline attribute if exists.
6185 if (InNoInlineAttributedStmt)
6186 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoInline);
6187
6188 // Add call-site always_inline attribute if exists.
6189 // Note: This corresponds to the [[clang::always_inline]] statement attribute.
6190 if (InAlwaysInlineAttributedStmt &&
6191 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
6192 Caller: CallerDecl, Callee: CalleeDecl))
6193 Attrs =
6194 Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::AlwaysInline);
6195
6196 // Remove call-site convergent attribute if requested.
6197 if (InNoConvergentAttributedStmt)
6198 Attrs =
6199 Attrs.removeFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::Convergent);
6200
6201 // Apply some call-site-specific attributes.
6202 // TODO: work this into building the attribute set.
6203
6204 // Apply always_inline to all calls within flatten functions.
6205 // FIXME: should this really take priority over __try, below?
6206 if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
6207 !InNoInlineAttributedStmt &&
6208 !(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>()) &&
6209 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
6210 Caller: CallerDecl, Callee: CalleeDecl)) {
6211 Attrs =
6212 Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::AlwaysInline);
6213 }
6214
6215 // Disable inlining inside SEH __try blocks.
6216 if (isSEHTryScope()) {
6217 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoInline);
6218 }
6219
6220 // Decide whether to use a call or an invoke.
6221 bool CannotThrow;
6222 if (currentFunctionUsesSEHTry()) {
6223 // SEH cares about asynchronous exceptions, so everything can "throw."
6224 CannotThrow = false;
6225 } else if (isCleanupPadScope() &&
6226 EHPersonality::get(CGF&: *this).isMSVCXXPersonality()) {
6227 // The MSVC++ personality will implicitly terminate the program if an
6228 // exception is thrown during a cleanup outside of a try/catch.
6229 // We don't need to model anything in IR to get this behavior.
6230 CannotThrow = true;
6231 } else {
6232 // Otherwise, nounwind call sites will never throw.
6233 CannotThrow = Attrs.hasFnAttr(Kind: llvm::Attribute::NoUnwind);
6234
6235 if (auto *FPtr = dyn_cast<llvm::Function>(Val: CalleePtr))
6236 if (FPtr->hasFnAttribute(Kind: llvm::Attribute::NoUnwind))
6237 CannotThrow = true;
6238 }
6239
6240 // If we made a temporary, be sure to clean up after ourselves. Note that we
6241 // can't depend on being inside of an ExprWithCleanups, so we need to manually
6242 // pop this cleanup later on. Being eager about this is OK, since this
6243 // temporary is 'invisible' outside of the callee.
6244 // Use the original alloca pointer (before any addrspacecast) for the
6245 // lifetime end marker, since lifetime intrinsics must reference the alloca
6246 // address space.
6247 if (NeedSRetLifetimeEnd)
6248 pushFullExprCleanup<CallLifetimeEnd>(kind: NormalEHLifetimeMarker, A: SRetAlloca);
6249
6250 llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();
6251
6252 SmallVector<llvm::OperandBundleDef, 1> BundleList =
6253 getBundlesForFunclet(Callee: CalleePtr);
6254
6255 if (SanOpts.has(K: SanitizerKind::KCFI) &&
6256 !isa_and_nonnull<FunctionDecl>(Val: TargetDecl))
6257 EmitKCFIOperandBundle(Callee: ConcreteCallee, Bundles&: BundleList);
6258
6259 // Add the pointer-authentication bundle.
6260 EmitPointerAuthOperandBundle(Info: ConcreteCallee.getPointerAuthInfo(), Bundles&: BundleList);
6261
6262 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl))
6263 if (FD->hasAttr<StrictFPAttr>())
6264 // All calls within a strictfp function are marked strictfp
6265 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::StrictFP);
6266
6267 AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
6268 Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
6269
6270 AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
6271 Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
6272
6273 // Emit the actual call/invoke instruction.
6274 llvm::CallBase *CI;
6275 if (!InvokeDest) {
6276 CI = Builder.CreateCall(FTy: IRFuncTy, Callee: CalleePtr, Args: IRCallArgs, OpBundles: BundleList);
6277 } else {
6278 llvm::BasicBlock *Cont = createBasicBlock(name: "invoke.cont");
6279 CI = Builder.CreateInvoke(Ty: IRFuncTy, Callee: CalleePtr, NormalDest: Cont, UnwindDest: InvokeDest, Args: IRCallArgs,
6280 OpBundles: BundleList);
6281 EmitBlock(BB: Cont);
6282 }
6283 if (CI->getCalledFunction() && CI->getCalledFunction()->hasName() &&
6284 CI->getCalledFunction()->getName().starts_with(Prefix: "_Z4sqrt")) {
6285 SetSqrtFPAccuracy(CI);
6286 }
6287 if (callOrInvoke) {
6288 *callOrInvoke = CI;
6289 if (CGM.getCodeGenOpts().CallGraphSection) {
6290 QualType CST;
6291 if (TargetDecl && TargetDecl->getFunctionType())
6292 CST = QualType(TargetDecl->getFunctionType(), 0);
6293 else if (const auto *FPT =
6294 Callee.getAbstractInfo().getCalleeFunctionProtoType())
6295 CST = QualType(FPT, 0);
6296 else
6297 llvm_unreachable(
6298 "Cannot find the callee type to generate callee_type metadata.");
6299
6300 // Set type identifier metadata of indirect calls for call graph section.
6301 if (!CST.isNull())
6302 CGM.createCalleeTypeMetadataForIcall(QT: CST, CB: *callOrInvoke);
6303 }
6304 }
6305
6306 // If this is within a function that has the guard(nocf) attribute and is an
6307 // indirect call, add the "guard_nocf" attribute to this call to indicate that
6308 // Control Flow Guard checks should not be added, even if the call is inlined.
6309 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
6310 if (const auto *A = FD->getAttr<CFGuardAttr>()) {
6311 if (A->getGuard() == CFGuardAttr::GuardArg::nocf &&
6312 !CI->getCalledFunction())
6313 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: "guard_nocf");
6314 }
6315 }
6316
6317 // Apply the attributes and calling convention.
6318 CI->setAttributes(Attrs);
6319 CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
6320
6321 // Apply various metadata.
6322
6323 if (!CI->getType()->isVoidTy())
6324 CI->setName("call");
6325
6326 if (CGM.shouldEmitConvergenceTokens() && CI->isConvergent())
6327 CI = addConvergenceControlToken(Input: CI);
6328
6329 // Update largest vector width from the return type.
6330 LargestVectorWidth =
6331 std::max(a: LargestVectorWidth, b: getMaxVectorWidth(Ty: CI->getType()));
6332
6333 // Insert instrumentation or attach profile metadata at indirect call sites.
6334 // For more details, see the comment before the definition of
6335 // IPVK_IndirectCallTarget in InstrProfData.inc.
6336 if (!CI->getCalledFunction())
6337 PGO->valueProfile(Builder, ValueKind: llvm::IPVK_IndirectCallTarget, ValueSite: CI, ValuePtr: CalleePtr);
6338
6339 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
6340 // optimizer it can aggressively ignore unwind edges.
6341 if (CGM.getLangOpts().ObjCAutoRefCount)
6342 AddObjCARCExceptionMetadata(Inst: CI);
6343
6344 // Set tail call kind if necessary.
6345 bool IsPPC = getTarget().getTriple().isPPC();
6346 bool IsMIPS = getTarget().getTriple().isMIPS();
6347 bool HasMips16 = false;
6348 if (IsMIPS) {
6349 const TargetOptions &TargetOpts = getTarget().getTargetOpts();
6350 HasMips16 = TargetOpts.FeatureMap.lookup(Key: "mips16");
6351 if (!HasMips16)
6352 HasMips16 = llvm::is_contained(Range: TargetOpts.Features, Element: "+mips16");
6353 }
6354 if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(Val: CI)) {
6355 if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
6356 Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
6357 else if (IsMustTail) {
6358 if (IsPPC) {
6359 if (getTarget().getTriple().isOSAIX())
6360 CGM.getDiags().Report(Loc, DiagID: diag::err_aix_musttail_unsupported);
6361 else if (!getTarget().hasFeature(Feature: "pcrelative-memops")) {
6362 if (getTarget().hasFeature(Feature: "longcall"))
6363 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail) << 0;
6364 else if (Call->isIndirectCall())
6365 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail) << 1;
6366 else if (isa_and_nonnull<FunctionDecl>(Val: TargetDecl)) {
6367 if (!cast<FunctionDecl>(Val: TargetDecl)->isDefined())
6368 // The undefined callee may be a forward declaration. Without
6369 // knowning all symbols in the module, we won't know the symbol is
6370 // defined or not. Collect all these symbols for later diagnosing.
6371 CGM.addUndefinedGlobalForTailCall(
6372 Global: {cast<FunctionDecl>(Val: TargetDecl), Loc});
6373 else {
6374 llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(
6375 GD: GlobalDecl(cast<FunctionDecl>(Val: TargetDecl)));
6376 if (llvm::GlobalValue::isWeakForLinker(Linkage) ||
6377 llvm::GlobalValue::isDiscardableIfUnused(Linkage))
6378 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail)
6379 << 2;
6380 }
6381 }
6382 }
6383 }
6384 if (IsMIPS) {
6385 if (HasMips16)
6386 CGM.getDiags().Report(Loc, DiagID: diag::err_mips_impossible_musttail) << 0;
6387 else if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl))
6388 CGM.addUndefinedGlobalForTailCall(Global: {FD, Loc});
6389 }
6390 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
6391 }
6392 }
6393
6394 // Add metadata for calls to MSAllocator functions
6395 if (getDebugInfo() && TargetDecl && TargetDecl->hasAttr<MSAllocatorAttr>())
6396 getDebugInfo()->addHeapAllocSiteMetadata(CallSite: CI, AllocatedTy: RetTy->getPointeeType(), Loc);
6397
6398 // Add srcloc metadata for [[gnu::error/warning]] diagnostics. When
6399 // ShowInliningChain is enabled, also track inline/static calls for the
6400 // heuristic fallback when debug info is not available. This heuristic is
6401 // conservative and best-effort since static or inline-annotated functions
6402 // are still not guaranteed to be inlined.
6403 if (TargetDecl) {
6404 bool NeedSrcLoc = TargetDecl->hasAttr<ErrorAttr>();
6405 if (!NeedSrcLoc && CGM.getCodeGenOpts().ShowInliningChain) {
6406 if (const auto *FD = dyn_cast<FunctionDecl>(Val: TargetDecl))
6407 NeedSrcLoc = FD->isInlined() || FD->hasAttr<AlwaysInlineAttr>() ||
6408 FD->getStorageClass() == SC_Static ||
6409 FD->isInAnonymousNamespace();
6410 }
6411 if (NeedSrcLoc) {
6412 auto *Line = llvm::ConstantInt::get(Ty: Int64Ty, V: Loc.getRawEncoding());
6413 auto *MD = llvm::ConstantAsMetadata::get(C: Line);
6414 CI->setMetadata(Kind: "srcloc", Node: llvm::MDNode::get(Context&: getLLVMContext(), MDs: {MD}));
6415 }
6416 }
6417
6418 // 4. Finish the call.
6419
6420 // If the call doesn't return, finish the basic block and clear the
6421 // insertion point; this allows the rest of IRGen to discard
6422 // unreachable code.
6423 if (CI->doesNotReturn()) {
6424 if (NeedSRetLifetimeEnd)
6425 PopCleanupBlock();
6426
6427 // Strip away the noreturn attribute to better diagnose unreachable UB.
6428 if (SanOpts.has(K: SanitizerKind::Unreachable)) {
6429 // Also remove from function since CallBase::hasFnAttr additionally checks
6430 // attributes of the called function.
6431 if (auto *F = CI->getCalledFunction())
6432 F->removeFnAttr(Kind: llvm::Attribute::NoReturn);
6433 CI->removeFnAttr(Kind: llvm::Attribute::NoReturn);
6434
6435 // Avoid incompatibility with ASan which relies on the `noreturn`
6436 // attribute to insert handler calls.
6437 if (SanOpts.hasOneOf(K: SanitizerKind::Address |
6438 SanitizerKind::KernelAddress)) {
6439 SanitizerScope SanScope(this);
6440 llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);
6441 Builder.SetInsertPoint(CI);
6442 auto *FnType = llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
6443 llvm::FunctionCallee Fn =
6444 CGM.CreateRuntimeFunction(Ty: FnType, Name: "__asan_handle_no_return");
6445 EmitNounwindRuntimeCall(callee: Fn);
6446 }
6447 }
6448
6449 EmitUnreachable(Loc);
6450 Builder.ClearInsertionPoint();
6451
6452 // FIXME: For now, emit a dummy basic block because expr emitters in
6453 // generally are not ready to handle emitting expressions at unreachable
6454 // points.
6455 EnsureInsertPoint();
6456
6457 // Return a reasonable RValue.
6458 return GetUndefRValue(Ty: RetTy);
6459 }
6460
6461 // If this is a musttail call, return immediately. We do not branch to the
6462 // epilogue in this case.
6463 if (IsMustTail) {
6464 for (auto it = EHStack.find(sp: CurrentCleanupScopeDepth); it != EHStack.end();
6465 ++it) {
6466 // A noexcept caller pushes an EHTerminateScope to call std::terminate()
6467 // if an exception escapes. A musttail call replaces the caller's frame,
6468 // removing this handler. This is safe if the callee is also nounwind:
6469 // the callee's own noexcept handler prevents any exception from reaching
6470 // where the caller's handler would have been.
6471 if (isa<EHTerminateScope>(Val: &*it)) {
6472 if (CI->doesNotThrow())
6473 continue;
6474 CGM.getDiags().Report(Loc: MustTailCall->getBeginLoc(),
6475 DiagID: diag::err_musttail_noexcept_mismatch);
6476 break;
6477 }
6478 EHCleanupScope *Cleanup = dyn_cast<EHCleanupScope>(Val: &*it);
6479 // Fake uses can be safely emitted immediately prior to the tail call, so
6480 // we choose to emit them just before the call here.
6481 if (Cleanup && Cleanup->isFakeUse()) {
6482 CGBuilderTy::InsertPointGuard IPG(Builder);
6483 Builder.SetInsertPoint(CI);
6484 Cleanup->getCleanup()->Emit(CGF&: *this, flags: EHScopeStack::Cleanup::Flags());
6485 } else if (!(Cleanup &&
6486 Cleanup->getCleanup()->isRedundantBeforeReturn())) {
6487 CGM.ErrorUnsupported(S: MustTailCall, Type: "tail call skipping over cleanups");
6488 }
6489 }
6490 if (CI->getType()->isVoidTy())
6491 Builder.CreateRetVoid();
6492 else
6493 Builder.CreateRet(V: CI);
6494 Builder.ClearInsertionPoint();
6495 EnsureInsertPoint();
6496 return GetUndefRValue(Ty: RetTy);
6497 }
6498
6499 // Perform the swifterror writeback.
6500 if (swiftErrorTemp.isValid()) {
6501 llvm::Value *errorResult = Builder.CreateLoad(Addr: swiftErrorTemp);
6502 Builder.CreateStore(Val: errorResult, Addr: swiftErrorArg);
6503 }
6504
6505 // Emit any call-associated writebacks immediately. Arguably this
6506 // should happen after any return-value munging.
6507 if (CallArgs.hasWritebacks())
6508 EmitWritebacks(args: CallArgs);
6509
6510 // The stack cleanup for inalloca arguments has to run out of the normal
6511 // lexical order, so deactivate it and run it manually here.
6512 CallArgs.freeArgumentMemory(CGF&: *this);
6513
6514 // Extract the return value.
6515 RValue Ret;
6516
6517 // If the current function is a virtual function pointer thunk, avoid copying
6518 // the return value of the musttail call to a temporary.
6519 if (IsVirtualFunctionPointerThunk) {
6520 Ret = RValue::get(V: CI);
6521 } else {
6522 Ret = [&] {
6523 switch (RetAI.getKind()) {
6524 case ABIArgInfo::CoerceAndExpand: {
6525 auto coercionType = RetAI.getCoerceAndExpandType();
6526
6527 Address addr = SRetPtr.withElementType(ElemTy: coercionType);
6528
6529 assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());
6530 bool requiresExtract = isa<llvm::StructType>(Val: CI->getType());
6531
6532 unsigned unpaddedIndex = 0;
6533 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
6534 llvm::Type *eltType = coercionType->getElementType(N: i);
6535 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
6536 continue;
6537 Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
6538 llvm::Value *elt = CI;
6539 if (requiresExtract)
6540 elt = Builder.CreateExtractValue(Agg: elt, Idxs: unpaddedIndex++);
6541 else
6542 assert(unpaddedIndex == 0);
6543 Builder.CreateStore(Val: elt, Addr: eltAddr);
6544 }
6545 [[fallthrough]];
6546 }
6547
6548 case ABIArgInfo::InAlloca:
6549 case ABIArgInfo::Indirect: {
6550 RValue ret = convertTempToRValue(addr: SRetPtr, type: RetTy, Loc: SourceLocation());
6551 if (NeedSRetLifetimeEnd)
6552 PopCleanupBlock();
6553 return ret;
6554 }
6555
6556 case ABIArgInfo::Ignore:
6557 // If we are ignoring an argument that had a result, make sure to
6558 // construct the appropriate return value for our caller.
6559 return GetUndefRValue(Ty: RetTy);
6560
6561 case ABIArgInfo::Extend:
6562 case ABIArgInfo::Direct: {
6563 llvm::Type *RetIRTy = ConvertType(T: RetTy);
6564 if (RetAI.getCoerceToType() == RetIRTy &&
6565 RetAI.getDirectOffset() == 0) {
6566 switch (getEvaluationKind(T: RetTy)) {
6567 case TEK_Complex: {
6568 llvm::Value *Real = Builder.CreateExtractValue(Agg: CI, Idxs: 0);
6569 llvm::Value *Imag = Builder.CreateExtractValue(Agg: CI, Idxs: 1);
6570 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
6571 }
6572 case TEK_Aggregate:
6573 break;
6574 case TEK_Scalar: {
6575 // If the argument doesn't match, perform a bitcast to coerce it.
6576 // This can happen due to trivial type mismatches.
6577 llvm::Value *V = CI;
6578 if (V->getType() != RetIRTy)
6579 V = Builder.CreateBitCast(V, DestTy: RetIRTy);
6580 return RValue::get(V);
6581 }
6582 }
6583 }
6584
6585 // If coercing a fixed vector from a scalable vector for ABI
6586 // compatibility, and the types match, use the llvm.vector.extract
6587 // intrinsic to perform the conversion.
6588 if (auto *FixedDstTy = dyn_cast<llvm::FixedVectorType>(Val: RetIRTy)) {
6589 llvm::Value *V = CI;
6590 if (auto *ScalableSrcTy =
6591 dyn_cast<llvm::ScalableVectorType>(Val: V->getType())) {
6592 if (FixedDstTy->getElementType() ==
6593 ScalableSrcTy->getElementType()) {
6594 V = Builder.CreateExtractVector(DstType: FixedDstTy, SrcVec: V, Idx: uint64_t(0),
6595 Name: "cast.fixed");
6596 return RValue::get(V);
6597 }
6598 }
6599 }
6600
6601 Address DestPtr = ReturnValue.getValue();
6602 bool DestIsVolatile = ReturnValue.isVolatile();
6603 uint64_t DestSize =
6604 getContext().getTypeInfoDataSizeInChars(T: RetTy).Width.getQuantity();
6605
6606 if (!DestPtr.isValid()) {
6607 DestPtr = CreateMemTempWithoutCast(T: RetTy, Name: "coerce");
6608 DestIsVolatile = false;
6609 DestSize = getContext().getTypeSizeInChars(T: RetTy).getQuantity();
6610 }
6611
6612 // An empty record can overlap other data (if declared with
6613 // no_unique_address); omit the store for such types - as there is no
6614 // actual data to store.
6615 if (!isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true)) {
6616 // If the value is offset in memory, apply the offset now.
6617 Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
6618 CreateCoercedStore(
6619 Src: CI, SrcFETy: RetTy, Dst: StorePtr,
6620 DstSize: llvm::TypeSize::getFixed(ExactSize: DestSize - RetAI.getDirectOffset()),
6621 DstIsVolatile: DestIsVolatile);
6622 }
6623
6624 return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation());
6625 }
6626
6627 case ABIArgInfo::TargetSpecific: {
6628 Address DestPtr = ReturnValue.getValue();
6629 Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
6630 bool DestIsVolatile = ReturnValue.isVolatile();
6631 if (!DestPtr.isValid()) {
6632 DestPtr = CreateMemTempWithoutCast(T: RetTy, Name: "target_coerce");
6633 DestIsVolatile = false;
6634 }
6635 CGM.getABIInfo().createCoercedStore(Val: CI, DstAddr: StorePtr, AI: RetAI, DestIsVolatile,
6636 CGF&: *this);
6637 return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation());
6638 }
6639
6640 case ABIArgInfo::Expand:
6641 case ABIArgInfo::IndirectAliased:
6642 llvm_unreachable("Invalid ABI kind for return argument");
6643 }
6644
6645 llvm_unreachable("Unhandled ABIArgInfo::Kind");
6646 }();
6647 }
6648
6649 // Emit the assume_aligned check on the return value.
6650 if (Ret.isScalar() && TargetDecl) {
6651 AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6652 AllocAlignAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6653 }
6654
6655 // Explicitly call CallLifetimeEnd::Emit just to re-use the code even though
6656 // we can't use the full cleanup mechanism.
6657 for (CallLifetimeEnd &LifetimeEnd : CallLifetimeEndAfterCall)
6658 LifetimeEnd.Emit(CGF&: *this, /*Flags=*/flags: {});
6659
6660 if (!ReturnValue.isExternallyDestructed() &&
6661 RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct)
6662 pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Ret.getAggregateAddress(),
6663 type: RetTy);
6664
6665 // Generate function declaration DISuprogram in order to be used
6666 // in debug info about call sites.
6667 if (CGDebugInfo *DI = getDebugInfo()) {
6668 // Ensure call site info would actually be emitted before collecting
6669 // further callee info.
6670 if (CalleeDecl && !CalleeDecl->hasAttr<NoDebugAttr>() &&
6671 DI->getCallSiteRelatedAttrs() != llvm::DINode::FlagZero) {
6672 CodeGenFunction CalleeCGF(CGM);
6673 const GlobalDecl &CalleeGlobalDecl =
6674 Callee.getAbstractInfo().getCalleeDecl();
6675 CalleeCGF.CurGD = CalleeGlobalDecl;
6676 FunctionArgList Args;
6677 QualType ResTy = CalleeCGF.BuildFunctionArgList(GD: CalleeGlobalDecl, Args);
6678 DI->EmitFuncDeclForCallSite(
6679 CallOrInvoke: CI, CalleeType: DI->getFunctionType(FD: CalleeDecl, RetTy: ResTy, Args), CalleeGlobalDecl);
6680 }
6681 // Generate call site target information.
6682 DI->addCallTargetIfVirtual(FD: CalleeDecl, CI);
6683 }
6684
6685 return Ret;
6686}
6687
6688CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const {
6689 if (isVirtual()) {
6690 const CallExpr *CE = getVirtualCallExpr();
6691 return CGF.CGM.getCXXABI().getVirtualFunctionPointer(
6692 CGF, GD: getVirtualMethodDecl(), This: getThisAddress(), Ty: getVirtualFunctionType(),
6693 Loc: CE ? CE->getBeginLoc() : SourceLocation());
6694 }
6695
6696 return *this;
6697}
6698
6699/* VarArg handling */
6700
6701RValue CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
6702 AggValueSlot Slot) {
6703 VAListAddr = VE->isMicrosoftABI() ? EmitMSVAListRef(E: VE->getSubExpr())
6704 : EmitVAListRef(E: VE->getSubExpr());
6705 QualType Ty = VE->getType();
6706 if (Ty->isVariablyModifiedType())
6707 EmitVariablyModifiedType(Ty);
6708 if (VE->isMicrosoftABI())
6709 return CGM.getABIInfo().EmitMSVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6710 return CGM.getABIInfo().EmitVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6711}
6712
6713DisableDebugLocationUpdates::DisableDebugLocationUpdates(CodeGenFunction &CGF)
6714 : CGF(CGF) {
6715 CGF.disableDebugInfo();
6716}
6717
6718DisableDebugLocationUpdates::~DisableDebugLocationUpdates() {
6719 CGF.enableDebugInfo();
6720}
6721