1//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// These classes wrap the information about a call or function
10// definition used to handle ABI compliancy.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CGCall.h"
15#include "ABIInfo.h"
16#include "ABIInfoImpl.h"
17#include "CGBlocks.h"
18#include "CGCXXABI.h"
19#include "CGCleanup.h"
20#include "CGDebugInfo.h"
21#include "CGRecordLayout.h"
22#include "CodeGenFunction.h"
23#include "CodeGenModule.h"
24#include "CodeGenPGO.h"
25#include "QualTypeMapper.h"
26#include "TargetInfo.h"
27#include "clang/AST/Attr.h"
28#include "clang/AST/Decl.h"
29#include "clang/AST/DeclCXX.h"
30#include "clang/AST/DeclObjC.h"
31#include "clang/AST/RecordLayout.h"
32#include "clang/Basic/CodeGenOptions.h"
33#include "clang/Basic/TargetInfo.h"
34#include "clang/CodeGen/CGFunctionInfo.h"
35#include "clang/CodeGen/SwiftCallingConv.h"
36#include "llvm/ABI/FunctionInfo.h"
37#include "llvm/ABI/IRTypeMapper.h"
38#include "llvm/ABI/TargetInfo.h"
39#include "llvm/ABI/Types.h"
40#include "llvm/ADT/STLExtras.h"
41#include "llvm/ADT/StringExtras.h"
42#include "llvm/Analysis/ValueTracking.h"
43#include "llvm/IR/Assumptions.h"
44#include "llvm/IR/AttributeMask.h"
45#include "llvm/IR/Attributes.h"
46#include "llvm/IR/CallingConv.h"
47#include "llvm/IR/DataLayout.h"
48#include "llvm/IR/DebugInfoMetadata.h"
49#include "llvm/IR/InlineAsm.h"
50#include "llvm/IR/IntrinsicInst.h"
51#include "llvm/IR/Intrinsics.h"
52#include "llvm/IR/Type.h"
53#include "llvm/Transforms/Utils/Local.h"
54#include <optional>
55using namespace clang;
56using namespace CodeGen;
57
58/***/
59
60unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
61 switch (CC) {
62 default:
63 return llvm::CallingConv::C;
64 case CC_X86StdCall:
65 return llvm::CallingConv::X86_StdCall;
66 case CC_X86FastCall:
67 return llvm::CallingConv::X86_FastCall;
68 case CC_X86RegCall:
69 return llvm::CallingConv::X86_RegCall;
70 case CC_X86ThisCall:
71 return llvm::CallingConv::X86_ThisCall;
72 case CC_Win64:
73 return llvm::CallingConv::Win64;
74 case CC_X86_64SysV:
75 return llvm::CallingConv::X86_64_SysV;
76 case CC_AAPCS:
77 return llvm::CallingConv::ARM_AAPCS;
78 case CC_AAPCS_VFP:
79 return llvm::CallingConv::ARM_AAPCS_VFP;
80 case CC_IntelOclBicc:
81 return llvm::CallingConv::Intel_OCL_BI;
82 // TODO: Add support for __pascal to LLVM.
83 case CC_X86Pascal:
84 return llvm::CallingConv::C;
85 // TODO: Add support for __vectorcall to LLVM.
86 case CC_X86VectorCall:
87 return llvm::CallingConv::X86_VectorCall;
88 case CC_AArch64VectorCall:
89 return llvm::CallingConv::AArch64_VectorCall;
90 case CC_AArch64SVEPCS:
91 return llvm::CallingConv::AArch64_SVE_VectorCall;
92 case CC_SpirFunction:
93 return llvm::CallingConv::SPIR_FUNC;
94 case CC_DeviceKernel:
95 return CGM.getTargetCodeGenInfo().getDeviceKernelCallingConv();
96 case CC_PreserveMost:
97 return llvm::CallingConv::PreserveMost;
98 case CC_PreserveAll:
99 return llvm::CallingConv::PreserveAll;
100 case CC_Swift:
101 return llvm::CallingConv::Swift;
102 case CC_SwiftAsync:
103 return llvm::CallingConv::SwiftTail;
104 case CC_M68kRTD:
105 return llvm::CallingConv::M68k_RTD;
106 case CC_PreserveNone:
107 return llvm::CallingConv::PreserveNone;
108 // clang-format off
109 case CC_RISCVVectorCall: return llvm::CallingConv::RISCV_VectorCall;
110 // clang-format on
111#define CC_VLS_CASE(ABI_VLEN) \
112 case CC_RISCVVLSCall_##ABI_VLEN: \
113 return llvm::CallingConv::RISCV_VLSCall_##ABI_VLEN;
114 CC_VLS_CASE(32)
115 CC_VLS_CASE(64)
116 CC_VLS_CASE(128)
117 CC_VLS_CASE(256)
118 CC_VLS_CASE(512)
119 CC_VLS_CASE(1024)
120 CC_VLS_CASE(2048)
121 CC_VLS_CASE(4096)
122 CC_VLS_CASE(8192)
123 CC_VLS_CASE(16384)
124 CC_VLS_CASE(32768)
125 CC_VLS_CASE(65536)
126#undef CC_VLS_CASE
127 }
128}
129
130/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
131/// qualification. Either or both of RD and MD may be null. A null RD indicates
132/// that there is no meaningful 'this' type, and a null MD can occur when
133/// calling a method pointer.
134CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
135 const CXXMethodDecl *MD) {
136 CanQualType RecTy;
137 if (RD)
138 RecTy = Context.getCanonicalTagType(TD: RD);
139 else
140 RecTy = Context.VoidTy;
141
142 if (MD)
143 RecTy = CanQualType::CreateUnsafe(Other: Context.getAddrSpaceQualType(
144 T: RecTy, AddressSpace: MD->getMethodQualifiers().getAddressSpace()));
145 return Context.getPointerType(T: RecTy);
146}
147
148/// Returns the canonical formal type of the given C++ method.
149static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
150 return MD->getType()
151 ->getCanonicalTypeUnqualified()
152 .getAs<FunctionProtoType>();
153}
154
155/// Returns the "extra-canonicalized" return type, which discards
156/// qualifiers on the return type. Codegen doesn't care about them,
157/// and it makes ABI code a little easier to be able to assume that
158/// all parameter and return types are top-level unqualified.
159static CanQualType GetReturnType(QualType RetTy) {
160 return RetTy->getCanonicalTypeUnqualified();
161}
162
163/// Arrange the argument and result information for a value of the given
164/// unprototyped freestanding function type.
165const CGFunctionInfo &
166CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
167 // When translating an unprototyped function type, always use a
168 // variadic type.
169 return arrangeLLVMFunctionInfo(returnType: FTNP->getReturnType().getUnqualifiedType(),
170 opts: FnInfoOpts::None, argTypes: {}, info: FTNP->getExtInfo(), paramInfos: {},
171 args: RequiredArgs(0), /*ABIInfoFD=*/nullptr);
172}
173
174static void addExtParameterInfosForCall(
175 llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
176 const FunctionProtoType *proto, unsigned prefixArgs, unsigned totalArgs) {
177 assert(proto->hasExtParameterInfos());
178 assert(paramInfos.size() <= prefixArgs);
179 assert(proto->getNumParams() + prefixArgs <= totalArgs);
180
181 paramInfos.reserve(N: totalArgs);
182
183 // Add default infos for any prefix args that don't already have infos.
184 paramInfos.resize(N: prefixArgs);
185
186 // Add infos for the prototype.
187 for (const auto &ParamInfo : proto->getExtParameterInfos()) {
188 paramInfos.push_back(Elt: ParamInfo);
189 // pass_object_size params have no parameter info.
190 if (ParamInfo.hasPassObjectSize())
191 paramInfos.emplace_back();
192 }
193
194 assert(paramInfos.size() <= totalArgs &&
195 "Did we forget to insert pass_object_size args?");
196 // Add default infos for the variadic and/or suffix arguments.
197 paramInfos.resize(N: totalArgs);
198}
199
200/// Adds the formal parameters in FPT to the given prefix. If any parameter in
201/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
202static void appendParameterTypes(
203 const CodeGenTypes &CGT, SmallVectorImpl<CanQualType> &prefix,
204 SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
205 CanQual<FunctionProtoType> FPT) {
206 // Fast path: don't touch param info if we don't need to.
207 if (!FPT->hasExtParameterInfos()) {
208 assert(paramInfos.empty() &&
209 "We have paramInfos, but the prototype doesn't?");
210 prefix.append(in_start: FPT->param_type_begin(), in_end: FPT->param_type_end());
211 return;
212 }
213
214 unsigned PrefixSize = prefix.size();
215 // In the vast majority of cases, we'll have precisely FPT->getNumParams()
216 // parameters; the only thing that can change this is the presence of
217 // pass_object_size. So, we preallocate for the common case.
218 prefix.reserve(N: prefix.size() + FPT->getNumParams());
219
220 auto ExtInfos = FPT->getExtParameterInfos();
221 assert(ExtInfos.size() == FPT->getNumParams());
222 for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
223 prefix.push_back(Elt: FPT->getParamType(i: I));
224 if (ExtInfos[I].hasPassObjectSize())
225 prefix.push_back(Elt: CGT.getContext().getCanonicalSizeType());
226 }
227
228 addExtParameterInfosForCall(paramInfos, proto: FPT.getTypePtr(), prefixArgs: PrefixSize,
229 totalArgs: prefix.size());
230}
231
232using ExtParameterInfoList =
233 SmallVector<FunctionProtoType::ExtParameterInfo, 16>;
234
235/// Arrange the LLVM function layout for a value of the given function
236/// type, on top of any implicit parameters already stored.
237static const CGFunctionInfo &
238arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
239 SmallVectorImpl<CanQualType> &prefix,
240 CanQual<FunctionProtoType> FTP) {
241 ExtParameterInfoList paramInfos;
242 RequiredArgs Required = RequiredArgs::forPrototypePlus(prototype: FTP, additional: prefix.size());
243 appendParameterTypes(CGT, prefix, paramInfos, FPT: FTP);
244 CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
245
246 FnInfoOpts opts =
247 instanceMethod ? FnInfoOpts::IsInstanceMethod : FnInfoOpts::None;
248 return CGT.arrangeLLVMFunctionInfo(returnType: resultType, opts, argTypes: prefix,
249 info: FTP->getExtInfo(), paramInfos, args: Required,
250 /*ABIInfoFD=*/nullptr);
251}
252
253using CanQualTypeList = SmallVector<CanQualType, 16>;
254
255/// Arrange the argument and result information for a value of the
256/// given freestanding function type.
257const CGFunctionInfo &
258CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
259 CanQualTypeList argTypes;
260 return ::arrangeLLVMFunctionInfo(CGT&: *this, /*instanceMethod=*/false, prefix&: argTypes,
261 FTP);
262}
263
264static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
265 bool IsTargetDefaultMSABI) {
266 // Set the appropriate calling convention for the Function.
267 if (D->hasAttr<StdCallAttr>())
268 return CC_X86StdCall;
269
270 if (D->hasAttr<FastCallAttr>())
271 return CC_X86FastCall;
272
273 if (D->hasAttr<RegCallAttr>())
274 return CC_X86RegCall;
275
276 if (D->hasAttr<ThisCallAttr>())
277 return CC_X86ThisCall;
278
279 if (D->hasAttr<VectorCallAttr>())
280 return CC_X86VectorCall;
281
282 if (D->hasAttr<PascalAttr>())
283 return CC_X86Pascal;
284
285 if (PcsAttr *PCS = D->getAttr<PcsAttr>())
286 return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
287
288 if (D->hasAttr<AArch64VectorPcsAttr>())
289 return CC_AArch64VectorCall;
290
291 if (D->hasAttr<AArch64SVEPcsAttr>())
292 return CC_AArch64SVEPCS;
293
294 if (D->hasAttr<DeviceKernelAttr>())
295 return CC_DeviceKernel;
296
297 if (D->hasAttr<IntelOclBiccAttr>())
298 return CC_IntelOclBicc;
299
300 if (D->hasAttr<MSABIAttr>())
301 return IsTargetDefaultMSABI ? CC_C : CC_Win64;
302
303 if (D->hasAttr<SysVABIAttr>())
304 return IsTargetDefaultMSABI ? CC_X86_64SysV : CC_C;
305
306 if (D->hasAttr<PreserveMostAttr>())
307 return CC_PreserveMost;
308
309 if (D->hasAttr<PreserveAllAttr>())
310 return CC_PreserveAll;
311
312 if (D->hasAttr<M68kRTDAttr>())
313 return CC_M68kRTD;
314
315 if (D->hasAttr<PreserveNoneAttr>())
316 return CC_PreserveNone;
317
318 if (D->hasAttr<RISCVVectorCCAttr>())
319 return CC_RISCVVectorCall;
320
321 if (RISCVVLSCCAttr *PCS = D->getAttr<RISCVVLSCCAttr>()) {
322 switch (PCS->getVectorWidth()) {
323 default:
324 llvm_unreachable("Invalid RISC-V VLS ABI VLEN");
325#define CC_VLS_CASE(ABI_VLEN) \
326 case ABI_VLEN: \
327 return CC_RISCVVLSCall_##ABI_VLEN;
328 CC_VLS_CASE(32)
329 CC_VLS_CASE(64)
330 CC_VLS_CASE(128)
331 CC_VLS_CASE(256)
332 CC_VLS_CASE(512)
333 CC_VLS_CASE(1024)
334 CC_VLS_CASE(2048)
335 CC_VLS_CASE(4096)
336 CC_VLS_CASE(8192)
337 CC_VLS_CASE(16384)
338 CC_VLS_CASE(32768)
339 CC_VLS_CASE(65536)
340#undef CC_VLS_CASE
341 }
342 }
343
344 return CC_C;
345}
346
347/// Arrange the argument and result information for a call to an
348/// unknown C++ non-static member function of the given abstract type.
349/// (A null RD means we don't have any meaningful "this" argument type,
350/// so fall back to a generic pointer type).
351/// The member function must be an ordinary function, i.e. not a
352/// constructor or destructor.
353const CGFunctionInfo &
354CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
355 const FunctionProtoType *FTP,
356 const CXXMethodDecl *MD) {
357 CanQualTypeList argTypes;
358
359 // Add the 'this' pointer.
360 argTypes.push_back(Elt: DeriveThisType(RD, MD));
361 auto CanonicalFTP =
362 FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>();
363 ExtParameterInfoList paramInfos;
364 RequiredArgs required = RequiredArgs::forPrototypePlus(
365 prototype: CanonicalFTP.getTypePtr(), additional: argTypes.size());
366 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: CanonicalFTP);
367 return arrangeLLVMFunctionInfo(
368 returnType: CanonicalFTP->getReturnType().getUnqualifiedType(),
369 opts: FnInfoOpts::IsInstanceMethod, argTypes, info: CanonicalFTP->getExtInfo(),
370 paramInfos, args: required, ABIInfoFD: MD);
371}
372
373/// Set calling convention for CUDA/HIP kernel.
374static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
375 const FunctionDecl *FD) {
376 if (FD->hasAttr<CUDAGlobalAttr>()) {
377 const FunctionType *FT = FTy->getAs<FunctionType>();
378 CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
379 FTy = FT->getCanonicalTypeUnqualified();
380 }
381}
382
383/// Arrange the argument and result information for a declaration or
384/// definition of the given C++ non-static member function. The
385/// member function must be an ordinary function, i.e. not a
386/// constructor or destructor.
387const CGFunctionInfo &
388CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
389 assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
390 assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
391
392 CanQualType FT = GetFormalType(MD).getAs<Type>();
393 setCUDAKernelCallingConvention(FTy&: FT, CGM, FD: MD);
394 auto prototype = FT.getAs<FunctionProtoType>();
395
396 if (MD->isImplicitObjectMemberFunction()) {
397 // The abstract case is perfectly fine.
398 const CXXRecordDecl *ThisType =
399 getCXXABI().getThisArgumentTypeForMethod(GD: MD);
400 return arrangeCXXMethodType(RD: ThisType, FTP: prototype.getTypePtr(), MD);
401 }
402
403 CanQualTypeList argTypes;
404 ExtParameterInfoList paramInfos;
405 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: prototype);
406 return arrangeLLVMFunctionInfo(
407 returnType: prototype->getReturnType().getUnqualifiedType(), opts: FnInfoOpts::None,
408 argTypes, info: prototype->getExtInfo(), paramInfos,
409 args: RequiredArgs::forPrototypePlus(prototype: prototype.getTypePtr(), additional: 0), ABIInfoFD: MD);
410}
411
412bool CodeGenTypes::inheritingCtorHasParams(
413 const InheritedConstructor &Inherited, CXXCtorType Type) {
414 // Parameters are unnecessary if we're constructing a base class subobject
415 // and the inherited constructor lives in a virtual base.
416 return Type == Ctor_Complete ||
417 !Inherited.getShadowDecl()->constructsVirtualBase() ||
418 !Target.getCXXABI().hasConstructorVariants();
419}
420
421const CGFunctionInfo &
422CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
423 auto *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
424
425 CanQualTypeList argTypes;
426 ExtParameterInfoList paramInfos;
427
428 const CXXRecordDecl *ThisType = getCXXABI().getThisArgumentTypeForMethod(GD);
429 argTypes.push_back(Elt: DeriveThisType(RD: ThisType, MD));
430
431 bool PassParams = true;
432
433 if (auto *CD = dyn_cast<CXXConstructorDecl>(Val: MD)) {
434 // A base class inheriting constructor doesn't get forwarded arguments
435 // needed to construct a virtual base (or base class thereof).
436 if (auto Inherited = CD->getInheritedConstructor())
437 PassParams = inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
438 }
439
440 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
441
442 // Add the formal parameters.
443 if (PassParams)
444 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: FTP);
445
446 CGCXXABI::AddedStructorArgCounts AddedArgs =
447 getCXXABI().buildStructorSignature(GD, ArgTys&: argTypes);
448 if (!paramInfos.empty()) {
449 // Note: prefix implies after the first param.
450 if (AddedArgs.Prefix)
451 paramInfos.insert(I: paramInfos.begin() + 1, NumToInsert: AddedArgs.Prefix,
452 Elt: FunctionProtoType::ExtParameterInfo{});
453 if (AddedArgs.Suffix)
454 paramInfos.append(NumInputs: AddedArgs.Suffix,
455 Elt: FunctionProtoType::ExtParameterInfo{});
456 }
457
458 RequiredArgs required =
459 (PassParams && MD->isVariadic() ? RequiredArgs(argTypes.size())
460 : RequiredArgs::All);
461
462 FunctionType::ExtInfo extInfo = FTP->getExtInfo();
463 CanQualType resultType = getCXXABI().HasThisReturn(GD) ? argTypes.front()
464 : getCXXABI().hasMostDerivedReturn(GD)
465 ? CGM.getContext().VoidPtrTy
466 : Context.VoidTy;
467 return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::IsInstanceMethod,
468 argTypes, info: extInfo, paramInfos, args: required, ABIInfoFD: MD);
469}
470
471static CanQualTypeList getArgTypesForCall(ASTContext &ctx,
472 const CallArgList &args) {
473 CanQualTypeList argTypes;
474 for (auto &arg : args)
475 argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg.Ty));
476 return argTypes;
477}
478
479static CanQualTypeList getArgTypesForDeclaration(ASTContext &ctx,
480 const FunctionArgList &args) {
481 CanQualTypeList argTypes;
482 for (auto &arg : args)
483 argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg->getType()));
484 return argTypes;
485}
486
487static ExtParameterInfoList
488getExtParameterInfosForCall(const FunctionProtoType *proto, unsigned prefixArgs,
489 unsigned totalArgs) {
490 ExtParameterInfoList result;
491 if (proto->hasExtParameterInfos()) {
492 addExtParameterInfosForCall(paramInfos&: result, proto, prefixArgs, totalArgs);
493 }
494 return result;
495}
496
497/// Arrange a call to a C++ method, passing the given arguments.
498///
499/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
500/// parameter.
501/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
502/// args.
503/// PassProtoArgs indicates whether `args` has args for the parameters in the
504/// given CXXConstructorDecl.
505const CGFunctionInfo &CodeGenTypes::arrangeCXXConstructorCall(
506 const CallArgList &args, const CXXConstructorDecl *D, CXXCtorType CtorKind,
507 unsigned ExtraPrefixArgs, unsigned ExtraSuffixArgs,
508 const FunctionDecl *ABIInfoFD, bool PassProtoArgs) {
509 CanQualTypeList ArgTypes;
510 for (const auto &Arg : args)
511 ArgTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
512
513 // +1 for implicit this, which should always be args[0].
514 unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
515
516 CanQual<FunctionProtoType> FPT = GetFormalType(MD: D);
517 RequiredArgs Required = PassProtoArgs
518 ? RequiredArgs::forPrototypePlus(
519 prototype: FPT, additional: TotalPrefixArgs + ExtraSuffixArgs)
520 : RequiredArgs::All;
521
522 GlobalDecl GD(D, CtorKind);
523 CanQualType ResultType = getCXXABI().HasThisReturn(GD) ? ArgTypes.front()
524 : getCXXABI().hasMostDerivedReturn(GD)
525 ? CGM.getContext().VoidPtrTy
526 : Context.VoidTy;
527
528 FunctionType::ExtInfo Info = FPT->getExtInfo();
529 ExtParameterInfoList ParamInfos;
530 // If the prototype args are elided, we should only have ABI-specific args,
531 // which never have param info.
532 if (PassProtoArgs && FPT->hasExtParameterInfos()) {
533 // ABI-specific suffix arguments are treated the same as variadic arguments.
534 addExtParameterInfosForCall(paramInfos&: ParamInfos, proto: FPT.getTypePtr(), prefixArgs: TotalPrefixArgs,
535 totalArgs: ArgTypes.size());
536 }
537
538 return arrangeLLVMFunctionInfo(returnType: ResultType, opts: FnInfoOpts::IsInstanceMethod,
539 argTypes: ArgTypes, info: Info, paramInfos: ParamInfos, args: Required,
540 ABIInfoFD);
541}
542
543/// Arrange the argument and result information for the declaration or
544/// definition of the given function.
545const CGFunctionInfo &
546CodeGenTypes::arrangeFunctionDeclaration(const GlobalDecl GD) {
547 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
548 if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD))
549 if (MD->isImplicitObjectMemberFunction())
550 return arrangeCXXMethodDeclaration(MD);
551
552 CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
553
554 assert(isa<FunctionType>(FTy));
555 setCUDAKernelCallingConvention(FTy, CGM, FD);
556
557 if (DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()) &&
558 GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
559 const FunctionType *FT = FTy->getAs<FunctionType>();
560 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
561 FTy = FT->getCanonicalTypeUnqualified();
562 }
563
564 // When declaring a function without a prototype, always use a
565 // non-variadic type.
566 if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
567 return arrangeLLVMFunctionInfo(returnType: noProto->getReturnType(), opts: FnInfoOpts::None,
568 argTypes: {}, info: noProto->getExtInfo(), paramInfos: {},
569 args: RequiredArgs::All, ABIInfoFD: FD);
570 }
571
572 CanQual<FunctionProtoType> FTP = FTy.castAs<FunctionProtoType>();
573 CanQualTypeList argTypes;
574 ExtParameterInfoList paramInfos;
575 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: FTP);
576 return arrangeLLVMFunctionInfo(returnType: FTP->getReturnType().getUnqualifiedType(),
577 opts: FnInfoOpts::None, argTypes, info: FTP->getExtInfo(),
578 paramInfos,
579 args: RequiredArgs::forPrototypePlus(prototype: FTP, additional: 0), ABIInfoFD: FD);
580}
581
582/// Arrange the argument and result information for the declaration or
583/// definition of an Objective-C method.
584const CGFunctionInfo &
585CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
586 // It happens that this is the same as a call with no optional
587 // arguments, except also using the formal 'self' type.
588 return arrangeObjCMessageSendSignature(MD, receiverType: MD->getSelfDecl()->getType());
589}
590
591/// Arrange the argument and result information for the function type
592/// through which to perform a send to the given Objective-C method,
593/// using the given receiver type. The receiver type is not always
594/// the 'self' type of the method or even an Objective-C pointer type.
595/// This is *not* the right method for actually performing such a
596/// message send, due to the possibility of optional arguments.
597const CGFunctionInfo &
598CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
599 QualType receiverType) {
600 CanQualTypeList argTys;
601 ExtParameterInfoList extParamInfos(MD->isDirectMethod() ? 1 : 2);
602 argTys.push_back(Elt: Context.getCanonicalParamType(T: receiverType));
603 if (!MD->isDirectMethod())
604 argTys.push_back(Elt: Context.getCanonicalParamType(T: Context.getObjCSelType()));
605 for (const auto *I : MD->parameters()) {
606 argTys.push_back(Elt: Context.getCanonicalParamType(T: I->getType()));
607 auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
608 NoEscape: I->hasAttr<NoEscapeAttr>());
609 extParamInfos.push_back(Elt: extParamInfo);
610 }
611
612 FunctionType::ExtInfo einfo;
613 bool IsTargetDefaultMSABI =
614 getContext().getTargetInfo().getTriple().isOSWindows() ||
615 getContext().getTargetInfo().getTriple().isUEFI();
616 einfo = einfo.withCallingConv(
617 cc: getCallingConventionForDecl(D: MD, IsTargetDefaultMSABI));
618
619 if (getContext().getLangOpts().ObjCAutoRefCount &&
620 MD->hasAttr<NSReturnsRetainedAttr>())
621 einfo = einfo.withProducesResult(producesResult: true);
622
623 RequiredArgs required =
624 (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);
625
626 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: MD->getReturnType()),
627 opts: FnInfoOpts::None, argTypes: argTys, info: einfo, paramInfos: extParamInfos,
628 args: required, /*ABIInfoFD=*/nullptr);
629}
630
631const CGFunctionInfo &
632CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
633 const CallArgList &args) {
634 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
635 FunctionType::ExtInfo einfo;
636
637 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: returnType), opts: FnInfoOpts::None,
638 argTypes, info: einfo, paramInfos: {}, args: RequiredArgs::All,
639 ABIInfoFD: nullptr);
640}
641
642const CGFunctionInfo &CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
643 // FIXME: Do we need to handle ObjCMethodDecl?
644 if (isa<CXXConstructorDecl>(Val: GD.getDecl()) ||
645 isa<CXXDestructorDecl>(Val: GD.getDecl()))
646 return arrangeCXXStructorDeclaration(GD);
647
648 return arrangeFunctionDeclaration(GD);
649}
650
651/// Arrange a thunk that takes 'this' as the first parameter followed by
652/// varargs. Return a void pointer, regardless of the actual return type.
653/// The body of the thunk will end in a musttail call to a function of the
654/// correct type, and the caller will bitcast the function to the correct
655/// prototype.
656const CGFunctionInfo &
657CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
658 assert(MD->isVirtual() && "only methods have thunks");
659 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
660 CanQualType ArgTys[] = {DeriveThisType(RD: MD->getParent(), MD)};
661 return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::None, argTypes: ArgTys,
662 info: FTP->getExtInfo(), paramInfos: {}, args: RequiredArgs(1), ABIInfoFD: MD);
663}
664
665const CGFunctionInfo &
666CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
667 CXXCtorType CT) {
668 assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
669
670 CanQual<FunctionProtoType> FTP = GetFormalType(MD: CD);
671 SmallVector<CanQualType, 2> ArgTys;
672 const CXXRecordDecl *RD = CD->getParent();
673 ArgTys.push_back(Elt: DeriveThisType(RD, MD: CD));
674 if (CT == Ctor_CopyingClosure)
675 ArgTys.push_back(Elt: *FTP->param_type_begin());
676 if (RD->getNumVBases() > 0)
677 ArgTys.push_back(Elt: Context.IntTy);
678 CallingConv CC = Context.getDefaultCallingConvention(
679 /*IsVariadic=*/false, /*IsCXXMethod=*/true);
680 return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::IsInstanceMethod,
681 argTypes: ArgTys, info: FunctionType::ExtInfo(CC), paramInfos: {},
682 args: RequiredArgs::All, /*ABIInfoFD=*/nullptr);
683}
684
685/// Arrange a call as unto a free function, except possibly with an
686/// additional number of formal parameters considered required.
687static const CGFunctionInfo &
688arrangeFreeFunctionLikeCall(CodeGenTypes &CGT, CodeGenModule &CGM,
689 const CallArgList &args, const FunctionType *fnType,
690 unsigned numExtraRequiredArgs, bool chainCall,
691 const FunctionDecl *ABIInfoFD) {
692 assert(args.size() >= numExtraRequiredArgs);
693
694 ExtParameterInfoList paramInfos;
695
696 // In most cases, there are no optional arguments.
697 RequiredArgs required = RequiredArgs::All;
698
699 // If we have a variadic prototype, the required arguments are the
700 // extra prefix plus the arguments in the prototype.
701 if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(Val: fnType)) {
702 if (proto->isVariadic())
703 required = RequiredArgs::forPrototypePlus(prototype: proto, additional: numExtraRequiredArgs);
704
705 if (proto->hasExtParameterInfos())
706 addExtParameterInfosForCall(paramInfos, proto, prefixArgs: numExtraRequiredArgs,
707 totalArgs: args.size());
708
709 // If we don't have a prototype at all, but we're supposed to
710 // explicitly use the variadic convention for unprototyped calls,
711 // treat all of the arguments as required but preserve the nominal
712 // possibility of variadics.
713 } else if (CGM.getTargetCodeGenInfo().isNoProtoCallVariadic(
714 args, fnType: cast<FunctionNoProtoType>(Val: fnType))) {
715 required = RequiredArgs(args.size());
716 }
717
718 CanQualTypeList argTypes;
719 for (const auto &arg : args)
720 argTypes.push_back(Elt: CGT.getContext().getCanonicalParamType(T: arg.Ty));
721 FnInfoOpts opts = chainCall ? FnInfoOpts::IsChainCall : FnInfoOpts::None;
722 return CGT.arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: fnType->getReturnType()),
723 opts, argTypes, info: fnType->getExtInfo(),
724 paramInfos, args: required, ABIInfoFD);
725}
726
727/// Figure out the rules for calling a function with the given formal
728/// type using the given arguments. The arguments are necessary
729/// because the function might be unprototyped, in which case it's
730/// target-dependent in crazy ways.
731const CGFunctionInfo &CodeGenTypes::arrangeFreeFunctionCall(
732 const CallArgList &args, const FunctionType *fnType, bool chainCall,
733 const FunctionDecl *ABIInfoFD) {
734 return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType,
735 numExtraRequiredArgs: chainCall ? 1 : 0, chainCall, ABIInfoFD);
736}
737
738/// A block function is essentially a free function with an
739/// extra implicit argument.
740const CGFunctionInfo &
741CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
742 const FunctionType *fnType) {
743 // FIXME: Pass the enclosing function's ABI information so block calls use
744 // the caller's target features.
745 return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType, numExtraRequiredArgs: 1,
746 /*chainCall=*/false, ABIInfoFD: nullptr);
747}
748
749const CGFunctionInfo &
750CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
751 const FunctionArgList &params) {
752 ExtParameterInfoList paramInfos =
753 getExtParameterInfosForCall(proto, prefixArgs: 1, totalArgs: params.size());
754 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args: params);
755
756 // FIXME: Use the block's target features when arranging its invoke function.
757 return arrangeLLVMFunctionInfo(
758 returnType: GetReturnType(RetTy: proto->getReturnType()), opts: FnInfoOpts::None, argTypes,
759 info: proto->getExtInfo(), paramInfos, args: RequiredArgs::forPrototypePlus(prototype: proto, additional: 1),
760 /*ABIInfoFD=*/nullptr);
761}
762
763const CGFunctionInfo &
764CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
765 const CallArgList &args) {
766 CanQualTypeList argTypes;
767 for (const auto &Arg : args)
768 argTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
769 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
770 argTypes, info: FunctionType::ExtInfo(),
771 /*paramInfos=*/{}, args: RequiredArgs::All, ABIInfoFD: nullptr);
772}
773
774const CGFunctionInfo &
775CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
776 const FunctionArgList &args) {
777 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
778
779 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
780 argTypes, info: FunctionType::ExtInfo(), paramInfos: {},
781 args: RequiredArgs::All, /*ABIInfoFD=*/nullptr);
782}
783
784const CGFunctionInfo &CodeGenTypes::arrangeBuiltinFunctionDeclaration(
785 CanQualType resultType, ArrayRef<CanQualType> argTypes) {
786 return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::None, argTypes,
787 info: FunctionType::ExtInfo(), paramInfos: {}, args: RequiredArgs::All,
788 /*ABIInfoFD=*/nullptr);
789}
790
791const CGFunctionInfo &CodeGenTypes::arrangeDeviceKernelCallerDeclaration(
792 QualType resultType, const FunctionArgList &args) {
793 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
794
795 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
796 argTypes,
797 info: FunctionType::ExtInfo(CC_DeviceKernel),
798 /*paramInfos=*/{}, args: RequiredArgs::All,
799 /*ABIInfoFD=*/nullptr);
800}
801
802/// Arrange a call to a C++ method, passing the given arguments.
803///
804/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
805/// does not count `this`.
806const CGFunctionInfo &CodeGenTypes::arrangeCXXMethodCall(
807 const CallArgList &args, const FunctionProtoType *proto,
808 RequiredArgs required, unsigned numPrefixArgs,
809 const FunctionDecl *ABIInfoFD) {
810 assert(numPrefixArgs + 1 <= args.size() &&
811 "Emitting a call with less args than the required prefix?");
812 // Add one to account for `this`. It's a bit awkward here, but we don't count
813 // `this` in similar places elsewhere.
814 ExtParameterInfoList paramInfos =
815 getExtParameterInfosForCall(proto, prefixArgs: numPrefixArgs + 1, totalArgs: args.size());
816
817 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
818
819 FunctionType::ExtInfo info = proto->getExtInfo();
820 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: proto->getReturnType()),
821 opts: FnInfoOpts::IsInstanceMethod, argTypes, info,
822 paramInfos, args: required, ABIInfoFD);
823}
824
825const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
826 return arrangeLLVMFunctionInfo(returnType: getContext().VoidTy, opts: FnInfoOpts::None, argTypes: {},
827 info: FunctionType::ExtInfo(), paramInfos: {}, args: RequiredArgs::All,
828 /*ABIInfoFD=*/nullptr);
829}
830
831const CGFunctionInfo &CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
832 const CallArgList &args,
833 const FunctionDecl *ABIInfoFD) {
834 assert(signature.arg_size() <= args.size());
835 unsigned X86ABIAVXLevel =
836 CGM.getABIInfo().getX86ABIAVXLevel(ABIInfoFD, signature.getExtInfo());
837 if (signature.arg_size() == args.size() &&
838 signature.getX86ABIAVXLevel() == X86ABIAVXLevel)
839 return signature;
840
841 ExtParameterInfoList paramInfos;
842 auto sigParamInfos = signature.getExtParameterInfos();
843 if (!sigParamInfos.empty()) {
844 paramInfos.append(in_start: sigParamInfos.begin(), in_end: sigParamInfos.end());
845 paramInfos.resize(N: args.size());
846 }
847
848 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
849
850 assert(signature.getRequiredArgs().allowsOptionalArgs());
851 FnInfoOpts opts = FnInfoOpts::None;
852 if (signature.isInstanceMethod())
853 opts |= FnInfoOpts::IsInstanceMethod;
854 if (signature.isChainCall())
855 opts |= FnInfoOpts::IsChainCall;
856 if (signature.isDelegateCall())
857 opts |= FnInfoOpts::IsDelegateCall;
858
859 const CGFunctionInfo *newFI = findOrInsertCGFunctionInfo(
860 isInstanceMethod: signature.isInstanceMethod(), isChainCall: signature.isChainCall(),
861 isDelegateCall: signature.isDelegateCall(), X86ABIAVXLevel, info: signature.getExtInfo(),
862 paramInfos, required: signature.getRequiredArgs(), resultType: signature.getReturnType(),
863 argTypes);
864 return *newFI;
865}
866
867namespace clang {
868namespace CodeGen {
869void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
870} // namespace CodeGen
871} // namespace clang
872
873#ifndef NDEBUG
874static const char *abiKindToString(ABIArgInfo::Kind K) {
875 switch (K) {
876 case ABIArgInfo::Direct:
877 return "Direct";
878 case ABIArgInfo::Extend:
879 return "Extend";
880 case ABIArgInfo::Indirect:
881 return "Indirect";
882 case ABIArgInfo::IndirectAliased:
883 return "IndirectAliased";
884 case ABIArgInfo::Ignore:
885 return "Ignore";
886 case ABIArgInfo::Expand:
887 return "Expand";
888 case ABIArgInfo::CoerceAndExpand:
889 return "CoerceAndExpand";
890 case ABIArgInfo::TargetSpecific:
891 return "TargetSpecific";
892 case ABIArgInfo::InAlloca:
893 return "InAlloca";
894 }
895 llvm_unreachable("Unknown kind");
896}
897#endif
898
899void CodeGenModule::computeABIInfoUsingLib(CGFunctionInfo &FI) {
900 SmallVector<const llvm::abi::Type *> MappedArgTypes;
901 MappedArgTypes.reserve(N: FI.arg_size());
902 for (const auto &Arg : FI.arguments())
903 MappedArgTypes.push_back(Elt: AbiMapper->convertType(QT: Arg.type));
904
905 std::optional<unsigned> NumRequired;
906 RequiredArgs Required = FI.getRequiredArgs();
907 if (Required.allowsOptionalArgs())
908 NumRequired = Required.getNumRequiredArgs();
909
910 auto AbiFI = llvm::abi::FunctionInfo::create(
911 CC: FI.getCallingConvention(), ReturnType: AbiMapper->convertType(QT: FI.getReturnType()),
912 ArgTypes: MappedArgTypes, NumRequired);
913
914 getLLVMABITargetInfo(TB&: AbiMapper->getTypeBuilder()).computeInfo(FI&: *AbiFI);
915
916#ifndef NDEBUG
917 // With assertions enabled, also compute info using Clang ABI logic,
918 // so we can ensure the results are consistent.
919 getABIInfo().computeInfo(FI);
920
921 auto ConvertABIArgInfo = [&](ABIArgInfo &Target,
922 const llvm::abi::ArgInfo &AbiInfo, QualType Type,
923 int ArgNo) {
924 auto Check = [&](bool Cond, llvm::function_ref<void()> MessageFn) {
925 if (Cond)
926 return;
927 if (ArgNo == -1)
928 llvm::dbgs() << "For return value of type ";
929 else
930 llvm::dbgs() << "For argument " << ArgNo << " of type ";
931 llvm::dbgs() << Type << ": ";
932 MessageFn();
933 llvm::dbgs() << "\n";
934 abort();
935 };
936 auto CheckSimple = [&](auto TargetVal, auto ResVal, StringRef What) {
937 Check(TargetVal == ResVal, [&]() {
938 llvm::dbgs() << What << " mismatch (expected: " << TargetVal
939 << ", given: " << ResVal << ")";
940 });
941 };
942
943 ABIArgInfo Res = convertABIArgInfo(AbiInfo, Type);
944 Check(Target.getKind() == Res.getKind(), [&]() {
945 llvm::dbgs() << "Kind mismatch (expected: "
946 << abiKindToString(Target.getKind())
947 << ", given: " << abiKindToString(Res.getKind()) << ")";
948 });
949
950 if (Res.canHaveCoerceToType()) {
951 // Normalize nullptr types.
952 llvm::Type *TargetType = Target.getCoerceToType();
953 llvm::Type *ResType = Res.getCoerceToType();
954 if (!TargetType)
955 TargetType = getTypes().ConvertType(Type);
956 if (!ResType)
957 ResType = getTypes().ConvertType(Type);
958
959 Check(TargetType == ResType, [&]() {
960 llvm::dbgs() << "CoerceToType mismatch (expected: " << *TargetType
961 << ", given: " << *ResType << ")";
962 });
963 }
964
965 switch (Res.getKind()) {
966 case ABIArgInfo::Extend:
967 CheckSimple(Target.isSignExt(), Res.isSignExt(), "SignExt");
968 CheckSimple(Target.isZeroExt(), Res.isZeroExt(), "ZeroExt");
969 [[fallthrough]];
970 case ABIArgInfo::Direct:
971 CheckSimple(Target.getDirectAlign(), Res.getDirectAlign(), "DirectAlign");
972 CheckSimple(Target.getDirectOffset(), Res.getDirectOffset(),
973 "DirectOffset");
974 break;
975 case ABIArgInfo::Indirect:
976 CheckSimple(Target.getIndirectByVal(), Res.getIndirectByVal(),
977 "IndirectByVal");
978 [[fallthrough]];
979 case ABIArgInfo::IndirectAliased:
980 CheckSimple(Target.getIndirectAddrSpace(), Res.getIndirectAddrSpace(),
981 "IndirectAddrSpace");
982 CheckSimple(Target.getIndirectRealign(), Res.getIndirectRealign(),
983 "IndirectRealign");
984 Check(Target.getIndirectAlign() == Res.getIndirectAlign(), [&]() {
985 llvm::dbgs() << "IndirectAlign mismatch (expected: "
986 << Target.getIndirectAlign().getQuantity()
987 << ", given: " << Res.getIndirectAlign().getQuantity()
988 << ")";
989 });
990 break;
991 default:
992 break;
993 }
994
995 Target = Res;
996 };
997#else
998 auto ConvertABIArgInfo =
999 [&](ABIArgInfo &Target, const llvm::abi::ArgInfo &AbiInfo, QualType Type,
1000 int ArgNo) { Target = convertABIArgInfo(AbiInfo, Type); };
1001#endif
1002
1003 ConvertABIArgInfo(FI.getReturnInfo(), AbiFI->getReturnInfo(),
1004 FI.getReturnType(), -1);
1005
1006 int ArgNo = 0;
1007 for (auto [CGArg, AbiArg] :
1008 llvm::zip_equal(t: FI.arguments(), u: AbiFI->arguments()))
1009 ConvertABIArgInfo(CGArg.info, AbiArg.Info, CGArg.type, ArgNo++);
1010}
1011
1012ABIArgInfo CodeGenModule::convertABIArgInfo(const llvm::abi::ArgInfo &AbiInfo,
1013 QualType Type) {
1014 switch (AbiInfo.getKind()) {
1015 case llvm::abi::ArgInfo::Direct: {
1016 llvm::Type *CoercedType = nullptr;
1017 if (AbiInfo.getCoerceToType())
1018 CoercedType = AbiReverseMapper->convertType(ABIType: AbiInfo.getCoerceToType());
1019 if (!CoercedType)
1020 CoercedType = getTypes().ConvertType(T: Type);
1021 return ABIArgInfo::getDirect(T: CoercedType, Offset: AbiInfo.getDirectOffset());
1022 }
1023 case llvm::abi::ArgInfo::Extend: {
1024 llvm::Type *CoercedType = nullptr;
1025 if (AbiInfo.getCoerceToType())
1026 CoercedType = AbiReverseMapper->convertType(ABIType: AbiInfo.getCoerceToType());
1027 if (!CoercedType)
1028 CoercedType = getTypes().ConvertType(T: Type);
1029 // A transparent union is passed as its first field, so the extend keys off
1030 // that field's integral type, matching the classifier's
1031 // useFirstFieldIfTransparentUnion. Passing the union type to
1032 // ABIArgInfo::getSignExtend would trip its integral-type assert.
1033 QualType ExtendType = useFirstFieldIfTransparentUnion(Ty: Type);
1034 if (AbiInfo.isSignExt())
1035 return ABIArgInfo::getSignExtend(Ty: ExtendType, T: CoercedType);
1036 if (AbiInfo.isZeroExt())
1037 return ABIArgInfo::getZeroExtend(Ty: ExtendType, T: CoercedType);
1038 return ABIArgInfo::getExtend(Ty: ExtendType, T: CoercedType);
1039 }
1040 case llvm::abi::ArgInfo::Indirect: {
1041 CharUnits Alignment =
1042 CharUnits::fromQuantity(Quantity: AbiInfo.getIndirectAlign().value());
1043 return ABIArgInfo::getIndirect(Alignment, AddrSpace: AbiInfo.getIndirectAddrSpace(),
1044 ByVal: AbiInfo.getIndirectByVal(),
1045 Realign: AbiInfo.getIndirectRealign());
1046 }
1047 case llvm::abi::ArgInfo::Ignore:
1048 return ABIArgInfo::getIgnore();
1049 }
1050 llvm_unreachable("Unexpected llvm::abi::ArgInfo kind");
1051}
1052
1053/// Arrange the argument and result information for an abstract value
1054/// of a given function type. This is the method which all of the
1055/// above functions ultimately defer to.
1056const CGFunctionInfo &CodeGenTypes::arrangeLLVMFunctionInfo(
1057 CanQualType resultType, FnInfoOpts opts, ArrayRef<CanQualType> argTypes,
1058 FunctionType::ExtInfo info,
1059 ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
1060 RequiredArgs required, const FunctionDecl *ABIInfoFD) {
1061 assert(llvm::all_of(argTypes,
1062 [](CanQualType T) { return T.isCanonicalAsParam(); }));
1063
1064 // Lookup or create unique function info.
1065 llvm::FoldingSetNodeID ID;
1066 bool isInstanceMethod =
1067 (opts & FnInfoOpts::IsInstanceMethod) == FnInfoOpts::IsInstanceMethod;
1068 bool isChainCall =
1069 (opts & FnInfoOpts::IsChainCall) == FnInfoOpts::IsChainCall;
1070 bool isDelegateCall =
1071 (opts & FnInfoOpts::IsDelegateCall) == FnInfoOpts::IsDelegateCall;
1072 unsigned X86ABIAVXLevel = CGM.getABIInfo().getX86ABIAVXLevel(ABIInfoFD, info);
1073
1074 const CGFunctionInfo *newFI = findOrInsertCGFunctionInfo(
1075 isInstanceMethod, isChainCall, isDelegateCall, X86ABIAVXLevel, info,
1076 paramInfos, required, resultType, argTypes);
1077 return *newFI;
1078}
1079
1080CGFunctionInfo *CodeGenTypes::findOrInsertCGFunctionInfo(
1081 bool isInstanceMethod, bool isChainCall, bool isDelegateCall,
1082 unsigned X86ABIAVXLevel, const FunctionType::ExtInfo &info,
1083 ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
1084 RequiredArgs required, CanQualType resultType,
1085 ArrayRef<CanQualType> argTypes) {
1086 llvm::FoldingSetNodeID ID;
1087 CGFunctionInfo::Profile(ID, InstanceMethod: isInstanceMethod, ChainCall: isChainCall, IsDelegateCall: isDelegateCall,
1088 X86ABIAVXLevel, info, paramInfos, required,
1089 resultType, argTypes);
1090
1091 void *insertPos = nullptr;
1092 CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, InsertPos&: insertPos);
1093 if (FI)
1094 return FI;
1095
1096 unsigned CC = ClangCallConvToLLVMCallConv(CC: info.getCC());
1097
1098 // Construct the function info. We co-allocate the ArgInfos.
1099 FI = CGFunctionInfo::create(llvmCC: CC, instanceMethod: isInstanceMethod, chainCall: isChainCall, delegateCall: isDelegateCall,
1100 X86ABIAVXLevel, extInfo: info, paramInfos, resultType,
1101 argTypes, required);
1102 FunctionInfos.InsertNode(N: FI, InsertPos: insertPos);
1103
1104 bool inserted = FunctionsBeingProcessed.insert(Ptr: FI).second;
1105 (void)inserted;
1106 assert(inserted && "Recursively being processed?");
1107
1108 // Compute ABI information.
1109 if (info.getCC() == CC_DeviceKernel &&
1110 (CC == llvm::CallingConv::SPIR_KERNEL || CC == llvm::CallingConv::C)) {
1111 // Force target independent argument handling for the host visible
1112 // kernel functions.
1113 //
1114 // For CPU targets, this currently only works for OpenCL.
1115 assert(CC != llvm::CallingConv::C || getContext().getLangOpts().OpenCL);
1116 computeSPIRKernelABIInfo(CGM, FI&: *FI);
1117 } else if (info.getCC() == CC_Swift || info.getCC() == CC_SwiftAsync) {
1118 swiftcall::computeABIInfo(CGM, FI&: *FI);
1119 } else if (CGM.shouldUseLLVMABILowering(CallingConv: CC)) {
1120 CGM.computeABIInfoUsingLib(FI&: *FI);
1121 } else {
1122 CGM.getABIInfo().computeInfo(FI&: *FI);
1123 }
1124
1125 // Loop over all of the computed argument and return value info. If any of
1126 // them are direct or extend without a specified coerce type, specify the
1127 // default now.
1128 ABIArgInfo &retInfo = FI->getReturnInfo();
1129 if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
1130 retInfo.setCoerceToType(ConvertType(T: FI->getReturnType()));
1131
1132 for (auto &I : FI->arguments())
1133 if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
1134 I.info.setCoerceToType(ConvertType(T: I.type));
1135
1136 bool erased = FunctionsBeingProcessed.erase(Ptr: FI);
1137 (void)erased;
1138 assert(erased && "Not in set?");
1139
1140 return FI;
1141}
1142
1143CGFunctionInfo *CGFunctionInfo::create(
1144 unsigned llvmCC, bool instanceMethod, bool chainCall, bool delegateCall,
1145 unsigned X86ABIAVXLevel, const FunctionType::ExtInfo &info,
1146 ArrayRef<ExtParameterInfo> paramInfos, CanQualType resultType,
1147 ArrayRef<CanQualType> argTypes, RequiredArgs required) {
1148 assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
1149 assert(!required.allowsOptionalArgs() ||
1150 required.getNumRequiredArgs() <= argTypes.size());
1151
1152 void *buffer = operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(
1153 Counts: argTypes.size() + 1, Counts: paramInfos.size()));
1154
1155 CGFunctionInfo *FI = new (buffer) CGFunctionInfo();
1156 FI->CallingConvention = llvmCC;
1157 FI->EffectiveCallingConvention = llvmCC;
1158 FI->ASTCallingConvention = info.getCC();
1159 FI->InstanceMethod = instanceMethod;
1160 FI->ChainCall = chainCall;
1161 FI->DelegateCall = delegateCall;
1162 FI->CmseNSCall = info.getCmseNSCall();
1163 FI->NoReturn = info.getNoReturn();
1164 FI->ReturnsRetained = info.getProducesResult();
1165 FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
1166 FI->NoCfCheck = info.getNoCfCheck();
1167 FI->Required = required;
1168 FI->HasRegParm = info.getHasRegParm();
1169 FI->RegParm = info.getRegParm();
1170 FI->X86ABIAVXLevel = X86ABIAVXLevel;
1171 FI->ArgStruct = nullptr;
1172 FI->ArgStructAlign = 0;
1173 FI->NumArgs = argTypes.size();
1174 FI->HasExtParameterInfos = !paramInfos.empty();
1175 FI->getArgsBuffer()[0].type = resultType;
1176 FI->MaxVectorWidth = 0;
1177 for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
1178 FI->getArgsBuffer()[i + 1].type = argTypes[i];
1179 for (unsigned i = 0, e = paramInfos.size(); i != e; ++i)
1180 FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
1181 return FI;
1182}
1183
1184/***/
1185
1186namespace {
1187// ABIArgInfo::Expand implementation.
1188
1189// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
1190struct TypeExpansion {
1191 enum TypeExpansionKind {
1192 // Elements of constant arrays are expanded recursively.
1193 TEK_ConstantArray,
1194 // Record fields are expanded recursively (but if record is a union, only
1195 // the field with the largest size is expanded).
1196 TEK_Record,
1197 // For complex types, real and imaginary parts are expanded recursively.
1198 TEK_Complex,
1199 // All other types are not expandable.
1200 TEK_None
1201 };
1202
1203 const TypeExpansionKind Kind;
1204
1205 TypeExpansion(TypeExpansionKind K) : Kind(K) {}
1206 virtual ~TypeExpansion() {}
1207};
1208
1209struct ConstantArrayExpansion : TypeExpansion {
1210 QualType EltTy;
1211 uint64_t NumElts;
1212
1213 ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
1214 : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
1215 static bool classof(const TypeExpansion *TE) {
1216 return TE->Kind == TEK_ConstantArray;
1217 }
1218};
1219
1220struct RecordExpansion : TypeExpansion {
1221 SmallVector<const CXXBaseSpecifier *, 1> Bases;
1222
1223 SmallVector<const FieldDecl *, 1> Fields;
1224
1225 RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
1226 SmallVector<const FieldDecl *, 1> &&Fields)
1227 : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
1228 Fields(std::move(Fields)) {}
1229 static bool classof(const TypeExpansion *TE) {
1230 return TE->Kind == TEK_Record;
1231 }
1232};
1233
1234struct ComplexExpansion : TypeExpansion {
1235 QualType EltTy;
1236
1237 ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
1238 static bool classof(const TypeExpansion *TE) {
1239 return TE->Kind == TEK_Complex;
1240 }
1241};
1242
1243struct NoExpansion : TypeExpansion {
1244 NoExpansion() : TypeExpansion(TEK_None) {}
1245 static bool classof(const TypeExpansion *TE) { return TE->Kind == TEK_None; }
1246};
1247} // namespace
1248
1249static std::unique_ptr<TypeExpansion>
1250getTypeExpansion(QualType Ty, const ASTContext &Context) {
1251 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty)) {
1252 return std::make_unique<ConstantArrayExpansion>(args: AT->getElementType(),
1253 args: AT->getZExtSize());
1254 }
1255 if (const auto *RD = Ty->getAsRecordDecl()) {
1256 SmallVector<const CXXBaseSpecifier *, 1> Bases;
1257 SmallVector<const FieldDecl *, 1> Fields;
1258 assert(!RD->hasFlexibleArrayMember() &&
1259 "Cannot expand structure with flexible array.");
1260 if (RD->isUnion()) {
1261 // Unions can be here only in degenerative cases - all the fields are same
1262 // after flattening. Thus we have to use the "largest" field.
1263 const FieldDecl *LargestFD = nullptr;
1264 CharUnits UnionSize = CharUnits::Zero();
1265
1266 for (const auto *FD : RD->fields()) {
1267 if (FD->isZeroLengthBitField())
1268 continue;
1269 assert(!FD->isBitField() &&
1270 "Cannot expand structure with bit-field members.");
1271 CharUnits FieldSize = Context.getTypeSizeInChars(T: FD->getType());
1272 if (UnionSize < FieldSize) {
1273 UnionSize = FieldSize;
1274 LargestFD = FD;
1275 }
1276 }
1277 if (LargestFD)
1278 Fields.push_back(Elt: LargestFD);
1279 } else {
1280 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
1281 assert(!CXXRD->isDynamicClass() &&
1282 "cannot expand vtable pointers in dynamic classes");
1283 llvm::append_range(C&: Bases, R: llvm::make_pointer_range(Range: CXXRD->bases()));
1284 }
1285
1286 for (const auto *FD : RD->fields()) {
1287 if (FD->isZeroLengthBitField())
1288 continue;
1289 assert(!FD->isBitField() &&
1290 "Cannot expand structure with bit-field members.");
1291 Fields.push_back(Elt: FD);
1292 }
1293 }
1294 return std::make_unique<RecordExpansion>(args: std::move(Bases),
1295 args: std::move(Fields));
1296 }
1297 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
1298 return std::make_unique<ComplexExpansion>(args: CT->getElementType());
1299 }
1300 return std::make_unique<NoExpansion>();
1301}
1302
1303static int getExpansionSize(QualType Ty, const ASTContext &Context) {
1304 auto Exp = getTypeExpansion(Ty, Context);
1305 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1306 return CAExp->NumElts * getExpansionSize(Ty: CAExp->EltTy, Context);
1307 }
1308 if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1309 int Res = 0;
1310 for (auto BS : RExp->Bases)
1311 Res += getExpansionSize(Ty: BS->getType(), Context);
1312 for (auto FD : RExp->Fields)
1313 Res += getExpansionSize(Ty: FD->getType(), Context);
1314 return Res;
1315 }
1316 if (isa<ComplexExpansion>(Val: Exp.get()))
1317 return 2;
1318 assert(isa<NoExpansion>(Exp.get()));
1319 return 1;
1320}
1321
1322void CodeGenTypes::getExpandedTypes(
1323 QualType Ty, SmallVectorImpl<llvm::Type *>::iterator &TI) {
1324 auto Exp = getTypeExpansion(Ty, Context);
1325 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1326 for (int i = 0, n = CAExp->NumElts; i < n; i++) {
1327 getExpandedTypes(Ty: CAExp->EltTy, TI);
1328 }
1329 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1330 for (auto BS : RExp->Bases)
1331 getExpandedTypes(Ty: BS->getType(), TI);
1332 for (auto FD : RExp->Fields)
1333 getExpandedTypes(Ty: FD->getType(), TI);
1334 } else if (auto CExp = dyn_cast<ComplexExpansion>(Val: Exp.get())) {
1335 llvm::Type *EltTy = ConvertType(T: CExp->EltTy);
1336 *TI++ = EltTy;
1337 *TI++ = EltTy;
1338 } else {
1339 assert(isa<NoExpansion>(Exp.get()));
1340 *TI++ = ConvertType(T: Ty);
1341 }
1342}
1343
1344static void forConstantArrayExpansion(CodeGenFunction &CGF,
1345 ConstantArrayExpansion *CAE,
1346 Address BaseAddr,
1347 llvm::function_ref<void(Address)> Fn) {
1348 for (int i = 0, n = CAE->NumElts; i < n; i++) {
1349 Address EltAddr = CGF.Builder.CreateConstGEP2_32(Addr: BaseAddr, Idx0: 0, Idx1: i);
1350 Fn(EltAddr);
1351 }
1352}
1353
1354void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1355 llvm::Function::arg_iterator &AI) {
1356 assert(LV.isSimple() &&
1357 "Unexpected non-simple lvalue during struct expansion.");
1358
1359 auto Exp = getTypeExpansion(Ty, Context: getContext());
1360 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1361 forConstantArrayExpansion(
1362 CGF&: *this, CAE: CAExp, BaseAddr: LV.getAddress(), Fn: [&](Address EltAddr) {
1363 LValue LV = MakeAddrLValue(Addr: EltAddr, T: CAExp->EltTy);
1364 ExpandTypeFromArgs(Ty: CAExp->EltTy, LV, AI);
1365 });
1366 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1367 Address This = LV.getAddress();
1368 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1369 // Perform a single step derived-to-base conversion.
1370 Address Base =
1371 GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1372 /*NullCheckValue=*/false, Loc: SourceLocation());
1373 LValue SubLV = MakeAddrLValue(Addr: Base, T: BS->getType());
1374
1375 // Recurse onto bases.
1376 ExpandTypeFromArgs(Ty: BS->getType(), LV: SubLV, AI);
1377 }
1378 for (auto FD : RExp->Fields) {
1379 // FIXME: What are the right qualifiers here?
1380 LValue SubLV = EmitLValueForFieldInitialization(Base: LV, Field: FD);
1381 ExpandTypeFromArgs(Ty: FD->getType(), LV: SubLV, AI);
1382 }
1383 } else if (isa<ComplexExpansion>(Val: Exp.get())) {
1384 auto realValue = &*AI++;
1385 auto imagValue = &*AI++;
1386 EmitStoreOfComplex(V: ComplexPairTy(realValue, imagValue), dest: LV, /*init*/ isInit: true);
1387 } else {
1388 // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1389 // primitive store.
1390 assert(isa<NoExpansion>(Exp.get()));
1391 llvm::Value *Arg = &*AI++;
1392 if (LV.isBitField()) {
1393 EmitStoreThroughLValue(Src: RValue::get(V: Arg), Dst: LV);
1394 } else {
1395 // TODO: currently there are some places are inconsistent in what LLVM
1396 // pointer type they use (see D118744). Once clang uses opaque pointers
1397 // all LLVM pointer types will be the same and we can remove this check.
1398 if (Arg->getType()->isPointerTy()) {
1399 Address Addr = LV.getAddress();
1400 Arg = Builder.CreateBitCast(V: Arg, DestTy: Addr.getElementType());
1401 }
1402 EmitStoreOfScalar(value: Arg, lvalue: LV);
1403 }
1404 }
1405}
1406
1407void CodeGenFunction::ExpandTypeToArgs(
1408 QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1409 SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1410 auto Exp = getTypeExpansion(Ty, Context: getContext());
1411 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1412 Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1413 : Arg.getKnownRValue().getAggregateAddress();
1414 forConstantArrayExpansion(CGF&: *this, CAE: CAExp, BaseAddr: Addr, Fn: [&](Address EltAddr) {
1415 CallArg EltArg =
1416 CallArg(convertTempToRValue(addr: EltAddr, type: CAExp->EltTy, Loc: SourceLocation()),
1417 CAExp->EltTy);
1418 ExpandTypeToArgs(Ty: CAExp->EltTy, Arg: EltArg, IRFuncTy, IRCallArgs,
1419 IRCallArgPos);
1420 });
1421 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1422 Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1423 : Arg.getKnownRValue().getAggregateAddress();
1424 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1425 // Perform a single step derived-to-base conversion.
1426 Address Base =
1427 GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1428 /*NullCheckValue=*/false, Loc: SourceLocation());
1429 CallArg BaseArg = CallArg(RValue::getAggregate(addr: Base), BS->getType());
1430
1431 // Recurse onto bases.
1432 ExpandTypeToArgs(Ty: BS->getType(), Arg: BaseArg, IRFuncTy, IRCallArgs,
1433 IRCallArgPos);
1434 }
1435
1436 LValue LV = MakeAddrLValue(Addr: This, T: Ty);
1437 for (auto FD : RExp->Fields) {
1438 CallArg FldArg =
1439 CallArg(EmitRValueForField(LV, FD, Loc: SourceLocation()), FD->getType());
1440 ExpandTypeToArgs(Ty: FD->getType(), Arg: FldArg, IRFuncTy, IRCallArgs,
1441 IRCallArgPos);
1442 }
1443 } else if (isa<ComplexExpansion>(Val: Exp.get())) {
1444 ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1445 IRCallArgs[IRCallArgPos++] = CV.first;
1446 IRCallArgs[IRCallArgPos++] = CV.second;
1447 } else {
1448 assert(isa<NoExpansion>(Exp.get()));
1449 auto RV = Arg.getKnownRValue();
1450 assert(RV.isScalar() &&
1451 "Unexpected non-scalar rvalue during struct expansion.");
1452
1453 // Insert a bitcast as needed.
1454 llvm::Value *V = RV.getScalarVal();
1455 if (IRCallArgPos < IRFuncTy->getNumParams() &&
1456 V->getType() != IRFuncTy->getParamType(i: IRCallArgPos))
1457 V = Builder.CreateBitCast(V, DestTy: IRFuncTy->getParamType(i: IRCallArgPos));
1458
1459 IRCallArgs[IRCallArgPos++] = V;
1460 }
1461}
1462
1463/// Create a temporary allocation for the purposes of coercion.
1464static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF,
1465 llvm::Type *Ty,
1466 CharUnits MinAlign,
1467 const Twine &Name = "tmp") {
1468 // Don't use an alignment that's worse than what LLVM would prefer.
1469 auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty);
1470 CharUnits Align = std::max(a: MinAlign, b: CharUnits::fromQuantity(Quantity: PrefAlign));
1471
1472 return CGF.CreateTempAlloca(Ty, align: Align, Name: Name + ".coerce");
1473}
1474
1475/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1476/// accessing some number of bytes out of it, try to gep into the struct to get
1477/// at its inner goodness. Dive as deep as possible without entering an element
1478/// with an in-memory size smaller than DstSize.
1479static Address EnterStructPointerForCoercedAccess(Address SrcPtr,
1480 llvm::StructType *SrcSTy,
1481 uint64_t DstSize,
1482 CodeGenFunction &CGF) {
1483 // We can't dive into a zero-element struct.
1484 if (SrcSTy->getNumElements() == 0)
1485 return SrcPtr;
1486
1487 llvm::Type *FirstElt = SrcSTy->getElementType(N: 0);
1488
1489 // If the first elt is at least as large as what we're looking for, or if the
1490 // first element is the same size as the whole struct, we can enter it. The
1491 // comparison must be made on the store size and not the alloca size. Using
1492 // the alloca size may overstate the size of the load.
1493 uint64_t FirstEltSize = CGF.CGM.getDataLayout().getTypeStoreSize(Ty: FirstElt);
1494 if (FirstEltSize < DstSize &&
1495 FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(Ty: SrcSTy))
1496 return SrcPtr;
1497
1498 // GEP into the first element.
1499 SrcPtr = CGF.Builder.CreateStructGEP(Addr: SrcPtr, Index: 0, Name: "coerce.dive");
1500
1501 // If the first element is a struct, recurse.
1502 llvm::Type *SrcTy = SrcPtr.getElementType();
1503 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy))
1504 return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1505
1506 return SrcPtr;
1507}
1508
1509/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1510/// are either integers or pointers. This does a truncation of the value if it
1511/// is too large or a zero extension if it is too small.
1512///
1513/// This behaves as if the value were coerced through memory, so on big-endian
1514/// targets the high bits are preserved in a truncation, while little-endian
1515/// targets preserve the low bits.
1516static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, llvm::Type *Ty,
1517 CodeGenFunction &CGF) {
1518 if (Val->getType() == Ty)
1519 return Val;
1520
1521 if (isa<llvm::PointerType>(Val: Val->getType())) {
1522 // If this is Pointer->Pointer avoid conversion to and from int.
1523 if (isa<llvm::PointerType>(Val: Ty))
1524 return CGF.Builder.CreateBitCast(V: Val, DestTy: Ty, Name: "coerce.val");
1525
1526 // Convert the pointer to an integer so we can play with its width.
1527 Val = CGF.Builder.CreatePtrToInt(V: Val, DestTy: CGF.IntPtrTy, Name: "coerce.val.pi");
1528 }
1529
1530 llvm::Type *DestIntTy = Ty;
1531 if (isa<llvm::PointerType>(Val: DestIntTy))
1532 DestIntTy = CGF.IntPtrTy;
1533
1534 if (Val->getType() != DestIntTy) {
1535 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1536 if (DL.isBigEndian()) {
1537 // Preserve the high bits on big-endian targets.
1538 // That is what memory coercion does.
1539 uint64_t SrcSize = DL.getTypeSizeInBits(Ty: Val->getType());
1540 uint64_t DstSize = DL.getTypeSizeInBits(Ty: DestIntTy);
1541
1542 if (SrcSize > DstSize) {
1543 Val = CGF.Builder.CreateLShr(LHS: Val, RHS: SrcSize - DstSize, Name: "coerce.highbits");
1544 Val = CGF.Builder.CreateTrunc(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1545 } else {
1546 Val = CGF.Builder.CreateZExt(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1547 Val = CGF.Builder.CreateShl(LHS: Val, RHS: DstSize - SrcSize, Name: "coerce.highbits");
1548 }
1549 } else {
1550 // Little-endian targets preserve the low bits. No shifts required.
1551 Val = CGF.Builder.CreateIntCast(V: Val, DestTy: DestIntTy, isSigned: false, Name: "coerce.val.ii");
1552 }
1553 }
1554
1555 if (isa<llvm::PointerType>(Val: Ty))
1556 Val = CGF.Builder.CreateIntToPtr(V: Val, DestTy: Ty, Name: "coerce.val.ip");
1557 return Val;
1558}
1559
1560static llvm::Value *CreatePFPCoercedLoad(Address Src, QualType SrcFETy,
1561 llvm::Type *Ty, CodeGenFunction &CGF) {
1562 std::vector<PFPField> PFPFields = CGF.getContext().findPFPFields(Ty: SrcFETy);
1563 if (PFPFields.empty())
1564 return nullptr;
1565
1566 auto LoadCoercedField = [&](CharUnits Offset,
1567 llvm::Type *FieldType) -> llvm::Value * {
1568 // Check whether the field at Offset is a PFP field. This function is called
1569 // in ascending order of offset, and PFPFields is sorted by offset. This
1570 // means that we only need to check the first element (and remove it from
1571 // PFPFields if matching).
1572 if (!PFPFields.empty() && PFPFields[0].Offset == Offset) {
1573 auto FieldAddr = CGF.EmitAddressOfPFPField(RecordPtr: Src, Field: PFPFields[0]);
1574 llvm::Value *FieldVal = CGF.Builder.CreateLoad(Addr: FieldAddr);
1575 if (isa<llvm::IntegerType>(Val: FieldType))
1576 FieldVal = CGF.Builder.CreatePtrToInt(V: FieldVal, DestTy: FieldType);
1577 PFPFields.erase(position: PFPFields.begin());
1578 return FieldVal;
1579 }
1580 auto FieldAddr =
1581 CGF.Builder
1582 .CreateConstInBoundsByteGEP(Addr: Src.withElementType(ElemTy: CGF.Int8Ty), Offset)
1583 .withElementType(ElemTy: FieldType);
1584 return CGF.Builder.CreateLoad(Addr: FieldAddr);
1585 };
1586
1587 // The types handled by this function are the only ones that may be generated
1588 // by AArch64ABIInfo::classify{Argument,Return}Type for struct types with
1589 // pointers. PFP is only supported on AArch64.
1590 if (isa<llvm::IntegerType>(Val: Ty) || isa<llvm::PointerType>(Val: Ty)) {
1591 auto Addr = CGF.EmitAddressOfPFPField(RecordPtr: Src, Field: PFPFields[0]);
1592 llvm::Value *Val = CGF.Builder.CreateLoad(Addr);
1593 if (isa<llvm::IntegerType>(Val: Ty))
1594 Val = CGF.Builder.CreatePtrToInt(V: Val, DestTy: Ty);
1595 return Val;
1596 }
1597 auto *AT = cast<llvm::ArrayType>(Val: Ty);
1598 auto *ET = AT->getElementType();
1599 CharUnits WordSize = CGF.getContext().toCharUnitsFromBits(
1600 BitSize: CGF.CGM.getDataLayout().getTypeSizeInBits(Ty: ET));
1601 CharUnits Offset = CharUnits::Zero();
1602 llvm::Value *Val = llvm::PoisonValue::get(T: AT);
1603 for (unsigned Idx = 0; Idx != AT->getNumElements(); ++Idx, Offset += WordSize)
1604 Val = CGF.Builder.CreateInsertValue(Agg: Val, Val: LoadCoercedField(Offset, ET), Idxs: Idx);
1605 return Val;
1606}
1607
1608/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1609/// a pointer to an object of type \arg Ty, known to be aligned to
1610/// \arg SrcAlign bytes.
1611///
1612/// This safely handles the case when the src type is smaller than the
1613/// destination type; in this situation the values of bits which not
1614/// present in the src are undefined.
1615static llvm::Value *CreateCoercedLoad(Address Src, QualType SrcFETy,
1616 llvm::Type *Ty, CodeGenFunction &CGF) {
1617 llvm::Type *SrcTy = Src.getElementType();
1618
1619 // If SrcTy and Ty are the same, just do a load.
1620 if (SrcTy == Ty)
1621 return CGF.Builder.CreateLoad(Addr: Src);
1622
1623 if (llvm::Value *V = CreatePFPCoercedLoad(Src, SrcFETy, Ty, CGF))
1624 return V;
1625
1626 llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1627
1628 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy)) {
1629 Src = EnterStructPointerForCoercedAccess(SrcPtr: Src, SrcSTy,
1630 DstSize: DstSize.getFixedValue(), CGF);
1631 SrcTy = Src.getElementType();
1632 }
1633
1634 llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1635
1636 // If the source and destination are integer or pointer types, just do an
1637 // extension or truncation to the desired type.
1638 if ((isa<llvm::IntegerType>(Val: Ty) || isa<llvm::PointerType>(Val: Ty)) &&
1639 (isa<llvm::IntegerType>(Val: SrcTy) || isa<llvm::PointerType>(Val: SrcTy))) {
1640 llvm::Value *Load = CGF.Builder.CreateLoad(Addr: Src);
1641 return CoerceIntOrPtrToIntOrPtr(Val: Load, Ty, CGF);
1642 }
1643
1644 // If load is legal, just bitcast the src pointer.
1645 if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1646 SrcSize.getFixedValue() >= DstSize.getFixedValue()) {
1647 // Generally SrcSize is never greater than DstSize, since this means we are
1648 // losing bits. However, this can happen in cases where the structure has
1649 // additional padding, for example due to a user specified alignment.
1650 //
1651 // FIXME: Assert that we aren't truncating non-padding bits when have access
1652 // to that information.
1653 Src = Src.withElementType(ElemTy: Ty);
1654 return CGF.Builder.CreateLoad(Addr: Src);
1655 }
1656
1657 // If coercing a fixed vector to a scalable vector for ABI compatibility, and
1658 // the types match, use the llvm.vector.insert intrinsic to perform the
1659 // conversion.
1660 if (auto *ScalableDstTy = dyn_cast<llvm::ScalableVectorType>(Val: Ty)) {
1661 if (auto *FixedSrcTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy)) {
1662 // If we are casting a fixed i8 vector to a scalable i1 predicate
1663 // vector, use a vector insert and bitcast the result.
1664 if (ScalableDstTy->getElementType()->isIntegerTy(BitWidth: 1) &&
1665 FixedSrcTy->getElementType()->isIntegerTy(BitWidth: 8)) {
1666 ScalableDstTy = llvm::ScalableVectorType::get(
1667 ElementType: FixedSrcTy->getElementType(),
1668 MinNumElts: llvm::divideCeil(
1669 Numerator: ScalableDstTy->getElementCount().getKnownMinValue(), Denominator: 8));
1670 }
1671 if (ScalableDstTy->getElementType() == FixedSrcTy->getElementType()) {
1672 auto *Load = CGF.Builder.CreateLoad(Addr: Src);
1673 auto *PoisonVec = llvm::PoisonValue::get(T: ScalableDstTy);
1674 llvm::Value *Result = CGF.Builder.CreateInsertVector(
1675 DstType: ScalableDstTy, SrcVec: PoisonVec, SubVec: Load, Idx: uint64_t(0), Name: "cast.scalable");
1676 ScalableDstTy = cast<llvm::ScalableVectorType>(
1677 Val: llvm::VectorType::getWithSizeAndScalar(SizeTy: ScalableDstTy, EltTy: Ty));
1678 if (Result->getType() != ScalableDstTy)
1679 Result = CGF.Builder.CreateBitCast(V: Result, DestTy: ScalableDstTy);
1680 if (Result->getType() != Ty)
1681 Result = CGF.Builder.CreateExtractVector(DstType: Ty, SrcVec: Result, Idx: uint64_t(0));
1682 return Result;
1683 }
1684 }
1685 }
1686
1687 // Otherwise do coercion through memory. This is stupid, but simple.
1688 RawAddress Tmp =
1689 CreateTempAllocaForCoercion(CGF, Ty, MinAlign: Src.getAlignment(), Name: Src.getName());
1690 CGF.Builder.CreateMemCpy(
1691 Dst: Tmp.getPointer(), DstAlign: Tmp.getAlignment().getAsAlign(),
1692 Src: Src.emitRawPointer(CGF), SrcAlign: Src.getAlignment().getAsAlign(),
1693 Size: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: SrcSize.getKnownMinValue()));
1694 return CGF.Builder.CreateLoad(Addr: Tmp);
1695}
1696
1697static bool CreatePFPCoercedStore(llvm::Value *Src, QualType SrcFETy,
1698 Address Dst, CodeGenFunction &CGF) {
1699 std::vector<PFPField> PFPFields = CGF.getContext().findPFPFields(Ty: SrcFETy);
1700 if (PFPFields.empty())
1701 return false;
1702
1703 llvm::Type *SrcTy = Src->getType();
1704 auto StoreCoercedField = [&](CharUnits Offset, llvm::Value *FieldVal) {
1705 if (!PFPFields.empty() && PFPFields[0].Offset == Offset) {
1706 auto FieldAddr = CGF.EmitAddressOfPFPField(RecordPtr: Dst, Field: PFPFields[0]);
1707 if (isa<llvm::IntegerType>(Val: FieldVal->getType()))
1708 FieldVal = CGF.Builder.CreateIntToPtr(V: FieldVal, DestTy: CGF.VoidPtrTy);
1709 CGF.Builder.CreateStore(Val: FieldVal, Addr: FieldAddr);
1710 PFPFields.erase(position: PFPFields.begin());
1711 } else {
1712 auto FieldAddr = CGF.Builder
1713 .CreateConstInBoundsByteGEP(
1714 Addr: Dst.withElementType(ElemTy: CGF.Int8Ty), Offset)
1715 .withElementType(ElemTy: FieldVal->getType());
1716 CGF.Builder.CreateStore(Val: FieldVal, Addr: FieldAddr);
1717 }
1718 };
1719
1720 // The types handled by this function are the only ones that may be generated
1721 // by AArch64ABIInfo::classify{Argument,Return}Type for struct types with
1722 // pointers. PFP is only supported on AArch64.
1723 if (isa<llvm::IntegerType>(Val: SrcTy) || isa<llvm::PointerType>(Val: SrcTy)) {
1724 if (isa<llvm::IntegerType>(Val: SrcTy))
1725 Src = CGF.Builder.CreateIntToPtr(V: Src, DestTy: CGF.VoidPtrTy);
1726 auto Addr = CGF.EmitAddressOfPFPField(RecordPtr: Dst, Field: PFPFields[0]);
1727 CGF.Builder.CreateStore(Val: Src, Addr);
1728 } else {
1729 auto *AT = cast<llvm::ArrayType>(Val: SrcTy);
1730 auto *ET = AT->getElementType();
1731 CharUnits WordSize = CGF.getContext().toCharUnitsFromBits(
1732 BitSize: CGF.CGM.getDataLayout().getTypeSizeInBits(Ty: ET));
1733 CharUnits Offset = CharUnits::Zero();
1734 for (unsigned i = 0; i != AT->getNumElements(); ++i, Offset += WordSize)
1735 StoreCoercedField(Offset, CGF.Builder.CreateExtractValue(Agg: Src, Idxs: i));
1736 }
1737 return true;
1738}
1739
1740void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, QualType SrcFETy,
1741 Address Dst, llvm::TypeSize DstSize,
1742 bool DstIsVolatile) {
1743 if (!DstSize)
1744 return;
1745
1746 llvm::Type *SrcTy = Src->getType();
1747 llvm::TypeSize SrcSize = CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1748
1749 // GEP into structs to try to make types match.
1750 // FIXME: This isn't really that useful with opaque types, but it impacts a
1751 // lot of regression tests.
1752 if (SrcTy != Dst.getElementType()) {
1753 if (llvm::StructType *DstSTy =
1754 dyn_cast<llvm::StructType>(Val: Dst.getElementType())) {
1755 assert(!SrcSize.isScalable());
1756 Dst = EnterStructPointerForCoercedAccess(SrcPtr: Dst, SrcSTy: DstSTy,
1757 DstSize: SrcSize.getFixedValue(), CGF&: *this);
1758 }
1759 }
1760
1761 if (CreatePFPCoercedStore(Src, SrcFETy, Dst, CGF&: *this))
1762 return;
1763
1764 if (SrcSize.isScalable() || SrcSize <= DstSize) {
1765 if (SrcTy->isIntegerTy() && Dst.getElementType()->isPointerTy() &&
1766 SrcSize == CGM.getDataLayout().getTypeAllocSize(Ty: Dst.getElementType())) {
1767 // If the value is supposed to be a pointer, convert it before storing it.
1768 Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: Dst.getElementType(), CGF&: *this);
1769 auto *I = Builder.CreateStore(Val: Src, Addr: Dst, IsVolatile: DstIsVolatile);
1770 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1771 } else if (llvm::StructType *STy =
1772 dyn_cast<llvm::StructType>(Val: Src->getType())) {
1773 // Prefer scalar stores to first-class aggregate stores.
1774 Dst = Dst.withElementType(ElemTy: SrcTy);
1775 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1776 Address EltPtr = Builder.CreateStructGEP(Addr: Dst, Index: i);
1777 llvm::Value *Elt = Builder.CreateExtractValue(Agg: Src, Idxs: i);
1778 auto *I = Builder.CreateStore(Val: Elt, Addr: EltPtr, IsVolatile: DstIsVolatile);
1779 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Elt);
1780 }
1781 } else {
1782 auto *I =
1783 Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: SrcTy), IsVolatile: DstIsVolatile);
1784 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1785 }
1786 } else if (SrcTy->isIntegerTy()) {
1787 // If the source is a simple integer, coerce it directly.
1788 llvm::Type *DstIntTy = Builder.getIntNTy(N: DstSize.getFixedValue() * 8);
1789 Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: DstIntTy, CGF&: *this);
1790 auto *I =
1791 Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: DstIntTy), IsVolatile: DstIsVolatile);
1792 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1793 } else {
1794 // Otherwise do coercion through memory. This is stupid, but
1795 // simple.
1796
1797 // Generally SrcSize is never greater than DstSize, since this means we are
1798 // losing bits. However, this can happen in cases where the structure has
1799 // additional padding, for example due to a user specified alignment.
1800 //
1801 // FIXME: Assert that we aren't truncating non-padding bits when have access
1802 // to that information.
1803 RawAddress Tmp =
1804 CreateTempAllocaForCoercion(CGF&: *this, Ty: SrcTy, MinAlign: Dst.getAlignment());
1805 Builder.CreateStore(Val: Src, Addr: Tmp);
1806 auto *I = Builder.CreateMemCpy(
1807 Dst: Dst.emitRawPointer(CGF&: *this), DstAlign: Dst.getAlignment().getAsAlign(),
1808 Src: Tmp.getPointer(), SrcAlign: Tmp.getAlignment().getAsAlign(),
1809 Size: Builder.CreateTypeSize(Ty: IntPtrTy, Size: DstSize));
1810 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1811 }
1812}
1813
1814static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1815 const ABIArgInfo &info) {
1816 if (unsigned offset = info.getDirectOffset()) {
1817 addr = addr.withElementType(ElemTy: CGF.Int8Ty);
1818 addr = CGF.Builder.CreateConstInBoundsByteGEP(
1819 Addr: addr, Offset: CharUnits::fromQuantity(Quantity: offset));
1820 addr = addr.withElementType(ElemTy: info.getCoerceToType());
1821 }
1822 return addr;
1823}
1824
1825static std::pair<llvm::Value *, bool>
1826CoerceScalableToFixed(CodeGenFunction &CGF, llvm::FixedVectorType *ToTy,
1827 llvm::ScalableVectorType *FromTy, llvm::Value *V,
1828 StringRef Name = "") {
1829 // If we are casting a scalable i1 predicate vector to a fixed i8
1830 // vector, first bitcast the source.
1831 if (FromTy->getElementType()->isIntegerTy(BitWidth: 1) &&
1832 ToTy->getElementType() == CGF.Builder.getInt8Ty()) {
1833 if (!FromTy->getElementCount().isKnownMultipleOf(RHS: 8)) {
1834 FromTy = llvm::ScalableVectorType::get(
1835 ElementType: FromTy->getElementType(),
1836 MinNumElts: llvm::alignTo<8>(Value: FromTy->getElementCount().getKnownMinValue()));
1837 llvm::Value *ZeroVec = llvm::Constant::getNullValue(Ty: FromTy);
1838 V = CGF.Builder.CreateInsertVector(DstType: FromTy, SrcVec: ZeroVec, SubVec: V, Idx: uint64_t(0));
1839 }
1840 FromTy = llvm::ScalableVectorType::get(
1841 ElementType: ToTy->getElementType(),
1842 MinNumElts: FromTy->getElementCount().getKnownMinValue() / 8);
1843 V = CGF.Builder.CreateBitCast(V, DestTy: FromTy);
1844 }
1845 if (FromTy->getElementType() == ToTy->getElementType()) {
1846 V->setName(Name + ".coerce");
1847 V = CGF.Builder.CreateExtractVector(DstType: ToTy, SrcVec: V, Idx: uint64_t(0), Name: "cast.fixed");
1848 return {V, true};
1849 }
1850 return {V, false};
1851}
1852
1853namespace {
1854
1855/// Encapsulates information about the way function arguments from
1856/// CGFunctionInfo should be passed to actual LLVM IR function.
1857class ClangToLLVMArgMapping {
1858 static const unsigned InvalidIndex = ~0U;
1859 unsigned InallocaArgNo;
1860 unsigned SRetArgNo;
1861 unsigned TotalIRArgs;
1862
1863 /// Arguments of LLVM IR function corresponding to single Clang argument.
1864 struct IRArgs {
1865 unsigned PaddingArgIndex;
1866 // Argument is expanded to IR arguments at positions
1867 // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1868 unsigned FirstArgIndex;
1869 unsigned NumberOfArgs;
1870
1871 IRArgs()
1872 : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1873 NumberOfArgs(0) {}
1874 };
1875
1876 SmallVector<IRArgs, 8> ArgInfo;
1877
1878public:
1879 ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1880 bool OnlyRequiredArgs = false)
1881 : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1882 ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1883 construct(Context, FI, OnlyRequiredArgs);
1884 }
1885
1886 bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1887 unsigned getInallocaArgNo() const {
1888 assert(hasInallocaArg());
1889 return InallocaArgNo;
1890 }
1891
1892 bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1893 unsigned getSRetArgNo() const {
1894 assert(hasSRetArg());
1895 return SRetArgNo;
1896 }
1897
1898 unsigned totalIRArgs() const { return TotalIRArgs; }
1899
1900 bool hasPaddingArg(unsigned ArgNo) const {
1901 assert(ArgNo < ArgInfo.size());
1902 return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1903 }
1904 unsigned getPaddingArgNo(unsigned ArgNo) const {
1905 assert(hasPaddingArg(ArgNo));
1906 return ArgInfo[ArgNo].PaddingArgIndex;
1907 }
1908
1909 /// Returns index of first IR argument corresponding to ArgNo, and their
1910 /// quantity.
1911 std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1912 assert(ArgNo < ArgInfo.size());
1913 return std::make_pair(x: ArgInfo[ArgNo].FirstArgIndex,
1914 y: ArgInfo[ArgNo].NumberOfArgs);
1915 }
1916
1917private:
1918 void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1919 bool OnlyRequiredArgs);
1920};
1921
1922void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1923 const CGFunctionInfo &FI,
1924 bool OnlyRequiredArgs) {
1925 unsigned IRArgNo = 0;
1926 bool SwapThisWithSRet = false;
1927 const ABIArgInfo &RetAI = FI.getReturnInfo();
1928
1929 if (RetAI.getKind() == ABIArgInfo::Indirect) {
1930 SwapThisWithSRet = RetAI.isSRetAfterThis();
1931 SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1932 }
1933
1934 unsigned ArgNo = 0;
1935 unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();
1936 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1937 ++I, ++ArgNo) {
1938 assert(I != FI.arg_end());
1939 QualType ArgType = I->type;
1940 const ABIArgInfo &AI = I->info;
1941 // Collect data about IR arguments corresponding to Clang argument ArgNo.
1942 auto &IRArgs = ArgInfo[ArgNo];
1943
1944 if (AI.getPaddingType())
1945 IRArgs.PaddingArgIndex = IRArgNo++;
1946
1947 switch (AI.getKind()) {
1948 case ABIArgInfo::TargetSpecific:
1949 case ABIArgInfo::Extend:
1950 case ABIArgInfo::Direct: {
1951 // FIXME: handle sseregparm someday...
1952 llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: AI.getCoerceToType());
1953 if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
1954 IRArgs.NumberOfArgs = STy->getNumElements();
1955 } else {
1956 IRArgs.NumberOfArgs = 1;
1957 }
1958 break;
1959 }
1960 case ABIArgInfo::Indirect:
1961 case ABIArgInfo::IndirectAliased:
1962 IRArgs.NumberOfArgs = 1;
1963 break;
1964 case ABIArgInfo::Ignore:
1965 case ABIArgInfo::InAlloca:
1966 // ignore and inalloca doesn't have matching LLVM parameters.
1967 IRArgs.NumberOfArgs = 0;
1968 break;
1969 case ABIArgInfo::CoerceAndExpand:
1970 IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1971 break;
1972 case ABIArgInfo::Expand:
1973 IRArgs.NumberOfArgs = getExpansionSize(Ty: ArgType, Context);
1974 break;
1975 }
1976
1977 if (IRArgs.NumberOfArgs > 0) {
1978 IRArgs.FirstArgIndex = IRArgNo;
1979 IRArgNo += IRArgs.NumberOfArgs;
1980 }
1981
1982 // Skip over the sret parameter when it comes second. We already handled it
1983 // above.
1984 if (IRArgNo == 1 && SwapThisWithSRet)
1985 IRArgNo++;
1986 }
1987 assert(ArgNo == ArgInfo.size());
1988
1989 if (FI.usesInAlloca())
1990 InallocaArgNo = IRArgNo++;
1991
1992 TotalIRArgs = IRArgNo;
1993}
1994} // namespace
1995
1996/***/
1997
1998bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1999 const auto &RI = FI.getReturnInfo();
2000 return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());
2001}
2002
2003bool CodeGenModule::ReturnTypeHasInReg(const CGFunctionInfo &FI) {
2004 const auto &RI = FI.getReturnInfo();
2005 return RI.getInReg();
2006}
2007
2008bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
2009 return ReturnTypeUsesSRet(FI) &&
2010 getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
2011}
2012
2013bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
2014 if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
2015 switch (BT->getKind()) {
2016 default:
2017 return false;
2018 case BuiltinType::Float:
2019 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Float);
2020 case BuiltinType::Double:
2021 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Double);
2022 case BuiltinType::LongDouble:
2023 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::LongDouble);
2024 }
2025 }
2026
2027 return false;
2028}
2029
2030bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
2031 if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
2032 if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
2033 if (BT->getKind() == BuiltinType::LongDouble)
2034 return getTarget().useObjCFP2RetForComplexLongDouble();
2035 }
2036 }
2037
2038 return false;
2039}
2040
2041llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
2042 const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
2043 return GetFunctionType(Info: FI);
2044}
2045
2046llvm::FunctionType *CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
2047
2048 bool Inserted = FunctionsBeingProcessed.insert(Ptr: &FI).second;
2049 (void)Inserted;
2050 assert(Inserted && "Recursively being processed?");
2051
2052 llvm::Type *resultType = nullptr;
2053 const ABIArgInfo &retAI = FI.getReturnInfo();
2054 switch (retAI.getKind()) {
2055 case ABIArgInfo::Expand:
2056 case ABIArgInfo::IndirectAliased:
2057 llvm_unreachable("Invalid ABI kind for return argument");
2058
2059 case ABIArgInfo::TargetSpecific:
2060 case ABIArgInfo::Extend:
2061 case ABIArgInfo::Direct:
2062 resultType = retAI.getCoerceToType();
2063 break;
2064
2065 case ABIArgInfo::InAlloca:
2066 if (retAI.getInAllocaSRet()) {
2067 // sret things on win32 aren't void, they return the sret pointer.
2068 QualType ret = FI.getReturnType();
2069 unsigned addressSpace = CGM.getTypes().getTargetAddressSpace(T: ret);
2070 resultType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: addressSpace);
2071 } else {
2072 resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
2073 }
2074 break;
2075
2076 case ABIArgInfo::Indirect:
2077 case ABIArgInfo::Ignore:
2078 resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
2079 break;
2080
2081 case ABIArgInfo::CoerceAndExpand:
2082 resultType = retAI.getUnpaddedCoerceAndExpandType();
2083 break;
2084 }
2085
2086 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
2087 SmallVector<llvm::Type *, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
2088
2089 // Add type for sret argument.
2090 if (IRFunctionArgs.hasSRetArg()) {
2091 ArgTypes[IRFunctionArgs.getSRetArgNo()] = llvm::PointerType::get(
2092 C&: getLLVMContext(), AddressSpace: FI.getReturnInfo().getIndirectAddrSpace());
2093 }
2094
2095 // Add type for inalloca argument.
2096 if (IRFunctionArgs.hasInallocaArg())
2097 ArgTypes[IRFunctionArgs.getInallocaArgNo()] =
2098 llvm::PointerType::getUnqual(C&: getLLVMContext());
2099
2100 // Add in all of the required arguments.
2101 unsigned ArgNo = 0;
2102 CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
2103 ie = it + FI.getNumRequiredArgs();
2104 for (; it != ie; ++it, ++ArgNo) {
2105 const ABIArgInfo &ArgInfo = it->info;
2106
2107 // Insert a padding type to ensure proper alignment.
2108 if (IRFunctionArgs.hasPaddingArg(ArgNo))
2109 ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2110 ArgInfo.getPaddingType();
2111
2112 unsigned FirstIRArg, NumIRArgs;
2113 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2114
2115 switch (ArgInfo.getKind()) {
2116 case ABIArgInfo::Ignore:
2117 case ABIArgInfo::InAlloca:
2118 assert(NumIRArgs == 0);
2119 break;
2120
2121 case ABIArgInfo::Indirect:
2122 assert(NumIRArgs == 1);
2123 // indirect arguments are always on the stack, which is alloca addr space.
2124 ArgTypes[FirstIRArg] = llvm::PointerType::get(
2125 C&: getLLVMContext(), AddressSpace: CGM.getDataLayout().getAllocaAddrSpace());
2126 break;
2127 case ABIArgInfo::IndirectAliased:
2128 assert(NumIRArgs == 1);
2129 ArgTypes[FirstIRArg] = llvm::PointerType::get(
2130 C&: getLLVMContext(), AddressSpace: ArgInfo.getIndirectAddrSpace());
2131 break;
2132 case ABIArgInfo::TargetSpecific:
2133 case ABIArgInfo::Extend:
2134 case ABIArgInfo::Direct: {
2135 // Fast-isel and the optimizer generally like scalar values better than
2136 // FCAs, so we flatten them if this is safe to do for this argument.
2137 llvm::Type *argType = ArgInfo.getCoerceToType();
2138 llvm::StructType *st = dyn_cast<llvm::StructType>(Val: argType);
2139 if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
2140 assert(NumIRArgs == st->getNumElements());
2141 for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
2142 ArgTypes[FirstIRArg + i] = st->getElementType(N: i);
2143 } else {
2144 assert(NumIRArgs == 1);
2145 ArgTypes[FirstIRArg] = argType;
2146 }
2147 break;
2148 }
2149
2150 case ABIArgInfo::CoerceAndExpand: {
2151 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
2152 for (auto *EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
2153 *ArgTypesIter++ = EltTy;
2154 }
2155 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
2156 break;
2157 }
2158
2159 case ABIArgInfo::Expand:
2160 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
2161 getExpandedTypes(Ty: it->type, TI&: ArgTypesIter);
2162 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
2163 break;
2164 }
2165 }
2166
2167 bool Erased = FunctionsBeingProcessed.erase(Ptr: &FI);
2168 (void)Erased;
2169 assert(Erased && "Not in set?");
2170
2171 return llvm::FunctionType::get(Result: resultType, Params: ArgTypes, isVarArg: FI.isVariadic());
2172}
2173
2174llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
2175 const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
2176 const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
2177
2178 if (!isFuncTypeConvertible(FT: FPT))
2179 return llvm::StructType::get(Context&: getLLVMContext());
2180
2181 return GetFunctionType(GD);
2182}
2183
2184static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
2185 llvm::AttrBuilder &FuncAttrs,
2186 const FunctionProtoType *FPT) {
2187 if (!FPT)
2188 return;
2189
2190 if (!isUnresolvedExceptionSpec(ESpecType: FPT->getExceptionSpecType()) &&
2191 FPT->isNothrow())
2192 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2193
2194 unsigned SMEBits = FPT->getAArch64SMEAttributes();
2195 if (SMEBits & FunctionType::SME_PStateSMEnabledMask)
2196 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_enabled");
2197 if (SMEBits & FunctionType::SME_PStateSMCompatibleMask)
2198 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_compatible");
2199 if (SMEBits & FunctionType::SME_AgnosticZAStateMask)
2200 FuncAttrs.addAttribute(A: "aarch64_za_state_agnostic");
2201
2202 // ZA
2203 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
2204 FuncAttrs.addAttribute(A: "aarch64_preserves_za");
2205 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_In)
2206 FuncAttrs.addAttribute(A: "aarch64_in_za");
2207 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Out)
2208 FuncAttrs.addAttribute(A: "aarch64_out_za");
2209 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_InOut)
2210 FuncAttrs.addAttribute(A: "aarch64_inout_za");
2211
2212 // ZT0
2213 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
2214 FuncAttrs.addAttribute(A: "aarch64_preserves_zt0");
2215 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_In)
2216 FuncAttrs.addAttribute(A: "aarch64_in_zt0");
2217 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Out)
2218 FuncAttrs.addAttribute(A: "aarch64_out_zt0");
2219 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_InOut)
2220 FuncAttrs.addAttribute(A: "aarch64_inout_zt0");
2221}
2222
2223static void AddAttributesFromOMPAssumes(llvm::AttrBuilder &FuncAttrs,
2224 const Decl *Callee) {
2225 if (!Callee)
2226 return;
2227
2228 SmallVector<StringRef, 4> Attrs;
2229
2230 for (const OMPAssumeAttr *AA : Callee->specific_attrs<OMPAssumeAttr>())
2231 AA->getAssumption().split(A&: Attrs, Separator: ",");
2232
2233 if (!Attrs.empty())
2234 FuncAttrs.addAttribute(A: llvm::AssumptionAttrKey,
2235 V: llvm::join(Begin: Attrs.begin(), End: Attrs.end(), Separator: ","));
2236}
2237
2238bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,
2239 QualType ReturnType) const {
2240 // We can't just discard the return value for a record type with a
2241 // complex destructor or a non-trivially copyable type.
2242 if (const RecordType *RT =
2243 ReturnType.getCanonicalType()->getAsCanonical<RecordType>()) {
2244 if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(Val: RT->getDecl()))
2245 return ClassDecl->hasTrivialDestructor();
2246 }
2247 return ReturnType.isTriviallyCopyableType(Context);
2248}
2249
2250static bool HasStrictReturn(const CodeGenModule &Module, QualType RetTy,
2251 const Decl *TargetDecl) {
2252 // As-is msan can not tolerate noundef mismatch between caller and
2253 // implementation. Mismatch is possible for e.g. indirect calls from C-caller
2254 // into C++. Such mismatches lead to confusing false reports. To avoid
2255 // expensive workaround on msan we enforce initialization event in uncommon
2256 // cases where it's allowed.
2257 if (Module.getLangOpts().Sanitize.has(K: SanitizerKind::Memory))
2258 return true;
2259 // C++ explicitly makes returning undefined values UB. C's rule only applies
2260 // to used values, so we never mark them noundef for now.
2261 if (!Module.getLangOpts().CPlusPlus)
2262 return false;
2263 if (TargetDecl) {
2264 if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2265 if (FDecl->isExternC())
2266 return false;
2267 } else if (const VarDecl *VDecl = dyn_cast<VarDecl>(Val: TargetDecl)) {
2268 // Function pointer.
2269 if (VDecl->isExternC())
2270 return false;
2271 }
2272 }
2273
2274 // We don't want to be too aggressive with the return checking, unless
2275 // it's explicit in the code opts or we're using an appropriate sanitizer.
2276 // Try to respect what the programmer intended.
2277 return Module.getCodeGenOpts().StrictReturn ||
2278 !Module.MayDropFunctionReturn(Context: Module.getContext(), ReturnType: RetTy) ||
2279 Module.getLangOpts().Sanitize.has(K: SanitizerKind::Return);
2280}
2281
2282/// Add denormal-fp-math and denormal-fp-math-f32 as appropriate for the
2283/// requested denormal behavior, accounting for the overriding behavior of the
2284/// -f32 case.
2285static void addDenormalModeAttrs(llvm::DenormalMode FPDenormalMode,
2286 llvm::DenormalMode FP32DenormalMode,
2287 llvm::AttrBuilder &FuncAttrs) {
2288 llvm::DenormalFPEnv FPEnv(FPDenormalMode, FP32DenormalMode);
2289 if (FPEnv != llvm::DenormalFPEnv::getDefault())
2290 FuncAttrs.addDenormalFPEnvAttr(Mode: FPEnv);
2291}
2292
2293/// Add default attributes to a function, which have merge semantics under
2294/// -mlink-builtin-bitcode and should not simply overwrite any existing
2295/// attributes in the linked library.
2296static void
2297addMergableDefaultFunctionAttributes(const CodeGenOptions &CodeGenOpts,
2298 llvm::AttrBuilder &FuncAttrs) {
2299 addDenormalModeAttrs(FPDenormalMode: CodeGenOpts.FPDenormalMode, FP32DenormalMode: CodeGenOpts.FP32DenormalMode,
2300 FuncAttrs);
2301}
2302
2303static void getTrivialDefaultFunctionAttributes(
2304 StringRef Name, bool HasOptnone, const CodeGenOptions &CodeGenOpts,
2305 const LangOptions &LangOpts, bool AttrOnCallSite,
2306 llvm::AttrBuilder &FuncAttrs) {
2307 // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
2308 if (!HasOptnone) {
2309 if (CodeGenOpts.OptimizeSize)
2310 FuncAttrs.addAttribute(Val: llvm::Attribute::OptimizeForSize);
2311 if (CodeGenOpts.OptimizeSize == 2)
2312 FuncAttrs.addAttribute(Val: llvm::Attribute::MinSize);
2313 }
2314
2315 if (CodeGenOpts.DisableRedZone)
2316 FuncAttrs.addAttribute(Val: llvm::Attribute::NoRedZone);
2317 if (CodeGenOpts.IndirectTlsSegRefs)
2318 FuncAttrs.addAttribute(A: "indirect-tls-seg-refs");
2319 if (CodeGenOpts.NoImplicitFloat)
2320 FuncAttrs.addAttribute(Val: llvm::Attribute::NoImplicitFloat);
2321
2322 if (AttrOnCallSite) {
2323 // Attributes that should go on the call site only.
2324 // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
2325 // the -fno-builtin-foo list.
2326 if (!CodeGenOpts.SimplifyLibCalls || LangOpts.isNoBuiltinFunc(Name))
2327 FuncAttrs.addAttribute(Val: llvm::Attribute::NoBuiltin);
2328 if (!CodeGenOpts.TrapFuncName.empty())
2329 FuncAttrs.addAttribute(A: "trap-func-name", V: CodeGenOpts.TrapFuncName);
2330 } else {
2331 switch (CodeGenOpts.getFramePointer()) {
2332 case CodeGenOptions::FramePointerKind::None:
2333 // This is the default behavior.
2334 break;
2335 case CodeGenOptions::FramePointerKind::Reserved:
2336 case CodeGenOptions::FramePointerKind::NonLeafNoReserve:
2337 case CodeGenOptions::FramePointerKind::NonLeaf:
2338 case CodeGenOptions::FramePointerKind::All:
2339 FuncAttrs.addAttribute(A: "frame-pointer",
2340 V: CodeGenOptions::getFramePointerKindName(
2341 Kind: CodeGenOpts.getFramePointer()));
2342 }
2343
2344 if (CodeGenOpts.LessPreciseFPMAD)
2345 FuncAttrs.addAttribute(A: "less-precise-fpmad", V: "true");
2346
2347 if (CodeGenOpts.NullPointerIsValid)
2348 FuncAttrs.addAttribute(Val: llvm::Attribute::NullPointerIsValid);
2349
2350 if (LangOpts.getDefaultExceptionMode() == LangOptions::FPE_Ignore)
2351 FuncAttrs.addAttribute(A: "no-trapping-math", V: "true");
2352
2353 // TODO: Are these all needed?
2354 // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
2355 if (CodeGenOpts.SoftFloat)
2356 FuncAttrs.addAttribute(A: "use-soft-float", V: "true");
2357 FuncAttrs.addAttribute(A: "stack-protector-buffer-size",
2358 V: llvm::utostr(X: CodeGenOpts.SSPBufferSize));
2359 if (LangOpts.NoSignedZero)
2360 FuncAttrs.addAttribute(A: "no-signed-zeros-fp-math", V: "true");
2361
2362 // TODO: Reciprocal estimate codegen options should apply to instructions?
2363 const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
2364 if (!Recips.empty())
2365 FuncAttrs.addAttribute(A: "reciprocal-estimates", V: llvm::join(R: Recips, Separator: ","));
2366
2367 if (!CodeGenOpts.PreferVectorWidth.empty() &&
2368 CodeGenOpts.PreferVectorWidth != "none")
2369 FuncAttrs.addAttribute(A: "prefer-vector-width",
2370 V: CodeGenOpts.PreferVectorWidth);
2371
2372 if (CodeGenOpts.StackRealignment)
2373 FuncAttrs.addAttribute(A: "stackrealign");
2374 if (CodeGenOpts.Backchain)
2375 FuncAttrs.addAttribute(A: "backchain");
2376 if (CodeGenOpts.EnableSegmentedStacks)
2377 FuncAttrs.addAttribute(A: "split-stack");
2378
2379 if (CodeGenOpts.SpeculativeLoadHardening)
2380 FuncAttrs.addAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2381
2382 // Add zero-call-used-regs attribute.
2383 switch (CodeGenOpts.getZeroCallUsedRegs()) {
2384 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
2385 FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2386 break;
2387 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
2388 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr-arg");
2389 break;
2390 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
2391 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr");
2392 break;
2393 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
2394 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-arg");
2395 break;
2396 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
2397 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used");
2398 break;
2399 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
2400 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr-arg");
2401 break;
2402 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
2403 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr");
2404 break;
2405 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
2406 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-arg");
2407 break;
2408 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
2409 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all");
2410 break;
2411 }
2412 }
2413
2414 if (LangOpts.assumeFunctionsAreConvergent()) {
2415 // Conservatively, mark all functions and calls in CUDA and OpenCL as
2416 // convergent (meaning, they may call an intrinsically convergent op, such
2417 // as __syncthreads() / barrier(), and so can't have certain optimizations
2418 // applied around them). LLVM will remove this attribute where it safely
2419 // can.
2420 FuncAttrs.addAttribute(Val: llvm::Attribute::Convergent);
2421 }
2422
2423 // TODO: NoUnwind attribute should be added for other GPU modes HIP,
2424 // OpenMP offload. AFAIK, neither of them support exceptions in device code.
2425 if ((LangOpts.CUDA && LangOpts.CUDAIsDevice) || LangOpts.OpenCL ||
2426 LangOpts.SYCLIsDevice) {
2427 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2428 }
2429
2430 if (CodeGenOpts.SaveRegParams && !AttrOnCallSite)
2431 FuncAttrs.addAttribute(A: "save-reg-params");
2432
2433 for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
2434 StringRef Var, Value;
2435 std::tie(args&: Var, args&: Value) = Attr.split(Separator: '=');
2436 FuncAttrs.addAttribute(A: Var, V: Value);
2437 }
2438
2439 TargetInfo::BranchProtectionInfo BPI(LangOpts);
2440 TargetCodeGenInfo::initBranchProtectionFnAttributes(BPI, FuncAttrs);
2441}
2442
2443/// Merges `target-features` from \TargetOpts and \F, and sets the result in
2444/// \FuncAttr
2445/// * features from \F are always kept
2446/// * a feature from \TargetOpts is kept if itself and its opposite are absent
2447/// from \F
2448static void
2449overrideFunctionFeaturesWithTargetFeatures(llvm::AttrBuilder &FuncAttr,
2450 const llvm::Function &F,
2451 const TargetOptions &TargetOpts) {
2452 auto FFeatures = F.getFnAttribute(Kind: "target-features");
2453
2454 llvm::StringSet<> MergedNames;
2455 SmallVector<StringRef> MergedFeatures;
2456 MergedFeatures.reserve(N: TargetOpts.Features.size());
2457
2458 auto AddUnmergedFeatures = [&](auto &&FeatureRange) {
2459 for (StringRef Feature : FeatureRange) {
2460 if (Feature.empty())
2461 continue;
2462 assert(Feature[0] == '+' || Feature[0] == '-');
2463 StringRef Name = Feature.drop_front(N: 1);
2464 bool Merged = !MergedNames.insert(key: Name).second;
2465 if (!Merged)
2466 MergedFeatures.push_back(Elt: Feature);
2467 }
2468 };
2469
2470 if (FFeatures.isValid())
2471 AddUnmergedFeatures(llvm::split(Str: FFeatures.getValueAsString(), Separator: ','));
2472 AddUnmergedFeatures(TargetOpts.Features);
2473
2474 if (!MergedFeatures.empty()) {
2475 llvm::sort(C&: MergedFeatures);
2476 FuncAttr.addAttribute(A: "target-features", V: llvm::join(R&: MergedFeatures, Separator: ","));
2477 }
2478}
2479
2480void CodeGen::mergeDefaultFunctionDefinitionAttributes(
2481 llvm::Function &F, const CodeGenOptions &CodeGenOpts,
2482 const LangOptions &LangOpts, const TargetOptions &TargetOpts,
2483 bool WillInternalize) {
2484
2485 llvm::AttrBuilder FuncAttrs(F.getContext());
2486 // Here we only extract the options that are relevant compared to the version
2487 // from GetCPUAndFeaturesAttributes.
2488 if (!TargetOpts.CPU.empty())
2489 FuncAttrs.addAttribute(A: "target-cpu", V: TargetOpts.CPU);
2490 if (!TargetOpts.TuneCPU.empty())
2491 FuncAttrs.addAttribute(A: "tune-cpu", V: TargetOpts.TuneCPU);
2492
2493 ::getTrivialDefaultFunctionAttributes(Name: F.getName(), HasOptnone: F.hasOptNone(),
2494 CodeGenOpts, LangOpts,
2495 /*AttrOnCallSite=*/false, FuncAttrs);
2496
2497 if (!WillInternalize && F.isInterposable()) {
2498 // Do not promote "dynamic" denormal-fp-math to this translation unit's
2499 // setting for weak functions that won't be internalized. The user has no
2500 // real control for how builtin bitcode is linked, so we shouldn't assume
2501 // later copies will use a consistent mode.
2502 F.addFnAttrs(Attrs: FuncAttrs);
2503 return;
2504 }
2505
2506 llvm::AttributeMask AttrsToRemove;
2507
2508 llvm::DenormalFPEnv OptsFPEnv(CodeGenOpts.FPDenormalMode,
2509 CodeGenOpts.FP32DenormalMode);
2510 llvm::DenormalFPEnv MergedFPEnv =
2511 OptsFPEnv.mergeCalleeMode(Callee: F.getDenormalFPEnv());
2512
2513 if (MergedFPEnv == llvm::DenormalFPEnv::getDefault()) {
2514 AttrsToRemove.addAttribute(Val: llvm::Attribute::DenormalFPEnv);
2515 } else {
2516 // Overwrite existing attribute
2517 FuncAttrs.addDenormalFPEnvAttr(Mode: MergedFPEnv);
2518 }
2519
2520 F.removeFnAttrs(Attrs: AttrsToRemove);
2521
2522 overrideFunctionFeaturesWithTargetFeatures(FuncAttr&: FuncAttrs, F, TargetOpts);
2523
2524 F.addFnAttrs(Attrs: FuncAttrs);
2525}
2526
2527void CodeGenModule::getTrivialDefaultFunctionAttributes(
2528 StringRef Name, bool HasOptnone, bool AttrOnCallSite,
2529 llvm::AttrBuilder &FuncAttrs) {
2530 ::getTrivialDefaultFunctionAttributes(Name, HasOptnone, CodeGenOpts: getCodeGenOpts(),
2531 LangOpts: getLangOpts(), AttrOnCallSite,
2532 FuncAttrs);
2533}
2534
2535void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
2536 bool HasOptnone,
2537 bool AttrOnCallSite,
2538 llvm::AttrBuilder &FuncAttrs) {
2539 getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,
2540 FuncAttrs);
2541
2542 if (!AttrOnCallSite)
2543 TargetCodeGenInfo::initPointerAuthFnAttributes(Opts: CodeGenOpts.PointerAuth,
2544 FuncAttrs);
2545
2546 // If we're just getting the default, get the default values for mergeable
2547 // attributes.
2548 if (!AttrOnCallSite)
2549 addMergableDefaultFunctionAttributes(CodeGenOpts, FuncAttrs);
2550}
2551
2552void CodeGenModule::addDefaultFunctionDefinitionAttributes(
2553 llvm::AttrBuilder &attrs) {
2554 getDefaultFunctionAttributes(/*function name*/ Name: "", /*optnone*/ HasOptnone: false,
2555 /*for call*/ AttrOnCallSite: false, FuncAttrs&: attrs);
2556 GetCPUAndFeaturesAttributes(GD: GlobalDecl(), AttrBuilder&: attrs);
2557}
2558
2559static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
2560 const LangOptions &LangOpts,
2561 const NoBuiltinAttr *NBA = nullptr) {
2562 auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
2563 SmallString<32> AttributeName;
2564 AttributeName += "no-builtin-";
2565 AttributeName += BuiltinName;
2566 FuncAttrs.addAttribute(A: AttributeName);
2567 };
2568
2569 // First, handle the language options passed through -fno-builtin.
2570 if (LangOpts.NoBuiltin) {
2571 // -fno-builtin disables them all.
2572 FuncAttrs.addAttribute(A: "no-builtins");
2573 return;
2574 }
2575
2576 // Then, add attributes for builtins specified through -fno-builtin-<name>.
2577 llvm::for_each(Range: LangOpts.NoBuiltinFuncs, F: AddNoBuiltinAttr);
2578
2579 // Now, let's check the __attribute__((no_builtin("...")) attribute added to
2580 // the source.
2581 if (!NBA)
2582 return;
2583
2584 // If there is a wildcard in the builtin names specified through the
2585 // attribute, disable them all.
2586 if (llvm::is_contained(Range: NBA->builtinNames(), Element: "*")) {
2587 FuncAttrs.addAttribute(A: "no-builtins");
2588 return;
2589 }
2590
2591 // And last, add the rest of the builtin names.
2592 llvm::for_each(Range: NBA->builtinNames(), F: AddNoBuiltinAttr);
2593}
2594
2595static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
2596 const llvm::DataLayout &DL, const ABIArgInfo &AI,
2597 bool CheckCoerce = true) {
2598 llvm::Type *Ty = Types.ConvertTypeForMem(T: QTy);
2599 if (AI.getKind() == ABIArgInfo::Indirect ||
2600 AI.getKind() == ABIArgInfo::IndirectAliased)
2601 return true;
2602 if (AI.getKind() == ABIArgInfo::Extend && !AI.isNoExt())
2603 return true;
2604 if (!DL.typeSizeEqualsStoreSize(Ty))
2605 // TODO: This will result in a modest amount of values not marked noundef
2606 // when they could be. We care about values that *invisibly* contain undef
2607 // bits from the perspective of LLVM IR.
2608 return false;
2609 if (CheckCoerce && AI.canHaveCoerceToType()) {
2610 llvm::Type *CoerceTy = AI.getCoerceToType();
2611 if (llvm::TypeSize::isKnownGT(LHS: DL.getTypeSizeInBits(Ty: CoerceTy),
2612 RHS: DL.getTypeSizeInBits(Ty)))
2613 // If we're coercing to a type with a greater size than the canonical one,
2614 // we're introducing new undef bits.
2615 // Coercing to a type of smaller or equal size is ok, as we know that
2616 // there's no internal padding (typeSizeEqualsStoreSize).
2617 return false;
2618 }
2619 if (QTy->isBitIntType())
2620 return true;
2621 if (QTy->isReferenceType())
2622 return true;
2623 if (QTy->isNullPtrType())
2624 return false;
2625 if (QTy->isMemberPointerType())
2626 // TODO: Some member pointers are `noundef`, but it depends on the ABI. For
2627 // now, never mark them.
2628 return false;
2629 if (QTy->isScalarType()) {
2630 if (const ComplexType *Complex = dyn_cast<ComplexType>(Val&: QTy))
2631 return DetermineNoUndef(QTy: Complex->getElementType(), Types, DL, AI, CheckCoerce: false);
2632 return true;
2633 }
2634 if (const VectorType *Vector = dyn_cast<VectorType>(Val&: QTy))
2635 return DetermineNoUndef(QTy: Vector->getElementType(), Types, DL, AI, CheckCoerce: false);
2636 if (const MatrixType *Matrix = dyn_cast<MatrixType>(Val&: QTy))
2637 return DetermineNoUndef(QTy: Matrix->getElementType(), Types, DL, AI, CheckCoerce: false);
2638 if (const ArrayType *Array = dyn_cast<ArrayType>(Val&: QTy))
2639 return DetermineNoUndef(QTy: Array->getElementType(), Types, DL, AI, CheckCoerce: false);
2640
2641 // TODO: Some structs may be `noundef`, in specific situations.
2642 return false;
2643}
2644
2645/// Check if the argument of a function has maybe_undef attribute.
2646static bool IsArgumentMaybeUndef(const Decl *TargetDecl,
2647 unsigned NumRequiredArgs, unsigned ArgNo) {
2648 const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
2649 if (!FD)
2650 return false;
2651
2652 // Assume variadic arguments do not have maybe_undef attribute.
2653 if (ArgNo >= NumRequiredArgs)
2654 return false;
2655
2656 // Check if argument has maybe_undef attribute.
2657 if (ArgNo < FD->getNumParams()) {
2658 const ParmVarDecl *Param = FD->getParamDecl(i: ArgNo);
2659 if (Param && Param->hasAttr<MaybeUndefAttr>())
2660 return true;
2661 }
2662
2663 return false;
2664}
2665
2666/// Test if it's legal to apply nofpclass for the given parameter type and it's
2667/// lowered IR type.
2668static bool canApplyNoFPClass(const ABIArgInfo &AI, QualType ParamType,
2669 bool IsReturn) {
2670 // Should only apply to FP types in the source, not ABI promoted.
2671 if (!ParamType->hasFloatingRepresentation())
2672 return false;
2673
2674 // The promoted-to IR type also needs to support nofpclass.
2675 llvm::Type *IRTy = AI.getCoerceToType();
2676 if (llvm::AttributeFuncs::isNoFPClassCompatibleType(Ty: IRTy))
2677 return true;
2678
2679 if (llvm::StructType *ST = dyn_cast<llvm::StructType>(Val: IRTy)) {
2680 return !IsReturn && AI.getCanBeFlattened() &&
2681 llvm::all_of(Range: ST->elements(),
2682 P: llvm::AttributeFuncs::isNoFPClassCompatibleType);
2683 }
2684
2685 return false;
2686}
2687
2688/// Return the nofpclass mask that can be applied to floating-point parameters.
2689static llvm::FPClassTest getNoFPClassTestMask(const LangOptions &LangOpts) {
2690 llvm::FPClassTest Mask = llvm::fcNone;
2691 if (LangOpts.NoHonorInfs)
2692 Mask |= llvm::fcInf;
2693 if (LangOpts.NoHonorNaNs)
2694 Mask |= llvm::fcNan;
2695 return Mask;
2696}
2697
2698void CodeGenModule::AdjustMemoryAttribute(StringRef Name,
2699 CGCalleeInfo CalleeInfo,
2700 llvm::AttributeList &Attrs) {
2701 if (Attrs.getMemoryEffects().getModRef() == llvm::ModRefInfo::NoModRef) {
2702 Attrs = Attrs.removeFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::Memory);
2703 llvm::Attribute MemoryAttr = llvm::Attribute::getWithMemoryEffects(
2704 Context&: getLLVMContext(), ME: llvm::MemoryEffects::writeOnly());
2705 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Attr: MemoryAttr);
2706 }
2707}
2708
2709/// Construct the IR attribute list of a function or call.
2710///
2711/// When adding an attribute, please consider where it should be handled:
2712///
2713/// - getDefaultFunctionAttributes is for attributes that are essentially
2714/// part of the global target configuration (but perhaps can be
2715/// overridden on a per-function basis). Adding attributes there
2716/// will cause them to also be set in frontends that build on Clang's
2717/// target-configuration logic, as well as for code defined in library
2718/// modules such as CUDA's libdevice.
2719///
2720/// - ConstructAttributeList builds on top of getDefaultFunctionAttributes
2721/// and adds declaration-specific, convention-specific, and
2722/// frontend-specific logic. The last is of particular importance:
2723/// attributes that restrict how the frontend generates code must be
2724/// added here rather than getDefaultFunctionAttributes.
2725///
2726void CodeGenModule::ConstructAttributeList(StringRef Name,
2727 const CGFunctionInfo &FI,
2728 CGCalleeInfo CalleeInfo,
2729 llvm::AttributeList &AttrList,
2730 unsigned &CallingConv,
2731 bool AttrOnCallSite, bool IsThunk) {
2732 llvm::AttrBuilder FuncAttrs(getLLVMContext());
2733 llvm::AttrBuilder RetAttrs(getLLVMContext());
2734
2735 // Collect function IR attributes from the CC lowering.
2736 // We'll collect the paramete and result attributes later.
2737 CallingConv = FI.getEffectiveCallingConvention();
2738 if (FI.isNoReturn())
2739 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2740 if (FI.isCmseNSCall())
2741 FuncAttrs.addAttribute(A: "cmse_nonsecure_call");
2742
2743 // Collect function IR attributes from the callee prototype if we have one.
2744 AddAttributesFromFunctionProtoType(Ctx&: getContext(), FuncAttrs,
2745 FPT: CalleeInfo.getCalleeFunctionProtoType());
2746 const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2747
2748 // Attach assumption attributes to the declaration. If this is a call
2749 // site, attach assumptions from the caller to the call as well.
2750 AddAttributesFromOMPAssumes(FuncAttrs, Callee: TargetDecl);
2751
2752 bool HasOptnone = false;
2753 // The NoBuiltinAttr attached to the target FunctionDecl.
2754 const NoBuiltinAttr *NBA = nullptr;
2755
2756 // Some ABIs may result in additional accesses to arguments that may
2757 // otherwise not be present.
2758 std::optional<llvm::Attribute::AttrKind> MemAttrForPtrArgs;
2759 bool AddedPotentialArgAccess = false;
2760 auto AddPotentialArgAccess = [&]() {
2761 AddedPotentialArgAccess = true;
2762 llvm::Attribute A = FuncAttrs.getAttribute(Kind: llvm::Attribute::Memory);
2763 if (A.isValid())
2764 FuncAttrs.addMemoryAttr(ME: A.getMemoryEffects() |
2765 llvm::MemoryEffects::argMemOnly());
2766 };
2767
2768 // Collect function IR attributes based on declaration-specific
2769 // information.
2770 // FIXME: handle sseregparm someday...
2771 if (TargetDecl) {
2772 if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2773 FuncAttrs.addAttribute(Val: llvm::Attribute::ReturnsTwice);
2774 if (TargetDecl->hasAttr<NoThrowAttr>())
2775 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2776 if (TargetDecl->hasAttr<NoReturnAttr>())
2777 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2778 if (TargetDecl->hasAttr<ColdAttr>())
2779 FuncAttrs.addAttribute(Val: llvm::Attribute::Cold);
2780 if (TargetDecl->hasAttr<HotAttr>())
2781 FuncAttrs.addAttribute(Val: llvm::Attribute::Hot);
2782 if (TargetDecl->hasAttr<NoDuplicateAttr>())
2783 FuncAttrs.addAttribute(Val: llvm::Attribute::NoDuplicate);
2784 if (TargetDecl->hasAttr<ConvergentAttr>())
2785 FuncAttrs.addAttribute(Val: llvm::Attribute::Convergent);
2786
2787 if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2788 AddAttributesFromFunctionProtoType(
2789 Ctx&: getContext(), FuncAttrs, FPT: Fn->getType()->getAs<FunctionProtoType>());
2790 if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
2791 // A sane operator new returns a non-aliasing pointer.
2792 auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2793 if (getCodeGenOpts().AssumeSaneOperatorNew &&
2794 (Kind == OO_New || Kind == OO_Array_New))
2795 RetAttrs.addAttribute(Val: llvm::Attribute::NoAlias);
2796 }
2797 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: Fn);
2798 const bool IsVirtualCall = MD && MD->isVirtual();
2799 // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2800 // virtual function. These attributes are not inherited by overloads.
2801 if (!(AttrOnCallSite && IsVirtualCall)) {
2802 if (Fn->isNoReturn())
2803 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2804 NBA = Fn->getAttr<NoBuiltinAttr>();
2805 }
2806 }
2807
2808 if (isa<FunctionDecl>(Val: TargetDecl) || isa<VarDecl>(Val: TargetDecl)) {
2809 // Only place nomerge attribute on call sites, never functions. This
2810 // allows it to work on indirect virtual function calls.
2811 if (AttrOnCallSite && TargetDecl->hasAttr<NoMergeAttr>())
2812 FuncAttrs.addAttribute(Val: llvm::Attribute::NoMerge);
2813 }
2814
2815 // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2816 if (TargetDecl->hasAttr<ConstAttr>()) {
2817 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::none());
2818 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2819 // gcc specifies that 'const' functions have greater restrictions than
2820 // 'pure' functions, so they also cannot have infinite loops.
2821 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
2822 MemAttrForPtrArgs = llvm::Attribute::ReadNone;
2823 } else if (TargetDecl->hasAttr<PureAttr>()) {
2824 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::readOnly());
2825 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2826 // gcc specifies that 'pure' functions cannot have infinite loops.
2827 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
2828 MemAttrForPtrArgs = llvm::Attribute::ReadOnly;
2829 } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2830 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::inaccessibleOrArgMemOnly());
2831 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2832 }
2833 if (const auto *RA = TargetDecl->getAttr<RestrictAttr>();
2834 RA && RA->getDeallocator() == nullptr)
2835 RetAttrs.addAttribute(Val: llvm::Attribute::NoAlias);
2836 if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2837 !CodeGenOpts.NullPointerIsValid)
2838 RetAttrs.addAttribute(Val: llvm::Attribute::NonNull);
2839 if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2840 FuncAttrs.addAttribute(A: "no_caller_saved_registers");
2841 if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2842 FuncAttrs.addAttribute(Val: llvm::Attribute::NoCfCheck);
2843 if (TargetDecl->hasAttr<LeafAttr>())
2844 FuncAttrs.addAttribute(Val: llvm::Attribute::NoCallback);
2845 if (TargetDecl->hasAttr<BPFFastCallAttr>())
2846 FuncAttrs.addAttribute(A: "bpf_fastcall");
2847
2848 HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2849 if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2850 std::optional<unsigned> NumElemsParam;
2851 if (AllocSize->getNumElemsParam().isValid())
2852 NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2853 FuncAttrs.addAllocSizeAttr(ElemSizeArg: AllocSize->getElemSizeParam().getLLVMIndex(),
2854 NumElemsArg: NumElemsParam);
2855 }
2856
2857 // OpenCL v2.0 Work groups may be whether uniform or not.
2858 // '-cl-uniform-work-group-size' compile option gets a hint
2859 // to the compiler that the global work-size be a multiple of
2860 // the work-group size specified to clEnqueueNDRangeKernel
2861 // (i.e. work groups are uniform).
2862 if (getLangOpts().OffloadUniformBlock)
2863 FuncAttrs.addAttribute(A: "uniform-work-group-size");
2864
2865 if (TargetDecl->hasAttr<ArmLocallyStreamingAttr>())
2866 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_body");
2867
2868 if (auto *ModularFormat = TargetDecl->getAttr<ModularFormatAttr>()) {
2869 FormatAttr *Format = TargetDecl->getAttr<FormatAttr>();
2870 StringRef Type = Format->getType()->getName();
2871 std::string FormatIdx = std::to_string(val: Format->getFormatIdx());
2872 std::string FirstArg = std::to_string(val: Format->getFirstArg());
2873 SmallVector<StringRef> Args = {
2874 Type, FormatIdx, FirstArg,
2875 ModularFormat->getModularImplFn()->getName(),
2876 ModularFormat->getImplName()};
2877 llvm::append_range(C&: Args, R: ModularFormat->aspects());
2878 FuncAttrs.addAttribute(A: "modular-format", V: llvm::join(R&: Args, Separator: ","));
2879 }
2880 }
2881
2882 // Attach "no-builtins" attributes to:
2883 // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2884 // * definitions: "no-builtins" or "no-builtin-<name>" only.
2885 // The attributes can come from:
2886 // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2887 // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2888 addNoBuiltinAttributes(FuncAttrs, LangOpts: getLangOpts(), NBA);
2889
2890 // Collect function IR attributes based on global settiings.
2891 getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2892
2893 // Override some default IR attributes based on declaration-specific
2894 // information.
2895 if (TargetDecl) {
2896 if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2897 FuncAttrs.removeAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2898 if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2899 FuncAttrs.addAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2900 if (TargetDecl->hasAttr<NoSplitStackAttr>())
2901 FuncAttrs.removeAttribute(A: "split-stack");
2902 if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
2903 // A function "__attribute__((...))" overrides the command-line flag.
2904 auto Kind =
2905 TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2906 FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2907 FuncAttrs.addAttribute(
2908 A: "zero-call-used-regs",
2909 V: ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Val: Kind));
2910 }
2911
2912 // Add NonLazyBind attribute to function declarations when -fno-plt
2913 // is used.
2914 // FIXME: what if we just haven't processed the function definition
2915 // yet, or if it's an external definition like C99 inline?
2916 if (CodeGenOpts.NoPLT) {
2917 if (auto *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2918 if (!Fn->isDefined() && !AttrOnCallSite) {
2919 FuncAttrs.addAttribute(Val: llvm::Attribute::NonLazyBind);
2920 }
2921 }
2922 }
2923 // Remove 'convergent' if requested.
2924 if (TargetDecl->hasAttr<NoConvergentAttr>())
2925 FuncAttrs.removeAttribute(Val: llvm::Attribute::Convergent);
2926 }
2927
2928 // Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2929 // functions with -funique-internal-linkage-names.
2930 if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames) {
2931 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
2932 if (!FD->isExternallyVisible())
2933 FuncAttrs.addAttribute(A: "sample-profile-suffix-elision-policy",
2934 V: "selected");
2935 }
2936 }
2937
2938 // Collect non-call-site function IR attributes from declaration-specific
2939 // information.
2940 if (!AttrOnCallSite) {
2941 if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>())
2942 FuncAttrs.addAttribute(A: "cmse_nonsecure_entry");
2943
2944 // Whether tail calls are enabled.
2945 auto shouldDisableTailCalls = [&] {
2946 // Should this be honored in getDefaultFunctionAttributes?
2947 if (CodeGenOpts.DisableTailCalls)
2948 return true;
2949
2950 if (!TargetDecl)
2951 return false;
2952
2953 if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2954 TargetDecl->hasAttr<AnyX86InterruptAttr>())
2955 return true;
2956
2957 if (CodeGenOpts.NoEscapingBlockTailCalls) {
2958 if (const auto *BD = dyn_cast<BlockDecl>(Val: TargetDecl))
2959 if (!BD->doesNotEscape())
2960 return true;
2961 }
2962
2963 return false;
2964 };
2965 if (shouldDisableTailCalls())
2966 FuncAttrs.addAttribute(A: "disable-tail-calls", V: "true");
2967
2968 // These functions require the returns_twice attribute for correct codegen,
2969 // but the attribute may not be added if -fno-builtin is specified. We
2970 // explicitly add that attribute here.
2971 static const llvm::StringSet<> ReturnsTwiceFn{
2972 "_setjmpex", "setjmp", "_setjmp", "vfork",
2973 "sigsetjmp", "__sigsetjmp", "savectx", "getcontext"};
2974 if (ReturnsTwiceFn.contains(key: Name))
2975 FuncAttrs.addAttribute(Val: llvm::Attribute::ReturnsTwice);
2976
2977 // CPU/feature overrides. addDefaultFunctionDefinitionAttributes
2978 // handles these separately to set them based on the global defaults.
2979 GetCPUAndFeaturesAttributes(GD: CalleeInfo.getCalleeDecl(), AttrBuilder&: FuncAttrs);
2980
2981 // Windows hotpatching support
2982 if (!MSHotPatchFunctions.empty()) {
2983 bool IsHotPatched = llvm::binary_search(Range&: MSHotPatchFunctions, Value&: Name);
2984 if (IsHotPatched)
2985 FuncAttrs.addAttribute(A: "marked_for_windows_hot_patching");
2986 }
2987 }
2988
2989 // Mark functions that are replaceable by the loader.
2990 if (CodeGenOpts.isLoaderReplaceableFunctionName(FuncName: Name))
2991 FuncAttrs.addAttribute(A: "loader-replaceable");
2992
2993 // Collect attributes from arguments and return values.
2994 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2995
2996 QualType RetTy = FI.getReturnType();
2997 const ABIArgInfo &RetAI = FI.getReturnInfo();
2998 const llvm::DataLayout &DL = getDataLayout();
2999
3000 // Determine if the return type could be partially undef
3001 if (CodeGenOpts.EnableNoundefAttrs &&
3002 HasStrictReturn(Module: *this, RetTy, TargetDecl)) {
3003 if (!RetTy->isVoidType() && RetAI.getKind() != ABIArgInfo::Indirect &&
3004 DetermineNoUndef(QTy: RetTy, Types&: getTypes(), DL, AI: RetAI))
3005 RetAttrs.addAttribute(Val: llvm::Attribute::NoUndef);
3006 }
3007
3008 switch (RetAI.getKind()) {
3009 case ABIArgInfo::Extend:
3010 if (RetAI.isSignExt())
3011 RetAttrs.addAttribute(Val: llvm::Attribute::SExt);
3012 else if (RetAI.isZeroExt())
3013 RetAttrs.addAttribute(Val: llvm::Attribute::ZExt);
3014 else
3015 RetAttrs.addAttribute(Val: llvm::Attribute::NoExt);
3016 [[fallthrough]];
3017 case ABIArgInfo::TargetSpecific:
3018 case ABIArgInfo::Direct:
3019 if (RetAI.getInReg())
3020 RetAttrs.addAttribute(Val: llvm::Attribute::InReg);
3021
3022 if (canApplyNoFPClass(AI: RetAI, ParamType: RetTy, IsReturn: true))
3023 RetAttrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
3024
3025 break;
3026 case ABIArgInfo::Ignore:
3027 break;
3028
3029 case ABIArgInfo::InAlloca:
3030 case ABIArgInfo::Indirect: {
3031 // inalloca and sret disable readnone and readonly
3032 AddPotentialArgAccess();
3033 break;
3034 }
3035
3036 case ABIArgInfo::CoerceAndExpand:
3037 break;
3038
3039 case ABIArgInfo::Expand:
3040 case ABIArgInfo::IndirectAliased:
3041 llvm_unreachable("Invalid ABI kind for return argument");
3042 }
3043
3044 if (!IsThunk) {
3045 // FIXME: fix this properly, https://reviews.llvm.org/D100388
3046 if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
3047 QualType PTy = RefTy->getPointeeType();
3048 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
3049 RetAttrs.addDereferenceableAttr(
3050 Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
3051 if (getTypes().getTargetAddressSpace(T: PTy) == 0 &&
3052 !CodeGenOpts.NullPointerIsValid)
3053 RetAttrs.addAttribute(Val: llvm::Attribute::NonNull);
3054 if (PTy->isObjectType()) {
3055 llvm::Align Alignment =
3056 getNaturalPointeeTypeAlignment(T: RetTy).getAsAlign();
3057 RetAttrs.addAlignmentAttr(Align: Alignment);
3058 }
3059 }
3060 }
3061
3062 bool hasUsedSRet = false;
3063 SmallVector<llvm::AttrBuilder, 4> ArgAttrs;
3064 for (unsigned I = 0; I < IRFunctionArgs.totalIRArgs(); ++I)
3065 ArgAttrs.emplace_back(Args&: getLLVMContext());
3066
3067 // Attach attributes to sret.
3068 if (IRFunctionArgs.hasSRetArg()) {
3069 llvm::AttrBuilder &SRETAttrs = ArgAttrs[IRFunctionArgs.getSRetArgNo()];
3070 SRETAttrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: RetTy));
3071 SRETAttrs.addAttribute(Val: llvm::Attribute::Writable);
3072 SRETAttrs.addAttribute(Val: llvm::Attribute::DeadOnUnwind);
3073 hasUsedSRet = true;
3074 if (RetAI.getInReg())
3075 SRETAttrs.addAttribute(Val: llvm::Attribute::InReg);
3076 SRETAttrs.addAlignmentAttr(Align: RetAI.getIndirectAlign().getQuantity());
3077 }
3078
3079 // Attach attributes to inalloca argument.
3080 if (IRFunctionArgs.hasInallocaArg()) {
3081 ArgAttrs[IRFunctionArgs.getInallocaArgNo()].addInAllocaAttr(
3082 Ty: FI.getArgStruct());
3083 }
3084
3085 // Apply `nonnull`, `dereferenceable(N)` and `align N` to the `this` argument,
3086 // unless this is a thunk function. Add dead_on_return to the `this` argument
3087 // in base class destructors to aid in DSE.
3088 // FIXME: fix this properly, https://reviews.llvm.org/D100388
3089 if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() &&
3090 !FI.arg_begin()->type->isVoidPointerType() && !IsThunk) {
3091 auto IRArgs = IRFunctionArgs.getIRArgs(ArgNo: 0);
3092
3093 assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
3094
3095 llvm::AttrBuilder &Attrs = ArgAttrs[IRArgs.first];
3096
3097 QualType ThisTy = FI.arg_begin()->type.getTypePtr()->getPointeeType();
3098 int64_t ThisSz = getMinimumObjectSize(Ty: ThisTy).getQuantity();
3099
3100 if (!CodeGenOpts.NullPointerIsValid &&
3101 getTypes().getTargetAddressSpace(T: FI.arg_begin()->type) == 0) {
3102 Attrs.addAttribute(Val: llvm::Attribute::NonNull);
3103 Attrs.addDereferenceableAttr(Bytes: ThisSz);
3104 } else {
3105 // FIXME dereferenceable should be correct here, regardless of
3106 // NullPointerIsValid. However, dereferenceable currently does not always
3107 // respect NullPointerIsValid and may imply nonnull and break the program.
3108 // See https://reviews.llvm.org/D66618 for discussions.
3109 Attrs.addDereferenceableOrNullAttr(Bytes: ThisSz);
3110 }
3111
3112 llvm::Align Alignment =
3113 getNaturalTypeAlignment(T: ThisTy, /*BaseInfo=*/nullptr,
3114 /*TBAAInfo=*/nullptr, /*forPointeeType=*/true)
3115 .getAsAlign();
3116 Attrs.addAlignmentAttr(Align: Alignment);
3117
3118 const auto *DD = dyn_cast_if_present<CXXDestructorDecl>(
3119 Val: CalleeInfo.getCalleeDecl().getDecl());
3120 // Do not annotate vector deleting destructors with dead_on_return as the
3121 // this pointer in that case points to an array which we cannot
3122 // statically know the size of. Also do not mark deleting destructors
3123 // dead_on_return as then we might delete stores inside of a user-defined
3124 // operator delete implementation if it gets inlined, which would be
3125 // incorrect as the object's lifetime has already ended and the operator
3126 // delete implementation is allowed to manipulate the underlying storage.
3127 if (DD &&
3128 CalleeInfo.getCalleeDecl().getDtorType() !=
3129 CXXDtorType::Dtor_VectorDeleting &&
3130 CalleeInfo.getCalleeDecl().getDtorType() !=
3131 CXXDtorType::Dtor_Deleting &&
3132 CodeGenOpts.StrictLifetimes) {
3133 const CXXRecordDecl *ClassDecl =
3134 dyn_cast<CXXRecordDecl>(Val: DD->getDeclContext());
3135 // We cannot add dead_on_return if we have virtual base classes because
3136 // they will generally still be live after the base object destructor.
3137 if (ClassDecl->getNumVBases() == 0)
3138 Attrs.addDeadOnReturnAttr(Info: llvm::DeadOnReturnInfo(
3139 Context.getASTRecordLayout(D: ClassDecl).getDataSize().getQuantity()));
3140 }
3141 }
3142
3143 unsigned ArgNo = 0;
3144 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(), E = FI.arg_end();
3145 I != E; ++I, ++ArgNo) {
3146 QualType ParamType = I->type;
3147 const ABIArgInfo &AI = I->info;
3148 llvm::AttrBuilder Attrs(getLLVMContext());
3149
3150 // Add attribute for padding argument, if necessary.
3151 if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
3152 if (AI.getPaddingInReg()) {
3153 ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)].addAttribute(
3154 Val: llvm::Attribute::InReg);
3155 }
3156 }
3157
3158 // Decide whether the argument we're handling could be partially undef
3159 if (CodeGenOpts.EnableNoundefAttrs &&
3160 DetermineNoUndef(QTy: ParamType, Types&: getTypes(), DL, AI)) {
3161 Attrs.addAttribute(Val: llvm::Attribute::NoUndef);
3162 }
3163
3164 // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
3165 // have the corresponding parameter variable. It doesn't make
3166 // sense to do it here because parameters are so messed up.
3167 switch (AI.getKind()) {
3168 case ABIArgInfo::Extend:
3169 if (AI.isSignExt())
3170 Attrs.addAttribute(Val: llvm::Attribute::SExt);
3171 else if (AI.isZeroExt())
3172 Attrs.addAttribute(Val: llvm::Attribute::ZExt);
3173 else
3174 Attrs.addAttribute(Val: llvm::Attribute::NoExt);
3175 [[fallthrough]];
3176 case ABIArgInfo::TargetSpecific:
3177 case ABIArgInfo::Direct:
3178 if (ArgNo == 0 && FI.isChainCall())
3179 Attrs.addAttribute(Val: llvm::Attribute::Nest);
3180 else if (AI.getInReg())
3181 Attrs.addAttribute(Val: llvm::Attribute::InReg);
3182 Attrs.addStackAlignmentAttr(Align: llvm::MaybeAlign(AI.getDirectAlign()));
3183
3184 if (canApplyNoFPClass(AI, ParamType, IsReturn: false))
3185 Attrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
3186 break;
3187 case ABIArgInfo::Indirect: {
3188 if (AI.getInReg())
3189 Attrs.addAttribute(Val: llvm::Attribute::InReg);
3190
3191 // HLSL out and inout parameters must not be marked with ByVal or
3192 // DeadOnReturn attributes because stores to these parameters by the
3193 // callee are visible to the caller.
3194 if (auto ParamABI = FI.getExtParameterInfo(argIndex: ArgNo).getABI();
3195 ParamABI != ParameterABI::HLSLOut &&
3196 ParamABI != ParameterABI::HLSLInOut) {
3197
3198 // Depending on the ABI, this may be either a byval or a dead_on_return
3199 // argument.
3200 if (AI.getIndirectByVal()) {
3201 Attrs.addByValAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
3202 } else {
3203 // Add dead_on_return when the object's lifetime ends in the callee.
3204 // This includes trivially-destructible objects, as well as objects
3205 // whose destruction / clean-up is carried out within the callee
3206 // (e.g., Obj-C ARC-managed structs, MSVC callee-destroyed objects).
3207 if (!ParamType.isDestructedType() || !ParamType->isRecordType() ||
3208 ParamType->castAsRecordDecl()->isParamDestroyedInCallee())
3209 Attrs.addDeadOnReturnAttr(Info: llvm::DeadOnReturnInfo());
3210 }
3211 }
3212
3213 auto *Decl = ParamType->getAsRecordDecl();
3214 if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
3215 Decl->getArgPassingRestrictions() ==
3216 RecordArgPassingKind::CanPassInRegs)
3217 // When calling the function, the pointer passed in will be the only
3218 // reference to the underlying object. Mark it accordingly.
3219 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
3220
3221 // TODO: We could add the byref attribute if not byval, but it would
3222 // require updating many testcases.
3223
3224 CharUnits Align = AI.getIndirectAlign();
3225
3226 // In a byval argument, it is important that the required
3227 // alignment of the type is honored, as LLVM might be creating a
3228 // *new* stack object, and needs to know what alignment to give
3229 // it. (Sometimes it can deduce a sensible alignment on its own,
3230 // but not if clang decides it must emit a packed struct, or the
3231 // user specifies increased alignment requirements.)
3232 //
3233 // This is different from indirect *not* byval, where an aligned copy is
3234 // already created by the caller, and the align attribute is purely
3235 // informative. However, this can still be useful information for
3236 // optimizations, such as giving us one necessary condition for checking
3237 // if a load to this pointer can be speculatively executed.
3238 assert(!Align.isZero());
3239 Attrs.addAlignmentAttr(Align: Align.getQuantity());
3240
3241 // byval disables readnone and readonly.
3242 AddPotentialArgAccess();
3243 break;
3244 }
3245 case ABIArgInfo::IndirectAliased: {
3246 CharUnits Align = AI.getIndirectAlign();
3247 Attrs.addByRefAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
3248 Attrs.addAlignmentAttr(Align: Align.getQuantity());
3249 break;
3250 }
3251 case ABIArgInfo::Ignore:
3252 case ABIArgInfo::Expand:
3253 case ABIArgInfo::CoerceAndExpand:
3254 break;
3255
3256 case ABIArgInfo::InAlloca:
3257 // inalloca disables readnone and readonly.
3258 AddPotentialArgAccess();
3259 continue;
3260 }
3261
3262 if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
3263 QualType PTy = RefTy->getPointeeType();
3264 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
3265 Attrs.addDereferenceableAttr(Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
3266 if (getTypes().getTargetAddressSpace(T: PTy) == 0 &&
3267 !CodeGenOpts.NullPointerIsValid)
3268 Attrs.addAttribute(Val: llvm::Attribute::NonNull);
3269 if (PTy->isObjectType()) {
3270 llvm::Align Alignment =
3271 getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
3272 Attrs.addAlignmentAttr(Align: Alignment);
3273 }
3274 }
3275
3276 // From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:
3277 // > For arguments to a __kernel function declared to be a pointer to a
3278 // > data type, the OpenCL compiler can assume that the pointee is always
3279 // > appropriately aligned as required by the data type.
3280 if (TargetDecl &&
3281 DeviceKernelAttr::isOpenCLSpelling(
3282 A: TargetDecl->getAttr<DeviceKernelAttr>()) &&
3283 ParamType->isPointerType()) {
3284 QualType PTy = ParamType->getPointeeType();
3285 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
3286 llvm::Align Alignment =
3287 getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
3288 Attrs.addAlignmentAttr(Align: Alignment);
3289 }
3290 }
3291
3292 switch (FI.getExtParameterInfo(argIndex: ArgNo).getABI()) {
3293 case ParameterABI::HLSLOut:
3294 case ParameterABI::HLSLInOut:
3295 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
3296 break;
3297 case ParameterABI::Ordinary:
3298 break;
3299
3300 case ParameterABI::SwiftIndirectResult: {
3301 // Add 'sret' if we haven't already used it for something, but
3302 // only if the result is void.
3303 if (!hasUsedSRet && RetTy->isVoidType()) {
3304 Attrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
3305 hasUsedSRet = true;
3306 }
3307
3308 // Add 'noalias' in either case.
3309 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
3310
3311 // Add 'dereferenceable' and 'alignment'.
3312 auto PTy = ParamType->getPointeeType();
3313 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
3314 auto info = getContext().getTypeInfoInChars(T: PTy);
3315 Attrs.addDereferenceableAttr(Bytes: info.Width.getQuantity());
3316 Attrs.addAlignmentAttr(Align: info.Align.getAsAlign());
3317 }
3318 break;
3319 }
3320
3321 case ParameterABI::SwiftErrorResult:
3322 Attrs.addAttribute(Val: llvm::Attribute::SwiftError);
3323 break;
3324
3325 case ParameterABI::SwiftContext:
3326 Attrs.addAttribute(Val: llvm::Attribute::SwiftSelf);
3327 break;
3328
3329 case ParameterABI::SwiftAsyncContext:
3330 Attrs.addAttribute(Val: llvm::Attribute::SwiftAsync);
3331 break;
3332 }
3333
3334 if (FI.getExtParameterInfo(argIndex: ArgNo).isNoEscape())
3335 Attrs.addCapturesAttr(
3336 CI: llvm::CaptureInfo(llvm::CaptureComponents::Address));
3337
3338 if (Attrs.hasAttributes()) {
3339 unsigned FirstIRArg, NumIRArgs;
3340 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3341 for (unsigned i = 0; i < NumIRArgs; i++)
3342 ArgAttrs[FirstIRArg + i].merge(B: Attrs);
3343 }
3344 }
3345 assert(ArgNo == FI.arg_size());
3346
3347 // We can't see all potential arguments in a varargs declaration; treat them
3348 // as if they can access memory.
3349 if (!AttrOnCallSite && FI.isVariadic())
3350 AddPotentialArgAccess();
3351
3352 ArgNo = 0;
3353 if (AddedPotentialArgAccess && MemAttrForPtrArgs) {
3354 llvm::FunctionType *FunctionType = getTypes().GetFunctionType(FI);
3355 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
3356 E = FI.arg_end();
3357 I != E; ++I, ++ArgNo) {
3358 if (I->info.isDirect() || I->info.isExpand() ||
3359 I->info.isCoerceAndExpand()) {
3360 unsigned FirstIRArg, NumIRArgs;
3361 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3362 for (unsigned i = FirstIRArg; i < FirstIRArg + NumIRArgs; ++i) {
3363 // The index may be out-of-bounds if the callee is a varargs
3364 // function.
3365 //
3366 // FIXME: We can compute the types of varargs arguments without going
3367 // through the function type, but the relevant code isn't exposed
3368 // in a way that can be called from here.
3369 if (i < FunctionType->getNumParams() &&
3370 FunctionType->getParamType(i)->isPointerTy()) {
3371 ArgAttrs[i].addAttribute(Val: *MemAttrForPtrArgs);
3372 }
3373 }
3374 }
3375 }
3376 }
3377
3378 SmallVector<llvm::AttributeSet, 4> ArgAttrSets;
3379 for (const llvm::AttrBuilder &Attrs : ArgAttrs)
3380 ArgAttrSets.push_back(Elt: llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs));
3381
3382 AttrList = llvm::AttributeList::get(
3383 C&: getLLVMContext(), FnAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: FuncAttrs),
3384 RetAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: RetAttrs), ArgAttrs: ArgAttrSets);
3385}
3386
3387/// An argument came in as a promoted argument; demote it back to its
3388/// declared type.
3389static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
3390 const VarDecl *var,
3391 llvm::Value *value) {
3392 llvm::Type *varType = CGF.ConvertType(T: var->getType());
3393
3394 // This can happen with promotions that actually don't change the
3395 // underlying type, like the enum promotions.
3396 if (value->getType() == varType)
3397 return value;
3398
3399 assert((varType->isIntegerTy() || varType->isFloatingPointTy()) &&
3400 "unexpected promotion type");
3401
3402 if (isa<llvm::IntegerType>(Val: varType))
3403 return CGF.Builder.CreateTrunc(V: value, DestTy: varType, Name: "arg.unpromote");
3404
3405 return CGF.Builder.CreateFPCast(V: value, DestTy: varType, Name: "arg.unpromote");
3406}
3407
3408/// Returns the attribute (either parameter attribute, or function
3409/// attribute), which declares argument ArgNo to be non-null.
3410static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
3411 QualType ArgType, unsigned ArgNo) {
3412 // FIXME: __attribute__((nonnull)) can also be applied to:
3413 // - references to pointers, where the pointee is known to be
3414 // nonnull (apparently a Clang extension)
3415 // - transparent unions containing pointers
3416 // In the former case, LLVM IR cannot represent the constraint. In
3417 // the latter case, we have no guarantee that the transparent union
3418 // is in fact passed as a pointer.
3419 if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType())
3420 return nullptr;
3421 // First, check attribute on parameter itself.
3422 if (PVD) {
3423 if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
3424 return ParmNNAttr;
3425 }
3426 // Check function attributes.
3427 if (!FD)
3428 return nullptr;
3429 for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
3430 if (NNAttr->isNonNull(IdxAST: ArgNo))
3431 return NNAttr;
3432 }
3433 return nullptr;
3434}
3435
3436namespace {
3437struct CopyBackSwiftError final : EHScopeStack::Cleanup {
3438 Address Temp;
3439 Address Arg;
3440 CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
3441 void Emit(CodeGenFunction &CGF, Flags flags) override {
3442 llvm::Value *errorValue = CGF.Builder.CreateLoad(Addr: Temp);
3443 CGF.Builder.CreateStore(Val: errorValue, Addr: Arg);
3444 }
3445};
3446} // namespace
3447
3448void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
3449 llvm::Function *Fn,
3450 const FunctionArgList &Args) {
3451 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())
3452 // Naked functions don't have prologues.
3453 return;
3454
3455 // If this is an implicit-return-zero function, go ahead and
3456 // initialize the return value. TODO: it might be nice to have
3457 // a more general mechanism for this that didn't require synthesized
3458 // return statements.
3459 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl)) {
3460 if (FD->hasImplicitReturnZero()) {
3461 QualType RetTy = FD->getReturnType().getUnqualifiedType();
3462 llvm::Type *LLVMTy = CGM.getTypes().ConvertType(T: RetTy);
3463 llvm::Constant *Zero = llvm::Constant::getNullValue(Ty: LLVMTy);
3464 Builder.CreateStore(Val: Zero, Addr: ReturnValue);
3465 }
3466 }
3467
3468 // FIXME: We no longer need the types from FunctionArgList; lift up and
3469 // simplify.
3470
3471 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
3472 assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
3473
3474 // If we're using inalloca, all the memory arguments are GEPs off of the last
3475 // parameter, which is a pointer to the complete memory area.
3476 Address ArgStruct = Address::invalid();
3477 if (IRFunctionArgs.hasInallocaArg())
3478 ArgStruct = Address(Fn->getArg(i: IRFunctionArgs.getInallocaArgNo()),
3479 FI.getArgStruct(), FI.getArgStructAlignment());
3480
3481 // Name the struct return parameter.
3482 if (IRFunctionArgs.hasSRetArg()) {
3483 auto AI = Fn->getArg(i: IRFunctionArgs.getSRetArgNo());
3484 AI->setName("agg.result");
3485 AI->addAttr(Kind: llvm::Attribute::NoAlias);
3486 }
3487
3488 // Track if we received the parameter as a pointer (indirect, byval, or
3489 // inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it
3490 // into a local alloca for us.
3491 SmallVector<ParamValue, 16> ArgVals;
3492 ArgVals.reserve(N: Args.size());
3493
3494 // Create a pointer value for every parameter declaration. This usually
3495 // entails copying one or more LLVM IR arguments into an alloca. Don't push
3496 // any cleanups or do anything that might unwind. We do that separately, so
3497 // we can push the cleanups in the correct order for the ABI.
3498 assert(FI.arg_size() == Args.size() &&
3499 "Mismatch between function signature & arguments.");
3500 unsigned ArgNo = 0;
3501 CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
3502 for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end(); i != e;
3503 ++i, ++info_it, ++ArgNo) {
3504 const VarDecl *Arg = *i;
3505 const ABIArgInfo &ArgI = info_it->info;
3506
3507 bool isPromoted =
3508 isa<ParmVarDecl>(Val: Arg) && cast<ParmVarDecl>(Val: Arg)->isKNRPromoted();
3509 // We are converting from ABIArgInfo type to VarDecl type directly, unless
3510 // the parameter is promoted. In this case we convert to
3511 // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
3512 QualType Ty = isPromoted ? info_it->type : Arg->getType();
3513 assert(hasScalarEvaluationKind(Ty) ==
3514 hasScalarEvaluationKind(Arg->getType()));
3515
3516 unsigned FirstIRArg, NumIRArgs;
3517 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3518
3519 switch (ArgI.getKind()) {
3520 case ABIArgInfo::InAlloca: {
3521 assert(NumIRArgs == 0);
3522 auto FieldIndex = ArgI.getInAllocaFieldIndex();
3523 Address V =
3524 Builder.CreateStructGEP(Addr: ArgStruct, Index: FieldIndex, Name: Arg->getName());
3525 if (ArgI.getInAllocaIndirect())
3526 V = Address(Builder.CreateLoad(Addr: V), ConvertTypeForMem(T: Ty),
3527 getContext().getTypeAlignInChars(T: Ty));
3528 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: V));
3529 break;
3530 }
3531
3532 case ABIArgInfo::Indirect:
3533 case ABIArgInfo::IndirectAliased: {
3534 assert(NumIRArgs == 1);
3535 Address ParamAddr = makeNaturalAddressForPointer(
3536 Ptr: Fn->getArg(i: FirstIRArg), T: Ty, Alignment: ArgI.getIndirectAlign(), ForPointeeType: false, BaseInfo: nullptr,
3537 TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
3538
3539 if (!hasScalarEvaluationKind(T: Ty)) {
3540 // Aggregates and complex variables are accessed by reference. All we
3541 // need to do is realign the value, if requested. Also, if the address
3542 // may be aliased, copy it to ensure that the parameter variable is
3543 // mutable and has a unique adress, as C requires.
3544 if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) {
3545 RawAddress AlignedTemp = CreateMemTempWithoutCast(T: Ty, Name: "coerce");
3546
3547 // Copy from the incoming argument pointer to the temporary with the
3548 // appropriate alignment.
3549 //
3550 // FIXME: We should have a common utility for generating an aggregate
3551 // copy.
3552 CharUnits Size = getContext().getTypeSizeInChars(T: Ty);
3553 Builder.CreateMemCpy(
3554 Dst: AlignedTemp.getPointer(), DstAlign: AlignedTemp.getAlignment().getAsAlign(),
3555 Src: ParamAddr.emitRawPointer(CGF&: *this),
3556 SrcAlign: ParamAddr.getAlignment().getAsAlign(),
3557 Size: llvm::ConstantInt::get(Ty: IntPtrTy, V: Size.getQuantity()));
3558 ParamAddr = AlignedTemp;
3559 }
3560 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: ParamAddr));
3561 } else {
3562 // Load scalar value from indirect argument.
3563 llvm::Value *V =
3564 EmitLoadOfScalar(Addr: ParamAddr, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3565
3566 if (isPromoted)
3567 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3568 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3569 }
3570 break;
3571 }
3572
3573 case ABIArgInfo::Extend:
3574 case ABIArgInfo::Direct: {
3575 auto AI = Fn->getArg(i: FirstIRArg);
3576 llvm::Type *LTy = ConvertType(T: Arg->getType());
3577
3578 // Prepare parameter attributes. So far, only attributes for pointer
3579 // parameters are prepared. See
3580 // http://llvm.org/docs/LangRef.html#paramattrs.
3581 if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy() &&
3582 ArgI.getCoerceToType()->isPointerTy()) {
3583 assert(NumIRArgs == 1);
3584
3585 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: Arg)) {
3586 // Set `nonnull` attribute if any.
3587 if (getNonNullAttr(FD: CurCodeDecl, PVD, ArgType: PVD->getType(),
3588 ArgNo: PVD->getFunctionScopeIndex()) &&
3589 !CGM.getCodeGenOpts().NullPointerIsValid)
3590 AI->addAttr(Kind: llvm::Attribute::NonNull);
3591
3592 QualType OTy = PVD->getOriginalType();
3593 if (const auto *ArrTy = getContext().getAsConstantArrayType(T: OTy)) {
3594 // A C99 array parameter declaration with the static keyword also
3595 // indicates dereferenceability, and if the size is constant we can
3596 // use the dereferenceable attribute (which requires the size in
3597 // bytes).
3598 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3599 QualType ETy = ArrTy->getElementType();
3600 llvm::Align Alignment =
3601 CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3602 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3603 .addAlignmentAttr(Align: Alignment));
3604 uint64_t ArrSize = ArrTy->getZExtSize();
3605 if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
3606 ArrSize) {
3607 llvm::AttrBuilder Attrs(getLLVMContext());
3608 Attrs.addDereferenceableAttr(
3609 Bytes: getContext().getTypeSizeInChars(T: ETy).getQuantity() *
3610 ArrSize);
3611 AI->addAttrs(B&: Attrs);
3612 } else if (getContext().getTargetInfo().getNullPointerValue(
3613 AddrSpace: ETy.getAddressSpace()) == 0 &&
3614 !CGM.getCodeGenOpts().NullPointerIsValid) {
3615 AI->addAttr(Kind: llvm::Attribute::NonNull);
3616 }
3617 }
3618 } else if (const auto *ArrTy =
3619 getContext().getAsVariableArrayType(T: OTy)) {
3620 // For C99 VLAs with the static keyword, we don't know the size so
3621 // we can't use the dereferenceable attribute, but in addrspace(0)
3622 // we know that it must be nonnull.
3623 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3624 QualType ETy = ArrTy->getElementType();
3625 llvm::Align Alignment =
3626 CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3627 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3628 .addAlignmentAttr(Align: Alignment));
3629 if (!getTypes().getTargetAddressSpace(T: ETy) &&
3630 !CGM.getCodeGenOpts().NullPointerIsValid)
3631 AI->addAttr(Kind: llvm::Attribute::NonNull);
3632 }
3633 }
3634
3635 // Set `align` attribute if any.
3636 const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
3637 if (!AVAttr)
3638 if (const auto *TOTy = OTy->getAs<TypedefType>())
3639 AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
3640 if (AVAttr && !SanOpts.has(K: SanitizerKind::Alignment)) {
3641 // If alignment-assumption sanitizer is enabled, we do *not* add
3642 // alignment attribute here, but emit normal alignment assumption,
3643 // so the UBSAN check could function.
3644 llvm::ConstantInt *AlignmentCI =
3645 cast<llvm::ConstantInt>(Val: EmitScalarExpr(E: AVAttr->getAlignment()));
3646 uint64_t AlignmentInt =
3647 AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment);
3648 if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
3649 AI->removeAttr(Kind: llvm::Attribute::AttrKind::Alignment);
3650 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3651 .addAlignmentAttr(Align: llvm::Align(AlignmentInt)));
3652 }
3653 }
3654 }
3655
3656 // Set 'noalias' if an argument type has the `restrict` qualifier.
3657 if (Arg->getType().isRestrictQualified())
3658 AI->addAttr(Kind: llvm::Attribute::NoAlias);
3659 }
3660
3661 // Prepare the argument value. If we have the trivial case, handle it
3662 // with no muss and fuss.
3663 if (!isa<llvm::StructType>(Val: ArgI.getCoerceToType()) &&
3664 ArgI.getCoerceToType() == ConvertType(T: Ty) &&
3665 ArgI.getDirectOffset() == 0) {
3666 assert(NumIRArgs == 1);
3667
3668 // LLVM expects swifterror parameters to be used in very restricted
3669 // ways. Copy the value into a less-restricted temporary.
3670 llvm::Value *V = AI;
3671 if (FI.getExtParameterInfo(argIndex: ArgNo).getABI() ==
3672 ParameterABI::SwiftErrorResult) {
3673 QualType pointeeTy = Ty->getPointeeType();
3674 assert(pointeeTy->isPointerType());
3675 RawAddress temp = CreateMemTempWithoutCast(
3676 T: pointeeTy, Align: getPointerAlign(), Name: "swifterror.temp");
3677 Address arg = makeNaturalAddressForPointer(
3678 Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
3679 llvm::Value *incomingErrorValue = Builder.CreateLoad(Addr: arg);
3680 Builder.CreateStore(Val: incomingErrorValue, Addr: temp);
3681 V = temp.getPointer();
3682
3683 // Push a cleanup to copy the value back at the end of the function.
3684 // The convention does not guarantee that the value will be written
3685 // back if the function exits with an unwind exception.
3686 EHStack.pushCleanup<CopyBackSwiftError>(Kind: NormalCleanup, A: temp, A: arg);
3687 }
3688
3689 // Ensure the argument is the correct type.
3690 if (V->getType() != ArgI.getCoerceToType())
3691 V = Builder.CreateBitCast(V, DestTy: ArgI.getCoerceToType());
3692
3693 if (isPromoted)
3694 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3695
3696 // Because of merging of function types from multiple decls it is
3697 // possible for the type of an argument to not match the corresponding
3698 // type in the function type. Since we are codegening the callee
3699 // in here, add a cast to the argument type.
3700 llvm::Type *LTy = ConvertType(T: Arg->getType());
3701 if (V->getType() != LTy)
3702 V = Builder.CreateBitCast(V, DestTy: LTy);
3703
3704 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3705 break;
3706 }
3707
3708 // VLST arguments are coerced to VLATs at the function boundary for
3709 // ABI consistency. If this is a VLST that was coerced to
3710 // a VLAT at the function boundary and the types match up, use
3711 // llvm.vector.extract to convert back to the original VLST.
3712 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: ConvertType(T: Ty))) {
3713 llvm::Value *ArgVal = Fn->getArg(i: FirstIRArg);
3714 if (auto *VecTyFrom =
3715 dyn_cast<llvm::ScalableVectorType>(Val: ArgVal->getType())) {
3716 auto [Coerced, Extracted] = CoerceScalableToFixed(
3717 CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: ArgVal, Name: Arg->getName());
3718 if (Extracted) {
3719 assert(NumIRArgs == 1);
3720 ArgVals.push_back(Elt: ParamValue::forDirect(value: Coerced));
3721 break;
3722 }
3723 }
3724 }
3725
3726 llvm::StructType *STy =
3727 dyn_cast<llvm::StructType>(Val: ArgI.getCoerceToType());
3728 Address Alloca = CreateMemTempWithoutCast(
3729 T: Ty, Align: getContext().getDeclAlign(D: Arg), Name: Arg->getName());
3730
3731 // Pointer to store into.
3732 Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3733
3734 // Fast-isel and the optimizer generally like scalar values better than
3735 // FCAs, so we flatten them if this is safe to do for this argument.
3736 if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
3737 STy->getNumElements() > 1) {
3738 llvm::TypeSize StructSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
3739 llvm::TypeSize PtrElementSize =
3740 CGM.getDataLayout().getTypeAllocSize(Ty: Ptr.getElementType());
3741 if (StructSize.isScalable()) {
3742 assert(STy->containsHomogeneousScalableVectorTypes() &&
3743 "ABI only supports structure with homogeneous scalable vector "
3744 "type");
3745 assert(StructSize == PtrElementSize &&
3746 "Only allow non-fractional movement of structure with"
3747 "homogeneous scalable vector type");
3748 assert(STy->getNumElements() == NumIRArgs);
3749
3750 llvm::Value *LoadedStructValue = llvm::PoisonValue::get(T: STy);
3751 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3752 auto *AI = Fn->getArg(i: FirstIRArg + i);
3753 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3754 LoadedStructValue =
3755 Builder.CreateInsertValue(Agg: LoadedStructValue, Val: AI, Idxs: i);
3756 }
3757
3758 Builder.CreateStore(Val: LoadedStructValue, Addr: Ptr);
3759 } else {
3760 uint64_t SrcSize = StructSize.getFixedValue();
3761 uint64_t DstSize = PtrElementSize.getFixedValue();
3762
3763 Address AddrToStoreInto = Address::invalid();
3764 if (SrcSize <= DstSize) {
3765 AddrToStoreInto = Ptr.withElementType(ElemTy: STy);
3766 } else {
3767 AddrToStoreInto =
3768 CreateTempAlloca(Ty: STy, align: Alloca.getAlignment(), Name: "coerce");
3769 }
3770
3771 assert(STy->getNumElements() == NumIRArgs);
3772 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3773 auto AI = Fn->getArg(i: FirstIRArg + i);
3774 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3775 Address EltPtr = Builder.CreateStructGEP(Addr: AddrToStoreInto, Index: i);
3776 Builder.CreateStore(Val: AI, Addr: EltPtr);
3777 }
3778
3779 if (SrcSize > DstSize) {
3780 Builder.CreateMemCpy(Dest: Ptr, Src: AddrToStoreInto, Size: DstSize);
3781 }
3782
3783 // Structures with PFP fields require a coerced store to add any
3784 // pointer signatures.
3785 if (getContext().hasPFPFields(Ty)) {
3786 llvm::Value *Struct = Builder.CreateLoad(Addr: Ptr);
3787 CreatePFPCoercedStore(Src: Struct, SrcFETy: Ty, Dst: Ptr, CGF&: *this);
3788 }
3789 }
3790 } else {
3791 // Simple case, just do a coerced store of the argument into the alloca.
3792 assert(NumIRArgs == 1);
3793 auto AI = Fn->getArg(i: FirstIRArg);
3794 AI->setName(Arg->getName() + ".coerce");
3795 CreateCoercedStore(
3796 Src: AI, SrcFETy: Ty, Dst: Ptr,
3797 DstSize: llvm::TypeSize::getFixed(
3798 ExactSize: getContext().getTypeSizeInChars(T: Ty).getQuantity() -
3799 ArgI.getDirectOffset()),
3800 /*DstIsVolatile=*/false);
3801 }
3802
3803 // Match to what EmitParmDecl is expecting for this type.
3804 if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3805 llvm::Value *V =
3806 EmitLoadOfScalar(Addr: Alloca, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3807 if (isPromoted)
3808 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3809 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3810 } else {
3811 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3812 }
3813 break;
3814 }
3815
3816 case ABIArgInfo::CoerceAndExpand: {
3817 // Reconstruct into a temporary.
3818 Address alloca =
3819 CreateMemTempWithoutCast(T: Ty, Align: getContext().getDeclAlign(D: Arg));
3820 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: alloca));
3821
3822 auto coercionType = ArgI.getCoerceAndExpandType();
3823 auto unpaddedCoercionType = ArgI.getUnpaddedCoerceAndExpandType();
3824 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
3825
3826 alloca = alloca.withElementType(ElemTy: coercionType);
3827
3828 unsigned argIndex = FirstIRArg;
3829 unsigned unpaddedIndex = 0;
3830 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3831 llvm::Type *eltType = coercionType->getElementType(N: i);
3832 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
3833 continue;
3834
3835 auto eltAddr = Builder.CreateStructGEP(Addr: alloca, Index: i);
3836 llvm::Value *elt = Fn->getArg(i: argIndex++);
3837
3838 auto paramType = unpaddedStruct
3839 ? unpaddedStruct->getElementType(N: unpaddedIndex++)
3840 : unpaddedCoercionType;
3841
3842 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: eltType)) {
3843 if (auto *VecTyFrom = dyn_cast<llvm::ScalableVectorType>(Val: paramType)) {
3844 bool Extracted;
3845 std::tie(args&: elt, args&: Extracted) = CoerceScalableToFixed(
3846 CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: elt, Name: elt->getName());
3847 assert(Extracted && "Unexpected scalable to fixed vector coercion");
3848 }
3849 }
3850 Builder.CreateStore(Val: elt, Addr: eltAddr);
3851 }
3852 assert(argIndex == FirstIRArg + NumIRArgs);
3853 break;
3854 }
3855
3856 case ABIArgInfo::Expand: {
3857 // If this structure was expanded into multiple arguments then
3858 // we need to create a temporary and reconstruct it from the
3859 // arguments.
3860 Address Alloca =
3861 CreateMemTempWithoutCast(T: Ty, Align: getContext().getDeclAlign(D: Arg));
3862 LValue LV = MakeAddrLValue(Addr: Alloca, T: Ty);
3863 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3864
3865 auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3866 ExpandTypeFromArgs(Ty, LV, AI&: FnArgIter);
3867 assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3868 for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
3869 auto AI = Fn->getArg(i: FirstIRArg + i);
3870 AI->setName(Arg->getName() + "." + Twine(i));
3871 }
3872 break;
3873 }
3874
3875 case ABIArgInfo::TargetSpecific: {
3876 auto *AI = Fn->getArg(i: FirstIRArg);
3877 AI->setName(Arg->getName() + ".target_coerce");
3878 Address Alloca = CreateMemTempWithoutCast(
3879 T: Ty, Align: getContext().getDeclAlign(D: Arg), Name: Arg->getName());
3880 Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3881 CGM.getABIInfo().createCoercedStore(Val: AI, DstAddr: Ptr, AI: ArgI, DestIsVolatile: false, CGF&: *this);
3882 if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3883 llvm::Value *V =
3884 EmitLoadOfScalar(Addr: Alloca, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3885 if (isPromoted) {
3886 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3887 }
3888 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3889 } else {
3890 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3891 }
3892 break;
3893 }
3894 case ABIArgInfo::Ignore:
3895 assert(NumIRArgs == 0);
3896 // Initialize the local variable appropriately.
3897 if (!hasScalarEvaluationKind(T: Ty)) {
3898 ArgVals.push_back(
3899 Elt: ParamValue::forIndirect(addr: CreateMemTempWithoutCast(T: Ty)));
3900 } else {
3901 llvm::Value *U = llvm::UndefValue::get(T: ConvertType(T: Arg->getType()));
3902 ArgVals.push_back(Elt: ParamValue::forDirect(value: U));
3903 }
3904 break;
3905 }
3906 }
3907
3908 if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3909 for (int I = Args.size() - 1; I >= 0; --I)
3910 EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3911 } else {
3912 for (unsigned I = 0, E = Args.size(); I != E; ++I)
3913 EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3914 }
3915}
3916
3917static void eraseUnusedBitCasts(llvm::Instruction *insn) {
3918 while (insn->use_empty()) {
3919 llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: insn);
3920 if (!bitcast)
3921 return;
3922
3923 // This is "safe" because we would have used a ConstantExpr otherwise.
3924 insn = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3925 bitcast->eraseFromParent();
3926 }
3927}
3928
3929/// Try to emit a fused autorelease of a return result.
3930static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
3931 llvm::Value *result) {
3932 // We must be immediately followed the cast.
3933 llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3934 if (BB->empty())
3935 return nullptr;
3936 if (&BB->back() != result)
3937 return nullptr;
3938
3939 llvm::Type *resultType = result->getType();
3940
3941 // result is in a BasicBlock and is therefore an Instruction.
3942 llvm::Instruction *generator = cast<llvm::Instruction>(Val: result);
3943
3944 SmallVector<llvm::Instruction *, 4> InstsToKill;
3945
3946 // Look for:
3947 // %generator = bitcast %type1* %generator2 to %type2*
3948 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: generator)) {
3949 // We would have emitted this as a constant if the operand weren't
3950 // an Instruction.
3951 generator = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3952
3953 // Require the generator to be immediately followed by the cast.
3954 if (generator->getNextNode() != bitcast)
3955 return nullptr;
3956
3957 InstsToKill.push_back(Elt: bitcast);
3958 }
3959
3960 // Look for:
3961 // %generator = call i8* @objc_retain(i8* %originalResult)
3962 // or
3963 // %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
3964 llvm::CallInst *call = dyn_cast<llvm::CallInst>(Val: generator);
3965 if (!call)
3966 return nullptr;
3967
3968 bool doRetainAutorelease;
3969
3970 if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
3971 doRetainAutorelease = true;
3972 } else if (call->getCalledOperand() ==
3973 CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
3974 doRetainAutorelease = false;
3975
3976 // If we emitted an assembly marker for this call (and the
3977 // ARCEntrypoints field should have been set if so), go looking
3978 // for that call. If we can't find it, we can't do this
3979 // optimization. But it should always be the immediately previous
3980 // instruction, unless we needed bitcasts around the call.
3981 if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
3982 llvm::Instruction *prev = call->getPrevNode();
3983 assert(prev);
3984 if (isa<llvm::BitCastInst>(Val: prev)) {
3985 prev = prev->getPrevNode();
3986 assert(prev);
3987 }
3988 assert(isa<llvm::CallInst>(prev));
3989 assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3990 CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
3991 InstsToKill.push_back(Elt: prev);
3992 }
3993 } else {
3994 return nullptr;
3995 }
3996
3997 result = call->getArgOperand(i: 0);
3998 InstsToKill.push_back(Elt: call);
3999
4000 // Keep killing bitcasts, for sanity. Note that we no longer care
4001 // about precise ordering as long as there's exactly one use.
4002 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: result)) {
4003 if (!bitcast->hasOneUse())
4004 break;
4005 InstsToKill.push_back(Elt: bitcast);
4006 result = bitcast->getOperand(i_nocapture: 0);
4007 }
4008
4009 // Delete all the unnecessary instructions, from latest to earliest.
4010 for (auto *I : InstsToKill)
4011 I->eraseFromParent();
4012
4013 // Do the fused retain/autorelease if we were asked to.
4014 if (doRetainAutorelease)
4015 result = CGF.EmitARCRetainAutoreleaseReturnValue(value: result);
4016
4017 // Cast back to the result type.
4018 return CGF.Builder.CreateBitCast(V: result, DestTy: resultType);
4019}
4020
4021/// If this is a +1 of the value of an immutable 'self', remove it.
4022static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
4023 llvm::Value *result) {
4024 // This is only applicable to a method with an immutable 'self'.
4025 const ObjCMethodDecl *method =
4026 dyn_cast_or_null<ObjCMethodDecl>(Val: CGF.CurCodeDecl);
4027 if (!method)
4028 return nullptr;
4029 const VarDecl *self = method->getSelfDecl();
4030 if (!self->getType().isConstQualified())
4031 return nullptr;
4032
4033 // Look for a retain call. Note: stripPointerCasts looks through returned arg
4034 // functions, which would cause us to miss the retain.
4035 llvm::CallInst *retainCall = dyn_cast<llvm::CallInst>(Val: result);
4036 if (!retainCall || retainCall->getCalledOperand() !=
4037 CGF.CGM.getObjCEntrypoints().objc_retain)
4038 return nullptr;
4039
4040 // Look for an ordinary load of 'self'.
4041 llvm::Value *retainedValue = retainCall->getArgOperand(i: 0);
4042 llvm::LoadInst *load =
4043 dyn_cast<llvm::LoadInst>(Val: retainedValue->stripPointerCasts());
4044 if (!load || load->isAtomic() || load->isVolatile() ||
4045 load->getPointerOperand() != CGF.GetAddrOfLocalVar(VD: self).getBasePointer())
4046 return nullptr;
4047
4048 // Okay! Burn it all down. This relies for correctness on the
4049 // assumption that the retain is emitted as part of the return and
4050 // that thereafter everything is used "linearly".
4051 llvm::Type *resultType = result->getType();
4052 eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: result));
4053 assert(retainCall->use_empty());
4054 retainCall->eraseFromParent();
4055 eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: retainedValue));
4056
4057 return CGF.Builder.CreateBitCast(V: load, DestTy: resultType);
4058}
4059
4060/// Emit an ARC autorelease of the result of a function.
4061///
4062/// \return the value to actually return from the function
4063static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
4064 llvm::Value *result) {
4065 // If we're returning 'self', kill the initial retain. This is a
4066 // heuristic attempt to "encourage correctness" in the really unfortunate
4067 // case where we have a return of self during a dealloc and we desperately
4068 // need to avoid the possible autorelease.
4069 if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
4070 return self;
4071
4072 // At -O0, try to emit a fused retain/autorelease.
4073 if (CGF.shouldUseFusedARCCalls())
4074 if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
4075 return fused;
4076
4077 return CGF.EmitARCAutoreleaseReturnValue(value: result);
4078}
4079
4080/// Heuristically search for a dominating store to the return-value slot.
4081static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
4082 llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer();
4083
4084 // Check if a User is a store which pointerOperand is the ReturnValue.
4085 // We are looking for stores to the ReturnValue, not for stores of the
4086 // ReturnValue to some other location.
4087 auto GetStoreIfValid = [&CGF,
4088 ReturnValuePtr](llvm::User *U) -> llvm::StoreInst * {
4089 auto *SI = dyn_cast<llvm::StoreInst>(Val: U);
4090 if (!SI || SI->getPointerOperand() != ReturnValuePtr ||
4091 SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())
4092 return nullptr;
4093 // These aren't actually possible for non-coerced returns, and we
4094 // only care about non-coerced returns on this code path.
4095 // All memory instructions inside __try block are volatile.
4096 assert(!SI->isAtomic() &&
4097 (!SI->isVolatile() || CGF.currentFunctionUsesSEHTry()));
4098 return SI;
4099 };
4100 // If there are multiple uses of the return-value slot, just check
4101 // for something immediately preceding the IP. Sometimes this can
4102 // happen with how we generate implicit-returns; it can also happen
4103 // with noreturn cleanups.
4104 if (!ReturnValuePtr->hasOneUse()) {
4105 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
4106 if (IP->empty())
4107 return nullptr;
4108
4109 // Look at directly preceding instruction, skipping bitcasts, lifetime
4110 // markers, and fake uses and their operands.
4111 const llvm::Instruction *LoadIntoFakeUse = nullptr;
4112 for (llvm::Instruction &I : llvm::reverse(C&: *IP)) {
4113 // Ignore instructions that are just loads for fake uses; the load should
4114 // immediately precede the fake use, so we only need to remember the
4115 // operand for the last fake use seen.
4116 if (LoadIntoFakeUse == &I)
4117 continue;
4118 if (isa<llvm::BitCastInst>(Val: &I))
4119 continue;
4120 if (auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &I)) {
4121 if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
4122 continue;
4123
4124 if (II->getIntrinsicID() == llvm::Intrinsic::fake_use) {
4125 LoadIntoFakeUse = dyn_cast<llvm::Instruction>(Val: II->getArgOperand(i: 0));
4126 continue;
4127 }
4128 }
4129 return GetStoreIfValid(&I);
4130 }
4131 return nullptr;
4132 }
4133
4134 llvm::StoreInst *store = GetStoreIfValid(ReturnValuePtr->user_back());
4135 if (!store)
4136 return nullptr;
4137
4138 // Now do a first-and-dirty dominance check: just walk up the
4139 // single-predecessors chain from the current insertion point.
4140 llvm::BasicBlock *StoreBB = store->getParent();
4141 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
4142 llvm::SmallPtrSet<llvm::BasicBlock *, 4> SeenBBs;
4143 while (IP != StoreBB) {
4144 if (!SeenBBs.insert(Ptr: IP).second || !(IP = IP->getSinglePredecessor()))
4145 return nullptr;
4146 }
4147
4148 // Okay, the store's basic block dominates the insertion point; we
4149 // can do our thing.
4150 return store;
4151}
4152
4153// Helper functions for EmitCMSEClearRecord
4154
4155// Set the bits corresponding to a field having width `BitWidth` and located at
4156// offset `BitOffset` (from the least significant bit) within a storage unit of
4157// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
4158// Use little-endian layout, i.e.`Bits[0]` is the LSB.
4159static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
4160 int BitWidth, int CharWidth) {
4161 assert(CharWidth <= 64);
4162 assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
4163
4164 int Pos = 0;
4165 if (BitOffset >= CharWidth) {
4166 Pos += BitOffset / CharWidth;
4167 BitOffset = BitOffset % CharWidth;
4168 }
4169
4170 const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
4171 if (BitOffset + BitWidth >= CharWidth) {
4172 Bits[Pos++] |= (Used << BitOffset) & Used;
4173 BitWidth -= CharWidth - BitOffset;
4174 BitOffset = 0;
4175 }
4176
4177 while (BitWidth >= CharWidth) {
4178 Bits[Pos++] = Used;
4179 BitWidth -= CharWidth;
4180 }
4181
4182 if (BitWidth > 0)
4183 Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
4184}
4185
4186// Set the bits corresponding to a field having width `BitWidth` and located at
4187// offset `BitOffset` (from the least significant bit) within a storage unit of
4188// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
4189// `Bits` corresponds to one target byte. Use target endian layout.
4190static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
4191 int StorageSize, int BitOffset, int BitWidth,
4192 int CharWidth, bool BigEndian) {
4193
4194 SmallVector<uint64_t, 8> TmpBits(StorageSize);
4195 setBitRange(Bits&: TmpBits, BitOffset, BitWidth, CharWidth);
4196
4197 if (BigEndian)
4198 std::reverse(first: TmpBits.begin(), last: TmpBits.end());
4199
4200 for (uint64_t V : TmpBits)
4201 Bits[StorageOffset++] |= V;
4202}
4203
4204static void setUsedBits(CodeGenModule &, QualType, int,
4205 SmallVectorImpl<uint64_t> &);
4206
4207// Set the bits in `Bits`, which correspond to the value representations of
4208// the actual members of the record type `RTy`. Note that this function does
4209// not handle base classes, virtual tables, etc, since they cannot happen in
4210// CMSE function arguments or return. The bit mask corresponds to the target
4211// memory layout, i.e. it's endian dependent.
4212static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
4213 SmallVectorImpl<uint64_t> &Bits) {
4214 ASTContext &Context = CGM.getContext();
4215 int CharWidth = Context.getCharWidth();
4216 const RecordDecl *RD = RTy->getDecl()->getDefinition();
4217 const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(D: RD);
4218 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
4219
4220 int Idx = 0;
4221 for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) {
4222 const FieldDecl *F = *I;
4223
4224 if (F->isUnnamedBitField() || F->isZeroLengthBitField() ||
4225 F->getType()->isIncompleteArrayType())
4226 continue;
4227
4228 if (F->isBitField()) {
4229 const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(FD: F);
4230 setBitRange(Bits, StorageOffset: Offset + BFI.StorageOffset.getQuantity(),
4231 StorageSize: BFI.StorageSize / CharWidth, BitOffset: BFI.Offset, BitWidth: BFI.Size, CharWidth,
4232 BigEndian: CGM.getDataLayout().isBigEndian());
4233 continue;
4234 }
4235
4236 setUsedBits(CGM, F->getType(),
4237 Offset + ASTLayout.getFieldOffset(FieldNo: Idx) / CharWidth, Bits);
4238 }
4239}
4240
4241// Set the bits in `Bits`, which correspond to the value representations of
4242// the elements of an array type `ATy`.
4243static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
4244 int Offset, SmallVectorImpl<uint64_t> &Bits) {
4245 const ASTContext &Context = CGM.getContext();
4246
4247 QualType ETy = Context.getBaseElementType(VAT: ATy);
4248 int Size = Context.getTypeSizeInChars(T: ETy).getQuantity();
4249 SmallVector<uint64_t, 4> TmpBits(Size);
4250 setUsedBits(CGM, ETy, 0, TmpBits);
4251
4252 for (int I = 0, N = Context.getConstantArrayElementCount(CA: ATy); I < N; ++I) {
4253 auto Src = TmpBits.begin();
4254 auto Dst = Bits.begin() + Offset + I * Size;
4255 for (int J = 0; J < Size; ++J)
4256 *Dst++ |= *Src++;
4257 }
4258}
4259
4260// Set the bits in `Bits`, which correspond to the value representations of
4261// the type `QTy`.
4262static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
4263 SmallVectorImpl<uint64_t> &Bits) {
4264 if (const auto *RTy = QTy->getAsCanonical<RecordType>())
4265 return setUsedBits(CGM, RTy, Offset, Bits);
4266
4267 ASTContext &Context = CGM.getContext();
4268 if (const auto *ATy = Context.getAsConstantArrayType(T: QTy))
4269 return setUsedBits(CGM, ATy, Offset, Bits);
4270
4271 int Size = Context.getTypeSizeInChars(T: QTy).getQuantity();
4272 if (Size <= 0)
4273 return;
4274
4275 std::fill_n(first: Bits.begin() + Offset, n: Size,
4276 value: (uint64_t(1) << Context.getCharWidth()) - 1);
4277}
4278
4279static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
4280 int Pos, int Size, int CharWidth,
4281 bool BigEndian) {
4282 assert(Size > 0);
4283 uint64_t Mask = 0;
4284 if (BigEndian) {
4285 for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
4286 ++P)
4287 Mask = (Mask << CharWidth) | *P;
4288 } else {
4289 auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
4290 do
4291 Mask = (Mask << CharWidth) | *--P;
4292 while (P != End);
4293 }
4294 return Mask;
4295}
4296
4297// Emit code to clear the bits in a record, which aren't a part of any user
4298// declared member, when the record is a function return.
4299llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
4300 llvm::IntegerType *ITy,
4301 QualType QTy) {
4302 assert(Src->getType() == ITy);
4303 assert(ITy->getScalarSizeInBits() <= 64);
4304
4305 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
4306 int Size = DataLayout.getTypeStoreSize(Ty: ITy);
4307 SmallVector<uint64_t, 4> Bits(Size);
4308 setUsedBits(CGM, RTy: QTy->castAsCanonical<RecordType>(), Offset: 0, Bits);
4309
4310 int CharWidth = CGM.getContext().getCharWidth();
4311 uint64_t Mask =
4312 buildMultiCharMask(Bits, Pos: 0, Size, CharWidth, BigEndian: DataLayout.isBigEndian());
4313
4314 return Builder.CreateAnd(LHS: Src, RHS: Mask, Name: "cmse.clear");
4315}
4316
4317// Emit code to clear the bits in a record, which aren't a part of any user
4318// declared member, when the record is a function argument.
4319llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
4320 llvm::ArrayType *ATy,
4321 QualType QTy) {
4322 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
4323 int Size = DataLayout.getTypeStoreSize(Ty: ATy);
4324 SmallVector<uint64_t, 16> Bits(Size);
4325 setUsedBits(CGM, RTy: QTy->castAsCanonical<RecordType>(), Offset: 0, Bits);
4326
4327 // Clear each element of the LLVM array.
4328 int CharWidth = CGM.getContext().getCharWidth();
4329 int CharsPerElt =
4330 ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
4331 int MaskIndex = 0;
4332 llvm::Value *R = llvm::PoisonValue::get(T: ATy);
4333 for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
4334 uint64_t Mask = buildMultiCharMask(Bits, Pos: MaskIndex, Size: CharsPerElt, CharWidth,
4335 BigEndian: DataLayout.isBigEndian());
4336 MaskIndex += CharsPerElt;
4337 llvm::Value *T0 = Builder.CreateExtractValue(Agg: Src, Idxs: I);
4338 llvm::Value *T1 = Builder.CreateAnd(LHS: T0, RHS: Mask, Name: "cmse.clear");
4339 R = Builder.CreateInsertValue(Agg: R, Val: T1, Idxs: I);
4340 }
4341
4342 return R;
4343}
4344
4345void CodeGenFunction::EmitFunctionEpilog(
4346 const CGFunctionInfo &FI, bool EmitRetDbgLoc, SourceLocation EndLoc,
4347 uint64_t RetKeyInstructionsSourceAtom) {
4348 if (FI.isNoReturn()) {
4349 // Noreturn functions don't return.
4350 EmitUnreachable(Loc: EndLoc);
4351 return;
4352 }
4353
4354 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {
4355 // Naked functions don't have epilogues.
4356 Builder.CreateUnreachable();
4357 return;
4358 }
4359
4360 // Functions with no result always return void.
4361 if (!ReturnValue.isValid()) {
4362 auto *I = Builder.CreateRetVoid();
4363 if (RetKeyInstructionsSourceAtom)
4364 addInstToSpecificSourceAtom(KeyInstruction: I, Backup: nullptr, Atom: RetKeyInstructionsSourceAtom);
4365 else
4366 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4367 return;
4368 }
4369
4370 llvm::DebugLoc RetDbgLoc;
4371 llvm::Value *RV = nullptr;
4372 QualType RetTy = FI.getReturnType();
4373 const ABIArgInfo &RetAI = FI.getReturnInfo();
4374
4375 switch (RetAI.getKind()) {
4376 case ABIArgInfo::InAlloca:
4377 // Aggregates get evaluated directly into the destination. Sometimes we
4378 // need to return the sret value in a register, though.
4379 assert(hasAggregateEvaluationKind(RetTy));
4380 if (RetAI.getInAllocaSRet()) {
4381 llvm::Function::arg_iterator EI = CurFn->arg_end();
4382 --EI;
4383 llvm::Value *ArgStruct = &*EI;
4384 llvm::Value *SRet = Builder.CreateStructGEP(
4385 Ty: FI.getArgStruct(), Ptr: ArgStruct, Idx: RetAI.getInAllocaFieldIndex());
4386 llvm::Type *Ty =
4387 cast<llvm::GetElementPtrInst>(Val: SRet)->getResultElementType();
4388 RV = Builder.CreateAlignedLoad(Ty, Addr: SRet, Align: getPointerAlign(), Name: "sret");
4389 }
4390 break;
4391
4392 case ABIArgInfo::Indirect: {
4393 auto AI = CurFn->arg_begin();
4394 if (RetAI.isSRetAfterThis())
4395 ++AI;
4396 switch (getEvaluationKind(T: RetTy)) {
4397 case TEK_Complex: {
4398 ComplexPairTy RT =
4399 EmitLoadOfComplex(src: MakeAddrLValue(Addr: ReturnValue, T: RetTy), loc: EndLoc);
4400 EmitStoreOfComplex(V: RT, dest: MakeNaturalAlignAddrLValue(V: &*AI, T: RetTy),
4401 /*isInit*/ true);
4402 break;
4403 }
4404 case TEK_Aggregate:
4405 // Do nothing; aggregates get evaluated directly into the destination.
4406 break;
4407 case TEK_Scalar: {
4408 LValueBaseInfo BaseInfo;
4409 TBAAAccessInfo TBAAInfo;
4410 CharUnits Alignment =
4411 CGM.getNaturalTypeAlignment(T: RetTy, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
4412 Address ArgAddr(&*AI, ConvertType(T: RetTy), Alignment);
4413 LValue ArgVal =
4414 LValue::MakeAddr(Addr: ArgAddr, type: RetTy, Context&: getContext(), BaseInfo, TBAAInfo);
4415 EmitStoreOfScalar(
4416 value: EmitLoadOfScalar(lvalue: MakeAddrLValue(Addr: ReturnValue, T: RetTy), Loc: EndLoc), lvalue: ArgVal,
4417 /*isInit*/ true);
4418 break;
4419 }
4420 }
4421 break;
4422 }
4423
4424 case ABIArgInfo::Extend:
4425 case ABIArgInfo::Direct:
4426 if (RetAI.getCoerceToType() == ConvertType(T: RetTy) &&
4427 RetAI.getDirectOffset() == 0) {
4428 // The internal return value temp always will have pointer-to-return-type
4429 // type, just do a load.
4430
4431 // If there is a dominating store to ReturnValue, we can elide
4432 // the load, zap the store, and usually zap the alloca.
4433 if (llvm::StoreInst *SI = findDominatingStoreToReturnValue(CGF&: *this)) {
4434 // Reuse the debug location from the store unless there is
4435 // cleanup code to be emitted between the store and return
4436 // instruction.
4437 if (EmitRetDbgLoc && !AutoreleaseResult)
4438 RetDbgLoc = SI->getDebugLoc();
4439 // Get the stored value and nuke the now-dead store.
4440 RV = SI->getValueOperand();
4441 SI->eraseFromParent();
4442
4443 // Otherwise, we have to do a simple load.
4444 } else {
4445 RV = Builder.CreateLoad(Addr: ReturnValue);
4446 }
4447 } else {
4448 // If the value is offset in memory, apply the offset now.
4449 Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
4450
4451 RV = CreateCoercedLoad(Src: V, SrcFETy: RetTy, Ty: RetAI.getCoerceToType(), CGF&: *this);
4452 }
4453
4454 // In ARC, end functions that return a retainable type with a call
4455 // to objc_autoreleaseReturnValue.
4456 if (AutoreleaseResult) {
4457#ifndef NDEBUG
4458 // Type::isObjCRetainabletype has to be called on a QualType that hasn't
4459 // been stripped of the typedefs, so we cannot use RetTy here. Get the
4460 // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
4461 // CurCodeDecl or BlockInfo.
4462 QualType RT;
4463
4464 if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
4465 RT = FD->getReturnType();
4466 else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
4467 RT = MD->getReturnType();
4468 else if (isa<BlockDecl>(CurCodeDecl))
4469 RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
4470 else
4471 llvm_unreachable("Unexpected function/method type");
4472
4473 assert(getLangOpts().ObjCAutoRefCount && !FI.isReturnsRetained() &&
4474 RT->isObjCRetainableType());
4475#endif
4476 RV = emitAutoreleaseOfResult(CGF&: *this, result: RV);
4477 }
4478
4479 break;
4480
4481 case ABIArgInfo::Ignore:
4482 break;
4483
4484 case ABIArgInfo::CoerceAndExpand: {
4485 auto coercionType = RetAI.getCoerceAndExpandType();
4486 auto unpaddedCoercionType = RetAI.getUnpaddedCoerceAndExpandType();
4487 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
4488
4489 // Load all of the coerced elements out into results.
4490 llvm::SmallVector<llvm::Value *, 4> results;
4491 Address addr = ReturnValue.withElementType(ElemTy: coercionType);
4492 unsigned unpaddedIndex = 0;
4493 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
4494 auto coercedEltType = coercionType->getElementType(N: i);
4495 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType: coercedEltType))
4496 continue;
4497
4498 auto eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
4499 llvm::Value *elt = CreateCoercedLoad(
4500 Src: eltAddr, SrcFETy: RetTy,
4501 Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
4502 : unpaddedCoercionType,
4503 CGF&: *this);
4504 results.push_back(Elt: elt);
4505 }
4506
4507 // If we have one result, it's the single direct result type.
4508 if (results.size() == 1) {
4509 RV = results[0];
4510
4511 // Otherwise, we need to make a first-class aggregate.
4512 } else {
4513 // Construct a return type that lacks padding elements.
4514 llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
4515
4516 RV = llvm::PoisonValue::get(T: returnType);
4517 for (unsigned i = 0, e = results.size(); i != e; ++i) {
4518 RV = Builder.CreateInsertValue(Agg: RV, Val: results[i], Idxs: i);
4519 }
4520 }
4521 break;
4522 }
4523 case ABIArgInfo::TargetSpecific: {
4524 Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
4525 RV = CGM.getABIInfo().createCoercedLoad(SrcAddr: V, AI: RetAI, CGF&: *this);
4526 break;
4527 }
4528 case ABIArgInfo::Expand:
4529 case ABIArgInfo::IndirectAliased:
4530 llvm_unreachable("Invalid ABI kind for return argument");
4531 }
4532
4533 llvm::Instruction *Ret;
4534 if (RV) {
4535 if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) {
4536 // For certain return types, clear padding bits, as they may reveal
4537 // sensitive information.
4538 // Small struct/union types are passed as integers.
4539 auto *ITy = dyn_cast<llvm::IntegerType>(Val: RV->getType());
4540 if (ITy != nullptr && isa<RecordType>(Val: RetTy.getCanonicalType()))
4541 RV = EmitCMSEClearRecord(Src: RV, ITy, QTy: RetTy);
4542 }
4543 EmitReturnValueCheck(RV);
4544 Ret = Builder.CreateRet(V: RV);
4545 } else {
4546 Ret = Builder.CreateRetVoid();
4547 }
4548
4549 if (RetDbgLoc)
4550 Ret->setDebugLoc(std::move(RetDbgLoc));
4551
4552 llvm::Value *Backup = RV ? Ret->getOperand(i: 0) : nullptr;
4553 if (RetKeyInstructionsSourceAtom)
4554 addInstToSpecificSourceAtom(KeyInstruction: Ret, Backup, Atom: RetKeyInstructionsSourceAtom);
4555 else
4556 addInstToNewSourceAtom(KeyInstruction: Ret, Backup);
4557}
4558
4559void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
4560 // A current decl may not be available when emitting vtable thunks.
4561 if (!CurCodeDecl)
4562 return;
4563
4564 // If the return block isn't reachable, neither is this check, so don't emit
4565 // it.
4566 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty())
4567 return;
4568
4569 ReturnsNonNullAttr *RetNNAttr = nullptr;
4570 if (SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute))
4571 RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
4572
4573 if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
4574 return;
4575
4576 // Prefer the returns_nonnull attribute if it's present.
4577 SourceLocation AttrLoc;
4578 SanitizerKind::SanitizerOrdinal CheckKind;
4579 SanitizerHandler Handler;
4580 if (RetNNAttr) {
4581 assert(!requiresReturnValueNullabilityCheck() &&
4582 "Cannot check nullability and the nonnull attribute");
4583 AttrLoc = RetNNAttr->getLocation();
4584 CheckKind = SanitizerKind::SO_ReturnsNonnullAttribute;
4585 Handler = SanitizerHandler::NonnullReturn;
4586 } else {
4587 if (auto *DD = dyn_cast<DeclaratorDecl>(Val: CurCodeDecl))
4588 if (auto *TSI = DD->getTypeSourceInfo())
4589 if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
4590 AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
4591 CheckKind = SanitizerKind::SO_NullabilityReturn;
4592 Handler = SanitizerHandler::NullabilityReturn;
4593 }
4594
4595 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4596
4597 // Make sure the "return" source location is valid. If we're checking a
4598 // nullability annotation, make sure the preconditions for the check are met.
4599 llvm::BasicBlock *Check = createBasicBlock(name: "nullcheck");
4600 llvm::BasicBlock *NoCheck = createBasicBlock(name: "no.nullcheck");
4601 llvm::Value *SLocPtr = Builder.CreateLoad(Addr: ReturnLocation, Name: "return.sloc.load");
4602 llvm::Value *CanNullCheck = Builder.CreateIsNotNull(Arg: SLocPtr);
4603 if (requiresReturnValueNullabilityCheck())
4604 CanNullCheck =
4605 Builder.CreateAnd(LHS: CanNullCheck, RHS: RetValNullabilityPrecondition);
4606 Builder.CreateCondBr(Cond: CanNullCheck, True: Check, False: NoCheck);
4607 EmitBlock(BB: Check);
4608
4609 // Now do the null check.
4610 llvm::Value *Cond = Builder.CreateIsNotNull(Arg: RV);
4611 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc: AttrLoc)};
4612 llvm::Value *DynamicData[] = {SLocPtr};
4613 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: DynamicData);
4614
4615 EmitBlock(BB: NoCheck);
4616
4617#ifndef NDEBUG
4618 // The return location should not be used after the check has been emitted.
4619 ReturnLocation = Address::invalid();
4620#endif
4621}
4622
4623static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
4624 const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
4625 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
4626}
4627
4628static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, QualType Ty) {
4629 // FIXME: Generate IR in one pass, rather than going back and fixing up these
4630 // placeholders.
4631 llvm::Type *IRTy = CGF.ConvertTypeForMem(T: Ty);
4632 llvm::Type *IRPtrTy = llvm::PointerType::getUnqual(C&: CGF.getLLVMContext());
4633 llvm::Value *Placeholder = llvm::PoisonValue::get(T: IRPtrTy);
4634
4635 // FIXME: When we generate this IR in one pass, we shouldn't need
4636 // this win32-specific alignment hack.
4637 CharUnits Align = CharUnits::fromQuantity(Quantity: 4);
4638 Placeholder = CGF.Builder.CreateAlignedLoad(Ty: IRPtrTy, Addr: Placeholder, Align);
4639
4640 return AggValueSlot::forAddr(
4641 addr: Address(Placeholder, IRTy, Align), quals: Ty.getQualifiers(),
4642 isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
4643 isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
4644}
4645
4646void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
4647 const VarDecl *param,
4648 SourceLocation loc) {
4649 // StartFunction converted the ABI-lowered parameter(s) into a
4650 // local alloca. We need to turn that into an r-value suitable
4651 // for EmitCall.
4652 Address local = GetAddrOfLocalVar(VD: param);
4653
4654 QualType type = param->getType();
4655
4656 // GetAddrOfLocalVar returns a pointer-to-pointer for references,
4657 // but the argument needs to be the original pointer.
4658 if (type->isReferenceType()) {
4659 args.add(rvalue: RValue::get(V: Builder.CreateLoad(Addr: local)), type);
4660
4661 // In ARC, move out of consumed arguments so that the release cleanup
4662 // entered by StartFunction doesn't cause an over-release. This isn't
4663 // optimal -O0 code generation, but it should get cleaned up when
4664 // optimization is enabled. This also assumes that delegate calls are
4665 // performed exactly once for a set of arguments, but that should be safe.
4666 } else if (getLangOpts().ObjCAutoRefCount &&
4667 param->hasAttr<NSConsumedAttr>() && type->isObjCRetainableType()) {
4668 llvm::Value *ptr = Builder.CreateLoad(Addr: local);
4669 auto null =
4670 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: ptr->getType()));
4671 Builder.CreateStore(Val: null, Addr: local);
4672 args.add(rvalue: RValue::get(V: ptr), type);
4673
4674 // For the most part, we just need to load the alloca, except that
4675 // aggregate r-values are actually pointers to temporaries.
4676 } else {
4677 args.add(rvalue: convertTempToRValue(addr: local, type, Loc: loc), type);
4678 }
4679
4680 // Deactivate the cleanup for the callee-destructed param that was pushed.
4681 if (type->isRecordType() && !CurFuncIsThunk &&
4682 type->castAsRecordDecl()->isParamDestroyedInCallee() &&
4683 param->needsDestruction(Ctx: getContext())) {
4684 EHScopeStack::stable_iterator cleanup =
4685 CalleeDestructedParamCleanups.lookup(Val: cast<ParmVarDecl>(Val: param));
4686 assert(cleanup.isValid() &&
4687 "cleanup for callee-destructed param not recorded");
4688 // This unreachable is a temporary marker which will be removed later.
4689 llvm::Instruction *isActive = Builder.CreateUnreachable();
4690 args.addArgCleanupDeactivation(Cleanup: cleanup, IsActiveIP: isActive);
4691 }
4692}
4693
4694static bool isProvablyNull(llvm::Value *addr) {
4695 return llvm::isa_and_nonnull<llvm::ConstantPointerNull>(Val: addr);
4696}
4697
4698static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) {
4699 return llvm::isKnownNonZero(V: Addr.getBasePointer(), Q: CGF.CGM.getDataLayout());
4700}
4701
4702/// Emit the actual writing-back of a writeback.
4703static void emitWriteback(CodeGenFunction &CGF,
4704 const CallArgList::Writeback &writeback) {
4705 const LValue &srcLV = writeback.Source;
4706 Address srcAddr = srcLV.getAddress();
4707 assert(!isProvablyNull(srcAddr.getBasePointer()) &&
4708 "shouldn't have writeback for provably null argument");
4709
4710 if (writeback.WritebackExpr) {
4711 CGF.EmitIgnoredExpr(E: writeback.WritebackExpr);
4712 CGF.EmitLifetimeEnd(Addr: writeback.Temporary.getBasePointer());
4713 return;
4714 }
4715
4716 llvm::BasicBlock *contBB = nullptr;
4717
4718 // If the argument wasn't provably non-null, we need to null check
4719 // before doing the store.
4720 bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4721
4722 if (!provablyNonNull) {
4723 llvm::BasicBlock *writebackBB = CGF.createBasicBlock(name: "icr.writeback");
4724 contBB = CGF.createBasicBlock(name: "icr.done");
4725
4726 llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4727 CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: writebackBB);
4728 CGF.EmitBlock(BB: writebackBB);
4729 }
4730
4731 // Load the value to writeback.
4732 llvm::Value *value = CGF.Builder.CreateLoad(Addr: writeback.Temporary);
4733
4734 // Cast it back, in case we're writing an id to a Foo* or something.
4735 value = CGF.Builder.CreateBitCast(V: value, DestTy: srcAddr.getElementType(),
4736 Name: "icr.writeback-cast");
4737
4738 // Perform the writeback.
4739
4740 // If we have a "to use" value, it's something we need to emit a use
4741 // of. This has to be carefully threaded in: if it's done after the
4742 // release it's potentially undefined behavior (and the optimizer
4743 // will ignore it), and if it happens before the retain then the
4744 // optimizer could move the release there.
4745 if (writeback.ToUse) {
4746 assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
4747
4748 // Retain the new value. No need to block-copy here: the block's
4749 // being passed up the stack.
4750 value = CGF.EmitARCRetainNonBlock(value);
4751
4752 // Emit the intrinsic use here.
4753 CGF.EmitARCIntrinsicUse(values: writeback.ToUse);
4754
4755 // Load the old value (primitively).
4756 llvm::Value *oldValue = CGF.EmitLoadOfScalar(lvalue: srcLV, Loc: SourceLocation());
4757
4758 // Put the new value in place (primitively).
4759 CGF.EmitStoreOfScalar(value, lvalue: srcLV, /*init*/ isInit: false);
4760
4761 // Release the old value.
4762 CGF.EmitARCRelease(value: oldValue, precise: srcLV.isARCPreciseLifetime());
4763
4764 // Otherwise, we can just do a normal lvalue store.
4765 } else {
4766 CGF.EmitStoreThroughLValue(Src: RValue::get(V: value), Dst: srcLV);
4767 }
4768
4769 // Jump to the continuation block.
4770 if (!provablyNonNull)
4771 CGF.EmitBlock(BB: contBB);
4772}
4773
4774static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
4775 const CallArgList &CallArgs) {
4776 ArrayRef<CallArgList::CallArgCleanup> Cleanups =
4777 CallArgs.getCleanupsToDeactivate();
4778 // Iterate in reverse to increase the likelihood of popping the cleanup.
4779 for (const auto &I : llvm::reverse(C&: Cleanups)) {
4780 CGF.DeactivateCleanupBlock(Cleanup: I.Cleanup, DominatingIP: I.IsActiveIP);
4781 I.IsActiveIP->eraseFromParent();
4782 }
4783}
4784
4785static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
4786 if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(Val: E->IgnoreParens()))
4787 if (uop->getOpcode() == UO_AddrOf)
4788 return uop->getSubExpr();
4789 return nullptr;
4790}
4791
4792/// Emit an argument that's being passed call-by-writeback. That is,
4793/// we are passing the address of an __autoreleased temporary; it
4794/// might be copy-initialized with the current value of the given
4795/// address, but it will definitely be copied out of after the call.
4796static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
4797 const ObjCIndirectCopyRestoreExpr *CRE) {
4798 LValue srcLV;
4799
4800 // Make an optimistic effort to emit the address as an l-value.
4801 // This can fail if the argument expression is more complicated.
4802 if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(E: CRE->getSubExpr())) {
4803 srcLV = CGF.EmitLValue(E: lvExpr);
4804
4805 // Otherwise, just emit it as a scalar.
4806 } else {
4807 Address srcAddr = CGF.EmitPointerWithAlignment(Addr: CRE->getSubExpr());
4808
4809 QualType srcAddrType =
4810 CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
4811 srcLV = CGF.MakeAddrLValue(Addr: srcAddr, T: srcAddrType);
4812 }
4813 Address srcAddr = srcLV.getAddress();
4814
4815 // The dest and src types don't necessarily match in LLVM terms
4816 // because of the crazy ObjC compatibility rules.
4817
4818 llvm::PointerType *destType =
4819 cast<llvm::PointerType>(Val: CGF.ConvertType(T: CRE->getType()));
4820 llvm::Type *destElemType =
4821 CGF.ConvertTypeForMem(T: CRE->getType()->getPointeeType());
4822
4823 // If the address is a constant null, just pass the appropriate null.
4824 if (isProvablyNull(addr: srcAddr.getBasePointer())) {
4825 args.add(rvalue: RValue::get(V: llvm::ConstantPointerNull::get(T: destType)),
4826 type: CRE->getType());
4827 return;
4828 }
4829
4830 // Create the temporary.
4831 Address temp =
4832 CGF.CreateTempAlloca(Ty: destElemType, align: CGF.getPointerAlign(), Name: "icr.temp");
4833 // Loading an l-value can introduce a cleanup if the l-value is __weak,
4834 // and that cleanup will be conditional if we can't prove that the l-value
4835 // isn't null, so we need to register a dominating point so that the cleanups
4836 // system will make valid IR.
4837 CodeGenFunction::ConditionalEvaluation condEval(CGF);
4838
4839 // Zero-initialize it if we're not doing a copy-initialization.
4840 bool shouldCopy = CRE->shouldCopy();
4841 if (!shouldCopy) {
4842 llvm::Value *null =
4843 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: destElemType));
4844 CGF.Builder.CreateStore(Val: null, Addr: temp);
4845 }
4846
4847 llvm::BasicBlock *contBB = nullptr;
4848 llvm::BasicBlock *originBB = nullptr;
4849
4850 // If the address is *not* known to be non-null, we need to switch.
4851 llvm::Value *finalArgument;
4852
4853 bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4854
4855 if (provablyNonNull) {
4856 finalArgument = temp.emitRawPointer(CGF);
4857 } else {
4858 llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4859
4860 finalArgument = CGF.Builder.CreateSelect(
4861 C: isNull, True: llvm::ConstantPointerNull::get(T: destType),
4862 False: temp.emitRawPointer(CGF), Name: "icr.argument");
4863
4864 // If we need to copy, then the load has to be conditional, which
4865 // means we need control flow.
4866 if (shouldCopy) {
4867 originBB = CGF.Builder.GetInsertBlock();
4868 contBB = CGF.createBasicBlock(name: "icr.cont");
4869 llvm::BasicBlock *copyBB = CGF.createBasicBlock(name: "icr.copy");
4870 CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: copyBB);
4871 CGF.EmitBlock(BB: copyBB);
4872 condEval.begin(CGF);
4873 }
4874 }
4875
4876 llvm::Value *valueToUse = nullptr;
4877
4878 // Perform a copy if necessary.
4879 if (shouldCopy) {
4880 RValue srcRV = CGF.EmitLoadOfLValue(V: srcLV, Loc: SourceLocation());
4881 assert(srcRV.isScalar());
4882
4883 llvm::Value *src = srcRV.getScalarVal();
4884 src = CGF.Builder.CreateBitCast(V: src, DestTy: destElemType, Name: "icr.cast");
4885
4886 // Use an ordinary store, not a store-to-lvalue.
4887 CGF.Builder.CreateStore(Val: src, Addr: temp);
4888
4889 // If optimization is enabled, and the value was held in a
4890 // __strong variable, we need to tell the optimizer that this
4891 // value has to stay alive until we're doing the store back.
4892 // This is because the temporary is effectively unretained,
4893 // and so otherwise we can violate the high-level semantics.
4894 if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4895 srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {
4896 valueToUse = src;
4897 }
4898 }
4899
4900 // Finish the control flow if we needed it.
4901 if (shouldCopy && !provablyNonNull) {
4902 llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
4903 CGF.EmitBlock(BB: contBB);
4904
4905 // Make a phi for the value to intrinsically use.
4906 if (valueToUse) {
4907 llvm::PHINode *phiToUse =
4908 CGF.Builder.CreatePHI(Ty: valueToUse->getType(), NumReservedValues: 2, Name: "icr.to-use");
4909 phiToUse->addIncoming(V: valueToUse, BB: copyBB);
4910 phiToUse->addIncoming(V: llvm::PoisonValue::get(T: valueToUse->getType()),
4911 BB: originBB);
4912 valueToUse = phiToUse;
4913 }
4914
4915 condEval.end(CGF);
4916 }
4917
4918 args.addWriteback(srcLV, temporary: temp, toUse: valueToUse);
4919 args.add(rvalue: RValue::get(V: finalArgument), type: CRE->getType());
4920}
4921
4922void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
4923 assert(!StackBase);
4924
4925 // Save the stack.
4926 StackBase = CGF.Builder.CreateStackSave(Name: "inalloca.save");
4927}
4928
4929void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
4930 if (StackBase) {
4931 // Restore the stack after the call.
4932 CGF.Builder.CreateStackRestore(Ptr: StackBase);
4933 }
4934}
4935
4936void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
4937 SourceLocation ArgLoc,
4938 AbstractCallee AC, unsigned ParmNum) {
4939 if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
4940 SanOpts.has(K: SanitizerKind::NullabilityArg)))
4941 return;
4942
4943 // The param decl may be missing in a variadic function.
4944 auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(I: ParmNum) : nullptr;
4945 unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
4946
4947 // Prefer the nonnull attribute if it's present.
4948 const NonNullAttr *NNAttr = nullptr;
4949 if (SanOpts.has(K: SanitizerKind::NonnullAttribute))
4950 NNAttr = getNonNullAttr(FD: AC.getDecl(), PVD, ArgType, ArgNo);
4951
4952 bool CanCheckNullability = false;
4953 if (SanOpts.has(K: SanitizerKind::NullabilityArg) && !NNAttr && PVD &&
4954 !PVD->getType()->isRecordType()) {
4955 auto Nullability = PVD->getType()->getNullability();
4956 CanCheckNullability = Nullability &&
4957 *Nullability == NullabilityKind::NonNull &&
4958 PVD->getTypeSourceInfo();
4959 }
4960
4961 if (!NNAttr && !CanCheckNullability)
4962 return;
4963
4964 SourceLocation AttrLoc;
4965 SanitizerKind::SanitizerOrdinal CheckKind;
4966 SanitizerHandler Handler;
4967 if (NNAttr) {
4968 AttrLoc = NNAttr->getLocation();
4969 CheckKind = SanitizerKind::SO_NonnullAttribute;
4970 Handler = SanitizerHandler::NonnullArg;
4971 } else {
4972 AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4973 CheckKind = SanitizerKind::SO_NullabilityArg;
4974 Handler = SanitizerHandler::NullabilityArg;
4975 }
4976
4977 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4978 llvm::Value *Cond = EmitNonNullRValueCheck(RV, T: ArgType);
4979 llvm::Constant *StaticData[] = {
4980 EmitCheckSourceLocation(Loc: ArgLoc),
4981 EmitCheckSourceLocation(Loc: AttrLoc),
4982 llvm::ConstantInt::get(Ty: Int32Ty, V: ArgNo + 1),
4983 };
4984 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: {});
4985}
4986
4987void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType,
4988 SourceLocation ArgLoc,
4989 AbstractCallee AC, unsigned ParmNum) {
4990 if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
4991 SanOpts.has(K: SanitizerKind::NullabilityArg)))
4992 return;
4993
4994 EmitNonNullArgCheck(RV: RValue::get(Addr, CGF&: *this), ArgType, ArgLoc, AC, ParmNum);
4995}
4996
4997// Check if the call is going to use the inalloca convention. This needs to
4998// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
4999// later, so we can't check it directly.
5000static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
5001 ArrayRef<QualType> ArgTypes) {
5002 // The Swift calling conventions don't go through the target-specific
5003 // argument classification, they never use inalloca.
5004 // TODO: Consider limiting inalloca use to only calling conventions supported
5005 // by MSVC.
5006 if (ExplicitCC == CC_Swift || ExplicitCC == CC_SwiftAsync)
5007 return false;
5008 if (!CGM.getTarget().getCXXABI().isMicrosoft())
5009 return false;
5010 return llvm::any_of(Range&: ArgTypes, P: [&](QualType Ty) {
5011 return isInAllocaArgument(ABI&: CGM.getCXXABI(), type: Ty);
5012 });
5013}
5014
5015#ifndef NDEBUG
5016// Determine whether the given argument is an Objective-C method
5017// that may have type parameters in its signature.
5018static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
5019 const DeclContext *dc = method->getDeclContext();
5020 if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {
5021 return classDecl->getTypeParamListAsWritten();
5022 }
5023
5024 if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
5025 return catDecl->getTypeParamList();
5026 }
5027
5028 return false;
5029}
5030#endif
5031
5032/// EmitCallArgs - Emit call arguments for a function.
5033void CodeGenFunction::EmitCallArgs(
5034 CallArgList &Args, PrototypeWrapper Prototype,
5035 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
5036 AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
5037 SmallVector<QualType, 16> ArgTypes;
5038
5039 assert((ParamsToSkip == 0 || Prototype.P) &&
5040 "Can't skip parameters if type info is not provided");
5041
5042 // This variable only captures *explicitly* written conventions, not those
5043 // applied by default via command line flags or target defaults, such as
5044 // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
5045 // require knowing if this is a C++ instance method or being able to see
5046 // unprototyped FunctionTypes.
5047 CallingConv ExplicitCC = CC_C;
5048
5049 // First, if a prototype was provided, use those argument types.
5050 bool IsVariadic = false;
5051 if (Prototype.P) {
5052 const auto *MD = dyn_cast<const ObjCMethodDecl *>(Val&: Prototype.P);
5053 if (MD) {
5054 IsVariadic = MD->isVariadic();
5055 ExplicitCC = getCallingConventionForDecl(
5056 D: MD, IsTargetDefaultMSABI: CGM.getTarget().getTriple().isOSWindows());
5057 ArgTypes.assign(in_start: MD->param_type_begin() + ParamsToSkip,
5058 in_end: MD->param_type_end());
5059 } else {
5060 const auto *FPT = cast<const FunctionProtoType *>(Val&: Prototype.P);
5061 IsVariadic = FPT->isVariadic();
5062 ExplicitCC = FPT->getExtInfo().getCC();
5063 ArgTypes.assign(in_start: FPT->param_type_begin() + ParamsToSkip,
5064 in_end: FPT->param_type_end());
5065 }
5066
5067#ifndef NDEBUG
5068 // Check that the prototyped types match the argument expression types.
5069 bool isGenericMethod = MD && isObjCMethodWithTypeParams(MD);
5070 CallExpr::const_arg_iterator Arg = ArgRange.begin();
5071 for (QualType Ty : ArgTypes) {
5072 assert(Arg != ArgRange.end() && "Running over edge of argument list!");
5073 QualType ParamTy = Ty.getNonReferenceType();
5074 QualType ArgTy = (*Arg)->getType();
5075 if (const auto *OBT = ParamTy->getAs<OverflowBehaviorType>())
5076 ParamTy = OBT->getUnderlyingType();
5077 if (const auto *OBT = ArgTy->getAs<OverflowBehaviorType>())
5078 ArgTy = OBT->getUnderlyingType();
5079 assert((isGenericMethod || Ty->isVariablyModifiedType() ||
5080 ParamTy->isObjCRetainableType() ||
5081 getContext().getCanonicalType(ParamTy).getTypePtr() ==
5082 getContext().getCanonicalType(ArgTy).getTypePtr()) &&
5083 "type mismatch in call argument!");
5084 ++Arg;
5085 }
5086
5087 // Either we've emitted all the call args, or we have a call to variadic
5088 // function.
5089 assert((Arg == ArgRange.end() || IsVariadic) &&
5090 "Extra arguments in non-variadic function!");
5091#endif
5092 }
5093
5094 // If we still have any arguments, emit them using the type of the argument.
5095 for (auto *A : llvm::drop_begin(RangeOrContainer&: ArgRange, N: ArgTypes.size()))
5096 ArgTypes.push_back(Elt: IsVariadic ? getVarArgType(Arg: A) : A->getType());
5097 assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
5098
5099 // We must evaluate arguments from right to left in the MS C++ ABI,
5100 // because arguments are destroyed left to right in the callee. As a special
5101 // case, there are certain language constructs that require left-to-right
5102 // evaluation, and in those cases we consider the evaluation order requirement
5103 // to trump the "destruction order is reverse construction order" guarantee.
5104 bool LeftToRight =
5105 CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
5106 ? Order == EvaluationOrder::ForceLeftToRight
5107 : Order != EvaluationOrder::ForceRightToLeft;
5108
5109 auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
5110 RValue EmittedArg) {
5111 if (!AC.hasFunctionDecl() || I >= AC.getNumParams())
5112 return;
5113 auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
5114 if (PS == nullptr)
5115 return;
5116
5117 const auto &Context = getContext();
5118 auto SizeTy = Context.getSizeType();
5119 auto T = Builder.getIntNTy(N: Context.getTypeSize(T: SizeTy));
5120 assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
5121 llvm::Value *V = evaluateOrEmitBuiltinObjectSize(
5122 E: Arg, Type: PS->getType(), ResType: T, EmittedE: EmittedArg.getScalarVal(), IsDynamic: PS->isDynamic());
5123 Args.add(rvalue: RValue::get(V), type: SizeTy);
5124 // If we're emitting args in reverse, be sure to do so with
5125 // pass_object_size, as well.
5126 if (!LeftToRight)
5127 std::swap(a&: Args.back(), b&: *(&Args.back() - 1));
5128 };
5129
5130 // Insert a stack save if we're going to need any inalloca args.
5131 if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
5132 assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
5133 "inalloca only supported on x86");
5134 Args.allocateArgumentMemory(CGF&: *this);
5135 }
5136
5137 // Evaluate each argument in the appropriate order.
5138 size_t CallArgsStart = Args.size();
5139 for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
5140 unsigned Idx = LeftToRight ? I : E - I - 1;
5141 CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
5142 unsigned InitialArgSize = Args.size();
5143 // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
5144 // the argument and parameter match or the objc method is parameterized.
5145 assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
5146 getContext().hasSameUnqualifiedType((*Arg)->getType(),
5147 ArgTypes[Idx]) ||
5148 (isa<ObjCMethodDecl>(AC.getDecl()) &&
5149 isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
5150 "Argument and parameter types don't match");
5151 EmitCallArg(args&: Args, E: *Arg, ArgType: ArgTypes[Idx]);
5152 // In particular, we depend on it being the last arg in Args, and the
5153 // objectsize bits depend on there only being one arg if !LeftToRight.
5154 assert(InitialArgSize + 1 == Args.size() &&
5155 "The code below depends on only adding one arg per EmitCallArg");
5156 (void)InitialArgSize;
5157 // Since pointer argument are never emitted as LValue, it is safe to emit
5158 // non-null argument check for r-value only.
5159 if (!Args.back().hasLValue()) {
5160 RValue RVArg = Args.back().getKnownRValue();
5161 EmitNonNullArgCheck(RV: RVArg, ArgType: ArgTypes[Idx], ArgLoc: (*Arg)->getExprLoc(), AC,
5162 ParmNum: ParamsToSkip + Idx);
5163 // @llvm.objectsize should never have side-effects and shouldn't need
5164 // destruction/cleanups, so we can safely "emit" it after its arg,
5165 // regardless of right-to-leftness
5166 MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
5167 }
5168 }
5169
5170 if (!LeftToRight) {
5171 // Un-reverse the arguments we just evaluated so they match up with the LLVM
5172 // IR function.
5173 std::reverse(first: Args.begin() + CallArgsStart, last: Args.end());
5174
5175 // Reverse the writebacks to match the MSVC ABI.
5176 Args.reverseWritebacks();
5177 }
5178}
5179
5180namespace {
5181
5182struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
5183 DestroyUnpassedArg(Address Addr, QualType Ty) : Addr(Addr), Ty(Ty) {}
5184
5185 Address Addr;
5186 QualType Ty;
5187
5188 void Emit(CodeGenFunction &CGF, Flags flags) override {
5189 QualType::DestructionKind DtorKind = Ty.isDestructedType();
5190 if (DtorKind == QualType::DK_cxx_destructor) {
5191 const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
5192 assert(!Dtor->isTrivial());
5193 CGF.EmitCXXDestructorCall(D: Dtor, Type: Dtor_Complete, /*for vbase*/ ForVirtualBase: false,
5194 /*Delegating=*/false, This: Addr, ThisTy: Ty);
5195 } else {
5196 CGF.callCStructDestructor(Dst: CGF.MakeAddrLValue(Addr, T: Ty));
5197 }
5198 }
5199};
5200
5201} // end anonymous namespace
5202
5203RValue CallArg::getRValue(CodeGenFunction &CGF) const {
5204 if (!HasLV)
5205 return RV;
5206 LValue Copy = CGF.MakeAddrLValue(Addr: CGF.CreateMemTempWithoutCast(T: Ty), T: Ty);
5207 CGF.EmitAggregateCopy(Dest: Copy, Src: LV, EltTy: Ty, MayOverlap: AggValueSlot::DoesNotOverlap,
5208 isVolatile: LV.isVolatile());
5209 IsUsed = true;
5210 return RValue::getAggregate(addr: Copy.getAddress());
5211}
5212
5213void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
5214 LValue Dst = CGF.MakeAddrLValue(Addr, T: Ty);
5215 if (!HasLV && RV.isScalar())
5216 CGF.EmitStoreOfScalar(value: RV.getScalarVal(), lvalue: Dst, /*isInit=*/true);
5217 else if (!HasLV && RV.isComplex())
5218 CGF.EmitStoreOfComplex(V: RV.getComplexVal(), dest: Dst, /*init=*/isInit: true);
5219 else {
5220 auto Addr = HasLV ? LV.getAddress() : RV.getAggregateAddress();
5221 LValue SrcLV = CGF.MakeAddrLValue(Addr, T: Ty);
5222 // We assume that call args are never copied into subobjects.
5223 CGF.EmitAggregateCopy(Dest: Dst, Src: SrcLV, EltTy: Ty, MayOverlap: AggValueSlot::DoesNotOverlap,
5224 isVolatile: HasLV ? LV.isVolatileQualified()
5225 : RV.isVolatileQualified());
5226 }
5227 IsUsed = true;
5228}
5229
5230void CodeGenFunction::EmitWritebacks(const CallArgList &args) {
5231 for (const auto &I : args.writebacks())
5232 emitWriteback(CGF&: *this, writeback: I);
5233}
5234
5235void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
5236 QualType type) {
5237 std::optional<DisableDebugLocationUpdates> Dis;
5238 if (isa<CXXDefaultArgExpr>(Val: E))
5239 Dis.emplace(args&: *this);
5240 if (const ObjCIndirectCopyRestoreExpr *CRE =
5241 dyn_cast<ObjCIndirectCopyRestoreExpr>(Val: E)) {
5242 assert(getLangOpts().ObjCAutoRefCount);
5243 return emitWritebackArg(CGF&: *this, args, CRE);
5244 }
5245
5246 // Add writeback for HLSLOutParamExpr.
5247 // Needs to be before the assert below because HLSLOutArgExpr is an LValue
5248 // and is not a reference.
5249 if (const HLSLOutArgExpr *OE = dyn_cast<HLSLOutArgExpr>(Val: E)) {
5250 EmitHLSLOutArgExpr(E: OE, Args&: args, Ty: type);
5251 return;
5252 }
5253
5254 assert(type->isReferenceType() == E->isGLValue() &&
5255 "reference binding to unmaterialized r-value!");
5256
5257 if (E->isGLValue()) {
5258 assert(E->getObjectKind() == OK_Ordinary);
5259 return args.add(rvalue: EmitReferenceBindingToExpr(E), type);
5260 }
5261
5262 bool HasAggregateEvalKind = hasAggregateEvaluationKind(T: type);
5263
5264 // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
5265 // However, we still have to push an EH-only cleanup in case we unwind before
5266 // we make it to the call.
5267 if (type->isRecordType() &&
5268 type->castAsRecordDecl()->isParamDestroyedInCallee()) {
5269 // If we're using inalloca, use the argument memory. Otherwise, use a
5270 // temporary.
5271 AggValueSlot Slot = args.isUsingInAlloca()
5272 ? createPlaceholderSlot(CGF&: *this, Ty: type)
5273 : CreateAggTemp(T: type, Name: "agg.tmp");
5274
5275 bool DestroyedInCallee = true, NeedsCleanup = true;
5276 if (const auto *RD = type->getAsCXXRecordDecl())
5277 DestroyedInCallee = RD->hasNonTrivialDestructor();
5278 else
5279 NeedsCleanup = type.isDestructedType();
5280
5281 if (DestroyedInCallee)
5282 Slot.setExternallyDestructed();
5283
5284 EmitAggExpr(E, AS: Slot);
5285 RValue RV = Slot.asRValue();
5286 args.add(rvalue: RV, type);
5287
5288 if (DestroyedInCallee && NeedsCleanup) {
5289 // Create a no-op GEP between the placeholder and the cleanup so we can
5290 // RAUW it successfully. It also serves as a marker of the first
5291 // instruction where the cleanup is active.
5292 pushFullExprCleanup<DestroyUnpassedArg>(kind: NormalAndEHCleanup,
5293 A: Slot.getAddress(), A: type);
5294 // This unreachable is a temporary marker which will be removed later.
5295 llvm::Instruction *IsActive =
5296 Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
5297 args.addArgCleanupDeactivation(Cleanup: EHStack.stable_begin(), IsActiveIP: IsActive);
5298 }
5299 return;
5300 }
5301
5302 if (HasAggregateEvalKind) {
5303 auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E);
5304 if (ICE && ICE->getCastKind() == CK_LValueToRValue &&
5305 ICE->getSubExpr()->getType().getAddressSpace() !=
5306 LangAS::hlsl_constant &&
5307 !type->isArrayParameterType() && !type.isNonTrivialToPrimitiveCopy()) {
5308 LValue L = EmitLValue(E: cast<CastExpr>(Val: E)->getSubExpr());
5309 assert(L.isSimple());
5310 args.addUncopiedAggregate(LV: L, type);
5311 return;
5312 }
5313 }
5314
5315 args.add(rvalue: EmitAnyExprToTemp(E), type);
5316}
5317
5318QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
5319 // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
5320 // implicitly widens null pointer constants that are arguments to varargs
5321 // functions to pointer-sized ints.
5322 if (!getTarget().getTriple().isOSWindows())
5323 return Arg->getType();
5324
5325 if (Arg->getType()->isIntegerType() &&
5326 getContext().getTypeSize(T: Arg->getType()) <
5327 getContext().getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default) &&
5328 Arg->isNullPointerConstant(Ctx&: getContext(),
5329 NPC: Expr::NPC_ValueDependentIsNotNull)) {
5330 return getContext().getIntPtrType();
5331 }
5332
5333 return Arg->getType();
5334}
5335
5336// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5337// optimizer it can aggressively ignore unwind edges.
5338void CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
5339 if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
5340 !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
5341 Inst->setMetadata(Kind: "clang.arc.no_objc_arc_exceptions",
5342 Node: CGM.getNoObjCARCExceptionsMetadata());
5343}
5344
5345/// Emits a call to the given no-arguments nounwind runtime function.
5346llvm::CallInst *
5347CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5348 const llvm::Twine &name) {
5349 return EmitNounwindRuntimeCall(callee, args: ArrayRef<llvm::Value *>(), name);
5350}
5351
5352/// Emits a call to the given nounwind runtime function.
5353llvm::CallInst *
5354CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5355 ArrayRef<Address> args,
5356 const llvm::Twine &name) {
5357 SmallVector<llvm::Value *, 3> values;
5358 for (auto arg : args)
5359 values.push_back(Elt: arg.emitRawPointer(CGF&: *this));
5360 return EmitNounwindRuntimeCall(callee, args: values, name);
5361}
5362
5363llvm::CallInst *
5364CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5365 ArrayRef<llvm::Value *> args,
5366 const llvm::Twine &name) {
5367 llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
5368 call->setDoesNotThrow();
5369 return call;
5370}
5371
5372/// Emits a simple call (never an invoke) to the given no-arguments
5373/// runtime function.
5374llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5375 const llvm::Twine &name) {
5376 return EmitRuntimeCall(callee, args: {}, name);
5377}
5378
5379// Calls which may throw must have operand bundles indicating which funclet
5380// they are nested within.
5381SmallVector<llvm::OperandBundleDef, 1>
5382CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
5383 // There is no need for a funclet operand bundle if we aren't inside a
5384 // funclet.
5385 if (!CurrentFuncletPad)
5386 return (SmallVector<llvm::OperandBundleDef, 1>());
5387
5388 // Skip intrinsics which cannot throw (as long as they don't lower into
5389 // regular function calls in the course of IR transformations).
5390 if (auto *CalleeFn = dyn_cast<llvm::Function>(Val: Callee->stripPointerCasts())) {
5391 if (CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) {
5392 auto IID = CalleeFn->getIntrinsicID();
5393 if (!llvm::IntrinsicInst::mayLowerToFunctionCall(IID))
5394 return (SmallVector<llvm::OperandBundleDef, 1>());
5395 }
5396 }
5397
5398 SmallVector<llvm::OperandBundleDef, 1> BundleList;
5399 BundleList.emplace_back(Args: "funclet", Args&: CurrentFuncletPad);
5400 return BundleList;
5401}
5402
5403/// Emits a simple call (never an invoke) to the given runtime function.
5404llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5405 ArrayRef<llvm::Value *> args,
5406 const llvm::Twine &name) {
5407 llvm::CallInst *call = Builder.CreateCall(
5408 Callee: callee, Args: args, OpBundles: getBundlesForFunclet(Callee: callee.getCallee()), Name: name);
5409 call->setCallingConv(getRuntimeCC());
5410
5411 if (CGM.shouldEmitConvergenceTokens() && call->isConvergent())
5412 return cast<llvm::CallInst>(Val: addConvergenceControlToken(Input: call));
5413 return call;
5414}
5415
5416llvm::CallInst *CodeGenFunction::EmitIntrinsicCall(llvm::Intrinsic::ID ID,
5417 const llvm::Twine &Name) {
5418 return EmitIntrinsicCall(ID, Types: {}, Args: {}, Name);
5419}
5420
5421llvm::CallInst *CodeGenFunction::EmitIntrinsicCall(llvm::Intrinsic::ID ID,
5422 ArrayRef<llvm::Value *> Args,
5423 const llvm::Twine &Name) {
5424 return EmitIntrinsicCall(ID, Types: {}, Args, Name);
5425}
5426
5427llvm::CallInst *CodeGenFunction::EmitIntrinsicCall(llvm::Intrinsic::ID ID,
5428 ArrayRef<llvm::Type *> Types,
5429 ArrayRef<llvm::Value *> Args,
5430 const llvm::Twine &Name) {
5431 llvm::Function *F =
5432 llvm::Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID, OverloadTys: Types);
5433 llvm::CallInst *Call =
5434 Builder.CreateCall(Callee: F, Args, OpBundles: getBundlesForFunclet(Callee: F), Name);
5435 if (CGM.shouldEmitConvergenceTokens() && Call->isConvergent())
5436 return cast<llvm::CallInst>(Val: addConvergenceControlToken(Input: Call));
5437 return Call;
5438}
5439
5440/// Emits a call or invoke to the given noreturn runtime function.
5441void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
5442 llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
5443 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5444 getBundlesForFunclet(Callee: callee.getCallee());
5445
5446 if (getInvokeDest()) {
5447 llvm::InvokeInst *invoke = Builder.CreateInvoke(
5448 Callee: callee, NormalDest: getUnreachableBlock(), UnwindDest: getInvokeDest(), Args: args, OpBundles: BundleList);
5449 invoke->setDoesNotReturn();
5450 invoke->setCallingConv(getRuntimeCC());
5451 } else {
5452 llvm::CallInst *call = Builder.CreateCall(Callee: callee, Args: args, OpBundles: BundleList);
5453 call->setDoesNotReturn();
5454 call->setCallingConv(getRuntimeCC());
5455 Builder.CreateUnreachable();
5456 }
5457}
5458
5459/// Emits a call or invoke instruction to the given nullary runtime function.
5460llvm::CallBase *
5461CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5462 const Twine &name) {
5463 return EmitRuntimeCallOrInvoke(callee, args: {}, name);
5464}
5465
5466/// Emits a call or invoke instruction to the given runtime function.
5467llvm::CallBase *
5468CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5469 ArrayRef<llvm::Value *> args,
5470 const Twine &name) {
5471 llvm::CallBase *call = EmitCallOrInvoke(Callee: callee, Args: args, Name: name);
5472 call->setCallingConv(getRuntimeCC());
5473 return call;
5474}
5475
5476/// Emits a call or invoke instruction to the given function, depending
5477/// on the current state of the EH stack.
5478llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
5479 ArrayRef<llvm::Value *> Args,
5480 const Twine &Name) {
5481 llvm::BasicBlock *InvokeDest = getInvokeDest();
5482 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5483 getBundlesForFunclet(Callee: Callee.getCallee());
5484
5485 llvm::CallBase *Inst;
5486 if (!InvokeDest)
5487 Inst = Builder.CreateCall(Callee, Args, OpBundles: BundleList, Name);
5488 else {
5489 llvm::BasicBlock *ContBB = createBasicBlock(name: "invoke.cont");
5490 Inst = Builder.CreateInvoke(Callee, NormalDest: ContBB, UnwindDest: InvokeDest, Args, OpBundles: BundleList,
5491 Name);
5492 EmitBlock(BB: ContBB);
5493 }
5494
5495 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5496 // optimizer it can aggressively ignore unwind edges.
5497 if (CGM.getLangOpts().ObjCAutoRefCount)
5498 AddObjCARCExceptionMetadata(Inst);
5499
5500 return Inst;
5501}
5502
5503void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
5504 llvm::Value *New) {
5505 DeferredReplacements.push_back(
5506 Elt: std::make_pair(x: llvm::WeakTrackingVH(Old), y&: New));
5507}
5508
5509namespace {
5510
5511/// Specify given \p NewAlign as the alignment of return value attribute. If
5512/// such attribute already exists, re-set it to the maximal one of two options.
5513[[nodiscard]] llvm::AttributeList
5514maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
5515 const llvm::AttributeList &Attrs,
5516 llvm::Align NewAlign) {
5517 llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
5518 if (CurAlign >= NewAlign)
5519 return Attrs;
5520 llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Context&: Ctx, Alignment: NewAlign);
5521 return Attrs.removeRetAttribute(C&: Ctx, Kind: llvm::Attribute::AttrKind::Alignment)
5522 .addRetAttribute(C&: Ctx, Attr: AlignAttr);
5523}
5524
5525template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
5526protected:
5527 CodeGenFunction &CGF;
5528
5529 /// We do nothing if this is, or becomes, nullptr.
5530 const AlignedAttrTy *AA = nullptr;
5531
5532 llvm::Value *Alignment = nullptr; // May or may not be a constant.
5533 llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
5534
5535 AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5536 : CGF(CGF_) {
5537 if (!FuncDecl)
5538 return;
5539 AA = FuncDecl->getAttr<AlignedAttrTy>();
5540 }
5541
5542public:
5543 /// If we can, materialize the alignment as an attribute on return value.
5544 [[nodiscard]] llvm::AttributeList
5545 TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
5546 if (!AA || OffsetCI || CGF.SanOpts.has(K: SanitizerKind::Alignment))
5547 return Attrs;
5548 const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Val: Alignment);
5549 if (!AlignmentCI)
5550 return Attrs;
5551 // We may legitimately have non-power-of-2 alignment here.
5552 // If so, this is UB land, emit it via `@llvm.assume` instead.
5553 if (!AlignmentCI->getValue().isPowerOf2())
5554 return Attrs;
5555 llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
5556 Ctx&: CGF.getLLVMContext(), Attrs,
5557 NewAlign: llvm::Align(
5558 AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment)));
5559 AA = nullptr; // We're done. Disallow doing anything else.
5560 return NewAttrs;
5561 }
5562
5563 /// Emit alignment assumption.
5564 /// This is a general fallback that we take if either there is an offset,
5565 /// or the alignment is variable or we are sanitizing for alignment.
5566 void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
5567 if (!AA)
5568 return;
5569 CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
5570 AA->getLocation(), Alignment, OffsetCI);
5571 AA = nullptr; // We're done. Disallow doing anything else.
5572 }
5573};
5574
5575/// Helper data structure to emit `AssumeAlignedAttr`.
5576class AssumeAlignedAttrEmitter final
5577 : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
5578public:
5579 AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5580 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5581 if (!AA)
5582 return;
5583 // It is guaranteed that the alignment/offset are constants.
5584 Alignment = cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: AA->getAlignment()));
5585 if (Expr *Offset = AA->getOffset()) {
5586 OffsetCI = cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: Offset));
5587 if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
5588 OffsetCI = nullptr;
5589 }
5590 }
5591};
5592
5593/// Helper data structure to emit `AllocAlignAttr`.
5594class AllocAlignAttrEmitter final
5595 : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
5596public:
5597 AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
5598 const CallArgList &CallArgs)
5599 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5600 if (!AA)
5601 return;
5602 // Alignment may or may not be a constant, and that is okay.
5603 Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
5604 .getRValue(CGF)
5605 .getScalarVal();
5606 }
5607};
5608
5609} // namespace
5610
5611static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
5612 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Ty))
5613 return VT->getPrimitiveSizeInBits().getKnownMinValue();
5614 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty))
5615 return getMaxVectorWidth(Ty: AT->getElementType());
5616
5617 unsigned MaxVectorWidth = 0;
5618 if (auto *ST = dyn_cast<llvm::StructType>(Val: Ty))
5619 for (auto *I : ST->elements())
5620 MaxVectorWidth = std::max(a: MaxVectorWidth, b: getMaxVectorWidth(Ty: I));
5621 return MaxVectorWidth;
5622}
5623
5624RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
5625 const CGCallee &Callee,
5626 ReturnValueSlot ReturnValue,
5627 const CallArgList &CallArgs,
5628 llvm::CallBase **callOrInvoke, bool IsMustTail,
5629 SourceLocation Loc,
5630 bool IsVirtualFunctionPointerThunk) {
5631 // FIXME: We no longer need the types from CallArgs; lift up and simplify.
5632
5633 assert(Callee.isOrdinary() || Callee.isVirtual());
5634
5635 // Handle struct-return functions by passing a pointer to the
5636 // location that we would like to return into.
5637 QualType RetTy = CallInfo.getReturnType();
5638 const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
5639
5640 llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(FI: CallInfo);
5641
5642 const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
5643 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
5644 // We can only guarantee that a function is called from the correct
5645 // context/function based on the appropriate target attributes,
5646 // so only check in the case where we have both always_inline and target
5647 // since otherwise we could be making a conditional call after a check for
5648 // the proper cpu features (and it won't cause code generation issues due to
5649 // function based code generation).
5650 if ((TargetDecl->hasAttr<AlwaysInlineAttr>() &&
5651 (TargetDecl->hasAttr<TargetAttr>() ||
5652 (CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>()))) ||
5653 (CurFuncDecl && CurFuncDecl->hasAttr<FlattenAttr>() &&
5654 (CurFuncDecl->hasAttr<TargetAttr>() ||
5655 TargetDecl->hasAttr<TargetAttr>())))
5656 checkTargetFeatures(Loc, TargetDecl: FD);
5657 }
5658
5659 // Some architectures (such as x86-64) have the ABI changed based on
5660 // attribute-target/features. Give them a chance to diagnose.
5661 const FunctionDecl *CallerDecl = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
5662 const FunctionDecl *CalleeDecl = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
5663 CGM.getTargetCodeGenInfo().checkFunctionCallABI(CGM, CallLoc: Loc, Caller: CallerDecl,
5664 Callee: CalleeDecl, Args: CallArgs, ReturnType: RetTy);
5665
5666 // 1. Set up the arguments.
5667
5668 // If we're using inalloca, insert the allocation after the stack save.
5669 // FIXME: Do this earlier rather than hacking it in here!
5670 RawAddress ArgMemory = RawAddress::invalid();
5671 if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
5672 const llvm::DataLayout &DL = CGM.getDataLayout();
5673 llvm::Instruction *IP = CallArgs.getStackBase();
5674 llvm::AllocaInst *AI;
5675 if (IP) {
5676 IP = IP->getNextNode();
5677 AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(), "argmem",
5678 IP->getIterator());
5679 } else {
5680 AI = CreateTempAlloca(Ty: ArgStruct, Name: "argmem");
5681 }
5682 auto Align = CallInfo.getArgStructAlignment();
5683 AI->setAlignment(Align.getAsAlign());
5684 AI->setUsedWithInAlloca(true);
5685 assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
5686 ArgMemory = RawAddress(AI, ArgStruct, Align);
5687 }
5688
5689 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
5690 SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
5691
5692 // If the call returns a temporary with struct return, create a temporary
5693 // alloca to hold the result, unless one is given to us.
5694 Address SRetPtr = Address::invalid();
5695 // Original alloca for lifetime markers
5696 Address SRetAlloca = Address::invalid();
5697 bool NeedSRetLifetimeEnd = false;
5698 if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
5699 // For virtual function pointer thunks and musttail calls, we must always
5700 // forward an incoming SRet pointer to the callee, because a local alloca
5701 // would be de-allocated before the call. These cases both guarantee that
5702 // there will be an incoming SRet argument of the correct type.
5703 if ((IsVirtualFunctionPointerThunk || IsMustTail) && RetAI.isIndirect()) {
5704 SRetPtr = makeNaturalAddressForPointer(Ptr: CurFn->arg_begin() +
5705 IRFunctionArgs.getSRetArgNo(),
5706 T: RetTy, Alignment: CharUnits::fromQuantity(Quantity: 1));
5707 } else if (!ReturnValue.isNull()) {
5708 SRetPtr = ReturnValue.getAddress();
5709 } else {
5710 SRetPtr = CreateMemTempWithoutCast(T: RetTy, Name: "tmp");
5711 if (HaveInsertPoint() && ReturnValue.isUnused()) {
5712 NeedSRetLifetimeEnd = EmitLifetimeStart(Addr: SRetPtr.getBasePointer());
5713 if (NeedSRetLifetimeEnd)
5714 SRetAlloca = SRetPtr;
5715 }
5716 }
5717 if (IRFunctionArgs.hasSRetArg()) {
5718 // A mismatch between the allocated return value's AS and the target's
5719 // chosen IndirectAS can happen e.g. when passing the this pointer through
5720 // a chain involving stores to / loads from the DefaultAS; we address this
5721 // here, symmetrically with the handling we have for normal pointer args.
5722 if (SRetPtr.getAddressSpace() != RetAI.getIndirectAddrSpace()) {
5723 llvm::Value *V = SRetPtr.getBasePointer();
5724 llvm::Type *Ty = llvm::PointerType::get(C&: getLLVMContext(),
5725 AddressSpace: RetAI.getIndirectAddrSpace());
5726
5727 SRetPtr = SRetPtr.withPointer(NewPointer: performAddrSpaceCast(Src: V, DestTy: Ty),
5728 IsKnownNonNull: SRetPtr.isKnownNonNull());
5729 }
5730 IRCallArgs[IRFunctionArgs.getSRetArgNo()] =
5731 getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy);
5732 } else if (RetAI.isInAlloca()) {
5733 Address Addr =
5734 Builder.CreateStructGEP(Addr: ArgMemory, Index: RetAI.getInAllocaFieldIndex());
5735 Builder.CreateStore(Val: getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy), Addr);
5736 }
5737 }
5738
5739 RawAddress swiftErrorTemp = RawAddress::invalid();
5740 Address swiftErrorArg = Address::invalid();
5741
5742 // When passing arguments using temporary allocas, we need to add the
5743 // appropriate lifetime markers. This vector keeps track of all the lifetime
5744 // markers that need to be ended right after the call.
5745 SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
5746
5747 // Translate all of the arguments as necessary to match the IR lowering.
5748 assert(CallInfo.arg_size() == CallArgs.size() &&
5749 "Mismatch between function signature & arguments.");
5750 unsigned ArgNo = 0;
5751 CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
5752 for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
5753 I != E; ++I, ++info_it, ++ArgNo) {
5754 const ABIArgInfo &ArgInfo = info_it->info;
5755
5756 // Insert a padding argument to ensure proper alignment.
5757 if (IRFunctionArgs.hasPaddingArg(ArgNo))
5758 IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
5759 llvm::UndefValue::get(T: ArgInfo.getPaddingType());
5760
5761 unsigned FirstIRArg, NumIRArgs;
5762 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
5763
5764 bool ArgHasMaybeUndefAttr =
5765 IsArgumentMaybeUndef(TargetDecl, NumRequiredArgs: CallInfo.getNumRequiredArgs(), ArgNo);
5766
5767 switch (ArgInfo.getKind()) {
5768 case ABIArgInfo::InAlloca: {
5769 assert(NumIRArgs == 0);
5770 assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
5771 if (I->isAggregate()) {
5772 RawAddress Addr = I->hasLValue()
5773 ? I->getKnownLValue().getAddress()
5774 : I->getKnownRValue().getAggregateAddress();
5775 llvm::Instruction *Placeholder =
5776 cast<llvm::Instruction>(Val: Addr.getPointer());
5777
5778 if (!ArgInfo.getInAllocaIndirect()) {
5779 // Replace the placeholder with the appropriate argument slot GEP.
5780 CGBuilderTy::InsertPoint IP = Builder.saveIP();
5781 Builder.SetInsertPoint(Placeholder);
5782 Addr = Builder.CreateStructGEP(Addr: ArgMemory,
5783 Index: ArgInfo.getInAllocaFieldIndex());
5784 Builder.restoreIP(IP);
5785 } else {
5786 // For indirect things such as overaligned structs, replace the
5787 // placeholder with a regular aggregate temporary alloca. Store the
5788 // address of this alloca into the struct.
5789 Addr =
5790 CreateMemTempWithoutCast(T: info_it->type, Name: "inalloca.indirect.tmp");
5791 Address ArgSlot = Builder.CreateStructGEP(
5792 Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5793 Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5794 }
5795 deferPlaceholderReplacement(Old: Placeholder, New: Addr.getPointer());
5796 } else if (ArgInfo.getInAllocaIndirect()) {
5797 // Make a temporary alloca and store the address of it into the argument
5798 // struct.
5799 RawAddress Addr = CreateMemTempWithoutCast(
5800 T: I->Ty, Align: getContext().getTypeAlignInChars(T: I->Ty),
5801 Name: "indirect-arg-temp");
5802 I->copyInto(CGF&: *this, Addr);
5803 Address ArgSlot =
5804 Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5805 Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5806 } else {
5807 // Store the RValue into the argument struct.
5808 Address Addr =
5809 Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5810 Addr = Addr.withElementType(ElemTy: ConvertTypeForMem(T: I->Ty));
5811 I->copyInto(CGF&: *this, Addr);
5812 }
5813 break;
5814 }
5815
5816 case ABIArgInfo::Indirect:
5817 case ABIArgInfo::IndirectAliased: {
5818 assert(NumIRArgs == 1);
5819 if (I->isAggregate()) {
5820 // We want to avoid creating an unnecessary temporary+copy here;
5821 // however, we need one in three cases:
5822 // 1. If the argument is not byval, and we are required to copy the
5823 // source. (This case doesn't occur on any common architecture.)
5824 // 2. If the argument is byval, RV is not sufficiently aligned, and
5825 // we cannot force it to be sufficiently aligned.
5826 // 3. If the argument is byval, but RV is not located in default
5827 // or alloca address space.
5828 Address Addr = I->hasLValue()
5829 ? I->getKnownLValue().getAddress()
5830 : I->getKnownRValue().getAggregateAddress();
5831 CharUnits Align = ArgInfo.getIndirectAlign();
5832 const llvm::DataLayout *TD = &CGM.getDataLayout();
5833
5834 assert((FirstIRArg >= IRFuncTy->getNumParams() ||
5835 IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
5836 TD->getAllocaAddrSpace()) &&
5837 "indirect argument must be in alloca address space");
5838
5839 bool NeedCopy = false;
5840 if (Addr.getAlignment() < Align &&
5841 llvm::getOrEnforceKnownAlignment(V: Addr.emitRawPointer(CGF&: *this),
5842 PrefAlign: Align.getAsAlign(),
5843 DL: *TD) < Align.getAsAlign()) {
5844 NeedCopy = true;
5845 } else if (I->hasLValue()) {
5846 auto LV = I->getKnownLValue();
5847
5848 bool isByValOrRef =
5849 ArgInfo.isIndirectAliased() || ArgInfo.getIndirectByVal();
5850
5851 if (!isByValOrRef ||
5852 (LV.getAlignment() < getContext().getTypeAlignInChars(T: I->Ty))) {
5853 NeedCopy = true;
5854 }
5855
5856 if (isByValOrRef && Addr.getType()->getAddressSpace() !=
5857 ArgInfo.getIndirectAddrSpace()) {
5858 NeedCopy = true;
5859 }
5860 }
5861
5862 if (!NeedCopy) {
5863 // Skip the extra memcpy call.
5864 llvm::Value *V = getAsNaturalPointerTo(Addr, PointeeType: I->Ty);
5865 auto *T = llvm::PointerType::get(C&: CGM.getLLVMContext(),
5866 AddressSpace: ArgInfo.getIndirectAddrSpace());
5867
5868 // FIXME: This should not depend on the language address spaces, and
5869 // only the contextual values. If the address space mismatches, see if
5870 // we can look through a cast to a compatible address space value,
5871 // otherwise emit a copy.
5872 llvm::Value *Val = performAddrSpaceCast(Src: V, DestTy: T);
5873 if (ArgHasMaybeUndefAttr)
5874 Val = Builder.CreateFreeze(V: Val);
5875 IRCallArgs[FirstIRArg] = Val;
5876 break;
5877 }
5878 } else if (I->getType()->isArrayParameterType()) {
5879 // Don't produce a temporary for ArrayParameterType arguments.
5880 // ArrayParameterType arguments are only created from
5881 // HLSL_ArrayRValue casts and HLSLOutArgExpr expressions, both
5882 // of which create temporaries already. This allows us to just use the
5883 // scalar for the decayed array pointer as the argument directly.
5884 IRCallArgs[FirstIRArg] = I->getKnownRValue().getScalarVal();
5885 break;
5886 }
5887
5888 // For non-aggregate args and aggregate args meeting conditions above
5889 // we need to create an aligned temporary, and copy to it.
5890 RawAddress AI = CreateMemTempWithoutCast(
5891 T: I->Ty, Align: ArgInfo.getIndirectAlign(), Name: "byval-temp");
5892 llvm::Value *Val = getAsNaturalPointerTo(Addr: AI, PointeeType: I->Ty);
5893 if (ArgHasMaybeUndefAttr)
5894 Val = Builder.CreateFreeze(V: Val);
5895 IRCallArgs[FirstIRArg] = Val;
5896
5897 // Emit lifetime markers for the temporary alloca and add cleanup code to
5898 // emit the end lifetime marker after the call.
5899 if (EmitLifetimeStart(Addr: AI.getPointer()))
5900 CallLifetimeEndAfterCall.emplace_back(Args&: AI);
5901
5902 // Generate the copy.
5903 I->copyInto(CGF&: *this, Addr: AI);
5904 break;
5905 }
5906
5907 case ABIArgInfo::Ignore:
5908 assert(NumIRArgs == 0);
5909 break;
5910
5911 case ABIArgInfo::Extend:
5912 case ABIArgInfo::Direct: {
5913 if (!isa<llvm::StructType>(Val: ArgInfo.getCoerceToType()) &&
5914 ArgInfo.getCoerceToType() == ConvertType(T: info_it->type) &&
5915 ArgInfo.getDirectOffset() == 0) {
5916 assert(NumIRArgs == 1);
5917 llvm::Value *V;
5918 if (!I->isAggregate())
5919 V = I->getKnownRValue().getScalarVal();
5920 else
5921 V = Builder.CreateLoad(
5922 Addr: I->hasLValue() ? I->getKnownLValue().getAddress()
5923 : I->getKnownRValue().getAggregateAddress());
5924
5925 // Implement swifterror by copying into a new swifterror argument.
5926 // We'll write back in the normal path out of the call.
5927 if (CallInfo.getExtParameterInfo(argIndex: ArgNo).getABI() ==
5928 ParameterABI::SwiftErrorResult) {
5929 assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
5930
5931 QualType pointeeTy = I->Ty->getPointeeType();
5932 swiftErrorArg = makeNaturalAddressForPointer(
5933 Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
5934
5935 swiftErrorTemp = CreateMemTempWithoutCast(
5936 T: pointeeTy, Align: getPointerAlign(), Name: "swifterror.temp");
5937 V = swiftErrorTemp.getPointer();
5938 cast<llvm::AllocaInst>(Val: V)->setSwiftError(true);
5939
5940 llvm::Value *errorValue = Builder.CreateLoad(Addr: swiftErrorArg);
5941 Builder.CreateStore(Val: errorValue, Addr: swiftErrorTemp);
5942 }
5943
5944 // We might have to widen integers, but we should never truncate.
5945 if (ArgInfo.getCoerceToType() != V->getType() &&
5946 V->getType()->isIntegerTy())
5947 V = Builder.CreateZExt(V, DestTy: ArgInfo.getCoerceToType());
5948
5949 // The only plausible mismatch here would be for pointer address spaces.
5950 // We assume that the target has a reasonable mapping for the DefaultAS
5951 // (it can be casted to from incoming specific ASes), and insert an AS
5952 // cast to address the mismatch.
5953 if (FirstIRArg < IRFuncTy->getNumParams() &&
5954 V->getType() != IRFuncTy->getParamType(i: FirstIRArg)) {
5955 assert(V->getType()->isPointerTy() && "Only pointers can mismatch!");
5956 V = performAddrSpaceCast(Src: V, DestTy: IRFuncTy->getParamType(i: FirstIRArg));
5957 }
5958
5959 if (ArgHasMaybeUndefAttr)
5960 V = Builder.CreateFreeze(V);
5961 IRCallArgs[FirstIRArg] = V;
5962 break;
5963 }
5964
5965 llvm::StructType *STy =
5966 dyn_cast<llvm::StructType>(Val: ArgInfo.getCoerceToType());
5967
5968 // FIXME: Avoid the conversion through memory if possible.
5969 Address Src = Address::invalid();
5970 if (!I->isAggregate()) {
5971 Src = CreateMemTempWithoutCast(T: I->Ty, Name: "coerce");
5972 I->copyInto(CGF&: *this, Addr: Src);
5973 } else {
5974 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
5975 : I->getKnownRValue().getAggregateAddress();
5976 }
5977
5978 // If the value is offset in memory, apply the offset now.
5979 Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
5980
5981 // Fast-isel and the optimizer generally like scalar values better than
5982 // FCAs, so we flatten them if this is safe to do for this argument.
5983 if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
5984 llvm::Type *SrcTy = Src.getElementType();
5985 llvm::TypeSize SrcTypeSize =
5986 CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
5987 llvm::TypeSize DstTypeSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
5988 if (SrcTypeSize.isScalable()) {
5989 assert(STy->containsHomogeneousScalableVectorTypes() &&
5990 "ABI only supports structure with homogeneous scalable vector "
5991 "type");
5992 assert(SrcTypeSize == DstTypeSize &&
5993 "Only allow non-fractional movement of structure with "
5994 "homogeneous scalable vector type");
5995 assert(NumIRArgs == STy->getNumElements());
5996
5997 llvm::Value *StoredStructValue =
5998 Builder.CreateLoad(Addr: Src, Name: Src.getName() + ".tuple");
5999 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
6000 llvm::Value *Extract = Builder.CreateExtractValue(
6001 Agg: StoredStructValue, Idxs: i, Name: Src.getName() + ".extract" + Twine(i));
6002 IRCallArgs[FirstIRArg + i] = Extract;
6003 }
6004 } else {
6005 uint64_t SrcSize = SrcTypeSize.getFixedValue();
6006 uint64_t DstSize = DstTypeSize.getFixedValue();
6007 bool HasPFPFields = getContext().hasPFPFields(Ty: I->Ty);
6008
6009 // If the source type is smaller than the destination type of the
6010 // coerce-to logic, copy the source value into a temp alloca the size
6011 // of the destination type to allow loading all of it. The bits past
6012 // the source value are left undef.
6013 if (HasPFPFields || SrcSize < DstSize) {
6014 Address TempAlloca = CreateTempAlloca(Ty: STy, align: Src.getAlignment(),
6015 Name: Src.getName() + ".coerce");
6016 if (HasPFPFields) {
6017 // Structures with PFP fields require a coerced load to remove any
6018 // pointer signatures.
6019 Builder.CreateStore(
6020 Val: CreatePFPCoercedLoad(Src, SrcFETy: I->Ty, Ty: ArgInfo.getCoerceToType(),
6021 CGF&: *this),
6022 Addr: TempAlloca);
6023 } else
6024 Builder.CreateMemCpy(Dest: TempAlloca, Src, Size: SrcSize);
6025 Src = TempAlloca;
6026 } else {
6027 Src = Src.withElementType(ElemTy: STy);
6028 }
6029
6030 assert(NumIRArgs == STy->getNumElements());
6031 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
6032 Address EltPtr = Builder.CreateStructGEP(Addr: Src, Index: i);
6033 llvm::Value *LI = Builder.CreateLoad(Addr: EltPtr);
6034 if (ArgHasMaybeUndefAttr)
6035 LI = Builder.CreateFreeze(V: LI);
6036 IRCallArgs[FirstIRArg + i] = LI;
6037 }
6038 }
6039 } else {
6040 // In the simple case, just pass the coerced loaded value.
6041 assert(NumIRArgs == 1);
6042 llvm::Value *Load =
6043 CreateCoercedLoad(Src, SrcFETy: I->Ty, Ty: ArgInfo.getCoerceToType(), CGF&: *this);
6044
6045 if (CallInfo.isCmseNSCall()) {
6046 // For certain parameter types, clear padding bits, as they may reveal
6047 // sensitive information.
6048 // Small struct/union types are passed as integer arrays.
6049 auto *ATy = dyn_cast<llvm::ArrayType>(Val: Load->getType());
6050 if (ATy != nullptr && isa<RecordType>(Val: I->Ty.getCanonicalType()))
6051 Load = EmitCMSEClearRecord(Src: Load, ATy, QTy: I->Ty);
6052 }
6053
6054 if (ArgHasMaybeUndefAttr)
6055 Load = Builder.CreateFreeze(V: Load);
6056 IRCallArgs[FirstIRArg] = Load;
6057 }
6058
6059 break;
6060 }
6061
6062 case ABIArgInfo::CoerceAndExpand: {
6063 auto coercionType = ArgInfo.getCoerceAndExpandType();
6064 auto layout = CGM.getDataLayout().getStructLayout(Ty: coercionType);
6065 auto unpaddedCoercionType = ArgInfo.getUnpaddedCoerceAndExpandType();
6066 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
6067
6068 Address addr = Address::invalid();
6069 RawAddress AllocaAddr = RawAddress::invalid();
6070 bool NeedLifetimeEnd = false;
6071 if (I->isAggregate()) {
6072 addr = I->hasLValue() ? I->getKnownLValue().getAddress()
6073 : I->getKnownRValue().getAggregateAddress();
6074
6075 } else {
6076 RValue RV = I->getKnownRValue();
6077 assert(RV.isScalar()); // complex should always just be direct
6078
6079 llvm::Type *scalarType = RV.getScalarVal()->getType();
6080 auto scalarAlign = CGM.getDataLayout().getPrefTypeAlign(Ty: scalarType);
6081
6082 // Materialize to a temporary.
6083 addr = CreateTempAlloca(Ty: RV.getScalarVal()->getType(),
6084 align: CharUnits::fromQuantity(Quantity: std::max(
6085 a: layout->getAlignment(), b: scalarAlign)),
6086 Name: "tmp",
6087 /*ArraySize=*/nullptr, Alloca: &AllocaAddr);
6088 NeedLifetimeEnd = EmitLifetimeStart(Addr: AllocaAddr.getPointer());
6089
6090 Builder.CreateStore(Val: RV.getScalarVal(), Addr: addr);
6091 }
6092
6093 addr = addr.withElementType(ElemTy: coercionType);
6094
6095 unsigned IRArgPos = FirstIRArg;
6096 unsigned unpaddedIndex = 0;
6097 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
6098 llvm::Type *eltType = coercionType->getElementType(N: i);
6099 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
6100 continue;
6101 Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
6102 llvm::Value *elt = CreateCoercedLoad(
6103 Src: eltAddr, SrcFETy: I->Ty,
6104 Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
6105 : unpaddedCoercionType,
6106 CGF&: *this);
6107 if (ArgHasMaybeUndefAttr)
6108 elt = Builder.CreateFreeze(V: elt);
6109 IRCallArgs[IRArgPos++] = elt;
6110 }
6111 assert(IRArgPos == FirstIRArg + NumIRArgs);
6112
6113 if (NeedLifetimeEnd)
6114 EmitLifetimeEnd(Addr: AllocaAddr.getPointer());
6115 break;
6116 }
6117
6118 case ABIArgInfo::Expand: {
6119 unsigned IRArgPos = FirstIRArg;
6120 ExpandTypeToArgs(Ty: I->Ty, Arg: *I, IRFuncTy, IRCallArgs, IRCallArgPos&: IRArgPos);
6121 assert(IRArgPos == FirstIRArg + NumIRArgs);
6122 break;
6123 }
6124
6125 case ABIArgInfo::TargetSpecific: {
6126 Address Src = Address::invalid();
6127 if (!I->isAggregate()) {
6128 Src = CreateMemTempWithoutCast(T: I->Ty, Name: "target_coerce");
6129 I->copyInto(CGF&: *this, Addr: Src);
6130 } else {
6131 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
6132 : I->getKnownRValue().getAggregateAddress();
6133 }
6134
6135 // If the value is offset in memory, apply the offset now.
6136 Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
6137 llvm::Value *Load =
6138 CGM.getABIInfo().createCoercedLoad(SrcAddr: Src, AI: ArgInfo, CGF&: *this);
6139 IRCallArgs[FirstIRArg] = Load;
6140 break;
6141 }
6142 }
6143 }
6144
6145 const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(CGF&: *this);
6146 llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
6147
6148 // If we're using inalloca, set up that argument.
6149 if (ArgMemory.isValid()) {
6150 llvm::Value *Arg = ArgMemory.getPointer();
6151 assert(IRFunctionArgs.hasInallocaArg());
6152 IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
6153 }
6154
6155 // 2. Prepare the function pointer.
6156
6157 // If the callee is a bitcast of a non-variadic function to have a
6158 // variadic function pointer type, check to see if we can remove the
6159 // bitcast. This comes up with unprototyped functions.
6160 //
6161 // This makes the IR nicer, but more importantly it ensures that we
6162 // can inline the function at -O0 if it is marked always_inline.
6163 auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
6164 llvm::Value *Ptr) -> llvm::Function * {
6165 if (!CalleeFT->isVarArg())
6166 return nullptr;
6167
6168 // Get underlying value if it's a bitcast
6169 if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Val: Ptr)) {
6170 if (CE->getOpcode() == llvm::Instruction::BitCast)
6171 Ptr = CE->getOperand(i_nocapture: 0);
6172 }
6173
6174 llvm::Function *OrigFn = dyn_cast<llvm::Function>(Val: Ptr);
6175 if (!OrigFn)
6176 return nullptr;
6177
6178 llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
6179
6180 // If the original type is variadic, or if any of the component types
6181 // disagree, we cannot remove the cast.
6182 if (OrigFT->isVarArg() ||
6183 OrigFT->getNumParams() != CalleeFT->getNumParams() ||
6184 OrigFT->getReturnType() != CalleeFT->getReturnType())
6185 return nullptr;
6186
6187 for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i)
6188 if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
6189 return nullptr;
6190
6191 return OrigFn;
6192 };
6193
6194 if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {
6195 CalleePtr = OrigFn;
6196 IRFuncTy = OrigFn->getFunctionType();
6197 }
6198
6199 // 3. Perform the actual call.
6200
6201 // Deactivate any cleanups that we're supposed to do immediately before
6202 // the call.
6203 if (!CallArgs.getCleanupsToDeactivate().empty())
6204 deactivateArgCleanupsBeforeCall(CGF&: *this, CallArgs);
6205
6206 // Update the largest vector width if any arguments have vector types.
6207 for (unsigned i = 0; i < IRCallArgs.size(); ++i)
6208 LargestVectorWidth = std::max(a: LargestVectorWidth,
6209 b: getMaxVectorWidth(Ty: IRCallArgs[i]->getType()));
6210
6211 // Compute the calling convention and attributes.
6212 unsigned CallingConv;
6213 llvm::AttributeList Attrs;
6214 CGM.ConstructAttributeList(Name: CalleePtr->getName(), FI: CallInfo,
6215 CalleeInfo: Callee.getAbstractInfo(), AttrList&: Attrs, CallingConv,
6216 /*AttrOnCallSite=*/true,
6217 /*IsThunk=*/false);
6218
6219 if (CallingConv == llvm::CallingConv::X86_VectorCall &&
6220 getTarget().getTriple().isWindowsArm64EC()) {
6221 CGM.Error(loc: Loc, error: "__vectorcall calling convention is not currently "
6222 "supported");
6223 }
6224
6225 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
6226 if (FD->hasAttr<StrictFPAttr>())
6227 // All calls within a strictfp function are marked strictfp
6228 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::StrictFP);
6229
6230 // If -ffast-math is enabled and the function is guarded by an
6231 // '__attribute__((optnone)) adjust the memory attribute so the BE emits the
6232 // library call instead of the intrinsic.
6233 if (FD->hasAttr<OptimizeNoneAttr>() && getLangOpts().FastMath)
6234 CGM.AdjustMemoryAttribute(Name: CalleePtr->getName(), CalleeInfo: Callee.getAbstractInfo(),
6235 Attrs);
6236 }
6237 // Add call-site nomerge attribute if exists.
6238 if (InNoMergeAttributedStmt)
6239 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoMerge);
6240
6241 // Add call-site noinline attribute if exists.
6242 if (InNoInlineAttributedStmt)
6243 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoInline);
6244
6245 // Add call-site always_inline attribute if exists.
6246 // Note: This corresponds to the [[clang::always_inline]] statement attribute.
6247 if (InAlwaysInlineAttributedStmt &&
6248 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
6249 Caller: CallerDecl, Callee: CalleeDecl))
6250 Attrs =
6251 Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::AlwaysInline);
6252
6253 // Remove call-site convergent attribute if requested.
6254 if (InNoConvergentAttributedStmt)
6255 Attrs =
6256 Attrs.removeFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::Convergent);
6257
6258 // Apply some call-site-specific attributes.
6259 // TODO: work this into building the attribute set.
6260
6261 // Apply always_inline to all calls within flatten functions.
6262 // FIXME: should this really take priority over __try, below?
6263 if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
6264 !InNoInlineAttributedStmt &&
6265 !(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>()) &&
6266 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
6267 Caller: CallerDecl, Callee: CalleeDecl)) {
6268 Attrs =
6269 Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::AlwaysInline);
6270 }
6271
6272 // Disable inlining inside SEH __try blocks.
6273 if (isSEHTryScope()) {
6274 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoInline);
6275 }
6276
6277 // Decide whether to use a call or an invoke.
6278 bool CannotThrow;
6279 if (currentFunctionUsesSEHTry()) {
6280 // SEH cares about asynchronous exceptions, so everything can "throw."
6281 CannotThrow = false;
6282 } else if (isCleanupPadScope() &&
6283 EHPersonality::get(CGF&: *this).isMSVCXXPersonality()) {
6284 // The MSVC++ personality will implicitly terminate the program if an
6285 // exception is thrown during a cleanup outside of a try/catch.
6286 // We don't need to model anything in IR to get this behavior.
6287 CannotThrow = true;
6288 } else {
6289 // Otherwise, nounwind call sites will never throw.
6290 CannotThrow = Attrs.hasFnAttr(Kind: llvm::Attribute::NoUnwind);
6291
6292 if (auto *FPtr = dyn_cast<llvm::Function>(Val: CalleePtr))
6293 if (FPtr->hasFnAttribute(Kind: llvm::Attribute::NoUnwind))
6294 CannotThrow = true;
6295 }
6296
6297 // If we made a temporary, be sure to clean up after ourselves. Note that we
6298 // can't depend on being inside of an ExprWithCleanups, so we need to manually
6299 // pop this cleanup later on. Being eager about this is OK, since this
6300 // temporary is 'invisible' outside of the callee.
6301 // Use the original alloca pointer (before any addrspacecast) for the
6302 // lifetime end marker, since lifetime intrinsics must reference the alloca
6303 // address space.
6304 if (NeedSRetLifetimeEnd)
6305 pushFullExprCleanup<CallLifetimeEnd>(kind: NormalEHLifetimeMarker, A: SRetAlloca);
6306
6307 llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();
6308
6309 SmallVector<llvm::OperandBundleDef, 1> BundleList =
6310 getBundlesForFunclet(Callee: CalleePtr);
6311
6312 if (SanOpts.has(K: SanitizerKind::KCFI) &&
6313 !isa_and_nonnull<FunctionDecl>(Val: TargetDecl))
6314 EmitKCFIOperandBundle(Callee: ConcreteCallee, Bundles&: BundleList);
6315
6316 // Add the pointer-authentication bundle.
6317 EmitPointerAuthOperandBundle(Info: ConcreteCallee.getPointerAuthInfo(), Bundles&: BundleList);
6318
6319 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl))
6320 if (FD->hasAttr<StrictFPAttr>())
6321 // All calls within a strictfp function are marked strictfp
6322 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::StrictFP);
6323
6324 AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
6325 Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
6326
6327 AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
6328 Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
6329
6330 // Emit the actual call/invoke instruction.
6331 llvm::CallBase *CI;
6332 if (!InvokeDest) {
6333 CI = Builder.CreateCall(FTy: IRFuncTy, Callee: CalleePtr, Args: IRCallArgs, OpBundles: BundleList);
6334 } else {
6335 llvm::BasicBlock *Cont = createBasicBlock(name: "invoke.cont");
6336 CI = Builder.CreateInvoke(Ty: IRFuncTy, Callee: CalleePtr, NormalDest: Cont, UnwindDest: InvokeDest, Args: IRCallArgs,
6337 OpBundles: BundleList);
6338 EmitBlock(BB: Cont);
6339 }
6340 if (CI->getCalledFunction() && CI->getCalledFunction()->hasName() &&
6341 CI->getCalledFunction()->getName().starts_with(Prefix: "_Z4sqrt")) {
6342 SetSqrtFPAccuracy(CI);
6343 }
6344 if (callOrInvoke) {
6345 *callOrInvoke = CI;
6346 if (CGM.getCodeGenOpts().CallGraphSection) {
6347 QualType CST;
6348 if (TargetDecl && TargetDecl->getFunctionType())
6349 CST = QualType(TargetDecl->getFunctionType(), 0);
6350 else if (const auto *FPT =
6351 Callee.getAbstractInfo().getCalleeFunctionProtoType())
6352 CST = QualType(FPT, 0);
6353 else
6354 llvm_unreachable(
6355 "Cannot find the callee type to generate callee_type metadata.");
6356
6357 // Set type identifier metadata of indirect calls for call graph section.
6358 if (!CST.isNull())
6359 CGM.createCalleeTypeMetadataForIcall(QT: CST, CB: *callOrInvoke);
6360 }
6361 }
6362
6363 // If this is within a function that has the guard(nocf) attribute and is an
6364 // indirect call, add the "guard_nocf" attribute to this call to indicate that
6365 // Control Flow Guard checks should not be added, even if the call is inlined.
6366 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
6367 if (const auto *A = FD->getAttr<CFGuardAttr>()) {
6368 if (A->getGuard() == CFGuardAttr::GuardArg::nocf &&
6369 !CI->getCalledFunction())
6370 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: "guard_nocf");
6371 }
6372 }
6373
6374 // Apply the attributes and calling convention.
6375 CI->setAttributes(Attrs);
6376 CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
6377
6378 // Apply various metadata.
6379
6380 if (!CI->getType()->isVoidTy())
6381 CI->setName("call");
6382
6383 if (CGM.shouldEmitConvergenceTokens() && CI->isConvergent())
6384 CI = addConvergenceControlToken(Input: CI);
6385
6386 // Update largest vector width from the return type.
6387 LargestVectorWidth =
6388 std::max(a: LargestVectorWidth, b: getMaxVectorWidth(Ty: CI->getType()));
6389
6390 // Insert instrumentation or attach profile metadata at indirect call sites.
6391 // For more details, see the comment before the definition of
6392 // IPVK_IndirectCallTarget in InstrProfData.inc.
6393 if (!CI->getCalledFunction())
6394 PGO->valueProfile(Builder, ValueKind: llvm::IPVK_IndirectCallTarget, ValueSite: CI, ValuePtr: CalleePtr);
6395
6396 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
6397 // optimizer it can aggressively ignore unwind edges.
6398 if (CGM.getLangOpts().ObjCAutoRefCount)
6399 AddObjCARCExceptionMetadata(Inst: CI);
6400
6401 // Set tail call kind if necessary.
6402 bool IsPPC = getTarget().getTriple().isPPC();
6403 bool IsMIPS = getTarget().getTriple().isMIPS();
6404 bool HasMips16 = false;
6405 if (IsMIPS) {
6406 const TargetOptions &TargetOpts = getTarget().getTargetOpts();
6407 HasMips16 = TargetOpts.FeatureMap.lookup(Key: "mips16");
6408 if (!HasMips16)
6409 HasMips16 = llvm::is_contained(Range: TargetOpts.Features, Element: "+mips16");
6410 }
6411 if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(Val: CI)) {
6412 if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
6413 Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
6414 else if (IsMustTail) {
6415 if (IsPPC) {
6416 if (getTarget().getTriple().isOSAIX())
6417 CGM.getDiags().Report(Loc, DiagID: diag::err_aix_musttail_unsupported);
6418 else if (!getTarget().hasFeature(Feature: "pcrelative-memops")) {
6419 if (getTarget().hasFeature(Feature: "longcall"))
6420 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail) << 0;
6421 else if (Call->isIndirectCall())
6422 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail) << 1;
6423 else if (isa_and_nonnull<FunctionDecl>(Val: TargetDecl)) {
6424 if (!cast<FunctionDecl>(Val: TargetDecl)->isDefined())
6425 // The undefined callee may be a forward declaration. Without
6426 // knowning all symbols in the module, we won't know the symbol is
6427 // defined or not. Collect all these symbols for later diagnosing.
6428 CGM.addUndefinedGlobalForTailCall(
6429 Global: {cast<FunctionDecl>(Val: TargetDecl), Loc});
6430 else {
6431 llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(
6432 GD: GlobalDecl(cast<FunctionDecl>(Val: TargetDecl)));
6433 if (llvm::GlobalValue::isWeakForLinker(Linkage) ||
6434 llvm::GlobalValue::isDiscardableIfUnused(Linkage))
6435 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail)
6436 << 2;
6437 }
6438 }
6439 }
6440 }
6441 if (IsMIPS) {
6442 if (HasMips16)
6443 CGM.getDiags().Report(Loc, DiagID: diag::err_mips_impossible_musttail) << 0;
6444 else if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl))
6445 CGM.addUndefinedGlobalForTailCall(Global: {FD, Loc});
6446 }
6447 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
6448 }
6449 }
6450
6451 // Add metadata for calls to MSAllocator functions
6452 if (getDebugInfo() && TargetDecl && TargetDecl->hasAttr<MSAllocatorAttr>())
6453 getDebugInfo()->addHeapAllocSiteMetadata(CallSite: CI, AllocatedTy: RetTy->getPointeeType(), Loc);
6454
6455 // Add srcloc metadata for [[gnu::error/warning]] diagnostics. When
6456 // ShowInliningChain is enabled, also track inline/static calls for the
6457 // heuristic fallback when debug info is not available. This heuristic is
6458 // conservative and best-effort since static or inline-annotated functions
6459 // are still not guaranteed to be inlined.
6460 if (TargetDecl) {
6461 bool NeedSrcLoc = TargetDecl->hasAttr<ErrorAttr>();
6462 if (!NeedSrcLoc && CGM.getCodeGenOpts().ShowInliningChain) {
6463 if (const auto *FD = dyn_cast<FunctionDecl>(Val: TargetDecl))
6464 NeedSrcLoc = FD->isInlined() || FD->hasAttr<AlwaysInlineAttr>() ||
6465 FD->getStorageClass() == SC_Static ||
6466 FD->isInAnonymousNamespace();
6467 }
6468 if (NeedSrcLoc) {
6469 auto *Line = llvm::ConstantInt::get(Ty: Int64Ty, V: Loc.getRawEncoding());
6470 auto *MD = llvm::ConstantAsMetadata::get(C: Line);
6471 CI->setMetadata(Kind: "srcloc", Node: llvm::MDNode::get(Context&: getLLVMContext(), MDs: {MD}));
6472 }
6473 }
6474
6475 // 4. Finish the call.
6476
6477 // If the call doesn't return, finish the basic block and clear the
6478 // insertion point; this allows the rest of IRGen to discard
6479 // unreachable code.
6480 if (CI->doesNotReturn()) {
6481 if (NeedSRetLifetimeEnd)
6482 PopCleanupBlock();
6483
6484 // Strip away the noreturn attribute to better diagnose unreachable UB.
6485 if (SanOpts.has(K: SanitizerKind::Unreachable)) {
6486 // Also remove from function since CallBase::hasFnAttr additionally checks
6487 // attributes of the called function.
6488 if (auto *F = CI->getCalledFunction())
6489 F->removeFnAttr(Kind: llvm::Attribute::NoReturn);
6490 CI->removeFnAttr(Kind: llvm::Attribute::NoReturn);
6491
6492 // Avoid incompatibility with ASan which relies on the `noreturn`
6493 // attribute to insert handler calls.
6494 if (SanOpts.hasOneOf(K: SanitizerKind::Address |
6495 SanitizerKind::KernelAddress)) {
6496 SanitizerScope SanScope(this);
6497 llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);
6498 Builder.SetInsertPoint(CI);
6499 auto *FnType = llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
6500 llvm::FunctionCallee Fn =
6501 CGM.CreateRuntimeFunction(Ty: FnType, Name: "__asan_handle_no_return");
6502 EmitNounwindRuntimeCall(callee: Fn);
6503 }
6504 }
6505
6506 EmitUnreachable(Loc);
6507 Builder.ClearInsertionPoint();
6508
6509 // FIXME: For now, emit a dummy basic block because expr emitters in
6510 // generally are not ready to handle emitting expressions at unreachable
6511 // points.
6512 EnsureInsertPoint();
6513
6514 // Return a reasonable RValue.
6515 return GetUndefRValue(Ty: RetTy);
6516 }
6517
6518 // If this is a musttail call, return immediately. We do not branch to the
6519 // epilogue in this case.
6520 if (IsMustTail) {
6521 for (auto it = EHStack.find(sp: CurrentCleanupScopeDepth); it != EHStack.end();
6522 ++it) {
6523 // A noexcept caller pushes an EHTerminateScope to call std::terminate()
6524 // if an exception escapes. A musttail call replaces the caller's frame,
6525 // removing this handler. This is safe if the callee is also nounwind:
6526 // the callee's own noexcept handler prevents any exception from reaching
6527 // where the caller's handler would have been.
6528 if (isa<EHTerminateScope>(Val: &*it)) {
6529 if (CI->doesNotThrow())
6530 continue;
6531 CGM.getDiags().Report(Loc: MustTailCall->getBeginLoc(),
6532 DiagID: diag::err_musttail_noexcept_mismatch);
6533 break;
6534 }
6535 EHCleanupScope *Cleanup = dyn_cast<EHCleanupScope>(Val: &*it);
6536 // Fake uses can be safely emitted immediately prior to the tail call, so
6537 // we choose to emit them just before the call here.
6538 if (Cleanup && Cleanup->isFakeUse()) {
6539 CGBuilderTy::InsertPointGuard IPG(Builder);
6540 Builder.SetInsertPoint(CI);
6541 Cleanup->getCleanup()->Emit(CGF&: *this, flags: EHScopeStack::Cleanup::Flags());
6542 } else if (!(Cleanup &&
6543 Cleanup->getCleanup()->isRedundantBeforeReturn())) {
6544 CGM.ErrorUnsupported(S: MustTailCall, Type: "tail call skipping over cleanups");
6545 }
6546 }
6547 if (CI->getType()->isVoidTy())
6548 Builder.CreateRetVoid();
6549 else
6550 Builder.CreateRet(V: CI);
6551 Builder.ClearInsertionPoint();
6552 EnsureInsertPoint();
6553 return GetUndefRValue(Ty: RetTy);
6554 }
6555
6556 // Perform the swifterror writeback.
6557 if (swiftErrorTemp.isValid()) {
6558 llvm::Value *errorResult = Builder.CreateLoad(Addr: swiftErrorTemp);
6559 Builder.CreateStore(Val: errorResult, Addr: swiftErrorArg);
6560 }
6561
6562 // Emit any call-associated writebacks immediately. Arguably this
6563 // should happen after any return-value munging.
6564 if (CallArgs.hasWritebacks())
6565 EmitWritebacks(args: CallArgs);
6566
6567 // The stack cleanup for inalloca arguments has to run out of the normal
6568 // lexical order, so deactivate it and run it manually here.
6569 CallArgs.freeArgumentMemory(CGF&: *this);
6570
6571 // Extract the return value.
6572 RValue Ret;
6573
6574 // If the current function is a virtual function pointer thunk, avoid copying
6575 // the return value of the musttail call to a temporary.
6576 if (IsVirtualFunctionPointerThunk) {
6577 Ret = RValue::get(V: CI);
6578 } else {
6579 Ret = [&] {
6580 switch (RetAI.getKind()) {
6581 case ABIArgInfo::CoerceAndExpand: {
6582 auto coercionType = RetAI.getCoerceAndExpandType();
6583
6584 Address addr = SRetPtr.withElementType(ElemTy: coercionType);
6585
6586 assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());
6587 bool requiresExtract = isa<llvm::StructType>(Val: CI->getType());
6588
6589 unsigned unpaddedIndex = 0;
6590 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
6591 llvm::Type *eltType = coercionType->getElementType(N: i);
6592 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
6593 continue;
6594 Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
6595 llvm::Value *elt = CI;
6596 if (requiresExtract)
6597 elt = Builder.CreateExtractValue(Agg: elt, Idxs: unpaddedIndex++);
6598 else
6599 assert(unpaddedIndex == 0);
6600 Builder.CreateStore(Val: elt, Addr: eltAddr);
6601 }
6602 [[fallthrough]];
6603 }
6604
6605 case ABIArgInfo::InAlloca:
6606 case ABIArgInfo::Indirect: {
6607 RValue ret = convertTempToRValue(addr: SRetPtr, type: RetTy, Loc: SourceLocation());
6608 if (NeedSRetLifetimeEnd)
6609 PopCleanupBlock();
6610 return ret;
6611 }
6612
6613 case ABIArgInfo::Ignore:
6614 // If we are ignoring an argument that had a result, make sure to
6615 // construct the appropriate return value for our caller.
6616 return GetUndefRValue(Ty: RetTy);
6617
6618 case ABIArgInfo::Extend:
6619 case ABIArgInfo::Direct: {
6620 llvm::Type *RetIRTy = ConvertType(T: RetTy);
6621 if (RetAI.getCoerceToType() == RetIRTy &&
6622 RetAI.getDirectOffset() == 0) {
6623 switch (getEvaluationKind(T: RetTy)) {
6624 case TEK_Complex: {
6625 llvm::Value *Real = Builder.CreateExtractValue(Agg: CI, Idxs: 0);
6626 llvm::Value *Imag = Builder.CreateExtractValue(Agg: CI, Idxs: 1);
6627 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
6628 }
6629 case TEK_Aggregate:
6630 break;
6631 case TEK_Scalar: {
6632 // If the argument doesn't match, perform a bitcast to coerce it.
6633 // This can happen due to trivial type mismatches.
6634 llvm::Value *V = CI;
6635 if (V->getType() != RetIRTy)
6636 V = Builder.CreateBitCast(V, DestTy: RetIRTy);
6637 return RValue::get(V);
6638 }
6639 }
6640 }
6641
6642 // If coercing a fixed vector from a scalable vector for ABI
6643 // compatibility, and the types match, use the llvm.vector.extract
6644 // intrinsic to perform the conversion.
6645 if (auto *FixedDstTy = dyn_cast<llvm::FixedVectorType>(Val: RetIRTy)) {
6646 llvm::Value *V = CI;
6647 if (auto *ScalableSrcTy =
6648 dyn_cast<llvm::ScalableVectorType>(Val: V->getType())) {
6649 if (FixedDstTy->getElementType() ==
6650 ScalableSrcTy->getElementType()) {
6651 V = Builder.CreateExtractVector(DstType: FixedDstTy, SrcVec: V, Idx: uint64_t(0),
6652 Name: "cast.fixed");
6653 return RValue::get(V);
6654 }
6655 }
6656 }
6657
6658 Address DestPtr = ReturnValue.getValue();
6659 bool DestIsVolatile = ReturnValue.isVolatile();
6660 uint64_t DestSize =
6661 getContext().getTypeInfoDataSizeInChars(T: RetTy).Width.getQuantity();
6662
6663 if (!DestPtr.isValid()) {
6664 DestPtr = CreateMemTempWithoutCast(T: RetTy, Name: "coerce");
6665 DestIsVolatile = false;
6666 DestSize = getContext().getTypeSizeInChars(T: RetTy).getQuantity();
6667 }
6668
6669 // An empty record can overlap other data (if declared with
6670 // no_unique_address); omit the store for such types - as there is no
6671 // actual data to store.
6672 if (!isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true)) {
6673 // If the value is offset in memory, apply the offset now.
6674 Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
6675 CreateCoercedStore(
6676 Src: CI, SrcFETy: RetTy, Dst: StorePtr,
6677 DstSize: llvm::TypeSize::getFixed(ExactSize: DestSize - RetAI.getDirectOffset()),
6678 DstIsVolatile: DestIsVolatile);
6679 }
6680
6681 return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation());
6682 }
6683
6684 case ABIArgInfo::TargetSpecific: {
6685 Address DestPtr = ReturnValue.getValue();
6686 Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
6687 bool DestIsVolatile = ReturnValue.isVolatile();
6688 if (!DestPtr.isValid()) {
6689 DestPtr = CreateMemTempWithoutCast(T: RetTy, Name: "target_coerce");
6690 DestIsVolatile = false;
6691 }
6692 CGM.getABIInfo().createCoercedStore(Val: CI, DstAddr: StorePtr, AI: RetAI, DestIsVolatile,
6693 CGF&: *this);
6694 return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation());
6695 }
6696
6697 case ABIArgInfo::Expand:
6698 case ABIArgInfo::IndirectAliased:
6699 llvm_unreachable("Invalid ABI kind for return argument");
6700 }
6701
6702 llvm_unreachable("Unhandled ABIArgInfo::Kind");
6703 }();
6704 }
6705
6706 // Emit the assume_aligned check on the return value.
6707 if (Ret.isScalar() && TargetDecl) {
6708 AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6709 AllocAlignAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6710 }
6711
6712 // Explicitly call CallLifetimeEnd::Emit just to re-use the code even though
6713 // we can't use the full cleanup mechanism.
6714 for (CallLifetimeEnd &LifetimeEnd : CallLifetimeEndAfterCall)
6715 LifetimeEnd.Emit(CGF&: *this, /*Flags=*/flags: {});
6716
6717 if (!ReturnValue.isExternallyDestructed() &&
6718 RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct)
6719 pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Ret.getAggregateAddress(),
6720 type: RetTy);
6721
6722 // Generate function declaration DISuprogram in order to be used
6723 // in debug info about call sites.
6724 if (CGDebugInfo *DI = getDebugInfo()) {
6725 // Ensure call site info would actually be emitted before collecting
6726 // further callee info.
6727 if (CalleeDecl && !CalleeDecl->hasAttr<NoDebugAttr>() &&
6728 DI->getCallSiteRelatedAttrs() != llvm::DINode::FlagZero) {
6729 CodeGenFunction CalleeCGF(CGM);
6730 const GlobalDecl &CalleeGlobalDecl =
6731 Callee.getAbstractInfo().getCalleeDecl();
6732 CalleeCGF.CurGD = CalleeGlobalDecl;
6733 FunctionArgList Args;
6734 QualType ResTy = CalleeCGF.BuildFunctionArgList(GD: CalleeGlobalDecl, Args);
6735 DI->EmitFuncDeclForCallSite(
6736 CallOrInvoke: CI, CalleeType: DI->getFunctionType(FD: CalleeDecl, RetTy: ResTy, Args), CalleeGlobalDecl);
6737 }
6738 // Generate call site target information.
6739 DI->addCallTargetIfVirtual(FD: CalleeDecl, CI);
6740 }
6741
6742 return Ret;
6743}
6744
6745CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const {
6746 if (isVirtual()) {
6747 const CallExpr *CE = getVirtualCallExpr();
6748 return CGF.CGM.getCXXABI().getVirtualFunctionPointer(
6749 CGF, GD: getVirtualMethodDecl(), This: getThisAddress(), Ty: getVirtualFunctionType(),
6750 Loc: CE ? CE->getBeginLoc() : SourceLocation());
6751 }
6752
6753 return *this;
6754}
6755
6756/* VarArg handling */
6757
6758RValue CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
6759 AggValueSlot Slot) {
6760 VAListAddr = VE->isMicrosoftABI()
6761 ? EmitMSVAListRef(E: VE->getSubExpr())
6762 : (VE->isZOSABI() ? EmitZOSVAListRef(E: VE->getSubExpr())
6763 : EmitVAListRef(E: VE->getSubExpr()));
6764 QualType Ty = VE->getType();
6765 if (Ty->isVariablyModifiedType())
6766 EmitVariablyModifiedType(Ty);
6767 if (VE->isMicrosoftABI())
6768 return CGM.getABIInfo().EmitMSVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6769 if (VE->isZOSABI())
6770 return CGM.getABIInfo().EmitZOSVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6771 return CGM.getABIInfo().EmitVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6772}
6773
6774DisableDebugLocationUpdates::DisableDebugLocationUpdates(CodeGenFunction &CGF)
6775 : CGF(CGF) {
6776 CGF.disableDebugInfo();
6777}
6778
6779DisableDebugLocationUpdates::~DisableDebugLocationUpdates() {
6780 CGF.enableDebugInfo();
6781}
6782