1//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// These classes wrap the information about a call or function
10// definition used to handle ABI compliancy.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CGCall.h"
15#include "ABIInfo.h"
16#include "ABIInfoImpl.h"
17#include "CGBlocks.h"
18#include "CGCXXABI.h"
19#include "CGCleanup.h"
20#include "CGDebugInfo.h"
21#include "CGRecordLayout.h"
22#include "CodeGenFunction.h"
23#include "CodeGenModule.h"
24#include "CodeGenPGO.h"
25#include "QualTypeMapper.h"
26#include "TargetInfo.h"
27#include "clang/AST/Attr.h"
28#include "clang/AST/Decl.h"
29#include "clang/AST/DeclCXX.h"
30#include "clang/AST/DeclObjC.h"
31#include "clang/AST/RecordLayout.h"
32#include "clang/Basic/CodeGenOptions.h"
33#include "clang/Basic/DiagnosticFrontend.h"
34#include "clang/Basic/TargetInfo.h"
35#include "clang/CodeGen/CGFunctionInfo.h"
36#include "clang/CodeGen/SwiftCallingConv.h"
37#include "llvm/ABI/FunctionInfo.h"
38#include "llvm/ABI/IRTypeMapper.h"
39#include "llvm/ABI/TargetInfo.h"
40#include "llvm/ABI/Types.h"
41#include "llvm/ADT/STLExtras.h"
42#include "llvm/ADT/StringExtras.h"
43#include "llvm/Analysis/ValueTracking.h"
44#include "llvm/IR/Assumptions.h"
45#include "llvm/IR/AttributeMask.h"
46#include "llvm/IR/Attributes.h"
47#include "llvm/IR/CallingConv.h"
48#include "llvm/IR/DataLayout.h"
49#include "llvm/IR/DebugInfoMetadata.h"
50#include "llvm/IR/InlineAsm.h"
51#include "llvm/IR/IntrinsicInst.h"
52#include "llvm/IR/Intrinsics.h"
53#include "llvm/IR/Type.h"
54#include "llvm/Transforms/Utils/Local.h"
55#include <optional>
56using namespace clang;
57using namespace CodeGen;
58
59/***/
60
61unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
62 switch (CC) {
63 case CC_C:
64 // On SPIR/SPIR-V, CC_C is the AST-level default calling convention, but
65 // it still needs to lower to spir_func so IR consumers can rely on the
66 // calling convention to distinguish device functions.
67 if (Target.getTriple().isSPIROrSPIRV())
68 return llvm::CallingConv::SPIR_FUNC;
69 return llvm::CallingConv::C;
70 case CC_X86StdCall:
71 return llvm::CallingConv::X86_StdCall;
72 case CC_X86FastCall:
73 return llvm::CallingConv::X86_FastCall;
74 case CC_X86RegCall:
75 return llvm::CallingConv::X86_RegCall;
76 case CC_X86ThisCall:
77 return llvm::CallingConv::X86_ThisCall;
78 case CC_Win64:
79 return llvm::CallingConv::Win64;
80 case CC_X86_64SysV:
81 return llvm::CallingConv::X86_64_SysV;
82 case CC_AAPCS:
83 return llvm::CallingConv::ARM_AAPCS;
84 case CC_AAPCS_VFP:
85 return llvm::CallingConv::ARM_AAPCS_VFP;
86 case CC_IntelOclBicc:
87 return llvm::CallingConv::Intel_OCL_BI;
88 // TODO: Add support for __pascal to LLVM.
89 case CC_X86Pascal:
90 return llvm::CallingConv::C;
91 // TODO: Add support for __vectorcall to LLVM.
92 case CC_X86VectorCall:
93 return llvm::CallingConv::X86_VectorCall;
94 case CC_AArch64VectorCall:
95 return llvm::CallingConv::AArch64_VectorCall;
96 case CC_AArch64SVEPCS:
97 return llvm::CallingConv::AArch64_SVE_VectorCall;
98 case CC_DeviceKernel:
99 return CGM.getTargetCodeGenInfo().getDeviceKernelCallingConv();
100 case CC_PreserveMost:
101 return llvm::CallingConv::PreserveMost;
102 case CC_PreserveAll:
103 return llvm::CallingConv::PreserveAll;
104 case CC_Swift:
105 return llvm::CallingConv::Swift;
106 case CC_SwiftAsync:
107 return llvm::CallingConv::SwiftTail;
108 case CC_M68kRTD:
109 return llvm::CallingConv::M68k_RTD;
110 case CC_PreserveNone:
111 return llvm::CallingConv::PreserveNone;
112 // clang-format off
113 case CC_RISCVVectorCall: return llvm::CallingConv::RISCV_VectorCall;
114 // clang-format on
115#define CC_VLS_CASE(ABI_VLEN) \
116 case CC_RISCVVLSCall_##ABI_VLEN: \
117 return llvm::CallingConv::RISCV_VLSCall_##ABI_VLEN;
118 CC_VLS_CASE(32)
119 CC_VLS_CASE(64)
120 CC_VLS_CASE(128)
121 CC_VLS_CASE(256)
122 CC_VLS_CASE(512)
123 CC_VLS_CASE(1024)
124 CC_VLS_CASE(2048)
125 CC_VLS_CASE(4096)
126 CC_VLS_CASE(8192)
127 CC_VLS_CASE(16384)
128 CC_VLS_CASE(32768)
129 CC_VLS_CASE(65536)
130#undef CC_VLS_CASE
131 }
132}
133
134/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
135/// qualification. Either or both of RD and MD may be null. A null RD indicates
136/// that there is no meaningful 'this' type, and a null MD can occur when
137/// calling a method pointer.
138CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
139 const CXXMethodDecl *MD) {
140 CanQualType RecTy;
141 if (RD)
142 RecTy = Context.getCanonicalTagType(TD: RD);
143 else
144 RecTy = Context.VoidTy;
145
146 if (MD)
147 RecTy = CanQualType::CreateUnsafe(Other: Context.getAddrSpaceQualType(
148 T: RecTy, AddressSpace: MD->getMethodQualifiers().getAddressSpace()));
149 return Context.getPointerType(T: RecTy);
150}
151
152/// Returns the canonical formal type of the given C++ method.
153static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
154 return MD->getType()
155 ->getCanonicalTypeUnqualified()
156 .getAs<FunctionProtoType>();
157}
158
159/// Returns the "extra-canonicalized" return type, which discards
160/// qualifiers on the return type. Codegen doesn't care about them,
161/// and it makes ABI code a little easier to be able to assume that
162/// all parameter and return types are top-level unqualified.
163static CanQualType GetReturnType(QualType RetTy) {
164 return RetTy->getCanonicalTypeUnqualified();
165}
166
167/// Arrange the argument and result information for a value of the given
168/// unprototyped freestanding function type.
169const CGFunctionInfo &
170CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
171 // When translating an unprototyped function type, always use a
172 // variadic type.
173 return arrangeLLVMFunctionInfo(returnType: FTNP->getReturnType().getUnqualifiedType(),
174 opts: FnInfoOpts::None, argTypes: {}, info: FTNP->getExtInfo(), paramInfos: {},
175 args: RequiredArgs(0), /*ABIInfoFD=*/nullptr);
176}
177
178static void addExtParameterInfosForCall(
179 llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
180 const FunctionProtoType *proto, unsigned prefixArgs, unsigned totalArgs) {
181 assert(proto->hasExtParameterInfos());
182 assert(paramInfos.size() <= prefixArgs);
183 assert(proto->getNumParams() + prefixArgs <= totalArgs);
184
185 paramInfos.reserve(N: totalArgs);
186
187 // Add default infos for any prefix args that don't already have infos.
188 paramInfos.resize(N: prefixArgs);
189
190 // Add infos for the prototype.
191 for (const auto &ParamInfo : proto->getExtParameterInfos()) {
192 paramInfos.push_back(Elt: ParamInfo);
193 // pass_object_size params have no parameter info.
194 if (ParamInfo.hasPassObjectSize())
195 paramInfos.emplace_back();
196 }
197
198 assert(paramInfos.size() <= totalArgs &&
199 "Did we forget to insert pass_object_size args?");
200 // Add default infos for the variadic and/or suffix arguments.
201 paramInfos.resize(N: totalArgs);
202}
203
204/// Adds the formal parameters in FPT to the given prefix. If any parameter in
205/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
206static void appendParameterTypes(
207 const CodeGenTypes &CGT, SmallVectorImpl<CanQualType> &prefix,
208 SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
209 CanQual<FunctionProtoType> FPT) {
210 // Fast path: don't touch param info if we don't need to.
211 if (!FPT->hasExtParameterInfos()) {
212 assert(paramInfos.empty() &&
213 "We have paramInfos, but the prototype doesn't?");
214 prefix.append(in_start: FPT->param_type_begin(), in_end: FPT->param_type_end());
215 return;
216 }
217
218 unsigned PrefixSize = prefix.size();
219 // In the vast majority of cases, we'll have precisely FPT->getNumParams()
220 // parameters; the only thing that can change this is the presence of
221 // pass_object_size. So, we preallocate for the common case.
222 prefix.reserve(N: prefix.size() + FPT->getNumParams());
223
224 auto ExtInfos = FPT->getExtParameterInfos();
225 assert(ExtInfos.size() == FPT->getNumParams());
226 for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
227 prefix.push_back(Elt: FPT->getParamType(i: I));
228 if (ExtInfos[I].hasPassObjectSize())
229 prefix.push_back(Elt: CGT.getContext().getCanonicalSizeType());
230 }
231
232 addExtParameterInfosForCall(paramInfos, proto: FPT.getTypePtr(), prefixArgs: PrefixSize,
233 totalArgs: prefix.size());
234}
235
236using ExtParameterInfoList =
237 SmallVector<FunctionProtoType::ExtParameterInfo, 16>;
238
239/// Arrange the LLVM function layout for a value of the given function
240/// type, on top of any implicit parameters already stored.
241static const CGFunctionInfo &
242arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
243 SmallVectorImpl<CanQualType> &prefix,
244 CanQual<FunctionProtoType> FTP) {
245 ExtParameterInfoList paramInfos;
246 RequiredArgs Required = RequiredArgs::forPrototypePlus(prototype: FTP, additional: prefix.size());
247 appendParameterTypes(CGT, prefix, paramInfos, FPT: FTP);
248 CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
249
250 FnInfoOpts opts =
251 instanceMethod ? FnInfoOpts::IsInstanceMethod : FnInfoOpts::None;
252 return CGT.arrangeLLVMFunctionInfo(returnType: resultType, opts, argTypes: prefix,
253 info: FTP->getExtInfo(), paramInfos, args: Required,
254 /*ABIInfoFD=*/nullptr);
255}
256
257using CanQualTypeList = SmallVector<CanQualType, 16>;
258
259/// Arrange the argument and result information for a value of the
260/// given freestanding function type.
261const CGFunctionInfo &
262CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
263 CanQualTypeList argTypes;
264 return ::arrangeLLVMFunctionInfo(CGT&: *this, /*instanceMethod=*/false, prefix&: argTypes,
265 FTP);
266}
267
268static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
269 bool IsTargetDefaultMSABI) {
270 // Set the appropriate calling convention for the Function.
271 if (D->hasAttr<StdCallAttr>())
272 return CC_X86StdCall;
273
274 if (D->hasAttr<FastCallAttr>())
275 return CC_X86FastCall;
276
277 if (D->hasAttr<RegCallAttr>())
278 return CC_X86RegCall;
279
280 if (D->hasAttr<ThisCallAttr>())
281 return CC_X86ThisCall;
282
283 if (D->hasAttr<VectorCallAttr>())
284 return CC_X86VectorCall;
285
286 if (D->hasAttr<PascalAttr>())
287 return CC_X86Pascal;
288
289 if (PcsAttr *PCS = D->getAttr<PcsAttr>())
290 return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
291
292 if (D->hasAttr<AArch64VectorPcsAttr>())
293 return CC_AArch64VectorCall;
294
295 if (D->hasAttr<AArch64SVEPcsAttr>())
296 return CC_AArch64SVEPCS;
297
298 if (D->hasAttr<DeviceKernelAttr>())
299 return CC_DeviceKernel;
300
301 if (D->hasAttr<IntelOclBiccAttr>())
302 return CC_IntelOclBicc;
303
304 if (D->hasAttr<MSABIAttr>())
305 return IsTargetDefaultMSABI ? CC_C : CC_Win64;
306
307 if (D->hasAttr<SysVABIAttr>())
308 return IsTargetDefaultMSABI ? CC_X86_64SysV : CC_C;
309
310 if (D->hasAttr<PreserveMostAttr>())
311 return CC_PreserveMost;
312
313 if (D->hasAttr<PreserveAllAttr>())
314 return CC_PreserveAll;
315
316 if (D->hasAttr<M68kRTDAttr>())
317 return CC_M68kRTD;
318
319 if (D->hasAttr<PreserveNoneAttr>())
320 return CC_PreserveNone;
321
322 if (D->hasAttr<RISCVVectorCCAttr>())
323 return CC_RISCVVectorCall;
324
325 if (RISCVVLSCCAttr *PCS = D->getAttr<RISCVVLSCCAttr>()) {
326 switch (PCS->getVectorWidth()) {
327 default:
328 llvm_unreachable("Invalid RISC-V VLS ABI VLEN");
329#define CC_VLS_CASE(ABI_VLEN) \
330 case ABI_VLEN: \
331 return CC_RISCVVLSCall_##ABI_VLEN;
332 CC_VLS_CASE(32)
333 CC_VLS_CASE(64)
334 CC_VLS_CASE(128)
335 CC_VLS_CASE(256)
336 CC_VLS_CASE(512)
337 CC_VLS_CASE(1024)
338 CC_VLS_CASE(2048)
339 CC_VLS_CASE(4096)
340 CC_VLS_CASE(8192)
341 CC_VLS_CASE(16384)
342 CC_VLS_CASE(32768)
343 CC_VLS_CASE(65536)
344#undef CC_VLS_CASE
345 }
346 }
347
348 return CC_C;
349}
350
351/// Arrange the argument and result information for a call to an
352/// unknown C++ non-static member function of the given abstract type.
353/// (A null RD means we don't have any meaningful "this" argument type,
354/// so fall back to a generic pointer type).
355/// The member function must be an ordinary function, i.e. not a
356/// constructor or destructor.
357const CGFunctionInfo &
358CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
359 const FunctionProtoType *FTP,
360 const CXXMethodDecl *MD) {
361 CanQualTypeList argTypes;
362
363 // Add the 'this' pointer.
364 argTypes.push_back(Elt: DeriveThisType(RD, MD));
365 auto CanonicalFTP =
366 FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>();
367 ExtParameterInfoList paramInfos;
368 RequiredArgs required = RequiredArgs::forPrototypePlus(
369 prototype: CanonicalFTP.getTypePtr(), additional: argTypes.size());
370 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: CanonicalFTP);
371 return arrangeLLVMFunctionInfo(
372 returnType: CanonicalFTP->getReturnType().getUnqualifiedType(),
373 opts: FnInfoOpts::IsInstanceMethod, argTypes, info: CanonicalFTP->getExtInfo(),
374 paramInfos, args: required, ABIInfoFD: MD);
375}
376
377/// Set calling convention for CUDA/HIP kernel.
378static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
379 const FunctionDecl *FD) {
380 if (FD->hasAttr<CUDAGlobalAttr>()) {
381 const FunctionType *FT = FTy->getAs<FunctionType>();
382 CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
383 FTy = FT->getCanonicalTypeUnqualified();
384 }
385}
386
387/// Arrange the argument and result information for a declaration or
388/// definition of the given C++ non-static member function. The
389/// member function must be an ordinary function, i.e. not a
390/// constructor or destructor.
391const CGFunctionInfo &
392CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
393 assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
394 assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
395
396 CanQualType FT = GetFormalType(MD).getAs<Type>();
397 setCUDAKernelCallingConvention(FTy&: FT, CGM, FD: MD);
398 auto prototype = FT.getAs<FunctionProtoType>();
399
400 if (MD->isImplicitObjectMemberFunction()) {
401 // The abstract case is perfectly fine.
402 const CXXRecordDecl *ThisType =
403 getCXXABI().getThisArgumentTypeForMethod(GD: MD);
404 return arrangeCXXMethodType(RD: ThisType, FTP: prototype.getTypePtr(), MD);
405 }
406
407 CanQualTypeList argTypes;
408 ExtParameterInfoList paramInfos;
409 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: prototype);
410 return arrangeLLVMFunctionInfo(
411 returnType: prototype->getReturnType().getUnqualifiedType(), opts: FnInfoOpts::None,
412 argTypes, info: prototype->getExtInfo(), paramInfos,
413 args: RequiredArgs::forPrototypePlus(prototype: prototype.getTypePtr(), additional: 0), ABIInfoFD: MD);
414}
415
416bool CodeGenTypes::inheritingCtorHasParams(
417 const InheritedConstructor &Inherited, CXXCtorType Type) {
418 // Parameters are unnecessary if we're constructing a base class subobject
419 // and the inherited constructor lives in a virtual base.
420 return Type == Ctor_Complete ||
421 !Inherited.getShadowDecl()->constructsVirtualBase() ||
422 !Target.getCXXABI().hasConstructorVariants();
423}
424
425const CGFunctionInfo &
426CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
427 auto *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
428
429 CanQualTypeList argTypes;
430 ExtParameterInfoList paramInfos;
431
432 const CXXRecordDecl *ThisType = getCXXABI().getThisArgumentTypeForMethod(GD);
433 argTypes.push_back(Elt: DeriveThisType(RD: ThisType, MD));
434
435 bool PassParams = true;
436
437 if (auto *CD = dyn_cast<CXXConstructorDecl>(Val: MD)) {
438 // A base class inheriting constructor doesn't get forwarded arguments
439 // needed to construct a virtual base (or base class thereof).
440 if (auto Inherited = CD->getInheritedConstructor())
441 PassParams = inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
442 }
443
444 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
445
446 // Add the formal parameters.
447 if (PassParams)
448 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: FTP);
449
450 CGCXXABI::AddedStructorArgCounts AddedArgs =
451 getCXXABI().buildStructorSignature(GD, ArgTys&: argTypes);
452 if (!paramInfos.empty()) {
453 // Note: prefix implies after the first param.
454 if (AddedArgs.Prefix)
455 paramInfos.insert(I: paramInfos.begin() + 1, NumToInsert: AddedArgs.Prefix,
456 Elt: FunctionProtoType::ExtParameterInfo{});
457 if (AddedArgs.Suffix)
458 paramInfos.append(NumInputs: AddedArgs.Suffix,
459 Elt: FunctionProtoType::ExtParameterInfo{});
460 }
461
462 RequiredArgs required =
463 (PassParams && MD->isVariadic() ? RequiredArgs(argTypes.size())
464 : RequiredArgs::All);
465
466 FunctionType::ExtInfo extInfo = FTP->getExtInfo();
467 CanQualType resultType = getCXXABI().HasThisReturn(GD) ? argTypes.front()
468 : getCXXABI().hasMostDerivedReturn(GD)
469 ? CGM.getContext().VoidPtrTy
470 : Context.VoidTy;
471 return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::IsInstanceMethod,
472 argTypes, info: extInfo, paramInfos, args: required, ABIInfoFD: MD);
473}
474
475static CanQualTypeList getArgTypesForCall(ASTContext &ctx,
476 const CallArgList &args) {
477 CanQualTypeList argTypes;
478 for (auto &arg : args)
479 argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg.Ty));
480 return argTypes;
481}
482
483static CanQualTypeList getArgTypesForDeclaration(ASTContext &ctx,
484 const FunctionArgList &args) {
485 CanQualTypeList argTypes;
486 for (auto &arg : args)
487 argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg->getType()));
488 return argTypes;
489}
490
491static ExtParameterInfoList
492getExtParameterInfosForCall(const FunctionProtoType *proto, unsigned prefixArgs,
493 unsigned totalArgs) {
494 ExtParameterInfoList result;
495 if (proto->hasExtParameterInfos()) {
496 addExtParameterInfosForCall(paramInfos&: result, proto, prefixArgs, totalArgs);
497 }
498 return result;
499}
500
501/// Arrange a call to a C++ method, passing the given arguments.
502///
503/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
504/// parameter.
505/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
506/// args.
507/// PassProtoArgs indicates whether `args` has args for the parameters in the
508/// given CXXConstructorDecl.
509const CGFunctionInfo &CodeGenTypes::arrangeCXXConstructorCall(
510 const CallArgList &args, const CXXConstructorDecl *D, CXXCtorType CtorKind,
511 unsigned ExtraPrefixArgs, unsigned ExtraSuffixArgs,
512 const FunctionDecl *ABIInfoFD, bool PassProtoArgs) {
513 CanQualTypeList ArgTypes;
514 for (const auto &Arg : args)
515 ArgTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
516
517 // +1 for implicit this, which should always be args[0].
518 unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
519
520 CanQual<FunctionProtoType> FPT = GetFormalType(MD: D);
521 RequiredArgs Required = PassProtoArgs
522 ? RequiredArgs::forPrototypePlus(
523 prototype: FPT, additional: TotalPrefixArgs + ExtraSuffixArgs)
524 : RequiredArgs::All;
525
526 GlobalDecl GD(D, CtorKind);
527 CanQualType ResultType = getCXXABI().HasThisReturn(GD) ? ArgTypes.front()
528 : getCXXABI().hasMostDerivedReturn(GD)
529 ? CGM.getContext().VoidPtrTy
530 : Context.VoidTy;
531
532 FunctionType::ExtInfo Info = FPT->getExtInfo();
533 ExtParameterInfoList ParamInfos;
534 // If the prototype args are elided, we should only have ABI-specific args,
535 // which never have param info.
536 if (PassProtoArgs && FPT->hasExtParameterInfos()) {
537 // ABI-specific suffix arguments are treated the same as variadic arguments.
538 addExtParameterInfosForCall(paramInfos&: ParamInfos, proto: FPT.getTypePtr(), prefixArgs: TotalPrefixArgs,
539 totalArgs: ArgTypes.size());
540 }
541
542 return arrangeLLVMFunctionInfo(returnType: ResultType, opts: FnInfoOpts::IsInstanceMethod,
543 argTypes: ArgTypes, info: Info, paramInfos: ParamInfos, args: Required,
544 ABIInfoFD);
545}
546
547/// Arrange the argument and result information for the declaration or
548/// definition of the given function.
549const CGFunctionInfo &
550CodeGenTypes::arrangeFunctionDeclaration(const GlobalDecl GD) {
551 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
552 if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD))
553 if (MD->isImplicitObjectMemberFunction())
554 return arrangeCXXMethodDeclaration(MD);
555
556 CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
557
558 assert(isa<FunctionType>(FTy));
559 setCUDAKernelCallingConvention(FTy, CGM, FD);
560
561 if (DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()) &&
562 GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
563 const FunctionType *FT = FTy->getAs<FunctionType>();
564 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
565 FTy = FT->getCanonicalTypeUnqualified();
566 }
567
568 // When declaring a function without a prototype, always use a
569 // non-variadic type.
570 if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
571 return arrangeLLVMFunctionInfo(returnType: noProto->getReturnType(), opts: FnInfoOpts::None,
572 argTypes: {}, info: noProto->getExtInfo(), paramInfos: {},
573 args: RequiredArgs::All, ABIInfoFD: FD);
574 }
575
576 CanQual<FunctionProtoType> FTP = FTy.castAs<FunctionProtoType>();
577 CanQualTypeList argTypes;
578 ExtParameterInfoList paramInfos;
579 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: FTP);
580 return arrangeLLVMFunctionInfo(returnType: FTP->getReturnType().getUnqualifiedType(),
581 opts: FnInfoOpts::None, argTypes, info: FTP->getExtInfo(),
582 paramInfos,
583 args: RequiredArgs::forPrototypePlus(prototype: FTP, additional: 0), ABIInfoFD: FD);
584}
585
586/// Arrange the argument and result information for the declaration or
587/// definition of an Objective-C method.
588const CGFunctionInfo &
589CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
590 // It happens that this is the same as a call with no optional
591 // arguments, except also using the formal 'self' type.
592 return arrangeObjCMessageSendSignature(MD, receiverType: MD->getSelfDecl()->getType());
593}
594
595/// Arrange the argument and result information for the function type
596/// through which to perform a send to the given Objective-C method,
597/// using the given receiver type. The receiver type is not always
598/// the 'self' type of the method or even an Objective-C pointer type.
599/// This is *not* the right method for actually performing such a
600/// message send, due to the possibility of optional arguments.
601const CGFunctionInfo &
602CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
603 QualType receiverType) {
604 CanQualTypeList argTys;
605 ExtParameterInfoList extParamInfos(MD->isDirectMethod() ? 1 : 2);
606 argTys.push_back(Elt: Context.getCanonicalParamType(T: receiverType));
607 if (!MD->isDirectMethod())
608 argTys.push_back(Elt: Context.getCanonicalParamType(T: Context.getObjCSelType()));
609 for (const auto *I : MD->parameters()) {
610 argTys.push_back(Elt: Context.getCanonicalParamType(T: I->getType()));
611 auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
612 NoEscape: I->hasAttr<NoEscapeAttr>());
613 extParamInfos.push_back(Elt: extParamInfo);
614 }
615
616 FunctionType::ExtInfo einfo;
617 bool IsTargetDefaultMSABI =
618 getContext().getTargetInfo().getTriple().isOSWindows() ||
619 getContext().getTargetInfo().getTriple().isUEFI();
620 einfo = einfo.withCallingConv(
621 cc: getCallingConventionForDecl(D: MD, IsTargetDefaultMSABI));
622
623 if (getContext().getLangOpts().ObjCAutoRefCount &&
624 MD->hasAttr<NSReturnsRetainedAttr>())
625 einfo = einfo.withProducesResult(producesResult: true);
626
627 RequiredArgs required =
628 (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);
629
630 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: MD->getReturnType()),
631 opts: FnInfoOpts::None, argTypes: argTys, info: einfo, paramInfos: extParamInfos,
632 args: required, /*ABIInfoFD=*/nullptr);
633}
634
635const CGFunctionInfo &
636CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
637 const CallArgList &args) {
638 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
639 FunctionType::ExtInfo einfo;
640
641 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: returnType), opts: FnInfoOpts::None,
642 argTypes, info: einfo, paramInfos: {}, args: RequiredArgs::All,
643 ABIInfoFD: nullptr);
644}
645
646const CGFunctionInfo &CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
647 // FIXME: Do we need to handle ObjCMethodDecl?
648 if (isa<CXXConstructorDecl>(Val: GD.getDecl()) ||
649 isa<CXXDestructorDecl>(Val: GD.getDecl()))
650 return arrangeCXXStructorDeclaration(GD);
651
652 return arrangeFunctionDeclaration(GD);
653}
654
655/// Arrange a thunk that takes 'this' as the first parameter followed by
656/// varargs. Return a void pointer, regardless of the actual return type.
657/// The body of the thunk will end in a musttail call to a function of the
658/// correct type, and the caller will bitcast the function to the correct
659/// prototype.
660const CGFunctionInfo &
661CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
662 assert(MD->isVirtual() && "only methods have thunks");
663 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
664 CanQualType ArgTys[] = {DeriveThisType(RD: MD->getParent(), MD)};
665 return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::None, argTypes: ArgTys,
666 info: FTP->getExtInfo(), paramInfos: {}, args: RequiredArgs(1), ABIInfoFD: MD);
667}
668
669const CGFunctionInfo &
670CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
671 CXXCtorType CT) {
672 assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
673
674 CanQual<FunctionProtoType> FTP = GetFormalType(MD: CD);
675 SmallVector<CanQualType, 2> ArgTys;
676 const CXXRecordDecl *RD = CD->getParent();
677 ArgTys.push_back(Elt: DeriveThisType(RD, MD: CD));
678 if (CT == Ctor_CopyingClosure)
679 ArgTys.push_back(Elt: *FTP->param_type_begin());
680 if (RD->getNumVBases() > 0)
681 ArgTys.push_back(Elt: Context.IntTy);
682 CallingConv CC = Context.getDefaultCallingConvention(
683 /*IsVariadic=*/false, /*IsCXXMethod=*/true);
684 return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::IsInstanceMethod,
685 argTypes: ArgTys, info: FunctionType::ExtInfo(CC), paramInfos: {},
686 args: RequiredArgs::All, /*ABIInfoFD=*/nullptr);
687}
688
689/// Arrange a call as unto a free function, except possibly with an
690/// additional number of formal parameters considered required.
691static const CGFunctionInfo &
692arrangeFreeFunctionLikeCall(CodeGenTypes &CGT, CodeGenModule &CGM,
693 const CallArgList &args, const FunctionType *fnType,
694 unsigned numExtraRequiredArgs, bool chainCall,
695 const FunctionDecl *ABIInfoFD) {
696 assert(args.size() >= numExtraRequiredArgs);
697
698 ExtParameterInfoList paramInfos;
699
700 // In most cases, there are no optional arguments.
701 RequiredArgs required = RequiredArgs::All;
702
703 // If we have a variadic prototype, the required arguments are the
704 // extra prefix plus the arguments in the prototype.
705 if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(Val: fnType)) {
706 if (proto->isVariadic())
707 required = RequiredArgs::forPrototypePlus(prototype: proto, additional: numExtraRequiredArgs);
708
709 if (proto->hasExtParameterInfos())
710 addExtParameterInfosForCall(paramInfos, proto, prefixArgs: numExtraRequiredArgs,
711 totalArgs: args.size());
712
713 // If we don't have a prototype at all, but we're supposed to
714 // explicitly use the variadic convention for unprototyped calls,
715 // treat all of the arguments as required but preserve the nominal
716 // possibility of variadics.
717 } else if (CGM.getTargetCodeGenInfo().isNoProtoCallVariadic(
718 args, fnType: cast<FunctionNoProtoType>(Val: fnType))) {
719 required = RequiredArgs(args.size());
720 }
721
722 CanQualTypeList argTypes;
723 for (const auto &arg : args)
724 argTypes.push_back(Elt: CGT.getContext().getCanonicalParamType(T: arg.Ty));
725 FnInfoOpts opts = chainCall ? FnInfoOpts::IsChainCall : FnInfoOpts::None;
726 return CGT.arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: fnType->getReturnType()),
727 opts, argTypes, info: fnType->getExtInfo(),
728 paramInfos, args: required, ABIInfoFD);
729}
730
731/// Figure out the rules for calling a function with the given formal
732/// type using the given arguments. The arguments are necessary
733/// because the function might be unprototyped, in which case it's
734/// target-dependent in crazy ways.
735const CGFunctionInfo &CodeGenTypes::arrangeFreeFunctionCall(
736 const CallArgList &args, const FunctionType *fnType, bool chainCall,
737 const FunctionDecl *ABIInfoFD) {
738 return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType,
739 numExtraRequiredArgs: chainCall ? 1 : 0, chainCall, ABIInfoFD);
740}
741
742/// A block function is essentially a free function with an
743/// extra implicit argument.
744const CGFunctionInfo &
745CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
746 const FunctionType *fnType) {
747 // FIXME: Pass the enclosing function's ABI information so block calls use
748 // the caller's target features.
749 return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType, numExtraRequiredArgs: 1,
750 /*chainCall=*/false, ABIInfoFD: nullptr);
751}
752
753const CGFunctionInfo &
754CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
755 const FunctionArgList &params) {
756 ExtParameterInfoList paramInfos =
757 getExtParameterInfosForCall(proto, prefixArgs: 1, totalArgs: params.size());
758 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args: params);
759
760 // FIXME: Use the block's target features when arranging its invoke function.
761 return arrangeLLVMFunctionInfo(
762 returnType: GetReturnType(RetTy: proto->getReturnType()), opts: FnInfoOpts::None, argTypes,
763 info: proto->getExtInfo(), paramInfos, args: RequiredArgs::forPrototypePlus(prototype: proto, additional: 1),
764 /*ABIInfoFD=*/nullptr);
765}
766
767const CGFunctionInfo &
768CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
769 const CallArgList &args) {
770 CanQualTypeList argTypes;
771 for (const auto &Arg : args)
772 argTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
773 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
774 argTypes, info: FunctionType::ExtInfo(),
775 /*paramInfos=*/{}, args: RequiredArgs::All, ABIInfoFD: nullptr);
776}
777
778const CGFunctionInfo &
779CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
780 const FunctionArgList &args) {
781 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
782
783 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
784 argTypes, info: FunctionType::ExtInfo(), paramInfos: {},
785 args: RequiredArgs::All, /*ABIInfoFD=*/nullptr);
786}
787
788const CGFunctionInfo &CodeGenTypes::arrangeBuiltinFunctionDeclaration(
789 CanQualType resultType, ArrayRef<CanQualType> argTypes) {
790 return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::None, argTypes,
791 info: FunctionType::ExtInfo(), paramInfos: {}, args: RequiredArgs::All,
792 /*ABIInfoFD=*/nullptr);
793}
794
795const CGFunctionInfo &CodeGenTypes::arrangeDeviceKernelCallerDeclaration(
796 QualType resultType, const FunctionArgList &args) {
797 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
798
799 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
800 argTypes,
801 info: FunctionType::ExtInfo(CC_DeviceKernel),
802 /*paramInfos=*/{}, args: RequiredArgs::All,
803 /*ABIInfoFD=*/nullptr);
804}
805
806/// Arrange a call to a C++ method, passing the given arguments.
807///
808/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
809/// does not count `this`.
810const CGFunctionInfo &CodeGenTypes::arrangeCXXMethodCall(
811 const CallArgList &args, const FunctionProtoType *proto,
812 RequiredArgs required, unsigned numPrefixArgs,
813 const FunctionDecl *ABIInfoFD) {
814 assert(numPrefixArgs + 1 <= args.size() &&
815 "Emitting a call with less args than the required prefix?");
816 // Add one to account for `this`. It's a bit awkward here, but we don't count
817 // `this` in similar places elsewhere.
818 ExtParameterInfoList paramInfos =
819 getExtParameterInfosForCall(proto, prefixArgs: numPrefixArgs + 1, totalArgs: args.size());
820
821 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
822
823 FunctionType::ExtInfo info = proto->getExtInfo();
824 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: proto->getReturnType()),
825 opts: FnInfoOpts::IsInstanceMethod, argTypes, info,
826 paramInfos, args: required, ABIInfoFD);
827}
828
829const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
830 return arrangeLLVMFunctionInfo(returnType: getContext().VoidTy, opts: FnInfoOpts::None, argTypes: {},
831 info: FunctionType::ExtInfo(), paramInfos: {}, args: RequiredArgs::All,
832 /*ABIInfoFD=*/nullptr);
833}
834
835const CGFunctionInfo &CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
836 const CallArgList &args,
837 const FunctionDecl *ABIInfoFD) {
838 assert(signature.arg_size() <= args.size());
839 unsigned X86ABIAVXLevel =
840 CGM.getABIInfo().getX86ABIAVXLevel(ABIInfoFD, signature.getExtInfo());
841 if (signature.arg_size() == args.size() &&
842 signature.getX86ABIAVXLevel() == X86ABIAVXLevel)
843 return signature;
844
845 ExtParameterInfoList paramInfos;
846 auto sigParamInfos = signature.getExtParameterInfos();
847 if (!sigParamInfos.empty()) {
848 paramInfos.append(in_start: sigParamInfos.begin(), in_end: sigParamInfos.end());
849 paramInfos.resize(N: args.size());
850 }
851
852 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
853
854 assert(signature.getRequiredArgs().allowsOptionalArgs());
855 FnInfoOpts opts = FnInfoOpts::None;
856 if (signature.isInstanceMethod())
857 opts |= FnInfoOpts::IsInstanceMethod;
858 if (signature.isChainCall())
859 opts |= FnInfoOpts::IsChainCall;
860 if (signature.isDelegateCall())
861 opts |= FnInfoOpts::IsDelegateCall;
862
863 const CGFunctionInfo *newFI = findOrInsertCGFunctionInfo(
864 isInstanceMethod: signature.isInstanceMethod(), isChainCall: signature.isChainCall(),
865 isDelegateCall: signature.isDelegateCall(), X86ABIAVXLevel, info: signature.getExtInfo(),
866 paramInfos, required: signature.getRequiredArgs(), resultType: signature.getReturnType(),
867 argTypes);
868 return *newFI;
869}
870
871namespace clang {
872namespace CodeGen {
873void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
874} // namespace CodeGen
875} // namespace clang
876
877#ifndef NDEBUG
878static const char *abiKindToString(ABIArgInfo::Kind K) {
879 switch (K) {
880 case ABIArgInfo::Direct:
881 return "Direct";
882 case ABIArgInfo::Extend:
883 return "Extend";
884 case ABIArgInfo::Indirect:
885 return "Indirect";
886 case ABIArgInfo::IndirectAliased:
887 return "IndirectAliased";
888 case ABIArgInfo::Ignore:
889 return "Ignore";
890 case ABIArgInfo::Expand:
891 return "Expand";
892 case ABIArgInfo::CoerceAndExpand:
893 return "CoerceAndExpand";
894 case ABIArgInfo::TargetSpecific:
895 return "TargetSpecific";
896 case ABIArgInfo::InAlloca:
897 return "InAlloca";
898 }
899 llvm_unreachable("Unknown kind");
900}
901#endif
902
903void CodeGenModule::computeABIInfoUsingLib(CGFunctionInfo &FI) {
904 SmallVector<const llvm::abi::Type *> MappedArgTypes;
905 MappedArgTypes.reserve(N: FI.arg_size());
906 for (const auto &Arg : FI.arguments())
907 MappedArgTypes.push_back(Elt: AbiMapper->convertType(QT: Arg.type));
908
909 RequiredArgs Required = FI.getRequiredArgs();
910 llvm::abi::RequiredArgs AbiRequired = llvm::abi::RequiredArgs::All;
911 if (Required.allowsOptionalArgs())
912 AbiRequired = llvm::abi::RequiredArgs(Required.getNumRequiredArgs());
913
914 auto AbiFI = llvm::abi::FunctionInfo::create(
915 CC: FI.getCallingConvention(), ReturnType: AbiMapper->convertType(QT: FI.getReturnType()),
916 ArgTypes: MappedArgTypes, Required: AbiRequired);
917
918 getLLVMABITargetInfo(TB&: AbiMapper->getTypeBuilder()).computeInfo(FI&: *AbiFI);
919
920#ifndef NDEBUG
921 // With assertions enabled, also compute info using Clang ABI logic,
922 // so we can ensure the results are consistent.
923 getABIInfo().computeInfo(FI);
924
925 auto ConvertABIArgInfo = [&](ABIArgInfo &Target,
926 const llvm::abi::ArgInfo &AbiInfo, QualType Type,
927 int ArgNo) {
928 auto Check = [&](bool Cond, llvm::function_ref<void()> MessageFn) {
929 if (Cond)
930 return;
931 if (ArgNo == -1)
932 llvm::dbgs() << "For return value of type ";
933 else
934 llvm::dbgs() << "For argument " << ArgNo << " of type ";
935 llvm::dbgs() << Type << ": ";
936 MessageFn();
937 llvm::dbgs() << "\n";
938 abort();
939 };
940 auto CheckSimple = [&](auto TargetVal, auto ResVal, StringRef What) {
941 Check(TargetVal == ResVal, [&]() {
942 llvm::dbgs() << What << " mismatch (expected: " << TargetVal
943 << ", given: " << ResVal << ")";
944 });
945 };
946
947 ABIArgInfo Res = convertABIArgInfo(AbiInfo, Type);
948 Check(Target.getKind() == Res.getKind(), [&]() {
949 llvm::dbgs() << "Kind mismatch (expected: "
950 << abiKindToString(Target.getKind())
951 << ", given: " << abiKindToString(Res.getKind()) << ")";
952 });
953
954 if (Res.canHaveCoerceToType()) {
955 // Normalize nullptr types.
956 llvm::Type *TargetType = Target.getCoerceToType();
957 llvm::Type *ResType = Res.getCoerceToType();
958 if (!TargetType)
959 TargetType = getTypes().ConvertType(Type);
960 if (!ResType)
961 ResType = getTypes().ConvertType(Type);
962
963 Check(TargetType == ResType, [&]() {
964 llvm::dbgs() << "CoerceToType mismatch (expected: " << *TargetType
965 << ", given: " << *ResType << ")";
966 });
967 }
968
969 switch (Res.getKind()) {
970 case ABIArgInfo::Extend:
971 CheckSimple(Target.isSignExt(), Res.isSignExt(), "SignExt");
972 CheckSimple(Target.isZeroExt(), Res.isZeroExt(), "ZeroExt");
973 [[fallthrough]];
974 case ABIArgInfo::Direct:
975 CheckSimple(Target.getDirectAlign(), Res.getDirectAlign(), "DirectAlign");
976 CheckSimple(Target.getDirectOffset(), Res.getDirectOffset(),
977 "DirectOffset");
978 break;
979 case ABIArgInfo::Indirect:
980 CheckSimple(Target.getIndirectByVal(), Res.getIndirectByVal(),
981 "IndirectByVal");
982 [[fallthrough]];
983 case ABIArgInfo::IndirectAliased:
984 CheckSimple(Target.getIndirectAddrSpace(), Res.getIndirectAddrSpace(),
985 "IndirectAddrSpace");
986 CheckSimple(Target.getIndirectRealign(), Res.getIndirectRealign(),
987 "IndirectRealign");
988 Check(Target.getIndirectAlign() == Res.getIndirectAlign(), [&]() {
989 llvm::dbgs() << "IndirectAlign mismatch (expected: "
990 << Target.getIndirectAlign().getQuantity()
991 << ", given: " << Res.getIndirectAlign().getQuantity()
992 << ")";
993 });
994 break;
995 default:
996 break;
997 }
998
999 Target = Res;
1000 };
1001#else
1002 auto ConvertABIArgInfo =
1003 [&](ABIArgInfo &Target, const llvm::abi::ArgInfo &AbiInfo, QualType Type,
1004 int ArgNo) { Target = convertABIArgInfo(AbiInfo, Type); };
1005#endif
1006
1007 ConvertABIArgInfo(FI.getReturnInfo(), AbiFI->getReturnInfo(),
1008 FI.getReturnType(), -1);
1009
1010 int ArgNo = 0;
1011 for (auto [CGArg, AbiArg] :
1012 llvm::zip_equal(t: FI.arguments(), u: AbiFI->arguments()))
1013 ConvertABIArgInfo(CGArg.info, AbiArg.Info, CGArg.type, ArgNo++);
1014}
1015
1016ABIArgInfo CodeGenModule::convertABIArgInfo(const llvm::abi::ArgInfo &AbiInfo,
1017 QualType Type) {
1018 switch (AbiInfo.getKind()) {
1019 case llvm::abi::ArgInfo::Direct: {
1020 llvm::Type *CoercedType = nullptr;
1021 if (AbiInfo.getCoerceToType())
1022 CoercedType = AbiReverseMapper->convertType(ABIType: AbiInfo.getCoerceToType());
1023 if (!CoercedType)
1024 CoercedType = getTypes().ConvertType(T: Type);
1025 return ABIArgInfo::getDirect(T: CoercedType, Offset: AbiInfo.getDirectOffset());
1026 }
1027 case llvm::abi::ArgInfo::Extend: {
1028 llvm::Type *CoercedType = nullptr;
1029 if (AbiInfo.getCoerceToType())
1030 CoercedType = AbiReverseMapper->convertType(ABIType: AbiInfo.getCoerceToType());
1031 if (!CoercedType)
1032 CoercedType = getTypes().ConvertType(T: Type);
1033 // A transparent union is passed as its first field, so the extend keys off
1034 // that field's integral type, matching the classifier's
1035 // useFirstFieldIfTransparentUnion. Passing the union type to
1036 // ABIArgInfo::getSignExtend would trip its integral-type assert.
1037 QualType ExtendType = useFirstFieldIfTransparentUnion(Ty: Type);
1038 if (AbiInfo.isSignExt())
1039 return ABIArgInfo::getSignExtend(Ty: ExtendType, T: CoercedType);
1040 if (AbiInfo.isZeroExt())
1041 return ABIArgInfo::getZeroExtend(Ty: ExtendType, T: CoercedType);
1042 return ABIArgInfo::getExtend(Ty: ExtendType, T: CoercedType);
1043 }
1044 case llvm::abi::ArgInfo::Indirect: {
1045 CharUnits Alignment =
1046 CharUnits::fromQuantity(Quantity: AbiInfo.getIndirectAlign().value());
1047 return ABIArgInfo::getIndirect(Alignment, AddrSpace: AbiInfo.getIndirectAddrSpace(),
1048 ByVal: AbiInfo.getIndirectByVal(),
1049 Realign: AbiInfo.getIndirectRealign());
1050 }
1051 case llvm::abi::ArgInfo::Ignore:
1052 return ABIArgInfo::getIgnore();
1053 }
1054 llvm_unreachable("Unexpected llvm::abi::ArgInfo kind");
1055}
1056
1057/// Arrange the argument and result information for an abstract value
1058/// of a given function type. This is the method which all of the
1059/// above functions ultimately defer to.
1060const CGFunctionInfo &CodeGenTypes::arrangeLLVMFunctionInfo(
1061 CanQualType resultType, FnInfoOpts opts, ArrayRef<CanQualType> argTypes,
1062 FunctionType::ExtInfo info,
1063 ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
1064 RequiredArgs required, const FunctionDecl *ABIInfoFD) {
1065 assert(llvm::all_of(argTypes,
1066 [](CanQualType T) { return T.isCanonicalAsParam(); }));
1067
1068 // Lookup or create unique function info.
1069 llvm::FoldingSetNodeID ID;
1070 bool isInstanceMethod =
1071 (opts & FnInfoOpts::IsInstanceMethod) == FnInfoOpts::IsInstanceMethod;
1072 bool isChainCall =
1073 (opts & FnInfoOpts::IsChainCall) == FnInfoOpts::IsChainCall;
1074 bool isDelegateCall =
1075 (opts & FnInfoOpts::IsDelegateCall) == FnInfoOpts::IsDelegateCall;
1076 unsigned X86ABIAVXLevel = CGM.getABIInfo().getX86ABIAVXLevel(ABIInfoFD, info);
1077
1078 const CGFunctionInfo *newFI = findOrInsertCGFunctionInfo(
1079 isInstanceMethod, isChainCall, isDelegateCall, X86ABIAVXLevel, info,
1080 paramInfos, required, resultType, argTypes);
1081 return *newFI;
1082}
1083
1084CGFunctionInfo *CodeGenTypes::findOrInsertCGFunctionInfo(
1085 bool isInstanceMethod, bool isChainCall, bool isDelegateCall,
1086 unsigned X86ABIAVXLevel, const FunctionType::ExtInfo &info,
1087 ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
1088 RequiredArgs required, CanQualType resultType,
1089 ArrayRef<CanQualType> argTypes) {
1090 llvm::FoldingSetNodeID ID;
1091 CGFunctionInfo::Profile(ID, InstanceMethod: isInstanceMethod, ChainCall: isChainCall, IsDelegateCall: isDelegateCall,
1092 X86ABIAVXLevel, info, paramInfos, required,
1093 resultType, argTypes);
1094
1095 void *insertPos = nullptr;
1096 CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, InsertPos&: insertPos);
1097 if (FI)
1098 return FI;
1099
1100 unsigned CC = ClangCallConvToLLVMCallConv(CC: info.getCC());
1101
1102 // Construct the function info. We co-allocate the ArgInfos.
1103 FI = CGFunctionInfo::create(llvmCC: CC, instanceMethod: isInstanceMethod, chainCall: isChainCall, delegateCall: isDelegateCall,
1104 X86ABIAVXLevel, extInfo: info, paramInfos, resultType,
1105 argTypes, required);
1106 FunctionInfos.InsertNode(N: FI, InsertPos: insertPos);
1107
1108 bool inserted = FunctionsBeingProcessed.insert(Ptr: FI).second;
1109 (void)inserted;
1110 assert(inserted && "Recursively being processed?");
1111
1112 // Compute ABI information.
1113 if (info.getCC() == CC_DeviceKernel &&
1114 (CC == llvm::CallingConv::SPIR_KERNEL || CC == llvm::CallingConv::C)) {
1115 // Force target independent argument handling for the host visible
1116 // kernel functions.
1117 //
1118 // For CPU targets, this currently only works for OpenCL.
1119 assert(CC != llvm::CallingConv::C || getContext().getLangOpts().OpenCL);
1120 computeSPIRKernelABIInfo(CGM, FI&: *FI);
1121 } else if (info.getCC() == CC_Swift || info.getCC() == CC_SwiftAsync) {
1122 swiftcall::computeABIInfo(CGM, FI&: *FI);
1123 } else if (CGM.shouldUseLLVMABILowering(CallingConv: CC)) {
1124 CGM.computeABIInfoUsingLib(FI&: *FI);
1125 } else {
1126 CGM.getABIInfo().computeInfo(FI&: *FI);
1127 }
1128
1129 // Loop over all of the computed argument and return value info. If any of
1130 // them are direct or extend without a specified coerce type, specify the
1131 // default now.
1132 ABIArgInfo &retInfo = FI->getReturnInfo();
1133 if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
1134 retInfo.setCoerceToType(ConvertType(T: FI->getReturnType()));
1135
1136 for (auto &I : FI->arguments())
1137 if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
1138 I.info.setCoerceToType(ConvertType(T: I.type));
1139
1140 bool erased = FunctionsBeingProcessed.erase(Ptr: FI);
1141 (void)erased;
1142 assert(erased && "Not in set?");
1143
1144 return FI;
1145}
1146
1147CGFunctionInfo *CGFunctionInfo::create(
1148 unsigned llvmCC, bool instanceMethod, bool chainCall, bool delegateCall,
1149 unsigned X86ABIAVXLevel, const FunctionType::ExtInfo &info,
1150 ArrayRef<ExtParameterInfo> paramInfos, CanQualType resultType,
1151 ArrayRef<CanQualType> argTypes, RequiredArgs required) {
1152 assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
1153 assert(!required.allowsOptionalArgs() ||
1154 required.getNumRequiredArgs() <= argTypes.size());
1155
1156 void *buffer = operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(
1157 Counts: argTypes.size() + 1, Counts: paramInfos.size()));
1158
1159 CGFunctionInfo *FI = new (buffer) CGFunctionInfo();
1160 FI->CallingConvention = llvmCC;
1161 FI->EffectiveCallingConvention = llvmCC;
1162 FI->ASTCallingConvention = info.getCC();
1163 FI->InstanceMethod = instanceMethod;
1164 FI->ChainCall = chainCall;
1165 FI->DelegateCall = delegateCall;
1166 FI->CmseNSCall = info.getCmseNSCall();
1167 FI->NoReturn = info.getNoReturn();
1168 FI->ReturnsRetained = info.getProducesResult();
1169 FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
1170 FI->NoCfCheck = info.getNoCfCheck();
1171 FI->Required = required;
1172 FI->HasRegParm = info.getHasRegParm();
1173 FI->RegParm = info.getRegParm();
1174 FI->X86ABIAVXLevel = X86ABIAVXLevel;
1175 FI->ArgStruct = nullptr;
1176 FI->ArgStructAlign = 0;
1177 FI->NumArgs = argTypes.size();
1178 FI->HasExtParameterInfos = !paramInfos.empty();
1179 FI->getArgsBuffer()[0].type = resultType;
1180 FI->MaxVectorWidth = 0;
1181 for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
1182 FI->getArgsBuffer()[i + 1].type = argTypes[i];
1183 for (unsigned i = 0, e = paramInfos.size(); i != e; ++i)
1184 FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
1185 return FI;
1186}
1187
1188/***/
1189
1190namespace {
1191// ABIArgInfo::Expand implementation.
1192
1193// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
1194struct TypeExpansion {
1195 enum TypeExpansionKind {
1196 // Elements of constant arrays are expanded recursively.
1197 TEK_ConstantArray,
1198 // Record fields are expanded recursively (but if record is a union, only
1199 // the field with the largest size is expanded).
1200 TEK_Record,
1201 // For complex types, real and imaginary parts are expanded recursively.
1202 TEK_Complex,
1203 // All other types are not expandable.
1204 TEK_None
1205 };
1206
1207 const TypeExpansionKind Kind;
1208
1209 TypeExpansion(TypeExpansionKind K) : Kind(K) {}
1210 virtual ~TypeExpansion() {}
1211};
1212
1213struct ConstantArrayExpansion : TypeExpansion {
1214 QualType EltTy;
1215 uint64_t NumElts;
1216
1217 ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
1218 : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
1219 static bool classof(const TypeExpansion *TE) {
1220 return TE->Kind == TEK_ConstantArray;
1221 }
1222};
1223
1224struct RecordExpansion : TypeExpansion {
1225 SmallVector<const CXXBaseSpecifier *, 1> Bases;
1226
1227 SmallVector<const FieldDecl *, 1> Fields;
1228
1229 RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
1230 SmallVector<const FieldDecl *, 1> &&Fields)
1231 : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
1232 Fields(std::move(Fields)) {}
1233 static bool classof(const TypeExpansion *TE) {
1234 return TE->Kind == TEK_Record;
1235 }
1236};
1237
1238struct ComplexExpansion : TypeExpansion {
1239 QualType EltTy;
1240
1241 ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
1242 static bool classof(const TypeExpansion *TE) {
1243 return TE->Kind == TEK_Complex;
1244 }
1245};
1246
1247struct NoExpansion : TypeExpansion {
1248 NoExpansion() : TypeExpansion(TEK_None) {}
1249 static bool classof(const TypeExpansion *TE) { return TE->Kind == TEK_None; }
1250};
1251} // namespace
1252
1253static std::unique_ptr<TypeExpansion>
1254getTypeExpansion(QualType Ty, const ASTContext &Context) {
1255 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty)) {
1256 return std::make_unique<ConstantArrayExpansion>(args: AT->getElementType(),
1257 args: AT->getZExtSize());
1258 }
1259 if (const auto *RD = Ty->getAsRecordDecl()) {
1260 SmallVector<const CXXBaseSpecifier *, 1> Bases;
1261 SmallVector<const FieldDecl *, 1> Fields;
1262 assert(!RD->hasFlexibleArrayMember() &&
1263 "Cannot expand structure with flexible array.");
1264 if (RD->isUnion()) {
1265 // Unions can be here only in degenerative cases - all the fields are same
1266 // after flattening. Thus we have to use the "largest" field.
1267 const FieldDecl *LargestFD = nullptr;
1268 CharUnits UnionSize = CharUnits::Zero();
1269
1270 for (const auto *FD : RD->fields()) {
1271 if (FD->isZeroLengthBitField())
1272 continue;
1273 assert(!FD->isBitField() &&
1274 "Cannot expand structure with bit-field members.");
1275 CharUnits FieldSize = Context.getTypeSizeInChars(T: FD->getType());
1276 if (UnionSize < FieldSize) {
1277 UnionSize = FieldSize;
1278 LargestFD = FD;
1279 }
1280 }
1281 if (LargestFD)
1282 Fields.push_back(Elt: LargestFD);
1283 } else {
1284 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
1285 assert(!CXXRD->isDynamicClass() &&
1286 "cannot expand vtable pointers in dynamic classes");
1287 llvm::append_range(C&: Bases, R: llvm::make_pointer_range(Range: CXXRD->bases()));
1288 }
1289
1290 for (const auto *FD : RD->fields()) {
1291 if (FD->isZeroLengthBitField())
1292 continue;
1293 assert(!FD->isBitField() &&
1294 "Cannot expand structure with bit-field members.");
1295 Fields.push_back(Elt: FD);
1296 }
1297 }
1298 return std::make_unique<RecordExpansion>(args: std::move(Bases),
1299 args: std::move(Fields));
1300 }
1301 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
1302 return std::make_unique<ComplexExpansion>(args: CT->getElementType());
1303 }
1304 return std::make_unique<NoExpansion>();
1305}
1306
1307static int getExpansionSize(QualType Ty, const ASTContext &Context) {
1308 auto Exp = getTypeExpansion(Ty, Context);
1309 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1310 return CAExp->NumElts * getExpansionSize(Ty: CAExp->EltTy, Context);
1311 }
1312 if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1313 int Res = 0;
1314 for (auto BS : RExp->Bases)
1315 Res += getExpansionSize(Ty: BS->getType(), Context);
1316 for (auto FD : RExp->Fields)
1317 Res += getExpansionSize(Ty: FD->getType(), Context);
1318 return Res;
1319 }
1320 if (isa<ComplexExpansion>(Val: Exp.get()))
1321 return 2;
1322 assert(isa<NoExpansion>(Exp.get()));
1323 return 1;
1324}
1325
1326void CodeGenTypes::getExpandedTypes(
1327 QualType Ty, SmallVectorImpl<llvm::Type *>::iterator &TI) {
1328 auto Exp = getTypeExpansion(Ty, Context);
1329 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1330 for (int i = 0, n = CAExp->NumElts; i < n; i++) {
1331 getExpandedTypes(Ty: CAExp->EltTy, TI);
1332 }
1333 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1334 for (auto BS : RExp->Bases)
1335 getExpandedTypes(Ty: BS->getType(), TI);
1336 for (auto FD : RExp->Fields)
1337 getExpandedTypes(Ty: FD->getType(), TI);
1338 } else if (auto CExp = dyn_cast<ComplexExpansion>(Val: Exp.get())) {
1339 llvm::Type *EltTy = ConvertType(T: CExp->EltTy);
1340 *TI++ = EltTy;
1341 *TI++ = EltTy;
1342 } else {
1343 assert(isa<NoExpansion>(Exp.get()));
1344 *TI++ = ConvertType(T: Ty);
1345 }
1346}
1347
1348static void forConstantArrayExpansion(CodeGenFunction &CGF,
1349 ConstantArrayExpansion *CAE,
1350 Address BaseAddr,
1351 llvm::function_ref<void(Address)> Fn) {
1352 for (int i = 0, n = CAE->NumElts; i < n; i++) {
1353 Address EltAddr = CGF.Builder.CreateConstGEP2_32(Addr: BaseAddr, Idx0: 0, Idx1: i);
1354 Fn(EltAddr);
1355 }
1356}
1357
1358void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1359 llvm::Function::arg_iterator &AI) {
1360 assert(LV.isSimple() &&
1361 "Unexpected non-simple lvalue during struct expansion.");
1362
1363 auto Exp = getTypeExpansion(Ty, Context: getContext());
1364 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1365 forConstantArrayExpansion(
1366 CGF&: *this, CAE: CAExp, BaseAddr: LV.getAddress(), Fn: [&](Address EltAddr) {
1367 LValue LV = MakeAddrLValue(Addr: EltAddr, T: CAExp->EltTy);
1368 ExpandTypeFromArgs(Ty: CAExp->EltTy, LV, AI);
1369 });
1370 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1371 Address This = LV.getAddress();
1372 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1373 // Perform a single step derived-to-base conversion.
1374 Address Base =
1375 GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1376 /*NullCheckValue=*/false, Loc: SourceLocation());
1377 LValue SubLV = MakeAddrLValue(Addr: Base, T: BS->getType());
1378
1379 // Recurse onto bases.
1380 ExpandTypeFromArgs(Ty: BS->getType(), LV: SubLV, AI);
1381 }
1382 for (auto FD : RExp->Fields) {
1383 // FIXME: What are the right qualifiers here?
1384 LValue SubLV = EmitLValueForFieldInitialization(Base: LV, Field: FD);
1385 ExpandTypeFromArgs(Ty: FD->getType(), LV: SubLV, AI);
1386 }
1387 } else if (isa<ComplexExpansion>(Val: Exp.get())) {
1388 auto realValue = &*AI++;
1389 auto imagValue = &*AI++;
1390 EmitStoreOfComplex(V: ComplexPairTy(realValue, imagValue), dest: LV, /*init*/ isInit: true);
1391 } else {
1392 // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1393 // primitive store.
1394 assert(isa<NoExpansion>(Exp.get()));
1395 llvm::Value *Arg = &*AI++;
1396 if (LV.isBitField()) {
1397 EmitStoreThroughLValue(Src: RValue::get(V: Arg), Dst: LV);
1398 } else {
1399 // TODO: currently there are some places are inconsistent in what LLVM
1400 // pointer type they use (see D118744). Once clang uses opaque pointers
1401 // all LLVM pointer types will be the same and we can remove this check.
1402 if (Arg->getType()->isPointerTy()) {
1403 Address Addr = LV.getAddress();
1404 Arg = Builder.CreateBitCast(V: Arg, DestTy: Addr.getElementType());
1405 }
1406 EmitStoreOfScalar(value: Arg, lvalue: LV);
1407 }
1408 }
1409}
1410
1411void CodeGenFunction::ExpandTypeToArgs(
1412 QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1413 SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1414 auto Exp = getTypeExpansion(Ty, Context: getContext());
1415 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1416 Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1417 : Arg.getKnownRValue().getAggregateAddress();
1418 forConstantArrayExpansion(CGF&: *this, CAE: CAExp, BaseAddr: Addr, Fn: [&](Address EltAddr) {
1419 CallArg EltArg =
1420 CallArg(convertTempToRValue(addr: EltAddr, type: CAExp->EltTy, Loc: SourceLocation()),
1421 CAExp->EltTy);
1422 ExpandTypeToArgs(Ty: CAExp->EltTy, Arg: EltArg, IRFuncTy, IRCallArgs,
1423 IRCallArgPos);
1424 });
1425 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1426 Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1427 : Arg.getKnownRValue().getAggregateAddress();
1428 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1429 // Perform a single step derived-to-base conversion.
1430 Address Base =
1431 GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1432 /*NullCheckValue=*/false, Loc: SourceLocation());
1433 CallArg BaseArg = CallArg(RValue::getAggregate(addr: Base), BS->getType());
1434
1435 // Recurse onto bases.
1436 ExpandTypeToArgs(Ty: BS->getType(), Arg: BaseArg, IRFuncTy, IRCallArgs,
1437 IRCallArgPos);
1438 }
1439
1440 LValue LV = MakeAddrLValue(Addr: This, T: Ty);
1441 for (auto FD : RExp->Fields) {
1442 CallArg FldArg =
1443 CallArg(EmitRValueForField(LV, FD, Loc: SourceLocation()), FD->getType());
1444 ExpandTypeToArgs(Ty: FD->getType(), Arg: FldArg, IRFuncTy, IRCallArgs,
1445 IRCallArgPos);
1446 }
1447 } else if (isa<ComplexExpansion>(Val: Exp.get())) {
1448 ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1449 IRCallArgs[IRCallArgPos++] = CV.first;
1450 IRCallArgs[IRCallArgPos++] = CV.second;
1451 } else {
1452 assert(isa<NoExpansion>(Exp.get()));
1453 auto RV = Arg.getKnownRValue();
1454 assert(RV.isScalar() &&
1455 "Unexpected non-scalar rvalue during struct expansion.");
1456
1457 // Insert a bitcast as needed.
1458 llvm::Value *V = RV.getScalarVal();
1459 if (IRCallArgPos < IRFuncTy->getNumParams() &&
1460 V->getType() != IRFuncTy->getParamType(i: IRCallArgPos))
1461 V = Builder.CreateBitCast(V, DestTy: IRFuncTy->getParamType(i: IRCallArgPos));
1462
1463 IRCallArgs[IRCallArgPos++] = V;
1464 }
1465}
1466
1467/// Create a temporary allocation for the purposes of coercion.
1468static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF,
1469 llvm::Type *Ty,
1470 CharUnits MinAlign,
1471 const Twine &Name = "tmp") {
1472 // Don't use an alignment that's worse than what LLVM would prefer.
1473 auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty);
1474 CharUnits Align = std::max(a: MinAlign, b: CharUnits::fromQuantity(Quantity: PrefAlign));
1475
1476 return CGF.CreateTempAlloca(Ty, align: Align, Name: Name + ".coerce");
1477}
1478
1479/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1480/// accessing some number of bytes out of it, try to gep into the struct to get
1481/// at its inner goodness. Dive as deep as possible without entering an element
1482/// with an in-memory size smaller than DstSize.
1483static Address EnterStructPointerForCoercedAccess(Address SrcPtr,
1484 llvm::StructType *SrcSTy,
1485 uint64_t DstSize,
1486 CodeGenFunction &CGF) {
1487 // We can't dive into a zero-element struct.
1488 if (SrcSTy->getNumElements() == 0)
1489 return SrcPtr;
1490
1491 llvm::Type *FirstElt = SrcSTy->getElementType(N: 0);
1492
1493 // If the first elt is at least as large as what we're looking for, or if the
1494 // first element is the same size as the whole struct, we can enter it. The
1495 // comparison must be made on the store size and not the alloca size. Using
1496 // the alloca size may overstate the size of the load.
1497 uint64_t FirstEltSize = CGF.CGM.getDataLayout().getTypeStoreSize(Ty: FirstElt);
1498 if (FirstEltSize < DstSize &&
1499 FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(Ty: SrcSTy))
1500 return SrcPtr;
1501
1502 // GEP into the first element.
1503 SrcPtr = CGF.Builder.CreateStructGEP(Addr: SrcPtr, Index: 0, Name: "coerce.dive");
1504
1505 // If the first element is a struct, recurse.
1506 llvm::Type *SrcTy = SrcPtr.getElementType();
1507 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy))
1508 return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1509
1510 return SrcPtr;
1511}
1512
1513/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1514/// are either integers or pointers. This does a truncation of the value if it
1515/// is too large or a zero extension if it is too small.
1516///
1517/// This behaves as if the value were coerced through memory, so on big-endian
1518/// targets the high bits are preserved in a truncation, while little-endian
1519/// targets preserve the low bits.
1520static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, llvm::Type *Ty,
1521 CodeGenFunction &CGF) {
1522 if (Val->getType() == Ty)
1523 return Val;
1524
1525 if (isa<llvm::PointerType>(Val: Val->getType())) {
1526 // If this is Pointer->Pointer avoid conversion to and from int.
1527 if (isa<llvm::PointerType>(Val: Ty))
1528 return CGF.Builder.CreateBitCast(V: Val, DestTy: Ty, Name: "coerce.val");
1529
1530 // Convert the pointer to an integer so we can play with its width.
1531 Val = CGF.Builder.CreatePtrToInt(V: Val, DestTy: CGF.IntPtrTy, Name: "coerce.val.pi");
1532 }
1533
1534 llvm::Type *DestIntTy = Ty;
1535 if (isa<llvm::PointerType>(Val: DestIntTy))
1536 DestIntTy = CGF.IntPtrTy;
1537
1538 if (Val->getType() != DestIntTy) {
1539 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1540 if (DL.isBigEndian()) {
1541 // Preserve the high bits on big-endian targets.
1542 // That is what memory coercion does.
1543 uint64_t SrcSize = DL.getTypeSizeInBits(Ty: Val->getType());
1544 uint64_t DstSize = DL.getTypeSizeInBits(Ty: DestIntTy);
1545
1546 if (SrcSize > DstSize) {
1547 Val = CGF.Builder.CreateLShr(LHS: Val, RHS: SrcSize - DstSize, Name: "coerce.highbits");
1548 Val = CGF.Builder.CreateTrunc(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1549 } else {
1550 Val = CGF.Builder.CreateZExt(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1551 Val = CGF.Builder.CreateShl(LHS: Val, RHS: DstSize - SrcSize, Name: "coerce.highbits");
1552 }
1553 } else {
1554 // Little-endian targets preserve the low bits. No shifts required.
1555 Val = CGF.Builder.CreateIntCast(V: Val, DestTy: DestIntTy, isSigned: false, Name: "coerce.val.ii");
1556 }
1557 }
1558
1559 if (isa<llvm::PointerType>(Val: Ty))
1560 Val = CGF.Builder.CreateIntToPtr(V: Val, DestTy: Ty, Name: "coerce.val.ip");
1561 return Val;
1562}
1563
1564static llvm::Value *CreatePFPCoercedLoad(Address Src, QualType SrcFETy,
1565 llvm::Type *Ty, CodeGenFunction &CGF) {
1566 std::vector<PFPField> PFPFields = CGF.getContext().findPFPFields(Ty: SrcFETy);
1567 if (PFPFields.empty())
1568 return nullptr;
1569
1570 auto LoadCoercedField = [&](CharUnits Offset,
1571 llvm::Type *FieldType) -> llvm::Value * {
1572 // Check whether the field at Offset is a PFP field. This function is called
1573 // in ascending order of offset, and PFPFields is sorted by offset. This
1574 // means that we only need to check the first element (and remove it from
1575 // PFPFields if matching).
1576 if (!PFPFields.empty() && PFPFields[0].Offset == Offset) {
1577 auto FieldAddr = CGF.EmitAddressOfPFPField(RecordPtr: Src, Field: PFPFields[0]);
1578 llvm::Value *FieldVal = CGF.Builder.CreateLoad(Addr: FieldAddr);
1579 if (isa<llvm::IntegerType>(Val: FieldType))
1580 FieldVal = CGF.Builder.CreatePtrToInt(V: FieldVal, DestTy: FieldType);
1581 PFPFields.erase(position: PFPFields.begin());
1582 return FieldVal;
1583 }
1584 auto FieldAddr =
1585 CGF.Builder
1586 .CreateConstInBoundsByteGEP(Addr: Src.withElementType(ElemTy: CGF.Int8Ty), Offset)
1587 .withElementType(ElemTy: FieldType);
1588 return CGF.Builder.CreateLoad(Addr: FieldAddr);
1589 };
1590
1591 // The types handled by this function are the only ones that may be generated
1592 // by AArch64ABIInfo::classify{Argument,Return}Type for struct types with
1593 // pointers. PFP is only supported on AArch64.
1594 if (isa<llvm::IntegerType>(Val: Ty) || isa<llvm::PointerType>(Val: Ty)) {
1595 auto Addr = CGF.EmitAddressOfPFPField(RecordPtr: Src, Field: PFPFields[0]);
1596 llvm::Value *Val = CGF.Builder.CreateLoad(Addr);
1597 if (isa<llvm::IntegerType>(Val: Ty))
1598 Val = CGF.Builder.CreatePtrToInt(V: Val, DestTy: Ty);
1599 return Val;
1600 }
1601 auto *AT = cast<llvm::ArrayType>(Val: Ty);
1602 auto *ET = AT->getElementType();
1603 CharUnits WordSize = CGF.getContext().toCharUnitsFromBits(
1604 BitSize: CGF.CGM.getDataLayout().getTypeSizeInBits(Ty: ET));
1605 CharUnits Offset = CharUnits::Zero();
1606 llvm::Value *Val = llvm::PoisonValue::get(T: AT);
1607 for (unsigned Idx = 0; Idx != AT->getNumElements(); ++Idx, Offset += WordSize)
1608 Val = CGF.Builder.CreateInsertValue(Agg: Val, Val: LoadCoercedField(Offset, ET), Idxs: Idx);
1609 return Val;
1610}
1611
1612/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1613/// a pointer to an object of type \arg Ty, known to be aligned to
1614/// \arg SrcAlign bytes.
1615///
1616/// This safely handles the case when the src type is smaller than the
1617/// destination type; in this situation the values of bits which not
1618/// present in the src are undefined.
1619static llvm::Value *CreateCoercedLoad(Address Src, QualType SrcFETy,
1620 llvm::Type *Ty, CodeGenFunction &CGF) {
1621 llvm::Type *SrcTy = Src.getElementType();
1622
1623 // If SrcTy and Ty are the same, just do a load.
1624 if (SrcTy == Ty)
1625 return CGF.Builder.CreateLoad(Addr: Src);
1626
1627 if (llvm::Value *V = CreatePFPCoercedLoad(Src, SrcFETy, Ty, CGF))
1628 return V;
1629
1630 llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1631
1632 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy)) {
1633 Src = EnterStructPointerForCoercedAccess(SrcPtr: Src, SrcSTy,
1634 DstSize: DstSize.getFixedValue(), CGF);
1635 SrcTy = Src.getElementType();
1636 }
1637
1638 llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1639
1640 // If the source and destination are integer or pointer types, just do an
1641 // extension or truncation to the desired type.
1642 if ((isa<llvm::IntegerType>(Val: Ty) || isa<llvm::PointerType>(Val: Ty)) &&
1643 (isa<llvm::IntegerType>(Val: SrcTy) || isa<llvm::PointerType>(Val: SrcTy))) {
1644 llvm::Value *Load = CGF.Builder.CreateLoad(Addr: Src);
1645 return CoerceIntOrPtrToIntOrPtr(Val: Load, Ty, CGF);
1646 }
1647
1648 // If load is legal, just bitcast the src pointer.
1649 if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1650 SrcSize.getFixedValue() >= DstSize.getFixedValue()) {
1651 // Generally SrcSize is never greater than DstSize, since this means we are
1652 // losing bits. However, this can happen in cases where the structure has
1653 // additional padding, for example due to a user specified alignment.
1654 //
1655 // FIXME: Assert that we aren't truncating non-padding bits when have access
1656 // to that information.
1657 Src = Src.withElementType(ElemTy: Ty);
1658 return CGF.Builder.CreateLoad(Addr: Src);
1659 }
1660
1661 // If coercing a fixed vector to a scalable vector for ABI compatibility, and
1662 // the types match, use the llvm.vector.insert intrinsic to perform the
1663 // conversion.
1664 if (auto *ScalableDstTy = dyn_cast<llvm::ScalableVectorType>(Val: Ty)) {
1665 if (auto *FixedSrcTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy)) {
1666 // If we are casting a fixed i8 vector to a scalable i1 predicate
1667 // vector, use a vector insert and bitcast the result.
1668 if (ScalableDstTy->getElementType()->isIntegerTy(BitWidth: 1) &&
1669 FixedSrcTy->getElementType()->isIntegerTy(BitWidth: 8)) {
1670 ScalableDstTy = llvm::ScalableVectorType::get(
1671 ElementType: FixedSrcTy->getElementType(),
1672 MinNumElts: llvm::divideCeil(
1673 Numerator: ScalableDstTy->getElementCount().getKnownMinValue(), Denominator: 8));
1674 }
1675 if (ScalableDstTy->getElementType() == FixedSrcTy->getElementType()) {
1676 auto *Load = CGF.Builder.CreateLoad(Addr: Src);
1677 auto *PoisonVec = llvm::PoisonValue::get(T: ScalableDstTy);
1678 llvm::Value *Result = CGF.Builder.CreateInsertVector(
1679 DstType: ScalableDstTy, SrcVec: PoisonVec, SubVec: Load, Idx: uint64_t(0), Name: "cast.scalable");
1680 ScalableDstTy = cast<llvm::ScalableVectorType>(
1681 Val: llvm::VectorType::getWithSizeAndScalar(SizeTy: ScalableDstTy, EltTy: Ty));
1682 if (Result->getType() != ScalableDstTy)
1683 Result = CGF.Builder.CreateBitCast(V: Result, DestTy: ScalableDstTy);
1684 if (Result->getType() != Ty)
1685 Result = CGF.Builder.CreateExtractVector(DstType: Ty, SrcVec: Result, Idx: uint64_t(0));
1686 return Result;
1687 }
1688 }
1689 }
1690
1691 // Otherwise do coercion through memory. This is stupid, but simple.
1692 RawAddress Tmp =
1693 CreateTempAllocaForCoercion(CGF, Ty, MinAlign: Src.getAlignment(), Name: Src.getName());
1694 CGF.Builder.CreateMemCpy(
1695 Dst: Tmp.getPointer(), DstAlign: Tmp.getAlignment().getAsAlign(),
1696 Src: Src.emitRawPointer(CGF), SrcAlign: Src.getAlignment().getAsAlign(),
1697 Size: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: SrcSize.getKnownMinValue()));
1698 return CGF.Builder.CreateLoad(Addr: Tmp);
1699}
1700
1701static bool CreatePFPCoercedStore(llvm::Value *Src, QualType SrcFETy,
1702 Address Dst, CodeGenFunction &CGF) {
1703 std::vector<PFPField> PFPFields = CGF.getContext().findPFPFields(Ty: SrcFETy);
1704 if (PFPFields.empty())
1705 return false;
1706
1707 llvm::Type *SrcTy = Src->getType();
1708 auto StoreCoercedField = [&](CharUnits Offset, llvm::Value *FieldVal) {
1709 if (!PFPFields.empty() && PFPFields[0].Offset == Offset) {
1710 auto FieldAddr = CGF.EmitAddressOfPFPField(RecordPtr: Dst, Field: PFPFields[0]);
1711 if (isa<llvm::IntegerType>(Val: FieldVal->getType()))
1712 FieldVal = CGF.Builder.CreateIntToPtr(V: FieldVal, DestTy: CGF.VoidPtrTy);
1713 CGF.Builder.CreateStore(Val: FieldVal, Addr: FieldAddr);
1714 PFPFields.erase(position: PFPFields.begin());
1715 } else {
1716 auto FieldAddr = CGF.Builder
1717 .CreateConstInBoundsByteGEP(
1718 Addr: Dst.withElementType(ElemTy: CGF.Int8Ty), Offset)
1719 .withElementType(ElemTy: FieldVal->getType());
1720 CGF.Builder.CreateStore(Val: FieldVal, Addr: FieldAddr);
1721 }
1722 };
1723
1724 // The types handled by this function are the only ones that may be generated
1725 // by AArch64ABIInfo::classify{Argument,Return}Type for struct types with
1726 // pointers. PFP is only supported on AArch64.
1727 if (isa<llvm::IntegerType>(Val: SrcTy) || isa<llvm::PointerType>(Val: SrcTy)) {
1728 if (isa<llvm::IntegerType>(Val: SrcTy))
1729 Src = CGF.Builder.CreateIntToPtr(V: Src, DestTy: CGF.VoidPtrTy);
1730 auto Addr = CGF.EmitAddressOfPFPField(RecordPtr: Dst, Field: PFPFields[0]);
1731 CGF.Builder.CreateStore(Val: Src, Addr);
1732 } else {
1733 auto *AT = cast<llvm::ArrayType>(Val: SrcTy);
1734 auto *ET = AT->getElementType();
1735 CharUnits WordSize = CGF.getContext().toCharUnitsFromBits(
1736 BitSize: CGF.CGM.getDataLayout().getTypeSizeInBits(Ty: ET));
1737 CharUnits Offset = CharUnits::Zero();
1738 for (unsigned i = 0; i != AT->getNumElements(); ++i, Offset += WordSize)
1739 StoreCoercedField(Offset, CGF.Builder.CreateExtractValue(Agg: Src, Idxs: i));
1740 }
1741 return true;
1742}
1743
1744void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, QualType SrcFETy,
1745 Address Dst, llvm::TypeSize DstSize,
1746 bool DstIsVolatile) {
1747 if (!DstSize)
1748 return;
1749
1750 llvm::Type *SrcTy = Src->getType();
1751 llvm::TypeSize SrcSize = CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1752
1753 // GEP into structs to try to make types match.
1754 // FIXME: This isn't really that useful with opaque types, but it impacts a
1755 // lot of regression tests.
1756 if (SrcTy != Dst.getElementType()) {
1757 if (llvm::StructType *DstSTy =
1758 dyn_cast<llvm::StructType>(Val: Dst.getElementType())) {
1759 assert(!SrcSize.isScalable());
1760 Dst = EnterStructPointerForCoercedAccess(SrcPtr: Dst, SrcSTy: DstSTy,
1761 DstSize: SrcSize.getFixedValue(), CGF&: *this);
1762 }
1763 }
1764
1765 if (CreatePFPCoercedStore(Src, SrcFETy, Dst, CGF&: *this))
1766 return;
1767
1768 if (SrcSize.isScalable() || SrcSize <= DstSize) {
1769 if (SrcTy->isIntegerTy() && Dst.getElementType()->isPointerTy() &&
1770 SrcSize == CGM.getDataLayout().getTypeAllocSize(Ty: Dst.getElementType())) {
1771 // If the value is supposed to be a pointer, convert it before storing it.
1772 Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: Dst.getElementType(), CGF&: *this);
1773 auto *I = Builder.CreateStore(Val: Src, Addr: Dst, IsVolatile: DstIsVolatile);
1774 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1775 } else if (llvm::StructType *STy =
1776 dyn_cast<llvm::StructType>(Val: Src->getType())) {
1777 // Prefer scalar stores to first-class aggregate stores.
1778 Dst = Dst.withElementType(ElemTy: SrcTy);
1779 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1780 Address EltPtr = Builder.CreateStructGEP(Addr: Dst, Index: i);
1781 llvm::Value *Elt = Builder.CreateExtractValue(Agg: Src, Idxs: i);
1782 auto *I = Builder.CreateStore(Val: Elt, Addr: EltPtr, IsVolatile: DstIsVolatile);
1783 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Elt);
1784 }
1785 } else {
1786 auto *I =
1787 Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: SrcTy), IsVolatile: DstIsVolatile);
1788 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1789 }
1790 } else if (SrcTy->isIntegerTy()) {
1791 // If the source is a simple integer, coerce it directly.
1792 llvm::Type *DstIntTy = Builder.getIntNTy(N: DstSize.getFixedValue() * 8);
1793 Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: DstIntTy, CGF&: *this);
1794 auto *I =
1795 Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: DstIntTy), IsVolatile: DstIsVolatile);
1796 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1797 } else {
1798 // Otherwise do coercion through memory. This is stupid, but
1799 // simple.
1800
1801 // Generally SrcSize is never greater than DstSize, since this means we are
1802 // losing bits. However, this can happen in cases where the structure has
1803 // additional padding, for example due to a user specified alignment.
1804 //
1805 // FIXME: Assert that we aren't truncating non-padding bits when have access
1806 // to that information.
1807 RawAddress Tmp =
1808 CreateTempAllocaForCoercion(CGF&: *this, Ty: SrcTy, MinAlign: Dst.getAlignment());
1809 Builder.CreateStore(Val: Src, Addr: Tmp);
1810 auto *I = Builder.CreateMemCpy(
1811 Dst: Dst.emitRawPointer(CGF&: *this), DstAlign: Dst.getAlignment().getAsAlign(),
1812 Src: Tmp.getPointer(), SrcAlign: Tmp.getAlignment().getAsAlign(),
1813 Size: Builder.CreateTypeSize(Ty: IntPtrTy, Size: DstSize));
1814 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1815 }
1816}
1817
1818static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1819 const ABIArgInfo &info) {
1820 if (unsigned offset = info.getDirectOffset()) {
1821 addr = addr.withElementType(ElemTy: CGF.Int8Ty);
1822 addr = CGF.Builder.CreateConstInBoundsByteGEP(
1823 Addr: addr, Offset: CharUnits::fromQuantity(Quantity: offset));
1824 addr = addr.withElementType(ElemTy: info.getCoerceToType());
1825 }
1826 return addr;
1827}
1828
1829static std::pair<llvm::Value *, bool>
1830CoerceScalableToFixed(CodeGenFunction &CGF, llvm::FixedVectorType *ToTy,
1831 llvm::ScalableVectorType *FromTy, llvm::Value *V,
1832 StringRef Name = "") {
1833 // If we are casting a scalable i1 predicate vector to a fixed i8
1834 // vector, first bitcast the source.
1835 if (FromTy->getElementType()->isIntegerTy(BitWidth: 1) &&
1836 ToTy->getElementType() == CGF.Builder.getInt8Ty()) {
1837 if (!FromTy->getElementCount().isKnownMultipleOf(RHS: 8)) {
1838 FromTy = llvm::ScalableVectorType::get(
1839 ElementType: FromTy->getElementType(),
1840 MinNumElts: llvm::alignTo<8>(Value: FromTy->getElementCount().getKnownMinValue()));
1841 llvm::Value *ZeroVec = llvm::Constant::getNullValue(Ty: FromTy);
1842 V = CGF.Builder.CreateInsertVector(DstType: FromTy, SrcVec: ZeroVec, SubVec: V, Idx: uint64_t(0));
1843 }
1844 FromTy = llvm::ScalableVectorType::get(
1845 ElementType: ToTy->getElementType(),
1846 MinNumElts: FromTy->getElementCount().getKnownMinValue() / 8);
1847 V = CGF.Builder.CreateBitCast(V, DestTy: FromTy);
1848 }
1849 if (FromTy->getElementType() == ToTy->getElementType()) {
1850 V->setName(Name + ".coerce");
1851 V = CGF.Builder.CreateExtractVector(DstType: ToTy, SrcVec: V, Idx: uint64_t(0), Name: "cast.fixed");
1852 return {V, true};
1853 }
1854 return {V, false};
1855}
1856
1857namespace {
1858
1859/// Encapsulates information about the way function arguments from
1860/// CGFunctionInfo should be passed to actual LLVM IR function.
1861class ClangToLLVMArgMapping {
1862 static const unsigned InvalidIndex = ~0U;
1863 unsigned InallocaArgNo;
1864 unsigned SRetArgNo;
1865 unsigned TotalIRArgs;
1866
1867 /// Arguments of LLVM IR function corresponding to single Clang argument.
1868 struct IRArgs {
1869 unsigned PaddingArgIndex;
1870 // Argument is expanded to IR arguments at positions
1871 // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1872 unsigned FirstArgIndex;
1873 unsigned NumberOfArgs;
1874
1875 IRArgs()
1876 : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1877 NumberOfArgs(0) {}
1878 };
1879
1880 SmallVector<IRArgs, 8> ArgInfo;
1881
1882public:
1883 ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1884 bool OnlyRequiredArgs = false)
1885 : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1886 ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1887 construct(Context, FI, OnlyRequiredArgs);
1888 }
1889
1890 bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1891 unsigned getInallocaArgNo() const {
1892 assert(hasInallocaArg());
1893 return InallocaArgNo;
1894 }
1895
1896 bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1897 unsigned getSRetArgNo() const {
1898 assert(hasSRetArg());
1899 return SRetArgNo;
1900 }
1901
1902 unsigned totalIRArgs() const { return TotalIRArgs; }
1903
1904 bool hasPaddingArg(unsigned ArgNo) const {
1905 assert(ArgNo < ArgInfo.size());
1906 return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1907 }
1908 unsigned getPaddingArgNo(unsigned ArgNo) const {
1909 assert(hasPaddingArg(ArgNo));
1910 return ArgInfo[ArgNo].PaddingArgIndex;
1911 }
1912
1913 /// Returns index of first IR argument corresponding to ArgNo, and their
1914 /// quantity.
1915 std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1916 assert(ArgNo < ArgInfo.size());
1917 return std::make_pair(x: ArgInfo[ArgNo].FirstArgIndex,
1918 y: ArgInfo[ArgNo].NumberOfArgs);
1919 }
1920
1921private:
1922 void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1923 bool OnlyRequiredArgs);
1924};
1925
1926void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1927 const CGFunctionInfo &FI,
1928 bool OnlyRequiredArgs) {
1929 unsigned IRArgNo = 0;
1930 bool SwapThisWithSRet = false;
1931 const ABIArgInfo &RetAI = FI.getReturnInfo();
1932
1933 if (RetAI.getKind() == ABIArgInfo::Indirect) {
1934 SwapThisWithSRet = RetAI.isSRetAfterThis();
1935 SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1936 }
1937
1938 unsigned ArgNo = 0;
1939 unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();
1940 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1941 ++I, ++ArgNo) {
1942 assert(I != FI.arg_end());
1943 QualType ArgType = I->type;
1944 const ABIArgInfo &AI = I->info;
1945 // Collect data about IR arguments corresponding to Clang argument ArgNo.
1946 auto &IRArgs = ArgInfo[ArgNo];
1947
1948 if (AI.getPaddingType())
1949 IRArgs.PaddingArgIndex = IRArgNo++;
1950
1951 switch (AI.getKind()) {
1952 case ABIArgInfo::TargetSpecific:
1953 case ABIArgInfo::Extend:
1954 case ABIArgInfo::Direct: {
1955 // FIXME: handle sseregparm someday...
1956 llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: AI.getCoerceToType());
1957 if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
1958 IRArgs.NumberOfArgs = STy->getNumElements();
1959 } else {
1960 IRArgs.NumberOfArgs = 1;
1961 }
1962 break;
1963 }
1964 case ABIArgInfo::Indirect:
1965 case ABIArgInfo::IndirectAliased:
1966 IRArgs.NumberOfArgs = 1;
1967 break;
1968 case ABIArgInfo::Ignore:
1969 case ABIArgInfo::InAlloca:
1970 // ignore and inalloca doesn't have matching LLVM parameters.
1971 IRArgs.NumberOfArgs = 0;
1972 break;
1973 case ABIArgInfo::CoerceAndExpand:
1974 IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1975 break;
1976 case ABIArgInfo::Expand:
1977 IRArgs.NumberOfArgs = getExpansionSize(Ty: ArgType, Context);
1978 break;
1979 }
1980
1981 if (IRArgs.NumberOfArgs > 0) {
1982 IRArgs.FirstArgIndex = IRArgNo;
1983 IRArgNo += IRArgs.NumberOfArgs;
1984 }
1985
1986 // Skip over the sret parameter when it comes second. We already handled it
1987 // above.
1988 if (IRArgNo == 1 && SwapThisWithSRet)
1989 IRArgNo++;
1990 }
1991 assert(ArgNo == ArgInfo.size());
1992
1993 if (FI.usesInAlloca())
1994 InallocaArgNo = IRArgNo++;
1995
1996 TotalIRArgs = IRArgNo;
1997}
1998} // namespace
1999
2000/***/
2001
2002bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
2003 const auto &RI = FI.getReturnInfo();
2004 return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());
2005}
2006
2007bool CodeGenModule::ReturnTypeHasInReg(const CGFunctionInfo &FI) {
2008 const auto &RI = FI.getReturnInfo();
2009 return RI.getInReg();
2010}
2011
2012bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
2013 return ReturnTypeUsesSRet(FI) &&
2014 getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
2015}
2016
2017bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
2018 if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
2019 switch (BT->getKind()) {
2020 default:
2021 return false;
2022 case BuiltinType::Float:
2023 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Float);
2024 case BuiltinType::Double:
2025 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Double);
2026 case BuiltinType::LongDouble:
2027 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::LongDouble);
2028 }
2029 }
2030
2031 return false;
2032}
2033
2034bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
2035 if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
2036 if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
2037 if (BT->getKind() == BuiltinType::LongDouble)
2038 return getTarget().useObjCFP2RetForComplexLongDouble();
2039 }
2040 }
2041
2042 return false;
2043}
2044
2045llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
2046 const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
2047 return GetFunctionType(Info: FI);
2048}
2049
2050llvm::FunctionType *CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
2051
2052 bool Inserted = FunctionsBeingProcessed.insert(Ptr: &FI).second;
2053 (void)Inserted;
2054 assert(Inserted && "Recursively being processed?");
2055
2056 llvm::Type *resultType = nullptr;
2057 const ABIArgInfo &retAI = FI.getReturnInfo();
2058 switch (retAI.getKind()) {
2059 case ABIArgInfo::Expand:
2060 case ABIArgInfo::IndirectAliased:
2061 llvm_unreachable("Invalid ABI kind for return argument");
2062
2063 case ABIArgInfo::TargetSpecific:
2064 case ABIArgInfo::Extend:
2065 case ABIArgInfo::Direct:
2066 resultType = retAI.getCoerceToType();
2067 break;
2068
2069 case ABIArgInfo::InAlloca:
2070 if (retAI.getInAllocaSRet()) {
2071 // sret things on win32 aren't void, they return the sret pointer.
2072 QualType ret = FI.getReturnType();
2073 unsigned addressSpace = CGM.getTypes().getTargetAddressSpace(T: ret);
2074 resultType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: addressSpace);
2075 } else {
2076 resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
2077 }
2078 break;
2079
2080 case ABIArgInfo::Indirect:
2081 case ABIArgInfo::Ignore:
2082 resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
2083 break;
2084
2085 case ABIArgInfo::CoerceAndExpand:
2086 resultType = retAI.getUnpaddedCoerceAndExpandType();
2087 break;
2088 }
2089
2090 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
2091 SmallVector<llvm::Type *, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
2092
2093 // Add type for sret argument.
2094 if (IRFunctionArgs.hasSRetArg()) {
2095 ArgTypes[IRFunctionArgs.getSRetArgNo()] = llvm::PointerType::get(
2096 C&: getLLVMContext(), AddressSpace: FI.getReturnInfo().getIndirectAddrSpace());
2097 }
2098
2099 // Add type for inalloca argument.
2100 if (IRFunctionArgs.hasInallocaArg())
2101 ArgTypes[IRFunctionArgs.getInallocaArgNo()] =
2102 llvm::PointerType::getUnqual(C&: getLLVMContext());
2103
2104 // Add in all of the required arguments.
2105 unsigned ArgNo = 0;
2106 CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
2107 ie = it + FI.getNumRequiredArgs();
2108 for (; it != ie; ++it, ++ArgNo) {
2109 const ABIArgInfo &ArgInfo = it->info;
2110
2111 // Insert a padding type to ensure proper alignment.
2112 if (IRFunctionArgs.hasPaddingArg(ArgNo))
2113 ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2114 ArgInfo.getPaddingType();
2115
2116 unsigned FirstIRArg, NumIRArgs;
2117 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2118
2119 switch (ArgInfo.getKind()) {
2120 case ABIArgInfo::Ignore:
2121 case ABIArgInfo::InAlloca:
2122 assert(NumIRArgs == 0);
2123 break;
2124
2125 case ABIArgInfo::Indirect:
2126 assert(NumIRArgs == 1);
2127 // indirect arguments are always on the stack, which is alloca addr space.
2128 ArgTypes[FirstIRArg] = llvm::PointerType::get(
2129 C&: getLLVMContext(), AddressSpace: CGM.getDataLayout().getAllocaAddrSpace());
2130 break;
2131 case ABIArgInfo::IndirectAliased:
2132 assert(NumIRArgs == 1);
2133 ArgTypes[FirstIRArg] = llvm::PointerType::get(
2134 C&: getLLVMContext(), AddressSpace: ArgInfo.getIndirectAddrSpace());
2135 break;
2136 case ABIArgInfo::TargetSpecific:
2137 case ABIArgInfo::Extend:
2138 case ABIArgInfo::Direct: {
2139 // Fast-isel and the optimizer generally like scalar values better than
2140 // FCAs, so we flatten them if this is safe to do for this argument.
2141 llvm::Type *argType = ArgInfo.getCoerceToType();
2142 llvm::StructType *st = dyn_cast<llvm::StructType>(Val: argType);
2143 if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
2144 assert(NumIRArgs == st->getNumElements());
2145 for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
2146 ArgTypes[FirstIRArg + i] = st->getElementType(N: i);
2147 } else {
2148 assert(NumIRArgs == 1);
2149 ArgTypes[FirstIRArg] = argType;
2150 }
2151 break;
2152 }
2153
2154 case ABIArgInfo::CoerceAndExpand: {
2155 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
2156 for (auto *EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
2157 *ArgTypesIter++ = EltTy;
2158 }
2159 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
2160 break;
2161 }
2162
2163 case ABIArgInfo::Expand:
2164 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
2165 getExpandedTypes(Ty: it->type, TI&: ArgTypesIter);
2166 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
2167 break;
2168 }
2169 }
2170
2171 bool Erased = FunctionsBeingProcessed.erase(Ptr: &FI);
2172 (void)Erased;
2173 assert(Erased && "Not in set?");
2174
2175 return llvm::FunctionType::get(Result: resultType, Params: ArgTypes, isVarArg: FI.isVariadic());
2176}
2177
2178llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
2179 const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
2180 const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
2181
2182 if (!isFuncTypeConvertible(FT: FPT))
2183 return llvm::StructType::get(Context&: getLLVMContext());
2184
2185 return GetFunctionType(GD);
2186}
2187
2188static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
2189 llvm::AttrBuilder &FuncAttrs,
2190 const FunctionProtoType *FPT) {
2191 if (!FPT)
2192 return;
2193
2194 if (!isUnresolvedExceptionSpec(ESpecType: FPT->getExceptionSpecType()) &&
2195 FPT->isNothrow())
2196 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2197
2198 unsigned SMEBits = FPT->getAArch64SMEAttributes();
2199 if (SMEBits & FunctionType::SME_PStateSMEnabledMask)
2200 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_enabled");
2201 if (SMEBits & FunctionType::SME_PStateSMCompatibleMask)
2202 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_compatible");
2203 if (SMEBits & FunctionType::SME_AgnosticZAStateMask)
2204 FuncAttrs.addAttribute(A: "aarch64_za_state_agnostic");
2205
2206 // ZA
2207 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
2208 FuncAttrs.addAttribute(A: "aarch64_preserves_za");
2209 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_In)
2210 FuncAttrs.addAttribute(A: "aarch64_in_za");
2211 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Out)
2212 FuncAttrs.addAttribute(A: "aarch64_out_za");
2213 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_InOut)
2214 FuncAttrs.addAttribute(A: "aarch64_inout_za");
2215
2216 // ZT0
2217 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
2218 FuncAttrs.addAttribute(A: "aarch64_preserves_zt0");
2219 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_In)
2220 FuncAttrs.addAttribute(A: "aarch64_in_zt0");
2221 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Out)
2222 FuncAttrs.addAttribute(A: "aarch64_out_zt0");
2223 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_InOut)
2224 FuncAttrs.addAttribute(A: "aarch64_inout_zt0");
2225}
2226
2227static void AddAttributesFromOMPAssumes(llvm::AttrBuilder &FuncAttrs,
2228 const Decl *Callee) {
2229 if (!Callee)
2230 return;
2231
2232 SmallVector<StringRef, 4> Attrs;
2233
2234 for (const OMPAssumeAttr *AA : Callee->specific_attrs<OMPAssumeAttr>())
2235 AA->getAssumption().split(A&: Attrs, Separator: ",");
2236
2237 if (!Attrs.empty())
2238 FuncAttrs.addAttribute(A: llvm::AssumptionAttrKey,
2239 V: llvm::join(Begin: Attrs.begin(), End: Attrs.end(), Separator: ","));
2240}
2241
2242bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,
2243 QualType ReturnType) const {
2244 // We can't just discard the return value for a record type with a
2245 // complex destructor or a non-trivially copyable type.
2246 if (const RecordType *RT =
2247 ReturnType.getCanonicalType()->getAsCanonical<RecordType>()) {
2248 if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(Val: RT->getDecl()))
2249 return ClassDecl->hasTrivialDestructor();
2250 }
2251 return ReturnType.isTriviallyCopyableType(Context);
2252}
2253
2254static bool HasStrictReturn(const CodeGenModule &Module, QualType RetTy,
2255 const Decl *TargetDecl) {
2256 // As-is msan can not tolerate noundef mismatch between caller and
2257 // implementation. Mismatch is possible for e.g. indirect calls from C-caller
2258 // into C++. Such mismatches lead to confusing false reports. To avoid
2259 // expensive workaround on msan we enforce initialization event in uncommon
2260 // cases where it's allowed.
2261 if (Module.getLangOpts().Sanitize.has(K: SanitizerKind::Memory))
2262 return true;
2263 // C++ explicitly makes returning undefined values UB. C's rule only applies
2264 // to used values, so we never mark them noundef for now.
2265 if (!Module.getLangOpts().CPlusPlus)
2266 return false;
2267 if (TargetDecl) {
2268 if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2269 if (FDecl->isExternC())
2270 return false;
2271 } else if (const VarDecl *VDecl = dyn_cast<VarDecl>(Val: TargetDecl)) {
2272 // Function pointer.
2273 if (VDecl->isExternC())
2274 return false;
2275 }
2276 }
2277
2278 // We don't want to be too aggressive with the return checking, unless
2279 // it's explicit in the code opts or we're using an appropriate sanitizer.
2280 // Try to respect what the programmer intended.
2281 return Module.getCodeGenOpts().StrictReturn ||
2282 !Module.MayDropFunctionReturn(Context: Module.getContext(), ReturnType: RetTy) ||
2283 Module.getLangOpts().Sanitize.has(K: SanitizerKind::Return);
2284}
2285
2286/// Add denormal-fp-math and denormal-fp-math-f32 as appropriate for the
2287/// requested denormal behavior, accounting for the overriding behavior of the
2288/// -f32 case.
2289static void addDenormalModeAttrs(llvm::DenormalMode FPDenormalMode,
2290 llvm::DenormalMode FP32DenormalMode,
2291 llvm::AttrBuilder &FuncAttrs) {
2292 llvm::DenormalFPEnv FPEnv(FPDenormalMode, FP32DenormalMode);
2293 if (FPEnv != llvm::DenormalFPEnv::getDefault())
2294 FuncAttrs.addDenormalFPEnvAttr(Mode: FPEnv);
2295}
2296
2297/// Add default attributes to a function, which have merge semantics under
2298/// -mlink-builtin-bitcode and should not simply overwrite any existing
2299/// attributes in the linked library.
2300static void
2301addMergableDefaultFunctionAttributes(const CodeGenOptions &CodeGenOpts,
2302 llvm::AttrBuilder &FuncAttrs) {
2303 addDenormalModeAttrs(FPDenormalMode: CodeGenOpts.FPDenormalMode, FP32DenormalMode: CodeGenOpts.FP32DenormalMode,
2304 FuncAttrs);
2305}
2306
2307static void getTrivialDefaultFunctionAttributes(
2308 StringRef Name, bool HasOptnone, const CodeGenOptions &CodeGenOpts,
2309 const LangOptions &LangOpts, bool AttrOnCallSite,
2310 llvm::AttrBuilder &FuncAttrs) {
2311 // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
2312 if (!HasOptnone) {
2313 if (CodeGenOpts.OptimizeSize)
2314 FuncAttrs.addAttribute(Val: llvm::Attribute::OptimizeForSize);
2315 if (CodeGenOpts.OptimizeSize == 2)
2316 FuncAttrs.addAttribute(Val: llvm::Attribute::MinSize);
2317 }
2318
2319 if (CodeGenOpts.DisableRedZone)
2320 FuncAttrs.addAttribute(Val: llvm::Attribute::NoRedZone);
2321 if (CodeGenOpts.IndirectTlsSegRefs)
2322 FuncAttrs.addAttribute(A: "indirect-tls-seg-refs");
2323 if (CodeGenOpts.NoImplicitFloat)
2324 FuncAttrs.addAttribute(Val: llvm::Attribute::NoImplicitFloat);
2325
2326 if (AttrOnCallSite) {
2327 // Attributes that should go on the call site only.
2328 // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
2329 // the -fno-builtin-foo list.
2330 if (!CodeGenOpts.SimplifyLibCalls || LangOpts.isNoBuiltinFunc(Name))
2331 FuncAttrs.addAttribute(Val: llvm::Attribute::NoBuiltin);
2332 if (!CodeGenOpts.TrapFuncName.empty())
2333 FuncAttrs.addAttribute(A: "trap-func-name", V: CodeGenOpts.TrapFuncName);
2334 } else {
2335 switch (CodeGenOpts.getFramePointer()) {
2336 case CodeGenOptions::FramePointerKind::None:
2337 // This is the default behavior.
2338 break;
2339 case CodeGenOptions::FramePointerKind::Reserved:
2340 case CodeGenOptions::FramePointerKind::NonLeafNoReserve:
2341 case CodeGenOptions::FramePointerKind::NonLeaf:
2342 case CodeGenOptions::FramePointerKind::All:
2343 FuncAttrs.addAttribute(A: "frame-pointer",
2344 V: CodeGenOptions::getFramePointerKindName(
2345 Kind: CodeGenOpts.getFramePointer()));
2346 }
2347
2348 if (CodeGenOpts.LessPreciseFPMAD)
2349 FuncAttrs.addAttribute(A: "less-precise-fpmad", V: "true");
2350
2351 if (CodeGenOpts.NullPointerIsValid)
2352 FuncAttrs.addAttribute(Val: llvm::Attribute::NullPointerIsValid);
2353
2354 if (LangOpts.getDefaultExceptionMode() == LangOptions::FPE_Ignore)
2355 FuncAttrs.addAttribute(A: "no-trapping-math", V: "true");
2356
2357 // TODO: Are these all needed?
2358 // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
2359 if (CodeGenOpts.SoftFloat)
2360 FuncAttrs.addAttribute(A: "use-soft-float", V: "true");
2361 FuncAttrs.addAttribute(A: "stack-protector-buffer-size",
2362 V: llvm::utostr(X: CodeGenOpts.SSPBufferSize));
2363 if (LangOpts.NoSignedZero)
2364 FuncAttrs.addAttribute(A: "no-signed-zeros-fp-math", V: "true");
2365
2366 // TODO: Reciprocal estimate codegen options should apply to instructions?
2367 const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
2368 if (!Recips.empty())
2369 FuncAttrs.addAttribute(A: "reciprocal-estimates", V: llvm::join(R: Recips, Separator: ","));
2370
2371 if (!CodeGenOpts.PreferVectorWidth.empty() &&
2372 CodeGenOpts.PreferVectorWidth != "none")
2373 FuncAttrs.addAttribute(A: "prefer-vector-width",
2374 V: CodeGenOpts.PreferVectorWidth);
2375
2376 if (CodeGenOpts.StackRealignment)
2377 FuncAttrs.addAttribute(A: "stackrealign");
2378 if (CodeGenOpts.Backchain)
2379 FuncAttrs.addAttribute(A: "backchain");
2380 if (CodeGenOpts.EnableSegmentedStacks)
2381 FuncAttrs.addAttribute(A: "split-stack");
2382
2383 if (CodeGenOpts.SpeculativeLoadHardening)
2384 FuncAttrs.addAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2385
2386 // Add zero-call-used-regs attribute.
2387 switch (CodeGenOpts.getZeroCallUsedRegs()) {
2388 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
2389 FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2390 break;
2391 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
2392 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr-arg");
2393 break;
2394 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
2395 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr");
2396 break;
2397 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
2398 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-arg");
2399 break;
2400 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
2401 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used");
2402 break;
2403 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
2404 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr-arg");
2405 break;
2406 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
2407 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr");
2408 break;
2409 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
2410 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-arg");
2411 break;
2412 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
2413 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all");
2414 break;
2415 }
2416 }
2417
2418 if (LangOpts.assumeFunctionsAreConvergent()) {
2419 // Conservatively, mark all functions and calls in CUDA and OpenCL as
2420 // convergent (meaning, they may call an intrinsically convergent op, such
2421 // as __syncthreads() / barrier(), and so can't have certain optimizations
2422 // applied around them). LLVM will remove this attribute where it safely
2423 // can.
2424 FuncAttrs.addAttribute(Val: llvm::Attribute::Convergent);
2425 }
2426
2427 // TODO: NoUnwind attribute should be added for other GPU modes HIP,
2428 // OpenMP offload. AFAIK, neither of them support exceptions in device code.
2429 if ((LangOpts.CUDA && LangOpts.CUDAIsDevice) || LangOpts.OpenCL ||
2430 LangOpts.SYCLIsDevice) {
2431 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2432 }
2433
2434 if (CodeGenOpts.SaveRegParams && !AttrOnCallSite)
2435 FuncAttrs.addAttribute(A: "save-reg-params");
2436
2437 for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
2438 StringRef Var, Value;
2439 std::tie(args&: Var, args&: Value) = Attr.split(Separator: '=');
2440 FuncAttrs.addAttribute(A: Var, V: Value);
2441 }
2442
2443 TargetInfo::BranchProtectionInfo BPI(LangOpts);
2444 TargetCodeGenInfo::initBranchProtectionFnAttributes(BPI, FuncAttrs);
2445}
2446
2447/// Merges `target-features` from \TargetOpts and \F, and sets the result in
2448/// \FuncAttr
2449/// * features from \F are always kept
2450/// * a feature from \TargetOpts is kept if itself and its opposite are absent
2451/// from \F
2452static void
2453overrideFunctionFeaturesWithTargetFeatures(llvm::AttrBuilder &FuncAttr,
2454 const llvm::Function &F,
2455 const TargetOptions &TargetOpts) {
2456 auto FFeatures = F.getFnAttribute(Kind: "target-features");
2457
2458 llvm::StringSet<> MergedNames;
2459 SmallVector<StringRef> MergedFeatures;
2460 MergedFeatures.reserve(N: TargetOpts.Features.size());
2461
2462 auto AddUnmergedFeatures = [&](auto &&FeatureRange) {
2463 for (StringRef Feature : FeatureRange) {
2464 if (Feature.empty())
2465 continue;
2466 assert(Feature[0] == '+' || Feature[0] == '-');
2467 StringRef Name = Feature.drop_front(N: 1);
2468 bool Merged = !MergedNames.insert(key: Name).second;
2469 if (!Merged)
2470 MergedFeatures.push_back(Elt: Feature);
2471 }
2472 };
2473
2474 if (FFeatures.isValid())
2475 AddUnmergedFeatures(llvm::split(Str: FFeatures.getValueAsString(), Separator: ','));
2476 AddUnmergedFeatures(TargetOpts.Features);
2477
2478 if (!MergedFeatures.empty()) {
2479 llvm::sort(C&: MergedFeatures);
2480 FuncAttr.addAttribute(A: "target-features", V: llvm::join(R&: MergedFeatures, Separator: ","));
2481 }
2482}
2483
2484void CodeGen::mergeDefaultFunctionDefinitionAttributes(
2485 llvm::Function &F, const CodeGenOptions &CodeGenOpts,
2486 const LangOptions &LangOpts, const TargetOptions &TargetOpts,
2487 bool WillInternalize) {
2488
2489 llvm::AttrBuilder FuncAttrs(F.getContext());
2490 // Here we only extract the options that are relevant compared to the version
2491 // from GetCPUAndFeaturesAttributes.
2492 if (!TargetOpts.CPU.empty())
2493 FuncAttrs.addAttribute(A: "target-cpu", V: TargetOpts.CPU);
2494 if (!TargetOpts.TuneCPU.empty())
2495 FuncAttrs.addAttribute(A: "tune-cpu", V: TargetOpts.TuneCPU);
2496
2497 ::getTrivialDefaultFunctionAttributes(Name: F.getName(), HasOptnone: F.hasOptNone(),
2498 CodeGenOpts, LangOpts,
2499 /*AttrOnCallSite=*/false, FuncAttrs);
2500
2501 if (!WillInternalize && F.isInterposable()) {
2502 // Do not promote "dynamic" denormal-fp-math to this translation unit's
2503 // setting for weak functions that won't be internalized. The user has no
2504 // real control for how builtin bitcode is linked, so we shouldn't assume
2505 // later copies will use a consistent mode.
2506 F.addFnAttrs(Attrs: FuncAttrs);
2507 return;
2508 }
2509
2510 llvm::AttributeMask AttrsToRemove;
2511
2512 llvm::DenormalFPEnv OptsFPEnv(CodeGenOpts.FPDenormalMode,
2513 CodeGenOpts.FP32DenormalMode);
2514 llvm::DenormalFPEnv MergedFPEnv =
2515 OptsFPEnv.mergeCalleeMode(Callee: F.getDenormalFPEnv());
2516
2517 if (MergedFPEnv == llvm::DenormalFPEnv::getDefault()) {
2518 AttrsToRemove.addAttribute(Val: llvm::Attribute::DenormalFPEnv);
2519 } else {
2520 // Overwrite existing attribute
2521 FuncAttrs.addDenormalFPEnvAttr(Mode: MergedFPEnv);
2522 }
2523
2524 F.removeFnAttrs(Attrs: AttrsToRemove);
2525
2526 overrideFunctionFeaturesWithTargetFeatures(FuncAttr&: FuncAttrs, F, TargetOpts);
2527
2528 F.addFnAttrs(Attrs: FuncAttrs);
2529}
2530
2531void CodeGenModule::getTrivialDefaultFunctionAttributes(
2532 StringRef Name, bool HasOptnone, bool AttrOnCallSite,
2533 llvm::AttrBuilder &FuncAttrs) {
2534 ::getTrivialDefaultFunctionAttributes(Name, HasOptnone, CodeGenOpts: getCodeGenOpts(),
2535 LangOpts: getLangOpts(), AttrOnCallSite,
2536 FuncAttrs);
2537}
2538
2539void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
2540 bool HasOptnone,
2541 bool AttrOnCallSite,
2542 llvm::AttrBuilder &FuncAttrs) {
2543 getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,
2544 FuncAttrs);
2545
2546 if (!AttrOnCallSite)
2547 TargetCodeGenInfo::initPointerAuthFnAttributes(Opts: CodeGenOpts.PointerAuth,
2548 FuncAttrs);
2549
2550 // If we're just getting the default, get the default values for mergeable
2551 // attributes.
2552 if (!AttrOnCallSite)
2553 addMergableDefaultFunctionAttributes(CodeGenOpts, FuncAttrs);
2554}
2555
2556void CodeGenModule::addDefaultFunctionDefinitionAttributes(
2557 llvm::AttrBuilder &attrs) {
2558 getDefaultFunctionAttributes(/*function name*/ Name: "", /*optnone*/ HasOptnone: false,
2559 /*for call*/ AttrOnCallSite: false, FuncAttrs&: attrs);
2560 GetCPUAndFeaturesAttributes(GD: GlobalDecl(), AttrBuilder&: attrs);
2561}
2562
2563static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
2564 const LangOptions &LangOpts,
2565 const NoBuiltinAttr *NBA = nullptr) {
2566 auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
2567 SmallString<32> AttributeName;
2568 AttributeName += "no-builtin-";
2569 AttributeName += BuiltinName;
2570 FuncAttrs.addAttribute(A: AttributeName);
2571 };
2572
2573 // First, handle the language options passed through -fno-builtin.
2574 if (LangOpts.NoBuiltin) {
2575 // -fno-builtin disables them all.
2576 FuncAttrs.addAttribute(A: "no-builtins");
2577 return;
2578 }
2579
2580 // Then, add attributes for builtins specified through -fno-builtin-<name>.
2581 llvm::for_each(Range: LangOpts.NoBuiltinFuncs, F: AddNoBuiltinAttr);
2582
2583 // Now, let's check the __attribute__((no_builtin("...")) attribute added to
2584 // the source.
2585 if (!NBA)
2586 return;
2587
2588 // If there is a wildcard in the builtin names specified through the
2589 // attribute, disable them all.
2590 if (llvm::is_contained(Range: NBA->builtinNames(), Element: "*")) {
2591 FuncAttrs.addAttribute(A: "no-builtins");
2592 return;
2593 }
2594
2595 // And last, add the rest of the builtin names.
2596 llvm::for_each(Range: NBA->builtinNames(), F: AddNoBuiltinAttr);
2597}
2598
2599static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
2600 const llvm::DataLayout &DL, const ABIArgInfo &AI,
2601 bool CheckCoerce = true) {
2602 llvm::Type *Ty = Types.ConvertTypeForMem(T: QTy);
2603 if (AI.getKind() == ABIArgInfo::Indirect ||
2604 AI.getKind() == ABIArgInfo::IndirectAliased)
2605 return true;
2606 if (AI.getKind() == ABIArgInfo::Extend && !AI.isNoExt())
2607 return true;
2608 if (!DL.typeSizeEqualsStoreSize(Ty))
2609 // TODO: This will result in a modest amount of values not marked noundef
2610 // when they could be. We care about values that *invisibly* contain undef
2611 // bits from the perspective of LLVM IR.
2612 return false;
2613 if (CheckCoerce && AI.canHaveCoerceToType()) {
2614 llvm::Type *CoerceTy = AI.getCoerceToType();
2615 if (llvm::TypeSize::isKnownGT(LHS: DL.getTypeSizeInBits(Ty: CoerceTy),
2616 RHS: DL.getTypeSizeInBits(Ty)))
2617 // If we're coercing to a type with a greater size than the canonical one,
2618 // we're introducing new undef bits.
2619 // Coercing to a type of smaller or equal size is ok, as we know that
2620 // there's no internal padding (typeSizeEqualsStoreSize).
2621 return false;
2622 }
2623 if (QTy->isBitIntType())
2624 return true;
2625 if (QTy->isReferenceType())
2626 return true;
2627 if (QTy->isNullPtrType())
2628 return false;
2629 if (QTy->isMemberPointerType())
2630 // TODO: Some member pointers are `noundef`, but it depends on the ABI. For
2631 // now, never mark them.
2632 return false;
2633 if (QTy->isScalarType()) {
2634 if (const ComplexType *Complex = dyn_cast<ComplexType>(Val&: QTy))
2635 return DetermineNoUndef(QTy: Complex->getElementType(), Types, DL, AI, CheckCoerce: false);
2636 return true;
2637 }
2638 if (const VectorType *Vector = dyn_cast<VectorType>(Val&: QTy))
2639 return DetermineNoUndef(QTy: Vector->getElementType(), Types, DL, AI, CheckCoerce: false);
2640 if (const MatrixType *Matrix = dyn_cast<MatrixType>(Val&: QTy))
2641 return DetermineNoUndef(QTy: Matrix->getElementType(), Types, DL, AI, CheckCoerce: false);
2642 if (const ArrayType *Array = dyn_cast<ArrayType>(Val&: QTy))
2643 return DetermineNoUndef(QTy: Array->getElementType(), Types, DL, AI, CheckCoerce: false);
2644
2645 // TODO: Some structs may be `noundef`, in specific situations.
2646 return false;
2647}
2648
2649/// Check if the argument of a function has maybe_undef attribute.
2650static bool IsArgumentMaybeUndef(const Decl *TargetDecl,
2651 unsigned NumRequiredArgs, unsigned ArgNo) {
2652 const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
2653 if (!FD)
2654 return false;
2655
2656 // Assume variadic arguments do not have maybe_undef attribute.
2657 if (ArgNo >= NumRequiredArgs)
2658 return false;
2659
2660 // Check if argument has maybe_undef attribute.
2661 if (ArgNo < FD->getNumParams()) {
2662 const ParmVarDecl *Param = FD->getParamDecl(i: ArgNo);
2663 if (Param && Param->hasAttr<MaybeUndefAttr>())
2664 return true;
2665 }
2666
2667 return false;
2668}
2669
2670/// Test if it's legal to apply nofpclass for the given parameter type and it's
2671/// lowered IR type.
2672static bool canApplyNoFPClass(const ABIArgInfo &AI, QualType ParamType,
2673 bool IsReturn) {
2674 // Should only apply to FP types in the source, not ABI promoted.
2675 if (!ParamType->hasFloatingRepresentation())
2676 return false;
2677
2678 // The promoted-to IR type also needs to support nofpclass.
2679 llvm::Type *IRTy = AI.getCoerceToType();
2680 if (llvm::AttributeFuncs::isNoFPClassCompatibleType(Ty: IRTy))
2681 return true;
2682
2683 if (llvm::StructType *ST = dyn_cast<llvm::StructType>(Val: IRTy)) {
2684 return !IsReturn && AI.getCanBeFlattened() &&
2685 llvm::all_of(Range: ST->elements(),
2686 P: llvm::AttributeFuncs::isNoFPClassCompatibleType);
2687 }
2688
2689 return false;
2690}
2691
2692/// Return the nofpclass mask that can be applied to floating-point parameters.
2693static llvm::FPClassTest getNoFPClassTestMask(const LangOptions &LangOpts) {
2694 llvm::FPClassTest Mask = llvm::fcNone;
2695 if (LangOpts.NoHonorInfs)
2696 Mask |= llvm::fcInf;
2697 if (LangOpts.NoHonorNaNs)
2698 Mask |= llvm::fcNan;
2699 return Mask;
2700}
2701
2702void CodeGenModule::AdjustMemoryAttribute(StringRef Name,
2703 CGCalleeInfo CalleeInfo,
2704 llvm::AttributeList &Attrs) {
2705 if (Attrs.getMemoryEffects().getModRef() == llvm::ModRefInfo::NoModRef) {
2706 Attrs = Attrs.removeFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::Memory);
2707 llvm::Attribute MemoryAttr = llvm::Attribute::getWithMemoryEffects(
2708 Context&: getLLVMContext(), ME: llvm::MemoryEffects::writeOnly());
2709 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Attr: MemoryAttr);
2710 }
2711}
2712
2713/// Construct the IR attribute list of a function or call.
2714///
2715/// When adding an attribute, please consider where it should be handled:
2716///
2717/// - getDefaultFunctionAttributes is for attributes that are essentially
2718/// part of the global target configuration (but perhaps can be
2719/// overridden on a per-function basis). Adding attributes there
2720/// will cause them to also be set in frontends that build on Clang's
2721/// target-configuration logic, as well as for code defined in library
2722/// modules such as CUDA's libdevice.
2723///
2724/// - ConstructAttributeList builds on top of getDefaultFunctionAttributes
2725/// and adds declaration-specific, convention-specific, and
2726/// frontend-specific logic. The last is of particular importance:
2727/// attributes that restrict how the frontend generates code must be
2728/// added here rather than getDefaultFunctionAttributes.
2729///
2730void CodeGenModule::ConstructAttributeList(StringRef Name,
2731 const CGFunctionInfo &FI,
2732 CGCalleeInfo CalleeInfo,
2733 llvm::AttributeList &AttrList,
2734 unsigned &CallingConv,
2735 bool AttrOnCallSite, bool IsThunk) {
2736 llvm::AttrBuilder FuncAttrs(getLLVMContext());
2737 llvm::AttrBuilder RetAttrs(getLLVMContext());
2738
2739 // Collect function IR attributes from the CC lowering.
2740 // We'll collect the paramete and result attributes later.
2741 CallingConv = FI.getEffectiveCallingConvention();
2742 if (FI.isNoReturn())
2743 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2744 if (FI.isCmseNSCall())
2745 FuncAttrs.addAttribute(A: "cmse_nonsecure_call");
2746
2747 // Collect function IR attributes from the callee prototype if we have one.
2748 AddAttributesFromFunctionProtoType(Ctx&: getContext(), FuncAttrs,
2749 FPT: CalleeInfo.getCalleeFunctionProtoType());
2750 const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2751
2752 // Attach assumption attributes to the declaration. If this is a call
2753 // site, attach assumptions from the caller to the call as well.
2754 AddAttributesFromOMPAssumes(FuncAttrs, Callee: TargetDecl);
2755
2756 bool HasOptnone = false;
2757 // The NoBuiltinAttr attached to the target FunctionDecl.
2758 const NoBuiltinAttr *NBA = nullptr;
2759
2760 // Some ABIs may result in additional accesses to arguments that may
2761 // otherwise not be present.
2762 std::optional<llvm::Attribute::AttrKind> MemAttrForPtrArgs;
2763 bool AddedPotentialArgAccess = false;
2764 auto AddPotentialArgAccess = [&]() {
2765 AddedPotentialArgAccess = true;
2766 llvm::Attribute A = FuncAttrs.getAttribute(Kind: llvm::Attribute::Memory);
2767 if (A.isValid())
2768 FuncAttrs.addMemoryAttr(ME: A.getMemoryEffects() |
2769 llvm::MemoryEffects::argMemOnly());
2770 };
2771
2772 // Collect function IR attributes based on declaration-specific
2773 // information.
2774 // FIXME: handle sseregparm someday...
2775 if (TargetDecl) {
2776 if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2777 FuncAttrs.addAttribute(Val: llvm::Attribute::ReturnsTwice);
2778 if (TargetDecl->hasAttr<NoThrowAttr>())
2779 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2780 if (TargetDecl->hasAttr<NoReturnAttr>())
2781 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2782 if (TargetDecl->hasAttr<ColdAttr>())
2783 FuncAttrs.addAttribute(Val: llvm::Attribute::Cold);
2784 if (TargetDecl->hasAttr<HotAttr>())
2785 FuncAttrs.addAttribute(Val: llvm::Attribute::Hot);
2786 if (TargetDecl->hasAttr<NoDuplicateAttr>())
2787 FuncAttrs.addAttribute(Val: llvm::Attribute::NoDuplicate);
2788 if (TargetDecl->hasAttr<ConvergentAttr>())
2789 FuncAttrs.addAttribute(Val: llvm::Attribute::Convergent);
2790
2791 if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2792 AddAttributesFromFunctionProtoType(
2793 Ctx&: getContext(), FuncAttrs, FPT: Fn->getType()->getAs<FunctionProtoType>());
2794 if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
2795 // A sane operator new returns a non-aliasing pointer and does not
2796 // read or write accessible memory.
2797 if (getCodeGenOpts().AssumeSaneOperatorNew &&
2798 Fn->getDeclName().isAnyOperatorNew()) {
2799 RetAttrs.addAttribute(Val: llvm::Attribute::NoAlias);
2800 // FIXME: inaccessiblemem could cause issues if LTO makes the
2801 // previously inaccessible memory accessible after linking.
2802 FuncAttrs.addMemoryAttr(
2803 ME: llvm::MemoryEffects::inaccessibleOrErrnoMemOnly(
2804 InaccessibleMR: llvm::ModRefInfo::ModRef, ErrnoMR: llvm::ModRefInfo::Mod));
2805 }
2806 }
2807 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: Fn);
2808 const bool IsVirtualCall = MD && MD->isVirtual();
2809 // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2810 // virtual function. These attributes are not inherited by overloads.
2811 if (!(AttrOnCallSite && IsVirtualCall)) {
2812 if (Fn->isNoReturn())
2813 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2814 NBA = Fn->getAttr<NoBuiltinAttr>();
2815 }
2816 }
2817
2818 if (isa<FunctionDecl>(Val: TargetDecl) || isa<VarDecl>(Val: TargetDecl)) {
2819 // Only place nomerge attribute on call sites, never functions. This
2820 // allows it to work on indirect virtual function calls.
2821 if (AttrOnCallSite && TargetDecl->hasAttr<NoMergeAttr>())
2822 FuncAttrs.addAttribute(Val: llvm::Attribute::NoMerge);
2823 }
2824
2825 // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2826 if (TargetDecl->hasAttr<ConstAttr>()) {
2827 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::none());
2828 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2829 // gcc specifies that 'const' functions have greater restrictions than
2830 // 'pure' functions, so they also cannot have infinite loops.
2831 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
2832 MemAttrForPtrArgs = llvm::Attribute::ReadNone;
2833 } else if (TargetDecl->hasAttr<PureAttr>()) {
2834 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::readOnly());
2835 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2836 // gcc specifies that 'pure' functions cannot have infinite loops.
2837 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
2838 MemAttrForPtrArgs = llvm::Attribute::ReadOnly;
2839 } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2840 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::inaccessibleOrArgMemOnly());
2841 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2842 }
2843 if (const auto *RA = TargetDecl->getAttr<RestrictAttr>();
2844 RA && RA->getDeallocator() == nullptr)
2845 RetAttrs.addAttribute(Val: llvm::Attribute::NoAlias);
2846 if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2847 !CodeGenOpts.NullPointerIsValid)
2848 RetAttrs.addAttribute(Val: llvm::Attribute::NonNull);
2849 if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2850 FuncAttrs.addAttribute(A: "no_caller_saved_registers");
2851 if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2852 FuncAttrs.addAttribute(Val: llvm::Attribute::NoCfCheck);
2853 if (TargetDecl->hasAttr<LeafAttr>())
2854 FuncAttrs.addAttribute(Val: llvm::Attribute::NoCallback);
2855 if (TargetDecl->hasAttr<BPFFastCallAttr>())
2856 FuncAttrs.addAttribute(A: "bpf_fastcall");
2857
2858 HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2859 if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2860 std::optional<unsigned> NumElemsParam;
2861 if (AllocSize->getNumElemsParam().isValid())
2862 NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2863 FuncAttrs.addAllocSizeAttr(ElemSizeArg: AllocSize->getElemSizeParam().getLLVMIndex(),
2864 NumElemsArg: NumElemsParam);
2865 }
2866
2867 // OpenCL v2.0 Work groups may be whether uniform or not.
2868 // '-cl-uniform-work-group-size' compile option gets a hint
2869 // to the compiler that the global work-size be a multiple of
2870 // the work-group size specified to clEnqueueNDRangeKernel
2871 // (i.e. work groups are uniform).
2872 if (getLangOpts().OffloadUniformBlock)
2873 FuncAttrs.addAttribute(A: "uniform-work-group-size");
2874
2875 if (TargetDecl->hasAttr<ArmLocallyStreamingAttr>())
2876 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_body");
2877
2878 if (auto *ModularFormat = TargetDecl->getAttr<ModularFormatAttr>()) {
2879 FormatAttr *Format = TargetDecl->getAttr<FormatAttr>();
2880 StringRef Type = Format->getType()->getName();
2881 std::string FormatIdx = std::to_string(val: Format->getFormatIdx());
2882 std::string FirstArg = std::to_string(val: Format->getFirstArg());
2883 SmallVector<StringRef> Args = {
2884 Type, FormatIdx, FirstArg,
2885 ModularFormat->getModularImplFn()->getName(),
2886 ModularFormat->getImplName()};
2887 llvm::append_range(C&: Args, R: ModularFormat->aspects());
2888 FuncAttrs.addAttribute(A: "modular-format", V: llvm::join(R&: Args, Separator: ","));
2889 }
2890 }
2891
2892 // Attach "no-builtins" attributes to:
2893 // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2894 // * definitions: "no-builtins" or "no-builtin-<name>" only.
2895 // The attributes can come from:
2896 // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2897 // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2898 addNoBuiltinAttributes(FuncAttrs, LangOpts: getLangOpts(), NBA);
2899
2900 // Collect function IR attributes based on global settiings.
2901 getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2902
2903 // Override some default IR attributes based on declaration-specific
2904 // information.
2905 if (TargetDecl) {
2906 if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2907 FuncAttrs.removeAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2908 if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2909 FuncAttrs.addAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2910 if (TargetDecl->hasAttr<NoSplitStackAttr>())
2911 FuncAttrs.removeAttribute(A: "split-stack");
2912 if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
2913 // A function "__attribute__((...))" overrides the command-line flag.
2914 auto Kind =
2915 TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2916 FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2917 FuncAttrs.addAttribute(
2918 A: "zero-call-used-regs",
2919 V: ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Val: Kind));
2920 }
2921
2922 // Add NonLazyBind attribute to function declarations when -fno-plt
2923 // is used.
2924 // FIXME: what if we just haven't processed the function definition
2925 // yet, or if it's an external definition like C99 inline?
2926 if (CodeGenOpts.NoPLT) {
2927 if (auto *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2928 if (!Fn->isDefined() && !AttrOnCallSite) {
2929 FuncAttrs.addAttribute(Val: llvm::Attribute::NonLazyBind);
2930 }
2931 }
2932 }
2933 // Remove 'convergent' if requested.
2934 if (TargetDecl->hasAttr<NoConvergentAttr>())
2935 FuncAttrs.removeAttribute(Val: llvm::Attribute::Convergent);
2936 }
2937
2938 // Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2939 // functions with -funique-internal-linkage-names.
2940 if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames) {
2941 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
2942 if (!FD->isExternallyVisible())
2943 FuncAttrs.addAttribute(A: "sample-profile-suffix-elision-policy",
2944 V: "selected");
2945 }
2946 }
2947
2948 // Collect non-call-site function IR attributes from declaration-specific
2949 // information.
2950 if (!AttrOnCallSite) {
2951 if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>())
2952 FuncAttrs.addAttribute(A: "cmse_nonsecure_entry");
2953
2954 // Whether tail calls are enabled.
2955 auto shouldDisableTailCalls = [&] {
2956 // Should this be honored in getDefaultFunctionAttributes?
2957 if (CodeGenOpts.DisableTailCalls)
2958 return true;
2959
2960 if (!TargetDecl)
2961 return false;
2962
2963 if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2964 TargetDecl->hasAttr<AnyX86InterruptAttr>())
2965 return true;
2966
2967 if (CodeGenOpts.NoEscapingBlockTailCalls) {
2968 if (const auto *BD = dyn_cast<BlockDecl>(Val: TargetDecl))
2969 if (!BD->doesNotEscape())
2970 return true;
2971 }
2972
2973 return false;
2974 };
2975 if (shouldDisableTailCalls())
2976 FuncAttrs.addAttribute(A: "disable-tail-calls", V: "true");
2977
2978 // These functions require the returns_twice attribute for correct codegen,
2979 // but the attribute may not be added if -fno-builtin is specified. We
2980 // explicitly add that attribute here.
2981 static const llvm::StringSet<> ReturnsTwiceFn{
2982 "_setjmpex", "setjmp", "_setjmp", "vfork",
2983 "sigsetjmp", "__sigsetjmp", "savectx", "getcontext"};
2984 if (ReturnsTwiceFn.contains(key: Name))
2985 FuncAttrs.addAttribute(Val: llvm::Attribute::ReturnsTwice);
2986
2987 // CPU/feature overrides. addDefaultFunctionDefinitionAttributes
2988 // handles these separately to set them based on the global defaults.
2989 GetCPUAndFeaturesAttributes(GD: CalleeInfo.getCalleeDecl(), AttrBuilder&: FuncAttrs);
2990
2991 // Windows hotpatching support
2992 if (!MSHotPatchFunctions.empty()) {
2993 bool IsHotPatched = llvm::binary_search(Range&: MSHotPatchFunctions, Value&: Name);
2994 if (IsHotPatched)
2995 FuncAttrs.addAttribute(A: "marked_for_windows_hot_patching");
2996 }
2997 }
2998
2999 // Mark functions that are replaceable by the loader.
3000 if (CodeGenOpts.isLoaderReplaceableFunctionName(FuncName: Name))
3001 FuncAttrs.addAttribute(A: "loader-replaceable");
3002
3003 // Collect attributes from arguments and return values.
3004 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
3005
3006 QualType RetTy = FI.getReturnType();
3007 const ABIArgInfo &RetAI = FI.getReturnInfo();
3008 const llvm::DataLayout &DL = getDataLayout();
3009
3010 // Determine if the return type could be partially undef
3011 if (CodeGenOpts.EnableNoundefAttrs &&
3012 HasStrictReturn(Module: *this, RetTy, TargetDecl)) {
3013 if (!RetTy->isVoidType() && RetAI.getKind() != ABIArgInfo::Indirect &&
3014 DetermineNoUndef(QTy: RetTy, Types&: getTypes(), DL, AI: RetAI))
3015 RetAttrs.addAttribute(Val: llvm::Attribute::NoUndef);
3016 }
3017
3018 switch (RetAI.getKind()) {
3019 case ABIArgInfo::Extend:
3020 if (RetAI.isSignExt())
3021 RetAttrs.addAttribute(Val: llvm::Attribute::SExt);
3022 else if (RetAI.isZeroExt())
3023 RetAttrs.addAttribute(Val: llvm::Attribute::ZExt);
3024 else
3025 RetAttrs.addAttribute(Val: llvm::Attribute::NoExt);
3026 [[fallthrough]];
3027 case ABIArgInfo::TargetSpecific:
3028 case ABIArgInfo::Direct:
3029 if (RetAI.getInReg())
3030 RetAttrs.addAttribute(Val: llvm::Attribute::InReg);
3031
3032 if (canApplyNoFPClass(AI: RetAI, ParamType: RetTy, IsReturn: true))
3033 RetAttrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
3034
3035 break;
3036 case ABIArgInfo::Ignore:
3037 break;
3038
3039 case ABIArgInfo::InAlloca:
3040 case ABIArgInfo::Indirect: {
3041 // inalloca and sret disable readnone and readonly
3042 AddPotentialArgAccess();
3043 break;
3044 }
3045
3046 case ABIArgInfo::CoerceAndExpand:
3047 break;
3048
3049 case ABIArgInfo::Expand:
3050 case ABIArgInfo::IndirectAliased:
3051 llvm_unreachable("Invalid ABI kind for return argument");
3052 }
3053
3054 if (!IsThunk) {
3055 // FIXME: fix this properly, https://reviews.llvm.org/D100388
3056 if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
3057 QualType PTy = RefTy->getPointeeType();
3058 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
3059 RetAttrs.addDereferenceableAttr(
3060 Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
3061 if (getTypes().getTargetAddressSpace(T: PTy) == 0 &&
3062 !CodeGenOpts.NullPointerIsValid)
3063 RetAttrs.addAttribute(Val: llvm::Attribute::NonNull);
3064 if (PTy->isObjectType()) {
3065 llvm::Align Alignment =
3066 getNaturalPointeeTypeAlignment(T: RetTy).getAsAlign();
3067 RetAttrs.addAlignmentAttr(Align: Alignment);
3068 }
3069 }
3070 }
3071
3072 bool hasUsedSRet = false;
3073 SmallVector<llvm::AttrBuilder, 4> ArgAttrs;
3074 for (unsigned I = 0; I < IRFunctionArgs.totalIRArgs(); ++I)
3075 ArgAttrs.emplace_back(Args&: getLLVMContext());
3076
3077 // Attach attributes to sret.
3078 if (IRFunctionArgs.hasSRetArg()) {
3079 llvm::AttrBuilder &SRETAttrs = ArgAttrs[IRFunctionArgs.getSRetArgNo()];
3080 SRETAttrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: RetTy));
3081 SRETAttrs.addAttribute(Val: llvm::Attribute::Writable);
3082 SRETAttrs.addAttribute(Val: llvm::Attribute::DeadOnUnwind);
3083 hasUsedSRet = true;
3084 if (RetAI.getInReg())
3085 SRETAttrs.addAttribute(Val: llvm::Attribute::InReg);
3086 SRETAttrs.addAlignmentAttr(Align: RetAI.getIndirectAlign().getQuantity());
3087 }
3088
3089 // Attach attributes to inalloca argument.
3090 if (IRFunctionArgs.hasInallocaArg()) {
3091 ArgAttrs[IRFunctionArgs.getInallocaArgNo()].addInAllocaAttr(
3092 Ty: FI.getArgStruct());
3093 }
3094
3095 // Apply `nonnull`, `dereferenceable(N)` and `align N` to the `this` argument,
3096 // unless this is a thunk function. Add dead_on_return to the `this` argument
3097 // in base class destructors to aid in DSE.
3098 // FIXME: fix this properly, https://reviews.llvm.org/D100388
3099 if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() &&
3100 !FI.arg_begin()->type->isVoidPointerType() && !IsThunk) {
3101 auto IRArgs = IRFunctionArgs.getIRArgs(ArgNo: 0);
3102
3103 assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
3104
3105 llvm::AttrBuilder &Attrs = ArgAttrs[IRArgs.first];
3106
3107 QualType ThisTy = FI.arg_begin()->type.getTypePtr()->getPointeeType();
3108 int64_t ThisSz = getMinimumObjectSize(Ty: ThisTy).getQuantity();
3109
3110 if (!CodeGenOpts.NullPointerIsValid &&
3111 getTypes().getTargetAddressSpace(T: FI.arg_begin()->type) == 0) {
3112 Attrs.addAttribute(Val: llvm::Attribute::NonNull);
3113 Attrs.addDereferenceableAttr(Bytes: ThisSz);
3114 } else {
3115 // FIXME dereferenceable should be correct here, regardless of
3116 // NullPointerIsValid. However, dereferenceable currently does not always
3117 // respect NullPointerIsValid and may imply nonnull and break the program.
3118 // See https://reviews.llvm.org/D66618 for discussions.
3119 Attrs.addDereferenceableOrNullAttr(Bytes: ThisSz);
3120 }
3121
3122 llvm::Align Alignment =
3123 getNaturalTypeAlignment(T: ThisTy, /*BaseInfo=*/nullptr,
3124 /*TBAAInfo=*/nullptr, /*forPointeeType=*/true)
3125 .getAsAlign();
3126 Attrs.addAlignmentAttr(Align: Alignment);
3127
3128 const auto *DD = dyn_cast_if_present<CXXDestructorDecl>(
3129 Val: CalleeInfo.getCalleeDecl().getDecl());
3130 // Do not annotate vector deleting destructors with dead_on_return as the
3131 // this pointer in that case points to an array which we cannot
3132 // statically know the size of. Also do not mark deleting destructors
3133 // dead_on_return as then we might delete stores inside of a user-defined
3134 // operator delete implementation if it gets inlined, which would be
3135 // incorrect as the object's lifetime has already ended and the operator
3136 // delete implementation is allowed to manipulate the underlying storage.
3137 if (DD &&
3138 CalleeInfo.getCalleeDecl().getDtorType() !=
3139 CXXDtorType::Dtor_VectorDeleting &&
3140 CalleeInfo.getCalleeDecl().getDtorType() !=
3141 CXXDtorType::Dtor_Deleting &&
3142 CodeGenOpts.StrictLifetimes) {
3143 const CXXRecordDecl *ClassDecl =
3144 dyn_cast<CXXRecordDecl>(Val: DD->getDeclContext());
3145 // We cannot add dead_on_return if we have virtual base classes because
3146 // they will generally still be live after the base object destructor.
3147 if (ClassDecl->getNumVBases() == 0)
3148 Attrs.addDeadOnReturnAttr(Info: llvm::DeadOnReturnInfo(
3149 Context.getASTRecordLayout(D: ClassDecl).getDataSize().getQuantity()));
3150 }
3151 }
3152
3153 unsigned ArgNo = 0;
3154 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(), E = FI.arg_end();
3155 I != E; ++I, ++ArgNo) {
3156 QualType ParamType = I->type;
3157 const ABIArgInfo &AI = I->info;
3158 llvm::AttrBuilder Attrs(getLLVMContext());
3159
3160 // Add attribute for padding argument, if necessary.
3161 if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
3162 if (AI.getPaddingInReg()) {
3163 ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)].addAttribute(
3164 Val: llvm::Attribute::InReg);
3165 }
3166 }
3167
3168 // Decide whether the argument we're handling could be partially undef
3169 if (CodeGenOpts.EnableNoundefAttrs &&
3170 DetermineNoUndef(QTy: ParamType, Types&: getTypes(), DL, AI)) {
3171 Attrs.addAttribute(Val: llvm::Attribute::NoUndef);
3172 }
3173
3174 // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
3175 // have the corresponding parameter variable. It doesn't make
3176 // sense to do it here because parameters are so messed up.
3177 switch (AI.getKind()) {
3178 case ABIArgInfo::Extend:
3179 if (AI.isSignExt())
3180 Attrs.addAttribute(Val: llvm::Attribute::SExt);
3181 else if (AI.isZeroExt())
3182 Attrs.addAttribute(Val: llvm::Attribute::ZExt);
3183 else
3184 Attrs.addAttribute(Val: llvm::Attribute::NoExt);
3185 [[fallthrough]];
3186 case ABIArgInfo::TargetSpecific:
3187 case ABIArgInfo::Direct:
3188 if (ArgNo == 0 && FI.isChainCall())
3189 Attrs.addAttribute(Val: llvm::Attribute::Nest);
3190 else if (AI.getInReg())
3191 Attrs.addAttribute(Val: llvm::Attribute::InReg);
3192 Attrs.addStackAlignmentAttr(Align: llvm::MaybeAlign(AI.getDirectAlign()));
3193
3194 if (canApplyNoFPClass(AI, ParamType, IsReturn: false))
3195 Attrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
3196 break;
3197 case ABIArgInfo::Indirect: {
3198 assert(!ParamType->isIncompleteType() &&
3199 "Pass-by-value parameter has incomplete definition?");
3200
3201 if (AI.getInReg())
3202 Attrs.addAttribute(Val: llvm::Attribute::InReg);
3203
3204 // HLSL out and inout parameters must not be marked with ByVal or
3205 // DeadOnReturn attributes because stores to these parameters by the
3206 // callee are visible to the caller.
3207 if (auto ParamABI = FI.getExtParameterInfo(argIndex: ArgNo).getABI();
3208 ParamABI != ParameterABI::HLSLOut &&
3209 ParamABI != ParameterABI::HLSLInOut) {
3210
3211 // Depending on the ABI, this may be either a byval or a dead_on_return
3212 // argument.
3213 if (AI.getIndirectByVal()) {
3214 Attrs.addByValAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
3215 } else {
3216 // Add dead_on_return when the object's lifetime ends in the callee.
3217 // This includes trivially-destructible objects, as well as objects
3218 // whose destruction / clean-up is carried out within the callee
3219 // (e.g., Obj-C ARC-managed structs, MSVC callee-destroyed objects).
3220 if (!ParamType.isDestructedType() || !ParamType->isRecordType() ||
3221 ParamType->castAsRecordDecl()->isParamDestroyedInCallee())
3222 Attrs.addDeadOnReturnAttr(Info: llvm::DeadOnReturnInfo());
3223 }
3224 }
3225
3226 auto *Decl = ParamType->getAsRecordDecl();
3227 if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
3228 Decl->getArgPassingRestrictions() ==
3229 RecordArgPassingKind::CanPassInRegs)
3230 // When calling the function, the pointer passed in will be the only
3231 // reference to the underlying object. Mark it accordingly.
3232 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
3233
3234 // TODO: We could add the byref attribute if not byval, but it would
3235 // require updating many testcases.
3236
3237 CharUnits Align = AI.getIndirectAlign();
3238
3239 // In a byval argument, it is important that the required
3240 // alignment of the type is honored, as LLVM might be creating a
3241 // *new* stack object, and needs to know what alignment to give
3242 // it. (Sometimes it can deduce a sensible alignment on its own,
3243 // but not if clang decides it must emit a packed struct, or the
3244 // user specifies increased alignment requirements.)
3245 //
3246 // This is different from indirect *not* byval, where an aligned copy is
3247 // already created by the caller, and the align attribute is purely
3248 // informative. However, this can still be useful information for
3249 // optimizations, such as giving us one necessary condition for checking
3250 // if a load to this pointer can be speculatively executed.
3251 assert(!Align.isZero());
3252 Attrs.addAlignmentAttr(Align: Align.getQuantity());
3253
3254 // The `nofree` and `dereferenceable` attributes can already be inferred
3255 // for `byval` arguments. We'll need to provide additional hints
3256 // otherwise.
3257 if (!AI.getIndirectByVal()) {
3258 // Both 6.9.1 of the C standard and [basic.stc.auto] of the C++ standard
3259 // require parameters to have automatic storage duration. Therefore, the
3260 // underlying object of this pointer will not be freed during the
3261 // function's execution.
3262 Attrs.addAttribute(Val: llvm::Attribute::NoFreeObj);
3263 Attrs.addDereferenceableAttr(
3264 Bytes: Context.getTypeSizeInChars(T: ParamType).getQuantity());
3265 }
3266
3267 // byval disables readnone and readonly.
3268 AddPotentialArgAccess();
3269 break;
3270 }
3271 case ABIArgInfo::IndirectAliased: {
3272 CharUnits Align = AI.getIndirectAlign();
3273 Attrs.addByRefAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
3274 Attrs.addAlignmentAttr(Align: Align.getQuantity());
3275 break;
3276 }
3277 case ABIArgInfo::Ignore:
3278 case ABIArgInfo::Expand:
3279 case ABIArgInfo::CoerceAndExpand:
3280 break;
3281
3282 case ABIArgInfo::InAlloca:
3283 // inalloca disables readnone and readonly.
3284 AddPotentialArgAccess();
3285 continue;
3286 }
3287
3288 if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
3289 QualType PTy = RefTy->getPointeeType();
3290 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
3291 Attrs.addDereferenceableAttr(Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
3292 if (getTypes().getTargetAddressSpace(T: PTy) == 0 &&
3293 !CodeGenOpts.NullPointerIsValid)
3294 Attrs.addAttribute(Val: llvm::Attribute::NonNull);
3295 if (PTy->isObjectType()) {
3296 llvm::Align Alignment =
3297 getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
3298 Attrs.addAlignmentAttr(Align: Alignment);
3299 }
3300 }
3301
3302 // From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:
3303 // > For arguments to a __kernel function declared to be a pointer to a
3304 // > data type, the OpenCL compiler can assume that the pointee is always
3305 // > appropriately aligned as required by the data type.
3306 if (TargetDecl &&
3307 DeviceKernelAttr::isOpenCLSpelling(
3308 A: TargetDecl->getAttr<DeviceKernelAttr>()) &&
3309 ParamType->isPointerType()) {
3310 QualType PTy = ParamType->getPointeeType();
3311 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
3312 llvm::Align Alignment =
3313 getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
3314 Attrs.addAlignmentAttr(Align: Alignment);
3315 }
3316 }
3317
3318 switch (FI.getExtParameterInfo(argIndex: ArgNo).getABI()) {
3319 case ParameterABI::HLSLOut:
3320 case ParameterABI::HLSLInOut:
3321 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
3322 break;
3323 case ParameterABI::Ordinary:
3324 break;
3325
3326 case ParameterABI::SwiftIndirectResult: {
3327 // Add 'sret' if we haven't already used it for something, but
3328 // only if the result is void.
3329 if (!hasUsedSRet && RetTy->isVoidType()) {
3330 Attrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
3331 hasUsedSRet = true;
3332 }
3333
3334 // Add 'noalias' in either case.
3335 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
3336
3337 // Add 'dereferenceable' and 'alignment'.
3338 auto PTy = ParamType->getPointeeType();
3339 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
3340 auto info = getContext().getTypeInfoInChars(T: PTy);
3341 Attrs.addDereferenceableAttr(Bytes: info.Width.getQuantity());
3342 Attrs.addAlignmentAttr(Align: info.Align.getAsAlign());
3343 }
3344 break;
3345 }
3346
3347 case ParameterABI::SwiftErrorResult:
3348 Attrs.addAttribute(Val: llvm::Attribute::SwiftError);
3349 break;
3350
3351 case ParameterABI::SwiftContext:
3352 Attrs.addAttribute(Val: llvm::Attribute::SwiftSelf);
3353 break;
3354
3355 case ParameterABI::SwiftAsyncContext:
3356 Attrs.addAttribute(Val: llvm::Attribute::SwiftAsync);
3357 break;
3358 }
3359
3360 if (FI.getExtParameterInfo(argIndex: ArgNo).isNoEscape())
3361 Attrs.addCapturesAttr(
3362 CI: llvm::CaptureInfo(llvm::CaptureComponents::Address));
3363
3364 if (Attrs.hasAttributes()) {
3365 unsigned FirstIRArg, NumIRArgs;
3366 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3367 for (unsigned i = 0; i < NumIRArgs; i++)
3368 ArgAttrs[FirstIRArg + i].merge(B: Attrs);
3369 }
3370 }
3371 assert(ArgNo == FI.arg_size());
3372
3373 // We can't see all potential arguments in a varargs declaration; treat them
3374 // as if they can access memory.
3375 if (!AttrOnCallSite && FI.isVariadic())
3376 AddPotentialArgAccess();
3377
3378 ArgNo = 0;
3379 if (AddedPotentialArgAccess && MemAttrForPtrArgs) {
3380 llvm::FunctionType *FunctionType = getTypes().GetFunctionType(FI);
3381 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
3382 E = FI.arg_end();
3383 I != E; ++I, ++ArgNo) {
3384 if (I->info.isDirect() || I->info.isExpand() ||
3385 I->info.isCoerceAndExpand()) {
3386 unsigned FirstIRArg, NumIRArgs;
3387 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3388 for (unsigned i = FirstIRArg; i < FirstIRArg + NumIRArgs; ++i) {
3389 // The index may be out-of-bounds if the callee is a varargs
3390 // function.
3391 //
3392 // FIXME: We can compute the types of varargs arguments without going
3393 // through the function type, but the relevant code isn't exposed
3394 // in a way that can be called from here.
3395 if (i < FunctionType->getNumParams() &&
3396 FunctionType->getParamType(i)->isPointerTy()) {
3397 ArgAttrs[i].addAttribute(Val: *MemAttrForPtrArgs);
3398 }
3399 }
3400 }
3401 }
3402 }
3403
3404 SmallVector<llvm::AttributeSet, 4> ArgAttrSets;
3405 for (const llvm::AttrBuilder &Attrs : ArgAttrs)
3406 ArgAttrSets.push_back(Elt: llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs));
3407
3408 AttrList = llvm::AttributeList::get(
3409 C&: getLLVMContext(), FnAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: FuncAttrs),
3410 RetAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: RetAttrs), ArgAttrs: ArgAttrSets);
3411}
3412
3413/// An argument came in as a promoted argument; demote it back to its
3414/// declared type.
3415static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
3416 const VarDecl *var,
3417 llvm::Value *value) {
3418 llvm::Type *varType = CGF.ConvertType(T: var->getType());
3419
3420 // This can happen with promotions that actually don't change the
3421 // underlying type, like the enum promotions.
3422 if (value->getType() == varType)
3423 return value;
3424
3425 assert((varType->isIntegerTy() || varType->isFloatingPointTy()) &&
3426 "unexpected promotion type");
3427
3428 if (isa<llvm::IntegerType>(Val: varType))
3429 return CGF.Builder.CreateTrunc(V: value, DestTy: varType, Name: "arg.unpromote");
3430
3431 return CGF.Builder.CreateFPCast(V: value, DestTy: varType, Name: "arg.unpromote");
3432}
3433
3434/// Returns the attribute (either parameter attribute, or function
3435/// attribute), which declares argument ArgNo to be non-null.
3436static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
3437 QualType ArgType, unsigned ArgNo) {
3438 // FIXME: __attribute__((nonnull)) can also be applied to:
3439 // - references to pointers, where the pointee is known to be
3440 // nonnull (apparently a Clang extension)
3441 // - transparent unions containing pointers
3442 // In the former case, LLVM IR cannot represent the constraint. In
3443 // the latter case, we have no guarantee that the transparent union
3444 // is in fact passed as a pointer.
3445 if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType())
3446 return nullptr;
3447 // First, check attribute on parameter itself.
3448 if (PVD) {
3449 if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
3450 return ParmNNAttr;
3451 }
3452 // Check function attributes.
3453 if (!FD)
3454 return nullptr;
3455 for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
3456 if (NNAttr->isNonNull(IdxAST: ArgNo))
3457 return NNAttr;
3458 }
3459 return nullptr;
3460}
3461
3462namespace {
3463struct CopyBackSwiftError final : EHScopeStack::Cleanup {
3464 Address Temp;
3465 Address Arg;
3466 CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
3467 void Emit(CodeGenFunction &CGF, Flags flags) override {
3468 llvm::Value *errorValue = CGF.Builder.CreateLoad(Addr: Temp);
3469 CGF.Builder.CreateStore(Val: errorValue, Addr: Arg);
3470 }
3471};
3472} // namespace
3473
3474void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
3475 llvm::Function *Fn,
3476 const FunctionArgList &Args) {
3477 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())
3478 // Naked functions don't have prologues.
3479 return;
3480
3481 // If this is an implicit-return-zero function, go ahead and
3482 // initialize the return value. TODO: it might be nice to have
3483 // a more general mechanism for this that didn't require synthesized
3484 // return statements.
3485 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl)) {
3486 if (FD->hasImplicitReturnZero()) {
3487 QualType RetTy = FD->getReturnType().getUnqualifiedType();
3488 llvm::Type *LLVMTy = CGM.getTypes().ConvertType(T: RetTy);
3489 llvm::Constant *Zero = llvm::Constant::getNullValue(Ty: LLVMTy);
3490 Builder.CreateStore(Val: Zero, Addr: ReturnValue);
3491 }
3492 }
3493
3494 // FIXME: We no longer need the types from FunctionArgList; lift up and
3495 // simplify.
3496
3497 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
3498 assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
3499
3500 // If we're using inalloca, all the memory arguments are GEPs off of the last
3501 // parameter, which is a pointer to the complete memory area.
3502 Address ArgStruct = Address::invalid();
3503 if (IRFunctionArgs.hasInallocaArg())
3504 ArgStruct = Address(Fn->getArg(i: IRFunctionArgs.getInallocaArgNo()),
3505 FI.getArgStruct(), FI.getArgStructAlignment());
3506
3507 // Name the struct return parameter.
3508 if (IRFunctionArgs.hasSRetArg()) {
3509 auto AI = Fn->getArg(i: IRFunctionArgs.getSRetArgNo());
3510 AI->setName("agg.result");
3511 AI->addAttr(Kind: llvm::Attribute::NoAlias);
3512 }
3513
3514 // Track if we received the parameter as a pointer (indirect, byval, or
3515 // inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it
3516 // into a local alloca for us.
3517 SmallVector<ParamValue, 16> ArgVals;
3518 ArgVals.reserve(N: Args.size());
3519
3520 // Create a pointer value for every parameter declaration. This usually
3521 // entails copying one or more LLVM IR arguments into an alloca. Don't push
3522 // any cleanups or do anything that might unwind. We do that separately, so
3523 // we can push the cleanups in the correct order for the ABI.
3524 assert(FI.arg_size() == Args.size() &&
3525 "Mismatch between function signature & arguments.");
3526 unsigned ArgNo = 0;
3527 CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
3528 for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end(); i != e;
3529 ++i, ++info_it, ++ArgNo) {
3530 const VarDecl *Arg = *i;
3531 const ABIArgInfo &ArgI = info_it->info;
3532
3533 bool isPromoted =
3534 isa<ParmVarDecl>(Val: Arg) && cast<ParmVarDecl>(Val: Arg)->isKNRPromoted();
3535 // We are converting from ABIArgInfo type to VarDecl type directly, unless
3536 // the parameter is promoted. In this case we convert to
3537 // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
3538 QualType Ty = isPromoted ? info_it->type : Arg->getType();
3539 assert(hasScalarEvaluationKind(Ty) ==
3540 hasScalarEvaluationKind(Arg->getType()));
3541
3542 unsigned FirstIRArg, NumIRArgs;
3543 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3544
3545 switch (ArgI.getKind()) {
3546 case ABIArgInfo::InAlloca: {
3547 assert(NumIRArgs == 0);
3548 auto FieldIndex = ArgI.getInAllocaFieldIndex();
3549 Address V =
3550 Builder.CreateStructGEP(Addr: ArgStruct, Index: FieldIndex, Name: Arg->getName());
3551 if (ArgI.getInAllocaIndirect())
3552 V = Address(Builder.CreateLoad(Addr: V), ConvertTypeForMem(T: Ty),
3553 getContext().getTypeAlignInChars(T: Ty));
3554 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: V));
3555 break;
3556 }
3557
3558 case ABIArgInfo::Indirect:
3559 case ABIArgInfo::IndirectAliased: {
3560 assert(NumIRArgs == 1);
3561 Address ParamAddr = makeNaturalAddressForPointer(
3562 Ptr: Fn->getArg(i: FirstIRArg), T: Ty, Alignment: ArgI.getIndirectAlign(), ForPointeeType: false, BaseInfo: nullptr,
3563 TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
3564
3565 if (!hasScalarEvaluationKind(T: Ty)) {
3566 // Aggregates and complex variables are accessed by reference. All we
3567 // need to do is realign the value, if requested. Also, if the address
3568 // may be aliased, copy it to ensure that the parameter variable is
3569 // mutable and has a unique adress, as C requires.
3570 if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) {
3571 RawAddress AlignedTemp = CreateMemTempWithoutCast(T: Ty, Name: "coerce");
3572
3573 // Copy from the incoming argument pointer to the temporary with the
3574 // appropriate alignment.
3575 //
3576 // FIXME: We should have a common utility for generating an aggregate
3577 // copy.
3578 CharUnits Size = getContext().getTypeSizeInChars(T: Ty);
3579 Builder.CreateMemCpy(
3580 Dst: AlignedTemp.getPointer(), DstAlign: AlignedTemp.getAlignment().getAsAlign(),
3581 Src: ParamAddr.emitRawPointer(CGF&: *this),
3582 SrcAlign: ParamAddr.getAlignment().getAsAlign(),
3583 Size: llvm::ConstantInt::get(Ty: IntPtrTy, V: Size.getQuantity()));
3584 ParamAddr = AlignedTemp;
3585 }
3586 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: ParamAddr));
3587 } else {
3588 // Load scalar value from indirect argument.
3589 llvm::Value *V =
3590 EmitLoadOfScalar(Addr: ParamAddr, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3591
3592 if (isPromoted)
3593 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3594 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3595 }
3596 break;
3597 }
3598
3599 case ABIArgInfo::Extend:
3600 case ABIArgInfo::Direct: {
3601 auto AI = Fn->getArg(i: FirstIRArg);
3602 llvm::Type *LTy = ConvertType(T: Arg->getType());
3603
3604 // Prepare parameter attributes. So far, only attributes for pointer
3605 // parameters are prepared. See
3606 // http://llvm.org/docs/LangRef.html#paramattrs.
3607 if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy() &&
3608 ArgI.getCoerceToType()->isPointerTy()) {
3609 assert(NumIRArgs == 1);
3610
3611 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: Arg)) {
3612 // Set `nonnull` attribute if any.
3613 if (getNonNullAttr(FD: CurCodeDecl, PVD, ArgType: PVD->getType(),
3614 ArgNo: PVD->getFunctionScopeIndex()) &&
3615 !CGM.getCodeGenOpts().NullPointerIsValid)
3616 AI->addAttr(Kind: llvm::Attribute::NonNull);
3617
3618 QualType OTy = PVD->getOriginalType();
3619 if (const auto *ArrTy = getContext().getAsConstantArrayType(T: OTy)) {
3620 // A C99 array parameter declaration with the static keyword also
3621 // indicates dereferenceability, and if the size is constant we can
3622 // use the dereferenceable attribute (which requires the size in
3623 // bytes).
3624 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3625 QualType ETy = ArrTy->getElementType();
3626 llvm::Align Alignment =
3627 CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3628 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3629 .addAlignmentAttr(Align: Alignment));
3630 uint64_t ArrSize = ArrTy->getZExtSize();
3631 if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
3632 ArrSize) {
3633 llvm::AttrBuilder Attrs(getLLVMContext());
3634 Attrs.addDereferenceableAttr(
3635 Bytes: getContext().getTypeSizeInChars(T: ETy).getQuantity() *
3636 ArrSize);
3637 AI->addAttrs(B&: Attrs);
3638 } else if (getContext().getTargetInfo().getNullPointerValue(
3639 AddrSpace: ETy.getAddressSpace()) == 0 &&
3640 !CGM.getCodeGenOpts().NullPointerIsValid) {
3641 AI->addAttr(Kind: llvm::Attribute::NonNull);
3642 }
3643 }
3644 } else if (const auto *ArrTy =
3645 getContext().getAsVariableArrayType(T: OTy)) {
3646 // For C99 VLAs with the static keyword, we don't know the size so
3647 // we can't use the dereferenceable attribute, but in addrspace(0)
3648 // we know that it must be nonnull.
3649 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3650 QualType ETy = ArrTy->getElementType();
3651 llvm::Align Alignment =
3652 CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3653 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3654 .addAlignmentAttr(Align: Alignment));
3655 if (!getTypes().getTargetAddressSpace(T: ETy) &&
3656 !CGM.getCodeGenOpts().NullPointerIsValid)
3657 AI->addAttr(Kind: llvm::Attribute::NonNull);
3658 }
3659 }
3660
3661 // Set `align` attribute if any.
3662 const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
3663 if (!AVAttr)
3664 if (const auto *TOTy = OTy->getAs<TypedefType>())
3665 AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
3666 if (AVAttr && !SanOpts.has(K: SanitizerKind::Alignment)) {
3667 // If alignment-assumption sanitizer is enabled, we do *not* add
3668 // alignment attribute here, but emit normal alignment assumption,
3669 // so the UBSAN check could function.
3670 llvm::ConstantInt *AlignmentCI =
3671 cast<llvm::ConstantInt>(Val: EmitScalarExpr(E: AVAttr->getAlignment()));
3672 uint64_t AlignmentInt =
3673 AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment);
3674 if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
3675 AI->removeAttr(Kind: llvm::Attribute::AttrKind::Alignment);
3676 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3677 .addAlignmentAttr(Align: llvm::Align(AlignmentInt)));
3678 }
3679 }
3680 }
3681
3682 // Set 'noalias' if an argument type has the `restrict` qualifier.
3683 if (Arg->getType().isRestrictQualified())
3684 AI->addAttr(Kind: llvm::Attribute::NoAlias);
3685 }
3686
3687 // Prepare the argument value. If we have the trivial case, handle it
3688 // with no muss and fuss.
3689 if (!isa<llvm::StructType>(Val: ArgI.getCoerceToType()) &&
3690 ArgI.getCoerceToType() == ConvertType(T: Ty) &&
3691 ArgI.getDirectOffset() == 0) {
3692 assert(NumIRArgs == 1);
3693
3694 // LLVM expects swifterror parameters to be used in very restricted
3695 // ways. Copy the value into a less-restricted temporary.
3696 llvm::Value *V = AI;
3697 if (FI.getExtParameterInfo(argIndex: ArgNo).getABI() ==
3698 ParameterABI::SwiftErrorResult) {
3699 QualType pointeeTy = Ty->getPointeeType();
3700 assert(pointeeTy->isPointerType());
3701 RawAddress temp = CreateMemTempWithoutCast(
3702 T: pointeeTy, Align: getPointerAlign(), Name: "swifterror.temp");
3703 Address arg = makeNaturalAddressForPointer(
3704 Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
3705 llvm::Value *incomingErrorValue = Builder.CreateLoad(Addr: arg);
3706 Builder.CreateStore(Val: incomingErrorValue, Addr: temp);
3707 V = temp.getPointer();
3708
3709 // Push a cleanup to copy the value back at the end of the function.
3710 // The convention does not guarantee that the value will be written
3711 // back if the function exits with an unwind exception.
3712 EHStack.pushCleanup<CopyBackSwiftError>(Kind: NormalCleanup, A: temp, A: arg);
3713 }
3714
3715 // Ensure the argument is the correct type.
3716 if (V->getType() != ArgI.getCoerceToType())
3717 V = Builder.CreateBitCast(V, DestTy: ArgI.getCoerceToType());
3718
3719 if (isPromoted)
3720 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3721
3722 // Because of merging of function types from multiple decls it is
3723 // possible for the type of an argument to not match the corresponding
3724 // type in the function type. Since we are codegening the callee
3725 // in here, add a cast to the argument type.
3726 llvm::Type *LTy = ConvertType(T: Arg->getType());
3727 if (V->getType() != LTy)
3728 V = Builder.CreateBitCast(V, DestTy: LTy);
3729
3730 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3731 break;
3732 }
3733
3734 // VLST arguments are coerced to VLATs at the function boundary for
3735 // ABI consistency. If this is a VLST that was coerced to
3736 // a VLAT at the function boundary and the types match up, use
3737 // llvm.vector.extract to convert back to the original VLST.
3738 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: ConvertType(T: Ty))) {
3739 llvm::Value *ArgVal = Fn->getArg(i: FirstIRArg);
3740 if (auto *VecTyFrom =
3741 dyn_cast<llvm::ScalableVectorType>(Val: ArgVal->getType())) {
3742 auto [Coerced, Extracted] = CoerceScalableToFixed(
3743 CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: ArgVal, Name: Arg->getName());
3744 if (Extracted) {
3745 assert(NumIRArgs == 1);
3746 ArgVals.push_back(Elt: ParamValue::forDirect(value: Coerced));
3747 break;
3748 }
3749 }
3750 }
3751
3752 llvm::StructType *STy =
3753 dyn_cast<llvm::StructType>(Val: ArgI.getCoerceToType());
3754 Address Alloca = CreateMemTempWithoutCast(
3755 T: Ty, Align: getContext().getDeclAlign(D: Arg), Name: Arg->getName());
3756
3757 // Pointer to store into.
3758 Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3759
3760 // Fast-isel and the optimizer generally like scalar values better than
3761 // FCAs, so we flatten them if this is safe to do for this argument.
3762 if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
3763 STy->getNumElements() > 1) {
3764 llvm::TypeSize StructSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
3765 llvm::TypeSize PtrElementSize =
3766 CGM.getDataLayout().getTypeAllocSize(Ty: Ptr.getElementType());
3767 if (StructSize.isScalable()) {
3768 assert(STy->containsHomogeneousScalableVectorTypes() &&
3769 "ABI only supports structure with homogeneous scalable vector "
3770 "type");
3771 assert(StructSize == PtrElementSize &&
3772 "Only allow non-fractional movement of structure with"
3773 "homogeneous scalable vector type");
3774 assert(STy->getNumElements() == NumIRArgs);
3775
3776 llvm::Value *LoadedStructValue = llvm::PoisonValue::get(T: STy);
3777 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3778 auto *AI = Fn->getArg(i: FirstIRArg + i);
3779 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3780 LoadedStructValue =
3781 Builder.CreateInsertValue(Agg: LoadedStructValue, Val: AI, Idxs: i);
3782 }
3783
3784 Builder.CreateStore(Val: LoadedStructValue, Addr: Ptr);
3785 } else {
3786 uint64_t SrcSize = StructSize.getFixedValue();
3787 uint64_t DstSize = PtrElementSize.getFixedValue();
3788
3789 Address AddrToStoreInto = Address::invalid();
3790 if (SrcSize <= DstSize) {
3791 AddrToStoreInto = Ptr.withElementType(ElemTy: STy);
3792 } else {
3793 AddrToStoreInto =
3794 CreateTempAlloca(Ty: STy, align: Alloca.getAlignment(), Name: "coerce");
3795 }
3796
3797 assert(STy->getNumElements() == NumIRArgs);
3798 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3799 auto AI = Fn->getArg(i: FirstIRArg + i);
3800 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3801 Address EltPtr = Builder.CreateStructGEP(Addr: AddrToStoreInto, Index: i);
3802 Builder.CreateStore(Val: AI, Addr: EltPtr);
3803 }
3804
3805 if (SrcSize > DstSize) {
3806 Builder.CreateMemCpy(Dest: Ptr, Src: AddrToStoreInto, Size: DstSize);
3807 }
3808
3809 // Structures with PFP fields require a coerced store to add any
3810 // pointer signatures.
3811 if (getContext().hasPFPFields(Ty)) {
3812 llvm::Value *Struct = Builder.CreateLoad(Addr: Ptr);
3813 CreatePFPCoercedStore(Src: Struct, SrcFETy: Ty, Dst: Ptr, CGF&: *this);
3814 }
3815 }
3816 } else {
3817 // Simple case, just do a coerced store of the argument into the alloca.
3818 assert(NumIRArgs == 1);
3819 auto AI = Fn->getArg(i: FirstIRArg);
3820 AI->setName(Arg->getName() + ".coerce");
3821 CreateCoercedStore(
3822 Src: AI, SrcFETy: Ty, Dst: Ptr,
3823 DstSize: llvm::TypeSize::getFixed(
3824 ExactSize: getContext().getTypeSizeInChars(T: Ty).getQuantity() -
3825 ArgI.getDirectOffset()),
3826 /*DstIsVolatile=*/false);
3827 }
3828
3829 // Match to what EmitParmDecl is expecting for this type.
3830 if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3831 llvm::Value *V =
3832 EmitLoadOfScalar(Addr: Alloca, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3833 if (isPromoted)
3834 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3835 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3836 } else {
3837 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3838 }
3839 break;
3840 }
3841
3842 case ABIArgInfo::CoerceAndExpand: {
3843 // Reconstruct into a temporary.
3844 Address alloca =
3845 CreateMemTempWithoutCast(T: Ty, Align: getContext().getDeclAlign(D: Arg));
3846 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: alloca));
3847
3848 auto coercionType = ArgI.getCoerceAndExpandType();
3849 auto unpaddedCoercionType = ArgI.getUnpaddedCoerceAndExpandType();
3850 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
3851
3852 alloca = alloca.withElementType(ElemTy: coercionType);
3853
3854 unsigned argIndex = FirstIRArg;
3855 unsigned unpaddedIndex = 0;
3856 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3857 llvm::Type *eltType = coercionType->getElementType(N: i);
3858 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
3859 continue;
3860
3861 auto eltAddr = Builder.CreateStructGEP(Addr: alloca, Index: i);
3862 llvm::Value *elt = Fn->getArg(i: argIndex++);
3863
3864 auto paramType = unpaddedStruct
3865 ? unpaddedStruct->getElementType(N: unpaddedIndex++)
3866 : unpaddedCoercionType;
3867
3868 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: eltType)) {
3869 if (auto *VecTyFrom = dyn_cast<llvm::ScalableVectorType>(Val: paramType)) {
3870 bool Extracted;
3871 std::tie(args&: elt, args&: Extracted) = CoerceScalableToFixed(
3872 CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: elt, Name: elt->getName());
3873 assert(Extracted && "Unexpected scalable to fixed vector coercion");
3874 }
3875 }
3876 Builder.CreateStore(Val: elt, Addr: eltAddr);
3877 }
3878 assert(argIndex == FirstIRArg + NumIRArgs);
3879 break;
3880 }
3881
3882 case ABIArgInfo::Expand: {
3883 // If this structure was expanded into multiple arguments then
3884 // we need to create a temporary and reconstruct it from the
3885 // arguments.
3886 Address Alloca =
3887 CreateMemTempWithoutCast(T: Ty, Align: getContext().getDeclAlign(D: Arg));
3888 LValue LV = MakeAddrLValue(Addr: Alloca, T: Ty);
3889 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3890
3891 auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3892 ExpandTypeFromArgs(Ty, LV, AI&: FnArgIter);
3893 assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3894 for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
3895 auto AI = Fn->getArg(i: FirstIRArg + i);
3896 AI->setName(Arg->getName() + "." + Twine(i));
3897 }
3898 break;
3899 }
3900
3901 case ABIArgInfo::TargetSpecific: {
3902 auto *AI = Fn->getArg(i: FirstIRArg);
3903 AI->setName(Arg->getName() + ".target_coerce");
3904 Address Alloca = CreateMemTempWithoutCast(
3905 T: Ty, Align: getContext().getDeclAlign(D: Arg), Name: Arg->getName());
3906 Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3907 CGM.getABIInfo().createCoercedStore(Val: AI, DstAddr: Ptr, AI: ArgI, DestIsVolatile: false, CGF&: *this);
3908 if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3909 llvm::Value *V =
3910 EmitLoadOfScalar(Addr: Alloca, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3911 if (isPromoted) {
3912 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3913 }
3914 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3915 } else {
3916 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3917 }
3918 break;
3919 }
3920 case ABIArgInfo::Ignore:
3921 assert(NumIRArgs == 0);
3922 // Initialize the local variable appropriately.
3923 if (!hasScalarEvaluationKind(T: Ty)) {
3924 ArgVals.push_back(
3925 Elt: ParamValue::forIndirect(addr: CreateMemTempWithoutCast(T: Ty)));
3926 } else {
3927 llvm::Value *U = llvm::UndefValue::get(T: ConvertType(T: Arg->getType()));
3928 ArgVals.push_back(Elt: ParamValue::forDirect(value: U));
3929 }
3930 break;
3931 }
3932 }
3933
3934 if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3935 for (int I = Args.size() - 1; I >= 0; --I)
3936 EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3937 } else {
3938 for (unsigned I = 0, E = Args.size(); I != E; ++I)
3939 EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3940 }
3941}
3942
3943static void eraseUnusedBitCasts(llvm::Instruction *insn) {
3944 while (insn->use_empty()) {
3945 llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: insn);
3946 if (!bitcast)
3947 return;
3948
3949 // This is "safe" because we would have used a ConstantExpr otherwise.
3950 insn = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3951 bitcast->eraseFromParent();
3952 }
3953}
3954
3955/// Try to emit a fused autorelease of a return result.
3956static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
3957 llvm::Value *result) {
3958 // We must be immediately followed the cast.
3959 llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3960 if (BB->empty())
3961 return nullptr;
3962 if (&BB->back() != result)
3963 return nullptr;
3964
3965 llvm::Type *resultType = result->getType();
3966
3967 // result is in a BasicBlock and is therefore an Instruction.
3968 llvm::Instruction *generator = cast<llvm::Instruction>(Val: result);
3969
3970 SmallVector<llvm::Instruction *, 4> InstsToKill;
3971
3972 // Look for:
3973 // %generator = bitcast %type1* %generator2 to %type2*
3974 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: generator)) {
3975 // We would have emitted this as a constant if the operand weren't
3976 // an Instruction.
3977 generator = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3978
3979 // Require the generator to be immediately followed by the cast.
3980 if (generator->getNextNode() != bitcast)
3981 return nullptr;
3982
3983 InstsToKill.push_back(Elt: bitcast);
3984 }
3985
3986 // Look for:
3987 // %generator = call i8* @objc_retain(i8* %originalResult)
3988 // or
3989 // %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
3990 llvm::CallInst *call = dyn_cast<llvm::CallInst>(Val: generator);
3991 if (!call)
3992 return nullptr;
3993
3994 bool doRetainAutorelease;
3995
3996 if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
3997 doRetainAutorelease = true;
3998 } else if (call->getCalledOperand() ==
3999 CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
4000 doRetainAutorelease = false;
4001
4002 // If we emitted an assembly marker for this call (and the
4003 // ARCEntrypoints field should have been set if so), go looking
4004 // for that call. If we can't find it, we can't do this
4005 // optimization. But it should always be the immediately previous
4006 // instruction, unless we needed bitcasts around the call.
4007 if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
4008 llvm::Instruction *prev = call->getPrevNode();
4009 assert(prev);
4010 if (isa<llvm::BitCastInst>(Val: prev)) {
4011 prev = prev->getPrevNode();
4012 assert(prev);
4013 }
4014 assert(isa<llvm::CallInst>(prev));
4015 assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
4016 CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
4017 InstsToKill.push_back(Elt: prev);
4018 }
4019 } else {
4020 return nullptr;
4021 }
4022
4023 result = call->getArgOperand(i: 0);
4024 InstsToKill.push_back(Elt: call);
4025
4026 // Keep killing bitcasts, for sanity. Note that we no longer care
4027 // about precise ordering as long as there's exactly one use.
4028 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: result)) {
4029 if (!bitcast->hasOneUse())
4030 break;
4031 InstsToKill.push_back(Elt: bitcast);
4032 result = bitcast->getOperand(i_nocapture: 0);
4033 }
4034
4035 // Delete all the unnecessary instructions, from latest to earliest.
4036 for (auto *I : InstsToKill)
4037 I->eraseFromParent();
4038
4039 // Do the fused retain/autorelease if we were asked to.
4040 if (doRetainAutorelease)
4041 result = CGF.EmitARCRetainAutoreleaseReturnValue(value: result);
4042
4043 // Cast back to the result type.
4044 return CGF.Builder.CreateBitCast(V: result, DestTy: resultType);
4045}
4046
4047/// If this is a +1 of the value of an immutable 'self', remove it.
4048static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
4049 llvm::Value *result) {
4050 // This is only applicable to a method with an immutable 'self'.
4051 const ObjCMethodDecl *method =
4052 dyn_cast_or_null<ObjCMethodDecl>(Val: CGF.CurCodeDecl);
4053 if (!method)
4054 return nullptr;
4055 const VarDecl *self = method->getSelfDecl();
4056 if (!self->getType().isConstQualified())
4057 return nullptr;
4058
4059 // Look for a retain call. Note: stripPointerCasts looks through returned arg
4060 // functions, which would cause us to miss the retain.
4061 llvm::CallInst *retainCall = dyn_cast<llvm::CallInst>(Val: result);
4062 if (!retainCall || retainCall->getCalledOperand() !=
4063 CGF.CGM.getObjCEntrypoints().objc_retain)
4064 return nullptr;
4065
4066 // Look for an ordinary load of 'self'.
4067 llvm::Value *retainedValue = retainCall->getArgOperand(i: 0);
4068 llvm::LoadInst *load =
4069 dyn_cast<llvm::LoadInst>(Val: retainedValue->stripPointerCasts());
4070 if (!load || load->isAtomic() || load->isVolatile() ||
4071 load->getPointerOperand() != CGF.GetAddrOfLocalVar(VD: self).getBasePointer())
4072 return nullptr;
4073
4074 // Okay! Burn it all down. This relies for correctness on the
4075 // assumption that the retain is emitted as part of the return and
4076 // that thereafter everything is used "linearly".
4077 llvm::Type *resultType = result->getType();
4078 eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: result));
4079 assert(retainCall->use_empty());
4080 retainCall->eraseFromParent();
4081 eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: retainedValue));
4082
4083 return CGF.Builder.CreateBitCast(V: load, DestTy: resultType);
4084}
4085
4086/// Emit an ARC autorelease of the result of a function.
4087///
4088/// \return the value to actually return from the function
4089static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
4090 llvm::Value *result) {
4091 // If we're returning 'self', kill the initial retain. This is a
4092 // heuristic attempt to "encourage correctness" in the really unfortunate
4093 // case where we have a return of self during a dealloc and we desperately
4094 // need to avoid the possible autorelease.
4095 if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
4096 return self;
4097
4098 // At -O0, try to emit a fused retain/autorelease.
4099 if (CGF.shouldUseFusedARCCalls())
4100 if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
4101 return fused;
4102
4103 return CGF.EmitARCAutoreleaseReturnValue(value: result);
4104}
4105
4106/// Heuristically search for a dominating store to the return-value slot.
4107static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
4108 llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer();
4109
4110 // Check if a User is a store which pointerOperand is the ReturnValue.
4111 // We are looking for stores to the ReturnValue, not for stores of the
4112 // ReturnValue to some other location.
4113 auto GetStoreIfValid = [&CGF,
4114 ReturnValuePtr](llvm::User *U) -> llvm::StoreInst * {
4115 auto *SI = dyn_cast<llvm::StoreInst>(Val: U);
4116 if (!SI || SI->getPointerOperand() != ReturnValuePtr ||
4117 SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())
4118 return nullptr;
4119 // These aren't actually possible for non-coerced returns, and we
4120 // only care about non-coerced returns on this code path.
4121 // All memory instructions inside __try block are volatile.
4122 assert(!SI->isAtomic() &&
4123 (!SI->isVolatile() || CGF.currentFunctionUsesSEHTry()));
4124 return SI;
4125 };
4126 // If there are multiple uses of the return-value slot, just check
4127 // for something immediately preceding the IP. Sometimes this can
4128 // happen with how we generate implicit-returns; it can also happen
4129 // with noreturn cleanups.
4130 if (!ReturnValuePtr->hasOneUse()) {
4131 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
4132 if (IP->empty())
4133 return nullptr;
4134
4135 // Look at directly preceding instruction, skipping bitcasts, lifetime
4136 // markers, and fake uses and their operands.
4137 const llvm::Instruction *LoadIntoFakeUse = nullptr;
4138 for (llvm::Instruction &I : llvm::reverse(C&: *IP)) {
4139 // Ignore instructions that are just loads for fake uses; the load should
4140 // immediately precede the fake use, so we only need to remember the
4141 // operand for the last fake use seen.
4142 if (LoadIntoFakeUse == &I)
4143 continue;
4144 if (isa<llvm::BitCastInst>(Val: &I))
4145 continue;
4146 if (auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &I)) {
4147 if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
4148 continue;
4149
4150 if (II->getIntrinsicID() == llvm::Intrinsic::fake_use) {
4151 LoadIntoFakeUse = dyn_cast<llvm::Instruction>(Val: II->getArgOperand(i: 0));
4152 continue;
4153 }
4154 }
4155 return GetStoreIfValid(&I);
4156 }
4157 return nullptr;
4158 }
4159
4160 llvm::StoreInst *store = GetStoreIfValid(ReturnValuePtr->user_back());
4161 if (!store)
4162 return nullptr;
4163
4164 // Now do a first-and-dirty dominance check: just walk up the
4165 // single-predecessors chain from the current insertion point.
4166 llvm::BasicBlock *StoreBB = store->getParent();
4167 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
4168 llvm::SmallPtrSet<llvm::BasicBlock *, 4> SeenBBs;
4169 while (IP != StoreBB) {
4170 if (!SeenBBs.insert(Ptr: IP).second || !(IP = IP->getSinglePredecessor()))
4171 return nullptr;
4172 }
4173
4174 // Okay, the store's basic block dominates the insertion point; we
4175 // can do our thing.
4176 return store;
4177}
4178
4179// Helper functions for EmitCMSEClearRecord
4180
4181// Set the bits corresponding to a field having width `BitWidth` and located at
4182// offset `BitOffset` (from the least significant bit) within a storage unit of
4183// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
4184// Use little-endian layout, i.e.`Bits[0]` is the LSB.
4185static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
4186 int BitWidth, int CharWidth) {
4187 assert(CharWidth <= 64);
4188 assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
4189
4190 int Pos = 0;
4191 if (BitOffset >= CharWidth) {
4192 Pos += BitOffset / CharWidth;
4193 BitOffset = BitOffset % CharWidth;
4194 }
4195
4196 const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
4197 if (BitOffset + BitWidth >= CharWidth) {
4198 Bits[Pos++] |= (Used << BitOffset) & Used;
4199 BitWidth -= CharWidth - BitOffset;
4200 BitOffset = 0;
4201 }
4202
4203 while (BitWidth >= CharWidth) {
4204 Bits[Pos++] = Used;
4205 BitWidth -= CharWidth;
4206 }
4207
4208 if (BitWidth > 0)
4209 Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
4210}
4211
4212// Set the bits corresponding to a field having width `BitWidth` and located at
4213// offset `BitOffset` (from the least significant bit) within a storage unit of
4214// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
4215// `Bits` corresponds to one target byte. Use target endian layout.
4216static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
4217 int StorageSize, int BitOffset, int BitWidth,
4218 int CharWidth, bool BigEndian) {
4219
4220 SmallVector<uint64_t, 8> TmpBits(StorageSize);
4221 setBitRange(Bits&: TmpBits, BitOffset, BitWidth, CharWidth);
4222
4223 if (BigEndian)
4224 std::reverse(first: TmpBits.begin(), last: TmpBits.end());
4225
4226 for (uint64_t V : TmpBits)
4227 Bits[StorageOffset++] |= V;
4228}
4229
4230static void setUsedBits(CodeGenModule &, QualType, int,
4231 SmallVectorImpl<uint64_t> &);
4232
4233// Set the bits in `Bits`, which correspond to the value representations of
4234// the actual members of the record type `RTy`. Note that this function does
4235// not handle base classes, virtual tables, etc, since they cannot happen in
4236// CMSE function arguments or return. The bit mask corresponds to the target
4237// memory layout, i.e. it's endian dependent.
4238static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
4239 SmallVectorImpl<uint64_t> &Bits) {
4240 ASTContext &Context = CGM.getContext();
4241 int CharWidth = Context.getCharWidth();
4242 const RecordDecl *RD = RTy->getDecl()->getDefinition();
4243 const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(D: RD);
4244 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
4245
4246 int Idx = 0;
4247 for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) {
4248 const FieldDecl *F = *I;
4249
4250 if (F->isUnnamedBitField() || F->isZeroLengthBitField() ||
4251 F->getType()->isIncompleteArrayType())
4252 continue;
4253
4254 if (F->isBitField()) {
4255 const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(FD: F);
4256 setBitRange(Bits, StorageOffset: Offset + BFI.StorageOffset.getQuantity(),
4257 StorageSize: BFI.StorageSize / CharWidth, BitOffset: BFI.Offset, BitWidth: BFI.Size, CharWidth,
4258 BigEndian: CGM.getDataLayout().isBigEndian());
4259 continue;
4260 }
4261
4262 setUsedBits(CGM, F->getType(),
4263 Offset + ASTLayout.getFieldOffset(FieldNo: Idx) / CharWidth, Bits);
4264 }
4265}
4266
4267// Set the bits in `Bits`, which correspond to the value representations of
4268// the elements of an array type `ATy`.
4269static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
4270 int Offset, SmallVectorImpl<uint64_t> &Bits) {
4271 const ASTContext &Context = CGM.getContext();
4272
4273 QualType ETy = Context.getBaseElementType(VAT: ATy);
4274 int Size = Context.getTypeSizeInChars(T: ETy).getQuantity();
4275 SmallVector<uint64_t, 4> TmpBits(Size);
4276 setUsedBits(CGM, ETy, 0, TmpBits);
4277
4278 for (int I = 0, N = Context.getConstantArrayElementCount(CA: ATy); I < N; ++I) {
4279 auto Src = TmpBits.begin();
4280 auto Dst = Bits.begin() + Offset + I * Size;
4281 for (int J = 0; J < Size; ++J)
4282 *Dst++ |= *Src++;
4283 }
4284}
4285
4286// Set the bits in `Bits`, which correspond to the value representations of
4287// the type `QTy`.
4288static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
4289 SmallVectorImpl<uint64_t> &Bits) {
4290 if (const auto *RTy = QTy->getAsCanonical<RecordType>())
4291 return setUsedBits(CGM, RTy, Offset, Bits);
4292
4293 ASTContext &Context = CGM.getContext();
4294 if (const auto *ATy = Context.getAsConstantArrayType(T: QTy))
4295 return setUsedBits(CGM, ATy, Offset, Bits);
4296
4297 int Size = Context.getTypeSizeInChars(T: QTy).getQuantity();
4298 if (Size <= 0)
4299 return;
4300
4301 std::fill_n(first: Bits.begin() + Offset, n: Size,
4302 value: (uint64_t(1) << Context.getCharWidth()) - 1);
4303}
4304
4305static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
4306 int Pos, int Size, int CharWidth,
4307 bool BigEndian) {
4308 assert(Size > 0);
4309 uint64_t Mask = 0;
4310 if (BigEndian) {
4311 for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
4312 ++P)
4313 Mask = (Mask << CharWidth) | *P;
4314 } else {
4315 auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
4316 do
4317 Mask = (Mask << CharWidth) | *--P;
4318 while (P != End);
4319 }
4320 return Mask;
4321}
4322
4323// Emit code to clear the bits in a record, which aren't a part of any user
4324// declared member, when the record is a function return.
4325llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
4326 llvm::IntegerType *ITy,
4327 QualType QTy) {
4328 assert(Src->getType() == ITy);
4329 assert(ITy->getScalarSizeInBits() <= 64);
4330
4331 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
4332 int Size = DataLayout.getTypeStoreSize(Ty: ITy);
4333 SmallVector<uint64_t, 4> Bits(Size);
4334 setUsedBits(CGM, RTy: QTy->castAsCanonical<RecordType>(), Offset: 0, Bits);
4335
4336 int CharWidth = CGM.getContext().getCharWidth();
4337 uint64_t Mask =
4338 buildMultiCharMask(Bits, Pos: 0, Size, CharWidth, BigEndian: DataLayout.isBigEndian());
4339
4340 return Builder.CreateAnd(LHS: Src, RHS: Mask, Name: "cmse.clear");
4341}
4342
4343// Emit code to clear the bits in a record, which aren't a part of any user
4344// declared member, when the record is a function argument.
4345llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
4346 llvm::ArrayType *ATy,
4347 QualType QTy) {
4348 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
4349 int Size = DataLayout.getTypeStoreSize(Ty: ATy);
4350 SmallVector<uint64_t, 16> Bits(Size);
4351 setUsedBits(CGM, RTy: QTy->castAsCanonical<RecordType>(), Offset: 0, Bits);
4352
4353 // Clear each element of the LLVM array.
4354 int CharWidth = CGM.getContext().getCharWidth();
4355 int CharsPerElt =
4356 ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
4357 int MaskIndex = 0;
4358 llvm::Value *R = llvm::PoisonValue::get(T: ATy);
4359 for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
4360 uint64_t Mask = buildMultiCharMask(Bits, Pos: MaskIndex, Size: CharsPerElt, CharWidth,
4361 BigEndian: DataLayout.isBigEndian());
4362 MaskIndex += CharsPerElt;
4363 llvm::Value *T0 = Builder.CreateExtractValue(Agg: Src, Idxs: I);
4364 llvm::Value *T1 = Builder.CreateAnd(LHS: T0, RHS: Mask, Name: "cmse.clear");
4365 R = Builder.CreateInsertValue(Agg: R, Val: T1, Idxs: I);
4366 }
4367
4368 return R;
4369}
4370
4371void CodeGenFunction::EmitFunctionEpilog(
4372 const CGFunctionInfo &FI, bool EmitRetDbgLoc, SourceLocation EndLoc,
4373 uint64_t RetKeyInstructionsSourceAtom) {
4374 if (FI.isNoReturn()) {
4375 // Noreturn functions don't return.
4376 EmitUnreachable(Loc: EndLoc);
4377 return;
4378 }
4379
4380 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {
4381 // Naked functions don't have epilogues.
4382 Builder.CreateUnreachable();
4383 return;
4384 }
4385
4386 // Functions with no result always return void.
4387 if (!ReturnValue.isValid()) {
4388 auto *I = Builder.CreateRetVoid();
4389 if (RetKeyInstructionsSourceAtom)
4390 addInstToSpecificSourceAtom(KeyInstruction: I, Backup: nullptr, Atom: RetKeyInstructionsSourceAtom);
4391 else
4392 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4393 return;
4394 }
4395
4396 llvm::DebugLoc RetDbgLoc;
4397 llvm::Value *RV = nullptr;
4398 QualType RetTy = FI.getReturnType();
4399 const ABIArgInfo &RetAI = FI.getReturnInfo();
4400
4401 switch (RetAI.getKind()) {
4402 case ABIArgInfo::InAlloca:
4403 // Aggregates get evaluated directly into the destination. Sometimes we
4404 // need to return the sret value in a register, though.
4405 assert(hasAggregateEvaluationKind(RetTy));
4406 if (RetAI.getInAllocaSRet()) {
4407 llvm::Function::arg_iterator EI = CurFn->arg_end();
4408 --EI;
4409 llvm::Value *ArgStruct = &*EI;
4410 llvm::Value *SRet = Builder.CreateStructGEP(
4411 Ty: FI.getArgStruct(), Ptr: ArgStruct, Idx: RetAI.getInAllocaFieldIndex());
4412 llvm::Type *Ty =
4413 cast<llvm::GetElementPtrInst>(Val: SRet)->getResultElementType();
4414 RV = Builder.CreateAlignedLoad(Ty, Addr: SRet, Align: getPointerAlign(), Name: "sret");
4415 }
4416 break;
4417
4418 case ABIArgInfo::Indirect: {
4419 auto AI = CurFn->arg_begin();
4420 if (RetAI.isSRetAfterThis())
4421 ++AI;
4422 switch (getEvaluationKind(T: RetTy)) {
4423 case TEK_Complex: {
4424 ComplexPairTy RT =
4425 EmitLoadOfComplex(src: MakeAddrLValue(Addr: ReturnValue, T: RetTy), loc: EndLoc);
4426 EmitStoreOfComplex(V: RT, dest: MakeNaturalAlignAddrLValue(V: &*AI, T: RetTy),
4427 /*isInit*/ true);
4428 break;
4429 }
4430 case TEK_Aggregate:
4431 // Do nothing; aggregates get evaluated directly into the destination.
4432 break;
4433 case TEK_Scalar: {
4434 LValueBaseInfo BaseInfo;
4435 TBAAAccessInfo TBAAInfo;
4436 CharUnits Alignment =
4437 CGM.getNaturalTypeAlignment(T: RetTy, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
4438 Address ArgAddr(&*AI, ConvertType(T: RetTy), Alignment);
4439 LValue ArgVal =
4440 LValue::MakeAddr(Addr: ArgAddr, type: RetTy, Context&: getContext(), BaseInfo, TBAAInfo);
4441 EmitStoreOfScalar(
4442 value: EmitLoadOfScalar(lvalue: MakeAddrLValue(Addr: ReturnValue, T: RetTy), Loc: EndLoc), lvalue: ArgVal,
4443 /*isInit*/ true);
4444 break;
4445 }
4446 }
4447 break;
4448 }
4449
4450 case ABIArgInfo::Extend:
4451 case ABIArgInfo::Direct:
4452 if (RetAI.getCoerceToType() == ConvertType(T: RetTy) &&
4453 RetAI.getDirectOffset() == 0) {
4454 // The internal return value temp always will have pointer-to-return-type
4455 // type, just do a load.
4456
4457 // If there is a dominating store to ReturnValue, we can elide
4458 // the load, zap the store, and usually zap the alloca.
4459 if (llvm::StoreInst *SI = findDominatingStoreToReturnValue(CGF&: *this)) {
4460 // Reuse the debug location from the store unless there is
4461 // cleanup code to be emitted between the store and return
4462 // instruction.
4463 if (EmitRetDbgLoc && !AutoreleaseResult)
4464 RetDbgLoc = SI->getDebugLoc();
4465 // Get the stored value and nuke the now-dead store.
4466 RV = SI->getValueOperand();
4467 SI->eraseFromParent();
4468
4469 // Otherwise, we have to do a simple load.
4470 } else {
4471 RV = Builder.CreateLoad(Addr: ReturnValue);
4472 }
4473 } else {
4474 // If the value is offset in memory, apply the offset now.
4475 Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
4476
4477 RV = CreateCoercedLoad(Src: V, SrcFETy: RetTy, Ty: RetAI.getCoerceToType(), CGF&: *this);
4478 }
4479
4480 // In ARC, end functions that return a retainable type with a call
4481 // to objc_autoreleaseReturnValue.
4482 if (AutoreleaseResult) {
4483#ifndef NDEBUG
4484 // Type::isObjCRetainabletype has to be called on a QualType that hasn't
4485 // been stripped of the typedefs, so we cannot use RetTy here. Get the
4486 // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
4487 // CurCodeDecl or BlockInfo.
4488 QualType RT;
4489
4490 if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
4491 RT = FD->getReturnType();
4492 else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
4493 RT = MD->getReturnType();
4494 else if (isa<BlockDecl>(CurCodeDecl))
4495 RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
4496 else
4497 llvm_unreachable("Unexpected function/method type");
4498
4499 assert(getLangOpts().ObjCAutoRefCount && !FI.isReturnsRetained() &&
4500 RT->isObjCRetainableType());
4501#endif
4502 RV = emitAutoreleaseOfResult(CGF&: *this, result: RV);
4503 }
4504
4505 break;
4506
4507 case ABIArgInfo::Ignore:
4508 break;
4509
4510 case ABIArgInfo::CoerceAndExpand: {
4511 auto coercionType = RetAI.getCoerceAndExpandType();
4512 auto unpaddedCoercionType = RetAI.getUnpaddedCoerceAndExpandType();
4513 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
4514
4515 // Load all of the coerced elements out into results.
4516 llvm::SmallVector<llvm::Value *, 4> results;
4517 Address addr = ReturnValue.withElementType(ElemTy: coercionType);
4518 unsigned unpaddedIndex = 0;
4519 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
4520 auto coercedEltType = coercionType->getElementType(N: i);
4521 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType: coercedEltType))
4522 continue;
4523
4524 auto eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
4525 llvm::Value *elt = CreateCoercedLoad(
4526 Src: eltAddr, SrcFETy: RetTy,
4527 Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
4528 : unpaddedCoercionType,
4529 CGF&: *this);
4530 results.push_back(Elt: elt);
4531 }
4532
4533 // If we have one result, it's the single direct result type.
4534 if (results.size() == 1) {
4535 RV = results[0];
4536
4537 // Otherwise, we need to make a first-class aggregate.
4538 } else {
4539 // Construct a return type that lacks padding elements.
4540 llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
4541
4542 RV = llvm::PoisonValue::get(T: returnType);
4543 for (unsigned i = 0, e = results.size(); i != e; ++i) {
4544 RV = Builder.CreateInsertValue(Agg: RV, Val: results[i], Idxs: i);
4545 }
4546 }
4547 break;
4548 }
4549 case ABIArgInfo::TargetSpecific: {
4550 Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
4551 RV = CGM.getABIInfo().createCoercedLoad(SrcAddr: V, AI: RetAI, CGF&: *this);
4552 break;
4553 }
4554 case ABIArgInfo::Expand:
4555 case ABIArgInfo::IndirectAliased:
4556 llvm_unreachable("Invalid ABI kind for return argument");
4557 }
4558
4559 llvm::Instruction *Ret;
4560 if (RV) {
4561 if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) {
4562 // For certain return types, clear padding bits, as they may reveal
4563 // sensitive information.
4564 // Small struct/union types are passed as integers.
4565 auto *ITy = dyn_cast<llvm::IntegerType>(Val: RV->getType());
4566 if (ITy != nullptr && isa<RecordType>(Val: RetTy.getCanonicalType()))
4567 RV = EmitCMSEClearRecord(Src: RV, ITy, QTy: RetTy);
4568 }
4569 EmitReturnValueCheck(RV);
4570 Ret = Builder.CreateRet(V: RV);
4571 } else {
4572 Ret = Builder.CreateRetVoid();
4573 }
4574
4575 if (RetDbgLoc)
4576 Ret->setDebugLoc(std::move(RetDbgLoc));
4577
4578 llvm::Value *Backup = RV ? Ret->getOperand(i: 0) : nullptr;
4579 if (RetKeyInstructionsSourceAtom)
4580 addInstToSpecificSourceAtom(KeyInstruction: Ret, Backup, Atom: RetKeyInstructionsSourceAtom);
4581 else
4582 addInstToNewSourceAtom(KeyInstruction: Ret, Backup);
4583}
4584
4585void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
4586 // A current decl may not be available when emitting vtable thunks.
4587 if (!CurCodeDecl)
4588 return;
4589
4590 // If the return block isn't reachable, neither is this check, so don't emit
4591 // it.
4592 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty())
4593 return;
4594
4595 ReturnsNonNullAttr *RetNNAttr = nullptr;
4596 if (SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute))
4597 RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
4598
4599 if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
4600 return;
4601
4602 // Prefer the returns_nonnull attribute if it's present.
4603 SourceLocation AttrLoc;
4604 SanitizerKind::SanitizerOrdinal CheckKind;
4605 SanitizerHandler Handler;
4606 if (RetNNAttr) {
4607 assert(!requiresReturnValueNullabilityCheck() &&
4608 "Cannot check nullability and the nonnull attribute");
4609 AttrLoc = RetNNAttr->getLocation();
4610 CheckKind = SanitizerKind::SO_ReturnsNonnullAttribute;
4611 Handler = SanitizerHandler::NonnullReturn;
4612 } else {
4613 if (auto *DD = dyn_cast<DeclaratorDecl>(Val: CurCodeDecl))
4614 if (auto *TSI = DD->getTypeSourceInfo())
4615 if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
4616 AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
4617 CheckKind = SanitizerKind::SO_NullabilityReturn;
4618 Handler = SanitizerHandler::NullabilityReturn;
4619 }
4620
4621 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4622
4623 // Make sure the "return" source location is valid. If we're checking a
4624 // nullability annotation, make sure the preconditions for the check are met.
4625 llvm::BasicBlock *Check = createBasicBlock(name: "nullcheck");
4626 llvm::BasicBlock *NoCheck = createBasicBlock(name: "no.nullcheck");
4627 llvm::Value *SLocPtr = Builder.CreateLoad(Addr: ReturnLocation, Name: "return.sloc.load");
4628 llvm::Value *CanNullCheck = Builder.CreateIsNotNull(Arg: SLocPtr);
4629 if (requiresReturnValueNullabilityCheck())
4630 CanNullCheck =
4631 Builder.CreateAnd(LHS: CanNullCheck, RHS: RetValNullabilityPrecondition);
4632 Builder.CreateCondBr(Cond: CanNullCheck, True: Check, False: NoCheck);
4633 EmitBlock(BB: Check);
4634
4635 // Now do the null check.
4636 llvm::Value *Cond = Builder.CreateIsNotNull(Arg: RV);
4637 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc: AttrLoc)};
4638 llvm::Value *DynamicData[] = {SLocPtr};
4639 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: DynamicData);
4640
4641 EmitBlock(BB: NoCheck);
4642
4643#ifndef NDEBUG
4644 // The return location should not be used after the check has been emitted.
4645 ReturnLocation = Address::invalid();
4646#endif
4647}
4648
4649static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
4650 const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
4651 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
4652}
4653
4654static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, QualType Ty) {
4655 // FIXME: Generate IR in one pass, rather than going back and fixing up these
4656 // placeholders.
4657 llvm::Type *IRTy = CGF.ConvertTypeForMem(T: Ty);
4658 llvm::Type *IRPtrTy = llvm::PointerType::getUnqual(C&: CGF.getLLVMContext());
4659 llvm::Value *Placeholder = llvm::PoisonValue::get(T: IRPtrTy);
4660
4661 // FIXME: When we generate this IR in one pass, we shouldn't need
4662 // this win32-specific alignment hack.
4663 CharUnits Align = CharUnits::fromQuantity(Quantity: 4);
4664 Placeholder = CGF.Builder.CreateAlignedLoad(Ty: IRPtrTy, Addr: Placeholder, Align);
4665
4666 return AggValueSlot::forAddr(
4667 addr: Address(Placeholder, IRTy, Align), quals: Ty.getQualifiers(),
4668 isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
4669 isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
4670}
4671
4672void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
4673 const VarDecl *param,
4674 SourceLocation loc) {
4675 // StartFunction converted the ABI-lowered parameter(s) into a
4676 // local alloca. We need to turn that into an r-value suitable
4677 // for EmitCall.
4678 Address local = GetAddrOfLocalVar(VD: param);
4679
4680 QualType type = param->getType();
4681
4682 // GetAddrOfLocalVar returns a pointer-to-pointer for references,
4683 // but the argument needs to be the original pointer.
4684 if (type->isReferenceType()) {
4685 args.add(rvalue: RValue::get(V: Builder.CreateLoad(Addr: local)), type);
4686
4687 // In ARC, move out of consumed arguments so that the release cleanup
4688 // entered by StartFunction doesn't cause an over-release. This isn't
4689 // optimal -O0 code generation, but it should get cleaned up when
4690 // optimization is enabled. This also assumes that delegate calls are
4691 // performed exactly once for a set of arguments, but that should be safe.
4692 } else if (getLangOpts().ObjCAutoRefCount &&
4693 param->hasAttr<NSConsumedAttr>() && type->isObjCRetainableType()) {
4694 llvm::Value *ptr = Builder.CreateLoad(Addr: local);
4695 auto null =
4696 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: ptr->getType()));
4697 Builder.CreateStore(Val: null, Addr: local);
4698 args.add(rvalue: RValue::get(V: ptr), type);
4699
4700 // For the most part, we just need to load the alloca, except that
4701 // aggregate r-values are actually pointers to temporaries.
4702 } else {
4703 args.add(rvalue: convertTempToRValue(addr: local, type, Loc: loc), type);
4704 }
4705
4706 // Deactivate the cleanup for the callee-destructed param that was pushed.
4707 if (type->isRecordType() && !CurFuncIsThunk &&
4708 type->castAsRecordDecl()->isParamDestroyedInCallee() &&
4709 param->needsDestruction(Ctx: getContext())) {
4710 EHScopeStack::stable_iterator cleanup =
4711 CalleeDestructedParamCleanups.lookup(Val: cast<ParmVarDecl>(Val: param));
4712 assert(cleanup.isValid() &&
4713 "cleanup for callee-destructed param not recorded");
4714 // This unreachable is a temporary marker which will be removed later.
4715 llvm::Instruction *isActive = Builder.CreateUnreachable();
4716 args.addArgCleanupDeactivation(Cleanup: cleanup, IsActiveIP: isActive);
4717 }
4718}
4719
4720static bool isProvablyNull(llvm::Value *addr) {
4721 return llvm::isa_and_nonnull<llvm::ConstantPointerNull>(Val: addr);
4722}
4723
4724static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) {
4725 return llvm::isKnownNonZero(V: Addr.getBasePointer(), Q: CGF.CGM.getDataLayout());
4726}
4727
4728/// Emit the actual writing-back of a writeback.
4729static void emitWriteback(CodeGenFunction &CGF,
4730 const CallArgList::Writeback &writeback) {
4731 const LValue &srcLV = writeback.Source;
4732 Address srcAddr = srcLV.getAddress();
4733 assert(!isProvablyNull(srcAddr.getBasePointer()) &&
4734 "shouldn't have writeback for provably null argument");
4735
4736 if (writeback.WritebackExpr) {
4737 CGF.EmitIgnoredExpr(E: writeback.WritebackExpr);
4738 CGF.EmitLifetimeEnd(Addr: writeback.Temporary.getBasePointer());
4739 return;
4740 }
4741
4742 llvm::BasicBlock *contBB = nullptr;
4743
4744 // If the argument wasn't provably non-null, we need to null check
4745 // before doing the store.
4746 bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4747
4748 if (!provablyNonNull) {
4749 llvm::BasicBlock *writebackBB = CGF.createBasicBlock(name: "icr.writeback");
4750 contBB = CGF.createBasicBlock(name: "icr.done");
4751
4752 llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4753 CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: writebackBB);
4754 CGF.EmitBlock(BB: writebackBB);
4755 }
4756
4757 // Load the value to writeback.
4758 llvm::Value *value = CGF.Builder.CreateLoad(Addr: writeback.Temporary);
4759
4760 // Cast it back, in case we're writing an id to a Foo* or something.
4761 value = CGF.Builder.CreateBitCast(V: value, DestTy: srcAddr.getElementType(),
4762 Name: "icr.writeback-cast");
4763
4764 // Perform the writeback.
4765
4766 // If we have a "to use" value, it's something we need to emit a use
4767 // of. This has to be carefully threaded in: if it's done after the
4768 // release it's potentially undefined behavior (and the optimizer
4769 // will ignore it), and if it happens before the retain then the
4770 // optimizer could move the release there.
4771 if (writeback.ToUse) {
4772 assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
4773
4774 // Retain the new value. No need to block-copy here: the block's
4775 // being passed up the stack.
4776 value = CGF.EmitARCRetainNonBlock(value);
4777
4778 // Emit the intrinsic use here.
4779 CGF.EmitARCIntrinsicUse(values: writeback.ToUse);
4780
4781 // Load the old value (primitively).
4782 llvm::Value *oldValue = CGF.EmitLoadOfScalar(lvalue: srcLV, Loc: SourceLocation());
4783
4784 // Put the new value in place (primitively).
4785 CGF.EmitStoreOfScalar(value, lvalue: srcLV, /*init*/ isInit: false);
4786
4787 // Release the old value.
4788 CGF.EmitARCRelease(value: oldValue, precise: srcLV.isARCPreciseLifetime());
4789
4790 // Otherwise, we can just do a normal lvalue store.
4791 } else {
4792 CGF.EmitStoreThroughLValue(Src: RValue::get(V: value), Dst: srcLV);
4793 }
4794
4795 // Jump to the continuation block.
4796 if (!provablyNonNull)
4797 CGF.EmitBlock(BB: contBB);
4798}
4799
4800static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
4801 const CallArgList &CallArgs) {
4802 ArrayRef<CallArgList::CallArgCleanup> Cleanups =
4803 CallArgs.getCleanupsToDeactivate();
4804 // Iterate in reverse to increase the likelihood of popping the cleanup.
4805 for (const auto &I : llvm::reverse(C&: Cleanups)) {
4806 CGF.DeactivateCleanupBlock(Cleanup: I.Cleanup, DominatingIP: I.IsActiveIP);
4807 I.IsActiveIP->eraseFromParent();
4808 }
4809}
4810
4811static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
4812 if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(Val: E->IgnoreParens()))
4813 if (uop->getOpcode() == UO_AddrOf)
4814 return uop->getSubExpr();
4815 return nullptr;
4816}
4817
4818/// Emit an argument that's being passed call-by-writeback. That is,
4819/// we are passing the address of an __autoreleased temporary; it
4820/// might be copy-initialized with the current value of the given
4821/// address, but it will definitely be copied out of after the call.
4822static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
4823 const ObjCIndirectCopyRestoreExpr *CRE) {
4824 LValue srcLV;
4825
4826 // Make an optimistic effort to emit the address as an l-value.
4827 // This can fail if the argument expression is more complicated.
4828 if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(E: CRE->getSubExpr())) {
4829 srcLV = CGF.EmitLValue(E: lvExpr);
4830
4831 // Otherwise, just emit it as a scalar.
4832 } else {
4833 Address srcAddr = CGF.EmitPointerWithAlignment(Addr: CRE->getSubExpr());
4834
4835 QualType srcAddrType =
4836 CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
4837 srcLV = CGF.MakeAddrLValue(Addr: srcAddr, T: srcAddrType);
4838 }
4839 Address srcAddr = srcLV.getAddress();
4840
4841 // The dest and src types don't necessarily match in LLVM terms
4842 // because of the crazy ObjC compatibility rules.
4843
4844 llvm::PointerType *destType =
4845 cast<llvm::PointerType>(Val: CGF.ConvertType(T: CRE->getType()));
4846 llvm::Type *destElemType =
4847 CGF.ConvertTypeForMem(T: CRE->getType()->getPointeeType());
4848
4849 // If the address is a constant null, just pass the appropriate null.
4850 if (isProvablyNull(addr: srcAddr.getBasePointer())) {
4851 args.add(rvalue: RValue::get(V: llvm::ConstantPointerNull::get(T: destType)),
4852 type: CRE->getType());
4853 return;
4854 }
4855
4856 // Create the temporary.
4857 Address temp =
4858 CGF.CreateTempAlloca(Ty: destElemType, align: CGF.getPointerAlign(), Name: "icr.temp");
4859 // Loading an l-value can introduce a cleanup if the l-value is __weak,
4860 // and that cleanup will be conditional if we can't prove that the l-value
4861 // isn't null, so we need to register a dominating point so that the cleanups
4862 // system will make valid IR.
4863 CodeGenFunction::ConditionalEvaluation condEval(CGF);
4864
4865 // Zero-initialize it if we're not doing a copy-initialization.
4866 bool shouldCopy = CRE->shouldCopy();
4867 if (!shouldCopy) {
4868 llvm::Value *null =
4869 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: destElemType));
4870 CGF.Builder.CreateStore(Val: null, Addr: temp);
4871 }
4872
4873 llvm::BasicBlock *contBB = nullptr;
4874 llvm::BasicBlock *originBB = nullptr;
4875
4876 // If the address is *not* known to be non-null, we need to switch.
4877 llvm::Value *finalArgument;
4878
4879 bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4880
4881 if (provablyNonNull) {
4882 finalArgument = temp.emitRawPointer(CGF);
4883 } else {
4884 llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4885
4886 finalArgument = CGF.Builder.CreateSelect(
4887 C: isNull, True: llvm::ConstantPointerNull::get(T: destType),
4888 False: temp.emitRawPointer(CGF), Name: "icr.argument");
4889
4890 // If we need to copy, then the load has to be conditional, which
4891 // means we need control flow.
4892 if (shouldCopy) {
4893 originBB = CGF.Builder.GetInsertBlock();
4894 contBB = CGF.createBasicBlock(name: "icr.cont");
4895 llvm::BasicBlock *copyBB = CGF.createBasicBlock(name: "icr.copy");
4896 CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: copyBB);
4897 CGF.EmitBlock(BB: copyBB);
4898 condEval.begin(CGF);
4899 }
4900 }
4901
4902 llvm::Value *valueToUse = nullptr;
4903
4904 // Perform a copy if necessary.
4905 if (shouldCopy) {
4906 RValue srcRV = CGF.EmitLoadOfLValue(V: srcLV, Loc: SourceLocation());
4907 assert(srcRV.isScalar());
4908
4909 llvm::Value *src = srcRV.getScalarVal();
4910 src = CGF.Builder.CreateBitCast(V: src, DestTy: destElemType, Name: "icr.cast");
4911
4912 // Use an ordinary store, not a store-to-lvalue.
4913 CGF.Builder.CreateStore(Val: src, Addr: temp);
4914
4915 // If optimization is enabled, and the value was held in a
4916 // __strong variable, we need to tell the optimizer that this
4917 // value has to stay alive until we're doing the store back.
4918 // This is because the temporary is effectively unretained,
4919 // and so otherwise we can violate the high-level semantics.
4920 if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4921 srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {
4922 valueToUse = src;
4923 }
4924 }
4925
4926 // Finish the control flow if we needed it.
4927 if (shouldCopy && !provablyNonNull) {
4928 llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
4929 CGF.EmitBlock(BB: contBB);
4930
4931 // Make a phi for the value to intrinsically use.
4932 if (valueToUse) {
4933 llvm::PHINode *phiToUse =
4934 CGF.Builder.CreatePHI(Ty: valueToUse->getType(), NumReservedValues: 2, Name: "icr.to-use");
4935 phiToUse->addIncoming(V: valueToUse, BB: copyBB);
4936 phiToUse->addIncoming(V: llvm::PoisonValue::get(T: valueToUse->getType()),
4937 BB: originBB);
4938 valueToUse = phiToUse;
4939 }
4940
4941 condEval.end(CGF);
4942 }
4943
4944 args.addWriteback(srcLV, temporary: temp, toUse: valueToUse);
4945 args.add(rvalue: RValue::get(V: finalArgument), type: CRE->getType());
4946}
4947
4948void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
4949 assert(!StackBase);
4950
4951 // Save the stack.
4952 StackBase = CGF.Builder.CreateStackSave(Name: "inalloca.save");
4953}
4954
4955void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
4956 if (StackBase) {
4957 // Restore the stack after the call.
4958 CGF.Builder.CreateStackRestore(Ptr: StackBase);
4959 }
4960}
4961
4962void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
4963 SourceLocation ArgLoc,
4964 AbstractCallee AC, unsigned ParmNum) {
4965 if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
4966 SanOpts.has(K: SanitizerKind::NullabilityArg)))
4967 return;
4968
4969 // The param decl may be missing in a variadic function.
4970 auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(I: ParmNum) : nullptr;
4971 unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
4972
4973 // Prefer the nonnull attribute if it's present.
4974 const NonNullAttr *NNAttr = nullptr;
4975 if (SanOpts.has(K: SanitizerKind::NonnullAttribute))
4976 NNAttr = getNonNullAttr(FD: AC.getDecl(), PVD, ArgType, ArgNo);
4977
4978 bool CanCheckNullability = false;
4979 if (SanOpts.has(K: SanitizerKind::NullabilityArg) && !NNAttr && PVD &&
4980 !PVD->getType()->isRecordType()) {
4981 auto Nullability = PVD->getType()->getNullability();
4982 CanCheckNullability = Nullability &&
4983 *Nullability == NullabilityKind::NonNull &&
4984 PVD->getTypeSourceInfo();
4985 }
4986
4987 if (!NNAttr && !CanCheckNullability)
4988 return;
4989
4990 SourceLocation AttrLoc;
4991 SanitizerKind::SanitizerOrdinal CheckKind;
4992 SanitizerHandler Handler;
4993 if (NNAttr) {
4994 AttrLoc = NNAttr->getLocation();
4995 CheckKind = SanitizerKind::SO_NonnullAttribute;
4996 Handler = SanitizerHandler::NonnullArg;
4997 } else {
4998 AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4999 CheckKind = SanitizerKind::SO_NullabilityArg;
5000 Handler = SanitizerHandler::NullabilityArg;
5001 }
5002
5003 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
5004 llvm::Value *Cond = EmitNonNullRValueCheck(RV, T: ArgType);
5005 llvm::Constant *StaticData[] = {
5006 EmitCheckSourceLocation(Loc: ArgLoc),
5007 EmitCheckSourceLocation(Loc: AttrLoc),
5008 llvm::ConstantInt::get(Ty: Int32Ty, V: ArgNo + 1),
5009 };
5010 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: {});
5011}
5012
5013void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType,
5014 SourceLocation ArgLoc,
5015 AbstractCallee AC, unsigned ParmNum) {
5016 if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
5017 SanOpts.has(K: SanitizerKind::NullabilityArg)))
5018 return;
5019
5020 EmitNonNullArgCheck(RV: RValue::get(Addr, CGF&: *this), ArgType, ArgLoc, AC, ParmNum);
5021}
5022
5023// Check if the call is going to use the inalloca convention. This needs to
5024// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
5025// later, so we can't check it directly.
5026static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
5027 ArrayRef<QualType> ArgTypes) {
5028 // The Swift calling conventions don't go through the target-specific
5029 // argument classification, they never use inalloca.
5030 // TODO: Consider limiting inalloca use to only calling conventions supported
5031 // by MSVC.
5032 if (ExplicitCC == CC_Swift || ExplicitCC == CC_SwiftAsync)
5033 return false;
5034 if (!CGM.getTarget().getCXXABI().isMicrosoft())
5035 return false;
5036 return llvm::any_of(Range&: ArgTypes, P: [&](QualType Ty) {
5037 return isInAllocaArgument(ABI&: CGM.getCXXABI(), type: Ty);
5038 });
5039}
5040
5041#ifndef NDEBUG
5042// Determine whether the given argument is an Objective-C method
5043// that may have type parameters in its signature.
5044static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
5045 const DeclContext *dc = method->getDeclContext();
5046 if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {
5047 return classDecl->getTypeParamListAsWritten();
5048 }
5049
5050 if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
5051 return catDecl->getTypeParamList();
5052 }
5053
5054 return false;
5055}
5056#endif
5057
5058/// EmitCallArgs - Emit call arguments for a function.
5059void CodeGenFunction::EmitCallArgs(
5060 CallArgList &Args, PrototypeWrapper Prototype,
5061 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
5062 AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
5063 SmallVector<QualType, 16> ArgTypes;
5064
5065 assert((ParamsToSkip == 0 || Prototype.P) &&
5066 "Can't skip parameters if type info is not provided");
5067
5068 // This variable only captures *explicitly* written conventions, not those
5069 // applied by default via command line flags or target defaults, such as
5070 // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
5071 // require knowing if this is a C++ instance method or being able to see
5072 // unprototyped FunctionTypes.
5073 CallingConv ExplicitCC = CC_C;
5074
5075 // First, if a prototype was provided, use those argument types.
5076 bool IsVariadic = false;
5077 if (Prototype.P) {
5078 const auto *MD = dyn_cast<const ObjCMethodDecl *>(Val&: Prototype.P);
5079 if (MD) {
5080 IsVariadic = MD->isVariadic();
5081 ExplicitCC = getCallingConventionForDecl(
5082 D: MD, IsTargetDefaultMSABI: CGM.getTarget().getTriple().isOSWindows());
5083 ArgTypes.assign(in_start: MD->param_type_begin() + ParamsToSkip,
5084 in_end: MD->param_type_end());
5085 } else {
5086 const auto *FPT = cast<const FunctionProtoType *>(Val&: Prototype.P);
5087 IsVariadic = FPT->isVariadic();
5088 ExplicitCC = FPT->getExtInfo().getCC();
5089 ArgTypes.assign(in_start: FPT->param_type_begin() + ParamsToSkip,
5090 in_end: FPT->param_type_end());
5091 }
5092
5093#ifndef NDEBUG
5094 // Check that the prototyped types match the argument expression types.
5095 bool isGenericMethod = MD && isObjCMethodWithTypeParams(MD);
5096 CallExpr::const_arg_iterator Arg = ArgRange.begin();
5097 for (QualType Ty : ArgTypes) {
5098 assert(Arg != ArgRange.end() && "Running over edge of argument list!");
5099 QualType ParamTy = Ty.getNonReferenceType();
5100 QualType ArgTy = (*Arg)->getType();
5101 if (const auto *OBT = ParamTy->getAs<OverflowBehaviorType>())
5102 ParamTy = OBT->getUnderlyingType();
5103 if (const auto *OBT = ArgTy->getAs<OverflowBehaviorType>())
5104 ArgTy = OBT->getUnderlyingType();
5105 assert((isGenericMethod || Ty->isVariablyModifiedType() ||
5106 ParamTy->isObjCRetainableType() ||
5107 getContext().getCanonicalType(ParamTy).getTypePtr() ==
5108 getContext().getCanonicalType(ArgTy).getTypePtr()) &&
5109 "type mismatch in call argument!");
5110 ++Arg;
5111 }
5112
5113 // Either we've emitted all the call args, or we have a call to variadic
5114 // function.
5115 assert((Arg == ArgRange.end() || IsVariadic) &&
5116 "Extra arguments in non-variadic function!");
5117#endif
5118 }
5119
5120 // If we still have any arguments, emit them using the type of the argument.
5121 for (auto *A : llvm::drop_begin(RangeOrContainer&: ArgRange, N: ArgTypes.size()))
5122 ArgTypes.push_back(Elt: IsVariadic ? getVarArgType(Arg: A) : A->getType());
5123 assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
5124
5125 // We must evaluate arguments from right to left in the MS C++ ABI,
5126 // because arguments are destroyed left to right in the callee. As a special
5127 // case, there are certain language constructs that require left-to-right
5128 // evaluation, and in those cases we consider the evaluation order requirement
5129 // to trump the "destruction order is reverse construction order" guarantee.
5130 bool LeftToRight =
5131 CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
5132 ? Order == EvaluationOrder::ForceLeftToRight
5133 : Order != EvaluationOrder::ForceRightToLeft;
5134
5135 auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
5136 RValue EmittedArg) {
5137 if (!AC.hasFunctionDecl() || I >= AC.getNumParams())
5138 return;
5139 auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
5140 if (PS == nullptr)
5141 return;
5142
5143 const auto &Context = getContext();
5144 auto SizeTy = Context.getSizeType();
5145 auto T = Builder.getIntNTy(N: Context.getTypeSize(T: SizeTy));
5146 assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
5147 llvm::Value *V = evaluateOrEmitBuiltinObjectSize(
5148 E: Arg, Type: PS->getType(), ResType: T, EmittedE: EmittedArg.getScalarVal(), IsDynamic: PS->isDynamic());
5149 Args.add(rvalue: RValue::get(V), type: SizeTy);
5150 // If we're emitting args in reverse, be sure to do so with
5151 // pass_object_size, as well.
5152 if (!LeftToRight)
5153 std::swap(a&: Args.back(), b&: *(&Args.back() - 1));
5154 };
5155
5156 // Insert a stack save if we're going to need any inalloca args.
5157 if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
5158 assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
5159 "inalloca only supported on x86");
5160 Args.allocateArgumentMemory(CGF&: *this);
5161 }
5162
5163 // Evaluate each argument in the appropriate order.
5164 size_t CallArgsStart = Args.size();
5165 for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
5166 unsigned Idx = LeftToRight ? I : E - I - 1;
5167 CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
5168 unsigned InitialArgSize = Args.size();
5169 // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
5170 // the argument and parameter match or the objc method is parameterized.
5171 assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
5172 getContext().hasSameUnqualifiedType((*Arg)->getType(),
5173 ArgTypes[Idx]) ||
5174 (isa<ObjCMethodDecl>(AC.getDecl()) &&
5175 isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
5176 "Argument and parameter types don't match");
5177 EmitCallArg(args&: Args, E: *Arg, ArgType: ArgTypes[Idx]);
5178 // In particular, we depend on it being the last arg in Args, and the
5179 // objectsize bits depend on there only being one arg if !LeftToRight.
5180 assert(InitialArgSize + 1 == Args.size() &&
5181 "The code below depends on only adding one arg per EmitCallArg");
5182 (void)InitialArgSize;
5183 // Since pointer argument are never emitted as LValue, it is safe to emit
5184 // non-null argument check for r-value only.
5185 if (!Args.back().hasLValue()) {
5186 RValue RVArg = Args.back().getKnownRValue();
5187 EmitNonNullArgCheck(RV: RVArg, ArgType: ArgTypes[Idx], ArgLoc: (*Arg)->getExprLoc(), AC,
5188 ParmNum: ParamsToSkip + Idx);
5189 // @llvm.objectsize should never have side-effects and shouldn't need
5190 // destruction/cleanups, so we can safely "emit" it after its arg,
5191 // regardless of right-to-leftness
5192 MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
5193 }
5194 }
5195
5196 if (!LeftToRight) {
5197 // Un-reverse the arguments we just evaluated so they match up with the LLVM
5198 // IR function.
5199 std::reverse(first: Args.begin() + CallArgsStart, last: Args.end());
5200
5201 // Reverse the writebacks to match the MSVC ABI.
5202 Args.reverseWritebacks();
5203 }
5204}
5205
5206namespace {
5207
5208struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
5209 DestroyUnpassedArg(Address Addr, QualType Ty) : Addr(Addr), Ty(Ty) {}
5210
5211 Address Addr;
5212 QualType Ty;
5213
5214 void Emit(CodeGenFunction &CGF, Flags flags) override {
5215 QualType::DestructionKind DtorKind = Ty.isDestructedType();
5216 if (DtorKind == QualType::DK_cxx_destructor) {
5217 const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
5218 assert(!Dtor->isTrivial());
5219 CGF.EmitCXXDestructorCall(D: Dtor, Type: Dtor_Complete, /*for vbase*/ ForVirtualBase: false,
5220 /*Delegating=*/false, This: Addr, ThisTy: Ty);
5221 } else {
5222 CGF.callCStructDestructor(Dst: CGF.MakeAddrLValue(Addr, T: Ty));
5223 }
5224 }
5225};
5226
5227} // end anonymous namespace
5228
5229RValue CallArg::getRValue(CodeGenFunction &CGF) const {
5230 if (!HasLV)
5231 return RV;
5232 LValue Copy = CGF.MakeAddrLValue(Addr: CGF.CreateMemTempWithoutCast(T: Ty), T: Ty);
5233 CGF.EmitAggregateCopy(Dest: Copy, Src: LV, EltTy: Ty, MayOverlap: AggValueSlot::DoesNotOverlap,
5234 isVolatile: LV.isVolatile());
5235 IsUsed = true;
5236 return RValue::getAggregate(addr: Copy.getAddress());
5237}
5238
5239void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
5240 LValue Dst = CGF.MakeAddrLValue(Addr, T: Ty);
5241 if (!HasLV && RV.isScalar())
5242 CGF.EmitStoreOfScalar(value: RV.getScalarVal(), lvalue: Dst, /*isInit=*/true);
5243 else if (!HasLV && RV.isComplex())
5244 CGF.EmitStoreOfComplex(V: RV.getComplexVal(), dest: Dst, /*init=*/isInit: true);
5245 else {
5246 auto Addr = HasLV ? LV.getAddress() : RV.getAggregateAddress();
5247 LValue SrcLV = CGF.MakeAddrLValue(Addr, T: Ty);
5248 // We assume that call args are never copied into subobjects.
5249 CGF.EmitAggregateCopy(Dest: Dst, Src: SrcLV, EltTy: Ty, MayOverlap: AggValueSlot::DoesNotOverlap,
5250 isVolatile: HasLV ? LV.isVolatileQualified()
5251 : RV.isVolatileQualified());
5252 }
5253 IsUsed = true;
5254}
5255
5256void CodeGenFunction::EmitWritebacks(const CallArgList &args) {
5257 for (const auto &I : args.writebacks())
5258 emitWriteback(CGF&: *this, writeback: I);
5259}
5260
5261void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
5262 QualType type) {
5263 std::optional<DisableDebugLocationUpdates> Dis;
5264 if (isa<CXXDefaultArgExpr>(Val: E))
5265 Dis.emplace(args&: *this);
5266 if (const ObjCIndirectCopyRestoreExpr *CRE =
5267 dyn_cast<ObjCIndirectCopyRestoreExpr>(Val: E)) {
5268 assert(getLangOpts().ObjCAutoRefCount);
5269 return emitWritebackArg(CGF&: *this, args, CRE);
5270 }
5271
5272 // Add writeback for HLSLOutParamExpr.
5273 // Needs to be before the assert below because HLSLOutArgExpr is an LValue
5274 // and is not a reference.
5275 if (const HLSLOutArgExpr *OE = dyn_cast<HLSLOutArgExpr>(Val: E)) {
5276 EmitHLSLOutArgExpr(E: OE, Args&: args, Ty: type);
5277 return;
5278 }
5279
5280 assert(type->isReferenceType() == E->isGLValue() &&
5281 "reference binding to unmaterialized r-value!");
5282
5283 if (E->isGLValue()) {
5284 assert(E->getObjectKind() == OK_Ordinary);
5285 return args.add(rvalue: EmitReferenceBindingToExpr(E), type);
5286 }
5287
5288 bool HasAggregateEvalKind = hasAggregateEvaluationKind(T: type);
5289
5290 // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
5291 // However, we still have to push an EH-only cleanup in case we unwind before
5292 // we make it to the call.
5293 if (type->isRecordType() &&
5294 type->castAsRecordDecl()->isParamDestroyedInCallee()) {
5295 // If we're using inalloca, use the argument memory. Otherwise, use a
5296 // temporary.
5297 AggValueSlot Slot = args.isUsingInAlloca()
5298 ? createPlaceholderSlot(CGF&: *this, Ty: type)
5299 : CreateAggTemp(T: type, Name: "agg.tmp");
5300
5301 bool DestroyedInCallee = true, NeedsCleanup = true;
5302 if (const auto *RD = type->getAsCXXRecordDecl())
5303 DestroyedInCallee = RD->hasNonTrivialDestructor();
5304 else
5305 NeedsCleanup = type.isDestructedType();
5306
5307 if (DestroyedInCallee)
5308 Slot.setExternallyDestructed();
5309
5310 EmitAggExpr(E, AS: Slot);
5311 RValue RV = Slot.asRValue();
5312 args.add(rvalue: RV, type);
5313
5314 if (DestroyedInCallee && NeedsCleanup) {
5315 // Create a no-op GEP between the placeholder and the cleanup so we can
5316 // RAUW it successfully. It also serves as a marker of the first
5317 // instruction where the cleanup is active.
5318 pushFullExprCleanup<DestroyUnpassedArg>(kind: NormalAndEHCleanup,
5319 A: Slot.getAddress(), A: type);
5320 // This unreachable is a temporary marker which will be removed later.
5321 llvm::Instruction *IsActive =
5322 Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
5323 args.addArgCleanupDeactivation(Cleanup: EHStack.stable_begin(), IsActiveIP: IsActive);
5324 }
5325 return;
5326 }
5327
5328 if (HasAggregateEvalKind) {
5329 auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E);
5330 if (ICE && ICE->getCastKind() == CK_LValueToRValue &&
5331 ICE->getSubExpr()->getType().getAddressSpace() !=
5332 LangAS::hlsl_constant &&
5333 !type->isArrayParameterType() && !type.isNonTrivialToPrimitiveCopy()) {
5334 LValue L = EmitLValue(E: cast<CastExpr>(Val: E)->getSubExpr());
5335 assert(L.isSimple());
5336 args.addUncopiedAggregate(LV: L, type);
5337 return;
5338 }
5339 }
5340
5341 args.add(rvalue: EmitAnyExprToTemp(E), type);
5342}
5343
5344QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
5345 // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
5346 // implicitly widens null pointer constants that are arguments to varargs
5347 // functions to pointer-sized ints.
5348 if (!getTarget().getTriple().isOSWindows())
5349 return Arg->getType();
5350
5351 if (Arg->getType()->isIntegerType() &&
5352 getContext().getTypeSize(T: Arg->getType()) <
5353 getContext().getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default) &&
5354 Arg->isNullPointerConstant(Ctx&: getContext(),
5355 NPC: Expr::NPC_ValueDependentIsNotNull)) {
5356 return getContext().getIntPtrType();
5357 }
5358
5359 return Arg->getType();
5360}
5361
5362// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5363// optimizer it can aggressively ignore unwind edges.
5364void CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
5365 if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
5366 !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
5367 Inst->setMetadata(Kind: "clang.arc.no_objc_arc_exceptions",
5368 Node: CGM.getNoObjCARCExceptionsMetadata());
5369}
5370
5371/// Emits a call to the given no-arguments nounwind runtime function.
5372llvm::CallInst *
5373CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5374 const llvm::Twine &name) {
5375 return EmitNounwindRuntimeCall(callee, args: ArrayRef<llvm::Value *>(), name);
5376}
5377
5378/// Emits a call to the given nounwind runtime function.
5379llvm::CallInst *
5380CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5381 ArrayRef<Address> args,
5382 const llvm::Twine &name) {
5383 SmallVector<llvm::Value *, 3> values;
5384 for (auto arg : args)
5385 values.push_back(Elt: arg.emitRawPointer(CGF&: *this));
5386 return EmitNounwindRuntimeCall(callee, args: values, name);
5387}
5388
5389llvm::CallInst *
5390CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5391 ArrayRef<llvm::Value *> args,
5392 const llvm::Twine &name) {
5393 llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
5394 call->setDoesNotThrow();
5395 return call;
5396}
5397
5398/// Emits a simple call (never an invoke) to the given no-arguments
5399/// runtime function.
5400llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5401 const llvm::Twine &name) {
5402 return EmitRuntimeCall(callee, args: {}, name);
5403}
5404
5405// Calls which may throw must have operand bundles indicating which funclet
5406// they are nested within.
5407SmallVector<llvm::OperandBundleDef, 1>
5408CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
5409 // There is no need for a funclet operand bundle if we aren't inside a
5410 // funclet.
5411 if (!CurrentFuncletPad)
5412 return (SmallVector<llvm::OperandBundleDef, 1>());
5413
5414 // Skip intrinsics which cannot throw (as long as they don't lower into
5415 // regular function calls in the course of IR transformations).
5416 if (auto *CalleeFn = dyn_cast<llvm::Function>(Val: Callee->stripPointerCasts())) {
5417 if (CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) {
5418 auto IID = CalleeFn->getIntrinsicID();
5419 if (!llvm::IntrinsicInst::mayLowerToFunctionCall(IID))
5420 return (SmallVector<llvm::OperandBundleDef, 1>());
5421 }
5422 }
5423
5424 SmallVector<llvm::OperandBundleDef, 1> BundleList;
5425 BundleList.emplace_back(Args: "funclet", Args&: CurrentFuncletPad);
5426 return BundleList;
5427}
5428
5429/// Emits a simple call (never an invoke) to the given runtime function.
5430llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5431 ArrayRef<llvm::Value *> args,
5432 const llvm::Twine &name) {
5433 llvm::CallInst *call = Builder.CreateCall(
5434 Callee: callee, Args: args, OpBundles: getBundlesForFunclet(Callee: callee.getCallee()), Name: name);
5435 call->setCallingConv(getRuntimeCC());
5436
5437 if (CGM.shouldEmitConvergenceTokens() && call->isConvergent())
5438 return cast<llvm::CallInst>(Val: addConvergenceControlToken(Input: call));
5439 return call;
5440}
5441
5442llvm::CallInst *CodeGenFunction::EmitIntrinsicCall(llvm::Intrinsic::ID ID,
5443 const llvm::Twine &Name) {
5444 return EmitIntrinsicCall(ID, Args: {}, RetTy: {}, Name);
5445}
5446
5447llvm::CallInst *CodeGenFunction::EmitIntrinsicCall(llvm::Intrinsic::ID ID,
5448 ArrayRef<llvm::Value *> Args,
5449 const llvm::Twine &Name) {
5450 return EmitIntrinsicCall(ID, Types: {}, Args, Name);
5451}
5452
5453llvm::CallInst *CodeGenFunction::EmitIntrinsicCall(llvm::Intrinsic::ID ID,
5454 ArrayRef<llvm::Type *> Types,
5455 ArrayRef<llvm::Value *> Args,
5456 const llvm::Twine &Name) {
5457 llvm::Function *F =
5458 llvm::Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID, OverloadTys: Types);
5459 llvm::CallInst *Call =
5460 Builder.CreateCall(Callee: F, Args, OpBundles: getBundlesForFunclet(Callee: F), Name);
5461 if (CGM.shouldEmitConvergenceTokens() && Call->isConvergent())
5462 return cast<llvm::CallInst>(Val: addConvergenceControlToken(Input: Call));
5463 return Call;
5464}
5465
5466llvm::CallInst *CodeGenFunction::EmitIntrinsicCall(llvm::Intrinsic::ID ID,
5467 ArrayRef<llvm::Value *> Args,
5468 llvm::Type *RetTy,
5469 const llvm::Twine &Name) {
5470 SmallVector<llvm::Type *> ArgTys;
5471 ArgTys.reserve(N: Args.size());
5472 for (llvm::Value *Arg : Args)
5473 ArgTys.push_back(Elt: Arg->getType());
5474 llvm::Function *F = llvm::Intrinsic::getOrInsertDeclaration(
5475 M: &CGM.getModule(), IID: ID, RetTy, ArgTys);
5476 llvm::CallInst *Call =
5477 Builder.CreateCall(Callee: F, Args, OpBundles: getBundlesForFunclet(Callee: F), Name);
5478 if (CGM.shouldEmitConvergenceTokens() && Call->isConvergent())
5479 return cast<llvm::CallInst>(Val: addConvergenceControlToken(Input: Call));
5480 return Call;
5481}
5482
5483/// Emits a call or invoke to the given noreturn runtime function.
5484void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
5485 llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
5486 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5487 getBundlesForFunclet(Callee: callee.getCallee());
5488
5489 if (getInvokeDest()) {
5490 llvm::InvokeInst *invoke = Builder.CreateInvoke(
5491 Callee: callee, NormalDest: getUnreachableBlock(), UnwindDest: getInvokeDest(), Args: args, OpBundles: BundleList);
5492 invoke->setDoesNotReturn();
5493 invoke->setCallingConv(getRuntimeCC());
5494 } else {
5495 llvm::CallInst *call = Builder.CreateCall(Callee: callee, Args: args, OpBundles: BundleList);
5496 call->setDoesNotReturn();
5497 call->setCallingConv(getRuntimeCC());
5498 Builder.CreateUnreachable();
5499 }
5500}
5501
5502/// Emits a call or invoke instruction to the given nullary runtime function.
5503llvm::CallBase *
5504CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5505 const Twine &name) {
5506 return EmitRuntimeCallOrInvoke(callee, args: {}, name);
5507}
5508
5509/// Emits a call or invoke instruction to the given runtime function.
5510llvm::CallBase *
5511CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5512 ArrayRef<llvm::Value *> args,
5513 const Twine &name) {
5514 llvm::CallBase *call = EmitCallOrInvoke(Callee: callee, Args: args, Name: name);
5515 call->setCallingConv(getRuntimeCC());
5516 return call;
5517}
5518
5519/// Emits a call or invoke instruction to the given function, depending
5520/// on the current state of the EH stack.
5521llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
5522 ArrayRef<llvm::Value *> Args,
5523 const Twine &Name) {
5524 llvm::BasicBlock *InvokeDest = getInvokeDest();
5525 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5526 getBundlesForFunclet(Callee: Callee.getCallee());
5527
5528 llvm::CallBase *Inst;
5529 if (!InvokeDest)
5530 Inst = Builder.CreateCall(Callee, Args, OpBundles: BundleList, Name);
5531 else {
5532 llvm::BasicBlock *ContBB = createBasicBlock(name: "invoke.cont");
5533 Inst = Builder.CreateInvoke(Callee, NormalDest: ContBB, UnwindDest: InvokeDest, Args, OpBundles: BundleList,
5534 Name);
5535 EmitBlock(BB: ContBB);
5536 }
5537
5538 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5539 // optimizer it can aggressively ignore unwind edges.
5540 if (CGM.getLangOpts().ObjCAutoRefCount)
5541 AddObjCARCExceptionMetadata(Inst);
5542
5543 return Inst;
5544}
5545
5546void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
5547 llvm::Value *New) {
5548 DeferredReplacements.push_back(
5549 Elt: std::make_pair(x: llvm::WeakTrackingVH(Old), y&: New));
5550}
5551
5552namespace {
5553
5554/// Specify given \p NewAlign as the alignment of return value attribute. If
5555/// such attribute already exists, re-set it to the maximal one of two options.
5556[[nodiscard]] llvm::AttributeList
5557maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
5558 const llvm::AttributeList &Attrs,
5559 llvm::Align NewAlign) {
5560 llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
5561 if (CurAlign >= NewAlign)
5562 return Attrs;
5563 llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Context&: Ctx, Alignment: NewAlign);
5564 return Attrs.removeRetAttribute(C&: Ctx, Kind: llvm::Attribute::AttrKind::Alignment)
5565 .addRetAttribute(C&: Ctx, Attr: AlignAttr);
5566}
5567
5568template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
5569protected:
5570 CodeGenFunction &CGF;
5571
5572 /// We do nothing if this is, or becomes, nullptr.
5573 const AlignedAttrTy *AA = nullptr;
5574
5575 llvm::Value *Alignment = nullptr; // May or may not be a constant.
5576 llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
5577
5578 AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5579 : CGF(CGF_) {
5580 if (!FuncDecl)
5581 return;
5582 AA = FuncDecl->getAttr<AlignedAttrTy>();
5583 }
5584
5585public:
5586 /// If we can, materialize the alignment as an attribute on return value.
5587 [[nodiscard]] llvm::AttributeList
5588 TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
5589 if (!AA || OffsetCI || CGF.SanOpts.has(K: SanitizerKind::Alignment))
5590 return Attrs;
5591 const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Val: Alignment);
5592 if (!AlignmentCI)
5593 return Attrs;
5594 // We may legitimately have non-power-of-2 alignment here.
5595 // If so, this is UB land, emit it via `@llvm.assume` instead.
5596 if (!AlignmentCI->getValue().isPowerOf2())
5597 return Attrs;
5598 llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
5599 Ctx&: CGF.getLLVMContext(), Attrs,
5600 NewAlign: llvm::Align(
5601 AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment)));
5602 AA = nullptr; // We're done. Disallow doing anything else.
5603 return NewAttrs;
5604 }
5605
5606 /// Emit alignment assumption.
5607 /// This is a general fallback that we take if either there is an offset,
5608 /// or the alignment is variable or we are sanitizing for alignment.
5609 void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
5610 if (!AA)
5611 return;
5612 CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
5613 AA->getLocation(), Alignment, OffsetCI);
5614 AA = nullptr; // We're done. Disallow doing anything else.
5615 }
5616};
5617
5618/// Helper data structure to emit `AssumeAlignedAttr`.
5619class AssumeAlignedAttrEmitter final
5620 : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
5621public:
5622 AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5623 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5624 if (!AA)
5625 return;
5626 // It is guaranteed that the alignment/offset are constants.
5627 Alignment = cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: AA->getAlignment()));
5628 if (Expr *Offset = AA->getOffset()) {
5629 OffsetCI = cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: Offset));
5630 if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
5631 OffsetCI = nullptr;
5632 }
5633 }
5634};
5635
5636/// Helper data structure to emit `AllocAlignAttr`.
5637class AllocAlignAttrEmitter final
5638 : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
5639public:
5640 AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
5641 const CallArgList &CallArgs)
5642 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5643 if (!AA)
5644 return;
5645 // Alignment may or may not be a constant, and that is okay.
5646 Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
5647 .getRValue(CGF)
5648 .getScalarVal();
5649 }
5650};
5651
5652} // namespace
5653
5654static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
5655 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Ty))
5656 return VT->getPrimitiveSizeInBits().getKnownMinValue();
5657 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty))
5658 return getMaxVectorWidth(Ty: AT->getElementType());
5659
5660 unsigned MaxVectorWidth = 0;
5661 if (auto *ST = dyn_cast<llvm::StructType>(Val: Ty))
5662 for (auto *I : ST->elements())
5663 MaxVectorWidth = std::max(a: MaxVectorWidth, b: getMaxVectorWidth(Ty: I));
5664 return MaxVectorWidth;
5665}
5666
5667RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
5668 const CGCallee &Callee,
5669 ReturnValueSlot ReturnValue,
5670 const CallArgList &CallArgs,
5671 llvm::CallBase **callOrInvoke, bool IsMustTail,
5672 SourceLocation Loc,
5673 bool IsVirtualFunctionPointerThunk) {
5674 // FIXME: We no longer need the types from CallArgs; lift up and simplify.
5675
5676 assert(Callee.isOrdinary() || Callee.isVirtual());
5677
5678 // Handle struct-return functions by passing a pointer to the
5679 // location that we would like to return into.
5680 QualType RetTy = CallInfo.getReturnType();
5681 const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
5682
5683 llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(FI: CallInfo);
5684
5685 const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
5686 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
5687 // We can only guarantee that a function is called from the correct
5688 // context/function based on the appropriate target attributes,
5689 // so only check in the case where we have both always_inline and target
5690 // since otherwise we could be making a conditional call after a check for
5691 // the proper cpu features (and it won't cause code generation issues due to
5692 // function based code generation).
5693 if ((TargetDecl->hasAttr<AlwaysInlineAttr>() &&
5694 (TargetDecl->hasAttr<TargetAttr>() ||
5695 (CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>()))) ||
5696 (CurFuncDecl && CurFuncDecl->hasAttr<FlattenAttr>() &&
5697 (CurFuncDecl->hasAttr<TargetAttr>() ||
5698 TargetDecl->hasAttr<TargetAttr>())))
5699 checkTargetFeatures(Loc, TargetDecl: FD);
5700 }
5701
5702 // Some architectures (such as x86-64) have the ABI changed based on
5703 // attribute-target/features. Give them a chance to diagnose.
5704 const FunctionDecl *CallerDecl = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
5705 const FunctionDecl *CalleeDecl = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
5706 CGM.getTargetCodeGenInfo().checkFunctionCallABI(CGM, CallLoc: Loc, Caller: CallerDecl,
5707 Callee: CalleeDecl, Args: CallArgs, ReturnType: RetTy);
5708
5709 // 1. Set up the arguments.
5710
5711 // If we're using inalloca, insert the allocation after the stack save.
5712 // FIXME: Do this earlier rather than hacking it in here!
5713 RawAddress ArgMemory = RawAddress::invalid();
5714 if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
5715 const llvm::DataLayout &DL = CGM.getDataLayout();
5716 llvm::Instruction *IP = CallArgs.getStackBase();
5717 llvm::AllocaInst *AI;
5718 if (IP) {
5719 IP = IP->getNextNode();
5720 AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(), "argmem",
5721 IP->getIterator());
5722 } else {
5723 AI = CreateTempAlloca(Ty: ArgStruct, Name: "argmem");
5724 }
5725 auto Align = CallInfo.getArgStructAlignment();
5726 AI->setAlignment(Align.getAsAlign());
5727 AI->setUsedWithInAlloca(true);
5728 assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
5729 ArgMemory = RawAddress(AI, ArgStruct, Align);
5730 }
5731
5732 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
5733 SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
5734
5735 // If the call returns a temporary with struct return, create a temporary
5736 // alloca to hold the result, unless one is given to us.
5737 Address SRetPtr = Address::invalid();
5738 // Original alloca for lifetime markers
5739 Address SRetAlloca = Address::invalid();
5740 bool NeedSRetLifetimeEnd = false;
5741 if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
5742 // For virtual function pointer thunks and musttail calls, we must always
5743 // forward an incoming SRet pointer to the callee, because a local alloca
5744 // would be de-allocated before the call. These cases both guarantee that
5745 // there will be an incoming SRet argument of the correct type.
5746 if ((IsVirtualFunctionPointerThunk || IsMustTail) && RetAI.isIndirect()) {
5747 SRetPtr = makeNaturalAddressForPointer(Ptr: CurFn->arg_begin() +
5748 IRFunctionArgs.getSRetArgNo(),
5749 T: RetTy, Alignment: CharUnits::fromQuantity(Quantity: 1));
5750 } else if (!ReturnValue.isNull()) {
5751 SRetPtr = ReturnValue.getAddress();
5752 } else {
5753 SRetPtr = CreateMemTempWithoutCast(T: RetTy, Name: "tmp");
5754 if (HaveInsertPoint() && ReturnValue.isUnused()) {
5755 NeedSRetLifetimeEnd = EmitLifetimeStart(Addr: SRetPtr.getBasePointer());
5756 if (NeedSRetLifetimeEnd)
5757 SRetAlloca = SRetPtr;
5758 }
5759 }
5760 if (IRFunctionArgs.hasSRetArg()) {
5761 // A mismatch between the allocated return value's AS and the target's
5762 // chosen IndirectAS can happen e.g. when passing the this pointer through
5763 // a chain involving stores to / loads from the DefaultAS; we address this
5764 // here, symmetrically with the handling we have for normal pointer args.
5765 if (SRetPtr.getAddressSpace() != RetAI.getIndirectAddrSpace()) {
5766 llvm::Value *V = SRetPtr.getBasePointer();
5767 llvm::Type *Ty = llvm::PointerType::get(C&: getLLVMContext(),
5768 AddressSpace: RetAI.getIndirectAddrSpace());
5769
5770 SRetPtr = SRetPtr.withPointer(NewPointer: performAddrSpaceCast(Src: V, DestTy: Ty),
5771 IsKnownNonNull: SRetPtr.isKnownNonNull());
5772 }
5773 IRCallArgs[IRFunctionArgs.getSRetArgNo()] =
5774 getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy);
5775 } else if (RetAI.isInAlloca()) {
5776 Address Addr =
5777 Builder.CreateStructGEP(Addr: ArgMemory, Index: RetAI.getInAllocaFieldIndex());
5778 Builder.CreateStore(Val: getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy), Addr);
5779 }
5780 }
5781
5782 RawAddress swiftErrorTemp = RawAddress::invalid();
5783 Address swiftErrorArg = Address::invalid();
5784
5785 // When passing arguments using temporary allocas, we need to add the
5786 // appropriate lifetime markers. This vector keeps track of all the lifetime
5787 // markers that need to be ended right after the call.
5788 SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
5789
5790 // Translate all of the arguments as necessary to match the IR lowering.
5791 assert(CallInfo.arg_size() == CallArgs.size() &&
5792 "Mismatch between function signature & arguments.");
5793 unsigned ArgNo = 0;
5794 CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
5795 for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
5796 I != E; ++I, ++info_it, ++ArgNo) {
5797 const ABIArgInfo &ArgInfo = info_it->info;
5798
5799 // Insert a padding argument to ensure proper alignment.
5800 if (IRFunctionArgs.hasPaddingArg(ArgNo))
5801 IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
5802 llvm::UndefValue::get(T: ArgInfo.getPaddingType());
5803
5804 unsigned FirstIRArg, NumIRArgs;
5805 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
5806
5807 bool ArgHasMaybeUndefAttr =
5808 IsArgumentMaybeUndef(TargetDecl, NumRequiredArgs: CallInfo.getNumRequiredArgs(), ArgNo);
5809
5810 switch (ArgInfo.getKind()) {
5811 case ABIArgInfo::InAlloca: {
5812 assert(NumIRArgs == 0);
5813 assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
5814 if (I->isAggregate()) {
5815 RawAddress Addr = I->hasLValue()
5816 ? I->getKnownLValue().getAddress()
5817 : I->getKnownRValue().getAggregateAddress();
5818 llvm::Instruction *Placeholder =
5819 cast<llvm::Instruction>(Val: Addr.getPointer());
5820
5821 if (!ArgInfo.getInAllocaIndirect()) {
5822 // Replace the placeholder with the appropriate argument slot GEP.
5823 CGBuilderTy::InsertPoint IP = Builder.saveIP();
5824 Builder.SetInsertPoint(Placeholder);
5825 Addr = Builder.CreateStructGEP(Addr: ArgMemory,
5826 Index: ArgInfo.getInAllocaFieldIndex());
5827 Builder.restoreIP(IP);
5828 } else {
5829 // For indirect things such as overaligned structs, replace the
5830 // placeholder with a regular aggregate temporary alloca. Store the
5831 // address of this alloca into the struct.
5832 Addr =
5833 CreateMemTempWithoutCast(T: info_it->type, Name: "inalloca.indirect.tmp");
5834 Address ArgSlot = Builder.CreateStructGEP(
5835 Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5836 Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5837 }
5838 deferPlaceholderReplacement(Old: Placeholder, New: Addr.getPointer());
5839 } else if (ArgInfo.getInAllocaIndirect()) {
5840 // Make a temporary alloca and store the address of it into the argument
5841 // struct.
5842 RawAddress Addr = CreateMemTempWithoutCast(
5843 T: I->Ty, Align: getContext().getTypeAlignInChars(T: I->Ty),
5844 Name: "indirect-arg-temp");
5845 I->copyInto(CGF&: *this, Addr);
5846 Address ArgSlot =
5847 Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5848 Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5849 } else {
5850 // Store the RValue into the argument struct.
5851 Address Addr =
5852 Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5853 Addr = Addr.withElementType(ElemTy: ConvertTypeForMem(T: I->Ty));
5854 I->copyInto(CGF&: *this, Addr);
5855 }
5856 break;
5857 }
5858
5859 case ABIArgInfo::Indirect:
5860 case ABIArgInfo::IndirectAliased: {
5861 assert(NumIRArgs == 1);
5862 if (I->isAggregate()) {
5863 // We want to avoid creating an unnecessary temporary+copy here;
5864 // however, we need one in three cases:
5865 // 1. If the argument is not byval, and we are required to copy the
5866 // source. (This case doesn't occur on any common architecture.)
5867 // 2. If the argument is byval, RV is not sufficiently aligned, and
5868 // we cannot force it to be sufficiently aligned.
5869 // 3. If the argument is byval, but RV is not located in default
5870 // or alloca address space.
5871 Address Addr = I->hasLValue()
5872 ? I->getKnownLValue().getAddress()
5873 : I->getKnownRValue().getAggregateAddress();
5874 CharUnits Align = ArgInfo.getIndirectAlign();
5875 const llvm::DataLayout *TD = &CGM.getDataLayout();
5876
5877 assert((FirstIRArg >= IRFuncTy->getNumParams() ||
5878 IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
5879 TD->getAllocaAddrSpace()) &&
5880 "indirect argument must be in alloca address space");
5881
5882 bool NeedCopy = false;
5883 if (Addr.getAlignment() < Align &&
5884 llvm::getOrEnforceKnownAlignment(V: Addr.emitRawPointer(CGF&: *this),
5885 PrefAlign: Align.getAsAlign(),
5886 DL: *TD) < Align.getAsAlign()) {
5887 NeedCopy = true;
5888 } else if (I->hasLValue()) {
5889 auto LV = I->getKnownLValue();
5890
5891 bool isByValOrRef =
5892 ArgInfo.isIndirectAliased() || ArgInfo.getIndirectByVal();
5893
5894 if (!isByValOrRef ||
5895 (LV.getAlignment() < getContext().getTypeAlignInChars(T: I->Ty))) {
5896 NeedCopy = true;
5897 }
5898
5899 if (isByValOrRef && Addr.getType()->getAddressSpace() !=
5900 ArgInfo.getIndirectAddrSpace()) {
5901 NeedCopy = true;
5902 }
5903 }
5904
5905 if (!NeedCopy) {
5906 // Skip the extra memcpy call.
5907 llvm::Value *V = getAsNaturalPointerTo(Addr, PointeeType: I->Ty);
5908 auto *T = llvm::PointerType::get(C&: CGM.getLLVMContext(),
5909 AddressSpace: ArgInfo.getIndirectAddrSpace());
5910
5911 // FIXME: This should not depend on the language address spaces, and
5912 // only the contextual values. If the address space mismatches, see if
5913 // we can look through a cast to a compatible address space value,
5914 // otherwise emit a copy.
5915 llvm::Value *Val = performAddrSpaceCast(Src: V, DestTy: T);
5916 if (ArgHasMaybeUndefAttr)
5917 Val = Builder.CreateFreeze(V: Val);
5918 IRCallArgs[FirstIRArg] = Val;
5919 break;
5920 }
5921 } else if (I->getType()->isArrayParameterType()) {
5922 // Don't produce a temporary for ArrayParameterType arguments.
5923 // ArrayParameterType arguments are only created from
5924 // HLSL_ArrayRValue casts and HLSLOutArgExpr expressions, both
5925 // of which create temporaries already. This allows us to just use the
5926 // scalar for the decayed array pointer as the argument directly.
5927 IRCallArgs[FirstIRArg] = I->getKnownRValue().getScalarVal();
5928 break;
5929 }
5930
5931 // For non-aggregate args and aggregate args meeting conditions above
5932 // we need to create an aligned temporary, and copy to it.
5933 RawAddress AI = CreateMemTempWithoutCast(
5934 T: I->Ty, Align: ArgInfo.getIndirectAlign(), Name: "byval-temp");
5935 llvm::Value *Val = getAsNaturalPointerTo(Addr: AI, PointeeType: I->Ty);
5936 if (ArgHasMaybeUndefAttr)
5937 Val = Builder.CreateFreeze(V: Val);
5938 IRCallArgs[FirstIRArg] = Val;
5939
5940 // Emit lifetime markers for the temporary alloca and add cleanup code to
5941 // emit the end lifetime marker after the call.
5942 if (EmitLifetimeStart(Addr: AI.getPointer()))
5943 CallLifetimeEndAfterCall.emplace_back(Args&: AI);
5944
5945 // Generate the copy.
5946 I->copyInto(CGF&: *this, Addr: AI);
5947 break;
5948 }
5949
5950 case ABIArgInfo::Ignore:
5951 assert(NumIRArgs == 0);
5952 break;
5953
5954 case ABIArgInfo::Extend:
5955 case ABIArgInfo::Direct: {
5956 if (!isa<llvm::StructType>(Val: ArgInfo.getCoerceToType()) &&
5957 ArgInfo.getCoerceToType() == ConvertType(T: info_it->type) &&
5958 ArgInfo.getDirectOffset() == 0) {
5959 assert(NumIRArgs == 1);
5960 llvm::Value *V;
5961 if (!I->isAggregate())
5962 V = I->getKnownRValue().getScalarVal();
5963 else
5964 V = Builder.CreateLoad(
5965 Addr: I->hasLValue() ? I->getKnownLValue().getAddress()
5966 : I->getKnownRValue().getAggregateAddress());
5967
5968 // Implement swifterror by copying into a new swifterror argument.
5969 // We'll write back in the normal path out of the call.
5970 if (CallInfo.getExtParameterInfo(argIndex: ArgNo).getABI() ==
5971 ParameterABI::SwiftErrorResult) {
5972 assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
5973
5974 QualType pointeeTy = I->Ty->getPointeeType();
5975 swiftErrorArg = makeNaturalAddressForPointer(
5976 Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
5977
5978 swiftErrorTemp = CreateMemTempWithoutCast(
5979 T: pointeeTy, Align: getPointerAlign(), Name: "swifterror.temp");
5980 V = swiftErrorTemp.getPointer();
5981 cast<llvm::AllocaInst>(Val: V)->setSwiftError(true);
5982
5983 llvm::Value *errorValue = Builder.CreateLoad(Addr: swiftErrorArg);
5984 Builder.CreateStore(Val: errorValue, Addr: swiftErrorTemp);
5985 }
5986
5987 // We might have to widen integers, but we should never truncate.
5988 if (ArgInfo.getCoerceToType() != V->getType() &&
5989 V->getType()->isIntegerTy())
5990 V = Builder.CreateZExt(V, DestTy: ArgInfo.getCoerceToType());
5991
5992 // The only plausible mismatch here would be for pointer address spaces.
5993 // We assume that the target has a reasonable mapping for the DefaultAS
5994 // (it can be casted to from incoming specific ASes), and insert an AS
5995 // cast to address the mismatch.
5996 if (FirstIRArg < IRFuncTy->getNumParams() &&
5997 V->getType() != IRFuncTy->getParamType(i: FirstIRArg)) {
5998 assert(V->getType()->isPointerTy() && "Only pointers can mismatch!");
5999 V = performAddrSpaceCast(Src: V, DestTy: IRFuncTy->getParamType(i: FirstIRArg));
6000 }
6001
6002 if (ArgHasMaybeUndefAttr)
6003 V = Builder.CreateFreeze(V);
6004 IRCallArgs[FirstIRArg] = V;
6005 break;
6006 }
6007
6008 llvm::StructType *STy =
6009 dyn_cast<llvm::StructType>(Val: ArgInfo.getCoerceToType());
6010
6011 // FIXME: Avoid the conversion through memory if possible.
6012 Address Src = Address::invalid();
6013 if (!I->isAggregate()) {
6014 Src = CreateMemTempWithoutCast(T: I->Ty, Name: "coerce");
6015 I->copyInto(CGF&: *this, Addr: Src);
6016 } else {
6017 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
6018 : I->getKnownRValue().getAggregateAddress();
6019 }
6020
6021 // If the value is offset in memory, apply the offset now.
6022 Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
6023
6024 // Fast-isel and the optimizer generally like scalar values better than
6025 // FCAs, so we flatten them if this is safe to do for this argument.
6026 if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
6027 llvm::Type *SrcTy = Src.getElementType();
6028 llvm::TypeSize SrcTypeSize =
6029 CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
6030 llvm::TypeSize DstTypeSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
6031 if (SrcTypeSize.isScalable()) {
6032 assert(STy->containsHomogeneousScalableVectorTypes() &&
6033 "ABI only supports structure with homogeneous scalable vector "
6034 "type");
6035 assert(SrcTypeSize == DstTypeSize &&
6036 "Only allow non-fractional movement of structure with "
6037 "homogeneous scalable vector type");
6038 assert(NumIRArgs == STy->getNumElements());
6039
6040 llvm::Value *StoredStructValue =
6041 Builder.CreateLoad(Addr: Src, Name: Src.getName() + ".tuple");
6042 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
6043 llvm::Value *Extract = Builder.CreateExtractValue(
6044 Agg: StoredStructValue, Idxs: i, Name: Src.getName() + ".extract" + Twine(i));
6045 IRCallArgs[FirstIRArg + i] = Extract;
6046 }
6047 } else {
6048 uint64_t SrcSize = SrcTypeSize.getFixedValue();
6049 uint64_t DstSize = DstTypeSize.getFixedValue();
6050 bool HasPFPFields = getContext().hasPFPFields(Ty: I->Ty);
6051
6052 // If the source type is smaller than the destination type of the
6053 // coerce-to logic, copy the source value into a temp alloca the size
6054 // of the destination type to allow loading all of it. The bits past
6055 // the source value are left undef.
6056 if (HasPFPFields || SrcSize < DstSize) {
6057 Address TempAlloca = CreateTempAlloca(Ty: STy, align: Src.getAlignment(),
6058 Name: Src.getName() + ".coerce");
6059 if (HasPFPFields) {
6060 // Structures with PFP fields require a coerced load to remove any
6061 // pointer signatures.
6062 Builder.CreateStore(
6063 Val: CreatePFPCoercedLoad(Src, SrcFETy: I->Ty, Ty: ArgInfo.getCoerceToType(),
6064 CGF&: *this),
6065 Addr: TempAlloca);
6066 } else
6067 Builder.CreateMemCpy(Dest: TempAlloca, Src, Size: SrcSize);
6068 Src = TempAlloca;
6069 } else {
6070 Src = Src.withElementType(ElemTy: STy);
6071 }
6072
6073 assert(NumIRArgs == STy->getNumElements());
6074 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
6075 Address EltPtr = Builder.CreateStructGEP(Addr: Src, Index: i);
6076 llvm::Value *LI = Builder.CreateLoad(Addr: EltPtr);
6077 if (ArgHasMaybeUndefAttr)
6078 LI = Builder.CreateFreeze(V: LI);
6079 IRCallArgs[FirstIRArg + i] = LI;
6080 }
6081 }
6082 } else {
6083 // In the simple case, just pass the coerced loaded value.
6084 assert(NumIRArgs == 1);
6085 llvm::Value *Load =
6086 CreateCoercedLoad(Src, SrcFETy: I->Ty, Ty: ArgInfo.getCoerceToType(), CGF&: *this);
6087
6088 if (CallInfo.isCmseNSCall()) {
6089 // For certain parameter types, clear padding bits, as they may reveal
6090 // sensitive information.
6091 // Small struct/union types are passed as integer arrays.
6092 auto *ATy = dyn_cast<llvm::ArrayType>(Val: Load->getType());
6093 if (ATy != nullptr && isa<RecordType>(Val: I->Ty.getCanonicalType()))
6094 Load = EmitCMSEClearRecord(Src: Load, ATy, QTy: I->Ty);
6095 }
6096
6097 if (ArgHasMaybeUndefAttr)
6098 Load = Builder.CreateFreeze(V: Load);
6099 IRCallArgs[FirstIRArg] = Load;
6100 }
6101
6102 break;
6103 }
6104
6105 case ABIArgInfo::CoerceAndExpand: {
6106 auto coercionType = ArgInfo.getCoerceAndExpandType();
6107 auto layout = CGM.getDataLayout().getStructLayout(Ty: coercionType);
6108 auto unpaddedCoercionType = ArgInfo.getUnpaddedCoerceAndExpandType();
6109 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
6110
6111 Address addr = Address::invalid();
6112 RawAddress AllocaAddr = RawAddress::invalid();
6113 bool NeedLifetimeEnd = false;
6114 if (I->isAggregate()) {
6115 addr = I->hasLValue() ? I->getKnownLValue().getAddress()
6116 : I->getKnownRValue().getAggregateAddress();
6117
6118 } else {
6119 RValue RV = I->getKnownRValue();
6120 assert(RV.isScalar()); // complex should always just be direct
6121
6122 llvm::Type *scalarType = RV.getScalarVal()->getType();
6123 auto scalarAlign = CGM.getDataLayout().getPrefTypeAlign(Ty: scalarType);
6124
6125 // Materialize to a temporary.
6126 addr = CreateTempAlloca(Ty: RV.getScalarVal()->getType(),
6127 align: CharUnits::fromQuantity(Quantity: std::max(
6128 a: layout->getAlignment(), b: scalarAlign)),
6129 Name: "tmp",
6130 /*ArraySize=*/nullptr, Alloca: &AllocaAddr);
6131 NeedLifetimeEnd = EmitLifetimeStart(Addr: AllocaAddr.getPointer());
6132
6133 Builder.CreateStore(Val: RV.getScalarVal(), Addr: addr);
6134 }
6135
6136 addr = addr.withElementType(ElemTy: coercionType);
6137
6138 unsigned IRArgPos = FirstIRArg;
6139 unsigned unpaddedIndex = 0;
6140 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
6141 llvm::Type *eltType = coercionType->getElementType(N: i);
6142 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
6143 continue;
6144 Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
6145 llvm::Value *elt = CreateCoercedLoad(
6146 Src: eltAddr, SrcFETy: I->Ty,
6147 Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
6148 : unpaddedCoercionType,
6149 CGF&: *this);
6150 if (ArgHasMaybeUndefAttr)
6151 elt = Builder.CreateFreeze(V: elt);
6152 IRCallArgs[IRArgPos++] = elt;
6153 }
6154 assert(IRArgPos == FirstIRArg + NumIRArgs);
6155
6156 if (NeedLifetimeEnd)
6157 EmitLifetimeEnd(Addr: AllocaAddr.getPointer());
6158 break;
6159 }
6160
6161 case ABIArgInfo::Expand: {
6162 unsigned IRArgPos = FirstIRArg;
6163 ExpandTypeToArgs(Ty: I->Ty, Arg: *I, IRFuncTy, IRCallArgs, IRCallArgPos&: IRArgPos);
6164 assert(IRArgPos == FirstIRArg + NumIRArgs);
6165 break;
6166 }
6167
6168 case ABIArgInfo::TargetSpecific: {
6169 Address Src = Address::invalid();
6170 if (!I->isAggregate()) {
6171 Src = CreateMemTempWithoutCast(T: I->Ty, Name: "target_coerce");
6172 I->copyInto(CGF&: *this, Addr: Src);
6173 } else {
6174 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
6175 : I->getKnownRValue().getAggregateAddress();
6176 }
6177
6178 // If the value is offset in memory, apply the offset now.
6179 Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
6180 llvm::Value *Load =
6181 CGM.getABIInfo().createCoercedLoad(SrcAddr: Src, AI: ArgInfo, CGF&: *this);
6182 IRCallArgs[FirstIRArg] = Load;
6183 break;
6184 }
6185 }
6186 }
6187
6188 const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(CGF&: *this);
6189 llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
6190
6191 // If we're using inalloca, set up that argument.
6192 if (ArgMemory.isValid()) {
6193 llvm::Value *Arg = ArgMemory.getPointer();
6194 assert(IRFunctionArgs.hasInallocaArg());
6195 IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
6196 }
6197
6198 // 2. Prepare the function pointer.
6199
6200 // If the callee is a bitcast of a non-variadic function to have a
6201 // variadic function pointer type, check to see if we can remove the
6202 // bitcast. This comes up with unprototyped functions.
6203 //
6204 // This makes the IR nicer, but more importantly it ensures that we
6205 // can inline the function at -O0 if it is marked always_inline.
6206 auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
6207 llvm::Value *Ptr) -> llvm::Function * {
6208 if (!CalleeFT->isVarArg())
6209 return nullptr;
6210
6211 // Get underlying value if it's a bitcast
6212 if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Val: Ptr)) {
6213 if (CE->getOpcode() == llvm::Instruction::BitCast)
6214 Ptr = CE->getOperand(i_nocapture: 0);
6215 }
6216
6217 llvm::Function *OrigFn = dyn_cast<llvm::Function>(Val: Ptr);
6218 if (!OrigFn)
6219 return nullptr;
6220
6221 llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
6222
6223 // If the original type is variadic, or if any of the component types
6224 // disagree, we cannot remove the cast.
6225 if (OrigFT->isVarArg() ||
6226 OrigFT->getNumParams() != CalleeFT->getNumParams() ||
6227 OrigFT->getReturnType() != CalleeFT->getReturnType())
6228 return nullptr;
6229
6230 for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i)
6231 if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
6232 return nullptr;
6233
6234 return OrigFn;
6235 };
6236
6237 if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {
6238 CalleePtr = OrigFn;
6239 IRFuncTy = OrigFn->getFunctionType();
6240 }
6241
6242 // 3. Perform the actual call.
6243
6244 // Deactivate any cleanups that we're supposed to do immediately before
6245 // the call.
6246 if (!CallArgs.getCleanupsToDeactivate().empty())
6247 deactivateArgCleanupsBeforeCall(CGF&: *this, CallArgs);
6248
6249 // Update the largest vector width if any arguments have vector types.
6250 for (unsigned i = 0; i < IRCallArgs.size(); ++i)
6251 LargestVectorWidth = std::max(a: LargestVectorWidth,
6252 b: getMaxVectorWidth(Ty: IRCallArgs[i]->getType()));
6253
6254 // Compute the calling convention and attributes.
6255 unsigned CallingConv;
6256 llvm::AttributeList Attrs;
6257 CGM.ConstructAttributeList(Name: CalleePtr->getName(), FI: CallInfo,
6258 CalleeInfo: Callee.getAbstractInfo(), AttrList&: Attrs, CallingConv,
6259 /*AttrOnCallSite=*/true,
6260 /*IsThunk=*/false);
6261
6262 if (CallingConv == llvm::CallingConv::X86_VectorCall &&
6263 getTarget().getTriple().isWindowsArm64EC()) {
6264 CGM.Error(loc: Loc, error: "__vectorcall calling convention is not currently "
6265 "supported");
6266 }
6267
6268 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
6269 if (FD->hasAttr<StrictFPAttr>())
6270 // All calls within a strictfp function are marked strictfp
6271 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::StrictFP);
6272
6273 // If -ffast-math is enabled and the function is guarded by an
6274 // '__attribute__((optnone)) adjust the memory attribute so the BE emits the
6275 // library call instead of the intrinsic.
6276 if (FD->hasAttr<OptimizeNoneAttr>() && getLangOpts().FastMath)
6277 CGM.AdjustMemoryAttribute(Name: CalleePtr->getName(), CalleeInfo: Callee.getAbstractInfo(),
6278 Attrs);
6279 }
6280 // Add call-site nomerge attribute if exists.
6281 if (InNoMergeAttributedStmt)
6282 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoMerge);
6283
6284 // Add call-site noinline attribute if exists.
6285 if (InNoInlineAttributedStmt)
6286 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoInline);
6287
6288 // Add call-site always_inline attribute if exists.
6289 // Note: This corresponds to the [[clang::always_inline]] statement attribute.
6290 if (InAlwaysInlineAttributedStmt &&
6291 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
6292 Caller: CallerDecl, Callee: CalleeDecl))
6293 Attrs =
6294 Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::AlwaysInline);
6295
6296 // Remove call-site convergent attribute if requested.
6297 if (InNoConvergentAttributedStmt)
6298 Attrs =
6299 Attrs.removeFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::Convergent);
6300
6301 // Apply some call-site-specific attributes.
6302 // TODO: work this into building the attribute set.
6303
6304 // Apply always_inline to all calls within flatten functions.
6305 // FIXME: should this really take priority over __try, below?
6306 if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
6307 !InNoInlineAttributedStmt &&
6308 !(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>()) &&
6309 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
6310 Caller: CallerDecl, Callee: CalleeDecl)) {
6311 Attrs =
6312 Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::AlwaysInline);
6313 }
6314
6315 // Disable inlining inside SEH __try blocks.
6316 if (isSEHTryScope()) {
6317 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoInline);
6318 }
6319
6320 // Decide whether to use a call or an invoke.
6321 bool CannotThrow;
6322 if (currentFunctionUsesSEHTry()) {
6323 // SEH cares about asynchronous exceptions, so everything can "throw."
6324 CannotThrow = false;
6325 } else if (isCleanupPadScope() &&
6326 EHPersonality::get(CGF&: *this).isMSVCXXPersonality()) {
6327 // The MSVC++ personality will implicitly terminate the program if an
6328 // exception is thrown during a cleanup outside of a try/catch.
6329 // We don't need to model anything in IR to get this behavior.
6330 CannotThrow = true;
6331 } else {
6332 // Otherwise, nounwind call sites will never throw.
6333 CannotThrow = Attrs.hasFnAttr(Kind: llvm::Attribute::NoUnwind);
6334
6335 if (auto *FPtr = dyn_cast<llvm::Function>(Val: CalleePtr))
6336 if (FPtr->hasFnAttribute(Kind: llvm::Attribute::NoUnwind))
6337 CannotThrow = true;
6338 }
6339
6340 // If we made a temporary, be sure to clean up after ourselves. Note that we
6341 // can't depend on being inside of an ExprWithCleanups, so we need to manually
6342 // pop this cleanup later on. Being eager about this is OK, since this
6343 // temporary is 'invisible' outside of the callee.
6344 // Use the original alloca pointer (before any addrspacecast) for the
6345 // lifetime end marker, since lifetime intrinsics must reference the alloca
6346 // address space.
6347 if (NeedSRetLifetimeEnd)
6348 pushFullExprCleanup<CallLifetimeEnd>(kind: NormalEHLifetimeMarker, A: SRetAlloca);
6349
6350 llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();
6351
6352 SmallVector<llvm::OperandBundleDef, 1> BundleList =
6353 getBundlesForFunclet(Callee: CalleePtr);
6354
6355 if (SanOpts.has(K: SanitizerKind::KCFI) &&
6356 !isa_and_nonnull<FunctionDecl>(Val: TargetDecl))
6357 EmitKCFIOperandBundle(Callee: ConcreteCallee, Bundles&: BundleList);
6358
6359 // Add the pointer-authentication bundle.
6360 EmitPointerAuthOperandBundle(Info: ConcreteCallee.getPointerAuthInfo(), Bundles&: BundleList);
6361
6362 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl))
6363 if (FD->hasAttr<StrictFPAttr>())
6364 // All calls within a strictfp function are marked strictfp
6365 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::StrictFP);
6366
6367 AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
6368 Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
6369
6370 AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
6371 Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
6372
6373 // Emit the actual call/invoke instruction.
6374 llvm::CallBase *CI;
6375 if (!InvokeDest) {
6376 CI = Builder.CreateCall(FTy: IRFuncTy, Callee: CalleePtr, Args: IRCallArgs, OpBundles: BundleList);
6377 } else {
6378 llvm::BasicBlock *Cont = createBasicBlock(name: "invoke.cont");
6379 CI = Builder.CreateInvoke(Ty: IRFuncTy, Callee: CalleePtr, NormalDest: Cont, UnwindDest: InvokeDest, Args: IRCallArgs,
6380 OpBundles: BundleList);
6381 EmitBlock(BB: Cont);
6382 }
6383 if (CI->getCalledFunction() && CI->getCalledFunction()->hasName() &&
6384 CI->getCalledFunction()->getName().starts_with(Prefix: "_Z4sqrt")) {
6385 SetSqrtFPAccuracy(CI);
6386 }
6387 if (callOrInvoke) {
6388 *callOrInvoke = CI;
6389 if (CGM.getCodeGenOpts().CallGraphSection) {
6390 QualType CST;
6391 if (TargetDecl && TargetDecl->getFunctionType())
6392 CST = QualType(TargetDecl->getFunctionType(), 0);
6393 else if (const auto *FPT =
6394 Callee.getAbstractInfo().getCalleeFunctionProtoType())
6395 CST = QualType(FPT, 0);
6396 else if (const auto *FT =
6397 Callee.getAbstractInfo().getCalleeFunctionType())
6398 CST = QualType(FT, 0);
6399 else
6400 llvm_unreachable(
6401 "Cannot find the callee type to generate callee_type metadata.");
6402
6403 // Set type identifier metadata of indirect calls for call graph section.
6404 if (!CST.isNull()) {
6405 if (!CST->isFunctionProtoType()) {
6406 // Reconstruct a prototype for unprototyped callees from the argument
6407 // types passed at the call site (after default argument promotion).
6408 //
6409 // Basic Rationale & K&R-Style Definitions:
6410 // The argument types in CallArgs have already undergone C default
6411 // argument promotion (e.g., char/short -> int, float -> double).
6412 // Furthermore, for a K&R-style
6413 // definition (e.g., void foo(x) short x; { ... }), canonical C ABI
6414 // semantics expect the promoted type (int) at the call boundary
6415 // and implicitly cast down to the declared type (short) inside the
6416 // function. Therefore, signature computation at K&R definition
6417 // sites must also apply default argument promotion (yielding
6418 // void(int), not void(short)) so definition and call sites match.
6419 //
6420 // Signature Strictness & Normalization:
6421 // Since type identifier matching relies on exact hash equality, any
6422 // tolerance for C compatibility rules must be done by normalizing
6423 // types before hashing.
6424 // - Standard C allows certain exceptions for unprototyped calls (and
6425 // variadic va_arg), such as differences in signedness (e.g.,
6426 // passing an int to an unsigned int parameter) or
6427 // interchangeability of enum types with their underlying integer
6428 // types.
6429 // - Existing CFI normalization (e.g.,
6430 // -fsanitize-cfi-icall-experimental-normalize-integers) normalizes
6431 // types by bit-width and signedness (e.g., int vs long on LP64,
6432 // which C does not treat as compatible), but does not normalize
6433 // away signedness or enum mismatches.
6434 // - In the future, whether to normalize away signedness, enums, or
6435 // integer bit-widths depends on whether call graph analysis should
6436 // err on the side of inclusion (admitting any C-valid call) or
6437 // strictness (like CFI). Any normalization applied here at the call
6438 // site must remain strictly matched with definition-site
6439 // type signature computation.
6440 if (const auto *FNPT = CST->getAs<FunctionNoProtoType>()) {
6441 SmallVector<QualType, 8> ParamTypes;
6442 // CallArgs already contains default-promoted argument types for
6443 // unprototyped calls.
6444 for (const CallArg &Arg : CallArgs)
6445 ParamTypes.push_back(Elt: Arg.getType());
6446 FunctionProtoType::ExtProtoInfo EPI;
6447 CST = getContext().getFunctionType(ResultTy: FNPT->getReturnType(),
6448 Args: ParamTypes, EPI);
6449 }
6450
6451 llvm::Metadata *MD =
6452 CGM.CreateMetadataIdentifierForCallGraphType(T: CST);
6453 StringRef TypeStr;
6454 if (auto *MDS = dyn_cast_or_null<llvm::MDString>(Val: MD))
6455 TypeStr = MDS->getString();
6456
6457 CGM.getDiags().Report(Loc, DiagID: diag::warn_cgs_no_proto) << CST << TypeStr;
6458 }
6459 CGM.createCalleeTypeMetadataForIcall(QT: CST, CB: *callOrInvoke);
6460 }
6461 }
6462 }
6463
6464 // If this is within a function that has the guard(nocf) attribute and is an
6465 // indirect call, add the "guard_nocf" attribute to this call to indicate that
6466 // Control Flow Guard checks should not be added, even if the call is inlined.
6467 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
6468 if (const auto *A = FD->getAttr<CFGuardAttr>()) {
6469 if (A->getGuard() == CFGuardAttr::GuardArg::nocf &&
6470 !CI->getCalledFunction())
6471 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: "guard_nocf");
6472 }
6473 }
6474
6475 // Apply the attributes and calling convention.
6476 CI->setAttributes(Attrs);
6477 CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
6478
6479 // Apply various metadata.
6480
6481 if (!CI->getType()->isVoidTy())
6482 CI->setName("call");
6483
6484 if (CGM.shouldEmitConvergenceTokens() && CI->isConvergent())
6485 CI = addConvergenceControlToken(Input: CI);
6486
6487 // Update largest vector width from the return type.
6488 LargestVectorWidth =
6489 std::max(a: LargestVectorWidth, b: getMaxVectorWidth(Ty: CI->getType()));
6490
6491 // Insert instrumentation or attach profile metadata at indirect call sites.
6492 // For more details, see the comment before the definition of
6493 // IPVK_IndirectCallTarget in InstrProfData.inc.
6494 if (!CI->getCalledFunction())
6495 PGO->valueProfile(Builder, ValueKind: llvm::IPVK_IndirectCallTarget, ValueSite: CI, ValuePtr: CalleePtr);
6496
6497 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
6498 // optimizer it can aggressively ignore unwind edges.
6499 if (CGM.getLangOpts().ObjCAutoRefCount)
6500 AddObjCARCExceptionMetadata(Inst: CI);
6501
6502 // Set tail call kind if necessary.
6503 bool IsPPC = getTarget().getTriple().isPPC();
6504 bool IsMIPS = getTarget().getTriple().isMIPS();
6505 bool HasMips16 = false;
6506 if (IsMIPS) {
6507 const TargetOptions &TargetOpts = getTarget().getTargetOpts();
6508 HasMips16 = TargetOpts.FeatureMap.lookup(Key: "mips16");
6509 if (!HasMips16)
6510 HasMips16 = llvm::is_contained(Range: TargetOpts.Features, Element: "+mips16");
6511 }
6512 if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(Val: CI)) {
6513 if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
6514 Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
6515 else if (IsMustTail) {
6516 if (IsPPC) {
6517 if (getTarget().getTriple().isOSAIX())
6518 CGM.getDiags().Report(Loc, DiagID: diag::err_aix_musttail_unsupported);
6519 else if (!getTarget().hasFeature(Feature: "pcrelative-memops")) {
6520 if (getTarget().hasFeature(Feature: "longcall"))
6521 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail) << 0;
6522 else if (Call->isIndirectCall())
6523 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail) << 1;
6524 else if (isa_and_nonnull<FunctionDecl>(Val: TargetDecl)) {
6525 if (!cast<FunctionDecl>(Val: TargetDecl)->isDefined())
6526 // The undefined callee may be a forward declaration. Without
6527 // knowning all symbols in the module, we won't know the symbol is
6528 // defined or not. Collect all these symbols for later diagnosing.
6529 CGM.addUndefinedGlobalForTailCall(
6530 Global: {cast<FunctionDecl>(Val: TargetDecl), Loc});
6531 else {
6532 llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(
6533 GD: GlobalDecl(cast<FunctionDecl>(Val: TargetDecl)));
6534 if (llvm::GlobalValue::isWeakForLinker(Linkage) ||
6535 llvm::GlobalValue::isDiscardableIfUnused(Linkage))
6536 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail)
6537 << 2;
6538 }
6539 }
6540 }
6541 }
6542 if (IsMIPS) {
6543 if (HasMips16)
6544 CGM.getDiags().Report(Loc, DiagID: diag::err_mips_impossible_musttail) << 0;
6545 else if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl))
6546 CGM.addUndefinedGlobalForTailCall(Global: {FD, Loc});
6547 }
6548 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
6549 }
6550 }
6551
6552 // Add metadata for calls to MSAllocator functions
6553 if (getDebugInfo() && TargetDecl && TargetDecl->hasAttr<MSAllocatorAttr>())
6554 getDebugInfo()->addHeapAllocSiteMetadata(CallSite: CI, AllocatedTy: RetTy->getPointeeType(), Loc);
6555
6556 // Add srcloc metadata for [[gnu::error/warning]] diagnostics. When
6557 // ShowInliningChain is enabled, also track inline/static calls for the
6558 // heuristic fallback when debug info is not available. This heuristic is
6559 // conservative and best-effort since static or inline-annotated functions
6560 // are still not guaranteed to be inlined.
6561 if (TargetDecl) {
6562 bool NeedSrcLoc = TargetDecl->hasAttr<ErrorAttr>();
6563 if (!NeedSrcLoc && CGM.getCodeGenOpts().ShowInliningChain) {
6564 if (const auto *FD = dyn_cast<FunctionDecl>(Val: TargetDecl))
6565 NeedSrcLoc = FD->isInlined() || FD->hasAttr<AlwaysInlineAttr>() ||
6566 FD->getStorageClass() == SC_Static ||
6567 FD->isInAnonymousNamespace();
6568 }
6569 if (NeedSrcLoc) {
6570 auto *Line = llvm::ConstantInt::get(Ty: Int64Ty, V: Loc.getRawEncoding());
6571 auto *MD = llvm::ConstantAsMetadata::get(C: Line);
6572 CI->setMetadata(Kind: "srcloc", Node: llvm::MDNode::get(Context&: getLLVMContext(), MDs: {MD}));
6573 }
6574 }
6575
6576 // 4. Finish the call.
6577
6578 // If the call doesn't return, finish the basic block and clear the
6579 // insertion point; this allows the rest of IRGen to discard
6580 // unreachable code.
6581 if (CI->doesNotReturn()) {
6582 if (NeedSRetLifetimeEnd)
6583 PopCleanupBlock();
6584
6585 // Strip away the noreturn attribute to better diagnose unreachable UB.
6586 if (SanOpts.has(K: SanitizerKind::Unreachable)) {
6587 // Also remove from function since CallBase::hasFnAttr additionally checks
6588 // attributes of the called function.
6589 if (auto *F = CI->getCalledFunction())
6590 F->removeFnAttr(Kind: llvm::Attribute::NoReturn);
6591 CI->removeFnAttr(Kind: llvm::Attribute::NoReturn);
6592
6593 // Avoid incompatibility with ASan which relies on the `noreturn`
6594 // attribute to insert handler calls.
6595 if (SanOpts.hasOneOf(K: SanitizerKind::Address |
6596 SanitizerKind::KernelAddress)) {
6597 SanitizerScope SanScope(this);
6598 llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);
6599 Builder.SetInsertPoint(CI);
6600 auto *FnType = llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
6601 llvm::FunctionCallee Fn =
6602 CGM.CreateRuntimeFunction(Ty: FnType, Name: "__asan_handle_no_return");
6603 EmitNounwindRuntimeCall(callee: Fn);
6604 }
6605 }
6606
6607 EmitUnreachable(Loc);
6608 Builder.ClearInsertionPoint();
6609
6610 // FIXME: For now, emit a dummy basic block because expr emitters in
6611 // generally are not ready to handle emitting expressions at unreachable
6612 // points.
6613 EnsureInsertPoint();
6614
6615 // Return a reasonable RValue.
6616 return GetUndefRValue(Ty: RetTy);
6617 }
6618
6619 // If this is a musttail call, return immediately. We do not branch to the
6620 // epilogue in this case.
6621 if (IsMustTail) {
6622 for (auto it = EHStack.find(sp: CurrentCleanupScopeDepth); it != EHStack.end();
6623 ++it) {
6624 // A noexcept caller pushes an EHTerminateScope to call std::terminate()
6625 // if an exception escapes. A musttail call replaces the caller's frame,
6626 // removing this handler. This is safe if the callee is also nounwind:
6627 // the callee's own noexcept handler prevents any exception from reaching
6628 // where the caller's handler would have been.
6629 if (isa<EHTerminateScope>(Val: &*it)) {
6630 if (CI->doesNotThrow())
6631 continue;
6632 CGM.getDiags().Report(Loc: MustTailCall->getBeginLoc(),
6633 DiagID: diag::err_musttail_noexcept_mismatch);
6634 break;
6635 }
6636 EHCleanupScope *Cleanup = dyn_cast<EHCleanupScope>(Val: &*it);
6637 // Fake uses can be safely emitted immediately prior to the tail call, so
6638 // we choose to emit them just before the call here.
6639 if (Cleanup && Cleanup->isFakeUse()) {
6640 CGBuilderTy::InsertPointGuard IPG(Builder);
6641 Builder.SetInsertPoint(CI);
6642 Cleanup->getCleanup()->Emit(CGF&: *this, flags: EHScopeStack::Cleanup::Flags());
6643 } else if (!(Cleanup &&
6644 Cleanup->getCleanup()->isRedundantBeforeReturn())) {
6645 CGM.ErrorUnsupported(S: MustTailCall, Type: "tail call skipping over cleanups");
6646 }
6647 }
6648 if (CI->getType()->isVoidTy())
6649 Builder.CreateRetVoid();
6650 else
6651 Builder.CreateRet(V: CI);
6652 Builder.ClearInsertionPoint();
6653 EnsureInsertPoint();
6654 return GetUndefRValue(Ty: RetTy);
6655 }
6656
6657 // Perform the swifterror writeback.
6658 if (swiftErrorTemp.isValid()) {
6659 llvm::Value *errorResult = Builder.CreateLoad(Addr: swiftErrorTemp);
6660 Builder.CreateStore(Val: errorResult, Addr: swiftErrorArg);
6661 }
6662
6663 // Emit any call-associated writebacks immediately. Arguably this
6664 // should happen after any return-value munging.
6665 if (CallArgs.hasWritebacks())
6666 EmitWritebacks(args: CallArgs);
6667
6668 // The stack cleanup for inalloca arguments has to run out of the normal
6669 // lexical order, so deactivate it and run it manually here.
6670 CallArgs.freeArgumentMemory(CGF&: *this);
6671
6672 // Extract the return value.
6673 RValue Ret;
6674
6675 // If the current function is a virtual function pointer thunk, avoid copying
6676 // the return value of the musttail call to a temporary.
6677 if (IsVirtualFunctionPointerThunk) {
6678 Ret = RValue::get(V: CI);
6679 } else {
6680 Ret = [&] {
6681 switch (RetAI.getKind()) {
6682 case ABIArgInfo::CoerceAndExpand: {
6683 auto coercionType = RetAI.getCoerceAndExpandType();
6684
6685 Address addr = SRetPtr.withElementType(ElemTy: coercionType);
6686
6687 assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());
6688 bool requiresExtract = isa<llvm::StructType>(Val: CI->getType());
6689
6690 unsigned unpaddedIndex = 0;
6691 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
6692 llvm::Type *eltType = coercionType->getElementType(N: i);
6693 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
6694 continue;
6695 Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
6696 llvm::Value *elt = CI;
6697 if (requiresExtract)
6698 elt = Builder.CreateExtractValue(Agg: elt, Idxs: unpaddedIndex++);
6699 else
6700 assert(unpaddedIndex == 0);
6701 Builder.CreateStore(Val: elt, Addr: eltAddr);
6702 }
6703 [[fallthrough]];
6704 }
6705
6706 case ABIArgInfo::InAlloca:
6707 case ABIArgInfo::Indirect: {
6708 RValue ret = convertTempToRValue(addr: SRetPtr, type: RetTy, Loc: SourceLocation());
6709 if (NeedSRetLifetimeEnd)
6710 PopCleanupBlock();
6711 return ret;
6712 }
6713
6714 case ABIArgInfo::Ignore:
6715 // If we are ignoring an argument that had a result, make sure to
6716 // construct the appropriate return value for our caller.
6717 return GetUndefRValue(Ty: RetTy);
6718
6719 case ABIArgInfo::Extend:
6720 case ABIArgInfo::Direct: {
6721 llvm::Type *RetIRTy = ConvertType(T: RetTy);
6722 if (RetAI.getCoerceToType() == RetIRTy &&
6723 RetAI.getDirectOffset() == 0) {
6724 switch (getEvaluationKind(T: RetTy)) {
6725 case TEK_Complex: {
6726 llvm::Value *Real = Builder.CreateExtractValue(Agg: CI, Idxs: 0);
6727 llvm::Value *Imag = Builder.CreateExtractValue(Agg: CI, Idxs: 1);
6728 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
6729 }
6730 case TEK_Aggregate:
6731 break;
6732 case TEK_Scalar: {
6733 // If the argument doesn't match, perform a bitcast to coerce it.
6734 // This can happen due to trivial type mismatches.
6735 llvm::Value *V = CI;
6736 if (V->getType() != RetIRTy)
6737 V = Builder.CreateBitCast(V, DestTy: RetIRTy);
6738 return RValue::get(V);
6739 }
6740 }
6741 }
6742
6743 // If coercing a fixed vector from a scalable vector for ABI
6744 // compatibility, and the types match, use the llvm.vector.extract
6745 // intrinsic to perform the conversion.
6746 if (auto *FixedDstTy = dyn_cast<llvm::FixedVectorType>(Val: RetIRTy)) {
6747 llvm::Value *V = CI;
6748 if (auto *ScalableSrcTy =
6749 dyn_cast<llvm::ScalableVectorType>(Val: V->getType())) {
6750 if (FixedDstTy->getElementType() ==
6751 ScalableSrcTy->getElementType()) {
6752 V = Builder.CreateExtractVector(DstType: FixedDstTy, SrcVec: V, Idx: uint64_t(0),
6753 Name: "cast.fixed");
6754 return RValue::get(V);
6755 }
6756 }
6757 }
6758
6759 Address DestPtr = ReturnValue.getValue();
6760 bool DestIsVolatile = ReturnValue.isVolatile();
6761 uint64_t DestSize =
6762 getContext().getTypeInfoDataSizeInChars(T: RetTy).Width.getQuantity();
6763
6764 if (!DestPtr.isValid()) {
6765 DestPtr = CreateMemTempWithoutCast(T: RetTy, Name: "coerce");
6766 DestIsVolatile = false;
6767 DestSize = getContext().getTypeSizeInChars(T: RetTy).getQuantity();
6768 }
6769
6770 // An empty record can overlap other data (if declared with
6771 // no_unique_address); omit the store for such types - as there is no
6772 // actual data to store.
6773 if (!isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true)) {
6774 // If the value is offset in memory, apply the offset now.
6775 Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
6776 CreateCoercedStore(
6777 Src: CI, SrcFETy: RetTy, Dst: StorePtr,
6778 DstSize: llvm::TypeSize::getFixed(ExactSize: DestSize - RetAI.getDirectOffset()),
6779 DstIsVolatile: DestIsVolatile);
6780 }
6781
6782 return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation());
6783 }
6784
6785 case ABIArgInfo::TargetSpecific: {
6786 Address DestPtr = ReturnValue.getValue();
6787 Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
6788 bool DestIsVolatile = ReturnValue.isVolatile();
6789 if (!DestPtr.isValid()) {
6790 DestPtr = CreateMemTempWithoutCast(T: RetTy, Name: "target_coerce");
6791 DestIsVolatile = false;
6792 }
6793 CGM.getABIInfo().createCoercedStore(Val: CI, DstAddr: StorePtr, AI: RetAI, DestIsVolatile,
6794 CGF&: *this);
6795 return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation());
6796 }
6797
6798 case ABIArgInfo::Expand:
6799 case ABIArgInfo::IndirectAliased:
6800 llvm_unreachable("Invalid ABI kind for return argument");
6801 }
6802
6803 llvm_unreachable("Unhandled ABIArgInfo::Kind");
6804 }();
6805 }
6806
6807 // Emit the assume_aligned check on the return value.
6808 if (Ret.isScalar() && TargetDecl) {
6809 AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6810 AllocAlignAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6811 }
6812
6813 // Explicitly call CallLifetimeEnd::Emit just to re-use the code even though
6814 // we can't use the full cleanup mechanism.
6815 for (CallLifetimeEnd &LifetimeEnd : CallLifetimeEndAfterCall)
6816 LifetimeEnd.Emit(CGF&: *this, /*Flags=*/flags: {});
6817
6818 if (!ReturnValue.isExternallyDestructed() &&
6819 RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct)
6820 pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Ret.getAggregateAddress(),
6821 type: RetTy);
6822
6823 // Generate function declaration DISuprogram in order to be used
6824 // in debug info about call sites.
6825 if (CGDebugInfo *DI = getDebugInfo()) {
6826 // Ensure call site info would actually be emitted before collecting
6827 // further callee info.
6828 if (CalleeDecl && !CalleeDecl->hasAttr<NoDebugAttr>() &&
6829 DI->getCallSiteRelatedAttrs() != llvm::DINode::FlagZero) {
6830 CodeGenFunction CalleeCGF(CGM);
6831 const GlobalDecl &CalleeGlobalDecl =
6832 Callee.getAbstractInfo().getCalleeDecl();
6833 CalleeCGF.CurGD = CalleeGlobalDecl;
6834 FunctionArgList Args;
6835 QualType ResTy = CalleeCGF.BuildFunctionArgList(GD: CalleeGlobalDecl, Args);
6836 DI->EmitFuncDeclForCallSite(
6837 CallOrInvoke: CI, CalleeType: DI->getFunctionType(FD: CalleeDecl, RetTy: ResTy, Args), CalleeGlobalDecl);
6838 }
6839 // Generate call site target information.
6840 DI->addCallTargetIfVirtual(FD: CalleeDecl, CI);
6841 }
6842
6843 return Ret;
6844}
6845
6846CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const {
6847 if (isVirtual()) {
6848 const CallExpr *CE = getVirtualCallExpr();
6849 return CGF.CGM.getCXXABI().getVirtualFunctionPointer(
6850 CGF, GD: getVirtualMethodDecl(), This: getThisAddress(), Ty: getVirtualFunctionType(),
6851 Loc: CE ? CE->getBeginLoc() : SourceLocation());
6852 }
6853
6854 return *this;
6855}
6856
6857/* VarArg handling */
6858
6859RValue CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
6860 AggValueSlot Slot) {
6861 VAListAddr = VE->isMicrosoftABI()
6862 ? EmitMSVAListRef(E: VE->getSubExpr())
6863 : (VE->isZOSABI() ? EmitZOSVAListRef(E: VE->getSubExpr())
6864 : EmitVAListRef(E: VE->getSubExpr()));
6865 QualType Ty = VE->getType();
6866 if (Ty->isVariablyModifiedType())
6867 EmitVariablyModifiedType(Ty);
6868 if (VE->isMicrosoftABI())
6869 return CGM.getABIInfo().EmitMSVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6870 if (VE->isZOSABI())
6871 return CGM.getABIInfo().EmitZOSVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6872 return CGM.getABIInfo().EmitVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6873}
6874
6875DisableDebugLocationUpdates::DisableDebugLocationUpdates(CodeGenFunction &CGF)
6876 : CGF(CGF) {
6877 CGF.disableDebugInfo();
6878}
6879
6880DisableDebugLocationUpdates::~DisableDebugLocationUpdates() {
6881 CGF.enableDebugInfo();
6882}
6883