1//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// These classes wrap the information about a call or function
10// definition used to handle ABI compliancy.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CGCall.h"
15#include "ABIInfo.h"
16#include "ABIInfoImpl.h"
17#include "CGBlocks.h"
18#include "CGCXXABI.h"
19#include "CGCleanup.h"
20#include "CGDebugInfo.h"
21#include "CGRecordLayout.h"
22#include "CodeGenFunction.h"
23#include "CodeGenModule.h"
24#include "CodeGenPGO.h"
25#include "TargetInfo.h"
26#include "clang/AST/Attr.h"
27#include "clang/AST/Decl.h"
28#include "clang/AST/DeclCXX.h"
29#include "clang/AST/DeclObjC.h"
30#include "clang/Basic/CodeGenOptions.h"
31#include "clang/Basic/TargetInfo.h"
32#include "clang/CodeGen/CGFunctionInfo.h"
33#include "clang/CodeGen/SwiftCallingConv.h"
34#include "llvm/ADT/StringExtras.h"
35#include "llvm/Analysis/ValueTracking.h"
36#include "llvm/IR/Assumptions.h"
37#include "llvm/IR/AttributeMask.h"
38#include "llvm/IR/Attributes.h"
39#include "llvm/IR/CallingConv.h"
40#include "llvm/IR/DataLayout.h"
41#include "llvm/IR/InlineAsm.h"
42#include "llvm/IR/IntrinsicInst.h"
43#include "llvm/IR/Intrinsics.h"
44#include "llvm/IR/Type.h"
45#include "llvm/Transforms/Utils/Local.h"
46#include <optional>
47using namespace clang;
48using namespace CodeGen;
49
50/***/
51
52unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
53 switch (CC) {
54 default:
55 return llvm::CallingConv::C;
56 case CC_X86StdCall:
57 return llvm::CallingConv::X86_StdCall;
58 case CC_X86FastCall:
59 return llvm::CallingConv::X86_FastCall;
60 case CC_X86RegCall:
61 return llvm::CallingConv::X86_RegCall;
62 case CC_X86ThisCall:
63 return llvm::CallingConv::X86_ThisCall;
64 case CC_Win64:
65 return llvm::CallingConv::Win64;
66 case CC_X86_64SysV:
67 return llvm::CallingConv::X86_64_SysV;
68 case CC_AAPCS:
69 return llvm::CallingConv::ARM_AAPCS;
70 case CC_AAPCS_VFP:
71 return llvm::CallingConv::ARM_AAPCS_VFP;
72 case CC_IntelOclBicc:
73 return llvm::CallingConv::Intel_OCL_BI;
74 // TODO: Add support for __pascal to LLVM.
75 case CC_X86Pascal:
76 return llvm::CallingConv::C;
77 // TODO: Add support for __vectorcall to LLVM.
78 case CC_X86VectorCall:
79 return llvm::CallingConv::X86_VectorCall;
80 case CC_AArch64VectorCall:
81 return llvm::CallingConv::AArch64_VectorCall;
82 case CC_AArch64SVEPCS:
83 return llvm::CallingConv::AArch64_SVE_VectorCall;
84 case CC_SpirFunction:
85 return llvm::CallingConv::SPIR_FUNC;
86 case CC_DeviceKernel:
87 return CGM.getTargetCodeGenInfo().getDeviceKernelCallingConv();
88 case CC_PreserveMost:
89 return llvm::CallingConv::PreserveMost;
90 case CC_PreserveAll:
91 return llvm::CallingConv::PreserveAll;
92 case CC_Swift:
93 return llvm::CallingConv::Swift;
94 case CC_SwiftAsync:
95 return llvm::CallingConv::SwiftTail;
96 case CC_M68kRTD:
97 return llvm::CallingConv::M68k_RTD;
98 case CC_PreserveNone:
99 return llvm::CallingConv::PreserveNone;
100 // clang-format off
101 case CC_RISCVVectorCall: return llvm::CallingConv::RISCV_VectorCall;
102 // clang-format on
103#define CC_VLS_CASE(ABI_VLEN) \
104 case CC_RISCVVLSCall_##ABI_VLEN: \
105 return llvm::CallingConv::RISCV_VLSCall_##ABI_VLEN;
106 CC_VLS_CASE(32)
107 CC_VLS_CASE(64)
108 CC_VLS_CASE(128)
109 CC_VLS_CASE(256)
110 CC_VLS_CASE(512)
111 CC_VLS_CASE(1024)
112 CC_VLS_CASE(2048)
113 CC_VLS_CASE(4096)
114 CC_VLS_CASE(8192)
115 CC_VLS_CASE(16384)
116 CC_VLS_CASE(32768)
117 CC_VLS_CASE(65536)
118#undef CC_VLS_CASE
119 }
120}
121
122/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
123/// qualification. Either or both of RD and MD may be null. A null RD indicates
124/// that there is no meaningful 'this' type, and a null MD can occur when
125/// calling a method pointer.
126CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
127 const CXXMethodDecl *MD) {
128 QualType RecTy;
129 if (RD)
130 RecTy = Context.getTagDeclType(Decl: RD)->getCanonicalTypeInternal();
131 else
132 RecTy = Context.VoidTy;
133
134 if (MD)
135 RecTy = Context.getAddrSpaceQualType(
136 T: RecTy, AddressSpace: MD->getMethodQualifiers().getAddressSpace());
137 return Context.getPointerType(T: CanQualType::CreateUnsafe(Other: RecTy));
138}
139
140/// Returns the canonical formal type of the given C++ method.
141static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
142 return MD->getType()
143 ->getCanonicalTypeUnqualified()
144 .getAs<FunctionProtoType>();
145}
146
147/// Returns the "extra-canonicalized" return type, which discards
148/// qualifiers on the return type. Codegen doesn't care about them,
149/// and it makes ABI code a little easier to be able to assume that
150/// all parameter and return types are top-level unqualified.
151static CanQualType GetReturnType(QualType RetTy) {
152 return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
153}
154
155/// Arrange the argument and result information for a value of the given
156/// unprototyped freestanding function type.
157const CGFunctionInfo &
158CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
159 // When translating an unprototyped function type, always use a
160 // variadic type.
161 return arrangeLLVMFunctionInfo(returnType: FTNP->getReturnType().getUnqualifiedType(),
162 opts: FnInfoOpts::None, argTypes: {}, info: FTNP->getExtInfo(), paramInfos: {},
163 args: RequiredArgs(0));
164}
165
166static void addExtParameterInfosForCall(
167 llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
168 const FunctionProtoType *proto, unsigned prefixArgs, unsigned totalArgs) {
169 assert(proto->hasExtParameterInfos());
170 assert(paramInfos.size() <= prefixArgs);
171 assert(proto->getNumParams() + prefixArgs <= totalArgs);
172
173 paramInfos.reserve(N: totalArgs);
174
175 // Add default infos for any prefix args that don't already have infos.
176 paramInfos.resize(N: prefixArgs);
177
178 // Add infos for the prototype.
179 for (const auto &ParamInfo : proto->getExtParameterInfos()) {
180 paramInfos.push_back(Elt: ParamInfo);
181 // pass_object_size params have no parameter info.
182 if (ParamInfo.hasPassObjectSize())
183 paramInfos.emplace_back();
184 }
185
186 assert(paramInfos.size() <= totalArgs &&
187 "Did we forget to insert pass_object_size args?");
188 // Add default infos for the variadic and/or suffix arguments.
189 paramInfos.resize(N: totalArgs);
190}
191
192/// Adds the formal parameters in FPT to the given prefix. If any parameter in
193/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
194static void appendParameterTypes(
195 const CodeGenTypes &CGT, SmallVectorImpl<CanQualType> &prefix,
196 SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
197 CanQual<FunctionProtoType> FPT) {
198 // Fast path: don't touch param info if we don't need to.
199 if (!FPT->hasExtParameterInfos()) {
200 assert(paramInfos.empty() &&
201 "We have paramInfos, but the prototype doesn't?");
202 prefix.append(in_start: FPT->param_type_begin(), in_end: FPT->param_type_end());
203 return;
204 }
205
206 unsigned PrefixSize = prefix.size();
207 // In the vast majority of cases, we'll have precisely FPT->getNumParams()
208 // parameters; the only thing that can change this is the presence of
209 // pass_object_size. So, we preallocate for the common case.
210 prefix.reserve(N: prefix.size() + FPT->getNumParams());
211
212 auto ExtInfos = FPT->getExtParameterInfos();
213 assert(ExtInfos.size() == FPT->getNumParams());
214 for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
215 prefix.push_back(Elt: FPT->getParamType(i: I));
216 if (ExtInfos[I].hasPassObjectSize())
217 prefix.push_back(Elt: CGT.getContext().getSizeType());
218 }
219
220 addExtParameterInfosForCall(paramInfos, proto: FPT.getTypePtr(), prefixArgs: PrefixSize,
221 totalArgs: prefix.size());
222}
223
224using ExtParameterInfoList =
225 SmallVector<FunctionProtoType::ExtParameterInfo, 16>;
226
227/// Arrange the LLVM function layout for a value of the given function
228/// type, on top of any implicit parameters already stored.
229static const CGFunctionInfo &
230arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
231 SmallVectorImpl<CanQualType> &prefix,
232 CanQual<FunctionProtoType> FTP) {
233 ExtParameterInfoList paramInfos;
234 RequiredArgs Required = RequiredArgs::forPrototypePlus(prototype: FTP, additional: prefix.size());
235 appendParameterTypes(CGT, prefix, paramInfos, FPT: FTP);
236 CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
237
238 FnInfoOpts opts =
239 instanceMethod ? FnInfoOpts::IsInstanceMethod : FnInfoOpts::None;
240 return CGT.arrangeLLVMFunctionInfo(returnType: resultType, opts, argTypes: prefix,
241 info: FTP->getExtInfo(), paramInfos, args: Required);
242}
243
244using CanQualTypeList = SmallVector<CanQualType, 16>;
245
246/// Arrange the argument and result information for a value of the
247/// given freestanding function type.
248const CGFunctionInfo &
249CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
250 CanQualTypeList argTypes;
251 return ::arrangeLLVMFunctionInfo(CGT&: *this, /*instanceMethod=*/false, prefix&: argTypes,
252 FTP);
253}
254
255static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
256 bool IsTargetDefaultMSABI) {
257 // Set the appropriate calling convention for the Function.
258 if (D->hasAttr<StdCallAttr>())
259 return CC_X86StdCall;
260
261 if (D->hasAttr<FastCallAttr>())
262 return CC_X86FastCall;
263
264 if (D->hasAttr<RegCallAttr>())
265 return CC_X86RegCall;
266
267 if (D->hasAttr<ThisCallAttr>())
268 return CC_X86ThisCall;
269
270 if (D->hasAttr<VectorCallAttr>())
271 return CC_X86VectorCall;
272
273 if (D->hasAttr<PascalAttr>())
274 return CC_X86Pascal;
275
276 if (PcsAttr *PCS = D->getAttr<PcsAttr>())
277 return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
278
279 if (D->hasAttr<AArch64VectorPcsAttr>())
280 return CC_AArch64VectorCall;
281
282 if (D->hasAttr<AArch64SVEPcsAttr>())
283 return CC_AArch64SVEPCS;
284
285 if (D->hasAttr<DeviceKernelAttr>())
286 return CC_DeviceKernel;
287
288 if (D->hasAttr<IntelOclBiccAttr>())
289 return CC_IntelOclBicc;
290
291 if (D->hasAttr<MSABIAttr>())
292 return IsTargetDefaultMSABI ? CC_C : CC_Win64;
293
294 if (D->hasAttr<SysVABIAttr>())
295 return IsTargetDefaultMSABI ? CC_X86_64SysV : CC_C;
296
297 if (D->hasAttr<PreserveMostAttr>())
298 return CC_PreserveMost;
299
300 if (D->hasAttr<PreserveAllAttr>())
301 return CC_PreserveAll;
302
303 if (D->hasAttr<M68kRTDAttr>())
304 return CC_M68kRTD;
305
306 if (D->hasAttr<PreserveNoneAttr>())
307 return CC_PreserveNone;
308
309 if (D->hasAttr<RISCVVectorCCAttr>())
310 return CC_RISCVVectorCall;
311
312 if (RISCVVLSCCAttr *PCS = D->getAttr<RISCVVLSCCAttr>()) {
313 switch (PCS->getVectorWidth()) {
314 default:
315 llvm_unreachable("Invalid RISC-V VLS ABI VLEN");
316#define CC_VLS_CASE(ABI_VLEN) \
317 case ABI_VLEN: \
318 return CC_RISCVVLSCall_##ABI_VLEN;
319 CC_VLS_CASE(32)
320 CC_VLS_CASE(64)
321 CC_VLS_CASE(128)
322 CC_VLS_CASE(256)
323 CC_VLS_CASE(512)
324 CC_VLS_CASE(1024)
325 CC_VLS_CASE(2048)
326 CC_VLS_CASE(4096)
327 CC_VLS_CASE(8192)
328 CC_VLS_CASE(16384)
329 CC_VLS_CASE(32768)
330 CC_VLS_CASE(65536)
331#undef CC_VLS_CASE
332 }
333 }
334
335 return CC_C;
336}
337
338/// Arrange the argument and result information for a call to an
339/// unknown C++ non-static member function of the given abstract type.
340/// (A null RD means we don't have any meaningful "this" argument type,
341/// so fall back to a generic pointer type).
342/// The member function must be an ordinary function, i.e. not a
343/// constructor or destructor.
344const CGFunctionInfo &
345CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
346 const FunctionProtoType *FTP,
347 const CXXMethodDecl *MD) {
348 CanQualTypeList argTypes;
349
350 // Add the 'this' pointer.
351 argTypes.push_back(Elt: DeriveThisType(RD, MD));
352
353 return ::arrangeLLVMFunctionInfo(
354 CGT&: *this, /*instanceMethod=*/true, prefix&: argTypes,
355 FTP: FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
356}
357
358/// Set calling convention for CUDA/HIP kernel.
359static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
360 const FunctionDecl *FD) {
361 if (FD->hasAttr<CUDAGlobalAttr>()) {
362 const FunctionType *FT = FTy->getAs<FunctionType>();
363 CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
364 FTy = FT->getCanonicalTypeUnqualified();
365 }
366}
367
368/// Arrange the argument and result information for a declaration or
369/// definition of the given C++ non-static member function. The
370/// member function must be an ordinary function, i.e. not a
371/// constructor or destructor.
372const CGFunctionInfo &
373CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
374 assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
375 assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
376
377 CanQualType FT = GetFormalType(MD).getAs<Type>();
378 setCUDAKernelCallingConvention(FTy&: FT, CGM, FD: MD);
379 auto prototype = FT.getAs<FunctionProtoType>();
380
381 if (MD->isImplicitObjectMemberFunction()) {
382 // The abstract case is perfectly fine.
383 const CXXRecordDecl *ThisType =
384 getCXXABI().getThisArgumentTypeForMethod(GD: MD);
385 return arrangeCXXMethodType(RD: ThisType, FTP: prototype.getTypePtr(), MD);
386 }
387
388 return arrangeFreeFunctionType(FTP: prototype);
389}
390
391bool CodeGenTypes::inheritingCtorHasParams(
392 const InheritedConstructor &Inherited, CXXCtorType Type) {
393 // Parameters are unnecessary if we're constructing a base class subobject
394 // and the inherited constructor lives in a virtual base.
395 return Type == Ctor_Complete ||
396 !Inherited.getShadowDecl()->constructsVirtualBase() ||
397 !Target.getCXXABI().hasConstructorVariants();
398}
399
400const CGFunctionInfo &
401CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
402 auto *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
403
404 CanQualTypeList argTypes;
405 ExtParameterInfoList paramInfos;
406
407 const CXXRecordDecl *ThisType = getCXXABI().getThisArgumentTypeForMethod(GD);
408 argTypes.push_back(Elt: DeriveThisType(RD: ThisType, MD));
409
410 bool PassParams = true;
411
412 if (auto *CD = dyn_cast<CXXConstructorDecl>(Val: MD)) {
413 // A base class inheriting constructor doesn't get forwarded arguments
414 // needed to construct a virtual base (or base class thereof).
415 if (auto Inherited = CD->getInheritedConstructor())
416 PassParams = inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
417 }
418
419 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
420
421 // Add the formal parameters.
422 if (PassParams)
423 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: FTP);
424
425 CGCXXABI::AddedStructorArgCounts AddedArgs =
426 getCXXABI().buildStructorSignature(GD, ArgTys&: argTypes);
427 if (!paramInfos.empty()) {
428 // Note: prefix implies after the first param.
429 if (AddedArgs.Prefix)
430 paramInfos.insert(I: paramInfos.begin() + 1, NumToInsert: AddedArgs.Prefix,
431 Elt: FunctionProtoType::ExtParameterInfo{});
432 if (AddedArgs.Suffix)
433 paramInfos.append(NumInputs: AddedArgs.Suffix,
434 Elt: FunctionProtoType::ExtParameterInfo{});
435 }
436
437 RequiredArgs required =
438 (PassParams && MD->isVariadic() ? RequiredArgs(argTypes.size())
439 : RequiredArgs::All);
440
441 FunctionType::ExtInfo extInfo = FTP->getExtInfo();
442 CanQualType resultType = getCXXABI().HasThisReturn(GD) ? argTypes.front()
443 : getCXXABI().hasMostDerivedReturn(GD)
444 ? CGM.getContext().VoidPtrTy
445 : Context.VoidTy;
446 return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::IsInstanceMethod,
447 argTypes, info: extInfo, paramInfos, args: required);
448}
449
450static CanQualTypeList getArgTypesForCall(ASTContext &ctx,
451 const CallArgList &args) {
452 CanQualTypeList argTypes;
453 for (auto &arg : args)
454 argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg.Ty));
455 return argTypes;
456}
457
458static CanQualTypeList getArgTypesForDeclaration(ASTContext &ctx,
459 const FunctionArgList &args) {
460 CanQualTypeList argTypes;
461 for (auto &arg : args)
462 argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg->getType()));
463 return argTypes;
464}
465
466static ExtParameterInfoList
467getExtParameterInfosForCall(const FunctionProtoType *proto, unsigned prefixArgs,
468 unsigned totalArgs) {
469 ExtParameterInfoList result;
470 if (proto->hasExtParameterInfos()) {
471 addExtParameterInfosForCall(paramInfos&: result, proto, prefixArgs, totalArgs);
472 }
473 return result;
474}
475
476/// Arrange a call to a C++ method, passing the given arguments.
477///
478/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
479/// parameter.
480/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
481/// args.
482/// PassProtoArgs indicates whether `args` has args for the parameters in the
483/// given CXXConstructorDecl.
484const CGFunctionInfo &CodeGenTypes::arrangeCXXConstructorCall(
485 const CallArgList &args, const CXXConstructorDecl *D, CXXCtorType CtorKind,
486 unsigned ExtraPrefixArgs, unsigned ExtraSuffixArgs, bool PassProtoArgs) {
487 CanQualTypeList ArgTypes;
488 for (const auto &Arg : args)
489 ArgTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
490
491 // +1 for implicit this, which should always be args[0].
492 unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
493
494 CanQual<FunctionProtoType> FPT = GetFormalType(MD: D);
495 RequiredArgs Required = PassProtoArgs
496 ? RequiredArgs::forPrototypePlus(
497 prototype: FPT, additional: TotalPrefixArgs + ExtraSuffixArgs)
498 : RequiredArgs::All;
499
500 GlobalDecl GD(D, CtorKind);
501 CanQualType ResultType = getCXXABI().HasThisReturn(GD) ? ArgTypes.front()
502 : getCXXABI().hasMostDerivedReturn(GD)
503 ? CGM.getContext().VoidPtrTy
504 : Context.VoidTy;
505
506 FunctionType::ExtInfo Info = FPT->getExtInfo();
507 ExtParameterInfoList ParamInfos;
508 // If the prototype args are elided, we should only have ABI-specific args,
509 // which never have param info.
510 if (PassProtoArgs && FPT->hasExtParameterInfos()) {
511 // ABI-specific suffix arguments are treated the same as variadic arguments.
512 addExtParameterInfosForCall(paramInfos&: ParamInfos, proto: FPT.getTypePtr(), prefixArgs: TotalPrefixArgs,
513 totalArgs: ArgTypes.size());
514 }
515
516 return arrangeLLVMFunctionInfo(returnType: ResultType, opts: FnInfoOpts::IsInstanceMethod,
517 argTypes: ArgTypes, info: Info, paramInfos: ParamInfos, args: Required);
518}
519
520/// Arrange the argument and result information for the declaration or
521/// definition of the given function.
522const CGFunctionInfo &
523CodeGenTypes::arrangeFunctionDeclaration(const GlobalDecl GD) {
524 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
525 if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD))
526 if (MD->isImplicitObjectMemberFunction())
527 return arrangeCXXMethodDeclaration(MD);
528
529 CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
530
531 assert(isa<FunctionType>(FTy));
532 setCUDAKernelCallingConvention(FTy, CGM, FD);
533
534 if (DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()) &&
535 GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
536 const FunctionType *FT = FTy->getAs<FunctionType>();
537 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
538 FTy = FT->getCanonicalTypeUnqualified();
539 }
540
541 // When declaring a function without a prototype, always use a
542 // non-variadic type.
543 if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
544 return arrangeLLVMFunctionInfo(returnType: noProto->getReturnType(), opts: FnInfoOpts::None,
545 argTypes: {}, info: noProto->getExtInfo(), paramInfos: {},
546 args: RequiredArgs::All);
547 }
548
549 return arrangeFreeFunctionType(FTP: FTy.castAs<FunctionProtoType>());
550}
551
552/// Arrange the argument and result information for the declaration or
553/// definition of an Objective-C method.
554const CGFunctionInfo &
555CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
556 // It happens that this is the same as a call with no optional
557 // arguments, except also using the formal 'self' type.
558 return arrangeObjCMessageSendSignature(MD, receiverType: MD->getSelfDecl()->getType());
559}
560
561/// Arrange the argument and result information for the function type
562/// through which to perform a send to the given Objective-C method,
563/// using the given receiver type. The receiver type is not always
564/// the 'self' type of the method or even an Objective-C pointer type.
565/// This is *not* the right method for actually performing such a
566/// message send, due to the possibility of optional arguments.
567const CGFunctionInfo &
568CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
569 QualType receiverType) {
570 CanQualTypeList argTys;
571 ExtParameterInfoList extParamInfos(MD->isDirectMethod() ? 1 : 2);
572 argTys.push_back(Elt: Context.getCanonicalParamType(T: receiverType));
573 if (!MD->isDirectMethod())
574 argTys.push_back(Elt: Context.getCanonicalParamType(T: Context.getObjCSelType()));
575 for (const auto *I : MD->parameters()) {
576 argTys.push_back(Elt: Context.getCanonicalParamType(T: I->getType()));
577 auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
578 NoEscape: I->hasAttr<NoEscapeAttr>());
579 extParamInfos.push_back(Elt: extParamInfo);
580 }
581
582 FunctionType::ExtInfo einfo;
583 bool IsTargetDefaultMSABI =
584 getContext().getTargetInfo().getTriple().isOSWindows() ||
585 getContext().getTargetInfo().getTriple().isUEFI();
586 einfo = einfo.withCallingConv(
587 cc: getCallingConventionForDecl(D: MD, IsTargetDefaultMSABI));
588
589 if (getContext().getLangOpts().ObjCAutoRefCount &&
590 MD->hasAttr<NSReturnsRetainedAttr>())
591 einfo = einfo.withProducesResult(producesResult: true);
592
593 RequiredArgs required =
594 (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);
595
596 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: MD->getReturnType()),
597 opts: FnInfoOpts::None, argTypes: argTys, info: einfo, paramInfos: extParamInfos,
598 args: required);
599}
600
601const CGFunctionInfo &
602CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
603 const CallArgList &args) {
604 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
605 FunctionType::ExtInfo einfo;
606
607 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: returnType), opts: FnInfoOpts::None,
608 argTypes, info: einfo, paramInfos: {}, args: RequiredArgs::All);
609}
610
611const CGFunctionInfo &CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
612 // FIXME: Do we need to handle ObjCMethodDecl?
613 if (isa<CXXConstructorDecl>(Val: GD.getDecl()) ||
614 isa<CXXDestructorDecl>(Val: GD.getDecl()))
615 return arrangeCXXStructorDeclaration(GD);
616
617 return arrangeFunctionDeclaration(GD);
618}
619
620/// Arrange a thunk that takes 'this' as the first parameter followed by
621/// varargs. Return a void pointer, regardless of the actual return type.
622/// The body of the thunk will end in a musttail call to a function of the
623/// correct type, and the caller will bitcast the function to the correct
624/// prototype.
625const CGFunctionInfo &
626CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
627 assert(MD->isVirtual() && "only methods have thunks");
628 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
629 CanQualType ArgTys[] = {DeriveThisType(RD: MD->getParent(), MD)};
630 return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::None, argTypes: ArgTys,
631 info: FTP->getExtInfo(), paramInfos: {}, args: RequiredArgs(1));
632}
633
634const CGFunctionInfo &
635CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
636 CXXCtorType CT) {
637 assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
638
639 CanQual<FunctionProtoType> FTP = GetFormalType(MD: CD);
640 SmallVector<CanQualType, 2> ArgTys;
641 const CXXRecordDecl *RD = CD->getParent();
642 ArgTys.push_back(Elt: DeriveThisType(RD, MD: CD));
643 if (CT == Ctor_CopyingClosure)
644 ArgTys.push_back(Elt: *FTP->param_type_begin());
645 if (RD->getNumVBases() > 0)
646 ArgTys.push_back(Elt: Context.IntTy);
647 CallingConv CC = Context.getDefaultCallingConvention(
648 /*IsVariadic=*/false, /*IsCXXMethod=*/true);
649 return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::IsInstanceMethod,
650 argTypes: ArgTys, info: FunctionType::ExtInfo(CC), paramInfos: {},
651 args: RequiredArgs::All);
652}
653
654/// Arrange a call as unto a free function, except possibly with an
655/// additional number of formal parameters considered required.
656static const CGFunctionInfo &
657arrangeFreeFunctionLikeCall(CodeGenTypes &CGT, CodeGenModule &CGM,
658 const CallArgList &args, const FunctionType *fnType,
659 unsigned numExtraRequiredArgs, bool chainCall) {
660 assert(args.size() >= numExtraRequiredArgs);
661
662 ExtParameterInfoList paramInfos;
663
664 // In most cases, there are no optional arguments.
665 RequiredArgs required = RequiredArgs::All;
666
667 // If we have a variadic prototype, the required arguments are the
668 // extra prefix plus the arguments in the prototype.
669 if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(Val: fnType)) {
670 if (proto->isVariadic())
671 required = RequiredArgs::forPrototypePlus(prototype: proto, additional: numExtraRequiredArgs);
672
673 if (proto->hasExtParameterInfos())
674 addExtParameterInfosForCall(paramInfos, proto, prefixArgs: numExtraRequiredArgs,
675 totalArgs: args.size());
676
677 // If we don't have a prototype at all, but we're supposed to
678 // explicitly use the variadic convention for unprototyped calls,
679 // treat all of the arguments as required but preserve the nominal
680 // possibility of variadics.
681 } else if (CGM.getTargetCodeGenInfo().isNoProtoCallVariadic(
682 args, fnType: cast<FunctionNoProtoType>(Val: fnType))) {
683 required = RequiredArgs(args.size());
684 }
685
686 CanQualTypeList argTypes;
687 for (const auto &arg : args)
688 argTypes.push_back(Elt: CGT.getContext().getCanonicalParamType(T: arg.Ty));
689 FnInfoOpts opts = chainCall ? FnInfoOpts::IsChainCall : FnInfoOpts::None;
690 return CGT.arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: fnType->getReturnType()),
691 opts, argTypes, info: fnType->getExtInfo(),
692 paramInfos, args: required);
693}
694
695/// Figure out the rules for calling a function with the given formal
696/// type using the given arguments. The arguments are necessary
697/// because the function might be unprototyped, in which case it's
698/// target-dependent in crazy ways.
699const CGFunctionInfo &CodeGenTypes::arrangeFreeFunctionCall(
700 const CallArgList &args, const FunctionType *fnType, bool chainCall) {
701 return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType,
702 numExtraRequiredArgs: chainCall ? 1 : 0, chainCall);
703}
704
705/// A block function is essentially a free function with an
706/// extra implicit argument.
707const CGFunctionInfo &
708CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
709 const FunctionType *fnType) {
710 return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType, numExtraRequiredArgs: 1,
711 /*chainCall=*/false);
712}
713
714const CGFunctionInfo &
715CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
716 const FunctionArgList &params) {
717 ExtParameterInfoList paramInfos =
718 getExtParameterInfosForCall(proto, prefixArgs: 1, totalArgs: params.size());
719 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args: params);
720
721 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: proto->getReturnType()),
722 opts: FnInfoOpts::None, argTypes,
723 info: proto->getExtInfo(), paramInfos,
724 args: RequiredArgs::forPrototypePlus(prototype: proto, additional: 1));
725}
726
727const CGFunctionInfo &
728CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
729 const CallArgList &args) {
730 CanQualTypeList argTypes;
731 for (const auto &Arg : args)
732 argTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
733 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
734 argTypes, info: FunctionType::ExtInfo(),
735 /*paramInfos=*/{}, args: RequiredArgs::All);
736}
737
738const CGFunctionInfo &
739CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
740 const FunctionArgList &args) {
741 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
742
743 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
744 argTypes, info: FunctionType::ExtInfo(), paramInfos: {},
745 args: RequiredArgs::All);
746}
747
748const CGFunctionInfo &CodeGenTypes::arrangeBuiltinFunctionDeclaration(
749 CanQualType resultType, ArrayRef<CanQualType> argTypes) {
750 return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::None, argTypes,
751 info: FunctionType::ExtInfo(), paramInfos: {},
752 args: RequiredArgs::All);
753}
754
755const CGFunctionInfo &
756CodeGenTypes::arrangeSYCLKernelCallerDeclaration(QualType resultType,
757 const FunctionArgList &args) {
758 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
759
760 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
761 argTypes,
762 info: FunctionType::ExtInfo(CC_DeviceKernel),
763 /*paramInfos=*/{}, args: RequiredArgs::All);
764}
765
766/// Arrange a call to a C++ method, passing the given arguments.
767///
768/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
769/// does not count `this`.
770const CGFunctionInfo &CodeGenTypes::arrangeCXXMethodCall(
771 const CallArgList &args, const FunctionProtoType *proto,
772 RequiredArgs required, unsigned numPrefixArgs) {
773 assert(numPrefixArgs + 1 <= args.size() &&
774 "Emitting a call with less args than the required prefix?");
775 // Add one to account for `this`. It's a bit awkward here, but we don't count
776 // `this` in similar places elsewhere.
777 ExtParameterInfoList paramInfos =
778 getExtParameterInfosForCall(proto, prefixArgs: numPrefixArgs + 1, totalArgs: args.size());
779
780 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
781
782 FunctionType::ExtInfo info = proto->getExtInfo();
783 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: proto->getReturnType()),
784 opts: FnInfoOpts::IsInstanceMethod, argTypes, info,
785 paramInfos, args: required);
786}
787
788const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
789 return arrangeLLVMFunctionInfo(returnType: getContext().VoidTy, opts: FnInfoOpts::None, argTypes: {},
790 info: FunctionType::ExtInfo(), paramInfos: {},
791 args: RequiredArgs::All);
792}
793
794const CGFunctionInfo &CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
795 const CallArgList &args) {
796 assert(signature.arg_size() <= args.size());
797 if (signature.arg_size() == args.size())
798 return signature;
799
800 ExtParameterInfoList paramInfos;
801 auto sigParamInfos = signature.getExtParameterInfos();
802 if (!sigParamInfos.empty()) {
803 paramInfos.append(in_start: sigParamInfos.begin(), in_end: sigParamInfos.end());
804 paramInfos.resize(N: args.size());
805 }
806
807 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
808
809 assert(signature.getRequiredArgs().allowsOptionalArgs());
810 FnInfoOpts opts = FnInfoOpts::None;
811 if (signature.isInstanceMethod())
812 opts |= FnInfoOpts::IsInstanceMethod;
813 if (signature.isChainCall())
814 opts |= FnInfoOpts::IsChainCall;
815 if (signature.isDelegateCall())
816 opts |= FnInfoOpts::IsDelegateCall;
817 return arrangeLLVMFunctionInfo(returnType: signature.getReturnType(), opts, argTypes,
818 info: signature.getExtInfo(), paramInfos,
819 args: signature.getRequiredArgs());
820}
821
822namespace clang {
823namespace CodeGen {
824void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
825}
826} // namespace clang
827
828/// Arrange the argument and result information for an abstract value
829/// of a given function type. This is the method which all of the
830/// above functions ultimately defer to.
831const CGFunctionInfo &CodeGenTypes::arrangeLLVMFunctionInfo(
832 CanQualType resultType, FnInfoOpts opts, ArrayRef<CanQualType> argTypes,
833 FunctionType::ExtInfo info,
834 ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
835 RequiredArgs required) {
836 assert(llvm::all_of(argTypes,
837 [](CanQualType T) { return T.isCanonicalAsParam(); }));
838
839 // Lookup or create unique function info.
840 llvm::FoldingSetNodeID ID;
841 bool isInstanceMethod =
842 (opts & FnInfoOpts::IsInstanceMethod) == FnInfoOpts::IsInstanceMethod;
843 bool isChainCall =
844 (opts & FnInfoOpts::IsChainCall) == FnInfoOpts::IsChainCall;
845 bool isDelegateCall =
846 (opts & FnInfoOpts::IsDelegateCall) == FnInfoOpts::IsDelegateCall;
847 CGFunctionInfo::Profile(ID, InstanceMethod: isInstanceMethod, ChainCall: isChainCall, IsDelegateCall: isDelegateCall,
848 info, paramInfos, required, resultType, argTypes);
849
850 void *insertPos = nullptr;
851 CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, InsertPos&: insertPos);
852 if (FI)
853 return *FI;
854
855 unsigned CC = ClangCallConvToLLVMCallConv(CC: info.getCC());
856
857 // Construct the function info. We co-allocate the ArgInfos.
858 FI = CGFunctionInfo::create(llvmCC: CC, instanceMethod: isInstanceMethod, chainCall: isChainCall, delegateCall: isDelegateCall,
859 extInfo: info, paramInfos, resultType, argTypes, required);
860 FunctionInfos.InsertNode(N: FI, InsertPos: insertPos);
861
862 bool inserted = FunctionsBeingProcessed.insert(Ptr: FI).second;
863 (void)inserted;
864 assert(inserted && "Recursively being processed?");
865
866 // Compute ABI information.
867 if (CC == llvm::CallingConv::SPIR_KERNEL) {
868 // Force target independent argument handling for the host visible
869 // kernel functions.
870 computeSPIRKernelABIInfo(CGM, FI&: *FI);
871 } else if (info.getCC() == CC_Swift || info.getCC() == CC_SwiftAsync) {
872 swiftcall::computeABIInfo(CGM, FI&: *FI);
873 } else {
874 CGM.getABIInfo().computeInfo(FI&: *FI);
875 }
876
877 // Loop over all of the computed argument and return value info. If any of
878 // them are direct or extend without a specified coerce type, specify the
879 // default now.
880 ABIArgInfo &retInfo = FI->getReturnInfo();
881 if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
882 retInfo.setCoerceToType(ConvertType(T: FI->getReturnType()));
883
884 for (auto &I : FI->arguments())
885 if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
886 I.info.setCoerceToType(ConvertType(T: I.type));
887
888 bool erased = FunctionsBeingProcessed.erase(Ptr: FI);
889 (void)erased;
890 assert(erased && "Not in set?");
891
892 return *FI;
893}
894
895CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC, bool instanceMethod,
896 bool chainCall, bool delegateCall,
897 const FunctionType::ExtInfo &info,
898 ArrayRef<ExtParameterInfo> paramInfos,
899 CanQualType resultType,
900 ArrayRef<CanQualType> argTypes,
901 RequiredArgs required) {
902 assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
903 assert(!required.allowsOptionalArgs() ||
904 required.getNumRequiredArgs() <= argTypes.size());
905
906 void *buffer = operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(
907 Counts: argTypes.size() + 1, Counts: paramInfos.size()));
908
909 CGFunctionInfo *FI = new (buffer) CGFunctionInfo();
910 FI->CallingConvention = llvmCC;
911 FI->EffectiveCallingConvention = llvmCC;
912 FI->ASTCallingConvention = info.getCC();
913 FI->InstanceMethod = instanceMethod;
914 FI->ChainCall = chainCall;
915 FI->DelegateCall = delegateCall;
916 FI->CmseNSCall = info.getCmseNSCall();
917 FI->NoReturn = info.getNoReturn();
918 FI->ReturnsRetained = info.getProducesResult();
919 FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
920 FI->NoCfCheck = info.getNoCfCheck();
921 FI->Required = required;
922 FI->HasRegParm = info.getHasRegParm();
923 FI->RegParm = info.getRegParm();
924 FI->ArgStruct = nullptr;
925 FI->ArgStructAlign = 0;
926 FI->NumArgs = argTypes.size();
927 FI->HasExtParameterInfos = !paramInfos.empty();
928 FI->getArgsBuffer()[0].type = resultType;
929 FI->MaxVectorWidth = 0;
930 for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
931 FI->getArgsBuffer()[i + 1].type = argTypes[i];
932 for (unsigned i = 0, e = paramInfos.size(); i != e; ++i)
933 FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
934 return FI;
935}
936
937/***/
938
939namespace {
940// ABIArgInfo::Expand implementation.
941
942// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
943struct TypeExpansion {
944 enum TypeExpansionKind {
945 // Elements of constant arrays are expanded recursively.
946 TEK_ConstantArray,
947 // Record fields are expanded recursively (but if record is a union, only
948 // the field with the largest size is expanded).
949 TEK_Record,
950 // For complex types, real and imaginary parts are expanded recursively.
951 TEK_Complex,
952 // All other types are not expandable.
953 TEK_None
954 };
955
956 const TypeExpansionKind Kind;
957
958 TypeExpansion(TypeExpansionKind K) : Kind(K) {}
959 virtual ~TypeExpansion() {}
960};
961
962struct ConstantArrayExpansion : TypeExpansion {
963 QualType EltTy;
964 uint64_t NumElts;
965
966 ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
967 : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
968 static bool classof(const TypeExpansion *TE) {
969 return TE->Kind == TEK_ConstantArray;
970 }
971};
972
973struct RecordExpansion : TypeExpansion {
974 SmallVector<const CXXBaseSpecifier *, 1> Bases;
975
976 SmallVector<const FieldDecl *, 1> Fields;
977
978 RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
979 SmallVector<const FieldDecl *, 1> &&Fields)
980 : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
981 Fields(std::move(Fields)) {}
982 static bool classof(const TypeExpansion *TE) {
983 return TE->Kind == TEK_Record;
984 }
985};
986
987struct ComplexExpansion : TypeExpansion {
988 QualType EltTy;
989
990 ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
991 static bool classof(const TypeExpansion *TE) {
992 return TE->Kind == TEK_Complex;
993 }
994};
995
996struct NoExpansion : TypeExpansion {
997 NoExpansion() : TypeExpansion(TEK_None) {}
998 static bool classof(const TypeExpansion *TE) { return TE->Kind == TEK_None; }
999};
1000} // namespace
1001
1002static std::unique_ptr<TypeExpansion>
1003getTypeExpansion(QualType Ty, const ASTContext &Context) {
1004 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty)) {
1005 return std::make_unique<ConstantArrayExpansion>(args: AT->getElementType(),
1006 args: AT->getZExtSize());
1007 }
1008 if (const RecordType *RT = Ty->getAs<RecordType>()) {
1009 SmallVector<const CXXBaseSpecifier *, 1> Bases;
1010 SmallVector<const FieldDecl *, 1> Fields;
1011 const RecordDecl *RD = RT->getDecl();
1012 assert(!RD->hasFlexibleArrayMember() &&
1013 "Cannot expand structure with flexible array.");
1014 if (RD->isUnion()) {
1015 // Unions can be here only in degenerative cases - all the fields are same
1016 // after flattening. Thus we have to use the "largest" field.
1017 const FieldDecl *LargestFD = nullptr;
1018 CharUnits UnionSize = CharUnits::Zero();
1019
1020 for (const auto *FD : RD->fields()) {
1021 if (FD->isZeroLengthBitField())
1022 continue;
1023 assert(!FD->isBitField() &&
1024 "Cannot expand structure with bit-field members.");
1025 CharUnits FieldSize = Context.getTypeSizeInChars(T: FD->getType());
1026 if (UnionSize < FieldSize) {
1027 UnionSize = FieldSize;
1028 LargestFD = FD;
1029 }
1030 }
1031 if (LargestFD)
1032 Fields.push_back(Elt: LargestFD);
1033 } else {
1034 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
1035 assert(!CXXRD->isDynamicClass() &&
1036 "cannot expand vtable pointers in dynamic classes");
1037 llvm::append_range(C&: Bases, R: llvm::make_pointer_range(Range: CXXRD->bases()));
1038 }
1039
1040 for (const auto *FD : RD->fields()) {
1041 if (FD->isZeroLengthBitField())
1042 continue;
1043 assert(!FD->isBitField() &&
1044 "Cannot expand structure with bit-field members.");
1045 Fields.push_back(Elt: FD);
1046 }
1047 }
1048 return std::make_unique<RecordExpansion>(args: std::move(Bases),
1049 args: std::move(Fields));
1050 }
1051 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
1052 return std::make_unique<ComplexExpansion>(args: CT->getElementType());
1053 }
1054 return std::make_unique<NoExpansion>();
1055}
1056
1057static int getExpansionSize(QualType Ty, const ASTContext &Context) {
1058 auto Exp = getTypeExpansion(Ty, Context);
1059 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1060 return CAExp->NumElts * getExpansionSize(Ty: CAExp->EltTy, Context);
1061 }
1062 if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1063 int Res = 0;
1064 for (auto BS : RExp->Bases)
1065 Res += getExpansionSize(Ty: BS->getType(), Context);
1066 for (auto FD : RExp->Fields)
1067 Res += getExpansionSize(Ty: FD->getType(), Context);
1068 return Res;
1069 }
1070 if (isa<ComplexExpansion>(Val: Exp.get()))
1071 return 2;
1072 assert(isa<NoExpansion>(Exp.get()));
1073 return 1;
1074}
1075
1076void CodeGenTypes::getExpandedTypes(
1077 QualType Ty, SmallVectorImpl<llvm::Type *>::iterator &TI) {
1078 auto Exp = getTypeExpansion(Ty, Context);
1079 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1080 for (int i = 0, n = CAExp->NumElts; i < n; i++) {
1081 getExpandedTypes(Ty: CAExp->EltTy, TI);
1082 }
1083 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1084 for (auto BS : RExp->Bases)
1085 getExpandedTypes(Ty: BS->getType(), TI);
1086 for (auto FD : RExp->Fields)
1087 getExpandedTypes(Ty: FD->getType(), TI);
1088 } else if (auto CExp = dyn_cast<ComplexExpansion>(Val: Exp.get())) {
1089 llvm::Type *EltTy = ConvertType(T: CExp->EltTy);
1090 *TI++ = EltTy;
1091 *TI++ = EltTy;
1092 } else {
1093 assert(isa<NoExpansion>(Exp.get()));
1094 *TI++ = ConvertType(T: Ty);
1095 }
1096}
1097
1098static void forConstantArrayExpansion(CodeGenFunction &CGF,
1099 ConstantArrayExpansion *CAE,
1100 Address BaseAddr,
1101 llvm::function_ref<void(Address)> Fn) {
1102 for (int i = 0, n = CAE->NumElts; i < n; i++) {
1103 Address EltAddr = CGF.Builder.CreateConstGEP2_32(Addr: BaseAddr, Idx0: 0, Idx1: i);
1104 Fn(EltAddr);
1105 }
1106}
1107
1108void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1109 llvm::Function::arg_iterator &AI) {
1110 assert(LV.isSimple() &&
1111 "Unexpected non-simple lvalue during struct expansion.");
1112
1113 auto Exp = getTypeExpansion(Ty, Context: getContext());
1114 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1115 forConstantArrayExpansion(
1116 CGF&: *this, CAE: CAExp, BaseAddr: LV.getAddress(), Fn: [&](Address EltAddr) {
1117 LValue LV = MakeAddrLValue(Addr: EltAddr, T: CAExp->EltTy);
1118 ExpandTypeFromArgs(Ty: CAExp->EltTy, LV, AI);
1119 });
1120 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1121 Address This = LV.getAddress();
1122 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1123 // Perform a single step derived-to-base conversion.
1124 Address Base =
1125 GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1126 /*NullCheckValue=*/false, Loc: SourceLocation());
1127 LValue SubLV = MakeAddrLValue(Addr: Base, T: BS->getType());
1128
1129 // Recurse onto bases.
1130 ExpandTypeFromArgs(Ty: BS->getType(), LV: SubLV, AI);
1131 }
1132 for (auto FD : RExp->Fields) {
1133 // FIXME: What are the right qualifiers here?
1134 LValue SubLV = EmitLValueForFieldInitialization(Base: LV, Field: FD);
1135 ExpandTypeFromArgs(Ty: FD->getType(), LV: SubLV, AI);
1136 }
1137 } else if (isa<ComplexExpansion>(Val: Exp.get())) {
1138 auto realValue = &*AI++;
1139 auto imagValue = &*AI++;
1140 EmitStoreOfComplex(V: ComplexPairTy(realValue, imagValue), dest: LV, /*init*/ isInit: true);
1141 } else {
1142 // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1143 // primitive store.
1144 assert(isa<NoExpansion>(Exp.get()));
1145 llvm::Value *Arg = &*AI++;
1146 if (LV.isBitField()) {
1147 EmitStoreThroughLValue(Src: RValue::get(V: Arg), Dst: LV);
1148 } else {
1149 // TODO: currently there are some places are inconsistent in what LLVM
1150 // pointer type they use (see D118744). Once clang uses opaque pointers
1151 // all LLVM pointer types will be the same and we can remove this check.
1152 if (Arg->getType()->isPointerTy()) {
1153 Address Addr = LV.getAddress();
1154 Arg = Builder.CreateBitCast(V: Arg, DestTy: Addr.getElementType());
1155 }
1156 EmitStoreOfScalar(value: Arg, lvalue: LV);
1157 }
1158 }
1159}
1160
1161void CodeGenFunction::ExpandTypeToArgs(
1162 QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1163 SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1164 auto Exp = getTypeExpansion(Ty, Context: getContext());
1165 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1166 Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1167 : Arg.getKnownRValue().getAggregateAddress();
1168 forConstantArrayExpansion(CGF&: *this, CAE: CAExp, BaseAddr: Addr, Fn: [&](Address EltAddr) {
1169 CallArg EltArg =
1170 CallArg(convertTempToRValue(addr: EltAddr, type: CAExp->EltTy, Loc: SourceLocation()),
1171 CAExp->EltTy);
1172 ExpandTypeToArgs(Ty: CAExp->EltTy, Arg: EltArg, IRFuncTy, IRCallArgs,
1173 IRCallArgPos);
1174 });
1175 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1176 Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1177 : Arg.getKnownRValue().getAggregateAddress();
1178 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1179 // Perform a single step derived-to-base conversion.
1180 Address Base =
1181 GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1182 /*NullCheckValue=*/false, Loc: SourceLocation());
1183 CallArg BaseArg = CallArg(RValue::getAggregate(addr: Base), BS->getType());
1184
1185 // Recurse onto bases.
1186 ExpandTypeToArgs(Ty: BS->getType(), Arg: BaseArg, IRFuncTy, IRCallArgs,
1187 IRCallArgPos);
1188 }
1189
1190 LValue LV = MakeAddrLValue(Addr: This, T: Ty);
1191 for (auto FD : RExp->Fields) {
1192 CallArg FldArg =
1193 CallArg(EmitRValueForField(LV, FD, Loc: SourceLocation()), FD->getType());
1194 ExpandTypeToArgs(Ty: FD->getType(), Arg: FldArg, IRFuncTy, IRCallArgs,
1195 IRCallArgPos);
1196 }
1197 } else if (isa<ComplexExpansion>(Val: Exp.get())) {
1198 ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1199 IRCallArgs[IRCallArgPos++] = CV.first;
1200 IRCallArgs[IRCallArgPos++] = CV.second;
1201 } else {
1202 assert(isa<NoExpansion>(Exp.get()));
1203 auto RV = Arg.getKnownRValue();
1204 assert(RV.isScalar() &&
1205 "Unexpected non-scalar rvalue during struct expansion.");
1206
1207 // Insert a bitcast as needed.
1208 llvm::Value *V = RV.getScalarVal();
1209 if (IRCallArgPos < IRFuncTy->getNumParams() &&
1210 V->getType() != IRFuncTy->getParamType(i: IRCallArgPos))
1211 V = Builder.CreateBitCast(V, DestTy: IRFuncTy->getParamType(i: IRCallArgPos));
1212
1213 IRCallArgs[IRCallArgPos++] = V;
1214 }
1215}
1216
1217/// Create a temporary allocation for the purposes of coercion.
1218static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF,
1219 llvm::Type *Ty,
1220 CharUnits MinAlign,
1221 const Twine &Name = "tmp") {
1222 // Don't use an alignment that's worse than what LLVM would prefer.
1223 auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty);
1224 CharUnits Align = std::max(a: MinAlign, b: CharUnits::fromQuantity(Quantity: PrefAlign));
1225
1226 return CGF.CreateTempAlloca(Ty, align: Align, Name: Name + ".coerce");
1227}
1228
1229/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1230/// accessing some number of bytes out of it, try to gep into the struct to get
1231/// at its inner goodness. Dive as deep as possible without entering an element
1232/// with an in-memory size smaller than DstSize.
1233static Address EnterStructPointerForCoercedAccess(Address SrcPtr,
1234 llvm::StructType *SrcSTy,
1235 uint64_t DstSize,
1236 CodeGenFunction &CGF) {
1237 // We can't dive into a zero-element struct.
1238 if (SrcSTy->getNumElements() == 0)
1239 return SrcPtr;
1240
1241 llvm::Type *FirstElt = SrcSTy->getElementType(N: 0);
1242
1243 // If the first elt is at least as large as what we're looking for, or if the
1244 // first element is the same size as the whole struct, we can enter it. The
1245 // comparison must be made on the store size and not the alloca size. Using
1246 // the alloca size may overstate the size of the load.
1247 uint64_t FirstEltSize = CGF.CGM.getDataLayout().getTypeStoreSize(Ty: FirstElt);
1248 if (FirstEltSize < DstSize &&
1249 FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(Ty: SrcSTy))
1250 return SrcPtr;
1251
1252 // GEP into the first element.
1253 SrcPtr = CGF.Builder.CreateStructGEP(Addr: SrcPtr, Index: 0, Name: "coerce.dive");
1254
1255 // If the first element is a struct, recurse.
1256 llvm::Type *SrcTy = SrcPtr.getElementType();
1257 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy))
1258 return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1259
1260 return SrcPtr;
1261}
1262
1263/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1264/// are either integers or pointers. This does a truncation of the value if it
1265/// is too large or a zero extension if it is too small.
1266///
1267/// This behaves as if the value were coerced through memory, so on big-endian
1268/// targets the high bits are preserved in a truncation, while little-endian
1269/// targets preserve the low bits.
1270static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, llvm::Type *Ty,
1271 CodeGenFunction &CGF) {
1272 if (Val->getType() == Ty)
1273 return Val;
1274
1275 if (isa<llvm::PointerType>(Val: Val->getType())) {
1276 // If this is Pointer->Pointer avoid conversion to and from int.
1277 if (isa<llvm::PointerType>(Val: Ty))
1278 return CGF.Builder.CreateBitCast(V: Val, DestTy: Ty, Name: "coerce.val");
1279
1280 // Convert the pointer to an integer so we can play with its width.
1281 Val = CGF.Builder.CreatePtrToInt(V: Val, DestTy: CGF.IntPtrTy, Name: "coerce.val.pi");
1282 }
1283
1284 llvm::Type *DestIntTy = Ty;
1285 if (isa<llvm::PointerType>(Val: DestIntTy))
1286 DestIntTy = CGF.IntPtrTy;
1287
1288 if (Val->getType() != DestIntTy) {
1289 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1290 if (DL.isBigEndian()) {
1291 // Preserve the high bits on big-endian targets.
1292 // That is what memory coercion does.
1293 uint64_t SrcSize = DL.getTypeSizeInBits(Ty: Val->getType());
1294 uint64_t DstSize = DL.getTypeSizeInBits(Ty: DestIntTy);
1295
1296 if (SrcSize > DstSize) {
1297 Val = CGF.Builder.CreateLShr(LHS: Val, RHS: SrcSize - DstSize, Name: "coerce.highbits");
1298 Val = CGF.Builder.CreateTrunc(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1299 } else {
1300 Val = CGF.Builder.CreateZExt(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1301 Val = CGF.Builder.CreateShl(LHS: Val, RHS: DstSize - SrcSize, Name: "coerce.highbits");
1302 }
1303 } else {
1304 // Little-endian targets preserve the low bits. No shifts required.
1305 Val = CGF.Builder.CreateIntCast(V: Val, DestTy: DestIntTy, isSigned: false, Name: "coerce.val.ii");
1306 }
1307 }
1308
1309 if (isa<llvm::PointerType>(Val: Ty))
1310 Val = CGF.Builder.CreateIntToPtr(V: Val, DestTy: Ty, Name: "coerce.val.ip");
1311 return Val;
1312}
1313
1314/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1315/// a pointer to an object of type \arg Ty, known to be aligned to
1316/// \arg SrcAlign bytes.
1317///
1318/// This safely handles the case when the src type is smaller than the
1319/// destination type; in this situation the values of bits which not
1320/// present in the src are undefined.
1321static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
1322 CodeGenFunction &CGF) {
1323 llvm::Type *SrcTy = Src.getElementType();
1324
1325 // If SrcTy and Ty are the same, just do a load.
1326 if (SrcTy == Ty)
1327 return CGF.Builder.CreateLoad(Addr: Src);
1328
1329 llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1330
1331 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy)) {
1332 Src = EnterStructPointerForCoercedAccess(SrcPtr: Src, SrcSTy,
1333 DstSize: DstSize.getFixedValue(), CGF);
1334 SrcTy = Src.getElementType();
1335 }
1336
1337 llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1338
1339 // If the source and destination are integer or pointer types, just do an
1340 // extension or truncation to the desired type.
1341 if ((isa<llvm::IntegerType>(Val: Ty) || isa<llvm::PointerType>(Val: Ty)) &&
1342 (isa<llvm::IntegerType>(Val: SrcTy) || isa<llvm::PointerType>(Val: SrcTy))) {
1343 llvm::Value *Load = CGF.Builder.CreateLoad(Addr: Src);
1344 return CoerceIntOrPtrToIntOrPtr(Val: Load, Ty, CGF);
1345 }
1346
1347 // If load is legal, just bitcast the src pointer.
1348 if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1349 SrcSize.getFixedValue() >= DstSize.getFixedValue()) {
1350 // Generally SrcSize is never greater than DstSize, since this means we are
1351 // losing bits. However, this can happen in cases where the structure has
1352 // additional padding, for example due to a user specified alignment.
1353 //
1354 // FIXME: Assert that we aren't truncating non-padding bits when have access
1355 // to that information.
1356 Src = Src.withElementType(ElemTy: Ty);
1357 return CGF.Builder.CreateLoad(Addr: Src);
1358 }
1359
1360 // If coercing a fixed vector to a scalable vector for ABI compatibility, and
1361 // the types match, use the llvm.vector.insert intrinsic to perform the
1362 // conversion.
1363 if (auto *ScalableDstTy = dyn_cast<llvm::ScalableVectorType>(Val: Ty)) {
1364 if (auto *FixedSrcTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy)) {
1365 // If we are casting a fixed i8 vector to a scalable i1 predicate
1366 // vector, use a vector insert and bitcast the result.
1367 if (ScalableDstTy->getElementType()->isIntegerTy(Bitwidth: 1) &&
1368 FixedSrcTy->getElementType()->isIntegerTy(Bitwidth: 8)) {
1369 ScalableDstTy = llvm::ScalableVectorType::get(
1370 ElementType: FixedSrcTy->getElementType(),
1371 MinNumElts: llvm::divideCeil(
1372 Numerator: ScalableDstTy->getElementCount().getKnownMinValue(), Denominator: 8));
1373 }
1374 if (ScalableDstTy->getElementType() == FixedSrcTy->getElementType()) {
1375 auto *Load = CGF.Builder.CreateLoad(Addr: Src);
1376 auto *PoisonVec = llvm::PoisonValue::get(T: ScalableDstTy);
1377 llvm::Value *Result = CGF.Builder.CreateInsertVector(
1378 DstType: ScalableDstTy, SrcVec: PoisonVec, SubVec: Load, Idx: uint64_t(0), Name: "cast.scalable");
1379 ScalableDstTy = cast<llvm::ScalableVectorType>(
1380 Val: llvm::VectorType::getWithSizeAndScalar(SizeTy: ScalableDstTy, EltTy: Ty));
1381 if (Result->getType() != ScalableDstTy)
1382 Result = CGF.Builder.CreateBitCast(V: Result, DestTy: ScalableDstTy);
1383 if (Result->getType() != Ty)
1384 Result = CGF.Builder.CreateExtractVector(DstType: Ty, SrcVec: Result, Idx: uint64_t(0));
1385 return Result;
1386 }
1387 }
1388 }
1389
1390 // Otherwise do coercion through memory. This is stupid, but simple.
1391 RawAddress Tmp =
1392 CreateTempAllocaForCoercion(CGF, Ty, MinAlign: Src.getAlignment(), Name: Src.getName());
1393 CGF.Builder.CreateMemCpy(
1394 Dst: Tmp.getPointer(), DstAlign: Tmp.getAlignment().getAsAlign(),
1395 Src: Src.emitRawPointer(CGF), SrcAlign: Src.getAlignment().getAsAlign(),
1396 Size: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: SrcSize.getKnownMinValue()));
1397 return CGF.Builder.CreateLoad(Addr: Tmp);
1398}
1399
1400void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, Address Dst,
1401 llvm::TypeSize DstSize,
1402 bool DstIsVolatile) {
1403 if (!DstSize)
1404 return;
1405
1406 llvm::Type *SrcTy = Src->getType();
1407 llvm::TypeSize SrcSize = CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1408
1409 // GEP into structs to try to make types match.
1410 // FIXME: This isn't really that useful with opaque types, but it impacts a
1411 // lot of regression tests.
1412 if (SrcTy != Dst.getElementType()) {
1413 if (llvm::StructType *DstSTy =
1414 dyn_cast<llvm::StructType>(Val: Dst.getElementType())) {
1415 assert(!SrcSize.isScalable());
1416 Dst = EnterStructPointerForCoercedAccess(SrcPtr: Dst, SrcSTy: DstSTy,
1417 DstSize: SrcSize.getFixedValue(), CGF&: *this);
1418 }
1419 }
1420
1421 if (SrcSize.isScalable() || SrcSize <= DstSize) {
1422 if (SrcTy->isIntegerTy() && Dst.getElementType()->isPointerTy() &&
1423 SrcSize == CGM.getDataLayout().getTypeAllocSize(Ty: Dst.getElementType())) {
1424 // If the value is supposed to be a pointer, convert it before storing it.
1425 Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: Dst.getElementType(), CGF&: *this);
1426 auto *I = Builder.CreateStore(Val: Src, Addr: Dst, IsVolatile: DstIsVolatile);
1427 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1428 } else if (llvm::StructType *STy =
1429 dyn_cast<llvm::StructType>(Val: Src->getType())) {
1430 // Prefer scalar stores to first-class aggregate stores.
1431 Dst = Dst.withElementType(ElemTy: SrcTy);
1432 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1433 Address EltPtr = Builder.CreateStructGEP(Addr: Dst, Index: i);
1434 llvm::Value *Elt = Builder.CreateExtractValue(Agg: Src, Idxs: i);
1435 auto *I = Builder.CreateStore(Val: Elt, Addr: EltPtr, IsVolatile: DstIsVolatile);
1436 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Elt);
1437 }
1438 } else {
1439 auto *I =
1440 Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: SrcTy), IsVolatile: DstIsVolatile);
1441 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1442 }
1443 } else if (SrcTy->isIntegerTy()) {
1444 // If the source is a simple integer, coerce it directly.
1445 llvm::Type *DstIntTy = Builder.getIntNTy(N: DstSize.getFixedValue() * 8);
1446 Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: DstIntTy, CGF&: *this);
1447 auto *I =
1448 Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: DstIntTy), IsVolatile: DstIsVolatile);
1449 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1450 } else {
1451 // Otherwise do coercion through memory. This is stupid, but
1452 // simple.
1453
1454 // Generally SrcSize is never greater than DstSize, since this means we are
1455 // losing bits. However, this can happen in cases where the structure has
1456 // additional padding, for example due to a user specified alignment.
1457 //
1458 // FIXME: Assert that we aren't truncating non-padding bits when have access
1459 // to that information.
1460 RawAddress Tmp =
1461 CreateTempAllocaForCoercion(CGF&: *this, Ty: SrcTy, MinAlign: Dst.getAlignment());
1462 Builder.CreateStore(Val: Src, Addr: Tmp);
1463 auto *I = Builder.CreateMemCpy(
1464 Dst: Dst.emitRawPointer(CGF&: *this), DstAlign: Dst.getAlignment().getAsAlign(),
1465 Src: Tmp.getPointer(), SrcAlign: Tmp.getAlignment().getAsAlign(),
1466 Size: Builder.CreateTypeSize(Ty: IntPtrTy, Size: DstSize));
1467 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1468 }
1469}
1470
1471static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1472 const ABIArgInfo &info) {
1473 if (unsigned offset = info.getDirectOffset()) {
1474 addr = addr.withElementType(ElemTy: CGF.Int8Ty);
1475 addr = CGF.Builder.CreateConstInBoundsByteGEP(
1476 Addr: addr, Offset: CharUnits::fromQuantity(Quantity: offset));
1477 addr = addr.withElementType(ElemTy: info.getCoerceToType());
1478 }
1479 return addr;
1480}
1481
1482static std::pair<llvm::Value *, bool>
1483CoerceScalableToFixed(CodeGenFunction &CGF, llvm::FixedVectorType *ToTy,
1484 llvm::ScalableVectorType *FromTy, llvm::Value *V,
1485 StringRef Name = "") {
1486 // If we are casting a scalable i1 predicate vector to a fixed i8
1487 // vector, first bitcast the source.
1488 if (FromTy->getElementType()->isIntegerTy(Bitwidth: 1) &&
1489 ToTy->getElementType() == CGF.Builder.getInt8Ty()) {
1490 if (!FromTy->getElementCount().isKnownMultipleOf(RHS: 8)) {
1491 FromTy = llvm::ScalableVectorType::get(
1492 ElementType: FromTy->getElementType(),
1493 MinNumElts: llvm::alignTo<8>(Value: FromTy->getElementCount().getKnownMinValue()));
1494 llvm::Value *ZeroVec = llvm::Constant::getNullValue(Ty: FromTy);
1495 V = CGF.Builder.CreateInsertVector(DstType: FromTy, SrcVec: ZeroVec, SubVec: V, Idx: uint64_t(0));
1496 }
1497 FromTy = llvm::ScalableVectorType::get(
1498 ElementType: ToTy->getElementType(),
1499 MinNumElts: FromTy->getElementCount().getKnownMinValue() / 8);
1500 V = CGF.Builder.CreateBitCast(V, DestTy: FromTy);
1501 }
1502 if (FromTy->getElementType() == ToTy->getElementType()) {
1503 V->setName(Name + ".coerce");
1504 V = CGF.Builder.CreateExtractVector(DstType: ToTy, SrcVec: V, Idx: uint64_t(0), Name: "cast.fixed");
1505 return {V, true};
1506 }
1507 return {V, false};
1508}
1509
1510namespace {
1511
1512/// Encapsulates information about the way function arguments from
1513/// CGFunctionInfo should be passed to actual LLVM IR function.
1514class ClangToLLVMArgMapping {
1515 static const unsigned InvalidIndex = ~0U;
1516 unsigned InallocaArgNo;
1517 unsigned SRetArgNo;
1518 unsigned TotalIRArgs;
1519
1520 /// Arguments of LLVM IR function corresponding to single Clang argument.
1521 struct IRArgs {
1522 unsigned PaddingArgIndex;
1523 // Argument is expanded to IR arguments at positions
1524 // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1525 unsigned FirstArgIndex;
1526 unsigned NumberOfArgs;
1527
1528 IRArgs()
1529 : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1530 NumberOfArgs(0) {}
1531 };
1532
1533 SmallVector<IRArgs, 8> ArgInfo;
1534
1535public:
1536 ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1537 bool OnlyRequiredArgs = false)
1538 : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1539 ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1540 construct(Context, FI, OnlyRequiredArgs);
1541 }
1542
1543 bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1544 unsigned getInallocaArgNo() const {
1545 assert(hasInallocaArg());
1546 return InallocaArgNo;
1547 }
1548
1549 bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1550 unsigned getSRetArgNo() const {
1551 assert(hasSRetArg());
1552 return SRetArgNo;
1553 }
1554
1555 unsigned totalIRArgs() const { return TotalIRArgs; }
1556
1557 bool hasPaddingArg(unsigned ArgNo) const {
1558 assert(ArgNo < ArgInfo.size());
1559 return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1560 }
1561 unsigned getPaddingArgNo(unsigned ArgNo) const {
1562 assert(hasPaddingArg(ArgNo));
1563 return ArgInfo[ArgNo].PaddingArgIndex;
1564 }
1565
1566 /// Returns index of first IR argument corresponding to ArgNo, and their
1567 /// quantity.
1568 std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1569 assert(ArgNo < ArgInfo.size());
1570 return std::make_pair(x: ArgInfo[ArgNo].FirstArgIndex,
1571 y: ArgInfo[ArgNo].NumberOfArgs);
1572 }
1573
1574private:
1575 void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1576 bool OnlyRequiredArgs);
1577};
1578
1579void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1580 const CGFunctionInfo &FI,
1581 bool OnlyRequiredArgs) {
1582 unsigned IRArgNo = 0;
1583 bool SwapThisWithSRet = false;
1584 const ABIArgInfo &RetAI = FI.getReturnInfo();
1585
1586 if (RetAI.getKind() == ABIArgInfo::Indirect) {
1587 SwapThisWithSRet = RetAI.isSRetAfterThis();
1588 SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1589 }
1590
1591 unsigned ArgNo = 0;
1592 unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();
1593 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1594 ++I, ++ArgNo) {
1595 assert(I != FI.arg_end());
1596 QualType ArgType = I->type;
1597 const ABIArgInfo &AI = I->info;
1598 // Collect data about IR arguments corresponding to Clang argument ArgNo.
1599 auto &IRArgs = ArgInfo[ArgNo];
1600
1601 if (AI.getPaddingType())
1602 IRArgs.PaddingArgIndex = IRArgNo++;
1603
1604 switch (AI.getKind()) {
1605 case ABIArgInfo::Extend:
1606 case ABIArgInfo::Direct: {
1607 // FIXME: handle sseregparm someday...
1608 llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: AI.getCoerceToType());
1609 if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
1610 IRArgs.NumberOfArgs = STy->getNumElements();
1611 } else {
1612 IRArgs.NumberOfArgs = 1;
1613 }
1614 break;
1615 }
1616 case ABIArgInfo::Indirect:
1617 case ABIArgInfo::IndirectAliased:
1618 IRArgs.NumberOfArgs = 1;
1619 break;
1620 case ABIArgInfo::Ignore:
1621 case ABIArgInfo::InAlloca:
1622 // ignore and inalloca doesn't have matching LLVM parameters.
1623 IRArgs.NumberOfArgs = 0;
1624 break;
1625 case ABIArgInfo::CoerceAndExpand:
1626 IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1627 break;
1628 case ABIArgInfo::Expand:
1629 IRArgs.NumberOfArgs = getExpansionSize(Ty: ArgType, Context);
1630 break;
1631 }
1632
1633 if (IRArgs.NumberOfArgs > 0) {
1634 IRArgs.FirstArgIndex = IRArgNo;
1635 IRArgNo += IRArgs.NumberOfArgs;
1636 }
1637
1638 // Skip over the sret parameter when it comes second. We already handled it
1639 // above.
1640 if (IRArgNo == 1 && SwapThisWithSRet)
1641 IRArgNo++;
1642 }
1643 assert(ArgNo == ArgInfo.size());
1644
1645 if (FI.usesInAlloca())
1646 InallocaArgNo = IRArgNo++;
1647
1648 TotalIRArgs = IRArgNo;
1649}
1650} // namespace
1651
1652/***/
1653
1654bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1655 const auto &RI = FI.getReturnInfo();
1656 return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());
1657}
1658
1659bool CodeGenModule::ReturnTypeHasInReg(const CGFunctionInfo &FI) {
1660 const auto &RI = FI.getReturnInfo();
1661 return RI.getInReg();
1662}
1663
1664bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1665 return ReturnTypeUsesSRet(FI) &&
1666 getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
1667}
1668
1669bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1670 if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1671 switch (BT->getKind()) {
1672 default:
1673 return false;
1674 case BuiltinType::Float:
1675 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Float);
1676 case BuiltinType::Double:
1677 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Double);
1678 case BuiltinType::LongDouble:
1679 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::LongDouble);
1680 }
1681 }
1682
1683 return false;
1684}
1685
1686bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1687 if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1688 if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1689 if (BT->getKind() == BuiltinType::LongDouble)
1690 return getTarget().useObjCFP2RetForComplexLongDouble();
1691 }
1692 }
1693
1694 return false;
1695}
1696
1697llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1698 const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1699 return GetFunctionType(Info: FI);
1700}
1701
1702llvm::FunctionType *CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1703
1704 bool Inserted = FunctionsBeingProcessed.insert(Ptr: &FI).second;
1705 (void)Inserted;
1706 assert(Inserted && "Recursively being processed?");
1707
1708 llvm::Type *resultType = nullptr;
1709 const ABIArgInfo &retAI = FI.getReturnInfo();
1710 switch (retAI.getKind()) {
1711 case ABIArgInfo::Expand:
1712 case ABIArgInfo::IndirectAliased:
1713 llvm_unreachable("Invalid ABI kind for return argument");
1714
1715 case ABIArgInfo::Extend:
1716 case ABIArgInfo::Direct:
1717 resultType = retAI.getCoerceToType();
1718 break;
1719
1720 case ABIArgInfo::InAlloca:
1721 if (retAI.getInAllocaSRet()) {
1722 // sret things on win32 aren't void, they return the sret pointer.
1723 QualType ret = FI.getReturnType();
1724 unsigned addressSpace = CGM.getTypes().getTargetAddressSpace(T: ret);
1725 resultType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: addressSpace);
1726 } else {
1727 resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
1728 }
1729 break;
1730
1731 case ABIArgInfo::Indirect:
1732 case ABIArgInfo::Ignore:
1733 resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
1734 break;
1735
1736 case ABIArgInfo::CoerceAndExpand:
1737 resultType = retAI.getUnpaddedCoerceAndExpandType();
1738 break;
1739 }
1740
1741 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1742 SmallVector<llvm::Type *, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1743
1744 // Add type for sret argument.
1745 if (IRFunctionArgs.hasSRetArg()) {
1746 ArgTypes[IRFunctionArgs.getSRetArgNo()] = llvm::PointerType::get(
1747 C&: getLLVMContext(), AddressSpace: FI.getReturnInfo().getIndirectAddrSpace());
1748 }
1749
1750 // Add type for inalloca argument.
1751 if (IRFunctionArgs.hasInallocaArg())
1752 ArgTypes[IRFunctionArgs.getInallocaArgNo()] =
1753 llvm::PointerType::getUnqual(C&: getLLVMContext());
1754
1755 // Add in all of the required arguments.
1756 unsigned ArgNo = 0;
1757 CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1758 ie = it + FI.getNumRequiredArgs();
1759 for (; it != ie; ++it, ++ArgNo) {
1760 const ABIArgInfo &ArgInfo = it->info;
1761
1762 // Insert a padding type to ensure proper alignment.
1763 if (IRFunctionArgs.hasPaddingArg(ArgNo))
1764 ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1765 ArgInfo.getPaddingType();
1766
1767 unsigned FirstIRArg, NumIRArgs;
1768 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1769
1770 switch (ArgInfo.getKind()) {
1771 case ABIArgInfo::Ignore:
1772 case ABIArgInfo::InAlloca:
1773 assert(NumIRArgs == 0);
1774 break;
1775
1776 case ABIArgInfo::Indirect:
1777 assert(NumIRArgs == 1);
1778 // indirect arguments are always on the stack, which is alloca addr space.
1779 ArgTypes[FirstIRArg] = llvm::PointerType::get(
1780 C&: getLLVMContext(), AddressSpace: CGM.getDataLayout().getAllocaAddrSpace());
1781 break;
1782 case ABIArgInfo::IndirectAliased:
1783 assert(NumIRArgs == 1);
1784 ArgTypes[FirstIRArg] = llvm::PointerType::get(
1785 C&: getLLVMContext(), AddressSpace: ArgInfo.getIndirectAddrSpace());
1786 break;
1787 case ABIArgInfo::Extend:
1788 case ABIArgInfo::Direct: {
1789 // Fast-isel and the optimizer generally like scalar values better than
1790 // FCAs, so we flatten them if this is safe to do for this argument.
1791 llvm::Type *argType = ArgInfo.getCoerceToType();
1792 llvm::StructType *st = dyn_cast<llvm::StructType>(Val: argType);
1793 if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
1794 assert(NumIRArgs == st->getNumElements());
1795 for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
1796 ArgTypes[FirstIRArg + i] = st->getElementType(N: i);
1797 } else {
1798 assert(NumIRArgs == 1);
1799 ArgTypes[FirstIRArg] = argType;
1800 }
1801 break;
1802 }
1803
1804 case ABIArgInfo::CoerceAndExpand: {
1805 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1806 for (auto *EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1807 *ArgTypesIter++ = EltTy;
1808 }
1809 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1810 break;
1811 }
1812
1813 case ABIArgInfo::Expand:
1814 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1815 getExpandedTypes(Ty: it->type, TI&: ArgTypesIter);
1816 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1817 break;
1818 }
1819 }
1820
1821 bool Erased = FunctionsBeingProcessed.erase(Ptr: &FI);
1822 (void)Erased;
1823 assert(Erased && "Not in set?");
1824
1825 return llvm::FunctionType::get(Result: resultType, Params: ArgTypes, isVarArg: FI.isVariadic());
1826}
1827
1828llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1829 const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
1830 const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
1831
1832 if (!isFuncTypeConvertible(FT: FPT))
1833 return llvm::StructType::get(Context&: getLLVMContext());
1834
1835 return GetFunctionType(GD);
1836}
1837
1838static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
1839 llvm::AttrBuilder &FuncAttrs,
1840 const FunctionProtoType *FPT) {
1841 if (!FPT)
1842 return;
1843
1844 if (!isUnresolvedExceptionSpec(ESpecType: FPT->getExceptionSpecType()) &&
1845 FPT->isNothrow())
1846 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
1847
1848 unsigned SMEBits = FPT->getAArch64SMEAttributes();
1849 if (SMEBits & FunctionType::SME_PStateSMEnabledMask)
1850 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_enabled");
1851 if (SMEBits & FunctionType::SME_PStateSMCompatibleMask)
1852 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_compatible");
1853 if (SMEBits & FunctionType::SME_AgnosticZAStateMask)
1854 FuncAttrs.addAttribute(A: "aarch64_za_state_agnostic");
1855
1856 // ZA
1857 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
1858 FuncAttrs.addAttribute(A: "aarch64_preserves_za");
1859 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_In)
1860 FuncAttrs.addAttribute(A: "aarch64_in_za");
1861 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Out)
1862 FuncAttrs.addAttribute(A: "aarch64_out_za");
1863 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_InOut)
1864 FuncAttrs.addAttribute(A: "aarch64_inout_za");
1865
1866 // ZT0
1867 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
1868 FuncAttrs.addAttribute(A: "aarch64_preserves_zt0");
1869 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_In)
1870 FuncAttrs.addAttribute(A: "aarch64_in_zt0");
1871 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Out)
1872 FuncAttrs.addAttribute(A: "aarch64_out_zt0");
1873 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_InOut)
1874 FuncAttrs.addAttribute(A: "aarch64_inout_zt0");
1875}
1876
1877static void AddAttributesFromOMPAssumes(llvm::AttrBuilder &FuncAttrs,
1878 const Decl *Callee) {
1879 if (!Callee)
1880 return;
1881
1882 SmallVector<StringRef, 4> Attrs;
1883
1884 for (const OMPAssumeAttr *AA : Callee->specific_attrs<OMPAssumeAttr>())
1885 AA->getAssumption().split(A&: Attrs, Separator: ",");
1886
1887 if (!Attrs.empty())
1888 FuncAttrs.addAttribute(A: llvm::AssumptionAttrKey,
1889 V: llvm::join(Begin: Attrs.begin(), End: Attrs.end(), Separator: ","));
1890}
1891
1892bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,
1893 QualType ReturnType) const {
1894 // We can't just discard the return value for a record type with a
1895 // complex destructor or a non-trivially copyable type.
1896 if (const RecordType *RT =
1897 ReturnType.getCanonicalType()->getAs<RecordType>()) {
1898 if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(Val: RT->getDecl()))
1899 return ClassDecl->hasTrivialDestructor();
1900 }
1901 return ReturnType.isTriviallyCopyableType(Context);
1902}
1903
1904static bool HasStrictReturn(const CodeGenModule &Module, QualType RetTy,
1905 const Decl *TargetDecl) {
1906 // As-is msan can not tolerate noundef mismatch between caller and
1907 // implementation. Mismatch is possible for e.g. indirect calls from C-caller
1908 // into C++. Such mismatches lead to confusing false reports. To avoid
1909 // expensive workaround on msan we enforce initialization event in uncommon
1910 // cases where it's allowed.
1911 if (Module.getLangOpts().Sanitize.has(K: SanitizerKind::Memory))
1912 return true;
1913 // C++ explicitly makes returning undefined values UB. C's rule only applies
1914 // to used values, so we never mark them noundef for now.
1915 if (!Module.getLangOpts().CPlusPlus)
1916 return false;
1917 if (TargetDecl) {
1918 if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
1919 if (FDecl->isExternC())
1920 return false;
1921 } else if (const VarDecl *VDecl = dyn_cast<VarDecl>(Val: TargetDecl)) {
1922 // Function pointer.
1923 if (VDecl->isExternC())
1924 return false;
1925 }
1926 }
1927
1928 // We don't want to be too aggressive with the return checking, unless
1929 // it's explicit in the code opts or we're using an appropriate sanitizer.
1930 // Try to respect what the programmer intended.
1931 return Module.getCodeGenOpts().StrictReturn ||
1932 !Module.MayDropFunctionReturn(Context: Module.getContext(), ReturnType: RetTy) ||
1933 Module.getLangOpts().Sanitize.has(K: SanitizerKind::Return);
1934}
1935
1936/// Add denormal-fp-math and denormal-fp-math-f32 as appropriate for the
1937/// requested denormal behavior, accounting for the overriding behavior of the
1938/// -f32 case.
1939static void addDenormalModeAttrs(llvm::DenormalMode FPDenormalMode,
1940 llvm::DenormalMode FP32DenormalMode,
1941 llvm::AttrBuilder &FuncAttrs) {
1942 if (FPDenormalMode != llvm::DenormalMode::getDefault())
1943 FuncAttrs.addAttribute(A: "denormal-fp-math", V: FPDenormalMode.str());
1944
1945 if (FP32DenormalMode != FPDenormalMode && FP32DenormalMode.isValid())
1946 FuncAttrs.addAttribute(A: "denormal-fp-math-f32", V: FP32DenormalMode.str());
1947}
1948
1949/// Add default attributes to a function, which have merge semantics under
1950/// -mlink-builtin-bitcode and should not simply overwrite any existing
1951/// attributes in the linked library.
1952static void
1953addMergableDefaultFunctionAttributes(const CodeGenOptions &CodeGenOpts,
1954 llvm::AttrBuilder &FuncAttrs) {
1955 addDenormalModeAttrs(FPDenormalMode: CodeGenOpts.FPDenormalMode, FP32DenormalMode: CodeGenOpts.FP32DenormalMode,
1956 FuncAttrs);
1957}
1958
1959static void getTrivialDefaultFunctionAttributes(
1960 StringRef Name, bool HasOptnone, const CodeGenOptions &CodeGenOpts,
1961 const LangOptions &LangOpts, bool AttrOnCallSite,
1962 llvm::AttrBuilder &FuncAttrs) {
1963 // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
1964 if (!HasOptnone) {
1965 if (CodeGenOpts.OptimizeSize)
1966 FuncAttrs.addAttribute(Val: llvm::Attribute::OptimizeForSize);
1967 if (CodeGenOpts.OptimizeSize == 2)
1968 FuncAttrs.addAttribute(Val: llvm::Attribute::MinSize);
1969 }
1970
1971 if (CodeGenOpts.DisableRedZone)
1972 FuncAttrs.addAttribute(Val: llvm::Attribute::NoRedZone);
1973 if (CodeGenOpts.IndirectTlsSegRefs)
1974 FuncAttrs.addAttribute(A: "indirect-tls-seg-refs");
1975 if (CodeGenOpts.NoImplicitFloat)
1976 FuncAttrs.addAttribute(Val: llvm::Attribute::NoImplicitFloat);
1977
1978 if (AttrOnCallSite) {
1979 // Attributes that should go on the call site only.
1980 // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
1981 // the -fno-builtin-foo list.
1982 if (!CodeGenOpts.SimplifyLibCalls || LangOpts.isNoBuiltinFunc(Name))
1983 FuncAttrs.addAttribute(Val: llvm::Attribute::NoBuiltin);
1984 if (!CodeGenOpts.TrapFuncName.empty())
1985 FuncAttrs.addAttribute(A: "trap-func-name", V: CodeGenOpts.TrapFuncName);
1986 } else {
1987 switch (CodeGenOpts.getFramePointer()) {
1988 case CodeGenOptions::FramePointerKind::None:
1989 // This is the default behavior.
1990 break;
1991 case CodeGenOptions::FramePointerKind::Reserved:
1992 case CodeGenOptions::FramePointerKind::NonLeaf:
1993 case CodeGenOptions::FramePointerKind::All:
1994 FuncAttrs.addAttribute(A: "frame-pointer",
1995 V: CodeGenOptions::getFramePointerKindName(
1996 Kind: CodeGenOpts.getFramePointer()));
1997 }
1998
1999 if (CodeGenOpts.LessPreciseFPMAD)
2000 FuncAttrs.addAttribute(A: "less-precise-fpmad", V: "true");
2001
2002 if (CodeGenOpts.NullPointerIsValid)
2003 FuncAttrs.addAttribute(Val: llvm::Attribute::NullPointerIsValid);
2004
2005 if (LangOpts.getDefaultExceptionMode() == LangOptions::FPE_Ignore)
2006 FuncAttrs.addAttribute(A: "no-trapping-math", V: "true");
2007
2008 // TODO: Are these all needed?
2009 // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
2010 if (LangOpts.NoHonorInfs)
2011 FuncAttrs.addAttribute(A: "no-infs-fp-math", V: "true");
2012 if (LangOpts.NoHonorNaNs)
2013 FuncAttrs.addAttribute(A: "no-nans-fp-math", V: "true");
2014 if (LangOpts.ApproxFunc)
2015 FuncAttrs.addAttribute(A: "approx-func-fp-math", V: "true");
2016 if (LangOpts.AllowFPReassoc && LangOpts.AllowRecip &&
2017 LangOpts.NoSignedZero && LangOpts.ApproxFunc &&
2018 (LangOpts.getDefaultFPContractMode() ==
2019 LangOptions::FPModeKind::FPM_Fast ||
2020 LangOpts.getDefaultFPContractMode() ==
2021 LangOptions::FPModeKind::FPM_FastHonorPragmas))
2022 FuncAttrs.addAttribute(A: "unsafe-fp-math", V: "true");
2023 if (CodeGenOpts.SoftFloat)
2024 FuncAttrs.addAttribute(A: "use-soft-float", V: "true");
2025 FuncAttrs.addAttribute(A: "stack-protector-buffer-size",
2026 V: llvm::utostr(X: CodeGenOpts.SSPBufferSize));
2027 if (LangOpts.NoSignedZero)
2028 FuncAttrs.addAttribute(A: "no-signed-zeros-fp-math", V: "true");
2029
2030 // TODO: Reciprocal estimate codegen options should apply to instructions?
2031 const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
2032 if (!Recips.empty())
2033 FuncAttrs.addAttribute(A: "reciprocal-estimates", V: llvm::join(R: Recips, Separator: ","));
2034
2035 if (!CodeGenOpts.PreferVectorWidth.empty() &&
2036 CodeGenOpts.PreferVectorWidth != "none")
2037 FuncAttrs.addAttribute(A: "prefer-vector-width",
2038 V: CodeGenOpts.PreferVectorWidth);
2039
2040 if (CodeGenOpts.StackRealignment)
2041 FuncAttrs.addAttribute(A: "stackrealign");
2042 if (CodeGenOpts.Backchain)
2043 FuncAttrs.addAttribute(A: "backchain");
2044 if (CodeGenOpts.EnableSegmentedStacks)
2045 FuncAttrs.addAttribute(A: "split-stack");
2046
2047 if (CodeGenOpts.SpeculativeLoadHardening)
2048 FuncAttrs.addAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2049
2050 // Add zero-call-used-regs attribute.
2051 switch (CodeGenOpts.getZeroCallUsedRegs()) {
2052 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
2053 FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2054 break;
2055 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
2056 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr-arg");
2057 break;
2058 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
2059 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr");
2060 break;
2061 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
2062 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-arg");
2063 break;
2064 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
2065 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used");
2066 break;
2067 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
2068 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr-arg");
2069 break;
2070 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
2071 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr");
2072 break;
2073 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
2074 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-arg");
2075 break;
2076 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
2077 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all");
2078 break;
2079 }
2080 }
2081
2082 if (LangOpts.assumeFunctionsAreConvergent()) {
2083 // Conservatively, mark all functions and calls in CUDA and OpenCL as
2084 // convergent (meaning, they may call an intrinsically convergent op, such
2085 // as __syncthreads() / barrier(), and so can't have certain optimizations
2086 // applied around them). LLVM will remove this attribute where it safely
2087 // can.
2088 FuncAttrs.addAttribute(Val: llvm::Attribute::Convergent);
2089 }
2090
2091 // TODO: NoUnwind attribute should be added for other GPU modes HIP,
2092 // OpenMP offload. AFAIK, neither of them support exceptions in device code.
2093 if ((LangOpts.CUDA && LangOpts.CUDAIsDevice) || LangOpts.OpenCL ||
2094 LangOpts.SYCLIsDevice) {
2095 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2096 }
2097
2098 if (CodeGenOpts.SaveRegParams && !AttrOnCallSite)
2099 FuncAttrs.addAttribute(A: "save-reg-params");
2100
2101 for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
2102 StringRef Var, Value;
2103 std::tie(args&: Var, args&: Value) = Attr.split(Separator: '=');
2104 FuncAttrs.addAttribute(A: Var, V: Value);
2105 }
2106
2107 TargetInfo::BranchProtectionInfo BPI(LangOpts);
2108 TargetCodeGenInfo::initBranchProtectionFnAttributes(BPI, FuncAttrs);
2109}
2110
2111/// Merges `target-features` from \TargetOpts and \F, and sets the result in
2112/// \FuncAttr
2113/// * features from \F are always kept
2114/// * a feature from \TargetOpts is kept if itself and its opposite are absent
2115/// from \F
2116static void
2117overrideFunctionFeaturesWithTargetFeatures(llvm::AttrBuilder &FuncAttr,
2118 const llvm::Function &F,
2119 const TargetOptions &TargetOpts) {
2120 auto FFeatures = F.getFnAttribute(Kind: "target-features");
2121
2122 llvm::StringSet<> MergedNames;
2123 SmallVector<StringRef> MergedFeatures;
2124 MergedFeatures.reserve(N: TargetOpts.Features.size());
2125
2126 auto AddUnmergedFeatures = [&](auto &&FeatureRange) {
2127 for (StringRef Feature : FeatureRange) {
2128 if (Feature.empty())
2129 continue;
2130 assert(Feature[0] == '+' || Feature[0] == '-');
2131 StringRef Name = Feature.drop_front(N: 1);
2132 bool Merged = !MergedNames.insert(key: Name).second;
2133 if (!Merged)
2134 MergedFeatures.push_back(Elt: Feature);
2135 }
2136 };
2137
2138 if (FFeatures.isValid())
2139 AddUnmergedFeatures(llvm::split(Str: FFeatures.getValueAsString(), Separator: ','));
2140 AddUnmergedFeatures(TargetOpts.Features);
2141
2142 if (!MergedFeatures.empty()) {
2143 llvm::sort(C&: MergedFeatures);
2144 FuncAttr.addAttribute(A: "target-features", V: llvm::join(R&: MergedFeatures, Separator: ","));
2145 }
2146}
2147
2148void CodeGen::mergeDefaultFunctionDefinitionAttributes(
2149 llvm::Function &F, const CodeGenOptions &CodeGenOpts,
2150 const LangOptions &LangOpts, const TargetOptions &TargetOpts,
2151 bool WillInternalize) {
2152
2153 llvm::AttrBuilder FuncAttrs(F.getContext());
2154 // Here we only extract the options that are relevant compared to the version
2155 // from GetCPUAndFeaturesAttributes.
2156 if (!TargetOpts.CPU.empty())
2157 FuncAttrs.addAttribute(A: "target-cpu", V: TargetOpts.CPU);
2158 if (!TargetOpts.TuneCPU.empty())
2159 FuncAttrs.addAttribute(A: "tune-cpu", V: TargetOpts.TuneCPU);
2160
2161 ::getTrivialDefaultFunctionAttributes(Name: F.getName(), HasOptnone: F.hasOptNone(),
2162 CodeGenOpts, LangOpts,
2163 /*AttrOnCallSite=*/false, FuncAttrs);
2164
2165 if (!WillInternalize && F.isInterposable()) {
2166 // Do not promote "dynamic" denormal-fp-math to this translation unit's
2167 // setting for weak functions that won't be internalized. The user has no
2168 // real control for how builtin bitcode is linked, so we shouldn't assume
2169 // later copies will use a consistent mode.
2170 F.addFnAttrs(Attrs: FuncAttrs);
2171 return;
2172 }
2173
2174 llvm::AttributeMask AttrsToRemove;
2175
2176 llvm::DenormalMode DenormModeToMerge = F.getDenormalModeRaw();
2177 llvm::DenormalMode DenormModeToMergeF32 = F.getDenormalModeF32Raw();
2178 llvm::DenormalMode Merged =
2179 CodeGenOpts.FPDenormalMode.mergeCalleeMode(Callee: DenormModeToMerge);
2180 llvm::DenormalMode MergedF32 = CodeGenOpts.FP32DenormalMode;
2181
2182 if (DenormModeToMergeF32.isValid()) {
2183 MergedF32 =
2184 CodeGenOpts.FP32DenormalMode.mergeCalleeMode(Callee: DenormModeToMergeF32);
2185 }
2186
2187 if (Merged == llvm::DenormalMode::getDefault()) {
2188 AttrsToRemove.addAttribute(A: "denormal-fp-math");
2189 } else if (Merged != DenormModeToMerge) {
2190 // Overwrite existing attribute
2191 FuncAttrs.addAttribute(A: "denormal-fp-math",
2192 V: CodeGenOpts.FPDenormalMode.str());
2193 }
2194
2195 if (MergedF32 == llvm::DenormalMode::getDefault()) {
2196 AttrsToRemove.addAttribute(A: "denormal-fp-math-f32");
2197 } else if (MergedF32 != DenormModeToMergeF32) {
2198 // Overwrite existing attribute
2199 FuncAttrs.addAttribute(A: "denormal-fp-math-f32",
2200 V: CodeGenOpts.FP32DenormalMode.str());
2201 }
2202
2203 F.removeFnAttrs(Attrs: AttrsToRemove);
2204 addDenormalModeAttrs(FPDenormalMode: Merged, FP32DenormalMode: MergedF32, FuncAttrs);
2205
2206 overrideFunctionFeaturesWithTargetFeatures(FuncAttr&: FuncAttrs, F, TargetOpts);
2207
2208 F.addFnAttrs(Attrs: FuncAttrs);
2209}
2210
2211void CodeGenModule::getTrivialDefaultFunctionAttributes(
2212 StringRef Name, bool HasOptnone, bool AttrOnCallSite,
2213 llvm::AttrBuilder &FuncAttrs) {
2214 ::getTrivialDefaultFunctionAttributes(Name, HasOptnone, CodeGenOpts: getCodeGenOpts(),
2215 LangOpts: getLangOpts(), AttrOnCallSite,
2216 FuncAttrs);
2217}
2218
2219void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
2220 bool HasOptnone,
2221 bool AttrOnCallSite,
2222 llvm::AttrBuilder &FuncAttrs) {
2223 getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,
2224 FuncAttrs);
2225
2226 if (!AttrOnCallSite)
2227 TargetCodeGenInfo::initPointerAuthFnAttributes(Opts: CodeGenOpts.PointerAuth,
2228 FuncAttrs);
2229
2230 // If we're just getting the default, get the default values for mergeable
2231 // attributes.
2232 if (!AttrOnCallSite)
2233 addMergableDefaultFunctionAttributes(CodeGenOpts, FuncAttrs);
2234}
2235
2236void CodeGenModule::addDefaultFunctionDefinitionAttributes(
2237 llvm::AttrBuilder &attrs) {
2238 getDefaultFunctionAttributes(/*function name*/ Name: "", /*optnone*/ HasOptnone: false,
2239 /*for call*/ AttrOnCallSite: false, FuncAttrs&: attrs);
2240 GetCPUAndFeaturesAttributes(GD: GlobalDecl(), AttrBuilder&: attrs);
2241}
2242
2243static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
2244 const LangOptions &LangOpts,
2245 const NoBuiltinAttr *NBA = nullptr) {
2246 auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
2247 SmallString<32> AttributeName;
2248 AttributeName += "no-builtin-";
2249 AttributeName += BuiltinName;
2250 FuncAttrs.addAttribute(A: AttributeName);
2251 };
2252
2253 // First, handle the language options passed through -fno-builtin.
2254 if (LangOpts.NoBuiltin) {
2255 // -fno-builtin disables them all.
2256 FuncAttrs.addAttribute(A: "no-builtins");
2257 return;
2258 }
2259
2260 // Then, add attributes for builtins specified through -fno-builtin-<name>.
2261 llvm::for_each(Range: LangOpts.NoBuiltinFuncs, F: AddNoBuiltinAttr);
2262
2263 // Now, let's check the __attribute__((no_builtin("...")) attribute added to
2264 // the source.
2265 if (!NBA)
2266 return;
2267
2268 // If there is a wildcard in the builtin names specified through the
2269 // attribute, disable them all.
2270 if (llvm::is_contained(Range: NBA->builtinNames(), Element: "*")) {
2271 FuncAttrs.addAttribute(A: "no-builtins");
2272 return;
2273 }
2274
2275 // And last, add the rest of the builtin names.
2276 llvm::for_each(Range: NBA->builtinNames(), F: AddNoBuiltinAttr);
2277}
2278
2279static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
2280 const llvm::DataLayout &DL, const ABIArgInfo &AI,
2281 bool CheckCoerce = true) {
2282 llvm::Type *Ty = Types.ConvertTypeForMem(T: QTy);
2283 if (AI.getKind() == ABIArgInfo::Indirect ||
2284 AI.getKind() == ABIArgInfo::IndirectAliased)
2285 return true;
2286 if (AI.getKind() == ABIArgInfo::Extend && !AI.isNoExt())
2287 return true;
2288 if (!DL.typeSizeEqualsStoreSize(Ty))
2289 // TODO: This will result in a modest amount of values not marked noundef
2290 // when they could be. We care about values that *invisibly* contain undef
2291 // bits from the perspective of LLVM IR.
2292 return false;
2293 if (CheckCoerce && AI.canHaveCoerceToType()) {
2294 llvm::Type *CoerceTy = AI.getCoerceToType();
2295 if (llvm::TypeSize::isKnownGT(LHS: DL.getTypeSizeInBits(Ty: CoerceTy),
2296 RHS: DL.getTypeSizeInBits(Ty)))
2297 // If we're coercing to a type with a greater size than the canonical one,
2298 // we're introducing new undef bits.
2299 // Coercing to a type of smaller or equal size is ok, as we know that
2300 // there's no internal padding (typeSizeEqualsStoreSize).
2301 return false;
2302 }
2303 if (QTy->isBitIntType())
2304 return true;
2305 if (QTy->isReferenceType())
2306 return true;
2307 if (QTy->isNullPtrType())
2308 return false;
2309 if (QTy->isMemberPointerType())
2310 // TODO: Some member pointers are `noundef`, but it depends on the ABI. For
2311 // now, never mark them.
2312 return false;
2313 if (QTy->isScalarType()) {
2314 if (const ComplexType *Complex = dyn_cast<ComplexType>(Val&: QTy))
2315 return DetermineNoUndef(QTy: Complex->getElementType(), Types, DL, AI, CheckCoerce: false);
2316 return true;
2317 }
2318 if (const VectorType *Vector = dyn_cast<VectorType>(Val&: QTy))
2319 return DetermineNoUndef(QTy: Vector->getElementType(), Types, DL, AI, CheckCoerce: false);
2320 if (const MatrixType *Matrix = dyn_cast<MatrixType>(Val&: QTy))
2321 return DetermineNoUndef(QTy: Matrix->getElementType(), Types, DL, AI, CheckCoerce: false);
2322 if (const ArrayType *Array = dyn_cast<ArrayType>(Val&: QTy))
2323 return DetermineNoUndef(QTy: Array->getElementType(), Types, DL, AI, CheckCoerce: false);
2324
2325 // TODO: Some structs may be `noundef`, in specific situations.
2326 return false;
2327}
2328
2329/// Check if the argument of a function has maybe_undef attribute.
2330static bool IsArgumentMaybeUndef(const Decl *TargetDecl,
2331 unsigned NumRequiredArgs, unsigned ArgNo) {
2332 const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
2333 if (!FD)
2334 return false;
2335
2336 // Assume variadic arguments do not have maybe_undef attribute.
2337 if (ArgNo >= NumRequiredArgs)
2338 return false;
2339
2340 // Check if argument has maybe_undef attribute.
2341 if (ArgNo < FD->getNumParams()) {
2342 const ParmVarDecl *Param = FD->getParamDecl(i: ArgNo);
2343 if (Param && Param->hasAttr<MaybeUndefAttr>())
2344 return true;
2345 }
2346
2347 return false;
2348}
2349
2350/// Test if it's legal to apply nofpclass for the given parameter type and it's
2351/// lowered IR type.
2352static bool canApplyNoFPClass(const ABIArgInfo &AI, QualType ParamType,
2353 bool IsReturn) {
2354 // Should only apply to FP types in the source, not ABI promoted.
2355 if (!ParamType->hasFloatingRepresentation())
2356 return false;
2357
2358 // The promoted-to IR type also needs to support nofpclass.
2359 llvm::Type *IRTy = AI.getCoerceToType();
2360 if (llvm::AttributeFuncs::isNoFPClassCompatibleType(Ty: IRTy))
2361 return true;
2362
2363 if (llvm::StructType *ST = dyn_cast<llvm::StructType>(Val: IRTy)) {
2364 return !IsReturn && AI.getCanBeFlattened() &&
2365 llvm::all_of(Range: ST->elements(),
2366 P: llvm::AttributeFuncs::isNoFPClassCompatibleType);
2367 }
2368
2369 return false;
2370}
2371
2372/// Return the nofpclass mask that can be applied to floating-point parameters.
2373static llvm::FPClassTest getNoFPClassTestMask(const LangOptions &LangOpts) {
2374 llvm::FPClassTest Mask = llvm::fcNone;
2375 if (LangOpts.NoHonorInfs)
2376 Mask |= llvm::fcInf;
2377 if (LangOpts.NoHonorNaNs)
2378 Mask |= llvm::fcNan;
2379 return Mask;
2380}
2381
2382void CodeGenModule::AdjustMemoryAttribute(StringRef Name,
2383 CGCalleeInfo CalleeInfo,
2384 llvm::AttributeList &Attrs) {
2385 if (Attrs.getMemoryEffects().getModRef() == llvm::ModRefInfo::NoModRef) {
2386 Attrs = Attrs.removeFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::Memory);
2387 llvm::Attribute MemoryAttr = llvm::Attribute::getWithMemoryEffects(
2388 Context&: getLLVMContext(), ME: llvm::MemoryEffects::writeOnly());
2389 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Attr: MemoryAttr);
2390 }
2391}
2392
2393/// Construct the IR attribute list of a function or call.
2394///
2395/// When adding an attribute, please consider where it should be handled:
2396///
2397/// - getDefaultFunctionAttributes is for attributes that are essentially
2398/// part of the global target configuration (but perhaps can be
2399/// overridden on a per-function basis). Adding attributes there
2400/// will cause them to also be set in frontends that build on Clang's
2401/// target-configuration logic, as well as for code defined in library
2402/// modules such as CUDA's libdevice.
2403///
2404/// - ConstructAttributeList builds on top of getDefaultFunctionAttributes
2405/// and adds declaration-specific, convention-specific, and
2406/// frontend-specific logic. The last is of particular importance:
2407/// attributes that restrict how the frontend generates code must be
2408/// added here rather than getDefaultFunctionAttributes.
2409///
2410void CodeGenModule::ConstructAttributeList(StringRef Name,
2411 const CGFunctionInfo &FI,
2412 CGCalleeInfo CalleeInfo,
2413 llvm::AttributeList &AttrList,
2414 unsigned &CallingConv,
2415 bool AttrOnCallSite, bool IsThunk) {
2416 llvm::AttrBuilder FuncAttrs(getLLVMContext());
2417 llvm::AttrBuilder RetAttrs(getLLVMContext());
2418
2419 // Collect function IR attributes from the CC lowering.
2420 // We'll collect the paramete and result attributes later.
2421 CallingConv = FI.getEffectiveCallingConvention();
2422 if (FI.isNoReturn())
2423 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2424 if (FI.isCmseNSCall())
2425 FuncAttrs.addAttribute(A: "cmse_nonsecure_call");
2426
2427 // Collect function IR attributes from the callee prototype if we have one.
2428 AddAttributesFromFunctionProtoType(Ctx&: getContext(), FuncAttrs,
2429 FPT: CalleeInfo.getCalleeFunctionProtoType());
2430 const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2431
2432 // Attach assumption attributes to the declaration. If this is a call
2433 // site, attach assumptions from the caller to the call as well.
2434 AddAttributesFromOMPAssumes(FuncAttrs, Callee: TargetDecl);
2435
2436 bool HasOptnone = false;
2437 // The NoBuiltinAttr attached to the target FunctionDecl.
2438 const NoBuiltinAttr *NBA = nullptr;
2439
2440 // Some ABIs may result in additional accesses to arguments that may
2441 // otherwise not be present.
2442 auto AddPotentialArgAccess = [&]() {
2443 llvm::Attribute A = FuncAttrs.getAttribute(Kind: llvm::Attribute::Memory);
2444 if (A.isValid())
2445 FuncAttrs.addMemoryAttr(ME: A.getMemoryEffects() |
2446 llvm::MemoryEffects::argMemOnly());
2447 };
2448
2449 // Collect function IR attributes based on declaration-specific
2450 // information.
2451 // FIXME: handle sseregparm someday...
2452 if (TargetDecl) {
2453 if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2454 FuncAttrs.addAttribute(Val: llvm::Attribute::ReturnsTwice);
2455 if (TargetDecl->hasAttr<NoThrowAttr>())
2456 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2457 if (TargetDecl->hasAttr<NoReturnAttr>())
2458 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2459 if (TargetDecl->hasAttr<ColdAttr>())
2460 FuncAttrs.addAttribute(Val: llvm::Attribute::Cold);
2461 if (TargetDecl->hasAttr<HotAttr>())
2462 FuncAttrs.addAttribute(Val: llvm::Attribute::Hot);
2463 if (TargetDecl->hasAttr<NoDuplicateAttr>())
2464 FuncAttrs.addAttribute(Val: llvm::Attribute::NoDuplicate);
2465 if (TargetDecl->hasAttr<ConvergentAttr>())
2466 FuncAttrs.addAttribute(Val: llvm::Attribute::Convergent);
2467
2468 if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2469 AddAttributesFromFunctionProtoType(
2470 Ctx&: getContext(), FuncAttrs, FPT: Fn->getType()->getAs<FunctionProtoType>());
2471 if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
2472 // A sane operator new returns a non-aliasing pointer.
2473 auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2474 if (getCodeGenOpts().AssumeSaneOperatorNew &&
2475 (Kind == OO_New || Kind == OO_Array_New))
2476 RetAttrs.addAttribute(Val: llvm::Attribute::NoAlias);
2477 }
2478 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: Fn);
2479 const bool IsVirtualCall = MD && MD->isVirtual();
2480 // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2481 // virtual function. These attributes are not inherited by overloads.
2482 if (!(AttrOnCallSite && IsVirtualCall)) {
2483 if (Fn->isNoReturn())
2484 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2485 NBA = Fn->getAttr<NoBuiltinAttr>();
2486 }
2487 }
2488
2489 if (isa<FunctionDecl>(Val: TargetDecl) || isa<VarDecl>(Val: TargetDecl)) {
2490 // Only place nomerge attribute on call sites, never functions. This
2491 // allows it to work on indirect virtual function calls.
2492 if (AttrOnCallSite && TargetDecl->hasAttr<NoMergeAttr>())
2493 FuncAttrs.addAttribute(Val: llvm::Attribute::NoMerge);
2494 }
2495
2496 // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2497 if (TargetDecl->hasAttr<ConstAttr>()) {
2498 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::none());
2499 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2500 // gcc specifies that 'const' functions have greater restrictions than
2501 // 'pure' functions, so they also cannot have infinite loops.
2502 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
2503 } else if (TargetDecl->hasAttr<PureAttr>()) {
2504 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::readOnly());
2505 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2506 // gcc specifies that 'pure' functions cannot have infinite loops.
2507 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
2508 } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2509 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::inaccessibleOrArgMemOnly());
2510 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2511 }
2512 if (const auto *RA = TargetDecl->getAttr<RestrictAttr>();
2513 RA && RA->getDeallocator() == nullptr)
2514 RetAttrs.addAttribute(Val: llvm::Attribute::NoAlias);
2515 if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2516 !CodeGenOpts.NullPointerIsValid)
2517 RetAttrs.addAttribute(Val: llvm::Attribute::NonNull);
2518 if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2519 FuncAttrs.addAttribute(A: "no_caller_saved_registers");
2520 if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2521 FuncAttrs.addAttribute(Val: llvm::Attribute::NoCfCheck);
2522 if (TargetDecl->hasAttr<LeafAttr>())
2523 FuncAttrs.addAttribute(Val: llvm::Attribute::NoCallback);
2524 if (TargetDecl->hasAttr<BPFFastCallAttr>())
2525 FuncAttrs.addAttribute(A: "bpf_fastcall");
2526
2527 HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2528 if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2529 std::optional<unsigned> NumElemsParam;
2530 if (AllocSize->getNumElemsParam().isValid())
2531 NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2532 FuncAttrs.addAllocSizeAttr(ElemSizeArg: AllocSize->getElemSizeParam().getLLVMIndex(),
2533 NumElemsArg: NumElemsParam);
2534 }
2535
2536 if (DeviceKernelAttr::isOpenCLSpelling(
2537 A: TargetDecl->getAttr<DeviceKernelAttr>()) &&
2538 CallingConv != CallingConv::CC_C &&
2539 CallingConv != CallingConv::CC_SpirFunction) {
2540 // Check CallingConv to avoid adding uniform-work-group-size attribute to
2541 // OpenCL Kernel Stub
2542 if (getLangOpts().OpenCLVersion <= 120) {
2543 // OpenCL v1.2 Work groups are always uniform
2544 FuncAttrs.addAttribute(A: "uniform-work-group-size", V: "true");
2545 } else {
2546 // OpenCL v2.0 Work groups may be whether uniform or not.
2547 // '-cl-uniform-work-group-size' compile option gets a hint
2548 // to the compiler that the global work-size be a multiple of
2549 // the work-group size specified to clEnqueueNDRangeKernel
2550 // (i.e. work groups are uniform).
2551 FuncAttrs.addAttribute(
2552 A: "uniform-work-group-size",
2553 V: llvm::toStringRef(B: getLangOpts().OffloadUniformBlock));
2554 }
2555 }
2556
2557 if (TargetDecl->hasAttr<CUDAGlobalAttr>() &&
2558 getLangOpts().OffloadUniformBlock)
2559 FuncAttrs.addAttribute(A: "uniform-work-group-size", V: "true");
2560
2561 if (TargetDecl->hasAttr<ArmLocallyStreamingAttr>())
2562 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_body");
2563 }
2564
2565 // Attach "no-builtins" attributes to:
2566 // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2567 // * definitions: "no-builtins" or "no-builtin-<name>" only.
2568 // The attributes can come from:
2569 // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2570 // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2571 addNoBuiltinAttributes(FuncAttrs, LangOpts: getLangOpts(), NBA);
2572
2573 // Collect function IR attributes based on global settiings.
2574 getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2575
2576 // Override some default IR attributes based on declaration-specific
2577 // information.
2578 if (TargetDecl) {
2579 if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2580 FuncAttrs.removeAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2581 if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2582 FuncAttrs.addAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2583 if (TargetDecl->hasAttr<NoSplitStackAttr>())
2584 FuncAttrs.removeAttribute(A: "split-stack");
2585 if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
2586 // A function "__attribute__((...))" overrides the command-line flag.
2587 auto Kind =
2588 TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2589 FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2590 FuncAttrs.addAttribute(
2591 A: "zero-call-used-regs",
2592 V: ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Val: Kind));
2593 }
2594
2595 // Add NonLazyBind attribute to function declarations when -fno-plt
2596 // is used.
2597 // FIXME: what if we just haven't processed the function definition
2598 // yet, or if it's an external definition like C99 inline?
2599 if (CodeGenOpts.NoPLT) {
2600 if (auto *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2601 if (!Fn->isDefined() && !AttrOnCallSite) {
2602 FuncAttrs.addAttribute(Val: llvm::Attribute::NonLazyBind);
2603 }
2604 }
2605 }
2606 // Remove 'convergent' if requested.
2607 if (TargetDecl->hasAttr<NoConvergentAttr>())
2608 FuncAttrs.removeAttribute(Val: llvm::Attribute::Convergent);
2609 }
2610
2611 // Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2612 // functions with -funique-internal-linkage-names.
2613 if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames) {
2614 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
2615 if (!FD->isExternallyVisible())
2616 FuncAttrs.addAttribute(A: "sample-profile-suffix-elision-policy",
2617 V: "selected");
2618 }
2619 }
2620
2621 // Collect non-call-site function IR attributes from declaration-specific
2622 // information.
2623 if (!AttrOnCallSite) {
2624 if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>())
2625 FuncAttrs.addAttribute(A: "cmse_nonsecure_entry");
2626
2627 // Whether tail calls are enabled.
2628 auto shouldDisableTailCalls = [&] {
2629 // Should this be honored in getDefaultFunctionAttributes?
2630 if (CodeGenOpts.DisableTailCalls)
2631 return true;
2632
2633 if (!TargetDecl)
2634 return false;
2635
2636 if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2637 TargetDecl->hasAttr<AnyX86InterruptAttr>())
2638 return true;
2639
2640 if (CodeGenOpts.NoEscapingBlockTailCalls) {
2641 if (const auto *BD = dyn_cast<BlockDecl>(Val: TargetDecl))
2642 if (!BD->doesNotEscape())
2643 return true;
2644 }
2645
2646 return false;
2647 };
2648 if (shouldDisableTailCalls())
2649 FuncAttrs.addAttribute(A: "disable-tail-calls", V: "true");
2650
2651 // These functions require the returns_twice attribute for correct codegen,
2652 // but the attribute may not be added if -fno-builtin is specified. We
2653 // explicitly add that attribute here.
2654 static const llvm::StringSet<> ReturnsTwiceFn{
2655 "_setjmpex", "setjmp", "_setjmp", "vfork",
2656 "sigsetjmp", "__sigsetjmp", "savectx", "getcontext"};
2657 if (ReturnsTwiceFn.contains(key: Name))
2658 FuncAttrs.addAttribute(Val: llvm::Attribute::ReturnsTwice);
2659
2660 // CPU/feature overrides. addDefaultFunctionDefinitionAttributes
2661 // handles these separately to set them based on the global defaults.
2662 GetCPUAndFeaturesAttributes(GD: CalleeInfo.getCalleeDecl(), AttrBuilder&: FuncAttrs);
2663
2664 // Windows hotpatching support
2665 if (!MSHotPatchFunctions.empty()) {
2666 bool IsHotPatched = llvm::binary_search(Range&: MSHotPatchFunctions, Value&: Name);
2667 if (IsHotPatched)
2668 FuncAttrs.addAttribute(A: "marked_for_windows_hot_patching");
2669 }
2670 }
2671
2672 // Mark functions that are replaceable by the loader.
2673 if (CodeGenOpts.isLoaderReplaceableFunctionName(FuncName: Name))
2674 FuncAttrs.addAttribute(A: "loader-replaceable");
2675
2676 // Collect attributes from arguments and return values.
2677 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2678
2679 QualType RetTy = FI.getReturnType();
2680 const ABIArgInfo &RetAI = FI.getReturnInfo();
2681 const llvm::DataLayout &DL = getDataLayout();
2682
2683 // Determine if the return type could be partially undef
2684 if (CodeGenOpts.EnableNoundefAttrs &&
2685 HasStrictReturn(Module: *this, RetTy, TargetDecl)) {
2686 if (!RetTy->isVoidType() && RetAI.getKind() != ABIArgInfo::Indirect &&
2687 DetermineNoUndef(QTy: RetTy, Types&: getTypes(), DL, AI: RetAI))
2688 RetAttrs.addAttribute(Val: llvm::Attribute::NoUndef);
2689 }
2690
2691 switch (RetAI.getKind()) {
2692 case ABIArgInfo::Extend:
2693 if (RetAI.isSignExt())
2694 RetAttrs.addAttribute(Val: llvm::Attribute::SExt);
2695 else if (RetAI.isZeroExt())
2696 RetAttrs.addAttribute(Val: llvm::Attribute::ZExt);
2697 else
2698 RetAttrs.addAttribute(Val: llvm::Attribute::NoExt);
2699 [[fallthrough]];
2700 case ABIArgInfo::Direct:
2701 if (RetAI.getInReg())
2702 RetAttrs.addAttribute(Val: llvm::Attribute::InReg);
2703
2704 if (canApplyNoFPClass(AI: RetAI, ParamType: RetTy, IsReturn: true))
2705 RetAttrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
2706
2707 break;
2708 case ABIArgInfo::Ignore:
2709 break;
2710
2711 case ABIArgInfo::InAlloca:
2712 case ABIArgInfo::Indirect: {
2713 // inalloca and sret disable readnone and readonly
2714 AddPotentialArgAccess();
2715 break;
2716 }
2717
2718 case ABIArgInfo::CoerceAndExpand:
2719 break;
2720
2721 case ABIArgInfo::Expand:
2722 case ABIArgInfo::IndirectAliased:
2723 llvm_unreachable("Invalid ABI kind for return argument");
2724 }
2725
2726 if (!IsThunk) {
2727 // FIXME: fix this properly, https://reviews.llvm.org/D100388
2728 if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2729 QualType PTy = RefTy->getPointeeType();
2730 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2731 RetAttrs.addDereferenceableAttr(
2732 Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
2733 if (getTypes().getTargetAddressSpace(T: PTy) == 0 &&
2734 !CodeGenOpts.NullPointerIsValid)
2735 RetAttrs.addAttribute(Val: llvm::Attribute::NonNull);
2736 if (PTy->isObjectType()) {
2737 llvm::Align Alignment =
2738 getNaturalPointeeTypeAlignment(T: RetTy).getAsAlign();
2739 RetAttrs.addAlignmentAttr(Align: Alignment);
2740 }
2741 }
2742 }
2743
2744 bool hasUsedSRet = false;
2745 SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
2746
2747 // Attach attributes to sret.
2748 if (IRFunctionArgs.hasSRetArg()) {
2749 llvm::AttrBuilder SRETAttrs(getLLVMContext());
2750 SRETAttrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: RetTy));
2751 SRETAttrs.addAttribute(Val: llvm::Attribute::Writable);
2752 SRETAttrs.addAttribute(Val: llvm::Attribute::DeadOnUnwind);
2753 hasUsedSRet = true;
2754 if (RetAI.getInReg())
2755 SRETAttrs.addAttribute(Val: llvm::Attribute::InReg);
2756 SRETAttrs.addAlignmentAttr(Align: RetAI.getIndirectAlign().getQuantity());
2757 ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2758 llvm::AttributeSet::get(C&: getLLVMContext(), B: SRETAttrs);
2759 }
2760
2761 // Attach attributes to inalloca argument.
2762 if (IRFunctionArgs.hasInallocaArg()) {
2763 llvm::AttrBuilder Attrs(getLLVMContext());
2764 Attrs.addInAllocaAttr(Ty: FI.getArgStruct());
2765 ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2766 llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs);
2767 }
2768
2769 // Apply `nonnull`, `dereferenceable(N)` and `align N` to the `this` argument,
2770 // unless this is a thunk function.
2771 // FIXME: fix this properly, https://reviews.llvm.org/D100388
2772 if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() &&
2773 !FI.arg_begin()->type->isVoidPointerType() && !IsThunk) {
2774 auto IRArgs = IRFunctionArgs.getIRArgs(ArgNo: 0);
2775
2776 assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
2777
2778 llvm::AttrBuilder Attrs(getLLVMContext());
2779
2780 QualType ThisTy = FI.arg_begin()->type.getTypePtr()->getPointeeType();
2781
2782 if (!CodeGenOpts.NullPointerIsValid &&
2783 getTypes().getTargetAddressSpace(T: FI.arg_begin()->type) == 0) {
2784 Attrs.addAttribute(Val: llvm::Attribute::NonNull);
2785 Attrs.addDereferenceableAttr(Bytes: getMinimumObjectSize(Ty: ThisTy).getQuantity());
2786 } else {
2787 // FIXME dereferenceable should be correct here, regardless of
2788 // NullPointerIsValid. However, dereferenceable currently does not always
2789 // respect NullPointerIsValid and may imply nonnull and break the program.
2790 // See https://reviews.llvm.org/D66618 for discussions.
2791 Attrs.addDereferenceableOrNullAttr(
2792 Bytes: getMinimumObjectSize(
2793 Ty: FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
2794 .getQuantity());
2795 }
2796
2797 llvm::Align Alignment =
2798 getNaturalTypeAlignment(T: ThisTy, /*BaseInfo=*/nullptr,
2799 /*TBAAInfo=*/nullptr, /*forPointeeType=*/true)
2800 .getAsAlign();
2801 Attrs.addAlignmentAttr(Align: Alignment);
2802
2803 ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs);
2804 }
2805
2806 unsigned ArgNo = 0;
2807 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(), E = FI.arg_end();
2808 I != E; ++I, ++ArgNo) {
2809 QualType ParamType = I->type;
2810 const ABIArgInfo &AI = I->info;
2811 llvm::AttrBuilder Attrs(getLLVMContext());
2812
2813 // Add attribute for padding argument, if necessary.
2814 if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2815 if (AI.getPaddingInReg()) {
2816 ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2817 llvm::AttributeSet::get(C&: getLLVMContext(),
2818 B: llvm::AttrBuilder(getLLVMContext())
2819 .addAttribute(Val: llvm::Attribute::InReg));
2820 }
2821 }
2822
2823 // Decide whether the argument we're handling could be partially undef
2824 if (CodeGenOpts.EnableNoundefAttrs &&
2825 DetermineNoUndef(QTy: ParamType, Types&: getTypes(), DL, AI)) {
2826 Attrs.addAttribute(Val: llvm::Attribute::NoUndef);
2827 }
2828
2829 // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2830 // have the corresponding parameter variable. It doesn't make
2831 // sense to do it here because parameters are so messed up.
2832 switch (AI.getKind()) {
2833 case ABIArgInfo::Extend:
2834 if (AI.isSignExt())
2835 Attrs.addAttribute(Val: llvm::Attribute::SExt);
2836 else if (AI.isZeroExt())
2837 Attrs.addAttribute(Val: llvm::Attribute::ZExt);
2838 else
2839 Attrs.addAttribute(Val: llvm::Attribute::NoExt);
2840 [[fallthrough]];
2841 case ABIArgInfo::Direct:
2842 if (ArgNo == 0 && FI.isChainCall())
2843 Attrs.addAttribute(Val: llvm::Attribute::Nest);
2844 else if (AI.getInReg())
2845 Attrs.addAttribute(Val: llvm::Attribute::InReg);
2846 Attrs.addStackAlignmentAttr(Align: llvm::MaybeAlign(AI.getDirectAlign()));
2847
2848 if (canApplyNoFPClass(AI, ParamType, IsReturn: false))
2849 Attrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
2850 break;
2851 case ABIArgInfo::Indirect: {
2852 if (AI.getInReg())
2853 Attrs.addAttribute(Val: llvm::Attribute::InReg);
2854
2855 if (AI.getIndirectByVal())
2856 Attrs.addByValAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
2857
2858 auto *Decl = ParamType->getAsRecordDecl();
2859 if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
2860 Decl->getArgPassingRestrictions() ==
2861 RecordArgPassingKind::CanPassInRegs)
2862 // When calling the function, the pointer passed in will be the only
2863 // reference to the underlying object. Mark it accordingly.
2864 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
2865
2866 // TODO: We could add the byref attribute if not byval, but it would
2867 // require updating many testcases.
2868
2869 CharUnits Align = AI.getIndirectAlign();
2870
2871 // In a byval argument, it is important that the required
2872 // alignment of the type is honored, as LLVM might be creating a
2873 // *new* stack object, and needs to know what alignment to give
2874 // it. (Sometimes it can deduce a sensible alignment on its own,
2875 // but not if clang decides it must emit a packed struct, or the
2876 // user specifies increased alignment requirements.)
2877 //
2878 // This is different from indirect *not* byval, where the object
2879 // exists already, and the align attribute is purely
2880 // informative.
2881 assert(!Align.isZero());
2882
2883 // For now, only add this when we have a byval argument.
2884 // TODO: be less lazy about updating test cases.
2885 if (AI.getIndirectByVal())
2886 Attrs.addAlignmentAttr(Align: Align.getQuantity());
2887
2888 // byval disables readnone and readonly.
2889 AddPotentialArgAccess();
2890 break;
2891 }
2892 case ABIArgInfo::IndirectAliased: {
2893 CharUnits Align = AI.getIndirectAlign();
2894 Attrs.addByRefAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
2895 Attrs.addAlignmentAttr(Align: Align.getQuantity());
2896 break;
2897 }
2898 case ABIArgInfo::Ignore:
2899 case ABIArgInfo::Expand:
2900 case ABIArgInfo::CoerceAndExpand:
2901 break;
2902
2903 case ABIArgInfo::InAlloca:
2904 // inalloca disables readnone and readonly.
2905 AddPotentialArgAccess();
2906 continue;
2907 }
2908
2909 if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
2910 QualType PTy = RefTy->getPointeeType();
2911 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2912 Attrs.addDereferenceableAttr(Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
2913 if (getTypes().getTargetAddressSpace(T: PTy) == 0 &&
2914 !CodeGenOpts.NullPointerIsValid)
2915 Attrs.addAttribute(Val: llvm::Attribute::NonNull);
2916 if (PTy->isObjectType()) {
2917 llvm::Align Alignment =
2918 getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
2919 Attrs.addAlignmentAttr(Align: Alignment);
2920 }
2921 }
2922
2923 // From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:
2924 // > For arguments to a __kernel function declared to be a pointer to a
2925 // > data type, the OpenCL compiler can assume that the pointee is always
2926 // > appropriately aligned as required by the data type.
2927 if (TargetDecl &&
2928 DeviceKernelAttr::isOpenCLSpelling(
2929 A: TargetDecl->getAttr<DeviceKernelAttr>()) &&
2930 ParamType->isPointerType()) {
2931 QualType PTy = ParamType->getPointeeType();
2932 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2933 llvm::Align Alignment =
2934 getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
2935 Attrs.addAlignmentAttr(Align: Alignment);
2936 }
2937 }
2938
2939 switch (FI.getExtParameterInfo(argIndex: ArgNo).getABI()) {
2940 case ParameterABI::HLSLOut:
2941 case ParameterABI::HLSLInOut:
2942 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
2943 break;
2944 case ParameterABI::Ordinary:
2945 break;
2946
2947 case ParameterABI::SwiftIndirectResult: {
2948 // Add 'sret' if we haven't already used it for something, but
2949 // only if the result is void.
2950 if (!hasUsedSRet && RetTy->isVoidType()) {
2951 Attrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
2952 hasUsedSRet = true;
2953 }
2954
2955 // Add 'noalias' in either case.
2956 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
2957
2958 // Add 'dereferenceable' and 'alignment'.
2959 auto PTy = ParamType->getPointeeType();
2960 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2961 auto info = getContext().getTypeInfoInChars(T: PTy);
2962 Attrs.addDereferenceableAttr(Bytes: info.Width.getQuantity());
2963 Attrs.addAlignmentAttr(Align: info.Align.getAsAlign());
2964 }
2965 break;
2966 }
2967
2968 case ParameterABI::SwiftErrorResult:
2969 Attrs.addAttribute(Val: llvm::Attribute::SwiftError);
2970 break;
2971
2972 case ParameterABI::SwiftContext:
2973 Attrs.addAttribute(Val: llvm::Attribute::SwiftSelf);
2974 break;
2975
2976 case ParameterABI::SwiftAsyncContext:
2977 Attrs.addAttribute(Val: llvm::Attribute::SwiftAsync);
2978 break;
2979 }
2980
2981 if (FI.getExtParameterInfo(argIndex: ArgNo).isNoEscape())
2982 Attrs.addCapturesAttr(CI: llvm::CaptureInfo::none());
2983
2984 if (Attrs.hasAttributes()) {
2985 unsigned FirstIRArg, NumIRArgs;
2986 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2987 for (unsigned i = 0; i < NumIRArgs; i++)
2988 ArgAttrs[FirstIRArg + i] = ArgAttrs[FirstIRArg + i].addAttributes(
2989 C&: getLLVMContext(), AS: llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs));
2990 }
2991 }
2992 assert(ArgNo == FI.arg_size());
2993
2994 AttrList = llvm::AttributeList::get(
2995 C&: getLLVMContext(), FnAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: FuncAttrs),
2996 RetAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: RetAttrs), ArgAttrs);
2997}
2998
2999/// An argument came in as a promoted argument; demote it back to its
3000/// declared type.
3001static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
3002 const VarDecl *var,
3003 llvm::Value *value) {
3004 llvm::Type *varType = CGF.ConvertType(T: var->getType());
3005
3006 // This can happen with promotions that actually don't change the
3007 // underlying type, like the enum promotions.
3008 if (value->getType() == varType)
3009 return value;
3010
3011 assert((varType->isIntegerTy() || varType->isFloatingPointTy()) &&
3012 "unexpected promotion type");
3013
3014 if (isa<llvm::IntegerType>(Val: varType))
3015 return CGF.Builder.CreateTrunc(V: value, DestTy: varType, Name: "arg.unpromote");
3016
3017 return CGF.Builder.CreateFPCast(V: value, DestTy: varType, Name: "arg.unpromote");
3018}
3019
3020/// Returns the attribute (either parameter attribute, or function
3021/// attribute), which declares argument ArgNo to be non-null.
3022static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
3023 QualType ArgType, unsigned ArgNo) {
3024 // FIXME: __attribute__((nonnull)) can also be applied to:
3025 // - references to pointers, where the pointee is known to be
3026 // nonnull (apparently a Clang extension)
3027 // - transparent unions containing pointers
3028 // In the former case, LLVM IR cannot represent the constraint. In
3029 // the latter case, we have no guarantee that the transparent union
3030 // is in fact passed as a pointer.
3031 if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType())
3032 return nullptr;
3033 // First, check attribute on parameter itself.
3034 if (PVD) {
3035 if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
3036 return ParmNNAttr;
3037 }
3038 // Check function attributes.
3039 if (!FD)
3040 return nullptr;
3041 for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
3042 if (NNAttr->isNonNull(IdxAST: ArgNo))
3043 return NNAttr;
3044 }
3045 return nullptr;
3046}
3047
3048namespace {
3049struct CopyBackSwiftError final : EHScopeStack::Cleanup {
3050 Address Temp;
3051 Address Arg;
3052 CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
3053 void Emit(CodeGenFunction &CGF, Flags flags) override {
3054 llvm::Value *errorValue = CGF.Builder.CreateLoad(Addr: Temp);
3055 CGF.Builder.CreateStore(Val: errorValue, Addr: Arg);
3056 }
3057};
3058} // namespace
3059
3060void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
3061 llvm::Function *Fn,
3062 const FunctionArgList &Args) {
3063 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())
3064 // Naked functions don't have prologues.
3065 return;
3066
3067 // If this is an implicit-return-zero function, go ahead and
3068 // initialize the return value. TODO: it might be nice to have
3069 // a more general mechanism for this that didn't require synthesized
3070 // return statements.
3071 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl)) {
3072 if (FD->hasImplicitReturnZero()) {
3073 QualType RetTy = FD->getReturnType().getUnqualifiedType();
3074 llvm::Type *LLVMTy = CGM.getTypes().ConvertType(T: RetTy);
3075 llvm::Constant *Zero = llvm::Constant::getNullValue(Ty: LLVMTy);
3076 Builder.CreateStore(Val: Zero, Addr: ReturnValue);
3077 }
3078 }
3079
3080 // FIXME: We no longer need the types from FunctionArgList; lift up and
3081 // simplify.
3082
3083 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
3084 assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
3085
3086 // If we're using inalloca, all the memory arguments are GEPs off of the last
3087 // parameter, which is a pointer to the complete memory area.
3088 Address ArgStruct = Address::invalid();
3089 if (IRFunctionArgs.hasInallocaArg())
3090 ArgStruct = Address(Fn->getArg(i: IRFunctionArgs.getInallocaArgNo()),
3091 FI.getArgStruct(), FI.getArgStructAlignment());
3092
3093 // Name the struct return parameter.
3094 if (IRFunctionArgs.hasSRetArg()) {
3095 auto AI = Fn->getArg(i: IRFunctionArgs.getSRetArgNo());
3096 AI->setName("agg.result");
3097 AI->addAttr(Kind: llvm::Attribute::NoAlias);
3098 }
3099
3100 // Track if we received the parameter as a pointer (indirect, byval, or
3101 // inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it
3102 // into a local alloca for us.
3103 SmallVector<ParamValue, 16> ArgVals;
3104 ArgVals.reserve(N: Args.size());
3105
3106 // Create a pointer value for every parameter declaration. This usually
3107 // entails copying one or more LLVM IR arguments into an alloca. Don't push
3108 // any cleanups or do anything that might unwind. We do that separately, so
3109 // we can push the cleanups in the correct order for the ABI.
3110 assert(FI.arg_size() == Args.size() &&
3111 "Mismatch between function signature & arguments.");
3112 unsigned ArgNo = 0;
3113 CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
3114 for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end(); i != e;
3115 ++i, ++info_it, ++ArgNo) {
3116 const VarDecl *Arg = *i;
3117 const ABIArgInfo &ArgI = info_it->info;
3118
3119 bool isPromoted =
3120 isa<ParmVarDecl>(Val: Arg) && cast<ParmVarDecl>(Val: Arg)->isKNRPromoted();
3121 // We are converting from ABIArgInfo type to VarDecl type directly, unless
3122 // the parameter is promoted. In this case we convert to
3123 // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
3124 QualType Ty = isPromoted ? info_it->type : Arg->getType();
3125 assert(hasScalarEvaluationKind(Ty) ==
3126 hasScalarEvaluationKind(Arg->getType()));
3127
3128 unsigned FirstIRArg, NumIRArgs;
3129 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3130
3131 switch (ArgI.getKind()) {
3132 case ABIArgInfo::InAlloca: {
3133 assert(NumIRArgs == 0);
3134 auto FieldIndex = ArgI.getInAllocaFieldIndex();
3135 Address V =
3136 Builder.CreateStructGEP(Addr: ArgStruct, Index: FieldIndex, Name: Arg->getName());
3137 if (ArgI.getInAllocaIndirect())
3138 V = Address(Builder.CreateLoad(Addr: V), ConvertTypeForMem(T: Ty),
3139 getContext().getTypeAlignInChars(T: Ty));
3140 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: V));
3141 break;
3142 }
3143
3144 case ABIArgInfo::Indirect:
3145 case ABIArgInfo::IndirectAliased: {
3146 assert(NumIRArgs == 1);
3147 Address ParamAddr = makeNaturalAddressForPointer(
3148 Ptr: Fn->getArg(i: FirstIRArg), T: Ty, Alignment: ArgI.getIndirectAlign(), ForPointeeType: false, BaseInfo: nullptr,
3149 TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
3150
3151 if (!hasScalarEvaluationKind(T: Ty)) {
3152 // Aggregates and complex variables are accessed by reference. All we
3153 // need to do is realign the value, if requested. Also, if the address
3154 // may be aliased, copy it to ensure that the parameter variable is
3155 // mutable and has a unique adress, as C requires.
3156 if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) {
3157 RawAddress AlignedTemp = CreateMemTemp(T: Ty, Name: "coerce");
3158
3159 // Copy from the incoming argument pointer to the temporary with the
3160 // appropriate alignment.
3161 //
3162 // FIXME: We should have a common utility for generating an aggregate
3163 // copy.
3164 CharUnits Size = getContext().getTypeSizeInChars(T: Ty);
3165 Builder.CreateMemCpy(
3166 Dst: AlignedTemp.getPointer(), DstAlign: AlignedTemp.getAlignment().getAsAlign(),
3167 Src: ParamAddr.emitRawPointer(CGF&: *this),
3168 SrcAlign: ParamAddr.getAlignment().getAsAlign(),
3169 Size: llvm::ConstantInt::get(Ty: IntPtrTy, V: Size.getQuantity()));
3170 ParamAddr = AlignedTemp;
3171 }
3172 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: ParamAddr));
3173 } else {
3174 // Load scalar value from indirect argument.
3175 llvm::Value *V =
3176 EmitLoadOfScalar(Addr: ParamAddr, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3177
3178 if (isPromoted)
3179 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3180 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3181 }
3182 break;
3183 }
3184
3185 case ABIArgInfo::Extend:
3186 case ABIArgInfo::Direct: {
3187 auto AI = Fn->getArg(i: FirstIRArg);
3188 llvm::Type *LTy = ConvertType(T: Arg->getType());
3189
3190 // Prepare parameter attributes. So far, only attributes for pointer
3191 // parameters are prepared. See
3192 // http://llvm.org/docs/LangRef.html#paramattrs.
3193 if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy() &&
3194 ArgI.getCoerceToType()->isPointerTy()) {
3195 assert(NumIRArgs == 1);
3196
3197 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: Arg)) {
3198 // Set `nonnull` attribute if any.
3199 if (getNonNullAttr(FD: CurCodeDecl, PVD, ArgType: PVD->getType(),
3200 ArgNo: PVD->getFunctionScopeIndex()) &&
3201 !CGM.getCodeGenOpts().NullPointerIsValid)
3202 AI->addAttr(Kind: llvm::Attribute::NonNull);
3203
3204 QualType OTy = PVD->getOriginalType();
3205 if (const auto *ArrTy = getContext().getAsConstantArrayType(T: OTy)) {
3206 // A C99 array parameter declaration with the static keyword also
3207 // indicates dereferenceability, and if the size is constant we can
3208 // use the dereferenceable attribute (which requires the size in
3209 // bytes).
3210 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3211 QualType ETy = ArrTy->getElementType();
3212 llvm::Align Alignment =
3213 CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3214 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3215 .addAlignmentAttr(Align: Alignment));
3216 uint64_t ArrSize = ArrTy->getZExtSize();
3217 if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
3218 ArrSize) {
3219 llvm::AttrBuilder Attrs(getLLVMContext());
3220 Attrs.addDereferenceableAttr(
3221 Bytes: getContext().getTypeSizeInChars(T: ETy).getQuantity() *
3222 ArrSize);
3223 AI->addAttrs(B&: Attrs);
3224 } else if (getContext().getTargetInfo().getNullPointerValue(
3225 AddrSpace: ETy.getAddressSpace()) == 0 &&
3226 !CGM.getCodeGenOpts().NullPointerIsValid) {
3227 AI->addAttr(Kind: llvm::Attribute::NonNull);
3228 }
3229 }
3230 } else if (const auto *ArrTy =
3231 getContext().getAsVariableArrayType(T: OTy)) {
3232 // For C99 VLAs with the static keyword, we don't know the size so
3233 // we can't use the dereferenceable attribute, but in addrspace(0)
3234 // we know that it must be nonnull.
3235 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3236 QualType ETy = ArrTy->getElementType();
3237 llvm::Align Alignment =
3238 CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3239 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3240 .addAlignmentAttr(Align: Alignment));
3241 if (!getTypes().getTargetAddressSpace(T: ETy) &&
3242 !CGM.getCodeGenOpts().NullPointerIsValid)
3243 AI->addAttr(Kind: llvm::Attribute::NonNull);
3244 }
3245 }
3246
3247 // Set `align` attribute if any.
3248 const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
3249 if (!AVAttr)
3250 if (const auto *TOTy = OTy->getAs<TypedefType>())
3251 AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
3252 if (AVAttr && !SanOpts.has(K: SanitizerKind::Alignment)) {
3253 // If alignment-assumption sanitizer is enabled, we do *not* add
3254 // alignment attribute here, but emit normal alignment assumption,
3255 // so the UBSAN check could function.
3256 llvm::ConstantInt *AlignmentCI =
3257 cast<llvm::ConstantInt>(Val: EmitScalarExpr(E: AVAttr->getAlignment()));
3258 uint64_t AlignmentInt =
3259 AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment);
3260 if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
3261 AI->removeAttr(Kind: llvm::Attribute::AttrKind::Alignment);
3262 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3263 .addAlignmentAttr(Align: llvm::Align(AlignmentInt)));
3264 }
3265 }
3266 }
3267
3268 // Set 'noalias' if an argument type has the `restrict` qualifier.
3269 if (Arg->getType().isRestrictQualified())
3270 AI->addAttr(Kind: llvm::Attribute::NoAlias);
3271 }
3272
3273 // Prepare the argument value. If we have the trivial case, handle it
3274 // with no muss and fuss.
3275 if (!isa<llvm::StructType>(Val: ArgI.getCoerceToType()) &&
3276 ArgI.getCoerceToType() == ConvertType(T: Ty) &&
3277 ArgI.getDirectOffset() == 0) {
3278 assert(NumIRArgs == 1);
3279
3280 // LLVM expects swifterror parameters to be used in very restricted
3281 // ways. Copy the value into a less-restricted temporary.
3282 llvm::Value *V = AI;
3283 if (FI.getExtParameterInfo(argIndex: ArgNo).getABI() ==
3284 ParameterABI::SwiftErrorResult) {
3285 QualType pointeeTy = Ty->getPointeeType();
3286 assert(pointeeTy->isPointerType());
3287 RawAddress temp =
3288 CreateMemTemp(T: pointeeTy, Align: getPointerAlign(), Name: "swifterror.temp");
3289 Address arg = makeNaturalAddressForPointer(
3290 Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
3291 llvm::Value *incomingErrorValue = Builder.CreateLoad(Addr: arg);
3292 Builder.CreateStore(Val: incomingErrorValue, Addr: temp);
3293 V = temp.getPointer();
3294
3295 // Push a cleanup to copy the value back at the end of the function.
3296 // The convention does not guarantee that the value will be written
3297 // back if the function exits with an unwind exception.
3298 EHStack.pushCleanup<CopyBackSwiftError>(Kind: NormalCleanup, A: temp, A: arg);
3299 }
3300
3301 // Ensure the argument is the correct type.
3302 if (V->getType() != ArgI.getCoerceToType())
3303 V = Builder.CreateBitCast(V, DestTy: ArgI.getCoerceToType());
3304
3305 if (isPromoted)
3306 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3307
3308 // Because of merging of function types from multiple decls it is
3309 // possible for the type of an argument to not match the corresponding
3310 // type in the function type. Since we are codegening the callee
3311 // in here, add a cast to the argument type.
3312 llvm::Type *LTy = ConvertType(T: Arg->getType());
3313 if (V->getType() != LTy)
3314 V = Builder.CreateBitCast(V, DestTy: LTy);
3315
3316 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3317 break;
3318 }
3319
3320 // VLST arguments are coerced to VLATs at the function boundary for
3321 // ABI consistency. If this is a VLST that was coerced to
3322 // a VLAT at the function boundary and the types match up, use
3323 // llvm.vector.extract to convert back to the original VLST.
3324 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: ConvertType(T: Ty))) {
3325 llvm::Value *ArgVal = Fn->getArg(i: FirstIRArg);
3326 if (auto *VecTyFrom =
3327 dyn_cast<llvm::ScalableVectorType>(Val: ArgVal->getType())) {
3328 auto [Coerced, Extracted] = CoerceScalableToFixed(
3329 CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: ArgVal, Name: Arg->getName());
3330 if (Extracted) {
3331 assert(NumIRArgs == 1);
3332 ArgVals.push_back(Elt: ParamValue::forDirect(value: Coerced));
3333 break;
3334 }
3335 }
3336 }
3337
3338 // Struct of fixed-length vectors and struct of array of fixed-length
3339 // vector in VLS calling convention are coerced to vector tuple
3340 // type(represented as TargetExtType) and scalable vector type
3341 // respectively, they're no longer handled as struct.
3342 if (ArgI.isDirect() && isa<llvm::StructType>(Val: ConvertType(T: Ty)) &&
3343 (isa<llvm::TargetExtType>(Val: ArgI.getCoerceToType()) ||
3344 isa<llvm::ScalableVectorType>(Val: ArgI.getCoerceToType()))) {
3345 ArgVals.push_back(Elt: ParamValue::forDirect(value: AI));
3346 break;
3347 }
3348
3349 llvm::StructType *STy =
3350 dyn_cast<llvm::StructType>(Val: ArgI.getCoerceToType());
3351 Address Alloca =
3352 CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(D: Arg), Name: Arg->getName());
3353
3354 // Pointer to store into.
3355 Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3356
3357 // Fast-isel and the optimizer generally like scalar values better than
3358 // FCAs, so we flatten them if this is safe to do for this argument.
3359 if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
3360 STy->getNumElements() > 1) {
3361 llvm::TypeSize StructSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
3362 llvm::TypeSize PtrElementSize =
3363 CGM.getDataLayout().getTypeAllocSize(Ty: Ptr.getElementType());
3364 if (StructSize.isScalable()) {
3365 assert(STy->containsHomogeneousScalableVectorTypes() &&
3366 "ABI only supports structure with homogeneous scalable vector "
3367 "type");
3368 assert(StructSize == PtrElementSize &&
3369 "Only allow non-fractional movement of structure with"
3370 "homogeneous scalable vector type");
3371 assert(STy->getNumElements() == NumIRArgs);
3372
3373 llvm::Value *LoadedStructValue = llvm::PoisonValue::get(T: STy);
3374 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3375 auto *AI = Fn->getArg(i: FirstIRArg + i);
3376 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3377 LoadedStructValue =
3378 Builder.CreateInsertValue(Agg: LoadedStructValue, Val: AI, Idxs: i);
3379 }
3380
3381 Builder.CreateStore(Val: LoadedStructValue, Addr: Ptr);
3382 } else {
3383 uint64_t SrcSize = StructSize.getFixedValue();
3384 uint64_t DstSize = PtrElementSize.getFixedValue();
3385
3386 Address AddrToStoreInto = Address::invalid();
3387 if (SrcSize <= DstSize) {
3388 AddrToStoreInto = Ptr.withElementType(ElemTy: STy);
3389 } else {
3390 AddrToStoreInto =
3391 CreateTempAlloca(Ty: STy, align: Alloca.getAlignment(), Name: "coerce");
3392 }
3393
3394 assert(STy->getNumElements() == NumIRArgs);
3395 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3396 auto AI = Fn->getArg(i: FirstIRArg + i);
3397 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3398 Address EltPtr = Builder.CreateStructGEP(Addr: AddrToStoreInto, Index: i);
3399 Builder.CreateStore(Val: AI, Addr: EltPtr);
3400 }
3401
3402 if (SrcSize > DstSize) {
3403 Builder.CreateMemCpy(Dest: Ptr, Src: AddrToStoreInto, Size: DstSize);
3404 }
3405 }
3406 } else {
3407 // Simple case, just do a coerced store of the argument into the alloca.
3408 assert(NumIRArgs == 1);
3409 auto AI = Fn->getArg(i: FirstIRArg);
3410 AI->setName(Arg->getName() + ".coerce");
3411 CreateCoercedStore(
3412 Src: AI, Dst: Ptr,
3413 DstSize: llvm::TypeSize::getFixed(
3414 ExactSize: getContext().getTypeSizeInChars(T: Ty).getQuantity() -
3415 ArgI.getDirectOffset()),
3416 /*DstIsVolatile=*/false);
3417 }
3418
3419 // Match to what EmitParmDecl is expecting for this type.
3420 if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3421 llvm::Value *V =
3422 EmitLoadOfScalar(Addr: Alloca, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3423 if (isPromoted)
3424 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3425 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3426 } else {
3427 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3428 }
3429 break;
3430 }
3431
3432 case ABIArgInfo::CoerceAndExpand: {
3433 // Reconstruct into a temporary.
3434 Address alloca = CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(D: Arg));
3435 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: alloca));
3436
3437 auto coercionType = ArgI.getCoerceAndExpandType();
3438 auto unpaddedCoercionType = ArgI.getUnpaddedCoerceAndExpandType();
3439 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
3440
3441 alloca = alloca.withElementType(ElemTy: coercionType);
3442
3443 unsigned argIndex = FirstIRArg;
3444 unsigned unpaddedIndex = 0;
3445 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3446 llvm::Type *eltType = coercionType->getElementType(N: i);
3447 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
3448 continue;
3449
3450 auto eltAddr = Builder.CreateStructGEP(Addr: alloca, Index: i);
3451 llvm::Value *elt = Fn->getArg(i: argIndex++);
3452
3453 auto paramType = unpaddedStruct
3454 ? unpaddedStruct->getElementType(N: unpaddedIndex++)
3455 : unpaddedCoercionType;
3456
3457 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: eltType)) {
3458 if (auto *VecTyFrom = dyn_cast<llvm::ScalableVectorType>(Val: paramType)) {
3459 bool Extracted;
3460 std::tie(args&: elt, args&: Extracted) = CoerceScalableToFixed(
3461 CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: elt, Name: elt->getName());
3462 assert(Extracted && "Unexpected scalable to fixed vector coercion");
3463 }
3464 }
3465 Builder.CreateStore(Val: elt, Addr: eltAddr);
3466 }
3467 assert(argIndex == FirstIRArg + NumIRArgs);
3468 break;
3469 }
3470
3471 case ABIArgInfo::Expand: {
3472 // If this structure was expanded into multiple arguments then
3473 // we need to create a temporary and reconstruct it from the
3474 // arguments.
3475 Address Alloca = CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(D: Arg));
3476 LValue LV = MakeAddrLValue(Addr: Alloca, T: Ty);
3477 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3478
3479 auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3480 ExpandTypeFromArgs(Ty, LV, AI&: FnArgIter);
3481 assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3482 for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
3483 auto AI = Fn->getArg(i: FirstIRArg + i);
3484 AI->setName(Arg->getName() + "." + Twine(i));
3485 }
3486 break;
3487 }
3488
3489 case ABIArgInfo::Ignore:
3490 assert(NumIRArgs == 0);
3491 // Initialize the local variable appropriately.
3492 if (!hasScalarEvaluationKind(T: Ty)) {
3493 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: CreateMemTemp(T: Ty)));
3494 } else {
3495 llvm::Value *U = llvm::UndefValue::get(T: ConvertType(T: Arg->getType()));
3496 ArgVals.push_back(Elt: ParamValue::forDirect(value: U));
3497 }
3498 break;
3499 }
3500 }
3501
3502 if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3503 for (int I = Args.size() - 1; I >= 0; --I)
3504 EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3505 } else {
3506 for (unsigned I = 0, E = Args.size(); I != E; ++I)
3507 EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3508 }
3509}
3510
3511static void eraseUnusedBitCasts(llvm::Instruction *insn) {
3512 while (insn->use_empty()) {
3513 llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: insn);
3514 if (!bitcast)
3515 return;
3516
3517 // This is "safe" because we would have used a ConstantExpr otherwise.
3518 insn = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3519 bitcast->eraseFromParent();
3520 }
3521}
3522
3523/// Try to emit a fused autorelease of a return result.
3524static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
3525 llvm::Value *result) {
3526 // We must be immediately followed the cast.
3527 llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3528 if (BB->empty())
3529 return nullptr;
3530 if (&BB->back() != result)
3531 return nullptr;
3532
3533 llvm::Type *resultType = result->getType();
3534
3535 // result is in a BasicBlock and is therefore an Instruction.
3536 llvm::Instruction *generator = cast<llvm::Instruction>(Val: result);
3537
3538 SmallVector<llvm::Instruction *, 4> InstsToKill;
3539
3540 // Look for:
3541 // %generator = bitcast %type1* %generator2 to %type2*
3542 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: generator)) {
3543 // We would have emitted this as a constant if the operand weren't
3544 // an Instruction.
3545 generator = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3546
3547 // Require the generator to be immediately followed by the cast.
3548 if (generator->getNextNode() != bitcast)
3549 return nullptr;
3550
3551 InstsToKill.push_back(Elt: bitcast);
3552 }
3553
3554 // Look for:
3555 // %generator = call i8* @objc_retain(i8* %originalResult)
3556 // or
3557 // %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
3558 llvm::CallInst *call = dyn_cast<llvm::CallInst>(Val: generator);
3559 if (!call)
3560 return nullptr;
3561
3562 bool doRetainAutorelease;
3563
3564 if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
3565 doRetainAutorelease = true;
3566 } else if (call->getCalledOperand() ==
3567 CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
3568 doRetainAutorelease = false;
3569
3570 // If we emitted an assembly marker for this call (and the
3571 // ARCEntrypoints field should have been set if so), go looking
3572 // for that call. If we can't find it, we can't do this
3573 // optimization. But it should always be the immediately previous
3574 // instruction, unless we needed bitcasts around the call.
3575 if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
3576 llvm::Instruction *prev = call->getPrevNode();
3577 assert(prev);
3578 if (isa<llvm::BitCastInst>(Val: prev)) {
3579 prev = prev->getPrevNode();
3580 assert(prev);
3581 }
3582 assert(isa<llvm::CallInst>(prev));
3583 assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3584 CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
3585 InstsToKill.push_back(Elt: prev);
3586 }
3587 } else {
3588 return nullptr;
3589 }
3590
3591 result = call->getArgOperand(i: 0);
3592 InstsToKill.push_back(Elt: call);
3593
3594 // Keep killing bitcasts, for sanity. Note that we no longer care
3595 // about precise ordering as long as there's exactly one use.
3596 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: result)) {
3597 if (!bitcast->hasOneUse())
3598 break;
3599 InstsToKill.push_back(Elt: bitcast);
3600 result = bitcast->getOperand(i_nocapture: 0);
3601 }
3602
3603 // Delete all the unnecessary instructions, from latest to earliest.
3604 for (auto *I : InstsToKill)
3605 I->eraseFromParent();
3606
3607 // Do the fused retain/autorelease if we were asked to.
3608 if (doRetainAutorelease)
3609 result = CGF.EmitARCRetainAutoreleaseReturnValue(value: result);
3610
3611 // Cast back to the result type.
3612 return CGF.Builder.CreateBitCast(V: result, DestTy: resultType);
3613}
3614
3615/// If this is a +1 of the value of an immutable 'self', remove it.
3616static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
3617 llvm::Value *result) {
3618 // This is only applicable to a method with an immutable 'self'.
3619 const ObjCMethodDecl *method =
3620 dyn_cast_or_null<ObjCMethodDecl>(Val: CGF.CurCodeDecl);
3621 if (!method)
3622 return nullptr;
3623 const VarDecl *self = method->getSelfDecl();
3624 if (!self->getType().isConstQualified())
3625 return nullptr;
3626
3627 // Look for a retain call. Note: stripPointerCasts looks through returned arg
3628 // functions, which would cause us to miss the retain.
3629 llvm::CallInst *retainCall = dyn_cast<llvm::CallInst>(Val: result);
3630 if (!retainCall || retainCall->getCalledOperand() !=
3631 CGF.CGM.getObjCEntrypoints().objc_retain)
3632 return nullptr;
3633
3634 // Look for an ordinary load of 'self'.
3635 llvm::Value *retainedValue = retainCall->getArgOperand(i: 0);
3636 llvm::LoadInst *load =
3637 dyn_cast<llvm::LoadInst>(Val: retainedValue->stripPointerCasts());
3638 if (!load || load->isAtomic() || load->isVolatile() ||
3639 load->getPointerOperand() != CGF.GetAddrOfLocalVar(VD: self).getBasePointer())
3640 return nullptr;
3641
3642 // Okay! Burn it all down. This relies for correctness on the
3643 // assumption that the retain is emitted as part of the return and
3644 // that thereafter everything is used "linearly".
3645 llvm::Type *resultType = result->getType();
3646 eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: result));
3647 assert(retainCall->use_empty());
3648 retainCall->eraseFromParent();
3649 eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: retainedValue));
3650
3651 return CGF.Builder.CreateBitCast(V: load, DestTy: resultType);
3652}
3653
3654/// Emit an ARC autorelease of the result of a function.
3655///
3656/// \return the value to actually return from the function
3657static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
3658 llvm::Value *result) {
3659 // If we're returning 'self', kill the initial retain. This is a
3660 // heuristic attempt to "encourage correctness" in the really unfortunate
3661 // case where we have a return of self during a dealloc and we desperately
3662 // need to avoid the possible autorelease.
3663 if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
3664 return self;
3665
3666 // At -O0, try to emit a fused retain/autorelease.
3667 if (CGF.shouldUseFusedARCCalls())
3668 if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
3669 return fused;
3670
3671 return CGF.EmitARCAutoreleaseReturnValue(value: result);
3672}
3673
3674/// Heuristically search for a dominating store to the return-value slot.
3675static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
3676 llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer();
3677
3678 // Check if a User is a store which pointerOperand is the ReturnValue.
3679 // We are looking for stores to the ReturnValue, not for stores of the
3680 // ReturnValue to some other location.
3681 auto GetStoreIfValid = [&CGF,
3682 ReturnValuePtr](llvm::User *U) -> llvm::StoreInst * {
3683 auto *SI = dyn_cast<llvm::StoreInst>(Val: U);
3684 if (!SI || SI->getPointerOperand() != ReturnValuePtr ||
3685 SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())
3686 return nullptr;
3687 // These aren't actually possible for non-coerced returns, and we
3688 // only care about non-coerced returns on this code path.
3689 // All memory instructions inside __try block are volatile.
3690 assert(!SI->isAtomic() &&
3691 (!SI->isVolatile() || CGF.currentFunctionUsesSEHTry()));
3692 return SI;
3693 };
3694 // If there are multiple uses of the return-value slot, just check
3695 // for something immediately preceding the IP. Sometimes this can
3696 // happen with how we generate implicit-returns; it can also happen
3697 // with noreturn cleanups.
3698 if (!ReturnValuePtr->hasOneUse()) {
3699 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3700 if (IP->empty())
3701 return nullptr;
3702
3703 // Look at directly preceding instruction, skipping bitcasts, lifetime
3704 // markers, and fake uses and their operands.
3705 const llvm::Instruction *LoadIntoFakeUse = nullptr;
3706 for (llvm::Instruction &I : llvm::reverse(C&: *IP)) {
3707 // Ignore instructions that are just loads for fake uses; the load should
3708 // immediately precede the fake use, so we only need to remember the
3709 // operand for the last fake use seen.
3710 if (LoadIntoFakeUse == &I)
3711 continue;
3712 if (isa<llvm::BitCastInst>(Val: &I))
3713 continue;
3714 if (auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &I)) {
3715 if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
3716 continue;
3717
3718 if (II->getIntrinsicID() == llvm::Intrinsic::fake_use) {
3719 LoadIntoFakeUse = dyn_cast<llvm::Instruction>(Val: II->getArgOperand(i: 0));
3720 continue;
3721 }
3722 }
3723 return GetStoreIfValid(&I);
3724 }
3725 return nullptr;
3726 }
3727
3728 llvm::StoreInst *store = GetStoreIfValid(ReturnValuePtr->user_back());
3729 if (!store)
3730 return nullptr;
3731
3732 // Now do a first-and-dirty dominance check: just walk up the
3733 // single-predecessors chain from the current insertion point.
3734 llvm::BasicBlock *StoreBB = store->getParent();
3735 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3736 llvm::SmallPtrSet<llvm::BasicBlock *, 4> SeenBBs;
3737 while (IP != StoreBB) {
3738 if (!SeenBBs.insert(Ptr: IP).second || !(IP = IP->getSinglePredecessor()))
3739 return nullptr;
3740 }
3741
3742 // Okay, the store's basic block dominates the insertion point; we
3743 // can do our thing.
3744 return store;
3745}
3746
3747// Helper functions for EmitCMSEClearRecord
3748
3749// Set the bits corresponding to a field having width `BitWidth` and located at
3750// offset `BitOffset` (from the least significant bit) within a storage unit of
3751// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
3752// Use little-endian layout, i.e.`Bits[0]` is the LSB.
3753static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
3754 int BitWidth, int CharWidth) {
3755 assert(CharWidth <= 64);
3756 assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3757
3758 int Pos = 0;
3759 if (BitOffset >= CharWidth) {
3760 Pos += BitOffset / CharWidth;
3761 BitOffset = BitOffset % CharWidth;
3762 }
3763
3764 const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
3765 if (BitOffset + BitWidth >= CharWidth) {
3766 Bits[Pos++] |= (Used << BitOffset) & Used;
3767 BitWidth -= CharWidth - BitOffset;
3768 BitOffset = 0;
3769 }
3770
3771 while (BitWidth >= CharWidth) {
3772 Bits[Pos++] = Used;
3773 BitWidth -= CharWidth;
3774 }
3775
3776 if (BitWidth > 0)
3777 Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
3778}
3779
3780// Set the bits corresponding to a field having width `BitWidth` and located at
3781// offset `BitOffset` (from the least significant bit) within a storage unit of
3782// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
3783// `Bits` corresponds to one target byte. Use target endian layout.
3784static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
3785 int StorageSize, int BitOffset, int BitWidth,
3786 int CharWidth, bool BigEndian) {
3787
3788 SmallVector<uint64_t, 8> TmpBits(StorageSize);
3789 setBitRange(Bits&: TmpBits, BitOffset, BitWidth, CharWidth);
3790
3791 if (BigEndian)
3792 std::reverse(first: TmpBits.begin(), last: TmpBits.end());
3793
3794 for (uint64_t V : TmpBits)
3795 Bits[StorageOffset++] |= V;
3796}
3797
3798static void setUsedBits(CodeGenModule &, QualType, int,
3799 SmallVectorImpl<uint64_t> &);
3800
3801// Set the bits in `Bits`, which correspond to the value representations of
3802// the actual members of the record type `RTy`. Note that this function does
3803// not handle base classes, virtual tables, etc, since they cannot happen in
3804// CMSE function arguments or return. The bit mask corresponds to the target
3805// memory layout, i.e. it's endian dependent.
3806static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
3807 SmallVectorImpl<uint64_t> &Bits) {
3808 ASTContext &Context = CGM.getContext();
3809 int CharWidth = Context.getCharWidth();
3810 const RecordDecl *RD = RTy->getDecl()->getDefinition();
3811 const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(D: RD);
3812 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
3813
3814 int Idx = 0;
3815 for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) {
3816 const FieldDecl *F = *I;
3817
3818 if (F->isUnnamedBitField() || F->isZeroLengthBitField() ||
3819 F->getType()->isIncompleteArrayType())
3820 continue;
3821
3822 if (F->isBitField()) {
3823 const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(FD: F);
3824 setBitRange(Bits, StorageOffset: Offset + BFI.StorageOffset.getQuantity(),
3825 StorageSize: BFI.StorageSize / CharWidth, BitOffset: BFI.Offset, BitWidth: BFI.Size, CharWidth,
3826 BigEndian: CGM.getDataLayout().isBigEndian());
3827 continue;
3828 }
3829
3830 setUsedBits(CGM, F->getType(),
3831 Offset + ASTLayout.getFieldOffset(FieldNo: Idx) / CharWidth, Bits);
3832 }
3833}
3834
3835// Set the bits in `Bits`, which correspond to the value representations of
3836// the elements of an array type `ATy`.
3837static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
3838 int Offset, SmallVectorImpl<uint64_t> &Bits) {
3839 const ASTContext &Context = CGM.getContext();
3840
3841 QualType ETy = Context.getBaseElementType(VAT: ATy);
3842 int Size = Context.getTypeSizeInChars(T: ETy).getQuantity();
3843 SmallVector<uint64_t, 4> TmpBits(Size);
3844 setUsedBits(CGM, ETy, 0, TmpBits);
3845
3846 for (int I = 0, N = Context.getConstantArrayElementCount(CA: ATy); I < N; ++I) {
3847 auto Src = TmpBits.begin();
3848 auto Dst = Bits.begin() + Offset + I * Size;
3849 for (int J = 0; J < Size; ++J)
3850 *Dst++ |= *Src++;
3851 }
3852}
3853
3854// Set the bits in `Bits`, which correspond to the value representations of
3855// the type `QTy`.
3856static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
3857 SmallVectorImpl<uint64_t> &Bits) {
3858 if (const auto *RTy = QTy->getAs<RecordType>())
3859 return setUsedBits(CGM, RTy, Offset, Bits);
3860
3861 ASTContext &Context = CGM.getContext();
3862 if (const auto *ATy = Context.getAsConstantArrayType(T: QTy))
3863 return setUsedBits(CGM, ATy, Offset, Bits);
3864
3865 int Size = Context.getTypeSizeInChars(T: QTy).getQuantity();
3866 if (Size <= 0)
3867 return;
3868
3869 std::fill_n(first: Bits.begin() + Offset, n: Size,
3870 value: (uint64_t(1) << Context.getCharWidth()) - 1);
3871}
3872
3873static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
3874 int Pos, int Size, int CharWidth,
3875 bool BigEndian) {
3876 assert(Size > 0);
3877 uint64_t Mask = 0;
3878 if (BigEndian) {
3879 for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
3880 ++P)
3881 Mask = (Mask << CharWidth) | *P;
3882 } else {
3883 auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
3884 do
3885 Mask = (Mask << CharWidth) | *--P;
3886 while (P != End);
3887 }
3888 return Mask;
3889}
3890
3891// Emit code to clear the bits in a record, which aren't a part of any user
3892// declared member, when the record is a function return.
3893llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3894 llvm::IntegerType *ITy,
3895 QualType QTy) {
3896 assert(Src->getType() == ITy);
3897 assert(ITy->getScalarSizeInBits() <= 64);
3898
3899 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3900 int Size = DataLayout.getTypeStoreSize(Ty: ITy);
3901 SmallVector<uint64_t, 4> Bits(Size);
3902 setUsedBits(CGM, RTy: QTy->castAs<RecordType>(), Offset: 0, Bits);
3903
3904 int CharWidth = CGM.getContext().getCharWidth();
3905 uint64_t Mask =
3906 buildMultiCharMask(Bits, Pos: 0, Size, CharWidth, BigEndian: DataLayout.isBigEndian());
3907
3908 return Builder.CreateAnd(LHS: Src, RHS: Mask, Name: "cmse.clear");
3909}
3910
3911// Emit code to clear the bits in a record, which aren't a part of any user
3912// declared member, when the record is a function argument.
3913llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3914 llvm::ArrayType *ATy,
3915 QualType QTy) {
3916 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3917 int Size = DataLayout.getTypeStoreSize(Ty: ATy);
3918 SmallVector<uint64_t, 16> Bits(Size);
3919 setUsedBits(CGM, RTy: QTy->castAs<RecordType>(), Offset: 0, Bits);
3920
3921 // Clear each element of the LLVM array.
3922 int CharWidth = CGM.getContext().getCharWidth();
3923 int CharsPerElt =
3924 ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
3925 int MaskIndex = 0;
3926 llvm::Value *R = llvm::PoisonValue::get(T: ATy);
3927 for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
3928 uint64_t Mask = buildMultiCharMask(Bits, Pos: MaskIndex, Size: CharsPerElt, CharWidth,
3929 BigEndian: DataLayout.isBigEndian());
3930 MaskIndex += CharsPerElt;
3931 llvm::Value *T0 = Builder.CreateExtractValue(Agg: Src, Idxs: I);
3932 llvm::Value *T1 = Builder.CreateAnd(LHS: T0, RHS: Mask, Name: "cmse.clear");
3933 R = Builder.CreateInsertValue(Agg: R, Val: T1, Idxs: I);
3934 }
3935
3936 return R;
3937}
3938
3939void CodeGenFunction::EmitFunctionEpilog(
3940 const CGFunctionInfo &FI, bool EmitRetDbgLoc, SourceLocation EndLoc,
3941 uint64_t RetKeyInstructionsSourceAtom) {
3942 if (FI.isNoReturn()) {
3943 // Noreturn functions don't return.
3944 EmitUnreachable(Loc: EndLoc);
3945 return;
3946 }
3947
3948 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {
3949 // Naked functions don't have epilogues.
3950 Builder.CreateUnreachable();
3951 return;
3952 }
3953
3954 // Functions with no result always return void.
3955 if (!ReturnValue.isValid()) {
3956 auto *I = Builder.CreateRetVoid();
3957 if (RetKeyInstructionsSourceAtom)
3958 addInstToSpecificSourceAtom(KeyInstruction: I, Backup: nullptr, Atom: RetKeyInstructionsSourceAtom);
3959 else
3960 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
3961 return;
3962 }
3963
3964 llvm::DebugLoc RetDbgLoc;
3965 llvm::Value *RV = nullptr;
3966 QualType RetTy = FI.getReturnType();
3967 const ABIArgInfo &RetAI = FI.getReturnInfo();
3968
3969 switch (RetAI.getKind()) {
3970 case ABIArgInfo::InAlloca:
3971 // Aggregates get evaluated directly into the destination. Sometimes we
3972 // need to return the sret value in a register, though.
3973 assert(hasAggregateEvaluationKind(RetTy));
3974 if (RetAI.getInAllocaSRet()) {
3975 llvm::Function::arg_iterator EI = CurFn->arg_end();
3976 --EI;
3977 llvm::Value *ArgStruct = &*EI;
3978 llvm::Value *SRet = Builder.CreateStructGEP(
3979 Ty: FI.getArgStruct(), Ptr: ArgStruct, Idx: RetAI.getInAllocaFieldIndex());
3980 llvm::Type *Ty =
3981 cast<llvm::GetElementPtrInst>(Val: SRet)->getResultElementType();
3982 RV = Builder.CreateAlignedLoad(Ty, Addr: SRet, Align: getPointerAlign(), Name: "sret");
3983 }
3984 break;
3985
3986 case ABIArgInfo::Indirect: {
3987 auto AI = CurFn->arg_begin();
3988 if (RetAI.isSRetAfterThis())
3989 ++AI;
3990 switch (getEvaluationKind(T: RetTy)) {
3991 case TEK_Complex: {
3992 ComplexPairTy RT =
3993 EmitLoadOfComplex(src: MakeAddrLValue(Addr: ReturnValue, T: RetTy), loc: EndLoc);
3994 EmitStoreOfComplex(V: RT, dest: MakeNaturalAlignAddrLValue(V: &*AI, T: RetTy),
3995 /*isInit*/ true);
3996 break;
3997 }
3998 case TEK_Aggregate:
3999 // Do nothing; aggregates get evaluated directly into the destination.
4000 break;
4001 case TEK_Scalar: {
4002 LValueBaseInfo BaseInfo;
4003 TBAAAccessInfo TBAAInfo;
4004 CharUnits Alignment =
4005 CGM.getNaturalTypeAlignment(T: RetTy, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
4006 Address ArgAddr(&*AI, ConvertType(T: RetTy), Alignment);
4007 LValue ArgVal =
4008 LValue::MakeAddr(Addr: ArgAddr, type: RetTy, Context&: getContext(), BaseInfo, TBAAInfo);
4009 EmitStoreOfScalar(
4010 value: EmitLoadOfScalar(lvalue: MakeAddrLValue(Addr: ReturnValue, T: RetTy), Loc: EndLoc), lvalue: ArgVal,
4011 /*isInit*/ true);
4012 break;
4013 }
4014 }
4015 break;
4016 }
4017
4018 case ABIArgInfo::Extend:
4019 case ABIArgInfo::Direct:
4020 if (RetAI.getCoerceToType() == ConvertType(T: RetTy) &&
4021 RetAI.getDirectOffset() == 0) {
4022 // The internal return value temp always will have pointer-to-return-type
4023 // type, just do a load.
4024
4025 // If there is a dominating store to ReturnValue, we can elide
4026 // the load, zap the store, and usually zap the alloca.
4027 if (llvm::StoreInst *SI = findDominatingStoreToReturnValue(CGF&: *this)) {
4028 // Reuse the debug location from the store unless there is
4029 // cleanup code to be emitted between the store and return
4030 // instruction.
4031 if (EmitRetDbgLoc && !AutoreleaseResult)
4032 RetDbgLoc = SI->getDebugLoc();
4033 // Get the stored value and nuke the now-dead store.
4034 RV = SI->getValueOperand();
4035 SI->eraseFromParent();
4036
4037 // Otherwise, we have to do a simple load.
4038 } else {
4039 RV = Builder.CreateLoad(Addr: ReturnValue);
4040 }
4041 } else {
4042 // If the value is offset in memory, apply the offset now.
4043 Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
4044
4045 RV = CreateCoercedLoad(Src: V, Ty: RetAI.getCoerceToType(), CGF&: *this);
4046 }
4047
4048 // In ARC, end functions that return a retainable type with a call
4049 // to objc_autoreleaseReturnValue.
4050 if (AutoreleaseResult) {
4051#ifndef NDEBUG
4052 // Type::isObjCRetainabletype has to be called on a QualType that hasn't
4053 // been stripped of the typedefs, so we cannot use RetTy here. Get the
4054 // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
4055 // CurCodeDecl or BlockInfo.
4056 QualType RT;
4057
4058 if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
4059 RT = FD->getReturnType();
4060 else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
4061 RT = MD->getReturnType();
4062 else if (isa<BlockDecl>(CurCodeDecl))
4063 RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
4064 else
4065 llvm_unreachable("Unexpected function/method type");
4066
4067 assert(getLangOpts().ObjCAutoRefCount && !FI.isReturnsRetained() &&
4068 RT->isObjCRetainableType());
4069#endif
4070 RV = emitAutoreleaseOfResult(CGF&: *this, result: RV);
4071 }
4072
4073 break;
4074
4075 case ABIArgInfo::Ignore:
4076 break;
4077
4078 case ABIArgInfo::CoerceAndExpand: {
4079 auto coercionType = RetAI.getCoerceAndExpandType();
4080 auto unpaddedCoercionType = RetAI.getUnpaddedCoerceAndExpandType();
4081 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
4082
4083 // Load all of the coerced elements out into results.
4084 llvm::SmallVector<llvm::Value *, 4> results;
4085 Address addr = ReturnValue.withElementType(ElemTy: coercionType);
4086 unsigned unpaddedIndex = 0;
4087 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
4088 auto coercedEltType = coercionType->getElementType(N: i);
4089 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType: coercedEltType))
4090 continue;
4091
4092 auto eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
4093 llvm::Value *elt = CreateCoercedLoad(
4094 Src: eltAddr,
4095 Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
4096 : unpaddedCoercionType,
4097 CGF&: *this);
4098 results.push_back(Elt: elt);
4099 }
4100
4101 // If we have one result, it's the single direct result type.
4102 if (results.size() == 1) {
4103 RV = results[0];
4104
4105 // Otherwise, we need to make a first-class aggregate.
4106 } else {
4107 // Construct a return type that lacks padding elements.
4108 llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
4109
4110 RV = llvm::PoisonValue::get(T: returnType);
4111 for (unsigned i = 0, e = results.size(); i != e; ++i) {
4112 RV = Builder.CreateInsertValue(Agg: RV, Val: results[i], Idxs: i);
4113 }
4114 }
4115 break;
4116 }
4117 case ABIArgInfo::Expand:
4118 case ABIArgInfo::IndirectAliased:
4119 llvm_unreachable("Invalid ABI kind for return argument");
4120 }
4121
4122 llvm::Instruction *Ret;
4123 if (RV) {
4124 if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) {
4125 // For certain return types, clear padding bits, as they may reveal
4126 // sensitive information.
4127 // Small struct/union types are passed as integers.
4128 auto *ITy = dyn_cast<llvm::IntegerType>(Val: RV->getType());
4129 if (ITy != nullptr && isa<RecordType>(Val: RetTy.getCanonicalType()))
4130 RV = EmitCMSEClearRecord(Src: RV, ITy, QTy: RetTy);
4131 }
4132 EmitReturnValueCheck(RV);
4133 Ret = Builder.CreateRet(V: RV);
4134 } else {
4135 Ret = Builder.CreateRetVoid();
4136 }
4137
4138 if (RetDbgLoc)
4139 Ret->setDebugLoc(std::move(RetDbgLoc));
4140
4141 llvm::Value *Backup = RV ? Ret->getOperand(i: 0) : nullptr;
4142 if (RetKeyInstructionsSourceAtom)
4143 addInstToSpecificSourceAtom(KeyInstruction: Ret, Backup, Atom: RetKeyInstructionsSourceAtom);
4144 else
4145 addInstToNewSourceAtom(KeyInstruction: Ret, Backup);
4146}
4147
4148void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
4149 // A current decl may not be available when emitting vtable thunks.
4150 if (!CurCodeDecl)
4151 return;
4152
4153 // If the return block isn't reachable, neither is this check, so don't emit
4154 // it.
4155 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty())
4156 return;
4157
4158 ReturnsNonNullAttr *RetNNAttr = nullptr;
4159 if (SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute))
4160 RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
4161
4162 if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
4163 return;
4164
4165 // Prefer the returns_nonnull attribute if it's present.
4166 SourceLocation AttrLoc;
4167 SanitizerKind::SanitizerOrdinal CheckKind;
4168 SanitizerHandler Handler;
4169 if (RetNNAttr) {
4170 assert(!requiresReturnValueNullabilityCheck() &&
4171 "Cannot check nullability and the nonnull attribute");
4172 AttrLoc = RetNNAttr->getLocation();
4173 CheckKind = SanitizerKind::SO_ReturnsNonnullAttribute;
4174 Handler = SanitizerHandler::NonnullReturn;
4175 } else {
4176 if (auto *DD = dyn_cast<DeclaratorDecl>(Val: CurCodeDecl))
4177 if (auto *TSI = DD->getTypeSourceInfo())
4178 if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
4179 AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
4180 CheckKind = SanitizerKind::SO_NullabilityReturn;
4181 Handler = SanitizerHandler::NullabilityReturn;
4182 }
4183
4184 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4185
4186 // Make sure the "return" source location is valid. If we're checking a
4187 // nullability annotation, make sure the preconditions for the check are met.
4188 llvm::BasicBlock *Check = createBasicBlock(name: "nullcheck");
4189 llvm::BasicBlock *NoCheck = createBasicBlock(name: "no.nullcheck");
4190 llvm::Value *SLocPtr = Builder.CreateLoad(Addr: ReturnLocation, Name: "return.sloc.load");
4191 llvm::Value *CanNullCheck = Builder.CreateIsNotNull(Arg: SLocPtr);
4192 if (requiresReturnValueNullabilityCheck())
4193 CanNullCheck =
4194 Builder.CreateAnd(LHS: CanNullCheck, RHS: RetValNullabilityPrecondition);
4195 Builder.CreateCondBr(Cond: CanNullCheck, True: Check, False: NoCheck);
4196 EmitBlock(BB: Check);
4197
4198 // Now do the null check.
4199 llvm::Value *Cond = Builder.CreateIsNotNull(Arg: RV);
4200 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc: AttrLoc)};
4201 llvm::Value *DynamicData[] = {SLocPtr};
4202 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: DynamicData);
4203
4204 EmitBlock(BB: NoCheck);
4205
4206#ifndef NDEBUG
4207 // The return location should not be used after the check has been emitted.
4208 ReturnLocation = Address::invalid();
4209#endif
4210}
4211
4212static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
4213 const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
4214 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
4215}
4216
4217static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, QualType Ty) {
4218 // FIXME: Generate IR in one pass, rather than going back and fixing up these
4219 // placeholders.
4220 llvm::Type *IRTy = CGF.ConvertTypeForMem(T: Ty);
4221 llvm::Type *IRPtrTy = llvm::PointerType::getUnqual(C&: CGF.getLLVMContext());
4222 llvm::Value *Placeholder = llvm::PoisonValue::get(T: IRPtrTy);
4223
4224 // FIXME: When we generate this IR in one pass, we shouldn't need
4225 // this win32-specific alignment hack.
4226 CharUnits Align = CharUnits::fromQuantity(Quantity: 4);
4227 Placeholder = CGF.Builder.CreateAlignedLoad(Ty: IRPtrTy, Addr: Placeholder, Align);
4228
4229 return AggValueSlot::forAddr(
4230 addr: Address(Placeholder, IRTy, Align), quals: Ty.getQualifiers(),
4231 isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
4232 isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
4233}
4234
4235void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
4236 const VarDecl *param,
4237 SourceLocation loc) {
4238 // StartFunction converted the ABI-lowered parameter(s) into a
4239 // local alloca. We need to turn that into an r-value suitable
4240 // for EmitCall.
4241 Address local = GetAddrOfLocalVar(VD: param);
4242
4243 QualType type = param->getType();
4244
4245 // GetAddrOfLocalVar returns a pointer-to-pointer for references,
4246 // but the argument needs to be the original pointer.
4247 if (type->isReferenceType()) {
4248 args.add(rvalue: RValue::get(V: Builder.CreateLoad(Addr: local)), type);
4249
4250 // In ARC, move out of consumed arguments so that the release cleanup
4251 // entered by StartFunction doesn't cause an over-release. This isn't
4252 // optimal -O0 code generation, but it should get cleaned up when
4253 // optimization is enabled. This also assumes that delegate calls are
4254 // performed exactly once for a set of arguments, but that should be safe.
4255 } else if (getLangOpts().ObjCAutoRefCount &&
4256 param->hasAttr<NSConsumedAttr>() && type->isObjCRetainableType()) {
4257 llvm::Value *ptr = Builder.CreateLoad(Addr: local);
4258 auto null =
4259 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: ptr->getType()));
4260 Builder.CreateStore(Val: null, Addr: local);
4261 args.add(rvalue: RValue::get(V: ptr), type);
4262
4263 // For the most part, we just need to load the alloca, except that
4264 // aggregate r-values are actually pointers to temporaries.
4265 } else {
4266 args.add(rvalue: convertTempToRValue(addr: local, type, Loc: loc), type);
4267 }
4268
4269 // Deactivate the cleanup for the callee-destructed param that was pushed.
4270 if (type->isRecordType() && !CurFuncIsThunk &&
4271 type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee() &&
4272 param->needsDestruction(Ctx: getContext())) {
4273 EHScopeStack::stable_iterator cleanup =
4274 CalleeDestructedParamCleanups.lookup(Val: cast<ParmVarDecl>(Val: param));
4275 assert(cleanup.isValid() &&
4276 "cleanup for callee-destructed param not recorded");
4277 // This unreachable is a temporary marker which will be removed later.
4278 llvm::Instruction *isActive = Builder.CreateUnreachable();
4279 args.addArgCleanupDeactivation(Cleanup: cleanup, IsActiveIP: isActive);
4280 }
4281}
4282
4283static bool isProvablyNull(llvm::Value *addr) {
4284 return llvm::isa_and_nonnull<llvm::ConstantPointerNull>(Val: addr);
4285}
4286
4287static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) {
4288 return llvm::isKnownNonZero(V: Addr.getBasePointer(), Q: CGF.CGM.getDataLayout());
4289}
4290
4291/// Emit the actual writing-back of a writeback.
4292static void emitWriteback(CodeGenFunction &CGF,
4293 const CallArgList::Writeback &writeback) {
4294 const LValue &srcLV = writeback.Source;
4295 Address srcAddr = srcLV.getAddress();
4296 assert(!isProvablyNull(srcAddr.getBasePointer()) &&
4297 "shouldn't have writeback for provably null argument");
4298
4299 if (writeback.WritebackExpr) {
4300 CGF.EmitIgnoredExpr(E: writeback.WritebackExpr);
4301
4302 if (writeback.LifetimeSz)
4303 CGF.EmitLifetimeEnd(Size: writeback.LifetimeSz,
4304 Addr: writeback.Temporary.getBasePointer());
4305 return;
4306 }
4307
4308 llvm::BasicBlock *contBB = nullptr;
4309
4310 // If the argument wasn't provably non-null, we need to null check
4311 // before doing the store.
4312 bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4313
4314 if (!provablyNonNull) {
4315 llvm::BasicBlock *writebackBB = CGF.createBasicBlock(name: "icr.writeback");
4316 contBB = CGF.createBasicBlock(name: "icr.done");
4317
4318 llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4319 CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: writebackBB);
4320 CGF.EmitBlock(BB: writebackBB);
4321 }
4322
4323 // Load the value to writeback.
4324 llvm::Value *value = CGF.Builder.CreateLoad(Addr: writeback.Temporary);
4325
4326 // Cast it back, in case we're writing an id to a Foo* or something.
4327 value = CGF.Builder.CreateBitCast(V: value, DestTy: srcAddr.getElementType(),
4328 Name: "icr.writeback-cast");
4329
4330 // Perform the writeback.
4331
4332 // If we have a "to use" value, it's something we need to emit a use
4333 // of. This has to be carefully threaded in: if it's done after the
4334 // release it's potentially undefined behavior (and the optimizer
4335 // will ignore it), and if it happens before the retain then the
4336 // optimizer could move the release there.
4337 if (writeback.ToUse) {
4338 assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
4339
4340 // Retain the new value. No need to block-copy here: the block's
4341 // being passed up the stack.
4342 value = CGF.EmitARCRetainNonBlock(value);
4343
4344 // Emit the intrinsic use here.
4345 CGF.EmitARCIntrinsicUse(values: writeback.ToUse);
4346
4347 // Load the old value (primitively).
4348 llvm::Value *oldValue = CGF.EmitLoadOfScalar(lvalue: srcLV, Loc: SourceLocation());
4349
4350 // Put the new value in place (primitively).
4351 CGF.EmitStoreOfScalar(value, lvalue: srcLV, /*init*/ isInit: false);
4352
4353 // Release the old value.
4354 CGF.EmitARCRelease(value: oldValue, precise: srcLV.isARCPreciseLifetime());
4355
4356 // Otherwise, we can just do a normal lvalue store.
4357 } else {
4358 CGF.EmitStoreThroughLValue(Src: RValue::get(V: value), Dst: srcLV);
4359 }
4360
4361 // Jump to the continuation block.
4362 if (!provablyNonNull)
4363 CGF.EmitBlock(BB: contBB);
4364}
4365
4366static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
4367 const CallArgList &CallArgs) {
4368 ArrayRef<CallArgList::CallArgCleanup> Cleanups =
4369 CallArgs.getCleanupsToDeactivate();
4370 // Iterate in reverse to increase the likelihood of popping the cleanup.
4371 for (const auto &I : llvm::reverse(C&: Cleanups)) {
4372 CGF.DeactivateCleanupBlock(Cleanup: I.Cleanup, DominatingIP: I.IsActiveIP);
4373 I.IsActiveIP->eraseFromParent();
4374 }
4375}
4376
4377static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
4378 if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(Val: E->IgnoreParens()))
4379 if (uop->getOpcode() == UO_AddrOf)
4380 return uop->getSubExpr();
4381 return nullptr;
4382}
4383
4384/// Emit an argument that's being passed call-by-writeback. That is,
4385/// we are passing the address of an __autoreleased temporary; it
4386/// might be copy-initialized with the current value of the given
4387/// address, but it will definitely be copied out of after the call.
4388static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
4389 const ObjCIndirectCopyRestoreExpr *CRE) {
4390 LValue srcLV;
4391
4392 // Make an optimistic effort to emit the address as an l-value.
4393 // This can fail if the argument expression is more complicated.
4394 if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(E: CRE->getSubExpr())) {
4395 srcLV = CGF.EmitLValue(E: lvExpr);
4396
4397 // Otherwise, just emit it as a scalar.
4398 } else {
4399 Address srcAddr = CGF.EmitPointerWithAlignment(Addr: CRE->getSubExpr());
4400
4401 QualType srcAddrType =
4402 CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
4403 srcLV = CGF.MakeAddrLValue(Addr: srcAddr, T: srcAddrType);
4404 }
4405 Address srcAddr = srcLV.getAddress();
4406
4407 // The dest and src types don't necessarily match in LLVM terms
4408 // because of the crazy ObjC compatibility rules.
4409
4410 llvm::PointerType *destType =
4411 cast<llvm::PointerType>(Val: CGF.ConvertType(T: CRE->getType()));
4412 llvm::Type *destElemType =
4413 CGF.ConvertTypeForMem(T: CRE->getType()->getPointeeType());
4414
4415 // If the address is a constant null, just pass the appropriate null.
4416 if (isProvablyNull(addr: srcAddr.getBasePointer())) {
4417 args.add(rvalue: RValue::get(V: llvm::ConstantPointerNull::get(T: destType)),
4418 type: CRE->getType());
4419 return;
4420 }
4421
4422 // Create the temporary.
4423 Address temp =
4424 CGF.CreateTempAlloca(Ty: destElemType, align: CGF.getPointerAlign(), Name: "icr.temp");
4425 // Loading an l-value can introduce a cleanup if the l-value is __weak,
4426 // and that cleanup will be conditional if we can't prove that the l-value
4427 // isn't null, so we need to register a dominating point so that the cleanups
4428 // system will make valid IR.
4429 CodeGenFunction::ConditionalEvaluation condEval(CGF);
4430
4431 // Zero-initialize it if we're not doing a copy-initialization.
4432 bool shouldCopy = CRE->shouldCopy();
4433 if (!shouldCopy) {
4434 llvm::Value *null =
4435 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: destElemType));
4436 CGF.Builder.CreateStore(Val: null, Addr: temp);
4437 }
4438
4439 llvm::BasicBlock *contBB = nullptr;
4440 llvm::BasicBlock *originBB = nullptr;
4441
4442 // If the address is *not* known to be non-null, we need to switch.
4443 llvm::Value *finalArgument;
4444
4445 bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4446
4447 if (provablyNonNull) {
4448 finalArgument = temp.emitRawPointer(CGF);
4449 } else {
4450 llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4451
4452 finalArgument = CGF.Builder.CreateSelect(
4453 C: isNull, True: llvm::ConstantPointerNull::get(T: destType),
4454 False: temp.emitRawPointer(CGF), Name: "icr.argument");
4455
4456 // If we need to copy, then the load has to be conditional, which
4457 // means we need control flow.
4458 if (shouldCopy) {
4459 originBB = CGF.Builder.GetInsertBlock();
4460 contBB = CGF.createBasicBlock(name: "icr.cont");
4461 llvm::BasicBlock *copyBB = CGF.createBasicBlock(name: "icr.copy");
4462 CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: copyBB);
4463 CGF.EmitBlock(BB: copyBB);
4464 condEval.begin(CGF);
4465 }
4466 }
4467
4468 llvm::Value *valueToUse = nullptr;
4469
4470 // Perform a copy if necessary.
4471 if (shouldCopy) {
4472 RValue srcRV = CGF.EmitLoadOfLValue(V: srcLV, Loc: SourceLocation());
4473 assert(srcRV.isScalar());
4474
4475 llvm::Value *src = srcRV.getScalarVal();
4476 src = CGF.Builder.CreateBitCast(V: src, DestTy: destElemType, Name: "icr.cast");
4477
4478 // Use an ordinary store, not a store-to-lvalue.
4479 CGF.Builder.CreateStore(Val: src, Addr: temp);
4480
4481 // If optimization is enabled, and the value was held in a
4482 // __strong variable, we need to tell the optimizer that this
4483 // value has to stay alive until we're doing the store back.
4484 // This is because the temporary is effectively unretained,
4485 // and so otherwise we can violate the high-level semantics.
4486 if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4487 srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {
4488 valueToUse = src;
4489 }
4490 }
4491
4492 // Finish the control flow if we needed it.
4493 if (shouldCopy && !provablyNonNull) {
4494 llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
4495 CGF.EmitBlock(BB: contBB);
4496
4497 // Make a phi for the value to intrinsically use.
4498 if (valueToUse) {
4499 llvm::PHINode *phiToUse =
4500 CGF.Builder.CreatePHI(Ty: valueToUse->getType(), NumReservedValues: 2, Name: "icr.to-use");
4501 phiToUse->addIncoming(V: valueToUse, BB: copyBB);
4502 phiToUse->addIncoming(V: llvm::PoisonValue::get(T: valueToUse->getType()),
4503 BB: originBB);
4504 valueToUse = phiToUse;
4505 }
4506
4507 condEval.end(CGF);
4508 }
4509
4510 args.addWriteback(srcLV, temporary: temp, toUse: valueToUse);
4511 args.add(rvalue: RValue::get(V: finalArgument), type: CRE->getType());
4512}
4513
4514void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
4515 assert(!StackBase);
4516
4517 // Save the stack.
4518 StackBase = CGF.Builder.CreateStackSave(Name: "inalloca.save");
4519}
4520
4521void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
4522 if (StackBase) {
4523 // Restore the stack after the call.
4524 CGF.Builder.CreateStackRestore(Ptr: StackBase);
4525 }
4526}
4527
4528void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
4529 SourceLocation ArgLoc,
4530 AbstractCallee AC, unsigned ParmNum) {
4531 if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
4532 SanOpts.has(K: SanitizerKind::NullabilityArg)))
4533 return;
4534
4535 // The param decl may be missing in a variadic function.
4536 auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(I: ParmNum) : nullptr;
4537 unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
4538
4539 // Prefer the nonnull attribute if it's present.
4540 const NonNullAttr *NNAttr = nullptr;
4541 if (SanOpts.has(K: SanitizerKind::NonnullAttribute))
4542 NNAttr = getNonNullAttr(FD: AC.getDecl(), PVD, ArgType, ArgNo);
4543
4544 bool CanCheckNullability = false;
4545 if (SanOpts.has(K: SanitizerKind::NullabilityArg) && !NNAttr && PVD &&
4546 !PVD->getType()->isRecordType()) {
4547 auto Nullability = PVD->getType()->getNullability();
4548 CanCheckNullability = Nullability &&
4549 *Nullability == NullabilityKind::NonNull &&
4550 PVD->getTypeSourceInfo();
4551 }
4552
4553 if (!NNAttr && !CanCheckNullability)
4554 return;
4555
4556 SourceLocation AttrLoc;
4557 SanitizerKind::SanitizerOrdinal CheckKind;
4558 SanitizerHandler Handler;
4559 if (NNAttr) {
4560 AttrLoc = NNAttr->getLocation();
4561 CheckKind = SanitizerKind::SO_NonnullAttribute;
4562 Handler = SanitizerHandler::NonnullArg;
4563 } else {
4564 AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4565 CheckKind = SanitizerKind::SO_NullabilityArg;
4566 Handler = SanitizerHandler::NullabilityArg;
4567 }
4568
4569 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4570 llvm::Value *Cond = EmitNonNullRValueCheck(RV, T: ArgType);
4571 llvm::Constant *StaticData[] = {
4572 EmitCheckSourceLocation(Loc: ArgLoc),
4573 EmitCheckSourceLocation(Loc: AttrLoc),
4574 llvm::ConstantInt::get(Ty: Int32Ty, V: ArgNo + 1),
4575 };
4576 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: {});
4577}
4578
4579void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType,
4580 SourceLocation ArgLoc,
4581 AbstractCallee AC, unsigned ParmNum) {
4582 if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
4583 SanOpts.has(K: SanitizerKind::NullabilityArg)))
4584 return;
4585
4586 EmitNonNullArgCheck(RV: RValue::get(Addr, CGF&: *this), ArgType, ArgLoc, AC, ParmNum);
4587}
4588
4589// Check if the call is going to use the inalloca convention. This needs to
4590// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
4591// later, so we can't check it directly.
4592static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
4593 ArrayRef<QualType> ArgTypes) {
4594 // The Swift calling conventions don't go through the target-specific
4595 // argument classification, they never use inalloca.
4596 // TODO: Consider limiting inalloca use to only calling conventions supported
4597 // by MSVC.
4598 if (ExplicitCC == CC_Swift || ExplicitCC == CC_SwiftAsync)
4599 return false;
4600 if (!CGM.getTarget().getCXXABI().isMicrosoft())
4601 return false;
4602 return llvm::any_of(Range&: ArgTypes, P: [&](QualType Ty) {
4603 return isInAllocaArgument(ABI&: CGM.getCXXABI(), type: Ty);
4604 });
4605}
4606
4607#ifndef NDEBUG
4608// Determine whether the given argument is an Objective-C method
4609// that may have type parameters in its signature.
4610static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
4611 const DeclContext *dc = method->getDeclContext();
4612 if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {
4613 return classDecl->getTypeParamListAsWritten();
4614 }
4615
4616 if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
4617 return catDecl->getTypeParamList();
4618 }
4619
4620 return false;
4621}
4622#endif
4623
4624/// EmitCallArgs - Emit call arguments for a function.
4625void CodeGenFunction::EmitCallArgs(
4626 CallArgList &Args, PrototypeWrapper Prototype,
4627 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4628 AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
4629 SmallVector<QualType, 16> ArgTypes;
4630
4631 assert((ParamsToSkip == 0 || Prototype.P) &&
4632 "Can't skip parameters if type info is not provided");
4633
4634 // This variable only captures *explicitly* written conventions, not those
4635 // applied by default via command line flags or target defaults, such as
4636 // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
4637 // require knowing if this is a C++ instance method or being able to see
4638 // unprototyped FunctionTypes.
4639 CallingConv ExplicitCC = CC_C;
4640
4641 // First, if a prototype was provided, use those argument types.
4642 bool IsVariadic = false;
4643 if (Prototype.P) {
4644 const auto *MD = dyn_cast<const ObjCMethodDecl *>(Val&: Prototype.P);
4645 if (MD) {
4646 IsVariadic = MD->isVariadic();
4647 ExplicitCC = getCallingConventionForDecl(
4648 D: MD, IsTargetDefaultMSABI: CGM.getTarget().getTriple().isOSWindows());
4649 ArgTypes.assign(in_start: MD->param_type_begin() + ParamsToSkip,
4650 in_end: MD->param_type_end());
4651 } else {
4652 const auto *FPT = cast<const FunctionProtoType *>(Val&: Prototype.P);
4653 IsVariadic = FPT->isVariadic();
4654 ExplicitCC = FPT->getExtInfo().getCC();
4655 ArgTypes.assign(in_start: FPT->param_type_begin() + ParamsToSkip,
4656 in_end: FPT->param_type_end());
4657 }
4658
4659#ifndef NDEBUG
4660 // Check that the prototyped types match the argument expression types.
4661 bool isGenericMethod = MD && isObjCMethodWithTypeParams(MD);
4662 CallExpr::const_arg_iterator Arg = ArgRange.begin();
4663 for (QualType Ty : ArgTypes) {
4664 assert(Arg != ArgRange.end() && "Running over edge of argument list!");
4665 assert(
4666 (isGenericMethod || Ty->isVariablyModifiedType() ||
4667 Ty.getNonReferenceType()->isObjCRetainableType() ||
4668 getContext()
4669 .getCanonicalType(Ty.getNonReferenceType())
4670 .getTypePtr() ==
4671 getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) &&
4672 "type mismatch in call argument!");
4673 ++Arg;
4674 }
4675
4676 // Either we've emitted all the call args, or we have a call to variadic
4677 // function.
4678 assert((Arg == ArgRange.end() || IsVariadic) &&
4679 "Extra arguments in non-variadic function!");
4680#endif
4681 }
4682
4683 // If we still have any arguments, emit them using the type of the argument.
4684 for (auto *A : llvm::drop_begin(RangeOrContainer&: ArgRange, N: ArgTypes.size()))
4685 ArgTypes.push_back(Elt: IsVariadic ? getVarArgType(Arg: A) : A->getType());
4686 assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
4687
4688 // We must evaluate arguments from right to left in the MS C++ ABI,
4689 // because arguments are destroyed left to right in the callee. As a special
4690 // case, there are certain language constructs that require left-to-right
4691 // evaluation, and in those cases we consider the evaluation order requirement
4692 // to trump the "destruction order is reverse construction order" guarantee.
4693 bool LeftToRight =
4694 CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
4695 ? Order == EvaluationOrder::ForceLeftToRight
4696 : Order != EvaluationOrder::ForceRightToLeft;
4697
4698 auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
4699 RValue EmittedArg) {
4700 if (!AC.hasFunctionDecl() || I >= AC.getNumParams())
4701 return;
4702 auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
4703 if (PS == nullptr)
4704 return;
4705
4706 const auto &Context = getContext();
4707 auto SizeTy = Context.getSizeType();
4708 auto T = Builder.getIntNTy(N: Context.getTypeSize(T: SizeTy));
4709 assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
4710 llvm::Value *V = evaluateOrEmitBuiltinObjectSize(
4711 E: Arg, Type: PS->getType(), ResType: T, EmittedE: EmittedArg.getScalarVal(), IsDynamic: PS->isDynamic());
4712 Args.add(rvalue: RValue::get(V), type: SizeTy);
4713 // If we're emitting args in reverse, be sure to do so with
4714 // pass_object_size, as well.
4715 if (!LeftToRight)
4716 std::swap(a&: Args.back(), b&: *(&Args.back() - 1));
4717 };
4718
4719 // Insert a stack save if we're going to need any inalloca args.
4720 if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
4721 assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
4722 "inalloca only supported on x86");
4723 Args.allocateArgumentMemory(CGF&: *this);
4724 }
4725
4726 // Evaluate each argument in the appropriate order.
4727 size_t CallArgsStart = Args.size();
4728 for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
4729 unsigned Idx = LeftToRight ? I : E - I - 1;
4730 CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
4731 unsigned InitialArgSize = Args.size();
4732 // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
4733 // the argument and parameter match or the objc method is parameterized.
4734 assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
4735 getContext().hasSameUnqualifiedType((*Arg)->getType(),
4736 ArgTypes[Idx]) ||
4737 (isa<ObjCMethodDecl>(AC.getDecl()) &&
4738 isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
4739 "Argument and parameter types don't match");
4740 EmitCallArg(args&: Args, E: *Arg, ArgType: ArgTypes[Idx]);
4741 // In particular, we depend on it being the last arg in Args, and the
4742 // objectsize bits depend on there only being one arg if !LeftToRight.
4743 assert(InitialArgSize + 1 == Args.size() &&
4744 "The code below depends on only adding one arg per EmitCallArg");
4745 (void)InitialArgSize;
4746 // Since pointer argument are never emitted as LValue, it is safe to emit
4747 // non-null argument check for r-value only.
4748 if (!Args.back().hasLValue()) {
4749 RValue RVArg = Args.back().getKnownRValue();
4750 EmitNonNullArgCheck(RV: RVArg, ArgType: ArgTypes[Idx], ArgLoc: (*Arg)->getExprLoc(), AC,
4751 ParmNum: ParamsToSkip + Idx);
4752 // @llvm.objectsize should never have side-effects and shouldn't need
4753 // destruction/cleanups, so we can safely "emit" it after its arg,
4754 // regardless of right-to-leftness
4755 MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
4756 }
4757 }
4758
4759 if (!LeftToRight) {
4760 // Un-reverse the arguments we just evaluated so they match up with the LLVM
4761 // IR function.
4762 std::reverse(first: Args.begin() + CallArgsStart, last: Args.end());
4763
4764 // Reverse the writebacks to match the MSVC ABI.
4765 Args.reverseWritebacks();
4766 }
4767}
4768
4769namespace {
4770
4771struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
4772 DestroyUnpassedArg(Address Addr, QualType Ty) : Addr(Addr), Ty(Ty) {}
4773
4774 Address Addr;
4775 QualType Ty;
4776
4777 void Emit(CodeGenFunction &CGF, Flags flags) override {
4778 QualType::DestructionKind DtorKind = Ty.isDestructedType();
4779 if (DtorKind == QualType::DK_cxx_destructor) {
4780 const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
4781 assert(!Dtor->isTrivial());
4782 CGF.EmitCXXDestructorCall(D: Dtor, Type: Dtor_Complete, /*for vbase*/ ForVirtualBase: false,
4783 /*Delegating=*/false, This: Addr, ThisTy: Ty);
4784 } else {
4785 CGF.callCStructDestructor(Dst: CGF.MakeAddrLValue(Addr, T: Ty));
4786 }
4787 }
4788};
4789
4790struct DisableDebugLocationUpdates {
4791 CodeGenFunction &CGF;
4792 bool disabledDebugInfo;
4793 DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) {
4794 if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(Val: E) && CGF.getDebugInfo()))
4795 CGF.disableDebugInfo();
4796 }
4797 ~DisableDebugLocationUpdates() {
4798 if (disabledDebugInfo)
4799 CGF.enableDebugInfo();
4800 }
4801};
4802
4803} // end anonymous namespace
4804
4805RValue CallArg::getRValue(CodeGenFunction &CGF) const {
4806 if (!HasLV)
4807 return RV;
4808 LValue Copy = CGF.MakeAddrLValue(Addr: CGF.CreateMemTemp(T: Ty), T: Ty);
4809 CGF.EmitAggregateCopy(Dest: Copy, Src: LV, EltTy: Ty, MayOverlap: AggValueSlot::DoesNotOverlap,
4810 isVolatile: LV.isVolatile());
4811 IsUsed = true;
4812 return RValue::getAggregate(addr: Copy.getAddress());
4813}
4814
4815void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
4816 LValue Dst = CGF.MakeAddrLValue(Addr, T: Ty);
4817 if (!HasLV && RV.isScalar())
4818 CGF.EmitStoreOfScalar(value: RV.getScalarVal(), lvalue: Dst, /*isInit=*/true);
4819 else if (!HasLV && RV.isComplex())
4820 CGF.EmitStoreOfComplex(V: RV.getComplexVal(), dest: Dst, /*init=*/isInit: true);
4821 else {
4822 auto Addr = HasLV ? LV.getAddress() : RV.getAggregateAddress();
4823 LValue SrcLV = CGF.MakeAddrLValue(Addr, T: Ty);
4824 // We assume that call args are never copied into subobjects.
4825 CGF.EmitAggregateCopy(Dest: Dst, Src: SrcLV, EltTy: Ty, MayOverlap: AggValueSlot::DoesNotOverlap,
4826 isVolatile: HasLV ? LV.isVolatileQualified()
4827 : RV.isVolatileQualified());
4828 }
4829 IsUsed = true;
4830}
4831
4832void CodeGenFunction::EmitWritebacks(const CallArgList &args) {
4833 for (const auto &I : args.writebacks())
4834 emitWriteback(CGF&: *this, writeback: I);
4835}
4836
4837void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
4838 QualType type) {
4839 DisableDebugLocationUpdates Dis(*this, E);
4840 if (const ObjCIndirectCopyRestoreExpr *CRE =
4841 dyn_cast<ObjCIndirectCopyRestoreExpr>(Val: E)) {
4842 assert(getLangOpts().ObjCAutoRefCount);
4843 return emitWritebackArg(CGF&: *this, args, CRE);
4844 }
4845
4846 // Add writeback for HLSLOutParamExpr.
4847 // Needs to be before the assert below because HLSLOutArgExpr is an LValue
4848 // and is not a reference.
4849 if (const HLSLOutArgExpr *OE = dyn_cast<HLSLOutArgExpr>(Val: E)) {
4850 EmitHLSLOutArgExpr(E: OE, Args&: args, Ty: type);
4851 return;
4852 }
4853
4854 assert(type->isReferenceType() == E->isGLValue() &&
4855 "reference binding to unmaterialized r-value!");
4856
4857 if (E->isGLValue()) {
4858 assert(E->getObjectKind() == OK_Ordinary);
4859 return args.add(rvalue: EmitReferenceBindingToExpr(E), type);
4860 }
4861
4862 bool HasAggregateEvalKind = hasAggregateEvaluationKind(T: type);
4863
4864 // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
4865 // However, we still have to push an EH-only cleanup in case we unwind before
4866 // we make it to the call.
4867 if (type->isRecordType() &&
4868 type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()) {
4869 // If we're using inalloca, use the argument memory. Otherwise, use a
4870 // temporary.
4871 AggValueSlot Slot = args.isUsingInAlloca()
4872 ? createPlaceholderSlot(CGF&: *this, Ty: type)
4873 : CreateAggTemp(T: type, Name: "agg.tmp");
4874
4875 bool DestroyedInCallee = true, NeedsCleanup = true;
4876 if (const auto *RD = type->getAsCXXRecordDecl())
4877 DestroyedInCallee = RD->hasNonTrivialDestructor();
4878 else
4879 NeedsCleanup = type.isDestructedType();
4880
4881 if (DestroyedInCallee)
4882 Slot.setExternallyDestructed();
4883
4884 EmitAggExpr(E, AS: Slot);
4885 RValue RV = Slot.asRValue();
4886 args.add(rvalue: RV, type);
4887
4888 if (DestroyedInCallee && NeedsCleanup) {
4889 // Create a no-op GEP between the placeholder and the cleanup so we can
4890 // RAUW it successfully. It also serves as a marker of the first
4891 // instruction where the cleanup is active.
4892 pushFullExprCleanup<DestroyUnpassedArg>(kind: NormalAndEHCleanup,
4893 A: Slot.getAddress(), A: type);
4894 // This unreachable is a temporary marker which will be removed later.
4895 llvm::Instruction *IsActive =
4896 Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
4897 args.addArgCleanupDeactivation(Cleanup: EHStack.stable_begin(), IsActiveIP: IsActive);
4898 }
4899 return;
4900 }
4901
4902 if (HasAggregateEvalKind && isa<ImplicitCastExpr>(Val: E) &&
4903 cast<CastExpr>(Val: E)->getCastKind() == CK_LValueToRValue &&
4904 !type->isArrayParameterType() && !type.isNonTrivialToPrimitiveCopy()) {
4905 LValue L = EmitLValue(E: cast<CastExpr>(Val: E)->getSubExpr());
4906 assert(L.isSimple());
4907 args.addUncopiedAggregate(LV: L, type);
4908 return;
4909 }
4910
4911 args.add(rvalue: EmitAnyExprToTemp(E), type);
4912}
4913
4914QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
4915 // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
4916 // implicitly widens null pointer constants that are arguments to varargs
4917 // functions to pointer-sized ints.
4918 if (!getTarget().getTriple().isOSWindows())
4919 return Arg->getType();
4920
4921 if (Arg->getType()->isIntegerType() &&
4922 getContext().getTypeSize(T: Arg->getType()) <
4923 getContext().getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default) &&
4924 Arg->isNullPointerConstant(Ctx&: getContext(),
4925 NPC: Expr::NPC_ValueDependentIsNotNull)) {
4926 return getContext().getIntPtrType();
4927 }
4928
4929 return Arg->getType();
4930}
4931
4932// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4933// optimizer it can aggressively ignore unwind edges.
4934void CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
4935 if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4936 !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
4937 Inst->setMetadata(Kind: "clang.arc.no_objc_arc_exceptions",
4938 Node: CGM.getNoObjCARCExceptionsMetadata());
4939}
4940
4941/// Emits a call to the given no-arguments nounwind runtime function.
4942llvm::CallInst *
4943CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4944 const llvm::Twine &name) {
4945 return EmitNounwindRuntimeCall(callee, args: ArrayRef<llvm::Value *>(), name);
4946}
4947
4948/// Emits a call to the given nounwind runtime function.
4949llvm::CallInst *
4950CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4951 ArrayRef<Address> args,
4952 const llvm::Twine &name) {
4953 SmallVector<llvm::Value *, 3> values;
4954 for (auto arg : args)
4955 values.push_back(Elt: arg.emitRawPointer(CGF&: *this));
4956 return EmitNounwindRuntimeCall(callee, args: values, name);
4957}
4958
4959llvm::CallInst *
4960CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4961 ArrayRef<llvm::Value *> args,
4962 const llvm::Twine &name) {
4963 llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
4964 call->setDoesNotThrow();
4965 return call;
4966}
4967
4968/// Emits a simple call (never an invoke) to the given no-arguments
4969/// runtime function.
4970llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4971 const llvm::Twine &name) {
4972 return EmitRuntimeCall(callee, args: {}, name);
4973}
4974
4975// Calls which may throw must have operand bundles indicating which funclet
4976// they are nested within.
4977SmallVector<llvm::OperandBundleDef, 1>
4978CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
4979 // There is no need for a funclet operand bundle if we aren't inside a
4980 // funclet.
4981 if (!CurrentFuncletPad)
4982 return (SmallVector<llvm::OperandBundleDef, 1>());
4983
4984 // Skip intrinsics which cannot throw (as long as they don't lower into
4985 // regular function calls in the course of IR transformations).
4986 if (auto *CalleeFn = dyn_cast<llvm::Function>(Val: Callee->stripPointerCasts())) {
4987 if (CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) {
4988 auto IID = CalleeFn->getIntrinsicID();
4989 if (!llvm::IntrinsicInst::mayLowerToFunctionCall(IID))
4990 return (SmallVector<llvm::OperandBundleDef, 1>());
4991 }
4992 }
4993
4994 SmallVector<llvm::OperandBundleDef, 1> BundleList;
4995 BundleList.emplace_back(Args: "funclet", Args&: CurrentFuncletPad);
4996 return BundleList;
4997}
4998
4999/// Emits a simple call (never an invoke) to the given runtime function.
5000llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5001 ArrayRef<llvm::Value *> args,
5002 const llvm::Twine &name) {
5003 llvm::CallInst *call = Builder.CreateCall(
5004 Callee: callee, Args: args, OpBundles: getBundlesForFunclet(Callee: callee.getCallee()), Name: name);
5005 call->setCallingConv(getRuntimeCC());
5006
5007 if (CGM.shouldEmitConvergenceTokens() && call->isConvergent())
5008 return cast<llvm::CallInst>(Val: addConvergenceControlToken(Input: call));
5009 return call;
5010}
5011
5012/// Emits a call or invoke to the given noreturn runtime function.
5013void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
5014 llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
5015 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5016 getBundlesForFunclet(Callee: callee.getCallee());
5017
5018 if (getInvokeDest()) {
5019 llvm::InvokeInst *invoke = Builder.CreateInvoke(
5020 Callee: callee, NormalDest: getUnreachableBlock(), UnwindDest: getInvokeDest(), Args: args, OpBundles: BundleList);
5021 invoke->setDoesNotReturn();
5022 invoke->setCallingConv(getRuntimeCC());
5023 } else {
5024 llvm::CallInst *call = Builder.CreateCall(Callee: callee, Args: args, OpBundles: BundleList);
5025 call->setDoesNotReturn();
5026 call->setCallingConv(getRuntimeCC());
5027 Builder.CreateUnreachable();
5028 }
5029}
5030
5031/// Emits a call or invoke instruction to the given nullary runtime function.
5032llvm::CallBase *
5033CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5034 const Twine &name) {
5035 return EmitRuntimeCallOrInvoke(callee, args: {}, name);
5036}
5037
5038/// Emits a call or invoke instruction to the given runtime function.
5039llvm::CallBase *
5040CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5041 ArrayRef<llvm::Value *> args,
5042 const Twine &name) {
5043 llvm::CallBase *call = EmitCallOrInvoke(Callee: callee, Args: args, Name: name);
5044 call->setCallingConv(getRuntimeCC());
5045 return call;
5046}
5047
5048/// Emits a call or invoke instruction to the given function, depending
5049/// on the current state of the EH stack.
5050llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
5051 ArrayRef<llvm::Value *> Args,
5052 const Twine &Name) {
5053 llvm::BasicBlock *InvokeDest = getInvokeDest();
5054 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5055 getBundlesForFunclet(Callee: Callee.getCallee());
5056
5057 llvm::CallBase *Inst;
5058 if (!InvokeDest)
5059 Inst = Builder.CreateCall(Callee, Args, OpBundles: BundleList, Name);
5060 else {
5061 llvm::BasicBlock *ContBB = createBasicBlock(name: "invoke.cont");
5062 Inst = Builder.CreateInvoke(Callee, NormalDest: ContBB, UnwindDest: InvokeDest, Args, OpBundles: BundleList,
5063 Name);
5064 EmitBlock(BB: ContBB);
5065 }
5066
5067 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5068 // optimizer it can aggressively ignore unwind edges.
5069 if (CGM.getLangOpts().ObjCAutoRefCount)
5070 AddObjCARCExceptionMetadata(Inst);
5071
5072 return Inst;
5073}
5074
5075void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
5076 llvm::Value *New) {
5077 DeferredReplacements.push_back(
5078 Elt: std::make_pair(x: llvm::WeakTrackingVH(Old), y&: New));
5079}
5080
5081namespace {
5082
5083/// Specify given \p NewAlign as the alignment of return value attribute. If
5084/// such attribute already exists, re-set it to the maximal one of two options.
5085[[nodiscard]] llvm::AttributeList
5086maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
5087 const llvm::AttributeList &Attrs,
5088 llvm::Align NewAlign) {
5089 llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
5090 if (CurAlign >= NewAlign)
5091 return Attrs;
5092 llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Context&: Ctx, Alignment: NewAlign);
5093 return Attrs.removeRetAttribute(C&: Ctx, Kind: llvm::Attribute::AttrKind::Alignment)
5094 .addRetAttribute(C&: Ctx, Attr: AlignAttr);
5095}
5096
5097template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
5098protected:
5099 CodeGenFunction &CGF;
5100
5101 /// We do nothing if this is, or becomes, nullptr.
5102 const AlignedAttrTy *AA = nullptr;
5103
5104 llvm::Value *Alignment = nullptr; // May or may not be a constant.
5105 llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
5106
5107 AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5108 : CGF(CGF_) {
5109 if (!FuncDecl)
5110 return;
5111 AA = FuncDecl->getAttr<AlignedAttrTy>();
5112 }
5113
5114public:
5115 /// If we can, materialize the alignment as an attribute on return value.
5116 [[nodiscard]] llvm::AttributeList
5117 TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
5118 if (!AA || OffsetCI || CGF.SanOpts.has(K: SanitizerKind::Alignment))
5119 return Attrs;
5120 const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Val: Alignment);
5121 if (!AlignmentCI)
5122 return Attrs;
5123 // We may legitimately have non-power-of-2 alignment here.
5124 // If so, this is UB land, emit it via `@llvm.assume` instead.
5125 if (!AlignmentCI->getValue().isPowerOf2())
5126 return Attrs;
5127 llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
5128 Ctx&: CGF.getLLVMContext(), Attrs,
5129 NewAlign: llvm::Align(
5130 AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment)));
5131 AA = nullptr; // We're done. Disallow doing anything else.
5132 return NewAttrs;
5133 }
5134
5135 /// Emit alignment assumption.
5136 /// This is a general fallback that we take if either there is an offset,
5137 /// or the alignment is variable or we are sanitizing for alignment.
5138 void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
5139 if (!AA)
5140 return;
5141 CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
5142 AA->getLocation(), Alignment, OffsetCI);
5143 AA = nullptr; // We're done. Disallow doing anything else.
5144 }
5145};
5146
5147/// Helper data structure to emit `AssumeAlignedAttr`.
5148class AssumeAlignedAttrEmitter final
5149 : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
5150public:
5151 AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5152 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5153 if (!AA)
5154 return;
5155 // It is guaranteed that the alignment/offset are constants.
5156 Alignment = cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: AA->getAlignment()));
5157 if (Expr *Offset = AA->getOffset()) {
5158 OffsetCI = cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: Offset));
5159 if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
5160 OffsetCI = nullptr;
5161 }
5162 }
5163};
5164
5165/// Helper data structure to emit `AllocAlignAttr`.
5166class AllocAlignAttrEmitter final
5167 : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
5168public:
5169 AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
5170 const CallArgList &CallArgs)
5171 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5172 if (!AA)
5173 return;
5174 // Alignment may or may not be a constant, and that is okay.
5175 Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
5176 .getRValue(CGF)
5177 .getScalarVal();
5178 }
5179};
5180
5181} // namespace
5182
5183static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
5184 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Ty))
5185 return VT->getPrimitiveSizeInBits().getKnownMinValue();
5186 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty))
5187 return getMaxVectorWidth(Ty: AT->getElementType());
5188
5189 unsigned MaxVectorWidth = 0;
5190 if (auto *ST = dyn_cast<llvm::StructType>(Val: Ty))
5191 for (auto *I : ST->elements())
5192 MaxVectorWidth = std::max(a: MaxVectorWidth, b: getMaxVectorWidth(Ty: I));
5193 return MaxVectorWidth;
5194}
5195
5196RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
5197 const CGCallee &Callee,
5198 ReturnValueSlot ReturnValue,
5199 const CallArgList &CallArgs,
5200 llvm::CallBase **callOrInvoke, bool IsMustTail,
5201 SourceLocation Loc,
5202 bool IsVirtualFunctionPointerThunk) {
5203 // FIXME: We no longer need the types from CallArgs; lift up and simplify.
5204
5205 assert(Callee.isOrdinary() || Callee.isVirtual());
5206
5207 // Handle struct-return functions by passing a pointer to the
5208 // location that we would like to return into.
5209 QualType RetTy = CallInfo.getReturnType();
5210 const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
5211
5212 llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(FI: CallInfo);
5213
5214 const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
5215 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
5216 // We can only guarantee that a function is called from the correct
5217 // context/function based on the appropriate target attributes,
5218 // so only check in the case where we have both always_inline and target
5219 // since otherwise we could be making a conditional call after a check for
5220 // the proper cpu features (and it won't cause code generation issues due to
5221 // function based code generation).
5222 if (TargetDecl->hasAttr<AlwaysInlineAttr>() &&
5223 (TargetDecl->hasAttr<TargetAttr>() ||
5224 (CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>())))
5225 checkTargetFeatures(Loc, TargetDecl: FD);
5226 }
5227
5228 // Some architectures (such as x86-64) have the ABI changed based on
5229 // attribute-target/features. Give them a chance to diagnose.
5230 const FunctionDecl *CallerDecl = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
5231 const FunctionDecl *CalleeDecl = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
5232 CGM.getTargetCodeGenInfo().checkFunctionCallABI(CGM, CallLoc: Loc, Caller: CallerDecl,
5233 Callee: CalleeDecl, Args: CallArgs, ReturnType: RetTy);
5234
5235 // 1. Set up the arguments.
5236
5237 // If we're using inalloca, insert the allocation after the stack save.
5238 // FIXME: Do this earlier rather than hacking it in here!
5239 RawAddress ArgMemory = RawAddress::invalid();
5240 if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
5241 const llvm::DataLayout &DL = CGM.getDataLayout();
5242 llvm::Instruction *IP = CallArgs.getStackBase();
5243 llvm::AllocaInst *AI;
5244 if (IP) {
5245 IP = IP->getNextNode();
5246 AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(), "argmem",
5247 IP->getIterator());
5248 } else {
5249 AI = CreateTempAlloca(Ty: ArgStruct, Name: "argmem");
5250 }
5251 auto Align = CallInfo.getArgStructAlignment();
5252 AI->setAlignment(Align.getAsAlign());
5253 AI->setUsedWithInAlloca(true);
5254 assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
5255 ArgMemory = RawAddress(AI, ArgStruct, Align);
5256 }
5257
5258 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
5259 SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
5260
5261 // If the call returns a temporary with struct return, create a temporary
5262 // alloca to hold the result, unless one is given to us.
5263 Address SRetPtr = Address::invalid();
5264 llvm::Value *UnusedReturnSizePtr = nullptr;
5265 if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
5266 // For virtual function pointer thunks and musttail calls, we must always
5267 // forward an incoming SRet pointer to the callee, because a local alloca
5268 // would be de-allocated before the call. These cases both guarantee that
5269 // there will be an incoming SRet argument of the correct type.
5270 if ((IsVirtualFunctionPointerThunk || IsMustTail) && RetAI.isIndirect()) {
5271 SRetPtr = makeNaturalAddressForPointer(Ptr: CurFn->arg_begin() +
5272 IRFunctionArgs.getSRetArgNo(),
5273 T: RetTy, Alignment: CharUnits::fromQuantity(Quantity: 1));
5274 } else if (!ReturnValue.isNull()) {
5275 SRetPtr = ReturnValue.getAddress();
5276 } else {
5277 SRetPtr = CreateMemTempWithoutCast(T: RetTy, Name: "tmp");
5278 if (HaveInsertPoint() && ReturnValue.isUnused()) {
5279 llvm::TypeSize size =
5280 CGM.getDataLayout().getTypeAllocSize(Ty: ConvertTypeForMem(T: RetTy));
5281 UnusedReturnSizePtr = EmitLifetimeStart(Size: size, Addr: SRetPtr.getBasePointer());
5282 }
5283 }
5284 if (IRFunctionArgs.hasSRetArg()) {
5285 // A mismatch between the allocated return value's AS and the target's
5286 // chosen IndirectAS can happen e.g. when passing the this pointer through
5287 // a chain involving stores to / loads from the DefaultAS; we address this
5288 // here, symmetrically with the handling we have for normal pointer args.
5289 if (SRetPtr.getAddressSpace() != RetAI.getIndirectAddrSpace()) {
5290 llvm::Value *V = SRetPtr.getBasePointer();
5291 LangAS SAS = getLangASFromTargetAS(TargetAS: SRetPtr.getAddressSpace());
5292 llvm::Type *Ty = llvm::PointerType::get(C&: getLLVMContext(),
5293 AddressSpace: RetAI.getIndirectAddrSpace());
5294
5295 SRetPtr = SRetPtr.withPointer(
5296 NewPointer: getTargetHooks().performAddrSpaceCast(CGF&: *this, V, SrcAddr: SAS, DestTy: Ty, IsNonNull: true),
5297 IsKnownNonNull: SRetPtr.isKnownNonNull());
5298 }
5299 IRCallArgs[IRFunctionArgs.getSRetArgNo()] =
5300 getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy);
5301 } else if (RetAI.isInAlloca()) {
5302 Address Addr =
5303 Builder.CreateStructGEP(Addr: ArgMemory, Index: RetAI.getInAllocaFieldIndex());
5304 Builder.CreateStore(Val: getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy), Addr);
5305 }
5306 }
5307
5308 RawAddress swiftErrorTemp = RawAddress::invalid();
5309 Address swiftErrorArg = Address::invalid();
5310
5311 // When passing arguments using temporary allocas, we need to add the
5312 // appropriate lifetime markers. This vector keeps track of all the lifetime
5313 // markers that need to be ended right after the call.
5314 SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
5315
5316 // Translate all of the arguments as necessary to match the IR lowering.
5317 assert(CallInfo.arg_size() == CallArgs.size() &&
5318 "Mismatch between function signature & arguments.");
5319 unsigned ArgNo = 0;
5320 CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
5321 for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
5322 I != E; ++I, ++info_it, ++ArgNo) {
5323 const ABIArgInfo &ArgInfo = info_it->info;
5324
5325 // Insert a padding argument to ensure proper alignment.
5326 if (IRFunctionArgs.hasPaddingArg(ArgNo))
5327 IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
5328 llvm::UndefValue::get(T: ArgInfo.getPaddingType());
5329
5330 unsigned FirstIRArg, NumIRArgs;
5331 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
5332
5333 bool ArgHasMaybeUndefAttr =
5334 IsArgumentMaybeUndef(TargetDecl, NumRequiredArgs: CallInfo.getNumRequiredArgs(), ArgNo);
5335
5336 switch (ArgInfo.getKind()) {
5337 case ABIArgInfo::InAlloca: {
5338 assert(NumIRArgs == 0);
5339 assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
5340 if (I->isAggregate()) {
5341 RawAddress Addr = I->hasLValue()
5342 ? I->getKnownLValue().getAddress()
5343 : I->getKnownRValue().getAggregateAddress();
5344 llvm::Instruction *Placeholder =
5345 cast<llvm::Instruction>(Val: Addr.getPointer());
5346
5347 if (!ArgInfo.getInAllocaIndirect()) {
5348 // Replace the placeholder with the appropriate argument slot GEP.
5349 CGBuilderTy::InsertPoint IP = Builder.saveIP();
5350 Builder.SetInsertPoint(Placeholder);
5351 Addr = Builder.CreateStructGEP(Addr: ArgMemory,
5352 Index: ArgInfo.getInAllocaFieldIndex());
5353 Builder.restoreIP(IP);
5354 } else {
5355 // For indirect things such as overaligned structs, replace the
5356 // placeholder with a regular aggregate temporary alloca. Store the
5357 // address of this alloca into the struct.
5358 Addr = CreateMemTemp(T: info_it->type, Name: "inalloca.indirect.tmp");
5359 Address ArgSlot = Builder.CreateStructGEP(
5360 Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5361 Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5362 }
5363 deferPlaceholderReplacement(Old: Placeholder, New: Addr.getPointer());
5364 } else if (ArgInfo.getInAllocaIndirect()) {
5365 // Make a temporary alloca and store the address of it into the argument
5366 // struct.
5367 RawAddress Addr = CreateMemTempWithoutCast(
5368 T: I->Ty, Align: getContext().getTypeAlignInChars(T: I->Ty),
5369 Name: "indirect-arg-temp");
5370 I->copyInto(CGF&: *this, Addr);
5371 Address ArgSlot =
5372 Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5373 Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5374 } else {
5375 // Store the RValue into the argument struct.
5376 Address Addr =
5377 Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5378 Addr = Addr.withElementType(ElemTy: ConvertTypeForMem(T: I->Ty));
5379 I->copyInto(CGF&: *this, Addr);
5380 }
5381 break;
5382 }
5383
5384 case ABIArgInfo::Indirect:
5385 case ABIArgInfo::IndirectAliased: {
5386 assert(NumIRArgs == 1);
5387 if (I->isAggregate()) {
5388 // We want to avoid creating an unnecessary temporary+copy here;
5389 // however, we need one in three cases:
5390 // 1. If the argument is not byval, and we are required to copy the
5391 // source. (This case doesn't occur on any common architecture.)
5392 // 2. If the argument is byval, RV is not sufficiently aligned, and
5393 // we cannot force it to be sufficiently aligned.
5394 // 3. If the argument is byval, but RV is not located in default
5395 // or alloca address space.
5396 Address Addr = I->hasLValue()
5397 ? I->getKnownLValue().getAddress()
5398 : I->getKnownRValue().getAggregateAddress();
5399 CharUnits Align = ArgInfo.getIndirectAlign();
5400 const llvm::DataLayout *TD = &CGM.getDataLayout();
5401
5402 assert((FirstIRArg >= IRFuncTy->getNumParams() ||
5403 IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
5404 TD->getAllocaAddrSpace()) &&
5405 "indirect argument must be in alloca address space");
5406
5407 bool NeedCopy = false;
5408 if (Addr.getAlignment() < Align &&
5409 llvm::getOrEnforceKnownAlignment(V: Addr.emitRawPointer(CGF&: *this),
5410 PrefAlign: Align.getAsAlign(),
5411 DL: *TD) < Align.getAsAlign()) {
5412 NeedCopy = true;
5413 } else if (I->hasLValue()) {
5414 auto LV = I->getKnownLValue();
5415
5416 bool isByValOrRef =
5417 ArgInfo.isIndirectAliased() || ArgInfo.getIndirectByVal();
5418
5419 if (!isByValOrRef ||
5420 (LV.getAlignment() < getContext().getTypeAlignInChars(T: I->Ty))) {
5421 NeedCopy = true;
5422 }
5423
5424 if (isByValOrRef && Addr.getType()->getAddressSpace() !=
5425 ArgInfo.getIndirectAddrSpace()) {
5426 NeedCopy = true;
5427 }
5428 }
5429
5430 if (!NeedCopy) {
5431 // Skip the extra memcpy call.
5432 llvm::Value *V = getAsNaturalPointerTo(Addr, PointeeType: I->Ty);
5433 auto *T = llvm::PointerType::get(C&: CGM.getLLVMContext(),
5434 AddressSpace: ArgInfo.getIndirectAddrSpace());
5435
5436 // FIXME: This should not depend on the language address spaces, and
5437 // only the contextual values. If the address space mismatches, see if
5438 // we can look through a cast to a compatible address space value,
5439 // otherwise emit a copy.
5440 llvm::Value *Val = getTargetHooks().performAddrSpaceCast(
5441 CGF&: *this, V, SrcAddr: I->Ty.getAddressSpace(), DestTy: T, IsNonNull: true);
5442 if (ArgHasMaybeUndefAttr)
5443 Val = Builder.CreateFreeze(V: Val);
5444 IRCallArgs[FirstIRArg] = Val;
5445 break;
5446 }
5447 } else if (I->getType()->isArrayParameterType()) {
5448 // Don't produce a temporary for ArrayParameterType arguments.
5449 // ArrayParameterType arguments are only created from
5450 // HLSL_ArrayRValue casts and HLSLOutArgExpr expressions, both
5451 // of which create temporaries already. This allows us to just use the
5452 // scalar for the decayed array pointer as the argument directly.
5453 IRCallArgs[FirstIRArg] = I->getKnownRValue().getScalarVal();
5454 break;
5455 }
5456
5457 // For non-aggregate args and aggregate args meeting conditions above
5458 // we need to create an aligned temporary, and copy to it.
5459 RawAddress AI = CreateMemTempWithoutCast(
5460 T: I->Ty, Align: ArgInfo.getIndirectAlign(), Name: "byval-temp");
5461 llvm::Value *Val = getAsNaturalPointerTo(Addr: AI, PointeeType: I->Ty);
5462 if (ArgHasMaybeUndefAttr)
5463 Val = Builder.CreateFreeze(V: Val);
5464 IRCallArgs[FirstIRArg] = Val;
5465
5466 // Emit lifetime markers for the temporary alloca.
5467 llvm::TypeSize ByvalTempElementSize =
5468 CGM.getDataLayout().getTypeAllocSize(Ty: AI.getElementType());
5469 llvm::Value *LifetimeSize =
5470 EmitLifetimeStart(Size: ByvalTempElementSize, Addr: AI.getPointer());
5471
5472 // Add cleanup code to emit the end lifetime marker after the call.
5473 if (LifetimeSize) // In case we disabled lifetime markers.
5474 CallLifetimeEndAfterCall.emplace_back(Args&: AI, Args&: LifetimeSize);
5475
5476 // Generate the copy.
5477 I->copyInto(CGF&: *this, Addr: AI);
5478 break;
5479 }
5480
5481 case ABIArgInfo::Ignore:
5482 assert(NumIRArgs == 0);
5483 break;
5484
5485 case ABIArgInfo::Extend:
5486 case ABIArgInfo::Direct: {
5487 if (!isa<llvm::StructType>(Val: ArgInfo.getCoerceToType()) &&
5488 ArgInfo.getCoerceToType() == ConvertType(T: info_it->type) &&
5489 ArgInfo.getDirectOffset() == 0) {
5490 assert(NumIRArgs == 1);
5491 llvm::Value *V;
5492 if (!I->isAggregate())
5493 V = I->getKnownRValue().getScalarVal();
5494 else
5495 V = Builder.CreateLoad(
5496 Addr: I->hasLValue() ? I->getKnownLValue().getAddress()
5497 : I->getKnownRValue().getAggregateAddress());
5498
5499 // Implement swifterror by copying into a new swifterror argument.
5500 // We'll write back in the normal path out of the call.
5501 if (CallInfo.getExtParameterInfo(argIndex: ArgNo).getABI() ==
5502 ParameterABI::SwiftErrorResult) {
5503 assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
5504
5505 QualType pointeeTy = I->Ty->getPointeeType();
5506 swiftErrorArg = makeNaturalAddressForPointer(
5507 Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
5508
5509 swiftErrorTemp =
5510 CreateMemTemp(T: pointeeTy, Align: getPointerAlign(), Name: "swifterror.temp");
5511 V = swiftErrorTemp.getPointer();
5512 cast<llvm::AllocaInst>(Val: V)->setSwiftError(true);
5513
5514 llvm::Value *errorValue = Builder.CreateLoad(Addr: swiftErrorArg);
5515 Builder.CreateStore(Val: errorValue, Addr: swiftErrorTemp);
5516 }
5517
5518 // We might have to widen integers, but we should never truncate.
5519 if (ArgInfo.getCoerceToType() != V->getType() &&
5520 V->getType()->isIntegerTy())
5521 V = Builder.CreateZExt(V, DestTy: ArgInfo.getCoerceToType());
5522
5523 // The only plausible mismatch here would be for pointer address spaces.
5524 // We assume that the target has a reasonable mapping for the DefaultAS
5525 // (it can be casted to from incoming specific ASes), and insert an AS
5526 // cast to address the mismatch.
5527 if (FirstIRArg < IRFuncTy->getNumParams() &&
5528 V->getType() != IRFuncTy->getParamType(i: FirstIRArg)) {
5529 assert(V->getType()->isPointerTy() && "Only pointers can mismatch!");
5530 auto ActualAS = I->Ty.getAddressSpace();
5531 V = getTargetHooks().performAddrSpaceCast(
5532 CGF&: *this, V, SrcAddr: ActualAS, DestTy: IRFuncTy->getParamType(i: FirstIRArg));
5533 }
5534
5535 if (ArgHasMaybeUndefAttr)
5536 V = Builder.CreateFreeze(V);
5537 IRCallArgs[FirstIRArg] = V;
5538 break;
5539 }
5540
5541 llvm::StructType *STy =
5542 dyn_cast<llvm::StructType>(Val: ArgInfo.getCoerceToType());
5543
5544 // FIXME: Avoid the conversion through memory if possible.
5545 Address Src = Address::invalid();
5546 if (!I->isAggregate()) {
5547 Src = CreateMemTemp(T: I->Ty, Name: "coerce");
5548 I->copyInto(CGF&: *this, Addr: Src);
5549 } else {
5550 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
5551 : I->getKnownRValue().getAggregateAddress();
5552 }
5553
5554 // If the value is offset in memory, apply the offset now.
5555 Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
5556
5557 // Fast-isel and the optimizer generally like scalar values better than
5558 // FCAs, so we flatten them if this is safe to do for this argument.
5559 if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
5560 llvm::Type *SrcTy = Src.getElementType();
5561 llvm::TypeSize SrcTypeSize =
5562 CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
5563 llvm::TypeSize DstTypeSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
5564 if (SrcTypeSize.isScalable()) {
5565 assert(STy->containsHomogeneousScalableVectorTypes() &&
5566 "ABI only supports structure with homogeneous scalable vector "
5567 "type");
5568 assert(SrcTypeSize == DstTypeSize &&
5569 "Only allow non-fractional movement of structure with "
5570 "homogeneous scalable vector type");
5571 assert(NumIRArgs == STy->getNumElements());
5572
5573 llvm::Value *StoredStructValue =
5574 Builder.CreateLoad(Addr: Src, Name: Src.getName() + ".tuple");
5575 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5576 llvm::Value *Extract = Builder.CreateExtractValue(
5577 Agg: StoredStructValue, Idxs: i, Name: Src.getName() + ".extract" + Twine(i));
5578 IRCallArgs[FirstIRArg + i] = Extract;
5579 }
5580 } else {
5581 uint64_t SrcSize = SrcTypeSize.getFixedValue();
5582 uint64_t DstSize = DstTypeSize.getFixedValue();
5583
5584 // If the source type is smaller than the destination type of the
5585 // coerce-to logic, copy the source value into a temp alloca the size
5586 // of the destination type to allow loading all of it. The bits past
5587 // the source value are left undef.
5588 if (SrcSize < DstSize) {
5589 Address TempAlloca = CreateTempAlloca(Ty: STy, align: Src.getAlignment(),
5590 Name: Src.getName() + ".coerce");
5591 Builder.CreateMemCpy(Dest: TempAlloca, Src, Size: SrcSize);
5592 Src = TempAlloca;
5593 } else {
5594 Src = Src.withElementType(ElemTy: STy);
5595 }
5596
5597 assert(NumIRArgs == STy->getNumElements());
5598 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5599 Address EltPtr = Builder.CreateStructGEP(Addr: Src, Index: i);
5600 llvm::Value *LI = Builder.CreateLoad(Addr: EltPtr);
5601 if (ArgHasMaybeUndefAttr)
5602 LI = Builder.CreateFreeze(V: LI);
5603 IRCallArgs[FirstIRArg + i] = LI;
5604 }
5605 }
5606 } else {
5607 // In the simple case, just pass the coerced loaded value.
5608 assert(NumIRArgs == 1);
5609 llvm::Value *Load =
5610 CreateCoercedLoad(Src, Ty: ArgInfo.getCoerceToType(), CGF&: *this);
5611
5612 if (CallInfo.isCmseNSCall()) {
5613 // For certain parameter types, clear padding bits, as they may reveal
5614 // sensitive information.
5615 // Small struct/union types are passed as integer arrays.
5616 auto *ATy = dyn_cast<llvm::ArrayType>(Val: Load->getType());
5617 if (ATy != nullptr && isa<RecordType>(Val: I->Ty.getCanonicalType()))
5618 Load = EmitCMSEClearRecord(Src: Load, ATy, QTy: I->Ty);
5619 }
5620
5621 if (ArgHasMaybeUndefAttr)
5622 Load = Builder.CreateFreeze(V: Load);
5623 IRCallArgs[FirstIRArg] = Load;
5624 }
5625
5626 break;
5627 }
5628
5629 case ABIArgInfo::CoerceAndExpand: {
5630 auto coercionType = ArgInfo.getCoerceAndExpandType();
5631 auto layout = CGM.getDataLayout().getStructLayout(Ty: coercionType);
5632 auto unpaddedCoercionType = ArgInfo.getUnpaddedCoerceAndExpandType();
5633 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
5634
5635 llvm::Value *tempSize = nullptr;
5636 Address addr = Address::invalid();
5637 RawAddress AllocaAddr = RawAddress::invalid();
5638 if (I->isAggregate()) {
5639 addr = I->hasLValue() ? I->getKnownLValue().getAddress()
5640 : I->getKnownRValue().getAggregateAddress();
5641
5642 } else {
5643 RValue RV = I->getKnownRValue();
5644 assert(RV.isScalar()); // complex should always just be direct
5645
5646 llvm::Type *scalarType = RV.getScalarVal()->getType();
5647 auto scalarSize = CGM.getDataLayout().getTypeAllocSize(Ty: scalarType);
5648 auto scalarAlign = CGM.getDataLayout().getPrefTypeAlign(Ty: scalarType);
5649
5650 // Materialize to a temporary.
5651 addr = CreateTempAlloca(Ty: RV.getScalarVal()->getType(),
5652 align: CharUnits::fromQuantity(Quantity: std::max(
5653 a: layout->getAlignment(), b: scalarAlign)),
5654 Name: "tmp",
5655 /*ArraySize=*/nullptr, Alloca: &AllocaAddr);
5656 tempSize = EmitLifetimeStart(Size: scalarSize, Addr: AllocaAddr.getPointer());
5657
5658 Builder.CreateStore(Val: RV.getScalarVal(), Addr: addr);
5659 }
5660
5661 addr = addr.withElementType(ElemTy: coercionType);
5662
5663 unsigned IRArgPos = FirstIRArg;
5664 unsigned unpaddedIndex = 0;
5665 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
5666 llvm::Type *eltType = coercionType->getElementType(N: i);
5667 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
5668 continue;
5669 Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
5670 llvm::Value *elt = CreateCoercedLoad(
5671 Src: eltAddr,
5672 Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
5673 : unpaddedCoercionType,
5674 CGF&: *this);
5675 if (ArgHasMaybeUndefAttr)
5676 elt = Builder.CreateFreeze(V: elt);
5677 IRCallArgs[IRArgPos++] = elt;
5678 }
5679 assert(IRArgPos == FirstIRArg + NumIRArgs);
5680
5681 if (tempSize) {
5682 EmitLifetimeEnd(Size: tempSize, Addr: AllocaAddr.getPointer());
5683 }
5684
5685 break;
5686 }
5687
5688 case ABIArgInfo::Expand: {
5689 unsigned IRArgPos = FirstIRArg;
5690 ExpandTypeToArgs(Ty: I->Ty, Arg: *I, IRFuncTy, IRCallArgs, IRCallArgPos&: IRArgPos);
5691 assert(IRArgPos == FirstIRArg + NumIRArgs);
5692 break;
5693 }
5694 }
5695 }
5696
5697 const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(CGF&: *this);
5698 llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
5699
5700 // If we're using inalloca, set up that argument.
5701 if (ArgMemory.isValid()) {
5702 llvm::Value *Arg = ArgMemory.getPointer();
5703 assert(IRFunctionArgs.hasInallocaArg());
5704 IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
5705 }
5706
5707 // 2. Prepare the function pointer.
5708
5709 // If the callee is a bitcast of a non-variadic function to have a
5710 // variadic function pointer type, check to see if we can remove the
5711 // bitcast. This comes up with unprototyped functions.
5712 //
5713 // This makes the IR nicer, but more importantly it ensures that we
5714 // can inline the function at -O0 if it is marked always_inline.
5715 auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
5716 llvm::Value *Ptr) -> llvm::Function * {
5717 if (!CalleeFT->isVarArg())
5718 return nullptr;
5719
5720 // Get underlying value if it's a bitcast
5721 if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Val: Ptr)) {
5722 if (CE->getOpcode() == llvm::Instruction::BitCast)
5723 Ptr = CE->getOperand(i_nocapture: 0);
5724 }
5725
5726 llvm::Function *OrigFn = dyn_cast<llvm::Function>(Val: Ptr);
5727 if (!OrigFn)
5728 return nullptr;
5729
5730 llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
5731
5732 // If the original type is variadic, or if any of the component types
5733 // disagree, we cannot remove the cast.
5734 if (OrigFT->isVarArg() ||
5735 OrigFT->getNumParams() != CalleeFT->getNumParams() ||
5736 OrigFT->getReturnType() != CalleeFT->getReturnType())
5737 return nullptr;
5738
5739 for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i)
5740 if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
5741 return nullptr;
5742
5743 return OrigFn;
5744 };
5745
5746 if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {
5747 CalleePtr = OrigFn;
5748 IRFuncTy = OrigFn->getFunctionType();
5749 }
5750
5751 // 3. Perform the actual call.
5752
5753 // Deactivate any cleanups that we're supposed to do immediately before
5754 // the call.
5755 if (!CallArgs.getCleanupsToDeactivate().empty())
5756 deactivateArgCleanupsBeforeCall(CGF&: *this, CallArgs);
5757
5758 // Update the largest vector width if any arguments have vector types.
5759 for (unsigned i = 0; i < IRCallArgs.size(); ++i)
5760 LargestVectorWidth = std::max(a: LargestVectorWidth,
5761 b: getMaxVectorWidth(Ty: IRCallArgs[i]->getType()));
5762
5763 // Compute the calling convention and attributes.
5764 unsigned CallingConv;
5765 llvm::AttributeList Attrs;
5766 CGM.ConstructAttributeList(Name: CalleePtr->getName(), FI: CallInfo,
5767 CalleeInfo: Callee.getAbstractInfo(), AttrList&: Attrs, CallingConv,
5768 /*AttrOnCallSite=*/true,
5769 /*IsThunk=*/false);
5770
5771 if (CallingConv == llvm::CallingConv::X86_VectorCall &&
5772 getTarget().getTriple().isWindowsArm64EC()) {
5773 CGM.Error(loc: Loc, error: "__vectorcall calling convention is not currently "
5774 "supported");
5775 }
5776
5777 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
5778 if (FD->hasAttr<StrictFPAttr>())
5779 // All calls within a strictfp function are marked strictfp
5780 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::StrictFP);
5781
5782 // If -ffast-math is enabled and the function is guarded by an
5783 // '__attribute__((optnone)) adjust the memory attribute so the BE emits the
5784 // library call instead of the intrinsic.
5785 if (FD->hasAttr<OptimizeNoneAttr>() && getLangOpts().FastMath)
5786 CGM.AdjustMemoryAttribute(Name: CalleePtr->getName(), CalleeInfo: Callee.getAbstractInfo(),
5787 Attrs);
5788 }
5789 // Add call-site nomerge attribute if exists.
5790 if (InNoMergeAttributedStmt)
5791 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoMerge);
5792
5793 // Add call-site noinline attribute if exists.
5794 if (InNoInlineAttributedStmt)
5795 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoInline);
5796
5797 // Add call-site always_inline attribute if exists.
5798 // Note: This corresponds to the [[clang::always_inline]] statement attribute.
5799 if (InAlwaysInlineAttributedStmt &&
5800 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
5801 Caller: CallerDecl, Callee: CalleeDecl))
5802 Attrs =
5803 Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::AlwaysInline);
5804
5805 // Remove call-site convergent attribute if requested.
5806 if (InNoConvergentAttributedStmt)
5807 Attrs =
5808 Attrs.removeFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::Convergent);
5809
5810 // Apply some call-site-specific attributes.
5811 // TODO: work this into building the attribute set.
5812
5813 // Apply always_inline to all calls within flatten functions.
5814 // FIXME: should this really take priority over __try, below?
5815 if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
5816 !InNoInlineAttributedStmt &&
5817 !(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>()) &&
5818 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
5819 Caller: CallerDecl, Callee: CalleeDecl)) {
5820 Attrs =
5821 Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::AlwaysInline);
5822 }
5823
5824 // Disable inlining inside SEH __try blocks.
5825 if (isSEHTryScope()) {
5826 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoInline);
5827 }
5828
5829 // Decide whether to use a call or an invoke.
5830 bool CannotThrow;
5831 if (currentFunctionUsesSEHTry()) {
5832 // SEH cares about asynchronous exceptions, so everything can "throw."
5833 CannotThrow = false;
5834 } else if (isCleanupPadScope() &&
5835 EHPersonality::get(CGF&: *this).isMSVCXXPersonality()) {
5836 // The MSVC++ personality will implicitly terminate the program if an
5837 // exception is thrown during a cleanup outside of a try/catch.
5838 // We don't need to model anything in IR to get this behavior.
5839 CannotThrow = true;
5840 } else {
5841 // Otherwise, nounwind call sites will never throw.
5842 CannotThrow = Attrs.hasFnAttr(Kind: llvm::Attribute::NoUnwind);
5843
5844 if (auto *FPtr = dyn_cast<llvm::Function>(Val: CalleePtr))
5845 if (FPtr->hasFnAttribute(Kind: llvm::Attribute::NoUnwind))
5846 CannotThrow = true;
5847 }
5848
5849 // If we made a temporary, be sure to clean up after ourselves. Note that we
5850 // can't depend on being inside of an ExprWithCleanups, so we need to manually
5851 // pop this cleanup later on. Being eager about this is OK, since this
5852 // temporary is 'invisible' outside of the callee.
5853 if (UnusedReturnSizePtr)
5854 pushFullExprCleanup<CallLifetimeEnd>(kind: NormalEHLifetimeMarker, A: SRetPtr,
5855 A: UnusedReturnSizePtr);
5856
5857 llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();
5858
5859 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5860 getBundlesForFunclet(Callee: CalleePtr);
5861
5862 if (SanOpts.has(K: SanitizerKind::KCFI) &&
5863 !isa_and_nonnull<FunctionDecl>(Val: TargetDecl))
5864 EmitKCFIOperandBundle(Callee: ConcreteCallee, Bundles&: BundleList);
5865
5866 // Add the pointer-authentication bundle.
5867 EmitPointerAuthOperandBundle(Info: ConcreteCallee.getPointerAuthInfo(), Bundles&: BundleList);
5868
5869 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl))
5870 if (FD->hasAttr<StrictFPAttr>())
5871 // All calls within a strictfp function are marked strictfp
5872 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::StrictFP);
5873
5874 AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
5875 Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
5876
5877 AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
5878 Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
5879
5880 // Emit the actual call/invoke instruction.
5881 llvm::CallBase *CI;
5882 if (!InvokeDest) {
5883 CI = Builder.CreateCall(FTy: IRFuncTy, Callee: CalleePtr, Args: IRCallArgs, OpBundles: BundleList);
5884 } else {
5885 llvm::BasicBlock *Cont = createBasicBlock(name: "invoke.cont");
5886 CI = Builder.CreateInvoke(Ty: IRFuncTy, Callee: CalleePtr, NormalDest: Cont, UnwindDest: InvokeDest, Args: IRCallArgs,
5887 OpBundles: BundleList);
5888 EmitBlock(BB: Cont);
5889 }
5890 if (CI->getCalledFunction() && CI->getCalledFunction()->hasName() &&
5891 CI->getCalledFunction()->getName().starts_with(Prefix: "_Z4sqrt")) {
5892 SetSqrtFPAccuracy(CI);
5893 }
5894 if (callOrInvoke)
5895 *callOrInvoke = CI;
5896
5897 // If this is within a function that has the guard(nocf) attribute and is an
5898 // indirect call, add the "guard_nocf" attribute to this call to indicate that
5899 // Control Flow Guard checks should not be added, even if the call is inlined.
5900 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
5901 if (const auto *A = FD->getAttr<CFGuardAttr>()) {
5902 if (A->getGuard() == CFGuardAttr::GuardArg::nocf &&
5903 !CI->getCalledFunction())
5904 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: "guard_nocf");
5905 }
5906 }
5907
5908 // Apply the attributes and calling convention.
5909 CI->setAttributes(Attrs);
5910 CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
5911
5912 // Apply various metadata.
5913
5914 if (!CI->getType()->isVoidTy())
5915 CI->setName("call");
5916
5917 if (CGM.shouldEmitConvergenceTokens() && CI->isConvergent())
5918 CI = addConvergenceControlToken(Input: CI);
5919
5920 // Update largest vector width from the return type.
5921 LargestVectorWidth =
5922 std::max(a: LargestVectorWidth, b: getMaxVectorWidth(Ty: CI->getType()));
5923
5924 // Insert instrumentation or attach profile metadata at indirect call sites.
5925 // For more details, see the comment before the definition of
5926 // IPVK_IndirectCallTarget in InstrProfData.inc.
5927 if (!CI->getCalledFunction())
5928 PGO->valueProfile(Builder, ValueKind: llvm::IPVK_IndirectCallTarget, ValueSite: CI, ValuePtr: CalleePtr);
5929
5930 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5931 // optimizer it can aggressively ignore unwind edges.
5932 if (CGM.getLangOpts().ObjCAutoRefCount)
5933 AddObjCARCExceptionMetadata(Inst: CI);
5934
5935 // Set tail call kind if necessary.
5936 if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(Val: CI)) {
5937 if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
5938 Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
5939 else if (IsMustTail) {
5940 if (getTarget().getTriple().isPPC()) {
5941 if (getTarget().getTriple().isOSAIX())
5942 CGM.getDiags().Report(Loc, DiagID: diag::err_aix_musttail_unsupported);
5943 else if (!getTarget().hasFeature(Feature: "pcrelative-memops")) {
5944 if (getTarget().hasFeature(Feature: "longcall"))
5945 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail) << 0;
5946 else if (Call->isIndirectCall())
5947 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail) << 1;
5948 else if (isa_and_nonnull<FunctionDecl>(Val: TargetDecl)) {
5949 if (!cast<FunctionDecl>(Val: TargetDecl)->isDefined())
5950 // The undefined callee may be a forward declaration. Without
5951 // knowning all symbols in the module, we won't know the symbol is
5952 // defined or not. Collect all these symbols for later diagnosing.
5953 CGM.addUndefinedGlobalForTailCall(
5954 Global: {cast<FunctionDecl>(Val: TargetDecl), Loc});
5955 else {
5956 llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(
5957 GD: GlobalDecl(cast<FunctionDecl>(Val: TargetDecl)));
5958 if (llvm::GlobalValue::isWeakForLinker(Linkage) ||
5959 llvm::GlobalValue::isDiscardableIfUnused(Linkage))
5960 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail)
5961 << 2;
5962 }
5963 }
5964 }
5965 }
5966 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
5967 }
5968 }
5969
5970 // Add metadata for calls to MSAllocator functions
5971 if (getDebugInfo() && TargetDecl && TargetDecl->hasAttr<MSAllocatorAttr>())
5972 getDebugInfo()->addHeapAllocSiteMetadata(CallSite: CI, AllocatedTy: RetTy->getPointeeType(), Loc);
5973
5974 // Add metadata if calling an __attribute__((error(""))) or warning fn.
5975 if (TargetDecl && TargetDecl->hasAttr<ErrorAttr>()) {
5976 llvm::ConstantInt *Line =
5977 llvm::ConstantInt::get(Ty: Int64Ty, V: Loc.getRawEncoding());
5978 llvm::ConstantAsMetadata *MD = llvm::ConstantAsMetadata::get(C: Line);
5979 llvm::MDTuple *MDT = llvm::MDNode::get(Context&: getLLVMContext(), MDs: {MD});
5980 CI->setMetadata(Kind: "srcloc", Node: MDT);
5981 }
5982
5983 // 4. Finish the call.
5984
5985 // If the call doesn't return, finish the basic block and clear the
5986 // insertion point; this allows the rest of IRGen to discard
5987 // unreachable code.
5988 if (CI->doesNotReturn()) {
5989 if (UnusedReturnSizePtr)
5990 PopCleanupBlock();
5991
5992 // Strip away the noreturn attribute to better diagnose unreachable UB.
5993 if (SanOpts.has(K: SanitizerKind::Unreachable)) {
5994 // Also remove from function since CallBase::hasFnAttr additionally checks
5995 // attributes of the called function.
5996 if (auto *F = CI->getCalledFunction())
5997 F->removeFnAttr(Kind: llvm::Attribute::NoReturn);
5998 CI->removeFnAttr(Kind: llvm::Attribute::NoReturn);
5999
6000 // Avoid incompatibility with ASan which relies on the `noreturn`
6001 // attribute to insert handler calls.
6002 if (SanOpts.hasOneOf(K: SanitizerKind::Address |
6003 SanitizerKind::KernelAddress)) {
6004 SanitizerScope SanScope(this);
6005 llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);
6006 Builder.SetInsertPoint(CI);
6007 auto *FnType = llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
6008 llvm::FunctionCallee Fn =
6009 CGM.CreateRuntimeFunction(Ty: FnType, Name: "__asan_handle_no_return");
6010 EmitNounwindRuntimeCall(callee: Fn);
6011 }
6012 }
6013
6014 EmitUnreachable(Loc);
6015 Builder.ClearInsertionPoint();
6016
6017 // FIXME: For now, emit a dummy basic block because expr emitters in
6018 // generally are not ready to handle emitting expressions at unreachable
6019 // points.
6020 EnsureInsertPoint();
6021
6022 // Return a reasonable RValue.
6023 return GetUndefRValue(Ty: RetTy);
6024 }
6025
6026 // If this is a musttail call, return immediately. We do not branch to the
6027 // epilogue in this case.
6028 if (IsMustTail) {
6029 for (auto it = EHStack.find(sp: CurrentCleanupScopeDepth); it != EHStack.end();
6030 ++it) {
6031 EHCleanupScope *Cleanup = dyn_cast<EHCleanupScope>(Val: &*it);
6032 // Fake uses can be safely emitted immediately prior to the tail call, so
6033 // we choose to emit them just before the call here.
6034 if (Cleanup && Cleanup->isFakeUse()) {
6035 CGBuilderTy::InsertPointGuard IPG(Builder);
6036 Builder.SetInsertPoint(CI);
6037 Cleanup->getCleanup()->Emit(CGF&: *this, flags: EHScopeStack::Cleanup::Flags());
6038 } else if (!(Cleanup &&
6039 Cleanup->getCleanup()->isRedundantBeforeReturn())) {
6040 CGM.ErrorUnsupported(S: MustTailCall, Type: "tail call skipping over cleanups");
6041 }
6042 }
6043 if (CI->getType()->isVoidTy())
6044 Builder.CreateRetVoid();
6045 else
6046 Builder.CreateRet(V: CI);
6047 Builder.ClearInsertionPoint();
6048 EnsureInsertPoint();
6049 return GetUndefRValue(Ty: RetTy);
6050 }
6051
6052 // Perform the swifterror writeback.
6053 if (swiftErrorTemp.isValid()) {
6054 llvm::Value *errorResult = Builder.CreateLoad(Addr: swiftErrorTemp);
6055 Builder.CreateStore(Val: errorResult, Addr: swiftErrorArg);
6056 }
6057
6058 // Emit any call-associated writebacks immediately. Arguably this
6059 // should happen after any return-value munging.
6060 if (CallArgs.hasWritebacks())
6061 EmitWritebacks(args: CallArgs);
6062
6063 // The stack cleanup for inalloca arguments has to run out of the normal
6064 // lexical order, so deactivate it and run it manually here.
6065 CallArgs.freeArgumentMemory(CGF&: *this);
6066
6067 // Extract the return value.
6068 RValue Ret;
6069
6070 // If the current function is a virtual function pointer thunk, avoid copying
6071 // the return value of the musttail call to a temporary.
6072 if (IsVirtualFunctionPointerThunk) {
6073 Ret = RValue::get(V: CI);
6074 } else {
6075 Ret = [&] {
6076 switch (RetAI.getKind()) {
6077 case ABIArgInfo::CoerceAndExpand: {
6078 auto coercionType = RetAI.getCoerceAndExpandType();
6079
6080 Address addr = SRetPtr.withElementType(ElemTy: coercionType);
6081
6082 assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());
6083 bool requiresExtract = isa<llvm::StructType>(Val: CI->getType());
6084
6085 unsigned unpaddedIndex = 0;
6086 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
6087 llvm::Type *eltType = coercionType->getElementType(N: i);
6088 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
6089 continue;
6090 Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
6091 llvm::Value *elt = CI;
6092 if (requiresExtract)
6093 elt = Builder.CreateExtractValue(Agg: elt, Idxs: unpaddedIndex++);
6094 else
6095 assert(unpaddedIndex == 0);
6096 Builder.CreateStore(Val: elt, Addr: eltAddr);
6097 }
6098 [[fallthrough]];
6099 }
6100
6101 case ABIArgInfo::InAlloca:
6102 case ABIArgInfo::Indirect: {
6103 RValue ret = convertTempToRValue(addr: SRetPtr, type: RetTy, Loc: SourceLocation());
6104 if (UnusedReturnSizePtr)
6105 PopCleanupBlock();
6106 return ret;
6107 }
6108
6109 case ABIArgInfo::Ignore:
6110 // If we are ignoring an argument that had a result, make sure to
6111 // construct the appropriate return value for our caller.
6112 return GetUndefRValue(Ty: RetTy);
6113
6114 case ABIArgInfo::Extend:
6115 case ABIArgInfo::Direct: {
6116 llvm::Type *RetIRTy = ConvertType(T: RetTy);
6117 if (RetAI.getCoerceToType() == RetIRTy &&
6118 RetAI.getDirectOffset() == 0) {
6119 switch (getEvaluationKind(T: RetTy)) {
6120 case TEK_Complex: {
6121 llvm::Value *Real = Builder.CreateExtractValue(Agg: CI, Idxs: 0);
6122 llvm::Value *Imag = Builder.CreateExtractValue(Agg: CI, Idxs: 1);
6123 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
6124 }
6125 case TEK_Aggregate:
6126 break;
6127 case TEK_Scalar: {
6128 // If the argument doesn't match, perform a bitcast to coerce it.
6129 // This can happen due to trivial type mismatches.
6130 llvm::Value *V = CI;
6131 if (V->getType() != RetIRTy)
6132 V = Builder.CreateBitCast(V, DestTy: RetIRTy);
6133 return RValue::get(V);
6134 }
6135 }
6136 }
6137
6138 // If coercing a fixed vector from a scalable vector for ABI
6139 // compatibility, and the types match, use the llvm.vector.extract
6140 // intrinsic to perform the conversion.
6141 if (auto *FixedDstTy = dyn_cast<llvm::FixedVectorType>(Val: RetIRTy)) {
6142 llvm::Value *V = CI;
6143 if (auto *ScalableSrcTy =
6144 dyn_cast<llvm::ScalableVectorType>(Val: V->getType())) {
6145 if (FixedDstTy->getElementType() ==
6146 ScalableSrcTy->getElementType()) {
6147 V = Builder.CreateExtractVector(DstType: FixedDstTy, SrcVec: V, Idx: uint64_t(0),
6148 Name: "cast.fixed");
6149 return RValue::get(V);
6150 }
6151 }
6152 }
6153
6154 Address DestPtr = ReturnValue.getValue();
6155 bool DestIsVolatile = ReturnValue.isVolatile();
6156 uint64_t DestSize =
6157 getContext().getTypeInfoDataSizeInChars(T: RetTy).Width.getQuantity();
6158
6159 if (!DestPtr.isValid()) {
6160 DestPtr = CreateMemTemp(T: RetTy, Name: "coerce");
6161 DestIsVolatile = false;
6162 DestSize = getContext().getTypeSizeInChars(T: RetTy).getQuantity();
6163 }
6164
6165 // An empty record can overlap other data (if declared with
6166 // no_unique_address); omit the store for such types - as there is no
6167 // actual data to store.
6168 if (!isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true)) {
6169 // If the value is offset in memory, apply the offset now.
6170 Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
6171 CreateCoercedStore(
6172 Src: CI, Dst: StorePtr,
6173 DstSize: llvm::TypeSize::getFixed(ExactSize: DestSize - RetAI.getDirectOffset()),
6174 DstIsVolatile: DestIsVolatile);
6175 }
6176
6177 return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation());
6178 }
6179
6180 case ABIArgInfo::Expand:
6181 case ABIArgInfo::IndirectAliased:
6182 llvm_unreachable("Invalid ABI kind for return argument");
6183 }
6184
6185 llvm_unreachable("Unhandled ABIArgInfo::Kind");
6186 }();
6187 }
6188
6189 // Emit the assume_aligned check on the return value.
6190 if (Ret.isScalar() && TargetDecl) {
6191 AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6192 AllocAlignAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6193 }
6194
6195 // Explicitly call CallLifetimeEnd::Emit just to re-use the code even though
6196 // we can't use the full cleanup mechanism.
6197 for (CallLifetimeEnd &LifetimeEnd : CallLifetimeEndAfterCall)
6198 LifetimeEnd.Emit(CGF&: *this, /*Flags=*/flags: {});
6199
6200 if (!ReturnValue.isExternallyDestructed() &&
6201 RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct)
6202 pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Ret.getAggregateAddress(),
6203 type: RetTy);
6204
6205 return Ret;
6206}
6207
6208CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const {
6209 if (isVirtual()) {
6210 const CallExpr *CE = getVirtualCallExpr();
6211 return CGF.CGM.getCXXABI().getVirtualFunctionPointer(
6212 CGF, GD: getVirtualMethodDecl(), This: getThisAddress(), Ty: getVirtualFunctionType(),
6213 Loc: CE ? CE->getBeginLoc() : SourceLocation());
6214 }
6215
6216 return *this;
6217}
6218
6219/* VarArg handling */
6220
6221RValue CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
6222 AggValueSlot Slot) {
6223 VAListAddr = VE->isMicrosoftABI() ? EmitMSVAListRef(E: VE->getSubExpr())
6224 : EmitVAListRef(E: VE->getSubExpr());
6225 QualType Ty = VE->getType();
6226 if (Ty->isVariablyModifiedType())
6227 EmitVariablyModifiedType(Ty);
6228 if (VE->isMicrosoftABI())
6229 return CGM.getABIInfo().EmitMSVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6230 return CGM.getABIInfo().EmitVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6231}
6232