1//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// These classes wrap the information about a call or function
10// definition used to handle ABI compliancy.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CGCall.h"
15#include "ABIInfo.h"
16#include "ABIInfoImpl.h"
17#include "CGBlocks.h"
18#include "CGCXXABI.h"
19#include "CGCleanup.h"
20#include "CGDebugInfo.h"
21#include "CGRecordLayout.h"
22#include "CodeGenFunction.h"
23#include "CodeGenModule.h"
24#include "CodeGenPGO.h"
25#include "TargetInfo.h"
26#include "clang/AST/Attr.h"
27#include "clang/AST/Decl.h"
28#include "clang/AST/DeclCXX.h"
29#include "clang/AST/DeclObjC.h"
30#include "clang/Basic/CodeGenOptions.h"
31#include "clang/Basic/TargetInfo.h"
32#include "clang/CodeGen/CGFunctionInfo.h"
33#include "clang/CodeGen/SwiftCallingConv.h"
34#include "llvm/ADT/STLExtras.h"
35#include "llvm/ADT/StringExtras.h"
36#include "llvm/Analysis/ValueTracking.h"
37#include "llvm/IR/Assumptions.h"
38#include "llvm/IR/AttributeMask.h"
39#include "llvm/IR/Attributes.h"
40#include "llvm/IR/CallingConv.h"
41#include "llvm/IR/DataLayout.h"
42#include "llvm/IR/DebugInfoMetadata.h"
43#include "llvm/IR/InlineAsm.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/Intrinsics.h"
46#include "llvm/IR/Type.h"
47#include "llvm/Transforms/Utils/Local.h"
48#include <optional>
49using namespace clang;
50using namespace CodeGen;
51
52/***/
53
54unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
55 switch (CC) {
56 default:
57 return llvm::CallingConv::C;
58 case CC_X86StdCall:
59 return llvm::CallingConv::X86_StdCall;
60 case CC_X86FastCall:
61 return llvm::CallingConv::X86_FastCall;
62 case CC_X86RegCall:
63 return llvm::CallingConv::X86_RegCall;
64 case CC_X86ThisCall:
65 return llvm::CallingConv::X86_ThisCall;
66 case CC_Win64:
67 return llvm::CallingConv::Win64;
68 case CC_X86_64SysV:
69 return llvm::CallingConv::X86_64_SysV;
70 case CC_AAPCS:
71 return llvm::CallingConv::ARM_AAPCS;
72 case CC_AAPCS_VFP:
73 return llvm::CallingConv::ARM_AAPCS_VFP;
74 case CC_IntelOclBicc:
75 return llvm::CallingConv::Intel_OCL_BI;
76 // TODO: Add support for __pascal to LLVM.
77 case CC_X86Pascal:
78 return llvm::CallingConv::C;
79 // TODO: Add support for __vectorcall to LLVM.
80 case CC_X86VectorCall:
81 return llvm::CallingConv::X86_VectorCall;
82 case CC_AArch64VectorCall:
83 return llvm::CallingConv::AArch64_VectorCall;
84 case CC_AArch64SVEPCS:
85 return llvm::CallingConv::AArch64_SVE_VectorCall;
86 case CC_SpirFunction:
87 return llvm::CallingConv::SPIR_FUNC;
88 case CC_DeviceKernel:
89 return CGM.getTargetCodeGenInfo().getDeviceKernelCallingConv();
90 case CC_PreserveMost:
91 return llvm::CallingConv::PreserveMost;
92 case CC_PreserveAll:
93 return llvm::CallingConv::PreserveAll;
94 case CC_Swift:
95 return llvm::CallingConv::Swift;
96 case CC_SwiftAsync:
97 return llvm::CallingConv::SwiftTail;
98 case CC_M68kRTD:
99 return llvm::CallingConv::M68k_RTD;
100 case CC_PreserveNone:
101 return llvm::CallingConv::PreserveNone;
102 // clang-format off
103 case CC_RISCVVectorCall: return llvm::CallingConv::RISCV_VectorCall;
104 // clang-format on
105#define CC_VLS_CASE(ABI_VLEN) \
106 case CC_RISCVVLSCall_##ABI_VLEN: \
107 return llvm::CallingConv::RISCV_VLSCall_##ABI_VLEN;
108 CC_VLS_CASE(32)
109 CC_VLS_CASE(64)
110 CC_VLS_CASE(128)
111 CC_VLS_CASE(256)
112 CC_VLS_CASE(512)
113 CC_VLS_CASE(1024)
114 CC_VLS_CASE(2048)
115 CC_VLS_CASE(4096)
116 CC_VLS_CASE(8192)
117 CC_VLS_CASE(16384)
118 CC_VLS_CASE(32768)
119 CC_VLS_CASE(65536)
120#undef CC_VLS_CASE
121 }
122}
123
124/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
125/// qualification. Either or both of RD and MD may be null. A null RD indicates
126/// that there is no meaningful 'this' type, and a null MD can occur when
127/// calling a method pointer.
128CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
129 const CXXMethodDecl *MD) {
130 CanQualType RecTy;
131 if (RD)
132 RecTy = Context.getCanonicalTagType(TD: RD);
133 else
134 RecTy = Context.VoidTy;
135
136 if (MD)
137 RecTy = CanQualType::CreateUnsafe(Other: Context.getAddrSpaceQualType(
138 T: RecTy, AddressSpace: MD->getMethodQualifiers().getAddressSpace()));
139 return Context.getPointerType(T: RecTy);
140}
141
142/// Returns the canonical formal type of the given C++ method.
143static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
144 return MD->getType()
145 ->getCanonicalTypeUnqualified()
146 .getAs<FunctionProtoType>();
147}
148
149/// Returns the "extra-canonicalized" return type, which discards
150/// qualifiers on the return type. Codegen doesn't care about them,
151/// and it makes ABI code a little easier to be able to assume that
152/// all parameter and return types are top-level unqualified.
153static CanQualType GetReturnType(QualType RetTy) {
154 return RetTy->getCanonicalTypeUnqualified();
155}
156
157/// Arrange the argument and result information for a value of the given
158/// unprototyped freestanding function type.
159const CGFunctionInfo &
160CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
161 // When translating an unprototyped function type, always use a
162 // variadic type.
163 return arrangeLLVMFunctionInfo(returnType: FTNP->getReturnType().getUnqualifiedType(),
164 opts: FnInfoOpts::None, argTypes: {}, info: FTNP->getExtInfo(), paramInfos: {},
165 args: RequiredArgs(0));
166}
167
168static void addExtParameterInfosForCall(
169 llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
170 const FunctionProtoType *proto, unsigned prefixArgs, unsigned totalArgs) {
171 assert(proto->hasExtParameterInfos());
172 assert(paramInfos.size() <= prefixArgs);
173 assert(proto->getNumParams() + prefixArgs <= totalArgs);
174
175 paramInfos.reserve(N: totalArgs);
176
177 // Add default infos for any prefix args that don't already have infos.
178 paramInfos.resize(N: prefixArgs);
179
180 // Add infos for the prototype.
181 for (const auto &ParamInfo : proto->getExtParameterInfos()) {
182 paramInfos.push_back(Elt: ParamInfo);
183 // pass_object_size params have no parameter info.
184 if (ParamInfo.hasPassObjectSize())
185 paramInfos.emplace_back();
186 }
187
188 assert(paramInfos.size() <= totalArgs &&
189 "Did we forget to insert pass_object_size args?");
190 // Add default infos for the variadic and/or suffix arguments.
191 paramInfos.resize(N: totalArgs);
192}
193
194/// Adds the formal parameters in FPT to the given prefix. If any parameter in
195/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
196static void appendParameterTypes(
197 const CodeGenTypes &CGT, SmallVectorImpl<CanQualType> &prefix,
198 SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
199 CanQual<FunctionProtoType> FPT) {
200 // Fast path: don't touch param info if we don't need to.
201 if (!FPT->hasExtParameterInfos()) {
202 assert(paramInfos.empty() &&
203 "We have paramInfos, but the prototype doesn't?");
204 prefix.append(in_start: FPT->param_type_begin(), in_end: FPT->param_type_end());
205 return;
206 }
207
208 unsigned PrefixSize = prefix.size();
209 // In the vast majority of cases, we'll have precisely FPT->getNumParams()
210 // parameters; the only thing that can change this is the presence of
211 // pass_object_size. So, we preallocate for the common case.
212 prefix.reserve(N: prefix.size() + FPT->getNumParams());
213
214 auto ExtInfos = FPT->getExtParameterInfos();
215 assert(ExtInfos.size() == FPT->getNumParams());
216 for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
217 prefix.push_back(Elt: FPT->getParamType(i: I));
218 if (ExtInfos[I].hasPassObjectSize())
219 prefix.push_back(Elt: CGT.getContext().getCanonicalSizeType());
220 }
221
222 addExtParameterInfosForCall(paramInfos, proto: FPT.getTypePtr(), prefixArgs: PrefixSize,
223 totalArgs: prefix.size());
224}
225
226using ExtParameterInfoList =
227 SmallVector<FunctionProtoType::ExtParameterInfo, 16>;
228
229/// Arrange the LLVM function layout for a value of the given function
230/// type, on top of any implicit parameters already stored.
231static const CGFunctionInfo &
232arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
233 SmallVectorImpl<CanQualType> &prefix,
234 CanQual<FunctionProtoType> FTP) {
235 ExtParameterInfoList paramInfos;
236 RequiredArgs Required = RequiredArgs::forPrototypePlus(prototype: FTP, additional: prefix.size());
237 appendParameterTypes(CGT, prefix, paramInfos, FPT: FTP);
238 CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
239
240 FnInfoOpts opts =
241 instanceMethod ? FnInfoOpts::IsInstanceMethod : FnInfoOpts::None;
242 return CGT.arrangeLLVMFunctionInfo(returnType: resultType, opts, argTypes: prefix,
243 info: FTP->getExtInfo(), paramInfos, args: Required);
244}
245
246using CanQualTypeList = SmallVector<CanQualType, 16>;
247
248/// Arrange the argument and result information for a value of the
249/// given freestanding function type.
250const CGFunctionInfo &
251CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
252 CanQualTypeList argTypes;
253 return ::arrangeLLVMFunctionInfo(CGT&: *this, /*instanceMethod=*/false, prefix&: argTypes,
254 FTP);
255}
256
257static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
258 bool IsTargetDefaultMSABI) {
259 // Set the appropriate calling convention for the Function.
260 if (D->hasAttr<StdCallAttr>())
261 return CC_X86StdCall;
262
263 if (D->hasAttr<FastCallAttr>())
264 return CC_X86FastCall;
265
266 if (D->hasAttr<RegCallAttr>())
267 return CC_X86RegCall;
268
269 if (D->hasAttr<ThisCallAttr>())
270 return CC_X86ThisCall;
271
272 if (D->hasAttr<VectorCallAttr>())
273 return CC_X86VectorCall;
274
275 if (D->hasAttr<PascalAttr>())
276 return CC_X86Pascal;
277
278 if (PcsAttr *PCS = D->getAttr<PcsAttr>())
279 return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
280
281 if (D->hasAttr<AArch64VectorPcsAttr>())
282 return CC_AArch64VectorCall;
283
284 if (D->hasAttr<AArch64SVEPcsAttr>())
285 return CC_AArch64SVEPCS;
286
287 if (D->hasAttr<DeviceKernelAttr>())
288 return CC_DeviceKernel;
289
290 if (D->hasAttr<IntelOclBiccAttr>())
291 return CC_IntelOclBicc;
292
293 if (D->hasAttr<MSABIAttr>())
294 return IsTargetDefaultMSABI ? CC_C : CC_Win64;
295
296 if (D->hasAttr<SysVABIAttr>())
297 return IsTargetDefaultMSABI ? CC_X86_64SysV : CC_C;
298
299 if (D->hasAttr<PreserveMostAttr>())
300 return CC_PreserveMost;
301
302 if (D->hasAttr<PreserveAllAttr>())
303 return CC_PreserveAll;
304
305 if (D->hasAttr<M68kRTDAttr>())
306 return CC_M68kRTD;
307
308 if (D->hasAttr<PreserveNoneAttr>())
309 return CC_PreserveNone;
310
311 if (D->hasAttr<RISCVVectorCCAttr>())
312 return CC_RISCVVectorCall;
313
314 if (RISCVVLSCCAttr *PCS = D->getAttr<RISCVVLSCCAttr>()) {
315 switch (PCS->getVectorWidth()) {
316 default:
317 llvm_unreachable("Invalid RISC-V VLS ABI VLEN");
318#define CC_VLS_CASE(ABI_VLEN) \
319 case ABI_VLEN: \
320 return CC_RISCVVLSCall_##ABI_VLEN;
321 CC_VLS_CASE(32)
322 CC_VLS_CASE(64)
323 CC_VLS_CASE(128)
324 CC_VLS_CASE(256)
325 CC_VLS_CASE(512)
326 CC_VLS_CASE(1024)
327 CC_VLS_CASE(2048)
328 CC_VLS_CASE(4096)
329 CC_VLS_CASE(8192)
330 CC_VLS_CASE(16384)
331 CC_VLS_CASE(32768)
332 CC_VLS_CASE(65536)
333#undef CC_VLS_CASE
334 }
335 }
336
337 return CC_C;
338}
339
340/// Arrange the argument and result information for a call to an
341/// unknown C++ non-static member function of the given abstract type.
342/// (A null RD means we don't have any meaningful "this" argument type,
343/// so fall back to a generic pointer type).
344/// The member function must be an ordinary function, i.e. not a
345/// constructor or destructor.
346const CGFunctionInfo &
347CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
348 const FunctionProtoType *FTP,
349 const CXXMethodDecl *MD) {
350 CanQualTypeList argTypes;
351
352 // Add the 'this' pointer.
353 argTypes.push_back(Elt: DeriveThisType(RD, MD));
354
355 return ::arrangeLLVMFunctionInfo(
356 CGT&: *this, /*instanceMethod=*/true, prefix&: argTypes,
357 FTP: FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
358}
359
360/// Set calling convention for CUDA/HIP kernel.
361static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
362 const FunctionDecl *FD) {
363 if (FD->hasAttr<CUDAGlobalAttr>()) {
364 const FunctionType *FT = FTy->getAs<FunctionType>();
365 CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
366 FTy = FT->getCanonicalTypeUnqualified();
367 }
368}
369
370/// Arrange the argument and result information for a declaration or
371/// definition of the given C++ non-static member function. The
372/// member function must be an ordinary function, i.e. not a
373/// constructor or destructor.
374const CGFunctionInfo &
375CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
376 assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
377 assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
378
379 CanQualType FT = GetFormalType(MD).getAs<Type>();
380 setCUDAKernelCallingConvention(FTy&: FT, CGM, FD: MD);
381 auto prototype = FT.getAs<FunctionProtoType>();
382
383 if (MD->isImplicitObjectMemberFunction()) {
384 // The abstract case is perfectly fine.
385 const CXXRecordDecl *ThisType =
386 getCXXABI().getThisArgumentTypeForMethod(GD: MD);
387 return arrangeCXXMethodType(RD: ThisType, FTP: prototype.getTypePtr(), MD);
388 }
389
390 return arrangeFreeFunctionType(FTP: prototype);
391}
392
393bool CodeGenTypes::inheritingCtorHasParams(
394 const InheritedConstructor &Inherited, CXXCtorType Type) {
395 // Parameters are unnecessary if we're constructing a base class subobject
396 // and the inherited constructor lives in a virtual base.
397 return Type == Ctor_Complete ||
398 !Inherited.getShadowDecl()->constructsVirtualBase() ||
399 !Target.getCXXABI().hasConstructorVariants();
400}
401
402const CGFunctionInfo &
403CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
404 auto *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
405
406 CanQualTypeList argTypes;
407 ExtParameterInfoList paramInfos;
408
409 const CXXRecordDecl *ThisType = getCXXABI().getThisArgumentTypeForMethod(GD);
410 argTypes.push_back(Elt: DeriveThisType(RD: ThisType, MD));
411
412 bool PassParams = true;
413
414 if (auto *CD = dyn_cast<CXXConstructorDecl>(Val: MD)) {
415 // A base class inheriting constructor doesn't get forwarded arguments
416 // needed to construct a virtual base (or base class thereof).
417 if (auto Inherited = CD->getInheritedConstructor())
418 PassParams = inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
419 }
420
421 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
422
423 // Add the formal parameters.
424 if (PassParams)
425 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: FTP);
426
427 CGCXXABI::AddedStructorArgCounts AddedArgs =
428 getCXXABI().buildStructorSignature(GD, ArgTys&: argTypes);
429 if (!paramInfos.empty()) {
430 // Note: prefix implies after the first param.
431 if (AddedArgs.Prefix)
432 paramInfos.insert(I: paramInfos.begin() + 1, NumToInsert: AddedArgs.Prefix,
433 Elt: FunctionProtoType::ExtParameterInfo{});
434 if (AddedArgs.Suffix)
435 paramInfos.append(NumInputs: AddedArgs.Suffix,
436 Elt: FunctionProtoType::ExtParameterInfo{});
437 }
438
439 RequiredArgs required =
440 (PassParams && MD->isVariadic() ? RequiredArgs(argTypes.size())
441 : RequiredArgs::All);
442
443 FunctionType::ExtInfo extInfo = FTP->getExtInfo();
444 CanQualType resultType = getCXXABI().HasThisReturn(GD) ? argTypes.front()
445 : getCXXABI().hasMostDerivedReturn(GD)
446 ? CGM.getContext().VoidPtrTy
447 : Context.VoidTy;
448 return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::IsInstanceMethod,
449 argTypes, info: extInfo, paramInfos, args: required);
450}
451
452static CanQualTypeList getArgTypesForCall(ASTContext &ctx,
453 const CallArgList &args) {
454 CanQualTypeList argTypes;
455 for (auto &arg : args)
456 argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg.Ty));
457 return argTypes;
458}
459
460static CanQualTypeList getArgTypesForDeclaration(ASTContext &ctx,
461 const FunctionArgList &args) {
462 CanQualTypeList argTypes;
463 for (auto &arg : args)
464 argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg->getType()));
465 return argTypes;
466}
467
468static ExtParameterInfoList
469getExtParameterInfosForCall(const FunctionProtoType *proto, unsigned prefixArgs,
470 unsigned totalArgs) {
471 ExtParameterInfoList result;
472 if (proto->hasExtParameterInfos()) {
473 addExtParameterInfosForCall(paramInfos&: result, proto, prefixArgs, totalArgs);
474 }
475 return result;
476}
477
478/// Arrange a call to a C++ method, passing the given arguments.
479///
480/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
481/// parameter.
482/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
483/// args.
484/// PassProtoArgs indicates whether `args` has args for the parameters in the
485/// given CXXConstructorDecl.
486const CGFunctionInfo &CodeGenTypes::arrangeCXXConstructorCall(
487 const CallArgList &args, const CXXConstructorDecl *D, CXXCtorType CtorKind,
488 unsigned ExtraPrefixArgs, unsigned ExtraSuffixArgs, bool PassProtoArgs) {
489 CanQualTypeList ArgTypes;
490 for (const auto &Arg : args)
491 ArgTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
492
493 // +1 for implicit this, which should always be args[0].
494 unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
495
496 CanQual<FunctionProtoType> FPT = GetFormalType(MD: D);
497 RequiredArgs Required = PassProtoArgs
498 ? RequiredArgs::forPrototypePlus(
499 prototype: FPT, additional: TotalPrefixArgs + ExtraSuffixArgs)
500 : RequiredArgs::All;
501
502 GlobalDecl GD(D, CtorKind);
503 CanQualType ResultType = getCXXABI().HasThisReturn(GD) ? ArgTypes.front()
504 : getCXXABI().hasMostDerivedReturn(GD)
505 ? CGM.getContext().VoidPtrTy
506 : Context.VoidTy;
507
508 FunctionType::ExtInfo Info = FPT->getExtInfo();
509 ExtParameterInfoList ParamInfos;
510 // If the prototype args are elided, we should only have ABI-specific args,
511 // which never have param info.
512 if (PassProtoArgs && FPT->hasExtParameterInfos()) {
513 // ABI-specific suffix arguments are treated the same as variadic arguments.
514 addExtParameterInfosForCall(paramInfos&: ParamInfos, proto: FPT.getTypePtr(), prefixArgs: TotalPrefixArgs,
515 totalArgs: ArgTypes.size());
516 }
517
518 return arrangeLLVMFunctionInfo(returnType: ResultType, opts: FnInfoOpts::IsInstanceMethod,
519 argTypes: ArgTypes, info: Info, paramInfos: ParamInfos, args: Required);
520}
521
522/// Arrange the argument and result information for the declaration or
523/// definition of the given function.
524const CGFunctionInfo &
525CodeGenTypes::arrangeFunctionDeclaration(const GlobalDecl GD) {
526 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
527 if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD))
528 if (MD->isImplicitObjectMemberFunction())
529 return arrangeCXXMethodDeclaration(MD);
530
531 CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
532
533 assert(isa<FunctionType>(FTy));
534 setCUDAKernelCallingConvention(FTy, CGM, FD);
535
536 if (DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()) &&
537 GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
538 const FunctionType *FT = FTy->getAs<FunctionType>();
539 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
540 FTy = FT->getCanonicalTypeUnqualified();
541 }
542
543 // When declaring a function without a prototype, always use a
544 // non-variadic type.
545 if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
546 return arrangeLLVMFunctionInfo(returnType: noProto->getReturnType(), opts: FnInfoOpts::None,
547 argTypes: {}, info: noProto->getExtInfo(), paramInfos: {},
548 args: RequiredArgs::All);
549 }
550
551 return arrangeFreeFunctionType(FTP: FTy.castAs<FunctionProtoType>());
552}
553
554/// Arrange the argument and result information for the declaration or
555/// definition of an Objective-C method.
556const CGFunctionInfo &
557CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
558 // It happens that this is the same as a call with no optional
559 // arguments, except also using the formal 'self' type.
560 return arrangeObjCMessageSendSignature(MD, receiverType: MD->getSelfDecl()->getType());
561}
562
563/// Arrange the argument and result information for the function type
564/// through which to perform a send to the given Objective-C method,
565/// using the given receiver type. The receiver type is not always
566/// the 'self' type of the method or even an Objective-C pointer type.
567/// This is *not* the right method for actually performing such a
568/// message send, due to the possibility of optional arguments.
569const CGFunctionInfo &
570CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
571 QualType receiverType) {
572 CanQualTypeList argTys;
573 ExtParameterInfoList extParamInfos(MD->isDirectMethod() ? 1 : 2);
574 argTys.push_back(Elt: Context.getCanonicalParamType(T: receiverType));
575 if (!MD->isDirectMethod())
576 argTys.push_back(Elt: Context.getCanonicalParamType(T: Context.getObjCSelType()));
577 for (const auto *I : MD->parameters()) {
578 argTys.push_back(Elt: Context.getCanonicalParamType(T: I->getType()));
579 auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
580 NoEscape: I->hasAttr<NoEscapeAttr>());
581 extParamInfos.push_back(Elt: extParamInfo);
582 }
583
584 FunctionType::ExtInfo einfo;
585 bool IsTargetDefaultMSABI =
586 getContext().getTargetInfo().getTriple().isOSWindows() ||
587 getContext().getTargetInfo().getTriple().isUEFI();
588 einfo = einfo.withCallingConv(
589 cc: getCallingConventionForDecl(D: MD, IsTargetDefaultMSABI));
590
591 if (getContext().getLangOpts().ObjCAutoRefCount &&
592 MD->hasAttr<NSReturnsRetainedAttr>())
593 einfo = einfo.withProducesResult(producesResult: true);
594
595 RequiredArgs required =
596 (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);
597
598 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: MD->getReturnType()),
599 opts: FnInfoOpts::None, argTypes: argTys, info: einfo, paramInfos: extParamInfos,
600 args: required);
601}
602
603const CGFunctionInfo &
604CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
605 const CallArgList &args) {
606 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
607 FunctionType::ExtInfo einfo;
608
609 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: returnType), opts: FnInfoOpts::None,
610 argTypes, info: einfo, paramInfos: {}, args: RequiredArgs::All);
611}
612
613const CGFunctionInfo &CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
614 // FIXME: Do we need to handle ObjCMethodDecl?
615 if (isa<CXXConstructorDecl>(Val: GD.getDecl()) ||
616 isa<CXXDestructorDecl>(Val: GD.getDecl()))
617 return arrangeCXXStructorDeclaration(GD);
618
619 return arrangeFunctionDeclaration(GD);
620}
621
622/// Arrange a thunk that takes 'this' as the first parameter followed by
623/// varargs. Return a void pointer, regardless of the actual return type.
624/// The body of the thunk will end in a musttail call to a function of the
625/// correct type, and the caller will bitcast the function to the correct
626/// prototype.
627const CGFunctionInfo &
628CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
629 assert(MD->isVirtual() && "only methods have thunks");
630 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
631 CanQualType ArgTys[] = {DeriveThisType(RD: MD->getParent(), MD)};
632 return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::None, argTypes: ArgTys,
633 info: FTP->getExtInfo(), paramInfos: {}, args: RequiredArgs(1));
634}
635
636const CGFunctionInfo &
637CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
638 CXXCtorType CT) {
639 assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
640
641 CanQual<FunctionProtoType> FTP = GetFormalType(MD: CD);
642 SmallVector<CanQualType, 2> ArgTys;
643 const CXXRecordDecl *RD = CD->getParent();
644 ArgTys.push_back(Elt: DeriveThisType(RD, MD: CD));
645 if (CT == Ctor_CopyingClosure)
646 ArgTys.push_back(Elt: *FTP->param_type_begin());
647 if (RD->getNumVBases() > 0)
648 ArgTys.push_back(Elt: Context.IntTy);
649 CallingConv CC = Context.getDefaultCallingConvention(
650 /*IsVariadic=*/false, /*IsCXXMethod=*/true);
651 return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::IsInstanceMethod,
652 argTypes: ArgTys, info: FunctionType::ExtInfo(CC), paramInfos: {},
653 args: RequiredArgs::All);
654}
655
656/// Arrange a call as unto a free function, except possibly with an
657/// additional number of formal parameters considered required.
658static const CGFunctionInfo &
659arrangeFreeFunctionLikeCall(CodeGenTypes &CGT, CodeGenModule &CGM,
660 const CallArgList &args, const FunctionType *fnType,
661 unsigned numExtraRequiredArgs, bool chainCall) {
662 assert(args.size() >= numExtraRequiredArgs);
663
664 ExtParameterInfoList paramInfos;
665
666 // In most cases, there are no optional arguments.
667 RequiredArgs required = RequiredArgs::All;
668
669 // If we have a variadic prototype, the required arguments are the
670 // extra prefix plus the arguments in the prototype.
671 if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(Val: fnType)) {
672 if (proto->isVariadic())
673 required = RequiredArgs::forPrototypePlus(prototype: proto, additional: numExtraRequiredArgs);
674
675 if (proto->hasExtParameterInfos())
676 addExtParameterInfosForCall(paramInfos, proto, prefixArgs: numExtraRequiredArgs,
677 totalArgs: args.size());
678
679 // If we don't have a prototype at all, but we're supposed to
680 // explicitly use the variadic convention for unprototyped calls,
681 // treat all of the arguments as required but preserve the nominal
682 // possibility of variadics.
683 } else if (CGM.getTargetCodeGenInfo().isNoProtoCallVariadic(
684 args, fnType: cast<FunctionNoProtoType>(Val: fnType))) {
685 required = RequiredArgs(args.size());
686 }
687
688 CanQualTypeList argTypes;
689 for (const auto &arg : args)
690 argTypes.push_back(Elt: CGT.getContext().getCanonicalParamType(T: arg.Ty));
691 FnInfoOpts opts = chainCall ? FnInfoOpts::IsChainCall : FnInfoOpts::None;
692 return CGT.arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: fnType->getReturnType()),
693 opts, argTypes, info: fnType->getExtInfo(),
694 paramInfos, args: required);
695}
696
697/// Figure out the rules for calling a function with the given formal
698/// type using the given arguments. The arguments are necessary
699/// because the function might be unprototyped, in which case it's
700/// target-dependent in crazy ways.
701const CGFunctionInfo &CodeGenTypes::arrangeFreeFunctionCall(
702 const CallArgList &args, const FunctionType *fnType, bool chainCall) {
703 return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType,
704 numExtraRequiredArgs: chainCall ? 1 : 0, chainCall);
705}
706
707/// A block function is essentially a free function with an
708/// extra implicit argument.
709const CGFunctionInfo &
710CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
711 const FunctionType *fnType) {
712 return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType, numExtraRequiredArgs: 1,
713 /*chainCall=*/false);
714}
715
716const CGFunctionInfo &
717CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
718 const FunctionArgList &params) {
719 ExtParameterInfoList paramInfos =
720 getExtParameterInfosForCall(proto, prefixArgs: 1, totalArgs: params.size());
721 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args: params);
722
723 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: proto->getReturnType()),
724 opts: FnInfoOpts::None, argTypes,
725 info: proto->getExtInfo(), paramInfos,
726 args: RequiredArgs::forPrototypePlus(prototype: proto, additional: 1));
727}
728
729const CGFunctionInfo &
730CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
731 const CallArgList &args) {
732 CanQualTypeList argTypes;
733 for (const auto &Arg : args)
734 argTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
735 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
736 argTypes, info: FunctionType::ExtInfo(),
737 /*paramInfos=*/{}, args: RequiredArgs::All);
738}
739
740const CGFunctionInfo &
741CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
742 const FunctionArgList &args) {
743 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
744
745 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
746 argTypes, info: FunctionType::ExtInfo(), paramInfos: {},
747 args: RequiredArgs::All);
748}
749
750const CGFunctionInfo &CodeGenTypes::arrangeBuiltinFunctionDeclaration(
751 CanQualType resultType, ArrayRef<CanQualType> argTypes) {
752 return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::None, argTypes,
753 info: FunctionType::ExtInfo(), paramInfos: {},
754 args: RequiredArgs::All);
755}
756
757const CGFunctionInfo &CodeGenTypes::arrangeDeviceKernelCallerDeclaration(
758 QualType resultType, const FunctionArgList &args) {
759 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
760
761 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
762 argTypes,
763 info: FunctionType::ExtInfo(CC_DeviceKernel),
764 /*paramInfos=*/{}, args: RequiredArgs::All);
765}
766
767/// Arrange a call to a C++ method, passing the given arguments.
768///
769/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
770/// does not count `this`.
771const CGFunctionInfo &CodeGenTypes::arrangeCXXMethodCall(
772 const CallArgList &args, const FunctionProtoType *proto,
773 RequiredArgs required, unsigned numPrefixArgs) {
774 assert(numPrefixArgs + 1 <= args.size() &&
775 "Emitting a call with less args than the required prefix?");
776 // Add one to account for `this`. It's a bit awkward here, but we don't count
777 // `this` in similar places elsewhere.
778 ExtParameterInfoList paramInfos =
779 getExtParameterInfosForCall(proto, prefixArgs: numPrefixArgs + 1, totalArgs: args.size());
780
781 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
782
783 FunctionType::ExtInfo info = proto->getExtInfo();
784 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: proto->getReturnType()),
785 opts: FnInfoOpts::IsInstanceMethod, argTypes, info,
786 paramInfos, args: required);
787}
788
789const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
790 return arrangeLLVMFunctionInfo(returnType: getContext().VoidTy, opts: FnInfoOpts::None, argTypes: {},
791 info: FunctionType::ExtInfo(), paramInfos: {},
792 args: RequiredArgs::All);
793}
794
795const CGFunctionInfo &CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
796 const CallArgList &args) {
797 assert(signature.arg_size() <= args.size());
798 if (signature.arg_size() == args.size())
799 return signature;
800
801 ExtParameterInfoList paramInfos;
802 auto sigParamInfos = signature.getExtParameterInfos();
803 if (!sigParamInfos.empty()) {
804 paramInfos.append(in_start: sigParamInfos.begin(), in_end: sigParamInfos.end());
805 paramInfos.resize(N: args.size());
806 }
807
808 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
809
810 assert(signature.getRequiredArgs().allowsOptionalArgs());
811 FnInfoOpts opts = FnInfoOpts::None;
812 if (signature.isInstanceMethod())
813 opts |= FnInfoOpts::IsInstanceMethod;
814 if (signature.isChainCall())
815 opts |= FnInfoOpts::IsChainCall;
816 if (signature.isDelegateCall())
817 opts |= FnInfoOpts::IsDelegateCall;
818 return arrangeLLVMFunctionInfo(returnType: signature.getReturnType(), opts, argTypes,
819 info: signature.getExtInfo(), paramInfos,
820 args: signature.getRequiredArgs());
821}
822
823namespace clang {
824namespace CodeGen {
825void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
826}
827} // namespace clang
828
829/// Arrange the argument and result information for an abstract value
830/// of a given function type. This is the method which all of the
831/// above functions ultimately defer to.
832const CGFunctionInfo &CodeGenTypes::arrangeLLVMFunctionInfo(
833 CanQualType resultType, FnInfoOpts opts, ArrayRef<CanQualType> argTypes,
834 FunctionType::ExtInfo info,
835 ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
836 RequiredArgs required) {
837 assert(llvm::all_of(argTypes,
838 [](CanQualType T) { return T.isCanonicalAsParam(); }));
839
840 // Lookup or create unique function info.
841 llvm::FoldingSetNodeID ID;
842 bool isInstanceMethod =
843 (opts & FnInfoOpts::IsInstanceMethod) == FnInfoOpts::IsInstanceMethod;
844 bool isChainCall =
845 (opts & FnInfoOpts::IsChainCall) == FnInfoOpts::IsChainCall;
846 bool isDelegateCall =
847 (opts & FnInfoOpts::IsDelegateCall) == FnInfoOpts::IsDelegateCall;
848 CGFunctionInfo::Profile(ID, InstanceMethod: isInstanceMethod, ChainCall: isChainCall, IsDelegateCall: isDelegateCall,
849 info, paramInfos, required, resultType, argTypes);
850
851 void *insertPos = nullptr;
852 CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, InsertPos&: insertPos);
853 if (FI)
854 return *FI;
855
856 unsigned CC = ClangCallConvToLLVMCallConv(CC: info.getCC());
857
858 // Construct the function info. We co-allocate the ArgInfos.
859 FI = CGFunctionInfo::create(llvmCC: CC, instanceMethod: isInstanceMethod, chainCall: isChainCall, delegateCall: isDelegateCall,
860 extInfo: info, paramInfos, resultType, argTypes, required);
861 FunctionInfos.InsertNode(N: FI, InsertPos: insertPos);
862
863 bool inserted = FunctionsBeingProcessed.insert(Ptr: FI).second;
864 (void)inserted;
865 assert(inserted && "Recursively being processed?");
866
867 // Compute ABI information.
868 if (CC == llvm::CallingConv::SPIR_KERNEL) {
869 // Force target independent argument handling for the host visible
870 // kernel functions.
871 computeSPIRKernelABIInfo(CGM, FI&: *FI);
872 } else if (info.getCC() == CC_Swift || info.getCC() == CC_SwiftAsync) {
873 swiftcall::computeABIInfo(CGM, FI&: *FI);
874 } else {
875 CGM.getABIInfo().computeInfo(FI&: *FI);
876 }
877
878 // Loop over all of the computed argument and return value info. If any of
879 // them are direct or extend without a specified coerce type, specify the
880 // default now.
881 ABIArgInfo &retInfo = FI->getReturnInfo();
882 if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
883 retInfo.setCoerceToType(ConvertType(T: FI->getReturnType()));
884
885 for (auto &I : FI->arguments())
886 if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
887 I.info.setCoerceToType(ConvertType(T: I.type));
888
889 bool erased = FunctionsBeingProcessed.erase(Ptr: FI);
890 (void)erased;
891 assert(erased && "Not in set?");
892
893 return *FI;
894}
895
896CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC, bool instanceMethod,
897 bool chainCall, bool delegateCall,
898 const FunctionType::ExtInfo &info,
899 ArrayRef<ExtParameterInfo> paramInfos,
900 CanQualType resultType,
901 ArrayRef<CanQualType> argTypes,
902 RequiredArgs required) {
903 assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
904 assert(!required.allowsOptionalArgs() ||
905 required.getNumRequiredArgs() <= argTypes.size());
906
907 void *buffer = operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(
908 Counts: argTypes.size() + 1, Counts: paramInfos.size()));
909
910 CGFunctionInfo *FI = new (buffer) CGFunctionInfo();
911 FI->CallingConvention = llvmCC;
912 FI->EffectiveCallingConvention = llvmCC;
913 FI->ASTCallingConvention = info.getCC();
914 FI->InstanceMethod = instanceMethod;
915 FI->ChainCall = chainCall;
916 FI->DelegateCall = delegateCall;
917 FI->CmseNSCall = info.getCmseNSCall();
918 FI->NoReturn = info.getNoReturn();
919 FI->ReturnsRetained = info.getProducesResult();
920 FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
921 FI->NoCfCheck = info.getNoCfCheck();
922 FI->Required = required;
923 FI->HasRegParm = info.getHasRegParm();
924 FI->RegParm = info.getRegParm();
925 FI->ArgStruct = nullptr;
926 FI->ArgStructAlign = 0;
927 FI->NumArgs = argTypes.size();
928 FI->HasExtParameterInfos = !paramInfos.empty();
929 FI->getArgsBuffer()[0].type = resultType;
930 FI->MaxVectorWidth = 0;
931 for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
932 FI->getArgsBuffer()[i + 1].type = argTypes[i];
933 for (unsigned i = 0, e = paramInfos.size(); i != e; ++i)
934 FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
935 return FI;
936}
937
938/***/
939
940namespace {
941// ABIArgInfo::Expand implementation.
942
943// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
944struct TypeExpansion {
945 enum TypeExpansionKind {
946 // Elements of constant arrays are expanded recursively.
947 TEK_ConstantArray,
948 // Record fields are expanded recursively (but if record is a union, only
949 // the field with the largest size is expanded).
950 TEK_Record,
951 // For complex types, real and imaginary parts are expanded recursively.
952 TEK_Complex,
953 // All other types are not expandable.
954 TEK_None
955 };
956
957 const TypeExpansionKind Kind;
958
959 TypeExpansion(TypeExpansionKind K) : Kind(K) {}
960 virtual ~TypeExpansion() {}
961};
962
963struct ConstantArrayExpansion : TypeExpansion {
964 QualType EltTy;
965 uint64_t NumElts;
966
967 ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
968 : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
969 static bool classof(const TypeExpansion *TE) {
970 return TE->Kind == TEK_ConstantArray;
971 }
972};
973
974struct RecordExpansion : TypeExpansion {
975 SmallVector<const CXXBaseSpecifier *, 1> Bases;
976
977 SmallVector<const FieldDecl *, 1> Fields;
978
979 RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
980 SmallVector<const FieldDecl *, 1> &&Fields)
981 : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
982 Fields(std::move(Fields)) {}
983 static bool classof(const TypeExpansion *TE) {
984 return TE->Kind == TEK_Record;
985 }
986};
987
988struct ComplexExpansion : TypeExpansion {
989 QualType EltTy;
990
991 ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
992 static bool classof(const TypeExpansion *TE) {
993 return TE->Kind == TEK_Complex;
994 }
995};
996
997struct NoExpansion : TypeExpansion {
998 NoExpansion() : TypeExpansion(TEK_None) {}
999 static bool classof(const TypeExpansion *TE) { return TE->Kind == TEK_None; }
1000};
1001} // namespace
1002
1003static std::unique_ptr<TypeExpansion>
1004getTypeExpansion(QualType Ty, const ASTContext &Context) {
1005 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty)) {
1006 return std::make_unique<ConstantArrayExpansion>(args: AT->getElementType(),
1007 args: AT->getZExtSize());
1008 }
1009 if (const auto *RD = Ty->getAsRecordDecl()) {
1010 SmallVector<const CXXBaseSpecifier *, 1> Bases;
1011 SmallVector<const FieldDecl *, 1> Fields;
1012 assert(!RD->hasFlexibleArrayMember() &&
1013 "Cannot expand structure with flexible array.");
1014 if (RD->isUnion()) {
1015 // Unions can be here only in degenerative cases - all the fields are same
1016 // after flattening. Thus we have to use the "largest" field.
1017 const FieldDecl *LargestFD = nullptr;
1018 CharUnits UnionSize = CharUnits::Zero();
1019
1020 for (const auto *FD : RD->fields()) {
1021 if (FD->isZeroLengthBitField())
1022 continue;
1023 assert(!FD->isBitField() &&
1024 "Cannot expand structure with bit-field members.");
1025 CharUnits FieldSize = Context.getTypeSizeInChars(T: FD->getType());
1026 if (UnionSize < FieldSize) {
1027 UnionSize = FieldSize;
1028 LargestFD = FD;
1029 }
1030 }
1031 if (LargestFD)
1032 Fields.push_back(Elt: LargestFD);
1033 } else {
1034 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
1035 assert(!CXXRD->isDynamicClass() &&
1036 "cannot expand vtable pointers in dynamic classes");
1037 llvm::append_range(C&: Bases, R: llvm::make_pointer_range(Range: CXXRD->bases()));
1038 }
1039
1040 for (const auto *FD : RD->fields()) {
1041 if (FD->isZeroLengthBitField())
1042 continue;
1043 assert(!FD->isBitField() &&
1044 "Cannot expand structure with bit-field members.");
1045 Fields.push_back(Elt: FD);
1046 }
1047 }
1048 return std::make_unique<RecordExpansion>(args: std::move(Bases),
1049 args: std::move(Fields));
1050 }
1051 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
1052 return std::make_unique<ComplexExpansion>(args: CT->getElementType());
1053 }
1054 return std::make_unique<NoExpansion>();
1055}
1056
1057static int getExpansionSize(QualType Ty, const ASTContext &Context) {
1058 auto Exp = getTypeExpansion(Ty, Context);
1059 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1060 return CAExp->NumElts * getExpansionSize(Ty: CAExp->EltTy, Context);
1061 }
1062 if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1063 int Res = 0;
1064 for (auto BS : RExp->Bases)
1065 Res += getExpansionSize(Ty: BS->getType(), Context);
1066 for (auto FD : RExp->Fields)
1067 Res += getExpansionSize(Ty: FD->getType(), Context);
1068 return Res;
1069 }
1070 if (isa<ComplexExpansion>(Val: Exp.get()))
1071 return 2;
1072 assert(isa<NoExpansion>(Exp.get()));
1073 return 1;
1074}
1075
1076void CodeGenTypes::getExpandedTypes(
1077 QualType Ty, SmallVectorImpl<llvm::Type *>::iterator &TI) {
1078 auto Exp = getTypeExpansion(Ty, Context);
1079 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1080 for (int i = 0, n = CAExp->NumElts; i < n; i++) {
1081 getExpandedTypes(Ty: CAExp->EltTy, TI);
1082 }
1083 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1084 for (auto BS : RExp->Bases)
1085 getExpandedTypes(Ty: BS->getType(), TI);
1086 for (auto FD : RExp->Fields)
1087 getExpandedTypes(Ty: FD->getType(), TI);
1088 } else if (auto CExp = dyn_cast<ComplexExpansion>(Val: Exp.get())) {
1089 llvm::Type *EltTy = ConvertType(T: CExp->EltTy);
1090 *TI++ = EltTy;
1091 *TI++ = EltTy;
1092 } else {
1093 assert(isa<NoExpansion>(Exp.get()));
1094 *TI++ = ConvertType(T: Ty);
1095 }
1096}
1097
1098static void forConstantArrayExpansion(CodeGenFunction &CGF,
1099 ConstantArrayExpansion *CAE,
1100 Address BaseAddr,
1101 llvm::function_ref<void(Address)> Fn) {
1102 for (int i = 0, n = CAE->NumElts; i < n; i++) {
1103 Address EltAddr = CGF.Builder.CreateConstGEP2_32(Addr: BaseAddr, Idx0: 0, Idx1: i);
1104 Fn(EltAddr);
1105 }
1106}
1107
1108void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1109 llvm::Function::arg_iterator &AI) {
1110 assert(LV.isSimple() &&
1111 "Unexpected non-simple lvalue during struct expansion.");
1112
1113 auto Exp = getTypeExpansion(Ty, Context: getContext());
1114 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1115 forConstantArrayExpansion(
1116 CGF&: *this, CAE: CAExp, BaseAddr: LV.getAddress(), Fn: [&](Address EltAddr) {
1117 LValue LV = MakeAddrLValue(Addr: EltAddr, T: CAExp->EltTy);
1118 ExpandTypeFromArgs(Ty: CAExp->EltTy, LV, AI);
1119 });
1120 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1121 Address This = LV.getAddress();
1122 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1123 // Perform a single step derived-to-base conversion.
1124 Address Base =
1125 GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1126 /*NullCheckValue=*/false, Loc: SourceLocation());
1127 LValue SubLV = MakeAddrLValue(Addr: Base, T: BS->getType());
1128
1129 // Recurse onto bases.
1130 ExpandTypeFromArgs(Ty: BS->getType(), LV: SubLV, AI);
1131 }
1132 for (auto FD : RExp->Fields) {
1133 // FIXME: What are the right qualifiers here?
1134 LValue SubLV = EmitLValueForFieldInitialization(Base: LV, Field: FD);
1135 ExpandTypeFromArgs(Ty: FD->getType(), LV: SubLV, AI);
1136 }
1137 } else if (isa<ComplexExpansion>(Val: Exp.get())) {
1138 auto realValue = &*AI++;
1139 auto imagValue = &*AI++;
1140 EmitStoreOfComplex(V: ComplexPairTy(realValue, imagValue), dest: LV, /*init*/ isInit: true);
1141 } else {
1142 // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1143 // primitive store.
1144 assert(isa<NoExpansion>(Exp.get()));
1145 llvm::Value *Arg = &*AI++;
1146 if (LV.isBitField()) {
1147 EmitStoreThroughLValue(Src: RValue::get(V: Arg), Dst: LV);
1148 } else {
1149 // TODO: currently there are some places are inconsistent in what LLVM
1150 // pointer type they use (see D118744). Once clang uses opaque pointers
1151 // all LLVM pointer types will be the same and we can remove this check.
1152 if (Arg->getType()->isPointerTy()) {
1153 Address Addr = LV.getAddress();
1154 Arg = Builder.CreateBitCast(V: Arg, DestTy: Addr.getElementType());
1155 }
1156 EmitStoreOfScalar(value: Arg, lvalue: LV);
1157 }
1158 }
1159}
1160
1161void CodeGenFunction::ExpandTypeToArgs(
1162 QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1163 SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1164 auto Exp = getTypeExpansion(Ty, Context: getContext());
1165 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1166 Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1167 : Arg.getKnownRValue().getAggregateAddress();
1168 forConstantArrayExpansion(CGF&: *this, CAE: CAExp, BaseAddr: Addr, Fn: [&](Address EltAddr) {
1169 CallArg EltArg =
1170 CallArg(convertTempToRValue(addr: EltAddr, type: CAExp->EltTy, Loc: SourceLocation()),
1171 CAExp->EltTy);
1172 ExpandTypeToArgs(Ty: CAExp->EltTy, Arg: EltArg, IRFuncTy, IRCallArgs,
1173 IRCallArgPos);
1174 });
1175 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1176 Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1177 : Arg.getKnownRValue().getAggregateAddress();
1178 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1179 // Perform a single step derived-to-base conversion.
1180 Address Base =
1181 GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1182 /*NullCheckValue=*/false, Loc: SourceLocation());
1183 CallArg BaseArg = CallArg(RValue::getAggregate(addr: Base), BS->getType());
1184
1185 // Recurse onto bases.
1186 ExpandTypeToArgs(Ty: BS->getType(), Arg: BaseArg, IRFuncTy, IRCallArgs,
1187 IRCallArgPos);
1188 }
1189
1190 LValue LV = MakeAddrLValue(Addr: This, T: Ty);
1191 for (auto FD : RExp->Fields) {
1192 CallArg FldArg =
1193 CallArg(EmitRValueForField(LV, FD, Loc: SourceLocation()), FD->getType());
1194 ExpandTypeToArgs(Ty: FD->getType(), Arg: FldArg, IRFuncTy, IRCallArgs,
1195 IRCallArgPos);
1196 }
1197 } else if (isa<ComplexExpansion>(Val: Exp.get())) {
1198 ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1199 IRCallArgs[IRCallArgPos++] = CV.first;
1200 IRCallArgs[IRCallArgPos++] = CV.second;
1201 } else {
1202 assert(isa<NoExpansion>(Exp.get()));
1203 auto RV = Arg.getKnownRValue();
1204 assert(RV.isScalar() &&
1205 "Unexpected non-scalar rvalue during struct expansion.");
1206
1207 // Insert a bitcast as needed.
1208 llvm::Value *V = RV.getScalarVal();
1209 if (IRCallArgPos < IRFuncTy->getNumParams() &&
1210 V->getType() != IRFuncTy->getParamType(i: IRCallArgPos))
1211 V = Builder.CreateBitCast(V, DestTy: IRFuncTy->getParamType(i: IRCallArgPos));
1212
1213 IRCallArgs[IRCallArgPos++] = V;
1214 }
1215}
1216
1217/// Create a temporary allocation for the purposes of coercion.
1218static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF,
1219 llvm::Type *Ty,
1220 CharUnits MinAlign,
1221 const Twine &Name = "tmp") {
1222 // Don't use an alignment that's worse than what LLVM would prefer.
1223 auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty);
1224 CharUnits Align = std::max(a: MinAlign, b: CharUnits::fromQuantity(Quantity: PrefAlign));
1225
1226 return CGF.CreateTempAlloca(Ty, align: Align, Name: Name + ".coerce");
1227}
1228
1229/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1230/// accessing some number of bytes out of it, try to gep into the struct to get
1231/// at its inner goodness. Dive as deep as possible without entering an element
1232/// with an in-memory size smaller than DstSize.
1233static Address EnterStructPointerForCoercedAccess(Address SrcPtr,
1234 llvm::StructType *SrcSTy,
1235 uint64_t DstSize,
1236 CodeGenFunction &CGF) {
1237 // We can't dive into a zero-element struct.
1238 if (SrcSTy->getNumElements() == 0)
1239 return SrcPtr;
1240
1241 llvm::Type *FirstElt = SrcSTy->getElementType(N: 0);
1242
1243 // If the first elt is at least as large as what we're looking for, or if the
1244 // first element is the same size as the whole struct, we can enter it. The
1245 // comparison must be made on the store size and not the alloca size. Using
1246 // the alloca size may overstate the size of the load.
1247 uint64_t FirstEltSize = CGF.CGM.getDataLayout().getTypeStoreSize(Ty: FirstElt);
1248 if (FirstEltSize < DstSize &&
1249 FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(Ty: SrcSTy))
1250 return SrcPtr;
1251
1252 // GEP into the first element.
1253 SrcPtr = CGF.Builder.CreateStructGEP(Addr: SrcPtr, Index: 0, Name: "coerce.dive");
1254
1255 // If the first element is a struct, recurse.
1256 llvm::Type *SrcTy = SrcPtr.getElementType();
1257 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy))
1258 return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1259
1260 return SrcPtr;
1261}
1262
1263/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1264/// are either integers or pointers. This does a truncation of the value if it
1265/// is too large or a zero extension if it is too small.
1266///
1267/// This behaves as if the value were coerced through memory, so on big-endian
1268/// targets the high bits are preserved in a truncation, while little-endian
1269/// targets preserve the low bits.
1270static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, llvm::Type *Ty,
1271 CodeGenFunction &CGF) {
1272 if (Val->getType() == Ty)
1273 return Val;
1274
1275 if (isa<llvm::PointerType>(Val: Val->getType())) {
1276 // If this is Pointer->Pointer avoid conversion to and from int.
1277 if (isa<llvm::PointerType>(Val: Ty))
1278 return CGF.Builder.CreateBitCast(V: Val, DestTy: Ty, Name: "coerce.val");
1279
1280 // Convert the pointer to an integer so we can play with its width.
1281 Val = CGF.Builder.CreatePtrToInt(V: Val, DestTy: CGF.IntPtrTy, Name: "coerce.val.pi");
1282 }
1283
1284 llvm::Type *DestIntTy = Ty;
1285 if (isa<llvm::PointerType>(Val: DestIntTy))
1286 DestIntTy = CGF.IntPtrTy;
1287
1288 if (Val->getType() != DestIntTy) {
1289 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1290 if (DL.isBigEndian()) {
1291 // Preserve the high bits on big-endian targets.
1292 // That is what memory coercion does.
1293 uint64_t SrcSize = DL.getTypeSizeInBits(Ty: Val->getType());
1294 uint64_t DstSize = DL.getTypeSizeInBits(Ty: DestIntTy);
1295
1296 if (SrcSize > DstSize) {
1297 Val = CGF.Builder.CreateLShr(LHS: Val, RHS: SrcSize - DstSize, Name: "coerce.highbits");
1298 Val = CGF.Builder.CreateTrunc(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1299 } else {
1300 Val = CGF.Builder.CreateZExt(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1301 Val = CGF.Builder.CreateShl(LHS: Val, RHS: DstSize - SrcSize, Name: "coerce.highbits");
1302 }
1303 } else {
1304 // Little-endian targets preserve the low bits. No shifts required.
1305 Val = CGF.Builder.CreateIntCast(V: Val, DestTy: DestIntTy, isSigned: false, Name: "coerce.val.ii");
1306 }
1307 }
1308
1309 if (isa<llvm::PointerType>(Val: Ty))
1310 Val = CGF.Builder.CreateIntToPtr(V: Val, DestTy: Ty, Name: "coerce.val.ip");
1311 return Val;
1312}
1313
1314/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1315/// a pointer to an object of type \arg Ty, known to be aligned to
1316/// \arg SrcAlign bytes.
1317///
1318/// This safely handles the case when the src type is smaller than the
1319/// destination type; in this situation the values of bits which not
1320/// present in the src are undefined.
1321static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
1322 CodeGenFunction &CGF) {
1323 llvm::Type *SrcTy = Src.getElementType();
1324
1325 // If SrcTy and Ty are the same, just do a load.
1326 if (SrcTy == Ty)
1327 return CGF.Builder.CreateLoad(Addr: Src);
1328
1329 llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1330
1331 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy)) {
1332 Src = EnterStructPointerForCoercedAccess(SrcPtr: Src, SrcSTy,
1333 DstSize: DstSize.getFixedValue(), CGF);
1334 SrcTy = Src.getElementType();
1335 }
1336
1337 llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1338
1339 // If the source and destination are integer or pointer types, just do an
1340 // extension or truncation to the desired type.
1341 if ((isa<llvm::IntegerType>(Val: Ty) || isa<llvm::PointerType>(Val: Ty)) &&
1342 (isa<llvm::IntegerType>(Val: SrcTy) || isa<llvm::PointerType>(Val: SrcTy))) {
1343 llvm::Value *Load = CGF.Builder.CreateLoad(Addr: Src);
1344 return CoerceIntOrPtrToIntOrPtr(Val: Load, Ty, CGF);
1345 }
1346
1347 // If load is legal, just bitcast the src pointer.
1348 if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1349 SrcSize.getFixedValue() >= DstSize.getFixedValue()) {
1350 // Generally SrcSize is never greater than DstSize, since this means we are
1351 // losing bits. However, this can happen in cases where the structure has
1352 // additional padding, for example due to a user specified alignment.
1353 //
1354 // FIXME: Assert that we aren't truncating non-padding bits when have access
1355 // to that information.
1356 Src = Src.withElementType(ElemTy: Ty);
1357 return CGF.Builder.CreateLoad(Addr: Src);
1358 }
1359
1360 // If coercing a fixed vector to a scalable vector for ABI compatibility, and
1361 // the types match, use the llvm.vector.insert intrinsic to perform the
1362 // conversion.
1363 if (auto *ScalableDstTy = dyn_cast<llvm::ScalableVectorType>(Val: Ty)) {
1364 if (auto *FixedSrcTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy)) {
1365 // If we are casting a fixed i8 vector to a scalable i1 predicate
1366 // vector, use a vector insert and bitcast the result.
1367 if (ScalableDstTy->getElementType()->isIntegerTy(Bitwidth: 1) &&
1368 FixedSrcTy->getElementType()->isIntegerTy(Bitwidth: 8)) {
1369 ScalableDstTy = llvm::ScalableVectorType::get(
1370 ElementType: FixedSrcTy->getElementType(),
1371 MinNumElts: llvm::divideCeil(
1372 Numerator: ScalableDstTy->getElementCount().getKnownMinValue(), Denominator: 8));
1373 }
1374 if (ScalableDstTy->getElementType() == FixedSrcTy->getElementType()) {
1375 auto *Load = CGF.Builder.CreateLoad(Addr: Src);
1376 auto *PoisonVec = llvm::PoisonValue::get(T: ScalableDstTy);
1377 llvm::Value *Result = CGF.Builder.CreateInsertVector(
1378 DstType: ScalableDstTy, SrcVec: PoisonVec, SubVec: Load, Idx: uint64_t(0), Name: "cast.scalable");
1379 ScalableDstTy = cast<llvm::ScalableVectorType>(
1380 Val: llvm::VectorType::getWithSizeAndScalar(SizeTy: ScalableDstTy, EltTy: Ty));
1381 if (Result->getType() != ScalableDstTy)
1382 Result = CGF.Builder.CreateBitCast(V: Result, DestTy: ScalableDstTy);
1383 if (Result->getType() != Ty)
1384 Result = CGF.Builder.CreateExtractVector(DstType: Ty, SrcVec: Result, Idx: uint64_t(0));
1385 return Result;
1386 }
1387 }
1388 }
1389
1390 // Otherwise do coercion through memory. This is stupid, but simple.
1391 RawAddress Tmp =
1392 CreateTempAllocaForCoercion(CGF, Ty, MinAlign: Src.getAlignment(), Name: Src.getName());
1393 CGF.Builder.CreateMemCpy(
1394 Dst: Tmp.getPointer(), DstAlign: Tmp.getAlignment().getAsAlign(),
1395 Src: Src.emitRawPointer(CGF), SrcAlign: Src.getAlignment().getAsAlign(),
1396 Size: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: SrcSize.getKnownMinValue()));
1397 return CGF.Builder.CreateLoad(Addr: Tmp);
1398}
1399
1400void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, Address Dst,
1401 llvm::TypeSize DstSize,
1402 bool DstIsVolatile) {
1403 if (!DstSize)
1404 return;
1405
1406 llvm::Type *SrcTy = Src->getType();
1407 llvm::TypeSize SrcSize = CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1408
1409 // GEP into structs to try to make types match.
1410 // FIXME: This isn't really that useful with opaque types, but it impacts a
1411 // lot of regression tests.
1412 if (SrcTy != Dst.getElementType()) {
1413 if (llvm::StructType *DstSTy =
1414 dyn_cast<llvm::StructType>(Val: Dst.getElementType())) {
1415 assert(!SrcSize.isScalable());
1416 Dst = EnterStructPointerForCoercedAccess(SrcPtr: Dst, SrcSTy: DstSTy,
1417 DstSize: SrcSize.getFixedValue(), CGF&: *this);
1418 }
1419 }
1420
1421 if (SrcSize.isScalable() || SrcSize <= DstSize) {
1422 if (SrcTy->isIntegerTy() && Dst.getElementType()->isPointerTy() &&
1423 SrcSize == CGM.getDataLayout().getTypeAllocSize(Ty: Dst.getElementType())) {
1424 // If the value is supposed to be a pointer, convert it before storing it.
1425 Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: Dst.getElementType(), CGF&: *this);
1426 auto *I = Builder.CreateStore(Val: Src, Addr: Dst, IsVolatile: DstIsVolatile);
1427 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1428 } else if (llvm::StructType *STy =
1429 dyn_cast<llvm::StructType>(Val: Src->getType())) {
1430 // Prefer scalar stores to first-class aggregate stores.
1431 Dst = Dst.withElementType(ElemTy: SrcTy);
1432 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1433 Address EltPtr = Builder.CreateStructGEP(Addr: Dst, Index: i);
1434 llvm::Value *Elt = Builder.CreateExtractValue(Agg: Src, Idxs: i);
1435 auto *I = Builder.CreateStore(Val: Elt, Addr: EltPtr, IsVolatile: DstIsVolatile);
1436 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Elt);
1437 }
1438 } else {
1439 auto *I =
1440 Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: SrcTy), IsVolatile: DstIsVolatile);
1441 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1442 }
1443 } else if (SrcTy->isIntegerTy()) {
1444 // If the source is a simple integer, coerce it directly.
1445 llvm::Type *DstIntTy = Builder.getIntNTy(N: DstSize.getFixedValue() * 8);
1446 Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: DstIntTy, CGF&: *this);
1447 auto *I =
1448 Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: DstIntTy), IsVolatile: DstIsVolatile);
1449 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1450 } else {
1451 // Otherwise do coercion through memory. This is stupid, but
1452 // simple.
1453
1454 // Generally SrcSize is never greater than DstSize, since this means we are
1455 // losing bits. However, this can happen in cases where the structure has
1456 // additional padding, for example due to a user specified alignment.
1457 //
1458 // FIXME: Assert that we aren't truncating non-padding bits when have access
1459 // to that information.
1460 RawAddress Tmp =
1461 CreateTempAllocaForCoercion(CGF&: *this, Ty: SrcTy, MinAlign: Dst.getAlignment());
1462 Builder.CreateStore(Val: Src, Addr: Tmp);
1463 auto *I = Builder.CreateMemCpy(
1464 Dst: Dst.emitRawPointer(CGF&: *this), DstAlign: Dst.getAlignment().getAsAlign(),
1465 Src: Tmp.getPointer(), SrcAlign: Tmp.getAlignment().getAsAlign(),
1466 Size: Builder.CreateTypeSize(Ty: IntPtrTy, Size: DstSize));
1467 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1468 }
1469}
1470
1471static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1472 const ABIArgInfo &info) {
1473 if (unsigned offset = info.getDirectOffset()) {
1474 addr = addr.withElementType(ElemTy: CGF.Int8Ty);
1475 addr = CGF.Builder.CreateConstInBoundsByteGEP(
1476 Addr: addr, Offset: CharUnits::fromQuantity(Quantity: offset));
1477 addr = addr.withElementType(ElemTy: info.getCoerceToType());
1478 }
1479 return addr;
1480}
1481
1482static std::pair<llvm::Value *, bool>
1483CoerceScalableToFixed(CodeGenFunction &CGF, llvm::FixedVectorType *ToTy,
1484 llvm::ScalableVectorType *FromTy, llvm::Value *V,
1485 StringRef Name = "") {
1486 // If we are casting a scalable i1 predicate vector to a fixed i8
1487 // vector, first bitcast the source.
1488 if (FromTy->getElementType()->isIntegerTy(Bitwidth: 1) &&
1489 ToTy->getElementType() == CGF.Builder.getInt8Ty()) {
1490 if (!FromTy->getElementCount().isKnownMultipleOf(RHS: 8)) {
1491 FromTy = llvm::ScalableVectorType::get(
1492 ElementType: FromTy->getElementType(),
1493 MinNumElts: llvm::alignTo<8>(Value: FromTy->getElementCount().getKnownMinValue()));
1494 llvm::Value *ZeroVec = llvm::Constant::getNullValue(Ty: FromTy);
1495 V = CGF.Builder.CreateInsertVector(DstType: FromTy, SrcVec: ZeroVec, SubVec: V, Idx: uint64_t(0));
1496 }
1497 FromTy = llvm::ScalableVectorType::get(
1498 ElementType: ToTy->getElementType(),
1499 MinNumElts: FromTy->getElementCount().getKnownMinValue() / 8);
1500 V = CGF.Builder.CreateBitCast(V, DestTy: FromTy);
1501 }
1502 if (FromTy->getElementType() == ToTy->getElementType()) {
1503 V->setName(Name + ".coerce");
1504 V = CGF.Builder.CreateExtractVector(DstType: ToTy, SrcVec: V, Idx: uint64_t(0), Name: "cast.fixed");
1505 return {V, true};
1506 }
1507 return {V, false};
1508}
1509
1510namespace {
1511
1512/// Encapsulates information about the way function arguments from
1513/// CGFunctionInfo should be passed to actual LLVM IR function.
1514class ClangToLLVMArgMapping {
1515 static const unsigned InvalidIndex = ~0U;
1516 unsigned InallocaArgNo;
1517 unsigned SRetArgNo;
1518 unsigned TotalIRArgs;
1519
1520 /// Arguments of LLVM IR function corresponding to single Clang argument.
1521 struct IRArgs {
1522 unsigned PaddingArgIndex;
1523 // Argument is expanded to IR arguments at positions
1524 // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1525 unsigned FirstArgIndex;
1526 unsigned NumberOfArgs;
1527
1528 IRArgs()
1529 : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1530 NumberOfArgs(0) {}
1531 };
1532
1533 SmallVector<IRArgs, 8> ArgInfo;
1534
1535public:
1536 ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1537 bool OnlyRequiredArgs = false)
1538 : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1539 ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1540 construct(Context, FI, OnlyRequiredArgs);
1541 }
1542
1543 bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1544 unsigned getInallocaArgNo() const {
1545 assert(hasInallocaArg());
1546 return InallocaArgNo;
1547 }
1548
1549 bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1550 unsigned getSRetArgNo() const {
1551 assert(hasSRetArg());
1552 return SRetArgNo;
1553 }
1554
1555 unsigned totalIRArgs() const { return TotalIRArgs; }
1556
1557 bool hasPaddingArg(unsigned ArgNo) const {
1558 assert(ArgNo < ArgInfo.size());
1559 return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1560 }
1561 unsigned getPaddingArgNo(unsigned ArgNo) const {
1562 assert(hasPaddingArg(ArgNo));
1563 return ArgInfo[ArgNo].PaddingArgIndex;
1564 }
1565
1566 /// Returns index of first IR argument corresponding to ArgNo, and their
1567 /// quantity.
1568 std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1569 assert(ArgNo < ArgInfo.size());
1570 return std::make_pair(x: ArgInfo[ArgNo].FirstArgIndex,
1571 y: ArgInfo[ArgNo].NumberOfArgs);
1572 }
1573
1574private:
1575 void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1576 bool OnlyRequiredArgs);
1577};
1578
1579void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1580 const CGFunctionInfo &FI,
1581 bool OnlyRequiredArgs) {
1582 unsigned IRArgNo = 0;
1583 bool SwapThisWithSRet = false;
1584 const ABIArgInfo &RetAI = FI.getReturnInfo();
1585
1586 if (RetAI.getKind() == ABIArgInfo::Indirect) {
1587 SwapThisWithSRet = RetAI.isSRetAfterThis();
1588 SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1589 }
1590
1591 unsigned ArgNo = 0;
1592 unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();
1593 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1594 ++I, ++ArgNo) {
1595 assert(I != FI.arg_end());
1596 QualType ArgType = I->type;
1597 const ABIArgInfo &AI = I->info;
1598 // Collect data about IR arguments corresponding to Clang argument ArgNo.
1599 auto &IRArgs = ArgInfo[ArgNo];
1600
1601 if (AI.getPaddingType())
1602 IRArgs.PaddingArgIndex = IRArgNo++;
1603
1604 switch (AI.getKind()) {
1605 case ABIArgInfo::TargetSpecific:
1606 case ABIArgInfo::Extend:
1607 case ABIArgInfo::Direct: {
1608 // FIXME: handle sseregparm someday...
1609 llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: AI.getCoerceToType());
1610 if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
1611 IRArgs.NumberOfArgs = STy->getNumElements();
1612 } else {
1613 IRArgs.NumberOfArgs = 1;
1614 }
1615 break;
1616 }
1617 case ABIArgInfo::Indirect:
1618 case ABIArgInfo::IndirectAliased:
1619 IRArgs.NumberOfArgs = 1;
1620 break;
1621 case ABIArgInfo::Ignore:
1622 case ABIArgInfo::InAlloca:
1623 // ignore and inalloca doesn't have matching LLVM parameters.
1624 IRArgs.NumberOfArgs = 0;
1625 break;
1626 case ABIArgInfo::CoerceAndExpand:
1627 IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1628 break;
1629 case ABIArgInfo::Expand:
1630 IRArgs.NumberOfArgs = getExpansionSize(Ty: ArgType, Context);
1631 break;
1632 }
1633
1634 if (IRArgs.NumberOfArgs > 0) {
1635 IRArgs.FirstArgIndex = IRArgNo;
1636 IRArgNo += IRArgs.NumberOfArgs;
1637 }
1638
1639 // Skip over the sret parameter when it comes second. We already handled it
1640 // above.
1641 if (IRArgNo == 1 && SwapThisWithSRet)
1642 IRArgNo++;
1643 }
1644 assert(ArgNo == ArgInfo.size());
1645
1646 if (FI.usesInAlloca())
1647 InallocaArgNo = IRArgNo++;
1648
1649 TotalIRArgs = IRArgNo;
1650}
1651} // namespace
1652
1653/***/
1654
1655bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1656 const auto &RI = FI.getReturnInfo();
1657 return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());
1658}
1659
1660bool CodeGenModule::ReturnTypeHasInReg(const CGFunctionInfo &FI) {
1661 const auto &RI = FI.getReturnInfo();
1662 return RI.getInReg();
1663}
1664
1665bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1666 return ReturnTypeUsesSRet(FI) &&
1667 getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
1668}
1669
1670bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1671 if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1672 switch (BT->getKind()) {
1673 default:
1674 return false;
1675 case BuiltinType::Float:
1676 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Float);
1677 case BuiltinType::Double:
1678 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Double);
1679 case BuiltinType::LongDouble:
1680 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::LongDouble);
1681 }
1682 }
1683
1684 return false;
1685}
1686
1687bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1688 if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1689 if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1690 if (BT->getKind() == BuiltinType::LongDouble)
1691 return getTarget().useObjCFP2RetForComplexLongDouble();
1692 }
1693 }
1694
1695 return false;
1696}
1697
1698llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1699 const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1700 return GetFunctionType(Info: FI);
1701}
1702
1703llvm::FunctionType *CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1704
1705 bool Inserted = FunctionsBeingProcessed.insert(Ptr: &FI).second;
1706 (void)Inserted;
1707 assert(Inserted && "Recursively being processed?");
1708
1709 llvm::Type *resultType = nullptr;
1710 const ABIArgInfo &retAI = FI.getReturnInfo();
1711 switch (retAI.getKind()) {
1712 case ABIArgInfo::Expand:
1713 case ABIArgInfo::IndirectAliased:
1714 llvm_unreachable("Invalid ABI kind for return argument");
1715
1716 case ABIArgInfo::TargetSpecific:
1717 case ABIArgInfo::Extend:
1718 case ABIArgInfo::Direct:
1719 resultType = retAI.getCoerceToType();
1720 break;
1721
1722 case ABIArgInfo::InAlloca:
1723 if (retAI.getInAllocaSRet()) {
1724 // sret things on win32 aren't void, they return the sret pointer.
1725 QualType ret = FI.getReturnType();
1726 unsigned addressSpace = CGM.getTypes().getTargetAddressSpace(T: ret);
1727 resultType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: addressSpace);
1728 } else {
1729 resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
1730 }
1731 break;
1732
1733 case ABIArgInfo::Indirect:
1734 case ABIArgInfo::Ignore:
1735 resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
1736 break;
1737
1738 case ABIArgInfo::CoerceAndExpand:
1739 resultType = retAI.getUnpaddedCoerceAndExpandType();
1740 break;
1741 }
1742
1743 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1744 SmallVector<llvm::Type *, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1745
1746 // Add type for sret argument.
1747 if (IRFunctionArgs.hasSRetArg()) {
1748 ArgTypes[IRFunctionArgs.getSRetArgNo()] = llvm::PointerType::get(
1749 C&: getLLVMContext(), AddressSpace: FI.getReturnInfo().getIndirectAddrSpace());
1750 }
1751
1752 // Add type for inalloca argument.
1753 if (IRFunctionArgs.hasInallocaArg())
1754 ArgTypes[IRFunctionArgs.getInallocaArgNo()] =
1755 llvm::PointerType::getUnqual(C&: getLLVMContext());
1756
1757 // Add in all of the required arguments.
1758 unsigned ArgNo = 0;
1759 CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1760 ie = it + FI.getNumRequiredArgs();
1761 for (; it != ie; ++it, ++ArgNo) {
1762 const ABIArgInfo &ArgInfo = it->info;
1763
1764 // Insert a padding type to ensure proper alignment.
1765 if (IRFunctionArgs.hasPaddingArg(ArgNo))
1766 ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1767 ArgInfo.getPaddingType();
1768
1769 unsigned FirstIRArg, NumIRArgs;
1770 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1771
1772 switch (ArgInfo.getKind()) {
1773 case ABIArgInfo::Ignore:
1774 case ABIArgInfo::InAlloca:
1775 assert(NumIRArgs == 0);
1776 break;
1777
1778 case ABIArgInfo::Indirect:
1779 assert(NumIRArgs == 1);
1780 // indirect arguments are always on the stack, which is alloca addr space.
1781 ArgTypes[FirstIRArg] = llvm::PointerType::get(
1782 C&: getLLVMContext(), AddressSpace: CGM.getDataLayout().getAllocaAddrSpace());
1783 break;
1784 case ABIArgInfo::IndirectAliased:
1785 assert(NumIRArgs == 1);
1786 ArgTypes[FirstIRArg] = llvm::PointerType::get(
1787 C&: getLLVMContext(), AddressSpace: ArgInfo.getIndirectAddrSpace());
1788 break;
1789 case ABIArgInfo::TargetSpecific:
1790 case ABIArgInfo::Extend:
1791 case ABIArgInfo::Direct: {
1792 // Fast-isel and the optimizer generally like scalar values better than
1793 // FCAs, so we flatten them if this is safe to do for this argument.
1794 llvm::Type *argType = ArgInfo.getCoerceToType();
1795 llvm::StructType *st = dyn_cast<llvm::StructType>(Val: argType);
1796 if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
1797 assert(NumIRArgs == st->getNumElements());
1798 for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
1799 ArgTypes[FirstIRArg + i] = st->getElementType(N: i);
1800 } else {
1801 assert(NumIRArgs == 1);
1802 ArgTypes[FirstIRArg] = argType;
1803 }
1804 break;
1805 }
1806
1807 case ABIArgInfo::CoerceAndExpand: {
1808 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1809 for (auto *EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1810 *ArgTypesIter++ = EltTy;
1811 }
1812 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1813 break;
1814 }
1815
1816 case ABIArgInfo::Expand:
1817 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1818 getExpandedTypes(Ty: it->type, TI&: ArgTypesIter);
1819 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1820 break;
1821 }
1822 }
1823
1824 bool Erased = FunctionsBeingProcessed.erase(Ptr: &FI);
1825 (void)Erased;
1826 assert(Erased && "Not in set?");
1827
1828 return llvm::FunctionType::get(Result: resultType, Params: ArgTypes, isVarArg: FI.isVariadic());
1829}
1830
1831llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1832 const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
1833 const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
1834
1835 if (!isFuncTypeConvertible(FT: FPT))
1836 return llvm::StructType::get(Context&: getLLVMContext());
1837
1838 return GetFunctionType(GD);
1839}
1840
1841static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
1842 llvm::AttrBuilder &FuncAttrs,
1843 const FunctionProtoType *FPT) {
1844 if (!FPT)
1845 return;
1846
1847 if (!isUnresolvedExceptionSpec(ESpecType: FPT->getExceptionSpecType()) &&
1848 FPT->isNothrow())
1849 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
1850
1851 unsigned SMEBits = FPT->getAArch64SMEAttributes();
1852 if (SMEBits & FunctionType::SME_PStateSMEnabledMask)
1853 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_enabled");
1854 if (SMEBits & FunctionType::SME_PStateSMCompatibleMask)
1855 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_compatible");
1856 if (SMEBits & FunctionType::SME_AgnosticZAStateMask)
1857 FuncAttrs.addAttribute(A: "aarch64_za_state_agnostic");
1858
1859 // ZA
1860 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
1861 FuncAttrs.addAttribute(A: "aarch64_preserves_za");
1862 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_In)
1863 FuncAttrs.addAttribute(A: "aarch64_in_za");
1864 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Out)
1865 FuncAttrs.addAttribute(A: "aarch64_out_za");
1866 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_InOut)
1867 FuncAttrs.addAttribute(A: "aarch64_inout_za");
1868
1869 // ZT0
1870 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
1871 FuncAttrs.addAttribute(A: "aarch64_preserves_zt0");
1872 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_In)
1873 FuncAttrs.addAttribute(A: "aarch64_in_zt0");
1874 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Out)
1875 FuncAttrs.addAttribute(A: "aarch64_out_zt0");
1876 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_InOut)
1877 FuncAttrs.addAttribute(A: "aarch64_inout_zt0");
1878}
1879
1880static void AddAttributesFromOMPAssumes(llvm::AttrBuilder &FuncAttrs,
1881 const Decl *Callee) {
1882 if (!Callee)
1883 return;
1884
1885 SmallVector<StringRef, 4> Attrs;
1886
1887 for (const OMPAssumeAttr *AA : Callee->specific_attrs<OMPAssumeAttr>())
1888 AA->getAssumption().split(A&: Attrs, Separator: ",");
1889
1890 if (!Attrs.empty())
1891 FuncAttrs.addAttribute(A: llvm::AssumptionAttrKey,
1892 V: llvm::join(Begin: Attrs.begin(), End: Attrs.end(), Separator: ","));
1893}
1894
1895bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,
1896 QualType ReturnType) const {
1897 // We can't just discard the return value for a record type with a
1898 // complex destructor or a non-trivially copyable type.
1899 if (const RecordType *RT =
1900 ReturnType.getCanonicalType()->getAsCanonical<RecordType>()) {
1901 if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(Val: RT->getDecl()))
1902 return ClassDecl->hasTrivialDestructor();
1903 }
1904 return ReturnType.isTriviallyCopyableType(Context);
1905}
1906
1907static bool HasStrictReturn(const CodeGenModule &Module, QualType RetTy,
1908 const Decl *TargetDecl) {
1909 // As-is msan can not tolerate noundef mismatch between caller and
1910 // implementation. Mismatch is possible for e.g. indirect calls from C-caller
1911 // into C++. Such mismatches lead to confusing false reports. To avoid
1912 // expensive workaround on msan we enforce initialization event in uncommon
1913 // cases where it's allowed.
1914 if (Module.getLangOpts().Sanitize.has(K: SanitizerKind::Memory))
1915 return true;
1916 // C++ explicitly makes returning undefined values UB. C's rule only applies
1917 // to used values, so we never mark them noundef for now.
1918 if (!Module.getLangOpts().CPlusPlus)
1919 return false;
1920 if (TargetDecl) {
1921 if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
1922 if (FDecl->isExternC())
1923 return false;
1924 } else if (const VarDecl *VDecl = dyn_cast<VarDecl>(Val: TargetDecl)) {
1925 // Function pointer.
1926 if (VDecl->isExternC())
1927 return false;
1928 }
1929 }
1930
1931 // We don't want to be too aggressive with the return checking, unless
1932 // it's explicit in the code opts or we're using an appropriate sanitizer.
1933 // Try to respect what the programmer intended.
1934 return Module.getCodeGenOpts().StrictReturn ||
1935 !Module.MayDropFunctionReturn(Context: Module.getContext(), ReturnType: RetTy) ||
1936 Module.getLangOpts().Sanitize.has(K: SanitizerKind::Return);
1937}
1938
1939/// Add denormal-fp-math and denormal-fp-math-f32 as appropriate for the
1940/// requested denormal behavior, accounting for the overriding behavior of the
1941/// -f32 case.
1942static void addDenormalModeAttrs(llvm::DenormalMode FPDenormalMode,
1943 llvm::DenormalMode FP32DenormalMode,
1944 llvm::AttrBuilder &FuncAttrs) {
1945 if (FPDenormalMode != llvm::DenormalMode::getDefault())
1946 FuncAttrs.addAttribute(A: "denormal-fp-math", V: FPDenormalMode.str());
1947
1948 if (FP32DenormalMode != FPDenormalMode && FP32DenormalMode.isValid())
1949 FuncAttrs.addAttribute(A: "denormal-fp-math-f32", V: FP32DenormalMode.str());
1950}
1951
1952/// Add default attributes to a function, which have merge semantics under
1953/// -mlink-builtin-bitcode and should not simply overwrite any existing
1954/// attributes in the linked library.
1955static void
1956addMergableDefaultFunctionAttributes(const CodeGenOptions &CodeGenOpts,
1957 llvm::AttrBuilder &FuncAttrs) {
1958 addDenormalModeAttrs(FPDenormalMode: CodeGenOpts.FPDenormalMode, FP32DenormalMode: CodeGenOpts.FP32DenormalMode,
1959 FuncAttrs);
1960}
1961
1962static void getTrivialDefaultFunctionAttributes(
1963 StringRef Name, bool HasOptnone, const CodeGenOptions &CodeGenOpts,
1964 const LangOptions &LangOpts, bool AttrOnCallSite,
1965 llvm::AttrBuilder &FuncAttrs) {
1966 // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
1967 if (!HasOptnone) {
1968 if (CodeGenOpts.OptimizeSize)
1969 FuncAttrs.addAttribute(Val: llvm::Attribute::OptimizeForSize);
1970 if (CodeGenOpts.OptimizeSize == 2)
1971 FuncAttrs.addAttribute(Val: llvm::Attribute::MinSize);
1972 }
1973
1974 if (CodeGenOpts.DisableRedZone)
1975 FuncAttrs.addAttribute(Val: llvm::Attribute::NoRedZone);
1976 if (CodeGenOpts.IndirectTlsSegRefs)
1977 FuncAttrs.addAttribute(A: "indirect-tls-seg-refs");
1978 if (CodeGenOpts.NoImplicitFloat)
1979 FuncAttrs.addAttribute(Val: llvm::Attribute::NoImplicitFloat);
1980
1981 if (AttrOnCallSite) {
1982 // Attributes that should go on the call site only.
1983 // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
1984 // the -fno-builtin-foo list.
1985 if (!CodeGenOpts.SimplifyLibCalls || LangOpts.isNoBuiltinFunc(Name))
1986 FuncAttrs.addAttribute(Val: llvm::Attribute::NoBuiltin);
1987 if (!CodeGenOpts.TrapFuncName.empty())
1988 FuncAttrs.addAttribute(A: "trap-func-name", V: CodeGenOpts.TrapFuncName);
1989 } else {
1990 switch (CodeGenOpts.getFramePointer()) {
1991 case CodeGenOptions::FramePointerKind::None:
1992 // This is the default behavior.
1993 break;
1994 case CodeGenOptions::FramePointerKind::Reserved:
1995 case CodeGenOptions::FramePointerKind::NonLeafNoReserve:
1996 case CodeGenOptions::FramePointerKind::NonLeaf:
1997 case CodeGenOptions::FramePointerKind::All:
1998 FuncAttrs.addAttribute(A: "frame-pointer",
1999 V: CodeGenOptions::getFramePointerKindName(
2000 Kind: CodeGenOpts.getFramePointer()));
2001 }
2002
2003 if (CodeGenOpts.LessPreciseFPMAD)
2004 FuncAttrs.addAttribute(A: "less-precise-fpmad", V: "true");
2005
2006 if (CodeGenOpts.NullPointerIsValid)
2007 FuncAttrs.addAttribute(Val: llvm::Attribute::NullPointerIsValid);
2008
2009 if (LangOpts.getDefaultExceptionMode() == LangOptions::FPE_Ignore)
2010 FuncAttrs.addAttribute(A: "no-trapping-math", V: "true");
2011
2012 // TODO: Are these all needed?
2013 // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
2014 if (LangOpts.NoHonorInfs)
2015 FuncAttrs.addAttribute(A: "no-infs-fp-math", V: "true");
2016 if (LangOpts.NoHonorNaNs)
2017 FuncAttrs.addAttribute(A: "no-nans-fp-math", V: "true");
2018 if (CodeGenOpts.SoftFloat)
2019 FuncAttrs.addAttribute(A: "use-soft-float", V: "true");
2020 FuncAttrs.addAttribute(A: "stack-protector-buffer-size",
2021 V: llvm::utostr(X: CodeGenOpts.SSPBufferSize));
2022 if (LangOpts.NoSignedZero)
2023 FuncAttrs.addAttribute(A: "no-signed-zeros-fp-math", V: "true");
2024
2025 // TODO: Reciprocal estimate codegen options should apply to instructions?
2026 const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
2027 if (!Recips.empty())
2028 FuncAttrs.addAttribute(A: "reciprocal-estimates", V: llvm::join(R: Recips, Separator: ","));
2029
2030 if (!CodeGenOpts.PreferVectorWidth.empty() &&
2031 CodeGenOpts.PreferVectorWidth != "none")
2032 FuncAttrs.addAttribute(A: "prefer-vector-width",
2033 V: CodeGenOpts.PreferVectorWidth);
2034
2035 if (CodeGenOpts.StackRealignment)
2036 FuncAttrs.addAttribute(A: "stackrealign");
2037 if (CodeGenOpts.Backchain)
2038 FuncAttrs.addAttribute(A: "backchain");
2039 if (CodeGenOpts.EnableSegmentedStacks)
2040 FuncAttrs.addAttribute(A: "split-stack");
2041
2042 if (CodeGenOpts.SpeculativeLoadHardening)
2043 FuncAttrs.addAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2044
2045 // Add zero-call-used-regs attribute.
2046 switch (CodeGenOpts.getZeroCallUsedRegs()) {
2047 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
2048 FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2049 break;
2050 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
2051 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr-arg");
2052 break;
2053 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
2054 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr");
2055 break;
2056 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
2057 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-arg");
2058 break;
2059 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
2060 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used");
2061 break;
2062 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
2063 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr-arg");
2064 break;
2065 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
2066 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr");
2067 break;
2068 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
2069 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-arg");
2070 break;
2071 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
2072 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all");
2073 break;
2074 }
2075 }
2076
2077 if (LangOpts.assumeFunctionsAreConvergent()) {
2078 // Conservatively, mark all functions and calls in CUDA and OpenCL as
2079 // convergent (meaning, they may call an intrinsically convergent op, such
2080 // as __syncthreads() / barrier(), and so can't have certain optimizations
2081 // applied around them). LLVM will remove this attribute where it safely
2082 // can.
2083 FuncAttrs.addAttribute(Val: llvm::Attribute::Convergent);
2084 }
2085
2086 // TODO: NoUnwind attribute should be added for other GPU modes HIP,
2087 // OpenMP offload. AFAIK, neither of them support exceptions in device code.
2088 if ((LangOpts.CUDA && LangOpts.CUDAIsDevice) || LangOpts.OpenCL ||
2089 LangOpts.SYCLIsDevice) {
2090 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2091 }
2092
2093 if (CodeGenOpts.SaveRegParams && !AttrOnCallSite)
2094 FuncAttrs.addAttribute(A: "save-reg-params");
2095
2096 for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
2097 StringRef Var, Value;
2098 std::tie(args&: Var, args&: Value) = Attr.split(Separator: '=');
2099 FuncAttrs.addAttribute(A: Var, V: Value);
2100 }
2101
2102 TargetInfo::BranchProtectionInfo BPI(LangOpts);
2103 TargetCodeGenInfo::initBranchProtectionFnAttributes(BPI, FuncAttrs);
2104}
2105
2106/// Merges `target-features` from \TargetOpts and \F, and sets the result in
2107/// \FuncAttr
2108/// * features from \F are always kept
2109/// * a feature from \TargetOpts is kept if itself and its opposite are absent
2110/// from \F
2111static void
2112overrideFunctionFeaturesWithTargetFeatures(llvm::AttrBuilder &FuncAttr,
2113 const llvm::Function &F,
2114 const TargetOptions &TargetOpts) {
2115 auto FFeatures = F.getFnAttribute(Kind: "target-features");
2116
2117 llvm::StringSet<> MergedNames;
2118 SmallVector<StringRef> MergedFeatures;
2119 MergedFeatures.reserve(N: TargetOpts.Features.size());
2120
2121 auto AddUnmergedFeatures = [&](auto &&FeatureRange) {
2122 for (StringRef Feature : FeatureRange) {
2123 if (Feature.empty())
2124 continue;
2125 assert(Feature[0] == '+' || Feature[0] == '-');
2126 StringRef Name = Feature.drop_front(N: 1);
2127 bool Merged = !MergedNames.insert(key: Name).second;
2128 if (!Merged)
2129 MergedFeatures.push_back(Elt: Feature);
2130 }
2131 };
2132
2133 if (FFeatures.isValid())
2134 AddUnmergedFeatures(llvm::split(Str: FFeatures.getValueAsString(), Separator: ','));
2135 AddUnmergedFeatures(TargetOpts.Features);
2136
2137 if (!MergedFeatures.empty()) {
2138 llvm::sort(C&: MergedFeatures);
2139 FuncAttr.addAttribute(A: "target-features", V: llvm::join(R&: MergedFeatures, Separator: ","));
2140 }
2141}
2142
2143void CodeGen::mergeDefaultFunctionDefinitionAttributes(
2144 llvm::Function &F, const CodeGenOptions &CodeGenOpts,
2145 const LangOptions &LangOpts, const TargetOptions &TargetOpts,
2146 bool WillInternalize) {
2147
2148 llvm::AttrBuilder FuncAttrs(F.getContext());
2149 // Here we only extract the options that are relevant compared to the version
2150 // from GetCPUAndFeaturesAttributes.
2151 if (!TargetOpts.CPU.empty())
2152 FuncAttrs.addAttribute(A: "target-cpu", V: TargetOpts.CPU);
2153 if (!TargetOpts.TuneCPU.empty())
2154 FuncAttrs.addAttribute(A: "tune-cpu", V: TargetOpts.TuneCPU);
2155
2156 ::getTrivialDefaultFunctionAttributes(Name: F.getName(), HasOptnone: F.hasOptNone(),
2157 CodeGenOpts, LangOpts,
2158 /*AttrOnCallSite=*/false, FuncAttrs);
2159
2160 if (!WillInternalize && F.isInterposable()) {
2161 // Do not promote "dynamic" denormal-fp-math to this translation unit's
2162 // setting for weak functions that won't be internalized. The user has no
2163 // real control for how builtin bitcode is linked, so we shouldn't assume
2164 // later copies will use a consistent mode.
2165 F.addFnAttrs(Attrs: FuncAttrs);
2166 return;
2167 }
2168
2169 llvm::AttributeMask AttrsToRemove;
2170
2171 llvm::DenormalMode DenormModeToMerge = F.getDenormalModeRaw();
2172 llvm::DenormalMode DenormModeToMergeF32 = F.getDenormalModeF32Raw();
2173 llvm::DenormalMode Merged =
2174 CodeGenOpts.FPDenormalMode.mergeCalleeMode(Callee: DenormModeToMerge);
2175 llvm::DenormalMode MergedF32 = CodeGenOpts.FP32DenormalMode;
2176
2177 if (DenormModeToMergeF32.isValid()) {
2178 MergedF32 =
2179 CodeGenOpts.FP32DenormalMode.mergeCalleeMode(Callee: DenormModeToMergeF32);
2180 }
2181
2182 if (Merged == llvm::DenormalMode::getDefault()) {
2183 AttrsToRemove.addAttribute(A: "denormal-fp-math");
2184 } else if (Merged != DenormModeToMerge) {
2185 // Overwrite existing attribute
2186 FuncAttrs.addAttribute(A: "denormal-fp-math",
2187 V: CodeGenOpts.FPDenormalMode.str());
2188 }
2189
2190 if (MergedF32 == llvm::DenormalMode::getDefault()) {
2191 AttrsToRemove.addAttribute(A: "denormal-fp-math-f32");
2192 } else if (MergedF32 != DenormModeToMergeF32) {
2193 // Overwrite existing attribute
2194 FuncAttrs.addAttribute(A: "denormal-fp-math-f32",
2195 V: CodeGenOpts.FP32DenormalMode.str());
2196 }
2197
2198 F.removeFnAttrs(Attrs: AttrsToRemove);
2199 addDenormalModeAttrs(FPDenormalMode: Merged, FP32DenormalMode: MergedF32, FuncAttrs);
2200
2201 overrideFunctionFeaturesWithTargetFeatures(FuncAttr&: FuncAttrs, F, TargetOpts);
2202
2203 F.addFnAttrs(Attrs: FuncAttrs);
2204}
2205
2206void CodeGenModule::getTrivialDefaultFunctionAttributes(
2207 StringRef Name, bool HasOptnone, bool AttrOnCallSite,
2208 llvm::AttrBuilder &FuncAttrs) {
2209 ::getTrivialDefaultFunctionAttributes(Name, HasOptnone, CodeGenOpts: getCodeGenOpts(),
2210 LangOpts: getLangOpts(), AttrOnCallSite,
2211 FuncAttrs);
2212}
2213
2214void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
2215 bool HasOptnone,
2216 bool AttrOnCallSite,
2217 llvm::AttrBuilder &FuncAttrs) {
2218 getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,
2219 FuncAttrs);
2220
2221 if (!AttrOnCallSite)
2222 TargetCodeGenInfo::initPointerAuthFnAttributes(Opts: CodeGenOpts.PointerAuth,
2223 FuncAttrs);
2224
2225 // If we're just getting the default, get the default values for mergeable
2226 // attributes.
2227 if (!AttrOnCallSite)
2228 addMergableDefaultFunctionAttributes(CodeGenOpts, FuncAttrs);
2229}
2230
2231void CodeGenModule::addDefaultFunctionDefinitionAttributes(
2232 llvm::AttrBuilder &attrs) {
2233 getDefaultFunctionAttributes(/*function name*/ Name: "", /*optnone*/ HasOptnone: false,
2234 /*for call*/ AttrOnCallSite: false, FuncAttrs&: attrs);
2235 GetCPUAndFeaturesAttributes(GD: GlobalDecl(), AttrBuilder&: attrs);
2236}
2237
2238static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
2239 const LangOptions &LangOpts,
2240 const NoBuiltinAttr *NBA = nullptr) {
2241 auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
2242 SmallString<32> AttributeName;
2243 AttributeName += "no-builtin-";
2244 AttributeName += BuiltinName;
2245 FuncAttrs.addAttribute(A: AttributeName);
2246 };
2247
2248 // First, handle the language options passed through -fno-builtin.
2249 if (LangOpts.NoBuiltin) {
2250 // -fno-builtin disables them all.
2251 FuncAttrs.addAttribute(A: "no-builtins");
2252 return;
2253 }
2254
2255 // Then, add attributes for builtins specified through -fno-builtin-<name>.
2256 llvm::for_each(Range: LangOpts.NoBuiltinFuncs, F: AddNoBuiltinAttr);
2257
2258 // Now, let's check the __attribute__((no_builtin("...")) attribute added to
2259 // the source.
2260 if (!NBA)
2261 return;
2262
2263 // If there is a wildcard in the builtin names specified through the
2264 // attribute, disable them all.
2265 if (llvm::is_contained(Range: NBA->builtinNames(), Element: "*")) {
2266 FuncAttrs.addAttribute(A: "no-builtins");
2267 return;
2268 }
2269
2270 // And last, add the rest of the builtin names.
2271 llvm::for_each(Range: NBA->builtinNames(), F: AddNoBuiltinAttr);
2272}
2273
2274static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
2275 const llvm::DataLayout &DL, const ABIArgInfo &AI,
2276 bool CheckCoerce = true) {
2277 llvm::Type *Ty = Types.ConvertTypeForMem(T: QTy);
2278 if (AI.getKind() == ABIArgInfo::Indirect ||
2279 AI.getKind() == ABIArgInfo::IndirectAliased)
2280 return true;
2281 if (AI.getKind() == ABIArgInfo::Extend && !AI.isNoExt())
2282 return true;
2283 if (!DL.typeSizeEqualsStoreSize(Ty))
2284 // TODO: This will result in a modest amount of values not marked noundef
2285 // when they could be. We care about values that *invisibly* contain undef
2286 // bits from the perspective of LLVM IR.
2287 return false;
2288 if (CheckCoerce && AI.canHaveCoerceToType()) {
2289 llvm::Type *CoerceTy = AI.getCoerceToType();
2290 if (llvm::TypeSize::isKnownGT(LHS: DL.getTypeSizeInBits(Ty: CoerceTy),
2291 RHS: DL.getTypeSizeInBits(Ty)))
2292 // If we're coercing to a type with a greater size than the canonical one,
2293 // we're introducing new undef bits.
2294 // Coercing to a type of smaller or equal size is ok, as we know that
2295 // there's no internal padding (typeSizeEqualsStoreSize).
2296 return false;
2297 }
2298 if (QTy->isBitIntType())
2299 return true;
2300 if (QTy->isReferenceType())
2301 return true;
2302 if (QTy->isNullPtrType())
2303 return false;
2304 if (QTy->isMemberPointerType())
2305 // TODO: Some member pointers are `noundef`, but it depends on the ABI. For
2306 // now, never mark them.
2307 return false;
2308 if (QTy->isScalarType()) {
2309 if (const ComplexType *Complex = dyn_cast<ComplexType>(Val&: QTy))
2310 return DetermineNoUndef(QTy: Complex->getElementType(), Types, DL, AI, CheckCoerce: false);
2311 return true;
2312 }
2313 if (const VectorType *Vector = dyn_cast<VectorType>(Val&: QTy))
2314 return DetermineNoUndef(QTy: Vector->getElementType(), Types, DL, AI, CheckCoerce: false);
2315 if (const MatrixType *Matrix = dyn_cast<MatrixType>(Val&: QTy))
2316 return DetermineNoUndef(QTy: Matrix->getElementType(), Types, DL, AI, CheckCoerce: false);
2317 if (const ArrayType *Array = dyn_cast<ArrayType>(Val&: QTy))
2318 return DetermineNoUndef(QTy: Array->getElementType(), Types, DL, AI, CheckCoerce: false);
2319
2320 // TODO: Some structs may be `noundef`, in specific situations.
2321 return false;
2322}
2323
2324/// Check if the argument of a function has maybe_undef attribute.
2325static bool IsArgumentMaybeUndef(const Decl *TargetDecl,
2326 unsigned NumRequiredArgs, unsigned ArgNo) {
2327 const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
2328 if (!FD)
2329 return false;
2330
2331 // Assume variadic arguments do not have maybe_undef attribute.
2332 if (ArgNo >= NumRequiredArgs)
2333 return false;
2334
2335 // Check if argument has maybe_undef attribute.
2336 if (ArgNo < FD->getNumParams()) {
2337 const ParmVarDecl *Param = FD->getParamDecl(i: ArgNo);
2338 if (Param && Param->hasAttr<MaybeUndefAttr>())
2339 return true;
2340 }
2341
2342 return false;
2343}
2344
2345/// Test if it's legal to apply nofpclass for the given parameter type and it's
2346/// lowered IR type.
2347static bool canApplyNoFPClass(const ABIArgInfo &AI, QualType ParamType,
2348 bool IsReturn) {
2349 // Should only apply to FP types in the source, not ABI promoted.
2350 if (!ParamType->hasFloatingRepresentation())
2351 return false;
2352
2353 // The promoted-to IR type also needs to support nofpclass.
2354 llvm::Type *IRTy = AI.getCoerceToType();
2355 if (llvm::AttributeFuncs::isNoFPClassCompatibleType(Ty: IRTy))
2356 return true;
2357
2358 if (llvm::StructType *ST = dyn_cast<llvm::StructType>(Val: IRTy)) {
2359 return !IsReturn && AI.getCanBeFlattened() &&
2360 llvm::all_of(Range: ST->elements(),
2361 P: llvm::AttributeFuncs::isNoFPClassCompatibleType);
2362 }
2363
2364 return false;
2365}
2366
2367/// Return the nofpclass mask that can be applied to floating-point parameters.
2368static llvm::FPClassTest getNoFPClassTestMask(const LangOptions &LangOpts) {
2369 llvm::FPClassTest Mask = llvm::fcNone;
2370 if (LangOpts.NoHonorInfs)
2371 Mask |= llvm::fcInf;
2372 if (LangOpts.NoHonorNaNs)
2373 Mask |= llvm::fcNan;
2374 return Mask;
2375}
2376
2377void CodeGenModule::AdjustMemoryAttribute(StringRef Name,
2378 CGCalleeInfo CalleeInfo,
2379 llvm::AttributeList &Attrs) {
2380 if (Attrs.getMemoryEffects().getModRef() == llvm::ModRefInfo::NoModRef) {
2381 Attrs = Attrs.removeFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::Memory);
2382 llvm::Attribute MemoryAttr = llvm::Attribute::getWithMemoryEffects(
2383 Context&: getLLVMContext(), ME: llvm::MemoryEffects::writeOnly());
2384 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Attr: MemoryAttr);
2385 }
2386}
2387
2388/// Construct the IR attribute list of a function or call.
2389///
2390/// When adding an attribute, please consider where it should be handled:
2391///
2392/// - getDefaultFunctionAttributes is for attributes that are essentially
2393/// part of the global target configuration (but perhaps can be
2394/// overridden on a per-function basis). Adding attributes there
2395/// will cause them to also be set in frontends that build on Clang's
2396/// target-configuration logic, as well as for code defined in library
2397/// modules such as CUDA's libdevice.
2398///
2399/// - ConstructAttributeList builds on top of getDefaultFunctionAttributes
2400/// and adds declaration-specific, convention-specific, and
2401/// frontend-specific logic. The last is of particular importance:
2402/// attributes that restrict how the frontend generates code must be
2403/// added here rather than getDefaultFunctionAttributes.
2404///
2405void CodeGenModule::ConstructAttributeList(StringRef Name,
2406 const CGFunctionInfo &FI,
2407 CGCalleeInfo CalleeInfo,
2408 llvm::AttributeList &AttrList,
2409 unsigned &CallingConv,
2410 bool AttrOnCallSite, bool IsThunk) {
2411 llvm::AttrBuilder FuncAttrs(getLLVMContext());
2412 llvm::AttrBuilder RetAttrs(getLLVMContext());
2413
2414 // Collect function IR attributes from the CC lowering.
2415 // We'll collect the paramete and result attributes later.
2416 CallingConv = FI.getEffectiveCallingConvention();
2417 if (FI.isNoReturn())
2418 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2419 if (FI.isCmseNSCall())
2420 FuncAttrs.addAttribute(A: "cmse_nonsecure_call");
2421
2422 // Collect function IR attributes from the callee prototype if we have one.
2423 AddAttributesFromFunctionProtoType(Ctx&: getContext(), FuncAttrs,
2424 FPT: CalleeInfo.getCalleeFunctionProtoType());
2425 const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2426
2427 // Attach assumption attributes to the declaration. If this is a call
2428 // site, attach assumptions from the caller to the call as well.
2429 AddAttributesFromOMPAssumes(FuncAttrs, Callee: TargetDecl);
2430
2431 bool HasOptnone = false;
2432 // The NoBuiltinAttr attached to the target FunctionDecl.
2433 const NoBuiltinAttr *NBA = nullptr;
2434
2435 // Some ABIs may result in additional accesses to arguments that may
2436 // otherwise not be present.
2437 std::optional<llvm::Attribute::AttrKind> MemAttrForPtrArgs;
2438 bool AddedPotentialArgAccess = false;
2439 auto AddPotentialArgAccess = [&]() {
2440 AddedPotentialArgAccess = true;
2441 llvm::Attribute A = FuncAttrs.getAttribute(Kind: llvm::Attribute::Memory);
2442 if (A.isValid())
2443 FuncAttrs.addMemoryAttr(ME: A.getMemoryEffects() |
2444 llvm::MemoryEffects::argMemOnly());
2445 };
2446
2447 // Collect function IR attributes based on declaration-specific
2448 // information.
2449 // FIXME: handle sseregparm someday...
2450 if (TargetDecl) {
2451 if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2452 FuncAttrs.addAttribute(Val: llvm::Attribute::ReturnsTwice);
2453 if (TargetDecl->hasAttr<NoThrowAttr>())
2454 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2455 if (TargetDecl->hasAttr<NoReturnAttr>())
2456 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2457 if (TargetDecl->hasAttr<ColdAttr>())
2458 FuncAttrs.addAttribute(Val: llvm::Attribute::Cold);
2459 if (TargetDecl->hasAttr<HotAttr>())
2460 FuncAttrs.addAttribute(Val: llvm::Attribute::Hot);
2461 if (TargetDecl->hasAttr<NoDuplicateAttr>())
2462 FuncAttrs.addAttribute(Val: llvm::Attribute::NoDuplicate);
2463 if (TargetDecl->hasAttr<ConvergentAttr>())
2464 FuncAttrs.addAttribute(Val: llvm::Attribute::Convergent);
2465
2466 if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2467 AddAttributesFromFunctionProtoType(
2468 Ctx&: getContext(), FuncAttrs, FPT: Fn->getType()->getAs<FunctionProtoType>());
2469 if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
2470 // A sane operator new returns a non-aliasing pointer.
2471 auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2472 if (getCodeGenOpts().AssumeSaneOperatorNew &&
2473 (Kind == OO_New || Kind == OO_Array_New))
2474 RetAttrs.addAttribute(Val: llvm::Attribute::NoAlias);
2475 }
2476 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: Fn);
2477 const bool IsVirtualCall = MD && MD->isVirtual();
2478 // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2479 // virtual function. These attributes are not inherited by overloads.
2480 if (!(AttrOnCallSite && IsVirtualCall)) {
2481 if (Fn->isNoReturn())
2482 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2483 NBA = Fn->getAttr<NoBuiltinAttr>();
2484 }
2485 }
2486
2487 if (isa<FunctionDecl>(Val: TargetDecl) || isa<VarDecl>(Val: TargetDecl)) {
2488 // Only place nomerge attribute on call sites, never functions. This
2489 // allows it to work on indirect virtual function calls.
2490 if (AttrOnCallSite && TargetDecl->hasAttr<NoMergeAttr>())
2491 FuncAttrs.addAttribute(Val: llvm::Attribute::NoMerge);
2492 }
2493
2494 // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2495 if (TargetDecl->hasAttr<ConstAttr>()) {
2496 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::none());
2497 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2498 // gcc specifies that 'const' functions have greater restrictions than
2499 // 'pure' functions, so they also cannot have infinite loops.
2500 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
2501 MemAttrForPtrArgs = llvm::Attribute::ReadNone;
2502 } else if (TargetDecl->hasAttr<PureAttr>()) {
2503 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::readOnly());
2504 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2505 // gcc specifies that 'pure' functions cannot have infinite loops.
2506 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
2507 MemAttrForPtrArgs = llvm::Attribute::ReadOnly;
2508 } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2509 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::inaccessibleOrArgMemOnly());
2510 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2511 }
2512 if (const auto *RA = TargetDecl->getAttr<RestrictAttr>();
2513 RA && RA->getDeallocator() == nullptr)
2514 RetAttrs.addAttribute(Val: llvm::Attribute::NoAlias);
2515 if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2516 !CodeGenOpts.NullPointerIsValid)
2517 RetAttrs.addAttribute(Val: llvm::Attribute::NonNull);
2518 if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2519 FuncAttrs.addAttribute(A: "no_caller_saved_registers");
2520 if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2521 FuncAttrs.addAttribute(Val: llvm::Attribute::NoCfCheck);
2522 if (TargetDecl->hasAttr<LeafAttr>())
2523 FuncAttrs.addAttribute(Val: llvm::Attribute::NoCallback);
2524 if (TargetDecl->hasAttr<BPFFastCallAttr>())
2525 FuncAttrs.addAttribute(A: "bpf_fastcall");
2526
2527 HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2528 if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2529 std::optional<unsigned> NumElemsParam;
2530 if (AllocSize->getNumElemsParam().isValid())
2531 NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2532 FuncAttrs.addAllocSizeAttr(ElemSizeArg: AllocSize->getElemSizeParam().getLLVMIndex(),
2533 NumElemsArg: NumElemsParam);
2534 }
2535
2536 if (DeviceKernelAttr::isOpenCLSpelling(
2537 A: TargetDecl->getAttr<DeviceKernelAttr>()) &&
2538 CallingConv != CallingConv::CC_C &&
2539 CallingConv != CallingConv::CC_SpirFunction) {
2540 // Check CallingConv to avoid adding uniform-work-group-size attribute to
2541 // OpenCL Kernel Stub
2542 if (getLangOpts().OpenCLVersion <= 120) {
2543 // OpenCL v1.2 Work groups are always uniform
2544 FuncAttrs.addAttribute(A: "uniform-work-group-size", V: "true");
2545 } else {
2546 // OpenCL v2.0 Work groups may be whether uniform or not.
2547 // '-cl-uniform-work-group-size' compile option gets a hint
2548 // to the compiler that the global work-size be a multiple of
2549 // the work-group size specified to clEnqueueNDRangeKernel
2550 // (i.e. work groups are uniform).
2551 FuncAttrs.addAttribute(
2552 A: "uniform-work-group-size",
2553 V: llvm::toStringRef(B: getLangOpts().OffloadUniformBlock));
2554 }
2555 }
2556
2557 if (TargetDecl->hasAttr<CUDAGlobalAttr>() &&
2558 getLangOpts().OffloadUniformBlock)
2559 FuncAttrs.addAttribute(A: "uniform-work-group-size", V: "true");
2560
2561 if (TargetDecl->hasAttr<ArmLocallyStreamingAttr>())
2562 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_body");
2563
2564 if (auto *ModularFormat = TargetDecl->getAttr<ModularFormatAttr>()) {
2565 FormatAttr *Format = TargetDecl->getAttr<FormatAttr>();
2566 StringRef Type = Format->getType()->getName();
2567 std::string FormatIdx = std::to_string(val: Format->getFormatIdx());
2568 std::string FirstArg = std::to_string(val: Format->getFirstArg());
2569 SmallVector<StringRef> Args = {
2570 Type, FormatIdx, FirstArg,
2571 ModularFormat->getModularImplFn()->getName(),
2572 ModularFormat->getImplName()};
2573 llvm::append_range(C&: Args, R: ModularFormat->aspects());
2574 FuncAttrs.addAttribute(A: "modular-format", V: llvm::join(R&: Args, Separator: ","));
2575 }
2576 }
2577
2578 // Attach "no-builtins" attributes to:
2579 // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2580 // * definitions: "no-builtins" or "no-builtin-<name>" only.
2581 // The attributes can come from:
2582 // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2583 // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2584 addNoBuiltinAttributes(FuncAttrs, LangOpts: getLangOpts(), NBA);
2585
2586 // Collect function IR attributes based on global settiings.
2587 getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2588
2589 // Override some default IR attributes based on declaration-specific
2590 // information.
2591 if (TargetDecl) {
2592 if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2593 FuncAttrs.removeAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2594 if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2595 FuncAttrs.addAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2596 if (TargetDecl->hasAttr<NoSplitStackAttr>())
2597 FuncAttrs.removeAttribute(A: "split-stack");
2598 if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
2599 // A function "__attribute__((...))" overrides the command-line flag.
2600 auto Kind =
2601 TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2602 FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2603 FuncAttrs.addAttribute(
2604 A: "zero-call-used-regs",
2605 V: ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Val: Kind));
2606 }
2607
2608 // Add NonLazyBind attribute to function declarations when -fno-plt
2609 // is used.
2610 // FIXME: what if we just haven't processed the function definition
2611 // yet, or if it's an external definition like C99 inline?
2612 if (CodeGenOpts.NoPLT) {
2613 if (auto *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2614 if (!Fn->isDefined() && !AttrOnCallSite) {
2615 FuncAttrs.addAttribute(Val: llvm::Attribute::NonLazyBind);
2616 }
2617 }
2618 }
2619 // Remove 'convergent' if requested.
2620 if (TargetDecl->hasAttr<NoConvergentAttr>())
2621 FuncAttrs.removeAttribute(Val: llvm::Attribute::Convergent);
2622 }
2623
2624 // Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2625 // functions with -funique-internal-linkage-names.
2626 if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames) {
2627 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
2628 if (!FD->isExternallyVisible())
2629 FuncAttrs.addAttribute(A: "sample-profile-suffix-elision-policy",
2630 V: "selected");
2631 }
2632 }
2633
2634 // Collect non-call-site function IR attributes from declaration-specific
2635 // information.
2636 if (!AttrOnCallSite) {
2637 if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>())
2638 FuncAttrs.addAttribute(A: "cmse_nonsecure_entry");
2639
2640 // Whether tail calls are enabled.
2641 auto shouldDisableTailCalls = [&] {
2642 // Should this be honored in getDefaultFunctionAttributes?
2643 if (CodeGenOpts.DisableTailCalls)
2644 return true;
2645
2646 if (!TargetDecl)
2647 return false;
2648
2649 if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2650 TargetDecl->hasAttr<AnyX86InterruptAttr>())
2651 return true;
2652
2653 if (CodeGenOpts.NoEscapingBlockTailCalls) {
2654 if (const auto *BD = dyn_cast<BlockDecl>(Val: TargetDecl))
2655 if (!BD->doesNotEscape())
2656 return true;
2657 }
2658
2659 return false;
2660 };
2661 if (shouldDisableTailCalls())
2662 FuncAttrs.addAttribute(A: "disable-tail-calls", V: "true");
2663
2664 // These functions require the returns_twice attribute for correct codegen,
2665 // but the attribute may not be added if -fno-builtin is specified. We
2666 // explicitly add that attribute here.
2667 static const llvm::StringSet<> ReturnsTwiceFn{
2668 "_setjmpex", "setjmp", "_setjmp", "vfork",
2669 "sigsetjmp", "__sigsetjmp", "savectx", "getcontext"};
2670 if (ReturnsTwiceFn.contains(key: Name))
2671 FuncAttrs.addAttribute(Val: llvm::Attribute::ReturnsTwice);
2672
2673 // CPU/feature overrides. addDefaultFunctionDefinitionAttributes
2674 // handles these separately to set them based on the global defaults.
2675 GetCPUAndFeaturesAttributes(GD: CalleeInfo.getCalleeDecl(), AttrBuilder&: FuncAttrs);
2676
2677 // Windows hotpatching support
2678 if (!MSHotPatchFunctions.empty()) {
2679 bool IsHotPatched = llvm::binary_search(Range&: MSHotPatchFunctions, Value&: Name);
2680 if (IsHotPatched)
2681 FuncAttrs.addAttribute(A: "marked_for_windows_hot_patching");
2682 }
2683 }
2684
2685 // Mark functions that are replaceable by the loader.
2686 if (CodeGenOpts.isLoaderReplaceableFunctionName(FuncName: Name))
2687 FuncAttrs.addAttribute(A: "loader-replaceable");
2688
2689 // Collect attributes from arguments and return values.
2690 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2691
2692 QualType RetTy = FI.getReturnType();
2693 const ABIArgInfo &RetAI = FI.getReturnInfo();
2694 const llvm::DataLayout &DL = getDataLayout();
2695
2696 // Determine if the return type could be partially undef
2697 if (CodeGenOpts.EnableNoundefAttrs &&
2698 HasStrictReturn(Module: *this, RetTy, TargetDecl)) {
2699 if (!RetTy->isVoidType() && RetAI.getKind() != ABIArgInfo::Indirect &&
2700 DetermineNoUndef(QTy: RetTy, Types&: getTypes(), DL, AI: RetAI))
2701 RetAttrs.addAttribute(Val: llvm::Attribute::NoUndef);
2702 }
2703
2704 switch (RetAI.getKind()) {
2705 case ABIArgInfo::Extend:
2706 if (RetAI.isSignExt())
2707 RetAttrs.addAttribute(Val: llvm::Attribute::SExt);
2708 else if (RetAI.isZeroExt())
2709 RetAttrs.addAttribute(Val: llvm::Attribute::ZExt);
2710 else
2711 RetAttrs.addAttribute(Val: llvm::Attribute::NoExt);
2712 [[fallthrough]];
2713 case ABIArgInfo::TargetSpecific:
2714 case ABIArgInfo::Direct:
2715 if (RetAI.getInReg())
2716 RetAttrs.addAttribute(Val: llvm::Attribute::InReg);
2717
2718 if (canApplyNoFPClass(AI: RetAI, ParamType: RetTy, IsReturn: true))
2719 RetAttrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
2720
2721 break;
2722 case ABIArgInfo::Ignore:
2723 break;
2724
2725 case ABIArgInfo::InAlloca:
2726 case ABIArgInfo::Indirect: {
2727 // inalloca and sret disable readnone and readonly
2728 AddPotentialArgAccess();
2729 break;
2730 }
2731
2732 case ABIArgInfo::CoerceAndExpand:
2733 break;
2734
2735 case ABIArgInfo::Expand:
2736 case ABIArgInfo::IndirectAliased:
2737 llvm_unreachable("Invalid ABI kind for return argument");
2738 }
2739
2740 if (!IsThunk) {
2741 // FIXME: fix this properly, https://reviews.llvm.org/D100388
2742 if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2743 QualType PTy = RefTy->getPointeeType();
2744 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2745 RetAttrs.addDereferenceableAttr(
2746 Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
2747 if (getTypes().getTargetAddressSpace(T: PTy) == 0 &&
2748 !CodeGenOpts.NullPointerIsValid)
2749 RetAttrs.addAttribute(Val: llvm::Attribute::NonNull);
2750 if (PTy->isObjectType()) {
2751 llvm::Align Alignment =
2752 getNaturalPointeeTypeAlignment(T: RetTy).getAsAlign();
2753 RetAttrs.addAlignmentAttr(Align: Alignment);
2754 }
2755 }
2756 }
2757
2758 bool hasUsedSRet = false;
2759 SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
2760
2761 // Attach attributes to sret.
2762 if (IRFunctionArgs.hasSRetArg()) {
2763 llvm::AttrBuilder SRETAttrs(getLLVMContext());
2764 SRETAttrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: RetTy));
2765 SRETAttrs.addAttribute(Val: llvm::Attribute::Writable);
2766 SRETAttrs.addAttribute(Val: llvm::Attribute::DeadOnUnwind);
2767 hasUsedSRet = true;
2768 if (RetAI.getInReg())
2769 SRETAttrs.addAttribute(Val: llvm::Attribute::InReg);
2770 SRETAttrs.addAlignmentAttr(Align: RetAI.getIndirectAlign().getQuantity());
2771 ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2772 llvm::AttributeSet::get(C&: getLLVMContext(), B: SRETAttrs);
2773 }
2774
2775 // Attach attributes to inalloca argument.
2776 if (IRFunctionArgs.hasInallocaArg()) {
2777 llvm::AttrBuilder Attrs(getLLVMContext());
2778 Attrs.addInAllocaAttr(Ty: FI.getArgStruct());
2779 ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2780 llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs);
2781 }
2782
2783 // Apply `nonnull`, `dereferenceable(N)` and `align N` to the `this` argument,
2784 // unless this is a thunk function.
2785 // FIXME: fix this properly, https://reviews.llvm.org/D100388
2786 if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() &&
2787 !FI.arg_begin()->type->isVoidPointerType() && !IsThunk) {
2788 auto IRArgs = IRFunctionArgs.getIRArgs(ArgNo: 0);
2789
2790 assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
2791
2792 llvm::AttrBuilder Attrs(getLLVMContext());
2793
2794 QualType ThisTy = FI.arg_begin()->type.getTypePtr()->getPointeeType();
2795
2796 if (!CodeGenOpts.NullPointerIsValid &&
2797 getTypes().getTargetAddressSpace(T: FI.arg_begin()->type) == 0) {
2798 Attrs.addAttribute(Val: llvm::Attribute::NonNull);
2799 Attrs.addDereferenceableAttr(Bytes: getMinimumObjectSize(Ty: ThisTy).getQuantity());
2800 } else {
2801 // FIXME dereferenceable should be correct here, regardless of
2802 // NullPointerIsValid. However, dereferenceable currently does not always
2803 // respect NullPointerIsValid and may imply nonnull and break the program.
2804 // See https://reviews.llvm.org/D66618 for discussions.
2805 Attrs.addDereferenceableOrNullAttr(
2806 Bytes: getMinimumObjectSize(
2807 Ty: FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
2808 .getQuantity());
2809 }
2810
2811 llvm::Align Alignment =
2812 getNaturalTypeAlignment(T: ThisTy, /*BaseInfo=*/nullptr,
2813 /*TBAAInfo=*/nullptr, /*forPointeeType=*/true)
2814 .getAsAlign();
2815 Attrs.addAlignmentAttr(Align: Alignment);
2816
2817 ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs);
2818 }
2819
2820 unsigned ArgNo = 0;
2821 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(), E = FI.arg_end();
2822 I != E; ++I, ++ArgNo) {
2823 QualType ParamType = I->type;
2824 const ABIArgInfo &AI = I->info;
2825 llvm::AttrBuilder Attrs(getLLVMContext());
2826
2827 // Add attribute for padding argument, if necessary.
2828 if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2829 if (AI.getPaddingInReg()) {
2830 ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2831 llvm::AttributeSet::get(C&: getLLVMContext(),
2832 B: llvm::AttrBuilder(getLLVMContext())
2833 .addAttribute(Val: llvm::Attribute::InReg));
2834 }
2835 }
2836
2837 // Decide whether the argument we're handling could be partially undef
2838 if (CodeGenOpts.EnableNoundefAttrs &&
2839 DetermineNoUndef(QTy: ParamType, Types&: getTypes(), DL, AI)) {
2840 Attrs.addAttribute(Val: llvm::Attribute::NoUndef);
2841 }
2842
2843 // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2844 // have the corresponding parameter variable. It doesn't make
2845 // sense to do it here because parameters are so messed up.
2846 switch (AI.getKind()) {
2847 case ABIArgInfo::Extend:
2848 if (AI.isSignExt())
2849 Attrs.addAttribute(Val: llvm::Attribute::SExt);
2850 else if (AI.isZeroExt())
2851 Attrs.addAttribute(Val: llvm::Attribute::ZExt);
2852 else
2853 Attrs.addAttribute(Val: llvm::Attribute::NoExt);
2854 [[fallthrough]];
2855 case ABIArgInfo::TargetSpecific:
2856 case ABIArgInfo::Direct:
2857 if (ArgNo == 0 && FI.isChainCall())
2858 Attrs.addAttribute(Val: llvm::Attribute::Nest);
2859 else if (AI.getInReg())
2860 Attrs.addAttribute(Val: llvm::Attribute::InReg);
2861 Attrs.addStackAlignmentAttr(Align: llvm::MaybeAlign(AI.getDirectAlign()));
2862
2863 if (canApplyNoFPClass(AI, ParamType, IsReturn: false))
2864 Attrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
2865 break;
2866 case ABIArgInfo::Indirect: {
2867 if (AI.getInReg())
2868 Attrs.addAttribute(Val: llvm::Attribute::InReg);
2869
2870 // HLSL out and inout parameters must not be marked with ByVal or
2871 // DeadOnReturn attributes because stores to these parameters by the
2872 // callee are visible to the caller.
2873 if (auto ParamABI = FI.getExtParameterInfo(argIndex: ArgNo).getABI();
2874 ParamABI != ParameterABI::HLSLOut &&
2875 ParamABI != ParameterABI::HLSLInOut) {
2876
2877 // Depending on the ABI, this may be either a byval or a dead_on_return
2878 // argument.
2879 if (AI.getIndirectByVal()) {
2880 Attrs.addByValAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
2881 } else {
2882 // Add dead_on_return when the object's lifetime ends in the callee.
2883 // This includes trivially-destructible objects, as well as objects
2884 // whose destruction / clean-up is carried out within the callee
2885 // (e.g., Obj-C ARC-managed structs, MSVC callee-destroyed objects).
2886 if (!ParamType.isDestructedType() || !ParamType->isRecordType() ||
2887 ParamType->castAsRecordDecl()->isParamDestroyedInCallee())
2888 Attrs.addDeadOnReturnAttr(Info: llvm::DeadOnReturnInfo());
2889 }
2890 }
2891
2892 auto *Decl = ParamType->getAsRecordDecl();
2893 if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
2894 Decl->getArgPassingRestrictions() ==
2895 RecordArgPassingKind::CanPassInRegs)
2896 // When calling the function, the pointer passed in will be the only
2897 // reference to the underlying object. Mark it accordingly.
2898 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
2899
2900 // TODO: We could add the byref attribute if not byval, but it would
2901 // require updating many testcases.
2902
2903 CharUnits Align = AI.getIndirectAlign();
2904
2905 // In a byval argument, it is important that the required
2906 // alignment of the type is honored, as LLVM might be creating a
2907 // *new* stack object, and needs to know what alignment to give
2908 // it. (Sometimes it can deduce a sensible alignment on its own,
2909 // but not if clang decides it must emit a packed struct, or the
2910 // user specifies increased alignment requirements.)
2911 //
2912 // This is different from indirect *not* byval, where the object
2913 // exists already, and the align attribute is purely
2914 // informative.
2915 assert(!Align.isZero());
2916
2917 // For now, only add this when we have a byval argument.
2918 // TODO: be less lazy about updating test cases.
2919 if (AI.getIndirectByVal())
2920 Attrs.addAlignmentAttr(Align: Align.getQuantity());
2921
2922 // byval disables readnone and readonly.
2923 AddPotentialArgAccess();
2924 break;
2925 }
2926 case ABIArgInfo::IndirectAliased: {
2927 CharUnits Align = AI.getIndirectAlign();
2928 Attrs.addByRefAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
2929 Attrs.addAlignmentAttr(Align: Align.getQuantity());
2930 break;
2931 }
2932 case ABIArgInfo::Ignore:
2933 case ABIArgInfo::Expand:
2934 case ABIArgInfo::CoerceAndExpand:
2935 break;
2936
2937 case ABIArgInfo::InAlloca:
2938 // inalloca disables readnone and readonly.
2939 AddPotentialArgAccess();
2940 continue;
2941 }
2942
2943 if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
2944 QualType PTy = RefTy->getPointeeType();
2945 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2946 Attrs.addDereferenceableAttr(Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
2947 if (getTypes().getTargetAddressSpace(T: PTy) == 0 &&
2948 !CodeGenOpts.NullPointerIsValid)
2949 Attrs.addAttribute(Val: llvm::Attribute::NonNull);
2950 if (PTy->isObjectType()) {
2951 llvm::Align Alignment =
2952 getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
2953 Attrs.addAlignmentAttr(Align: Alignment);
2954 }
2955 }
2956
2957 // From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:
2958 // > For arguments to a __kernel function declared to be a pointer to a
2959 // > data type, the OpenCL compiler can assume that the pointee is always
2960 // > appropriately aligned as required by the data type.
2961 if (TargetDecl &&
2962 DeviceKernelAttr::isOpenCLSpelling(
2963 A: TargetDecl->getAttr<DeviceKernelAttr>()) &&
2964 ParamType->isPointerType()) {
2965 QualType PTy = ParamType->getPointeeType();
2966 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2967 llvm::Align Alignment =
2968 getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
2969 Attrs.addAlignmentAttr(Align: Alignment);
2970 }
2971 }
2972
2973 switch (FI.getExtParameterInfo(argIndex: ArgNo).getABI()) {
2974 case ParameterABI::HLSLOut:
2975 case ParameterABI::HLSLInOut:
2976 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
2977 break;
2978 case ParameterABI::Ordinary:
2979 break;
2980
2981 case ParameterABI::SwiftIndirectResult: {
2982 // Add 'sret' if we haven't already used it for something, but
2983 // only if the result is void.
2984 if (!hasUsedSRet && RetTy->isVoidType()) {
2985 Attrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
2986 hasUsedSRet = true;
2987 }
2988
2989 // Add 'noalias' in either case.
2990 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
2991
2992 // Add 'dereferenceable' and 'alignment'.
2993 auto PTy = ParamType->getPointeeType();
2994 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2995 auto info = getContext().getTypeInfoInChars(T: PTy);
2996 Attrs.addDereferenceableAttr(Bytes: info.Width.getQuantity());
2997 Attrs.addAlignmentAttr(Align: info.Align.getAsAlign());
2998 }
2999 break;
3000 }
3001
3002 case ParameterABI::SwiftErrorResult:
3003 Attrs.addAttribute(Val: llvm::Attribute::SwiftError);
3004 break;
3005
3006 case ParameterABI::SwiftContext:
3007 Attrs.addAttribute(Val: llvm::Attribute::SwiftSelf);
3008 break;
3009
3010 case ParameterABI::SwiftAsyncContext:
3011 Attrs.addAttribute(Val: llvm::Attribute::SwiftAsync);
3012 break;
3013 }
3014
3015 if (FI.getExtParameterInfo(argIndex: ArgNo).isNoEscape())
3016 Attrs.addCapturesAttr(CI: llvm::CaptureInfo::none());
3017
3018 if (Attrs.hasAttributes()) {
3019 unsigned FirstIRArg, NumIRArgs;
3020 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3021 for (unsigned i = 0; i < NumIRArgs; i++)
3022 ArgAttrs[FirstIRArg + i] = ArgAttrs[FirstIRArg + i].addAttributes(
3023 C&: getLLVMContext(), AS: llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs));
3024 }
3025 }
3026 assert(ArgNo == FI.arg_size());
3027
3028 ArgNo = 0;
3029 if (AddedPotentialArgAccess && MemAttrForPtrArgs) {
3030 llvm::FunctionType *FunctionType = getTypes().GetFunctionType(FI);
3031 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
3032 E = FI.arg_end();
3033 I != E; ++I, ++ArgNo) {
3034 if (I->info.isDirect() || I->info.isExpand() ||
3035 I->info.isCoerceAndExpand()) {
3036 unsigned FirstIRArg, NumIRArgs;
3037 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3038 for (unsigned i = FirstIRArg; i < FirstIRArg + NumIRArgs; ++i) {
3039 if (FunctionType->getParamType(i)->isPointerTy()) {
3040 ArgAttrs[i] =
3041 ArgAttrs[i].addAttribute(C&: getLLVMContext(), Kind: *MemAttrForPtrArgs);
3042 }
3043 }
3044 }
3045 }
3046 }
3047
3048 AttrList = llvm::AttributeList::get(
3049 C&: getLLVMContext(), FnAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: FuncAttrs),
3050 RetAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: RetAttrs), ArgAttrs);
3051}
3052
3053/// An argument came in as a promoted argument; demote it back to its
3054/// declared type.
3055static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
3056 const VarDecl *var,
3057 llvm::Value *value) {
3058 llvm::Type *varType = CGF.ConvertType(T: var->getType());
3059
3060 // This can happen with promotions that actually don't change the
3061 // underlying type, like the enum promotions.
3062 if (value->getType() == varType)
3063 return value;
3064
3065 assert((varType->isIntegerTy() || varType->isFloatingPointTy()) &&
3066 "unexpected promotion type");
3067
3068 if (isa<llvm::IntegerType>(Val: varType))
3069 return CGF.Builder.CreateTrunc(V: value, DestTy: varType, Name: "arg.unpromote");
3070
3071 return CGF.Builder.CreateFPCast(V: value, DestTy: varType, Name: "arg.unpromote");
3072}
3073
3074/// Returns the attribute (either parameter attribute, or function
3075/// attribute), which declares argument ArgNo to be non-null.
3076static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
3077 QualType ArgType, unsigned ArgNo) {
3078 // FIXME: __attribute__((nonnull)) can also be applied to:
3079 // - references to pointers, where the pointee is known to be
3080 // nonnull (apparently a Clang extension)
3081 // - transparent unions containing pointers
3082 // In the former case, LLVM IR cannot represent the constraint. In
3083 // the latter case, we have no guarantee that the transparent union
3084 // is in fact passed as a pointer.
3085 if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType())
3086 return nullptr;
3087 // First, check attribute on parameter itself.
3088 if (PVD) {
3089 if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
3090 return ParmNNAttr;
3091 }
3092 // Check function attributes.
3093 if (!FD)
3094 return nullptr;
3095 for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
3096 if (NNAttr->isNonNull(IdxAST: ArgNo))
3097 return NNAttr;
3098 }
3099 return nullptr;
3100}
3101
3102namespace {
3103struct CopyBackSwiftError final : EHScopeStack::Cleanup {
3104 Address Temp;
3105 Address Arg;
3106 CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
3107 void Emit(CodeGenFunction &CGF, Flags flags) override {
3108 llvm::Value *errorValue = CGF.Builder.CreateLoad(Addr: Temp);
3109 CGF.Builder.CreateStore(Val: errorValue, Addr: Arg);
3110 }
3111};
3112} // namespace
3113
3114void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
3115 llvm::Function *Fn,
3116 const FunctionArgList &Args) {
3117 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())
3118 // Naked functions don't have prologues.
3119 return;
3120
3121 // If this is an implicit-return-zero function, go ahead and
3122 // initialize the return value. TODO: it might be nice to have
3123 // a more general mechanism for this that didn't require synthesized
3124 // return statements.
3125 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl)) {
3126 if (FD->hasImplicitReturnZero()) {
3127 QualType RetTy = FD->getReturnType().getUnqualifiedType();
3128 llvm::Type *LLVMTy = CGM.getTypes().ConvertType(T: RetTy);
3129 llvm::Constant *Zero = llvm::Constant::getNullValue(Ty: LLVMTy);
3130 Builder.CreateStore(Val: Zero, Addr: ReturnValue);
3131 }
3132 }
3133
3134 // FIXME: We no longer need the types from FunctionArgList; lift up and
3135 // simplify.
3136
3137 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
3138 assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
3139
3140 // If we're using inalloca, all the memory arguments are GEPs off of the last
3141 // parameter, which is a pointer to the complete memory area.
3142 Address ArgStruct = Address::invalid();
3143 if (IRFunctionArgs.hasInallocaArg())
3144 ArgStruct = Address(Fn->getArg(i: IRFunctionArgs.getInallocaArgNo()),
3145 FI.getArgStruct(), FI.getArgStructAlignment());
3146
3147 // Name the struct return parameter.
3148 if (IRFunctionArgs.hasSRetArg()) {
3149 auto AI = Fn->getArg(i: IRFunctionArgs.getSRetArgNo());
3150 AI->setName("agg.result");
3151 AI->addAttr(Kind: llvm::Attribute::NoAlias);
3152 }
3153
3154 // Track if we received the parameter as a pointer (indirect, byval, or
3155 // inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it
3156 // into a local alloca for us.
3157 SmallVector<ParamValue, 16> ArgVals;
3158 ArgVals.reserve(N: Args.size());
3159
3160 // Create a pointer value for every parameter declaration. This usually
3161 // entails copying one or more LLVM IR arguments into an alloca. Don't push
3162 // any cleanups or do anything that might unwind. We do that separately, so
3163 // we can push the cleanups in the correct order for the ABI.
3164 assert(FI.arg_size() == Args.size() &&
3165 "Mismatch between function signature & arguments.");
3166 unsigned ArgNo = 0;
3167 CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
3168 for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end(); i != e;
3169 ++i, ++info_it, ++ArgNo) {
3170 const VarDecl *Arg = *i;
3171 const ABIArgInfo &ArgI = info_it->info;
3172
3173 bool isPromoted =
3174 isa<ParmVarDecl>(Val: Arg) && cast<ParmVarDecl>(Val: Arg)->isKNRPromoted();
3175 // We are converting from ABIArgInfo type to VarDecl type directly, unless
3176 // the parameter is promoted. In this case we convert to
3177 // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
3178 QualType Ty = isPromoted ? info_it->type : Arg->getType();
3179 assert(hasScalarEvaluationKind(Ty) ==
3180 hasScalarEvaluationKind(Arg->getType()));
3181
3182 unsigned FirstIRArg, NumIRArgs;
3183 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3184
3185 switch (ArgI.getKind()) {
3186 case ABIArgInfo::InAlloca: {
3187 assert(NumIRArgs == 0);
3188 auto FieldIndex = ArgI.getInAllocaFieldIndex();
3189 Address V =
3190 Builder.CreateStructGEP(Addr: ArgStruct, Index: FieldIndex, Name: Arg->getName());
3191 if (ArgI.getInAllocaIndirect())
3192 V = Address(Builder.CreateLoad(Addr: V), ConvertTypeForMem(T: Ty),
3193 getContext().getTypeAlignInChars(T: Ty));
3194 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: V));
3195 break;
3196 }
3197
3198 case ABIArgInfo::Indirect:
3199 case ABIArgInfo::IndirectAliased: {
3200 assert(NumIRArgs == 1);
3201 Address ParamAddr = makeNaturalAddressForPointer(
3202 Ptr: Fn->getArg(i: FirstIRArg), T: Ty, Alignment: ArgI.getIndirectAlign(), ForPointeeType: false, BaseInfo: nullptr,
3203 TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
3204
3205 if (!hasScalarEvaluationKind(T: Ty)) {
3206 // Aggregates and complex variables are accessed by reference. All we
3207 // need to do is realign the value, if requested. Also, if the address
3208 // may be aliased, copy it to ensure that the parameter variable is
3209 // mutable and has a unique adress, as C requires.
3210 if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) {
3211 RawAddress AlignedTemp = CreateMemTemp(T: Ty, Name: "coerce");
3212
3213 // Copy from the incoming argument pointer to the temporary with the
3214 // appropriate alignment.
3215 //
3216 // FIXME: We should have a common utility for generating an aggregate
3217 // copy.
3218 CharUnits Size = getContext().getTypeSizeInChars(T: Ty);
3219 Builder.CreateMemCpy(
3220 Dst: AlignedTemp.getPointer(), DstAlign: AlignedTemp.getAlignment().getAsAlign(),
3221 Src: ParamAddr.emitRawPointer(CGF&: *this),
3222 SrcAlign: ParamAddr.getAlignment().getAsAlign(),
3223 Size: llvm::ConstantInt::get(Ty: IntPtrTy, V: Size.getQuantity()));
3224 ParamAddr = AlignedTemp;
3225 }
3226 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: ParamAddr));
3227 } else {
3228 // Load scalar value from indirect argument.
3229 llvm::Value *V =
3230 EmitLoadOfScalar(Addr: ParamAddr, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3231
3232 if (isPromoted)
3233 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3234 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3235 }
3236 break;
3237 }
3238
3239 case ABIArgInfo::Extend:
3240 case ABIArgInfo::Direct: {
3241 auto AI = Fn->getArg(i: FirstIRArg);
3242 llvm::Type *LTy = ConvertType(T: Arg->getType());
3243
3244 // Prepare parameter attributes. So far, only attributes for pointer
3245 // parameters are prepared. See
3246 // http://llvm.org/docs/LangRef.html#paramattrs.
3247 if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy() &&
3248 ArgI.getCoerceToType()->isPointerTy()) {
3249 assert(NumIRArgs == 1);
3250
3251 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: Arg)) {
3252 // Set `nonnull` attribute if any.
3253 if (getNonNullAttr(FD: CurCodeDecl, PVD, ArgType: PVD->getType(),
3254 ArgNo: PVD->getFunctionScopeIndex()) &&
3255 !CGM.getCodeGenOpts().NullPointerIsValid)
3256 AI->addAttr(Kind: llvm::Attribute::NonNull);
3257
3258 QualType OTy = PVD->getOriginalType();
3259 if (const auto *ArrTy = getContext().getAsConstantArrayType(T: OTy)) {
3260 // A C99 array parameter declaration with the static keyword also
3261 // indicates dereferenceability, and if the size is constant we can
3262 // use the dereferenceable attribute (which requires the size in
3263 // bytes).
3264 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3265 QualType ETy = ArrTy->getElementType();
3266 llvm::Align Alignment =
3267 CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3268 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3269 .addAlignmentAttr(Align: Alignment));
3270 uint64_t ArrSize = ArrTy->getZExtSize();
3271 if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
3272 ArrSize) {
3273 llvm::AttrBuilder Attrs(getLLVMContext());
3274 Attrs.addDereferenceableAttr(
3275 Bytes: getContext().getTypeSizeInChars(T: ETy).getQuantity() *
3276 ArrSize);
3277 AI->addAttrs(B&: Attrs);
3278 } else if (getContext().getTargetInfo().getNullPointerValue(
3279 AddrSpace: ETy.getAddressSpace()) == 0 &&
3280 !CGM.getCodeGenOpts().NullPointerIsValid) {
3281 AI->addAttr(Kind: llvm::Attribute::NonNull);
3282 }
3283 }
3284 } else if (const auto *ArrTy =
3285 getContext().getAsVariableArrayType(T: OTy)) {
3286 // For C99 VLAs with the static keyword, we don't know the size so
3287 // we can't use the dereferenceable attribute, but in addrspace(0)
3288 // we know that it must be nonnull.
3289 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3290 QualType ETy = ArrTy->getElementType();
3291 llvm::Align Alignment =
3292 CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3293 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3294 .addAlignmentAttr(Align: Alignment));
3295 if (!getTypes().getTargetAddressSpace(T: ETy) &&
3296 !CGM.getCodeGenOpts().NullPointerIsValid)
3297 AI->addAttr(Kind: llvm::Attribute::NonNull);
3298 }
3299 }
3300
3301 // Set `align` attribute if any.
3302 const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
3303 if (!AVAttr)
3304 if (const auto *TOTy = OTy->getAs<TypedefType>())
3305 AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
3306 if (AVAttr && !SanOpts.has(K: SanitizerKind::Alignment)) {
3307 // If alignment-assumption sanitizer is enabled, we do *not* add
3308 // alignment attribute here, but emit normal alignment assumption,
3309 // so the UBSAN check could function.
3310 llvm::ConstantInt *AlignmentCI =
3311 cast<llvm::ConstantInt>(Val: EmitScalarExpr(E: AVAttr->getAlignment()));
3312 uint64_t AlignmentInt =
3313 AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment);
3314 if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
3315 AI->removeAttr(Kind: llvm::Attribute::AttrKind::Alignment);
3316 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3317 .addAlignmentAttr(Align: llvm::Align(AlignmentInt)));
3318 }
3319 }
3320 }
3321
3322 // Set 'noalias' if an argument type has the `restrict` qualifier.
3323 if (Arg->getType().isRestrictQualified())
3324 AI->addAttr(Kind: llvm::Attribute::NoAlias);
3325 }
3326
3327 // Prepare the argument value. If we have the trivial case, handle it
3328 // with no muss and fuss.
3329 if (!isa<llvm::StructType>(Val: ArgI.getCoerceToType()) &&
3330 ArgI.getCoerceToType() == ConvertType(T: Ty) &&
3331 ArgI.getDirectOffset() == 0) {
3332 assert(NumIRArgs == 1);
3333
3334 // LLVM expects swifterror parameters to be used in very restricted
3335 // ways. Copy the value into a less-restricted temporary.
3336 llvm::Value *V = AI;
3337 if (FI.getExtParameterInfo(argIndex: ArgNo).getABI() ==
3338 ParameterABI::SwiftErrorResult) {
3339 QualType pointeeTy = Ty->getPointeeType();
3340 assert(pointeeTy->isPointerType());
3341 RawAddress temp =
3342 CreateMemTemp(T: pointeeTy, Align: getPointerAlign(), Name: "swifterror.temp");
3343 Address arg = makeNaturalAddressForPointer(
3344 Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
3345 llvm::Value *incomingErrorValue = Builder.CreateLoad(Addr: arg);
3346 Builder.CreateStore(Val: incomingErrorValue, Addr: temp);
3347 V = temp.getPointer();
3348
3349 // Push a cleanup to copy the value back at the end of the function.
3350 // The convention does not guarantee that the value will be written
3351 // back if the function exits with an unwind exception.
3352 EHStack.pushCleanup<CopyBackSwiftError>(Kind: NormalCleanup, A: temp, A: arg);
3353 }
3354
3355 // Ensure the argument is the correct type.
3356 if (V->getType() != ArgI.getCoerceToType())
3357 V = Builder.CreateBitCast(V, DestTy: ArgI.getCoerceToType());
3358
3359 if (isPromoted)
3360 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3361
3362 // Because of merging of function types from multiple decls it is
3363 // possible for the type of an argument to not match the corresponding
3364 // type in the function type. Since we are codegening the callee
3365 // in here, add a cast to the argument type.
3366 llvm::Type *LTy = ConvertType(T: Arg->getType());
3367 if (V->getType() != LTy)
3368 V = Builder.CreateBitCast(V, DestTy: LTy);
3369
3370 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3371 break;
3372 }
3373
3374 // VLST arguments are coerced to VLATs at the function boundary for
3375 // ABI consistency. If this is a VLST that was coerced to
3376 // a VLAT at the function boundary and the types match up, use
3377 // llvm.vector.extract to convert back to the original VLST.
3378 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: ConvertType(T: Ty))) {
3379 llvm::Value *ArgVal = Fn->getArg(i: FirstIRArg);
3380 if (auto *VecTyFrom =
3381 dyn_cast<llvm::ScalableVectorType>(Val: ArgVal->getType())) {
3382 auto [Coerced, Extracted] = CoerceScalableToFixed(
3383 CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: ArgVal, Name: Arg->getName());
3384 if (Extracted) {
3385 assert(NumIRArgs == 1);
3386 ArgVals.push_back(Elt: ParamValue::forDirect(value: Coerced));
3387 break;
3388 }
3389 }
3390 }
3391
3392 llvm::StructType *STy =
3393 dyn_cast<llvm::StructType>(Val: ArgI.getCoerceToType());
3394 Address Alloca =
3395 CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(D: Arg), Name: Arg->getName());
3396
3397 // Pointer to store into.
3398 Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3399
3400 // Fast-isel and the optimizer generally like scalar values better than
3401 // FCAs, so we flatten them if this is safe to do for this argument.
3402 if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
3403 STy->getNumElements() > 1) {
3404 llvm::TypeSize StructSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
3405 llvm::TypeSize PtrElementSize =
3406 CGM.getDataLayout().getTypeAllocSize(Ty: Ptr.getElementType());
3407 if (StructSize.isScalable()) {
3408 assert(STy->containsHomogeneousScalableVectorTypes() &&
3409 "ABI only supports structure with homogeneous scalable vector "
3410 "type");
3411 assert(StructSize == PtrElementSize &&
3412 "Only allow non-fractional movement of structure with"
3413 "homogeneous scalable vector type");
3414 assert(STy->getNumElements() == NumIRArgs);
3415
3416 llvm::Value *LoadedStructValue = llvm::PoisonValue::get(T: STy);
3417 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3418 auto *AI = Fn->getArg(i: FirstIRArg + i);
3419 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3420 LoadedStructValue =
3421 Builder.CreateInsertValue(Agg: LoadedStructValue, Val: AI, Idxs: i);
3422 }
3423
3424 Builder.CreateStore(Val: LoadedStructValue, Addr: Ptr);
3425 } else {
3426 uint64_t SrcSize = StructSize.getFixedValue();
3427 uint64_t DstSize = PtrElementSize.getFixedValue();
3428
3429 Address AddrToStoreInto = Address::invalid();
3430 if (SrcSize <= DstSize) {
3431 AddrToStoreInto = Ptr.withElementType(ElemTy: STy);
3432 } else {
3433 AddrToStoreInto =
3434 CreateTempAlloca(Ty: STy, align: Alloca.getAlignment(), Name: "coerce");
3435 }
3436
3437 assert(STy->getNumElements() == NumIRArgs);
3438 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3439 auto AI = Fn->getArg(i: FirstIRArg + i);
3440 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3441 Address EltPtr = Builder.CreateStructGEP(Addr: AddrToStoreInto, Index: i);
3442 Builder.CreateStore(Val: AI, Addr: EltPtr);
3443 }
3444
3445 if (SrcSize > DstSize) {
3446 Builder.CreateMemCpy(Dest: Ptr, Src: AddrToStoreInto, Size: DstSize);
3447 }
3448 }
3449 } else {
3450 // Simple case, just do a coerced store of the argument into the alloca.
3451 assert(NumIRArgs == 1);
3452 auto AI = Fn->getArg(i: FirstIRArg);
3453 AI->setName(Arg->getName() + ".coerce");
3454 CreateCoercedStore(
3455 Src: AI, Dst: Ptr,
3456 DstSize: llvm::TypeSize::getFixed(
3457 ExactSize: getContext().getTypeSizeInChars(T: Ty).getQuantity() -
3458 ArgI.getDirectOffset()),
3459 /*DstIsVolatile=*/false);
3460 }
3461
3462 // Match to what EmitParmDecl is expecting for this type.
3463 if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3464 llvm::Value *V =
3465 EmitLoadOfScalar(Addr: Alloca, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3466 if (isPromoted)
3467 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3468 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3469 } else {
3470 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3471 }
3472 break;
3473 }
3474
3475 case ABIArgInfo::CoerceAndExpand: {
3476 // Reconstruct into a temporary.
3477 Address alloca = CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(D: Arg));
3478 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: alloca));
3479
3480 auto coercionType = ArgI.getCoerceAndExpandType();
3481 auto unpaddedCoercionType = ArgI.getUnpaddedCoerceAndExpandType();
3482 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
3483
3484 alloca = alloca.withElementType(ElemTy: coercionType);
3485
3486 unsigned argIndex = FirstIRArg;
3487 unsigned unpaddedIndex = 0;
3488 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3489 llvm::Type *eltType = coercionType->getElementType(N: i);
3490 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
3491 continue;
3492
3493 auto eltAddr = Builder.CreateStructGEP(Addr: alloca, Index: i);
3494 llvm::Value *elt = Fn->getArg(i: argIndex++);
3495
3496 auto paramType = unpaddedStruct
3497 ? unpaddedStruct->getElementType(N: unpaddedIndex++)
3498 : unpaddedCoercionType;
3499
3500 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: eltType)) {
3501 if (auto *VecTyFrom = dyn_cast<llvm::ScalableVectorType>(Val: paramType)) {
3502 bool Extracted;
3503 std::tie(args&: elt, args&: Extracted) = CoerceScalableToFixed(
3504 CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: elt, Name: elt->getName());
3505 assert(Extracted && "Unexpected scalable to fixed vector coercion");
3506 }
3507 }
3508 Builder.CreateStore(Val: elt, Addr: eltAddr);
3509 }
3510 assert(argIndex == FirstIRArg + NumIRArgs);
3511 break;
3512 }
3513
3514 case ABIArgInfo::Expand: {
3515 // If this structure was expanded into multiple arguments then
3516 // we need to create a temporary and reconstruct it from the
3517 // arguments.
3518 Address Alloca = CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(D: Arg));
3519 LValue LV = MakeAddrLValue(Addr: Alloca, T: Ty);
3520 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3521
3522 auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3523 ExpandTypeFromArgs(Ty, LV, AI&: FnArgIter);
3524 assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3525 for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
3526 auto AI = Fn->getArg(i: FirstIRArg + i);
3527 AI->setName(Arg->getName() + "." + Twine(i));
3528 }
3529 break;
3530 }
3531
3532 case ABIArgInfo::TargetSpecific: {
3533 auto *AI = Fn->getArg(i: FirstIRArg);
3534 AI->setName(Arg->getName() + ".target_coerce");
3535 Address Alloca =
3536 CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(D: Arg), Name: Arg->getName());
3537 Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3538 CGM.getABIInfo().createCoercedStore(Val: AI, DstAddr: Ptr, AI: ArgI, DestIsVolatile: false, CGF&: *this);
3539 if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3540 llvm::Value *V =
3541 EmitLoadOfScalar(Addr: Alloca, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3542 if (isPromoted) {
3543 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3544 }
3545 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3546 } else {
3547 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3548 }
3549 break;
3550 }
3551 case ABIArgInfo::Ignore:
3552 assert(NumIRArgs == 0);
3553 // Initialize the local variable appropriately.
3554 if (!hasScalarEvaluationKind(T: Ty)) {
3555 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: CreateMemTemp(T: Ty)));
3556 } else {
3557 llvm::Value *U = llvm::UndefValue::get(T: ConvertType(T: Arg->getType()));
3558 ArgVals.push_back(Elt: ParamValue::forDirect(value: U));
3559 }
3560 break;
3561 }
3562 }
3563
3564 if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3565 for (int I = Args.size() - 1; I >= 0; --I)
3566 EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3567 } else {
3568 for (unsigned I = 0, E = Args.size(); I != E; ++I)
3569 EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3570 }
3571}
3572
3573static void eraseUnusedBitCasts(llvm::Instruction *insn) {
3574 while (insn->use_empty()) {
3575 llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: insn);
3576 if (!bitcast)
3577 return;
3578
3579 // This is "safe" because we would have used a ConstantExpr otherwise.
3580 insn = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3581 bitcast->eraseFromParent();
3582 }
3583}
3584
3585/// Try to emit a fused autorelease of a return result.
3586static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
3587 llvm::Value *result) {
3588 // We must be immediately followed the cast.
3589 llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3590 if (BB->empty())
3591 return nullptr;
3592 if (&BB->back() != result)
3593 return nullptr;
3594
3595 llvm::Type *resultType = result->getType();
3596
3597 // result is in a BasicBlock and is therefore an Instruction.
3598 llvm::Instruction *generator = cast<llvm::Instruction>(Val: result);
3599
3600 SmallVector<llvm::Instruction *, 4> InstsToKill;
3601
3602 // Look for:
3603 // %generator = bitcast %type1* %generator2 to %type2*
3604 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: generator)) {
3605 // We would have emitted this as a constant if the operand weren't
3606 // an Instruction.
3607 generator = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3608
3609 // Require the generator to be immediately followed by the cast.
3610 if (generator->getNextNode() != bitcast)
3611 return nullptr;
3612
3613 InstsToKill.push_back(Elt: bitcast);
3614 }
3615
3616 // Look for:
3617 // %generator = call i8* @objc_retain(i8* %originalResult)
3618 // or
3619 // %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
3620 llvm::CallInst *call = dyn_cast<llvm::CallInst>(Val: generator);
3621 if (!call)
3622 return nullptr;
3623
3624 bool doRetainAutorelease;
3625
3626 if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
3627 doRetainAutorelease = true;
3628 } else if (call->getCalledOperand() ==
3629 CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
3630 doRetainAutorelease = false;
3631
3632 // If we emitted an assembly marker for this call (and the
3633 // ARCEntrypoints field should have been set if so), go looking
3634 // for that call. If we can't find it, we can't do this
3635 // optimization. But it should always be the immediately previous
3636 // instruction, unless we needed bitcasts around the call.
3637 if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
3638 llvm::Instruction *prev = call->getPrevNode();
3639 assert(prev);
3640 if (isa<llvm::BitCastInst>(Val: prev)) {
3641 prev = prev->getPrevNode();
3642 assert(prev);
3643 }
3644 assert(isa<llvm::CallInst>(prev));
3645 assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3646 CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
3647 InstsToKill.push_back(Elt: prev);
3648 }
3649 } else {
3650 return nullptr;
3651 }
3652
3653 result = call->getArgOperand(i: 0);
3654 InstsToKill.push_back(Elt: call);
3655
3656 // Keep killing bitcasts, for sanity. Note that we no longer care
3657 // about precise ordering as long as there's exactly one use.
3658 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: result)) {
3659 if (!bitcast->hasOneUse())
3660 break;
3661 InstsToKill.push_back(Elt: bitcast);
3662 result = bitcast->getOperand(i_nocapture: 0);
3663 }
3664
3665 // Delete all the unnecessary instructions, from latest to earliest.
3666 for (auto *I : InstsToKill)
3667 I->eraseFromParent();
3668
3669 // Do the fused retain/autorelease if we were asked to.
3670 if (doRetainAutorelease)
3671 result = CGF.EmitARCRetainAutoreleaseReturnValue(value: result);
3672
3673 // Cast back to the result type.
3674 return CGF.Builder.CreateBitCast(V: result, DestTy: resultType);
3675}
3676
3677/// If this is a +1 of the value of an immutable 'self', remove it.
3678static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
3679 llvm::Value *result) {
3680 // This is only applicable to a method with an immutable 'self'.
3681 const ObjCMethodDecl *method =
3682 dyn_cast_or_null<ObjCMethodDecl>(Val: CGF.CurCodeDecl);
3683 if (!method)
3684 return nullptr;
3685 const VarDecl *self = method->getSelfDecl();
3686 if (!self->getType().isConstQualified())
3687 return nullptr;
3688
3689 // Look for a retain call. Note: stripPointerCasts looks through returned arg
3690 // functions, which would cause us to miss the retain.
3691 llvm::CallInst *retainCall = dyn_cast<llvm::CallInst>(Val: result);
3692 if (!retainCall || retainCall->getCalledOperand() !=
3693 CGF.CGM.getObjCEntrypoints().objc_retain)
3694 return nullptr;
3695
3696 // Look for an ordinary load of 'self'.
3697 llvm::Value *retainedValue = retainCall->getArgOperand(i: 0);
3698 llvm::LoadInst *load =
3699 dyn_cast<llvm::LoadInst>(Val: retainedValue->stripPointerCasts());
3700 if (!load || load->isAtomic() || load->isVolatile() ||
3701 load->getPointerOperand() != CGF.GetAddrOfLocalVar(VD: self).getBasePointer())
3702 return nullptr;
3703
3704 // Okay! Burn it all down. This relies for correctness on the
3705 // assumption that the retain is emitted as part of the return and
3706 // that thereafter everything is used "linearly".
3707 llvm::Type *resultType = result->getType();
3708 eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: result));
3709 assert(retainCall->use_empty());
3710 retainCall->eraseFromParent();
3711 eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: retainedValue));
3712
3713 return CGF.Builder.CreateBitCast(V: load, DestTy: resultType);
3714}
3715
3716/// Emit an ARC autorelease of the result of a function.
3717///
3718/// \return the value to actually return from the function
3719static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
3720 llvm::Value *result) {
3721 // If we're returning 'self', kill the initial retain. This is a
3722 // heuristic attempt to "encourage correctness" in the really unfortunate
3723 // case where we have a return of self during a dealloc and we desperately
3724 // need to avoid the possible autorelease.
3725 if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
3726 return self;
3727
3728 // At -O0, try to emit a fused retain/autorelease.
3729 if (CGF.shouldUseFusedARCCalls())
3730 if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
3731 return fused;
3732
3733 return CGF.EmitARCAutoreleaseReturnValue(value: result);
3734}
3735
3736/// Heuristically search for a dominating store to the return-value slot.
3737static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
3738 llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer();
3739
3740 // Check if a User is a store which pointerOperand is the ReturnValue.
3741 // We are looking for stores to the ReturnValue, not for stores of the
3742 // ReturnValue to some other location.
3743 auto GetStoreIfValid = [&CGF,
3744 ReturnValuePtr](llvm::User *U) -> llvm::StoreInst * {
3745 auto *SI = dyn_cast<llvm::StoreInst>(Val: U);
3746 if (!SI || SI->getPointerOperand() != ReturnValuePtr ||
3747 SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())
3748 return nullptr;
3749 // These aren't actually possible for non-coerced returns, and we
3750 // only care about non-coerced returns on this code path.
3751 // All memory instructions inside __try block are volatile.
3752 assert(!SI->isAtomic() &&
3753 (!SI->isVolatile() || CGF.currentFunctionUsesSEHTry()));
3754 return SI;
3755 };
3756 // If there are multiple uses of the return-value slot, just check
3757 // for something immediately preceding the IP. Sometimes this can
3758 // happen with how we generate implicit-returns; it can also happen
3759 // with noreturn cleanups.
3760 if (!ReturnValuePtr->hasOneUse()) {
3761 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3762 if (IP->empty())
3763 return nullptr;
3764
3765 // Look at directly preceding instruction, skipping bitcasts, lifetime
3766 // markers, and fake uses and their operands.
3767 const llvm::Instruction *LoadIntoFakeUse = nullptr;
3768 for (llvm::Instruction &I : llvm::reverse(C&: *IP)) {
3769 // Ignore instructions that are just loads for fake uses; the load should
3770 // immediately precede the fake use, so we only need to remember the
3771 // operand for the last fake use seen.
3772 if (LoadIntoFakeUse == &I)
3773 continue;
3774 if (isa<llvm::BitCastInst>(Val: &I))
3775 continue;
3776 if (auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &I)) {
3777 if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
3778 continue;
3779
3780 if (II->getIntrinsicID() == llvm::Intrinsic::fake_use) {
3781 LoadIntoFakeUse = dyn_cast<llvm::Instruction>(Val: II->getArgOperand(i: 0));
3782 continue;
3783 }
3784 }
3785 return GetStoreIfValid(&I);
3786 }
3787 return nullptr;
3788 }
3789
3790 llvm::StoreInst *store = GetStoreIfValid(ReturnValuePtr->user_back());
3791 if (!store)
3792 return nullptr;
3793
3794 // Now do a first-and-dirty dominance check: just walk up the
3795 // single-predecessors chain from the current insertion point.
3796 llvm::BasicBlock *StoreBB = store->getParent();
3797 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3798 llvm::SmallPtrSet<llvm::BasicBlock *, 4> SeenBBs;
3799 while (IP != StoreBB) {
3800 if (!SeenBBs.insert(Ptr: IP).second || !(IP = IP->getSinglePredecessor()))
3801 return nullptr;
3802 }
3803
3804 // Okay, the store's basic block dominates the insertion point; we
3805 // can do our thing.
3806 return store;
3807}
3808
3809// Helper functions for EmitCMSEClearRecord
3810
3811// Set the bits corresponding to a field having width `BitWidth` and located at
3812// offset `BitOffset` (from the least significant bit) within a storage unit of
3813// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
3814// Use little-endian layout, i.e.`Bits[0]` is the LSB.
3815static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
3816 int BitWidth, int CharWidth) {
3817 assert(CharWidth <= 64);
3818 assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3819
3820 int Pos = 0;
3821 if (BitOffset >= CharWidth) {
3822 Pos += BitOffset / CharWidth;
3823 BitOffset = BitOffset % CharWidth;
3824 }
3825
3826 const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
3827 if (BitOffset + BitWidth >= CharWidth) {
3828 Bits[Pos++] |= (Used << BitOffset) & Used;
3829 BitWidth -= CharWidth - BitOffset;
3830 BitOffset = 0;
3831 }
3832
3833 while (BitWidth >= CharWidth) {
3834 Bits[Pos++] = Used;
3835 BitWidth -= CharWidth;
3836 }
3837
3838 if (BitWidth > 0)
3839 Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
3840}
3841
3842// Set the bits corresponding to a field having width `BitWidth` and located at
3843// offset `BitOffset` (from the least significant bit) within a storage unit of
3844// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
3845// `Bits` corresponds to one target byte. Use target endian layout.
3846static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
3847 int StorageSize, int BitOffset, int BitWidth,
3848 int CharWidth, bool BigEndian) {
3849
3850 SmallVector<uint64_t, 8> TmpBits(StorageSize);
3851 setBitRange(Bits&: TmpBits, BitOffset, BitWidth, CharWidth);
3852
3853 if (BigEndian)
3854 std::reverse(first: TmpBits.begin(), last: TmpBits.end());
3855
3856 for (uint64_t V : TmpBits)
3857 Bits[StorageOffset++] |= V;
3858}
3859
3860static void setUsedBits(CodeGenModule &, QualType, int,
3861 SmallVectorImpl<uint64_t> &);
3862
3863// Set the bits in `Bits`, which correspond to the value representations of
3864// the actual members of the record type `RTy`. Note that this function does
3865// not handle base classes, virtual tables, etc, since they cannot happen in
3866// CMSE function arguments or return. The bit mask corresponds to the target
3867// memory layout, i.e. it's endian dependent.
3868static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
3869 SmallVectorImpl<uint64_t> &Bits) {
3870 ASTContext &Context = CGM.getContext();
3871 int CharWidth = Context.getCharWidth();
3872 const RecordDecl *RD = RTy->getDecl()->getDefinition();
3873 const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(D: RD);
3874 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
3875
3876 int Idx = 0;
3877 for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) {
3878 const FieldDecl *F = *I;
3879
3880 if (F->isUnnamedBitField() || F->isZeroLengthBitField() ||
3881 F->getType()->isIncompleteArrayType())
3882 continue;
3883
3884 if (F->isBitField()) {
3885 const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(FD: F);
3886 setBitRange(Bits, StorageOffset: Offset + BFI.StorageOffset.getQuantity(),
3887 StorageSize: BFI.StorageSize / CharWidth, BitOffset: BFI.Offset, BitWidth: BFI.Size, CharWidth,
3888 BigEndian: CGM.getDataLayout().isBigEndian());
3889 continue;
3890 }
3891
3892 setUsedBits(CGM, F->getType(),
3893 Offset + ASTLayout.getFieldOffset(FieldNo: Idx) / CharWidth, Bits);
3894 }
3895}
3896
3897// Set the bits in `Bits`, which correspond to the value representations of
3898// the elements of an array type `ATy`.
3899static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
3900 int Offset, SmallVectorImpl<uint64_t> &Bits) {
3901 const ASTContext &Context = CGM.getContext();
3902
3903 QualType ETy = Context.getBaseElementType(VAT: ATy);
3904 int Size = Context.getTypeSizeInChars(T: ETy).getQuantity();
3905 SmallVector<uint64_t, 4> TmpBits(Size);
3906 setUsedBits(CGM, ETy, 0, TmpBits);
3907
3908 for (int I = 0, N = Context.getConstantArrayElementCount(CA: ATy); I < N; ++I) {
3909 auto Src = TmpBits.begin();
3910 auto Dst = Bits.begin() + Offset + I * Size;
3911 for (int J = 0; J < Size; ++J)
3912 *Dst++ |= *Src++;
3913 }
3914}
3915
3916// Set the bits in `Bits`, which correspond to the value representations of
3917// the type `QTy`.
3918static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
3919 SmallVectorImpl<uint64_t> &Bits) {
3920 if (const auto *RTy = QTy->getAsCanonical<RecordType>())
3921 return setUsedBits(CGM, RTy, Offset, Bits);
3922
3923 ASTContext &Context = CGM.getContext();
3924 if (const auto *ATy = Context.getAsConstantArrayType(T: QTy))
3925 return setUsedBits(CGM, ATy, Offset, Bits);
3926
3927 int Size = Context.getTypeSizeInChars(T: QTy).getQuantity();
3928 if (Size <= 0)
3929 return;
3930
3931 std::fill_n(first: Bits.begin() + Offset, n: Size,
3932 value: (uint64_t(1) << Context.getCharWidth()) - 1);
3933}
3934
3935static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
3936 int Pos, int Size, int CharWidth,
3937 bool BigEndian) {
3938 assert(Size > 0);
3939 uint64_t Mask = 0;
3940 if (BigEndian) {
3941 for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
3942 ++P)
3943 Mask = (Mask << CharWidth) | *P;
3944 } else {
3945 auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
3946 do
3947 Mask = (Mask << CharWidth) | *--P;
3948 while (P != End);
3949 }
3950 return Mask;
3951}
3952
3953// Emit code to clear the bits in a record, which aren't a part of any user
3954// declared member, when the record is a function return.
3955llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3956 llvm::IntegerType *ITy,
3957 QualType QTy) {
3958 assert(Src->getType() == ITy);
3959 assert(ITy->getScalarSizeInBits() <= 64);
3960
3961 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3962 int Size = DataLayout.getTypeStoreSize(Ty: ITy);
3963 SmallVector<uint64_t, 4> Bits(Size);
3964 setUsedBits(CGM, RTy: QTy->castAsCanonical<RecordType>(), Offset: 0, Bits);
3965
3966 int CharWidth = CGM.getContext().getCharWidth();
3967 uint64_t Mask =
3968 buildMultiCharMask(Bits, Pos: 0, Size, CharWidth, BigEndian: DataLayout.isBigEndian());
3969
3970 return Builder.CreateAnd(LHS: Src, RHS: Mask, Name: "cmse.clear");
3971}
3972
3973// Emit code to clear the bits in a record, which aren't a part of any user
3974// declared member, when the record is a function argument.
3975llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3976 llvm::ArrayType *ATy,
3977 QualType QTy) {
3978 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3979 int Size = DataLayout.getTypeStoreSize(Ty: ATy);
3980 SmallVector<uint64_t, 16> Bits(Size);
3981 setUsedBits(CGM, RTy: QTy->castAsCanonical<RecordType>(), Offset: 0, Bits);
3982
3983 // Clear each element of the LLVM array.
3984 int CharWidth = CGM.getContext().getCharWidth();
3985 int CharsPerElt =
3986 ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
3987 int MaskIndex = 0;
3988 llvm::Value *R = llvm::PoisonValue::get(T: ATy);
3989 for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
3990 uint64_t Mask = buildMultiCharMask(Bits, Pos: MaskIndex, Size: CharsPerElt, CharWidth,
3991 BigEndian: DataLayout.isBigEndian());
3992 MaskIndex += CharsPerElt;
3993 llvm::Value *T0 = Builder.CreateExtractValue(Agg: Src, Idxs: I);
3994 llvm::Value *T1 = Builder.CreateAnd(LHS: T0, RHS: Mask, Name: "cmse.clear");
3995 R = Builder.CreateInsertValue(Agg: R, Val: T1, Idxs: I);
3996 }
3997
3998 return R;
3999}
4000
4001void CodeGenFunction::EmitFunctionEpilog(
4002 const CGFunctionInfo &FI, bool EmitRetDbgLoc, SourceLocation EndLoc,
4003 uint64_t RetKeyInstructionsSourceAtom) {
4004 if (FI.isNoReturn()) {
4005 // Noreturn functions don't return.
4006 EmitUnreachable(Loc: EndLoc);
4007 return;
4008 }
4009
4010 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {
4011 // Naked functions don't have epilogues.
4012 Builder.CreateUnreachable();
4013 return;
4014 }
4015
4016 // Functions with no result always return void.
4017 if (!ReturnValue.isValid()) {
4018 auto *I = Builder.CreateRetVoid();
4019 if (RetKeyInstructionsSourceAtom)
4020 addInstToSpecificSourceAtom(KeyInstruction: I, Backup: nullptr, Atom: RetKeyInstructionsSourceAtom);
4021 else
4022 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4023 return;
4024 }
4025
4026 llvm::DebugLoc RetDbgLoc;
4027 llvm::Value *RV = nullptr;
4028 QualType RetTy = FI.getReturnType();
4029 const ABIArgInfo &RetAI = FI.getReturnInfo();
4030
4031 switch (RetAI.getKind()) {
4032 case ABIArgInfo::InAlloca:
4033 // Aggregates get evaluated directly into the destination. Sometimes we
4034 // need to return the sret value in a register, though.
4035 assert(hasAggregateEvaluationKind(RetTy));
4036 if (RetAI.getInAllocaSRet()) {
4037 llvm::Function::arg_iterator EI = CurFn->arg_end();
4038 --EI;
4039 llvm::Value *ArgStruct = &*EI;
4040 llvm::Value *SRet = Builder.CreateStructGEP(
4041 Ty: FI.getArgStruct(), Ptr: ArgStruct, Idx: RetAI.getInAllocaFieldIndex());
4042 llvm::Type *Ty =
4043 cast<llvm::GetElementPtrInst>(Val: SRet)->getResultElementType();
4044 RV = Builder.CreateAlignedLoad(Ty, Addr: SRet, Align: getPointerAlign(), Name: "sret");
4045 }
4046 break;
4047
4048 case ABIArgInfo::Indirect: {
4049 auto AI = CurFn->arg_begin();
4050 if (RetAI.isSRetAfterThis())
4051 ++AI;
4052 switch (getEvaluationKind(T: RetTy)) {
4053 case TEK_Complex: {
4054 ComplexPairTy RT =
4055 EmitLoadOfComplex(src: MakeAddrLValue(Addr: ReturnValue, T: RetTy), loc: EndLoc);
4056 EmitStoreOfComplex(V: RT, dest: MakeNaturalAlignAddrLValue(V: &*AI, T: RetTy),
4057 /*isInit*/ true);
4058 break;
4059 }
4060 case TEK_Aggregate:
4061 // Do nothing; aggregates get evaluated directly into the destination.
4062 break;
4063 case TEK_Scalar: {
4064 LValueBaseInfo BaseInfo;
4065 TBAAAccessInfo TBAAInfo;
4066 CharUnits Alignment =
4067 CGM.getNaturalTypeAlignment(T: RetTy, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
4068 Address ArgAddr(&*AI, ConvertType(T: RetTy), Alignment);
4069 LValue ArgVal =
4070 LValue::MakeAddr(Addr: ArgAddr, type: RetTy, Context&: getContext(), BaseInfo, TBAAInfo);
4071 EmitStoreOfScalar(
4072 value: EmitLoadOfScalar(lvalue: MakeAddrLValue(Addr: ReturnValue, T: RetTy), Loc: EndLoc), lvalue: ArgVal,
4073 /*isInit*/ true);
4074 break;
4075 }
4076 }
4077 break;
4078 }
4079
4080 case ABIArgInfo::Extend:
4081 case ABIArgInfo::Direct:
4082 if (RetAI.getCoerceToType() == ConvertType(T: RetTy) &&
4083 RetAI.getDirectOffset() == 0) {
4084 // The internal return value temp always will have pointer-to-return-type
4085 // type, just do a load.
4086
4087 // If there is a dominating store to ReturnValue, we can elide
4088 // the load, zap the store, and usually zap the alloca.
4089 if (llvm::StoreInst *SI = findDominatingStoreToReturnValue(CGF&: *this)) {
4090 // Reuse the debug location from the store unless there is
4091 // cleanup code to be emitted between the store and return
4092 // instruction.
4093 if (EmitRetDbgLoc && !AutoreleaseResult)
4094 RetDbgLoc = SI->getDebugLoc();
4095 // Get the stored value and nuke the now-dead store.
4096 RV = SI->getValueOperand();
4097 SI->eraseFromParent();
4098
4099 // Otherwise, we have to do a simple load.
4100 } else {
4101 RV = Builder.CreateLoad(Addr: ReturnValue);
4102 }
4103 } else {
4104 // If the value is offset in memory, apply the offset now.
4105 Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
4106
4107 RV = CreateCoercedLoad(Src: V, Ty: RetAI.getCoerceToType(), CGF&: *this);
4108 }
4109
4110 // In ARC, end functions that return a retainable type with a call
4111 // to objc_autoreleaseReturnValue.
4112 if (AutoreleaseResult) {
4113#ifndef NDEBUG
4114 // Type::isObjCRetainabletype has to be called on a QualType that hasn't
4115 // been stripped of the typedefs, so we cannot use RetTy here. Get the
4116 // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
4117 // CurCodeDecl or BlockInfo.
4118 QualType RT;
4119
4120 if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
4121 RT = FD->getReturnType();
4122 else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
4123 RT = MD->getReturnType();
4124 else if (isa<BlockDecl>(CurCodeDecl))
4125 RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
4126 else
4127 llvm_unreachable("Unexpected function/method type");
4128
4129 assert(getLangOpts().ObjCAutoRefCount && !FI.isReturnsRetained() &&
4130 RT->isObjCRetainableType());
4131#endif
4132 RV = emitAutoreleaseOfResult(CGF&: *this, result: RV);
4133 }
4134
4135 break;
4136
4137 case ABIArgInfo::Ignore:
4138 break;
4139
4140 case ABIArgInfo::CoerceAndExpand: {
4141 auto coercionType = RetAI.getCoerceAndExpandType();
4142 auto unpaddedCoercionType = RetAI.getUnpaddedCoerceAndExpandType();
4143 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
4144
4145 // Load all of the coerced elements out into results.
4146 llvm::SmallVector<llvm::Value *, 4> results;
4147 Address addr = ReturnValue.withElementType(ElemTy: coercionType);
4148 unsigned unpaddedIndex = 0;
4149 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
4150 auto coercedEltType = coercionType->getElementType(N: i);
4151 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType: coercedEltType))
4152 continue;
4153
4154 auto eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
4155 llvm::Value *elt = CreateCoercedLoad(
4156 Src: eltAddr,
4157 Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
4158 : unpaddedCoercionType,
4159 CGF&: *this);
4160 results.push_back(Elt: elt);
4161 }
4162
4163 // If we have one result, it's the single direct result type.
4164 if (results.size() == 1) {
4165 RV = results[0];
4166
4167 // Otherwise, we need to make a first-class aggregate.
4168 } else {
4169 // Construct a return type that lacks padding elements.
4170 llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
4171
4172 RV = llvm::PoisonValue::get(T: returnType);
4173 for (unsigned i = 0, e = results.size(); i != e; ++i) {
4174 RV = Builder.CreateInsertValue(Agg: RV, Val: results[i], Idxs: i);
4175 }
4176 }
4177 break;
4178 }
4179 case ABIArgInfo::TargetSpecific: {
4180 Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
4181 RV = CGM.getABIInfo().createCoercedLoad(SrcAddr: V, AI: RetAI, CGF&: *this);
4182 break;
4183 }
4184 case ABIArgInfo::Expand:
4185 case ABIArgInfo::IndirectAliased:
4186 llvm_unreachable("Invalid ABI kind for return argument");
4187 }
4188
4189 llvm::Instruction *Ret;
4190 if (RV) {
4191 if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) {
4192 // For certain return types, clear padding bits, as they may reveal
4193 // sensitive information.
4194 // Small struct/union types are passed as integers.
4195 auto *ITy = dyn_cast<llvm::IntegerType>(Val: RV->getType());
4196 if (ITy != nullptr && isa<RecordType>(Val: RetTy.getCanonicalType()))
4197 RV = EmitCMSEClearRecord(Src: RV, ITy, QTy: RetTy);
4198 }
4199 EmitReturnValueCheck(RV);
4200 Ret = Builder.CreateRet(V: RV);
4201 } else {
4202 Ret = Builder.CreateRetVoid();
4203 }
4204
4205 if (RetDbgLoc)
4206 Ret->setDebugLoc(std::move(RetDbgLoc));
4207
4208 llvm::Value *Backup = RV ? Ret->getOperand(i: 0) : nullptr;
4209 if (RetKeyInstructionsSourceAtom)
4210 addInstToSpecificSourceAtom(KeyInstruction: Ret, Backup, Atom: RetKeyInstructionsSourceAtom);
4211 else
4212 addInstToNewSourceAtom(KeyInstruction: Ret, Backup);
4213}
4214
4215void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
4216 // A current decl may not be available when emitting vtable thunks.
4217 if (!CurCodeDecl)
4218 return;
4219
4220 // If the return block isn't reachable, neither is this check, so don't emit
4221 // it.
4222 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty())
4223 return;
4224
4225 ReturnsNonNullAttr *RetNNAttr = nullptr;
4226 if (SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute))
4227 RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
4228
4229 if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
4230 return;
4231
4232 // Prefer the returns_nonnull attribute if it's present.
4233 SourceLocation AttrLoc;
4234 SanitizerKind::SanitizerOrdinal CheckKind;
4235 SanitizerHandler Handler;
4236 if (RetNNAttr) {
4237 assert(!requiresReturnValueNullabilityCheck() &&
4238 "Cannot check nullability and the nonnull attribute");
4239 AttrLoc = RetNNAttr->getLocation();
4240 CheckKind = SanitizerKind::SO_ReturnsNonnullAttribute;
4241 Handler = SanitizerHandler::NonnullReturn;
4242 } else {
4243 if (auto *DD = dyn_cast<DeclaratorDecl>(Val: CurCodeDecl))
4244 if (auto *TSI = DD->getTypeSourceInfo())
4245 if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
4246 AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
4247 CheckKind = SanitizerKind::SO_NullabilityReturn;
4248 Handler = SanitizerHandler::NullabilityReturn;
4249 }
4250
4251 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4252
4253 // Make sure the "return" source location is valid. If we're checking a
4254 // nullability annotation, make sure the preconditions for the check are met.
4255 llvm::BasicBlock *Check = createBasicBlock(name: "nullcheck");
4256 llvm::BasicBlock *NoCheck = createBasicBlock(name: "no.nullcheck");
4257 llvm::Value *SLocPtr = Builder.CreateLoad(Addr: ReturnLocation, Name: "return.sloc.load");
4258 llvm::Value *CanNullCheck = Builder.CreateIsNotNull(Arg: SLocPtr);
4259 if (requiresReturnValueNullabilityCheck())
4260 CanNullCheck =
4261 Builder.CreateAnd(LHS: CanNullCheck, RHS: RetValNullabilityPrecondition);
4262 Builder.CreateCondBr(Cond: CanNullCheck, True: Check, False: NoCheck);
4263 EmitBlock(BB: Check);
4264
4265 // Now do the null check.
4266 llvm::Value *Cond = Builder.CreateIsNotNull(Arg: RV);
4267 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc: AttrLoc)};
4268 llvm::Value *DynamicData[] = {SLocPtr};
4269 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: DynamicData);
4270
4271 EmitBlock(BB: NoCheck);
4272
4273#ifndef NDEBUG
4274 // The return location should not be used after the check has been emitted.
4275 ReturnLocation = Address::invalid();
4276#endif
4277}
4278
4279static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
4280 const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
4281 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
4282}
4283
4284static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, QualType Ty) {
4285 // FIXME: Generate IR in one pass, rather than going back and fixing up these
4286 // placeholders.
4287 llvm::Type *IRTy = CGF.ConvertTypeForMem(T: Ty);
4288 llvm::Type *IRPtrTy = llvm::PointerType::getUnqual(C&: CGF.getLLVMContext());
4289 llvm::Value *Placeholder = llvm::PoisonValue::get(T: IRPtrTy);
4290
4291 // FIXME: When we generate this IR in one pass, we shouldn't need
4292 // this win32-specific alignment hack.
4293 CharUnits Align = CharUnits::fromQuantity(Quantity: 4);
4294 Placeholder = CGF.Builder.CreateAlignedLoad(Ty: IRPtrTy, Addr: Placeholder, Align);
4295
4296 return AggValueSlot::forAddr(
4297 addr: Address(Placeholder, IRTy, Align), quals: Ty.getQualifiers(),
4298 isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
4299 isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
4300}
4301
4302void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
4303 const VarDecl *param,
4304 SourceLocation loc) {
4305 // StartFunction converted the ABI-lowered parameter(s) into a
4306 // local alloca. We need to turn that into an r-value suitable
4307 // for EmitCall.
4308 Address local = GetAddrOfLocalVar(VD: param);
4309
4310 QualType type = param->getType();
4311
4312 // GetAddrOfLocalVar returns a pointer-to-pointer for references,
4313 // but the argument needs to be the original pointer.
4314 if (type->isReferenceType()) {
4315 args.add(rvalue: RValue::get(V: Builder.CreateLoad(Addr: local)), type);
4316
4317 // In ARC, move out of consumed arguments so that the release cleanup
4318 // entered by StartFunction doesn't cause an over-release. This isn't
4319 // optimal -O0 code generation, but it should get cleaned up when
4320 // optimization is enabled. This also assumes that delegate calls are
4321 // performed exactly once for a set of arguments, but that should be safe.
4322 } else if (getLangOpts().ObjCAutoRefCount &&
4323 param->hasAttr<NSConsumedAttr>() && type->isObjCRetainableType()) {
4324 llvm::Value *ptr = Builder.CreateLoad(Addr: local);
4325 auto null =
4326 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: ptr->getType()));
4327 Builder.CreateStore(Val: null, Addr: local);
4328 args.add(rvalue: RValue::get(V: ptr), type);
4329
4330 // For the most part, we just need to load the alloca, except that
4331 // aggregate r-values are actually pointers to temporaries.
4332 } else {
4333 args.add(rvalue: convertTempToRValue(addr: local, type, Loc: loc), type);
4334 }
4335
4336 // Deactivate the cleanup for the callee-destructed param that was pushed.
4337 if (type->isRecordType() && !CurFuncIsThunk &&
4338 type->castAsRecordDecl()->isParamDestroyedInCallee() &&
4339 param->needsDestruction(Ctx: getContext())) {
4340 EHScopeStack::stable_iterator cleanup =
4341 CalleeDestructedParamCleanups.lookup(Val: cast<ParmVarDecl>(Val: param));
4342 assert(cleanup.isValid() &&
4343 "cleanup for callee-destructed param not recorded");
4344 // This unreachable is a temporary marker which will be removed later.
4345 llvm::Instruction *isActive = Builder.CreateUnreachable();
4346 args.addArgCleanupDeactivation(Cleanup: cleanup, IsActiveIP: isActive);
4347 }
4348}
4349
4350static bool isProvablyNull(llvm::Value *addr) {
4351 return llvm::isa_and_nonnull<llvm::ConstantPointerNull>(Val: addr);
4352}
4353
4354static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) {
4355 return llvm::isKnownNonZero(V: Addr.getBasePointer(), Q: CGF.CGM.getDataLayout());
4356}
4357
4358/// Emit the actual writing-back of a writeback.
4359static void emitWriteback(CodeGenFunction &CGF,
4360 const CallArgList::Writeback &writeback) {
4361 const LValue &srcLV = writeback.Source;
4362 Address srcAddr = srcLV.getAddress();
4363 assert(!isProvablyNull(srcAddr.getBasePointer()) &&
4364 "shouldn't have writeback for provably null argument");
4365
4366 if (writeback.WritebackExpr) {
4367 CGF.EmitIgnoredExpr(E: writeback.WritebackExpr);
4368 CGF.EmitLifetimeEnd(Addr: writeback.Temporary.getBasePointer());
4369 return;
4370 }
4371
4372 llvm::BasicBlock *contBB = nullptr;
4373
4374 // If the argument wasn't provably non-null, we need to null check
4375 // before doing the store.
4376 bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4377
4378 if (!provablyNonNull) {
4379 llvm::BasicBlock *writebackBB = CGF.createBasicBlock(name: "icr.writeback");
4380 contBB = CGF.createBasicBlock(name: "icr.done");
4381
4382 llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4383 CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: writebackBB);
4384 CGF.EmitBlock(BB: writebackBB);
4385 }
4386
4387 // Load the value to writeback.
4388 llvm::Value *value = CGF.Builder.CreateLoad(Addr: writeback.Temporary);
4389
4390 // Cast it back, in case we're writing an id to a Foo* or something.
4391 value = CGF.Builder.CreateBitCast(V: value, DestTy: srcAddr.getElementType(),
4392 Name: "icr.writeback-cast");
4393
4394 // Perform the writeback.
4395
4396 // If we have a "to use" value, it's something we need to emit a use
4397 // of. This has to be carefully threaded in: if it's done after the
4398 // release it's potentially undefined behavior (and the optimizer
4399 // will ignore it), and if it happens before the retain then the
4400 // optimizer could move the release there.
4401 if (writeback.ToUse) {
4402 assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
4403
4404 // Retain the new value. No need to block-copy here: the block's
4405 // being passed up the stack.
4406 value = CGF.EmitARCRetainNonBlock(value);
4407
4408 // Emit the intrinsic use here.
4409 CGF.EmitARCIntrinsicUse(values: writeback.ToUse);
4410
4411 // Load the old value (primitively).
4412 llvm::Value *oldValue = CGF.EmitLoadOfScalar(lvalue: srcLV, Loc: SourceLocation());
4413
4414 // Put the new value in place (primitively).
4415 CGF.EmitStoreOfScalar(value, lvalue: srcLV, /*init*/ isInit: false);
4416
4417 // Release the old value.
4418 CGF.EmitARCRelease(value: oldValue, precise: srcLV.isARCPreciseLifetime());
4419
4420 // Otherwise, we can just do a normal lvalue store.
4421 } else {
4422 CGF.EmitStoreThroughLValue(Src: RValue::get(V: value), Dst: srcLV);
4423 }
4424
4425 // Jump to the continuation block.
4426 if (!provablyNonNull)
4427 CGF.EmitBlock(BB: contBB);
4428}
4429
4430static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
4431 const CallArgList &CallArgs) {
4432 ArrayRef<CallArgList::CallArgCleanup> Cleanups =
4433 CallArgs.getCleanupsToDeactivate();
4434 // Iterate in reverse to increase the likelihood of popping the cleanup.
4435 for (const auto &I : llvm::reverse(C&: Cleanups)) {
4436 CGF.DeactivateCleanupBlock(Cleanup: I.Cleanup, DominatingIP: I.IsActiveIP);
4437 I.IsActiveIP->eraseFromParent();
4438 }
4439}
4440
4441static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
4442 if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(Val: E->IgnoreParens()))
4443 if (uop->getOpcode() == UO_AddrOf)
4444 return uop->getSubExpr();
4445 return nullptr;
4446}
4447
4448/// Emit an argument that's being passed call-by-writeback. That is,
4449/// we are passing the address of an __autoreleased temporary; it
4450/// might be copy-initialized with the current value of the given
4451/// address, but it will definitely be copied out of after the call.
4452static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
4453 const ObjCIndirectCopyRestoreExpr *CRE) {
4454 LValue srcLV;
4455
4456 // Make an optimistic effort to emit the address as an l-value.
4457 // This can fail if the argument expression is more complicated.
4458 if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(E: CRE->getSubExpr())) {
4459 srcLV = CGF.EmitLValue(E: lvExpr);
4460
4461 // Otherwise, just emit it as a scalar.
4462 } else {
4463 Address srcAddr = CGF.EmitPointerWithAlignment(Addr: CRE->getSubExpr());
4464
4465 QualType srcAddrType =
4466 CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
4467 srcLV = CGF.MakeAddrLValue(Addr: srcAddr, T: srcAddrType);
4468 }
4469 Address srcAddr = srcLV.getAddress();
4470
4471 // The dest and src types don't necessarily match in LLVM terms
4472 // because of the crazy ObjC compatibility rules.
4473
4474 llvm::PointerType *destType =
4475 cast<llvm::PointerType>(Val: CGF.ConvertType(T: CRE->getType()));
4476 llvm::Type *destElemType =
4477 CGF.ConvertTypeForMem(T: CRE->getType()->getPointeeType());
4478
4479 // If the address is a constant null, just pass the appropriate null.
4480 if (isProvablyNull(addr: srcAddr.getBasePointer())) {
4481 args.add(rvalue: RValue::get(V: llvm::ConstantPointerNull::get(T: destType)),
4482 type: CRE->getType());
4483 return;
4484 }
4485
4486 // Create the temporary.
4487 Address temp =
4488 CGF.CreateTempAlloca(Ty: destElemType, align: CGF.getPointerAlign(), Name: "icr.temp");
4489 // Loading an l-value can introduce a cleanup if the l-value is __weak,
4490 // and that cleanup will be conditional if we can't prove that the l-value
4491 // isn't null, so we need to register a dominating point so that the cleanups
4492 // system will make valid IR.
4493 CodeGenFunction::ConditionalEvaluation condEval(CGF);
4494
4495 // Zero-initialize it if we're not doing a copy-initialization.
4496 bool shouldCopy = CRE->shouldCopy();
4497 if (!shouldCopy) {
4498 llvm::Value *null =
4499 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: destElemType));
4500 CGF.Builder.CreateStore(Val: null, Addr: temp);
4501 }
4502
4503 llvm::BasicBlock *contBB = nullptr;
4504 llvm::BasicBlock *originBB = nullptr;
4505
4506 // If the address is *not* known to be non-null, we need to switch.
4507 llvm::Value *finalArgument;
4508
4509 bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4510
4511 if (provablyNonNull) {
4512 finalArgument = temp.emitRawPointer(CGF);
4513 } else {
4514 llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4515
4516 finalArgument = CGF.Builder.CreateSelect(
4517 C: isNull, True: llvm::ConstantPointerNull::get(T: destType),
4518 False: temp.emitRawPointer(CGF), Name: "icr.argument");
4519
4520 // If we need to copy, then the load has to be conditional, which
4521 // means we need control flow.
4522 if (shouldCopy) {
4523 originBB = CGF.Builder.GetInsertBlock();
4524 contBB = CGF.createBasicBlock(name: "icr.cont");
4525 llvm::BasicBlock *copyBB = CGF.createBasicBlock(name: "icr.copy");
4526 CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: copyBB);
4527 CGF.EmitBlock(BB: copyBB);
4528 condEval.begin(CGF);
4529 }
4530 }
4531
4532 llvm::Value *valueToUse = nullptr;
4533
4534 // Perform a copy if necessary.
4535 if (shouldCopy) {
4536 RValue srcRV = CGF.EmitLoadOfLValue(V: srcLV, Loc: SourceLocation());
4537 assert(srcRV.isScalar());
4538
4539 llvm::Value *src = srcRV.getScalarVal();
4540 src = CGF.Builder.CreateBitCast(V: src, DestTy: destElemType, Name: "icr.cast");
4541
4542 // Use an ordinary store, not a store-to-lvalue.
4543 CGF.Builder.CreateStore(Val: src, Addr: temp);
4544
4545 // If optimization is enabled, and the value was held in a
4546 // __strong variable, we need to tell the optimizer that this
4547 // value has to stay alive until we're doing the store back.
4548 // This is because the temporary is effectively unretained,
4549 // and so otherwise we can violate the high-level semantics.
4550 if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4551 srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {
4552 valueToUse = src;
4553 }
4554 }
4555
4556 // Finish the control flow if we needed it.
4557 if (shouldCopy && !provablyNonNull) {
4558 llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
4559 CGF.EmitBlock(BB: contBB);
4560
4561 // Make a phi for the value to intrinsically use.
4562 if (valueToUse) {
4563 llvm::PHINode *phiToUse =
4564 CGF.Builder.CreatePHI(Ty: valueToUse->getType(), NumReservedValues: 2, Name: "icr.to-use");
4565 phiToUse->addIncoming(V: valueToUse, BB: copyBB);
4566 phiToUse->addIncoming(V: llvm::PoisonValue::get(T: valueToUse->getType()),
4567 BB: originBB);
4568 valueToUse = phiToUse;
4569 }
4570
4571 condEval.end(CGF);
4572 }
4573
4574 args.addWriteback(srcLV, temporary: temp, toUse: valueToUse);
4575 args.add(rvalue: RValue::get(V: finalArgument), type: CRE->getType());
4576}
4577
4578void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
4579 assert(!StackBase);
4580
4581 // Save the stack.
4582 StackBase = CGF.Builder.CreateStackSave(Name: "inalloca.save");
4583}
4584
4585void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
4586 if (StackBase) {
4587 // Restore the stack after the call.
4588 CGF.Builder.CreateStackRestore(Ptr: StackBase);
4589 }
4590}
4591
4592void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
4593 SourceLocation ArgLoc,
4594 AbstractCallee AC, unsigned ParmNum) {
4595 if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
4596 SanOpts.has(K: SanitizerKind::NullabilityArg)))
4597 return;
4598
4599 // The param decl may be missing in a variadic function.
4600 auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(I: ParmNum) : nullptr;
4601 unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
4602
4603 // Prefer the nonnull attribute if it's present.
4604 const NonNullAttr *NNAttr = nullptr;
4605 if (SanOpts.has(K: SanitizerKind::NonnullAttribute))
4606 NNAttr = getNonNullAttr(FD: AC.getDecl(), PVD, ArgType, ArgNo);
4607
4608 bool CanCheckNullability = false;
4609 if (SanOpts.has(K: SanitizerKind::NullabilityArg) && !NNAttr && PVD &&
4610 !PVD->getType()->isRecordType()) {
4611 auto Nullability = PVD->getType()->getNullability();
4612 CanCheckNullability = Nullability &&
4613 *Nullability == NullabilityKind::NonNull &&
4614 PVD->getTypeSourceInfo();
4615 }
4616
4617 if (!NNAttr && !CanCheckNullability)
4618 return;
4619
4620 SourceLocation AttrLoc;
4621 SanitizerKind::SanitizerOrdinal CheckKind;
4622 SanitizerHandler Handler;
4623 if (NNAttr) {
4624 AttrLoc = NNAttr->getLocation();
4625 CheckKind = SanitizerKind::SO_NonnullAttribute;
4626 Handler = SanitizerHandler::NonnullArg;
4627 } else {
4628 AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4629 CheckKind = SanitizerKind::SO_NullabilityArg;
4630 Handler = SanitizerHandler::NullabilityArg;
4631 }
4632
4633 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4634 llvm::Value *Cond = EmitNonNullRValueCheck(RV, T: ArgType);
4635 llvm::Constant *StaticData[] = {
4636 EmitCheckSourceLocation(Loc: ArgLoc),
4637 EmitCheckSourceLocation(Loc: AttrLoc),
4638 llvm::ConstantInt::get(Ty: Int32Ty, V: ArgNo + 1),
4639 };
4640 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: {});
4641}
4642
4643void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType,
4644 SourceLocation ArgLoc,
4645 AbstractCallee AC, unsigned ParmNum) {
4646 if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
4647 SanOpts.has(K: SanitizerKind::NullabilityArg)))
4648 return;
4649
4650 EmitNonNullArgCheck(RV: RValue::get(Addr, CGF&: *this), ArgType, ArgLoc, AC, ParmNum);
4651}
4652
4653// Check if the call is going to use the inalloca convention. This needs to
4654// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
4655// later, so we can't check it directly.
4656static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
4657 ArrayRef<QualType> ArgTypes) {
4658 // The Swift calling conventions don't go through the target-specific
4659 // argument classification, they never use inalloca.
4660 // TODO: Consider limiting inalloca use to only calling conventions supported
4661 // by MSVC.
4662 if (ExplicitCC == CC_Swift || ExplicitCC == CC_SwiftAsync)
4663 return false;
4664 if (!CGM.getTarget().getCXXABI().isMicrosoft())
4665 return false;
4666 return llvm::any_of(Range&: ArgTypes, P: [&](QualType Ty) {
4667 return isInAllocaArgument(ABI&: CGM.getCXXABI(), type: Ty);
4668 });
4669}
4670
4671#ifndef NDEBUG
4672// Determine whether the given argument is an Objective-C method
4673// that may have type parameters in its signature.
4674static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
4675 const DeclContext *dc = method->getDeclContext();
4676 if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {
4677 return classDecl->getTypeParamListAsWritten();
4678 }
4679
4680 if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
4681 return catDecl->getTypeParamList();
4682 }
4683
4684 return false;
4685}
4686#endif
4687
4688/// EmitCallArgs - Emit call arguments for a function.
4689void CodeGenFunction::EmitCallArgs(
4690 CallArgList &Args, PrototypeWrapper Prototype,
4691 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4692 AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
4693 SmallVector<QualType, 16> ArgTypes;
4694
4695 assert((ParamsToSkip == 0 || Prototype.P) &&
4696 "Can't skip parameters if type info is not provided");
4697
4698 // This variable only captures *explicitly* written conventions, not those
4699 // applied by default via command line flags or target defaults, such as
4700 // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
4701 // require knowing if this is a C++ instance method or being able to see
4702 // unprototyped FunctionTypes.
4703 CallingConv ExplicitCC = CC_C;
4704
4705 // First, if a prototype was provided, use those argument types.
4706 bool IsVariadic = false;
4707 if (Prototype.P) {
4708 const auto *MD = dyn_cast<const ObjCMethodDecl *>(Val&: Prototype.P);
4709 if (MD) {
4710 IsVariadic = MD->isVariadic();
4711 ExplicitCC = getCallingConventionForDecl(
4712 D: MD, IsTargetDefaultMSABI: CGM.getTarget().getTriple().isOSWindows());
4713 ArgTypes.assign(in_start: MD->param_type_begin() + ParamsToSkip,
4714 in_end: MD->param_type_end());
4715 } else {
4716 const auto *FPT = cast<const FunctionProtoType *>(Val&: Prototype.P);
4717 IsVariadic = FPT->isVariadic();
4718 ExplicitCC = FPT->getExtInfo().getCC();
4719 ArgTypes.assign(in_start: FPT->param_type_begin() + ParamsToSkip,
4720 in_end: FPT->param_type_end());
4721 }
4722
4723#ifndef NDEBUG
4724 // Check that the prototyped types match the argument expression types.
4725 bool isGenericMethod = MD && isObjCMethodWithTypeParams(MD);
4726 CallExpr::const_arg_iterator Arg = ArgRange.begin();
4727 for (QualType Ty : ArgTypes) {
4728 assert(Arg != ArgRange.end() && "Running over edge of argument list!");
4729 assert(
4730 (isGenericMethod || Ty->isVariablyModifiedType() ||
4731 Ty.getNonReferenceType()->isObjCRetainableType() ||
4732 getContext()
4733 .getCanonicalType(Ty.getNonReferenceType())
4734 .getTypePtr() ==
4735 getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) &&
4736 "type mismatch in call argument!");
4737 ++Arg;
4738 }
4739
4740 // Either we've emitted all the call args, or we have a call to variadic
4741 // function.
4742 assert((Arg == ArgRange.end() || IsVariadic) &&
4743 "Extra arguments in non-variadic function!");
4744#endif
4745 }
4746
4747 // If we still have any arguments, emit them using the type of the argument.
4748 for (auto *A : llvm::drop_begin(RangeOrContainer&: ArgRange, N: ArgTypes.size()))
4749 ArgTypes.push_back(Elt: IsVariadic ? getVarArgType(Arg: A) : A->getType());
4750 assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
4751
4752 // We must evaluate arguments from right to left in the MS C++ ABI,
4753 // because arguments are destroyed left to right in the callee. As a special
4754 // case, there are certain language constructs that require left-to-right
4755 // evaluation, and in those cases we consider the evaluation order requirement
4756 // to trump the "destruction order is reverse construction order" guarantee.
4757 bool LeftToRight =
4758 CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
4759 ? Order == EvaluationOrder::ForceLeftToRight
4760 : Order != EvaluationOrder::ForceRightToLeft;
4761
4762 auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
4763 RValue EmittedArg) {
4764 if (!AC.hasFunctionDecl() || I >= AC.getNumParams())
4765 return;
4766 auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
4767 if (PS == nullptr)
4768 return;
4769
4770 const auto &Context = getContext();
4771 auto SizeTy = Context.getSizeType();
4772 auto T = Builder.getIntNTy(N: Context.getTypeSize(T: SizeTy));
4773 assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
4774 llvm::Value *V = evaluateOrEmitBuiltinObjectSize(
4775 E: Arg, Type: PS->getType(), ResType: T, EmittedE: EmittedArg.getScalarVal(), IsDynamic: PS->isDynamic());
4776 Args.add(rvalue: RValue::get(V), type: SizeTy);
4777 // If we're emitting args in reverse, be sure to do so with
4778 // pass_object_size, as well.
4779 if (!LeftToRight)
4780 std::swap(a&: Args.back(), b&: *(&Args.back() - 1));
4781 };
4782
4783 // Insert a stack save if we're going to need any inalloca args.
4784 if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
4785 assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
4786 "inalloca only supported on x86");
4787 Args.allocateArgumentMemory(CGF&: *this);
4788 }
4789
4790 // Evaluate each argument in the appropriate order.
4791 size_t CallArgsStart = Args.size();
4792 for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
4793 unsigned Idx = LeftToRight ? I : E - I - 1;
4794 CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
4795 unsigned InitialArgSize = Args.size();
4796 // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
4797 // the argument and parameter match or the objc method is parameterized.
4798 assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
4799 getContext().hasSameUnqualifiedType((*Arg)->getType(),
4800 ArgTypes[Idx]) ||
4801 (isa<ObjCMethodDecl>(AC.getDecl()) &&
4802 isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
4803 "Argument and parameter types don't match");
4804 EmitCallArg(args&: Args, E: *Arg, ArgType: ArgTypes[Idx]);
4805 // In particular, we depend on it being the last arg in Args, and the
4806 // objectsize bits depend on there only being one arg if !LeftToRight.
4807 assert(InitialArgSize + 1 == Args.size() &&
4808 "The code below depends on only adding one arg per EmitCallArg");
4809 (void)InitialArgSize;
4810 // Since pointer argument are never emitted as LValue, it is safe to emit
4811 // non-null argument check for r-value only.
4812 if (!Args.back().hasLValue()) {
4813 RValue RVArg = Args.back().getKnownRValue();
4814 EmitNonNullArgCheck(RV: RVArg, ArgType: ArgTypes[Idx], ArgLoc: (*Arg)->getExprLoc(), AC,
4815 ParmNum: ParamsToSkip + Idx);
4816 // @llvm.objectsize should never have side-effects and shouldn't need
4817 // destruction/cleanups, so we can safely "emit" it after its arg,
4818 // regardless of right-to-leftness
4819 MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
4820 }
4821 }
4822
4823 if (!LeftToRight) {
4824 // Un-reverse the arguments we just evaluated so they match up with the LLVM
4825 // IR function.
4826 std::reverse(first: Args.begin() + CallArgsStart, last: Args.end());
4827
4828 // Reverse the writebacks to match the MSVC ABI.
4829 Args.reverseWritebacks();
4830 }
4831}
4832
4833namespace {
4834
4835struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
4836 DestroyUnpassedArg(Address Addr, QualType Ty) : Addr(Addr), Ty(Ty) {}
4837
4838 Address Addr;
4839 QualType Ty;
4840
4841 void Emit(CodeGenFunction &CGF, Flags flags) override {
4842 QualType::DestructionKind DtorKind = Ty.isDestructedType();
4843 if (DtorKind == QualType::DK_cxx_destructor) {
4844 const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
4845 assert(!Dtor->isTrivial());
4846 CGF.EmitCXXDestructorCall(D: Dtor, Type: Dtor_Complete, /*for vbase*/ ForVirtualBase: false,
4847 /*Delegating=*/false, This: Addr, ThisTy: Ty);
4848 } else {
4849 CGF.callCStructDestructor(Dst: CGF.MakeAddrLValue(Addr, T: Ty));
4850 }
4851 }
4852};
4853
4854} // end anonymous namespace
4855
4856RValue CallArg::getRValue(CodeGenFunction &CGF) const {
4857 if (!HasLV)
4858 return RV;
4859 LValue Copy = CGF.MakeAddrLValue(Addr: CGF.CreateMemTemp(T: Ty), T: Ty);
4860 CGF.EmitAggregateCopy(Dest: Copy, Src: LV, EltTy: Ty, MayOverlap: AggValueSlot::DoesNotOverlap,
4861 isVolatile: LV.isVolatile());
4862 IsUsed = true;
4863 return RValue::getAggregate(addr: Copy.getAddress());
4864}
4865
4866void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
4867 LValue Dst = CGF.MakeAddrLValue(Addr, T: Ty);
4868 if (!HasLV && RV.isScalar())
4869 CGF.EmitStoreOfScalar(value: RV.getScalarVal(), lvalue: Dst, /*isInit=*/true);
4870 else if (!HasLV && RV.isComplex())
4871 CGF.EmitStoreOfComplex(V: RV.getComplexVal(), dest: Dst, /*init=*/isInit: true);
4872 else {
4873 auto Addr = HasLV ? LV.getAddress() : RV.getAggregateAddress();
4874 LValue SrcLV = CGF.MakeAddrLValue(Addr, T: Ty);
4875 // We assume that call args are never copied into subobjects.
4876 CGF.EmitAggregateCopy(Dest: Dst, Src: SrcLV, EltTy: Ty, MayOverlap: AggValueSlot::DoesNotOverlap,
4877 isVolatile: HasLV ? LV.isVolatileQualified()
4878 : RV.isVolatileQualified());
4879 }
4880 IsUsed = true;
4881}
4882
4883void CodeGenFunction::EmitWritebacks(const CallArgList &args) {
4884 for (const auto &I : args.writebacks())
4885 emitWriteback(CGF&: *this, writeback: I);
4886}
4887
4888void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
4889 QualType type) {
4890 std::optional<DisableDebugLocationUpdates> Dis;
4891 if (isa<CXXDefaultArgExpr>(Val: E))
4892 Dis.emplace(args&: *this);
4893 if (const ObjCIndirectCopyRestoreExpr *CRE =
4894 dyn_cast<ObjCIndirectCopyRestoreExpr>(Val: E)) {
4895 assert(getLangOpts().ObjCAutoRefCount);
4896 return emitWritebackArg(CGF&: *this, args, CRE);
4897 }
4898
4899 // Add writeback for HLSLOutParamExpr.
4900 // Needs to be before the assert below because HLSLOutArgExpr is an LValue
4901 // and is not a reference.
4902 if (const HLSLOutArgExpr *OE = dyn_cast<HLSLOutArgExpr>(Val: E)) {
4903 EmitHLSLOutArgExpr(E: OE, Args&: args, Ty: type);
4904 return;
4905 }
4906
4907 assert(type->isReferenceType() == E->isGLValue() &&
4908 "reference binding to unmaterialized r-value!");
4909
4910 if (E->isGLValue()) {
4911 assert(E->getObjectKind() == OK_Ordinary);
4912 return args.add(rvalue: EmitReferenceBindingToExpr(E), type);
4913 }
4914
4915 bool HasAggregateEvalKind = hasAggregateEvaluationKind(T: type);
4916
4917 // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
4918 // However, we still have to push an EH-only cleanup in case we unwind before
4919 // we make it to the call.
4920 if (type->isRecordType() &&
4921 type->castAsRecordDecl()->isParamDestroyedInCallee()) {
4922 // If we're using inalloca, use the argument memory. Otherwise, use a
4923 // temporary.
4924 AggValueSlot Slot = args.isUsingInAlloca()
4925 ? createPlaceholderSlot(CGF&: *this, Ty: type)
4926 : CreateAggTemp(T: type, Name: "agg.tmp");
4927
4928 bool DestroyedInCallee = true, NeedsCleanup = true;
4929 if (const auto *RD = type->getAsCXXRecordDecl())
4930 DestroyedInCallee = RD->hasNonTrivialDestructor();
4931 else
4932 NeedsCleanup = type.isDestructedType();
4933
4934 if (DestroyedInCallee)
4935 Slot.setExternallyDestructed();
4936
4937 EmitAggExpr(E, AS: Slot);
4938 RValue RV = Slot.asRValue();
4939 args.add(rvalue: RV, type);
4940
4941 if (DestroyedInCallee && NeedsCleanup) {
4942 // Create a no-op GEP between the placeholder and the cleanup so we can
4943 // RAUW it successfully. It also serves as a marker of the first
4944 // instruction where the cleanup is active.
4945 pushFullExprCleanup<DestroyUnpassedArg>(kind: NormalAndEHCleanup,
4946 A: Slot.getAddress(), A: type);
4947 // This unreachable is a temporary marker which will be removed later.
4948 llvm::Instruction *IsActive =
4949 Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
4950 args.addArgCleanupDeactivation(Cleanup: EHStack.stable_begin(), IsActiveIP: IsActive);
4951 }
4952 return;
4953 }
4954
4955 if (HasAggregateEvalKind && isa<ImplicitCastExpr>(Val: E) &&
4956 cast<CastExpr>(Val: E)->getCastKind() == CK_LValueToRValue &&
4957 !type->isArrayParameterType() && !type.isNonTrivialToPrimitiveCopy()) {
4958 LValue L = EmitLValue(E: cast<CastExpr>(Val: E)->getSubExpr());
4959 assert(L.isSimple());
4960 args.addUncopiedAggregate(LV: L, type);
4961 return;
4962 }
4963
4964 args.add(rvalue: EmitAnyExprToTemp(E), type);
4965}
4966
4967QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
4968 // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
4969 // implicitly widens null pointer constants that are arguments to varargs
4970 // functions to pointer-sized ints.
4971 if (!getTarget().getTriple().isOSWindows())
4972 return Arg->getType();
4973
4974 if (Arg->getType()->isIntegerType() &&
4975 getContext().getTypeSize(T: Arg->getType()) <
4976 getContext().getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default) &&
4977 Arg->isNullPointerConstant(Ctx&: getContext(),
4978 NPC: Expr::NPC_ValueDependentIsNotNull)) {
4979 return getContext().getIntPtrType();
4980 }
4981
4982 return Arg->getType();
4983}
4984
4985// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4986// optimizer it can aggressively ignore unwind edges.
4987void CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
4988 if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4989 !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
4990 Inst->setMetadata(Kind: "clang.arc.no_objc_arc_exceptions",
4991 Node: CGM.getNoObjCARCExceptionsMetadata());
4992}
4993
4994/// Emits a call to the given no-arguments nounwind runtime function.
4995llvm::CallInst *
4996CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4997 const llvm::Twine &name) {
4998 return EmitNounwindRuntimeCall(callee, args: ArrayRef<llvm::Value *>(), name);
4999}
5000
5001/// Emits a call to the given nounwind runtime function.
5002llvm::CallInst *
5003CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5004 ArrayRef<Address> args,
5005 const llvm::Twine &name) {
5006 SmallVector<llvm::Value *, 3> values;
5007 for (auto arg : args)
5008 values.push_back(Elt: arg.emitRawPointer(CGF&: *this));
5009 return EmitNounwindRuntimeCall(callee, args: values, name);
5010}
5011
5012llvm::CallInst *
5013CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5014 ArrayRef<llvm::Value *> args,
5015 const llvm::Twine &name) {
5016 llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
5017 call->setDoesNotThrow();
5018 return call;
5019}
5020
5021/// Emits a simple call (never an invoke) to the given no-arguments
5022/// runtime function.
5023llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5024 const llvm::Twine &name) {
5025 return EmitRuntimeCall(callee, args: {}, name);
5026}
5027
5028// Calls which may throw must have operand bundles indicating which funclet
5029// they are nested within.
5030SmallVector<llvm::OperandBundleDef, 1>
5031CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
5032 // There is no need for a funclet operand bundle if we aren't inside a
5033 // funclet.
5034 if (!CurrentFuncletPad)
5035 return (SmallVector<llvm::OperandBundleDef, 1>());
5036
5037 // Skip intrinsics which cannot throw (as long as they don't lower into
5038 // regular function calls in the course of IR transformations).
5039 if (auto *CalleeFn = dyn_cast<llvm::Function>(Val: Callee->stripPointerCasts())) {
5040 if (CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) {
5041 auto IID = CalleeFn->getIntrinsicID();
5042 if (!llvm::IntrinsicInst::mayLowerToFunctionCall(IID))
5043 return (SmallVector<llvm::OperandBundleDef, 1>());
5044 }
5045 }
5046
5047 SmallVector<llvm::OperandBundleDef, 1> BundleList;
5048 BundleList.emplace_back(Args: "funclet", Args&: CurrentFuncletPad);
5049 return BundleList;
5050}
5051
5052/// Emits a simple call (never an invoke) to the given runtime function.
5053llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5054 ArrayRef<llvm::Value *> args,
5055 const llvm::Twine &name) {
5056 llvm::CallInst *call = Builder.CreateCall(
5057 Callee: callee, Args: args, OpBundles: getBundlesForFunclet(Callee: callee.getCallee()), Name: name);
5058 call->setCallingConv(getRuntimeCC());
5059
5060 if (CGM.shouldEmitConvergenceTokens() && call->isConvergent())
5061 return cast<llvm::CallInst>(Val: addConvergenceControlToken(Input: call));
5062 return call;
5063}
5064
5065/// Emits a call or invoke to the given noreturn runtime function.
5066void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
5067 llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
5068 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5069 getBundlesForFunclet(Callee: callee.getCallee());
5070
5071 if (getInvokeDest()) {
5072 llvm::InvokeInst *invoke = Builder.CreateInvoke(
5073 Callee: callee, NormalDest: getUnreachableBlock(), UnwindDest: getInvokeDest(), Args: args, OpBundles: BundleList);
5074 invoke->setDoesNotReturn();
5075 invoke->setCallingConv(getRuntimeCC());
5076 } else {
5077 llvm::CallInst *call = Builder.CreateCall(Callee: callee, Args: args, OpBundles: BundleList);
5078 call->setDoesNotReturn();
5079 call->setCallingConv(getRuntimeCC());
5080 Builder.CreateUnreachable();
5081 }
5082}
5083
5084/// Emits a call or invoke instruction to the given nullary runtime function.
5085llvm::CallBase *
5086CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5087 const Twine &name) {
5088 return EmitRuntimeCallOrInvoke(callee, args: {}, name);
5089}
5090
5091/// Emits a call or invoke instruction to the given runtime function.
5092llvm::CallBase *
5093CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5094 ArrayRef<llvm::Value *> args,
5095 const Twine &name) {
5096 llvm::CallBase *call = EmitCallOrInvoke(Callee: callee, Args: args, Name: name);
5097 call->setCallingConv(getRuntimeCC());
5098 return call;
5099}
5100
5101/// Emits a call or invoke instruction to the given function, depending
5102/// on the current state of the EH stack.
5103llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
5104 ArrayRef<llvm::Value *> Args,
5105 const Twine &Name) {
5106 llvm::BasicBlock *InvokeDest = getInvokeDest();
5107 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5108 getBundlesForFunclet(Callee: Callee.getCallee());
5109
5110 llvm::CallBase *Inst;
5111 if (!InvokeDest)
5112 Inst = Builder.CreateCall(Callee, Args, OpBundles: BundleList, Name);
5113 else {
5114 llvm::BasicBlock *ContBB = createBasicBlock(name: "invoke.cont");
5115 Inst = Builder.CreateInvoke(Callee, NormalDest: ContBB, UnwindDest: InvokeDest, Args, OpBundles: BundleList,
5116 Name);
5117 EmitBlock(BB: ContBB);
5118 }
5119
5120 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5121 // optimizer it can aggressively ignore unwind edges.
5122 if (CGM.getLangOpts().ObjCAutoRefCount)
5123 AddObjCARCExceptionMetadata(Inst);
5124
5125 return Inst;
5126}
5127
5128void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
5129 llvm::Value *New) {
5130 DeferredReplacements.push_back(
5131 Elt: std::make_pair(x: llvm::WeakTrackingVH(Old), y&: New));
5132}
5133
5134namespace {
5135
5136/// Specify given \p NewAlign as the alignment of return value attribute. If
5137/// such attribute already exists, re-set it to the maximal one of two options.
5138[[nodiscard]] llvm::AttributeList
5139maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
5140 const llvm::AttributeList &Attrs,
5141 llvm::Align NewAlign) {
5142 llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
5143 if (CurAlign >= NewAlign)
5144 return Attrs;
5145 llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Context&: Ctx, Alignment: NewAlign);
5146 return Attrs.removeRetAttribute(C&: Ctx, Kind: llvm::Attribute::AttrKind::Alignment)
5147 .addRetAttribute(C&: Ctx, Attr: AlignAttr);
5148}
5149
5150template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
5151protected:
5152 CodeGenFunction &CGF;
5153
5154 /// We do nothing if this is, or becomes, nullptr.
5155 const AlignedAttrTy *AA = nullptr;
5156
5157 llvm::Value *Alignment = nullptr; // May or may not be a constant.
5158 llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
5159
5160 AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5161 : CGF(CGF_) {
5162 if (!FuncDecl)
5163 return;
5164 AA = FuncDecl->getAttr<AlignedAttrTy>();
5165 }
5166
5167public:
5168 /// If we can, materialize the alignment as an attribute on return value.
5169 [[nodiscard]] llvm::AttributeList
5170 TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
5171 if (!AA || OffsetCI || CGF.SanOpts.has(K: SanitizerKind::Alignment))
5172 return Attrs;
5173 const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Val: Alignment);
5174 if (!AlignmentCI)
5175 return Attrs;
5176 // We may legitimately have non-power-of-2 alignment here.
5177 // If so, this is UB land, emit it via `@llvm.assume` instead.
5178 if (!AlignmentCI->getValue().isPowerOf2())
5179 return Attrs;
5180 llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
5181 Ctx&: CGF.getLLVMContext(), Attrs,
5182 NewAlign: llvm::Align(
5183 AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment)));
5184 AA = nullptr; // We're done. Disallow doing anything else.
5185 return NewAttrs;
5186 }
5187
5188 /// Emit alignment assumption.
5189 /// This is a general fallback that we take if either there is an offset,
5190 /// or the alignment is variable or we are sanitizing for alignment.
5191 void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
5192 if (!AA)
5193 return;
5194 CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
5195 AA->getLocation(), Alignment, OffsetCI);
5196 AA = nullptr; // We're done. Disallow doing anything else.
5197 }
5198};
5199
5200/// Helper data structure to emit `AssumeAlignedAttr`.
5201class AssumeAlignedAttrEmitter final
5202 : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
5203public:
5204 AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5205 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5206 if (!AA)
5207 return;
5208 // It is guaranteed that the alignment/offset are constants.
5209 Alignment = cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: AA->getAlignment()));
5210 if (Expr *Offset = AA->getOffset()) {
5211 OffsetCI = cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: Offset));
5212 if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
5213 OffsetCI = nullptr;
5214 }
5215 }
5216};
5217
5218/// Helper data structure to emit `AllocAlignAttr`.
5219class AllocAlignAttrEmitter final
5220 : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
5221public:
5222 AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
5223 const CallArgList &CallArgs)
5224 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5225 if (!AA)
5226 return;
5227 // Alignment may or may not be a constant, and that is okay.
5228 Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
5229 .getRValue(CGF)
5230 .getScalarVal();
5231 }
5232};
5233
5234} // namespace
5235
5236static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
5237 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Ty))
5238 return VT->getPrimitiveSizeInBits().getKnownMinValue();
5239 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty))
5240 return getMaxVectorWidth(Ty: AT->getElementType());
5241
5242 unsigned MaxVectorWidth = 0;
5243 if (auto *ST = dyn_cast<llvm::StructType>(Val: Ty))
5244 for (auto *I : ST->elements())
5245 MaxVectorWidth = std::max(a: MaxVectorWidth, b: getMaxVectorWidth(Ty: I));
5246 return MaxVectorWidth;
5247}
5248
5249RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
5250 const CGCallee &Callee,
5251 ReturnValueSlot ReturnValue,
5252 const CallArgList &CallArgs,
5253 llvm::CallBase **callOrInvoke, bool IsMustTail,
5254 SourceLocation Loc,
5255 bool IsVirtualFunctionPointerThunk) {
5256 // FIXME: We no longer need the types from CallArgs; lift up and simplify.
5257
5258 assert(Callee.isOrdinary() || Callee.isVirtual());
5259
5260 // Handle struct-return functions by passing a pointer to the
5261 // location that we would like to return into.
5262 QualType RetTy = CallInfo.getReturnType();
5263 const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
5264
5265 llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(FI: CallInfo);
5266
5267 const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
5268 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
5269 // We can only guarantee that a function is called from the correct
5270 // context/function based on the appropriate target attributes,
5271 // so only check in the case where we have both always_inline and target
5272 // since otherwise we could be making a conditional call after a check for
5273 // the proper cpu features (and it won't cause code generation issues due to
5274 // function based code generation).
5275 if ((TargetDecl->hasAttr<AlwaysInlineAttr>() &&
5276 (TargetDecl->hasAttr<TargetAttr>() ||
5277 (CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>()))) ||
5278 (CurFuncDecl && CurFuncDecl->hasAttr<FlattenAttr>() &&
5279 (CurFuncDecl->hasAttr<TargetAttr>() ||
5280 TargetDecl->hasAttr<TargetAttr>())))
5281 checkTargetFeatures(Loc, TargetDecl: FD);
5282 }
5283
5284 // Some architectures (such as x86-64) have the ABI changed based on
5285 // attribute-target/features. Give them a chance to diagnose.
5286 const FunctionDecl *CallerDecl = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
5287 const FunctionDecl *CalleeDecl = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
5288 CGM.getTargetCodeGenInfo().checkFunctionCallABI(CGM, CallLoc: Loc, Caller: CallerDecl,
5289 Callee: CalleeDecl, Args: CallArgs, ReturnType: RetTy);
5290
5291 // 1. Set up the arguments.
5292
5293 // If we're using inalloca, insert the allocation after the stack save.
5294 // FIXME: Do this earlier rather than hacking it in here!
5295 RawAddress ArgMemory = RawAddress::invalid();
5296 if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
5297 const llvm::DataLayout &DL = CGM.getDataLayout();
5298 llvm::Instruction *IP = CallArgs.getStackBase();
5299 llvm::AllocaInst *AI;
5300 if (IP) {
5301 IP = IP->getNextNode();
5302 AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(), "argmem",
5303 IP->getIterator());
5304 } else {
5305 AI = CreateTempAlloca(Ty: ArgStruct, Name: "argmem");
5306 }
5307 auto Align = CallInfo.getArgStructAlignment();
5308 AI->setAlignment(Align.getAsAlign());
5309 AI->setUsedWithInAlloca(true);
5310 assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
5311 ArgMemory = RawAddress(AI, ArgStruct, Align);
5312 }
5313
5314 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
5315 SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
5316
5317 // If the call returns a temporary with struct return, create a temporary
5318 // alloca to hold the result, unless one is given to us.
5319 Address SRetPtr = Address::invalid();
5320 bool NeedSRetLifetimeEnd = false;
5321 if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
5322 // For virtual function pointer thunks and musttail calls, we must always
5323 // forward an incoming SRet pointer to the callee, because a local alloca
5324 // would be de-allocated before the call. These cases both guarantee that
5325 // there will be an incoming SRet argument of the correct type.
5326 if ((IsVirtualFunctionPointerThunk || IsMustTail) && RetAI.isIndirect()) {
5327 SRetPtr = makeNaturalAddressForPointer(Ptr: CurFn->arg_begin() +
5328 IRFunctionArgs.getSRetArgNo(),
5329 T: RetTy, Alignment: CharUnits::fromQuantity(Quantity: 1));
5330 } else if (!ReturnValue.isNull()) {
5331 SRetPtr = ReturnValue.getAddress();
5332 } else {
5333 SRetPtr = CreateMemTempWithoutCast(T: RetTy, Name: "tmp");
5334 if (HaveInsertPoint() && ReturnValue.isUnused())
5335 NeedSRetLifetimeEnd = EmitLifetimeStart(Addr: SRetPtr.getBasePointer());
5336 }
5337 if (IRFunctionArgs.hasSRetArg()) {
5338 // A mismatch between the allocated return value's AS and the target's
5339 // chosen IndirectAS can happen e.g. when passing the this pointer through
5340 // a chain involving stores to / loads from the DefaultAS; we address this
5341 // here, symmetrically with the handling we have for normal pointer args.
5342 if (SRetPtr.getAddressSpace() != RetAI.getIndirectAddrSpace()) {
5343 llvm::Value *V = SRetPtr.getBasePointer();
5344 LangAS SAS = getLangASFromTargetAS(TargetAS: SRetPtr.getAddressSpace());
5345 llvm::Type *Ty = llvm::PointerType::get(C&: getLLVMContext(),
5346 AddressSpace: RetAI.getIndirectAddrSpace());
5347
5348 SRetPtr = SRetPtr.withPointer(
5349 NewPointer: getTargetHooks().performAddrSpaceCast(CGF&: *this, V, SrcAddr: SAS, DestTy: Ty, IsNonNull: true),
5350 IsKnownNonNull: SRetPtr.isKnownNonNull());
5351 }
5352 IRCallArgs[IRFunctionArgs.getSRetArgNo()] =
5353 getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy);
5354 } else if (RetAI.isInAlloca()) {
5355 Address Addr =
5356 Builder.CreateStructGEP(Addr: ArgMemory, Index: RetAI.getInAllocaFieldIndex());
5357 Builder.CreateStore(Val: getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy), Addr);
5358 }
5359 }
5360
5361 RawAddress swiftErrorTemp = RawAddress::invalid();
5362 Address swiftErrorArg = Address::invalid();
5363
5364 // When passing arguments using temporary allocas, we need to add the
5365 // appropriate lifetime markers. This vector keeps track of all the lifetime
5366 // markers that need to be ended right after the call.
5367 SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
5368
5369 // Translate all of the arguments as necessary to match the IR lowering.
5370 assert(CallInfo.arg_size() == CallArgs.size() &&
5371 "Mismatch between function signature & arguments.");
5372 unsigned ArgNo = 0;
5373 CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
5374 for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
5375 I != E; ++I, ++info_it, ++ArgNo) {
5376 const ABIArgInfo &ArgInfo = info_it->info;
5377
5378 // Insert a padding argument to ensure proper alignment.
5379 if (IRFunctionArgs.hasPaddingArg(ArgNo))
5380 IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
5381 llvm::UndefValue::get(T: ArgInfo.getPaddingType());
5382
5383 unsigned FirstIRArg, NumIRArgs;
5384 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
5385
5386 bool ArgHasMaybeUndefAttr =
5387 IsArgumentMaybeUndef(TargetDecl, NumRequiredArgs: CallInfo.getNumRequiredArgs(), ArgNo);
5388
5389 switch (ArgInfo.getKind()) {
5390 case ABIArgInfo::InAlloca: {
5391 assert(NumIRArgs == 0);
5392 assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
5393 if (I->isAggregate()) {
5394 RawAddress Addr = I->hasLValue()
5395 ? I->getKnownLValue().getAddress()
5396 : I->getKnownRValue().getAggregateAddress();
5397 llvm::Instruction *Placeholder =
5398 cast<llvm::Instruction>(Val: Addr.getPointer());
5399
5400 if (!ArgInfo.getInAllocaIndirect()) {
5401 // Replace the placeholder with the appropriate argument slot GEP.
5402 CGBuilderTy::InsertPoint IP = Builder.saveIP();
5403 Builder.SetInsertPoint(Placeholder);
5404 Addr = Builder.CreateStructGEP(Addr: ArgMemory,
5405 Index: ArgInfo.getInAllocaFieldIndex());
5406 Builder.restoreIP(IP);
5407 } else {
5408 // For indirect things such as overaligned structs, replace the
5409 // placeholder with a regular aggregate temporary alloca. Store the
5410 // address of this alloca into the struct.
5411 Addr = CreateMemTemp(T: info_it->type, Name: "inalloca.indirect.tmp");
5412 Address ArgSlot = Builder.CreateStructGEP(
5413 Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5414 Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5415 }
5416 deferPlaceholderReplacement(Old: Placeholder, New: Addr.getPointer());
5417 } else if (ArgInfo.getInAllocaIndirect()) {
5418 // Make a temporary alloca and store the address of it into the argument
5419 // struct.
5420 RawAddress Addr = CreateMemTempWithoutCast(
5421 T: I->Ty, Align: getContext().getTypeAlignInChars(T: I->Ty),
5422 Name: "indirect-arg-temp");
5423 I->copyInto(CGF&: *this, Addr);
5424 Address ArgSlot =
5425 Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5426 Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5427 } else {
5428 // Store the RValue into the argument struct.
5429 Address Addr =
5430 Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5431 Addr = Addr.withElementType(ElemTy: ConvertTypeForMem(T: I->Ty));
5432 I->copyInto(CGF&: *this, Addr);
5433 }
5434 break;
5435 }
5436
5437 case ABIArgInfo::Indirect:
5438 case ABIArgInfo::IndirectAliased: {
5439 assert(NumIRArgs == 1);
5440 if (I->isAggregate()) {
5441 // We want to avoid creating an unnecessary temporary+copy here;
5442 // however, we need one in three cases:
5443 // 1. If the argument is not byval, and we are required to copy the
5444 // source. (This case doesn't occur on any common architecture.)
5445 // 2. If the argument is byval, RV is not sufficiently aligned, and
5446 // we cannot force it to be sufficiently aligned.
5447 // 3. If the argument is byval, but RV is not located in default
5448 // or alloca address space.
5449 Address Addr = I->hasLValue()
5450 ? I->getKnownLValue().getAddress()
5451 : I->getKnownRValue().getAggregateAddress();
5452 CharUnits Align = ArgInfo.getIndirectAlign();
5453 const llvm::DataLayout *TD = &CGM.getDataLayout();
5454
5455 assert((FirstIRArg >= IRFuncTy->getNumParams() ||
5456 IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
5457 TD->getAllocaAddrSpace()) &&
5458 "indirect argument must be in alloca address space");
5459
5460 bool NeedCopy = false;
5461 if (Addr.getAlignment() < Align &&
5462 llvm::getOrEnforceKnownAlignment(V: Addr.emitRawPointer(CGF&: *this),
5463 PrefAlign: Align.getAsAlign(),
5464 DL: *TD) < Align.getAsAlign()) {
5465 NeedCopy = true;
5466 } else if (I->hasLValue()) {
5467 auto LV = I->getKnownLValue();
5468
5469 bool isByValOrRef =
5470 ArgInfo.isIndirectAliased() || ArgInfo.getIndirectByVal();
5471
5472 if (!isByValOrRef ||
5473 (LV.getAlignment() < getContext().getTypeAlignInChars(T: I->Ty))) {
5474 NeedCopy = true;
5475 }
5476
5477 if (isByValOrRef && Addr.getType()->getAddressSpace() !=
5478 ArgInfo.getIndirectAddrSpace()) {
5479 NeedCopy = true;
5480 }
5481 }
5482
5483 if (!NeedCopy) {
5484 // Skip the extra memcpy call.
5485 llvm::Value *V = getAsNaturalPointerTo(Addr, PointeeType: I->Ty);
5486 auto *T = llvm::PointerType::get(C&: CGM.getLLVMContext(),
5487 AddressSpace: ArgInfo.getIndirectAddrSpace());
5488
5489 // FIXME: This should not depend on the language address spaces, and
5490 // only the contextual values. If the address space mismatches, see if
5491 // we can look through a cast to a compatible address space value,
5492 // otherwise emit a copy.
5493 llvm::Value *Val = getTargetHooks().performAddrSpaceCast(
5494 CGF&: *this, V, SrcAddr: I->Ty.getAddressSpace(), DestTy: T, IsNonNull: true);
5495 if (ArgHasMaybeUndefAttr)
5496 Val = Builder.CreateFreeze(V: Val);
5497 IRCallArgs[FirstIRArg] = Val;
5498 break;
5499 }
5500 } else if (I->getType()->isArrayParameterType()) {
5501 // Don't produce a temporary for ArrayParameterType arguments.
5502 // ArrayParameterType arguments are only created from
5503 // HLSL_ArrayRValue casts and HLSLOutArgExpr expressions, both
5504 // of which create temporaries already. This allows us to just use the
5505 // scalar for the decayed array pointer as the argument directly.
5506 IRCallArgs[FirstIRArg] = I->getKnownRValue().getScalarVal();
5507 break;
5508 }
5509
5510 // For non-aggregate args and aggregate args meeting conditions above
5511 // we need to create an aligned temporary, and copy to it.
5512 RawAddress AI = CreateMemTempWithoutCast(
5513 T: I->Ty, Align: ArgInfo.getIndirectAlign(), Name: "byval-temp");
5514 llvm::Value *Val = getAsNaturalPointerTo(Addr: AI, PointeeType: I->Ty);
5515 if (ArgHasMaybeUndefAttr)
5516 Val = Builder.CreateFreeze(V: Val);
5517 IRCallArgs[FirstIRArg] = Val;
5518
5519 // Emit lifetime markers for the temporary alloca and add cleanup code to
5520 // emit the end lifetime marker after the call.
5521 if (EmitLifetimeStart(Addr: AI.getPointer()))
5522 CallLifetimeEndAfterCall.emplace_back(Args&: AI);
5523
5524 // Generate the copy.
5525 I->copyInto(CGF&: *this, Addr: AI);
5526 break;
5527 }
5528
5529 case ABIArgInfo::Ignore:
5530 assert(NumIRArgs == 0);
5531 break;
5532
5533 case ABIArgInfo::Extend:
5534 case ABIArgInfo::Direct: {
5535 if (!isa<llvm::StructType>(Val: ArgInfo.getCoerceToType()) &&
5536 ArgInfo.getCoerceToType() == ConvertType(T: info_it->type) &&
5537 ArgInfo.getDirectOffset() == 0) {
5538 assert(NumIRArgs == 1);
5539 llvm::Value *V;
5540 if (!I->isAggregate())
5541 V = I->getKnownRValue().getScalarVal();
5542 else
5543 V = Builder.CreateLoad(
5544 Addr: I->hasLValue() ? I->getKnownLValue().getAddress()
5545 : I->getKnownRValue().getAggregateAddress());
5546
5547 // Implement swifterror by copying into a new swifterror argument.
5548 // We'll write back in the normal path out of the call.
5549 if (CallInfo.getExtParameterInfo(argIndex: ArgNo).getABI() ==
5550 ParameterABI::SwiftErrorResult) {
5551 assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
5552
5553 QualType pointeeTy = I->Ty->getPointeeType();
5554 swiftErrorArg = makeNaturalAddressForPointer(
5555 Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
5556
5557 swiftErrorTemp =
5558 CreateMemTemp(T: pointeeTy, Align: getPointerAlign(), Name: "swifterror.temp");
5559 V = swiftErrorTemp.getPointer();
5560 cast<llvm::AllocaInst>(Val: V)->setSwiftError(true);
5561
5562 llvm::Value *errorValue = Builder.CreateLoad(Addr: swiftErrorArg);
5563 Builder.CreateStore(Val: errorValue, Addr: swiftErrorTemp);
5564 }
5565
5566 // We might have to widen integers, but we should never truncate.
5567 if (ArgInfo.getCoerceToType() != V->getType() &&
5568 V->getType()->isIntegerTy())
5569 V = Builder.CreateZExt(V, DestTy: ArgInfo.getCoerceToType());
5570
5571 // The only plausible mismatch here would be for pointer address spaces.
5572 // We assume that the target has a reasonable mapping for the DefaultAS
5573 // (it can be casted to from incoming specific ASes), and insert an AS
5574 // cast to address the mismatch.
5575 if (FirstIRArg < IRFuncTy->getNumParams() &&
5576 V->getType() != IRFuncTy->getParamType(i: FirstIRArg)) {
5577 assert(V->getType()->isPointerTy() && "Only pointers can mismatch!");
5578 auto ActualAS = I->Ty.getAddressSpace();
5579 V = getTargetHooks().performAddrSpaceCast(
5580 CGF&: *this, V, SrcAddr: ActualAS, DestTy: IRFuncTy->getParamType(i: FirstIRArg));
5581 }
5582
5583 if (ArgHasMaybeUndefAttr)
5584 V = Builder.CreateFreeze(V);
5585 IRCallArgs[FirstIRArg] = V;
5586 break;
5587 }
5588
5589 llvm::StructType *STy =
5590 dyn_cast<llvm::StructType>(Val: ArgInfo.getCoerceToType());
5591
5592 // FIXME: Avoid the conversion through memory if possible.
5593 Address Src = Address::invalid();
5594 if (!I->isAggregate()) {
5595 Src = CreateMemTemp(T: I->Ty, Name: "coerce");
5596 I->copyInto(CGF&: *this, Addr: Src);
5597 } else {
5598 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
5599 : I->getKnownRValue().getAggregateAddress();
5600 }
5601
5602 // If the value is offset in memory, apply the offset now.
5603 Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
5604
5605 // Fast-isel and the optimizer generally like scalar values better than
5606 // FCAs, so we flatten them if this is safe to do for this argument.
5607 if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
5608 llvm::Type *SrcTy = Src.getElementType();
5609 llvm::TypeSize SrcTypeSize =
5610 CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
5611 llvm::TypeSize DstTypeSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
5612 if (SrcTypeSize.isScalable()) {
5613 assert(STy->containsHomogeneousScalableVectorTypes() &&
5614 "ABI only supports structure with homogeneous scalable vector "
5615 "type");
5616 assert(SrcTypeSize == DstTypeSize &&
5617 "Only allow non-fractional movement of structure with "
5618 "homogeneous scalable vector type");
5619 assert(NumIRArgs == STy->getNumElements());
5620
5621 llvm::Value *StoredStructValue =
5622 Builder.CreateLoad(Addr: Src, Name: Src.getName() + ".tuple");
5623 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5624 llvm::Value *Extract = Builder.CreateExtractValue(
5625 Agg: StoredStructValue, Idxs: i, Name: Src.getName() + ".extract" + Twine(i));
5626 IRCallArgs[FirstIRArg + i] = Extract;
5627 }
5628 } else {
5629 uint64_t SrcSize = SrcTypeSize.getFixedValue();
5630 uint64_t DstSize = DstTypeSize.getFixedValue();
5631
5632 // If the source type is smaller than the destination type of the
5633 // coerce-to logic, copy the source value into a temp alloca the size
5634 // of the destination type to allow loading all of it. The bits past
5635 // the source value are left undef.
5636 if (SrcSize < DstSize) {
5637 Address TempAlloca = CreateTempAlloca(Ty: STy, align: Src.getAlignment(),
5638 Name: Src.getName() + ".coerce");
5639 Builder.CreateMemCpy(Dest: TempAlloca, Src, Size: SrcSize);
5640 Src = TempAlloca;
5641 } else {
5642 Src = Src.withElementType(ElemTy: STy);
5643 }
5644
5645 assert(NumIRArgs == STy->getNumElements());
5646 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5647 Address EltPtr = Builder.CreateStructGEP(Addr: Src, Index: i);
5648 llvm::Value *LI = Builder.CreateLoad(Addr: EltPtr);
5649 if (ArgHasMaybeUndefAttr)
5650 LI = Builder.CreateFreeze(V: LI);
5651 IRCallArgs[FirstIRArg + i] = LI;
5652 }
5653 }
5654 } else {
5655 // In the simple case, just pass the coerced loaded value.
5656 assert(NumIRArgs == 1);
5657 llvm::Value *Load =
5658 CreateCoercedLoad(Src, Ty: ArgInfo.getCoerceToType(), CGF&: *this);
5659
5660 if (CallInfo.isCmseNSCall()) {
5661 // For certain parameter types, clear padding bits, as they may reveal
5662 // sensitive information.
5663 // Small struct/union types are passed as integer arrays.
5664 auto *ATy = dyn_cast<llvm::ArrayType>(Val: Load->getType());
5665 if (ATy != nullptr && isa<RecordType>(Val: I->Ty.getCanonicalType()))
5666 Load = EmitCMSEClearRecord(Src: Load, ATy, QTy: I->Ty);
5667 }
5668
5669 if (ArgHasMaybeUndefAttr)
5670 Load = Builder.CreateFreeze(V: Load);
5671 IRCallArgs[FirstIRArg] = Load;
5672 }
5673
5674 break;
5675 }
5676
5677 case ABIArgInfo::CoerceAndExpand: {
5678 auto coercionType = ArgInfo.getCoerceAndExpandType();
5679 auto layout = CGM.getDataLayout().getStructLayout(Ty: coercionType);
5680 auto unpaddedCoercionType = ArgInfo.getUnpaddedCoerceAndExpandType();
5681 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
5682
5683 Address addr = Address::invalid();
5684 RawAddress AllocaAddr = RawAddress::invalid();
5685 bool NeedLifetimeEnd = false;
5686 if (I->isAggregate()) {
5687 addr = I->hasLValue() ? I->getKnownLValue().getAddress()
5688 : I->getKnownRValue().getAggregateAddress();
5689
5690 } else {
5691 RValue RV = I->getKnownRValue();
5692 assert(RV.isScalar()); // complex should always just be direct
5693
5694 llvm::Type *scalarType = RV.getScalarVal()->getType();
5695 auto scalarAlign = CGM.getDataLayout().getPrefTypeAlign(Ty: scalarType);
5696
5697 // Materialize to a temporary.
5698 addr = CreateTempAlloca(Ty: RV.getScalarVal()->getType(),
5699 align: CharUnits::fromQuantity(Quantity: std::max(
5700 a: layout->getAlignment(), b: scalarAlign)),
5701 Name: "tmp",
5702 /*ArraySize=*/nullptr, Alloca: &AllocaAddr);
5703 NeedLifetimeEnd = EmitLifetimeStart(Addr: AllocaAddr.getPointer());
5704
5705 Builder.CreateStore(Val: RV.getScalarVal(), Addr: addr);
5706 }
5707
5708 addr = addr.withElementType(ElemTy: coercionType);
5709
5710 unsigned IRArgPos = FirstIRArg;
5711 unsigned unpaddedIndex = 0;
5712 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
5713 llvm::Type *eltType = coercionType->getElementType(N: i);
5714 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
5715 continue;
5716 Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
5717 llvm::Value *elt = CreateCoercedLoad(
5718 Src: eltAddr,
5719 Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
5720 : unpaddedCoercionType,
5721 CGF&: *this);
5722 if (ArgHasMaybeUndefAttr)
5723 elt = Builder.CreateFreeze(V: elt);
5724 IRCallArgs[IRArgPos++] = elt;
5725 }
5726 assert(IRArgPos == FirstIRArg + NumIRArgs);
5727
5728 if (NeedLifetimeEnd)
5729 EmitLifetimeEnd(Addr: AllocaAddr.getPointer());
5730 break;
5731 }
5732
5733 case ABIArgInfo::Expand: {
5734 unsigned IRArgPos = FirstIRArg;
5735 ExpandTypeToArgs(Ty: I->Ty, Arg: *I, IRFuncTy, IRCallArgs, IRCallArgPos&: IRArgPos);
5736 assert(IRArgPos == FirstIRArg + NumIRArgs);
5737 break;
5738 }
5739
5740 case ABIArgInfo::TargetSpecific: {
5741 Address Src = Address::invalid();
5742 if (!I->isAggregate()) {
5743 Src = CreateMemTemp(T: I->Ty, Name: "target_coerce");
5744 I->copyInto(CGF&: *this, Addr: Src);
5745 } else {
5746 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
5747 : I->getKnownRValue().getAggregateAddress();
5748 }
5749
5750 // If the value is offset in memory, apply the offset now.
5751 Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
5752 llvm::Value *Load =
5753 CGM.getABIInfo().createCoercedLoad(SrcAddr: Src, AI: ArgInfo, CGF&: *this);
5754 IRCallArgs[FirstIRArg] = Load;
5755 break;
5756 }
5757 }
5758 }
5759
5760 const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(CGF&: *this);
5761 llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
5762
5763 // If we're using inalloca, set up that argument.
5764 if (ArgMemory.isValid()) {
5765 llvm::Value *Arg = ArgMemory.getPointer();
5766 assert(IRFunctionArgs.hasInallocaArg());
5767 IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
5768 }
5769
5770 // 2. Prepare the function pointer.
5771
5772 // If the callee is a bitcast of a non-variadic function to have a
5773 // variadic function pointer type, check to see if we can remove the
5774 // bitcast. This comes up with unprototyped functions.
5775 //
5776 // This makes the IR nicer, but more importantly it ensures that we
5777 // can inline the function at -O0 if it is marked always_inline.
5778 auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
5779 llvm::Value *Ptr) -> llvm::Function * {
5780 if (!CalleeFT->isVarArg())
5781 return nullptr;
5782
5783 // Get underlying value if it's a bitcast
5784 if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Val: Ptr)) {
5785 if (CE->getOpcode() == llvm::Instruction::BitCast)
5786 Ptr = CE->getOperand(i_nocapture: 0);
5787 }
5788
5789 llvm::Function *OrigFn = dyn_cast<llvm::Function>(Val: Ptr);
5790 if (!OrigFn)
5791 return nullptr;
5792
5793 llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
5794
5795 // If the original type is variadic, or if any of the component types
5796 // disagree, we cannot remove the cast.
5797 if (OrigFT->isVarArg() ||
5798 OrigFT->getNumParams() != CalleeFT->getNumParams() ||
5799 OrigFT->getReturnType() != CalleeFT->getReturnType())
5800 return nullptr;
5801
5802 for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i)
5803 if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
5804 return nullptr;
5805
5806 return OrigFn;
5807 };
5808
5809 if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {
5810 CalleePtr = OrigFn;
5811 IRFuncTy = OrigFn->getFunctionType();
5812 }
5813
5814 // 3. Perform the actual call.
5815
5816 // Deactivate any cleanups that we're supposed to do immediately before
5817 // the call.
5818 if (!CallArgs.getCleanupsToDeactivate().empty())
5819 deactivateArgCleanupsBeforeCall(CGF&: *this, CallArgs);
5820
5821 // Update the largest vector width if any arguments have vector types.
5822 for (unsigned i = 0; i < IRCallArgs.size(); ++i)
5823 LargestVectorWidth = std::max(a: LargestVectorWidth,
5824 b: getMaxVectorWidth(Ty: IRCallArgs[i]->getType()));
5825
5826 // Compute the calling convention and attributes.
5827 unsigned CallingConv;
5828 llvm::AttributeList Attrs;
5829 CGM.ConstructAttributeList(Name: CalleePtr->getName(), FI: CallInfo,
5830 CalleeInfo: Callee.getAbstractInfo(), AttrList&: Attrs, CallingConv,
5831 /*AttrOnCallSite=*/true,
5832 /*IsThunk=*/false);
5833
5834 if (CallingConv == llvm::CallingConv::X86_VectorCall &&
5835 getTarget().getTriple().isWindowsArm64EC()) {
5836 CGM.Error(loc: Loc, error: "__vectorcall calling convention is not currently "
5837 "supported");
5838 }
5839
5840 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
5841 if (FD->hasAttr<StrictFPAttr>())
5842 // All calls within a strictfp function are marked strictfp
5843 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::StrictFP);
5844
5845 // If -ffast-math is enabled and the function is guarded by an
5846 // '__attribute__((optnone)) adjust the memory attribute so the BE emits the
5847 // library call instead of the intrinsic.
5848 if (FD->hasAttr<OptimizeNoneAttr>() && getLangOpts().FastMath)
5849 CGM.AdjustMemoryAttribute(Name: CalleePtr->getName(), CalleeInfo: Callee.getAbstractInfo(),
5850 Attrs);
5851 }
5852 // Add call-site nomerge attribute if exists.
5853 if (InNoMergeAttributedStmt)
5854 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoMerge);
5855
5856 // Add call-site noinline attribute if exists.
5857 if (InNoInlineAttributedStmt)
5858 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoInline);
5859
5860 // Add call-site always_inline attribute if exists.
5861 // Note: This corresponds to the [[clang::always_inline]] statement attribute.
5862 if (InAlwaysInlineAttributedStmt &&
5863 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
5864 Caller: CallerDecl, Callee: CalleeDecl))
5865 Attrs =
5866 Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::AlwaysInline);
5867
5868 // Remove call-site convergent attribute if requested.
5869 if (InNoConvergentAttributedStmt)
5870 Attrs =
5871 Attrs.removeFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::Convergent);
5872
5873 // Apply some call-site-specific attributes.
5874 // TODO: work this into building the attribute set.
5875
5876 // Apply always_inline to all calls within flatten functions.
5877 // FIXME: should this really take priority over __try, below?
5878 if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
5879 !InNoInlineAttributedStmt &&
5880 !(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>()) &&
5881 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
5882 Caller: CallerDecl, Callee: CalleeDecl)) {
5883 Attrs =
5884 Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::AlwaysInline);
5885 }
5886
5887 // Disable inlining inside SEH __try blocks.
5888 if (isSEHTryScope()) {
5889 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoInline);
5890 }
5891
5892 // Decide whether to use a call or an invoke.
5893 bool CannotThrow;
5894 if (currentFunctionUsesSEHTry()) {
5895 // SEH cares about asynchronous exceptions, so everything can "throw."
5896 CannotThrow = false;
5897 } else if (isCleanupPadScope() &&
5898 EHPersonality::get(CGF&: *this).isMSVCXXPersonality()) {
5899 // The MSVC++ personality will implicitly terminate the program if an
5900 // exception is thrown during a cleanup outside of a try/catch.
5901 // We don't need to model anything in IR to get this behavior.
5902 CannotThrow = true;
5903 } else {
5904 // Otherwise, nounwind call sites will never throw.
5905 CannotThrow = Attrs.hasFnAttr(Kind: llvm::Attribute::NoUnwind);
5906
5907 if (auto *FPtr = dyn_cast<llvm::Function>(Val: CalleePtr))
5908 if (FPtr->hasFnAttribute(Kind: llvm::Attribute::NoUnwind))
5909 CannotThrow = true;
5910 }
5911
5912 // If we made a temporary, be sure to clean up after ourselves. Note that we
5913 // can't depend on being inside of an ExprWithCleanups, so we need to manually
5914 // pop this cleanup later on. Being eager about this is OK, since this
5915 // temporary is 'invisible' outside of the callee.
5916 if (NeedSRetLifetimeEnd)
5917 pushFullExprCleanup<CallLifetimeEnd>(kind: NormalEHLifetimeMarker, A: SRetPtr);
5918
5919 llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();
5920
5921 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5922 getBundlesForFunclet(Callee: CalleePtr);
5923
5924 if (SanOpts.has(K: SanitizerKind::KCFI) &&
5925 !isa_and_nonnull<FunctionDecl>(Val: TargetDecl))
5926 EmitKCFIOperandBundle(Callee: ConcreteCallee, Bundles&: BundleList);
5927
5928 // Add the pointer-authentication bundle.
5929 EmitPointerAuthOperandBundle(Info: ConcreteCallee.getPointerAuthInfo(), Bundles&: BundleList);
5930
5931 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl))
5932 if (FD->hasAttr<StrictFPAttr>())
5933 // All calls within a strictfp function are marked strictfp
5934 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::StrictFP);
5935
5936 AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
5937 Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
5938
5939 AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
5940 Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
5941
5942 // Emit the actual call/invoke instruction.
5943 llvm::CallBase *CI;
5944 if (!InvokeDest) {
5945 CI = Builder.CreateCall(FTy: IRFuncTy, Callee: CalleePtr, Args: IRCallArgs, OpBundles: BundleList);
5946 } else {
5947 llvm::BasicBlock *Cont = createBasicBlock(name: "invoke.cont");
5948 CI = Builder.CreateInvoke(Ty: IRFuncTy, Callee: CalleePtr, NormalDest: Cont, UnwindDest: InvokeDest, Args: IRCallArgs,
5949 OpBundles: BundleList);
5950 EmitBlock(BB: Cont);
5951 }
5952 if (CI->getCalledFunction() && CI->getCalledFunction()->hasName() &&
5953 CI->getCalledFunction()->getName().starts_with(Prefix: "_Z4sqrt")) {
5954 SetSqrtFPAccuracy(CI);
5955 }
5956 if (callOrInvoke) {
5957 *callOrInvoke = CI;
5958 if (CGM.getCodeGenOpts().CallGraphSection) {
5959 QualType CST;
5960 if (TargetDecl && TargetDecl->getFunctionType())
5961 CST = QualType(TargetDecl->getFunctionType(), 0);
5962 else if (const auto *FPT =
5963 Callee.getAbstractInfo().getCalleeFunctionProtoType())
5964 CST = QualType(FPT, 0);
5965 else
5966 llvm_unreachable(
5967 "Cannot find the callee type to generate callee_type metadata.");
5968
5969 // Set type identifier metadata of indirect calls for call graph section.
5970 if (!CST.isNull())
5971 CGM.createCalleeTypeMetadataForIcall(QT: CST, CB: *callOrInvoke);
5972 }
5973 }
5974
5975 // If this is within a function that has the guard(nocf) attribute and is an
5976 // indirect call, add the "guard_nocf" attribute to this call to indicate that
5977 // Control Flow Guard checks should not be added, even if the call is inlined.
5978 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
5979 if (const auto *A = FD->getAttr<CFGuardAttr>()) {
5980 if (A->getGuard() == CFGuardAttr::GuardArg::nocf &&
5981 !CI->getCalledFunction())
5982 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: "guard_nocf");
5983 }
5984 }
5985
5986 // Apply the attributes and calling convention.
5987 CI->setAttributes(Attrs);
5988 CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
5989
5990 // Apply various metadata.
5991
5992 if (!CI->getType()->isVoidTy())
5993 CI->setName("call");
5994
5995 if (CGM.shouldEmitConvergenceTokens() && CI->isConvergent())
5996 CI = addConvergenceControlToken(Input: CI);
5997
5998 // Update largest vector width from the return type.
5999 LargestVectorWidth =
6000 std::max(a: LargestVectorWidth, b: getMaxVectorWidth(Ty: CI->getType()));
6001
6002 // Insert instrumentation or attach profile metadata at indirect call sites.
6003 // For more details, see the comment before the definition of
6004 // IPVK_IndirectCallTarget in InstrProfData.inc.
6005 if (!CI->getCalledFunction())
6006 PGO->valueProfile(Builder, ValueKind: llvm::IPVK_IndirectCallTarget, ValueSite: CI, ValuePtr: CalleePtr);
6007
6008 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
6009 // optimizer it can aggressively ignore unwind edges.
6010 if (CGM.getLangOpts().ObjCAutoRefCount)
6011 AddObjCARCExceptionMetadata(Inst: CI);
6012
6013 // Set tail call kind if necessary.
6014 bool IsPPC = getTarget().getTriple().isPPC();
6015 bool IsMIPS = getTarget().getTriple().isMIPS();
6016 bool HasMips16 = false;
6017 if (IsMIPS) {
6018 const TargetOptions &TargetOpts = getTarget().getTargetOpts();
6019 HasMips16 = TargetOpts.FeatureMap.lookup(Key: "mips16");
6020 if (!HasMips16)
6021 HasMips16 = llvm::is_contained(Range: TargetOpts.Features, Element: "+mips16");
6022 }
6023 if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(Val: CI)) {
6024 if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
6025 Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
6026 else if (IsMustTail) {
6027 if (IsPPC) {
6028 if (getTarget().getTriple().isOSAIX())
6029 CGM.getDiags().Report(Loc, DiagID: diag::err_aix_musttail_unsupported);
6030 else if (!getTarget().hasFeature(Feature: "pcrelative-memops")) {
6031 if (getTarget().hasFeature(Feature: "longcall"))
6032 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail) << 0;
6033 else if (Call->isIndirectCall())
6034 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail) << 1;
6035 else if (isa_and_nonnull<FunctionDecl>(Val: TargetDecl)) {
6036 if (!cast<FunctionDecl>(Val: TargetDecl)->isDefined())
6037 // The undefined callee may be a forward declaration. Without
6038 // knowning all symbols in the module, we won't know the symbol is
6039 // defined or not. Collect all these symbols for later diagnosing.
6040 CGM.addUndefinedGlobalForTailCall(
6041 Global: {cast<FunctionDecl>(Val: TargetDecl), Loc});
6042 else {
6043 llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(
6044 GD: GlobalDecl(cast<FunctionDecl>(Val: TargetDecl)));
6045 if (llvm::GlobalValue::isWeakForLinker(Linkage) ||
6046 llvm::GlobalValue::isDiscardableIfUnused(Linkage))
6047 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail)
6048 << 2;
6049 }
6050 }
6051 }
6052 }
6053 if (IsMIPS) {
6054 if (HasMips16)
6055 CGM.getDiags().Report(Loc, DiagID: diag::err_mips_impossible_musttail) << 0;
6056 else if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl))
6057 CGM.addUndefinedGlobalForTailCall(Global: {FD, Loc});
6058 }
6059 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
6060 }
6061 }
6062
6063 // Add metadata for calls to MSAllocator functions
6064 if (getDebugInfo() && TargetDecl && TargetDecl->hasAttr<MSAllocatorAttr>())
6065 getDebugInfo()->addHeapAllocSiteMetadata(CallSite: CI, AllocatedTy: RetTy->getPointeeType(), Loc);
6066
6067 // Add metadata if calling an __attribute__((error(""))) or warning fn.
6068 if (TargetDecl && TargetDecl->hasAttr<ErrorAttr>()) {
6069 llvm::ConstantInt *Line =
6070 llvm::ConstantInt::get(Ty: Int64Ty, V: Loc.getRawEncoding());
6071 llvm::ConstantAsMetadata *MD = llvm::ConstantAsMetadata::get(C: Line);
6072 llvm::MDTuple *MDT = llvm::MDNode::get(Context&: getLLVMContext(), MDs: {MD});
6073 CI->setMetadata(Kind: "srcloc", Node: MDT);
6074 }
6075
6076 // 4. Finish the call.
6077
6078 // If the call doesn't return, finish the basic block and clear the
6079 // insertion point; this allows the rest of IRGen to discard
6080 // unreachable code.
6081 if (CI->doesNotReturn()) {
6082 if (NeedSRetLifetimeEnd)
6083 PopCleanupBlock();
6084
6085 // Strip away the noreturn attribute to better diagnose unreachable UB.
6086 if (SanOpts.has(K: SanitizerKind::Unreachable)) {
6087 // Also remove from function since CallBase::hasFnAttr additionally checks
6088 // attributes of the called function.
6089 if (auto *F = CI->getCalledFunction())
6090 F->removeFnAttr(Kind: llvm::Attribute::NoReturn);
6091 CI->removeFnAttr(Kind: llvm::Attribute::NoReturn);
6092
6093 // Avoid incompatibility with ASan which relies on the `noreturn`
6094 // attribute to insert handler calls.
6095 if (SanOpts.hasOneOf(K: SanitizerKind::Address |
6096 SanitizerKind::KernelAddress)) {
6097 SanitizerScope SanScope(this);
6098 llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);
6099 Builder.SetInsertPoint(CI);
6100 auto *FnType = llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
6101 llvm::FunctionCallee Fn =
6102 CGM.CreateRuntimeFunction(Ty: FnType, Name: "__asan_handle_no_return");
6103 EmitNounwindRuntimeCall(callee: Fn);
6104 }
6105 }
6106
6107 EmitUnreachable(Loc);
6108 Builder.ClearInsertionPoint();
6109
6110 // FIXME: For now, emit a dummy basic block because expr emitters in
6111 // generally are not ready to handle emitting expressions at unreachable
6112 // points.
6113 EnsureInsertPoint();
6114
6115 // Return a reasonable RValue.
6116 return GetUndefRValue(Ty: RetTy);
6117 }
6118
6119 // If this is a musttail call, return immediately. We do not branch to the
6120 // epilogue in this case.
6121 if (IsMustTail) {
6122 for (auto it = EHStack.find(sp: CurrentCleanupScopeDepth); it != EHStack.end();
6123 ++it) {
6124 EHCleanupScope *Cleanup = dyn_cast<EHCleanupScope>(Val: &*it);
6125 // Fake uses can be safely emitted immediately prior to the tail call, so
6126 // we choose to emit them just before the call here.
6127 if (Cleanup && Cleanup->isFakeUse()) {
6128 CGBuilderTy::InsertPointGuard IPG(Builder);
6129 Builder.SetInsertPoint(CI);
6130 Cleanup->getCleanup()->Emit(CGF&: *this, flags: EHScopeStack::Cleanup::Flags());
6131 } else if (!(Cleanup &&
6132 Cleanup->getCleanup()->isRedundantBeforeReturn())) {
6133 CGM.ErrorUnsupported(S: MustTailCall, Type: "tail call skipping over cleanups");
6134 }
6135 }
6136 if (CI->getType()->isVoidTy())
6137 Builder.CreateRetVoid();
6138 else
6139 Builder.CreateRet(V: CI);
6140 Builder.ClearInsertionPoint();
6141 EnsureInsertPoint();
6142 return GetUndefRValue(Ty: RetTy);
6143 }
6144
6145 // Perform the swifterror writeback.
6146 if (swiftErrorTemp.isValid()) {
6147 llvm::Value *errorResult = Builder.CreateLoad(Addr: swiftErrorTemp);
6148 Builder.CreateStore(Val: errorResult, Addr: swiftErrorArg);
6149 }
6150
6151 // Emit any call-associated writebacks immediately. Arguably this
6152 // should happen after any return-value munging.
6153 if (CallArgs.hasWritebacks())
6154 EmitWritebacks(args: CallArgs);
6155
6156 // The stack cleanup for inalloca arguments has to run out of the normal
6157 // lexical order, so deactivate it and run it manually here.
6158 CallArgs.freeArgumentMemory(CGF&: *this);
6159
6160 // Extract the return value.
6161 RValue Ret;
6162
6163 // If the current function is a virtual function pointer thunk, avoid copying
6164 // the return value of the musttail call to a temporary.
6165 if (IsVirtualFunctionPointerThunk) {
6166 Ret = RValue::get(V: CI);
6167 } else {
6168 Ret = [&] {
6169 switch (RetAI.getKind()) {
6170 case ABIArgInfo::CoerceAndExpand: {
6171 auto coercionType = RetAI.getCoerceAndExpandType();
6172
6173 Address addr = SRetPtr.withElementType(ElemTy: coercionType);
6174
6175 assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());
6176 bool requiresExtract = isa<llvm::StructType>(Val: CI->getType());
6177
6178 unsigned unpaddedIndex = 0;
6179 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
6180 llvm::Type *eltType = coercionType->getElementType(N: i);
6181 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
6182 continue;
6183 Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
6184 llvm::Value *elt = CI;
6185 if (requiresExtract)
6186 elt = Builder.CreateExtractValue(Agg: elt, Idxs: unpaddedIndex++);
6187 else
6188 assert(unpaddedIndex == 0);
6189 Builder.CreateStore(Val: elt, Addr: eltAddr);
6190 }
6191 [[fallthrough]];
6192 }
6193
6194 case ABIArgInfo::InAlloca:
6195 case ABIArgInfo::Indirect: {
6196 RValue ret = convertTempToRValue(addr: SRetPtr, type: RetTy, Loc: SourceLocation());
6197 if (NeedSRetLifetimeEnd)
6198 PopCleanupBlock();
6199 return ret;
6200 }
6201
6202 case ABIArgInfo::Ignore:
6203 // If we are ignoring an argument that had a result, make sure to
6204 // construct the appropriate return value for our caller.
6205 return GetUndefRValue(Ty: RetTy);
6206
6207 case ABIArgInfo::Extend:
6208 case ABIArgInfo::Direct: {
6209 llvm::Type *RetIRTy = ConvertType(T: RetTy);
6210 if (RetAI.getCoerceToType() == RetIRTy &&
6211 RetAI.getDirectOffset() == 0) {
6212 switch (getEvaluationKind(T: RetTy)) {
6213 case TEK_Complex: {
6214 llvm::Value *Real = Builder.CreateExtractValue(Agg: CI, Idxs: 0);
6215 llvm::Value *Imag = Builder.CreateExtractValue(Agg: CI, Idxs: 1);
6216 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
6217 }
6218 case TEK_Aggregate:
6219 break;
6220 case TEK_Scalar: {
6221 // If the argument doesn't match, perform a bitcast to coerce it.
6222 // This can happen due to trivial type mismatches.
6223 llvm::Value *V = CI;
6224 if (V->getType() != RetIRTy)
6225 V = Builder.CreateBitCast(V, DestTy: RetIRTy);
6226 return RValue::get(V);
6227 }
6228 }
6229 }
6230
6231 // If coercing a fixed vector from a scalable vector for ABI
6232 // compatibility, and the types match, use the llvm.vector.extract
6233 // intrinsic to perform the conversion.
6234 if (auto *FixedDstTy = dyn_cast<llvm::FixedVectorType>(Val: RetIRTy)) {
6235 llvm::Value *V = CI;
6236 if (auto *ScalableSrcTy =
6237 dyn_cast<llvm::ScalableVectorType>(Val: V->getType())) {
6238 if (FixedDstTy->getElementType() ==
6239 ScalableSrcTy->getElementType()) {
6240 V = Builder.CreateExtractVector(DstType: FixedDstTy, SrcVec: V, Idx: uint64_t(0),
6241 Name: "cast.fixed");
6242 return RValue::get(V);
6243 }
6244 }
6245 }
6246
6247 Address DestPtr = ReturnValue.getValue();
6248 bool DestIsVolatile = ReturnValue.isVolatile();
6249 uint64_t DestSize =
6250 getContext().getTypeInfoDataSizeInChars(T: RetTy).Width.getQuantity();
6251
6252 if (!DestPtr.isValid()) {
6253 DestPtr = CreateMemTemp(T: RetTy, Name: "coerce");
6254 DestIsVolatile = false;
6255 DestSize = getContext().getTypeSizeInChars(T: RetTy).getQuantity();
6256 }
6257
6258 // An empty record can overlap other data (if declared with
6259 // no_unique_address); omit the store for such types - as there is no
6260 // actual data to store.
6261 if (!isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true)) {
6262 // If the value is offset in memory, apply the offset now.
6263 Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
6264 CreateCoercedStore(
6265 Src: CI, Dst: StorePtr,
6266 DstSize: llvm::TypeSize::getFixed(ExactSize: DestSize - RetAI.getDirectOffset()),
6267 DstIsVolatile: DestIsVolatile);
6268 }
6269
6270 return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation());
6271 }
6272
6273 case ABIArgInfo::TargetSpecific: {
6274 Address DestPtr = ReturnValue.getValue();
6275 Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
6276 bool DestIsVolatile = ReturnValue.isVolatile();
6277 if (!DestPtr.isValid()) {
6278 DestPtr = CreateMemTemp(T: RetTy, Name: "target_coerce");
6279 DestIsVolatile = false;
6280 }
6281 CGM.getABIInfo().createCoercedStore(Val: CI, DstAddr: StorePtr, AI: RetAI, DestIsVolatile,
6282 CGF&: *this);
6283 return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation());
6284 }
6285
6286 case ABIArgInfo::Expand:
6287 case ABIArgInfo::IndirectAliased:
6288 llvm_unreachable("Invalid ABI kind for return argument");
6289 }
6290
6291 llvm_unreachable("Unhandled ABIArgInfo::Kind");
6292 }();
6293 }
6294
6295 // Emit the assume_aligned check on the return value.
6296 if (Ret.isScalar() && TargetDecl) {
6297 AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6298 AllocAlignAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6299 }
6300
6301 // Explicitly call CallLifetimeEnd::Emit just to re-use the code even though
6302 // we can't use the full cleanup mechanism.
6303 for (CallLifetimeEnd &LifetimeEnd : CallLifetimeEndAfterCall)
6304 LifetimeEnd.Emit(CGF&: *this, /*Flags=*/flags: {});
6305
6306 if (!ReturnValue.isExternallyDestructed() &&
6307 RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct)
6308 pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Ret.getAggregateAddress(),
6309 type: RetTy);
6310
6311 // Generate function declaration DISuprogram in order to be used
6312 // in debug info about call sites.
6313 if (CGDebugInfo *DI = getDebugInfo()) {
6314 // Ensure call site info would actually be emitted before collecting
6315 // further callee info.
6316 if (CalleeDecl && !CalleeDecl->hasAttr<NoDebugAttr>() &&
6317 DI->getCallSiteRelatedAttrs() != llvm::DINode::FlagZero) {
6318 CodeGenFunction CalleeCGF(CGM);
6319 const GlobalDecl &CalleeGlobalDecl =
6320 Callee.getAbstractInfo().getCalleeDecl();
6321 CalleeCGF.CurGD = CalleeGlobalDecl;
6322 FunctionArgList Args;
6323 QualType ResTy = CalleeCGF.BuildFunctionArgList(GD: CalleeGlobalDecl, Args);
6324 DI->EmitFuncDeclForCallSite(
6325 CallOrInvoke: CI, CalleeType: DI->getFunctionType(FD: CalleeDecl, RetTy: ResTy, Args), CalleeGlobalDecl);
6326 }
6327 }
6328
6329 return Ret;
6330}
6331
6332CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const {
6333 if (isVirtual()) {
6334 const CallExpr *CE = getVirtualCallExpr();
6335 return CGF.CGM.getCXXABI().getVirtualFunctionPointer(
6336 CGF, GD: getVirtualMethodDecl(), This: getThisAddress(), Ty: getVirtualFunctionType(),
6337 Loc: CE ? CE->getBeginLoc() : SourceLocation());
6338 }
6339
6340 return *this;
6341}
6342
6343/* VarArg handling */
6344
6345RValue CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
6346 AggValueSlot Slot) {
6347 VAListAddr = VE->isMicrosoftABI() ? EmitMSVAListRef(E: VE->getSubExpr())
6348 : EmitVAListRef(E: VE->getSubExpr());
6349 QualType Ty = VE->getType();
6350 if (Ty->isVariablyModifiedType())
6351 EmitVariablyModifiedType(Ty);
6352 if (VE->isMicrosoftABI())
6353 return CGM.getABIInfo().EmitMSVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6354 return CGM.getABIInfo().EmitVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6355}
6356
6357DisableDebugLocationUpdates::DisableDebugLocationUpdates(CodeGenFunction &CGF)
6358 : CGF(CGF) {
6359 CGF.disableDebugInfo();
6360}
6361
6362DisableDebugLocationUpdates::~DisableDebugLocationUpdates() {
6363 CGF.enableDebugInfo();
6364}
6365