1//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// These classes wrap the information about a call or function
10// definition used to handle ABI compliancy.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CGCall.h"
15#include "ABIInfo.h"
16#include "ABIInfoImpl.h"
17#include "CGBlocks.h"
18#include "CGCXXABI.h"
19#include "CGCleanup.h"
20#include "CGDebugInfo.h"
21#include "CGRecordLayout.h"
22#include "CodeGenFunction.h"
23#include "CodeGenModule.h"
24#include "CodeGenPGO.h"
25#include "TargetInfo.h"
26#include "clang/AST/Attr.h"
27#include "clang/AST/Decl.h"
28#include "clang/AST/DeclCXX.h"
29#include "clang/AST/DeclObjC.h"
30#include "clang/AST/RecordLayout.h"
31#include "clang/Basic/CodeGenOptions.h"
32#include "clang/Basic/TargetInfo.h"
33#include "clang/CodeGen/CGFunctionInfo.h"
34#include "clang/CodeGen/SwiftCallingConv.h"
35#include "llvm/ADT/STLExtras.h"
36#include "llvm/ADT/StringExtras.h"
37#include "llvm/Analysis/ValueTracking.h"
38#include "llvm/IR/Assumptions.h"
39#include "llvm/IR/AttributeMask.h"
40#include "llvm/IR/Attributes.h"
41#include "llvm/IR/CallingConv.h"
42#include "llvm/IR/DataLayout.h"
43#include "llvm/IR/DebugInfoMetadata.h"
44#include "llvm/IR/InlineAsm.h"
45#include "llvm/IR/IntrinsicInst.h"
46#include "llvm/IR/Intrinsics.h"
47#include "llvm/IR/Type.h"
48#include "llvm/Transforms/Utils/Local.h"
49#include <optional>
50using namespace clang;
51using namespace CodeGen;
52
53/***/
54
55unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
56 switch (CC) {
57 default:
58 return llvm::CallingConv::C;
59 case CC_X86StdCall:
60 return llvm::CallingConv::X86_StdCall;
61 case CC_X86FastCall:
62 return llvm::CallingConv::X86_FastCall;
63 case CC_X86RegCall:
64 return llvm::CallingConv::X86_RegCall;
65 case CC_X86ThisCall:
66 return llvm::CallingConv::X86_ThisCall;
67 case CC_Win64:
68 return llvm::CallingConv::Win64;
69 case CC_X86_64SysV:
70 return llvm::CallingConv::X86_64_SysV;
71 case CC_AAPCS:
72 return llvm::CallingConv::ARM_AAPCS;
73 case CC_AAPCS_VFP:
74 return llvm::CallingConv::ARM_AAPCS_VFP;
75 case CC_IntelOclBicc:
76 return llvm::CallingConv::Intel_OCL_BI;
77 // TODO: Add support for __pascal to LLVM.
78 case CC_X86Pascal:
79 return llvm::CallingConv::C;
80 // TODO: Add support for __vectorcall to LLVM.
81 case CC_X86VectorCall:
82 return llvm::CallingConv::X86_VectorCall;
83 case CC_AArch64VectorCall:
84 return llvm::CallingConv::AArch64_VectorCall;
85 case CC_AArch64SVEPCS:
86 return llvm::CallingConv::AArch64_SVE_VectorCall;
87 case CC_SpirFunction:
88 return llvm::CallingConv::SPIR_FUNC;
89 case CC_DeviceKernel:
90 return CGM.getTargetCodeGenInfo().getDeviceKernelCallingConv();
91 case CC_PreserveMost:
92 return llvm::CallingConv::PreserveMost;
93 case CC_PreserveAll:
94 return llvm::CallingConv::PreserveAll;
95 case CC_Swift:
96 return llvm::CallingConv::Swift;
97 case CC_SwiftAsync:
98 return llvm::CallingConv::SwiftTail;
99 case CC_M68kRTD:
100 return llvm::CallingConv::M68k_RTD;
101 case CC_PreserveNone:
102 return llvm::CallingConv::PreserveNone;
103 // clang-format off
104 case CC_RISCVVectorCall: return llvm::CallingConv::RISCV_VectorCall;
105 // clang-format on
106#define CC_VLS_CASE(ABI_VLEN) \
107 case CC_RISCVVLSCall_##ABI_VLEN: \
108 return llvm::CallingConv::RISCV_VLSCall_##ABI_VLEN;
109 CC_VLS_CASE(32)
110 CC_VLS_CASE(64)
111 CC_VLS_CASE(128)
112 CC_VLS_CASE(256)
113 CC_VLS_CASE(512)
114 CC_VLS_CASE(1024)
115 CC_VLS_CASE(2048)
116 CC_VLS_CASE(4096)
117 CC_VLS_CASE(8192)
118 CC_VLS_CASE(16384)
119 CC_VLS_CASE(32768)
120 CC_VLS_CASE(65536)
121#undef CC_VLS_CASE
122 }
123}
124
125/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
126/// qualification. Either or both of RD and MD may be null. A null RD indicates
127/// that there is no meaningful 'this' type, and a null MD can occur when
128/// calling a method pointer.
129CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
130 const CXXMethodDecl *MD) {
131 CanQualType RecTy;
132 if (RD)
133 RecTy = Context.getCanonicalTagType(TD: RD);
134 else
135 RecTy = Context.VoidTy;
136
137 if (MD)
138 RecTy = CanQualType::CreateUnsafe(Other: Context.getAddrSpaceQualType(
139 T: RecTy, AddressSpace: MD->getMethodQualifiers().getAddressSpace()));
140 return Context.getPointerType(T: RecTy);
141}
142
143/// Returns the canonical formal type of the given C++ method.
144static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
145 return MD->getType()
146 ->getCanonicalTypeUnqualified()
147 .getAs<FunctionProtoType>();
148}
149
150/// Returns the "extra-canonicalized" return type, which discards
151/// qualifiers on the return type. Codegen doesn't care about them,
152/// and it makes ABI code a little easier to be able to assume that
153/// all parameter and return types are top-level unqualified.
154static CanQualType GetReturnType(QualType RetTy) {
155 return RetTy->getCanonicalTypeUnqualified();
156}
157
158/// Arrange the argument and result information for a value of the given
159/// unprototyped freestanding function type.
160const CGFunctionInfo &
161CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
162 // When translating an unprototyped function type, always use a
163 // variadic type.
164 return arrangeLLVMFunctionInfo(returnType: FTNP->getReturnType().getUnqualifiedType(),
165 opts: FnInfoOpts::None, argTypes: {}, info: FTNP->getExtInfo(), paramInfos: {},
166 args: RequiredArgs(0));
167}
168
169static void addExtParameterInfosForCall(
170 llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
171 const FunctionProtoType *proto, unsigned prefixArgs, unsigned totalArgs) {
172 assert(proto->hasExtParameterInfos());
173 assert(paramInfos.size() <= prefixArgs);
174 assert(proto->getNumParams() + prefixArgs <= totalArgs);
175
176 paramInfos.reserve(N: totalArgs);
177
178 // Add default infos for any prefix args that don't already have infos.
179 paramInfos.resize(N: prefixArgs);
180
181 // Add infos for the prototype.
182 for (const auto &ParamInfo : proto->getExtParameterInfos()) {
183 paramInfos.push_back(Elt: ParamInfo);
184 // pass_object_size params have no parameter info.
185 if (ParamInfo.hasPassObjectSize())
186 paramInfos.emplace_back();
187 }
188
189 assert(paramInfos.size() <= totalArgs &&
190 "Did we forget to insert pass_object_size args?");
191 // Add default infos for the variadic and/or suffix arguments.
192 paramInfos.resize(N: totalArgs);
193}
194
195/// Adds the formal parameters in FPT to the given prefix. If any parameter in
196/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
197static void appendParameterTypes(
198 const CodeGenTypes &CGT, SmallVectorImpl<CanQualType> &prefix,
199 SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
200 CanQual<FunctionProtoType> FPT) {
201 // Fast path: don't touch param info if we don't need to.
202 if (!FPT->hasExtParameterInfos()) {
203 assert(paramInfos.empty() &&
204 "We have paramInfos, but the prototype doesn't?");
205 prefix.append(in_start: FPT->param_type_begin(), in_end: FPT->param_type_end());
206 return;
207 }
208
209 unsigned PrefixSize = prefix.size();
210 // In the vast majority of cases, we'll have precisely FPT->getNumParams()
211 // parameters; the only thing that can change this is the presence of
212 // pass_object_size. So, we preallocate for the common case.
213 prefix.reserve(N: prefix.size() + FPT->getNumParams());
214
215 auto ExtInfos = FPT->getExtParameterInfos();
216 assert(ExtInfos.size() == FPT->getNumParams());
217 for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
218 prefix.push_back(Elt: FPT->getParamType(i: I));
219 if (ExtInfos[I].hasPassObjectSize())
220 prefix.push_back(Elt: CGT.getContext().getCanonicalSizeType());
221 }
222
223 addExtParameterInfosForCall(paramInfos, proto: FPT.getTypePtr(), prefixArgs: PrefixSize,
224 totalArgs: prefix.size());
225}
226
227using ExtParameterInfoList =
228 SmallVector<FunctionProtoType::ExtParameterInfo, 16>;
229
230/// Arrange the LLVM function layout for a value of the given function
231/// type, on top of any implicit parameters already stored.
232static const CGFunctionInfo &
233arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
234 SmallVectorImpl<CanQualType> &prefix,
235 CanQual<FunctionProtoType> FTP) {
236 ExtParameterInfoList paramInfos;
237 RequiredArgs Required = RequiredArgs::forPrototypePlus(prototype: FTP, additional: prefix.size());
238 appendParameterTypes(CGT, prefix, paramInfos, FPT: FTP);
239 CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
240
241 FnInfoOpts opts =
242 instanceMethod ? FnInfoOpts::IsInstanceMethod : FnInfoOpts::None;
243 return CGT.arrangeLLVMFunctionInfo(returnType: resultType, opts, argTypes: prefix,
244 info: FTP->getExtInfo(), paramInfos, args: Required);
245}
246
247using CanQualTypeList = SmallVector<CanQualType, 16>;
248
249/// Arrange the argument and result information for a value of the
250/// given freestanding function type.
251const CGFunctionInfo &
252CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
253 CanQualTypeList argTypes;
254 return ::arrangeLLVMFunctionInfo(CGT&: *this, /*instanceMethod=*/false, prefix&: argTypes,
255 FTP);
256}
257
258static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
259 bool IsTargetDefaultMSABI) {
260 // Set the appropriate calling convention for the Function.
261 if (D->hasAttr<StdCallAttr>())
262 return CC_X86StdCall;
263
264 if (D->hasAttr<FastCallAttr>())
265 return CC_X86FastCall;
266
267 if (D->hasAttr<RegCallAttr>())
268 return CC_X86RegCall;
269
270 if (D->hasAttr<ThisCallAttr>())
271 return CC_X86ThisCall;
272
273 if (D->hasAttr<VectorCallAttr>())
274 return CC_X86VectorCall;
275
276 if (D->hasAttr<PascalAttr>())
277 return CC_X86Pascal;
278
279 if (PcsAttr *PCS = D->getAttr<PcsAttr>())
280 return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
281
282 if (D->hasAttr<AArch64VectorPcsAttr>())
283 return CC_AArch64VectorCall;
284
285 if (D->hasAttr<AArch64SVEPcsAttr>())
286 return CC_AArch64SVEPCS;
287
288 if (D->hasAttr<DeviceKernelAttr>())
289 return CC_DeviceKernel;
290
291 if (D->hasAttr<IntelOclBiccAttr>())
292 return CC_IntelOclBicc;
293
294 if (D->hasAttr<MSABIAttr>())
295 return IsTargetDefaultMSABI ? CC_C : CC_Win64;
296
297 if (D->hasAttr<SysVABIAttr>())
298 return IsTargetDefaultMSABI ? CC_X86_64SysV : CC_C;
299
300 if (D->hasAttr<PreserveMostAttr>())
301 return CC_PreserveMost;
302
303 if (D->hasAttr<PreserveAllAttr>())
304 return CC_PreserveAll;
305
306 if (D->hasAttr<M68kRTDAttr>())
307 return CC_M68kRTD;
308
309 if (D->hasAttr<PreserveNoneAttr>())
310 return CC_PreserveNone;
311
312 if (D->hasAttr<RISCVVectorCCAttr>())
313 return CC_RISCVVectorCall;
314
315 if (RISCVVLSCCAttr *PCS = D->getAttr<RISCVVLSCCAttr>()) {
316 switch (PCS->getVectorWidth()) {
317 default:
318 llvm_unreachable("Invalid RISC-V VLS ABI VLEN");
319#define CC_VLS_CASE(ABI_VLEN) \
320 case ABI_VLEN: \
321 return CC_RISCVVLSCall_##ABI_VLEN;
322 CC_VLS_CASE(32)
323 CC_VLS_CASE(64)
324 CC_VLS_CASE(128)
325 CC_VLS_CASE(256)
326 CC_VLS_CASE(512)
327 CC_VLS_CASE(1024)
328 CC_VLS_CASE(2048)
329 CC_VLS_CASE(4096)
330 CC_VLS_CASE(8192)
331 CC_VLS_CASE(16384)
332 CC_VLS_CASE(32768)
333 CC_VLS_CASE(65536)
334#undef CC_VLS_CASE
335 }
336 }
337
338 return CC_C;
339}
340
341/// Arrange the argument and result information for a call to an
342/// unknown C++ non-static member function of the given abstract type.
343/// (A null RD means we don't have any meaningful "this" argument type,
344/// so fall back to a generic pointer type).
345/// The member function must be an ordinary function, i.e. not a
346/// constructor or destructor.
347const CGFunctionInfo &
348CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
349 const FunctionProtoType *FTP,
350 const CXXMethodDecl *MD) {
351 CanQualTypeList argTypes;
352
353 // Add the 'this' pointer.
354 argTypes.push_back(Elt: DeriveThisType(RD, MD));
355
356 return ::arrangeLLVMFunctionInfo(
357 CGT&: *this, /*instanceMethod=*/true, prefix&: argTypes,
358 FTP: FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
359}
360
361/// Set calling convention for CUDA/HIP kernel.
362static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
363 const FunctionDecl *FD) {
364 if (FD->hasAttr<CUDAGlobalAttr>()) {
365 const FunctionType *FT = FTy->getAs<FunctionType>();
366 CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
367 FTy = FT->getCanonicalTypeUnqualified();
368 }
369}
370
371/// Arrange the argument and result information for a declaration or
372/// definition of the given C++ non-static member function. The
373/// member function must be an ordinary function, i.e. not a
374/// constructor or destructor.
375const CGFunctionInfo &
376CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
377 assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
378 assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
379
380 CanQualType FT = GetFormalType(MD).getAs<Type>();
381 setCUDAKernelCallingConvention(FTy&: FT, CGM, FD: MD);
382 auto prototype = FT.getAs<FunctionProtoType>();
383
384 if (MD->isImplicitObjectMemberFunction()) {
385 // The abstract case is perfectly fine.
386 const CXXRecordDecl *ThisType =
387 getCXXABI().getThisArgumentTypeForMethod(GD: MD);
388 return arrangeCXXMethodType(RD: ThisType, FTP: prototype.getTypePtr(), MD);
389 }
390
391 return arrangeFreeFunctionType(FTP: prototype);
392}
393
394bool CodeGenTypes::inheritingCtorHasParams(
395 const InheritedConstructor &Inherited, CXXCtorType Type) {
396 // Parameters are unnecessary if we're constructing a base class subobject
397 // and the inherited constructor lives in a virtual base.
398 return Type == Ctor_Complete ||
399 !Inherited.getShadowDecl()->constructsVirtualBase() ||
400 !Target.getCXXABI().hasConstructorVariants();
401}
402
403const CGFunctionInfo &
404CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
405 auto *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
406
407 CanQualTypeList argTypes;
408 ExtParameterInfoList paramInfos;
409
410 const CXXRecordDecl *ThisType = getCXXABI().getThisArgumentTypeForMethod(GD);
411 argTypes.push_back(Elt: DeriveThisType(RD: ThisType, MD));
412
413 bool PassParams = true;
414
415 if (auto *CD = dyn_cast<CXXConstructorDecl>(Val: MD)) {
416 // A base class inheriting constructor doesn't get forwarded arguments
417 // needed to construct a virtual base (or base class thereof).
418 if (auto Inherited = CD->getInheritedConstructor())
419 PassParams = inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
420 }
421
422 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
423
424 // Add the formal parameters.
425 if (PassParams)
426 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: FTP);
427
428 CGCXXABI::AddedStructorArgCounts AddedArgs =
429 getCXXABI().buildStructorSignature(GD, ArgTys&: argTypes);
430 if (!paramInfos.empty()) {
431 // Note: prefix implies after the first param.
432 if (AddedArgs.Prefix)
433 paramInfos.insert(I: paramInfos.begin() + 1, NumToInsert: AddedArgs.Prefix,
434 Elt: FunctionProtoType::ExtParameterInfo{});
435 if (AddedArgs.Suffix)
436 paramInfos.append(NumInputs: AddedArgs.Suffix,
437 Elt: FunctionProtoType::ExtParameterInfo{});
438 }
439
440 RequiredArgs required =
441 (PassParams && MD->isVariadic() ? RequiredArgs(argTypes.size())
442 : RequiredArgs::All);
443
444 FunctionType::ExtInfo extInfo = FTP->getExtInfo();
445 CanQualType resultType = getCXXABI().HasThisReturn(GD) ? argTypes.front()
446 : getCXXABI().hasMostDerivedReturn(GD)
447 ? CGM.getContext().VoidPtrTy
448 : Context.VoidTy;
449 return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::IsInstanceMethod,
450 argTypes, info: extInfo, paramInfos, args: required);
451}
452
453static CanQualTypeList getArgTypesForCall(ASTContext &ctx,
454 const CallArgList &args) {
455 CanQualTypeList argTypes;
456 for (auto &arg : args)
457 argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg.Ty));
458 return argTypes;
459}
460
461static CanQualTypeList getArgTypesForDeclaration(ASTContext &ctx,
462 const FunctionArgList &args) {
463 CanQualTypeList argTypes;
464 for (auto &arg : args)
465 argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg->getType()));
466 return argTypes;
467}
468
469static ExtParameterInfoList
470getExtParameterInfosForCall(const FunctionProtoType *proto, unsigned prefixArgs,
471 unsigned totalArgs) {
472 ExtParameterInfoList result;
473 if (proto->hasExtParameterInfos()) {
474 addExtParameterInfosForCall(paramInfos&: result, proto, prefixArgs, totalArgs);
475 }
476 return result;
477}
478
479/// Arrange a call to a C++ method, passing the given arguments.
480///
481/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
482/// parameter.
483/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
484/// args.
485/// PassProtoArgs indicates whether `args` has args for the parameters in the
486/// given CXXConstructorDecl.
487const CGFunctionInfo &CodeGenTypes::arrangeCXXConstructorCall(
488 const CallArgList &args, const CXXConstructorDecl *D, CXXCtorType CtorKind,
489 unsigned ExtraPrefixArgs, unsigned ExtraSuffixArgs, bool PassProtoArgs) {
490 CanQualTypeList ArgTypes;
491 for (const auto &Arg : args)
492 ArgTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
493
494 // +1 for implicit this, which should always be args[0].
495 unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
496
497 CanQual<FunctionProtoType> FPT = GetFormalType(MD: D);
498 RequiredArgs Required = PassProtoArgs
499 ? RequiredArgs::forPrototypePlus(
500 prototype: FPT, additional: TotalPrefixArgs + ExtraSuffixArgs)
501 : RequiredArgs::All;
502
503 GlobalDecl GD(D, CtorKind);
504 CanQualType ResultType = getCXXABI().HasThisReturn(GD) ? ArgTypes.front()
505 : getCXXABI().hasMostDerivedReturn(GD)
506 ? CGM.getContext().VoidPtrTy
507 : Context.VoidTy;
508
509 FunctionType::ExtInfo Info = FPT->getExtInfo();
510 ExtParameterInfoList ParamInfos;
511 // If the prototype args are elided, we should only have ABI-specific args,
512 // which never have param info.
513 if (PassProtoArgs && FPT->hasExtParameterInfos()) {
514 // ABI-specific suffix arguments are treated the same as variadic arguments.
515 addExtParameterInfosForCall(paramInfos&: ParamInfos, proto: FPT.getTypePtr(), prefixArgs: TotalPrefixArgs,
516 totalArgs: ArgTypes.size());
517 }
518
519 return arrangeLLVMFunctionInfo(returnType: ResultType, opts: FnInfoOpts::IsInstanceMethod,
520 argTypes: ArgTypes, info: Info, paramInfos: ParamInfos, args: Required);
521}
522
523/// Arrange the argument and result information for the declaration or
524/// definition of the given function.
525const CGFunctionInfo &
526CodeGenTypes::arrangeFunctionDeclaration(const GlobalDecl GD) {
527 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
528 if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD))
529 if (MD->isImplicitObjectMemberFunction())
530 return arrangeCXXMethodDeclaration(MD);
531
532 CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
533
534 assert(isa<FunctionType>(FTy));
535 setCUDAKernelCallingConvention(FTy, CGM, FD);
536
537 if (DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()) &&
538 GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
539 const FunctionType *FT = FTy->getAs<FunctionType>();
540 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
541 FTy = FT->getCanonicalTypeUnqualified();
542 }
543
544 // When declaring a function without a prototype, always use a
545 // non-variadic type.
546 if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
547 return arrangeLLVMFunctionInfo(returnType: noProto->getReturnType(), opts: FnInfoOpts::None,
548 argTypes: {}, info: noProto->getExtInfo(), paramInfos: {},
549 args: RequiredArgs::All);
550 }
551
552 return arrangeFreeFunctionType(FTP: FTy.castAs<FunctionProtoType>());
553}
554
555/// Arrange the argument and result information for the declaration or
556/// definition of an Objective-C method.
557const CGFunctionInfo &
558CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
559 // It happens that this is the same as a call with no optional
560 // arguments, except also using the formal 'self' type.
561 return arrangeObjCMessageSendSignature(MD, receiverType: MD->getSelfDecl()->getType());
562}
563
564/// Arrange the argument and result information for the function type
565/// through which to perform a send to the given Objective-C method,
566/// using the given receiver type. The receiver type is not always
567/// the 'self' type of the method or even an Objective-C pointer type.
568/// This is *not* the right method for actually performing such a
569/// message send, due to the possibility of optional arguments.
570const CGFunctionInfo &
571CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
572 QualType receiverType) {
573 CanQualTypeList argTys;
574 ExtParameterInfoList extParamInfos(MD->isDirectMethod() ? 1 : 2);
575 argTys.push_back(Elt: Context.getCanonicalParamType(T: receiverType));
576 if (!MD->isDirectMethod())
577 argTys.push_back(Elt: Context.getCanonicalParamType(T: Context.getObjCSelType()));
578 for (const auto *I : MD->parameters()) {
579 argTys.push_back(Elt: Context.getCanonicalParamType(T: I->getType()));
580 auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
581 NoEscape: I->hasAttr<NoEscapeAttr>());
582 extParamInfos.push_back(Elt: extParamInfo);
583 }
584
585 FunctionType::ExtInfo einfo;
586 bool IsTargetDefaultMSABI =
587 getContext().getTargetInfo().getTriple().isOSWindows() ||
588 getContext().getTargetInfo().getTriple().isUEFI();
589 einfo = einfo.withCallingConv(
590 cc: getCallingConventionForDecl(D: MD, IsTargetDefaultMSABI));
591
592 if (getContext().getLangOpts().ObjCAutoRefCount &&
593 MD->hasAttr<NSReturnsRetainedAttr>())
594 einfo = einfo.withProducesResult(producesResult: true);
595
596 RequiredArgs required =
597 (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);
598
599 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: MD->getReturnType()),
600 opts: FnInfoOpts::None, argTypes: argTys, info: einfo, paramInfos: extParamInfos,
601 args: required);
602}
603
604const CGFunctionInfo &
605CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
606 const CallArgList &args) {
607 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
608 FunctionType::ExtInfo einfo;
609
610 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: returnType), opts: FnInfoOpts::None,
611 argTypes, info: einfo, paramInfos: {}, args: RequiredArgs::All);
612}
613
614const CGFunctionInfo &CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
615 // FIXME: Do we need to handle ObjCMethodDecl?
616 if (isa<CXXConstructorDecl>(Val: GD.getDecl()) ||
617 isa<CXXDestructorDecl>(Val: GD.getDecl()))
618 return arrangeCXXStructorDeclaration(GD);
619
620 return arrangeFunctionDeclaration(GD);
621}
622
623/// Arrange a thunk that takes 'this' as the first parameter followed by
624/// varargs. Return a void pointer, regardless of the actual return type.
625/// The body of the thunk will end in a musttail call to a function of the
626/// correct type, and the caller will bitcast the function to the correct
627/// prototype.
628const CGFunctionInfo &
629CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
630 assert(MD->isVirtual() && "only methods have thunks");
631 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
632 CanQualType ArgTys[] = {DeriveThisType(RD: MD->getParent(), MD)};
633 return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::None, argTypes: ArgTys,
634 info: FTP->getExtInfo(), paramInfos: {}, args: RequiredArgs(1));
635}
636
637const CGFunctionInfo &
638CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
639 CXXCtorType CT) {
640 assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
641
642 CanQual<FunctionProtoType> FTP = GetFormalType(MD: CD);
643 SmallVector<CanQualType, 2> ArgTys;
644 const CXXRecordDecl *RD = CD->getParent();
645 ArgTys.push_back(Elt: DeriveThisType(RD, MD: CD));
646 if (CT == Ctor_CopyingClosure)
647 ArgTys.push_back(Elt: *FTP->param_type_begin());
648 if (RD->getNumVBases() > 0)
649 ArgTys.push_back(Elt: Context.IntTy);
650 CallingConv CC = Context.getDefaultCallingConvention(
651 /*IsVariadic=*/false, /*IsCXXMethod=*/true);
652 return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::IsInstanceMethod,
653 argTypes: ArgTys, info: FunctionType::ExtInfo(CC), paramInfos: {},
654 args: RequiredArgs::All);
655}
656
657/// Arrange a call as unto a free function, except possibly with an
658/// additional number of formal parameters considered required.
659static const CGFunctionInfo &
660arrangeFreeFunctionLikeCall(CodeGenTypes &CGT, CodeGenModule &CGM,
661 const CallArgList &args, const FunctionType *fnType,
662 unsigned numExtraRequiredArgs, bool chainCall) {
663 assert(args.size() >= numExtraRequiredArgs);
664
665 ExtParameterInfoList paramInfos;
666
667 // In most cases, there are no optional arguments.
668 RequiredArgs required = RequiredArgs::All;
669
670 // If we have a variadic prototype, the required arguments are the
671 // extra prefix plus the arguments in the prototype.
672 if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(Val: fnType)) {
673 if (proto->isVariadic())
674 required = RequiredArgs::forPrototypePlus(prototype: proto, additional: numExtraRequiredArgs);
675
676 if (proto->hasExtParameterInfos())
677 addExtParameterInfosForCall(paramInfos, proto, prefixArgs: numExtraRequiredArgs,
678 totalArgs: args.size());
679
680 // If we don't have a prototype at all, but we're supposed to
681 // explicitly use the variadic convention for unprototyped calls,
682 // treat all of the arguments as required but preserve the nominal
683 // possibility of variadics.
684 } else if (CGM.getTargetCodeGenInfo().isNoProtoCallVariadic(
685 args, fnType: cast<FunctionNoProtoType>(Val: fnType))) {
686 required = RequiredArgs(args.size());
687 }
688
689 CanQualTypeList argTypes;
690 for (const auto &arg : args)
691 argTypes.push_back(Elt: CGT.getContext().getCanonicalParamType(T: arg.Ty));
692 FnInfoOpts opts = chainCall ? FnInfoOpts::IsChainCall : FnInfoOpts::None;
693 return CGT.arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: fnType->getReturnType()),
694 opts, argTypes, info: fnType->getExtInfo(),
695 paramInfos, args: required);
696}
697
698/// Figure out the rules for calling a function with the given formal
699/// type using the given arguments. The arguments are necessary
700/// because the function might be unprototyped, in which case it's
701/// target-dependent in crazy ways.
702const CGFunctionInfo &CodeGenTypes::arrangeFreeFunctionCall(
703 const CallArgList &args, const FunctionType *fnType, bool chainCall) {
704 return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType,
705 numExtraRequiredArgs: chainCall ? 1 : 0, chainCall);
706}
707
708/// A block function is essentially a free function with an
709/// extra implicit argument.
710const CGFunctionInfo &
711CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
712 const FunctionType *fnType) {
713 return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType, numExtraRequiredArgs: 1,
714 /*chainCall=*/false);
715}
716
717const CGFunctionInfo &
718CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
719 const FunctionArgList &params) {
720 ExtParameterInfoList paramInfos =
721 getExtParameterInfosForCall(proto, prefixArgs: 1, totalArgs: params.size());
722 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args: params);
723
724 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: proto->getReturnType()),
725 opts: FnInfoOpts::None, argTypes,
726 info: proto->getExtInfo(), paramInfos,
727 args: RequiredArgs::forPrototypePlus(prototype: proto, additional: 1));
728}
729
730const CGFunctionInfo &
731CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
732 const CallArgList &args) {
733 CanQualTypeList argTypes;
734 for (const auto &Arg : args)
735 argTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
736 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
737 argTypes, info: FunctionType::ExtInfo(),
738 /*paramInfos=*/{}, args: RequiredArgs::All);
739}
740
741const CGFunctionInfo &
742CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
743 const FunctionArgList &args) {
744 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
745
746 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
747 argTypes, info: FunctionType::ExtInfo(), paramInfos: {},
748 args: RequiredArgs::All);
749}
750
751const CGFunctionInfo &CodeGenTypes::arrangeBuiltinFunctionDeclaration(
752 CanQualType resultType, ArrayRef<CanQualType> argTypes) {
753 return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::None, argTypes,
754 info: FunctionType::ExtInfo(), paramInfos: {},
755 args: RequiredArgs::All);
756}
757
758const CGFunctionInfo &CodeGenTypes::arrangeDeviceKernelCallerDeclaration(
759 QualType resultType, const FunctionArgList &args) {
760 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
761
762 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
763 argTypes,
764 info: FunctionType::ExtInfo(CC_DeviceKernel),
765 /*paramInfos=*/{}, args: RequiredArgs::All);
766}
767
768/// Arrange a call to a C++ method, passing the given arguments.
769///
770/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
771/// does not count `this`.
772const CGFunctionInfo &CodeGenTypes::arrangeCXXMethodCall(
773 const CallArgList &args, const FunctionProtoType *proto,
774 RequiredArgs required, unsigned numPrefixArgs) {
775 assert(numPrefixArgs + 1 <= args.size() &&
776 "Emitting a call with less args than the required prefix?");
777 // Add one to account for `this`. It's a bit awkward here, but we don't count
778 // `this` in similar places elsewhere.
779 ExtParameterInfoList paramInfos =
780 getExtParameterInfosForCall(proto, prefixArgs: numPrefixArgs + 1, totalArgs: args.size());
781
782 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
783
784 FunctionType::ExtInfo info = proto->getExtInfo();
785 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: proto->getReturnType()),
786 opts: FnInfoOpts::IsInstanceMethod, argTypes, info,
787 paramInfos, args: required);
788}
789
790const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
791 return arrangeLLVMFunctionInfo(returnType: getContext().VoidTy, opts: FnInfoOpts::None, argTypes: {},
792 info: FunctionType::ExtInfo(), paramInfos: {},
793 args: RequiredArgs::All);
794}
795
796const CGFunctionInfo &CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
797 const CallArgList &args) {
798 assert(signature.arg_size() <= args.size());
799 if (signature.arg_size() == args.size())
800 return signature;
801
802 ExtParameterInfoList paramInfos;
803 auto sigParamInfos = signature.getExtParameterInfos();
804 if (!sigParamInfos.empty()) {
805 paramInfos.append(in_start: sigParamInfos.begin(), in_end: sigParamInfos.end());
806 paramInfos.resize(N: args.size());
807 }
808
809 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
810
811 assert(signature.getRequiredArgs().allowsOptionalArgs());
812 FnInfoOpts opts = FnInfoOpts::None;
813 if (signature.isInstanceMethod())
814 opts |= FnInfoOpts::IsInstanceMethod;
815 if (signature.isChainCall())
816 opts |= FnInfoOpts::IsChainCall;
817 if (signature.isDelegateCall())
818 opts |= FnInfoOpts::IsDelegateCall;
819 return arrangeLLVMFunctionInfo(returnType: signature.getReturnType(), opts, argTypes,
820 info: signature.getExtInfo(), paramInfos,
821 args: signature.getRequiredArgs());
822}
823
824namespace clang {
825namespace CodeGen {
826void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
827}
828} // namespace clang
829
830/// Arrange the argument and result information for an abstract value
831/// of a given function type. This is the method which all of the
832/// above functions ultimately defer to.
833const CGFunctionInfo &CodeGenTypes::arrangeLLVMFunctionInfo(
834 CanQualType resultType, FnInfoOpts opts, ArrayRef<CanQualType> argTypes,
835 FunctionType::ExtInfo info,
836 ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
837 RequiredArgs required) {
838 assert(llvm::all_of(argTypes,
839 [](CanQualType T) { return T.isCanonicalAsParam(); }));
840
841 // Lookup or create unique function info.
842 llvm::FoldingSetNodeID ID;
843 bool isInstanceMethod =
844 (opts & FnInfoOpts::IsInstanceMethod) == FnInfoOpts::IsInstanceMethod;
845 bool isChainCall =
846 (opts & FnInfoOpts::IsChainCall) == FnInfoOpts::IsChainCall;
847 bool isDelegateCall =
848 (opts & FnInfoOpts::IsDelegateCall) == FnInfoOpts::IsDelegateCall;
849 CGFunctionInfo::Profile(ID, InstanceMethod: isInstanceMethod, ChainCall: isChainCall, IsDelegateCall: isDelegateCall,
850 info, paramInfos, required, resultType, argTypes);
851
852 void *insertPos = nullptr;
853 CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, InsertPos&: insertPos);
854 if (FI)
855 return *FI;
856
857 unsigned CC = ClangCallConvToLLVMCallConv(CC: info.getCC());
858
859 // Construct the function info. We co-allocate the ArgInfos.
860 FI = CGFunctionInfo::create(llvmCC: CC, instanceMethod: isInstanceMethod, chainCall: isChainCall, delegateCall: isDelegateCall,
861 extInfo: info, paramInfos, resultType, argTypes, required);
862 FunctionInfos.InsertNode(N: FI, InsertPos: insertPos);
863
864 bool inserted = FunctionsBeingProcessed.insert(Ptr: FI).second;
865 (void)inserted;
866 assert(inserted && "Recursively being processed?");
867
868 // Compute ABI information.
869 if (CC == llvm::CallingConv::SPIR_KERNEL) {
870 // Force target independent argument handling for the host visible
871 // kernel functions.
872 computeSPIRKernelABIInfo(CGM, FI&: *FI);
873 } else if (info.getCC() == CC_Swift || info.getCC() == CC_SwiftAsync) {
874 swiftcall::computeABIInfo(CGM, FI&: *FI);
875 } else {
876 CGM.getABIInfo().computeInfo(FI&: *FI);
877 }
878
879 // Loop over all of the computed argument and return value info. If any of
880 // them are direct or extend without a specified coerce type, specify the
881 // default now.
882 ABIArgInfo &retInfo = FI->getReturnInfo();
883 if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
884 retInfo.setCoerceToType(ConvertType(T: FI->getReturnType()));
885
886 for (auto &I : FI->arguments())
887 if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
888 I.info.setCoerceToType(ConvertType(T: I.type));
889
890 bool erased = FunctionsBeingProcessed.erase(Ptr: FI);
891 (void)erased;
892 assert(erased && "Not in set?");
893
894 return *FI;
895}
896
897CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC, bool instanceMethod,
898 bool chainCall, bool delegateCall,
899 const FunctionType::ExtInfo &info,
900 ArrayRef<ExtParameterInfo> paramInfos,
901 CanQualType resultType,
902 ArrayRef<CanQualType> argTypes,
903 RequiredArgs required) {
904 assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
905 assert(!required.allowsOptionalArgs() ||
906 required.getNumRequiredArgs() <= argTypes.size());
907
908 void *buffer = operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(
909 Counts: argTypes.size() + 1, Counts: paramInfos.size()));
910
911 CGFunctionInfo *FI = new (buffer) CGFunctionInfo();
912 FI->CallingConvention = llvmCC;
913 FI->EffectiveCallingConvention = llvmCC;
914 FI->ASTCallingConvention = info.getCC();
915 FI->InstanceMethod = instanceMethod;
916 FI->ChainCall = chainCall;
917 FI->DelegateCall = delegateCall;
918 FI->CmseNSCall = info.getCmseNSCall();
919 FI->NoReturn = info.getNoReturn();
920 FI->ReturnsRetained = info.getProducesResult();
921 FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
922 FI->NoCfCheck = info.getNoCfCheck();
923 FI->Required = required;
924 FI->HasRegParm = info.getHasRegParm();
925 FI->RegParm = info.getRegParm();
926 FI->ArgStruct = nullptr;
927 FI->ArgStructAlign = 0;
928 FI->NumArgs = argTypes.size();
929 FI->HasExtParameterInfos = !paramInfos.empty();
930 FI->getArgsBuffer()[0].type = resultType;
931 FI->MaxVectorWidth = 0;
932 for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
933 FI->getArgsBuffer()[i + 1].type = argTypes[i];
934 for (unsigned i = 0, e = paramInfos.size(); i != e; ++i)
935 FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
936 return FI;
937}
938
939/***/
940
941namespace {
942// ABIArgInfo::Expand implementation.
943
944// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
945struct TypeExpansion {
946 enum TypeExpansionKind {
947 // Elements of constant arrays are expanded recursively.
948 TEK_ConstantArray,
949 // Record fields are expanded recursively (but if record is a union, only
950 // the field with the largest size is expanded).
951 TEK_Record,
952 // For complex types, real and imaginary parts are expanded recursively.
953 TEK_Complex,
954 // All other types are not expandable.
955 TEK_None
956 };
957
958 const TypeExpansionKind Kind;
959
960 TypeExpansion(TypeExpansionKind K) : Kind(K) {}
961 virtual ~TypeExpansion() {}
962};
963
964struct ConstantArrayExpansion : TypeExpansion {
965 QualType EltTy;
966 uint64_t NumElts;
967
968 ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
969 : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
970 static bool classof(const TypeExpansion *TE) {
971 return TE->Kind == TEK_ConstantArray;
972 }
973};
974
975struct RecordExpansion : TypeExpansion {
976 SmallVector<const CXXBaseSpecifier *, 1> Bases;
977
978 SmallVector<const FieldDecl *, 1> Fields;
979
980 RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
981 SmallVector<const FieldDecl *, 1> &&Fields)
982 : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
983 Fields(std::move(Fields)) {}
984 static bool classof(const TypeExpansion *TE) {
985 return TE->Kind == TEK_Record;
986 }
987};
988
989struct ComplexExpansion : TypeExpansion {
990 QualType EltTy;
991
992 ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
993 static bool classof(const TypeExpansion *TE) {
994 return TE->Kind == TEK_Complex;
995 }
996};
997
998struct NoExpansion : TypeExpansion {
999 NoExpansion() : TypeExpansion(TEK_None) {}
1000 static bool classof(const TypeExpansion *TE) { return TE->Kind == TEK_None; }
1001};
1002} // namespace
1003
1004static std::unique_ptr<TypeExpansion>
1005getTypeExpansion(QualType Ty, const ASTContext &Context) {
1006 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty)) {
1007 return std::make_unique<ConstantArrayExpansion>(args: AT->getElementType(),
1008 args: AT->getZExtSize());
1009 }
1010 if (const auto *RD = Ty->getAsRecordDecl()) {
1011 SmallVector<const CXXBaseSpecifier *, 1> Bases;
1012 SmallVector<const FieldDecl *, 1> Fields;
1013 assert(!RD->hasFlexibleArrayMember() &&
1014 "Cannot expand structure with flexible array.");
1015 if (RD->isUnion()) {
1016 // Unions can be here only in degenerative cases - all the fields are same
1017 // after flattening. Thus we have to use the "largest" field.
1018 const FieldDecl *LargestFD = nullptr;
1019 CharUnits UnionSize = CharUnits::Zero();
1020
1021 for (const auto *FD : RD->fields()) {
1022 if (FD->isZeroLengthBitField())
1023 continue;
1024 assert(!FD->isBitField() &&
1025 "Cannot expand structure with bit-field members.");
1026 CharUnits FieldSize = Context.getTypeSizeInChars(T: FD->getType());
1027 if (UnionSize < FieldSize) {
1028 UnionSize = FieldSize;
1029 LargestFD = FD;
1030 }
1031 }
1032 if (LargestFD)
1033 Fields.push_back(Elt: LargestFD);
1034 } else {
1035 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
1036 assert(!CXXRD->isDynamicClass() &&
1037 "cannot expand vtable pointers in dynamic classes");
1038 llvm::append_range(C&: Bases, R: llvm::make_pointer_range(Range: CXXRD->bases()));
1039 }
1040
1041 for (const auto *FD : RD->fields()) {
1042 if (FD->isZeroLengthBitField())
1043 continue;
1044 assert(!FD->isBitField() &&
1045 "Cannot expand structure with bit-field members.");
1046 Fields.push_back(Elt: FD);
1047 }
1048 }
1049 return std::make_unique<RecordExpansion>(args: std::move(Bases),
1050 args: std::move(Fields));
1051 }
1052 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
1053 return std::make_unique<ComplexExpansion>(args: CT->getElementType());
1054 }
1055 return std::make_unique<NoExpansion>();
1056}
1057
1058static int getExpansionSize(QualType Ty, const ASTContext &Context) {
1059 auto Exp = getTypeExpansion(Ty, Context);
1060 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1061 return CAExp->NumElts * getExpansionSize(Ty: CAExp->EltTy, Context);
1062 }
1063 if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1064 int Res = 0;
1065 for (auto BS : RExp->Bases)
1066 Res += getExpansionSize(Ty: BS->getType(), Context);
1067 for (auto FD : RExp->Fields)
1068 Res += getExpansionSize(Ty: FD->getType(), Context);
1069 return Res;
1070 }
1071 if (isa<ComplexExpansion>(Val: Exp.get()))
1072 return 2;
1073 assert(isa<NoExpansion>(Exp.get()));
1074 return 1;
1075}
1076
1077void CodeGenTypes::getExpandedTypes(
1078 QualType Ty, SmallVectorImpl<llvm::Type *>::iterator &TI) {
1079 auto Exp = getTypeExpansion(Ty, Context);
1080 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1081 for (int i = 0, n = CAExp->NumElts; i < n; i++) {
1082 getExpandedTypes(Ty: CAExp->EltTy, TI);
1083 }
1084 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1085 for (auto BS : RExp->Bases)
1086 getExpandedTypes(Ty: BS->getType(), TI);
1087 for (auto FD : RExp->Fields)
1088 getExpandedTypes(Ty: FD->getType(), TI);
1089 } else if (auto CExp = dyn_cast<ComplexExpansion>(Val: Exp.get())) {
1090 llvm::Type *EltTy = ConvertType(T: CExp->EltTy);
1091 *TI++ = EltTy;
1092 *TI++ = EltTy;
1093 } else {
1094 assert(isa<NoExpansion>(Exp.get()));
1095 *TI++ = ConvertType(T: Ty);
1096 }
1097}
1098
1099static void forConstantArrayExpansion(CodeGenFunction &CGF,
1100 ConstantArrayExpansion *CAE,
1101 Address BaseAddr,
1102 llvm::function_ref<void(Address)> Fn) {
1103 for (int i = 0, n = CAE->NumElts; i < n; i++) {
1104 Address EltAddr = CGF.Builder.CreateConstGEP2_32(Addr: BaseAddr, Idx0: 0, Idx1: i);
1105 Fn(EltAddr);
1106 }
1107}
1108
1109void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1110 llvm::Function::arg_iterator &AI) {
1111 assert(LV.isSimple() &&
1112 "Unexpected non-simple lvalue during struct expansion.");
1113
1114 auto Exp = getTypeExpansion(Ty, Context: getContext());
1115 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1116 forConstantArrayExpansion(
1117 CGF&: *this, CAE: CAExp, BaseAddr: LV.getAddress(), Fn: [&](Address EltAddr) {
1118 LValue LV = MakeAddrLValue(Addr: EltAddr, T: CAExp->EltTy);
1119 ExpandTypeFromArgs(Ty: CAExp->EltTy, LV, AI);
1120 });
1121 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1122 Address This = LV.getAddress();
1123 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1124 // Perform a single step derived-to-base conversion.
1125 Address Base =
1126 GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1127 /*NullCheckValue=*/false, Loc: SourceLocation());
1128 LValue SubLV = MakeAddrLValue(Addr: Base, T: BS->getType());
1129
1130 // Recurse onto bases.
1131 ExpandTypeFromArgs(Ty: BS->getType(), LV: SubLV, AI);
1132 }
1133 for (auto FD : RExp->Fields) {
1134 // FIXME: What are the right qualifiers here?
1135 LValue SubLV = EmitLValueForFieldInitialization(Base: LV, Field: FD);
1136 ExpandTypeFromArgs(Ty: FD->getType(), LV: SubLV, AI);
1137 }
1138 } else if (isa<ComplexExpansion>(Val: Exp.get())) {
1139 auto realValue = &*AI++;
1140 auto imagValue = &*AI++;
1141 EmitStoreOfComplex(V: ComplexPairTy(realValue, imagValue), dest: LV, /*init*/ isInit: true);
1142 } else {
1143 // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1144 // primitive store.
1145 assert(isa<NoExpansion>(Exp.get()));
1146 llvm::Value *Arg = &*AI++;
1147 if (LV.isBitField()) {
1148 EmitStoreThroughLValue(Src: RValue::get(V: Arg), Dst: LV);
1149 } else {
1150 // TODO: currently there are some places are inconsistent in what LLVM
1151 // pointer type they use (see D118744). Once clang uses opaque pointers
1152 // all LLVM pointer types will be the same and we can remove this check.
1153 if (Arg->getType()->isPointerTy()) {
1154 Address Addr = LV.getAddress();
1155 Arg = Builder.CreateBitCast(V: Arg, DestTy: Addr.getElementType());
1156 }
1157 EmitStoreOfScalar(value: Arg, lvalue: LV);
1158 }
1159 }
1160}
1161
1162void CodeGenFunction::ExpandTypeToArgs(
1163 QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1164 SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1165 auto Exp = getTypeExpansion(Ty, Context: getContext());
1166 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1167 Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1168 : Arg.getKnownRValue().getAggregateAddress();
1169 forConstantArrayExpansion(CGF&: *this, CAE: CAExp, BaseAddr: Addr, Fn: [&](Address EltAddr) {
1170 CallArg EltArg =
1171 CallArg(convertTempToRValue(addr: EltAddr, type: CAExp->EltTy, Loc: SourceLocation()),
1172 CAExp->EltTy);
1173 ExpandTypeToArgs(Ty: CAExp->EltTy, Arg: EltArg, IRFuncTy, IRCallArgs,
1174 IRCallArgPos);
1175 });
1176 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1177 Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1178 : Arg.getKnownRValue().getAggregateAddress();
1179 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1180 // Perform a single step derived-to-base conversion.
1181 Address Base =
1182 GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1183 /*NullCheckValue=*/false, Loc: SourceLocation());
1184 CallArg BaseArg = CallArg(RValue::getAggregate(addr: Base), BS->getType());
1185
1186 // Recurse onto bases.
1187 ExpandTypeToArgs(Ty: BS->getType(), Arg: BaseArg, IRFuncTy, IRCallArgs,
1188 IRCallArgPos);
1189 }
1190
1191 LValue LV = MakeAddrLValue(Addr: This, T: Ty);
1192 for (auto FD : RExp->Fields) {
1193 CallArg FldArg =
1194 CallArg(EmitRValueForField(LV, FD, Loc: SourceLocation()), FD->getType());
1195 ExpandTypeToArgs(Ty: FD->getType(), Arg: FldArg, IRFuncTy, IRCallArgs,
1196 IRCallArgPos);
1197 }
1198 } else if (isa<ComplexExpansion>(Val: Exp.get())) {
1199 ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1200 IRCallArgs[IRCallArgPos++] = CV.first;
1201 IRCallArgs[IRCallArgPos++] = CV.second;
1202 } else {
1203 assert(isa<NoExpansion>(Exp.get()));
1204 auto RV = Arg.getKnownRValue();
1205 assert(RV.isScalar() &&
1206 "Unexpected non-scalar rvalue during struct expansion.");
1207
1208 // Insert a bitcast as needed.
1209 llvm::Value *V = RV.getScalarVal();
1210 if (IRCallArgPos < IRFuncTy->getNumParams() &&
1211 V->getType() != IRFuncTy->getParamType(i: IRCallArgPos))
1212 V = Builder.CreateBitCast(V, DestTy: IRFuncTy->getParamType(i: IRCallArgPos));
1213
1214 IRCallArgs[IRCallArgPos++] = V;
1215 }
1216}
1217
1218/// Create a temporary allocation for the purposes of coercion.
1219static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF,
1220 llvm::Type *Ty,
1221 CharUnits MinAlign,
1222 const Twine &Name = "tmp") {
1223 // Don't use an alignment that's worse than what LLVM would prefer.
1224 auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty);
1225 CharUnits Align = std::max(a: MinAlign, b: CharUnits::fromQuantity(Quantity: PrefAlign));
1226
1227 return CGF.CreateTempAlloca(Ty, align: Align, Name: Name + ".coerce");
1228}
1229
1230/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1231/// accessing some number of bytes out of it, try to gep into the struct to get
1232/// at its inner goodness. Dive as deep as possible without entering an element
1233/// with an in-memory size smaller than DstSize.
1234static Address EnterStructPointerForCoercedAccess(Address SrcPtr,
1235 llvm::StructType *SrcSTy,
1236 uint64_t DstSize,
1237 CodeGenFunction &CGF) {
1238 // We can't dive into a zero-element struct.
1239 if (SrcSTy->getNumElements() == 0)
1240 return SrcPtr;
1241
1242 llvm::Type *FirstElt = SrcSTy->getElementType(N: 0);
1243
1244 // If the first elt is at least as large as what we're looking for, or if the
1245 // first element is the same size as the whole struct, we can enter it. The
1246 // comparison must be made on the store size and not the alloca size. Using
1247 // the alloca size may overstate the size of the load.
1248 uint64_t FirstEltSize = CGF.CGM.getDataLayout().getTypeStoreSize(Ty: FirstElt);
1249 if (FirstEltSize < DstSize &&
1250 FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(Ty: SrcSTy))
1251 return SrcPtr;
1252
1253 // GEP into the first element.
1254 SrcPtr = CGF.Builder.CreateStructGEP(Addr: SrcPtr, Index: 0, Name: "coerce.dive");
1255
1256 // If the first element is a struct, recurse.
1257 llvm::Type *SrcTy = SrcPtr.getElementType();
1258 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy))
1259 return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1260
1261 return SrcPtr;
1262}
1263
1264/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1265/// are either integers or pointers. This does a truncation of the value if it
1266/// is too large or a zero extension if it is too small.
1267///
1268/// This behaves as if the value were coerced through memory, so on big-endian
1269/// targets the high bits are preserved in a truncation, while little-endian
1270/// targets preserve the low bits.
1271static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, llvm::Type *Ty,
1272 CodeGenFunction &CGF) {
1273 if (Val->getType() == Ty)
1274 return Val;
1275
1276 if (isa<llvm::PointerType>(Val: Val->getType())) {
1277 // If this is Pointer->Pointer avoid conversion to and from int.
1278 if (isa<llvm::PointerType>(Val: Ty))
1279 return CGF.Builder.CreateBitCast(V: Val, DestTy: Ty, Name: "coerce.val");
1280
1281 // Convert the pointer to an integer so we can play with its width.
1282 Val = CGF.Builder.CreatePtrToInt(V: Val, DestTy: CGF.IntPtrTy, Name: "coerce.val.pi");
1283 }
1284
1285 llvm::Type *DestIntTy = Ty;
1286 if (isa<llvm::PointerType>(Val: DestIntTy))
1287 DestIntTy = CGF.IntPtrTy;
1288
1289 if (Val->getType() != DestIntTy) {
1290 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1291 if (DL.isBigEndian()) {
1292 // Preserve the high bits on big-endian targets.
1293 // That is what memory coercion does.
1294 uint64_t SrcSize = DL.getTypeSizeInBits(Ty: Val->getType());
1295 uint64_t DstSize = DL.getTypeSizeInBits(Ty: DestIntTy);
1296
1297 if (SrcSize > DstSize) {
1298 Val = CGF.Builder.CreateLShr(LHS: Val, RHS: SrcSize - DstSize, Name: "coerce.highbits");
1299 Val = CGF.Builder.CreateTrunc(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1300 } else {
1301 Val = CGF.Builder.CreateZExt(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1302 Val = CGF.Builder.CreateShl(LHS: Val, RHS: DstSize - SrcSize, Name: "coerce.highbits");
1303 }
1304 } else {
1305 // Little-endian targets preserve the low bits. No shifts required.
1306 Val = CGF.Builder.CreateIntCast(V: Val, DestTy: DestIntTy, isSigned: false, Name: "coerce.val.ii");
1307 }
1308 }
1309
1310 if (isa<llvm::PointerType>(Val: Ty))
1311 Val = CGF.Builder.CreateIntToPtr(V: Val, DestTy: Ty, Name: "coerce.val.ip");
1312 return Val;
1313}
1314
1315static llvm::Value *CreatePFPCoercedLoad(Address Src, QualType SrcFETy,
1316 llvm::Type *Ty, CodeGenFunction &CGF) {
1317 std::vector<PFPField> PFPFields = CGF.getContext().findPFPFields(Ty: SrcFETy);
1318 if (PFPFields.empty())
1319 return nullptr;
1320
1321 auto LoadCoercedField = [&](CharUnits Offset,
1322 llvm::Type *FieldType) -> llvm::Value * {
1323 // Check whether the field at Offset is a PFP field. This function is called
1324 // in ascending order of offset, and PFPFields is sorted by offset. This
1325 // means that we only need to check the first element (and remove it from
1326 // PFPFields if matching).
1327 if (!PFPFields.empty() && PFPFields[0].Offset == Offset) {
1328 auto FieldAddr = CGF.EmitAddressOfPFPField(RecordPtr: Src, Field: PFPFields[0]);
1329 llvm::Value *FieldVal = CGF.Builder.CreateLoad(Addr: FieldAddr);
1330 if (isa<llvm::IntegerType>(Val: FieldType))
1331 FieldVal = CGF.Builder.CreatePtrToInt(V: FieldVal, DestTy: FieldType);
1332 PFPFields.erase(position: PFPFields.begin());
1333 return FieldVal;
1334 }
1335 auto FieldAddr =
1336 CGF.Builder
1337 .CreateConstInBoundsByteGEP(Addr: Src.withElementType(ElemTy: CGF.Int8Ty), Offset)
1338 .withElementType(ElemTy: FieldType);
1339 return CGF.Builder.CreateLoad(Addr: FieldAddr);
1340 };
1341
1342 // The types handled by this function are the only ones that may be generated
1343 // by AArch64ABIInfo::classify{Argument,Return}Type for struct types with
1344 // pointers. PFP is only supported on AArch64.
1345 if (isa<llvm::IntegerType>(Val: Ty) || isa<llvm::PointerType>(Val: Ty)) {
1346 auto Addr = CGF.EmitAddressOfPFPField(RecordPtr: Src, Field: PFPFields[0]);
1347 llvm::Value *Val = CGF.Builder.CreateLoad(Addr);
1348 if (isa<llvm::IntegerType>(Val: Ty))
1349 Val = CGF.Builder.CreatePtrToInt(V: Val, DestTy: Ty);
1350 return Val;
1351 }
1352 auto *AT = cast<llvm::ArrayType>(Val: Ty);
1353 auto *ET = AT->getElementType();
1354 CharUnits WordSize = CGF.getContext().toCharUnitsFromBits(
1355 BitSize: CGF.CGM.getDataLayout().getTypeSizeInBits(Ty: ET));
1356 CharUnits Offset = CharUnits::Zero();
1357 llvm::Value *Val = llvm::PoisonValue::get(T: AT);
1358 for (unsigned Idx = 0; Idx != AT->getNumElements(); ++Idx, Offset += WordSize)
1359 Val = CGF.Builder.CreateInsertValue(Agg: Val, Val: LoadCoercedField(Offset, ET), Idxs: Idx);
1360 return Val;
1361}
1362
1363/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1364/// a pointer to an object of type \arg Ty, known to be aligned to
1365/// \arg SrcAlign bytes.
1366///
1367/// This safely handles the case when the src type is smaller than the
1368/// destination type; in this situation the values of bits which not
1369/// present in the src are undefined.
1370static llvm::Value *CreateCoercedLoad(Address Src, QualType SrcFETy,
1371 llvm::Type *Ty, CodeGenFunction &CGF) {
1372 llvm::Type *SrcTy = Src.getElementType();
1373
1374 // If SrcTy and Ty are the same, just do a load.
1375 if (SrcTy == Ty)
1376 return CGF.Builder.CreateLoad(Addr: Src);
1377
1378 if (llvm::Value *V = CreatePFPCoercedLoad(Src, SrcFETy, Ty, CGF))
1379 return V;
1380
1381 llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1382
1383 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy)) {
1384 Src = EnterStructPointerForCoercedAccess(SrcPtr: Src, SrcSTy,
1385 DstSize: DstSize.getFixedValue(), CGF);
1386 SrcTy = Src.getElementType();
1387 }
1388
1389 llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1390
1391 // If the source and destination are integer or pointer types, just do an
1392 // extension or truncation to the desired type.
1393 if ((isa<llvm::IntegerType>(Val: Ty) || isa<llvm::PointerType>(Val: Ty)) &&
1394 (isa<llvm::IntegerType>(Val: SrcTy) || isa<llvm::PointerType>(Val: SrcTy))) {
1395 llvm::Value *Load = CGF.Builder.CreateLoad(Addr: Src);
1396 return CoerceIntOrPtrToIntOrPtr(Val: Load, Ty, CGF);
1397 }
1398
1399 // If load is legal, just bitcast the src pointer.
1400 if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1401 SrcSize.getFixedValue() >= DstSize.getFixedValue()) {
1402 // Generally SrcSize is never greater than DstSize, since this means we are
1403 // losing bits. However, this can happen in cases where the structure has
1404 // additional padding, for example due to a user specified alignment.
1405 //
1406 // FIXME: Assert that we aren't truncating non-padding bits when have access
1407 // to that information.
1408 Src = Src.withElementType(ElemTy: Ty);
1409 return CGF.Builder.CreateLoad(Addr: Src);
1410 }
1411
1412 // If coercing a fixed vector to a scalable vector for ABI compatibility, and
1413 // the types match, use the llvm.vector.insert intrinsic to perform the
1414 // conversion.
1415 if (auto *ScalableDstTy = dyn_cast<llvm::ScalableVectorType>(Val: Ty)) {
1416 if (auto *FixedSrcTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy)) {
1417 // If we are casting a fixed i8 vector to a scalable i1 predicate
1418 // vector, use a vector insert and bitcast the result.
1419 if (ScalableDstTy->getElementType()->isIntegerTy(Bitwidth: 1) &&
1420 FixedSrcTy->getElementType()->isIntegerTy(Bitwidth: 8)) {
1421 ScalableDstTy = llvm::ScalableVectorType::get(
1422 ElementType: FixedSrcTy->getElementType(),
1423 MinNumElts: llvm::divideCeil(
1424 Numerator: ScalableDstTy->getElementCount().getKnownMinValue(), Denominator: 8));
1425 }
1426 if (ScalableDstTy->getElementType() == FixedSrcTy->getElementType()) {
1427 auto *Load = CGF.Builder.CreateLoad(Addr: Src);
1428 auto *PoisonVec = llvm::PoisonValue::get(T: ScalableDstTy);
1429 llvm::Value *Result = CGF.Builder.CreateInsertVector(
1430 DstType: ScalableDstTy, SrcVec: PoisonVec, SubVec: Load, Idx: uint64_t(0), Name: "cast.scalable");
1431 ScalableDstTy = cast<llvm::ScalableVectorType>(
1432 Val: llvm::VectorType::getWithSizeAndScalar(SizeTy: ScalableDstTy, EltTy: Ty));
1433 if (Result->getType() != ScalableDstTy)
1434 Result = CGF.Builder.CreateBitCast(V: Result, DestTy: ScalableDstTy);
1435 if (Result->getType() != Ty)
1436 Result = CGF.Builder.CreateExtractVector(DstType: Ty, SrcVec: Result, Idx: uint64_t(0));
1437 return Result;
1438 }
1439 }
1440 }
1441
1442 // Otherwise do coercion through memory. This is stupid, but simple.
1443 RawAddress Tmp =
1444 CreateTempAllocaForCoercion(CGF, Ty, MinAlign: Src.getAlignment(), Name: Src.getName());
1445 CGF.Builder.CreateMemCpy(
1446 Dst: Tmp.getPointer(), DstAlign: Tmp.getAlignment().getAsAlign(),
1447 Src: Src.emitRawPointer(CGF), SrcAlign: Src.getAlignment().getAsAlign(),
1448 Size: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: SrcSize.getKnownMinValue()));
1449 return CGF.Builder.CreateLoad(Addr: Tmp);
1450}
1451
1452static bool CreatePFPCoercedStore(llvm::Value *Src, QualType SrcFETy,
1453 Address Dst, CodeGenFunction &CGF) {
1454 std::vector<PFPField> PFPFields = CGF.getContext().findPFPFields(Ty: SrcFETy);
1455 if (PFPFields.empty())
1456 return false;
1457
1458 llvm::Type *SrcTy = Src->getType();
1459 auto StoreCoercedField = [&](CharUnits Offset, llvm::Value *FieldVal) {
1460 if (!PFPFields.empty() && PFPFields[0].Offset == Offset) {
1461 auto FieldAddr = CGF.EmitAddressOfPFPField(RecordPtr: Dst, Field: PFPFields[0]);
1462 if (isa<llvm::IntegerType>(Val: FieldVal->getType()))
1463 FieldVal = CGF.Builder.CreateIntToPtr(V: FieldVal, DestTy: CGF.VoidPtrTy);
1464 CGF.Builder.CreateStore(Val: FieldVal, Addr: FieldAddr);
1465 PFPFields.erase(position: PFPFields.begin());
1466 } else {
1467 auto FieldAddr = CGF.Builder
1468 .CreateConstInBoundsByteGEP(
1469 Addr: Dst.withElementType(ElemTy: CGF.Int8Ty), Offset)
1470 .withElementType(ElemTy: FieldVal->getType());
1471 CGF.Builder.CreateStore(Val: FieldVal, Addr: FieldAddr);
1472 }
1473 };
1474
1475 // The types handled by this function are the only ones that may be generated
1476 // by AArch64ABIInfo::classify{Argument,Return}Type for struct types with
1477 // pointers. PFP is only supported on AArch64.
1478 if (isa<llvm::IntegerType>(Val: SrcTy) || isa<llvm::PointerType>(Val: SrcTy)) {
1479 if (isa<llvm::IntegerType>(Val: SrcTy))
1480 Src = CGF.Builder.CreateIntToPtr(V: Src, DestTy: CGF.VoidPtrTy);
1481 auto Addr = CGF.EmitAddressOfPFPField(RecordPtr: Dst, Field: PFPFields[0]);
1482 CGF.Builder.CreateStore(Val: Src, Addr);
1483 } else {
1484 auto *AT = cast<llvm::ArrayType>(Val: SrcTy);
1485 auto *ET = AT->getElementType();
1486 CharUnits WordSize = CGF.getContext().toCharUnitsFromBits(
1487 BitSize: CGF.CGM.getDataLayout().getTypeSizeInBits(Ty: ET));
1488 CharUnits Offset = CharUnits::Zero();
1489 for (unsigned i = 0; i != AT->getNumElements(); ++i, Offset += WordSize)
1490 StoreCoercedField(Offset, CGF.Builder.CreateExtractValue(Agg: Src, Idxs: i));
1491 }
1492 return true;
1493}
1494
1495void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, QualType SrcFETy,
1496 Address Dst, llvm::TypeSize DstSize,
1497 bool DstIsVolatile) {
1498 if (!DstSize)
1499 return;
1500
1501 llvm::Type *SrcTy = Src->getType();
1502 llvm::TypeSize SrcSize = CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1503
1504 // GEP into structs to try to make types match.
1505 // FIXME: This isn't really that useful with opaque types, but it impacts a
1506 // lot of regression tests.
1507 if (SrcTy != Dst.getElementType()) {
1508 if (llvm::StructType *DstSTy =
1509 dyn_cast<llvm::StructType>(Val: Dst.getElementType())) {
1510 assert(!SrcSize.isScalable());
1511 Dst = EnterStructPointerForCoercedAccess(SrcPtr: Dst, SrcSTy: DstSTy,
1512 DstSize: SrcSize.getFixedValue(), CGF&: *this);
1513 }
1514 }
1515
1516 if (CreatePFPCoercedStore(Src, SrcFETy, Dst, CGF&: *this))
1517 return;
1518
1519 if (SrcSize.isScalable() || SrcSize <= DstSize) {
1520 if (SrcTy->isIntegerTy() && Dst.getElementType()->isPointerTy() &&
1521 SrcSize == CGM.getDataLayout().getTypeAllocSize(Ty: Dst.getElementType())) {
1522 // If the value is supposed to be a pointer, convert it before storing it.
1523 Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: Dst.getElementType(), CGF&: *this);
1524 auto *I = Builder.CreateStore(Val: Src, Addr: Dst, IsVolatile: DstIsVolatile);
1525 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1526 } else if (llvm::StructType *STy =
1527 dyn_cast<llvm::StructType>(Val: Src->getType())) {
1528 // Prefer scalar stores to first-class aggregate stores.
1529 Dst = Dst.withElementType(ElemTy: SrcTy);
1530 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1531 Address EltPtr = Builder.CreateStructGEP(Addr: Dst, Index: i);
1532 llvm::Value *Elt = Builder.CreateExtractValue(Agg: Src, Idxs: i);
1533 auto *I = Builder.CreateStore(Val: Elt, Addr: EltPtr, IsVolatile: DstIsVolatile);
1534 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Elt);
1535 }
1536 } else {
1537 auto *I =
1538 Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: SrcTy), IsVolatile: DstIsVolatile);
1539 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1540 }
1541 } else if (SrcTy->isIntegerTy()) {
1542 // If the source is a simple integer, coerce it directly.
1543 llvm::Type *DstIntTy = Builder.getIntNTy(N: DstSize.getFixedValue() * 8);
1544 Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: DstIntTy, CGF&: *this);
1545 auto *I =
1546 Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: DstIntTy), IsVolatile: DstIsVolatile);
1547 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1548 } else {
1549 // Otherwise do coercion through memory. This is stupid, but
1550 // simple.
1551
1552 // Generally SrcSize is never greater than DstSize, since this means we are
1553 // losing bits. However, this can happen in cases where the structure has
1554 // additional padding, for example due to a user specified alignment.
1555 //
1556 // FIXME: Assert that we aren't truncating non-padding bits when have access
1557 // to that information.
1558 RawAddress Tmp =
1559 CreateTempAllocaForCoercion(CGF&: *this, Ty: SrcTy, MinAlign: Dst.getAlignment());
1560 Builder.CreateStore(Val: Src, Addr: Tmp);
1561 auto *I = Builder.CreateMemCpy(
1562 Dst: Dst.emitRawPointer(CGF&: *this), DstAlign: Dst.getAlignment().getAsAlign(),
1563 Src: Tmp.getPointer(), SrcAlign: Tmp.getAlignment().getAsAlign(),
1564 Size: Builder.CreateTypeSize(Ty: IntPtrTy, Size: DstSize));
1565 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1566 }
1567}
1568
1569static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1570 const ABIArgInfo &info) {
1571 if (unsigned offset = info.getDirectOffset()) {
1572 addr = addr.withElementType(ElemTy: CGF.Int8Ty);
1573 addr = CGF.Builder.CreateConstInBoundsByteGEP(
1574 Addr: addr, Offset: CharUnits::fromQuantity(Quantity: offset));
1575 addr = addr.withElementType(ElemTy: info.getCoerceToType());
1576 }
1577 return addr;
1578}
1579
1580static std::pair<llvm::Value *, bool>
1581CoerceScalableToFixed(CodeGenFunction &CGF, llvm::FixedVectorType *ToTy,
1582 llvm::ScalableVectorType *FromTy, llvm::Value *V,
1583 StringRef Name = "") {
1584 // If we are casting a scalable i1 predicate vector to a fixed i8
1585 // vector, first bitcast the source.
1586 if (FromTy->getElementType()->isIntegerTy(Bitwidth: 1) &&
1587 ToTy->getElementType() == CGF.Builder.getInt8Ty()) {
1588 if (!FromTy->getElementCount().isKnownMultipleOf(RHS: 8)) {
1589 FromTy = llvm::ScalableVectorType::get(
1590 ElementType: FromTy->getElementType(),
1591 MinNumElts: llvm::alignTo<8>(Value: FromTy->getElementCount().getKnownMinValue()));
1592 llvm::Value *ZeroVec = llvm::Constant::getNullValue(Ty: FromTy);
1593 V = CGF.Builder.CreateInsertVector(DstType: FromTy, SrcVec: ZeroVec, SubVec: V, Idx: uint64_t(0));
1594 }
1595 FromTy = llvm::ScalableVectorType::get(
1596 ElementType: ToTy->getElementType(),
1597 MinNumElts: FromTy->getElementCount().getKnownMinValue() / 8);
1598 V = CGF.Builder.CreateBitCast(V, DestTy: FromTy);
1599 }
1600 if (FromTy->getElementType() == ToTy->getElementType()) {
1601 V->setName(Name + ".coerce");
1602 V = CGF.Builder.CreateExtractVector(DstType: ToTy, SrcVec: V, Idx: uint64_t(0), Name: "cast.fixed");
1603 return {V, true};
1604 }
1605 return {V, false};
1606}
1607
1608namespace {
1609
1610/// Encapsulates information about the way function arguments from
1611/// CGFunctionInfo should be passed to actual LLVM IR function.
1612class ClangToLLVMArgMapping {
1613 static const unsigned InvalidIndex = ~0U;
1614 unsigned InallocaArgNo;
1615 unsigned SRetArgNo;
1616 unsigned TotalIRArgs;
1617
1618 /// Arguments of LLVM IR function corresponding to single Clang argument.
1619 struct IRArgs {
1620 unsigned PaddingArgIndex;
1621 // Argument is expanded to IR arguments at positions
1622 // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1623 unsigned FirstArgIndex;
1624 unsigned NumberOfArgs;
1625
1626 IRArgs()
1627 : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1628 NumberOfArgs(0) {}
1629 };
1630
1631 SmallVector<IRArgs, 8> ArgInfo;
1632
1633public:
1634 ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1635 bool OnlyRequiredArgs = false)
1636 : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1637 ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1638 construct(Context, FI, OnlyRequiredArgs);
1639 }
1640
1641 bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1642 unsigned getInallocaArgNo() const {
1643 assert(hasInallocaArg());
1644 return InallocaArgNo;
1645 }
1646
1647 bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1648 unsigned getSRetArgNo() const {
1649 assert(hasSRetArg());
1650 return SRetArgNo;
1651 }
1652
1653 unsigned totalIRArgs() const { return TotalIRArgs; }
1654
1655 bool hasPaddingArg(unsigned ArgNo) const {
1656 assert(ArgNo < ArgInfo.size());
1657 return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1658 }
1659 unsigned getPaddingArgNo(unsigned ArgNo) const {
1660 assert(hasPaddingArg(ArgNo));
1661 return ArgInfo[ArgNo].PaddingArgIndex;
1662 }
1663
1664 /// Returns index of first IR argument corresponding to ArgNo, and their
1665 /// quantity.
1666 std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1667 assert(ArgNo < ArgInfo.size());
1668 return std::make_pair(x: ArgInfo[ArgNo].FirstArgIndex,
1669 y: ArgInfo[ArgNo].NumberOfArgs);
1670 }
1671
1672private:
1673 void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1674 bool OnlyRequiredArgs);
1675};
1676
1677void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1678 const CGFunctionInfo &FI,
1679 bool OnlyRequiredArgs) {
1680 unsigned IRArgNo = 0;
1681 bool SwapThisWithSRet = false;
1682 const ABIArgInfo &RetAI = FI.getReturnInfo();
1683
1684 if (RetAI.getKind() == ABIArgInfo::Indirect) {
1685 SwapThisWithSRet = RetAI.isSRetAfterThis();
1686 SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1687 }
1688
1689 unsigned ArgNo = 0;
1690 unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();
1691 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1692 ++I, ++ArgNo) {
1693 assert(I != FI.arg_end());
1694 QualType ArgType = I->type;
1695 const ABIArgInfo &AI = I->info;
1696 // Collect data about IR arguments corresponding to Clang argument ArgNo.
1697 auto &IRArgs = ArgInfo[ArgNo];
1698
1699 if (AI.getPaddingType())
1700 IRArgs.PaddingArgIndex = IRArgNo++;
1701
1702 switch (AI.getKind()) {
1703 case ABIArgInfo::TargetSpecific:
1704 case ABIArgInfo::Extend:
1705 case ABIArgInfo::Direct: {
1706 // FIXME: handle sseregparm someday...
1707 llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: AI.getCoerceToType());
1708 if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
1709 IRArgs.NumberOfArgs = STy->getNumElements();
1710 } else {
1711 IRArgs.NumberOfArgs = 1;
1712 }
1713 break;
1714 }
1715 case ABIArgInfo::Indirect:
1716 case ABIArgInfo::IndirectAliased:
1717 IRArgs.NumberOfArgs = 1;
1718 break;
1719 case ABIArgInfo::Ignore:
1720 case ABIArgInfo::InAlloca:
1721 // ignore and inalloca doesn't have matching LLVM parameters.
1722 IRArgs.NumberOfArgs = 0;
1723 break;
1724 case ABIArgInfo::CoerceAndExpand:
1725 IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1726 break;
1727 case ABIArgInfo::Expand:
1728 IRArgs.NumberOfArgs = getExpansionSize(Ty: ArgType, Context);
1729 break;
1730 }
1731
1732 if (IRArgs.NumberOfArgs > 0) {
1733 IRArgs.FirstArgIndex = IRArgNo;
1734 IRArgNo += IRArgs.NumberOfArgs;
1735 }
1736
1737 // Skip over the sret parameter when it comes second. We already handled it
1738 // above.
1739 if (IRArgNo == 1 && SwapThisWithSRet)
1740 IRArgNo++;
1741 }
1742 assert(ArgNo == ArgInfo.size());
1743
1744 if (FI.usesInAlloca())
1745 InallocaArgNo = IRArgNo++;
1746
1747 TotalIRArgs = IRArgNo;
1748}
1749} // namespace
1750
1751/***/
1752
1753bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1754 const auto &RI = FI.getReturnInfo();
1755 return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());
1756}
1757
1758bool CodeGenModule::ReturnTypeHasInReg(const CGFunctionInfo &FI) {
1759 const auto &RI = FI.getReturnInfo();
1760 return RI.getInReg();
1761}
1762
1763bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1764 return ReturnTypeUsesSRet(FI) &&
1765 getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
1766}
1767
1768bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1769 if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1770 switch (BT->getKind()) {
1771 default:
1772 return false;
1773 case BuiltinType::Float:
1774 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Float);
1775 case BuiltinType::Double:
1776 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Double);
1777 case BuiltinType::LongDouble:
1778 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::LongDouble);
1779 }
1780 }
1781
1782 return false;
1783}
1784
1785bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1786 if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1787 if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1788 if (BT->getKind() == BuiltinType::LongDouble)
1789 return getTarget().useObjCFP2RetForComplexLongDouble();
1790 }
1791 }
1792
1793 return false;
1794}
1795
1796llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1797 const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1798 return GetFunctionType(Info: FI);
1799}
1800
1801llvm::FunctionType *CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1802
1803 bool Inserted = FunctionsBeingProcessed.insert(Ptr: &FI).second;
1804 (void)Inserted;
1805 assert(Inserted && "Recursively being processed?");
1806
1807 llvm::Type *resultType = nullptr;
1808 const ABIArgInfo &retAI = FI.getReturnInfo();
1809 switch (retAI.getKind()) {
1810 case ABIArgInfo::Expand:
1811 case ABIArgInfo::IndirectAliased:
1812 llvm_unreachable("Invalid ABI kind for return argument");
1813
1814 case ABIArgInfo::TargetSpecific:
1815 case ABIArgInfo::Extend:
1816 case ABIArgInfo::Direct:
1817 resultType = retAI.getCoerceToType();
1818 break;
1819
1820 case ABIArgInfo::InAlloca:
1821 if (retAI.getInAllocaSRet()) {
1822 // sret things on win32 aren't void, they return the sret pointer.
1823 QualType ret = FI.getReturnType();
1824 unsigned addressSpace = CGM.getTypes().getTargetAddressSpace(T: ret);
1825 resultType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: addressSpace);
1826 } else {
1827 resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
1828 }
1829 break;
1830
1831 case ABIArgInfo::Indirect:
1832 case ABIArgInfo::Ignore:
1833 resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
1834 break;
1835
1836 case ABIArgInfo::CoerceAndExpand:
1837 resultType = retAI.getUnpaddedCoerceAndExpandType();
1838 break;
1839 }
1840
1841 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1842 SmallVector<llvm::Type *, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1843
1844 // Add type for sret argument.
1845 if (IRFunctionArgs.hasSRetArg()) {
1846 ArgTypes[IRFunctionArgs.getSRetArgNo()] = llvm::PointerType::get(
1847 C&: getLLVMContext(), AddressSpace: FI.getReturnInfo().getIndirectAddrSpace());
1848 }
1849
1850 // Add type for inalloca argument.
1851 if (IRFunctionArgs.hasInallocaArg())
1852 ArgTypes[IRFunctionArgs.getInallocaArgNo()] =
1853 llvm::PointerType::getUnqual(C&: getLLVMContext());
1854
1855 // Add in all of the required arguments.
1856 unsigned ArgNo = 0;
1857 CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1858 ie = it + FI.getNumRequiredArgs();
1859 for (; it != ie; ++it, ++ArgNo) {
1860 const ABIArgInfo &ArgInfo = it->info;
1861
1862 // Insert a padding type to ensure proper alignment.
1863 if (IRFunctionArgs.hasPaddingArg(ArgNo))
1864 ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1865 ArgInfo.getPaddingType();
1866
1867 unsigned FirstIRArg, NumIRArgs;
1868 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1869
1870 switch (ArgInfo.getKind()) {
1871 case ABIArgInfo::Ignore:
1872 case ABIArgInfo::InAlloca:
1873 assert(NumIRArgs == 0);
1874 break;
1875
1876 case ABIArgInfo::Indirect:
1877 assert(NumIRArgs == 1);
1878 // indirect arguments are always on the stack, which is alloca addr space.
1879 ArgTypes[FirstIRArg] = llvm::PointerType::get(
1880 C&: getLLVMContext(), AddressSpace: CGM.getDataLayout().getAllocaAddrSpace());
1881 break;
1882 case ABIArgInfo::IndirectAliased:
1883 assert(NumIRArgs == 1);
1884 ArgTypes[FirstIRArg] = llvm::PointerType::get(
1885 C&: getLLVMContext(), AddressSpace: ArgInfo.getIndirectAddrSpace());
1886 break;
1887 case ABIArgInfo::TargetSpecific:
1888 case ABIArgInfo::Extend:
1889 case ABIArgInfo::Direct: {
1890 // Fast-isel and the optimizer generally like scalar values better than
1891 // FCAs, so we flatten them if this is safe to do for this argument.
1892 llvm::Type *argType = ArgInfo.getCoerceToType();
1893 llvm::StructType *st = dyn_cast<llvm::StructType>(Val: argType);
1894 if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
1895 assert(NumIRArgs == st->getNumElements());
1896 for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
1897 ArgTypes[FirstIRArg + i] = st->getElementType(N: i);
1898 } else {
1899 assert(NumIRArgs == 1);
1900 ArgTypes[FirstIRArg] = argType;
1901 }
1902 break;
1903 }
1904
1905 case ABIArgInfo::CoerceAndExpand: {
1906 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1907 for (auto *EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1908 *ArgTypesIter++ = EltTy;
1909 }
1910 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1911 break;
1912 }
1913
1914 case ABIArgInfo::Expand:
1915 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1916 getExpandedTypes(Ty: it->type, TI&: ArgTypesIter);
1917 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1918 break;
1919 }
1920 }
1921
1922 bool Erased = FunctionsBeingProcessed.erase(Ptr: &FI);
1923 (void)Erased;
1924 assert(Erased && "Not in set?");
1925
1926 return llvm::FunctionType::get(Result: resultType, Params: ArgTypes, isVarArg: FI.isVariadic());
1927}
1928
1929llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1930 const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
1931 const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
1932
1933 if (!isFuncTypeConvertible(FT: FPT))
1934 return llvm::StructType::get(Context&: getLLVMContext());
1935
1936 return GetFunctionType(GD);
1937}
1938
1939static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
1940 llvm::AttrBuilder &FuncAttrs,
1941 const FunctionProtoType *FPT) {
1942 if (!FPT)
1943 return;
1944
1945 if (!isUnresolvedExceptionSpec(ESpecType: FPT->getExceptionSpecType()) &&
1946 FPT->isNothrow())
1947 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
1948
1949 unsigned SMEBits = FPT->getAArch64SMEAttributes();
1950 if (SMEBits & FunctionType::SME_PStateSMEnabledMask)
1951 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_enabled");
1952 if (SMEBits & FunctionType::SME_PStateSMCompatibleMask)
1953 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_compatible");
1954 if (SMEBits & FunctionType::SME_AgnosticZAStateMask)
1955 FuncAttrs.addAttribute(A: "aarch64_za_state_agnostic");
1956
1957 // ZA
1958 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
1959 FuncAttrs.addAttribute(A: "aarch64_preserves_za");
1960 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_In)
1961 FuncAttrs.addAttribute(A: "aarch64_in_za");
1962 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Out)
1963 FuncAttrs.addAttribute(A: "aarch64_out_za");
1964 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_InOut)
1965 FuncAttrs.addAttribute(A: "aarch64_inout_za");
1966
1967 // ZT0
1968 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
1969 FuncAttrs.addAttribute(A: "aarch64_preserves_zt0");
1970 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_In)
1971 FuncAttrs.addAttribute(A: "aarch64_in_zt0");
1972 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Out)
1973 FuncAttrs.addAttribute(A: "aarch64_out_zt0");
1974 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_InOut)
1975 FuncAttrs.addAttribute(A: "aarch64_inout_zt0");
1976}
1977
1978static void AddAttributesFromOMPAssumes(llvm::AttrBuilder &FuncAttrs,
1979 const Decl *Callee) {
1980 if (!Callee)
1981 return;
1982
1983 SmallVector<StringRef, 4> Attrs;
1984
1985 for (const OMPAssumeAttr *AA : Callee->specific_attrs<OMPAssumeAttr>())
1986 AA->getAssumption().split(A&: Attrs, Separator: ",");
1987
1988 if (!Attrs.empty())
1989 FuncAttrs.addAttribute(A: llvm::AssumptionAttrKey,
1990 V: llvm::join(Begin: Attrs.begin(), End: Attrs.end(), Separator: ","));
1991}
1992
1993bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,
1994 QualType ReturnType) const {
1995 // We can't just discard the return value for a record type with a
1996 // complex destructor or a non-trivially copyable type.
1997 if (const RecordType *RT =
1998 ReturnType.getCanonicalType()->getAsCanonical<RecordType>()) {
1999 if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(Val: RT->getDecl()))
2000 return ClassDecl->hasTrivialDestructor();
2001 }
2002 return ReturnType.isTriviallyCopyableType(Context);
2003}
2004
2005static bool HasStrictReturn(const CodeGenModule &Module, QualType RetTy,
2006 const Decl *TargetDecl) {
2007 // As-is msan can not tolerate noundef mismatch between caller and
2008 // implementation. Mismatch is possible for e.g. indirect calls from C-caller
2009 // into C++. Such mismatches lead to confusing false reports. To avoid
2010 // expensive workaround on msan we enforce initialization event in uncommon
2011 // cases where it's allowed.
2012 if (Module.getLangOpts().Sanitize.has(K: SanitizerKind::Memory))
2013 return true;
2014 // C++ explicitly makes returning undefined values UB. C's rule only applies
2015 // to used values, so we never mark them noundef for now.
2016 if (!Module.getLangOpts().CPlusPlus)
2017 return false;
2018 if (TargetDecl) {
2019 if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2020 if (FDecl->isExternC())
2021 return false;
2022 } else if (const VarDecl *VDecl = dyn_cast<VarDecl>(Val: TargetDecl)) {
2023 // Function pointer.
2024 if (VDecl->isExternC())
2025 return false;
2026 }
2027 }
2028
2029 // We don't want to be too aggressive with the return checking, unless
2030 // it's explicit in the code opts or we're using an appropriate sanitizer.
2031 // Try to respect what the programmer intended.
2032 return Module.getCodeGenOpts().StrictReturn ||
2033 !Module.MayDropFunctionReturn(Context: Module.getContext(), ReturnType: RetTy) ||
2034 Module.getLangOpts().Sanitize.has(K: SanitizerKind::Return);
2035}
2036
2037/// Add denormal-fp-math and denormal-fp-math-f32 as appropriate for the
2038/// requested denormal behavior, accounting for the overriding behavior of the
2039/// -f32 case.
2040static void addDenormalModeAttrs(llvm::DenormalMode FPDenormalMode,
2041 llvm::DenormalMode FP32DenormalMode,
2042 llvm::AttrBuilder &FuncAttrs) {
2043 llvm::DenormalFPEnv FPEnv(FPDenormalMode, FP32DenormalMode);
2044 if (FPEnv != llvm::DenormalFPEnv::getDefault())
2045 FuncAttrs.addDenormalFPEnvAttr(Mode: FPEnv);
2046}
2047
2048/// Add default attributes to a function, which have merge semantics under
2049/// -mlink-builtin-bitcode and should not simply overwrite any existing
2050/// attributes in the linked library.
2051static void
2052addMergableDefaultFunctionAttributes(const CodeGenOptions &CodeGenOpts,
2053 llvm::AttrBuilder &FuncAttrs) {
2054 addDenormalModeAttrs(FPDenormalMode: CodeGenOpts.FPDenormalMode, FP32DenormalMode: CodeGenOpts.FP32DenormalMode,
2055 FuncAttrs);
2056}
2057
2058static void getTrivialDefaultFunctionAttributes(
2059 StringRef Name, bool HasOptnone, const CodeGenOptions &CodeGenOpts,
2060 const LangOptions &LangOpts, bool AttrOnCallSite,
2061 llvm::AttrBuilder &FuncAttrs) {
2062 // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
2063 if (!HasOptnone) {
2064 if (CodeGenOpts.OptimizeSize)
2065 FuncAttrs.addAttribute(Val: llvm::Attribute::OptimizeForSize);
2066 if (CodeGenOpts.OptimizeSize == 2)
2067 FuncAttrs.addAttribute(Val: llvm::Attribute::MinSize);
2068 }
2069
2070 if (CodeGenOpts.DisableRedZone)
2071 FuncAttrs.addAttribute(Val: llvm::Attribute::NoRedZone);
2072 if (CodeGenOpts.IndirectTlsSegRefs)
2073 FuncAttrs.addAttribute(A: "indirect-tls-seg-refs");
2074 if (CodeGenOpts.NoImplicitFloat)
2075 FuncAttrs.addAttribute(Val: llvm::Attribute::NoImplicitFloat);
2076
2077 if (AttrOnCallSite) {
2078 // Attributes that should go on the call site only.
2079 // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
2080 // the -fno-builtin-foo list.
2081 if (!CodeGenOpts.SimplifyLibCalls || LangOpts.isNoBuiltinFunc(Name))
2082 FuncAttrs.addAttribute(Val: llvm::Attribute::NoBuiltin);
2083 if (!CodeGenOpts.TrapFuncName.empty())
2084 FuncAttrs.addAttribute(A: "trap-func-name", V: CodeGenOpts.TrapFuncName);
2085 } else {
2086 switch (CodeGenOpts.getFramePointer()) {
2087 case CodeGenOptions::FramePointerKind::None:
2088 // This is the default behavior.
2089 break;
2090 case CodeGenOptions::FramePointerKind::Reserved:
2091 case CodeGenOptions::FramePointerKind::NonLeafNoReserve:
2092 case CodeGenOptions::FramePointerKind::NonLeaf:
2093 case CodeGenOptions::FramePointerKind::All:
2094 FuncAttrs.addAttribute(A: "frame-pointer",
2095 V: CodeGenOptions::getFramePointerKindName(
2096 Kind: CodeGenOpts.getFramePointer()));
2097 }
2098
2099 if (CodeGenOpts.LessPreciseFPMAD)
2100 FuncAttrs.addAttribute(A: "less-precise-fpmad", V: "true");
2101
2102 if (CodeGenOpts.NullPointerIsValid)
2103 FuncAttrs.addAttribute(Val: llvm::Attribute::NullPointerIsValid);
2104
2105 if (LangOpts.getDefaultExceptionMode() == LangOptions::FPE_Ignore)
2106 FuncAttrs.addAttribute(A: "no-trapping-math", V: "true");
2107
2108 // TODO: Are these all needed?
2109 // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
2110 if (CodeGenOpts.SoftFloat)
2111 FuncAttrs.addAttribute(A: "use-soft-float", V: "true");
2112 FuncAttrs.addAttribute(A: "stack-protector-buffer-size",
2113 V: llvm::utostr(X: CodeGenOpts.SSPBufferSize));
2114 if (LangOpts.NoSignedZero)
2115 FuncAttrs.addAttribute(A: "no-signed-zeros-fp-math", V: "true");
2116
2117 // TODO: Reciprocal estimate codegen options should apply to instructions?
2118 const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
2119 if (!Recips.empty())
2120 FuncAttrs.addAttribute(A: "reciprocal-estimates", V: llvm::join(R: Recips, Separator: ","));
2121
2122 if (!CodeGenOpts.PreferVectorWidth.empty() &&
2123 CodeGenOpts.PreferVectorWidth != "none")
2124 FuncAttrs.addAttribute(A: "prefer-vector-width",
2125 V: CodeGenOpts.PreferVectorWidth);
2126
2127 if (CodeGenOpts.StackRealignment)
2128 FuncAttrs.addAttribute(A: "stackrealign");
2129 if (CodeGenOpts.Backchain)
2130 FuncAttrs.addAttribute(A: "backchain");
2131 if (CodeGenOpts.EnableSegmentedStacks)
2132 FuncAttrs.addAttribute(A: "split-stack");
2133
2134 if (CodeGenOpts.SpeculativeLoadHardening)
2135 FuncAttrs.addAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2136
2137 // Add zero-call-used-regs attribute.
2138 switch (CodeGenOpts.getZeroCallUsedRegs()) {
2139 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
2140 FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2141 break;
2142 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
2143 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr-arg");
2144 break;
2145 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
2146 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr");
2147 break;
2148 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
2149 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-arg");
2150 break;
2151 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
2152 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used");
2153 break;
2154 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
2155 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr-arg");
2156 break;
2157 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
2158 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr");
2159 break;
2160 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
2161 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-arg");
2162 break;
2163 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
2164 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all");
2165 break;
2166 }
2167 }
2168
2169 if (LangOpts.assumeFunctionsAreConvergent()) {
2170 // Conservatively, mark all functions and calls in CUDA and OpenCL as
2171 // convergent (meaning, they may call an intrinsically convergent op, such
2172 // as __syncthreads() / barrier(), and so can't have certain optimizations
2173 // applied around them). LLVM will remove this attribute where it safely
2174 // can.
2175 FuncAttrs.addAttribute(Val: llvm::Attribute::Convergent);
2176 }
2177
2178 // TODO: NoUnwind attribute should be added for other GPU modes HIP,
2179 // OpenMP offload. AFAIK, neither of them support exceptions in device code.
2180 if ((LangOpts.CUDA && LangOpts.CUDAIsDevice) || LangOpts.OpenCL ||
2181 LangOpts.SYCLIsDevice) {
2182 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2183 }
2184
2185 if (CodeGenOpts.SaveRegParams && !AttrOnCallSite)
2186 FuncAttrs.addAttribute(A: "save-reg-params");
2187
2188 for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
2189 StringRef Var, Value;
2190 std::tie(args&: Var, args&: Value) = Attr.split(Separator: '=');
2191 FuncAttrs.addAttribute(A: Var, V: Value);
2192 }
2193
2194 TargetInfo::BranchProtectionInfo BPI(LangOpts);
2195 TargetCodeGenInfo::initBranchProtectionFnAttributes(BPI, FuncAttrs);
2196}
2197
2198/// Merges `target-features` from \TargetOpts and \F, and sets the result in
2199/// \FuncAttr
2200/// * features from \F are always kept
2201/// * a feature from \TargetOpts is kept if itself and its opposite are absent
2202/// from \F
2203static void
2204overrideFunctionFeaturesWithTargetFeatures(llvm::AttrBuilder &FuncAttr,
2205 const llvm::Function &F,
2206 const TargetOptions &TargetOpts) {
2207 auto FFeatures = F.getFnAttribute(Kind: "target-features");
2208
2209 llvm::StringSet<> MergedNames;
2210 SmallVector<StringRef> MergedFeatures;
2211 MergedFeatures.reserve(N: TargetOpts.Features.size());
2212
2213 auto AddUnmergedFeatures = [&](auto &&FeatureRange) {
2214 for (StringRef Feature : FeatureRange) {
2215 if (Feature.empty())
2216 continue;
2217 assert(Feature[0] == '+' || Feature[0] == '-');
2218 StringRef Name = Feature.drop_front(N: 1);
2219 bool Merged = !MergedNames.insert(key: Name).second;
2220 if (!Merged)
2221 MergedFeatures.push_back(Elt: Feature);
2222 }
2223 };
2224
2225 if (FFeatures.isValid())
2226 AddUnmergedFeatures(llvm::split(Str: FFeatures.getValueAsString(), Separator: ','));
2227 AddUnmergedFeatures(TargetOpts.Features);
2228
2229 if (!MergedFeatures.empty()) {
2230 llvm::sort(C&: MergedFeatures);
2231 FuncAttr.addAttribute(A: "target-features", V: llvm::join(R&: MergedFeatures, Separator: ","));
2232 }
2233}
2234
2235void CodeGen::mergeDefaultFunctionDefinitionAttributes(
2236 llvm::Function &F, const CodeGenOptions &CodeGenOpts,
2237 const LangOptions &LangOpts, const TargetOptions &TargetOpts,
2238 bool WillInternalize) {
2239
2240 llvm::AttrBuilder FuncAttrs(F.getContext());
2241 // Here we only extract the options that are relevant compared to the version
2242 // from GetCPUAndFeaturesAttributes.
2243 if (!TargetOpts.CPU.empty())
2244 FuncAttrs.addAttribute(A: "target-cpu", V: TargetOpts.CPU);
2245 if (!TargetOpts.TuneCPU.empty())
2246 FuncAttrs.addAttribute(A: "tune-cpu", V: TargetOpts.TuneCPU);
2247
2248 ::getTrivialDefaultFunctionAttributes(Name: F.getName(), HasOptnone: F.hasOptNone(),
2249 CodeGenOpts, LangOpts,
2250 /*AttrOnCallSite=*/false, FuncAttrs);
2251
2252 if (!WillInternalize && F.isInterposable()) {
2253 // Do not promote "dynamic" denormal-fp-math to this translation unit's
2254 // setting for weak functions that won't be internalized. The user has no
2255 // real control for how builtin bitcode is linked, so we shouldn't assume
2256 // later copies will use a consistent mode.
2257 F.addFnAttrs(Attrs: FuncAttrs);
2258 return;
2259 }
2260
2261 llvm::AttributeMask AttrsToRemove;
2262
2263 llvm::DenormalFPEnv OptsFPEnv(CodeGenOpts.FPDenormalMode,
2264 CodeGenOpts.FP32DenormalMode);
2265 llvm::DenormalFPEnv MergedFPEnv =
2266 OptsFPEnv.mergeCalleeMode(Callee: F.getDenormalFPEnv());
2267
2268 if (MergedFPEnv == llvm::DenormalFPEnv::getDefault()) {
2269 AttrsToRemove.addAttribute(Val: llvm::Attribute::DenormalFPEnv);
2270 } else {
2271 // Overwrite existing attribute
2272 FuncAttrs.addDenormalFPEnvAttr(Mode: MergedFPEnv);
2273 }
2274
2275 F.removeFnAttrs(Attrs: AttrsToRemove);
2276
2277 overrideFunctionFeaturesWithTargetFeatures(FuncAttr&: FuncAttrs, F, TargetOpts);
2278
2279 F.addFnAttrs(Attrs: FuncAttrs);
2280}
2281
2282void CodeGenModule::getTrivialDefaultFunctionAttributes(
2283 StringRef Name, bool HasOptnone, bool AttrOnCallSite,
2284 llvm::AttrBuilder &FuncAttrs) {
2285 ::getTrivialDefaultFunctionAttributes(Name, HasOptnone, CodeGenOpts: getCodeGenOpts(),
2286 LangOpts: getLangOpts(), AttrOnCallSite,
2287 FuncAttrs);
2288}
2289
2290void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
2291 bool HasOptnone,
2292 bool AttrOnCallSite,
2293 llvm::AttrBuilder &FuncAttrs) {
2294 getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,
2295 FuncAttrs);
2296
2297 if (!AttrOnCallSite)
2298 TargetCodeGenInfo::initPointerAuthFnAttributes(Opts: CodeGenOpts.PointerAuth,
2299 FuncAttrs);
2300
2301 // If we're just getting the default, get the default values for mergeable
2302 // attributes.
2303 if (!AttrOnCallSite)
2304 addMergableDefaultFunctionAttributes(CodeGenOpts, FuncAttrs);
2305}
2306
2307void CodeGenModule::addDefaultFunctionDefinitionAttributes(
2308 llvm::AttrBuilder &attrs) {
2309 getDefaultFunctionAttributes(/*function name*/ Name: "", /*optnone*/ HasOptnone: false,
2310 /*for call*/ AttrOnCallSite: false, FuncAttrs&: attrs);
2311 GetCPUAndFeaturesAttributes(GD: GlobalDecl(), AttrBuilder&: attrs);
2312}
2313
2314static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
2315 const LangOptions &LangOpts,
2316 const NoBuiltinAttr *NBA = nullptr) {
2317 auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
2318 SmallString<32> AttributeName;
2319 AttributeName += "no-builtin-";
2320 AttributeName += BuiltinName;
2321 FuncAttrs.addAttribute(A: AttributeName);
2322 };
2323
2324 // First, handle the language options passed through -fno-builtin.
2325 if (LangOpts.NoBuiltin) {
2326 // -fno-builtin disables them all.
2327 FuncAttrs.addAttribute(A: "no-builtins");
2328 return;
2329 }
2330
2331 // Then, add attributes for builtins specified through -fno-builtin-<name>.
2332 llvm::for_each(Range: LangOpts.NoBuiltinFuncs, F: AddNoBuiltinAttr);
2333
2334 // Now, let's check the __attribute__((no_builtin("...")) attribute added to
2335 // the source.
2336 if (!NBA)
2337 return;
2338
2339 // If there is a wildcard in the builtin names specified through the
2340 // attribute, disable them all.
2341 if (llvm::is_contained(Range: NBA->builtinNames(), Element: "*")) {
2342 FuncAttrs.addAttribute(A: "no-builtins");
2343 return;
2344 }
2345
2346 // And last, add the rest of the builtin names.
2347 llvm::for_each(Range: NBA->builtinNames(), F: AddNoBuiltinAttr);
2348}
2349
2350static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
2351 const llvm::DataLayout &DL, const ABIArgInfo &AI,
2352 bool CheckCoerce = true) {
2353 llvm::Type *Ty = Types.ConvertTypeForMem(T: QTy);
2354 if (AI.getKind() == ABIArgInfo::Indirect ||
2355 AI.getKind() == ABIArgInfo::IndirectAliased)
2356 return true;
2357 if (AI.getKind() == ABIArgInfo::Extend && !AI.isNoExt())
2358 return true;
2359 if (!DL.typeSizeEqualsStoreSize(Ty))
2360 // TODO: This will result in a modest amount of values not marked noundef
2361 // when they could be. We care about values that *invisibly* contain undef
2362 // bits from the perspective of LLVM IR.
2363 return false;
2364 if (CheckCoerce && AI.canHaveCoerceToType()) {
2365 llvm::Type *CoerceTy = AI.getCoerceToType();
2366 if (llvm::TypeSize::isKnownGT(LHS: DL.getTypeSizeInBits(Ty: CoerceTy),
2367 RHS: DL.getTypeSizeInBits(Ty)))
2368 // If we're coercing to a type with a greater size than the canonical one,
2369 // we're introducing new undef bits.
2370 // Coercing to a type of smaller or equal size is ok, as we know that
2371 // there's no internal padding (typeSizeEqualsStoreSize).
2372 return false;
2373 }
2374 if (QTy->isBitIntType())
2375 return true;
2376 if (QTy->isReferenceType())
2377 return true;
2378 if (QTy->isNullPtrType())
2379 return false;
2380 if (QTy->isMemberPointerType())
2381 // TODO: Some member pointers are `noundef`, but it depends on the ABI. For
2382 // now, never mark them.
2383 return false;
2384 if (QTy->isScalarType()) {
2385 if (const ComplexType *Complex = dyn_cast<ComplexType>(Val&: QTy))
2386 return DetermineNoUndef(QTy: Complex->getElementType(), Types, DL, AI, CheckCoerce: false);
2387 return true;
2388 }
2389 if (const VectorType *Vector = dyn_cast<VectorType>(Val&: QTy))
2390 return DetermineNoUndef(QTy: Vector->getElementType(), Types, DL, AI, CheckCoerce: false);
2391 if (const MatrixType *Matrix = dyn_cast<MatrixType>(Val&: QTy))
2392 return DetermineNoUndef(QTy: Matrix->getElementType(), Types, DL, AI, CheckCoerce: false);
2393 if (const ArrayType *Array = dyn_cast<ArrayType>(Val&: QTy))
2394 return DetermineNoUndef(QTy: Array->getElementType(), Types, DL, AI, CheckCoerce: false);
2395
2396 // TODO: Some structs may be `noundef`, in specific situations.
2397 return false;
2398}
2399
2400/// Check if the argument of a function has maybe_undef attribute.
2401static bool IsArgumentMaybeUndef(const Decl *TargetDecl,
2402 unsigned NumRequiredArgs, unsigned ArgNo) {
2403 const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
2404 if (!FD)
2405 return false;
2406
2407 // Assume variadic arguments do not have maybe_undef attribute.
2408 if (ArgNo >= NumRequiredArgs)
2409 return false;
2410
2411 // Check if argument has maybe_undef attribute.
2412 if (ArgNo < FD->getNumParams()) {
2413 const ParmVarDecl *Param = FD->getParamDecl(i: ArgNo);
2414 if (Param && Param->hasAttr<MaybeUndefAttr>())
2415 return true;
2416 }
2417
2418 return false;
2419}
2420
2421/// Test if it's legal to apply nofpclass for the given parameter type and it's
2422/// lowered IR type.
2423static bool canApplyNoFPClass(const ABIArgInfo &AI, QualType ParamType,
2424 bool IsReturn) {
2425 // Should only apply to FP types in the source, not ABI promoted.
2426 if (!ParamType->hasFloatingRepresentation())
2427 return false;
2428
2429 // The promoted-to IR type also needs to support nofpclass.
2430 llvm::Type *IRTy = AI.getCoerceToType();
2431 if (llvm::AttributeFuncs::isNoFPClassCompatibleType(Ty: IRTy))
2432 return true;
2433
2434 if (llvm::StructType *ST = dyn_cast<llvm::StructType>(Val: IRTy)) {
2435 return !IsReturn && AI.getCanBeFlattened() &&
2436 llvm::all_of(Range: ST->elements(),
2437 P: llvm::AttributeFuncs::isNoFPClassCompatibleType);
2438 }
2439
2440 return false;
2441}
2442
2443/// Return the nofpclass mask that can be applied to floating-point parameters.
2444static llvm::FPClassTest getNoFPClassTestMask(const LangOptions &LangOpts) {
2445 llvm::FPClassTest Mask = llvm::fcNone;
2446 if (LangOpts.NoHonorInfs)
2447 Mask |= llvm::fcInf;
2448 if (LangOpts.NoHonorNaNs)
2449 Mask |= llvm::fcNan;
2450 return Mask;
2451}
2452
2453void CodeGenModule::AdjustMemoryAttribute(StringRef Name,
2454 CGCalleeInfo CalleeInfo,
2455 llvm::AttributeList &Attrs) {
2456 if (Attrs.getMemoryEffects().getModRef() == llvm::ModRefInfo::NoModRef) {
2457 Attrs = Attrs.removeFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::Memory);
2458 llvm::Attribute MemoryAttr = llvm::Attribute::getWithMemoryEffects(
2459 Context&: getLLVMContext(), ME: llvm::MemoryEffects::writeOnly());
2460 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Attr: MemoryAttr);
2461 }
2462}
2463
2464/// Construct the IR attribute list of a function or call.
2465///
2466/// When adding an attribute, please consider where it should be handled:
2467///
2468/// - getDefaultFunctionAttributes is for attributes that are essentially
2469/// part of the global target configuration (but perhaps can be
2470/// overridden on a per-function basis). Adding attributes there
2471/// will cause them to also be set in frontends that build on Clang's
2472/// target-configuration logic, as well as for code defined in library
2473/// modules such as CUDA's libdevice.
2474///
2475/// - ConstructAttributeList builds on top of getDefaultFunctionAttributes
2476/// and adds declaration-specific, convention-specific, and
2477/// frontend-specific logic. The last is of particular importance:
2478/// attributes that restrict how the frontend generates code must be
2479/// added here rather than getDefaultFunctionAttributes.
2480///
2481void CodeGenModule::ConstructAttributeList(StringRef Name,
2482 const CGFunctionInfo &FI,
2483 CGCalleeInfo CalleeInfo,
2484 llvm::AttributeList &AttrList,
2485 unsigned &CallingConv,
2486 bool AttrOnCallSite, bool IsThunk) {
2487 llvm::AttrBuilder FuncAttrs(getLLVMContext());
2488 llvm::AttrBuilder RetAttrs(getLLVMContext());
2489
2490 // Collect function IR attributes from the CC lowering.
2491 // We'll collect the paramete and result attributes later.
2492 CallingConv = FI.getEffectiveCallingConvention();
2493 if (FI.isNoReturn())
2494 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2495 if (FI.isCmseNSCall())
2496 FuncAttrs.addAttribute(A: "cmse_nonsecure_call");
2497
2498 // Collect function IR attributes from the callee prototype if we have one.
2499 AddAttributesFromFunctionProtoType(Ctx&: getContext(), FuncAttrs,
2500 FPT: CalleeInfo.getCalleeFunctionProtoType());
2501 const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2502
2503 // Attach assumption attributes to the declaration. If this is a call
2504 // site, attach assumptions from the caller to the call as well.
2505 AddAttributesFromOMPAssumes(FuncAttrs, Callee: TargetDecl);
2506
2507 bool HasOptnone = false;
2508 // The NoBuiltinAttr attached to the target FunctionDecl.
2509 const NoBuiltinAttr *NBA = nullptr;
2510
2511 // Some ABIs may result in additional accesses to arguments that may
2512 // otherwise not be present.
2513 std::optional<llvm::Attribute::AttrKind> MemAttrForPtrArgs;
2514 bool AddedPotentialArgAccess = false;
2515 auto AddPotentialArgAccess = [&]() {
2516 AddedPotentialArgAccess = true;
2517 llvm::Attribute A = FuncAttrs.getAttribute(Kind: llvm::Attribute::Memory);
2518 if (A.isValid())
2519 FuncAttrs.addMemoryAttr(ME: A.getMemoryEffects() |
2520 llvm::MemoryEffects::argMemOnly());
2521 };
2522
2523 // Collect function IR attributes based on declaration-specific
2524 // information.
2525 // FIXME: handle sseregparm someday...
2526 if (TargetDecl) {
2527 if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2528 FuncAttrs.addAttribute(Val: llvm::Attribute::ReturnsTwice);
2529 if (TargetDecl->hasAttr<NoThrowAttr>())
2530 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2531 if (TargetDecl->hasAttr<NoReturnAttr>())
2532 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2533 if (TargetDecl->hasAttr<ColdAttr>())
2534 FuncAttrs.addAttribute(Val: llvm::Attribute::Cold);
2535 if (TargetDecl->hasAttr<HotAttr>())
2536 FuncAttrs.addAttribute(Val: llvm::Attribute::Hot);
2537 if (TargetDecl->hasAttr<NoDuplicateAttr>())
2538 FuncAttrs.addAttribute(Val: llvm::Attribute::NoDuplicate);
2539 if (TargetDecl->hasAttr<ConvergentAttr>())
2540 FuncAttrs.addAttribute(Val: llvm::Attribute::Convergent);
2541
2542 if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2543 AddAttributesFromFunctionProtoType(
2544 Ctx&: getContext(), FuncAttrs, FPT: Fn->getType()->getAs<FunctionProtoType>());
2545 if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
2546 // A sane operator new returns a non-aliasing pointer.
2547 auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2548 if (getCodeGenOpts().AssumeSaneOperatorNew &&
2549 (Kind == OO_New || Kind == OO_Array_New))
2550 RetAttrs.addAttribute(Val: llvm::Attribute::NoAlias);
2551 }
2552 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: Fn);
2553 const bool IsVirtualCall = MD && MD->isVirtual();
2554 // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2555 // virtual function. These attributes are not inherited by overloads.
2556 if (!(AttrOnCallSite && IsVirtualCall)) {
2557 if (Fn->isNoReturn())
2558 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2559 NBA = Fn->getAttr<NoBuiltinAttr>();
2560 }
2561 }
2562
2563 if (isa<FunctionDecl>(Val: TargetDecl) || isa<VarDecl>(Val: TargetDecl)) {
2564 // Only place nomerge attribute on call sites, never functions. This
2565 // allows it to work on indirect virtual function calls.
2566 if (AttrOnCallSite && TargetDecl->hasAttr<NoMergeAttr>())
2567 FuncAttrs.addAttribute(Val: llvm::Attribute::NoMerge);
2568 }
2569
2570 // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2571 if (TargetDecl->hasAttr<ConstAttr>()) {
2572 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::none());
2573 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2574 // gcc specifies that 'const' functions have greater restrictions than
2575 // 'pure' functions, so they also cannot have infinite loops.
2576 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
2577 MemAttrForPtrArgs = llvm::Attribute::ReadNone;
2578 } else if (TargetDecl->hasAttr<PureAttr>()) {
2579 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::readOnly());
2580 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2581 // gcc specifies that 'pure' functions cannot have infinite loops.
2582 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
2583 MemAttrForPtrArgs = llvm::Attribute::ReadOnly;
2584 } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2585 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::inaccessibleOrArgMemOnly());
2586 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2587 }
2588 if (const auto *RA = TargetDecl->getAttr<RestrictAttr>();
2589 RA && RA->getDeallocator() == nullptr)
2590 RetAttrs.addAttribute(Val: llvm::Attribute::NoAlias);
2591 if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2592 !CodeGenOpts.NullPointerIsValid)
2593 RetAttrs.addAttribute(Val: llvm::Attribute::NonNull);
2594 if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2595 FuncAttrs.addAttribute(A: "no_caller_saved_registers");
2596 if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2597 FuncAttrs.addAttribute(Val: llvm::Attribute::NoCfCheck);
2598 if (TargetDecl->hasAttr<LeafAttr>())
2599 FuncAttrs.addAttribute(Val: llvm::Attribute::NoCallback);
2600 if (TargetDecl->hasAttr<BPFFastCallAttr>())
2601 FuncAttrs.addAttribute(A: "bpf_fastcall");
2602
2603 HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2604 if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2605 std::optional<unsigned> NumElemsParam;
2606 if (AllocSize->getNumElemsParam().isValid())
2607 NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2608 FuncAttrs.addAllocSizeAttr(ElemSizeArg: AllocSize->getElemSizeParam().getLLVMIndex(),
2609 NumElemsArg: NumElemsParam);
2610 }
2611
2612 // OpenCL v2.0 Work groups may be whether uniform or not.
2613 // '-cl-uniform-work-group-size' compile option gets a hint
2614 // to the compiler that the global work-size be a multiple of
2615 // the work-group size specified to clEnqueueNDRangeKernel
2616 // (i.e. work groups are uniform).
2617 if (getLangOpts().OffloadUniformBlock)
2618 FuncAttrs.addAttribute(A: "uniform-work-group-size");
2619
2620 if (TargetDecl->hasAttr<ArmLocallyStreamingAttr>())
2621 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_body");
2622
2623 if (auto *ModularFormat = TargetDecl->getAttr<ModularFormatAttr>()) {
2624 FormatAttr *Format = TargetDecl->getAttr<FormatAttr>();
2625 StringRef Type = Format->getType()->getName();
2626 std::string FormatIdx = std::to_string(val: Format->getFormatIdx());
2627 std::string FirstArg = std::to_string(val: Format->getFirstArg());
2628 SmallVector<StringRef> Args = {
2629 Type, FormatIdx, FirstArg,
2630 ModularFormat->getModularImplFn()->getName(),
2631 ModularFormat->getImplName()};
2632 llvm::append_range(C&: Args, R: ModularFormat->aspects());
2633 FuncAttrs.addAttribute(A: "modular-format", V: llvm::join(R&: Args, Separator: ","));
2634 }
2635 }
2636
2637 // Attach "no-builtins" attributes to:
2638 // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2639 // * definitions: "no-builtins" or "no-builtin-<name>" only.
2640 // The attributes can come from:
2641 // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2642 // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2643 addNoBuiltinAttributes(FuncAttrs, LangOpts: getLangOpts(), NBA);
2644
2645 // Collect function IR attributes based on global settiings.
2646 getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2647
2648 // Override some default IR attributes based on declaration-specific
2649 // information.
2650 if (TargetDecl) {
2651 if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2652 FuncAttrs.removeAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2653 if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2654 FuncAttrs.addAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2655 if (TargetDecl->hasAttr<NoSplitStackAttr>())
2656 FuncAttrs.removeAttribute(A: "split-stack");
2657 if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
2658 // A function "__attribute__((...))" overrides the command-line flag.
2659 auto Kind =
2660 TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2661 FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2662 FuncAttrs.addAttribute(
2663 A: "zero-call-used-regs",
2664 V: ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Val: Kind));
2665 }
2666
2667 // Add NonLazyBind attribute to function declarations when -fno-plt
2668 // is used.
2669 // FIXME: what if we just haven't processed the function definition
2670 // yet, or if it's an external definition like C99 inline?
2671 if (CodeGenOpts.NoPLT) {
2672 if (auto *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2673 if (!Fn->isDefined() && !AttrOnCallSite) {
2674 FuncAttrs.addAttribute(Val: llvm::Attribute::NonLazyBind);
2675 }
2676 }
2677 }
2678 // Remove 'convergent' if requested.
2679 if (TargetDecl->hasAttr<NoConvergentAttr>())
2680 FuncAttrs.removeAttribute(Val: llvm::Attribute::Convergent);
2681 }
2682
2683 // Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2684 // functions with -funique-internal-linkage-names.
2685 if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames) {
2686 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
2687 if (!FD->isExternallyVisible())
2688 FuncAttrs.addAttribute(A: "sample-profile-suffix-elision-policy",
2689 V: "selected");
2690 }
2691 }
2692
2693 // Collect non-call-site function IR attributes from declaration-specific
2694 // information.
2695 if (!AttrOnCallSite) {
2696 if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>())
2697 FuncAttrs.addAttribute(A: "cmse_nonsecure_entry");
2698
2699 // Whether tail calls are enabled.
2700 auto shouldDisableTailCalls = [&] {
2701 // Should this be honored in getDefaultFunctionAttributes?
2702 if (CodeGenOpts.DisableTailCalls)
2703 return true;
2704
2705 if (!TargetDecl)
2706 return false;
2707
2708 if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2709 TargetDecl->hasAttr<AnyX86InterruptAttr>())
2710 return true;
2711
2712 if (CodeGenOpts.NoEscapingBlockTailCalls) {
2713 if (const auto *BD = dyn_cast<BlockDecl>(Val: TargetDecl))
2714 if (!BD->doesNotEscape())
2715 return true;
2716 }
2717
2718 return false;
2719 };
2720 if (shouldDisableTailCalls())
2721 FuncAttrs.addAttribute(A: "disable-tail-calls", V: "true");
2722
2723 // These functions require the returns_twice attribute for correct codegen,
2724 // but the attribute may not be added if -fno-builtin is specified. We
2725 // explicitly add that attribute here.
2726 static const llvm::StringSet<> ReturnsTwiceFn{
2727 "_setjmpex", "setjmp", "_setjmp", "vfork",
2728 "sigsetjmp", "__sigsetjmp", "savectx", "getcontext"};
2729 if (ReturnsTwiceFn.contains(key: Name))
2730 FuncAttrs.addAttribute(Val: llvm::Attribute::ReturnsTwice);
2731
2732 // CPU/feature overrides. addDefaultFunctionDefinitionAttributes
2733 // handles these separately to set them based on the global defaults.
2734 GetCPUAndFeaturesAttributes(GD: CalleeInfo.getCalleeDecl(), AttrBuilder&: FuncAttrs);
2735
2736 // Windows hotpatching support
2737 if (!MSHotPatchFunctions.empty()) {
2738 bool IsHotPatched = llvm::binary_search(Range&: MSHotPatchFunctions, Value&: Name);
2739 if (IsHotPatched)
2740 FuncAttrs.addAttribute(A: "marked_for_windows_hot_patching");
2741 }
2742 }
2743
2744 // Mark functions that are replaceable by the loader.
2745 if (CodeGenOpts.isLoaderReplaceableFunctionName(FuncName: Name))
2746 FuncAttrs.addAttribute(A: "loader-replaceable");
2747
2748 // Collect attributes from arguments and return values.
2749 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2750
2751 QualType RetTy = FI.getReturnType();
2752 const ABIArgInfo &RetAI = FI.getReturnInfo();
2753 const llvm::DataLayout &DL = getDataLayout();
2754
2755 // Determine if the return type could be partially undef
2756 if (CodeGenOpts.EnableNoundefAttrs &&
2757 HasStrictReturn(Module: *this, RetTy, TargetDecl)) {
2758 if (!RetTy->isVoidType() && RetAI.getKind() != ABIArgInfo::Indirect &&
2759 DetermineNoUndef(QTy: RetTy, Types&: getTypes(), DL, AI: RetAI))
2760 RetAttrs.addAttribute(Val: llvm::Attribute::NoUndef);
2761 }
2762
2763 switch (RetAI.getKind()) {
2764 case ABIArgInfo::Extend:
2765 if (RetAI.isSignExt())
2766 RetAttrs.addAttribute(Val: llvm::Attribute::SExt);
2767 else if (RetAI.isZeroExt())
2768 RetAttrs.addAttribute(Val: llvm::Attribute::ZExt);
2769 else
2770 RetAttrs.addAttribute(Val: llvm::Attribute::NoExt);
2771 [[fallthrough]];
2772 case ABIArgInfo::TargetSpecific:
2773 case ABIArgInfo::Direct:
2774 if (RetAI.getInReg())
2775 RetAttrs.addAttribute(Val: llvm::Attribute::InReg);
2776
2777 if (canApplyNoFPClass(AI: RetAI, ParamType: RetTy, IsReturn: true))
2778 RetAttrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
2779
2780 break;
2781 case ABIArgInfo::Ignore:
2782 break;
2783
2784 case ABIArgInfo::InAlloca:
2785 case ABIArgInfo::Indirect: {
2786 // inalloca and sret disable readnone and readonly
2787 AddPotentialArgAccess();
2788 break;
2789 }
2790
2791 case ABIArgInfo::CoerceAndExpand:
2792 break;
2793
2794 case ABIArgInfo::Expand:
2795 case ABIArgInfo::IndirectAliased:
2796 llvm_unreachable("Invalid ABI kind for return argument");
2797 }
2798
2799 if (!IsThunk) {
2800 // FIXME: fix this properly, https://reviews.llvm.org/D100388
2801 if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2802 QualType PTy = RefTy->getPointeeType();
2803 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2804 RetAttrs.addDereferenceableAttr(
2805 Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
2806 if (getTypes().getTargetAddressSpace(T: PTy) == 0 &&
2807 !CodeGenOpts.NullPointerIsValid)
2808 RetAttrs.addAttribute(Val: llvm::Attribute::NonNull);
2809 if (PTy->isObjectType()) {
2810 llvm::Align Alignment =
2811 getNaturalPointeeTypeAlignment(T: RetTy).getAsAlign();
2812 RetAttrs.addAlignmentAttr(Align: Alignment);
2813 }
2814 }
2815 }
2816
2817 bool hasUsedSRet = false;
2818 SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
2819
2820 // Attach attributes to sret.
2821 if (IRFunctionArgs.hasSRetArg()) {
2822 llvm::AttrBuilder SRETAttrs(getLLVMContext());
2823 SRETAttrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: RetTy));
2824 SRETAttrs.addAttribute(Val: llvm::Attribute::Writable);
2825 SRETAttrs.addAttribute(Val: llvm::Attribute::DeadOnUnwind);
2826 hasUsedSRet = true;
2827 if (RetAI.getInReg())
2828 SRETAttrs.addAttribute(Val: llvm::Attribute::InReg);
2829 SRETAttrs.addAlignmentAttr(Align: RetAI.getIndirectAlign().getQuantity());
2830 ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2831 llvm::AttributeSet::get(C&: getLLVMContext(), B: SRETAttrs);
2832 }
2833
2834 // Attach attributes to inalloca argument.
2835 if (IRFunctionArgs.hasInallocaArg()) {
2836 llvm::AttrBuilder Attrs(getLLVMContext());
2837 Attrs.addInAllocaAttr(Ty: FI.getArgStruct());
2838 ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2839 llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs);
2840 }
2841
2842 // Apply `nonnull`, `dereferenceable(N)` and `align N` to the `this` argument,
2843 // unless this is a thunk function. Add dead_on_return to the `this` argument
2844 // in base class destructors to aid in DSE.
2845 // FIXME: fix this properly, https://reviews.llvm.org/D100388
2846 if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() &&
2847 !FI.arg_begin()->type->isVoidPointerType() && !IsThunk) {
2848 auto IRArgs = IRFunctionArgs.getIRArgs(ArgNo: 0);
2849
2850 assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
2851
2852 llvm::AttrBuilder Attrs(getLLVMContext());
2853
2854 QualType ThisTy = FI.arg_begin()->type.getTypePtr()->getPointeeType();
2855
2856 if (!CodeGenOpts.NullPointerIsValid &&
2857 getTypes().getTargetAddressSpace(T: FI.arg_begin()->type) == 0) {
2858 Attrs.addAttribute(Val: llvm::Attribute::NonNull);
2859 Attrs.addDereferenceableAttr(Bytes: getMinimumObjectSize(Ty: ThisTy).getQuantity());
2860 } else {
2861 // FIXME dereferenceable should be correct here, regardless of
2862 // NullPointerIsValid. However, dereferenceable currently does not always
2863 // respect NullPointerIsValid and may imply nonnull and break the program.
2864 // See https://reviews.llvm.org/D66618 for discussions.
2865 Attrs.addDereferenceableOrNullAttr(
2866 Bytes: getMinimumObjectSize(
2867 Ty: FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
2868 .getQuantity());
2869 }
2870
2871 llvm::Align Alignment =
2872 getNaturalTypeAlignment(T: ThisTy, /*BaseInfo=*/nullptr,
2873 /*TBAAInfo=*/nullptr, /*forPointeeType=*/true)
2874 .getAsAlign();
2875 Attrs.addAlignmentAttr(Align: Alignment);
2876
2877 const auto *DD = dyn_cast_if_present<CXXDestructorDecl>(
2878 Val: CalleeInfo.getCalleeDecl().getDecl());
2879 // Do not annotate vector deleting destructors with dead_on_return as the
2880 // this pointer in that case points to an array which we cannot
2881 // statically know the size of. Also do not mark deleting destructors
2882 // dead_on_return as then we might delete stores inside of a user-defined
2883 // operator delete implementation if it gets inlined, which would be
2884 // incorrect as the object's lifetime has already ended and the operator
2885 // delete implementation is allowed to manipulate the underlying storage.
2886 if (DD &&
2887 CalleeInfo.getCalleeDecl().getDtorType() !=
2888 CXXDtorType::Dtor_VectorDeleting &&
2889 CalleeInfo.getCalleeDecl().getDtorType() !=
2890 CXXDtorType::Dtor_Deleting &&
2891 CodeGenOpts.StrictLifetimes) {
2892 const CXXRecordDecl *ClassDecl =
2893 dyn_cast<CXXRecordDecl>(Val: DD->getDeclContext());
2894 // We cannot add dead_on_return if we have virtual base classes because
2895 // they will generally still be live after the base object destructor.
2896 if (ClassDecl->getNumVBases() == 0)
2897 Attrs.addDeadOnReturnAttr(Info: llvm::DeadOnReturnInfo(
2898 Context.getASTRecordLayout(D: ClassDecl).getDataSize().getQuantity()));
2899 }
2900
2901 ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs);
2902 }
2903
2904 unsigned ArgNo = 0;
2905 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(), E = FI.arg_end();
2906 I != E; ++I, ++ArgNo) {
2907 QualType ParamType = I->type;
2908 const ABIArgInfo &AI = I->info;
2909 llvm::AttrBuilder Attrs(getLLVMContext());
2910
2911 // Add attribute for padding argument, if necessary.
2912 if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2913 if (AI.getPaddingInReg()) {
2914 ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2915 llvm::AttributeSet::get(C&: getLLVMContext(),
2916 B: llvm::AttrBuilder(getLLVMContext())
2917 .addAttribute(Val: llvm::Attribute::InReg));
2918 }
2919 }
2920
2921 // Decide whether the argument we're handling could be partially undef
2922 if (CodeGenOpts.EnableNoundefAttrs &&
2923 DetermineNoUndef(QTy: ParamType, Types&: getTypes(), DL, AI)) {
2924 Attrs.addAttribute(Val: llvm::Attribute::NoUndef);
2925 }
2926
2927 // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2928 // have the corresponding parameter variable. It doesn't make
2929 // sense to do it here because parameters are so messed up.
2930 switch (AI.getKind()) {
2931 case ABIArgInfo::Extend:
2932 if (AI.isSignExt())
2933 Attrs.addAttribute(Val: llvm::Attribute::SExt);
2934 else if (AI.isZeroExt())
2935 Attrs.addAttribute(Val: llvm::Attribute::ZExt);
2936 else
2937 Attrs.addAttribute(Val: llvm::Attribute::NoExt);
2938 [[fallthrough]];
2939 case ABIArgInfo::TargetSpecific:
2940 case ABIArgInfo::Direct:
2941 if (ArgNo == 0 && FI.isChainCall())
2942 Attrs.addAttribute(Val: llvm::Attribute::Nest);
2943 else if (AI.getInReg())
2944 Attrs.addAttribute(Val: llvm::Attribute::InReg);
2945 Attrs.addStackAlignmentAttr(Align: llvm::MaybeAlign(AI.getDirectAlign()));
2946
2947 if (canApplyNoFPClass(AI, ParamType, IsReturn: false))
2948 Attrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
2949 break;
2950 case ABIArgInfo::Indirect: {
2951 if (AI.getInReg())
2952 Attrs.addAttribute(Val: llvm::Attribute::InReg);
2953
2954 // HLSL out and inout parameters must not be marked with ByVal or
2955 // DeadOnReturn attributes because stores to these parameters by the
2956 // callee are visible to the caller.
2957 if (auto ParamABI = FI.getExtParameterInfo(argIndex: ArgNo).getABI();
2958 ParamABI != ParameterABI::HLSLOut &&
2959 ParamABI != ParameterABI::HLSLInOut) {
2960
2961 // Depending on the ABI, this may be either a byval or a dead_on_return
2962 // argument.
2963 if (AI.getIndirectByVal()) {
2964 Attrs.addByValAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
2965 } else {
2966 // Add dead_on_return when the object's lifetime ends in the callee.
2967 // This includes trivially-destructible objects, as well as objects
2968 // whose destruction / clean-up is carried out within the callee
2969 // (e.g., Obj-C ARC-managed structs, MSVC callee-destroyed objects).
2970 if (!ParamType.isDestructedType() || !ParamType->isRecordType() ||
2971 ParamType->castAsRecordDecl()->isParamDestroyedInCallee())
2972 Attrs.addDeadOnReturnAttr(Info: llvm::DeadOnReturnInfo());
2973 }
2974 }
2975
2976 auto *Decl = ParamType->getAsRecordDecl();
2977 if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
2978 Decl->getArgPassingRestrictions() ==
2979 RecordArgPassingKind::CanPassInRegs)
2980 // When calling the function, the pointer passed in will be the only
2981 // reference to the underlying object. Mark it accordingly.
2982 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
2983
2984 // TODO: We could add the byref attribute if not byval, but it would
2985 // require updating many testcases.
2986
2987 CharUnits Align = AI.getIndirectAlign();
2988
2989 // In a byval argument, it is important that the required
2990 // alignment of the type is honored, as LLVM might be creating a
2991 // *new* stack object, and needs to know what alignment to give
2992 // it. (Sometimes it can deduce a sensible alignment on its own,
2993 // but not if clang decides it must emit a packed struct, or the
2994 // user specifies increased alignment requirements.)
2995 //
2996 // This is different from indirect *not* byval, where the object
2997 // exists already, and the align attribute is purely
2998 // informative.
2999 assert(!Align.isZero());
3000
3001 // For now, only add this when we have a byval argument.
3002 // TODO: be less lazy about updating test cases.
3003 if (AI.getIndirectByVal())
3004 Attrs.addAlignmentAttr(Align: Align.getQuantity());
3005
3006 // byval disables readnone and readonly.
3007 AddPotentialArgAccess();
3008 break;
3009 }
3010 case ABIArgInfo::IndirectAliased: {
3011 CharUnits Align = AI.getIndirectAlign();
3012 Attrs.addByRefAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
3013 Attrs.addAlignmentAttr(Align: Align.getQuantity());
3014 break;
3015 }
3016 case ABIArgInfo::Ignore:
3017 case ABIArgInfo::Expand:
3018 case ABIArgInfo::CoerceAndExpand:
3019 break;
3020
3021 case ABIArgInfo::InAlloca:
3022 // inalloca disables readnone and readonly.
3023 AddPotentialArgAccess();
3024 continue;
3025 }
3026
3027 if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
3028 QualType PTy = RefTy->getPointeeType();
3029 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
3030 Attrs.addDereferenceableAttr(Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
3031 if (getTypes().getTargetAddressSpace(T: PTy) == 0 &&
3032 !CodeGenOpts.NullPointerIsValid)
3033 Attrs.addAttribute(Val: llvm::Attribute::NonNull);
3034 if (PTy->isObjectType()) {
3035 llvm::Align Alignment =
3036 getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
3037 Attrs.addAlignmentAttr(Align: Alignment);
3038 }
3039 }
3040
3041 // From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:
3042 // > For arguments to a __kernel function declared to be a pointer to a
3043 // > data type, the OpenCL compiler can assume that the pointee is always
3044 // > appropriately aligned as required by the data type.
3045 if (TargetDecl &&
3046 DeviceKernelAttr::isOpenCLSpelling(
3047 A: TargetDecl->getAttr<DeviceKernelAttr>()) &&
3048 ParamType->isPointerType()) {
3049 QualType PTy = ParamType->getPointeeType();
3050 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
3051 llvm::Align Alignment =
3052 getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
3053 Attrs.addAlignmentAttr(Align: Alignment);
3054 }
3055 }
3056
3057 switch (FI.getExtParameterInfo(argIndex: ArgNo).getABI()) {
3058 case ParameterABI::HLSLOut:
3059 case ParameterABI::HLSLInOut:
3060 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
3061 break;
3062 case ParameterABI::Ordinary:
3063 break;
3064
3065 case ParameterABI::SwiftIndirectResult: {
3066 // Add 'sret' if we haven't already used it for something, but
3067 // only if the result is void.
3068 if (!hasUsedSRet && RetTy->isVoidType()) {
3069 Attrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
3070 hasUsedSRet = true;
3071 }
3072
3073 // Add 'noalias' in either case.
3074 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
3075
3076 // Add 'dereferenceable' and 'alignment'.
3077 auto PTy = ParamType->getPointeeType();
3078 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
3079 auto info = getContext().getTypeInfoInChars(T: PTy);
3080 Attrs.addDereferenceableAttr(Bytes: info.Width.getQuantity());
3081 Attrs.addAlignmentAttr(Align: info.Align.getAsAlign());
3082 }
3083 break;
3084 }
3085
3086 case ParameterABI::SwiftErrorResult:
3087 Attrs.addAttribute(Val: llvm::Attribute::SwiftError);
3088 break;
3089
3090 case ParameterABI::SwiftContext:
3091 Attrs.addAttribute(Val: llvm::Attribute::SwiftSelf);
3092 break;
3093
3094 case ParameterABI::SwiftAsyncContext:
3095 Attrs.addAttribute(Val: llvm::Attribute::SwiftAsync);
3096 break;
3097 }
3098
3099 if (FI.getExtParameterInfo(argIndex: ArgNo).isNoEscape())
3100 Attrs.addCapturesAttr(CI: llvm::CaptureInfo::none());
3101
3102 if (Attrs.hasAttributes()) {
3103 unsigned FirstIRArg, NumIRArgs;
3104 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3105 for (unsigned i = 0; i < NumIRArgs; i++)
3106 ArgAttrs[FirstIRArg + i] = ArgAttrs[FirstIRArg + i].addAttributes(
3107 C&: getLLVMContext(), AS: llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs));
3108 }
3109 }
3110 assert(ArgNo == FI.arg_size());
3111
3112 // We can't see all potential arguments in a varargs declaration; treat them
3113 // as if they can access memory.
3114 if (!AttrOnCallSite && FI.isVariadic())
3115 AddPotentialArgAccess();
3116
3117 ArgNo = 0;
3118 if (AddedPotentialArgAccess && MemAttrForPtrArgs) {
3119 llvm::FunctionType *FunctionType = getTypes().GetFunctionType(FI);
3120 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
3121 E = FI.arg_end();
3122 I != E; ++I, ++ArgNo) {
3123 if (I->info.isDirect() || I->info.isExpand() ||
3124 I->info.isCoerceAndExpand()) {
3125 unsigned FirstIRArg, NumIRArgs;
3126 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3127 for (unsigned i = FirstIRArg; i < FirstIRArg + NumIRArgs; ++i) {
3128 // The index may be out-of-bounds if the callee is a varargs
3129 // function.
3130 //
3131 // FIXME: We can compute the types of varargs arguments without going
3132 // through the function type, but the relevant code isn't exposed
3133 // in a way that can be called from here.
3134 if (i < FunctionType->getNumParams() &&
3135 FunctionType->getParamType(i)->isPointerTy()) {
3136 ArgAttrs[i] =
3137 ArgAttrs[i].addAttribute(C&: getLLVMContext(), Kind: *MemAttrForPtrArgs);
3138 }
3139 }
3140 }
3141 }
3142 }
3143
3144 AttrList = llvm::AttributeList::get(
3145 C&: getLLVMContext(), FnAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: FuncAttrs),
3146 RetAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: RetAttrs), ArgAttrs);
3147}
3148
3149/// An argument came in as a promoted argument; demote it back to its
3150/// declared type.
3151static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
3152 const VarDecl *var,
3153 llvm::Value *value) {
3154 llvm::Type *varType = CGF.ConvertType(T: var->getType());
3155
3156 // This can happen with promotions that actually don't change the
3157 // underlying type, like the enum promotions.
3158 if (value->getType() == varType)
3159 return value;
3160
3161 assert((varType->isIntegerTy() || varType->isFloatingPointTy()) &&
3162 "unexpected promotion type");
3163
3164 if (isa<llvm::IntegerType>(Val: varType))
3165 return CGF.Builder.CreateTrunc(V: value, DestTy: varType, Name: "arg.unpromote");
3166
3167 return CGF.Builder.CreateFPCast(V: value, DestTy: varType, Name: "arg.unpromote");
3168}
3169
3170/// Returns the attribute (either parameter attribute, or function
3171/// attribute), which declares argument ArgNo to be non-null.
3172static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
3173 QualType ArgType, unsigned ArgNo) {
3174 // FIXME: __attribute__((nonnull)) can also be applied to:
3175 // - references to pointers, where the pointee is known to be
3176 // nonnull (apparently a Clang extension)
3177 // - transparent unions containing pointers
3178 // In the former case, LLVM IR cannot represent the constraint. In
3179 // the latter case, we have no guarantee that the transparent union
3180 // is in fact passed as a pointer.
3181 if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType())
3182 return nullptr;
3183 // First, check attribute on parameter itself.
3184 if (PVD) {
3185 if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
3186 return ParmNNAttr;
3187 }
3188 // Check function attributes.
3189 if (!FD)
3190 return nullptr;
3191 for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
3192 if (NNAttr->isNonNull(IdxAST: ArgNo))
3193 return NNAttr;
3194 }
3195 return nullptr;
3196}
3197
3198namespace {
3199struct CopyBackSwiftError final : EHScopeStack::Cleanup {
3200 Address Temp;
3201 Address Arg;
3202 CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
3203 void Emit(CodeGenFunction &CGF, Flags flags) override {
3204 llvm::Value *errorValue = CGF.Builder.CreateLoad(Addr: Temp);
3205 CGF.Builder.CreateStore(Val: errorValue, Addr: Arg);
3206 }
3207};
3208} // namespace
3209
3210void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
3211 llvm::Function *Fn,
3212 const FunctionArgList &Args) {
3213 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())
3214 // Naked functions don't have prologues.
3215 return;
3216
3217 // If this is an implicit-return-zero function, go ahead and
3218 // initialize the return value. TODO: it might be nice to have
3219 // a more general mechanism for this that didn't require synthesized
3220 // return statements.
3221 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl)) {
3222 if (FD->hasImplicitReturnZero()) {
3223 QualType RetTy = FD->getReturnType().getUnqualifiedType();
3224 llvm::Type *LLVMTy = CGM.getTypes().ConvertType(T: RetTy);
3225 llvm::Constant *Zero = llvm::Constant::getNullValue(Ty: LLVMTy);
3226 Builder.CreateStore(Val: Zero, Addr: ReturnValue);
3227 }
3228 }
3229
3230 // FIXME: We no longer need the types from FunctionArgList; lift up and
3231 // simplify.
3232
3233 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
3234 assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
3235
3236 // If we're using inalloca, all the memory arguments are GEPs off of the last
3237 // parameter, which is a pointer to the complete memory area.
3238 Address ArgStruct = Address::invalid();
3239 if (IRFunctionArgs.hasInallocaArg())
3240 ArgStruct = Address(Fn->getArg(i: IRFunctionArgs.getInallocaArgNo()),
3241 FI.getArgStruct(), FI.getArgStructAlignment());
3242
3243 // Name the struct return parameter.
3244 if (IRFunctionArgs.hasSRetArg()) {
3245 auto AI = Fn->getArg(i: IRFunctionArgs.getSRetArgNo());
3246 AI->setName("agg.result");
3247 AI->addAttr(Kind: llvm::Attribute::NoAlias);
3248 }
3249
3250 // Track if we received the parameter as a pointer (indirect, byval, or
3251 // inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it
3252 // into a local alloca for us.
3253 SmallVector<ParamValue, 16> ArgVals;
3254 ArgVals.reserve(N: Args.size());
3255
3256 // Create a pointer value for every parameter declaration. This usually
3257 // entails copying one or more LLVM IR arguments into an alloca. Don't push
3258 // any cleanups or do anything that might unwind. We do that separately, so
3259 // we can push the cleanups in the correct order for the ABI.
3260 assert(FI.arg_size() == Args.size() &&
3261 "Mismatch between function signature & arguments.");
3262 unsigned ArgNo = 0;
3263 CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
3264 for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end(); i != e;
3265 ++i, ++info_it, ++ArgNo) {
3266 const VarDecl *Arg = *i;
3267 const ABIArgInfo &ArgI = info_it->info;
3268
3269 bool isPromoted =
3270 isa<ParmVarDecl>(Val: Arg) && cast<ParmVarDecl>(Val: Arg)->isKNRPromoted();
3271 // We are converting from ABIArgInfo type to VarDecl type directly, unless
3272 // the parameter is promoted. In this case we convert to
3273 // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
3274 QualType Ty = isPromoted ? info_it->type : Arg->getType();
3275 assert(hasScalarEvaluationKind(Ty) ==
3276 hasScalarEvaluationKind(Arg->getType()));
3277
3278 unsigned FirstIRArg, NumIRArgs;
3279 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3280
3281 switch (ArgI.getKind()) {
3282 case ABIArgInfo::InAlloca: {
3283 assert(NumIRArgs == 0);
3284 auto FieldIndex = ArgI.getInAllocaFieldIndex();
3285 Address V =
3286 Builder.CreateStructGEP(Addr: ArgStruct, Index: FieldIndex, Name: Arg->getName());
3287 if (ArgI.getInAllocaIndirect())
3288 V = Address(Builder.CreateLoad(Addr: V), ConvertTypeForMem(T: Ty),
3289 getContext().getTypeAlignInChars(T: Ty));
3290 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: V));
3291 break;
3292 }
3293
3294 case ABIArgInfo::Indirect:
3295 case ABIArgInfo::IndirectAliased: {
3296 assert(NumIRArgs == 1);
3297 Address ParamAddr = makeNaturalAddressForPointer(
3298 Ptr: Fn->getArg(i: FirstIRArg), T: Ty, Alignment: ArgI.getIndirectAlign(), ForPointeeType: false, BaseInfo: nullptr,
3299 TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
3300
3301 if (!hasScalarEvaluationKind(T: Ty)) {
3302 // Aggregates and complex variables are accessed by reference. All we
3303 // need to do is realign the value, if requested. Also, if the address
3304 // may be aliased, copy it to ensure that the parameter variable is
3305 // mutable and has a unique adress, as C requires.
3306 if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) {
3307 RawAddress AlignedTemp = CreateMemTemp(T: Ty, Name: "coerce");
3308
3309 // Copy from the incoming argument pointer to the temporary with the
3310 // appropriate alignment.
3311 //
3312 // FIXME: We should have a common utility for generating an aggregate
3313 // copy.
3314 CharUnits Size = getContext().getTypeSizeInChars(T: Ty);
3315 Builder.CreateMemCpy(
3316 Dst: AlignedTemp.getPointer(), DstAlign: AlignedTemp.getAlignment().getAsAlign(),
3317 Src: ParamAddr.emitRawPointer(CGF&: *this),
3318 SrcAlign: ParamAddr.getAlignment().getAsAlign(),
3319 Size: llvm::ConstantInt::get(Ty: IntPtrTy, V: Size.getQuantity()));
3320 ParamAddr = AlignedTemp;
3321 }
3322 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: ParamAddr));
3323 } else {
3324 // Load scalar value from indirect argument.
3325 llvm::Value *V =
3326 EmitLoadOfScalar(Addr: ParamAddr, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3327
3328 if (isPromoted)
3329 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3330 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3331 }
3332 break;
3333 }
3334
3335 case ABIArgInfo::Extend:
3336 case ABIArgInfo::Direct: {
3337 auto AI = Fn->getArg(i: FirstIRArg);
3338 llvm::Type *LTy = ConvertType(T: Arg->getType());
3339
3340 // Prepare parameter attributes. So far, only attributes for pointer
3341 // parameters are prepared. See
3342 // http://llvm.org/docs/LangRef.html#paramattrs.
3343 if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy() &&
3344 ArgI.getCoerceToType()->isPointerTy()) {
3345 assert(NumIRArgs == 1);
3346
3347 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: Arg)) {
3348 // Set `nonnull` attribute if any.
3349 if (getNonNullAttr(FD: CurCodeDecl, PVD, ArgType: PVD->getType(),
3350 ArgNo: PVD->getFunctionScopeIndex()) &&
3351 !CGM.getCodeGenOpts().NullPointerIsValid)
3352 AI->addAttr(Kind: llvm::Attribute::NonNull);
3353
3354 QualType OTy = PVD->getOriginalType();
3355 if (const auto *ArrTy = getContext().getAsConstantArrayType(T: OTy)) {
3356 // A C99 array parameter declaration with the static keyword also
3357 // indicates dereferenceability, and if the size is constant we can
3358 // use the dereferenceable attribute (which requires the size in
3359 // bytes).
3360 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3361 QualType ETy = ArrTy->getElementType();
3362 llvm::Align Alignment =
3363 CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3364 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3365 .addAlignmentAttr(Align: Alignment));
3366 uint64_t ArrSize = ArrTy->getZExtSize();
3367 if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
3368 ArrSize) {
3369 llvm::AttrBuilder Attrs(getLLVMContext());
3370 Attrs.addDereferenceableAttr(
3371 Bytes: getContext().getTypeSizeInChars(T: ETy).getQuantity() *
3372 ArrSize);
3373 AI->addAttrs(B&: Attrs);
3374 } else if (getContext().getTargetInfo().getNullPointerValue(
3375 AddrSpace: ETy.getAddressSpace()) == 0 &&
3376 !CGM.getCodeGenOpts().NullPointerIsValid) {
3377 AI->addAttr(Kind: llvm::Attribute::NonNull);
3378 }
3379 }
3380 } else if (const auto *ArrTy =
3381 getContext().getAsVariableArrayType(T: OTy)) {
3382 // For C99 VLAs with the static keyword, we don't know the size so
3383 // we can't use the dereferenceable attribute, but in addrspace(0)
3384 // we know that it must be nonnull.
3385 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3386 QualType ETy = ArrTy->getElementType();
3387 llvm::Align Alignment =
3388 CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3389 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3390 .addAlignmentAttr(Align: Alignment));
3391 if (!getTypes().getTargetAddressSpace(T: ETy) &&
3392 !CGM.getCodeGenOpts().NullPointerIsValid)
3393 AI->addAttr(Kind: llvm::Attribute::NonNull);
3394 }
3395 }
3396
3397 // Set `align` attribute if any.
3398 const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
3399 if (!AVAttr)
3400 if (const auto *TOTy = OTy->getAs<TypedefType>())
3401 AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
3402 if (AVAttr && !SanOpts.has(K: SanitizerKind::Alignment)) {
3403 // If alignment-assumption sanitizer is enabled, we do *not* add
3404 // alignment attribute here, but emit normal alignment assumption,
3405 // so the UBSAN check could function.
3406 llvm::ConstantInt *AlignmentCI =
3407 cast<llvm::ConstantInt>(Val: EmitScalarExpr(E: AVAttr->getAlignment()));
3408 uint64_t AlignmentInt =
3409 AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment);
3410 if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
3411 AI->removeAttr(Kind: llvm::Attribute::AttrKind::Alignment);
3412 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3413 .addAlignmentAttr(Align: llvm::Align(AlignmentInt)));
3414 }
3415 }
3416 }
3417
3418 // Set 'noalias' if an argument type has the `restrict` qualifier.
3419 if (Arg->getType().isRestrictQualified())
3420 AI->addAttr(Kind: llvm::Attribute::NoAlias);
3421 }
3422
3423 // Prepare the argument value. If we have the trivial case, handle it
3424 // with no muss and fuss.
3425 if (!isa<llvm::StructType>(Val: ArgI.getCoerceToType()) &&
3426 ArgI.getCoerceToType() == ConvertType(T: Ty) &&
3427 ArgI.getDirectOffset() == 0) {
3428 assert(NumIRArgs == 1);
3429
3430 // LLVM expects swifterror parameters to be used in very restricted
3431 // ways. Copy the value into a less-restricted temporary.
3432 llvm::Value *V = AI;
3433 if (FI.getExtParameterInfo(argIndex: ArgNo).getABI() ==
3434 ParameterABI::SwiftErrorResult) {
3435 QualType pointeeTy = Ty->getPointeeType();
3436 assert(pointeeTy->isPointerType());
3437 RawAddress temp =
3438 CreateMemTemp(T: pointeeTy, Align: getPointerAlign(), Name: "swifterror.temp");
3439 Address arg = makeNaturalAddressForPointer(
3440 Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
3441 llvm::Value *incomingErrorValue = Builder.CreateLoad(Addr: arg);
3442 Builder.CreateStore(Val: incomingErrorValue, Addr: temp);
3443 V = temp.getPointer();
3444
3445 // Push a cleanup to copy the value back at the end of the function.
3446 // The convention does not guarantee that the value will be written
3447 // back if the function exits with an unwind exception.
3448 EHStack.pushCleanup<CopyBackSwiftError>(Kind: NormalCleanup, A: temp, A: arg);
3449 }
3450
3451 // Ensure the argument is the correct type.
3452 if (V->getType() != ArgI.getCoerceToType())
3453 V = Builder.CreateBitCast(V, DestTy: ArgI.getCoerceToType());
3454
3455 if (isPromoted)
3456 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3457
3458 // Because of merging of function types from multiple decls it is
3459 // possible for the type of an argument to not match the corresponding
3460 // type in the function type. Since we are codegening the callee
3461 // in here, add a cast to the argument type.
3462 llvm::Type *LTy = ConvertType(T: Arg->getType());
3463 if (V->getType() != LTy)
3464 V = Builder.CreateBitCast(V, DestTy: LTy);
3465
3466 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3467 break;
3468 }
3469
3470 // VLST arguments are coerced to VLATs at the function boundary for
3471 // ABI consistency. If this is a VLST that was coerced to
3472 // a VLAT at the function boundary and the types match up, use
3473 // llvm.vector.extract to convert back to the original VLST.
3474 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: ConvertType(T: Ty))) {
3475 llvm::Value *ArgVal = Fn->getArg(i: FirstIRArg);
3476 if (auto *VecTyFrom =
3477 dyn_cast<llvm::ScalableVectorType>(Val: ArgVal->getType())) {
3478 auto [Coerced, Extracted] = CoerceScalableToFixed(
3479 CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: ArgVal, Name: Arg->getName());
3480 if (Extracted) {
3481 assert(NumIRArgs == 1);
3482 ArgVals.push_back(Elt: ParamValue::forDirect(value: Coerced));
3483 break;
3484 }
3485 }
3486 }
3487
3488 llvm::StructType *STy =
3489 dyn_cast<llvm::StructType>(Val: ArgI.getCoerceToType());
3490 Address Alloca =
3491 CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(D: Arg), Name: Arg->getName());
3492
3493 // Pointer to store into.
3494 Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3495
3496 // Fast-isel and the optimizer generally like scalar values better than
3497 // FCAs, so we flatten them if this is safe to do for this argument.
3498 if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
3499 STy->getNumElements() > 1) {
3500 llvm::TypeSize StructSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
3501 llvm::TypeSize PtrElementSize =
3502 CGM.getDataLayout().getTypeAllocSize(Ty: Ptr.getElementType());
3503 if (StructSize.isScalable()) {
3504 assert(STy->containsHomogeneousScalableVectorTypes() &&
3505 "ABI only supports structure with homogeneous scalable vector "
3506 "type");
3507 assert(StructSize == PtrElementSize &&
3508 "Only allow non-fractional movement of structure with"
3509 "homogeneous scalable vector type");
3510 assert(STy->getNumElements() == NumIRArgs);
3511
3512 llvm::Value *LoadedStructValue = llvm::PoisonValue::get(T: STy);
3513 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3514 auto *AI = Fn->getArg(i: FirstIRArg + i);
3515 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3516 LoadedStructValue =
3517 Builder.CreateInsertValue(Agg: LoadedStructValue, Val: AI, Idxs: i);
3518 }
3519
3520 Builder.CreateStore(Val: LoadedStructValue, Addr: Ptr);
3521 } else {
3522 uint64_t SrcSize = StructSize.getFixedValue();
3523 uint64_t DstSize = PtrElementSize.getFixedValue();
3524
3525 Address AddrToStoreInto = Address::invalid();
3526 if (SrcSize <= DstSize) {
3527 AddrToStoreInto = Ptr.withElementType(ElemTy: STy);
3528 } else {
3529 AddrToStoreInto =
3530 CreateTempAlloca(Ty: STy, align: Alloca.getAlignment(), Name: "coerce");
3531 }
3532
3533 assert(STy->getNumElements() == NumIRArgs);
3534 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3535 auto AI = Fn->getArg(i: FirstIRArg + i);
3536 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3537 Address EltPtr = Builder.CreateStructGEP(Addr: AddrToStoreInto, Index: i);
3538 Builder.CreateStore(Val: AI, Addr: EltPtr);
3539 }
3540
3541 if (SrcSize > DstSize) {
3542 Builder.CreateMemCpy(Dest: Ptr, Src: AddrToStoreInto, Size: DstSize);
3543 }
3544
3545 // Structures with PFP fields require a coerced store to add any
3546 // pointer signatures.
3547 if (getContext().hasPFPFields(Ty)) {
3548 llvm::Value *Struct = Builder.CreateLoad(Addr: Ptr);
3549 CreatePFPCoercedStore(Src: Struct, SrcFETy: Ty, Dst: Ptr, CGF&: *this);
3550 }
3551 }
3552 } else {
3553 // Simple case, just do a coerced store of the argument into the alloca.
3554 assert(NumIRArgs == 1);
3555 auto AI = Fn->getArg(i: FirstIRArg);
3556 AI->setName(Arg->getName() + ".coerce");
3557 CreateCoercedStore(
3558 Src: AI, SrcFETy: Ty, Dst: Ptr,
3559 DstSize: llvm::TypeSize::getFixed(
3560 ExactSize: getContext().getTypeSizeInChars(T: Ty).getQuantity() -
3561 ArgI.getDirectOffset()),
3562 /*DstIsVolatile=*/false);
3563 }
3564
3565 // Match to what EmitParmDecl is expecting for this type.
3566 if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3567 llvm::Value *V =
3568 EmitLoadOfScalar(Addr: Alloca, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3569 if (isPromoted)
3570 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3571 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3572 } else {
3573 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3574 }
3575 break;
3576 }
3577
3578 case ABIArgInfo::CoerceAndExpand: {
3579 // Reconstruct into a temporary.
3580 Address alloca = CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(D: Arg));
3581 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: alloca));
3582
3583 auto coercionType = ArgI.getCoerceAndExpandType();
3584 auto unpaddedCoercionType = ArgI.getUnpaddedCoerceAndExpandType();
3585 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
3586
3587 alloca = alloca.withElementType(ElemTy: coercionType);
3588
3589 unsigned argIndex = FirstIRArg;
3590 unsigned unpaddedIndex = 0;
3591 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3592 llvm::Type *eltType = coercionType->getElementType(N: i);
3593 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
3594 continue;
3595
3596 auto eltAddr = Builder.CreateStructGEP(Addr: alloca, Index: i);
3597 llvm::Value *elt = Fn->getArg(i: argIndex++);
3598
3599 auto paramType = unpaddedStruct
3600 ? unpaddedStruct->getElementType(N: unpaddedIndex++)
3601 : unpaddedCoercionType;
3602
3603 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: eltType)) {
3604 if (auto *VecTyFrom = dyn_cast<llvm::ScalableVectorType>(Val: paramType)) {
3605 bool Extracted;
3606 std::tie(args&: elt, args&: Extracted) = CoerceScalableToFixed(
3607 CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: elt, Name: elt->getName());
3608 assert(Extracted && "Unexpected scalable to fixed vector coercion");
3609 }
3610 }
3611 Builder.CreateStore(Val: elt, Addr: eltAddr);
3612 }
3613 assert(argIndex == FirstIRArg + NumIRArgs);
3614 break;
3615 }
3616
3617 case ABIArgInfo::Expand: {
3618 // If this structure was expanded into multiple arguments then
3619 // we need to create a temporary and reconstruct it from the
3620 // arguments.
3621 Address Alloca = CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(D: Arg));
3622 LValue LV = MakeAddrLValue(Addr: Alloca, T: Ty);
3623 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3624
3625 auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3626 ExpandTypeFromArgs(Ty, LV, AI&: FnArgIter);
3627 assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3628 for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
3629 auto AI = Fn->getArg(i: FirstIRArg + i);
3630 AI->setName(Arg->getName() + "." + Twine(i));
3631 }
3632 break;
3633 }
3634
3635 case ABIArgInfo::TargetSpecific: {
3636 auto *AI = Fn->getArg(i: FirstIRArg);
3637 AI->setName(Arg->getName() + ".target_coerce");
3638 Address Alloca =
3639 CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(D: Arg), Name: Arg->getName());
3640 Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3641 CGM.getABIInfo().createCoercedStore(Val: AI, DstAddr: Ptr, AI: ArgI, DestIsVolatile: false, CGF&: *this);
3642 if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3643 llvm::Value *V =
3644 EmitLoadOfScalar(Addr: Alloca, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3645 if (isPromoted) {
3646 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3647 }
3648 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3649 } else {
3650 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3651 }
3652 break;
3653 }
3654 case ABIArgInfo::Ignore:
3655 assert(NumIRArgs == 0);
3656 // Initialize the local variable appropriately.
3657 if (!hasScalarEvaluationKind(T: Ty)) {
3658 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: CreateMemTemp(T: Ty)));
3659 } else {
3660 llvm::Value *U = llvm::UndefValue::get(T: ConvertType(T: Arg->getType()));
3661 ArgVals.push_back(Elt: ParamValue::forDirect(value: U));
3662 }
3663 break;
3664 }
3665 }
3666
3667 if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3668 for (int I = Args.size() - 1; I >= 0; --I)
3669 EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3670 } else {
3671 for (unsigned I = 0, E = Args.size(); I != E; ++I)
3672 EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3673 }
3674}
3675
3676static void eraseUnusedBitCasts(llvm::Instruction *insn) {
3677 while (insn->use_empty()) {
3678 llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: insn);
3679 if (!bitcast)
3680 return;
3681
3682 // This is "safe" because we would have used a ConstantExpr otherwise.
3683 insn = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3684 bitcast->eraseFromParent();
3685 }
3686}
3687
3688/// Try to emit a fused autorelease of a return result.
3689static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
3690 llvm::Value *result) {
3691 // We must be immediately followed the cast.
3692 llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3693 if (BB->empty())
3694 return nullptr;
3695 if (&BB->back() != result)
3696 return nullptr;
3697
3698 llvm::Type *resultType = result->getType();
3699
3700 // result is in a BasicBlock and is therefore an Instruction.
3701 llvm::Instruction *generator = cast<llvm::Instruction>(Val: result);
3702
3703 SmallVector<llvm::Instruction *, 4> InstsToKill;
3704
3705 // Look for:
3706 // %generator = bitcast %type1* %generator2 to %type2*
3707 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: generator)) {
3708 // We would have emitted this as a constant if the operand weren't
3709 // an Instruction.
3710 generator = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3711
3712 // Require the generator to be immediately followed by the cast.
3713 if (generator->getNextNode() != bitcast)
3714 return nullptr;
3715
3716 InstsToKill.push_back(Elt: bitcast);
3717 }
3718
3719 // Look for:
3720 // %generator = call i8* @objc_retain(i8* %originalResult)
3721 // or
3722 // %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
3723 llvm::CallInst *call = dyn_cast<llvm::CallInst>(Val: generator);
3724 if (!call)
3725 return nullptr;
3726
3727 bool doRetainAutorelease;
3728
3729 if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
3730 doRetainAutorelease = true;
3731 } else if (call->getCalledOperand() ==
3732 CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
3733 doRetainAutorelease = false;
3734
3735 // If we emitted an assembly marker for this call (and the
3736 // ARCEntrypoints field should have been set if so), go looking
3737 // for that call. If we can't find it, we can't do this
3738 // optimization. But it should always be the immediately previous
3739 // instruction, unless we needed bitcasts around the call.
3740 if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
3741 llvm::Instruction *prev = call->getPrevNode();
3742 assert(prev);
3743 if (isa<llvm::BitCastInst>(Val: prev)) {
3744 prev = prev->getPrevNode();
3745 assert(prev);
3746 }
3747 assert(isa<llvm::CallInst>(prev));
3748 assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3749 CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
3750 InstsToKill.push_back(Elt: prev);
3751 }
3752 } else {
3753 return nullptr;
3754 }
3755
3756 result = call->getArgOperand(i: 0);
3757 InstsToKill.push_back(Elt: call);
3758
3759 // Keep killing bitcasts, for sanity. Note that we no longer care
3760 // about precise ordering as long as there's exactly one use.
3761 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: result)) {
3762 if (!bitcast->hasOneUse())
3763 break;
3764 InstsToKill.push_back(Elt: bitcast);
3765 result = bitcast->getOperand(i_nocapture: 0);
3766 }
3767
3768 // Delete all the unnecessary instructions, from latest to earliest.
3769 for (auto *I : InstsToKill)
3770 I->eraseFromParent();
3771
3772 // Do the fused retain/autorelease if we were asked to.
3773 if (doRetainAutorelease)
3774 result = CGF.EmitARCRetainAutoreleaseReturnValue(value: result);
3775
3776 // Cast back to the result type.
3777 return CGF.Builder.CreateBitCast(V: result, DestTy: resultType);
3778}
3779
3780/// If this is a +1 of the value of an immutable 'self', remove it.
3781static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
3782 llvm::Value *result) {
3783 // This is only applicable to a method with an immutable 'self'.
3784 const ObjCMethodDecl *method =
3785 dyn_cast_or_null<ObjCMethodDecl>(Val: CGF.CurCodeDecl);
3786 if (!method)
3787 return nullptr;
3788 const VarDecl *self = method->getSelfDecl();
3789 if (!self->getType().isConstQualified())
3790 return nullptr;
3791
3792 // Look for a retain call. Note: stripPointerCasts looks through returned arg
3793 // functions, which would cause us to miss the retain.
3794 llvm::CallInst *retainCall = dyn_cast<llvm::CallInst>(Val: result);
3795 if (!retainCall || retainCall->getCalledOperand() !=
3796 CGF.CGM.getObjCEntrypoints().objc_retain)
3797 return nullptr;
3798
3799 // Look for an ordinary load of 'self'.
3800 llvm::Value *retainedValue = retainCall->getArgOperand(i: 0);
3801 llvm::LoadInst *load =
3802 dyn_cast<llvm::LoadInst>(Val: retainedValue->stripPointerCasts());
3803 if (!load || load->isAtomic() || load->isVolatile() ||
3804 load->getPointerOperand() != CGF.GetAddrOfLocalVar(VD: self).getBasePointer())
3805 return nullptr;
3806
3807 // Okay! Burn it all down. This relies for correctness on the
3808 // assumption that the retain is emitted as part of the return and
3809 // that thereafter everything is used "linearly".
3810 llvm::Type *resultType = result->getType();
3811 eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: result));
3812 assert(retainCall->use_empty());
3813 retainCall->eraseFromParent();
3814 eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: retainedValue));
3815
3816 return CGF.Builder.CreateBitCast(V: load, DestTy: resultType);
3817}
3818
3819/// Emit an ARC autorelease of the result of a function.
3820///
3821/// \return the value to actually return from the function
3822static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
3823 llvm::Value *result) {
3824 // If we're returning 'self', kill the initial retain. This is a
3825 // heuristic attempt to "encourage correctness" in the really unfortunate
3826 // case where we have a return of self during a dealloc and we desperately
3827 // need to avoid the possible autorelease.
3828 if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
3829 return self;
3830
3831 // At -O0, try to emit a fused retain/autorelease.
3832 if (CGF.shouldUseFusedARCCalls())
3833 if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
3834 return fused;
3835
3836 return CGF.EmitARCAutoreleaseReturnValue(value: result);
3837}
3838
3839/// Heuristically search for a dominating store to the return-value slot.
3840static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
3841 llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer();
3842
3843 // Check if a User is a store which pointerOperand is the ReturnValue.
3844 // We are looking for stores to the ReturnValue, not for stores of the
3845 // ReturnValue to some other location.
3846 auto GetStoreIfValid = [&CGF,
3847 ReturnValuePtr](llvm::User *U) -> llvm::StoreInst * {
3848 auto *SI = dyn_cast<llvm::StoreInst>(Val: U);
3849 if (!SI || SI->getPointerOperand() != ReturnValuePtr ||
3850 SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())
3851 return nullptr;
3852 // These aren't actually possible for non-coerced returns, and we
3853 // only care about non-coerced returns on this code path.
3854 // All memory instructions inside __try block are volatile.
3855 assert(!SI->isAtomic() &&
3856 (!SI->isVolatile() || CGF.currentFunctionUsesSEHTry()));
3857 return SI;
3858 };
3859 // If there are multiple uses of the return-value slot, just check
3860 // for something immediately preceding the IP. Sometimes this can
3861 // happen with how we generate implicit-returns; it can also happen
3862 // with noreturn cleanups.
3863 if (!ReturnValuePtr->hasOneUse()) {
3864 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3865 if (IP->empty())
3866 return nullptr;
3867
3868 // Look at directly preceding instruction, skipping bitcasts, lifetime
3869 // markers, and fake uses and their operands.
3870 const llvm::Instruction *LoadIntoFakeUse = nullptr;
3871 for (llvm::Instruction &I : llvm::reverse(C&: *IP)) {
3872 // Ignore instructions that are just loads for fake uses; the load should
3873 // immediately precede the fake use, so we only need to remember the
3874 // operand for the last fake use seen.
3875 if (LoadIntoFakeUse == &I)
3876 continue;
3877 if (isa<llvm::BitCastInst>(Val: &I))
3878 continue;
3879 if (auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &I)) {
3880 if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
3881 continue;
3882
3883 if (II->getIntrinsicID() == llvm::Intrinsic::fake_use) {
3884 LoadIntoFakeUse = dyn_cast<llvm::Instruction>(Val: II->getArgOperand(i: 0));
3885 continue;
3886 }
3887 }
3888 return GetStoreIfValid(&I);
3889 }
3890 return nullptr;
3891 }
3892
3893 llvm::StoreInst *store = GetStoreIfValid(ReturnValuePtr->user_back());
3894 if (!store)
3895 return nullptr;
3896
3897 // Now do a first-and-dirty dominance check: just walk up the
3898 // single-predecessors chain from the current insertion point.
3899 llvm::BasicBlock *StoreBB = store->getParent();
3900 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3901 llvm::SmallPtrSet<llvm::BasicBlock *, 4> SeenBBs;
3902 while (IP != StoreBB) {
3903 if (!SeenBBs.insert(Ptr: IP).second || !(IP = IP->getSinglePredecessor()))
3904 return nullptr;
3905 }
3906
3907 // Okay, the store's basic block dominates the insertion point; we
3908 // can do our thing.
3909 return store;
3910}
3911
3912// Helper functions for EmitCMSEClearRecord
3913
3914// Set the bits corresponding to a field having width `BitWidth` and located at
3915// offset `BitOffset` (from the least significant bit) within a storage unit of
3916// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
3917// Use little-endian layout, i.e.`Bits[0]` is the LSB.
3918static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
3919 int BitWidth, int CharWidth) {
3920 assert(CharWidth <= 64);
3921 assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3922
3923 int Pos = 0;
3924 if (BitOffset >= CharWidth) {
3925 Pos += BitOffset / CharWidth;
3926 BitOffset = BitOffset % CharWidth;
3927 }
3928
3929 const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
3930 if (BitOffset + BitWidth >= CharWidth) {
3931 Bits[Pos++] |= (Used << BitOffset) & Used;
3932 BitWidth -= CharWidth - BitOffset;
3933 BitOffset = 0;
3934 }
3935
3936 while (BitWidth >= CharWidth) {
3937 Bits[Pos++] = Used;
3938 BitWidth -= CharWidth;
3939 }
3940
3941 if (BitWidth > 0)
3942 Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
3943}
3944
3945// Set the bits corresponding to a field having width `BitWidth` and located at
3946// offset `BitOffset` (from the least significant bit) within a storage unit of
3947// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
3948// `Bits` corresponds to one target byte. Use target endian layout.
3949static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
3950 int StorageSize, int BitOffset, int BitWidth,
3951 int CharWidth, bool BigEndian) {
3952
3953 SmallVector<uint64_t, 8> TmpBits(StorageSize);
3954 setBitRange(Bits&: TmpBits, BitOffset, BitWidth, CharWidth);
3955
3956 if (BigEndian)
3957 std::reverse(first: TmpBits.begin(), last: TmpBits.end());
3958
3959 for (uint64_t V : TmpBits)
3960 Bits[StorageOffset++] |= V;
3961}
3962
3963static void setUsedBits(CodeGenModule &, QualType, int,
3964 SmallVectorImpl<uint64_t> &);
3965
3966// Set the bits in `Bits`, which correspond to the value representations of
3967// the actual members of the record type `RTy`. Note that this function does
3968// not handle base classes, virtual tables, etc, since they cannot happen in
3969// CMSE function arguments or return. The bit mask corresponds to the target
3970// memory layout, i.e. it's endian dependent.
3971static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
3972 SmallVectorImpl<uint64_t> &Bits) {
3973 ASTContext &Context = CGM.getContext();
3974 int CharWidth = Context.getCharWidth();
3975 const RecordDecl *RD = RTy->getDecl()->getDefinition();
3976 const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(D: RD);
3977 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
3978
3979 int Idx = 0;
3980 for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) {
3981 const FieldDecl *F = *I;
3982
3983 if (F->isUnnamedBitField() || F->isZeroLengthBitField() ||
3984 F->getType()->isIncompleteArrayType())
3985 continue;
3986
3987 if (F->isBitField()) {
3988 const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(FD: F);
3989 setBitRange(Bits, StorageOffset: Offset + BFI.StorageOffset.getQuantity(),
3990 StorageSize: BFI.StorageSize / CharWidth, BitOffset: BFI.Offset, BitWidth: BFI.Size, CharWidth,
3991 BigEndian: CGM.getDataLayout().isBigEndian());
3992 continue;
3993 }
3994
3995 setUsedBits(CGM, F->getType(),
3996 Offset + ASTLayout.getFieldOffset(FieldNo: Idx) / CharWidth, Bits);
3997 }
3998}
3999
4000// Set the bits in `Bits`, which correspond to the value representations of
4001// the elements of an array type `ATy`.
4002static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
4003 int Offset, SmallVectorImpl<uint64_t> &Bits) {
4004 const ASTContext &Context = CGM.getContext();
4005
4006 QualType ETy = Context.getBaseElementType(VAT: ATy);
4007 int Size = Context.getTypeSizeInChars(T: ETy).getQuantity();
4008 SmallVector<uint64_t, 4> TmpBits(Size);
4009 setUsedBits(CGM, ETy, 0, TmpBits);
4010
4011 for (int I = 0, N = Context.getConstantArrayElementCount(CA: ATy); I < N; ++I) {
4012 auto Src = TmpBits.begin();
4013 auto Dst = Bits.begin() + Offset + I * Size;
4014 for (int J = 0; J < Size; ++J)
4015 *Dst++ |= *Src++;
4016 }
4017}
4018
4019// Set the bits in `Bits`, which correspond to the value representations of
4020// the type `QTy`.
4021static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
4022 SmallVectorImpl<uint64_t> &Bits) {
4023 if (const auto *RTy = QTy->getAsCanonical<RecordType>())
4024 return setUsedBits(CGM, RTy, Offset, Bits);
4025
4026 ASTContext &Context = CGM.getContext();
4027 if (const auto *ATy = Context.getAsConstantArrayType(T: QTy))
4028 return setUsedBits(CGM, ATy, Offset, Bits);
4029
4030 int Size = Context.getTypeSizeInChars(T: QTy).getQuantity();
4031 if (Size <= 0)
4032 return;
4033
4034 std::fill_n(first: Bits.begin() + Offset, n: Size,
4035 value: (uint64_t(1) << Context.getCharWidth()) - 1);
4036}
4037
4038static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
4039 int Pos, int Size, int CharWidth,
4040 bool BigEndian) {
4041 assert(Size > 0);
4042 uint64_t Mask = 0;
4043 if (BigEndian) {
4044 for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
4045 ++P)
4046 Mask = (Mask << CharWidth) | *P;
4047 } else {
4048 auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
4049 do
4050 Mask = (Mask << CharWidth) | *--P;
4051 while (P != End);
4052 }
4053 return Mask;
4054}
4055
4056// Emit code to clear the bits in a record, which aren't a part of any user
4057// declared member, when the record is a function return.
4058llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
4059 llvm::IntegerType *ITy,
4060 QualType QTy) {
4061 assert(Src->getType() == ITy);
4062 assert(ITy->getScalarSizeInBits() <= 64);
4063
4064 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
4065 int Size = DataLayout.getTypeStoreSize(Ty: ITy);
4066 SmallVector<uint64_t, 4> Bits(Size);
4067 setUsedBits(CGM, RTy: QTy->castAsCanonical<RecordType>(), Offset: 0, Bits);
4068
4069 int CharWidth = CGM.getContext().getCharWidth();
4070 uint64_t Mask =
4071 buildMultiCharMask(Bits, Pos: 0, Size, CharWidth, BigEndian: DataLayout.isBigEndian());
4072
4073 return Builder.CreateAnd(LHS: Src, RHS: Mask, Name: "cmse.clear");
4074}
4075
4076// Emit code to clear the bits in a record, which aren't a part of any user
4077// declared member, when the record is a function argument.
4078llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
4079 llvm::ArrayType *ATy,
4080 QualType QTy) {
4081 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
4082 int Size = DataLayout.getTypeStoreSize(Ty: ATy);
4083 SmallVector<uint64_t, 16> Bits(Size);
4084 setUsedBits(CGM, RTy: QTy->castAsCanonical<RecordType>(), Offset: 0, Bits);
4085
4086 // Clear each element of the LLVM array.
4087 int CharWidth = CGM.getContext().getCharWidth();
4088 int CharsPerElt =
4089 ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
4090 int MaskIndex = 0;
4091 llvm::Value *R = llvm::PoisonValue::get(T: ATy);
4092 for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
4093 uint64_t Mask = buildMultiCharMask(Bits, Pos: MaskIndex, Size: CharsPerElt, CharWidth,
4094 BigEndian: DataLayout.isBigEndian());
4095 MaskIndex += CharsPerElt;
4096 llvm::Value *T0 = Builder.CreateExtractValue(Agg: Src, Idxs: I);
4097 llvm::Value *T1 = Builder.CreateAnd(LHS: T0, RHS: Mask, Name: "cmse.clear");
4098 R = Builder.CreateInsertValue(Agg: R, Val: T1, Idxs: I);
4099 }
4100
4101 return R;
4102}
4103
4104void CodeGenFunction::EmitFunctionEpilog(
4105 const CGFunctionInfo &FI, bool EmitRetDbgLoc, SourceLocation EndLoc,
4106 uint64_t RetKeyInstructionsSourceAtom) {
4107 if (FI.isNoReturn()) {
4108 // Noreturn functions don't return.
4109 EmitUnreachable(Loc: EndLoc);
4110 return;
4111 }
4112
4113 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {
4114 // Naked functions don't have epilogues.
4115 Builder.CreateUnreachable();
4116 return;
4117 }
4118
4119 // Functions with no result always return void.
4120 if (!ReturnValue.isValid()) {
4121 auto *I = Builder.CreateRetVoid();
4122 if (RetKeyInstructionsSourceAtom)
4123 addInstToSpecificSourceAtom(KeyInstruction: I, Backup: nullptr, Atom: RetKeyInstructionsSourceAtom);
4124 else
4125 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4126 return;
4127 }
4128
4129 llvm::DebugLoc RetDbgLoc;
4130 llvm::Value *RV = nullptr;
4131 QualType RetTy = FI.getReturnType();
4132 const ABIArgInfo &RetAI = FI.getReturnInfo();
4133
4134 switch (RetAI.getKind()) {
4135 case ABIArgInfo::InAlloca:
4136 // Aggregates get evaluated directly into the destination. Sometimes we
4137 // need to return the sret value in a register, though.
4138 assert(hasAggregateEvaluationKind(RetTy));
4139 if (RetAI.getInAllocaSRet()) {
4140 llvm::Function::arg_iterator EI = CurFn->arg_end();
4141 --EI;
4142 llvm::Value *ArgStruct = &*EI;
4143 llvm::Value *SRet = Builder.CreateStructGEP(
4144 Ty: FI.getArgStruct(), Ptr: ArgStruct, Idx: RetAI.getInAllocaFieldIndex());
4145 llvm::Type *Ty =
4146 cast<llvm::GetElementPtrInst>(Val: SRet)->getResultElementType();
4147 RV = Builder.CreateAlignedLoad(Ty, Addr: SRet, Align: getPointerAlign(), Name: "sret");
4148 }
4149 break;
4150
4151 case ABIArgInfo::Indirect: {
4152 auto AI = CurFn->arg_begin();
4153 if (RetAI.isSRetAfterThis())
4154 ++AI;
4155 switch (getEvaluationKind(T: RetTy)) {
4156 case TEK_Complex: {
4157 ComplexPairTy RT =
4158 EmitLoadOfComplex(src: MakeAddrLValue(Addr: ReturnValue, T: RetTy), loc: EndLoc);
4159 EmitStoreOfComplex(V: RT, dest: MakeNaturalAlignAddrLValue(V: &*AI, T: RetTy),
4160 /*isInit*/ true);
4161 break;
4162 }
4163 case TEK_Aggregate:
4164 // Do nothing; aggregates get evaluated directly into the destination.
4165 break;
4166 case TEK_Scalar: {
4167 LValueBaseInfo BaseInfo;
4168 TBAAAccessInfo TBAAInfo;
4169 CharUnits Alignment =
4170 CGM.getNaturalTypeAlignment(T: RetTy, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
4171 Address ArgAddr(&*AI, ConvertType(T: RetTy), Alignment);
4172 LValue ArgVal =
4173 LValue::MakeAddr(Addr: ArgAddr, type: RetTy, Context&: getContext(), BaseInfo, TBAAInfo);
4174 EmitStoreOfScalar(
4175 value: EmitLoadOfScalar(lvalue: MakeAddrLValue(Addr: ReturnValue, T: RetTy), Loc: EndLoc), lvalue: ArgVal,
4176 /*isInit*/ true);
4177 break;
4178 }
4179 }
4180 break;
4181 }
4182
4183 case ABIArgInfo::Extend:
4184 case ABIArgInfo::Direct:
4185 if (RetAI.getCoerceToType() == ConvertType(T: RetTy) &&
4186 RetAI.getDirectOffset() == 0) {
4187 // The internal return value temp always will have pointer-to-return-type
4188 // type, just do a load.
4189
4190 // If there is a dominating store to ReturnValue, we can elide
4191 // the load, zap the store, and usually zap the alloca.
4192 if (llvm::StoreInst *SI = findDominatingStoreToReturnValue(CGF&: *this)) {
4193 // Reuse the debug location from the store unless there is
4194 // cleanup code to be emitted between the store and return
4195 // instruction.
4196 if (EmitRetDbgLoc && !AutoreleaseResult)
4197 RetDbgLoc = SI->getDebugLoc();
4198 // Get the stored value and nuke the now-dead store.
4199 RV = SI->getValueOperand();
4200 SI->eraseFromParent();
4201
4202 // Otherwise, we have to do a simple load.
4203 } else {
4204 RV = Builder.CreateLoad(Addr: ReturnValue);
4205 }
4206 } else {
4207 // If the value is offset in memory, apply the offset now.
4208 Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
4209
4210 RV = CreateCoercedLoad(Src: V, SrcFETy: RetTy, Ty: RetAI.getCoerceToType(), CGF&: *this);
4211 }
4212
4213 // In ARC, end functions that return a retainable type with a call
4214 // to objc_autoreleaseReturnValue.
4215 if (AutoreleaseResult) {
4216#ifndef NDEBUG
4217 // Type::isObjCRetainabletype has to be called on a QualType that hasn't
4218 // been stripped of the typedefs, so we cannot use RetTy here. Get the
4219 // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
4220 // CurCodeDecl or BlockInfo.
4221 QualType RT;
4222
4223 if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
4224 RT = FD->getReturnType();
4225 else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
4226 RT = MD->getReturnType();
4227 else if (isa<BlockDecl>(CurCodeDecl))
4228 RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
4229 else
4230 llvm_unreachable("Unexpected function/method type");
4231
4232 assert(getLangOpts().ObjCAutoRefCount && !FI.isReturnsRetained() &&
4233 RT->isObjCRetainableType());
4234#endif
4235 RV = emitAutoreleaseOfResult(CGF&: *this, result: RV);
4236 }
4237
4238 break;
4239
4240 case ABIArgInfo::Ignore:
4241 break;
4242
4243 case ABIArgInfo::CoerceAndExpand: {
4244 auto coercionType = RetAI.getCoerceAndExpandType();
4245 auto unpaddedCoercionType = RetAI.getUnpaddedCoerceAndExpandType();
4246 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
4247
4248 // Load all of the coerced elements out into results.
4249 llvm::SmallVector<llvm::Value *, 4> results;
4250 Address addr = ReturnValue.withElementType(ElemTy: coercionType);
4251 unsigned unpaddedIndex = 0;
4252 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
4253 auto coercedEltType = coercionType->getElementType(N: i);
4254 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType: coercedEltType))
4255 continue;
4256
4257 auto eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
4258 llvm::Value *elt = CreateCoercedLoad(
4259 Src: eltAddr, SrcFETy: RetTy,
4260 Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
4261 : unpaddedCoercionType,
4262 CGF&: *this);
4263 results.push_back(Elt: elt);
4264 }
4265
4266 // If we have one result, it's the single direct result type.
4267 if (results.size() == 1) {
4268 RV = results[0];
4269
4270 // Otherwise, we need to make a first-class aggregate.
4271 } else {
4272 // Construct a return type that lacks padding elements.
4273 llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
4274
4275 RV = llvm::PoisonValue::get(T: returnType);
4276 for (unsigned i = 0, e = results.size(); i != e; ++i) {
4277 RV = Builder.CreateInsertValue(Agg: RV, Val: results[i], Idxs: i);
4278 }
4279 }
4280 break;
4281 }
4282 case ABIArgInfo::TargetSpecific: {
4283 Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
4284 RV = CGM.getABIInfo().createCoercedLoad(SrcAddr: V, AI: RetAI, CGF&: *this);
4285 break;
4286 }
4287 case ABIArgInfo::Expand:
4288 case ABIArgInfo::IndirectAliased:
4289 llvm_unreachable("Invalid ABI kind for return argument");
4290 }
4291
4292 llvm::Instruction *Ret;
4293 if (RV) {
4294 if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) {
4295 // For certain return types, clear padding bits, as they may reveal
4296 // sensitive information.
4297 // Small struct/union types are passed as integers.
4298 auto *ITy = dyn_cast<llvm::IntegerType>(Val: RV->getType());
4299 if (ITy != nullptr && isa<RecordType>(Val: RetTy.getCanonicalType()))
4300 RV = EmitCMSEClearRecord(Src: RV, ITy, QTy: RetTy);
4301 }
4302 EmitReturnValueCheck(RV);
4303 Ret = Builder.CreateRet(V: RV);
4304 } else {
4305 Ret = Builder.CreateRetVoid();
4306 }
4307
4308 if (RetDbgLoc)
4309 Ret->setDebugLoc(std::move(RetDbgLoc));
4310
4311 llvm::Value *Backup = RV ? Ret->getOperand(i: 0) : nullptr;
4312 if (RetKeyInstructionsSourceAtom)
4313 addInstToSpecificSourceAtom(KeyInstruction: Ret, Backup, Atom: RetKeyInstructionsSourceAtom);
4314 else
4315 addInstToNewSourceAtom(KeyInstruction: Ret, Backup);
4316}
4317
4318void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
4319 // A current decl may not be available when emitting vtable thunks.
4320 if (!CurCodeDecl)
4321 return;
4322
4323 // If the return block isn't reachable, neither is this check, so don't emit
4324 // it.
4325 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty())
4326 return;
4327
4328 ReturnsNonNullAttr *RetNNAttr = nullptr;
4329 if (SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute))
4330 RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
4331
4332 if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
4333 return;
4334
4335 // Prefer the returns_nonnull attribute if it's present.
4336 SourceLocation AttrLoc;
4337 SanitizerKind::SanitizerOrdinal CheckKind;
4338 SanitizerHandler Handler;
4339 if (RetNNAttr) {
4340 assert(!requiresReturnValueNullabilityCheck() &&
4341 "Cannot check nullability and the nonnull attribute");
4342 AttrLoc = RetNNAttr->getLocation();
4343 CheckKind = SanitizerKind::SO_ReturnsNonnullAttribute;
4344 Handler = SanitizerHandler::NonnullReturn;
4345 } else {
4346 if (auto *DD = dyn_cast<DeclaratorDecl>(Val: CurCodeDecl))
4347 if (auto *TSI = DD->getTypeSourceInfo())
4348 if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
4349 AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
4350 CheckKind = SanitizerKind::SO_NullabilityReturn;
4351 Handler = SanitizerHandler::NullabilityReturn;
4352 }
4353
4354 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4355
4356 // Make sure the "return" source location is valid. If we're checking a
4357 // nullability annotation, make sure the preconditions for the check are met.
4358 llvm::BasicBlock *Check = createBasicBlock(name: "nullcheck");
4359 llvm::BasicBlock *NoCheck = createBasicBlock(name: "no.nullcheck");
4360 llvm::Value *SLocPtr = Builder.CreateLoad(Addr: ReturnLocation, Name: "return.sloc.load");
4361 llvm::Value *CanNullCheck = Builder.CreateIsNotNull(Arg: SLocPtr);
4362 if (requiresReturnValueNullabilityCheck())
4363 CanNullCheck =
4364 Builder.CreateAnd(LHS: CanNullCheck, RHS: RetValNullabilityPrecondition);
4365 Builder.CreateCondBr(Cond: CanNullCheck, True: Check, False: NoCheck);
4366 EmitBlock(BB: Check);
4367
4368 // Now do the null check.
4369 llvm::Value *Cond = Builder.CreateIsNotNull(Arg: RV);
4370 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc: AttrLoc)};
4371 llvm::Value *DynamicData[] = {SLocPtr};
4372 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: DynamicData);
4373
4374 EmitBlock(BB: NoCheck);
4375
4376#ifndef NDEBUG
4377 // The return location should not be used after the check has been emitted.
4378 ReturnLocation = Address::invalid();
4379#endif
4380}
4381
4382static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
4383 const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
4384 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
4385}
4386
4387static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, QualType Ty) {
4388 // FIXME: Generate IR in one pass, rather than going back and fixing up these
4389 // placeholders.
4390 llvm::Type *IRTy = CGF.ConvertTypeForMem(T: Ty);
4391 llvm::Type *IRPtrTy = llvm::PointerType::getUnqual(C&: CGF.getLLVMContext());
4392 llvm::Value *Placeholder = llvm::PoisonValue::get(T: IRPtrTy);
4393
4394 // FIXME: When we generate this IR in one pass, we shouldn't need
4395 // this win32-specific alignment hack.
4396 CharUnits Align = CharUnits::fromQuantity(Quantity: 4);
4397 Placeholder = CGF.Builder.CreateAlignedLoad(Ty: IRPtrTy, Addr: Placeholder, Align);
4398
4399 return AggValueSlot::forAddr(
4400 addr: Address(Placeholder, IRTy, Align), quals: Ty.getQualifiers(),
4401 isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
4402 isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
4403}
4404
4405void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
4406 const VarDecl *param,
4407 SourceLocation loc) {
4408 // StartFunction converted the ABI-lowered parameter(s) into a
4409 // local alloca. We need to turn that into an r-value suitable
4410 // for EmitCall.
4411 Address local = GetAddrOfLocalVar(VD: param);
4412
4413 QualType type = param->getType();
4414
4415 // GetAddrOfLocalVar returns a pointer-to-pointer for references,
4416 // but the argument needs to be the original pointer.
4417 if (type->isReferenceType()) {
4418 args.add(rvalue: RValue::get(V: Builder.CreateLoad(Addr: local)), type);
4419
4420 // In ARC, move out of consumed arguments so that the release cleanup
4421 // entered by StartFunction doesn't cause an over-release. This isn't
4422 // optimal -O0 code generation, but it should get cleaned up when
4423 // optimization is enabled. This also assumes that delegate calls are
4424 // performed exactly once for a set of arguments, but that should be safe.
4425 } else if (getLangOpts().ObjCAutoRefCount &&
4426 param->hasAttr<NSConsumedAttr>() && type->isObjCRetainableType()) {
4427 llvm::Value *ptr = Builder.CreateLoad(Addr: local);
4428 auto null =
4429 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: ptr->getType()));
4430 Builder.CreateStore(Val: null, Addr: local);
4431 args.add(rvalue: RValue::get(V: ptr), type);
4432
4433 // For the most part, we just need to load the alloca, except that
4434 // aggregate r-values are actually pointers to temporaries.
4435 } else {
4436 args.add(rvalue: convertTempToRValue(addr: local, type, Loc: loc), type);
4437 }
4438
4439 // Deactivate the cleanup for the callee-destructed param that was pushed.
4440 if (type->isRecordType() && !CurFuncIsThunk &&
4441 type->castAsRecordDecl()->isParamDestroyedInCallee() &&
4442 param->needsDestruction(Ctx: getContext())) {
4443 EHScopeStack::stable_iterator cleanup =
4444 CalleeDestructedParamCleanups.lookup(Val: cast<ParmVarDecl>(Val: param));
4445 assert(cleanup.isValid() &&
4446 "cleanup for callee-destructed param not recorded");
4447 // This unreachable is a temporary marker which will be removed later.
4448 llvm::Instruction *isActive = Builder.CreateUnreachable();
4449 args.addArgCleanupDeactivation(Cleanup: cleanup, IsActiveIP: isActive);
4450 }
4451}
4452
4453static bool isProvablyNull(llvm::Value *addr) {
4454 return llvm::isa_and_nonnull<llvm::ConstantPointerNull>(Val: addr);
4455}
4456
4457static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) {
4458 return llvm::isKnownNonZero(V: Addr.getBasePointer(), Q: CGF.CGM.getDataLayout());
4459}
4460
4461/// Emit the actual writing-back of a writeback.
4462static void emitWriteback(CodeGenFunction &CGF,
4463 const CallArgList::Writeback &writeback) {
4464 const LValue &srcLV = writeback.Source;
4465 Address srcAddr = srcLV.getAddress();
4466 assert(!isProvablyNull(srcAddr.getBasePointer()) &&
4467 "shouldn't have writeback for provably null argument");
4468
4469 if (writeback.WritebackExpr) {
4470 CGF.EmitIgnoredExpr(E: writeback.WritebackExpr);
4471 CGF.EmitLifetimeEnd(Addr: writeback.Temporary.getBasePointer());
4472 return;
4473 }
4474
4475 llvm::BasicBlock *contBB = nullptr;
4476
4477 // If the argument wasn't provably non-null, we need to null check
4478 // before doing the store.
4479 bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4480
4481 if (!provablyNonNull) {
4482 llvm::BasicBlock *writebackBB = CGF.createBasicBlock(name: "icr.writeback");
4483 contBB = CGF.createBasicBlock(name: "icr.done");
4484
4485 llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4486 CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: writebackBB);
4487 CGF.EmitBlock(BB: writebackBB);
4488 }
4489
4490 // Load the value to writeback.
4491 llvm::Value *value = CGF.Builder.CreateLoad(Addr: writeback.Temporary);
4492
4493 // Cast it back, in case we're writing an id to a Foo* or something.
4494 value = CGF.Builder.CreateBitCast(V: value, DestTy: srcAddr.getElementType(),
4495 Name: "icr.writeback-cast");
4496
4497 // Perform the writeback.
4498
4499 // If we have a "to use" value, it's something we need to emit a use
4500 // of. This has to be carefully threaded in: if it's done after the
4501 // release it's potentially undefined behavior (and the optimizer
4502 // will ignore it), and if it happens before the retain then the
4503 // optimizer could move the release there.
4504 if (writeback.ToUse) {
4505 assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
4506
4507 // Retain the new value. No need to block-copy here: the block's
4508 // being passed up the stack.
4509 value = CGF.EmitARCRetainNonBlock(value);
4510
4511 // Emit the intrinsic use here.
4512 CGF.EmitARCIntrinsicUse(values: writeback.ToUse);
4513
4514 // Load the old value (primitively).
4515 llvm::Value *oldValue = CGF.EmitLoadOfScalar(lvalue: srcLV, Loc: SourceLocation());
4516
4517 // Put the new value in place (primitively).
4518 CGF.EmitStoreOfScalar(value, lvalue: srcLV, /*init*/ isInit: false);
4519
4520 // Release the old value.
4521 CGF.EmitARCRelease(value: oldValue, precise: srcLV.isARCPreciseLifetime());
4522
4523 // Otherwise, we can just do a normal lvalue store.
4524 } else {
4525 CGF.EmitStoreThroughLValue(Src: RValue::get(V: value), Dst: srcLV);
4526 }
4527
4528 // Jump to the continuation block.
4529 if (!provablyNonNull)
4530 CGF.EmitBlock(BB: contBB);
4531}
4532
4533static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
4534 const CallArgList &CallArgs) {
4535 ArrayRef<CallArgList::CallArgCleanup> Cleanups =
4536 CallArgs.getCleanupsToDeactivate();
4537 // Iterate in reverse to increase the likelihood of popping the cleanup.
4538 for (const auto &I : llvm::reverse(C&: Cleanups)) {
4539 CGF.DeactivateCleanupBlock(Cleanup: I.Cleanup, DominatingIP: I.IsActiveIP);
4540 I.IsActiveIP->eraseFromParent();
4541 }
4542}
4543
4544static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
4545 if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(Val: E->IgnoreParens()))
4546 if (uop->getOpcode() == UO_AddrOf)
4547 return uop->getSubExpr();
4548 return nullptr;
4549}
4550
4551/// Emit an argument that's being passed call-by-writeback. That is,
4552/// we are passing the address of an __autoreleased temporary; it
4553/// might be copy-initialized with the current value of the given
4554/// address, but it will definitely be copied out of after the call.
4555static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
4556 const ObjCIndirectCopyRestoreExpr *CRE) {
4557 LValue srcLV;
4558
4559 // Make an optimistic effort to emit the address as an l-value.
4560 // This can fail if the argument expression is more complicated.
4561 if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(E: CRE->getSubExpr())) {
4562 srcLV = CGF.EmitLValue(E: lvExpr);
4563
4564 // Otherwise, just emit it as a scalar.
4565 } else {
4566 Address srcAddr = CGF.EmitPointerWithAlignment(Addr: CRE->getSubExpr());
4567
4568 QualType srcAddrType =
4569 CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
4570 srcLV = CGF.MakeAddrLValue(Addr: srcAddr, T: srcAddrType);
4571 }
4572 Address srcAddr = srcLV.getAddress();
4573
4574 // The dest and src types don't necessarily match in LLVM terms
4575 // because of the crazy ObjC compatibility rules.
4576
4577 llvm::PointerType *destType =
4578 cast<llvm::PointerType>(Val: CGF.ConvertType(T: CRE->getType()));
4579 llvm::Type *destElemType =
4580 CGF.ConvertTypeForMem(T: CRE->getType()->getPointeeType());
4581
4582 // If the address is a constant null, just pass the appropriate null.
4583 if (isProvablyNull(addr: srcAddr.getBasePointer())) {
4584 args.add(rvalue: RValue::get(V: llvm::ConstantPointerNull::get(T: destType)),
4585 type: CRE->getType());
4586 return;
4587 }
4588
4589 // Create the temporary.
4590 Address temp =
4591 CGF.CreateTempAlloca(Ty: destElemType, align: CGF.getPointerAlign(), Name: "icr.temp");
4592 // Loading an l-value can introduce a cleanup if the l-value is __weak,
4593 // and that cleanup will be conditional if we can't prove that the l-value
4594 // isn't null, so we need to register a dominating point so that the cleanups
4595 // system will make valid IR.
4596 CodeGenFunction::ConditionalEvaluation condEval(CGF);
4597
4598 // Zero-initialize it if we're not doing a copy-initialization.
4599 bool shouldCopy = CRE->shouldCopy();
4600 if (!shouldCopy) {
4601 llvm::Value *null =
4602 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: destElemType));
4603 CGF.Builder.CreateStore(Val: null, Addr: temp);
4604 }
4605
4606 llvm::BasicBlock *contBB = nullptr;
4607 llvm::BasicBlock *originBB = nullptr;
4608
4609 // If the address is *not* known to be non-null, we need to switch.
4610 llvm::Value *finalArgument;
4611
4612 bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4613
4614 if (provablyNonNull) {
4615 finalArgument = temp.emitRawPointer(CGF);
4616 } else {
4617 llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4618
4619 finalArgument = CGF.Builder.CreateSelect(
4620 C: isNull, True: llvm::ConstantPointerNull::get(T: destType),
4621 False: temp.emitRawPointer(CGF), Name: "icr.argument");
4622
4623 // If we need to copy, then the load has to be conditional, which
4624 // means we need control flow.
4625 if (shouldCopy) {
4626 originBB = CGF.Builder.GetInsertBlock();
4627 contBB = CGF.createBasicBlock(name: "icr.cont");
4628 llvm::BasicBlock *copyBB = CGF.createBasicBlock(name: "icr.copy");
4629 CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: copyBB);
4630 CGF.EmitBlock(BB: copyBB);
4631 condEval.begin(CGF);
4632 }
4633 }
4634
4635 llvm::Value *valueToUse = nullptr;
4636
4637 // Perform a copy if necessary.
4638 if (shouldCopy) {
4639 RValue srcRV = CGF.EmitLoadOfLValue(V: srcLV, Loc: SourceLocation());
4640 assert(srcRV.isScalar());
4641
4642 llvm::Value *src = srcRV.getScalarVal();
4643 src = CGF.Builder.CreateBitCast(V: src, DestTy: destElemType, Name: "icr.cast");
4644
4645 // Use an ordinary store, not a store-to-lvalue.
4646 CGF.Builder.CreateStore(Val: src, Addr: temp);
4647
4648 // If optimization is enabled, and the value was held in a
4649 // __strong variable, we need to tell the optimizer that this
4650 // value has to stay alive until we're doing the store back.
4651 // This is because the temporary is effectively unretained,
4652 // and so otherwise we can violate the high-level semantics.
4653 if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4654 srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {
4655 valueToUse = src;
4656 }
4657 }
4658
4659 // Finish the control flow if we needed it.
4660 if (shouldCopy && !provablyNonNull) {
4661 llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
4662 CGF.EmitBlock(BB: contBB);
4663
4664 // Make a phi for the value to intrinsically use.
4665 if (valueToUse) {
4666 llvm::PHINode *phiToUse =
4667 CGF.Builder.CreatePHI(Ty: valueToUse->getType(), NumReservedValues: 2, Name: "icr.to-use");
4668 phiToUse->addIncoming(V: valueToUse, BB: copyBB);
4669 phiToUse->addIncoming(V: llvm::PoisonValue::get(T: valueToUse->getType()),
4670 BB: originBB);
4671 valueToUse = phiToUse;
4672 }
4673
4674 condEval.end(CGF);
4675 }
4676
4677 args.addWriteback(srcLV, temporary: temp, toUse: valueToUse);
4678 args.add(rvalue: RValue::get(V: finalArgument), type: CRE->getType());
4679}
4680
4681void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
4682 assert(!StackBase);
4683
4684 // Save the stack.
4685 StackBase = CGF.Builder.CreateStackSave(Name: "inalloca.save");
4686}
4687
4688void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
4689 if (StackBase) {
4690 // Restore the stack after the call.
4691 CGF.Builder.CreateStackRestore(Ptr: StackBase);
4692 }
4693}
4694
4695void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
4696 SourceLocation ArgLoc,
4697 AbstractCallee AC, unsigned ParmNum) {
4698 if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
4699 SanOpts.has(K: SanitizerKind::NullabilityArg)))
4700 return;
4701
4702 // The param decl may be missing in a variadic function.
4703 auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(I: ParmNum) : nullptr;
4704 unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
4705
4706 // Prefer the nonnull attribute if it's present.
4707 const NonNullAttr *NNAttr = nullptr;
4708 if (SanOpts.has(K: SanitizerKind::NonnullAttribute))
4709 NNAttr = getNonNullAttr(FD: AC.getDecl(), PVD, ArgType, ArgNo);
4710
4711 bool CanCheckNullability = false;
4712 if (SanOpts.has(K: SanitizerKind::NullabilityArg) && !NNAttr && PVD &&
4713 !PVD->getType()->isRecordType()) {
4714 auto Nullability = PVD->getType()->getNullability();
4715 CanCheckNullability = Nullability &&
4716 *Nullability == NullabilityKind::NonNull &&
4717 PVD->getTypeSourceInfo();
4718 }
4719
4720 if (!NNAttr && !CanCheckNullability)
4721 return;
4722
4723 SourceLocation AttrLoc;
4724 SanitizerKind::SanitizerOrdinal CheckKind;
4725 SanitizerHandler Handler;
4726 if (NNAttr) {
4727 AttrLoc = NNAttr->getLocation();
4728 CheckKind = SanitizerKind::SO_NonnullAttribute;
4729 Handler = SanitizerHandler::NonnullArg;
4730 } else {
4731 AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4732 CheckKind = SanitizerKind::SO_NullabilityArg;
4733 Handler = SanitizerHandler::NullabilityArg;
4734 }
4735
4736 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4737 llvm::Value *Cond = EmitNonNullRValueCheck(RV, T: ArgType);
4738 llvm::Constant *StaticData[] = {
4739 EmitCheckSourceLocation(Loc: ArgLoc),
4740 EmitCheckSourceLocation(Loc: AttrLoc),
4741 llvm::ConstantInt::get(Ty: Int32Ty, V: ArgNo + 1),
4742 };
4743 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: {});
4744}
4745
4746void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType,
4747 SourceLocation ArgLoc,
4748 AbstractCallee AC, unsigned ParmNum) {
4749 if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
4750 SanOpts.has(K: SanitizerKind::NullabilityArg)))
4751 return;
4752
4753 EmitNonNullArgCheck(RV: RValue::get(Addr, CGF&: *this), ArgType, ArgLoc, AC, ParmNum);
4754}
4755
4756// Check if the call is going to use the inalloca convention. This needs to
4757// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
4758// later, so we can't check it directly.
4759static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
4760 ArrayRef<QualType> ArgTypes) {
4761 // The Swift calling conventions don't go through the target-specific
4762 // argument classification, they never use inalloca.
4763 // TODO: Consider limiting inalloca use to only calling conventions supported
4764 // by MSVC.
4765 if (ExplicitCC == CC_Swift || ExplicitCC == CC_SwiftAsync)
4766 return false;
4767 if (!CGM.getTarget().getCXXABI().isMicrosoft())
4768 return false;
4769 return llvm::any_of(Range&: ArgTypes, P: [&](QualType Ty) {
4770 return isInAllocaArgument(ABI&: CGM.getCXXABI(), type: Ty);
4771 });
4772}
4773
4774#ifndef NDEBUG
4775// Determine whether the given argument is an Objective-C method
4776// that may have type parameters in its signature.
4777static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
4778 const DeclContext *dc = method->getDeclContext();
4779 if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {
4780 return classDecl->getTypeParamListAsWritten();
4781 }
4782
4783 if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
4784 return catDecl->getTypeParamList();
4785 }
4786
4787 return false;
4788}
4789#endif
4790
4791/// EmitCallArgs - Emit call arguments for a function.
4792void CodeGenFunction::EmitCallArgs(
4793 CallArgList &Args, PrototypeWrapper Prototype,
4794 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4795 AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
4796 SmallVector<QualType, 16> ArgTypes;
4797
4798 assert((ParamsToSkip == 0 || Prototype.P) &&
4799 "Can't skip parameters if type info is not provided");
4800
4801 // This variable only captures *explicitly* written conventions, not those
4802 // applied by default via command line flags or target defaults, such as
4803 // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
4804 // require knowing if this is a C++ instance method or being able to see
4805 // unprototyped FunctionTypes.
4806 CallingConv ExplicitCC = CC_C;
4807
4808 // First, if a prototype was provided, use those argument types.
4809 bool IsVariadic = false;
4810 if (Prototype.P) {
4811 const auto *MD = dyn_cast<const ObjCMethodDecl *>(Val&: Prototype.P);
4812 if (MD) {
4813 IsVariadic = MD->isVariadic();
4814 ExplicitCC = getCallingConventionForDecl(
4815 D: MD, IsTargetDefaultMSABI: CGM.getTarget().getTriple().isOSWindows());
4816 ArgTypes.assign(in_start: MD->param_type_begin() + ParamsToSkip,
4817 in_end: MD->param_type_end());
4818 } else {
4819 const auto *FPT = cast<const FunctionProtoType *>(Val&: Prototype.P);
4820 IsVariadic = FPT->isVariadic();
4821 ExplicitCC = FPT->getExtInfo().getCC();
4822 ArgTypes.assign(in_start: FPT->param_type_begin() + ParamsToSkip,
4823 in_end: FPT->param_type_end());
4824 }
4825
4826#ifndef NDEBUG
4827 // Check that the prototyped types match the argument expression types.
4828 bool isGenericMethod = MD && isObjCMethodWithTypeParams(MD);
4829 CallExpr::const_arg_iterator Arg = ArgRange.begin();
4830 for (QualType Ty : ArgTypes) {
4831 assert(Arg != ArgRange.end() && "Running over edge of argument list!");
4832 QualType ParamTy = Ty.getNonReferenceType();
4833 QualType ArgTy = (*Arg)->getType();
4834 if (const auto *OBT = ParamTy->getAs<OverflowBehaviorType>())
4835 ParamTy = OBT->getUnderlyingType();
4836 if (const auto *OBT = ArgTy->getAs<OverflowBehaviorType>())
4837 ArgTy = OBT->getUnderlyingType();
4838 assert((isGenericMethod || Ty->isVariablyModifiedType() ||
4839 ParamTy->isObjCRetainableType() ||
4840 getContext().getCanonicalType(ParamTy).getTypePtr() ==
4841 getContext().getCanonicalType(ArgTy).getTypePtr()) &&
4842 "type mismatch in call argument!");
4843 ++Arg;
4844 }
4845
4846 // Either we've emitted all the call args, or we have a call to variadic
4847 // function.
4848 assert((Arg == ArgRange.end() || IsVariadic) &&
4849 "Extra arguments in non-variadic function!");
4850#endif
4851 }
4852
4853 // If we still have any arguments, emit them using the type of the argument.
4854 for (auto *A : llvm::drop_begin(RangeOrContainer&: ArgRange, N: ArgTypes.size()))
4855 ArgTypes.push_back(Elt: IsVariadic ? getVarArgType(Arg: A) : A->getType());
4856 assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
4857
4858 // We must evaluate arguments from right to left in the MS C++ ABI,
4859 // because arguments are destroyed left to right in the callee. As a special
4860 // case, there are certain language constructs that require left-to-right
4861 // evaluation, and in those cases we consider the evaluation order requirement
4862 // to trump the "destruction order is reverse construction order" guarantee.
4863 bool LeftToRight =
4864 CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
4865 ? Order == EvaluationOrder::ForceLeftToRight
4866 : Order != EvaluationOrder::ForceRightToLeft;
4867
4868 auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
4869 RValue EmittedArg) {
4870 if (!AC.hasFunctionDecl() || I >= AC.getNumParams())
4871 return;
4872 auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
4873 if (PS == nullptr)
4874 return;
4875
4876 const auto &Context = getContext();
4877 auto SizeTy = Context.getSizeType();
4878 auto T = Builder.getIntNTy(N: Context.getTypeSize(T: SizeTy));
4879 assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
4880 llvm::Value *V = evaluateOrEmitBuiltinObjectSize(
4881 E: Arg, Type: PS->getType(), ResType: T, EmittedE: EmittedArg.getScalarVal(), IsDynamic: PS->isDynamic());
4882 Args.add(rvalue: RValue::get(V), type: SizeTy);
4883 // If we're emitting args in reverse, be sure to do so with
4884 // pass_object_size, as well.
4885 if (!LeftToRight)
4886 std::swap(a&: Args.back(), b&: *(&Args.back() - 1));
4887 };
4888
4889 // Insert a stack save if we're going to need any inalloca args.
4890 if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
4891 assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
4892 "inalloca only supported on x86");
4893 Args.allocateArgumentMemory(CGF&: *this);
4894 }
4895
4896 // Evaluate each argument in the appropriate order.
4897 size_t CallArgsStart = Args.size();
4898 for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
4899 unsigned Idx = LeftToRight ? I : E - I - 1;
4900 CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
4901 unsigned InitialArgSize = Args.size();
4902 // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
4903 // the argument and parameter match or the objc method is parameterized.
4904 assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
4905 getContext().hasSameUnqualifiedType((*Arg)->getType(),
4906 ArgTypes[Idx]) ||
4907 (isa<ObjCMethodDecl>(AC.getDecl()) &&
4908 isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
4909 "Argument and parameter types don't match");
4910 EmitCallArg(args&: Args, E: *Arg, ArgType: ArgTypes[Idx]);
4911 // In particular, we depend on it being the last arg in Args, and the
4912 // objectsize bits depend on there only being one arg if !LeftToRight.
4913 assert(InitialArgSize + 1 == Args.size() &&
4914 "The code below depends on only adding one arg per EmitCallArg");
4915 (void)InitialArgSize;
4916 // Since pointer argument are never emitted as LValue, it is safe to emit
4917 // non-null argument check for r-value only.
4918 if (!Args.back().hasLValue()) {
4919 RValue RVArg = Args.back().getKnownRValue();
4920 EmitNonNullArgCheck(RV: RVArg, ArgType: ArgTypes[Idx], ArgLoc: (*Arg)->getExprLoc(), AC,
4921 ParmNum: ParamsToSkip + Idx);
4922 // @llvm.objectsize should never have side-effects and shouldn't need
4923 // destruction/cleanups, so we can safely "emit" it after its arg,
4924 // regardless of right-to-leftness
4925 MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
4926 }
4927 }
4928
4929 if (!LeftToRight) {
4930 // Un-reverse the arguments we just evaluated so they match up with the LLVM
4931 // IR function.
4932 std::reverse(first: Args.begin() + CallArgsStart, last: Args.end());
4933
4934 // Reverse the writebacks to match the MSVC ABI.
4935 Args.reverseWritebacks();
4936 }
4937}
4938
4939namespace {
4940
4941struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
4942 DestroyUnpassedArg(Address Addr, QualType Ty) : Addr(Addr), Ty(Ty) {}
4943
4944 Address Addr;
4945 QualType Ty;
4946
4947 void Emit(CodeGenFunction &CGF, Flags flags) override {
4948 QualType::DestructionKind DtorKind = Ty.isDestructedType();
4949 if (DtorKind == QualType::DK_cxx_destructor) {
4950 const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
4951 assert(!Dtor->isTrivial());
4952 CGF.EmitCXXDestructorCall(D: Dtor, Type: Dtor_Complete, /*for vbase*/ ForVirtualBase: false,
4953 /*Delegating=*/false, This: Addr, ThisTy: Ty);
4954 } else {
4955 CGF.callCStructDestructor(Dst: CGF.MakeAddrLValue(Addr, T: Ty));
4956 }
4957 }
4958};
4959
4960} // end anonymous namespace
4961
4962RValue CallArg::getRValue(CodeGenFunction &CGF) const {
4963 if (!HasLV)
4964 return RV;
4965 LValue Copy = CGF.MakeAddrLValue(Addr: CGF.CreateMemTemp(T: Ty), T: Ty);
4966 CGF.EmitAggregateCopy(Dest: Copy, Src: LV, EltTy: Ty, MayOverlap: AggValueSlot::DoesNotOverlap,
4967 isVolatile: LV.isVolatile());
4968 IsUsed = true;
4969 return RValue::getAggregate(addr: Copy.getAddress());
4970}
4971
4972void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
4973 LValue Dst = CGF.MakeAddrLValue(Addr, T: Ty);
4974 if (!HasLV && RV.isScalar())
4975 CGF.EmitStoreOfScalar(value: RV.getScalarVal(), lvalue: Dst, /*isInit=*/true);
4976 else if (!HasLV && RV.isComplex())
4977 CGF.EmitStoreOfComplex(V: RV.getComplexVal(), dest: Dst, /*init=*/isInit: true);
4978 else {
4979 auto Addr = HasLV ? LV.getAddress() : RV.getAggregateAddress();
4980 LValue SrcLV = CGF.MakeAddrLValue(Addr, T: Ty);
4981 // We assume that call args are never copied into subobjects.
4982 CGF.EmitAggregateCopy(Dest: Dst, Src: SrcLV, EltTy: Ty, MayOverlap: AggValueSlot::DoesNotOverlap,
4983 isVolatile: HasLV ? LV.isVolatileQualified()
4984 : RV.isVolatileQualified());
4985 }
4986 IsUsed = true;
4987}
4988
4989void CodeGenFunction::EmitWritebacks(const CallArgList &args) {
4990 for (const auto &I : args.writebacks())
4991 emitWriteback(CGF&: *this, writeback: I);
4992}
4993
4994void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
4995 QualType type) {
4996 std::optional<DisableDebugLocationUpdates> Dis;
4997 if (isa<CXXDefaultArgExpr>(Val: E))
4998 Dis.emplace(args&: *this);
4999 if (const ObjCIndirectCopyRestoreExpr *CRE =
5000 dyn_cast<ObjCIndirectCopyRestoreExpr>(Val: E)) {
5001 assert(getLangOpts().ObjCAutoRefCount);
5002 return emitWritebackArg(CGF&: *this, args, CRE);
5003 }
5004
5005 // Add writeback for HLSLOutParamExpr.
5006 // Needs to be before the assert below because HLSLOutArgExpr is an LValue
5007 // and is not a reference.
5008 if (const HLSLOutArgExpr *OE = dyn_cast<HLSLOutArgExpr>(Val: E)) {
5009 EmitHLSLOutArgExpr(E: OE, Args&: args, Ty: type);
5010 return;
5011 }
5012
5013 assert(type->isReferenceType() == E->isGLValue() &&
5014 "reference binding to unmaterialized r-value!");
5015
5016 if (E->isGLValue()) {
5017 assert(E->getObjectKind() == OK_Ordinary);
5018 return args.add(rvalue: EmitReferenceBindingToExpr(E), type);
5019 }
5020
5021 bool HasAggregateEvalKind = hasAggregateEvaluationKind(T: type);
5022
5023 // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
5024 // However, we still have to push an EH-only cleanup in case we unwind before
5025 // we make it to the call.
5026 if (type->isRecordType() &&
5027 type->castAsRecordDecl()->isParamDestroyedInCallee()) {
5028 // If we're using inalloca, use the argument memory. Otherwise, use a
5029 // temporary.
5030 AggValueSlot Slot = args.isUsingInAlloca()
5031 ? createPlaceholderSlot(CGF&: *this, Ty: type)
5032 : CreateAggTemp(T: type, Name: "agg.tmp");
5033
5034 bool DestroyedInCallee = true, NeedsCleanup = true;
5035 if (const auto *RD = type->getAsCXXRecordDecl())
5036 DestroyedInCallee = RD->hasNonTrivialDestructor();
5037 else
5038 NeedsCleanup = type.isDestructedType();
5039
5040 if (DestroyedInCallee)
5041 Slot.setExternallyDestructed();
5042
5043 EmitAggExpr(E, AS: Slot);
5044 RValue RV = Slot.asRValue();
5045 args.add(rvalue: RV, type);
5046
5047 if (DestroyedInCallee && NeedsCleanup) {
5048 // Create a no-op GEP between the placeholder and the cleanup so we can
5049 // RAUW it successfully. It also serves as a marker of the first
5050 // instruction where the cleanup is active.
5051 pushFullExprCleanup<DestroyUnpassedArg>(kind: NormalAndEHCleanup,
5052 A: Slot.getAddress(), A: type);
5053 // This unreachable is a temporary marker which will be removed later.
5054 llvm::Instruction *IsActive =
5055 Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
5056 args.addArgCleanupDeactivation(Cleanup: EHStack.stable_begin(), IsActiveIP: IsActive);
5057 }
5058 return;
5059 }
5060
5061 if (HasAggregateEvalKind && isa<ImplicitCastExpr>(Val: E) &&
5062 cast<CastExpr>(Val: E)->getCastKind() == CK_LValueToRValue &&
5063 !type->isArrayParameterType() && !type.isNonTrivialToPrimitiveCopy()) {
5064 LValue L = EmitLValue(E: cast<CastExpr>(Val: E)->getSubExpr());
5065 assert(L.isSimple());
5066 args.addUncopiedAggregate(LV: L, type);
5067 return;
5068 }
5069
5070 args.add(rvalue: EmitAnyExprToTemp(E), type);
5071}
5072
5073QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
5074 // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
5075 // implicitly widens null pointer constants that are arguments to varargs
5076 // functions to pointer-sized ints.
5077 if (!getTarget().getTriple().isOSWindows())
5078 return Arg->getType();
5079
5080 if (Arg->getType()->isIntegerType() &&
5081 getContext().getTypeSize(T: Arg->getType()) <
5082 getContext().getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default) &&
5083 Arg->isNullPointerConstant(Ctx&: getContext(),
5084 NPC: Expr::NPC_ValueDependentIsNotNull)) {
5085 return getContext().getIntPtrType();
5086 }
5087
5088 return Arg->getType();
5089}
5090
5091// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5092// optimizer it can aggressively ignore unwind edges.
5093void CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
5094 if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
5095 !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
5096 Inst->setMetadata(Kind: "clang.arc.no_objc_arc_exceptions",
5097 Node: CGM.getNoObjCARCExceptionsMetadata());
5098}
5099
5100/// Emits a call to the given no-arguments nounwind runtime function.
5101llvm::CallInst *
5102CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5103 const llvm::Twine &name) {
5104 return EmitNounwindRuntimeCall(callee, args: ArrayRef<llvm::Value *>(), name);
5105}
5106
5107/// Emits a call to the given nounwind runtime function.
5108llvm::CallInst *
5109CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5110 ArrayRef<Address> args,
5111 const llvm::Twine &name) {
5112 SmallVector<llvm::Value *, 3> values;
5113 for (auto arg : args)
5114 values.push_back(Elt: arg.emitRawPointer(CGF&: *this));
5115 return EmitNounwindRuntimeCall(callee, args: values, name);
5116}
5117
5118llvm::CallInst *
5119CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5120 ArrayRef<llvm::Value *> args,
5121 const llvm::Twine &name) {
5122 llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
5123 call->setDoesNotThrow();
5124 return call;
5125}
5126
5127/// Emits a simple call (never an invoke) to the given no-arguments
5128/// runtime function.
5129llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5130 const llvm::Twine &name) {
5131 return EmitRuntimeCall(callee, args: {}, name);
5132}
5133
5134// Calls which may throw must have operand bundles indicating which funclet
5135// they are nested within.
5136SmallVector<llvm::OperandBundleDef, 1>
5137CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
5138 // There is no need for a funclet operand bundle if we aren't inside a
5139 // funclet.
5140 if (!CurrentFuncletPad)
5141 return (SmallVector<llvm::OperandBundleDef, 1>());
5142
5143 // Skip intrinsics which cannot throw (as long as they don't lower into
5144 // regular function calls in the course of IR transformations).
5145 if (auto *CalleeFn = dyn_cast<llvm::Function>(Val: Callee->stripPointerCasts())) {
5146 if (CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) {
5147 auto IID = CalleeFn->getIntrinsicID();
5148 if (!llvm::IntrinsicInst::mayLowerToFunctionCall(IID))
5149 return (SmallVector<llvm::OperandBundleDef, 1>());
5150 }
5151 }
5152
5153 SmallVector<llvm::OperandBundleDef, 1> BundleList;
5154 BundleList.emplace_back(Args: "funclet", Args&: CurrentFuncletPad);
5155 return BundleList;
5156}
5157
5158/// Emits a simple call (never an invoke) to the given runtime function.
5159llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5160 ArrayRef<llvm::Value *> args,
5161 const llvm::Twine &name) {
5162 llvm::CallInst *call = Builder.CreateCall(
5163 Callee: callee, Args: args, OpBundles: getBundlesForFunclet(Callee: callee.getCallee()), Name: name);
5164 call->setCallingConv(getRuntimeCC());
5165
5166 if (CGM.shouldEmitConvergenceTokens() && call->isConvergent())
5167 return cast<llvm::CallInst>(Val: addConvergenceControlToken(Input: call));
5168 return call;
5169}
5170
5171/// Emits a call or invoke to the given noreturn runtime function.
5172void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
5173 llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
5174 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5175 getBundlesForFunclet(Callee: callee.getCallee());
5176
5177 if (getInvokeDest()) {
5178 llvm::InvokeInst *invoke = Builder.CreateInvoke(
5179 Callee: callee, NormalDest: getUnreachableBlock(), UnwindDest: getInvokeDest(), Args: args, OpBundles: BundleList);
5180 invoke->setDoesNotReturn();
5181 invoke->setCallingConv(getRuntimeCC());
5182 } else {
5183 llvm::CallInst *call = Builder.CreateCall(Callee: callee, Args: args, OpBundles: BundleList);
5184 call->setDoesNotReturn();
5185 call->setCallingConv(getRuntimeCC());
5186 Builder.CreateUnreachable();
5187 }
5188}
5189
5190/// Emits a call or invoke instruction to the given nullary runtime function.
5191llvm::CallBase *
5192CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5193 const Twine &name) {
5194 return EmitRuntimeCallOrInvoke(callee, args: {}, name);
5195}
5196
5197/// Emits a call or invoke instruction to the given runtime function.
5198llvm::CallBase *
5199CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5200 ArrayRef<llvm::Value *> args,
5201 const Twine &name) {
5202 llvm::CallBase *call = EmitCallOrInvoke(Callee: callee, Args: args, Name: name);
5203 call->setCallingConv(getRuntimeCC());
5204 return call;
5205}
5206
5207/// Emits a call or invoke instruction to the given function, depending
5208/// on the current state of the EH stack.
5209llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
5210 ArrayRef<llvm::Value *> Args,
5211 const Twine &Name) {
5212 llvm::BasicBlock *InvokeDest = getInvokeDest();
5213 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5214 getBundlesForFunclet(Callee: Callee.getCallee());
5215
5216 llvm::CallBase *Inst;
5217 if (!InvokeDest)
5218 Inst = Builder.CreateCall(Callee, Args, OpBundles: BundleList, Name);
5219 else {
5220 llvm::BasicBlock *ContBB = createBasicBlock(name: "invoke.cont");
5221 Inst = Builder.CreateInvoke(Callee, NormalDest: ContBB, UnwindDest: InvokeDest, Args, OpBundles: BundleList,
5222 Name);
5223 EmitBlock(BB: ContBB);
5224 }
5225
5226 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5227 // optimizer it can aggressively ignore unwind edges.
5228 if (CGM.getLangOpts().ObjCAutoRefCount)
5229 AddObjCARCExceptionMetadata(Inst);
5230
5231 return Inst;
5232}
5233
5234void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
5235 llvm::Value *New) {
5236 DeferredReplacements.push_back(
5237 Elt: std::make_pair(x: llvm::WeakTrackingVH(Old), y&: New));
5238}
5239
5240namespace {
5241
5242/// Specify given \p NewAlign as the alignment of return value attribute. If
5243/// such attribute already exists, re-set it to the maximal one of two options.
5244[[nodiscard]] llvm::AttributeList
5245maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
5246 const llvm::AttributeList &Attrs,
5247 llvm::Align NewAlign) {
5248 llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
5249 if (CurAlign >= NewAlign)
5250 return Attrs;
5251 llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Context&: Ctx, Alignment: NewAlign);
5252 return Attrs.removeRetAttribute(C&: Ctx, Kind: llvm::Attribute::AttrKind::Alignment)
5253 .addRetAttribute(C&: Ctx, Attr: AlignAttr);
5254}
5255
5256template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
5257protected:
5258 CodeGenFunction &CGF;
5259
5260 /// We do nothing if this is, or becomes, nullptr.
5261 const AlignedAttrTy *AA = nullptr;
5262
5263 llvm::Value *Alignment = nullptr; // May or may not be a constant.
5264 llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
5265
5266 AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5267 : CGF(CGF_) {
5268 if (!FuncDecl)
5269 return;
5270 AA = FuncDecl->getAttr<AlignedAttrTy>();
5271 }
5272
5273public:
5274 /// If we can, materialize the alignment as an attribute on return value.
5275 [[nodiscard]] llvm::AttributeList
5276 TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
5277 if (!AA || OffsetCI || CGF.SanOpts.has(K: SanitizerKind::Alignment))
5278 return Attrs;
5279 const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Val: Alignment);
5280 if (!AlignmentCI)
5281 return Attrs;
5282 // We may legitimately have non-power-of-2 alignment here.
5283 // If so, this is UB land, emit it via `@llvm.assume` instead.
5284 if (!AlignmentCI->getValue().isPowerOf2())
5285 return Attrs;
5286 llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
5287 Ctx&: CGF.getLLVMContext(), Attrs,
5288 NewAlign: llvm::Align(
5289 AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment)));
5290 AA = nullptr; // We're done. Disallow doing anything else.
5291 return NewAttrs;
5292 }
5293
5294 /// Emit alignment assumption.
5295 /// This is a general fallback that we take if either there is an offset,
5296 /// or the alignment is variable or we are sanitizing for alignment.
5297 void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
5298 if (!AA)
5299 return;
5300 CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
5301 AA->getLocation(), Alignment, OffsetCI);
5302 AA = nullptr; // We're done. Disallow doing anything else.
5303 }
5304};
5305
5306/// Helper data structure to emit `AssumeAlignedAttr`.
5307class AssumeAlignedAttrEmitter final
5308 : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
5309public:
5310 AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5311 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5312 if (!AA)
5313 return;
5314 // It is guaranteed that the alignment/offset are constants.
5315 Alignment = cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: AA->getAlignment()));
5316 if (Expr *Offset = AA->getOffset()) {
5317 OffsetCI = cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: Offset));
5318 if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
5319 OffsetCI = nullptr;
5320 }
5321 }
5322};
5323
5324/// Helper data structure to emit `AllocAlignAttr`.
5325class AllocAlignAttrEmitter final
5326 : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
5327public:
5328 AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
5329 const CallArgList &CallArgs)
5330 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5331 if (!AA)
5332 return;
5333 // Alignment may or may not be a constant, and that is okay.
5334 Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
5335 .getRValue(CGF)
5336 .getScalarVal();
5337 }
5338};
5339
5340} // namespace
5341
5342static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
5343 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Ty))
5344 return VT->getPrimitiveSizeInBits().getKnownMinValue();
5345 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty))
5346 return getMaxVectorWidth(Ty: AT->getElementType());
5347
5348 unsigned MaxVectorWidth = 0;
5349 if (auto *ST = dyn_cast<llvm::StructType>(Val: Ty))
5350 for (auto *I : ST->elements())
5351 MaxVectorWidth = std::max(a: MaxVectorWidth, b: getMaxVectorWidth(Ty: I));
5352 return MaxVectorWidth;
5353}
5354
5355RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
5356 const CGCallee &Callee,
5357 ReturnValueSlot ReturnValue,
5358 const CallArgList &CallArgs,
5359 llvm::CallBase **callOrInvoke, bool IsMustTail,
5360 SourceLocation Loc,
5361 bool IsVirtualFunctionPointerThunk) {
5362 // FIXME: We no longer need the types from CallArgs; lift up and simplify.
5363
5364 assert(Callee.isOrdinary() || Callee.isVirtual());
5365
5366 // Handle struct-return functions by passing a pointer to the
5367 // location that we would like to return into.
5368 QualType RetTy = CallInfo.getReturnType();
5369 const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
5370
5371 llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(FI: CallInfo);
5372
5373 const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
5374 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
5375 // We can only guarantee that a function is called from the correct
5376 // context/function based on the appropriate target attributes,
5377 // so only check in the case where we have both always_inline and target
5378 // since otherwise we could be making a conditional call after a check for
5379 // the proper cpu features (and it won't cause code generation issues due to
5380 // function based code generation).
5381 if ((TargetDecl->hasAttr<AlwaysInlineAttr>() &&
5382 (TargetDecl->hasAttr<TargetAttr>() ||
5383 (CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>()))) ||
5384 (CurFuncDecl && CurFuncDecl->hasAttr<FlattenAttr>() &&
5385 (CurFuncDecl->hasAttr<TargetAttr>() ||
5386 TargetDecl->hasAttr<TargetAttr>())))
5387 checkTargetFeatures(Loc, TargetDecl: FD);
5388 }
5389
5390 // Some architectures (such as x86-64) have the ABI changed based on
5391 // attribute-target/features. Give them a chance to diagnose.
5392 const FunctionDecl *CallerDecl = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
5393 const FunctionDecl *CalleeDecl = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
5394 CGM.getTargetCodeGenInfo().checkFunctionCallABI(CGM, CallLoc: Loc, Caller: CallerDecl,
5395 Callee: CalleeDecl, Args: CallArgs, ReturnType: RetTy);
5396
5397 // 1. Set up the arguments.
5398
5399 // If we're using inalloca, insert the allocation after the stack save.
5400 // FIXME: Do this earlier rather than hacking it in here!
5401 RawAddress ArgMemory = RawAddress::invalid();
5402 if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
5403 const llvm::DataLayout &DL = CGM.getDataLayout();
5404 llvm::Instruction *IP = CallArgs.getStackBase();
5405 llvm::AllocaInst *AI;
5406 if (IP) {
5407 IP = IP->getNextNode();
5408 AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(), "argmem",
5409 IP->getIterator());
5410 } else {
5411 AI = CreateTempAlloca(Ty: ArgStruct, Name: "argmem");
5412 }
5413 auto Align = CallInfo.getArgStructAlignment();
5414 AI->setAlignment(Align.getAsAlign());
5415 AI->setUsedWithInAlloca(true);
5416 assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
5417 ArgMemory = RawAddress(AI, ArgStruct, Align);
5418 }
5419
5420 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
5421 SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
5422
5423 // If the call returns a temporary with struct return, create a temporary
5424 // alloca to hold the result, unless one is given to us.
5425 Address SRetPtr = Address::invalid();
5426 bool NeedSRetLifetimeEnd = false;
5427 if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
5428 // For virtual function pointer thunks and musttail calls, we must always
5429 // forward an incoming SRet pointer to the callee, because a local alloca
5430 // would be de-allocated before the call. These cases both guarantee that
5431 // there will be an incoming SRet argument of the correct type.
5432 if ((IsVirtualFunctionPointerThunk || IsMustTail) && RetAI.isIndirect()) {
5433 SRetPtr = makeNaturalAddressForPointer(Ptr: CurFn->arg_begin() +
5434 IRFunctionArgs.getSRetArgNo(),
5435 T: RetTy, Alignment: CharUnits::fromQuantity(Quantity: 1));
5436 } else if (!ReturnValue.isNull()) {
5437 SRetPtr = ReturnValue.getAddress();
5438 } else {
5439 SRetPtr = CreateMemTempWithoutCast(T: RetTy, Name: "tmp");
5440 if (HaveInsertPoint() && ReturnValue.isUnused())
5441 NeedSRetLifetimeEnd = EmitLifetimeStart(Addr: SRetPtr.getBasePointer());
5442 }
5443 if (IRFunctionArgs.hasSRetArg()) {
5444 // A mismatch between the allocated return value's AS and the target's
5445 // chosen IndirectAS can happen e.g. when passing the this pointer through
5446 // a chain involving stores to / loads from the DefaultAS; we address this
5447 // here, symmetrically with the handling we have for normal pointer args.
5448 if (SRetPtr.getAddressSpace() != RetAI.getIndirectAddrSpace()) {
5449 llvm::Value *V = SRetPtr.getBasePointer();
5450 llvm::Type *Ty = llvm::PointerType::get(C&: getLLVMContext(),
5451 AddressSpace: RetAI.getIndirectAddrSpace());
5452
5453 SRetPtr = SRetPtr.withPointer(NewPointer: performAddrSpaceCast(Src: V, DestTy: Ty),
5454 IsKnownNonNull: SRetPtr.isKnownNonNull());
5455 }
5456 IRCallArgs[IRFunctionArgs.getSRetArgNo()] =
5457 getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy);
5458 } else if (RetAI.isInAlloca()) {
5459 Address Addr =
5460 Builder.CreateStructGEP(Addr: ArgMemory, Index: RetAI.getInAllocaFieldIndex());
5461 Builder.CreateStore(Val: getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy), Addr);
5462 }
5463 }
5464
5465 RawAddress swiftErrorTemp = RawAddress::invalid();
5466 Address swiftErrorArg = Address::invalid();
5467
5468 // When passing arguments using temporary allocas, we need to add the
5469 // appropriate lifetime markers. This vector keeps track of all the lifetime
5470 // markers that need to be ended right after the call.
5471 SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
5472
5473 // Translate all of the arguments as necessary to match the IR lowering.
5474 assert(CallInfo.arg_size() == CallArgs.size() &&
5475 "Mismatch between function signature & arguments.");
5476 unsigned ArgNo = 0;
5477 CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
5478 for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
5479 I != E; ++I, ++info_it, ++ArgNo) {
5480 const ABIArgInfo &ArgInfo = info_it->info;
5481
5482 // Insert a padding argument to ensure proper alignment.
5483 if (IRFunctionArgs.hasPaddingArg(ArgNo))
5484 IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
5485 llvm::UndefValue::get(T: ArgInfo.getPaddingType());
5486
5487 unsigned FirstIRArg, NumIRArgs;
5488 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
5489
5490 bool ArgHasMaybeUndefAttr =
5491 IsArgumentMaybeUndef(TargetDecl, NumRequiredArgs: CallInfo.getNumRequiredArgs(), ArgNo);
5492
5493 switch (ArgInfo.getKind()) {
5494 case ABIArgInfo::InAlloca: {
5495 assert(NumIRArgs == 0);
5496 assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
5497 if (I->isAggregate()) {
5498 RawAddress Addr = I->hasLValue()
5499 ? I->getKnownLValue().getAddress()
5500 : I->getKnownRValue().getAggregateAddress();
5501 llvm::Instruction *Placeholder =
5502 cast<llvm::Instruction>(Val: Addr.getPointer());
5503
5504 if (!ArgInfo.getInAllocaIndirect()) {
5505 // Replace the placeholder with the appropriate argument slot GEP.
5506 CGBuilderTy::InsertPoint IP = Builder.saveIP();
5507 Builder.SetInsertPoint(Placeholder);
5508 Addr = Builder.CreateStructGEP(Addr: ArgMemory,
5509 Index: ArgInfo.getInAllocaFieldIndex());
5510 Builder.restoreIP(IP);
5511 } else {
5512 // For indirect things such as overaligned structs, replace the
5513 // placeholder with a regular aggregate temporary alloca. Store the
5514 // address of this alloca into the struct.
5515 Addr = CreateMemTemp(T: info_it->type, Name: "inalloca.indirect.tmp");
5516 Address ArgSlot = Builder.CreateStructGEP(
5517 Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5518 Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5519 }
5520 deferPlaceholderReplacement(Old: Placeholder, New: Addr.getPointer());
5521 } else if (ArgInfo.getInAllocaIndirect()) {
5522 // Make a temporary alloca and store the address of it into the argument
5523 // struct.
5524 RawAddress Addr = CreateMemTempWithoutCast(
5525 T: I->Ty, Align: getContext().getTypeAlignInChars(T: I->Ty),
5526 Name: "indirect-arg-temp");
5527 I->copyInto(CGF&: *this, Addr);
5528 Address ArgSlot =
5529 Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5530 Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5531 } else {
5532 // Store the RValue into the argument struct.
5533 Address Addr =
5534 Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5535 Addr = Addr.withElementType(ElemTy: ConvertTypeForMem(T: I->Ty));
5536 I->copyInto(CGF&: *this, Addr);
5537 }
5538 break;
5539 }
5540
5541 case ABIArgInfo::Indirect:
5542 case ABIArgInfo::IndirectAliased: {
5543 assert(NumIRArgs == 1);
5544 if (I->isAggregate()) {
5545 // We want to avoid creating an unnecessary temporary+copy here;
5546 // however, we need one in three cases:
5547 // 1. If the argument is not byval, and we are required to copy the
5548 // source. (This case doesn't occur on any common architecture.)
5549 // 2. If the argument is byval, RV is not sufficiently aligned, and
5550 // we cannot force it to be sufficiently aligned.
5551 // 3. If the argument is byval, but RV is not located in default
5552 // or alloca address space.
5553 Address Addr = I->hasLValue()
5554 ? I->getKnownLValue().getAddress()
5555 : I->getKnownRValue().getAggregateAddress();
5556 CharUnits Align = ArgInfo.getIndirectAlign();
5557 const llvm::DataLayout *TD = &CGM.getDataLayout();
5558
5559 assert((FirstIRArg >= IRFuncTy->getNumParams() ||
5560 IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
5561 TD->getAllocaAddrSpace()) &&
5562 "indirect argument must be in alloca address space");
5563
5564 bool NeedCopy = false;
5565 if (Addr.getAlignment() < Align &&
5566 llvm::getOrEnforceKnownAlignment(V: Addr.emitRawPointer(CGF&: *this),
5567 PrefAlign: Align.getAsAlign(),
5568 DL: *TD) < Align.getAsAlign()) {
5569 NeedCopy = true;
5570 } else if (I->hasLValue()) {
5571 auto LV = I->getKnownLValue();
5572
5573 bool isByValOrRef =
5574 ArgInfo.isIndirectAliased() || ArgInfo.getIndirectByVal();
5575
5576 if (!isByValOrRef ||
5577 (LV.getAlignment() < getContext().getTypeAlignInChars(T: I->Ty))) {
5578 NeedCopy = true;
5579 }
5580
5581 if (isByValOrRef && Addr.getType()->getAddressSpace() !=
5582 ArgInfo.getIndirectAddrSpace()) {
5583 NeedCopy = true;
5584 }
5585 }
5586
5587 if (!NeedCopy) {
5588 // Skip the extra memcpy call.
5589 llvm::Value *V = getAsNaturalPointerTo(Addr, PointeeType: I->Ty);
5590 auto *T = llvm::PointerType::get(C&: CGM.getLLVMContext(),
5591 AddressSpace: ArgInfo.getIndirectAddrSpace());
5592
5593 // FIXME: This should not depend on the language address spaces, and
5594 // only the contextual values. If the address space mismatches, see if
5595 // we can look through a cast to a compatible address space value,
5596 // otherwise emit a copy.
5597 llvm::Value *Val = performAddrSpaceCast(Src: V, DestTy: T);
5598 if (ArgHasMaybeUndefAttr)
5599 Val = Builder.CreateFreeze(V: Val);
5600 IRCallArgs[FirstIRArg] = Val;
5601 break;
5602 }
5603 } else if (I->getType()->isArrayParameterType()) {
5604 // Don't produce a temporary for ArrayParameterType arguments.
5605 // ArrayParameterType arguments are only created from
5606 // HLSL_ArrayRValue casts and HLSLOutArgExpr expressions, both
5607 // of which create temporaries already. This allows us to just use the
5608 // scalar for the decayed array pointer as the argument directly.
5609 IRCallArgs[FirstIRArg] = I->getKnownRValue().getScalarVal();
5610 break;
5611 }
5612
5613 // For non-aggregate args and aggregate args meeting conditions above
5614 // we need to create an aligned temporary, and copy to it.
5615 RawAddress AI = CreateMemTempWithoutCast(
5616 T: I->Ty, Align: ArgInfo.getIndirectAlign(), Name: "byval-temp");
5617 llvm::Value *Val = getAsNaturalPointerTo(Addr: AI, PointeeType: I->Ty);
5618 if (ArgHasMaybeUndefAttr)
5619 Val = Builder.CreateFreeze(V: Val);
5620 IRCallArgs[FirstIRArg] = Val;
5621
5622 // Emit lifetime markers for the temporary alloca and add cleanup code to
5623 // emit the end lifetime marker after the call.
5624 if (EmitLifetimeStart(Addr: AI.getPointer()))
5625 CallLifetimeEndAfterCall.emplace_back(Args&: AI);
5626
5627 // Generate the copy.
5628 I->copyInto(CGF&: *this, Addr: AI);
5629 break;
5630 }
5631
5632 case ABIArgInfo::Ignore:
5633 assert(NumIRArgs == 0);
5634 break;
5635
5636 case ABIArgInfo::Extend:
5637 case ABIArgInfo::Direct: {
5638 if (!isa<llvm::StructType>(Val: ArgInfo.getCoerceToType()) &&
5639 ArgInfo.getCoerceToType() == ConvertType(T: info_it->type) &&
5640 ArgInfo.getDirectOffset() == 0) {
5641 assert(NumIRArgs == 1);
5642 llvm::Value *V;
5643 if (!I->isAggregate())
5644 V = I->getKnownRValue().getScalarVal();
5645 else
5646 V = Builder.CreateLoad(
5647 Addr: I->hasLValue() ? I->getKnownLValue().getAddress()
5648 : I->getKnownRValue().getAggregateAddress());
5649
5650 // Implement swifterror by copying into a new swifterror argument.
5651 // We'll write back in the normal path out of the call.
5652 if (CallInfo.getExtParameterInfo(argIndex: ArgNo).getABI() ==
5653 ParameterABI::SwiftErrorResult) {
5654 assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
5655
5656 QualType pointeeTy = I->Ty->getPointeeType();
5657 swiftErrorArg = makeNaturalAddressForPointer(
5658 Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
5659
5660 swiftErrorTemp =
5661 CreateMemTemp(T: pointeeTy, Align: getPointerAlign(), Name: "swifterror.temp");
5662 V = swiftErrorTemp.getPointer();
5663 cast<llvm::AllocaInst>(Val: V)->setSwiftError(true);
5664
5665 llvm::Value *errorValue = Builder.CreateLoad(Addr: swiftErrorArg);
5666 Builder.CreateStore(Val: errorValue, Addr: swiftErrorTemp);
5667 }
5668
5669 // We might have to widen integers, but we should never truncate.
5670 if (ArgInfo.getCoerceToType() != V->getType() &&
5671 V->getType()->isIntegerTy())
5672 V = Builder.CreateZExt(V, DestTy: ArgInfo.getCoerceToType());
5673
5674 // The only plausible mismatch here would be for pointer address spaces.
5675 // We assume that the target has a reasonable mapping for the DefaultAS
5676 // (it can be casted to from incoming specific ASes), and insert an AS
5677 // cast to address the mismatch.
5678 if (FirstIRArg < IRFuncTy->getNumParams() &&
5679 V->getType() != IRFuncTy->getParamType(i: FirstIRArg)) {
5680 assert(V->getType()->isPointerTy() && "Only pointers can mismatch!");
5681 V = performAddrSpaceCast(Src: V, DestTy: IRFuncTy->getParamType(i: FirstIRArg));
5682 }
5683
5684 if (ArgHasMaybeUndefAttr)
5685 V = Builder.CreateFreeze(V);
5686 IRCallArgs[FirstIRArg] = V;
5687 break;
5688 }
5689
5690 llvm::StructType *STy =
5691 dyn_cast<llvm::StructType>(Val: ArgInfo.getCoerceToType());
5692
5693 // FIXME: Avoid the conversion through memory if possible.
5694 Address Src = Address::invalid();
5695 if (!I->isAggregate()) {
5696 Src = CreateMemTemp(T: I->Ty, Name: "coerce");
5697 I->copyInto(CGF&: *this, Addr: Src);
5698 } else {
5699 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
5700 : I->getKnownRValue().getAggregateAddress();
5701 }
5702
5703 // If the value is offset in memory, apply the offset now.
5704 Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
5705
5706 // Fast-isel and the optimizer generally like scalar values better than
5707 // FCAs, so we flatten them if this is safe to do for this argument.
5708 if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
5709 llvm::Type *SrcTy = Src.getElementType();
5710 llvm::TypeSize SrcTypeSize =
5711 CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
5712 llvm::TypeSize DstTypeSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
5713 if (SrcTypeSize.isScalable()) {
5714 assert(STy->containsHomogeneousScalableVectorTypes() &&
5715 "ABI only supports structure with homogeneous scalable vector "
5716 "type");
5717 assert(SrcTypeSize == DstTypeSize &&
5718 "Only allow non-fractional movement of structure with "
5719 "homogeneous scalable vector type");
5720 assert(NumIRArgs == STy->getNumElements());
5721
5722 llvm::Value *StoredStructValue =
5723 Builder.CreateLoad(Addr: Src, Name: Src.getName() + ".tuple");
5724 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5725 llvm::Value *Extract = Builder.CreateExtractValue(
5726 Agg: StoredStructValue, Idxs: i, Name: Src.getName() + ".extract" + Twine(i));
5727 IRCallArgs[FirstIRArg + i] = Extract;
5728 }
5729 } else {
5730 uint64_t SrcSize = SrcTypeSize.getFixedValue();
5731 uint64_t DstSize = DstTypeSize.getFixedValue();
5732 bool HasPFPFields = getContext().hasPFPFields(Ty: I->Ty);
5733
5734 // If the source type is smaller than the destination type of the
5735 // coerce-to logic, copy the source value into a temp alloca the size
5736 // of the destination type to allow loading all of it. The bits past
5737 // the source value are left undef.
5738 if (HasPFPFields || SrcSize < DstSize) {
5739 Address TempAlloca = CreateTempAlloca(Ty: STy, align: Src.getAlignment(),
5740 Name: Src.getName() + ".coerce");
5741 if (HasPFPFields) {
5742 // Structures with PFP fields require a coerced load to remove any
5743 // pointer signatures.
5744 Builder.CreateStore(
5745 Val: CreatePFPCoercedLoad(Src, SrcFETy: I->Ty, Ty: ArgInfo.getCoerceToType(),
5746 CGF&: *this),
5747 Addr: TempAlloca);
5748 } else
5749 Builder.CreateMemCpy(Dest: TempAlloca, Src, Size: SrcSize);
5750 Src = TempAlloca;
5751 } else {
5752 Src = Src.withElementType(ElemTy: STy);
5753 }
5754
5755 assert(NumIRArgs == STy->getNumElements());
5756 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5757 Address EltPtr = Builder.CreateStructGEP(Addr: Src, Index: i);
5758 llvm::Value *LI = Builder.CreateLoad(Addr: EltPtr);
5759 if (ArgHasMaybeUndefAttr)
5760 LI = Builder.CreateFreeze(V: LI);
5761 IRCallArgs[FirstIRArg + i] = LI;
5762 }
5763 }
5764 } else {
5765 // In the simple case, just pass the coerced loaded value.
5766 assert(NumIRArgs == 1);
5767 llvm::Value *Load =
5768 CreateCoercedLoad(Src, SrcFETy: I->Ty, Ty: ArgInfo.getCoerceToType(), CGF&: *this);
5769
5770 if (CallInfo.isCmseNSCall()) {
5771 // For certain parameter types, clear padding bits, as they may reveal
5772 // sensitive information.
5773 // Small struct/union types are passed as integer arrays.
5774 auto *ATy = dyn_cast<llvm::ArrayType>(Val: Load->getType());
5775 if (ATy != nullptr && isa<RecordType>(Val: I->Ty.getCanonicalType()))
5776 Load = EmitCMSEClearRecord(Src: Load, ATy, QTy: I->Ty);
5777 }
5778
5779 if (ArgHasMaybeUndefAttr)
5780 Load = Builder.CreateFreeze(V: Load);
5781 IRCallArgs[FirstIRArg] = Load;
5782 }
5783
5784 break;
5785 }
5786
5787 case ABIArgInfo::CoerceAndExpand: {
5788 auto coercionType = ArgInfo.getCoerceAndExpandType();
5789 auto layout = CGM.getDataLayout().getStructLayout(Ty: coercionType);
5790 auto unpaddedCoercionType = ArgInfo.getUnpaddedCoerceAndExpandType();
5791 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
5792
5793 Address addr = Address::invalid();
5794 RawAddress AllocaAddr = RawAddress::invalid();
5795 bool NeedLifetimeEnd = false;
5796 if (I->isAggregate()) {
5797 addr = I->hasLValue() ? I->getKnownLValue().getAddress()
5798 : I->getKnownRValue().getAggregateAddress();
5799
5800 } else {
5801 RValue RV = I->getKnownRValue();
5802 assert(RV.isScalar()); // complex should always just be direct
5803
5804 llvm::Type *scalarType = RV.getScalarVal()->getType();
5805 auto scalarAlign = CGM.getDataLayout().getPrefTypeAlign(Ty: scalarType);
5806
5807 // Materialize to a temporary.
5808 addr = CreateTempAlloca(Ty: RV.getScalarVal()->getType(),
5809 align: CharUnits::fromQuantity(Quantity: std::max(
5810 a: layout->getAlignment(), b: scalarAlign)),
5811 Name: "tmp",
5812 /*ArraySize=*/nullptr, Alloca: &AllocaAddr);
5813 NeedLifetimeEnd = EmitLifetimeStart(Addr: AllocaAddr.getPointer());
5814
5815 Builder.CreateStore(Val: RV.getScalarVal(), Addr: addr);
5816 }
5817
5818 addr = addr.withElementType(ElemTy: coercionType);
5819
5820 unsigned IRArgPos = FirstIRArg;
5821 unsigned unpaddedIndex = 0;
5822 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
5823 llvm::Type *eltType = coercionType->getElementType(N: i);
5824 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
5825 continue;
5826 Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
5827 llvm::Value *elt = CreateCoercedLoad(
5828 Src: eltAddr, SrcFETy: I->Ty,
5829 Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
5830 : unpaddedCoercionType,
5831 CGF&: *this);
5832 if (ArgHasMaybeUndefAttr)
5833 elt = Builder.CreateFreeze(V: elt);
5834 IRCallArgs[IRArgPos++] = elt;
5835 }
5836 assert(IRArgPos == FirstIRArg + NumIRArgs);
5837
5838 if (NeedLifetimeEnd)
5839 EmitLifetimeEnd(Addr: AllocaAddr.getPointer());
5840 break;
5841 }
5842
5843 case ABIArgInfo::Expand: {
5844 unsigned IRArgPos = FirstIRArg;
5845 ExpandTypeToArgs(Ty: I->Ty, Arg: *I, IRFuncTy, IRCallArgs, IRCallArgPos&: IRArgPos);
5846 assert(IRArgPos == FirstIRArg + NumIRArgs);
5847 break;
5848 }
5849
5850 case ABIArgInfo::TargetSpecific: {
5851 Address Src = Address::invalid();
5852 if (!I->isAggregate()) {
5853 Src = CreateMemTemp(T: I->Ty, Name: "target_coerce");
5854 I->copyInto(CGF&: *this, Addr: Src);
5855 } else {
5856 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
5857 : I->getKnownRValue().getAggregateAddress();
5858 }
5859
5860 // If the value is offset in memory, apply the offset now.
5861 Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
5862 llvm::Value *Load =
5863 CGM.getABIInfo().createCoercedLoad(SrcAddr: Src, AI: ArgInfo, CGF&: *this);
5864 IRCallArgs[FirstIRArg] = Load;
5865 break;
5866 }
5867 }
5868 }
5869
5870 const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(CGF&: *this);
5871 llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
5872
5873 // If we're using inalloca, set up that argument.
5874 if (ArgMemory.isValid()) {
5875 llvm::Value *Arg = ArgMemory.getPointer();
5876 assert(IRFunctionArgs.hasInallocaArg());
5877 IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
5878 }
5879
5880 // 2. Prepare the function pointer.
5881
5882 // If the callee is a bitcast of a non-variadic function to have a
5883 // variadic function pointer type, check to see if we can remove the
5884 // bitcast. This comes up with unprototyped functions.
5885 //
5886 // This makes the IR nicer, but more importantly it ensures that we
5887 // can inline the function at -O0 if it is marked always_inline.
5888 auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
5889 llvm::Value *Ptr) -> llvm::Function * {
5890 if (!CalleeFT->isVarArg())
5891 return nullptr;
5892
5893 // Get underlying value if it's a bitcast
5894 if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Val: Ptr)) {
5895 if (CE->getOpcode() == llvm::Instruction::BitCast)
5896 Ptr = CE->getOperand(i_nocapture: 0);
5897 }
5898
5899 llvm::Function *OrigFn = dyn_cast<llvm::Function>(Val: Ptr);
5900 if (!OrigFn)
5901 return nullptr;
5902
5903 llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
5904
5905 // If the original type is variadic, or if any of the component types
5906 // disagree, we cannot remove the cast.
5907 if (OrigFT->isVarArg() ||
5908 OrigFT->getNumParams() != CalleeFT->getNumParams() ||
5909 OrigFT->getReturnType() != CalleeFT->getReturnType())
5910 return nullptr;
5911
5912 for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i)
5913 if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
5914 return nullptr;
5915
5916 return OrigFn;
5917 };
5918
5919 if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {
5920 CalleePtr = OrigFn;
5921 IRFuncTy = OrigFn->getFunctionType();
5922 }
5923
5924 // 3. Perform the actual call.
5925
5926 // Deactivate any cleanups that we're supposed to do immediately before
5927 // the call.
5928 if (!CallArgs.getCleanupsToDeactivate().empty())
5929 deactivateArgCleanupsBeforeCall(CGF&: *this, CallArgs);
5930
5931 // Update the largest vector width if any arguments have vector types.
5932 for (unsigned i = 0; i < IRCallArgs.size(); ++i)
5933 LargestVectorWidth = std::max(a: LargestVectorWidth,
5934 b: getMaxVectorWidth(Ty: IRCallArgs[i]->getType()));
5935
5936 // Compute the calling convention and attributes.
5937 unsigned CallingConv;
5938 llvm::AttributeList Attrs;
5939 CGM.ConstructAttributeList(Name: CalleePtr->getName(), FI: CallInfo,
5940 CalleeInfo: Callee.getAbstractInfo(), AttrList&: Attrs, CallingConv,
5941 /*AttrOnCallSite=*/true,
5942 /*IsThunk=*/false);
5943
5944 if (CallingConv == llvm::CallingConv::X86_VectorCall &&
5945 getTarget().getTriple().isWindowsArm64EC()) {
5946 CGM.Error(loc: Loc, error: "__vectorcall calling convention is not currently "
5947 "supported");
5948 }
5949
5950 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
5951 if (FD->hasAttr<StrictFPAttr>())
5952 // All calls within a strictfp function are marked strictfp
5953 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::StrictFP);
5954
5955 // If -ffast-math is enabled and the function is guarded by an
5956 // '__attribute__((optnone)) adjust the memory attribute so the BE emits the
5957 // library call instead of the intrinsic.
5958 if (FD->hasAttr<OptimizeNoneAttr>() && getLangOpts().FastMath)
5959 CGM.AdjustMemoryAttribute(Name: CalleePtr->getName(), CalleeInfo: Callee.getAbstractInfo(),
5960 Attrs);
5961 }
5962 // Add call-site nomerge attribute if exists.
5963 if (InNoMergeAttributedStmt)
5964 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoMerge);
5965
5966 // Add call-site noinline attribute if exists.
5967 if (InNoInlineAttributedStmt)
5968 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoInline);
5969
5970 // Add call-site always_inline attribute if exists.
5971 // Note: This corresponds to the [[clang::always_inline]] statement attribute.
5972 if (InAlwaysInlineAttributedStmt &&
5973 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
5974 Caller: CallerDecl, Callee: CalleeDecl))
5975 Attrs =
5976 Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::AlwaysInline);
5977
5978 // Remove call-site convergent attribute if requested.
5979 if (InNoConvergentAttributedStmt)
5980 Attrs =
5981 Attrs.removeFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::Convergent);
5982
5983 // Apply some call-site-specific attributes.
5984 // TODO: work this into building the attribute set.
5985
5986 // Apply always_inline to all calls within flatten functions.
5987 // FIXME: should this really take priority over __try, below?
5988 if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
5989 !InNoInlineAttributedStmt &&
5990 !(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>()) &&
5991 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
5992 Caller: CallerDecl, Callee: CalleeDecl)) {
5993 Attrs =
5994 Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::AlwaysInline);
5995 }
5996
5997 // Disable inlining inside SEH __try blocks.
5998 if (isSEHTryScope()) {
5999 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoInline);
6000 }
6001
6002 // Decide whether to use a call or an invoke.
6003 bool CannotThrow;
6004 if (currentFunctionUsesSEHTry()) {
6005 // SEH cares about asynchronous exceptions, so everything can "throw."
6006 CannotThrow = false;
6007 } else if (isCleanupPadScope() &&
6008 EHPersonality::get(CGF&: *this).isMSVCXXPersonality()) {
6009 // The MSVC++ personality will implicitly terminate the program if an
6010 // exception is thrown during a cleanup outside of a try/catch.
6011 // We don't need to model anything in IR to get this behavior.
6012 CannotThrow = true;
6013 } else {
6014 // Otherwise, nounwind call sites will never throw.
6015 CannotThrow = Attrs.hasFnAttr(Kind: llvm::Attribute::NoUnwind);
6016
6017 if (auto *FPtr = dyn_cast<llvm::Function>(Val: CalleePtr))
6018 if (FPtr->hasFnAttribute(Kind: llvm::Attribute::NoUnwind))
6019 CannotThrow = true;
6020 }
6021
6022 // If we made a temporary, be sure to clean up after ourselves. Note that we
6023 // can't depend on being inside of an ExprWithCleanups, so we need to manually
6024 // pop this cleanup later on. Being eager about this is OK, since this
6025 // temporary is 'invisible' outside of the callee.
6026 if (NeedSRetLifetimeEnd)
6027 pushFullExprCleanup<CallLifetimeEnd>(kind: NormalEHLifetimeMarker, A: SRetPtr);
6028
6029 llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();
6030
6031 SmallVector<llvm::OperandBundleDef, 1> BundleList =
6032 getBundlesForFunclet(Callee: CalleePtr);
6033
6034 if (SanOpts.has(K: SanitizerKind::KCFI) &&
6035 !isa_and_nonnull<FunctionDecl>(Val: TargetDecl))
6036 EmitKCFIOperandBundle(Callee: ConcreteCallee, Bundles&: BundleList);
6037
6038 // Add the pointer-authentication bundle.
6039 EmitPointerAuthOperandBundle(Info: ConcreteCallee.getPointerAuthInfo(), Bundles&: BundleList);
6040
6041 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl))
6042 if (FD->hasAttr<StrictFPAttr>())
6043 // All calls within a strictfp function are marked strictfp
6044 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::StrictFP);
6045
6046 AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
6047 Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
6048
6049 AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
6050 Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
6051
6052 // Emit the actual call/invoke instruction.
6053 llvm::CallBase *CI;
6054 if (!InvokeDest) {
6055 CI = Builder.CreateCall(FTy: IRFuncTy, Callee: CalleePtr, Args: IRCallArgs, OpBundles: BundleList);
6056 } else {
6057 llvm::BasicBlock *Cont = createBasicBlock(name: "invoke.cont");
6058 CI = Builder.CreateInvoke(Ty: IRFuncTy, Callee: CalleePtr, NormalDest: Cont, UnwindDest: InvokeDest, Args: IRCallArgs,
6059 OpBundles: BundleList);
6060 EmitBlock(BB: Cont);
6061 }
6062 if (CI->getCalledFunction() && CI->getCalledFunction()->hasName() &&
6063 CI->getCalledFunction()->getName().starts_with(Prefix: "_Z4sqrt")) {
6064 SetSqrtFPAccuracy(CI);
6065 }
6066 if (callOrInvoke) {
6067 *callOrInvoke = CI;
6068 if (CGM.getCodeGenOpts().CallGraphSection) {
6069 QualType CST;
6070 if (TargetDecl && TargetDecl->getFunctionType())
6071 CST = QualType(TargetDecl->getFunctionType(), 0);
6072 else if (const auto *FPT =
6073 Callee.getAbstractInfo().getCalleeFunctionProtoType())
6074 CST = QualType(FPT, 0);
6075 else
6076 llvm_unreachable(
6077 "Cannot find the callee type to generate callee_type metadata.");
6078
6079 // Set type identifier metadata of indirect calls for call graph section.
6080 if (!CST.isNull())
6081 CGM.createCalleeTypeMetadataForIcall(QT: CST, CB: *callOrInvoke);
6082 }
6083 }
6084
6085 // If this is within a function that has the guard(nocf) attribute and is an
6086 // indirect call, add the "guard_nocf" attribute to this call to indicate that
6087 // Control Flow Guard checks should not be added, even if the call is inlined.
6088 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
6089 if (const auto *A = FD->getAttr<CFGuardAttr>()) {
6090 if (A->getGuard() == CFGuardAttr::GuardArg::nocf &&
6091 !CI->getCalledFunction())
6092 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: "guard_nocf");
6093 }
6094 }
6095
6096 // Apply the attributes and calling convention.
6097 CI->setAttributes(Attrs);
6098 CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
6099
6100 // Apply various metadata.
6101
6102 if (!CI->getType()->isVoidTy())
6103 CI->setName("call");
6104
6105 if (CGM.shouldEmitConvergenceTokens() && CI->isConvergent())
6106 CI = addConvergenceControlToken(Input: CI);
6107
6108 // Update largest vector width from the return type.
6109 LargestVectorWidth =
6110 std::max(a: LargestVectorWidth, b: getMaxVectorWidth(Ty: CI->getType()));
6111
6112 // Insert instrumentation or attach profile metadata at indirect call sites.
6113 // For more details, see the comment before the definition of
6114 // IPVK_IndirectCallTarget in InstrProfData.inc.
6115 if (!CI->getCalledFunction())
6116 PGO->valueProfile(Builder, ValueKind: llvm::IPVK_IndirectCallTarget, ValueSite: CI, ValuePtr: CalleePtr);
6117
6118 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
6119 // optimizer it can aggressively ignore unwind edges.
6120 if (CGM.getLangOpts().ObjCAutoRefCount)
6121 AddObjCARCExceptionMetadata(Inst: CI);
6122
6123 // Set tail call kind if necessary.
6124 bool IsPPC = getTarget().getTriple().isPPC();
6125 bool IsMIPS = getTarget().getTriple().isMIPS();
6126 bool HasMips16 = false;
6127 if (IsMIPS) {
6128 const TargetOptions &TargetOpts = getTarget().getTargetOpts();
6129 HasMips16 = TargetOpts.FeatureMap.lookup(Key: "mips16");
6130 if (!HasMips16)
6131 HasMips16 = llvm::is_contained(Range: TargetOpts.Features, Element: "+mips16");
6132 }
6133 if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(Val: CI)) {
6134 if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
6135 Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
6136 else if (IsMustTail) {
6137 if (IsPPC) {
6138 if (getTarget().getTriple().isOSAIX())
6139 CGM.getDiags().Report(Loc, DiagID: diag::err_aix_musttail_unsupported);
6140 else if (!getTarget().hasFeature(Feature: "pcrelative-memops")) {
6141 if (getTarget().hasFeature(Feature: "longcall"))
6142 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail) << 0;
6143 else if (Call->isIndirectCall())
6144 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail) << 1;
6145 else if (isa_and_nonnull<FunctionDecl>(Val: TargetDecl)) {
6146 if (!cast<FunctionDecl>(Val: TargetDecl)->isDefined())
6147 // The undefined callee may be a forward declaration. Without
6148 // knowning all symbols in the module, we won't know the symbol is
6149 // defined or not. Collect all these symbols for later diagnosing.
6150 CGM.addUndefinedGlobalForTailCall(
6151 Global: {cast<FunctionDecl>(Val: TargetDecl), Loc});
6152 else {
6153 llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(
6154 GD: GlobalDecl(cast<FunctionDecl>(Val: TargetDecl)));
6155 if (llvm::GlobalValue::isWeakForLinker(Linkage) ||
6156 llvm::GlobalValue::isDiscardableIfUnused(Linkage))
6157 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail)
6158 << 2;
6159 }
6160 }
6161 }
6162 }
6163 if (IsMIPS) {
6164 if (HasMips16)
6165 CGM.getDiags().Report(Loc, DiagID: diag::err_mips_impossible_musttail) << 0;
6166 else if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl))
6167 CGM.addUndefinedGlobalForTailCall(Global: {FD, Loc});
6168 }
6169 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
6170 }
6171 }
6172
6173 // Add metadata for calls to MSAllocator functions
6174 if (getDebugInfo() && TargetDecl && TargetDecl->hasAttr<MSAllocatorAttr>())
6175 getDebugInfo()->addHeapAllocSiteMetadata(CallSite: CI, AllocatedTy: RetTy->getPointeeType(), Loc);
6176
6177 // Add srcloc metadata for [[gnu::error/warning]] diagnostics. When
6178 // ShowInliningChain is enabled, also track inline/static calls for the
6179 // heuristic fallback when debug info is not available. This heuristic is
6180 // conservative and best-effort since static or inline-annotated functions
6181 // are still not guaranteed to be inlined.
6182 if (TargetDecl) {
6183 bool NeedSrcLoc = TargetDecl->hasAttr<ErrorAttr>();
6184 if (!NeedSrcLoc && CGM.getCodeGenOpts().ShowInliningChain) {
6185 if (const auto *FD = dyn_cast<FunctionDecl>(Val: TargetDecl))
6186 NeedSrcLoc = FD->isInlined() || FD->hasAttr<AlwaysInlineAttr>() ||
6187 FD->getStorageClass() == SC_Static ||
6188 FD->isInAnonymousNamespace();
6189 }
6190 if (NeedSrcLoc) {
6191 auto *Line = llvm::ConstantInt::get(Ty: Int64Ty, V: Loc.getRawEncoding());
6192 auto *MD = llvm::ConstantAsMetadata::get(C: Line);
6193 CI->setMetadata(Kind: "srcloc", Node: llvm::MDNode::get(Context&: getLLVMContext(), MDs: {MD}));
6194 }
6195 }
6196
6197 // 4. Finish the call.
6198
6199 // If the call doesn't return, finish the basic block and clear the
6200 // insertion point; this allows the rest of IRGen to discard
6201 // unreachable code.
6202 if (CI->doesNotReturn()) {
6203 if (NeedSRetLifetimeEnd)
6204 PopCleanupBlock();
6205
6206 // Strip away the noreturn attribute to better diagnose unreachable UB.
6207 if (SanOpts.has(K: SanitizerKind::Unreachable)) {
6208 // Also remove from function since CallBase::hasFnAttr additionally checks
6209 // attributes of the called function.
6210 if (auto *F = CI->getCalledFunction())
6211 F->removeFnAttr(Kind: llvm::Attribute::NoReturn);
6212 CI->removeFnAttr(Kind: llvm::Attribute::NoReturn);
6213
6214 // Avoid incompatibility with ASan which relies on the `noreturn`
6215 // attribute to insert handler calls.
6216 if (SanOpts.hasOneOf(K: SanitizerKind::Address |
6217 SanitizerKind::KernelAddress)) {
6218 SanitizerScope SanScope(this);
6219 llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);
6220 Builder.SetInsertPoint(CI);
6221 auto *FnType = llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
6222 llvm::FunctionCallee Fn =
6223 CGM.CreateRuntimeFunction(Ty: FnType, Name: "__asan_handle_no_return");
6224 EmitNounwindRuntimeCall(callee: Fn);
6225 }
6226 }
6227
6228 EmitUnreachable(Loc);
6229 Builder.ClearInsertionPoint();
6230
6231 // FIXME: For now, emit a dummy basic block because expr emitters in
6232 // generally are not ready to handle emitting expressions at unreachable
6233 // points.
6234 EnsureInsertPoint();
6235
6236 // Return a reasonable RValue.
6237 return GetUndefRValue(Ty: RetTy);
6238 }
6239
6240 // If this is a musttail call, return immediately. We do not branch to the
6241 // epilogue in this case.
6242 if (IsMustTail) {
6243 for (auto it = EHStack.find(sp: CurrentCleanupScopeDepth); it != EHStack.end();
6244 ++it) {
6245 EHCleanupScope *Cleanup = dyn_cast<EHCleanupScope>(Val: &*it);
6246 // Fake uses can be safely emitted immediately prior to the tail call, so
6247 // we choose to emit them just before the call here.
6248 if (Cleanup && Cleanup->isFakeUse()) {
6249 CGBuilderTy::InsertPointGuard IPG(Builder);
6250 Builder.SetInsertPoint(CI);
6251 Cleanup->getCleanup()->Emit(CGF&: *this, flags: EHScopeStack::Cleanup::Flags());
6252 } else if (!(Cleanup &&
6253 Cleanup->getCleanup()->isRedundantBeforeReturn())) {
6254 CGM.ErrorUnsupported(S: MustTailCall, Type: "tail call skipping over cleanups");
6255 }
6256 }
6257 if (CI->getType()->isVoidTy())
6258 Builder.CreateRetVoid();
6259 else
6260 Builder.CreateRet(V: CI);
6261 Builder.ClearInsertionPoint();
6262 EnsureInsertPoint();
6263 return GetUndefRValue(Ty: RetTy);
6264 }
6265
6266 // Perform the swifterror writeback.
6267 if (swiftErrorTemp.isValid()) {
6268 llvm::Value *errorResult = Builder.CreateLoad(Addr: swiftErrorTemp);
6269 Builder.CreateStore(Val: errorResult, Addr: swiftErrorArg);
6270 }
6271
6272 // Emit any call-associated writebacks immediately. Arguably this
6273 // should happen after any return-value munging.
6274 if (CallArgs.hasWritebacks())
6275 EmitWritebacks(args: CallArgs);
6276
6277 // The stack cleanup for inalloca arguments has to run out of the normal
6278 // lexical order, so deactivate it and run it manually here.
6279 CallArgs.freeArgumentMemory(CGF&: *this);
6280
6281 // Extract the return value.
6282 RValue Ret;
6283
6284 // If the current function is a virtual function pointer thunk, avoid copying
6285 // the return value of the musttail call to a temporary.
6286 if (IsVirtualFunctionPointerThunk) {
6287 Ret = RValue::get(V: CI);
6288 } else {
6289 Ret = [&] {
6290 switch (RetAI.getKind()) {
6291 case ABIArgInfo::CoerceAndExpand: {
6292 auto coercionType = RetAI.getCoerceAndExpandType();
6293
6294 Address addr = SRetPtr.withElementType(ElemTy: coercionType);
6295
6296 assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());
6297 bool requiresExtract = isa<llvm::StructType>(Val: CI->getType());
6298
6299 unsigned unpaddedIndex = 0;
6300 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
6301 llvm::Type *eltType = coercionType->getElementType(N: i);
6302 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
6303 continue;
6304 Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
6305 llvm::Value *elt = CI;
6306 if (requiresExtract)
6307 elt = Builder.CreateExtractValue(Agg: elt, Idxs: unpaddedIndex++);
6308 else
6309 assert(unpaddedIndex == 0);
6310 Builder.CreateStore(Val: elt, Addr: eltAddr);
6311 }
6312 [[fallthrough]];
6313 }
6314
6315 case ABIArgInfo::InAlloca:
6316 case ABIArgInfo::Indirect: {
6317 RValue ret = convertTempToRValue(addr: SRetPtr, type: RetTy, Loc: SourceLocation());
6318 if (NeedSRetLifetimeEnd)
6319 PopCleanupBlock();
6320 return ret;
6321 }
6322
6323 case ABIArgInfo::Ignore:
6324 // If we are ignoring an argument that had a result, make sure to
6325 // construct the appropriate return value for our caller.
6326 return GetUndefRValue(Ty: RetTy);
6327
6328 case ABIArgInfo::Extend:
6329 case ABIArgInfo::Direct: {
6330 llvm::Type *RetIRTy = ConvertType(T: RetTy);
6331 if (RetAI.getCoerceToType() == RetIRTy &&
6332 RetAI.getDirectOffset() == 0) {
6333 switch (getEvaluationKind(T: RetTy)) {
6334 case TEK_Complex: {
6335 llvm::Value *Real = Builder.CreateExtractValue(Agg: CI, Idxs: 0);
6336 llvm::Value *Imag = Builder.CreateExtractValue(Agg: CI, Idxs: 1);
6337 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
6338 }
6339 case TEK_Aggregate:
6340 break;
6341 case TEK_Scalar: {
6342 // If the argument doesn't match, perform a bitcast to coerce it.
6343 // This can happen due to trivial type mismatches.
6344 llvm::Value *V = CI;
6345 if (V->getType() != RetIRTy)
6346 V = Builder.CreateBitCast(V, DestTy: RetIRTy);
6347 return RValue::get(V);
6348 }
6349 }
6350 }
6351
6352 // If coercing a fixed vector from a scalable vector for ABI
6353 // compatibility, and the types match, use the llvm.vector.extract
6354 // intrinsic to perform the conversion.
6355 if (auto *FixedDstTy = dyn_cast<llvm::FixedVectorType>(Val: RetIRTy)) {
6356 llvm::Value *V = CI;
6357 if (auto *ScalableSrcTy =
6358 dyn_cast<llvm::ScalableVectorType>(Val: V->getType())) {
6359 if (FixedDstTy->getElementType() ==
6360 ScalableSrcTy->getElementType()) {
6361 V = Builder.CreateExtractVector(DstType: FixedDstTy, SrcVec: V, Idx: uint64_t(0),
6362 Name: "cast.fixed");
6363 return RValue::get(V);
6364 }
6365 }
6366 }
6367
6368 Address DestPtr = ReturnValue.getValue();
6369 bool DestIsVolatile = ReturnValue.isVolatile();
6370 uint64_t DestSize =
6371 getContext().getTypeInfoDataSizeInChars(T: RetTy).Width.getQuantity();
6372
6373 if (!DestPtr.isValid()) {
6374 DestPtr = CreateMemTemp(T: RetTy, Name: "coerce");
6375 DestIsVolatile = false;
6376 DestSize = getContext().getTypeSizeInChars(T: RetTy).getQuantity();
6377 }
6378
6379 // An empty record can overlap other data (if declared with
6380 // no_unique_address); omit the store for such types - as there is no
6381 // actual data to store.
6382 if (!isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true)) {
6383 // If the value is offset in memory, apply the offset now.
6384 Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
6385 CreateCoercedStore(
6386 Src: CI, SrcFETy: RetTy, Dst: StorePtr,
6387 DstSize: llvm::TypeSize::getFixed(ExactSize: DestSize - RetAI.getDirectOffset()),
6388 DstIsVolatile: DestIsVolatile);
6389 }
6390
6391 return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation());
6392 }
6393
6394 case ABIArgInfo::TargetSpecific: {
6395 Address DestPtr = ReturnValue.getValue();
6396 Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
6397 bool DestIsVolatile = ReturnValue.isVolatile();
6398 if (!DestPtr.isValid()) {
6399 DestPtr = CreateMemTemp(T: RetTy, Name: "target_coerce");
6400 DestIsVolatile = false;
6401 }
6402 CGM.getABIInfo().createCoercedStore(Val: CI, DstAddr: StorePtr, AI: RetAI, DestIsVolatile,
6403 CGF&: *this);
6404 return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation());
6405 }
6406
6407 case ABIArgInfo::Expand:
6408 case ABIArgInfo::IndirectAliased:
6409 llvm_unreachable("Invalid ABI kind for return argument");
6410 }
6411
6412 llvm_unreachable("Unhandled ABIArgInfo::Kind");
6413 }();
6414 }
6415
6416 // Emit the assume_aligned check on the return value.
6417 if (Ret.isScalar() && TargetDecl) {
6418 AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6419 AllocAlignAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6420 }
6421
6422 // Explicitly call CallLifetimeEnd::Emit just to re-use the code even though
6423 // we can't use the full cleanup mechanism.
6424 for (CallLifetimeEnd &LifetimeEnd : CallLifetimeEndAfterCall)
6425 LifetimeEnd.Emit(CGF&: *this, /*Flags=*/flags: {});
6426
6427 if (!ReturnValue.isExternallyDestructed() &&
6428 RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct)
6429 pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Ret.getAggregateAddress(),
6430 type: RetTy);
6431
6432 // Generate function declaration DISuprogram in order to be used
6433 // in debug info about call sites.
6434 if (CGDebugInfo *DI = getDebugInfo()) {
6435 // Ensure call site info would actually be emitted before collecting
6436 // further callee info.
6437 if (CalleeDecl && !CalleeDecl->hasAttr<NoDebugAttr>() &&
6438 DI->getCallSiteRelatedAttrs() != llvm::DINode::FlagZero) {
6439 CodeGenFunction CalleeCGF(CGM);
6440 const GlobalDecl &CalleeGlobalDecl =
6441 Callee.getAbstractInfo().getCalleeDecl();
6442 CalleeCGF.CurGD = CalleeGlobalDecl;
6443 FunctionArgList Args;
6444 QualType ResTy = CalleeCGF.BuildFunctionArgList(GD: CalleeGlobalDecl, Args);
6445 DI->EmitFuncDeclForCallSite(
6446 CallOrInvoke: CI, CalleeType: DI->getFunctionType(FD: CalleeDecl, RetTy: ResTy, Args), CalleeGlobalDecl);
6447 }
6448 // Generate call site target information.
6449 DI->addCallTargetIfVirtual(FD: CalleeDecl, CI);
6450 }
6451
6452 return Ret;
6453}
6454
6455CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const {
6456 if (isVirtual()) {
6457 const CallExpr *CE = getVirtualCallExpr();
6458 return CGF.CGM.getCXXABI().getVirtualFunctionPointer(
6459 CGF, GD: getVirtualMethodDecl(), This: getThisAddress(), Ty: getVirtualFunctionType(),
6460 Loc: CE ? CE->getBeginLoc() : SourceLocation());
6461 }
6462
6463 return *this;
6464}
6465
6466/* VarArg handling */
6467
6468RValue CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
6469 AggValueSlot Slot) {
6470 VAListAddr = VE->isMicrosoftABI() ? EmitMSVAListRef(E: VE->getSubExpr())
6471 : EmitVAListRef(E: VE->getSubExpr());
6472 QualType Ty = VE->getType();
6473 if (Ty->isVariablyModifiedType())
6474 EmitVariablyModifiedType(Ty);
6475 if (VE->isMicrosoftABI())
6476 return CGM.getABIInfo().EmitMSVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6477 return CGM.getABIInfo().EmitVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6478}
6479
6480DisableDebugLocationUpdates::DisableDebugLocationUpdates(CodeGenFunction &CGF)
6481 : CGF(CGF) {
6482 CGF.disableDebugInfo();
6483}
6484
6485DisableDebugLocationUpdates::~DisableDebugLocationUpdates() {
6486 CGF.enableDebugInfo();
6487}
6488