1//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// These classes wrap the information about a call or function
10// definition used to handle ABI compliancy.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CGCall.h"
15#include "ABIInfo.h"
16#include "ABIInfoImpl.h"
17#include "CGBlocks.h"
18#include "CGCXXABI.h"
19#include "CGCleanup.h"
20#include "CGDebugInfo.h"
21#include "CGRecordLayout.h"
22#include "CodeGenFunction.h"
23#include "CodeGenModule.h"
24#include "CodeGenPGO.h"
25#include "TargetInfo.h"
26#include "clang/AST/Attr.h"
27#include "clang/AST/Decl.h"
28#include "clang/AST/DeclCXX.h"
29#include "clang/AST/DeclObjC.h"
30#include "clang/AST/RecordLayout.h"
31#include "clang/Basic/CodeGenOptions.h"
32#include "clang/Basic/TargetInfo.h"
33#include "clang/CodeGen/CGFunctionInfo.h"
34#include "clang/CodeGen/SwiftCallingConv.h"
35#include "llvm/ADT/STLExtras.h"
36#include "llvm/ADT/StringExtras.h"
37#include "llvm/Analysis/ValueTracking.h"
38#include "llvm/IR/Assumptions.h"
39#include "llvm/IR/AttributeMask.h"
40#include "llvm/IR/Attributes.h"
41#include "llvm/IR/CallingConv.h"
42#include "llvm/IR/DataLayout.h"
43#include "llvm/IR/DebugInfoMetadata.h"
44#include "llvm/IR/InlineAsm.h"
45#include "llvm/IR/IntrinsicInst.h"
46#include "llvm/IR/Intrinsics.h"
47#include "llvm/IR/Type.h"
48#include "llvm/Transforms/Utils/Local.h"
49#include <optional>
50using namespace clang;
51using namespace CodeGen;
52
53/***/
54
55unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
56 switch (CC) {
57 default:
58 return llvm::CallingConv::C;
59 case CC_X86StdCall:
60 return llvm::CallingConv::X86_StdCall;
61 case CC_X86FastCall:
62 return llvm::CallingConv::X86_FastCall;
63 case CC_X86RegCall:
64 return llvm::CallingConv::X86_RegCall;
65 case CC_X86ThisCall:
66 return llvm::CallingConv::X86_ThisCall;
67 case CC_Win64:
68 return llvm::CallingConv::Win64;
69 case CC_X86_64SysV:
70 return llvm::CallingConv::X86_64_SysV;
71 case CC_AAPCS:
72 return llvm::CallingConv::ARM_AAPCS;
73 case CC_AAPCS_VFP:
74 return llvm::CallingConv::ARM_AAPCS_VFP;
75 case CC_IntelOclBicc:
76 return llvm::CallingConv::Intel_OCL_BI;
77 // TODO: Add support for __pascal to LLVM.
78 case CC_X86Pascal:
79 return llvm::CallingConv::C;
80 // TODO: Add support for __vectorcall to LLVM.
81 case CC_X86VectorCall:
82 return llvm::CallingConv::X86_VectorCall;
83 case CC_AArch64VectorCall:
84 return llvm::CallingConv::AArch64_VectorCall;
85 case CC_AArch64SVEPCS:
86 return llvm::CallingConv::AArch64_SVE_VectorCall;
87 case CC_SpirFunction:
88 return llvm::CallingConv::SPIR_FUNC;
89 case CC_DeviceKernel:
90 return CGM.getTargetCodeGenInfo().getDeviceKernelCallingConv();
91 case CC_PreserveMost:
92 return llvm::CallingConv::PreserveMost;
93 case CC_PreserveAll:
94 return llvm::CallingConv::PreserveAll;
95 case CC_Swift:
96 return llvm::CallingConv::Swift;
97 case CC_SwiftAsync:
98 return llvm::CallingConv::SwiftTail;
99 case CC_M68kRTD:
100 return llvm::CallingConv::M68k_RTD;
101 case CC_PreserveNone:
102 return llvm::CallingConv::PreserveNone;
103 // clang-format off
104 case CC_RISCVVectorCall: return llvm::CallingConv::RISCV_VectorCall;
105 // clang-format on
106#define CC_VLS_CASE(ABI_VLEN) \
107 case CC_RISCVVLSCall_##ABI_VLEN: \
108 return llvm::CallingConv::RISCV_VLSCall_##ABI_VLEN;
109 CC_VLS_CASE(32)
110 CC_VLS_CASE(64)
111 CC_VLS_CASE(128)
112 CC_VLS_CASE(256)
113 CC_VLS_CASE(512)
114 CC_VLS_CASE(1024)
115 CC_VLS_CASE(2048)
116 CC_VLS_CASE(4096)
117 CC_VLS_CASE(8192)
118 CC_VLS_CASE(16384)
119 CC_VLS_CASE(32768)
120 CC_VLS_CASE(65536)
121#undef CC_VLS_CASE
122 }
123}
124
125/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
126/// qualification. Either or both of RD and MD may be null. A null RD indicates
127/// that there is no meaningful 'this' type, and a null MD can occur when
128/// calling a method pointer.
129CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
130 const CXXMethodDecl *MD) {
131 CanQualType RecTy;
132 if (RD)
133 RecTy = Context.getCanonicalTagType(TD: RD);
134 else
135 RecTy = Context.VoidTy;
136
137 if (MD)
138 RecTy = CanQualType::CreateUnsafe(Other: Context.getAddrSpaceQualType(
139 T: RecTy, AddressSpace: MD->getMethodQualifiers().getAddressSpace()));
140 return Context.getPointerType(T: RecTy);
141}
142
143/// Returns the canonical formal type of the given C++ method.
144static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
145 return MD->getType()
146 ->getCanonicalTypeUnqualified()
147 .getAs<FunctionProtoType>();
148}
149
150/// Returns the "extra-canonicalized" return type, which discards
151/// qualifiers on the return type. Codegen doesn't care about them,
152/// and it makes ABI code a little easier to be able to assume that
153/// all parameter and return types are top-level unqualified.
154static CanQualType GetReturnType(QualType RetTy) {
155 return RetTy->getCanonicalTypeUnqualified();
156}
157
158/// Arrange the argument and result information for a value of the given
159/// unprototyped freestanding function type.
160const CGFunctionInfo &
161CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
162 // When translating an unprototyped function type, always use a
163 // variadic type.
164 return arrangeLLVMFunctionInfo(returnType: FTNP->getReturnType().getUnqualifiedType(),
165 opts: FnInfoOpts::None, argTypes: {}, info: FTNP->getExtInfo(), paramInfos: {},
166 args: RequiredArgs(0));
167}
168
169static void addExtParameterInfosForCall(
170 llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
171 const FunctionProtoType *proto, unsigned prefixArgs, unsigned totalArgs) {
172 assert(proto->hasExtParameterInfos());
173 assert(paramInfos.size() <= prefixArgs);
174 assert(proto->getNumParams() + prefixArgs <= totalArgs);
175
176 paramInfos.reserve(N: totalArgs);
177
178 // Add default infos for any prefix args that don't already have infos.
179 paramInfos.resize(N: prefixArgs);
180
181 // Add infos for the prototype.
182 for (const auto &ParamInfo : proto->getExtParameterInfos()) {
183 paramInfos.push_back(Elt: ParamInfo);
184 // pass_object_size params have no parameter info.
185 if (ParamInfo.hasPassObjectSize())
186 paramInfos.emplace_back();
187 }
188
189 assert(paramInfos.size() <= totalArgs &&
190 "Did we forget to insert pass_object_size args?");
191 // Add default infos for the variadic and/or suffix arguments.
192 paramInfos.resize(N: totalArgs);
193}
194
195/// Adds the formal parameters in FPT to the given prefix. If any parameter in
196/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
197static void appendParameterTypes(
198 const CodeGenTypes &CGT, SmallVectorImpl<CanQualType> &prefix,
199 SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
200 CanQual<FunctionProtoType> FPT) {
201 // Fast path: don't touch param info if we don't need to.
202 if (!FPT->hasExtParameterInfos()) {
203 assert(paramInfos.empty() &&
204 "We have paramInfos, but the prototype doesn't?");
205 prefix.append(in_start: FPT->param_type_begin(), in_end: FPT->param_type_end());
206 return;
207 }
208
209 unsigned PrefixSize = prefix.size();
210 // In the vast majority of cases, we'll have precisely FPT->getNumParams()
211 // parameters; the only thing that can change this is the presence of
212 // pass_object_size. So, we preallocate for the common case.
213 prefix.reserve(N: prefix.size() + FPT->getNumParams());
214
215 auto ExtInfos = FPT->getExtParameterInfos();
216 assert(ExtInfos.size() == FPT->getNumParams());
217 for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
218 prefix.push_back(Elt: FPT->getParamType(i: I));
219 if (ExtInfos[I].hasPassObjectSize())
220 prefix.push_back(Elt: CGT.getContext().getCanonicalSizeType());
221 }
222
223 addExtParameterInfosForCall(paramInfos, proto: FPT.getTypePtr(), prefixArgs: PrefixSize,
224 totalArgs: prefix.size());
225}
226
227using ExtParameterInfoList =
228 SmallVector<FunctionProtoType::ExtParameterInfo, 16>;
229
230/// Arrange the LLVM function layout for a value of the given function
231/// type, on top of any implicit parameters already stored.
232static const CGFunctionInfo &
233arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
234 SmallVectorImpl<CanQualType> &prefix,
235 CanQual<FunctionProtoType> FTP) {
236 ExtParameterInfoList paramInfos;
237 RequiredArgs Required = RequiredArgs::forPrototypePlus(prototype: FTP, additional: prefix.size());
238 appendParameterTypes(CGT, prefix, paramInfos, FPT: FTP);
239 CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
240
241 FnInfoOpts opts =
242 instanceMethod ? FnInfoOpts::IsInstanceMethod : FnInfoOpts::None;
243 return CGT.arrangeLLVMFunctionInfo(returnType: resultType, opts, argTypes: prefix,
244 info: FTP->getExtInfo(), paramInfos, args: Required);
245}
246
247using CanQualTypeList = SmallVector<CanQualType, 16>;
248
249/// Arrange the argument and result information for a value of the
250/// given freestanding function type.
251const CGFunctionInfo &
252CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
253 CanQualTypeList argTypes;
254 return ::arrangeLLVMFunctionInfo(CGT&: *this, /*instanceMethod=*/false, prefix&: argTypes,
255 FTP);
256}
257
258static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
259 bool IsTargetDefaultMSABI) {
260 // Set the appropriate calling convention for the Function.
261 if (D->hasAttr<StdCallAttr>())
262 return CC_X86StdCall;
263
264 if (D->hasAttr<FastCallAttr>())
265 return CC_X86FastCall;
266
267 if (D->hasAttr<RegCallAttr>())
268 return CC_X86RegCall;
269
270 if (D->hasAttr<ThisCallAttr>())
271 return CC_X86ThisCall;
272
273 if (D->hasAttr<VectorCallAttr>())
274 return CC_X86VectorCall;
275
276 if (D->hasAttr<PascalAttr>())
277 return CC_X86Pascal;
278
279 if (PcsAttr *PCS = D->getAttr<PcsAttr>())
280 return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
281
282 if (D->hasAttr<AArch64VectorPcsAttr>())
283 return CC_AArch64VectorCall;
284
285 if (D->hasAttr<AArch64SVEPcsAttr>())
286 return CC_AArch64SVEPCS;
287
288 if (D->hasAttr<DeviceKernelAttr>())
289 return CC_DeviceKernel;
290
291 if (D->hasAttr<IntelOclBiccAttr>())
292 return CC_IntelOclBicc;
293
294 if (D->hasAttr<MSABIAttr>())
295 return IsTargetDefaultMSABI ? CC_C : CC_Win64;
296
297 if (D->hasAttr<SysVABIAttr>())
298 return IsTargetDefaultMSABI ? CC_X86_64SysV : CC_C;
299
300 if (D->hasAttr<PreserveMostAttr>())
301 return CC_PreserveMost;
302
303 if (D->hasAttr<PreserveAllAttr>())
304 return CC_PreserveAll;
305
306 if (D->hasAttr<M68kRTDAttr>())
307 return CC_M68kRTD;
308
309 if (D->hasAttr<PreserveNoneAttr>())
310 return CC_PreserveNone;
311
312 if (D->hasAttr<RISCVVectorCCAttr>())
313 return CC_RISCVVectorCall;
314
315 if (RISCVVLSCCAttr *PCS = D->getAttr<RISCVVLSCCAttr>()) {
316 switch (PCS->getVectorWidth()) {
317 default:
318 llvm_unreachable("Invalid RISC-V VLS ABI VLEN");
319#define CC_VLS_CASE(ABI_VLEN) \
320 case ABI_VLEN: \
321 return CC_RISCVVLSCall_##ABI_VLEN;
322 CC_VLS_CASE(32)
323 CC_VLS_CASE(64)
324 CC_VLS_CASE(128)
325 CC_VLS_CASE(256)
326 CC_VLS_CASE(512)
327 CC_VLS_CASE(1024)
328 CC_VLS_CASE(2048)
329 CC_VLS_CASE(4096)
330 CC_VLS_CASE(8192)
331 CC_VLS_CASE(16384)
332 CC_VLS_CASE(32768)
333 CC_VLS_CASE(65536)
334#undef CC_VLS_CASE
335 }
336 }
337
338 return CC_C;
339}
340
341/// Arrange the argument and result information for a call to an
342/// unknown C++ non-static member function of the given abstract type.
343/// (A null RD means we don't have any meaningful "this" argument type,
344/// so fall back to a generic pointer type).
345/// The member function must be an ordinary function, i.e. not a
346/// constructor or destructor.
347const CGFunctionInfo &
348CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
349 const FunctionProtoType *FTP,
350 const CXXMethodDecl *MD) {
351 CanQualTypeList argTypes;
352
353 // Add the 'this' pointer.
354 argTypes.push_back(Elt: DeriveThisType(RD, MD));
355
356 return ::arrangeLLVMFunctionInfo(
357 CGT&: *this, /*instanceMethod=*/true, prefix&: argTypes,
358 FTP: FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
359}
360
361/// Set calling convention for CUDA/HIP kernel.
362static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
363 const FunctionDecl *FD) {
364 if (FD->hasAttr<CUDAGlobalAttr>()) {
365 const FunctionType *FT = FTy->getAs<FunctionType>();
366 CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
367 FTy = FT->getCanonicalTypeUnqualified();
368 }
369}
370
371/// Arrange the argument and result information for a declaration or
372/// definition of the given C++ non-static member function. The
373/// member function must be an ordinary function, i.e. not a
374/// constructor or destructor.
375const CGFunctionInfo &
376CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
377 assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
378 assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
379
380 CanQualType FT = GetFormalType(MD).getAs<Type>();
381 setCUDAKernelCallingConvention(FTy&: FT, CGM, FD: MD);
382 auto prototype = FT.getAs<FunctionProtoType>();
383
384 if (MD->isImplicitObjectMemberFunction()) {
385 // The abstract case is perfectly fine.
386 const CXXRecordDecl *ThisType =
387 getCXXABI().getThisArgumentTypeForMethod(GD: MD);
388 return arrangeCXXMethodType(RD: ThisType, FTP: prototype.getTypePtr(), MD);
389 }
390
391 return arrangeFreeFunctionType(FTP: prototype);
392}
393
394bool CodeGenTypes::inheritingCtorHasParams(
395 const InheritedConstructor &Inherited, CXXCtorType Type) {
396 // Parameters are unnecessary if we're constructing a base class subobject
397 // and the inherited constructor lives in a virtual base.
398 return Type == Ctor_Complete ||
399 !Inherited.getShadowDecl()->constructsVirtualBase() ||
400 !Target.getCXXABI().hasConstructorVariants();
401}
402
403const CGFunctionInfo &
404CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
405 auto *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
406
407 CanQualTypeList argTypes;
408 ExtParameterInfoList paramInfos;
409
410 const CXXRecordDecl *ThisType = getCXXABI().getThisArgumentTypeForMethod(GD);
411 argTypes.push_back(Elt: DeriveThisType(RD: ThisType, MD));
412
413 bool PassParams = true;
414
415 if (auto *CD = dyn_cast<CXXConstructorDecl>(Val: MD)) {
416 // A base class inheriting constructor doesn't get forwarded arguments
417 // needed to construct a virtual base (or base class thereof).
418 if (auto Inherited = CD->getInheritedConstructor())
419 PassParams = inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
420 }
421
422 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
423
424 // Add the formal parameters.
425 if (PassParams)
426 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: FTP);
427
428 CGCXXABI::AddedStructorArgCounts AddedArgs =
429 getCXXABI().buildStructorSignature(GD, ArgTys&: argTypes);
430 if (!paramInfos.empty()) {
431 // Note: prefix implies after the first param.
432 if (AddedArgs.Prefix)
433 paramInfos.insert(I: paramInfos.begin() + 1, NumToInsert: AddedArgs.Prefix,
434 Elt: FunctionProtoType::ExtParameterInfo{});
435 if (AddedArgs.Suffix)
436 paramInfos.append(NumInputs: AddedArgs.Suffix,
437 Elt: FunctionProtoType::ExtParameterInfo{});
438 }
439
440 RequiredArgs required =
441 (PassParams && MD->isVariadic() ? RequiredArgs(argTypes.size())
442 : RequiredArgs::All);
443
444 FunctionType::ExtInfo extInfo = FTP->getExtInfo();
445 CanQualType resultType = getCXXABI().HasThisReturn(GD) ? argTypes.front()
446 : getCXXABI().hasMostDerivedReturn(GD)
447 ? CGM.getContext().VoidPtrTy
448 : Context.VoidTy;
449 return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::IsInstanceMethod,
450 argTypes, info: extInfo, paramInfos, args: required);
451}
452
453static CanQualTypeList getArgTypesForCall(ASTContext &ctx,
454 const CallArgList &args) {
455 CanQualTypeList argTypes;
456 for (auto &arg : args)
457 argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg.Ty));
458 return argTypes;
459}
460
461static CanQualTypeList getArgTypesForDeclaration(ASTContext &ctx,
462 const FunctionArgList &args) {
463 CanQualTypeList argTypes;
464 for (auto &arg : args)
465 argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg->getType()));
466 return argTypes;
467}
468
469static ExtParameterInfoList
470getExtParameterInfosForCall(const FunctionProtoType *proto, unsigned prefixArgs,
471 unsigned totalArgs) {
472 ExtParameterInfoList result;
473 if (proto->hasExtParameterInfos()) {
474 addExtParameterInfosForCall(paramInfos&: result, proto, prefixArgs, totalArgs);
475 }
476 return result;
477}
478
479/// Arrange a call to a C++ method, passing the given arguments.
480///
481/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
482/// parameter.
483/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
484/// args.
485/// PassProtoArgs indicates whether `args` has args for the parameters in the
486/// given CXXConstructorDecl.
487const CGFunctionInfo &CodeGenTypes::arrangeCXXConstructorCall(
488 const CallArgList &args, const CXXConstructorDecl *D, CXXCtorType CtorKind,
489 unsigned ExtraPrefixArgs, unsigned ExtraSuffixArgs, bool PassProtoArgs) {
490 CanQualTypeList ArgTypes;
491 for (const auto &Arg : args)
492 ArgTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
493
494 // +1 for implicit this, which should always be args[0].
495 unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
496
497 CanQual<FunctionProtoType> FPT = GetFormalType(MD: D);
498 RequiredArgs Required = PassProtoArgs
499 ? RequiredArgs::forPrototypePlus(
500 prototype: FPT, additional: TotalPrefixArgs + ExtraSuffixArgs)
501 : RequiredArgs::All;
502
503 GlobalDecl GD(D, CtorKind);
504 CanQualType ResultType = getCXXABI().HasThisReturn(GD) ? ArgTypes.front()
505 : getCXXABI().hasMostDerivedReturn(GD)
506 ? CGM.getContext().VoidPtrTy
507 : Context.VoidTy;
508
509 FunctionType::ExtInfo Info = FPT->getExtInfo();
510 ExtParameterInfoList ParamInfos;
511 // If the prototype args are elided, we should only have ABI-specific args,
512 // which never have param info.
513 if (PassProtoArgs && FPT->hasExtParameterInfos()) {
514 // ABI-specific suffix arguments are treated the same as variadic arguments.
515 addExtParameterInfosForCall(paramInfos&: ParamInfos, proto: FPT.getTypePtr(), prefixArgs: TotalPrefixArgs,
516 totalArgs: ArgTypes.size());
517 }
518
519 return arrangeLLVMFunctionInfo(returnType: ResultType, opts: FnInfoOpts::IsInstanceMethod,
520 argTypes: ArgTypes, info: Info, paramInfos: ParamInfos, args: Required);
521}
522
523/// Arrange the argument and result information for the declaration or
524/// definition of the given function.
525const CGFunctionInfo &
526CodeGenTypes::arrangeFunctionDeclaration(const GlobalDecl GD) {
527 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
528 if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD))
529 if (MD->isImplicitObjectMemberFunction())
530 return arrangeCXXMethodDeclaration(MD);
531
532 CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
533
534 assert(isa<FunctionType>(FTy));
535 setCUDAKernelCallingConvention(FTy, CGM, FD);
536
537 if (DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()) &&
538 GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
539 const FunctionType *FT = FTy->getAs<FunctionType>();
540 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
541 FTy = FT->getCanonicalTypeUnqualified();
542 }
543
544 // When declaring a function without a prototype, always use a
545 // non-variadic type.
546 if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
547 return arrangeLLVMFunctionInfo(returnType: noProto->getReturnType(), opts: FnInfoOpts::None,
548 argTypes: {}, info: noProto->getExtInfo(), paramInfos: {},
549 args: RequiredArgs::All);
550 }
551
552 return arrangeFreeFunctionType(FTP: FTy.castAs<FunctionProtoType>());
553}
554
555/// Arrange the argument and result information for the declaration or
556/// definition of an Objective-C method.
557const CGFunctionInfo &
558CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
559 // It happens that this is the same as a call with no optional
560 // arguments, except also using the formal 'self' type.
561 return arrangeObjCMessageSendSignature(MD, receiverType: MD->getSelfDecl()->getType());
562}
563
564/// Arrange the argument and result information for the function type
565/// through which to perform a send to the given Objective-C method,
566/// using the given receiver type. The receiver type is not always
567/// the 'self' type of the method or even an Objective-C pointer type.
568/// This is *not* the right method for actually performing such a
569/// message send, due to the possibility of optional arguments.
570const CGFunctionInfo &
571CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
572 QualType receiverType) {
573 CanQualTypeList argTys;
574 ExtParameterInfoList extParamInfos(MD->isDirectMethod() ? 1 : 2);
575 argTys.push_back(Elt: Context.getCanonicalParamType(T: receiverType));
576 if (!MD->isDirectMethod())
577 argTys.push_back(Elt: Context.getCanonicalParamType(T: Context.getObjCSelType()));
578 for (const auto *I : MD->parameters()) {
579 argTys.push_back(Elt: Context.getCanonicalParamType(T: I->getType()));
580 auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
581 NoEscape: I->hasAttr<NoEscapeAttr>());
582 extParamInfos.push_back(Elt: extParamInfo);
583 }
584
585 FunctionType::ExtInfo einfo;
586 bool IsTargetDefaultMSABI =
587 getContext().getTargetInfo().getTriple().isOSWindows() ||
588 getContext().getTargetInfo().getTriple().isUEFI();
589 einfo = einfo.withCallingConv(
590 cc: getCallingConventionForDecl(D: MD, IsTargetDefaultMSABI));
591
592 if (getContext().getLangOpts().ObjCAutoRefCount &&
593 MD->hasAttr<NSReturnsRetainedAttr>())
594 einfo = einfo.withProducesResult(producesResult: true);
595
596 RequiredArgs required =
597 (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);
598
599 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: MD->getReturnType()),
600 opts: FnInfoOpts::None, argTypes: argTys, info: einfo, paramInfos: extParamInfos,
601 args: required);
602}
603
604const CGFunctionInfo &
605CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
606 const CallArgList &args) {
607 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
608 FunctionType::ExtInfo einfo;
609
610 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: returnType), opts: FnInfoOpts::None,
611 argTypes, info: einfo, paramInfos: {}, args: RequiredArgs::All);
612}
613
614const CGFunctionInfo &CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
615 // FIXME: Do we need to handle ObjCMethodDecl?
616 if (isa<CXXConstructorDecl>(Val: GD.getDecl()) ||
617 isa<CXXDestructorDecl>(Val: GD.getDecl()))
618 return arrangeCXXStructorDeclaration(GD);
619
620 return arrangeFunctionDeclaration(GD);
621}
622
623/// Arrange a thunk that takes 'this' as the first parameter followed by
624/// varargs. Return a void pointer, regardless of the actual return type.
625/// The body of the thunk will end in a musttail call to a function of the
626/// correct type, and the caller will bitcast the function to the correct
627/// prototype.
628const CGFunctionInfo &
629CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
630 assert(MD->isVirtual() && "only methods have thunks");
631 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
632 CanQualType ArgTys[] = {DeriveThisType(RD: MD->getParent(), MD)};
633 return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::None, argTypes: ArgTys,
634 info: FTP->getExtInfo(), paramInfos: {}, args: RequiredArgs(1));
635}
636
637const CGFunctionInfo &
638CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
639 CXXCtorType CT) {
640 assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
641
642 CanQual<FunctionProtoType> FTP = GetFormalType(MD: CD);
643 SmallVector<CanQualType, 2> ArgTys;
644 const CXXRecordDecl *RD = CD->getParent();
645 ArgTys.push_back(Elt: DeriveThisType(RD, MD: CD));
646 if (CT == Ctor_CopyingClosure)
647 ArgTys.push_back(Elt: *FTP->param_type_begin());
648 if (RD->getNumVBases() > 0)
649 ArgTys.push_back(Elt: Context.IntTy);
650 CallingConv CC = Context.getDefaultCallingConvention(
651 /*IsVariadic=*/false, /*IsCXXMethod=*/true);
652 return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::IsInstanceMethod,
653 argTypes: ArgTys, info: FunctionType::ExtInfo(CC), paramInfos: {},
654 args: RequiredArgs::All);
655}
656
657/// Arrange a call as unto a free function, except possibly with an
658/// additional number of formal parameters considered required.
659static const CGFunctionInfo &
660arrangeFreeFunctionLikeCall(CodeGenTypes &CGT, CodeGenModule &CGM,
661 const CallArgList &args, const FunctionType *fnType,
662 unsigned numExtraRequiredArgs, bool chainCall) {
663 assert(args.size() >= numExtraRequiredArgs);
664
665 ExtParameterInfoList paramInfos;
666
667 // In most cases, there are no optional arguments.
668 RequiredArgs required = RequiredArgs::All;
669
670 // If we have a variadic prototype, the required arguments are the
671 // extra prefix plus the arguments in the prototype.
672 if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(Val: fnType)) {
673 if (proto->isVariadic())
674 required = RequiredArgs::forPrototypePlus(prototype: proto, additional: numExtraRequiredArgs);
675
676 if (proto->hasExtParameterInfos())
677 addExtParameterInfosForCall(paramInfos, proto, prefixArgs: numExtraRequiredArgs,
678 totalArgs: args.size());
679
680 // If we don't have a prototype at all, but we're supposed to
681 // explicitly use the variadic convention for unprototyped calls,
682 // treat all of the arguments as required but preserve the nominal
683 // possibility of variadics.
684 } else if (CGM.getTargetCodeGenInfo().isNoProtoCallVariadic(
685 args, fnType: cast<FunctionNoProtoType>(Val: fnType))) {
686 required = RequiredArgs(args.size());
687 }
688
689 CanQualTypeList argTypes;
690 for (const auto &arg : args)
691 argTypes.push_back(Elt: CGT.getContext().getCanonicalParamType(T: arg.Ty));
692 FnInfoOpts opts = chainCall ? FnInfoOpts::IsChainCall : FnInfoOpts::None;
693 return CGT.arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: fnType->getReturnType()),
694 opts, argTypes, info: fnType->getExtInfo(),
695 paramInfos, args: required);
696}
697
698/// Figure out the rules for calling a function with the given formal
699/// type using the given arguments. The arguments are necessary
700/// because the function might be unprototyped, in which case it's
701/// target-dependent in crazy ways.
702const CGFunctionInfo &CodeGenTypes::arrangeFreeFunctionCall(
703 const CallArgList &args, const FunctionType *fnType, bool chainCall) {
704 return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType,
705 numExtraRequiredArgs: chainCall ? 1 : 0, chainCall);
706}
707
708/// A block function is essentially a free function with an
709/// extra implicit argument.
710const CGFunctionInfo &
711CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
712 const FunctionType *fnType) {
713 return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType, numExtraRequiredArgs: 1,
714 /*chainCall=*/false);
715}
716
717const CGFunctionInfo &
718CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
719 const FunctionArgList &params) {
720 ExtParameterInfoList paramInfos =
721 getExtParameterInfosForCall(proto, prefixArgs: 1, totalArgs: params.size());
722 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args: params);
723
724 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: proto->getReturnType()),
725 opts: FnInfoOpts::None, argTypes,
726 info: proto->getExtInfo(), paramInfos,
727 args: RequiredArgs::forPrototypePlus(prototype: proto, additional: 1));
728}
729
730const CGFunctionInfo &
731CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
732 const CallArgList &args) {
733 CanQualTypeList argTypes;
734 for (const auto &Arg : args)
735 argTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
736 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
737 argTypes, info: FunctionType::ExtInfo(),
738 /*paramInfos=*/{}, args: RequiredArgs::All);
739}
740
741const CGFunctionInfo &
742CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
743 const FunctionArgList &args) {
744 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
745
746 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
747 argTypes, info: FunctionType::ExtInfo(), paramInfos: {},
748 args: RequiredArgs::All);
749}
750
751const CGFunctionInfo &CodeGenTypes::arrangeBuiltinFunctionDeclaration(
752 CanQualType resultType, ArrayRef<CanQualType> argTypes) {
753 return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::None, argTypes,
754 info: FunctionType::ExtInfo(), paramInfos: {},
755 args: RequiredArgs::All);
756}
757
758const CGFunctionInfo &CodeGenTypes::arrangeDeviceKernelCallerDeclaration(
759 QualType resultType, const FunctionArgList &args) {
760 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
761
762 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
763 argTypes,
764 info: FunctionType::ExtInfo(CC_DeviceKernel),
765 /*paramInfos=*/{}, args: RequiredArgs::All);
766}
767
768/// Arrange a call to a C++ method, passing the given arguments.
769///
770/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
771/// does not count `this`.
772const CGFunctionInfo &CodeGenTypes::arrangeCXXMethodCall(
773 const CallArgList &args, const FunctionProtoType *proto,
774 RequiredArgs required, unsigned numPrefixArgs) {
775 assert(numPrefixArgs + 1 <= args.size() &&
776 "Emitting a call with less args than the required prefix?");
777 // Add one to account for `this`. It's a bit awkward here, but we don't count
778 // `this` in similar places elsewhere.
779 ExtParameterInfoList paramInfos =
780 getExtParameterInfosForCall(proto, prefixArgs: numPrefixArgs + 1, totalArgs: args.size());
781
782 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
783
784 FunctionType::ExtInfo info = proto->getExtInfo();
785 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: proto->getReturnType()),
786 opts: FnInfoOpts::IsInstanceMethod, argTypes, info,
787 paramInfos, args: required);
788}
789
790const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
791 return arrangeLLVMFunctionInfo(returnType: getContext().VoidTy, opts: FnInfoOpts::None, argTypes: {},
792 info: FunctionType::ExtInfo(), paramInfos: {},
793 args: RequiredArgs::All);
794}
795
796const CGFunctionInfo &CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
797 const CallArgList &args) {
798 assert(signature.arg_size() <= args.size());
799 if (signature.arg_size() == args.size())
800 return signature;
801
802 ExtParameterInfoList paramInfos;
803 auto sigParamInfos = signature.getExtParameterInfos();
804 if (!sigParamInfos.empty()) {
805 paramInfos.append(in_start: sigParamInfos.begin(), in_end: sigParamInfos.end());
806 paramInfos.resize(N: args.size());
807 }
808
809 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
810
811 assert(signature.getRequiredArgs().allowsOptionalArgs());
812 FnInfoOpts opts = FnInfoOpts::None;
813 if (signature.isInstanceMethod())
814 opts |= FnInfoOpts::IsInstanceMethod;
815 if (signature.isChainCall())
816 opts |= FnInfoOpts::IsChainCall;
817 if (signature.isDelegateCall())
818 opts |= FnInfoOpts::IsDelegateCall;
819 return arrangeLLVMFunctionInfo(returnType: signature.getReturnType(), opts, argTypes,
820 info: signature.getExtInfo(), paramInfos,
821 args: signature.getRequiredArgs());
822}
823
824namespace clang {
825namespace CodeGen {
826void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
827}
828} // namespace clang
829
830/// Arrange the argument and result information for an abstract value
831/// of a given function type. This is the method which all of the
832/// above functions ultimately defer to.
833const CGFunctionInfo &CodeGenTypes::arrangeLLVMFunctionInfo(
834 CanQualType resultType, FnInfoOpts opts, ArrayRef<CanQualType> argTypes,
835 FunctionType::ExtInfo info,
836 ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
837 RequiredArgs required) {
838 assert(llvm::all_of(argTypes,
839 [](CanQualType T) { return T.isCanonicalAsParam(); }));
840
841 // Lookup or create unique function info.
842 llvm::FoldingSetNodeID ID;
843 bool isInstanceMethod =
844 (opts & FnInfoOpts::IsInstanceMethod) == FnInfoOpts::IsInstanceMethod;
845 bool isChainCall =
846 (opts & FnInfoOpts::IsChainCall) == FnInfoOpts::IsChainCall;
847 bool isDelegateCall =
848 (opts & FnInfoOpts::IsDelegateCall) == FnInfoOpts::IsDelegateCall;
849 CGFunctionInfo::Profile(ID, InstanceMethod: isInstanceMethod, ChainCall: isChainCall, IsDelegateCall: isDelegateCall,
850 info, paramInfos, required, resultType, argTypes);
851
852 void *insertPos = nullptr;
853 CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, InsertPos&: insertPos);
854 if (FI)
855 return *FI;
856
857 unsigned CC = ClangCallConvToLLVMCallConv(CC: info.getCC());
858
859 // Construct the function info. We co-allocate the ArgInfos.
860 FI = CGFunctionInfo::create(llvmCC: CC, instanceMethod: isInstanceMethod, chainCall: isChainCall, delegateCall: isDelegateCall,
861 extInfo: info, paramInfos, resultType, argTypes, required);
862 FunctionInfos.InsertNode(N: FI, InsertPos: insertPos);
863
864 bool inserted = FunctionsBeingProcessed.insert(Ptr: FI).second;
865 (void)inserted;
866 assert(inserted && "Recursively being processed?");
867
868 // Compute ABI information.
869 if (CC == llvm::CallingConv::SPIR_KERNEL) {
870 // Force target independent argument handling for the host visible
871 // kernel functions.
872 computeSPIRKernelABIInfo(CGM, FI&: *FI);
873 } else if (info.getCC() == CC_Swift || info.getCC() == CC_SwiftAsync) {
874 swiftcall::computeABIInfo(CGM, FI&: *FI);
875 } else {
876 CGM.getABIInfo().computeInfo(FI&: *FI);
877 }
878
879 // Loop over all of the computed argument and return value info. If any of
880 // them are direct or extend without a specified coerce type, specify the
881 // default now.
882 ABIArgInfo &retInfo = FI->getReturnInfo();
883 if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
884 retInfo.setCoerceToType(ConvertType(T: FI->getReturnType()));
885
886 for (auto &I : FI->arguments())
887 if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
888 I.info.setCoerceToType(ConvertType(T: I.type));
889
890 bool erased = FunctionsBeingProcessed.erase(Ptr: FI);
891 (void)erased;
892 assert(erased && "Not in set?");
893
894 return *FI;
895}
896
897CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC, bool instanceMethod,
898 bool chainCall, bool delegateCall,
899 const FunctionType::ExtInfo &info,
900 ArrayRef<ExtParameterInfo> paramInfos,
901 CanQualType resultType,
902 ArrayRef<CanQualType> argTypes,
903 RequiredArgs required) {
904 assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
905 assert(!required.allowsOptionalArgs() ||
906 required.getNumRequiredArgs() <= argTypes.size());
907
908 void *buffer = operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(
909 Counts: argTypes.size() + 1, Counts: paramInfos.size()));
910
911 CGFunctionInfo *FI = new (buffer) CGFunctionInfo();
912 FI->CallingConvention = llvmCC;
913 FI->EffectiveCallingConvention = llvmCC;
914 FI->ASTCallingConvention = info.getCC();
915 FI->InstanceMethod = instanceMethod;
916 FI->ChainCall = chainCall;
917 FI->DelegateCall = delegateCall;
918 FI->CmseNSCall = info.getCmseNSCall();
919 FI->NoReturn = info.getNoReturn();
920 FI->ReturnsRetained = info.getProducesResult();
921 FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
922 FI->NoCfCheck = info.getNoCfCheck();
923 FI->Required = required;
924 FI->HasRegParm = info.getHasRegParm();
925 FI->RegParm = info.getRegParm();
926 FI->ArgStruct = nullptr;
927 FI->ArgStructAlign = 0;
928 FI->NumArgs = argTypes.size();
929 FI->HasExtParameterInfos = !paramInfos.empty();
930 FI->getArgsBuffer()[0].type = resultType;
931 FI->MaxVectorWidth = 0;
932 for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
933 FI->getArgsBuffer()[i + 1].type = argTypes[i];
934 for (unsigned i = 0, e = paramInfos.size(); i != e; ++i)
935 FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
936 return FI;
937}
938
939/***/
940
941namespace {
942// ABIArgInfo::Expand implementation.
943
944// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
945struct TypeExpansion {
946 enum TypeExpansionKind {
947 // Elements of constant arrays are expanded recursively.
948 TEK_ConstantArray,
949 // Record fields are expanded recursively (but if record is a union, only
950 // the field with the largest size is expanded).
951 TEK_Record,
952 // For complex types, real and imaginary parts are expanded recursively.
953 TEK_Complex,
954 // All other types are not expandable.
955 TEK_None
956 };
957
958 const TypeExpansionKind Kind;
959
960 TypeExpansion(TypeExpansionKind K) : Kind(K) {}
961 virtual ~TypeExpansion() {}
962};
963
964struct ConstantArrayExpansion : TypeExpansion {
965 QualType EltTy;
966 uint64_t NumElts;
967
968 ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
969 : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
970 static bool classof(const TypeExpansion *TE) {
971 return TE->Kind == TEK_ConstantArray;
972 }
973};
974
975struct RecordExpansion : TypeExpansion {
976 SmallVector<const CXXBaseSpecifier *, 1> Bases;
977
978 SmallVector<const FieldDecl *, 1> Fields;
979
980 RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
981 SmallVector<const FieldDecl *, 1> &&Fields)
982 : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
983 Fields(std::move(Fields)) {}
984 static bool classof(const TypeExpansion *TE) {
985 return TE->Kind == TEK_Record;
986 }
987};
988
989struct ComplexExpansion : TypeExpansion {
990 QualType EltTy;
991
992 ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
993 static bool classof(const TypeExpansion *TE) {
994 return TE->Kind == TEK_Complex;
995 }
996};
997
998struct NoExpansion : TypeExpansion {
999 NoExpansion() : TypeExpansion(TEK_None) {}
1000 static bool classof(const TypeExpansion *TE) { return TE->Kind == TEK_None; }
1001};
1002} // namespace
1003
1004static std::unique_ptr<TypeExpansion>
1005getTypeExpansion(QualType Ty, const ASTContext &Context) {
1006 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty)) {
1007 return std::make_unique<ConstantArrayExpansion>(args: AT->getElementType(),
1008 args: AT->getZExtSize());
1009 }
1010 if (const auto *RD = Ty->getAsRecordDecl()) {
1011 SmallVector<const CXXBaseSpecifier *, 1> Bases;
1012 SmallVector<const FieldDecl *, 1> Fields;
1013 assert(!RD->hasFlexibleArrayMember() &&
1014 "Cannot expand structure with flexible array.");
1015 if (RD->isUnion()) {
1016 // Unions can be here only in degenerative cases - all the fields are same
1017 // after flattening. Thus we have to use the "largest" field.
1018 const FieldDecl *LargestFD = nullptr;
1019 CharUnits UnionSize = CharUnits::Zero();
1020
1021 for (const auto *FD : RD->fields()) {
1022 if (FD->isZeroLengthBitField())
1023 continue;
1024 assert(!FD->isBitField() &&
1025 "Cannot expand structure with bit-field members.");
1026 CharUnits FieldSize = Context.getTypeSizeInChars(T: FD->getType());
1027 if (UnionSize < FieldSize) {
1028 UnionSize = FieldSize;
1029 LargestFD = FD;
1030 }
1031 }
1032 if (LargestFD)
1033 Fields.push_back(Elt: LargestFD);
1034 } else {
1035 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
1036 assert(!CXXRD->isDynamicClass() &&
1037 "cannot expand vtable pointers in dynamic classes");
1038 llvm::append_range(C&: Bases, R: llvm::make_pointer_range(Range: CXXRD->bases()));
1039 }
1040
1041 for (const auto *FD : RD->fields()) {
1042 if (FD->isZeroLengthBitField())
1043 continue;
1044 assert(!FD->isBitField() &&
1045 "Cannot expand structure with bit-field members.");
1046 Fields.push_back(Elt: FD);
1047 }
1048 }
1049 return std::make_unique<RecordExpansion>(args: std::move(Bases),
1050 args: std::move(Fields));
1051 }
1052 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
1053 return std::make_unique<ComplexExpansion>(args: CT->getElementType());
1054 }
1055 return std::make_unique<NoExpansion>();
1056}
1057
1058static int getExpansionSize(QualType Ty, const ASTContext &Context) {
1059 auto Exp = getTypeExpansion(Ty, Context);
1060 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1061 return CAExp->NumElts * getExpansionSize(Ty: CAExp->EltTy, Context);
1062 }
1063 if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1064 int Res = 0;
1065 for (auto BS : RExp->Bases)
1066 Res += getExpansionSize(Ty: BS->getType(), Context);
1067 for (auto FD : RExp->Fields)
1068 Res += getExpansionSize(Ty: FD->getType(), Context);
1069 return Res;
1070 }
1071 if (isa<ComplexExpansion>(Val: Exp.get()))
1072 return 2;
1073 assert(isa<NoExpansion>(Exp.get()));
1074 return 1;
1075}
1076
1077void CodeGenTypes::getExpandedTypes(
1078 QualType Ty, SmallVectorImpl<llvm::Type *>::iterator &TI) {
1079 auto Exp = getTypeExpansion(Ty, Context);
1080 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1081 for (int i = 0, n = CAExp->NumElts; i < n; i++) {
1082 getExpandedTypes(Ty: CAExp->EltTy, TI);
1083 }
1084 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1085 for (auto BS : RExp->Bases)
1086 getExpandedTypes(Ty: BS->getType(), TI);
1087 for (auto FD : RExp->Fields)
1088 getExpandedTypes(Ty: FD->getType(), TI);
1089 } else if (auto CExp = dyn_cast<ComplexExpansion>(Val: Exp.get())) {
1090 llvm::Type *EltTy = ConvertType(T: CExp->EltTy);
1091 *TI++ = EltTy;
1092 *TI++ = EltTy;
1093 } else {
1094 assert(isa<NoExpansion>(Exp.get()));
1095 *TI++ = ConvertType(T: Ty);
1096 }
1097}
1098
1099static void forConstantArrayExpansion(CodeGenFunction &CGF,
1100 ConstantArrayExpansion *CAE,
1101 Address BaseAddr,
1102 llvm::function_ref<void(Address)> Fn) {
1103 for (int i = 0, n = CAE->NumElts; i < n; i++) {
1104 Address EltAddr = CGF.Builder.CreateConstGEP2_32(Addr: BaseAddr, Idx0: 0, Idx1: i);
1105 Fn(EltAddr);
1106 }
1107}
1108
1109void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1110 llvm::Function::arg_iterator &AI) {
1111 assert(LV.isSimple() &&
1112 "Unexpected non-simple lvalue during struct expansion.");
1113
1114 auto Exp = getTypeExpansion(Ty, Context: getContext());
1115 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1116 forConstantArrayExpansion(
1117 CGF&: *this, CAE: CAExp, BaseAddr: LV.getAddress(), Fn: [&](Address EltAddr) {
1118 LValue LV = MakeAddrLValue(Addr: EltAddr, T: CAExp->EltTy);
1119 ExpandTypeFromArgs(Ty: CAExp->EltTy, LV, AI);
1120 });
1121 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1122 Address This = LV.getAddress();
1123 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1124 // Perform a single step derived-to-base conversion.
1125 Address Base =
1126 GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1127 /*NullCheckValue=*/false, Loc: SourceLocation());
1128 LValue SubLV = MakeAddrLValue(Addr: Base, T: BS->getType());
1129
1130 // Recurse onto bases.
1131 ExpandTypeFromArgs(Ty: BS->getType(), LV: SubLV, AI);
1132 }
1133 for (auto FD : RExp->Fields) {
1134 // FIXME: What are the right qualifiers here?
1135 LValue SubLV = EmitLValueForFieldInitialization(Base: LV, Field: FD);
1136 ExpandTypeFromArgs(Ty: FD->getType(), LV: SubLV, AI);
1137 }
1138 } else if (isa<ComplexExpansion>(Val: Exp.get())) {
1139 auto realValue = &*AI++;
1140 auto imagValue = &*AI++;
1141 EmitStoreOfComplex(V: ComplexPairTy(realValue, imagValue), dest: LV, /*init*/ isInit: true);
1142 } else {
1143 // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1144 // primitive store.
1145 assert(isa<NoExpansion>(Exp.get()));
1146 llvm::Value *Arg = &*AI++;
1147 if (LV.isBitField()) {
1148 EmitStoreThroughLValue(Src: RValue::get(V: Arg), Dst: LV);
1149 } else {
1150 // TODO: currently there are some places are inconsistent in what LLVM
1151 // pointer type they use (see D118744). Once clang uses opaque pointers
1152 // all LLVM pointer types will be the same and we can remove this check.
1153 if (Arg->getType()->isPointerTy()) {
1154 Address Addr = LV.getAddress();
1155 Arg = Builder.CreateBitCast(V: Arg, DestTy: Addr.getElementType());
1156 }
1157 EmitStoreOfScalar(value: Arg, lvalue: LV);
1158 }
1159 }
1160}
1161
1162void CodeGenFunction::ExpandTypeToArgs(
1163 QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1164 SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1165 auto Exp = getTypeExpansion(Ty, Context: getContext());
1166 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1167 Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1168 : Arg.getKnownRValue().getAggregateAddress();
1169 forConstantArrayExpansion(CGF&: *this, CAE: CAExp, BaseAddr: Addr, Fn: [&](Address EltAddr) {
1170 CallArg EltArg =
1171 CallArg(convertTempToRValue(addr: EltAddr, type: CAExp->EltTy, Loc: SourceLocation()),
1172 CAExp->EltTy);
1173 ExpandTypeToArgs(Ty: CAExp->EltTy, Arg: EltArg, IRFuncTy, IRCallArgs,
1174 IRCallArgPos);
1175 });
1176 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1177 Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1178 : Arg.getKnownRValue().getAggregateAddress();
1179 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1180 // Perform a single step derived-to-base conversion.
1181 Address Base =
1182 GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1183 /*NullCheckValue=*/false, Loc: SourceLocation());
1184 CallArg BaseArg = CallArg(RValue::getAggregate(addr: Base), BS->getType());
1185
1186 // Recurse onto bases.
1187 ExpandTypeToArgs(Ty: BS->getType(), Arg: BaseArg, IRFuncTy, IRCallArgs,
1188 IRCallArgPos);
1189 }
1190
1191 LValue LV = MakeAddrLValue(Addr: This, T: Ty);
1192 for (auto FD : RExp->Fields) {
1193 CallArg FldArg =
1194 CallArg(EmitRValueForField(LV, FD, Loc: SourceLocation()), FD->getType());
1195 ExpandTypeToArgs(Ty: FD->getType(), Arg: FldArg, IRFuncTy, IRCallArgs,
1196 IRCallArgPos);
1197 }
1198 } else if (isa<ComplexExpansion>(Val: Exp.get())) {
1199 ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1200 IRCallArgs[IRCallArgPos++] = CV.first;
1201 IRCallArgs[IRCallArgPos++] = CV.second;
1202 } else {
1203 assert(isa<NoExpansion>(Exp.get()));
1204 auto RV = Arg.getKnownRValue();
1205 assert(RV.isScalar() &&
1206 "Unexpected non-scalar rvalue during struct expansion.");
1207
1208 // Insert a bitcast as needed.
1209 llvm::Value *V = RV.getScalarVal();
1210 if (IRCallArgPos < IRFuncTy->getNumParams() &&
1211 V->getType() != IRFuncTy->getParamType(i: IRCallArgPos))
1212 V = Builder.CreateBitCast(V, DestTy: IRFuncTy->getParamType(i: IRCallArgPos));
1213
1214 IRCallArgs[IRCallArgPos++] = V;
1215 }
1216}
1217
1218/// Create a temporary allocation for the purposes of coercion.
1219static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF,
1220 llvm::Type *Ty,
1221 CharUnits MinAlign,
1222 const Twine &Name = "tmp") {
1223 // Don't use an alignment that's worse than what LLVM would prefer.
1224 auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty);
1225 CharUnits Align = std::max(a: MinAlign, b: CharUnits::fromQuantity(Quantity: PrefAlign));
1226
1227 return CGF.CreateTempAlloca(Ty, align: Align, Name: Name + ".coerce");
1228}
1229
1230/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1231/// accessing some number of bytes out of it, try to gep into the struct to get
1232/// at its inner goodness. Dive as deep as possible without entering an element
1233/// with an in-memory size smaller than DstSize.
1234static Address EnterStructPointerForCoercedAccess(Address SrcPtr,
1235 llvm::StructType *SrcSTy,
1236 uint64_t DstSize,
1237 CodeGenFunction &CGF) {
1238 // We can't dive into a zero-element struct.
1239 if (SrcSTy->getNumElements() == 0)
1240 return SrcPtr;
1241
1242 llvm::Type *FirstElt = SrcSTy->getElementType(N: 0);
1243
1244 // If the first elt is at least as large as what we're looking for, or if the
1245 // first element is the same size as the whole struct, we can enter it. The
1246 // comparison must be made on the store size and not the alloca size. Using
1247 // the alloca size may overstate the size of the load.
1248 uint64_t FirstEltSize = CGF.CGM.getDataLayout().getTypeStoreSize(Ty: FirstElt);
1249 if (FirstEltSize < DstSize &&
1250 FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(Ty: SrcSTy))
1251 return SrcPtr;
1252
1253 // GEP into the first element.
1254 SrcPtr = CGF.Builder.CreateStructGEP(Addr: SrcPtr, Index: 0, Name: "coerce.dive");
1255
1256 // If the first element is a struct, recurse.
1257 llvm::Type *SrcTy = SrcPtr.getElementType();
1258 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy))
1259 return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1260
1261 return SrcPtr;
1262}
1263
1264/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1265/// are either integers or pointers. This does a truncation of the value if it
1266/// is too large or a zero extension if it is too small.
1267///
1268/// This behaves as if the value were coerced through memory, so on big-endian
1269/// targets the high bits are preserved in a truncation, while little-endian
1270/// targets preserve the low bits.
1271static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, llvm::Type *Ty,
1272 CodeGenFunction &CGF) {
1273 if (Val->getType() == Ty)
1274 return Val;
1275
1276 if (isa<llvm::PointerType>(Val: Val->getType())) {
1277 // If this is Pointer->Pointer avoid conversion to and from int.
1278 if (isa<llvm::PointerType>(Val: Ty))
1279 return CGF.Builder.CreateBitCast(V: Val, DestTy: Ty, Name: "coerce.val");
1280
1281 // Convert the pointer to an integer so we can play with its width.
1282 Val = CGF.Builder.CreatePtrToInt(V: Val, DestTy: CGF.IntPtrTy, Name: "coerce.val.pi");
1283 }
1284
1285 llvm::Type *DestIntTy = Ty;
1286 if (isa<llvm::PointerType>(Val: DestIntTy))
1287 DestIntTy = CGF.IntPtrTy;
1288
1289 if (Val->getType() != DestIntTy) {
1290 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1291 if (DL.isBigEndian()) {
1292 // Preserve the high bits on big-endian targets.
1293 // That is what memory coercion does.
1294 uint64_t SrcSize = DL.getTypeSizeInBits(Ty: Val->getType());
1295 uint64_t DstSize = DL.getTypeSizeInBits(Ty: DestIntTy);
1296
1297 if (SrcSize > DstSize) {
1298 Val = CGF.Builder.CreateLShr(LHS: Val, RHS: SrcSize - DstSize, Name: "coerce.highbits");
1299 Val = CGF.Builder.CreateTrunc(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1300 } else {
1301 Val = CGF.Builder.CreateZExt(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1302 Val = CGF.Builder.CreateShl(LHS: Val, RHS: DstSize - SrcSize, Name: "coerce.highbits");
1303 }
1304 } else {
1305 // Little-endian targets preserve the low bits. No shifts required.
1306 Val = CGF.Builder.CreateIntCast(V: Val, DestTy: DestIntTy, isSigned: false, Name: "coerce.val.ii");
1307 }
1308 }
1309
1310 if (isa<llvm::PointerType>(Val: Ty))
1311 Val = CGF.Builder.CreateIntToPtr(V: Val, DestTy: Ty, Name: "coerce.val.ip");
1312 return Val;
1313}
1314
1315static llvm::Value *CreatePFPCoercedLoad(Address Src, QualType SrcFETy,
1316 llvm::Type *Ty, CodeGenFunction &CGF) {
1317 std::vector<PFPField> PFPFields = CGF.getContext().findPFPFields(Ty: SrcFETy);
1318 if (PFPFields.empty())
1319 return nullptr;
1320
1321 auto LoadCoercedField = [&](CharUnits Offset,
1322 llvm::Type *FieldType) -> llvm::Value * {
1323 // Check whether the field at Offset is a PFP field. This function is called
1324 // in ascending order of offset, and PFPFields is sorted by offset. This
1325 // means that we only need to check the first element (and remove it from
1326 // PFPFields if matching).
1327 if (!PFPFields.empty() && PFPFields[0].Offset == Offset) {
1328 auto FieldAddr = CGF.EmitAddressOfPFPField(RecordPtr: Src, Field: PFPFields[0]);
1329 llvm::Value *FieldVal = CGF.Builder.CreateLoad(Addr: FieldAddr);
1330 if (isa<llvm::IntegerType>(Val: FieldType))
1331 FieldVal = CGF.Builder.CreatePtrToInt(V: FieldVal, DestTy: FieldType);
1332 PFPFields.erase(position: PFPFields.begin());
1333 return FieldVal;
1334 }
1335 auto FieldAddr =
1336 CGF.Builder
1337 .CreateConstInBoundsByteGEP(Addr: Src.withElementType(ElemTy: CGF.Int8Ty), Offset)
1338 .withElementType(ElemTy: FieldType);
1339 return CGF.Builder.CreateLoad(Addr: FieldAddr);
1340 };
1341
1342 // The types handled by this function are the only ones that may be generated
1343 // by AArch64ABIInfo::classify{Argument,Return}Type for struct types with
1344 // pointers. PFP is only supported on AArch64.
1345 if (isa<llvm::IntegerType>(Val: Ty) || isa<llvm::PointerType>(Val: Ty)) {
1346 auto Addr = CGF.EmitAddressOfPFPField(RecordPtr: Src, Field: PFPFields[0]);
1347 llvm::Value *Val = CGF.Builder.CreateLoad(Addr);
1348 if (isa<llvm::IntegerType>(Val: Ty))
1349 Val = CGF.Builder.CreatePtrToInt(V: Val, DestTy: Ty);
1350 return Val;
1351 }
1352 auto *AT = cast<llvm::ArrayType>(Val: Ty);
1353 auto *ET = AT->getElementType();
1354 CharUnits WordSize = CGF.getContext().toCharUnitsFromBits(
1355 BitSize: CGF.CGM.getDataLayout().getTypeSizeInBits(Ty: ET));
1356 CharUnits Offset = CharUnits::Zero();
1357 llvm::Value *Val = llvm::PoisonValue::get(T: AT);
1358 for (unsigned Idx = 0; Idx != AT->getNumElements(); ++Idx, Offset += WordSize)
1359 Val = CGF.Builder.CreateInsertValue(Agg: Val, Val: LoadCoercedField(Offset, ET), Idxs: Idx);
1360 return Val;
1361}
1362
1363/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1364/// a pointer to an object of type \arg Ty, known to be aligned to
1365/// \arg SrcAlign bytes.
1366///
1367/// This safely handles the case when the src type is smaller than the
1368/// destination type; in this situation the values of bits which not
1369/// present in the src are undefined.
1370static llvm::Value *CreateCoercedLoad(Address Src, QualType SrcFETy,
1371 llvm::Type *Ty, CodeGenFunction &CGF) {
1372 llvm::Type *SrcTy = Src.getElementType();
1373
1374 // If SrcTy and Ty are the same, just do a load.
1375 if (SrcTy == Ty)
1376 return CGF.Builder.CreateLoad(Addr: Src);
1377
1378 if (llvm::Value *V = CreatePFPCoercedLoad(Src, SrcFETy, Ty, CGF))
1379 return V;
1380
1381 llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1382
1383 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy)) {
1384 Src = EnterStructPointerForCoercedAccess(SrcPtr: Src, SrcSTy,
1385 DstSize: DstSize.getFixedValue(), CGF);
1386 SrcTy = Src.getElementType();
1387 }
1388
1389 llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1390
1391 // If the source and destination are integer or pointer types, just do an
1392 // extension or truncation to the desired type.
1393 if ((isa<llvm::IntegerType>(Val: Ty) || isa<llvm::PointerType>(Val: Ty)) &&
1394 (isa<llvm::IntegerType>(Val: SrcTy) || isa<llvm::PointerType>(Val: SrcTy))) {
1395 llvm::Value *Load = CGF.Builder.CreateLoad(Addr: Src);
1396 return CoerceIntOrPtrToIntOrPtr(Val: Load, Ty, CGF);
1397 }
1398
1399 // If load is legal, just bitcast the src pointer.
1400 if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1401 SrcSize.getFixedValue() >= DstSize.getFixedValue()) {
1402 // Generally SrcSize is never greater than DstSize, since this means we are
1403 // losing bits. However, this can happen in cases where the structure has
1404 // additional padding, for example due to a user specified alignment.
1405 //
1406 // FIXME: Assert that we aren't truncating non-padding bits when have access
1407 // to that information.
1408 Src = Src.withElementType(ElemTy: Ty);
1409 return CGF.Builder.CreateLoad(Addr: Src);
1410 }
1411
1412 // If coercing a fixed vector to a scalable vector for ABI compatibility, and
1413 // the types match, use the llvm.vector.insert intrinsic to perform the
1414 // conversion.
1415 if (auto *ScalableDstTy = dyn_cast<llvm::ScalableVectorType>(Val: Ty)) {
1416 if (auto *FixedSrcTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy)) {
1417 // If we are casting a fixed i8 vector to a scalable i1 predicate
1418 // vector, use a vector insert and bitcast the result.
1419 if (ScalableDstTy->getElementType()->isIntegerTy(Bitwidth: 1) &&
1420 FixedSrcTy->getElementType()->isIntegerTy(Bitwidth: 8)) {
1421 ScalableDstTy = llvm::ScalableVectorType::get(
1422 ElementType: FixedSrcTy->getElementType(),
1423 MinNumElts: llvm::divideCeil(
1424 Numerator: ScalableDstTy->getElementCount().getKnownMinValue(), Denominator: 8));
1425 }
1426 if (ScalableDstTy->getElementType() == FixedSrcTy->getElementType()) {
1427 auto *Load = CGF.Builder.CreateLoad(Addr: Src);
1428 auto *PoisonVec = llvm::PoisonValue::get(T: ScalableDstTy);
1429 llvm::Value *Result = CGF.Builder.CreateInsertVector(
1430 DstType: ScalableDstTy, SrcVec: PoisonVec, SubVec: Load, Idx: uint64_t(0), Name: "cast.scalable");
1431 ScalableDstTy = cast<llvm::ScalableVectorType>(
1432 Val: llvm::VectorType::getWithSizeAndScalar(SizeTy: ScalableDstTy, EltTy: Ty));
1433 if (Result->getType() != ScalableDstTy)
1434 Result = CGF.Builder.CreateBitCast(V: Result, DestTy: ScalableDstTy);
1435 if (Result->getType() != Ty)
1436 Result = CGF.Builder.CreateExtractVector(DstType: Ty, SrcVec: Result, Idx: uint64_t(0));
1437 return Result;
1438 }
1439 }
1440 }
1441
1442 // Otherwise do coercion through memory. This is stupid, but simple.
1443 RawAddress Tmp =
1444 CreateTempAllocaForCoercion(CGF, Ty, MinAlign: Src.getAlignment(), Name: Src.getName());
1445 CGF.Builder.CreateMemCpy(
1446 Dst: Tmp.getPointer(), DstAlign: Tmp.getAlignment().getAsAlign(),
1447 Src: Src.emitRawPointer(CGF), SrcAlign: Src.getAlignment().getAsAlign(),
1448 Size: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: SrcSize.getKnownMinValue()));
1449 return CGF.Builder.CreateLoad(Addr: Tmp);
1450}
1451
1452static bool CreatePFPCoercedStore(llvm::Value *Src, QualType SrcFETy,
1453 Address Dst, CodeGenFunction &CGF) {
1454 std::vector<PFPField> PFPFields = CGF.getContext().findPFPFields(Ty: SrcFETy);
1455 if (PFPFields.empty())
1456 return false;
1457
1458 llvm::Type *SrcTy = Src->getType();
1459 auto StoreCoercedField = [&](CharUnits Offset, llvm::Value *FieldVal) {
1460 if (!PFPFields.empty() && PFPFields[0].Offset == Offset) {
1461 auto FieldAddr = CGF.EmitAddressOfPFPField(RecordPtr: Dst, Field: PFPFields[0]);
1462 if (isa<llvm::IntegerType>(Val: FieldVal->getType()))
1463 FieldVal = CGF.Builder.CreateIntToPtr(V: FieldVal, DestTy: CGF.VoidPtrTy);
1464 CGF.Builder.CreateStore(Val: FieldVal, Addr: FieldAddr);
1465 PFPFields.erase(position: PFPFields.begin());
1466 } else {
1467 auto FieldAddr = CGF.Builder
1468 .CreateConstInBoundsByteGEP(
1469 Addr: Dst.withElementType(ElemTy: CGF.Int8Ty), Offset)
1470 .withElementType(ElemTy: FieldVal->getType());
1471 CGF.Builder.CreateStore(Val: FieldVal, Addr: FieldAddr);
1472 }
1473 };
1474
1475 // The types handled by this function are the only ones that may be generated
1476 // by AArch64ABIInfo::classify{Argument,Return}Type for struct types with
1477 // pointers. PFP is only supported on AArch64.
1478 if (isa<llvm::IntegerType>(Val: SrcTy) || isa<llvm::PointerType>(Val: SrcTy)) {
1479 if (isa<llvm::IntegerType>(Val: SrcTy))
1480 Src = CGF.Builder.CreateIntToPtr(V: Src, DestTy: CGF.VoidPtrTy);
1481 auto Addr = CGF.EmitAddressOfPFPField(RecordPtr: Dst, Field: PFPFields[0]);
1482 CGF.Builder.CreateStore(Val: Src, Addr);
1483 } else {
1484 auto *AT = cast<llvm::ArrayType>(Val: SrcTy);
1485 auto *ET = AT->getElementType();
1486 CharUnits WordSize = CGF.getContext().toCharUnitsFromBits(
1487 BitSize: CGF.CGM.getDataLayout().getTypeSizeInBits(Ty: ET));
1488 CharUnits Offset = CharUnits::Zero();
1489 for (unsigned i = 0; i != AT->getNumElements(); ++i, Offset += WordSize)
1490 StoreCoercedField(Offset, CGF.Builder.CreateExtractValue(Agg: Src, Idxs: i));
1491 }
1492 return true;
1493}
1494
1495void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, QualType SrcFETy,
1496 Address Dst, llvm::TypeSize DstSize,
1497 bool DstIsVolatile) {
1498 if (!DstSize)
1499 return;
1500
1501 llvm::Type *SrcTy = Src->getType();
1502 llvm::TypeSize SrcSize = CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1503
1504 // GEP into structs to try to make types match.
1505 // FIXME: This isn't really that useful with opaque types, but it impacts a
1506 // lot of regression tests.
1507 if (SrcTy != Dst.getElementType()) {
1508 if (llvm::StructType *DstSTy =
1509 dyn_cast<llvm::StructType>(Val: Dst.getElementType())) {
1510 assert(!SrcSize.isScalable());
1511 Dst = EnterStructPointerForCoercedAccess(SrcPtr: Dst, SrcSTy: DstSTy,
1512 DstSize: SrcSize.getFixedValue(), CGF&: *this);
1513 }
1514 }
1515
1516 if (CreatePFPCoercedStore(Src, SrcFETy, Dst, CGF&: *this))
1517 return;
1518
1519 if (SrcSize.isScalable() || SrcSize <= DstSize) {
1520 if (SrcTy->isIntegerTy() && Dst.getElementType()->isPointerTy() &&
1521 SrcSize == CGM.getDataLayout().getTypeAllocSize(Ty: Dst.getElementType())) {
1522 // If the value is supposed to be a pointer, convert it before storing it.
1523 Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: Dst.getElementType(), CGF&: *this);
1524 auto *I = Builder.CreateStore(Val: Src, Addr: Dst, IsVolatile: DstIsVolatile);
1525 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1526 } else if (llvm::StructType *STy =
1527 dyn_cast<llvm::StructType>(Val: Src->getType())) {
1528 // Prefer scalar stores to first-class aggregate stores.
1529 Dst = Dst.withElementType(ElemTy: SrcTy);
1530 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1531 Address EltPtr = Builder.CreateStructGEP(Addr: Dst, Index: i);
1532 llvm::Value *Elt = Builder.CreateExtractValue(Agg: Src, Idxs: i);
1533 auto *I = Builder.CreateStore(Val: Elt, Addr: EltPtr, IsVolatile: DstIsVolatile);
1534 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Elt);
1535 }
1536 } else {
1537 auto *I =
1538 Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: SrcTy), IsVolatile: DstIsVolatile);
1539 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1540 }
1541 } else if (SrcTy->isIntegerTy()) {
1542 // If the source is a simple integer, coerce it directly.
1543 llvm::Type *DstIntTy = Builder.getIntNTy(N: DstSize.getFixedValue() * 8);
1544 Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: DstIntTy, CGF&: *this);
1545 auto *I =
1546 Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: DstIntTy), IsVolatile: DstIsVolatile);
1547 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1548 } else {
1549 // Otherwise do coercion through memory. This is stupid, but
1550 // simple.
1551
1552 // Generally SrcSize is never greater than DstSize, since this means we are
1553 // losing bits. However, this can happen in cases where the structure has
1554 // additional padding, for example due to a user specified alignment.
1555 //
1556 // FIXME: Assert that we aren't truncating non-padding bits when have access
1557 // to that information.
1558 RawAddress Tmp =
1559 CreateTempAllocaForCoercion(CGF&: *this, Ty: SrcTy, MinAlign: Dst.getAlignment());
1560 Builder.CreateStore(Val: Src, Addr: Tmp);
1561 auto *I = Builder.CreateMemCpy(
1562 Dst: Dst.emitRawPointer(CGF&: *this), DstAlign: Dst.getAlignment().getAsAlign(),
1563 Src: Tmp.getPointer(), SrcAlign: Tmp.getAlignment().getAsAlign(),
1564 Size: Builder.CreateTypeSize(Ty: IntPtrTy, Size: DstSize));
1565 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1566 }
1567}
1568
1569static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1570 const ABIArgInfo &info) {
1571 if (unsigned offset = info.getDirectOffset()) {
1572 addr = addr.withElementType(ElemTy: CGF.Int8Ty);
1573 addr = CGF.Builder.CreateConstInBoundsByteGEP(
1574 Addr: addr, Offset: CharUnits::fromQuantity(Quantity: offset));
1575 addr = addr.withElementType(ElemTy: info.getCoerceToType());
1576 }
1577 return addr;
1578}
1579
1580static std::pair<llvm::Value *, bool>
1581CoerceScalableToFixed(CodeGenFunction &CGF, llvm::FixedVectorType *ToTy,
1582 llvm::ScalableVectorType *FromTy, llvm::Value *V,
1583 StringRef Name = "") {
1584 // If we are casting a scalable i1 predicate vector to a fixed i8
1585 // vector, first bitcast the source.
1586 if (FromTy->getElementType()->isIntegerTy(Bitwidth: 1) &&
1587 ToTy->getElementType() == CGF.Builder.getInt8Ty()) {
1588 if (!FromTy->getElementCount().isKnownMultipleOf(RHS: 8)) {
1589 FromTy = llvm::ScalableVectorType::get(
1590 ElementType: FromTy->getElementType(),
1591 MinNumElts: llvm::alignTo<8>(Value: FromTy->getElementCount().getKnownMinValue()));
1592 llvm::Value *ZeroVec = llvm::Constant::getNullValue(Ty: FromTy);
1593 V = CGF.Builder.CreateInsertVector(DstType: FromTy, SrcVec: ZeroVec, SubVec: V, Idx: uint64_t(0));
1594 }
1595 FromTy = llvm::ScalableVectorType::get(
1596 ElementType: ToTy->getElementType(),
1597 MinNumElts: FromTy->getElementCount().getKnownMinValue() / 8);
1598 V = CGF.Builder.CreateBitCast(V, DestTy: FromTy);
1599 }
1600 if (FromTy->getElementType() == ToTy->getElementType()) {
1601 V->setName(Name + ".coerce");
1602 V = CGF.Builder.CreateExtractVector(DstType: ToTy, SrcVec: V, Idx: uint64_t(0), Name: "cast.fixed");
1603 return {V, true};
1604 }
1605 return {V, false};
1606}
1607
1608namespace {
1609
1610/// Encapsulates information about the way function arguments from
1611/// CGFunctionInfo should be passed to actual LLVM IR function.
1612class ClangToLLVMArgMapping {
1613 static const unsigned InvalidIndex = ~0U;
1614 unsigned InallocaArgNo;
1615 unsigned SRetArgNo;
1616 unsigned TotalIRArgs;
1617
1618 /// Arguments of LLVM IR function corresponding to single Clang argument.
1619 struct IRArgs {
1620 unsigned PaddingArgIndex;
1621 // Argument is expanded to IR arguments at positions
1622 // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1623 unsigned FirstArgIndex;
1624 unsigned NumberOfArgs;
1625
1626 IRArgs()
1627 : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1628 NumberOfArgs(0) {}
1629 };
1630
1631 SmallVector<IRArgs, 8> ArgInfo;
1632
1633public:
1634 ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1635 bool OnlyRequiredArgs = false)
1636 : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1637 ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1638 construct(Context, FI, OnlyRequiredArgs);
1639 }
1640
1641 bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1642 unsigned getInallocaArgNo() const {
1643 assert(hasInallocaArg());
1644 return InallocaArgNo;
1645 }
1646
1647 bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1648 unsigned getSRetArgNo() const {
1649 assert(hasSRetArg());
1650 return SRetArgNo;
1651 }
1652
1653 unsigned totalIRArgs() const { return TotalIRArgs; }
1654
1655 bool hasPaddingArg(unsigned ArgNo) const {
1656 assert(ArgNo < ArgInfo.size());
1657 return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1658 }
1659 unsigned getPaddingArgNo(unsigned ArgNo) const {
1660 assert(hasPaddingArg(ArgNo));
1661 return ArgInfo[ArgNo].PaddingArgIndex;
1662 }
1663
1664 /// Returns index of first IR argument corresponding to ArgNo, and their
1665 /// quantity.
1666 std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1667 assert(ArgNo < ArgInfo.size());
1668 return std::make_pair(x: ArgInfo[ArgNo].FirstArgIndex,
1669 y: ArgInfo[ArgNo].NumberOfArgs);
1670 }
1671
1672private:
1673 void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1674 bool OnlyRequiredArgs);
1675};
1676
1677void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1678 const CGFunctionInfo &FI,
1679 bool OnlyRequiredArgs) {
1680 unsigned IRArgNo = 0;
1681 bool SwapThisWithSRet = false;
1682 const ABIArgInfo &RetAI = FI.getReturnInfo();
1683
1684 if (RetAI.getKind() == ABIArgInfo::Indirect) {
1685 SwapThisWithSRet = RetAI.isSRetAfterThis();
1686 SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1687 }
1688
1689 unsigned ArgNo = 0;
1690 unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();
1691 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1692 ++I, ++ArgNo) {
1693 assert(I != FI.arg_end());
1694 QualType ArgType = I->type;
1695 const ABIArgInfo &AI = I->info;
1696 // Collect data about IR arguments corresponding to Clang argument ArgNo.
1697 auto &IRArgs = ArgInfo[ArgNo];
1698
1699 if (AI.getPaddingType())
1700 IRArgs.PaddingArgIndex = IRArgNo++;
1701
1702 switch (AI.getKind()) {
1703 case ABIArgInfo::TargetSpecific:
1704 case ABIArgInfo::Extend:
1705 case ABIArgInfo::Direct: {
1706 // FIXME: handle sseregparm someday...
1707 llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: AI.getCoerceToType());
1708 if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
1709 IRArgs.NumberOfArgs = STy->getNumElements();
1710 } else {
1711 IRArgs.NumberOfArgs = 1;
1712 }
1713 break;
1714 }
1715 case ABIArgInfo::Indirect:
1716 case ABIArgInfo::IndirectAliased:
1717 IRArgs.NumberOfArgs = 1;
1718 break;
1719 case ABIArgInfo::Ignore:
1720 case ABIArgInfo::InAlloca:
1721 // ignore and inalloca doesn't have matching LLVM parameters.
1722 IRArgs.NumberOfArgs = 0;
1723 break;
1724 case ABIArgInfo::CoerceAndExpand:
1725 IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1726 break;
1727 case ABIArgInfo::Expand:
1728 IRArgs.NumberOfArgs = getExpansionSize(Ty: ArgType, Context);
1729 break;
1730 }
1731
1732 if (IRArgs.NumberOfArgs > 0) {
1733 IRArgs.FirstArgIndex = IRArgNo;
1734 IRArgNo += IRArgs.NumberOfArgs;
1735 }
1736
1737 // Skip over the sret parameter when it comes second. We already handled it
1738 // above.
1739 if (IRArgNo == 1 && SwapThisWithSRet)
1740 IRArgNo++;
1741 }
1742 assert(ArgNo == ArgInfo.size());
1743
1744 if (FI.usesInAlloca())
1745 InallocaArgNo = IRArgNo++;
1746
1747 TotalIRArgs = IRArgNo;
1748}
1749} // namespace
1750
1751/***/
1752
1753bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1754 const auto &RI = FI.getReturnInfo();
1755 return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());
1756}
1757
1758bool CodeGenModule::ReturnTypeHasInReg(const CGFunctionInfo &FI) {
1759 const auto &RI = FI.getReturnInfo();
1760 return RI.getInReg();
1761}
1762
1763bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1764 return ReturnTypeUsesSRet(FI) &&
1765 getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
1766}
1767
1768bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1769 if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1770 switch (BT->getKind()) {
1771 default:
1772 return false;
1773 case BuiltinType::Float:
1774 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Float);
1775 case BuiltinType::Double:
1776 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Double);
1777 case BuiltinType::LongDouble:
1778 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::LongDouble);
1779 }
1780 }
1781
1782 return false;
1783}
1784
1785bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1786 if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1787 if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1788 if (BT->getKind() == BuiltinType::LongDouble)
1789 return getTarget().useObjCFP2RetForComplexLongDouble();
1790 }
1791 }
1792
1793 return false;
1794}
1795
1796llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1797 const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1798 return GetFunctionType(Info: FI);
1799}
1800
1801llvm::FunctionType *CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1802
1803 bool Inserted = FunctionsBeingProcessed.insert(Ptr: &FI).second;
1804 (void)Inserted;
1805 assert(Inserted && "Recursively being processed?");
1806
1807 llvm::Type *resultType = nullptr;
1808 const ABIArgInfo &retAI = FI.getReturnInfo();
1809 switch (retAI.getKind()) {
1810 case ABIArgInfo::Expand:
1811 case ABIArgInfo::IndirectAliased:
1812 llvm_unreachable("Invalid ABI kind for return argument");
1813
1814 case ABIArgInfo::TargetSpecific:
1815 case ABIArgInfo::Extend:
1816 case ABIArgInfo::Direct:
1817 resultType = retAI.getCoerceToType();
1818 break;
1819
1820 case ABIArgInfo::InAlloca:
1821 if (retAI.getInAllocaSRet()) {
1822 // sret things on win32 aren't void, they return the sret pointer.
1823 QualType ret = FI.getReturnType();
1824 unsigned addressSpace = CGM.getTypes().getTargetAddressSpace(T: ret);
1825 resultType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: addressSpace);
1826 } else {
1827 resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
1828 }
1829 break;
1830
1831 case ABIArgInfo::Indirect:
1832 case ABIArgInfo::Ignore:
1833 resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
1834 break;
1835
1836 case ABIArgInfo::CoerceAndExpand:
1837 resultType = retAI.getUnpaddedCoerceAndExpandType();
1838 break;
1839 }
1840
1841 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1842 SmallVector<llvm::Type *, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1843
1844 // Add type for sret argument.
1845 if (IRFunctionArgs.hasSRetArg()) {
1846 ArgTypes[IRFunctionArgs.getSRetArgNo()] = llvm::PointerType::get(
1847 C&: getLLVMContext(), AddressSpace: FI.getReturnInfo().getIndirectAddrSpace());
1848 }
1849
1850 // Add type for inalloca argument.
1851 if (IRFunctionArgs.hasInallocaArg())
1852 ArgTypes[IRFunctionArgs.getInallocaArgNo()] =
1853 llvm::PointerType::getUnqual(C&: getLLVMContext());
1854
1855 // Add in all of the required arguments.
1856 unsigned ArgNo = 0;
1857 CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1858 ie = it + FI.getNumRequiredArgs();
1859 for (; it != ie; ++it, ++ArgNo) {
1860 const ABIArgInfo &ArgInfo = it->info;
1861
1862 // Insert a padding type to ensure proper alignment.
1863 if (IRFunctionArgs.hasPaddingArg(ArgNo))
1864 ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1865 ArgInfo.getPaddingType();
1866
1867 unsigned FirstIRArg, NumIRArgs;
1868 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1869
1870 switch (ArgInfo.getKind()) {
1871 case ABIArgInfo::Ignore:
1872 case ABIArgInfo::InAlloca:
1873 assert(NumIRArgs == 0);
1874 break;
1875
1876 case ABIArgInfo::Indirect:
1877 assert(NumIRArgs == 1);
1878 // indirect arguments are always on the stack, which is alloca addr space.
1879 ArgTypes[FirstIRArg] = llvm::PointerType::get(
1880 C&: getLLVMContext(), AddressSpace: CGM.getDataLayout().getAllocaAddrSpace());
1881 break;
1882 case ABIArgInfo::IndirectAliased:
1883 assert(NumIRArgs == 1);
1884 ArgTypes[FirstIRArg] = llvm::PointerType::get(
1885 C&: getLLVMContext(), AddressSpace: ArgInfo.getIndirectAddrSpace());
1886 break;
1887 case ABIArgInfo::TargetSpecific:
1888 case ABIArgInfo::Extend:
1889 case ABIArgInfo::Direct: {
1890 // Fast-isel and the optimizer generally like scalar values better than
1891 // FCAs, so we flatten them if this is safe to do for this argument.
1892 llvm::Type *argType = ArgInfo.getCoerceToType();
1893 llvm::StructType *st = dyn_cast<llvm::StructType>(Val: argType);
1894 if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
1895 assert(NumIRArgs == st->getNumElements());
1896 for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
1897 ArgTypes[FirstIRArg + i] = st->getElementType(N: i);
1898 } else {
1899 assert(NumIRArgs == 1);
1900 ArgTypes[FirstIRArg] = argType;
1901 }
1902 break;
1903 }
1904
1905 case ABIArgInfo::CoerceAndExpand: {
1906 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1907 for (auto *EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1908 *ArgTypesIter++ = EltTy;
1909 }
1910 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1911 break;
1912 }
1913
1914 case ABIArgInfo::Expand:
1915 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1916 getExpandedTypes(Ty: it->type, TI&: ArgTypesIter);
1917 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1918 break;
1919 }
1920 }
1921
1922 bool Erased = FunctionsBeingProcessed.erase(Ptr: &FI);
1923 (void)Erased;
1924 assert(Erased && "Not in set?");
1925
1926 return llvm::FunctionType::get(Result: resultType, Params: ArgTypes, isVarArg: FI.isVariadic());
1927}
1928
1929llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1930 const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
1931 const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
1932
1933 if (!isFuncTypeConvertible(FT: FPT))
1934 return llvm::StructType::get(Context&: getLLVMContext());
1935
1936 return GetFunctionType(GD);
1937}
1938
1939static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
1940 llvm::AttrBuilder &FuncAttrs,
1941 const FunctionProtoType *FPT) {
1942 if (!FPT)
1943 return;
1944
1945 if (!isUnresolvedExceptionSpec(ESpecType: FPT->getExceptionSpecType()) &&
1946 FPT->isNothrow())
1947 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
1948
1949 unsigned SMEBits = FPT->getAArch64SMEAttributes();
1950 if (SMEBits & FunctionType::SME_PStateSMEnabledMask)
1951 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_enabled");
1952 if (SMEBits & FunctionType::SME_PStateSMCompatibleMask)
1953 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_compatible");
1954 if (SMEBits & FunctionType::SME_AgnosticZAStateMask)
1955 FuncAttrs.addAttribute(A: "aarch64_za_state_agnostic");
1956
1957 // ZA
1958 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
1959 FuncAttrs.addAttribute(A: "aarch64_preserves_za");
1960 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_In)
1961 FuncAttrs.addAttribute(A: "aarch64_in_za");
1962 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Out)
1963 FuncAttrs.addAttribute(A: "aarch64_out_za");
1964 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_InOut)
1965 FuncAttrs.addAttribute(A: "aarch64_inout_za");
1966
1967 // ZT0
1968 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
1969 FuncAttrs.addAttribute(A: "aarch64_preserves_zt0");
1970 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_In)
1971 FuncAttrs.addAttribute(A: "aarch64_in_zt0");
1972 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Out)
1973 FuncAttrs.addAttribute(A: "aarch64_out_zt0");
1974 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_InOut)
1975 FuncAttrs.addAttribute(A: "aarch64_inout_zt0");
1976}
1977
1978static void AddAttributesFromOMPAssumes(llvm::AttrBuilder &FuncAttrs,
1979 const Decl *Callee) {
1980 if (!Callee)
1981 return;
1982
1983 SmallVector<StringRef, 4> Attrs;
1984
1985 for (const OMPAssumeAttr *AA : Callee->specific_attrs<OMPAssumeAttr>())
1986 AA->getAssumption().split(A&: Attrs, Separator: ",");
1987
1988 if (!Attrs.empty())
1989 FuncAttrs.addAttribute(A: llvm::AssumptionAttrKey,
1990 V: llvm::join(Begin: Attrs.begin(), End: Attrs.end(), Separator: ","));
1991}
1992
1993bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,
1994 QualType ReturnType) const {
1995 // We can't just discard the return value for a record type with a
1996 // complex destructor or a non-trivially copyable type.
1997 if (const RecordType *RT =
1998 ReturnType.getCanonicalType()->getAsCanonical<RecordType>()) {
1999 if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(Val: RT->getDecl()))
2000 return ClassDecl->hasTrivialDestructor();
2001 }
2002 return ReturnType.isTriviallyCopyableType(Context);
2003}
2004
2005static bool HasStrictReturn(const CodeGenModule &Module, QualType RetTy,
2006 const Decl *TargetDecl) {
2007 // As-is msan can not tolerate noundef mismatch between caller and
2008 // implementation. Mismatch is possible for e.g. indirect calls from C-caller
2009 // into C++. Such mismatches lead to confusing false reports. To avoid
2010 // expensive workaround on msan we enforce initialization event in uncommon
2011 // cases where it's allowed.
2012 if (Module.getLangOpts().Sanitize.has(K: SanitizerKind::Memory))
2013 return true;
2014 // C++ explicitly makes returning undefined values UB. C's rule only applies
2015 // to used values, so we never mark them noundef for now.
2016 if (!Module.getLangOpts().CPlusPlus)
2017 return false;
2018 if (TargetDecl) {
2019 if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2020 if (FDecl->isExternC())
2021 return false;
2022 } else if (const VarDecl *VDecl = dyn_cast<VarDecl>(Val: TargetDecl)) {
2023 // Function pointer.
2024 if (VDecl->isExternC())
2025 return false;
2026 }
2027 }
2028
2029 // We don't want to be too aggressive with the return checking, unless
2030 // it's explicit in the code opts or we're using an appropriate sanitizer.
2031 // Try to respect what the programmer intended.
2032 return Module.getCodeGenOpts().StrictReturn ||
2033 !Module.MayDropFunctionReturn(Context: Module.getContext(), ReturnType: RetTy) ||
2034 Module.getLangOpts().Sanitize.has(K: SanitizerKind::Return);
2035}
2036
2037/// Add denormal-fp-math and denormal-fp-math-f32 as appropriate for the
2038/// requested denormal behavior, accounting for the overriding behavior of the
2039/// -f32 case.
2040static void addDenormalModeAttrs(llvm::DenormalMode FPDenormalMode,
2041 llvm::DenormalMode FP32DenormalMode,
2042 llvm::AttrBuilder &FuncAttrs) {
2043 llvm::DenormalFPEnv FPEnv(FPDenormalMode, FP32DenormalMode);
2044 if (FPEnv != llvm::DenormalFPEnv::getDefault())
2045 FuncAttrs.addDenormalFPEnvAttr(Mode: FPEnv);
2046}
2047
2048/// Add default attributes to a function, which have merge semantics under
2049/// -mlink-builtin-bitcode and should not simply overwrite any existing
2050/// attributes in the linked library.
2051static void
2052addMergableDefaultFunctionAttributes(const CodeGenOptions &CodeGenOpts,
2053 llvm::AttrBuilder &FuncAttrs) {
2054 addDenormalModeAttrs(FPDenormalMode: CodeGenOpts.FPDenormalMode, FP32DenormalMode: CodeGenOpts.FP32DenormalMode,
2055 FuncAttrs);
2056}
2057
2058static void getTrivialDefaultFunctionAttributes(
2059 StringRef Name, bool HasOptnone, const CodeGenOptions &CodeGenOpts,
2060 const LangOptions &LangOpts, bool AttrOnCallSite,
2061 llvm::AttrBuilder &FuncAttrs) {
2062 // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
2063 if (!HasOptnone) {
2064 if (CodeGenOpts.OptimizeSize)
2065 FuncAttrs.addAttribute(Val: llvm::Attribute::OptimizeForSize);
2066 if (CodeGenOpts.OptimizeSize == 2)
2067 FuncAttrs.addAttribute(Val: llvm::Attribute::MinSize);
2068 }
2069
2070 if (CodeGenOpts.DisableRedZone)
2071 FuncAttrs.addAttribute(Val: llvm::Attribute::NoRedZone);
2072 if (CodeGenOpts.IndirectTlsSegRefs)
2073 FuncAttrs.addAttribute(A: "indirect-tls-seg-refs");
2074 if (CodeGenOpts.NoImplicitFloat)
2075 FuncAttrs.addAttribute(Val: llvm::Attribute::NoImplicitFloat);
2076
2077 if (AttrOnCallSite) {
2078 // Attributes that should go on the call site only.
2079 // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
2080 // the -fno-builtin-foo list.
2081 if (!CodeGenOpts.SimplifyLibCalls || LangOpts.isNoBuiltinFunc(Name))
2082 FuncAttrs.addAttribute(Val: llvm::Attribute::NoBuiltin);
2083 if (!CodeGenOpts.TrapFuncName.empty())
2084 FuncAttrs.addAttribute(A: "trap-func-name", V: CodeGenOpts.TrapFuncName);
2085 } else {
2086 switch (CodeGenOpts.getFramePointer()) {
2087 case CodeGenOptions::FramePointerKind::None:
2088 // This is the default behavior.
2089 break;
2090 case CodeGenOptions::FramePointerKind::Reserved:
2091 case CodeGenOptions::FramePointerKind::NonLeafNoReserve:
2092 case CodeGenOptions::FramePointerKind::NonLeaf:
2093 case CodeGenOptions::FramePointerKind::All:
2094 FuncAttrs.addAttribute(A: "frame-pointer",
2095 V: CodeGenOptions::getFramePointerKindName(
2096 Kind: CodeGenOpts.getFramePointer()));
2097 }
2098
2099 if (CodeGenOpts.LessPreciseFPMAD)
2100 FuncAttrs.addAttribute(A: "less-precise-fpmad", V: "true");
2101
2102 if (CodeGenOpts.NullPointerIsValid)
2103 FuncAttrs.addAttribute(Val: llvm::Attribute::NullPointerIsValid);
2104
2105 if (LangOpts.getDefaultExceptionMode() == LangOptions::FPE_Ignore)
2106 FuncAttrs.addAttribute(A: "no-trapping-math", V: "true");
2107
2108 // TODO: Are these all needed?
2109 // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
2110 if (LangOpts.NoHonorNaNs)
2111 FuncAttrs.addAttribute(A: "no-nans-fp-math", V: "true");
2112 if (CodeGenOpts.SoftFloat)
2113 FuncAttrs.addAttribute(A: "use-soft-float", V: "true");
2114 FuncAttrs.addAttribute(A: "stack-protector-buffer-size",
2115 V: llvm::utostr(X: CodeGenOpts.SSPBufferSize));
2116 if (LangOpts.NoSignedZero)
2117 FuncAttrs.addAttribute(A: "no-signed-zeros-fp-math", V: "true");
2118
2119 // TODO: Reciprocal estimate codegen options should apply to instructions?
2120 const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
2121 if (!Recips.empty())
2122 FuncAttrs.addAttribute(A: "reciprocal-estimates", V: llvm::join(R: Recips, Separator: ","));
2123
2124 if (!CodeGenOpts.PreferVectorWidth.empty() &&
2125 CodeGenOpts.PreferVectorWidth != "none")
2126 FuncAttrs.addAttribute(A: "prefer-vector-width",
2127 V: CodeGenOpts.PreferVectorWidth);
2128
2129 if (CodeGenOpts.StackRealignment)
2130 FuncAttrs.addAttribute(A: "stackrealign");
2131 if (CodeGenOpts.Backchain)
2132 FuncAttrs.addAttribute(A: "backchain");
2133 if (CodeGenOpts.EnableSegmentedStacks)
2134 FuncAttrs.addAttribute(A: "split-stack");
2135
2136 if (CodeGenOpts.SpeculativeLoadHardening)
2137 FuncAttrs.addAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2138
2139 // Add zero-call-used-regs attribute.
2140 switch (CodeGenOpts.getZeroCallUsedRegs()) {
2141 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
2142 FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2143 break;
2144 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
2145 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr-arg");
2146 break;
2147 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
2148 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr");
2149 break;
2150 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
2151 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-arg");
2152 break;
2153 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
2154 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used");
2155 break;
2156 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
2157 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr-arg");
2158 break;
2159 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
2160 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr");
2161 break;
2162 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
2163 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-arg");
2164 break;
2165 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
2166 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all");
2167 break;
2168 }
2169 }
2170
2171 if (LangOpts.assumeFunctionsAreConvergent()) {
2172 // Conservatively, mark all functions and calls in CUDA and OpenCL as
2173 // convergent (meaning, they may call an intrinsically convergent op, such
2174 // as __syncthreads() / barrier(), and so can't have certain optimizations
2175 // applied around them). LLVM will remove this attribute where it safely
2176 // can.
2177 FuncAttrs.addAttribute(Val: llvm::Attribute::Convergent);
2178 }
2179
2180 // TODO: NoUnwind attribute should be added for other GPU modes HIP,
2181 // OpenMP offload. AFAIK, neither of them support exceptions in device code.
2182 if ((LangOpts.CUDA && LangOpts.CUDAIsDevice) || LangOpts.OpenCL ||
2183 LangOpts.SYCLIsDevice) {
2184 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2185 }
2186
2187 if (CodeGenOpts.SaveRegParams && !AttrOnCallSite)
2188 FuncAttrs.addAttribute(A: "save-reg-params");
2189
2190 for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
2191 StringRef Var, Value;
2192 std::tie(args&: Var, args&: Value) = Attr.split(Separator: '=');
2193 FuncAttrs.addAttribute(A: Var, V: Value);
2194 }
2195
2196 TargetInfo::BranchProtectionInfo BPI(LangOpts);
2197 TargetCodeGenInfo::initBranchProtectionFnAttributes(BPI, FuncAttrs);
2198}
2199
2200/// Merges `target-features` from \TargetOpts and \F, and sets the result in
2201/// \FuncAttr
2202/// * features from \F are always kept
2203/// * a feature from \TargetOpts is kept if itself and its opposite are absent
2204/// from \F
2205static void
2206overrideFunctionFeaturesWithTargetFeatures(llvm::AttrBuilder &FuncAttr,
2207 const llvm::Function &F,
2208 const TargetOptions &TargetOpts) {
2209 auto FFeatures = F.getFnAttribute(Kind: "target-features");
2210
2211 llvm::StringSet<> MergedNames;
2212 SmallVector<StringRef> MergedFeatures;
2213 MergedFeatures.reserve(N: TargetOpts.Features.size());
2214
2215 auto AddUnmergedFeatures = [&](auto &&FeatureRange) {
2216 for (StringRef Feature : FeatureRange) {
2217 if (Feature.empty())
2218 continue;
2219 assert(Feature[0] == '+' || Feature[0] == '-');
2220 StringRef Name = Feature.drop_front(N: 1);
2221 bool Merged = !MergedNames.insert(key: Name).second;
2222 if (!Merged)
2223 MergedFeatures.push_back(Elt: Feature);
2224 }
2225 };
2226
2227 if (FFeatures.isValid())
2228 AddUnmergedFeatures(llvm::split(Str: FFeatures.getValueAsString(), Separator: ','));
2229 AddUnmergedFeatures(TargetOpts.Features);
2230
2231 if (!MergedFeatures.empty()) {
2232 llvm::sort(C&: MergedFeatures);
2233 FuncAttr.addAttribute(A: "target-features", V: llvm::join(R&: MergedFeatures, Separator: ","));
2234 }
2235}
2236
2237void CodeGen::mergeDefaultFunctionDefinitionAttributes(
2238 llvm::Function &F, const CodeGenOptions &CodeGenOpts,
2239 const LangOptions &LangOpts, const TargetOptions &TargetOpts,
2240 bool WillInternalize) {
2241
2242 llvm::AttrBuilder FuncAttrs(F.getContext());
2243 // Here we only extract the options that are relevant compared to the version
2244 // from GetCPUAndFeaturesAttributes.
2245 if (!TargetOpts.CPU.empty())
2246 FuncAttrs.addAttribute(A: "target-cpu", V: TargetOpts.CPU);
2247 if (!TargetOpts.TuneCPU.empty())
2248 FuncAttrs.addAttribute(A: "tune-cpu", V: TargetOpts.TuneCPU);
2249
2250 ::getTrivialDefaultFunctionAttributes(Name: F.getName(), HasOptnone: F.hasOptNone(),
2251 CodeGenOpts, LangOpts,
2252 /*AttrOnCallSite=*/false, FuncAttrs);
2253
2254 if (!WillInternalize && F.isInterposable()) {
2255 // Do not promote "dynamic" denormal-fp-math to this translation unit's
2256 // setting for weak functions that won't be internalized. The user has no
2257 // real control for how builtin bitcode is linked, so we shouldn't assume
2258 // later copies will use a consistent mode.
2259 F.addFnAttrs(Attrs: FuncAttrs);
2260 return;
2261 }
2262
2263 llvm::AttributeMask AttrsToRemove;
2264
2265 llvm::DenormalFPEnv OptsFPEnv(CodeGenOpts.FPDenormalMode,
2266 CodeGenOpts.FP32DenormalMode);
2267 llvm::DenormalFPEnv MergedFPEnv =
2268 OptsFPEnv.mergeCalleeMode(Callee: F.getDenormalFPEnv());
2269
2270 if (MergedFPEnv == llvm::DenormalFPEnv::getDefault()) {
2271 AttrsToRemove.addAttribute(Val: llvm::Attribute::DenormalFPEnv);
2272 } else {
2273 // Overwrite existing attribute
2274 FuncAttrs.addDenormalFPEnvAttr(Mode: MergedFPEnv);
2275 }
2276
2277 F.removeFnAttrs(Attrs: AttrsToRemove);
2278
2279 overrideFunctionFeaturesWithTargetFeatures(FuncAttr&: FuncAttrs, F, TargetOpts);
2280
2281 F.addFnAttrs(Attrs: FuncAttrs);
2282}
2283
2284void CodeGenModule::getTrivialDefaultFunctionAttributes(
2285 StringRef Name, bool HasOptnone, bool AttrOnCallSite,
2286 llvm::AttrBuilder &FuncAttrs) {
2287 ::getTrivialDefaultFunctionAttributes(Name, HasOptnone, CodeGenOpts: getCodeGenOpts(),
2288 LangOpts: getLangOpts(), AttrOnCallSite,
2289 FuncAttrs);
2290}
2291
2292void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
2293 bool HasOptnone,
2294 bool AttrOnCallSite,
2295 llvm::AttrBuilder &FuncAttrs) {
2296 getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,
2297 FuncAttrs);
2298
2299 if (!AttrOnCallSite)
2300 TargetCodeGenInfo::initPointerAuthFnAttributes(Opts: CodeGenOpts.PointerAuth,
2301 FuncAttrs);
2302
2303 // If we're just getting the default, get the default values for mergeable
2304 // attributes.
2305 if (!AttrOnCallSite)
2306 addMergableDefaultFunctionAttributes(CodeGenOpts, FuncAttrs);
2307}
2308
2309void CodeGenModule::addDefaultFunctionDefinitionAttributes(
2310 llvm::AttrBuilder &attrs) {
2311 getDefaultFunctionAttributes(/*function name*/ Name: "", /*optnone*/ HasOptnone: false,
2312 /*for call*/ AttrOnCallSite: false, FuncAttrs&: attrs);
2313 GetCPUAndFeaturesAttributes(GD: GlobalDecl(), AttrBuilder&: attrs);
2314}
2315
2316static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
2317 const LangOptions &LangOpts,
2318 const NoBuiltinAttr *NBA = nullptr) {
2319 auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
2320 SmallString<32> AttributeName;
2321 AttributeName += "no-builtin-";
2322 AttributeName += BuiltinName;
2323 FuncAttrs.addAttribute(A: AttributeName);
2324 };
2325
2326 // First, handle the language options passed through -fno-builtin.
2327 if (LangOpts.NoBuiltin) {
2328 // -fno-builtin disables them all.
2329 FuncAttrs.addAttribute(A: "no-builtins");
2330 return;
2331 }
2332
2333 // Then, add attributes for builtins specified through -fno-builtin-<name>.
2334 llvm::for_each(Range: LangOpts.NoBuiltinFuncs, F: AddNoBuiltinAttr);
2335
2336 // Now, let's check the __attribute__((no_builtin("...")) attribute added to
2337 // the source.
2338 if (!NBA)
2339 return;
2340
2341 // If there is a wildcard in the builtin names specified through the
2342 // attribute, disable them all.
2343 if (llvm::is_contained(Range: NBA->builtinNames(), Element: "*")) {
2344 FuncAttrs.addAttribute(A: "no-builtins");
2345 return;
2346 }
2347
2348 // And last, add the rest of the builtin names.
2349 llvm::for_each(Range: NBA->builtinNames(), F: AddNoBuiltinAttr);
2350}
2351
2352static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
2353 const llvm::DataLayout &DL, const ABIArgInfo &AI,
2354 bool CheckCoerce = true) {
2355 llvm::Type *Ty = Types.ConvertTypeForMem(T: QTy);
2356 if (AI.getKind() == ABIArgInfo::Indirect ||
2357 AI.getKind() == ABIArgInfo::IndirectAliased)
2358 return true;
2359 if (AI.getKind() == ABIArgInfo::Extend && !AI.isNoExt())
2360 return true;
2361 if (!DL.typeSizeEqualsStoreSize(Ty))
2362 // TODO: This will result in a modest amount of values not marked noundef
2363 // when they could be. We care about values that *invisibly* contain undef
2364 // bits from the perspective of LLVM IR.
2365 return false;
2366 if (CheckCoerce && AI.canHaveCoerceToType()) {
2367 llvm::Type *CoerceTy = AI.getCoerceToType();
2368 if (llvm::TypeSize::isKnownGT(LHS: DL.getTypeSizeInBits(Ty: CoerceTy),
2369 RHS: DL.getTypeSizeInBits(Ty)))
2370 // If we're coercing to a type with a greater size than the canonical one,
2371 // we're introducing new undef bits.
2372 // Coercing to a type of smaller or equal size is ok, as we know that
2373 // there's no internal padding (typeSizeEqualsStoreSize).
2374 return false;
2375 }
2376 if (QTy->isBitIntType())
2377 return true;
2378 if (QTy->isReferenceType())
2379 return true;
2380 if (QTy->isNullPtrType())
2381 return false;
2382 if (QTy->isMemberPointerType())
2383 // TODO: Some member pointers are `noundef`, but it depends on the ABI. For
2384 // now, never mark them.
2385 return false;
2386 if (QTy->isScalarType()) {
2387 if (const ComplexType *Complex = dyn_cast<ComplexType>(Val&: QTy))
2388 return DetermineNoUndef(QTy: Complex->getElementType(), Types, DL, AI, CheckCoerce: false);
2389 return true;
2390 }
2391 if (const VectorType *Vector = dyn_cast<VectorType>(Val&: QTy))
2392 return DetermineNoUndef(QTy: Vector->getElementType(), Types, DL, AI, CheckCoerce: false);
2393 if (const MatrixType *Matrix = dyn_cast<MatrixType>(Val&: QTy))
2394 return DetermineNoUndef(QTy: Matrix->getElementType(), Types, DL, AI, CheckCoerce: false);
2395 if (const ArrayType *Array = dyn_cast<ArrayType>(Val&: QTy))
2396 return DetermineNoUndef(QTy: Array->getElementType(), Types, DL, AI, CheckCoerce: false);
2397
2398 // TODO: Some structs may be `noundef`, in specific situations.
2399 return false;
2400}
2401
2402/// Check if the argument of a function has maybe_undef attribute.
2403static bool IsArgumentMaybeUndef(const Decl *TargetDecl,
2404 unsigned NumRequiredArgs, unsigned ArgNo) {
2405 const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
2406 if (!FD)
2407 return false;
2408
2409 // Assume variadic arguments do not have maybe_undef attribute.
2410 if (ArgNo >= NumRequiredArgs)
2411 return false;
2412
2413 // Check if argument has maybe_undef attribute.
2414 if (ArgNo < FD->getNumParams()) {
2415 const ParmVarDecl *Param = FD->getParamDecl(i: ArgNo);
2416 if (Param && Param->hasAttr<MaybeUndefAttr>())
2417 return true;
2418 }
2419
2420 return false;
2421}
2422
2423/// Test if it's legal to apply nofpclass for the given parameter type and it's
2424/// lowered IR type.
2425static bool canApplyNoFPClass(const ABIArgInfo &AI, QualType ParamType,
2426 bool IsReturn) {
2427 // Should only apply to FP types in the source, not ABI promoted.
2428 if (!ParamType->hasFloatingRepresentation())
2429 return false;
2430
2431 // The promoted-to IR type also needs to support nofpclass.
2432 llvm::Type *IRTy = AI.getCoerceToType();
2433 if (llvm::AttributeFuncs::isNoFPClassCompatibleType(Ty: IRTy))
2434 return true;
2435
2436 if (llvm::StructType *ST = dyn_cast<llvm::StructType>(Val: IRTy)) {
2437 return !IsReturn && AI.getCanBeFlattened() &&
2438 llvm::all_of(Range: ST->elements(),
2439 P: llvm::AttributeFuncs::isNoFPClassCompatibleType);
2440 }
2441
2442 return false;
2443}
2444
2445/// Return the nofpclass mask that can be applied to floating-point parameters.
2446static llvm::FPClassTest getNoFPClassTestMask(const LangOptions &LangOpts) {
2447 llvm::FPClassTest Mask = llvm::fcNone;
2448 if (LangOpts.NoHonorInfs)
2449 Mask |= llvm::fcInf;
2450 if (LangOpts.NoHonorNaNs)
2451 Mask |= llvm::fcNan;
2452 return Mask;
2453}
2454
2455void CodeGenModule::AdjustMemoryAttribute(StringRef Name,
2456 CGCalleeInfo CalleeInfo,
2457 llvm::AttributeList &Attrs) {
2458 if (Attrs.getMemoryEffects().getModRef() == llvm::ModRefInfo::NoModRef) {
2459 Attrs = Attrs.removeFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::Memory);
2460 llvm::Attribute MemoryAttr = llvm::Attribute::getWithMemoryEffects(
2461 Context&: getLLVMContext(), ME: llvm::MemoryEffects::writeOnly());
2462 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Attr: MemoryAttr);
2463 }
2464}
2465
2466/// Construct the IR attribute list of a function or call.
2467///
2468/// When adding an attribute, please consider where it should be handled:
2469///
2470/// - getDefaultFunctionAttributes is for attributes that are essentially
2471/// part of the global target configuration (but perhaps can be
2472/// overridden on a per-function basis). Adding attributes there
2473/// will cause them to also be set in frontends that build on Clang's
2474/// target-configuration logic, as well as for code defined in library
2475/// modules such as CUDA's libdevice.
2476///
2477/// - ConstructAttributeList builds on top of getDefaultFunctionAttributes
2478/// and adds declaration-specific, convention-specific, and
2479/// frontend-specific logic. The last is of particular importance:
2480/// attributes that restrict how the frontend generates code must be
2481/// added here rather than getDefaultFunctionAttributes.
2482///
2483void CodeGenModule::ConstructAttributeList(StringRef Name,
2484 const CGFunctionInfo &FI,
2485 CGCalleeInfo CalleeInfo,
2486 llvm::AttributeList &AttrList,
2487 unsigned &CallingConv,
2488 bool AttrOnCallSite, bool IsThunk) {
2489 llvm::AttrBuilder FuncAttrs(getLLVMContext());
2490 llvm::AttrBuilder RetAttrs(getLLVMContext());
2491
2492 // Collect function IR attributes from the CC lowering.
2493 // We'll collect the paramete and result attributes later.
2494 CallingConv = FI.getEffectiveCallingConvention();
2495 if (FI.isNoReturn())
2496 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2497 if (FI.isCmseNSCall())
2498 FuncAttrs.addAttribute(A: "cmse_nonsecure_call");
2499
2500 // Collect function IR attributes from the callee prototype if we have one.
2501 AddAttributesFromFunctionProtoType(Ctx&: getContext(), FuncAttrs,
2502 FPT: CalleeInfo.getCalleeFunctionProtoType());
2503 const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2504
2505 // Attach assumption attributes to the declaration. If this is a call
2506 // site, attach assumptions from the caller to the call as well.
2507 AddAttributesFromOMPAssumes(FuncAttrs, Callee: TargetDecl);
2508
2509 bool HasOptnone = false;
2510 // The NoBuiltinAttr attached to the target FunctionDecl.
2511 const NoBuiltinAttr *NBA = nullptr;
2512
2513 // Some ABIs may result in additional accesses to arguments that may
2514 // otherwise not be present.
2515 std::optional<llvm::Attribute::AttrKind> MemAttrForPtrArgs;
2516 bool AddedPotentialArgAccess = false;
2517 auto AddPotentialArgAccess = [&]() {
2518 AddedPotentialArgAccess = true;
2519 llvm::Attribute A = FuncAttrs.getAttribute(Kind: llvm::Attribute::Memory);
2520 if (A.isValid())
2521 FuncAttrs.addMemoryAttr(ME: A.getMemoryEffects() |
2522 llvm::MemoryEffects::argMemOnly());
2523 };
2524
2525 // Collect function IR attributes based on declaration-specific
2526 // information.
2527 // FIXME: handle sseregparm someday...
2528 if (TargetDecl) {
2529 if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2530 FuncAttrs.addAttribute(Val: llvm::Attribute::ReturnsTwice);
2531 if (TargetDecl->hasAttr<NoThrowAttr>())
2532 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2533 if (TargetDecl->hasAttr<NoReturnAttr>())
2534 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2535 if (TargetDecl->hasAttr<ColdAttr>())
2536 FuncAttrs.addAttribute(Val: llvm::Attribute::Cold);
2537 if (TargetDecl->hasAttr<HotAttr>())
2538 FuncAttrs.addAttribute(Val: llvm::Attribute::Hot);
2539 if (TargetDecl->hasAttr<NoDuplicateAttr>())
2540 FuncAttrs.addAttribute(Val: llvm::Attribute::NoDuplicate);
2541 if (TargetDecl->hasAttr<ConvergentAttr>())
2542 FuncAttrs.addAttribute(Val: llvm::Attribute::Convergent);
2543
2544 if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2545 AddAttributesFromFunctionProtoType(
2546 Ctx&: getContext(), FuncAttrs, FPT: Fn->getType()->getAs<FunctionProtoType>());
2547 if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
2548 // A sane operator new returns a non-aliasing pointer.
2549 auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2550 if (getCodeGenOpts().AssumeSaneOperatorNew &&
2551 (Kind == OO_New || Kind == OO_Array_New))
2552 RetAttrs.addAttribute(Val: llvm::Attribute::NoAlias);
2553 }
2554 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: Fn);
2555 const bool IsVirtualCall = MD && MD->isVirtual();
2556 // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2557 // virtual function. These attributes are not inherited by overloads.
2558 if (!(AttrOnCallSite && IsVirtualCall)) {
2559 if (Fn->isNoReturn())
2560 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2561 NBA = Fn->getAttr<NoBuiltinAttr>();
2562 }
2563 }
2564
2565 if (isa<FunctionDecl>(Val: TargetDecl) || isa<VarDecl>(Val: TargetDecl)) {
2566 // Only place nomerge attribute on call sites, never functions. This
2567 // allows it to work on indirect virtual function calls.
2568 if (AttrOnCallSite && TargetDecl->hasAttr<NoMergeAttr>())
2569 FuncAttrs.addAttribute(Val: llvm::Attribute::NoMerge);
2570 }
2571
2572 // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2573 if (TargetDecl->hasAttr<ConstAttr>()) {
2574 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::none());
2575 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2576 // gcc specifies that 'const' functions have greater restrictions than
2577 // 'pure' functions, so they also cannot have infinite loops.
2578 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
2579 MemAttrForPtrArgs = llvm::Attribute::ReadNone;
2580 } else if (TargetDecl->hasAttr<PureAttr>()) {
2581 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::readOnly());
2582 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2583 // gcc specifies that 'pure' functions cannot have infinite loops.
2584 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
2585 MemAttrForPtrArgs = llvm::Attribute::ReadOnly;
2586 } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2587 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::inaccessibleOrArgMemOnly());
2588 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2589 }
2590 if (const auto *RA = TargetDecl->getAttr<RestrictAttr>();
2591 RA && RA->getDeallocator() == nullptr)
2592 RetAttrs.addAttribute(Val: llvm::Attribute::NoAlias);
2593 if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2594 !CodeGenOpts.NullPointerIsValid)
2595 RetAttrs.addAttribute(Val: llvm::Attribute::NonNull);
2596 if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2597 FuncAttrs.addAttribute(A: "no_caller_saved_registers");
2598 if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2599 FuncAttrs.addAttribute(Val: llvm::Attribute::NoCfCheck);
2600 if (TargetDecl->hasAttr<LeafAttr>())
2601 FuncAttrs.addAttribute(Val: llvm::Attribute::NoCallback);
2602 if (TargetDecl->hasAttr<BPFFastCallAttr>())
2603 FuncAttrs.addAttribute(A: "bpf_fastcall");
2604
2605 HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2606 if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2607 std::optional<unsigned> NumElemsParam;
2608 if (AllocSize->getNumElemsParam().isValid())
2609 NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2610 FuncAttrs.addAllocSizeAttr(ElemSizeArg: AllocSize->getElemSizeParam().getLLVMIndex(),
2611 NumElemsArg: NumElemsParam);
2612 }
2613
2614 if (DeviceKernelAttr::isOpenCLSpelling(
2615 A: TargetDecl->getAttr<DeviceKernelAttr>()) &&
2616 CallingConv != CallingConv::CC_C &&
2617 CallingConv != CallingConv::CC_SpirFunction) {
2618 // Check CallingConv to avoid adding uniform-work-group-size attribute to
2619 // OpenCL Kernel Stub
2620 if (getLangOpts().OpenCLVersion <= 120) {
2621 // OpenCL v1.2 Work groups are always uniform
2622 FuncAttrs.addAttribute(A: "uniform-work-group-size", V: "true");
2623 } else {
2624 // OpenCL v2.0 Work groups may be whether uniform or not.
2625 // '-cl-uniform-work-group-size' compile option gets a hint
2626 // to the compiler that the global work-size be a multiple of
2627 // the work-group size specified to clEnqueueNDRangeKernel
2628 // (i.e. work groups are uniform).
2629 FuncAttrs.addAttribute(
2630 A: "uniform-work-group-size",
2631 V: llvm::toStringRef(B: getLangOpts().OffloadUniformBlock));
2632 }
2633 }
2634
2635 if (TargetDecl->hasAttr<CUDAGlobalAttr>() &&
2636 getLangOpts().OffloadUniformBlock)
2637 FuncAttrs.addAttribute(A: "uniform-work-group-size", V: "true");
2638
2639 if (TargetDecl->hasAttr<ArmLocallyStreamingAttr>())
2640 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_body");
2641
2642 if (auto *ModularFormat = TargetDecl->getAttr<ModularFormatAttr>()) {
2643 FormatAttr *Format = TargetDecl->getAttr<FormatAttr>();
2644 StringRef Type = Format->getType()->getName();
2645 std::string FormatIdx = std::to_string(val: Format->getFormatIdx());
2646 std::string FirstArg = std::to_string(val: Format->getFirstArg());
2647 SmallVector<StringRef> Args = {
2648 Type, FormatIdx, FirstArg,
2649 ModularFormat->getModularImplFn()->getName(),
2650 ModularFormat->getImplName()};
2651 llvm::append_range(C&: Args, R: ModularFormat->aspects());
2652 FuncAttrs.addAttribute(A: "modular-format", V: llvm::join(R&: Args, Separator: ","));
2653 }
2654 }
2655
2656 // Attach "no-builtins" attributes to:
2657 // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2658 // * definitions: "no-builtins" or "no-builtin-<name>" only.
2659 // The attributes can come from:
2660 // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2661 // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2662 addNoBuiltinAttributes(FuncAttrs, LangOpts: getLangOpts(), NBA);
2663
2664 // Collect function IR attributes based on global settiings.
2665 getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2666
2667 // Override some default IR attributes based on declaration-specific
2668 // information.
2669 if (TargetDecl) {
2670 if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2671 FuncAttrs.removeAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2672 if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2673 FuncAttrs.addAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2674 if (TargetDecl->hasAttr<NoSplitStackAttr>())
2675 FuncAttrs.removeAttribute(A: "split-stack");
2676 if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
2677 // A function "__attribute__((...))" overrides the command-line flag.
2678 auto Kind =
2679 TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2680 FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2681 FuncAttrs.addAttribute(
2682 A: "zero-call-used-regs",
2683 V: ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Val: Kind));
2684 }
2685
2686 // Add NonLazyBind attribute to function declarations when -fno-plt
2687 // is used.
2688 // FIXME: what if we just haven't processed the function definition
2689 // yet, or if it's an external definition like C99 inline?
2690 if (CodeGenOpts.NoPLT) {
2691 if (auto *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2692 if (!Fn->isDefined() && !AttrOnCallSite) {
2693 FuncAttrs.addAttribute(Val: llvm::Attribute::NonLazyBind);
2694 }
2695 }
2696 }
2697 // Remove 'convergent' if requested.
2698 if (TargetDecl->hasAttr<NoConvergentAttr>())
2699 FuncAttrs.removeAttribute(Val: llvm::Attribute::Convergent);
2700 }
2701
2702 // Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2703 // functions with -funique-internal-linkage-names.
2704 if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames) {
2705 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
2706 if (!FD->isExternallyVisible())
2707 FuncAttrs.addAttribute(A: "sample-profile-suffix-elision-policy",
2708 V: "selected");
2709 }
2710 }
2711
2712 // Collect non-call-site function IR attributes from declaration-specific
2713 // information.
2714 if (!AttrOnCallSite) {
2715 if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>())
2716 FuncAttrs.addAttribute(A: "cmse_nonsecure_entry");
2717
2718 // Whether tail calls are enabled.
2719 auto shouldDisableTailCalls = [&] {
2720 // Should this be honored in getDefaultFunctionAttributes?
2721 if (CodeGenOpts.DisableTailCalls)
2722 return true;
2723
2724 if (!TargetDecl)
2725 return false;
2726
2727 if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2728 TargetDecl->hasAttr<AnyX86InterruptAttr>())
2729 return true;
2730
2731 if (CodeGenOpts.NoEscapingBlockTailCalls) {
2732 if (const auto *BD = dyn_cast<BlockDecl>(Val: TargetDecl))
2733 if (!BD->doesNotEscape())
2734 return true;
2735 }
2736
2737 return false;
2738 };
2739 if (shouldDisableTailCalls())
2740 FuncAttrs.addAttribute(A: "disable-tail-calls", V: "true");
2741
2742 // These functions require the returns_twice attribute for correct codegen,
2743 // but the attribute may not be added if -fno-builtin is specified. We
2744 // explicitly add that attribute here.
2745 static const llvm::StringSet<> ReturnsTwiceFn{
2746 "_setjmpex", "setjmp", "_setjmp", "vfork",
2747 "sigsetjmp", "__sigsetjmp", "savectx", "getcontext"};
2748 if (ReturnsTwiceFn.contains(key: Name))
2749 FuncAttrs.addAttribute(Val: llvm::Attribute::ReturnsTwice);
2750
2751 // CPU/feature overrides. addDefaultFunctionDefinitionAttributes
2752 // handles these separately to set them based on the global defaults.
2753 GetCPUAndFeaturesAttributes(GD: CalleeInfo.getCalleeDecl(), AttrBuilder&: FuncAttrs);
2754
2755 // Windows hotpatching support
2756 if (!MSHotPatchFunctions.empty()) {
2757 bool IsHotPatched = llvm::binary_search(Range&: MSHotPatchFunctions, Value&: Name);
2758 if (IsHotPatched)
2759 FuncAttrs.addAttribute(A: "marked_for_windows_hot_patching");
2760 }
2761 }
2762
2763 // Mark functions that are replaceable by the loader.
2764 if (CodeGenOpts.isLoaderReplaceableFunctionName(FuncName: Name))
2765 FuncAttrs.addAttribute(A: "loader-replaceable");
2766
2767 // Collect attributes from arguments and return values.
2768 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2769
2770 QualType RetTy = FI.getReturnType();
2771 const ABIArgInfo &RetAI = FI.getReturnInfo();
2772 const llvm::DataLayout &DL = getDataLayout();
2773
2774 // Determine if the return type could be partially undef
2775 if (CodeGenOpts.EnableNoundefAttrs &&
2776 HasStrictReturn(Module: *this, RetTy, TargetDecl)) {
2777 if (!RetTy->isVoidType() && RetAI.getKind() != ABIArgInfo::Indirect &&
2778 DetermineNoUndef(QTy: RetTy, Types&: getTypes(), DL, AI: RetAI))
2779 RetAttrs.addAttribute(Val: llvm::Attribute::NoUndef);
2780 }
2781
2782 switch (RetAI.getKind()) {
2783 case ABIArgInfo::Extend:
2784 if (RetAI.isSignExt())
2785 RetAttrs.addAttribute(Val: llvm::Attribute::SExt);
2786 else if (RetAI.isZeroExt())
2787 RetAttrs.addAttribute(Val: llvm::Attribute::ZExt);
2788 else
2789 RetAttrs.addAttribute(Val: llvm::Attribute::NoExt);
2790 [[fallthrough]];
2791 case ABIArgInfo::TargetSpecific:
2792 case ABIArgInfo::Direct:
2793 if (RetAI.getInReg())
2794 RetAttrs.addAttribute(Val: llvm::Attribute::InReg);
2795
2796 if (canApplyNoFPClass(AI: RetAI, ParamType: RetTy, IsReturn: true))
2797 RetAttrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
2798
2799 break;
2800 case ABIArgInfo::Ignore:
2801 break;
2802
2803 case ABIArgInfo::InAlloca:
2804 case ABIArgInfo::Indirect: {
2805 // inalloca and sret disable readnone and readonly
2806 AddPotentialArgAccess();
2807 break;
2808 }
2809
2810 case ABIArgInfo::CoerceAndExpand:
2811 break;
2812
2813 case ABIArgInfo::Expand:
2814 case ABIArgInfo::IndirectAliased:
2815 llvm_unreachable("Invalid ABI kind for return argument");
2816 }
2817
2818 if (!IsThunk) {
2819 // FIXME: fix this properly, https://reviews.llvm.org/D100388
2820 if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2821 QualType PTy = RefTy->getPointeeType();
2822 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2823 RetAttrs.addDereferenceableAttr(
2824 Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
2825 if (getTypes().getTargetAddressSpace(T: PTy) == 0 &&
2826 !CodeGenOpts.NullPointerIsValid)
2827 RetAttrs.addAttribute(Val: llvm::Attribute::NonNull);
2828 if (PTy->isObjectType()) {
2829 llvm::Align Alignment =
2830 getNaturalPointeeTypeAlignment(T: RetTy).getAsAlign();
2831 RetAttrs.addAlignmentAttr(Align: Alignment);
2832 }
2833 }
2834 }
2835
2836 bool hasUsedSRet = false;
2837 SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
2838
2839 // Attach attributes to sret.
2840 if (IRFunctionArgs.hasSRetArg()) {
2841 llvm::AttrBuilder SRETAttrs(getLLVMContext());
2842 SRETAttrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: RetTy));
2843 SRETAttrs.addAttribute(Val: llvm::Attribute::Writable);
2844 SRETAttrs.addAttribute(Val: llvm::Attribute::DeadOnUnwind);
2845 hasUsedSRet = true;
2846 if (RetAI.getInReg())
2847 SRETAttrs.addAttribute(Val: llvm::Attribute::InReg);
2848 SRETAttrs.addAlignmentAttr(Align: RetAI.getIndirectAlign().getQuantity());
2849 ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2850 llvm::AttributeSet::get(C&: getLLVMContext(), B: SRETAttrs);
2851 }
2852
2853 // Attach attributes to inalloca argument.
2854 if (IRFunctionArgs.hasInallocaArg()) {
2855 llvm::AttrBuilder Attrs(getLLVMContext());
2856 Attrs.addInAllocaAttr(Ty: FI.getArgStruct());
2857 ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2858 llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs);
2859 }
2860
2861 // Apply `nonnull`, `dereferenceable(N)` and `align N` to the `this` argument,
2862 // unless this is a thunk function. Add dead_on_return to the `this` argument
2863 // in base class destructors to aid in DSE.
2864 // FIXME: fix this properly, https://reviews.llvm.org/D100388
2865 if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() &&
2866 !FI.arg_begin()->type->isVoidPointerType() && !IsThunk) {
2867 auto IRArgs = IRFunctionArgs.getIRArgs(ArgNo: 0);
2868
2869 assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
2870
2871 llvm::AttrBuilder Attrs(getLLVMContext());
2872
2873 QualType ThisTy = FI.arg_begin()->type.getTypePtr()->getPointeeType();
2874
2875 if (!CodeGenOpts.NullPointerIsValid &&
2876 getTypes().getTargetAddressSpace(T: FI.arg_begin()->type) == 0) {
2877 Attrs.addAttribute(Val: llvm::Attribute::NonNull);
2878 Attrs.addDereferenceableAttr(Bytes: getMinimumObjectSize(Ty: ThisTy).getQuantity());
2879 } else {
2880 // FIXME dereferenceable should be correct here, regardless of
2881 // NullPointerIsValid. However, dereferenceable currently does not always
2882 // respect NullPointerIsValid and may imply nonnull and break the program.
2883 // See https://reviews.llvm.org/D66618 for discussions.
2884 Attrs.addDereferenceableOrNullAttr(
2885 Bytes: getMinimumObjectSize(
2886 Ty: FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
2887 .getQuantity());
2888 }
2889
2890 llvm::Align Alignment =
2891 getNaturalTypeAlignment(T: ThisTy, /*BaseInfo=*/nullptr,
2892 /*TBAAInfo=*/nullptr, /*forPointeeType=*/true)
2893 .getAsAlign();
2894 Attrs.addAlignmentAttr(Align: Alignment);
2895
2896 const auto *DD = dyn_cast_if_present<CXXDestructorDecl>(
2897 Val: CalleeInfo.getCalleeDecl().getDecl());
2898 // Do not annotate vector deleting destructors with dead_on_return as the
2899 // this pointer in that case points to an array which we cannot
2900 // statically know the size of.
2901 if (DD &&
2902 CalleeInfo.getCalleeDecl().getDtorType() !=
2903 CXXDtorType::Dtor_VectorDeleting &&
2904 CodeGenOpts.StrictLifetimes) {
2905 const CXXRecordDecl *ClassDecl =
2906 dyn_cast<CXXRecordDecl>(Val: DD->getDeclContext());
2907 // TODO(boomanaiden154): We are being intentionally conservative here
2908 // as we gain experience with this optimization. We should remove the
2909 // condition for non-virtual bases after more testing. We cannot add
2910 // dead_on_return if we have virtual base classes because they will
2911 // generally still be live after the base object destructor.
2912 if (ClassDecl->getNumBases() == 0 && ClassDecl->getNumVBases() == 0)
2913 Attrs.addDeadOnReturnAttr(Info: llvm::DeadOnReturnInfo(
2914 Context.getASTRecordLayout(D: ClassDecl).getDataSize().getQuantity()));
2915 }
2916
2917 ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs);
2918 }
2919
2920 unsigned ArgNo = 0;
2921 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(), E = FI.arg_end();
2922 I != E; ++I, ++ArgNo) {
2923 QualType ParamType = I->type;
2924 const ABIArgInfo &AI = I->info;
2925 llvm::AttrBuilder Attrs(getLLVMContext());
2926
2927 // Add attribute for padding argument, if necessary.
2928 if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2929 if (AI.getPaddingInReg()) {
2930 ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2931 llvm::AttributeSet::get(C&: getLLVMContext(),
2932 B: llvm::AttrBuilder(getLLVMContext())
2933 .addAttribute(Val: llvm::Attribute::InReg));
2934 }
2935 }
2936
2937 // Decide whether the argument we're handling could be partially undef
2938 if (CodeGenOpts.EnableNoundefAttrs &&
2939 DetermineNoUndef(QTy: ParamType, Types&: getTypes(), DL, AI)) {
2940 Attrs.addAttribute(Val: llvm::Attribute::NoUndef);
2941 }
2942
2943 // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2944 // have the corresponding parameter variable. It doesn't make
2945 // sense to do it here because parameters are so messed up.
2946 switch (AI.getKind()) {
2947 case ABIArgInfo::Extend:
2948 if (AI.isSignExt())
2949 Attrs.addAttribute(Val: llvm::Attribute::SExt);
2950 else if (AI.isZeroExt())
2951 Attrs.addAttribute(Val: llvm::Attribute::ZExt);
2952 else
2953 Attrs.addAttribute(Val: llvm::Attribute::NoExt);
2954 [[fallthrough]];
2955 case ABIArgInfo::TargetSpecific:
2956 case ABIArgInfo::Direct:
2957 if (ArgNo == 0 && FI.isChainCall())
2958 Attrs.addAttribute(Val: llvm::Attribute::Nest);
2959 else if (AI.getInReg())
2960 Attrs.addAttribute(Val: llvm::Attribute::InReg);
2961 Attrs.addStackAlignmentAttr(Align: llvm::MaybeAlign(AI.getDirectAlign()));
2962
2963 if (canApplyNoFPClass(AI, ParamType, IsReturn: false))
2964 Attrs.addNoFPClassAttr(NoFPClassMask: getNoFPClassTestMask(LangOpts: getLangOpts()));
2965 break;
2966 case ABIArgInfo::Indirect: {
2967 if (AI.getInReg())
2968 Attrs.addAttribute(Val: llvm::Attribute::InReg);
2969
2970 // HLSL out and inout parameters must not be marked with ByVal or
2971 // DeadOnReturn attributes because stores to these parameters by the
2972 // callee are visible to the caller.
2973 if (auto ParamABI = FI.getExtParameterInfo(argIndex: ArgNo).getABI();
2974 ParamABI != ParameterABI::HLSLOut &&
2975 ParamABI != ParameterABI::HLSLInOut) {
2976
2977 // Depending on the ABI, this may be either a byval or a dead_on_return
2978 // argument.
2979 if (AI.getIndirectByVal()) {
2980 Attrs.addByValAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
2981 } else {
2982 // Add dead_on_return when the object's lifetime ends in the callee.
2983 // This includes trivially-destructible objects, as well as objects
2984 // whose destruction / clean-up is carried out within the callee
2985 // (e.g., Obj-C ARC-managed structs, MSVC callee-destroyed objects).
2986 if (!ParamType.isDestructedType() || !ParamType->isRecordType() ||
2987 ParamType->castAsRecordDecl()->isParamDestroyedInCallee())
2988 Attrs.addDeadOnReturnAttr(Info: llvm::DeadOnReturnInfo());
2989 }
2990 }
2991
2992 auto *Decl = ParamType->getAsRecordDecl();
2993 if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
2994 Decl->getArgPassingRestrictions() ==
2995 RecordArgPassingKind::CanPassInRegs)
2996 // When calling the function, the pointer passed in will be the only
2997 // reference to the underlying object. Mark it accordingly.
2998 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
2999
3000 // TODO: We could add the byref attribute if not byval, but it would
3001 // require updating many testcases.
3002
3003 CharUnits Align = AI.getIndirectAlign();
3004
3005 // In a byval argument, it is important that the required
3006 // alignment of the type is honored, as LLVM might be creating a
3007 // *new* stack object, and needs to know what alignment to give
3008 // it. (Sometimes it can deduce a sensible alignment on its own,
3009 // but not if clang decides it must emit a packed struct, or the
3010 // user specifies increased alignment requirements.)
3011 //
3012 // This is different from indirect *not* byval, where the object
3013 // exists already, and the align attribute is purely
3014 // informative.
3015 assert(!Align.isZero());
3016
3017 // For now, only add this when we have a byval argument.
3018 // TODO: be less lazy about updating test cases.
3019 if (AI.getIndirectByVal())
3020 Attrs.addAlignmentAttr(Align: Align.getQuantity());
3021
3022 // byval disables readnone and readonly.
3023 AddPotentialArgAccess();
3024 break;
3025 }
3026 case ABIArgInfo::IndirectAliased: {
3027 CharUnits Align = AI.getIndirectAlign();
3028 Attrs.addByRefAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
3029 Attrs.addAlignmentAttr(Align: Align.getQuantity());
3030 break;
3031 }
3032 case ABIArgInfo::Ignore:
3033 case ABIArgInfo::Expand:
3034 case ABIArgInfo::CoerceAndExpand:
3035 break;
3036
3037 case ABIArgInfo::InAlloca:
3038 // inalloca disables readnone and readonly.
3039 AddPotentialArgAccess();
3040 continue;
3041 }
3042
3043 if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
3044 QualType PTy = RefTy->getPointeeType();
3045 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
3046 Attrs.addDereferenceableAttr(Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
3047 if (getTypes().getTargetAddressSpace(T: PTy) == 0 &&
3048 !CodeGenOpts.NullPointerIsValid)
3049 Attrs.addAttribute(Val: llvm::Attribute::NonNull);
3050 if (PTy->isObjectType()) {
3051 llvm::Align Alignment =
3052 getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
3053 Attrs.addAlignmentAttr(Align: Alignment);
3054 }
3055 }
3056
3057 // From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:
3058 // > For arguments to a __kernel function declared to be a pointer to a
3059 // > data type, the OpenCL compiler can assume that the pointee is always
3060 // > appropriately aligned as required by the data type.
3061 if (TargetDecl &&
3062 DeviceKernelAttr::isOpenCLSpelling(
3063 A: TargetDecl->getAttr<DeviceKernelAttr>()) &&
3064 ParamType->isPointerType()) {
3065 QualType PTy = ParamType->getPointeeType();
3066 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
3067 llvm::Align Alignment =
3068 getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
3069 Attrs.addAlignmentAttr(Align: Alignment);
3070 }
3071 }
3072
3073 switch (FI.getExtParameterInfo(argIndex: ArgNo).getABI()) {
3074 case ParameterABI::HLSLOut:
3075 case ParameterABI::HLSLInOut:
3076 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
3077 break;
3078 case ParameterABI::Ordinary:
3079 break;
3080
3081 case ParameterABI::SwiftIndirectResult: {
3082 // Add 'sret' if we haven't already used it for something, but
3083 // only if the result is void.
3084 if (!hasUsedSRet && RetTy->isVoidType()) {
3085 Attrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
3086 hasUsedSRet = true;
3087 }
3088
3089 // Add 'noalias' in either case.
3090 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
3091
3092 // Add 'dereferenceable' and 'alignment'.
3093 auto PTy = ParamType->getPointeeType();
3094 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
3095 auto info = getContext().getTypeInfoInChars(T: PTy);
3096 Attrs.addDereferenceableAttr(Bytes: info.Width.getQuantity());
3097 Attrs.addAlignmentAttr(Align: info.Align.getAsAlign());
3098 }
3099 break;
3100 }
3101
3102 case ParameterABI::SwiftErrorResult:
3103 Attrs.addAttribute(Val: llvm::Attribute::SwiftError);
3104 break;
3105
3106 case ParameterABI::SwiftContext:
3107 Attrs.addAttribute(Val: llvm::Attribute::SwiftSelf);
3108 break;
3109
3110 case ParameterABI::SwiftAsyncContext:
3111 Attrs.addAttribute(Val: llvm::Attribute::SwiftAsync);
3112 break;
3113 }
3114
3115 if (FI.getExtParameterInfo(argIndex: ArgNo).isNoEscape())
3116 Attrs.addCapturesAttr(CI: llvm::CaptureInfo::none());
3117
3118 if (Attrs.hasAttributes()) {
3119 unsigned FirstIRArg, NumIRArgs;
3120 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3121 for (unsigned i = 0; i < NumIRArgs; i++)
3122 ArgAttrs[FirstIRArg + i] = ArgAttrs[FirstIRArg + i].addAttributes(
3123 C&: getLLVMContext(), AS: llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs));
3124 }
3125 }
3126 assert(ArgNo == FI.arg_size());
3127
3128 ArgNo = 0;
3129 if (AddedPotentialArgAccess && MemAttrForPtrArgs) {
3130 llvm::FunctionType *FunctionType = getTypes().GetFunctionType(FI);
3131 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
3132 E = FI.arg_end();
3133 I != E; ++I, ++ArgNo) {
3134 if (I->info.isDirect() || I->info.isExpand() ||
3135 I->info.isCoerceAndExpand()) {
3136 unsigned FirstIRArg, NumIRArgs;
3137 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3138 for (unsigned i = FirstIRArg; i < FirstIRArg + NumIRArgs; ++i) {
3139 if (FunctionType->getParamType(i)->isPointerTy()) {
3140 ArgAttrs[i] =
3141 ArgAttrs[i].addAttribute(C&: getLLVMContext(), Kind: *MemAttrForPtrArgs);
3142 }
3143 }
3144 }
3145 }
3146 }
3147
3148 AttrList = llvm::AttributeList::get(
3149 C&: getLLVMContext(), FnAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: FuncAttrs),
3150 RetAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: RetAttrs), ArgAttrs);
3151}
3152
3153/// An argument came in as a promoted argument; demote it back to its
3154/// declared type.
3155static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
3156 const VarDecl *var,
3157 llvm::Value *value) {
3158 llvm::Type *varType = CGF.ConvertType(T: var->getType());
3159
3160 // This can happen with promotions that actually don't change the
3161 // underlying type, like the enum promotions.
3162 if (value->getType() == varType)
3163 return value;
3164
3165 assert((varType->isIntegerTy() || varType->isFloatingPointTy()) &&
3166 "unexpected promotion type");
3167
3168 if (isa<llvm::IntegerType>(Val: varType))
3169 return CGF.Builder.CreateTrunc(V: value, DestTy: varType, Name: "arg.unpromote");
3170
3171 return CGF.Builder.CreateFPCast(V: value, DestTy: varType, Name: "arg.unpromote");
3172}
3173
3174/// Returns the attribute (either parameter attribute, or function
3175/// attribute), which declares argument ArgNo to be non-null.
3176static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
3177 QualType ArgType, unsigned ArgNo) {
3178 // FIXME: __attribute__((nonnull)) can also be applied to:
3179 // - references to pointers, where the pointee is known to be
3180 // nonnull (apparently a Clang extension)
3181 // - transparent unions containing pointers
3182 // In the former case, LLVM IR cannot represent the constraint. In
3183 // the latter case, we have no guarantee that the transparent union
3184 // is in fact passed as a pointer.
3185 if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType())
3186 return nullptr;
3187 // First, check attribute on parameter itself.
3188 if (PVD) {
3189 if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
3190 return ParmNNAttr;
3191 }
3192 // Check function attributes.
3193 if (!FD)
3194 return nullptr;
3195 for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
3196 if (NNAttr->isNonNull(IdxAST: ArgNo))
3197 return NNAttr;
3198 }
3199 return nullptr;
3200}
3201
3202namespace {
3203struct CopyBackSwiftError final : EHScopeStack::Cleanup {
3204 Address Temp;
3205 Address Arg;
3206 CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
3207 void Emit(CodeGenFunction &CGF, Flags flags) override {
3208 llvm::Value *errorValue = CGF.Builder.CreateLoad(Addr: Temp);
3209 CGF.Builder.CreateStore(Val: errorValue, Addr: Arg);
3210 }
3211};
3212} // namespace
3213
3214void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
3215 llvm::Function *Fn,
3216 const FunctionArgList &Args) {
3217 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())
3218 // Naked functions don't have prologues.
3219 return;
3220
3221 // If this is an implicit-return-zero function, go ahead and
3222 // initialize the return value. TODO: it might be nice to have
3223 // a more general mechanism for this that didn't require synthesized
3224 // return statements.
3225 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl)) {
3226 if (FD->hasImplicitReturnZero()) {
3227 QualType RetTy = FD->getReturnType().getUnqualifiedType();
3228 llvm::Type *LLVMTy = CGM.getTypes().ConvertType(T: RetTy);
3229 llvm::Constant *Zero = llvm::Constant::getNullValue(Ty: LLVMTy);
3230 Builder.CreateStore(Val: Zero, Addr: ReturnValue);
3231 }
3232 }
3233
3234 // FIXME: We no longer need the types from FunctionArgList; lift up and
3235 // simplify.
3236
3237 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
3238 assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
3239
3240 // If we're using inalloca, all the memory arguments are GEPs off of the last
3241 // parameter, which is a pointer to the complete memory area.
3242 Address ArgStruct = Address::invalid();
3243 if (IRFunctionArgs.hasInallocaArg())
3244 ArgStruct = Address(Fn->getArg(i: IRFunctionArgs.getInallocaArgNo()),
3245 FI.getArgStruct(), FI.getArgStructAlignment());
3246
3247 // Name the struct return parameter.
3248 if (IRFunctionArgs.hasSRetArg()) {
3249 auto AI = Fn->getArg(i: IRFunctionArgs.getSRetArgNo());
3250 AI->setName("agg.result");
3251 AI->addAttr(Kind: llvm::Attribute::NoAlias);
3252 }
3253
3254 // Track if we received the parameter as a pointer (indirect, byval, or
3255 // inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it
3256 // into a local alloca for us.
3257 SmallVector<ParamValue, 16> ArgVals;
3258 ArgVals.reserve(N: Args.size());
3259
3260 // Create a pointer value for every parameter declaration. This usually
3261 // entails copying one or more LLVM IR arguments into an alloca. Don't push
3262 // any cleanups or do anything that might unwind. We do that separately, so
3263 // we can push the cleanups in the correct order for the ABI.
3264 assert(FI.arg_size() == Args.size() &&
3265 "Mismatch between function signature & arguments.");
3266 unsigned ArgNo = 0;
3267 CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
3268 for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end(); i != e;
3269 ++i, ++info_it, ++ArgNo) {
3270 const VarDecl *Arg = *i;
3271 const ABIArgInfo &ArgI = info_it->info;
3272
3273 bool isPromoted =
3274 isa<ParmVarDecl>(Val: Arg) && cast<ParmVarDecl>(Val: Arg)->isKNRPromoted();
3275 // We are converting from ABIArgInfo type to VarDecl type directly, unless
3276 // the parameter is promoted. In this case we convert to
3277 // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
3278 QualType Ty = isPromoted ? info_it->type : Arg->getType();
3279 assert(hasScalarEvaluationKind(Ty) ==
3280 hasScalarEvaluationKind(Arg->getType()));
3281
3282 unsigned FirstIRArg, NumIRArgs;
3283 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3284
3285 switch (ArgI.getKind()) {
3286 case ABIArgInfo::InAlloca: {
3287 assert(NumIRArgs == 0);
3288 auto FieldIndex = ArgI.getInAllocaFieldIndex();
3289 Address V =
3290 Builder.CreateStructGEP(Addr: ArgStruct, Index: FieldIndex, Name: Arg->getName());
3291 if (ArgI.getInAllocaIndirect())
3292 V = Address(Builder.CreateLoad(Addr: V), ConvertTypeForMem(T: Ty),
3293 getContext().getTypeAlignInChars(T: Ty));
3294 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: V));
3295 break;
3296 }
3297
3298 case ABIArgInfo::Indirect:
3299 case ABIArgInfo::IndirectAliased: {
3300 assert(NumIRArgs == 1);
3301 Address ParamAddr = makeNaturalAddressForPointer(
3302 Ptr: Fn->getArg(i: FirstIRArg), T: Ty, Alignment: ArgI.getIndirectAlign(), ForPointeeType: false, BaseInfo: nullptr,
3303 TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
3304
3305 if (!hasScalarEvaluationKind(T: Ty)) {
3306 // Aggregates and complex variables are accessed by reference. All we
3307 // need to do is realign the value, if requested. Also, if the address
3308 // may be aliased, copy it to ensure that the parameter variable is
3309 // mutable and has a unique adress, as C requires.
3310 if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) {
3311 RawAddress AlignedTemp = CreateMemTemp(T: Ty, Name: "coerce");
3312
3313 // Copy from the incoming argument pointer to the temporary with the
3314 // appropriate alignment.
3315 //
3316 // FIXME: We should have a common utility for generating an aggregate
3317 // copy.
3318 CharUnits Size = getContext().getTypeSizeInChars(T: Ty);
3319 Builder.CreateMemCpy(
3320 Dst: AlignedTemp.getPointer(), DstAlign: AlignedTemp.getAlignment().getAsAlign(),
3321 Src: ParamAddr.emitRawPointer(CGF&: *this),
3322 SrcAlign: ParamAddr.getAlignment().getAsAlign(),
3323 Size: llvm::ConstantInt::get(Ty: IntPtrTy, V: Size.getQuantity()));
3324 ParamAddr = AlignedTemp;
3325 }
3326 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: ParamAddr));
3327 } else {
3328 // Load scalar value from indirect argument.
3329 llvm::Value *V =
3330 EmitLoadOfScalar(Addr: ParamAddr, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3331
3332 if (isPromoted)
3333 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3334 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3335 }
3336 break;
3337 }
3338
3339 case ABIArgInfo::Extend:
3340 case ABIArgInfo::Direct: {
3341 auto AI = Fn->getArg(i: FirstIRArg);
3342 llvm::Type *LTy = ConvertType(T: Arg->getType());
3343
3344 // Prepare parameter attributes. So far, only attributes for pointer
3345 // parameters are prepared. See
3346 // http://llvm.org/docs/LangRef.html#paramattrs.
3347 if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy() &&
3348 ArgI.getCoerceToType()->isPointerTy()) {
3349 assert(NumIRArgs == 1);
3350
3351 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: Arg)) {
3352 // Set `nonnull` attribute if any.
3353 if (getNonNullAttr(FD: CurCodeDecl, PVD, ArgType: PVD->getType(),
3354 ArgNo: PVD->getFunctionScopeIndex()) &&
3355 !CGM.getCodeGenOpts().NullPointerIsValid)
3356 AI->addAttr(Kind: llvm::Attribute::NonNull);
3357
3358 QualType OTy = PVD->getOriginalType();
3359 if (const auto *ArrTy = getContext().getAsConstantArrayType(T: OTy)) {
3360 // A C99 array parameter declaration with the static keyword also
3361 // indicates dereferenceability, and if the size is constant we can
3362 // use the dereferenceable attribute (which requires the size in
3363 // bytes).
3364 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3365 QualType ETy = ArrTy->getElementType();
3366 llvm::Align Alignment =
3367 CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3368 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3369 .addAlignmentAttr(Align: Alignment));
3370 uint64_t ArrSize = ArrTy->getZExtSize();
3371 if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
3372 ArrSize) {
3373 llvm::AttrBuilder Attrs(getLLVMContext());
3374 Attrs.addDereferenceableAttr(
3375 Bytes: getContext().getTypeSizeInChars(T: ETy).getQuantity() *
3376 ArrSize);
3377 AI->addAttrs(B&: Attrs);
3378 } else if (getContext().getTargetInfo().getNullPointerValue(
3379 AddrSpace: ETy.getAddressSpace()) == 0 &&
3380 !CGM.getCodeGenOpts().NullPointerIsValid) {
3381 AI->addAttr(Kind: llvm::Attribute::NonNull);
3382 }
3383 }
3384 } else if (const auto *ArrTy =
3385 getContext().getAsVariableArrayType(T: OTy)) {
3386 // For C99 VLAs with the static keyword, we don't know the size so
3387 // we can't use the dereferenceable attribute, but in addrspace(0)
3388 // we know that it must be nonnull.
3389 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3390 QualType ETy = ArrTy->getElementType();
3391 llvm::Align Alignment =
3392 CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3393 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3394 .addAlignmentAttr(Align: Alignment));
3395 if (!getTypes().getTargetAddressSpace(T: ETy) &&
3396 !CGM.getCodeGenOpts().NullPointerIsValid)
3397 AI->addAttr(Kind: llvm::Attribute::NonNull);
3398 }
3399 }
3400
3401 // Set `align` attribute if any.
3402 const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
3403 if (!AVAttr)
3404 if (const auto *TOTy = OTy->getAs<TypedefType>())
3405 AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
3406 if (AVAttr && !SanOpts.has(K: SanitizerKind::Alignment)) {
3407 // If alignment-assumption sanitizer is enabled, we do *not* add
3408 // alignment attribute here, but emit normal alignment assumption,
3409 // so the UBSAN check could function.
3410 llvm::ConstantInt *AlignmentCI =
3411 cast<llvm::ConstantInt>(Val: EmitScalarExpr(E: AVAttr->getAlignment()));
3412 uint64_t AlignmentInt =
3413 AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment);
3414 if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
3415 AI->removeAttr(Kind: llvm::Attribute::AttrKind::Alignment);
3416 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3417 .addAlignmentAttr(Align: llvm::Align(AlignmentInt)));
3418 }
3419 }
3420 }
3421
3422 // Set 'noalias' if an argument type has the `restrict` qualifier.
3423 if (Arg->getType().isRestrictQualified())
3424 AI->addAttr(Kind: llvm::Attribute::NoAlias);
3425 }
3426
3427 // Prepare the argument value. If we have the trivial case, handle it
3428 // with no muss and fuss.
3429 if (!isa<llvm::StructType>(Val: ArgI.getCoerceToType()) &&
3430 ArgI.getCoerceToType() == ConvertType(T: Ty) &&
3431 ArgI.getDirectOffset() == 0) {
3432 assert(NumIRArgs == 1);
3433
3434 // LLVM expects swifterror parameters to be used in very restricted
3435 // ways. Copy the value into a less-restricted temporary.
3436 llvm::Value *V = AI;
3437 if (FI.getExtParameterInfo(argIndex: ArgNo).getABI() ==
3438 ParameterABI::SwiftErrorResult) {
3439 QualType pointeeTy = Ty->getPointeeType();
3440 assert(pointeeTy->isPointerType());
3441 RawAddress temp =
3442 CreateMemTemp(T: pointeeTy, Align: getPointerAlign(), Name: "swifterror.temp");
3443 Address arg = makeNaturalAddressForPointer(
3444 Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
3445 llvm::Value *incomingErrorValue = Builder.CreateLoad(Addr: arg);
3446 Builder.CreateStore(Val: incomingErrorValue, Addr: temp);
3447 V = temp.getPointer();
3448
3449 // Push a cleanup to copy the value back at the end of the function.
3450 // The convention does not guarantee that the value will be written
3451 // back if the function exits with an unwind exception.
3452 EHStack.pushCleanup<CopyBackSwiftError>(Kind: NormalCleanup, A: temp, A: arg);
3453 }
3454
3455 // Ensure the argument is the correct type.
3456 if (V->getType() != ArgI.getCoerceToType())
3457 V = Builder.CreateBitCast(V, DestTy: ArgI.getCoerceToType());
3458
3459 if (isPromoted)
3460 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3461
3462 // Because of merging of function types from multiple decls it is
3463 // possible for the type of an argument to not match the corresponding
3464 // type in the function type. Since we are codegening the callee
3465 // in here, add a cast to the argument type.
3466 llvm::Type *LTy = ConvertType(T: Arg->getType());
3467 if (V->getType() != LTy)
3468 V = Builder.CreateBitCast(V, DestTy: LTy);
3469
3470 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3471 break;
3472 }
3473
3474 // VLST arguments are coerced to VLATs at the function boundary for
3475 // ABI consistency. If this is a VLST that was coerced to
3476 // a VLAT at the function boundary and the types match up, use
3477 // llvm.vector.extract to convert back to the original VLST.
3478 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: ConvertType(T: Ty))) {
3479 llvm::Value *ArgVal = Fn->getArg(i: FirstIRArg);
3480 if (auto *VecTyFrom =
3481 dyn_cast<llvm::ScalableVectorType>(Val: ArgVal->getType())) {
3482 auto [Coerced, Extracted] = CoerceScalableToFixed(
3483 CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: ArgVal, Name: Arg->getName());
3484 if (Extracted) {
3485 assert(NumIRArgs == 1);
3486 ArgVals.push_back(Elt: ParamValue::forDirect(value: Coerced));
3487 break;
3488 }
3489 }
3490 }
3491
3492 llvm::StructType *STy =
3493 dyn_cast<llvm::StructType>(Val: ArgI.getCoerceToType());
3494 Address Alloca =
3495 CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(D: Arg), Name: Arg->getName());
3496
3497 // Pointer to store into.
3498 Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3499
3500 // Fast-isel and the optimizer generally like scalar values better than
3501 // FCAs, so we flatten them if this is safe to do for this argument.
3502 if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
3503 STy->getNumElements() > 1) {
3504 llvm::TypeSize StructSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
3505 llvm::TypeSize PtrElementSize =
3506 CGM.getDataLayout().getTypeAllocSize(Ty: Ptr.getElementType());
3507 if (StructSize.isScalable()) {
3508 assert(STy->containsHomogeneousScalableVectorTypes() &&
3509 "ABI only supports structure with homogeneous scalable vector "
3510 "type");
3511 assert(StructSize == PtrElementSize &&
3512 "Only allow non-fractional movement of structure with"
3513 "homogeneous scalable vector type");
3514 assert(STy->getNumElements() == NumIRArgs);
3515
3516 llvm::Value *LoadedStructValue = llvm::PoisonValue::get(T: STy);
3517 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3518 auto *AI = Fn->getArg(i: FirstIRArg + i);
3519 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3520 LoadedStructValue =
3521 Builder.CreateInsertValue(Agg: LoadedStructValue, Val: AI, Idxs: i);
3522 }
3523
3524 Builder.CreateStore(Val: LoadedStructValue, Addr: Ptr);
3525 } else {
3526 uint64_t SrcSize = StructSize.getFixedValue();
3527 uint64_t DstSize = PtrElementSize.getFixedValue();
3528
3529 Address AddrToStoreInto = Address::invalid();
3530 if (SrcSize <= DstSize) {
3531 AddrToStoreInto = Ptr.withElementType(ElemTy: STy);
3532 } else {
3533 AddrToStoreInto =
3534 CreateTempAlloca(Ty: STy, align: Alloca.getAlignment(), Name: "coerce");
3535 }
3536
3537 assert(STy->getNumElements() == NumIRArgs);
3538 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3539 auto AI = Fn->getArg(i: FirstIRArg + i);
3540 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3541 Address EltPtr = Builder.CreateStructGEP(Addr: AddrToStoreInto, Index: i);
3542 Builder.CreateStore(Val: AI, Addr: EltPtr);
3543 }
3544
3545 if (SrcSize > DstSize) {
3546 Builder.CreateMemCpy(Dest: Ptr, Src: AddrToStoreInto, Size: DstSize);
3547 }
3548
3549 // Structures with PFP fields require a coerced store to add any
3550 // pointer signatures.
3551 if (getContext().hasPFPFields(Ty)) {
3552 llvm::Value *Struct = Builder.CreateLoad(Addr: Ptr);
3553 CreatePFPCoercedStore(Src: Struct, SrcFETy: Ty, Dst: Ptr, CGF&: *this);
3554 }
3555 }
3556 } else {
3557 // Simple case, just do a coerced store of the argument into the alloca.
3558 assert(NumIRArgs == 1);
3559 auto AI = Fn->getArg(i: FirstIRArg);
3560 AI->setName(Arg->getName() + ".coerce");
3561 CreateCoercedStore(
3562 Src: AI, SrcFETy: Ty, Dst: Ptr,
3563 DstSize: llvm::TypeSize::getFixed(
3564 ExactSize: getContext().getTypeSizeInChars(T: Ty).getQuantity() -
3565 ArgI.getDirectOffset()),
3566 /*DstIsVolatile=*/false);
3567 }
3568
3569 // Match to what EmitParmDecl is expecting for this type.
3570 if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3571 llvm::Value *V =
3572 EmitLoadOfScalar(Addr: Alloca, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3573 if (isPromoted)
3574 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3575 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3576 } else {
3577 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3578 }
3579 break;
3580 }
3581
3582 case ABIArgInfo::CoerceAndExpand: {
3583 // Reconstruct into a temporary.
3584 Address alloca = CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(D: Arg));
3585 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: alloca));
3586
3587 auto coercionType = ArgI.getCoerceAndExpandType();
3588 auto unpaddedCoercionType = ArgI.getUnpaddedCoerceAndExpandType();
3589 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
3590
3591 alloca = alloca.withElementType(ElemTy: coercionType);
3592
3593 unsigned argIndex = FirstIRArg;
3594 unsigned unpaddedIndex = 0;
3595 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3596 llvm::Type *eltType = coercionType->getElementType(N: i);
3597 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
3598 continue;
3599
3600 auto eltAddr = Builder.CreateStructGEP(Addr: alloca, Index: i);
3601 llvm::Value *elt = Fn->getArg(i: argIndex++);
3602
3603 auto paramType = unpaddedStruct
3604 ? unpaddedStruct->getElementType(N: unpaddedIndex++)
3605 : unpaddedCoercionType;
3606
3607 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: eltType)) {
3608 if (auto *VecTyFrom = dyn_cast<llvm::ScalableVectorType>(Val: paramType)) {
3609 bool Extracted;
3610 std::tie(args&: elt, args&: Extracted) = CoerceScalableToFixed(
3611 CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: elt, Name: elt->getName());
3612 assert(Extracted && "Unexpected scalable to fixed vector coercion");
3613 }
3614 }
3615 Builder.CreateStore(Val: elt, Addr: eltAddr);
3616 }
3617 assert(argIndex == FirstIRArg + NumIRArgs);
3618 break;
3619 }
3620
3621 case ABIArgInfo::Expand: {
3622 // If this structure was expanded into multiple arguments then
3623 // we need to create a temporary and reconstruct it from the
3624 // arguments.
3625 Address Alloca = CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(D: Arg));
3626 LValue LV = MakeAddrLValue(Addr: Alloca, T: Ty);
3627 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3628
3629 auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3630 ExpandTypeFromArgs(Ty, LV, AI&: FnArgIter);
3631 assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3632 for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
3633 auto AI = Fn->getArg(i: FirstIRArg + i);
3634 AI->setName(Arg->getName() + "." + Twine(i));
3635 }
3636 break;
3637 }
3638
3639 case ABIArgInfo::TargetSpecific: {
3640 auto *AI = Fn->getArg(i: FirstIRArg);
3641 AI->setName(Arg->getName() + ".target_coerce");
3642 Address Alloca =
3643 CreateMemTemp(T: Ty, Align: getContext().getDeclAlign(D: Arg), Name: Arg->getName());
3644 Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3645 CGM.getABIInfo().createCoercedStore(Val: AI, DstAddr: Ptr, AI: ArgI, DestIsVolatile: false, CGF&: *this);
3646 if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3647 llvm::Value *V =
3648 EmitLoadOfScalar(Addr: Alloca, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3649 if (isPromoted) {
3650 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3651 }
3652 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3653 } else {
3654 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3655 }
3656 break;
3657 }
3658 case ABIArgInfo::Ignore:
3659 assert(NumIRArgs == 0);
3660 // Initialize the local variable appropriately.
3661 if (!hasScalarEvaluationKind(T: Ty)) {
3662 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: CreateMemTemp(T: Ty)));
3663 } else {
3664 llvm::Value *U = llvm::UndefValue::get(T: ConvertType(T: Arg->getType()));
3665 ArgVals.push_back(Elt: ParamValue::forDirect(value: U));
3666 }
3667 break;
3668 }
3669 }
3670
3671 if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3672 for (int I = Args.size() - 1; I >= 0; --I)
3673 EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3674 } else {
3675 for (unsigned I = 0, E = Args.size(); I != E; ++I)
3676 EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3677 }
3678}
3679
3680static void eraseUnusedBitCasts(llvm::Instruction *insn) {
3681 while (insn->use_empty()) {
3682 llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: insn);
3683 if (!bitcast)
3684 return;
3685
3686 // This is "safe" because we would have used a ConstantExpr otherwise.
3687 insn = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3688 bitcast->eraseFromParent();
3689 }
3690}
3691
3692/// Try to emit a fused autorelease of a return result.
3693static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
3694 llvm::Value *result) {
3695 // We must be immediately followed the cast.
3696 llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3697 if (BB->empty())
3698 return nullptr;
3699 if (&BB->back() != result)
3700 return nullptr;
3701
3702 llvm::Type *resultType = result->getType();
3703
3704 // result is in a BasicBlock and is therefore an Instruction.
3705 llvm::Instruction *generator = cast<llvm::Instruction>(Val: result);
3706
3707 SmallVector<llvm::Instruction *, 4> InstsToKill;
3708
3709 // Look for:
3710 // %generator = bitcast %type1* %generator2 to %type2*
3711 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: generator)) {
3712 // We would have emitted this as a constant if the operand weren't
3713 // an Instruction.
3714 generator = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3715
3716 // Require the generator to be immediately followed by the cast.
3717 if (generator->getNextNode() != bitcast)
3718 return nullptr;
3719
3720 InstsToKill.push_back(Elt: bitcast);
3721 }
3722
3723 // Look for:
3724 // %generator = call i8* @objc_retain(i8* %originalResult)
3725 // or
3726 // %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
3727 llvm::CallInst *call = dyn_cast<llvm::CallInst>(Val: generator);
3728 if (!call)
3729 return nullptr;
3730
3731 bool doRetainAutorelease;
3732
3733 if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
3734 doRetainAutorelease = true;
3735 } else if (call->getCalledOperand() ==
3736 CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
3737 doRetainAutorelease = false;
3738
3739 // If we emitted an assembly marker for this call (and the
3740 // ARCEntrypoints field should have been set if so), go looking
3741 // for that call. If we can't find it, we can't do this
3742 // optimization. But it should always be the immediately previous
3743 // instruction, unless we needed bitcasts around the call.
3744 if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
3745 llvm::Instruction *prev = call->getPrevNode();
3746 assert(prev);
3747 if (isa<llvm::BitCastInst>(Val: prev)) {
3748 prev = prev->getPrevNode();
3749 assert(prev);
3750 }
3751 assert(isa<llvm::CallInst>(prev));
3752 assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3753 CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
3754 InstsToKill.push_back(Elt: prev);
3755 }
3756 } else {
3757 return nullptr;
3758 }
3759
3760 result = call->getArgOperand(i: 0);
3761 InstsToKill.push_back(Elt: call);
3762
3763 // Keep killing bitcasts, for sanity. Note that we no longer care
3764 // about precise ordering as long as there's exactly one use.
3765 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: result)) {
3766 if (!bitcast->hasOneUse())
3767 break;
3768 InstsToKill.push_back(Elt: bitcast);
3769 result = bitcast->getOperand(i_nocapture: 0);
3770 }
3771
3772 // Delete all the unnecessary instructions, from latest to earliest.
3773 for (auto *I : InstsToKill)
3774 I->eraseFromParent();
3775
3776 // Do the fused retain/autorelease if we were asked to.
3777 if (doRetainAutorelease)
3778 result = CGF.EmitARCRetainAutoreleaseReturnValue(value: result);
3779
3780 // Cast back to the result type.
3781 return CGF.Builder.CreateBitCast(V: result, DestTy: resultType);
3782}
3783
3784/// If this is a +1 of the value of an immutable 'self', remove it.
3785static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
3786 llvm::Value *result) {
3787 // This is only applicable to a method with an immutable 'self'.
3788 const ObjCMethodDecl *method =
3789 dyn_cast_or_null<ObjCMethodDecl>(Val: CGF.CurCodeDecl);
3790 if (!method)
3791 return nullptr;
3792 const VarDecl *self = method->getSelfDecl();
3793 if (!self->getType().isConstQualified())
3794 return nullptr;
3795
3796 // Look for a retain call. Note: stripPointerCasts looks through returned arg
3797 // functions, which would cause us to miss the retain.
3798 llvm::CallInst *retainCall = dyn_cast<llvm::CallInst>(Val: result);
3799 if (!retainCall || retainCall->getCalledOperand() !=
3800 CGF.CGM.getObjCEntrypoints().objc_retain)
3801 return nullptr;
3802
3803 // Look for an ordinary load of 'self'.
3804 llvm::Value *retainedValue = retainCall->getArgOperand(i: 0);
3805 llvm::LoadInst *load =
3806 dyn_cast<llvm::LoadInst>(Val: retainedValue->stripPointerCasts());
3807 if (!load || load->isAtomic() || load->isVolatile() ||
3808 load->getPointerOperand() != CGF.GetAddrOfLocalVar(VD: self).getBasePointer())
3809 return nullptr;
3810
3811 // Okay! Burn it all down. This relies for correctness on the
3812 // assumption that the retain is emitted as part of the return and
3813 // that thereafter everything is used "linearly".
3814 llvm::Type *resultType = result->getType();
3815 eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: result));
3816 assert(retainCall->use_empty());
3817 retainCall->eraseFromParent();
3818 eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: retainedValue));
3819
3820 return CGF.Builder.CreateBitCast(V: load, DestTy: resultType);
3821}
3822
3823/// Emit an ARC autorelease of the result of a function.
3824///
3825/// \return the value to actually return from the function
3826static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
3827 llvm::Value *result) {
3828 // If we're returning 'self', kill the initial retain. This is a
3829 // heuristic attempt to "encourage correctness" in the really unfortunate
3830 // case where we have a return of self during a dealloc and we desperately
3831 // need to avoid the possible autorelease.
3832 if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
3833 return self;
3834
3835 // At -O0, try to emit a fused retain/autorelease.
3836 if (CGF.shouldUseFusedARCCalls())
3837 if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
3838 return fused;
3839
3840 return CGF.EmitARCAutoreleaseReturnValue(value: result);
3841}
3842
3843/// Heuristically search for a dominating store to the return-value slot.
3844static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
3845 llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer();
3846
3847 // Check if a User is a store which pointerOperand is the ReturnValue.
3848 // We are looking for stores to the ReturnValue, not for stores of the
3849 // ReturnValue to some other location.
3850 auto GetStoreIfValid = [&CGF,
3851 ReturnValuePtr](llvm::User *U) -> llvm::StoreInst * {
3852 auto *SI = dyn_cast<llvm::StoreInst>(Val: U);
3853 if (!SI || SI->getPointerOperand() != ReturnValuePtr ||
3854 SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())
3855 return nullptr;
3856 // These aren't actually possible for non-coerced returns, and we
3857 // only care about non-coerced returns on this code path.
3858 // All memory instructions inside __try block are volatile.
3859 assert(!SI->isAtomic() &&
3860 (!SI->isVolatile() || CGF.currentFunctionUsesSEHTry()));
3861 return SI;
3862 };
3863 // If there are multiple uses of the return-value slot, just check
3864 // for something immediately preceding the IP. Sometimes this can
3865 // happen with how we generate implicit-returns; it can also happen
3866 // with noreturn cleanups.
3867 if (!ReturnValuePtr->hasOneUse()) {
3868 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3869 if (IP->empty())
3870 return nullptr;
3871
3872 // Look at directly preceding instruction, skipping bitcasts, lifetime
3873 // markers, and fake uses and their operands.
3874 const llvm::Instruction *LoadIntoFakeUse = nullptr;
3875 for (llvm::Instruction &I : llvm::reverse(C&: *IP)) {
3876 // Ignore instructions that are just loads for fake uses; the load should
3877 // immediately precede the fake use, so we only need to remember the
3878 // operand for the last fake use seen.
3879 if (LoadIntoFakeUse == &I)
3880 continue;
3881 if (isa<llvm::BitCastInst>(Val: &I))
3882 continue;
3883 if (auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &I)) {
3884 if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
3885 continue;
3886
3887 if (II->getIntrinsicID() == llvm::Intrinsic::fake_use) {
3888 LoadIntoFakeUse = dyn_cast<llvm::Instruction>(Val: II->getArgOperand(i: 0));
3889 continue;
3890 }
3891 }
3892 return GetStoreIfValid(&I);
3893 }
3894 return nullptr;
3895 }
3896
3897 llvm::StoreInst *store = GetStoreIfValid(ReturnValuePtr->user_back());
3898 if (!store)
3899 return nullptr;
3900
3901 // Now do a first-and-dirty dominance check: just walk up the
3902 // single-predecessors chain from the current insertion point.
3903 llvm::BasicBlock *StoreBB = store->getParent();
3904 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3905 llvm::SmallPtrSet<llvm::BasicBlock *, 4> SeenBBs;
3906 while (IP != StoreBB) {
3907 if (!SeenBBs.insert(Ptr: IP).second || !(IP = IP->getSinglePredecessor()))
3908 return nullptr;
3909 }
3910
3911 // Okay, the store's basic block dominates the insertion point; we
3912 // can do our thing.
3913 return store;
3914}
3915
3916// Helper functions for EmitCMSEClearRecord
3917
3918// Set the bits corresponding to a field having width `BitWidth` and located at
3919// offset `BitOffset` (from the least significant bit) within a storage unit of
3920// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
3921// Use little-endian layout, i.e.`Bits[0]` is the LSB.
3922static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
3923 int BitWidth, int CharWidth) {
3924 assert(CharWidth <= 64);
3925 assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3926
3927 int Pos = 0;
3928 if (BitOffset >= CharWidth) {
3929 Pos += BitOffset / CharWidth;
3930 BitOffset = BitOffset % CharWidth;
3931 }
3932
3933 const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
3934 if (BitOffset + BitWidth >= CharWidth) {
3935 Bits[Pos++] |= (Used << BitOffset) & Used;
3936 BitWidth -= CharWidth - BitOffset;
3937 BitOffset = 0;
3938 }
3939
3940 while (BitWidth >= CharWidth) {
3941 Bits[Pos++] = Used;
3942 BitWidth -= CharWidth;
3943 }
3944
3945 if (BitWidth > 0)
3946 Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
3947}
3948
3949// Set the bits corresponding to a field having width `BitWidth` and located at
3950// offset `BitOffset` (from the least significant bit) within a storage unit of
3951// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
3952// `Bits` corresponds to one target byte. Use target endian layout.
3953static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
3954 int StorageSize, int BitOffset, int BitWidth,
3955 int CharWidth, bool BigEndian) {
3956
3957 SmallVector<uint64_t, 8> TmpBits(StorageSize);
3958 setBitRange(Bits&: TmpBits, BitOffset, BitWidth, CharWidth);
3959
3960 if (BigEndian)
3961 std::reverse(first: TmpBits.begin(), last: TmpBits.end());
3962
3963 for (uint64_t V : TmpBits)
3964 Bits[StorageOffset++] |= V;
3965}
3966
3967static void setUsedBits(CodeGenModule &, QualType, int,
3968 SmallVectorImpl<uint64_t> &);
3969
3970// Set the bits in `Bits`, which correspond to the value representations of
3971// the actual members of the record type `RTy`. Note that this function does
3972// not handle base classes, virtual tables, etc, since they cannot happen in
3973// CMSE function arguments or return. The bit mask corresponds to the target
3974// memory layout, i.e. it's endian dependent.
3975static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
3976 SmallVectorImpl<uint64_t> &Bits) {
3977 ASTContext &Context = CGM.getContext();
3978 int CharWidth = Context.getCharWidth();
3979 const RecordDecl *RD = RTy->getDecl()->getDefinition();
3980 const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(D: RD);
3981 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
3982
3983 int Idx = 0;
3984 for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) {
3985 const FieldDecl *F = *I;
3986
3987 if (F->isUnnamedBitField() || F->isZeroLengthBitField() ||
3988 F->getType()->isIncompleteArrayType())
3989 continue;
3990
3991 if (F->isBitField()) {
3992 const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(FD: F);
3993 setBitRange(Bits, StorageOffset: Offset + BFI.StorageOffset.getQuantity(),
3994 StorageSize: BFI.StorageSize / CharWidth, BitOffset: BFI.Offset, BitWidth: BFI.Size, CharWidth,
3995 BigEndian: CGM.getDataLayout().isBigEndian());
3996 continue;
3997 }
3998
3999 setUsedBits(CGM, F->getType(),
4000 Offset + ASTLayout.getFieldOffset(FieldNo: Idx) / CharWidth, Bits);
4001 }
4002}
4003
4004// Set the bits in `Bits`, which correspond to the value representations of
4005// the elements of an array type `ATy`.
4006static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
4007 int Offset, SmallVectorImpl<uint64_t> &Bits) {
4008 const ASTContext &Context = CGM.getContext();
4009
4010 QualType ETy = Context.getBaseElementType(VAT: ATy);
4011 int Size = Context.getTypeSizeInChars(T: ETy).getQuantity();
4012 SmallVector<uint64_t, 4> TmpBits(Size);
4013 setUsedBits(CGM, ETy, 0, TmpBits);
4014
4015 for (int I = 0, N = Context.getConstantArrayElementCount(CA: ATy); I < N; ++I) {
4016 auto Src = TmpBits.begin();
4017 auto Dst = Bits.begin() + Offset + I * Size;
4018 for (int J = 0; J < Size; ++J)
4019 *Dst++ |= *Src++;
4020 }
4021}
4022
4023// Set the bits in `Bits`, which correspond to the value representations of
4024// the type `QTy`.
4025static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
4026 SmallVectorImpl<uint64_t> &Bits) {
4027 if (const auto *RTy = QTy->getAsCanonical<RecordType>())
4028 return setUsedBits(CGM, RTy, Offset, Bits);
4029
4030 ASTContext &Context = CGM.getContext();
4031 if (const auto *ATy = Context.getAsConstantArrayType(T: QTy))
4032 return setUsedBits(CGM, ATy, Offset, Bits);
4033
4034 int Size = Context.getTypeSizeInChars(T: QTy).getQuantity();
4035 if (Size <= 0)
4036 return;
4037
4038 std::fill_n(first: Bits.begin() + Offset, n: Size,
4039 value: (uint64_t(1) << Context.getCharWidth()) - 1);
4040}
4041
4042static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
4043 int Pos, int Size, int CharWidth,
4044 bool BigEndian) {
4045 assert(Size > 0);
4046 uint64_t Mask = 0;
4047 if (BigEndian) {
4048 for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
4049 ++P)
4050 Mask = (Mask << CharWidth) | *P;
4051 } else {
4052 auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
4053 do
4054 Mask = (Mask << CharWidth) | *--P;
4055 while (P != End);
4056 }
4057 return Mask;
4058}
4059
4060// Emit code to clear the bits in a record, which aren't a part of any user
4061// declared member, when the record is a function return.
4062llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
4063 llvm::IntegerType *ITy,
4064 QualType QTy) {
4065 assert(Src->getType() == ITy);
4066 assert(ITy->getScalarSizeInBits() <= 64);
4067
4068 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
4069 int Size = DataLayout.getTypeStoreSize(Ty: ITy);
4070 SmallVector<uint64_t, 4> Bits(Size);
4071 setUsedBits(CGM, RTy: QTy->castAsCanonical<RecordType>(), Offset: 0, Bits);
4072
4073 int CharWidth = CGM.getContext().getCharWidth();
4074 uint64_t Mask =
4075 buildMultiCharMask(Bits, Pos: 0, Size, CharWidth, BigEndian: DataLayout.isBigEndian());
4076
4077 return Builder.CreateAnd(LHS: Src, RHS: Mask, Name: "cmse.clear");
4078}
4079
4080// Emit code to clear the bits in a record, which aren't a part of any user
4081// declared member, when the record is a function argument.
4082llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
4083 llvm::ArrayType *ATy,
4084 QualType QTy) {
4085 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
4086 int Size = DataLayout.getTypeStoreSize(Ty: ATy);
4087 SmallVector<uint64_t, 16> Bits(Size);
4088 setUsedBits(CGM, RTy: QTy->castAsCanonical<RecordType>(), Offset: 0, Bits);
4089
4090 // Clear each element of the LLVM array.
4091 int CharWidth = CGM.getContext().getCharWidth();
4092 int CharsPerElt =
4093 ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
4094 int MaskIndex = 0;
4095 llvm::Value *R = llvm::PoisonValue::get(T: ATy);
4096 for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
4097 uint64_t Mask = buildMultiCharMask(Bits, Pos: MaskIndex, Size: CharsPerElt, CharWidth,
4098 BigEndian: DataLayout.isBigEndian());
4099 MaskIndex += CharsPerElt;
4100 llvm::Value *T0 = Builder.CreateExtractValue(Agg: Src, Idxs: I);
4101 llvm::Value *T1 = Builder.CreateAnd(LHS: T0, RHS: Mask, Name: "cmse.clear");
4102 R = Builder.CreateInsertValue(Agg: R, Val: T1, Idxs: I);
4103 }
4104
4105 return R;
4106}
4107
4108void CodeGenFunction::EmitFunctionEpilog(
4109 const CGFunctionInfo &FI, bool EmitRetDbgLoc, SourceLocation EndLoc,
4110 uint64_t RetKeyInstructionsSourceAtom) {
4111 if (FI.isNoReturn()) {
4112 // Noreturn functions don't return.
4113 EmitUnreachable(Loc: EndLoc);
4114 return;
4115 }
4116
4117 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {
4118 // Naked functions don't have epilogues.
4119 Builder.CreateUnreachable();
4120 return;
4121 }
4122
4123 // Functions with no result always return void.
4124 if (!ReturnValue.isValid()) {
4125 auto *I = Builder.CreateRetVoid();
4126 if (RetKeyInstructionsSourceAtom)
4127 addInstToSpecificSourceAtom(KeyInstruction: I, Backup: nullptr, Atom: RetKeyInstructionsSourceAtom);
4128 else
4129 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4130 return;
4131 }
4132
4133 llvm::DebugLoc RetDbgLoc;
4134 llvm::Value *RV = nullptr;
4135 QualType RetTy = FI.getReturnType();
4136 const ABIArgInfo &RetAI = FI.getReturnInfo();
4137
4138 switch (RetAI.getKind()) {
4139 case ABIArgInfo::InAlloca:
4140 // Aggregates get evaluated directly into the destination. Sometimes we
4141 // need to return the sret value in a register, though.
4142 assert(hasAggregateEvaluationKind(RetTy));
4143 if (RetAI.getInAllocaSRet()) {
4144 llvm::Function::arg_iterator EI = CurFn->arg_end();
4145 --EI;
4146 llvm::Value *ArgStruct = &*EI;
4147 llvm::Value *SRet = Builder.CreateStructGEP(
4148 Ty: FI.getArgStruct(), Ptr: ArgStruct, Idx: RetAI.getInAllocaFieldIndex());
4149 llvm::Type *Ty =
4150 cast<llvm::GetElementPtrInst>(Val: SRet)->getResultElementType();
4151 RV = Builder.CreateAlignedLoad(Ty, Addr: SRet, Align: getPointerAlign(), Name: "sret");
4152 }
4153 break;
4154
4155 case ABIArgInfo::Indirect: {
4156 auto AI = CurFn->arg_begin();
4157 if (RetAI.isSRetAfterThis())
4158 ++AI;
4159 switch (getEvaluationKind(T: RetTy)) {
4160 case TEK_Complex: {
4161 ComplexPairTy RT =
4162 EmitLoadOfComplex(src: MakeAddrLValue(Addr: ReturnValue, T: RetTy), loc: EndLoc);
4163 EmitStoreOfComplex(V: RT, dest: MakeNaturalAlignAddrLValue(V: &*AI, T: RetTy),
4164 /*isInit*/ true);
4165 break;
4166 }
4167 case TEK_Aggregate:
4168 // Do nothing; aggregates get evaluated directly into the destination.
4169 break;
4170 case TEK_Scalar: {
4171 LValueBaseInfo BaseInfo;
4172 TBAAAccessInfo TBAAInfo;
4173 CharUnits Alignment =
4174 CGM.getNaturalTypeAlignment(T: RetTy, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
4175 Address ArgAddr(&*AI, ConvertType(T: RetTy), Alignment);
4176 LValue ArgVal =
4177 LValue::MakeAddr(Addr: ArgAddr, type: RetTy, Context&: getContext(), BaseInfo, TBAAInfo);
4178 EmitStoreOfScalar(
4179 value: EmitLoadOfScalar(lvalue: MakeAddrLValue(Addr: ReturnValue, T: RetTy), Loc: EndLoc), lvalue: ArgVal,
4180 /*isInit*/ true);
4181 break;
4182 }
4183 }
4184 break;
4185 }
4186
4187 case ABIArgInfo::Extend:
4188 case ABIArgInfo::Direct:
4189 if (RetAI.getCoerceToType() == ConvertType(T: RetTy) &&
4190 RetAI.getDirectOffset() == 0) {
4191 // The internal return value temp always will have pointer-to-return-type
4192 // type, just do a load.
4193
4194 // If there is a dominating store to ReturnValue, we can elide
4195 // the load, zap the store, and usually zap the alloca.
4196 if (llvm::StoreInst *SI = findDominatingStoreToReturnValue(CGF&: *this)) {
4197 // Reuse the debug location from the store unless there is
4198 // cleanup code to be emitted between the store and return
4199 // instruction.
4200 if (EmitRetDbgLoc && !AutoreleaseResult)
4201 RetDbgLoc = SI->getDebugLoc();
4202 // Get the stored value and nuke the now-dead store.
4203 RV = SI->getValueOperand();
4204 SI->eraseFromParent();
4205
4206 // Otherwise, we have to do a simple load.
4207 } else {
4208 RV = Builder.CreateLoad(Addr: ReturnValue);
4209 }
4210 } else {
4211 // If the value is offset in memory, apply the offset now.
4212 Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
4213
4214 RV = CreateCoercedLoad(Src: V, SrcFETy: RetTy, Ty: RetAI.getCoerceToType(), CGF&: *this);
4215 }
4216
4217 // In ARC, end functions that return a retainable type with a call
4218 // to objc_autoreleaseReturnValue.
4219 if (AutoreleaseResult) {
4220#ifndef NDEBUG
4221 // Type::isObjCRetainabletype has to be called on a QualType that hasn't
4222 // been stripped of the typedefs, so we cannot use RetTy here. Get the
4223 // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
4224 // CurCodeDecl or BlockInfo.
4225 QualType RT;
4226
4227 if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
4228 RT = FD->getReturnType();
4229 else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
4230 RT = MD->getReturnType();
4231 else if (isa<BlockDecl>(CurCodeDecl))
4232 RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
4233 else
4234 llvm_unreachable("Unexpected function/method type");
4235
4236 assert(getLangOpts().ObjCAutoRefCount && !FI.isReturnsRetained() &&
4237 RT->isObjCRetainableType());
4238#endif
4239 RV = emitAutoreleaseOfResult(CGF&: *this, result: RV);
4240 }
4241
4242 break;
4243
4244 case ABIArgInfo::Ignore:
4245 break;
4246
4247 case ABIArgInfo::CoerceAndExpand: {
4248 auto coercionType = RetAI.getCoerceAndExpandType();
4249 auto unpaddedCoercionType = RetAI.getUnpaddedCoerceAndExpandType();
4250 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
4251
4252 // Load all of the coerced elements out into results.
4253 llvm::SmallVector<llvm::Value *, 4> results;
4254 Address addr = ReturnValue.withElementType(ElemTy: coercionType);
4255 unsigned unpaddedIndex = 0;
4256 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
4257 auto coercedEltType = coercionType->getElementType(N: i);
4258 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType: coercedEltType))
4259 continue;
4260
4261 auto eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
4262 llvm::Value *elt = CreateCoercedLoad(
4263 Src: eltAddr, SrcFETy: RetTy,
4264 Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
4265 : unpaddedCoercionType,
4266 CGF&: *this);
4267 results.push_back(Elt: elt);
4268 }
4269
4270 // If we have one result, it's the single direct result type.
4271 if (results.size() == 1) {
4272 RV = results[0];
4273
4274 // Otherwise, we need to make a first-class aggregate.
4275 } else {
4276 // Construct a return type that lacks padding elements.
4277 llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
4278
4279 RV = llvm::PoisonValue::get(T: returnType);
4280 for (unsigned i = 0, e = results.size(); i != e; ++i) {
4281 RV = Builder.CreateInsertValue(Agg: RV, Val: results[i], Idxs: i);
4282 }
4283 }
4284 break;
4285 }
4286 case ABIArgInfo::TargetSpecific: {
4287 Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
4288 RV = CGM.getABIInfo().createCoercedLoad(SrcAddr: V, AI: RetAI, CGF&: *this);
4289 break;
4290 }
4291 case ABIArgInfo::Expand:
4292 case ABIArgInfo::IndirectAliased:
4293 llvm_unreachable("Invalid ABI kind for return argument");
4294 }
4295
4296 llvm::Instruction *Ret;
4297 if (RV) {
4298 if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) {
4299 // For certain return types, clear padding bits, as they may reveal
4300 // sensitive information.
4301 // Small struct/union types are passed as integers.
4302 auto *ITy = dyn_cast<llvm::IntegerType>(Val: RV->getType());
4303 if (ITy != nullptr && isa<RecordType>(Val: RetTy.getCanonicalType()))
4304 RV = EmitCMSEClearRecord(Src: RV, ITy, QTy: RetTy);
4305 }
4306 EmitReturnValueCheck(RV);
4307 Ret = Builder.CreateRet(V: RV);
4308 } else {
4309 Ret = Builder.CreateRetVoid();
4310 }
4311
4312 if (RetDbgLoc)
4313 Ret->setDebugLoc(std::move(RetDbgLoc));
4314
4315 llvm::Value *Backup = RV ? Ret->getOperand(i: 0) : nullptr;
4316 if (RetKeyInstructionsSourceAtom)
4317 addInstToSpecificSourceAtom(KeyInstruction: Ret, Backup, Atom: RetKeyInstructionsSourceAtom);
4318 else
4319 addInstToNewSourceAtom(KeyInstruction: Ret, Backup);
4320}
4321
4322void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
4323 // A current decl may not be available when emitting vtable thunks.
4324 if (!CurCodeDecl)
4325 return;
4326
4327 // If the return block isn't reachable, neither is this check, so don't emit
4328 // it.
4329 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty())
4330 return;
4331
4332 ReturnsNonNullAttr *RetNNAttr = nullptr;
4333 if (SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute))
4334 RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
4335
4336 if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
4337 return;
4338
4339 // Prefer the returns_nonnull attribute if it's present.
4340 SourceLocation AttrLoc;
4341 SanitizerKind::SanitizerOrdinal CheckKind;
4342 SanitizerHandler Handler;
4343 if (RetNNAttr) {
4344 assert(!requiresReturnValueNullabilityCheck() &&
4345 "Cannot check nullability and the nonnull attribute");
4346 AttrLoc = RetNNAttr->getLocation();
4347 CheckKind = SanitizerKind::SO_ReturnsNonnullAttribute;
4348 Handler = SanitizerHandler::NonnullReturn;
4349 } else {
4350 if (auto *DD = dyn_cast<DeclaratorDecl>(Val: CurCodeDecl))
4351 if (auto *TSI = DD->getTypeSourceInfo())
4352 if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
4353 AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
4354 CheckKind = SanitizerKind::SO_NullabilityReturn;
4355 Handler = SanitizerHandler::NullabilityReturn;
4356 }
4357
4358 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4359
4360 // Make sure the "return" source location is valid. If we're checking a
4361 // nullability annotation, make sure the preconditions for the check are met.
4362 llvm::BasicBlock *Check = createBasicBlock(name: "nullcheck");
4363 llvm::BasicBlock *NoCheck = createBasicBlock(name: "no.nullcheck");
4364 llvm::Value *SLocPtr = Builder.CreateLoad(Addr: ReturnLocation, Name: "return.sloc.load");
4365 llvm::Value *CanNullCheck = Builder.CreateIsNotNull(Arg: SLocPtr);
4366 if (requiresReturnValueNullabilityCheck())
4367 CanNullCheck =
4368 Builder.CreateAnd(LHS: CanNullCheck, RHS: RetValNullabilityPrecondition);
4369 Builder.CreateCondBr(Cond: CanNullCheck, True: Check, False: NoCheck);
4370 EmitBlock(BB: Check);
4371
4372 // Now do the null check.
4373 llvm::Value *Cond = Builder.CreateIsNotNull(Arg: RV);
4374 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc: AttrLoc)};
4375 llvm::Value *DynamicData[] = {SLocPtr};
4376 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: DynamicData);
4377
4378 EmitBlock(BB: NoCheck);
4379
4380#ifndef NDEBUG
4381 // The return location should not be used after the check has been emitted.
4382 ReturnLocation = Address::invalid();
4383#endif
4384}
4385
4386static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
4387 const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
4388 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
4389}
4390
4391static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, QualType Ty) {
4392 // FIXME: Generate IR in one pass, rather than going back and fixing up these
4393 // placeholders.
4394 llvm::Type *IRTy = CGF.ConvertTypeForMem(T: Ty);
4395 llvm::Type *IRPtrTy = llvm::PointerType::getUnqual(C&: CGF.getLLVMContext());
4396 llvm::Value *Placeholder = llvm::PoisonValue::get(T: IRPtrTy);
4397
4398 // FIXME: When we generate this IR in one pass, we shouldn't need
4399 // this win32-specific alignment hack.
4400 CharUnits Align = CharUnits::fromQuantity(Quantity: 4);
4401 Placeholder = CGF.Builder.CreateAlignedLoad(Ty: IRPtrTy, Addr: Placeholder, Align);
4402
4403 return AggValueSlot::forAddr(
4404 addr: Address(Placeholder, IRTy, Align), quals: Ty.getQualifiers(),
4405 isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
4406 isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
4407}
4408
4409void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
4410 const VarDecl *param,
4411 SourceLocation loc) {
4412 // StartFunction converted the ABI-lowered parameter(s) into a
4413 // local alloca. We need to turn that into an r-value suitable
4414 // for EmitCall.
4415 Address local = GetAddrOfLocalVar(VD: param);
4416
4417 QualType type = param->getType();
4418
4419 // GetAddrOfLocalVar returns a pointer-to-pointer for references,
4420 // but the argument needs to be the original pointer.
4421 if (type->isReferenceType()) {
4422 args.add(rvalue: RValue::get(V: Builder.CreateLoad(Addr: local)), type);
4423
4424 // In ARC, move out of consumed arguments so that the release cleanup
4425 // entered by StartFunction doesn't cause an over-release. This isn't
4426 // optimal -O0 code generation, but it should get cleaned up when
4427 // optimization is enabled. This also assumes that delegate calls are
4428 // performed exactly once for a set of arguments, but that should be safe.
4429 } else if (getLangOpts().ObjCAutoRefCount &&
4430 param->hasAttr<NSConsumedAttr>() && type->isObjCRetainableType()) {
4431 llvm::Value *ptr = Builder.CreateLoad(Addr: local);
4432 auto null =
4433 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: ptr->getType()));
4434 Builder.CreateStore(Val: null, Addr: local);
4435 args.add(rvalue: RValue::get(V: ptr), type);
4436
4437 // For the most part, we just need to load the alloca, except that
4438 // aggregate r-values are actually pointers to temporaries.
4439 } else {
4440 args.add(rvalue: convertTempToRValue(addr: local, type, Loc: loc), type);
4441 }
4442
4443 // Deactivate the cleanup for the callee-destructed param that was pushed.
4444 if (type->isRecordType() && !CurFuncIsThunk &&
4445 type->castAsRecordDecl()->isParamDestroyedInCallee() &&
4446 param->needsDestruction(Ctx: getContext())) {
4447 EHScopeStack::stable_iterator cleanup =
4448 CalleeDestructedParamCleanups.lookup(Val: cast<ParmVarDecl>(Val: param));
4449 assert(cleanup.isValid() &&
4450 "cleanup for callee-destructed param not recorded");
4451 // This unreachable is a temporary marker which will be removed later.
4452 llvm::Instruction *isActive = Builder.CreateUnreachable();
4453 args.addArgCleanupDeactivation(Cleanup: cleanup, IsActiveIP: isActive);
4454 }
4455}
4456
4457static bool isProvablyNull(llvm::Value *addr) {
4458 return llvm::isa_and_nonnull<llvm::ConstantPointerNull>(Val: addr);
4459}
4460
4461static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) {
4462 return llvm::isKnownNonZero(V: Addr.getBasePointer(), Q: CGF.CGM.getDataLayout());
4463}
4464
4465/// Emit the actual writing-back of a writeback.
4466static void emitWriteback(CodeGenFunction &CGF,
4467 const CallArgList::Writeback &writeback) {
4468 const LValue &srcLV = writeback.Source;
4469 Address srcAddr = srcLV.getAddress();
4470 assert(!isProvablyNull(srcAddr.getBasePointer()) &&
4471 "shouldn't have writeback for provably null argument");
4472
4473 if (writeback.WritebackExpr) {
4474 CGF.EmitIgnoredExpr(E: writeback.WritebackExpr);
4475 CGF.EmitLifetimeEnd(Addr: writeback.Temporary.getBasePointer());
4476 return;
4477 }
4478
4479 llvm::BasicBlock *contBB = nullptr;
4480
4481 // If the argument wasn't provably non-null, we need to null check
4482 // before doing the store.
4483 bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4484
4485 if (!provablyNonNull) {
4486 llvm::BasicBlock *writebackBB = CGF.createBasicBlock(name: "icr.writeback");
4487 contBB = CGF.createBasicBlock(name: "icr.done");
4488
4489 llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4490 CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: writebackBB);
4491 CGF.EmitBlock(BB: writebackBB);
4492 }
4493
4494 // Load the value to writeback.
4495 llvm::Value *value = CGF.Builder.CreateLoad(Addr: writeback.Temporary);
4496
4497 // Cast it back, in case we're writing an id to a Foo* or something.
4498 value = CGF.Builder.CreateBitCast(V: value, DestTy: srcAddr.getElementType(),
4499 Name: "icr.writeback-cast");
4500
4501 // Perform the writeback.
4502
4503 // If we have a "to use" value, it's something we need to emit a use
4504 // of. This has to be carefully threaded in: if it's done after the
4505 // release it's potentially undefined behavior (and the optimizer
4506 // will ignore it), and if it happens before the retain then the
4507 // optimizer could move the release there.
4508 if (writeback.ToUse) {
4509 assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
4510
4511 // Retain the new value. No need to block-copy here: the block's
4512 // being passed up the stack.
4513 value = CGF.EmitARCRetainNonBlock(value);
4514
4515 // Emit the intrinsic use here.
4516 CGF.EmitARCIntrinsicUse(values: writeback.ToUse);
4517
4518 // Load the old value (primitively).
4519 llvm::Value *oldValue = CGF.EmitLoadOfScalar(lvalue: srcLV, Loc: SourceLocation());
4520
4521 // Put the new value in place (primitively).
4522 CGF.EmitStoreOfScalar(value, lvalue: srcLV, /*init*/ isInit: false);
4523
4524 // Release the old value.
4525 CGF.EmitARCRelease(value: oldValue, precise: srcLV.isARCPreciseLifetime());
4526
4527 // Otherwise, we can just do a normal lvalue store.
4528 } else {
4529 CGF.EmitStoreThroughLValue(Src: RValue::get(V: value), Dst: srcLV);
4530 }
4531
4532 // Jump to the continuation block.
4533 if (!provablyNonNull)
4534 CGF.EmitBlock(BB: contBB);
4535}
4536
4537static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
4538 const CallArgList &CallArgs) {
4539 ArrayRef<CallArgList::CallArgCleanup> Cleanups =
4540 CallArgs.getCleanupsToDeactivate();
4541 // Iterate in reverse to increase the likelihood of popping the cleanup.
4542 for (const auto &I : llvm::reverse(C&: Cleanups)) {
4543 CGF.DeactivateCleanupBlock(Cleanup: I.Cleanup, DominatingIP: I.IsActiveIP);
4544 I.IsActiveIP->eraseFromParent();
4545 }
4546}
4547
4548static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
4549 if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(Val: E->IgnoreParens()))
4550 if (uop->getOpcode() == UO_AddrOf)
4551 return uop->getSubExpr();
4552 return nullptr;
4553}
4554
4555/// Emit an argument that's being passed call-by-writeback. That is,
4556/// we are passing the address of an __autoreleased temporary; it
4557/// might be copy-initialized with the current value of the given
4558/// address, but it will definitely be copied out of after the call.
4559static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
4560 const ObjCIndirectCopyRestoreExpr *CRE) {
4561 LValue srcLV;
4562
4563 // Make an optimistic effort to emit the address as an l-value.
4564 // This can fail if the argument expression is more complicated.
4565 if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(E: CRE->getSubExpr())) {
4566 srcLV = CGF.EmitLValue(E: lvExpr);
4567
4568 // Otherwise, just emit it as a scalar.
4569 } else {
4570 Address srcAddr = CGF.EmitPointerWithAlignment(Addr: CRE->getSubExpr());
4571
4572 QualType srcAddrType =
4573 CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
4574 srcLV = CGF.MakeAddrLValue(Addr: srcAddr, T: srcAddrType);
4575 }
4576 Address srcAddr = srcLV.getAddress();
4577
4578 // The dest and src types don't necessarily match in LLVM terms
4579 // because of the crazy ObjC compatibility rules.
4580
4581 llvm::PointerType *destType =
4582 cast<llvm::PointerType>(Val: CGF.ConvertType(T: CRE->getType()));
4583 llvm::Type *destElemType =
4584 CGF.ConvertTypeForMem(T: CRE->getType()->getPointeeType());
4585
4586 // If the address is a constant null, just pass the appropriate null.
4587 if (isProvablyNull(addr: srcAddr.getBasePointer())) {
4588 args.add(rvalue: RValue::get(V: llvm::ConstantPointerNull::get(T: destType)),
4589 type: CRE->getType());
4590 return;
4591 }
4592
4593 // Create the temporary.
4594 Address temp =
4595 CGF.CreateTempAlloca(Ty: destElemType, align: CGF.getPointerAlign(), Name: "icr.temp");
4596 // Loading an l-value can introduce a cleanup if the l-value is __weak,
4597 // and that cleanup will be conditional if we can't prove that the l-value
4598 // isn't null, so we need to register a dominating point so that the cleanups
4599 // system will make valid IR.
4600 CodeGenFunction::ConditionalEvaluation condEval(CGF);
4601
4602 // Zero-initialize it if we're not doing a copy-initialization.
4603 bool shouldCopy = CRE->shouldCopy();
4604 if (!shouldCopy) {
4605 llvm::Value *null =
4606 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: destElemType));
4607 CGF.Builder.CreateStore(Val: null, Addr: temp);
4608 }
4609
4610 llvm::BasicBlock *contBB = nullptr;
4611 llvm::BasicBlock *originBB = nullptr;
4612
4613 // If the address is *not* known to be non-null, we need to switch.
4614 llvm::Value *finalArgument;
4615
4616 bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4617
4618 if (provablyNonNull) {
4619 finalArgument = temp.emitRawPointer(CGF);
4620 } else {
4621 llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4622
4623 finalArgument = CGF.Builder.CreateSelect(
4624 C: isNull, True: llvm::ConstantPointerNull::get(T: destType),
4625 False: temp.emitRawPointer(CGF), Name: "icr.argument");
4626
4627 // If we need to copy, then the load has to be conditional, which
4628 // means we need control flow.
4629 if (shouldCopy) {
4630 originBB = CGF.Builder.GetInsertBlock();
4631 contBB = CGF.createBasicBlock(name: "icr.cont");
4632 llvm::BasicBlock *copyBB = CGF.createBasicBlock(name: "icr.copy");
4633 CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: copyBB);
4634 CGF.EmitBlock(BB: copyBB);
4635 condEval.begin(CGF);
4636 }
4637 }
4638
4639 llvm::Value *valueToUse = nullptr;
4640
4641 // Perform a copy if necessary.
4642 if (shouldCopy) {
4643 RValue srcRV = CGF.EmitLoadOfLValue(V: srcLV, Loc: SourceLocation());
4644 assert(srcRV.isScalar());
4645
4646 llvm::Value *src = srcRV.getScalarVal();
4647 src = CGF.Builder.CreateBitCast(V: src, DestTy: destElemType, Name: "icr.cast");
4648
4649 // Use an ordinary store, not a store-to-lvalue.
4650 CGF.Builder.CreateStore(Val: src, Addr: temp);
4651
4652 // If optimization is enabled, and the value was held in a
4653 // __strong variable, we need to tell the optimizer that this
4654 // value has to stay alive until we're doing the store back.
4655 // This is because the temporary is effectively unretained,
4656 // and so otherwise we can violate the high-level semantics.
4657 if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4658 srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {
4659 valueToUse = src;
4660 }
4661 }
4662
4663 // Finish the control flow if we needed it.
4664 if (shouldCopy && !provablyNonNull) {
4665 llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
4666 CGF.EmitBlock(BB: contBB);
4667
4668 // Make a phi for the value to intrinsically use.
4669 if (valueToUse) {
4670 llvm::PHINode *phiToUse =
4671 CGF.Builder.CreatePHI(Ty: valueToUse->getType(), NumReservedValues: 2, Name: "icr.to-use");
4672 phiToUse->addIncoming(V: valueToUse, BB: copyBB);
4673 phiToUse->addIncoming(V: llvm::PoisonValue::get(T: valueToUse->getType()),
4674 BB: originBB);
4675 valueToUse = phiToUse;
4676 }
4677
4678 condEval.end(CGF);
4679 }
4680
4681 args.addWriteback(srcLV, temporary: temp, toUse: valueToUse);
4682 args.add(rvalue: RValue::get(V: finalArgument), type: CRE->getType());
4683}
4684
4685void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
4686 assert(!StackBase);
4687
4688 // Save the stack.
4689 StackBase = CGF.Builder.CreateStackSave(Name: "inalloca.save");
4690}
4691
4692void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
4693 if (StackBase) {
4694 // Restore the stack after the call.
4695 CGF.Builder.CreateStackRestore(Ptr: StackBase);
4696 }
4697}
4698
4699void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
4700 SourceLocation ArgLoc,
4701 AbstractCallee AC, unsigned ParmNum) {
4702 if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
4703 SanOpts.has(K: SanitizerKind::NullabilityArg)))
4704 return;
4705
4706 // The param decl may be missing in a variadic function.
4707 auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(I: ParmNum) : nullptr;
4708 unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
4709
4710 // Prefer the nonnull attribute if it's present.
4711 const NonNullAttr *NNAttr = nullptr;
4712 if (SanOpts.has(K: SanitizerKind::NonnullAttribute))
4713 NNAttr = getNonNullAttr(FD: AC.getDecl(), PVD, ArgType, ArgNo);
4714
4715 bool CanCheckNullability = false;
4716 if (SanOpts.has(K: SanitizerKind::NullabilityArg) && !NNAttr && PVD &&
4717 !PVD->getType()->isRecordType()) {
4718 auto Nullability = PVD->getType()->getNullability();
4719 CanCheckNullability = Nullability &&
4720 *Nullability == NullabilityKind::NonNull &&
4721 PVD->getTypeSourceInfo();
4722 }
4723
4724 if (!NNAttr && !CanCheckNullability)
4725 return;
4726
4727 SourceLocation AttrLoc;
4728 SanitizerKind::SanitizerOrdinal CheckKind;
4729 SanitizerHandler Handler;
4730 if (NNAttr) {
4731 AttrLoc = NNAttr->getLocation();
4732 CheckKind = SanitizerKind::SO_NonnullAttribute;
4733 Handler = SanitizerHandler::NonnullArg;
4734 } else {
4735 AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4736 CheckKind = SanitizerKind::SO_NullabilityArg;
4737 Handler = SanitizerHandler::NullabilityArg;
4738 }
4739
4740 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4741 llvm::Value *Cond = EmitNonNullRValueCheck(RV, T: ArgType);
4742 llvm::Constant *StaticData[] = {
4743 EmitCheckSourceLocation(Loc: ArgLoc),
4744 EmitCheckSourceLocation(Loc: AttrLoc),
4745 llvm::ConstantInt::get(Ty: Int32Ty, V: ArgNo + 1),
4746 };
4747 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: {});
4748}
4749
4750void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType,
4751 SourceLocation ArgLoc,
4752 AbstractCallee AC, unsigned ParmNum) {
4753 if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
4754 SanOpts.has(K: SanitizerKind::NullabilityArg)))
4755 return;
4756
4757 EmitNonNullArgCheck(RV: RValue::get(Addr, CGF&: *this), ArgType, ArgLoc, AC, ParmNum);
4758}
4759
4760// Check if the call is going to use the inalloca convention. This needs to
4761// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
4762// later, so we can't check it directly.
4763static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
4764 ArrayRef<QualType> ArgTypes) {
4765 // The Swift calling conventions don't go through the target-specific
4766 // argument classification, they never use inalloca.
4767 // TODO: Consider limiting inalloca use to only calling conventions supported
4768 // by MSVC.
4769 if (ExplicitCC == CC_Swift || ExplicitCC == CC_SwiftAsync)
4770 return false;
4771 if (!CGM.getTarget().getCXXABI().isMicrosoft())
4772 return false;
4773 return llvm::any_of(Range&: ArgTypes, P: [&](QualType Ty) {
4774 return isInAllocaArgument(ABI&: CGM.getCXXABI(), type: Ty);
4775 });
4776}
4777
4778#ifndef NDEBUG
4779// Determine whether the given argument is an Objective-C method
4780// that may have type parameters in its signature.
4781static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
4782 const DeclContext *dc = method->getDeclContext();
4783 if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {
4784 return classDecl->getTypeParamListAsWritten();
4785 }
4786
4787 if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
4788 return catDecl->getTypeParamList();
4789 }
4790
4791 return false;
4792}
4793#endif
4794
4795/// EmitCallArgs - Emit call arguments for a function.
4796void CodeGenFunction::EmitCallArgs(
4797 CallArgList &Args, PrototypeWrapper Prototype,
4798 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4799 AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
4800 SmallVector<QualType, 16> ArgTypes;
4801
4802 assert((ParamsToSkip == 0 || Prototype.P) &&
4803 "Can't skip parameters if type info is not provided");
4804
4805 // This variable only captures *explicitly* written conventions, not those
4806 // applied by default via command line flags or target defaults, such as
4807 // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
4808 // require knowing if this is a C++ instance method or being able to see
4809 // unprototyped FunctionTypes.
4810 CallingConv ExplicitCC = CC_C;
4811
4812 // First, if a prototype was provided, use those argument types.
4813 bool IsVariadic = false;
4814 if (Prototype.P) {
4815 const auto *MD = dyn_cast<const ObjCMethodDecl *>(Val&: Prototype.P);
4816 if (MD) {
4817 IsVariadic = MD->isVariadic();
4818 ExplicitCC = getCallingConventionForDecl(
4819 D: MD, IsTargetDefaultMSABI: CGM.getTarget().getTriple().isOSWindows());
4820 ArgTypes.assign(in_start: MD->param_type_begin() + ParamsToSkip,
4821 in_end: MD->param_type_end());
4822 } else {
4823 const auto *FPT = cast<const FunctionProtoType *>(Val&: Prototype.P);
4824 IsVariadic = FPT->isVariadic();
4825 ExplicitCC = FPT->getExtInfo().getCC();
4826 ArgTypes.assign(in_start: FPT->param_type_begin() + ParamsToSkip,
4827 in_end: FPT->param_type_end());
4828 }
4829
4830#ifndef NDEBUG
4831 // Check that the prototyped types match the argument expression types.
4832 bool isGenericMethod = MD && isObjCMethodWithTypeParams(MD);
4833 CallExpr::const_arg_iterator Arg = ArgRange.begin();
4834 for (QualType Ty : ArgTypes) {
4835 assert(Arg != ArgRange.end() && "Running over edge of argument list!");
4836 QualType ParamTy = Ty.getNonReferenceType();
4837 QualType ArgTy = (*Arg)->getType();
4838 if (const auto *OBT = ParamTy->getAs<OverflowBehaviorType>())
4839 ParamTy = OBT->getUnderlyingType();
4840 if (const auto *OBT = ArgTy->getAs<OverflowBehaviorType>())
4841 ArgTy = OBT->getUnderlyingType();
4842 assert((isGenericMethod || Ty->isVariablyModifiedType() ||
4843 ParamTy->isObjCRetainableType() ||
4844 getContext().getCanonicalType(ParamTy).getTypePtr() ==
4845 getContext().getCanonicalType(ArgTy).getTypePtr()) &&
4846 "type mismatch in call argument!");
4847 ++Arg;
4848 }
4849
4850 // Either we've emitted all the call args, or we have a call to variadic
4851 // function.
4852 assert((Arg == ArgRange.end() || IsVariadic) &&
4853 "Extra arguments in non-variadic function!");
4854#endif
4855 }
4856
4857 // If we still have any arguments, emit them using the type of the argument.
4858 for (auto *A : llvm::drop_begin(RangeOrContainer&: ArgRange, N: ArgTypes.size()))
4859 ArgTypes.push_back(Elt: IsVariadic ? getVarArgType(Arg: A) : A->getType());
4860 assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
4861
4862 // We must evaluate arguments from right to left in the MS C++ ABI,
4863 // because arguments are destroyed left to right in the callee. As a special
4864 // case, there are certain language constructs that require left-to-right
4865 // evaluation, and in those cases we consider the evaluation order requirement
4866 // to trump the "destruction order is reverse construction order" guarantee.
4867 bool LeftToRight =
4868 CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
4869 ? Order == EvaluationOrder::ForceLeftToRight
4870 : Order != EvaluationOrder::ForceRightToLeft;
4871
4872 auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
4873 RValue EmittedArg) {
4874 if (!AC.hasFunctionDecl() || I >= AC.getNumParams())
4875 return;
4876 auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
4877 if (PS == nullptr)
4878 return;
4879
4880 const auto &Context = getContext();
4881 auto SizeTy = Context.getSizeType();
4882 auto T = Builder.getIntNTy(N: Context.getTypeSize(T: SizeTy));
4883 assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
4884 llvm::Value *V = evaluateOrEmitBuiltinObjectSize(
4885 E: Arg, Type: PS->getType(), ResType: T, EmittedE: EmittedArg.getScalarVal(), IsDynamic: PS->isDynamic());
4886 Args.add(rvalue: RValue::get(V), type: SizeTy);
4887 // If we're emitting args in reverse, be sure to do so with
4888 // pass_object_size, as well.
4889 if (!LeftToRight)
4890 std::swap(a&: Args.back(), b&: *(&Args.back() - 1));
4891 };
4892
4893 // Insert a stack save if we're going to need any inalloca args.
4894 if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
4895 assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
4896 "inalloca only supported on x86");
4897 Args.allocateArgumentMemory(CGF&: *this);
4898 }
4899
4900 // Evaluate each argument in the appropriate order.
4901 size_t CallArgsStart = Args.size();
4902 for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
4903 unsigned Idx = LeftToRight ? I : E - I - 1;
4904 CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
4905 unsigned InitialArgSize = Args.size();
4906 // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
4907 // the argument and parameter match or the objc method is parameterized.
4908 assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
4909 getContext().hasSameUnqualifiedType((*Arg)->getType(),
4910 ArgTypes[Idx]) ||
4911 (isa<ObjCMethodDecl>(AC.getDecl()) &&
4912 isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
4913 "Argument and parameter types don't match");
4914 EmitCallArg(args&: Args, E: *Arg, ArgType: ArgTypes[Idx]);
4915 // In particular, we depend on it being the last arg in Args, and the
4916 // objectsize bits depend on there only being one arg if !LeftToRight.
4917 assert(InitialArgSize + 1 == Args.size() &&
4918 "The code below depends on only adding one arg per EmitCallArg");
4919 (void)InitialArgSize;
4920 // Since pointer argument are never emitted as LValue, it is safe to emit
4921 // non-null argument check for r-value only.
4922 if (!Args.back().hasLValue()) {
4923 RValue RVArg = Args.back().getKnownRValue();
4924 EmitNonNullArgCheck(RV: RVArg, ArgType: ArgTypes[Idx], ArgLoc: (*Arg)->getExprLoc(), AC,
4925 ParmNum: ParamsToSkip + Idx);
4926 // @llvm.objectsize should never have side-effects and shouldn't need
4927 // destruction/cleanups, so we can safely "emit" it after its arg,
4928 // regardless of right-to-leftness
4929 MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
4930 }
4931 }
4932
4933 if (!LeftToRight) {
4934 // Un-reverse the arguments we just evaluated so they match up with the LLVM
4935 // IR function.
4936 std::reverse(first: Args.begin() + CallArgsStart, last: Args.end());
4937
4938 // Reverse the writebacks to match the MSVC ABI.
4939 Args.reverseWritebacks();
4940 }
4941}
4942
4943namespace {
4944
4945struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
4946 DestroyUnpassedArg(Address Addr, QualType Ty) : Addr(Addr), Ty(Ty) {}
4947
4948 Address Addr;
4949 QualType Ty;
4950
4951 void Emit(CodeGenFunction &CGF, Flags flags) override {
4952 QualType::DestructionKind DtorKind = Ty.isDestructedType();
4953 if (DtorKind == QualType::DK_cxx_destructor) {
4954 const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
4955 assert(!Dtor->isTrivial());
4956 CGF.EmitCXXDestructorCall(D: Dtor, Type: Dtor_Complete, /*for vbase*/ ForVirtualBase: false,
4957 /*Delegating=*/false, This: Addr, ThisTy: Ty);
4958 } else {
4959 CGF.callCStructDestructor(Dst: CGF.MakeAddrLValue(Addr, T: Ty));
4960 }
4961 }
4962};
4963
4964} // end anonymous namespace
4965
4966RValue CallArg::getRValue(CodeGenFunction &CGF) const {
4967 if (!HasLV)
4968 return RV;
4969 LValue Copy = CGF.MakeAddrLValue(Addr: CGF.CreateMemTemp(T: Ty), T: Ty);
4970 CGF.EmitAggregateCopy(Dest: Copy, Src: LV, EltTy: Ty, MayOverlap: AggValueSlot::DoesNotOverlap,
4971 isVolatile: LV.isVolatile());
4972 IsUsed = true;
4973 return RValue::getAggregate(addr: Copy.getAddress());
4974}
4975
4976void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
4977 LValue Dst = CGF.MakeAddrLValue(Addr, T: Ty);
4978 if (!HasLV && RV.isScalar())
4979 CGF.EmitStoreOfScalar(value: RV.getScalarVal(), lvalue: Dst, /*isInit=*/true);
4980 else if (!HasLV && RV.isComplex())
4981 CGF.EmitStoreOfComplex(V: RV.getComplexVal(), dest: Dst, /*init=*/isInit: true);
4982 else {
4983 auto Addr = HasLV ? LV.getAddress() : RV.getAggregateAddress();
4984 LValue SrcLV = CGF.MakeAddrLValue(Addr, T: Ty);
4985 // We assume that call args are never copied into subobjects.
4986 CGF.EmitAggregateCopy(Dest: Dst, Src: SrcLV, EltTy: Ty, MayOverlap: AggValueSlot::DoesNotOverlap,
4987 isVolatile: HasLV ? LV.isVolatileQualified()
4988 : RV.isVolatileQualified());
4989 }
4990 IsUsed = true;
4991}
4992
4993void CodeGenFunction::EmitWritebacks(const CallArgList &args) {
4994 for (const auto &I : args.writebacks())
4995 emitWriteback(CGF&: *this, writeback: I);
4996}
4997
4998void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
4999 QualType type) {
5000 std::optional<DisableDebugLocationUpdates> Dis;
5001 if (isa<CXXDefaultArgExpr>(Val: E))
5002 Dis.emplace(args&: *this);
5003 if (const ObjCIndirectCopyRestoreExpr *CRE =
5004 dyn_cast<ObjCIndirectCopyRestoreExpr>(Val: E)) {
5005 assert(getLangOpts().ObjCAutoRefCount);
5006 return emitWritebackArg(CGF&: *this, args, CRE);
5007 }
5008
5009 // Add writeback for HLSLOutParamExpr.
5010 // Needs to be before the assert below because HLSLOutArgExpr is an LValue
5011 // and is not a reference.
5012 if (const HLSLOutArgExpr *OE = dyn_cast<HLSLOutArgExpr>(Val: E)) {
5013 EmitHLSLOutArgExpr(E: OE, Args&: args, Ty: type);
5014 return;
5015 }
5016
5017 assert(type->isReferenceType() == E->isGLValue() &&
5018 "reference binding to unmaterialized r-value!");
5019
5020 if (E->isGLValue()) {
5021 assert(E->getObjectKind() == OK_Ordinary);
5022 return args.add(rvalue: EmitReferenceBindingToExpr(E), type);
5023 }
5024
5025 bool HasAggregateEvalKind = hasAggregateEvaluationKind(T: type);
5026
5027 // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
5028 // However, we still have to push an EH-only cleanup in case we unwind before
5029 // we make it to the call.
5030 if (type->isRecordType() &&
5031 type->castAsRecordDecl()->isParamDestroyedInCallee()) {
5032 // If we're using inalloca, use the argument memory. Otherwise, use a
5033 // temporary.
5034 AggValueSlot Slot = args.isUsingInAlloca()
5035 ? createPlaceholderSlot(CGF&: *this, Ty: type)
5036 : CreateAggTemp(T: type, Name: "agg.tmp");
5037
5038 bool DestroyedInCallee = true, NeedsCleanup = true;
5039 if (const auto *RD = type->getAsCXXRecordDecl())
5040 DestroyedInCallee = RD->hasNonTrivialDestructor();
5041 else
5042 NeedsCleanup = type.isDestructedType();
5043
5044 if (DestroyedInCallee)
5045 Slot.setExternallyDestructed();
5046
5047 EmitAggExpr(E, AS: Slot);
5048 RValue RV = Slot.asRValue();
5049 args.add(rvalue: RV, type);
5050
5051 if (DestroyedInCallee && NeedsCleanup) {
5052 // Create a no-op GEP between the placeholder and the cleanup so we can
5053 // RAUW it successfully. It also serves as a marker of the first
5054 // instruction where the cleanup is active.
5055 pushFullExprCleanup<DestroyUnpassedArg>(kind: NormalAndEHCleanup,
5056 A: Slot.getAddress(), A: type);
5057 // This unreachable is a temporary marker which will be removed later.
5058 llvm::Instruction *IsActive =
5059 Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
5060 args.addArgCleanupDeactivation(Cleanup: EHStack.stable_begin(), IsActiveIP: IsActive);
5061 }
5062 return;
5063 }
5064
5065 if (HasAggregateEvalKind && isa<ImplicitCastExpr>(Val: E) &&
5066 cast<CastExpr>(Val: E)->getCastKind() == CK_LValueToRValue &&
5067 !type->isArrayParameterType() && !type.isNonTrivialToPrimitiveCopy()) {
5068 LValue L = EmitLValue(E: cast<CastExpr>(Val: E)->getSubExpr());
5069 assert(L.isSimple());
5070 args.addUncopiedAggregate(LV: L, type);
5071 return;
5072 }
5073
5074 args.add(rvalue: EmitAnyExprToTemp(E), type);
5075}
5076
5077QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
5078 // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
5079 // implicitly widens null pointer constants that are arguments to varargs
5080 // functions to pointer-sized ints.
5081 if (!getTarget().getTriple().isOSWindows())
5082 return Arg->getType();
5083
5084 if (Arg->getType()->isIntegerType() &&
5085 getContext().getTypeSize(T: Arg->getType()) <
5086 getContext().getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default) &&
5087 Arg->isNullPointerConstant(Ctx&: getContext(),
5088 NPC: Expr::NPC_ValueDependentIsNotNull)) {
5089 return getContext().getIntPtrType();
5090 }
5091
5092 return Arg->getType();
5093}
5094
5095// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5096// optimizer it can aggressively ignore unwind edges.
5097void CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
5098 if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
5099 !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
5100 Inst->setMetadata(Kind: "clang.arc.no_objc_arc_exceptions",
5101 Node: CGM.getNoObjCARCExceptionsMetadata());
5102}
5103
5104/// Emits a call to the given no-arguments nounwind runtime function.
5105llvm::CallInst *
5106CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5107 const llvm::Twine &name) {
5108 return EmitNounwindRuntimeCall(callee, args: ArrayRef<llvm::Value *>(), name);
5109}
5110
5111/// Emits a call to the given nounwind runtime function.
5112llvm::CallInst *
5113CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5114 ArrayRef<Address> args,
5115 const llvm::Twine &name) {
5116 SmallVector<llvm::Value *, 3> values;
5117 for (auto arg : args)
5118 values.push_back(Elt: arg.emitRawPointer(CGF&: *this));
5119 return EmitNounwindRuntimeCall(callee, args: values, name);
5120}
5121
5122llvm::CallInst *
5123CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5124 ArrayRef<llvm::Value *> args,
5125 const llvm::Twine &name) {
5126 llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
5127 call->setDoesNotThrow();
5128 return call;
5129}
5130
5131/// Emits a simple call (never an invoke) to the given no-arguments
5132/// runtime function.
5133llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5134 const llvm::Twine &name) {
5135 return EmitRuntimeCall(callee, args: {}, name);
5136}
5137
5138// Calls which may throw must have operand bundles indicating which funclet
5139// they are nested within.
5140SmallVector<llvm::OperandBundleDef, 1>
5141CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
5142 // There is no need for a funclet operand bundle if we aren't inside a
5143 // funclet.
5144 if (!CurrentFuncletPad)
5145 return (SmallVector<llvm::OperandBundleDef, 1>());
5146
5147 // Skip intrinsics which cannot throw (as long as they don't lower into
5148 // regular function calls in the course of IR transformations).
5149 if (auto *CalleeFn = dyn_cast<llvm::Function>(Val: Callee->stripPointerCasts())) {
5150 if (CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) {
5151 auto IID = CalleeFn->getIntrinsicID();
5152 if (!llvm::IntrinsicInst::mayLowerToFunctionCall(IID))
5153 return (SmallVector<llvm::OperandBundleDef, 1>());
5154 }
5155 }
5156
5157 SmallVector<llvm::OperandBundleDef, 1> BundleList;
5158 BundleList.emplace_back(Args: "funclet", Args&: CurrentFuncletPad);
5159 return BundleList;
5160}
5161
5162/// Emits a simple call (never an invoke) to the given runtime function.
5163llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5164 ArrayRef<llvm::Value *> args,
5165 const llvm::Twine &name) {
5166 llvm::CallInst *call = Builder.CreateCall(
5167 Callee: callee, Args: args, OpBundles: getBundlesForFunclet(Callee: callee.getCallee()), Name: name);
5168 call->setCallingConv(getRuntimeCC());
5169
5170 if (CGM.shouldEmitConvergenceTokens() && call->isConvergent())
5171 return cast<llvm::CallInst>(Val: addConvergenceControlToken(Input: call));
5172 return call;
5173}
5174
5175/// Emits a call or invoke to the given noreturn runtime function.
5176void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
5177 llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
5178 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5179 getBundlesForFunclet(Callee: callee.getCallee());
5180
5181 if (getInvokeDest()) {
5182 llvm::InvokeInst *invoke = Builder.CreateInvoke(
5183 Callee: callee, NormalDest: getUnreachableBlock(), UnwindDest: getInvokeDest(), Args: args, OpBundles: BundleList);
5184 invoke->setDoesNotReturn();
5185 invoke->setCallingConv(getRuntimeCC());
5186 } else {
5187 llvm::CallInst *call = Builder.CreateCall(Callee: callee, Args: args, OpBundles: BundleList);
5188 call->setDoesNotReturn();
5189 call->setCallingConv(getRuntimeCC());
5190 Builder.CreateUnreachable();
5191 }
5192}
5193
5194/// Emits a call or invoke instruction to the given nullary runtime function.
5195llvm::CallBase *
5196CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5197 const Twine &name) {
5198 return EmitRuntimeCallOrInvoke(callee, args: {}, name);
5199}
5200
5201/// Emits a call or invoke instruction to the given runtime function.
5202llvm::CallBase *
5203CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5204 ArrayRef<llvm::Value *> args,
5205 const Twine &name) {
5206 llvm::CallBase *call = EmitCallOrInvoke(Callee: callee, Args: args, Name: name);
5207 call->setCallingConv(getRuntimeCC());
5208 return call;
5209}
5210
5211/// Emits a call or invoke instruction to the given function, depending
5212/// on the current state of the EH stack.
5213llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
5214 ArrayRef<llvm::Value *> Args,
5215 const Twine &Name) {
5216 llvm::BasicBlock *InvokeDest = getInvokeDest();
5217 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5218 getBundlesForFunclet(Callee: Callee.getCallee());
5219
5220 llvm::CallBase *Inst;
5221 if (!InvokeDest)
5222 Inst = Builder.CreateCall(Callee, Args, OpBundles: BundleList, Name);
5223 else {
5224 llvm::BasicBlock *ContBB = createBasicBlock(name: "invoke.cont");
5225 Inst = Builder.CreateInvoke(Callee, NormalDest: ContBB, UnwindDest: InvokeDest, Args, OpBundles: BundleList,
5226 Name);
5227 EmitBlock(BB: ContBB);
5228 }
5229
5230 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5231 // optimizer it can aggressively ignore unwind edges.
5232 if (CGM.getLangOpts().ObjCAutoRefCount)
5233 AddObjCARCExceptionMetadata(Inst);
5234
5235 return Inst;
5236}
5237
5238void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
5239 llvm::Value *New) {
5240 DeferredReplacements.push_back(
5241 Elt: std::make_pair(x: llvm::WeakTrackingVH(Old), y&: New));
5242}
5243
5244namespace {
5245
5246/// Specify given \p NewAlign as the alignment of return value attribute. If
5247/// such attribute already exists, re-set it to the maximal one of two options.
5248[[nodiscard]] llvm::AttributeList
5249maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
5250 const llvm::AttributeList &Attrs,
5251 llvm::Align NewAlign) {
5252 llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
5253 if (CurAlign >= NewAlign)
5254 return Attrs;
5255 llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Context&: Ctx, Alignment: NewAlign);
5256 return Attrs.removeRetAttribute(C&: Ctx, Kind: llvm::Attribute::AttrKind::Alignment)
5257 .addRetAttribute(C&: Ctx, Attr: AlignAttr);
5258}
5259
5260template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
5261protected:
5262 CodeGenFunction &CGF;
5263
5264 /// We do nothing if this is, or becomes, nullptr.
5265 const AlignedAttrTy *AA = nullptr;
5266
5267 llvm::Value *Alignment = nullptr; // May or may not be a constant.
5268 llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
5269
5270 AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5271 : CGF(CGF_) {
5272 if (!FuncDecl)
5273 return;
5274 AA = FuncDecl->getAttr<AlignedAttrTy>();
5275 }
5276
5277public:
5278 /// If we can, materialize the alignment as an attribute on return value.
5279 [[nodiscard]] llvm::AttributeList
5280 TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
5281 if (!AA || OffsetCI || CGF.SanOpts.has(K: SanitizerKind::Alignment))
5282 return Attrs;
5283 const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Val: Alignment);
5284 if (!AlignmentCI)
5285 return Attrs;
5286 // We may legitimately have non-power-of-2 alignment here.
5287 // If so, this is UB land, emit it via `@llvm.assume` instead.
5288 if (!AlignmentCI->getValue().isPowerOf2())
5289 return Attrs;
5290 llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
5291 Ctx&: CGF.getLLVMContext(), Attrs,
5292 NewAlign: llvm::Align(
5293 AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment)));
5294 AA = nullptr; // We're done. Disallow doing anything else.
5295 return NewAttrs;
5296 }
5297
5298 /// Emit alignment assumption.
5299 /// This is a general fallback that we take if either there is an offset,
5300 /// or the alignment is variable or we are sanitizing for alignment.
5301 void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
5302 if (!AA)
5303 return;
5304 CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
5305 AA->getLocation(), Alignment, OffsetCI);
5306 AA = nullptr; // We're done. Disallow doing anything else.
5307 }
5308};
5309
5310/// Helper data structure to emit `AssumeAlignedAttr`.
5311class AssumeAlignedAttrEmitter final
5312 : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
5313public:
5314 AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5315 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5316 if (!AA)
5317 return;
5318 // It is guaranteed that the alignment/offset are constants.
5319 Alignment = cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: AA->getAlignment()));
5320 if (Expr *Offset = AA->getOffset()) {
5321 OffsetCI = cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: Offset));
5322 if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
5323 OffsetCI = nullptr;
5324 }
5325 }
5326};
5327
5328/// Helper data structure to emit `AllocAlignAttr`.
5329class AllocAlignAttrEmitter final
5330 : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
5331public:
5332 AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
5333 const CallArgList &CallArgs)
5334 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5335 if (!AA)
5336 return;
5337 // Alignment may or may not be a constant, and that is okay.
5338 Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
5339 .getRValue(CGF)
5340 .getScalarVal();
5341 }
5342};
5343
5344} // namespace
5345
5346static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
5347 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Ty))
5348 return VT->getPrimitiveSizeInBits().getKnownMinValue();
5349 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty))
5350 return getMaxVectorWidth(Ty: AT->getElementType());
5351
5352 unsigned MaxVectorWidth = 0;
5353 if (auto *ST = dyn_cast<llvm::StructType>(Val: Ty))
5354 for (auto *I : ST->elements())
5355 MaxVectorWidth = std::max(a: MaxVectorWidth, b: getMaxVectorWidth(Ty: I));
5356 return MaxVectorWidth;
5357}
5358
5359RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
5360 const CGCallee &Callee,
5361 ReturnValueSlot ReturnValue,
5362 const CallArgList &CallArgs,
5363 llvm::CallBase **callOrInvoke, bool IsMustTail,
5364 SourceLocation Loc,
5365 bool IsVirtualFunctionPointerThunk) {
5366 // FIXME: We no longer need the types from CallArgs; lift up and simplify.
5367
5368 assert(Callee.isOrdinary() || Callee.isVirtual());
5369
5370 // Handle struct-return functions by passing a pointer to the
5371 // location that we would like to return into.
5372 QualType RetTy = CallInfo.getReturnType();
5373 const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
5374
5375 llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(FI: CallInfo);
5376
5377 const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
5378 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
5379 // We can only guarantee that a function is called from the correct
5380 // context/function based on the appropriate target attributes,
5381 // so only check in the case where we have both always_inline and target
5382 // since otherwise we could be making a conditional call after a check for
5383 // the proper cpu features (and it won't cause code generation issues due to
5384 // function based code generation).
5385 if ((TargetDecl->hasAttr<AlwaysInlineAttr>() &&
5386 (TargetDecl->hasAttr<TargetAttr>() ||
5387 (CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>()))) ||
5388 (CurFuncDecl && CurFuncDecl->hasAttr<FlattenAttr>() &&
5389 (CurFuncDecl->hasAttr<TargetAttr>() ||
5390 TargetDecl->hasAttr<TargetAttr>())))
5391 checkTargetFeatures(Loc, TargetDecl: FD);
5392 }
5393
5394 // Some architectures (such as x86-64) have the ABI changed based on
5395 // attribute-target/features. Give them a chance to diagnose.
5396 const FunctionDecl *CallerDecl = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
5397 const FunctionDecl *CalleeDecl = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
5398 CGM.getTargetCodeGenInfo().checkFunctionCallABI(CGM, CallLoc: Loc, Caller: CallerDecl,
5399 Callee: CalleeDecl, Args: CallArgs, ReturnType: RetTy);
5400
5401 // 1. Set up the arguments.
5402
5403 // If we're using inalloca, insert the allocation after the stack save.
5404 // FIXME: Do this earlier rather than hacking it in here!
5405 RawAddress ArgMemory = RawAddress::invalid();
5406 if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
5407 const llvm::DataLayout &DL = CGM.getDataLayout();
5408 llvm::Instruction *IP = CallArgs.getStackBase();
5409 llvm::AllocaInst *AI;
5410 if (IP) {
5411 IP = IP->getNextNode();
5412 AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(), "argmem",
5413 IP->getIterator());
5414 } else {
5415 AI = CreateTempAlloca(Ty: ArgStruct, Name: "argmem");
5416 }
5417 auto Align = CallInfo.getArgStructAlignment();
5418 AI->setAlignment(Align.getAsAlign());
5419 AI->setUsedWithInAlloca(true);
5420 assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
5421 ArgMemory = RawAddress(AI, ArgStruct, Align);
5422 }
5423
5424 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
5425 SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
5426
5427 // If the call returns a temporary with struct return, create a temporary
5428 // alloca to hold the result, unless one is given to us.
5429 Address SRetPtr = Address::invalid();
5430 bool NeedSRetLifetimeEnd = false;
5431 if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
5432 // For virtual function pointer thunks and musttail calls, we must always
5433 // forward an incoming SRet pointer to the callee, because a local alloca
5434 // would be de-allocated before the call. These cases both guarantee that
5435 // there will be an incoming SRet argument of the correct type.
5436 if ((IsVirtualFunctionPointerThunk || IsMustTail) && RetAI.isIndirect()) {
5437 SRetPtr = makeNaturalAddressForPointer(Ptr: CurFn->arg_begin() +
5438 IRFunctionArgs.getSRetArgNo(),
5439 T: RetTy, Alignment: CharUnits::fromQuantity(Quantity: 1));
5440 } else if (!ReturnValue.isNull()) {
5441 SRetPtr = ReturnValue.getAddress();
5442 } else {
5443 SRetPtr = CreateMemTempWithoutCast(T: RetTy, Name: "tmp");
5444 if (HaveInsertPoint() && ReturnValue.isUnused())
5445 NeedSRetLifetimeEnd = EmitLifetimeStart(Addr: SRetPtr.getBasePointer());
5446 }
5447 if (IRFunctionArgs.hasSRetArg()) {
5448 // A mismatch between the allocated return value's AS and the target's
5449 // chosen IndirectAS can happen e.g. when passing the this pointer through
5450 // a chain involving stores to / loads from the DefaultAS; we address this
5451 // here, symmetrically with the handling we have for normal pointer args.
5452 if (SRetPtr.getAddressSpace() != RetAI.getIndirectAddrSpace()) {
5453 llvm::Value *V = SRetPtr.getBasePointer();
5454 llvm::Type *Ty = llvm::PointerType::get(C&: getLLVMContext(),
5455 AddressSpace: RetAI.getIndirectAddrSpace());
5456
5457 SRetPtr = SRetPtr.withPointer(NewPointer: performAddrSpaceCast(Src: V, DestTy: Ty),
5458 IsKnownNonNull: SRetPtr.isKnownNonNull());
5459 }
5460 IRCallArgs[IRFunctionArgs.getSRetArgNo()] =
5461 getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy);
5462 } else if (RetAI.isInAlloca()) {
5463 Address Addr =
5464 Builder.CreateStructGEP(Addr: ArgMemory, Index: RetAI.getInAllocaFieldIndex());
5465 Builder.CreateStore(Val: getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy), Addr);
5466 }
5467 }
5468
5469 RawAddress swiftErrorTemp = RawAddress::invalid();
5470 Address swiftErrorArg = Address::invalid();
5471
5472 // When passing arguments using temporary allocas, we need to add the
5473 // appropriate lifetime markers. This vector keeps track of all the lifetime
5474 // markers that need to be ended right after the call.
5475 SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
5476
5477 // Translate all of the arguments as necessary to match the IR lowering.
5478 assert(CallInfo.arg_size() == CallArgs.size() &&
5479 "Mismatch between function signature & arguments.");
5480 unsigned ArgNo = 0;
5481 CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
5482 for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
5483 I != E; ++I, ++info_it, ++ArgNo) {
5484 const ABIArgInfo &ArgInfo = info_it->info;
5485
5486 // Insert a padding argument to ensure proper alignment.
5487 if (IRFunctionArgs.hasPaddingArg(ArgNo))
5488 IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
5489 llvm::UndefValue::get(T: ArgInfo.getPaddingType());
5490
5491 unsigned FirstIRArg, NumIRArgs;
5492 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
5493
5494 bool ArgHasMaybeUndefAttr =
5495 IsArgumentMaybeUndef(TargetDecl, NumRequiredArgs: CallInfo.getNumRequiredArgs(), ArgNo);
5496
5497 switch (ArgInfo.getKind()) {
5498 case ABIArgInfo::InAlloca: {
5499 assert(NumIRArgs == 0);
5500 assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
5501 if (I->isAggregate()) {
5502 RawAddress Addr = I->hasLValue()
5503 ? I->getKnownLValue().getAddress()
5504 : I->getKnownRValue().getAggregateAddress();
5505 llvm::Instruction *Placeholder =
5506 cast<llvm::Instruction>(Val: Addr.getPointer());
5507
5508 if (!ArgInfo.getInAllocaIndirect()) {
5509 // Replace the placeholder with the appropriate argument slot GEP.
5510 CGBuilderTy::InsertPoint IP = Builder.saveIP();
5511 Builder.SetInsertPoint(Placeholder);
5512 Addr = Builder.CreateStructGEP(Addr: ArgMemory,
5513 Index: ArgInfo.getInAllocaFieldIndex());
5514 Builder.restoreIP(IP);
5515 } else {
5516 // For indirect things such as overaligned structs, replace the
5517 // placeholder with a regular aggregate temporary alloca. Store the
5518 // address of this alloca into the struct.
5519 Addr = CreateMemTemp(T: info_it->type, Name: "inalloca.indirect.tmp");
5520 Address ArgSlot = Builder.CreateStructGEP(
5521 Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5522 Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5523 }
5524 deferPlaceholderReplacement(Old: Placeholder, New: Addr.getPointer());
5525 } else if (ArgInfo.getInAllocaIndirect()) {
5526 // Make a temporary alloca and store the address of it into the argument
5527 // struct.
5528 RawAddress Addr = CreateMemTempWithoutCast(
5529 T: I->Ty, Align: getContext().getTypeAlignInChars(T: I->Ty),
5530 Name: "indirect-arg-temp");
5531 I->copyInto(CGF&: *this, Addr);
5532 Address ArgSlot =
5533 Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5534 Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5535 } else {
5536 // Store the RValue into the argument struct.
5537 Address Addr =
5538 Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5539 Addr = Addr.withElementType(ElemTy: ConvertTypeForMem(T: I->Ty));
5540 I->copyInto(CGF&: *this, Addr);
5541 }
5542 break;
5543 }
5544
5545 case ABIArgInfo::Indirect:
5546 case ABIArgInfo::IndirectAliased: {
5547 assert(NumIRArgs == 1);
5548 if (I->isAggregate()) {
5549 // We want to avoid creating an unnecessary temporary+copy here;
5550 // however, we need one in three cases:
5551 // 1. If the argument is not byval, and we are required to copy the
5552 // source. (This case doesn't occur on any common architecture.)
5553 // 2. If the argument is byval, RV is not sufficiently aligned, and
5554 // we cannot force it to be sufficiently aligned.
5555 // 3. If the argument is byval, but RV is not located in default
5556 // or alloca address space.
5557 Address Addr = I->hasLValue()
5558 ? I->getKnownLValue().getAddress()
5559 : I->getKnownRValue().getAggregateAddress();
5560 CharUnits Align = ArgInfo.getIndirectAlign();
5561 const llvm::DataLayout *TD = &CGM.getDataLayout();
5562
5563 assert((FirstIRArg >= IRFuncTy->getNumParams() ||
5564 IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
5565 TD->getAllocaAddrSpace()) &&
5566 "indirect argument must be in alloca address space");
5567
5568 bool NeedCopy = false;
5569 if (Addr.getAlignment() < Align &&
5570 llvm::getOrEnforceKnownAlignment(V: Addr.emitRawPointer(CGF&: *this),
5571 PrefAlign: Align.getAsAlign(),
5572 DL: *TD) < Align.getAsAlign()) {
5573 NeedCopy = true;
5574 } else if (I->hasLValue()) {
5575 auto LV = I->getKnownLValue();
5576
5577 bool isByValOrRef =
5578 ArgInfo.isIndirectAliased() || ArgInfo.getIndirectByVal();
5579
5580 if (!isByValOrRef ||
5581 (LV.getAlignment() < getContext().getTypeAlignInChars(T: I->Ty))) {
5582 NeedCopy = true;
5583 }
5584
5585 if (isByValOrRef && Addr.getType()->getAddressSpace() !=
5586 ArgInfo.getIndirectAddrSpace()) {
5587 NeedCopy = true;
5588 }
5589 }
5590
5591 if (!NeedCopy) {
5592 // Skip the extra memcpy call.
5593 llvm::Value *V = getAsNaturalPointerTo(Addr, PointeeType: I->Ty);
5594 auto *T = llvm::PointerType::get(C&: CGM.getLLVMContext(),
5595 AddressSpace: ArgInfo.getIndirectAddrSpace());
5596
5597 // FIXME: This should not depend on the language address spaces, and
5598 // only the contextual values. If the address space mismatches, see if
5599 // we can look through a cast to a compatible address space value,
5600 // otherwise emit a copy.
5601 llvm::Value *Val = performAddrSpaceCast(Src: V, DestTy: T);
5602 if (ArgHasMaybeUndefAttr)
5603 Val = Builder.CreateFreeze(V: Val);
5604 IRCallArgs[FirstIRArg] = Val;
5605 break;
5606 }
5607 } else if (I->getType()->isArrayParameterType()) {
5608 // Don't produce a temporary for ArrayParameterType arguments.
5609 // ArrayParameterType arguments are only created from
5610 // HLSL_ArrayRValue casts and HLSLOutArgExpr expressions, both
5611 // of which create temporaries already. This allows us to just use the
5612 // scalar for the decayed array pointer as the argument directly.
5613 IRCallArgs[FirstIRArg] = I->getKnownRValue().getScalarVal();
5614 break;
5615 }
5616
5617 // For non-aggregate args and aggregate args meeting conditions above
5618 // we need to create an aligned temporary, and copy to it.
5619 RawAddress AI = CreateMemTempWithoutCast(
5620 T: I->Ty, Align: ArgInfo.getIndirectAlign(), Name: "byval-temp");
5621 llvm::Value *Val = getAsNaturalPointerTo(Addr: AI, PointeeType: I->Ty);
5622 if (ArgHasMaybeUndefAttr)
5623 Val = Builder.CreateFreeze(V: Val);
5624 IRCallArgs[FirstIRArg] = Val;
5625
5626 // Emit lifetime markers for the temporary alloca and add cleanup code to
5627 // emit the end lifetime marker after the call.
5628 if (EmitLifetimeStart(Addr: AI.getPointer()))
5629 CallLifetimeEndAfterCall.emplace_back(Args&: AI);
5630
5631 // Generate the copy.
5632 I->copyInto(CGF&: *this, Addr: AI);
5633 break;
5634 }
5635
5636 case ABIArgInfo::Ignore:
5637 assert(NumIRArgs == 0);
5638 break;
5639
5640 case ABIArgInfo::Extend:
5641 case ABIArgInfo::Direct: {
5642 if (!isa<llvm::StructType>(Val: ArgInfo.getCoerceToType()) &&
5643 ArgInfo.getCoerceToType() == ConvertType(T: info_it->type) &&
5644 ArgInfo.getDirectOffset() == 0) {
5645 assert(NumIRArgs == 1);
5646 llvm::Value *V;
5647 if (!I->isAggregate())
5648 V = I->getKnownRValue().getScalarVal();
5649 else
5650 V = Builder.CreateLoad(
5651 Addr: I->hasLValue() ? I->getKnownLValue().getAddress()
5652 : I->getKnownRValue().getAggregateAddress());
5653
5654 // Implement swifterror by copying into a new swifterror argument.
5655 // We'll write back in the normal path out of the call.
5656 if (CallInfo.getExtParameterInfo(argIndex: ArgNo).getABI() ==
5657 ParameterABI::SwiftErrorResult) {
5658 assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
5659
5660 QualType pointeeTy = I->Ty->getPointeeType();
5661 swiftErrorArg = makeNaturalAddressForPointer(
5662 Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
5663
5664 swiftErrorTemp =
5665 CreateMemTemp(T: pointeeTy, Align: getPointerAlign(), Name: "swifterror.temp");
5666 V = swiftErrorTemp.getPointer();
5667 cast<llvm::AllocaInst>(Val: V)->setSwiftError(true);
5668
5669 llvm::Value *errorValue = Builder.CreateLoad(Addr: swiftErrorArg);
5670 Builder.CreateStore(Val: errorValue, Addr: swiftErrorTemp);
5671 }
5672
5673 // We might have to widen integers, but we should never truncate.
5674 if (ArgInfo.getCoerceToType() != V->getType() &&
5675 V->getType()->isIntegerTy())
5676 V = Builder.CreateZExt(V, DestTy: ArgInfo.getCoerceToType());
5677
5678 // The only plausible mismatch here would be for pointer address spaces.
5679 // We assume that the target has a reasonable mapping for the DefaultAS
5680 // (it can be casted to from incoming specific ASes), and insert an AS
5681 // cast to address the mismatch.
5682 if (FirstIRArg < IRFuncTy->getNumParams() &&
5683 V->getType() != IRFuncTy->getParamType(i: FirstIRArg)) {
5684 assert(V->getType()->isPointerTy() && "Only pointers can mismatch!");
5685 V = performAddrSpaceCast(Src: V, DestTy: IRFuncTy->getParamType(i: FirstIRArg));
5686 }
5687
5688 if (ArgHasMaybeUndefAttr)
5689 V = Builder.CreateFreeze(V);
5690 IRCallArgs[FirstIRArg] = V;
5691 break;
5692 }
5693
5694 llvm::StructType *STy =
5695 dyn_cast<llvm::StructType>(Val: ArgInfo.getCoerceToType());
5696
5697 // FIXME: Avoid the conversion through memory if possible.
5698 Address Src = Address::invalid();
5699 if (!I->isAggregate()) {
5700 Src = CreateMemTemp(T: I->Ty, Name: "coerce");
5701 I->copyInto(CGF&: *this, Addr: Src);
5702 } else {
5703 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
5704 : I->getKnownRValue().getAggregateAddress();
5705 }
5706
5707 // If the value is offset in memory, apply the offset now.
5708 Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
5709
5710 // Fast-isel and the optimizer generally like scalar values better than
5711 // FCAs, so we flatten them if this is safe to do for this argument.
5712 if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
5713 llvm::Type *SrcTy = Src.getElementType();
5714 llvm::TypeSize SrcTypeSize =
5715 CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
5716 llvm::TypeSize DstTypeSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
5717 if (SrcTypeSize.isScalable()) {
5718 assert(STy->containsHomogeneousScalableVectorTypes() &&
5719 "ABI only supports structure with homogeneous scalable vector "
5720 "type");
5721 assert(SrcTypeSize == DstTypeSize &&
5722 "Only allow non-fractional movement of structure with "
5723 "homogeneous scalable vector type");
5724 assert(NumIRArgs == STy->getNumElements());
5725
5726 llvm::Value *StoredStructValue =
5727 Builder.CreateLoad(Addr: Src, Name: Src.getName() + ".tuple");
5728 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5729 llvm::Value *Extract = Builder.CreateExtractValue(
5730 Agg: StoredStructValue, Idxs: i, Name: Src.getName() + ".extract" + Twine(i));
5731 IRCallArgs[FirstIRArg + i] = Extract;
5732 }
5733 } else {
5734 uint64_t SrcSize = SrcTypeSize.getFixedValue();
5735 uint64_t DstSize = DstTypeSize.getFixedValue();
5736 bool HasPFPFields = getContext().hasPFPFields(Ty: I->Ty);
5737
5738 // If the source type is smaller than the destination type of the
5739 // coerce-to logic, copy the source value into a temp alloca the size
5740 // of the destination type to allow loading all of it. The bits past
5741 // the source value are left undef.
5742 if (HasPFPFields || SrcSize < DstSize) {
5743 Address TempAlloca = CreateTempAlloca(Ty: STy, align: Src.getAlignment(),
5744 Name: Src.getName() + ".coerce");
5745 if (HasPFPFields) {
5746 // Structures with PFP fields require a coerced load to remove any
5747 // pointer signatures.
5748 Builder.CreateStore(
5749 Val: CreatePFPCoercedLoad(Src, SrcFETy: I->Ty, Ty: ArgInfo.getCoerceToType(),
5750 CGF&: *this),
5751 Addr: TempAlloca);
5752 } else
5753 Builder.CreateMemCpy(Dest: TempAlloca, Src, Size: SrcSize);
5754 Src = TempAlloca;
5755 } else {
5756 Src = Src.withElementType(ElemTy: STy);
5757 }
5758
5759 assert(NumIRArgs == STy->getNumElements());
5760 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5761 Address EltPtr = Builder.CreateStructGEP(Addr: Src, Index: i);
5762 llvm::Value *LI = Builder.CreateLoad(Addr: EltPtr);
5763 if (ArgHasMaybeUndefAttr)
5764 LI = Builder.CreateFreeze(V: LI);
5765 IRCallArgs[FirstIRArg + i] = LI;
5766 }
5767 }
5768 } else {
5769 // In the simple case, just pass the coerced loaded value.
5770 assert(NumIRArgs == 1);
5771 llvm::Value *Load =
5772 CreateCoercedLoad(Src, SrcFETy: I->Ty, Ty: ArgInfo.getCoerceToType(), CGF&: *this);
5773
5774 if (CallInfo.isCmseNSCall()) {
5775 // For certain parameter types, clear padding bits, as they may reveal
5776 // sensitive information.
5777 // Small struct/union types are passed as integer arrays.
5778 auto *ATy = dyn_cast<llvm::ArrayType>(Val: Load->getType());
5779 if (ATy != nullptr && isa<RecordType>(Val: I->Ty.getCanonicalType()))
5780 Load = EmitCMSEClearRecord(Src: Load, ATy, QTy: I->Ty);
5781 }
5782
5783 if (ArgHasMaybeUndefAttr)
5784 Load = Builder.CreateFreeze(V: Load);
5785 IRCallArgs[FirstIRArg] = Load;
5786 }
5787
5788 break;
5789 }
5790
5791 case ABIArgInfo::CoerceAndExpand: {
5792 auto coercionType = ArgInfo.getCoerceAndExpandType();
5793 auto layout = CGM.getDataLayout().getStructLayout(Ty: coercionType);
5794 auto unpaddedCoercionType = ArgInfo.getUnpaddedCoerceAndExpandType();
5795 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
5796
5797 Address addr = Address::invalid();
5798 RawAddress AllocaAddr = RawAddress::invalid();
5799 bool NeedLifetimeEnd = false;
5800 if (I->isAggregate()) {
5801 addr = I->hasLValue() ? I->getKnownLValue().getAddress()
5802 : I->getKnownRValue().getAggregateAddress();
5803
5804 } else {
5805 RValue RV = I->getKnownRValue();
5806 assert(RV.isScalar()); // complex should always just be direct
5807
5808 llvm::Type *scalarType = RV.getScalarVal()->getType();
5809 auto scalarAlign = CGM.getDataLayout().getPrefTypeAlign(Ty: scalarType);
5810
5811 // Materialize to a temporary.
5812 addr = CreateTempAlloca(Ty: RV.getScalarVal()->getType(),
5813 align: CharUnits::fromQuantity(Quantity: std::max(
5814 a: layout->getAlignment(), b: scalarAlign)),
5815 Name: "tmp",
5816 /*ArraySize=*/nullptr, Alloca: &AllocaAddr);
5817 NeedLifetimeEnd = EmitLifetimeStart(Addr: AllocaAddr.getPointer());
5818
5819 Builder.CreateStore(Val: RV.getScalarVal(), Addr: addr);
5820 }
5821
5822 addr = addr.withElementType(ElemTy: coercionType);
5823
5824 unsigned IRArgPos = FirstIRArg;
5825 unsigned unpaddedIndex = 0;
5826 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
5827 llvm::Type *eltType = coercionType->getElementType(N: i);
5828 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
5829 continue;
5830 Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
5831 llvm::Value *elt = CreateCoercedLoad(
5832 Src: eltAddr, SrcFETy: I->Ty,
5833 Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
5834 : unpaddedCoercionType,
5835 CGF&: *this);
5836 if (ArgHasMaybeUndefAttr)
5837 elt = Builder.CreateFreeze(V: elt);
5838 IRCallArgs[IRArgPos++] = elt;
5839 }
5840 assert(IRArgPos == FirstIRArg + NumIRArgs);
5841
5842 if (NeedLifetimeEnd)
5843 EmitLifetimeEnd(Addr: AllocaAddr.getPointer());
5844 break;
5845 }
5846
5847 case ABIArgInfo::Expand: {
5848 unsigned IRArgPos = FirstIRArg;
5849 ExpandTypeToArgs(Ty: I->Ty, Arg: *I, IRFuncTy, IRCallArgs, IRCallArgPos&: IRArgPos);
5850 assert(IRArgPos == FirstIRArg + NumIRArgs);
5851 break;
5852 }
5853
5854 case ABIArgInfo::TargetSpecific: {
5855 Address Src = Address::invalid();
5856 if (!I->isAggregate()) {
5857 Src = CreateMemTemp(T: I->Ty, Name: "target_coerce");
5858 I->copyInto(CGF&: *this, Addr: Src);
5859 } else {
5860 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
5861 : I->getKnownRValue().getAggregateAddress();
5862 }
5863
5864 // If the value is offset in memory, apply the offset now.
5865 Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
5866 llvm::Value *Load =
5867 CGM.getABIInfo().createCoercedLoad(SrcAddr: Src, AI: ArgInfo, CGF&: *this);
5868 IRCallArgs[FirstIRArg] = Load;
5869 break;
5870 }
5871 }
5872 }
5873
5874 const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(CGF&: *this);
5875 llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
5876
5877 // If we're using inalloca, set up that argument.
5878 if (ArgMemory.isValid()) {
5879 llvm::Value *Arg = ArgMemory.getPointer();
5880 assert(IRFunctionArgs.hasInallocaArg());
5881 IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
5882 }
5883
5884 // 2. Prepare the function pointer.
5885
5886 // If the callee is a bitcast of a non-variadic function to have a
5887 // variadic function pointer type, check to see if we can remove the
5888 // bitcast. This comes up with unprototyped functions.
5889 //
5890 // This makes the IR nicer, but more importantly it ensures that we
5891 // can inline the function at -O0 if it is marked always_inline.
5892 auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
5893 llvm::Value *Ptr) -> llvm::Function * {
5894 if (!CalleeFT->isVarArg())
5895 return nullptr;
5896
5897 // Get underlying value if it's a bitcast
5898 if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Val: Ptr)) {
5899 if (CE->getOpcode() == llvm::Instruction::BitCast)
5900 Ptr = CE->getOperand(i_nocapture: 0);
5901 }
5902
5903 llvm::Function *OrigFn = dyn_cast<llvm::Function>(Val: Ptr);
5904 if (!OrigFn)
5905 return nullptr;
5906
5907 llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
5908
5909 // If the original type is variadic, or if any of the component types
5910 // disagree, we cannot remove the cast.
5911 if (OrigFT->isVarArg() ||
5912 OrigFT->getNumParams() != CalleeFT->getNumParams() ||
5913 OrigFT->getReturnType() != CalleeFT->getReturnType())
5914 return nullptr;
5915
5916 for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i)
5917 if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
5918 return nullptr;
5919
5920 return OrigFn;
5921 };
5922
5923 if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {
5924 CalleePtr = OrigFn;
5925 IRFuncTy = OrigFn->getFunctionType();
5926 }
5927
5928 // 3. Perform the actual call.
5929
5930 // Deactivate any cleanups that we're supposed to do immediately before
5931 // the call.
5932 if (!CallArgs.getCleanupsToDeactivate().empty())
5933 deactivateArgCleanupsBeforeCall(CGF&: *this, CallArgs);
5934
5935 // Update the largest vector width if any arguments have vector types.
5936 for (unsigned i = 0; i < IRCallArgs.size(); ++i)
5937 LargestVectorWidth = std::max(a: LargestVectorWidth,
5938 b: getMaxVectorWidth(Ty: IRCallArgs[i]->getType()));
5939
5940 // Compute the calling convention and attributes.
5941 unsigned CallingConv;
5942 llvm::AttributeList Attrs;
5943 CGM.ConstructAttributeList(Name: CalleePtr->getName(), FI: CallInfo,
5944 CalleeInfo: Callee.getAbstractInfo(), AttrList&: Attrs, CallingConv,
5945 /*AttrOnCallSite=*/true,
5946 /*IsThunk=*/false);
5947
5948 if (CallingConv == llvm::CallingConv::X86_VectorCall &&
5949 getTarget().getTriple().isWindowsArm64EC()) {
5950 CGM.Error(loc: Loc, error: "__vectorcall calling convention is not currently "
5951 "supported");
5952 }
5953
5954 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
5955 if (FD->hasAttr<StrictFPAttr>())
5956 // All calls within a strictfp function are marked strictfp
5957 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::StrictFP);
5958
5959 // If -ffast-math is enabled and the function is guarded by an
5960 // '__attribute__((optnone)) adjust the memory attribute so the BE emits the
5961 // library call instead of the intrinsic.
5962 if (FD->hasAttr<OptimizeNoneAttr>() && getLangOpts().FastMath)
5963 CGM.AdjustMemoryAttribute(Name: CalleePtr->getName(), CalleeInfo: Callee.getAbstractInfo(),
5964 Attrs);
5965 }
5966 // Add call-site nomerge attribute if exists.
5967 if (InNoMergeAttributedStmt)
5968 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoMerge);
5969
5970 // Add call-site noinline attribute if exists.
5971 if (InNoInlineAttributedStmt)
5972 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoInline);
5973
5974 // Add call-site always_inline attribute if exists.
5975 // Note: This corresponds to the [[clang::always_inline]] statement attribute.
5976 if (InAlwaysInlineAttributedStmt &&
5977 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
5978 Caller: CallerDecl, Callee: CalleeDecl))
5979 Attrs =
5980 Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::AlwaysInline);
5981
5982 // Remove call-site convergent attribute if requested.
5983 if (InNoConvergentAttributedStmt)
5984 Attrs =
5985 Attrs.removeFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::Convergent);
5986
5987 // Apply some call-site-specific attributes.
5988 // TODO: work this into building the attribute set.
5989
5990 // Apply always_inline to all calls within flatten functions.
5991 // FIXME: should this really take priority over __try, below?
5992 if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
5993 !InNoInlineAttributedStmt &&
5994 !(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>()) &&
5995 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
5996 Caller: CallerDecl, Callee: CalleeDecl)) {
5997 Attrs =
5998 Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::AlwaysInline);
5999 }
6000
6001 // Disable inlining inside SEH __try blocks.
6002 if (isSEHTryScope()) {
6003 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoInline);
6004 }
6005
6006 // Decide whether to use a call or an invoke.
6007 bool CannotThrow;
6008 if (currentFunctionUsesSEHTry()) {
6009 // SEH cares about asynchronous exceptions, so everything can "throw."
6010 CannotThrow = false;
6011 } else if (isCleanupPadScope() &&
6012 EHPersonality::get(CGF&: *this).isMSVCXXPersonality()) {
6013 // The MSVC++ personality will implicitly terminate the program if an
6014 // exception is thrown during a cleanup outside of a try/catch.
6015 // We don't need to model anything in IR to get this behavior.
6016 CannotThrow = true;
6017 } else {
6018 // Otherwise, nounwind call sites will never throw.
6019 CannotThrow = Attrs.hasFnAttr(Kind: llvm::Attribute::NoUnwind);
6020
6021 if (auto *FPtr = dyn_cast<llvm::Function>(Val: CalleePtr))
6022 if (FPtr->hasFnAttribute(Kind: llvm::Attribute::NoUnwind))
6023 CannotThrow = true;
6024 }
6025
6026 // If we made a temporary, be sure to clean up after ourselves. Note that we
6027 // can't depend on being inside of an ExprWithCleanups, so we need to manually
6028 // pop this cleanup later on. Being eager about this is OK, since this
6029 // temporary is 'invisible' outside of the callee.
6030 if (NeedSRetLifetimeEnd)
6031 pushFullExprCleanup<CallLifetimeEnd>(kind: NormalEHLifetimeMarker, A: SRetPtr);
6032
6033 llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();
6034
6035 SmallVector<llvm::OperandBundleDef, 1> BundleList =
6036 getBundlesForFunclet(Callee: CalleePtr);
6037
6038 if (SanOpts.has(K: SanitizerKind::KCFI) &&
6039 !isa_and_nonnull<FunctionDecl>(Val: TargetDecl))
6040 EmitKCFIOperandBundle(Callee: ConcreteCallee, Bundles&: BundleList);
6041
6042 // Add the pointer-authentication bundle.
6043 EmitPointerAuthOperandBundle(Info: ConcreteCallee.getPointerAuthInfo(), Bundles&: BundleList);
6044
6045 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl))
6046 if (FD->hasAttr<StrictFPAttr>())
6047 // All calls within a strictfp function are marked strictfp
6048 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::StrictFP);
6049
6050 AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
6051 Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
6052
6053 AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
6054 Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
6055
6056 // Emit the actual call/invoke instruction.
6057 llvm::CallBase *CI;
6058 if (!InvokeDest) {
6059 CI = Builder.CreateCall(FTy: IRFuncTy, Callee: CalleePtr, Args: IRCallArgs, OpBundles: BundleList);
6060 } else {
6061 llvm::BasicBlock *Cont = createBasicBlock(name: "invoke.cont");
6062 CI = Builder.CreateInvoke(Ty: IRFuncTy, Callee: CalleePtr, NormalDest: Cont, UnwindDest: InvokeDest, Args: IRCallArgs,
6063 OpBundles: BundleList);
6064 EmitBlock(BB: Cont);
6065 }
6066 if (CI->getCalledFunction() && CI->getCalledFunction()->hasName() &&
6067 CI->getCalledFunction()->getName().starts_with(Prefix: "_Z4sqrt")) {
6068 SetSqrtFPAccuracy(CI);
6069 }
6070 if (callOrInvoke) {
6071 *callOrInvoke = CI;
6072 if (CGM.getCodeGenOpts().CallGraphSection) {
6073 QualType CST;
6074 if (TargetDecl && TargetDecl->getFunctionType())
6075 CST = QualType(TargetDecl->getFunctionType(), 0);
6076 else if (const auto *FPT =
6077 Callee.getAbstractInfo().getCalleeFunctionProtoType())
6078 CST = QualType(FPT, 0);
6079 else
6080 llvm_unreachable(
6081 "Cannot find the callee type to generate callee_type metadata.");
6082
6083 // Set type identifier metadata of indirect calls for call graph section.
6084 if (!CST.isNull())
6085 CGM.createCalleeTypeMetadataForIcall(QT: CST, CB: *callOrInvoke);
6086 }
6087 }
6088
6089 // If this is within a function that has the guard(nocf) attribute and is an
6090 // indirect call, add the "guard_nocf" attribute to this call to indicate that
6091 // Control Flow Guard checks should not be added, even if the call is inlined.
6092 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
6093 if (const auto *A = FD->getAttr<CFGuardAttr>()) {
6094 if (A->getGuard() == CFGuardAttr::GuardArg::nocf &&
6095 !CI->getCalledFunction())
6096 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: "guard_nocf");
6097 }
6098 }
6099
6100 // Apply the attributes and calling convention.
6101 CI->setAttributes(Attrs);
6102 CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
6103
6104 // Apply various metadata.
6105
6106 if (!CI->getType()->isVoidTy())
6107 CI->setName("call");
6108
6109 if (CGM.shouldEmitConvergenceTokens() && CI->isConvergent())
6110 CI = addConvergenceControlToken(Input: CI);
6111
6112 // Update largest vector width from the return type.
6113 LargestVectorWidth =
6114 std::max(a: LargestVectorWidth, b: getMaxVectorWidth(Ty: CI->getType()));
6115
6116 // Insert instrumentation or attach profile metadata at indirect call sites.
6117 // For more details, see the comment before the definition of
6118 // IPVK_IndirectCallTarget in InstrProfData.inc.
6119 if (!CI->getCalledFunction())
6120 PGO->valueProfile(Builder, ValueKind: llvm::IPVK_IndirectCallTarget, ValueSite: CI, ValuePtr: CalleePtr);
6121
6122 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
6123 // optimizer it can aggressively ignore unwind edges.
6124 if (CGM.getLangOpts().ObjCAutoRefCount)
6125 AddObjCARCExceptionMetadata(Inst: CI);
6126
6127 // Set tail call kind if necessary.
6128 bool IsPPC = getTarget().getTriple().isPPC();
6129 bool IsMIPS = getTarget().getTriple().isMIPS();
6130 bool HasMips16 = false;
6131 if (IsMIPS) {
6132 const TargetOptions &TargetOpts = getTarget().getTargetOpts();
6133 HasMips16 = TargetOpts.FeatureMap.lookup(Key: "mips16");
6134 if (!HasMips16)
6135 HasMips16 = llvm::is_contained(Range: TargetOpts.Features, Element: "+mips16");
6136 }
6137 if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(Val: CI)) {
6138 if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
6139 Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
6140 else if (IsMustTail) {
6141 if (IsPPC) {
6142 if (getTarget().getTriple().isOSAIX())
6143 CGM.getDiags().Report(Loc, DiagID: diag::err_aix_musttail_unsupported);
6144 else if (!getTarget().hasFeature(Feature: "pcrelative-memops")) {
6145 if (getTarget().hasFeature(Feature: "longcall"))
6146 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail) << 0;
6147 else if (Call->isIndirectCall())
6148 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail) << 1;
6149 else if (isa_and_nonnull<FunctionDecl>(Val: TargetDecl)) {
6150 if (!cast<FunctionDecl>(Val: TargetDecl)->isDefined())
6151 // The undefined callee may be a forward declaration. Without
6152 // knowning all symbols in the module, we won't know the symbol is
6153 // defined or not. Collect all these symbols for later diagnosing.
6154 CGM.addUndefinedGlobalForTailCall(
6155 Global: {cast<FunctionDecl>(Val: TargetDecl), Loc});
6156 else {
6157 llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(
6158 GD: GlobalDecl(cast<FunctionDecl>(Val: TargetDecl)));
6159 if (llvm::GlobalValue::isWeakForLinker(Linkage) ||
6160 llvm::GlobalValue::isDiscardableIfUnused(Linkage))
6161 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail)
6162 << 2;
6163 }
6164 }
6165 }
6166 }
6167 if (IsMIPS) {
6168 if (HasMips16)
6169 CGM.getDiags().Report(Loc, DiagID: diag::err_mips_impossible_musttail) << 0;
6170 else if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl))
6171 CGM.addUndefinedGlobalForTailCall(Global: {FD, Loc});
6172 }
6173 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
6174 }
6175 }
6176
6177 // Add metadata for calls to MSAllocator functions
6178 if (getDebugInfo() && TargetDecl && TargetDecl->hasAttr<MSAllocatorAttr>())
6179 getDebugInfo()->addHeapAllocSiteMetadata(CallSite: CI, AllocatedTy: RetTy->getPointeeType(), Loc);
6180
6181 // Add metadata if calling an __attribute__((error(""))) or warning fn.
6182 if (TargetDecl && TargetDecl->hasAttr<ErrorAttr>()) {
6183 llvm::ConstantInt *Line =
6184 llvm::ConstantInt::get(Ty: Int64Ty, V: Loc.getRawEncoding());
6185 llvm::ConstantAsMetadata *MD = llvm::ConstantAsMetadata::get(C: Line);
6186 llvm::MDTuple *MDT = llvm::MDNode::get(Context&: getLLVMContext(), MDs: {MD});
6187 CI->setMetadata(Kind: "srcloc", Node: MDT);
6188 }
6189
6190 // 4. Finish the call.
6191
6192 // If the call doesn't return, finish the basic block and clear the
6193 // insertion point; this allows the rest of IRGen to discard
6194 // unreachable code.
6195 if (CI->doesNotReturn()) {
6196 if (NeedSRetLifetimeEnd)
6197 PopCleanupBlock();
6198
6199 // Strip away the noreturn attribute to better diagnose unreachable UB.
6200 if (SanOpts.has(K: SanitizerKind::Unreachable)) {
6201 // Also remove from function since CallBase::hasFnAttr additionally checks
6202 // attributes of the called function.
6203 if (auto *F = CI->getCalledFunction())
6204 F->removeFnAttr(Kind: llvm::Attribute::NoReturn);
6205 CI->removeFnAttr(Kind: llvm::Attribute::NoReturn);
6206
6207 // Avoid incompatibility with ASan which relies on the `noreturn`
6208 // attribute to insert handler calls.
6209 if (SanOpts.hasOneOf(K: SanitizerKind::Address |
6210 SanitizerKind::KernelAddress)) {
6211 SanitizerScope SanScope(this);
6212 llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);
6213 Builder.SetInsertPoint(CI);
6214 auto *FnType = llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
6215 llvm::FunctionCallee Fn =
6216 CGM.CreateRuntimeFunction(Ty: FnType, Name: "__asan_handle_no_return");
6217 EmitNounwindRuntimeCall(callee: Fn);
6218 }
6219 }
6220
6221 EmitUnreachable(Loc);
6222 Builder.ClearInsertionPoint();
6223
6224 // FIXME: For now, emit a dummy basic block because expr emitters in
6225 // generally are not ready to handle emitting expressions at unreachable
6226 // points.
6227 EnsureInsertPoint();
6228
6229 // Return a reasonable RValue.
6230 return GetUndefRValue(Ty: RetTy);
6231 }
6232
6233 // If this is a musttail call, return immediately. We do not branch to the
6234 // epilogue in this case.
6235 if (IsMustTail) {
6236 for (auto it = EHStack.find(sp: CurrentCleanupScopeDepth); it != EHStack.end();
6237 ++it) {
6238 EHCleanupScope *Cleanup = dyn_cast<EHCleanupScope>(Val: &*it);
6239 // Fake uses can be safely emitted immediately prior to the tail call, so
6240 // we choose to emit them just before the call here.
6241 if (Cleanup && Cleanup->isFakeUse()) {
6242 CGBuilderTy::InsertPointGuard IPG(Builder);
6243 Builder.SetInsertPoint(CI);
6244 Cleanup->getCleanup()->Emit(CGF&: *this, flags: EHScopeStack::Cleanup::Flags());
6245 } else if (!(Cleanup &&
6246 Cleanup->getCleanup()->isRedundantBeforeReturn())) {
6247 CGM.ErrorUnsupported(S: MustTailCall, Type: "tail call skipping over cleanups");
6248 }
6249 }
6250 if (CI->getType()->isVoidTy())
6251 Builder.CreateRetVoid();
6252 else
6253 Builder.CreateRet(V: CI);
6254 Builder.ClearInsertionPoint();
6255 EnsureInsertPoint();
6256 return GetUndefRValue(Ty: RetTy);
6257 }
6258
6259 // Perform the swifterror writeback.
6260 if (swiftErrorTemp.isValid()) {
6261 llvm::Value *errorResult = Builder.CreateLoad(Addr: swiftErrorTemp);
6262 Builder.CreateStore(Val: errorResult, Addr: swiftErrorArg);
6263 }
6264
6265 // Emit any call-associated writebacks immediately. Arguably this
6266 // should happen after any return-value munging.
6267 if (CallArgs.hasWritebacks())
6268 EmitWritebacks(args: CallArgs);
6269
6270 // The stack cleanup for inalloca arguments has to run out of the normal
6271 // lexical order, so deactivate it and run it manually here.
6272 CallArgs.freeArgumentMemory(CGF&: *this);
6273
6274 // Extract the return value.
6275 RValue Ret;
6276
6277 // If the current function is a virtual function pointer thunk, avoid copying
6278 // the return value of the musttail call to a temporary.
6279 if (IsVirtualFunctionPointerThunk) {
6280 Ret = RValue::get(V: CI);
6281 } else {
6282 Ret = [&] {
6283 switch (RetAI.getKind()) {
6284 case ABIArgInfo::CoerceAndExpand: {
6285 auto coercionType = RetAI.getCoerceAndExpandType();
6286
6287 Address addr = SRetPtr.withElementType(ElemTy: coercionType);
6288
6289 assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());
6290 bool requiresExtract = isa<llvm::StructType>(Val: CI->getType());
6291
6292 unsigned unpaddedIndex = 0;
6293 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
6294 llvm::Type *eltType = coercionType->getElementType(N: i);
6295 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
6296 continue;
6297 Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
6298 llvm::Value *elt = CI;
6299 if (requiresExtract)
6300 elt = Builder.CreateExtractValue(Agg: elt, Idxs: unpaddedIndex++);
6301 else
6302 assert(unpaddedIndex == 0);
6303 Builder.CreateStore(Val: elt, Addr: eltAddr);
6304 }
6305 [[fallthrough]];
6306 }
6307
6308 case ABIArgInfo::InAlloca:
6309 case ABIArgInfo::Indirect: {
6310 RValue ret = convertTempToRValue(addr: SRetPtr, type: RetTy, Loc: SourceLocation());
6311 if (NeedSRetLifetimeEnd)
6312 PopCleanupBlock();
6313 return ret;
6314 }
6315
6316 case ABIArgInfo::Ignore:
6317 // If we are ignoring an argument that had a result, make sure to
6318 // construct the appropriate return value for our caller.
6319 return GetUndefRValue(Ty: RetTy);
6320
6321 case ABIArgInfo::Extend:
6322 case ABIArgInfo::Direct: {
6323 llvm::Type *RetIRTy = ConvertType(T: RetTy);
6324 if (RetAI.getCoerceToType() == RetIRTy &&
6325 RetAI.getDirectOffset() == 0) {
6326 switch (getEvaluationKind(T: RetTy)) {
6327 case TEK_Complex: {
6328 llvm::Value *Real = Builder.CreateExtractValue(Agg: CI, Idxs: 0);
6329 llvm::Value *Imag = Builder.CreateExtractValue(Agg: CI, Idxs: 1);
6330 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
6331 }
6332 case TEK_Aggregate:
6333 break;
6334 case TEK_Scalar: {
6335 // If the argument doesn't match, perform a bitcast to coerce it.
6336 // This can happen due to trivial type mismatches.
6337 llvm::Value *V = CI;
6338 if (V->getType() != RetIRTy)
6339 V = Builder.CreateBitCast(V, DestTy: RetIRTy);
6340 return RValue::get(V);
6341 }
6342 }
6343 }
6344
6345 // If coercing a fixed vector from a scalable vector for ABI
6346 // compatibility, and the types match, use the llvm.vector.extract
6347 // intrinsic to perform the conversion.
6348 if (auto *FixedDstTy = dyn_cast<llvm::FixedVectorType>(Val: RetIRTy)) {
6349 llvm::Value *V = CI;
6350 if (auto *ScalableSrcTy =
6351 dyn_cast<llvm::ScalableVectorType>(Val: V->getType())) {
6352 if (FixedDstTy->getElementType() ==
6353 ScalableSrcTy->getElementType()) {
6354 V = Builder.CreateExtractVector(DstType: FixedDstTy, SrcVec: V, Idx: uint64_t(0),
6355 Name: "cast.fixed");
6356 return RValue::get(V);
6357 }
6358 }
6359 }
6360
6361 Address DestPtr = ReturnValue.getValue();
6362 bool DestIsVolatile = ReturnValue.isVolatile();
6363 uint64_t DestSize =
6364 getContext().getTypeInfoDataSizeInChars(T: RetTy).Width.getQuantity();
6365
6366 if (!DestPtr.isValid()) {
6367 DestPtr = CreateMemTemp(T: RetTy, Name: "coerce");
6368 DestIsVolatile = false;
6369 DestSize = getContext().getTypeSizeInChars(T: RetTy).getQuantity();
6370 }
6371
6372 // An empty record can overlap other data (if declared with
6373 // no_unique_address); omit the store for such types - as there is no
6374 // actual data to store.
6375 if (!isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true)) {
6376 // If the value is offset in memory, apply the offset now.
6377 Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
6378 CreateCoercedStore(
6379 Src: CI, SrcFETy: RetTy, Dst: StorePtr,
6380 DstSize: llvm::TypeSize::getFixed(ExactSize: DestSize - RetAI.getDirectOffset()),
6381 DstIsVolatile: DestIsVolatile);
6382 }
6383
6384 return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation());
6385 }
6386
6387 case ABIArgInfo::TargetSpecific: {
6388 Address DestPtr = ReturnValue.getValue();
6389 Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
6390 bool DestIsVolatile = ReturnValue.isVolatile();
6391 if (!DestPtr.isValid()) {
6392 DestPtr = CreateMemTemp(T: RetTy, Name: "target_coerce");
6393 DestIsVolatile = false;
6394 }
6395 CGM.getABIInfo().createCoercedStore(Val: CI, DstAddr: StorePtr, AI: RetAI, DestIsVolatile,
6396 CGF&: *this);
6397 return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation());
6398 }
6399
6400 case ABIArgInfo::Expand:
6401 case ABIArgInfo::IndirectAliased:
6402 llvm_unreachable("Invalid ABI kind for return argument");
6403 }
6404
6405 llvm_unreachable("Unhandled ABIArgInfo::Kind");
6406 }();
6407 }
6408
6409 // Emit the assume_aligned check on the return value.
6410 if (Ret.isScalar() && TargetDecl) {
6411 AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6412 AllocAlignAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6413 }
6414
6415 // Explicitly call CallLifetimeEnd::Emit just to re-use the code even though
6416 // we can't use the full cleanup mechanism.
6417 for (CallLifetimeEnd &LifetimeEnd : CallLifetimeEndAfterCall)
6418 LifetimeEnd.Emit(CGF&: *this, /*Flags=*/flags: {});
6419
6420 if (!ReturnValue.isExternallyDestructed() &&
6421 RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct)
6422 pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Ret.getAggregateAddress(),
6423 type: RetTy);
6424
6425 // Generate function declaration DISuprogram in order to be used
6426 // in debug info about call sites.
6427 if (CGDebugInfo *DI = getDebugInfo()) {
6428 // Ensure call site info would actually be emitted before collecting
6429 // further callee info.
6430 if (CalleeDecl && !CalleeDecl->hasAttr<NoDebugAttr>() &&
6431 DI->getCallSiteRelatedAttrs() != llvm::DINode::FlagZero) {
6432 CodeGenFunction CalleeCGF(CGM);
6433 const GlobalDecl &CalleeGlobalDecl =
6434 Callee.getAbstractInfo().getCalleeDecl();
6435 CalleeCGF.CurGD = CalleeGlobalDecl;
6436 FunctionArgList Args;
6437 QualType ResTy = CalleeCGF.BuildFunctionArgList(GD: CalleeGlobalDecl, Args);
6438 DI->EmitFuncDeclForCallSite(
6439 CallOrInvoke: CI, CalleeType: DI->getFunctionType(FD: CalleeDecl, RetTy: ResTy, Args), CalleeGlobalDecl);
6440 }
6441 }
6442
6443 return Ret;
6444}
6445
6446CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const {
6447 if (isVirtual()) {
6448 const CallExpr *CE = getVirtualCallExpr();
6449 return CGF.CGM.getCXXABI().getVirtualFunctionPointer(
6450 CGF, GD: getVirtualMethodDecl(), This: getThisAddress(), Ty: getVirtualFunctionType(),
6451 Loc: CE ? CE->getBeginLoc() : SourceLocation());
6452 }
6453
6454 return *this;
6455}
6456
6457/* VarArg handling */
6458
6459RValue CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
6460 AggValueSlot Slot) {
6461 VAListAddr = VE->isMicrosoftABI() ? EmitMSVAListRef(E: VE->getSubExpr())
6462 : EmitVAListRef(E: VE->getSubExpr());
6463 QualType Ty = VE->getType();
6464 if (Ty->isVariablyModifiedType())
6465 EmitVariablyModifiedType(Ty);
6466 if (VE->isMicrosoftABI())
6467 return CGM.getABIInfo().EmitMSVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6468 return CGM.getABIInfo().EmitVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6469}
6470
6471DisableDebugLocationUpdates::DisableDebugLocationUpdates(CodeGenFunction &CGF)
6472 : CGF(CGF) {
6473 CGF.disableDebugInfo();
6474}
6475
6476DisableDebugLocationUpdates::~DisableDebugLocationUpdates() {
6477 CGF.enableDebugInfo();
6478}
6479