1//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// These classes wrap the information about a call or function
10// definition used to handle ABI compliancy.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CGCall.h"
15#include "ABIInfo.h"
16#include "ABIInfoImpl.h"
17#include "CGBlocks.h"
18#include "CGCXXABI.h"
19#include "CGCleanup.h"
20#include "CGDebugInfo.h"
21#include "CGRecordLayout.h"
22#include "CodeGenFunction.h"
23#include "CodeGenModule.h"
24#include "CodeGenPGO.h"
25#include "QualTypeMapper.h"
26#include "TargetInfo.h"
27#include "clang/AST/Attr.h"
28#include "clang/AST/Decl.h"
29#include "clang/AST/DeclCXX.h"
30#include "clang/AST/DeclObjC.h"
31#include "clang/AST/RecordLayout.h"
32#include "clang/Basic/CodeGenOptions.h"
33#include "clang/Basic/DiagnosticFrontend.h"
34#include "clang/Basic/TargetInfo.h"
35#include "clang/CodeGen/CGFunctionInfo.h"
36#include "clang/CodeGen/SwiftCallingConv.h"
37#include "clang/CodeGenUtils/CallUtils.h"
38#include "llvm/ABI/FunctionInfo.h"
39#include "llvm/ABI/IRTypeMapper.h"
40#include "llvm/ABI/TargetInfo.h"
41#include "llvm/ABI/Types.h"
42#include "llvm/ADT/STLExtras.h"
43#include "llvm/ADT/StringExtras.h"
44#include "llvm/Analysis/ValueTracking.h"
45#include "llvm/IR/Assumptions.h"
46#include "llvm/IR/AttributeMask.h"
47#include "llvm/IR/Attributes.h"
48#include "llvm/IR/CallingConv.h"
49#include "llvm/IR/DataLayout.h"
50#include "llvm/IR/DebugInfoMetadata.h"
51#include "llvm/IR/InlineAsm.h"
52#include "llvm/IR/IntrinsicInst.h"
53#include "llvm/IR/Intrinsics.h"
54#include "llvm/IR/Type.h"
55#include "llvm/Transforms/Utils/Local.h"
56#include <optional>
57using namespace clang;
58using namespace CodeGen;
59
60/***/
61
62unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
63 switch (CC) {
64 case CC_C:
65 // On SPIR/SPIR-V, CC_C is the AST-level default calling convention, but
66 // it still needs to lower to spir_func so IR consumers can rely on the
67 // calling convention to distinguish device functions.
68 if (Target.getTriple().isSPIROrSPIRV())
69 return llvm::CallingConv::SPIR_FUNC;
70 return llvm::CallingConv::C;
71 case CC_X86StdCall:
72 return llvm::CallingConv::X86_StdCall;
73 case CC_X86FastCall:
74 return llvm::CallingConv::X86_FastCall;
75 case CC_X86RegCall:
76 return llvm::CallingConv::X86_RegCall;
77 case CC_X86ThisCall:
78 return llvm::CallingConv::X86_ThisCall;
79 case CC_Win64:
80 return llvm::CallingConv::Win64;
81 case CC_X86_64SysV:
82 return llvm::CallingConv::X86_64_SysV;
83 case CC_AAPCS:
84 return llvm::CallingConv::ARM_AAPCS;
85 case CC_AAPCS_VFP:
86 return llvm::CallingConv::ARM_AAPCS_VFP;
87 case CC_IntelOclBicc:
88 return llvm::CallingConv::Intel_OCL_BI;
89 // TODO: Add support for __pascal to LLVM.
90 case CC_X86Pascal:
91 return llvm::CallingConv::C;
92 // TODO: Add support for __vectorcall to LLVM.
93 case CC_X86VectorCall:
94 return llvm::CallingConv::X86_VectorCall;
95 case CC_AArch64VectorCall:
96 return llvm::CallingConv::AArch64_VectorCall;
97 case CC_AArch64SVEPCS:
98 return llvm::CallingConv::AArch64_SVE_VectorCall;
99 case CC_DeviceKernel:
100 return CGM.getTargetCodeGenInfo().getDeviceKernelCallingConv();
101 case CC_PreserveMost:
102 return llvm::CallingConv::PreserveMost;
103 case CC_PreserveAll:
104 return llvm::CallingConv::PreserveAll;
105 case CC_Swift:
106 return llvm::CallingConv::Swift;
107 case CC_SwiftAsync:
108 return llvm::CallingConv::SwiftTail;
109 case CC_M68kRTD:
110 return llvm::CallingConv::M68k_RTD;
111 case CC_PreserveNone:
112 return llvm::CallingConv::PreserveNone;
113 // clang-format off
114 case CC_RISCVVectorCall: return llvm::CallingConv::RISCV_VectorCall;
115 // clang-format on
116#define CC_VLS_CASE(ABI_VLEN) \
117 case CC_RISCVVLSCall_##ABI_VLEN: \
118 return llvm::CallingConv::RISCV_VLSCall_##ABI_VLEN;
119 CC_VLS_CASE(32)
120 CC_VLS_CASE(64)
121 CC_VLS_CASE(128)
122 CC_VLS_CASE(256)
123 CC_VLS_CASE(512)
124 CC_VLS_CASE(1024)
125 CC_VLS_CASE(2048)
126 CC_VLS_CASE(4096)
127 CC_VLS_CASE(8192)
128 CC_VLS_CASE(16384)
129 CC_VLS_CASE(32768)
130 CC_VLS_CASE(65536)
131#undef CC_VLS_CASE
132 }
133 llvm_unreachable("unhandled calling convention");
134}
135
136/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
137/// qualification. Either or both of RD and MD may be null. A null RD indicates
138/// that there is no meaningful 'this' type, and a null MD can occur when
139/// calling a method pointer.
140CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
141 const CXXMethodDecl *MD) {
142 CanQualType RecTy;
143 if (RD)
144 RecTy = Context.getCanonicalTagType(TD: RD);
145 else
146 RecTy = Context.VoidTy;
147
148 if (MD)
149 RecTy = CanQualType::CreateUnsafe(Other: Context.getAddrSpaceQualType(
150 T: RecTy, AddressSpace: MD->getMethodQualifiers().getAddressSpace()));
151 return Context.getPointerType(T: RecTy);
152}
153
154/// Returns the "extra-canonicalized" return type, which discards
155/// qualifiers on the return type. Codegen doesn't care about them,
156/// and it makes ABI code a little easier to be able to assume that
157/// all parameter and return types are top-level unqualified.
158static CanQualType GetReturnType(QualType RetTy) {
159 return RetTy->getCanonicalTypeUnqualified();
160}
161
162/// Arrange the argument and result information for a value of the given
163/// unprototyped freestanding function type.
164const CGFunctionInfo &
165CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
166 // When translating an unprototyped function type, always use a
167 // variadic type.
168 return arrangeLLVMFunctionInfo(returnType: FTNP->getReturnType().getUnqualifiedType(),
169 opts: FnInfoOpts::None, argTypes: {}, info: FTNP->getExtInfo(), paramInfos: {},
170 args: RequiredArgs(0), /*ABIInfoFD=*/nullptr);
171}
172
173static void addExtParameterInfosForCall(
174 llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
175 const FunctionProtoType *proto, unsigned prefixArgs, unsigned totalArgs) {
176 assert(proto->hasExtParameterInfos());
177 assert(paramInfos.size() <= prefixArgs);
178 assert(proto->getNumParams() + prefixArgs <= totalArgs);
179
180 paramInfos.reserve(N: totalArgs);
181
182 // Add default infos for any prefix args that don't already have infos.
183 paramInfos.resize(N: prefixArgs);
184
185 // Add infos for the prototype.
186 for (const auto &ParamInfo : proto->getExtParameterInfos()) {
187 paramInfos.push_back(Elt: ParamInfo);
188 // pass_object_size params have no parameter info.
189 if (ParamInfo.hasPassObjectSize())
190 paramInfos.emplace_back();
191 }
192
193 assert(paramInfos.size() <= totalArgs &&
194 "Did we forget to insert pass_object_size args?");
195 // Add default infos for the variadic and/or suffix arguments.
196 paramInfos.resize(N: totalArgs);
197}
198
199/// Adds the formal parameters in FPT to the given prefix. If any parameter in
200/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
201static void appendParameterTypes(
202 const CodeGenTypes &CGT, SmallVectorImpl<CanQualType> &prefix,
203 SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
204 CanQual<FunctionProtoType> FPT) {
205 // Fast path: don't touch param info if we don't need to.
206 if (!FPT->hasExtParameterInfos()) {
207 assert(paramInfos.empty() &&
208 "We have paramInfos, but the prototype doesn't?");
209 prefix.append(in_start: FPT->param_type_begin(), in_end: FPT->param_type_end());
210 return;
211 }
212
213 unsigned PrefixSize = prefix.size();
214 // In the vast majority of cases, we'll have precisely FPT->getNumParams()
215 // parameters; the only thing that can change this is the presence of
216 // pass_object_size. So, we preallocate for the common case.
217 prefix.reserve(N: prefix.size() + FPT->getNumParams());
218
219 auto ExtInfos = FPT->getExtParameterInfos();
220 assert(ExtInfos.size() == FPT->getNumParams());
221 for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
222 prefix.push_back(Elt: FPT->getParamType(i: I));
223 if (ExtInfos[I].hasPassObjectSize())
224 prefix.push_back(Elt: CGT.getContext().getCanonicalSizeType());
225 }
226
227 addExtParameterInfosForCall(paramInfos, proto: FPT.getTypePtr(), prefixArgs: PrefixSize,
228 totalArgs: prefix.size());
229}
230
231using ExtParameterInfoList =
232 SmallVector<FunctionProtoType::ExtParameterInfo, 16>;
233
234/// Arrange the LLVM function layout for a value of the given function
235/// type, on top of any implicit parameters already stored.
236static const CGFunctionInfo &
237arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
238 SmallVectorImpl<CanQualType> &prefix,
239 CanQual<FunctionProtoType> FTP) {
240 ExtParameterInfoList paramInfos;
241 RequiredArgs Required = RequiredArgs::forPrototypePlus(prototype: FTP, additional: prefix.size());
242 appendParameterTypes(CGT, prefix, paramInfos, FPT: FTP);
243 CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
244
245 FnInfoOpts opts =
246 instanceMethod ? FnInfoOpts::IsInstanceMethod : FnInfoOpts::None;
247 return CGT.arrangeLLVMFunctionInfo(returnType: resultType, opts, argTypes: prefix,
248 info: FTP->getExtInfo(), paramInfos, args: Required,
249 /*ABIInfoFD=*/nullptr);
250}
251
252using CanQualTypeList = SmallVector<CanQualType, 16>;
253
254/// Arrange the argument and result information for a value of the
255/// given freestanding function type.
256const CGFunctionInfo &
257CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
258 CanQualTypeList argTypes;
259 return ::arrangeLLVMFunctionInfo(CGT&: *this, /*instanceMethod=*/false, prefix&: argTypes,
260 FTP);
261}
262
263static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
264 bool IsTargetDefaultMSABI) {
265 // Set the appropriate calling convention for the Function.
266 if (D->hasAttr<StdCallAttr>())
267 return CC_X86StdCall;
268
269 if (D->hasAttr<FastCallAttr>())
270 return CC_X86FastCall;
271
272 if (D->hasAttr<RegCallAttr>())
273 return CC_X86RegCall;
274
275 if (D->hasAttr<ThisCallAttr>())
276 return CC_X86ThisCall;
277
278 if (D->hasAttr<VectorCallAttr>())
279 return CC_X86VectorCall;
280
281 if (D->hasAttr<PascalAttr>())
282 return CC_X86Pascal;
283
284 if (PcsAttr *PCS = D->getAttr<PcsAttr>())
285 return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
286
287 if (D->hasAttr<AArch64VectorPcsAttr>())
288 return CC_AArch64VectorCall;
289
290 if (D->hasAttr<AArch64SVEPcsAttr>())
291 return CC_AArch64SVEPCS;
292
293 if (D->hasAttr<DeviceKernelAttr>())
294 return CC_DeviceKernel;
295
296 if (D->hasAttr<IntelOclBiccAttr>())
297 return CC_IntelOclBicc;
298
299 if (D->hasAttr<MSABIAttr>())
300 return IsTargetDefaultMSABI ? CC_C : CC_Win64;
301
302 if (D->hasAttr<SysVABIAttr>())
303 return IsTargetDefaultMSABI ? CC_X86_64SysV : CC_C;
304
305 if (D->hasAttr<PreserveMostAttr>())
306 return CC_PreserveMost;
307
308 if (D->hasAttr<PreserveAllAttr>())
309 return CC_PreserveAll;
310
311 if (D->hasAttr<M68kRTDAttr>())
312 return CC_M68kRTD;
313
314 if (D->hasAttr<PreserveNoneAttr>())
315 return CC_PreserveNone;
316
317 if (D->hasAttr<RISCVVectorCCAttr>())
318 return CC_RISCVVectorCall;
319
320 if (RISCVVLSCCAttr *PCS = D->getAttr<RISCVVLSCCAttr>()) {
321 switch (PCS->getVectorWidth()) {
322 default:
323 llvm_unreachable("Invalid RISC-V VLS ABI VLEN");
324#define CC_VLS_CASE(ABI_VLEN) \
325 case ABI_VLEN: \
326 return CC_RISCVVLSCall_##ABI_VLEN;
327 CC_VLS_CASE(32)
328 CC_VLS_CASE(64)
329 CC_VLS_CASE(128)
330 CC_VLS_CASE(256)
331 CC_VLS_CASE(512)
332 CC_VLS_CASE(1024)
333 CC_VLS_CASE(2048)
334 CC_VLS_CASE(4096)
335 CC_VLS_CASE(8192)
336 CC_VLS_CASE(16384)
337 CC_VLS_CASE(32768)
338 CC_VLS_CASE(65536)
339#undef CC_VLS_CASE
340 }
341 }
342
343 return CC_C;
344}
345
346/// Arrange the argument and result information for a call to an
347/// unknown C++ non-static member function of the given abstract type.
348/// (A null RD means we don't have any meaningful "this" argument type,
349/// so fall back to a generic pointer type).
350/// The member function must be an ordinary function, i.e. not a
351/// constructor or destructor.
352const CGFunctionInfo &
353CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
354 const FunctionProtoType *FTP,
355 const CXXMethodDecl *MD) {
356 CanQualTypeList argTypes;
357
358 // Add the 'this' pointer.
359 argTypes.push_back(Elt: DeriveThisType(RD, MD));
360 auto CanonicalFTP =
361 FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>();
362 ExtParameterInfoList paramInfos;
363 RequiredArgs required = RequiredArgs::forPrototypePlus(
364 prototype: CanonicalFTP.getTypePtr(), additional: argTypes.size());
365 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: CanonicalFTP);
366 return arrangeLLVMFunctionInfo(
367 returnType: CanonicalFTP->getReturnType().getUnqualifiedType(),
368 opts: FnInfoOpts::IsInstanceMethod, argTypes, info: CanonicalFTP->getExtInfo(),
369 paramInfos, args: required, ABIInfoFD: MD);
370}
371
372/// Set calling convention for CUDA/HIP kernel.
373static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
374 const FunctionDecl *FD) {
375 if (FD->hasAttr<CUDAGlobalAttr>()) {
376 const FunctionType *FT = FTy->getAs<FunctionType>();
377 CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
378 FTy = FT->getCanonicalTypeUnqualified();
379 }
380}
381
382/// Arrange the argument and result information for a declaration or
383/// definition of the given C++ non-static member function. The
384/// member function must be an ordinary function, i.e. not a
385/// constructor or destructor.
386const CGFunctionInfo &
387CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
388 assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
389 assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
390
391 CanQualType FT = CodeGenUtils::getFormalType(MD).getAs<Type>();
392 setCUDAKernelCallingConvention(FTy&: FT, CGM, FD: MD);
393 auto prototype = FT.getAs<FunctionProtoType>();
394
395 if (MD->isImplicitObjectMemberFunction()) {
396 // The abstract case is perfectly fine.
397 const CXXRecordDecl *ThisType =
398 getCXXABI().getThisArgumentTypeForMethod(GD: MD);
399 return arrangeCXXMethodType(RD: ThisType, FTP: prototype.getTypePtr(), MD);
400 }
401
402 CanQualTypeList argTypes;
403 ExtParameterInfoList paramInfos;
404 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: prototype);
405 return arrangeLLVMFunctionInfo(
406 returnType: prototype->getReturnType().getUnqualifiedType(), opts: FnInfoOpts::None,
407 argTypes, info: prototype->getExtInfo(), paramInfos,
408 args: RequiredArgs::forPrototypePlus(prototype: prototype.getTypePtr(), additional: 0), ABIInfoFD: MD);
409}
410
411bool CodeGenTypes::inheritingCtorHasParams(
412 const InheritedConstructor &Inherited, CXXCtorType Type) {
413 // Parameters are unnecessary if we're constructing a base class subobject
414 // and the inherited constructor lives in a virtual base.
415 return Type == Ctor_Complete ||
416 !Inherited.getShadowDecl()->constructsVirtualBase() ||
417 !Target.getCXXABI().hasConstructorVariants();
418}
419
420const CGFunctionInfo &
421CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
422 auto *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
423
424 CanQualTypeList argTypes;
425 ExtParameterInfoList paramInfos;
426
427 const CXXRecordDecl *ThisType = getCXXABI().getThisArgumentTypeForMethod(GD);
428 argTypes.push_back(Elt: DeriveThisType(RD: ThisType, MD));
429
430 bool PassParams = true;
431
432 if (auto *CD = dyn_cast<CXXConstructorDecl>(Val: MD)) {
433 // A base class inheriting constructor doesn't get forwarded arguments
434 // needed to construct a virtual base (or base class thereof).
435 if (auto Inherited = CD->getInheritedConstructor())
436 PassParams = inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
437 }
438
439 CanQual<FunctionProtoType> FTP = CodeGenUtils::getFormalType(MD);
440
441 // Add the formal parameters.
442 if (PassParams)
443 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: FTP);
444
445 CGCXXABI::AddedStructorArgCounts AddedArgs =
446 getCXXABI().buildStructorSignature(GD, ArgTys&: argTypes);
447 if (!paramInfos.empty()) {
448 // Note: prefix implies after the first param.
449 if (AddedArgs.Prefix)
450 paramInfos.insert(I: paramInfos.begin() + 1, NumToInsert: AddedArgs.Prefix,
451 Elt: FunctionProtoType::ExtParameterInfo{});
452 if (AddedArgs.Suffix)
453 paramInfos.append(NumInputs: AddedArgs.Suffix,
454 Elt: FunctionProtoType::ExtParameterInfo{});
455 }
456
457 RequiredArgs required =
458 (PassParams && MD->isVariadic() ? RequiredArgs(argTypes.size())
459 : RequiredArgs::All);
460
461 FunctionType::ExtInfo extInfo = FTP->getExtInfo();
462 CanQualType resultType = getCXXABI().HasThisReturn(GD) ? argTypes.front()
463 : getCXXABI().hasMostDerivedReturn(GD)
464 ? CGM.getContext().VoidPtrTy
465 : Context.VoidTy;
466 return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::IsInstanceMethod,
467 argTypes, info: extInfo, paramInfos, args: required, ABIInfoFD: MD);
468}
469
470static CanQualTypeList getArgTypesForCall(ASTContext &ctx,
471 const CallArgList &args) {
472 CanQualTypeList argTypes;
473 for (auto &arg : args)
474 argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg.Ty));
475 return argTypes;
476}
477
478static CanQualTypeList getArgTypesForDeclaration(ASTContext &ctx,
479 const FunctionArgList &args) {
480 CanQualTypeList argTypes;
481 for (auto &arg : args)
482 argTypes.push_back(Elt: ctx.getCanonicalParamType(T: arg->getType()));
483 return argTypes;
484}
485
486static ExtParameterInfoList
487getExtParameterInfosForCall(const FunctionProtoType *proto, unsigned prefixArgs,
488 unsigned totalArgs) {
489 ExtParameterInfoList result;
490 if (proto->hasExtParameterInfos()) {
491 addExtParameterInfosForCall(paramInfos&: result, proto, prefixArgs, totalArgs);
492 }
493 return result;
494}
495
496/// Arrange a call to a C++ method, passing the given arguments.
497///
498/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
499/// parameter.
500/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
501/// args.
502/// PassProtoArgs indicates whether `args` has args for the parameters in the
503/// given CXXConstructorDecl.
504const CGFunctionInfo &CodeGenTypes::arrangeCXXConstructorCall(
505 const CallArgList &args, const CXXConstructorDecl *D, CXXCtorType CtorKind,
506 unsigned ExtraPrefixArgs, unsigned ExtraSuffixArgs,
507 const FunctionDecl *ABIInfoFD, bool PassProtoArgs) {
508 CanQualTypeList ArgTypes;
509 for (const auto &Arg : args)
510 ArgTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
511
512 // +1 for implicit this, which should always be args[0].
513 unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
514
515 CanQual<FunctionProtoType> FPT = CodeGenUtils::getFormalType(MD: D);
516 RequiredArgs Required = PassProtoArgs
517 ? RequiredArgs::forPrototypePlus(
518 prototype: FPT, additional: TotalPrefixArgs + ExtraSuffixArgs)
519 : RequiredArgs::All;
520
521 GlobalDecl GD(D, CtorKind);
522 CanQualType ResultType = getCXXABI().HasThisReturn(GD) ? ArgTypes.front()
523 : getCXXABI().hasMostDerivedReturn(GD)
524 ? CGM.getContext().VoidPtrTy
525 : Context.VoidTy;
526
527 FunctionType::ExtInfo Info = FPT->getExtInfo();
528 ExtParameterInfoList ParamInfos;
529 // If the prototype args are elided, we should only have ABI-specific args,
530 // which never have param info.
531 if (PassProtoArgs && FPT->hasExtParameterInfos()) {
532 // ABI-specific suffix arguments are treated the same as variadic arguments.
533 addExtParameterInfosForCall(paramInfos&: ParamInfos, proto: FPT.getTypePtr(), prefixArgs: TotalPrefixArgs,
534 totalArgs: ArgTypes.size());
535 }
536
537 return arrangeLLVMFunctionInfo(returnType: ResultType, opts: FnInfoOpts::IsInstanceMethod,
538 argTypes: ArgTypes, info: Info, paramInfos: ParamInfos, args: Required,
539 ABIInfoFD);
540}
541
542/// Arrange the argument and result information for the declaration or
543/// definition of the given function.
544const CGFunctionInfo &
545CodeGenTypes::arrangeFunctionDeclaration(const GlobalDecl GD) {
546 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
547 if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD))
548 if (MD->isImplicitObjectMemberFunction())
549 return arrangeCXXMethodDeclaration(MD);
550
551 CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
552
553 assert(isa<FunctionType>(FTy));
554 setCUDAKernelCallingConvention(FTy, CGM, FD);
555
556 if (DeviceKernelAttr::isOpenCLSpelling(A: FD->getAttr<DeviceKernelAttr>()) &&
557 GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
558 const FunctionType *FT = FTy->getAs<FunctionType>();
559 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
560 FTy = FT->getCanonicalTypeUnqualified();
561 }
562
563 // When declaring a function without a prototype, always use a
564 // non-variadic type.
565 if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
566 return arrangeLLVMFunctionInfo(returnType: noProto->getReturnType(), opts: FnInfoOpts::None,
567 argTypes: {}, info: noProto->getExtInfo(), paramInfos: {},
568 args: RequiredArgs::All, ABIInfoFD: FD);
569 }
570
571 CanQual<FunctionProtoType> FTP = FTy.castAs<FunctionProtoType>();
572 CanQualTypeList argTypes;
573 ExtParameterInfoList paramInfos;
574 appendParameterTypes(CGT: *this, prefix&: argTypes, paramInfos, FPT: FTP);
575 return arrangeLLVMFunctionInfo(returnType: FTP->getReturnType().getUnqualifiedType(),
576 opts: FnInfoOpts::None, argTypes, info: FTP->getExtInfo(),
577 paramInfos,
578 args: RequiredArgs::forPrototypePlus(prototype: FTP, additional: 0), ABIInfoFD: FD);
579}
580
581/// Arrange the argument and result information for the declaration or
582/// definition of an Objective-C method.
583const CGFunctionInfo &
584CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
585 // It happens that this is the same as a call with no optional
586 // arguments, except also using the formal 'self' type.
587 return arrangeObjCMessageSendSignature(MD, receiverType: MD->getSelfDecl()->getType());
588}
589
590/// Arrange the argument and result information for the function type
591/// through which to perform a send to the given Objective-C method,
592/// using the given receiver type. The receiver type is not always
593/// the 'self' type of the method or even an Objective-C pointer type.
594/// This is *not* the right method for actually performing such a
595/// message send, due to the possibility of optional arguments.
596const CGFunctionInfo &
597CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
598 QualType receiverType) {
599 CanQualTypeList argTys;
600 ExtParameterInfoList extParamInfos(MD->isDirectMethod() ? 1 : 2);
601 argTys.push_back(Elt: Context.getCanonicalParamType(T: receiverType));
602 if (!MD->isDirectMethod())
603 argTys.push_back(Elt: Context.getCanonicalParamType(T: Context.getObjCSelType()));
604 for (const auto *I : MD->parameters()) {
605 argTys.push_back(Elt: Context.getCanonicalParamType(T: I->getType()));
606 auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
607 NoEscape: I->hasAttr<NoEscapeAttr>());
608 extParamInfos.push_back(Elt: extParamInfo);
609 }
610
611 FunctionType::ExtInfo einfo;
612 bool IsTargetDefaultMSABI =
613 getContext().getTargetInfo().getTriple().isOSWindows() ||
614 getContext().getTargetInfo().getTriple().isUEFI();
615 einfo = einfo.withCallingConv(
616 cc: getCallingConventionForDecl(D: MD, IsTargetDefaultMSABI));
617
618 if (getContext().getLangOpts().ObjCAutoRefCount &&
619 MD->hasAttr<NSReturnsRetainedAttr>())
620 einfo = einfo.withProducesResult(producesResult: true);
621
622 RequiredArgs required =
623 (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);
624
625 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: MD->getReturnType()),
626 opts: FnInfoOpts::None, argTypes: argTys, info: einfo, paramInfos: extParamInfos,
627 args: required, /*ABIInfoFD=*/nullptr);
628}
629
630const CGFunctionInfo &
631CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
632 const CallArgList &args) {
633 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
634 FunctionType::ExtInfo einfo;
635
636 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: returnType), opts: FnInfoOpts::None,
637 argTypes, info: einfo, paramInfos: {}, args: RequiredArgs::All,
638 ABIInfoFD: nullptr);
639}
640
641const CGFunctionInfo &CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
642 // FIXME: Do we need to handle ObjCMethodDecl?
643 if (isa<CXXConstructorDecl>(Val: GD.getDecl()) ||
644 isa<CXXDestructorDecl>(Val: GD.getDecl()))
645 return arrangeCXXStructorDeclaration(GD);
646
647 return arrangeFunctionDeclaration(GD);
648}
649
650/// Arrange a thunk that takes 'this' as the first parameter followed by
651/// varargs. Return a void pointer, regardless of the actual return type.
652/// The body of the thunk will end in a musttail call to a function of the
653/// correct type, and the caller will bitcast the function to the correct
654/// prototype.
655const CGFunctionInfo &
656CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
657 assert(MD->isVirtual() && "only methods have thunks");
658 CanQual<FunctionProtoType> FTP = CodeGenUtils::getFormalType(MD);
659 CanQualType ArgTys[] = {DeriveThisType(RD: MD->getParent(), MD)};
660 return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::None, argTypes: ArgTys,
661 info: FTP->getExtInfo(), paramInfos: {}, args: RequiredArgs(1), ABIInfoFD: MD);
662}
663
664const CGFunctionInfo &
665CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
666 CXXCtorType CT) {
667 assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
668
669 CanQual<FunctionProtoType> FTP = CodeGenUtils::getFormalType(MD: CD);
670 SmallVector<CanQualType, 2> ArgTys;
671 const CXXRecordDecl *RD = CD->getParent();
672 ArgTys.push_back(Elt: DeriveThisType(RD, MD: CD));
673 if (CT == Ctor_CopyingClosure)
674 ArgTys.push_back(Elt: *FTP->param_type_begin());
675 if (RD->getNumVBases() > 0)
676 ArgTys.push_back(Elt: Context.IntTy);
677 CallingConv CC = Context.getDefaultCallingConvention(
678 /*IsVariadic=*/false, /*IsCXXMethod=*/true);
679 return arrangeLLVMFunctionInfo(returnType: Context.VoidTy, opts: FnInfoOpts::IsInstanceMethod,
680 argTypes: ArgTys, info: FunctionType::ExtInfo(CC), paramInfos: {},
681 args: RequiredArgs::All, /*ABIInfoFD=*/nullptr);
682}
683
684/// Arrange a call as unto a free function, except possibly with an
685/// additional number of formal parameters considered required.
686static const CGFunctionInfo &
687arrangeFreeFunctionLikeCall(CodeGenTypes &CGT, CodeGenModule &CGM,
688 const CallArgList &args, const FunctionType *fnType,
689 unsigned numExtraRequiredArgs, bool chainCall,
690 const FunctionDecl *ABIInfoFD) {
691 assert(args.size() >= numExtraRequiredArgs);
692
693 ExtParameterInfoList paramInfos;
694
695 // In most cases, there are no optional arguments.
696 RequiredArgs required = RequiredArgs::All;
697
698 // If we have a variadic prototype, the required arguments are the
699 // extra prefix plus the arguments in the prototype.
700 if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(Val: fnType)) {
701 if (proto->isVariadic())
702 required = RequiredArgs::forPrototypePlus(prototype: proto, additional: numExtraRequiredArgs);
703
704 if (proto->hasExtParameterInfos())
705 addExtParameterInfosForCall(paramInfos, proto, prefixArgs: numExtraRequiredArgs,
706 totalArgs: args.size());
707
708 // If we don't have a prototype at all, but we're supposed to
709 // explicitly use the variadic convention for unprototyped calls,
710 // treat all of the arguments as required but preserve the nominal
711 // possibility of variadics.
712 } else if (CGM.getTargetCodeGenInfo().isNoProtoCallVariadic(
713 args, fnType: cast<FunctionNoProtoType>(Val: fnType))) {
714 required = RequiredArgs(args.size());
715 }
716
717 CanQualTypeList argTypes;
718 for (const auto &arg : args)
719 argTypes.push_back(Elt: CGT.getContext().getCanonicalParamType(T: arg.Ty));
720 FnInfoOpts opts = chainCall ? FnInfoOpts::IsChainCall : FnInfoOpts::None;
721 return CGT.arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: fnType->getReturnType()),
722 opts, argTypes, info: fnType->getExtInfo(),
723 paramInfos, args: required, ABIInfoFD);
724}
725
726/// Figure out the rules for calling a function with the given formal
727/// type using the given arguments. The arguments are necessary
728/// because the function might be unprototyped, in which case it's
729/// target-dependent in crazy ways.
730const CGFunctionInfo &CodeGenTypes::arrangeFreeFunctionCall(
731 const CallArgList &args, const FunctionType *fnType, bool chainCall,
732 const FunctionDecl *ABIInfoFD) {
733 return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType,
734 numExtraRequiredArgs: chainCall ? 1 : 0, chainCall, ABIInfoFD);
735}
736
737/// A block function is essentially a free function with an
738/// extra implicit argument.
739const CGFunctionInfo &
740CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
741 const FunctionType *fnType) {
742 // FIXME: Pass the enclosing function's ABI information so block calls use
743 // the caller's target features.
744 return arrangeFreeFunctionLikeCall(CGT&: *this, CGM, args, fnType, numExtraRequiredArgs: 1,
745 /*chainCall=*/false, ABIInfoFD: nullptr);
746}
747
748const CGFunctionInfo &
749CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
750 const FunctionArgList &params) {
751 ExtParameterInfoList paramInfos =
752 getExtParameterInfosForCall(proto, prefixArgs: 1, totalArgs: params.size());
753 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args: params);
754
755 // FIXME: Use the block's target features when arranging its invoke function.
756 return arrangeLLVMFunctionInfo(
757 returnType: GetReturnType(RetTy: proto->getReturnType()), opts: FnInfoOpts::None, argTypes,
758 info: proto->getExtInfo(), paramInfos, args: RequiredArgs::forPrototypePlus(prototype: proto, additional: 1),
759 /*ABIInfoFD=*/nullptr);
760}
761
762const CGFunctionInfo &
763CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
764 const CallArgList &args) {
765 CanQualTypeList argTypes;
766 for (const auto &Arg : args)
767 argTypes.push_back(Elt: Context.getCanonicalParamType(T: Arg.Ty));
768 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
769 argTypes, info: FunctionType::ExtInfo(),
770 /*paramInfos=*/{}, args: RequiredArgs::All, ABIInfoFD: nullptr);
771}
772
773const CGFunctionInfo &
774CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
775 const FunctionArgList &args) {
776 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
777
778 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
779 argTypes, info: FunctionType::ExtInfo(), paramInfos: {},
780 args: RequiredArgs::All, /*ABIInfoFD=*/nullptr);
781}
782
783const CGFunctionInfo &CodeGenTypes::arrangeBuiltinFunctionDeclaration(
784 CanQualType resultType, ArrayRef<CanQualType> argTypes) {
785 return arrangeLLVMFunctionInfo(returnType: resultType, opts: FnInfoOpts::None, argTypes,
786 info: FunctionType::ExtInfo(), paramInfos: {}, args: RequiredArgs::All,
787 /*ABIInfoFD=*/nullptr);
788}
789
790const CGFunctionInfo &CodeGenTypes::arrangeDeviceKernelCallerDeclaration(
791 QualType resultType, const FunctionArgList &args) {
792 CanQualTypeList argTypes = getArgTypesForDeclaration(ctx&: Context, args);
793
794 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: resultType), opts: FnInfoOpts::None,
795 argTypes,
796 info: FunctionType::ExtInfo(CC_DeviceKernel),
797 /*paramInfos=*/{}, args: RequiredArgs::All,
798 /*ABIInfoFD=*/nullptr);
799}
800
801/// Arrange a call to a C++ method, passing the given arguments.
802///
803/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
804/// does not count `this`.
805const CGFunctionInfo &CodeGenTypes::arrangeCXXMethodCall(
806 const CallArgList &args, const FunctionProtoType *proto,
807 RequiredArgs required, unsigned numPrefixArgs,
808 const FunctionDecl *ABIInfoFD) {
809 assert(numPrefixArgs + 1 <= args.size() &&
810 "Emitting a call with less args than the required prefix?");
811 // Add one to account for `this`. It's a bit awkward here, but we don't count
812 // `this` in similar places elsewhere.
813 ExtParameterInfoList paramInfos =
814 getExtParameterInfosForCall(proto, prefixArgs: numPrefixArgs + 1, totalArgs: args.size());
815
816 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
817
818 FunctionType::ExtInfo info = proto->getExtInfo();
819 return arrangeLLVMFunctionInfo(returnType: GetReturnType(RetTy: proto->getReturnType()),
820 opts: FnInfoOpts::IsInstanceMethod, argTypes, info,
821 paramInfos, args: required, ABIInfoFD);
822}
823
824const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
825 return arrangeLLVMFunctionInfo(returnType: getContext().VoidTy, opts: FnInfoOpts::None, argTypes: {},
826 info: FunctionType::ExtInfo(), paramInfos: {}, args: RequiredArgs::All,
827 /*ABIInfoFD=*/nullptr);
828}
829
830const CGFunctionInfo &CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
831 const CallArgList &args,
832 const FunctionDecl *ABIInfoFD) {
833 assert(signature.arg_size() <= args.size());
834 unsigned X86ABIAVXLevel =
835 CGM.getABIInfo().getX86ABIAVXLevel(ABIInfoFD, signature.getExtInfo());
836 if (signature.arg_size() == args.size() &&
837 signature.getX86ABIAVXLevel() == X86ABIAVXLevel)
838 return signature;
839
840 ExtParameterInfoList paramInfos;
841 auto sigParamInfos = signature.getExtParameterInfos();
842 if (!sigParamInfos.empty()) {
843 paramInfos.append(in_start: sigParamInfos.begin(), in_end: sigParamInfos.end());
844 paramInfos.resize(N: args.size());
845 }
846
847 CanQualTypeList argTypes = getArgTypesForCall(ctx&: Context, args);
848
849 assert(signature.getRequiredArgs().allowsOptionalArgs());
850 FnInfoOpts opts = FnInfoOpts::None;
851 if (signature.isInstanceMethod())
852 opts |= FnInfoOpts::IsInstanceMethod;
853 if (signature.isChainCall())
854 opts |= FnInfoOpts::IsChainCall;
855 if (signature.isDelegateCall())
856 opts |= FnInfoOpts::IsDelegateCall;
857
858 const CGFunctionInfo *newFI = findOrInsertCGFunctionInfo(
859 isInstanceMethod: signature.isInstanceMethod(), isChainCall: signature.isChainCall(),
860 isDelegateCall: signature.isDelegateCall(), X86ABIAVXLevel, info: signature.getExtInfo(),
861 paramInfos, required: signature.getRequiredArgs(), resultType: signature.getReturnType(),
862 argTypes);
863 return *newFI;
864}
865
866namespace clang {
867namespace CodeGen {
868void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
869} // namespace CodeGen
870} // namespace clang
871
872#ifndef NDEBUG
873static const char *abiKindToString(ABIArgInfo::Kind K) {
874 switch (K) {
875 case ABIArgInfo::Direct:
876 return "Direct";
877 case ABIArgInfo::Extend:
878 return "Extend";
879 case ABIArgInfo::Indirect:
880 return "Indirect";
881 case ABIArgInfo::IndirectAliased:
882 return "IndirectAliased";
883 case ABIArgInfo::Ignore:
884 return "Ignore";
885 case ABIArgInfo::Expand:
886 return "Expand";
887 case ABIArgInfo::CoerceAndExpand:
888 return "CoerceAndExpand";
889 case ABIArgInfo::TargetSpecific:
890 return "TargetSpecific";
891 case ABIArgInfo::InAlloca:
892 return "InAlloca";
893 }
894 llvm_unreachable("Unknown kind");
895}
896#endif
897
898void CodeGenModule::computeABIInfoUsingLib(CGFunctionInfo &FI) {
899 SmallVector<const llvm::abi::Type *> MappedArgTypes;
900 MappedArgTypes.reserve(N: FI.arg_size());
901 for (const auto &Arg : FI.arguments())
902 MappedArgTypes.push_back(Elt: AbiMapper->convertType(QT: Arg.type));
903
904 RequiredArgs Required = FI.getRequiredArgs();
905 llvm::abi::RequiredArgs AbiRequired = llvm::abi::RequiredArgs::All;
906 if (Required.allowsOptionalArgs())
907 AbiRequired = llvm::abi::RequiredArgs(Required.getNumRequiredArgs());
908
909 auto AbiFI = llvm::abi::FunctionInfo::create(
910 CC: FI.getCallingConvention(), ReturnType: AbiMapper->convertType(QT: FI.getReturnType()),
911 ArgTypes: MappedArgTypes, Required: AbiRequired);
912
913 getLLVMABITargetInfo(TB&: AbiMapper->getTypeBuilder()).computeInfo(FI&: *AbiFI);
914
915#ifndef NDEBUG
916 // With assertions enabled, also compute info using Clang ABI logic,
917 // so we can ensure the results are consistent.
918 getABIInfo().computeInfo(FI);
919
920 auto ConvertABIArgInfo = [&](ABIArgInfo &Target,
921 const llvm::abi::ArgInfo &AbiInfo, QualType Type,
922 int ArgNo) {
923 auto Check = [&](bool Cond, llvm::function_ref<void()> MessageFn) {
924 if (Cond)
925 return;
926 if (ArgNo == -1)
927 llvm::dbgs() << "For return value of type ";
928 else
929 llvm::dbgs() << "For argument " << ArgNo << " of type ";
930 llvm::dbgs() << Type << ": ";
931 MessageFn();
932 llvm::dbgs() << "\n";
933 abort();
934 };
935 auto CheckSimple = [&](auto TargetVal, auto ResVal, StringRef What) {
936 Check(TargetVal == ResVal, [&]() {
937 llvm::dbgs() << What << " mismatch (expected: " << TargetVal
938 << ", given: " << ResVal << ")";
939 });
940 };
941
942 ABIArgInfo Res = convertABIArgInfo(AbiInfo, Type);
943 Check(Target.getKind() == Res.getKind(), [&]() {
944 llvm::dbgs() << "Kind mismatch (expected: "
945 << abiKindToString(Target.getKind())
946 << ", given: " << abiKindToString(Res.getKind()) << ")";
947 });
948
949 if (Res.canHaveCoerceToType()) {
950 // Normalize nullptr types.
951 llvm::Type *TargetType = Target.getCoerceToType();
952 llvm::Type *ResType = Res.getCoerceToType();
953 if (!TargetType)
954 TargetType = getTypes().ConvertType(Type);
955 if (!ResType)
956 ResType = getTypes().ConvertType(Type);
957
958 Check(TargetType == ResType, [&]() {
959 llvm::dbgs() << "CoerceToType mismatch (expected: " << *TargetType
960 << ", given: " << *ResType << ")";
961 });
962 }
963
964 switch (Res.getKind()) {
965 case ABIArgInfo::Extend:
966 CheckSimple(Target.isSignExt(), Res.isSignExt(), "SignExt");
967 CheckSimple(Target.isZeroExt(), Res.isZeroExt(), "ZeroExt");
968 CheckSimple(Target.isConservative(), Res.isConservative(),
969 "Conservative");
970 [[fallthrough]];
971 case ABIArgInfo::Direct:
972 CheckSimple(Target.getDirectAlign(), Res.getDirectAlign(), "DirectAlign");
973 CheckSimple(Target.getDirectOffset(), Res.getDirectOffset(),
974 "DirectOffset");
975 // Extend falls through to here, and only Direct carries the flag.
976 if (Res.isDirect())
977 CheckSimple(Target.getCanBeFlattened(), Res.getCanBeFlattened(),
978 "CanBeFlattened");
979 break;
980 case ABIArgInfo::Indirect:
981 CheckSimple(Target.getIndirectByVal(), Res.getIndirectByVal(),
982 "IndirectByVal");
983 [[fallthrough]];
984 case ABIArgInfo::IndirectAliased:
985 CheckSimple(Target.getIndirectAddrSpace(), Res.getIndirectAddrSpace(),
986 "IndirectAddrSpace");
987 CheckSimple(Target.getIndirectRealign(), Res.getIndirectRealign(),
988 "IndirectRealign");
989 Check(Target.getIndirectAlign() == Res.getIndirectAlign(), [&]() {
990 llvm::dbgs() << "IndirectAlign mismatch (expected: "
991 << Target.getIndirectAlign().getQuantity()
992 << ", given: " << Res.getIndirectAlign().getQuantity()
993 << ")";
994 });
995 break;
996 case ABIArgInfo::CoerceAndExpand:
997 Check(Target.getUnpaddedCoerceAndExpandType() ==
998 Res.getUnpaddedCoerceAndExpandType(),
999 [&]() {
1000 llvm::dbgs() << "UnpaddedCoerceAndExpandType mismatch (expected: "
1001 << *Target.getUnpaddedCoerceAndExpandType()
1002 << ", given: "
1003 << *Res.getUnpaddedCoerceAndExpandType() << ")";
1004 });
1005 break;
1006 default:
1007 break;
1008 }
1009
1010 Target = Res;
1011 };
1012#else
1013 auto ConvertABIArgInfo =
1014 [&](ABIArgInfo &Target, const llvm::abi::ArgInfo &AbiInfo, QualType Type,
1015 int ArgNo) { Target = convertABIArgInfo(AbiInfo, Type); };
1016#endif
1017
1018 ConvertABIArgInfo(FI.getReturnInfo(), AbiFI->getReturnInfo(),
1019 FI.getReturnType(), -1);
1020
1021 int ArgNo = 0;
1022 for (auto [CGArg, AbiArg] :
1023 llvm::zip_equal(t: FI.arguments(), u: AbiFI->arguments()))
1024 ConvertABIArgInfo(CGArg.info, AbiArg.Info, CGArg.type, ArgNo++);
1025}
1026
1027ABIArgInfo CodeGenModule::convertABIArgInfo(const llvm::abi::ArgInfo &AbiInfo,
1028 QualType Type) {
1029 switch (AbiInfo.getKind()) {
1030 case llvm::abi::ArgInfo::Direct: {
1031 llvm::Type *CoercedType = nullptr;
1032 if (AbiInfo.getCoerceToType())
1033 CoercedType = AbiReverseMapper->convertType(ABIType: AbiInfo.getCoerceToType());
1034 if (!CoercedType)
1035 CoercedType = getTypes().ConvertType(T: Type);
1036 unsigned DirectAlign = 0;
1037 if (llvm::MaybeAlign Align = AbiInfo.getDirectAlign())
1038 DirectAlign = Align->value();
1039 // TODO: Move Padding into the ABIArgInfo struct when we add support for
1040 // targets that need a different setting than we have here.
1041 return ABIArgInfo::getDirect(T: CoercedType, Offset: AbiInfo.getDirectOffset(),
1042 /*Padding=*/nullptr,
1043 CanBeFlattened: AbiInfo.getCanBeFlattened(), Align: DirectAlign);
1044 }
1045 case llvm::abi::ArgInfo::Extend: {
1046 llvm::Type *CoercedType = nullptr;
1047 if (AbiInfo.getCoerceToType())
1048 CoercedType = AbiReverseMapper->convertType(ABIType: AbiInfo.getCoerceToType());
1049 if (!CoercedType)
1050 CoercedType = getTypes().ConvertType(T: Type);
1051 // A transparent union is passed as its first field, so the extend keys off
1052 // that field's integral type, matching the classifier's
1053 // useFirstFieldIfTransparentUnion. Passing the union type to
1054 // ABIArgInfo::getSignExtend would trip its integral-type assert.
1055 QualType ExtendType = useFirstFieldIfTransparentUnion(Ty: Type);
1056 ABIArgInfo Res;
1057 if (AbiInfo.isSignExt())
1058 Res = ABIArgInfo::getSignExtend(Ty: ExtendType, T: CoercedType);
1059 else if (AbiInfo.isZeroExt())
1060 Res = ABIArgInfo::getZeroExtend(Ty: ExtendType, T: CoercedType);
1061 else
1062 Res = ABIArgInfo::getNoExtend(T: cast<llvm::IntegerType>(Val: CoercedType));
1063 Res.setConservative(AbiInfo.isConservative());
1064 return Res;
1065 }
1066 case llvm::abi::ArgInfo::Indirect: {
1067 CharUnits Alignment =
1068 CharUnits::fromQuantity(Quantity: AbiInfo.getIndirectAlign().value());
1069 return ABIArgInfo::getIndirect(Alignment, AddrSpace: AbiInfo.getIndirectAddrSpace(),
1070 ByVal: AbiInfo.getIndirectByVal(),
1071 Realign: AbiInfo.getIndirectRealign());
1072 }
1073 case llvm::abi::ArgInfo::IndirectAliased: {
1074 // Aliased indirect carries an address space but never byval.
1075 CharUnits Alignment =
1076 CharUnits::fromQuantity(Quantity: AbiInfo.getIndirectAlign().value());
1077 return ABIArgInfo::getIndirectAliased(Alignment,
1078 AddrSpace: AbiInfo.getIndirectAddrSpace(),
1079 Realign: AbiInfo.getIndirectRealign());
1080 }
1081 case llvm::abi::ArgInfo::Ignore:
1082 return ABIArgInfo::getIgnore();
1083 case llvm::abi::ArgInfo::CoerceAndExpand: {
1084 llvm::StructType *CoercedType = llvm::cast<llvm::StructType>(
1085 Val: AbiReverseMapper->convertType(ABIType: AbiInfo.getCoerceToType()));
1086 llvm::Type *UnpaddedType =
1087 AbiReverseMapper->convertType(ABIType: AbiInfo.getUnpaddedCoerceAndExpandType());
1088 return ABIArgInfo::getCoerceAndExpand(coerceToType: CoercedType, unpaddedCoerceToType: UnpaddedType);
1089 }
1090 }
1091 llvm_unreachable("Unexpected llvm::abi::ArgInfo kind");
1092}
1093
1094/// Arrange the argument and result information for an abstract value
1095/// of a given function type. This is the method which all of the
1096/// above functions ultimately defer to.
1097const CGFunctionInfo &CodeGenTypes::arrangeLLVMFunctionInfo(
1098 CanQualType resultType, FnInfoOpts opts, ArrayRef<CanQualType> argTypes,
1099 FunctionType::ExtInfo info,
1100 ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
1101 RequiredArgs required, const FunctionDecl *ABIInfoFD) {
1102 assert(llvm::all_of(argTypes,
1103 [](CanQualType T) { return T.isCanonicalAsParam(); }));
1104
1105 // Lookup or create unique function info.
1106 bool isInstanceMethod =
1107 (opts & FnInfoOpts::IsInstanceMethod) == FnInfoOpts::IsInstanceMethod;
1108 bool isChainCall =
1109 (opts & FnInfoOpts::IsChainCall) == FnInfoOpts::IsChainCall;
1110 bool isDelegateCall =
1111 (opts & FnInfoOpts::IsDelegateCall) == FnInfoOpts::IsDelegateCall;
1112 unsigned X86ABIAVXLevel = CGM.getABIInfo().getX86ABIAVXLevel(ABIInfoFD, info);
1113
1114 const CGFunctionInfo *newFI = findOrInsertCGFunctionInfo(
1115 isInstanceMethod, isChainCall, isDelegateCall, X86ABIAVXLevel, info,
1116 paramInfos, required, resultType, argTypes);
1117 return *newFI;
1118}
1119
1120CGFunctionInfo *CodeGenTypes::findOrInsertCGFunctionInfo(
1121 bool isInstanceMethod, bool isChainCall, bool isDelegateCall,
1122 unsigned X86ABIAVXLevel, const FunctionType::ExtInfo &info,
1123 ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
1124 RequiredArgs required, CanQualType resultType,
1125 ArrayRef<CanQualType> argTypes) {
1126 llvm::FoldingSetNodeID ID;
1127 CGFunctionInfo::Profile(ID, InstanceMethod: isInstanceMethod, ChainCall: isChainCall, IsDelegateCall: isDelegateCall,
1128 X86ABIAVXLevel, info, paramInfos, required,
1129 resultType, argTypes);
1130
1131 llvm::FoldingSetInsertToken InsertToken;
1132 CGFunctionInfo *FI = FunctionInfos.lookup(ID, Token&: InsertToken);
1133 if (FI)
1134 return FI;
1135
1136 unsigned CC = ClangCallConvToLLVMCallConv(CC: info.getCC());
1137
1138 // Construct the function info. We co-allocate the ArgInfos.
1139 FI = CGFunctionInfo::create(llvmCC: CC, instanceMethod: isInstanceMethod, chainCall: isChainCall, delegateCall: isDelegateCall,
1140 X86ABIAVXLevel, extInfo: info, paramInfos, resultType,
1141 argTypes, required);
1142 FunctionInfos.insert(N: FI, Token: InsertToken);
1143
1144 bool inserted = FunctionsBeingProcessed.insert(Ptr: FI).second;
1145 (void)inserted;
1146 assert(inserted && "Recursively being processed?");
1147
1148 // Compute ABI information.
1149 if (info.getCC() == CC_DeviceKernel &&
1150 (CC == llvm::CallingConv::SPIR_KERNEL || CC == llvm::CallingConv::C)) {
1151 // Force target independent argument handling for the host visible
1152 // kernel functions.
1153 //
1154 // For CPU targets, this currently only works for OpenCL.
1155 assert(CC != llvm::CallingConv::C || getContext().getLangOpts().OpenCL);
1156 // SPIR-V lowers OpenCL types to target extension types, which the ABI
1157 // library cannot tell apart from the kernel pointers it coerces yet, so
1158 // such kernels keep Clang's own lowering.
1159 auto IsSPIRVOpenCLType = [&](const CGFunctionInfoArgInfo &Arg) {
1160 return CGM.getTriple().isSPIRV() &&
1161 Arg.type.getTypePtr()->isOpenCLSpecificType();
1162 };
1163 if (CC == llvm::CallingConv::SPIR_KERNEL &&
1164 CGM.shouldUseLLVMABILowering(CallingConv: CC) &&
1165 llvm::none_of(Range: FI->arguments(), P: IsSPIRVOpenCLType))
1166 CGM.computeABIInfoUsingLib(FI&: *FI);
1167 else
1168 computeSPIRKernelABIInfo(CGM, FI&: *FI);
1169 } else if (info.getCC() == CC_Swift || info.getCC() == CC_SwiftAsync) {
1170 swiftcall::computeABIInfo(CGM, FI&: *FI);
1171 } else if (CGM.shouldUseLLVMABILowering(CallingConv: CC)) {
1172 CGM.computeABIInfoUsingLib(FI&: *FI);
1173 } else {
1174 CGM.getABIInfo().computeInfo(FI&: *FI);
1175 }
1176
1177 // Loop over all of the computed argument and return value info. If any of
1178 // them are direct or extend without a specified coerce type, specify the
1179 // default now.
1180 ABIArgInfo &retInfo = FI->getReturnInfo();
1181 if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
1182 retInfo.setCoerceToType(ConvertType(T: FI->getReturnType()));
1183
1184 for (auto &I : FI->arguments())
1185 if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
1186 I.info.setCoerceToType(ConvertType(T: I.type));
1187
1188 bool erased = FunctionsBeingProcessed.erase(Ptr: FI);
1189 (void)erased;
1190 assert(erased && "Not in set?");
1191
1192 return FI;
1193}
1194
1195CGFunctionInfo *CGFunctionInfo::create(
1196 unsigned llvmCC, bool instanceMethod, bool chainCall, bool delegateCall,
1197 unsigned X86ABIAVXLevel, const FunctionType::ExtInfo &info,
1198 ArrayRef<ExtParameterInfo> paramInfos, CanQualType resultType,
1199 ArrayRef<CanQualType> argTypes, RequiredArgs required) {
1200 assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
1201 assert(!required.allowsOptionalArgs() ||
1202 required.getNumRequiredArgs() <= argTypes.size());
1203
1204 void *buffer = operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(
1205 Counts: argTypes.size() + 1, Counts: paramInfos.size()));
1206
1207 CGFunctionInfo *FI = new (buffer) CGFunctionInfo();
1208 FI->CallingConvention = llvmCC;
1209 FI->EffectiveCallingConvention = llvmCC;
1210 FI->ASTCallingConvention = info.getCC();
1211 FI->InstanceMethod = instanceMethod;
1212 FI->ChainCall = chainCall;
1213 FI->DelegateCall = delegateCall;
1214 FI->CmseNSCall = info.getCmseNSCall();
1215 FI->NoReturn = info.getNoReturn();
1216 FI->ReturnsRetained = info.getProducesResult();
1217 FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
1218 FI->NoCfCheck = info.getNoCfCheck();
1219 FI->Required = required;
1220 FI->HasRegParm = info.getHasRegParm();
1221 FI->RegParm = info.getRegParm();
1222 FI->X86ABIAVXLevel = X86ABIAVXLevel;
1223 FI->ArgStruct = nullptr;
1224 FI->ArgStructAlign = 0;
1225 FI->NumArgs = argTypes.size();
1226 FI->HasExtParameterInfos = !paramInfos.empty();
1227 FI->getArgsBuffer()[0].type = resultType;
1228 FI->MaxVectorWidth = 0;
1229 for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
1230 FI->getArgsBuffer()[i + 1].type = argTypes[i];
1231 for (unsigned i = 0, e = paramInfos.size(); i != e; ++i)
1232 FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
1233 return FI;
1234}
1235
1236/***/
1237
1238namespace {
1239// ABIArgInfo::Expand implementation.
1240
1241// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
1242struct TypeExpansion {
1243 enum TypeExpansionKind {
1244 // Elements of constant arrays are expanded recursively.
1245 TEK_ConstantArray,
1246 // Record fields are expanded recursively (but if record is a union, only
1247 // the field with the largest size is expanded).
1248 TEK_Record,
1249 // For complex types, real and imaginary parts are expanded recursively.
1250 TEK_Complex,
1251 // All other types are not expandable.
1252 TEK_None
1253 };
1254
1255 const TypeExpansionKind Kind;
1256
1257 TypeExpansion(TypeExpansionKind K) : Kind(K) {}
1258 virtual ~TypeExpansion() {}
1259};
1260
1261struct ConstantArrayExpansion : TypeExpansion {
1262 QualType EltTy;
1263 uint64_t NumElts;
1264
1265 ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
1266 : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
1267 static bool classof(const TypeExpansion *TE) {
1268 return TE->Kind == TEK_ConstantArray;
1269 }
1270};
1271
1272struct RecordExpansion : TypeExpansion {
1273 SmallVector<const CXXBaseSpecifier *, 1> Bases;
1274
1275 SmallVector<const FieldDecl *, 1> Fields;
1276
1277 RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
1278 SmallVector<const FieldDecl *, 1> &&Fields)
1279 : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
1280 Fields(std::move(Fields)) {}
1281 static bool classof(const TypeExpansion *TE) {
1282 return TE->Kind == TEK_Record;
1283 }
1284};
1285
1286struct ComplexExpansion : TypeExpansion {
1287 QualType EltTy;
1288
1289 ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
1290 static bool classof(const TypeExpansion *TE) {
1291 return TE->Kind == TEK_Complex;
1292 }
1293};
1294
1295struct NoExpansion : TypeExpansion {
1296 NoExpansion() : TypeExpansion(TEK_None) {}
1297 static bool classof(const TypeExpansion *TE) { return TE->Kind == TEK_None; }
1298};
1299} // namespace
1300
1301static std::unique_ptr<TypeExpansion>
1302getTypeExpansion(QualType Ty, const ASTContext &Context) {
1303 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(T: Ty)) {
1304 return std::make_unique<ConstantArrayExpansion>(args: AT->getElementType(),
1305 args: AT->getZExtSize());
1306 }
1307 if (const auto *RD = Ty->getAsRecordDecl()) {
1308 SmallVector<const CXXBaseSpecifier *, 1> Bases;
1309 SmallVector<const FieldDecl *, 1> Fields;
1310 assert(!RD->hasFlexibleArrayMember() &&
1311 "Cannot expand structure with flexible array.");
1312 if (RD->isUnion()) {
1313 // Unions can be here only in degenerative cases - all the fields are same
1314 // after flattening. Thus we have to use the "largest" field.
1315 const FieldDecl *LargestFD = nullptr;
1316 CharUnits UnionSize = CharUnits::Zero();
1317
1318 for (const auto *FD : RD->fields()) {
1319 if (FD->isZeroLengthBitField())
1320 continue;
1321 assert(!FD->isBitField() &&
1322 "Cannot expand structure with bit-field members.");
1323 CharUnits FieldSize = Context.getTypeSizeInChars(T: FD->getType());
1324 if (UnionSize < FieldSize) {
1325 UnionSize = FieldSize;
1326 LargestFD = FD;
1327 }
1328 }
1329 if (LargestFD)
1330 Fields.push_back(Elt: LargestFD);
1331 } else {
1332 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD)) {
1333 assert(!CXXRD->isDynamicClass() &&
1334 "cannot expand vtable pointers in dynamic classes");
1335 llvm::append_range(C&: Bases, R: llvm::make_pointer_range(Range: CXXRD->bases()));
1336 }
1337
1338 for (const auto *FD : RD->fields()) {
1339 if (FD->isZeroLengthBitField())
1340 continue;
1341 assert(!FD->isBitField() &&
1342 "Cannot expand structure with bit-field members.");
1343 Fields.push_back(Elt: FD);
1344 }
1345 }
1346 return std::make_unique<RecordExpansion>(args: std::move(Bases),
1347 args: std::move(Fields));
1348 }
1349 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
1350 return std::make_unique<ComplexExpansion>(args: CT->getElementType());
1351 }
1352 return std::make_unique<NoExpansion>();
1353}
1354
1355static int getExpansionSize(QualType Ty, const ASTContext &Context) {
1356 auto Exp = getTypeExpansion(Ty, Context);
1357 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1358 return CAExp->NumElts * getExpansionSize(Ty: CAExp->EltTy, Context);
1359 }
1360 if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1361 int Res = 0;
1362 for (auto BS : RExp->Bases)
1363 Res += getExpansionSize(Ty: BS->getType(), Context);
1364 for (auto FD : RExp->Fields)
1365 Res += getExpansionSize(Ty: FD->getType(), Context);
1366 return Res;
1367 }
1368 if (isa<ComplexExpansion>(Val: Exp.get()))
1369 return 2;
1370 assert(isa<NoExpansion>(Exp.get()));
1371 return 1;
1372}
1373
1374void CodeGenTypes::getExpandedTypes(
1375 QualType Ty, SmallVectorImpl<llvm::Type *>::iterator &TI) {
1376 auto Exp = getTypeExpansion(Ty, Context);
1377 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1378 for (int i = 0, n = CAExp->NumElts; i < n; i++) {
1379 getExpandedTypes(Ty: CAExp->EltTy, TI);
1380 }
1381 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1382 for (auto BS : RExp->Bases)
1383 getExpandedTypes(Ty: BS->getType(), TI);
1384 for (auto FD : RExp->Fields)
1385 getExpandedTypes(Ty: FD->getType(), TI);
1386 } else if (auto CExp = dyn_cast<ComplexExpansion>(Val: Exp.get())) {
1387 llvm::Type *EltTy = ConvertType(T: CExp->EltTy);
1388 *TI++ = EltTy;
1389 *TI++ = EltTy;
1390 } else {
1391 assert(isa<NoExpansion>(Exp.get()));
1392 *TI++ = ConvertType(T: Ty);
1393 }
1394}
1395
1396static void forConstantArrayExpansion(CodeGenFunction &CGF,
1397 ConstantArrayExpansion *CAE,
1398 Address BaseAddr,
1399 llvm::function_ref<void(Address)> Fn) {
1400 for (int i = 0, n = CAE->NumElts; i < n; i++) {
1401 Address EltAddr = CGF.Builder.CreateConstGEP2_32(Addr: BaseAddr, Idx0: 0, Idx1: i);
1402 Fn(EltAddr);
1403 }
1404}
1405
1406void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1407 llvm::Function::arg_iterator &AI) {
1408 assert(LV.isSimple() &&
1409 "Unexpected non-simple lvalue during struct expansion.");
1410
1411 auto Exp = getTypeExpansion(Ty, Context: getContext());
1412 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1413 forConstantArrayExpansion(
1414 CGF&: *this, CAE: CAExp, BaseAddr: LV.getAddress(), Fn: [&](Address EltAddr) {
1415 LValue LV = MakeAddrLValue(Addr: EltAddr, T: CAExp->EltTy);
1416 ExpandTypeFromArgs(Ty: CAExp->EltTy, LV, AI);
1417 });
1418 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1419 Address This = LV.getAddress();
1420 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1421 // Perform a single step derived-to-base conversion.
1422 Address Base =
1423 GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1424 /*NullCheckValue=*/false, Loc: SourceLocation());
1425 LValue SubLV = MakeAddrLValue(Addr: Base, T: BS->getType());
1426
1427 // Recurse onto bases.
1428 ExpandTypeFromArgs(Ty: BS->getType(), LV: SubLV, AI);
1429 }
1430 for (auto FD : RExp->Fields) {
1431 // FIXME: What are the right qualifiers here?
1432 LValue SubLV = EmitLValueForFieldInitialization(Base: LV, Field: FD);
1433 ExpandTypeFromArgs(Ty: FD->getType(), LV: SubLV, AI);
1434 }
1435 } else if (isa<ComplexExpansion>(Val: Exp.get())) {
1436 auto realValue = &*AI++;
1437 auto imagValue = &*AI++;
1438 EmitStoreOfComplex(V: ComplexPairTy(realValue, imagValue), dest: LV, /*init*/ isInit: true);
1439 } else {
1440 // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1441 // primitive store.
1442 assert(isa<NoExpansion>(Exp.get()));
1443 llvm::Value *Arg = &*AI++;
1444 if (LV.isBitField()) {
1445 EmitStoreThroughLValue(Src: RValue::get(V: Arg), Dst: LV);
1446 } else {
1447 // TODO: currently there are some places are inconsistent in what LLVM
1448 // pointer type they use (see D118744). Once clang uses opaque pointers
1449 // all LLVM pointer types will be the same and we can remove this check.
1450 if (Arg->getType()->isPointerTy()) {
1451 Address Addr = LV.getAddress();
1452 Arg = Builder.CreateBitCast(V: Arg, DestTy: Addr.getElementType());
1453 }
1454 EmitStoreOfScalar(value: Arg, lvalue: LV);
1455 }
1456 }
1457}
1458
1459void CodeGenFunction::ExpandTypeToArgs(
1460 QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1461 SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1462 auto Exp = getTypeExpansion(Ty, Context: getContext());
1463 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Val: Exp.get())) {
1464 Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1465 : Arg.getKnownRValue().getAggregateAddress();
1466 forConstantArrayExpansion(CGF&: *this, CAE: CAExp, BaseAddr: Addr, Fn: [&](Address EltAddr) {
1467 CallArg EltArg =
1468 CallArg(convertTempToRValue(addr: EltAddr, type: CAExp->EltTy, Loc: SourceLocation()),
1469 CAExp->EltTy);
1470 ExpandTypeToArgs(Ty: CAExp->EltTy, Arg: EltArg, IRFuncTy, IRCallArgs,
1471 IRCallArgPos);
1472 });
1473 } else if (auto RExp = dyn_cast<RecordExpansion>(Val: Exp.get())) {
1474 Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1475 : Arg.getKnownRValue().getAggregateAddress();
1476 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1477 // Perform a single step derived-to-base conversion.
1478 Address Base =
1479 GetAddressOfBaseClass(Value: This, Derived: Ty->getAsCXXRecordDecl(), PathBegin: &BS, PathEnd: &BS + 1,
1480 /*NullCheckValue=*/false, Loc: SourceLocation());
1481 CallArg BaseArg = CallArg(RValue::getAggregate(addr: Base), BS->getType());
1482
1483 // Recurse onto bases.
1484 ExpandTypeToArgs(Ty: BS->getType(), Arg: BaseArg, IRFuncTy, IRCallArgs,
1485 IRCallArgPos);
1486 }
1487
1488 LValue LV = MakeAddrLValue(Addr: This, T: Ty);
1489 for (auto FD : RExp->Fields) {
1490 CallArg FldArg =
1491 CallArg(EmitRValueForField(LV, FD, Loc: SourceLocation()), FD->getType());
1492 ExpandTypeToArgs(Ty: FD->getType(), Arg: FldArg, IRFuncTy, IRCallArgs,
1493 IRCallArgPos);
1494 }
1495 } else if (isa<ComplexExpansion>(Val: Exp.get())) {
1496 ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1497 IRCallArgs[IRCallArgPos++] = CV.first;
1498 IRCallArgs[IRCallArgPos++] = CV.second;
1499 } else {
1500 assert(isa<NoExpansion>(Exp.get()));
1501 auto RV = Arg.getKnownRValue();
1502 assert(RV.isScalar() &&
1503 "Unexpected non-scalar rvalue during struct expansion.");
1504
1505 // Insert a bitcast as needed.
1506 llvm::Value *V = RV.getScalarVal();
1507 if (IRCallArgPos < IRFuncTy->getNumParams() &&
1508 V->getType() != IRFuncTy->getParamType(i: IRCallArgPos))
1509 V = Builder.CreateBitCast(V, DestTy: IRFuncTy->getParamType(i: IRCallArgPos));
1510
1511 IRCallArgs[IRCallArgPos++] = V;
1512 }
1513}
1514
1515/// Create a temporary allocation for the purposes of coercion.
1516static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF,
1517 llvm::Type *Ty,
1518 CharUnits MinAlign,
1519 const Twine &Name = "tmp") {
1520 // Don't use an alignment that's worse than what LLVM would prefer.
1521 auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty);
1522 CharUnits Align = std::max(a: MinAlign, b: CharUnits::fromQuantity(Quantity: PrefAlign));
1523
1524 return CGF.CreateTempAlloca(Ty, align: Align, Name: Name + ".coerce");
1525}
1526
1527/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1528/// accessing some number of bytes out of it, try to gep into the struct to get
1529/// at its inner goodness. Dive as deep as possible without entering an element
1530/// with an in-memory size smaller than DstSize.
1531static Address EnterStructPointerForCoercedAccess(Address SrcPtr,
1532 llvm::StructType *SrcSTy,
1533 uint64_t DstSize,
1534 CodeGenFunction &CGF) {
1535 // We can't dive into a zero-element struct.
1536 if (SrcSTy->getNumElements() == 0)
1537 return SrcPtr;
1538
1539 llvm::Type *FirstElt = SrcSTy->getElementType(N: 0);
1540
1541 // If the first elt is at least as large as what we're looking for, or if the
1542 // first element is the same size as the whole struct, we can enter it. The
1543 // comparison must be made on the store size and not the alloca size. Using
1544 // the alloca size may overstate the size of the load.
1545 uint64_t FirstEltSize = CGF.CGM.getDataLayout().getTypeStoreSize(Ty: FirstElt);
1546 if (FirstEltSize < DstSize &&
1547 FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(Ty: SrcSTy))
1548 return SrcPtr;
1549
1550 // GEP into the first element.
1551 SrcPtr = CGF.Builder.CreateStructGEP(Addr: SrcPtr, Index: 0, Name: "coerce.dive");
1552
1553 // If the first element is a struct, recurse.
1554 llvm::Type *SrcTy = SrcPtr.getElementType();
1555 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy))
1556 return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1557
1558 return SrcPtr;
1559}
1560
1561/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1562/// are either integers or pointers. This does a truncation of the value if it
1563/// is too large or a zero extension if it is too small.
1564///
1565/// This behaves as if the value were coerced through memory, so on big-endian
1566/// targets the high bits are preserved in a truncation, while little-endian
1567/// targets preserve the low bits.
1568static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, llvm::Type *Ty,
1569 CodeGenFunction &CGF) {
1570 if (Val->getType() == Ty)
1571 return Val;
1572
1573 if (isa<llvm::PointerType>(Val: Val->getType())) {
1574 // If this is Pointer->Pointer avoid conversion to and from int.
1575 if (isa<llvm::PointerType>(Val: Ty))
1576 return CGF.Builder.CreateBitCast(V: Val, DestTy: Ty, Name: "coerce.val");
1577
1578 // Convert the pointer to an integer so we can play with its width.
1579 Val = CGF.Builder.CreatePtrToInt(V: Val, DestTy: CGF.IntPtrTy, Name: "coerce.val.pi");
1580 }
1581
1582 llvm::Type *DestIntTy = Ty;
1583 if (isa<llvm::PointerType>(Val: DestIntTy))
1584 DestIntTy = CGF.IntPtrTy;
1585
1586 if (Val->getType() != DestIntTy) {
1587 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1588 if (DL.isBigEndian()) {
1589 // Preserve the high bits on big-endian targets.
1590 // That is what memory coercion does.
1591 uint64_t SrcSize = DL.getTypeSizeInBits(Ty: Val->getType());
1592 uint64_t DstSize = DL.getTypeSizeInBits(Ty: DestIntTy);
1593
1594 if (SrcSize > DstSize) {
1595 Val = CGF.Builder.CreateLShr(LHS: Val, RHS: SrcSize - DstSize, Name: "coerce.highbits");
1596 Val = CGF.Builder.CreateTrunc(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1597 } else {
1598 Val = CGF.Builder.CreateZExt(V: Val, DestTy: DestIntTy, Name: "coerce.val.ii");
1599 Val = CGF.Builder.CreateShl(LHS: Val, RHS: DstSize - SrcSize, Name: "coerce.highbits");
1600 }
1601 } else {
1602 // Little-endian targets preserve the low bits. No shifts required.
1603 Val = CGF.Builder.CreateIntCast(V: Val, DestTy: DestIntTy, isSigned: false, Name: "coerce.val.ii");
1604 }
1605 }
1606
1607 if (isa<llvm::PointerType>(Val: Ty))
1608 Val = CGF.Builder.CreateIntToPtr(V: Val, DestTy: Ty, Name: "coerce.val.ip");
1609 return Val;
1610}
1611
1612static llvm::Value *CreatePFPCoercedLoad(Address Src, QualType SrcFETy,
1613 llvm::Type *Ty, CodeGenFunction &CGF) {
1614 std::vector<PFPField> PFPFields = CGF.getContext().findPFPFields(Ty: SrcFETy);
1615 if (PFPFields.empty())
1616 return nullptr;
1617
1618 auto LoadCoercedField = [&](CharUnits Offset,
1619 llvm::Type *FieldType) -> llvm::Value * {
1620 // Check whether the field at Offset is a PFP field. This function is called
1621 // in ascending order of offset, and PFPFields is sorted by offset. This
1622 // means that we only need to check the first element (and remove it from
1623 // PFPFields if matching).
1624 if (!PFPFields.empty() && PFPFields[0].Offset == Offset) {
1625 auto FieldAddr = CGF.EmitAddressOfPFPField(RecordPtr: Src, Field: PFPFields[0]);
1626 llvm::Value *FieldVal = CGF.Builder.CreateLoad(Addr: FieldAddr);
1627 if (isa<llvm::IntegerType>(Val: FieldType))
1628 FieldVal = CGF.Builder.CreatePtrToInt(V: FieldVal, DestTy: FieldType);
1629 PFPFields.erase(position: PFPFields.begin());
1630 return FieldVal;
1631 }
1632 auto FieldAddr =
1633 CGF.Builder
1634 .CreateConstInBoundsByteGEP(Addr: Src.withElementType(ElemTy: CGF.Int8Ty), Offset)
1635 .withElementType(ElemTy: FieldType);
1636 return CGF.Builder.CreateLoad(Addr: FieldAddr);
1637 };
1638
1639 // The types handled by this function are the only ones that may be generated
1640 // by AArch64ABIInfo::classify{Argument,Return}Type for struct types with
1641 // pointers. PFP is only supported on AArch64.
1642 if (isa<llvm::IntegerType>(Val: Ty) || isa<llvm::PointerType>(Val: Ty)) {
1643 auto Addr = CGF.EmitAddressOfPFPField(RecordPtr: Src, Field: PFPFields[0]);
1644 llvm::Value *Val = CGF.Builder.CreateLoad(Addr);
1645 if (isa<llvm::IntegerType>(Val: Ty))
1646 Val = CGF.Builder.CreatePtrToInt(V: Val, DestTy: Ty);
1647 return Val;
1648 }
1649 auto *AT = cast<llvm::ArrayType>(Val: Ty);
1650 auto *ET = AT->getElementType();
1651 CharUnits WordSize = CGF.getContext().toCharUnitsFromBits(
1652 BitSize: CGF.CGM.getDataLayout().getTypeSizeInBits(Ty: ET));
1653 CharUnits Offset = CharUnits::Zero();
1654 llvm::Value *Val = llvm::PoisonValue::get(T: AT);
1655 for (unsigned Idx = 0; Idx != AT->getNumElements(); ++Idx, Offset += WordSize)
1656 Val = CGF.Builder.CreateInsertValue(Agg: Val, Val: LoadCoercedField(Offset, ET), Idxs: Idx);
1657 return Val;
1658}
1659
1660/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1661/// a pointer to an object of type \arg Ty, known to be aligned to
1662/// \arg SrcAlign bytes.
1663///
1664/// This safely handles the case when the src type is smaller than the
1665/// destination type; in this situation the values of bits which not
1666/// present in the src are undefined.
1667static llvm::Value *CreateCoercedLoad(Address Src, QualType SrcFETy,
1668 llvm::Type *Ty, CodeGenFunction &CGF) {
1669 llvm::Type *SrcTy = Src.getElementType();
1670
1671 // If SrcTy and Ty are the same, just do a load.
1672 if (SrcTy == Ty)
1673 return CGF.Builder.CreateLoad(Addr: Src);
1674
1675 if (llvm::Value *V = CreatePFPCoercedLoad(Src, SrcFETy, Ty, CGF))
1676 return V;
1677
1678 llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1679
1680 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(Val: SrcTy)) {
1681 Src = EnterStructPointerForCoercedAccess(SrcPtr: Src, SrcSTy,
1682 DstSize: DstSize.getFixedValue(), CGF);
1683 SrcTy = Src.getElementType();
1684 }
1685
1686 llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1687
1688 // If the source and destination are integer or pointer types, just do an
1689 // extension or truncation to the desired type.
1690 if ((isa<llvm::IntegerType>(Val: Ty) || isa<llvm::PointerType>(Val: Ty)) &&
1691 (isa<llvm::IntegerType>(Val: SrcTy) || isa<llvm::PointerType>(Val: SrcTy))) {
1692 llvm::Value *Load = CGF.Builder.CreateLoad(Addr: Src);
1693 return CoerceIntOrPtrToIntOrPtr(Val: Load, Ty, CGF);
1694 }
1695
1696 // If load is legal, just bitcast the src pointer.
1697 if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1698 SrcSize.getFixedValue() >= DstSize.getFixedValue()) {
1699 // Generally SrcSize is never greater than DstSize, since this means we are
1700 // losing bits. However, this can happen in cases where the structure has
1701 // additional padding, for example due to a user specified alignment.
1702 //
1703 // FIXME: Assert that we aren't truncating non-padding bits when have access
1704 // to that information.
1705 Src = Src.withElementType(ElemTy: Ty);
1706 return CGF.Builder.CreateLoad(Addr: Src);
1707 }
1708
1709 // If coercing a fixed vector to a scalable vector for ABI compatibility, and
1710 // the types match, use the llvm.vector.insert intrinsic to perform the
1711 // conversion.
1712 if (auto *ScalableDstTy = dyn_cast<llvm::ScalableVectorType>(Val: Ty)) {
1713 if (auto *FixedSrcTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy)) {
1714 // If we are casting a fixed i8 vector to a scalable i1 predicate
1715 // vector, use a vector insert and bitcast the result.
1716 if (ScalableDstTy->getElementType()->isIntegerTy(BitWidth: 1) &&
1717 FixedSrcTy->getElementType()->isIntegerTy(BitWidth: 8)) {
1718 ScalableDstTy = llvm::ScalableVectorType::get(
1719 ElementType: FixedSrcTy->getElementType(),
1720 MinNumElts: llvm::divideCeil(
1721 Numerator: ScalableDstTy->getElementCount().getKnownMinValue(), Denominator: 8));
1722 }
1723 if (ScalableDstTy->getElementType() == FixedSrcTy->getElementType()) {
1724 auto *Load = CGF.Builder.CreateLoad(Addr: Src);
1725 auto *PoisonVec = llvm::PoisonValue::get(T: ScalableDstTy);
1726 llvm::Value *Result = CGF.Builder.CreateInsertVector(
1727 DstType: ScalableDstTy, SrcVec: PoisonVec, SubVec: Load, Idx: uint64_t(0), Name: "cast.scalable");
1728 ScalableDstTy = cast<llvm::ScalableVectorType>(
1729 Val: llvm::VectorType::getWithSizeAndScalar(SizeTy: ScalableDstTy, EltTy: Ty));
1730 if (Result->getType() != ScalableDstTy)
1731 Result = CGF.Builder.CreateBitCast(V: Result, DestTy: ScalableDstTy);
1732 if (Result->getType() != Ty)
1733 Result = CGF.Builder.CreateExtractVector(DstType: Ty, SrcVec: Result, Idx: uint64_t(0));
1734 return Result;
1735 }
1736 }
1737 }
1738
1739 // Otherwise do coercion through memory. This is stupid, but simple.
1740 RawAddress Tmp =
1741 CreateTempAllocaForCoercion(CGF, Ty, MinAlign: Src.getAlignment(), Name: Src.getName());
1742 CGF.Builder.CreateMemCpy(
1743 Dst: Tmp.getPointer(), DstAlign: Tmp.getAlignment().getAsAlign(),
1744 Src: Src.emitRawPointer(CGF), SrcAlign: Src.getAlignment().getAsAlign(),
1745 Size: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: SrcSize.getKnownMinValue()));
1746 return CGF.Builder.CreateLoad(Addr: Tmp);
1747}
1748
1749static bool CreatePFPCoercedStore(llvm::Value *Src, QualType SrcFETy,
1750 Address Dst, CodeGenFunction &CGF) {
1751 std::vector<PFPField> PFPFields = CGF.getContext().findPFPFields(Ty: SrcFETy);
1752 if (PFPFields.empty())
1753 return false;
1754
1755 llvm::Type *SrcTy = Src->getType();
1756 auto StoreCoercedField = [&](CharUnits Offset, llvm::Value *FieldVal) {
1757 if (!PFPFields.empty() && PFPFields[0].Offset == Offset) {
1758 auto FieldAddr = CGF.EmitAddressOfPFPField(RecordPtr: Dst, Field: PFPFields[0]);
1759 if (isa<llvm::IntegerType>(Val: FieldVal->getType()))
1760 FieldVal = CGF.Builder.CreateIntToPtr(V: FieldVal, DestTy: CGF.VoidPtrTy);
1761 CGF.Builder.CreateStore(Val: FieldVal, Addr: FieldAddr);
1762 PFPFields.erase(position: PFPFields.begin());
1763 } else {
1764 auto FieldAddr = CGF.Builder
1765 .CreateConstInBoundsByteGEP(
1766 Addr: Dst.withElementType(ElemTy: CGF.Int8Ty), Offset)
1767 .withElementType(ElemTy: FieldVal->getType());
1768 CGF.Builder.CreateStore(Val: FieldVal, Addr: FieldAddr);
1769 }
1770 };
1771
1772 // The types handled by this function are the only ones that may be generated
1773 // by AArch64ABIInfo::classify{Argument,Return}Type for struct types with
1774 // pointers. PFP is only supported on AArch64.
1775 if (isa<llvm::IntegerType>(Val: SrcTy) || isa<llvm::PointerType>(Val: SrcTy)) {
1776 if (isa<llvm::IntegerType>(Val: SrcTy))
1777 Src = CGF.Builder.CreateIntToPtr(V: Src, DestTy: CGF.VoidPtrTy);
1778 auto Addr = CGF.EmitAddressOfPFPField(RecordPtr: Dst, Field: PFPFields[0]);
1779 CGF.Builder.CreateStore(Val: Src, Addr);
1780 } else {
1781 auto *AT = cast<llvm::ArrayType>(Val: SrcTy);
1782 auto *ET = AT->getElementType();
1783 CharUnits WordSize = CGF.getContext().toCharUnitsFromBits(
1784 BitSize: CGF.CGM.getDataLayout().getTypeSizeInBits(Ty: ET));
1785 CharUnits Offset = CharUnits::Zero();
1786 for (unsigned i = 0; i != AT->getNumElements(); ++i, Offset += WordSize)
1787 StoreCoercedField(Offset, CGF.Builder.CreateExtractValue(Agg: Src, Idxs: i));
1788 }
1789 return true;
1790}
1791
1792void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, QualType SrcFETy,
1793 Address Dst, llvm::TypeSize DstSize,
1794 bool DstIsVolatile) {
1795 if (!DstSize)
1796 return;
1797
1798 llvm::Type *SrcTy = Src->getType();
1799 llvm::TypeSize SrcSize = CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
1800
1801 // GEP into structs to try to make types match.
1802 // FIXME: This isn't really that useful with opaque types, but it impacts a
1803 // lot of regression tests.
1804 if (SrcTy != Dst.getElementType()) {
1805 if (llvm::StructType *DstSTy =
1806 dyn_cast<llvm::StructType>(Val: Dst.getElementType())) {
1807 assert(!SrcSize.isScalable());
1808 Dst = EnterStructPointerForCoercedAccess(SrcPtr: Dst, SrcSTy: DstSTy,
1809 DstSize: SrcSize.getFixedValue(), CGF&: *this);
1810 }
1811 }
1812
1813 if (CreatePFPCoercedStore(Src, SrcFETy, Dst, CGF&: *this))
1814 return;
1815
1816 if (SrcSize.isScalable() || SrcSize <= DstSize) {
1817 if (SrcTy->isIntegerTy() && Dst.getElementType()->isPointerTy() &&
1818 SrcSize == CGM.getDataLayout().getTypeAllocSize(Ty: Dst.getElementType())) {
1819 // If the value is supposed to be a pointer, convert it before storing it.
1820 Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: Dst.getElementType(), CGF&: *this);
1821 auto *I = Builder.CreateStore(Val: Src, Addr: Dst, IsVolatile: DstIsVolatile);
1822 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1823 } else if (llvm::StructType *STy =
1824 dyn_cast<llvm::StructType>(Val: Src->getType())) {
1825 // Prefer scalar stores to first-class aggregate stores.
1826 Dst = Dst.withElementType(ElemTy: SrcTy);
1827 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1828 Address EltPtr = Builder.CreateStructGEP(Addr: Dst, Index: i);
1829 llvm::Value *Elt = Builder.CreateExtractValue(Agg: Src, Idxs: i);
1830 auto *I = Builder.CreateStore(Val: Elt, Addr: EltPtr, IsVolatile: DstIsVolatile);
1831 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Elt);
1832 }
1833 } else {
1834 auto *I =
1835 Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: SrcTy), IsVolatile: DstIsVolatile);
1836 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1837 }
1838 } else if (SrcTy->isIntegerTy()) {
1839 // If the source is a simple integer, coerce it directly.
1840 llvm::Type *DstIntTy = Builder.getIntNTy(N: DstSize.getFixedValue() * 8);
1841 Src = CoerceIntOrPtrToIntOrPtr(Val: Src, Ty: DstIntTy, CGF&: *this);
1842 auto *I =
1843 Builder.CreateStore(Val: Src, Addr: Dst.withElementType(ElemTy: DstIntTy), IsVolatile: DstIsVolatile);
1844 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1845 } else {
1846 // Otherwise do coercion through memory. This is stupid, but
1847 // simple.
1848
1849 // Generally SrcSize is never greater than DstSize, since this means we are
1850 // losing bits. However, this can happen in cases where the structure has
1851 // additional padding, for example due to a user specified alignment.
1852 //
1853 // FIXME: Assert that we aren't truncating non-padding bits when have access
1854 // to that information.
1855 RawAddress Tmp =
1856 CreateTempAllocaForCoercion(CGF&: *this, Ty: SrcTy, MinAlign: Dst.getAlignment());
1857 Builder.CreateStore(Val: Src, Addr: Tmp);
1858 auto *I = Builder.CreateMemCpy(
1859 Dst: Dst.emitRawPointer(CGF&: *this), DstAlign: Dst.getAlignment().getAsAlign(),
1860 Src: Tmp.getPointer(), SrcAlign: Tmp.getAlignment().getAsAlign(),
1861 Size: Builder.CreateTypeSize(Ty: IntPtrTy, Size: DstSize));
1862 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: Src);
1863 }
1864}
1865
1866static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1867 const ABIArgInfo &info) {
1868 if (unsigned offset = info.getDirectOffset()) {
1869 addr = addr.withElementType(ElemTy: CGF.Int8Ty);
1870 addr = CGF.Builder.CreateConstInBoundsByteGEP(
1871 Addr: addr, Offset: CharUnits::fromQuantity(Quantity: offset));
1872 addr = addr.withElementType(ElemTy: info.getCoerceToType());
1873 }
1874 return addr;
1875}
1876
1877static std::pair<llvm::Value *, bool>
1878CoerceScalableToFixed(CodeGenFunction &CGF, llvm::FixedVectorType *ToTy,
1879 llvm::ScalableVectorType *FromTy, llvm::Value *V,
1880 StringRef Name = "") {
1881 // If we are casting a scalable i1 predicate vector to a fixed i8
1882 // vector, first bitcast the source.
1883 if (FromTy->getElementType()->isIntegerTy(BitWidth: 1) &&
1884 ToTy->getElementType() == CGF.Builder.getInt8Ty()) {
1885 if (!FromTy->getElementCount().isKnownMultipleOf(RHS: 8)) {
1886 FromTy = llvm::ScalableVectorType::get(
1887 ElementType: FromTy->getElementType(),
1888 MinNumElts: llvm::alignTo<8>(Value: FromTy->getElementCount().getKnownMinValue()));
1889 llvm::Value *ZeroVec = llvm::Constant::getNullValue(Ty: FromTy);
1890 V = CGF.Builder.CreateInsertVector(DstType: FromTy, SrcVec: ZeroVec, SubVec: V, Idx: uint64_t(0));
1891 }
1892 FromTy = llvm::ScalableVectorType::get(
1893 ElementType: ToTy->getElementType(),
1894 MinNumElts: FromTy->getElementCount().getKnownMinValue() / 8);
1895 V = CGF.Builder.CreateBitCast(V, DestTy: FromTy);
1896 }
1897 if (FromTy->getElementType() == ToTy->getElementType()) {
1898 V->setName(Name + ".coerce");
1899 V = CGF.Builder.CreateExtractVector(DstType: ToTy, SrcVec: V, Idx: uint64_t(0), Name: "cast.fixed");
1900 return {V, true};
1901 }
1902 return {V, false};
1903}
1904
1905namespace {
1906
1907/// Encapsulates information about the way function arguments from
1908/// CGFunctionInfo should be passed to actual LLVM IR function.
1909class ClangToLLVMArgMapping {
1910 static const unsigned InvalidIndex = ~0U;
1911 unsigned InallocaArgNo;
1912 unsigned SRetArgNo;
1913 unsigned TotalIRArgs;
1914
1915 /// Arguments of LLVM IR function corresponding to single Clang argument.
1916 struct IRArgs {
1917 unsigned PaddingArgIndex;
1918 // Argument is expanded to IR arguments at positions
1919 // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1920 unsigned FirstArgIndex;
1921 unsigned NumberOfArgs;
1922
1923 IRArgs()
1924 : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1925 NumberOfArgs(0) {}
1926 };
1927
1928 SmallVector<IRArgs, 8> ArgInfo;
1929
1930public:
1931 ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1932 bool OnlyRequiredArgs = false)
1933 : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1934 ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1935 construct(Context, FI, OnlyRequiredArgs);
1936 }
1937
1938 bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1939 unsigned getInallocaArgNo() const {
1940 assert(hasInallocaArg());
1941 return InallocaArgNo;
1942 }
1943
1944 bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1945 unsigned getSRetArgNo() const {
1946 assert(hasSRetArg());
1947 return SRetArgNo;
1948 }
1949
1950 unsigned totalIRArgs() const { return TotalIRArgs; }
1951
1952 bool hasPaddingArg(unsigned ArgNo) const {
1953 assert(ArgNo < ArgInfo.size());
1954 return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1955 }
1956 unsigned getPaddingArgNo(unsigned ArgNo) const {
1957 assert(hasPaddingArg(ArgNo));
1958 return ArgInfo[ArgNo].PaddingArgIndex;
1959 }
1960
1961 /// Returns index of first IR argument corresponding to ArgNo, and their
1962 /// quantity.
1963 std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1964 assert(ArgNo < ArgInfo.size());
1965 return std::make_pair(x: ArgInfo[ArgNo].FirstArgIndex,
1966 y: ArgInfo[ArgNo].NumberOfArgs);
1967 }
1968
1969private:
1970 void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1971 bool OnlyRequiredArgs);
1972};
1973
1974void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1975 const CGFunctionInfo &FI,
1976 bool OnlyRequiredArgs) {
1977 unsigned IRArgNo = 0;
1978 bool SwapThisWithSRet = false;
1979 const ABIArgInfo &RetAI = FI.getReturnInfo();
1980
1981 if (RetAI.getKind() == ABIArgInfo::Indirect) {
1982 SwapThisWithSRet = RetAI.isSRetAfterThis();
1983 SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1984 }
1985
1986 unsigned ArgNo = 0;
1987 unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();
1988 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1989 ++I, ++ArgNo) {
1990 assert(I != FI.arg_end());
1991 QualType ArgType = I->type;
1992 const ABIArgInfo &AI = I->info;
1993 // Collect data about IR arguments corresponding to Clang argument ArgNo.
1994 auto &IRArgs = ArgInfo[ArgNo];
1995
1996 if (AI.getPaddingType())
1997 IRArgs.PaddingArgIndex = IRArgNo++;
1998
1999 switch (AI.getKind()) {
2000 case ABIArgInfo::TargetSpecific:
2001 case ABIArgInfo::Extend:
2002 case ABIArgInfo::Direct: {
2003 // FIXME: handle sseregparm someday...
2004 llvm::StructType *STy = dyn_cast<llvm::StructType>(Val: AI.getCoerceToType());
2005 if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
2006 IRArgs.NumberOfArgs = STy->getNumElements();
2007 } else {
2008 IRArgs.NumberOfArgs = 1;
2009 }
2010 break;
2011 }
2012 case ABIArgInfo::Indirect:
2013 case ABIArgInfo::IndirectAliased:
2014 IRArgs.NumberOfArgs = 1;
2015 break;
2016 case ABIArgInfo::Ignore:
2017 case ABIArgInfo::InAlloca:
2018 // ignore and inalloca doesn't have matching LLVM parameters.
2019 IRArgs.NumberOfArgs = 0;
2020 break;
2021 case ABIArgInfo::CoerceAndExpand:
2022 IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
2023 break;
2024 case ABIArgInfo::Expand:
2025 IRArgs.NumberOfArgs = getExpansionSize(Ty: ArgType, Context);
2026 break;
2027 }
2028
2029 if (IRArgs.NumberOfArgs > 0) {
2030 IRArgs.FirstArgIndex = IRArgNo;
2031 IRArgNo += IRArgs.NumberOfArgs;
2032 }
2033
2034 // Skip over the sret parameter when it comes second. We already handled it
2035 // above.
2036 if (IRArgNo == 1 && SwapThisWithSRet)
2037 IRArgNo++;
2038 }
2039 assert(ArgNo == ArgInfo.size());
2040
2041 if (FI.usesInAlloca())
2042 InallocaArgNo = IRArgNo++;
2043
2044 TotalIRArgs = IRArgNo;
2045}
2046} // namespace
2047
2048/***/
2049
2050bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
2051 const auto &RI = FI.getReturnInfo();
2052 return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());
2053}
2054
2055bool CodeGenModule::ReturnTypeHasInReg(const CGFunctionInfo &FI) {
2056 const auto &RI = FI.getReturnInfo();
2057 return RI.getInReg();
2058}
2059
2060bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
2061 return ReturnTypeUsesSRet(FI) &&
2062 getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
2063}
2064
2065bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
2066 if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
2067 switch (BT->getKind()) {
2068 default:
2069 return false;
2070 case BuiltinType::Float:
2071 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Float);
2072 case BuiltinType::Double:
2073 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::Double);
2074 case BuiltinType::LongDouble:
2075 return getTarget().useObjCFPRetForRealType(T: FloatModeKind::LongDouble);
2076 }
2077 }
2078
2079 return false;
2080}
2081
2082bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
2083 if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
2084 if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
2085 if (BT->getKind() == BuiltinType::LongDouble)
2086 return getTarget().useObjCFP2RetForComplexLongDouble();
2087 }
2088 }
2089
2090 return false;
2091}
2092
2093llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
2094 const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
2095 return GetFunctionType(Info: FI);
2096}
2097
2098llvm::FunctionType *CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
2099
2100 bool Inserted = FunctionsBeingProcessed.insert(Ptr: &FI).second;
2101 (void)Inserted;
2102 assert(Inserted && "Recursively being processed?");
2103
2104 llvm::Type *resultType = nullptr;
2105 const ABIArgInfo &retAI = FI.getReturnInfo();
2106 switch (retAI.getKind()) {
2107 case ABIArgInfo::Expand:
2108 case ABIArgInfo::IndirectAliased:
2109 llvm_unreachable("Invalid ABI kind for return argument");
2110
2111 case ABIArgInfo::TargetSpecific:
2112 case ABIArgInfo::Extend:
2113 case ABIArgInfo::Direct:
2114 resultType = retAI.getCoerceToType();
2115 break;
2116
2117 case ABIArgInfo::InAlloca:
2118 if (retAI.getInAllocaSRet()) {
2119 // sret things on win32 aren't void, they return the sret pointer.
2120 QualType ret = FI.getReturnType();
2121 unsigned addressSpace = CGM.getTypes().getTargetAddressSpace(T: ret);
2122 resultType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: addressSpace);
2123 } else {
2124 resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
2125 }
2126 break;
2127
2128 case ABIArgInfo::Indirect:
2129 case ABIArgInfo::Ignore:
2130 resultType = llvm::Type::getVoidTy(C&: getLLVMContext());
2131 break;
2132
2133 case ABIArgInfo::CoerceAndExpand:
2134 resultType = retAI.getUnpaddedCoerceAndExpandType();
2135 break;
2136 }
2137
2138 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
2139 SmallVector<llvm::Type *, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
2140
2141 // Add type for sret argument.
2142 if (IRFunctionArgs.hasSRetArg()) {
2143 ArgTypes[IRFunctionArgs.getSRetArgNo()] = llvm::PointerType::get(
2144 C&: getLLVMContext(), AddressSpace: FI.getReturnInfo().getIndirectAddrSpace());
2145 }
2146
2147 // Add type for inalloca argument.
2148 if (IRFunctionArgs.hasInallocaArg())
2149 ArgTypes[IRFunctionArgs.getInallocaArgNo()] =
2150 llvm::PointerType::getUnqual(C&: getLLVMContext());
2151
2152 // Add in all of the required arguments.
2153 unsigned ArgNo = 0;
2154 CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
2155 ie = it + FI.getNumRequiredArgs();
2156 for (; it != ie; ++it, ++ArgNo) {
2157 const ABIArgInfo &ArgInfo = it->info;
2158
2159 // Insert a padding type to ensure proper alignment.
2160 if (IRFunctionArgs.hasPaddingArg(ArgNo))
2161 ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2162 ArgInfo.getPaddingType();
2163
2164 unsigned FirstIRArg, NumIRArgs;
2165 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2166
2167 switch (ArgInfo.getKind()) {
2168 case ABIArgInfo::Ignore:
2169 case ABIArgInfo::InAlloca:
2170 assert(NumIRArgs == 0);
2171 break;
2172
2173 case ABIArgInfo::Indirect:
2174 assert(NumIRArgs == 1);
2175 // indirect arguments are always on the stack, which is alloca addr space.
2176 ArgTypes[FirstIRArg] = llvm::PointerType::get(
2177 C&: getLLVMContext(), AddressSpace: CGM.getDataLayout().getAllocaAddrSpace());
2178 break;
2179 case ABIArgInfo::IndirectAliased:
2180 assert(NumIRArgs == 1);
2181 ArgTypes[FirstIRArg] = llvm::PointerType::get(
2182 C&: getLLVMContext(), AddressSpace: ArgInfo.getIndirectAddrSpace());
2183 break;
2184 case ABIArgInfo::TargetSpecific:
2185 case ABIArgInfo::Extend:
2186 case ABIArgInfo::Direct: {
2187 // Fast-isel and the optimizer generally like scalar values better than
2188 // FCAs, so we flatten them if this is safe to do for this argument.
2189 llvm::Type *argType = ArgInfo.getCoerceToType();
2190 llvm::StructType *st = dyn_cast<llvm::StructType>(Val: argType);
2191 if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
2192 assert(NumIRArgs == st->getNumElements());
2193 for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
2194 ArgTypes[FirstIRArg + i] = st->getElementType(N: i);
2195 } else {
2196 assert(NumIRArgs == 1);
2197 ArgTypes[FirstIRArg] = argType;
2198 }
2199 break;
2200 }
2201
2202 case ABIArgInfo::CoerceAndExpand: {
2203 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
2204 for (auto *EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
2205 *ArgTypesIter++ = EltTy;
2206 }
2207 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
2208 break;
2209 }
2210
2211 case ABIArgInfo::Expand:
2212 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
2213 getExpandedTypes(Ty: it->type, TI&: ArgTypesIter);
2214 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
2215 break;
2216 }
2217 }
2218
2219 bool Erased = FunctionsBeingProcessed.erase(Ptr: &FI);
2220 (void)Erased;
2221 assert(Erased && "Not in set?");
2222
2223 return llvm::FunctionType::get(Result: resultType, Params: ArgTypes, isVarArg: FI.isVariadic());
2224}
2225
2226llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
2227 const CXXMethodDecl *MD = cast<CXXMethodDecl>(Val: GD.getDecl());
2228 const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
2229
2230 if (!isFuncTypeConvertible(FT: FPT))
2231 return llvm::StructType::get(Context&: getLLVMContext());
2232
2233 return GetFunctionType(GD);
2234}
2235
2236static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
2237 llvm::AttrBuilder &FuncAttrs,
2238 const FunctionProtoType *FPT) {
2239 if (!FPT)
2240 return;
2241
2242 if (!isUnresolvedExceptionSpec(ESpecType: FPT->getExceptionSpecType()) &&
2243 FPT->isNothrow())
2244 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2245
2246 unsigned SMEBits = FPT->getAArch64SMEAttributes();
2247 if (SMEBits & FunctionType::SME_PStateSMEnabledMask)
2248 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_enabled");
2249 if (SMEBits & FunctionType::SME_PStateSMCompatibleMask)
2250 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_compatible");
2251 if (SMEBits & FunctionType::SME_AgnosticZAStateMask)
2252 FuncAttrs.addAttribute(A: "aarch64_za_state_agnostic");
2253
2254 // ZA
2255 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
2256 FuncAttrs.addAttribute(A: "aarch64_preserves_za");
2257 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_In)
2258 FuncAttrs.addAttribute(A: "aarch64_in_za");
2259 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_Out)
2260 FuncAttrs.addAttribute(A: "aarch64_out_za");
2261 if (FunctionType::getArmZAState(AttrBits: SMEBits) == FunctionType::ARM_InOut)
2262 FuncAttrs.addAttribute(A: "aarch64_inout_za");
2263
2264 // ZT0
2265 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Preserves)
2266 FuncAttrs.addAttribute(A: "aarch64_preserves_zt0");
2267 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_In)
2268 FuncAttrs.addAttribute(A: "aarch64_in_zt0");
2269 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_Out)
2270 FuncAttrs.addAttribute(A: "aarch64_out_zt0");
2271 if (FunctionType::getArmZT0State(AttrBits: SMEBits) == FunctionType::ARM_InOut)
2272 FuncAttrs.addAttribute(A: "aarch64_inout_zt0");
2273}
2274
2275static void AddAttributesFromOMPAssumes(llvm::AttrBuilder &FuncAttrs,
2276 const Decl *Callee) {
2277 if (!Callee)
2278 return;
2279
2280 SmallVector<StringRef, 4> Attrs;
2281
2282 for (const OMPAssumeAttr *AA : Callee->specific_attrs<OMPAssumeAttr>())
2283 AA->getAssumption().split(A&: Attrs, Separator: ",");
2284
2285 if (!Attrs.empty())
2286 FuncAttrs.addAttribute(A: llvm::AssumptionAttrKey,
2287 V: llvm::join(Begin: Attrs.begin(), End: Attrs.end(), Separator: ","));
2288}
2289
2290bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,
2291 QualType ReturnType) const {
2292 // We can't just discard the return value for a record type with a
2293 // complex destructor or a non-trivially copyable type.
2294 if (const RecordType *RT =
2295 ReturnType.getCanonicalType()->getAsCanonical<RecordType>()) {
2296 if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(Val: RT->getDecl()))
2297 return ClassDecl->hasTrivialDestructor();
2298 }
2299 return ReturnType.isTriviallyCopyableType(Context);
2300}
2301
2302static bool HasStrictReturn(const CodeGenModule &Module, QualType RetTy,
2303 const Decl *TargetDecl) {
2304 // As-is msan can not tolerate noundef mismatch between caller and
2305 // implementation. Mismatch is possible for e.g. indirect calls from C-caller
2306 // into C++. Such mismatches lead to confusing false reports. To avoid
2307 // expensive workaround on msan we enforce initialization event in uncommon
2308 // cases where it's allowed.
2309 if (Module.getLangOpts().Sanitize.has(K: SanitizerKind::Memory))
2310 return true;
2311 // C++ explicitly makes returning undefined values UB. C's rule only applies
2312 // to used values, so we never mark them noundef for now.
2313 if (!Module.getLangOpts().CPlusPlus)
2314 return false;
2315 if (TargetDecl) {
2316 if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2317 if (FDecl->isExternC())
2318 return false;
2319 } else if (const VarDecl *VDecl = dyn_cast<VarDecl>(Val: TargetDecl)) {
2320 // Function pointer.
2321 if (VDecl->isExternC())
2322 return false;
2323 }
2324 }
2325
2326 // We don't want to be too aggressive with the return checking, unless
2327 // it's explicit in the code opts or we're using an appropriate sanitizer.
2328 // Try to respect what the programmer intended.
2329 return Module.getCodeGenOpts().StrictReturn ||
2330 !Module.MayDropFunctionReturn(Context: Module.getContext(), ReturnType: RetTy) ||
2331 Module.getLangOpts().Sanitize.has(K: SanitizerKind::Return);
2332}
2333
2334/// Add denormal-fp-math and denormal-fp-math-f32 as appropriate for the
2335/// requested denormal behavior, accounting for the overriding behavior of the
2336/// -f32 case.
2337static void addDenormalModeAttrs(llvm::DenormalMode FPDenormalMode,
2338 llvm::DenormalMode FP32DenormalMode,
2339 llvm::AttrBuilder &FuncAttrs) {
2340 llvm::DenormalFPEnv FPEnv(FPDenormalMode, FP32DenormalMode);
2341 if (FPEnv != llvm::DenormalFPEnv::getDefault())
2342 FuncAttrs.addDenormalFPEnvAttr(Mode: FPEnv);
2343}
2344
2345/// Add default attributes to a function, which have merge semantics under
2346/// -mlink-builtin-bitcode and should not simply overwrite any existing
2347/// attributes in the linked library.
2348static void
2349addMergableDefaultFunctionAttributes(const CodeGenOptions &CodeGenOpts,
2350 llvm::AttrBuilder &FuncAttrs) {
2351 addDenormalModeAttrs(FPDenormalMode: CodeGenOpts.FPDenormalMode, FP32DenormalMode: CodeGenOpts.FP32DenormalMode,
2352 FuncAttrs);
2353}
2354
2355static void getTrivialDefaultFunctionAttributes(
2356 StringRef Name, bool HasOptnone, const CodeGenOptions &CodeGenOpts,
2357 const LangOptions &LangOpts, bool AttrOnCallSite,
2358 llvm::AttrBuilder &FuncAttrs) {
2359 // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
2360 if (!HasOptnone) {
2361 if (CodeGenOpts.OptimizeSize)
2362 FuncAttrs.addAttribute(Val: llvm::Attribute::OptimizeForSize);
2363 if (CodeGenOpts.OptimizeSize == 2)
2364 FuncAttrs.addAttribute(Val: llvm::Attribute::MinSize);
2365 }
2366
2367 if (CodeGenOpts.DisableRedZone)
2368 FuncAttrs.addAttribute(Val: llvm::Attribute::NoRedZone);
2369 if (CodeGenOpts.IndirectTlsSegRefs)
2370 FuncAttrs.addAttribute(A: "indirect-tls-seg-refs");
2371 if (CodeGenOpts.NoImplicitFloat)
2372 FuncAttrs.addAttribute(Val: llvm::Attribute::NoImplicitFloat);
2373
2374 if (AttrOnCallSite) {
2375 // Attributes that should go on the call site only.
2376 // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
2377 // the -fno-builtin-foo list.
2378 if (!CodeGenOpts.SimplifyLibCalls || LangOpts.isNoBuiltinFunc(Name))
2379 FuncAttrs.addAttribute(Val: llvm::Attribute::NoBuiltin);
2380 if (!CodeGenOpts.TrapFuncName.empty())
2381 FuncAttrs.addAttribute(A: "trap-func-name", V: CodeGenOpts.TrapFuncName);
2382 } else {
2383 switch (CodeGenOpts.getFramePointer()) {
2384 case CodeGenOptions::FramePointerKind::None:
2385 // This is the default behavior.
2386 break;
2387 case CodeGenOptions::FramePointerKind::Reserved:
2388 case CodeGenOptions::FramePointerKind::NonLeafNoReserve:
2389 case CodeGenOptions::FramePointerKind::NonLeaf:
2390 case CodeGenOptions::FramePointerKind::All:
2391 FuncAttrs.addAttribute(A: "frame-pointer",
2392 V: CodeGenOptions::getFramePointerKindName(
2393 Kind: CodeGenOpts.getFramePointer()));
2394 }
2395
2396 if (CodeGenOpts.LessPreciseFPMAD)
2397 FuncAttrs.addAttribute(A: "less-precise-fpmad", V: "true");
2398
2399 if (CodeGenOpts.NullPointerIsValid)
2400 FuncAttrs.addAttribute(Val: llvm::Attribute::NullPointerIsValid);
2401
2402 if (LangOpts.getDefaultExceptionMode() == LangOptions::FPE_Ignore)
2403 FuncAttrs.addAttribute(A: "no-trapping-math", V: "true");
2404
2405 // TODO: Are these all needed?
2406 // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
2407 if (CodeGenOpts.SoftFloat)
2408 FuncAttrs.addAttribute(A: "use-soft-float", V: "true");
2409 FuncAttrs.addAttribute(A: "stack-protector-buffer-size",
2410 V: llvm::utostr(X: CodeGenOpts.SSPBufferSize));
2411 if (LangOpts.NoSignedZero)
2412 FuncAttrs.addAttribute(A: "no-signed-zeros-fp-math", V: "true");
2413
2414 // TODO: Reciprocal estimate codegen options should apply to instructions?
2415 const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
2416 if (!Recips.empty())
2417 FuncAttrs.addAttribute(A: "reciprocal-estimates", V: llvm::join(R: Recips, Separator: ","));
2418
2419 if (!CodeGenOpts.PreferVectorWidth.empty() &&
2420 CodeGenOpts.PreferVectorWidth != "none")
2421 FuncAttrs.addAttribute(A: "prefer-vector-width",
2422 V: CodeGenOpts.PreferVectorWidth);
2423
2424 if (CodeGenOpts.StackRealignment)
2425 FuncAttrs.addAttribute(A: "stackrealign");
2426 if (CodeGenOpts.Backchain)
2427 FuncAttrs.addAttribute(A: "backchain");
2428 if (CodeGenOpts.EnableSegmentedStacks)
2429 FuncAttrs.addAttribute(A: "split-stack");
2430
2431 if (CodeGenOpts.SpeculativeLoadHardening)
2432 FuncAttrs.addAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2433
2434 // Add zero-call-used-regs attribute.
2435 switch (CodeGenOpts.getZeroCallUsedRegs()) {
2436 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
2437 FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2438 break;
2439 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
2440 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr-arg");
2441 break;
2442 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
2443 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-gpr");
2444 break;
2445 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
2446 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used-arg");
2447 break;
2448 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
2449 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "used");
2450 break;
2451 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
2452 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr-arg");
2453 break;
2454 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
2455 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-gpr");
2456 break;
2457 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
2458 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all-arg");
2459 break;
2460 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
2461 FuncAttrs.addAttribute(A: "zero-call-used-regs", V: "all");
2462 break;
2463 }
2464 }
2465
2466 if (LangOpts.assumeFunctionsAreConvergent()) {
2467 // Conservatively, mark all functions and calls in CUDA and OpenCL as
2468 // convergent (meaning, they may call an intrinsically convergent op, such
2469 // as __syncthreads() / barrier(), and so can't have certain optimizations
2470 // applied around them). LLVM will remove this attribute where it safely
2471 // can.
2472 FuncAttrs.addAttribute(Val: llvm::Attribute::Convergent);
2473 }
2474
2475 // TODO: NoUnwind attribute should be added for other GPU modes HIP,
2476 // OpenMP offload. AFAIK, neither of them support exceptions in device code.
2477 if ((LangOpts.CUDA && LangOpts.CUDAIsDevice) || LangOpts.OpenCL ||
2478 LangOpts.SYCLIsDevice) {
2479 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2480 }
2481
2482 if (CodeGenOpts.SaveRegParams && !AttrOnCallSite)
2483 FuncAttrs.addAttribute(A: "save-reg-params");
2484
2485 for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
2486 StringRef Var, Value;
2487 std::tie(args&: Var, args&: Value) = Attr.split(Separator: '=');
2488 FuncAttrs.addAttribute(A: Var, V: Value);
2489 }
2490
2491 if (!AttrOnCallSite) {
2492 TargetInfo::BranchProtectionInfo BPI(LangOpts);
2493 TargetCodeGenInfo::initBranchProtectionFnAttributes(BPI, FuncAttrs);
2494 }
2495}
2496
2497/// Merges `target-features` from \TargetOpts and \F, and sets the result in
2498/// \FuncAttr
2499/// * features from \F are always kept
2500/// * a feature from \TargetOpts is kept if itself and its opposite are absent
2501/// from \F
2502static void
2503overrideFunctionFeaturesWithTargetFeatures(llvm::AttrBuilder &FuncAttr,
2504 const llvm::Function &F,
2505 const TargetOptions &TargetOpts) {
2506 auto FFeatures = F.getFnAttribute(Kind: "target-features");
2507
2508 llvm::StringSet<> MergedNames;
2509 SmallVector<StringRef> MergedFeatures;
2510 MergedFeatures.reserve(N: TargetOpts.Features.size());
2511
2512 auto AddUnmergedFeatures = [&](auto &&FeatureRange) {
2513 for (StringRef Feature : FeatureRange) {
2514 if (Feature.empty())
2515 continue;
2516 assert(Feature[0] == '+' || Feature[0] == '-');
2517 StringRef Name = Feature.drop_front(N: 1);
2518 bool Merged = !MergedNames.insert(key: Name).second;
2519 if (!Merged)
2520 MergedFeatures.push_back(Elt: Feature);
2521 }
2522 };
2523
2524 if (FFeatures.isValid())
2525 AddUnmergedFeatures(llvm::split(Str: FFeatures.getValueAsString(), Separator: ','));
2526 AddUnmergedFeatures(TargetOpts.Features);
2527
2528 if (!MergedFeatures.empty()) {
2529 llvm::sort(C&: MergedFeatures);
2530 FuncAttr.addAttribute(A: "target-features", V: llvm::join(R&: MergedFeatures, Separator: ","));
2531 }
2532}
2533
2534void CodeGen::mergeDefaultFunctionDefinitionAttributes(
2535 llvm::Function &F, const CodeGenOptions &CodeGenOpts,
2536 const LangOptions &LangOpts, const TargetOptions &TargetOpts,
2537 bool WillInternalize) {
2538
2539 llvm::AttrBuilder FuncAttrs(F.getContext());
2540 // Here we only extract the options that are relevant compared to the version
2541 // from GetCPUAndFeaturesAttributes.
2542 if (!TargetOpts.CPU.empty())
2543 FuncAttrs.addAttribute(A: "target-cpu", V: TargetOpts.CPU);
2544 if (!TargetOpts.TuneCPU.empty())
2545 FuncAttrs.addAttribute(A: "tune-cpu", V: TargetOpts.TuneCPU);
2546
2547 ::getTrivialDefaultFunctionAttributes(Name: F.getName(), HasOptnone: F.hasOptNone(),
2548 CodeGenOpts, LangOpts,
2549 /*AttrOnCallSite=*/false, FuncAttrs);
2550
2551 if (!WillInternalize && F.isInterposable()) {
2552 // Do not promote "dynamic" denormal-fp-math to this translation unit's
2553 // setting for weak functions that won't be internalized. The user has no
2554 // real control for how builtin bitcode is linked, so we shouldn't assume
2555 // later copies will use a consistent mode.
2556 F.addFnAttrs(Attrs: FuncAttrs);
2557 return;
2558 }
2559
2560 llvm::AttributeMask AttrsToRemove;
2561
2562 llvm::DenormalFPEnv OptsFPEnv(CodeGenOpts.FPDenormalMode,
2563 CodeGenOpts.FP32DenormalMode);
2564 llvm::DenormalFPEnv MergedFPEnv =
2565 OptsFPEnv.mergeCalleeMode(Callee: F.getDenormalFPEnv());
2566
2567 if (MergedFPEnv == llvm::DenormalFPEnv::getDefault()) {
2568 AttrsToRemove.addAttribute(Val: llvm::Attribute::DenormalFPEnv);
2569 } else {
2570 // Overwrite existing attribute
2571 FuncAttrs.addDenormalFPEnvAttr(Mode: MergedFPEnv);
2572 }
2573
2574 F.removeFnAttrs(Attrs: AttrsToRemove);
2575
2576 overrideFunctionFeaturesWithTargetFeatures(FuncAttr&: FuncAttrs, F, TargetOpts);
2577
2578 F.addFnAttrs(Attrs: FuncAttrs);
2579}
2580
2581void CodeGenModule::getTrivialDefaultFunctionAttributes(
2582 StringRef Name, bool HasOptnone, bool AttrOnCallSite,
2583 llvm::AttrBuilder &FuncAttrs) {
2584 ::getTrivialDefaultFunctionAttributes(Name, HasOptnone, CodeGenOpts: getCodeGenOpts(),
2585 LangOpts: getLangOpts(), AttrOnCallSite,
2586 FuncAttrs);
2587}
2588
2589void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
2590 bool HasOptnone,
2591 bool AttrOnCallSite,
2592 llvm::AttrBuilder &FuncAttrs) {
2593 getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,
2594 FuncAttrs);
2595
2596 if (!AttrOnCallSite)
2597 TargetCodeGenInfo::initPointerAuthFnAttributes(Opts: CodeGenOpts.PointerAuth,
2598 FuncAttrs);
2599
2600 // If we're just getting the default, get the default values for mergeable
2601 // attributes.
2602 if (!AttrOnCallSite)
2603 addMergableDefaultFunctionAttributes(CodeGenOpts, FuncAttrs);
2604}
2605
2606void CodeGenModule::addDefaultFunctionDefinitionAttributes(
2607 llvm::AttrBuilder &attrs) {
2608 getDefaultFunctionAttributes(/*function name*/ Name: "", /*optnone*/ HasOptnone: false,
2609 /*for call*/ AttrOnCallSite: false, FuncAttrs&: attrs);
2610 GetCPUAndFeaturesAttributes(GD: GlobalDecl(), AttrBuilder&: attrs);
2611}
2612
2613static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
2614 const LangOptions &LangOpts,
2615 const NoBuiltinAttr *NBA = nullptr) {
2616 auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
2617 SmallString<32> AttributeName;
2618 AttributeName += "no-builtin-";
2619 AttributeName += BuiltinName;
2620 FuncAttrs.addAttribute(A: AttributeName);
2621 };
2622
2623 // First, handle the language options passed through -fno-builtin.
2624 if (LangOpts.NoBuiltin) {
2625 // -fno-builtin disables them all.
2626 FuncAttrs.addAttribute(A: "no-builtins");
2627 return;
2628 }
2629
2630 // Then, add attributes for builtins specified through -fno-builtin-<name>.
2631 llvm::for_each(Range: LangOpts.NoBuiltinFuncs, F: AddNoBuiltinAttr);
2632
2633 // Now, let's check the __attribute__((no_builtin("...")) attribute added to
2634 // the source.
2635 if (!NBA)
2636 return;
2637
2638 // If there is a wildcard in the builtin names specified through the
2639 // attribute, disable them all.
2640 if (llvm::is_contained(Range: NBA->builtinNames(), Element: "*")) {
2641 FuncAttrs.addAttribute(A: "no-builtins");
2642 return;
2643 }
2644
2645 // And last, add the rest of the builtin names.
2646 llvm::for_each(Range: NBA->builtinNames(), F: AddNoBuiltinAttr);
2647}
2648
2649static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
2650 const llvm::DataLayout &DL, const ABIArgInfo &AI,
2651 bool CheckCoerce = true) {
2652 llvm::Type *Ty = Types.ConvertTypeForMem(T: QTy);
2653 if (AI.getKind() == ABIArgInfo::Indirect ||
2654 AI.getKind() == ABIArgInfo::IndirectAliased)
2655 return true;
2656 if (AI.getKind() == ABIArgInfo::Extend && !AI.isNoExt())
2657 return true;
2658 if (!DL.typeSizeEqualsStoreSize(Ty))
2659 // TODO: This will result in a modest amount of values not marked noundef
2660 // when they could be. We care about values that *invisibly* contain undef
2661 // bits from the perspective of LLVM IR.
2662 return false;
2663 if (CheckCoerce && AI.canHaveCoerceToType()) {
2664 llvm::Type *CoerceTy = AI.getCoerceToType();
2665 if (llvm::TypeSize::isKnownGT(LHS: DL.getTypeSizeInBits(Ty: CoerceTy),
2666 RHS: DL.getTypeSizeInBits(Ty)))
2667 // If we're coercing to a type with a greater size than the canonical one,
2668 // we're introducing new undef bits.
2669 // Coercing to a type of smaller or equal size is ok, as we know that
2670 // there's no internal padding (typeSizeEqualsStoreSize).
2671 return false;
2672 }
2673 if (QTy->isBitIntType())
2674 return true;
2675 if (QTy->isReferenceType())
2676 return true;
2677 if (QTy->isNullPtrType())
2678 return false;
2679 if (QTy->isMemberPointerType())
2680 // TODO: Some member pointers are `noundef`, but it depends on the ABI. For
2681 // now, never mark them.
2682 return false;
2683 if (QTy->isScalarType()) {
2684 if (const ComplexType *Complex = dyn_cast<ComplexType>(Val&: QTy))
2685 return DetermineNoUndef(QTy: Complex->getElementType(), Types, DL, AI, CheckCoerce: false);
2686 return true;
2687 }
2688 if (const VectorType *Vector = dyn_cast<VectorType>(Val&: QTy))
2689 return DetermineNoUndef(QTy: Vector->getElementType(), Types, DL, AI, CheckCoerce: false);
2690 if (const MatrixType *Matrix = dyn_cast<MatrixType>(Val&: QTy))
2691 return DetermineNoUndef(QTy: Matrix->getElementType(), Types, DL, AI, CheckCoerce: false);
2692 if (const ArrayType *Array = dyn_cast<ArrayType>(Val&: QTy))
2693 return DetermineNoUndef(QTy: Array->getElementType(), Types, DL, AI, CheckCoerce: false);
2694
2695 // TODO: Some structs may be `noundef`, in specific situations.
2696 return false;
2697}
2698
2699/// Test if it's legal to apply nofpclass for the given parameter type and it's
2700/// lowered IR type.
2701static bool canApplyNoFPClass(const ABIArgInfo &AI, QualType ParamType,
2702 bool IsReturn) {
2703 // Should only apply to FP types in the source, not ABI promoted.
2704 if (!ParamType->hasFloatingRepresentation())
2705 return false;
2706
2707 // The promoted-to IR type also needs to support nofpclass.
2708 llvm::Type *IRTy = AI.getCoerceToType();
2709 if (llvm::AttributeFuncs::isNoFPClassCompatibleType(Ty: IRTy))
2710 return true;
2711
2712 if (llvm::StructType *ST = dyn_cast<llvm::StructType>(Val: IRTy)) {
2713 return !IsReturn && AI.getCanBeFlattened() &&
2714 llvm::all_of(Range: ST->elements(),
2715 P: llvm::AttributeFuncs::isNoFPClassCompatibleType);
2716 }
2717
2718 return false;
2719}
2720
2721void CodeGenModule::AdjustMemoryAttribute(StringRef Name,
2722 CGCalleeInfo CalleeInfo,
2723 llvm::AttributeList &Attrs) {
2724 if (Attrs.getMemoryEffects().getModRef() == llvm::ModRefInfo::NoModRef) {
2725 Attrs = Attrs.removeFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::Memory);
2726 llvm::Attribute MemoryAttr = llvm::Attribute::getWithMemoryEffects(
2727 Context&: getLLVMContext(), ME: llvm::MemoryEffects::writeOnly());
2728 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Attr: MemoryAttr);
2729 }
2730}
2731
2732/// Construct the IR attribute list of a function or call.
2733///
2734/// When adding an attribute, please consider where it should be handled:
2735///
2736/// - getDefaultFunctionAttributes is for attributes that are essentially
2737/// part of the global target configuration (but perhaps can be
2738/// overridden on a per-function basis). Adding attributes there
2739/// will cause them to also be set in frontends that build on Clang's
2740/// target-configuration logic, as well as for code defined in library
2741/// modules such as CUDA's libdevice.
2742///
2743/// - ConstructAttributeList builds on top of getDefaultFunctionAttributes
2744/// and adds declaration-specific, convention-specific, and
2745/// frontend-specific logic. The last is of particular importance:
2746/// attributes that restrict how the frontend generates code must be
2747/// added here rather than getDefaultFunctionAttributes.
2748///
2749void CodeGenModule::ConstructAttributeList(StringRef Name,
2750 const CGFunctionInfo &FI,
2751 CGCalleeInfo CalleeInfo,
2752 llvm::AttributeList &AttrList,
2753 unsigned &CallingConv,
2754 bool AttrOnCallSite, bool IsThunk) {
2755 llvm::AttrBuilder FuncAttrs(getLLVMContext());
2756 llvm::AttrBuilder RetAttrs(getLLVMContext());
2757
2758 // Collect function IR attributes from the CC lowering.
2759 // We'll collect the paramete and result attributes later.
2760 CallingConv = FI.getEffectiveCallingConvention();
2761 if (FI.isNoReturn())
2762 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2763 if (FI.isCmseNSCall())
2764 FuncAttrs.addAttribute(A: "cmse_nonsecure_call");
2765
2766 // Collect function IR attributes from the callee prototype if we have one.
2767 AddAttributesFromFunctionProtoType(Ctx&: getContext(), FuncAttrs,
2768 FPT: CalleeInfo.getCalleeFunctionProtoType());
2769 const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2770
2771 // Attach assumption attributes to the declaration. If this is a call
2772 // site, attach assumptions from the caller to the call as well.
2773 AddAttributesFromOMPAssumes(FuncAttrs, Callee: TargetDecl);
2774
2775 bool HasOptnone = false;
2776 // The NoBuiltinAttr attached to the target FunctionDecl.
2777 const NoBuiltinAttr *NBA = nullptr;
2778
2779 // Some ABIs may result in additional accesses to arguments that may
2780 // otherwise not be present.
2781 std::optional<llvm::Attribute::AttrKind> MemAttrForPtrArgs;
2782 bool AddedPotentialArgAccess = false;
2783 auto AddPotentialArgAccess = [&]() {
2784 AddedPotentialArgAccess = true;
2785 llvm::Attribute A = FuncAttrs.getAttribute(Kind: llvm::Attribute::Memory);
2786 if (A.isValid())
2787 FuncAttrs.addMemoryAttr(ME: A.getMemoryEffects() |
2788 llvm::MemoryEffects::argMemOnly());
2789 };
2790
2791 // Collect function IR attributes based on declaration-specific
2792 // information.
2793 // FIXME: handle sseregparm someday...
2794 if (TargetDecl) {
2795 if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2796 FuncAttrs.addAttribute(Val: llvm::Attribute::ReturnsTwice);
2797 if (TargetDecl->hasAttr<NoThrowAttr>())
2798 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2799 if (TargetDecl->hasAttr<NoReturnAttr>())
2800 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2801 if (TargetDecl->hasAttr<ColdAttr>())
2802 FuncAttrs.addAttribute(Val: llvm::Attribute::Cold);
2803 if (TargetDecl->hasAttr<HotAttr>())
2804 FuncAttrs.addAttribute(Val: llvm::Attribute::Hot);
2805 if (TargetDecl->hasAttr<NoDuplicateAttr>())
2806 FuncAttrs.addAttribute(Val: llvm::Attribute::NoDuplicate);
2807 if (TargetDecl->hasAttr<ConvergentAttr>())
2808 FuncAttrs.addAttribute(Val: llvm::Attribute::Convergent);
2809
2810 if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2811 AddAttributesFromFunctionProtoType(
2812 Ctx&: getContext(), FuncAttrs, FPT: Fn->getType()->getAs<FunctionProtoType>());
2813 if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
2814 // A sane operator new returns a non-aliasing pointer.
2815 if (getCodeGenOpts().AssumeSaneOperatorNew &&
2816 Fn->getDeclName().isAnyOperatorNew())
2817 RetAttrs.addAttribute(Val: llvm::Attribute::NoAlias);
2818 }
2819 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: Fn);
2820 const bool IsVirtualCall = MD && MD->isVirtual();
2821 // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2822 // virtual function. These attributes are not inherited by overloads.
2823 if (!(AttrOnCallSite && IsVirtualCall)) {
2824 if (Fn->isNoReturn())
2825 FuncAttrs.addAttribute(Val: llvm::Attribute::NoReturn);
2826 NBA = Fn->getAttr<NoBuiltinAttr>();
2827 }
2828 }
2829
2830 if (isa<FunctionDecl>(Val: TargetDecl) || isa<VarDecl>(Val: TargetDecl)) {
2831 // Only place nomerge attribute on call sites, never functions. This
2832 // allows it to work on indirect virtual function calls.
2833 if (AttrOnCallSite && TargetDecl->hasAttr<NoMergeAttr>())
2834 FuncAttrs.addAttribute(Val: llvm::Attribute::NoMerge);
2835 }
2836
2837 // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2838 if (TargetDecl->hasAttr<ConstAttr>()) {
2839 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::none());
2840 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2841 // gcc specifies that 'const' functions have greater restrictions than
2842 // 'pure' functions, so they also cannot have infinite loops.
2843 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
2844 MemAttrForPtrArgs = llvm::Attribute::ReadNone;
2845 } else if (TargetDecl->hasAttr<PureAttr>()) {
2846 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::readOnly());
2847 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2848 // gcc specifies that 'pure' functions cannot have infinite loops.
2849 FuncAttrs.addAttribute(Val: llvm::Attribute::WillReturn);
2850 MemAttrForPtrArgs = llvm::Attribute::ReadOnly;
2851 } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2852 FuncAttrs.addMemoryAttr(ME: llvm::MemoryEffects::inaccessibleOrArgMemOnly());
2853 FuncAttrs.addAttribute(Val: llvm::Attribute::NoUnwind);
2854 }
2855 if (const auto *RA = TargetDecl->getAttr<RestrictAttr>();
2856 RA && RA->getDeallocator() == nullptr)
2857 RetAttrs.addAttribute(Val: llvm::Attribute::NoAlias);
2858 if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2859 !CodeGenOpts.NullPointerIsValid)
2860 RetAttrs.addAttribute(Val: llvm::Attribute::NonNull);
2861 if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2862 FuncAttrs.addAttribute(A: "no_caller_saved_registers");
2863 if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2864 FuncAttrs.addAttribute(Val: llvm::Attribute::NoCfCheck);
2865 if (TargetDecl->hasAttr<LeafAttr>())
2866 FuncAttrs.addAttribute(Val: llvm::Attribute::NoCallback);
2867 if (TargetDecl->hasAttr<BPFFastCallAttr>())
2868 FuncAttrs.addAttribute(A: "bpf_fastcall");
2869
2870 HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2871 if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2872 std::optional<unsigned> NumElemsParam;
2873 if (AllocSize->getNumElemsParam().isValid())
2874 NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2875 FuncAttrs.addAllocSizeAttr(ElemSizeArg: AllocSize->getElemSizeParam().getLLVMIndex(),
2876 NumElemsArg: NumElemsParam);
2877 }
2878
2879 // OpenCL v2.0 Work groups may be whether uniform or not.
2880 // '-cl-uniform-work-group-size' compile option gets a hint
2881 // to the compiler that the global work-size be a multiple of
2882 // the work-group size specified to clEnqueueNDRangeKernel
2883 // (i.e. work groups are uniform).
2884 if (getLangOpts().OffloadUniformBlock)
2885 FuncAttrs.addAttribute(A: "uniform-work-group-size");
2886
2887 if (TargetDecl->hasAttr<ArmLocallyStreamingAttr>())
2888 FuncAttrs.addAttribute(A: "aarch64_pstate_sm_body");
2889
2890 if (auto *ModularFormat = TargetDecl->getAttr<ModularFormatAttr>()) {
2891 FormatAttr *Format = TargetDecl->getAttr<FormatAttr>();
2892 StringRef Type = Format->getType()->getName();
2893 std::string FormatIdx = std::to_string(val: Format->getFormatIdx());
2894 std::string FirstArg = std::to_string(val: Format->getFirstArg());
2895 SmallVector<StringRef> Args = {
2896 Type, FormatIdx, FirstArg,
2897 ModularFormat->getModularImplFn()->getName(),
2898 ModularFormat->getImplName()};
2899 llvm::append_range(C&: Args, R: ModularFormat->aspects());
2900 FuncAttrs.addAttribute(A: "modular-format", V: llvm::join(R&: Args, Separator: ","));
2901 }
2902 }
2903
2904 // Attach "no-builtins" attributes to:
2905 // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2906 // * definitions: "no-builtins" or "no-builtin-<name>" only.
2907 // The attributes can come from:
2908 // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2909 // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2910 addNoBuiltinAttributes(FuncAttrs, LangOpts: getLangOpts(), NBA);
2911
2912 // Collect function IR attributes based on global settiings.
2913 getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2914
2915 // Override some default IR attributes based on declaration-specific
2916 // information.
2917 if (TargetDecl) {
2918 if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2919 FuncAttrs.removeAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2920 if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2921 FuncAttrs.addAttribute(Val: llvm::Attribute::SpeculativeLoadHardening);
2922 if (TargetDecl->hasAttr<NoSplitStackAttr>())
2923 FuncAttrs.removeAttribute(A: "split-stack");
2924 if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
2925 // A function "__attribute__((...))" overrides the command-line flag.
2926 auto Kind =
2927 TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2928 FuncAttrs.removeAttribute(A: "zero-call-used-regs");
2929 FuncAttrs.addAttribute(
2930 A: "zero-call-used-regs",
2931 V: ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Val: Kind));
2932 }
2933
2934 // Add NonLazyBind attribute to function declarations when -fno-plt
2935 // is used.
2936 // FIXME: what if we just haven't processed the function definition
2937 // yet, or if it's an external definition like C99 inline?
2938 if (CodeGenOpts.NoPLT) {
2939 if (auto *Fn = dyn_cast<FunctionDecl>(Val: TargetDecl)) {
2940 if (!Fn->isDefined() && !AttrOnCallSite) {
2941 FuncAttrs.addAttribute(Val: llvm::Attribute::NonLazyBind);
2942 }
2943 }
2944 }
2945 // Remove 'convergent' if requested.
2946 if (TargetDecl->hasAttr<NoConvergentAttr>())
2947 FuncAttrs.removeAttribute(Val: llvm::Attribute::Convergent);
2948 }
2949
2950 // Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2951 // functions with -funique-internal-linkage-names.
2952 if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames) {
2953 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
2954 if (!FD->isExternallyVisible())
2955 FuncAttrs.addAttribute(A: "sample-profile-suffix-elision-policy",
2956 V: "selected");
2957 }
2958 }
2959
2960 // Collect non-call-site function IR attributes from declaration-specific
2961 // information.
2962 if (!AttrOnCallSite) {
2963 if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>())
2964 FuncAttrs.addAttribute(A: "cmse_nonsecure_entry");
2965
2966 // Whether tail calls are enabled.
2967 auto shouldDisableTailCalls = [&] {
2968 // Should this be honored in getDefaultFunctionAttributes?
2969 if (CodeGenOpts.DisableTailCalls)
2970 return true;
2971
2972 if (!TargetDecl)
2973 return false;
2974
2975 if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2976 TargetDecl->hasAttr<AnyX86InterruptAttr>())
2977 return true;
2978
2979 if (CodeGenOpts.NoEscapingBlockTailCalls) {
2980 if (const auto *BD = dyn_cast<BlockDecl>(Val: TargetDecl))
2981 if (!BD->doesNotEscape())
2982 return true;
2983 }
2984
2985 return false;
2986 };
2987 if (shouldDisableTailCalls())
2988 FuncAttrs.addAttribute(A: "disable-tail-calls", V: "true");
2989
2990 // These functions require the returns_twice attribute for correct codegen,
2991 // but the attribute may not be added if -fno-builtin is specified. We
2992 // explicitly add that attribute here.
2993 static const llvm::StringSet<> ReturnsTwiceFn{
2994 "_setjmpex", "setjmp", "_setjmp", "vfork",
2995 "sigsetjmp", "__sigsetjmp", "savectx", "getcontext"};
2996 if (ReturnsTwiceFn.contains(key: Name))
2997 FuncAttrs.addAttribute(Val: llvm::Attribute::ReturnsTwice);
2998
2999 // CPU/feature overrides. addDefaultFunctionDefinitionAttributes
3000 // handles these separately to set them based on the global defaults.
3001 GetCPUAndFeaturesAttributes(GD: CalleeInfo.getCalleeDecl(), AttrBuilder&: FuncAttrs);
3002
3003 // Windows hotpatching support
3004 if (!MSHotPatchFunctions.empty()) {
3005 bool IsHotPatched = llvm::binary_search(Range&: MSHotPatchFunctions, Value&: Name);
3006 if (IsHotPatched)
3007 FuncAttrs.addAttribute(A: "marked_for_windows_hot_patching");
3008 }
3009 }
3010
3011 // Mark functions that are replaceable by the loader.
3012 if (CodeGenOpts.isLoaderReplaceableFunctionName(FuncName: Name))
3013 FuncAttrs.addAttribute(A: "loader-replaceable");
3014
3015 // Collect attributes from arguments and return values.
3016 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
3017
3018 QualType RetTy = FI.getReturnType();
3019 const ABIArgInfo &RetAI = FI.getReturnInfo();
3020 const llvm::DataLayout &DL = getDataLayout();
3021
3022 // Determine if the return type could be partially undef
3023 if (CodeGenOpts.EnableNoundefAttrs &&
3024 HasStrictReturn(Module: *this, RetTy, TargetDecl)) {
3025 if (!RetTy->isVoidType() && RetAI.getKind() != ABIArgInfo::Indirect &&
3026 DetermineNoUndef(QTy: RetTy, Types&: getTypes(), DL, AI: RetAI))
3027 RetAttrs.addAttribute(Val: llvm::Attribute::NoUndef);
3028 }
3029
3030 switch (RetAI.getKind()) {
3031 case ABIArgInfo::Extend:
3032 assert(!RetAI.isConservative() && "Conservative return not supported");
3033 if (RetAI.isSignExt())
3034 RetAttrs.addAttribute(Val: llvm::Attribute::SExt);
3035 else if (RetAI.isZeroExt())
3036 RetAttrs.addAttribute(Val: llvm::Attribute::ZExt);
3037 else
3038 RetAttrs.addAttribute(Val: llvm::Attribute::NoExt);
3039 [[fallthrough]];
3040 case ABIArgInfo::TargetSpecific:
3041 case ABIArgInfo::Direct:
3042 if (RetAI.getInReg())
3043 RetAttrs.addAttribute(Val: llvm::Attribute::InReg);
3044
3045 if (canApplyNoFPClass(AI: RetAI, ParamType: RetTy, IsReturn: true))
3046 RetAttrs.addNoFPClassAttr(
3047 NoFPClassMask: CodeGenUtils::getNoFPClassTestMask(LangOpts: getLangOpts()));
3048
3049 break;
3050 case ABIArgInfo::Ignore:
3051 break;
3052
3053 case ABIArgInfo::InAlloca:
3054 case ABIArgInfo::Indirect: {
3055 // inalloca and sret disable readnone and readonly
3056 AddPotentialArgAccess();
3057 break;
3058 }
3059
3060 case ABIArgInfo::CoerceAndExpand:
3061 break;
3062
3063 case ABIArgInfo::Expand:
3064 case ABIArgInfo::IndirectAliased:
3065 llvm_unreachable("Invalid ABI kind for return argument");
3066 }
3067
3068 if (!IsThunk) {
3069 // FIXME: fix this properly, https://reviews.llvm.org/D100388
3070 if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
3071 QualType PTy = RefTy->getPointeeType();
3072 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
3073 RetAttrs.addDereferenceableAttr(
3074 Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
3075 if (getTypes().getTargetAddressSpace(T: PTy) == 0 &&
3076 !CodeGenOpts.NullPointerIsValid)
3077 RetAttrs.addAttribute(Val: llvm::Attribute::NonNull);
3078 if (PTy->isObjectType()) {
3079 llvm::Align Alignment =
3080 getNaturalPointeeTypeAlignment(T: RetTy).getAsAlign();
3081 RetAttrs.addAlignmentAttr(Align: Alignment);
3082 }
3083 }
3084 }
3085
3086 bool hasUsedSRet = false;
3087 SmallVector<llvm::AttrBuilder, 4> ArgAttrs;
3088 for (unsigned I = 0; I < IRFunctionArgs.totalIRArgs(); ++I)
3089 ArgAttrs.emplace_back(Args&: getLLVMContext());
3090
3091 // Attach attributes to sret.
3092 if (IRFunctionArgs.hasSRetArg()) {
3093 llvm::AttrBuilder &SRETAttrs = ArgAttrs[IRFunctionArgs.getSRetArgNo()];
3094 SRETAttrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: RetTy));
3095 SRETAttrs.addAttribute(Val: llvm::Attribute::Writable);
3096 SRETAttrs.addAttribute(Val: llvm::Attribute::DeadOnUnwind);
3097 hasUsedSRet = true;
3098 if (RetAI.getInReg())
3099 SRETAttrs.addAttribute(Val: llvm::Attribute::InReg);
3100 SRETAttrs.addAlignmentAttr(Align: RetAI.getIndirectAlign().getQuantity());
3101 }
3102
3103 // Attach attributes to inalloca argument.
3104 if (IRFunctionArgs.hasInallocaArg()) {
3105 ArgAttrs[IRFunctionArgs.getInallocaArgNo()].addInAllocaAttr(
3106 Ty: FI.getArgStruct());
3107 }
3108
3109 // Apply `nonnull`, `dereferenceable(N)` and `align N` to the `this` argument,
3110 // unless this is a thunk function. Add dead_on_return to the `this` argument
3111 // in base class destructors to aid in DSE.
3112 // FIXME: fix this properly, https://reviews.llvm.org/D100388
3113 if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() &&
3114 !FI.arg_begin()->type->isVoidPointerType() && !IsThunk) {
3115 auto IRArgs = IRFunctionArgs.getIRArgs(ArgNo: 0);
3116
3117 assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
3118
3119 llvm::AttrBuilder &Attrs = ArgAttrs[IRArgs.first];
3120
3121 QualType ThisTy = FI.arg_begin()->type.getTypePtr()->getPointeeType();
3122 int64_t ThisSz = getMinimumObjectSize(Ty: ThisTy).getQuantity();
3123
3124 if (!CodeGenOpts.NullPointerIsValid &&
3125 getTypes().getTargetAddressSpace(T: FI.arg_begin()->type) == 0) {
3126 Attrs.addAttribute(Val: llvm::Attribute::NonNull);
3127 Attrs.addDereferenceableAttr(Bytes: ThisSz);
3128 } else {
3129 // FIXME dereferenceable should be correct here, regardless of
3130 // NullPointerIsValid. However, dereferenceable currently does not always
3131 // respect NullPointerIsValid and may imply nonnull and break the program.
3132 // See https://reviews.llvm.org/D66618 for discussions.
3133 Attrs.addDereferenceableOrNullAttr(Bytes: ThisSz);
3134 }
3135
3136 llvm::Align Alignment =
3137 getNaturalTypeAlignment(T: ThisTy, /*BaseInfo=*/nullptr,
3138 /*TBAAInfo=*/nullptr, /*forPointeeType=*/true)
3139 .getAsAlign();
3140 Attrs.addAlignmentAttr(Align: Alignment);
3141
3142 const auto *DD = dyn_cast_if_present<CXXDestructorDecl>(
3143 Val: CalleeInfo.getCalleeDecl().getDecl());
3144 // Do not annotate vector deleting destructors with dead_on_return as the
3145 // this pointer in that case points to an array which we cannot
3146 // statically know the size of. Also do not mark deleting destructors
3147 // dead_on_return as then we might delete stores inside of a user-defined
3148 // operator delete implementation if it gets inlined, which would be
3149 // incorrect as the object's lifetime has already ended and the operator
3150 // delete implementation is allowed to manipulate the underlying storage.
3151 if (DD &&
3152 CalleeInfo.getCalleeDecl().getDtorType() !=
3153 CXXDtorType::Dtor_VectorDeleting &&
3154 CalleeInfo.getCalleeDecl().getDtorType() !=
3155 CXXDtorType::Dtor_Deleting &&
3156 CodeGenOpts.StrictLifetimes) {
3157 const CXXRecordDecl *ClassDecl =
3158 dyn_cast<CXXRecordDecl>(Val: DD->getDeclContext());
3159 // We cannot add dead_on_return if we have virtual base classes because
3160 // they will generally still be live after the base object destructor.
3161 if (ClassDecl->getNumVBases() == 0)
3162 Attrs.addDeadOnReturnAttr(Info: llvm::DeadOnReturnInfo(
3163 Context.getASTRecordLayout(D: ClassDecl).getDataSize().getQuantity()));
3164 }
3165 }
3166
3167 if (const auto *Fn = dyn_cast_if_present<FunctionDecl>(Val: TargetDecl)) {
3168 if (Fn->isReplaceableGlobalAllocationFunction() &&
3169 Fn->getDeclName().isAnyOperatorDelete()) {
3170 auto IRArgs = IRFunctionArgs.getIRArgs(ArgNo: 0);
3171 assert(IRArgs.second == 1 && "Expected only a single pointer to delete.");
3172 ArgAttrs[IRArgs.first].addAttribute(Val: llvm::Attribute::AllocatedPointer);
3173 }
3174 }
3175
3176 unsigned ArgNo = 0;
3177 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(), E = FI.arg_end();
3178 I != E; ++I, ++ArgNo) {
3179 QualType ParamType = I->type;
3180 const ABIArgInfo &AI = I->info;
3181 llvm::AttrBuilder Attrs(getLLVMContext());
3182
3183 // Add attribute for padding argument, if necessary.
3184 if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
3185 if (AI.getPaddingInReg()) {
3186 ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)].addAttribute(
3187 Val: llvm::Attribute::InReg);
3188 }
3189 }
3190
3191 // Decide whether the argument we're handling could be partially undef
3192 if (CodeGenOpts.EnableNoundefAttrs &&
3193 DetermineNoUndef(QTy: ParamType, Types&: getTypes(), DL, AI)) {
3194 Attrs.addAttribute(Val: llvm::Attribute::NoUndef);
3195 }
3196
3197 // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
3198 // have the corresponding parameter variable. It doesn't make
3199 // sense to do it here because parameters are so messed up.
3200 switch (AI.getKind()) {
3201 case ABIArgInfo::Extend:
3202 // Under conservative extension, add sext/zext only on the call site; the
3203 // callee does not get to assume extension.
3204 if (AttrOnCallSite || !AI.isConservative()) {
3205 if (AI.isSignExt())
3206 Attrs.addAttribute(Val: llvm::Attribute::SExt);
3207 else if (AI.isZeroExt())
3208 Attrs.addAttribute(Val: llvm::Attribute::ZExt);
3209 else
3210 Attrs.addAttribute(Val: llvm::Attribute::NoExt);
3211 }
3212 [[fallthrough]];
3213 case ABIArgInfo::TargetSpecific:
3214 case ABIArgInfo::Direct:
3215 if (ArgNo == 0 && FI.isChainCall())
3216 Attrs.addAttribute(Val: llvm::Attribute::Nest);
3217 else if (AI.getInReg())
3218 Attrs.addAttribute(Val: llvm::Attribute::InReg);
3219 Attrs.addStackAlignmentAttr(Align: llvm::MaybeAlign(AI.getDirectAlign()));
3220
3221 if (canApplyNoFPClass(AI, ParamType, IsReturn: false))
3222 Attrs.addNoFPClassAttr(
3223 NoFPClassMask: CodeGenUtils::getNoFPClassTestMask(LangOpts: getLangOpts()));
3224 break;
3225 case ABIArgInfo::Indirect: {
3226 assert(!ParamType->isIncompleteType() &&
3227 "Pass-by-value parameter has incomplete definition?");
3228
3229 if (AI.getInReg())
3230 Attrs.addAttribute(Val: llvm::Attribute::InReg);
3231
3232 // HLSL out and inout parameters must not be marked with ByVal or
3233 // DeadOnReturn attributes because stores to these parameters by the
3234 // callee are visible to the caller.
3235 if (auto ParamABI = FI.getExtParameterInfo(argIndex: ArgNo).getABI();
3236 ParamABI != ParameterABI::HLSLOut &&
3237 ParamABI != ParameterABI::HLSLInOut) {
3238
3239 // Depending on the ABI, this may be either a byval or a dead_on_return
3240 // argument.
3241 if (AI.getIndirectByVal()) {
3242 Attrs.addByValAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
3243 } else {
3244 // Add dead_on_return when the object's lifetime ends in the callee.
3245 // This includes trivially-destructible objects, as well as objects
3246 // whose destruction / clean-up is carried out within the callee
3247 // (e.g., Obj-C ARC-managed structs, MSVC callee-destroyed objects).
3248 if (!ParamType.isDestructedType() || !ParamType->isRecordType() ||
3249 ParamType->castAsRecordDecl()->isParamDestroyedInCallee())
3250 Attrs.addDeadOnReturnAttr(Info: llvm::DeadOnReturnInfo());
3251 }
3252 }
3253
3254 auto *Decl = ParamType->getAsRecordDecl();
3255 if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
3256 Decl->getArgPassingRestrictions() ==
3257 RecordArgPassingKind::CanPassInRegs)
3258 // When calling the function, the pointer passed in will be the only
3259 // reference to the underlying object. Mark it accordingly.
3260 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
3261
3262 // TODO: We could add the byref attribute if not byval, but it would
3263 // require updating many testcases.
3264
3265 CharUnits Align = AI.getIndirectAlign();
3266
3267 // In a byval argument, it is important that the required
3268 // alignment of the type is honored, as LLVM might be creating a
3269 // *new* stack object, and needs to know what alignment to give
3270 // it. (Sometimes it can deduce a sensible alignment on its own,
3271 // but not if clang decides it must emit a packed struct, or the
3272 // user specifies increased alignment requirements.)
3273 //
3274 // This is different from indirect *not* byval, where an aligned copy is
3275 // already created by the caller, and the align attribute is purely
3276 // informative. However, this can still be useful information for
3277 // optimizations, such as giving us one necessary condition for checking
3278 // if a load to this pointer can be speculatively executed.
3279 assert(!Align.isZero());
3280 Attrs.addAlignmentAttr(Align: Align.getQuantity());
3281
3282 // The `nofree` and `dereferenceable` attributes can already be inferred
3283 // for `byval` arguments. We'll need to provide additional hints
3284 // otherwise.
3285 if (!AI.getIndirectByVal()) {
3286 // Both 6.9.1 of the C standard and [basic.stc.auto] of the C++ standard
3287 // require parameters to have automatic storage duration. Therefore, the
3288 // underlying object of this pointer will not be freed during the
3289 // function's execution.
3290 Attrs.addAttribute(Val: llvm::Attribute::NoFreeObj);
3291 Attrs.addDereferenceableAttr(
3292 Bytes: Context.getTypeSizeInChars(T: ParamType).getQuantity());
3293 }
3294
3295 // byval disables readnone and readonly.
3296 AddPotentialArgAccess();
3297 break;
3298 }
3299 case ABIArgInfo::IndirectAliased: {
3300 CharUnits Align = AI.getIndirectAlign();
3301 Attrs.addByRefAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
3302 Attrs.addAlignmentAttr(Align: Align.getQuantity());
3303 break;
3304 }
3305 case ABIArgInfo::Ignore:
3306 case ABIArgInfo::Expand:
3307 case ABIArgInfo::CoerceAndExpand:
3308 break;
3309
3310 case ABIArgInfo::InAlloca:
3311 // inalloca disables readnone and readonly.
3312 AddPotentialArgAccess();
3313 continue;
3314 }
3315
3316 if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
3317 QualType PTy = RefTy->getPointeeType();
3318 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
3319 Attrs.addDereferenceableAttr(Bytes: getMinimumObjectSize(Ty: PTy).getQuantity());
3320 if (getTypes().getTargetAddressSpace(T: PTy) == 0 &&
3321 !CodeGenOpts.NullPointerIsValid)
3322 Attrs.addAttribute(Val: llvm::Attribute::NonNull);
3323 if (PTy->isObjectType()) {
3324 llvm::Align Alignment =
3325 getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
3326 Attrs.addAlignmentAttr(Align: Alignment);
3327 }
3328 }
3329
3330 // From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:
3331 // > For arguments to a __kernel function declared to be a pointer to a
3332 // > data type, the OpenCL compiler can assume that the pointee is always
3333 // > appropriately aligned as required by the data type.
3334 if (TargetDecl &&
3335 DeviceKernelAttr::isOpenCLSpelling(
3336 A: TargetDecl->getAttr<DeviceKernelAttr>()) &&
3337 ParamType->isPointerType()) {
3338 QualType PTy = ParamType->getPointeeType();
3339 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
3340 llvm::Align Alignment =
3341 getNaturalPointeeTypeAlignment(T: ParamType).getAsAlign();
3342 Attrs.addAlignmentAttr(Align: Alignment);
3343 }
3344 }
3345
3346 switch (FI.getExtParameterInfo(argIndex: ArgNo).getABI()) {
3347 case ParameterABI::HLSLOut:
3348 case ParameterABI::HLSLInOut:
3349 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
3350 break;
3351 case ParameterABI::Ordinary:
3352 break;
3353
3354 case ParameterABI::SwiftIndirectResult: {
3355 // Add 'sret' if we haven't already used it for something, but
3356 // only if the result is void.
3357 if (!hasUsedSRet && RetTy->isVoidType()) {
3358 Attrs.addStructRetAttr(Ty: getTypes().ConvertTypeForMem(T: ParamType));
3359 hasUsedSRet = true;
3360 }
3361
3362 // Add 'noalias' in either case.
3363 Attrs.addAttribute(Val: llvm::Attribute::NoAlias);
3364
3365 // Add 'dereferenceable' and 'alignment'.
3366 auto PTy = ParamType->getPointeeType();
3367 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
3368 auto info = getContext().getTypeInfoInChars(T: PTy);
3369 Attrs.addDereferenceableAttr(Bytes: info.Width.getQuantity());
3370 Attrs.addAlignmentAttr(Align: info.Align.getAsAlign());
3371 }
3372 break;
3373 }
3374
3375 case ParameterABI::SwiftErrorResult:
3376 Attrs.addAttribute(Val: llvm::Attribute::SwiftError);
3377 break;
3378
3379 case ParameterABI::SwiftContext:
3380 Attrs.addAttribute(Val: llvm::Attribute::SwiftSelf);
3381 break;
3382
3383 case ParameterABI::SwiftAsyncContext:
3384 Attrs.addAttribute(Val: llvm::Attribute::SwiftAsync);
3385 break;
3386 }
3387
3388 if (FI.getExtParameterInfo(argIndex: ArgNo).isNoEscape())
3389 Attrs.addCapturesAttr(
3390 CI: llvm::CaptureInfo(llvm::CaptureComponents::Address));
3391
3392 if (Attrs.hasAttributes()) {
3393 unsigned FirstIRArg, NumIRArgs;
3394 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3395 for (unsigned i = 0; i < NumIRArgs; i++)
3396 ArgAttrs[FirstIRArg + i].merge(B: Attrs);
3397 }
3398 }
3399 assert(ArgNo == FI.arg_size());
3400
3401 // We can't see all potential arguments in a varargs declaration; treat them
3402 // as if they can access memory.
3403 if (!AttrOnCallSite && FI.isVariadic())
3404 AddPotentialArgAccess();
3405
3406 ArgNo = 0;
3407 if (AddedPotentialArgAccess && MemAttrForPtrArgs) {
3408 llvm::FunctionType *FunctionType = getTypes().GetFunctionType(FI);
3409 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
3410 E = FI.arg_end();
3411 I != E; ++I, ++ArgNo) {
3412 if (I->info.isDirect() || I->info.isExpand() ||
3413 I->info.isCoerceAndExpand()) {
3414 unsigned FirstIRArg, NumIRArgs;
3415 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3416 for (unsigned i = FirstIRArg; i < FirstIRArg + NumIRArgs; ++i) {
3417 // The index may be out-of-bounds if the callee is a varargs
3418 // function.
3419 //
3420 // FIXME: We can compute the types of varargs arguments without going
3421 // through the function type, but the relevant code isn't exposed
3422 // in a way that can be called from here.
3423 if (i < FunctionType->getNumParams() &&
3424 FunctionType->getParamType(i)->isPointerTy()) {
3425 ArgAttrs[i].addAttribute(Val: *MemAttrForPtrArgs);
3426 }
3427 }
3428 }
3429 }
3430 }
3431
3432 SmallVector<llvm::AttributeSet, 4> ArgAttrSets;
3433 for (const llvm::AttrBuilder &Attrs : ArgAttrs)
3434 ArgAttrSets.push_back(Elt: llvm::AttributeSet::get(C&: getLLVMContext(), B: Attrs));
3435
3436 AttrList = llvm::AttributeList::get(
3437 C&: getLLVMContext(), FnAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: FuncAttrs),
3438 RetAttrs: llvm::AttributeSet::get(C&: getLLVMContext(), B: RetAttrs), ArgAttrs: ArgAttrSets);
3439}
3440
3441/// An argument came in as a promoted argument; demote it back to its
3442/// declared type.
3443static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
3444 const VarDecl *var,
3445 llvm::Value *value) {
3446 llvm::Type *varType = CGF.ConvertType(T: var->getType());
3447
3448 // This can happen with promotions that actually don't change the
3449 // underlying type, like the enum promotions.
3450 if (value->getType() == varType)
3451 return value;
3452
3453 assert((varType->isIntegerTy() || varType->isFloatingPointTy()) &&
3454 "unexpected promotion type");
3455
3456 if (isa<llvm::IntegerType>(Val: varType))
3457 return CGF.Builder.CreateTrunc(V: value, DestTy: varType, Name: "arg.unpromote");
3458
3459 return CGF.Builder.CreateFPCast(V: value, DestTy: varType, Name: "arg.unpromote");
3460}
3461
3462/// Returns the attribute (either parameter attribute, or function
3463/// attribute), which declares argument ArgNo to be non-null.
3464static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
3465 QualType ArgType, unsigned ArgNo) {
3466 // FIXME: __attribute__((nonnull)) can also be applied to:
3467 // - references to pointers, where the pointee is known to be
3468 // nonnull (apparently a Clang extension)
3469 // - transparent unions containing pointers
3470 // In the former case, LLVM IR cannot represent the constraint. In
3471 // the latter case, we have no guarantee that the transparent union
3472 // is in fact passed as a pointer.
3473 if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType())
3474 return nullptr;
3475 // First, check attribute on parameter itself.
3476 if (PVD) {
3477 if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
3478 return ParmNNAttr;
3479 }
3480 // Check function attributes.
3481 if (!FD)
3482 return nullptr;
3483 for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
3484 if (NNAttr->isNonNull(IdxAST: ArgNo))
3485 return NNAttr;
3486 }
3487 return nullptr;
3488}
3489
3490namespace {
3491struct CopyBackSwiftError final : EHScopeStack::Cleanup {
3492 Address Temp;
3493 Address Arg;
3494 CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
3495 void Emit(CodeGenFunction &CGF, Flags flags) override {
3496 llvm::Value *errorValue = CGF.Builder.CreateLoad(Addr: Temp);
3497 CGF.Builder.CreateStore(Val: errorValue, Addr: Arg);
3498 }
3499};
3500} // namespace
3501
3502void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
3503 llvm::Function *Fn,
3504 const FunctionArgList &Args) {
3505 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())
3506 // Naked functions don't have prologues.
3507 return;
3508
3509 // If this is an implicit-return-zero function, go ahead and
3510 // initialize the return value. TODO: it might be nice to have
3511 // a more general mechanism for this that didn't require synthesized
3512 // return statements.
3513 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl)) {
3514 if (FD->hasImplicitReturnZero()) {
3515 QualType RetTy = FD->getReturnType().getUnqualifiedType();
3516 llvm::Type *LLVMTy = CGM.getTypes().ConvertType(T: RetTy);
3517 llvm::Constant *Zero = llvm::Constant::getNullValue(Ty: LLVMTy);
3518 Builder.CreateStore(Val: Zero, Addr: ReturnValue);
3519 }
3520 }
3521
3522 // FIXME: We no longer need the types from FunctionArgList; lift up and
3523 // simplify.
3524
3525 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
3526 assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
3527
3528 // If we're using inalloca, all the memory arguments are GEPs off of the last
3529 // parameter, which is a pointer to the complete memory area.
3530 Address ArgStruct = Address::invalid();
3531 if (IRFunctionArgs.hasInallocaArg())
3532 ArgStruct = Address(Fn->getArg(i: IRFunctionArgs.getInallocaArgNo()),
3533 FI.getArgStruct(), FI.getArgStructAlignment());
3534
3535 // Name the struct return parameter.
3536 if (IRFunctionArgs.hasSRetArg()) {
3537 auto AI = Fn->getArg(i: IRFunctionArgs.getSRetArgNo());
3538 AI->setName("agg.result");
3539 AI->addAttr(Kind: llvm::Attribute::NoAlias);
3540 }
3541
3542 // Track if we received the parameter as a pointer (indirect, byval, or
3543 // inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it
3544 // into a local alloca for us.
3545 SmallVector<ParamValue, 16> ArgVals;
3546 ArgVals.reserve(N: Args.size());
3547
3548 // Create a pointer value for every parameter declaration. This usually
3549 // entails copying one or more LLVM IR arguments into an alloca. Don't push
3550 // any cleanups or do anything that might unwind. We do that separately, so
3551 // we can push the cleanups in the correct order for the ABI.
3552 assert(FI.arg_size() == Args.size() &&
3553 "Mismatch between function signature & arguments.");
3554 unsigned ArgNo = 0;
3555 CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
3556 for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end(); i != e;
3557 ++i, ++info_it, ++ArgNo) {
3558 const VarDecl *Arg = *i;
3559 const ABIArgInfo &ArgI = info_it->info;
3560
3561 bool isPromoted =
3562 isa<ParmVarDecl>(Val: Arg) && cast<ParmVarDecl>(Val: Arg)->isKNRPromoted();
3563 // We are converting from ABIArgInfo type to VarDecl type directly, unless
3564 // the parameter is promoted. In this case we convert to
3565 // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
3566 QualType Ty = isPromoted ? info_it->type : Arg->getType();
3567 assert(hasScalarEvaluationKind(Ty) ==
3568 hasScalarEvaluationKind(Arg->getType()));
3569
3570 unsigned FirstIRArg, NumIRArgs;
3571 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3572
3573 switch (ArgI.getKind()) {
3574 case ABIArgInfo::InAlloca: {
3575 assert(NumIRArgs == 0);
3576 auto FieldIndex = ArgI.getInAllocaFieldIndex();
3577 Address V =
3578 Builder.CreateStructGEP(Addr: ArgStruct, Index: FieldIndex, Name: Arg->getName());
3579 if (ArgI.getInAllocaIndirect())
3580 V = Address(Builder.CreateLoad(Addr: V), ConvertTypeForMem(T: Ty),
3581 getContext().getTypeAlignInChars(T: Ty));
3582 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: V));
3583 break;
3584 }
3585
3586 case ABIArgInfo::Indirect:
3587 case ABIArgInfo::IndirectAliased: {
3588 assert(NumIRArgs == 1);
3589 Address ParamAddr = makeNaturalAddressForPointer(
3590 Ptr: Fn->getArg(i: FirstIRArg), T: Ty, Alignment: ArgI.getIndirectAlign(), ForPointeeType: false, BaseInfo: nullptr,
3591 TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
3592
3593 if (!hasScalarEvaluationKind(T: Ty)) {
3594 // Aggregates and complex variables are accessed by reference. All we
3595 // need to do is realign the value, if requested. Also, if the address
3596 // may be aliased, copy it to ensure that the parameter variable is
3597 // mutable and has a unique adress, as C requires.
3598 if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) {
3599 RawAddress AlignedTemp = CreateMemTempWithoutCast(T: Ty, Name: "coerce");
3600
3601 // Copy from the incoming argument pointer to the temporary with the
3602 // appropriate alignment.
3603 //
3604 // FIXME: We should have a common utility for generating an aggregate
3605 // copy.
3606 CharUnits Size = getContext().getTypeSizeInChars(T: Ty);
3607 Builder.CreateMemCpy(
3608 Dst: AlignedTemp.getPointer(), DstAlign: AlignedTemp.getAlignment().getAsAlign(),
3609 Src: ParamAddr.emitRawPointer(CGF&: *this),
3610 SrcAlign: ParamAddr.getAlignment().getAsAlign(),
3611 Size: llvm::ConstantInt::get(Ty: IntPtrTy, V: Size.getQuantity()));
3612 ParamAddr = AlignedTemp;
3613 }
3614 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: ParamAddr));
3615 } else {
3616 // Load scalar value from indirect argument.
3617 llvm::Value *V =
3618 EmitLoadOfScalar(Addr: ParamAddr, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3619
3620 if (isPromoted)
3621 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3622 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3623 }
3624 break;
3625 }
3626
3627 case ABIArgInfo::Extend:
3628 case ABIArgInfo::Direct: {
3629 auto AI = Fn->getArg(i: FirstIRArg);
3630 llvm::Type *LTy = ConvertType(T: Arg->getType());
3631
3632 // Prepare parameter attributes. So far, only attributes for pointer
3633 // parameters are prepared. See
3634 // http://llvm.org/docs/LangRef.html#paramattrs.
3635 if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy() &&
3636 ArgI.getCoerceToType()->isPointerTy()) {
3637 assert(NumIRArgs == 1);
3638
3639 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: Arg)) {
3640 // Set `nonnull` attribute if any.
3641 if (getNonNullAttr(FD: CurCodeDecl, PVD, ArgType: PVD->getType(),
3642 ArgNo: PVD->getFunctionScopeIndex()) &&
3643 !CGM.getCodeGenOpts().NullPointerIsValid)
3644 AI->addAttr(Kind: llvm::Attribute::NonNull);
3645
3646 QualType OTy = PVD->getOriginalType();
3647 if (const auto *ArrTy = getContext().getAsConstantArrayType(T: OTy)) {
3648 // A C99 array parameter declaration with the static keyword also
3649 // indicates dereferenceability, and if the size is constant we can
3650 // use the dereferenceable attribute (which requires the size in
3651 // bytes).
3652 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3653 QualType ETy = ArrTy->getElementType();
3654 llvm::Align Alignment =
3655 CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3656 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3657 .addAlignmentAttr(Align: Alignment));
3658 uint64_t ArrSize = ArrTy->getZExtSize();
3659 if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
3660 ArrSize) {
3661 llvm::AttrBuilder Attrs(getLLVMContext());
3662 Attrs.addDereferenceableAttr(
3663 Bytes: getContext().getTypeSizeInChars(T: ETy).getQuantity() *
3664 ArrSize);
3665 AI->addAttrs(B&: Attrs);
3666 } else if (getContext().getTargetInfo().getNullPointerValue(
3667 AddrSpace: ETy.getAddressSpace()) == 0 &&
3668 !CGM.getCodeGenOpts().NullPointerIsValid) {
3669 AI->addAttr(Kind: llvm::Attribute::NonNull);
3670 }
3671 }
3672 } else if (const auto *ArrTy =
3673 getContext().getAsVariableArrayType(T: OTy)) {
3674 // For C99 VLAs with the static keyword, we don't know the size so
3675 // we can't use the dereferenceable attribute, but in addrspace(0)
3676 // we know that it must be nonnull.
3677 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3678 QualType ETy = ArrTy->getElementType();
3679 llvm::Align Alignment =
3680 CGM.getNaturalTypeAlignment(T: ETy).getAsAlign();
3681 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3682 .addAlignmentAttr(Align: Alignment));
3683 if (!getTypes().getTargetAddressSpace(T: ETy) &&
3684 !CGM.getCodeGenOpts().NullPointerIsValid)
3685 AI->addAttr(Kind: llvm::Attribute::NonNull);
3686 }
3687 }
3688
3689 // Set `align` attribute if any.
3690 const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
3691 if (!AVAttr)
3692 if (const auto *TOTy = OTy->getAs<TypedefType>())
3693 AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
3694 if (AVAttr && !SanOpts.has(K: SanitizerKind::Alignment)) {
3695 // If alignment-assumption sanitizer is enabled, we do *not* add
3696 // alignment attribute here, but emit normal alignment assumption,
3697 // so the UBSAN check could function.
3698 llvm::ConstantInt *AlignmentCI =
3699 cast<llvm::ConstantInt>(Val: EmitScalarExpr(E: AVAttr->getAlignment()));
3700 uint64_t AlignmentInt =
3701 AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment);
3702 if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
3703 AI->removeAttr(Kind: llvm::Attribute::AttrKind::Alignment);
3704 AI->addAttrs(B&: llvm::AttrBuilder(getLLVMContext())
3705 .addAlignmentAttr(Align: llvm::Align(AlignmentInt)));
3706 }
3707 }
3708 }
3709
3710 // Set 'noalias' if an argument type has the `restrict` qualifier.
3711 if (Arg->getType().isRestrictQualified())
3712 AI->addAttr(Kind: llvm::Attribute::NoAlias);
3713 }
3714
3715 // Prepare the argument value. If we have the trivial case, handle it
3716 // with no muss and fuss.
3717 if (!isa<llvm::StructType>(Val: ArgI.getCoerceToType()) &&
3718 ArgI.getCoerceToType() == ConvertType(T: Ty) &&
3719 ArgI.getDirectOffset() == 0) {
3720 assert(NumIRArgs == 1);
3721
3722 // LLVM expects swifterror parameters to be used in very restricted
3723 // ways. Copy the value into a less-restricted temporary.
3724 llvm::Value *V = AI;
3725 if (FI.getExtParameterInfo(argIndex: ArgNo).getABI() ==
3726 ParameterABI::SwiftErrorResult) {
3727 QualType pointeeTy = Ty->getPointeeType();
3728 assert(pointeeTy->isPointerType());
3729 RawAddress temp = CreateMemTempWithoutCast(
3730 T: pointeeTy, Align: getPointerAlign(), Name: "swifterror.temp");
3731 Address arg = makeNaturalAddressForPointer(
3732 Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
3733 llvm::Value *incomingErrorValue = Builder.CreateLoad(Addr: arg);
3734 Builder.CreateStore(Val: incomingErrorValue, Addr: temp);
3735 V = temp.getPointer();
3736
3737 // Push a cleanup to copy the value back at the end of the function.
3738 // The convention does not guarantee that the value will be written
3739 // back if the function exits with an unwind exception.
3740 EHStack.pushCleanup<CopyBackSwiftError>(Kind: NormalCleanup, A: temp, A: arg);
3741 }
3742
3743 // Ensure the argument is the correct type.
3744 if (V->getType() != ArgI.getCoerceToType())
3745 V = Builder.CreateBitCast(V, DestTy: ArgI.getCoerceToType());
3746
3747 if (isPromoted)
3748 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3749
3750 // Because of merging of function types from multiple decls it is
3751 // possible for the type of an argument to not match the corresponding
3752 // type in the function type. Since we are codegening the callee
3753 // in here, add a cast to the argument type.
3754 llvm::Type *LTy = ConvertType(T: Arg->getType());
3755 if (V->getType() != LTy)
3756 V = Builder.CreateBitCast(V, DestTy: LTy);
3757
3758 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3759 break;
3760 }
3761
3762 // VLST arguments are coerced to VLATs at the function boundary for
3763 // ABI consistency. If this is a VLST that was coerced to
3764 // a VLAT at the function boundary and the types match up, use
3765 // llvm.vector.extract to convert back to the original VLST.
3766 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: ConvertType(T: Ty))) {
3767 llvm::Value *ArgVal = Fn->getArg(i: FirstIRArg);
3768 if (auto *VecTyFrom =
3769 dyn_cast<llvm::ScalableVectorType>(Val: ArgVal->getType())) {
3770 auto [Coerced, Extracted] = CoerceScalableToFixed(
3771 CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: ArgVal, Name: Arg->getName());
3772 if (Extracted) {
3773 assert(NumIRArgs == 1);
3774 ArgVals.push_back(Elt: ParamValue::forDirect(value: Coerced));
3775 break;
3776 }
3777 }
3778 }
3779
3780 llvm::StructType *STy =
3781 dyn_cast<llvm::StructType>(Val: ArgI.getCoerceToType());
3782 Address Alloca = CreateMemTempWithoutCast(
3783 T: Ty, Align: getContext().getDeclAlign(D: Arg), Name: Arg->getName());
3784
3785 // Pointer to store into.
3786 Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3787
3788 // Fast-isel and the optimizer generally like scalar values better than
3789 // FCAs, so we flatten them if this is safe to do for this argument.
3790 if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
3791 STy->getNumElements() > 1) {
3792 llvm::TypeSize StructSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
3793 llvm::TypeSize PtrElementSize =
3794 CGM.getDataLayout().getTypeAllocSize(Ty: Ptr.getElementType());
3795 if (StructSize.isScalable()) {
3796 assert(STy->containsHomogeneousScalableVectorTypes() &&
3797 "ABI only supports structure with homogeneous scalable vector "
3798 "type");
3799 assert(StructSize == PtrElementSize &&
3800 "Only allow non-fractional movement of structure with"
3801 "homogeneous scalable vector type");
3802 assert(STy->getNumElements() == NumIRArgs);
3803
3804 llvm::Value *LoadedStructValue = llvm::PoisonValue::get(T: STy);
3805 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3806 auto *AI = Fn->getArg(i: FirstIRArg + i);
3807 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3808 LoadedStructValue =
3809 Builder.CreateInsertValue(Agg: LoadedStructValue, Val: AI, Idxs: i);
3810 }
3811
3812 Builder.CreateStore(Val: LoadedStructValue, Addr: Ptr);
3813 } else {
3814 uint64_t SrcSize = StructSize.getFixedValue();
3815 uint64_t DstSize = PtrElementSize.getFixedValue();
3816
3817 Address AddrToStoreInto = Address::invalid();
3818 if (SrcSize <= DstSize) {
3819 AddrToStoreInto = Ptr.withElementType(ElemTy: STy);
3820 } else {
3821 AddrToStoreInto =
3822 CreateTempAlloca(Ty: STy, align: Alloca.getAlignment(), Name: "coerce");
3823 }
3824
3825 assert(STy->getNumElements() == NumIRArgs);
3826 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3827 auto AI = Fn->getArg(i: FirstIRArg + i);
3828 AI->setName(Arg->getName() + ".coerce" + Twine(i));
3829 Address EltPtr = Builder.CreateStructGEP(Addr: AddrToStoreInto, Index: i);
3830 Builder.CreateStore(Val: AI, Addr: EltPtr);
3831 }
3832
3833 if (SrcSize > DstSize) {
3834 Builder.CreateMemCpy(Dest: Ptr, Src: AddrToStoreInto, Size: DstSize);
3835 }
3836
3837 // Structures with PFP fields require a coerced store to add any
3838 // pointer signatures.
3839 if (getContext().hasPFPFields(Ty)) {
3840 llvm::Value *Struct = Builder.CreateLoad(Addr: Ptr);
3841 CreatePFPCoercedStore(Src: Struct, SrcFETy: Ty, Dst: Ptr, CGF&: *this);
3842 }
3843 }
3844 } else {
3845 // Simple case, just do a coerced store of the argument into the alloca.
3846 assert(NumIRArgs == 1);
3847 auto AI = Fn->getArg(i: FirstIRArg);
3848 AI->setName(Arg->getName() + ".coerce");
3849 CreateCoercedStore(
3850 Src: AI, SrcFETy: Ty, Dst: Ptr,
3851 DstSize: llvm::TypeSize::getFixed(
3852 ExactSize: getContext().getTypeSizeInChars(T: Ty).getQuantity() -
3853 ArgI.getDirectOffset()),
3854 /*DstIsVolatile=*/false);
3855 }
3856
3857 // Match to what EmitParmDecl is expecting for this type.
3858 if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3859 llvm::Value *V =
3860 EmitLoadOfScalar(Addr: Alloca, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3861 if (isPromoted)
3862 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3863 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3864 } else {
3865 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3866 }
3867 break;
3868 }
3869
3870 case ABIArgInfo::CoerceAndExpand: {
3871 // Reconstruct into a temporary.
3872 Address alloca =
3873 CreateMemTempWithoutCast(T: Ty, Align: getContext().getDeclAlign(D: Arg));
3874 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: alloca));
3875
3876 auto coercionType = ArgI.getCoerceAndExpandType();
3877 auto unpaddedCoercionType = ArgI.getUnpaddedCoerceAndExpandType();
3878 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
3879
3880 alloca = alloca.withElementType(ElemTy: coercionType);
3881
3882 unsigned argIndex = FirstIRArg;
3883 unsigned unpaddedIndex = 0;
3884 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3885 llvm::Type *eltType = coercionType->getElementType(N: i);
3886 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
3887 continue;
3888
3889 auto eltAddr = Builder.CreateStructGEP(Addr: alloca, Index: i);
3890 llvm::Value *elt = Fn->getArg(i: argIndex++);
3891
3892 auto paramType = unpaddedStruct
3893 ? unpaddedStruct->getElementType(N: unpaddedIndex++)
3894 : unpaddedCoercionType;
3895
3896 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(Val: eltType)) {
3897 if (auto *VecTyFrom = dyn_cast<llvm::ScalableVectorType>(Val: paramType)) {
3898 bool Extracted;
3899 std::tie(args&: elt, args&: Extracted) = CoerceScalableToFixed(
3900 CGF&: *this, ToTy: VecTyTo, FromTy: VecTyFrom, V: elt, Name: elt->getName());
3901 assert(Extracted && "Unexpected scalable to fixed vector coercion");
3902 }
3903 }
3904 Builder.CreateStore(Val: elt, Addr: eltAddr);
3905 }
3906 assert(argIndex == FirstIRArg + NumIRArgs);
3907 break;
3908 }
3909
3910 case ABIArgInfo::Expand: {
3911 // If this structure was expanded into multiple arguments then
3912 // we need to create a temporary and reconstruct it from the
3913 // arguments.
3914 Address Alloca =
3915 CreateMemTempWithoutCast(T: Ty, Align: getContext().getDeclAlign(D: Arg));
3916 LValue LV = MakeAddrLValue(Addr: Alloca, T: Ty);
3917 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3918
3919 auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3920 ExpandTypeFromArgs(Ty, LV, AI&: FnArgIter);
3921 assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3922 for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
3923 auto AI = Fn->getArg(i: FirstIRArg + i);
3924 AI->setName(Arg->getName() + "." + Twine(i));
3925 }
3926 break;
3927 }
3928
3929 case ABIArgInfo::TargetSpecific: {
3930 auto *AI = Fn->getArg(i: FirstIRArg);
3931 AI->setName(Arg->getName() + ".target_coerce");
3932 Address Alloca = CreateMemTempWithoutCast(
3933 T: Ty, Align: getContext().getDeclAlign(D: Arg), Name: Arg->getName());
3934 Address Ptr = emitAddressAtOffset(CGF&: *this, addr: Alloca, info: ArgI);
3935 CGM.getABIInfo().createCoercedStore(Val: AI, DstAddr: Ptr, AI: ArgI, DestIsVolatile: false, CGF&: *this);
3936 if (CodeGenFunction::hasScalarEvaluationKind(T: Ty)) {
3937 llvm::Value *V =
3938 EmitLoadOfScalar(Addr: Alloca, Volatile: false, Ty, Loc: Arg->getBeginLoc());
3939 if (isPromoted) {
3940 V = emitArgumentDemotion(CGF&: *this, var: Arg, value: V);
3941 }
3942 ArgVals.push_back(Elt: ParamValue::forDirect(value: V));
3943 } else {
3944 ArgVals.push_back(Elt: ParamValue::forIndirect(addr: Alloca));
3945 }
3946 break;
3947 }
3948 case ABIArgInfo::Ignore:
3949 assert(NumIRArgs == 0);
3950 // Initialize the local variable appropriately.
3951 if (!hasScalarEvaluationKind(T: Ty)) {
3952 ArgVals.push_back(
3953 Elt: ParamValue::forIndirect(addr: CreateMemTempWithoutCast(T: Ty)));
3954 } else {
3955 llvm::Value *U = llvm::UndefValue::get(T: ConvertType(T: Arg->getType()));
3956 ArgVals.push_back(Elt: ParamValue::forDirect(value: U));
3957 }
3958 break;
3959 }
3960 }
3961
3962 if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3963 for (int I = Args.size() - 1; I >= 0; --I)
3964 EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3965 } else {
3966 for (unsigned I = 0, E = Args.size(); I != E; ++I)
3967 EmitParmDecl(D: *Args[I], Arg: ArgVals[I], ArgNo: I + 1);
3968 }
3969}
3970
3971static void eraseUnusedBitCasts(llvm::Instruction *insn) {
3972 while (insn->use_empty()) {
3973 llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: insn);
3974 if (!bitcast)
3975 return;
3976
3977 // This is "safe" because we would have used a ConstantExpr otherwise.
3978 insn = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
3979 bitcast->eraseFromParent();
3980 }
3981}
3982
3983/// Try to emit a fused autorelease of a return result.
3984static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
3985 llvm::Value *result) {
3986 // We must be immediately followed the cast.
3987 llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3988 if (BB->empty())
3989 return nullptr;
3990 if (&BB->back() != result)
3991 return nullptr;
3992
3993 llvm::Type *resultType = result->getType();
3994
3995 // result is in a BasicBlock and is therefore an Instruction.
3996 llvm::Instruction *generator = cast<llvm::Instruction>(Val: result);
3997
3998 SmallVector<llvm::Instruction *, 4> InstsToKill;
3999
4000 // Look for:
4001 // %generator = bitcast %type1* %generator2 to %type2*
4002 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: generator)) {
4003 // We would have emitted this as a constant if the operand weren't
4004 // an Instruction.
4005 generator = cast<llvm::Instruction>(Val: bitcast->getOperand(i_nocapture: 0));
4006
4007 // Require the generator to be immediately followed by the cast.
4008 if (generator->getNextNode() != bitcast)
4009 return nullptr;
4010
4011 InstsToKill.push_back(Elt: bitcast);
4012 }
4013
4014 // Look for:
4015 // %generator = call i8* @objc_retain(i8* %originalResult)
4016 // or
4017 // %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
4018 llvm::CallInst *call = dyn_cast<llvm::CallInst>(Val: generator);
4019 if (!call)
4020 return nullptr;
4021
4022 bool doRetainAutorelease;
4023
4024 if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
4025 doRetainAutorelease = true;
4026 } else if (call->getCalledOperand() ==
4027 CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
4028 doRetainAutorelease = false;
4029
4030 // If we emitted an assembly marker for this call (and the
4031 // ARCEntrypoints field should have been set if so), go looking
4032 // for that call. If we can't find it, we can't do this
4033 // optimization. But it should always be the immediately previous
4034 // instruction, unless we needed bitcasts around the call.
4035 if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
4036 llvm::Instruction *prev = call->getPrevNode();
4037 assert(prev);
4038 if (isa<llvm::BitCastInst>(Val: prev)) {
4039 prev = prev->getPrevNode();
4040 assert(prev);
4041 }
4042 assert(isa<llvm::CallInst>(prev));
4043 assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
4044 CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
4045 InstsToKill.push_back(Elt: prev);
4046 }
4047 } else {
4048 return nullptr;
4049 }
4050
4051 result = call->getArgOperand(i: 0);
4052 InstsToKill.push_back(Elt: call);
4053
4054 // Keep killing bitcasts, for sanity. Note that we no longer care
4055 // about precise ordering as long as there's exactly one use.
4056 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(Val: result)) {
4057 if (!bitcast->hasOneUse())
4058 break;
4059 InstsToKill.push_back(Elt: bitcast);
4060 result = bitcast->getOperand(i_nocapture: 0);
4061 }
4062
4063 // Delete all the unnecessary instructions, from latest to earliest.
4064 for (auto *I : InstsToKill)
4065 I->eraseFromParent();
4066
4067 // Do the fused retain/autorelease if we were asked to.
4068 if (doRetainAutorelease)
4069 result = CGF.EmitARCRetainAutoreleaseReturnValue(value: result);
4070
4071 // Cast back to the result type.
4072 return CGF.Builder.CreateBitCast(V: result, DestTy: resultType);
4073}
4074
4075/// If this is a +1 of the value of an immutable 'self', remove it.
4076static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
4077 llvm::Value *result) {
4078 // This is only applicable to a method with an immutable 'self'.
4079 const ObjCMethodDecl *method =
4080 dyn_cast_or_null<ObjCMethodDecl>(Val: CGF.CurCodeDecl);
4081 if (!method)
4082 return nullptr;
4083 const VarDecl *self = method->getSelfDecl();
4084 if (!self->getType().isConstQualified())
4085 return nullptr;
4086
4087 // Look for a retain call. Note: stripPointerCasts looks through returned arg
4088 // functions, which would cause us to miss the retain.
4089 llvm::CallInst *retainCall = dyn_cast<llvm::CallInst>(Val: result);
4090 if (!retainCall || retainCall->getCalledOperand() !=
4091 CGF.CGM.getObjCEntrypoints().objc_retain)
4092 return nullptr;
4093
4094 // Look for an ordinary load of 'self'.
4095 llvm::Value *retainedValue = retainCall->getArgOperand(i: 0);
4096 llvm::LoadInst *load =
4097 dyn_cast<llvm::LoadInst>(Val: retainedValue->stripPointerCasts());
4098 if (!load || load->isAtomic() || load->isVolatile() ||
4099 load->getPointerOperand() != CGF.GetAddrOfLocalVar(VD: self).getBasePointer())
4100 return nullptr;
4101
4102 // Okay! Burn it all down. This relies for correctness on the
4103 // assumption that the retain is emitted as part of the return and
4104 // that thereafter everything is used "linearly".
4105 llvm::Type *resultType = result->getType();
4106 eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: result));
4107 assert(retainCall->use_empty());
4108 retainCall->eraseFromParent();
4109 eraseUnusedBitCasts(insn: cast<llvm::Instruction>(Val: retainedValue));
4110
4111 return CGF.Builder.CreateBitCast(V: load, DestTy: resultType);
4112}
4113
4114/// Emit an ARC autorelease of the result of a function.
4115///
4116/// \return the value to actually return from the function
4117static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
4118 llvm::Value *result) {
4119 // If we're returning 'self', kill the initial retain. This is a
4120 // heuristic attempt to "encourage correctness" in the really unfortunate
4121 // case where we have a return of self during a dealloc and we desperately
4122 // need to avoid the possible autorelease.
4123 if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
4124 return self;
4125
4126 // At -O0, try to emit a fused retain/autorelease.
4127 if (CGF.shouldUseFusedARCCalls())
4128 if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
4129 return fused;
4130
4131 return CGF.EmitARCAutoreleaseReturnValue(value: result);
4132}
4133
4134/// Heuristically search for a dominating store to the return-value slot.
4135static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
4136 llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer();
4137
4138 // Check if a User is a store which pointerOperand is the ReturnValue.
4139 // We are looking for stores to the ReturnValue, not for stores of the
4140 // ReturnValue to some other location.
4141 auto GetStoreIfValid = [&CGF,
4142 ReturnValuePtr](llvm::User *U) -> llvm::StoreInst * {
4143 auto *SI = dyn_cast<llvm::StoreInst>(Val: U);
4144 if (!SI || SI->getPointerOperand() != ReturnValuePtr ||
4145 SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())
4146 return nullptr;
4147 // These aren't actually possible for non-coerced returns, and we
4148 // only care about non-coerced returns on this code path.
4149 // All memory instructions inside __try block are volatile.
4150 assert(!SI->isAtomic() &&
4151 (!SI->isVolatile() || CGF.currentFunctionUsesSEHTry()));
4152 return SI;
4153 };
4154 // If there are multiple uses of the return-value slot, just check
4155 // for something immediately preceding the IP. Sometimes this can
4156 // happen with how we generate implicit-returns; it can also happen
4157 // with noreturn cleanups.
4158 if (!ReturnValuePtr->hasOneUse()) {
4159 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
4160 if (IP->empty())
4161 return nullptr;
4162
4163 // Look at directly preceding instruction, skipping bitcasts, lifetime
4164 // markers, and fake uses and their operands.
4165 const llvm::Instruction *LoadIntoFakeUse = nullptr;
4166 for (llvm::Instruction &I : llvm::reverse(C&: *IP)) {
4167 // Ignore instructions that are just loads for fake uses; the load should
4168 // immediately precede the fake use, so we only need to remember the
4169 // operand for the last fake use seen.
4170 if (LoadIntoFakeUse == &I)
4171 continue;
4172 if (isa<llvm::BitCastInst>(Val: &I))
4173 continue;
4174 if (auto *II = dyn_cast<llvm::IntrinsicInst>(Val: &I)) {
4175 if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
4176 continue;
4177
4178 if (II->getIntrinsicID() == llvm::Intrinsic::fake_use) {
4179 LoadIntoFakeUse = dyn_cast<llvm::Instruction>(Val: II->getArgOperand(i: 0));
4180 continue;
4181 }
4182 }
4183 return GetStoreIfValid(&I);
4184 }
4185 return nullptr;
4186 }
4187
4188 llvm::StoreInst *store = GetStoreIfValid(ReturnValuePtr->user_back());
4189 if (!store)
4190 return nullptr;
4191
4192 // Now do a first-and-dirty dominance check: just walk up the
4193 // single-predecessors chain from the current insertion point.
4194 llvm::BasicBlock *StoreBB = store->getParent();
4195 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
4196 llvm::SmallPtrSet<llvm::BasicBlock *, 4> SeenBBs;
4197 while (IP != StoreBB) {
4198 if (!SeenBBs.insert(Ptr: IP).second || !(IP = IP->getSinglePredecessor()))
4199 return nullptr;
4200 }
4201
4202 // Okay, the store's basic block dominates the insertion point; we
4203 // can do our thing.
4204 return store;
4205}
4206
4207// Helper functions for EmitCMSEClearRecord
4208
4209// Set the bits corresponding to a field having width `BitWidth` and located at
4210// offset `BitOffset` (from the least significant bit) within a storage unit of
4211// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
4212// Use little-endian layout, i.e.`Bits[0]` is the LSB.
4213static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
4214 int BitWidth, int CharWidth) {
4215 assert(CharWidth <= 64);
4216 assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
4217
4218 int Pos = 0;
4219 if (BitOffset >= CharWidth) {
4220 Pos += BitOffset / CharWidth;
4221 BitOffset = BitOffset % CharWidth;
4222 }
4223
4224 const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
4225 if (BitOffset + BitWidth >= CharWidth) {
4226 Bits[Pos++] |= (Used << BitOffset) & Used;
4227 BitWidth -= CharWidth - BitOffset;
4228 BitOffset = 0;
4229 }
4230
4231 while (BitWidth >= CharWidth) {
4232 Bits[Pos++] = Used;
4233 BitWidth -= CharWidth;
4234 }
4235
4236 if (BitWidth > 0)
4237 Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
4238}
4239
4240// Set the bits corresponding to a field having width `BitWidth` and located at
4241// offset `BitOffset` (from the least significant bit) within a storage unit of
4242// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
4243// `Bits` corresponds to one target byte. Use target endian layout.
4244static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
4245 int StorageSize, int BitOffset, int BitWidth,
4246 int CharWidth, bool BigEndian) {
4247
4248 SmallVector<uint64_t, 8> TmpBits(StorageSize);
4249 setBitRange(Bits&: TmpBits, BitOffset, BitWidth, CharWidth);
4250
4251 if (BigEndian)
4252 std::reverse(first: TmpBits.begin(), last: TmpBits.end());
4253
4254 for (uint64_t V : TmpBits)
4255 Bits[StorageOffset++] |= V;
4256}
4257
4258static void setUsedBits(CodeGenModule &, QualType, int,
4259 SmallVectorImpl<uint64_t> &);
4260
4261// Set the bits in `Bits`, which correspond to the value representations of
4262// the actual members of the record type `RTy`. Note that this function does
4263// not handle base classes, virtual tables, etc, since they cannot happen in
4264// CMSE function arguments or return. The bit mask corresponds to the target
4265// memory layout, i.e. it's endian dependent.
4266static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
4267 SmallVectorImpl<uint64_t> &Bits) {
4268 ASTContext &Context = CGM.getContext();
4269 int CharWidth = Context.getCharWidth();
4270 const RecordDecl *RD = RTy->getDecl()->getDefinition();
4271 const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(D: RD);
4272 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
4273
4274 int Idx = 0;
4275 for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) {
4276 const FieldDecl *F = *I;
4277
4278 if (F->isUnnamedBitField() || F->isZeroLengthBitField() ||
4279 F->getType()->isIncompleteArrayType())
4280 continue;
4281
4282 if (F->isBitField()) {
4283 const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(FD: F);
4284 setBitRange(Bits, StorageOffset: Offset + BFI.StorageOffset.getQuantity(),
4285 StorageSize: BFI.StorageSize / CharWidth, BitOffset: BFI.Offset, BitWidth: BFI.Size, CharWidth,
4286 BigEndian: CGM.getDataLayout().isBigEndian());
4287 continue;
4288 }
4289
4290 setUsedBits(CGM, F->getType(),
4291 Offset + ASTLayout.getFieldOffset(FieldNo: Idx) / CharWidth, Bits);
4292 }
4293}
4294
4295// Set the bits in `Bits`, which correspond to the value representations of
4296// the elements of an array type `ATy`.
4297static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
4298 int Offset, SmallVectorImpl<uint64_t> &Bits) {
4299 const ASTContext &Context = CGM.getContext();
4300
4301 QualType ETy = Context.getBaseElementType(VAT: ATy);
4302 int Size = Context.getTypeSizeInChars(T: ETy).getQuantity();
4303 SmallVector<uint64_t, 4> TmpBits(Size);
4304 setUsedBits(CGM, ETy, 0, TmpBits);
4305
4306 for (int I = 0, N = Context.getConstantArrayElementCount(CA: ATy); I < N; ++I) {
4307 auto Src = TmpBits.begin();
4308 auto Dst = Bits.begin() + Offset + I * Size;
4309 for (int J = 0; J < Size; ++J)
4310 *Dst++ |= *Src++;
4311 }
4312}
4313
4314// Set the bits in `Bits`, which correspond to the value representations of
4315// the type `QTy`.
4316static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
4317 SmallVectorImpl<uint64_t> &Bits) {
4318 if (const auto *RTy = QTy->getAsCanonical<RecordType>())
4319 return setUsedBits(CGM, RTy, Offset, Bits);
4320
4321 ASTContext &Context = CGM.getContext();
4322 if (const auto *ATy = Context.getAsConstantArrayType(T: QTy))
4323 return setUsedBits(CGM, ATy, Offset, Bits);
4324
4325 int Size = Context.getTypeSizeInChars(T: QTy).getQuantity();
4326 if (Size <= 0)
4327 return;
4328
4329 std::fill_n(first: Bits.begin() + Offset, n: Size,
4330 value: (uint64_t(1) << Context.getCharWidth()) - 1);
4331}
4332
4333static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
4334 int Pos, int Size, int CharWidth,
4335 bool BigEndian) {
4336 assert(Size > 0);
4337 uint64_t Mask = 0;
4338 if (BigEndian) {
4339 for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
4340 ++P)
4341 Mask = (Mask << CharWidth) | *P;
4342 } else {
4343 auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
4344 do
4345 Mask = (Mask << CharWidth) | *--P;
4346 while (P != End);
4347 }
4348 return Mask;
4349}
4350
4351// Emit code to clear the bits in a record, which aren't a part of any user
4352// declared member, when the record is a function return.
4353llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
4354 llvm::IntegerType *ITy,
4355 QualType QTy) {
4356 assert(Src->getType() == ITy);
4357 assert(ITy->getScalarSizeInBits() <= 64);
4358
4359 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
4360 int Size = DataLayout.getTypeStoreSize(Ty: ITy);
4361 SmallVector<uint64_t, 4> Bits(Size);
4362 setUsedBits(CGM, RTy: QTy->castAsCanonical<RecordType>(), Offset: 0, Bits);
4363
4364 int CharWidth = CGM.getContext().getCharWidth();
4365 uint64_t Mask =
4366 buildMultiCharMask(Bits, Pos: 0, Size, CharWidth, BigEndian: DataLayout.isBigEndian());
4367
4368 return Builder.CreateAnd(LHS: Src, RHS: Mask, Name: "cmse.clear");
4369}
4370
4371// Emit code to clear the bits in a record, which aren't a part of any user
4372// declared member, when the record is a function argument.
4373llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
4374 llvm::ArrayType *ATy,
4375 QualType QTy) {
4376 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
4377 int Size = DataLayout.getTypeStoreSize(Ty: ATy);
4378 SmallVector<uint64_t, 16> Bits(Size);
4379 setUsedBits(CGM, RTy: QTy->castAsCanonical<RecordType>(), Offset: 0, Bits);
4380
4381 // Clear each element of the LLVM array.
4382 int CharWidth = CGM.getContext().getCharWidth();
4383 int CharsPerElt =
4384 ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
4385 int MaskIndex = 0;
4386 llvm::Value *R = llvm::PoisonValue::get(T: ATy);
4387 for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
4388 uint64_t Mask = buildMultiCharMask(Bits, Pos: MaskIndex, Size: CharsPerElt, CharWidth,
4389 BigEndian: DataLayout.isBigEndian());
4390 MaskIndex += CharsPerElt;
4391 llvm::Value *T0 = Builder.CreateExtractValue(Agg: Src, Idxs: I);
4392 llvm::Value *T1 = Builder.CreateAnd(LHS: T0, RHS: Mask, Name: "cmse.clear");
4393 R = Builder.CreateInsertValue(Agg: R, Val: T1, Idxs: I);
4394 }
4395
4396 return R;
4397}
4398
4399void CodeGenFunction::EmitFunctionEpilog(
4400 const CGFunctionInfo &FI, bool EmitRetDbgLoc, SourceLocation EndLoc,
4401 uint64_t RetKeyInstructionsSourceAtom) {
4402 if (FI.isNoReturn()) {
4403 // Noreturn functions don't return.
4404 EmitUnreachable(Loc: EndLoc);
4405 return;
4406 }
4407
4408 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {
4409 // Naked functions don't have epilogues.
4410 Builder.CreateUnreachable();
4411 return;
4412 }
4413
4414 // Functions with no result always return void.
4415 if (!ReturnValue.isValid()) {
4416 auto *I = Builder.CreateRetVoid();
4417 if (RetKeyInstructionsSourceAtom)
4418 addInstToSpecificSourceAtom(KeyInstruction: I, Backup: nullptr, Atom: RetKeyInstructionsSourceAtom);
4419 else
4420 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4421 return;
4422 }
4423
4424 llvm::DebugLoc RetDbgLoc;
4425 llvm::Value *RV = nullptr;
4426 QualType RetTy = FI.getReturnType();
4427 const ABIArgInfo &RetAI = FI.getReturnInfo();
4428
4429 switch (RetAI.getKind()) {
4430 case ABIArgInfo::InAlloca:
4431 // Aggregates get evaluated directly into the destination. Sometimes we
4432 // need to return the sret value in a register, though.
4433 assert(hasAggregateEvaluationKind(RetTy));
4434 if (RetAI.getInAllocaSRet()) {
4435 llvm::Function::arg_iterator EI = CurFn->arg_end();
4436 --EI;
4437 llvm::Value *ArgStruct = &*EI;
4438 llvm::Value *SRet = Builder.CreateStructGEP(
4439 Ty: FI.getArgStruct(), Ptr: ArgStruct, Idx: RetAI.getInAllocaFieldIndex());
4440 llvm::Type *Ty =
4441 cast<llvm::GetElementPtrInst>(Val: SRet)->getResultElementType();
4442 RV = Builder.CreateAlignedLoad(Ty, Addr: SRet, Align: getPointerAlign(), Name: "sret");
4443 }
4444 break;
4445
4446 case ABIArgInfo::Indirect: {
4447 auto AI = CurFn->arg_begin();
4448 if (RetAI.isSRetAfterThis())
4449 ++AI;
4450 switch (getEvaluationKind(T: RetTy)) {
4451 case TEK_Complex: {
4452 ComplexPairTy RT =
4453 EmitLoadOfComplex(src: MakeAddrLValue(Addr: ReturnValue, T: RetTy), loc: EndLoc);
4454 EmitStoreOfComplex(V: RT, dest: MakeNaturalAlignAddrLValue(V: &*AI, T: RetTy),
4455 /*isInit*/ true);
4456 break;
4457 }
4458 case TEK_Aggregate:
4459 // Do nothing; aggregates get evaluated directly into the destination.
4460 break;
4461 case TEK_Scalar: {
4462 LValueBaseInfo BaseInfo;
4463 TBAAAccessInfo TBAAInfo;
4464 CharUnits Alignment =
4465 CGM.getNaturalTypeAlignment(T: RetTy, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
4466 Address ArgAddr(&*AI, ConvertType(T: RetTy), Alignment);
4467 LValue ArgVal =
4468 LValue::MakeAddr(Addr: ArgAddr, type: RetTy, Context&: getContext(), BaseInfo, TBAAInfo);
4469 EmitStoreOfScalar(
4470 value: EmitLoadOfScalar(lvalue: MakeAddrLValue(Addr: ReturnValue, T: RetTy), Loc: EndLoc), lvalue: ArgVal,
4471 /*isInit*/ true);
4472 break;
4473 }
4474 }
4475 break;
4476 }
4477
4478 case ABIArgInfo::Extend:
4479 case ABIArgInfo::Direct:
4480 if (RetAI.getCoerceToType() == ConvertType(T: RetTy) &&
4481 RetAI.getDirectOffset() == 0) {
4482 // The internal return value temp always will have pointer-to-return-type
4483 // type, just do a load.
4484
4485 // If there is a dominating store to ReturnValue, we can elide
4486 // the load, zap the store, and usually zap the alloca.
4487 if (llvm::StoreInst *SI = findDominatingStoreToReturnValue(CGF&: *this)) {
4488 // Reuse the debug location from the store unless there is
4489 // cleanup code to be emitted between the store and return
4490 // instruction.
4491 if (EmitRetDbgLoc && !AutoreleaseResult)
4492 RetDbgLoc = SI->getDebugLoc();
4493 // Get the stored value and nuke the now-dead store.
4494 RV = SI->getValueOperand();
4495 SI->eraseFromParent();
4496
4497 // Otherwise, we have to do a simple load.
4498 } else {
4499 RV = Builder.CreateLoad(Addr: ReturnValue);
4500 }
4501 } else {
4502 // If the value is offset in memory, apply the offset now.
4503 Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
4504
4505 RV = CreateCoercedLoad(Src: V, SrcFETy: RetTy, Ty: RetAI.getCoerceToType(), CGF&: *this);
4506 }
4507
4508 // In ARC, end functions that return a retainable type with a call
4509 // to objc_autoreleaseReturnValue.
4510 if (AutoreleaseResult) {
4511#ifndef NDEBUG
4512 // Type::isObjCRetainabletype has to be called on a QualType that hasn't
4513 // been stripped of the typedefs, so we cannot use RetTy here. Get the
4514 // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
4515 // CurCodeDecl or BlockInfo.
4516 QualType RT;
4517
4518 if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
4519 RT = FD->getReturnType();
4520 else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
4521 RT = MD->getReturnType();
4522 else if (isa<BlockDecl>(CurCodeDecl))
4523 RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
4524 else
4525 llvm_unreachable("Unexpected function/method type");
4526
4527 assert(getLangOpts().ObjCAutoRefCount && !FI.isReturnsRetained() &&
4528 RT->isObjCRetainableType());
4529#endif
4530 RV = emitAutoreleaseOfResult(CGF&: *this, result: RV);
4531 }
4532
4533 break;
4534
4535 case ABIArgInfo::Ignore:
4536 break;
4537
4538 case ABIArgInfo::CoerceAndExpand: {
4539 auto coercionType = RetAI.getCoerceAndExpandType();
4540 auto unpaddedCoercionType = RetAI.getUnpaddedCoerceAndExpandType();
4541 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
4542
4543 // Load all of the coerced elements out into results.
4544 llvm::SmallVector<llvm::Value *, 4> results;
4545 Address addr = ReturnValue.withElementType(ElemTy: coercionType);
4546 unsigned unpaddedIndex = 0;
4547 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
4548 auto coercedEltType = coercionType->getElementType(N: i);
4549 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType: coercedEltType))
4550 continue;
4551
4552 auto eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
4553 llvm::Value *elt = CreateCoercedLoad(
4554 Src: eltAddr, SrcFETy: RetTy,
4555 Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
4556 : unpaddedCoercionType,
4557 CGF&: *this);
4558 results.push_back(Elt: elt);
4559 }
4560
4561 // If we have one result, it's the single direct result type.
4562 if (results.size() == 1) {
4563 RV = results[0];
4564
4565 // Otherwise, we need to make a first-class aggregate.
4566 } else {
4567 // Construct a return type that lacks padding elements.
4568 llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
4569
4570 RV = llvm::PoisonValue::get(T: returnType);
4571 for (unsigned i = 0, e = results.size(); i != e; ++i) {
4572 RV = Builder.CreateInsertValue(Agg: RV, Val: results[i], Idxs: i);
4573 }
4574 }
4575 break;
4576 }
4577 case ABIArgInfo::TargetSpecific: {
4578 Address V = emitAddressAtOffset(CGF&: *this, addr: ReturnValue, info: RetAI);
4579 RV = CGM.getABIInfo().createCoercedLoad(SrcAddr: V, AI: RetAI, CGF&: *this);
4580 break;
4581 }
4582 case ABIArgInfo::Expand:
4583 case ABIArgInfo::IndirectAliased:
4584 llvm_unreachable("Invalid ABI kind for return argument");
4585 }
4586
4587 llvm::Instruction *Ret;
4588 if (RV) {
4589 if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) {
4590 // For certain return types, clear padding bits, as they may reveal
4591 // sensitive information.
4592 // Small struct/union types are passed as integers.
4593 auto *ITy = dyn_cast<llvm::IntegerType>(Val: RV->getType());
4594 if (ITy != nullptr && isa<RecordType>(Val: RetTy.getCanonicalType()))
4595 RV = EmitCMSEClearRecord(Src: RV, ITy, QTy: RetTy);
4596 }
4597 EmitReturnValueCheck(RV);
4598 Ret = Builder.CreateRet(V: RV);
4599 } else {
4600 Ret = Builder.CreateRetVoid();
4601 }
4602
4603 if (RetDbgLoc)
4604 Ret->setDebugLoc(std::move(RetDbgLoc));
4605
4606 llvm::Value *Backup = RV ? Ret->getOperand(i: 0) : nullptr;
4607 if (RetKeyInstructionsSourceAtom)
4608 addInstToSpecificSourceAtom(KeyInstruction: Ret, Backup, Atom: RetKeyInstructionsSourceAtom);
4609 else
4610 addInstToNewSourceAtom(KeyInstruction: Ret, Backup);
4611}
4612
4613void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
4614 // A current decl may not be available when emitting vtable thunks.
4615 if (!CurCodeDecl)
4616 return;
4617
4618 // If the return block isn't reachable, neither is this check, so don't emit
4619 // it.
4620 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty())
4621 return;
4622
4623 if (SanOpts.has(K: SanitizerKind::Alignment)) {
4624 if (const AllocAlignAttr *AA = CurCodeDecl->getAttr<AllocAlignAttr>()) {
4625 if (isa<FunctionDecl>(Val: CurCodeDecl)) {
4626 unsigned PI = AA->getParamIndex().getLLVMIndex();
4627 llvm::Value *Alignment = CurFn->getArg(i: PI);
4628 emitReturnAlignmentAssumptionCheck(Ptr: RV, Ty: FnRetTy, AssumptionLoc: AA->getLocation(),
4629 Alignment);
4630 }
4631 }
4632 if (const AssumeAlignedAttr *AA =
4633 CurCodeDecl->getAttr<AssumeAlignedAttr>()) {
4634 llvm::Value *Alignment = EmitScalarExpr(E: AA->getAlignment());
4635 llvm::ConstantInt *OffsetCI = nullptr;
4636 if (Expr *Offset = AA->getOffset()) {
4637 OffsetCI = cast<llvm::ConstantInt>(Val: EmitScalarExpr(E: Offset));
4638 if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
4639 OffsetCI = nullptr;
4640 }
4641 emitReturnAlignmentAssumptionCheck(Ptr: RV, Ty: FnRetTy, AssumptionLoc: AA->getLocation(),
4642 Alignment, OffsetValue: OffsetCI);
4643 }
4644 }
4645
4646 ReturnsNonNullAttr *RetNNAttr = nullptr;
4647 if (SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute))
4648 RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
4649
4650 if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
4651 return;
4652
4653 // Prefer the returns_nonnull attribute if it's present.
4654 SourceLocation AttrLoc;
4655 SanitizerKind::SanitizerOrdinal CheckKind;
4656 SanitizerHandler Handler;
4657 if (RetNNAttr) {
4658 assert(!requiresReturnValueNullabilityCheck() &&
4659 "Cannot check nullability and the nonnull attribute");
4660 AttrLoc = RetNNAttr->getLocation();
4661 CheckKind = SanitizerKind::SO_ReturnsNonnullAttribute;
4662 Handler = SanitizerHandler::NonnullReturn;
4663 } else {
4664 if (auto *DD = dyn_cast<DeclaratorDecl>(Val: CurCodeDecl))
4665 if (auto *TSI = DD->getTypeSourceInfo())
4666 if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
4667 AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
4668 CheckKind = SanitizerKind::SO_NullabilityReturn;
4669 Handler = SanitizerHandler::NullabilityReturn;
4670 }
4671
4672 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
4673
4674 // Make sure the "return" source location is valid. If we're checking a
4675 // nullability annotation, make sure the preconditions for the check are met.
4676 llvm::BasicBlock *Check = createBasicBlock(name: "nullcheck");
4677 llvm::BasicBlock *NoCheck = createBasicBlock(name: "no.nullcheck");
4678 llvm::Value *SLocPtr = Builder.CreateLoad(Addr: ReturnLocation, Name: "return.sloc.load");
4679 llvm::Value *CanNullCheck = Builder.CreateIsNotNull(Arg: SLocPtr);
4680 if (requiresReturnValueNullabilityCheck())
4681 CanNullCheck =
4682 Builder.CreateAnd(LHS: CanNullCheck, RHS: RetValNullabilityPrecondition);
4683 Builder.CreateCondBr(Cond: CanNullCheck, True: Check, False: NoCheck);
4684 EmitBlock(BB: Check);
4685
4686 // Now do the null check.
4687 llvm::Value *Cond = Builder.CreateIsNotNull(Arg: RV);
4688 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc: AttrLoc)};
4689 llvm::Value *DynamicData[] = {SLocPtr};
4690 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: DynamicData);
4691
4692 EmitBlock(BB: NoCheck);
4693
4694#ifndef NDEBUG
4695 // The return location should not be used after the check has been emitted.
4696 ReturnLocation = Address::invalid();
4697#endif
4698}
4699
4700static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
4701 const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
4702 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
4703}
4704
4705static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, QualType Ty) {
4706 // FIXME: Generate IR in one pass, rather than going back and fixing up these
4707 // placeholders.
4708 llvm::Type *IRTy = CGF.ConvertTypeForMem(T: Ty);
4709 llvm::Type *IRPtrTy = llvm::PointerType::getUnqual(C&: CGF.getLLVMContext());
4710 llvm::Value *Placeholder = llvm::PoisonValue::get(T: IRPtrTy);
4711
4712 // FIXME: When we generate this IR in one pass, we shouldn't need
4713 // this win32-specific alignment hack.
4714 CharUnits Align = CharUnits::fromQuantity(Quantity: 4);
4715 Placeholder = CGF.Builder.CreateAlignedLoad(Ty: IRPtrTy, Addr: Placeholder, Align);
4716
4717 return AggValueSlot::forAddr(
4718 addr: Address(Placeholder, IRTy, Align), quals: Ty.getQualifiers(),
4719 isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
4720 isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
4721}
4722
4723void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
4724 const VarDecl *param,
4725 SourceLocation loc) {
4726 // StartFunction converted the ABI-lowered parameter(s) into a
4727 // local alloca. We need to turn that into an r-value suitable
4728 // for EmitCall.
4729 Address local = GetAddrOfLocalVar(VD: param);
4730
4731 QualType type = param->getType();
4732
4733 // GetAddrOfLocalVar returns a pointer-to-pointer for references,
4734 // but the argument needs to be the original pointer.
4735 if (type->isReferenceType()) {
4736 args.add(rvalue: RValue::get(V: Builder.CreateLoad(Addr: local)), type);
4737
4738 // In ARC, move out of consumed arguments so that the release cleanup
4739 // entered by StartFunction doesn't cause an over-release. This isn't
4740 // optimal -O0 code generation, but it should get cleaned up when
4741 // optimization is enabled. This also assumes that delegate calls are
4742 // performed exactly once for a set of arguments, but that should be safe.
4743 } else if (getLangOpts().ObjCAutoRefCount &&
4744 param->hasAttr<NSConsumedAttr>() && type->isObjCRetainableType()) {
4745 llvm::Value *ptr = Builder.CreateLoad(Addr: local);
4746 auto null =
4747 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: ptr->getType()));
4748 Builder.CreateStore(Val: null, Addr: local);
4749 args.add(rvalue: RValue::get(V: ptr), type);
4750
4751 // For the most part, we just need to load the alloca, except that
4752 // aggregate r-values are actually pointers to temporaries.
4753 } else {
4754 args.add(rvalue: convertTempToRValue(addr: local, type, Loc: loc), type);
4755 }
4756
4757 // Deactivate the cleanup for the callee-destructed param that was pushed.
4758 if (type->isRecordType() && !CurFuncIsThunk &&
4759 type->castAsRecordDecl()->isParamDestroyedInCallee() &&
4760 param->needsDestruction(Ctx: getContext())) {
4761 EHScopeStack::stable_iterator cleanup =
4762 CalleeDestructedParamCleanups.lookup(Val: cast<ParmVarDecl>(Val: param));
4763 assert(cleanup.isValid() &&
4764 "cleanup for callee-destructed param not recorded");
4765 // This load is a temporary marker which will be removed later.
4766 llvm::Instruction *isActive =
4767 Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
4768 args.addArgCleanupDeactivation(Cleanup: cleanup, IsActiveIP: isActive);
4769 }
4770}
4771
4772static bool isProvablyNull(llvm::Value *addr) {
4773 return llvm::isa_and_nonnull<llvm::ConstantPointerNull>(Val: addr);
4774}
4775
4776static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) {
4777 return llvm::isKnownNonZero(V: Addr.getBasePointer(), Q: CGF.CGM.getDataLayout());
4778}
4779
4780/// Emit the actual writing-back of a writeback.
4781static void emitWriteback(CodeGenFunction &CGF,
4782 const CallArgList::Writeback &writeback) {
4783 const LValue &srcLV = writeback.Source;
4784 Address srcAddr = srcLV.getAddress();
4785 assert(!isProvablyNull(srcAddr.getBasePointer()) &&
4786 "shouldn't have writeback for provably null argument");
4787
4788 if (writeback.WritebackExpr) {
4789 CGF.EmitIgnoredExpr(E: writeback.WritebackExpr);
4790 CGF.EmitLifetimeEnd(Addr: writeback.Temporary.getBasePointer());
4791 return;
4792 }
4793
4794 llvm::BasicBlock *contBB = nullptr;
4795
4796 // If the argument wasn't provably non-null, we need to null check
4797 // before doing the store.
4798 bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4799
4800 if (!provablyNonNull) {
4801 llvm::BasicBlock *writebackBB = CGF.createBasicBlock(name: "icr.writeback");
4802 contBB = CGF.createBasicBlock(name: "icr.done");
4803
4804 llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4805 CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: writebackBB);
4806 CGF.EmitBlock(BB: writebackBB);
4807 }
4808
4809 // Load the value to writeback.
4810 llvm::Value *value = CGF.Builder.CreateLoad(Addr: writeback.Temporary);
4811
4812 // Cast it back, in case we're writing an id to a Foo* or something.
4813 value = CGF.Builder.CreateBitCast(V: value, DestTy: srcAddr.getElementType(),
4814 Name: "icr.writeback-cast");
4815
4816 // Perform the writeback.
4817
4818 // If we have a "to use" value, it's something we need to emit a use
4819 // of. This has to be carefully threaded in: if it's done after the
4820 // release it's potentially undefined behavior (and the optimizer
4821 // will ignore it), and if it happens before the retain then the
4822 // optimizer could move the release there.
4823 if (writeback.ToUse) {
4824 assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
4825
4826 // Retain the new value. No need to block-copy here: the block's
4827 // being passed up the stack.
4828 value = CGF.EmitARCRetainNonBlock(value);
4829
4830 // Emit the intrinsic use here.
4831 CGF.EmitARCIntrinsicUse(values: writeback.ToUse);
4832
4833 // Load the old value (primitively).
4834 llvm::Value *oldValue = CGF.EmitLoadOfScalar(lvalue: srcLV, Loc: SourceLocation());
4835
4836 // Put the new value in place (primitively).
4837 CGF.EmitStoreOfScalar(value, lvalue: srcLV, /*init*/ isInit: false);
4838
4839 // Release the old value.
4840 CGF.EmitARCRelease(value: oldValue, precise: srcLV.isARCPreciseLifetime());
4841
4842 // Otherwise, we can just do a normal lvalue store.
4843 } else {
4844 CGF.EmitStoreThroughLValue(Src: RValue::get(V: value), Dst: srcLV);
4845 }
4846
4847 // Jump to the continuation block.
4848 if (!provablyNonNull)
4849 CGF.EmitBlock(BB: contBB);
4850}
4851
4852static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
4853 const CallArgList &CallArgs) {
4854 ArrayRef<CallArgList::CallArgCleanup> Cleanups =
4855 CallArgs.getCleanupsToDeactivate();
4856 // Iterate in reverse to increase the likelihood of popping the cleanup.
4857 for (const auto &I : llvm::reverse(C&: Cleanups)) {
4858 CGF.DeactivateCleanupBlock(Cleanup: I.Cleanup, DominatingIP: I.IsActiveIP);
4859 I.IsActiveIP->eraseFromParent();
4860 }
4861}
4862
4863static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
4864 if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(Val: E->IgnoreParens()))
4865 if (uop->getOpcode() == UO_AddrOf)
4866 return uop->getSubExpr();
4867 return nullptr;
4868}
4869
4870/// Emit an argument that's being passed call-by-writeback. That is,
4871/// we are passing the address of an __autoreleased temporary; it
4872/// might be copy-initialized with the current value of the given
4873/// address, but it will definitely be copied out of after the call.
4874static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
4875 const ObjCIndirectCopyRestoreExpr *CRE) {
4876 LValue srcLV;
4877
4878 // Make an optimistic effort to emit the address as an l-value.
4879 // This can fail if the argument expression is more complicated.
4880 if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(E: CRE->getSubExpr())) {
4881 srcLV = CGF.EmitLValue(E: lvExpr);
4882
4883 // Otherwise, just emit it as a scalar.
4884 } else {
4885 Address srcAddr = CGF.EmitPointerWithAlignment(Addr: CRE->getSubExpr());
4886
4887 QualType srcAddrType =
4888 CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
4889 srcLV = CGF.MakeAddrLValue(Addr: srcAddr, T: srcAddrType);
4890 }
4891 Address srcAddr = srcLV.getAddress();
4892
4893 // The dest and src types don't necessarily match in LLVM terms
4894 // because of the crazy ObjC compatibility rules.
4895
4896 llvm::PointerType *destType =
4897 cast<llvm::PointerType>(Val: CGF.ConvertType(T: CRE->getType()));
4898 llvm::Type *destElemType =
4899 CGF.ConvertTypeForMem(T: CRE->getType()->getPointeeType());
4900
4901 // If the address is a constant null, just pass the appropriate null.
4902 if (isProvablyNull(addr: srcAddr.getBasePointer())) {
4903 args.add(rvalue: RValue::get(V: llvm::ConstantPointerNull::get(T: destType)),
4904 type: CRE->getType());
4905 return;
4906 }
4907
4908 // Create the temporary.
4909 Address temp =
4910 CGF.CreateTempAlloca(Ty: destElemType, align: CGF.getPointerAlign(), Name: "icr.temp");
4911 // Loading an l-value can introduce a cleanup if the l-value is __weak,
4912 // and that cleanup will be conditional if we can't prove that the l-value
4913 // isn't null, so we need to register a dominating point so that the cleanups
4914 // system will make valid IR.
4915 CodeGenFunction::ConditionalEvaluation condEval(CGF);
4916
4917 // Zero-initialize it if we're not doing a copy-initialization.
4918 bool shouldCopy = CRE->shouldCopy();
4919 if (!shouldCopy) {
4920 llvm::Value *null =
4921 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: destElemType));
4922 CGF.Builder.CreateStore(Val: null, Addr: temp);
4923 }
4924
4925 llvm::BasicBlock *contBB = nullptr;
4926 llvm::BasicBlock *originBB = nullptr;
4927
4928 // If the address is *not* known to be non-null, we need to switch.
4929 llvm::Value *finalArgument;
4930
4931 bool provablyNonNull = isProvablyNonNull(Addr: srcAddr, CGF);
4932
4933 if (provablyNonNull) {
4934 finalArgument = temp.emitRawPointer(CGF);
4935 } else {
4936 llvm::Value *isNull = CGF.Builder.CreateIsNull(Addr: srcAddr, Name: "icr.isnull");
4937
4938 finalArgument = CGF.Builder.CreateSelect(
4939 C: isNull, True: llvm::ConstantPointerNull::get(T: destType),
4940 False: temp.emitRawPointer(CGF), Name: "icr.argument");
4941
4942 // If we need to copy, then the load has to be conditional, which
4943 // means we need control flow.
4944 if (shouldCopy) {
4945 originBB = CGF.Builder.GetInsertBlock();
4946 contBB = CGF.createBasicBlock(name: "icr.cont");
4947 llvm::BasicBlock *copyBB = CGF.createBasicBlock(name: "icr.copy");
4948 CGF.Builder.CreateCondBr(Cond: isNull, True: contBB, False: copyBB);
4949 CGF.EmitBlock(BB: copyBB);
4950 condEval.begin(CGF);
4951 }
4952 }
4953
4954 llvm::Value *valueToUse = nullptr;
4955
4956 // Perform a copy if necessary.
4957 if (shouldCopy) {
4958 RValue srcRV = CGF.EmitLoadOfLValue(V: srcLV, Loc: SourceLocation());
4959 assert(srcRV.isScalar());
4960
4961 llvm::Value *src = srcRV.getScalarVal();
4962 src = CGF.Builder.CreateBitCast(V: src, DestTy: destElemType, Name: "icr.cast");
4963
4964 // Use an ordinary store, not a store-to-lvalue.
4965 CGF.Builder.CreateStore(Val: src, Addr: temp);
4966
4967 // If optimization is enabled, and the value was held in a
4968 // __strong variable, we need to tell the optimizer that this
4969 // value has to stay alive until we're doing the store back.
4970 // This is because the temporary is effectively unretained,
4971 // and so otherwise we can violate the high-level semantics.
4972 if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4973 srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {
4974 valueToUse = src;
4975 }
4976 }
4977
4978 // Finish the control flow if we needed it.
4979 if (shouldCopy && !provablyNonNull) {
4980 llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
4981 CGF.EmitBlock(BB: contBB);
4982
4983 // Make a phi for the value to intrinsically use.
4984 if (valueToUse) {
4985 llvm::PHINode *phiToUse =
4986 CGF.Builder.CreatePHI(Ty: valueToUse->getType(), NumReservedValues: 2, Name: "icr.to-use");
4987 phiToUse->addIncoming(V: valueToUse, BB: copyBB);
4988 phiToUse->addIncoming(V: llvm::PoisonValue::get(T: valueToUse->getType()),
4989 BB: originBB);
4990 valueToUse = phiToUse;
4991 }
4992
4993 condEval.end(CGF);
4994 }
4995
4996 args.addWriteback(srcLV, temporary: temp, toUse: valueToUse);
4997 args.add(rvalue: RValue::get(V: finalArgument), type: CRE->getType());
4998}
4999
5000void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
5001 assert(!StackBase);
5002
5003 // Save the stack.
5004 StackBase = CGF.Builder.CreateStackSave(Name: "inalloca.save");
5005}
5006
5007void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
5008 if (StackBase) {
5009 // Restore the stack after the call.
5010 CGF.Builder.CreateStackRestore(Ptr: StackBase);
5011 }
5012}
5013
5014void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
5015 SourceLocation ArgLoc,
5016 AbstractCallee AC, unsigned ParmNum) {
5017 if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
5018 SanOpts.has(K: SanitizerKind::NullabilityArg)))
5019 return;
5020
5021 // The param decl may be missing in a variadic function.
5022 auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(I: ParmNum) : nullptr;
5023 unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
5024
5025 // Prefer the nonnull attribute if it's present.
5026 const NonNullAttr *NNAttr = nullptr;
5027 if (SanOpts.has(K: SanitizerKind::NonnullAttribute))
5028 NNAttr = getNonNullAttr(FD: AC.getDecl(), PVD, ArgType, ArgNo);
5029
5030 bool CanCheckNullability = false;
5031 if (SanOpts.has(K: SanitizerKind::NullabilityArg) && !NNAttr && PVD &&
5032 !PVD->getType()->isRecordType()) {
5033 auto Nullability = PVD->getType()->getNullability();
5034 CanCheckNullability = Nullability &&
5035 *Nullability == NullabilityKind::NonNull &&
5036 PVD->getTypeSourceInfo();
5037 }
5038
5039 if (!NNAttr && !CanCheckNullability)
5040 return;
5041
5042 SourceLocation AttrLoc;
5043 SanitizerKind::SanitizerOrdinal CheckKind;
5044 SanitizerHandler Handler;
5045 if (NNAttr) {
5046 AttrLoc = NNAttr->getLocation();
5047 CheckKind = SanitizerKind::SO_NonnullAttribute;
5048 Handler = SanitizerHandler::NonnullArg;
5049 } else {
5050 AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
5051 CheckKind = SanitizerKind::SO_NullabilityArg;
5052 Handler = SanitizerHandler::NullabilityArg;
5053 }
5054
5055 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);
5056 llvm::Value *Cond = EmitNonNullRValueCheck(RV, T: ArgType);
5057 llvm::Constant *StaticData[] = {
5058 EmitCheckSourceLocation(Loc: ArgLoc),
5059 EmitCheckSourceLocation(Loc: AttrLoc),
5060 llvm::ConstantInt::get(Ty: Int32Ty, V: ArgNo + 1),
5061 };
5062 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckKind), Check: Handler, StaticArgs: StaticData, DynamicArgs: {});
5063}
5064
5065void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType,
5066 SourceLocation ArgLoc,
5067 AbstractCallee AC, unsigned ParmNum) {
5068 if (!AC.getDecl() || !(SanOpts.has(K: SanitizerKind::NonnullAttribute) ||
5069 SanOpts.has(K: SanitizerKind::NullabilityArg)))
5070 return;
5071
5072 EmitNonNullArgCheck(RV: RValue::get(Addr, CGF&: *this), ArgType, ArgLoc, AC, ParmNum);
5073}
5074
5075// Check if the call is going to use the inalloca convention. This needs to
5076// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
5077// later, so we can't check it directly.
5078static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
5079 ArrayRef<QualType> ArgTypes) {
5080 // The Swift calling conventions don't go through the target-specific
5081 // argument classification, they never use inalloca.
5082 // TODO: Consider limiting inalloca use to only calling conventions supported
5083 // by MSVC.
5084 if (ExplicitCC == CC_Swift || ExplicitCC == CC_SwiftAsync)
5085 return false;
5086 if (!CGM.getTarget().getCXXABI().isMicrosoft())
5087 return false;
5088 return llvm::any_of(Range&: ArgTypes, P: [&](QualType Ty) {
5089 return isInAllocaArgument(ABI&: CGM.getCXXABI(), type: Ty);
5090 });
5091}
5092
5093#ifndef NDEBUG
5094// Determine whether the given argument is an Objective-C method
5095// that may have type parameters in its signature.
5096static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
5097 const DeclContext *dc = method->getDeclContext();
5098 if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {
5099 return classDecl->getTypeParamListAsWritten();
5100 }
5101
5102 if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
5103 return catDecl->getTypeParamList();
5104 }
5105
5106 return false;
5107}
5108#endif
5109
5110/// EmitCallArgs - Emit call arguments for a function.
5111void CodeGenFunction::EmitCallArgs(
5112 CallArgList &Args, PrototypeWrapper Prototype,
5113 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
5114 AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
5115 SmallVector<QualType, 16> ArgTypes;
5116
5117 assert((ParamsToSkip == 0 || Prototype.P) &&
5118 "Can't skip parameters if type info is not provided");
5119
5120 // This variable only captures *explicitly* written conventions, not those
5121 // applied by default via command line flags or target defaults, such as
5122 // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
5123 // require knowing if this is a C++ instance method or being able to see
5124 // unprototyped FunctionTypes.
5125 CallingConv ExplicitCC = CC_C;
5126
5127 // First, if a prototype was provided, use those argument types.
5128 bool IsVariadic = false;
5129 if (Prototype.P) {
5130 const auto *MD = dyn_cast<const ObjCMethodDecl *>(Val&: Prototype.P);
5131 if (MD) {
5132 IsVariadic = MD->isVariadic();
5133 ExplicitCC = getCallingConventionForDecl(
5134 D: MD, IsTargetDefaultMSABI: CGM.getTarget().getTriple().isOSWindows());
5135 ArgTypes.assign(in_start: MD->param_type_begin() + ParamsToSkip,
5136 in_end: MD->param_type_end());
5137 } else {
5138 const auto *FPT = cast<const FunctionProtoType *>(Val&: Prototype.P);
5139 IsVariadic = FPT->isVariadic();
5140 ExplicitCC = FPT->getExtInfo().getCC();
5141 ArgTypes.assign(in_start: FPT->param_type_begin() + ParamsToSkip,
5142 in_end: FPT->param_type_end());
5143 }
5144
5145#ifndef NDEBUG
5146 // Check that the prototyped types match the argument expression types.
5147 bool isGenericMethod = MD && isObjCMethodWithTypeParams(MD);
5148 CallExpr::const_arg_iterator Arg = ArgRange.begin();
5149 for (QualType Ty : ArgTypes) {
5150 assert(Arg != ArgRange.end() && "Running over edge of argument list!");
5151 QualType ParamTy = Ty.getNonReferenceType();
5152 QualType ArgTy = (*Arg)->getType();
5153 if (const auto *OBT = ParamTy->getAs<OverflowBehaviorType>())
5154 ParamTy = OBT->getUnderlyingType();
5155 if (const auto *OBT = ArgTy->getAs<OverflowBehaviorType>())
5156 ArgTy = OBT->getUnderlyingType();
5157 assert((isGenericMethod || Ty->isVariablyModifiedType() ||
5158 ParamTy->isObjCRetainableType() ||
5159 getContext().getCanonicalType(ParamTy).getTypePtr() ==
5160 getContext().getCanonicalType(ArgTy).getTypePtr()) &&
5161 "type mismatch in call argument!");
5162 ++Arg;
5163 }
5164
5165 // Either we've emitted all the call args, or we have a call to variadic
5166 // function.
5167 assert((Arg == ArgRange.end() || IsVariadic) &&
5168 "Extra arguments in non-variadic function!");
5169#endif
5170 }
5171
5172 // If we still have any arguments, emit them using the type of the argument.
5173 for (auto *A : llvm::drop_begin(RangeOrContainer&: ArgRange, N: ArgTypes.size()))
5174 ArgTypes.push_back(Elt: IsVariadic ? getVarArgType(Arg: A) : A->getType());
5175 assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
5176
5177 // We must evaluate arguments from right to left in the MS C++ ABI,
5178 // because arguments are destroyed left to right in the callee. As a special
5179 // case, there are certain language constructs that require left-to-right
5180 // evaluation, and in those cases we consider the evaluation order requirement
5181 // to trump the "destruction order is reverse construction order" guarantee.
5182 bool LeftToRight = CodeGenUtils::shouldEvaluateArgsLeftToRight(
5183 Order, ABI: CGM.getTarget().getCXXABI());
5184
5185 auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
5186 RValue EmittedArg) {
5187 if (!AC.hasFunctionDecl() || I >= AC.getNumParams())
5188 return;
5189 auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
5190 if (PS == nullptr)
5191 return;
5192
5193 const auto &Context = getContext();
5194 auto SizeTy = Context.getSizeType();
5195 auto T = Builder.getIntNTy(N: Context.getTypeSize(T: SizeTy));
5196 assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
5197 llvm::Value *V = evaluateOrEmitBuiltinObjectSize(
5198 E: Arg, Type: PS->getType(), ResType: T, EmittedE: EmittedArg.getScalarVal(), IsDynamic: PS->isDynamic());
5199 Args.add(rvalue: RValue::get(V), type: SizeTy);
5200 // If we're emitting args in reverse, be sure to do so with
5201 // pass_object_size, as well.
5202 if (!LeftToRight)
5203 std::swap(a&: Args.back(), b&: *(&Args.back() - 1));
5204 };
5205
5206 // Insert a stack save if we're going to need any inalloca args.
5207 if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
5208 assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
5209 "inalloca only supported on x86");
5210 Args.allocateArgumentMemory(CGF&: *this);
5211 }
5212
5213 // Evaluate each argument in the appropriate order.
5214 size_t CallArgsStart = Args.size();
5215 for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
5216 unsigned Idx = LeftToRight ? I : E - I - 1;
5217 CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
5218 unsigned InitialArgSize = Args.size();
5219 // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
5220 // the argument and parameter match or the objc method is parameterized.
5221 assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
5222 getContext().hasSameUnqualifiedType((*Arg)->getType(),
5223 ArgTypes[Idx]) ||
5224 (isa<ObjCMethodDecl>(AC.getDecl()) &&
5225 isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
5226 "Argument and parameter types don't match");
5227 EmitCallArg(args&: Args, E: *Arg, ArgType: ArgTypes[Idx]);
5228 // In particular, we depend on it being the last arg in Args, and the
5229 // objectsize bits depend on there only being one arg if !LeftToRight.
5230 assert(InitialArgSize + 1 == Args.size() &&
5231 "The code below depends on only adding one arg per EmitCallArg");
5232 (void)InitialArgSize;
5233 // Since pointer argument are never emitted as LValue, it is safe to emit
5234 // non-null argument check for r-value only.
5235 if (!Args.back().hasLValue()) {
5236 RValue RVArg = Args.back().getKnownRValue();
5237 EmitNonNullArgCheck(RV: RVArg, ArgType: ArgTypes[Idx], ArgLoc: (*Arg)->getExprLoc(), AC,
5238 ParmNum: ParamsToSkip + Idx);
5239 // @llvm.objectsize should never have side-effects and shouldn't need
5240 // destruction/cleanups, so we can safely "emit" it after its arg,
5241 // regardless of right-to-leftness
5242 MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
5243 }
5244 }
5245
5246 if (!LeftToRight) {
5247 // Un-reverse the arguments we just evaluated so they match up with the LLVM
5248 // IR function.
5249 std::reverse(first: Args.begin() + CallArgsStart, last: Args.end());
5250
5251 // Reverse the writebacks to match the MSVC ABI.
5252 Args.reverseWritebacks();
5253 }
5254}
5255
5256namespace {
5257
5258struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
5259 DestroyUnpassedArg(Address Addr, QualType Ty) : Addr(Addr), Ty(Ty) {}
5260
5261 Address Addr;
5262 QualType Ty;
5263
5264 void Emit(CodeGenFunction &CGF, Flags flags) override {
5265 QualType::DestructionKind DtorKind = Ty.isDestructedType();
5266 if (DtorKind == QualType::DK_cxx_destructor) {
5267 const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
5268 assert(!Dtor->isTrivial());
5269 CGF.EmitCXXDestructorCall(D: Dtor, Type: Dtor_Complete, /*for vbase*/ ForVirtualBase: false,
5270 /*Delegating=*/false, This: Addr, ThisTy: Ty);
5271 } else {
5272 CGF.callCStructDestructor(Dst: CGF.MakeAddrLValue(Addr, T: Ty));
5273 }
5274 }
5275};
5276
5277} // end anonymous namespace
5278
5279RValue CallArg::getRValue(CodeGenFunction &CGF) const {
5280 if (!HasLV)
5281 return RV;
5282 LValue Copy = CGF.MakeAddrLValue(Addr: CGF.CreateMemTempWithoutCast(T: Ty), T: Ty);
5283 CGF.EmitAggregateCopy(Dest: Copy, Src: LV, EltTy: Ty, MayOverlap: AggValueSlot::DoesNotOverlap,
5284 isVolatile: LV.isVolatile());
5285 IsUsed = true;
5286 return RValue::getAggregate(addr: Copy.getAddress());
5287}
5288
5289void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
5290 LValue Dst = CGF.MakeAddrLValue(Addr, T: Ty);
5291 if (!HasLV && RV.isScalar())
5292 CGF.EmitStoreOfScalar(value: RV.getScalarVal(), lvalue: Dst, /*isInit=*/true);
5293 else if (!HasLV && RV.isComplex())
5294 CGF.EmitStoreOfComplex(V: RV.getComplexVal(), dest: Dst, /*init=*/isInit: true);
5295 else {
5296 auto Addr = HasLV ? LV.getAddress() : RV.getAggregateAddress();
5297 LValue SrcLV = CGF.MakeAddrLValue(Addr, T: Ty);
5298 // We assume that call args are never copied into subobjects.
5299 CGF.EmitAggregateCopy(Dest: Dst, Src: SrcLV, EltTy: Ty, MayOverlap: AggValueSlot::DoesNotOverlap,
5300 isVolatile: HasLV ? LV.isVolatileQualified()
5301 : RV.isVolatileQualified());
5302 }
5303 IsUsed = true;
5304}
5305
5306void CodeGenFunction::EmitWritebacks(const CallArgList &args) {
5307 for (const auto &I : args.writebacks())
5308 emitWriteback(CGF&: *this, writeback: I);
5309}
5310
5311void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
5312 QualType type) {
5313 std::optional<DisableDebugLocationUpdates> Dis;
5314 if (isa<CXXDefaultArgExpr>(Val: E))
5315 Dis.emplace(args&: *this);
5316 if (const ObjCIndirectCopyRestoreExpr *CRE =
5317 dyn_cast<ObjCIndirectCopyRestoreExpr>(Val: E)) {
5318 assert(getLangOpts().ObjCAutoRefCount);
5319 return emitWritebackArg(CGF&: *this, args, CRE);
5320 }
5321
5322 // Add writeback for HLSLOutParamExpr.
5323 // Needs to be before the assert below because HLSLOutArgExpr is an LValue
5324 // and is not a reference.
5325 if (const HLSLOutArgExpr *OE = dyn_cast<HLSLOutArgExpr>(Val: E)) {
5326 EmitHLSLOutArgExpr(E: OE, Args&: args, Ty: type);
5327 return;
5328 }
5329
5330 assert(type->isReferenceType() == E->isGLValue() &&
5331 "reference binding to unmaterialized r-value!");
5332
5333 if (E->isGLValue()) {
5334 assert(E->getObjectKind() == OK_Ordinary);
5335 return args.add(rvalue: EmitReferenceBindingToExpr(E), type);
5336 }
5337
5338 bool HasAggregateEvalKind = hasAggregateEvaluationKind(T: type);
5339
5340 // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
5341 // However, we still have to push an EH-only cleanup in case we unwind before
5342 // we make it to the call.
5343 if (type->isRecordType() &&
5344 type->castAsRecordDecl()->isParamDestroyedInCallee()) {
5345 // If we're using inalloca, use the argument memory. Otherwise, use a
5346 // temporary.
5347 AggValueSlot Slot = args.isUsingInAlloca()
5348 ? createPlaceholderSlot(CGF&: *this, Ty: type)
5349 : CreateAggTemp(T: type, Name: "agg.tmp");
5350
5351 bool DestroyedInCallee = true, NeedsCleanup = true;
5352 if (const auto *RD = type->getAsCXXRecordDecl())
5353 DestroyedInCallee = RD->hasNonTrivialDestructor();
5354 else
5355 NeedsCleanup = type.isDestructedType();
5356
5357 if (DestroyedInCallee)
5358 Slot.setExternallyDestructed();
5359
5360 EmitAggExpr(E, AS: Slot);
5361 RValue RV = Slot.asRValue();
5362 args.add(rvalue: RV, type);
5363
5364 if (DestroyedInCallee && NeedsCleanup) {
5365 // Create a no-op GEP between the placeholder and the cleanup so we can
5366 // RAUW it successfully. It also serves as a marker of the first
5367 // instruction where the cleanup is active.
5368 pushFullExprCleanup<DestroyUnpassedArg>(kind: NormalAndEHCleanup,
5369 A: Slot.getAddress(), A: type);
5370 // This unreachable is a temporary marker which will be removed later.
5371 llvm::Instruction *IsActive =
5372 Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
5373 args.addArgCleanupDeactivation(Cleanup: EHStack.stable_begin(), IsActiveIP: IsActive);
5374 }
5375 return;
5376 }
5377
5378 if (HasAggregateEvalKind) {
5379 auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E);
5380 if (ICE && ICE->getCastKind() == CK_LValueToRValue &&
5381 ICE->getSubExpr()->getType().getAddressSpace() !=
5382 LangAS::hlsl_constant &&
5383 !type->isArrayParameterType() && !type.isNonTrivialToPrimitiveCopy()) {
5384 LValue L = EmitLValue(E: cast<CastExpr>(Val: E)->getSubExpr());
5385 assert(L.isSimple());
5386 args.addUncopiedAggregate(LV: L, type);
5387 return;
5388 }
5389 }
5390
5391 args.add(rvalue: EmitAnyExprToTemp(E), type);
5392}
5393
5394QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
5395 // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
5396 // implicitly widens null pointer constants that are arguments to varargs
5397 // functions to pointer-sized ints.
5398 if (!getTarget().getTriple().isOSWindows())
5399 return Arg->getType();
5400
5401 if (Arg->getType()->isIntegerType() &&
5402 getContext().getTypeSize(T: Arg->getType()) <
5403 getContext().getTargetInfo().getPointerWidth(AddrSpace: LangAS::Default) &&
5404 Arg->isNullPointerConstant(Ctx&: getContext(),
5405 NPC: Expr::NPC_ValueDependentIsNotNull)) {
5406 return getContext().getIntPtrType();
5407 }
5408
5409 return Arg->getType();
5410}
5411
5412// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5413// optimizer it can aggressively ignore unwind edges.
5414void CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
5415 if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
5416 !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
5417 Inst->setMetadata(Kind: "clang.arc.no_objc_arc_exceptions",
5418 Node: CGM.getNoObjCARCExceptionsMetadata());
5419}
5420
5421/// Emits a call to the given no-arguments nounwind runtime function.
5422llvm::CallInst *
5423CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5424 const llvm::Twine &name) {
5425 return EmitNounwindRuntimeCall(callee, args: ArrayRef<llvm::Value *>(), name);
5426}
5427
5428/// Emits a call to the given nounwind runtime function.
5429llvm::CallInst *
5430CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5431 ArrayRef<Address> args,
5432 const llvm::Twine &name) {
5433 SmallVector<llvm::Value *, 3> values;
5434 for (auto arg : args)
5435 values.push_back(Elt: arg.emitRawPointer(CGF&: *this));
5436 return EmitNounwindRuntimeCall(callee, args: values, name);
5437}
5438
5439llvm::CallInst *
5440CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
5441 ArrayRef<llvm::Value *> args,
5442 const llvm::Twine &name) {
5443 llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
5444 call->setDoesNotThrow();
5445 return call;
5446}
5447
5448/// Emits a simple call (never an invoke) to the given no-arguments
5449/// runtime function.
5450llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5451 const llvm::Twine &name) {
5452 return EmitRuntimeCall(callee, args: {}, name);
5453}
5454
5455// Calls which may throw must have operand bundles indicating which funclet
5456// they are nested within.
5457SmallVector<llvm::OperandBundleDef, 1>
5458CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
5459 // There is no need for a funclet operand bundle if we aren't inside a
5460 // funclet.
5461 if (!CurrentFuncletPad)
5462 return (SmallVector<llvm::OperandBundleDef, 1>());
5463
5464 // Skip intrinsics which cannot throw (as long as they don't lower into
5465 // regular function calls in the course of IR transformations).
5466 if (auto *CalleeFn = dyn_cast<llvm::Function>(Val: Callee->stripPointerCasts())) {
5467 if (CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) {
5468 auto IID = CalleeFn->getIntrinsicID();
5469 if (!llvm::IntrinsicInst::mayLowerToFunctionCall(IID))
5470 return (SmallVector<llvm::OperandBundleDef, 1>());
5471 }
5472 }
5473
5474 SmallVector<llvm::OperandBundleDef, 1> BundleList;
5475 BundleList.emplace_back(Args: "funclet", Args&: CurrentFuncletPad);
5476 return BundleList;
5477}
5478
5479static bool isConvergentIntrinsic(const llvm::Function *F) {
5480 assert(F->isIntrinsic() && "expected an intrinsic declaration");
5481 return F->isConvergent();
5482}
5483
5484/// Emits a simple call (never an invoke) to the given runtime function.
5485llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
5486 ArrayRef<llvm::Value *> args,
5487 const llvm::Twine &name,
5488 bool IsConvergent) {
5489 SmallVector<llvm::OperandBundleDef> BundleList =
5490 getBundlesForFunclet(Callee: callee.getCallee());
5491 if (IsConvergent)
5492 addConvergenceControlBundle(Bundles&: BundleList);
5493
5494 llvm::CallInst *call = Builder.CreateCall(Callee: callee, Args: args, OpBundles: BundleList, Name: name);
5495 call->setCallingConv(getRuntimeCC());
5496 assert((IsConvergent == call->isConvergent() ||
5497 (!IsConvergent && !CGM.shouldEmitConvergenceTokens())) &&
5498 "runtime call convergence doesn't match its declaration");
5499 return call;
5500}
5501
5502llvm::CallInst *CodeGenFunction::EmitIntrinsicCall(llvm::Intrinsic::ID ID,
5503 const llvm::Twine &Name) {
5504 return EmitIntrinsicCall(ID, Args: {}, RetTy: {}, Name);
5505}
5506
5507llvm::CallInst *CodeGenFunction::EmitIntrinsicCall(llvm::Intrinsic::ID ID,
5508 ArrayRef<llvm::Value *> Args,
5509 const llvm::Twine &Name) {
5510 return EmitIntrinsicCall(ID, Types: {}, Args, Name);
5511}
5512
5513llvm::CallInst *CodeGenFunction::EmitIntrinsicCall(llvm::Intrinsic::ID ID,
5514 ArrayRef<llvm::Type *> Types,
5515 ArrayRef<llvm::Value *> Args,
5516 const llvm::Twine &Name) {
5517 llvm::Function *F =
5518 llvm::Intrinsic::getOrInsertDeclaration(M: &CGM.getModule(), id: ID, OverloadTys: Types);
5519 SmallVector<llvm::OperandBundleDef> BundleList = getBundlesForFunclet(Callee: F);
5520 if (isConvergentIntrinsic(F))
5521 addConvergenceControlBundle(Bundles&: BundleList);
5522 return Builder.CreateCall(Callee: F, Args, OpBundles: BundleList, Name);
5523}
5524
5525llvm::CallInst *CodeGenFunction::EmitIntrinsicCall(llvm::Intrinsic::ID ID,
5526 ArrayRef<llvm::Value *> Args,
5527 llvm::Type *RetTy,
5528 const llvm::Twine &Name) {
5529 SmallVector<llvm::Type *> ArgTys;
5530 ArgTys.reserve(N: Args.size());
5531 for (llvm::Value *Arg : Args)
5532 ArgTys.push_back(Elt: Arg->getType());
5533 llvm::Function *F = llvm::Intrinsic::getOrInsertDeclaration(
5534 M: &CGM.getModule(), IID: ID, RetTy, ArgTys);
5535 SmallVector<llvm::OperandBundleDef> BundleList = getBundlesForFunclet(Callee: F);
5536 if (isConvergentIntrinsic(F))
5537 addConvergenceControlBundle(Bundles&: BundleList);
5538 return Builder.CreateCall(Callee: F, Args, OpBundles: BundleList, Name);
5539}
5540
5541/// Emits a call or invoke to the given noreturn runtime function.
5542void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
5543 llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
5544 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5545 getBundlesForFunclet(Callee: callee.getCallee());
5546
5547 if (getInvokeDest()) {
5548 llvm::InvokeInst *invoke = Builder.CreateInvoke(
5549 Callee: callee, NormalDest: getUnreachableBlock(), UnwindDest: getInvokeDest(), Args: args, OpBundles: BundleList);
5550 invoke->setDoesNotReturn();
5551 invoke->setCallingConv(getRuntimeCC());
5552 } else {
5553 llvm::CallInst *call = Builder.CreateCall(Callee: callee, Args: args, OpBundles: BundleList);
5554 call->setDoesNotReturn();
5555 call->setCallingConv(getRuntimeCC());
5556 Builder.CreateUnreachable();
5557 }
5558}
5559
5560/// Emits a call or invoke instruction to the given nullary runtime function.
5561llvm::CallBase *
5562CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5563 const Twine &name) {
5564 return EmitRuntimeCallOrInvoke(callee, args: {}, name);
5565}
5566
5567/// Emits a call or invoke instruction to the given runtime function.
5568llvm::CallBase *
5569CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
5570 ArrayRef<llvm::Value *> args,
5571 const Twine &name) {
5572 llvm::CallBase *call = EmitCallOrInvoke(Callee: callee, Args: args, Name: name);
5573 call->setCallingConv(getRuntimeCC());
5574 return call;
5575}
5576
5577/// Emits a call or invoke instruction to the given function, depending
5578/// on the current state of the EH stack.
5579llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
5580 ArrayRef<llvm::Value *> Args,
5581 const Twine &Name) {
5582 llvm::BasicBlock *InvokeDest = getInvokeDest();
5583 SmallVector<llvm::OperandBundleDef, 1> BundleList =
5584 getBundlesForFunclet(Callee: Callee.getCallee());
5585
5586 llvm::CallBase *Inst;
5587 if (!InvokeDest)
5588 Inst = Builder.CreateCall(Callee, Args, OpBundles: BundleList, Name);
5589 else {
5590 llvm::BasicBlock *ContBB = createBasicBlock(name: "invoke.cont");
5591 Inst = Builder.CreateInvoke(Callee, NormalDest: ContBB, UnwindDest: InvokeDest, Args, OpBundles: BundleList,
5592 Name);
5593 EmitBlock(BB: ContBB);
5594 }
5595
5596 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5597 // optimizer it can aggressively ignore unwind edges.
5598 if (CGM.getLangOpts().ObjCAutoRefCount)
5599 AddObjCARCExceptionMetadata(Inst);
5600
5601 return Inst;
5602}
5603
5604void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
5605 llvm::Value *New) {
5606 DeferredReplacements.push_back(
5607 Elt: std::make_pair(x: llvm::WeakTrackingVH(Old), y&: New));
5608}
5609
5610namespace {
5611
5612/// Specify given \p NewAlign as the alignment of return value attribute. If
5613/// such attribute already exists, re-set it to the maximal one of two options.
5614[[nodiscard]] llvm::AttributeList
5615maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
5616 const llvm::AttributeList &Attrs,
5617 llvm::Align NewAlign) {
5618 llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
5619 if (CurAlign >= NewAlign)
5620 return Attrs;
5621 llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Context&: Ctx, Alignment: NewAlign);
5622 return Attrs.removeRetAttribute(C&: Ctx, Kind: llvm::Attribute::AttrKind::Alignment)
5623 .addRetAttribute(C&: Ctx, Attr: AlignAttr);
5624}
5625
5626template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
5627protected:
5628 CodeGenFunction &CGF;
5629
5630 /// We do nothing if this is, or becomes, nullptr.
5631 const AlignedAttrTy *AA = nullptr;
5632
5633 llvm::Value *Alignment = nullptr; // May or may not be a constant.
5634 llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
5635
5636 AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5637 : CGF(CGF_) {
5638 if (!FuncDecl)
5639 return;
5640 AA = FuncDecl->getAttr<AlignedAttrTy>();
5641 }
5642
5643public:
5644 /// If we can, materialize the alignment as an attribute on return value.
5645 [[nodiscard]] llvm::AttributeList
5646 TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
5647 if (!AA || OffsetCI || CGF.SanOpts.has(K: SanitizerKind::Alignment))
5648 return Attrs;
5649 const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Val: Alignment);
5650 if (!AlignmentCI)
5651 return Attrs;
5652 // We may legitimately have non-power-of-2 alignment here.
5653 // If so, this is UB land, emit it via `@llvm.assume` instead.
5654 if (!AlignmentCI->getValue().isPowerOf2())
5655 return Attrs;
5656 llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
5657 Ctx&: CGF.getLLVMContext(), Attrs,
5658 NewAlign: llvm::Align(
5659 AlignmentCI->getLimitedValue(Limit: llvm::Value::MaximumAlignment)));
5660 AA = nullptr; // We're done. Disallow doing anything else.
5661 return NewAttrs;
5662 }
5663
5664 /// Emit alignment assumption.
5665 /// This is a general fallback that we take if either there is an offset,
5666 /// or the alignment is variable or we are sanitizing for alignment.
5667 void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
5668 if (!AA)
5669 return;
5670 CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
5671 AA->getLocation(), Alignment, OffsetCI);
5672 AA = nullptr; // We're done. Disallow doing anything else.
5673 }
5674};
5675
5676/// Helper data structure to emit `AssumeAlignedAttr`.
5677class AssumeAlignedAttrEmitter final
5678 : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
5679public:
5680 AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
5681 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5682 if (!AA)
5683 return;
5684 // It is guaranteed that the alignment/offset are constants.
5685 Alignment = cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: AA->getAlignment()));
5686 if (Expr *Offset = AA->getOffset()) {
5687 OffsetCI = cast<llvm::ConstantInt>(Val: CGF.EmitScalarExpr(E: Offset));
5688 if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
5689 OffsetCI = nullptr;
5690 }
5691 }
5692};
5693
5694/// Helper data structure to emit `AllocAlignAttr`.
5695class AllocAlignAttrEmitter final
5696 : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
5697public:
5698 AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
5699 const CallArgList &CallArgs)
5700 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
5701 if (!AA)
5702 return;
5703 // Alignment may or may not be a constant, and that is okay.
5704 Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
5705 .getRValue(CGF)
5706 .getScalarVal();
5707 }
5708
5709 [[nodiscard]] llvm::AttributeList
5710 TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs,
5711 const ClangToLLVMArgMapping &IRFunctionArgs) {
5712 llvm::AttributeList NewAttrs = Attrs;
5713 if (AA) {
5714 auto [FirstIRArg, NumIRArgs] =
5715 IRFunctionArgs.getIRArgs(ArgNo: AA->getParamIndex().getLLVMIndex());
5716 if (NumIRArgs == 1)
5717 NewAttrs = NewAttrs.addParamAttribute(C&: CGF.getLLVMContext(), ArgNo: FirstIRArg,
5718 Kind: llvm::Attribute::AllocAlign);
5719 }
5720 // NOTE: we shouldn't _need_ to emit this `align` attribute here, but there
5721 // are some interesting cases where LLVM can manage to delete the arguments
5722 // to a function that has the `alloc_align` attribute on it, and then the
5723 // alignment gets lost. An example of this is the failure you'll get in
5724 // clang/test/Headers/mm_malloc.c if this `align` is omitted, because
5725 // _mm_malloc ends up getting its arguments turned into constants that get
5726 // passed to posix_memalign, which we don't have logic to handle in LLVM.
5727 return AbstractAssumeAlignedAttrEmitter<
5728 AllocAlignAttr>::TryEmitAsCallSiteAttribute(Attrs: NewAttrs);
5729 }
5730};
5731
5732} // namespace
5733
5734static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
5735 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Ty))
5736 return VT->getPrimitiveSizeInBits().getKnownMinValue();
5737 if (auto *AT = dyn_cast<llvm::ArrayType>(Val: Ty))
5738 return getMaxVectorWidth(Ty: AT->getElementType());
5739
5740 unsigned MaxVectorWidth = 0;
5741 if (auto *ST = dyn_cast<llvm::StructType>(Val: Ty))
5742 for (auto *I : ST->elements())
5743 MaxVectorWidth = std::max(a: MaxVectorWidth, b: getMaxVectorWidth(Ty: I));
5744 return MaxVectorWidth;
5745}
5746
5747RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
5748 const CGCallee &Callee,
5749 ReturnValueSlot ReturnValue,
5750 const CallArgList &CallArgs,
5751 llvm::CallBase **callOrInvoke, bool IsMustTail,
5752 SourceLocation Loc,
5753 bool IsVirtualFunctionPointerThunk) {
5754 // FIXME: We no longer need the types from CallArgs; lift up and simplify.
5755
5756 assert(Callee.isOrdinary() || Callee.isVirtual());
5757
5758 // Handle struct-return functions by passing a pointer to the
5759 // location that we would like to return into.
5760 QualType RetTy = CallInfo.getReturnType();
5761 const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
5762
5763 llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(FI: CallInfo);
5764
5765 const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
5766 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl)) {
5767 // We can only guarantee that a function is called from the correct
5768 // context/function based on the appropriate target attributes,
5769 // so only check in the case where we have both always_inline and target
5770 // since otherwise we could be making a conditional call after a check for
5771 // the proper cpu features (and it won't cause code generation issues due to
5772 // function based code generation).
5773 if ((TargetDecl->hasAttr<AlwaysInlineAttr>() &&
5774 (TargetDecl->hasAttr<TargetAttr>() ||
5775 (CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>()))) ||
5776 (CurFuncDecl && CurFuncDecl->hasAttr<FlattenAttr>() &&
5777 (CurFuncDecl->hasAttr<TargetAttr>() ||
5778 TargetDecl->hasAttr<TargetAttr>())))
5779 checkTargetFeatures(Loc, TargetDecl: FD);
5780 }
5781
5782 // Some architectures (such as x86-64) have the ABI changed based on
5783 // attribute-target/features. Give them a chance to diagnose.
5784 const FunctionDecl *CallerDecl = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
5785 const FunctionDecl *CalleeDecl = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl);
5786 CGM.getTargetCodeGenInfo().checkFunctionCallABI(CGM, CallLoc: Loc, Caller: CallerDecl,
5787 Callee: CalleeDecl, Args: CallArgs, ReturnType: RetTy);
5788
5789 // 1. Set up the arguments.
5790
5791 // If we're using inalloca, insert the allocation after the stack save.
5792 // FIXME: Do this earlier rather than hacking it in here!
5793 RawAddress ArgMemory = RawAddress::invalid();
5794 if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
5795 const llvm::DataLayout &DL = CGM.getDataLayout();
5796 llvm::Instruction *IP = CallArgs.getStackBase();
5797 llvm::AllocaInst *AI;
5798 if (IP) {
5799 IP = IP->getNextNode();
5800 AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(), "argmem",
5801 IP->getIterator());
5802 } else {
5803 AI = CreateTempAlloca(Ty: ArgStruct, Name: "argmem");
5804 }
5805 auto Align = CallInfo.getArgStructAlignment();
5806 AI->setAlignment(Align.getAsAlign());
5807 AI->setUsedWithInAlloca(true);
5808 assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
5809 ArgMemory = RawAddress(AI, ArgStruct, Align);
5810 }
5811
5812 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
5813 SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
5814
5815 // If the call returns a temporary with struct return, create a temporary
5816 // alloca to hold the result, unless one is given to us.
5817 Address SRetPtr = Address::invalid();
5818 // Original alloca for lifetime markers
5819 Address SRetAlloca = Address::invalid();
5820 bool NeedSRetLifetimeEnd = false;
5821 if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
5822 // For virtual function pointer thunks and musttail calls, we must always
5823 // forward an incoming SRet pointer to the callee, because a local alloca
5824 // would be de-allocated before the call. These cases both guarantee that
5825 // there will be an incoming SRet argument of the correct type.
5826 if ((IsVirtualFunctionPointerThunk || IsMustTail) && RetAI.isIndirect()) {
5827 SRetPtr = makeNaturalAddressForPointer(Ptr: CurFn->arg_begin() +
5828 IRFunctionArgs.getSRetArgNo(),
5829 T: RetTy, Alignment: CharUnits::fromQuantity(Quantity: 1));
5830 } else if (!ReturnValue.isNull()) {
5831 SRetPtr = ReturnValue.getAddress();
5832 } else {
5833 SRetPtr = CreateMemTempWithoutCast(T: RetTy, Name: "tmp");
5834 if (HaveInsertPoint() && ReturnValue.isUnused()) {
5835 NeedSRetLifetimeEnd = EmitLifetimeStart(Addr: SRetPtr.getBasePointer());
5836 if (NeedSRetLifetimeEnd)
5837 SRetAlloca = SRetPtr;
5838 }
5839 }
5840 if (IRFunctionArgs.hasSRetArg()) {
5841 // A mismatch between the allocated return value's AS and the target's
5842 // chosen IndirectAS can happen e.g. when passing the this pointer through
5843 // a chain involving stores to / loads from the DefaultAS; we address this
5844 // here, symmetrically with the handling we have for normal pointer args.
5845 if (SRetPtr.getAddressSpace() != RetAI.getIndirectAddrSpace()) {
5846 llvm::Value *V = SRetPtr.getBasePointer();
5847 llvm::Type *Ty = llvm::PointerType::get(C&: getLLVMContext(),
5848 AddressSpace: RetAI.getIndirectAddrSpace());
5849
5850 SRetPtr = SRetPtr.withPointer(NewPointer: performAddrSpaceCast(Src: V, DestTy: Ty),
5851 IsKnownNonNull: SRetPtr.isKnownNonNull());
5852 }
5853 IRCallArgs[IRFunctionArgs.getSRetArgNo()] =
5854 getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy);
5855 } else if (RetAI.isInAlloca()) {
5856 Address Addr =
5857 Builder.CreateStructGEP(Addr: ArgMemory, Index: RetAI.getInAllocaFieldIndex());
5858 Builder.CreateStore(Val: getAsNaturalPointerTo(Addr: SRetPtr, PointeeType: RetTy), Addr);
5859 }
5860 }
5861
5862 RawAddress swiftErrorTemp = RawAddress::invalid();
5863 Address swiftErrorArg = Address::invalid();
5864
5865 // When passing arguments using temporary allocas, we need to add the
5866 // appropriate lifetime markers. This vector keeps track of all the lifetime
5867 // markers that need to be ended right after the call.
5868 SmallVector<llvm::Value *, 2> CallLifetimeEndAfterCall;
5869
5870 // Translate all of the arguments as necessary to match the IR lowering.
5871 assert(CallInfo.arg_size() == CallArgs.size() &&
5872 "Mismatch between function signature & arguments.");
5873 unsigned ArgNo = 0;
5874 CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
5875 for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
5876 I != E; ++I, ++info_it, ++ArgNo) {
5877 const ABIArgInfo &ArgInfo = info_it->info;
5878
5879 // Insert a padding argument to ensure proper alignment.
5880 if (IRFunctionArgs.hasPaddingArg(ArgNo))
5881 IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
5882 llvm::UndefValue::get(T: ArgInfo.getPaddingType());
5883
5884 unsigned FirstIRArg, NumIRArgs;
5885 std::tie(args&: FirstIRArg, args&: NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
5886
5887 bool ArgHasMaybeUndefAttr = CodeGenUtils::isArgumentMaybeUndef(
5888 TargetDecl, NumRequiredArgs: CallInfo.getNumRequiredArgs(), ArgNo);
5889
5890 switch (ArgInfo.getKind()) {
5891 case ABIArgInfo::InAlloca: {
5892 assert(NumIRArgs == 0);
5893 assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
5894 if (I->isAggregate()) {
5895 RawAddress Addr = I->hasLValue()
5896 ? I->getKnownLValue().getAddress()
5897 : I->getKnownRValue().getAggregateAddress();
5898 llvm::Instruction *Placeholder =
5899 cast<llvm::Instruction>(Val: Addr.getPointer());
5900
5901 if (!ArgInfo.getInAllocaIndirect()) {
5902 // Replace the placeholder with the appropriate argument slot GEP.
5903 CGBuilderTy::InsertPoint IP = Builder.saveIP();
5904 Builder.SetInsertPoint(Placeholder);
5905 Addr = Builder.CreateStructGEP(Addr: ArgMemory,
5906 Index: ArgInfo.getInAllocaFieldIndex());
5907 Builder.restoreIP(IP);
5908 } else {
5909 // For indirect things such as overaligned structs, replace the
5910 // placeholder with a regular aggregate temporary alloca. Store the
5911 // address of this alloca into the struct.
5912 Addr =
5913 CreateMemTempWithoutCast(T: info_it->type, Name: "inalloca.indirect.tmp");
5914 Address ArgSlot = Builder.CreateStructGEP(
5915 Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5916 Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5917 }
5918 deferPlaceholderReplacement(Old: Placeholder, New: Addr.getPointer());
5919 } else if (ArgInfo.getInAllocaIndirect()) {
5920 // Make a temporary alloca and store the address of it into the argument
5921 // struct.
5922 RawAddress Addr = CreateMemTempWithoutCast(
5923 T: I->Ty, Align: getContext().getTypeAlignInChars(T: I->Ty),
5924 Name: "indirect-arg-temp");
5925 I->copyInto(CGF&: *this, Addr);
5926 Address ArgSlot =
5927 Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5928 Builder.CreateStore(Val: Addr.getPointer(), Addr: ArgSlot);
5929 } else {
5930 // Store the RValue into the argument struct.
5931 Address Addr =
5932 Builder.CreateStructGEP(Addr: ArgMemory, Index: ArgInfo.getInAllocaFieldIndex());
5933 Addr = Addr.withElementType(ElemTy: ConvertTypeForMem(T: I->Ty));
5934 I->copyInto(CGF&: *this, Addr);
5935 }
5936 break;
5937 }
5938
5939 case ABIArgInfo::Indirect:
5940 case ABIArgInfo::IndirectAliased: {
5941 assert(NumIRArgs == 1);
5942 if (I->isAggregate()) {
5943 // We want to avoid creating an unnecessary temporary+copy here;
5944 // however, we need one in three cases:
5945 // 1. If the argument is not byval, and we are required to copy the
5946 // source. (This case doesn't occur on any common architecture.)
5947 // 2. If the argument is byval, RV is not sufficiently aligned, and
5948 // we cannot force it to be sufficiently aligned.
5949 // 3. If the argument is byval, but RV is not located in default
5950 // or alloca address space.
5951 Address Addr = I->hasLValue()
5952 ? I->getKnownLValue().getAddress()
5953 : I->getKnownRValue().getAggregateAddress();
5954 CharUnits Align = ArgInfo.getIndirectAlign();
5955 const llvm::DataLayout *TD = &CGM.getDataLayout();
5956
5957 assert((FirstIRArg >= IRFuncTy->getNumParams() ||
5958 IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
5959 TD->getAllocaAddrSpace()) &&
5960 "indirect argument must be in alloca address space");
5961
5962 bool NeedCopy = false;
5963 if (Addr.getAlignment() < Align &&
5964 llvm::getOrEnforceKnownAlignment(V: Addr.emitRawPointer(CGF&: *this),
5965 PrefAlign: Align.getAsAlign(),
5966 DL: *TD) < Align.getAsAlign()) {
5967 NeedCopy = true;
5968 } else if (I->hasLValue()) {
5969 auto LV = I->getKnownLValue();
5970
5971 bool isByValOrRef =
5972 ArgInfo.isIndirectAliased() || ArgInfo.getIndirectByVal();
5973
5974 if (!isByValOrRef ||
5975 (LV.getAlignment() < getContext().getTypeAlignInChars(T: I->Ty))) {
5976 NeedCopy = true;
5977 }
5978
5979 if (isByValOrRef && Addr.getType()->getAddressSpace() !=
5980 ArgInfo.getIndirectAddrSpace()) {
5981 NeedCopy = true;
5982 }
5983 }
5984
5985 if (!NeedCopy) {
5986 // Skip the extra memcpy call.
5987 llvm::Value *V = getAsNaturalPointerTo(Addr, PointeeType: I->Ty);
5988 auto *T = llvm::PointerType::get(C&: CGM.getLLVMContext(),
5989 AddressSpace: ArgInfo.getIndirectAddrSpace());
5990
5991 // FIXME: This should not depend on the language address spaces, and
5992 // only the contextual values. If the address space mismatches, see if
5993 // we can look through a cast to a compatible address space value,
5994 // otherwise emit a copy.
5995 llvm::Value *Val = performAddrSpaceCast(Src: V, DestTy: T);
5996 if (ArgHasMaybeUndefAttr)
5997 Val = Builder.CreateFreeze(V: Val);
5998 IRCallArgs[FirstIRArg] = Val;
5999 break;
6000 }
6001 } else if (I->getType()->isArrayParameterType()) {
6002 // Don't produce a temporary for ArrayParameterType arguments.
6003 // ArrayParameterType arguments are only created from
6004 // HLSL_ArrayRValue casts and HLSLOutArgExpr expressions, both
6005 // of which create temporaries already. This allows us to just use the
6006 // scalar for the decayed array pointer as the argument directly.
6007 IRCallArgs[FirstIRArg] = I->getKnownRValue().getScalarVal();
6008 break;
6009 }
6010
6011 // For non-aggregate args and aggregate args meeting conditions above
6012 // we need to create an aligned temporary, and copy to it.
6013 RawAddress AI = CreateMemTempWithoutCast(
6014 T: I->Ty, Align: ArgInfo.getIndirectAlign(), Name: "byval-temp");
6015 llvm::Value *Val = getAsNaturalPointerTo(Addr: AI, PointeeType: I->Ty);
6016 if (ArgHasMaybeUndefAttr)
6017 Val = Builder.CreateFreeze(V: Val);
6018 IRCallArgs[FirstIRArg] = Val;
6019
6020 // Emit lifetime markers for the temporary alloca and add cleanup code to
6021 // emit the end lifetime marker after the call.
6022 if (EmitLifetimeStart(Addr: AI.getPointer()))
6023 CallLifetimeEndAfterCall.push_back(Elt: AI.getPointer());
6024
6025 // Generate the copy.
6026 I->copyInto(CGF&: *this, Addr: AI);
6027 break;
6028 }
6029
6030 case ABIArgInfo::Ignore:
6031 assert(NumIRArgs == 0);
6032 break;
6033
6034 case ABIArgInfo::Extend:
6035 case ABIArgInfo::Direct: {
6036 if (!isa<llvm::StructType>(Val: ArgInfo.getCoerceToType()) &&
6037 ArgInfo.getCoerceToType() == ConvertType(T: info_it->type) &&
6038 ArgInfo.getDirectOffset() == 0) {
6039 assert(NumIRArgs == 1);
6040 llvm::Value *V;
6041 if (!I->isAggregate())
6042 V = I->getKnownRValue().getScalarVal();
6043 else
6044 V = Builder.CreateLoad(
6045 Addr: I->hasLValue() ? I->getKnownLValue().getAddress()
6046 : I->getKnownRValue().getAggregateAddress());
6047
6048 // Implement swifterror by copying into a new swifterror argument.
6049 // We'll write back in the normal path out of the call.
6050 if (CallInfo.getExtParameterInfo(argIndex: ArgNo).getABI() ==
6051 ParameterABI::SwiftErrorResult) {
6052 assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
6053
6054 QualType pointeeTy = I->Ty->getPointeeType();
6055 swiftErrorArg = makeNaturalAddressForPointer(
6056 Ptr: V, T: pointeeTy, Alignment: getContext().getTypeAlignInChars(T: pointeeTy));
6057
6058 swiftErrorTemp = CreateMemTempWithoutCast(
6059 T: pointeeTy, Align: getPointerAlign(), Name: "swifterror.temp");
6060 V = swiftErrorTemp.getPointer();
6061 cast<llvm::AllocaInst>(Val: V)->setSwiftError(true);
6062
6063 llvm::Value *errorValue = Builder.CreateLoad(Addr: swiftErrorArg);
6064 Builder.CreateStore(Val: errorValue, Addr: swiftErrorTemp);
6065 }
6066
6067 // We might have to widen integers, but we should never truncate.
6068 if (ArgInfo.getCoerceToType() != V->getType() &&
6069 V->getType()->isIntegerTy())
6070 V = Builder.CreateZExt(V, DestTy: ArgInfo.getCoerceToType());
6071
6072 // The only plausible mismatch here would be for pointer address spaces.
6073 // We assume that the target has a reasonable mapping for the DefaultAS
6074 // (it can be casted to from incoming specific ASes), and insert an AS
6075 // cast to address the mismatch.
6076 if (FirstIRArg < IRFuncTy->getNumParams() &&
6077 V->getType() != IRFuncTy->getParamType(i: FirstIRArg)) {
6078 assert(V->getType()->isPointerTy() && "Only pointers can mismatch!");
6079 V = performAddrSpaceCast(Src: V, DestTy: IRFuncTy->getParamType(i: FirstIRArg));
6080 }
6081
6082 if (ArgHasMaybeUndefAttr)
6083 V = Builder.CreateFreeze(V);
6084 IRCallArgs[FirstIRArg] = V;
6085 break;
6086 }
6087
6088 llvm::StructType *STy =
6089 dyn_cast<llvm::StructType>(Val: ArgInfo.getCoerceToType());
6090
6091 // FIXME: Avoid the conversion through memory if possible.
6092 Address Src = Address::invalid();
6093 if (!I->isAggregate()) {
6094 Src = CreateMemTempWithoutCast(T: I->Ty, Name: "coerce");
6095 I->copyInto(CGF&: *this, Addr: Src);
6096 } else {
6097 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
6098 : I->getKnownRValue().getAggregateAddress();
6099 }
6100
6101 // If the value is offset in memory, apply the offset now.
6102 Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
6103
6104 // Fast-isel and the optimizer generally like scalar values better than
6105 // FCAs, so we flatten them if this is safe to do for this argument.
6106 if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
6107 llvm::Type *SrcTy = Src.getElementType();
6108 llvm::TypeSize SrcTypeSize =
6109 CGM.getDataLayout().getTypeAllocSize(Ty: SrcTy);
6110 llvm::TypeSize DstTypeSize = CGM.getDataLayout().getTypeAllocSize(Ty: STy);
6111 if (SrcTypeSize.isScalable()) {
6112 assert(STy->containsHomogeneousScalableVectorTypes() &&
6113 "ABI only supports structure with homogeneous scalable vector "
6114 "type");
6115 assert(SrcTypeSize == DstTypeSize &&
6116 "Only allow non-fractional movement of structure with "
6117 "homogeneous scalable vector type");
6118 assert(NumIRArgs == STy->getNumElements());
6119
6120 llvm::Value *StoredStructValue =
6121 Builder.CreateLoad(Addr: Src, Name: Src.getName() + ".tuple");
6122 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
6123 llvm::Value *Extract = Builder.CreateExtractValue(
6124 Agg: StoredStructValue, Idxs: i, Name: Src.getName() + ".extract" + Twine(i));
6125 IRCallArgs[FirstIRArg + i] = Extract;
6126 }
6127 } else {
6128 uint64_t SrcSize = SrcTypeSize.getFixedValue();
6129 uint64_t DstSize = DstTypeSize.getFixedValue();
6130 bool HasPFPFields = getContext().hasPFPFields(Ty: I->Ty);
6131
6132 // If the source type is smaller than the destination type of the
6133 // coerce-to logic, copy the source value into a temp alloca the size
6134 // of the destination type to allow loading all of it. The bits past
6135 // the source value are left undef.
6136 if (HasPFPFields || SrcSize < DstSize) {
6137 Address TempAlloca = CreateTempAlloca(Ty: STy, align: Src.getAlignment(),
6138 Name: Src.getName() + ".coerce");
6139 if (HasPFPFields) {
6140 // Structures with PFP fields require a coerced load to remove any
6141 // pointer signatures.
6142 Builder.CreateStore(
6143 Val: CreatePFPCoercedLoad(Src, SrcFETy: I->Ty, Ty: ArgInfo.getCoerceToType(),
6144 CGF&: *this),
6145 Addr: TempAlloca);
6146 } else
6147 Builder.CreateMemCpy(Dest: TempAlloca, Src, Size: SrcSize);
6148 Src = TempAlloca;
6149 } else {
6150 Src = Src.withElementType(ElemTy: STy);
6151 }
6152
6153 assert(NumIRArgs == STy->getNumElements());
6154 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
6155 Address EltPtr = Builder.CreateStructGEP(Addr: Src, Index: i);
6156 llvm::Value *LI = Builder.CreateLoad(Addr: EltPtr);
6157 if (ArgHasMaybeUndefAttr)
6158 LI = Builder.CreateFreeze(V: LI);
6159 IRCallArgs[FirstIRArg + i] = LI;
6160 }
6161 }
6162 } else {
6163 // In the simple case, just pass the coerced loaded value.
6164 assert(NumIRArgs == 1);
6165 llvm::Value *Load =
6166 CreateCoercedLoad(Src, SrcFETy: I->Ty, Ty: ArgInfo.getCoerceToType(), CGF&: *this);
6167
6168 if (CallInfo.isCmseNSCall()) {
6169 // For certain parameter types, clear padding bits, as they may reveal
6170 // sensitive information.
6171 // Small struct/union types are passed as integer arrays.
6172 auto *ATy = dyn_cast<llvm::ArrayType>(Val: Load->getType());
6173 if (ATy != nullptr && isa<RecordType>(Val: I->Ty.getCanonicalType()))
6174 Load = EmitCMSEClearRecord(Src: Load, ATy, QTy: I->Ty);
6175 }
6176
6177 if (ArgHasMaybeUndefAttr)
6178 Load = Builder.CreateFreeze(V: Load);
6179 IRCallArgs[FirstIRArg] = Load;
6180 }
6181
6182 break;
6183 }
6184
6185 case ABIArgInfo::CoerceAndExpand: {
6186 auto coercionType = ArgInfo.getCoerceAndExpandType();
6187 auto layout = CGM.getDataLayout().getStructLayout(Ty: coercionType);
6188 auto unpaddedCoercionType = ArgInfo.getUnpaddedCoerceAndExpandType();
6189 auto *unpaddedStruct = dyn_cast<llvm::StructType>(Val: unpaddedCoercionType);
6190
6191 Address addr = Address::invalid();
6192 RawAddress AllocaAddr = RawAddress::invalid();
6193 bool NeedLifetimeEnd = false;
6194 if (I->isAggregate()) {
6195 addr = I->hasLValue() ? I->getKnownLValue().getAddress()
6196 : I->getKnownRValue().getAggregateAddress();
6197
6198 } else {
6199 RValue RV = I->getKnownRValue();
6200 assert(RV.isScalar()); // complex should always just be direct
6201
6202 llvm::Type *scalarType = RV.getScalarVal()->getType();
6203 auto scalarAlign = CGM.getDataLayout().getPrefTypeAlign(Ty: scalarType);
6204
6205 // Materialize to a temporary.
6206 addr = CreateTempAlloca(Ty: RV.getScalarVal()->getType(),
6207 align: CharUnits::fromQuantity(Quantity: std::max(
6208 a: layout->getAlignment(), b: scalarAlign)),
6209 Name: "tmp",
6210 /*ArraySize=*/nullptr, Alloca: &AllocaAddr);
6211 NeedLifetimeEnd = EmitLifetimeStart(Addr: AllocaAddr.getPointer());
6212
6213 Builder.CreateStore(Val: RV.getScalarVal(), Addr: addr);
6214 }
6215
6216 addr = addr.withElementType(ElemTy: coercionType);
6217
6218 unsigned IRArgPos = FirstIRArg;
6219 unsigned unpaddedIndex = 0;
6220 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
6221 llvm::Type *eltType = coercionType->getElementType(N: i);
6222 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
6223 continue;
6224 Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
6225 llvm::Value *elt = CreateCoercedLoad(
6226 Src: eltAddr, SrcFETy: I->Ty,
6227 Ty: unpaddedStruct ? unpaddedStruct->getElementType(N: unpaddedIndex++)
6228 : unpaddedCoercionType,
6229 CGF&: *this);
6230 if (ArgHasMaybeUndefAttr)
6231 elt = Builder.CreateFreeze(V: elt);
6232 IRCallArgs[IRArgPos++] = elt;
6233 }
6234 assert(IRArgPos == FirstIRArg + NumIRArgs);
6235
6236 if (NeedLifetimeEnd)
6237 EmitLifetimeEnd(Addr: AllocaAddr.getPointer());
6238 break;
6239 }
6240
6241 case ABIArgInfo::Expand: {
6242 unsigned IRArgPos = FirstIRArg;
6243 ExpandTypeToArgs(Ty: I->Ty, Arg: *I, IRFuncTy, IRCallArgs, IRCallArgPos&: IRArgPos);
6244 assert(IRArgPos == FirstIRArg + NumIRArgs);
6245 break;
6246 }
6247
6248 case ABIArgInfo::TargetSpecific: {
6249 Address Src = Address::invalid();
6250 if (!I->isAggregate()) {
6251 Src = CreateMemTempWithoutCast(T: I->Ty, Name: "target_coerce");
6252 I->copyInto(CGF&: *this, Addr: Src);
6253 } else {
6254 Src = I->hasLValue() ? I->getKnownLValue().getAddress()
6255 : I->getKnownRValue().getAggregateAddress();
6256 }
6257
6258 // If the value is offset in memory, apply the offset now.
6259 Src = emitAddressAtOffset(CGF&: *this, addr: Src, info: ArgInfo);
6260 llvm::Value *Load =
6261 CGM.getABIInfo().createCoercedLoad(SrcAddr: Src, AI: ArgInfo, CGF&: *this);
6262 IRCallArgs[FirstIRArg] = Load;
6263 break;
6264 }
6265 }
6266 }
6267
6268 const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(CGF&: *this);
6269 llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
6270
6271 // If we're using inalloca, set up that argument.
6272 if (ArgMemory.isValid()) {
6273 llvm::Value *Arg = ArgMemory.getPointer();
6274 assert(IRFunctionArgs.hasInallocaArg());
6275 IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
6276 }
6277
6278 // 2. Prepare the function pointer.
6279
6280 // If the callee is a bitcast of a non-variadic function to have a
6281 // variadic function pointer type, check to see if we can remove the
6282 // bitcast. This comes up with unprototyped functions.
6283 //
6284 // This makes the IR nicer, but more importantly it ensures that we
6285 // can inline the function at -O0 if it is marked always_inline.
6286 auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
6287 llvm::Value *Ptr) -> llvm::Function * {
6288 if (!CalleeFT->isVarArg())
6289 return nullptr;
6290
6291 // Get underlying value if it's a bitcast
6292 if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Val: Ptr)) {
6293 if (CE->getOpcode() == llvm::Instruction::BitCast)
6294 Ptr = CE->getOperand(i_nocapture: 0);
6295 }
6296
6297 llvm::Function *OrigFn = dyn_cast<llvm::Function>(Val: Ptr);
6298 if (!OrigFn)
6299 return nullptr;
6300
6301 llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
6302
6303 // If the original type is variadic, or if any of the component types
6304 // disagree, we cannot remove the cast.
6305 if (OrigFT->isVarArg() ||
6306 OrigFT->getNumParams() != CalleeFT->getNumParams() ||
6307 OrigFT->getReturnType() != CalleeFT->getReturnType())
6308 return nullptr;
6309
6310 for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i)
6311 if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
6312 return nullptr;
6313
6314 return OrigFn;
6315 };
6316
6317 if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {
6318 CalleePtr = OrigFn;
6319 IRFuncTy = OrigFn->getFunctionType();
6320 }
6321
6322 // 3. Perform the actual call.
6323
6324 // Deactivate any cleanups that we're supposed to do immediately before
6325 // the call.
6326 if (!CallArgs.getCleanupsToDeactivate().empty())
6327 deactivateArgCleanupsBeforeCall(CGF&: *this, CallArgs);
6328
6329 // Update the largest vector width if any arguments have vector types.
6330 for (unsigned i = 0; i < IRCallArgs.size(); ++i)
6331 LargestVectorWidth = std::max(a: LargestVectorWidth,
6332 b: getMaxVectorWidth(Ty: IRCallArgs[i]->getType()));
6333
6334 // Compute the calling convention and attributes.
6335 unsigned CallingConv;
6336 llvm::AttributeList Attrs;
6337 CGM.ConstructAttributeList(Name: CalleePtr->getName(), FI: CallInfo,
6338 CalleeInfo: Callee.getAbstractInfo(), AttrList&: Attrs, CallingConv,
6339 /*AttrOnCallSite=*/true,
6340 /*IsThunk=*/false);
6341
6342 if (CallingConv == llvm::CallingConv::X86_VectorCall &&
6343 getTarget().getTriple().isWindowsArm64EC()) {
6344 CGM.Error(loc: Loc, error: "__vectorcall calling convention is not currently "
6345 "supported");
6346 }
6347
6348 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
6349 if (FD->hasAttr<StrictFPAttr>())
6350 // All calls within a strictfp function are marked strictfp
6351 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::StrictFP);
6352
6353 // If -ffast-math is enabled and the function is guarded by an
6354 // '__attribute__((optnone)) adjust the memory attribute so the BE emits the
6355 // library call instead of the intrinsic.
6356 if (FD->hasAttr<OptimizeNoneAttr>() && getLangOpts().FastMath)
6357 CGM.AdjustMemoryAttribute(Name: CalleePtr->getName(), CalleeInfo: Callee.getAbstractInfo(),
6358 Attrs);
6359 }
6360 // Add call-site nomerge attribute if exists.
6361 if (InNoMergeAttributedStmt)
6362 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoMerge);
6363
6364 // Add call-site noinline attribute if exists.
6365 if (InNoInlineAttributedStmt)
6366 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoInline);
6367
6368 // Add call-site always_inline attribute if exists.
6369 // Note: This corresponds to the [[clang::always_inline]] statement attribute.
6370 if (InAlwaysInlineAttributedStmt &&
6371 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
6372 Caller: CallerDecl, Callee: CalleeDecl))
6373 Attrs =
6374 Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::AlwaysInline);
6375
6376 // Apply some call-site-specific attributes.
6377 // TODO: work this into building the attribute set.
6378
6379 // Apply always_inline to all calls within flatten functions.
6380 // FIXME: should this really take priority over __try, below?
6381 if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
6382 !InNoInlineAttributedStmt &&
6383 !(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>()) &&
6384 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(
6385 Caller: CallerDecl, Callee: CalleeDecl)) {
6386 Attrs =
6387 Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::AlwaysInline);
6388 }
6389
6390 // Disable inlining inside SEH __try blocks.
6391 if (isSEHTryScope()) {
6392 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::NoInline);
6393 }
6394
6395 // Decide whether to use a call or an invoke.
6396 bool CannotThrow;
6397 if (currentFunctionUsesSEHTry()) {
6398 // SEH cares about asynchronous exceptions, so everything can "throw."
6399 CannotThrow = false;
6400 } else if (isCleanupPadScope() &&
6401 getEHPersonality(CGF&: *this).isMSVCXXPersonality()) {
6402 // The MSVC++ personality will implicitly terminate the program if an
6403 // exception is thrown during a cleanup outside of a try/catch.
6404 // We don't need to model anything in IR to get this behavior.
6405 CannotThrow = true;
6406 } else {
6407 // Otherwise, nounwind call sites will never throw.
6408 CannotThrow = Attrs.hasFnAttr(Kind: llvm::Attribute::NoUnwind);
6409
6410 if (auto *FPtr = dyn_cast<llvm::Function>(Val: CalleePtr))
6411 if (FPtr->hasFnAttribute(Kind: llvm::Attribute::NoUnwind))
6412 CannotThrow = true;
6413 }
6414
6415 // If we made a temporary, be sure to clean up after ourselves. Note that we
6416 // can't depend on being inside of an ExprWithCleanups, so we need to manually
6417 // pop this cleanup later on. Being eager about this is OK, since this
6418 // temporary is 'invisible' outside of the callee.
6419 // Use the original alloca pointer (before any addrspacecast) for the
6420 // lifetime end marker, since lifetime intrinsics must reference the alloca
6421 // address space.
6422 if (NeedSRetLifetimeEnd)
6423 pushFullExprCleanup<CallLifetimeEnd>(kind: NormalEHLifetimeMarker, A: SRetAlloca);
6424
6425 llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();
6426
6427 SmallVector<llvm::OperandBundleDef, 1> BundleList =
6428 getBundlesForFunclet(Callee: CalleePtr);
6429
6430 if (SanOpts.has(K: SanitizerKind::KCFI) &&
6431 !isa_and_nonnull<FunctionDecl>(Val: TargetDecl))
6432 EmitKCFIOperandBundle(Callee: ConcreteCallee, Bundles&: BundleList);
6433
6434 // Add the pointer-authentication bundle.
6435 EmitPointerAuthOperandBundle(Info: ConcreteCallee.getPointerAuthInfo(), Bundles&: BundleList);
6436
6437 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl))
6438 if (FD->hasAttr<StrictFPAttr>())
6439 // All calls within a strictfp function are marked strictfp
6440 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: llvm::Attribute::StrictFP);
6441
6442 AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
6443 Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
6444
6445 AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
6446 Attrs =
6447 AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs, IRFunctionArgs);
6448
6449 if (Attrs.hasFnAttr(Kind: llvm::Attribute::Convergent)) {
6450 addConvergenceControlBundle(Bundles&: BundleList);
6451
6452 // Remove call-site convergent attribute if requested, unless the call must
6453 // stay convergent to match its convergence token and the callee doesn't
6454 // already make it convergent.
6455 //
6456 // FIXME: This means [[clang::noconvergent]] can't make an indirect call
6457 // non-convergent when using convergence tokens. A call without a token can
6458 // later be made direct to a convergent function, which mixes controlled
6459 // and uncontrolled convergence and fails the verifier.
6460 auto *CalleeFn = dyn_cast<llvm::Function>(Val: CalleePtr);
6461 if (InNoConvergentAttributedStmt &&
6462 (!CGM.shouldEmitConvergenceTokens() ||
6463 (CalleeFn && CalleeFn->isConvergent()))) {
6464 Attrs = Attrs.removeFnAttribute(C&: getLLVMContext(),
6465 Kind: llvm::Attribute::Convergent);
6466 }
6467 }
6468
6469 // Emit the actual call/invoke instruction.
6470 llvm::CallBase *CI;
6471 if (!InvokeDest) {
6472 CI = Builder.CreateCall(FTy: IRFuncTy, Callee: CalleePtr, Args: IRCallArgs, OpBundles: BundleList);
6473 } else {
6474 llvm::BasicBlock *Cont = createBasicBlock(name: "invoke.cont");
6475 CI = Builder.CreateInvoke(Ty: IRFuncTy, Callee: CalleePtr, NormalDest: Cont, UnwindDest: InvokeDest, Args: IRCallArgs,
6476 OpBundles: BundleList);
6477 EmitBlock(BB: Cont);
6478 }
6479 if (CI->getCalledFunction() && CI->getCalledFunction()->hasName() &&
6480 CI->getCalledFunction()->getName().starts_with(Prefix: "_Z4sqrt")) {
6481 SetSqrtFPAccuracy(CI);
6482 }
6483 if (callOrInvoke) {
6484 *callOrInvoke = CI;
6485 if (CGM.getCodeGenOpts().CallGraphSection) {
6486 QualType CST;
6487 if (TargetDecl && TargetDecl->getFunctionType())
6488 CST = QualType(TargetDecl->getFunctionType(), 0);
6489 else if (const auto *FPT =
6490 Callee.getAbstractInfo().getCalleeFunctionProtoType())
6491 CST = QualType(FPT, 0);
6492 else if (const auto *FT =
6493 Callee.getAbstractInfo().getCalleeFunctionType())
6494 CST = QualType(FT, 0);
6495 else
6496 llvm_unreachable(
6497 "Cannot find the callee type to generate callee_type metadata.");
6498
6499 // Set type identifier metadata of indirect calls for call graph section.
6500 if (!CST.isNull()) {
6501 if (!CST->isFunctionProtoType()) {
6502 // Reconstruct a prototype for unprototyped callees from the argument
6503 // types passed at the call site (after default argument promotion).
6504 //
6505 // Basic Rationale & K&R-Style Definitions:
6506 // The argument types in CallArgs have already undergone C default
6507 // argument promotion (e.g., char/short -> int, float -> double).
6508 // Furthermore, for a K&R-style
6509 // definition (e.g., void foo(x) short x; { ... }), canonical C ABI
6510 // semantics expect the promoted type (int) at the call boundary
6511 // and implicitly cast down to the declared type (short) inside the
6512 // function. Therefore, signature computation at K&R definition
6513 // sites must also apply default argument promotion (yielding
6514 // void(int), not void(short)) so definition and call sites match.
6515 //
6516 // Signature Strictness & Normalization:
6517 // Since type identifier matching relies on exact hash equality, any
6518 // tolerance for C compatibility rules must be done by normalizing
6519 // types before hashing.
6520 // - Standard C allows certain exceptions for unprototyped calls (and
6521 // variadic va_arg), such as differences in signedness (e.g.,
6522 // passing an int to an unsigned int parameter) or
6523 // interchangeability of enum types with their underlying integer
6524 // types.
6525 // - Existing CFI normalization (e.g.,
6526 // -fsanitize-cfi-icall-experimental-normalize-integers) normalizes
6527 // types by bit-width and signedness (e.g., int vs long on LP64,
6528 // which C does not treat as compatible), but does not normalize
6529 // away signedness or enum mismatches.
6530 // - In the future, whether to normalize away signedness, enums, or
6531 // integer bit-widths depends on whether call graph analysis should
6532 // err on the side of inclusion (admitting any C-valid call) or
6533 // strictness (like CFI). Any normalization applied here at the call
6534 // site must remain strictly matched with definition-site
6535 // type signature computation.
6536 if (const auto *FNPT = CST->getAs<FunctionNoProtoType>()) {
6537 SmallVector<QualType, 8> ParamTypes;
6538 // CallArgs already contains default-promoted argument types for
6539 // unprototyped calls.
6540 for (const CallArg &Arg : CallArgs)
6541 ParamTypes.push_back(Elt: Arg.getType());
6542 CST = CGM.ReconstructCallGraphPrototype(FNPT, ParamTypes);
6543 }
6544
6545 llvm::Metadata *MD =
6546 CGM.CreateMetadataIdentifierForCallGraphType(T: CST);
6547 StringRef TypeStr;
6548 if (auto *MDS = dyn_cast_or_null<llvm::MDString>(Val: MD))
6549 TypeStr = MDS->getString();
6550
6551 CGM.getDiags().Report(Loc, DiagID: diag::warn_cgs_no_proto) << CST << TypeStr;
6552 }
6553 CGM.createCalleeTypeMetadataForIcall(QT: CST, CB: *callOrInvoke);
6554 }
6555 }
6556 }
6557
6558 // If this is within a function that has the guard(nocf) attribute and is an
6559 // indirect call, add the "guard_nocf" attribute to this call to indicate that
6560 // Control Flow Guard checks should not be added, even if the call is inlined.
6561 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl)) {
6562 if (const auto *A = FD->getAttr<CFGuardAttr>()) {
6563 if (A->getGuard() == CFGuardAttr::GuardArg::nocf &&
6564 !CI->getCalledFunction())
6565 Attrs = Attrs.addFnAttribute(C&: getLLVMContext(), Kind: "guard_nocf");
6566 }
6567 }
6568
6569 // Apply the attributes and calling convention.
6570 CI->setAttributes(Attrs);
6571 CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
6572
6573 // Apply various metadata.
6574
6575 if (!CI->getType()->isVoidTy())
6576 CI->setName("call");
6577
6578 // Update largest vector width from the return type.
6579 LargestVectorWidth =
6580 std::max(a: LargestVectorWidth, b: getMaxVectorWidth(Ty: CI->getType()));
6581
6582 // Insert instrumentation or attach profile metadata at indirect call sites.
6583 // For more details, see the comment before the definition of
6584 // IPVK_IndirectCallTarget in InstrProfData.inc.
6585 if (!CI->getCalledFunction())
6586 PGO->valueProfile(Builder, ValueKind: llvm::IPVK_IndirectCallTarget, ValueSite: CI, ValuePtr: CalleePtr);
6587
6588 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
6589 // optimizer it can aggressively ignore unwind edges.
6590 if (CGM.getLangOpts().ObjCAutoRefCount)
6591 AddObjCARCExceptionMetadata(Inst: CI);
6592
6593 // Set tail call kind if necessary.
6594 bool IsPPC = getTarget().getTriple().isPPC();
6595 bool IsMIPS = getTarget().getTriple().isMIPS();
6596 bool HasMips16 = false;
6597 if (IsMIPS) {
6598 const TargetOptions &TargetOpts = getTarget().getTargetOpts();
6599 HasMips16 = TargetOpts.FeatureMap.lookup(Key: "mips16");
6600 if (!HasMips16)
6601 HasMips16 = llvm::is_contained(Range: TargetOpts.Features, Element: "+mips16");
6602 }
6603 if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(Val: CI)) {
6604 if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
6605 Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
6606 else if (IsMustTail) {
6607 if (IsPPC) {
6608 if (getTarget().getTriple().isOSAIX())
6609 CGM.getDiags().Report(Loc, DiagID: diag::err_aix_musttail_unsupported);
6610 else if (!getTarget().hasFeature(Feature: "pcrelative-memops")) {
6611 if (getTarget().hasFeature(Feature: "longcall"))
6612 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail) << 0;
6613 else if (Call->isIndirectCall())
6614 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail) << 1;
6615 else if (isa_and_nonnull<FunctionDecl>(Val: TargetDecl)) {
6616 if (!cast<FunctionDecl>(Val: TargetDecl)->isDefined())
6617 // The undefined callee may be a forward declaration. Without
6618 // knowning all symbols in the module, we won't know the symbol is
6619 // defined or not. Collect all these symbols for later diagnosing.
6620 CGM.addUndefinedGlobalForTailCall(
6621 Global: {cast<FunctionDecl>(Val: TargetDecl), Loc});
6622 else {
6623 llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(
6624 GD: GlobalDecl(cast<FunctionDecl>(Val: TargetDecl)));
6625 if (llvm::GlobalValue::isWeakForLinker(Linkage) ||
6626 llvm::GlobalValue::isDiscardableIfUnused(Linkage))
6627 CGM.getDiags().Report(Loc, DiagID: diag::err_ppc_impossible_musttail)
6628 << 2;
6629 }
6630 }
6631 }
6632 }
6633 if (IsMIPS) {
6634 if (HasMips16)
6635 CGM.getDiags().Report(Loc, DiagID: diag::err_mips_impossible_musttail) << 0;
6636 else if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: TargetDecl))
6637 CGM.addUndefinedGlobalForTailCall(Global: {FD, Loc});
6638 }
6639 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
6640 }
6641 }
6642
6643 // Add metadata for calls to MSAllocator functions
6644 if (getDebugInfo() && TargetDecl && TargetDecl->hasAttr<MSAllocatorAttr>())
6645 getDebugInfo()->addHeapAllocSiteMetadata(CallSite: CI, AllocatedTy: RetTy->getPointeeType(), Loc);
6646
6647 // Add srcloc metadata for [[gnu::error/warning]] diagnostics. When
6648 // ShowInliningChain is enabled, also track inline/static calls for the
6649 // heuristic fallback when debug info is not available. This heuristic is
6650 // conservative and best-effort since static or inline-annotated functions
6651 // are still not guaranteed to be inlined.
6652 if (TargetDecl) {
6653 bool NeedSrcLoc = TargetDecl->hasAttr<ErrorAttr>();
6654 if (!NeedSrcLoc && CGM.getCodeGenOpts().ShowInliningChain) {
6655 if (const auto *FD = dyn_cast<FunctionDecl>(Val: TargetDecl))
6656 NeedSrcLoc = FD->isInlined() || FD->hasAttr<AlwaysInlineAttr>() ||
6657 FD->getStorageClass() == SC_Static ||
6658 FD->isInAnonymousNamespace();
6659 }
6660 if (NeedSrcLoc) {
6661 auto *Line = llvm::ConstantInt::get(Ty: Int64Ty, V: Loc.getRawEncoding());
6662 auto *MD = llvm::ConstantAsMetadata::get(C: Line);
6663 CI->setMetadata(Kind: "srcloc", Node: llvm::MDNode::get(Context&: getLLVMContext(), MDs: {MD}));
6664 }
6665 }
6666
6667 // 4. Finish the call.
6668
6669 // If the call doesn't return, finish the basic block and clear the
6670 // insertion point; this allows the rest of IRGen to discard
6671 // unreachable code.
6672 if (CI->doesNotReturn()) {
6673 if (NeedSRetLifetimeEnd)
6674 PopCleanupBlock();
6675
6676 // Strip away the noreturn attribute to better diagnose unreachable UB.
6677 if (SanOpts.has(K: SanitizerKind::Unreachable)) {
6678 // Also remove from function since CallBase::hasFnAttr additionally checks
6679 // attributes of the called function.
6680 if (auto *F = CI->getCalledFunction())
6681 F->removeFnAttr(Kind: llvm::Attribute::NoReturn);
6682 CI->removeFnAttr(Kind: llvm::Attribute::NoReturn);
6683
6684 // Avoid incompatibility with ASan which relies on the `noreturn`
6685 // attribute to insert handler calls.
6686 if (SanOpts.hasOneOf(K: SanitizerKind::Address |
6687 SanitizerKind::KernelAddress)) {
6688 SanitizerScope SanScope(this);
6689 llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);
6690 Builder.SetInsertPoint(CI);
6691 auto *FnType = llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg=*/false);
6692 llvm::FunctionCallee Fn =
6693 CGM.CreateRuntimeFunction(Ty: FnType, Name: "__asan_handle_no_return");
6694 EmitNounwindRuntimeCall(callee: Fn);
6695 }
6696 }
6697
6698 EmitUnreachable(Loc);
6699 Builder.ClearInsertionPoint();
6700
6701 // FIXME: For now, emit a dummy basic block because expr emitters in
6702 // generally are not ready to handle emitting expressions at unreachable
6703 // points.
6704 EnsureInsertPoint();
6705
6706 // Return a reasonable RValue.
6707 return GetUndefRValue(Ty: RetTy);
6708 }
6709
6710 // If this is a musttail call, return immediately. We do not branch to the
6711 // epilogue in this case.
6712 if (IsMustTail) {
6713 for (auto it = EHStack.find(sp: CurrentCleanupScopeDepth); it != EHStack.end();
6714 ++it) {
6715 // A noexcept caller pushes an EHTerminateScope to call std::terminate()
6716 // if an exception escapes. A musttail call replaces the caller's frame,
6717 // removing this handler. This is safe if the callee is also nounwind:
6718 // the callee's own noexcept handler prevents any exception from reaching
6719 // where the caller's handler would have been.
6720 if (isa<EHTerminateScope>(Val: &*it)) {
6721 if (CI->doesNotThrow())
6722 continue;
6723 CGM.getDiags().Report(Loc: MustTailCall->getBeginLoc(),
6724 DiagID: diag::err_musttail_noexcept_mismatch);
6725 break;
6726 }
6727 if (auto *Cleanup = dyn_cast<EHCleanupScope>(Val: &*it)) {
6728 // Fake uses can be safely emitted immediately prior to the tail call,
6729 // so we choose to emit them just before the call here.
6730 if (Cleanup->isFakeUse()) {
6731 CGBuilderTy::InsertPointGuard IPG(Builder);
6732 Builder.SetInsertPoint(CI);
6733 Cleanup->getCleanup()->Emit(CGF&: *this, flags: EHScopeStack::Cleanup::Flags());
6734 continue;
6735 }
6736 if (Cleanup->isRedundantBeforeReturn())
6737 continue;
6738 }
6739 CGM.ErrorUnsupported(S: MustTailCall, Type: "tail call skipping over cleanups");
6740 }
6741 if (CI->getType()->isVoidTy())
6742 Builder.CreateRetVoid();
6743 else
6744 Builder.CreateRet(V: CI);
6745 Builder.ClearInsertionPoint();
6746 EnsureInsertPoint();
6747 return GetUndefRValue(Ty: RetTy);
6748 }
6749
6750 // Perform the swifterror writeback.
6751 if (swiftErrorTemp.isValid()) {
6752 llvm::Value *errorResult = Builder.CreateLoad(Addr: swiftErrorTemp);
6753 Builder.CreateStore(Val: errorResult, Addr: swiftErrorArg);
6754 }
6755
6756 // Emit any call-associated writebacks immediately. Arguably this
6757 // should happen after any return-value munging.
6758 if (CallArgs.hasWritebacks())
6759 EmitWritebacks(args: CallArgs);
6760
6761 // The stack cleanup for inalloca arguments has to run out of the normal
6762 // lexical order, so deactivate it and run it manually here.
6763 CallArgs.freeArgumentMemory(CGF&: *this);
6764
6765 // Extract the return value.
6766 RValue Ret;
6767
6768 // If the current function is a virtual function pointer thunk, avoid copying
6769 // the return value of the musttail call to a temporary.
6770 if (IsVirtualFunctionPointerThunk) {
6771 Ret = RValue::get(V: CI);
6772 } else {
6773 Ret = [&] {
6774 switch (RetAI.getKind()) {
6775 case ABIArgInfo::CoerceAndExpand: {
6776 auto coercionType = RetAI.getCoerceAndExpandType();
6777
6778 Address addr = SRetPtr.withElementType(ElemTy: coercionType);
6779
6780 assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());
6781 bool requiresExtract = isa<llvm::StructType>(Val: CI->getType());
6782
6783 unsigned unpaddedIndex = 0;
6784 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
6785 llvm::Type *eltType = coercionType->getElementType(N: i);
6786 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
6787 continue;
6788 Address eltAddr = Builder.CreateStructGEP(Addr: addr, Index: i);
6789 llvm::Value *elt = CI;
6790 if (requiresExtract)
6791 elt = Builder.CreateExtractValue(Agg: elt, Idxs: unpaddedIndex++);
6792 else
6793 assert(unpaddedIndex == 0);
6794 Builder.CreateStore(Val: elt, Addr: eltAddr);
6795 }
6796 [[fallthrough]];
6797 }
6798
6799 case ABIArgInfo::InAlloca:
6800 case ABIArgInfo::Indirect: {
6801 RValue ret = convertTempToRValue(addr: SRetPtr, type: RetTy, Loc: SourceLocation());
6802 if (NeedSRetLifetimeEnd)
6803 PopCleanupBlock();
6804 return ret;
6805 }
6806
6807 case ABIArgInfo::Ignore:
6808 // If we are ignoring an argument that had a result, make sure to
6809 // construct the appropriate return value for our caller.
6810 return GetUndefRValue(Ty: RetTy);
6811
6812 case ABIArgInfo::Extend:
6813 case ABIArgInfo::Direct: {
6814 llvm::Type *RetIRTy = ConvertType(T: RetTy);
6815 if (RetAI.getCoerceToType() == RetIRTy &&
6816 RetAI.getDirectOffset() == 0) {
6817 switch (getEvaluationKind(T: RetTy)) {
6818 case TEK_Complex: {
6819 llvm::Value *Real = Builder.CreateExtractValue(Agg: CI, Idxs: 0);
6820 llvm::Value *Imag = Builder.CreateExtractValue(Agg: CI, Idxs: 1);
6821 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
6822 }
6823 case TEK_Aggregate:
6824 break;
6825 case TEK_Scalar: {
6826 // If the argument doesn't match, perform a bitcast to coerce it.
6827 // This can happen due to trivial type mismatches.
6828 llvm::Value *V = CI;
6829 if (V->getType() != RetIRTy)
6830 V = Builder.CreateBitCast(V, DestTy: RetIRTy);
6831 return RValue::get(V);
6832 }
6833 }
6834 }
6835
6836 // If coercing a fixed vector from a scalable vector for ABI
6837 // compatibility, and the types match, use the llvm.vector.extract
6838 // intrinsic to perform the conversion.
6839 if (auto *FixedDstTy = dyn_cast<llvm::FixedVectorType>(Val: RetIRTy)) {
6840 llvm::Value *V = CI;
6841 if (auto *ScalableSrcTy =
6842 dyn_cast<llvm::ScalableVectorType>(Val: V->getType())) {
6843 if (FixedDstTy->getElementType() ==
6844 ScalableSrcTy->getElementType()) {
6845 V = Builder.CreateExtractVector(DstType: FixedDstTy, SrcVec: V, Idx: uint64_t(0),
6846 Name: "cast.fixed");
6847 return RValue::get(V);
6848 }
6849 }
6850 }
6851
6852 Address DestPtr = ReturnValue.getValue();
6853 bool DestIsVolatile = ReturnValue.isVolatile();
6854 uint64_t DestSize =
6855 getContext().getTypeInfoDataSizeInChars(T: RetTy).Width.getQuantity();
6856
6857 if (!DestPtr.isValid()) {
6858 DestPtr = CreateMemTempWithoutCast(T: RetTy, Name: "coerce");
6859 DestIsVolatile = false;
6860 DestSize = getContext().getTypeSizeInChars(T: RetTy).getQuantity();
6861 }
6862
6863 // An empty record can overlap other data (if declared with
6864 // no_unique_address); omit the store for such types - as there is no
6865 // actual data to store.
6866 if (!isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true)) {
6867 // If the value is offset in memory, apply the offset now.
6868 Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
6869 CreateCoercedStore(
6870 Src: CI, SrcFETy: RetTy, Dst: StorePtr,
6871 DstSize: llvm::TypeSize::getFixed(ExactSize: DestSize - RetAI.getDirectOffset()),
6872 DstIsVolatile: DestIsVolatile);
6873 }
6874
6875 return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation());
6876 }
6877
6878 case ABIArgInfo::TargetSpecific: {
6879 Address DestPtr = ReturnValue.getValue();
6880 Address StorePtr = emitAddressAtOffset(CGF&: *this, addr: DestPtr, info: RetAI);
6881 bool DestIsVolatile = ReturnValue.isVolatile();
6882 if (!DestPtr.isValid()) {
6883 DestPtr = CreateMemTempWithoutCast(T: RetTy, Name: "target_coerce");
6884 DestIsVolatile = false;
6885 }
6886 CGM.getABIInfo().createCoercedStore(Val: CI, DstAddr: StorePtr, AI: RetAI, DestIsVolatile,
6887 CGF&: *this);
6888 return convertTempToRValue(addr: DestPtr, type: RetTy, Loc: SourceLocation());
6889 }
6890
6891 case ABIArgInfo::Expand:
6892 case ABIArgInfo::IndirectAliased:
6893 llvm_unreachable("Invalid ABI kind for return argument");
6894 }
6895
6896 llvm_unreachable("Unhandled ABIArgInfo::Kind");
6897 }();
6898 }
6899
6900 // Emit the assume_aligned check on the return value.
6901 if (Ret.isScalar() && TargetDecl) {
6902 AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
6903 }
6904
6905 // Emit the end lifetime markers for any temporary allocas created for the
6906 // call arguments.
6907 for (llvm::Value *LifetimeEndAddr : CallLifetimeEndAfterCall)
6908 EmitLifetimeEnd(Addr: LifetimeEndAddr);
6909
6910 if (!ReturnValue.isExternallyDestructed() &&
6911 RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct)
6912 pushDestroy(dtorKind: QualType::DK_nontrivial_c_struct, addr: Ret.getAggregateAddress(),
6913 type: RetTy);
6914
6915 // Generate function declaration DISuprogram in order to be used
6916 // in debug info about call sites.
6917 if (CGDebugInfo *DI = getDebugInfo()) {
6918 // Ensure call site info would actually be emitted before collecting
6919 // further callee info.
6920 if (CalleeDecl && !CalleeDecl->hasAttr<NoDebugAttr>() &&
6921 DI->getCallSiteRelatedAttrs() != llvm::DINode::FlagZero) {
6922 CodeGenFunction CalleeCGF(CGM);
6923 const GlobalDecl &CalleeGlobalDecl =
6924 Callee.getAbstractInfo().getCalleeDecl();
6925 CalleeCGF.CurGD = CalleeGlobalDecl;
6926 FunctionArgList Args;
6927 QualType ResTy = CalleeCGF.BuildFunctionArgList(GD: CalleeGlobalDecl, Args);
6928 DI->EmitFuncDeclForCallSite(
6929 CallOrInvoke: CI, CalleeType: DI->getFunctionType(FD: CalleeDecl, RetTy: ResTy, Args), CalleeGlobalDecl);
6930 }
6931 // Generate call site target information.
6932 DI->addCallTargetIfVirtual(FD: CalleeDecl, CI);
6933 }
6934
6935 return Ret;
6936}
6937
6938CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const {
6939 if (isVirtual()) {
6940 const CallExpr *CE = getVirtualCallExpr();
6941 return CGF.CGM.getCXXABI().getVirtualFunctionPointer(
6942 CGF, GD: getVirtualMethodDecl(), This: getThisAddress(), Ty: getVirtualFunctionType(),
6943 Loc: CE ? CE->getBeginLoc() : SourceLocation());
6944 }
6945
6946 return *this;
6947}
6948
6949/* VarArg handling */
6950
6951RValue CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
6952 AggValueSlot Slot) {
6953 VAListAddr = VE->isMicrosoftABI()
6954 ? EmitMSVAListRef(E: VE->getSubExpr())
6955 : (VE->isZOSABI() ? EmitZOSVAListRef(E: VE->getSubExpr())
6956 : EmitVAListRef(E: VE->getSubExpr()));
6957 QualType Ty = VE->getType();
6958 if (Ty->isVariablyModifiedType())
6959 EmitVariablyModifiedType(Ty);
6960 if (VE->isMicrosoftABI())
6961 return CGM.getABIInfo().EmitMSVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6962 if (VE->isZOSABI())
6963 return CGM.getABIInfo().EmitZOSVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6964 return CGM.getABIInfo().EmitVAArg(CGF&: *this, VAListAddr, Ty, Slot);
6965}
6966
6967DisableDebugLocationUpdates::DisableDebugLocationUpdates(CodeGenFunction &CGF)
6968 : CGF(CGF) {
6969 CGF.disableDebugInfo();
6970}
6971
6972DisableDebugLocationUpdates::~DisableDebugLocationUpdates() {
6973 CGF.enableDebugInfo();
6974}
6975