1//===--- CodeGenFunction.cpp - Emit LLVM Code from ASTs for a Function ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This coordinates the per-function state used while generating code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CodeGenFunction.h"
14#include "CGBlocks.h"
15#include "CGCUDARuntime.h"
16#include "CGCXXABI.h"
17#include "CGCleanup.h"
18#include "CGDebugInfo.h"
19#include "CGHLSLRuntime.h"
20#include "CGOpenMPRuntime.h"
21#include "CodeGenModule.h"
22#include "CodeGenPGO.h"
23#include "TargetInfo.h"
24#include "clang/AST/ASTContext.h"
25#include "clang/AST/ASTLambda.h"
26#include "clang/AST/Attr.h"
27#include "clang/AST/Decl.h"
28#include "clang/AST/DeclCXX.h"
29#include "clang/AST/Expr.h"
30#include "clang/AST/IgnoreExpr.h"
31#include "clang/AST/StmtCXX.h"
32#include "clang/AST/StmtObjC.h"
33#include "clang/Basic/Builtins.h"
34#include "clang/Basic/CodeGenOptions.h"
35#include "clang/Basic/DiagnosticFrontend.h"
36#include "clang/Basic/TargetBuiltins.h"
37#include "clang/Basic/TargetInfo.h"
38#include "clang/CodeGen/CGFunctionInfo.h"
39#include "clang/CodeGenUtils/CodeGenUtils.h"
40#include "llvm/ADT/ArrayRef.h"
41#include "llvm/ADT/ScopeExit.h"
42#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
43#include "llvm/IR/DataLayout.h"
44#include "llvm/IR/Dominators.h"
45#include "llvm/IR/FPEnv.h"
46#include "llvm/IR/Instruction.h"
47#include "llvm/IR/IntrinsicInst.h"
48#include "llvm/IR/Intrinsics.h"
49#include "llvm/IR/IntrinsicsPowerPC.h"
50#include "llvm/IR/MDBuilder.h"
51#include "llvm/Support/CRC.h"
52#include "llvm/Support/SaveAndRestore.h"
53#include "llvm/Support/SipHash.h"
54#include "llvm/Support/xxhash.h"
55#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
56#include "llvm/Transforms/Utils/PromoteMemToReg.h"
57#include <optional>
58
59using namespace clang;
60using namespace CodeGen;
61
62/// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time
63/// markers.
64static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts,
65 const LangOptions &LangOpts) {
66 if (CGOpts.DisableLifetimeMarkers)
67 return false;
68
69 // Sanitizers may use markers.
70 if (CGOpts.SanitizeAddressUseAfterScope ||
71 LangOpts.Sanitize.has(K: SanitizerKind::HWAddress) ||
72 LangOpts.Sanitize.has(K: SanitizerKind::Memory) ||
73 LangOpts.Sanitize.has(K: SanitizerKind::MemtagStack))
74 return true;
75
76 // For now, only in optimized builds.
77 return CGOpts.OptimizationLevel != 0;
78}
79
80CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
81 : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
82 Builder(cgm, cgm.getModule().getContext(), CGBuilderInserterTy(this)),
83 SanOpts(CGM.getLangOpts().Sanitize), CurFPFeatures(CGM.getLangOpts()),
84 DebugInfo(CGM.getModuleDebugInfo()),
85 PGO(std::make_unique<CodeGenPGO>(args&: cgm)),
86 ShouldEmitLifetimeMarkers(
87 shouldEmitLifetimeMarkers(CGOpts: CGM.getCodeGenOpts(), LangOpts: CGM.getLangOpts())) {
88 if (!suppressNewContext)
89 CGM.getCXXABI().getMangleContext().startNewFunction();
90 EHStack.setCGF(this);
91
92 SetFastMathFlags(CurFPFeatures);
93}
94
95const FunctionDecl *CodeGenFunction::getCurrentFunctionDecl() const {
96 const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
97 if (!FD)
98 FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl);
99 return FD;
100}
101
102CodeGenFunction::~CodeGenFunction() {
103 assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup");
104 assert(DeferredDeactivationCleanupStack.empty() &&
105 "missed to deactivate a cleanup");
106
107 if (getLangOpts().OpenMP && CurFn)
108 CGM.getOpenMPRuntime().functionFinished(CGF&: *this);
109
110 // If we have an OpenMPIRBuilder we want to finalize functions (incl.
111 // outlining etc) at some point. Doing it once the function codegen is done
112 // seems to be a reasonable spot. We do it here, as opposed to the deletion
113 // time of the CodeGenModule, because we have to ensure the IR has not yet
114 // been "emitted" to the outside, thus, modifications are still sensible.
115 if (CGM.getLangOpts().OpenMPIRBuilder && CurFn)
116 CGM.getOpenMPRuntime().getOMPBuilder().finalize(Fn: CurFn);
117}
118
119// Map the LangOption for exception behavior into
120// the corresponding enum in the IR.
121llvm::fp::ExceptionBehavior
122clang::ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind) {
123
124 switch (Kind) {
125 case LangOptions::FPE_Ignore: return llvm::fp::ebIgnore;
126 case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap;
127 case LangOptions::FPE_Strict: return llvm::fp::ebStrict;
128 default:
129 llvm_unreachable("Unsupported FP Exception Behavior");
130 }
131}
132
133void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) {
134 llvm::FastMathFlags FMF;
135 FMF.setAllowReassoc(FPFeatures.getAllowFPReassociate());
136 FMF.setNoNaNs(FPFeatures.getNoHonorNaNs());
137 FMF.setNoInfs(FPFeatures.getNoHonorInfs());
138 FMF.setNoSignedZeros(FPFeatures.getNoSignedZero());
139 FMF.setAllowReciprocal(FPFeatures.getAllowReciprocal());
140 FMF.setApproxFunc(FPFeatures.getAllowApproxFunc());
141 FMF.setAllowContract(FPFeatures.allowFPContractAcrossStatement());
142 Builder.setFastMathFlags(FMF);
143}
144
145CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
146 const Expr *E)
147 : CGF(CGF) {
148 ConstructorHelper(FPFeatures: E->getFPFeaturesInEffect(LO: CGF.getLangOpts()));
149}
150
151CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
152 FPOptions FPFeatures)
153 : CGF(CGF) {
154 ConstructorHelper(FPFeatures);
155}
156
157void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) {
158 OldFPFeatures = CGF.CurFPFeatures;
159 CGF.CurFPFeatures = FPFeatures;
160
161 OldExcept = CGF.Builder.getDefaultConstrainedExcept();
162 OldRounding = CGF.Builder.getDefaultConstrainedRounding();
163
164 if (OldFPFeatures == FPFeatures)
165 return;
166
167 FMFGuard.emplace(args&: CGF.Builder);
168
169 llvm::RoundingMode NewRoundingBehavior = FPFeatures.getRoundingMode();
170 CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);
171 auto NewExceptionBehavior =
172 ToConstrainedExceptMD(Kind: FPFeatures.getExceptionMode());
173 CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);
174
175 CGF.SetFastMathFlags(FPFeatures);
176
177 assert((CGF.CurFuncDecl == nullptr || CGF.Builder.getIsFPConstrained() ||
178 isa<CXXConstructorDecl>(CGF.CurFuncDecl) ||
179 isa<CXXDestructorDecl>(CGF.CurFuncDecl) ||
180 (NewExceptionBehavior == llvm::fp::ebIgnore &&
181 NewRoundingBehavior == llvm::RoundingMode::NearestTiesToEven)) &&
182 "FPConstrained should be enabled on entire function");
183
184 auto mergeFnAttrValue = [&](StringRef Name, bool Value) {
185 auto OldValue =
186 CGF.CurFn->getFnAttribute(Kind: Name).getValueAsBool();
187 auto NewValue = OldValue & Value;
188 if (OldValue != NewValue)
189 CGF.CurFn->addFnAttr(Kind: Name, Val: llvm::toStringRef(B: NewValue));
190 };
191 mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.getNoSignedZero());
192}
193
194CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
195 CGF.CurFPFeatures = OldFPFeatures;
196 CGF.Builder.setDefaultConstrainedExcept(OldExcept);
197 CGF.Builder.setDefaultConstrainedRounding(OldRounding);
198}
199
200static LValue
201makeNaturalAlignAddrLValue(llvm::Value *V, QualType T, bool ForPointeeType,
202 bool MightBeSigned, CodeGenFunction &CGF,
203 KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
204 LValueBaseInfo BaseInfo;
205 TBAAAccessInfo TBAAInfo;
206 CharUnits Alignment =
207 CGF.CGM.getNaturalTypeAlignment(T, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo, forPointeeType: ForPointeeType);
208 Address Addr =
209 MightBeSigned
210 ? CGF.makeNaturalAddressForPointer(Ptr: V, T, Alignment, ForPointeeType: false, BaseInfo: nullptr,
211 TBAAInfo: nullptr, IsKnownNonNull)
212 : Address(V, CGF.ConvertTypeForMem(T), Alignment, IsKnownNonNull);
213 return CGF.MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
214}
215
216LValue
217CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
218 KnownNonNull_t IsKnownNonNull) {
219 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,
220 /*MightBeSigned*/ true, CGF&: *this,
221 IsKnownNonNull);
222}
223
224LValue
225CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
226 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,
227 /*MightBeSigned*/ true, CGF&: *this);
228}
229
230LValue CodeGenFunction::MakeNaturalAlignRawAddrLValue(llvm::Value *V,
231 QualType T) {
232 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,
233 /*MightBeSigned*/ false, CGF&: *this);
234}
235
236LValue CodeGenFunction::MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V,
237 QualType T) {
238 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,
239 /*MightBeSigned*/ false, CGF&: *this);
240}
241
242llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
243 return CGM.getTypes().ConvertTypeForMem(T);
244}
245
246llvm::Type *CodeGenFunction::ConvertType(QualType T) {
247 return CGM.getTypes().ConvertType(T);
248}
249
250llvm::Type *CodeGenFunction::convertTypeForLoadStore(QualType ASTTy,
251 llvm::Type *LLVMTy) {
252 return CGM.getTypes().convertTypeForLoadStore(T: ASTTy, LLVMTy);
253}
254
255TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
256 type = type.getCanonicalType();
257 while (true) {
258 switch (type->getTypeClass()) {
259#define TYPE(name, parent)
260#define ABSTRACT_TYPE(name, parent)
261#define NON_CANONICAL_TYPE(name, parent) case Type::name:
262#define DEPENDENT_TYPE(name, parent) case Type::name:
263#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
264#include "clang/AST/TypeNodes.inc"
265 llvm_unreachable("non-canonical or dependent type in IR-generation");
266
267 case Type::Auto:
268 case Type::DeducedTemplateSpecialization:
269 llvm_unreachable("undeduced type in IR-generation");
270
271 // Various scalar types.
272 case Type::Builtin:
273 case Type::Pointer:
274 case Type::BlockPointer:
275 case Type::LValueReference:
276 case Type::RValueReference:
277 case Type::MemberPointer:
278 case Type::Vector:
279 case Type::ExtVector:
280 case Type::ConstantMatrix:
281 case Type::FunctionProto:
282 case Type::FunctionNoProto:
283 case Type::Enum:
284 case Type::ObjCObjectPointer:
285 case Type::Pipe:
286 case Type::BitInt:
287 case Type::HLSLAttributedResource:
288 case Type::HLSLInlineSpirv:
289 case Type::OverflowBehavior:
290 return TEK_Scalar;
291
292 // Complexes.
293 case Type::Complex:
294 return TEK_Complex;
295
296 // Arrays, records, and Objective-C objects.
297 case Type::ConstantArray:
298 case Type::IncompleteArray:
299 case Type::VariableArray:
300 case Type::Record:
301 case Type::ObjCObject:
302 case Type::ObjCInterface:
303 case Type::ArrayParameter:
304 return TEK_Aggregate;
305
306 // We operate on atomic values according to their underlying type.
307 case Type::Atomic:
308 type = cast<AtomicType>(Val&: type)->getValueType();
309 continue;
310 }
311 llvm_unreachable("unknown type kind!");
312 }
313}
314
315llvm::DebugLoc CodeGenFunction::EmitReturnBlock() {
316 // For cleanliness, we try to avoid emitting the return block for
317 // simple cases.
318 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
319
320 if (CurBB) {
321 assert(!CurBB->hasTerminator() && "Unexpected terminated block.");
322
323 // We have a valid insert point, reuse it if it is empty or there are no
324 // explicit jumps to the return block.
325 if (CurBB->empty() || ReturnBlock.getBlock()->use_empty()) {
326 ReturnBlock.getBlock()->replaceAllUsesWith(V: CurBB);
327 delete ReturnBlock.getBlock();
328 ReturnBlock = JumpDest();
329 } else
330 EmitBlock(BB: ReturnBlock.getBlock());
331 return llvm::DebugLoc();
332 }
333
334 // Otherwise, if the return block is the target of a single direct
335 // branch then we can just put the code in that block instead. This
336 // cleans up functions which started with a unified return block.
337 if (ReturnBlock.getBlock()->hasOneUse()) {
338 auto *BI =
339 dyn_cast<llvm::UncondBrInst>(Val: *ReturnBlock.getBlock()->user_begin());
340 if (BI && BI->getSuccessor(i: 0) == ReturnBlock.getBlock()) {
341 // Record/return the DebugLoc of the simple 'return' expression to be used
342 // later by the actual 'ret' instruction.
343 llvm::DebugLoc Loc = BI->getDebugLoc();
344 Builder.SetInsertPoint(BI->getParent());
345 BI->eraseFromParent();
346 delete ReturnBlock.getBlock();
347 ReturnBlock = JumpDest();
348 return Loc;
349 }
350 }
351
352 // FIXME: We are at an unreachable point, there is no reason to emit the block
353 // unless it has uses. However, we still need a place to put the debug
354 // region.end for now.
355
356 EmitBlock(BB: ReturnBlock.getBlock());
357 return llvm::DebugLoc();
358}
359
360static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
361 if (!BB) return;
362 if (!BB->use_empty()) {
363 CGF.CurFn->insert(Position: CGF.CurFn->end(), BB);
364 return;
365 }
366 delete BB;
367}
368
369void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
370 assert(BreakContinueStack.empty() &&
371 "mismatched push/pop in break/continue stack!");
372 assert(LifetimeExtendedCleanupStack.empty() &&
373 "mismatched push/pop of cleanups in EHStack!");
374 assert(DeferredDeactivationCleanupStack.empty() &&
375 "mismatched activate/deactivate of cleanups!");
376
377 if (CGM.shouldEmitConvergenceTokens()) {
378 ConvergenceTokenStack.pop_back();
379 assert(ConvergenceTokenStack.empty() &&
380 "mismatched push/pop in convergence stack!");
381 }
382
383 bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
384 && NumSimpleReturnExprs == NumReturnExprs
385 && ReturnBlock.getBlock()->use_empty();
386 // Usually the return expression is evaluated before the cleanup
387 // code. If the function contains only a simple return statement,
388 // such as a constant, the location before the cleanup code becomes
389 // the last useful breakpoint in the function, because the simple
390 // return expression will be evaluated after the cleanup code. To be
391 // safe, set the debug location for cleanup code to the location of
392 // the return statement. Otherwise the cleanup code should be at the
393 // end of the function's lexical scope.
394 //
395 // If there are multiple branches to the return block, the branch
396 // instructions will get the location of the return statements and
397 // all will be fine.
398 if (CGDebugInfo *DI = getDebugInfo()) {
399 if (OnlySimpleReturnStmts)
400 DI->EmitLocation(Builder, Loc: LastStopPoint);
401 else
402 DI->EmitLocation(Builder, Loc: EndLoc);
403 }
404
405 // Pop any cleanups that might have been associated with the
406 // parameters. Do this in whatever block we're currently in; it's
407 // important to do this before we enter the return block or return
408 // edges will be *really* confused.
409 bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth;
410 bool HasOnlyNoopCleanups =
411 HasCleanups && EHStack.containsOnlyNoopCleanups(Old: PrologueCleanupDepth);
412 bool EmitRetDbgLoc = !HasCleanups || HasOnlyNoopCleanups;
413
414 std::optional<ApplyDebugLocation> OAL;
415 if (HasCleanups) {
416 // Make sure the line table doesn't jump back into the body for
417 // the ret after it's been at EndLoc.
418 if (CGDebugInfo *DI = getDebugInfo()) {
419 if (OnlySimpleReturnStmts)
420 DI->EmitLocation(Builder, Loc: EndLoc);
421 else
422 // We may not have a valid end location. Try to apply it anyway, and
423 // fall back to an artificial location if needed.
424 OAL = ApplyDebugLocation::CreateDefaultArtificial(CGF&: *this, TemporaryLocation: EndLoc);
425 }
426
427 PopCleanupBlocks(OldCleanupStackSize: PrologueCleanupDepth);
428 }
429
430 // Emit function epilog (to return).
431 llvm::DebugLoc Loc = EmitReturnBlock();
432
433 if (ShouldInstrumentFunction()) {
434 if (CGM.getCodeGenOpts().InstrumentFunctions)
435 CurFn->addFnAttr(Kind: "instrument-function-exit", Val: "__cyg_profile_func_exit");
436 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
437 CurFn->addFnAttr(Kind: "instrument-function-exit-inlined",
438 Val: "__cyg_profile_func_exit");
439 }
440
441 // Emit debug descriptor for function end.
442 if (CGDebugInfo *DI = getDebugInfo())
443 DI->EmitFunctionEnd(Builder, Fn: CurFn);
444
445 // Reset the debug location to that of the simple 'return' expression, if any
446 // rather than that of the end of the function's scope '}'.
447 uint64_t RetKeyInstructionsAtomGroup = Loc ? Loc->getAtomGroup() : 0;
448 ApplyDebugLocation AL(*this, Loc);
449 EmitFunctionEpilog(FI: *CurFnInfo, EmitRetDbgLoc, EndLoc,
450 RetKeyInstructionsSourceAtom: RetKeyInstructionsAtomGroup);
451 EmitEndEHSpec(D: CurCodeDecl);
452
453 assert(EHStack.empty() &&
454 "did not remove all scopes from cleanup stack!");
455
456 // If someone did an indirect goto, emit the indirect goto block at the end of
457 // the function.
458 if (IndirectBranch) {
459 EmitBlock(BB: IndirectBranch->getParent());
460 Builder.ClearInsertionPoint();
461 }
462
463 // If some of our locals escaped, insert a call to llvm.localescape in the
464 // entry block.
465 if (!EscapedLocals.empty()) {
466 // Invert the map from local to index into a simple vector. There should be
467 // no holes.
468 SmallVector<llvm::Value *, 4> EscapeArgs;
469 EscapeArgs.resize(N: EscapedLocals.size());
470 for (auto &Pair : EscapedLocals)
471 EscapeArgs[Pair.second] = Pair.first;
472 llvm::Function *FrameEscapeFn = llvm::Intrinsic::getOrInsertDeclaration(
473 M: &CGM.getModule(), id: llvm::Intrinsic::localescape);
474 CGBuilderTy(CGM, AllocaInsertPt).CreateCall(Callee: FrameEscapeFn, Args: EscapeArgs);
475 }
476
477 // Remove the AllocaInsertPt instruction, which is just a convenience for us.
478 llvm::Instruction *Ptr = AllocaInsertPt;
479 AllocaInsertPt = nullptr;
480 Ptr->eraseFromParent();
481
482 // PostAllocaInsertPt, if created, was lazily created when it was required,
483 // remove it now since it was just created for our own convenience.
484 if (PostAllocaInsertPt) {
485 llvm::Instruction *PostPtr = PostAllocaInsertPt;
486 PostAllocaInsertPt = nullptr;
487 PostPtr->eraseFromParent();
488 }
489
490 // If someone took the address of a label but never did an indirect goto, we
491 // made a zero entry PHI node, which is illegal, zap it now.
492 if (IndirectBranch) {
493 llvm::PHINode *PN = cast<llvm::PHINode>(Val: IndirectBranch->getAddress());
494 if (PN->getNumIncomingValues() == 0) {
495 PN->replaceAllUsesWith(V: llvm::PoisonValue::get(T: PN->getType()));
496 PN->eraseFromParent();
497 }
498 }
499
500 EmitIfUsed(CGF&: *this, BB: EHResumeBlock);
501 EmitIfUsed(CGF&: *this, BB: TerminateLandingPad);
502 EmitIfUsed(CGF&: *this, BB: TerminateHandler);
503 EmitIfUsed(CGF&: *this, BB: UnreachableBlock);
504
505 for (const auto &FuncletAndParent : TerminateFunclets)
506 EmitIfUsed(CGF&: *this, BB: FuncletAndParent.second);
507
508 if (CGM.getCodeGenOpts().EmitDeclMetadata)
509 EmitDeclMetadata();
510
511 for (const auto &R : DeferredReplacements) {
512 if (llvm::Value *Old = R.first) {
513 Old->replaceAllUsesWith(V: R.second);
514 cast<llvm::Instruction>(Val: Old)->eraseFromParent();
515 }
516 }
517 DeferredReplacements.clear();
518
519 // Eliminate CleanupDestSlot alloca by replacing it with SSA values and
520 // PHIs if the current function is a coroutine. We don't do it for all
521 // functions as it may result in slight increase in numbers of instructions
522 // if compiled with no optimizations. We do it for coroutine as the lifetime
523 // of CleanupDestSlot alloca make correct coroutine frame building very
524 // difficult.
525 if (NormalCleanupDest.isValid() && isCoroutine()) {
526 llvm::DominatorTree DT(*CurFn);
527 llvm::PromoteMemToReg(
528 Allocas: cast<llvm::AllocaInst>(Val: NormalCleanupDest.getPointer()), DT);
529 NormalCleanupDest = Address::invalid();
530 }
531
532 // Scan function arguments for vector width.
533 for (llvm::Argument &A : CurFn->args())
534 if (auto *VT = dyn_cast<llvm::VectorType>(Val: A.getType()))
535 LargestVectorWidth =
536 std::max(a: (uint64_t)LargestVectorWidth,
537 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
538
539 // Update vector width based on return type.
540 if (auto *VT = dyn_cast<llvm::VectorType>(Val: CurFn->getReturnType()))
541 LargestVectorWidth =
542 std::max(a: (uint64_t)LargestVectorWidth,
543 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
544
545 if (CurFnInfo->getMaxVectorWidth() > LargestVectorWidth)
546 LargestVectorWidth = CurFnInfo->getMaxVectorWidth();
547
548 // Add the min-legal-vector-width attribute. This contains the max width from:
549 // 1. min-vector-width attribute used in the source program.
550 // 2. Any builtins used that have a vector width specified.
551 // 3. Values passed in and out of inline assembly.
552 // 4. Width of vector arguments and return types for this function.
553 // 5. Width of vector arguments and return types for functions called by this
554 // function.
555 if (getContext().getTargetInfo().getTriple().isX86())
556 CurFn->addFnAttr(Kind: "min-legal-vector-width",
557 Val: llvm::utostr(X: LargestVectorWidth));
558
559 // If we generated an unreachable return block, delete it now.
560 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()) {
561 Builder.ClearInsertionPoint();
562 ReturnBlock.getBlock()->eraseFromParent();
563 }
564 if (ReturnValue.isValid()) {
565 auto *RetAlloca =
566 dyn_cast<llvm::AllocaInst>(Val: ReturnValue.emitRawPointer(CGF&: *this));
567 if (RetAlloca && RetAlloca->use_empty()) {
568 RetAlloca->eraseFromParent();
569 ReturnValue = Address::invalid();
570 }
571 }
572}
573
574/// ShouldInstrumentFunction - Return true if the current function should be
575/// instrumented with __cyg_profile_func_* calls
576bool CodeGenFunction::ShouldInstrumentFunction() {
577 if (!CGM.getCodeGenOpts().InstrumentFunctions &&
578 !CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining &&
579 !CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
580 return false;
581 if (!CurFuncDecl || CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>())
582 return false;
583 return true;
584}
585
586bool CodeGenFunction::ShouldSkipSanitizerInstrumentation() {
587 if (!CurFuncDecl)
588 return false;
589 return CurFuncDecl->hasAttr<DisableSanitizerInstrumentationAttr>();
590}
591
592/// ShouldXRayInstrument - Return true if the current function should be
593/// instrumented with XRay nop sleds.
594bool CodeGenFunction::ShouldXRayInstrumentFunction() const {
595 return CGM.getCodeGenOpts().XRayInstrumentFunctions;
596}
597
598/// AlwaysEmitXRayCustomEvents - Return true if we should emit IR for calls to
599/// the __xray_customevent(...) builtin calls, when doing XRay instrumentation.
600bool CodeGenFunction::AlwaysEmitXRayCustomEvents() const {
601 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
602 (CGM.getCodeGenOpts().XRayAlwaysEmitCustomEvents ||
603 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
604 XRayInstrKind::Custom);
605}
606
607bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const {
608 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
609 (CGM.getCodeGenOpts().XRayAlwaysEmitTypedEvents ||
610 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
611 XRayInstrKind::Typed);
612}
613
614llvm::ConstantInt *
615CodeGenFunction::getUBSanFunctionTypeHash(QualType Ty) const {
616 // Remove any (C++17) exception specifications, to allow calling e.g. a
617 // noexcept function through a non-noexcept pointer.
618 if (!Ty->isFunctionNoProtoType())
619 Ty = getContext().getFunctionTypeWithExceptionSpec(Orig: Ty, ESI: EST_None);
620 std::string Mangled;
621 llvm::raw_string_ostream Out(Mangled);
622 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: Ty, Out, NormalizeIntegers: false);
623 return llvm::ConstantInt::get(
624 Ty: CGM.Int32Ty, V: static_cast<uint32_t>(llvm::xxh3_64bits(data: Mangled)));
625}
626
627void CodeGenFunction::EmitKernelMetadata(const FunctionDecl *FD,
628 llvm::Function *Fn) {
629 if (!FD->hasAttr<DeviceKernelAttr>() && !FD->hasAttr<CUDAGlobalAttr>())
630 return;
631
632 llvm::LLVMContext &Context = getLLVMContext();
633
634 CGM.GenKernelArgMetadata(FN: Fn, FD, CGF: this);
635
636 if (!(getLangOpts().OpenCL ||
637 (getLangOpts().CUDA &&
638 getContext().getTargetInfo().getTriple().isSPIRV())))
639 return;
640
641 if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {
642 QualType HintQTy = A->getTypeHint();
643 const ExtVectorType *HintEltQTy = HintQTy->getAs<ExtVectorType>();
644 bool IsSignedInteger =
645 HintQTy->isSignedIntegerType() ||
646 (HintEltQTy && HintEltQTy->getElementType()->isSignedIntegerType());
647 llvm::Metadata *AttrMDArgs[] = {
648 llvm::ConstantAsMetadata::get(C: llvm::PoisonValue::get(
649 T: CGM.getTypes().ConvertType(T: A->getTypeHint()))),
650 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
651 Ty: llvm::IntegerType::get(C&: Context, NumBits: 32),
652 V: llvm::APInt(32, (uint64_t)(IsSignedInteger ? 1 : 0))))};
653 Fn->setMetadata(Kind: "vec_type_hint", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
654 }
655
656 if (const WorkGroupSizeHintAttr *A = FD->getAttr<WorkGroupSizeHintAttr>()) {
657 auto Eval = [&](Expr *E) {
658 return E->EvaluateKnownConstInt(Ctx: FD->getASTContext()).getExtValue();
659 };
660 llvm::Metadata *AttrMDArgs[] = {
661 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getXDim()))),
662 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getYDim()))),
663 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getZDim())))};
664 Fn->setMetadata(Kind: "work_group_size_hint", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
665 }
666
667 if (const ReqdWorkGroupSizeAttr *A = FD->getAttr<ReqdWorkGroupSizeAttr>()) {
668 auto Eval = [&](Expr *E) {
669 return E->EvaluateKnownConstInt(Ctx: FD->getASTContext()).getExtValue();
670 };
671 llvm::Metadata *AttrMDArgs[] = {
672 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getXDim()))),
673 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getYDim()))),
674 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getZDim())))};
675 Fn->setMetadata(Kind: "reqd_work_group_size", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
676 }
677
678 if (const OpenCLIntelReqdSubGroupSizeAttr *A =
679 FD->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
680 llvm::Metadata *AttrMDArgs[] = {
681 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: A->getSubGroupSize()))};
682 Fn->setMetadata(Kind: "intel_reqd_sub_group_size",
683 Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
684 }
685}
686
687/// Determine whether the function F ends with a return stmt.
688static bool endsWithReturn(const Decl* F) {
689 const Stmt *Body = nullptr;
690 if (auto *FD = dyn_cast_or_null<FunctionDecl>(Val: F))
691 Body = FD->getBody();
692 else if (auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(Val: F))
693 Body = OMD->getBody();
694
695 if (auto *CS = dyn_cast_or_null<CompoundStmt>(Val: Body)) {
696 auto LastStmt = CS->body_rbegin();
697 if (LastStmt != CS->body_rend())
698 return isa<ReturnStmt>(Val: *LastStmt);
699 }
700 return false;
701}
702
703void CodeGenFunction::markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {
704 if (SanOpts.has(K: SanitizerKind::Thread)) {
705 Fn->addFnAttr(Kind: "sanitize_thread_no_checking_at_run_time");
706 Fn->removeFnAttr(Kind: llvm::Attribute::SanitizeThread);
707 }
708}
709
710/// Check if the return value of this function requires sanitization.
711bool CodeGenFunction::requiresReturnValueCheck() const {
712 return requiresReturnValueNullabilityCheck() ||
713 (SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute) && CurCodeDecl &&
714 CurCodeDecl->getAttr<ReturnsNonNullAttr>());
715}
716
717static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) {
718 auto *MD = dyn_cast_or_null<CXXMethodDecl>(Val: D);
719 if (!MD || !MD->getDeclName().getAsIdentifierInfo() ||
720 !MD->getDeclName().getAsIdentifierInfo()->isStr(Str: "allocate") ||
721 (MD->getNumParams() != 1 && MD->getNumParams() != 2))
722 return false;
723
724 if (!Ctx.hasSameType(T1: MD->parameters()[0]->getType(), T2: Ctx.getSizeType()))
725 return false;
726
727 if (MD->getNumParams() == 2) {
728 auto *PT = MD->parameters()[1]->getType()->getAs<PointerType>();
729 if (!PT || !PT->isVoidPointerType() ||
730 !PT->getPointeeType().isConstQualified())
731 return false;
732 }
733
734 return true;
735}
736
737bool CodeGenFunction::isInAllocaArgument(CGCXXABI &ABI, QualType Ty) {
738 const CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
739 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
740}
741
742bool CodeGenFunction::hasInAllocaArg(const CXXMethodDecl *MD) {
743 return getTarget().getTriple().getArch() == llvm::Triple::x86 &&
744 getTarget().getCXXABI().isMicrosoft() &&
745 llvm::any_of(Range: MD->parameters(), P: [&](ParmVarDecl *P) {
746 return isInAllocaArgument(ABI&: CGM.getCXXABI(), Ty: P->getType());
747 });
748}
749
750/// Return the UBSan prologue signature for \p FD if one is available.
751static llvm::Constant *getPrologueSignature(CodeGenModule &CGM,
752 const FunctionDecl *FD) {
753 if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: FD))
754 if (!MD->isStatic())
755 return nullptr;
756 return CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM);
757}
758
759void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
760 llvm::Function *Fn,
761 const CGFunctionInfo &FnInfo,
762 const FunctionArgList &Args,
763 SourceLocation Loc,
764 SourceLocation StartLoc) {
765 assert(!CurFn &&
766 "Do not use a CodeGenFunction object for more than one function");
767
768 const Decl *D = GD.getDecl();
769
770 DidCallStackSave = false;
771 CurCodeDecl = D;
772 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
773 if (FD && FD->usesSEHTry())
774 CurSEHParent = GD;
775 CurFuncDecl = (D ? D->getNonClosureContext() : nullptr);
776 FnRetTy = RetTy;
777 CurFn = Fn;
778 CurFnInfo = &FnInfo;
779 assert(CurFn->isDeclaration() && "Function already has body?");
780
781 // If this function is ignored for any of the enabled sanitizers,
782 // disable the sanitizer for the function.
783 do {
784#define SANITIZER(NAME, ID) \
785 if (SanOpts.empty()) \
786 break; \
787 if (SanOpts.has(SanitizerKind::ID)) \
788 if (CGM.isInNoSanitizeList(SanitizerKind::ID, Fn, Loc)) \
789 SanOpts.set(SanitizerKind::ID, false);
790
791#include "clang/Basic/Sanitizers.def"
792#undef SANITIZER
793 } while (false);
794
795 if (D) {
796 const bool SanitizeBounds = SanOpts.hasOneOf(K: SanitizerKind::Bounds);
797 SanitizerMask no_sanitize_mask;
798 bool NoSanitizeCoverage = false;
799
800 for (auto *Attr : D->specific_attrs<NoSanitizeAttr>()) {
801 no_sanitize_mask |= Attr->getMask();
802 // SanitizeCoverage is not handled by SanOpts.
803 if (Attr->hasCoverage())
804 NoSanitizeCoverage = true;
805 }
806
807 // Apply the no_sanitize* attributes to SanOpts.
808 SanOpts.Mask &= ~no_sanitize_mask;
809 if (no_sanitize_mask & SanitizerKind::Address)
810 SanOpts.set(K: SanitizerKind::KernelAddress, Value: false);
811 if (no_sanitize_mask & SanitizerKind::KernelAddress)
812 SanOpts.set(K: SanitizerKind::Address, Value: false);
813 if (no_sanitize_mask & SanitizerKind::HWAddress)
814 SanOpts.set(K: SanitizerKind::KernelHWAddress, Value: false);
815 if (no_sanitize_mask & SanitizerKind::KernelHWAddress)
816 SanOpts.set(K: SanitizerKind::HWAddress, Value: false);
817
818 if (SanitizeBounds && !SanOpts.hasOneOf(K: SanitizerKind::Bounds))
819 Fn->addFnAttr(Kind: llvm::Attribute::NoSanitizeBounds);
820
821 if (NoSanitizeCoverage && CGM.getCodeGenOpts().hasSanitizeCoverage())
822 Fn->addFnAttr(Kind: llvm::Attribute::NoSanitizeCoverage);
823
824 // Some passes need the non-negated no_sanitize attribute. Pass them on.
825 if (CGM.getCodeGenOpts().hasSanitizeBinaryMetadata()) {
826 if (no_sanitize_mask & SanitizerKind::Thread)
827 Fn->addFnAttr(Kind: "no_sanitize_thread");
828 }
829 }
830
831 if (ShouldSkipSanitizerInstrumentation()) {
832 CurFn->addFnAttr(Kind: llvm::Attribute::DisableSanitizerInstrumentation);
833 } else {
834 // Apply sanitizer attributes to the function.
835 if (SanOpts.hasOneOf(K: SanitizerKind::Address | SanitizerKind::KernelAddress))
836 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAddress);
837 if (SanOpts.hasOneOf(K: SanitizerKind::HWAddress |
838 SanitizerKind::KernelHWAddress))
839 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeHWAddress);
840 if (SanOpts.has(K: SanitizerKind::MemtagStack))
841 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemTag);
842 if (SanOpts.has(K: SanitizerKind::Thread))
843 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeThread);
844 if (SanOpts.has(K: SanitizerKind::Type))
845 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeType);
846 if (SanOpts.has(K: SanitizerKind::NumericalStability))
847 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeNumericalStability);
848 if (SanOpts.hasOneOf(K: SanitizerKind::Memory | SanitizerKind::KernelMemory))
849 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemory);
850 if (SanOpts.has(K: SanitizerKind::AllocToken))
851 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAllocToken);
852 }
853 if (SanOpts.has(K: SanitizerKind::SafeStack))
854 Fn->addFnAttr(Kind: llvm::Attribute::SafeStack);
855 if (SanOpts.has(K: SanitizerKind::ShadowCallStack))
856 Fn->addFnAttr(Kind: llvm::Attribute::ShadowCallStack);
857
858 if (SanOpts.has(K: SanitizerKind::Realtime))
859 if (FD && FD->getASTContext().hasAnyFunctionEffects())
860 for (const FunctionEffectWithCondition &Fe : FD->getFunctionEffects()) {
861 if (Fe.Effect.kind() == FunctionEffect::Kind::NonBlocking)
862 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeRealtime);
863 else if (Fe.Effect.kind() == FunctionEffect::Kind::Blocking)
864 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeRealtimeBlocking);
865 }
866
867 // Apply fuzzing attribute to the function.
868 if (SanOpts.hasOneOf(K: SanitizerKind::Fuzzer | SanitizerKind::FuzzerNoLink))
869 Fn->addFnAttr(Kind: llvm::Attribute::OptForFuzzing);
870
871 // Ignore TSan memory acesses from within ObjC/ObjC++ dealloc, initialize,
872 // .cxx_destruct, __destroy_helper_block_ and all of their calees at run time.
873 if (SanOpts.has(K: SanitizerKind::Thread)) {
874 if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(Val: D)) {
875 const IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(argIndex: 0);
876 if (OMD->getMethodFamily() == OMF_dealloc ||
877 OMD->getMethodFamily() == OMF_initialize ||
878 (OMD->getSelector().isUnarySelector() && II->isStr(Str: ".cxx_destruct"))) {
879 markAsIgnoreThreadCheckingAtRuntime(Fn);
880 }
881 }
882 }
883
884 // Ignore unrelated casts in STL allocate() since the allocator must cast
885 // from void* to T* before object initialization completes. Don't match on the
886 // namespace because not all allocators are in std::
887 if (D && SanOpts.has(K: SanitizerKind::CFIUnrelatedCast)) {
888 if (matchesStlAllocatorFn(D, Ctx: getContext()))
889 SanOpts.Mask &= ~SanitizerKind::CFIUnrelatedCast;
890 }
891
892 // Ignore null checks in coroutine functions since the coroutines passes
893 // are not aware of how to move the extra UBSan instructions across the split
894 // coroutine boundaries.
895 if (D && SanOpts.has(K: SanitizerKind::Null))
896 if (FD && FD->getBody() &&
897 FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass)
898 SanOpts.Mask &= ~SanitizerKind::Null;
899
900 // Apply xray attributes to the function (as a string, for now)
901 bool AlwaysXRayAttr = false;
902 if (const auto *XRayAttr = D ? D->getAttr<XRayInstrumentAttr>() : nullptr) {
903 if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
904 K: XRayInstrKind::FunctionEntry) ||
905 CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
906 K: XRayInstrKind::FunctionExit)) {
907 if (XRayAttr->alwaysXRayInstrument() && ShouldXRayInstrumentFunction()) {
908 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-always");
909 AlwaysXRayAttr = true;
910 }
911 if (XRayAttr->neverXRayInstrument())
912 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-never");
913 if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())
914 if (ShouldXRayInstrumentFunction())
915 Fn->addFnAttr(Kind: "xray-log-args",
916 Val: llvm::utostr(X: LogArgs->getArgumentCount()));
917 }
918 } else {
919 if (ShouldXRayInstrumentFunction() && !CGM.imbueXRayAttrs(Fn, Loc))
920 Fn->addFnAttr(
921 Kind: "xray-instruction-threshold",
922 Val: llvm::itostr(X: CGM.getCodeGenOpts().XRayInstructionThreshold));
923 }
924
925 if (ShouldXRayInstrumentFunction()) {
926 if (CGM.getCodeGenOpts().XRayIgnoreLoops)
927 Fn->addFnAttr(Kind: "xray-ignore-loops");
928
929 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
930 K: XRayInstrKind::FunctionExit))
931 Fn->addFnAttr(Kind: "xray-skip-exit");
932
933 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
934 K: XRayInstrKind::FunctionEntry))
935 Fn->addFnAttr(Kind: "xray-skip-entry");
936
937 auto FuncGroups = CGM.getCodeGenOpts().XRayTotalFunctionGroups;
938 if (FuncGroups > 1) {
939 auto FuncName = llvm::ArrayRef<uint8_t>(CurFn->getName().bytes_begin(),
940 CurFn->getName().bytes_end());
941 auto Group = crc32(Data: FuncName) % FuncGroups;
942 if (Group != CGM.getCodeGenOpts().XRaySelectedFunctionGroup &&
943 !AlwaysXRayAttr)
944 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-never");
945 }
946 }
947
948 if (CGM.getCodeGenOpts().getProfileInstr() !=
949 llvm::driver::ProfileInstrKind::ProfileNone) {
950 switch (CGM.isFunctionBlockedFromProfileInstr(Fn, Loc)) {
951 case ProfileList::Skip:
952 Fn->addFnAttr(Kind: llvm::Attribute::SkipProfile);
953 break;
954 case ProfileList::Forbid:
955 Fn->addFnAttr(Kind: llvm::Attribute::NoProfile);
956 break;
957 case ProfileList::Allow:
958 break;
959 }
960 }
961
962 unsigned Count, Offset;
963 StringRef Section;
964 if (const auto *Attr =
965 D ? D->getAttr<PatchableFunctionEntryAttr>() : nullptr) {
966 Count = Attr->getCount();
967 Offset = Attr->getOffset();
968 Section = Attr->getSection();
969 } else {
970 Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;
971 Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;
972 }
973 if (Section.empty())
974 Section = CGM.getCodeGenOpts().PatchableFunctionEntrySection;
975 if (Count && Offset <= Count) {
976 Fn->addFnAttr(Kind: "patchable-function-entry", Val: std::to_string(val: Count - Offset));
977 if (Offset)
978 Fn->addFnAttr(Kind: "patchable-function-prefix", Val: std::to_string(val: Offset));
979 if (!Section.empty())
980 Fn->addFnAttr(Kind: "patchable-function-entry-section", Val: Section);
981 }
982 // Instruct that functions for COFF/CodeView targets should start with a
983 // patchable instruction, but only on x86/x64. Don't forward this to ARM/ARM64
984 // backends as they don't need it -- instructions on these architectures are
985 // always atomically patchable at runtime.
986 if (CGM.getCodeGenOpts().HotPatch &&
987 getContext().getTargetInfo().getTriple().isX86() &&
988 getContext().getTargetInfo().getTriple().getEnvironment() !=
989 llvm::Triple::CODE16)
990 Fn->addFnAttr(Kind: "patchable-function", Val: "prologue-short-redirect");
991
992 // Add no-jump-tables value.
993 if (CGM.getCodeGenOpts().NoUseJumpTables)
994 Fn->addFnAttr(Kind: "no-jump-tables", Val: "true");
995
996 // Add no-inline-line-tables value.
997 if (CGM.getCodeGenOpts().NoInlineLineTables)
998 Fn->addFnAttr(Kind: "no-inline-line-tables");
999
1000 // Add profile-sample-accurate value.
1001 if (CGM.getCodeGenOpts().ProfileSampleAccurate)
1002 Fn->addFnAttr(Kind: "profile-sample-accurate");
1003
1004 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
1005 Fn->addFnAttr(Kind: "use-sample-profile");
1006
1007 if (D && D->hasAttr<CFICanonicalJumpTableAttr>())
1008 Fn->addFnAttr(Kind: "cfi-canonical-jump-table");
1009
1010 if (D && D->hasAttr<NoProfileFunctionAttr>())
1011 Fn->addFnAttr(Kind: llvm::Attribute::NoProfile);
1012
1013 if (D && D->hasAttr<HybridPatchableAttr>())
1014 Fn->addFnAttr(Kind: llvm::Attribute::HybridPatchable);
1015
1016 if (D) {
1017 // Function attributes take precedence over command line flags.
1018 if (auto *A = D->getAttr<FunctionReturnThunksAttr>()) {
1019 switch (A->getThunkType()) {
1020 case FunctionReturnThunksAttr::Kind::Keep:
1021 break;
1022 case FunctionReturnThunksAttr::Kind::Extern:
1023 Fn->addFnAttr(Kind: llvm::Attribute::FnRetThunkExtern);
1024 break;
1025 }
1026 } else if (CGM.getCodeGenOpts().FunctionReturnThunks)
1027 Fn->addFnAttr(Kind: llvm::Attribute::FnRetThunkExtern);
1028 }
1029
1030 if (FD && (getLangOpts().OpenCL ||
1031 (getLangOpts().CUDA &&
1032 getContext().getTargetInfo().getTriple().isSPIRV()) ||
1033 ((getLangOpts().HIP || getLangOpts().OffloadViaLLVM) &&
1034 getLangOpts().CUDAIsDevice))) {
1035 // Add metadata for a kernel function.
1036 EmitKernelMetadata(FD, Fn);
1037 }
1038
1039 if (FD && FD->hasAttr<ClspvLibclcBuiltinAttr>()) {
1040 Fn->setMetadata(Kind: "clspv_libclc_builtin",
1041 Node: llvm::MDNode::get(Context&: getLLVMContext(), MDs: {}));
1042 }
1043
1044 // If we are checking function types, emit a function type signature as
1045 // prologue data. Kernel functions have strict alignment requirements and
1046 // cannot be call indirectly so we do not instrument them.
1047 if (FD && SanOpts.has(K: SanitizerKind::Function) &&
1048 !FD->getType()->isCFIUncheckedCalleeFunctionType() &&
1049 llvm::isCallableCC(CC: Fn->getCallingConv())) {
1050 if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {
1051 llvm::LLVMContext &Ctx = Fn->getContext();
1052 llvm::MDBuilder MDB(Ctx);
1053 Fn->setMetadata(
1054 KindID: llvm::LLVMContext::MD_func_sanitize,
1055 Node: MDB.createRTTIPointerPrologue(
1056 PrologueSig, RTTI: getUBSanFunctionTypeHash(Ty: FD->getType())));
1057 }
1058 }
1059
1060 // If we're checking nullability, we need to know whether we can check the
1061 // return value. Initialize the flag to 'true' and refine it in EmitParmDecl.
1062 if (SanOpts.has(K: SanitizerKind::NullabilityReturn)) {
1063 auto Nullability = FnRetTy->getNullability();
1064 if (Nullability && *Nullability == NullabilityKind::NonNull &&
1065 !FnRetTy->isRecordType()) {
1066 if (!(SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute) &&
1067 CurCodeDecl && CurCodeDecl->getAttr<ReturnsNonNullAttr>()))
1068 RetValNullabilityPrecondition =
1069 llvm::ConstantInt::getTrue(Context&: getLLVMContext());
1070 }
1071 }
1072
1073 // If we're in C++ mode and the function name is "main", it is guaranteed
1074 // to be norecurse by the standard (3.6.1.3 "The function main shall not be
1075 // used within a program").
1076 //
1077 // OpenCL C 2.0 v2.2-11 s6.9.i:
1078 // Recursion is not supported.
1079 //
1080 // HLSL
1081 // Recursion is not supported.
1082 //
1083 // SYCL v1.2.1 s3.10:
1084 // kernels cannot include RTTI information, exception classes,
1085 // recursive code, virtual functions or make use of C++ libraries that
1086 // are not compiled for the device.
1087 if (FD &&
1088 ((getLangOpts().CPlusPlus && FD->isMain()) || getLangOpts().OpenCL ||
1089 getLangOpts().HLSL || getLangOpts().SYCLIsDevice ||
1090 (getLangOpts().CUDA && FD->hasAttr<CUDAGlobalAttr>())))
1091 Fn->addFnAttr(Kind: llvm::Attribute::NoRecurse);
1092
1093 llvm::RoundingMode RM = getLangOpts().getDefaultRoundingMode();
1094 llvm::fp::ExceptionBehavior FPExceptionBehavior =
1095 ToConstrainedExceptMD(Kind: getLangOpts().getDefaultExceptionMode());
1096 Builder.setDefaultConstrainedRounding(RM);
1097 Builder.setDefaultConstrainedExcept(FPExceptionBehavior);
1098 if ((FD && (FD->UsesFPIntrin() || FD->hasAttr<StrictFPAttr>())) ||
1099 (!FD && (FPExceptionBehavior != llvm::fp::ebIgnore ||
1100 RM != llvm::RoundingMode::NearestTiesToEven))) {
1101 Builder.setIsFPConstrained(true);
1102 Fn->addFnAttr(Kind: llvm::Attribute::StrictFP);
1103 }
1104
1105 // If a custom alignment is used, force realigning to this alignment on
1106 // any main function which certainly will need it.
1107 if (FD && ((FD->isMain() || FD->isMSVCRTEntryPoint()) &&
1108 CGM.getCodeGenOpts().StackAlignment))
1109 Fn->addFnAttr(Kind: "stackrealign");
1110
1111 // "main" doesn't need to zero out call-used registers.
1112 if (FD && FD->isMain())
1113 Fn->removeFnAttr(Kind: "zero-call-used-regs");
1114
1115 // Add vscale_range attribute if appropriate.
1116 llvm::StringMap<bool> FeatureMap;
1117 auto IsArmStreaming = TargetInfo::ArmStreamingKind::NotStreaming;
1118 if (FD) {
1119 getContext().getFunctionFeatureMap(FeatureMap, FD);
1120 if (const auto *T = FD->getType()->getAs<FunctionProtoType>())
1121 if (T->getAArch64SMEAttributes() &
1122 FunctionType::SME_PStateSMCompatibleMask)
1123 IsArmStreaming = TargetInfo::ArmStreamingKind::StreamingCompatible;
1124
1125 if (IsArmStreamingFunction(FD, IncludeLocallyStreaming: true))
1126 IsArmStreaming = TargetInfo::ArmStreamingKind::Streaming;
1127 }
1128 std::optional<std::pair<unsigned, unsigned>> VScaleRange =
1129 getContext().getTargetInfo().getVScaleRange(LangOpts: getLangOpts(), Mode: IsArmStreaming,
1130 FeatureMap: &FeatureMap);
1131 if (VScaleRange) {
1132 CurFn->addFnAttr(Attr: llvm::Attribute::getWithVScaleRangeArgs(
1133 Context&: getLLVMContext(), MinValue: VScaleRange->first, MaxValue: VScaleRange->second));
1134 }
1135
1136 llvm::BasicBlock *EntryBB = createBasicBlock(name: "entry", parent: CurFn);
1137
1138 // Create a marker to make it easy to insert allocas into the entryblock
1139 // later. Don't create this with the builder, because we don't want it
1140 // folded.
1141 llvm::Value *Poison = llvm::PoisonValue::get(T: Int32Ty);
1142 AllocaInsertPt = new llvm::BitCastInst(Poison, Int32Ty, "allocapt", EntryBB);
1143
1144 ReturnBlock = getJumpDestInCurrentScope(Name: "return");
1145
1146 Builder.SetInsertPoint(EntryBB);
1147
1148 // If we're checking the return value, allocate space for a pointer to a
1149 // precise source location of the checked return statement.
1150 if (requiresReturnValueCheck()) {
1151 ReturnLocation = CreateDefaultAlignTempAlloca(Ty: Int8PtrTy, Name: "return.sloc.ptr");
1152 Builder.CreateStore(Val: llvm::ConstantPointerNull::get(T: Int8PtrTy),
1153 Addr: ReturnLocation);
1154 }
1155
1156 // Emit subprogram debug descriptor.
1157 if (CGDebugInfo *DI = getDebugInfo()) {
1158 // Reconstruct the type from the argument list so that implicit parameters,
1159 // such as 'this' and 'vtt', show up in the debug info. Preserve the calling
1160 // convention.
1161 DI->emitFunctionStart(GD, Loc, ScopeLoc: StartLoc,
1162 FnType: DI->getFunctionType(FD, RetTy, Args), Fn: CurFn,
1163 CurFnIsThunk: CurFuncIsThunk);
1164 }
1165
1166 if (ShouldInstrumentFunction()) {
1167 if (CGM.getCodeGenOpts().InstrumentFunctions)
1168 CurFn->addFnAttr(Kind: "instrument-function-entry", Val: "__cyg_profile_func_enter");
1169 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
1170 CurFn->addFnAttr(Kind: "instrument-function-entry-inlined",
1171 Val: "__cyg_profile_func_enter");
1172 if (CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
1173 CurFn->addFnAttr(Kind: "instrument-function-entry-inlined",
1174 Val: "__cyg_profile_func_enter_bare");
1175 }
1176
1177 // Since emitting the mcount call here impacts optimizations such as function
1178 // inlining, we just add an attribute to insert a mcount call in backend.
1179 // The attribute "counting-function" is set to mcount function name which is
1180 // architecture dependent.
1181 if (CGM.getCodeGenOpts().InstrumentForProfiling) {
1182 // Calls to fentry/mcount should not be generated if function has
1183 // the no_instrument_function attribute.
1184 if (!CurFuncDecl || !CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()) {
1185 if (CGM.getCodeGenOpts().CallFEntry)
1186 Fn->addFnAttr(Kind: "fentry-call", Val: "true");
1187 else {
1188 Fn->addFnAttr(Kind: "instrument-function-entry-inlined",
1189 Val: getTarget().getMCountName());
1190 }
1191 if (CGM.getCodeGenOpts().MNopMCount) {
1192 if (!CGM.getCodeGenOpts().CallFEntry)
1193 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_without_opt)
1194 << "-mnop-mcount" << "-mfentry";
1195 Fn->addFnAttr(Kind: "mnop-mcount");
1196 }
1197
1198 if (CGM.getCodeGenOpts().RecordMCount) {
1199 if (!CGM.getCodeGenOpts().CallFEntry)
1200 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_without_opt)
1201 << "-mrecord-mcount" << "-mfentry";
1202 Fn->addFnAttr(Kind: "mrecord-mcount");
1203 }
1204 }
1205 }
1206
1207 if (CGM.getCodeGenOpts().PackedStack) {
1208 if (getContext().getTargetInfo().getTriple().getArch() !=
1209 llvm::Triple::systemz)
1210 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_on_target)
1211 << "-mpacked-stack";
1212 Fn->addFnAttr(Kind: "packed-stack");
1213 }
1214
1215 if (!CGM.getCodeGenOpts().ZOSPPA1Name)
1216 Fn->addFnAttr(Kind: "zos-ppa1-name", Val: "");
1217
1218 if (CGM.getCodeGenOpts().WarnStackSize != UINT_MAX &&
1219 !CGM.getDiags().isIgnored(DiagID: diag::warn_fe_backend_frame_larger_than, Loc))
1220 Fn->addFnAttr(Kind: "warn-stack-size",
1221 Val: std::to_string(val: CGM.getCodeGenOpts().WarnStackSize));
1222
1223 if (RetTy->isVoidType()) {
1224 // Void type; nothing to return.
1225 ReturnValue = Address::invalid();
1226
1227 // Count the implicit return.
1228 if (!endsWithReturn(F: D))
1229 ++NumReturnExprs;
1230 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1231 // Indirect return; emit returned value directly into sret slot.
1232 // This reduces code size, and affects correctness in C++.
1233 auto AI = CurFn->arg_begin();
1234 if (CurFnInfo->getReturnInfo().isSRetAfterThis())
1235 ++AI;
1236 ReturnValue = makeNaturalAddressForPointer(
1237 Ptr: &*AI, T: RetTy, Alignment: CurFnInfo->getReturnInfo().getIndirectAlign(), ForPointeeType: false,
1238 BaseInfo: nullptr, TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
1239 if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
1240 ReturnValuePointer =
1241 CreateDefaultAlignTempAlloca(Ty: ReturnValue.getType(), Name: "result.ptr");
1242 Builder.CreateStore(Val: ReturnValue.emitRawPointer(CGF&: *this),
1243 Addr: ReturnValuePointer);
1244 }
1245 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
1246 !hasScalarEvaluationKind(T: CurFnInfo->getReturnType())) {
1247 // Load the sret pointer from the argument struct and return into that.
1248 unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();
1249 llvm::Function::arg_iterator EI = CurFn->arg_end();
1250 --EI;
1251 llvm::Value *Addr = Builder.CreateStructGEP(
1252 Ty: CurFnInfo->getArgStruct(), Ptr: &*EI, Idx);
1253 llvm::Type *Ty =
1254 cast<llvm::GetElementPtrInst>(Val: Addr)->getResultElementType();
1255 ReturnValuePointer = Address(Addr, Ty, getPointerAlign());
1256 Addr = Builder.CreateAlignedLoad(Ty, Addr, Align: getPointerAlign(), Name: "agg.result");
1257 ReturnValue = Address(Addr, ConvertType(T: RetTy),
1258 CGM.getNaturalTypeAlignment(T: RetTy), KnownNonNull);
1259 } else {
1260 ReturnValue = CreateIRTempWithoutCast(T: RetTy, Name: "retval");
1261
1262 // Tell the epilog emitter to autorelease the result. We do this
1263 // now so that various specialized functions can suppress it
1264 // during their IR-generation.
1265 if (getLangOpts().ObjCAutoRefCount &&
1266 !CurFnInfo->isReturnsRetained() &&
1267 RetTy->isObjCRetainableType())
1268 AutoreleaseResult = true;
1269 }
1270
1271 EmitStartEHSpec(D: CurCodeDecl);
1272
1273 PrologueCleanupDepth = EHStack.stable_begin();
1274
1275 // Emit OpenMP specific initialization of the device functions.
1276 if (getLangOpts().OpenMP && CurCodeDecl)
1277 CGM.getOpenMPRuntime().emitFunctionProlog(CGF&: *this, D: CurCodeDecl);
1278
1279 if (FD && getLangOpts().HLSL) {
1280 // Handle emitting HLSL entry functions.
1281 if (FD->hasAttr<HLSLShaderAttr>()) {
1282 CGM.getHLSLRuntime().emitEntryFunction(FD, Fn);
1283 }
1284 }
1285
1286 EmitFunctionProlog(FI: *CurFnInfo, Fn: CurFn, Args);
1287
1288 if (const CXXMethodDecl *MD = dyn_cast_if_present<CXXMethodDecl>(Val: D);
1289 MD && !MD->isStatic()) {
1290 bool IsInLambda =
1291 MD->getParent()->isLambda() && MD->getOverloadedOperator() == OO_Call;
1292 if (MD->isImplicitObjectMemberFunction())
1293 CGM.getCXXABI().EmitInstanceFunctionProlog(CGF&: *this);
1294 if (IsInLambda) {
1295 // We're in a lambda; figure out the captures.
1296 MD->getParent()->getCaptureFields(Captures&: LambdaCaptureFields,
1297 ThisCapture&: LambdaThisCaptureField);
1298 if (LambdaThisCaptureField) {
1299 // If the lambda captures the object referred to by '*this' - either by
1300 // value or by reference, make sure CXXThisValue points to the correct
1301 // object.
1302
1303 // Get the lvalue for the field (which is a copy of the enclosing object
1304 // or contains the address of the enclosing object).
1305 LValue ThisFieldLValue = EmitLValueForLambdaField(Field: LambdaThisCaptureField);
1306 if (!LambdaThisCaptureField->getType()->isPointerType()) {
1307 // If the enclosing object was captured by value, just use its
1308 // address. Sign this pointer.
1309 CXXThisValue = ThisFieldLValue.getPointer(CGF&: *this);
1310 } else {
1311 // Load the lvalue pointed to by the field, since '*this' was captured
1312 // by reference.
1313 CXXThisValue =
1314 EmitLoadOfLValue(V: ThisFieldLValue, Loc: SourceLocation()).getScalarVal();
1315 }
1316 }
1317 for (auto *FD : MD->getParent()->fields()) {
1318 if (FD->hasCapturedVLAType()) {
1319 auto *ExprArg = EmitLoadOfLValue(V: EmitLValueForLambdaField(Field: FD),
1320 Loc: SourceLocation()).getScalarVal();
1321 auto VAT = FD->getCapturedVLAType();
1322 VLASizeMap[VAT->getSizeExpr()] = ExprArg;
1323 }
1324 }
1325 } else if (MD->isImplicitObjectMemberFunction()) {
1326 // Not in a lambda; just use 'this' from the method.
1327 // FIXME: Should we generate a new load for each use of 'this'? The
1328 // fast register allocator would be happier...
1329 CXXThisValue = CXXABIThisValue;
1330 }
1331
1332 // Check the 'this' pointer once per function, if it's available.
1333 if (CXXABIThisValue) {
1334 SanitizerSet SkippedChecks;
1335 SkippedChecks.set(K: SanitizerKind::ObjectSize, Value: true);
1336 QualType ThisTy = MD->getThisType();
1337
1338 // If this is the call operator of a lambda with no captures, it
1339 // may have a static invoker function, which may call this operator with
1340 // a null 'this' pointer.
1341 if (isLambdaCallOperator(MD) && MD->getParent()->isCapturelessLambda())
1342 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
1343
1344 EmitTypeCheck(
1345 TCK: isa<CXXConstructorDecl>(Val: MD) ? TCK_ConstructorCall : TCK_MemberCall,
1346 Loc, V: CXXABIThisValue, Type: ThisTy, Alignment: CXXABIThisAlignment, SkippedChecks);
1347 }
1348 }
1349
1350 // If any of the arguments have a variably modified type, make sure to
1351 // emit the type size, but only if the function is not naked. Naked functions
1352 // have no prolog to run this evaluation.
1353 if (!FD || !FD->hasAttr<NakedAttr>()) {
1354 for (const VarDecl *VD : Args) {
1355 // Dig out the type as written from ParmVarDecls; it's unclear whether
1356 // the standard (C99 6.9.1p10) requires this, but we're following the
1357 // precedent set by gcc.
1358 QualType Ty;
1359 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: VD))
1360 Ty = PVD->getOriginalType();
1361 else
1362 Ty = VD->getType();
1363
1364 if (Ty->isVariablyModifiedType())
1365 EmitVariablyModifiedType(Ty);
1366 }
1367 }
1368 // Emit a location at the end of the prologue.
1369 if (CGDebugInfo *DI = getDebugInfo())
1370 DI->EmitLocation(Builder, Loc: StartLoc);
1371 // TODO: Do we need to handle this in two places like we do with
1372 // target-features/target-cpu?
1373 if (CurFuncDecl)
1374 if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>())
1375 LargestVectorWidth = VecWidth->getVectorWidth();
1376
1377 if (CGM.shouldEmitConvergenceTokens())
1378 ConvergenceTokenStack.push_back(Elt: getOrEmitConvergenceEntryToken(F: CurFn));
1379}
1380
1381void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {
1382 incrementProfileCounter(S: Body);
1383 maybeCreateMCDCCondBitmap();
1384 if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Val: Body))
1385 EmitCompoundStmtWithoutScope(S: *S);
1386 else
1387 EmitStmt(S: Body);
1388}
1389
1390/// When instrumenting to collect profile data, the counts for some blocks
1391/// such as switch cases need to not include the fall-through counts, so
1392/// emit a branch around the instrumentation code. When not instrumenting,
1393/// this just calls EmitBlock().
1394void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
1395 const Stmt *S) {
1396 llvm::BasicBlock *SkipCountBB = nullptr;
1397 if (HaveInsertPoint() && CGM.getCodeGenOpts().hasProfileClangInstr()) {
1398 // When instrumenting for profiling, the fallthrough to certain
1399 // statements needs to skip over the instrumentation code so that we
1400 // get an accurate count.
1401 SkipCountBB = createBasicBlock(name: "skipcount");
1402 EmitBranch(Block: SkipCountBB);
1403 }
1404 EmitBlock(BB);
1405 uint64_t CurrentCount = getCurrentProfileCount();
1406 incrementProfileCounter(ExecSkip: UseExecPath, S);
1407 setCurrentProfileCount(getCurrentProfileCount() + CurrentCount);
1408 if (SkipCountBB)
1409 EmitBlock(BB: SkipCountBB);
1410}
1411
1412/// Tries to mark the given function nounwind based on the
1413/// non-existence of any throwing calls within it. We believe this is
1414/// lightweight enough to do at -O0.
1415static void TryMarkNoThrow(llvm::Function *F) {
1416 // LLVM treats 'nounwind' on a function as part of the type, so we
1417 // can't do this on functions that can be overwritten.
1418 if (F->isInterposable()) return;
1419
1420 for (llvm::BasicBlock &BB : *F)
1421 for (llvm::Instruction &I : BB)
1422 if (I.mayThrow())
1423 return;
1424
1425 F->setDoesNotThrow();
1426}
1427
1428QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD,
1429 FunctionArgList &Args) {
1430 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
1431 QualType ResTy = FD->getReturnType();
1432
1433 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD);
1434 if (MD && MD->isImplicitObjectMemberFunction()) {
1435 if (CGM.getCXXABI().HasThisReturn(GD))
1436 ResTy = MD->getThisType();
1437 else if (CGM.getCXXABI().hasMostDerivedReturn(GD))
1438 ResTy = CGM.getContext().VoidPtrTy;
1439 CGM.getCXXABI().buildThisParam(CGF&: *this, Params&: Args);
1440 }
1441
1442 // The base version of an inheriting constructor whose constructed base is a
1443 // virtual base is not passed any arguments (because it doesn't actually call
1444 // the inherited constructor).
1445 bool PassedParams = true;
1446 if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(Val: FD))
1447 if (auto Inherited = CD->getInheritedConstructor())
1448 PassedParams =
1449 getTypes().inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
1450
1451 if (PassedParams) {
1452 for (auto *Param : FD->parameters()) {
1453 Args.push_back(Elt: Param);
1454 if (!Param->hasAttr<PassObjectSizeAttr>())
1455 continue;
1456
1457 auto *Implicit = ImplicitParamDecl::Create(
1458 C&: getContext(), DC: Param->getDeclContext(), IdLoc: Param->getLocation(),
1459 /*Id=*/nullptr, T: getContext().getSizeType(), ParamKind: ImplicitParamKind::Other);
1460 SizeArguments[Param] = Implicit;
1461 Args.push_back(Elt: Implicit);
1462 }
1463 }
1464
1465 if (MD && (isa<CXXConstructorDecl>(Val: MD) || isa<CXXDestructorDecl>(Val: MD)))
1466 CGM.getCXXABI().addImplicitStructorParams(CGF&: *this, ResTy, Params&: Args);
1467
1468 return ResTy;
1469}
1470
1471void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
1472 const CGFunctionInfo &FnInfo) {
1473 assert(Fn && "generating code for null Function");
1474 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
1475 CurGD = GD;
1476
1477 FunctionArgList Args;
1478 QualType ResTy = BuildFunctionArgList(GD, Args);
1479
1480 CGM.getTargetCodeGenInfo().checkFunctionABI(CGM, Decl: FD);
1481
1482 if (FD->isInlineBuiltinDeclaration()) {
1483 // When generating code for a builtin with an inline declaration, use a
1484 // mangled name to hold the actual body, while keeping an external
1485 // definition in case the function pointer is referenced somewhere.
1486 std::string FDInlineName = (Fn->getName() + ".inline").str();
1487 llvm::Module *M = Fn->getParent();
1488 llvm::Function *Clone = M->getFunction(Name: FDInlineName);
1489 if (!Clone) {
1490 Clone = llvm::Function::Create(Ty: Fn->getFunctionType(),
1491 Linkage: llvm::GlobalValue::InternalLinkage,
1492 AddrSpace: Fn->getAddressSpace(), N: FDInlineName, M);
1493 Clone->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1494 }
1495 Fn->setLinkage(llvm::GlobalValue::ExternalLinkage);
1496 Fn = Clone;
1497 } else {
1498 // Detect the unusual situation where an inline version is shadowed by a
1499 // non-inline version. In that case we should pick the external one
1500 // everywhere. That's GCC behavior too. Unfortunately, I cannot find a way
1501 // to detect that situation before we reach codegen, so do some late
1502 // replacement.
1503 for (const FunctionDecl *PD = FD->getPreviousDecl(); PD;
1504 PD = PD->getPreviousDecl()) {
1505 if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) {
1506 std::string FDInlineName = (Fn->getName() + ".inline").str();
1507 llvm::Module *M = Fn->getParent();
1508 if (llvm::Function *Clone = M->getFunction(Name: FDInlineName)) {
1509 Clone->replaceAllUsesWith(V: Fn);
1510 Clone->eraseFromParent();
1511 }
1512 break;
1513 }
1514 }
1515 }
1516
1517 // Check if we should generate debug info for this function.
1518 if (FD->hasAttr<NoDebugAttr>()) {
1519 // Clear non-distinct debug info that was possibly attached to the function
1520 // due to an earlier declaration without the nodebug attribute
1521 Fn->setSubprogram(nullptr);
1522 // Disable debug info indefinitely for this function
1523 DebugInfo = nullptr;
1524 }
1525 // Finalize function debug info on exit.
1526 llvm::scope_exit Cleanup([this] {
1527 if (CGDebugInfo *DI = getDebugInfo())
1528 DI->completeFunction();
1529 });
1530
1531 // The function might not have a body if we're generating thunks for a
1532 // function declaration.
1533 SourceRange BodyRange;
1534 if (Stmt *Body = FD->getBody())
1535 BodyRange = Body->getSourceRange();
1536 else
1537 BodyRange = FD->getLocation();
1538 CurEHLocation = BodyRange.getEnd();
1539
1540 // Use the location of the start of the function to determine where
1541 // the function definition is located. By default use the location
1542 // of the declaration as the location for the subprogram. A function
1543 // may lack a declaration in the source code if it is created by code
1544 // gen. (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).
1545 SourceLocation Loc = FD->getLocation();
1546
1547 // If this is a function specialization then use the pattern body
1548 // as the location for the function.
1549 if (const FunctionDecl *SpecDecl = FD->getTemplateInstantiationPattern())
1550 if (SpecDecl->hasBody(Definition&: SpecDecl))
1551 Loc = SpecDecl->getLocation();
1552
1553 Stmt *Body = FD->getBody();
1554
1555 if (Body) {
1556 // Coroutines always emit lifetime markers.
1557 if (isa<CoroutineBodyStmt>(Val: Body))
1558 ShouldEmitLifetimeMarkers = true;
1559
1560 // Initialize helper which will detect jumps which can cause invalid
1561 // lifetime markers.
1562 if (ShouldEmitLifetimeMarkers)
1563 Bypasses.Init(CGM, Body);
1564 }
1565
1566 // Emit the standard function prologue.
1567 StartFunction(GD, RetTy: ResTy, Fn, FnInfo, Args, Loc, StartLoc: BodyRange.getBegin());
1568
1569 // Save parameters for coroutine function.
1570 if (Body && isa_and_nonnull<CoroutineBodyStmt>(Val: Body))
1571 llvm::append_range(C&: FnArgs, R: FD->parameters());
1572
1573 // Ensure that the function adheres to the forward progress guarantee, which
1574 // is required by certain optimizations.
1575 // In C++11 and up, the attribute will be removed if the body contains a
1576 // trivial empty loop.
1577 if (checkIfFunctionMustProgress())
1578 CurFn->addFnAttr(Kind: llvm::Attribute::MustProgress);
1579
1580 // Generate the body of the function.
1581 PGO->assignRegionCounters(GD, Fn: CurFn);
1582 if (isa<CXXDestructorDecl>(Val: FD))
1583 EmitDestructorBody(Args);
1584 else if (isa<CXXConstructorDecl>(Val: FD))
1585 EmitConstructorBody(Args);
1586 else if (getLangOpts().CUDA &&
1587 !getLangOpts().CUDAIsDevice &&
1588 FD->hasAttr<CUDAGlobalAttr>())
1589 CGM.getCUDARuntime().emitDeviceStub(CGF&: *this, Args);
1590 else if (isa<CXXMethodDecl>(Val: FD) &&
1591 cast<CXXMethodDecl>(Val: FD)->isLambdaStaticInvoker()) {
1592 // The lambda static invoker function is special, because it forwards or
1593 // clones the body of the function call operator (but is actually static).
1594 EmitLambdaStaticInvokeBody(MD: cast<CXXMethodDecl>(Val: FD));
1595 } else if (isa<CXXMethodDecl>(Val: FD) &&
1596 isLambdaCallOperator(MD: cast<CXXMethodDecl>(Val: FD)) &&
1597 !FnInfo.isDelegateCall() &&
1598 cast<CXXMethodDecl>(Val: FD)->getParent()->getLambdaStaticInvoker() &&
1599 hasInAllocaArg(MD: cast<CXXMethodDecl>(Val: FD))) {
1600 // If emitting a lambda with static invoker on X86 Windows, change
1601 // the call operator body.
1602 // Make sure that this is a call operator with an inalloca arg and check
1603 // for delegate call to make sure this is the original call op and not the
1604 // new forwarding function for the static invoker.
1605 EmitLambdaInAllocaCallOpBody(MD: cast<CXXMethodDecl>(Val: FD));
1606 } else if (FD->isDefaulted() && isa<CXXMethodDecl>(Val: FD) &&
1607 (cast<CXXMethodDecl>(Val: FD)->isCopyAssignmentOperator() ||
1608 cast<CXXMethodDecl>(Val: FD)->isMoveAssignmentOperator())) {
1609 // Implicit copy-assignment gets the same special treatment as implicit
1610 // copy-constructors.
1611 emitImplicitAssignmentOperatorBody(Args);
1612 } else if (DeviceKernelAttr::isOpenCLSpelling(
1613 A: FD->getAttr<DeviceKernelAttr>()) &&
1614 GD.getKernelReferenceKind() == KernelReferenceKind::Kernel) {
1615 CallArgList CallArgs;
1616 for (unsigned i = 0; i < Args.size(); ++i) {
1617 Address ArgAddr = GetAddrOfLocalVar(VD: Args[i]);
1618 QualType ArgQualType = Args[i]->getType();
1619 RValue ArgRValue = convertTempToRValue(addr: ArgAddr, type: ArgQualType, Loc);
1620 CallArgs.add(rvalue: ArgRValue, type: ArgQualType);
1621 }
1622 GlobalDecl GDStub = GlobalDecl(FD, KernelReferenceKind::Stub);
1623 const FunctionType *FT = cast<FunctionType>(Val: FD->getType());
1624 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
1625 const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
1626 Args: CallArgs, Ty: FT, /*ChainCall=*/false, ABIInfoFD: getCurrentFunctionDecl());
1627 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
1628 llvm::Constant *GDStubFunctionPointer =
1629 CGM.getRawFunctionPointer(GD: GDStub, Ty: FTy);
1630 CGCallee GDStubCallee = CGCallee::forDirect(functionPtr: GDStubFunctionPointer, abstractInfo: GDStub);
1631 EmitCall(CallInfo: FnInfo, Callee: GDStubCallee, ReturnValue: ReturnValueSlot(), Args: CallArgs, CallOrInvoke: nullptr, IsMustTail: false,
1632 Loc);
1633 } else if (Body) {
1634 EmitFunctionBody(Body);
1635 } else
1636 llvm_unreachable("no definition for emitted function");
1637
1638 // C++11 [stmt.return]p2:
1639 // Flowing off the end of a function [...] results in undefined behavior in
1640 // a value-returning function.
1641 // C11 6.9.1p12:
1642 // If the '}' that terminates a function is reached, and the value of the
1643 // function call is used by the caller, the behavior is undefined.
1644 if (getLangOpts().CPlusPlus && !FD->hasImplicitReturnZero() && !SawAsmBlock &&
1645 !FD->getReturnType()->isVoidType() && Builder.GetInsertBlock()) {
1646 bool ShouldEmitUnreachable =
1647 CGM.getCodeGenOpts().StrictReturn ||
1648 !CGM.MayDropFunctionReturn(Context: FD->getASTContext(), ReturnType: FD->getReturnType());
1649 if (SanOpts.has(K: SanitizerKind::Return)) {
1650 auto CheckOrdinal = SanitizerKind::SO_Return;
1651 auto CheckHandler = SanitizerHandler::MissingReturn;
1652 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
1653 llvm::Value *IsFalse = Builder.getFalse();
1654 EmitCheck(Checked: std::make_pair(x&: IsFalse, y&: CheckOrdinal), Check: CheckHandler,
1655 StaticArgs: EmitCheckSourceLocation(Loc: FD->getLocation()), DynamicArgs: {});
1656 } else if (ShouldEmitUnreachable) {
1657 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1658 EmitTrapCall(IntrID: llvm::Intrinsic::trap);
1659 }
1660 if (SanOpts.has(K: SanitizerKind::Return) || ShouldEmitUnreachable) {
1661 Builder.CreateUnreachable();
1662 Builder.ClearInsertionPoint();
1663 }
1664 }
1665
1666 // Emit the standard function epilogue.
1667 FinishFunction(EndLoc: BodyRange.getEnd());
1668
1669 PGO->verifyCounterMap();
1670
1671 if (CurCodeDecl->hasAttr<PersonalityAttr>()) {
1672 StringRef Identifier =
1673 CurCodeDecl->getAttr<PersonalityAttr>()->getRoutine()->getName();
1674 llvm::FunctionCallee PersonalityRoutine =
1675 CGM.CreateRuntimeFunction(Ty: llvm::FunctionType::get(Result: CGM.Int32Ty, isVarArg: true),
1676 Name: Identifier, ExtraAttrs: {}, /*local=*/Local: true);
1677 Fn->setPersonalityFn(cast<llvm::Constant>(Val: PersonalityRoutine.getCallee()));
1678 }
1679
1680 // If we haven't marked the function nothrow through other means, do
1681 // a quick pass now to see if we can.
1682 if (!CurFn->doesNotThrow())
1683 TryMarkNoThrow(F: CurFn);
1684}
1685
1686/// ContainsLabel - Return true if the statement contains a label in it. If
1687/// this statement is not executed normally, it not containing a label means
1688/// that we can just remove the code.
1689bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
1690 // Null statement, not a label!
1691 if (!S) return false;
1692
1693 // If this is a label, we have to emit the code, consider something like:
1694 // if (0) { ... foo: bar(); } goto foo;
1695 //
1696 // TODO: If anyone cared, we could track __label__'s, since we know that you
1697 // can't jump to one from outside their declared region.
1698 if (isa<LabelStmt>(Val: S))
1699 return true;
1700
1701 // If this is a case/default statement, and we haven't seen a switch, we have
1702 // to emit the code.
1703 if (isa<SwitchCase>(Val: S) && !IgnoreCaseStmts)
1704 return true;
1705
1706 // If this is a switch statement, we want to ignore cases below it.
1707 if (isa<SwitchStmt>(Val: S))
1708 IgnoreCaseStmts = true;
1709
1710 // Scan subexpressions for verboten labels.
1711 for (const Stmt *SubStmt : S->children())
1712 if (ContainsLabel(S: SubStmt, IgnoreCaseStmts))
1713 return true;
1714
1715 return false;
1716}
1717
1718/// containsBreak - Return true if the statement contains a break out of it.
1719/// If the statement (recursively) contains a switch or loop with a break
1720/// inside of it, this is fine.
1721bool CodeGenFunction::containsBreak(const Stmt *S) {
1722 // Null statement, not a label!
1723 if (!S) return false;
1724
1725 // If this is a switch or loop that defines its own break scope, then we can
1726 // include it and anything inside of it.
1727 if (isa<SwitchStmt>(Val: S) || isa<WhileStmt>(Val: S) || isa<DoStmt>(Val: S) ||
1728 isa<ForStmt>(Val: S))
1729 return false;
1730
1731 if (isa<BreakStmt>(Val: S))
1732 return true;
1733
1734 // Scan subexpressions for verboten breaks.
1735 for (const Stmt *SubStmt : S->children())
1736 if (containsBreak(S: SubStmt))
1737 return true;
1738
1739 return false;
1740}
1741
1742bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) {
1743 if (!S) return false;
1744
1745 // Some statement kinds add a scope and thus never add a decl to the current
1746 // scope. Note, this list is longer than the list of statements that might
1747 // have an unscoped decl nested within them, but this way is conservatively
1748 // correct even if more statement kinds are added.
1749 if (isa<IfStmt>(Val: S) || isa<SwitchStmt>(Val: S) || isa<WhileStmt>(Val: S) ||
1750 isa<DoStmt>(Val: S) || isa<ForStmt>(Val: S) || isa<CompoundStmt>(Val: S) ||
1751 isa<CXXForRangeStmt>(Val: S) || isa<CXXTryStmt>(Val: S) ||
1752 isa<ObjCForCollectionStmt>(Val: S) || isa<ObjCAtTryStmt>(Val: S))
1753 return false;
1754
1755 if (isa<DeclStmt>(Val: S))
1756 return true;
1757
1758 for (const Stmt *SubStmt : S->children())
1759 if (mightAddDeclToScope(S: SubStmt))
1760 return true;
1761
1762 return false;
1763}
1764
1765/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1766/// to a constant, or if it does but contains a label, return false. If it
1767/// constant folds return true and set the boolean result in Result.
1768bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1769 bool &ResultBool,
1770 bool AllowLabels) {
1771 // If MC/DC is enabled, disable folding so that we can instrument all
1772 // conditions to yield complete test vectors. We still keep track of
1773 // folded conditions during region mapping and visualization.
1774 if (!AllowLabels && CGM.getCodeGenOpts().hasProfileClangInstr() &&
1775 CGM.getCodeGenOpts().MCDCCoverage)
1776 return false;
1777
1778 llvm::APSInt ResultInt;
1779 if (!ConstantFoldsToSimpleInteger(Cond, Result&: ResultInt, AllowLabels))
1780 return false;
1781
1782 ResultBool = ResultInt.getBoolValue();
1783 return true;
1784}
1785
1786/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1787/// to a constant, or if it does but contains a label, return false. If it
1788/// constant folds return true and set the folded value.
1789bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1790 llvm::APSInt &ResultInt,
1791 bool AllowLabels) {
1792 // FIXME: Rename and handle conversion of other evaluatable things
1793 // to bool.
1794 Expr::EvalResult Result;
1795 if (!Cond->EvaluateAsInt(Result, Ctx: getContext()))
1796 return false; // Not foldable, not integer or not fully evaluatable.
1797
1798 llvm::APSInt Int = Result.Val.getInt();
1799 if (!AllowLabels && CodeGenFunction::ContainsLabel(S: Cond))
1800 return false; // Contains a label.
1801
1802 PGO->markStmtMaybeUsed(S: Cond);
1803 ResultInt = std::move(Int);
1804 return true;
1805}
1806
1807/// Strip parentheses and simplistic logical-NOT operators.
1808const Expr *CodeGenFunction::stripCond(const Expr *C) {
1809 while (true) {
1810 const Expr *SC = IgnoreExprNodes(
1811 E: C, Fns&: IgnoreParensSingleStep, Fns&: IgnoreUOpLNotSingleStep,
1812 Fns&: IgnoreBuiltinExpectSingleStep, Fns&: IgnoreImplicitCastsSingleStep);
1813 if (C == SC)
1814 return SC;
1815 C = SC;
1816 }
1817}
1818
1819/// Determine whether the given condition is an instrumentable condition
1820/// (i.e. no "&&" or "||").
1821bool CodeGenFunction::isInstrumentedCondition(const Expr *C) {
1822 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: stripCond(C));
1823 return (!BOp || !BOp->isLogicalOp());
1824}
1825
1826/// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
1827/// increments a profile counter based on the semantics of the given logical
1828/// operator opcode. This is used to instrument branch condition coverage for
1829/// logical operators.
1830void CodeGenFunction::EmitBranchToCounterBlock(
1831 const Expr *Cond, BinaryOperator::Opcode LOp, llvm::BasicBlock *TrueBlock,
1832 llvm::BasicBlock *FalseBlock, uint64_t TrueCount /* = 0 */,
1833 Stmt::Likelihood LH /* =None */, const Expr *CntrIdx /* = nullptr */) {
1834 // If not instrumenting, just emit a branch.
1835 bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr();
1836 if (!InstrumentRegions || !isInstrumentedCondition(C: Cond))
1837 return EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount, LH);
1838
1839 const Stmt *CntrStmt = (CntrIdx ? CntrIdx : Cond);
1840
1841 llvm::BasicBlock *ThenBlock = nullptr;
1842 llvm::BasicBlock *ElseBlock = nullptr;
1843 llvm::BasicBlock *NextBlock = nullptr;
1844
1845 // Create the block we'll use to increment the appropriate counter.
1846 llvm::BasicBlock *CounterIncrBlock = createBasicBlock(name: "lop.rhscnt");
1847
1848 llvm::BasicBlock *SkipIncrBlock =
1849 (hasSkipCounter(S: CntrStmt) ? createBasicBlock(name: "lop.rhsskip") : nullptr);
1850 llvm::BasicBlock *SkipNextBlock = nullptr;
1851
1852 // Set block pointers according to Logical-AND (BO_LAnd) semantics. This
1853 // means we need to evaluate the condition and increment the counter on TRUE:
1854 //
1855 // if (Cond)
1856 // goto CounterIncrBlock;
1857 // else
1858 // goto FalseBlock;
1859 //
1860 // CounterIncrBlock:
1861 // Counter++;
1862 // goto TrueBlock;
1863
1864 if (LOp == BO_LAnd) {
1865 SkipNextBlock = FalseBlock;
1866 ThenBlock = CounterIncrBlock;
1867 ElseBlock = (SkipIncrBlock ? SkipIncrBlock : SkipNextBlock);
1868 NextBlock = TrueBlock;
1869 }
1870
1871 // Set block pointers according to Logical-OR (BO_LOr) semantics. This means
1872 // we need to evaluate the condition and increment the counter on FALSE:
1873 //
1874 // if (Cond)
1875 // goto TrueBlock;
1876 // else
1877 // goto CounterIncrBlock;
1878 //
1879 // CounterIncrBlock:
1880 // Counter++;
1881 // goto FalseBlock;
1882
1883 else if (LOp == BO_LOr) {
1884 SkipNextBlock = TrueBlock;
1885 ThenBlock = (SkipIncrBlock ? SkipIncrBlock : SkipNextBlock);
1886 ElseBlock = CounterIncrBlock;
1887 NextBlock = FalseBlock;
1888 } else {
1889 llvm_unreachable("Expected Opcode must be that of a Logical Operator");
1890 }
1891
1892 // Emit Branch based on condition.
1893 EmitBranchOnBoolExpr(Cond, TrueBlock: ThenBlock, FalseBlock: ElseBlock, TrueCount, LH);
1894
1895 if (SkipIncrBlock) {
1896 EmitBlock(BB: SkipIncrBlock);
1897 incrementProfileCounter(ExecSkip: UseSkipPath, S: CntrStmt);
1898 EmitBranch(Block: SkipNextBlock);
1899 }
1900
1901 // Emit the block containing the counter increment(s).
1902 EmitBlock(BB: CounterIncrBlock);
1903
1904 // Increment corresponding counter; if index not provided, use Cond as index.
1905 incrementProfileCounter(ExecSkip: UseExecPath, S: CntrStmt);
1906
1907 // Go to the next block.
1908 EmitBranch(Block: NextBlock);
1909}
1910
1911/// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an if
1912/// statement) to the specified blocks. Based on the condition, this might try
1913/// to simplify the codegen of the conditional based on the branch.
1914/// \param LH The value of the likelihood attribute on the True branch.
1915/// \param ConditionalOp Used by MC/DC code coverage to track the result of the
1916/// ConditionalOperator (ternary) through a recursive call for the operator's
1917/// LHS and RHS nodes.
1918void CodeGenFunction::EmitBranchOnBoolExpr(
1919 const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock,
1920 uint64_t TrueCount, Stmt::Likelihood LH, const Expr *ConditionalOp,
1921 const VarDecl *ConditionalDecl) {
1922 Cond = Cond->IgnoreParens();
1923
1924 if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Val: Cond)) {
1925 bool HasSkip = hasSkipCounter(S: CondBOp);
1926
1927 // Handle X && Y in a condition.
1928 if (CondBOp->getOpcode() == BO_LAnd) {
1929 // If we have "1 && X", simplify the code. "0 && X" would have constant
1930 // folded if the case was simple enough.
1931 bool ConstantBool = false;
1932 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getLHS(), ResultBool&: ConstantBool) &&
1933 ConstantBool) {
1934 // br(1 && X) -> br(X).
1935 incrementProfileCounter(S: CondBOp);
1936 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LAnd, TrueBlock,
1937 FalseBlock, TrueCount, LH);
1938 return;
1939 }
1940
1941 // If we have "X && 1", simplify the code to use an uncond branch.
1942 // "X && 0" would have been constant folded to 0.
1943 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getRHS(), ResultBool&: ConstantBool) &&
1944 ConstantBool) {
1945 // br(X && 1) -> br(X).
1946 EmitBranchToCounterBlock(Cond: CondBOp->getLHS(), LOp: BO_LAnd, TrueBlock,
1947 FalseBlock, TrueCount, LH, CntrIdx: CondBOp);
1948 return;
1949 }
1950
1951 // Emit the LHS as a conditional. If the LHS conditional is false, we
1952 // want to jump to the FalseBlock.
1953 llvm::BasicBlock *LHSTrue = createBasicBlock(name: "land.lhs.true");
1954 llvm::BasicBlock *LHSFalse =
1955 (HasSkip ? createBasicBlock(name: "land.lhsskip") : FalseBlock);
1956 // The counter tells us how often we evaluate RHS, and all of TrueCount
1957 // can be propagated to that branch.
1958 uint64_t RHSCount = getProfileCount(S: CondBOp->getRHS());
1959
1960 ConditionalEvaluation eval(*this);
1961 {
1962 ApplyDebugLocation DL(*this, Cond);
1963 // Propagate the likelihood attribute like __builtin_expect
1964 // __builtin_expect(X && Y, 1) -> X and Y are likely
1965 // __builtin_expect(X && Y, 0) -> only Y is unlikely
1966 EmitBranchOnBoolExpr(Cond: CondBOp->getLHS(), TrueBlock: LHSTrue, FalseBlock: LHSFalse, TrueCount: RHSCount,
1967 LH: LH == Stmt::LH_Unlikely ? Stmt::LH_None : LH);
1968 if (HasSkip) {
1969 EmitBlock(BB: LHSFalse);
1970 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondBOp);
1971 EmitBranch(Block: FalseBlock);
1972 }
1973 EmitBlock(BB: LHSTrue);
1974 }
1975
1976 incrementProfileCounter(ExecSkip: UseExecPath, S: CondBOp);
1977 setCurrentProfileCount(getProfileCount(S: CondBOp->getRHS()));
1978
1979 // Any temporaries created here are conditional.
1980 eval.begin(CGF&: *this);
1981 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LAnd, TrueBlock,
1982 FalseBlock, TrueCount, LH);
1983 eval.end(CGF&: *this);
1984 return;
1985 }
1986
1987 if (CondBOp->getOpcode() == BO_LOr) {
1988 // If we have "0 || X", simplify the code. "1 || X" would have constant
1989 // folded if the case was simple enough.
1990 bool ConstantBool = false;
1991 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getLHS(), ResultBool&: ConstantBool) &&
1992 !ConstantBool) {
1993 // br(0 || X) -> br(X).
1994 incrementProfileCounter(S: CondBOp);
1995 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LOr, TrueBlock,
1996 FalseBlock, TrueCount, LH);
1997 return;
1998 }
1999
2000 // If we have "X || 0", simplify the code to use an uncond branch.
2001 // "X || 1" would have been constant folded to 1.
2002 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getRHS(), ResultBool&: ConstantBool) &&
2003 !ConstantBool) {
2004 // br(X || 0) -> br(X).
2005 EmitBranchToCounterBlock(Cond: CondBOp->getLHS(), LOp: BO_LOr, TrueBlock,
2006 FalseBlock, TrueCount, LH, CntrIdx: CondBOp);
2007 return;
2008 }
2009 // Emit the LHS as a conditional. If the LHS conditional is true, we
2010 // want to jump to the TrueBlock.
2011 llvm::BasicBlock *LHSTrue =
2012 (HasSkip ? createBasicBlock(name: "lor.lhsskip") : TrueBlock);
2013 llvm::BasicBlock *LHSFalse = createBasicBlock(name: "lor.lhs.false");
2014 // We have the count for entry to the RHS and for the whole expression
2015 // being true, so we can divy up True count between the short circuit and
2016 // the RHS.
2017 uint64_t LHSCount =
2018 getCurrentProfileCount() - getProfileCount(S: CondBOp->getRHS());
2019 uint64_t RHSCount = TrueCount - LHSCount;
2020
2021 ConditionalEvaluation eval(*this);
2022 {
2023 // Propagate the likelihood attribute like __builtin_expect
2024 // __builtin_expect(X || Y, 1) -> only Y is likely
2025 // __builtin_expect(X || Y, 0) -> both X and Y are unlikely
2026 ApplyDebugLocation DL(*this, Cond);
2027 EmitBranchOnBoolExpr(Cond: CondBOp->getLHS(), TrueBlock: LHSTrue, FalseBlock: LHSFalse, TrueCount: LHSCount,
2028 LH: LH == Stmt::LH_Likely ? Stmt::LH_None : LH);
2029 if (HasSkip) {
2030 EmitBlock(BB: LHSTrue);
2031 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondBOp);
2032 EmitBranch(Block: TrueBlock);
2033 }
2034 EmitBlock(BB: LHSFalse);
2035 }
2036
2037 incrementProfileCounter(ExecSkip: UseExecPath, S: CondBOp);
2038 setCurrentProfileCount(getProfileCount(S: CondBOp->getRHS()));
2039
2040 // Any temporaries created here are conditional.
2041 eval.begin(CGF&: *this);
2042 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LOr, TrueBlock, FalseBlock,
2043 TrueCount: RHSCount, LH);
2044
2045 eval.end(CGF&: *this);
2046 return;
2047 }
2048 }
2049
2050 if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Val: Cond)) {
2051 // br(!x, t, f) -> br(x, f, t)
2052 // Avoid doing this optimization when instrumenting a condition for MC/DC.
2053 // LNot is taken as part of the condition for simplicity, and changing its
2054 // sense negatively impacts test vector tracking.
2055 bool MCDCCondition = CGM.getCodeGenOpts().hasProfileClangInstr() &&
2056 CGM.getCodeGenOpts().MCDCCoverage &&
2057 isInstrumentedCondition(C: Cond);
2058 if (CondUOp->getOpcode() == UO_LNot && !MCDCCondition) {
2059 // Negate the count.
2060 uint64_t FalseCount = getCurrentProfileCount() - TrueCount;
2061 // The values of the enum are chosen to make this negation possible.
2062 LH = static_cast<Stmt::Likelihood>(-LH);
2063 // Negate the condition and swap the destination blocks.
2064 return EmitBranchOnBoolExpr(Cond: CondUOp->getSubExpr(), TrueBlock: FalseBlock, FalseBlock: TrueBlock,
2065 TrueCount: FalseCount, LH);
2066 }
2067 }
2068
2069 if (const ConditionalOperator *CondOp = dyn_cast<ConditionalOperator>(Val: Cond)) {
2070 // br(c ? x : y, t, f) -> br(c, br(x, t, f), br(y, t, f))
2071 llvm::BasicBlock *LHSBlock = createBasicBlock(name: "cond.true");
2072 llvm::BasicBlock *RHSBlock = createBasicBlock(name: "cond.false");
2073
2074 // The ConditionalOperator itself has no likelihood information for its
2075 // true and false branches. This matches the behavior of __builtin_expect.
2076 ConditionalEvaluation cond(*this);
2077 EmitBranchOnBoolExpr(Cond: CondOp->getCond(), TrueBlock: LHSBlock, FalseBlock: RHSBlock,
2078 TrueCount: getProfileCount(S: CondOp), LH: Stmt::LH_None);
2079
2080 // When computing PGO branch weights, we only know the overall count for
2081 // the true block. This code is essentially doing tail duplication of the
2082 // naive code-gen, introducing new edges for which counts are not
2083 // available. Divide the counts proportionally between the LHS and RHS of
2084 // the conditional operator.
2085 uint64_t LHSScaledTrueCount = 0;
2086 if (TrueCount) {
2087 double LHSRatio =
2088 getProfileCount(S: CondOp) / (double)getCurrentProfileCount();
2089 LHSScaledTrueCount = TrueCount * LHSRatio;
2090 }
2091
2092 cond.begin(CGF&: *this);
2093 EmitBlock(BB: LHSBlock);
2094 incrementProfileCounter(ExecSkip: UseExecPath, S: CondOp);
2095 {
2096 ApplyDebugLocation DL(*this, Cond);
2097 EmitBranchOnBoolExpr(Cond: CondOp->getLHS(), TrueBlock, FalseBlock,
2098 TrueCount: LHSScaledTrueCount, LH, ConditionalOp: CondOp);
2099 }
2100 cond.end(CGF&: *this);
2101
2102 cond.begin(CGF&: *this);
2103 EmitBlock(BB: RHSBlock);
2104 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondOp);
2105 EmitBranchOnBoolExpr(Cond: CondOp->getRHS(), TrueBlock, FalseBlock,
2106 TrueCount: TrueCount - LHSScaledTrueCount, LH, ConditionalOp: CondOp);
2107 cond.end(CGF&: *this);
2108
2109 return;
2110 }
2111
2112 if (const CXXThrowExpr *Throw = dyn_cast<CXXThrowExpr>(Val: Cond)) {
2113 // Conditional operator handling can give us a throw expression as a
2114 // condition for a case like:
2115 // br(c ? throw x : y, t, f) -> br(c, br(throw x, t, f), br(y, t, f)
2116 // Fold this to:
2117 // br(c, throw x, br(y, t, f))
2118 EmitCXXThrowExpr(E: Throw, /*KeepInsertionPoint*/false);
2119 return;
2120 }
2121
2122 // Emit the code with the fully general case.
2123 llvm::Value *CondV;
2124 {
2125 ApplyDebugLocation DL(*this, Cond);
2126 CondV = EvaluateExprAsBool(E: Cond);
2127 }
2128
2129 MaybeEmitDeferredVarDeclInit(var: ConditionalDecl);
2130
2131 // If not at the top of the logical operator nest, update MCDC temp with the
2132 // boolean result of the evaluated condition.
2133 {
2134 const Expr *MCDCBaseExpr = Cond;
2135 // When a nested ConditionalOperator (ternary) is encountered in a boolean
2136 // expression, MC/DC tracks the result of the ternary, and this is tied to
2137 // the ConditionalOperator expression and not the ternary's LHS or RHS. If
2138 // this is the case, the ConditionalOperator expression is passed through
2139 // the ConditionalOp parameter and then used as the MCDC base expression.
2140 if (ConditionalOp)
2141 MCDCBaseExpr = ConditionalOp;
2142
2143 if (isMCDCBranchExpr(E: stripCond(C: MCDCBaseExpr)) &&
2144 !isMCDCDecisionExpr(E: stripCond(C: Cond)))
2145 maybeUpdateMCDCCondBitmap(E: MCDCBaseExpr, Val: CondV);
2146 }
2147
2148 llvm::MDNode *Weights = nullptr;
2149 llvm::MDNode *Unpredictable = nullptr;
2150
2151 // If the branch has a condition wrapped by __builtin_unpredictable,
2152 // create metadata that specifies that the branch is unpredictable.
2153 // Don't bother if not optimizing because that metadata would not be used.
2154 auto *Call = dyn_cast<CallExpr>(Val: Cond->IgnoreImpCasts());
2155 if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
2156 auto *FD = dyn_cast_or_null<FunctionDecl>(Val: Call->getCalleeDecl());
2157 if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
2158 llvm::MDBuilder MDHelper(getLLVMContext());
2159 Unpredictable = MDHelper.createUnpredictable();
2160 }
2161 }
2162
2163 // If there is a Likelihood knowledge for the cond, lower it.
2164 // Note that if not optimizing this won't emit anything.
2165 llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(Cond: CondV, LH);
2166 if (CondV != NewCondV)
2167 CondV = NewCondV;
2168 else {
2169 // Otherwise, lower profile counts. Note that we do this even at -O0.
2170 uint64_t CurrentCount = std::max(a: getCurrentProfileCount(), b: TrueCount);
2171 Weights = createProfileWeights(TrueCount, FalseCount: CurrentCount - TrueCount);
2172 }
2173
2174 llvm::Instruction *BrInst = Builder.CreateCondBr(Cond: CondV, True: TrueBlock, False: FalseBlock,
2175 BranchWeights: Weights, Unpredictable);
2176 addInstToNewSourceAtom(KeyInstruction: BrInst, Backup: CondV);
2177
2178 switch (HLSLControlFlowAttr) {
2179 case HLSLControlFlowHintAttr::Microsoft_branch:
2180 case HLSLControlFlowHintAttr::Microsoft_flatten: {
2181 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2182
2183 llvm::ConstantInt *BranchHintConstant =
2184 HLSLControlFlowAttr ==
2185 HLSLControlFlowHintAttr::Spelling::Microsoft_branch
2186 ? llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 1)
2187 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 2);
2188
2189 SmallVector<llvm::Metadata *, 2> Vals(
2190 {MDHelper.createString(Str: "hlsl.controlflow.hint"),
2191 MDHelper.createConstant(C: BranchHintConstant)});
2192 BrInst->setMetadata(Kind: "hlsl.controlflow.hint",
2193 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Vals));
2194 break;
2195 }
2196 // This is required to avoid warnings during compilation
2197 case HLSLControlFlowHintAttr::SpellingNotCalculated:
2198 break;
2199 }
2200}
2201
2202llvm::Value *CodeGenFunction::EmitScalarOrConstFoldImmArg(unsigned ICEArguments,
2203 unsigned Idx,
2204 const CallExpr *E) {
2205 llvm::Value *Arg = nullptr;
2206 if ((ICEArguments & (1 << Idx)) == 0) {
2207 Arg = EmitScalarExpr(E: E->getArg(Arg: Idx));
2208 } else {
2209 // If this is required to be a constant, constant fold it so that we
2210 // know that the generated intrinsic gets a ConstantInt.
2211 std::optional<llvm::APSInt> Result =
2212 E->getArg(Arg: Idx)->getIntegerConstantExpr(Ctx: getContext());
2213 assert(Result && "Expected argument to be a constant");
2214 Arg = llvm::ConstantInt::get(Context&: getLLVMContext(), V: *Result);
2215 }
2216 return Arg;
2217}
2218
2219/// ErrorUnsupported - Print out an error that codegen doesn't support the
2220/// specified stmt yet.
2221void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) {
2222 CGM.ErrorUnsupported(S, Type);
2223}
2224
2225/// emitNonZeroVLAInit - Emit the "zero" initialization of a
2226/// variable-length array whose elements have a non-zero bit-pattern.
2227///
2228/// \param baseType the inner-most element type of the array
2229/// \param src - a char* pointing to the bit-pattern for a single
2230/// base element of the array
2231/// \param sizeInChars - the total size of the VLA, in chars
2232static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
2233 Address dest, Address src,
2234 llvm::Value *sizeInChars) {
2235 CGBuilderTy &Builder = CGF.Builder;
2236
2237 CharUnits baseSize = CGF.getContext().getTypeSizeInChars(T: baseType);
2238 llvm::Value *baseSizeInChars
2239 = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: baseSize.getQuantity());
2240
2241 Address begin = dest.withElementType(ElemTy: CGF.Int8Ty);
2242 llvm::Value *end = Builder.CreateInBoundsGEP(Ty: begin.getElementType(),
2243 Ptr: begin.emitRawPointer(CGF),
2244 IdxList: sizeInChars, Name: "vla.end");
2245
2246 llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
2247 llvm::BasicBlock *loopBB = CGF.createBasicBlock(name: "vla-init.loop");
2248 llvm::BasicBlock *contBB = CGF.createBasicBlock(name: "vla-init.cont");
2249
2250 // Make a loop over the VLA. C99 guarantees that the VLA element
2251 // count must be nonzero.
2252 CGF.EmitBlock(BB: loopBB);
2253
2254 llvm::PHINode *cur = Builder.CreatePHI(Ty: begin.getType(), NumReservedValues: 2, Name: "vla.cur");
2255 cur->addIncoming(V: begin.emitRawPointer(CGF), BB: originBB);
2256
2257 CharUnits curAlign =
2258 dest.getAlignment().alignmentOfArrayElement(elementSize: baseSize);
2259
2260 // memcpy the individual element bit-pattern.
2261 Builder.CreateMemCpy(Dest: Address(cur, CGF.Int8Ty, curAlign), Src: src, Size: baseSizeInChars,
2262 /*volatile*/ IsVolatile: false);
2263
2264 // Go to the next element.
2265 llvm::Value *next =
2266 Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: cur, IdxList: baseSizeInChars, Name: "vla.next");
2267
2268 // Leave if that's the end of the VLA.
2269 llvm::Value *done = Builder.CreateICmpEQ(LHS: next, RHS: end, Name: "vla-init.isdone");
2270 Builder.CreateCondBr(Cond: done, True: contBB, False: loopBB);
2271 cur->addIncoming(V: next, BB: loopBB);
2272
2273 CGF.EmitBlock(BB: contBB);
2274}
2275
2276Address CodeGenFunction::EmitAddressOfPFPField(Address RecordPtr,
2277 const PFPField &Field) {
2278 return EmitAddressOfPFPField(
2279 RecordPtr,
2280 FieldPtr: Builder.CreateConstInBoundsByteGEP(Addr: RecordPtr.withElementType(ElemTy: Int8Ty),
2281 Offset: Field.Offset),
2282 Field: Field.Field);
2283}
2284
2285Address CodeGenFunction::EmitAddressOfPFPField(Address RecordPtr,
2286 Address PtrPtr,
2287 const FieldDecl *Field) {
2288 llvm::Value *Disc;
2289 if (CGM.getContext().arePFPFieldsTriviallyCopyable(RD: Field->getParent())) {
2290 uint64_t FieldSignature =
2291 llvm::getPointerAuthStableSipHash(S: CGM.getPFPFieldName(FD: Field));
2292 Disc = llvm::ConstantInt::get(Ty: CGM.Int64Ty, V: FieldSignature);
2293 } else
2294 Disc = Builder.CreatePtrToInt(V: RecordPtr.getBasePointer(), DestTy: CGM.Int64Ty);
2295
2296 llvm::GlobalValue *DS = CGM.getPFPDeactivationSymbol(FD: Field);
2297 llvm::OperandBundleDef DSBundle("deactivation-symbol", DS);
2298 llvm::Value *Args[] = {PtrPtr.getBasePointer(), Disc, Builder.getTrue()};
2299 return Address(
2300 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::protected_field_ptr,
2301 Tys: PtrPtr.getType()),
2302 Args, OpBundles: DSBundle),
2303 VoidPtrTy, PtrPtr.getAlignment());
2304}
2305
2306void
2307CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
2308 // Ignore empty classes in C++.
2309 if (getLangOpts().CPlusPlus)
2310 if (const auto *RD = Ty->getAsCXXRecordDecl(); RD && RD->isEmpty())
2311 return;
2312
2313 if (DestPtr.getElementType() != Int8Ty)
2314 DestPtr = DestPtr.withElementType(ElemTy: Int8Ty);
2315
2316 // Get size and alignment info for this aggregate.
2317 CharUnits size = getContext().getTypeSizeInChars(T: Ty);
2318
2319 llvm::Value *SizeVal;
2320 const VariableArrayType *vla;
2321
2322 // Don't bother emitting a zero-byte memset.
2323 if (size.isZero()) {
2324 // But note that getTypeInfo returns 0 for a VLA.
2325 if (const VariableArrayType *vlaType =
2326 dyn_cast_or_null<VariableArrayType>(
2327 Val: getContext().getAsArrayType(T: Ty))) {
2328 auto VlaSize = getVLASize(vla: vlaType);
2329 SizeVal = VlaSize.NumElts;
2330 CharUnits eltSize = getContext().getTypeSizeInChars(T: VlaSize.Type);
2331 if (!eltSize.isOne())
2332 SizeVal = Builder.CreateNUWMul(LHS: SizeVal, RHS: CGM.getSize(numChars: eltSize));
2333 vla = vlaType;
2334 } else {
2335 return;
2336 }
2337 } else {
2338 SizeVal = CGM.getSize(numChars: size);
2339 vla = nullptr;
2340 }
2341
2342 // If the type contains a pointer to data member we can't memset it to zero.
2343 // Instead, create a null constant and copy it to the destination.
2344 // TODO: there are other patterns besides zero that we can usefully memset,
2345 // like -1, which happens to be the pattern used by member-pointers.
2346 if (!CGM.getTypes().isZeroInitializable(T: Ty)) {
2347 // For a VLA, emit a single element, then splat that over the VLA.
2348 if (vla) Ty = getContext().getBaseElementType(VAT: vla);
2349
2350 llvm::Constant *NullConstant = CGM.EmitNullConstant(T: Ty);
2351
2352 llvm::GlobalVariable *NullVariable =
2353 new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
2354 /*isConstant=*/true,
2355 llvm::GlobalVariable::PrivateLinkage,
2356 NullConstant, Twine());
2357 CharUnits NullAlign = DestPtr.getAlignment();
2358 NullVariable->setAlignment(NullAlign.getAsAlign());
2359 Address SrcPtr(NullVariable, Builder.getInt8Ty(), NullAlign);
2360
2361 if (vla) return emitNonZeroVLAInit(CGF&: *this, baseType: Ty, dest: DestPtr, src: SrcPtr, sizeInChars: SizeVal);
2362
2363 // Get and call the appropriate llvm.memcpy overload.
2364 Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: SizeVal, IsVolatile: false);
2365 } else {
2366 // Otherwise, just memset the whole thing to zero. This is legal
2367 // because in LLVM, all default initializers (other than the ones we just
2368 // handled above, and the case handled below) are guaranteed to have a bit
2369 // pattern of all zeros.
2370 Builder.CreateMemSet(Dest: DestPtr, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
2371 }
2372
2373 // With the pointer field protection feature, null pointers do not have a bit
2374 // pattern of zero in memory, so we must initialize them separately.
2375 for (auto &Field : getContext().findPFPFields(Ty)) {
2376 auto addr = EmitAddressOfPFPField(RecordPtr: DestPtr, Field);
2377 Builder.CreateStore(Val: llvm::ConstantPointerNull::get(T: VoidPtrTy), Addr: addr);
2378 }
2379}
2380
2381llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
2382 // Make sure that there is a block for the indirect goto.
2383 if (!IndirectBranch)
2384 GetIndirectGotoBlock();
2385
2386 llvm::BasicBlock *BB = getJumpDestForLabel(S: L).getBlock();
2387
2388 // Make sure the indirect branch includes all of the address-taken blocks.
2389 IndirectBranch->addDestination(Dest: BB);
2390 return llvm::BlockAddress::get(Ty: CurFn->getType(), BB);
2391}
2392
2393llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {
2394 // If we already made the indirect branch for indirect goto, return its block.
2395 if (IndirectBranch) return IndirectBranch->getParent();
2396
2397 CGBuilderTy TmpBuilder(CGM, createBasicBlock(name: "indirectgoto"));
2398
2399 // Create the PHI node that indirect gotos will add entries to.
2400 llvm::Value *DestVal = TmpBuilder.CreatePHI(Ty: Int8PtrTy, NumReservedValues: 0,
2401 Name: "indirect.goto.dest");
2402
2403 // Create the indirect branch instruction.
2404 IndirectBranch = TmpBuilder.CreateIndirectBr(Addr: DestVal);
2405 return IndirectBranch->getParent();
2406}
2407
2408/// Computes the length of an array in elements, as well as the base
2409/// element type and a properly-typed first element pointer.
2410llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
2411 QualType &baseType,
2412 Address &addr) {
2413 const ArrayType *arrayType = origArrayType;
2414
2415 // If it's a VLA, we have to load the stored size. Note that
2416 // this is the size of the VLA in bytes, not its size in elements.
2417 llvm::Value *numVLAElements = nullptr;
2418 if (isa<VariableArrayType>(Val: arrayType)) {
2419 numVLAElements = getVLASize(vla: cast<VariableArrayType>(Val: arrayType)).NumElts;
2420
2421 // Walk into all VLAs. This doesn't require changes to addr,
2422 // which has type T* where T is the first non-VLA element type.
2423 do {
2424 QualType elementType = arrayType->getElementType();
2425 arrayType = getContext().getAsArrayType(T: elementType);
2426
2427 // If we only have VLA components, 'addr' requires no adjustment.
2428 if (!arrayType) {
2429 baseType = elementType;
2430 return numVLAElements;
2431 }
2432 } while (isa<VariableArrayType>(Val: arrayType));
2433
2434 // We get out here only if we find a constant array type
2435 // inside the VLA.
2436 }
2437
2438 // We have some number of constant-length arrays, so addr should
2439 // have LLVM type [M x [N x [...]]]*. Build a GEP that walks
2440 // down to the first element of addr.
2441 SmallVector<llvm::Value*, 8> gepIndices;
2442
2443 // GEP down to the array type.
2444 llvm::ConstantInt *zero = Builder.getInt32(C: 0);
2445 gepIndices.push_back(Elt: zero);
2446
2447 uint64_t countFromCLAs = 1;
2448 QualType eltType;
2449
2450 llvm::ArrayType *llvmArrayType =
2451 dyn_cast<llvm::ArrayType>(Val: addr.getElementType());
2452 while (llvmArrayType) {
2453 assert(isa<ConstantArrayType>(arrayType));
2454 assert(cast<ConstantArrayType>(arrayType)->getZExtSize() ==
2455 llvmArrayType->getNumElements());
2456
2457 gepIndices.push_back(Elt: zero);
2458 countFromCLAs *= llvmArrayType->getNumElements();
2459 eltType = arrayType->getElementType();
2460
2461 llvmArrayType =
2462 dyn_cast<llvm::ArrayType>(Val: llvmArrayType->getElementType());
2463 arrayType = getContext().getAsArrayType(T: arrayType->getElementType());
2464 assert((!llvmArrayType || arrayType) &&
2465 "LLVM and Clang types are out-of-synch");
2466 }
2467
2468 if (arrayType) {
2469 // From this point onwards, the Clang array type has been emitted
2470 // as some other type (probably a packed struct). Compute the array
2471 // size, and just emit the 'begin' expression as a bitcast.
2472 while (arrayType) {
2473 countFromCLAs *= cast<ConstantArrayType>(Val: arrayType)->getZExtSize();
2474 eltType = arrayType->getElementType();
2475 arrayType = getContext().getAsArrayType(T: eltType);
2476 }
2477
2478 llvm::Type *baseType = ConvertType(T: eltType);
2479 addr = addr.withElementType(ElemTy: baseType);
2480 } else {
2481 // Create the actual GEP.
2482 addr = Address(Builder.CreateInBoundsGEP(Ty: addr.getElementType(),
2483 Ptr: addr.emitRawPointer(CGF&: *this),
2484 IdxList: gepIndices, Name: "array.begin"),
2485 ConvertTypeForMem(T: eltType), addr.getAlignment());
2486 }
2487
2488 baseType = eltType;
2489
2490 llvm::Value *numElements
2491 = llvm::ConstantInt::get(Ty: SizeTy, V: countFromCLAs);
2492
2493 // If we had any VLA dimensions, factor them in.
2494 if (numVLAElements)
2495 numElements = Builder.CreateNUWMul(LHS: numVLAElements, RHS: numElements);
2496
2497 return numElements;
2498}
2499
2500CodeGenFunction::VlaSizePair CodeGenFunction::getVLASize(QualType type) {
2501 const VariableArrayType *vla = getContext().getAsVariableArrayType(T: type);
2502 assert(vla && "type was not a variable array type!");
2503 return getVLASize(vla);
2504}
2505
2506CodeGenFunction::VlaSizePair
2507CodeGenFunction::getVLASize(const VariableArrayType *type) {
2508 // The number of elements so far; always size_t.
2509 llvm::Value *numElements = nullptr;
2510
2511 QualType elementType;
2512 do {
2513 elementType = type->getElementType();
2514 llvm::Value *vlaSize = VLASizeMap[type->getSizeExpr()];
2515 assert(vlaSize && "no size for VLA!");
2516 assert(vlaSize->getType() == SizeTy);
2517
2518 if (!numElements) {
2519 numElements = vlaSize;
2520 } else {
2521 // It's undefined behavior if this wraps around, so mark it that way.
2522 // FIXME: Teach -fsanitize=undefined to trap this.
2523 numElements = Builder.CreateNUWMul(LHS: numElements, RHS: vlaSize);
2524 }
2525 } while ((type = getContext().getAsVariableArrayType(T: elementType)));
2526
2527 return { numElements, elementType };
2528}
2529
2530CodeGenFunction::VlaSizePair
2531CodeGenFunction::getVLAElements1D(QualType type) {
2532 const VariableArrayType *vla = getContext().getAsVariableArrayType(T: type);
2533 assert(vla && "type was not a variable array type!");
2534 return getVLAElements1D(vla);
2535}
2536
2537CodeGenFunction::VlaSizePair
2538CodeGenFunction::getVLAElements1D(const VariableArrayType *Vla) {
2539 llvm::Value *VlaSize = VLASizeMap[Vla->getSizeExpr()];
2540 assert(VlaSize && "no size for VLA!");
2541 assert(VlaSize->getType() == SizeTy);
2542 return { VlaSize, Vla->getElementType() };
2543}
2544
2545void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
2546 assert(type->isVariablyModifiedType() &&
2547 "Must pass variably modified type to EmitVLASizes!");
2548
2549 EnsureInsertPoint();
2550
2551 // We're going to walk down into the type and look for VLA
2552 // expressions.
2553 do {
2554 assert(type->isVariablyModifiedType());
2555
2556 const Type *ty = type.getTypePtr();
2557 switch (ty->getTypeClass()) {
2558
2559#define TYPE(Class, Base)
2560#define ABSTRACT_TYPE(Class, Base)
2561#define NON_CANONICAL_TYPE(Class, Base)
2562#define DEPENDENT_TYPE(Class, Base) case Type::Class:
2563#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
2564#include "clang/AST/TypeNodes.inc"
2565 llvm_unreachable("unexpected dependent type!");
2566
2567 // These types are never variably-modified.
2568 case Type::Builtin:
2569 case Type::Complex:
2570 case Type::Vector:
2571 case Type::ExtVector:
2572 case Type::ConstantMatrix:
2573 case Type::Record:
2574 case Type::Enum:
2575 case Type::Using:
2576 case Type::TemplateSpecialization:
2577 case Type::ObjCTypeParam:
2578 case Type::ObjCObject:
2579 case Type::ObjCInterface:
2580 case Type::ObjCObjectPointer:
2581 case Type::BitInt:
2582 case Type::HLSLInlineSpirv:
2583 case Type::PredefinedSugar:
2584 llvm_unreachable("type class is never variably-modified!");
2585
2586 case Type::Adjusted:
2587 type = cast<AdjustedType>(Val: ty)->getAdjustedType();
2588 break;
2589
2590 case Type::Decayed:
2591 type = cast<DecayedType>(Val: ty)->getPointeeType();
2592 break;
2593
2594 case Type::Pointer:
2595 type = cast<PointerType>(Val: ty)->getPointeeType();
2596 break;
2597
2598 case Type::BlockPointer:
2599 type = cast<BlockPointerType>(Val: ty)->getPointeeType();
2600 break;
2601
2602 case Type::LValueReference:
2603 case Type::RValueReference:
2604 type = cast<ReferenceType>(Val: ty)->getPointeeType();
2605 break;
2606
2607 case Type::MemberPointer:
2608 type = cast<MemberPointerType>(Val: ty)->getPointeeType();
2609 break;
2610
2611 case Type::ArrayParameter:
2612 case Type::ConstantArray:
2613 case Type::IncompleteArray:
2614 // Losing element qualification here is fine.
2615 type = cast<ArrayType>(Val: ty)->getElementType();
2616 break;
2617
2618 case Type::VariableArray: {
2619 // Losing element qualification here is fine.
2620 const VariableArrayType *vat = cast<VariableArrayType>(Val: ty);
2621
2622 // Unknown size indication requires no size computation.
2623 // Otherwise, evaluate and record it.
2624 if (const Expr *sizeExpr = vat->getSizeExpr()) {
2625 // It's possible that we might have emitted this already,
2626 // e.g. with a typedef and a pointer to it.
2627 llvm::Value *&entry = VLASizeMap[sizeExpr];
2628 if (!entry) {
2629 llvm::Value *size = EmitScalarExpr(E: sizeExpr);
2630
2631 // C11 6.7.6.2p5:
2632 // If the size is an expression that is not an integer constant
2633 // expression [...] each time it is evaluated it shall have a value
2634 // greater than zero.
2635 if (SanOpts.has(K: SanitizerKind::VLABound)) {
2636 auto CheckOrdinal = SanitizerKind::SO_VLABound;
2637 auto CheckHandler = SanitizerHandler::VLABoundNotPositive;
2638 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2639 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: size->getType());
2640 clang::QualType SEType = sizeExpr->getType();
2641 llvm::Value *CheckCondition =
2642 SEType->isSignedIntegerType()
2643 ? Builder.CreateICmpSGT(LHS: size, RHS: Zero)
2644 : Builder.CreateICmpUGT(LHS: size, RHS: Zero);
2645 llvm::Constant *StaticArgs[] = {
2646 EmitCheckSourceLocation(Loc: sizeExpr->getBeginLoc()),
2647 EmitCheckTypeDescriptor(T: SEType)};
2648 EmitCheck(Checked: std::make_pair(x&: CheckCondition, y&: CheckOrdinal),
2649 Check: CheckHandler, StaticArgs, DynamicArgs: size);
2650 }
2651
2652 // Always zexting here would be wrong if it weren't
2653 // undefined behavior to have a negative bound.
2654 // FIXME: What about when size's type is larger than size_t?
2655 entry = Builder.CreateIntCast(V: size, DestTy: SizeTy, /*signed*/ isSigned: false);
2656 }
2657 }
2658 type = vat->getElementType();
2659 break;
2660 }
2661
2662 case Type::FunctionProto:
2663 case Type::FunctionNoProto:
2664 type = cast<FunctionType>(Val: ty)->getReturnType();
2665 break;
2666
2667 case Type::Paren:
2668 case Type::TypeOf:
2669 case Type::UnaryTransform:
2670 case Type::Attributed:
2671 case Type::BTFTagAttributed:
2672 case Type::OverflowBehavior:
2673 case Type::HLSLAttributedResource:
2674 case Type::SubstTemplateTypeParm:
2675 case Type::MacroQualified:
2676 case Type::CountAttributed:
2677 case Type::LateParsedAttr:
2678 // Keep walking after single level desugaring.
2679 type = type.getSingleStepDesugaredType(Context: getContext());
2680 break;
2681
2682 case Type::Typedef:
2683 case Type::Decltype:
2684 case Type::Auto:
2685 case Type::DeducedTemplateSpecialization:
2686 case Type::PackIndexing:
2687 // Stop walking: nothing to do.
2688 return;
2689
2690 case Type::TypeOfExpr:
2691 // Stop walking: emit typeof expression.
2692 EmitIgnoredExpr(E: cast<TypeOfExprType>(Val: ty)->getUnderlyingExpr());
2693 return;
2694
2695 case Type::Atomic:
2696 type = cast<AtomicType>(Val: ty)->getValueType();
2697 break;
2698
2699 case Type::Pipe:
2700 type = cast<PipeType>(Val: ty)->getElementType();
2701 break;
2702 }
2703 } while (type->isVariablyModifiedType());
2704}
2705
2706Address CodeGenFunction::EmitVAListRef(const Expr* E) {
2707 if (getContext().getBuiltinVaListType()->isArrayType())
2708 return EmitPointerWithAlignment(Addr: E);
2709 return EmitLValue(E).getAddress();
2710}
2711
2712Address CodeGenFunction::EmitMSVAListRef(const Expr *E) {
2713 return EmitLValue(E).getAddress();
2714}
2715
2716Address CodeGenFunction::EmitZOSVAListRef(const Expr *E) {
2717 return EmitPointerWithAlignment(Addr: E);
2718}
2719
2720void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E,
2721 const APValue &Init) {
2722 assert(Init.hasValue() && "Invalid DeclRefExpr initializer!");
2723 if (CGDebugInfo *Dbg = getDebugInfo())
2724 if (CGM.getCodeGenOpts().hasReducedDebugInfo())
2725 Dbg->EmitGlobalVariable(VD: E->getDecl(), Init);
2726}
2727
2728CodeGenFunction::PeepholeProtection
2729CodeGenFunction::protectFromPeepholes(RValue rvalue) {
2730 // At the moment, the only aggressive peephole we do in IR gen
2731 // is trunc(zext) folding, but if we add more, we can easily
2732 // extend this protection.
2733
2734 if (!rvalue.isScalar()) return PeepholeProtection();
2735 llvm::Value *value = rvalue.getScalarVal();
2736 if (!isa<llvm::ZExtInst>(Val: value)) return PeepholeProtection();
2737
2738 // Just make an extra bitcast.
2739 assert(HaveInsertPoint());
2740 llvm::Instruction *inst = new llvm::BitCastInst(value, value->getType(), "",
2741 Builder.GetInsertBlock());
2742
2743 PeepholeProtection protection;
2744 protection.Inst = inst;
2745 return protection;
2746}
2747
2748void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {
2749 if (!protection.Inst) return;
2750
2751 // In theory, we could try to duplicate the peepholes now, but whatever.
2752 protection.Inst->eraseFromParent();
2753}
2754
2755void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2756 QualType Ty, SourceLocation Loc,
2757 SourceLocation AssumptionLoc,
2758 llvm::Value *Alignment,
2759 llvm::Value *OffsetValue) {
2760 if (Alignment->getType() != IntPtrTy)
2761 Alignment =
2762 Builder.CreateIntCast(V: Alignment, DestTy: IntPtrTy, isSigned: false, Name: "casted.align");
2763 if (OffsetValue && OffsetValue->getType() != IntPtrTy)
2764 OffsetValue =
2765 Builder.CreateIntCast(V: OffsetValue, DestTy: IntPtrTy, isSigned: true, Name: "casted.offset");
2766 llvm::Value *TheCheck = nullptr;
2767 if (SanOpts.has(K: SanitizerKind::Alignment)) {
2768 llvm::Value *PtrIntValue =
2769 Builder.CreatePtrToInt(V: PtrValue, DestTy: IntPtrTy, Name: "ptrint");
2770
2771 if (OffsetValue) {
2772 bool IsOffsetZero = false;
2773 if (const auto *CI = dyn_cast<llvm::ConstantInt>(Val: OffsetValue))
2774 IsOffsetZero = CI->isZero();
2775
2776 if (!IsOffsetZero)
2777 PtrIntValue = Builder.CreateSub(LHS: PtrIntValue, RHS: OffsetValue, Name: "offsetptr");
2778 }
2779
2780 llvm::Value *Zero = llvm::ConstantInt::get(Ty: IntPtrTy, V: 0);
2781 llvm::Value *Mask =
2782 Builder.CreateSub(LHS: Alignment, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: 1));
2783 llvm::Value *MaskedPtr = Builder.CreateAnd(LHS: PtrIntValue, RHS: Mask, Name: "maskedptr");
2784 TheCheck = Builder.CreateICmpEQ(LHS: MaskedPtr, RHS: Zero, Name: "maskcond");
2785 }
2786 llvm::Instruction *Assumption = Builder.CreateAlignmentAssumption(
2787 DL: CGM.getDataLayout(), PtrValue, Alignment, OffsetValue);
2788
2789 if (!SanOpts.has(K: SanitizerKind::Alignment))
2790 return;
2791 emitAlignmentAssumptionCheck(Ptr: PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2792 OffsetValue, TheCheck, Assumption);
2793}
2794
2795void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2796 const Expr *E,
2797 SourceLocation AssumptionLoc,
2798 llvm::Value *Alignment,
2799 llvm::Value *OffsetValue) {
2800 QualType Ty = E->getType();
2801 SourceLocation Loc = E->getExprLoc();
2802
2803 emitAlignmentAssumption(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2804 OffsetValue);
2805}
2806
2807llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn,
2808 llvm::Value *AnnotatedVal,
2809 StringRef AnnotationStr,
2810 SourceLocation Location,
2811 const AnnotateAttr *Attr) {
2812 SmallVector<llvm::Value *, 5> Args = {
2813 AnnotatedVal,
2814 CGM.EmitAnnotationString(Str: AnnotationStr),
2815 CGM.EmitAnnotationUnit(Loc: Location),
2816 CGM.EmitAnnotationLineNo(L: Location),
2817 };
2818 if (Attr)
2819 Args.push_back(Elt: CGM.EmitAnnotationArgs(Attr));
2820 return Builder.CreateCall(Callee: AnnotationFn, Args);
2821}
2822
2823void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
2824 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2825 for (const auto *I : D->specific_attrs<AnnotateAttr>())
2826 EmitAnnotationCall(AnnotationFn: CGM.getIntrinsic(IID: llvm::Intrinsic::var_annotation,
2827 Tys: {V->getType(), CGM.ConstGlobalsPtrTy}),
2828 AnnotatedVal: V, AnnotationStr: I->getAnnotation(), Location: D->getLocation(), Attr: I);
2829}
2830
2831Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
2832 Address Addr) {
2833 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2834 llvm::Value *V = Addr.emitRawPointer(CGF&: *this);
2835 llvm::Type *VTy = V->getType();
2836 auto *PTy = dyn_cast<llvm::PointerType>(Val: VTy);
2837 unsigned AS = PTy ? PTy->getAddressSpace() : 0;
2838 llvm::PointerType *IntrinTy =
2839 llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: AS);
2840 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::ptr_annotation,
2841 Tys: {IntrinTy, CGM.ConstGlobalsPtrTy});
2842
2843 for (const auto *I : D->specific_attrs<AnnotateAttr>()) {
2844 // FIXME Always emit the cast inst so we can differentiate between
2845 // annotation on the first field of a struct and annotation on the struct
2846 // itself.
2847 if (VTy != IntrinTy)
2848 V = Builder.CreateBitCast(V, DestTy: IntrinTy);
2849 V = EmitAnnotationCall(AnnotationFn: F, AnnotatedVal: V, AnnotationStr: I->getAnnotation(), Location: D->getLocation(), Attr: I);
2850 V = Builder.CreateBitCast(V, DestTy: VTy);
2851 }
2852
2853 return Address(V, Addr.getElementType(), Addr.getAlignment());
2854}
2855
2856CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
2857
2858CodeGenFunction::SanitizerScope::SanitizerScope(CodeGenFunction *CGF)
2859 : CGF(CGF) {
2860 assert(!CGF->IsSanitizerScope);
2861 CGF->IsSanitizerScope = true;
2862}
2863
2864CodeGenFunction::SanitizerScope::~SanitizerScope() {
2865 CGF->IsSanitizerScope = false;
2866}
2867
2868void CodeGenFunction::InsertHelper(llvm::Instruction *I,
2869 const llvm::Twine &Name,
2870 llvm::BasicBlock::iterator InsertPt) const {
2871 LoopStack.InsertHelper(I);
2872 if (IsSanitizerScope)
2873 I->setNoSanitizeMetadata();
2874}
2875
2876void CGBuilderInserter::InsertHelper(
2877 llvm::Instruction *I, const llvm::Twine &Name,
2878 llvm::BasicBlock::iterator InsertPt) const {
2879 llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, InsertPt);
2880 if (CGF)
2881 CGF->InsertHelper(I, Name, InsertPt);
2882}
2883
2884// Emits an error if we don't have a valid set of target features for the
2885// called function.
2886void CodeGenFunction::checkTargetFeatures(const CallExpr *E,
2887 const FunctionDecl *TargetDecl) {
2888 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
2889 CodeGenUtils::checkTargetFeatures(Ctx&: CGM.getContext(), Diags&: CGM.getDiags(),
2890 LangOpts: getLangOpts(), E, Caller: FD, TargetDecl);
2891}
2892
2893// Emits an error if we don't have a valid set of target features for the
2894// called function.
2895void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
2896 const FunctionDecl *TargetDecl) {
2897 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
2898 CodeGenUtils::checkTargetFeatures(Ctx&: CGM.getContext(), Diags&: CGM.getDiags(),
2899 LangOpts: getLangOpts(), Loc, Caller: FD, TargetDecl);
2900}
2901
2902void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
2903 if (!CGM.getCodeGenOpts().SanitizeStats)
2904 return;
2905
2906 llvm::IRBuilder<> IRB(Builder.GetInsertBlock(), Builder.GetInsertPoint());
2907 IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation());
2908 CGM.getSanStats().create(B&: IRB, SK: SSK);
2909}
2910
2911void CodeGenFunction::EmitKCFIOperandBundle(
2912 const CGCallee &Callee, SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
2913 const CGCalleeInfo &CI = Callee.getAbstractInfo();
2914 const FunctionProtoType *FP = CI.getCalleeFunctionProtoType();
2915 if (!FP)
2916 return;
2917
2918 StringRef Salt;
2919 if (const auto &Info = FP->getExtraAttributeInfo())
2920 Salt = Info.CFISalt;
2921
2922 Bundles.emplace_back(Args: "kcfi", Args: CGM.CreateKCFITypeId(T: FP->desugar(), Salt));
2923}
2924
2925llvm::Value *
2926CodeGenFunction::FormAArch64ResolverCondition(const FMVResolverOption &RO) {
2927 return RO.Features.empty() ? nullptr : EmitAArch64CpuSupports(FeatureStrs: RO.Features);
2928}
2929
2930llvm::Value *
2931CodeGenFunction::FormX86ResolverCondition(const FMVResolverOption &RO) {
2932 llvm::Value *Condition = nullptr;
2933
2934 if (RO.Architecture) {
2935 StringRef Arch = *RO.Architecture;
2936 // If arch= specifies an x86-64 micro-architecture level, test the feature
2937 // with __builtin_cpu_supports, otherwise use __builtin_cpu_is.
2938 if (Arch.starts_with(Prefix: "x86-64"))
2939 Condition = EmitX86CpuSupports(FeatureStrs: {Arch});
2940 else
2941 Condition = EmitX86CpuIs(CPUStr: Arch);
2942 }
2943
2944 if (!RO.Features.empty()) {
2945 llvm::Value *FeatureCond = EmitX86CpuSupports(FeatureStrs: RO.Features);
2946 Condition =
2947 Condition ? Builder.CreateAnd(LHS: Condition, RHS: FeatureCond) : FeatureCond;
2948 }
2949 return Condition;
2950}
2951
2952static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
2953 llvm::Function *Resolver,
2954 CGBuilderTy &Builder,
2955 llvm::Function *FuncToReturn,
2956 bool SupportsIFunc) {
2957 if (SupportsIFunc) {
2958 Builder.CreateRet(V: FuncToReturn);
2959 return;
2960 }
2961
2962 llvm::SmallVector<llvm::Value *, 10> Args(
2963 llvm::make_pointer_range(Range: Resolver->args()));
2964
2965 llvm::CallInst *Result = Builder.CreateCall(Callee: FuncToReturn, Args);
2966 Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
2967
2968 if (Resolver->getReturnType()->isVoidTy())
2969 Builder.CreateRetVoid();
2970 else
2971 Builder.CreateRet(V: Result);
2972}
2973
2974void CodeGenFunction::EmitMultiVersionResolver(
2975 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
2976 llvm::SaveAndRestore<llvm::Function *> savedCurFn(CurFn, Resolver);
2977 llvm::Triple::ArchType ArchType =
2978 getContext().getTargetInfo().getTriple().getArch();
2979
2980 switch (ArchType) {
2981 case llvm::Triple::x86:
2982 case llvm::Triple::x86_64:
2983 EmitX86MultiVersionResolver(Resolver, Options);
2984 return;
2985 case llvm::Triple::aarch64:
2986 EmitAArch64MultiVersionResolver(Resolver, Options);
2987 return;
2988 case llvm::Triple::riscv32:
2989 case llvm::Triple::riscv64:
2990 case llvm::Triple::riscv32be:
2991 case llvm::Triple::riscv64be:
2992 EmitRISCVMultiVersionResolver(Resolver, Options);
2993 return;
2994 case llvm::Triple::ppc:
2995 case llvm::Triple::ppc64:
2996 if (getContext().getTargetInfo().getTriple().isOSAIX()) {
2997 EmitPPCAIXMultiVersionResolver(Resolver, Options);
2998 return;
2999 }
3000 [[fallthrough]];
3001 default:
3002 assert(false &&
3003 "Only implemented for x86, AArch64, RISC-V, and PowerPC AIX");
3004 }
3005}
3006
3007/**
3008 * define internal ptr @foo.resolver() {
3009 * entry:
3010 * %is_version_1 = __builtin_cpu_supports(version_1)
3011 * br i1 %1, label %if.version_1, label %if.else_2
3012 *
3013 * if.version_1:
3014 * ret ptr @foo.version_1
3015 *
3016 * if.else_2:
3017 * %is_version_2 = __builtin_cpu_supports(version_2)
3018 * ...
3019 * if.else: ; preds = %entry
3020 * ret ptr @foo.default
3021 * }
3022 */
3023void CodeGenFunction::EmitPPCAIXMultiVersionResolver(
3024 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3025
3026 // entry:
3027 llvm::BasicBlock *CurBlock = createBasicBlock(name: "entry", parent: Resolver);
3028
3029 SmallVector<std::pair<llvm::Value *, llvm::BasicBlock *>, 3> PhiArgs;
3030 for (const FMVResolverOption &RO : Options) {
3031 Builder.SetInsertPoint(CurBlock);
3032 // The 'default' or 'generic' case.
3033 if (!RO.Architecture && RO.Features.empty()) {
3034 // if.else:
3035 // ret ptr @foo.default
3036 assert(&RO == Options.end() - 1 &&
3037 "Default or Generic case must be last");
3038 Builder.CreateRet(V: RO.Function);
3039 return;
3040 }
3041 // if.else_n:
3042 // %is_version_n = __builtin_cpu_supports(version_n)
3043 // br i1 %is_version_n, label %if.version_n, label %if.else_n+1
3044 //
3045 // if.version_n:
3046 // ret ptr @foo_version_n
3047 assert(RO.Features.size() == 1 &&
3048 "for now one feature requirement per version");
3049
3050 assert(RO.Features[0].starts_with("cpu="));
3051 StringRef CPU = RO.Features[0].split(Separator: "=").second.trim();
3052 StringRef Feature = llvm::StringSwitch<StringRef>(CPU)
3053 .Case(S: "pwr7", Value: "arch_2_06")
3054 .Case(S: "pwr8", Value: "arch_2_07")
3055 .Case(S: "pwr9", Value: "arch_3_00")
3056 .Case(S: "pwr10", Value: "arch_3_1")
3057 .Case(S: "pwr11", Value: "arch_3_1")
3058 .Default(Value: "error");
3059
3060 llvm::Value *Condition = EmitPPCBuiltinCpu(
3061 BuiltinID: Builtin::BI__builtin_cpu_supports, ReturnType: Builder.getInt1Ty(), CPUStr: Feature);
3062
3063 llvm::BasicBlock *ThenBlock = createBasicBlock(name: "if.version", parent: Resolver);
3064 CurBlock = createBasicBlock(name: "if.else", parent: Resolver);
3065 Builder.CreateCondBr(Cond: Condition, True: ThenBlock, False: CurBlock);
3066
3067 Builder.SetInsertPoint(ThenBlock);
3068 Builder.CreateRet(V: RO.Function);
3069 }
3070
3071 llvm_unreachable("Default case missing");
3072}
3073
3074void CodeGenFunction::EmitRISCVMultiVersionResolver(
3075 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3076
3077 if (getContext().getTargetInfo().getTriple().getOS() !=
3078 llvm::Triple::OSType::Linux) {
3079 CGM.getDiags().Report(DiagID: diag::err_os_unsupport_riscv_fmv);
3080 return;
3081 }
3082
3083 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3084 Builder.SetInsertPoint(CurBlock);
3085 EmitRISCVCpuInit();
3086
3087 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3088 bool HasDefault = false;
3089 unsigned DefaultIndex = 0;
3090
3091 // Check the each candidate function.
3092 for (unsigned Index = 0; Index < Options.size(); Index++) {
3093
3094 if (Options[Index].Features.empty()) {
3095 HasDefault = true;
3096 DefaultIndex = Index;
3097 continue;
3098 }
3099
3100 Builder.SetInsertPoint(CurBlock);
3101
3102 // FeaturesCondition: The bitmask of the required extension has been
3103 // enabled by the runtime object.
3104 // (__riscv_feature_bits.features[i] & REQUIRED_BITMASK) ==
3105 // REQUIRED_BITMASK
3106 //
3107 // When condition is met, return this version of the function.
3108 // Otherwise, try the next version.
3109 //
3110 // if (FeaturesConditionVersion1)
3111 // return Version1;
3112 // else if (FeaturesConditionVersion2)
3113 // return Version2;
3114 // else if (FeaturesConditionVersion3)
3115 // return Version3;
3116 // ...
3117 // else
3118 // return DefaultVersion;
3119
3120 // TODO: Add a condition to check the length before accessing elements.
3121 // Without checking the length first, we may access an incorrect memory
3122 // address when using different versions.
3123 llvm::SmallVector<StringRef, 8> CurrTargetAttrFeats;
3124 llvm::SmallVector<std::string, 8> TargetAttrFeats;
3125
3126 for (StringRef Feat : Options[Index].Features) {
3127 std::vector<std::string> FeatStr =
3128 getContext().getTargetInfo().parseTargetAttr(Str: Feat).Features;
3129
3130 assert(FeatStr.size() == 1 && "Feature string not delimited");
3131
3132 std::string &CurrFeat = FeatStr.front();
3133 if (CurrFeat[0] == '+')
3134 TargetAttrFeats.push_back(Elt: CurrFeat.substr(pos: 1));
3135 }
3136
3137 if (TargetAttrFeats.empty())
3138 continue;
3139
3140 for (std::string &Feat : TargetAttrFeats)
3141 CurrTargetAttrFeats.push_back(Elt: Feat);
3142
3143 Builder.SetInsertPoint(CurBlock);
3144 llvm::Value *FeatsCondition = EmitRISCVCpuSupports(FeaturesStrs: CurrTargetAttrFeats);
3145
3146 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3147 CGBuilderTy RetBuilder(CGM, RetBlock);
3148 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder,
3149 FuncToReturn: Options[Index].Function, SupportsIFunc);
3150 llvm::BasicBlock *ElseBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3151
3152 Builder.SetInsertPoint(CurBlock);
3153 Builder.CreateCondBr(Cond: FeatsCondition, True: RetBlock, False: ElseBlock);
3154
3155 CurBlock = ElseBlock;
3156 }
3157
3158 // Finally, emit the default one.
3159 if (HasDefault) {
3160 Builder.SetInsertPoint(CurBlock);
3161 CreateMultiVersionResolverReturn(
3162 CGM, Resolver, Builder, FuncToReturn: Options[DefaultIndex].Function, SupportsIFunc);
3163 return;
3164 }
3165
3166 // If no generic/default, emit an unreachable.
3167 Builder.SetInsertPoint(CurBlock);
3168 EmitTrapCallAndMakeUnreachable();
3169}
3170
3171void CodeGenFunction::EmitAArch64MultiVersionResolver(
3172 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3173 assert(!Options.empty() && "No multiversion resolver options found");
3174 assert(Options.back().Features.size() == 0 && "Default case must be last");
3175 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3176 assert(SupportsIFunc &&
3177 "Multiversion resolver requires target IFUNC support");
3178 bool AArch64CpuInitialized = false;
3179 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3180
3181 for (const FMVResolverOption &RO : Options) {
3182 Builder.SetInsertPoint(CurBlock);
3183 llvm::Value *Condition = FormAArch64ResolverCondition(RO);
3184
3185 // The 'default' or 'all features enabled' case.
3186 if (!Condition) {
3187 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, FuncToReturn: RO.Function,
3188 SupportsIFunc);
3189 return;
3190 }
3191
3192 if (!AArch64CpuInitialized) {
3193 Builder.SetInsertPoint(TheBB: CurBlock, IP: CurBlock->begin());
3194 EmitAArch64CpuInit();
3195 AArch64CpuInitialized = true;
3196 Builder.SetInsertPoint(CurBlock);
3197 }
3198
3199 // Skip unreachable versions.
3200 if (RO.Function == nullptr)
3201 continue;
3202
3203 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3204 CGBuilderTy RetBuilder(CGM, RetBlock);
3205 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder, FuncToReturn: RO.Function,
3206 SupportsIFunc);
3207 CurBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3208 Builder.CreateCondBr(Cond: Condition, True: RetBlock, False: CurBlock);
3209 }
3210
3211 // If no default, emit an unreachable.
3212 Builder.SetInsertPoint(CurBlock);
3213 EmitTrapCallAndMakeUnreachable();
3214}
3215
3216void CodeGenFunction::EmitX86MultiVersionResolver(
3217 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3218
3219 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3220
3221 // Main function's basic block.
3222 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3223 Builder.SetInsertPoint(CurBlock);
3224 EmitX86CpuInit();
3225
3226 for (const FMVResolverOption &RO : Options) {
3227 Builder.SetInsertPoint(CurBlock);
3228 llvm::Value *Condition = FormX86ResolverCondition(RO);
3229
3230 // The 'default' or 'generic' case.
3231 if (!Condition) {
3232 assert(&RO == Options.end() - 1 &&
3233 "Default or Generic case must be last");
3234 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, FuncToReturn: RO.Function,
3235 SupportsIFunc);
3236 return;
3237 }
3238
3239 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3240 CGBuilderTy RetBuilder(CGM, RetBlock);
3241 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder, FuncToReturn: RO.Function,
3242 SupportsIFunc);
3243 CurBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3244 Builder.CreateCondBr(Cond: Condition, True: RetBlock, False: CurBlock);
3245 }
3246
3247 // If no generic/default, emit an unreachable.
3248 Builder.SetInsertPoint(CurBlock);
3249 EmitTrapCallAndMakeUnreachable();
3250}
3251
3252// Loc - where the diagnostic will point, where in the source code this
3253// alignment has failed.
3254// SecondaryLoc - if present (will be present if sufficiently different from
3255// Loc), the diagnostic will additionally point a "Note:" to this location.
3256// It should be the location where the __attribute__((assume_aligned))
3257// was written e.g.
3258void CodeGenFunction::emitAlignmentAssumptionCheck(
3259 llvm::Value *Ptr, QualType Ty, SourceLocation Loc,
3260 SourceLocation SecondaryLoc, llvm::Value *Alignment,
3261 llvm::Value *OffsetValue, llvm::Value *TheCheck,
3262 llvm::Instruction *Assumption) {
3263 assert(isa_and_nonnull<llvm::CallInst>(Assumption) &&
3264 cast<llvm::CallInst>(Assumption)->getCalledOperand() ==
3265 llvm::Intrinsic::getOrInsertDeclaration(
3266 Builder.GetInsertBlock()->getParent()->getParent(),
3267 llvm::Intrinsic::assume) &&
3268 "Assumption should be a call to llvm.assume().");
3269 assert(&(Builder.GetInsertBlock()->back()) == Assumption &&
3270 "Assumption should be the last instruction of the basic block, "
3271 "since the basic block is still being generated.");
3272
3273 if (!SanOpts.has(K: SanitizerKind::Alignment))
3274 return;
3275
3276 // Don't check pointers to volatile data. The behavior here is implementation-
3277 // defined.
3278 if (Ty->getPointeeType().isVolatileQualified())
3279 return;
3280
3281 // We need to temorairly remove the assumption so we can insert the
3282 // sanitizer check before it, else the check will be dropped by optimizations.
3283 Assumption->removeFromParent();
3284
3285 {
3286 auto CheckOrdinal = SanitizerKind::SO_Alignment;
3287 auto CheckHandler = SanitizerHandler::AlignmentAssumption;
3288 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
3289
3290 if (!OffsetValue)
3291 OffsetValue = Builder.getInt1(V: false); // no offset.
3292
3293 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc),
3294 EmitCheckSourceLocation(Loc: SecondaryLoc),
3295 EmitCheckTypeDescriptor(T: Ty)};
3296 llvm::Value *DynamicData[] = {Ptr, Alignment, OffsetValue};
3297 EmitCheck(Checked: {std::make_pair(x&: TheCheck, y&: CheckOrdinal)}, Check: CheckHandler,
3298 StaticArgs: StaticData, DynamicArgs: DynamicData);
3299 }
3300
3301 // We are now in the (new, empty) "cont" basic block.
3302 // Reintroduce the assumption.
3303 Builder.Insert(I: Assumption);
3304 // FIXME: Assumption still has it's original basic block as it's Parent.
3305}
3306
3307llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
3308 if (CGDebugInfo *DI = getDebugInfo())
3309 return DI->SourceLocToDebugLoc(Loc: Location);
3310
3311 return llvm::DebugLoc();
3312}
3313
3314llvm::Value *
3315CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
3316 Stmt::Likelihood LH) {
3317 switch (LH) {
3318 case Stmt::LH_None:
3319 return Cond;
3320 case Stmt::LH_Likely:
3321 case Stmt::LH_Unlikely:
3322 // Don't generate llvm.expect on -O0 as the backend won't use it for
3323 // anything.
3324 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3325 return Cond;
3326 llvm::Type *CondTy = Cond->getType();
3327 assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");
3328 llvm::Function *FnExpect =
3329 CGM.getIntrinsic(IID: llvm::Intrinsic::expect, Tys: CondTy);
3330 llvm::Value *ExpectedValueOfCond =
3331 llvm::ConstantInt::getBool(Ty: CondTy, V: LH == Stmt::LH_Likely);
3332 return Builder.CreateCall(Callee: FnExpect, Args: {Cond, ExpectedValueOfCond},
3333 Name: Cond->getName() + ".expval");
3334 }
3335 llvm_unreachable("Unknown Likelihood");
3336}
3337
3338llvm::Value *CodeGenFunction::emitBoolVecConversion(llvm::Value *SrcVec,
3339 unsigned NumElementsDst,
3340 const llvm::Twine &Name) {
3341 auto *SrcTy = cast<llvm::FixedVectorType>(Val: SrcVec->getType());
3342 unsigned NumElementsSrc = SrcTy->getNumElements();
3343 if (NumElementsSrc == NumElementsDst)
3344 return SrcVec;
3345
3346 std::vector<int> ShuffleMask(NumElementsDst, -1);
3347 for (unsigned MaskIdx = 0;
3348 MaskIdx < std::min<>(a: NumElementsDst, b: NumElementsSrc); ++MaskIdx)
3349 ShuffleMask[MaskIdx] = MaskIdx;
3350
3351 return Builder.CreateShuffleVector(V: SrcVec, Mask: ShuffleMask, Name);
3352}
3353
3354void CodeGenFunction::EmitPointerAuthOperandBundle(
3355 const CGPointerAuthInfo &PointerAuth,
3356 SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
3357 if (!PointerAuth.isSigned())
3358 return;
3359
3360 auto *Key = Builder.getInt32(C: PointerAuth.getKey());
3361
3362 llvm::Value *Discriminator = PointerAuth.getDiscriminator();
3363 if (!Discriminator)
3364 Discriminator = Builder.getSize(N: 0);
3365
3366 llvm::Value *Args[] = {Key, Discriminator};
3367 Bundles.emplace_back(Args: "ptrauth", Args);
3368}
3369
3370static llvm::Value *EmitPointerAuthCommon(CodeGenFunction &CGF,
3371 const CGPointerAuthInfo &PointerAuth,
3372 llvm::Value *Pointer,
3373 unsigned IntrinsicID) {
3374 if (!PointerAuth)
3375 return Pointer;
3376
3377 auto Key = CGF.Builder.getInt32(C: PointerAuth.getKey());
3378
3379 llvm::Value *Discriminator = PointerAuth.getDiscriminator();
3380 if (!Discriminator) {
3381 Discriminator = CGF.Builder.getSize(N: 0);
3382 }
3383
3384 // Convert the pointer to intptr_t before signing it.
3385 auto OrigType = Pointer->getType();
3386 Pointer = CGF.Builder.CreatePtrToInt(V: Pointer, DestTy: CGF.IntPtrTy);
3387
3388 // call i64 @llvm.ptrauth.sign.i64(i64 %pointer, i32 %key, i64 %discriminator)
3389 auto Intrinsic = CGF.CGM.getIntrinsic(IID: IntrinsicID);
3390 Pointer = CGF.EmitRuntimeCall(callee: Intrinsic, args: {Pointer, Key, Discriminator});
3391
3392 // Convert back to the original type.
3393 Pointer = CGF.Builder.CreateIntToPtr(V: Pointer, DestTy: OrigType);
3394 return Pointer;
3395}
3396
3397llvm::Value *
3398CodeGenFunction::EmitPointerAuthSign(const CGPointerAuthInfo &PointerAuth,
3399 llvm::Value *Pointer) {
3400 if (!PointerAuth.shouldSign())
3401 return Pointer;
3402 return EmitPointerAuthCommon(CGF&: *this, PointerAuth, Pointer,
3403 IntrinsicID: llvm::Intrinsic::ptrauth_sign);
3404}
3405
3406static llvm::Value *EmitStrip(CodeGenFunction &CGF,
3407 const CGPointerAuthInfo &PointerAuth,
3408 llvm::Value *Pointer) {
3409 auto StripIntrinsic = CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::ptrauth_strip);
3410
3411 auto Key = CGF.Builder.getInt32(C: PointerAuth.getKey());
3412 // Convert the pointer to intptr_t before signing it.
3413 auto OrigType = Pointer->getType();
3414 Pointer = CGF.EmitRuntimeCall(
3415 callee: StripIntrinsic, args: {CGF.Builder.CreatePtrToInt(V: Pointer, DestTy: CGF.IntPtrTy), Key});
3416 return CGF.Builder.CreateIntToPtr(V: Pointer, DestTy: OrigType);
3417}
3418
3419llvm::Value *
3420CodeGenFunction::EmitPointerAuthAuth(const CGPointerAuthInfo &PointerAuth,
3421 llvm::Value *Pointer) {
3422 if (PointerAuth.shouldStrip()) {
3423 return EmitStrip(CGF&: *this, PointerAuth, Pointer);
3424 }
3425 if (!PointerAuth.shouldAuth()) {
3426 return Pointer;
3427 }
3428
3429 return EmitPointerAuthCommon(CGF&: *this, PointerAuth, Pointer,
3430 IntrinsicID: llvm::Intrinsic::ptrauth_auth);
3431}
3432
3433void CodeGenFunction::addInstToCurrentSourceAtom(
3434 llvm::Instruction *KeyInstruction, llvm::Value *Backup) {
3435 if (CGDebugInfo *DI = getDebugInfo())
3436 DI->addInstToCurrentSourceAtom(KeyInstruction, Backup);
3437}
3438
3439void CodeGenFunction::addInstToSpecificSourceAtom(
3440 llvm::Instruction *KeyInstruction, llvm::Value *Backup, uint64_t Atom) {
3441 if (CGDebugInfo *DI = getDebugInfo())
3442 DI->addInstToSpecificSourceAtom(KeyInstruction, Backup, Atom);
3443}
3444
3445void CodeGenFunction::addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,
3446 llvm::Value *Backup) {
3447 if (CGDebugInfo *DI = getDebugInfo()) {
3448 ApplyAtomGroup Grp(getDebugInfo());
3449 DI->addInstToCurrentSourceAtom(KeyInstruction, Backup);
3450 }
3451}
3452
3453void CodeGenFunction::emitPFPPostCopyUpdates(Address DestPtr, Address SrcPtr,
3454 QualType Ty) {
3455 for (auto &Field : getContext().findPFPFields(Ty)) {
3456 if (getContext().arePFPFieldsTriviallyCopyable(RD: Field.Field->getParent()))
3457 continue;
3458 auto DestFieldPtr = EmitAddressOfPFPField(RecordPtr: DestPtr, Field);
3459 auto SrcFieldPtr = EmitAddressOfPFPField(RecordPtr: SrcPtr, Field);
3460 Builder.CreateStore(Val: Builder.CreateLoad(Addr: SrcFieldPtr), Addr: DestFieldPtr);
3461 }
3462}
3463