1//===--- CodeGenFunction.cpp - Emit LLVM Code from ASTs for a Function ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This coordinates the per-function state used while generating code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CodeGenFunction.h"
14#include "CGBlocks.h"
15#include "CGCUDARuntime.h"
16#include "CGCXXABI.h"
17#include "CGCleanup.h"
18#include "CGDebugInfo.h"
19#include "CGHLSLRuntime.h"
20#include "CGOpenMPRuntime.h"
21#include "CodeGenModule.h"
22#include "CodeGenPGO.h"
23#include "TargetInfo.h"
24#include "clang/AST/ASTContext.h"
25#include "clang/AST/ASTLambda.h"
26#include "clang/AST/Attr.h"
27#include "clang/AST/Decl.h"
28#include "clang/AST/DeclCXX.h"
29#include "clang/AST/Expr.h"
30#include "clang/AST/IgnoreExpr.h"
31#include "clang/AST/StmtCXX.h"
32#include "clang/AST/StmtObjC.h"
33#include "clang/Basic/Builtins.h"
34#include "clang/Basic/CodeGenOptions.h"
35#include "clang/Basic/DiagnosticFrontend.h"
36#include "clang/Basic/TargetBuiltins.h"
37#include "clang/Basic/TargetInfo.h"
38#include "clang/CodeGen/CGFunctionInfo.h"
39#include "clang/CodeGenUtils/FunctionUtils.h"
40#include "llvm/ADT/ArrayRef.h"
41#include "llvm/ADT/ScopeExit.h"
42#include "llvm/ADT/StringExtras.h"
43#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
44#include "llvm/IR/DataLayout.h"
45#include "llvm/IR/Dominators.h"
46#include "llvm/IR/FPEnv.h"
47#include "llvm/IR/Instruction.h"
48#include "llvm/IR/IntrinsicInst.h"
49#include "llvm/IR/Intrinsics.h"
50#include "llvm/IR/IntrinsicsPowerPC.h"
51#include "llvm/IR/MDBuilder.h"
52#include "llvm/Support/CRC.h"
53#include "llvm/Support/SaveAndRestore.h"
54#include "llvm/Support/SipHash.h"
55#include "llvm/Support/xxhash.h"
56#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
57#include "llvm/Transforms/Utils/PromoteMemToReg.h"
58#include <optional>
59
60using namespace clang;
61using namespace CodeGen;
62
63CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
64 : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
65 Builder(cgm, cgm.getModule().getContext(), CGBuilderInserterTy(this)),
66 SanOpts(CGM.getLangOpts().Sanitize), CurFPFeatures(CGM.getLangOpts()),
67 DebugInfo(CGM.getModuleDebugInfo()),
68 PGO(std::make_unique<CodeGenPGO>(args&: cgm)),
69 ShouldEmitLifetimeMarkers(CodeGenUtils::shouldEmitLifetimeMarkers(
70 CGOpts: CGM.getCodeGenOpts(), LangOpts: CGM.getLangOpts())) {
71 if (!suppressNewContext)
72 CGM.getCXXABI().getMangleContext().startNewFunction();
73 EHStack.setCGF(this);
74
75 SetFastMathFlags(CurFPFeatures);
76}
77
78const FunctionDecl *CodeGenFunction::getCurrentFunctionDecl() const {
79 const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
80 if (!FD)
81 FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl);
82 return FD;
83}
84
85CodeGenFunction::~CodeGenFunction() {
86 assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup");
87 assert(DeferredDeactivationCleanupStack.empty() &&
88 "missed to deactivate a cleanup");
89
90 if (getLangOpts().OpenMP && CurFn)
91 CGM.getOpenMPRuntime().functionFinished(CGF&: *this);
92
93 // If we have an OpenMPIRBuilder we want to finalize functions (incl.
94 // outlining etc) at some point. Doing it once the function codegen is done
95 // seems to be a reasonable spot. We do it here, as opposed to the deletion
96 // time of the CodeGenModule, because we have to ensure the IR has not yet
97 // been "emitted" to the outside, thus, modifications are still sensible.
98 if (CGM.getLangOpts().OpenMPIRBuilder && CurFn)
99 CGM.getOpenMPRuntime().getOMPBuilder().finalize(Fn: CurFn);
100}
101
102// Map the LangOption for exception behavior into
103// the corresponding enum in the IR.
104llvm::fp::ExceptionBehavior
105clang::ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind) {
106
107 switch (Kind) {
108 case LangOptions::FPE_Ignore: return llvm::fp::ebIgnore;
109 case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap;
110 case LangOptions::FPE_Strict: return llvm::fp::ebStrict;
111 default:
112 llvm_unreachable("Unsupported FP Exception Behavior");
113 }
114}
115
116void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) {
117 llvm::FastMathFlags FMF;
118 FMF.setAllowReassoc(FPFeatures.getAllowFPReassociate());
119 FMF.setNoNaNs(FPFeatures.getNoHonorNaNs());
120 FMF.setNoInfs(FPFeatures.getNoHonorInfs());
121 FMF.setNoSignedZeros(FPFeatures.getNoSignedZero());
122 FMF.setAllowReciprocal(FPFeatures.getAllowReciprocal());
123 FMF.setApproxFunc(FPFeatures.getAllowApproxFunc());
124 FMF.setAllowContract(FPFeatures.allowFPContractAcrossStatement());
125 Builder.setFastMathFlags(FMF);
126}
127
128CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
129 const Expr *E)
130 : CGF(CGF) {
131 ConstructorHelper(FPFeatures: E->getFPFeaturesInEffect(LO: CGF.getLangOpts()));
132}
133
134CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
135 FPOptions FPFeatures)
136 : CGF(CGF) {
137 ConstructorHelper(FPFeatures);
138}
139
140void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) {
141 OldFPFeatures = CGF.CurFPFeatures;
142 CGF.CurFPFeatures = FPFeatures;
143
144 OldExcept = CGF.Builder.getDefaultConstrainedExcept();
145 OldRounding = CGF.Builder.getDefaultConstrainedRounding();
146
147 if (OldFPFeatures == FPFeatures)
148 return;
149
150 FMFGuard.emplace(args&: CGF.Builder);
151
152 llvm::RoundingMode NewRoundingBehavior = FPFeatures.getRoundingMode();
153 CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);
154 auto NewExceptionBehavior =
155 ToConstrainedExceptMD(Kind: FPFeatures.getExceptionMode());
156 CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);
157
158 CGF.SetFastMathFlags(FPFeatures);
159
160 assert((CGF.CurFuncDecl == nullptr || CGF.Builder.getIsFPConstrained() ||
161 isa<CXXConstructorDecl>(CGF.CurFuncDecl) ||
162 isa<CXXDestructorDecl>(CGF.CurFuncDecl) ||
163 (NewExceptionBehavior == llvm::fp::ebIgnore &&
164 NewRoundingBehavior == llvm::RoundingMode::NearestTiesToEven)) &&
165 "FPConstrained should be enabled on entire function");
166
167 auto mergeFnAttrValue = [&](StringRef Name, bool Value) {
168 auto OldValue =
169 CGF.CurFn->getFnAttribute(Kind: Name).getValueAsBool();
170 auto NewValue = OldValue & Value;
171 if (OldValue != NewValue)
172 CGF.CurFn->addFnAttr(Kind: Name, Val: llvm::toStringRef(B: NewValue));
173 };
174 mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.getNoSignedZero());
175}
176
177CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
178 CGF.CurFPFeatures = OldFPFeatures;
179 CGF.Builder.setDefaultConstrainedExcept(OldExcept);
180 CGF.Builder.setDefaultConstrainedRounding(OldRounding);
181}
182
183static LValue
184makeNaturalAlignAddrLValue(llvm::Value *V, QualType T, bool ForPointeeType,
185 bool MightBeSigned, CodeGenFunction &CGF,
186 KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
187 LValueBaseInfo BaseInfo;
188 TBAAAccessInfo TBAAInfo;
189 CharUnits Alignment =
190 CGF.CGM.getNaturalTypeAlignment(T, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo, forPointeeType: ForPointeeType);
191 Address Addr =
192 MightBeSigned
193 ? CGF.makeNaturalAddressForPointer(Ptr: V, T, Alignment, ForPointeeType: false, BaseInfo: nullptr,
194 TBAAInfo: nullptr, IsKnownNonNull)
195 : Address(V, CGF.ConvertTypeForMem(T), Alignment, IsKnownNonNull);
196 return CGF.MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
197}
198
199LValue
200CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
201 KnownNonNull_t IsKnownNonNull) {
202 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,
203 /*MightBeSigned*/ true, CGF&: *this,
204 IsKnownNonNull);
205}
206
207LValue
208CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
209 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,
210 /*MightBeSigned*/ true, CGF&: *this);
211}
212
213LValue CodeGenFunction::MakeNaturalAlignRawAddrLValue(llvm::Value *V,
214 QualType T) {
215 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,
216 /*MightBeSigned*/ false, CGF&: *this);
217}
218
219LValue CodeGenFunction::MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V,
220 QualType T) {
221 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,
222 /*MightBeSigned*/ false, CGF&: *this);
223}
224
225llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
226 return CGM.getTypes().ConvertTypeForMem(T);
227}
228
229llvm::Type *CodeGenFunction::ConvertType(QualType T) {
230 return CGM.getTypes().ConvertType(T);
231}
232
233llvm::Type *CodeGenFunction::convertTypeForLoadStore(QualType ASTTy,
234 llvm::Type *LLVMTy) {
235 return CGM.getTypes().convertTypeForLoadStore(T: ASTTy, LLVMTy);
236}
237
238TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
239 type = type.getCanonicalType();
240 while (true) {
241 switch (type->getTypeClass()) {
242#define TYPE(name, parent)
243#define ABSTRACT_TYPE(name, parent)
244#define NON_CANONICAL_TYPE(name, parent) case Type::name:
245#define DEPENDENT_TYPE(name, parent) case Type::name:
246#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
247#include "clang/AST/TypeNodes.inc"
248 llvm_unreachable("non-canonical or dependent type in IR-generation");
249
250 case Type::Auto:
251 case Type::DeducedTemplateSpecialization:
252 llvm_unreachable("undeduced type in IR-generation");
253
254 // Various scalar types.
255 case Type::Builtin:
256 case Type::Pointer:
257 case Type::BlockPointer:
258 case Type::LValueReference:
259 case Type::RValueReference:
260 case Type::MemberPointer:
261 case Type::Vector:
262 case Type::ExtVector:
263 case Type::ConstantMatrix:
264 case Type::FunctionProto:
265 case Type::FunctionNoProto:
266 case Type::Enum:
267 case Type::ObjCObjectPointer:
268 case Type::Pipe:
269 case Type::BitInt:
270 case Type::HLSLAttributedResource:
271 case Type::HLSLInlineSpirv:
272 case Type::OverflowBehavior:
273 return TEK_Scalar;
274
275 // Complexes.
276 case Type::Complex:
277 return TEK_Complex;
278
279 // Arrays, records, and Objective-C objects.
280 case Type::ConstantArray:
281 case Type::IncompleteArray:
282 case Type::VariableArray:
283 case Type::Record:
284 case Type::ObjCObject:
285 case Type::ObjCInterface:
286 case Type::ArrayParameter:
287 return TEK_Aggregate;
288
289 // We operate on atomic values according to their underlying type.
290 case Type::Atomic:
291 type = cast<AtomicType>(Val&: type)->getValueType();
292 continue;
293 }
294 llvm_unreachable("unknown type kind!");
295 }
296}
297
298llvm::DebugLoc CodeGenFunction::EmitReturnBlock() {
299 // For cleanliness, we try to avoid emitting the return block for
300 // simple cases.
301 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
302
303 if (CurBB) {
304 assert(!CurBB->hasTerminator() && "Unexpected terminated block.");
305
306 // We have a valid insert point, reuse it if it is empty or there are no
307 // explicit jumps to the return block.
308 if (CurBB->empty() || ReturnBlock.getBlock()->use_empty()) {
309 ReturnBlock.getBlock()->replaceAllUsesWith(V: CurBB);
310 delete ReturnBlock.getBlock();
311 ReturnBlock = JumpDest();
312 } else
313 EmitBlock(BB: ReturnBlock.getBlock());
314 return llvm::DebugLoc();
315 }
316
317 // Otherwise, if the return block is the target of a single direct
318 // branch then we can just put the code in that block instead. This
319 // cleans up functions which started with a unified return block.
320 if (ReturnBlock.getBlock()->hasOneUse()) {
321 auto *BI =
322 dyn_cast<llvm::UncondBrInst>(Val: *ReturnBlock.getBlock()->user_begin());
323 if (BI && BI->getSuccessor(i: 0) == ReturnBlock.getBlock()) {
324 // Record/return the DebugLoc of the simple 'return' expression to be used
325 // later by the actual 'ret' instruction.
326 llvm::DebugLoc Loc = BI->getDebugLoc();
327 Builder.SetInsertPoint(BI->getParent());
328 BI->eraseFromParent();
329 delete ReturnBlock.getBlock();
330 ReturnBlock = JumpDest();
331 return Loc;
332 }
333 }
334
335 // FIXME: We are at an unreachable point, there is no reason to emit the block
336 // unless it has uses. However, we still need a place to put the debug
337 // region.end for now.
338
339 EmitBlock(BB: ReturnBlock.getBlock());
340 return llvm::DebugLoc();
341}
342
343static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
344 if (!BB) return;
345 if (!BB->use_empty()) {
346 CGF.CurFn->insert(Position: CGF.CurFn->end(), BB);
347 return;
348 }
349 delete BB;
350}
351
352void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
353 assert(BreakContinueStack.empty() &&
354 "mismatched push/pop in break/continue stack!");
355 assert(LifetimeExtendedCleanupStack.empty() &&
356 "mismatched push/pop of cleanups in EHStack!");
357 assert(DeferredDeactivationCleanupStack.empty() &&
358 "mismatched activate/deactivate of cleanups!");
359
360 if (CGM.shouldEmitConvergenceTokens()) {
361 ConvergenceTokenStack.pop_back();
362 assert(ConvergenceTokenStack.empty() &&
363 "mismatched push/pop in convergence stack!");
364 }
365
366 bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
367 && NumSimpleReturnExprs == NumReturnExprs
368 && ReturnBlock.getBlock()->use_empty();
369 // Usually the return expression is evaluated before the cleanup
370 // code. If the function contains only a simple return statement,
371 // such as a constant, the location before the cleanup code becomes
372 // the last useful breakpoint in the function, because the simple
373 // return expression will be evaluated after the cleanup code. To be
374 // safe, set the debug location for cleanup code to the location of
375 // the return statement. Otherwise the cleanup code should be at the
376 // end of the function's lexical scope.
377 //
378 // If there are multiple branches to the return block, the branch
379 // instructions will get the location of the return statements and
380 // all will be fine.
381 if (CGDebugInfo *DI = getDebugInfo()) {
382 if (OnlySimpleReturnStmts)
383 DI->EmitLocation(Builder, Loc: LastStopPoint);
384 else
385 DI->EmitLocation(Builder, Loc: EndLoc);
386 }
387
388 // Pop any cleanups that might have been associated with the
389 // parameters. Do this in whatever block we're currently in; it's
390 // important to do this before we enter the return block or return
391 // edges will be *really* confused.
392 bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth;
393 bool HasOnlyNoopCleanups =
394 HasCleanups && EHStack.containsOnlyNoopCleanups(Old: PrologueCleanupDepth);
395 bool EmitRetDbgLoc = !HasCleanups || HasOnlyNoopCleanups;
396
397 std::optional<ApplyDebugLocation> OAL;
398 if (HasCleanups) {
399 // Make sure the line table doesn't jump back into the body for
400 // the ret after it's been at EndLoc.
401 if (CGDebugInfo *DI = getDebugInfo()) {
402 if (OnlySimpleReturnStmts)
403 DI->EmitLocation(Builder, Loc: EndLoc);
404 else
405 // We may not have a valid end location. Try to apply it anyway, and
406 // fall back to an artificial location if needed.
407 OAL = ApplyDebugLocation::CreateDefaultArtificial(CGF&: *this, TemporaryLocation: EndLoc);
408 }
409
410 PopCleanupBlocks(OldCleanupStackSize: PrologueCleanupDepth);
411 }
412
413 // Emit function epilog (to return).
414 llvm::DebugLoc Loc = EmitReturnBlock();
415
416 if (ShouldInstrumentFunction()) {
417 if (CGM.getCodeGenOpts().InstrumentFunctions)
418 CurFn->addFnAttr(Kind: "instrument-function-exit", Val: "__cyg_profile_func_exit");
419 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
420 CurFn->addFnAttr(Kind: "instrument-function-exit-inlined",
421 Val: "__cyg_profile_func_exit");
422 }
423
424 // Emit debug descriptor for function end.
425 if (CGDebugInfo *DI = getDebugInfo())
426 DI->EmitFunctionEnd(Builder, Fn: CurFn);
427
428 // Reset the debug location to that of the simple 'return' expression, if any
429 // rather than that of the end of the function's scope '}'.
430 uint64_t RetKeyInstructionsAtomGroup = Loc ? Loc->getAtomGroup() : 0;
431 ApplyDebugLocation AL(*this, Loc);
432 EmitFunctionEpilog(FI: *CurFnInfo, EmitRetDbgLoc, EndLoc,
433 RetKeyInstructionsSourceAtom: RetKeyInstructionsAtomGroup);
434 EmitEndEHSpec(D: CurCodeDecl);
435
436 assert(EHStack.empty() &&
437 "did not remove all scopes from cleanup stack!");
438
439 // If someone did an indirect goto, emit the indirect goto block at the end of
440 // the function.
441 if (IndirectBranch) {
442 EmitBlock(BB: IndirectBranch->getParent());
443 Builder.ClearInsertionPoint();
444 }
445
446 // If some of our locals escaped, insert a call to llvm.localescape in the
447 // entry block.
448 if (!EscapedLocals.empty()) {
449 // Invert the map from local to index into a simple vector. There should be
450 // no holes.
451 SmallVector<llvm::Value *, 4> EscapeArgs;
452 EscapeArgs.resize(N: EscapedLocals.size());
453 for (auto &Pair : EscapedLocals)
454 EscapeArgs[Pair.second] = Pair.first;
455 llvm::Function *FrameEscapeFn = llvm::Intrinsic::getOrInsertDeclaration(
456 M: &CGM.getModule(), id: llvm::Intrinsic::localescape);
457 CGBuilderTy(CGM, AllocaInsertPt).CreateCall(Callee: FrameEscapeFn, Args: EscapeArgs);
458 }
459
460 // Remove the AllocaInsertPt instruction, which is just a convenience for us.
461 llvm::Instruction *Ptr = AllocaInsertPt;
462 AllocaInsertPt = nullptr;
463 Ptr->eraseFromParent();
464
465 // PostAllocaInsertPt, if created, was lazily created when it was required,
466 // remove it now since it was just created for our own convenience.
467 if (PostAllocaInsertPt) {
468 llvm::Instruction *PostPtr = PostAllocaInsertPt;
469 PostAllocaInsertPt = nullptr;
470 PostPtr->eraseFromParent();
471 }
472
473 // If someone took the address of a label but never did an indirect goto, we
474 // made a zero entry PHI node, which is illegal, zap it now.
475 if (IndirectBranch) {
476 llvm::PHINode *PN = cast<llvm::PHINode>(Val: IndirectBranch->getAddress());
477 if (PN->getNumIncomingValues() == 0) {
478 PN->replaceAllUsesWith(V: llvm::PoisonValue::get(T: PN->getType()));
479 PN->eraseFromParent();
480 }
481 }
482
483 EmitIfUsed(CGF&: *this, BB: EHResumeBlock);
484 EmitIfUsed(CGF&: *this, BB: TerminateLandingPad);
485 EmitIfUsed(CGF&: *this, BB: TerminateHandler);
486 EmitIfUsed(CGF&: *this, BB: UnreachableBlock);
487
488 for (const auto &FuncletAndParent : TerminateFunclets)
489 EmitIfUsed(CGF&: *this, BB: FuncletAndParent.second);
490
491 if (CGM.getCodeGenOpts().EmitDeclMetadata)
492 EmitDeclMetadata();
493
494 for (const auto &R : DeferredReplacements) {
495 if (llvm::Value *Old = R.first) {
496 Old->replaceAllUsesWith(V: R.second);
497 cast<llvm::Instruction>(Val: Old)->eraseFromParent();
498 }
499 }
500 DeferredReplacements.clear();
501
502 // Eliminate CleanupDestSlot alloca by replacing it with SSA values and
503 // PHIs if the current function is a coroutine. We don't do it for all
504 // functions as it may result in slight increase in numbers of instructions
505 // if compiled with no optimizations. We do it for coroutine as the lifetime
506 // of CleanupDestSlot alloca make correct coroutine frame building very
507 // difficult.
508 if (NormalCleanupDest.isValid() && isCoroutine()) {
509 llvm::DominatorTree DT(*CurFn);
510 llvm::PromoteMemToReg(
511 Allocas: cast<llvm::AllocaInst>(Val: NormalCleanupDest.getPointer()), DT);
512 NormalCleanupDest = Address::invalid();
513 }
514
515 // Scan function arguments for vector width.
516 for (llvm::Argument &A : CurFn->args())
517 if (auto *VT = dyn_cast<llvm::VectorType>(Val: A.getType()))
518 LargestVectorWidth =
519 std::max(a: (uint64_t)LargestVectorWidth,
520 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
521
522 // Update vector width based on return type.
523 if (auto *VT = dyn_cast<llvm::VectorType>(Val: CurFn->getReturnType()))
524 LargestVectorWidth =
525 std::max(a: (uint64_t)LargestVectorWidth,
526 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
527
528 if (CurFnInfo->getMaxVectorWidth() > LargestVectorWidth)
529 LargestVectorWidth = CurFnInfo->getMaxVectorWidth();
530
531 // Add the min-legal-vector-width attribute. This contains the max width from:
532 // 1. min-vector-width attribute used in the source program.
533 // 2. Any builtins used that have a vector width specified.
534 // 3. Values passed in and out of inline assembly.
535 // 4. Width of vector arguments and return types for this function.
536 // 5. Width of vector arguments and return types for functions called by this
537 // function.
538 if (getContext().getTargetInfo().getTriple().isX86())
539 CurFn->addFnAttr(Kind: "min-legal-vector-width",
540 Val: llvm::utostr(X: LargestVectorWidth));
541
542 // If we generated an unreachable return block, delete it now.
543 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()) {
544 Builder.ClearInsertionPoint();
545 ReturnBlock.getBlock()->eraseFromParent();
546 }
547 if (ReturnValue.isValid()) {
548 auto *RetAlloca =
549 dyn_cast<llvm::AllocaInst>(Val: ReturnValue.emitRawPointer(CGF&: *this));
550 if (RetAlloca && RetAlloca->use_empty()) {
551 RetAlloca->eraseFromParent();
552 ReturnValue = Address::invalid();
553 }
554 }
555}
556
557/// ShouldInstrumentFunction - Return true if the current function should be
558/// instrumented with __cyg_profile_func_* calls
559bool CodeGenFunction::ShouldInstrumentFunction() {
560 if (!CGM.getCodeGenOpts().InstrumentFunctions &&
561 !CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining &&
562 !CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
563 return false;
564 if (!CurFuncDecl || CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>())
565 return false;
566 return true;
567}
568
569bool CodeGenFunction::ShouldSkipSanitizerInstrumentation() {
570 if (!CurFuncDecl)
571 return false;
572 return CurFuncDecl->hasAttr<DisableSanitizerInstrumentationAttr>();
573}
574
575/// ShouldXRayInstrument - Return true if the current function should be
576/// instrumented with XRay nop sleds.
577bool CodeGenFunction::ShouldXRayInstrumentFunction() const {
578 return CGM.getCodeGenOpts().XRayInstrumentFunctions;
579}
580
581/// AlwaysEmitXRayCustomEvents - Return true if we should emit IR for calls to
582/// the __xray_customevent(...) builtin calls, when doing XRay instrumentation.
583bool CodeGenFunction::AlwaysEmitXRayCustomEvents() const {
584 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
585 (CGM.getCodeGenOpts().XRayAlwaysEmitCustomEvents ||
586 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
587 XRayInstrKind::Custom);
588}
589
590bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const {
591 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
592 (CGM.getCodeGenOpts().XRayAlwaysEmitTypedEvents ||
593 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
594 XRayInstrKind::Typed);
595}
596
597llvm::ConstantInt *
598CodeGenFunction::getUBSanFunctionTypeHash(QualType Ty) const {
599 // Remove any (C++17) exception specifications, to allow calling e.g. a
600 // noexcept function through a non-noexcept pointer.
601 if (!Ty->isFunctionNoProtoType())
602 Ty = getContext().getFunctionTypeWithExceptionSpec(Orig: Ty, ESI: EST_None);
603 std::string Mangled;
604 llvm::raw_string_ostream Out(Mangled);
605 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: Ty, Out, NormalizeIntegers: false);
606 return llvm::ConstantInt::get(
607 Ty: CGM.Int32Ty, V: static_cast<uint32_t>(llvm::xxh3_64bits(data: Mangled)));
608}
609
610void CodeGenFunction::EmitKernelMetadata(const FunctionDecl *FD,
611 llvm::Function *Fn) {
612 if (!FD->hasAttr<DeviceKernelAttr>() && !FD->hasAttr<CUDAGlobalAttr>())
613 return;
614
615 llvm::LLVMContext &Context = getLLVMContext();
616
617 CGM.GenKernelArgMetadata(FN: Fn, FD, CGF: this);
618
619 if (!(getLangOpts().OpenCL ||
620 (getLangOpts().CUDA &&
621 getContext().getTargetInfo().getTriple().isSPIRV())))
622 return;
623
624 if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {
625 QualType HintQTy = A->getTypeHint();
626 const ExtVectorType *HintEltQTy = HintQTy->getAs<ExtVectorType>();
627 bool IsSignedInteger =
628 HintQTy->isSignedIntegerType() ||
629 (HintEltQTy && HintEltQTy->getElementType()->isSignedIntegerType());
630 llvm::Metadata *AttrMDArgs[] = {
631 llvm::ConstantAsMetadata::get(C: llvm::PoisonValue::get(
632 T: CGM.getTypes().ConvertType(T: A->getTypeHint()))),
633 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
634 Ty: llvm::IntegerType::get(C&: Context, NumBits: 32),
635 V: llvm::APInt(32, (uint64_t)(IsSignedInteger ? 1 : 0))))};
636 Fn->setMetadata(Kind: "vec_type_hint", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
637 }
638
639 if (const WorkGroupSizeHintAttr *A = FD->getAttr<WorkGroupSizeHintAttr>()) {
640 auto Eval = [&](Expr *E) {
641 return E->EvaluateKnownConstInt(Ctx: FD->getASTContext()).getExtValue();
642 };
643 llvm::Metadata *AttrMDArgs[] = {
644 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getXDim()))),
645 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getYDim()))),
646 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getZDim())))};
647 Fn->setMetadata(Kind: "work_group_size_hint", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
648 }
649
650 if (const ReqdWorkGroupSizeAttr *A = FD->getAttr<ReqdWorkGroupSizeAttr>()) {
651 auto Eval = [&](Expr *E) {
652 return E->EvaluateKnownConstInt(Ctx: FD->getASTContext()).getExtValue();
653 };
654 llvm::Metadata *AttrMDArgs[] = {
655 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getXDim()))),
656 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getYDim()))),
657 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getZDim())))};
658 Fn->setMetadata(Kind: "reqd_work_group_size", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
659 }
660
661 if (const OpenCLIntelReqdSubGroupSizeAttr *A =
662 FD->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
663 llvm::Metadata *AttrMDArgs[] = {
664 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: A->getSubGroupSize()))};
665 Fn->setMetadata(Kind: "intel_reqd_sub_group_size",
666 Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
667 }
668}
669
670/// Determine whether the function F ends with a return stmt.
671static bool endsWithReturn(const Decl* F) {
672 const Stmt *Body = nullptr;
673 if (auto *FD = dyn_cast_or_null<FunctionDecl>(Val: F))
674 Body = FD->getBody();
675 else if (auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(Val: F))
676 Body = OMD->getBody();
677
678 if (auto *CS = dyn_cast_or_null<CompoundStmt>(Val: Body)) {
679 auto LastStmt = CS->body_rbegin();
680 if (LastStmt != CS->body_rend())
681 return isa<ReturnStmt>(Val: *LastStmt);
682 }
683 return false;
684}
685
686void CodeGenFunction::markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {
687 if (SanOpts.has(K: SanitizerKind::Thread)) {
688 Fn->addFnAttr(Kind: "sanitize_thread_no_checking_at_run_time");
689 Fn->removeFnAttr(Kind: llvm::Attribute::SanitizeThread);
690 }
691 if (SanOpts.has(K: SanitizerKind::Concurrency)) {
692 Fn->addFnAttr(Kind: "sanitize_concurrency_no_checking_at_run_time");
693 Fn->removeFnAttr(Kind: llvm::Attribute::SanitizeConcurrency);
694 }
695}
696
697/// Check if the return value of this function requires sanitization.
698bool CodeGenFunction::requiresReturnValueCheck() const {
699 return requiresReturnValueNullabilityCheck() ||
700 (SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute) && CurCodeDecl &&
701 CurCodeDecl->getAttr<ReturnsNonNullAttr>()) ||
702 (SanOpts.has(K: SanitizerKind::Alignment) && CurCodeDecl &&
703 ((isa<FunctionDecl>(Val: CurCodeDecl) &&
704 CurCodeDecl->getAttr<AllocAlignAttr>()) ||
705 CurCodeDecl->getAttr<AssumeAlignedAttr>()));
706}
707
708static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) {
709 auto *MD = dyn_cast_or_null<CXXMethodDecl>(Val: D);
710 if (!MD || !MD->getDeclName().getAsIdentifierInfo() ||
711 !MD->getDeclName().getAsIdentifierInfo()->isStr(Str: "allocate") ||
712 (MD->getNumParams() != 1 && MD->getNumParams() != 2))
713 return false;
714
715 if (!Ctx.hasSameType(T1: MD->parameters()[0]->getType(), T2: Ctx.getSizeType()))
716 return false;
717
718 if (MD->getNumParams() == 2) {
719 auto *PT = MD->parameters()[1]->getType()->getAs<PointerType>();
720 if (!PT || !PT->isVoidPointerType() ||
721 !PT->getPointeeType().isConstQualified())
722 return false;
723 }
724
725 return true;
726}
727
728bool CodeGenFunction::isInAllocaArgument(CGCXXABI &ABI, QualType Ty) {
729 const CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
730 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
731}
732
733bool CodeGenFunction::hasInAllocaArg(const CXXMethodDecl *MD) {
734 return getTarget().getTriple().getArch() == llvm::Triple::x86 &&
735 getTarget().getCXXABI().isMicrosoft() &&
736 llvm::any_of(Range: MD->parameters(), P: [&](ParmVarDecl *P) {
737 return isInAllocaArgument(ABI&: CGM.getCXXABI(), Ty: P->getType());
738 });
739}
740
741/// Return the UBSan prologue signature for \p FD if one is available.
742static llvm::Constant *getPrologueSignature(CodeGenModule &CGM,
743 const FunctionDecl *FD) {
744 if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: FD))
745 if (!MD->isStatic())
746 return nullptr;
747 return CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM);
748}
749
750void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
751 llvm::Function *Fn,
752 const CGFunctionInfo &FnInfo,
753 const FunctionArgList &Args,
754 SourceLocation Loc,
755 SourceLocation StartLoc) {
756 assert(!CurFn &&
757 "Do not use a CodeGenFunction object for more than one function");
758
759 const Decl *D = GD.getDecl();
760
761 DidCallStackSave = false;
762 CurCodeDecl = D;
763 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
764 if (FD && FD->usesSEHTry())
765 CurSEHParent = GD;
766 CurFuncDecl = (D ? D->getNonClosureContext() : nullptr);
767 FnRetTy = RetTy;
768 CurFn = Fn;
769 CurFnInfo = &FnInfo;
770 assert(CurFn->isDeclaration() && "Function already has body?");
771
772 // If this function is ignored for any of the enabled sanitizers,
773 // disable the sanitizer for the function.
774 do {
775#define SANITIZER(NAME, ID) \
776 if (SanOpts.empty()) \
777 break; \
778 if (SanOpts.has(SanitizerKind::ID)) \
779 if (CGM.isInNoSanitizeList(SanitizerKind::ID, Fn, Loc)) \
780 SanOpts.set(SanitizerKind::ID, false);
781
782#include "clang/Basic/Sanitizers.def"
783#undef SANITIZER
784 } while (false);
785
786 if (D) {
787 const bool SanitizeBounds = SanOpts.hasOneOf(K: SanitizerKind::Bounds);
788 SanitizerMask no_sanitize_mask;
789 bool NoSanitizeCoverage = false;
790
791 for (auto *Attr : D->specific_attrs<NoSanitizeAttr>()) {
792 no_sanitize_mask |= Attr->getMask();
793 // SanitizeCoverage is not handled by SanOpts.
794 if (Attr->hasCoverage())
795 NoSanitizeCoverage = true;
796 }
797
798 // Apply the no_sanitize* attributes to SanOpts.
799 SanOpts.Mask &= ~no_sanitize_mask;
800 if (no_sanitize_mask & SanitizerKind::Address)
801 SanOpts.set(K: SanitizerKind::KernelAddress, Value: false);
802 if (no_sanitize_mask & SanitizerKind::KernelAddress)
803 SanOpts.set(K: SanitizerKind::Address, Value: false);
804 if (no_sanitize_mask & SanitizerKind::HWAddress)
805 SanOpts.set(K: SanitizerKind::KernelHWAddress, Value: false);
806 if (no_sanitize_mask & SanitizerKind::KernelHWAddress)
807 SanOpts.set(K: SanitizerKind::HWAddress, Value: false);
808
809 if (SanitizeBounds && !SanOpts.hasOneOf(K: SanitizerKind::Bounds))
810 Fn->addFnAttr(Kind: llvm::Attribute::NoSanitizeBounds);
811
812 if (NoSanitizeCoverage && CGM.getCodeGenOpts().hasSanitizeCoverage())
813 Fn->addFnAttr(Kind: llvm::Attribute::NoSanitizeCoverage);
814
815 // Some passes need the non-negated no_sanitize attribute. Pass them on.
816 if (CGM.getCodeGenOpts().hasSanitizeBinaryMetadata()) {
817 if (no_sanitize_mask & SanitizerKind::Thread)
818 Fn->addFnAttr(Kind: "no_sanitize_thread");
819 }
820 }
821
822 if (ShouldSkipSanitizerInstrumentation()) {
823 CurFn->addFnAttr(Kind: llvm::Attribute::DisableSanitizerInstrumentation);
824 } else {
825 // Apply sanitizer attributes to the function.
826 if (SanOpts.hasOneOf(K: SanitizerKind::Address | SanitizerKind::KernelAddress))
827 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAddress);
828 if (SanOpts.hasOneOf(K: SanitizerKind::HWAddress |
829 SanitizerKind::KernelHWAddress))
830 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeHWAddress);
831 if (SanOpts.has(K: SanitizerKind::MemtagStack))
832 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemTag);
833 if (SanOpts.has(K: SanitizerKind::Thread))
834 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeThread);
835 if (SanOpts.has(K: SanitizerKind::Concurrency))
836 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeConcurrency);
837 if (SanOpts.has(K: SanitizerKind::Type))
838 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeType);
839 if (SanOpts.has(K: SanitizerKind::NumericalStability))
840 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeNumericalStability);
841 if (SanOpts.hasOneOf(K: SanitizerKind::Memory | SanitizerKind::KernelMemory))
842 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemory);
843 if (SanOpts.has(K: SanitizerKind::AllocToken))
844 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAllocToken);
845 }
846 if (SanOpts.has(K: SanitizerKind::SafeStack))
847 Fn->addFnAttr(Kind: llvm::Attribute::SafeStack);
848 if (SanOpts.has(K: SanitizerKind::ShadowCallStack))
849 Fn->addFnAttr(Kind: llvm::Attribute::ShadowCallStack);
850
851 if (SanOpts.has(K: SanitizerKind::Realtime))
852 if (FD && FD->getASTContext().hasAnyFunctionEffects())
853 for (const FunctionEffectWithCondition &Fe : FD->getFunctionEffects()) {
854 if (Fe.Effect.kind() == FunctionEffect::Kind::NonBlocking)
855 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeRealtime);
856 else if (Fe.Effect.kind() == FunctionEffect::Kind::Blocking)
857 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeRealtimeBlocking);
858 }
859
860 // Apply fuzzing attribute to the function.
861 if (SanOpts.hasOneOf(K: SanitizerKind::Fuzzer | SanitizerKind::FuzzerNoLink))
862 Fn->addFnAttr(Kind: llvm::Attribute::OptForFuzzing);
863
864 // Ignore TSan/CSan memory accesses from within ObjC/ObjC++ dealloc,
865 // initialize, .cxx_destruct, __destroy_helper_block_ and all of their callees
866 // at run time.
867 if (SanOpts.hasOneOf(K: SanitizerKind::Thread | SanitizerKind::Concurrency)) {
868 if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(Val: D)) {
869 const IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(argIndex: 0);
870 if (OMD->getMethodFamily() == OMF_dealloc ||
871 OMD->getMethodFamily() == OMF_initialize ||
872 (OMD->getSelector().isUnarySelector() && II->isStr(Str: ".cxx_destruct"))) {
873 markAsIgnoreThreadCheckingAtRuntime(Fn);
874 }
875 }
876 }
877
878 // Ignore unrelated casts in STL allocate() since the allocator must cast
879 // from void* to T* before object initialization completes. Don't match on the
880 // namespace because not all allocators are in std::
881 if (D && SanOpts.has(K: SanitizerKind::CFIUnrelatedCast)) {
882 if (matchesStlAllocatorFn(D, Ctx: getContext()))
883 SanOpts.Mask &= ~SanitizerKind::CFIUnrelatedCast;
884 }
885
886 // Ignore null checks in coroutine functions since the coroutines passes
887 // are not aware of how to move the extra UBSan instructions across the split
888 // coroutine boundaries.
889 if (D && SanOpts.has(K: SanitizerKind::Null))
890 if (FD && FD->getBody() &&
891 FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass)
892 SanOpts.Mask &= ~SanitizerKind::Null;
893
894 // Apply xray attributes to the function (as a string, for now)
895 bool AlwaysXRayAttr = false;
896 if (const auto *XRayAttr = D ? D->getAttr<XRayInstrumentAttr>() : nullptr) {
897 if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
898 K: XRayInstrKind::FunctionEntry) ||
899 CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
900 K: XRayInstrKind::FunctionExit)) {
901 if (XRayAttr->alwaysXRayInstrument() && ShouldXRayInstrumentFunction()) {
902 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-always");
903 AlwaysXRayAttr = true;
904 }
905 if (XRayAttr->neverXRayInstrument())
906 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-never");
907 if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())
908 if (ShouldXRayInstrumentFunction())
909 Fn->addFnAttr(Kind: "xray-log-args",
910 Val: llvm::utostr(X: LogArgs->getArgumentCount()));
911 }
912 } else {
913 if (ShouldXRayInstrumentFunction() && !CGM.imbueXRayAttrs(Fn, Loc))
914 Fn->addFnAttr(
915 Kind: "xray-instruction-threshold",
916 Val: llvm::itostr(X: CGM.getCodeGenOpts().XRayInstructionThreshold));
917 }
918
919 if (ShouldXRayInstrumentFunction()) {
920 if (CGM.getCodeGenOpts().XRayIgnoreLoops)
921 Fn->addFnAttr(Kind: "xray-ignore-loops");
922
923 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
924 K: XRayInstrKind::FunctionExit))
925 Fn->addFnAttr(Kind: "xray-skip-exit");
926
927 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
928 K: XRayInstrKind::FunctionEntry))
929 Fn->addFnAttr(Kind: "xray-skip-entry");
930
931 auto FuncGroups = CGM.getCodeGenOpts().XRayTotalFunctionGroups;
932 if (FuncGroups > 1) {
933 auto FuncName = llvm::ArrayRef<uint8_t>(CurFn->getName().bytes_begin(),
934 CurFn->getName().bytes_end());
935 auto Group = crc32(Data: FuncName) % FuncGroups;
936 if (Group != CGM.getCodeGenOpts().XRaySelectedFunctionGroup &&
937 !AlwaysXRayAttr)
938 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-never");
939 }
940 }
941
942 if (CGM.getCodeGenOpts().getProfileInstr() !=
943 llvm::driver::ProfileInstrKind::ProfileNone) {
944 switch (CGM.isFunctionBlockedFromProfileInstr(Fn, Loc)) {
945 case ProfileList::Skip:
946 Fn->addFnAttr(Kind: llvm::Attribute::SkipProfile);
947 break;
948 case ProfileList::Forbid:
949 Fn->addFnAttr(Kind: llvm::Attribute::NoProfile);
950 break;
951 case ProfileList::Allow:
952 break;
953 }
954 }
955
956 unsigned Count, Offset;
957 StringRef Section;
958 if (const auto *Attr =
959 D ? D->getAttr<PatchableFunctionEntryAttr>() : nullptr) {
960 Count = Attr->getCount();
961 Offset = Attr->getOffset();
962 Section = Attr->getSection();
963 } else {
964 Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;
965 Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;
966 }
967 if (Section.empty())
968 Section = CGM.getCodeGenOpts().PatchableFunctionEntrySection;
969 if (Count && Offset <= Count) {
970 Fn->addFnAttr(Kind: "patchable-function-entry", Val: std::to_string(val: Count - Offset));
971 if (Offset)
972 Fn->addFnAttr(Kind: "patchable-function-prefix", Val: std::to_string(val: Offset));
973 if (!Section.empty())
974 Fn->addFnAttr(Kind: "patchable-function-entry-section", Val: Section);
975 }
976 // Instruct that functions for COFF/CodeView targets should start with a
977 // patchable instruction, but only on x86/x64. Don't forward this to ARM/ARM64
978 // backends as they don't need it -- instructions on these architectures are
979 // always atomically patchable at runtime.
980 if (CGM.getCodeGenOpts().HotPatch &&
981 getContext().getTargetInfo().getTriple().isX86() &&
982 getContext().getTargetInfo().getTriple().getEnvironment() !=
983 llvm::Triple::CODE16)
984 Fn->addFnAttr(Kind: "patchable-function", Val: "prologue-short-redirect");
985
986 // Add no-jump-tables value.
987 if (CGM.getCodeGenOpts().NoUseJumpTables)
988 Fn->addFnAttr(Kind: "no-jump-tables", Val: "true");
989
990 // Add no-inline-line-tables value.
991 if (CGM.getCodeGenOpts().NoInlineLineTables)
992 Fn->addFnAttr(Kind: "no-inline-line-tables");
993
994 // Add profile-sample-accurate value.
995 if (CGM.getCodeGenOpts().ProfileSampleAccurate)
996 Fn->addFnAttr(Kind: "profile-sample-accurate");
997
998 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
999 Fn->addFnAttr(Kind: "use-sample-profile");
1000
1001 if (D && D->hasAttr<CFICanonicalJumpTableAttr>())
1002 Fn->addFnAttr(Kind: "cfi-canonical-jump-table");
1003
1004 if (D && D->hasAttr<NoProfileFunctionAttr>())
1005 Fn->addFnAttr(Kind: llvm::Attribute::NoProfile);
1006
1007 if (D && D->hasAttr<HybridPatchableAttr>())
1008 Fn->addFnAttr(Kind: llvm::Attribute::HybridPatchable);
1009
1010 if (D) {
1011 // Function attributes take precedence over command line flags.
1012 if (auto *A = D->getAttr<FunctionReturnThunksAttr>()) {
1013 switch (A->getThunkType()) {
1014 case FunctionReturnThunksAttr::Kind::Keep:
1015 break;
1016 case FunctionReturnThunksAttr::Kind::Extern:
1017 Fn->addFnAttr(Kind: llvm::Attribute::FnRetThunkExtern);
1018 break;
1019 }
1020 } else if (CGM.getCodeGenOpts().FunctionReturnThunks)
1021 Fn->addFnAttr(Kind: llvm::Attribute::FnRetThunkExtern);
1022 }
1023
1024 if (FD && (getLangOpts().OpenCL ||
1025 (getLangOpts().CUDA &&
1026 getContext().getTargetInfo().getTriple().isSPIRV()) ||
1027 ((getLangOpts().HIP || getLangOpts().OffloadViaLLVM) &&
1028 getLangOpts().CUDAIsDevice))) {
1029 // Add metadata for a kernel function.
1030 EmitKernelMetadata(FD, Fn);
1031 }
1032
1033 if (FD && FD->hasAttr<ClspvLibclcBuiltinAttr>()) {
1034 Fn->setMetadata(Kind: "clspv_libclc_builtin",
1035 Node: llvm::MDNode::get(Context&: getLLVMContext(), MDs: {}));
1036 }
1037
1038 // If we are checking function types, emit a function type signature as
1039 // prologue data. Kernel functions have strict alignment requirements and
1040 // cannot be call indirectly so we do not instrument them.
1041 if (FD && SanOpts.has(K: SanitizerKind::Function) &&
1042 !FD->getType()->isCFIUncheckedCalleeFunctionType() &&
1043 llvm::isCallableCC(CC: Fn->getCallingConv())) {
1044 if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {
1045 llvm::LLVMContext &Ctx = Fn->getContext();
1046 llvm::MDBuilder MDB(Ctx);
1047 Fn->setMetadata(
1048 KindID: llvm::LLVMContext::MD_func_sanitize,
1049 Node: MDB.createRTTIPointerPrologue(
1050 PrologueSig, RTTI: getUBSanFunctionTypeHash(Ty: FD->getType())));
1051 }
1052 }
1053
1054 // If we're checking nullability, we need to know whether we can check the
1055 // return value. Initialize the flag to 'true' and refine it in EmitParmDecl.
1056 if (SanOpts.has(K: SanitizerKind::NullabilityReturn)) {
1057 auto Nullability = FnRetTy->getNullability();
1058 if (Nullability && *Nullability == NullabilityKind::NonNull &&
1059 !FnRetTy->isRecordType()) {
1060 if (!(SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute) &&
1061 CurCodeDecl && CurCodeDecl->getAttr<ReturnsNonNullAttr>()))
1062 RetValNullabilityPrecondition =
1063 llvm::ConstantInt::getTrue(Context&: getLLVMContext());
1064 }
1065 }
1066
1067 // Annotate C++ special member functions so that CopyProfPass can instrument
1068 // them, provided the object is at least as large as the size threshold.
1069 if (CGM.getCodeGenOpts().CopyProf) {
1070 if (const auto *MD = dyn_cast_or_null<CXXMethodDecl>(Val: D)) {
1071 StringRef Attr;
1072 if (const auto *CD = dyn_cast<CXXConstructorDecl>(Val: MD)) {
1073 if (!CD->isMoveConstructor())
1074 Attr =
1075 CD->isCopyConstructor() ? "copyprof-copy-ctor" : "copyprof-ctor";
1076 } else if (isa<CXXDestructorDecl>(Val: MD)) {
1077 Attr = "copyprof-dtor";
1078 } else if (MD->isCopyAssignmentOperator()) {
1079 Attr = "copyprof-copy-assign-op";
1080 }
1081 if (!Attr.empty()) {
1082 // Finally, add the object size in bytes to the annotation.
1083 // A special member function always has an implicit object parameter, so
1084 // its type is guaranteed to be complete here.
1085 CharUnits ObjSize = getContext().getTypeSizeInChars(
1086 T: MD->getFunctionObjectParameterType());
1087 if (ObjSize.getQuantity() >=
1088 CGM.getCodeGenOpts().CopyProfStaticSizeThreshold)
1089 Fn->addFnAttr(Kind: Attr, Val: llvm::utostr(X: ObjSize.getQuantity()));
1090 }
1091 }
1092 }
1093
1094 // If we're in C++ mode and the function name is "main", it is guaranteed
1095 // to be norecurse by the standard (3.6.1.3 "The function main shall not be
1096 // used within a program").
1097 //
1098 // OpenCL C 2.0 v2.2-11 s6.9.i:
1099 // Recursion is not supported.
1100 //
1101 // HLSL
1102 // Recursion is not supported.
1103 //
1104 // SYCL v1.2.1 s3.10:
1105 // kernels cannot include RTTI information, exception classes,
1106 // recursive code, virtual functions or make use of C++ libraries that
1107 // are not compiled for the device.
1108 if (FD &&
1109 ((getLangOpts().CPlusPlus && FD->isMain()) || getLangOpts().OpenCL ||
1110 getLangOpts().HLSL || getLangOpts().SYCLIsDevice ||
1111 (getLangOpts().CUDA && FD->hasAttr<CUDAGlobalAttr>())))
1112 Fn->addFnAttr(Kind: llvm::Attribute::NoRecurse);
1113
1114 llvm::RoundingMode RM = getLangOpts().getDefaultRoundingMode();
1115 llvm::fp::ExceptionBehavior FPExceptionBehavior =
1116 ToConstrainedExceptMD(Kind: getLangOpts().getDefaultExceptionMode());
1117 Builder.setDefaultConstrainedRounding(RM);
1118 Builder.setDefaultConstrainedExcept(FPExceptionBehavior);
1119 if ((FD && (FD->UsesFPIntrin() || FD->hasAttr<StrictFPAttr>())) ||
1120 (!FD && (FPExceptionBehavior != llvm::fp::ebIgnore ||
1121 RM != llvm::RoundingMode::NearestTiesToEven))) {
1122 Builder.setIsFPConstrained(true);
1123 Fn->addFnAttr(Kind: llvm::Attribute::StrictFP);
1124 }
1125
1126 // If a custom alignment is used, force realigning to this alignment on
1127 // any main function which certainly will need it.
1128 if (FD && ((FD->isMain() || FD->isMSVCRTEntryPoint()) &&
1129 CGM.getCodeGenOpts().StackAlignment))
1130 Fn->addFnAttr(Kind: "stackrealign");
1131
1132 // "main" doesn't need to zero out call-used registers.
1133 if (FD && FD->isMain())
1134 Fn->removeFnAttr(Kind: "zero-call-used-regs");
1135
1136 // Add vscale_range attribute if appropriate.
1137 auto IsArmStreaming = TargetInfo::ArmStreamingKind::NotStreaming;
1138 if (FD) {
1139 if (const auto *T = FD->getType()->getAs<FunctionProtoType>())
1140 if (T->getAArch64SMEAttributes() &
1141 FunctionType::SME_PStateSMCompatibleMask)
1142 IsArmStreaming = TargetInfo::ArmStreamingKind::StreamingCompatible;
1143
1144 if (IsArmStreamingFunction(FD, IncludeLocallyStreaming: true))
1145 IsArmStreaming = TargetInfo::ArmStreamingKind::Streaming;
1146 }
1147 const auto *FeatureMap =
1148 FD ? &getContext().getFunctionFeatureMap(FD) : nullptr;
1149 if (auto VScaleRange = getContext().getTargetInfo().getVScaleRange(
1150 LangOpts: getLangOpts(), Mode: IsArmStreaming, FeatureMap))
1151 CurFn->addFnAttr(Attr: llvm::Attribute::getWithVScaleRangeArgs(
1152 Context&: getLLVMContext(), MinValue: VScaleRange->first, MaxValue: VScaleRange->second));
1153
1154 llvm::BasicBlock *EntryBB = createBasicBlock(name: "entry", parent: CurFn);
1155
1156 // Create a marker to make it easy to insert allocas into the entryblock
1157 // later. Don't create this with the builder, because we don't want it
1158 // folded.
1159 llvm::Value *Poison = llvm::PoisonValue::get(T: Int32Ty);
1160 AllocaInsertPt = new llvm::BitCastInst(Poison, Int32Ty, "allocapt", EntryBB);
1161
1162 ReturnBlock = getJumpDestInCurrentScope(Name: "return");
1163
1164 Builder.SetInsertPoint(EntryBB);
1165
1166 // If we're checking the return value, allocate space for a pointer to a
1167 // precise source location of the checked return statement.
1168 if (requiresReturnValueCheck()) {
1169 ReturnLocation = CreateDefaultAlignTempAlloca(Ty: Int8PtrTy, Name: "return.sloc.ptr");
1170 Builder.CreateStore(Val: llvm::ConstantPointerNull::get(T: Int8PtrTy),
1171 Addr: ReturnLocation);
1172 }
1173
1174 // Emit subprogram debug descriptor.
1175 if (CGDebugInfo *DI = getDebugInfo()) {
1176 // Reconstruct the type from the argument list so that implicit parameters,
1177 // such as 'this' and 'vtt', show up in the debug info. Preserve the calling
1178 // convention.
1179 DI->emitFunctionStart(GD, Loc, ScopeLoc: StartLoc,
1180 FnType: DI->getFunctionType(FD, RetTy, Args), Fn: CurFn,
1181 CurFnIsThunk: CurFuncIsThunk);
1182 }
1183
1184 if (ShouldInstrumentFunction()) {
1185 if (CGM.getCodeGenOpts().InstrumentFunctions)
1186 CurFn->addFnAttr(Kind: "instrument-function-entry", Val: "__cyg_profile_func_enter");
1187 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
1188 CurFn->addFnAttr(Kind: "instrument-function-entry-inlined",
1189 Val: "__cyg_profile_func_enter");
1190 if (CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
1191 CurFn->addFnAttr(Kind: "instrument-function-entry-inlined",
1192 Val: "__cyg_profile_func_enter_bare");
1193 }
1194
1195 // Since emitting the mcount call here impacts optimizations such as function
1196 // inlining, we just add an attribute to insert a mcount call in backend.
1197 // The attribute "counting-function" is set to mcount function name which is
1198 // architecture dependent.
1199 if (CGM.getCodeGenOpts().InstrumentForProfiling) {
1200 // Calls to fentry/mcount should not be generated if function has
1201 // the no_instrument_function attribute.
1202 if (!CurFuncDecl || !CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()) {
1203 if (CGM.getCodeGenOpts().CallFEntry)
1204 Fn->addFnAttr(Kind: "fentry-call", Val: "true");
1205 else {
1206 Fn->addFnAttr(Kind: "instrument-function-entry-inlined",
1207 Val: getTarget().getMCountName());
1208 }
1209 if (CGM.getCodeGenOpts().MNopMCount) {
1210 if (!CGM.getCodeGenOpts().CallFEntry)
1211 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_without_opt)
1212 << "-mnop-mcount" << "-mfentry";
1213 Fn->addFnAttr(Kind: "mnop-mcount");
1214 }
1215
1216 if (CGM.getCodeGenOpts().RecordMCount) {
1217 if (!CGM.getCodeGenOpts().CallFEntry)
1218 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_without_opt)
1219 << "-mrecord-mcount" << "-mfentry";
1220 Fn->addFnAttr(Kind: "mrecord-mcount");
1221 }
1222 }
1223 }
1224
1225 if (CGM.getCodeGenOpts().PackedStack) {
1226 if (getContext().getTargetInfo().getTriple().getArch() !=
1227 llvm::Triple::systemz)
1228 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_on_target)
1229 << "-mpacked-stack";
1230 Fn->addFnAttr(Kind: "packed-stack");
1231 }
1232
1233 if (!CGM.getCodeGenOpts().ZOSPPA1Name)
1234 Fn->addFnAttr(Kind: "zos-ppa1-name", Val: "");
1235 else if (getContext().getTargetInfo().getTriple().isOSzOS() && FD &&
1236 FD->hasAttr<AsmLabelAttr>()) {
1237 std::string SrcName = CGM.getMangledNameIgnoringAsmLabel(GD, ND: FD);
1238 Fn->addFnAttr(Kind: "zos-ppa1-name", Val: SrcName);
1239 }
1240
1241 if (CGM.getCodeGenOpts().WarnStackSize != UINT_MAX &&
1242 !CGM.getDiags().isIgnored(DiagID: diag::warn_fe_backend_frame_larger_than, Loc))
1243 Fn->addFnAttr(Kind: "warn-stack-size",
1244 Val: std::to_string(val: CGM.getCodeGenOpts().WarnStackSize));
1245
1246 if (RetTy->isVoidType()) {
1247 // Void type; nothing to return.
1248 ReturnValue = Address::invalid();
1249
1250 // Count the implicit return.
1251 if (!endsWithReturn(F: D))
1252 ++NumReturnExprs;
1253 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1254 // Indirect return; emit returned value directly into sret slot.
1255 // This reduces code size, and affects correctness in C++.
1256 auto AI = CurFn->arg_begin();
1257 if (CurFnInfo->getReturnInfo().isSRetAfterThis())
1258 ++AI;
1259 ReturnValue = makeNaturalAddressForPointer(
1260 Ptr: &*AI, T: RetTy, Alignment: CurFnInfo->getReturnInfo().getIndirectAlign(), ForPointeeType: false,
1261 BaseInfo: nullptr, TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
1262 if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
1263 ReturnValuePointer =
1264 CreateDefaultAlignTempAlloca(Ty: ReturnValue.getType(), Name: "result.ptr");
1265 Builder.CreateStore(Val: ReturnValue.emitRawPointer(CGF&: *this),
1266 Addr: ReturnValuePointer);
1267 }
1268 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
1269 !hasScalarEvaluationKind(T: CurFnInfo->getReturnType())) {
1270 // Load the sret pointer from the argument struct and return into that.
1271 unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();
1272 llvm::Function::arg_iterator EI = CurFn->arg_end();
1273 --EI;
1274 llvm::Value *Addr = Builder.CreateStructGEP(
1275 Ty: CurFnInfo->getArgStruct(), Ptr: &*EI, Idx);
1276 llvm::Type *Ty =
1277 cast<llvm::GetElementPtrInst>(Val: Addr)->getResultElementType();
1278 ReturnValuePointer = Address(Addr, Ty, getPointerAlign());
1279 Addr = Builder.CreateAlignedLoad(Ty, Addr, Align: getPointerAlign(), Name: "agg.result");
1280 ReturnValue = Address(Addr, ConvertType(T: RetTy),
1281 CGM.getNaturalTypeAlignment(T: RetTy), KnownNonNull);
1282 } else {
1283 ReturnValue = CreateIRTempWithoutCast(T: RetTy, Name: "retval");
1284
1285 // Tell the epilog emitter to autorelease the result. We do this
1286 // now so that various specialized functions can suppress it
1287 // during their IR-generation.
1288 if (getLangOpts().ObjCAutoRefCount &&
1289 !CurFnInfo->isReturnsRetained() &&
1290 RetTy->isObjCRetainableType())
1291 AutoreleaseResult = true;
1292 }
1293
1294 EmitStartEHSpec(D: CurCodeDecl);
1295
1296 PrologueCleanupDepth = EHStack.stable_begin();
1297
1298 // Emit OpenMP specific initialization of the device functions.
1299 if (getLangOpts().OpenMP && CurCodeDecl)
1300 CGM.getOpenMPRuntime().emitFunctionProlog(CGF&: *this, D: CurCodeDecl);
1301
1302 if (FD && getLangOpts().HLSL) {
1303 // Handle emitting HLSL entry functions.
1304 if (FD->hasAttr<HLSLShaderAttr>()) {
1305 CGM.getHLSLRuntime().emitEntryFunction(FD, Fn);
1306 }
1307 }
1308
1309 EmitFunctionProlog(FI: *CurFnInfo, Fn: CurFn, Args);
1310
1311 if (const CXXMethodDecl *MD = dyn_cast_if_present<CXXMethodDecl>(Val: D);
1312 MD && !MD->isStatic()) {
1313 bool IsInLambda =
1314 MD->getParent()->isLambda() && MD->getOverloadedOperator() == OO_Call;
1315 if (MD->isImplicitObjectMemberFunction())
1316 CGM.getCXXABI().EmitInstanceFunctionProlog(CGF&: *this);
1317 if (IsInLambda) {
1318 // We're in a lambda; figure out the captures.
1319 MD->getParent()->getCaptureFields(Captures&: LambdaCaptureFields,
1320 ThisCapture&: LambdaThisCaptureField);
1321 if (LambdaThisCaptureField) {
1322 // If the lambda captures the object referred to by '*this' - either by
1323 // value or by reference, make sure CXXThisValue points to the correct
1324 // object.
1325
1326 // Get the lvalue for the field (which is a copy of the enclosing object
1327 // or contains the address of the enclosing object).
1328 LValue ThisFieldLValue = EmitLValueForLambdaField(Field: LambdaThisCaptureField);
1329 if (!LambdaThisCaptureField->getType()->isPointerType()) {
1330 // If the enclosing object was captured by value, just use its
1331 // address. Sign this pointer.
1332 CXXThisValue = ThisFieldLValue.getPointer(CGF&: *this);
1333 } else {
1334 // Load the lvalue pointed to by the field, since '*this' was captured
1335 // by reference.
1336 CXXThisValue =
1337 EmitLoadOfLValue(V: ThisFieldLValue, Loc: SourceLocation()).getScalarVal();
1338 }
1339 }
1340 for (auto *FD : MD->getParent()->fields()) {
1341 if (FD->hasCapturedVLAType()) {
1342 auto *ExprArg = EmitLoadOfLValue(V: EmitLValueForLambdaField(Field: FD),
1343 Loc: SourceLocation()).getScalarVal();
1344 auto VAT = FD->getCapturedVLAType();
1345 VLASizeMap[VAT->getSizeExpr()] = ExprArg;
1346 }
1347 }
1348 } else if (MD->isImplicitObjectMemberFunction()) {
1349 // Not in a lambda; just use 'this' from the method.
1350 // FIXME: Should we generate a new load for each use of 'this'? The
1351 // fast register allocator would be happier...
1352 CXXThisValue = CXXABIThisValue;
1353 }
1354
1355 // Check the 'this' pointer once per function, if it's available.
1356 if (CXXABIThisValue) {
1357 SanitizerSet SkippedChecks;
1358 SkippedChecks.set(K: SanitizerKind::ObjectSize, Value: true);
1359 QualType ThisTy = MD->getThisType();
1360
1361 // If this is the call operator of a lambda with no captures, it
1362 // may have a static invoker function, which may call this operator with
1363 // a null 'this' pointer.
1364 if (isLambdaCallOperator(MD) && MD->getParent()->isCapturelessLambda())
1365 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
1366
1367 EmitTypeCheck(
1368 TCK: isa<CXXConstructorDecl>(Val: MD) ? TCK_ConstructorCall : TCK_MemberCall,
1369 Loc, V: CXXABIThisValue, Type: ThisTy, Alignment: CXXABIThisAlignment, SkippedChecks);
1370 }
1371 }
1372
1373 // If any of the arguments have a variably modified type, make sure to
1374 // emit the type size, but only if the function is not naked. Naked functions
1375 // have no prolog to run this evaluation.
1376 if (!FD || !FD->hasAttr<NakedAttr>()) {
1377 for (const VarDecl *VD : Args) {
1378 // Dig out the type as written from ParmVarDecls; it's unclear whether
1379 // the standard (C99 6.9.1p10) requires this, but we're following the
1380 // precedent set by gcc.
1381 QualType Ty;
1382 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: VD))
1383 Ty = PVD->getOriginalType();
1384 else
1385 Ty = VD->getType();
1386
1387 if (Ty->isVariablyModifiedType())
1388 EmitVariablyModifiedType(Ty);
1389 }
1390 }
1391 // Emit a location at the end of the prologue.
1392 if (CGDebugInfo *DI = getDebugInfo())
1393 DI->EmitLocation(Builder, Loc: StartLoc);
1394 // TODO: Do we need to handle this in two places like we do with
1395 // target-features/target-cpu?
1396 if (CurFuncDecl)
1397 if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>())
1398 LargestVectorWidth = VecWidth->getVectorWidth();
1399
1400 if (CGM.shouldEmitConvergenceTokens())
1401 ConvergenceTokenStack.push_back(Elt: getOrEmitConvergenceEntryToken(F: CurFn));
1402}
1403
1404void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {
1405 incrementProfileCounter(S: Body);
1406 maybeCreateMCDCCondBitmap();
1407 if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Val: Body))
1408 EmitCompoundStmtWithoutScope(S: *S);
1409 else
1410 EmitStmt(S: Body);
1411}
1412
1413/// When instrumenting to collect profile data, the counts for some blocks
1414/// such as switch cases need to not include the fall-through counts, so
1415/// emit a branch around the instrumentation code. When not instrumenting,
1416/// this just calls EmitBlock().
1417void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
1418 const Stmt *S) {
1419 llvm::BasicBlock *SkipCountBB = nullptr;
1420 if (HaveInsertPoint() && CGM.getCodeGenOpts().hasProfileClangInstr()) {
1421 // When instrumenting for profiling, the fallthrough to certain
1422 // statements needs to skip over the instrumentation code so that we
1423 // get an accurate count.
1424 SkipCountBB = createBasicBlock(name: "skipcount");
1425 EmitBranch(Block: SkipCountBB);
1426 }
1427 EmitBlock(BB);
1428 uint64_t CurrentCount = getCurrentProfileCount();
1429 incrementProfileCounter(ExecSkip: UseExecPath, S);
1430 setCurrentProfileCount(getCurrentProfileCount() + CurrentCount);
1431 if (SkipCountBB)
1432 EmitBlock(BB: SkipCountBB);
1433}
1434
1435/// Tries to mark the given function nounwind based on the
1436/// non-existence of any throwing calls within it. We believe this is
1437/// lightweight enough to do at -O0.
1438static void TryMarkNoThrow(llvm::Function *F) {
1439 // LLVM treats 'nounwind' on a function as part of the type, so we
1440 // can't do this on functions that can be overwritten.
1441 if (F->isInterposable()) return;
1442
1443 for (llvm::BasicBlock &BB : *F)
1444 for (llvm::Instruction &I : BB)
1445 if (I.mayThrow())
1446 return;
1447
1448 F->setDoesNotThrow();
1449}
1450
1451QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD,
1452 FunctionArgList &Args) {
1453 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
1454 QualType ResTy = FD->getReturnType();
1455
1456 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD);
1457 if (MD && MD->isImplicitObjectMemberFunction()) {
1458 if (CGM.getCXXABI().HasThisReturn(GD))
1459 ResTy = MD->getThisType();
1460 else if (CGM.getCXXABI().hasMostDerivedReturn(GD))
1461 ResTy = CGM.getContext().VoidPtrTy;
1462 CGM.getCXXABI().buildThisParam(CGF&: *this, Params&: Args);
1463 }
1464
1465 // The base version of an inheriting constructor whose constructed base is a
1466 // virtual base is not passed any arguments (because it doesn't actually call
1467 // the inherited constructor).
1468 bool PassedParams = true;
1469 if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(Val: FD))
1470 if (auto Inherited = CD->getInheritedConstructor())
1471 PassedParams =
1472 getTypes().inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
1473
1474 if (PassedParams) {
1475 for (auto *Param : FD->parameters()) {
1476 Args.push_back(Elt: Param);
1477 if (!Param->hasAttr<PassObjectSizeAttr>())
1478 continue;
1479
1480 auto *Implicit = ImplicitParamDecl::Create(
1481 C&: getContext(), DC: Param->getDeclContext(), IdLoc: Param->getLocation(),
1482 /*Id=*/nullptr, T: getContext().getSizeType(), ParamKind: ImplicitParamKind::Other);
1483 SizeArguments[Param] = Implicit;
1484 Args.push_back(Elt: Implicit);
1485 }
1486 }
1487
1488 if (MD && (isa<CXXConstructorDecl>(Val: MD) || isa<CXXDestructorDecl>(Val: MD)))
1489 CGM.getCXXABI().addImplicitStructorParams(CGF&: *this, ResTy, Params&: Args);
1490
1491 return ResTy;
1492}
1493
1494void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
1495 const CGFunctionInfo &FnInfo) {
1496 assert(Fn && "generating code for null Function");
1497 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
1498 CurGD = GD;
1499
1500 FunctionArgList Args;
1501 QualType ResTy = BuildFunctionArgList(GD, Args);
1502
1503 CGM.getTargetCodeGenInfo().checkFunctionABI(CGM, Decl: FD);
1504
1505 if (FD->isInlineBuiltinDeclaration()) {
1506 // When generating code for a builtin with an inline declaration, use a
1507 // mangled name to hold the actual body, while keeping an external
1508 // definition in case the function pointer is referenced somewhere.
1509 std::string FDInlineName = (Fn->getName() + ".inline").str();
1510 llvm::Module *M = Fn->getParent();
1511 llvm::Function *Clone = M->getFunction(Name: FDInlineName);
1512 if (!Clone) {
1513 Clone = llvm::Function::Create(Ty: Fn->getFunctionType(),
1514 Linkage: llvm::GlobalValue::InternalLinkage,
1515 AddrSpace: Fn->getAddressSpace(), N: FDInlineName, M);
1516 Clone->setCallingConv(Fn->getCallingConv());
1517 Clone->setAttributes(Fn->getAttributes());
1518 Clone->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1519 }
1520 Fn->setLinkage(llvm::GlobalValue::ExternalLinkage);
1521 Fn = Clone;
1522 } else {
1523 // Detect the unusual situation where an inline version is shadowed by a
1524 // non-inline version. In that case we should pick the external one
1525 // everywhere. That's GCC behavior too. Unfortunately, I cannot find a way
1526 // to detect that situation before we reach codegen, so do some late
1527 // replacement.
1528 for (const FunctionDecl *PD = FD->getPreviousDecl(); PD;
1529 PD = PD->getPreviousDecl()) {
1530 if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) {
1531 std::string FDInlineName = (Fn->getName() + ".inline").str();
1532 llvm::Module *M = Fn->getParent();
1533 if (llvm::Function *Clone = M->getFunction(Name: FDInlineName)) {
1534 Clone->replaceAllUsesWith(V: Fn);
1535 Clone->eraseFromParent();
1536 }
1537 break;
1538 }
1539 }
1540 }
1541
1542 // Check if we should generate debug info for this function.
1543 if (FD->hasAttr<NoDebugAttr>()) {
1544 // Clear non-distinct debug info that was possibly attached to the function
1545 // due to an earlier declaration without the nodebug attribute
1546 Fn->setSubprogram(nullptr);
1547 // Disable debug info indefinitely for this function
1548 DebugInfo = nullptr;
1549 }
1550 // Finalize function debug info on exit.
1551 llvm::scope_exit Cleanup([this] {
1552 if (CGDebugInfo *DI = getDebugInfo())
1553 DI->completeFunction();
1554 });
1555
1556 // The function might not have a body if we're generating thunks for a
1557 // function declaration.
1558 SourceRange BodyRange;
1559 if (Stmt *Body = FD->getBody())
1560 BodyRange = Body->getSourceRange();
1561 else
1562 BodyRange = FD->getLocation();
1563 CurEHLocation = BodyRange.getEnd();
1564
1565 // Use the location of the start of the function to determine where
1566 // the function definition is located. By default use the location
1567 // of the declaration as the location for the subprogram. A function
1568 // may lack a declaration in the source code if it is created by code
1569 // gen. (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).
1570 SourceLocation Loc = FD->getLocation();
1571
1572 // If this is a function specialization then use the pattern body
1573 // as the location for the function.
1574 if (const FunctionDecl *SpecDecl = FD->getTemplateInstantiationPattern())
1575 if (SpecDecl->hasBody(Definition&: SpecDecl))
1576 Loc = SpecDecl->getLocation();
1577
1578 Stmt *Body = FD->getBody();
1579
1580 if (Body) {
1581 // Coroutines always emit lifetime markers.
1582 if (isa<CoroutineBodyStmt>(Val: Body))
1583 ShouldEmitLifetimeMarkers = true;
1584
1585 // Detect jumps that invalidate lifetime markers or bypass auto-var-init.
1586 bool NeedsBypassDetection =
1587 ShouldEmitLifetimeMarkers ||
1588 (CGM.getLangOpts().getTrivialAutoVarInit() !=
1589 LangOptions::TrivialAutoVarInitKind::Uninitialized);
1590 if (NeedsBypassDetection)
1591 Bypasses.Init(CGM, Body);
1592 }
1593
1594 // Emit the standard function prologue.
1595 StartFunction(GD, RetTy: ResTy, Fn, FnInfo, Args, Loc, StartLoc: BodyRange.getBegin());
1596
1597 // Save parameters for coroutine function.
1598 if (Body && isa_and_nonnull<CoroutineBodyStmt>(Val: Body))
1599 llvm::append_range(C&: FnArgs, R: FD->parameters());
1600
1601 // Ensure that the function adheres to the forward progress guarantee, which
1602 // is required by certain optimizations.
1603 // In C++11 and up, the attribute will be removed if the body contains a
1604 // trivial empty loop.
1605 if (checkIfFunctionMustProgress())
1606 CurFn->addFnAttr(Kind: llvm::Attribute::MustProgress);
1607
1608 // Generate the body of the function.
1609 PGO->assignRegionCounters(GD, Fn: CurFn);
1610 if (isa<CXXDestructorDecl>(Val: FD))
1611 EmitDestructorBody(Args);
1612 else if (isa<CXXConstructorDecl>(Val: FD))
1613 EmitConstructorBody(Args);
1614 else if (getLangOpts().CUDA &&
1615 !getLangOpts().CUDAIsDevice &&
1616 FD->hasAttr<CUDAGlobalAttr>())
1617 CGM.getCUDARuntime().emitDeviceStub(CGF&: *this, Args);
1618 else if (isa<CXXMethodDecl>(Val: FD) &&
1619 cast<CXXMethodDecl>(Val: FD)->isLambdaStaticInvoker()) {
1620 // The lambda static invoker function is special, because it forwards or
1621 // clones the body of the function call operator (but is actually static).
1622 EmitLambdaStaticInvokeBody(MD: cast<CXXMethodDecl>(Val: FD));
1623 } else if (isa<CXXMethodDecl>(Val: FD) &&
1624 isLambdaCallOperator(MD: cast<CXXMethodDecl>(Val: FD)) &&
1625 !FnInfo.isDelegateCall() &&
1626 cast<CXXMethodDecl>(Val: FD)->getParent()->getLambdaStaticInvoker() &&
1627 hasInAllocaArg(MD: cast<CXXMethodDecl>(Val: FD))) {
1628 // If emitting a lambda with static invoker on X86 Windows, change
1629 // the call operator body.
1630 // Make sure that this is a call operator with an inalloca arg and check
1631 // for delegate call to make sure this is the original call op and not the
1632 // new forwarding function for the static invoker.
1633 EmitLambdaInAllocaCallOpBody(MD: cast<CXXMethodDecl>(Val: FD));
1634 } else if (FD->isDefaulted() && isa<CXXMethodDecl>(Val: FD) &&
1635 (cast<CXXMethodDecl>(Val: FD)->isCopyAssignmentOperator() ||
1636 cast<CXXMethodDecl>(Val: FD)->isMoveAssignmentOperator())) {
1637 // Implicit copy-assignment gets the same special treatment as implicit
1638 // copy-constructors.
1639 emitImplicitAssignmentOperatorBody(Args);
1640 } else if (DeviceKernelAttr::isOpenCLSpelling(
1641 A: FD->getAttr<DeviceKernelAttr>()) &&
1642 GD.getKernelReferenceKind() == KernelReferenceKind::Kernel) {
1643 CallArgList CallArgs;
1644 for (unsigned i = 0; i < Args.size(); ++i) {
1645 Address ArgAddr = GetAddrOfLocalVar(VD: Args[i]);
1646 QualType ArgQualType = Args[i]->getType();
1647 RValue ArgRValue = convertTempToRValue(addr: ArgAddr, type: ArgQualType, Loc);
1648 CallArgs.add(rvalue: ArgRValue, type: ArgQualType);
1649 }
1650 GlobalDecl GDStub = GlobalDecl(FD, KernelReferenceKind::Stub);
1651 const FunctionType *FT = cast<FunctionType>(Val: FD->getType());
1652 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
1653 const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
1654 Args: CallArgs, Ty: FT, /*ChainCall=*/false, ABIInfoFD: getCurrentFunctionDecl());
1655 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
1656 llvm::Constant *GDStubFunctionPointer =
1657 CGM.getRawFunctionPointer(GD: GDStub, Ty: FTy);
1658 CGCallee GDStubCallee = CGCallee::forDirect(functionPtr: GDStubFunctionPointer, abstractInfo: GDStub);
1659 EmitCall(CallInfo: FnInfo, Callee: GDStubCallee, ReturnValue: ReturnValueSlot(), Args: CallArgs, CallOrInvoke: nullptr, IsMustTail: false,
1660 Loc);
1661 } else if (Body) {
1662 EmitFunctionBody(Body);
1663 } else
1664 llvm_unreachable("no definition for emitted function");
1665
1666 // C++11 [stmt.return]p2:
1667 // Flowing off the end of a function [...] results in undefined behavior in
1668 // a value-returning function.
1669 // C11 6.9.1p12:
1670 // If the '}' that terminates a function is reached, and the value of the
1671 // function call is used by the caller, the behavior is undefined.
1672 if (getLangOpts().CPlusPlus && !FD->hasImplicitReturnZero() && !SawAsmBlock &&
1673 !FD->getReturnType()->isVoidType() && Builder.GetInsertBlock()) {
1674 bool ShouldEmitUnreachable =
1675 CGM.getCodeGenOpts().StrictReturn ||
1676 !CGM.MayDropFunctionReturn(Context: FD->getASTContext(), ReturnType: FD->getReturnType());
1677 if (SanOpts.has(K: SanitizerKind::Return)) {
1678 auto CheckOrdinal = SanitizerKind::SO_Return;
1679 auto CheckHandler = SanitizerHandler::MissingReturn;
1680 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
1681 llvm::Value *IsFalse = Builder.getFalse();
1682 EmitCheck(Checked: std::make_pair(x&: IsFalse, y&: CheckOrdinal), Check: CheckHandler,
1683 StaticArgs: EmitCheckSourceLocation(Loc: FD->getLocation()), DynamicArgs: {});
1684 } else if (ShouldEmitUnreachable) {
1685 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1686 EmitTrapCall(IntrID: llvm::Intrinsic::trap);
1687 }
1688 if (SanOpts.has(K: SanitizerKind::Return) || ShouldEmitUnreachable) {
1689 Builder.CreateUnreachable();
1690 Builder.ClearInsertionPoint();
1691 }
1692 }
1693
1694 // Emit the standard function epilogue.
1695 FinishFunction(EndLoc: BodyRange.getEnd());
1696
1697 PGO->verifyCounterMap();
1698
1699 if (CurCodeDecl->hasAttr<PersonalityAttr>()) {
1700 StringRef Identifier =
1701 CurCodeDecl->getAttr<PersonalityAttr>()->getRoutine()->getName();
1702 llvm::FunctionCallee PersonalityRoutine =
1703 CGM.CreateRuntimeFunction(Ty: llvm::FunctionType::get(Result: CGM.Int32Ty, isVarArg: true),
1704 Name: Identifier, ExtraAttrs: {}, /*local=*/Local: true);
1705 Fn->setPersonalityFn(cast<llvm::Constant>(Val: PersonalityRoutine.getCallee()));
1706 }
1707
1708 // If we haven't marked the function nothrow through other means, do
1709 // a quick pass now to see if we can.
1710 if (!CurFn->doesNotThrow())
1711 TryMarkNoThrow(F: CurFn);
1712}
1713
1714/// ContainsLabel - Return true if the statement contains a label in it. If
1715/// this statement is not executed normally, it not containing a label means
1716/// that we can just remove the code.
1717bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
1718 // Null statement, not a label!
1719 if (!S) return false;
1720
1721 // If this is a label, we have to emit the code, consider something like:
1722 // if (0) { ... foo: bar(); } goto foo;
1723 //
1724 // TODO: If anyone cared, we could track __label__'s, since we know that you
1725 // can't jump to one from outside their declared region.
1726 if (isa<LabelStmt>(Val: S))
1727 return true;
1728
1729 // If this is a case/default statement, and we haven't seen a switch, we have
1730 // to emit the code.
1731 if (isa<SwitchCase>(Val: S) && !IgnoreCaseStmts)
1732 return true;
1733
1734 // If this is a switch statement, we want to ignore cases below it.
1735 if (isa<SwitchStmt>(Val: S))
1736 IgnoreCaseStmts = true;
1737
1738 // Scan subexpressions for verboten labels.
1739 for (const Stmt *SubStmt : S->children())
1740 if (ContainsLabel(S: SubStmt, IgnoreCaseStmts))
1741 return true;
1742
1743 return false;
1744}
1745
1746/// containsBreak - Return true if the statement contains a break out of it.
1747/// If the statement (recursively) contains a switch or loop with a break
1748/// inside of it, this is fine.
1749bool CodeGenFunction::containsBreak(const Stmt *S) {
1750 // Null statement, not a label!
1751 if (!S) return false;
1752
1753 // If this is a switch or loop that defines its own break scope, then we can
1754 // include it and anything inside of it.
1755 if (isa<SwitchStmt>(Val: S) || isa<WhileStmt>(Val: S) || isa<DoStmt>(Val: S) ||
1756 isa<ForStmt>(Val: S))
1757 return false;
1758
1759 if (isa<BreakStmt>(Val: S))
1760 return true;
1761
1762 // Scan subexpressions for verboten breaks.
1763 for (const Stmt *SubStmt : S->children())
1764 if (containsBreak(S: SubStmt))
1765 return true;
1766
1767 return false;
1768}
1769
1770bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) {
1771 if (!S) return false;
1772
1773 // Some statement kinds add a scope and thus never add a decl to the current
1774 // scope. Note, this list is longer than the list of statements that might
1775 // have an unscoped decl nested within them, but this way is conservatively
1776 // correct even if more statement kinds are added.
1777 if (isa<IfStmt>(Val: S) || isa<SwitchStmt>(Val: S) || isa<WhileStmt>(Val: S) ||
1778 isa<DoStmt>(Val: S) || isa<ForStmt>(Val: S) || isa<CompoundStmt>(Val: S) ||
1779 isa<CXXForRangeStmt>(Val: S) || isa<CXXTryStmt>(Val: S) ||
1780 isa<ObjCForCollectionStmt>(Val: S) || isa<ObjCAtTryStmt>(Val: S))
1781 return false;
1782
1783 if (isa<DeclStmt>(Val: S))
1784 return true;
1785
1786 for (const Stmt *SubStmt : S->children())
1787 if (mightAddDeclToScope(S: SubStmt))
1788 return true;
1789
1790 return false;
1791}
1792
1793/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1794/// to a constant, or if it does but contains a label, return false. If it
1795/// constant folds return true and set the boolean result in Result.
1796bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1797 bool &ResultBool,
1798 bool AllowLabels) {
1799 // If MC/DC is enabled, disable folding so that we can instrument all
1800 // conditions to yield complete test vectors. We still keep track of
1801 // folded conditions during region mapping and visualization.
1802 if (!AllowLabels && CGM.getCodeGenOpts().hasProfileClangInstr() &&
1803 CGM.getCodeGenOpts().MCDCCoverage)
1804 return false;
1805
1806 llvm::APSInt ResultInt;
1807 if (!ConstantFoldsToSimpleInteger(Cond, Result&: ResultInt, AllowLabels))
1808 return false;
1809
1810 ResultBool = ResultInt.getBoolValue();
1811 return true;
1812}
1813
1814/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1815/// to a constant, or if it does but contains a label, return false. If it
1816/// constant folds return true and set the folded value.
1817bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1818 llvm::APSInt &ResultInt,
1819 bool AllowLabels) {
1820 // FIXME: Rename and handle conversion of other evaluatable things
1821 // to bool.
1822 Expr::EvalResult Result;
1823 if (!Cond->EvaluateAsInt(Result, Ctx: getContext()))
1824 return false; // Not foldable, not integer or not fully evaluatable.
1825
1826 llvm::APSInt Int = Result.Val.getInt();
1827 if (!AllowLabels && CodeGenFunction::ContainsLabel(S: Cond))
1828 return false; // Contains a label.
1829
1830 PGO->markStmtMaybeUsed(S: Cond);
1831 ResultInt = std::move(Int);
1832 return true;
1833}
1834
1835/// Strip parentheses and simplistic logical-NOT operators.
1836const Expr *CodeGenFunction::stripCond(const Expr *C) {
1837 while (true) {
1838 const Expr *SC = IgnoreExprNodes(
1839 E: C, Fns&: IgnoreParensSingleStep, Fns&: IgnoreUOpLNotSingleStep,
1840 Fns&: IgnoreBuiltinExpectSingleStep, Fns&: IgnoreImplicitCastsSingleStep);
1841 if (C == SC)
1842 return SC;
1843 C = SC;
1844 }
1845}
1846
1847/// Determine whether the given condition is an instrumentable condition
1848/// (i.e. no "&&" or "||").
1849bool CodeGenFunction::isInstrumentedCondition(const Expr *C) {
1850 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: stripCond(C));
1851 return (!BOp || !BOp->isLogicalOp());
1852}
1853
1854/// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
1855/// increments a profile counter based on the semantics of the given logical
1856/// operator opcode. This is used to instrument branch condition coverage for
1857/// logical operators.
1858void CodeGenFunction::EmitBranchToCounterBlock(
1859 const Expr *Cond, BinaryOperator::Opcode LOp, llvm::BasicBlock *TrueBlock,
1860 llvm::BasicBlock *FalseBlock, uint64_t TrueCount /* = 0 */,
1861 Stmt::Likelihood LH /* =None */, const Expr *CntrIdx /* = nullptr */) {
1862 // If not instrumenting, just emit a branch.
1863 bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr();
1864 if (!InstrumentRegions || !isInstrumentedCondition(C: Cond))
1865 return EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount, LH);
1866
1867 const Stmt *CntrStmt = (CntrIdx ? CntrIdx : Cond);
1868
1869 llvm::BasicBlock *ThenBlock = nullptr;
1870 llvm::BasicBlock *ElseBlock = nullptr;
1871 llvm::BasicBlock *NextBlock = nullptr;
1872
1873 // Create the block we'll use to increment the appropriate counter.
1874 llvm::BasicBlock *CounterIncrBlock = createBasicBlock(name: "lop.rhscnt");
1875
1876 llvm::BasicBlock *SkipIncrBlock =
1877 (hasSkipCounter(S: CntrStmt) ? createBasicBlock(name: "lop.rhsskip") : nullptr);
1878 llvm::BasicBlock *SkipNextBlock = nullptr;
1879
1880 // Set block pointers according to Logical-AND (BO_LAnd) semantics. This
1881 // means we need to evaluate the condition and increment the counter on TRUE:
1882 //
1883 // if (Cond)
1884 // goto CounterIncrBlock;
1885 // else
1886 // goto FalseBlock;
1887 //
1888 // CounterIncrBlock:
1889 // Counter++;
1890 // goto TrueBlock;
1891
1892 if (LOp == BO_LAnd) {
1893 SkipNextBlock = FalseBlock;
1894 ThenBlock = CounterIncrBlock;
1895 ElseBlock = (SkipIncrBlock ? SkipIncrBlock : SkipNextBlock);
1896 NextBlock = TrueBlock;
1897 }
1898
1899 // Set block pointers according to Logical-OR (BO_LOr) semantics. This means
1900 // we need to evaluate the condition and increment the counter on FALSE:
1901 //
1902 // if (Cond)
1903 // goto TrueBlock;
1904 // else
1905 // goto CounterIncrBlock;
1906 //
1907 // CounterIncrBlock:
1908 // Counter++;
1909 // goto FalseBlock;
1910
1911 else if (LOp == BO_LOr) {
1912 SkipNextBlock = TrueBlock;
1913 ThenBlock = (SkipIncrBlock ? SkipIncrBlock : SkipNextBlock);
1914 ElseBlock = CounterIncrBlock;
1915 NextBlock = FalseBlock;
1916 } else {
1917 llvm_unreachable("Expected Opcode must be that of a Logical Operator");
1918 }
1919
1920 // Emit Branch based on condition.
1921 EmitBranchOnBoolExpr(Cond, TrueBlock: ThenBlock, FalseBlock: ElseBlock, TrueCount, LH);
1922
1923 if (SkipIncrBlock) {
1924 EmitBlock(BB: SkipIncrBlock);
1925 incrementProfileCounter(ExecSkip: UseSkipPath, S: CntrStmt);
1926 EmitBranch(Block: SkipNextBlock);
1927 }
1928
1929 // Emit the block containing the counter increment(s).
1930 EmitBlock(BB: CounterIncrBlock);
1931
1932 // Increment corresponding counter; if index not provided, use Cond as index.
1933 incrementProfileCounter(ExecSkip: UseExecPath, S: CntrStmt);
1934
1935 // Go to the next block.
1936 EmitBranch(Block: NextBlock);
1937}
1938
1939/// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an if
1940/// statement) to the specified blocks. Based on the condition, this might try
1941/// to simplify the codegen of the conditional based on the branch.
1942/// \param LH The value of the likelihood attribute on the True branch.
1943/// \param ConditionalOp Used by MC/DC code coverage to track the result of the
1944/// ConditionalOperator (ternary) through a recursive call for the operator's
1945/// LHS and RHS nodes.
1946void CodeGenFunction::EmitBranchOnBoolExpr(
1947 const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock,
1948 uint64_t TrueCount, Stmt::Likelihood LH, const Expr *ConditionalOp,
1949 const VarDecl *ConditionalDecl) {
1950 Cond = Cond->IgnoreParens();
1951
1952 if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Val: Cond)) {
1953 bool HasSkip = hasSkipCounter(S: CondBOp);
1954
1955 // Handle X && Y in a condition.
1956 if (CondBOp->getOpcode() == BO_LAnd) {
1957 // If we have "1 && X", simplify the code. "0 && X" would have constant
1958 // folded if the case was simple enough.
1959 bool ConstantBool = false;
1960 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getLHS(), ResultBool&: ConstantBool) &&
1961 ConstantBool) {
1962 // br(1 && X) -> br(X).
1963 incrementProfileCounter(S: CondBOp);
1964 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LAnd, TrueBlock,
1965 FalseBlock, TrueCount, LH);
1966 return;
1967 }
1968
1969 // If we have "X && 1", simplify the code to use an uncond branch.
1970 // "X && 0" would have been constant folded to 0.
1971 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getRHS(), ResultBool&: ConstantBool) &&
1972 ConstantBool) {
1973 // br(X && 1) -> br(X).
1974 EmitBranchToCounterBlock(Cond: CondBOp->getLHS(), LOp: BO_LAnd, TrueBlock,
1975 FalseBlock, TrueCount, LH, CntrIdx: CondBOp);
1976 return;
1977 }
1978
1979 // Emit the LHS as a conditional. If the LHS conditional is false, we
1980 // want to jump to the FalseBlock.
1981 llvm::BasicBlock *LHSTrue = createBasicBlock(name: "land.lhs.true");
1982 llvm::BasicBlock *LHSFalse =
1983 (HasSkip ? createBasicBlock(name: "land.lhsskip") : FalseBlock);
1984 // The counter tells us how often we evaluate RHS, and all of TrueCount
1985 // can be propagated to that branch.
1986 uint64_t RHSCount = getProfileCount(S: CondBOp->getRHS());
1987
1988 ConditionalEvaluation eval(*this);
1989 {
1990 ApplyDebugLocation DL(*this, Cond);
1991 // Propagate the likelihood attribute like __builtin_expect
1992 // __builtin_expect(X && Y, 1) -> X and Y are likely
1993 // __builtin_expect(X && Y, 0) -> only Y is unlikely
1994 EmitBranchOnBoolExpr(Cond: CondBOp->getLHS(), TrueBlock: LHSTrue, FalseBlock: LHSFalse, TrueCount: RHSCount,
1995 LH: LH == Stmt::LH_Unlikely ? Stmt::LH_None : LH);
1996 if (HasSkip) {
1997 EmitBlock(BB: LHSFalse);
1998 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondBOp);
1999 EmitBranch(Block: FalseBlock);
2000 }
2001 EmitBlock(BB: LHSTrue);
2002 }
2003
2004 incrementProfileCounter(ExecSkip: UseExecPath, S: CondBOp);
2005 setCurrentProfileCount(getProfileCount(S: CondBOp->getRHS()));
2006
2007 // Any temporaries created here are conditional.
2008 eval.begin(CGF&: *this);
2009 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LAnd, TrueBlock,
2010 FalseBlock, TrueCount, LH);
2011 eval.end(CGF&: *this);
2012 return;
2013 }
2014
2015 if (CondBOp->getOpcode() == BO_LOr) {
2016 // If we have "0 || X", simplify the code. "1 || X" would have constant
2017 // folded if the case was simple enough.
2018 bool ConstantBool = false;
2019 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getLHS(), ResultBool&: ConstantBool) &&
2020 !ConstantBool) {
2021 // br(0 || X) -> br(X).
2022 incrementProfileCounter(S: CondBOp);
2023 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LOr, TrueBlock,
2024 FalseBlock, TrueCount, LH);
2025 return;
2026 }
2027
2028 // If we have "X || 0", simplify the code to use an uncond branch.
2029 // "X || 1" would have been constant folded to 1.
2030 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getRHS(), ResultBool&: ConstantBool) &&
2031 !ConstantBool) {
2032 // br(X || 0) -> br(X).
2033 EmitBranchToCounterBlock(Cond: CondBOp->getLHS(), LOp: BO_LOr, TrueBlock,
2034 FalseBlock, TrueCount, LH, CntrIdx: CondBOp);
2035 return;
2036 }
2037 // Emit the LHS as a conditional. If the LHS conditional is true, we
2038 // want to jump to the TrueBlock.
2039 llvm::BasicBlock *LHSTrue =
2040 (HasSkip ? createBasicBlock(name: "lor.lhsskip") : TrueBlock);
2041 llvm::BasicBlock *LHSFalse = createBasicBlock(name: "lor.lhs.false");
2042 // We have the count for entry to the RHS and for the whole expression
2043 // being true, so we can divy up True count between the short circuit and
2044 // the RHS.
2045 uint64_t LHSCount =
2046 getCurrentProfileCount() - getProfileCount(S: CondBOp->getRHS());
2047 uint64_t RHSCount = TrueCount - LHSCount;
2048
2049 ConditionalEvaluation eval(*this);
2050 {
2051 // Propagate the likelihood attribute like __builtin_expect
2052 // __builtin_expect(X || Y, 1) -> only Y is likely
2053 // __builtin_expect(X || Y, 0) -> both X and Y are unlikely
2054 ApplyDebugLocation DL(*this, Cond);
2055 EmitBranchOnBoolExpr(Cond: CondBOp->getLHS(), TrueBlock: LHSTrue, FalseBlock: LHSFalse, TrueCount: LHSCount,
2056 LH: LH == Stmt::LH_Likely ? Stmt::LH_None : LH);
2057 if (HasSkip) {
2058 EmitBlock(BB: LHSTrue);
2059 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondBOp);
2060 EmitBranch(Block: TrueBlock);
2061 }
2062 EmitBlock(BB: LHSFalse);
2063 }
2064
2065 incrementProfileCounter(ExecSkip: UseExecPath, S: CondBOp);
2066 setCurrentProfileCount(getProfileCount(S: CondBOp->getRHS()));
2067
2068 // Any temporaries created here are conditional.
2069 eval.begin(CGF&: *this);
2070 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LOr, TrueBlock, FalseBlock,
2071 TrueCount: RHSCount, LH);
2072
2073 eval.end(CGF&: *this);
2074 return;
2075 }
2076 }
2077
2078 if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Val: Cond)) {
2079 // br(!x, t, f) -> br(x, f, t)
2080 // Avoid doing this optimization when instrumenting a condition for MC/DC.
2081 // LNot is taken as part of the condition for simplicity, and changing its
2082 // sense negatively impacts test vector tracking.
2083 bool MCDCCondition = CGM.getCodeGenOpts().hasProfileClangInstr() &&
2084 CGM.getCodeGenOpts().MCDCCoverage &&
2085 isInstrumentedCondition(C: Cond);
2086 if (CondUOp->getOpcode() == UO_LNot && !MCDCCondition) {
2087 // Negate the count.
2088 uint64_t FalseCount = getCurrentProfileCount() - TrueCount;
2089 // The values of the enum are chosen to make this negation possible.
2090 LH = static_cast<Stmt::Likelihood>(-LH);
2091 // Negate the condition and swap the destination blocks.
2092 return EmitBranchOnBoolExpr(Cond: CondUOp->getSubExpr(), TrueBlock: FalseBlock, FalseBlock: TrueBlock,
2093 TrueCount: FalseCount, LH);
2094 }
2095 }
2096
2097 if (const ConditionalOperator *CondOp = dyn_cast<ConditionalOperator>(Val: Cond)) {
2098 // br(c ? x : y, t, f) -> br(c, br(x, t, f), br(y, t, f))
2099 llvm::BasicBlock *LHSBlock = createBasicBlock(name: "cond.true");
2100 llvm::BasicBlock *RHSBlock = createBasicBlock(name: "cond.false");
2101
2102 // The ConditionalOperator itself has no likelihood information for its
2103 // true and false branches. This matches the behavior of __builtin_expect.
2104 ConditionalEvaluation cond(*this);
2105 EmitBranchOnBoolExpr(Cond: CondOp->getCond(), TrueBlock: LHSBlock, FalseBlock: RHSBlock,
2106 TrueCount: getProfileCount(S: CondOp), LH: Stmt::LH_None);
2107
2108 // When computing PGO branch weights, we only know the overall count for
2109 // the true block. This code is essentially doing tail duplication of the
2110 // naive code-gen, introducing new edges for which counts are not
2111 // available. Divide the counts proportionally between the LHS and RHS of
2112 // the conditional operator.
2113 uint64_t LHSScaledTrueCount = 0;
2114 if (TrueCount) {
2115 double LHSRatio =
2116 getProfileCount(S: CondOp) / (double)getCurrentProfileCount();
2117 LHSScaledTrueCount = TrueCount * LHSRatio;
2118 }
2119
2120 cond.begin(CGF&: *this);
2121 EmitBlock(BB: LHSBlock);
2122 incrementProfileCounter(ExecSkip: UseExecPath, S: CondOp);
2123 {
2124 ApplyDebugLocation DL(*this, Cond);
2125 EmitBranchOnBoolExpr(Cond: CondOp->getLHS(), TrueBlock, FalseBlock,
2126 TrueCount: LHSScaledTrueCount, LH, ConditionalOp: CondOp);
2127 }
2128 cond.end(CGF&: *this);
2129
2130 cond.begin(CGF&: *this);
2131 EmitBlock(BB: RHSBlock);
2132 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondOp);
2133 EmitBranchOnBoolExpr(Cond: CondOp->getRHS(), TrueBlock, FalseBlock,
2134 TrueCount: TrueCount - LHSScaledTrueCount, LH, ConditionalOp: CondOp);
2135 cond.end(CGF&: *this);
2136
2137 return;
2138 }
2139
2140 if (const CXXThrowExpr *Throw = dyn_cast<CXXThrowExpr>(Val: Cond)) {
2141 // Conditional operator handling can give us a throw expression as a
2142 // condition for a case like:
2143 // br(c ? throw x : y, t, f) -> br(c, br(throw x, t, f), br(y, t, f)
2144 // Fold this to:
2145 // br(c, throw x, br(y, t, f))
2146 EmitCXXThrowExpr(E: Throw, /*KeepInsertionPoint*/false);
2147 return;
2148 }
2149
2150 // Emit the code with the fully general case.
2151 llvm::Value *CondV;
2152 {
2153 ApplyDebugLocation DL(*this, Cond);
2154 CondV = EvaluateExprAsBool(E: Cond);
2155 }
2156
2157 MaybeEmitDeferredVarDeclInit(var: ConditionalDecl);
2158
2159 // If not at the top of the logical operator nest, update MCDC temp with the
2160 // boolean result of the evaluated condition.
2161 {
2162 const Expr *MCDCBaseExpr = Cond;
2163 // When a nested ConditionalOperator (ternary) is encountered in a boolean
2164 // expression, MC/DC tracks the result of the ternary, and this is tied to
2165 // the ConditionalOperator expression and not the ternary's LHS or RHS. If
2166 // this is the case, the ConditionalOperator expression is passed through
2167 // the ConditionalOp parameter and then used as the MCDC base expression.
2168 if (ConditionalOp)
2169 MCDCBaseExpr = ConditionalOp;
2170
2171 if (isMCDCBranchExpr(E: stripCond(C: MCDCBaseExpr)) &&
2172 !isMCDCDecisionExpr(E: stripCond(C: Cond)))
2173 maybeUpdateMCDCCondBitmap(E: MCDCBaseExpr, Val: CondV);
2174 }
2175
2176 llvm::MDNode *Weights = nullptr;
2177 llvm::MDNode *Unpredictable = nullptr;
2178
2179 // If the branch has a condition wrapped by __builtin_unpredictable,
2180 // create metadata that specifies that the branch is unpredictable.
2181 // Don't bother if not optimizing because that metadata would not be used.
2182 auto *Call = dyn_cast<CallExpr>(Val: Cond->IgnoreImpCasts());
2183 if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
2184 auto *FD = dyn_cast_or_null<FunctionDecl>(Val: Call->getCalleeDecl());
2185 if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
2186 llvm::MDBuilder MDHelper(getLLVMContext());
2187 Unpredictable = MDHelper.createUnpredictable();
2188 }
2189 }
2190
2191 // If there is a Likelihood knowledge for the cond, lower it.
2192 // Note that if not optimizing this won't emit anything.
2193 llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(Cond: CondV, LH);
2194 if (CondV != NewCondV)
2195 CondV = NewCondV;
2196 else {
2197 // Otherwise, lower profile counts. Note that we do this even at -O0.
2198 uint64_t CurrentCount = std::max(a: getCurrentProfileCount(), b: TrueCount);
2199 Weights = createProfileWeights(TrueCount, FalseCount: CurrentCount - TrueCount);
2200 }
2201
2202 llvm::Instruction *BrInst = Builder.CreateCondBr(Cond: CondV, True: TrueBlock, False: FalseBlock,
2203 BranchWeights: Weights, Unpredictable);
2204 addInstToNewSourceAtom(KeyInstruction: BrInst, Backup: CondV);
2205
2206 switch (HLSLControlFlowAttr) {
2207 case HLSLControlFlowHintAttr::Microsoft_branch:
2208 case HLSLControlFlowHintAttr::Microsoft_flatten: {
2209 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2210
2211 llvm::ConstantInt *BranchHintConstant =
2212 HLSLControlFlowAttr ==
2213 HLSLControlFlowHintAttr::Spelling::Microsoft_branch
2214 ? llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 1)
2215 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 2);
2216
2217 SmallVector<llvm::Metadata *, 2> Vals(
2218 {MDHelper.createString(Str: "hlsl.controlflow.hint"),
2219 MDHelper.createConstant(C: BranchHintConstant)});
2220 BrInst->setMetadata(Kind: "hlsl.controlflow.hint",
2221 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Vals));
2222 break;
2223 }
2224 // This is required to avoid warnings during compilation
2225 case HLSLControlFlowHintAttr::SpellingNotCalculated:
2226 break;
2227 }
2228}
2229
2230llvm::Value *CodeGenFunction::EmitScalarOrConstFoldImmArg(unsigned ICEArguments,
2231 unsigned Idx,
2232 const CallExpr *E) {
2233 llvm::Value *Arg = nullptr;
2234 if ((ICEArguments & (1 << Idx)) == 0) {
2235 Arg = EmitScalarExpr(E: E->getArg(Arg: Idx));
2236 } else {
2237 // If this is required to be a constant, constant fold it so that we
2238 // know that the generated intrinsic gets a ConstantInt.
2239 std::optional<llvm::APSInt> Result =
2240 E->getArg(Arg: Idx)->getIntegerConstantExpr(Ctx: getContext());
2241 assert(Result && "Expected argument to be a constant");
2242 Arg = llvm::ConstantInt::get(Context&: getLLVMContext(), V: *Result);
2243 }
2244 return Arg;
2245}
2246
2247/// ErrorUnsupported - Print out an error that codegen doesn't support the
2248/// specified stmt yet.
2249void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) {
2250 CGM.ErrorUnsupported(S, Type);
2251}
2252
2253/// emitNonZeroVLAInit - Emit the "zero" initialization of a
2254/// variable-length array whose elements have a non-zero bit-pattern.
2255///
2256/// \param baseType the inner-most element type of the array
2257/// \param src - a char* pointing to the bit-pattern for a single
2258/// base element of the array
2259/// \param sizeInChars - the total size of the VLA, in chars
2260static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
2261 Address dest, Address src,
2262 llvm::Value *sizeInChars) {
2263 CGBuilderTy &Builder = CGF.Builder;
2264
2265 CharUnits baseSize = CGF.getContext().getTypeSizeInChars(T: baseType);
2266 llvm::Value *baseSizeInChars
2267 = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: baseSize.getQuantity());
2268
2269 Address begin = dest.withElementType(ElemTy: CGF.Int8Ty);
2270 llvm::Value *end = Builder.CreateInBoundsGEP(Ty: begin.getElementType(),
2271 Ptr: begin.emitRawPointer(CGF),
2272 IdxList: sizeInChars, Name: "vla.end");
2273
2274 llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
2275 llvm::BasicBlock *loopBB = CGF.createBasicBlock(name: "vla-init.loop");
2276 llvm::BasicBlock *contBB = CGF.createBasicBlock(name: "vla-init.cont");
2277
2278 // Make a loop over the VLA. C99 guarantees that the VLA element
2279 // count must be nonzero.
2280 CGF.EmitBlock(BB: loopBB);
2281
2282 llvm::PHINode *cur = Builder.CreatePHI(Ty: begin.getType(), NumReservedValues: 2, Name: "vla.cur");
2283 cur->addIncoming(V: begin.emitRawPointer(CGF), BB: originBB);
2284
2285 CharUnits curAlign =
2286 dest.getAlignment().alignmentOfArrayElement(elementSize: baseSize);
2287
2288 // memcpy the individual element bit-pattern.
2289 Builder.CreateMemCpy(Dest: Address(cur, CGF.Int8Ty, curAlign), Src: src, Size: baseSizeInChars,
2290 /*volatile*/ IsVolatile: false);
2291
2292 // Go to the next element.
2293 llvm::Value *next =
2294 Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: cur, IdxList: baseSizeInChars, Name: "vla.next");
2295
2296 // Leave if that's the end of the VLA.
2297 llvm::Value *done = Builder.CreateICmpEQ(LHS: next, RHS: end, Name: "vla-init.isdone");
2298 Builder.CreateCondBr(Cond: done, True: contBB, False: loopBB);
2299 cur->addIncoming(V: next, BB: loopBB);
2300
2301 CGF.EmitBlock(BB: contBB);
2302}
2303
2304Address CodeGenFunction::EmitAddressOfPFPField(Address RecordPtr,
2305 const PFPField &Field) {
2306 return EmitAddressOfPFPField(
2307 RecordPtr,
2308 FieldPtr: Builder.CreateConstInBoundsByteGEP(Addr: RecordPtr.withElementType(ElemTy: Int8Ty),
2309 Offset: Field.Offset),
2310 Field: Field.Field);
2311}
2312
2313Address CodeGenFunction::EmitAddressOfPFPField(Address RecordPtr,
2314 Address PtrPtr,
2315 const FieldDecl *Field) {
2316 llvm::Value *Disc;
2317 if (CGM.getContext().arePFPFieldsTriviallyCopyable(RD: Field->getParent())) {
2318 uint64_t FieldSignature =
2319 llvm::getPointerAuthStableSipHash(S: CGM.getPFPFieldName(FD: Field));
2320 Disc = llvm::ConstantInt::get(Ty: CGM.Int64Ty, V: FieldSignature);
2321 } else
2322 Disc = Builder.CreatePtrToInt(V: RecordPtr.getBasePointer(), DestTy: CGM.Int64Ty);
2323
2324 llvm::GlobalValue *DS = CGM.getPFPDeactivationSymbol(FD: Field);
2325 llvm::OperandBundleDef DSBundle("deactivation-symbol", DS);
2326 llvm::Value *Args[] = {PtrPtr.getBasePointer(), Disc, Builder.getTrue()};
2327 return Address(
2328 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::protected_field_ptr,
2329 Tys: PtrPtr.getType()),
2330 Args, OpBundles: DSBundle),
2331 VoidPtrTy, PtrPtr.getAlignment());
2332}
2333
2334void
2335CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
2336 // Ignore empty classes in C++.
2337 if (getLangOpts().CPlusPlus)
2338 if (const auto *RD = Ty->getAsCXXRecordDecl(); RD && RD->isEmpty())
2339 return;
2340
2341 if (DestPtr.getElementType() != Int8Ty)
2342 DestPtr = DestPtr.withElementType(ElemTy: Int8Ty);
2343
2344 // Get size and alignment info for this aggregate.
2345 CharUnits size = getContext().getTypeSizeInChars(T: Ty);
2346
2347 llvm::Value *SizeVal;
2348 const VariableArrayType *vla;
2349
2350 // Don't bother emitting a zero-byte memset.
2351 if (size.isZero()) {
2352 // But note that getTypeInfo returns 0 for a VLA.
2353 if (const VariableArrayType *vlaType =
2354 dyn_cast_or_null<VariableArrayType>(
2355 Val: getContext().getAsArrayType(T: Ty))) {
2356 auto VlaSize = getVLASize(vla: vlaType);
2357 SizeVal = VlaSize.NumElts;
2358 CharUnits eltSize = getContext().getTypeSizeInChars(T: VlaSize.Type);
2359 if (!eltSize.isOne())
2360 SizeVal = Builder.CreateNUWMul(LHS: SizeVal, RHS: CGM.getSize(numChars: eltSize));
2361 vla = vlaType;
2362 } else {
2363 return;
2364 }
2365 } else {
2366 SizeVal = CGM.getSize(numChars: size);
2367 vla = nullptr;
2368 }
2369
2370 // If the type contains a pointer to data member we can't memset it to zero.
2371 // Instead, create a null constant and copy it to the destination.
2372 // TODO: there are other patterns besides zero that we can usefully memset,
2373 // like -1, which happens to be the pattern used by member-pointers.
2374 if (!CGM.getTypes().isZeroInitializable(T: Ty)) {
2375 // For a VLA, emit a single element, then splat that over the VLA.
2376 if (vla) Ty = getContext().getBaseElementType(VAT: vla);
2377
2378 llvm::Constant *NullConstant = CGM.EmitNullConstant(T: Ty);
2379
2380 llvm::GlobalVariable *NullVariable =
2381 new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
2382 /*isConstant=*/true,
2383 llvm::GlobalVariable::PrivateLinkage,
2384 NullConstant, Twine());
2385 CharUnits NullAlign = DestPtr.getAlignment();
2386 NullVariable->setAlignment(NullAlign.getAsAlign());
2387 Address SrcPtr(NullVariable, Builder.getInt8Ty(), NullAlign);
2388
2389 if (vla) return emitNonZeroVLAInit(CGF&: *this, baseType: Ty, dest: DestPtr, src: SrcPtr, sizeInChars: SizeVal);
2390
2391 // Get and call the appropriate llvm.memcpy overload.
2392 Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: SizeVal, IsVolatile: false);
2393 } else {
2394 // Otherwise, just memset the whole thing to zero. This is legal
2395 // because in LLVM, all default initializers (other than the ones we just
2396 // handled above, and the case handled below) are guaranteed to have a bit
2397 // pattern of all zeros.
2398 Builder.CreateMemSet(Dest: DestPtr, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
2399 }
2400
2401 // With the pointer field protection feature, null pointers do not have a bit
2402 // pattern of zero in memory, so we must initialize them separately.
2403 for (auto &Field : getContext().findPFPFields(Ty)) {
2404 auto addr = EmitAddressOfPFPField(RecordPtr: DestPtr, Field);
2405 Builder.CreateStore(Val: llvm::ConstantPointerNull::get(T: VoidPtrTy), Addr: addr);
2406 }
2407}
2408
2409llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
2410 // Make sure that there is a block for the indirect goto.
2411 if (!IndirectBranch)
2412 GetIndirectGotoBlock();
2413
2414 llvm::BasicBlock *BB = getJumpDestForLabel(S: L).getBlock();
2415
2416 // Make sure the indirect branch includes all of the address-taken blocks.
2417 IndirectBranch->addDestination(Dest: BB);
2418 return llvm::BlockAddress::get(Ty: CurFn->getType(), BB);
2419}
2420
2421llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {
2422 // If we already made the indirect branch for indirect goto, return its block.
2423 if (IndirectBranch) return IndirectBranch->getParent();
2424
2425 CGBuilderTy TmpBuilder(CGM, createBasicBlock(name: "indirectgoto"));
2426
2427 // Create the PHI node that indirect gotos will add entries to.
2428 llvm::Value *DestVal = TmpBuilder.CreatePHI(Ty: Int8PtrTy, NumReservedValues: 0,
2429 Name: "indirect.goto.dest");
2430
2431 // Create the indirect branch instruction.
2432 IndirectBranch = TmpBuilder.CreateIndirectBr(Addr: DestVal);
2433 return IndirectBranch->getParent();
2434}
2435
2436/// Computes the length of an array in elements, as well as the base
2437/// element type and a properly-typed first element pointer.
2438llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
2439 QualType &baseType,
2440 Address &addr) {
2441 const ArrayType *arrayType = origArrayType;
2442
2443 // If it's a VLA, we have to load the stored size. Note that
2444 // this is the size of the VLA in bytes, not its size in elements.
2445 llvm::Value *numVLAElements = nullptr;
2446 if (isa<VariableArrayType>(Val: arrayType)) {
2447 numVLAElements = getVLASize(vla: cast<VariableArrayType>(Val: arrayType)).NumElts;
2448
2449 // Walk into all VLAs. This doesn't require changes to addr,
2450 // which has type T* where T is the first non-VLA element type.
2451 do {
2452 QualType elementType = arrayType->getElementType();
2453 arrayType = getContext().getAsArrayType(T: elementType);
2454
2455 // If we only have VLA components, 'addr' requires no adjustment.
2456 if (!arrayType) {
2457 baseType = elementType;
2458 return numVLAElements;
2459 }
2460 } while (isa<VariableArrayType>(Val: arrayType));
2461
2462 // We get out here only if we find a constant array type
2463 // inside the VLA.
2464 }
2465
2466 // We have some number of constant-length arrays, so addr should
2467 // have LLVM type [M x [N x [...]]]*. Build a GEP that walks
2468 // down to the first element of addr.
2469 SmallVector<llvm::Value*, 8> gepIndices;
2470
2471 // GEP down to the array type.
2472 llvm::ConstantInt *zero = Builder.getInt32(C: 0);
2473 gepIndices.push_back(Elt: zero);
2474
2475 uint64_t countFromCLAs = 1;
2476 QualType eltType;
2477
2478 llvm::ArrayType *llvmArrayType =
2479 dyn_cast<llvm::ArrayType>(Val: addr.getElementType());
2480 while (llvmArrayType) {
2481 assert(isa<ConstantArrayType>(arrayType));
2482 assert(cast<ConstantArrayType>(arrayType)->getZExtSize() ==
2483 llvmArrayType->getNumElements());
2484
2485 gepIndices.push_back(Elt: zero);
2486 countFromCLAs *= llvmArrayType->getNumElements();
2487 eltType = arrayType->getElementType();
2488
2489 llvmArrayType =
2490 dyn_cast<llvm::ArrayType>(Val: llvmArrayType->getElementType());
2491 arrayType = getContext().getAsArrayType(T: arrayType->getElementType());
2492 assert((!llvmArrayType || arrayType) &&
2493 "LLVM and Clang types are out-of-synch");
2494 }
2495
2496 if (arrayType) {
2497 // From this point onwards, the Clang array type has been emitted
2498 // as some other type (probably a packed struct). Compute the array
2499 // size, and just emit the 'begin' expression as a bitcast.
2500 while (arrayType) {
2501 countFromCLAs *= cast<ConstantArrayType>(Val: arrayType)->getZExtSize();
2502 eltType = arrayType->getElementType();
2503 arrayType = getContext().getAsArrayType(T: eltType);
2504 }
2505
2506 llvm::Type *baseType = ConvertType(T: eltType);
2507 addr = addr.withElementType(ElemTy: baseType);
2508 } else {
2509 // Create the actual GEP.
2510 addr = Address(Builder.CreateInBoundsGEP(Ty: addr.getElementType(),
2511 Ptr: addr.emitRawPointer(CGF&: *this),
2512 IdxList: gepIndices, Name: "array.begin"),
2513 ConvertTypeForMem(T: eltType), addr.getAlignment());
2514 }
2515
2516 baseType = eltType;
2517
2518 llvm::Value *numElements
2519 = llvm::ConstantInt::get(Ty: SizeTy, V: countFromCLAs);
2520
2521 // If we had any VLA dimensions, factor them in.
2522 if (numVLAElements)
2523 numElements = Builder.CreateNUWMul(LHS: numVLAElements, RHS: numElements);
2524
2525 return numElements;
2526}
2527
2528CodeGenFunction::VlaSizePair CodeGenFunction::getVLASize(QualType type) {
2529 const VariableArrayType *vla = getContext().getAsVariableArrayType(T: type);
2530 assert(vla && "type was not a variable array type!");
2531 return getVLASize(vla);
2532}
2533
2534CodeGenFunction::VlaSizePair
2535CodeGenFunction::getVLASize(const VariableArrayType *type) {
2536 // The number of elements so far; always size_t.
2537 llvm::Value *numElements = nullptr;
2538
2539 QualType elementType;
2540 do {
2541 elementType = type->getElementType();
2542 llvm::Value *vlaSize = VLASizeMap[type->getSizeExpr()];
2543 assert(vlaSize && "no size for VLA!");
2544 assert(vlaSize->getType() == SizeTy);
2545
2546 if (!numElements) {
2547 numElements = vlaSize;
2548 } else {
2549 // It's undefined behavior if this wraps around, so mark it that way.
2550 // FIXME: Teach -fsanitize=undefined to trap this.
2551 numElements = Builder.CreateNUWMul(LHS: numElements, RHS: vlaSize);
2552 }
2553 } while ((type = getContext().getAsVariableArrayType(T: elementType)));
2554
2555 return { numElements, elementType };
2556}
2557
2558CodeGenFunction::VlaSizePair
2559CodeGenFunction::getVLAElements1D(QualType type) {
2560 const VariableArrayType *vla = getContext().getAsVariableArrayType(T: type);
2561 assert(vla && "type was not a variable array type!");
2562 return getVLAElements1D(vla);
2563}
2564
2565CodeGenFunction::VlaSizePair
2566CodeGenFunction::getVLAElements1D(const VariableArrayType *Vla) {
2567 llvm::Value *VlaSize = VLASizeMap[Vla->getSizeExpr()];
2568 assert(VlaSize && "no size for VLA!");
2569 assert(VlaSize->getType() == SizeTy);
2570 return { VlaSize, Vla->getElementType() };
2571}
2572
2573void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
2574 assert(type->isVariablyModifiedType() &&
2575 "Must pass variably modified type to EmitVLASizes!");
2576
2577 EnsureInsertPoint();
2578
2579 // We're going to walk down into the type and look for VLA
2580 // expressions.
2581 do {
2582 assert(type->isVariablyModifiedType());
2583
2584 const Type *ty = type.getTypePtr();
2585 switch (ty->getTypeClass()) {
2586
2587#define TYPE(Class, Base)
2588#define ABSTRACT_TYPE(Class, Base)
2589#define NON_CANONICAL_TYPE(Class, Base)
2590#define DEPENDENT_TYPE(Class, Base) case Type::Class:
2591#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
2592#include "clang/AST/TypeNodes.inc"
2593 llvm_unreachable("unexpected dependent type!");
2594
2595 // These types are never variably-modified.
2596 case Type::Builtin:
2597 case Type::Complex:
2598 case Type::Vector:
2599 case Type::ExtVector:
2600 case Type::ConstantMatrix:
2601 case Type::Record:
2602 case Type::Enum:
2603 case Type::Using:
2604 case Type::TemplateSpecialization:
2605 case Type::ObjCTypeParam:
2606 case Type::ObjCObject:
2607 case Type::ObjCInterface:
2608 case Type::ObjCObjectPointer:
2609 case Type::BitInt:
2610 case Type::HLSLInlineSpirv:
2611 case Type::PredefinedSugar:
2612 llvm_unreachable("type class is never variably-modified!");
2613
2614 case Type::Adjusted:
2615 type = cast<AdjustedType>(Val: ty)->getAdjustedType();
2616 break;
2617
2618 case Type::Decayed:
2619 type = cast<DecayedType>(Val: ty)->getPointeeType();
2620 break;
2621
2622 case Type::Pointer:
2623 type = cast<PointerType>(Val: ty)->getPointeeType();
2624 break;
2625
2626 case Type::BlockPointer:
2627 type = cast<BlockPointerType>(Val: ty)->getPointeeType();
2628 break;
2629
2630 case Type::LValueReference:
2631 case Type::RValueReference:
2632 type = cast<ReferenceType>(Val: ty)->getPointeeType();
2633 break;
2634
2635 case Type::MemberPointer:
2636 type = cast<MemberPointerType>(Val: ty)->getPointeeType();
2637 break;
2638
2639 case Type::ArrayParameter:
2640 case Type::ConstantArray:
2641 case Type::IncompleteArray:
2642 // Losing element qualification here is fine.
2643 type = cast<ArrayType>(Val: ty)->getElementType();
2644 break;
2645
2646 case Type::VariableArray: {
2647 // Losing element qualification here is fine.
2648 const VariableArrayType *vat = cast<VariableArrayType>(Val: ty);
2649
2650 // Unknown size indication requires no size computation.
2651 // Otherwise, evaluate and record it.
2652 if (const Expr *sizeExpr = vat->getSizeExpr()) {
2653 // It's possible that we might have emitted this already,
2654 // e.g. with a typedef and a pointer to it.
2655 llvm::Value *&entry = VLASizeMap[sizeExpr];
2656 if (!entry) {
2657 llvm::Value *size = EmitScalarExpr(E: sizeExpr);
2658
2659 // C11 6.7.6.2p5:
2660 // If the size is an expression that is not an integer constant
2661 // expression [...] each time it is evaluated it shall have a value
2662 // greater than zero.
2663 if (SanOpts.has(K: SanitizerKind::VLABound)) {
2664 auto CheckOrdinal = SanitizerKind::SO_VLABound;
2665 auto CheckHandler = SanitizerHandler::VLABoundNotPositive;
2666 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2667 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: size->getType());
2668 clang::QualType SEType = sizeExpr->getType();
2669 llvm::Value *CheckCondition =
2670 SEType->isSignedIntegerType()
2671 ? Builder.CreateICmpSGT(LHS: size, RHS: Zero)
2672 : Builder.CreateICmpUGT(LHS: size, RHS: Zero);
2673 llvm::Constant *StaticArgs[] = {
2674 EmitCheckSourceLocation(Loc: sizeExpr->getBeginLoc()),
2675 EmitCheckTypeDescriptor(T: SEType)};
2676 EmitCheck(Checked: std::make_pair(x&: CheckCondition, y&: CheckOrdinal),
2677 Check: CheckHandler, StaticArgs, DynamicArgs: size);
2678 }
2679
2680 // Always zexting here would be wrong if it weren't
2681 // undefined behavior to have a negative bound.
2682 // FIXME: What about when size's type is larger than size_t?
2683 entry = Builder.CreateIntCast(V: size, DestTy: SizeTy, /*signed*/ isSigned: false);
2684 }
2685 }
2686 type = vat->getElementType();
2687 break;
2688 }
2689
2690 case Type::FunctionProto:
2691 case Type::FunctionNoProto:
2692 type = cast<FunctionType>(Val: ty)->getReturnType();
2693 break;
2694
2695 case Type::Paren:
2696 case Type::TypeOf:
2697 case Type::UnaryTransform:
2698 case Type::Attributed:
2699 case Type::BTFTagAttributed:
2700 case Type::OverflowBehavior:
2701 case Type::HLSLAttributedResource:
2702 case Type::SubstTemplateTypeParm:
2703 case Type::MacroQualified:
2704 case Type::CountAttributed:
2705 case Type::LateParsedAttr:
2706 // Keep walking after single level desugaring.
2707 type = type.getSingleStepDesugaredType(Context: getContext());
2708 break;
2709
2710 case Type::Typedef:
2711 case Type::Decltype:
2712 case Type::Auto:
2713 case Type::DeducedTemplateSpecialization:
2714 case Type::PackIndexing:
2715 // Stop walking: nothing to do.
2716 return;
2717
2718 case Type::TypeOfExpr:
2719 // Stop walking: emit typeof expression.
2720 EmitIgnoredExpr(E: cast<TypeOfExprType>(Val: ty)->getUnderlyingExpr());
2721 return;
2722
2723 case Type::Atomic:
2724 type = cast<AtomicType>(Val: ty)->getValueType();
2725 break;
2726
2727 case Type::Pipe:
2728 type = cast<PipeType>(Val: ty)->getElementType();
2729 break;
2730 }
2731 } while (type->isVariablyModifiedType());
2732}
2733
2734Address CodeGenFunction::EmitVAListRef(const Expr* E) {
2735 if (getContext().getBuiltinVaListType()->isArrayType())
2736 return EmitPointerWithAlignment(Addr: E);
2737 return EmitLValue(E).getAddress();
2738}
2739
2740Address CodeGenFunction::EmitMSVAListRef(const Expr *E) {
2741 return EmitLValue(E).getAddress();
2742}
2743
2744Address CodeGenFunction::EmitZOSVAListRef(const Expr *E) {
2745 return EmitPointerWithAlignment(Addr: E);
2746}
2747
2748void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E,
2749 const APValue &Init) {
2750 assert(Init.hasValue() && "Invalid DeclRefExpr initializer!");
2751 if (CGDebugInfo *Dbg = getDebugInfo())
2752 if (CGM.getCodeGenOpts().hasReducedDebugInfo())
2753 Dbg->EmitGlobalVariable(VD: E->getDecl(), Init);
2754}
2755
2756CodeGenFunction::PeepholeProtection
2757CodeGenFunction::protectFromPeepholes(RValue rvalue) {
2758 // At the moment, the only aggressive peephole we do in IR gen
2759 // is trunc(zext) folding, but if we add more, we can easily
2760 // extend this protection.
2761
2762 if (!rvalue.isScalar()) return PeepholeProtection();
2763 llvm::Value *value = rvalue.getScalarVal();
2764 if (!isa<llvm::ZExtInst>(Val: value)) return PeepholeProtection();
2765
2766 // Just make an extra bitcast.
2767 assert(HaveInsertPoint());
2768 llvm::Instruction *inst = new llvm::BitCastInst(value, value->getType(), "",
2769 Builder.GetInsertBlock());
2770
2771 PeepholeProtection protection;
2772 protection.Inst = inst;
2773 return protection;
2774}
2775
2776void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {
2777 if (!protection.Inst) return;
2778
2779 // In theory, we could try to duplicate the peepholes now, but whatever.
2780 protection.Inst->eraseFromParent();
2781}
2782
2783void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2784 QualType Ty, SourceLocation Loc,
2785 SourceLocation AssumptionLoc,
2786 llvm::Value *Alignment,
2787 llvm::Value *OffsetValue) {
2788 if (Alignment->getType() != IntPtrTy)
2789 Alignment =
2790 Builder.CreateIntCast(V: Alignment, DestTy: IntPtrTy, isSigned: false, Name: "casted.align");
2791 if (OffsetValue && OffsetValue->getType() != IntPtrTy)
2792 OffsetValue =
2793 Builder.CreateIntCast(V: OffsetValue, DestTy: IntPtrTy, isSigned: true, Name: "casted.offset");
2794 if (SanOpts.has(K: SanitizerKind::Alignment))
2795 emitAlignmentAssumptionCheck(Ptr: PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2796 OffsetValue);
2797 Builder.CreateAlignmentAssumption(DL: CGM.getDataLayout(), PtrValue, Alignment,
2798 OffsetValue);
2799}
2800
2801void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2802 const Expr *E,
2803 SourceLocation AssumptionLoc,
2804 llvm::Value *Alignment,
2805 llvm::Value *OffsetValue) {
2806 QualType Ty = E->getType();
2807 SourceLocation Loc = E->getExprLoc();
2808
2809 emitAlignmentAssumption(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2810 OffsetValue);
2811}
2812
2813llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn,
2814 llvm::Value *AnnotatedVal,
2815 StringRef AnnotationStr,
2816 SourceLocation Location,
2817 const AnnotateAttr *Attr) {
2818 SmallVector<llvm::Value *, 5> Args = {
2819 AnnotatedVal,
2820 CGM.EmitAnnotationString(Str: AnnotationStr),
2821 CGM.EmitAnnotationUnit(Loc: Location),
2822 CGM.EmitAnnotationLineNo(L: Location),
2823 };
2824 if (Attr)
2825 Args.push_back(Elt: CGM.EmitAnnotationArgs(Attr));
2826 return Builder.CreateCall(Callee: AnnotationFn, Args);
2827}
2828
2829void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
2830 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2831 for (const auto *I : D->specific_attrs<AnnotateAttr>())
2832 EmitAnnotationCall(AnnotationFn: CGM.getIntrinsic(IID: llvm::Intrinsic::var_annotation,
2833 Tys: {V->getType(), CGM.ConstGlobalsPtrTy}),
2834 AnnotatedVal: V, AnnotationStr: I->getAnnotation(), Location: D->getLocation(), Attr: I);
2835}
2836
2837Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
2838 Address Addr) {
2839 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2840 llvm::Value *V = Addr.emitRawPointer(CGF&: *this);
2841 llvm::Type *VTy = V->getType();
2842 auto *PTy = dyn_cast<llvm::PointerType>(Val: VTy);
2843 unsigned AS = PTy ? PTy->getAddressSpace() : 0;
2844 llvm::PointerType *IntrinTy =
2845 llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: AS);
2846 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::ptr_annotation,
2847 Tys: {IntrinTy, CGM.ConstGlobalsPtrTy});
2848
2849 for (const auto *I : D->specific_attrs<AnnotateAttr>()) {
2850 // FIXME Always emit the cast inst so we can differentiate between
2851 // annotation on the first field of a struct and annotation on the struct
2852 // itself.
2853 if (VTy != IntrinTy)
2854 V = Builder.CreateBitCast(V, DestTy: IntrinTy);
2855 V = EmitAnnotationCall(AnnotationFn: F, AnnotatedVal: V, AnnotationStr: I->getAnnotation(), Location: D->getLocation(), Attr: I);
2856 V = Builder.CreateBitCast(V, DestTy: VTy);
2857 }
2858
2859 return Address(V, Addr.getElementType(), Addr.getAlignment());
2860}
2861
2862CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
2863
2864CodeGenFunction::SanitizerScope::SanitizerScope(CodeGenFunction *CGF)
2865 : CGF(CGF) {
2866 assert(!CGF->IsSanitizerScope);
2867 CGF->IsSanitizerScope = true;
2868}
2869
2870CodeGenFunction::SanitizerScope::~SanitizerScope() {
2871 CGF->IsSanitizerScope = false;
2872}
2873
2874void CodeGenFunction::InsertHelper(llvm::Instruction *I,
2875 const llvm::Twine &Name,
2876 llvm::BasicBlock::iterator InsertPt) const {
2877 LoopStack.InsertHelper(I);
2878 if (IsSanitizerScope)
2879 I->setNoSanitizeMetadata();
2880}
2881
2882void CGBuilderInserter::InsertHelper(
2883 llvm::Instruction *I, const llvm::Twine &Name,
2884 llvm::BasicBlock::iterator InsertPt) const {
2885 llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, InsertPt);
2886 if (CGF)
2887 CGF->InsertHelper(I, Name, InsertPt);
2888}
2889
2890// Emits an error if we don't have a valid set of target features for the
2891// called function.
2892void CodeGenFunction::checkTargetFeatures(const CallExpr *E,
2893 const FunctionDecl *TargetDecl) {
2894 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
2895 CodeGenUtils::checkTargetFeatures(Ctx&: CGM.getContext(), Diags&: CGM.getDiags(),
2896 LangOpts: getLangOpts(), E, Caller: FD, TargetDecl);
2897}
2898
2899// Emits an error if we don't have a valid set of target features for the
2900// called function.
2901void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
2902 const FunctionDecl *TargetDecl) {
2903 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
2904 CodeGenUtils::checkTargetFeatures(Ctx&: CGM.getContext(), Diags&: CGM.getDiags(),
2905 LangOpts: getLangOpts(), Loc, Caller: FD, TargetDecl);
2906}
2907
2908void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
2909 if (!CGM.getCodeGenOpts().SanitizeStats)
2910 return;
2911
2912 llvm::IRBuilder<> IRB(Builder.GetInsertPoint());
2913 IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation());
2914 CGM.getSanStats().create(B&: IRB, SK: SSK);
2915}
2916
2917void CodeGenFunction::EmitKCFIOperandBundle(
2918 const CGCallee &Callee, SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
2919 const CGCalleeInfo &CI = Callee.getAbstractInfo();
2920 const FunctionProtoType *FP = CI.getCalleeFunctionProtoType();
2921 if (!FP)
2922 return;
2923
2924 StringRef Salt;
2925 if (const auto &Info = FP->getExtraAttributeInfo())
2926 Salt = Info.CFISalt;
2927
2928 Bundles.emplace_back(Args: "kcfi", Args: CGM.CreateKCFITypeId(T: FP->desugar(), Salt));
2929}
2930
2931llvm::Value *
2932CodeGenFunction::FormAArch64ResolverCondition(const FMVResolverOption &RO) {
2933 return RO.Features.empty() ? nullptr : EmitAArch64CpuSupports(FeatureStrs: RO.Features);
2934}
2935
2936llvm::Value *
2937CodeGenFunction::FormX86ResolverCondition(const FMVResolverOption &RO) {
2938 llvm::Value *Condition = nullptr;
2939
2940 if (RO.Architecture) {
2941 StringRef Arch = *RO.Architecture;
2942 // If arch= specifies an x86-64 micro-architecture level, test the feature
2943 // with __builtin_cpu_supports, otherwise use __builtin_cpu_is.
2944 if (Arch.starts_with(Prefix: "x86-64"))
2945 Condition = EmitX86CpuSupports(FeatureStrs: {Arch});
2946 else
2947 Condition = EmitX86CpuIs(CPUStr: Arch);
2948 }
2949
2950 if (!RO.Features.empty()) {
2951 llvm::Value *FeatureCond = EmitX86CpuSupports(FeatureStrs: RO.Features);
2952 Condition =
2953 Condition ? Builder.CreateAnd(LHS: Condition, RHS: FeatureCond) : FeatureCond;
2954 }
2955 return Condition;
2956}
2957
2958static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
2959 llvm::Function *Resolver,
2960 CGBuilderTy &Builder,
2961 llvm::Function *FuncToReturn,
2962 bool SupportsIFunc) {
2963 if (SupportsIFunc) {
2964 Builder.CreateRet(V: FuncToReturn);
2965 return;
2966 }
2967
2968 llvm::SmallVector<llvm::Value *, 10> Args(
2969 llvm::make_pointer_range(Range: Resolver->args()));
2970
2971 llvm::CallInst *Result = Builder.CreateCall(Callee: FuncToReturn, Args);
2972 Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
2973
2974 if (Resolver->getReturnType()->isVoidTy())
2975 Builder.CreateRetVoid();
2976 else
2977 Builder.CreateRet(V: Result);
2978}
2979
2980void CodeGenFunction::EmitMultiVersionResolver(
2981 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
2982 llvm::SaveAndRestore<llvm::Function *> savedCurFn(CurFn, Resolver);
2983 llvm::Triple::ArchType ArchType =
2984 getContext().getTargetInfo().getTriple().getArch();
2985
2986 switch (ArchType) {
2987 case llvm::Triple::x86:
2988 case llvm::Triple::x86_64:
2989 EmitX86MultiVersionResolver(Resolver, Options);
2990 return;
2991 case llvm::Triple::aarch64:
2992 EmitAArch64MultiVersionResolver(Resolver, Options);
2993 return;
2994 case llvm::Triple::riscv32:
2995 case llvm::Triple::riscv64:
2996 case llvm::Triple::riscv32be:
2997 case llvm::Triple::riscv64be:
2998 EmitRISCVMultiVersionResolver(Resolver, Options);
2999 return;
3000 case llvm::Triple::ppc:
3001 case llvm::Triple::ppc64:
3002 if (getContext().getTargetInfo().getTriple().isOSAIX()) {
3003 EmitPPCAIXMultiVersionResolver(Resolver, Options);
3004 return;
3005 }
3006 [[fallthrough]];
3007 default:
3008 assert(false &&
3009 "Only implemented for x86, AArch64, RISC-V, and PowerPC AIX");
3010 }
3011}
3012
3013/**
3014 * define internal ptr @foo.resolver() {
3015 * entry:
3016 * %is_version_1 = __builtin_cpu_supports(version_1)
3017 * br i1 %1, label %if.version_1, label %if.else_2
3018 *
3019 * if.version_1:
3020 * ret ptr @foo.version_1
3021 *
3022 * if.else_2:
3023 * %is_version_2 = __builtin_cpu_supports(version_2)
3024 * ...
3025 * if.else: ; preds = %entry
3026 * ret ptr @foo.default
3027 * }
3028 */
3029void CodeGenFunction::EmitPPCAIXMultiVersionResolver(
3030 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3031
3032 // entry:
3033 llvm::BasicBlock *CurBlock = createBasicBlock(name: "entry", parent: Resolver);
3034
3035 for (const FMVResolverOption &RO : Options) {
3036 Builder.SetInsertPoint(CurBlock);
3037 // The 'default' or 'generic' case.
3038 if (!RO.Architecture && RO.Features.empty()) {
3039 // if.else:
3040 // ret ptr @foo.default
3041 assert(&RO == Options.end() - 1 &&
3042 "Default or Generic case must be last");
3043 Builder.CreateRet(V: RO.Function);
3044 return;
3045 }
3046 // if.else_n:
3047 // %is_version_n = __builtin_cpu_supports(version_n)
3048 // br i1 %is_version_n, label %if.version_n, label %if.else_n+1
3049 //
3050 // if.version_n:
3051 // ret ptr @foo_version_n
3052 assert(RO.Features.size() == 1 &&
3053 "for now one feature requirement per version");
3054
3055 StringRef FeatureStr = RO.Features[0];
3056 StringRef BuiltinCpuSupportsArg;
3057 bool IsNegated = false;
3058
3059 if (FeatureStr.starts_with(Prefix: "cpu=")) {
3060 // CPU specification - map to ISA level
3061 StringRef CPU = FeatureStr.split(Separator: "=").second.trim();
3062 BuiltinCpuSupportsArg = llvm::StringSwitch<StringRef>(CPU)
3063#define PPC_AIX_CLONES_CPU(CPU_NAME, AIX_BUILTIN_CPU_SUPPORTS_NAME, _) \
3064 .Case(CPU_NAME, AIX_BUILTIN_CPU_SUPPORTS_NAME)
3065#include "llvm/TargetParser/PPCTargetParser.def"
3066 .Default(Value: "error");
3067 } else {
3068 // Feature strings arrive here already normalized:
3069 // - Positive features: just the name (e.g., "altivec")
3070 // - Negated features: "no-" prefix (e.g., "no-altivec")
3071 if (FeatureStr.starts_with(Prefix: "no-")) {
3072 IsNegated = true;
3073 FeatureStr = FeatureStr.drop_front(N: 3);
3074 }
3075
3076 // Map feature names to __builtin_cpu_supports() strings
3077 BuiltinCpuSupportsArg =
3078 llvm::StringSwitch<StringRef>(FeatureStr)
3079#define PPC_AIX_CLONES_FEATURE(FEATURE_NAME, AIX_BUILTIN_CPU_SUPPORTS_NAME, _, \
3080 __) \
3081 .Case(FEATURE_NAME, AIX_BUILTIN_CPU_SUPPORTS_NAME)
3082#include "llvm/TargetParser/PPCTargetParser.def"
3083 // Features without runtime checks return empty string
3084 .Default(Value: "");
3085
3086 // All features in target_clones must have runtime detection
3087 assert(!BuiltinCpuSupportsArg.empty() &&
3088 "Feature without runtime detection should have been rejected in "
3089 "Sema");
3090 }
3091
3092 assert(getContext().getTargetInfo().validateCpuSupports(
3093 BuiltinCpuSupportsArg));
3094
3095 llvm::Value *Condition =
3096 EmitPPCBuiltinCpu(BuiltinID: Builtin::BI__builtin_cpu_supports,
3097 ReturnType: Builder.getInt1Ty(), CPUStr: BuiltinCpuSupportsArg);
3098
3099 // Negate the condition if this is a negated feature
3100 if (IsNegated) {
3101 Condition = Builder.CreateNot(V: Condition, Name: "neg");
3102 }
3103
3104 llvm::BasicBlock *ThenBlock = createBasicBlock(name: "if.version", parent: Resolver);
3105 CurBlock = createBasicBlock(name: "if.else", parent: Resolver);
3106 Builder.CreateCondBr(Cond: Condition, True: ThenBlock, False: CurBlock);
3107
3108 Builder.SetInsertPoint(ThenBlock);
3109 Builder.CreateRet(V: RO.Function);
3110 }
3111
3112 llvm_unreachable("Default case missing");
3113}
3114
3115void CodeGenFunction::EmitRISCVMultiVersionResolver(
3116 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3117
3118 if (getContext().getTargetInfo().getTriple().getOS() !=
3119 llvm::Triple::OSType::Linux) {
3120 CGM.getDiags().Report(DiagID: diag::err_os_unsupport_riscv_fmv);
3121 return;
3122 }
3123
3124 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3125 Builder.SetInsertPoint(CurBlock);
3126 EmitRISCVCpuInit();
3127
3128 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3129 bool HasDefault = false;
3130 unsigned DefaultIndex = 0;
3131
3132 // Check the each candidate function.
3133 for (unsigned Index = 0; Index < Options.size(); Index++) {
3134
3135 if (Options[Index].Features.empty()) {
3136 HasDefault = true;
3137 DefaultIndex = Index;
3138 continue;
3139 }
3140
3141 Builder.SetInsertPoint(CurBlock);
3142
3143 // FeaturesCondition: The bitmask of the required extension has been
3144 // enabled by the runtime object.
3145 // (__riscv_feature_bits.features[i] & REQUIRED_BITMASK) ==
3146 // REQUIRED_BITMASK
3147 //
3148 // When condition is met, return this version of the function.
3149 // Otherwise, try the next version.
3150 //
3151 // if (FeaturesConditionVersion1)
3152 // return Version1;
3153 // else if (FeaturesConditionVersion2)
3154 // return Version2;
3155 // else if (FeaturesConditionVersion3)
3156 // return Version3;
3157 // ...
3158 // else
3159 // return DefaultVersion;
3160
3161 // TODO: Add a condition to check the length before accessing elements.
3162 // Without checking the length first, we may access an incorrect memory
3163 // address when using different versions.
3164 llvm::SmallVector<StringRef, 8> CurrTargetAttrFeats;
3165 llvm::SmallVector<std::string, 8> TargetAttrFeats;
3166
3167 for (StringRef Feat : Options[Index].Features) {
3168 std::vector<std::string> FeatStr =
3169 getContext().getTargetInfo().parseTargetAttr(Str: Feat).Features;
3170
3171 assert(FeatStr.size() == 1 && "Feature string not delimited");
3172
3173 std::string &CurrFeat = FeatStr.front();
3174 if (CurrFeat[0] == '+')
3175 TargetAttrFeats.push_back(Elt: CurrFeat.substr(pos: 1));
3176 }
3177
3178 if (TargetAttrFeats.empty())
3179 continue;
3180
3181 for (std::string &Feat : TargetAttrFeats)
3182 CurrTargetAttrFeats.push_back(Elt: Feat);
3183
3184 Builder.SetInsertPoint(CurBlock);
3185 llvm::Value *FeatsCondition = EmitRISCVCpuSupports(FeaturesStrs: CurrTargetAttrFeats);
3186
3187 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3188 CGBuilderTy RetBuilder(CGM, RetBlock);
3189 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder,
3190 FuncToReturn: Options[Index].Function, SupportsIFunc);
3191 llvm::BasicBlock *ElseBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3192
3193 Builder.SetInsertPoint(CurBlock);
3194 Builder.CreateCondBr(Cond: FeatsCondition, True: RetBlock, False: ElseBlock);
3195
3196 CurBlock = ElseBlock;
3197 }
3198
3199 // Finally, emit the default one.
3200 if (HasDefault) {
3201 Builder.SetInsertPoint(CurBlock);
3202 CreateMultiVersionResolverReturn(
3203 CGM, Resolver, Builder, FuncToReturn: Options[DefaultIndex].Function, SupportsIFunc);
3204 return;
3205 }
3206
3207 // If no generic/default, emit an unreachable.
3208 Builder.SetInsertPoint(CurBlock);
3209 EmitTrapCallAndMakeUnreachable();
3210}
3211
3212void CodeGenFunction::EmitAArch64MultiVersionResolver(
3213 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3214 assert(!Options.empty() && "No multiversion resolver options found");
3215 assert(Options.back().Features.size() == 0 && "Default case must be last");
3216 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3217 assert(SupportsIFunc &&
3218 "Multiversion resolver requires target IFUNC support");
3219 bool AArch64CpuInitialized = false;
3220 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3221
3222 for (const FMVResolverOption &RO : Options) {
3223 // Skip unreachable versions.
3224 if (RO.Function == nullptr)
3225 continue;
3226
3227 Builder.SetInsertPoint(CurBlock);
3228 llvm::Value *Condition = FormAArch64ResolverCondition(RO);
3229
3230 // The 'default' or 'all features enabled' case.
3231 if (!Condition) {
3232 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, FuncToReturn: RO.Function,
3233 SupportsIFunc);
3234 return;
3235 }
3236
3237 if (!AArch64CpuInitialized) {
3238 Builder.SetInsertPoint(CurBlock->begin());
3239 EmitAArch64CpuInit();
3240 AArch64CpuInitialized = true;
3241 Builder.SetInsertPoint(CurBlock);
3242 }
3243
3244 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3245 CGBuilderTy RetBuilder(CGM, RetBlock);
3246 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder, FuncToReturn: RO.Function,
3247 SupportsIFunc);
3248 CurBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3249 Builder.CreateCondBr(Cond: Condition, True: RetBlock, False: CurBlock);
3250 }
3251
3252 // If no default, emit an unreachable.
3253 Builder.SetInsertPoint(CurBlock);
3254 EmitTrapCallAndMakeUnreachable();
3255}
3256
3257void CodeGenFunction::EmitX86MultiVersionResolver(
3258 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3259
3260 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3261
3262 // Main function's basic block.
3263 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3264 Builder.SetInsertPoint(CurBlock);
3265 EmitX86CpuInit();
3266
3267 for (const FMVResolverOption &RO : Options) {
3268 Builder.SetInsertPoint(CurBlock);
3269 llvm::Value *Condition = FormX86ResolverCondition(RO);
3270
3271 // The 'default' or 'generic' case.
3272 if (!Condition) {
3273 assert(&RO == Options.end() - 1 &&
3274 "Default or Generic case must be last");
3275 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, FuncToReturn: RO.Function,
3276 SupportsIFunc);
3277 return;
3278 }
3279
3280 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3281 CGBuilderTy RetBuilder(CGM, RetBlock);
3282 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder, FuncToReturn: RO.Function,
3283 SupportsIFunc);
3284 CurBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3285 Builder.CreateCondBr(Cond: Condition, True: RetBlock, False: CurBlock);
3286 }
3287
3288 // If no generic/default, emit an unreachable.
3289 Builder.SetInsertPoint(CurBlock);
3290 EmitTrapCallAndMakeUnreachable();
3291}
3292
3293// Loc / DynamicLoc - where the diagnostic will point, where in the source code
3294// this alignment has failed.
3295// AssumptionLoc - if present (will be present if sufficiently different from
3296// Loc), the diagnostic will additionally point a "Note:" to this location.
3297// It should be the location where the __attribute__((assume_aligned))
3298// was written e.g.
3299static void emitAlignmentAssumptionCheck(CodeGenFunction &CGF, llvm::Value *Ptr,
3300 QualType Ty, SourceLocation Loc,
3301 llvm::Value *DynamicLoc,
3302 SourceLocation AssumptionLoc,
3303 llvm::Value *Alignment,
3304 llvm::Value *OffsetValue) {
3305 if (!CGF.SanOpts.has(K: SanitizerKind::Alignment))
3306 return;
3307
3308 CGBuilderTy &Builder = CGF.Builder;
3309 llvm::IntegerType *IntPtrTy = CGF.IntPtrTy;
3310 if (Alignment->getType() != IntPtrTy)
3311 Alignment =
3312 Builder.CreateIntCast(V: Alignment, DestTy: IntPtrTy, isSigned: false, Name: "casted.align");
3313 if (OffsetValue && OffsetValue->getType() != IntPtrTy)
3314 OffsetValue =
3315 Builder.CreateIntCast(V: OffsetValue, DestTy: IntPtrTy, isSigned: true, Name: "casted.offset");
3316
3317 llvm::Value *PtrIntValue = Builder.CreatePtrToInt(V: Ptr, DestTy: IntPtrTy, Name: "ptrint");
3318
3319 if (OffsetValue) {
3320 bool IsOffsetZero = false;
3321 if (const auto *CI = dyn_cast<llvm::ConstantInt>(Val: OffsetValue))
3322 IsOffsetZero = CI->isZero();
3323
3324 if (!IsOffsetZero)
3325 PtrIntValue = Builder.CreateSub(LHS: PtrIntValue, RHS: OffsetValue, Name: "offsetptr");
3326 }
3327
3328 llvm::Value *Zero = llvm::ConstantInt::get(Ty: IntPtrTy, V: 0);
3329 llvm::Value *Mask =
3330 Builder.CreateSub(LHS: Alignment, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: 1));
3331 llvm::Value *MaskedPtr = Builder.CreateAnd(LHS: PtrIntValue, RHS: Mask, Name: "maskedptr");
3332 llvm::Value *TheCheck = Builder.CreateICmpEQ(LHS: MaskedPtr, RHS: Zero, Name: "maskcond");
3333
3334 // Don't check pointers to volatile data. The behavior here is implementation-
3335 // defined.
3336 if (Ty->getPointeeType().isVolatileQualified())
3337 return;
3338
3339 auto CheckOrdinal = SanitizerKind::SO_Alignment;
3340 auto CheckHandler = !DynamicLoc ? SanitizerHandler::AlignmentAssumption
3341 : SanitizerHandler::ReturnAlignmentAssumption;
3342 SanitizerDebugLocation SanScope(&CGF, {CheckOrdinal}, CheckHandler);
3343
3344 if (!OffsetValue)
3345 OffsetValue = Builder.getInt1(V: false); // no offset.
3346
3347 if (!DynamicLoc) {
3348 llvm::Constant *StaticData[] = {CGF.EmitCheckSourceLocation(Loc),
3349 CGF.EmitCheckSourceLocation(Loc: AssumptionLoc),
3350 CGF.EmitCheckTypeDescriptor(T: Ty)};
3351 llvm::Value *DynamicData[] = {Ptr, Alignment, OffsetValue};
3352 CGF.EmitCheck(Checked: {std::make_pair(x&: TheCheck, y&: CheckOrdinal)}, Check: CheckHandler,
3353 StaticArgs: StaticData, DynamicArgs: DynamicData);
3354 } else {
3355 llvm::Constant *StaticData[] = {CGF.EmitCheckSourceLocation(Loc: AssumptionLoc),
3356 CGF.EmitCheckTypeDescriptor(T: Ty)};
3357 llvm::Value *DynamicData[] = {DynamicLoc, Ptr, Alignment, OffsetValue};
3358 CGF.EmitCheck(Checked: {std::make_pair(x&: TheCheck, y&: CheckOrdinal)}, Check: CheckHandler,
3359 StaticArgs: StaticData, DynamicArgs: DynamicData);
3360 }
3361}
3362
3363void CodeGenFunction::emitAlignmentAssumptionCheck(llvm::Value *Ptr,
3364 QualType Ty,
3365 SourceLocation Loc,
3366 SourceLocation AssumptionLoc,
3367 llvm::Value *Alignment,
3368 llvm::Value *OffsetValue) {
3369 ::emitAlignmentAssumptionCheck(CGF&: *this, Ptr, Ty, Loc, DynamicLoc: nullptr, AssumptionLoc,
3370 Alignment, OffsetValue);
3371}
3372
3373void CodeGenFunction::emitReturnAlignmentAssumptionCheck(
3374 llvm::Value *Ptr, QualType Ty, SourceLocation AssumptionLoc,
3375 llvm::Value *Alignment, llvm::Value *OffsetValue) {
3376 llvm::Value *SLocPtr = Builder.CreateLoad(Addr: ReturnLocation, Name: "return.sloc.load");
3377 ::emitAlignmentAssumptionCheck(CGF&: *this, Ptr, Ty, Loc: SourceLocation(), DynamicLoc: SLocPtr,
3378 AssumptionLoc, Alignment, OffsetValue);
3379}
3380
3381llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
3382 if (CGDebugInfo *DI = getDebugInfo())
3383 return DI->SourceLocToDebugLoc(Loc: Location);
3384
3385 return llvm::DebugLoc();
3386}
3387
3388llvm::Value *
3389CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
3390 Stmt::Likelihood LH) {
3391 switch (LH) {
3392 case Stmt::LH_None:
3393 return Cond;
3394 case Stmt::LH_Likely:
3395 case Stmt::LH_Unlikely:
3396 // Don't generate llvm.expect on -O0 as the backend won't use it for
3397 // anything.
3398 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3399 return Cond;
3400 llvm::Type *CondTy = Cond->getType();
3401 assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");
3402 llvm::Function *FnExpect =
3403 CGM.getIntrinsic(IID: llvm::Intrinsic::expect, Tys: CondTy);
3404 llvm::Value *ExpectedValueOfCond =
3405 llvm::ConstantInt::getBool(Ty: CondTy, V: LH == Stmt::LH_Likely);
3406 return Builder.CreateCall(Callee: FnExpect, Args: {Cond, ExpectedValueOfCond},
3407 Name: Cond->getName() + ".expval");
3408 }
3409 llvm_unreachable("Unknown Likelihood");
3410}
3411
3412llvm::Value *CodeGenFunction::emitBoolVecConversion(llvm::Value *SrcVec,
3413 unsigned NumElementsDst,
3414 const llvm::Twine &Name) {
3415 auto *SrcTy = cast<llvm::FixedVectorType>(Val: SrcVec->getType());
3416 unsigned NumElementsSrc = SrcTy->getNumElements();
3417 if (NumElementsSrc == NumElementsDst)
3418 return SrcVec;
3419
3420 std::vector<int> ShuffleMask(NumElementsDst, -1);
3421 for (unsigned MaskIdx = 0;
3422 MaskIdx < std::min<>(a: NumElementsDst, b: NumElementsSrc); ++MaskIdx)
3423 ShuffleMask[MaskIdx] = MaskIdx;
3424
3425 return Builder.CreateShuffleVector(V: SrcVec, Mask: ShuffleMask, Name);
3426}
3427
3428void CodeGenFunction::EmitPointerAuthOperandBundle(
3429 const CGPointerAuthInfo &PointerAuth,
3430 SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
3431 if (!PointerAuth.isSigned())
3432 return;
3433
3434 auto *Key = Builder.getInt32(C: PointerAuth.getKey());
3435
3436 llvm::Value *Discriminator = PointerAuth.getDiscriminator();
3437 if (!Discriminator)
3438 Discriminator = Builder.getSize(N: 0);
3439
3440 llvm::Value *Args[] = {Key, Discriminator};
3441 Bundles.emplace_back(Args: "ptrauth", Args);
3442}
3443
3444static llvm::Value *EmitPointerAuthCommon(CodeGenFunction &CGF,
3445 const CGPointerAuthInfo &PointerAuth,
3446 llvm::Value *Pointer,
3447 unsigned IntrinsicID) {
3448 if (!PointerAuth)
3449 return Pointer;
3450
3451 auto Key = CGF.Builder.getInt32(C: PointerAuth.getKey());
3452
3453 llvm::Value *Discriminator = PointerAuth.getDiscriminator();
3454 if (!Discriminator) {
3455 Discriminator = CGF.Builder.getSize(N: 0);
3456 }
3457
3458 // Convert the pointer to intptr_t before signing it.
3459 auto OrigType = Pointer->getType();
3460 Pointer = CGF.Builder.CreatePtrToInt(V: Pointer, DestTy: CGF.IntPtrTy);
3461
3462 // call i64 @llvm.ptrauth.sign.i64(i64 %pointer, i32 %key, i64 %discriminator)
3463 auto Intrinsic = CGF.CGM.getIntrinsic(IID: IntrinsicID);
3464 Pointer = CGF.EmitRuntimeCall(callee: Intrinsic, args: {Pointer, Key, Discriminator});
3465
3466 // Convert back to the original type.
3467 Pointer = CGF.Builder.CreateIntToPtr(V: Pointer, DestTy: OrigType);
3468 return Pointer;
3469}
3470
3471llvm::Value *
3472CodeGenFunction::EmitPointerAuthSign(const CGPointerAuthInfo &PointerAuth,
3473 llvm::Value *Pointer) {
3474 if (!PointerAuth.shouldSign())
3475 return Pointer;
3476 return EmitPointerAuthCommon(CGF&: *this, PointerAuth, Pointer,
3477 IntrinsicID: llvm::Intrinsic::ptrauth_sign);
3478}
3479
3480static llvm::Value *EmitStrip(CodeGenFunction &CGF,
3481 const CGPointerAuthInfo &PointerAuth,
3482 llvm::Value *Pointer) {
3483 auto StripIntrinsic = CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::ptrauth_strip);
3484
3485 auto Key = CGF.Builder.getInt32(C: PointerAuth.getKey());
3486 // Convert the pointer to intptr_t before signing it.
3487 auto OrigType = Pointer->getType();
3488 Pointer = CGF.EmitRuntimeCall(
3489 callee: StripIntrinsic, args: {CGF.Builder.CreatePtrToInt(V: Pointer, DestTy: CGF.IntPtrTy), Key});
3490 return CGF.Builder.CreateIntToPtr(V: Pointer, DestTy: OrigType);
3491}
3492
3493llvm::Value *
3494CodeGenFunction::EmitPointerAuthAuth(const CGPointerAuthInfo &PointerAuth,
3495 llvm::Value *Pointer) {
3496 if (PointerAuth.shouldStrip()) {
3497 return EmitStrip(CGF&: *this, PointerAuth, Pointer);
3498 }
3499 if (!PointerAuth.shouldAuth()) {
3500 return Pointer;
3501 }
3502
3503 return EmitPointerAuthCommon(CGF&: *this, PointerAuth, Pointer,
3504 IntrinsicID: llvm::Intrinsic::ptrauth_auth);
3505}
3506
3507void CodeGenFunction::addInstToCurrentSourceAtom(
3508 llvm::Instruction *KeyInstruction, llvm::Value *Backup) {
3509 if (CGDebugInfo *DI = getDebugInfo())
3510 DI->addInstToCurrentSourceAtom(KeyInstruction, Backup);
3511}
3512
3513void CodeGenFunction::addInstToSpecificSourceAtom(
3514 llvm::Instruction *KeyInstruction, llvm::Value *Backup, uint64_t Atom) {
3515 if (CGDebugInfo *DI = getDebugInfo())
3516 DI->addInstToSpecificSourceAtom(KeyInstruction, Backup, Atom);
3517}
3518
3519void CodeGenFunction::addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,
3520 llvm::Value *Backup) {
3521 if (CGDebugInfo *DI = getDebugInfo()) {
3522 ApplyAtomGroup Grp(getDebugInfo());
3523 DI->addInstToCurrentSourceAtom(KeyInstruction, Backup);
3524 }
3525}
3526
3527void CodeGenFunction::emitPFPPostCopyUpdates(Address DestPtr, Address SrcPtr,
3528 QualType Ty) {
3529 for (auto &Field : getContext().findPFPFields(Ty)) {
3530 if (getContext().arePFPFieldsTriviallyCopyable(RD: Field.Field->getParent()))
3531 continue;
3532 auto DestFieldPtr = EmitAddressOfPFPField(RecordPtr: DestPtr, Field);
3533 auto SrcFieldPtr = EmitAddressOfPFPField(RecordPtr: SrcPtr, Field);
3534 Builder.CreateStore(Val: Builder.CreateLoad(Addr: SrcFieldPtr), Addr: DestFieldPtr);
3535 }
3536}
3537