1//===--- CodeGenFunction.cpp - Emit LLVM Code from ASTs for a Function ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This coordinates the per-function state used while generating code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CodeGenFunction.h"
14#include "CGBlocks.h"
15#include "CGCUDARuntime.h"
16#include "CGCXXABI.h"
17#include "CGCleanup.h"
18#include "CGDebugInfo.h"
19#include "CGHLSLRuntime.h"
20#include "CGOpenMPRuntime.h"
21#include "CodeGenModule.h"
22#include "CodeGenPGO.h"
23#include "TargetInfo.h"
24#include "clang/AST/ASTContext.h"
25#include "clang/AST/ASTLambda.h"
26#include "clang/AST/Attr.h"
27#include "clang/AST/Decl.h"
28#include "clang/AST/DeclCXX.h"
29#include "clang/AST/Expr.h"
30#include "clang/AST/IgnoreExpr.h"
31#include "clang/AST/StmtCXX.h"
32#include "clang/AST/StmtObjC.h"
33#include "clang/Basic/Builtins.h"
34#include "clang/Basic/CodeGenOptions.h"
35#include "clang/Basic/DiagnosticFrontend.h"
36#include "clang/Basic/TargetBuiltins.h"
37#include "clang/Basic/TargetInfo.h"
38#include "clang/CodeGen/CGFunctionInfo.h"
39#include "llvm/ADT/ArrayRef.h"
40#include "llvm/ADT/ScopeExit.h"
41#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
42#include "llvm/IR/DataLayout.h"
43#include "llvm/IR/Dominators.h"
44#include "llvm/IR/FPEnv.h"
45#include "llvm/IR/Instruction.h"
46#include "llvm/IR/IntrinsicInst.h"
47#include "llvm/IR/Intrinsics.h"
48#include "llvm/IR/MDBuilder.h"
49#include "llvm/Support/CRC.h"
50#include "llvm/Support/SipHash.h"
51#include "llvm/Support/xxhash.h"
52#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
53#include "llvm/Transforms/Utils/PromoteMemToReg.h"
54#include <optional>
55
56using namespace clang;
57using namespace CodeGen;
58
59/// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time
60/// markers.
61static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts,
62 const LangOptions &LangOpts) {
63 if (CGOpts.DisableLifetimeMarkers)
64 return false;
65
66 // Sanitizers may use markers.
67 if (CGOpts.SanitizeAddressUseAfterScope ||
68 LangOpts.Sanitize.has(K: SanitizerKind::HWAddress) ||
69 LangOpts.Sanitize.has(K: SanitizerKind::Memory) ||
70 LangOpts.Sanitize.has(K: SanitizerKind::MemtagStack))
71 return true;
72
73 // For now, only in optimized builds.
74 return CGOpts.OptimizationLevel != 0;
75}
76
77CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
78 : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
79 Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(),
80 CGBuilderInserterTy(this)),
81 SanOpts(CGM.getLangOpts().Sanitize), CurFPFeatures(CGM.getLangOpts()),
82 DebugInfo(CGM.getModuleDebugInfo()),
83 PGO(std::make_unique<CodeGenPGO>(args&: cgm)),
84 ShouldEmitLifetimeMarkers(
85 shouldEmitLifetimeMarkers(CGOpts: CGM.getCodeGenOpts(), LangOpts: CGM.getLangOpts())) {
86 if (!suppressNewContext)
87 CGM.getCXXABI().getMangleContext().startNewFunction();
88 EHStack.setCGF(this);
89
90 SetFastMathFlags(CurFPFeatures);
91}
92
93CodeGenFunction::~CodeGenFunction() {
94 assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup");
95 assert(DeferredDeactivationCleanupStack.empty() &&
96 "missed to deactivate a cleanup");
97
98 if (getLangOpts().OpenMP && CurFn)
99 CGM.getOpenMPRuntime().functionFinished(CGF&: *this);
100
101 // If we have an OpenMPIRBuilder we want to finalize functions (incl.
102 // outlining etc) at some point. Doing it once the function codegen is done
103 // seems to be a reasonable spot. We do it here, as opposed to the deletion
104 // time of the CodeGenModule, because we have to ensure the IR has not yet
105 // been "emitted" to the outside, thus, modifications are still sensible.
106 if (CGM.getLangOpts().OpenMPIRBuilder && CurFn)
107 CGM.getOpenMPRuntime().getOMPBuilder().finalize(Fn: CurFn);
108}
109
110// Map the LangOption for exception behavior into
111// the corresponding enum in the IR.
112llvm::fp::ExceptionBehavior
113clang::ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind) {
114
115 switch (Kind) {
116 case LangOptions::FPE_Ignore: return llvm::fp::ebIgnore;
117 case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap;
118 case LangOptions::FPE_Strict: return llvm::fp::ebStrict;
119 default:
120 llvm_unreachable("Unsupported FP Exception Behavior");
121 }
122}
123
124void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) {
125 llvm::FastMathFlags FMF;
126 FMF.setAllowReassoc(FPFeatures.getAllowFPReassociate());
127 FMF.setNoNaNs(FPFeatures.getNoHonorNaNs());
128 FMF.setNoInfs(FPFeatures.getNoHonorInfs());
129 FMF.setNoSignedZeros(FPFeatures.getNoSignedZero());
130 FMF.setAllowReciprocal(FPFeatures.getAllowReciprocal());
131 FMF.setApproxFunc(FPFeatures.getAllowApproxFunc());
132 FMF.setAllowContract(FPFeatures.allowFPContractAcrossStatement());
133 Builder.setFastMathFlags(FMF);
134}
135
136CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
137 const Expr *E)
138 : CGF(CGF) {
139 ConstructorHelper(FPFeatures: E->getFPFeaturesInEffect(LO: CGF.getLangOpts()));
140}
141
142CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
143 FPOptions FPFeatures)
144 : CGF(CGF) {
145 ConstructorHelper(FPFeatures);
146}
147
148void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) {
149 OldFPFeatures = CGF.CurFPFeatures;
150 CGF.CurFPFeatures = FPFeatures;
151
152 OldExcept = CGF.Builder.getDefaultConstrainedExcept();
153 OldRounding = CGF.Builder.getDefaultConstrainedRounding();
154
155 if (OldFPFeatures == FPFeatures)
156 return;
157
158 FMFGuard.emplace(args&: CGF.Builder);
159
160 llvm::RoundingMode NewRoundingBehavior = FPFeatures.getRoundingMode();
161 CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);
162 auto NewExceptionBehavior =
163 ToConstrainedExceptMD(Kind: FPFeatures.getExceptionMode());
164 CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);
165
166 CGF.SetFastMathFlags(FPFeatures);
167
168 assert((CGF.CurFuncDecl == nullptr || CGF.Builder.getIsFPConstrained() ||
169 isa<CXXConstructorDecl>(CGF.CurFuncDecl) ||
170 isa<CXXDestructorDecl>(CGF.CurFuncDecl) ||
171 (NewExceptionBehavior == llvm::fp::ebIgnore &&
172 NewRoundingBehavior == llvm::RoundingMode::NearestTiesToEven)) &&
173 "FPConstrained should be enabled on entire function");
174
175 auto mergeFnAttrValue = [&](StringRef Name, bool Value) {
176 auto OldValue =
177 CGF.CurFn->getFnAttribute(Kind: Name).getValueAsBool();
178 auto NewValue = OldValue & Value;
179 if (OldValue != NewValue)
180 CGF.CurFn->addFnAttr(Kind: Name, Val: llvm::toStringRef(B: NewValue));
181 };
182 mergeFnAttrValue("no-nans-fp-math", FPFeatures.getNoHonorNaNs());
183 mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.getNoSignedZero());
184}
185
186CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
187 CGF.CurFPFeatures = OldFPFeatures;
188 CGF.Builder.setDefaultConstrainedExcept(OldExcept);
189 CGF.Builder.setDefaultConstrainedRounding(OldRounding);
190}
191
192static LValue
193makeNaturalAlignAddrLValue(llvm::Value *V, QualType T, bool ForPointeeType,
194 bool MightBeSigned, CodeGenFunction &CGF,
195 KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
196 LValueBaseInfo BaseInfo;
197 TBAAAccessInfo TBAAInfo;
198 CharUnits Alignment =
199 CGF.CGM.getNaturalTypeAlignment(T, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo, forPointeeType: ForPointeeType);
200 Address Addr =
201 MightBeSigned
202 ? CGF.makeNaturalAddressForPointer(Ptr: V, T, Alignment, ForPointeeType: false, BaseInfo: nullptr,
203 TBAAInfo: nullptr, IsKnownNonNull)
204 : Address(V, CGF.ConvertTypeForMem(T), Alignment, IsKnownNonNull);
205 return CGF.MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
206}
207
208LValue
209CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
210 KnownNonNull_t IsKnownNonNull) {
211 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,
212 /*MightBeSigned*/ true, CGF&: *this,
213 IsKnownNonNull);
214}
215
216LValue
217CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
218 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,
219 /*MightBeSigned*/ true, CGF&: *this);
220}
221
222LValue CodeGenFunction::MakeNaturalAlignRawAddrLValue(llvm::Value *V,
223 QualType T) {
224 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,
225 /*MightBeSigned*/ false, CGF&: *this);
226}
227
228LValue CodeGenFunction::MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V,
229 QualType T) {
230 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,
231 /*MightBeSigned*/ false, CGF&: *this);
232}
233
234llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
235 return CGM.getTypes().ConvertTypeForMem(T);
236}
237
238llvm::Type *CodeGenFunction::ConvertType(QualType T) {
239 return CGM.getTypes().ConvertType(T);
240}
241
242llvm::Type *CodeGenFunction::convertTypeForLoadStore(QualType ASTTy,
243 llvm::Type *LLVMTy) {
244 return CGM.getTypes().convertTypeForLoadStore(T: ASTTy, LLVMTy);
245}
246
247TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
248 type = type.getCanonicalType();
249 while (true) {
250 switch (type->getTypeClass()) {
251#define TYPE(name, parent)
252#define ABSTRACT_TYPE(name, parent)
253#define NON_CANONICAL_TYPE(name, parent) case Type::name:
254#define DEPENDENT_TYPE(name, parent) case Type::name:
255#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
256#include "clang/AST/TypeNodes.inc"
257 llvm_unreachable("non-canonical or dependent type in IR-generation");
258
259 case Type::Auto:
260 case Type::DeducedTemplateSpecialization:
261 llvm_unreachable("undeduced type in IR-generation");
262
263 // Various scalar types.
264 case Type::Builtin:
265 case Type::Pointer:
266 case Type::BlockPointer:
267 case Type::LValueReference:
268 case Type::RValueReference:
269 case Type::MemberPointer:
270 case Type::Vector:
271 case Type::ExtVector:
272 case Type::ConstantMatrix:
273 case Type::FunctionProto:
274 case Type::FunctionNoProto:
275 case Type::Enum:
276 case Type::ObjCObjectPointer:
277 case Type::Pipe:
278 case Type::BitInt:
279 case Type::HLSLAttributedResource:
280 case Type::HLSLInlineSpirv:
281 case Type::OverflowBehavior:
282 return TEK_Scalar;
283
284 // Complexes.
285 case Type::Complex:
286 return TEK_Complex;
287
288 // Arrays, records, and Objective-C objects.
289 case Type::ConstantArray:
290 case Type::IncompleteArray:
291 case Type::VariableArray:
292 case Type::Record:
293 case Type::ObjCObject:
294 case Type::ObjCInterface:
295 case Type::ArrayParameter:
296 return TEK_Aggregate;
297
298 // We operate on atomic values according to their underlying type.
299 case Type::Atomic:
300 type = cast<AtomicType>(Val&: type)->getValueType();
301 continue;
302 }
303 llvm_unreachable("unknown type kind!");
304 }
305}
306
307llvm::DebugLoc CodeGenFunction::EmitReturnBlock() {
308 // For cleanliness, we try to avoid emitting the return block for
309 // simple cases.
310 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
311
312 if (CurBB) {
313 assert(!CurBB->getTerminator() && "Unexpected terminated block.");
314
315 // We have a valid insert point, reuse it if it is empty or there are no
316 // explicit jumps to the return block.
317 if (CurBB->empty() || ReturnBlock.getBlock()->use_empty()) {
318 ReturnBlock.getBlock()->replaceAllUsesWith(V: CurBB);
319 delete ReturnBlock.getBlock();
320 ReturnBlock = JumpDest();
321 } else
322 EmitBlock(BB: ReturnBlock.getBlock());
323 return llvm::DebugLoc();
324 }
325
326 // Otherwise, if the return block is the target of a single direct
327 // branch then we can just put the code in that block instead. This
328 // cleans up functions which started with a unified return block.
329 if (ReturnBlock.getBlock()->hasOneUse()) {
330 llvm::BranchInst *BI =
331 dyn_cast<llvm::BranchInst>(Val: *ReturnBlock.getBlock()->user_begin());
332 if (BI && BI->isUnconditional() &&
333 BI->getSuccessor(i: 0) == ReturnBlock.getBlock()) {
334 // Record/return the DebugLoc of the simple 'return' expression to be used
335 // later by the actual 'ret' instruction.
336 llvm::DebugLoc Loc = BI->getDebugLoc();
337 Builder.SetInsertPoint(BI->getParent());
338 BI->eraseFromParent();
339 delete ReturnBlock.getBlock();
340 ReturnBlock = JumpDest();
341 return Loc;
342 }
343 }
344
345 // FIXME: We are at an unreachable point, there is no reason to emit the block
346 // unless it has uses. However, we still need a place to put the debug
347 // region.end for now.
348
349 EmitBlock(BB: ReturnBlock.getBlock());
350 return llvm::DebugLoc();
351}
352
353static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
354 if (!BB) return;
355 if (!BB->use_empty()) {
356 CGF.CurFn->insert(Position: CGF.CurFn->end(), BB);
357 return;
358 }
359 delete BB;
360}
361
362void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
363 assert(BreakContinueStack.empty() &&
364 "mismatched push/pop in break/continue stack!");
365 assert(LifetimeExtendedCleanupStack.empty() &&
366 "mismatched push/pop of cleanups in EHStack!");
367 assert(DeferredDeactivationCleanupStack.empty() &&
368 "mismatched activate/deactivate of cleanups!");
369
370 if (CGM.shouldEmitConvergenceTokens()) {
371 ConvergenceTokenStack.pop_back();
372 assert(ConvergenceTokenStack.empty() &&
373 "mismatched push/pop in convergence stack!");
374 }
375
376 bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
377 && NumSimpleReturnExprs == NumReturnExprs
378 && ReturnBlock.getBlock()->use_empty();
379 // Usually the return expression is evaluated before the cleanup
380 // code. If the function contains only a simple return statement,
381 // such as a constant, the location before the cleanup code becomes
382 // the last useful breakpoint in the function, because the simple
383 // return expression will be evaluated after the cleanup code. To be
384 // safe, set the debug location for cleanup code to the location of
385 // the return statement. Otherwise the cleanup code should be at the
386 // end of the function's lexical scope.
387 //
388 // If there are multiple branches to the return block, the branch
389 // instructions will get the location of the return statements and
390 // all will be fine.
391 if (CGDebugInfo *DI = getDebugInfo()) {
392 if (OnlySimpleReturnStmts)
393 DI->EmitLocation(Builder, Loc: LastStopPoint);
394 else
395 DI->EmitLocation(Builder, Loc: EndLoc);
396 }
397
398 // Pop any cleanups that might have been associated with the
399 // parameters. Do this in whatever block we're currently in; it's
400 // important to do this before we enter the return block or return
401 // edges will be *really* confused.
402 bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth;
403 bool HasOnlyNoopCleanups =
404 HasCleanups && EHStack.containsOnlyNoopCleanups(Old: PrologueCleanupDepth);
405 bool EmitRetDbgLoc = !HasCleanups || HasOnlyNoopCleanups;
406
407 std::optional<ApplyDebugLocation> OAL;
408 if (HasCleanups) {
409 // Make sure the line table doesn't jump back into the body for
410 // the ret after it's been at EndLoc.
411 if (CGDebugInfo *DI = getDebugInfo()) {
412 if (OnlySimpleReturnStmts)
413 DI->EmitLocation(Builder, Loc: EndLoc);
414 else
415 // We may not have a valid end location. Try to apply it anyway, and
416 // fall back to an artificial location if needed.
417 OAL = ApplyDebugLocation::CreateDefaultArtificial(CGF&: *this, TemporaryLocation: EndLoc);
418 }
419
420 PopCleanupBlocks(OldCleanupStackSize: PrologueCleanupDepth);
421 }
422
423 // Emit function epilog (to return).
424 llvm::DebugLoc Loc = EmitReturnBlock();
425
426 if (ShouldInstrumentFunction()) {
427 if (CGM.getCodeGenOpts().InstrumentFunctions)
428 CurFn->addFnAttr(Kind: "instrument-function-exit", Val: "__cyg_profile_func_exit");
429 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
430 CurFn->addFnAttr(Kind: "instrument-function-exit-inlined",
431 Val: "__cyg_profile_func_exit");
432 }
433
434 // Emit debug descriptor for function end.
435 if (CGDebugInfo *DI = getDebugInfo())
436 DI->EmitFunctionEnd(Builder, Fn: CurFn);
437
438 // Reset the debug location to that of the simple 'return' expression, if any
439 // rather than that of the end of the function's scope '}'.
440 uint64_t RetKeyInstructionsAtomGroup = Loc ? Loc->getAtomGroup() : 0;
441 ApplyDebugLocation AL(*this, Loc);
442 EmitFunctionEpilog(FI: *CurFnInfo, EmitRetDbgLoc, EndLoc,
443 RetKeyInstructionsSourceAtom: RetKeyInstructionsAtomGroup);
444 EmitEndEHSpec(D: CurCodeDecl);
445
446 assert(EHStack.empty() &&
447 "did not remove all scopes from cleanup stack!");
448
449 // If someone did an indirect goto, emit the indirect goto block at the end of
450 // the function.
451 if (IndirectBranch) {
452 EmitBlock(BB: IndirectBranch->getParent());
453 Builder.ClearInsertionPoint();
454 }
455
456 // If some of our locals escaped, insert a call to llvm.localescape in the
457 // entry block.
458 if (!EscapedLocals.empty()) {
459 // Invert the map from local to index into a simple vector. There should be
460 // no holes.
461 SmallVector<llvm::Value *, 4> EscapeArgs;
462 EscapeArgs.resize(N: EscapedLocals.size());
463 for (auto &Pair : EscapedLocals)
464 EscapeArgs[Pair.second] = Pair.first;
465 llvm::Function *FrameEscapeFn = llvm::Intrinsic::getOrInsertDeclaration(
466 M: &CGM.getModule(), id: llvm::Intrinsic::localescape);
467 CGBuilderTy(*this, AllocaInsertPt).CreateCall(Callee: FrameEscapeFn, Args: EscapeArgs);
468 }
469
470 // Remove the AllocaInsertPt instruction, which is just a convenience for us.
471 llvm::Instruction *Ptr = AllocaInsertPt;
472 AllocaInsertPt = nullptr;
473 Ptr->eraseFromParent();
474
475 // PostAllocaInsertPt, if created, was lazily created when it was required,
476 // remove it now since it was just created for our own convenience.
477 if (PostAllocaInsertPt) {
478 llvm::Instruction *PostPtr = PostAllocaInsertPt;
479 PostAllocaInsertPt = nullptr;
480 PostPtr->eraseFromParent();
481 }
482
483 // If someone took the address of a label but never did an indirect goto, we
484 // made a zero entry PHI node, which is illegal, zap it now.
485 if (IndirectBranch) {
486 llvm::PHINode *PN = cast<llvm::PHINode>(Val: IndirectBranch->getAddress());
487 if (PN->getNumIncomingValues() == 0) {
488 PN->replaceAllUsesWith(V: llvm::PoisonValue::get(T: PN->getType()));
489 PN->eraseFromParent();
490 }
491 }
492
493 EmitIfUsed(CGF&: *this, BB: EHResumeBlock);
494 EmitIfUsed(CGF&: *this, BB: TerminateLandingPad);
495 EmitIfUsed(CGF&: *this, BB: TerminateHandler);
496 EmitIfUsed(CGF&: *this, BB: UnreachableBlock);
497
498 for (const auto &FuncletAndParent : TerminateFunclets)
499 EmitIfUsed(CGF&: *this, BB: FuncletAndParent.second);
500
501 if (CGM.getCodeGenOpts().EmitDeclMetadata)
502 EmitDeclMetadata();
503
504 for (const auto &R : DeferredReplacements) {
505 if (llvm::Value *Old = R.first) {
506 Old->replaceAllUsesWith(V: R.second);
507 cast<llvm::Instruction>(Val: Old)->eraseFromParent();
508 }
509 }
510 DeferredReplacements.clear();
511
512 // Eliminate CleanupDestSlot alloca by replacing it with SSA values and
513 // PHIs if the current function is a coroutine. We don't do it for all
514 // functions as it may result in slight increase in numbers of instructions
515 // if compiled with no optimizations. We do it for coroutine as the lifetime
516 // of CleanupDestSlot alloca make correct coroutine frame building very
517 // difficult.
518 if (NormalCleanupDest.isValid() && isCoroutine()) {
519 llvm::DominatorTree DT(*CurFn);
520 llvm::PromoteMemToReg(
521 Allocas: cast<llvm::AllocaInst>(Val: NormalCleanupDest.getPointer()), DT);
522 NormalCleanupDest = Address::invalid();
523 }
524
525 // Scan function arguments for vector width.
526 for (llvm::Argument &A : CurFn->args())
527 if (auto *VT = dyn_cast<llvm::VectorType>(Val: A.getType()))
528 LargestVectorWidth =
529 std::max(a: (uint64_t)LargestVectorWidth,
530 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
531
532 // Update vector width based on return type.
533 if (auto *VT = dyn_cast<llvm::VectorType>(Val: CurFn->getReturnType()))
534 LargestVectorWidth =
535 std::max(a: (uint64_t)LargestVectorWidth,
536 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
537
538 if (CurFnInfo->getMaxVectorWidth() > LargestVectorWidth)
539 LargestVectorWidth = CurFnInfo->getMaxVectorWidth();
540
541 // Add the min-legal-vector-width attribute. This contains the max width from:
542 // 1. min-vector-width attribute used in the source program.
543 // 2. Any builtins used that have a vector width specified.
544 // 3. Values passed in and out of inline assembly.
545 // 4. Width of vector arguments and return types for this function.
546 // 5. Width of vector arguments and return types for functions called by this
547 // function.
548 if (getContext().getTargetInfo().getTriple().isX86())
549 CurFn->addFnAttr(Kind: "min-legal-vector-width",
550 Val: llvm::utostr(X: LargestVectorWidth));
551
552 // If we generated an unreachable return block, delete it now.
553 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()) {
554 Builder.ClearInsertionPoint();
555 ReturnBlock.getBlock()->eraseFromParent();
556 }
557 if (ReturnValue.isValid()) {
558 auto *RetAlloca =
559 dyn_cast<llvm::AllocaInst>(Val: ReturnValue.emitRawPointer(CGF&: *this));
560 if (RetAlloca && RetAlloca->use_empty()) {
561 RetAlloca->eraseFromParent();
562 ReturnValue = Address::invalid();
563 }
564 }
565}
566
567/// ShouldInstrumentFunction - Return true if the current function should be
568/// instrumented with __cyg_profile_func_* calls
569bool CodeGenFunction::ShouldInstrumentFunction() {
570 if (!CGM.getCodeGenOpts().InstrumentFunctions &&
571 !CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining &&
572 !CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
573 return false;
574 if (!CurFuncDecl || CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>())
575 return false;
576 return true;
577}
578
579bool CodeGenFunction::ShouldSkipSanitizerInstrumentation() {
580 if (!CurFuncDecl)
581 return false;
582 return CurFuncDecl->hasAttr<DisableSanitizerInstrumentationAttr>();
583}
584
585/// ShouldXRayInstrument - Return true if the current function should be
586/// instrumented with XRay nop sleds.
587bool CodeGenFunction::ShouldXRayInstrumentFunction() const {
588 return CGM.getCodeGenOpts().XRayInstrumentFunctions;
589}
590
591/// AlwaysEmitXRayCustomEvents - Return true if we should emit IR for calls to
592/// the __xray_customevent(...) builtin calls, when doing XRay instrumentation.
593bool CodeGenFunction::AlwaysEmitXRayCustomEvents() const {
594 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
595 (CGM.getCodeGenOpts().XRayAlwaysEmitCustomEvents ||
596 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
597 XRayInstrKind::Custom);
598}
599
600bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const {
601 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
602 (CGM.getCodeGenOpts().XRayAlwaysEmitTypedEvents ||
603 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
604 XRayInstrKind::Typed);
605}
606
607llvm::ConstantInt *
608CodeGenFunction::getUBSanFunctionTypeHash(QualType Ty) const {
609 // Remove any (C++17) exception specifications, to allow calling e.g. a
610 // noexcept function through a non-noexcept pointer.
611 if (!Ty->isFunctionNoProtoType())
612 Ty = getContext().getFunctionTypeWithExceptionSpec(Orig: Ty, ESI: EST_None);
613 std::string Mangled;
614 llvm::raw_string_ostream Out(Mangled);
615 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: Ty, Out, NormalizeIntegers: false);
616 return llvm::ConstantInt::get(
617 Ty: CGM.Int32Ty, V: static_cast<uint32_t>(llvm::xxh3_64bits(data: Mangled)));
618}
619
620void CodeGenFunction::EmitKernelMetadata(const FunctionDecl *FD,
621 llvm::Function *Fn) {
622 if (!FD->hasAttr<DeviceKernelAttr>() && !FD->hasAttr<CUDAGlobalAttr>())
623 return;
624
625 llvm::LLVMContext &Context = getLLVMContext();
626
627 CGM.GenKernelArgMetadata(FN: Fn, FD, CGF: this);
628
629 if (!(getLangOpts().OpenCL ||
630 (getLangOpts().CUDA &&
631 getContext().getTargetInfo().getTriple().isSPIRV())))
632 return;
633
634 if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {
635 QualType HintQTy = A->getTypeHint();
636 const ExtVectorType *HintEltQTy = HintQTy->getAs<ExtVectorType>();
637 bool IsSignedInteger =
638 HintQTy->isSignedIntegerType() ||
639 (HintEltQTy && HintEltQTy->getElementType()->isSignedIntegerType());
640 llvm::Metadata *AttrMDArgs[] = {
641 llvm::ConstantAsMetadata::get(C: llvm::PoisonValue::get(
642 T: CGM.getTypes().ConvertType(T: A->getTypeHint()))),
643 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
644 Ty: llvm::IntegerType::get(C&: Context, NumBits: 32),
645 V: llvm::APInt(32, (uint64_t)(IsSignedInteger ? 1 : 0))))};
646 Fn->setMetadata(Kind: "vec_type_hint", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
647 }
648
649 if (const WorkGroupSizeHintAttr *A = FD->getAttr<WorkGroupSizeHintAttr>()) {
650 auto Eval = [&](Expr *E) {
651 return E->EvaluateKnownConstInt(Ctx: FD->getASTContext()).getExtValue();
652 };
653 llvm::Metadata *AttrMDArgs[] = {
654 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getXDim()))),
655 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getYDim()))),
656 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getZDim())))};
657 Fn->setMetadata(Kind: "work_group_size_hint", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
658 }
659
660 if (const ReqdWorkGroupSizeAttr *A = FD->getAttr<ReqdWorkGroupSizeAttr>()) {
661 auto Eval = [&](Expr *E) {
662 return E->EvaluateKnownConstInt(Ctx: FD->getASTContext()).getExtValue();
663 };
664 llvm::Metadata *AttrMDArgs[] = {
665 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getXDim()))),
666 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getYDim()))),
667 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getZDim())))};
668 Fn->setMetadata(Kind: "reqd_work_group_size", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
669 }
670
671 if (const OpenCLIntelReqdSubGroupSizeAttr *A =
672 FD->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
673 llvm::Metadata *AttrMDArgs[] = {
674 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: A->getSubGroupSize()))};
675 Fn->setMetadata(Kind: "intel_reqd_sub_group_size",
676 Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
677 }
678}
679
680/// Determine whether the function F ends with a return stmt.
681static bool endsWithReturn(const Decl* F) {
682 const Stmt *Body = nullptr;
683 if (auto *FD = dyn_cast_or_null<FunctionDecl>(Val: F))
684 Body = FD->getBody();
685 else if (auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(Val: F))
686 Body = OMD->getBody();
687
688 if (auto *CS = dyn_cast_or_null<CompoundStmt>(Val: Body)) {
689 auto LastStmt = CS->body_rbegin();
690 if (LastStmt != CS->body_rend())
691 return isa<ReturnStmt>(Val: *LastStmt);
692 }
693 return false;
694}
695
696void CodeGenFunction::markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {
697 if (SanOpts.has(K: SanitizerKind::Thread)) {
698 Fn->addFnAttr(Kind: "sanitize_thread_no_checking_at_run_time");
699 Fn->removeFnAttr(Kind: llvm::Attribute::SanitizeThread);
700 }
701}
702
703/// Check if the return value of this function requires sanitization.
704bool CodeGenFunction::requiresReturnValueCheck() const {
705 return requiresReturnValueNullabilityCheck() ||
706 (SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute) && CurCodeDecl &&
707 CurCodeDecl->getAttr<ReturnsNonNullAttr>());
708}
709
710static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) {
711 auto *MD = dyn_cast_or_null<CXXMethodDecl>(Val: D);
712 if (!MD || !MD->getDeclName().getAsIdentifierInfo() ||
713 !MD->getDeclName().getAsIdentifierInfo()->isStr(Str: "allocate") ||
714 (MD->getNumParams() != 1 && MD->getNumParams() != 2))
715 return false;
716
717 if (!Ctx.hasSameType(T1: MD->parameters()[0]->getType(), T2: Ctx.getSizeType()))
718 return false;
719
720 if (MD->getNumParams() == 2) {
721 auto *PT = MD->parameters()[1]->getType()->getAs<PointerType>();
722 if (!PT || !PT->isVoidPointerType() ||
723 !PT->getPointeeType().isConstQualified())
724 return false;
725 }
726
727 return true;
728}
729
730bool CodeGenFunction::isInAllocaArgument(CGCXXABI &ABI, QualType Ty) {
731 const CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
732 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
733}
734
735bool CodeGenFunction::hasInAllocaArg(const CXXMethodDecl *MD) {
736 return getTarget().getTriple().getArch() == llvm::Triple::x86 &&
737 getTarget().getCXXABI().isMicrosoft() &&
738 llvm::any_of(Range: MD->parameters(), P: [&](ParmVarDecl *P) {
739 return isInAllocaArgument(ABI&: CGM.getCXXABI(), Ty: P->getType());
740 });
741}
742
743/// Return the UBSan prologue signature for \p FD if one is available.
744static llvm::Constant *getPrologueSignature(CodeGenModule &CGM,
745 const FunctionDecl *FD) {
746 if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: FD))
747 if (!MD->isStatic())
748 return nullptr;
749 return CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM);
750}
751
752void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
753 llvm::Function *Fn,
754 const CGFunctionInfo &FnInfo,
755 const FunctionArgList &Args,
756 SourceLocation Loc,
757 SourceLocation StartLoc) {
758 assert(!CurFn &&
759 "Do not use a CodeGenFunction object for more than one function");
760
761 const Decl *D = GD.getDecl();
762
763 DidCallStackSave = false;
764 CurCodeDecl = D;
765 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
766 if (FD && FD->usesSEHTry())
767 CurSEHParent = GD;
768 CurFuncDecl = (D ? D->getNonClosureContext() : nullptr);
769 FnRetTy = RetTy;
770 CurFn = Fn;
771 CurFnInfo = &FnInfo;
772 assert(CurFn->isDeclaration() && "Function already has body?");
773
774 // If this function is ignored for any of the enabled sanitizers,
775 // disable the sanitizer for the function.
776 do {
777#define SANITIZER(NAME, ID) \
778 if (SanOpts.empty()) \
779 break; \
780 if (SanOpts.has(SanitizerKind::ID)) \
781 if (CGM.isInNoSanitizeList(SanitizerKind::ID, Fn, Loc)) \
782 SanOpts.set(SanitizerKind::ID, false);
783
784#include "clang/Basic/Sanitizers.def"
785#undef SANITIZER
786 } while (false);
787
788 if (D) {
789 const bool SanitizeBounds = SanOpts.hasOneOf(K: SanitizerKind::Bounds);
790 SanitizerMask no_sanitize_mask;
791 bool NoSanitizeCoverage = false;
792
793 for (auto *Attr : D->specific_attrs<NoSanitizeAttr>()) {
794 no_sanitize_mask |= Attr->getMask();
795 // SanitizeCoverage is not handled by SanOpts.
796 if (Attr->hasCoverage())
797 NoSanitizeCoverage = true;
798 }
799
800 // Apply the no_sanitize* attributes to SanOpts.
801 SanOpts.Mask &= ~no_sanitize_mask;
802 if (no_sanitize_mask & SanitizerKind::Address)
803 SanOpts.set(K: SanitizerKind::KernelAddress, Value: false);
804 if (no_sanitize_mask & SanitizerKind::KernelAddress)
805 SanOpts.set(K: SanitizerKind::Address, Value: false);
806 if (no_sanitize_mask & SanitizerKind::HWAddress)
807 SanOpts.set(K: SanitizerKind::KernelHWAddress, Value: false);
808 if (no_sanitize_mask & SanitizerKind::KernelHWAddress)
809 SanOpts.set(K: SanitizerKind::HWAddress, Value: false);
810
811 if (SanitizeBounds && !SanOpts.hasOneOf(K: SanitizerKind::Bounds))
812 Fn->addFnAttr(Kind: llvm::Attribute::NoSanitizeBounds);
813
814 if (NoSanitizeCoverage && CGM.getCodeGenOpts().hasSanitizeCoverage())
815 Fn->addFnAttr(Kind: llvm::Attribute::NoSanitizeCoverage);
816
817 // Some passes need the non-negated no_sanitize attribute. Pass them on.
818 if (CGM.getCodeGenOpts().hasSanitizeBinaryMetadata()) {
819 if (no_sanitize_mask & SanitizerKind::Thread)
820 Fn->addFnAttr(Kind: "no_sanitize_thread");
821 }
822 }
823
824 if (ShouldSkipSanitizerInstrumentation()) {
825 CurFn->addFnAttr(Kind: llvm::Attribute::DisableSanitizerInstrumentation);
826 } else {
827 // Apply sanitizer attributes to the function.
828 if (SanOpts.hasOneOf(K: SanitizerKind::Address | SanitizerKind::KernelAddress))
829 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAddress);
830 if (SanOpts.hasOneOf(K: SanitizerKind::HWAddress |
831 SanitizerKind::KernelHWAddress))
832 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeHWAddress);
833 if (SanOpts.has(K: SanitizerKind::MemtagStack))
834 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemTag);
835 if (SanOpts.has(K: SanitizerKind::Thread))
836 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeThread);
837 if (SanOpts.has(K: SanitizerKind::Type))
838 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeType);
839 if (SanOpts.has(K: SanitizerKind::NumericalStability))
840 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeNumericalStability);
841 if (SanOpts.hasOneOf(K: SanitizerKind::Memory | SanitizerKind::KernelMemory))
842 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemory);
843 if (SanOpts.has(K: SanitizerKind::AllocToken))
844 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAllocToken);
845 }
846 if (SanOpts.has(K: SanitizerKind::SafeStack))
847 Fn->addFnAttr(Kind: llvm::Attribute::SafeStack);
848 if (SanOpts.has(K: SanitizerKind::ShadowCallStack))
849 Fn->addFnAttr(Kind: llvm::Attribute::ShadowCallStack);
850
851 if (SanOpts.has(K: SanitizerKind::Realtime))
852 if (FD && FD->getASTContext().hasAnyFunctionEffects())
853 for (const FunctionEffectWithCondition &Fe : FD->getFunctionEffects()) {
854 if (Fe.Effect.kind() == FunctionEffect::Kind::NonBlocking)
855 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeRealtime);
856 else if (Fe.Effect.kind() == FunctionEffect::Kind::Blocking)
857 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeRealtimeBlocking);
858 }
859
860 // Apply fuzzing attribute to the function.
861 if (SanOpts.hasOneOf(K: SanitizerKind::Fuzzer | SanitizerKind::FuzzerNoLink))
862 Fn->addFnAttr(Kind: llvm::Attribute::OptForFuzzing);
863
864 // Ignore TSan memory acesses from within ObjC/ObjC++ dealloc, initialize,
865 // .cxx_destruct, __destroy_helper_block_ and all of their calees at run time.
866 if (SanOpts.has(K: SanitizerKind::Thread)) {
867 if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(Val: D)) {
868 const IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(argIndex: 0);
869 if (OMD->getMethodFamily() == OMF_dealloc ||
870 OMD->getMethodFamily() == OMF_initialize ||
871 (OMD->getSelector().isUnarySelector() && II->isStr(Str: ".cxx_destruct"))) {
872 markAsIgnoreThreadCheckingAtRuntime(Fn);
873 }
874 }
875 }
876
877 // Ignore unrelated casts in STL allocate() since the allocator must cast
878 // from void* to T* before object initialization completes. Don't match on the
879 // namespace because not all allocators are in std::
880 if (D && SanOpts.has(K: SanitizerKind::CFIUnrelatedCast)) {
881 if (matchesStlAllocatorFn(D, Ctx: getContext()))
882 SanOpts.Mask &= ~SanitizerKind::CFIUnrelatedCast;
883 }
884
885 // Ignore null checks in coroutine functions since the coroutines passes
886 // are not aware of how to move the extra UBSan instructions across the split
887 // coroutine boundaries.
888 if (D && SanOpts.has(K: SanitizerKind::Null))
889 if (FD && FD->getBody() &&
890 FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass)
891 SanOpts.Mask &= ~SanitizerKind::Null;
892
893 // Apply xray attributes to the function (as a string, for now)
894 bool AlwaysXRayAttr = false;
895 if (const auto *XRayAttr = D ? D->getAttr<XRayInstrumentAttr>() : nullptr) {
896 if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
897 K: XRayInstrKind::FunctionEntry) ||
898 CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
899 K: XRayInstrKind::FunctionExit)) {
900 if (XRayAttr->alwaysXRayInstrument() && ShouldXRayInstrumentFunction()) {
901 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-always");
902 AlwaysXRayAttr = true;
903 }
904 if (XRayAttr->neverXRayInstrument())
905 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-never");
906 if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())
907 if (ShouldXRayInstrumentFunction())
908 Fn->addFnAttr(Kind: "xray-log-args",
909 Val: llvm::utostr(X: LogArgs->getArgumentCount()));
910 }
911 } else {
912 if (ShouldXRayInstrumentFunction() && !CGM.imbueXRayAttrs(Fn, Loc))
913 Fn->addFnAttr(
914 Kind: "xray-instruction-threshold",
915 Val: llvm::itostr(X: CGM.getCodeGenOpts().XRayInstructionThreshold));
916 }
917
918 if (ShouldXRayInstrumentFunction()) {
919 if (CGM.getCodeGenOpts().XRayIgnoreLoops)
920 Fn->addFnAttr(Kind: "xray-ignore-loops");
921
922 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
923 K: XRayInstrKind::FunctionExit))
924 Fn->addFnAttr(Kind: "xray-skip-exit");
925
926 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
927 K: XRayInstrKind::FunctionEntry))
928 Fn->addFnAttr(Kind: "xray-skip-entry");
929
930 auto FuncGroups = CGM.getCodeGenOpts().XRayTotalFunctionGroups;
931 if (FuncGroups > 1) {
932 auto FuncName = llvm::ArrayRef<uint8_t>(CurFn->getName().bytes_begin(),
933 CurFn->getName().bytes_end());
934 auto Group = crc32(Data: FuncName) % FuncGroups;
935 if (Group != CGM.getCodeGenOpts().XRaySelectedFunctionGroup &&
936 !AlwaysXRayAttr)
937 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-never");
938 }
939 }
940
941 if (CGM.getCodeGenOpts().getProfileInstr() !=
942 llvm::driver::ProfileInstrKind::ProfileNone) {
943 switch (CGM.isFunctionBlockedFromProfileInstr(Fn, Loc)) {
944 case ProfileList::Skip:
945 Fn->addFnAttr(Kind: llvm::Attribute::SkipProfile);
946 break;
947 case ProfileList::Forbid:
948 Fn->addFnAttr(Kind: llvm::Attribute::NoProfile);
949 break;
950 case ProfileList::Allow:
951 break;
952 }
953 }
954
955 unsigned Count, Offset;
956 StringRef Section;
957 if (const auto *Attr =
958 D ? D->getAttr<PatchableFunctionEntryAttr>() : nullptr) {
959 Count = Attr->getCount();
960 Offset = Attr->getOffset();
961 Section = Attr->getSection();
962 } else {
963 Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;
964 Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;
965 }
966 if (Section.empty())
967 Section = CGM.getCodeGenOpts().PatchableFunctionEntrySection;
968 if (Count && Offset <= Count) {
969 Fn->addFnAttr(Kind: "patchable-function-entry", Val: std::to_string(val: Count - Offset));
970 if (Offset)
971 Fn->addFnAttr(Kind: "patchable-function-prefix", Val: std::to_string(val: Offset));
972 if (!Section.empty())
973 Fn->addFnAttr(Kind: "patchable-function-entry-section", Val: Section);
974 }
975 // Instruct that functions for COFF/CodeView targets should start with a
976 // patchable instruction, but only on x86/x64. Don't forward this to ARM/ARM64
977 // backends as they don't need it -- instructions on these architectures are
978 // always atomically patchable at runtime.
979 if (CGM.getCodeGenOpts().HotPatch &&
980 getContext().getTargetInfo().getTriple().isX86() &&
981 getContext().getTargetInfo().getTriple().getEnvironment() !=
982 llvm::Triple::CODE16)
983 Fn->addFnAttr(Kind: "patchable-function", Val: "prologue-short-redirect");
984
985 // Add no-jump-tables value.
986 if (CGM.getCodeGenOpts().NoUseJumpTables)
987 Fn->addFnAttr(Kind: "no-jump-tables", Val: "true");
988
989 // Add no-inline-line-tables value.
990 if (CGM.getCodeGenOpts().NoInlineLineTables)
991 Fn->addFnAttr(Kind: "no-inline-line-tables");
992
993 // Add profile-sample-accurate value.
994 if (CGM.getCodeGenOpts().ProfileSampleAccurate)
995 Fn->addFnAttr(Kind: "profile-sample-accurate");
996
997 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
998 Fn->addFnAttr(Kind: "use-sample-profile");
999
1000 if (D && D->hasAttr<CFICanonicalJumpTableAttr>())
1001 Fn->addFnAttr(Kind: "cfi-canonical-jump-table");
1002
1003 if (D && D->hasAttr<NoProfileFunctionAttr>())
1004 Fn->addFnAttr(Kind: llvm::Attribute::NoProfile);
1005
1006 if (D && D->hasAttr<HybridPatchableAttr>())
1007 Fn->addFnAttr(Kind: llvm::Attribute::HybridPatchable);
1008
1009 if (D) {
1010 // Function attributes take precedence over command line flags.
1011 if (auto *A = D->getAttr<FunctionReturnThunksAttr>()) {
1012 switch (A->getThunkType()) {
1013 case FunctionReturnThunksAttr::Kind::Keep:
1014 break;
1015 case FunctionReturnThunksAttr::Kind::Extern:
1016 Fn->addFnAttr(Kind: llvm::Attribute::FnRetThunkExtern);
1017 break;
1018 }
1019 } else if (CGM.getCodeGenOpts().FunctionReturnThunks)
1020 Fn->addFnAttr(Kind: llvm::Attribute::FnRetThunkExtern);
1021 }
1022
1023 if (FD && (getLangOpts().OpenCL ||
1024 (getLangOpts().CUDA &&
1025 getContext().getTargetInfo().getTriple().isSPIRV()) ||
1026 ((getLangOpts().HIP || getLangOpts().OffloadViaLLVM) &&
1027 getLangOpts().CUDAIsDevice))) {
1028 // Add metadata for a kernel function.
1029 EmitKernelMetadata(FD, Fn);
1030 }
1031
1032 if (FD && FD->hasAttr<ClspvLibclcBuiltinAttr>()) {
1033 Fn->setMetadata(Kind: "clspv_libclc_builtin",
1034 Node: llvm::MDNode::get(Context&: getLLVMContext(), MDs: {}));
1035 }
1036
1037 // If we are checking function types, emit a function type signature as
1038 // prologue data.
1039 if (FD && SanOpts.has(K: SanitizerKind::Function) &&
1040 !FD->getType()->isCFIUncheckedCalleeFunctionType()) {
1041 if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {
1042 llvm::LLVMContext &Ctx = Fn->getContext();
1043 llvm::MDBuilder MDB(Ctx);
1044 Fn->setMetadata(
1045 KindID: llvm::LLVMContext::MD_func_sanitize,
1046 Node: MDB.createRTTIPointerPrologue(
1047 PrologueSig, RTTI: getUBSanFunctionTypeHash(Ty: FD->getType())));
1048 }
1049 }
1050
1051 // If we're checking nullability, we need to know whether we can check the
1052 // return value. Initialize the flag to 'true' and refine it in EmitParmDecl.
1053 if (SanOpts.has(K: SanitizerKind::NullabilityReturn)) {
1054 auto Nullability = FnRetTy->getNullability();
1055 if (Nullability && *Nullability == NullabilityKind::NonNull &&
1056 !FnRetTy->isRecordType()) {
1057 if (!(SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute) &&
1058 CurCodeDecl && CurCodeDecl->getAttr<ReturnsNonNullAttr>()))
1059 RetValNullabilityPrecondition =
1060 llvm::ConstantInt::getTrue(Context&: getLLVMContext());
1061 }
1062 }
1063
1064 // If we're in C++ mode and the function name is "main", it is guaranteed
1065 // to be norecurse by the standard (3.6.1.3 "The function main shall not be
1066 // used within a program").
1067 //
1068 // OpenCL C 2.0 v2.2-11 s6.9.i:
1069 // Recursion is not supported.
1070 //
1071 // HLSL
1072 // Recursion is not supported.
1073 //
1074 // SYCL v1.2.1 s3.10:
1075 // kernels cannot include RTTI information, exception classes,
1076 // recursive code, virtual functions or make use of C++ libraries that
1077 // are not compiled for the device.
1078 if (FD &&
1079 ((getLangOpts().CPlusPlus && FD->isMain()) || getLangOpts().OpenCL ||
1080 getLangOpts().HLSL || getLangOpts().SYCLIsDevice ||
1081 (getLangOpts().CUDA && FD->hasAttr<CUDAGlobalAttr>())))
1082 Fn->addFnAttr(Kind: llvm::Attribute::NoRecurse);
1083
1084 llvm::RoundingMode RM = getLangOpts().getDefaultRoundingMode();
1085 llvm::fp::ExceptionBehavior FPExceptionBehavior =
1086 ToConstrainedExceptMD(Kind: getLangOpts().getDefaultExceptionMode());
1087 Builder.setDefaultConstrainedRounding(RM);
1088 Builder.setDefaultConstrainedExcept(FPExceptionBehavior);
1089 if ((FD && (FD->UsesFPIntrin() || FD->hasAttr<StrictFPAttr>())) ||
1090 (!FD && (FPExceptionBehavior != llvm::fp::ebIgnore ||
1091 RM != llvm::RoundingMode::NearestTiesToEven))) {
1092 Builder.setIsFPConstrained(true);
1093 Fn->addFnAttr(Kind: llvm::Attribute::StrictFP);
1094 }
1095
1096 // If a custom alignment is used, force realigning to this alignment on
1097 // any main function which certainly will need it.
1098 if (FD && ((FD->isMain() || FD->isMSVCRTEntryPoint()) &&
1099 CGM.getCodeGenOpts().StackAlignment))
1100 Fn->addFnAttr(Kind: "stackrealign");
1101
1102 // "main" doesn't need to zero out call-used registers.
1103 if (FD && FD->isMain())
1104 Fn->removeFnAttr(Kind: "zero-call-used-regs");
1105
1106 // Add vscale_range attribute if appropriate.
1107 llvm::StringMap<bool> FeatureMap;
1108 auto IsArmStreaming = TargetInfo::ArmStreamingKind::NotStreaming;
1109 if (FD) {
1110 getContext().getFunctionFeatureMap(FeatureMap, FD);
1111 if (const auto *T = FD->getType()->getAs<FunctionProtoType>())
1112 if (T->getAArch64SMEAttributes() &
1113 FunctionType::SME_PStateSMCompatibleMask)
1114 IsArmStreaming = TargetInfo::ArmStreamingKind::StreamingCompatible;
1115
1116 if (IsArmStreamingFunction(FD, IncludeLocallyStreaming: true))
1117 IsArmStreaming = TargetInfo::ArmStreamingKind::Streaming;
1118 }
1119 std::optional<std::pair<unsigned, unsigned>> VScaleRange =
1120 getContext().getTargetInfo().getVScaleRange(LangOpts: getLangOpts(), Mode: IsArmStreaming,
1121 FeatureMap: &FeatureMap);
1122 if (VScaleRange) {
1123 CurFn->addFnAttr(Attr: llvm::Attribute::getWithVScaleRangeArgs(
1124 Context&: getLLVMContext(), MinValue: VScaleRange->first, MaxValue: VScaleRange->second));
1125 }
1126
1127 llvm::BasicBlock *EntryBB = createBasicBlock(name: "entry", parent: CurFn);
1128
1129 // Create a marker to make it easy to insert allocas into the entryblock
1130 // later. Don't create this with the builder, because we don't want it
1131 // folded.
1132 llvm::Value *Poison = llvm::PoisonValue::get(T: Int32Ty);
1133 AllocaInsertPt = new llvm::BitCastInst(Poison, Int32Ty, "allocapt", EntryBB);
1134
1135 ReturnBlock = getJumpDestInCurrentScope(Name: "return");
1136
1137 Builder.SetInsertPoint(EntryBB);
1138
1139 // If we're checking the return value, allocate space for a pointer to a
1140 // precise source location of the checked return statement.
1141 if (requiresReturnValueCheck()) {
1142 ReturnLocation = CreateDefaultAlignTempAlloca(Ty: Int8PtrTy, Name: "return.sloc.ptr");
1143 Builder.CreateStore(Val: llvm::ConstantPointerNull::get(T: Int8PtrTy),
1144 Addr: ReturnLocation);
1145 }
1146
1147 // Emit subprogram debug descriptor.
1148 if (CGDebugInfo *DI = getDebugInfo()) {
1149 // Reconstruct the type from the argument list so that implicit parameters,
1150 // such as 'this' and 'vtt', show up in the debug info. Preserve the calling
1151 // convention.
1152 DI->emitFunctionStart(GD, Loc, ScopeLoc: StartLoc,
1153 FnType: DI->getFunctionType(FD, RetTy, Args), Fn: CurFn,
1154 CurFnIsThunk: CurFuncIsThunk);
1155 }
1156
1157 if (ShouldInstrumentFunction()) {
1158 if (CGM.getCodeGenOpts().InstrumentFunctions)
1159 CurFn->addFnAttr(Kind: "instrument-function-entry", Val: "__cyg_profile_func_enter");
1160 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
1161 CurFn->addFnAttr(Kind: "instrument-function-entry-inlined",
1162 Val: "__cyg_profile_func_enter");
1163 if (CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
1164 CurFn->addFnAttr(Kind: "instrument-function-entry-inlined",
1165 Val: "__cyg_profile_func_enter_bare");
1166 }
1167
1168 // Since emitting the mcount call here impacts optimizations such as function
1169 // inlining, we just add an attribute to insert a mcount call in backend.
1170 // The attribute "counting-function" is set to mcount function name which is
1171 // architecture dependent.
1172 if (CGM.getCodeGenOpts().InstrumentForProfiling) {
1173 // Calls to fentry/mcount should not be generated if function has
1174 // the no_instrument_function attribute.
1175 if (!CurFuncDecl || !CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()) {
1176 if (CGM.getCodeGenOpts().CallFEntry)
1177 Fn->addFnAttr(Kind: "fentry-call", Val: "true");
1178 else {
1179 Fn->addFnAttr(Kind: "instrument-function-entry-inlined",
1180 Val: getTarget().getMCountName());
1181 }
1182 if (CGM.getCodeGenOpts().MNopMCount) {
1183 if (!CGM.getCodeGenOpts().CallFEntry)
1184 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_without_opt)
1185 << "-mnop-mcount" << "-mfentry";
1186 Fn->addFnAttr(Kind: "mnop-mcount");
1187 }
1188
1189 if (CGM.getCodeGenOpts().RecordMCount) {
1190 if (!CGM.getCodeGenOpts().CallFEntry)
1191 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_without_opt)
1192 << "-mrecord-mcount" << "-mfentry";
1193 Fn->addFnAttr(Kind: "mrecord-mcount");
1194 }
1195 }
1196 }
1197
1198 if (CGM.getCodeGenOpts().PackedStack) {
1199 if (getContext().getTargetInfo().getTriple().getArch() !=
1200 llvm::Triple::systemz)
1201 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_on_target)
1202 << "-mpacked-stack";
1203 Fn->addFnAttr(Kind: "packed-stack");
1204 }
1205
1206 if (CGM.getCodeGenOpts().WarnStackSize != UINT_MAX &&
1207 !CGM.getDiags().isIgnored(DiagID: diag::warn_fe_backend_frame_larger_than, Loc))
1208 Fn->addFnAttr(Kind: "warn-stack-size",
1209 Val: std::to_string(val: CGM.getCodeGenOpts().WarnStackSize));
1210
1211 if (RetTy->isVoidType()) {
1212 // Void type; nothing to return.
1213 ReturnValue = Address::invalid();
1214
1215 // Count the implicit return.
1216 if (!endsWithReturn(F: D))
1217 ++NumReturnExprs;
1218 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1219 // Indirect return; emit returned value directly into sret slot.
1220 // This reduces code size, and affects correctness in C++.
1221 auto AI = CurFn->arg_begin();
1222 if (CurFnInfo->getReturnInfo().isSRetAfterThis())
1223 ++AI;
1224 ReturnValue = makeNaturalAddressForPointer(
1225 Ptr: &*AI, T: RetTy, Alignment: CurFnInfo->getReturnInfo().getIndirectAlign(), ForPointeeType: false,
1226 BaseInfo: nullptr, TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
1227 if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
1228 ReturnValuePointer =
1229 CreateDefaultAlignTempAlloca(Ty: ReturnValue.getType(), Name: "result.ptr");
1230 Builder.CreateStore(Val: ReturnValue.emitRawPointer(CGF&: *this),
1231 Addr: ReturnValuePointer);
1232 }
1233 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
1234 !hasScalarEvaluationKind(T: CurFnInfo->getReturnType())) {
1235 // Load the sret pointer from the argument struct and return into that.
1236 unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();
1237 llvm::Function::arg_iterator EI = CurFn->arg_end();
1238 --EI;
1239 llvm::Value *Addr = Builder.CreateStructGEP(
1240 Ty: CurFnInfo->getArgStruct(), Ptr: &*EI, Idx);
1241 llvm::Type *Ty =
1242 cast<llvm::GetElementPtrInst>(Val: Addr)->getResultElementType();
1243 ReturnValuePointer = Address(Addr, Ty, getPointerAlign());
1244 Addr = Builder.CreateAlignedLoad(Ty, Addr, Align: getPointerAlign(), Name: "agg.result");
1245 ReturnValue = Address(Addr, ConvertType(T: RetTy),
1246 CGM.getNaturalTypeAlignment(T: RetTy), KnownNonNull);
1247 } else {
1248 ReturnValue = CreateIRTempWithoutCast(T: RetTy, Name: "retval");
1249
1250 // Tell the epilog emitter to autorelease the result. We do this
1251 // now so that various specialized functions can suppress it
1252 // during their IR-generation.
1253 if (getLangOpts().ObjCAutoRefCount &&
1254 !CurFnInfo->isReturnsRetained() &&
1255 RetTy->isObjCRetainableType())
1256 AutoreleaseResult = true;
1257 }
1258
1259 EmitStartEHSpec(D: CurCodeDecl);
1260
1261 PrologueCleanupDepth = EHStack.stable_begin();
1262
1263 // Emit OpenMP specific initialization of the device functions.
1264 if (getLangOpts().OpenMP && CurCodeDecl)
1265 CGM.getOpenMPRuntime().emitFunctionProlog(CGF&: *this, D: CurCodeDecl);
1266
1267 if (FD && getLangOpts().HLSL) {
1268 // Handle emitting HLSL entry functions.
1269 if (FD->hasAttr<HLSLShaderAttr>()) {
1270 CGM.getHLSLRuntime().emitEntryFunction(FD, Fn);
1271 }
1272 }
1273
1274 EmitFunctionProlog(FI: *CurFnInfo, Fn: CurFn, Args);
1275
1276 if (const CXXMethodDecl *MD = dyn_cast_if_present<CXXMethodDecl>(Val: D);
1277 MD && !MD->isStatic()) {
1278 bool IsInLambda =
1279 MD->getParent()->isLambda() && MD->getOverloadedOperator() == OO_Call;
1280 if (MD->isImplicitObjectMemberFunction())
1281 CGM.getCXXABI().EmitInstanceFunctionProlog(CGF&: *this);
1282 if (IsInLambda) {
1283 // We're in a lambda; figure out the captures.
1284 MD->getParent()->getCaptureFields(Captures&: LambdaCaptureFields,
1285 ThisCapture&: LambdaThisCaptureField);
1286 if (LambdaThisCaptureField) {
1287 // If the lambda captures the object referred to by '*this' - either by
1288 // value or by reference, make sure CXXThisValue points to the correct
1289 // object.
1290
1291 // Get the lvalue for the field (which is a copy of the enclosing object
1292 // or contains the address of the enclosing object).
1293 LValue ThisFieldLValue = EmitLValueForLambdaField(Field: LambdaThisCaptureField);
1294 if (!LambdaThisCaptureField->getType()->isPointerType()) {
1295 // If the enclosing object was captured by value, just use its
1296 // address. Sign this pointer.
1297 CXXThisValue = ThisFieldLValue.getPointer(CGF&: *this);
1298 } else {
1299 // Load the lvalue pointed to by the field, since '*this' was captured
1300 // by reference.
1301 CXXThisValue =
1302 EmitLoadOfLValue(V: ThisFieldLValue, Loc: SourceLocation()).getScalarVal();
1303 }
1304 }
1305 for (auto *FD : MD->getParent()->fields()) {
1306 if (FD->hasCapturedVLAType()) {
1307 auto *ExprArg = EmitLoadOfLValue(V: EmitLValueForLambdaField(Field: FD),
1308 Loc: SourceLocation()).getScalarVal();
1309 auto VAT = FD->getCapturedVLAType();
1310 VLASizeMap[VAT->getSizeExpr()] = ExprArg;
1311 }
1312 }
1313 } else if (MD->isImplicitObjectMemberFunction()) {
1314 // Not in a lambda; just use 'this' from the method.
1315 // FIXME: Should we generate a new load for each use of 'this'? The
1316 // fast register allocator would be happier...
1317 CXXThisValue = CXXABIThisValue;
1318 }
1319
1320 // Check the 'this' pointer once per function, if it's available.
1321 if (CXXABIThisValue) {
1322 SanitizerSet SkippedChecks;
1323 SkippedChecks.set(K: SanitizerKind::ObjectSize, Value: true);
1324 QualType ThisTy = MD->getThisType();
1325
1326 // If this is the call operator of a lambda with no captures, it
1327 // may have a static invoker function, which may call this operator with
1328 // a null 'this' pointer.
1329 if (isLambdaCallOperator(MD) && MD->getParent()->isCapturelessLambda())
1330 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
1331
1332 EmitTypeCheck(
1333 TCK: isa<CXXConstructorDecl>(Val: MD) ? TCK_ConstructorCall : TCK_MemberCall,
1334 Loc, V: CXXABIThisValue, Type: ThisTy, Alignment: CXXABIThisAlignment, SkippedChecks);
1335 }
1336 }
1337
1338 // If any of the arguments have a variably modified type, make sure to
1339 // emit the type size, but only if the function is not naked. Naked functions
1340 // have no prolog to run this evaluation.
1341 if (!FD || !FD->hasAttr<NakedAttr>()) {
1342 for (const VarDecl *VD : Args) {
1343 // Dig out the type as written from ParmVarDecls; it's unclear whether
1344 // the standard (C99 6.9.1p10) requires this, but we're following the
1345 // precedent set by gcc.
1346 QualType Ty;
1347 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: VD))
1348 Ty = PVD->getOriginalType();
1349 else
1350 Ty = VD->getType();
1351
1352 if (Ty->isVariablyModifiedType())
1353 EmitVariablyModifiedType(Ty);
1354 }
1355 }
1356 // Emit a location at the end of the prologue.
1357 if (CGDebugInfo *DI = getDebugInfo())
1358 DI->EmitLocation(Builder, Loc: StartLoc);
1359 // TODO: Do we need to handle this in two places like we do with
1360 // target-features/target-cpu?
1361 if (CurFuncDecl)
1362 if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>())
1363 LargestVectorWidth = VecWidth->getVectorWidth();
1364
1365 if (CGM.shouldEmitConvergenceTokens())
1366 ConvergenceTokenStack.push_back(Elt: getOrEmitConvergenceEntryToken(F: CurFn));
1367}
1368
1369void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {
1370 incrementProfileCounter(S: Body);
1371 maybeCreateMCDCCondBitmap();
1372 if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Val: Body))
1373 EmitCompoundStmtWithoutScope(S: *S);
1374 else
1375 EmitStmt(S: Body);
1376}
1377
1378/// When instrumenting to collect profile data, the counts for some blocks
1379/// such as switch cases need to not include the fall-through counts, so
1380/// emit a branch around the instrumentation code. When not instrumenting,
1381/// this just calls EmitBlock().
1382void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
1383 const Stmt *S) {
1384 llvm::BasicBlock *SkipCountBB = nullptr;
1385 if (HaveInsertPoint() && CGM.getCodeGenOpts().hasProfileClangInstr()) {
1386 // When instrumenting for profiling, the fallthrough to certain
1387 // statements needs to skip over the instrumentation code so that we
1388 // get an accurate count.
1389 SkipCountBB = createBasicBlock(name: "skipcount");
1390 EmitBranch(Block: SkipCountBB);
1391 }
1392 EmitBlock(BB);
1393 uint64_t CurrentCount = getCurrentProfileCount();
1394 incrementProfileCounter(ExecSkip: UseExecPath, S);
1395 setCurrentProfileCount(getCurrentProfileCount() + CurrentCount);
1396 if (SkipCountBB)
1397 EmitBlock(BB: SkipCountBB);
1398}
1399
1400/// Tries to mark the given function nounwind based on the
1401/// non-existence of any throwing calls within it. We believe this is
1402/// lightweight enough to do at -O0.
1403static void TryMarkNoThrow(llvm::Function *F) {
1404 // LLVM treats 'nounwind' on a function as part of the type, so we
1405 // can't do this on functions that can be overwritten.
1406 if (F->isInterposable()) return;
1407
1408 for (llvm::BasicBlock &BB : *F)
1409 for (llvm::Instruction &I : BB)
1410 if (I.mayThrow())
1411 return;
1412
1413 F->setDoesNotThrow();
1414}
1415
1416QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD,
1417 FunctionArgList &Args) {
1418 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
1419 QualType ResTy = FD->getReturnType();
1420
1421 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD);
1422 if (MD && MD->isImplicitObjectMemberFunction()) {
1423 if (CGM.getCXXABI().HasThisReturn(GD))
1424 ResTy = MD->getThisType();
1425 else if (CGM.getCXXABI().hasMostDerivedReturn(GD))
1426 ResTy = CGM.getContext().VoidPtrTy;
1427 CGM.getCXXABI().buildThisParam(CGF&: *this, Params&: Args);
1428 }
1429
1430 // The base version of an inheriting constructor whose constructed base is a
1431 // virtual base is not passed any arguments (because it doesn't actually call
1432 // the inherited constructor).
1433 bool PassedParams = true;
1434 if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(Val: FD))
1435 if (auto Inherited = CD->getInheritedConstructor())
1436 PassedParams =
1437 getTypes().inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
1438
1439 if (PassedParams) {
1440 for (auto *Param : FD->parameters()) {
1441 Args.push_back(Elt: Param);
1442 if (!Param->hasAttr<PassObjectSizeAttr>())
1443 continue;
1444
1445 auto *Implicit = ImplicitParamDecl::Create(
1446 C&: getContext(), DC: Param->getDeclContext(), IdLoc: Param->getLocation(),
1447 /*Id=*/nullptr, T: getContext().getSizeType(), ParamKind: ImplicitParamKind::Other);
1448 SizeArguments[Param] = Implicit;
1449 Args.push_back(Elt: Implicit);
1450 }
1451 }
1452
1453 if (MD && (isa<CXXConstructorDecl>(Val: MD) || isa<CXXDestructorDecl>(Val: MD)))
1454 CGM.getCXXABI().addImplicitStructorParams(CGF&: *this, ResTy, Params&: Args);
1455
1456 return ResTy;
1457}
1458
1459void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
1460 const CGFunctionInfo &FnInfo) {
1461 assert(Fn && "generating code for null Function");
1462 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
1463 CurGD = GD;
1464
1465 FunctionArgList Args;
1466 QualType ResTy = BuildFunctionArgList(GD, Args);
1467
1468 CGM.getTargetCodeGenInfo().checkFunctionABI(CGM, Decl: FD);
1469
1470 if (FD->isInlineBuiltinDeclaration()) {
1471 // When generating code for a builtin with an inline declaration, use a
1472 // mangled name to hold the actual body, while keeping an external
1473 // definition in case the function pointer is referenced somewhere.
1474 std::string FDInlineName = (Fn->getName() + ".inline").str();
1475 llvm::Module *M = Fn->getParent();
1476 llvm::Function *Clone = M->getFunction(Name: FDInlineName);
1477 if (!Clone) {
1478 Clone = llvm::Function::Create(Ty: Fn->getFunctionType(),
1479 Linkage: llvm::GlobalValue::InternalLinkage,
1480 AddrSpace: Fn->getAddressSpace(), N: FDInlineName, M);
1481 Clone->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1482 }
1483 Fn->setLinkage(llvm::GlobalValue::ExternalLinkage);
1484 Fn = Clone;
1485 } else {
1486 // Detect the unusual situation where an inline version is shadowed by a
1487 // non-inline version. In that case we should pick the external one
1488 // everywhere. That's GCC behavior too. Unfortunately, I cannot find a way
1489 // to detect that situation before we reach codegen, so do some late
1490 // replacement.
1491 for (const FunctionDecl *PD = FD->getPreviousDecl(); PD;
1492 PD = PD->getPreviousDecl()) {
1493 if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) {
1494 std::string FDInlineName = (Fn->getName() + ".inline").str();
1495 llvm::Module *M = Fn->getParent();
1496 if (llvm::Function *Clone = M->getFunction(Name: FDInlineName)) {
1497 Clone->replaceAllUsesWith(V: Fn);
1498 Clone->eraseFromParent();
1499 }
1500 break;
1501 }
1502 }
1503 }
1504
1505 // Check if we should generate debug info for this function.
1506 if (FD->hasAttr<NoDebugAttr>()) {
1507 // Clear non-distinct debug info that was possibly attached to the function
1508 // due to an earlier declaration without the nodebug attribute
1509 Fn->setSubprogram(nullptr);
1510 // Disable debug info indefinitely for this function
1511 DebugInfo = nullptr;
1512 }
1513 // Finalize function debug info on exit.
1514 llvm::scope_exit Cleanup([this] {
1515 if (CGDebugInfo *DI = getDebugInfo())
1516 DI->completeFunction();
1517 });
1518
1519 // The function might not have a body if we're generating thunks for a
1520 // function declaration.
1521 SourceRange BodyRange;
1522 if (Stmt *Body = FD->getBody())
1523 BodyRange = Body->getSourceRange();
1524 else
1525 BodyRange = FD->getLocation();
1526 CurEHLocation = BodyRange.getEnd();
1527
1528 // Use the location of the start of the function to determine where
1529 // the function definition is located. By default use the location
1530 // of the declaration as the location for the subprogram. A function
1531 // may lack a declaration in the source code if it is created by code
1532 // gen. (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).
1533 SourceLocation Loc = FD->getLocation();
1534
1535 // If this is a function specialization then use the pattern body
1536 // as the location for the function.
1537 if (const FunctionDecl *SpecDecl = FD->getTemplateInstantiationPattern())
1538 if (SpecDecl->hasBody(Definition&: SpecDecl))
1539 Loc = SpecDecl->getLocation();
1540
1541 Stmt *Body = FD->getBody();
1542
1543 if (Body) {
1544 // Coroutines always emit lifetime markers.
1545 if (isa<CoroutineBodyStmt>(Val: Body))
1546 ShouldEmitLifetimeMarkers = true;
1547
1548 // Initialize helper which will detect jumps which can cause invalid
1549 // lifetime markers.
1550 if (ShouldEmitLifetimeMarkers)
1551 Bypasses.Init(CGM, Body);
1552 }
1553
1554 // Emit the standard function prologue.
1555 StartFunction(GD, RetTy: ResTy, Fn, FnInfo, Args, Loc, StartLoc: BodyRange.getBegin());
1556
1557 // Save parameters for coroutine function.
1558 if (Body && isa_and_nonnull<CoroutineBodyStmt>(Val: Body))
1559 llvm::append_range(C&: FnArgs, R: FD->parameters());
1560
1561 // Ensure that the function adheres to the forward progress guarantee, which
1562 // is required by certain optimizations.
1563 // In C++11 and up, the attribute will be removed if the body contains a
1564 // trivial empty loop.
1565 if (checkIfFunctionMustProgress())
1566 CurFn->addFnAttr(Kind: llvm::Attribute::MustProgress);
1567
1568 // Generate the body of the function.
1569 PGO->assignRegionCounters(GD, Fn: CurFn);
1570 if (isa<CXXDestructorDecl>(Val: FD))
1571 EmitDestructorBody(Args);
1572 else if (isa<CXXConstructorDecl>(Val: FD))
1573 EmitConstructorBody(Args);
1574 else if (getLangOpts().CUDA &&
1575 !getLangOpts().CUDAIsDevice &&
1576 FD->hasAttr<CUDAGlobalAttr>())
1577 CGM.getCUDARuntime().emitDeviceStub(CGF&: *this, Args);
1578 else if (isa<CXXMethodDecl>(Val: FD) &&
1579 cast<CXXMethodDecl>(Val: FD)->isLambdaStaticInvoker()) {
1580 // The lambda static invoker function is special, because it forwards or
1581 // clones the body of the function call operator (but is actually static).
1582 EmitLambdaStaticInvokeBody(MD: cast<CXXMethodDecl>(Val: FD));
1583 } else if (isa<CXXMethodDecl>(Val: FD) &&
1584 isLambdaCallOperator(MD: cast<CXXMethodDecl>(Val: FD)) &&
1585 !FnInfo.isDelegateCall() &&
1586 cast<CXXMethodDecl>(Val: FD)->getParent()->getLambdaStaticInvoker() &&
1587 hasInAllocaArg(MD: cast<CXXMethodDecl>(Val: FD))) {
1588 // If emitting a lambda with static invoker on X86 Windows, change
1589 // the call operator body.
1590 // Make sure that this is a call operator with an inalloca arg and check
1591 // for delegate call to make sure this is the original call op and not the
1592 // new forwarding function for the static invoker.
1593 EmitLambdaInAllocaCallOpBody(MD: cast<CXXMethodDecl>(Val: FD));
1594 } else if (FD->isDefaulted() && isa<CXXMethodDecl>(Val: FD) &&
1595 (cast<CXXMethodDecl>(Val: FD)->isCopyAssignmentOperator() ||
1596 cast<CXXMethodDecl>(Val: FD)->isMoveAssignmentOperator())) {
1597 // Implicit copy-assignment gets the same special treatment as implicit
1598 // copy-constructors.
1599 emitImplicitAssignmentOperatorBody(Args);
1600 } else if (DeviceKernelAttr::isOpenCLSpelling(
1601 A: FD->getAttr<DeviceKernelAttr>()) &&
1602 GD.getKernelReferenceKind() == KernelReferenceKind::Kernel) {
1603 CallArgList CallArgs;
1604 for (unsigned i = 0; i < Args.size(); ++i) {
1605 Address ArgAddr = GetAddrOfLocalVar(VD: Args[i]);
1606 QualType ArgQualType = Args[i]->getType();
1607 RValue ArgRValue = convertTempToRValue(addr: ArgAddr, type: ArgQualType, Loc);
1608 CallArgs.add(rvalue: ArgRValue, type: ArgQualType);
1609 }
1610 GlobalDecl GDStub = GlobalDecl(FD, KernelReferenceKind::Stub);
1611 const FunctionType *FT = cast<FunctionType>(Val: FD->getType());
1612 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
1613 const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
1614 Args: CallArgs, Ty: FT, /*ChainCall=*/false);
1615 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
1616 llvm::Constant *GDStubFunctionPointer =
1617 CGM.getRawFunctionPointer(GD: GDStub, Ty: FTy);
1618 CGCallee GDStubCallee = CGCallee::forDirect(functionPtr: GDStubFunctionPointer, abstractInfo: GDStub);
1619 EmitCall(CallInfo: FnInfo, Callee: GDStubCallee, ReturnValue: ReturnValueSlot(), Args: CallArgs, CallOrInvoke: nullptr, IsMustTail: false,
1620 Loc);
1621 } else if (Body) {
1622 EmitFunctionBody(Body);
1623 } else
1624 llvm_unreachable("no definition for emitted function");
1625
1626 // C++11 [stmt.return]p2:
1627 // Flowing off the end of a function [...] results in undefined behavior in
1628 // a value-returning function.
1629 // C11 6.9.1p12:
1630 // If the '}' that terminates a function is reached, and the value of the
1631 // function call is used by the caller, the behavior is undefined.
1632 if (getLangOpts().CPlusPlus && !FD->hasImplicitReturnZero() && !SawAsmBlock &&
1633 !FD->getReturnType()->isVoidType() && Builder.GetInsertBlock()) {
1634 bool ShouldEmitUnreachable =
1635 CGM.getCodeGenOpts().StrictReturn ||
1636 !CGM.MayDropFunctionReturn(Context: FD->getASTContext(), ReturnType: FD->getReturnType());
1637 if (SanOpts.has(K: SanitizerKind::Return)) {
1638 auto CheckOrdinal = SanitizerKind::SO_Return;
1639 auto CheckHandler = SanitizerHandler::MissingReturn;
1640 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
1641 llvm::Value *IsFalse = Builder.getFalse();
1642 EmitCheck(Checked: std::make_pair(x&: IsFalse, y&: CheckOrdinal), Check: CheckHandler,
1643 StaticArgs: EmitCheckSourceLocation(Loc: FD->getLocation()), DynamicArgs: {});
1644 } else if (ShouldEmitUnreachable) {
1645 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1646 EmitTrapCall(IntrID: llvm::Intrinsic::trap);
1647 }
1648 if (SanOpts.has(K: SanitizerKind::Return) || ShouldEmitUnreachable) {
1649 Builder.CreateUnreachable();
1650 Builder.ClearInsertionPoint();
1651 }
1652 }
1653
1654 // Emit the standard function epilogue.
1655 FinishFunction(EndLoc: BodyRange.getEnd());
1656
1657 PGO->verifyCounterMap();
1658
1659 // If we haven't marked the function nothrow through other means, do
1660 // a quick pass now to see if we can.
1661 if (!CurFn->doesNotThrow())
1662 TryMarkNoThrow(F: CurFn);
1663}
1664
1665/// ContainsLabel - Return true if the statement contains a label in it. If
1666/// this statement is not executed normally, it not containing a label means
1667/// that we can just remove the code.
1668bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
1669 // Null statement, not a label!
1670 if (!S) return false;
1671
1672 // If this is a label, we have to emit the code, consider something like:
1673 // if (0) { ... foo: bar(); } goto foo;
1674 //
1675 // TODO: If anyone cared, we could track __label__'s, since we know that you
1676 // can't jump to one from outside their declared region.
1677 if (isa<LabelStmt>(Val: S))
1678 return true;
1679
1680 // If this is a case/default statement, and we haven't seen a switch, we have
1681 // to emit the code.
1682 if (isa<SwitchCase>(Val: S) && !IgnoreCaseStmts)
1683 return true;
1684
1685 // If this is a switch statement, we want to ignore cases below it.
1686 if (isa<SwitchStmt>(Val: S))
1687 IgnoreCaseStmts = true;
1688
1689 // Scan subexpressions for verboten labels.
1690 for (const Stmt *SubStmt : S->children())
1691 if (ContainsLabel(S: SubStmt, IgnoreCaseStmts))
1692 return true;
1693
1694 return false;
1695}
1696
1697/// containsBreak - Return true if the statement contains a break out of it.
1698/// If the statement (recursively) contains a switch or loop with a break
1699/// inside of it, this is fine.
1700bool CodeGenFunction::containsBreak(const Stmt *S) {
1701 // Null statement, not a label!
1702 if (!S) return false;
1703
1704 // If this is a switch or loop that defines its own break scope, then we can
1705 // include it and anything inside of it.
1706 if (isa<SwitchStmt>(Val: S) || isa<WhileStmt>(Val: S) || isa<DoStmt>(Val: S) ||
1707 isa<ForStmt>(Val: S))
1708 return false;
1709
1710 if (isa<BreakStmt>(Val: S))
1711 return true;
1712
1713 // Scan subexpressions for verboten breaks.
1714 for (const Stmt *SubStmt : S->children())
1715 if (containsBreak(S: SubStmt))
1716 return true;
1717
1718 return false;
1719}
1720
1721bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) {
1722 if (!S) return false;
1723
1724 // Some statement kinds add a scope and thus never add a decl to the current
1725 // scope. Note, this list is longer than the list of statements that might
1726 // have an unscoped decl nested within them, but this way is conservatively
1727 // correct even if more statement kinds are added.
1728 if (isa<IfStmt>(Val: S) || isa<SwitchStmt>(Val: S) || isa<WhileStmt>(Val: S) ||
1729 isa<DoStmt>(Val: S) || isa<ForStmt>(Val: S) || isa<CompoundStmt>(Val: S) ||
1730 isa<CXXForRangeStmt>(Val: S) || isa<CXXTryStmt>(Val: S) ||
1731 isa<ObjCForCollectionStmt>(Val: S) || isa<ObjCAtTryStmt>(Val: S))
1732 return false;
1733
1734 if (isa<DeclStmt>(Val: S))
1735 return true;
1736
1737 for (const Stmt *SubStmt : S->children())
1738 if (mightAddDeclToScope(S: SubStmt))
1739 return true;
1740
1741 return false;
1742}
1743
1744/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1745/// to a constant, or if it does but contains a label, return false. If it
1746/// constant folds return true and set the boolean result in Result.
1747bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1748 bool &ResultBool,
1749 bool AllowLabels) {
1750 // If MC/DC is enabled, disable folding so that we can instrument all
1751 // conditions to yield complete test vectors. We still keep track of
1752 // folded conditions during region mapping and visualization.
1753 if (!AllowLabels && CGM.getCodeGenOpts().hasProfileClangInstr() &&
1754 CGM.getCodeGenOpts().MCDCCoverage)
1755 return false;
1756
1757 llvm::APSInt ResultInt;
1758 if (!ConstantFoldsToSimpleInteger(Cond, Result&: ResultInt, AllowLabels))
1759 return false;
1760
1761 ResultBool = ResultInt.getBoolValue();
1762 return true;
1763}
1764
1765/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1766/// to a constant, or if it does but contains a label, return false. If it
1767/// constant folds return true and set the folded value.
1768bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1769 llvm::APSInt &ResultInt,
1770 bool AllowLabels) {
1771 // FIXME: Rename and handle conversion of other evaluatable things
1772 // to bool.
1773 Expr::EvalResult Result;
1774 if (!Cond->EvaluateAsInt(Result, Ctx: getContext()))
1775 return false; // Not foldable, not integer or not fully evaluatable.
1776
1777 llvm::APSInt Int = Result.Val.getInt();
1778 if (!AllowLabels && CodeGenFunction::ContainsLabel(S: Cond))
1779 return false; // Contains a label.
1780
1781 PGO->markStmtMaybeUsed(S: Cond);
1782 ResultInt = Int;
1783 return true;
1784}
1785
1786/// Strip parentheses and simplistic logical-NOT operators.
1787const Expr *CodeGenFunction::stripCond(const Expr *C) {
1788 while (true) {
1789 const Expr *SC = IgnoreExprNodes(
1790 E: C, Fns&: IgnoreParensSingleStep, Fns&: IgnoreUOpLNotSingleStep,
1791 Fns&: IgnoreBuiltinExpectSingleStep, Fns&: IgnoreImplicitCastsSingleStep);
1792 if (C == SC)
1793 return SC;
1794 C = SC;
1795 }
1796}
1797
1798/// Determine whether the given condition is an instrumentable condition
1799/// (i.e. no "&&" or "||").
1800bool CodeGenFunction::isInstrumentedCondition(const Expr *C) {
1801 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: stripCond(C));
1802 return (!BOp || !BOp->isLogicalOp());
1803}
1804
1805/// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
1806/// increments a profile counter based on the semantics of the given logical
1807/// operator opcode. This is used to instrument branch condition coverage for
1808/// logical operators.
1809void CodeGenFunction::EmitBranchToCounterBlock(
1810 const Expr *Cond, BinaryOperator::Opcode LOp, llvm::BasicBlock *TrueBlock,
1811 llvm::BasicBlock *FalseBlock, uint64_t TrueCount /* = 0 */,
1812 Stmt::Likelihood LH /* =None */, const Expr *CntrIdx /* = nullptr */) {
1813 // If not instrumenting, just emit a branch.
1814 bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr();
1815 if (!InstrumentRegions || !isInstrumentedCondition(C: Cond))
1816 return EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount, LH);
1817
1818 const Stmt *CntrStmt = (CntrIdx ? CntrIdx : Cond);
1819
1820 llvm::BasicBlock *ThenBlock = nullptr;
1821 llvm::BasicBlock *ElseBlock = nullptr;
1822 llvm::BasicBlock *NextBlock = nullptr;
1823
1824 // Create the block we'll use to increment the appropriate counter.
1825 llvm::BasicBlock *CounterIncrBlock = createBasicBlock(name: "lop.rhscnt");
1826
1827 llvm::BasicBlock *SkipIncrBlock =
1828 (hasSkipCounter(S: CntrStmt) ? createBasicBlock(name: "lop.rhsskip") : nullptr);
1829 llvm::BasicBlock *SkipNextBlock = nullptr;
1830
1831 // Set block pointers according to Logical-AND (BO_LAnd) semantics. This
1832 // means we need to evaluate the condition and increment the counter on TRUE:
1833 //
1834 // if (Cond)
1835 // goto CounterIncrBlock;
1836 // else
1837 // goto FalseBlock;
1838 //
1839 // CounterIncrBlock:
1840 // Counter++;
1841 // goto TrueBlock;
1842
1843 if (LOp == BO_LAnd) {
1844 SkipNextBlock = FalseBlock;
1845 ThenBlock = CounterIncrBlock;
1846 ElseBlock = (SkipIncrBlock ? SkipIncrBlock : SkipNextBlock);
1847 NextBlock = TrueBlock;
1848 }
1849
1850 // Set block pointers according to Logical-OR (BO_LOr) semantics. This means
1851 // we need to evaluate the condition and increment the counter on FALSE:
1852 //
1853 // if (Cond)
1854 // goto TrueBlock;
1855 // else
1856 // goto CounterIncrBlock;
1857 //
1858 // CounterIncrBlock:
1859 // Counter++;
1860 // goto FalseBlock;
1861
1862 else if (LOp == BO_LOr) {
1863 SkipNextBlock = TrueBlock;
1864 ThenBlock = (SkipIncrBlock ? SkipIncrBlock : SkipNextBlock);
1865 ElseBlock = CounterIncrBlock;
1866 NextBlock = FalseBlock;
1867 } else {
1868 llvm_unreachable("Expected Opcode must be that of a Logical Operator");
1869 }
1870
1871 // Emit Branch based on condition.
1872 EmitBranchOnBoolExpr(Cond, TrueBlock: ThenBlock, FalseBlock: ElseBlock, TrueCount, LH);
1873
1874 if (SkipIncrBlock) {
1875 EmitBlock(BB: SkipIncrBlock);
1876 incrementProfileCounter(ExecSkip: UseSkipPath, S: CntrStmt);
1877 EmitBranch(Block: SkipNextBlock);
1878 }
1879
1880 // Emit the block containing the counter increment(s).
1881 EmitBlock(BB: CounterIncrBlock);
1882
1883 // Increment corresponding counter; if index not provided, use Cond as index.
1884 incrementProfileCounter(ExecSkip: UseExecPath, S: CntrStmt);
1885
1886 // Go to the next block.
1887 EmitBranch(Block: NextBlock);
1888}
1889
1890/// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an if
1891/// statement) to the specified blocks. Based on the condition, this might try
1892/// to simplify the codegen of the conditional based on the branch.
1893/// \param LH The value of the likelihood attribute on the True branch.
1894/// \param ConditionalOp Used by MC/DC code coverage to track the result of the
1895/// ConditionalOperator (ternary) through a recursive call for the operator's
1896/// LHS and RHS nodes.
1897void CodeGenFunction::EmitBranchOnBoolExpr(
1898 const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock,
1899 uint64_t TrueCount, Stmt::Likelihood LH, const Expr *ConditionalOp,
1900 const VarDecl *ConditionalDecl) {
1901 Cond = Cond->IgnoreParens();
1902
1903 if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Val: Cond)) {
1904 bool HasSkip = hasSkipCounter(S: CondBOp);
1905
1906 // Handle X && Y in a condition.
1907 if (CondBOp->getOpcode() == BO_LAnd) {
1908 // If we have "1 && X", simplify the code. "0 && X" would have constant
1909 // folded if the case was simple enough.
1910 bool ConstantBool = false;
1911 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getLHS(), ResultBool&: ConstantBool) &&
1912 ConstantBool) {
1913 // br(1 && X) -> br(X).
1914 incrementProfileCounter(S: CondBOp);
1915 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LAnd, TrueBlock,
1916 FalseBlock, TrueCount, LH);
1917 return;
1918 }
1919
1920 // If we have "X && 1", simplify the code to use an uncond branch.
1921 // "X && 0" would have been constant folded to 0.
1922 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getRHS(), ResultBool&: ConstantBool) &&
1923 ConstantBool) {
1924 // br(X && 1) -> br(X).
1925 EmitBranchToCounterBlock(Cond: CondBOp->getLHS(), LOp: BO_LAnd, TrueBlock,
1926 FalseBlock, TrueCount, LH, CntrIdx: CondBOp);
1927 return;
1928 }
1929
1930 // Emit the LHS as a conditional. If the LHS conditional is false, we
1931 // want to jump to the FalseBlock.
1932 llvm::BasicBlock *LHSTrue = createBasicBlock(name: "land.lhs.true");
1933 llvm::BasicBlock *LHSFalse =
1934 (HasSkip ? createBasicBlock(name: "land.lhsskip") : FalseBlock);
1935 // The counter tells us how often we evaluate RHS, and all of TrueCount
1936 // can be propagated to that branch.
1937 uint64_t RHSCount = getProfileCount(S: CondBOp->getRHS());
1938
1939 ConditionalEvaluation eval(*this);
1940 {
1941 ApplyDebugLocation DL(*this, Cond);
1942 // Propagate the likelihood attribute like __builtin_expect
1943 // __builtin_expect(X && Y, 1) -> X and Y are likely
1944 // __builtin_expect(X && Y, 0) -> only Y is unlikely
1945 EmitBranchOnBoolExpr(Cond: CondBOp->getLHS(), TrueBlock: LHSTrue, FalseBlock: LHSFalse, TrueCount: RHSCount,
1946 LH: LH == Stmt::LH_Unlikely ? Stmt::LH_None : LH);
1947 if (HasSkip) {
1948 EmitBlock(BB: LHSFalse);
1949 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondBOp);
1950 EmitBranch(Block: FalseBlock);
1951 }
1952 EmitBlock(BB: LHSTrue);
1953 }
1954
1955 incrementProfileCounter(ExecSkip: UseExecPath, S: CondBOp);
1956 setCurrentProfileCount(getProfileCount(S: CondBOp->getRHS()));
1957
1958 // Any temporaries created here are conditional.
1959 eval.begin(CGF&: *this);
1960 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LAnd, TrueBlock,
1961 FalseBlock, TrueCount, LH);
1962 eval.end(CGF&: *this);
1963 return;
1964 }
1965
1966 if (CondBOp->getOpcode() == BO_LOr) {
1967 // If we have "0 || X", simplify the code. "1 || X" would have constant
1968 // folded if the case was simple enough.
1969 bool ConstantBool = false;
1970 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getLHS(), ResultBool&: ConstantBool) &&
1971 !ConstantBool) {
1972 // br(0 || X) -> br(X).
1973 incrementProfileCounter(S: CondBOp);
1974 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LOr, TrueBlock,
1975 FalseBlock, TrueCount, LH);
1976 return;
1977 }
1978
1979 // If we have "X || 0", simplify the code to use an uncond branch.
1980 // "X || 1" would have been constant folded to 1.
1981 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getRHS(), ResultBool&: ConstantBool) &&
1982 !ConstantBool) {
1983 // br(X || 0) -> br(X).
1984 EmitBranchToCounterBlock(Cond: CondBOp->getLHS(), LOp: BO_LOr, TrueBlock,
1985 FalseBlock, TrueCount, LH, CntrIdx: CondBOp);
1986 return;
1987 }
1988 // Emit the LHS as a conditional. If the LHS conditional is true, we
1989 // want to jump to the TrueBlock.
1990 llvm::BasicBlock *LHSTrue =
1991 (HasSkip ? createBasicBlock(name: "lor.lhsskip") : TrueBlock);
1992 llvm::BasicBlock *LHSFalse = createBasicBlock(name: "lor.lhs.false");
1993 // We have the count for entry to the RHS and for the whole expression
1994 // being true, so we can divy up True count between the short circuit and
1995 // the RHS.
1996 uint64_t LHSCount =
1997 getCurrentProfileCount() - getProfileCount(S: CondBOp->getRHS());
1998 uint64_t RHSCount = TrueCount - LHSCount;
1999
2000 ConditionalEvaluation eval(*this);
2001 {
2002 // Propagate the likelihood attribute like __builtin_expect
2003 // __builtin_expect(X || Y, 1) -> only Y is likely
2004 // __builtin_expect(X || Y, 0) -> both X and Y are unlikely
2005 ApplyDebugLocation DL(*this, Cond);
2006 EmitBranchOnBoolExpr(Cond: CondBOp->getLHS(), TrueBlock: LHSTrue, FalseBlock: LHSFalse, TrueCount: LHSCount,
2007 LH: LH == Stmt::LH_Likely ? Stmt::LH_None : LH);
2008 if (HasSkip) {
2009 EmitBlock(BB: LHSTrue);
2010 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondBOp);
2011 EmitBranch(Block: TrueBlock);
2012 }
2013 EmitBlock(BB: LHSFalse);
2014 }
2015
2016 incrementProfileCounter(ExecSkip: UseExecPath, S: CondBOp);
2017 setCurrentProfileCount(getProfileCount(S: CondBOp->getRHS()));
2018
2019 // Any temporaries created here are conditional.
2020 eval.begin(CGF&: *this);
2021 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LOr, TrueBlock, FalseBlock,
2022 TrueCount: RHSCount, LH);
2023
2024 eval.end(CGF&: *this);
2025 return;
2026 }
2027 }
2028
2029 if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Val: Cond)) {
2030 // br(!x, t, f) -> br(x, f, t)
2031 // Avoid doing this optimization when instrumenting a condition for MC/DC.
2032 // LNot is taken as part of the condition for simplicity, and changing its
2033 // sense negatively impacts test vector tracking.
2034 bool MCDCCondition = CGM.getCodeGenOpts().hasProfileClangInstr() &&
2035 CGM.getCodeGenOpts().MCDCCoverage &&
2036 isInstrumentedCondition(C: Cond);
2037 if (CondUOp->getOpcode() == UO_LNot && !MCDCCondition) {
2038 // Negate the count.
2039 uint64_t FalseCount = getCurrentProfileCount() - TrueCount;
2040 // The values of the enum are chosen to make this negation possible.
2041 LH = static_cast<Stmt::Likelihood>(-LH);
2042 // Negate the condition and swap the destination blocks.
2043 return EmitBranchOnBoolExpr(Cond: CondUOp->getSubExpr(), TrueBlock: FalseBlock, FalseBlock: TrueBlock,
2044 TrueCount: FalseCount, LH);
2045 }
2046 }
2047
2048 if (const ConditionalOperator *CondOp = dyn_cast<ConditionalOperator>(Val: Cond)) {
2049 // br(c ? x : y, t, f) -> br(c, br(x, t, f), br(y, t, f))
2050 llvm::BasicBlock *LHSBlock = createBasicBlock(name: "cond.true");
2051 llvm::BasicBlock *RHSBlock = createBasicBlock(name: "cond.false");
2052
2053 // The ConditionalOperator itself has no likelihood information for its
2054 // true and false branches. This matches the behavior of __builtin_expect.
2055 ConditionalEvaluation cond(*this);
2056 EmitBranchOnBoolExpr(Cond: CondOp->getCond(), TrueBlock: LHSBlock, FalseBlock: RHSBlock,
2057 TrueCount: getProfileCount(S: CondOp), LH: Stmt::LH_None);
2058
2059 // When computing PGO branch weights, we only know the overall count for
2060 // the true block. This code is essentially doing tail duplication of the
2061 // naive code-gen, introducing new edges for which counts are not
2062 // available. Divide the counts proportionally between the LHS and RHS of
2063 // the conditional operator.
2064 uint64_t LHSScaledTrueCount = 0;
2065 if (TrueCount) {
2066 double LHSRatio =
2067 getProfileCount(S: CondOp) / (double)getCurrentProfileCount();
2068 LHSScaledTrueCount = TrueCount * LHSRatio;
2069 }
2070
2071 cond.begin(CGF&: *this);
2072 EmitBlock(BB: LHSBlock);
2073 incrementProfileCounter(ExecSkip: UseExecPath, S: CondOp);
2074 {
2075 ApplyDebugLocation DL(*this, Cond);
2076 EmitBranchOnBoolExpr(Cond: CondOp->getLHS(), TrueBlock, FalseBlock,
2077 TrueCount: LHSScaledTrueCount, LH, ConditionalOp: CondOp);
2078 }
2079 cond.end(CGF&: *this);
2080
2081 cond.begin(CGF&: *this);
2082 EmitBlock(BB: RHSBlock);
2083 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondOp);
2084 EmitBranchOnBoolExpr(Cond: CondOp->getRHS(), TrueBlock, FalseBlock,
2085 TrueCount: TrueCount - LHSScaledTrueCount, LH, ConditionalOp: CondOp);
2086 cond.end(CGF&: *this);
2087
2088 return;
2089 }
2090
2091 if (const CXXThrowExpr *Throw = dyn_cast<CXXThrowExpr>(Val: Cond)) {
2092 // Conditional operator handling can give us a throw expression as a
2093 // condition for a case like:
2094 // br(c ? throw x : y, t, f) -> br(c, br(throw x, t, f), br(y, t, f)
2095 // Fold this to:
2096 // br(c, throw x, br(y, t, f))
2097 EmitCXXThrowExpr(E: Throw, /*KeepInsertionPoint*/false);
2098 return;
2099 }
2100
2101 // Emit the code with the fully general case.
2102 llvm::Value *CondV;
2103 {
2104 ApplyDebugLocation DL(*this, Cond);
2105 CondV = EvaluateExprAsBool(E: Cond);
2106 }
2107
2108 MaybeEmitDeferredVarDeclInit(var: ConditionalDecl);
2109
2110 // If not at the top of the logical operator nest, update MCDC temp with the
2111 // boolean result of the evaluated condition.
2112 {
2113 const Expr *MCDCBaseExpr = Cond;
2114 // When a nested ConditionalOperator (ternary) is encountered in a boolean
2115 // expression, MC/DC tracks the result of the ternary, and this is tied to
2116 // the ConditionalOperator expression and not the ternary's LHS or RHS. If
2117 // this is the case, the ConditionalOperator expression is passed through
2118 // the ConditionalOp parameter and then used as the MCDC base expression.
2119 if (ConditionalOp)
2120 MCDCBaseExpr = ConditionalOp;
2121
2122 if (isMCDCBranchExpr(E: stripCond(C: MCDCBaseExpr)) &&
2123 !isMCDCDecisionExpr(E: stripCond(C: Cond)))
2124 maybeUpdateMCDCCondBitmap(E: MCDCBaseExpr, Val: CondV);
2125 }
2126
2127 llvm::MDNode *Weights = nullptr;
2128 llvm::MDNode *Unpredictable = nullptr;
2129
2130 // If the branch has a condition wrapped by __builtin_unpredictable,
2131 // create metadata that specifies that the branch is unpredictable.
2132 // Don't bother if not optimizing because that metadata would not be used.
2133 auto *Call = dyn_cast<CallExpr>(Val: Cond->IgnoreImpCasts());
2134 if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
2135 auto *FD = dyn_cast_or_null<FunctionDecl>(Val: Call->getCalleeDecl());
2136 if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
2137 llvm::MDBuilder MDHelper(getLLVMContext());
2138 Unpredictable = MDHelper.createUnpredictable();
2139 }
2140 }
2141
2142 // If there is a Likelihood knowledge for the cond, lower it.
2143 // Note that if not optimizing this won't emit anything.
2144 llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(Cond: CondV, LH);
2145 if (CondV != NewCondV)
2146 CondV = NewCondV;
2147 else {
2148 // Otherwise, lower profile counts. Note that we do this even at -O0.
2149 uint64_t CurrentCount = std::max(a: getCurrentProfileCount(), b: TrueCount);
2150 Weights = createProfileWeights(TrueCount, FalseCount: CurrentCount - TrueCount);
2151 }
2152
2153 llvm::Instruction *BrInst = Builder.CreateCondBr(Cond: CondV, True: TrueBlock, False: FalseBlock,
2154 BranchWeights: Weights, Unpredictable);
2155 addInstToNewSourceAtom(KeyInstruction: BrInst, Backup: CondV);
2156
2157 switch (HLSLControlFlowAttr) {
2158 case HLSLControlFlowHintAttr::Microsoft_branch:
2159 case HLSLControlFlowHintAttr::Microsoft_flatten: {
2160 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2161
2162 llvm::ConstantInt *BranchHintConstant =
2163 HLSLControlFlowAttr ==
2164 HLSLControlFlowHintAttr::Spelling::Microsoft_branch
2165 ? llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 1)
2166 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 2);
2167
2168 SmallVector<llvm::Metadata *, 2> Vals(
2169 {MDHelper.createString(Str: "hlsl.controlflow.hint"),
2170 MDHelper.createConstant(C: BranchHintConstant)});
2171 BrInst->setMetadata(Kind: "hlsl.controlflow.hint",
2172 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Vals));
2173 break;
2174 }
2175 // This is required to avoid warnings during compilation
2176 case HLSLControlFlowHintAttr::SpellingNotCalculated:
2177 break;
2178 }
2179}
2180
2181llvm::Value *CodeGenFunction::EmitScalarOrConstFoldImmArg(unsigned ICEArguments,
2182 unsigned Idx,
2183 const CallExpr *E) {
2184 llvm::Value *Arg = nullptr;
2185 if ((ICEArguments & (1 << Idx)) == 0) {
2186 Arg = EmitScalarExpr(E: E->getArg(Arg: Idx));
2187 } else {
2188 // If this is required to be a constant, constant fold it so that we
2189 // know that the generated intrinsic gets a ConstantInt.
2190 std::optional<llvm::APSInt> Result =
2191 E->getArg(Arg: Idx)->getIntegerConstantExpr(Ctx: getContext());
2192 assert(Result && "Expected argument to be a constant");
2193 Arg = llvm::ConstantInt::get(Context&: getLLVMContext(), V: *Result);
2194 }
2195 return Arg;
2196}
2197
2198/// ErrorUnsupported - Print out an error that codegen doesn't support the
2199/// specified stmt yet.
2200void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) {
2201 CGM.ErrorUnsupported(S, Type);
2202}
2203
2204/// emitNonZeroVLAInit - Emit the "zero" initialization of a
2205/// variable-length array whose elements have a non-zero bit-pattern.
2206///
2207/// \param baseType the inner-most element type of the array
2208/// \param src - a char* pointing to the bit-pattern for a single
2209/// base element of the array
2210/// \param sizeInChars - the total size of the VLA, in chars
2211static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
2212 Address dest, Address src,
2213 llvm::Value *sizeInChars) {
2214 CGBuilderTy &Builder = CGF.Builder;
2215
2216 CharUnits baseSize = CGF.getContext().getTypeSizeInChars(T: baseType);
2217 llvm::Value *baseSizeInChars
2218 = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: baseSize.getQuantity());
2219
2220 Address begin = dest.withElementType(ElemTy: CGF.Int8Ty);
2221 llvm::Value *end = Builder.CreateInBoundsGEP(Ty: begin.getElementType(),
2222 Ptr: begin.emitRawPointer(CGF),
2223 IdxList: sizeInChars, Name: "vla.end");
2224
2225 llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
2226 llvm::BasicBlock *loopBB = CGF.createBasicBlock(name: "vla-init.loop");
2227 llvm::BasicBlock *contBB = CGF.createBasicBlock(name: "vla-init.cont");
2228
2229 // Make a loop over the VLA. C99 guarantees that the VLA element
2230 // count must be nonzero.
2231 CGF.EmitBlock(BB: loopBB);
2232
2233 llvm::PHINode *cur = Builder.CreatePHI(Ty: begin.getType(), NumReservedValues: 2, Name: "vla.cur");
2234 cur->addIncoming(V: begin.emitRawPointer(CGF), BB: originBB);
2235
2236 CharUnits curAlign =
2237 dest.getAlignment().alignmentOfArrayElement(elementSize: baseSize);
2238
2239 // memcpy the individual element bit-pattern.
2240 Builder.CreateMemCpy(Dest: Address(cur, CGF.Int8Ty, curAlign), Src: src, Size: baseSizeInChars,
2241 /*volatile*/ IsVolatile: false);
2242
2243 // Go to the next element.
2244 llvm::Value *next =
2245 Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: cur, IdxList: baseSizeInChars, Name: "vla.next");
2246
2247 // Leave if that's the end of the VLA.
2248 llvm::Value *done = Builder.CreateICmpEQ(LHS: next, RHS: end, Name: "vla-init.isdone");
2249 Builder.CreateCondBr(Cond: done, True: contBB, False: loopBB);
2250 cur->addIncoming(V: next, BB: loopBB);
2251
2252 CGF.EmitBlock(BB: contBB);
2253}
2254
2255Address CodeGenFunction::EmitAddressOfPFPField(Address RecordPtr,
2256 const PFPField &Field) {
2257 return EmitAddressOfPFPField(
2258 RecordPtr,
2259 FieldPtr: Builder.CreateConstInBoundsByteGEP(Addr: RecordPtr.withElementType(ElemTy: Int8Ty),
2260 Offset: Field.Offset),
2261 Field: Field.Field);
2262}
2263
2264Address CodeGenFunction::EmitAddressOfPFPField(Address RecordPtr,
2265 Address PtrPtr,
2266 const FieldDecl *Field) {
2267 llvm::Value *Disc;
2268 if (CGM.getContext().arePFPFieldsTriviallyCopyable(RD: Field->getParent())) {
2269 uint64_t FieldSignature =
2270 llvm::getPointerAuthStableSipHash(S: CGM.getPFPFieldName(FD: Field));
2271 Disc = llvm::ConstantInt::get(Ty: CGM.Int64Ty, V: FieldSignature);
2272 } else
2273 Disc = Builder.CreatePtrToInt(V: RecordPtr.getBasePointer(), DestTy: CGM.Int64Ty);
2274
2275 llvm::GlobalValue *DS = CGM.getPFPDeactivationSymbol(FD: Field);
2276 llvm::OperandBundleDef DSBundle("deactivation-symbol", DS);
2277 llvm::Value *Args[] = {PtrPtr.getBasePointer(), Disc, Builder.getTrue()};
2278 return Address(
2279 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::protected_field_ptr,
2280 Tys: PtrPtr.getType()),
2281 Args, OpBundles: DSBundle),
2282 VoidPtrTy, PtrPtr.getAlignment());
2283}
2284
2285void
2286CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
2287 // Ignore empty classes in C++.
2288 if (getLangOpts().CPlusPlus)
2289 if (const auto *RD = Ty->getAsCXXRecordDecl(); RD && RD->isEmpty())
2290 return;
2291
2292 if (DestPtr.getElementType() != Int8Ty)
2293 DestPtr = DestPtr.withElementType(ElemTy: Int8Ty);
2294
2295 // Get size and alignment info for this aggregate.
2296 CharUnits size = getContext().getTypeSizeInChars(T: Ty);
2297
2298 llvm::Value *SizeVal;
2299 const VariableArrayType *vla;
2300
2301 // Don't bother emitting a zero-byte memset.
2302 if (size.isZero()) {
2303 // But note that getTypeInfo returns 0 for a VLA.
2304 if (const VariableArrayType *vlaType =
2305 dyn_cast_or_null<VariableArrayType>(
2306 Val: getContext().getAsArrayType(T: Ty))) {
2307 auto VlaSize = getVLASize(vla: vlaType);
2308 SizeVal = VlaSize.NumElts;
2309 CharUnits eltSize = getContext().getTypeSizeInChars(T: VlaSize.Type);
2310 if (!eltSize.isOne())
2311 SizeVal = Builder.CreateNUWMul(LHS: SizeVal, RHS: CGM.getSize(numChars: eltSize));
2312 vla = vlaType;
2313 } else {
2314 return;
2315 }
2316 } else {
2317 SizeVal = CGM.getSize(numChars: size);
2318 vla = nullptr;
2319 }
2320
2321 // If the type contains a pointer to data member we can't memset it to zero.
2322 // Instead, create a null constant and copy it to the destination.
2323 // TODO: there are other patterns besides zero that we can usefully memset,
2324 // like -1, which happens to be the pattern used by member-pointers.
2325 if (!CGM.getTypes().isZeroInitializable(T: Ty)) {
2326 // For a VLA, emit a single element, then splat that over the VLA.
2327 if (vla) Ty = getContext().getBaseElementType(VAT: vla);
2328
2329 llvm::Constant *NullConstant = CGM.EmitNullConstant(T: Ty);
2330
2331 llvm::GlobalVariable *NullVariable =
2332 new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
2333 /*isConstant=*/true,
2334 llvm::GlobalVariable::PrivateLinkage,
2335 NullConstant, Twine());
2336 CharUnits NullAlign = DestPtr.getAlignment();
2337 NullVariable->setAlignment(NullAlign.getAsAlign());
2338 Address SrcPtr(NullVariable, Builder.getInt8Ty(), NullAlign);
2339
2340 if (vla) return emitNonZeroVLAInit(CGF&: *this, baseType: Ty, dest: DestPtr, src: SrcPtr, sizeInChars: SizeVal);
2341
2342 // Get and call the appropriate llvm.memcpy overload.
2343 Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: SizeVal, IsVolatile: false);
2344 } else {
2345 // Otherwise, just memset the whole thing to zero. This is legal
2346 // because in LLVM, all default initializers (other than the ones we just
2347 // handled above, and the case handled below) are guaranteed to have a bit
2348 // pattern of all zeros.
2349 Builder.CreateMemSet(Dest: DestPtr, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
2350 }
2351
2352 // With the pointer field protection feature, null pointers do not have a bit
2353 // pattern of zero in memory, so we must initialize them separately.
2354 for (auto &Field : getContext().findPFPFields(Ty)) {
2355 auto addr = EmitAddressOfPFPField(RecordPtr: DestPtr, Field);
2356 Builder.CreateStore(Val: llvm::ConstantPointerNull::get(T: VoidPtrTy), Addr: addr);
2357 }
2358}
2359
2360llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
2361 // Make sure that there is a block for the indirect goto.
2362 if (!IndirectBranch)
2363 GetIndirectGotoBlock();
2364
2365 llvm::BasicBlock *BB = getJumpDestForLabel(S: L).getBlock();
2366
2367 // Make sure the indirect branch includes all of the address-taken blocks.
2368 IndirectBranch->addDestination(Dest: BB);
2369 return llvm::BlockAddress::get(Ty: CurFn->getType(), BB);
2370}
2371
2372llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {
2373 // If we already made the indirect branch for indirect goto, return its block.
2374 if (IndirectBranch) return IndirectBranch->getParent();
2375
2376 CGBuilderTy TmpBuilder(*this, createBasicBlock(name: "indirectgoto"));
2377
2378 // Create the PHI node that indirect gotos will add entries to.
2379 llvm::Value *DestVal = TmpBuilder.CreatePHI(Ty: Int8PtrTy, NumReservedValues: 0,
2380 Name: "indirect.goto.dest");
2381
2382 // Create the indirect branch instruction.
2383 IndirectBranch = TmpBuilder.CreateIndirectBr(Addr: DestVal);
2384 return IndirectBranch->getParent();
2385}
2386
2387/// Computes the length of an array in elements, as well as the base
2388/// element type and a properly-typed first element pointer.
2389llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
2390 QualType &baseType,
2391 Address &addr) {
2392 const ArrayType *arrayType = origArrayType;
2393
2394 // If it's a VLA, we have to load the stored size. Note that
2395 // this is the size of the VLA in bytes, not its size in elements.
2396 llvm::Value *numVLAElements = nullptr;
2397 if (isa<VariableArrayType>(Val: arrayType)) {
2398 numVLAElements = getVLASize(vla: cast<VariableArrayType>(Val: arrayType)).NumElts;
2399
2400 // Walk into all VLAs. This doesn't require changes to addr,
2401 // which has type T* where T is the first non-VLA element type.
2402 do {
2403 QualType elementType = arrayType->getElementType();
2404 arrayType = getContext().getAsArrayType(T: elementType);
2405
2406 // If we only have VLA components, 'addr' requires no adjustment.
2407 if (!arrayType) {
2408 baseType = elementType;
2409 return numVLAElements;
2410 }
2411 } while (isa<VariableArrayType>(Val: arrayType));
2412
2413 // We get out here only if we find a constant array type
2414 // inside the VLA.
2415 }
2416
2417 // We have some number of constant-length arrays, so addr should
2418 // have LLVM type [M x [N x [...]]]*. Build a GEP that walks
2419 // down to the first element of addr.
2420 SmallVector<llvm::Value*, 8> gepIndices;
2421
2422 // GEP down to the array type.
2423 llvm::ConstantInt *zero = Builder.getInt32(C: 0);
2424 gepIndices.push_back(Elt: zero);
2425
2426 uint64_t countFromCLAs = 1;
2427 QualType eltType;
2428
2429 llvm::ArrayType *llvmArrayType =
2430 dyn_cast<llvm::ArrayType>(Val: addr.getElementType());
2431 while (llvmArrayType) {
2432 assert(isa<ConstantArrayType>(arrayType));
2433 assert(cast<ConstantArrayType>(arrayType)->getZExtSize() ==
2434 llvmArrayType->getNumElements());
2435
2436 gepIndices.push_back(Elt: zero);
2437 countFromCLAs *= llvmArrayType->getNumElements();
2438 eltType = arrayType->getElementType();
2439
2440 llvmArrayType =
2441 dyn_cast<llvm::ArrayType>(Val: llvmArrayType->getElementType());
2442 arrayType = getContext().getAsArrayType(T: arrayType->getElementType());
2443 assert((!llvmArrayType || arrayType) &&
2444 "LLVM and Clang types are out-of-synch");
2445 }
2446
2447 if (arrayType) {
2448 // From this point onwards, the Clang array type has been emitted
2449 // as some other type (probably a packed struct). Compute the array
2450 // size, and just emit the 'begin' expression as a bitcast.
2451 while (arrayType) {
2452 countFromCLAs *= cast<ConstantArrayType>(Val: arrayType)->getZExtSize();
2453 eltType = arrayType->getElementType();
2454 arrayType = getContext().getAsArrayType(T: eltType);
2455 }
2456
2457 llvm::Type *baseType = ConvertType(T: eltType);
2458 addr = addr.withElementType(ElemTy: baseType);
2459 } else {
2460 // Create the actual GEP.
2461 addr = Address(Builder.CreateInBoundsGEP(Ty: addr.getElementType(),
2462 Ptr: addr.emitRawPointer(CGF&: *this),
2463 IdxList: gepIndices, Name: "array.begin"),
2464 ConvertTypeForMem(T: eltType), addr.getAlignment());
2465 }
2466
2467 baseType = eltType;
2468
2469 llvm::Value *numElements
2470 = llvm::ConstantInt::get(Ty: SizeTy, V: countFromCLAs);
2471
2472 // If we had any VLA dimensions, factor them in.
2473 if (numVLAElements)
2474 numElements = Builder.CreateNUWMul(LHS: numVLAElements, RHS: numElements);
2475
2476 return numElements;
2477}
2478
2479CodeGenFunction::VlaSizePair CodeGenFunction::getVLASize(QualType type) {
2480 const VariableArrayType *vla = getContext().getAsVariableArrayType(T: type);
2481 assert(vla && "type was not a variable array type!");
2482 return getVLASize(vla);
2483}
2484
2485CodeGenFunction::VlaSizePair
2486CodeGenFunction::getVLASize(const VariableArrayType *type) {
2487 // The number of elements so far; always size_t.
2488 llvm::Value *numElements = nullptr;
2489
2490 QualType elementType;
2491 do {
2492 elementType = type->getElementType();
2493 llvm::Value *vlaSize = VLASizeMap[type->getSizeExpr()];
2494 assert(vlaSize && "no size for VLA!");
2495 assert(vlaSize->getType() == SizeTy);
2496
2497 if (!numElements) {
2498 numElements = vlaSize;
2499 } else {
2500 // It's undefined behavior if this wraps around, so mark it that way.
2501 // FIXME: Teach -fsanitize=undefined to trap this.
2502 numElements = Builder.CreateNUWMul(LHS: numElements, RHS: vlaSize);
2503 }
2504 } while ((type = getContext().getAsVariableArrayType(T: elementType)));
2505
2506 return { numElements, elementType };
2507}
2508
2509CodeGenFunction::VlaSizePair
2510CodeGenFunction::getVLAElements1D(QualType type) {
2511 const VariableArrayType *vla = getContext().getAsVariableArrayType(T: type);
2512 assert(vla && "type was not a variable array type!");
2513 return getVLAElements1D(vla);
2514}
2515
2516CodeGenFunction::VlaSizePair
2517CodeGenFunction::getVLAElements1D(const VariableArrayType *Vla) {
2518 llvm::Value *VlaSize = VLASizeMap[Vla->getSizeExpr()];
2519 assert(VlaSize && "no size for VLA!");
2520 assert(VlaSize->getType() == SizeTy);
2521 return { VlaSize, Vla->getElementType() };
2522}
2523
2524void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
2525 assert(type->isVariablyModifiedType() &&
2526 "Must pass variably modified type to EmitVLASizes!");
2527
2528 EnsureInsertPoint();
2529
2530 // We're going to walk down into the type and look for VLA
2531 // expressions.
2532 do {
2533 assert(type->isVariablyModifiedType());
2534
2535 const Type *ty = type.getTypePtr();
2536 switch (ty->getTypeClass()) {
2537
2538#define TYPE(Class, Base)
2539#define ABSTRACT_TYPE(Class, Base)
2540#define NON_CANONICAL_TYPE(Class, Base)
2541#define DEPENDENT_TYPE(Class, Base) case Type::Class:
2542#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
2543#include "clang/AST/TypeNodes.inc"
2544 llvm_unreachable("unexpected dependent type!");
2545
2546 // These types are never variably-modified.
2547 case Type::Builtin:
2548 case Type::Complex:
2549 case Type::Vector:
2550 case Type::ExtVector:
2551 case Type::ConstantMatrix:
2552 case Type::Record:
2553 case Type::Enum:
2554 case Type::Using:
2555 case Type::TemplateSpecialization:
2556 case Type::ObjCTypeParam:
2557 case Type::ObjCObject:
2558 case Type::ObjCInterface:
2559 case Type::ObjCObjectPointer:
2560 case Type::BitInt:
2561 case Type::HLSLInlineSpirv:
2562 case Type::PredefinedSugar:
2563 llvm_unreachable("type class is never variably-modified!");
2564
2565 case Type::Adjusted:
2566 type = cast<AdjustedType>(Val: ty)->getAdjustedType();
2567 break;
2568
2569 case Type::Decayed:
2570 type = cast<DecayedType>(Val: ty)->getPointeeType();
2571 break;
2572
2573 case Type::Pointer:
2574 type = cast<PointerType>(Val: ty)->getPointeeType();
2575 break;
2576
2577 case Type::BlockPointer:
2578 type = cast<BlockPointerType>(Val: ty)->getPointeeType();
2579 break;
2580
2581 case Type::LValueReference:
2582 case Type::RValueReference:
2583 type = cast<ReferenceType>(Val: ty)->getPointeeType();
2584 break;
2585
2586 case Type::MemberPointer:
2587 type = cast<MemberPointerType>(Val: ty)->getPointeeType();
2588 break;
2589
2590 case Type::ArrayParameter:
2591 case Type::ConstantArray:
2592 case Type::IncompleteArray:
2593 // Losing element qualification here is fine.
2594 type = cast<ArrayType>(Val: ty)->getElementType();
2595 break;
2596
2597 case Type::VariableArray: {
2598 // Losing element qualification here is fine.
2599 const VariableArrayType *vat = cast<VariableArrayType>(Val: ty);
2600
2601 // Unknown size indication requires no size computation.
2602 // Otherwise, evaluate and record it.
2603 if (const Expr *sizeExpr = vat->getSizeExpr()) {
2604 // It's possible that we might have emitted this already,
2605 // e.g. with a typedef and a pointer to it.
2606 llvm::Value *&entry = VLASizeMap[sizeExpr];
2607 if (!entry) {
2608 llvm::Value *size = EmitScalarExpr(E: sizeExpr);
2609
2610 // C11 6.7.6.2p5:
2611 // If the size is an expression that is not an integer constant
2612 // expression [...] each time it is evaluated it shall have a value
2613 // greater than zero.
2614 if (SanOpts.has(K: SanitizerKind::VLABound)) {
2615 auto CheckOrdinal = SanitizerKind::SO_VLABound;
2616 auto CheckHandler = SanitizerHandler::VLABoundNotPositive;
2617 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2618 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: size->getType());
2619 clang::QualType SEType = sizeExpr->getType();
2620 llvm::Value *CheckCondition =
2621 SEType->isSignedIntegerType()
2622 ? Builder.CreateICmpSGT(LHS: size, RHS: Zero)
2623 : Builder.CreateICmpUGT(LHS: size, RHS: Zero);
2624 llvm::Constant *StaticArgs[] = {
2625 EmitCheckSourceLocation(Loc: sizeExpr->getBeginLoc()),
2626 EmitCheckTypeDescriptor(T: SEType)};
2627 EmitCheck(Checked: std::make_pair(x&: CheckCondition, y&: CheckOrdinal),
2628 Check: CheckHandler, StaticArgs, DynamicArgs: size);
2629 }
2630
2631 // Always zexting here would be wrong if it weren't
2632 // undefined behavior to have a negative bound.
2633 // FIXME: What about when size's type is larger than size_t?
2634 entry = Builder.CreateIntCast(V: size, DestTy: SizeTy, /*signed*/ isSigned: false);
2635 }
2636 }
2637 type = vat->getElementType();
2638 break;
2639 }
2640
2641 case Type::FunctionProto:
2642 case Type::FunctionNoProto:
2643 type = cast<FunctionType>(Val: ty)->getReturnType();
2644 break;
2645
2646 case Type::Paren:
2647 case Type::TypeOf:
2648 case Type::UnaryTransform:
2649 case Type::Attributed:
2650 case Type::BTFTagAttributed:
2651 case Type::OverflowBehavior:
2652 case Type::HLSLAttributedResource:
2653 case Type::SubstTemplateTypeParm:
2654 case Type::MacroQualified:
2655 case Type::CountAttributed:
2656 // Keep walking after single level desugaring.
2657 type = type.getSingleStepDesugaredType(Context: getContext());
2658 break;
2659
2660 case Type::Typedef:
2661 case Type::Decltype:
2662 case Type::Auto:
2663 case Type::DeducedTemplateSpecialization:
2664 case Type::PackIndexing:
2665 // Stop walking: nothing to do.
2666 return;
2667
2668 case Type::TypeOfExpr:
2669 // Stop walking: emit typeof expression.
2670 EmitIgnoredExpr(E: cast<TypeOfExprType>(Val: ty)->getUnderlyingExpr());
2671 return;
2672
2673 case Type::Atomic:
2674 type = cast<AtomicType>(Val: ty)->getValueType();
2675 break;
2676
2677 case Type::Pipe:
2678 type = cast<PipeType>(Val: ty)->getElementType();
2679 break;
2680 }
2681 } while (type->isVariablyModifiedType());
2682}
2683
2684Address CodeGenFunction::EmitVAListRef(const Expr* E) {
2685 if (getContext().getBuiltinVaListType()->isArrayType())
2686 return EmitPointerWithAlignment(Addr: E);
2687 return EmitLValue(E).getAddress();
2688}
2689
2690Address CodeGenFunction::EmitMSVAListRef(const Expr *E) {
2691 return EmitLValue(E).getAddress();
2692}
2693
2694void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E,
2695 const APValue &Init) {
2696 assert(Init.hasValue() && "Invalid DeclRefExpr initializer!");
2697 if (CGDebugInfo *Dbg = getDebugInfo())
2698 if (CGM.getCodeGenOpts().hasReducedDebugInfo())
2699 Dbg->EmitGlobalVariable(VD: E->getDecl(), Init);
2700}
2701
2702CodeGenFunction::PeepholeProtection
2703CodeGenFunction::protectFromPeepholes(RValue rvalue) {
2704 // At the moment, the only aggressive peephole we do in IR gen
2705 // is trunc(zext) folding, but if we add more, we can easily
2706 // extend this protection.
2707
2708 if (!rvalue.isScalar()) return PeepholeProtection();
2709 llvm::Value *value = rvalue.getScalarVal();
2710 if (!isa<llvm::ZExtInst>(Val: value)) return PeepholeProtection();
2711
2712 // Just make an extra bitcast.
2713 assert(HaveInsertPoint());
2714 llvm::Instruction *inst = new llvm::BitCastInst(value, value->getType(), "",
2715 Builder.GetInsertBlock());
2716
2717 PeepholeProtection protection;
2718 protection.Inst = inst;
2719 return protection;
2720}
2721
2722void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {
2723 if (!protection.Inst) return;
2724
2725 // In theory, we could try to duplicate the peepholes now, but whatever.
2726 protection.Inst->eraseFromParent();
2727}
2728
2729void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2730 QualType Ty, SourceLocation Loc,
2731 SourceLocation AssumptionLoc,
2732 llvm::Value *Alignment,
2733 llvm::Value *OffsetValue) {
2734 if (Alignment->getType() != IntPtrTy)
2735 Alignment =
2736 Builder.CreateIntCast(V: Alignment, DestTy: IntPtrTy, isSigned: false, Name: "casted.align");
2737 if (OffsetValue && OffsetValue->getType() != IntPtrTy)
2738 OffsetValue =
2739 Builder.CreateIntCast(V: OffsetValue, DestTy: IntPtrTy, isSigned: true, Name: "casted.offset");
2740 llvm::Value *TheCheck = nullptr;
2741 if (SanOpts.has(K: SanitizerKind::Alignment)) {
2742 llvm::Value *PtrIntValue =
2743 Builder.CreatePtrToInt(V: PtrValue, DestTy: IntPtrTy, Name: "ptrint");
2744
2745 if (OffsetValue) {
2746 bool IsOffsetZero = false;
2747 if (const auto *CI = dyn_cast<llvm::ConstantInt>(Val: OffsetValue))
2748 IsOffsetZero = CI->isZero();
2749
2750 if (!IsOffsetZero)
2751 PtrIntValue = Builder.CreateSub(LHS: PtrIntValue, RHS: OffsetValue, Name: "offsetptr");
2752 }
2753
2754 llvm::Value *Zero = llvm::ConstantInt::get(Ty: IntPtrTy, V: 0);
2755 llvm::Value *Mask =
2756 Builder.CreateSub(LHS: Alignment, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: 1));
2757 llvm::Value *MaskedPtr = Builder.CreateAnd(LHS: PtrIntValue, RHS: Mask, Name: "maskedptr");
2758 TheCheck = Builder.CreateICmpEQ(LHS: MaskedPtr, RHS: Zero, Name: "maskcond");
2759 }
2760 llvm::Instruction *Assumption = Builder.CreateAlignmentAssumption(
2761 DL: CGM.getDataLayout(), PtrValue, Alignment, OffsetValue);
2762
2763 if (!SanOpts.has(K: SanitizerKind::Alignment))
2764 return;
2765 emitAlignmentAssumptionCheck(Ptr: PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2766 OffsetValue, TheCheck, Assumption);
2767}
2768
2769void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2770 const Expr *E,
2771 SourceLocation AssumptionLoc,
2772 llvm::Value *Alignment,
2773 llvm::Value *OffsetValue) {
2774 QualType Ty = E->getType();
2775 SourceLocation Loc = E->getExprLoc();
2776
2777 emitAlignmentAssumption(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2778 OffsetValue);
2779}
2780
2781llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn,
2782 llvm::Value *AnnotatedVal,
2783 StringRef AnnotationStr,
2784 SourceLocation Location,
2785 const AnnotateAttr *Attr) {
2786 SmallVector<llvm::Value *, 5> Args = {
2787 AnnotatedVal,
2788 CGM.EmitAnnotationString(Str: AnnotationStr),
2789 CGM.EmitAnnotationUnit(Loc: Location),
2790 CGM.EmitAnnotationLineNo(L: Location),
2791 };
2792 if (Attr)
2793 Args.push_back(Elt: CGM.EmitAnnotationArgs(Attr));
2794 return Builder.CreateCall(Callee: AnnotationFn, Args);
2795}
2796
2797void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
2798 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2799 for (const auto *I : D->specific_attrs<AnnotateAttr>())
2800 EmitAnnotationCall(AnnotationFn: CGM.getIntrinsic(IID: llvm::Intrinsic::var_annotation,
2801 Tys: {V->getType(), CGM.ConstGlobalsPtrTy}),
2802 AnnotatedVal: V, AnnotationStr: I->getAnnotation(), Location: D->getLocation(), Attr: I);
2803}
2804
2805Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
2806 Address Addr) {
2807 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2808 llvm::Value *V = Addr.emitRawPointer(CGF&: *this);
2809 llvm::Type *VTy = V->getType();
2810 auto *PTy = dyn_cast<llvm::PointerType>(Val: VTy);
2811 unsigned AS = PTy ? PTy->getAddressSpace() : 0;
2812 llvm::PointerType *IntrinTy =
2813 llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: AS);
2814 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::ptr_annotation,
2815 Tys: {IntrinTy, CGM.ConstGlobalsPtrTy});
2816
2817 for (const auto *I : D->specific_attrs<AnnotateAttr>()) {
2818 // FIXME Always emit the cast inst so we can differentiate between
2819 // annotation on the first field of a struct and annotation on the struct
2820 // itself.
2821 if (VTy != IntrinTy)
2822 V = Builder.CreateBitCast(V, DestTy: IntrinTy);
2823 V = EmitAnnotationCall(AnnotationFn: F, AnnotatedVal: V, AnnotationStr: I->getAnnotation(), Location: D->getLocation(), Attr: I);
2824 V = Builder.CreateBitCast(V, DestTy: VTy);
2825 }
2826
2827 return Address(V, Addr.getElementType(), Addr.getAlignment());
2828}
2829
2830CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
2831
2832CodeGenFunction::SanitizerScope::SanitizerScope(CodeGenFunction *CGF)
2833 : CGF(CGF) {
2834 assert(!CGF->IsSanitizerScope);
2835 CGF->IsSanitizerScope = true;
2836}
2837
2838CodeGenFunction::SanitizerScope::~SanitizerScope() {
2839 CGF->IsSanitizerScope = false;
2840}
2841
2842void CodeGenFunction::InsertHelper(llvm::Instruction *I,
2843 const llvm::Twine &Name,
2844 llvm::BasicBlock::iterator InsertPt) const {
2845 LoopStack.InsertHelper(I);
2846 if (IsSanitizerScope)
2847 I->setNoSanitizeMetadata();
2848}
2849
2850void CGBuilderInserter::InsertHelper(
2851 llvm::Instruction *I, const llvm::Twine &Name,
2852 llvm::BasicBlock::iterator InsertPt) const {
2853 llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, InsertPt);
2854 if (CGF)
2855 CGF->InsertHelper(I, Name, InsertPt);
2856}
2857
2858// Emits an error if we don't have a valid set of target features for the
2859// called function.
2860void CodeGenFunction::checkTargetFeatures(const CallExpr *E,
2861 const FunctionDecl *TargetDecl) {
2862 // SemaChecking cannot handle below x86 builtins because they have different
2863 // parameter ranges with different TargetAttribute of caller.
2864 if (CGM.getContext().getTargetInfo().getTriple().isX86()) {
2865 unsigned BuiltinID = TargetDecl->getBuiltinID();
2866 if (BuiltinID == X86::BI__builtin_ia32_cmpps ||
2867 BuiltinID == X86::BI__builtin_ia32_cmpss ||
2868 BuiltinID == X86::BI__builtin_ia32_cmppd ||
2869 BuiltinID == X86::BI__builtin_ia32_cmpsd) {
2870 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
2871 llvm::StringMap<bool> TargetFetureMap;
2872 CGM.getContext().getFunctionFeatureMap(FeatureMap&: TargetFetureMap, FD);
2873 llvm::APSInt Result =
2874 *(E->getArg(Arg: 2)->getIntegerConstantExpr(Ctx: CGM.getContext()));
2875 if (Result.getSExtValue() > 7 && !TargetFetureMap.lookup(Key: "avx"))
2876 CGM.getDiags().Report(Loc: E->getBeginLoc(), DiagID: diag::err_builtin_needs_feature)
2877 << TargetDecl->getDeclName() << "avx";
2878 }
2879 }
2880 return checkTargetFeatures(Loc: E->getBeginLoc(), TargetDecl);
2881}
2882
2883// Emits an error if we don't have a valid set of target features for the
2884// called function.
2885void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
2886 const FunctionDecl *TargetDecl) {
2887 // Early exit if this is an indirect call.
2888 if (!TargetDecl)
2889 return;
2890
2891 // Get the current enclosing function if it exists. If it doesn't
2892 // we can't check the target features anyhow.
2893 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
2894 if (!FD)
2895 return;
2896
2897 bool IsAlwaysInline = TargetDecl->hasAttr<AlwaysInlineAttr>();
2898 bool IsFlatten = FD && FD->hasAttr<FlattenAttr>();
2899
2900 // Grab the required features for the call. For a builtin this is listed in
2901 // the td file with the default cpu, for an always_inline function this is any
2902 // listed cpu and any listed features.
2903 unsigned BuiltinID = TargetDecl->getBuiltinID();
2904 std::string MissingFeature;
2905 llvm::StringMap<bool> CallerFeatureMap;
2906 CGM.getContext().getFunctionFeatureMap(FeatureMap&: CallerFeatureMap, FD);
2907 // When compiling in HipStdPar mode we have to be conservative in rejecting
2908 // target specific features in the FE, and defer the possible error to the
2909 // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
2910 // referenced by an accelerator executable function, we emit an error.
2911 bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;
2912 if (BuiltinID) {
2913 StringRef FeatureList(CGM.getContext().BuiltinInfo.getRequiredFeatures(ID: BuiltinID));
2914 if (!Builtin::evaluateRequiredTargetFeatures(
2915 RequiredFatures: FeatureList, TargetFetureMap: CallerFeatureMap) && !IsHipStdPar) {
2916 CGM.getDiags().Report(Loc, DiagID: diag::err_builtin_needs_feature)
2917 << TargetDecl->getDeclName()
2918 << FeatureList;
2919 }
2920 } else if (!TargetDecl->isMultiVersion() &&
2921 TargetDecl->hasAttr<TargetAttr>()) {
2922 // Get the required features for the callee.
2923
2924 const TargetAttr *TD = TargetDecl->getAttr<TargetAttr>();
2925 ParsedTargetAttr ParsedAttr =
2926 CGM.getContext().filterFunctionTargetAttrs(TD);
2927
2928 SmallVector<StringRef, 1> ReqFeatures;
2929 llvm::StringMap<bool> CalleeFeatureMap;
2930 CGM.getContext().getFunctionFeatureMap(FeatureMap&: CalleeFeatureMap, TargetDecl);
2931
2932 for (const auto &F : ParsedAttr.Features) {
2933 if (F[0] == '+' && CalleeFeatureMap.lookup(Key: F.substr(pos: 1)))
2934 ReqFeatures.push_back(Elt: StringRef(F).substr(Start: 1));
2935 }
2936
2937 for (const auto &F : CalleeFeatureMap) {
2938 // Only positive features are "required".
2939 if (F.getValue())
2940 ReqFeatures.push_back(Elt: F.getKey());
2941 }
2942 if (!llvm::all_of(Range&: ReqFeatures,
2943 P: [&](StringRef Feature) {
2944 if (!CallerFeatureMap.lookup(Key: Feature)) {
2945 MissingFeature = Feature.str();
2946 return false;
2947 }
2948 return true;
2949 }) &&
2950 !IsHipStdPar) {
2951 if (IsAlwaysInline)
2952 CGM.getDiags().Report(Loc, DiagID: diag::err_function_needs_feature)
2953 << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
2954 else if (IsFlatten)
2955 CGM.getDiags().Report(Loc, DiagID: diag::err_flatten_function_needs_feature)
2956 << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
2957 }
2958
2959 } else if (!FD->isMultiVersion() && FD->hasAttr<TargetAttr>()) {
2960 llvm::StringMap<bool> CalleeFeatureMap;
2961 CGM.getContext().getFunctionFeatureMap(FeatureMap&: CalleeFeatureMap, TargetDecl);
2962
2963 for (const auto &F : CalleeFeatureMap) {
2964 if (F.getValue() &&
2965 (!CallerFeatureMap.lookup(Key: F.getKey()) ||
2966 !CallerFeatureMap.find(Key: F.getKey())->getValue()) &&
2967 !IsHipStdPar) {
2968 if (IsAlwaysInline)
2969 CGM.getDiags().Report(Loc, DiagID: diag::err_function_needs_feature)
2970 << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();
2971 else if (IsFlatten)
2972 CGM.getDiags().Report(Loc, DiagID: diag::err_flatten_function_needs_feature)
2973 << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();
2974 }
2975 }
2976 }
2977}
2978
2979void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
2980 if (!CGM.getCodeGenOpts().SanitizeStats)
2981 return;
2982
2983 llvm::IRBuilder<> IRB(Builder.GetInsertBlock(), Builder.GetInsertPoint());
2984 IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation());
2985 CGM.getSanStats().create(B&: IRB, SK: SSK);
2986}
2987
2988void CodeGenFunction::EmitKCFIOperandBundle(
2989 const CGCallee &Callee, SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
2990 const CGCalleeInfo &CI = Callee.getAbstractInfo();
2991 const FunctionProtoType *FP = CI.getCalleeFunctionProtoType();
2992 if (!FP)
2993 return;
2994
2995 StringRef Salt;
2996 if (const auto &Info = FP->getExtraAttributeInfo())
2997 Salt = Info.CFISalt;
2998
2999 Bundles.emplace_back(Args: "kcfi", Args: CGM.CreateKCFITypeId(T: FP->desugar(), Salt));
3000}
3001
3002llvm::Value *
3003CodeGenFunction::FormAArch64ResolverCondition(const FMVResolverOption &RO) {
3004 return RO.Features.empty() ? nullptr : EmitAArch64CpuSupports(FeatureStrs: RO.Features);
3005}
3006
3007llvm::Value *
3008CodeGenFunction::FormX86ResolverCondition(const FMVResolverOption &RO) {
3009 llvm::Value *Condition = nullptr;
3010
3011 if (RO.Architecture) {
3012 StringRef Arch = *RO.Architecture;
3013 // If arch= specifies an x86-64 micro-architecture level, test the feature
3014 // with __builtin_cpu_supports, otherwise use __builtin_cpu_is.
3015 if (Arch.starts_with(Prefix: "x86-64"))
3016 Condition = EmitX86CpuSupports(FeatureStrs: {Arch});
3017 else
3018 Condition = EmitX86CpuIs(CPUStr: Arch);
3019 }
3020
3021 if (!RO.Features.empty()) {
3022 llvm::Value *FeatureCond = EmitX86CpuSupports(FeatureStrs: RO.Features);
3023 Condition =
3024 Condition ? Builder.CreateAnd(LHS: Condition, RHS: FeatureCond) : FeatureCond;
3025 }
3026 return Condition;
3027}
3028
3029static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
3030 llvm::Function *Resolver,
3031 CGBuilderTy &Builder,
3032 llvm::Function *FuncToReturn,
3033 bool SupportsIFunc) {
3034 if (SupportsIFunc) {
3035 Builder.CreateRet(V: FuncToReturn);
3036 return;
3037 }
3038
3039 llvm::SmallVector<llvm::Value *, 10> Args(
3040 llvm::make_pointer_range(Range: Resolver->args()));
3041
3042 llvm::CallInst *Result = Builder.CreateCall(Callee: FuncToReturn, Args);
3043 Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
3044
3045 if (Resolver->getReturnType()->isVoidTy())
3046 Builder.CreateRetVoid();
3047 else
3048 Builder.CreateRet(V: Result);
3049}
3050
3051void CodeGenFunction::EmitMultiVersionResolver(
3052 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3053
3054 llvm::Triple::ArchType ArchType =
3055 getContext().getTargetInfo().getTriple().getArch();
3056
3057 switch (ArchType) {
3058 case llvm::Triple::x86:
3059 case llvm::Triple::x86_64:
3060 EmitX86MultiVersionResolver(Resolver, Options);
3061 return;
3062 case llvm::Triple::aarch64:
3063 EmitAArch64MultiVersionResolver(Resolver, Options);
3064 return;
3065 case llvm::Triple::riscv32:
3066 case llvm::Triple::riscv64:
3067 case llvm::Triple::riscv32be:
3068 case llvm::Triple::riscv64be:
3069 EmitRISCVMultiVersionResolver(Resolver, Options);
3070 return;
3071
3072 default:
3073 assert(false && "Only implemented for x86, AArch64 and RISC-V targets");
3074 }
3075}
3076
3077void CodeGenFunction::EmitRISCVMultiVersionResolver(
3078 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3079
3080 if (getContext().getTargetInfo().getTriple().getOS() !=
3081 llvm::Triple::OSType::Linux) {
3082 CGM.getDiags().Report(DiagID: diag::err_os_unsupport_riscv_fmv);
3083 return;
3084 }
3085
3086 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3087 Builder.SetInsertPoint(CurBlock);
3088 EmitRISCVCpuInit();
3089
3090 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3091 bool HasDefault = false;
3092 unsigned DefaultIndex = 0;
3093
3094 // Check the each candidate function.
3095 for (unsigned Index = 0; Index < Options.size(); Index++) {
3096
3097 if (Options[Index].Features.empty()) {
3098 HasDefault = true;
3099 DefaultIndex = Index;
3100 continue;
3101 }
3102
3103 Builder.SetInsertPoint(CurBlock);
3104
3105 // FeaturesCondition: The bitmask of the required extension has been
3106 // enabled by the runtime object.
3107 // (__riscv_feature_bits.features[i] & REQUIRED_BITMASK) ==
3108 // REQUIRED_BITMASK
3109 //
3110 // When condition is met, return this version of the function.
3111 // Otherwise, try the next version.
3112 //
3113 // if (FeaturesConditionVersion1)
3114 // return Version1;
3115 // else if (FeaturesConditionVersion2)
3116 // return Version2;
3117 // else if (FeaturesConditionVersion3)
3118 // return Version3;
3119 // ...
3120 // else
3121 // return DefaultVersion;
3122
3123 // TODO: Add a condition to check the length before accessing elements.
3124 // Without checking the length first, we may access an incorrect memory
3125 // address when using different versions.
3126 llvm::SmallVector<StringRef, 8> CurrTargetAttrFeats;
3127 llvm::SmallVector<std::string, 8> TargetAttrFeats;
3128
3129 for (StringRef Feat : Options[Index].Features) {
3130 std::vector<std::string> FeatStr =
3131 getContext().getTargetInfo().parseTargetAttr(Str: Feat).Features;
3132
3133 assert(FeatStr.size() == 1 && "Feature string not delimited");
3134
3135 std::string &CurrFeat = FeatStr.front();
3136 if (CurrFeat[0] == '+')
3137 TargetAttrFeats.push_back(Elt: CurrFeat.substr(pos: 1));
3138 }
3139
3140 if (TargetAttrFeats.empty())
3141 continue;
3142
3143 for (std::string &Feat : TargetAttrFeats)
3144 CurrTargetAttrFeats.push_back(Elt: Feat);
3145
3146 Builder.SetInsertPoint(CurBlock);
3147 llvm::Value *FeatsCondition = EmitRISCVCpuSupports(FeaturesStrs: CurrTargetAttrFeats);
3148
3149 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3150 CGBuilderTy RetBuilder(*this, RetBlock);
3151 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder,
3152 FuncToReturn: Options[Index].Function, SupportsIFunc);
3153 llvm::BasicBlock *ElseBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3154
3155 Builder.SetInsertPoint(CurBlock);
3156 Builder.CreateCondBr(Cond: FeatsCondition, True: RetBlock, False: ElseBlock);
3157
3158 CurBlock = ElseBlock;
3159 }
3160
3161 // Finally, emit the default one.
3162 if (HasDefault) {
3163 Builder.SetInsertPoint(CurBlock);
3164 CreateMultiVersionResolverReturn(
3165 CGM, Resolver, Builder, FuncToReturn: Options[DefaultIndex].Function, SupportsIFunc);
3166 return;
3167 }
3168
3169 // If no generic/default, emit an unreachable.
3170 Builder.SetInsertPoint(CurBlock);
3171 llvm::CallInst *TrapCall = EmitTrapCall(IntrID: llvm::Intrinsic::trap);
3172 TrapCall->setDoesNotReturn();
3173 TrapCall->setDoesNotThrow();
3174 Builder.CreateUnreachable();
3175 Builder.ClearInsertionPoint();
3176}
3177
3178void CodeGenFunction::EmitAArch64MultiVersionResolver(
3179 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3180 assert(!Options.empty() && "No multiversion resolver options found");
3181 assert(Options.back().Features.size() == 0 && "Default case must be last");
3182 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3183 assert(SupportsIFunc &&
3184 "Multiversion resolver requires target IFUNC support");
3185 bool AArch64CpuInitialized = false;
3186 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3187
3188 for (const FMVResolverOption &RO : Options) {
3189 Builder.SetInsertPoint(CurBlock);
3190 llvm::Value *Condition = FormAArch64ResolverCondition(RO);
3191
3192 // The 'default' or 'all features enabled' case.
3193 if (!Condition) {
3194 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, FuncToReturn: RO.Function,
3195 SupportsIFunc);
3196 return;
3197 }
3198
3199 if (!AArch64CpuInitialized) {
3200 Builder.SetInsertPoint(TheBB: CurBlock, IP: CurBlock->begin());
3201 EmitAArch64CpuInit();
3202 AArch64CpuInitialized = true;
3203 Builder.SetInsertPoint(CurBlock);
3204 }
3205
3206 // Skip unreachable versions.
3207 if (RO.Function == nullptr)
3208 continue;
3209
3210 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3211 CGBuilderTy RetBuilder(*this, RetBlock);
3212 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder, FuncToReturn: RO.Function,
3213 SupportsIFunc);
3214 CurBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3215 Builder.CreateCondBr(Cond: Condition, True: RetBlock, False: CurBlock);
3216 }
3217
3218 // If no default, emit an unreachable.
3219 Builder.SetInsertPoint(CurBlock);
3220 llvm::CallInst *TrapCall = EmitTrapCall(IntrID: llvm::Intrinsic::trap);
3221 TrapCall->setDoesNotReturn();
3222 TrapCall->setDoesNotThrow();
3223 Builder.CreateUnreachable();
3224 Builder.ClearInsertionPoint();
3225}
3226
3227void CodeGenFunction::EmitX86MultiVersionResolver(
3228 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3229
3230 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3231
3232 // Main function's basic block.
3233 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3234 Builder.SetInsertPoint(CurBlock);
3235 EmitX86CpuInit();
3236
3237 for (const FMVResolverOption &RO : Options) {
3238 Builder.SetInsertPoint(CurBlock);
3239 llvm::Value *Condition = FormX86ResolverCondition(RO);
3240
3241 // The 'default' or 'generic' case.
3242 if (!Condition) {
3243 assert(&RO == Options.end() - 1 &&
3244 "Default or Generic case must be last");
3245 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, FuncToReturn: RO.Function,
3246 SupportsIFunc);
3247 return;
3248 }
3249
3250 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3251 CGBuilderTy RetBuilder(*this, RetBlock);
3252 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder, FuncToReturn: RO.Function,
3253 SupportsIFunc);
3254 CurBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3255 Builder.CreateCondBr(Cond: Condition, True: RetBlock, False: CurBlock);
3256 }
3257
3258 // If no generic/default, emit an unreachable.
3259 Builder.SetInsertPoint(CurBlock);
3260 llvm::CallInst *TrapCall = EmitTrapCall(IntrID: llvm::Intrinsic::trap);
3261 TrapCall->setDoesNotReturn();
3262 TrapCall->setDoesNotThrow();
3263 Builder.CreateUnreachable();
3264 Builder.ClearInsertionPoint();
3265}
3266
3267// Loc - where the diagnostic will point, where in the source code this
3268// alignment has failed.
3269// SecondaryLoc - if present (will be present if sufficiently different from
3270// Loc), the diagnostic will additionally point a "Note:" to this location.
3271// It should be the location where the __attribute__((assume_aligned))
3272// was written e.g.
3273void CodeGenFunction::emitAlignmentAssumptionCheck(
3274 llvm::Value *Ptr, QualType Ty, SourceLocation Loc,
3275 SourceLocation SecondaryLoc, llvm::Value *Alignment,
3276 llvm::Value *OffsetValue, llvm::Value *TheCheck,
3277 llvm::Instruction *Assumption) {
3278 assert(isa_and_nonnull<llvm::CallInst>(Assumption) &&
3279 cast<llvm::CallInst>(Assumption)->getCalledOperand() ==
3280 llvm::Intrinsic::getOrInsertDeclaration(
3281 Builder.GetInsertBlock()->getParent()->getParent(),
3282 llvm::Intrinsic::assume) &&
3283 "Assumption should be a call to llvm.assume().");
3284 assert(&(Builder.GetInsertBlock()->back()) == Assumption &&
3285 "Assumption should be the last instruction of the basic block, "
3286 "since the basic block is still being generated.");
3287
3288 if (!SanOpts.has(K: SanitizerKind::Alignment))
3289 return;
3290
3291 // Don't check pointers to volatile data. The behavior here is implementation-
3292 // defined.
3293 if (Ty->getPointeeType().isVolatileQualified())
3294 return;
3295
3296 // We need to temorairly remove the assumption so we can insert the
3297 // sanitizer check before it, else the check will be dropped by optimizations.
3298 Assumption->removeFromParent();
3299
3300 {
3301 auto CheckOrdinal = SanitizerKind::SO_Alignment;
3302 auto CheckHandler = SanitizerHandler::AlignmentAssumption;
3303 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
3304
3305 if (!OffsetValue)
3306 OffsetValue = Builder.getInt1(V: false); // no offset.
3307
3308 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc),
3309 EmitCheckSourceLocation(Loc: SecondaryLoc),
3310 EmitCheckTypeDescriptor(T: Ty)};
3311 llvm::Value *DynamicData[] = {Ptr, Alignment, OffsetValue};
3312 EmitCheck(Checked: {std::make_pair(x&: TheCheck, y&: CheckOrdinal)}, Check: CheckHandler,
3313 StaticArgs: StaticData, DynamicArgs: DynamicData);
3314 }
3315
3316 // We are now in the (new, empty) "cont" basic block.
3317 // Reintroduce the assumption.
3318 Builder.Insert(I: Assumption);
3319 // FIXME: Assumption still has it's original basic block as it's Parent.
3320}
3321
3322llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
3323 if (CGDebugInfo *DI = getDebugInfo())
3324 return DI->SourceLocToDebugLoc(Loc: Location);
3325
3326 return llvm::DebugLoc();
3327}
3328
3329llvm::Value *
3330CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
3331 Stmt::Likelihood LH) {
3332 switch (LH) {
3333 case Stmt::LH_None:
3334 return Cond;
3335 case Stmt::LH_Likely:
3336 case Stmt::LH_Unlikely:
3337 // Don't generate llvm.expect on -O0 as the backend won't use it for
3338 // anything.
3339 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3340 return Cond;
3341 llvm::Type *CondTy = Cond->getType();
3342 assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");
3343 llvm::Function *FnExpect =
3344 CGM.getIntrinsic(IID: llvm::Intrinsic::expect, Tys: CondTy);
3345 llvm::Value *ExpectedValueOfCond =
3346 llvm::ConstantInt::getBool(Ty: CondTy, V: LH == Stmt::LH_Likely);
3347 return Builder.CreateCall(Callee: FnExpect, Args: {Cond, ExpectedValueOfCond},
3348 Name: Cond->getName() + ".expval");
3349 }
3350 llvm_unreachable("Unknown Likelihood");
3351}
3352
3353llvm::Value *CodeGenFunction::emitBoolVecConversion(llvm::Value *SrcVec,
3354 unsigned NumElementsDst,
3355 const llvm::Twine &Name) {
3356 auto *SrcTy = cast<llvm::FixedVectorType>(Val: SrcVec->getType());
3357 unsigned NumElementsSrc = SrcTy->getNumElements();
3358 if (NumElementsSrc == NumElementsDst)
3359 return SrcVec;
3360
3361 std::vector<int> ShuffleMask(NumElementsDst, -1);
3362 for (unsigned MaskIdx = 0;
3363 MaskIdx < std::min<>(a: NumElementsDst, b: NumElementsSrc); ++MaskIdx)
3364 ShuffleMask[MaskIdx] = MaskIdx;
3365
3366 return Builder.CreateShuffleVector(V: SrcVec, Mask: ShuffleMask, Name);
3367}
3368
3369void CodeGenFunction::EmitPointerAuthOperandBundle(
3370 const CGPointerAuthInfo &PointerAuth,
3371 SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
3372 if (!PointerAuth.isSigned())
3373 return;
3374
3375 auto *Key = Builder.getInt32(C: PointerAuth.getKey());
3376
3377 llvm::Value *Discriminator = PointerAuth.getDiscriminator();
3378 if (!Discriminator)
3379 Discriminator = Builder.getSize(N: 0);
3380
3381 llvm::Value *Args[] = {Key, Discriminator};
3382 Bundles.emplace_back(Args: "ptrauth", Args);
3383}
3384
3385static llvm::Value *EmitPointerAuthCommon(CodeGenFunction &CGF,
3386 const CGPointerAuthInfo &PointerAuth,
3387 llvm::Value *Pointer,
3388 unsigned IntrinsicID) {
3389 if (!PointerAuth)
3390 return Pointer;
3391
3392 auto Key = CGF.Builder.getInt32(C: PointerAuth.getKey());
3393
3394 llvm::Value *Discriminator = PointerAuth.getDiscriminator();
3395 if (!Discriminator) {
3396 Discriminator = CGF.Builder.getSize(N: 0);
3397 }
3398
3399 // Convert the pointer to intptr_t before signing it.
3400 auto OrigType = Pointer->getType();
3401 Pointer = CGF.Builder.CreatePtrToInt(V: Pointer, DestTy: CGF.IntPtrTy);
3402
3403 // call i64 @llvm.ptrauth.sign.i64(i64 %pointer, i32 %key, i64 %discriminator)
3404 auto Intrinsic = CGF.CGM.getIntrinsic(IID: IntrinsicID);
3405 Pointer = CGF.EmitRuntimeCall(callee: Intrinsic, args: {Pointer, Key, Discriminator});
3406
3407 // Convert back to the original type.
3408 Pointer = CGF.Builder.CreateIntToPtr(V: Pointer, DestTy: OrigType);
3409 return Pointer;
3410}
3411
3412llvm::Value *
3413CodeGenFunction::EmitPointerAuthSign(const CGPointerAuthInfo &PointerAuth,
3414 llvm::Value *Pointer) {
3415 if (!PointerAuth.shouldSign())
3416 return Pointer;
3417 return EmitPointerAuthCommon(CGF&: *this, PointerAuth, Pointer,
3418 IntrinsicID: llvm::Intrinsic::ptrauth_sign);
3419}
3420
3421static llvm::Value *EmitStrip(CodeGenFunction &CGF,
3422 const CGPointerAuthInfo &PointerAuth,
3423 llvm::Value *Pointer) {
3424 auto StripIntrinsic = CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::ptrauth_strip);
3425
3426 auto Key = CGF.Builder.getInt32(C: PointerAuth.getKey());
3427 // Convert the pointer to intptr_t before signing it.
3428 auto OrigType = Pointer->getType();
3429 Pointer = CGF.EmitRuntimeCall(
3430 callee: StripIntrinsic, args: {CGF.Builder.CreatePtrToInt(V: Pointer, DestTy: CGF.IntPtrTy), Key});
3431 return CGF.Builder.CreateIntToPtr(V: Pointer, DestTy: OrigType);
3432}
3433
3434llvm::Value *
3435CodeGenFunction::EmitPointerAuthAuth(const CGPointerAuthInfo &PointerAuth,
3436 llvm::Value *Pointer) {
3437 if (PointerAuth.shouldStrip()) {
3438 return EmitStrip(CGF&: *this, PointerAuth, Pointer);
3439 }
3440 if (!PointerAuth.shouldAuth()) {
3441 return Pointer;
3442 }
3443
3444 return EmitPointerAuthCommon(CGF&: *this, PointerAuth, Pointer,
3445 IntrinsicID: llvm::Intrinsic::ptrauth_auth);
3446}
3447
3448void CodeGenFunction::addInstToCurrentSourceAtom(
3449 llvm::Instruction *KeyInstruction, llvm::Value *Backup) {
3450 if (CGDebugInfo *DI = getDebugInfo())
3451 DI->addInstToCurrentSourceAtom(KeyInstruction, Backup);
3452}
3453
3454void CodeGenFunction::addInstToSpecificSourceAtom(
3455 llvm::Instruction *KeyInstruction, llvm::Value *Backup, uint64_t Atom) {
3456 if (CGDebugInfo *DI = getDebugInfo())
3457 DI->addInstToSpecificSourceAtom(KeyInstruction, Backup, Atom);
3458}
3459
3460void CodeGenFunction::addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,
3461 llvm::Value *Backup) {
3462 if (CGDebugInfo *DI = getDebugInfo()) {
3463 ApplyAtomGroup Grp(getDebugInfo());
3464 DI->addInstToCurrentSourceAtom(KeyInstruction, Backup);
3465 }
3466}
3467
3468void CodeGenFunction::emitPFPPostCopyUpdates(Address DestPtr, Address SrcPtr,
3469 QualType Ty) {
3470 for (auto &Field : getContext().findPFPFields(Ty)) {
3471 if (getContext().arePFPFieldsTriviallyCopyable(RD: Field.Field->getParent()))
3472 continue;
3473 auto DestFieldPtr = EmitAddressOfPFPField(RecordPtr: DestPtr, Field);
3474 auto SrcFieldPtr = EmitAddressOfPFPField(RecordPtr: SrcPtr, Field);
3475 Builder.CreateStore(Val: Builder.CreateLoad(Addr: SrcFieldPtr), Addr: DestFieldPtr);
3476 }
3477}
3478