1//===--- CodeGenFunction.cpp - Emit LLVM Code from ASTs for a Function ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This coordinates the per-function state used while generating code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CodeGenFunction.h"
14#include "CGBlocks.h"
15#include "CGCUDARuntime.h"
16#include "CGCXXABI.h"
17#include "CGCleanup.h"
18#include "CGDebugInfo.h"
19#include "CGHLSLRuntime.h"
20#include "CGOpenMPRuntime.h"
21#include "CodeGenModule.h"
22#include "CodeGenPGO.h"
23#include "TargetInfo.h"
24#include "clang/AST/ASTContext.h"
25#include "clang/AST/ASTLambda.h"
26#include "clang/AST/Attr.h"
27#include "clang/AST/Decl.h"
28#include "clang/AST/DeclCXX.h"
29#include "clang/AST/Expr.h"
30#include "clang/AST/IgnoreExpr.h"
31#include "clang/AST/StmtCXX.h"
32#include "clang/AST/StmtObjC.h"
33#include "clang/Basic/Builtins.h"
34#include "clang/Basic/CodeGenOptions.h"
35#include "clang/Basic/DiagnosticFrontend.h"
36#include "clang/Basic/TargetBuiltins.h"
37#include "clang/Basic/TargetInfo.h"
38#include "clang/CodeGen/CGFunctionInfo.h"
39#include "llvm/ADT/ArrayRef.h"
40#include "llvm/ADT/ScopeExit.h"
41#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
42#include "llvm/IR/DataLayout.h"
43#include "llvm/IR/Dominators.h"
44#include "llvm/IR/FPEnv.h"
45#include "llvm/IR/Instruction.h"
46#include "llvm/IR/IntrinsicInst.h"
47#include "llvm/IR/Intrinsics.h"
48#include "llvm/IR/MDBuilder.h"
49#include "llvm/Support/CRC.h"
50#include "llvm/Support/SipHash.h"
51#include "llvm/Support/xxhash.h"
52#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
53#include "llvm/Transforms/Utils/PromoteMemToReg.h"
54#include <optional>
55
56using namespace clang;
57using namespace CodeGen;
58
59/// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time
60/// markers.
61static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts,
62 const LangOptions &LangOpts) {
63 if (CGOpts.DisableLifetimeMarkers)
64 return false;
65
66 // Sanitizers may use markers.
67 if (CGOpts.SanitizeAddressUseAfterScope ||
68 LangOpts.Sanitize.has(K: SanitizerKind::HWAddress) ||
69 LangOpts.Sanitize.has(K: SanitizerKind::Memory) ||
70 LangOpts.Sanitize.has(K: SanitizerKind::MemtagStack))
71 return true;
72
73 // For now, only in optimized builds.
74 return CGOpts.OptimizationLevel != 0;
75}
76
77CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
78 : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
79 Builder(cgm, cgm.getModule().getContext(), CGBuilderInserterTy(this)),
80 SanOpts(CGM.getLangOpts().Sanitize), CurFPFeatures(CGM.getLangOpts()),
81 DebugInfo(CGM.getModuleDebugInfo()),
82 PGO(std::make_unique<CodeGenPGO>(args&: cgm)),
83 ShouldEmitLifetimeMarkers(
84 shouldEmitLifetimeMarkers(CGOpts: CGM.getCodeGenOpts(), LangOpts: CGM.getLangOpts())) {
85 if (!suppressNewContext)
86 CGM.getCXXABI().getMangleContext().startNewFunction();
87 EHStack.setCGF(this);
88
89 SetFastMathFlags(CurFPFeatures);
90}
91
92CodeGenFunction::~CodeGenFunction() {
93 assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup");
94 assert(DeferredDeactivationCleanupStack.empty() &&
95 "missed to deactivate a cleanup");
96
97 if (getLangOpts().OpenMP && CurFn)
98 CGM.getOpenMPRuntime().functionFinished(CGF&: *this);
99
100 // If we have an OpenMPIRBuilder we want to finalize functions (incl.
101 // outlining etc) at some point. Doing it once the function codegen is done
102 // seems to be a reasonable spot. We do it here, as opposed to the deletion
103 // time of the CodeGenModule, because we have to ensure the IR has not yet
104 // been "emitted" to the outside, thus, modifications are still sensible.
105 if (CGM.getLangOpts().OpenMPIRBuilder && CurFn)
106 CGM.getOpenMPRuntime().getOMPBuilder().finalize(Fn: CurFn);
107}
108
109// Map the LangOption for exception behavior into
110// the corresponding enum in the IR.
111llvm::fp::ExceptionBehavior
112clang::ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind) {
113
114 switch (Kind) {
115 case LangOptions::FPE_Ignore: return llvm::fp::ebIgnore;
116 case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap;
117 case LangOptions::FPE_Strict: return llvm::fp::ebStrict;
118 default:
119 llvm_unreachable("Unsupported FP Exception Behavior");
120 }
121}
122
123void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) {
124 llvm::FastMathFlags FMF;
125 FMF.setAllowReassoc(FPFeatures.getAllowFPReassociate());
126 FMF.setNoNaNs(FPFeatures.getNoHonorNaNs());
127 FMF.setNoInfs(FPFeatures.getNoHonorInfs());
128 FMF.setNoSignedZeros(FPFeatures.getNoSignedZero());
129 FMF.setAllowReciprocal(FPFeatures.getAllowReciprocal());
130 FMF.setApproxFunc(FPFeatures.getAllowApproxFunc());
131 FMF.setAllowContract(FPFeatures.allowFPContractAcrossStatement());
132 Builder.setFastMathFlags(FMF);
133}
134
135CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
136 const Expr *E)
137 : CGF(CGF) {
138 ConstructorHelper(FPFeatures: E->getFPFeaturesInEffect(LO: CGF.getLangOpts()));
139}
140
141CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
142 FPOptions FPFeatures)
143 : CGF(CGF) {
144 ConstructorHelper(FPFeatures);
145}
146
147void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) {
148 OldFPFeatures = CGF.CurFPFeatures;
149 CGF.CurFPFeatures = FPFeatures;
150
151 OldExcept = CGF.Builder.getDefaultConstrainedExcept();
152 OldRounding = CGF.Builder.getDefaultConstrainedRounding();
153
154 if (OldFPFeatures == FPFeatures)
155 return;
156
157 FMFGuard.emplace(args&: CGF.Builder);
158
159 llvm::RoundingMode NewRoundingBehavior = FPFeatures.getRoundingMode();
160 CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);
161 auto NewExceptionBehavior =
162 ToConstrainedExceptMD(Kind: FPFeatures.getExceptionMode());
163 CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);
164
165 CGF.SetFastMathFlags(FPFeatures);
166
167 assert((CGF.CurFuncDecl == nullptr || CGF.Builder.getIsFPConstrained() ||
168 isa<CXXConstructorDecl>(CGF.CurFuncDecl) ||
169 isa<CXXDestructorDecl>(CGF.CurFuncDecl) ||
170 (NewExceptionBehavior == llvm::fp::ebIgnore &&
171 NewRoundingBehavior == llvm::RoundingMode::NearestTiesToEven)) &&
172 "FPConstrained should be enabled on entire function");
173
174 auto mergeFnAttrValue = [&](StringRef Name, bool Value) {
175 auto OldValue =
176 CGF.CurFn->getFnAttribute(Kind: Name).getValueAsBool();
177 auto NewValue = OldValue & Value;
178 if (OldValue != NewValue)
179 CGF.CurFn->addFnAttr(Kind: Name, Val: llvm::toStringRef(B: NewValue));
180 };
181 mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.getNoSignedZero());
182}
183
184CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
185 CGF.CurFPFeatures = OldFPFeatures;
186 CGF.Builder.setDefaultConstrainedExcept(OldExcept);
187 CGF.Builder.setDefaultConstrainedRounding(OldRounding);
188}
189
190static LValue
191makeNaturalAlignAddrLValue(llvm::Value *V, QualType T, bool ForPointeeType,
192 bool MightBeSigned, CodeGenFunction &CGF,
193 KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
194 LValueBaseInfo BaseInfo;
195 TBAAAccessInfo TBAAInfo;
196 CharUnits Alignment =
197 CGF.CGM.getNaturalTypeAlignment(T, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo, forPointeeType: ForPointeeType);
198 Address Addr =
199 MightBeSigned
200 ? CGF.makeNaturalAddressForPointer(Ptr: V, T, Alignment, ForPointeeType: false, BaseInfo: nullptr,
201 TBAAInfo: nullptr, IsKnownNonNull)
202 : Address(V, CGF.ConvertTypeForMem(T), Alignment, IsKnownNonNull);
203 return CGF.MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
204}
205
206LValue
207CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
208 KnownNonNull_t IsKnownNonNull) {
209 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,
210 /*MightBeSigned*/ true, CGF&: *this,
211 IsKnownNonNull);
212}
213
214LValue
215CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
216 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,
217 /*MightBeSigned*/ true, CGF&: *this);
218}
219
220LValue CodeGenFunction::MakeNaturalAlignRawAddrLValue(llvm::Value *V,
221 QualType T) {
222 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,
223 /*MightBeSigned*/ false, CGF&: *this);
224}
225
226LValue CodeGenFunction::MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V,
227 QualType T) {
228 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,
229 /*MightBeSigned*/ false, CGF&: *this);
230}
231
232llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
233 return CGM.getTypes().ConvertTypeForMem(T);
234}
235
236llvm::Type *CodeGenFunction::ConvertType(QualType T) {
237 return CGM.getTypes().ConvertType(T);
238}
239
240llvm::Type *CodeGenFunction::convertTypeForLoadStore(QualType ASTTy,
241 llvm::Type *LLVMTy) {
242 return CGM.getTypes().convertTypeForLoadStore(T: ASTTy, LLVMTy);
243}
244
245TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
246 type = type.getCanonicalType();
247 while (true) {
248 switch (type->getTypeClass()) {
249#define TYPE(name, parent)
250#define ABSTRACT_TYPE(name, parent)
251#define NON_CANONICAL_TYPE(name, parent) case Type::name:
252#define DEPENDENT_TYPE(name, parent) case Type::name:
253#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
254#include "clang/AST/TypeNodes.inc"
255 llvm_unreachable("non-canonical or dependent type in IR-generation");
256
257 case Type::Auto:
258 case Type::DeducedTemplateSpecialization:
259 llvm_unreachable("undeduced type in IR-generation");
260
261 // Various scalar types.
262 case Type::Builtin:
263 case Type::Pointer:
264 case Type::BlockPointer:
265 case Type::LValueReference:
266 case Type::RValueReference:
267 case Type::MemberPointer:
268 case Type::Vector:
269 case Type::ExtVector:
270 case Type::ConstantMatrix:
271 case Type::FunctionProto:
272 case Type::FunctionNoProto:
273 case Type::Enum:
274 case Type::ObjCObjectPointer:
275 case Type::Pipe:
276 case Type::BitInt:
277 case Type::HLSLAttributedResource:
278 case Type::HLSLInlineSpirv:
279 case Type::OverflowBehavior:
280 return TEK_Scalar;
281
282 // Complexes.
283 case Type::Complex:
284 return TEK_Complex;
285
286 // Arrays, records, and Objective-C objects.
287 case Type::ConstantArray:
288 case Type::IncompleteArray:
289 case Type::VariableArray:
290 case Type::Record:
291 case Type::ObjCObject:
292 case Type::ObjCInterface:
293 case Type::ArrayParameter:
294 return TEK_Aggregate;
295
296 // We operate on atomic values according to their underlying type.
297 case Type::Atomic:
298 type = cast<AtomicType>(Val&: type)->getValueType();
299 continue;
300 }
301 llvm_unreachable("unknown type kind!");
302 }
303}
304
305llvm::DebugLoc CodeGenFunction::EmitReturnBlock() {
306 // For cleanliness, we try to avoid emitting the return block for
307 // simple cases.
308 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
309
310 if (CurBB) {
311 assert(!CurBB->getTerminator() && "Unexpected terminated block.");
312
313 // We have a valid insert point, reuse it if it is empty or there are no
314 // explicit jumps to the return block.
315 if (CurBB->empty() || ReturnBlock.getBlock()->use_empty()) {
316 ReturnBlock.getBlock()->replaceAllUsesWith(V: CurBB);
317 delete ReturnBlock.getBlock();
318 ReturnBlock = JumpDest();
319 } else
320 EmitBlock(BB: ReturnBlock.getBlock());
321 return llvm::DebugLoc();
322 }
323
324 // Otherwise, if the return block is the target of a single direct
325 // branch then we can just put the code in that block instead. This
326 // cleans up functions which started with a unified return block.
327 if (ReturnBlock.getBlock()->hasOneUse()) {
328 llvm::BranchInst *BI =
329 dyn_cast<llvm::BranchInst>(Val: *ReturnBlock.getBlock()->user_begin());
330 if (BI && BI->isUnconditional() &&
331 BI->getSuccessor(Idx: 0) == ReturnBlock.getBlock()) {
332 // Record/return the DebugLoc of the simple 'return' expression to be used
333 // later by the actual 'ret' instruction.
334 llvm::DebugLoc Loc = BI->getDebugLoc();
335 Builder.SetInsertPoint(BI->getParent());
336 BI->eraseFromParent();
337 delete ReturnBlock.getBlock();
338 ReturnBlock = JumpDest();
339 return Loc;
340 }
341 }
342
343 // FIXME: We are at an unreachable point, there is no reason to emit the block
344 // unless it has uses. However, we still need a place to put the debug
345 // region.end for now.
346
347 EmitBlock(BB: ReturnBlock.getBlock());
348 return llvm::DebugLoc();
349}
350
351static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
352 if (!BB) return;
353 if (!BB->use_empty()) {
354 CGF.CurFn->insert(Position: CGF.CurFn->end(), BB);
355 return;
356 }
357 delete BB;
358}
359
360void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
361 assert(BreakContinueStack.empty() &&
362 "mismatched push/pop in break/continue stack!");
363 assert(LifetimeExtendedCleanupStack.empty() &&
364 "mismatched push/pop of cleanups in EHStack!");
365 assert(DeferredDeactivationCleanupStack.empty() &&
366 "mismatched activate/deactivate of cleanups!");
367
368 if (CGM.shouldEmitConvergenceTokens()) {
369 ConvergenceTokenStack.pop_back();
370 assert(ConvergenceTokenStack.empty() &&
371 "mismatched push/pop in convergence stack!");
372 }
373
374 bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
375 && NumSimpleReturnExprs == NumReturnExprs
376 && ReturnBlock.getBlock()->use_empty();
377 // Usually the return expression is evaluated before the cleanup
378 // code. If the function contains only a simple return statement,
379 // such as a constant, the location before the cleanup code becomes
380 // the last useful breakpoint in the function, because the simple
381 // return expression will be evaluated after the cleanup code. To be
382 // safe, set the debug location for cleanup code to the location of
383 // the return statement. Otherwise the cleanup code should be at the
384 // end of the function's lexical scope.
385 //
386 // If there are multiple branches to the return block, the branch
387 // instructions will get the location of the return statements and
388 // all will be fine.
389 if (CGDebugInfo *DI = getDebugInfo()) {
390 if (OnlySimpleReturnStmts)
391 DI->EmitLocation(Builder, Loc: LastStopPoint);
392 else
393 DI->EmitLocation(Builder, Loc: EndLoc);
394 }
395
396 // Pop any cleanups that might have been associated with the
397 // parameters. Do this in whatever block we're currently in; it's
398 // important to do this before we enter the return block or return
399 // edges will be *really* confused.
400 bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth;
401 bool HasOnlyNoopCleanups =
402 HasCleanups && EHStack.containsOnlyNoopCleanups(Old: PrologueCleanupDepth);
403 bool EmitRetDbgLoc = !HasCleanups || HasOnlyNoopCleanups;
404
405 std::optional<ApplyDebugLocation> OAL;
406 if (HasCleanups) {
407 // Make sure the line table doesn't jump back into the body for
408 // the ret after it's been at EndLoc.
409 if (CGDebugInfo *DI = getDebugInfo()) {
410 if (OnlySimpleReturnStmts)
411 DI->EmitLocation(Builder, Loc: EndLoc);
412 else
413 // We may not have a valid end location. Try to apply it anyway, and
414 // fall back to an artificial location if needed.
415 OAL = ApplyDebugLocation::CreateDefaultArtificial(CGF&: *this, TemporaryLocation: EndLoc);
416 }
417
418 PopCleanupBlocks(OldCleanupStackSize: PrologueCleanupDepth);
419 }
420
421 // Emit function epilog (to return).
422 llvm::DebugLoc Loc = EmitReturnBlock();
423
424 if (ShouldInstrumentFunction()) {
425 if (CGM.getCodeGenOpts().InstrumentFunctions)
426 CurFn->addFnAttr(Kind: "instrument-function-exit", Val: "__cyg_profile_func_exit");
427 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
428 CurFn->addFnAttr(Kind: "instrument-function-exit-inlined",
429 Val: "__cyg_profile_func_exit");
430 }
431
432 // Emit debug descriptor for function end.
433 if (CGDebugInfo *DI = getDebugInfo())
434 DI->EmitFunctionEnd(Builder, Fn: CurFn);
435
436 // Reset the debug location to that of the simple 'return' expression, if any
437 // rather than that of the end of the function's scope '}'.
438 uint64_t RetKeyInstructionsAtomGroup = Loc ? Loc->getAtomGroup() : 0;
439 ApplyDebugLocation AL(*this, Loc);
440 EmitFunctionEpilog(FI: *CurFnInfo, EmitRetDbgLoc, EndLoc,
441 RetKeyInstructionsSourceAtom: RetKeyInstructionsAtomGroup);
442 EmitEndEHSpec(D: CurCodeDecl);
443
444 assert(EHStack.empty() &&
445 "did not remove all scopes from cleanup stack!");
446
447 // If someone did an indirect goto, emit the indirect goto block at the end of
448 // the function.
449 if (IndirectBranch) {
450 EmitBlock(BB: IndirectBranch->getParent());
451 Builder.ClearInsertionPoint();
452 }
453
454 // If some of our locals escaped, insert a call to llvm.localescape in the
455 // entry block.
456 if (!EscapedLocals.empty()) {
457 // Invert the map from local to index into a simple vector. There should be
458 // no holes.
459 SmallVector<llvm::Value *, 4> EscapeArgs;
460 EscapeArgs.resize(N: EscapedLocals.size());
461 for (auto &Pair : EscapedLocals)
462 EscapeArgs[Pair.second] = Pair.first;
463 llvm::Function *FrameEscapeFn = llvm::Intrinsic::getOrInsertDeclaration(
464 M: &CGM.getModule(), id: llvm::Intrinsic::localescape);
465 CGBuilderTy(CGM, AllocaInsertPt).CreateCall(Callee: FrameEscapeFn, Args: EscapeArgs);
466 }
467
468 // Remove the AllocaInsertPt instruction, which is just a convenience for us.
469 llvm::Instruction *Ptr = AllocaInsertPt;
470 AllocaInsertPt = nullptr;
471 Ptr->eraseFromParent();
472
473 // PostAllocaInsertPt, if created, was lazily created when it was required,
474 // remove it now since it was just created for our own convenience.
475 if (PostAllocaInsertPt) {
476 llvm::Instruction *PostPtr = PostAllocaInsertPt;
477 PostAllocaInsertPt = nullptr;
478 PostPtr->eraseFromParent();
479 }
480
481 // If someone took the address of a label but never did an indirect goto, we
482 // made a zero entry PHI node, which is illegal, zap it now.
483 if (IndirectBranch) {
484 llvm::PHINode *PN = cast<llvm::PHINode>(Val: IndirectBranch->getAddress());
485 if (PN->getNumIncomingValues() == 0) {
486 PN->replaceAllUsesWith(V: llvm::PoisonValue::get(T: PN->getType()));
487 PN->eraseFromParent();
488 }
489 }
490
491 EmitIfUsed(CGF&: *this, BB: EHResumeBlock);
492 EmitIfUsed(CGF&: *this, BB: TerminateLandingPad);
493 EmitIfUsed(CGF&: *this, BB: TerminateHandler);
494 EmitIfUsed(CGF&: *this, BB: UnreachableBlock);
495
496 for (const auto &FuncletAndParent : TerminateFunclets)
497 EmitIfUsed(CGF&: *this, BB: FuncletAndParent.second);
498
499 if (CGM.getCodeGenOpts().EmitDeclMetadata)
500 EmitDeclMetadata();
501
502 for (const auto &R : DeferredReplacements) {
503 if (llvm::Value *Old = R.first) {
504 Old->replaceAllUsesWith(V: R.second);
505 cast<llvm::Instruction>(Val: Old)->eraseFromParent();
506 }
507 }
508 DeferredReplacements.clear();
509
510 // Eliminate CleanupDestSlot alloca by replacing it with SSA values and
511 // PHIs if the current function is a coroutine. We don't do it for all
512 // functions as it may result in slight increase in numbers of instructions
513 // if compiled with no optimizations. We do it for coroutine as the lifetime
514 // of CleanupDestSlot alloca make correct coroutine frame building very
515 // difficult.
516 if (NormalCleanupDest.isValid() && isCoroutine()) {
517 llvm::DominatorTree DT(*CurFn);
518 llvm::PromoteMemToReg(
519 Allocas: cast<llvm::AllocaInst>(Val: NormalCleanupDest.getPointer()), DT);
520 NormalCleanupDest = Address::invalid();
521 }
522
523 // Scan function arguments for vector width.
524 for (llvm::Argument &A : CurFn->args())
525 if (auto *VT = dyn_cast<llvm::VectorType>(Val: A.getType()))
526 LargestVectorWidth =
527 std::max(a: (uint64_t)LargestVectorWidth,
528 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
529
530 // Update vector width based on return type.
531 if (auto *VT = dyn_cast<llvm::VectorType>(Val: CurFn->getReturnType()))
532 LargestVectorWidth =
533 std::max(a: (uint64_t)LargestVectorWidth,
534 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
535
536 if (CurFnInfo->getMaxVectorWidth() > LargestVectorWidth)
537 LargestVectorWidth = CurFnInfo->getMaxVectorWidth();
538
539 // Add the min-legal-vector-width attribute. This contains the max width from:
540 // 1. min-vector-width attribute used in the source program.
541 // 2. Any builtins used that have a vector width specified.
542 // 3. Values passed in and out of inline assembly.
543 // 4. Width of vector arguments and return types for this function.
544 // 5. Width of vector arguments and return types for functions called by this
545 // function.
546 if (getContext().getTargetInfo().getTriple().isX86())
547 CurFn->addFnAttr(Kind: "min-legal-vector-width",
548 Val: llvm::utostr(X: LargestVectorWidth));
549
550 // If we generated an unreachable return block, delete it now.
551 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()) {
552 Builder.ClearInsertionPoint();
553 ReturnBlock.getBlock()->eraseFromParent();
554 }
555 if (ReturnValue.isValid()) {
556 auto *RetAlloca =
557 dyn_cast<llvm::AllocaInst>(Val: ReturnValue.emitRawPointer(CGF&: *this));
558 if (RetAlloca && RetAlloca->use_empty()) {
559 RetAlloca->eraseFromParent();
560 ReturnValue = Address::invalid();
561 }
562 }
563}
564
565/// ShouldInstrumentFunction - Return true if the current function should be
566/// instrumented with __cyg_profile_func_* calls
567bool CodeGenFunction::ShouldInstrumentFunction() {
568 if (!CGM.getCodeGenOpts().InstrumentFunctions &&
569 !CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining &&
570 !CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
571 return false;
572 if (!CurFuncDecl || CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>())
573 return false;
574 return true;
575}
576
577bool CodeGenFunction::ShouldSkipSanitizerInstrumentation() {
578 if (!CurFuncDecl)
579 return false;
580 return CurFuncDecl->hasAttr<DisableSanitizerInstrumentationAttr>();
581}
582
583/// ShouldXRayInstrument - Return true if the current function should be
584/// instrumented with XRay nop sleds.
585bool CodeGenFunction::ShouldXRayInstrumentFunction() const {
586 return CGM.getCodeGenOpts().XRayInstrumentFunctions;
587}
588
589/// AlwaysEmitXRayCustomEvents - Return true if we should emit IR for calls to
590/// the __xray_customevent(...) builtin calls, when doing XRay instrumentation.
591bool CodeGenFunction::AlwaysEmitXRayCustomEvents() const {
592 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
593 (CGM.getCodeGenOpts().XRayAlwaysEmitCustomEvents ||
594 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
595 XRayInstrKind::Custom);
596}
597
598bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const {
599 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
600 (CGM.getCodeGenOpts().XRayAlwaysEmitTypedEvents ||
601 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
602 XRayInstrKind::Typed);
603}
604
605llvm::ConstantInt *
606CodeGenFunction::getUBSanFunctionTypeHash(QualType Ty) const {
607 // Remove any (C++17) exception specifications, to allow calling e.g. a
608 // noexcept function through a non-noexcept pointer.
609 if (!Ty->isFunctionNoProtoType())
610 Ty = getContext().getFunctionTypeWithExceptionSpec(Orig: Ty, ESI: EST_None);
611 std::string Mangled;
612 llvm::raw_string_ostream Out(Mangled);
613 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: Ty, Out, NormalizeIntegers: false);
614 return llvm::ConstantInt::get(
615 Ty: CGM.Int32Ty, V: static_cast<uint32_t>(llvm::xxh3_64bits(data: Mangled)));
616}
617
618void CodeGenFunction::EmitKernelMetadata(const FunctionDecl *FD,
619 llvm::Function *Fn) {
620 if (!FD->hasAttr<DeviceKernelAttr>() && !FD->hasAttr<CUDAGlobalAttr>())
621 return;
622
623 llvm::LLVMContext &Context = getLLVMContext();
624
625 CGM.GenKernelArgMetadata(FN: Fn, FD, CGF: this);
626
627 if (!(getLangOpts().OpenCL ||
628 (getLangOpts().CUDA &&
629 getContext().getTargetInfo().getTriple().isSPIRV())))
630 return;
631
632 if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {
633 QualType HintQTy = A->getTypeHint();
634 const ExtVectorType *HintEltQTy = HintQTy->getAs<ExtVectorType>();
635 bool IsSignedInteger =
636 HintQTy->isSignedIntegerType() ||
637 (HintEltQTy && HintEltQTy->getElementType()->isSignedIntegerType());
638 llvm::Metadata *AttrMDArgs[] = {
639 llvm::ConstantAsMetadata::get(C: llvm::PoisonValue::get(
640 T: CGM.getTypes().ConvertType(T: A->getTypeHint()))),
641 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
642 Ty: llvm::IntegerType::get(C&: Context, NumBits: 32),
643 V: llvm::APInt(32, (uint64_t)(IsSignedInteger ? 1 : 0))))};
644 Fn->setMetadata(Kind: "vec_type_hint", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
645 }
646
647 if (const WorkGroupSizeHintAttr *A = FD->getAttr<WorkGroupSizeHintAttr>()) {
648 auto Eval = [&](Expr *E) {
649 return E->EvaluateKnownConstInt(Ctx: FD->getASTContext()).getExtValue();
650 };
651 llvm::Metadata *AttrMDArgs[] = {
652 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getXDim()))),
653 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getYDim()))),
654 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getZDim())))};
655 Fn->setMetadata(Kind: "work_group_size_hint", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
656 }
657
658 if (const ReqdWorkGroupSizeAttr *A = FD->getAttr<ReqdWorkGroupSizeAttr>()) {
659 auto Eval = [&](Expr *E) {
660 return E->EvaluateKnownConstInt(Ctx: FD->getASTContext()).getExtValue();
661 };
662 llvm::Metadata *AttrMDArgs[] = {
663 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getXDim()))),
664 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getYDim()))),
665 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getZDim())))};
666 Fn->setMetadata(Kind: "reqd_work_group_size", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
667 }
668
669 if (const OpenCLIntelReqdSubGroupSizeAttr *A =
670 FD->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
671 llvm::Metadata *AttrMDArgs[] = {
672 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: A->getSubGroupSize()))};
673 Fn->setMetadata(Kind: "intel_reqd_sub_group_size",
674 Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
675 }
676}
677
678/// Determine whether the function F ends with a return stmt.
679static bool endsWithReturn(const Decl* F) {
680 const Stmt *Body = nullptr;
681 if (auto *FD = dyn_cast_or_null<FunctionDecl>(Val: F))
682 Body = FD->getBody();
683 else if (auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(Val: F))
684 Body = OMD->getBody();
685
686 if (auto *CS = dyn_cast_or_null<CompoundStmt>(Val: Body)) {
687 auto LastStmt = CS->body_rbegin();
688 if (LastStmt != CS->body_rend())
689 return isa<ReturnStmt>(Val: *LastStmt);
690 }
691 return false;
692}
693
694void CodeGenFunction::markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {
695 if (SanOpts.has(K: SanitizerKind::Thread)) {
696 Fn->addFnAttr(Kind: "sanitize_thread_no_checking_at_run_time");
697 Fn->removeFnAttr(Kind: llvm::Attribute::SanitizeThread);
698 }
699}
700
701/// Check if the return value of this function requires sanitization.
702bool CodeGenFunction::requiresReturnValueCheck() const {
703 return requiresReturnValueNullabilityCheck() ||
704 (SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute) && CurCodeDecl &&
705 CurCodeDecl->getAttr<ReturnsNonNullAttr>());
706}
707
708static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) {
709 auto *MD = dyn_cast_or_null<CXXMethodDecl>(Val: D);
710 if (!MD || !MD->getDeclName().getAsIdentifierInfo() ||
711 !MD->getDeclName().getAsIdentifierInfo()->isStr(Str: "allocate") ||
712 (MD->getNumParams() != 1 && MD->getNumParams() != 2))
713 return false;
714
715 if (!Ctx.hasSameType(T1: MD->parameters()[0]->getType(), T2: Ctx.getSizeType()))
716 return false;
717
718 if (MD->getNumParams() == 2) {
719 auto *PT = MD->parameters()[1]->getType()->getAs<PointerType>();
720 if (!PT || !PT->isVoidPointerType() ||
721 !PT->getPointeeType().isConstQualified())
722 return false;
723 }
724
725 return true;
726}
727
728bool CodeGenFunction::isInAllocaArgument(CGCXXABI &ABI, QualType Ty) {
729 const CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
730 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
731}
732
733bool CodeGenFunction::hasInAllocaArg(const CXXMethodDecl *MD) {
734 return getTarget().getTriple().getArch() == llvm::Triple::x86 &&
735 getTarget().getCXXABI().isMicrosoft() &&
736 llvm::any_of(Range: MD->parameters(), P: [&](ParmVarDecl *P) {
737 return isInAllocaArgument(ABI&: CGM.getCXXABI(), Ty: P->getType());
738 });
739}
740
741/// Return the UBSan prologue signature for \p FD if one is available.
742static llvm::Constant *getPrologueSignature(CodeGenModule &CGM,
743 const FunctionDecl *FD) {
744 if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: FD))
745 if (!MD->isStatic())
746 return nullptr;
747 return CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM);
748}
749
750void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
751 llvm::Function *Fn,
752 const CGFunctionInfo &FnInfo,
753 const FunctionArgList &Args,
754 SourceLocation Loc,
755 SourceLocation StartLoc) {
756 assert(!CurFn &&
757 "Do not use a CodeGenFunction object for more than one function");
758
759 const Decl *D = GD.getDecl();
760
761 DidCallStackSave = false;
762 CurCodeDecl = D;
763 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
764 if (FD && FD->usesSEHTry())
765 CurSEHParent = GD;
766 CurFuncDecl = (D ? D->getNonClosureContext() : nullptr);
767 FnRetTy = RetTy;
768 CurFn = Fn;
769 CurFnInfo = &FnInfo;
770 assert(CurFn->isDeclaration() && "Function already has body?");
771
772 // If this function is ignored for any of the enabled sanitizers,
773 // disable the sanitizer for the function.
774 do {
775#define SANITIZER(NAME, ID) \
776 if (SanOpts.empty()) \
777 break; \
778 if (SanOpts.has(SanitizerKind::ID)) \
779 if (CGM.isInNoSanitizeList(SanitizerKind::ID, Fn, Loc)) \
780 SanOpts.set(SanitizerKind::ID, false);
781
782#include "clang/Basic/Sanitizers.def"
783#undef SANITIZER
784 } while (false);
785
786 if (D) {
787 const bool SanitizeBounds = SanOpts.hasOneOf(K: SanitizerKind::Bounds);
788 SanitizerMask no_sanitize_mask;
789 bool NoSanitizeCoverage = false;
790
791 for (auto *Attr : D->specific_attrs<NoSanitizeAttr>()) {
792 no_sanitize_mask |= Attr->getMask();
793 // SanitizeCoverage is not handled by SanOpts.
794 if (Attr->hasCoverage())
795 NoSanitizeCoverage = true;
796 }
797
798 // Apply the no_sanitize* attributes to SanOpts.
799 SanOpts.Mask &= ~no_sanitize_mask;
800 if (no_sanitize_mask & SanitizerKind::Address)
801 SanOpts.set(K: SanitizerKind::KernelAddress, Value: false);
802 if (no_sanitize_mask & SanitizerKind::KernelAddress)
803 SanOpts.set(K: SanitizerKind::Address, Value: false);
804 if (no_sanitize_mask & SanitizerKind::HWAddress)
805 SanOpts.set(K: SanitizerKind::KernelHWAddress, Value: false);
806 if (no_sanitize_mask & SanitizerKind::KernelHWAddress)
807 SanOpts.set(K: SanitizerKind::HWAddress, Value: false);
808
809 if (SanitizeBounds && !SanOpts.hasOneOf(K: SanitizerKind::Bounds))
810 Fn->addFnAttr(Kind: llvm::Attribute::NoSanitizeBounds);
811
812 if (NoSanitizeCoverage && CGM.getCodeGenOpts().hasSanitizeCoverage())
813 Fn->addFnAttr(Kind: llvm::Attribute::NoSanitizeCoverage);
814
815 // Some passes need the non-negated no_sanitize attribute. Pass them on.
816 if (CGM.getCodeGenOpts().hasSanitizeBinaryMetadata()) {
817 if (no_sanitize_mask & SanitizerKind::Thread)
818 Fn->addFnAttr(Kind: "no_sanitize_thread");
819 }
820 }
821
822 if (ShouldSkipSanitizerInstrumentation()) {
823 CurFn->addFnAttr(Kind: llvm::Attribute::DisableSanitizerInstrumentation);
824 } else {
825 // Apply sanitizer attributes to the function.
826 if (SanOpts.hasOneOf(K: SanitizerKind::Address | SanitizerKind::KernelAddress))
827 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAddress);
828 if (SanOpts.hasOneOf(K: SanitizerKind::HWAddress |
829 SanitizerKind::KernelHWAddress))
830 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeHWAddress);
831 if (SanOpts.has(K: SanitizerKind::MemtagStack))
832 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemTag);
833 if (SanOpts.has(K: SanitizerKind::Thread))
834 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeThread);
835 if (SanOpts.has(K: SanitizerKind::Type))
836 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeType);
837 if (SanOpts.has(K: SanitizerKind::NumericalStability))
838 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeNumericalStability);
839 if (SanOpts.hasOneOf(K: SanitizerKind::Memory | SanitizerKind::KernelMemory))
840 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemory);
841 if (SanOpts.has(K: SanitizerKind::AllocToken))
842 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAllocToken);
843 }
844 if (SanOpts.has(K: SanitizerKind::SafeStack))
845 Fn->addFnAttr(Kind: llvm::Attribute::SafeStack);
846 if (SanOpts.has(K: SanitizerKind::ShadowCallStack))
847 Fn->addFnAttr(Kind: llvm::Attribute::ShadowCallStack);
848
849 if (SanOpts.has(K: SanitizerKind::Realtime))
850 if (FD && FD->getASTContext().hasAnyFunctionEffects())
851 for (const FunctionEffectWithCondition &Fe : FD->getFunctionEffects()) {
852 if (Fe.Effect.kind() == FunctionEffect::Kind::NonBlocking)
853 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeRealtime);
854 else if (Fe.Effect.kind() == FunctionEffect::Kind::Blocking)
855 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeRealtimeBlocking);
856 }
857
858 // Apply fuzzing attribute to the function.
859 if (SanOpts.hasOneOf(K: SanitizerKind::Fuzzer | SanitizerKind::FuzzerNoLink))
860 Fn->addFnAttr(Kind: llvm::Attribute::OptForFuzzing);
861
862 // Ignore TSan memory acesses from within ObjC/ObjC++ dealloc, initialize,
863 // .cxx_destruct, __destroy_helper_block_ and all of their calees at run time.
864 if (SanOpts.has(K: SanitizerKind::Thread)) {
865 if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(Val: D)) {
866 const IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(argIndex: 0);
867 if (OMD->getMethodFamily() == OMF_dealloc ||
868 OMD->getMethodFamily() == OMF_initialize ||
869 (OMD->getSelector().isUnarySelector() && II->isStr(Str: ".cxx_destruct"))) {
870 markAsIgnoreThreadCheckingAtRuntime(Fn);
871 }
872 }
873 }
874
875 // Ignore unrelated casts in STL allocate() since the allocator must cast
876 // from void* to T* before object initialization completes. Don't match on the
877 // namespace because not all allocators are in std::
878 if (D && SanOpts.has(K: SanitizerKind::CFIUnrelatedCast)) {
879 if (matchesStlAllocatorFn(D, Ctx: getContext()))
880 SanOpts.Mask &= ~SanitizerKind::CFIUnrelatedCast;
881 }
882
883 // Ignore null checks in coroutine functions since the coroutines passes
884 // are not aware of how to move the extra UBSan instructions across the split
885 // coroutine boundaries.
886 if (D && SanOpts.has(K: SanitizerKind::Null))
887 if (FD && FD->getBody() &&
888 FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass)
889 SanOpts.Mask &= ~SanitizerKind::Null;
890
891 // Apply xray attributes to the function (as a string, for now)
892 bool AlwaysXRayAttr = false;
893 if (const auto *XRayAttr = D ? D->getAttr<XRayInstrumentAttr>() : nullptr) {
894 if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
895 K: XRayInstrKind::FunctionEntry) ||
896 CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
897 K: XRayInstrKind::FunctionExit)) {
898 if (XRayAttr->alwaysXRayInstrument() && ShouldXRayInstrumentFunction()) {
899 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-always");
900 AlwaysXRayAttr = true;
901 }
902 if (XRayAttr->neverXRayInstrument())
903 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-never");
904 if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())
905 if (ShouldXRayInstrumentFunction())
906 Fn->addFnAttr(Kind: "xray-log-args",
907 Val: llvm::utostr(X: LogArgs->getArgumentCount()));
908 }
909 } else {
910 if (ShouldXRayInstrumentFunction() && !CGM.imbueXRayAttrs(Fn, Loc))
911 Fn->addFnAttr(
912 Kind: "xray-instruction-threshold",
913 Val: llvm::itostr(X: CGM.getCodeGenOpts().XRayInstructionThreshold));
914 }
915
916 if (ShouldXRayInstrumentFunction()) {
917 if (CGM.getCodeGenOpts().XRayIgnoreLoops)
918 Fn->addFnAttr(Kind: "xray-ignore-loops");
919
920 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
921 K: XRayInstrKind::FunctionExit))
922 Fn->addFnAttr(Kind: "xray-skip-exit");
923
924 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
925 K: XRayInstrKind::FunctionEntry))
926 Fn->addFnAttr(Kind: "xray-skip-entry");
927
928 auto FuncGroups = CGM.getCodeGenOpts().XRayTotalFunctionGroups;
929 if (FuncGroups > 1) {
930 auto FuncName = llvm::ArrayRef<uint8_t>(CurFn->getName().bytes_begin(),
931 CurFn->getName().bytes_end());
932 auto Group = crc32(Data: FuncName) % FuncGroups;
933 if (Group != CGM.getCodeGenOpts().XRaySelectedFunctionGroup &&
934 !AlwaysXRayAttr)
935 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-never");
936 }
937 }
938
939 if (CGM.getCodeGenOpts().getProfileInstr() !=
940 llvm::driver::ProfileInstrKind::ProfileNone) {
941 switch (CGM.isFunctionBlockedFromProfileInstr(Fn, Loc)) {
942 case ProfileList::Skip:
943 Fn->addFnAttr(Kind: llvm::Attribute::SkipProfile);
944 break;
945 case ProfileList::Forbid:
946 Fn->addFnAttr(Kind: llvm::Attribute::NoProfile);
947 break;
948 case ProfileList::Allow:
949 break;
950 }
951 }
952
953 unsigned Count, Offset;
954 StringRef Section;
955 if (const auto *Attr =
956 D ? D->getAttr<PatchableFunctionEntryAttr>() : nullptr) {
957 Count = Attr->getCount();
958 Offset = Attr->getOffset();
959 Section = Attr->getSection();
960 } else {
961 Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;
962 Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;
963 }
964 if (Section.empty())
965 Section = CGM.getCodeGenOpts().PatchableFunctionEntrySection;
966 if (Count && Offset <= Count) {
967 Fn->addFnAttr(Kind: "patchable-function-entry", Val: std::to_string(val: Count - Offset));
968 if (Offset)
969 Fn->addFnAttr(Kind: "patchable-function-prefix", Val: std::to_string(val: Offset));
970 if (!Section.empty())
971 Fn->addFnAttr(Kind: "patchable-function-entry-section", Val: Section);
972 }
973 // Instruct that functions for COFF/CodeView targets should start with a
974 // patchable instruction, but only on x86/x64. Don't forward this to ARM/ARM64
975 // backends as they don't need it -- instructions on these architectures are
976 // always atomically patchable at runtime.
977 if (CGM.getCodeGenOpts().HotPatch &&
978 getContext().getTargetInfo().getTriple().isX86() &&
979 getContext().getTargetInfo().getTriple().getEnvironment() !=
980 llvm::Triple::CODE16)
981 Fn->addFnAttr(Kind: "patchable-function", Val: "prologue-short-redirect");
982
983 // Add no-jump-tables value.
984 if (CGM.getCodeGenOpts().NoUseJumpTables)
985 Fn->addFnAttr(Kind: "no-jump-tables", Val: "true");
986
987 // Add no-inline-line-tables value.
988 if (CGM.getCodeGenOpts().NoInlineLineTables)
989 Fn->addFnAttr(Kind: "no-inline-line-tables");
990
991 // Add profile-sample-accurate value.
992 if (CGM.getCodeGenOpts().ProfileSampleAccurate)
993 Fn->addFnAttr(Kind: "profile-sample-accurate");
994
995 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
996 Fn->addFnAttr(Kind: "use-sample-profile");
997
998 if (D && D->hasAttr<CFICanonicalJumpTableAttr>())
999 Fn->addFnAttr(Kind: "cfi-canonical-jump-table");
1000
1001 if (D && D->hasAttr<NoProfileFunctionAttr>())
1002 Fn->addFnAttr(Kind: llvm::Attribute::NoProfile);
1003
1004 if (D && D->hasAttr<HybridPatchableAttr>())
1005 Fn->addFnAttr(Kind: llvm::Attribute::HybridPatchable);
1006
1007 if (D) {
1008 // Function attributes take precedence over command line flags.
1009 if (auto *A = D->getAttr<FunctionReturnThunksAttr>()) {
1010 switch (A->getThunkType()) {
1011 case FunctionReturnThunksAttr::Kind::Keep:
1012 break;
1013 case FunctionReturnThunksAttr::Kind::Extern:
1014 Fn->addFnAttr(Kind: llvm::Attribute::FnRetThunkExtern);
1015 break;
1016 }
1017 } else if (CGM.getCodeGenOpts().FunctionReturnThunks)
1018 Fn->addFnAttr(Kind: llvm::Attribute::FnRetThunkExtern);
1019 }
1020
1021 if (FD && (getLangOpts().OpenCL ||
1022 (getLangOpts().CUDA &&
1023 getContext().getTargetInfo().getTriple().isSPIRV()) ||
1024 ((getLangOpts().HIP || getLangOpts().OffloadViaLLVM) &&
1025 getLangOpts().CUDAIsDevice))) {
1026 // Add metadata for a kernel function.
1027 EmitKernelMetadata(FD, Fn);
1028 }
1029
1030 if (FD && FD->hasAttr<ClspvLibclcBuiltinAttr>()) {
1031 Fn->setMetadata(Kind: "clspv_libclc_builtin",
1032 Node: llvm::MDNode::get(Context&: getLLVMContext(), MDs: {}));
1033 }
1034
1035 // If we are checking function types, emit a function type signature as
1036 // prologue data.
1037 if (FD && SanOpts.has(K: SanitizerKind::Function) &&
1038 !FD->getType()->isCFIUncheckedCalleeFunctionType()) {
1039 if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {
1040 llvm::LLVMContext &Ctx = Fn->getContext();
1041 llvm::MDBuilder MDB(Ctx);
1042 Fn->setMetadata(
1043 KindID: llvm::LLVMContext::MD_func_sanitize,
1044 Node: MDB.createRTTIPointerPrologue(
1045 PrologueSig, RTTI: getUBSanFunctionTypeHash(Ty: FD->getType())));
1046 }
1047 }
1048
1049 // If we're checking nullability, we need to know whether we can check the
1050 // return value. Initialize the flag to 'true' and refine it in EmitParmDecl.
1051 if (SanOpts.has(K: SanitizerKind::NullabilityReturn)) {
1052 auto Nullability = FnRetTy->getNullability();
1053 if (Nullability && *Nullability == NullabilityKind::NonNull &&
1054 !FnRetTy->isRecordType()) {
1055 if (!(SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute) &&
1056 CurCodeDecl && CurCodeDecl->getAttr<ReturnsNonNullAttr>()))
1057 RetValNullabilityPrecondition =
1058 llvm::ConstantInt::getTrue(Context&: getLLVMContext());
1059 }
1060 }
1061
1062 // If we're in C++ mode and the function name is "main", it is guaranteed
1063 // to be norecurse by the standard (3.6.1.3 "The function main shall not be
1064 // used within a program").
1065 //
1066 // OpenCL C 2.0 v2.2-11 s6.9.i:
1067 // Recursion is not supported.
1068 //
1069 // HLSL
1070 // Recursion is not supported.
1071 //
1072 // SYCL v1.2.1 s3.10:
1073 // kernels cannot include RTTI information, exception classes,
1074 // recursive code, virtual functions or make use of C++ libraries that
1075 // are not compiled for the device.
1076 if (FD &&
1077 ((getLangOpts().CPlusPlus && FD->isMain()) || getLangOpts().OpenCL ||
1078 getLangOpts().HLSL || getLangOpts().SYCLIsDevice ||
1079 (getLangOpts().CUDA && FD->hasAttr<CUDAGlobalAttr>())))
1080 Fn->addFnAttr(Kind: llvm::Attribute::NoRecurse);
1081
1082 llvm::RoundingMode RM = getLangOpts().getDefaultRoundingMode();
1083 llvm::fp::ExceptionBehavior FPExceptionBehavior =
1084 ToConstrainedExceptMD(Kind: getLangOpts().getDefaultExceptionMode());
1085 Builder.setDefaultConstrainedRounding(RM);
1086 Builder.setDefaultConstrainedExcept(FPExceptionBehavior);
1087 if ((FD && (FD->UsesFPIntrin() || FD->hasAttr<StrictFPAttr>())) ||
1088 (!FD && (FPExceptionBehavior != llvm::fp::ebIgnore ||
1089 RM != llvm::RoundingMode::NearestTiesToEven))) {
1090 Builder.setIsFPConstrained(true);
1091 Fn->addFnAttr(Kind: llvm::Attribute::StrictFP);
1092 }
1093
1094 // If a custom alignment is used, force realigning to this alignment on
1095 // any main function which certainly will need it.
1096 if (FD && ((FD->isMain() || FD->isMSVCRTEntryPoint()) &&
1097 CGM.getCodeGenOpts().StackAlignment))
1098 Fn->addFnAttr(Kind: "stackrealign");
1099
1100 // "main" doesn't need to zero out call-used registers.
1101 if (FD && FD->isMain())
1102 Fn->removeFnAttr(Kind: "zero-call-used-regs");
1103
1104 // Add vscale_range attribute if appropriate.
1105 llvm::StringMap<bool> FeatureMap;
1106 auto IsArmStreaming = TargetInfo::ArmStreamingKind::NotStreaming;
1107 if (FD) {
1108 getContext().getFunctionFeatureMap(FeatureMap, FD);
1109 if (const auto *T = FD->getType()->getAs<FunctionProtoType>())
1110 if (T->getAArch64SMEAttributes() &
1111 FunctionType::SME_PStateSMCompatibleMask)
1112 IsArmStreaming = TargetInfo::ArmStreamingKind::StreamingCompatible;
1113
1114 if (IsArmStreamingFunction(FD, IncludeLocallyStreaming: true))
1115 IsArmStreaming = TargetInfo::ArmStreamingKind::Streaming;
1116 }
1117 std::optional<std::pair<unsigned, unsigned>> VScaleRange =
1118 getContext().getTargetInfo().getVScaleRange(LangOpts: getLangOpts(), Mode: IsArmStreaming,
1119 FeatureMap: &FeatureMap);
1120 if (VScaleRange) {
1121 CurFn->addFnAttr(Attr: llvm::Attribute::getWithVScaleRangeArgs(
1122 Context&: getLLVMContext(), MinValue: VScaleRange->first, MaxValue: VScaleRange->second));
1123 }
1124
1125 llvm::BasicBlock *EntryBB = createBasicBlock(name: "entry", parent: CurFn);
1126
1127 // Create a marker to make it easy to insert allocas into the entryblock
1128 // later. Don't create this with the builder, because we don't want it
1129 // folded.
1130 llvm::Value *Poison = llvm::PoisonValue::get(T: Int32Ty);
1131 AllocaInsertPt = new llvm::BitCastInst(Poison, Int32Ty, "allocapt", EntryBB);
1132
1133 ReturnBlock = getJumpDestInCurrentScope(Name: "return");
1134
1135 Builder.SetInsertPoint(EntryBB);
1136
1137 // If we're checking the return value, allocate space for a pointer to a
1138 // precise source location of the checked return statement.
1139 if (requiresReturnValueCheck()) {
1140 ReturnLocation = CreateDefaultAlignTempAlloca(Ty: Int8PtrTy, Name: "return.sloc.ptr");
1141 Builder.CreateStore(Val: llvm::ConstantPointerNull::get(T: Int8PtrTy),
1142 Addr: ReturnLocation);
1143 }
1144
1145 // Emit subprogram debug descriptor.
1146 if (CGDebugInfo *DI = getDebugInfo()) {
1147 // Reconstruct the type from the argument list so that implicit parameters,
1148 // such as 'this' and 'vtt', show up in the debug info. Preserve the calling
1149 // convention.
1150 DI->emitFunctionStart(GD, Loc, ScopeLoc: StartLoc,
1151 FnType: DI->getFunctionType(FD, RetTy, Args), Fn: CurFn,
1152 CurFnIsThunk: CurFuncIsThunk);
1153 }
1154
1155 if (ShouldInstrumentFunction()) {
1156 if (CGM.getCodeGenOpts().InstrumentFunctions)
1157 CurFn->addFnAttr(Kind: "instrument-function-entry", Val: "__cyg_profile_func_enter");
1158 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
1159 CurFn->addFnAttr(Kind: "instrument-function-entry-inlined",
1160 Val: "__cyg_profile_func_enter");
1161 if (CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
1162 CurFn->addFnAttr(Kind: "instrument-function-entry-inlined",
1163 Val: "__cyg_profile_func_enter_bare");
1164 }
1165
1166 // Since emitting the mcount call here impacts optimizations such as function
1167 // inlining, we just add an attribute to insert a mcount call in backend.
1168 // The attribute "counting-function" is set to mcount function name which is
1169 // architecture dependent.
1170 if (CGM.getCodeGenOpts().InstrumentForProfiling) {
1171 // Calls to fentry/mcount should not be generated if function has
1172 // the no_instrument_function attribute.
1173 if (!CurFuncDecl || !CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()) {
1174 if (CGM.getCodeGenOpts().CallFEntry)
1175 Fn->addFnAttr(Kind: "fentry-call", Val: "true");
1176 else {
1177 Fn->addFnAttr(Kind: "instrument-function-entry-inlined",
1178 Val: getTarget().getMCountName());
1179 }
1180 if (CGM.getCodeGenOpts().MNopMCount) {
1181 if (!CGM.getCodeGenOpts().CallFEntry)
1182 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_without_opt)
1183 << "-mnop-mcount" << "-mfentry";
1184 Fn->addFnAttr(Kind: "mnop-mcount");
1185 }
1186
1187 if (CGM.getCodeGenOpts().RecordMCount) {
1188 if (!CGM.getCodeGenOpts().CallFEntry)
1189 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_without_opt)
1190 << "-mrecord-mcount" << "-mfentry";
1191 Fn->addFnAttr(Kind: "mrecord-mcount");
1192 }
1193 }
1194 }
1195
1196 if (CGM.getCodeGenOpts().PackedStack) {
1197 if (getContext().getTargetInfo().getTriple().getArch() !=
1198 llvm::Triple::systemz)
1199 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_on_target)
1200 << "-mpacked-stack";
1201 Fn->addFnAttr(Kind: "packed-stack");
1202 }
1203
1204 if (CGM.getCodeGenOpts().WarnStackSize != UINT_MAX &&
1205 !CGM.getDiags().isIgnored(DiagID: diag::warn_fe_backend_frame_larger_than, Loc))
1206 Fn->addFnAttr(Kind: "warn-stack-size",
1207 Val: std::to_string(val: CGM.getCodeGenOpts().WarnStackSize));
1208
1209 if (RetTy->isVoidType()) {
1210 // Void type; nothing to return.
1211 ReturnValue = Address::invalid();
1212
1213 // Count the implicit return.
1214 if (!endsWithReturn(F: D))
1215 ++NumReturnExprs;
1216 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1217 // Indirect return; emit returned value directly into sret slot.
1218 // This reduces code size, and affects correctness in C++.
1219 auto AI = CurFn->arg_begin();
1220 if (CurFnInfo->getReturnInfo().isSRetAfterThis())
1221 ++AI;
1222 ReturnValue = makeNaturalAddressForPointer(
1223 Ptr: &*AI, T: RetTy, Alignment: CurFnInfo->getReturnInfo().getIndirectAlign(), ForPointeeType: false,
1224 BaseInfo: nullptr, TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
1225 if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
1226 ReturnValuePointer =
1227 CreateDefaultAlignTempAlloca(Ty: ReturnValue.getType(), Name: "result.ptr");
1228 Builder.CreateStore(Val: ReturnValue.emitRawPointer(CGF&: *this),
1229 Addr: ReturnValuePointer);
1230 }
1231 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
1232 !hasScalarEvaluationKind(T: CurFnInfo->getReturnType())) {
1233 // Load the sret pointer from the argument struct and return into that.
1234 unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();
1235 llvm::Function::arg_iterator EI = CurFn->arg_end();
1236 --EI;
1237 llvm::Value *Addr = Builder.CreateStructGEP(
1238 Ty: CurFnInfo->getArgStruct(), Ptr: &*EI, Idx);
1239 llvm::Type *Ty =
1240 cast<llvm::GetElementPtrInst>(Val: Addr)->getResultElementType();
1241 ReturnValuePointer = Address(Addr, Ty, getPointerAlign());
1242 Addr = Builder.CreateAlignedLoad(Ty, Addr, Align: getPointerAlign(), Name: "agg.result");
1243 ReturnValue = Address(Addr, ConvertType(T: RetTy),
1244 CGM.getNaturalTypeAlignment(T: RetTy), KnownNonNull);
1245 } else {
1246 ReturnValue = CreateIRTempWithoutCast(T: RetTy, Name: "retval");
1247
1248 // Tell the epilog emitter to autorelease the result. We do this
1249 // now so that various specialized functions can suppress it
1250 // during their IR-generation.
1251 if (getLangOpts().ObjCAutoRefCount &&
1252 !CurFnInfo->isReturnsRetained() &&
1253 RetTy->isObjCRetainableType())
1254 AutoreleaseResult = true;
1255 }
1256
1257 EmitStartEHSpec(D: CurCodeDecl);
1258
1259 PrologueCleanupDepth = EHStack.stable_begin();
1260
1261 // Emit OpenMP specific initialization of the device functions.
1262 if (getLangOpts().OpenMP && CurCodeDecl)
1263 CGM.getOpenMPRuntime().emitFunctionProlog(CGF&: *this, D: CurCodeDecl);
1264
1265 if (FD && getLangOpts().HLSL) {
1266 // Handle emitting HLSL entry functions.
1267 if (FD->hasAttr<HLSLShaderAttr>()) {
1268 CGM.getHLSLRuntime().emitEntryFunction(FD, Fn);
1269 }
1270 }
1271
1272 EmitFunctionProlog(FI: *CurFnInfo, Fn: CurFn, Args);
1273
1274 if (const CXXMethodDecl *MD = dyn_cast_if_present<CXXMethodDecl>(Val: D);
1275 MD && !MD->isStatic()) {
1276 bool IsInLambda =
1277 MD->getParent()->isLambda() && MD->getOverloadedOperator() == OO_Call;
1278 if (MD->isImplicitObjectMemberFunction())
1279 CGM.getCXXABI().EmitInstanceFunctionProlog(CGF&: *this);
1280 if (IsInLambda) {
1281 // We're in a lambda; figure out the captures.
1282 MD->getParent()->getCaptureFields(Captures&: LambdaCaptureFields,
1283 ThisCapture&: LambdaThisCaptureField);
1284 if (LambdaThisCaptureField) {
1285 // If the lambda captures the object referred to by '*this' - either by
1286 // value or by reference, make sure CXXThisValue points to the correct
1287 // object.
1288
1289 // Get the lvalue for the field (which is a copy of the enclosing object
1290 // or contains the address of the enclosing object).
1291 LValue ThisFieldLValue = EmitLValueForLambdaField(Field: LambdaThisCaptureField);
1292 if (!LambdaThisCaptureField->getType()->isPointerType()) {
1293 // If the enclosing object was captured by value, just use its
1294 // address. Sign this pointer.
1295 CXXThisValue = ThisFieldLValue.getPointer(CGF&: *this);
1296 } else {
1297 // Load the lvalue pointed to by the field, since '*this' was captured
1298 // by reference.
1299 CXXThisValue =
1300 EmitLoadOfLValue(V: ThisFieldLValue, Loc: SourceLocation()).getScalarVal();
1301 }
1302 }
1303 for (auto *FD : MD->getParent()->fields()) {
1304 if (FD->hasCapturedVLAType()) {
1305 auto *ExprArg = EmitLoadOfLValue(V: EmitLValueForLambdaField(Field: FD),
1306 Loc: SourceLocation()).getScalarVal();
1307 auto VAT = FD->getCapturedVLAType();
1308 VLASizeMap[VAT->getSizeExpr()] = ExprArg;
1309 }
1310 }
1311 } else if (MD->isImplicitObjectMemberFunction()) {
1312 // Not in a lambda; just use 'this' from the method.
1313 // FIXME: Should we generate a new load for each use of 'this'? The
1314 // fast register allocator would be happier...
1315 CXXThisValue = CXXABIThisValue;
1316 }
1317
1318 // Check the 'this' pointer once per function, if it's available.
1319 if (CXXABIThisValue) {
1320 SanitizerSet SkippedChecks;
1321 SkippedChecks.set(K: SanitizerKind::ObjectSize, Value: true);
1322 QualType ThisTy = MD->getThisType();
1323
1324 // If this is the call operator of a lambda with no captures, it
1325 // may have a static invoker function, which may call this operator with
1326 // a null 'this' pointer.
1327 if (isLambdaCallOperator(MD) && MD->getParent()->isCapturelessLambda())
1328 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
1329
1330 EmitTypeCheck(
1331 TCK: isa<CXXConstructorDecl>(Val: MD) ? TCK_ConstructorCall : TCK_MemberCall,
1332 Loc, V: CXXABIThisValue, Type: ThisTy, Alignment: CXXABIThisAlignment, SkippedChecks);
1333 }
1334 }
1335
1336 // If any of the arguments have a variably modified type, make sure to
1337 // emit the type size, but only if the function is not naked. Naked functions
1338 // have no prolog to run this evaluation.
1339 if (!FD || !FD->hasAttr<NakedAttr>()) {
1340 for (const VarDecl *VD : Args) {
1341 // Dig out the type as written from ParmVarDecls; it's unclear whether
1342 // the standard (C99 6.9.1p10) requires this, but we're following the
1343 // precedent set by gcc.
1344 QualType Ty;
1345 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: VD))
1346 Ty = PVD->getOriginalType();
1347 else
1348 Ty = VD->getType();
1349
1350 if (Ty->isVariablyModifiedType())
1351 EmitVariablyModifiedType(Ty);
1352 }
1353 }
1354 // Emit a location at the end of the prologue.
1355 if (CGDebugInfo *DI = getDebugInfo())
1356 DI->EmitLocation(Builder, Loc: StartLoc);
1357 // TODO: Do we need to handle this in two places like we do with
1358 // target-features/target-cpu?
1359 if (CurFuncDecl)
1360 if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>())
1361 LargestVectorWidth = VecWidth->getVectorWidth();
1362
1363 if (CGM.shouldEmitConvergenceTokens())
1364 ConvergenceTokenStack.push_back(Elt: getOrEmitConvergenceEntryToken(F: CurFn));
1365}
1366
1367void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {
1368 incrementProfileCounter(S: Body);
1369 maybeCreateMCDCCondBitmap();
1370 if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Val: Body))
1371 EmitCompoundStmtWithoutScope(S: *S);
1372 else
1373 EmitStmt(S: Body);
1374}
1375
1376/// When instrumenting to collect profile data, the counts for some blocks
1377/// such as switch cases need to not include the fall-through counts, so
1378/// emit a branch around the instrumentation code. When not instrumenting,
1379/// this just calls EmitBlock().
1380void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
1381 const Stmt *S) {
1382 llvm::BasicBlock *SkipCountBB = nullptr;
1383 if (HaveInsertPoint() && CGM.getCodeGenOpts().hasProfileClangInstr()) {
1384 // When instrumenting for profiling, the fallthrough to certain
1385 // statements needs to skip over the instrumentation code so that we
1386 // get an accurate count.
1387 SkipCountBB = createBasicBlock(name: "skipcount");
1388 EmitBranch(Block: SkipCountBB);
1389 }
1390 EmitBlock(BB);
1391 uint64_t CurrentCount = getCurrentProfileCount();
1392 incrementProfileCounter(ExecSkip: UseExecPath, S);
1393 setCurrentProfileCount(getCurrentProfileCount() + CurrentCount);
1394 if (SkipCountBB)
1395 EmitBlock(BB: SkipCountBB);
1396}
1397
1398/// Tries to mark the given function nounwind based on the
1399/// non-existence of any throwing calls within it. We believe this is
1400/// lightweight enough to do at -O0.
1401static void TryMarkNoThrow(llvm::Function *F) {
1402 // LLVM treats 'nounwind' on a function as part of the type, so we
1403 // can't do this on functions that can be overwritten.
1404 if (F->isInterposable()) return;
1405
1406 for (llvm::BasicBlock &BB : *F)
1407 for (llvm::Instruction &I : BB)
1408 if (I.mayThrow())
1409 return;
1410
1411 F->setDoesNotThrow();
1412}
1413
1414QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD,
1415 FunctionArgList &Args) {
1416 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
1417 QualType ResTy = FD->getReturnType();
1418
1419 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD);
1420 if (MD && MD->isImplicitObjectMemberFunction()) {
1421 if (CGM.getCXXABI().HasThisReturn(GD))
1422 ResTy = MD->getThisType();
1423 else if (CGM.getCXXABI().hasMostDerivedReturn(GD))
1424 ResTy = CGM.getContext().VoidPtrTy;
1425 CGM.getCXXABI().buildThisParam(CGF&: *this, Params&: Args);
1426 }
1427
1428 // The base version of an inheriting constructor whose constructed base is a
1429 // virtual base is not passed any arguments (because it doesn't actually call
1430 // the inherited constructor).
1431 bool PassedParams = true;
1432 if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(Val: FD))
1433 if (auto Inherited = CD->getInheritedConstructor())
1434 PassedParams =
1435 getTypes().inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
1436
1437 if (PassedParams) {
1438 for (auto *Param : FD->parameters()) {
1439 Args.push_back(Elt: Param);
1440 if (!Param->hasAttr<PassObjectSizeAttr>())
1441 continue;
1442
1443 auto *Implicit = ImplicitParamDecl::Create(
1444 C&: getContext(), DC: Param->getDeclContext(), IdLoc: Param->getLocation(),
1445 /*Id=*/nullptr, T: getContext().getSizeType(), ParamKind: ImplicitParamKind::Other);
1446 SizeArguments[Param] = Implicit;
1447 Args.push_back(Elt: Implicit);
1448 }
1449 }
1450
1451 if (MD && (isa<CXXConstructorDecl>(Val: MD) || isa<CXXDestructorDecl>(Val: MD)))
1452 CGM.getCXXABI().addImplicitStructorParams(CGF&: *this, ResTy, Params&: Args);
1453
1454 return ResTy;
1455}
1456
1457void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
1458 const CGFunctionInfo &FnInfo) {
1459 assert(Fn && "generating code for null Function");
1460 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
1461 CurGD = GD;
1462
1463 FunctionArgList Args;
1464 QualType ResTy = BuildFunctionArgList(GD, Args);
1465
1466 CGM.getTargetCodeGenInfo().checkFunctionABI(CGM, Decl: FD);
1467
1468 if (FD->isInlineBuiltinDeclaration()) {
1469 // When generating code for a builtin with an inline declaration, use a
1470 // mangled name to hold the actual body, while keeping an external
1471 // definition in case the function pointer is referenced somewhere.
1472 std::string FDInlineName = (Fn->getName() + ".inline").str();
1473 llvm::Module *M = Fn->getParent();
1474 llvm::Function *Clone = M->getFunction(Name: FDInlineName);
1475 if (!Clone) {
1476 Clone = llvm::Function::Create(Ty: Fn->getFunctionType(),
1477 Linkage: llvm::GlobalValue::InternalLinkage,
1478 AddrSpace: Fn->getAddressSpace(), N: FDInlineName, M);
1479 Clone->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1480 }
1481 Fn->setLinkage(llvm::GlobalValue::ExternalLinkage);
1482 Fn = Clone;
1483 } else {
1484 // Detect the unusual situation where an inline version is shadowed by a
1485 // non-inline version. In that case we should pick the external one
1486 // everywhere. That's GCC behavior too. Unfortunately, I cannot find a way
1487 // to detect that situation before we reach codegen, so do some late
1488 // replacement.
1489 for (const FunctionDecl *PD = FD->getPreviousDecl(); PD;
1490 PD = PD->getPreviousDecl()) {
1491 if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) {
1492 std::string FDInlineName = (Fn->getName() + ".inline").str();
1493 llvm::Module *M = Fn->getParent();
1494 if (llvm::Function *Clone = M->getFunction(Name: FDInlineName)) {
1495 Clone->replaceAllUsesWith(V: Fn);
1496 Clone->eraseFromParent();
1497 }
1498 break;
1499 }
1500 }
1501 }
1502
1503 // Check if we should generate debug info for this function.
1504 if (FD->hasAttr<NoDebugAttr>()) {
1505 // Clear non-distinct debug info that was possibly attached to the function
1506 // due to an earlier declaration without the nodebug attribute
1507 Fn->setSubprogram(nullptr);
1508 // Disable debug info indefinitely for this function
1509 DebugInfo = nullptr;
1510 }
1511 // Finalize function debug info on exit.
1512 llvm::scope_exit Cleanup([this] {
1513 if (CGDebugInfo *DI = getDebugInfo())
1514 DI->completeFunction();
1515 });
1516
1517 // The function might not have a body if we're generating thunks for a
1518 // function declaration.
1519 SourceRange BodyRange;
1520 if (Stmt *Body = FD->getBody())
1521 BodyRange = Body->getSourceRange();
1522 else
1523 BodyRange = FD->getLocation();
1524 CurEHLocation = BodyRange.getEnd();
1525
1526 // Use the location of the start of the function to determine where
1527 // the function definition is located. By default use the location
1528 // of the declaration as the location for the subprogram. A function
1529 // may lack a declaration in the source code if it is created by code
1530 // gen. (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).
1531 SourceLocation Loc = FD->getLocation();
1532
1533 // If this is a function specialization then use the pattern body
1534 // as the location for the function.
1535 if (const FunctionDecl *SpecDecl = FD->getTemplateInstantiationPattern())
1536 if (SpecDecl->hasBody(Definition&: SpecDecl))
1537 Loc = SpecDecl->getLocation();
1538
1539 Stmt *Body = FD->getBody();
1540
1541 if (Body) {
1542 // Coroutines always emit lifetime markers.
1543 if (isa<CoroutineBodyStmt>(Val: Body))
1544 ShouldEmitLifetimeMarkers = true;
1545
1546 // Initialize helper which will detect jumps which can cause invalid
1547 // lifetime markers.
1548 if (ShouldEmitLifetimeMarkers)
1549 Bypasses.Init(CGM, Body);
1550 }
1551
1552 // Emit the standard function prologue.
1553 StartFunction(GD, RetTy: ResTy, Fn, FnInfo, Args, Loc, StartLoc: BodyRange.getBegin());
1554
1555 // Save parameters for coroutine function.
1556 if (Body && isa_and_nonnull<CoroutineBodyStmt>(Val: Body))
1557 llvm::append_range(C&: FnArgs, R: FD->parameters());
1558
1559 // Ensure that the function adheres to the forward progress guarantee, which
1560 // is required by certain optimizations.
1561 // In C++11 and up, the attribute will be removed if the body contains a
1562 // trivial empty loop.
1563 if (checkIfFunctionMustProgress())
1564 CurFn->addFnAttr(Kind: llvm::Attribute::MustProgress);
1565
1566 // Generate the body of the function.
1567 PGO->assignRegionCounters(GD, Fn: CurFn);
1568 if (isa<CXXDestructorDecl>(Val: FD))
1569 EmitDestructorBody(Args);
1570 else if (isa<CXXConstructorDecl>(Val: FD))
1571 EmitConstructorBody(Args);
1572 else if (getLangOpts().CUDA &&
1573 !getLangOpts().CUDAIsDevice &&
1574 FD->hasAttr<CUDAGlobalAttr>())
1575 CGM.getCUDARuntime().emitDeviceStub(CGF&: *this, Args);
1576 else if (isa<CXXMethodDecl>(Val: FD) &&
1577 cast<CXXMethodDecl>(Val: FD)->isLambdaStaticInvoker()) {
1578 // The lambda static invoker function is special, because it forwards or
1579 // clones the body of the function call operator (but is actually static).
1580 EmitLambdaStaticInvokeBody(MD: cast<CXXMethodDecl>(Val: FD));
1581 } else if (isa<CXXMethodDecl>(Val: FD) &&
1582 isLambdaCallOperator(MD: cast<CXXMethodDecl>(Val: FD)) &&
1583 !FnInfo.isDelegateCall() &&
1584 cast<CXXMethodDecl>(Val: FD)->getParent()->getLambdaStaticInvoker() &&
1585 hasInAllocaArg(MD: cast<CXXMethodDecl>(Val: FD))) {
1586 // If emitting a lambda with static invoker on X86 Windows, change
1587 // the call operator body.
1588 // Make sure that this is a call operator with an inalloca arg and check
1589 // for delegate call to make sure this is the original call op and not the
1590 // new forwarding function for the static invoker.
1591 EmitLambdaInAllocaCallOpBody(MD: cast<CXXMethodDecl>(Val: FD));
1592 } else if (FD->isDefaulted() && isa<CXXMethodDecl>(Val: FD) &&
1593 (cast<CXXMethodDecl>(Val: FD)->isCopyAssignmentOperator() ||
1594 cast<CXXMethodDecl>(Val: FD)->isMoveAssignmentOperator())) {
1595 // Implicit copy-assignment gets the same special treatment as implicit
1596 // copy-constructors.
1597 emitImplicitAssignmentOperatorBody(Args);
1598 } else if (DeviceKernelAttr::isOpenCLSpelling(
1599 A: FD->getAttr<DeviceKernelAttr>()) &&
1600 GD.getKernelReferenceKind() == KernelReferenceKind::Kernel) {
1601 CallArgList CallArgs;
1602 for (unsigned i = 0; i < Args.size(); ++i) {
1603 Address ArgAddr = GetAddrOfLocalVar(VD: Args[i]);
1604 QualType ArgQualType = Args[i]->getType();
1605 RValue ArgRValue = convertTempToRValue(addr: ArgAddr, type: ArgQualType, Loc);
1606 CallArgs.add(rvalue: ArgRValue, type: ArgQualType);
1607 }
1608 GlobalDecl GDStub = GlobalDecl(FD, KernelReferenceKind::Stub);
1609 const FunctionType *FT = cast<FunctionType>(Val: FD->getType());
1610 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
1611 const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
1612 Args: CallArgs, Ty: FT, /*ChainCall=*/false);
1613 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
1614 llvm::Constant *GDStubFunctionPointer =
1615 CGM.getRawFunctionPointer(GD: GDStub, Ty: FTy);
1616 CGCallee GDStubCallee = CGCallee::forDirect(functionPtr: GDStubFunctionPointer, abstractInfo: GDStub);
1617 EmitCall(CallInfo: FnInfo, Callee: GDStubCallee, ReturnValue: ReturnValueSlot(), Args: CallArgs, CallOrInvoke: nullptr, IsMustTail: false,
1618 Loc);
1619 } else if (Body) {
1620 EmitFunctionBody(Body);
1621 } else
1622 llvm_unreachable("no definition for emitted function");
1623
1624 // C++11 [stmt.return]p2:
1625 // Flowing off the end of a function [...] results in undefined behavior in
1626 // a value-returning function.
1627 // C11 6.9.1p12:
1628 // If the '}' that terminates a function is reached, and the value of the
1629 // function call is used by the caller, the behavior is undefined.
1630 if (getLangOpts().CPlusPlus && !FD->hasImplicitReturnZero() && !SawAsmBlock &&
1631 !FD->getReturnType()->isVoidType() && Builder.GetInsertBlock()) {
1632 bool ShouldEmitUnreachable =
1633 CGM.getCodeGenOpts().StrictReturn ||
1634 !CGM.MayDropFunctionReturn(Context: FD->getASTContext(), ReturnType: FD->getReturnType());
1635 if (SanOpts.has(K: SanitizerKind::Return)) {
1636 auto CheckOrdinal = SanitizerKind::SO_Return;
1637 auto CheckHandler = SanitizerHandler::MissingReturn;
1638 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
1639 llvm::Value *IsFalse = Builder.getFalse();
1640 EmitCheck(Checked: std::make_pair(x&: IsFalse, y&: CheckOrdinal), Check: CheckHandler,
1641 StaticArgs: EmitCheckSourceLocation(Loc: FD->getLocation()), DynamicArgs: {});
1642 } else if (ShouldEmitUnreachable) {
1643 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1644 EmitTrapCall(IntrID: llvm::Intrinsic::trap);
1645 }
1646 if (SanOpts.has(K: SanitizerKind::Return) || ShouldEmitUnreachable) {
1647 Builder.CreateUnreachable();
1648 Builder.ClearInsertionPoint();
1649 }
1650 }
1651
1652 // Emit the standard function epilogue.
1653 FinishFunction(EndLoc: BodyRange.getEnd());
1654
1655 PGO->verifyCounterMap();
1656
1657 if (CurCodeDecl->hasAttr<PersonalityAttr>()) {
1658 StringRef Identifier =
1659 CurCodeDecl->getAttr<PersonalityAttr>()->getRoutine()->getName();
1660 llvm::FunctionCallee PersonalityRoutine =
1661 CGM.CreateRuntimeFunction(Ty: llvm::FunctionType::get(Result: CGM.Int32Ty, isVarArg: true),
1662 Name: Identifier, ExtraAttrs: {}, /*local=*/Local: true);
1663 Fn->setPersonalityFn(cast<llvm::Constant>(Val: PersonalityRoutine.getCallee()));
1664 }
1665
1666 // If we haven't marked the function nothrow through other means, do
1667 // a quick pass now to see if we can.
1668 if (!CurFn->doesNotThrow())
1669 TryMarkNoThrow(F: CurFn);
1670}
1671
1672/// ContainsLabel - Return true if the statement contains a label in it. If
1673/// this statement is not executed normally, it not containing a label means
1674/// that we can just remove the code.
1675bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
1676 // Null statement, not a label!
1677 if (!S) return false;
1678
1679 // If this is a label, we have to emit the code, consider something like:
1680 // if (0) { ... foo: bar(); } goto foo;
1681 //
1682 // TODO: If anyone cared, we could track __label__'s, since we know that you
1683 // can't jump to one from outside their declared region.
1684 if (isa<LabelStmt>(Val: S))
1685 return true;
1686
1687 // If this is a case/default statement, and we haven't seen a switch, we have
1688 // to emit the code.
1689 if (isa<SwitchCase>(Val: S) && !IgnoreCaseStmts)
1690 return true;
1691
1692 // If this is a switch statement, we want to ignore cases below it.
1693 if (isa<SwitchStmt>(Val: S))
1694 IgnoreCaseStmts = true;
1695
1696 // Scan subexpressions for verboten labels.
1697 for (const Stmt *SubStmt : S->children())
1698 if (ContainsLabel(S: SubStmt, IgnoreCaseStmts))
1699 return true;
1700
1701 return false;
1702}
1703
1704/// containsBreak - Return true if the statement contains a break out of it.
1705/// If the statement (recursively) contains a switch or loop with a break
1706/// inside of it, this is fine.
1707bool CodeGenFunction::containsBreak(const Stmt *S) {
1708 // Null statement, not a label!
1709 if (!S) return false;
1710
1711 // If this is a switch or loop that defines its own break scope, then we can
1712 // include it and anything inside of it.
1713 if (isa<SwitchStmt>(Val: S) || isa<WhileStmt>(Val: S) || isa<DoStmt>(Val: S) ||
1714 isa<ForStmt>(Val: S))
1715 return false;
1716
1717 if (isa<BreakStmt>(Val: S))
1718 return true;
1719
1720 // Scan subexpressions for verboten breaks.
1721 for (const Stmt *SubStmt : S->children())
1722 if (containsBreak(S: SubStmt))
1723 return true;
1724
1725 return false;
1726}
1727
1728bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) {
1729 if (!S) return false;
1730
1731 // Some statement kinds add a scope and thus never add a decl to the current
1732 // scope. Note, this list is longer than the list of statements that might
1733 // have an unscoped decl nested within them, but this way is conservatively
1734 // correct even if more statement kinds are added.
1735 if (isa<IfStmt>(Val: S) || isa<SwitchStmt>(Val: S) || isa<WhileStmt>(Val: S) ||
1736 isa<DoStmt>(Val: S) || isa<ForStmt>(Val: S) || isa<CompoundStmt>(Val: S) ||
1737 isa<CXXForRangeStmt>(Val: S) || isa<CXXTryStmt>(Val: S) ||
1738 isa<ObjCForCollectionStmt>(Val: S) || isa<ObjCAtTryStmt>(Val: S))
1739 return false;
1740
1741 if (isa<DeclStmt>(Val: S))
1742 return true;
1743
1744 for (const Stmt *SubStmt : S->children())
1745 if (mightAddDeclToScope(S: SubStmt))
1746 return true;
1747
1748 return false;
1749}
1750
1751/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1752/// to a constant, or if it does but contains a label, return false. If it
1753/// constant folds return true and set the boolean result in Result.
1754bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1755 bool &ResultBool,
1756 bool AllowLabels) {
1757 // If MC/DC is enabled, disable folding so that we can instrument all
1758 // conditions to yield complete test vectors. We still keep track of
1759 // folded conditions during region mapping and visualization.
1760 if (!AllowLabels && CGM.getCodeGenOpts().hasProfileClangInstr() &&
1761 CGM.getCodeGenOpts().MCDCCoverage)
1762 return false;
1763
1764 llvm::APSInt ResultInt;
1765 if (!ConstantFoldsToSimpleInteger(Cond, Result&: ResultInt, AllowLabels))
1766 return false;
1767
1768 ResultBool = ResultInt.getBoolValue();
1769 return true;
1770}
1771
1772/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1773/// to a constant, or if it does but contains a label, return false. If it
1774/// constant folds return true and set the folded value.
1775bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1776 llvm::APSInt &ResultInt,
1777 bool AllowLabels) {
1778 // FIXME: Rename and handle conversion of other evaluatable things
1779 // to bool.
1780 Expr::EvalResult Result;
1781 if (!Cond->EvaluateAsInt(Result, Ctx: getContext()))
1782 return false; // Not foldable, not integer or not fully evaluatable.
1783
1784 llvm::APSInt Int = Result.Val.getInt();
1785 if (!AllowLabels && CodeGenFunction::ContainsLabel(S: Cond))
1786 return false; // Contains a label.
1787
1788 PGO->markStmtMaybeUsed(S: Cond);
1789 ResultInt = std::move(Int);
1790 return true;
1791}
1792
1793/// Strip parentheses and simplistic logical-NOT operators.
1794const Expr *CodeGenFunction::stripCond(const Expr *C) {
1795 while (true) {
1796 const Expr *SC = IgnoreExprNodes(
1797 E: C, Fns&: IgnoreParensSingleStep, Fns&: IgnoreUOpLNotSingleStep,
1798 Fns&: IgnoreBuiltinExpectSingleStep, Fns&: IgnoreImplicitCastsSingleStep);
1799 if (C == SC)
1800 return SC;
1801 C = SC;
1802 }
1803}
1804
1805/// Determine whether the given condition is an instrumentable condition
1806/// (i.e. no "&&" or "||").
1807bool CodeGenFunction::isInstrumentedCondition(const Expr *C) {
1808 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: stripCond(C));
1809 return (!BOp || !BOp->isLogicalOp());
1810}
1811
1812/// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
1813/// increments a profile counter based on the semantics of the given logical
1814/// operator opcode. This is used to instrument branch condition coverage for
1815/// logical operators.
1816void CodeGenFunction::EmitBranchToCounterBlock(
1817 const Expr *Cond, BinaryOperator::Opcode LOp, llvm::BasicBlock *TrueBlock,
1818 llvm::BasicBlock *FalseBlock, uint64_t TrueCount /* = 0 */,
1819 Stmt::Likelihood LH /* =None */, const Expr *CntrIdx /* = nullptr */) {
1820 // If not instrumenting, just emit a branch.
1821 bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr();
1822 if (!InstrumentRegions || !isInstrumentedCondition(C: Cond))
1823 return EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount, LH);
1824
1825 const Stmt *CntrStmt = (CntrIdx ? CntrIdx : Cond);
1826
1827 llvm::BasicBlock *ThenBlock = nullptr;
1828 llvm::BasicBlock *ElseBlock = nullptr;
1829 llvm::BasicBlock *NextBlock = nullptr;
1830
1831 // Create the block we'll use to increment the appropriate counter.
1832 llvm::BasicBlock *CounterIncrBlock = createBasicBlock(name: "lop.rhscnt");
1833
1834 llvm::BasicBlock *SkipIncrBlock =
1835 (hasSkipCounter(S: CntrStmt) ? createBasicBlock(name: "lop.rhsskip") : nullptr);
1836 llvm::BasicBlock *SkipNextBlock = nullptr;
1837
1838 // Set block pointers according to Logical-AND (BO_LAnd) semantics. This
1839 // means we need to evaluate the condition and increment the counter on TRUE:
1840 //
1841 // if (Cond)
1842 // goto CounterIncrBlock;
1843 // else
1844 // goto FalseBlock;
1845 //
1846 // CounterIncrBlock:
1847 // Counter++;
1848 // goto TrueBlock;
1849
1850 if (LOp == BO_LAnd) {
1851 SkipNextBlock = FalseBlock;
1852 ThenBlock = CounterIncrBlock;
1853 ElseBlock = (SkipIncrBlock ? SkipIncrBlock : SkipNextBlock);
1854 NextBlock = TrueBlock;
1855 }
1856
1857 // Set block pointers according to Logical-OR (BO_LOr) semantics. This means
1858 // we need to evaluate the condition and increment the counter on FALSE:
1859 //
1860 // if (Cond)
1861 // goto TrueBlock;
1862 // else
1863 // goto CounterIncrBlock;
1864 //
1865 // CounterIncrBlock:
1866 // Counter++;
1867 // goto FalseBlock;
1868
1869 else if (LOp == BO_LOr) {
1870 SkipNextBlock = TrueBlock;
1871 ThenBlock = (SkipIncrBlock ? SkipIncrBlock : SkipNextBlock);
1872 ElseBlock = CounterIncrBlock;
1873 NextBlock = FalseBlock;
1874 } else {
1875 llvm_unreachable("Expected Opcode must be that of a Logical Operator");
1876 }
1877
1878 // Emit Branch based on condition.
1879 EmitBranchOnBoolExpr(Cond, TrueBlock: ThenBlock, FalseBlock: ElseBlock, TrueCount, LH);
1880
1881 if (SkipIncrBlock) {
1882 EmitBlock(BB: SkipIncrBlock);
1883 incrementProfileCounter(ExecSkip: UseSkipPath, S: CntrStmt);
1884 EmitBranch(Block: SkipNextBlock);
1885 }
1886
1887 // Emit the block containing the counter increment(s).
1888 EmitBlock(BB: CounterIncrBlock);
1889
1890 // Increment corresponding counter; if index not provided, use Cond as index.
1891 incrementProfileCounter(ExecSkip: UseExecPath, S: CntrStmt);
1892
1893 // Go to the next block.
1894 EmitBranch(Block: NextBlock);
1895}
1896
1897/// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an if
1898/// statement) to the specified blocks. Based on the condition, this might try
1899/// to simplify the codegen of the conditional based on the branch.
1900/// \param LH The value of the likelihood attribute on the True branch.
1901/// \param ConditionalOp Used by MC/DC code coverage to track the result of the
1902/// ConditionalOperator (ternary) through a recursive call for the operator's
1903/// LHS and RHS nodes.
1904void CodeGenFunction::EmitBranchOnBoolExpr(
1905 const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock,
1906 uint64_t TrueCount, Stmt::Likelihood LH, const Expr *ConditionalOp,
1907 const VarDecl *ConditionalDecl) {
1908 Cond = Cond->IgnoreParens();
1909
1910 if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Val: Cond)) {
1911 bool HasSkip = hasSkipCounter(S: CondBOp);
1912
1913 // Handle X && Y in a condition.
1914 if (CondBOp->getOpcode() == BO_LAnd) {
1915 // If we have "1 && X", simplify the code. "0 && X" would have constant
1916 // folded if the case was simple enough.
1917 bool ConstantBool = false;
1918 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getLHS(), ResultBool&: ConstantBool) &&
1919 ConstantBool) {
1920 // br(1 && X) -> br(X).
1921 incrementProfileCounter(S: CondBOp);
1922 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LAnd, TrueBlock,
1923 FalseBlock, TrueCount, LH);
1924 return;
1925 }
1926
1927 // If we have "X && 1", simplify the code to use an uncond branch.
1928 // "X && 0" would have been constant folded to 0.
1929 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getRHS(), ResultBool&: ConstantBool) &&
1930 ConstantBool) {
1931 // br(X && 1) -> br(X).
1932 EmitBranchToCounterBlock(Cond: CondBOp->getLHS(), LOp: BO_LAnd, TrueBlock,
1933 FalseBlock, TrueCount, LH, CntrIdx: CondBOp);
1934 return;
1935 }
1936
1937 // Emit the LHS as a conditional. If the LHS conditional is false, we
1938 // want to jump to the FalseBlock.
1939 llvm::BasicBlock *LHSTrue = createBasicBlock(name: "land.lhs.true");
1940 llvm::BasicBlock *LHSFalse =
1941 (HasSkip ? createBasicBlock(name: "land.lhsskip") : FalseBlock);
1942 // The counter tells us how often we evaluate RHS, and all of TrueCount
1943 // can be propagated to that branch.
1944 uint64_t RHSCount = getProfileCount(S: CondBOp->getRHS());
1945
1946 ConditionalEvaluation eval(*this);
1947 {
1948 ApplyDebugLocation DL(*this, Cond);
1949 // Propagate the likelihood attribute like __builtin_expect
1950 // __builtin_expect(X && Y, 1) -> X and Y are likely
1951 // __builtin_expect(X && Y, 0) -> only Y is unlikely
1952 EmitBranchOnBoolExpr(Cond: CondBOp->getLHS(), TrueBlock: LHSTrue, FalseBlock: LHSFalse, TrueCount: RHSCount,
1953 LH: LH == Stmt::LH_Unlikely ? Stmt::LH_None : LH);
1954 if (HasSkip) {
1955 EmitBlock(BB: LHSFalse);
1956 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondBOp);
1957 EmitBranch(Block: FalseBlock);
1958 }
1959 EmitBlock(BB: LHSTrue);
1960 }
1961
1962 incrementProfileCounter(ExecSkip: UseExecPath, S: CondBOp);
1963 setCurrentProfileCount(getProfileCount(S: CondBOp->getRHS()));
1964
1965 // Any temporaries created here are conditional.
1966 eval.begin(CGF&: *this);
1967 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LAnd, TrueBlock,
1968 FalseBlock, TrueCount, LH);
1969 eval.end(CGF&: *this);
1970 return;
1971 }
1972
1973 if (CondBOp->getOpcode() == BO_LOr) {
1974 // If we have "0 || X", simplify the code. "1 || X" would have constant
1975 // folded if the case was simple enough.
1976 bool ConstantBool = false;
1977 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getLHS(), ResultBool&: ConstantBool) &&
1978 !ConstantBool) {
1979 // br(0 || X) -> br(X).
1980 incrementProfileCounter(S: CondBOp);
1981 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LOr, TrueBlock,
1982 FalseBlock, TrueCount, LH);
1983 return;
1984 }
1985
1986 // If we have "X || 0", simplify the code to use an uncond branch.
1987 // "X || 1" would have been constant folded to 1.
1988 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getRHS(), ResultBool&: ConstantBool) &&
1989 !ConstantBool) {
1990 // br(X || 0) -> br(X).
1991 EmitBranchToCounterBlock(Cond: CondBOp->getLHS(), LOp: BO_LOr, TrueBlock,
1992 FalseBlock, TrueCount, LH, CntrIdx: CondBOp);
1993 return;
1994 }
1995 // Emit the LHS as a conditional. If the LHS conditional is true, we
1996 // want to jump to the TrueBlock.
1997 llvm::BasicBlock *LHSTrue =
1998 (HasSkip ? createBasicBlock(name: "lor.lhsskip") : TrueBlock);
1999 llvm::BasicBlock *LHSFalse = createBasicBlock(name: "lor.lhs.false");
2000 // We have the count for entry to the RHS and for the whole expression
2001 // being true, so we can divy up True count between the short circuit and
2002 // the RHS.
2003 uint64_t LHSCount =
2004 getCurrentProfileCount() - getProfileCount(S: CondBOp->getRHS());
2005 uint64_t RHSCount = TrueCount - LHSCount;
2006
2007 ConditionalEvaluation eval(*this);
2008 {
2009 // Propagate the likelihood attribute like __builtin_expect
2010 // __builtin_expect(X || Y, 1) -> only Y is likely
2011 // __builtin_expect(X || Y, 0) -> both X and Y are unlikely
2012 ApplyDebugLocation DL(*this, Cond);
2013 EmitBranchOnBoolExpr(Cond: CondBOp->getLHS(), TrueBlock: LHSTrue, FalseBlock: LHSFalse, TrueCount: LHSCount,
2014 LH: LH == Stmt::LH_Likely ? Stmt::LH_None : LH);
2015 if (HasSkip) {
2016 EmitBlock(BB: LHSTrue);
2017 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondBOp);
2018 EmitBranch(Block: TrueBlock);
2019 }
2020 EmitBlock(BB: LHSFalse);
2021 }
2022
2023 incrementProfileCounter(ExecSkip: UseExecPath, S: CondBOp);
2024 setCurrentProfileCount(getProfileCount(S: CondBOp->getRHS()));
2025
2026 // Any temporaries created here are conditional.
2027 eval.begin(CGF&: *this);
2028 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LOr, TrueBlock, FalseBlock,
2029 TrueCount: RHSCount, LH);
2030
2031 eval.end(CGF&: *this);
2032 return;
2033 }
2034 }
2035
2036 if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Val: Cond)) {
2037 // br(!x, t, f) -> br(x, f, t)
2038 // Avoid doing this optimization when instrumenting a condition for MC/DC.
2039 // LNot is taken as part of the condition for simplicity, and changing its
2040 // sense negatively impacts test vector tracking.
2041 bool MCDCCondition = CGM.getCodeGenOpts().hasProfileClangInstr() &&
2042 CGM.getCodeGenOpts().MCDCCoverage &&
2043 isInstrumentedCondition(C: Cond);
2044 if (CondUOp->getOpcode() == UO_LNot && !MCDCCondition) {
2045 // Negate the count.
2046 uint64_t FalseCount = getCurrentProfileCount() - TrueCount;
2047 // The values of the enum are chosen to make this negation possible.
2048 LH = static_cast<Stmt::Likelihood>(-LH);
2049 // Negate the condition and swap the destination blocks.
2050 return EmitBranchOnBoolExpr(Cond: CondUOp->getSubExpr(), TrueBlock: FalseBlock, FalseBlock: TrueBlock,
2051 TrueCount: FalseCount, LH);
2052 }
2053 }
2054
2055 if (const ConditionalOperator *CondOp = dyn_cast<ConditionalOperator>(Val: Cond)) {
2056 // br(c ? x : y, t, f) -> br(c, br(x, t, f), br(y, t, f))
2057 llvm::BasicBlock *LHSBlock = createBasicBlock(name: "cond.true");
2058 llvm::BasicBlock *RHSBlock = createBasicBlock(name: "cond.false");
2059
2060 // The ConditionalOperator itself has no likelihood information for its
2061 // true and false branches. This matches the behavior of __builtin_expect.
2062 ConditionalEvaluation cond(*this);
2063 EmitBranchOnBoolExpr(Cond: CondOp->getCond(), TrueBlock: LHSBlock, FalseBlock: RHSBlock,
2064 TrueCount: getProfileCount(S: CondOp), LH: Stmt::LH_None);
2065
2066 // When computing PGO branch weights, we only know the overall count for
2067 // the true block. This code is essentially doing tail duplication of the
2068 // naive code-gen, introducing new edges for which counts are not
2069 // available. Divide the counts proportionally between the LHS and RHS of
2070 // the conditional operator.
2071 uint64_t LHSScaledTrueCount = 0;
2072 if (TrueCount) {
2073 double LHSRatio =
2074 getProfileCount(S: CondOp) / (double)getCurrentProfileCount();
2075 LHSScaledTrueCount = TrueCount * LHSRatio;
2076 }
2077
2078 cond.begin(CGF&: *this);
2079 EmitBlock(BB: LHSBlock);
2080 incrementProfileCounter(ExecSkip: UseExecPath, S: CondOp);
2081 {
2082 ApplyDebugLocation DL(*this, Cond);
2083 EmitBranchOnBoolExpr(Cond: CondOp->getLHS(), TrueBlock, FalseBlock,
2084 TrueCount: LHSScaledTrueCount, LH, ConditionalOp: CondOp);
2085 }
2086 cond.end(CGF&: *this);
2087
2088 cond.begin(CGF&: *this);
2089 EmitBlock(BB: RHSBlock);
2090 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondOp);
2091 EmitBranchOnBoolExpr(Cond: CondOp->getRHS(), TrueBlock, FalseBlock,
2092 TrueCount: TrueCount - LHSScaledTrueCount, LH, ConditionalOp: CondOp);
2093 cond.end(CGF&: *this);
2094
2095 return;
2096 }
2097
2098 if (const CXXThrowExpr *Throw = dyn_cast<CXXThrowExpr>(Val: Cond)) {
2099 // Conditional operator handling can give us a throw expression as a
2100 // condition for a case like:
2101 // br(c ? throw x : y, t, f) -> br(c, br(throw x, t, f), br(y, t, f)
2102 // Fold this to:
2103 // br(c, throw x, br(y, t, f))
2104 EmitCXXThrowExpr(E: Throw, /*KeepInsertionPoint*/false);
2105 return;
2106 }
2107
2108 // Emit the code with the fully general case.
2109 llvm::Value *CondV;
2110 {
2111 ApplyDebugLocation DL(*this, Cond);
2112 CondV = EvaluateExprAsBool(E: Cond);
2113 }
2114
2115 MaybeEmitDeferredVarDeclInit(var: ConditionalDecl);
2116
2117 // If not at the top of the logical operator nest, update MCDC temp with the
2118 // boolean result of the evaluated condition.
2119 {
2120 const Expr *MCDCBaseExpr = Cond;
2121 // When a nested ConditionalOperator (ternary) is encountered in a boolean
2122 // expression, MC/DC tracks the result of the ternary, and this is tied to
2123 // the ConditionalOperator expression and not the ternary's LHS or RHS. If
2124 // this is the case, the ConditionalOperator expression is passed through
2125 // the ConditionalOp parameter and then used as the MCDC base expression.
2126 if (ConditionalOp)
2127 MCDCBaseExpr = ConditionalOp;
2128
2129 if (isMCDCBranchExpr(E: stripCond(C: MCDCBaseExpr)) &&
2130 !isMCDCDecisionExpr(E: stripCond(C: Cond)))
2131 maybeUpdateMCDCCondBitmap(E: MCDCBaseExpr, Val: CondV);
2132 }
2133
2134 llvm::MDNode *Weights = nullptr;
2135 llvm::MDNode *Unpredictable = nullptr;
2136
2137 // If the branch has a condition wrapped by __builtin_unpredictable,
2138 // create metadata that specifies that the branch is unpredictable.
2139 // Don't bother if not optimizing because that metadata would not be used.
2140 auto *Call = dyn_cast<CallExpr>(Val: Cond->IgnoreImpCasts());
2141 if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
2142 auto *FD = dyn_cast_or_null<FunctionDecl>(Val: Call->getCalleeDecl());
2143 if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
2144 llvm::MDBuilder MDHelper(getLLVMContext());
2145 Unpredictable = MDHelper.createUnpredictable();
2146 }
2147 }
2148
2149 // If there is a Likelihood knowledge for the cond, lower it.
2150 // Note that if not optimizing this won't emit anything.
2151 llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(Cond: CondV, LH);
2152 if (CondV != NewCondV)
2153 CondV = NewCondV;
2154 else {
2155 // Otherwise, lower profile counts. Note that we do this even at -O0.
2156 uint64_t CurrentCount = std::max(a: getCurrentProfileCount(), b: TrueCount);
2157 Weights = createProfileWeights(TrueCount, FalseCount: CurrentCount - TrueCount);
2158 }
2159
2160 llvm::Instruction *BrInst = Builder.CreateCondBr(Cond: CondV, True: TrueBlock, False: FalseBlock,
2161 BranchWeights: Weights, Unpredictable);
2162 addInstToNewSourceAtom(KeyInstruction: BrInst, Backup: CondV);
2163
2164 switch (HLSLControlFlowAttr) {
2165 case HLSLControlFlowHintAttr::Microsoft_branch:
2166 case HLSLControlFlowHintAttr::Microsoft_flatten: {
2167 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2168
2169 llvm::ConstantInt *BranchHintConstant =
2170 HLSLControlFlowAttr ==
2171 HLSLControlFlowHintAttr::Spelling::Microsoft_branch
2172 ? llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 1)
2173 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 2);
2174
2175 SmallVector<llvm::Metadata *, 2> Vals(
2176 {MDHelper.createString(Str: "hlsl.controlflow.hint"),
2177 MDHelper.createConstant(C: BranchHintConstant)});
2178 BrInst->setMetadata(Kind: "hlsl.controlflow.hint",
2179 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Vals));
2180 break;
2181 }
2182 // This is required to avoid warnings during compilation
2183 case HLSLControlFlowHintAttr::SpellingNotCalculated:
2184 break;
2185 }
2186}
2187
2188llvm::Value *CodeGenFunction::EmitScalarOrConstFoldImmArg(unsigned ICEArguments,
2189 unsigned Idx,
2190 const CallExpr *E) {
2191 llvm::Value *Arg = nullptr;
2192 if ((ICEArguments & (1 << Idx)) == 0) {
2193 Arg = EmitScalarExpr(E: E->getArg(Arg: Idx));
2194 } else {
2195 // If this is required to be a constant, constant fold it so that we
2196 // know that the generated intrinsic gets a ConstantInt.
2197 std::optional<llvm::APSInt> Result =
2198 E->getArg(Arg: Idx)->getIntegerConstantExpr(Ctx: getContext());
2199 assert(Result && "Expected argument to be a constant");
2200 Arg = llvm::ConstantInt::get(Context&: getLLVMContext(), V: *Result);
2201 }
2202 return Arg;
2203}
2204
2205/// ErrorUnsupported - Print out an error that codegen doesn't support the
2206/// specified stmt yet.
2207void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) {
2208 CGM.ErrorUnsupported(S, Type);
2209}
2210
2211/// emitNonZeroVLAInit - Emit the "zero" initialization of a
2212/// variable-length array whose elements have a non-zero bit-pattern.
2213///
2214/// \param baseType the inner-most element type of the array
2215/// \param src - a char* pointing to the bit-pattern for a single
2216/// base element of the array
2217/// \param sizeInChars - the total size of the VLA, in chars
2218static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
2219 Address dest, Address src,
2220 llvm::Value *sizeInChars) {
2221 CGBuilderTy &Builder = CGF.Builder;
2222
2223 CharUnits baseSize = CGF.getContext().getTypeSizeInChars(T: baseType);
2224 llvm::Value *baseSizeInChars
2225 = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: baseSize.getQuantity());
2226
2227 Address begin = dest.withElementType(ElemTy: CGF.Int8Ty);
2228 llvm::Value *end = Builder.CreateInBoundsGEP(Ty: begin.getElementType(),
2229 Ptr: begin.emitRawPointer(CGF),
2230 IdxList: sizeInChars, Name: "vla.end");
2231
2232 llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
2233 llvm::BasicBlock *loopBB = CGF.createBasicBlock(name: "vla-init.loop");
2234 llvm::BasicBlock *contBB = CGF.createBasicBlock(name: "vla-init.cont");
2235
2236 // Make a loop over the VLA. C99 guarantees that the VLA element
2237 // count must be nonzero.
2238 CGF.EmitBlock(BB: loopBB);
2239
2240 llvm::PHINode *cur = Builder.CreatePHI(Ty: begin.getType(), NumReservedValues: 2, Name: "vla.cur");
2241 cur->addIncoming(V: begin.emitRawPointer(CGF), BB: originBB);
2242
2243 CharUnits curAlign =
2244 dest.getAlignment().alignmentOfArrayElement(elementSize: baseSize);
2245
2246 // memcpy the individual element bit-pattern.
2247 Builder.CreateMemCpy(Dest: Address(cur, CGF.Int8Ty, curAlign), Src: src, Size: baseSizeInChars,
2248 /*volatile*/ IsVolatile: false);
2249
2250 // Go to the next element.
2251 llvm::Value *next =
2252 Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: cur, IdxList: baseSizeInChars, Name: "vla.next");
2253
2254 // Leave if that's the end of the VLA.
2255 llvm::Value *done = Builder.CreateICmpEQ(LHS: next, RHS: end, Name: "vla-init.isdone");
2256 Builder.CreateCondBr(Cond: done, True: contBB, False: loopBB);
2257 cur->addIncoming(V: next, BB: loopBB);
2258
2259 CGF.EmitBlock(BB: contBB);
2260}
2261
2262Address CodeGenFunction::EmitAddressOfPFPField(Address RecordPtr,
2263 const PFPField &Field) {
2264 return EmitAddressOfPFPField(
2265 RecordPtr,
2266 FieldPtr: Builder.CreateConstInBoundsByteGEP(Addr: RecordPtr.withElementType(ElemTy: Int8Ty),
2267 Offset: Field.Offset),
2268 Field: Field.Field);
2269}
2270
2271Address CodeGenFunction::EmitAddressOfPFPField(Address RecordPtr,
2272 Address PtrPtr,
2273 const FieldDecl *Field) {
2274 llvm::Value *Disc;
2275 if (CGM.getContext().arePFPFieldsTriviallyCopyable(RD: Field->getParent())) {
2276 uint64_t FieldSignature =
2277 llvm::getPointerAuthStableSipHash(S: CGM.getPFPFieldName(FD: Field));
2278 Disc = llvm::ConstantInt::get(Ty: CGM.Int64Ty, V: FieldSignature);
2279 } else
2280 Disc = Builder.CreatePtrToInt(V: RecordPtr.getBasePointer(), DestTy: CGM.Int64Ty);
2281
2282 llvm::GlobalValue *DS = CGM.getPFPDeactivationSymbol(FD: Field);
2283 llvm::OperandBundleDef DSBundle("deactivation-symbol", DS);
2284 llvm::Value *Args[] = {PtrPtr.getBasePointer(), Disc, Builder.getTrue()};
2285 return Address(
2286 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::protected_field_ptr,
2287 Tys: PtrPtr.getType()),
2288 Args, OpBundles: DSBundle),
2289 VoidPtrTy, PtrPtr.getAlignment());
2290}
2291
2292void
2293CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
2294 // Ignore empty classes in C++.
2295 if (getLangOpts().CPlusPlus)
2296 if (const auto *RD = Ty->getAsCXXRecordDecl(); RD && RD->isEmpty())
2297 return;
2298
2299 if (DestPtr.getElementType() != Int8Ty)
2300 DestPtr = DestPtr.withElementType(ElemTy: Int8Ty);
2301
2302 // Get size and alignment info for this aggregate.
2303 CharUnits size = getContext().getTypeSizeInChars(T: Ty);
2304
2305 llvm::Value *SizeVal;
2306 const VariableArrayType *vla;
2307
2308 // Don't bother emitting a zero-byte memset.
2309 if (size.isZero()) {
2310 // But note that getTypeInfo returns 0 for a VLA.
2311 if (const VariableArrayType *vlaType =
2312 dyn_cast_or_null<VariableArrayType>(
2313 Val: getContext().getAsArrayType(T: Ty))) {
2314 auto VlaSize = getVLASize(vla: vlaType);
2315 SizeVal = VlaSize.NumElts;
2316 CharUnits eltSize = getContext().getTypeSizeInChars(T: VlaSize.Type);
2317 if (!eltSize.isOne())
2318 SizeVal = Builder.CreateNUWMul(LHS: SizeVal, RHS: CGM.getSize(numChars: eltSize));
2319 vla = vlaType;
2320 } else {
2321 return;
2322 }
2323 } else {
2324 SizeVal = CGM.getSize(numChars: size);
2325 vla = nullptr;
2326 }
2327
2328 // If the type contains a pointer to data member we can't memset it to zero.
2329 // Instead, create a null constant and copy it to the destination.
2330 // TODO: there are other patterns besides zero that we can usefully memset,
2331 // like -1, which happens to be the pattern used by member-pointers.
2332 if (!CGM.getTypes().isZeroInitializable(T: Ty)) {
2333 // For a VLA, emit a single element, then splat that over the VLA.
2334 if (vla) Ty = getContext().getBaseElementType(VAT: vla);
2335
2336 llvm::Constant *NullConstant = CGM.EmitNullConstant(T: Ty);
2337
2338 llvm::GlobalVariable *NullVariable =
2339 new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
2340 /*isConstant=*/true,
2341 llvm::GlobalVariable::PrivateLinkage,
2342 NullConstant, Twine());
2343 CharUnits NullAlign = DestPtr.getAlignment();
2344 NullVariable->setAlignment(NullAlign.getAsAlign());
2345 Address SrcPtr(NullVariable, Builder.getInt8Ty(), NullAlign);
2346
2347 if (vla) return emitNonZeroVLAInit(CGF&: *this, baseType: Ty, dest: DestPtr, src: SrcPtr, sizeInChars: SizeVal);
2348
2349 // Get and call the appropriate llvm.memcpy overload.
2350 Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: SizeVal, IsVolatile: false);
2351 } else {
2352 // Otherwise, just memset the whole thing to zero. This is legal
2353 // because in LLVM, all default initializers (other than the ones we just
2354 // handled above, and the case handled below) are guaranteed to have a bit
2355 // pattern of all zeros.
2356 Builder.CreateMemSet(Dest: DestPtr, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
2357 }
2358
2359 // With the pointer field protection feature, null pointers do not have a bit
2360 // pattern of zero in memory, so we must initialize them separately.
2361 for (auto &Field : getContext().findPFPFields(Ty)) {
2362 auto addr = EmitAddressOfPFPField(RecordPtr: DestPtr, Field);
2363 Builder.CreateStore(Val: llvm::ConstantPointerNull::get(T: VoidPtrTy), Addr: addr);
2364 }
2365}
2366
2367llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
2368 // Make sure that there is a block for the indirect goto.
2369 if (!IndirectBranch)
2370 GetIndirectGotoBlock();
2371
2372 llvm::BasicBlock *BB = getJumpDestForLabel(S: L).getBlock();
2373
2374 // Make sure the indirect branch includes all of the address-taken blocks.
2375 IndirectBranch->addDestination(Dest: BB);
2376 return llvm::BlockAddress::get(Ty: CurFn->getType(), BB);
2377}
2378
2379llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {
2380 // If we already made the indirect branch for indirect goto, return its block.
2381 if (IndirectBranch) return IndirectBranch->getParent();
2382
2383 CGBuilderTy TmpBuilder(CGM, createBasicBlock(name: "indirectgoto"));
2384
2385 // Create the PHI node that indirect gotos will add entries to.
2386 llvm::Value *DestVal = TmpBuilder.CreatePHI(Ty: Int8PtrTy, NumReservedValues: 0,
2387 Name: "indirect.goto.dest");
2388
2389 // Create the indirect branch instruction.
2390 IndirectBranch = TmpBuilder.CreateIndirectBr(Addr: DestVal);
2391 return IndirectBranch->getParent();
2392}
2393
2394/// Computes the length of an array in elements, as well as the base
2395/// element type and a properly-typed first element pointer.
2396llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
2397 QualType &baseType,
2398 Address &addr) {
2399 const ArrayType *arrayType = origArrayType;
2400
2401 // If it's a VLA, we have to load the stored size. Note that
2402 // this is the size of the VLA in bytes, not its size in elements.
2403 llvm::Value *numVLAElements = nullptr;
2404 if (isa<VariableArrayType>(Val: arrayType)) {
2405 numVLAElements = getVLASize(vla: cast<VariableArrayType>(Val: arrayType)).NumElts;
2406
2407 // Walk into all VLAs. This doesn't require changes to addr,
2408 // which has type T* where T is the first non-VLA element type.
2409 do {
2410 QualType elementType = arrayType->getElementType();
2411 arrayType = getContext().getAsArrayType(T: elementType);
2412
2413 // If we only have VLA components, 'addr' requires no adjustment.
2414 if (!arrayType) {
2415 baseType = elementType;
2416 return numVLAElements;
2417 }
2418 } while (isa<VariableArrayType>(Val: arrayType));
2419
2420 // We get out here only if we find a constant array type
2421 // inside the VLA.
2422 }
2423
2424 // We have some number of constant-length arrays, so addr should
2425 // have LLVM type [M x [N x [...]]]*. Build a GEP that walks
2426 // down to the first element of addr.
2427 SmallVector<llvm::Value*, 8> gepIndices;
2428
2429 // GEP down to the array type.
2430 llvm::ConstantInt *zero = Builder.getInt32(C: 0);
2431 gepIndices.push_back(Elt: zero);
2432
2433 uint64_t countFromCLAs = 1;
2434 QualType eltType;
2435
2436 llvm::ArrayType *llvmArrayType =
2437 dyn_cast<llvm::ArrayType>(Val: addr.getElementType());
2438 while (llvmArrayType) {
2439 assert(isa<ConstantArrayType>(arrayType));
2440 assert(cast<ConstantArrayType>(arrayType)->getZExtSize() ==
2441 llvmArrayType->getNumElements());
2442
2443 gepIndices.push_back(Elt: zero);
2444 countFromCLAs *= llvmArrayType->getNumElements();
2445 eltType = arrayType->getElementType();
2446
2447 llvmArrayType =
2448 dyn_cast<llvm::ArrayType>(Val: llvmArrayType->getElementType());
2449 arrayType = getContext().getAsArrayType(T: arrayType->getElementType());
2450 assert((!llvmArrayType || arrayType) &&
2451 "LLVM and Clang types are out-of-synch");
2452 }
2453
2454 if (arrayType) {
2455 // From this point onwards, the Clang array type has been emitted
2456 // as some other type (probably a packed struct). Compute the array
2457 // size, and just emit the 'begin' expression as a bitcast.
2458 while (arrayType) {
2459 countFromCLAs *= cast<ConstantArrayType>(Val: arrayType)->getZExtSize();
2460 eltType = arrayType->getElementType();
2461 arrayType = getContext().getAsArrayType(T: eltType);
2462 }
2463
2464 llvm::Type *baseType = ConvertType(T: eltType);
2465 addr = addr.withElementType(ElemTy: baseType);
2466 } else {
2467 // Create the actual GEP.
2468 addr = Address(Builder.CreateInBoundsGEP(Ty: addr.getElementType(),
2469 Ptr: addr.emitRawPointer(CGF&: *this),
2470 IdxList: gepIndices, Name: "array.begin"),
2471 ConvertTypeForMem(T: eltType), addr.getAlignment());
2472 }
2473
2474 baseType = eltType;
2475
2476 llvm::Value *numElements
2477 = llvm::ConstantInt::get(Ty: SizeTy, V: countFromCLAs);
2478
2479 // If we had any VLA dimensions, factor them in.
2480 if (numVLAElements)
2481 numElements = Builder.CreateNUWMul(LHS: numVLAElements, RHS: numElements);
2482
2483 return numElements;
2484}
2485
2486CodeGenFunction::VlaSizePair CodeGenFunction::getVLASize(QualType type) {
2487 const VariableArrayType *vla = getContext().getAsVariableArrayType(T: type);
2488 assert(vla && "type was not a variable array type!");
2489 return getVLASize(vla);
2490}
2491
2492CodeGenFunction::VlaSizePair
2493CodeGenFunction::getVLASize(const VariableArrayType *type) {
2494 // The number of elements so far; always size_t.
2495 llvm::Value *numElements = nullptr;
2496
2497 QualType elementType;
2498 do {
2499 elementType = type->getElementType();
2500 llvm::Value *vlaSize = VLASizeMap[type->getSizeExpr()];
2501 assert(vlaSize && "no size for VLA!");
2502 assert(vlaSize->getType() == SizeTy);
2503
2504 if (!numElements) {
2505 numElements = vlaSize;
2506 } else {
2507 // It's undefined behavior if this wraps around, so mark it that way.
2508 // FIXME: Teach -fsanitize=undefined to trap this.
2509 numElements = Builder.CreateNUWMul(LHS: numElements, RHS: vlaSize);
2510 }
2511 } while ((type = getContext().getAsVariableArrayType(T: elementType)));
2512
2513 return { numElements, elementType };
2514}
2515
2516CodeGenFunction::VlaSizePair
2517CodeGenFunction::getVLAElements1D(QualType type) {
2518 const VariableArrayType *vla = getContext().getAsVariableArrayType(T: type);
2519 assert(vla && "type was not a variable array type!");
2520 return getVLAElements1D(vla);
2521}
2522
2523CodeGenFunction::VlaSizePair
2524CodeGenFunction::getVLAElements1D(const VariableArrayType *Vla) {
2525 llvm::Value *VlaSize = VLASizeMap[Vla->getSizeExpr()];
2526 assert(VlaSize && "no size for VLA!");
2527 assert(VlaSize->getType() == SizeTy);
2528 return { VlaSize, Vla->getElementType() };
2529}
2530
2531void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
2532 assert(type->isVariablyModifiedType() &&
2533 "Must pass variably modified type to EmitVLASizes!");
2534
2535 EnsureInsertPoint();
2536
2537 // We're going to walk down into the type and look for VLA
2538 // expressions.
2539 do {
2540 assert(type->isVariablyModifiedType());
2541
2542 const Type *ty = type.getTypePtr();
2543 switch (ty->getTypeClass()) {
2544
2545#define TYPE(Class, Base)
2546#define ABSTRACT_TYPE(Class, Base)
2547#define NON_CANONICAL_TYPE(Class, Base)
2548#define DEPENDENT_TYPE(Class, Base) case Type::Class:
2549#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
2550#include "clang/AST/TypeNodes.inc"
2551 llvm_unreachable("unexpected dependent type!");
2552
2553 // These types are never variably-modified.
2554 case Type::Builtin:
2555 case Type::Complex:
2556 case Type::Vector:
2557 case Type::ExtVector:
2558 case Type::ConstantMatrix:
2559 case Type::Record:
2560 case Type::Enum:
2561 case Type::Using:
2562 case Type::TemplateSpecialization:
2563 case Type::ObjCTypeParam:
2564 case Type::ObjCObject:
2565 case Type::ObjCInterface:
2566 case Type::ObjCObjectPointer:
2567 case Type::BitInt:
2568 case Type::HLSLInlineSpirv:
2569 case Type::PredefinedSugar:
2570 llvm_unreachable("type class is never variably-modified!");
2571
2572 case Type::Adjusted:
2573 type = cast<AdjustedType>(Val: ty)->getAdjustedType();
2574 break;
2575
2576 case Type::Decayed:
2577 type = cast<DecayedType>(Val: ty)->getPointeeType();
2578 break;
2579
2580 case Type::Pointer:
2581 type = cast<PointerType>(Val: ty)->getPointeeType();
2582 break;
2583
2584 case Type::BlockPointer:
2585 type = cast<BlockPointerType>(Val: ty)->getPointeeType();
2586 break;
2587
2588 case Type::LValueReference:
2589 case Type::RValueReference:
2590 type = cast<ReferenceType>(Val: ty)->getPointeeType();
2591 break;
2592
2593 case Type::MemberPointer:
2594 type = cast<MemberPointerType>(Val: ty)->getPointeeType();
2595 break;
2596
2597 case Type::ArrayParameter:
2598 case Type::ConstantArray:
2599 case Type::IncompleteArray:
2600 // Losing element qualification here is fine.
2601 type = cast<ArrayType>(Val: ty)->getElementType();
2602 break;
2603
2604 case Type::VariableArray: {
2605 // Losing element qualification here is fine.
2606 const VariableArrayType *vat = cast<VariableArrayType>(Val: ty);
2607
2608 // Unknown size indication requires no size computation.
2609 // Otherwise, evaluate and record it.
2610 if (const Expr *sizeExpr = vat->getSizeExpr()) {
2611 // It's possible that we might have emitted this already,
2612 // e.g. with a typedef and a pointer to it.
2613 llvm::Value *&entry = VLASizeMap[sizeExpr];
2614 if (!entry) {
2615 llvm::Value *size = EmitScalarExpr(E: sizeExpr);
2616
2617 // C11 6.7.6.2p5:
2618 // If the size is an expression that is not an integer constant
2619 // expression [...] each time it is evaluated it shall have a value
2620 // greater than zero.
2621 if (SanOpts.has(K: SanitizerKind::VLABound)) {
2622 auto CheckOrdinal = SanitizerKind::SO_VLABound;
2623 auto CheckHandler = SanitizerHandler::VLABoundNotPositive;
2624 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2625 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: size->getType());
2626 clang::QualType SEType = sizeExpr->getType();
2627 llvm::Value *CheckCondition =
2628 SEType->isSignedIntegerType()
2629 ? Builder.CreateICmpSGT(LHS: size, RHS: Zero)
2630 : Builder.CreateICmpUGT(LHS: size, RHS: Zero);
2631 llvm::Constant *StaticArgs[] = {
2632 EmitCheckSourceLocation(Loc: sizeExpr->getBeginLoc()),
2633 EmitCheckTypeDescriptor(T: SEType)};
2634 EmitCheck(Checked: std::make_pair(x&: CheckCondition, y&: CheckOrdinal),
2635 Check: CheckHandler, StaticArgs, DynamicArgs: size);
2636 }
2637
2638 // Always zexting here would be wrong if it weren't
2639 // undefined behavior to have a negative bound.
2640 // FIXME: What about when size's type is larger than size_t?
2641 entry = Builder.CreateIntCast(V: size, DestTy: SizeTy, /*signed*/ isSigned: false);
2642 }
2643 }
2644 type = vat->getElementType();
2645 break;
2646 }
2647
2648 case Type::FunctionProto:
2649 case Type::FunctionNoProto:
2650 type = cast<FunctionType>(Val: ty)->getReturnType();
2651 break;
2652
2653 case Type::Paren:
2654 case Type::TypeOf:
2655 case Type::UnaryTransform:
2656 case Type::Attributed:
2657 case Type::BTFTagAttributed:
2658 case Type::OverflowBehavior:
2659 case Type::HLSLAttributedResource:
2660 case Type::SubstTemplateTypeParm:
2661 case Type::MacroQualified:
2662 case Type::CountAttributed:
2663 // Keep walking after single level desugaring.
2664 type = type.getSingleStepDesugaredType(Context: getContext());
2665 break;
2666
2667 case Type::Typedef:
2668 case Type::Decltype:
2669 case Type::Auto:
2670 case Type::DeducedTemplateSpecialization:
2671 case Type::PackIndexing:
2672 // Stop walking: nothing to do.
2673 return;
2674
2675 case Type::TypeOfExpr:
2676 // Stop walking: emit typeof expression.
2677 EmitIgnoredExpr(E: cast<TypeOfExprType>(Val: ty)->getUnderlyingExpr());
2678 return;
2679
2680 case Type::Atomic:
2681 type = cast<AtomicType>(Val: ty)->getValueType();
2682 break;
2683
2684 case Type::Pipe:
2685 type = cast<PipeType>(Val: ty)->getElementType();
2686 break;
2687 }
2688 } while (type->isVariablyModifiedType());
2689}
2690
2691Address CodeGenFunction::EmitVAListRef(const Expr* E) {
2692 if (getContext().getBuiltinVaListType()->isArrayType())
2693 return EmitPointerWithAlignment(Addr: E);
2694 return EmitLValue(E).getAddress();
2695}
2696
2697Address CodeGenFunction::EmitMSVAListRef(const Expr *E) {
2698 return EmitLValue(E).getAddress();
2699}
2700
2701void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E,
2702 const APValue &Init) {
2703 assert(Init.hasValue() && "Invalid DeclRefExpr initializer!");
2704 if (CGDebugInfo *Dbg = getDebugInfo())
2705 if (CGM.getCodeGenOpts().hasReducedDebugInfo())
2706 Dbg->EmitGlobalVariable(VD: E->getDecl(), Init);
2707}
2708
2709CodeGenFunction::PeepholeProtection
2710CodeGenFunction::protectFromPeepholes(RValue rvalue) {
2711 // At the moment, the only aggressive peephole we do in IR gen
2712 // is trunc(zext) folding, but if we add more, we can easily
2713 // extend this protection.
2714
2715 if (!rvalue.isScalar()) return PeepholeProtection();
2716 llvm::Value *value = rvalue.getScalarVal();
2717 if (!isa<llvm::ZExtInst>(Val: value)) return PeepholeProtection();
2718
2719 // Just make an extra bitcast.
2720 assert(HaveInsertPoint());
2721 llvm::Instruction *inst = new llvm::BitCastInst(value, value->getType(), "",
2722 Builder.GetInsertBlock());
2723
2724 PeepholeProtection protection;
2725 protection.Inst = inst;
2726 return protection;
2727}
2728
2729void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {
2730 if (!protection.Inst) return;
2731
2732 // In theory, we could try to duplicate the peepholes now, but whatever.
2733 protection.Inst->eraseFromParent();
2734}
2735
2736void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2737 QualType Ty, SourceLocation Loc,
2738 SourceLocation AssumptionLoc,
2739 llvm::Value *Alignment,
2740 llvm::Value *OffsetValue) {
2741 if (Alignment->getType() != IntPtrTy)
2742 Alignment =
2743 Builder.CreateIntCast(V: Alignment, DestTy: IntPtrTy, isSigned: false, Name: "casted.align");
2744 if (OffsetValue && OffsetValue->getType() != IntPtrTy)
2745 OffsetValue =
2746 Builder.CreateIntCast(V: OffsetValue, DestTy: IntPtrTy, isSigned: true, Name: "casted.offset");
2747 llvm::Value *TheCheck = nullptr;
2748 if (SanOpts.has(K: SanitizerKind::Alignment)) {
2749 llvm::Value *PtrIntValue =
2750 Builder.CreatePtrToInt(V: PtrValue, DestTy: IntPtrTy, Name: "ptrint");
2751
2752 if (OffsetValue) {
2753 bool IsOffsetZero = false;
2754 if (const auto *CI = dyn_cast<llvm::ConstantInt>(Val: OffsetValue))
2755 IsOffsetZero = CI->isZero();
2756
2757 if (!IsOffsetZero)
2758 PtrIntValue = Builder.CreateSub(LHS: PtrIntValue, RHS: OffsetValue, Name: "offsetptr");
2759 }
2760
2761 llvm::Value *Zero = llvm::ConstantInt::get(Ty: IntPtrTy, V: 0);
2762 llvm::Value *Mask =
2763 Builder.CreateSub(LHS: Alignment, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: 1));
2764 llvm::Value *MaskedPtr = Builder.CreateAnd(LHS: PtrIntValue, RHS: Mask, Name: "maskedptr");
2765 TheCheck = Builder.CreateICmpEQ(LHS: MaskedPtr, RHS: Zero, Name: "maskcond");
2766 }
2767 llvm::Instruction *Assumption = Builder.CreateAlignmentAssumption(
2768 DL: CGM.getDataLayout(), PtrValue, Alignment, OffsetValue);
2769
2770 if (!SanOpts.has(K: SanitizerKind::Alignment))
2771 return;
2772 emitAlignmentAssumptionCheck(Ptr: PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2773 OffsetValue, TheCheck, Assumption);
2774}
2775
2776void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2777 const Expr *E,
2778 SourceLocation AssumptionLoc,
2779 llvm::Value *Alignment,
2780 llvm::Value *OffsetValue) {
2781 QualType Ty = E->getType();
2782 SourceLocation Loc = E->getExprLoc();
2783
2784 emitAlignmentAssumption(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2785 OffsetValue);
2786}
2787
2788llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn,
2789 llvm::Value *AnnotatedVal,
2790 StringRef AnnotationStr,
2791 SourceLocation Location,
2792 const AnnotateAttr *Attr) {
2793 SmallVector<llvm::Value *, 5> Args = {
2794 AnnotatedVal,
2795 CGM.EmitAnnotationString(Str: AnnotationStr),
2796 CGM.EmitAnnotationUnit(Loc: Location),
2797 CGM.EmitAnnotationLineNo(L: Location),
2798 };
2799 if (Attr)
2800 Args.push_back(Elt: CGM.EmitAnnotationArgs(Attr));
2801 return Builder.CreateCall(Callee: AnnotationFn, Args);
2802}
2803
2804void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
2805 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2806 for (const auto *I : D->specific_attrs<AnnotateAttr>())
2807 EmitAnnotationCall(AnnotationFn: CGM.getIntrinsic(IID: llvm::Intrinsic::var_annotation,
2808 Tys: {V->getType(), CGM.ConstGlobalsPtrTy}),
2809 AnnotatedVal: V, AnnotationStr: I->getAnnotation(), Location: D->getLocation(), Attr: I);
2810}
2811
2812Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
2813 Address Addr) {
2814 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2815 llvm::Value *V = Addr.emitRawPointer(CGF&: *this);
2816 llvm::Type *VTy = V->getType();
2817 auto *PTy = dyn_cast<llvm::PointerType>(Val: VTy);
2818 unsigned AS = PTy ? PTy->getAddressSpace() : 0;
2819 llvm::PointerType *IntrinTy =
2820 llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: AS);
2821 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::ptr_annotation,
2822 Tys: {IntrinTy, CGM.ConstGlobalsPtrTy});
2823
2824 for (const auto *I : D->specific_attrs<AnnotateAttr>()) {
2825 // FIXME Always emit the cast inst so we can differentiate between
2826 // annotation on the first field of a struct and annotation on the struct
2827 // itself.
2828 if (VTy != IntrinTy)
2829 V = Builder.CreateBitCast(V, DestTy: IntrinTy);
2830 V = EmitAnnotationCall(AnnotationFn: F, AnnotatedVal: V, AnnotationStr: I->getAnnotation(), Location: D->getLocation(), Attr: I);
2831 V = Builder.CreateBitCast(V, DestTy: VTy);
2832 }
2833
2834 return Address(V, Addr.getElementType(), Addr.getAlignment());
2835}
2836
2837CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
2838
2839CodeGenFunction::SanitizerScope::SanitizerScope(CodeGenFunction *CGF)
2840 : CGF(CGF) {
2841 assert(!CGF->IsSanitizerScope);
2842 CGF->IsSanitizerScope = true;
2843}
2844
2845CodeGenFunction::SanitizerScope::~SanitizerScope() {
2846 CGF->IsSanitizerScope = false;
2847}
2848
2849void CodeGenFunction::InsertHelper(llvm::Instruction *I,
2850 const llvm::Twine &Name,
2851 llvm::BasicBlock::iterator InsertPt) const {
2852 LoopStack.InsertHelper(I);
2853 if (IsSanitizerScope)
2854 I->setNoSanitizeMetadata();
2855}
2856
2857void CGBuilderInserter::InsertHelper(
2858 llvm::Instruction *I, const llvm::Twine &Name,
2859 llvm::BasicBlock::iterator InsertPt) const {
2860 llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, InsertPt);
2861 if (CGF)
2862 CGF->InsertHelper(I, Name, InsertPt);
2863}
2864
2865// Emits an error if we don't have a valid set of target features for the
2866// called function.
2867void CodeGenFunction::checkTargetFeatures(const CallExpr *E,
2868 const FunctionDecl *TargetDecl) {
2869 // SemaChecking cannot handle below x86 builtins because they have different
2870 // parameter ranges with different TargetAttribute of caller.
2871 if (CGM.getContext().getTargetInfo().getTriple().isX86()) {
2872 unsigned BuiltinID = TargetDecl->getBuiltinID();
2873 if (BuiltinID == X86::BI__builtin_ia32_cmpps ||
2874 BuiltinID == X86::BI__builtin_ia32_cmpss ||
2875 BuiltinID == X86::BI__builtin_ia32_cmppd ||
2876 BuiltinID == X86::BI__builtin_ia32_cmpsd) {
2877 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
2878 llvm::StringMap<bool> TargetFetureMap;
2879 CGM.getContext().getFunctionFeatureMap(FeatureMap&: TargetFetureMap, FD);
2880 llvm::APSInt Result =
2881 *(E->getArg(Arg: 2)->getIntegerConstantExpr(Ctx: CGM.getContext()));
2882 if (Result.getSExtValue() > 7 && !TargetFetureMap.lookup(Key: "avx"))
2883 CGM.getDiags().Report(Loc: E->getBeginLoc(), DiagID: diag::err_builtin_needs_feature)
2884 << TargetDecl->getDeclName() << "avx";
2885 }
2886 }
2887 return checkTargetFeatures(Loc: E->getBeginLoc(), TargetDecl);
2888}
2889
2890// Emits an error if we don't have a valid set of target features for the
2891// called function.
2892void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
2893 const FunctionDecl *TargetDecl) {
2894 // Early exit if this is an indirect call.
2895 if (!TargetDecl)
2896 return;
2897
2898 // Get the current enclosing function if it exists. If it doesn't
2899 // we can't check the target features anyhow.
2900 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
2901 if (!FD)
2902 return;
2903
2904 bool IsAlwaysInline = TargetDecl->hasAttr<AlwaysInlineAttr>();
2905 bool IsFlatten = FD && FD->hasAttr<FlattenAttr>();
2906
2907 // Grab the required features for the call. For a builtin this is listed in
2908 // the td file with the default cpu, for an always_inline function this is any
2909 // listed cpu and any listed features.
2910 unsigned BuiltinID = TargetDecl->getBuiltinID();
2911 std::string MissingFeature;
2912 llvm::StringMap<bool> CallerFeatureMap;
2913 CGM.getContext().getFunctionFeatureMap(FeatureMap&: CallerFeatureMap, FD);
2914 // When compiling in HipStdPar mode we have to be conservative in rejecting
2915 // target specific features in the FE, and defer the possible error to the
2916 // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
2917 // referenced by an accelerator executable function, we emit an error.
2918 bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;
2919 if (BuiltinID) {
2920 StringRef FeatureList(CGM.getContext().BuiltinInfo.getRequiredFeatures(ID: BuiltinID));
2921 if (!Builtin::evaluateRequiredTargetFeatures(
2922 RequiredFatures: FeatureList, TargetFetureMap: CallerFeatureMap) && !IsHipStdPar) {
2923 CGM.getDiags().Report(Loc, DiagID: diag::err_builtin_needs_feature)
2924 << TargetDecl->getDeclName()
2925 << FeatureList;
2926 }
2927 } else if (!TargetDecl->isMultiVersion() &&
2928 TargetDecl->hasAttr<TargetAttr>()) {
2929 // Get the required features for the callee.
2930
2931 const TargetAttr *TD = TargetDecl->getAttr<TargetAttr>();
2932 ParsedTargetAttr ParsedAttr =
2933 CGM.getContext().filterFunctionTargetAttrs(TD);
2934
2935 SmallVector<StringRef, 1> ReqFeatures;
2936 llvm::StringMap<bool> CalleeFeatureMap;
2937 CGM.getContext().getFunctionFeatureMap(FeatureMap&: CalleeFeatureMap, TargetDecl);
2938
2939 for (const auto &F : ParsedAttr.Features) {
2940 if (F[0] == '+' && CalleeFeatureMap.lookup(Key: F.substr(pos: 1)))
2941 ReqFeatures.push_back(Elt: StringRef(F).substr(Start: 1));
2942 }
2943
2944 for (const auto &F : CalleeFeatureMap) {
2945 // Only positive features are "required".
2946 if (F.getValue())
2947 ReqFeatures.push_back(Elt: F.getKey());
2948 }
2949 if (!llvm::all_of(Range&: ReqFeatures,
2950 P: [&](StringRef Feature) {
2951 if (!CallerFeatureMap.lookup(Key: Feature)) {
2952 MissingFeature = Feature.str();
2953 return false;
2954 }
2955 return true;
2956 }) &&
2957 !IsHipStdPar) {
2958 if (IsAlwaysInline)
2959 CGM.getDiags().Report(Loc, DiagID: diag::err_function_needs_feature)
2960 << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
2961 else if (IsFlatten)
2962 CGM.getDiags().Report(Loc, DiagID: diag::err_flatten_function_needs_feature)
2963 << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
2964 }
2965
2966 } else if (!FD->isMultiVersion() && FD->hasAttr<TargetAttr>()) {
2967 llvm::StringMap<bool> CalleeFeatureMap;
2968 CGM.getContext().getFunctionFeatureMap(FeatureMap&: CalleeFeatureMap, TargetDecl);
2969
2970 for (const auto &F : CalleeFeatureMap) {
2971 if (F.getValue() &&
2972 (!CallerFeatureMap.lookup(Key: F.getKey()) ||
2973 !CallerFeatureMap.find(Key: F.getKey())->getValue()) &&
2974 !IsHipStdPar) {
2975 if (IsAlwaysInline)
2976 CGM.getDiags().Report(Loc, DiagID: diag::err_function_needs_feature)
2977 << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();
2978 else if (IsFlatten)
2979 CGM.getDiags().Report(Loc, DiagID: diag::err_flatten_function_needs_feature)
2980 << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();
2981 }
2982 }
2983 }
2984}
2985
2986void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
2987 if (!CGM.getCodeGenOpts().SanitizeStats)
2988 return;
2989
2990 llvm::IRBuilder<> IRB(Builder.GetInsertBlock(), Builder.GetInsertPoint());
2991 IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation());
2992 CGM.getSanStats().create(B&: IRB, SK: SSK);
2993}
2994
2995void CodeGenFunction::EmitKCFIOperandBundle(
2996 const CGCallee &Callee, SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
2997 const CGCalleeInfo &CI = Callee.getAbstractInfo();
2998 const FunctionProtoType *FP = CI.getCalleeFunctionProtoType();
2999 if (!FP)
3000 return;
3001
3002 StringRef Salt;
3003 if (const auto &Info = FP->getExtraAttributeInfo())
3004 Salt = Info.CFISalt;
3005
3006 Bundles.emplace_back(Args: "kcfi", Args: CGM.CreateKCFITypeId(T: FP->desugar(), Salt));
3007}
3008
3009llvm::Value *
3010CodeGenFunction::FormAArch64ResolverCondition(const FMVResolverOption &RO) {
3011 return RO.Features.empty() ? nullptr : EmitAArch64CpuSupports(FeatureStrs: RO.Features);
3012}
3013
3014llvm::Value *
3015CodeGenFunction::FormX86ResolverCondition(const FMVResolverOption &RO) {
3016 llvm::Value *Condition = nullptr;
3017
3018 if (RO.Architecture) {
3019 StringRef Arch = *RO.Architecture;
3020 // If arch= specifies an x86-64 micro-architecture level, test the feature
3021 // with __builtin_cpu_supports, otherwise use __builtin_cpu_is.
3022 if (Arch.starts_with(Prefix: "x86-64"))
3023 Condition = EmitX86CpuSupports(FeatureStrs: {Arch});
3024 else
3025 Condition = EmitX86CpuIs(CPUStr: Arch);
3026 }
3027
3028 if (!RO.Features.empty()) {
3029 llvm::Value *FeatureCond = EmitX86CpuSupports(FeatureStrs: RO.Features);
3030 Condition =
3031 Condition ? Builder.CreateAnd(LHS: Condition, RHS: FeatureCond) : FeatureCond;
3032 }
3033 return Condition;
3034}
3035
3036static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
3037 llvm::Function *Resolver,
3038 CGBuilderTy &Builder,
3039 llvm::Function *FuncToReturn,
3040 bool SupportsIFunc) {
3041 if (SupportsIFunc) {
3042 Builder.CreateRet(V: FuncToReturn);
3043 return;
3044 }
3045
3046 llvm::SmallVector<llvm::Value *, 10> Args(
3047 llvm::make_pointer_range(Range: Resolver->args()));
3048
3049 llvm::CallInst *Result = Builder.CreateCall(Callee: FuncToReturn, Args);
3050 Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
3051
3052 if (Resolver->getReturnType()->isVoidTy())
3053 Builder.CreateRetVoid();
3054 else
3055 Builder.CreateRet(V: Result);
3056}
3057
3058void CodeGenFunction::EmitMultiVersionResolver(
3059 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3060
3061 llvm::Triple::ArchType ArchType =
3062 getContext().getTargetInfo().getTriple().getArch();
3063
3064 switch (ArchType) {
3065 case llvm::Triple::x86:
3066 case llvm::Triple::x86_64:
3067 EmitX86MultiVersionResolver(Resolver, Options);
3068 return;
3069 case llvm::Triple::aarch64:
3070 EmitAArch64MultiVersionResolver(Resolver, Options);
3071 return;
3072 case llvm::Triple::riscv32:
3073 case llvm::Triple::riscv64:
3074 case llvm::Triple::riscv32be:
3075 case llvm::Triple::riscv64be:
3076 EmitRISCVMultiVersionResolver(Resolver, Options);
3077 return;
3078
3079 default:
3080 assert(false && "Only implemented for x86, AArch64 and RISC-V targets");
3081 }
3082}
3083
3084void CodeGenFunction::EmitRISCVMultiVersionResolver(
3085 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3086
3087 if (getContext().getTargetInfo().getTriple().getOS() !=
3088 llvm::Triple::OSType::Linux) {
3089 CGM.getDiags().Report(DiagID: diag::err_os_unsupport_riscv_fmv);
3090 return;
3091 }
3092
3093 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3094 Builder.SetInsertPoint(CurBlock);
3095 EmitRISCVCpuInit();
3096
3097 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3098 bool HasDefault = false;
3099 unsigned DefaultIndex = 0;
3100
3101 // Check the each candidate function.
3102 for (unsigned Index = 0; Index < Options.size(); Index++) {
3103
3104 if (Options[Index].Features.empty()) {
3105 HasDefault = true;
3106 DefaultIndex = Index;
3107 continue;
3108 }
3109
3110 Builder.SetInsertPoint(CurBlock);
3111
3112 // FeaturesCondition: The bitmask of the required extension has been
3113 // enabled by the runtime object.
3114 // (__riscv_feature_bits.features[i] & REQUIRED_BITMASK) ==
3115 // REQUIRED_BITMASK
3116 //
3117 // When condition is met, return this version of the function.
3118 // Otherwise, try the next version.
3119 //
3120 // if (FeaturesConditionVersion1)
3121 // return Version1;
3122 // else if (FeaturesConditionVersion2)
3123 // return Version2;
3124 // else if (FeaturesConditionVersion3)
3125 // return Version3;
3126 // ...
3127 // else
3128 // return DefaultVersion;
3129
3130 // TODO: Add a condition to check the length before accessing elements.
3131 // Without checking the length first, we may access an incorrect memory
3132 // address when using different versions.
3133 llvm::SmallVector<StringRef, 8> CurrTargetAttrFeats;
3134 llvm::SmallVector<std::string, 8> TargetAttrFeats;
3135
3136 for (StringRef Feat : Options[Index].Features) {
3137 std::vector<std::string> FeatStr =
3138 getContext().getTargetInfo().parseTargetAttr(Str: Feat).Features;
3139
3140 assert(FeatStr.size() == 1 && "Feature string not delimited");
3141
3142 std::string &CurrFeat = FeatStr.front();
3143 if (CurrFeat[0] == '+')
3144 TargetAttrFeats.push_back(Elt: CurrFeat.substr(pos: 1));
3145 }
3146
3147 if (TargetAttrFeats.empty())
3148 continue;
3149
3150 for (std::string &Feat : TargetAttrFeats)
3151 CurrTargetAttrFeats.push_back(Elt: Feat);
3152
3153 Builder.SetInsertPoint(CurBlock);
3154 llvm::Value *FeatsCondition = EmitRISCVCpuSupports(FeaturesStrs: CurrTargetAttrFeats);
3155
3156 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3157 CGBuilderTy RetBuilder(CGM, RetBlock);
3158 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder,
3159 FuncToReturn: Options[Index].Function, SupportsIFunc);
3160 llvm::BasicBlock *ElseBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3161
3162 Builder.SetInsertPoint(CurBlock);
3163 Builder.CreateCondBr(Cond: FeatsCondition, True: RetBlock, False: ElseBlock);
3164
3165 CurBlock = ElseBlock;
3166 }
3167
3168 // Finally, emit the default one.
3169 if (HasDefault) {
3170 Builder.SetInsertPoint(CurBlock);
3171 CreateMultiVersionResolverReturn(
3172 CGM, Resolver, Builder, FuncToReturn: Options[DefaultIndex].Function, SupportsIFunc);
3173 return;
3174 }
3175
3176 // If no generic/default, emit an unreachable.
3177 Builder.SetInsertPoint(CurBlock);
3178 llvm::CallInst *TrapCall = EmitTrapCall(IntrID: llvm::Intrinsic::trap);
3179 TrapCall->setDoesNotReturn();
3180 TrapCall->setDoesNotThrow();
3181 Builder.CreateUnreachable();
3182 Builder.ClearInsertionPoint();
3183}
3184
3185void CodeGenFunction::EmitAArch64MultiVersionResolver(
3186 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3187 assert(!Options.empty() && "No multiversion resolver options found");
3188 assert(Options.back().Features.size() == 0 && "Default case must be last");
3189 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3190 assert(SupportsIFunc &&
3191 "Multiversion resolver requires target IFUNC support");
3192 bool AArch64CpuInitialized = false;
3193 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3194
3195 for (const FMVResolverOption &RO : Options) {
3196 Builder.SetInsertPoint(CurBlock);
3197 llvm::Value *Condition = FormAArch64ResolverCondition(RO);
3198
3199 // The 'default' or 'all features enabled' case.
3200 if (!Condition) {
3201 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, FuncToReturn: RO.Function,
3202 SupportsIFunc);
3203 return;
3204 }
3205
3206 if (!AArch64CpuInitialized) {
3207 Builder.SetInsertPoint(TheBB: CurBlock, IP: CurBlock->begin());
3208 EmitAArch64CpuInit();
3209 AArch64CpuInitialized = true;
3210 Builder.SetInsertPoint(CurBlock);
3211 }
3212
3213 // Skip unreachable versions.
3214 if (RO.Function == nullptr)
3215 continue;
3216
3217 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3218 CGBuilderTy RetBuilder(CGM, RetBlock);
3219 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder, FuncToReturn: RO.Function,
3220 SupportsIFunc);
3221 CurBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3222 Builder.CreateCondBr(Cond: Condition, True: RetBlock, False: CurBlock);
3223 }
3224
3225 // If no default, emit an unreachable.
3226 Builder.SetInsertPoint(CurBlock);
3227 llvm::CallInst *TrapCall = EmitTrapCall(IntrID: llvm::Intrinsic::trap);
3228 TrapCall->setDoesNotReturn();
3229 TrapCall->setDoesNotThrow();
3230 Builder.CreateUnreachable();
3231 Builder.ClearInsertionPoint();
3232}
3233
3234void CodeGenFunction::EmitX86MultiVersionResolver(
3235 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3236
3237 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3238
3239 // Main function's basic block.
3240 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3241 Builder.SetInsertPoint(CurBlock);
3242 EmitX86CpuInit();
3243
3244 for (const FMVResolverOption &RO : Options) {
3245 Builder.SetInsertPoint(CurBlock);
3246 llvm::Value *Condition = FormX86ResolverCondition(RO);
3247
3248 // The 'default' or 'generic' case.
3249 if (!Condition) {
3250 assert(&RO == Options.end() - 1 &&
3251 "Default or Generic case must be last");
3252 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, FuncToReturn: RO.Function,
3253 SupportsIFunc);
3254 return;
3255 }
3256
3257 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3258 CGBuilderTy RetBuilder(CGM, RetBlock);
3259 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder, FuncToReturn: RO.Function,
3260 SupportsIFunc);
3261 CurBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3262 Builder.CreateCondBr(Cond: Condition, True: RetBlock, False: CurBlock);
3263 }
3264
3265 // If no generic/default, emit an unreachable.
3266 Builder.SetInsertPoint(CurBlock);
3267 llvm::CallInst *TrapCall = EmitTrapCall(IntrID: llvm::Intrinsic::trap);
3268 TrapCall->setDoesNotReturn();
3269 TrapCall->setDoesNotThrow();
3270 Builder.CreateUnreachable();
3271 Builder.ClearInsertionPoint();
3272}
3273
3274// Loc - where the diagnostic will point, where in the source code this
3275// alignment has failed.
3276// SecondaryLoc - if present (will be present if sufficiently different from
3277// Loc), the diagnostic will additionally point a "Note:" to this location.
3278// It should be the location where the __attribute__((assume_aligned))
3279// was written e.g.
3280void CodeGenFunction::emitAlignmentAssumptionCheck(
3281 llvm::Value *Ptr, QualType Ty, SourceLocation Loc,
3282 SourceLocation SecondaryLoc, llvm::Value *Alignment,
3283 llvm::Value *OffsetValue, llvm::Value *TheCheck,
3284 llvm::Instruction *Assumption) {
3285 assert(isa_and_nonnull<llvm::CallInst>(Assumption) &&
3286 cast<llvm::CallInst>(Assumption)->getCalledOperand() ==
3287 llvm::Intrinsic::getOrInsertDeclaration(
3288 Builder.GetInsertBlock()->getParent()->getParent(),
3289 llvm::Intrinsic::assume) &&
3290 "Assumption should be a call to llvm.assume().");
3291 assert(&(Builder.GetInsertBlock()->back()) == Assumption &&
3292 "Assumption should be the last instruction of the basic block, "
3293 "since the basic block is still being generated.");
3294
3295 if (!SanOpts.has(K: SanitizerKind::Alignment))
3296 return;
3297
3298 // Don't check pointers to volatile data. The behavior here is implementation-
3299 // defined.
3300 if (Ty->getPointeeType().isVolatileQualified())
3301 return;
3302
3303 // We need to temorairly remove the assumption so we can insert the
3304 // sanitizer check before it, else the check will be dropped by optimizations.
3305 Assumption->removeFromParent();
3306
3307 {
3308 auto CheckOrdinal = SanitizerKind::SO_Alignment;
3309 auto CheckHandler = SanitizerHandler::AlignmentAssumption;
3310 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
3311
3312 if (!OffsetValue)
3313 OffsetValue = Builder.getInt1(V: false); // no offset.
3314
3315 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc),
3316 EmitCheckSourceLocation(Loc: SecondaryLoc),
3317 EmitCheckTypeDescriptor(T: Ty)};
3318 llvm::Value *DynamicData[] = {Ptr, Alignment, OffsetValue};
3319 EmitCheck(Checked: {std::make_pair(x&: TheCheck, y&: CheckOrdinal)}, Check: CheckHandler,
3320 StaticArgs: StaticData, DynamicArgs: DynamicData);
3321 }
3322
3323 // We are now in the (new, empty) "cont" basic block.
3324 // Reintroduce the assumption.
3325 Builder.Insert(I: Assumption);
3326 // FIXME: Assumption still has it's original basic block as it's Parent.
3327}
3328
3329llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
3330 if (CGDebugInfo *DI = getDebugInfo())
3331 return DI->SourceLocToDebugLoc(Loc: Location);
3332
3333 return llvm::DebugLoc();
3334}
3335
3336llvm::Value *
3337CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
3338 Stmt::Likelihood LH) {
3339 switch (LH) {
3340 case Stmt::LH_None:
3341 return Cond;
3342 case Stmt::LH_Likely:
3343 case Stmt::LH_Unlikely:
3344 // Don't generate llvm.expect on -O0 as the backend won't use it for
3345 // anything.
3346 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3347 return Cond;
3348 llvm::Type *CondTy = Cond->getType();
3349 assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");
3350 llvm::Function *FnExpect =
3351 CGM.getIntrinsic(IID: llvm::Intrinsic::expect, Tys: CondTy);
3352 llvm::Value *ExpectedValueOfCond =
3353 llvm::ConstantInt::getBool(Ty: CondTy, V: LH == Stmt::LH_Likely);
3354 return Builder.CreateCall(Callee: FnExpect, Args: {Cond, ExpectedValueOfCond},
3355 Name: Cond->getName() + ".expval");
3356 }
3357 llvm_unreachable("Unknown Likelihood");
3358}
3359
3360llvm::Value *CodeGenFunction::emitBoolVecConversion(llvm::Value *SrcVec,
3361 unsigned NumElementsDst,
3362 const llvm::Twine &Name) {
3363 auto *SrcTy = cast<llvm::FixedVectorType>(Val: SrcVec->getType());
3364 unsigned NumElementsSrc = SrcTy->getNumElements();
3365 if (NumElementsSrc == NumElementsDst)
3366 return SrcVec;
3367
3368 std::vector<int> ShuffleMask(NumElementsDst, -1);
3369 for (unsigned MaskIdx = 0;
3370 MaskIdx < std::min<>(a: NumElementsDst, b: NumElementsSrc); ++MaskIdx)
3371 ShuffleMask[MaskIdx] = MaskIdx;
3372
3373 return Builder.CreateShuffleVector(V: SrcVec, Mask: ShuffleMask, Name);
3374}
3375
3376void CodeGenFunction::EmitPointerAuthOperandBundle(
3377 const CGPointerAuthInfo &PointerAuth,
3378 SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
3379 if (!PointerAuth.isSigned())
3380 return;
3381
3382 auto *Key = Builder.getInt32(C: PointerAuth.getKey());
3383
3384 llvm::Value *Discriminator = PointerAuth.getDiscriminator();
3385 if (!Discriminator)
3386 Discriminator = Builder.getSize(N: 0);
3387
3388 llvm::Value *Args[] = {Key, Discriminator};
3389 Bundles.emplace_back(Args: "ptrauth", Args);
3390}
3391
3392static llvm::Value *EmitPointerAuthCommon(CodeGenFunction &CGF,
3393 const CGPointerAuthInfo &PointerAuth,
3394 llvm::Value *Pointer,
3395 unsigned IntrinsicID) {
3396 if (!PointerAuth)
3397 return Pointer;
3398
3399 auto Key = CGF.Builder.getInt32(C: PointerAuth.getKey());
3400
3401 llvm::Value *Discriminator = PointerAuth.getDiscriminator();
3402 if (!Discriminator) {
3403 Discriminator = CGF.Builder.getSize(N: 0);
3404 }
3405
3406 // Convert the pointer to intptr_t before signing it.
3407 auto OrigType = Pointer->getType();
3408 Pointer = CGF.Builder.CreatePtrToInt(V: Pointer, DestTy: CGF.IntPtrTy);
3409
3410 // call i64 @llvm.ptrauth.sign.i64(i64 %pointer, i32 %key, i64 %discriminator)
3411 auto Intrinsic = CGF.CGM.getIntrinsic(IID: IntrinsicID);
3412 Pointer = CGF.EmitRuntimeCall(callee: Intrinsic, args: {Pointer, Key, Discriminator});
3413
3414 // Convert back to the original type.
3415 Pointer = CGF.Builder.CreateIntToPtr(V: Pointer, DestTy: OrigType);
3416 return Pointer;
3417}
3418
3419llvm::Value *
3420CodeGenFunction::EmitPointerAuthSign(const CGPointerAuthInfo &PointerAuth,
3421 llvm::Value *Pointer) {
3422 if (!PointerAuth.shouldSign())
3423 return Pointer;
3424 return EmitPointerAuthCommon(CGF&: *this, PointerAuth, Pointer,
3425 IntrinsicID: llvm::Intrinsic::ptrauth_sign);
3426}
3427
3428static llvm::Value *EmitStrip(CodeGenFunction &CGF,
3429 const CGPointerAuthInfo &PointerAuth,
3430 llvm::Value *Pointer) {
3431 auto StripIntrinsic = CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::ptrauth_strip);
3432
3433 auto Key = CGF.Builder.getInt32(C: PointerAuth.getKey());
3434 // Convert the pointer to intptr_t before signing it.
3435 auto OrigType = Pointer->getType();
3436 Pointer = CGF.EmitRuntimeCall(
3437 callee: StripIntrinsic, args: {CGF.Builder.CreatePtrToInt(V: Pointer, DestTy: CGF.IntPtrTy), Key});
3438 return CGF.Builder.CreateIntToPtr(V: Pointer, DestTy: OrigType);
3439}
3440
3441llvm::Value *
3442CodeGenFunction::EmitPointerAuthAuth(const CGPointerAuthInfo &PointerAuth,
3443 llvm::Value *Pointer) {
3444 if (PointerAuth.shouldStrip()) {
3445 return EmitStrip(CGF&: *this, PointerAuth, Pointer);
3446 }
3447 if (!PointerAuth.shouldAuth()) {
3448 return Pointer;
3449 }
3450
3451 return EmitPointerAuthCommon(CGF&: *this, PointerAuth, Pointer,
3452 IntrinsicID: llvm::Intrinsic::ptrauth_auth);
3453}
3454
3455void CodeGenFunction::addInstToCurrentSourceAtom(
3456 llvm::Instruction *KeyInstruction, llvm::Value *Backup) {
3457 if (CGDebugInfo *DI = getDebugInfo())
3458 DI->addInstToCurrentSourceAtom(KeyInstruction, Backup);
3459}
3460
3461void CodeGenFunction::addInstToSpecificSourceAtom(
3462 llvm::Instruction *KeyInstruction, llvm::Value *Backup, uint64_t Atom) {
3463 if (CGDebugInfo *DI = getDebugInfo())
3464 DI->addInstToSpecificSourceAtom(KeyInstruction, Backup, Atom);
3465}
3466
3467void CodeGenFunction::addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,
3468 llvm::Value *Backup) {
3469 if (CGDebugInfo *DI = getDebugInfo()) {
3470 ApplyAtomGroup Grp(getDebugInfo());
3471 DI->addInstToCurrentSourceAtom(KeyInstruction, Backup);
3472 }
3473}
3474
3475void CodeGenFunction::emitPFPPostCopyUpdates(Address DestPtr, Address SrcPtr,
3476 QualType Ty) {
3477 for (auto &Field : getContext().findPFPFields(Ty)) {
3478 if (getContext().arePFPFieldsTriviallyCopyable(RD: Field.Field->getParent()))
3479 continue;
3480 auto DestFieldPtr = EmitAddressOfPFPField(RecordPtr: DestPtr, Field);
3481 auto SrcFieldPtr = EmitAddressOfPFPField(RecordPtr: SrcPtr, Field);
3482 Builder.CreateStore(Val: Builder.CreateLoad(Addr: SrcFieldPtr), Addr: DestFieldPtr);
3483 }
3484}
3485