1//===--- CodeGenFunction.cpp - Emit LLVM Code from ASTs for a Function ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This coordinates the per-function state used while generating code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CodeGenFunction.h"
14#include "CGBlocks.h"
15#include "CGCUDARuntime.h"
16#include "CGCXXABI.h"
17#include "CGCleanup.h"
18#include "CGDebugInfo.h"
19#include "CGHLSLRuntime.h"
20#include "CGOpenMPRuntime.h"
21#include "CodeGenModule.h"
22#include "CodeGenPGO.h"
23#include "TargetInfo.h"
24#include "clang/AST/ASTContext.h"
25#include "clang/AST/ASTLambda.h"
26#include "clang/AST/Attr.h"
27#include "clang/AST/Decl.h"
28#include "clang/AST/DeclCXX.h"
29#include "clang/AST/Expr.h"
30#include "clang/AST/StmtCXX.h"
31#include "clang/AST/StmtObjC.h"
32#include "clang/Basic/Builtins.h"
33#include "clang/Basic/CodeGenOptions.h"
34#include "clang/Basic/TargetBuiltins.h"
35#include "clang/Basic/TargetInfo.h"
36#include "clang/CodeGen/CGFunctionInfo.h"
37#include "clang/Frontend/FrontendDiagnostic.h"
38#include "llvm/ADT/ArrayRef.h"
39#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
40#include "llvm/IR/DataLayout.h"
41#include "llvm/IR/Dominators.h"
42#include "llvm/IR/FPEnv.h"
43#include "llvm/IR/IntrinsicInst.h"
44#include "llvm/IR/Intrinsics.h"
45#include "llvm/IR/MDBuilder.h"
46#include "llvm/IR/Operator.h"
47#include "llvm/Support/CRC.h"
48#include "llvm/Support/xxhash.h"
49#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
50#include "llvm/Transforms/Utils/PromoteMemToReg.h"
51#include <optional>
52
53using namespace clang;
54using namespace CodeGen;
55
56namespace llvm {
57extern cl::opt<bool> EnableSingleByteCoverage;
58} // namespace llvm
59
60/// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time
61/// markers.
62static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts,
63 const LangOptions &LangOpts) {
64 if (CGOpts.DisableLifetimeMarkers)
65 return false;
66
67 // Sanitizers may use markers.
68 if (CGOpts.SanitizeAddressUseAfterScope ||
69 LangOpts.Sanitize.has(K: SanitizerKind::HWAddress) ||
70 LangOpts.Sanitize.has(K: SanitizerKind::Memory))
71 return true;
72
73 // For now, only in optimized builds.
74 return CGOpts.OptimizationLevel != 0;
75}
76
77CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
78 : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
79 Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(),
80 CGBuilderInserterTy(this)),
81 SanOpts(CGM.getLangOpts().Sanitize), CurFPFeatures(CGM.getLangOpts()),
82 DebugInfo(CGM.getModuleDebugInfo()), PGO(cgm),
83 ShouldEmitLifetimeMarkers(
84 shouldEmitLifetimeMarkers(CGOpts: CGM.getCodeGenOpts(), LangOpts: CGM.getLangOpts())) {
85 if (!suppressNewContext)
86 CGM.getCXXABI().getMangleContext().startNewFunction();
87 EHStack.setCGF(this);
88
89 SetFastMathFlags(CurFPFeatures);
90}
91
92CodeGenFunction::~CodeGenFunction() {
93 assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup");
94 assert(DeferredDeactivationCleanupStack.empty() &&
95 "missed to deactivate a cleanup");
96
97 if (getLangOpts().OpenMP && CurFn)
98 CGM.getOpenMPRuntime().functionFinished(CGF&: *this);
99
100 // If we have an OpenMPIRBuilder we want to finalize functions (incl.
101 // outlining etc) at some point. Doing it once the function codegen is done
102 // seems to be a reasonable spot. We do it here, as opposed to the deletion
103 // time of the CodeGenModule, because we have to ensure the IR has not yet
104 // been "emitted" to the outside, thus, modifications are still sensible.
105 if (CGM.getLangOpts().OpenMPIRBuilder && CurFn)
106 CGM.getOpenMPRuntime().getOMPBuilder().finalize(Fn: CurFn);
107}
108
109// Map the LangOption for exception behavior into
110// the corresponding enum in the IR.
111llvm::fp::ExceptionBehavior
112clang::ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind) {
113
114 switch (Kind) {
115 case LangOptions::FPE_Ignore: return llvm::fp::ebIgnore;
116 case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap;
117 case LangOptions::FPE_Strict: return llvm::fp::ebStrict;
118 default:
119 llvm_unreachable("Unsupported FP Exception Behavior");
120 }
121}
122
123void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) {
124 llvm::FastMathFlags FMF;
125 FMF.setAllowReassoc(FPFeatures.getAllowFPReassociate());
126 FMF.setNoNaNs(FPFeatures.getNoHonorNaNs());
127 FMF.setNoInfs(FPFeatures.getNoHonorInfs());
128 FMF.setNoSignedZeros(FPFeatures.getNoSignedZero());
129 FMF.setAllowReciprocal(FPFeatures.getAllowReciprocal());
130 FMF.setApproxFunc(FPFeatures.getAllowApproxFunc());
131 FMF.setAllowContract(FPFeatures.allowFPContractAcrossStatement());
132 Builder.setFastMathFlags(FMF);
133}
134
135CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
136 const Expr *E)
137 : CGF(CGF) {
138 ConstructorHelper(FPFeatures: E->getFPFeaturesInEffect(LO: CGF.getLangOpts()));
139}
140
141CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
142 FPOptions FPFeatures)
143 : CGF(CGF) {
144 ConstructorHelper(FPFeatures);
145}
146
147void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) {
148 OldFPFeatures = CGF.CurFPFeatures;
149 CGF.CurFPFeatures = FPFeatures;
150
151 OldExcept = CGF.Builder.getDefaultConstrainedExcept();
152 OldRounding = CGF.Builder.getDefaultConstrainedRounding();
153
154 if (OldFPFeatures == FPFeatures)
155 return;
156
157 FMFGuard.emplace(args&: CGF.Builder);
158
159 llvm::RoundingMode NewRoundingBehavior = FPFeatures.getRoundingMode();
160 CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);
161 auto NewExceptionBehavior =
162 ToConstrainedExceptMD(Kind: static_cast<LangOptions::FPExceptionModeKind>(
163 FPFeatures.getExceptionMode()));
164 CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);
165
166 CGF.SetFastMathFlags(FPFeatures);
167
168 assert((CGF.CurFuncDecl == nullptr || CGF.Builder.getIsFPConstrained() ||
169 isa<CXXConstructorDecl>(CGF.CurFuncDecl) ||
170 isa<CXXDestructorDecl>(CGF.CurFuncDecl) ||
171 (NewExceptionBehavior == llvm::fp::ebIgnore &&
172 NewRoundingBehavior == llvm::RoundingMode::NearestTiesToEven)) &&
173 "FPConstrained should be enabled on entire function");
174
175 auto mergeFnAttrValue = [&](StringRef Name, bool Value) {
176 auto OldValue =
177 CGF.CurFn->getFnAttribute(Kind: Name).getValueAsBool();
178 auto NewValue = OldValue & Value;
179 if (OldValue != NewValue)
180 CGF.CurFn->addFnAttr(Kind: Name, Val: llvm::toStringRef(B: NewValue));
181 };
182 mergeFnAttrValue("no-infs-fp-math", FPFeatures.getNoHonorInfs());
183 mergeFnAttrValue("no-nans-fp-math", FPFeatures.getNoHonorNaNs());
184 mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.getNoSignedZero());
185 mergeFnAttrValue(
186 "unsafe-fp-math",
187 FPFeatures.getAllowFPReassociate() && FPFeatures.getAllowReciprocal() &&
188 FPFeatures.getAllowApproxFunc() && FPFeatures.getNoSignedZero() &&
189 FPFeatures.allowFPContractAcrossStatement());
190}
191
192CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
193 CGF.CurFPFeatures = OldFPFeatures;
194 CGF.Builder.setDefaultConstrainedExcept(OldExcept);
195 CGF.Builder.setDefaultConstrainedRounding(OldRounding);
196}
197
198static LValue
199makeNaturalAlignAddrLValue(llvm::Value *V, QualType T, bool ForPointeeType,
200 bool MightBeSigned, CodeGenFunction &CGF,
201 KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
202 LValueBaseInfo BaseInfo;
203 TBAAAccessInfo TBAAInfo;
204 CharUnits Alignment =
205 CGF.CGM.getNaturalTypeAlignment(T, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo, forPointeeType: ForPointeeType);
206 Address Addr =
207 MightBeSigned
208 ? CGF.makeNaturalAddressForPointer(Ptr: V, T, Alignment, ForPointeeType: false, BaseInfo: nullptr,
209 TBAAInfo: nullptr, IsKnownNonNull)
210 : Address(V, CGF.ConvertTypeForMem(T), Alignment, IsKnownNonNull);
211 return CGF.MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
212}
213
214LValue
215CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
216 KnownNonNull_t IsKnownNonNull) {
217 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,
218 /*MightBeSigned*/ true, CGF&: *this,
219 IsKnownNonNull);
220}
221
222LValue
223CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
224 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,
225 /*MightBeSigned*/ true, CGF&: *this);
226}
227
228LValue CodeGenFunction::MakeNaturalAlignRawAddrLValue(llvm::Value *V,
229 QualType T) {
230 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,
231 /*MightBeSigned*/ false, CGF&: *this);
232}
233
234LValue CodeGenFunction::MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V,
235 QualType T) {
236 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,
237 /*MightBeSigned*/ false, CGF&: *this);
238}
239
240llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
241 return CGM.getTypes().ConvertTypeForMem(T);
242}
243
244llvm::Type *CodeGenFunction::ConvertType(QualType T) {
245 return CGM.getTypes().ConvertType(T);
246}
247
248llvm::Type *CodeGenFunction::convertTypeForLoadStore(QualType ASTTy,
249 llvm::Type *LLVMTy) {
250 return CGM.getTypes().convertTypeForLoadStore(T: ASTTy, LLVMTy);
251}
252
253TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
254 type = type.getCanonicalType();
255 while (true) {
256 switch (type->getTypeClass()) {
257#define TYPE(name, parent)
258#define ABSTRACT_TYPE(name, parent)
259#define NON_CANONICAL_TYPE(name, parent) case Type::name:
260#define DEPENDENT_TYPE(name, parent) case Type::name:
261#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
262#include "clang/AST/TypeNodes.inc"
263 llvm_unreachable("non-canonical or dependent type in IR-generation");
264
265 case Type::Auto:
266 case Type::DeducedTemplateSpecialization:
267 llvm_unreachable("undeduced type in IR-generation");
268
269 // Various scalar types.
270 case Type::Builtin:
271 case Type::Pointer:
272 case Type::BlockPointer:
273 case Type::LValueReference:
274 case Type::RValueReference:
275 case Type::MemberPointer:
276 case Type::Vector:
277 case Type::ExtVector:
278 case Type::ConstantMatrix:
279 case Type::FunctionProto:
280 case Type::FunctionNoProto:
281 case Type::Enum:
282 case Type::ObjCObjectPointer:
283 case Type::Pipe:
284 case Type::BitInt:
285 return TEK_Scalar;
286
287 // Complexes.
288 case Type::Complex:
289 return TEK_Complex;
290
291 // Arrays, records, and Objective-C objects.
292 case Type::ConstantArray:
293 case Type::IncompleteArray:
294 case Type::VariableArray:
295 case Type::Record:
296 case Type::ObjCObject:
297 case Type::ObjCInterface:
298 case Type::ArrayParameter:
299 return TEK_Aggregate;
300
301 // We operate on atomic values according to their underlying type.
302 case Type::Atomic:
303 type = cast<AtomicType>(Val&: type)->getValueType();
304 continue;
305 }
306 llvm_unreachable("unknown type kind!");
307 }
308}
309
310llvm::DebugLoc CodeGenFunction::EmitReturnBlock() {
311 // For cleanliness, we try to avoid emitting the return block for
312 // simple cases.
313 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
314
315 if (CurBB) {
316 assert(!CurBB->getTerminator() && "Unexpected terminated block.");
317
318 // We have a valid insert point, reuse it if it is empty or there are no
319 // explicit jumps to the return block.
320 if (CurBB->empty() || ReturnBlock.getBlock()->use_empty()) {
321 ReturnBlock.getBlock()->replaceAllUsesWith(V: CurBB);
322 delete ReturnBlock.getBlock();
323 ReturnBlock = JumpDest();
324 } else
325 EmitBlock(BB: ReturnBlock.getBlock());
326 return llvm::DebugLoc();
327 }
328
329 // Otherwise, if the return block is the target of a single direct
330 // branch then we can just put the code in that block instead. This
331 // cleans up functions which started with a unified return block.
332 if (ReturnBlock.getBlock()->hasOneUse()) {
333 llvm::BranchInst *BI =
334 dyn_cast<llvm::BranchInst>(Val: *ReturnBlock.getBlock()->user_begin());
335 if (BI && BI->isUnconditional() &&
336 BI->getSuccessor(i: 0) == ReturnBlock.getBlock()) {
337 // Record/return the DebugLoc of the simple 'return' expression to be used
338 // later by the actual 'ret' instruction.
339 llvm::DebugLoc Loc = BI->getDebugLoc();
340 Builder.SetInsertPoint(BI->getParent());
341 BI->eraseFromParent();
342 delete ReturnBlock.getBlock();
343 ReturnBlock = JumpDest();
344 return Loc;
345 }
346 }
347
348 // FIXME: We are at an unreachable point, there is no reason to emit the block
349 // unless it has uses. However, we still need a place to put the debug
350 // region.end for now.
351
352 EmitBlock(BB: ReturnBlock.getBlock());
353 return llvm::DebugLoc();
354}
355
356static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
357 if (!BB) return;
358 if (!BB->use_empty()) {
359 CGF.CurFn->insert(Position: CGF.CurFn->end(), BB);
360 return;
361 }
362 delete BB;
363}
364
365void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
366 assert(BreakContinueStack.empty() &&
367 "mismatched push/pop in break/continue stack!");
368 assert(LifetimeExtendedCleanupStack.empty() &&
369 "mismatched push/pop of cleanups in EHStack!");
370 assert(DeferredDeactivationCleanupStack.empty() &&
371 "mismatched activate/deactivate of cleanups!");
372
373 if (CGM.shouldEmitConvergenceTokens()) {
374 ConvergenceTokenStack.pop_back();
375 assert(ConvergenceTokenStack.empty() &&
376 "mismatched push/pop in convergence stack!");
377 }
378
379 bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
380 && NumSimpleReturnExprs == NumReturnExprs
381 && ReturnBlock.getBlock()->use_empty();
382 // Usually the return expression is evaluated before the cleanup
383 // code. If the function contains only a simple return statement,
384 // such as a constant, the location before the cleanup code becomes
385 // the last useful breakpoint in the function, because the simple
386 // return expression will be evaluated after the cleanup code. To be
387 // safe, set the debug location for cleanup code to the location of
388 // the return statement. Otherwise the cleanup code should be at the
389 // end of the function's lexical scope.
390 //
391 // If there are multiple branches to the return block, the branch
392 // instructions will get the location of the return statements and
393 // all will be fine.
394 if (CGDebugInfo *DI = getDebugInfo()) {
395 if (OnlySimpleReturnStmts)
396 DI->EmitLocation(Builder, Loc: LastStopPoint);
397 else
398 DI->EmitLocation(Builder, Loc: EndLoc);
399 }
400
401 // Pop any cleanups that might have been associated with the
402 // parameters. Do this in whatever block we're currently in; it's
403 // important to do this before we enter the return block or return
404 // edges will be *really* confused.
405 bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth;
406 bool HasOnlyLifetimeMarkers =
407 HasCleanups && EHStack.containsOnlyLifetimeMarkers(Old: PrologueCleanupDepth);
408 bool EmitRetDbgLoc = !HasCleanups || HasOnlyLifetimeMarkers;
409
410 std::optional<ApplyDebugLocation> OAL;
411 if (HasCleanups) {
412 // Make sure the line table doesn't jump back into the body for
413 // the ret after it's been at EndLoc.
414 if (CGDebugInfo *DI = getDebugInfo()) {
415 if (OnlySimpleReturnStmts)
416 DI->EmitLocation(Builder, Loc: EndLoc);
417 else
418 // We may not have a valid end location. Try to apply it anyway, and
419 // fall back to an artificial location if needed.
420 OAL = ApplyDebugLocation::CreateDefaultArtificial(CGF&: *this, TemporaryLocation: EndLoc);
421 }
422
423 PopCleanupBlocks(OldCleanupStackSize: PrologueCleanupDepth);
424 }
425
426 // Emit function epilog (to return).
427 llvm::DebugLoc Loc = EmitReturnBlock();
428
429 if (ShouldInstrumentFunction()) {
430 if (CGM.getCodeGenOpts().InstrumentFunctions)
431 CurFn->addFnAttr(Kind: "instrument-function-exit", Val: "__cyg_profile_func_exit");
432 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
433 CurFn->addFnAttr(Kind: "instrument-function-exit-inlined",
434 Val: "__cyg_profile_func_exit");
435 }
436
437 // Emit debug descriptor for function end.
438 if (CGDebugInfo *DI = getDebugInfo())
439 DI->EmitFunctionEnd(Builder, Fn: CurFn);
440
441 // Reset the debug location to that of the simple 'return' expression, if any
442 // rather than that of the end of the function's scope '}'.
443 ApplyDebugLocation AL(*this, Loc);
444 EmitFunctionEpilog(FI: *CurFnInfo, EmitRetDbgLoc, EndLoc);
445 EmitEndEHSpec(D: CurCodeDecl);
446
447 assert(EHStack.empty() &&
448 "did not remove all scopes from cleanup stack!");
449
450 // If someone did an indirect goto, emit the indirect goto block at the end of
451 // the function.
452 if (IndirectBranch) {
453 EmitBlock(BB: IndirectBranch->getParent());
454 Builder.ClearInsertionPoint();
455 }
456
457 // If some of our locals escaped, insert a call to llvm.localescape in the
458 // entry block.
459 if (!EscapedLocals.empty()) {
460 // Invert the map from local to index into a simple vector. There should be
461 // no holes.
462 SmallVector<llvm::Value *, 4> EscapeArgs;
463 EscapeArgs.resize(N: EscapedLocals.size());
464 for (auto &Pair : EscapedLocals)
465 EscapeArgs[Pair.second] = Pair.first;
466 llvm::Function *FrameEscapeFn = llvm::Intrinsic::getDeclaration(
467 M: &CGM.getModule(), id: llvm::Intrinsic::localescape);
468 CGBuilderTy(*this, AllocaInsertPt).CreateCall(Callee: FrameEscapeFn, Args: EscapeArgs);
469 }
470
471 // Remove the AllocaInsertPt instruction, which is just a convenience for us.
472 llvm::Instruction *Ptr = AllocaInsertPt;
473 AllocaInsertPt = nullptr;
474 Ptr->eraseFromParent();
475
476 // PostAllocaInsertPt, if created, was lazily created when it was required,
477 // remove it now since it was just created for our own convenience.
478 if (PostAllocaInsertPt) {
479 llvm::Instruction *PostPtr = PostAllocaInsertPt;
480 PostAllocaInsertPt = nullptr;
481 PostPtr->eraseFromParent();
482 }
483
484 // If someone took the address of a label but never did an indirect goto, we
485 // made a zero entry PHI node, which is illegal, zap it now.
486 if (IndirectBranch) {
487 llvm::PHINode *PN = cast<llvm::PHINode>(Val: IndirectBranch->getAddress());
488 if (PN->getNumIncomingValues() == 0) {
489 PN->replaceAllUsesWith(V: llvm::UndefValue::get(T: PN->getType()));
490 PN->eraseFromParent();
491 }
492 }
493
494 EmitIfUsed(CGF&: *this, BB: EHResumeBlock);
495 EmitIfUsed(CGF&: *this, BB: TerminateLandingPad);
496 EmitIfUsed(CGF&: *this, BB: TerminateHandler);
497 EmitIfUsed(CGF&: *this, BB: UnreachableBlock);
498
499 for (const auto &FuncletAndParent : TerminateFunclets)
500 EmitIfUsed(CGF&: *this, BB: FuncletAndParent.second);
501
502 if (CGM.getCodeGenOpts().EmitDeclMetadata)
503 EmitDeclMetadata();
504
505 for (const auto &R : DeferredReplacements) {
506 if (llvm::Value *Old = R.first) {
507 Old->replaceAllUsesWith(V: R.second);
508 cast<llvm::Instruction>(Val: Old)->eraseFromParent();
509 }
510 }
511 DeferredReplacements.clear();
512
513 // Eliminate CleanupDestSlot alloca by replacing it with SSA values and
514 // PHIs if the current function is a coroutine. We don't do it for all
515 // functions as it may result in slight increase in numbers of instructions
516 // if compiled with no optimizations. We do it for coroutine as the lifetime
517 // of CleanupDestSlot alloca make correct coroutine frame building very
518 // difficult.
519 if (NormalCleanupDest.isValid() && isCoroutine()) {
520 llvm::DominatorTree DT(*CurFn);
521 llvm::PromoteMemToReg(
522 Allocas: cast<llvm::AllocaInst>(Val: NormalCleanupDest.getPointer()), DT);
523 NormalCleanupDest = Address::invalid();
524 }
525
526 // Scan function arguments for vector width.
527 for (llvm::Argument &A : CurFn->args())
528 if (auto *VT = dyn_cast<llvm::VectorType>(Val: A.getType()))
529 LargestVectorWidth =
530 std::max(a: (uint64_t)LargestVectorWidth,
531 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
532
533 // Update vector width based on return type.
534 if (auto *VT = dyn_cast<llvm::VectorType>(Val: CurFn->getReturnType()))
535 LargestVectorWidth =
536 std::max(a: (uint64_t)LargestVectorWidth,
537 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
538
539 if (CurFnInfo->getMaxVectorWidth() > LargestVectorWidth)
540 LargestVectorWidth = CurFnInfo->getMaxVectorWidth();
541
542 // Add the min-legal-vector-width attribute. This contains the max width from:
543 // 1. min-vector-width attribute used in the source program.
544 // 2. Any builtins used that have a vector width specified.
545 // 3. Values passed in and out of inline assembly.
546 // 4. Width of vector arguments and return types for this function.
547 // 5. Width of vector arguments and return types for functions called by this
548 // function.
549 if (getContext().getTargetInfo().getTriple().isX86())
550 CurFn->addFnAttr(Kind: "min-legal-vector-width",
551 Val: llvm::utostr(X: LargestVectorWidth));
552
553 // Add vscale_range attribute if appropriate.
554 std::optional<std::pair<unsigned, unsigned>> VScaleRange =
555 getContext().getTargetInfo().getVScaleRange(LangOpts: getLangOpts());
556 if (VScaleRange) {
557 CurFn->addFnAttr(Attr: llvm::Attribute::getWithVScaleRangeArgs(
558 Context&: getLLVMContext(), MinValue: VScaleRange->first, MaxValue: VScaleRange->second));
559 }
560
561 // If we generated an unreachable return block, delete it now.
562 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()) {
563 Builder.ClearInsertionPoint();
564 ReturnBlock.getBlock()->eraseFromParent();
565 }
566 if (ReturnValue.isValid()) {
567 auto *RetAlloca =
568 dyn_cast<llvm::AllocaInst>(Val: ReturnValue.emitRawPointer(CGF&: *this));
569 if (RetAlloca && RetAlloca->use_empty()) {
570 RetAlloca->eraseFromParent();
571 ReturnValue = Address::invalid();
572 }
573 }
574}
575
576/// ShouldInstrumentFunction - Return true if the current function should be
577/// instrumented with __cyg_profile_func_* calls
578bool CodeGenFunction::ShouldInstrumentFunction() {
579 if (!CGM.getCodeGenOpts().InstrumentFunctions &&
580 !CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining &&
581 !CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
582 return false;
583 if (!CurFuncDecl || CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>())
584 return false;
585 return true;
586}
587
588bool CodeGenFunction::ShouldSkipSanitizerInstrumentation() {
589 if (!CurFuncDecl)
590 return false;
591 return CurFuncDecl->hasAttr<DisableSanitizerInstrumentationAttr>();
592}
593
594/// ShouldXRayInstrument - Return true if the current function should be
595/// instrumented with XRay nop sleds.
596bool CodeGenFunction::ShouldXRayInstrumentFunction() const {
597 return CGM.getCodeGenOpts().XRayInstrumentFunctions;
598}
599
600/// AlwaysEmitXRayCustomEvents - Return true if we should emit IR for calls to
601/// the __xray_customevent(...) builtin calls, when doing XRay instrumentation.
602bool CodeGenFunction::AlwaysEmitXRayCustomEvents() const {
603 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
604 (CGM.getCodeGenOpts().XRayAlwaysEmitCustomEvents ||
605 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
606 XRayInstrKind::Custom);
607}
608
609bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const {
610 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
611 (CGM.getCodeGenOpts().XRayAlwaysEmitTypedEvents ||
612 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
613 XRayInstrKind::Typed);
614}
615
616llvm::ConstantInt *
617CodeGenFunction::getUBSanFunctionTypeHash(QualType Ty) const {
618 // Remove any (C++17) exception specifications, to allow calling e.g. a
619 // noexcept function through a non-noexcept pointer.
620 if (!Ty->isFunctionNoProtoType())
621 Ty = getContext().getFunctionTypeWithExceptionSpec(Orig: Ty, ESI: EST_None);
622 std::string Mangled;
623 llvm::raw_string_ostream Out(Mangled);
624 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: Ty, Out, NormalizeIntegers: false);
625 return llvm::ConstantInt::get(
626 Ty: CGM.Int32Ty, V: static_cast<uint32_t>(llvm::xxh3_64bits(data: Mangled)));
627}
628
629void CodeGenFunction::EmitKernelMetadata(const FunctionDecl *FD,
630 llvm::Function *Fn) {
631 if (!FD->hasAttr<OpenCLKernelAttr>() && !FD->hasAttr<CUDAGlobalAttr>())
632 return;
633
634 llvm::LLVMContext &Context = getLLVMContext();
635
636 CGM.GenKernelArgMetadata(FN: Fn, FD, CGF: this);
637
638 if (!getLangOpts().OpenCL)
639 return;
640
641 if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {
642 QualType HintQTy = A->getTypeHint();
643 const ExtVectorType *HintEltQTy = HintQTy->getAs<ExtVectorType>();
644 bool IsSignedInteger =
645 HintQTy->isSignedIntegerType() ||
646 (HintEltQTy && HintEltQTy->getElementType()->isSignedIntegerType());
647 llvm::Metadata *AttrMDArgs[] = {
648 llvm::ConstantAsMetadata::get(C: llvm::UndefValue::get(
649 T: CGM.getTypes().ConvertType(T: A->getTypeHint()))),
650 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
651 Ty: llvm::IntegerType::get(C&: Context, NumBits: 32),
652 V: llvm::APInt(32, (uint64_t)(IsSignedInteger ? 1 : 0))))};
653 Fn->setMetadata(Kind: "vec_type_hint", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
654 }
655
656 if (const WorkGroupSizeHintAttr *A = FD->getAttr<WorkGroupSizeHintAttr>()) {
657 llvm::Metadata *AttrMDArgs[] = {
658 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: A->getXDim())),
659 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: A->getYDim())),
660 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: A->getZDim()))};
661 Fn->setMetadata(Kind: "work_group_size_hint", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
662 }
663
664 if (const ReqdWorkGroupSizeAttr *A = FD->getAttr<ReqdWorkGroupSizeAttr>()) {
665 llvm::Metadata *AttrMDArgs[] = {
666 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: A->getXDim())),
667 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: A->getYDim())),
668 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: A->getZDim()))};
669 Fn->setMetadata(Kind: "reqd_work_group_size", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
670 }
671
672 if (const OpenCLIntelReqdSubGroupSizeAttr *A =
673 FD->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
674 llvm::Metadata *AttrMDArgs[] = {
675 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: A->getSubGroupSize()))};
676 Fn->setMetadata(Kind: "intel_reqd_sub_group_size",
677 Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
678 }
679}
680
681/// Determine whether the function F ends with a return stmt.
682static bool endsWithReturn(const Decl* F) {
683 const Stmt *Body = nullptr;
684 if (auto *FD = dyn_cast_or_null<FunctionDecl>(Val: F))
685 Body = FD->getBody();
686 else if (auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(Val: F))
687 Body = OMD->getBody();
688
689 if (auto *CS = dyn_cast_or_null<CompoundStmt>(Val: Body)) {
690 auto LastStmt = CS->body_rbegin();
691 if (LastStmt != CS->body_rend())
692 return isa<ReturnStmt>(Val: *LastStmt);
693 }
694 return false;
695}
696
697void CodeGenFunction::markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {
698 if (SanOpts.has(K: SanitizerKind::Thread)) {
699 Fn->addFnAttr(Kind: "sanitize_thread_no_checking_at_run_time");
700 Fn->removeFnAttr(Kind: llvm::Attribute::SanitizeThread);
701 }
702}
703
704/// Check if the return value of this function requires sanitization.
705bool CodeGenFunction::requiresReturnValueCheck() const {
706 return requiresReturnValueNullabilityCheck() ||
707 (SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute) && CurCodeDecl &&
708 CurCodeDecl->getAttr<ReturnsNonNullAttr>());
709}
710
711static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) {
712 auto *MD = dyn_cast_or_null<CXXMethodDecl>(Val: D);
713 if (!MD || !MD->getDeclName().getAsIdentifierInfo() ||
714 !MD->getDeclName().getAsIdentifierInfo()->isStr(Str: "allocate") ||
715 (MD->getNumParams() != 1 && MD->getNumParams() != 2))
716 return false;
717
718 if (MD->parameters()[0]->getType().getCanonicalType() != Ctx.getSizeType())
719 return false;
720
721 if (MD->getNumParams() == 2) {
722 auto *PT = MD->parameters()[1]->getType()->getAs<PointerType>();
723 if (!PT || !PT->isVoidPointerType() ||
724 !PT->getPointeeType().isConstQualified())
725 return false;
726 }
727
728 return true;
729}
730
731bool CodeGenFunction::isInAllocaArgument(CGCXXABI &ABI, QualType Ty) {
732 const CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
733 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
734}
735
736bool CodeGenFunction::hasInAllocaArg(const CXXMethodDecl *MD) {
737 return getTarget().getTriple().getArch() == llvm::Triple::x86 &&
738 getTarget().getCXXABI().isMicrosoft() &&
739 llvm::any_of(Range: MD->parameters(), P: [&](ParmVarDecl *P) {
740 return isInAllocaArgument(ABI&: CGM.getCXXABI(), Ty: P->getType());
741 });
742}
743
744/// Return the UBSan prologue signature for \p FD if one is available.
745static llvm::Constant *getPrologueSignature(CodeGenModule &CGM,
746 const FunctionDecl *FD) {
747 if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: FD))
748 if (!MD->isStatic())
749 return nullptr;
750 return CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM);
751}
752
753void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
754 llvm::Function *Fn,
755 const CGFunctionInfo &FnInfo,
756 const FunctionArgList &Args,
757 SourceLocation Loc,
758 SourceLocation StartLoc) {
759 assert(!CurFn &&
760 "Do not use a CodeGenFunction object for more than one function");
761
762 const Decl *D = GD.getDecl();
763
764 DidCallStackSave = false;
765 CurCodeDecl = D;
766 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
767 if (FD && FD->usesSEHTry())
768 CurSEHParent = GD;
769 CurFuncDecl = (D ? D->getNonClosureContext() : nullptr);
770 FnRetTy = RetTy;
771 CurFn = Fn;
772 CurFnInfo = &FnInfo;
773 assert(CurFn->isDeclaration() && "Function already has body?");
774
775 // If this function is ignored for any of the enabled sanitizers,
776 // disable the sanitizer for the function.
777 do {
778#define SANITIZER(NAME, ID) \
779 if (SanOpts.empty()) \
780 break; \
781 if (SanOpts.has(SanitizerKind::ID)) \
782 if (CGM.isInNoSanitizeList(SanitizerKind::ID, Fn, Loc)) \
783 SanOpts.set(SanitizerKind::ID, false);
784
785#include "clang/Basic/Sanitizers.def"
786#undef SANITIZER
787 } while (false);
788
789 if (D) {
790 const bool SanitizeBounds = SanOpts.hasOneOf(K: SanitizerKind::Bounds);
791 SanitizerMask no_sanitize_mask;
792 bool NoSanitizeCoverage = false;
793
794 for (auto *Attr : D->specific_attrs<NoSanitizeAttr>()) {
795 no_sanitize_mask |= Attr->getMask();
796 // SanitizeCoverage is not handled by SanOpts.
797 if (Attr->hasCoverage())
798 NoSanitizeCoverage = true;
799 }
800
801 // Apply the no_sanitize* attributes to SanOpts.
802 SanOpts.Mask &= ~no_sanitize_mask;
803 if (no_sanitize_mask & SanitizerKind::Address)
804 SanOpts.set(K: SanitizerKind::KernelAddress, Value: false);
805 if (no_sanitize_mask & SanitizerKind::KernelAddress)
806 SanOpts.set(K: SanitizerKind::Address, Value: false);
807 if (no_sanitize_mask & SanitizerKind::HWAddress)
808 SanOpts.set(K: SanitizerKind::KernelHWAddress, Value: false);
809 if (no_sanitize_mask & SanitizerKind::KernelHWAddress)
810 SanOpts.set(K: SanitizerKind::HWAddress, Value: false);
811
812 if (SanitizeBounds && !SanOpts.hasOneOf(K: SanitizerKind::Bounds))
813 Fn->addFnAttr(Kind: llvm::Attribute::NoSanitizeBounds);
814
815 if (NoSanitizeCoverage && CGM.getCodeGenOpts().hasSanitizeCoverage())
816 Fn->addFnAttr(Kind: llvm::Attribute::NoSanitizeCoverage);
817
818 // Some passes need the non-negated no_sanitize attribute. Pass them on.
819 if (CGM.getCodeGenOpts().hasSanitizeBinaryMetadata()) {
820 if (no_sanitize_mask & SanitizerKind::Thread)
821 Fn->addFnAttr(Kind: "no_sanitize_thread");
822 }
823 }
824
825 if (ShouldSkipSanitizerInstrumentation()) {
826 CurFn->addFnAttr(Kind: llvm::Attribute::DisableSanitizerInstrumentation);
827 } else {
828 // Apply sanitizer attributes to the function.
829 if (SanOpts.hasOneOf(K: SanitizerKind::Address | SanitizerKind::KernelAddress))
830 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAddress);
831 if (SanOpts.hasOneOf(K: SanitizerKind::HWAddress |
832 SanitizerKind::KernelHWAddress))
833 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeHWAddress);
834 if (SanOpts.has(K: SanitizerKind::MemtagStack))
835 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemTag);
836 if (SanOpts.has(K: SanitizerKind::Thread))
837 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeThread);
838 if (SanOpts.has(K: SanitizerKind::NumericalStability))
839 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeNumericalStability);
840 if (SanOpts.hasOneOf(K: SanitizerKind::Memory | SanitizerKind::KernelMemory))
841 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemory);
842 }
843 if (SanOpts.has(K: SanitizerKind::SafeStack))
844 Fn->addFnAttr(Kind: llvm::Attribute::SafeStack);
845 if (SanOpts.has(K: SanitizerKind::ShadowCallStack))
846 Fn->addFnAttr(Kind: llvm::Attribute::ShadowCallStack);
847
848 // Apply fuzzing attribute to the function.
849 if (SanOpts.hasOneOf(K: SanitizerKind::Fuzzer | SanitizerKind::FuzzerNoLink))
850 Fn->addFnAttr(Kind: llvm::Attribute::OptForFuzzing);
851
852 // Ignore TSan memory acesses from within ObjC/ObjC++ dealloc, initialize,
853 // .cxx_destruct, __destroy_helper_block_ and all of their calees at run time.
854 if (SanOpts.has(K: SanitizerKind::Thread)) {
855 if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(Val: D)) {
856 const IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(argIndex: 0);
857 if (OMD->getMethodFamily() == OMF_dealloc ||
858 OMD->getMethodFamily() == OMF_initialize ||
859 (OMD->getSelector().isUnarySelector() && II->isStr(Str: ".cxx_destruct"))) {
860 markAsIgnoreThreadCheckingAtRuntime(Fn);
861 }
862 }
863 }
864
865 // Ignore unrelated casts in STL allocate() since the allocator must cast
866 // from void* to T* before object initialization completes. Don't match on the
867 // namespace because not all allocators are in std::
868 if (D && SanOpts.has(K: SanitizerKind::CFIUnrelatedCast)) {
869 if (matchesStlAllocatorFn(D, Ctx: getContext()))
870 SanOpts.Mask &= ~SanitizerKind::CFIUnrelatedCast;
871 }
872
873 // Ignore null checks in coroutine functions since the coroutines passes
874 // are not aware of how to move the extra UBSan instructions across the split
875 // coroutine boundaries.
876 if (D && SanOpts.has(K: SanitizerKind::Null))
877 if (FD && FD->getBody() &&
878 FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass)
879 SanOpts.Mask &= ~SanitizerKind::Null;
880
881 // Add pointer authentication attributes.
882 const CodeGenOptions &CodeGenOpts = CGM.getCodeGenOpts();
883 if (CodeGenOpts.PointerAuth.ReturnAddresses)
884 Fn->addFnAttr(Kind: "ptrauth-returns");
885 if (CodeGenOpts.PointerAuth.FunctionPointers)
886 Fn->addFnAttr(Kind: "ptrauth-calls");
887 if (CodeGenOpts.PointerAuth.AuthTraps)
888 Fn->addFnAttr(Kind: "ptrauth-auth-traps");
889 if (CodeGenOpts.PointerAuth.IndirectGotos)
890 Fn->addFnAttr(Kind: "ptrauth-indirect-gotos");
891
892 // Apply xray attributes to the function (as a string, for now)
893 bool AlwaysXRayAttr = false;
894 if (const auto *XRayAttr = D ? D->getAttr<XRayInstrumentAttr>() : nullptr) {
895 if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
896 K: XRayInstrKind::FunctionEntry) ||
897 CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
898 K: XRayInstrKind::FunctionExit)) {
899 if (XRayAttr->alwaysXRayInstrument() && ShouldXRayInstrumentFunction()) {
900 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-always");
901 AlwaysXRayAttr = true;
902 }
903 if (XRayAttr->neverXRayInstrument())
904 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-never");
905 if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())
906 if (ShouldXRayInstrumentFunction())
907 Fn->addFnAttr(Kind: "xray-log-args",
908 Val: llvm::utostr(X: LogArgs->getArgumentCount()));
909 }
910 } else {
911 if (ShouldXRayInstrumentFunction() && !CGM.imbueXRayAttrs(Fn, Loc))
912 Fn->addFnAttr(
913 Kind: "xray-instruction-threshold",
914 Val: llvm::itostr(X: CGM.getCodeGenOpts().XRayInstructionThreshold));
915 }
916
917 if (ShouldXRayInstrumentFunction()) {
918 if (CGM.getCodeGenOpts().XRayIgnoreLoops)
919 Fn->addFnAttr(Kind: "xray-ignore-loops");
920
921 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
922 K: XRayInstrKind::FunctionExit))
923 Fn->addFnAttr(Kind: "xray-skip-exit");
924
925 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
926 K: XRayInstrKind::FunctionEntry))
927 Fn->addFnAttr(Kind: "xray-skip-entry");
928
929 auto FuncGroups = CGM.getCodeGenOpts().XRayTotalFunctionGroups;
930 if (FuncGroups > 1) {
931 auto FuncName = llvm::ArrayRef<uint8_t>(CurFn->getName().bytes_begin(),
932 CurFn->getName().bytes_end());
933 auto Group = crc32(Data: FuncName) % FuncGroups;
934 if (Group != CGM.getCodeGenOpts().XRaySelectedFunctionGroup &&
935 !AlwaysXRayAttr)
936 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-never");
937 }
938 }
939
940 if (CGM.getCodeGenOpts().getProfileInstr() != CodeGenOptions::ProfileNone) {
941 switch (CGM.isFunctionBlockedFromProfileInstr(Fn, Loc)) {
942 case ProfileList::Skip:
943 Fn->addFnAttr(Kind: llvm::Attribute::SkipProfile);
944 break;
945 case ProfileList::Forbid:
946 Fn->addFnAttr(Kind: llvm::Attribute::NoProfile);
947 break;
948 case ProfileList::Allow:
949 break;
950 }
951 }
952
953 unsigned Count, Offset;
954 if (const auto *Attr =
955 D ? D->getAttr<PatchableFunctionEntryAttr>() : nullptr) {
956 Count = Attr->getCount();
957 Offset = Attr->getOffset();
958 } else {
959 Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;
960 Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;
961 }
962 if (Count && Offset <= Count) {
963 Fn->addFnAttr(Kind: "patchable-function-entry", Val: std::to_string(val: Count - Offset));
964 if (Offset)
965 Fn->addFnAttr(Kind: "patchable-function-prefix", Val: std::to_string(val: Offset));
966 }
967 // Instruct that functions for COFF/CodeView targets should start with a
968 // patchable instruction, but only on x86/x64. Don't forward this to ARM/ARM64
969 // backends as they don't need it -- instructions on these architectures are
970 // always atomically patchable at runtime.
971 if (CGM.getCodeGenOpts().HotPatch &&
972 getContext().getTargetInfo().getTriple().isX86() &&
973 getContext().getTargetInfo().getTriple().getEnvironment() !=
974 llvm::Triple::CODE16)
975 Fn->addFnAttr(Kind: "patchable-function", Val: "prologue-short-redirect");
976
977 // Add no-jump-tables value.
978 if (CGM.getCodeGenOpts().NoUseJumpTables)
979 Fn->addFnAttr(Kind: "no-jump-tables", Val: "true");
980
981 // Add no-inline-line-tables value.
982 if (CGM.getCodeGenOpts().NoInlineLineTables)
983 Fn->addFnAttr(Kind: "no-inline-line-tables");
984
985 // Add profile-sample-accurate value.
986 if (CGM.getCodeGenOpts().ProfileSampleAccurate)
987 Fn->addFnAttr(Kind: "profile-sample-accurate");
988
989 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
990 Fn->addFnAttr(Kind: "use-sample-profile");
991
992 if (D && D->hasAttr<CFICanonicalJumpTableAttr>())
993 Fn->addFnAttr(Kind: "cfi-canonical-jump-table");
994
995 if (D && D->hasAttr<NoProfileFunctionAttr>())
996 Fn->addFnAttr(Kind: llvm::Attribute::NoProfile);
997
998 if (D && D->hasAttr<HybridPatchableAttr>())
999 Fn->addFnAttr(Kind: llvm::Attribute::HybridPatchable);
1000
1001 if (D) {
1002 // Function attributes take precedence over command line flags.
1003 if (auto *A = D->getAttr<FunctionReturnThunksAttr>()) {
1004 switch (A->getThunkType()) {
1005 case FunctionReturnThunksAttr::Kind::Keep:
1006 break;
1007 case FunctionReturnThunksAttr::Kind::Extern:
1008 Fn->addFnAttr(Kind: llvm::Attribute::FnRetThunkExtern);
1009 break;
1010 }
1011 } else if (CGM.getCodeGenOpts().FunctionReturnThunks)
1012 Fn->addFnAttr(Kind: llvm::Attribute::FnRetThunkExtern);
1013 }
1014
1015 if (FD && (getLangOpts().OpenCL ||
1016 (getLangOpts().HIP && getLangOpts().CUDAIsDevice))) {
1017 // Add metadata for a kernel function.
1018 EmitKernelMetadata(FD, Fn);
1019 }
1020
1021 if (FD && FD->hasAttr<ClspvLibclcBuiltinAttr>()) {
1022 Fn->setMetadata(Kind: "clspv_libclc_builtin",
1023 Node: llvm::MDNode::get(Context&: getLLVMContext(), MDs: {}));
1024 }
1025
1026 // If we are checking function types, emit a function type signature as
1027 // prologue data.
1028 if (FD && SanOpts.has(K: SanitizerKind::Function)) {
1029 if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {
1030 llvm::LLVMContext &Ctx = Fn->getContext();
1031 llvm::MDBuilder MDB(Ctx);
1032 Fn->setMetadata(
1033 KindID: llvm::LLVMContext::MD_func_sanitize,
1034 Node: MDB.createRTTIPointerPrologue(
1035 PrologueSig, RTTI: getUBSanFunctionTypeHash(Ty: FD->getType())));
1036 }
1037 }
1038
1039 // If we're checking nullability, we need to know whether we can check the
1040 // return value. Initialize the flag to 'true' and refine it in EmitParmDecl.
1041 if (SanOpts.has(K: SanitizerKind::NullabilityReturn)) {
1042 auto Nullability = FnRetTy->getNullability();
1043 if (Nullability && *Nullability == NullabilityKind::NonNull &&
1044 !FnRetTy->isRecordType()) {
1045 if (!(SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute) &&
1046 CurCodeDecl && CurCodeDecl->getAttr<ReturnsNonNullAttr>()))
1047 RetValNullabilityPrecondition =
1048 llvm::ConstantInt::getTrue(Context&: getLLVMContext());
1049 }
1050 }
1051
1052 // If we're in C++ mode and the function name is "main", it is guaranteed
1053 // to be norecurse by the standard (3.6.1.3 "The function main shall not be
1054 // used within a program").
1055 //
1056 // OpenCL C 2.0 v2.2-11 s6.9.i:
1057 // Recursion is not supported.
1058 //
1059 // SYCL v1.2.1 s3.10:
1060 // kernels cannot include RTTI information, exception classes,
1061 // recursive code, virtual functions or make use of C++ libraries that
1062 // are not compiled for the device.
1063 if (FD && ((getLangOpts().CPlusPlus && FD->isMain()) ||
1064 getLangOpts().OpenCL || getLangOpts().SYCLIsDevice ||
1065 (getLangOpts().CUDA && FD->hasAttr<CUDAGlobalAttr>())))
1066 Fn->addFnAttr(Kind: llvm::Attribute::NoRecurse);
1067
1068 llvm::RoundingMode RM = getLangOpts().getDefaultRoundingMode();
1069 llvm::fp::ExceptionBehavior FPExceptionBehavior =
1070 ToConstrainedExceptMD(Kind: getLangOpts().getDefaultExceptionMode());
1071 Builder.setDefaultConstrainedRounding(RM);
1072 Builder.setDefaultConstrainedExcept(FPExceptionBehavior);
1073 if ((FD && (FD->UsesFPIntrin() || FD->hasAttr<StrictFPAttr>())) ||
1074 (!FD && (FPExceptionBehavior != llvm::fp::ebIgnore ||
1075 RM != llvm::RoundingMode::NearestTiesToEven))) {
1076 Builder.setIsFPConstrained(true);
1077 Fn->addFnAttr(Kind: llvm::Attribute::StrictFP);
1078 }
1079
1080 // If a custom alignment is used, force realigning to this alignment on
1081 // any main function which certainly will need it.
1082 if (FD && ((FD->isMain() || FD->isMSVCRTEntryPoint()) &&
1083 CGM.getCodeGenOpts().StackAlignment))
1084 Fn->addFnAttr(Kind: "stackrealign");
1085
1086 // "main" doesn't need to zero out call-used registers.
1087 if (FD && FD->isMain())
1088 Fn->removeFnAttr(Kind: "zero-call-used-regs");
1089
1090 llvm::BasicBlock *EntryBB = createBasicBlock(name: "entry", parent: CurFn);
1091
1092 // Create a marker to make it easy to insert allocas into the entryblock
1093 // later. Don't create this with the builder, because we don't want it
1094 // folded.
1095 llvm::Value *Undef = llvm::UndefValue::get(T: Int32Ty);
1096 AllocaInsertPt = new llvm::BitCastInst(Undef, Int32Ty, "allocapt", EntryBB);
1097
1098 ReturnBlock = getJumpDestInCurrentScope(Name: "return");
1099
1100 Builder.SetInsertPoint(EntryBB);
1101
1102 // If we're checking the return value, allocate space for a pointer to a
1103 // precise source location of the checked return statement.
1104 if (requiresReturnValueCheck()) {
1105 ReturnLocation = CreateDefaultAlignTempAlloca(Ty: Int8PtrTy, Name: "return.sloc.ptr");
1106 Builder.CreateStore(Val: llvm::ConstantPointerNull::get(T: Int8PtrTy),
1107 Addr: ReturnLocation);
1108 }
1109
1110 // Emit subprogram debug descriptor.
1111 if (CGDebugInfo *DI = getDebugInfo()) {
1112 // Reconstruct the type from the argument list so that implicit parameters,
1113 // such as 'this' and 'vtt', show up in the debug info. Preserve the calling
1114 // convention.
1115 DI->emitFunctionStart(GD, Loc, ScopeLoc: StartLoc,
1116 FnType: DI->getFunctionType(FD, RetTy, Args), Fn: CurFn,
1117 CurFnIsThunk: CurFuncIsThunk);
1118 }
1119
1120 if (ShouldInstrumentFunction()) {
1121 if (CGM.getCodeGenOpts().InstrumentFunctions)
1122 CurFn->addFnAttr(Kind: "instrument-function-entry", Val: "__cyg_profile_func_enter");
1123 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
1124 CurFn->addFnAttr(Kind: "instrument-function-entry-inlined",
1125 Val: "__cyg_profile_func_enter");
1126 if (CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
1127 CurFn->addFnAttr(Kind: "instrument-function-entry-inlined",
1128 Val: "__cyg_profile_func_enter_bare");
1129 }
1130
1131 // Since emitting the mcount call here impacts optimizations such as function
1132 // inlining, we just add an attribute to insert a mcount call in backend.
1133 // The attribute "counting-function" is set to mcount function name which is
1134 // architecture dependent.
1135 if (CGM.getCodeGenOpts().InstrumentForProfiling) {
1136 // Calls to fentry/mcount should not be generated if function has
1137 // the no_instrument_function attribute.
1138 if (!CurFuncDecl || !CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()) {
1139 if (CGM.getCodeGenOpts().CallFEntry)
1140 Fn->addFnAttr(Kind: "fentry-call", Val: "true");
1141 else {
1142 Fn->addFnAttr(Kind: "instrument-function-entry-inlined",
1143 Val: getTarget().getMCountName());
1144 }
1145 if (CGM.getCodeGenOpts().MNopMCount) {
1146 if (!CGM.getCodeGenOpts().CallFEntry)
1147 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_without_opt)
1148 << "-mnop-mcount" << "-mfentry";
1149 Fn->addFnAttr(Kind: "mnop-mcount");
1150 }
1151
1152 if (CGM.getCodeGenOpts().RecordMCount) {
1153 if (!CGM.getCodeGenOpts().CallFEntry)
1154 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_without_opt)
1155 << "-mrecord-mcount" << "-mfentry";
1156 Fn->addFnAttr(Kind: "mrecord-mcount");
1157 }
1158 }
1159 }
1160
1161 if (CGM.getCodeGenOpts().PackedStack) {
1162 if (getContext().getTargetInfo().getTriple().getArch() !=
1163 llvm::Triple::systemz)
1164 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_on_target)
1165 << "-mpacked-stack";
1166 Fn->addFnAttr(Kind: "packed-stack");
1167 }
1168
1169 if (CGM.getCodeGenOpts().WarnStackSize != UINT_MAX &&
1170 !CGM.getDiags().isIgnored(DiagID: diag::warn_fe_backend_frame_larger_than, Loc))
1171 Fn->addFnAttr(Kind: "warn-stack-size",
1172 Val: std::to_string(val: CGM.getCodeGenOpts().WarnStackSize));
1173
1174 if (RetTy->isVoidType()) {
1175 // Void type; nothing to return.
1176 ReturnValue = Address::invalid();
1177
1178 // Count the implicit return.
1179 if (!endsWithReturn(F: D))
1180 ++NumReturnExprs;
1181 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1182 // Indirect return; emit returned value directly into sret slot.
1183 // This reduces code size, and affects correctness in C++.
1184 auto AI = CurFn->arg_begin();
1185 if (CurFnInfo->getReturnInfo().isSRetAfterThis())
1186 ++AI;
1187 ReturnValue = makeNaturalAddressForPointer(
1188 Ptr: &*AI, T: RetTy, Alignment: CurFnInfo->getReturnInfo().getIndirectAlign(), ForPointeeType: false,
1189 BaseInfo: nullptr, TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
1190 if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
1191 ReturnValuePointer =
1192 CreateDefaultAlignTempAlloca(Ty: ReturnValue.getType(), Name: "result.ptr");
1193 Builder.CreateStore(Val: ReturnValue.emitRawPointer(CGF&: *this),
1194 Addr: ReturnValuePointer);
1195 }
1196 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
1197 !hasScalarEvaluationKind(T: CurFnInfo->getReturnType())) {
1198 // Load the sret pointer from the argument struct and return into that.
1199 unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();
1200 llvm::Function::arg_iterator EI = CurFn->arg_end();
1201 --EI;
1202 llvm::Value *Addr = Builder.CreateStructGEP(
1203 Ty: CurFnInfo->getArgStruct(), Ptr: &*EI, Idx);
1204 llvm::Type *Ty =
1205 cast<llvm::GetElementPtrInst>(Val: Addr)->getResultElementType();
1206 ReturnValuePointer = Address(Addr, Ty, getPointerAlign());
1207 Addr = Builder.CreateAlignedLoad(Ty, Addr, Align: getPointerAlign(), Name: "agg.result");
1208 ReturnValue = Address(Addr, ConvertType(T: RetTy),
1209 CGM.getNaturalTypeAlignment(T: RetTy), KnownNonNull);
1210 } else {
1211 ReturnValue = CreateIRTemp(T: RetTy, Name: "retval");
1212
1213 // Tell the epilog emitter to autorelease the result. We do this
1214 // now so that various specialized functions can suppress it
1215 // during their IR-generation.
1216 if (getLangOpts().ObjCAutoRefCount &&
1217 !CurFnInfo->isReturnsRetained() &&
1218 RetTy->isObjCRetainableType())
1219 AutoreleaseResult = true;
1220 }
1221
1222 EmitStartEHSpec(D: CurCodeDecl);
1223
1224 PrologueCleanupDepth = EHStack.stable_begin();
1225
1226 // Emit OpenMP specific initialization of the device functions.
1227 if (getLangOpts().OpenMP && CurCodeDecl)
1228 CGM.getOpenMPRuntime().emitFunctionProlog(CGF&: *this, D: CurCodeDecl);
1229
1230 // Handle emitting HLSL entry functions.
1231 if (D && D->hasAttr<HLSLShaderAttr>())
1232 CGM.getHLSLRuntime().emitEntryFunction(FD, Fn);
1233
1234 EmitFunctionProlog(FI: *CurFnInfo, Fn: CurFn, Args);
1235
1236 if (const CXXMethodDecl *MD = dyn_cast_if_present<CXXMethodDecl>(Val: D);
1237 MD && !MD->isStatic()) {
1238 bool IsInLambda =
1239 MD->getParent()->isLambda() && MD->getOverloadedOperator() == OO_Call;
1240 if (MD->isImplicitObjectMemberFunction())
1241 CGM.getCXXABI().EmitInstanceFunctionProlog(CGF&: *this);
1242 if (IsInLambda) {
1243 // We're in a lambda; figure out the captures.
1244 MD->getParent()->getCaptureFields(Captures&: LambdaCaptureFields,
1245 ThisCapture&: LambdaThisCaptureField);
1246 if (LambdaThisCaptureField) {
1247 // If the lambda captures the object referred to by '*this' - either by
1248 // value or by reference, make sure CXXThisValue points to the correct
1249 // object.
1250
1251 // Get the lvalue for the field (which is a copy of the enclosing object
1252 // or contains the address of the enclosing object).
1253 LValue ThisFieldLValue = EmitLValueForLambdaField(Field: LambdaThisCaptureField);
1254 if (!LambdaThisCaptureField->getType()->isPointerType()) {
1255 // If the enclosing object was captured by value, just use its
1256 // address. Sign this pointer.
1257 CXXThisValue = ThisFieldLValue.getPointer(CGF&: *this);
1258 } else {
1259 // Load the lvalue pointed to by the field, since '*this' was captured
1260 // by reference.
1261 CXXThisValue =
1262 EmitLoadOfLValue(V: ThisFieldLValue, Loc: SourceLocation()).getScalarVal();
1263 }
1264 }
1265 for (auto *FD : MD->getParent()->fields()) {
1266 if (FD->hasCapturedVLAType()) {
1267 auto *ExprArg = EmitLoadOfLValue(V: EmitLValueForLambdaField(Field: FD),
1268 Loc: SourceLocation()).getScalarVal();
1269 auto VAT = FD->getCapturedVLAType();
1270 VLASizeMap[VAT->getSizeExpr()] = ExprArg;
1271 }
1272 }
1273 } else if (MD->isImplicitObjectMemberFunction()) {
1274 // Not in a lambda; just use 'this' from the method.
1275 // FIXME: Should we generate a new load for each use of 'this'? The
1276 // fast register allocator would be happier...
1277 CXXThisValue = CXXABIThisValue;
1278 }
1279
1280 // Check the 'this' pointer once per function, if it's available.
1281 if (CXXABIThisValue) {
1282 SanitizerSet SkippedChecks;
1283 SkippedChecks.set(K: SanitizerKind::ObjectSize, Value: true);
1284 QualType ThisTy = MD->getThisType();
1285
1286 // If this is the call operator of a lambda with no captures, it
1287 // may have a static invoker function, which may call this operator with
1288 // a null 'this' pointer.
1289 if (isLambdaCallOperator(MD) && MD->getParent()->isCapturelessLambda())
1290 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
1291
1292 EmitTypeCheck(
1293 TCK: isa<CXXConstructorDecl>(Val: MD) ? TCK_ConstructorCall : TCK_MemberCall,
1294 Loc, V: CXXABIThisValue, Type: ThisTy, Alignment: CXXABIThisAlignment, SkippedChecks);
1295 }
1296 }
1297
1298 // If any of the arguments have a variably modified type, make sure to
1299 // emit the type size, but only if the function is not naked. Naked functions
1300 // have no prolog to run this evaluation.
1301 if (!FD || !FD->hasAttr<NakedAttr>()) {
1302 for (const VarDecl *VD : Args) {
1303 // Dig out the type as written from ParmVarDecls; it's unclear whether
1304 // the standard (C99 6.9.1p10) requires this, but we're following the
1305 // precedent set by gcc.
1306 QualType Ty;
1307 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: VD))
1308 Ty = PVD->getOriginalType();
1309 else
1310 Ty = VD->getType();
1311
1312 if (Ty->isVariablyModifiedType())
1313 EmitVariablyModifiedType(Ty);
1314 }
1315 }
1316 // Emit a location at the end of the prologue.
1317 if (CGDebugInfo *DI = getDebugInfo())
1318 DI->EmitLocation(Builder, Loc: StartLoc);
1319 // TODO: Do we need to handle this in two places like we do with
1320 // target-features/target-cpu?
1321 if (CurFuncDecl)
1322 if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>())
1323 LargestVectorWidth = VecWidth->getVectorWidth();
1324
1325 if (CGM.shouldEmitConvergenceTokens())
1326 ConvergenceTokenStack.push_back(Elt: getOrEmitConvergenceEntryToken(F: CurFn));
1327}
1328
1329void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {
1330 incrementProfileCounter(S: Body);
1331 maybeCreateMCDCCondBitmap();
1332 if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Val: Body))
1333 EmitCompoundStmtWithoutScope(S: *S);
1334 else
1335 EmitStmt(S: Body);
1336}
1337
1338/// When instrumenting to collect profile data, the counts for some blocks
1339/// such as switch cases need to not include the fall-through counts, so
1340/// emit a branch around the instrumentation code. When not instrumenting,
1341/// this just calls EmitBlock().
1342void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
1343 const Stmt *S) {
1344 llvm::BasicBlock *SkipCountBB = nullptr;
1345 // Do not skip over the instrumentation when single byte coverage mode is
1346 // enabled.
1347 if (HaveInsertPoint() && CGM.getCodeGenOpts().hasProfileClangInstr() &&
1348 !llvm::EnableSingleByteCoverage) {
1349 // When instrumenting for profiling, the fallthrough to certain
1350 // statements needs to skip over the instrumentation code so that we
1351 // get an accurate count.
1352 SkipCountBB = createBasicBlock(name: "skipcount");
1353 EmitBranch(Block: SkipCountBB);
1354 }
1355 EmitBlock(BB);
1356 uint64_t CurrentCount = getCurrentProfileCount();
1357 incrementProfileCounter(S);
1358 setCurrentProfileCount(getCurrentProfileCount() + CurrentCount);
1359 if (SkipCountBB)
1360 EmitBlock(BB: SkipCountBB);
1361}
1362
1363/// Tries to mark the given function nounwind based on the
1364/// non-existence of any throwing calls within it. We believe this is
1365/// lightweight enough to do at -O0.
1366static void TryMarkNoThrow(llvm::Function *F) {
1367 // LLVM treats 'nounwind' on a function as part of the type, so we
1368 // can't do this on functions that can be overwritten.
1369 if (F->isInterposable()) return;
1370
1371 for (llvm::BasicBlock &BB : *F)
1372 for (llvm::Instruction &I : BB)
1373 if (I.mayThrow())
1374 return;
1375
1376 F->setDoesNotThrow();
1377}
1378
1379QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD,
1380 FunctionArgList &Args) {
1381 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
1382 QualType ResTy = FD->getReturnType();
1383
1384 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD);
1385 if (MD && MD->isImplicitObjectMemberFunction()) {
1386 if (CGM.getCXXABI().HasThisReturn(GD))
1387 ResTy = MD->getThisType();
1388 else if (CGM.getCXXABI().hasMostDerivedReturn(GD))
1389 ResTy = CGM.getContext().VoidPtrTy;
1390 CGM.getCXXABI().buildThisParam(CGF&: *this, Params&: Args);
1391 }
1392
1393 // The base version of an inheriting constructor whose constructed base is a
1394 // virtual base is not passed any arguments (because it doesn't actually call
1395 // the inherited constructor).
1396 bool PassedParams = true;
1397 if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(Val: FD))
1398 if (auto Inherited = CD->getInheritedConstructor())
1399 PassedParams =
1400 getTypes().inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
1401
1402 if (PassedParams) {
1403 for (auto *Param : FD->parameters()) {
1404 Args.push_back(Elt: Param);
1405 if (!Param->hasAttr<PassObjectSizeAttr>())
1406 continue;
1407
1408 auto *Implicit = ImplicitParamDecl::Create(
1409 C&: getContext(), DC: Param->getDeclContext(), IdLoc: Param->getLocation(),
1410 /*Id=*/nullptr, T: getContext().getSizeType(), ParamKind: ImplicitParamKind::Other);
1411 SizeArguments[Param] = Implicit;
1412 Args.push_back(Elt: Implicit);
1413 }
1414 }
1415
1416 if (MD && (isa<CXXConstructorDecl>(Val: MD) || isa<CXXDestructorDecl>(Val: MD)))
1417 CGM.getCXXABI().addImplicitStructorParams(CGF&: *this, ResTy, Params&: Args);
1418
1419 return ResTy;
1420}
1421
1422void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
1423 const CGFunctionInfo &FnInfo) {
1424 assert(Fn && "generating code for null Function");
1425 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
1426 CurGD = GD;
1427
1428 FunctionArgList Args;
1429 QualType ResTy = BuildFunctionArgList(GD, Args);
1430
1431 CGM.getTargetCodeGenInfo().checkFunctionABI(CGM, Decl: FD);
1432
1433 if (FD->isInlineBuiltinDeclaration()) {
1434 // When generating code for a builtin with an inline declaration, use a
1435 // mangled name to hold the actual body, while keeping an external
1436 // definition in case the function pointer is referenced somewhere.
1437 std::string FDInlineName = (Fn->getName() + ".inline").str();
1438 llvm::Module *M = Fn->getParent();
1439 llvm::Function *Clone = M->getFunction(Name: FDInlineName);
1440 if (!Clone) {
1441 Clone = llvm::Function::Create(Ty: Fn->getFunctionType(),
1442 Linkage: llvm::GlobalValue::InternalLinkage,
1443 AddrSpace: Fn->getAddressSpace(), N: FDInlineName, M);
1444 Clone->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1445 }
1446 Fn->setLinkage(llvm::GlobalValue::ExternalLinkage);
1447 Fn = Clone;
1448 } else {
1449 // Detect the unusual situation where an inline version is shadowed by a
1450 // non-inline version. In that case we should pick the external one
1451 // everywhere. That's GCC behavior too. Unfortunately, I cannot find a way
1452 // to detect that situation before we reach codegen, so do some late
1453 // replacement.
1454 for (const FunctionDecl *PD = FD->getPreviousDecl(); PD;
1455 PD = PD->getPreviousDecl()) {
1456 if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) {
1457 std::string FDInlineName = (Fn->getName() + ".inline").str();
1458 llvm::Module *M = Fn->getParent();
1459 if (llvm::Function *Clone = M->getFunction(Name: FDInlineName)) {
1460 Clone->replaceAllUsesWith(V: Fn);
1461 Clone->eraseFromParent();
1462 }
1463 break;
1464 }
1465 }
1466 }
1467
1468 // Check if we should generate debug info for this function.
1469 if (FD->hasAttr<NoDebugAttr>()) {
1470 // Clear non-distinct debug info that was possibly attached to the function
1471 // due to an earlier declaration without the nodebug attribute
1472 Fn->setSubprogram(nullptr);
1473 // Disable debug info indefinitely for this function
1474 DebugInfo = nullptr;
1475 }
1476
1477 // The function might not have a body if we're generating thunks for a
1478 // function declaration.
1479 SourceRange BodyRange;
1480 if (Stmt *Body = FD->getBody())
1481 BodyRange = Body->getSourceRange();
1482 else
1483 BodyRange = FD->getLocation();
1484 CurEHLocation = BodyRange.getEnd();
1485
1486 // Use the location of the start of the function to determine where
1487 // the function definition is located. By default use the location
1488 // of the declaration as the location for the subprogram. A function
1489 // may lack a declaration in the source code if it is created by code
1490 // gen. (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).
1491 SourceLocation Loc = FD->getLocation();
1492
1493 // If this is a function specialization then use the pattern body
1494 // as the location for the function.
1495 if (const FunctionDecl *SpecDecl = FD->getTemplateInstantiationPattern())
1496 if (SpecDecl->hasBody(Definition&: SpecDecl))
1497 Loc = SpecDecl->getLocation();
1498
1499 Stmt *Body = FD->getBody();
1500
1501 if (Body) {
1502 // Coroutines always emit lifetime markers.
1503 if (isa<CoroutineBodyStmt>(Val: Body))
1504 ShouldEmitLifetimeMarkers = true;
1505
1506 // Initialize helper which will detect jumps which can cause invalid
1507 // lifetime markers.
1508 if (ShouldEmitLifetimeMarkers)
1509 Bypasses.Init(Body);
1510 }
1511
1512 // Emit the standard function prologue.
1513 StartFunction(GD, RetTy: ResTy, Fn, FnInfo, Args, Loc, StartLoc: BodyRange.getBegin());
1514
1515 // Save parameters for coroutine function.
1516 if (Body && isa_and_nonnull<CoroutineBodyStmt>(Val: Body))
1517 llvm::append_range(C&: FnArgs, R: FD->parameters());
1518
1519 // Ensure that the function adheres to the forward progress guarantee, which
1520 // is required by certain optimizations.
1521 // In C++11 and up, the attribute will be removed if the body contains a
1522 // trivial empty loop.
1523 if (checkIfFunctionMustProgress())
1524 CurFn->addFnAttr(Kind: llvm::Attribute::MustProgress);
1525
1526 // Generate the body of the function.
1527 PGO.assignRegionCounters(GD, Fn: CurFn);
1528 if (isa<CXXDestructorDecl>(Val: FD))
1529 EmitDestructorBody(Args);
1530 else if (isa<CXXConstructorDecl>(Val: FD))
1531 EmitConstructorBody(Args);
1532 else if (getLangOpts().CUDA &&
1533 !getLangOpts().CUDAIsDevice &&
1534 FD->hasAttr<CUDAGlobalAttr>())
1535 CGM.getCUDARuntime().emitDeviceStub(CGF&: *this, Args);
1536 else if (isa<CXXMethodDecl>(Val: FD) &&
1537 cast<CXXMethodDecl>(Val: FD)->isLambdaStaticInvoker()) {
1538 // The lambda static invoker function is special, because it forwards or
1539 // clones the body of the function call operator (but is actually static).
1540 EmitLambdaStaticInvokeBody(MD: cast<CXXMethodDecl>(Val: FD));
1541 } else if (isa<CXXMethodDecl>(Val: FD) &&
1542 isLambdaCallOperator(MD: cast<CXXMethodDecl>(Val: FD)) &&
1543 !FnInfo.isDelegateCall() &&
1544 cast<CXXMethodDecl>(Val: FD)->getParent()->getLambdaStaticInvoker() &&
1545 hasInAllocaArg(MD: cast<CXXMethodDecl>(Val: FD))) {
1546 // If emitting a lambda with static invoker on X86 Windows, change
1547 // the call operator body.
1548 // Make sure that this is a call operator with an inalloca arg and check
1549 // for delegate call to make sure this is the original call op and not the
1550 // new forwarding function for the static invoker.
1551 EmitLambdaInAllocaCallOpBody(MD: cast<CXXMethodDecl>(Val: FD));
1552 } else if (FD->isDefaulted() && isa<CXXMethodDecl>(Val: FD) &&
1553 (cast<CXXMethodDecl>(Val: FD)->isCopyAssignmentOperator() ||
1554 cast<CXXMethodDecl>(Val: FD)->isMoveAssignmentOperator())) {
1555 // Implicit copy-assignment gets the same special treatment as implicit
1556 // copy-constructors.
1557 emitImplicitAssignmentOperatorBody(Args);
1558 } else if (Body) {
1559 EmitFunctionBody(Body);
1560 } else
1561 llvm_unreachable("no definition for emitted function");
1562
1563 // C++11 [stmt.return]p2:
1564 // Flowing off the end of a function [...] results in undefined behavior in
1565 // a value-returning function.
1566 // C11 6.9.1p12:
1567 // If the '}' that terminates a function is reached, and the value of the
1568 // function call is used by the caller, the behavior is undefined.
1569 if (getLangOpts().CPlusPlus && !FD->hasImplicitReturnZero() && !SawAsmBlock &&
1570 !FD->getReturnType()->isVoidType() && Builder.GetInsertBlock()) {
1571 bool ShouldEmitUnreachable =
1572 CGM.getCodeGenOpts().StrictReturn ||
1573 !CGM.MayDropFunctionReturn(Context: FD->getASTContext(), ReturnType: FD->getReturnType());
1574 if (SanOpts.has(K: SanitizerKind::Return)) {
1575 SanitizerScope SanScope(this);
1576 llvm::Value *IsFalse = Builder.getFalse();
1577 EmitCheck(Checked: std::make_pair(x&: IsFalse, y: SanitizerKind::Return),
1578 Check: SanitizerHandler::MissingReturn,
1579 StaticArgs: EmitCheckSourceLocation(Loc: FD->getLocation()), DynamicArgs: std::nullopt);
1580 } else if (ShouldEmitUnreachable) {
1581 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1582 EmitTrapCall(IntrID: llvm::Intrinsic::trap);
1583 }
1584 if (SanOpts.has(K: SanitizerKind::Return) || ShouldEmitUnreachable) {
1585 Builder.CreateUnreachable();
1586 Builder.ClearInsertionPoint();
1587 }
1588 }
1589
1590 // Emit the standard function epilogue.
1591 FinishFunction(EndLoc: BodyRange.getEnd());
1592
1593 // If we haven't marked the function nothrow through other means, do
1594 // a quick pass now to see if we can.
1595 if (!CurFn->doesNotThrow())
1596 TryMarkNoThrow(F: CurFn);
1597}
1598
1599/// ContainsLabel - Return true if the statement contains a label in it. If
1600/// this statement is not executed normally, it not containing a label means
1601/// that we can just remove the code.
1602bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
1603 // Null statement, not a label!
1604 if (!S) return false;
1605
1606 // If this is a label, we have to emit the code, consider something like:
1607 // if (0) { ... foo: bar(); } goto foo;
1608 //
1609 // TODO: If anyone cared, we could track __label__'s, since we know that you
1610 // can't jump to one from outside their declared region.
1611 if (isa<LabelStmt>(Val: S))
1612 return true;
1613
1614 // If this is a case/default statement, and we haven't seen a switch, we have
1615 // to emit the code.
1616 if (isa<SwitchCase>(Val: S) && !IgnoreCaseStmts)
1617 return true;
1618
1619 // If this is a switch statement, we want to ignore cases below it.
1620 if (isa<SwitchStmt>(Val: S))
1621 IgnoreCaseStmts = true;
1622
1623 // Scan subexpressions for verboten labels.
1624 for (const Stmt *SubStmt : S->children())
1625 if (ContainsLabel(S: SubStmt, IgnoreCaseStmts))
1626 return true;
1627
1628 return false;
1629}
1630
1631/// containsBreak - Return true if the statement contains a break out of it.
1632/// If the statement (recursively) contains a switch or loop with a break
1633/// inside of it, this is fine.
1634bool CodeGenFunction::containsBreak(const Stmt *S) {
1635 // Null statement, not a label!
1636 if (!S) return false;
1637
1638 // If this is a switch or loop that defines its own break scope, then we can
1639 // include it and anything inside of it.
1640 if (isa<SwitchStmt>(Val: S) || isa<WhileStmt>(Val: S) || isa<DoStmt>(Val: S) ||
1641 isa<ForStmt>(Val: S))
1642 return false;
1643
1644 if (isa<BreakStmt>(Val: S))
1645 return true;
1646
1647 // Scan subexpressions for verboten breaks.
1648 for (const Stmt *SubStmt : S->children())
1649 if (containsBreak(S: SubStmt))
1650 return true;
1651
1652 return false;
1653}
1654
1655bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) {
1656 if (!S) return false;
1657
1658 // Some statement kinds add a scope and thus never add a decl to the current
1659 // scope. Note, this list is longer than the list of statements that might
1660 // have an unscoped decl nested within them, but this way is conservatively
1661 // correct even if more statement kinds are added.
1662 if (isa<IfStmt>(Val: S) || isa<SwitchStmt>(Val: S) || isa<WhileStmt>(Val: S) ||
1663 isa<DoStmt>(Val: S) || isa<ForStmt>(Val: S) || isa<CompoundStmt>(Val: S) ||
1664 isa<CXXForRangeStmt>(Val: S) || isa<CXXTryStmt>(Val: S) ||
1665 isa<ObjCForCollectionStmt>(Val: S) || isa<ObjCAtTryStmt>(Val: S))
1666 return false;
1667
1668 if (isa<DeclStmt>(Val: S))
1669 return true;
1670
1671 for (const Stmt *SubStmt : S->children())
1672 if (mightAddDeclToScope(S: SubStmt))
1673 return true;
1674
1675 return false;
1676}
1677
1678/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1679/// to a constant, or if it does but contains a label, return false. If it
1680/// constant folds return true and set the boolean result in Result.
1681bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1682 bool &ResultBool,
1683 bool AllowLabels) {
1684 // If MC/DC is enabled, disable folding so that we can instrument all
1685 // conditions to yield complete test vectors. We still keep track of
1686 // folded conditions during region mapping and visualization.
1687 if (!AllowLabels && CGM.getCodeGenOpts().hasProfileClangInstr() &&
1688 CGM.getCodeGenOpts().MCDCCoverage)
1689 return false;
1690
1691 llvm::APSInt ResultInt;
1692 if (!ConstantFoldsToSimpleInteger(Cond, Result&: ResultInt, AllowLabels))
1693 return false;
1694
1695 ResultBool = ResultInt.getBoolValue();
1696 return true;
1697}
1698
1699/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1700/// to a constant, or if it does but contains a label, return false. If it
1701/// constant folds return true and set the folded value.
1702bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1703 llvm::APSInt &ResultInt,
1704 bool AllowLabels) {
1705 // FIXME: Rename and handle conversion of other evaluatable things
1706 // to bool.
1707 Expr::EvalResult Result;
1708 if (!Cond->EvaluateAsInt(Result, Ctx: getContext()))
1709 return false; // Not foldable, not integer or not fully evaluatable.
1710
1711 llvm::APSInt Int = Result.Val.getInt();
1712 if (!AllowLabels && CodeGenFunction::ContainsLabel(S: Cond))
1713 return false; // Contains a label.
1714
1715 ResultInt = Int;
1716 return true;
1717}
1718
1719/// Strip parentheses and simplistic logical-NOT operators.
1720const Expr *CodeGenFunction::stripCond(const Expr *C) {
1721 while (const UnaryOperator *Op = dyn_cast<UnaryOperator>(Val: C->IgnoreParens())) {
1722 if (Op->getOpcode() != UO_LNot)
1723 break;
1724 C = Op->getSubExpr();
1725 }
1726 return C->IgnoreParens();
1727}
1728
1729/// Determine whether the given condition is an instrumentable condition
1730/// (i.e. no "&&" or "||").
1731bool CodeGenFunction::isInstrumentedCondition(const Expr *C) {
1732 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: stripCond(C));
1733 return (!BOp || !BOp->isLogicalOp());
1734}
1735
1736/// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
1737/// increments a profile counter based on the semantics of the given logical
1738/// operator opcode. This is used to instrument branch condition coverage for
1739/// logical operators.
1740void CodeGenFunction::EmitBranchToCounterBlock(
1741 const Expr *Cond, BinaryOperator::Opcode LOp, llvm::BasicBlock *TrueBlock,
1742 llvm::BasicBlock *FalseBlock, uint64_t TrueCount /* = 0 */,
1743 Stmt::Likelihood LH /* =None */, const Expr *CntrIdx /* = nullptr */) {
1744 // If not instrumenting, just emit a branch.
1745 bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr();
1746 if (!InstrumentRegions || !isInstrumentedCondition(C: Cond))
1747 return EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount, LH);
1748
1749 llvm::BasicBlock *ThenBlock = nullptr;
1750 llvm::BasicBlock *ElseBlock = nullptr;
1751 llvm::BasicBlock *NextBlock = nullptr;
1752
1753 // Create the block we'll use to increment the appropriate counter.
1754 llvm::BasicBlock *CounterIncrBlock = createBasicBlock(name: "lop.rhscnt");
1755
1756 // Set block pointers according to Logical-AND (BO_LAnd) semantics. This
1757 // means we need to evaluate the condition and increment the counter on TRUE:
1758 //
1759 // if (Cond)
1760 // goto CounterIncrBlock;
1761 // else
1762 // goto FalseBlock;
1763 //
1764 // CounterIncrBlock:
1765 // Counter++;
1766 // goto TrueBlock;
1767
1768 if (LOp == BO_LAnd) {
1769 ThenBlock = CounterIncrBlock;
1770 ElseBlock = FalseBlock;
1771 NextBlock = TrueBlock;
1772 }
1773
1774 // Set block pointers according to Logical-OR (BO_LOr) semantics. This means
1775 // we need to evaluate the condition and increment the counter on FALSE:
1776 //
1777 // if (Cond)
1778 // goto TrueBlock;
1779 // else
1780 // goto CounterIncrBlock;
1781 //
1782 // CounterIncrBlock:
1783 // Counter++;
1784 // goto FalseBlock;
1785
1786 else if (LOp == BO_LOr) {
1787 ThenBlock = TrueBlock;
1788 ElseBlock = CounterIncrBlock;
1789 NextBlock = FalseBlock;
1790 } else {
1791 llvm_unreachable("Expected Opcode must be that of a Logical Operator");
1792 }
1793
1794 // Emit Branch based on condition.
1795 EmitBranchOnBoolExpr(Cond, TrueBlock: ThenBlock, FalseBlock: ElseBlock, TrueCount, LH);
1796
1797 // Emit the block containing the counter increment(s).
1798 EmitBlock(BB: CounterIncrBlock);
1799
1800 // Increment corresponding counter; if index not provided, use Cond as index.
1801 incrementProfileCounter(S: CntrIdx ? CntrIdx : Cond);
1802
1803 // Go to the next block.
1804 EmitBranch(Block: NextBlock);
1805}
1806
1807/// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an if
1808/// statement) to the specified blocks. Based on the condition, this might try
1809/// to simplify the codegen of the conditional based on the branch.
1810/// \param LH The value of the likelihood attribute on the True branch.
1811/// \param ConditionalOp Used by MC/DC code coverage to track the result of the
1812/// ConditionalOperator (ternary) through a recursive call for the operator's
1813/// LHS and RHS nodes.
1814void CodeGenFunction::EmitBranchOnBoolExpr(
1815 const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock,
1816 uint64_t TrueCount, Stmt::Likelihood LH, const Expr *ConditionalOp) {
1817 Cond = Cond->IgnoreParens();
1818
1819 if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Val: Cond)) {
1820 // Handle X && Y in a condition.
1821 if (CondBOp->getOpcode() == BO_LAnd) {
1822 MCDCLogOpStack.push_back(Elt: CondBOp);
1823
1824 // If we have "1 && X", simplify the code. "0 && X" would have constant
1825 // folded if the case was simple enough.
1826 bool ConstantBool = false;
1827 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getLHS(), ResultBool&: ConstantBool) &&
1828 ConstantBool) {
1829 // br(1 && X) -> br(X).
1830 incrementProfileCounter(S: CondBOp);
1831 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LAnd, TrueBlock,
1832 FalseBlock, TrueCount, LH);
1833 MCDCLogOpStack.pop_back();
1834 return;
1835 }
1836
1837 // If we have "X && 1", simplify the code to use an uncond branch.
1838 // "X && 0" would have been constant folded to 0.
1839 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getRHS(), ResultBool&: ConstantBool) &&
1840 ConstantBool) {
1841 // br(X && 1) -> br(X).
1842 EmitBranchToCounterBlock(Cond: CondBOp->getLHS(), LOp: BO_LAnd, TrueBlock,
1843 FalseBlock, TrueCount, LH, CntrIdx: CondBOp);
1844 MCDCLogOpStack.pop_back();
1845 return;
1846 }
1847
1848 // Emit the LHS as a conditional. If the LHS conditional is false, we
1849 // want to jump to the FalseBlock.
1850 llvm::BasicBlock *LHSTrue = createBasicBlock(name: "land.lhs.true");
1851 // The counter tells us how often we evaluate RHS, and all of TrueCount
1852 // can be propagated to that branch.
1853 uint64_t RHSCount = getProfileCount(S: CondBOp->getRHS());
1854
1855 ConditionalEvaluation eval(*this);
1856 {
1857 ApplyDebugLocation DL(*this, Cond);
1858 // Propagate the likelihood attribute like __builtin_expect
1859 // __builtin_expect(X && Y, 1) -> X and Y are likely
1860 // __builtin_expect(X && Y, 0) -> only Y is unlikely
1861 EmitBranchOnBoolExpr(Cond: CondBOp->getLHS(), TrueBlock: LHSTrue, FalseBlock, TrueCount: RHSCount,
1862 LH: LH == Stmt::LH_Unlikely ? Stmt::LH_None : LH);
1863 EmitBlock(BB: LHSTrue);
1864 }
1865
1866 incrementProfileCounter(S: CondBOp);
1867 setCurrentProfileCount(getProfileCount(S: CondBOp->getRHS()));
1868
1869 // Any temporaries created here are conditional.
1870 eval.begin(CGF&: *this);
1871 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LAnd, TrueBlock,
1872 FalseBlock, TrueCount, LH);
1873 eval.end(CGF&: *this);
1874 MCDCLogOpStack.pop_back();
1875 return;
1876 }
1877
1878 if (CondBOp->getOpcode() == BO_LOr) {
1879 MCDCLogOpStack.push_back(Elt: CondBOp);
1880
1881 // If we have "0 || X", simplify the code. "1 || X" would have constant
1882 // folded if the case was simple enough.
1883 bool ConstantBool = false;
1884 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getLHS(), ResultBool&: ConstantBool) &&
1885 !ConstantBool) {
1886 // br(0 || X) -> br(X).
1887 incrementProfileCounter(S: CondBOp);
1888 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LOr, TrueBlock,
1889 FalseBlock, TrueCount, LH);
1890 MCDCLogOpStack.pop_back();
1891 return;
1892 }
1893
1894 // If we have "X || 0", simplify the code to use an uncond branch.
1895 // "X || 1" would have been constant folded to 1.
1896 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getRHS(), ResultBool&: ConstantBool) &&
1897 !ConstantBool) {
1898 // br(X || 0) -> br(X).
1899 EmitBranchToCounterBlock(Cond: CondBOp->getLHS(), LOp: BO_LOr, TrueBlock,
1900 FalseBlock, TrueCount, LH, CntrIdx: CondBOp);
1901 MCDCLogOpStack.pop_back();
1902 return;
1903 }
1904 // Emit the LHS as a conditional. If the LHS conditional is true, we
1905 // want to jump to the TrueBlock.
1906 llvm::BasicBlock *LHSFalse = createBasicBlock(name: "lor.lhs.false");
1907 // We have the count for entry to the RHS and for the whole expression
1908 // being true, so we can divy up True count between the short circuit and
1909 // the RHS.
1910 uint64_t LHSCount =
1911 getCurrentProfileCount() - getProfileCount(S: CondBOp->getRHS());
1912 uint64_t RHSCount = TrueCount - LHSCount;
1913
1914 ConditionalEvaluation eval(*this);
1915 {
1916 // Propagate the likelihood attribute like __builtin_expect
1917 // __builtin_expect(X || Y, 1) -> only Y is likely
1918 // __builtin_expect(X || Y, 0) -> both X and Y are unlikely
1919 ApplyDebugLocation DL(*this, Cond);
1920 EmitBranchOnBoolExpr(Cond: CondBOp->getLHS(), TrueBlock, FalseBlock: LHSFalse, TrueCount: LHSCount,
1921 LH: LH == Stmt::LH_Likely ? Stmt::LH_None : LH);
1922 EmitBlock(BB: LHSFalse);
1923 }
1924
1925 incrementProfileCounter(S: CondBOp);
1926 setCurrentProfileCount(getProfileCount(S: CondBOp->getRHS()));
1927
1928 // Any temporaries created here are conditional.
1929 eval.begin(CGF&: *this);
1930 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LOr, TrueBlock, FalseBlock,
1931 TrueCount: RHSCount, LH);
1932
1933 eval.end(CGF&: *this);
1934 MCDCLogOpStack.pop_back();
1935 return;
1936 }
1937 }
1938
1939 if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Val: Cond)) {
1940 // br(!x, t, f) -> br(x, f, t)
1941 // Avoid doing this optimization when instrumenting a condition for MC/DC.
1942 // LNot is taken as part of the condition for simplicity, and changing its
1943 // sense negatively impacts test vector tracking.
1944 bool MCDCCondition = CGM.getCodeGenOpts().hasProfileClangInstr() &&
1945 CGM.getCodeGenOpts().MCDCCoverage &&
1946 isInstrumentedCondition(C: Cond);
1947 if (CondUOp->getOpcode() == UO_LNot && !MCDCCondition) {
1948 // Negate the count.
1949 uint64_t FalseCount = getCurrentProfileCount() - TrueCount;
1950 // The values of the enum are chosen to make this negation possible.
1951 LH = static_cast<Stmt::Likelihood>(-LH);
1952 // Negate the condition and swap the destination blocks.
1953 return EmitBranchOnBoolExpr(Cond: CondUOp->getSubExpr(), TrueBlock: FalseBlock, FalseBlock: TrueBlock,
1954 TrueCount: FalseCount, LH);
1955 }
1956 }
1957
1958 if (const ConditionalOperator *CondOp = dyn_cast<ConditionalOperator>(Val: Cond)) {
1959 // br(c ? x : y, t, f) -> br(c, br(x, t, f), br(y, t, f))
1960 llvm::BasicBlock *LHSBlock = createBasicBlock(name: "cond.true");
1961 llvm::BasicBlock *RHSBlock = createBasicBlock(name: "cond.false");
1962
1963 // The ConditionalOperator itself has no likelihood information for its
1964 // true and false branches. This matches the behavior of __builtin_expect.
1965 ConditionalEvaluation cond(*this);
1966 EmitBranchOnBoolExpr(Cond: CondOp->getCond(), TrueBlock: LHSBlock, FalseBlock: RHSBlock,
1967 TrueCount: getProfileCount(S: CondOp), LH: Stmt::LH_None);
1968
1969 // When computing PGO branch weights, we only know the overall count for
1970 // the true block. This code is essentially doing tail duplication of the
1971 // naive code-gen, introducing new edges for which counts are not
1972 // available. Divide the counts proportionally between the LHS and RHS of
1973 // the conditional operator.
1974 uint64_t LHSScaledTrueCount = 0;
1975 if (TrueCount) {
1976 double LHSRatio =
1977 getProfileCount(S: CondOp) / (double)getCurrentProfileCount();
1978 LHSScaledTrueCount = TrueCount * LHSRatio;
1979 }
1980
1981 cond.begin(CGF&: *this);
1982 EmitBlock(BB: LHSBlock);
1983 incrementProfileCounter(S: CondOp);
1984 {
1985 ApplyDebugLocation DL(*this, Cond);
1986 EmitBranchOnBoolExpr(Cond: CondOp->getLHS(), TrueBlock, FalseBlock,
1987 TrueCount: LHSScaledTrueCount, LH, ConditionalOp: CondOp);
1988 }
1989 cond.end(CGF&: *this);
1990
1991 cond.begin(CGF&: *this);
1992 EmitBlock(BB: RHSBlock);
1993 EmitBranchOnBoolExpr(Cond: CondOp->getRHS(), TrueBlock, FalseBlock,
1994 TrueCount: TrueCount - LHSScaledTrueCount, LH, ConditionalOp: CondOp);
1995 cond.end(CGF&: *this);
1996
1997 return;
1998 }
1999
2000 if (const CXXThrowExpr *Throw = dyn_cast<CXXThrowExpr>(Val: Cond)) {
2001 // Conditional operator handling can give us a throw expression as a
2002 // condition for a case like:
2003 // br(c ? throw x : y, t, f) -> br(c, br(throw x, t, f), br(y, t, f)
2004 // Fold this to:
2005 // br(c, throw x, br(y, t, f))
2006 EmitCXXThrowExpr(E: Throw, /*KeepInsertionPoint*/false);
2007 return;
2008 }
2009
2010 // Emit the code with the fully general case.
2011 llvm::Value *CondV;
2012 {
2013 ApplyDebugLocation DL(*this, Cond);
2014 CondV = EvaluateExprAsBool(E: Cond);
2015 }
2016
2017 // If not at the top of the logical operator nest, update MCDC temp with the
2018 // boolean result of the evaluated condition.
2019 if (!MCDCLogOpStack.empty()) {
2020 const Expr *MCDCBaseExpr = Cond;
2021 // When a nested ConditionalOperator (ternary) is encountered in a boolean
2022 // expression, MC/DC tracks the result of the ternary, and this is tied to
2023 // the ConditionalOperator expression and not the ternary's LHS or RHS. If
2024 // this is the case, the ConditionalOperator expression is passed through
2025 // the ConditionalOp parameter and then used as the MCDC base expression.
2026 if (ConditionalOp)
2027 MCDCBaseExpr = ConditionalOp;
2028
2029 maybeUpdateMCDCCondBitmap(E: MCDCBaseExpr, Val: CondV);
2030 }
2031
2032 llvm::MDNode *Weights = nullptr;
2033 llvm::MDNode *Unpredictable = nullptr;
2034
2035 // If the branch has a condition wrapped by __builtin_unpredictable,
2036 // create metadata that specifies that the branch is unpredictable.
2037 // Don't bother if not optimizing because that metadata would not be used.
2038 auto *Call = dyn_cast<CallExpr>(Val: Cond->IgnoreImpCasts());
2039 if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
2040 auto *FD = dyn_cast_or_null<FunctionDecl>(Val: Call->getCalleeDecl());
2041 if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
2042 llvm::MDBuilder MDHelper(getLLVMContext());
2043 Unpredictable = MDHelper.createUnpredictable();
2044 }
2045 }
2046
2047 // If there is a Likelihood knowledge for the cond, lower it.
2048 // Note that if not optimizing this won't emit anything.
2049 llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(Cond: CondV, LH);
2050 if (CondV != NewCondV)
2051 CondV = NewCondV;
2052 else {
2053 // Otherwise, lower profile counts. Note that we do this even at -O0.
2054 uint64_t CurrentCount = std::max(a: getCurrentProfileCount(), b: TrueCount);
2055 Weights = createProfileWeights(TrueCount, FalseCount: CurrentCount - TrueCount);
2056 }
2057
2058 Builder.CreateCondBr(Cond: CondV, True: TrueBlock, False: FalseBlock, BranchWeights: Weights, Unpredictable);
2059}
2060
2061/// ErrorUnsupported - Print out an error that codegen doesn't support the
2062/// specified stmt yet.
2063void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) {
2064 CGM.ErrorUnsupported(S, Type);
2065}
2066
2067/// emitNonZeroVLAInit - Emit the "zero" initialization of a
2068/// variable-length array whose elements have a non-zero bit-pattern.
2069///
2070/// \param baseType the inner-most element type of the array
2071/// \param src - a char* pointing to the bit-pattern for a single
2072/// base element of the array
2073/// \param sizeInChars - the total size of the VLA, in chars
2074static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
2075 Address dest, Address src,
2076 llvm::Value *sizeInChars) {
2077 CGBuilderTy &Builder = CGF.Builder;
2078
2079 CharUnits baseSize = CGF.getContext().getTypeSizeInChars(T: baseType);
2080 llvm::Value *baseSizeInChars
2081 = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: baseSize.getQuantity());
2082
2083 Address begin = dest.withElementType(ElemTy: CGF.Int8Ty);
2084 llvm::Value *end = Builder.CreateInBoundsGEP(Ty: begin.getElementType(),
2085 Ptr: begin.emitRawPointer(CGF),
2086 IdxList: sizeInChars, Name: "vla.end");
2087
2088 llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
2089 llvm::BasicBlock *loopBB = CGF.createBasicBlock(name: "vla-init.loop");
2090 llvm::BasicBlock *contBB = CGF.createBasicBlock(name: "vla-init.cont");
2091
2092 // Make a loop over the VLA. C99 guarantees that the VLA element
2093 // count must be nonzero.
2094 CGF.EmitBlock(BB: loopBB);
2095
2096 llvm::PHINode *cur = Builder.CreatePHI(Ty: begin.getType(), NumReservedValues: 2, Name: "vla.cur");
2097 cur->addIncoming(V: begin.emitRawPointer(CGF), BB: originBB);
2098
2099 CharUnits curAlign =
2100 dest.getAlignment().alignmentOfArrayElement(elementSize: baseSize);
2101
2102 // memcpy the individual element bit-pattern.
2103 Builder.CreateMemCpy(Dest: Address(cur, CGF.Int8Ty, curAlign), Src: src, Size: baseSizeInChars,
2104 /*volatile*/ IsVolatile: false);
2105
2106 // Go to the next element.
2107 llvm::Value *next =
2108 Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: cur, IdxList: baseSizeInChars, Name: "vla.next");
2109
2110 // Leave if that's the end of the VLA.
2111 llvm::Value *done = Builder.CreateICmpEQ(LHS: next, RHS: end, Name: "vla-init.isdone");
2112 Builder.CreateCondBr(Cond: done, True: contBB, False: loopBB);
2113 cur->addIncoming(V: next, BB: loopBB);
2114
2115 CGF.EmitBlock(BB: contBB);
2116}
2117
2118void
2119CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
2120 // Ignore empty classes in C++.
2121 if (getLangOpts().CPlusPlus) {
2122 if (const RecordType *RT = Ty->getAs<RecordType>()) {
2123 if (cast<CXXRecordDecl>(Val: RT->getDecl())->isEmpty())
2124 return;
2125 }
2126 }
2127
2128 if (DestPtr.getElementType() != Int8Ty)
2129 DestPtr = DestPtr.withElementType(ElemTy: Int8Ty);
2130
2131 // Get size and alignment info for this aggregate.
2132 CharUnits size = getContext().getTypeSizeInChars(T: Ty);
2133
2134 llvm::Value *SizeVal;
2135 const VariableArrayType *vla;
2136
2137 // Don't bother emitting a zero-byte memset.
2138 if (size.isZero()) {
2139 // But note that getTypeInfo returns 0 for a VLA.
2140 if (const VariableArrayType *vlaType =
2141 dyn_cast_or_null<VariableArrayType>(
2142 Val: getContext().getAsArrayType(T: Ty))) {
2143 auto VlaSize = getVLASize(vla: vlaType);
2144 SizeVal = VlaSize.NumElts;
2145 CharUnits eltSize = getContext().getTypeSizeInChars(T: VlaSize.Type);
2146 if (!eltSize.isOne())
2147 SizeVal = Builder.CreateNUWMul(LHS: SizeVal, RHS: CGM.getSize(numChars: eltSize));
2148 vla = vlaType;
2149 } else {
2150 return;
2151 }
2152 } else {
2153 SizeVal = CGM.getSize(numChars: size);
2154 vla = nullptr;
2155 }
2156
2157 // If the type contains a pointer to data member we can't memset it to zero.
2158 // Instead, create a null constant and copy it to the destination.
2159 // TODO: there are other patterns besides zero that we can usefully memset,
2160 // like -1, which happens to be the pattern used by member-pointers.
2161 if (!CGM.getTypes().isZeroInitializable(T: Ty)) {
2162 // For a VLA, emit a single element, then splat that over the VLA.
2163 if (vla) Ty = getContext().getBaseElementType(VAT: vla);
2164
2165 llvm::Constant *NullConstant = CGM.EmitNullConstant(T: Ty);
2166
2167 llvm::GlobalVariable *NullVariable =
2168 new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
2169 /*isConstant=*/true,
2170 llvm::GlobalVariable::PrivateLinkage,
2171 NullConstant, Twine());
2172 CharUnits NullAlign = DestPtr.getAlignment();
2173 NullVariable->setAlignment(NullAlign.getAsAlign());
2174 Address SrcPtr(NullVariable, Builder.getInt8Ty(), NullAlign);
2175
2176 if (vla) return emitNonZeroVLAInit(CGF&: *this, baseType: Ty, dest: DestPtr, src: SrcPtr, sizeInChars: SizeVal);
2177
2178 // Get and call the appropriate llvm.memcpy overload.
2179 Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: SizeVal, IsVolatile: false);
2180 return;
2181 }
2182
2183 // Otherwise, just memset the whole thing to zero. This is legal
2184 // because in LLVM, all default initializers (other than the ones we just
2185 // handled above) are guaranteed to have a bit pattern of all zeros.
2186 Builder.CreateMemSet(Dest: DestPtr, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
2187}
2188
2189llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
2190 // Make sure that there is a block for the indirect goto.
2191 if (!IndirectBranch)
2192 GetIndirectGotoBlock();
2193
2194 llvm::BasicBlock *BB = getJumpDestForLabel(S: L).getBlock();
2195
2196 // Make sure the indirect branch includes all of the address-taken blocks.
2197 IndirectBranch->addDestination(Dest: BB);
2198 return llvm::BlockAddress::get(F: CurFn, BB);
2199}
2200
2201llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {
2202 // If we already made the indirect branch for indirect goto, return its block.
2203 if (IndirectBranch) return IndirectBranch->getParent();
2204
2205 CGBuilderTy TmpBuilder(*this, createBasicBlock(name: "indirectgoto"));
2206
2207 // Create the PHI node that indirect gotos will add entries to.
2208 llvm::Value *DestVal = TmpBuilder.CreatePHI(Ty: Int8PtrTy, NumReservedValues: 0,
2209 Name: "indirect.goto.dest");
2210
2211 // Create the indirect branch instruction.
2212 IndirectBranch = TmpBuilder.CreateIndirectBr(Addr: DestVal);
2213 return IndirectBranch->getParent();
2214}
2215
2216/// Computes the length of an array in elements, as well as the base
2217/// element type and a properly-typed first element pointer.
2218llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
2219 QualType &baseType,
2220 Address &addr) {
2221 const ArrayType *arrayType = origArrayType;
2222
2223 // If it's a VLA, we have to load the stored size. Note that
2224 // this is the size of the VLA in bytes, not its size in elements.
2225 llvm::Value *numVLAElements = nullptr;
2226 if (isa<VariableArrayType>(Val: arrayType)) {
2227 numVLAElements = getVLASize(vla: cast<VariableArrayType>(Val: arrayType)).NumElts;
2228
2229 // Walk into all VLAs. This doesn't require changes to addr,
2230 // which has type T* where T is the first non-VLA element type.
2231 do {
2232 QualType elementType = arrayType->getElementType();
2233 arrayType = getContext().getAsArrayType(T: elementType);
2234
2235 // If we only have VLA components, 'addr' requires no adjustment.
2236 if (!arrayType) {
2237 baseType = elementType;
2238 return numVLAElements;
2239 }
2240 } while (isa<VariableArrayType>(Val: arrayType));
2241
2242 // We get out here only if we find a constant array type
2243 // inside the VLA.
2244 }
2245
2246 // We have some number of constant-length arrays, so addr should
2247 // have LLVM type [M x [N x [...]]]*. Build a GEP that walks
2248 // down to the first element of addr.
2249 SmallVector<llvm::Value*, 8> gepIndices;
2250
2251 // GEP down to the array type.
2252 llvm::ConstantInt *zero = Builder.getInt32(C: 0);
2253 gepIndices.push_back(Elt: zero);
2254
2255 uint64_t countFromCLAs = 1;
2256 QualType eltType;
2257
2258 llvm::ArrayType *llvmArrayType =
2259 dyn_cast<llvm::ArrayType>(Val: addr.getElementType());
2260 while (llvmArrayType) {
2261 assert(isa<ConstantArrayType>(arrayType));
2262 assert(cast<ConstantArrayType>(arrayType)->getZExtSize() ==
2263 llvmArrayType->getNumElements());
2264
2265 gepIndices.push_back(Elt: zero);
2266 countFromCLAs *= llvmArrayType->getNumElements();
2267 eltType = arrayType->getElementType();
2268
2269 llvmArrayType =
2270 dyn_cast<llvm::ArrayType>(Val: llvmArrayType->getElementType());
2271 arrayType = getContext().getAsArrayType(T: arrayType->getElementType());
2272 assert((!llvmArrayType || arrayType) &&
2273 "LLVM and Clang types are out-of-synch");
2274 }
2275
2276 if (arrayType) {
2277 // From this point onwards, the Clang array type has been emitted
2278 // as some other type (probably a packed struct). Compute the array
2279 // size, and just emit the 'begin' expression as a bitcast.
2280 while (arrayType) {
2281 countFromCLAs *= cast<ConstantArrayType>(Val: arrayType)->getZExtSize();
2282 eltType = arrayType->getElementType();
2283 arrayType = getContext().getAsArrayType(T: eltType);
2284 }
2285
2286 llvm::Type *baseType = ConvertType(T: eltType);
2287 addr = addr.withElementType(ElemTy: baseType);
2288 } else {
2289 // Create the actual GEP.
2290 addr = Address(Builder.CreateInBoundsGEP(Ty: addr.getElementType(),
2291 Ptr: addr.emitRawPointer(CGF&: *this),
2292 IdxList: gepIndices, Name: "array.begin"),
2293 ConvertTypeForMem(T: eltType), addr.getAlignment());
2294 }
2295
2296 baseType = eltType;
2297
2298 llvm::Value *numElements
2299 = llvm::ConstantInt::get(Ty: SizeTy, V: countFromCLAs);
2300
2301 // If we had any VLA dimensions, factor them in.
2302 if (numVLAElements)
2303 numElements = Builder.CreateNUWMul(LHS: numVLAElements, RHS: numElements);
2304
2305 return numElements;
2306}
2307
2308CodeGenFunction::VlaSizePair CodeGenFunction::getVLASize(QualType type) {
2309 const VariableArrayType *vla = getContext().getAsVariableArrayType(T: type);
2310 assert(vla && "type was not a variable array type!");
2311 return getVLASize(vla);
2312}
2313
2314CodeGenFunction::VlaSizePair
2315CodeGenFunction::getVLASize(const VariableArrayType *type) {
2316 // The number of elements so far; always size_t.
2317 llvm::Value *numElements = nullptr;
2318
2319 QualType elementType;
2320 do {
2321 elementType = type->getElementType();
2322 llvm::Value *vlaSize = VLASizeMap[type->getSizeExpr()];
2323 assert(vlaSize && "no size for VLA!");
2324 assert(vlaSize->getType() == SizeTy);
2325
2326 if (!numElements) {
2327 numElements = vlaSize;
2328 } else {
2329 // It's undefined behavior if this wraps around, so mark it that way.
2330 // FIXME: Teach -fsanitize=undefined to trap this.
2331 numElements = Builder.CreateNUWMul(LHS: numElements, RHS: vlaSize);
2332 }
2333 } while ((type = getContext().getAsVariableArrayType(T: elementType)));
2334
2335 return { numElements, elementType };
2336}
2337
2338CodeGenFunction::VlaSizePair
2339CodeGenFunction::getVLAElements1D(QualType type) {
2340 const VariableArrayType *vla = getContext().getAsVariableArrayType(T: type);
2341 assert(vla && "type was not a variable array type!");
2342 return getVLAElements1D(vla);
2343}
2344
2345CodeGenFunction::VlaSizePair
2346CodeGenFunction::getVLAElements1D(const VariableArrayType *Vla) {
2347 llvm::Value *VlaSize = VLASizeMap[Vla->getSizeExpr()];
2348 assert(VlaSize && "no size for VLA!");
2349 assert(VlaSize->getType() == SizeTy);
2350 return { VlaSize, Vla->getElementType() };
2351}
2352
2353void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
2354 assert(type->isVariablyModifiedType() &&
2355 "Must pass variably modified type to EmitVLASizes!");
2356
2357 EnsureInsertPoint();
2358
2359 // We're going to walk down into the type and look for VLA
2360 // expressions.
2361 do {
2362 assert(type->isVariablyModifiedType());
2363
2364 const Type *ty = type.getTypePtr();
2365 switch (ty->getTypeClass()) {
2366
2367#define TYPE(Class, Base)
2368#define ABSTRACT_TYPE(Class, Base)
2369#define NON_CANONICAL_TYPE(Class, Base)
2370#define DEPENDENT_TYPE(Class, Base) case Type::Class:
2371#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
2372#include "clang/AST/TypeNodes.inc"
2373 llvm_unreachable("unexpected dependent type!");
2374
2375 // These types are never variably-modified.
2376 case Type::Builtin:
2377 case Type::Complex:
2378 case Type::Vector:
2379 case Type::ExtVector:
2380 case Type::ConstantMatrix:
2381 case Type::Record:
2382 case Type::Enum:
2383 case Type::Using:
2384 case Type::TemplateSpecialization:
2385 case Type::ObjCTypeParam:
2386 case Type::ObjCObject:
2387 case Type::ObjCInterface:
2388 case Type::ObjCObjectPointer:
2389 case Type::BitInt:
2390 llvm_unreachable("type class is never variably-modified!");
2391
2392 case Type::Elaborated:
2393 type = cast<ElaboratedType>(Val: ty)->getNamedType();
2394 break;
2395
2396 case Type::Adjusted:
2397 type = cast<AdjustedType>(Val: ty)->getAdjustedType();
2398 break;
2399
2400 case Type::Decayed:
2401 type = cast<DecayedType>(Val: ty)->getPointeeType();
2402 break;
2403
2404 case Type::Pointer:
2405 type = cast<PointerType>(Val: ty)->getPointeeType();
2406 break;
2407
2408 case Type::BlockPointer:
2409 type = cast<BlockPointerType>(Val: ty)->getPointeeType();
2410 break;
2411
2412 case Type::LValueReference:
2413 case Type::RValueReference:
2414 type = cast<ReferenceType>(Val: ty)->getPointeeType();
2415 break;
2416
2417 case Type::MemberPointer:
2418 type = cast<MemberPointerType>(Val: ty)->getPointeeType();
2419 break;
2420
2421 case Type::ArrayParameter:
2422 case Type::ConstantArray:
2423 case Type::IncompleteArray:
2424 // Losing element qualification here is fine.
2425 type = cast<ArrayType>(Val: ty)->getElementType();
2426 break;
2427
2428 case Type::VariableArray: {
2429 // Losing element qualification here is fine.
2430 const VariableArrayType *vat = cast<VariableArrayType>(Val: ty);
2431
2432 // Unknown size indication requires no size computation.
2433 // Otherwise, evaluate and record it.
2434 if (const Expr *sizeExpr = vat->getSizeExpr()) {
2435 // It's possible that we might have emitted this already,
2436 // e.g. with a typedef and a pointer to it.
2437 llvm::Value *&entry = VLASizeMap[sizeExpr];
2438 if (!entry) {
2439 llvm::Value *size = EmitScalarExpr(E: sizeExpr);
2440
2441 // C11 6.7.6.2p5:
2442 // If the size is an expression that is not an integer constant
2443 // expression [...] each time it is evaluated it shall have a value
2444 // greater than zero.
2445 if (SanOpts.has(K: SanitizerKind::VLABound)) {
2446 SanitizerScope SanScope(this);
2447 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: size->getType());
2448 clang::QualType SEType = sizeExpr->getType();
2449 llvm::Value *CheckCondition =
2450 SEType->isSignedIntegerType()
2451 ? Builder.CreateICmpSGT(LHS: size, RHS: Zero)
2452 : Builder.CreateICmpUGT(LHS: size, RHS: Zero);
2453 llvm::Constant *StaticArgs[] = {
2454 EmitCheckSourceLocation(Loc: sizeExpr->getBeginLoc()),
2455 EmitCheckTypeDescriptor(T: SEType)};
2456 EmitCheck(Checked: std::make_pair(x&: CheckCondition, y: SanitizerKind::VLABound),
2457 Check: SanitizerHandler::VLABoundNotPositive, StaticArgs, DynamicArgs: size);
2458 }
2459
2460 // Always zexting here would be wrong if it weren't
2461 // undefined behavior to have a negative bound.
2462 // FIXME: What about when size's type is larger than size_t?
2463 entry = Builder.CreateIntCast(V: size, DestTy: SizeTy, /*signed*/ isSigned: false);
2464 }
2465 }
2466 type = vat->getElementType();
2467 break;
2468 }
2469
2470 case Type::FunctionProto:
2471 case Type::FunctionNoProto:
2472 type = cast<FunctionType>(Val: ty)->getReturnType();
2473 break;
2474
2475 case Type::Paren:
2476 case Type::TypeOf:
2477 case Type::UnaryTransform:
2478 case Type::Attributed:
2479 case Type::BTFTagAttributed:
2480 case Type::SubstTemplateTypeParm:
2481 case Type::MacroQualified:
2482 case Type::CountAttributed:
2483 // Keep walking after single level desugaring.
2484 type = type.getSingleStepDesugaredType(Context: getContext());
2485 break;
2486
2487 case Type::Typedef:
2488 case Type::Decltype:
2489 case Type::Auto:
2490 case Type::DeducedTemplateSpecialization:
2491 case Type::PackIndexing:
2492 // Stop walking: nothing to do.
2493 return;
2494
2495 case Type::TypeOfExpr:
2496 // Stop walking: emit typeof expression.
2497 EmitIgnoredExpr(E: cast<TypeOfExprType>(Val: ty)->getUnderlyingExpr());
2498 return;
2499
2500 case Type::Atomic:
2501 type = cast<AtomicType>(Val: ty)->getValueType();
2502 break;
2503
2504 case Type::Pipe:
2505 type = cast<PipeType>(Val: ty)->getElementType();
2506 break;
2507 }
2508 } while (type->isVariablyModifiedType());
2509}
2510
2511Address CodeGenFunction::EmitVAListRef(const Expr* E) {
2512 if (getContext().getBuiltinVaListType()->isArrayType())
2513 return EmitPointerWithAlignment(Addr: E);
2514 return EmitLValue(E).getAddress();
2515}
2516
2517Address CodeGenFunction::EmitMSVAListRef(const Expr *E) {
2518 return EmitLValue(E).getAddress();
2519}
2520
2521void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E,
2522 const APValue &Init) {
2523 assert(Init.hasValue() && "Invalid DeclRefExpr initializer!");
2524 if (CGDebugInfo *Dbg = getDebugInfo())
2525 if (CGM.getCodeGenOpts().hasReducedDebugInfo())
2526 Dbg->EmitGlobalVariable(VD: E->getDecl(), Init);
2527}
2528
2529CodeGenFunction::PeepholeProtection
2530CodeGenFunction::protectFromPeepholes(RValue rvalue) {
2531 // At the moment, the only aggressive peephole we do in IR gen
2532 // is trunc(zext) folding, but if we add more, we can easily
2533 // extend this protection.
2534
2535 if (!rvalue.isScalar()) return PeepholeProtection();
2536 llvm::Value *value = rvalue.getScalarVal();
2537 if (!isa<llvm::ZExtInst>(Val: value)) return PeepholeProtection();
2538
2539 // Just make an extra bitcast.
2540 assert(HaveInsertPoint());
2541 llvm::Instruction *inst = new llvm::BitCastInst(value, value->getType(), "",
2542 Builder.GetInsertBlock());
2543
2544 PeepholeProtection protection;
2545 protection.Inst = inst;
2546 return protection;
2547}
2548
2549void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {
2550 if (!protection.Inst) return;
2551
2552 // In theory, we could try to duplicate the peepholes now, but whatever.
2553 protection.Inst->eraseFromParent();
2554}
2555
2556void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2557 QualType Ty, SourceLocation Loc,
2558 SourceLocation AssumptionLoc,
2559 llvm::Value *Alignment,
2560 llvm::Value *OffsetValue) {
2561 if (Alignment->getType() != IntPtrTy)
2562 Alignment =
2563 Builder.CreateIntCast(V: Alignment, DestTy: IntPtrTy, isSigned: false, Name: "casted.align");
2564 if (OffsetValue && OffsetValue->getType() != IntPtrTy)
2565 OffsetValue =
2566 Builder.CreateIntCast(V: OffsetValue, DestTy: IntPtrTy, isSigned: true, Name: "casted.offset");
2567 llvm::Value *TheCheck = nullptr;
2568 if (SanOpts.has(K: SanitizerKind::Alignment)) {
2569 llvm::Value *PtrIntValue =
2570 Builder.CreatePtrToInt(V: PtrValue, DestTy: IntPtrTy, Name: "ptrint");
2571
2572 if (OffsetValue) {
2573 bool IsOffsetZero = false;
2574 if (const auto *CI = dyn_cast<llvm::ConstantInt>(Val: OffsetValue))
2575 IsOffsetZero = CI->isZero();
2576
2577 if (!IsOffsetZero)
2578 PtrIntValue = Builder.CreateSub(LHS: PtrIntValue, RHS: OffsetValue, Name: "offsetptr");
2579 }
2580
2581 llvm::Value *Zero = llvm::ConstantInt::get(Ty: IntPtrTy, V: 0);
2582 llvm::Value *Mask =
2583 Builder.CreateSub(LHS: Alignment, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: 1));
2584 llvm::Value *MaskedPtr = Builder.CreateAnd(LHS: PtrIntValue, RHS: Mask, Name: "maskedptr");
2585 TheCheck = Builder.CreateICmpEQ(LHS: MaskedPtr, RHS: Zero, Name: "maskcond");
2586 }
2587 llvm::Instruction *Assumption = Builder.CreateAlignmentAssumption(
2588 DL: CGM.getDataLayout(), PtrValue, Alignment, OffsetValue);
2589
2590 if (!SanOpts.has(K: SanitizerKind::Alignment))
2591 return;
2592 emitAlignmentAssumptionCheck(Ptr: PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2593 OffsetValue, TheCheck, Assumption);
2594}
2595
2596void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2597 const Expr *E,
2598 SourceLocation AssumptionLoc,
2599 llvm::Value *Alignment,
2600 llvm::Value *OffsetValue) {
2601 QualType Ty = E->getType();
2602 SourceLocation Loc = E->getExprLoc();
2603
2604 emitAlignmentAssumption(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2605 OffsetValue);
2606}
2607
2608llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn,
2609 llvm::Value *AnnotatedVal,
2610 StringRef AnnotationStr,
2611 SourceLocation Location,
2612 const AnnotateAttr *Attr) {
2613 SmallVector<llvm::Value *, 5> Args = {
2614 AnnotatedVal,
2615 CGM.EmitAnnotationString(Str: AnnotationStr),
2616 CGM.EmitAnnotationUnit(Loc: Location),
2617 CGM.EmitAnnotationLineNo(L: Location),
2618 };
2619 if (Attr)
2620 Args.push_back(Elt: CGM.EmitAnnotationArgs(Attr));
2621 return Builder.CreateCall(Callee: AnnotationFn, Args);
2622}
2623
2624void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
2625 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2626 for (const auto *I : D->specific_attrs<AnnotateAttr>())
2627 EmitAnnotationCall(AnnotationFn: CGM.getIntrinsic(IID: llvm::Intrinsic::var_annotation,
2628 Tys: {V->getType(), CGM.ConstGlobalsPtrTy}),
2629 AnnotatedVal: V, AnnotationStr: I->getAnnotation(), Location: D->getLocation(), Attr: I);
2630}
2631
2632Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
2633 Address Addr) {
2634 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2635 llvm::Value *V = Addr.emitRawPointer(CGF&: *this);
2636 llvm::Type *VTy = V->getType();
2637 auto *PTy = dyn_cast<llvm::PointerType>(Val: VTy);
2638 unsigned AS = PTy ? PTy->getAddressSpace() : 0;
2639 llvm::PointerType *IntrinTy =
2640 llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: AS);
2641 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::ptr_annotation,
2642 Tys: {IntrinTy, CGM.ConstGlobalsPtrTy});
2643
2644 for (const auto *I : D->specific_attrs<AnnotateAttr>()) {
2645 // FIXME Always emit the cast inst so we can differentiate between
2646 // annotation on the first field of a struct and annotation on the struct
2647 // itself.
2648 if (VTy != IntrinTy)
2649 V = Builder.CreateBitCast(V, DestTy: IntrinTy);
2650 V = EmitAnnotationCall(AnnotationFn: F, AnnotatedVal: V, AnnotationStr: I->getAnnotation(), Location: D->getLocation(), Attr: I);
2651 V = Builder.CreateBitCast(V, DestTy: VTy);
2652 }
2653
2654 return Address(V, Addr.getElementType(), Addr.getAlignment());
2655}
2656
2657CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
2658
2659CodeGenFunction::SanitizerScope::SanitizerScope(CodeGenFunction *CGF)
2660 : CGF(CGF) {
2661 assert(!CGF->IsSanitizerScope);
2662 CGF->IsSanitizerScope = true;
2663}
2664
2665CodeGenFunction::SanitizerScope::~SanitizerScope() {
2666 CGF->IsSanitizerScope = false;
2667}
2668
2669void CodeGenFunction::InsertHelper(llvm::Instruction *I,
2670 const llvm::Twine &Name,
2671 llvm::BasicBlock::iterator InsertPt) const {
2672 LoopStack.InsertHelper(I);
2673 if (IsSanitizerScope)
2674 I->setNoSanitizeMetadata();
2675}
2676
2677void CGBuilderInserter::InsertHelper(
2678 llvm::Instruction *I, const llvm::Twine &Name,
2679 llvm::BasicBlock::iterator InsertPt) const {
2680 llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, InsertPt);
2681 if (CGF)
2682 CGF->InsertHelper(I, Name, InsertPt);
2683}
2684
2685// Emits an error if we don't have a valid set of target features for the
2686// called function.
2687void CodeGenFunction::checkTargetFeatures(const CallExpr *E,
2688 const FunctionDecl *TargetDecl) {
2689 // SemaChecking cannot handle below x86 builtins because they have different
2690 // parameter ranges with different TargetAttribute of caller.
2691 if (CGM.getContext().getTargetInfo().getTriple().isX86()) {
2692 unsigned BuiltinID = TargetDecl->getBuiltinID();
2693 if (BuiltinID == X86::BI__builtin_ia32_cmpps ||
2694 BuiltinID == X86::BI__builtin_ia32_cmpss ||
2695 BuiltinID == X86::BI__builtin_ia32_cmppd ||
2696 BuiltinID == X86::BI__builtin_ia32_cmpsd) {
2697 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
2698 llvm::StringMap<bool> TargetFetureMap;
2699 CGM.getContext().getFunctionFeatureMap(FeatureMap&: TargetFetureMap, FD);
2700 llvm::APSInt Result =
2701 *(E->getArg(Arg: 2)->getIntegerConstantExpr(Ctx: CGM.getContext()));
2702 if (Result.getSExtValue() > 7 && !TargetFetureMap.lookup(Key: "avx"))
2703 CGM.getDiags().Report(Loc: E->getBeginLoc(), DiagID: diag::err_builtin_needs_feature)
2704 << TargetDecl->getDeclName() << "avx";
2705 }
2706 }
2707 return checkTargetFeatures(Loc: E->getBeginLoc(), TargetDecl);
2708}
2709
2710// Emits an error if we don't have a valid set of target features for the
2711// called function.
2712void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
2713 const FunctionDecl *TargetDecl) {
2714 // Early exit if this is an indirect call.
2715 if (!TargetDecl)
2716 return;
2717
2718 // Get the current enclosing function if it exists. If it doesn't
2719 // we can't check the target features anyhow.
2720 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
2721 if (!FD)
2722 return;
2723
2724 // Grab the required features for the call. For a builtin this is listed in
2725 // the td file with the default cpu, for an always_inline function this is any
2726 // listed cpu and any listed features.
2727 unsigned BuiltinID = TargetDecl->getBuiltinID();
2728 std::string MissingFeature;
2729 llvm::StringMap<bool> CallerFeatureMap;
2730 CGM.getContext().getFunctionFeatureMap(FeatureMap&: CallerFeatureMap, FD);
2731 // When compiling in HipStdPar mode we have to be conservative in rejecting
2732 // target specific features in the FE, and defer the possible error to the
2733 // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
2734 // referenced by an accelerator executable function, we emit an error.
2735 bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;
2736 if (BuiltinID) {
2737 StringRef FeatureList(CGM.getContext().BuiltinInfo.getRequiredFeatures(ID: BuiltinID));
2738 if (!Builtin::evaluateRequiredTargetFeatures(
2739 RequiredFatures: FeatureList, TargetFetureMap: CallerFeatureMap) && !IsHipStdPar) {
2740 CGM.getDiags().Report(Loc, DiagID: diag::err_builtin_needs_feature)
2741 << TargetDecl->getDeclName()
2742 << FeatureList;
2743 }
2744 } else if (!TargetDecl->isMultiVersion() &&
2745 TargetDecl->hasAttr<TargetAttr>()) {
2746 // Get the required features for the callee.
2747
2748 const TargetAttr *TD = TargetDecl->getAttr<TargetAttr>();
2749 ParsedTargetAttr ParsedAttr =
2750 CGM.getContext().filterFunctionTargetAttrs(TD);
2751
2752 SmallVector<StringRef, 1> ReqFeatures;
2753 llvm::StringMap<bool> CalleeFeatureMap;
2754 CGM.getContext().getFunctionFeatureMap(FeatureMap&: CalleeFeatureMap, TargetDecl);
2755
2756 for (const auto &F : ParsedAttr.Features) {
2757 if (F[0] == '+' && CalleeFeatureMap.lookup(Key: F.substr(pos: 1)))
2758 ReqFeatures.push_back(Elt: StringRef(F).substr(Start: 1));
2759 }
2760
2761 for (const auto &F : CalleeFeatureMap) {
2762 // Only positive features are "required".
2763 if (F.getValue())
2764 ReqFeatures.push_back(Elt: F.getKey());
2765 }
2766 if (!llvm::all_of(Range&: ReqFeatures, P: [&](StringRef Feature) {
2767 if (!CallerFeatureMap.lookup(Key: Feature)) {
2768 MissingFeature = Feature.str();
2769 return false;
2770 }
2771 return true;
2772 }) && !IsHipStdPar)
2773 CGM.getDiags().Report(Loc, DiagID: diag::err_function_needs_feature)
2774 << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
2775 } else if (!FD->isMultiVersion() && FD->hasAttr<TargetAttr>()) {
2776 llvm::StringMap<bool> CalleeFeatureMap;
2777 CGM.getContext().getFunctionFeatureMap(FeatureMap&: CalleeFeatureMap, TargetDecl);
2778
2779 for (const auto &F : CalleeFeatureMap) {
2780 if (F.getValue() && (!CallerFeatureMap.lookup(Key: F.getKey()) ||
2781 !CallerFeatureMap.find(Key: F.getKey())->getValue()) &&
2782 !IsHipStdPar)
2783 CGM.getDiags().Report(Loc, DiagID: diag::err_function_needs_feature)
2784 << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();
2785 }
2786 }
2787}
2788
2789void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
2790 if (!CGM.getCodeGenOpts().SanitizeStats)
2791 return;
2792
2793 llvm::IRBuilder<> IRB(Builder.GetInsertBlock(), Builder.GetInsertPoint());
2794 IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation());
2795 CGM.getSanStats().create(B&: IRB, SK: SSK);
2796}
2797
2798void CodeGenFunction::EmitKCFIOperandBundle(
2799 const CGCallee &Callee, SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
2800 const FunctionProtoType *FP =
2801 Callee.getAbstractInfo().getCalleeFunctionProtoType();
2802 if (FP)
2803 Bundles.emplace_back(Args: "kcfi", Args: CGM.CreateKCFITypeId(T: FP->desugar()));
2804}
2805
2806llvm::Value *CodeGenFunction::FormAArch64ResolverCondition(
2807 const MultiVersionResolverOption &RO) {
2808 llvm::SmallVector<StringRef, 8> CondFeatures;
2809 for (const StringRef &Feature : RO.Conditions.Features)
2810 CondFeatures.push_back(Elt: Feature);
2811 if (!CondFeatures.empty()) {
2812 return EmitAArch64CpuSupports(FeatureStrs: CondFeatures);
2813 }
2814 return nullptr;
2815}
2816
2817llvm::Value *CodeGenFunction::FormX86ResolverCondition(
2818 const MultiVersionResolverOption &RO) {
2819 llvm::Value *Condition = nullptr;
2820
2821 if (!RO.Conditions.Architecture.empty()) {
2822 StringRef Arch = RO.Conditions.Architecture;
2823 // If arch= specifies an x86-64 micro-architecture level, test the feature
2824 // with __builtin_cpu_supports, otherwise use __builtin_cpu_is.
2825 if (Arch.starts_with(Prefix: "x86-64"))
2826 Condition = EmitX86CpuSupports(FeatureStrs: {Arch});
2827 else
2828 Condition = EmitX86CpuIs(CPUStr: Arch);
2829 }
2830
2831 if (!RO.Conditions.Features.empty()) {
2832 llvm::Value *FeatureCond = EmitX86CpuSupports(FeatureStrs: RO.Conditions.Features);
2833 Condition =
2834 Condition ? Builder.CreateAnd(LHS: Condition, RHS: FeatureCond) : FeatureCond;
2835 }
2836 return Condition;
2837}
2838
2839static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
2840 llvm::Function *Resolver,
2841 CGBuilderTy &Builder,
2842 llvm::Function *FuncToReturn,
2843 bool SupportsIFunc) {
2844 if (SupportsIFunc) {
2845 Builder.CreateRet(V: FuncToReturn);
2846 return;
2847 }
2848
2849 llvm::SmallVector<llvm::Value *, 10> Args(
2850 llvm::make_pointer_range(Range: Resolver->args()));
2851
2852 llvm::CallInst *Result = Builder.CreateCall(Callee: FuncToReturn, Args);
2853 Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
2854
2855 if (Resolver->getReturnType()->isVoidTy())
2856 Builder.CreateRetVoid();
2857 else
2858 Builder.CreateRet(V: Result);
2859}
2860
2861void CodeGenFunction::EmitMultiVersionResolver(
2862 llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
2863
2864 llvm::Triple::ArchType ArchType =
2865 getContext().getTargetInfo().getTriple().getArch();
2866
2867 switch (ArchType) {
2868 case llvm::Triple::x86:
2869 case llvm::Triple::x86_64:
2870 EmitX86MultiVersionResolver(Resolver, Options);
2871 return;
2872 case llvm::Triple::aarch64:
2873 EmitAArch64MultiVersionResolver(Resolver, Options);
2874 return;
2875
2876 default:
2877 assert(false && "Only implemented for x86 and AArch64 targets");
2878 }
2879}
2880
2881void CodeGenFunction::EmitAArch64MultiVersionResolver(
2882 llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
2883 assert(!Options.empty() && "No multiversion resolver options found");
2884 assert(Options.back().Conditions.Features.size() == 0 &&
2885 "Default case must be last");
2886 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
2887 assert(SupportsIFunc &&
2888 "Multiversion resolver requires target IFUNC support");
2889 bool AArch64CpuInitialized = false;
2890 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
2891
2892 for (const MultiVersionResolverOption &RO : Options) {
2893 Builder.SetInsertPoint(CurBlock);
2894 llvm::Value *Condition = FormAArch64ResolverCondition(RO);
2895
2896 // The 'default' or 'all features enabled' case.
2897 if (!Condition) {
2898 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, FuncToReturn: RO.Function,
2899 SupportsIFunc);
2900 return;
2901 }
2902
2903 if (!AArch64CpuInitialized) {
2904 Builder.SetInsertPoint(TheBB: CurBlock, IP: CurBlock->begin());
2905 EmitAArch64CpuInit();
2906 AArch64CpuInitialized = true;
2907 Builder.SetInsertPoint(CurBlock);
2908 }
2909
2910 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
2911 CGBuilderTy RetBuilder(*this, RetBlock);
2912 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder, FuncToReturn: RO.Function,
2913 SupportsIFunc);
2914 CurBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
2915 Builder.CreateCondBr(Cond: Condition, True: RetBlock, False: CurBlock);
2916 }
2917
2918 // If no default, emit an unreachable.
2919 Builder.SetInsertPoint(CurBlock);
2920 llvm::CallInst *TrapCall = EmitTrapCall(IntrID: llvm::Intrinsic::trap);
2921 TrapCall->setDoesNotReturn();
2922 TrapCall->setDoesNotThrow();
2923 Builder.CreateUnreachable();
2924 Builder.ClearInsertionPoint();
2925}
2926
2927void CodeGenFunction::EmitX86MultiVersionResolver(
2928 llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
2929
2930 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
2931
2932 // Main function's basic block.
2933 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
2934 Builder.SetInsertPoint(CurBlock);
2935 EmitX86CpuInit();
2936
2937 for (const MultiVersionResolverOption &RO : Options) {
2938 Builder.SetInsertPoint(CurBlock);
2939 llvm::Value *Condition = FormX86ResolverCondition(RO);
2940
2941 // The 'default' or 'generic' case.
2942 if (!Condition) {
2943 assert(&RO == Options.end() - 1 &&
2944 "Default or Generic case must be last");
2945 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, FuncToReturn: RO.Function,
2946 SupportsIFunc);
2947 return;
2948 }
2949
2950 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
2951 CGBuilderTy RetBuilder(*this, RetBlock);
2952 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder, FuncToReturn: RO.Function,
2953 SupportsIFunc);
2954 CurBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
2955 Builder.CreateCondBr(Cond: Condition, True: RetBlock, False: CurBlock);
2956 }
2957
2958 // If no generic/default, emit an unreachable.
2959 Builder.SetInsertPoint(CurBlock);
2960 llvm::CallInst *TrapCall = EmitTrapCall(IntrID: llvm::Intrinsic::trap);
2961 TrapCall->setDoesNotReturn();
2962 TrapCall->setDoesNotThrow();
2963 Builder.CreateUnreachable();
2964 Builder.ClearInsertionPoint();
2965}
2966
2967// Loc - where the diagnostic will point, where in the source code this
2968// alignment has failed.
2969// SecondaryLoc - if present (will be present if sufficiently different from
2970// Loc), the diagnostic will additionally point a "Note:" to this location.
2971// It should be the location where the __attribute__((assume_aligned))
2972// was written e.g.
2973void CodeGenFunction::emitAlignmentAssumptionCheck(
2974 llvm::Value *Ptr, QualType Ty, SourceLocation Loc,
2975 SourceLocation SecondaryLoc, llvm::Value *Alignment,
2976 llvm::Value *OffsetValue, llvm::Value *TheCheck,
2977 llvm::Instruction *Assumption) {
2978 assert(isa_and_nonnull<llvm::CallInst>(Assumption) &&
2979 cast<llvm::CallInst>(Assumption)->getCalledOperand() ==
2980 llvm::Intrinsic::getDeclaration(
2981 Builder.GetInsertBlock()->getParent()->getParent(),
2982 llvm::Intrinsic::assume) &&
2983 "Assumption should be a call to llvm.assume().");
2984 assert(&(Builder.GetInsertBlock()->back()) == Assumption &&
2985 "Assumption should be the last instruction of the basic block, "
2986 "since the basic block is still being generated.");
2987
2988 if (!SanOpts.has(K: SanitizerKind::Alignment))
2989 return;
2990
2991 // Don't check pointers to volatile data. The behavior here is implementation-
2992 // defined.
2993 if (Ty->getPointeeType().isVolatileQualified())
2994 return;
2995
2996 // We need to temorairly remove the assumption so we can insert the
2997 // sanitizer check before it, else the check will be dropped by optimizations.
2998 Assumption->removeFromParent();
2999
3000 {
3001 SanitizerScope SanScope(this);
3002
3003 if (!OffsetValue)
3004 OffsetValue = Builder.getInt1(V: false); // no offset.
3005
3006 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc),
3007 EmitCheckSourceLocation(Loc: SecondaryLoc),
3008 EmitCheckTypeDescriptor(T: Ty)};
3009 llvm::Value *DynamicData[] = {EmitCheckValue(V: Ptr),
3010 EmitCheckValue(V: Alignment),
3011 EmitCheckValue(V: OffsetValue)};
3012 EmitCheck(Checked: {std::make_pair(x&: TheCheck, y: SanitizerKind::Alignment)},
3013 Check: SanitizerHandler::AlignmentAssumption, StaticArgs: StaticData, DynamicArgs: DynamicData);
3014 }
3015
3016 // We are now in the (new, empty) "cont" basic block.
3017 // Reintroduce the assumption.
3018 Builder.Insert(I: Assumption);
3019 // FIXME: Assumption still has it's original basic block as it's Parent.
3020}
3021
3022llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
3023 if (CGDebugInfo *DI = getDebugInfo())
3024 return DI->SourceLocToDebugLoc(Loc: Location);
3025
3026 return llvm::DebugLoc();
3027}
3028
3029llvm::Value *
3030CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
3031 Stmt::Likelihood LH) {
3032 switch (LH) {
3033 case Stmt::LH_None:
3034 return Cond;
3035 case Stmt::LH_Likely:
3036 case Stmt::LH_Unlikely:
3037 // Don't generate llvm.expect on -O0 as the backend won't use it for
3038 // anything.
3039 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3040 return Cond;
3041 llvm::Type *CondTy = Cond->getType();
3042 assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");
3043 llvm::Function *FnExpect =
3044 CGM.getIntrinsic(IID: llvm::Intrinsic::expect, Tys: CondTy);
3045 llvm::Value *ExpectedValueOfCond =
3046 llvm::ConstantInt::getBool(Ty: CondTy, V: LH == Stmt::LH_Likely);
3047 return Builder.CreateCall(Callee: FnExpect, Args: {Cond, ExpectedValueOfCond},
3048 Name: Cond->getName() + ".expval");
3049 }
3050 llvm_unreachable("Unknown Likelihood");
3051}
3052
3053llvm::Value *CodeGenFunction::emitBoolVecConversion(llvm::Value *SrcVec,
3054 unsigned NumElementsDst,
3055 const llvm::Twine &Name) {
3056 auto *SrcTy = cast<llvm::FixedVectorType>(Val: SrcVec->getType());
3057 unsigned NumElementsSrc = SrcTy->getNumElements();
3058 if (NumElementsSrc == NumElementsDst)
3059 return SrcVec;
3060
3061 std::vector<int> ShuffleMask(NumElementsDst, -1);
3062 for (unsigned MaskIdx = 0;
3063 MaskIdx < std::min<>(a: NumElementsDst, b: NumElementsSrc); ++MaskIdx)
3064 ShuffleMask[MaskIdx] = MaskIdx;
3065
3066 return Builder.CreateShuffleVector(V: SrcVec, Mask: ShuffleMask, Name);
3067}
3068
3069void CodeGenFunction::EmitPointerAuthOperandBundle(
3070 const CGPointerAuthInfo &PointerAuth,
3071 SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
3072 if (!PointerAuth.isSigned())
3073 return;
3074
3075 auto *Key = Builder.getInt32(C: PointerAuth.getKey());
3076
3077 llvm::Value *Discriminator = PointerAuth.getDiscriminator();
3078 if (!Discriminator)
3079 Discriminator = Builder.getSize(N: 0);
3080
3081 llvm::Value *Args[] = {Key, Discriminator};
3082 Bundles.emplace_back(Args: "ptrauth", Args);
3083}
3084
3085static llvm::Value *EmitPointerAuthCommon(CodeGenFunction &CGF,
3086 const CGPointerAuthInfo &PointerAuth,
3087 llvm::Value *Pointer,
3088 unsigned IntrinsicID) {
3089 if (!PointerAuth)
3090 return Pointer;
3091
3092 auto Key = CGF.Builder.getInt32(C: PointerAuth.getKey());
3093
3094 llvm::Value *Discriminator = PointerAuth.getDiscriminator();
3095 if (!Discriminator) {
3096 Discriminator = CGF.Builder.getSize(N: 0);
3097 }
3098
3099 // Convert the pointer to intptr_t before signing it.
3100 auto OrigType = Pointer->getType();
3101 Pointer = CGF.Builder.CreatePtrToInt(V: Pointer, DestTy: CGF.IntPtrTy);
3102
3103 // call i64 @llvm.ptrauth.sign.i64(i64 %pointer, i32 %key, i64 %discriminator)
3104 auto Intrinsic = CGF.CGM.getIntrinsic(IID: IntrinsicID);
3105 Pointer = CGF.EmitRuntimeCall(callee: Intrinsic, args: {Pointer, Key, Discriminator});
3106
3107 // Convert back to the original type.
3108 Pointer = CGF.Builder.CreateIntToPtr(V: Pointer, DestTy: OrigType);
3109 return Pointer;
3110}
3111
3112llvm::Value *
3113CodeGenFunction::EmitPointerAuthSign(const CGPointerAuthInfo &PointerAuth,
3114 llvm::Value *Pointer) {
3115 if (!PointerAuth.shouldSign())
3116 return Pointer;
3117 return EmitPointerAuthCommon(CGF&: *this, PointerAuth, Pointer,
3118 IntrinsicID: llvm::Intrinsic::ptrauth_sign);
3119}
3120
3121static llvm::Value *EmitStrip(CodeGenFunction &CGF,
3122 const CGPointerAuthInfo &PointerAuth,
3123 llvm::Value *Pointer) {
3124 auto StripIntrinsic = CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::ptrauth_strip);
3125
3126 auto Key = CGF.Builder.getInt32(C: PointerAuth.getKey());
3127 // Convert the pointer to intptr_t before signing it.
3128 auto OrigType = Pointer->getType();
3129 Pointer = CGF.EmitRuntimeCall(
3130 callee: StripIntrinsic, args: {CGF.Builder.CreatePtrToInt(V: Pointer, DestTy: CGF.IntPtrTy), Key});
3131 return CGF.Builder.CreateIntToPtr(V: Pointer, DestTy: OrigType);
3132}
3133
3134llvm::Value *
3135CodeGenFunction::EmitPointerAuthAuth(const CGPointerAuthInfo &PointerAuth,
3136 llvm::Value *Pointer) {
3137 if (PointerAuth.shouldStrip()) {
3138 return EmitStrip(CGF&: *this, PointerAuth, Pointer);
3139 }
3140 if (!PointerAuth.shouldAuth()) {
3141 return Pointer;
3142 }
3143
3144 return EmitPointerAuthCommon(CGF&: *this, PointerAuth, Pointer,
3145 IntrinsicID: llvm::Intrinsic::ptrauth_auth);
3146}
3147