1//===--- CodeGenFunction.cpp - Emit LLVM Code from ASTs for a Function ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This coordinates the per-function state used while generating code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CodeGenFunction.h"
14#include "CGBlocks.h"
15#include "CGCUDARuntime.h"
16#include "CGCXXABI.h"
17#include "CGCleanup.h"
18#include "CGDebugInfo.h"
19#include "CGHLSLRuntime.h"
20#include "CGOpenMPRuntime.h"
21#include "CodeGenModule.h"
22#include "CodeGenPGO.h"
23#include "TargetInfo.h"
24#include "clang/AST/ASTContext.h"
25#include "clang/AST/ASTLambda.h"
26#include "clang/AST/Attr.h"
27#include "clang/AST/Decl.h"
28#include "clang/AST/DeclCXX.h"
29#include "clang/AST/Expr.h"
30#include "clang/AST/IgnoreExpr.h"
31#include "clang/AST/StmtCXX.h"
32#include "clang/AST/StmtObjC.h"
33#include "clang/Basic/Builtins.h"
34#include "clang/Basic/CodeGenOptions.h"
35#include "clang/Basic/DiagnosticFrontend.h"
36#include "clang/Basic/TargetBuiltins.h"
37#include "clang/Basic/TargetInfo.h"
38#include "clang/CodeGen/CGFunctionInfo.h"
39#include "llvm/ADT/ArrayRef.h"
40#include "llvm/ADT/ScopeExit.h"
41#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
42#include "llvm/IR/DataLayout.h"
43#include "llvm/IR/Dominators.h"
44#include "llvm/IR/FPEnv.h"
45#include "llvm/IR/Instruction.h"
46#include "llvm/IR/IntrinsicInst.h"
47#include "llvm/IR/Intrinsics.h"
48#include "llvm/IR/IntrinsicsPowerPC.h"
49#include "llvm/IR/MDBuilder.h"
50#include "llvm/Support/CRC.h"
51#include "llvm/Support/SaveAndRestore.h"
52#include "llvm/Support/SipHash.h"
53#include "llvm/Support/xxhash.h"
54#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
55#include "llvm/Transforms/Utils/PromoteMemToReg.h"
56#include <optional>
57
58using namespace clang;
59using namespace CodeGen;
60
61/// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time
62/// markers.
63static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts,
64 const LangOptions &LangOpts) {
65 if (CGOpts.DisableLifetimeMarkers)
66 return false;
67
68 // Sanitizers may use markers.
69 if (CGOpts.SanitizeAddressUseAfterScope ||
70 LangOpts.Sanitize.has(K: SanitizerKind::HWAddress) ||
71 LangOpts.Sanitize.has(K: SanitizerKind::Memory) ||
72 LangOpts.Sanitize.has(K: SanitizerKind::MemtagStack))
73 return true;
74
75 // For now, only in optimized builds.
76 return CGOpts.OptimizationLevel != 0;
77}
78
79CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
80 : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
81 Builder(cgm, cgm.getModule().getContext(), CGBuilderInserterTy(this)),
82 SanOpts(CGM.getLangOpts().Sanitize), CurFPFeatures(CGM.getLangOpts()),
83 DebugInfo(CGM.getModuleDebugInfo()),
84 PGO(std::make_unique<CodeGenPGO>(args&: cgm)),
85 ShouldEmitLifetimeMarkers(
86 shouldEmitLifetimeMarkers(CGOpts: CGM.getCodeGenOpts(), LangOpts: CGM.getLangOpts())) {
87 if (!suppressNewContext)
88 CGM.getCXXABI().getMangleContext().startNewFunction();
89 EHStack.setCGF(this);
90
91 SetFastMathFlags(CurFPFeatures);
92}
93
94const FunctionDecl *CodeGenFunction::getCurrentFunctionDecl() const {
95 const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
96 if (!FD)
97 FD = dyn_cast_or_null<FunctionDecl>(Val: CurFuncDecl);
98 return FD;
99}
100
101CodeGenFunction::~CodeGenFunction() {
102 assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup");
103 assert(DeferredDeactivationCleanupStack.empty() &&
104 "missed to deactivate a cleanup");
105
106 if (getLangOpts().OpenMP && CurFn)
107 CGM.getOpenMPRuntime().functionFinished(CGF&: *this);
108
109 // If we have an OpenMPIRBuilder we want to finalize functions (incl.
110 // outlining etc) at some point. Doing it once the function codegen is done
111 // seems to be a reasonable spot. We do it here, as opposed to the deletion
112 // time of the CodeGenModule, because we have to ensure the IR has not yet
113 // been "emitted" to the outside, thus, modifications are still sensible.
114 if (CGM.getLangOpts().OpenMPIRBuilder && CurFn)
115 CGM.getOpenMPRuntime().getOMPBuilder().finalize(Fn: CurFn);
116}
117
118// Map the LangOption for exception behavior into
119// the corresponding enum in the IR.
120llvm::fp::ExceptionBehavior
121clang::ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind) {
122
123 switch (Kind) {
124 case LangOptions::FPE_Ignore: return llvm::fp::ebIgnore;
125 case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap;
126 case LangOptions::FPE_Strict: return llvm::fp::ebStrict;
127 default:
128 llvm_unreachable("Unsupported FP Exception Behavior");
129 }
130}
131
132void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) {
133 llvm::FastMathFlags FMF;
134 FMF.setAllowReassoc(FPFeatures.getAllowFPReassociate());
135 FMF.setNoNaNs(FPFeatures.getNoHonorNaNs());
136 FMF.setNoInfs(FPFeatures.getNoHonorInfs());
137 FMF.setNoSignedZeros(FPFeatures.getNoSignedZero());
138 FMF.setAllowReciprocal(FPFeatures.getAllowReciprocal());
139 FMF.setApproxFunc(FPFeatures.getAllowApproxFunc());
140 FMF.setAllowContract(FPFeatures.allowFPContractAcrossStatement());
141 Builder.setFastMathFlags(FMF);
142}
143
144CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
145 const Expr *E)
146 : CGF(CGF) {
147 ConstructorHelper(FPFeatures: E->getFPFeaturesInEffect(LO: CGF.getLangOpts()));
148}
149
150CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
151 FPOptions FPFeatures)
152 : CGF(CGF) {
153 ConstructorHelper(FPFeatures);
154}
155
156void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) {
157 OldFPFeatures = CGF.CurFPFeatures;
158 CGF.CurFPFeatures = FPFeatures;
159
160 OldExcept = CGF.Builder.getDefaultConstrainedExcept();
161 OldRounding = CGF.Builder.getDefaultConstrainedRounding();
162
163 if (OldFPFeatures == FPFeatures)
164 return;
165
166 FMFGuard.emplace(args&: CGF.Builder);
167
168 llvm::RoundingMode NewRoundingBehavior = FPFeatures.getRoundingMode();
169 CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);
170 auto NewExceptionBehavior =
171 ToConstrainedExceptMD(Kind: FPFeatures.getExceptionMode());
172 CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);
173
174 CGF.SetFastMathFlags(FPFeatures);
175
176 assert((CGF.CurFuncDecl == nullptr || CGF.Builder.getIsFPConstrained() ||
177 isa<CXXConstructorDecl>(CGF.CurFuncDecl) ||
178 isa<CXXDestructorDecl>(CGF.CurFuncDecl) ||
179 (NewExceptionBehavior == llvm::fp::ebIgnore &&
180 NewRoundingBehavior == llvm::RoundingMode::NearestTiesToEven)) &&
181 "FPConstrained should be enabled on entire function");
182
183 auto mergeFnAttrValue = [&](StringRef Name, bool Value) {
184 auto OldValue =
185 CGF.CurFn->getFnAttribute(Kind: Name).getValueAsBool();
186 auto NewValue = OldValue & Value;
187 if (OldValue != NewValue)
188 CGF.CurFn->addFnAttr(Kind: Name, Val: llvm::toStringRef(B: NewValue));
189 };
190 mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.getNoSignedZero());
191}
192
193CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
194 CGF.CurFPFeatures = OldFPFeatures;
195 CGF.Builder.setDefaultConstrainedExcept(OldExcept);
196 CGF.Builder.setDefaultConstrainedRounding(OldRounding);
197}
198
199static LValue
200makeNaturalAlignAddrLValue(llvm::Value *V, QualType T, bool ForPointeeType,
201 bool MightBeSigned, CodeGenFunction &CGF,
202 KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
203 LValueBaseInfo BaseInfo;
204 TBAAAccessInfo TBAAInfo;
205 CharUnits Alignment =
206 CGF.CGM.getNaturalTypeAlignment(T, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo, forPointeeType: ForPointeeType);
207 Address Addr =
208 MightBeSigned
209 ? CGF.makeNaturalAddressForPointer(Ptr: V, T, Alignment, ForPointeeType: false, BaseInfo: nullptr,
210 TBAAInfo: nullptr, IsKnownNonNull)
211 : Address(V, CGF.ConvertTypeForMem(T), Alignment, IsKnownNonNull);
212 return CGF.MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
213}
214
215LValue
216CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
217 KnownNonNull_t IsKnownNonNull) {
218 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,
219 /*MightBeSigned*/ true, CGF&: *this,
220 IsKnownNonNull);
221}
222
223LValue
224CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
225 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,
226 /*MightBeSigned*/ true, CGF&: *this);
227}
228
229LValue CodeGenFunction::MakeNaturalAlignRawAddrLValue(llvm::Value *V,
230 QualType T) {
231 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,
232 /*MightBeSigned*/ false, CGF&: *this);
233}
234
235LValue CodeGenFunction::MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V,
236 QualType T) {
237 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,
238 /*MightBeSigned*/ false, CGF&: *this);
239}
240
241llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
242 return CGM.getTypes().ConvertTypeForMem(T);
243}
244
245llvm::Type *CodeGenFunction::ConvertType(QualType T) {
246 return CGM.getTypes().ConvertType(T);
247}
248
249llvm::Type *CodeGenFunction::convertTypeForLoadStore(QualType ASTTy,
250 llvm::Type *LLVMTy) {
251 return CGM.getTypes().convertTypeForLoadStore(T: ASTTy, LLVMTy);
252}
253
254TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
255 type = type.getCanonicalType();
256 while (true) {
257 switch (type->getTypeClass()) {
258#define TYPE(name, parent)
259#define ABSTRACT_TYPE(name, parent)
260#define NON_CANONICAL_TYPE(name, parent) case Type::name:
261#define DEPENDENT_TYPE(name, parent) case Type::name:
262#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
263#include "clang/AST/TypeNodes.inc"
264 llvm_unreachable("non-canonical or dependent type in IR-generation");
265
266 case Type::Auto:
267 case Type::DeducedTemplateSpecialization:
268 llvm_unreachable("undeduced type in IR-generation");
269
270 // Various scalar types.
271 case Type::Builtin:
272 case Type::Pointer:
273 case Type::BlockPointer:
274 case Type::LValueReference:
275 case Type::RValueReference:
276 case Type::MemberPointer:
277 case Type::Vector:
278 case Type::ExtVector:
279 case Type::ConstantMatrix:
280 case Type::FunctionProto:
281 case Type::FunctionNoProto:
282 case Type::Enum:
283 case Type::ObjCObjectPointer:
284 case Type::Pipe:
285 case Type::BitInt:
286 case Type::HLSLAttributedResource:
287 case Type::HLSLInlineSpirv:
288 case Type::OverflowBehavior:
289 return TEK_Scalar;
290
291 // Complexes.
292 case Type::Complex:
293 return TEK_Complex;
294
295 // Arrays, records, and Objective-C objects.
296 case Type::ConstantArray:
297 case Type::IncompleteArray:
298 case Type::VariableArray:
299 case Type::Record:
300 case Type::ObjCObject:
301 case Type::ObjCInterface:
302 case Type::ArrayParameter:
303 return TEK_Aggregate;
304
305 // We operate on atomic values according to their underlying type.
306 case Type::Atomic:
307 type = cast<AtomicType>(Val&: type)->getValueType();
308 continue;
309 }
310 llvm_unreachable("unknown type kind!");
311 }
312}
313
314llvm::DebugLoc CodeGenFunction::EmitReturnBlock() {
315 // For cleanliness, we try to avoid emitting the return block for
316 // simple cases.
317 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
318
319 if (CurBB) {
320 assert(!CurBB->hasTerminator() && "Unexpected terminated block.");
321
322 // We have a valid insert point, reuse it if it is empty or there are no
323 // explicit jumps to the return block.
324 if (CurBB->empty() || ReturnBlock.getBlock()->use_empty()) {
325 ReturnBlock.getBlock()->replaceAllUsesWith(V: CurBB);
326 delete ReturnBlock.getBlock();
327 ReturnBlock = JumpDest();
328 } else
329 EmitBlock(BB: ReturnBlock.getBlock());
330 return llvm::DebugLoc();
331 }
332
333 // Otherwise, if the return block is the target of a single direct
334 // branch then we can just put the code in that block instead. This
335 // cleans up functions which started with a unified return block.
336 if (ReturnBlock.getBlock()->hasOneUse()) {
337 auto *BI =
338 dyn_cast<llvm::UncondBrInst>(Val: *ReturnBlock.getBlock()->user_begin());
339 if (BI && BI->getSuccessor(i: 0) == ReturnBlock.getBlock()) {
340 // Record/return the DebugLoc of the simple 'return' expression to be used
341 // later by the actual 'ret' instruction.
342 llvm::DebugLoc Loc = BI->getDebugLoc();
343 Builder.SetInsertPoint(BI->getParent());
344 BI->eraseFromParent();
345 delete ReturnBlock.getBlock();
346 ReturnBlock = JumpDest();
347 return Loc;
348 }
349 }
350
351 // FIXME: We are at an unreachable point, there is no reason to emit the block
352 // unless it has uses. However, we still need a place to put the debug
353 // region.end for now.
354
355 EmitBlock(BB: ReturnBlock.getBlock());
356 return llvm::DebugLoc();
357}
358
359static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
360 if (!BB) return;
361 if (!BB->use_empty()) {
362 CGF.CurFn->insert(Position: CGF.CurFn->end(), BB);
363 return;
364 }
365 delete BB;
366}
367
368void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
369 assert(BreakContinueStack.empty() &&
370 "mismatched push/pop in break/continue stack!");
371 assert(LifetimeExtendedCleanupStack.empty() &&
372 "mismatched push/pop of cleanups in EHStack!");
373 assert(DeferredDeactivationCleanupStack.empty() &&
374 "mismatched activate/deactivate of cleanups!");
375
376 if (CGM.shouldEmitConvergenceTokens()) {
377 ConvergenceTokenStack.pop_back();
378 assert(ConvergenceTokenStack.empty() &&
379 "mismatched push/pop in convergence stack!");
380 }
381
382 bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
383 && NumSimpleReturnExprs == NumReturnExprs
384 && ReturnBlock.getBlock()->use_empty();
385 // Usually the return expression is evaluated before the cleanup
386 // code. If the function contains only a simple return statement,
387 // such as a constant, the location before the cleanup code becomes
388 // the last useful breakpoint in the function, because the simple
389 // return expression will be evaluated after the cleanup code. To be
390 // safe, set the debug location for cleanup code to the location of
391 // the return statement. Otherwise the cleanup code should be at the
392 // end of the function's lexical scope.
393 //
394 // If there are multiple branches to the return block, the branch
395 // instructions will get the location of the return statements and
396 // all will be fine.
397 if (CGDebugInfo *DI = getDebugInfo()) {
398 if (OnlySimpleReturnStmts)
399 DI->EmitLocation(Builder, Loc: LastStopPoint);
400 else
401 DI->EmitLocation(Builder, Loc: EndLoc);
402 }
403
404 // Pop any cleanups that might have been associated with the
405 // parameters. Do this in whatever block we're currently in; it's
406 // important to do this before we enter the return block or return
407 // edges will be *really* confused.
408 bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth;
409 bool HasOnlyNoopCleanups =
410 HasCleanups && EHStack.containsOnlyNoopCleanups(Old: PrologueCleanupDepth);
411 bool EmitRetDbgLoc = !HasCleanups || HasOnlyNoopCleanups;
412
413 std::optional<ApplyDebugLocation> OAL;
414 if (HasCleanups) {
415 // Make sure the line table doesn't jump back into the body for
416 // the ret after it's been at EndLoc.
417 if (CGDebugInfo *DI = getDebugInfo()) {
418 if (OnlySimpleReturnStmts)
419 DI->EmitLocation(Builder, Loc: EndLoc);
420 else
421 // We may not have a valid end location. Try to apply it anyway, and
422 // fall back to an artificial location if needed.
423 OAL = ApplyDebugLocation::CreateDefaultArtificial(CGF&: *this, TemporaryLocation: EndLoc);
424 }
425
426 PopCleanupBlocks(OldCleanupStackSize: PrologueCleanupDepth);
427 }
428
429 // Emit function epilog (to return).
430 llvm::DebugLoc Loc = EmitReturnBlock();
431
432 if (ShouldInstrumentFunction()) {
433 if (CGM.getCodeGenOpts().InstrumentFunctions)
434 CurFn->addFnAttr(Kind: "instrument-function-exit", Val: "__cyg_profile_func_exit");
435 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
436 CurFn->addFnAttr(Kind: "instrument-function-exit-inlined",
437 Val: "__cyg_profile_func_exit");
438 }
439
440 // Emit debug descriptor for function end.
441 if (CGDebugInfo *DI = getDebugInfo())
442 DI->EmitFunctionEnd(Builder, Fn: CurFn);
443
444 // Reset the debug location to that of the simple 'return' expression, if any
445 // rather than that of the end of the function's scope '}'.
446 uint64_t RetKeyInstructionsAtomGroup = Loc ? Loc->getAtomGroup() : 0;
447 ApplyDebugLocation AL(*this, Loc);
448 EmitFunctionEpilog(FI: *CurFnInfo, EmitRetDbgLoc, EndLoc,
449 RetKeyInstructionsSourceAtom: RetKeyInstructionsAtomGroup);
450 EmitEndEHSpec(D: CurCodeDecl);
451
452 assert(EHStack.empty() &&
453 "did not remove all scopes from cleanup stack!");
454
455 // If someone did an indirect goto, emit the indirect goto block at the end of
456 // the function.
457 if (IndirectBranch) {
458 EmitBlock(BB: IndirectBranch->getParent());
459 Builder.ClearInsertionPoint();
460 }
461
462 // If some of our locals escaped, insert a call to llvm.localescape in the
463 // entry block.
464 if (!EscapedLocals.empty()) {
465 // Invert the map from local to index into a simple vector. There should be
466 // no holes.
467 SmallVector<llvm::Value *, 4> EscapeArgs;
468 EscapeArgs.resize(N: EscapedLocals.size());
469 for (auto &Pair : EscapedLocals)
470 EscapeArgs[Pair.second] = Pair.first;
471 llvm::Function *FrameEscapeFn = llvm::Intrinsic::getOrInsertDeclaration(
472 M: &CGM.getModule(), id: llvm::Intrinsic::localescape);
473 CGBuilderTy(CGM, AllocaInsertPt).CreateCall(Callee: FrameEscapeFn, Args: EscapeArgs);
474 }
475
476 // Remove the AllocaInsertPt instruction, which is just a convenience for us.
477 llvm::Instruction *Ptr = AllocaInsertPt;
478 AllocaInsertPt = nullptr;
479 Ptr->eraseFromParent();
480
481 // PostAllocaInsertPt, if created, was lazily created when it was required,
482 // remove it now since it was just created for our own convenience.
483 if (PostAllocaInsertPt) {
484 llvm::Instruction *PostPtr = PostAllocaInsertPt;
485 PostAllocaInsertPt = nullptr;
486 PostPtr->eraseFromParent();
487 }
488
489 // If someone took the address of a label but never did an indirect goto, we
490 // made a zero entry PHI node, which is illegal, zap it now.
491 if (IndirectBranch) {
492 llvm::PHINode *PN = cast<llvm::PHINode>(Val: IndirectBranch->getAddress());
493 if (PN->getNumIncomingValues() == 0) {
494 PN->replaceAllUsesWith(V: llvm::PoisonValue::get(T: PN->getType()));
495 PN->eraseFromParent();
496 }
497 }
498
499 EmitIfUsed(CGF&: *this, BB: EHResumeBlock);
500 EmitIfUsed(CGF&: *this, BB: TerminateLandingPad);
501 EmitIfUsed(CGF&: *this, BB: TerminateHandler);
502 EmitIfUsed(CGF&: *this, BB: UnreachableBlock);
503
504 for (const auto &FuncletAndParent : TerminateFunclets)
505 EmitIfUsed(CGF&: *this, BB: FuncletAndParent.second);
506
507 if (CGM.getCodeGenOpts().EmitDeclMetadata)
508 EmitDeclMetadata();
509
510 for (const auto &R : DeferredReplacements) {
511 if (llvm::Value *Old = R.first) {
512 Old->replaceAllUsesWith(V: R.second);
513 cast<llvm::Instruction>(Val: Old)->eraseFromParent();
514 }
515 }
516 DeferredReplacements.clear();
517
518 // Eliminate CleanupDestSlot alloca by replacing it with SSA values and
519 // PHIs if the current function is a coroutine. We don't do it for all
520 // functions as it may result in slight increase in numbers of instructions
521 // if compiled with no optimizations. We do it for coroutine as the lifetime
522 // of CleanupDestSlot alloca make correct coroutine frame building very
523 // difficult.
524 if (NormalCleanupDest.isValid() && isCoroutine()) {
525 llvm::DominatorTree DT(*CurFn);
526 llvm::PromoteMemToReg(
527 Allocas: cast<llvm::AllocaInst>(Val: NormalCleanupDest.getPointer()), DT);
528 NormalCleanupDest = Address::invalid();
529 }
530
531 // Scan function arguments for vector width.
532 for (llvm::Argument &A : CurFn->args())
533 if (auto *VT = dyn_cast<llvm::VectorType>(Val: A.getType()))
534 LargestVectorWidth =
535 std::max(a: (uint64_t)LargestVectorWidth,
536 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
537
538 // Update vector width based on return type.
539 if (auto *VT = dyn_cast<llvm::VectorType>(Val: CurFn->getReturnType()))
540 LargestVectorWidth =
541 std::max(a: (uint64_t)LargestVectorWidth,
542 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
543
544 if (CurFnInfo->getMaxVectorWidth() > LargestVectorWidth)
545 LargestVectorWidth = CurFnInfo->getMaxVectorWidth();
546
547 // Add the min-legal-vector-width attribute. This contains the max width from:
548 // 1. min-vector-width attribute used in the source program.
549 // 2. Any builtins used that have a vector width specified.
550 // 3. Values passed in and out of inline assembly.
551 // 4. Width of vector arguments and return types for this function.
552 // 5. Width of vector arguments and return types for functions called by this
553 // function.
554 if (getContext().getTargetInfo().getTriple().isX86())
555 CurFn->addFnAttr(Kind: "min-legal-vector-width",
556 Val: llvm::utostr(X: LargestVectorWidth));
557
558 // If we generated an unreachable return block, delete it now.
559 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()) {
560 Builder.ClearInsertionPoint();
561 ReturnBlock.getBlock()->eraseFromParent();
562 }
563 if (ReturnValue.isValid()) {
564 auto *RetAlloca =
565 dyn_cast<llvm::AllocaInst>(Val: ReturnValue.emitRawPointer(CGF&: *this));
566 if (RetAlloca && RetAlloca->use_empty()) {
567 RetAlloca->eraseFromParent();
568 ReturnValue = Address::invalid();
569 }
570 }
571}
572
573/// ShouldInstrumentFunction - Return true if the current function should be
574/// instrumented with __cyg_profile_func_* calls
575bool CodeGenFunction::ShouldInstrumentFunction() {
576 if (!CGM.getCodeGenOpts().InstrumentFunctions &&
577 !CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining &&
578 !CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
579 return false;
580 if (!CurFuncDecl || CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>())
581 return false;
582 return true;
583}
584
585bool CodeGenFunction::ShouldSkipSanitizerInstrumentation() {
586 if (!CurFuncDecl)
587 return false;
588 return CurFuncDecl->hasAttr<DisableSanitizerInstrumentationAttr>();
589}
590
591/// ShouldXRayInstrument - Return true if the current function should be
592/// instrumented with XRay nop sleds.
593bool CodeGenFunction::ShouldXRayInstrumentFunction() const {
594 return CGM.getCodeGenOpts().XRayInstrumentFunctions;
595}
596
597/// AlwaysEmitXRayCustomEvents - Return true if we should emit IR for calls to
598/// the __xray_customevent(...) builtin calls, when doing XRay instrumentation.
599bool CodeGenFunction::AlwaysEmitXRayCustomEvents() const {
600 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
601 (CGM.getCodeGenOpts().XRayAlwaysEmitCustomEvents ||
602 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
603 XRayInstrKind::Custom);
604}
605
606bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const {
607 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
608 (CGM.getCodeGenOpts().XRayAlwaysEmitTypedEvents ||
609 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
610 XRayInstrKind::Typed);
611}
612
613llvm::ConstantInt *
614CodeGenFunction::getUBSanFunctionTypeHash(QualType Ty) const {
615 // Remove any (C++17) exception specifications, to allow calling e.g. a
616 // noexcept function through a non-noexcept pointer.
617 if (!Ty->isFunctionNoProtoType())
618 Ty = getContext().getFunctionTypeWithExceptionSpec(Orig: Ty, ESI: EST_None);
619 std::string Mangled;
620 llvm::raw_string_ostream Out(Mangled);
621 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: Ty, Out, NormalizeIntegers: false);
622 return llvm::ConstantInt::get(
623 Ty: CGM.Int32Ty, V: static_cast<uint32_t>(llvm::xxh3_64bits(data: Mangled)));
624}
625
626void CodeGenFunction::EmitKernelMetadata(const FunctionDecl *FD,
627 llvm::Function *Fn) {
628 if (!FD->hasAttr<DeviceKernelAttr>() && !FD->hasAttr<CUDAGlobalAttr>())
629 return;
630
631 llvm::LLVMContext &Context = getLLVMContext();
632
633 CGM.GenKernelArgMetadata(FN: Fn, FD, CGF: this);
634
635 if (!(getLangOpts().OpenCL ||
636 (getLangOpts().CUDA &&
637 getContext().getTargetInfo().getTriple().isSPIRV())))
638 return;
639
640 if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {
641 QualType HintQTy = A->getTypeHint();
642 const ExtVectorType *HintEltQTy = HintQTy->getAs<ExtVectorType>();
643 bool IsSignedInteger =
644 HintQTy->isSignedIntegerType() ||
645 (HintEltQTy && HintEltQTy->getElementType()->isSignedIntegerType());
646 llvm::Metadata *AttrMDArgs[] = {
647 llvm::ConstantAsMetadata::get(C: llvm::PoisonValue::get(
648 T: CGM.getTypes().ConvertType(T: A->getTypeHint()))),
649 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
650 Ty: llvm::IntegerType::get(C&: Context, NumBits: 32),
651 V: llvm::APInt(32, (uint64_t)(IsSignedInteger ? 1 : 0))))};
652 Fn->setMetadata(Kind: "vec_type_hint", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
653 }
654
655 if (const WorkGroupSizeHintAttr *A = FD->getAttr<WorkGroupSizeHintAttr>()) {
656 auto Eval = [&](Expr *E) {
657 return E->EvaluateKnownConstInt(Ctx: FD->getASTContext()).getExtValue();
658 };
659 llvm::Metadata *AttrMDArgs[] = {
660 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getXDim()))),
661 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getYDim()))),
662 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getZDim())))};
663 Fn->setMetadata(Kind: "work_group_size_hint", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
664 }
665
666 if (const ReqdWorkGroupSizeAttr *A = FD->getAttr<ReqdWorkGroupSizeAttr>()) {
667 auto Eval = [&](Expr *E) {
668 return E->EvaluateKnownConstInt(Ctx: FD->getASTContext()).getExtValue();
669 };
670 llvm::Metadata *AttrMDArgs[] = {
671 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getXDim()))),
672 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getYDim()))),
673 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getZDim())))};
674 Fn->setMetadata(Kind: "reqd_work_group_size", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
675 }
676
677 if (const OpenCLIntelReqdSubGroupSizeAttr *A =
678 FD->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
679 llvm::Metadata *AttrMDArgs[] = {
680 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: A->getSubGroupSize()))};
681 Fn->setMetadata(Kind: "intel_reqd_sub_group_size",
682 Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
683 }
684}
685
686/// Determine whether the function F ends with a return stmt.
687static bool endsWithReturn(const Decl* F) {
688 const Stmt *Body = nullptr;
689 if (auto *FD = dyn_cast_or_null<FunctionDecl>(Val: F))
690 Body = FD->getBody();
691 else if (auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(Val: F))
692 Body = OMD->getBody();
693
694 if (auto *CS = dyn_cast_or_null<CompoundStmt>(Val: Body)) {
695 auto LastStmt = CS->body_rbegin();
696 if (LastStmt != CS->body_rend())
697 return isa<ReturnStmt>(Val: *LastStmt);
698 }
699 return false;
700}
701
702void CodeGenFunction::markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {
703 if (SanOpts.has(K: SanitizerKind::Thread)) {
704 Fn->addFnAttr(Kind: "sanitize_thread_no_checking_at_run_time");
705 Fn->removeFnAttr(Kind: llvm::Attribute::SanitizeThread);
706 }
707}
708
709/// Check if the return value of this function requires sanitization.
710bool CodeGenFunction::requiresReturnValueCheck() const {
711 return requiresReturnValueNullabilityCheck() ||
712 (SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute) && CurCodeDecl &&
713 CurCodeDecl->getAttr<ReturnsNonNullAttr>());
714}
715
716static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) {
717 auto *MD = dyn_cast_or_null<CXXMethodDecl>(Val: D);
718 if (!MD || !MD->getDeclName().getAsIdentifierInfo() ||
719 !MD->getDeclName().getAsIdentifierInfo()->isStr(Str: "allocate") ||
720 (MD->getNumParams() != 1 && MD->getNumParams() != 2))
721 return false;
722
723 if (!Ctx.hasSameType(T1: MD->parameters()[0]->getType(), T2: Ctx.getSizeType()))
724 return false;
725
726 if (MD->getNumParams() == 2) {
727 auto *PT = MD->parameters()[1]->getType()->getAs<PointerType>();
728 if (!PT || !PT->isVoidPointerType() ||
729 !PT->getPointeeType().isConstQualified())
730 return false;
731 }
732
733 return true;
734}
735
736bool CodeGenFunction::isInAllocaArgument(CGCXXABI &ABI, QualType Ty) {
737 const CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
738 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
739}
740
741bool CodeGenFunction::hasInAllocaArg(const CXXMethodDecl *MD) {
742 return getTarget().getTriple().getArch() == llvm::Triple::x86 &&
743 getTarget().getCXXABI().isMicrosoft() &&
744 llvm::any_of(Range: MD->parameters(), P: [&](ParmVarDecl *P) {
745 return isInAllocaArgument(ABI&: CGM.getCXXABI(), Ty: P->getType());
746 });
747}
748
749/// Return the UBSan prologue signature for \p FD if one is available.
750static llvm::Constant *getPrologueSignature(CodeGenModule &CGM,
751 const FunctionDecl *FD) {
752 if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: FD))
753 if (!MD->isStatic())
754 return nullptr;
755 return CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM);
756}
757
758void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
759 llvm::Function *Fn,
760 const CGFunctionInfo &FnInfo,
761 const FunctionArgList &Args,
762 SourceLocation Loc,
763 SourceLocation StartLoc) {
764 assert(!CurFn &&
765 "Do not use a CodeGenFunction object for more than one function");
766
767 const Decl *D = GD.getDecl();
768
769 DidCallStackSave = false;
770 CurCodeDecl = D;
771 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
772 if (FD && FD->usesSEHTry())
773 CurSEHParent = GD;
774 CurFuncDecl = (D ? D->getNonClosureContext() : nullptr);
775 FnRetTy = RetTy;
776 CurFn = Fn;
777 CurFnInfo = &FnInfo;
778 assert(CurFn->isDeclaration() && "Function already has body?");
779
780 // If this function is ignored for any of the enabled sanitizers,
781 // disable the sanitizer for the function.
782 do {
783#define SANITIZER(NAME, ID) \
784 if (SanOpts.empty()) \
785 break; \
786 if (SanOpts.has(SanitizerKind::ID)) \
787 if (CGM.isInNoSanitizeList(SanitizerKind::ID, Fn, Loc)) \
788 SanOpts.set(SanitizerKind::ID, false);
789
790#include "clang/Basic/Sanitizers.def"
791#undef SANITIZER
792 } while (false);
793
794 if (D) {
795 const bool SanitizeBounds = SanOpts.hasOneOf(K: SanitizerKind::Bounds);
796 SanitizerMask no_sanitize_mask;
797 bool NoSanitizeCoverage = false;
798
799 for (auto *Attr : D->specific_attrs<NoSanitizeAttr>()) {
800 no_sanitize_mask |= Attr->getMask();
801 // SanitizeCoverage is not handled by SanOpts.
802 if (Attr->hasCoverage())
803 NoSanitizeCoverage = true;
804 }
805
806 // Apply the no_sanitize* attributes to SanOpts.
807 SanOpts.Mask &= ~no_sanitize_mask;
808 if (no_sanitize_mask & SanitizerKind::Address)
809 SanOpts.set(K: SanitizerKind::KernelAddress, Value: false);
810 if (no_sanitize_mask & SanitizerKind::KernelAddress)
811 SanOpts.set(K: SanitizerKind::Address, Value: false);
812 if (no_sanitize_mask & SanitizerKind::HWAddress)
813 SanOpts.set(K: SanitizerKind::KernelHWAddress, Value: false);
814 if (no_sanitize_mask & SanitizerKind::KernelHWAddress)
815 SanOpts.set(K: SanitizerKind::HWAddress, Value: false);
816
817 if (SanitizeBounds && !SanOpts.hasOneOf(K: SanitizerKind::Bounds))
818 Fn->addFnAttr(Kind: llvm::Attribute::NoSanitizeBounds);
819
820 if (NoSanitizeCoverage && CGM.getCodeGenOpts().hasSanitizeCoverage())
821 Fn->addFnAttr(Kind: llvm::Attribute::NoSanitizeCoverage);
822
823 // Some passes need the non-negated no_sanitize attribute. Pass them on.
824 if (CGM.getCodeGenOpts().hasSanitizeBinaryMetadata()) {
825 if (no_sanitize_mask & SanitizerKind::Thread)
826 Fn->addFnAttr(Kind: "no_sanitize_thread");
827 }
828 }
829
830 if (ShouldSkipSanitizerInstrumentation()) {
831 CurFn->addFnAttr(Kind: llvm::Attribute::DisableSanitizerInstrumentation);
832 } else {
833 // Apply sanitizer attributes to the function.
834 if (SanOpts.hasOneOf(K: SanitizerKind::Address | SanitizerKind::KernelAddress))
835 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAddress);
836 if (SanOpts.hasOneOf(K: SanitizerKind::HWAddress |
837 SanitizerKind::KernelHWAddress))
838 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeHWAddress);
839 if (SanOpts.has(K: SanitizerKind::MemtagStack))
840 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemTag);
841 if (SanOpts.has(K: SanitizerKind::Thread))
842 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeThread);
843 if (SanOpts.has(K: SanitizerKind::Type))
844 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeType);
845 if (SanOpts.has(K: SanitizerKind::NumericalStability))
846 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeNumericalStability);
847 if (SanOpts.hasOneOf(K: SanitizerKind::Memory | SanitizerKind::KernelMemory))
848 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemory);
849 if (SanOpts.has(K: SanitizerKind::AllocToken))
850 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAllocToken);
851 }
852 if (SanOpts.has(K: SanitizerKind::SafeStack))
853 Fn->addFnAttr(Kind: llvm::Attribute::SafeStack);
854 if (SanOpts.has(K: SanitizerKind::ShadowCallStack))
855 Fn->addFnAttr(Kind: llvm::Attribute::ShadowCallStack);
856
857 if (SanOpts.has(K: SanitizerKind::Realtime))
858 if (FD && FD->getASTContext().hasAnyFunctionEffects())
859 for (const FunctionEffectWithCondition &Fe : FD->getFunctionEffects()) {
860 if (Fe.Effect.kind() == FunctionEffect::Kind::NonBlocking)
861 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeRealtime);
862 else if (Fe.Effect.kind() == FunctionEffect::Kind::Blocking)
863 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeRealtimeBlocking);
864 }
865
866 // Apply fuzzing attribute to the function.
867 if (SanOpts.hasOneOf(K: SanitizerKind::Fuzzer | SanitizerKind::FuzzerNoLink))
868 Fn->addFnAttr(Kind: llvm::Attribute::OptForFuzzing);
869
870 // Ignore TSan memory acesses from within ObjC/ObjC++ dealloc, initialize,
871 // .cxx_destruct, __destroy_helper_block_ and all of their calees at run time.
872 if (SanOpts.has(K: SanitizerKind::Thread)) {
873 if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(Val: D)) {
874 const IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(argIndex: 0);
875 if (OMD->getMethodFamily() == OMF_dealloc ||
876 OMD->getMethodFamily() == OMF_initialize ||
877 (OMD->getSelector().isUnarySelector() && II->isStr(Str: ".cxx_destruct"))) {
878 markAsIgnoreThreadCheckingAtRuntime(Fn);
879 }
880 }
881 }
882
883 // Ignore unrelated casts in STL allocate() since the allocator must cast
884 // from void* to T* before object initialization completes. Don't match on the
885 // namespace because not all allocators are in std::
886 if (D && SanOpts.has(K: SanitizerKind::CFIUnrelatedCast)) {
887 if (matchesStlAllocatorFn(D, Ctx: getContext()))
888 SanOpts.Mask &= ~SanitizerKind::CFIUnrelatedCast;
889 }
890
891 // Ignore null checks in coroutine functions since the coroutines passes
892 // are not aware of how to move the extra UBSan instructions across the split
893 // coroutine boundaries.
894 if (D && SanOpts.has(K: SanitizerKind::Null))
895 if (FD && FD->getBody() &&
896 FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass)
897 SanOpts.Mask &= ~SanitizerKind::Null;
898
899 // Apply xray attributes to the function (as a string, for now)
900 bool AlwaysXRayAttr = false;
901 if (const auto *XRayAttr = D ? D->getAttr<XRayInstrumentAttr>() : nullptr) {
902 if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
903 K: XRayInstrKind::FunctionEntry) ||
904 CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
905 K: XRayInstrKind::FunctionExit)) {
906 if (XRayAttr->alwaysXRayInstrument() && ShouldXRayInstrumentFunction()) {
907 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-always");
908 AlwaysXRayAttr = true;
909 }
910 if (XRayAttr->neverXRayInstrument())
911 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-never");
912 if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())
913 if (ShouldXRayInstrumentFunction())
914 Fn->addFnAttr(Kind: "xray-log-args",
915 Val: llvm::utostr(X: LogArgs->getArgumentCount()));
916 }
917 } else {
918 if (ShouldXRayInstrumentFunction() && !CGM.imbueXRayAttrs(Fn, Loc))
919 Fn->addFnAttr(
920 Kind: "xray-instruction-threshold",
921 Val: llvm::itostr(X: CGM.getCodeGenOpts().XRayInstructionThreshold));
922 }
923
924 if (ShouldXRayInstrumentFunction()) {
925 if (CGM.getCodeGenOpts().XRayIgnoreLoops)
926 Fn->addFnAttr(Kind: "xray-ignore-loops");
927
928 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
929 K: XRayInstrKind::FunctionExit))
930 Fn->addFnAttr(Kind: "xray-skip-exit");
931
932 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
933 K: XRayInstrKind::FunctionEntry))
934 Fn->addFnAttr(Kind: "xray-skip-entry");
935
936 auto FuncGroups = CGM.getCodeGenOpts().XRayTotalFunctionGroups;
937 if (FuncGroups > 1) {
938 auto FuncName = llvm::ArrayRef<uint8_t>(CurFn->getName().bytes_begin(),
939 CurFn->getName().bytes_end());
940 auto Group = crc32(Data: FuncName) % FuncGroups;
941 if (Group != CGM.getCodeGenOpts().XRaySelectedFunctionGroup &&
942 !AlwaysXRayAttr)
943 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-never");
944 }
945 }
946
947 if (CGM.getCodeGenOpts().getProfileInstr() !=
948 llvm::driver::ProfileInstrKind::ProfileNone) {
949 switch (CGM.isFunctionBlockedFromProfileInstr(Fn, Loc)) {
950 case ProfileList::Skip:
951 Fn->addFnAttr(Kind: llvm::Attribute::SkipProfile);
952 break;
953 case ProfileList::Forbid:
954 Fn->addFnAttr(Kind: llvm::Attribute::NoProfile);
955 break;
956 case ProfileList::Allow:
957 break;
958 }
959 }
960
961 unsigned Count, Offset;
962 StringRef Section;
963 if (const auto *Attr =
964 D ? D->getAttr<PatchableFunctionEntryAttr>() : nullptr) {
965 Count = Attr->getCount();
966 Offset = Attr->getOffset();
967 Section = Attr->getSection();
968 } else {
969 Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;
970 Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;
971 }
972 if (Section.empty())
973 Section = CGM.getCodeGenOpts().PatchableFunctionEntrySection;
974 if (Count && Offset <= Count) {
975 Fn->addFnAttr(Kind: "patchable-function-entry", Val: std::to_string(val: Count - Offset));
976 if (Offset)
977 Fn->addFnAttr(Kind: "patchable-function-prefix", Val: std::to_string(val: Offset));
978 if (!Section.empty())
979 Fn->addFnAttr(Kind: "patchable-function-entry-section", Val: Section);
980 }
981 // Instruct that functions for COFF/CodeView targets should start with a
982 // patchable instruction, but only on x86/x64. Don't forward this to ARM/ARM64
983 // backends as they don't need it -- instructions on these architectures are
984 // always atomically patchable at runtime.
985 if (CGM.getCodeGenOpts().HotPatch &&
986 getContext().getTargetInfo().getTriple().isX86() &&
987 getContext().getTargetInfo().getTriple().getEnvironment() !=
988 llvm::Triple::CODE16)
989 Fn->addFnAttr(Kind: "patchable-function", Val: "prologue-short-redirect");
990
991 // Add no-jump-tables value.
992 if (CGM.getCodeGenOpts().NoUseJumpTables)
993 Fn->addFnAttr(Kind: "no-jump-tables", Val: "true");
994
995 // Add no-inline-line-tables value.
996 if (CGM.getCodeGenOpts().NoInlineLineTables)
997 Fn->addFnAttr(Kind: "no-inline-line-tables");
998
999 // Add profile-sample-accurate value.
1000 if (CGM.getCodeGenOpts().ProfileSampleAccurate)
1001 Fn->addFnAttr(Kind: "profile-sample-accurate");
1002
1003 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
1004 Fn->addFnAttr(Kind: "use-sample-profile");
1005
1006 if (D && D->hasAttr<CFICanonicalJumpTableAttr>())
1007 Fn->addFnAttr(Kind: "cfi-canonical-jump-table");
1008
1009 if (D && D->hasAttr<NoProfileFunctionAttr>())
1010 Fn->addFnAttr(Kind: llvm::Attribute::NoProfile);
1011
1012 if (D && D->hasAttr<HybridPatchableAttr>())
1013 Fn->addFnAttr(Kind: llvm::Attribute::HybridPatchable);
1014
1015 if (D) {
1016 // Function attributes take precedence over command line flags.
1017 if (auto *A = D->getAttr<FunctionReturnThunksAttr>()) {
1018 switch (A->getThunkType()) {
1019 case FunctionReturnThunksAttr::Kind::Keep:
1020 break;
1021 case FunctionReturnThunksAttr::Kind::Extern:
1022 Fn->addFnAttr(Kind: llvm::Attribute::FnRetThunkExtern);
1023 break;
1024 }
1025 } else if (CGM.getCodeGenOpts().FunctionReturnThunks)
1026 Fn->addFnAttr(Kind: llvm::Attribute::FnRetThunkExtern);
1027 }
1028
1029 if (FD && (getLangOpts().OpenCL ||
1030 (getLangOpts().CUDA &&
1031 getContext().getTargetInfo().getTriple().isSPIRV()) ||
1032 ((getLangOpts().HIP || getLangOpts().OffloadViaLLVM) &&
1033 getLangOpts().CUDAIsDevice))) {
1034 // Add metadata for a kernel function.
1035 EmitKernelMetadata(FD, Fn);
1036 }
1037
1038 if (FD && FD->hasAttr<ClspvLibclcBuiltinAttr>()) {
1039 Fn->setMetadata(Kind: "clspv_libclc_builtin",
1040 Node: llvm::MDNode::get(Context&: getLLVMContext(), MDs: {}));
1041 }
1042
1043 // If we are checking function types, emit a function type signature as
1044 // prologue data. Kernel functions have strict alignment requirements and
1045 // cannot be call indirectly so we do not instrument them.
1046 if (FD && SanOpts.has(K: SanitizerKind::Function) &&
1047 !FD->getType()->isCFIUncheckedCalleeFunctionType() &&
1048 llvm::isCallableCC(CC: Fn->getCallingConv())) {
1049 if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {
1050 llvm::LLVMContext &Ctx = Fn->getContext();
1051 llvm::MDBuilder MDB(Ctx);
1052 Fn->setMetadata(
1053 KindID: llvm::LLVMContext::MD_func_sanitize,
1054 Node: MDB.createRTTIPointerPrologue(
1055 PrologueSig, RTTI: getUBSanFunctionTypeHash(Ty: FD->getType())));
1056 }
1057 }
1058
1059 // If we're checking nullability, we need to know whether we can check the
1060 // return value. Initialize the flag to 'true' and refine it in EmitParmDecl.
1061 if (SanOpts.has(K: SanitizerKind::NullabilityReturn)) {
1062 auto Nullability = FnRetTy->getNullability();
1063 if (Nullability && *Nullability == NullabilityKind::NonNull &&
1064 !FnRetTy->isRecordType()) {
1065 if (!(SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute) &&
1066 CurCodeDecl && CurCodeDecl->getAttr<ReturnsNonNullAttr>()))
1067 RetValNullabilityPrecondition =
1068 llvm::ConstantInt::getTrue(Context&: getLLVMContext());
1069 }
1070 }
1071
1072 // If we're in C++ mode and the function name is "main", it is guaranteed
1073 // to be norecurse by the standard (3.6.1.3 "The function main shall not be
1074 // used within a program").
1075 //
1076 // OpenCL C 2.0 v2.2-11 s6.9.i:
1077 // Recursion is not supported.
1078 //
1079 // HLSL
1080 // Recursion is not supported.
1081 //
1082 // SYCL v1.2.1 s3.10:
1083 // kernels cannot include RTTI information, exception classes,
1084 // recursive code, virtual functions or make use of C++ libraries that
1085 // are not compiled for the device.
1086 if (FD &&
1087 ((getLangOpts().CPlusPlus && FD->isMain()) || getLangOpts().OpenCL ||
1088 getLangOpts().HLSL || getLangOpts().SYCLIsDevice ||
1089 (getLangOpts().CUDA && FD->hasAttr<CUDAGlobalAttr>())))
1090 Fn->addFnAttr(Kind: llvm::Attribute::NoRecurse);
1091
1092 llvm::RoundingMode RM = getLangOpts().getDefaultRoundingMode();
1093 llvm::fp::ExceptionBehavior FPExceptionBehavior =
1094 ToConstrainedExceptMD(Kind: getLangOpts().getDefaultExceptionMode());
1095 Builder.setDefaultConstrainedRounding(RM);
1096 Builder.setDefaultConstrainedExcept(FPExceptionBehavior);
1097 if ((FD && (FD->UsesFPIntrin() || FD->hasAttr<StrictFPAttr>())) ||
1098 (!FD && (FPExceptionBehavior != llvm::fp::ebIgnore ||
1099 RM != llvm::RoundingMode::NearestTiesToEven))) {
1100 Builder.setIsFPConstrained(true);
1101 Fn->addFnAttr(Kind: llvm::Attribute::StrictFP);
1102 }
1103
1104 // If a custom alignment is used, force realigning to this alignment on
1105 // any main function which certainly will need it.
1106 if (FD && ((FD->isMain() || FD->isMSVCRTEntryPoint()) &&
1107 CGM.getCodeGenOpts().StackAlignment))
1108 Fn->addFnAttr(Kind: "stackrealign");
1109
1110 // "main" doesn't need to zero out call-used registers.
1111 if (FD && FD->isMain())
1112 Fn->removeFnAttr(Kind: "zero-call-used-regs");
1113
1114 // Add vscale_range attribute if appropriate.
1115 llvm::StringMap<bool> FeatureMap;
1116 auto IsArmStreaming = TargetInfo::ArmStreamingKind::NotStreaming;
1117 if (FD) {
1118 getContext().getFunctionFeatureMap(FeatureMap, FD);
1119 if (const auto *T = FD->getType()->getAs<FunctionProtoType>())
1120 if (T->getAArch64SMEAttributes() &
1121 FunctionType::SME_PStateSMCompatibleMask)
1122 IsArmStreaming = TargetInfo::ArmStreamingKind::StreamingCompatible;
1123
1124 if (IsArmStreamingFunction(FD, IncludeLocallyStreaming: true))
1125 IsArmStreaming = TargetInfo::ArmStreamingKind::Streaming;
1126 }
1127 std::optional<std::pair<unsigned, unsigned>> VScaleRange =
1128 getContext().getTargetInfo().getVScaleRange(LangOpts: getLangOpts(), Mode: IsArmStreaming,
1129 FeatureMap: &FeatureMap);
1130 if (VScaleRange) {
1131 CurFn->addFnAttr(Attr: llvm::Attribute::getWithVScaleRangeArgs(
1132 Context&: getLLVMContext(), MinValue: VScaleRange->first, MaxValue: VScaleRange->second));
1133 }
1134
1135 llvm::BasicBlock *EntryBB = createBasicBlock(name: "entry", parent: CurFn);
1136
1137 // Create a marker to make it easy to insert allocas into the entryblock
1138 // later. Don't create this with the builder, because we don't want it
1139 // folded.
1140 llvm::Value *Poison = llvm::PoisonValue::get(T: Int32Ty);
1141 AllocaInsertPt = new llvm::BitCastInst(Poison, Int32Ty, "allocapt", EntryBB);
1142
1143 ReturnBlock = getJumpDestInCurrentScope(Name: "return");
1144
1145 Builder.SetInsertPoint(EntryBB);
1146
1147 // If we're checking the return value, allocate space for a pointer to a
1148 // precise source location of the checked return statement.
1149 if (requiresReturnValueCheck()) {
1150 ReturnLocation = CreateDefaultAlignTempAlloca(Ty: Int8PtrTy, Name: "return.sloc.ptr");
1151 Builder.CreateStore(Val: llvm::ConstantPointerNull::get(T: Int8PtrTy),
1152 Addr: ReturnLocation);
1153 }
1154
1155 // Emit subprogram debug descriptor.
1156 if (CGDebugInfo *DI = getDebugInfo()) {
1157 // Reconstruct the type from the argument list so that implicit parameters,
1158 // such as 'this' and 'vtt', show up in the debug info. Preserve the calling
1159 // convention.
1160 DI->emitFunctionStart(GD, Loc, ScopeLoc: StartLoc,
1161 FnType: DI->getFunctionType(FD, RetTy, Args), Fn: CurFn,
1162 CurFnIsThunk: CurFuncIsThunk);
1163 }
1164
1165 if (ShouldInstrumentFunction()) {
1166 if (CGM.getCodeGenOpts().InstrumentFunctions)
1167 CurFn->addFnAttr(Kind: "instrument-function-entry", Val: "__cyg_profile_func_enter");
1168 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
1169 CurFn->addFnAttr(Kind: "instrument-function-entry-inlined",
1170 Val: "__cyg_profile_func_enter");
1171 if (CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
1172 CurFn->addFnAttr(Kind: "instrument-function-entry-inlined",
1173 Val: "__cyg_profile_func_enter_bare");
1174 }
1175
1176 // Since emitting the mcount call here impacts optimizations such as function
1177 // inlining, we just add an attribute to insert a mcount call in backend.
1178 // The attribute "counting-function" is set to mcount function name which is
1179 // architecture dependent.
1180 if (CGM.getCodeGenOpts().InstrumentForProfiling) {
1181 // Calls to fentry/mcount should not be generated if function has
1182 // the no_instrument_function attribute.
1183 if (!CurFuncDecl || !CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()) {
1184 if (CGM.getCodeGenOpts().CallFEntry)
1185 Fn->addFnAttr(Kind: "fentry-call", Val: "true");
1186 else {
1187 Fn->addFnAttr(Kind: "instrument-function-entry-inlined",
1188 Val: getTarget().getMCountName());
1189 }
1190 if (CGM.getCodeGenOpts().MNopMCount) {
1191 if (!CGM.getCodeGenOpts().CallFEntry)
1192 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_without_opt)
1193 << "-mnop-mcount" << "-mfentry";
1194 Fn->addFnAttr(Kind: "mnop-mcount");
1195 }
1196
1197 if (CGM.getCodeGenOpts().RecordMCount) {
1198 if (!CGM.getCodeGenOpts().CallFEntry)
1199 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_without_opt)
1200 << "-mrecord-mcount" << "-mfentry";
1201 Fn->addFnAttr(Kind: "mrecord-mcount");
1202 }
1203 }
1204 }
1205
1206 if (CGM.getCodeGenOpts().PackedStack) {
1207 if (getContext().getTargetInfo().getTriple().getArch() !=
1208 llvm::Triple::systemz)
1209 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_on_target)
1210 << "-mpacked-stack";
1211 Fn->addFnAttr(Kind: "packed-stack");
1212 }
1213
1214 if (!CGM.getCodeGenOpts().ZOSPPA1Name)
1215 Fn->addFnAttr(Kind: "zos-ppa1-name", Val: "");
1216
1217 if (CGM.getCodeGenOpts().WarnStackSize != UINT_MAX &&
1218 !CGM.getDiags().isIgnored(DiagID: diag::warn_fe_backend_frame_larger_than, Loc))
1219 Fn->addFnAttr(Kind: "warn-stack-size",
1220 Val: std::to_string(val: CGM.getCodeGenOpts().WarnStackSize));
1221
1222 if (RetTy->isVoidType()) {
1223 // Void type; nothing to return.
1224 ReturnValue = Address::invalid();
1225
1226 // Count the implicit return.
1227 if (!endsWithReturn(F: D))
1228 ++NumReturnExprs;
1229 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1230 // Indirect return; emit returned value directly into sret slot.
1231 // This reduces code size, and affects correctness in C++.
1232 auto AI = CurFn->arg_begin();
1233 if (CurFnInfo->getReturnInfo().isSRetAfterThis())
1234 ++AI;
1235 ReturnValue = makeNaturalAddressForPointer(
1236 Ptr: &*AI, T: RetTy, Alignment: CurFnInfo->getReturnInfo().getIndirectAlign(), ForPointeeType: false,
1237 BaseInfo: nullptr, TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
1238 if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
1239 ReturnValuePointer =
1240 CreateDefaultAlignTempAlloca(Ty: ReturnValue.getType(), Name: "result.ptr");
1241 Builder.CreateStore(Val: ReturnValue.emitRawPointer(CGF&: *this),
1242 Addr: ReturnValuePointer);
1243 }
1244 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
1245 !hasScalarEvaluationKind(T: CurFnInfo->getReturnType())) {
1246 // Load the sret pointer from the argument struct and return into that.
1247 unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();
1248 llvm::Function::arg_iterator EI = CurFn->arg_end();
1249 --EI;
1250 llvm::Value *Addr = Builder.CreateStructGEP(
1251 Ty: CurFnInfo->getArgStruct(), Ptr: &*EI, Idx);
1252 llvm::Type *Ty =
1253 cast<llvm::GetElementPtrInst>(Val: Addr)->getResultElementType();
1254 ReturnValuePointer = Address(Addr, Ty, getPointerAlign());
1255 Addr = Builder.CreateAlignedLoad(Ty, Addr, Align: getPointerAlign(), Name: "agg.result");
1256 ReturnValue = Address(Addr, ConvertType(T: RetTy),
1257 CGM.getNaturalTypeAlignment(T: RetTy), KnownNonNull);
1258 } else {
1259 ReturnValue = CreateIRTempWithoutCast(T: RetTy, Name: "retval");
1260
1261 // Tell the epilog emitter to autorelease the result. We do this
1262 // now so that various specialized functions can suppress it
1263 // during their IR-generation.
1264 if (getLangOpts().ObjCAutoRefCount &&
1265 !CurFnInfo->isReturnsRetained() &&
1266 RetTy->isObjCRetainableType())
1267 AutoreleaseResult = true;
1268 }
1269
1270 EmitStartEHSpec(D: CurCodeDecl);
1271
1272 PrologueCleanupDepth = EHStack.stable_begin();
1273
1274 // Emit OpenMP specific initialization of the device functions.
1275 if (getLangOpts().OpenMP && CurCodeDecl)
1276 CGM.getOpenMPRuntime().emitFunctionProlog(CGF&: *this, D: CurCodeDecl);
1277
1278 if (FD && getLangOpts().HLSL) {
1279 // Handle emitting HLSL entry functions.
1280 if (FD->hasAttr<HLSLShaderAttr>()) {
1281 CGM.getHLSLRuntime().emitEntryFunction(FD, Fn);
1282 }
1283 }
1284
1285 EmitFunctionProlog(FI: *CurFnInfo, Fn: CurFn, Args);
1286
1287 if (const CXXMethodDecl *MD = dyn_cast_if_present<CXXMethodDecl>(Val: D);
1288 MD && !MD->isStatic()) {
1289 bool IsInLambda =
1290 MD->getParent()->isLambda() && MD->getOverloadedOperator() == OO_Call;
1291 if (MD->isImplicitObjectMemberFunction())
1292 CGM.getCXXABI().EmitInstanceFunctionProlog(CGF&: *this);
1293 if (IsInLambda) {
1294 // We're in a lambda; figure out the captures.
1295 MD->getParent()->getCaptureFields(Captures&: LambdaCaptureFields,
1296 ThisCapture&: LambdaThisCaptureField);
1297 if (LambdaThisCaptureField) {
1298 // If the lambda captures the object referred to by '*this' - either by
1299 // value or by reference, make sure CXXThisValue points to the correct
1300 // object.
1301
1302 // Get the lvalue for the field (which is a copy of the enclosing object
1303 // or contains the address of the enclosing object).
1304 LValue ThisFieldLValue = EmitLValueForLambdaField(Field: LambdaThisCaptureField);
1305 if (!LambdaThisCaptureField->getType()->isPointerType()) {
1306 // If the enclosing object was captured by value, just use its
1307 // address. Sign this pointer.
1308 CXXThisValue = ThisFieldLValue.getPointer(CGF&: *this);
1309 } else {
1310 // Load the lvalue pointed to by the field, since '*this' was captured
1311 // by reference.
1312 CXXThisValue =
1313 EmitLoadOfLValue(V: ThisFieldLValue, Loc: SourceLocation()).getScalarVal();
1314 }
1315 }
1316 for (auto *FD : MD->getParent()->fields()) {
1317 if (FD->hasCapturedVLAType()) {
1318 auto *ExprArg = EmitLoadOfLValue(V: EmitLValueForLambdaField(Field: FD),
1319 Loc: SourceLocation()).getScalarVal();
1320 auto VAT = FD->getCapturedVLAType();
1321 VLASizeMap[VAT->getSizeExpr()] = ExprArg;
1322 }
1323 }
1324 } else if (MD->isImplicitObjectMemberFunction()) {
1325 // Not in a lambda; just use 'this' from the method.
1326 // FIXME: Should we generate a new load for each use of 'this'? The
1327 // fast register allocator would be happier...
1328 CXXThisValue = CXXABIThisValue;
1329 }
1330
1331 // Check the 'this' pointer once per function, if it's available.
1332 if (CXXABIThisValue) {
1333 SanitizerSet SkippedChecks;
1334 SkippedChecks.set(K: SanitizerKind::ObjectSize, Value: true);
1335 QualType ThisTy = MD->getThisType();
1336
1337 // If this is the call operator of a lambda with no captures, it
1338 // may have a static invoker function, which may call this operator with
1339 // a null 'this' pointer.
1340 if (isLambdaCallOperator(MD) && MD->getParent()->isCapturelessLambda())
1341 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
1342
1343 EmitTypeCheck(
1344 TCK: isa<CXXConstructorDecl>(Val: MD) ? TCK_ConstructorCall : TCK_MemberCall,
1345 Loc, V: CXXABIThisValue, Type: ThisTy, Alignment: CXXABIThisAlignment, SkippedChecks);
1346 }
1347 }
1348
1349 // If any of the arguments have a variably modified type, make sure to
1350 // emit the type size, but only if the function is not naked. Naked functions
1351 // have no prolog to run this evaluation.
1352 if (!FD || !FD->hasAttr<NakedAttr>()) {
1353 for (const VarDecl *VD : Args) {
1354 // Dig out the type as written from ParmVarDecls; it's unclear whether
1355 // the standard (C99 6.9.1p10) requires this, but we're following the
1356 // precedent set by gcc.
1357 QualType Ty;
1358 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: VD))
1359 Ty = PVD->getOriginalType();
1360 else
1361 Ty = VD->getType();
1362
1363 if (Ty->isVariablyModifiedType())
1364 EmitVariablyModifiedType(Ty);
1365 }
1366 }
1367 // Emit a location at the end of the prologue.
1368 if (CGDebugInfo *DI = getDebugInfo())
1369 DI->EmitLocation(Builder, Loc: StartLoc);
1370 // TODO: Do we need to handle this in two places like we do with
1371 // target-features/target-cpu?
1372 if (CurFuncDecl)
1373 if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>())
1374 LargestVectorWidth = VecWidth->getVectorWidth();
1375
1376 if (CGM.shouldEmitConvergenceTokens())
1377 ConvergenceTokenStack.push_back(Elt: getOrEmitConvergenceEntryToken(F: CurFn));
1378}
1379
1380void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {
1381 incrementProfileCounter(S: Body);
1382 maybeCreateMCDCCondBitmap();
1383 if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Val: Body))
1384 EmitCompoundStmtWithoutScope(S: *S);
1385 else
1386 EmitStmt(S: Body);
1387}
1388
1389/// When instrumenting to collect profile data, the counts for some blocks
1390/// such as switch cases need to not include the fall-through counts, so
1391/// emit a branch around the instrumentation code. When not instrumenting,
1392/// this just calls EmitBlock().
1393void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
1394 const Stmt *S) {
1395 llvm::BasicBlock *SkipCountBB = nullptr;
1396 if (HaveInsertPoint() && CGM.getCodeGenOpts().hasProfileClangInstr()) {
1397 // When instrumenting for profiling, the fallthrough to certain
1398 // statements needs to skip over the instrumentation code so that we
1399 // get an accurate count.
1400 SkipCountBB = createBasicBlock(name: "skipcount");
1401 EmitBranch(Block: SkipCountBB);
1402 }
1403 EmitBlock(BB);
1404 uint64_t CurrentCount = getCurrentProfileCount();
1405 incrementProfileCounter(ExecSkip: UseExecPath, S);
1406 setCurrentProfileCount(getCurrentProfileCount() + CurrentCount);
1407 if (SkipCountBB)
1408 EmitBlock(BB: SkipCountBB);
1409}
1410
1411/// Tries to mark the given function nounwind based on the
1412/// non-existence of any throwing calls within it. We believe this is
1413/// lightweight enough to do at -O0.
1414static void TryMarkNoThrow(llvm::Function *F) {
1415 // LLVM treats 'nounwind' on a function as part of the type, so we
1416 // can't do this on functions that can be overwritten.
1417 if (F->isInterposable()) return;
1418
1419 for (llvm::BasicBlock &BB : *F)
1420 for (llvm::Instruction &I : BB)
1421 if (I.mayThrow())
1422 return;
1423
1424 F->setDoesNotThrow();
1425}
1426
1427QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD,
1428 FunctionArgList &Args) {
1429 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
1430 QualType ResTy = FD->getReturnType();
1431
1432 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD);
1433 if (MD && MD->isImplicitObjectMemberFunction()) {
1434 if (CGM.getCXXABI().HasThisReturn(GD))
1435 ResTy = MD->getThisType();
1436 else if (CGM.getCXXABI().hasMostDerivedReturn(GD))
1437 ResTy = CGM.getContext().VoidPtrTy;
1438 CGM.getCXXABI().buildThisParam(CGF&: *this, Params&: Args);
1439 }
1440
1441 // The base version of an inheriting constructor whose constructed base is a
1442 // virtual base is not passed any arguments (because it doesn't actually call
1443 // the inherited constructor).
1444 bool PassedParams = true;
1445 if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(Val: FD))
1446 if (auto Inherited = CD->getInheritedConstructor())
1447 PassedParams =
1448 getTypes().inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
1449
1450 if (PassedParams) {
1451 for (auto *Param : FD->parameters()) {
1452 Args.push_back(Elt: Param);
1453 if (!Param->hasAttr<PassObjectSizeAttr>())
1454 continue;
1455
1456 auto *Implicit = ImplicitParamDecl::Create(
1457 C&: getContext(), DC: Param->getDeclContext(), IdLoc: Param->getLocation(),
1458 /*Id=*/nullptr, T: getContext().getSizeType(), ParamKind: ImplicitParamKind::Other);
1459 SizeArguments[Param] = Implicit;
1460 Args.push_back(Elt: Implicit);
1461 }
1462 }
1463
1464 if (MD && (isa<CXXConstructorDecl>(Val: MD) || isa<CXXDestructorDecl>(Val: MD)))
1465 CGM.getCXXABI().addImplicitStructorParams(CGF&: *this, ResTy, Params&: Args);
1466
1467 return ResTy;
1468}
1469
1470void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
1471 const CGFunctionInfo &FnInfo) {
1472 assert(Fn && "generating code for null Function");
1473 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
1474 CurGD = GD;
1475
1476 FunctionArgList Args;
1477 QualType ResTy = BuildFunctionArgList(GD, Args);
1478
1479 CGM.getTargetCodeGenInfo().checkFunctionABI(CGM, Decl: FD);
1480
1481 if (FD->isInlineBuiltinDeclaration()) {
1482 // When generating code for a builtin with an inline declaration, use a
1483 // mangled name to hold the actual body, while keeping an external
1484 // definition in case the function pointer is referenced somewhere.
1485 std::string FDInlineName = (Fn->getName() + ".inline").str();
1486 llvm::Module *M = Fn->getParent();
1487 llvm::Function *Clone = M->getFunction(Name: FDInlineName);
1488 if (!Clone) {
1489 Clone = llvm::Function::Create(Ty: Fn->getFunctionType(),
1490 Linkage: llvm::GlobalValue::InternalLinkage,
1491 AddrSpace: Fn->getAddressSpace(), N: FDInlineName, M);
1492 Clone->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1493 }
1494 Fn->setLinkage(llvm::GlobalValue::ExternalLinkage);
1495 Fn = Clone;
1496 } else {
1497 // Detect the unusual situation where an inline version is shadowed by a
1498 // non-inline version. In that case we should pick the external one
1499 // everywhere. That's GCC behavior too. Unfortunately, I cannot find a way
1500 // to detect that situation before we reach codegen, so do some late
1501 // replacement.
1502 for (const FunctionDecl *PD = FD->getPreviousDecl(); PD;
1503 PD = PD->getPreviousDecl()) {
1504 if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) {
1505 std::string FDInlineName = (Fn->getName() + ".inline").str();
1506 llvm::Module *M = Fn->getParent();
1507 if (llvm::Function *Clone = M->getFunction(Name: FDInlineName)) {
1508 Clone->replaceAllUsesWith(V: Fn);
1509 Clone->eraseFromParent();
1510 }
1511 break;
1512 }
1513 }
1514 }
1515
1516 // Check if we should generate debug info for this function.
1517 if (FD->hasAttr<NoDebugAttr>()) {
1518 // Clear non-distinct debug info that was possibly attached to the function
1519 // due to an earlier declaration without the nodebug attribute
1520 Fn->setSubprogram(nullptr);
1521 // Disable debug info indefinitely for this function
1522 DebugInfo = nullptr;
1523 }
1524 // Finalize function debug info on exit.
1525 llvm::scope_exit Cleanup([this] {
1526 if (CGDebugInfo *DI = getDebugInfo())
1527 DI->completeFunction();
1528 });
1529
1530 // The function might not have a body if we're generating thunks for a
1531 // function declaration.
1532 SourceRange BodyRange;
1533 if (Stmt *Body = FD->getBody())
1534 BodyRange = Body->getSourceRange();
1535 else
1536 BodyRange = FD->getLocation();
1537 CurEHLocation = BodyRange.getEnd();
1538
1539 // Use the location of the start of the function to determine where
1540 // the function definition is located. By default use the location
1541 // of the declaration as the location for the subprogram. A function
1542 // may lack a declaration in the source code if it is created by code
1543 // gen. (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).
1544 SourceLocation Loc = FD->getLocation();
1545
1546 // If this is a function specialization then use the pattern body
1547 // as the location for the function.
1548 if (const FunctionDecl *SpecDecl = FD->getTemplateInstantiationPattern())
1549 if (SpecDecl->hasBody(Definition&: SpecDecl))
1550 Loc = SpecDecl->getLocation();
1551
1552 Stmt *Body = FD->getBody();
1553
1554 if (Body) {
1555 // Coroutines always emit lifetime markers.
1556 if (isa<CoroutineBodyStmt>(Val: Body))
1557 ShouldEmitLifetimeMarkers = true;
1558
1559 // Initialize helper which will detect jumps which can cause invalid
1560 // lifetime markers.
1561 if (ShouldEmitLifetimeMarkers)
1562 Bypasses.Init(CGM, Body);
1563 }
1564
1565 // Emit the standard function prologue.
1566 StartFunction(GD, RetTy: ResTy, Fn, FnInfo, Args, Loc, StartLoc: BodyRange.getBegin());
1567
1568 // Save parameters for coroutine function.
1569 if (Body && isa_and_nonnull<CoroutineBodyStmt>(Val: Body))
1570 llvm::append_range(C&: FnArgs, R: FD->parameters());
1571
1572 // Ensure that the function adheres to the forward progress guarantee, which
1573 // is required by certain optimizations.
1574 // In C++11 and up, the attribute will be removed if the body contains a
1575 // trivial empty loop.
1576 if (checkIfFunctionMustProgress())
1577 CurFn->addFnAttr(Kind: llvm::Attribute::MustProgress);
1578
1579 // Generate the body of the function.
1580 PGO->assignRegionCounters(GD, Fn: CurFn);
1581 if (isa<CXXDestructorDecl>(Val: FD))
1582 EmitDestructorBody(Args);
1583 else if (isa<CXXConstructorDecl>(Val: FD))
1584 EmitConstructorBody(Args);
1585 else if (getLangOpts().CUDA &&
1586 !getLangOpts().CUDAIsDevice &&
1587 FD->hasAttr<CUDAGlobalAttr>())
1588 CGM.getCUDARuntime().emitDeviceStub(CGF&: *this, Args);
1589 else if (isa<CXXMethodDecl>(Val: FD) &&
1590 cast<CXXMethodDecl>(Val: FD)->isLambdaStaticInvoker()) {
1591 // The lambda static invoker function is special, because it forwards or
1592 // clones the body of the function call operator (but is actually static).
1593 EmitLambdaStaticInvokeBody(MD: cast<CXXMethodDecl>(Val: FD));
1594 } else if (isa<CXXMethodDecl>(Val: FD) &&
1595 isLambdaCallOperator(MD: cast<CXXMethodDecl>(Val: FD)) &&
1596 !FnInfo.isDelegateCall() &&
1597 cast<CXXMethodDecl>(Val: FD)->getParent()->getLambdaStaticInvoker() &&
1598 hasInAllocaArg(MD: cast<CXXMethodDecl>(Val: FD))) {
1599 // If emitting a lambda with static invoker on X86 Windows, change
1600 // the call operator body.
1601 // Make sure that this is a call operator with an inalloca arg and check
1602 // for delegate call to make sure this is the original call op and not the
1603 // new forwarding function for the static invoker.
1604 EmitLambdaInAllocaCallOpBody(MD: cast<CXXMethodDecl>(Val: FD));
1605 } else if (FD->isDefaulted() && isa<CXXMethodDecl>(Val: FD) &&
1606 (cast<CXXMethodDecl>(Val: FD)->isCopyAssignmentOperator() ||
1607 cast<CXXMethodDecl>(Val: FD)->isMoveAssignmentOperator())) {
1608 // Implicit copy-assignment gets the same special treatment as implicit
1609 // copy-constructors.
1610 emitImplicitAssignmentOperatorBody(Args);
1611 } else if (DeviceKernelAttr::isOpenCLSpelling(
1612 A: FD->getAttr<DeviceKernelAttr>()) &&
1613 GD.getKernelReferenceKind() == KernelReferenceKind::Kernel) {
1614 CallArgList CallArgs;
1615 for (unsigned i = 0; i < Args.size(); ++i) {
1616 Address ArgAddr = GetAddrOfLocalVar(VD: Args[i]);
1617 QualType ArgQualType = Args[i]->getType();
1618 RValue ArgRValue = convertTempToRValue(addr: ArgAddr, type: ArgQualType, Loc);
1619 CallArgs.add(rvalue: ArgRValue, type: ArgQualType);
1620 }
1621 GlobalDecl GDStub = GlobalDecl(FD, KernelReferenceKind::Stub);
1622 const FunctionType *FT = cast<FunctionType>(Val: FD->getType());
1623 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
1624 const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
1625 Args: CallArgs, Ty: FT, /*ChainCall=*/false, ABIInfoFD: getCurrentFunctionDecl());
1626 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
1627 llvm::Constant *GDStubFunctionPointer =
1628 CGM.getRawFunctionPointer(GD: GDStub, Ty: FTy);
1629 CGCallee GDStubCallee = CGCallee::forDirect(functionPtr: GDStubFunctionPointer, abstractInfo: GDStub);
1630 EmitCall(CallInfo: FnInfo, Callee: GDStubCallee, ReturnValue: ReturnValueSlot(), Args: CallArgs, CallOrInvoke: nullptr, IsMustTail: false,
1631 Loc);
1632 } else if (Body) {
1633 EmitFunctionBody(Body);
1634 } else
1635 llvm_unreachable("no definition for emitted function");
1636
1637 // C++11 [stmt.return]p2:
1638 // Flowing off the end of a function [...] results in undefined behavior in
1639 // a value-returning function.
1640 // C11 6.9.1p12:
1641 // If the '}' that terminates a function is reached, and the value of the
1642 // function call is used by the caller, the behavior is undefined.
1643 if (getLangOpts().CPlusPlus && !FD->hasImplicitReturnZero() && !SawAsmBlock &&
1644 !FD->getReturnType()->isVoidType() && Builder.GetInsertBlock()) {
1645 bool ShouldEmitUnreachable =
1646 CGM.getCodeGenOpts().StrictReturn ||
1647 !CGM.MayDropFunctionReturn(Context: FD->getASTContext(), ReturnType: FD->getReturnType());
1648 if (SanOpts.has(K: SanitizerKind::Return)) {
1649 auto CheckOrdinal = SanitizerKind::SO_Return;
1650 auto CheckHandler = SanitizerHandler::MissingReturn;
1651 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
1652 llvm::Value *IsFalse = Builder.getFalse();
1653 EmitCheck(Checked: std::make_pair(x&: IsFalse, y&: CheckOrdinal), Check: CheckHandler,
1654 StaticArgs: EmitCheckSourceLocation(Loc: FD->getLocation()), DynamicArgs: {});
1655 } else if (ShouldEmitUnreachable) {
1656 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1657 EmitTrapCall(IntrID: llvm::Intrinsic::trap);
1658 }
1659 if (SanOpts.has(K: SanitizerKind::Return) || ShouldEmitUnreachable) {
1660 Builder.CreateUnreachable();
1661 Builder.ClearInsertionPoint();
1662 }
1663 }
1664
1665 // Emit the standard function epilogue.
1666 FinishFunction(EndLoc: BodyRange.getEnd());
1667
1668 PGO->verifyCounterMap();
1669
1670 if (CurCodeDecl->hasAttr<PersonalityAttr>()) {
1671 StringRef Identifier =
1672 CurCodeDecl->getAttr<PersonalityAttr>()->getRoutine()->getName();
1673 llvm::FunctionCallee PersonalityRoutine =
1674 CGM.CreateRuntimeFunction(Ty: llvm::FunctionType::get(Result: CGM.Int32Ty, isVarArg: true),
1675 Name: Identifier, ExtraAttrs: {}, /*local=*/Local: true);
1676 Fn->setPersonalityFn(cast<llvm::Constant>(Val: PersonalityRoutine.getCallee()));
1677 }
1678
1679 // If we haven't marked the function nothrow through other means, do
1680 // a quick pass now to see if we can.
1681 if (!CurFn->doesNotThrow())
1682 TryMarkNoThrow(F: CurFn);
1683}
1684
1685/// ContainsLabel - Return true if the statement contains a label in it. If
1686/// this statement is not executed normally, it not containing a label means
1687/// that we can just remove the code.
1688bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
1689 // Null statement, not a label!
1690 if (!S) return false;
1691
1692 // If this is a label, we have to emit the code, consider something like:
1693 // if (0) { ... foo: bar(); } goto foo;
1694 //
1695 // TODO: If anyone cared, we could track __label__'s, since we know that you
1696 // can't jump to one from outside their declared region.
1697 if (isa<LabelStmt>(Val: S))
1698 return true;
1699
1700 // If this is a case/default statement, and we haven't seen a switch, we have
1701 // to emit the code.
1702 if (isa<SwitchCase>(Val: S) && !IgnoreCaseStmts)
1703 return true;
1704
1705 // If this is a switch statement, we want to ignore cases below it.
1706 if (isa<SwitchStmt>(Val: S))
1707 IgnoreCaseStmts = true;
1708
1709 // Scan subexpressions for verboten labels.
1710 for (const Stmt *SubStmt : S->children())
1711 if (ContainsLabel(S: SubStmt, IgnoreCaseStmts))
1712 return true;
1713
1714 return false;
1715}
1716
1717/// containsBreak - Return true if the statement contains a break out of it.
1718/// If the statement (recursively) contains a switch or loop with a break
1719/// inside of it, this is fine.
1720bool CodeGenFunction::containsBreak(const Stmt *S) {
1721 // Null statement, not a label!
1722 if (!S) return false;
1723
1724 // If this is a switch or loop that defines its own break scope, then we can
1725 // include it and anything inside of it.
1726 if (isa<SwitchStmt>(Val: S) || isa<WhileStmt>(Val: S) || isa<DoStmt>(Val: S) ||
1727 isa<ForStmt>(Val: S))
1728 return false;
1729
1730 if (isa<BreakStmt>(Val: S))
1731 return true;
1732
1733 // Scan subexpressions for verboten breaks.
1734 for (const Stmt *SubStmt : S->children())
1735 if (containsBreak(S: SubStmt))
1736 return true;
1737
1738 return false;
1739}
1740
1741bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) {
1742 if (!S) return false;
1743
1744 // Some statement kinds add a scope and thus never add a decl to the current
1745 // scope. Note, this list is longer than the list of statements that might
1746 // have an unscoped decl nested within them, but this way is conservatively
1747 // correct even if more statement kinds are added.
1748 if (isa<IfStmt>(Val: S) || isa<SwitchStmt>(Val: S) || isa<WhileStmt>(Val: S) ||
1749 isa<DoStmt>(Val: S) || isa<ForStmt>(Val: S) || isa<CompoundStmt>(Val: S) ||
1750 isa<CXXForRangeStmt>(Val: S) || isa<CXXTryStmt>(Val: S) ||
1751 isa<ObjCForCollectionStmt>(Val: S) || isa<ObjCAtTryStmt>(Val: S))
1752 return false;
1753
1754 if (isa<DeclStmt>(Val: S))
1755 return true;
1756
1757 for (const Stmt *SubStmt : S->children())
1758 if (mightAddDeclToScope(S: SubStmt))
1759 return true;
1760
1761 return false;
1762}
1763
1764/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1765/// to a constant, or if it does but contains a label, return false. If it
1766/// constant folds return true and set the boolean result in Result.
1767bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1768 bool &ResultBool,
1769 bool AllowLabels) {
1770 // If MC/DC is enabled, disable folding so that we can instrument all
1771 // conditions to yield complete test vectors. We still keep track of
1772 // folded conditions during region mapping and visualization.
1773 if (!AllowLabels && CGM.getCodeGenOpts().hasProfileClangInstr() &&
1774 CGM.getCodeGenOpts().MCDCCoverage)
1775 return false;
1776
1777 llvm::APSInt ResultInt;
1778 if (!ConstantFoldsToSimpleInteger(Cond, Result&: ResultInt, AllowLabels))
1779 return false;
1780
1781 ResultBool = ResultInt.getBoolValue();
1782 return true;
1783}
1784
1785/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1786/// to a constant, or if it does but contains a label, return false. If it
1787/// constant folds return true and set the folded value.
1788bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1789 llvm::APSInt &ResultInt,
1790 bool AllowLabels) {
1791 // FIXME: Rename and handle conversion of other evaluatable things
1792 // to bool.
1793 Expr::EvalResult Result;
1794 if (!Cond->EvaluateAsInt(Result, Ctx: getContext()))
1795 return false; // Not foldable, not integer or not fully evaluatable.
1796
1797 llvm::APSInt Int = Result.Val.getInt();
1798 if (!AllowLabels && CodeGenFunction::ContainsLabel(S: Cond))
1799 return false; // Contains a label.
1800
1801 PGO->markStmtMaybeUsed(S: Cond);
1802 ResultInt = std::move(Int);
1803 return true;
1804}
1805
1806/// Strip parentheses and simplistic logical-NOT operators.
1807const Expr *CodeGenFunction::stripCond(const Expr *C) {
1808 while (true) {
1809 const Expr *SC = IgnoreExprNodes(
1810 E: C, Fns&: IgnoreParensSingleStep, Fns&: IgnoreUOpLNotSingleStep,
1811 Fns&: IgnoreBuiltinExpectSingleStep, Fns&: IgnoreImplicitCastsSingleStep);
1812 if (C == SC)
1813 return SC;
1814 C = SC;
1815 }
1816}
1817
1818/// Determine whether the given condition is an instrumentable condition
1819/// (i.e. no "&&" or "||").
1820bool CodeGenFunction::isInstrumentedCondition(const Expr *C) {
1821 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: stripCond(C));
1822 return (!BOp || !BOp->isLogicalOp());
1823}
1824
1825/// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
1826/// increments a profile counter based on the semantics of the given logical
1827/// operator opcode. This is used to instrument branch condition coverage for
1828/// logical operators.
1829void CodeGenFunction::EmitBranchToCounterBlock(
1830 const Expr *Cond, BinaryOperator::Opcode LOp, llvm::BasicBlock *TrueBlock,
1831 llvm::BasicBlock *FalseBlock, uint64_t TrueCount /* = 0 */,
1832 Stmt::Likelihood LH /* =None */, const Expr *CntrIdx /* = nullptr */) {
1833 // If not instrumenting, just emit a branch.
1834 bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr();
1835 if (!InstrumentRegions || !isInstrumentedCondition(C: Cond))
1836 return EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount, LH);
1837
1838 const Stmt *CntrStmt = (CntrIdx ? CntrIdx : Cond);
1839
1840 llvm::BasicBlock *ThenBlock = nullptr;
1841 llvm::BasicBlock *ElseBlock = nullptr;
1842 llvm::BasicBlock *NextBlock = nullptr;
1843
1844 // Create the block we'll use to increment the appropriate counter.
1845 llvm::BasicBlock *CounterIncrBlock = createBasicBlock(name: "lop.rhscnt");
1846
1847 llvm::BasicBlock *SkipIncrBlock =
1848 (hasSkipCounter(S: CntrStmt) ? createBasicBlock(name: "lop.rhsskip") : nullptr);
1849 llvm::BasicBlock *SkipNextBlock = nullptr;
1850
1851 // Set block pointers according to Logical-AND (BO_LAnd) semantics. This
1852 // means we need to evaluate the condition and increment the counter on TRUE:
1853 //
1854 // if (Cond)
1855 // goto CounterIncrBlock;
1856 // else
1857 // goto FalseBlock;
1858 //
1859 // CounterIncrBlock:
1860 // Counter++;
1861 // goto TrueBlock;
1862
1863 if (LOp == BO_LAnd) {
1864 SkipNextBlock = FalseBlock;
1865 ThenBlock = CounterIncrBlock;
1866 ElseBlock = (SkipIncrBlock ? SkipIncrBlock : SkipNextBlock);
1867 NextBlock = TrueBlock;
1868 }
1869
1870 // Set block pointers according to Logical-OR (BO_LOr) semantics. This means
1871 // we need to evaluate the condition and increment the counter on FALSE:
1872 //
1873 // if (Cond)
1874 // goto TrueBlock;
1875 // else
1876 // goto CounterIncrBlock;
1877 //
1878 // CounterIncrBlock:
1879 // Counter++;
1880 // goto FalseBlock;
1881
1882 else if (LOp == BO_LOr) {
1883 SkipNextBlock = TrueBlock;
1884 ThenBlock = (SkipIncrBlock ? SkipIncrBlock : SkipNextBlock);
1885 ElseBlock = CounterIncrBlock;
1886 NextBlock = FalseBlock;
1887 } else {
1888 llvm_unreachable("Expected Opcode must be that of a Logical Operator");
1889 }
1890
1891 // Emit Branch based on condition.
1892 EmitBranchOnBoolExpr(Cond, TrueBlock: ThenBlock, FalseBlock: ElseBlock, TrueCount, LH);
1893
1894 if (SkipIncrBlock) {
1895 EmitBlock(BB: SkipIncrBlock);
1896 incrementProfileCounter(ExecSkip: UseSkipPath, S: CntrStmt);
1897 EmitBranch(Block: SkipNextBlock);
1898 }
1899
1900 // Emit the block containing the counter increment(s).
1901 EmitBlock(BB: CounterIncrBlock);
1902
1903 // Increment corresponding counter; if index not provided, use Cond as index.
1904 incrementProfileCounter(ExecSkip: UseExecPath, S: CntrStmt);
1905
1906 // Go to the next block.
1907 EmitBranch(Block: NextBlock);
1908}
1909
1910/// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an if
1911/// statement) to the specified blocks. Based on the condition, this might try
1912/// to simplify the codegen of the conditional based on the branch.
1913/// \param LH The value of the likelihood attribute on the True branch.
1914/// \param ConditionalOp Used by MC/DC code coverage to track the result of the
1915/// ConditionalOperator (ternary) through a recursive call for the operator's
1916/// LHS and RHS nodes.
1917void CodeGenFunction::EmitBranchOnBoolExpr(
1918 const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock,
1919 uint64_t TrueCount, Stmt::Likelihood LH, const Expr *ConditionalOp,
1920 const VarDecl *ConditionalDecl) {
1921 Cond = Cond->IgnoreParens();
1922
1923 if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Val: Cond)) {
1924 bool HasSkip = hasSkipCounter(S: CondBOp);
1925
1926 // Handle X && Y in a condition.
1927 if (CondBOp->getOpcode() == BO_LAnd) {
1928 // If we have "1 && X", simplify the code. "0 && X" would have constant
1929 // folded if the case was simple enough.
1930 bool ConstantBool = false;
1931 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getLHS(), ResultBool&: ConstantBool) &&
1932 ConstantBool) {
1933 // br(1 && X) -> br(X).
1934 incrementProfileCounter(S: CondBOp);
1935 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LAnd, TrueBlock,
1936 FalseBlock, TrueCount, LH);
1937 return;
1938 }
1939
1940 // If we have "X && 1", simplify the code to use an uncond branch.
1941 // "X && 0" would have been constant folded to 0.
1942 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getRHS(), ResultBool&: ConstantBool) &&
1943 ConstantBool) {
1944 // br(X && 1) -> br(X).
1945 EmitBranchToCounterBlock(Cond: CondBOp->getLHS(), LOp: BO_LAnd, TrueBlock,
1946 FalseBlock, TrueCount, LH, CntrIdx: CondBOp);
1947 return;
1948 }
1949
1950 // Emit the LHS as a conditional. If the LHS conditional is false, we
1951 // want to jump to the FalseBlock.
1952 llvm::BasicBlock *LHSTrue = createBasicBlock(name: "land.lhs.true");
1953 llvm::BasicBlock *LHSFalse =
1954 (HasSkip ? createBasicBlock(name: "land.lhsskip") : FalseBlock);
1955 // The counter tells us how often we evaluate RHS, and all of TrueCount
1956 // can be propagated to that branch.
1957 uint64_t RHSCount = getProfileCount(S: CondBOp->getRHS());
1958
1959 ConditionalEvaluation eval(*this);
1960 {
1961 ApplyDebugLocation DL(*this, Cond);
1962 // Propagate the likelihood attribute like __builtin_expect
1963 // __builtin_expect(X && Y, 1) -> X and Y are likely
1964 // __builtin_expect(X && Y, 0) -> only Y is unlikely
1965 EmitBranchOnBoolExpr(Cond: CondBOp->getLHS(), TrueBlock: LHSTrue, FalseBlock: LHSFalse, TrueCount: RHSCount,
1966 LH: LH == Stmt::LH_Unlikely ? Stmt::LH_None : LH);
1967 if (HasSkip) {
1968 EmitBlock(BB: LHSFalse);
1969 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondBOp);
1970 EmitBranch(Block: FalseBlock);
1971 }
1972 EmitBlock(BB: LHSTrue);
1973 }
1974
1975 incrementProfileCounter(ExecSkip: UseExecPath, S: CondBOp);
1976 setCurrentProfileCount(getProfileCount(S: CondBOp->getRHS()));
1977
1978 // Any temporaries created here are conditional.
1979 eval.begin(CGF&: *this);
1980 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LAnd, TrueBlock,
1981 FalseBlock, TrueCount, LH);
1982 eval.end(CGF&: *this);
1983 return;
1984 }
1985
1986 if (CondBOp->getOpcode() == BO_LOr) {
1987 // If we have "0 || X", simplify the code. "1 || X" would have constant
1988 // folded if the case was simple enough.
1989 bool ConstantBool = false;
1990 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getLHS(), ResultBool&: ConstantBool) &&
1991 !ConstantBool) {
1992 // br(0 || X) -> br(X).
1993 incrementProfileCounter(S: CondBOp);
1994 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LOr, TrueBlock,
1995 FalseBlock, TrueCount, LH);
1996 return;
1997 }
1998
1999 // If we have "X || 0", simplify the code to use an uncond branch.
2000 // "X || 1" would have been constant folded to 1.
2001 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getRHS(), ResultBool&: ConstantBool) &&
2002 !ConstantBool) {
2003 // br(X || 0) -> br(X).
2004 EmitBranchToCounterBlock(Cond: CondBOp->getLHS(), LOp: BO_LOr, TrueBlock,
2005 FalseBlock, TrueCount, LH, CntrIdx: CondBOp);
2006 return;
2007 }
2008 // Emit the LHS as a conditional. If the LHS conditional is true, we
2009 // want to jump to the TrueBlock.
2010 llvm::BasicBlock *LHSTrue =
2011 (HasSkip ? createBasicBlock(name: "lor.lhsskip") : TrueBlock);
2012 llvm::BasicBlock *LHSFalse = createBasicBlock(name: "lor.lhs.false");
2013 // We have the count for entry to the RHS and for the whole expression
2014 // being true, so we can divy up True count between the short circuit and
2015 // the RHS.
2016 uint64_t LHSCount =
2017 getCurrentProfileCount() - getProfileCount(S: CondBOp->getRHS());
2018 uint64_t RHSCount = TrueCount - LHSCount;
2019
2020 ConditionalEvaluation eval(*this);
2021 {
2022 // Propagate the likelihood attribute like __builtin_expect
2023 // __builtin_expect(X || Y, 1) -> only Y is likely
2024 // __builtin_expect(X || Y, 0) -> both X and Y are unlikely
2025 ApplyDebugLocation DL(*this, Cond);
2026 EmitBranchOnBoolExpr(Cond: CondBOp->getLHS(), TrueBlock: LHSTrue, FalseBlock: LHSFalse, TrueCount: LHSCount,
2027 LH: LH == Stmt::LH_Likely ? Stmt::LH_None : LH);
2028 if (HasSkip) {
2029 EmitBlock(BB: LHSTrue);
2030 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondBOp);
2031 EmitBranch(Block: TrueBlock);
2032 }
2033 EmitBlock(BB: LHSFalse);
2034 }
2035
2036 incrementProfileCounter(ExecSkip: UseExecPath, S: CondBOp);
2037 setCurrentProfileCount(getProfileCount(S: CondBOp->getRHS()));
2038
2039 // Any temporaries created here are conditional.
2040 eval.begin(CGF&: *this);
2041 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LOr, TrueBlock, FalseBlock,
2042 TrueCount: RHSCount, LH);
2043
2044 eval.end(CGF&: *this);
2045 return;
2046 }
2047 }
2048
2049 if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Val: Cond)) {
2050 // br(!x, t, f) -> br(x, f, t)
2051 // Avoid doing this optimization when instrumenting a condition for MC/DC.
2052 // LNot is taken as part of the condition for simplicity, and changing its
2053 // sense negatively impacts test vector tracking.
2054 bool MCDCCondition = CGM.getCodeGenOpts().hasProfileClangInstr() &&
2055 CGM.getCodeGenOpts().MCDCCoverage &&
2056 isInstrumentedCondition(C: Cond);
2057 if (CondUOp->getOpcode() == UO_LNot && !MCDCCondition) {
2058 // Negate the count.
2059 uint64_t FalseCount = getCurrentProfileCount() - TrueCount;
2060 // The values of the enum are chosen to make this negation possible.
2061 LH = static_cast<Stmt::Likelihood>(-LH);
2062 // Negate the condition and swap the destination blocks.
2063 return EmitBranchOnBoolExpr(Cond: CondUOp->getSubExpr(), TrueBlock: FalseBlock, FalseBlock: TrueBlock,
2064 TrueCount: FalseCount, LH);
2065 }
2066 }
2067
2068 if (const ConditionalOperator *CondOp = dyn_cast<ConditionalOperator>(Val: Cond)) {
2069 // br(c ? x : y, t, f) -> br(c, br(x, t, f), br(y, t, f))
2070 llvm::BasicBlock *LHSBlock = createBasicBlock(name: "cond.true");
2071 llvm::BasicBlock *RHSBlock = createBasicBlock(name: "cond.false");
2072
2073 // The ConditionalOperator itself has no likelihood information for its
2074 // true and false branches. This matches the behavior of __builtin_expect.
2075 ConditionalEvaluation cond(*this);
2076 EmitBranchOnBoolExpr(Cond: CondOp->getCond(), TrueBlock: LHSBlock, FalseBlock: RHSBlock,
2077 TrueCount: getProfileCount(S: CondOp), LH: Stmt::LH_None);
2078
2079 // When computing PGO branch weights, we only know the overall count for
2080 // the true block. This code is essentially doing tail duplication of the
2081 // naive code-gen, introducing new edges for which counts are not
2082 // available. Divide the counts proportionally between the LHS and RHS of
2083 // the conditional operator.
2084 uint64_t LHSScaledTrueCount = 0;
2085 if (TrueCount) {
2086 double LHSRatio =
2087 getProfileCount(S: CondOp) / (double)getCurrentProfileCount();
2088 LHSScaledTrueCount = TrueCount * LHSRatio;
2089 }
2090
2091 cond.begin(CGF&: *this);
2092 EmitBlock(BB: LHSBlock);
2093 incrementProfileCounter(ExecSkip: UseExecPath, S: CondOp);
2094 {
2095 ApplyDebugLocation DL(*this, Cond);
2096 EmitBranchOnBoolExpr(Cond: CondOp->getLHS(), TrueBlock, FalseBlock,
2097 TrueCount: LHSScaledTrueCount, LH, ConditionalOp: CondOp);
2098 }
2099 cond.end(CGF&: *this);
2100
2101 cond.begin(CGF&: *this);
2102 EmitBlock(BB: RHSBlock);
2103 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondOp);
2104 EmitBranchOnBoolExpr(Cond: CondOp->getRHS(), TrueBlock, FalseBlock,
2105 TrueCount: TrueCount - LHSScaledTrueCount, LH, ConditionalOp: CondOp);
2106 cond.end(CGF&: *this);
2107
2108 return;
2109 }
2110
2111 if (const CXXThrowExpr *Throw = dyn_cast<CXXThrowExpr>(Val: Cond)) {
2112 // Conditional operator handling can give us a throw expression as a
2113 // condition for a case like:
2114 // br(c ? throw x : y, t, f) -> br(c, br(throw x, t, f), br(y, t, f)
2115 // Fold this to:
2116 // br(c, throw x, br(y, t, f))
2117 EmitCXXThrowExpr(E: Throw, /*KeepInsertionPoint*/false);
2118 return;
2119 }
2120
2121 // Emit the code with the fully general case.
2122 llvm::Value *CondV;
2123 {
2124 ApplyDebugLocation DL(*this, Cond);
2125 CondV = EvaluateExprAsBool(E: Cond);
2126 }
2127
2128 MaybeEmitDeferredVarDeclInit(var: ConditionalDecl);
2129
2130 // If not at the top of the logical operator nest, update MCDC temp with the
2131 // boolean result of the evaluated condition.
2132 {
2133 const Expr *MCDCBaseExpr = Cond;
2134 // When a nested ConditionalOperator (ternary) is encountered in a boolean
2135 // expression, MC/DC tracks the result of the ternary, and this is tied to
2136 // the ConditionalOperator expression and not the ternary's LHS or RHS. If
2137 // this is the case, the ConditionalOperator expression is passed through
2138 // the ConditionalOp parameter and then used as the MCDC base expression.
2139 if (ConditionalOp)
2140 MCDCBaseExpr = ConditionalOp;
2141
2142 if (isMCDCBranchExpr(E: stripCond(C: MCDCBaseExpr)) &&
2143 !isMCDCDecisionExpr(E: stripCond(C: Cond)))
2144 maybeUpdateMCDCCondBitmap(E: MCDCBaseExpr, Val: CondV);
2145 }
2146
2147 llvm::MDNode *Weights = nullptr;
2148 llvm::MDNode *Unpredictable = nullptr;
2149
2150 // If the branch has a condition wrapped by __builtin_unpredictable,
2151 // create metadata that specifies that the branch is unpredictable.
2152 // Don't bother if not optimizing because that metadata would not be used.
2153 auto *Call = dyn_cast<CallExpr>(Val: Cond->IgnoreImpCasts());
2154 if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
2155 auto *FD = dyn_cast_or_null<FunctionDecl>(Val: Call->getCalleeDecl());
2156 if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
2157 llvm::MDBuilder MDHelper(getLLVMContext());
2158 Unpredictable = MDHelper.createUnpredictable();
2159 }
2160 }
2161
2162 // If there is a Likelihood knowledge for the cond, lower it.
2163 // Note that if not optimizing this won't emit anything.
2164 llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(Cond: CondV, LH);
2165 if (CondV != NewCondV)
2166 CondV = NewCondV;
2167 else {
2168 // Otherwise, lower profile counts. Note that we do this even at -O0.
2169 uint64_t CurrentCount = std::max(a: getCurrentProfileCount(), b: TrueCount);
2170 Weights = createProfileWeights(TrueCount, FalseCount: CurrentCount - TrueCount);
2171 }
2172
2173 llvm::Instruction *BrInst = Builder.CreateCondBr(Cond: CondV, True: TrueBlock, False: FalseBlock,
2174 BranchWeights: Weights, Unpredictable);
2175 addInstToNewSourceAtom(KeyInstruction: BrInst, Backup: CondV);
2176
2177 switch (HLSLControlFlowAttr) {
2178 case HLSLControlFlowHintAttr::Microsoft_branch:
2179 case HLSLControlFlowHintAttr::Microsoft_flatten: {
2180 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2181
2182 llvm::ConstantInt *BranchHintConstant =
2183 HLSLControlFlowAttr ==
2184 HLSLControlFlowHintAttr::Spelling::Microsoft_branch
2185 ? llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 1)
2186 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 2);
2187
2188 SmallVector<llvm::Metadata *, 2> Vals(
2189 {MDHelper.createString(Str: "hlsl.controlflow.hint"),
2190 MDHelper.createConstant(C: BranchHintConstant)});
2191 BrInst->setMetadata(Kind: "hlsl.controlflow.hint",
2192 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Vals));
2193 break;
2194 }
2195 // This is required to avoid warnings during compilation
2196 case HLSLControlFlowHintAttr::SpellingNotCalculated:
2197 break;
2198 }
2199}
2200
2201llvm::Value *CodeGenFunction::EmitScalarOrConstFoldImmArg(unsigned ICEArguments,
2202 unsigned Idx,
2203 const CallExpr *E) {
2204 llvm::Value *Arg = nullptr;
2205 if ((ICEArguments & (1 << Idx)) == 0) {
2206 Arg = EmitScalarExpr(E: E->getArg(Arg: Idx));
2207 } else {
2208 // If this is required to be a constant, constant fold it so that we
2209 // know that the generated intrinsic gets a ConstantInt.
2210 std::optional<llvm::APSInt> Result =
2211 E->getArg(Arg: Idx)->getIntegerConstantExpr(Ctx: getContext());
2212 assert(Result && "Expected argument to be a constant");
2213 Arg = llvm::ConstantInt::get(Context&: getLLVMContext(), V: *Result);
2214 }
2215 return Arg;
2216}
2217
2218/// ErrorUnsupported - Print out an error that codegen doesn't support the
2219/// specified stmt yet.
2220void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) {
2221 CGM.ErrorUnsupported(S, Type);
2222}
2223
2224/// emitNonZeroVLAInit - Emit the "zero" initialization of a
2225/// variable-length array whose elements have a non-zero bit-pattern.
2226///
2227/// \param baseType the inner-most element type of the array
2228/// \param src - a char* pointing to the bit-pattern for a single
2229/// base element of the array
2230/// \param sizeInChars - the total size of the VLA, in chars
2231static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
2232 Address dest, Address src,
2233 llvm::Value *sizeInChars) {
2234 CGBuilderTy &Builder = CGF.Builder;
2235
2236 CharUnits baseSize = CGF.getContext().getTypeSizeInChars(T: baseType);
2237 llvm::Value *baseSizeInChars
2238 = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: baseSize.getQuantity());
2239
2240 Address begin = dest.withElementType(ElemTy: CGF.Int8Ty);
2241 llvm::Value *end = Builder.CreateInBoundsGEP(Ty: begin.getElementType(),
2242 Ptr: begin.emitRawPointer(CGF),
2243 IdxList: sizeInChars, Name: "vla.end");
2244
2245 llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
2246 llvm::BasicBlock *loopBB = CGF.createBasicBlock(name: "vla-init.loop");
2247 llvm::BasicBlock *contBB = CGF.createBasicBlock(name: "vla-init.cont");
2248
2249 // Make a loop over the VLA. C99 guarantees that the VLA element
2250 // count must be nonzero.
2251 CGF.EmitBlock(BB: loopBB);
2252
2253 llvm::PHINode *cur = Builder.CreatePHI(Ty: begin.getType(), NumReservedValues: 2, Name: "vla.cur");
2254 cur->addIncoming(V: begin.emitRawPointer(CGF), BB: originBB);
2255
2256 CharUnits curAlign =
2257 dest.getAlignment().alignmentOfArrayElement(elementSize: baseSize);
2258
2259 // memcpy the individual element bit-pattern.
2260 Builder.CreateMemCpy(Dest: Address(cur, CGF.Int8Ty, curAlign), Src: src, Size: baseSizeInChars,
2261 /*volatile*/ IsVolatile: false);
2262
2263 // Go to the next element.
2264 llvm::Value *next =
2265 Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: cur, IdxList: baseSizeInChars, Name: "vla.next");
2266
2267 // Leave if that's the end of the VLA.
2268 llvm::Value *done = Builder.CreateICmpEQ(LHS: next, RHS: end, Name: "vla-init.isdone");
2269 Builder.CreateCondBr(Cond: done, True: contBB, False: loopBB);
2270 cur->addIncoming(V: next, BB: loopBB);
2271
2272 CGF.EmitBlock(BB: contBB);
2273}
2274
2275Address CodeGenFunction::EmitAddressOfPFPField(Address RecordPtr,
2276 const PFPField &Field) {
2277 return EmitAddressOfPFPField(
2278 RecordPtr,
2279 FieldPtr: Builder.CreateConstInBoundsByteGEP(Addr: RecordPtr.withElementType(ElemTy: Int8Ty),
2280 Offset: Field.Offset),
2281 Field: Field.Field);
2282}
2283
2284Address CodeGenFunction::EmitAddressOfPFPField(Address RecordPtr,
2285 Address PtrPtr,
2286 const FieldDecl *Field) {
2287 llvm::Value *Disc;
2288 if (CGM.getContext().arePFPFieldsTriviallyCopyable(RD: Field->getParent())) {
2289 uint64_t FieldSignature =
2290 llvm::getPointerAuthStableSipHash(S: CGM.getPFPFieldName(FD: Field));
2291 Disc = llvm::ConstantInt::get(Ty: CGM.Int64Ty, V: FieldSignature);
2292 } else
2293 Disc = Builder.CreatePtrToInt(V: RecordPtr.getBasePointer(), DestTy: CGM.Int64Ty);
2294
2295 llvm::GlobalValue *DS = CGM.getPFPDeactivationSymbol(FD: Field);
2296 llvm::OperandBundleDef DSBundle("deactivation-symbol", DS);
2297 llvm::Value *Args[] = {PtrPtr.getBasePointer(), Disc, Builder.getTrue()};
2298 return Address(
2299 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: llvm::Intrinsic::protected_field_ptr,
2300 Tys: PtrPtr.getType()),
2301 Args, OpBundles: DSBundle),
2302 VoidPtrTy, PtrPtr.getAlignment());
2303}
2304
2305void
2306CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
2307 // Ignore empty classes in C++.
2308 if (getLangOpts().CPlusPlus)
2309 if (const auto *RD = Ty->getAsCXXRecordDecl(); RD && RD->isEmpty())
2310 return;
2311
2312 if (DestPtr.getElementType() != Int8Ty)
2313 DestPtr = DestPtr.withElementType(ElemTy: Int8Ty);
2314
2315 // Get size and alignment info for this aggregate.
2316 CharUnits size = getContext().getTypeSizeInChars(T: Ty);
2317
2318 llvm::Value *SizeVal;
2319 const VariableArrayType *vla;
2320
2321 // Don't bother emitting a zero-byte memset.
2322 if (size.isZero()) {
2323 // But note that getTypeInfo returns 0 for a VLA.
2324 if (const VariableArrayType *vlaType =
2325 dyn_cast_or_null<VariableArrayType>(
2326 Val: getContext().getAsArrayType(T: Ty))) {
2327 auto VlaSize = getVLASize(vla: vlaType);
2328 SizeVal = VlaSize.NumElts;
2329 CharUnits eltSize = getContext().getTypeSizeInChars(T: VlaSize.Type);
2330 if (!eltSize.isOne())
2331 SizeVal = Builder.CreateNUWMul(LHS: SizeVal, RHS: CGM.getSize(numChars: eltSize));
2332 vla = vlaType;
2333 } else {
2334 return;
2335 }
2336 } else {
2337 SizeVal = CGM.getSize(numChars: size);
2338 vla = nullptr;
2339 }
2340
2341 // If the type contains a pointer to data member we can't memset it to zero.
2342 // Instead, create a null constant and copy it to the destination.
2343 // TODO: there are other patterns besides zero that we can usefully memset,
2344 // like -1, which happens to be the pattern used by member-pointers.
2345 if (!CGM.getTypes().isZeroInitializable(T: Ty)) {
2346 // For a VLA, emit a single element, then splat that over the VLA.
2347 if (vla) Ty = getContext().getBaseElementType(VAT: vla);
2348
2349 llvm::Constant *NullConstant = CGM.EmitNullConstant(T: Ty);
2350
2351 llvm::GlobalVariable *NullVariable =
2352 new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
2353 /*isConstant=*/true,
2354 llvm::GlobalVariable::PrivateLinkage,
2355 NullConstant, Twine());
2356 CharUnits NullAlign = DestPtr.getAlignment();
2357 NullVariable->setAlignment(NullAlign.getAsAlign());
2358 Address SrcPtr(NullVariable, Builder.getInt8Ty(), NullAlign);
2359
2360 if (vla) return emitNonZeroVLAInit(CGF&: *this, baseType: Ty, dest: DestPtr, src: SrcPtr, sizeInChars: SizeVal);
2361
2362 // Get and call the appropriate llvm.memcpy overload.
2363 Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: SizeVal, IsVolatile: false);
2364 } else {
2365 // Otherwise, just memset the whole thing to zero. This is legal
2366 // because in LLVM, all default initializers (other than the ones we just
2367 // handled above, and the case handled below) are guaranteed to have a bit
2368 // pattern of all zeros.
2369 Builder.CreateMemSet(Dest: DestPtr, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
2370 }
2371
2372 // With the pointer field protection feature, null pointers do not have a bit
2373 // pattern of zero in memory, so we must initialize them separately.
2374 for (auto &Field : getContext().findPFPFields(Ty)) {
2375 auto addr = EmitAddressOfPFPField(RecordPtr: DestPtr, Field);
2376 Builder.CreateStore(Val: llvm::ConstantPointerNull::get(T: VoidPtrTy), Addr: addr);
2377 }
2378}
2379
2380llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
2381 // Make sure that there is a block for the indirect goto.
2382 if (!IndirectBranch)
2383 GetIndirectGotoBlock();
2384
2385 llvm::BasicBlock *BB = getJumpDestForLabel(S: L).getBlock();
2386
2387 // Make sure the indirect branch includes all of the address-taken blocks.
2388 IndirectBranch->addDestination(Dest: BB);
2389 return llvm::BlockAddress::get(Ty: CurFn->getType(), BB);
2390}
2391
2392llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {
2393 // If we already made the indirect branch for indirect goto, return its block.
2394 if (IndirectBranch) return IndirectBranch->getParent();
2395
2396 CGBuilderTy TmpBuilder(CGM, createBasicBlock(name: "indirectgoto"));
2397
2398 // Create the PHI node that indirect gotos will add entries to.
2399 llvm::Value *DestVal = TmpBuilder.CreatePHI(Ty: Int8PtrTy, NumReservedValues: 0,
2400 Name: "indirect.goto.dest");
2401
2402 // Create the indirect branch instruction.
2403 IndirectBranch = TmpBuilder.CreateIndirectBr(Addr: DestVal);
2404 return IndirectBranch->getParent();
2405}
2406
2407/// Computes the length of an array in elements, as well as the base
2408/// element type and a properly-typed first element pointer.
2409llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
2410 QualType &baseType,
2411 Address &addr) {
2412 const ArrayType *arrayType = origArrayType;
2413
2414 // If it's a VLA, we have to load the stored size. Note that
2415 // this is the size of the VLA in bytes, not its size in elements.
2416 llvm::Value *numVLAElements = nullptr;
2417 if (isa<VariableArrayType>(Val: arrayType)) {
2418 numVLAElements = getVLASize(vla: cast<VariableArrayType>(Val: arrayType)).NumElts;
2419
2420 // Walk into all VLAs. This doesn't require changes to addr,
2421 // which has type T* where T is the first non-VLA element type.
2422 do {
2423 QualType elementType = arrayType->getElementType();
2424 arrayType = getContext().getAsArrayType(T: elementType);
2425
2426 // If we only have VLA components, 'addr' requires no adjustment.
2427 if (!arrayType) {
2428 baseType = elementType;
2429 return numVLAElements;
2430 }
2431 } while (isa<VariableArrayType>(Val: arrayType));
2432
2433 // We get out here only if we find a constant array type
2434 // inside the VLA.
2435 }
2436
2437 // We have some number of constant-length arrays, so addr should
2438 // have LLVM type [M x [N x [...]]]*. Build a GEP that walks
2439 // down to the first element of addr.
2440 SmallVector<llvm::Value*, 8> gepIndices;
2441
2442 // GEP down to the array type.
2443 llvm::ConstantInt *zero = Builder.getInt32(C: 0);
2444 gepIndices.push_back(Elt: zero);
2445
2446 uint64_t countFromCLAs = 1;
2447 QualType eltType;
2448
2449 llvm::ArrayType *llvmArrayType =
2450 dyn_cast<llvm::ArrayType>(Val: addr.getElementType());
2451 while (llvmArrayType) {
2452 assert(isa<ConstantArrayType>(arrayType));
2453 assert(cast<ConstantArrayType>(arrayType)->getZExtSize() ==
2454 llvmArrayType->getNumElements());
2455
2456 gepIndices.push_back(Elt: zero);
2457 countFromCLAs *= llvmArrayType->getNumElements();
2458 eltType = arrayType->getElementType();
2459
2460 llvmArrayType =
2461 dyn_cast<llvm::ArrayType>(Val: llvmArrayType->getElementType());
2462 arrayType = getContext().getAsArrayType(T: arrayType->getElementType());
2463 assert((!llvmArrayType || arrayType) &&
2464 "LLVM and Clang types are out-of-synch");
2465 }
2466
2467 if (arrayType) {
2468 // From this point onwards, the Clang array type has been emitted
2469 // as some other type (probably a packed struct). Compute the array
2470 // size, and just emit the 'begin' expression as a bitcast.
2471 while (arrayType) {
2472 countFromCLAs *= cast<ConstantArrayType>(Val: arrayType)->getZExtSize();
2473 eltType = arrayType->getElementType();
2474 arrayType = getContext().getAsArrayType(T: eltType);
2475 }
2476
2477 llvm::Type *baseType = ConvertType(T: eltType);
2478 addr = addr.withElementType(ElemTy: baseType);
2479 } else {
2480 // Create the actual GEP.
2481 addr = Address(Builder.CreateInBoundsGEP(Ty: addr.getElementType(),
2482 Ptr: addr.emitRawPointer(CGF&: *this),
2483 IdxList: gepIndices, Name: "array.begin"),
2484 ConvertTypeForMem(T: eltType), addr.getAlignment());
2485 }
2486
2487 baseType = eltType;
2488
2489 llvm::Value *numElements
2490 = llvm::ConstantInt::get(Ty: SizeTy, V: countFromCLAs);
2491
2492 // If we had any VLA dimensions, factor them in.
2493 if (numVLAElements)
2494 numElements = Builder.CreateNUWMul(LHS: numVLAElements, RHS: numElements);
2495
2496 return numElements;
2497}
2498
2499CodeGenFunction::VlaSizePair CodeGenFunction::getVLASize(QualType type) {
2500 const VariableArrayType *vla = getContext().getAsVariableArrayType(T: type);
2501 assert(vla && "type was not a variable array type!");
2502 return getVLASize(vla);
2503}
2504
2505CodeGenFunction::VlaSizePair
2506CodeGenFunction::getVLASize(const VariableArrayType *type) {
2507 // The number of elements so far; always size_t.
2508 llvm::Value *numElements = nullptr;
2509
2510 QualType elementType;
2511 do {
2512 elementType = type->getElementType();
2513 llvm::Value *vlaSize = VLASizeMap[type->getSizeExpr()];
2514 assert(vlaSize && "no size for VLA!");
2515 assert(vlaSize->getType() == SizeTy);
2516
2517 if (!numElements) {
2518 numElements = vlaSize;
2519 } else {
2520 // It's undefined behavior if this wraps around, so mark it that way.
2521 // FIXME: Teach -fsanitize=undefined to trap this.
2522 numElements = Builder.CreateNUWMul(LHS: numElements, RHS: vlaSize);
2523 }
2524 } while ((type = getContext().getAsVariableArrayType(T: elementType)));
2525
2526 return { numElements, elementType };
2527}
2528
2529CodeGenFunction::VlaSizePair
2530CodeGenFunction::getVLAElements1D(QualType type) {
2531 const VariableArrayType *vla = getContext().getAsVariableArrayType(T: type);
2532 assert(vla && "type was not a variable array type!");
2533 return getVLAElements1D(vla);
2534}
2535
2536CodeGenFunction::VlaSizePair
2537CodeGenFunction::getVLAElements1D(const VariableArrayType *Vla) {
2538 llvm::Value *VlaSize = VLASizeMap[Vla->getSizeExpr()];
2539 assert(VlaSize && "no size for VLA!");
2540 assert(VlaSize->getType() == SizeTy);
2541 return { VlaSize, Vla->getElementType() };
2542}
2543
2544void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
2545 assert(type->isVariablyModifiedType() &&
2546 "Must pass variably modified type to EmitVLASizes!");
2547
2548 EnsureInsertPoint();
2549
2550 // We're going to walk down into the type and look for VLA
2551 // expressions.
2552 do {
2553 assert(type->isVariablyModifiedType());
2554
2555 const Type *ty = type.getTypePtr();
2556 switch (ty->getTypeClass()) {
2557
2558#define TYPE(Class, Base)
2559#define ABSTRACT_TYPE(Class, Base)
2560#define NON_CANONICAL_TYPE(Class, Base)
2561#define DEPENDENT_TYPE(Class, Base) case Type::Class:
2562#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
2563#include "clang/AST/TypeNodes.inc"
2564 llvm_unreachable("unexpected dependent type!");
2565
2566 // These types are never variably-modified.
2567 case Type::Builtin:
2568 case Type::Complex:
2569 case Type::Vector:
2570 case Type::ExtVector:
2571 case Type::ConstantMatrix:
2572 case Type::Record:
2573 case Type::Enum:
2574 case Type::Using:
2575 case Type::TemplateSpecialization:
2576 case Type::ObjCTypeParam:
2577 case Type::ObjCObject:
2578 case Type::ObjCInterface:
2579 case Type::ObjCObjectPointer:
2580 case Type::BitInt:
2581 case Type::HLSLInlineSpirv:
2582 case Type::PredefinedSugar:
2583 llvm_unreachable("type class is never variably-modified!");
2584
2585 case Type::Adjusted:
2586 type = cast<AdjustedType>(Val: ty)->getAdjustedType();
2587 break;
2588
2589 case Type::Decayed:
2590 type = cast<DecayedType>(Val: ty)->getPointeeType();
2591 break;
2592
2593 case Type::Pointer:
2594 type = cast<PointerType>(Val: ty)->getPointeeType();
2595 break;
2596
2597 case Type::BlockPointer:
2598 type = cast<BlockPointerType>(Val: ty)->getPointeeType();
2599 break;
2600
2601 case Type::LValueReference:
2602 case Type::RValueReference:
2603 type = cast<ReferenceType>(Val: ty)->getPointeeType();
2604 break;
2605
2606 case Type::MemberPointer:
2607 type = cast<MemberPointerType>(Val: ty)->getPointeeType();
2608 break;
2609
2610 case Type::ArrayParameter:
2611 case Type::ConstantArray:
2612 case Type::IncompleteArray:
2613 // Losing element qualification here is fine.
2614 type = cast<ArrayType>(Val: ty)->getElementType();
2615 break;
2616
2617 case Type::VariableArray: {
2618 // Losing element qualification here is fine.
2619 const VariableArrayType *vat = cast<VariableArrayType>(Val: ty);
2620
2621 // Unknown size indication requires no size computation.
2622 // Otherwise, evaluate and record it.
2623 if (const Expr *sizeExpr = vat->getSizeExpr()) {
2624 // It's possible that we might have emitted this already,
2625 // e.g. with a typedef and a pointer to it.
2626 llvm::Value *&entry = VLASizeMap[sizeExpr];
2627 if (!entry) {
2628 llvm::Value *size = EmitScalarExpr(E: sizeExpr);
2629
2630 // C11 6.7.6.2p5:
2631 // If the size is an expression that is not an integer constant
2632 // expression [...] each time it is evaluated it shall have a value
2633 // greater than zero.
2634 if (SanOpts.has(K: SanitizerKind::VLABound)) {
2635 auto CheckOrdinal = SanitizerKind::SO_VLABound;
2636 auto CheckHandler = SanitizerHandler::VLABoundNotPositive;
2637 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2638 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: size->getType());
2639 clang::QualType SEType = sizeExpr->getType();
2640 llvm::Value *CheckCondition =
2641 SEType->isSignedIntegerType()
2642 ? Builder.CreateICmpSGT(LHS: size, RHS: Zero)
2643 : Builder.CreateICmpUGT(LHS: size, RHS: Zero);
2644 llvm::Constant *StaticArgs[] = {
2645 EmitCheckSourceLocation(Loc: sizeExpr->getBeginLoc()),
2646 EmitCheckTypeDescriptor(T: SEType)};
2647 EmitCheck(Checked: std::make_pair(x&: CheckCondition, y&: CheckOrdinal),
2648 Check: CheckHandler, StaticArgs, DynamicArgs: size);
2649 }
2650
2651 // Always zexting here would be wrong if it weren't
2652 // undefined behavior to have a negative bound.
2653 // FIXME: What about when size's type is larger than size_t?
2654 entry = Builder.CreateIntCast(V: size, DestTy: SizeTy, /*signed*/ isSigned: false);
2655 }
2656 }
2657 type = vat->getElementType();
2658 break;
2659 }
2660
2661 case Type::FunctionProto:
2662 case Type::FunctionNoProto:
2663 type = cast<FunctionType>(Val: ty)->getReturnType();
2664 break;
2665
2666 case Type::Paren:
2667 case Type::TypeOf:
2668 case Type::UnaryTransform:
2669 case Type::Attributed:
2670 case Type::BTFTagAttributed:
2671 case Type::OverflowBehavior:
2672 case Type::HLSLAttributedResource:
2673 case Type::SubstTemplateTypeParm:
2674 case Type::MacroQualified:
2675 case Type::CountAttributed:
2676 // Keep walking after single level desugaring.
2677 type = type.getSingleStepDesugaredType(Context: getContext());
2678 break;
2679
2680 case Type::Typedef:
2681 case Type::Decltype:
2682 case Type::Auto:
2683 case Type::DeducedTemplateSpecialization:
2684 case Type::PackIndexing:
2685 // Stop walking: nothing to do.
2686 return;
2687
2688 case Type::TypeOfExpr:
2689 // Stop walking: emit typeof expression.
2690 EmitIgnoredExpr(E: cast<TypeOfExprType>(Val: ty)->getUnderlyingExpr());
2691 return;
2692
2693 case Type::Atomic:
2694 type = cast<AtomicType>(Val: ty)->getValueType();
2695 break;
2696
2697 case Type::Pipe:
2698 type = cast<PipeType>(Val: ty)->getElementType();
2699 break;
2700 }
2701 } while (type->isVariablyModifiedType());
2702}
2703
2704Address CodeGenFunction::EmitVAListRef(const Expr* E) {
2705 if (getContext().getBuiltinVaListType()->isArrayType())
2706 return EmitPointerWithAlignment(Addr: E);
2707 return EmitLValue(E).getAddress();
2708}
2709
2710Address CodeGenFunction::EmitMSVAListRef(const Expr *E) {
2711 return EmitLValue(E).getAddress();
2712}
2713
2714Address CodeGenFunction::EmitZOSVAListRef(const Expr *E) {
2715 return EmitPointerWithAlignment(Addr: E);
2716}
2717
2718void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E,
2719 const APValue &Init) {
2720 assert(Init.hasValue() && "Invalid DeclRefExpr initializer!");
2721 if (CGDebugInfo *Dbg = getDebugInfo())
2722 if (CGM.getCodeGenOpts().hasReducedDebugInfo())
2723 Dbg->EmitGlobalVariable(VD: E->getDecl(), Init);
2724}
2725
2726CodeGenFunction::PeepholeProtection
2727CodeGenFunction::protectFromPeepholes(RValue rvalue) {
2728 // At the moment, the only aggressive peephole we do in IR gen
2729 // is trunc(zext) folding, but if we add more, we can easily
2730 // extend this protection.
2731
2732 if (!rvalue.isScalar()) return PeepholeProtection();
2733 llvm::Value *value = rvalue.getScalarVal();
2734 if (!isa<llvm::ZExtInst>(Val: value)) return PeepholeProtection();
2735
2736 // Just make an extra bitcast.
2737 assert(HaveInsertPoint());
2738 llvm::Instruction *inst = new llvm::BitCastInst(value, value->getType(), "",
2739 Builder.GetInsertBlock());
2740
2741 PeepholeProtection protection;
2742 protection.Inst = inst;
2743 return protection;
2744}
2745
2746void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {
2747 if (!protection.Inst) return;
2748
2749 // In theory, we could try to duplicate the peepholes now, but whatever.
2750 protection.Inst->eraseFromParent();
2751}
2752
2753void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2754 QualType Ty, SourceLocation Loc,
2755 SourceLocation AssumptionLoc,
2756 llvm::Value *Alignment,
2757 llvm::Value *OffsetValue) {
2758 if (Alignment->getType() != IntPtrTy)
2759 Alignment =
2760 Builder.CreateIntCast(V: Alignment, DestTy: IntPtrTy, isSigned: false, Name: "casted.align");
2761 if (OffsetValue && OffsetValue->getType() != IntPtrTy)
2762 OffsetValue =
2763 Builder.CreateIntCast(V: OffsetValue, DestTy: IntPtrTy, isSigned: true, Name: "casted.offset");
2764 llvm::Value *TheCheck = nullptr;
2765 if (SanOpts.has(K: SanitizerKind::Alignment)) {
2766 llvm::Value *PtrIntValue =
2767 Builder.CreatePtrToInt(V: PtrValue, DestTy: IntPtrTy, Name: "ptrint");
2768
2769 if (OffsetValue) {
2770 bool IsOffsetZero = false;
2771 if (const auto *CI = dyn_cast<llvm::ConstantInt>(Val: OffsetValue))
2772 IsOffsetZero = CI->isZero();
2773
2774 if (!IsOffsetZero)
2775 PtrIntValue = Builder.CreateSub(LHS: PtrIntValue, RHS: OffsetValue, Name: "offsetptr");
2776 }
2777
2778 llvm::Value *Zero = llvm::ConstantInt::get(Ty: IntPtrTy, V: 0);
2779 llvm::Value *Mask =
2780 Builder.CreateSub(LHS: Alignment, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: 1));
2781 llvm::Value *MaskedPtr = Builder.CreateAnd(LHS: PtrIntValue, RHS: Mask, Name: "maskedptr");
2782 TheCheck = Builder.CreateICmpEQ(LHS: MaskedPtr, RHS: Zero, Name: "maskcond");
2783 }
2784 llvm::Instruction *Assumption = Builder.CreateAlignmentAssumption(
2785 DL: CGM.getDataLayout(), PtrValue, Alignment, OffsetValue);
2786
2787 if (!SanOpts.has(K: SanitizerKind::Alignment))
2788 return;
2789 emitAlignmentAssumptionCheck(Ptr: PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2790 OffsetValue, TheCheck, Assumption);
2791}
2792
2793void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2794 const Expr *E,
2795 SourceLocation AssumptionLoc,
2796 llvm::Value *Alignment,
2797 llvm::Value *OffsetValue) {
2798 QualType Ty = E->getType();
2799 SourceLocation Loc = E->getExprLoc();
2800
2801 emitAlignmentAssumption(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2802 OffsetValue);
2803}
2804
2805llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn,
2806 llvm::Value *AnnotatedVal,
2807 StringRef AnnotationStr,
2808 SourceLocation Location,
2809 const AnnotateAttr *Attr) {
2810 SmallVector<llvm::Value *, 5> Args = {
2811 AnnotatedVal,
2812 CGM.EmitAnnotationString(Str: AnnotationStr),
2813 CGM.EmitAnnotationUnit(Loc: Location),
2814 CGM.EmitAnnotationLineNo(L: Location),
2815 };
2816 if (Attr)
2817 Args.push_back(Elt: CGM.EmitAnnotationArgs(Attr));
2818 return Builder.CreateCall(Callee: AnnotationFn, Args);
2819}
2820
2821void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
2822 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2823 for (const auto *I : D->specific_attrs<AnnotateAttr>())
2824 EmitAnnotationCall(AnnotationFn: CGM.getIntrinsic(IID: llvm::Intrinsic::var_annotation,
2825 Tys: {V->getType(), CGM.ConstGlobalsPtrTy}),
2826 AnnotatedVal: V, AnnotationStr: I->getAnnotation(), Location: D->getLocation(), Attr: I);
2827}
2828
2829Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
2830 Address Addr) {
2831 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2832 llvm::Value *V = Addr.emitRawPointer(CGF&: *this);
2833 llvm::Type *VTy = V->getType();
2834 auto *PTy = dyn_cast<llvm::PointerType>(Val: VTy);
2835 unsigned AS = PTy ? PTy->getAddressSpace() : 0;
2836 llvm::PointerType *IntrinTy =
2837 llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: AS);
2838 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::ptr_annotation,
2839 Tys: {IntrinTy, CGM.ConstGlobalsPtrTy});
2840
2841 for (const auto *I : D->specific_attrs<AnnotateAttr>()) {
2842 // FIXME Always emit the cast inst so we can differentiate between
2843 // annotation on the first field of a struct and annotation on the struct
2844 // itself.
2845 if (VTy != IntrinTy)
2846 V = Builder.CreateBitCast(V, DestTy: IntrinTy);
2847 V = EmitAnnotationCall(AnnotationFn: F, AnnotatedVal: V, AnnotationStr: I->getAnnotation(), Location: D->getLocation(), Attr: I);
2848 V = Builder.CreateBitCast(V, DestTy: VTy);
2849 }
2850
2851 return Address(V, Addr.getElementType(), Addr.getAlignment());
2852}
2853
2854CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
2855
2856CodeGenFunction::SanitizerScope::SanitizerScope(CodeGenFunction *CGF)
2857 : CGF(CGF) {
2858 assert(!CGF->IsSanitizerScope);
2859 CGF->IsSanitizerScope = true;
2860}
2861
2862CodeGenFunction::SanitizerScope::~SanitizerScope() {
2863 CGF->IsSanitizerScope = false;
2864}
2865
2866void CodeGenFunction::InsertHelper(llvm::Instruction *I,
2867 const llvm::Twine &Name,
2868 llvm::BasicBlock::iterator InsertPt) const {
2869 LoopStack.InsertHelper(I);
2870 if (IsSanitizerScope)
2871 I->setNoSanitizeMetadata();
2872}
2873
2874void CGBuilderInserter::InsertHelper(
2875 llvm::Instruction *I, const llvm::Twine &Name,
2876 llvm::BasicBlock::iterator InsertPt) const {
2877 llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, InsertPt);
2878 if (CGF)
2879 CGF->InsertHelper(I, Name, InsertPt);
2880}
2881
2882// Emits an error if we don't have a valid set of target features for the
2883// called function.
2884void CodeGenFunction::checkTargetFeatures(const CallExpr *E,
2885 const FunctionDecl *TargetDecl) {
2886 // SemaChecking cannot handle below x86 builtins because they have different
2887 // parameter ranges with different TargetAttribute of caller.
2888 if (CGM.getContext().getTargetInfo().getTriple().isX86()) {
2889 unsigned BuiltinID = TargetDecl->getBuiltinID();
2890 if (BuiltinID == X86::BI__builtin_ia32_cmpps ||
2891 BuiltinID == X86::BI__builtin_ia32_cmpss ||
2892 BuiltinID == X86::BI__builtin_ia32_cmppd ||
2893 BuiltinID == X86::BI__builtin_ia32_cmpsd) {
2894 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
2895 llvm::StringMap<bool> TargetFetureMap;
2896 CGM.getContext().getFunctionFeatureMap(FeatureMap&: TargetFetureMap, FD);
2897 llvm::APSInt Result =
2898 *(E->getArg(Arg: 2)->getIntegerConstantExpr(Ctx: CGM.getContext()));
2899 if (Result.getSExtValue() > 7 && !TargetFetureMap.lookup(Key: "avx"))
2900 CGM.getDiags().Report(Loc: E->getBeginLoc(), DiagID: diag::err_builtin_needs_feature)
2901 << TargetDecl->getDeclName() << "avx";
2902 }
2903 }
2904 return checkTargetFeatures(Loc: E->getBeginLoc(), TargetDecl);
2905}
2906
2907// Emits an error if we don't have a valid set of target features for the
2908// called function.
2909void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
2910 const FunctionDecl *TargetDecl) {
2911 // Early exit if this is an indirect call.
2912 if (!TargetDecl)
2913 return;
2914
2915 // Get the current enclosing function if it exists. If it doesn't
2916 // we can't check the target features anyhow.
2917 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
2918 if (!FD)
2919 return;
2920
2921 bool IsAlwaysInline = TargetDecl->hasAttr<AlwaysInlineAttr>();
2922 bool IsFlatten = FD && FD->hasAttr<FlattenAttr>();
2923
2924 // Grab the required features for the call. For a builtin this is listed in
2925 // the td file with the default cpu, for an always_inline function this is any
2926 // listed cpu and any listed features.
2927 unsigned BuiltinID = TargetDecl->getBuiltinID();
2928 std::string MissingFeature;
2929 llvm::StringMap<bool> CallerFeatureMap;
2930 CGM.getContext().getFunctionFeatureMap(FeatureMap&: CallerFeatureMap, FD);
2931 // When compiling in HipStdPar mode we have to be conservative in rejecting
2932 // target specific features in the FE, and defer the possible error to the
2933 // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
2934 // referenced by an accelerator executable function, we emit an error.
2935 bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;
2936 if (BuiltinID) {
2937 StringRef FeatureList(CGM.getContext().BuiltinInfo.getRequiredFeatures(ID: BuiltinID));
2938 if (!Builtin::evaluateRequiredTargetFeatures(
2939 RequiredFatures: FeatureList, TargetFetureMap: CallerFeatureMap) && !IsHipStdPar) {
2940 CGM.getDiags().Report(Loc, DiagID: diag::err_builtin_needs_feature)
2941 << TargetDecl->getDeclName()
2942 << FeatureList;
2943 }
2944 } else if (!TargetDecl->isMultiVersion() &&
2945 TargetDecl->hasAttr<TargetAttr>()) {
2946 // Get the required features for the callee.
2947
2948 const TargetAttr *TD = TargetDecl->getAttr<TargetAttr>();
2949 ParsedTargetAttr ParsedAttr =
2950 CGM.getContext().filterFunctionTargetAttrs(TD);
2951
2952 SmallVector<StringRef, 1> ReqFeatures;
2953 llvm::StringMap<bool> CalleeFeatureMap;
2954 CGM.getContext().getFunctionFeatureMap(FeatureMap&: CalleeFeatureMap, TargetDecl);
2955
2956 for (const auto &F : ParsedAttr.Features) {
2957 if (F[0] == '+' && CalleeFeatureMap.lookup(Key: F.substr(pos: 1)))
2958 ReqFeatures.push_back(Elt: StringRef(F).substr(Start: 1));
2959 }
2960
2961 for (const auto &F : CalleeFeatureMap) {
2962 // Only positive features are "required".
2963 if (F.getValue())
2964 ReqFeatures.push_back(Elt: F.getKey());
2965 }
2966 if (!llvm::all_of(Range&: ReqFeatures,
2967 P: [&](StringRef Feature) {
2968 if (!CallerFeatureMap.lookup(Key: Feature)) {
2969 MissingFeature = Feature.str();
2970 return false;
2971 }
2972 return true;
2973 }) &&
2974 !IsHipStdPar) {
2975 if (IsAlwaysInline)
2976 CGM.getDiags().Report(Loc, DiagID: diag::err_function_needs_feature)
2977 << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
2978 else if (IsFlatten)
2979 CGM.getDiags().Report(Loc, DiagID: diag::err_flatten_function_needs_feature)
2980 << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
2981 }
2982
2983 } else if (!FD->isMultiVersion() && FD->hasAttr<TargetAttr>()) {
2984 llvm::StringMap<bool> CalleeFeatureMap;
2985 CGM.getContext().getFunctionFeatureMap(FeatureMap&: CalleeFeatureMap, TargetDecl);
2986
2987 for (const auto &F : CalleeFeatureMap) {
2988 if (F.getValue() &&
2989 (!CallerFeatureMap.lookup(Key: F.getKey()) ||
2990 !CallerFeatureMap.find(Key: F.getKey())->getValue()) &&
2991 !IsHipStdPar) {
2992 if (IsAlwaysInline)
2993 CGM.getDiags().Report(Loc, DiagID: diag::err_function_needs_feature)
2994 << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();
2995 else if (IsFlatten)
2996 CGM.getDiags().Report(Loc, DiagID: diag::err_flatten_function_needs_feature)
2997 << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();
2998 }
2999 }
3000 }
3001}
3002
3003void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
3004 if (!CGM.getCodeGenOpts().SanitizeStats)
3005 return;
3006
3007 llvm::IRBuilder<> IRB(Builder.GetInsertBlock(), Builder.GetInsertPoint());
3008 IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation());
3009 CGM.getSanStats().create(B&: IRB, SK: SSK);
3010}
3011
3012void CodeGenFunction::EmitKCFIOperandBundle(
3013 const CGCallee &Callee, SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
3014 const CGCalleeInfo &CI = Callee.getAbstractInfo();
3015 const FunctionProtoType *FP = CI.getCalleeFunctionProtoType();
3016 if (!FP)
3017 return;
3018
3019 StringRef Salt;
3020 if (const auto &Info = FP->getExtraAttributeInfo())
3021 Salt = Info.CFISalt;
3022
3023 Bundles.emplace_back(Args: "kcfi", Args: CGM.CreateKCFITypeId(T: FP->desugar(), Salt));
3024}
3025
3026llvm::Value *
3027CodeGenFunction::FormAArch64ResolverCondition(const FMVResolverOption &RO) {
3028 return RO.Features.empty() ? nullptr : EmitAArch64CpuSupports(FeatureStrs: RO.Features);
3029}
3030
3031llvm::Value *
3032CodeGenFunction::FormX86ResolverCondition(const FMVResolverOption &RO) {
3033 llvm::Value *Condition = nullptr;
3034
3035 if (RO.Architecture) {
3036 StringRef Arch = *RO.Architecture;
3037 // If arch= specifies an x86-64 micro-architecture level, test the feature
3038 // with __builtin_cpu_supports, otherwise use __builtin_cpu_is.
3039 if (Arch.starts_with(Prefix: "x86-64"))
3040 Condition = EmitX86CpuSupports(FeatureStrs: {Arch});
3041 else
3042 Condition = EmitX86CpuIs(CPUStr: Arch);
3043 }
3044
3045 if (!RO.Features.empty()) {
3046 llvm::Value *FeatureCond = EmitX86CpuSupports(FeatureStrs: RO.Features);
3047 Condition =
3048 Condition ? Builder.CreateAnd(LHS: Condition, RHS: FeatureCond) : FeatureCond;
3049 }
3050 return Condition;
3051}
3052
3053static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
3054 llvm::Function *Resolver,
3055 CGBuilderTy &Builder,
3056 llvm::Function *FuncToReturn,
3057 bool SupportsIFunc) {
3058 if (SupportsIFunc) {
3059 Builder.CreateRet(V: FuncToReturn);
3060 return;
3061 }
3062
3063 llvm::SmallVector<llvm::Value *, 10> Args(
3064 llvm::make_pointer_range(Range: Resolver->args()));
3065
3066 llvm::CallInst *Result = Builder.CreateCall(Callee: FuncToReturn, Args);
3067 Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
3068
3069 if (Resolver->getReturnType()->isVoidTy())
3070 Builder.CreateRetVoid();
3071 else
3072 Builder.CreateRet(V: Result);
3073}
3074
3075void CodeGenFunction::EmitMultiVersionResolver(
3076 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3077 llvm::SaveAndRestore<llvm::Function *> savedCurFn(CurFn, Resolver);
3078 llvm::Triple::ArchType ArchType =
3079 getContext().getTargetInfo().getTriple().getArch();
3080
3081 switch (ArchType) {
3082 case llvm::Triple::x86:
3083 case llvm::Triple::x86_64:
3084 EmitX86MultiVersionResolver(Resolver, Options);
3085 return;
3086 case llvm::Triple::aarch64:
3087 EmitAArch64MultiVersionResolver(Resolver, Options);
3088 return;
3089 case llvm::Triple::riscv32:
3090 case llvm::Triple::riscv64:
3091 case llvm::Triple::riscv32be:
3092 case llvm::Triple::riscv64be:
3093 EmitRISCVMultiVersionResolver(Resolver, Options);
3094 return;
3095 case llvm::Triple::ppc:
3096 case llvm::Triple::ppc64:
3097 if (getContext().getTargetInfo().getTriple().isOSAIX()) {
3098 EmitPPCAIXMultiVersionResolver(Resolver, Options);
3099 return;
3100 }
3101 [[fallthrough]];
3102 default:
3103 assert(false &&
3104 "Only implemented for x86, AArch64, RISC-V, and PowerPC AIX");
3105 }
3106}
3107
3108/**
3109 * define internal ptr @foo.resolver() {
3110 * entry:
3111 * %is_version_1 = __builtin_cpu_supports(version_1)
3112 * br i1 %1, label %if.version_1, label %if.else_2
3113 *
3114 * if.version_1:
3115 * ret ptr @foo.version_1
3116 *
3117 * if.else_2:
3118 * %is_version_2 = __builtin_cpu_supports(version_2)
3119 * ...
3120 * if.else: ; preds = %entry
3121 * ret ptr @foo.default
3122 * }
3123 */
3124void CodeGenFunction::EmitPPCAIXMultiVersionResolver(
3125 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3126
3127 // entry:
3128 llvm::BasicBlock *CurBlock = createBasicBlock(name: "entry", parent: Resolver);
3129
3130 SmallVector<std::pair<llvm::Value *, llvm::BasicBlock *>, 3> PhiArgs;
3131 for (const FMVResolverOption &RO : Options) {
3132 Builder.SetInsertPoint(CurBlock);
3133 // The 'default' or 'generic' case.
3134 if (!RO.Architecture && RO.Features.empty()) {
3135 // if.else:
3136 // ret ptr @foo.default
3137 assert(&RO == Options.end() - 1 &&
3138 "Default or Generic case must be last");
3139 Builder.CreateRet(V: RO.Function);
3140 return;
3141 }
3142 // if.else_n:
3143 // %is_version_n = __builtin_cpu_supports(version_n)
3144 // br i1 %is_version_n, label %if.version_n, label %if.else_n+1
3145 //
3146 // if.version_n:
3147 // ret ptr @foo_version_n
3148 assert(RO.Features.size() == 1 &&
3149 "for now one feature requirement per version");
3150
3151 assert(RO.Features[0].starts_with("cpu="));
3152 StringRef CPU = RO.Features[0].split(Separator: "=").second.trim();
3153 StringRef Feature = llvm::StringSwitch<StringRef>(CPU)
3154 .Case(S: "pwr7", Value: "arch_2_06")
3155 .Case(S: "pwr8", Value: "arch_2_07")
3156 .Case(S: "pwr9", Value: "arch_3_00")
3157 .Case(S: "pwr10", Value: "arch_3_1")
3158 .Case(S: "pwr11", Value: "arch_3_1")
3159 .Default(Value: "error");
3160
3161 llvm::Value *Condition = EmitPPCBuiltinCpu(
3162 BuiltinID: Builtin::BI__builtin_cpu_supports, ReturnType: Builder.getInt1Ty(), CPUStr: Feature);
3163
3164 llvm::BasicBlock *ThenBlock = createBasicBlock(name: "if.version", parent: Resolver);
3165 CurBlock = createBasicBlock(name: "if.else", parent: Resolver);
3166 Builder.CreateCondBr(Cond: Condition, True: ThenBlock, False: CurBlock);
3167
3168 Builder.SetInsertPoint(ThenBlock);
3169 Builder.CreateRet(V: RO.Function);
3170 }
3171
3172 llvm_unreachable("Default case missing");
3173}
3174
3175void CodeGenFunction::EmitRISCVMultiVersionResolver(
3176 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3177
3178 if (getContext().getTargetInfo().getTriple().getOS() !=
3179 llvm::Triple::OSType::Linux) {
3180 CGM.getDiags().Report(DiagID: diag::err_os_unsupport_riscv_fmv);
3181 return;
3182 }
3183
3184 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3185 Builder.SetInsertPoint(CurBlock);
3186 EmitRISCVCpuInit();
3187
3188 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3189 bool HasDefault = false;
3190 unsigned DefaultIndex = 0;
3191
3192 // Check the each candidate function.
3193 for (unsigned Index = 0; Index < Options.size(); Index++) {
3194
3195 if (Options[Index].Features.empty()) {
3196 HasDefault = true;
3197 DefaultIndex = Index;
3198 continue;
3199 }
3200
3201 Builder.SetInsertPoint(CurBlock);
3202
3203 // FeaturesCondition: The bitmask of the required extension has been
3204 // enabled by the runtime object.
3205 // (__riscv_feature_bits.features[i] & REQUIRED_BITMASK) ==
3206 // REQUIRED_BITMASK
3207 //
3208 // When condition is met, return this version of the function.
3209 // Otherwise, try the next version.
3210 //
3211 // if (FeaturesConditionVersion1)
3212 // return Version1;
3213 // else if (FeaturesConditionVersion2)
3214 // return Version2;
3215 // else if (FeaturesConditionVersion3)
3216 // return Version3;
3217 // ...
3218 // else
3219 // return DefaultVersion;
3220
3221 // TODO: Add a condition to check the length before accessing elements.
3222 // Without checking the length first, we may access an incorrect memory
3223 // address when using different versions.
3224 llvm::SmallVector<StringRef, 8> CurrTargetAttrFeats;
3225 llvm::SmallVector<std::string, 8> TargetAttrFeats;
3226
3227 for (StringRef Feat : Options[Index].Features) {
3228 std::vector<std::string> FeatStr =
3229 getContext().getTargetInfo().parseTargetAttr(Str: Feat).Features;
3230
3231 assert(FeatStr.size() == 1 && "Feature string not delimited");
3232
3233 std::string &CurrFeat = FeatStr.front();
3234 if (CurrFeat[0] == '+')
3235 TargetAttrFeats.push_back(Elt: CurrFeat.substr(pos: 1));
3236 }
3237
3238 if (TargetAttrFeats.empty())
3239 continue;
3240
3241 for (std::string &Feat : TargetAttrFeats)
3242 CurrTargetAttrFeats.push_back(Elt: Feat);
3243
3244 Builder.SetInsertPoint(CurBlock);
3245 llvm::Value *FeatsCondition = EmitRISCVCpuSupports(FeaturesStrs: CurrTargetAttrFeats);
3246
3247 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3248 CGBuilderTy RetBuilder(CGM, RetBlock);
3249 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder,
3250 FuncToReturn: Options[Index].Function, SupportsIFunc);
3251 llvm::BasicBlock *ElseBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3252
3253 Builder.SetInsertPoint(CurBlock);
3254 Builder.CreateCondBr(Cond: FeatsCondition, True: RetBlock, False: ElseBlock);
3255
3256 CurBlock = ElseBlock;
3257 }
3258
3259 // Finally, emit the default one.
3260 if (HasDefault) {
3261 Builder.SetInsertPoint(CurBlock);
3262 CreateMultiVersionResolverReturn(
3263 CGM, Resolver, Builder, FuncToReturn: Options[DefaultIndex].Function, SupportsIFunc);
3264 return;
3265 }
3266
3267 // If no generic/default, emit an unreachable.
3268 Builder.SetInsertPoint(CurBlock);
3269 llvm::CallInst *TrapCall = EmitTrapCall(IntrID: llvm::Intrinsic::trap);
3270 TrapCall->setDoesNotReturn();
3271 TrapCall->setDoesNotThrow();
3272 Builder.CreateUnreachable();
3273 Builder.ClearInsertionPoint();
3274}
3275
3276void CodeGenFunction::EmitAArch64MultiVersionResolver(
3277 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3278 assert(!Options.empty() && "No multiversion resolver options found");
3279 assert(Options.back().Features.size() == 0 && "Default case must be last");
3280 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3281 assert(SupportsIFunc &&
3282 "Multiversion resolver requires target IFUNC support");
3283 bool AArch64CpuInitialized = false;
3284 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3285
3286 for (const FMVResolverOption &RO : Options) {
3287 Builder.SetInsertPoint(CurBlock);
3288 llvm::Value *Condition = FormAArch64ResolverCondition(RO);
3289
3290 // The 'default' or 'all features enabled' case.
3291 if (!Condition) {
3292 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, FuncToReturn: RO.Function,
3293 SupportsIFunc);
3294 return;
3295 }
3296
3297 if (!AArch64CpuInitialized) {
3298 Builder.SetInsertPoint(TheBB: CurBlock, IP: CurBlock->begin());
3299 EmitAArch64CpuInit();
3300 AArch64CpuInitialized = true;
3301 Builder.SetInsertPoint(CurBlock);
3302 }
3303
3304 // Skip unreachable versions.
3305 if (RO.Function == nullptr)
3306 continue;
3307
3308 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3309 CGBuilderTy RetBuilder(CGM, RetBlock);
3310 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder, FuncToReturn: RO.Function,
3311 SupportsIFunc);
3312 CurBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3313 Builder.CreateCondBr(Cond: Condition, True: RetBlock, False: CurBlock);
3314 }
3315
3316 // If no default, emit an unreachable.
3317 Builder.SetInsertPoint(CurBlock);
3318 llvm::CallInst *TrapCall = EmitTrapCall(IntrID: llvm::Intrinsic::trap);
3319 TrapCall->setDoesNotReturn();
3320 TrapCall->setDoesNotThrow();
3321 Builder.CreateUnreachable();
3322 Builder.ClearInsertionPoint();
3323}
3324
3325void CodeGenFunction::EmitX86MultiVersionResolver(
3326 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3327
3328 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3329
3330 // Main function's basic block.
3331 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3332 Builder.SetInsertPoint(CurBlock);
3333 EmitX86CpuInit();
3334
3335 for (const FMVResolverOption &RO : Options) {
3336 Builder.SetInsertPoint(CurBlock);
3337 llvm::Value *Condition = FormX86ResolverCondition(RO);
3338
3339 // The 'default' or 'generic' case.
3340 if (!Condition) {
3341 assert(&RO == Options.end() - 1 &&
3342 "Default or Generic case must be last");
3343 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, FuncToReturn: RO.Function,
3344 SupportsIFunc);
3345 return;
3346 }
3347
3348 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3349 CGBuilderTy RetBuilder(CGM, RetBlock);
3350 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder, FuncToReturn: RO.Function,
3351 SupportsIFunc);
3352 CurBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3353 Builder.CreateCondBr(Cond: Condition, True: RetBlock, False: CurBlock);
3354 }
3355
3356 // If no generic/default, emit an unreachable.
3357 Builder.SetInsertPoint(CurBlock);
3358 llvm::CallInst *TrapCall = EmitTrapCall(IntrID: llvm::Intrinsic::trap);
3359 TrapCall->setDoesNotReturn();
3360 TrapCall->setDoesNotThrow();
3361 Builder.CreateUnreachable();
3362 Builder.ClearInsertionPoint();
3363}
3364
3365// Loc - where the diagnostic will point, where in the source code this
3366// alignment has failed.
3367// SecondaryLoc - if present (will be present if sufficiently different from
3368// Loc), the diagnostic will additionally point a "Note:" to this location.
3369// It should be the location where the __attribute__((assume_aligned))
3370// was written e.g.
3371void CodeGenFunction::emitAlignmentAssumptionCheck(
3372 llvm::Value *Ptr, QualType Ty, SourceLocation Loc,
3373 SourceLocation SecondaryLoc, llvm::Value *Alignment,
3374 llvm::Value *OffsetValue, llvm::Value *TheCheck,
3375 llvm::Instruction *Assumption) {
3376 assert(isa_and_nonnull<llvm::CallInst>(Assumption) &&
3377 cast<llvm::CallInst>(Assumption)->getCalledOperand() ==
3378 llvm::Intrinsic::getOrInsertDeclaration(
3379 Builder.GetInsertBlock()->getParent()->getParent(),
3380 llvm::Intrinsic::assume) &&
3381 "Assumption should be a call to llvm.assume().");
3382 assert(&(Builder.GetInsertBlock()->back()) == Assumption &&
3383 "Assumption should be the last instruction of the basic block, "
3384 "since the basic block is still being generated.");
3385
3386 if (!SanOpts.has(K: SanitizerKind::Alignment))
3387 return;
3388
3389 // Don't check pointers to volatile data. The behavior here is implementation-
3390 // defined.
3391 if (Ty->getPointeeType().isVolatileQualified())
3392 return;
3393
3394 // We need to temorairly remove the assumption so we can insert the
3395 // sanitizer check before it, else the check will be dropped by optimizations.
3396 Assumption->removeFromParent();
3397
3398 {
3399 auto CheckOrdinal = SanitizerKind::SO_Alignment;
3400 auto CheckHandler = SanitizerHandler::AlignmentAssumption;
3401 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
3402
3403 if (!OffsetValue)
3404 OffsetValue = Builder.getInt1(V: false); // no offset.
3405
3406 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc),
3407 EmitCheckSourceLocation(Loc: SecondaryLoc),
3408 EmitCheckTypeDescriptor(T: Ty)};
3409 llvm::Value *DynamicData[] = {Ptr, Alignment, OffsetValue};
3410 EmitCheck(Checked: {std::make_pair(x&: TheCheck, y&: CheckOrdinal)}, Check: CheckHandler,
3411 StaticArgs: StaticData, DynamicArgs: DynamicData);
3412 }
3413
3414 // We are now in the (new, empty) "cont" basic block.
3415 // Reintroduce the assumption.
3416 Builder.Insert(I: Assumption);
3417 // FIXME: Assumption still has it's original basic block as it's Parent.
3418}
3419
3420llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
3421 if (CGDebugInfo *DI = getDebugInfo())
3422 return DI->SourceLocToDebugLoc(Loc: Location);
3423
3424 return llvm::DebugLoc();
3425}
3426
3427llvm::Value *
3428CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
3429 Stmt::Likelihood LH) {
3430 switch (LH) {
3431 case Stmt::LH_None:
3432 return Cond;
3433 case Stmt::LH_Likely:
3434 case Stmt::LH_Unlikely:
3435 // Don't generate llvm.expect on -O0 as the backend won't use it for
3436 // anything.
3437 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3438 return Cond;
3439 llvm::Type *CondTy = Cond->getType();
3440 assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");
3441 llvm::Function *FnExpect =
3442 CGM.getIntrinsic(IID: llvm::Intrinsic::expect, Tys: CondTy);
3443 llvm::Value *ExpectedValueOfCond =
3444 llvm::ConstantInt::getBool(Ty: CondTy, V: LH == Stmt::LH_Likely);
3445 return Builder.CreateCall(Callee: FnExpect, Args: {Cond, ExpectedValueOfCond},
3446 Name: Cond->getName() + ".expval");
3447 }
3448 llvm_unreachable("Unknown Likelihood");
3449}
3450
3451llvm::Value *CodeGenFunction::emitBoolVecConversion(llvm::Value *SrcVec,
3452 unsigned NumElementsDst,
3453 const llvm::Twine &Name) {
3454 auto *SrcTy = cast<llvm::FixedVectorType>(Val: SrcVec->getType());
3455 unsigned NumElementsSrc = SrcTy->getNumElements();
3456 if (NumElementsSrc == NumElementsDst)
3457 return SrcVec;
3458
3459 std::vector<int> ShuffleMask(NumElementsDst, -1);
3460 for (unsigned MaskIdx = 0;
3461 MaskIdx < std::min<>(a: NumElementsDst, b: NumElementsSrc); ++MaskIdx)
3462 ShuffleMask[MaskIdx] = MaskIdx;
3463
3464 return Builder.CreateShuffleVector(V: SrcVec, Mask: ShuffleMask, Name);
3465}
3466
3467void CodeGenFunction::EmitPointerAuthOperandBundle(
3468 const CGPointerAuthInfo &PointerAuth,
3469 SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
3470 if (!PointerAuth.isSigned())
3471 return;
3472
3473 auto *Key = Builder.getInt32(C: PointerAuth.getKey());
3474
3475 llvm::Value *Discriminator = PointerAuth.getDiscriminator();
3476 if (!Discriminator)
3477 Discriminator = Builder.getSize(N: 0);
3478
3479 llvm::Value *Args[] = {Key, Discriminator};
3480 Bundles.emplace_back(Args: "ptrauth", Args);
3481}
3482
3483static llvm::Value *EmitPointerAuthCommon(CodeGenFunction &CGF,
3484 const CGPointerAuthInfo &PointerAuth,
3485 llvm::Value *Pointer,
3486 unsigned IntrinsicID) {
3487 if (!PointerAuth)
3488 return Pointer;
3489
3490 auto Key = CGF.Builder.getInt32(C: PointerAuth.getKey());
3491
3492 llvm::Value *Discriminator = PointerAuth.getDiscriminator();
3493 if (!Discriminator) {
3494 Discriminator = CGF.Builder.getSize(N: 0);
3495 }
3496
3497 // Convert the pointer to intptr_t before signing it.
3498 auto OrigType = Pointer->getType();
3499 Pointer = CGF.Builder.CreatePtrToInt(V: Pointer, DestTy: CGF.IntPtrTy);
3500
3501 // call i64 @llvm.ptrauth.sign.i64(i64 %pointer, i32 %key, i64 %discriminator)
3502 auto Intrinsic = CGF.CGM.getIntrinsic(IID: IntrinsicID);
3503 Pointer = CGF.EmitRuntimeCall(callee: Intrinsic, args: {Pointer, Key, Discriminator});
3504
3505 // Convert back to the original type.
3506 Pointer = CGF.Builder.CreateIntToPtr(V: Pointer, DestTy: OrigType);
3507 return Pointer;
3508}
3509
3510llvm::Value *
3511CodeGenFunction::EmitPointerAuthSign(const CGPointerAuthInfo &PointerAuth,
3512 llvm::Value *Pointer) {
3513 if (!PointerAuth.shouldSign())
3514 return Pointer;
3515 return EmitPointerAuthCommon(CGF&: *this, PointerAuth, Pointer,
3516 IntrinsicID: llvm::Intrinsic::ptrauth_sign);
3517}
3518
3519static llvm::Value *EmitStrip(CodeGenFunction &CGF,
3520 const CGPointerAuthInfo &PointerAuth,
3521 llvm::Value *Pointer) {
3522 auto StripIntrinsic = CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::ptrauth_strip);
3523
3524 auto Key = CGF.Builder.getInt32(C: PointerAuth.getKey());
3525 // Convert the pointer to intptr_t before signing it.
3526 auto OrigType = Pointer->getType();
3527 Pointer = CGF.EmitRuntimeCall(
3528 callee: StripIntrinsic, args: {CGF.Builder.CreatePtrToInt(V: Pointer, DestTy: CGF.IntPtrTy), Key});
3529 return CGF.Builder.CreateIntToPtr(V: Pointer, DestTy: OrigType);
3530}
3531
3532llvm::Value *
3533CodeGenFunction::EmitPointerAuthAuth(const CGPointerAuthInfo &PointerAuth,
3534 llvm::Value *Pointer) {
3535 if (PointerAuth.shouldStrip()) {
3536 return EmitStrip(CGF&: *this, PointerAuth, Pointer);
3537 }
3538 if (!PointerAuth.shouldAuth()) {
3539 return Pointer;
3540 }
3541
3542 return EmitPointerAuthCommon(CGF&: *this, PointerAuth, Pointer,
3543 IntrinsicID: llvm::Intrinsic::ptrauth_auth);
3544}
3545
3546void CodeGenFunction::addInstToCurrentSourceAtom(
3547 llvm::Instruction *KeyInstruction, llvm::Value *Backup) {
3548 if (CGDebugInfo *DI = getDebugInfo())
3549 DI->addInstToCurrentSourceAtom(KeyInstruction, Backup);
3550}
3551
3552void CodeGenFunction::addInstToSpecificSourceAtom(
3553 llvm::Instruction *KeyInstruction, llvm::Value *Backup, uint64_t Atom) {
3554 if (CGDebugInfo *DI = getDebugInfo())
3555 DI->addInstToSpecificSourceAtom(KeyInstruction, Backup, Atom);
3556}
3557
3558void CodeGenFunction::addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,
3559 llvm::Value *Backup) {
3560 if (CGDebugInfo *DI = getDebugInfo()) {
3561 ApplyAtomGroup Grp(getDebugInfo());
3562 DI->addInstToCurrentSourceAtom(KeyInstruction, Backup);
3563 }
3564}
3565
3566void CodeGenFunction::emitPFPPostCopyUpdates(Address DestPtr, Address SrcPtr,
3567 QualType Ty) {
3568 for (auto &Field : getContext().findPFPFields(Ty)) {
3569 if (getContext().arePFPFieldsTriviallyCopyable(RD: Field.Field->getParent()))
3570 continue;
3571 auto DestFieldPtr = EmitAddressOfPFPField(RecordPtr: DestPtr, Field);
3572 auto SrcFieldPtr = EmitAddressOfPFPField(RecordPtr: SrcPtr, Field);
3573 Builder.CreateStore(Val: Builder.CreateLoad(Addr: SrcFieldPtr), Addr: DestFieldPtr);
3574 }
3575}
3576