1//===--- CodeGenFunction.cpp - Emit LLVM Code from ASTs for a Function ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This coordinates the per-function state used while generating code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CodeGenFunction.h"
14#include "CGBlocks.h"
15#include "CGCUDARuntime.h"
16#include "CGCXXABI.h"
17#include "CGCleanup.h"
18#include "CGDebugInfo.h"
19#include "CGHLSLRuntime.h"
20#include "CGOpenMPRuntime.h"
21#include "CodeGenModule.h"
22#include "CodeGenPGO.h"
23#include "TargetInfo.h"
24#include "clang/AST/ASTContext.h"
25#include "clang/AST/ASTLambda.h"
26#include "clang/AST/Attr.h"
27#include "clang/AST/Decl.h"
28#include "clang/AST/DeclCXX.h"
29#include "clang/AST/Expr.h"
30#include "clang/AST/IgnoreExpr.h"
31#include "clang/AST/StmtCXX.h"
32#include "clang/AST/StmtObjC.h"
33#include "clang/Basic/Builtins.h"
34#include "clang/Basic/CodeGenOptions.h"
35#include "clang/Basic/DiagnosticFrontend.h"
36#include "clang/Basic/TargetBuiltins.h"
37#include "clang/Basic/TargetInfo.h"
38#include "clang/CodeGen/CGFunctionInfo.h"
39#include "llvm/ADT/ArrayRef.h"
40#include "llvm/ADT/ScopeExit.h"
41#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
42#include "llvm/IR/DataLayout.h"
43#include "llvm/IR/Dominators.h"
44#include "llvm/IR/FPEnv.h"
45#include "llvm/IR/Instruction.h"
46#include "llvm/IR/IntrinsicInst.h"
47#include "llvm/IR/Intrinsics.h"
48#include "llvm/IR/MDBuilder.h"
49#include "llvm/Support/CRC.h"
50#include "llvm/Support/xxhash.h"
51#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
52#include "llvm/Transforms/Utils/PromoteMemToReg.h"
53#include <optional>
54
55using namespace clang;
56using namespace CodeGen;
57
58/// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time
59/// markers.
60static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts,
61 const LangOptions &LangOpts) {
62 if (CGOpts.DisableLifetimeMarkers)
63 return false;
64
65 // Sanitizers may use markers.
66 if (CGOpts.SanitizeAddressUseAfterScope ||
67 LangOpts.Sanitize.has(K: SanitizerKind::HWAddress) ||
68 LangOpts.Sanitize.has(K: SanitizerKind::Memory) ||
69 LangOpts.Sanitize.has(K: SanitizerKind::MemtagStack))
70 return true;
71
72 // For now, only in optimized builds.
73 return CGOpts.OptimizationLevel != 0;
74}
75
76CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
77 : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
78 Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(),
79 CGBuilderInserterTy(this)),
80 SanOpts(CGM.getLangOpts().Sanitize), CurFPFeatures(CGM.getLangOpts()),
81 DebugInfo(CGM.getModuleDebugInfo()),
82 PGO(std::make_unique<CodeGenPGO>(args&: cgm)),
83 ShouldEmitLifetimeMarkers(
84 shouldEmitLifetimeMarkers(CGOpts: CGM.getCodeGenOpts(), LangOpts: CGM.getLangOpts())) {
85 if (!suppressNewContext)
86 CGM.getCXXABI().getMangleContext().startNewFunction();
87 EHStack.setCGF(this);
88
89 SetFastMathFlags(CurFPFeatures);
90}
91
92CodeGenFunction::~CodeGenFunction() {
93 assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup");
94 assert(DeferredDeactivationCleanupStack.empty() &&
95 "missed to deactivate a cleanup");
96
97 if (getLangOpts().OpenMP && CurFn)
98 CGM.getOpenMPRuntime().functionFinished(CGF&: *this);
99
100 // If we have an OpenMPIRBuilder we want to finalize functions (incl.
101 // outlining etc) at some point. Doing it once the function codegen is done
102 // seems to be a reasonable spot. We do it here, as opposed to the deletion
103 // time of the CodeGenModule, because we have to ensure the IR has not yet
104 // been "emitted" to the outside, thus, modifications are still sensible.
105 if (CGM.getLangOpts().OpenMPIRBuilder && CurFn)
106 CGM.getOpenMPRuntime().getOMPBuilder().finalize(Fn: CurFn);
107}
108
109// Map the LangOption for exception behavior into
110// the corresponding enum in the IR.
111llvm::fp::ExceptionBehavior
112clang::ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind) {
113
114 switch (Kind) {
115 case LangOptions::FPE_Ignore: return llvm::fp::ebIgnore;
116 case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap;
117 case LangOptions::FPE_Strict: return llvm::fp::ebStrict;
118 default:
119 llvm_unreachable("Unsupported FP Exception Behavior");
120 }
121}
122
123void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) {
124 llvm::FastMathFlags FMF;
125 FMF.setAllowReassoc(FPFeatures.getAllowFPReassociate());
126 FMF.setNoNaNs(FPFeatures.getNoHonorNaNs());
127 FMF.setNoInfs(FPFeatures.getNoHonorInfs());
128 FMF.setNoSignedZeros(FPFeatures.getNoSignedZero());
129 FMF.setAllowReciprocal(FPFeatures.getAllowReciprocal());
130 FMF.setApproxFunc(FPFeatures.getAllowApproxFunc());
131 FMF.setAllowContract(FPFeatures.allowFPContractAcrossStatement());
132 Builder.setFastMathFlags(FMF);
133}
134
135CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
136 const Expr *E)
137 : CGF(CGF) {
138 ConstructorHelper(FPFeatures: E->getFPFeaturesInEffect(LO: CGF.getLangOpts()));
139}
140
141CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
142 FPOptions FPFeatures)
143 : CGF(CGF) {
144 ConstructorHelper(FPFeatures);
145}
146
147void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) {
148 OldFPFeatures = CGF.CurFPFeatures;
149 CGF.CurFPFeatures = FPFeatures;
150
151 OldExcept = CGF.Builder.getDefaultConstrainedExcept();
152 OldRounding = CGF.Builder.getDefaultConstrainedRounding();
153
154 if (OldFPFeatures == FPFeatures)
155 return;
156
157 FMFGuard.emplace(args&: CGF.Builder);
158
159 llvm::RoundingMode NewRoundingBehavior = FPFeatures.getRoundingMode();
160 CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);
161 auto NewExceptionBehavior =
162 ToConstrainedExceptMD(Kind: FPFeatures.getExceptionMode());
163 CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);
164
165 CGF.SetFastMathFlags(FPFeatures);
166
167 assert((CGF.CurFuncDecl == nullptr || CGF.Builder.getIsFPConstrained() ||
168 isa<CXXConstructorDecl>(CGF.CurFuncDecl) ||
169 isa<CXXDestructorDecl>(CGF.CurFuncDecl) ||
170 (NewExceptionBehavior == llvm::fp::ebIgnore &&
171 NewRoundingBehavior == llvm::RoundingMode::NearestTiesToEven)) &&
172 "FPConstrained should be enabled on entire function");
173
174 auto mergeFnAttrValue = [&](StringRef Name, bool Value) {
175 auto OldValue =
176 CGF.CurFn->getFnAttribute(Kind: Name).getValueAsBool();
177 auto NewValue = OldValue & Value;
178 if (OldValue != NewValue)
179 CGF.CurFn->addFnAttr(Kind: Name, Val: llvm::toStringRef(B: NewValue));
180 };
181 mergeFnAttrValue("no-infs-fp-math", FPFeatures.getNoHonorInfs());
182 mergeFnAttrValue("no-nans-fp-math", FPFeatures.getNoHonorNaNs());
183 mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.getNoSignedZero());
184}
185
186CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
187 CGF.CurFPFeatures = OldFPFeatures;
188 CGF.Builder.setDefaultConstrainedExcept(OldExcept);
189 CGF.Builder.setDefaultConstrainedRounding(OldRounding);
190}
191
192static LValue
193makeNaturalAlignAddrLValue(llvm::Value *V, QualType T, bool ForPointeeType,
194 bool MightBeSigned, CodeGenFunction &CGF,
195 KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
196 LValueBaseInfo BaseInfo;
197 TBAAAccessInfo TBAAInfo;
198 CharUnits Alignment =
199 CGF.CGM.getNaturalTypeAlignment(T, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo, forPointeeType: ForPointeeType);
200 Address Addr =
201 MightBeSigned
202 ? CGF.makeNaturalAddressForPointer(Ptr: V, T, Alignment, ForPointeeType: false, BaseInfo: nullptr,
203 TBAAInfo: nullptr, IsKnownNonNull)
204 : Address(V, CGF.ConvertTypeForMem(T), Alignment, IsKnownNonNull);
205 return CGF.MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
206}
207
208LValue
209CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
210 KnownNonNull_t IsKnownNonNull) {
211 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,
212 /*MightBeSigned*/ true, CGF&: *this,
213 IsKnownNonNull);
214}
215
216LValue
217CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
218 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,
219 /*MightBeSigned*/ true, CGF&: *this);
220}
221
222LValue CodeGenFunction::MakeNaturalAlignRawAddrLValue(llvm::Value *V,
223 QualType T) {
224 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,
225 /*MightBeSigned*/ false, CGF&: *this);
226}
227
228LValue CodeGenFunction::MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V,
229 QualType T) {
230 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,
231 /*MightBeSigned*/ false, CGF&: *this);
232}
233
234llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
235 return CGM.getTypes().ConvertTypeForMem(T);
236}
237
238llvm::Type *CodeGenFunction::ConvertType(QualType T) {
239 return CGM.getTypes().ConvertType(T);
240}
241
242llvm::Type *CodeGenFunction::convertTypeForLoadStore(QualType ASTTy,
243 llvm::Type *LLVMTy) {
244 return CGM.getTypes().convertTypeForLoadStore(T: ASTTy, LLVMTy);
245}
246
247TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
248 type = type.getCanonicalType();
249 while (true) {
250 switch (type->getTypeClass()) {
251#define TYPE(name, parent)
252#define ABSTRACT_TYPE(name, parent)
253#define NON_CANONICAL_TYPE(name, parent) case Type::name:
254#define DEPENDENT_TYPE(name, parent) case Type::name:
255#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
256#include "clang/AST/TypeNodes.inc"
257 llvm_unreachable("non-canonical or dependent type in IR-generation");
258
259 case Type::Auto:
260 case Type::DeducedTemplateSpecialization:
261 llvm_unreachable("undeduced type in IR-generation");
262
263 // Various scalar types.
264 case Type::Builtin:
265 case Type::Pointer:
266 case Type::BlockPointer:
267 case Type::LValueReference:
268 case Type::RValueReference:
269 case Type::MemberPointer:
270 case Type::Vector:
271 case Type::ExtVector:
272 case Type::ConstantMatrix:
273 case Type::FunctionProto:
274 case Type::FunctionNoProto:
275 case Type::Enum:
276 case Type::ObjCObjectPointer:
277 case Type::Pipe:
278 case Type::BitInt:
279 case Type::HLSLAttributedResource:
280 case Type::HLSLInlineSpirv:
281 return TEK_Scalar;
282
283 // Complexes.
284 case Type::Complex:
285 return TEK_Complex;
286
287 // Arrays, records, and Objective-C objects.
288 case Type::ConstantArray:
289 case Type::IncompleteArray:
290 case Type::VariableArray:
291 case Type::Record:
292 case Type::ObjCObject:
293 case Type::ObjCInterface:
294 case Type::ArrayParameter:
295 return TEK_Aggregate;
296
297 // We operate on atomic values according to their underlying type.
298 case Type::Atomic:
299 type = cast<AtomicType>(Val&: type)->getValueType();
300 continue;
301 }
302 llvm_unreachable("unknown type kind!");
303 }
304}
305
306llvm::DebugLoc CodeGenFunction::EmitReturnBlock() {
307 // For cleanliness, we try to avoid emitting the return block for
308 // simple cases.
309 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
310
311 if (CurBB) {
312 assert(!CurBB->getTerminator() && "Unexpected terminated block.");
313
314 // We have a valid insert point, reuse it if it is empty or there are no
315 // explicit jumps to the return block.
316 if (CurBB->empty() || ReturnBlock.getBlock()->use_empty()) {
317 ReturnBlock.getBlock()->replaceAllUsesWith(V: CurBB);
318 delete ReturnBlock.getBlock();
319 ReturnBlock = JumpDest();
320 } else
321 EmitBlock(BB: ReturnBlock.getBlock());
322 return llvm::DebugLoc();
323 }
324
325 // Otherwise, if the return block is the target of a single direct
326 // branch then we can just put the code in that block instead. This
327 // cleans up functions which started with a unified return block.
328 if (ReturnBlock.getBlock()->hasOneUse()) {
329 llvm::BranchInst *BI =
330 dyn_cast<llvm::BranchInst>(Val: *ReturnBlock.getBlock()->user_begin());
331 if (BI && BI->isUnconditional() &&
332 BI->getSuccessor(i: 0) == ReturnBlock.getBlock()) {
333 // Record/return the DebugLoc of the simple 'return' expression to be used
334 // later by the actual 'ret' instruction.
335 llvm::DebugLoc Loc = BI->getDebugLoc();
336 Builder.SetInsertPoint(BI->getParent());
337 BI->eraseFromParent();
338 delete ReturnBlock.getBlock();
339 ReturnBlock = JumpDest();
340 return Loc;
341 }
342 }
343
344 // FIXME: We are at an unreachable point, there is no reason to emit the block
345 // unless it has uses. However, we still need a place to put the debug
346 // region.end for now.
347
348 EmitBlock(BB: ReturnBlock.getBlock());
349 return llvm::DebugLoc();
350}
351
352static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
353 if (!BB) return;
354 if (!BB->use_empty()) {
355 CGF.CurFn->insert(Position: CGF.CurFn->end(), BB);
356 return;
357 }
358 delete BB;
359}
360
361void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
362 assert(BreakContinueStack.empty() &&
363 "mismatched push/pop in break/continue stack!");
364 assert(LifetimeExtendedCleanupStack.empty() &&
365 "mismatched push/pop of cleanups in EHStack!");
366 assert(DeferredDeactivationCleanupStack.empty() &&
367 "mismatched activate/deactivate of cleanups!");
368
369 if (CGM.shouldEmitConvergenceTokens()) {
370 ConvergenceTokenStack.pop_back();
371 assert(ConvergenceTokenStack.empty() &&
372 "mismatched push/pop in convergence stack!");
373 }
374
375 bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
376 && NumSimpleReturnExprs == NumReturnExprs
377 && ReturnBlock.getBlock()->use_empty();
378 // Usually the return expression is evaluated before the cleanup
379 // code. If the function contains only a simple return statement,
380 // such as a constant, the location before the cleanup code becomes
381 // the last useful breakpoint in the function, because the simple
382 // return expression will be evaluated after the cleanup code. To be
383 // safe, set the debug location for cleanup code to the location of
384 // the return statement. Otherwise the cleanup code should be at the
385 // end of the function's lexical scope.
386 //
387 // If there are multiple branches to the return block, the branch
388 // instructions will get the location of the return statements and
389 // all will be fine.
390 if (CGDebugInfo *DI = getDebugInfo()) {
391 if (OnlySimpleReturnStmts)
392 DI->EmitLocation(Builder, Loc: LastStopPoint);
393 else
394 DI->EmitLocation(Builder, Loc: EndLoc);
395 }
396
397 // Pop any cleanups that might have been associated with the
398 // parameters. Do this in whatever block we're currently in; it's
399 // important to do this before we enter the return block or return
400 // edges will be *really* confused.
401 bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth;
402 bool HasOnlyNoopCleanups =
403 HasCleanups && EHStack.containsOnlyNoopCleanups(Old: PrologueCleanupDepth);
404 bool EmitRetDbgLoc = !HasCleanups || HasOnlyNoopCleanups;
405
406 std::optional<ApplyDebugLocation> OAL;
407 if (HasCleanups) {
408 // Make sure the line table doesn't jump back into the body for
409 // the ret after it's been at EndLoc.
410 if (CGDebugInfo *DI = getDebugInfo()) {
411 if (OnlySimpleReturnStmts)
412 DI->EmitLocation(Builder, Loc: EndLoc);
413 else
414 // We may not have a valid end location. Try to apply it anyway, and
415 // fall back to an artificial location if needed.
416 OAL = ApplyDebugLocation::CreateDefaultArtificial(CGF&: *this, TemporaryLocation: EndLoc);
417 }
418
419 PopCleanupBlocks(OldCleanupStackSize: PrologueCleanupDepth);
420 }
421
422 // Emit function epilog (to return).
423 llvm::DebugLoc Loc = EmitReturnBlock();
424
425 if (ShouldInstrumentFunction()) {
426 if (CGM.getCodeGenOpts().InstrumentFunctions)
427 CurFn->addFnAttr(Kind: "instrument-function-exit", Val: "__cyg_profile_func_exit");
428 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
429 CurFn->addFnAttr(Kind: "instrument-function-exit-inlined",
430 Val: "__cyg_profile_func_exit");
431 }
432
433 // Emit debug descriptor for function end.
434 if (CGDebugInfo *DI = getDebugInfo())
435 DI->EmitFunctionEnd(Builder, Fn: CurFn);
436
437 // Reset the debug location to that of the simple 'return' expression, if any
438 // rather than that of the end of the function's scope '}'.
439 uint64_t RetKeyInstructionsAtomGroup = Loc ? Loc->getAtomGroup() : 0;
440 ApplyDebugLocation AL(*this, Loc);
441 EmitFunctionEpilog(FI: *CurFnInfo, EmitRetDbgLoc, EndLoc,
442 RetKeyInstructionsSourceAtom: RetKeyInstructionsAtomGroup);
443 EmitEndEHSpec(D: CurCodeDecl);
444
445 assert(EHStack.empty() &&
446 "did not remove all scopes from cleanup stack!");
447
448 // If someone did an indirect goto, emit the indirect goto block at the end of
449 // the function.
450 if (IndirectBranch) {
451 EmitBlock(BB: IndirectBranch->getParent());
452 Builder.ClearInsertionPoint();
453 }
454
455 // If some of our locals escaped, insert a call to llvm.localescape in the
456 // entry block.
457 if (!EscapedLocals.empty()) {
458 // Invert the map from local to index into a simple vector. There should be
459 // no holes.
460 SmallVector<llvm::Value *, 4> EscapeArgs;
461 EscapeArgs.resize(N: EscapedLocals.size());
462 for (auto &Pair : EscapedLocals)
463 EscapeArgs[Pair.second] = Pair.first;
464 llvm::Function *FrameEscapeFn = llvm::Intrinsic::getOrInsertDeclaration(
465 M: &CGM.getModule(), id: llvm::Intrinsic::localescape);
466 CGBuilderTy(*this, AllocaInsertPt).CreateCall(Callee: FrameEscapeFn, Args: EscapeArgs);
467 }
468
469 // Remove the AllocaInsertPt instruction, which is just a convenience for us.
470 llvm::Instruction *Ptr = AllocaInsertPt;
471 AllocaInsertPt = nullptr;
472 Ptr->eraseFromParent();
473
474 // PostAllocaInsertPt, if created, was lazily created when it was required,
475 // remove it now since it was just created for our own convenience.
476 if (PostAllocaInsertPt) {
477 llvm::Instruction *PostPtr = PostAllocaInsertPt;
478 PostAllocaInsertPt = nullptr;
479 PostPtr->eraseFromParent();
480 }
481
482 // If someone took the address of a label but never did an indirect goto, we
483 // made a zero entry PHI node, which is illegal, zap it now.
484 if (IndirectBranch) {
485 llvm::PHINode *PN = cast<llvm::PHINode>(Val: IndirectBranch->getAddress());
486 if (PN->getNumIncomingValues() == 0) {
487 PN->replaceAllUsesWith(V: llvm::PoisonValue::get(T: PN->getType()));
488 PN->eraseFromParent();
489 }
490 }
491
492 EmitIfUsed(CGF&: *this, BB: EHResumeBlock);
493 EmitIfUsed(CGF&: *this, BB: TerminateLandingPad);
494 EmitIfUsed(CGF&: *this, BB: TerminateHandler);
495 EmitIfUsed(CGF&: *this, BB: UnreachableBlock);
496
497 for (const auto &FuncletAndParent : TerminateFunclets)
498 EmitIfUsed(CGF&: *this, BB: FuncletAndParent.second);
499
500 if (CGM.getCodeGenOpts().EmitDeclMetadata)
501 EmitDeclMetadata();
502
503 for (const auto &R : DeferredReplacements) {
504 if (llvm::Value *Old = R.first) {
505 Old->replaceAllUsesWith(V: R.second);
506 cast<llvm::Instruction>(Val: Old)->eraseFromParent();
507 }
508 }
509 DeferredReplacements.clear();
510
511 // Eliminate CleanupDestSlot alloca by replacing it with SSA values and
512 // PHIs if the current function is a coroutine. We don't do it for all
513 // functions as it may result in slight increase in numbers of instructions
514 // if compiled with no optimizations. We do it for coroutine as the lifetime
515 // of CleanupDestSlot alloca make correct coroutine frame building very
516 // difficult.
517 if (NormalCleanupDest.isValid() && isCoroutine()) {
518 llvm::DominatorTree DT(*CurFn);
519 llvm::PromoteMemToReg(
520 Allocas: cast<llvm::AllocaInst>(Val: NormalCleanupDest.getPointer()), DT);
521 NormalCleanupDest = Address::invalid();
522 }
523
524 // Scan function arguments for vector width.
525 for (llvm::Argument &A : CurFn->args())
526 if (auto *VT = dyn_cast<llvm::VectorType>(Val: A.getType()))
527 LargestVectorWidth =
528 std::max(a: (uint64_t)LargestVectorWidth,
529 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
530
531 // Update vector width based on return type.
532 if (auto *VT = dyn_cast<llvm::VectorType>(Val: CurFn->getReturnType()))
533 LargestVectorWidth =
534 std::max(a: (uint64_t)LargestVectorWidth,
535 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
536
537 if (CurFnInfo->getMaxVectorWidth() > LargestVectorWidth)
538 LargestVectorWidth = CurFnInfo->getMaxVectorWidth();
539
540 // Add the min-legal-vector-width attribute. This contains the max width from:
541 // 1. min-vector-width attribute used in the source program.
542 // 2. Any builtins used that have a vector width specified.
543 // 3. Values passed in and out of inline assembly.
544 // 4. Width of vector arguments and return types for this function.
545 // 5. Width of vector arguments and return types for functions called by this
546 // function.
547 if (getContext().getTargetInfo().getTriple().isX86())
548 CurFn->addFnAttr(Kind: "min-legal-vector-width",
549 Val: llvm::utostr(X: LargestVectorWidth));
550
551 // If we generated an unreachable return block, delete it now.
552 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()) {
553 Builder.ClearInsertionPoint();
554 ReturnBlock.getBlock()->eraseFromParent();
555 }
556 if (ReturnValue.isValid()) {
557 auto *RetAlloca =
558 dyn_cast<llvm::AllocaInst>(Val: ReturnValue.emitRawPointer(CGF&: *this));
559 if (RetAlloca && RetAlloca->use_empty()) {
560 RetAlloca->eraseFromParent();
561 ReturnValue = Address::invalid();
562 }
563 }
564}
565
566/// ShouldInstrumentFunction - Return true if the current function should be
567/// instrumented with __cyg_profile_func_* calls
568bool CodeGenFunction::ShouldInstrumentFunction() {
569 if (!CGM.getCodeGenOpts().InstrumentFunctions &&
570 !CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining &&
571 !CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
572 return false;
573 if (!CurFuncDecl || CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>())
574 return false;
575 return true;
576}
577
578bool CodeGenFunction::ShouldSkipSanitizerInstrumentation() {
579 if (!CurFuncDecl)
580 return false;
581 return CurFuncDecl->hasAttr<DisableSanitizerInstrumentationAttr>();
582}
583
584/// ShouldXRayInstrument - Return true if the current function should be
585/// instrumented with XRay nop sleds.
586bool CodeGenFunction::ShouldXRayInstrumentFunction() const {
587 return CGM.getCodeGenOpts().XRayInstrumentFunctions;
588}
589
590/// AlwaysEmitXRayCustomEvents - Return true if we should emit IR for calls to
591/// the __xray_customevent(...) builtin calls, when doing XRay instrumentation.
592bool CodeGenFunction::AlwaysEmitXRayCustomEvents() const {
593 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
594 (CGM.getCodeGenOpts().XRayAlwaysEmitCustomEvents ||
595 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
596 XRayInstrKind::Custom);
597}
598
599bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const {
600 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
601 (CGM.getCodeGenOpts().XRayAlwaysEmitTypedEvents ||
602 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
603 XRayInstrKind::Typed);
604}
605
606llvm::ConstantInt *
607CodeGenFunction::getUBSanFunctionTypeHash(QualType Ty) const {
608 // Remove any (C++17) exception specifications, to allow calling e.g. a
609 // noexcept function through a non-noexcept pointer.
610 if (!Ty->isFunctionNoProtoType())
611 Ty = getContext().getFunctionTypeWithExceptionSpec(Orig: Ty, ESI: EST_None);
612 std::string Mangled;
613 llvm::raw_string_ostream Out(Mangled);
614 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: Ty, Out, NormalizeIntegers: false);
615 return llvm::ConstantInt::get(
616 Ty: CGM.Int32Ty, V: static_cast<uint32_t>(llvm::xxh3_64bits(data: Mangled)));
617}
618
619void CodeGenFunction::EmitKernelMetadata(const FunctionDecl *FD,
620 llvm::Function *Fn) {
621 if (!FD->hasAttr<DeviceKernelAttr>() && !FD->hasAttr<CUDAGlobalAttr>())
622 return;
623
624 llvm::LLVMContext &Context = getLLVMContext();
625
626 CGM.GenKernelArgMetadata(FN: Fn, FD, CGF: this);
627
628 if (!(getLangOpts().OpenCL ||
629 (getLangOpts().CUDA &&
630 getContext().getTargetInfo().getTriple().isSPIRV())))
631 return;
632
633 if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {
634 QualType HintQTy = A->getTypeHint();
635 const ExtVectorType *HintEltQTy = HintQTy->getAs<ExtVectorType>();
636 bool IsSignedInteger =
637 HintQTy->isSignedIntegerType() ||
638 (HintEltQTy && HintEltQTy->getElementType()->isSignedIntegerType());
639 llvm::Metadata *AttrMDArgs[] = {
640 llvm::ConstantAsMetadata::get(C: llvm::PoisonValue::get(
641 T: CGM.getTypes().ConvertType(T: A->getTypeHint()))),
642 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
643 Ty: llvm::IntegerType::get(C&: Context, NumBits: 32),
644 V: llvm::APInt(32, (uint64_t)(IsSignedInteger ? 1 : 0))))};
645 Fn->setMetadata(Kind: "vec_type_hint", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
646 }
647
648 if (const WorkGroupSizeHintAttr *A = FD->getAttr<WorkGroupSizeHintAttr>()) {
649 auto Eval = [&](Expr *E) {
650 return E->EvaluateKnownConstInt(Ctx: FD->getASTContext()).getExtValue();
651 };
652 llvm::Metadata *AttrMDArgs[] = {
653 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getXDim()))),
654 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getYDim()))),
655 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getZDim())))};
656 Fn->setMetadata(Kind: "work_group_size_hint", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
657 }
658
659 if (const ReqdWorkGroupSizeAttr *A = FD->getAttr<ReqdWorkGroupSizeAttr>()) {
660 auto Eval = [&](Expr *E) {
661 return E->EvaluateKnownConstInt(Ctx: FD->getASTContext()).getExtValue();
662 };
663 llvm::Metadata *AttrMDArgs[] = {
664 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getXDim()))),
665 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getYDim()))),
666 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: Eval(A->getZDim())))};
667 Fn->setMetadata(Kind: "reqd_work_group_size", Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
668 }
669
670 if (const OpenCLIntelReqdSubGroupSizeAttr *A =
671 FD->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
672 llvm::Metadata *AttrMDArgs[] = {
673 llvm::ConstantAsMetadata::get(C: Builder.getInt32(C: A->getSubGroupSize()))};
674 Fn->setMetadata(Kind: "intel_reqd_sub_group_size",
675 Node: llvm::MDNode::get(Context, MDs: AttrMDArgs));
676 }
677}
678
679/// Determine whether the function F ends with a return stmt.
680static bool endsWithReturn(const Decl* F) {
681 const Stmt *Body = nullptr;
682 if (auto *FD = dyn_cast_or_null<FunctionDecl>(Val: F))
683 Body = FD->getBody();
684 else if (auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(Val: F))
685 Body = OMD->getBody();
686
687 if (auto *CS = dyn_cast_or_null<CompoundStmt>(Val: Body)) {
688 auto LastStmt = CS->body_rbegin();
689 if (LastStmt != CS->body_rend())
690 return isa<ReturnStmt>(Val: *LastStmt);
691 }
692 return false;
693}
694
695void CodeGenFunction::markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {
696 if (SanOpts.has(K: SanitizerKind::Thread)) {
697 Fn->addFnAttr(Kind: "sanitize_thread_no_checking_at_run_time");
698 Fn->removeFnAttr(Kind: llvm::Attribute::SanitizeThread);
699 }
700}
701
702/// Check if the return value of this function requires sanitization.
703bool CodeGenFunction::requiresReturnValueCheck() const {
704 return requiresReturnValueNullabilityCheck() ||
705 (SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute) && CurCodeDecl &&
706 CurCodeDecl->getAttr<ReturnsNonNullAttr>());
707}
708
709static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) {
710 auto *MD = dyn_cast_or_null<CXXMethodDecl>(Val: D);
711 if (!MD || !MD->getDeclName().getAsIdentifierInfo() ||
712 !MD->getDeclName().getAsIdentifierInfo()->isStr(Str: "allocate") ||
713 (MD->getNumParams() != 1 && MD->getNumParams() != 2))
714 return false;
715
716 if (!Ctx.hasSameType(T1: MD->parameters()[0]->getType(), T2: Ctx.getSizeType()))
717 return false;
718
719 if (MD->getNumParams() == 2) {
720 auto *PT = MD->parameters()[1]->getType()->getAs<PointerType>();
721 if (!PT || !PT->isVoidPointerType() ||
722 !PT->getPointeeType().isConstQualified())
723 return false;
724 }
725
726 return true;
727}
728
729bool CodeGenFunction::isInAllocaArgument(CGCXXABI &ABI, QualType Ty) {
730 const CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
731 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
732}
733
734bool CodeGenFunction::hasInAllocaArg(const CXXMethodDecl *MD) {
735 return getTarget().getTriple().getArch() == llvm::Triple::x86 &&
736 getTarget().getCXXABI().isMicrosoft() &&
737 llvm::any_of(Range: MD->parameters(), P: [&](ParmVarDecl *P) {
738 return isInAllocaArgument(ABI&: CGM.getCXXABI(), Ty: P->getType());
739 });
740}
741
742/// Return the UBSan prologue signature for \p FD if one is available.
743static llvm::Constant *getPrologueSignature(CodeGenModule &CGM,
744 const FunctionDecl *FD) {
745 if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: FD))
746 if (!MD->isStatic())
747 return nullptr;
748 return CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM);
749}
750
751void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
752 llvm::Function *Fn,
753 const CGFunctionInfo &FnInfo,
754 const FunctionArgList &Args,
755 SourceLocation Loc,
756 SourceLocation StartLoc) {
757 assert(!CurFn &&
758 "Do not use a CodeGenFunction object for more than one function");
759
760 const Decl *D = GD.getDecl();
761
762 DidCallStackSave = false;
763 CurCodeDecl = D;
764 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
765 if (FD && FD->usesSEHTry())
766 CurSEHParent = GD;
767 CurFuncDecl = (D ? D->getNonClosureContext() : nullptr);
768 FnRetTy = RetTy;
769 CurFn = Fn;
770 CurFnInfo = &FnInfo;
771 assert(CurFn->isDeclaration() && "Function already has body?");
772
773 // If this function is ignored for any of the enabled sanitizers,
774 // disable the sanitizer for the function.
775 do {
776#define SANITIZER(NAME, ID) \
777 if (SanOpts.empty()) \
778 break; \
779 if (SanOpts.has(SanitizerKind::ID)) \
780 if (CGM.isInNoSanitizeList(SanitizerKind::ID, Fn, Loc)) \
781 SanOpts.set(SanitizerKind::ID, false);
782
783#include "clang/Basic/Sanitizers.def"
784#undef SANITIZER
785 } while (false);
786
787 if (D) {
788 const bool SanitizeBounds = SanOpts.hasOneOf(K: SanitizerKind::Bounds);
789 SanitizerMask no_sanitize_mask;
790 bool NoSanitizeCoverage = false;
791
792 for (auto *Attr : D->specific_attrs<NoSanitizeAttr>()) {
793 no_sanitize_mask |= Attr->getMask();
794 // SanitizeCoverage is not handled by SanOpts.
795 if (Attr->hasCoverage())
796 NoSanitizeCoverage = true;
797 }
798
799 // Apply the no_sanitize* attributes to SanOpts.
800 SanOpts.Mask &= ~no_sanitize_mask;
801 if (no_sanitize_mask & SanitizerKind::Address)
802 SanOpts.set(K: SanitizerKind::KernelAddress, Value: false);
803 if (no_sanitize_mask & SanitizerKind::KernelAddress)
804 SanOpts.set(K: SanitizerKind::Address, Value: false);
805 if (no_sanitize_mask & SanitizerKind::HWAddress)
806 SanOpts.set(K: SanitizerKind::KernelHWAddress, Value: false);
807 if (no_sanitize_mask & SanitizerKind::KernelHWAddress)
808 SanOpts.set(K: SanitizerKind::HWAddress, Value: false);
809
810 if (SanitizeBounds && !SanOpts.hasOneOf(K: SanitizerKind::Bounds))
811 Fn->addFnAttr(Kind: llvm::Attribute::NoSanitizeBounds);
812
813 if (NoSanitizeCoverage && CGM.getCodeGenOpts().hasSanitizeCoverage())
814 Fn->addFnAttr(Kind: llvm::Attribute::NoSanitizeCoverage);
815
816 // Some passes need the non-negated no_sanitize attribute. Pass them on.
817 if (CGM.getCodeGenOpts().hasSanitizeBinaryMetadata()) {
818 if (no_sanitize_mask & SanitizerKind::Thread)
819 Fn->addFnAttr(Kind: "no_sanitize_thread");
820 }
821 }
822
823 if (ShouldSkipSanitizerInstrumentation()) {
824 CurFn->addFnAttr(Kind: llvm::Attribute::DisableSanitizerInstrumentation);
825 } else {
826 // Apply sanitizer attributes to the function.
827 if (SanOpts.hasOneOf(K: SanitizerKind::Address | SanitizerKind::KernelAddress))
828 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAddress);
829 if (SanOpts.hasOneOf(K: SanitizerKind::HWAddress |
830 SanitizerKind::KernelHWAddress))
831 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeHWAddress);
832 if (SanOpts.has(K: SanitizerKind::MemtagStack))
833 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemTag);
834 if (SanOpts.has(K: SanitizerKind::Thread))
835 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeThread);
836 if (SanOpts.has(K: SanitizerKind::Type))
837 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeType);
838 if (SanOpts.has(K: SanitizerKind::NumericalStability))
839 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeNumericalStability);
840 if (SanOpts.hasOneOf(K: SanitizerKind::Memory | SanitizerKind::KernelMemory))
841 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeMemory);
842 if (SanOpts.has(K: SanitizerKind::AllocToken))
843 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeAllocToken);
844 }
845 if (SanOpts.has(K: SanitizerKind::SafeStack))
846 Fn->addFnAttr(Kind: llvm::Attribute::SafeStack);
847 if (SanOpts.has(K: SanitizerKind::ShadowCallStack))
848 Fn->addFnAttr(Kind: llvm::Attribute::ShadowCallStack);
849
850 if (SanOpts.has(K: SanitizerKind::Realtime))
851 if (FD && FD->getASTContext().hasAnyFunctionEffects())
852 for (const FunctionEffectWithCondition &Fe : FD->getFunctionEffects()) {
853 if (Fe.Effect.kind() == FunctionEffect::Kind::NonBlocking)
854 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeRealtime);
855 else if (Fe.Effect.kind() == FunctionEffect::Kind::Blocking)
856 Fn->addFnAttr(Kind: llvm::Attribute::SanitizeRealtimeBlocking);
857 }
858
859 // Apply fuzzing attribute to the function.
860 if (SanOpts.hasOneOf(K: SanitizerKind::Fuzzer | SanitizerKind::FuzzerNoLink))
861 Fn->addFnAttr(Kind: llvm::Attribute::OptForFuzzing);
862
863 // Ignore TSan memory acesses from within ObjC/ObjC++ dealloc, initialize,
864 // .cxx_destruct, __destroy_helper_block_ and all of their calees at run time.
865 if (SanOpts.has(K: SanitizerKind::Thread)) {
866 if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(Val: D)) {
867 const IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(argIndex: 0);
868 if (OMD->getMethodFamily() == OMF_dealloc ||
869 OMD->getMethodFamily() == OMF_initialize ||
870 (OMD->getSelector().isUnarySelector() && II->isStr(Str: ".cxx_destruct"))) {
871 markAsIgnoreThreadCheckingAtRuntime(Fn);
872 }
873 }
874 }
875
876 // Ignore unrelated casts in STL allocate() since the allocator must cast
877 // from void* to T* before object initialization completes. Don't match on the
878 // namespace because not all allocators are in std::
879 if (D && SanOpts.has(K: SanitizerKind::CFIUnrelatedCast)) {
880 if (matchesStlAllocatorFn(D, Ctx: getContext()))
881 SanOpts.Mask &= ~SanitizerKind::CFIUnrelatedCast;
882 }
883
884 // Ignore null checks in coroutine functions since the coroutines passes
885 // are not aware of how to move the extra UBSan instructions across the split
886 // coroutine boundaries.
887 if (D && SanOpts.has(K: SanitizerKind::Null))
888 if (FD && FD->getBody() &&
889 FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass)
890 SanOpts.Mask &= ~SanitizerKind::Null;
891
892 // Apply xray attributes to the function (as a string, for now)
893 bool AlwaysXRayAttr = false;
894 if (const auto *XRayAttr = D ? D->getAttr<XRayInstrumentAttr>() : nullptr) {
895 if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
896 K: XRayInstrKind::FunctionEntry) ||
897 CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
898 K: XRayInstrKind::FunctionExit)) {
899 if (XRayAttr->alwaysXRayInstrument() && ShouldXRayInstrumentFunction()) {
900 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-always");
901 AlwaysXRayAttr = true;
902 }
903 if (XRayAttr->neverXRayInstrument())
904 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-never");
905 if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())
906 if (ShouldXRayInstrumentFunction())
907 Fn->addFnAttr(Kind: "xray-log-args",
908 Val: llvm::utostr(X: LogArgs->getArgumentCount()));
909 }
910 } else {
911 if (ShouldXRayInstrumentFunction() && !CGM.imbueXRayAttrs(Fn, Loc))
912 Fn->addFnAttr(
913 Kind: "xray-instruction-threshold",
914 Val: llvm::itostr(X: CGM.getCodeGenOpts().XRayInstructionThreshold));
915 }
916
917 if (ShouldXRayInstrumentFunction()) {
918 if (CGM.getCodeGenOpts().XRayIgnoreLoops)
919 Fn->addFnAttr(Kind: "xray-ignore-loops");
920
921 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
922 K: XRayInstrKind::FunctionExit))
923 Fn->addFnAttr(Kind: "xray-skip-exit");
924
925 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
926 K: XRayInstrKind::FunctionEntry))
927 Fn->addFnAttr(Kind: "xray-skip-entry");
928
929 auto FuncGroups = CGM.getCodeGenOpts().XRayTotalFunctionGroups;
930 if (FuncGroups > 1) {
931 auto FuncName = llvm::ArrayRef<uint8_t>(CurFn->getName().bytes_begin(),
932 CurFn->getName().bytes_end());
933 auto Group = crc32(Data: FuncName) % FuncGroups;
934 if (Group != CGM.getCodeGenOpts().XRaySelectedFunctionGroup &&
935 !AlwaysXRayAttr)
936 Fn->addFnAttr(Kind: "function-instrument", Val: "xray-never");
937 }
938 }
939
940 if (CGM.getCodeGenOpts().getProfileInstr() !=
941 llvm::driver::ProfileInstrKind::ProfileNone) {
942 switch (CGM.isFunctionBlockedFromProfileInstr(Fn, Loc)) {
943 case ProfileList::Skip:
944 Fn->addFnAttr(Kind: llvm::Attribute::SkipProfile);
945 break;
946 case ProfileList::Forbid:
947 Fn->addFnAttr(Kind: llvm::Attribute::NoProfile);
948 break;
949 case ProfileList::Allow:
950 break;
951 }
952 }
953
954 unsigned Count, Offset;
955 StringRef Section;
956 if (const auto *Attr =
957 D ? D->getAttr<PatchableFunctionEntryAttr>() : nullptr) {
958 Count = Attr->getCount();
959 Offset = Attr->getOffset();
960 Section = Attr->getSection();
961 } else {
962 Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;
963 Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;
964 }
965 if (Section.empty())
966 Section = CGM.getCodeGenOpts().PatchableFunctionEntrySection;
967 if (Count && Offset <= Count) {
968 Fn->addFnAttr(Kind: "patchable-function-entry", Val: std::to_string(val: Count - Offset));
969 if (Offset)
970 Fn->addFnAttr(Kind: "patchable-function-prefix", Val: std::to_string(val: Offset));
971 if (!Section.empty())
972 Fn->addFnAttr(Kind: "patchable-function-entry-section", Val: Section);
973 }
974 // Instruct that functions for COFF/CodeView targets should start with a
975 // patchable instruction, but only on x86/x64. Don't forward this to ARM/ARM64
976 // backends as they don't need it -- instructions on these architectures are
977 // always atomically patchable at runtime.
978 if (CGM.getCodeGenOpts().HotPatch &&
979 getContext().getTargetInfo().getTriple().isX86() &&
980 getContext().getTargetInfo().getTriple().getEnvironment() !=
981 llvm::Triple::CODE16)
982 Fn->addFnAttr(Kind: "patchable-function", Val: "prologue-short-redirect");
983
984 // Add no-jump-tables value.
985 if (CGM.getCodeGenOpts().NoUseJumpTables)
986 Fn->addFnAttr(Kind: "no-jump-tables", Val: "true");
987
988 // Add no-inline-line-tables value.
989 if (CGM.getCodeGenOpts().NoInlineLineTables)
990 Fn->addFnAttr(Kind: "no-inline-line-tables");
991
992 // Add profile-sample-accurate value.
993 if (CGM.getCodeGenOpts().ProfileSampleAccurate)
994 Fn->addFnAttr(Kind: "profile-sample-accurate");
995
996 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
997 Fn->addFnAttr(Kind: "use-sample-profile");
998
999 if (D && D->hasAttr<CFICanonicalJumpTableAttr>())
1000 Fn->addFnAttr(Kind: "cfi-canonical-jump-table");
1001
1002 if (D && D->hasAttr<NoProfileFunctionAttr>())
1003 Fn->addFnAttr(Kind: llvm::Attribute::NoProfile);
1004
1005 if (D && D->hasAttr<HybridPatchableAttr>())
1006 Fn->addFnAttr(Kind: llvm::Attribute::HybridPatchable);
1007
1008 if (D) {
1009 // Function attributes take precedence over command line flags.
1010 if (auto *A = D->getAttr<FunctionReturnThunksAttr>()) {
1011 switch (A->getThunkType()) {
1012 case FunctionReturnThunksAttr::Kind::Keep:
1013 break;
1014 case FunctionReturnThunksAttr::Kind::Extern:
1015 Fn->addFnAttr(Kind: llvm::Attribute::FnRetThunkExtern);
1016 break;
1017 }
1018 } else if (CGM.getCodeGenOpts().FunctionReturnThunks)
1019 Fn->addFnAttr(Kind: llvm::Attribute::FnRetThunkExtern);
1020 }
1021
1022 if (FD && (getLangOpts().OpenCL ||
1023 (getLangOpts().CUDA &&
1024 getContext().getTargetInfo().getTriple().isSPIRV()) ||
1025 ((getLangOpts().HIP || getLangOpts().OffloadViaLLVM) &&
1026 getLangOpts().CUDAIsDevice))) {
1027 // Add metadata for a kernel function.
1028 EmitKernelMetadata(FD, Fn);
1029 }
1030
1031 if (FD && FD->hasAttr<ClspvLibclcBuiltinAttr>()) {
1032 Fn->setMetadata(Kind: "clspv_libclc_builtin",
1033 Node: llvm::MDNode::get(Context&: getLLVMContext(), MDs: {}));
1034 }
1035
1036 // If we are checking function types, emit a function type signature as
1037 // prologue data.
1038 if (FD && SanOpts.has(K: SanitizerKind::Function) &&
1039 !FD->getType()->isCFIUncheckedCalleeFunctionType()) {
1040 if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {
1041 llvm::LLVMContext &Ctx = Fn->getContext();
1042 llvm::MDBuilder MDB(Ctx);
1043 Fn->setMetadata(
1044 KindID: llvm::LLVMContext::MD_func_sanitize,
1045 Node: MDB.createRTTIPointerPrologue(
1046 PrologueSig, RTTI: getUBSanFunctionTypeHash(Ty: FD->getType())));
1047 }
1048 }
1049
1050 // If we're checking nullability, we need to know whether we can check the
1051 // return value. Initialize the flag to 'true' and refine it in EmitParmDecl.
1052 if (SanOpts.has(K: SanitizerKind::NullabilityReturn)) {
1053 auto Nullability = FnRetTy->getNullability();
1054 if (Nullability && *Nullability == NullabilityKind::NonNull &&
1055 !FnRetTy->isRecordType()) {
1056 if (!(SanOpts.has(K: SanitizerKind::ReturnsNonnullAttribute) &&
1057 CurCodeDecl && CurCodeDecl->getAttr<ReturnsNonNullAttr>()))
1058 RetValNullabilityPrecondition =
1059 llvm::ConstantInt::getTrue(Context&: getLLVMContext());
1060 }
1061 }
1062
1063 // If we're in C++ mode and the function name is "main", it is guaranteed
1064 // to be norecurse by the standard (3.6.1.3 "The function main shall not be
1065 // used within a program").
1066 //
1067 // OpenCL C 2.0 v2.2-11 s6.9.i:
1068 // Recursion is not supported.
1069 //
1070 // HLSL
1071 // Recursion is not supported.
1072 //
1073 // SYCL v1.2.1 s3.10:
1074 // kernels cannot include RTTI information, exception classes,
1075 // recursive code, virtual functions or make use of C++ libraries that
1076 // are not compiled for the device.
1077 if (FD &&
1078 ((getLangOpts().CPlusPlus && FD->isMain()) || getLangOpts().OpenCL ||
1079 getLangOpts().HLSL || getLangOpts().SYCLIsDevice ||
1080 (getLangOpts().CUDA && FD->hasAttr<CUDAGlobalAttr>())))
1081 Fn->addFnAttr(Kind: llvm::Attribute::NoRecurse);
1082
1083 llvm::RoundingMode RM = getLangOpts().getDefaultRoundingMode();
1084 llvm::fp::ExceptionBehavior FPExceptionBehavior =
1085 ToConstrainedExceptMD(Kind: getLangOpts().getDefaultExceptionMode());
1086 Builder.setDefaultConstrainedRounding(RM);
1087 Builder.setDefaultConstrainedExcept(FPExceptionBehavior);
1088 if ((FD && (FD->UsesFPIntrin() || FD->hasAttr<StrictFPAttr>())) ||
1089 (!FD && (FPExceptionBehavior != llvm::fp::ebIgnore ||
1090 RM != llvm::RoundingMode::NearestTiesToEven))) {
1091 Builder.setIsFPConstrained(true);
1092 Fn->addFnAttr(Kind: llvm::Attribute::StrictFP);
1093 }
1094
1095 // If a custom alignment is used, force realigning to this alignment on
1096 // any main function which certainly will need it.
1097 if (FD && ((FD->isMain() || FD->isMSVCRTEntryPoint()) &&
1098 CGM.getCodeGenOpts().StackAlignment))
1099 Fn->addFnAttr(Kind: "stackrealign");
1100
1101 // "main" doesn't need to zero out call-used registers.
1102 if (FD && FD->isMain())
1103 Fn->removeFnAttr(Kind: "zero-call-used-regs");
1104
1105 // Add vscale_range attribute if appropriate.
1106 llvm::StringMap<bool> FeatureMap;
1107 auto IsArmStreaming = TargetInfo::ArmStreamingKind::NotStreaming;
1108 if (FD) {
1109 getContext().getFunctionFeatureMap(FeatureMap, FD);
1110 if (const auto *T = FD->getType()->getAs<FunctionProtoType>())
1111 if (T->getAArch64SMEAttributes() &
1112 FunctionType::SME_PStateSMCompatibleMask)
1113 IsArmStreaming = TargetInfo::ArmStreamingKind::StreamingCompatible;
1114
1115 if (IsArmStreamingFunction(FD, IncludeLocallyStreaming: true))
1116 IsArmStreaming = TargetInfo::ArmStreamingKind::Streaming;
1117 }
1118 std::optional<std::pair<unsigned, unsigned>> VScaleRange =
1119 getContext().getTargetInfo().getVScaleRange(LangOpts: getLangOpts(), Mode: IsArmStreaming,
1120 FeatureMap: &FeatureMap);
1121 if (VScaleRange) {
1122 CurFn->addFnAttr(Attr: llvm::Attribute::getWithVScaleRangeArgs(
1123 Context&: getLLVMContext(), MinValue: VScaleRange->first, MaxValue: VScaleRange->second));
1124 }
1125
1126 llvm::BasicBlock *EntryBB = createBasicBlock(name: "entry", parent: CurFn);
1127
1128 // Create a marker to make it easy to insert allocas into the entryblock
1129 // later. Don't create this with the builder, because we don't want it
1130 // folded.
1131 llvm::Value *Poison = llvm::PoisonValue::get(T: Int32Ty);
1132 AllocaInsertPt = new llvm::BitCastInst(Poison, Int32Ty, "allocapt", EntryBB);
1133
1134 ReturnBlock = getJumpDestInCurrentScope(Name: "return");
1135
1136 Builder.SetInsertPoint(EntryBB);
1137
1138 // If we're checking the return value, allocate space for a pointer to a
1139 // precise source location of the checked return statement.
1140 if (requiresReturnValueCheck()) {
1141 ReturnLocation = CreateDefaultAlignTempAlloca(Ty: Int8PtrTy, Name: "return.sloc.ptr");
1142 Builder.CreateStore(Val: llvm::ConstantPointerNull::get(T: Int8PtrTy),
1143 Addr: ReturnLocation);
1144 }
1145
1146 // Emit subprogram debug descriptor.
1147 if (CGDebugInfo *DI = getDebugInfo()) {
1148 // Reconstruct the type from the argument list so that implicit parameters,
1149 // such as 'this' and 'vtt', show up in the debug info. Preserve the calling
1150 // convention.
1151 DI->emitFunctionStart(GD, Loc, ScopeLoc: StartLoc,
1152 FnType: DI->getFunctionType(FD, RetTy, Args), Fn: CurFn,
1153 CurFnIsThunk: CurFuncIsThunk);
1154 }
1155
1156 if (ShouldInstrumentFunction()) {
1157 if (CGM.getCodeGenOpts().InstrumentFunctions)
1158 CurFn->addFnAttr(Kind: "instrument-function-entry", Val: "__cyg_profile_func_enter");
1159 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
1160 CurFn->addFnAttr(Kind: "instrument-function-entry-inlined",
1161 Val: "__cyg_profile_func_enter");
1162 if (CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
1163 CurFn->addFnAttr(Kind: "instrument-function-entry-inlined",
1164 Val: "__cyg_profile_func_enter_bare");
1165 }
1166
1167 // Since emitting the mcount call here impacts optimizations such as function
1168 // inlining, we just add an attribute to insert a mcount call in backend.
1169 // The attribute "counting-function" is set to mcount function name which is
1170 // architecture dependent.
1171 if (CGM.getCodeGenOpts().InstrumentForProfiling) {
1172 // Calls to fentry/mcount should not be generated if function has
1173 // the no_instrument_function attribute.
1174 if (!CurFuncDecl || !CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()) {
1175 if (CGM.getCodeGenOpts().CallFEntry)
1176 Fn->addFnAttr(Kind: "fentry-call", Val: "true");
1177 else {
1178 Fn->addFnAttr(Kind: "instrument-function-entry-inlined",
1179 Val: getTarget().getMCountName());
1180 }
1181 if (CGM.getCodeGenOpts().MNopMCount) {
1182 if (!CGM.getCodeGenOpts().CallFEntry)
1183 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_without_opt)
1184 << "-mnop-mcount" << "-mfentry";
1185 Fn->addFnAttr(Kind: "mnop-mcount");
1186 }
1187
1188 if (CGM.getCodeGenOpts().RecordMCount) {
1189 if (!CGM.getCodeGenOpts().CallFEntry)
1190 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_without_opt)
1191 << "-mrecord-mcount" << "-mfentry";
1192 Fn->addFnAttr(Kind: "mrecord-mcount");
1193 }
1194 }
1195 }
1196
1197 if (CGM.getCodeGenOpts().PackedStack) {
1198 if (getContext().getTargetInfo().getTriple().getArch() !=
1199 llvm::Triple::systemz)
1200 CGM.getDiags().Report(DiagID: diag::err_opt_not_valid_on_target)
1201 << "-mpacked-stack";
1202 Fn->addFnAttr(Kind: "packed-stack");
1203 }
1204
1205 if (CGM.getCodeGenOpts().WarnStackSize != UINT_MAX &&
1206 !CGM.getDiags().isIgnored(DiagID: diag::warn_fe_backend_frame_larger_than, Loc))
1207 Fn->addFnAttr(Kind: "warn-stack-size",
1208 Val: std::to_string(val: CGM.getCodeGenOpts().WarnStackSize));
1209
1210 if (RetTy->isVoidType()) {
1211 // Void type; nothing to return.
1212 ReturnValue = Address::invalid();
1213
1214 // Count the implicit return.
1215 if (!endsWithReturn(F: D))
1216 ++NumReturnExprs;
1217 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1218 // Indirect return; emit returned value directly into sret slot.
1219 // This reduces code size, and affects correctness in C++.
1220 auto AI = CurFn->arg_begin();
1221 if (CurFnInfo->getReturnInfo().isSRetAfterThis())
1222 ++AI;
1223 ReturnValue = makeNaturalAddressForPointer(
1224 Ptr: &*AI, T: RetTy, Alignment: CurFnInfo->getReturnInfo().getIndirectAlign(), ForPointeeType: false,
1225 BaseInfo: nullptr, TBAAInfo: nullptr, IsKnownNonNull: KnownNonNull);
1226 if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
1227 ReturnValuePointer =
1228 CreateDefaultAlignTempAlloca(Ty: ReturnValue.getType(), Name: "result.ptr");
1229 Builder.CreateStore(Val: ReturnValue.emitRawPointer(CGF&: *this),
1230 Addr: ReturnValuePointer);
1231 }
1232 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
1233 !hasScalarEvaluationKind(T: CurFnInfo->getReturnType())) {
1234 // Load the sret pointer from the argument struct and return into that.
1235 unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();
1236 llvm::Function::arg_iterator EI = CurFn->arg_end();
1237 --EI;
1238 llvm::Value *Addr = Builder.CreateStructGEP(
1239 Ty: CurFnInfo->getArgStruct(), Ptr: &*EI, Idx);
1240 llvm::Type *Ty =
1241 cast<llvm::GetElementPtrInst>(Val: Addr)->getResultElementType();
1242 ReturnValuePointer = Address(Addr, Ty, getPointerAlign());
1243 Addr = Builder.CreateAlignedLoad(Ty, Addr, Align: getPointerAlign(), Name: "agg.result");
1244 ReturnValue = Address(Addr, ConvertType(T: RetTy),
1245 CGM.getNaturalTypeAlignment(T: RetTy), KnownNonNull);
1246 } else {
1247 ReturnValue = CreateIRTemp(T: RetTy, Name: "retval");
1248
1249 // Tell the epilog emitter to autorelease the result. We do this
1250 // now so that various specialized functions can suppress it
1251 // during their IR-generation.
1252 if (getLangOpts().ObjCAutoRefCount &&
1253 !CurFnInfo->isReturnsRetained() &&
1254 RetTy->isObjCRetainableType())
1255 AutoreleaseResult = true;
1256 }
1257
1258 EmitStartEHSpec(D: CurCodeDecl);
1259
1260 PrologueCleanupDepth = EHStack.stable_begin();
1261
1262 // Emit OpenMP specific initialization of the device functions.
1263 if (getLangOpts().OpenMP && CurCodeDecl)
1264 CGM.getOpenMPRuntime().emitFunctionProlog(CGF&: *this, D: CurCodeDecl);
1265
1266 if (FD && getLangOpts().HLSL) {
1267 // Handle emitting HLSL entry functions.
1268 if (FD->hasAttr<HLSLShaderAttr>()) {
1269 CGM.getHLSLRuntime().emitEntryFunction(FD, Fn);
1270 }
1271 }
1272
1273 EmitFunctionProlog(FI: *CurFnInfo, Fn: CurFn, Args);
1274
1275 if (const CXXMethodDecl *MD = dyn_cast_if_present<CXXMethodDecl>(Val: D);
1276 MD && !MD->isStatic()) {
1277 bool IsInLambda =
1278 MD->getParent()->isLambda() && MD->getOverloadedOperator() == OO_Call;
1279 if (MD->isImplicitObjectMemberFunction())
1280 CGM.getCXXABI().EmitInstanceFunctionProlog(CGF&: *this);
1281 if (IsInLambda) {
1282 // We're in a lambda; figure out the captures.
1283 MD->getParent()->getCaptureFields(Captures&: LambdaCaptureFields,
1284 ThisCapture&: LambdaThisCaptureField);
1285 if (LambdaThisCaptureField) {
1286 // If the lambda captures the object referred to by '*this' - either by
1287 // value or by reference, make sure CXXThisValue points to the correct
1288 // object.
1289
1290 // Get the lvalue for the field (which is a copy of the enclosing object
1291 // or contains the address of the enclosing object).
1292 LValue ThisFieldLValue = EmitLValueForLambdaField(Field: LambdaThisCaptureField);
1293 if (!LambdaThisCaptureField->getType()->isPointerType()) {
1294 // If the enclosing object was captured by value, just use its
1295 // address. Sign this pointer.
1296 CXXThisValue = ThisFieldLValue.getPointer(CGF&: *this);
1297 } else {
1298 // Load the lvalue pointed to by the field, since '*this' was captured
1299 // by reference.
1300 CXXThisValue =
1301 EmitLoadOfLValue(V: ThisFieldLValue, Loc: SourceLocation()).getScalarVal();
1302 }
1303 }
1304 for (auto *FD : MD->getParent()->fields()) {
1305 if (FD->hasCapturedVLAType()) {
1306 auto *ExprArg = EmitLoadOfLValue(V: EmitLValueForLambdaField(Field: FD),
1307 Loc: SourceLocation()).getScalarVal();
1308 auto VAT = FD->getCapturedVLAType();
1309 VLASizeMap[VAT->getSizeExpr()] = ExprArg;
1310 }
1311 }
1312 } else if (MD->isImplicitObjectMemberFunction()) {
1313 // Not in a lambda; just use 'this' from the method.
1314 // FIXME: Should we generate a new load for each use of 'this'? The
1315 // fast register allocator would be happier...
1316 CXXThisValue = CXXABIThisValue;
1317 }
1318
1319 // Check the 'this' pointer once per function, if it's available.
1320 if (CXXABIThisValue) {
1321 SanitizerSet SkippedChecks;
1322 SkippedChecks.set(K: SanitizerKind::ObjectSize, Value: true);
1323 QualType ThisTy = MD->getThisType();
1324
1325 // If this is the call operator of a lambda with no captures, it
1326 // may have a static invoker function, which may call this operator with
1327 // a null 'this' pointer.
1328 if (isLambdaCallOperator(MD) && MD->getParent()->isCapturelessLambda())
1329 SkippedChecks.set(K: SanitizerKind::Null, Value: true);
1330
1331 EmitTypeCheck(
1332 TCK: isa<CXXConstructorDecl>(Val: MD) ? TCK_ConstructorCall : TCK_MemberCall,
1333 Loc, V: CXXABIThisValue, Type: ThisTy, Alignment: CXXABIThisAlignment, SkippedChecks);
1334 }
1335 }
1336
1337 // If any of the arguments have a variably modified type, make sure to
1338 // emit the type size, but only if the function is not naked. Naked functions
1339 // have no prolog to run this evaluation.
1340 if (!FD || !FD->hasAttr<NakedAttr>()) {
1341 for (const VarDecl *VD : Args) {
1342 // Dig out the type as written from ParmVarDecls; it's unclear whether
1343 // the standard (C99 6.9.1p10) requires this, but we're following the
1344 // precedent set by gcc.
1345 QualType Ty;
1346 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Val: VD))
1347 Ty = PVD->getOriginalType();
1348 else
1349 Ty = VD->getType();
1350
1351 if (Ty->isVariablyModifiedType())
1352 EmitVariablyModifiedType(Ty);
1353 }
1354 }
1355 // Emit a location at the end of the prologue.
1356 if (CGDebugInfo *DI = getDebugInfo())
1357 DI->EmitLocation(Builder, Loc: StartLoc);
1358 // TODO: Do we need to handle this in two places like we do with
1359 // target-features/target-cpu?
1360 if (CurFuncDecl)
1361 if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>())
1362 LargestVectorWidth = VecWidth->getVectorWidth();
1363
1364 if (CGM.shouldEmitConvergenceTokens())
1365 ConvergenceTokenStack.push_back(Elt: getOrEmitConvergenceEntryToken(F: CurFn));
1366}
1367
1368void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {
1369 incrementProfileCounter(S: Body);
1370 maybeCreateMCDCCondBitmap();
1371 if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Val: Body))
1372 EmitCompoundStmtWithoutScope(S: *S);
1373 else
1374 EmitStmt(S: Body);
1375}
1376
1377/// When instrumenting to collect profile data, the counts for some blocks
1378/// such as switch cases need to not include the fall-through counts, so
1379/// emit a branch around the instrumentation code. When not instrumenting,
1380/// this just calls EmitBlock().
1381void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
1382 const Stmt *S) {
1383 llvm::BasicBlock *SkipCountBB = nullptr;
1384 if (HaveInsertPoint() && CGM.getCodeGenOpts().hasProfileClangInstr()) {
1385 // When instrumenting for profiling, the fallthrough to certain
1386 // statements needs to skip over the instrumentation code so that we
1387 // get an accurate count.
1388 SkipCountBB = createBasicBlock(name: "skipcount");
1389 EmitBranch(Block: SkipCountBB);
1390 }
1391 EmitBlock(BB);
1392 uint64_t CurrentCount = getCurrentProfileCount();
1393 incrementProfileCounter(ExecSkip: UseExecPath, S);
1394 setCurrentProfileCount(getCurrentProfileCount() + CurrentCount);
1395 if (SkipCountBB)
1396 EmitBlock(BB: SkipCountBB);
1397}
1398
1399/// Tries to mark the given function nounwind based on the
1400/// non-existence of any throwing calls within it. We believe this is
1401/// lightweight enough to do at -O0.
1402static void TryMarkNoThrow(llvm::Function *F) {
1403 // LLVM treats 'nounwind' on a function as part of the type, so we
1404 // can't do this on functions that can be overwritten.
1405 if (F->isInterposable()) return;
1406
1407 for (llvm::BasicBlock &BB : *F)
1408 for (llvm::Instruction &I : BB)
1409 if (I.mayThrow())
1410 return;
1411
1412 F->setDoesNotThrow();
1413}
1414
1415QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD,
1416 FunctionArgList &Args) {
1417 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
1418 QualType ResTy = FD->getReturnType();
1419
1420 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Val: FD);
1421 if (MD && MD->isImplicitObjectMemberFunction()) {
1422 if (CGM.getCXXABI().HasThisReturn(GD))
1423 ResTy = MD->getThisType();
1424 else if (CGM.getCXXABI().hasMostDerivedReturn(GD))
1425 ResTy = CGM.getContext().VoidPtrTy;
1426 CGM.getCXXABI().buildThisParam(CGF&: *this, Params&: Args);
1427 }
1428
1429 // The base version of an inheriting constructor whose constructed base is a
1430 // virtual base is not passed any arguments (because it doesn't actually call
1431 // the inherited constructor).
1432 bool PassedParams = true;
1433 if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(Val: FD))
1434 if (auto Inherited = CD->getInheritedConstructor())
1435 PassedParams =
1436 getTypes().inheritingCtorHasParams(Inherited, Type: GD.getCtorType());
1437
1438 if (PassedParams) {
1439 for (auto *Param : FD->parameters()) {
1440 Args.push_back(Elt: Param);
1441 if (!Param->hasAttr<PassObjectSizeAttr>())
1442 continue;
1443
1444 auto *Implicit = ImplicitParamDecl::Create(
1445 C&: getContext(), DC: Param->getDeclContext(), IdLoc: Param->getLocation(),
1446 /*Id=*/nullptr, T: getContext().getSizeType(), ParamKind: ImplicitParamKind::Other);
1447 SizeArguments[Param] = Implicit;
1448 Args.push_back(Elt: Implicit);
1449 }
1450 }
1451
1452 if (MD && (isa<CXXConstructorDecl>(Val: MD) || isa<CXXDestructorDecl>(Val: MD)))
1453 CGM.getCXXABI().addImplicitStructorParams(CGF&: *this, ResTy, Params&: Args);
1454
1455 return ResTy;
1456}
1457
1458void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
1459 const CGFunctionInfo &FnInfo) {
1460 assert(Fn && "generating code for null Function");
1461 const FunctionDecl *FD = cast<FunctionDecl>(Val: GD.getDecl());
1462 CurGD = GD;
1463
1464 FunctionArgList Args;
1465 QualType ResTy = BuildFunctionArgList(GD, Args);
1466
1467 CGM.getTargetCodeGenInfo().checkFunctionABI(CGM, Decl: FD);
1468
1469 if (FD->isInlineBuiltinDeclaration()) {
1470 // When generating code for a builtin with an inline declaration, use a
1471 // mangled name to hold the actual body, while keeping an external
1472 // definition in case the function pointer is referenced somewhere.
1473 std::string FDInlineName = (Fn->getName() + ".inline").str();
1474 llvm::Module *M = Fn->getParent();
1475 llvm::Function *Clone = M->getFunction(Name: FDInlineName);
1476 if (!Clone) {
1477 Clone = llvm::Function::Create(Ty: Fn->getFunctionType(),
1478 Linkage: llvm::GlobalValue::InternalLinkage,
1479 AddrSpace: Fn->getAddressSpace(), N: FDInlineName, M);
1480 Clone->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1481 }
1482 Fn->setLinkage(llvm::GlobalValue::ExternalLinkage);
1483 Fn = Clone;
1484 } else {
1485 // Detect the unusual situation where an inline version is shadowed by a
1486 // non-inline version. In that case we should pick the external one
1487 // everywhere. That's GCC behavior too. Unfortunately, I cannot find a way
1488 // to detect that situation before we reach codegen, so do some late
1489 // replacement.
1490 for (const FunctionDecl *PD = FD->getPreviousDecl(); PD;
1491 PD = PD->getPreviousDecl()) {
1492 if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) {
1493 std::string FDInlineName = (Fn->getName() + ".inline").str();
1494 llvm::Module *M = Fn->getParent();
1495 if (llvm::Function *Clone = M->getFunction(Name: FDInlineName)) {
1496 Clone->replaceAllUsesWith(V: Fn);
1497 Clone->eraseFromParent();
1498 }
1499 break;
1500 }
1501 }
1502 }
1503
1504 // Check if we should generate debug info for this function.
1505 if (FD->hasAttr<NoDebugAttr>()) {
1506 // Clear non-distinct debug info that was possibly attached to the function
1507 // due to an earlier declaration without the nodebug attribute
1508 Fn->setSubprogram(nullptr);
1509 // Disable debug info indefinitely for this function
1510 DebugInfo = nullptr;
1511 }
1512 // Finalize function debug info on exit.
1513 llvm::scope_exit Cleanup([this] {
1514 if (CGDebugInfo *DI = getDebugInfo())
1515 DI->completeFunction();
1516 });
1517
1518 // The function might not have a body if we're generating thunks for a
1519 // function declaration.
1520 SourceRange BodyRange;
1521 if (Stmt *Body = FD->getBody())
1522 BodyRange = Body->getSourceRange();
1523 else
1524 BodyRange = FD->getLocation();
1525 CurEHLocation = BodyRange.getEnd();
1526
1527 // Use the location of the start of the function to determine where
1528 // the function definition is located. By default use the location
1529 // of the declaration as the location for the subprogram. A function
1530 // may lack a declaration in the source code if it is created by code
1531 // gen. (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).
1532 SourceLocation Loc = FD->getLocation();
1533
1534 // If this is a function specialization then use the pattern body
1535 // as the location for the function.
1536 if (const FunctionDecl *SpecDecl = FD->getTemplateInstantiationPattern())
1537 if (SpecDecl->hasBody(Definition&: SpecDecl))
1538 Loc = SpecDecl->getLocation();
1539
1540 Stmt *Body = FD->getBody();
1541
1542 if (Body) {
1543 // Coroutines always emit lifetime markers.
1544 if (isa<CoroutineBodyStmt>(Val: Body))
1545 ShouldEmitLifetimeMarkers = true;
1546
1547 // Initialize helper which will detect jumps which can cause invalid
1548 // lifetime markers.
1549 if (ShouldEmitLifetimeMarkers)
1550 Bypasses.Init(CGM, Body);
1551 }
1552
1553 // Emit the standard function prologue.
1554 StartFunction(GD, RetTy: ResTy, Fn, FnInfo, Args, Loc, StartLoc: BodyRange.getBegin());
1555
1556 // Save parameters for coroutine function.
1557 if (Body && isa_and_nonnull<CoroutineBodyStmt>(Val: Body))
1558 llvm::append_range(C&: FnArgs, R: FD->parameters());
1559
1560 // Ensure that the function adheres to the forward progress guarantee, which
1561 // is required by certain optimizations.
1562 // In C++11 and up, the attribute will be removed if the body contains a
1563 // trivial empty loop.
1564 if (checkIfFunctionMustProgress())
1565 CurFn->addFnAttr(Kind: llvm::Attribute::MustProgress);
1566
1567 // Generate the body of the function.
1568 PGO->assignRegionCounters(GD, Fn: CurFn);
1569 if (isa<CXXDestructorDecl>(Val: FD))
1570 EmitDestructorBody(Args);
1571 else if (isa<CXXConstructorDecl>(Val: FD))
1572 EmitConstructorBody(Args);
1573 else if (getLangOpts().CUDA &&
1574 !getLangOpts().CUDAIsDevice &&
1575 FD->hasAttr<CUDAGlobalAttr>())
1576 CGM.getCUDARuntime().emitDeviceStub(CGF&: *this, Args);
1577 else if (isa<CXXMethodDecl>(Val: FD) &&
1578 cast<CXXMethodDecl>(Val: FD)->isLambdaStaticInvoker()) {
1579 // The lambda static invoker function is special, because it forwards or
1580 // clones the body of the function call operator (but is actually static).
1581 EmitLambdaStaticInvokeBody(MD: cast<CXXMethodDecl>(Val: FD));
1582 } else if (isa<CXXMethodDecl>(Val: FD) &&
1583 isLambdaCallOperator(MD: cast<CXXMethodDecl>(Val: FD)) &&
1584 !FnInfo.isDelegateCall() &&
1585 cast<CXXMethodDecl>(Val: FD)->getParent()->getLambdaStaticInvoker() &&
1586 hasInAllocaArg(MD: cast<CXXMethodDecl>(Val: FD))) {
1587 // If emitting a lambda with static invoker on X86 Windows, change
1588 // the call operator body.
1589 // Make sure that this is a call operator with an inalloca arg and check
1590 // for delegate call to make sure this is the original call op and not the
1591 // new forwarding function for the static invoker.
1592 EmitLambdaInAllocaCallOpBody(MD: cast<CXXMethodDecl>(Val: FD));
1593 } else if (FD->isDefaulted() && isa<CXXMethodDecl>(Val: FD) &&
1594 (cast<CXXMethodDecl>(Val: FD)->isCopyAssignmentOperator() ||
1595 cast<CXXMethodDecl>(Val: FD)->isMoveAssignmentOperator())) {
1596 // Implicit copy-assignment gets the same special treatment as implicit
1597 // copy-constructors.
1598 emitImplicitAssignmentOperatorBody(Args);
1599 } else if (DeviceKernelAttr::isOpenCLSpelling(
1600 A: FD->getAttr<DeviceKernelAttr>()) &&
1601 GD.getKernelReferenceKind() == KernelReferenceKind::Kernel) {
1602 CallArgList CallArgs;
1603 for (unsigned i = 0; i < Args.size(); ++i) {
1604 Address ArgAddr = GetAddrOfLocalVar(VD: Args[i]);
1605 QualType ArgQualType = Args[i]->getType();
1606 RValue ArgRValue = convertTempToRValue(addr: ArgAddr, type: ArgQualType, Loc);
1607 CallArgs.add(rvalue: ArgRValue, type: ArgQualType);
1608 }
1609 GlobalDecl GDStub = GlobalDecl(FD, KernelReferenceKind::Stub);
1610 const FunctionType *FT = cast<FunctionType>(Val: FD->getType());
1611 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);
1612 const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
1613 Args: CallArgs, Ty: FT, /*ChainCall=*/false);
1614 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
1615 llvm::Constant *GDStubFunctionPointer =
1616 CGM.getRawFunctionPointer(GD: GDStub, Ty: FTy);
1617 CGCallee GDStubCallee = CGCallee::forDirect(functionPtr: GDStubFunctionPointer, abstractInfo: GDStub);
1618 EmitCall(CallInfo: FnInfo, Callee: GDStubCallee, ReturnValue: ReturnValueSlot(), Args: CallArgs, CallOrInvoke: nullptr, IsMustTail: false,
1619 Loc);
1620 } else if (Body) {
1621 EmitFunctionBody(Body);
1622 } else
1623 llvm_unreachable("no definition for emitted function");
1624
1625 // C++11 [stmt.return]p2:
1626 // Flowing off the end of a function [...] results in undefined behavior in
1627 // a value-returning function.
1628 // C11 6.9.1p12:
1629 // If the '}' that terminates a function is reached, and the value of the
1630 // function call is used by the caller, the behavior is undefined.
1631 if (getLangOpts().CPlusPlus && !FD->hasImplicitReturnZero() && !SawAsmBlock &&
1632 !FD->getReturnType()->isVoidType() && Builder.GetInsertBlock()) {
1633 bool ShouldEmitUnreachable =
1634 CGM.getCodeGenOpts().StrictReturn ||
1635 !CGM.MayDropFunctionReturn(Context: FD->getASTContext(), ReturnType: FD->getReturnType());
1636 if (SanOpts.has(K: SanitizerKind::Return)) {
1637 auto CheckOrdinal = SanitizerKind::SO_Return;
1638 auto CheckHandler = SanitizerHandler::MissingReturn;
1639 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
1640 llvm::Value *IsFalse = Builder.getFalse();
1641 EmitCheck(Checked: std::make_pair(x&: IsFalse, y&: CheckOrdinal), Check: CheckHandler,
1642 StaticArgs: EmitCheckSourceLocation(Loc: FD->getLocation()), DynamicArgs: {});
1643 } else if (ShouldEmitUnreachable) {
1644 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1645 EmitTrapCall(IntrID: llvm::Intrinsic::trap);
1646 }
1647 if (SanOpts.has(K: SanitizerKind::Return) || ShouldEmitUnreachable) {
1648 Builder.CreateUnreachable();
1649 Builder.ClearInsertionPoint();
1650 }
1651 }
1652
1653 // Emit the standard function epilogue.
1654 FinishFunction(EndLoc: BodyRange.getEnd());
1655
1656 PGO->verifyCounterMap();
1657
1658 // If we haven't marked the function nothrow through other means, do
1659 // a quick pass now to see if we can.
1660 if (!CurFn->doesNotThrow())
1661 TryMarkNoThrow(F: CurFn);
1662}
1663
1664/// ContainsLabel - Return true if the statement contains a label in it. If
1665/// this statement is not executed normally, it not containing a label means
1666/// that we can just remove the code.
1667bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
1668 // Null statement, not a label!
1669 if (!S) return false;
1670
1671 // If this is a label, we have to emit the code, consider something like:
1672 // if (0) { ... foo: bar(); } goto foo;
1673 //
1674 // TODO: If anyone cared, we could track __label__'s, since we know that you
1675 // can't jump to one from outside their declared region.
1676 if (isa<LabelStmt>(Val: S))
1677 return true;
1678
1679 // If this is a case/default statement, and we haven't seen a switch, we have
1680 // to emit the code.
1681 if (isa<SwitchCase>(Val: S) && !IgnoreCaseStmts)
1682 return true;
1683
1684 // If this is a switch statement, we want to ignore cases below it.
1685 if (isa<SwitchStmt>(Val: S))
1686 IgnoreCaseStmts = true;
1687
1688 // Scan subexpressions for verboten labels.
1689 for (const Stmt *SubStmt : S->children())
1690 if (ContainsLabel(S: SubStmt, IgnoreCaseStmts))
1691 return true;
1692
1693 return false;
1694}
1695
1696/// containsBreak - Return true if the statement contains a break out of it.
1697/// If the statement (recursively) contains a switch or loop with a break
1698/// inside of it, this is fine.
1699bool CodeGenFunction::containsBreak(const Stmt *S) {
1700 // Null statement, not a label!
1701 if (!S) return false;
1702
1703 // If this is a switch or loop that defines its own break scope, then we can
1704 // include it and anything inside of it.
1705 if (isa<SwitchStmt>(Val: S) || isa<WhileStmt>(Val: S) || isa<DoStmt>(Val: S) ||
1706 isa<ForStmt>(Val: S))
1707 return false;
1708
1709 if (isa<BreakStmt>(Val: S))
1710 return true;
1711
1712 // Scan subexpressions for verboten breaks.
1713 for (const Stmt *SubStmt : S->children())
1714 if (containsBreak(S: SubStmt))
1715 return true;
1716
1717 return false;
1718}
1719
1720bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) {
1721 if (!S) return false;
1722
1723 // Some statement kinds add a scope and thus never add a decl to the current
1724 // scope. Note, this list is longer than the list of statements that might
1725 // have an unscoped decl nested within them, but this way is conservatively
1726 // correct even if more statement kinds are added.
1727 if (isa<IfStmt>(Val: S) || isa<SwitchStmt>(Val: S) || isa<WhileStmt>(Val: S) ||
1728 isa<DoStmt>(Val: S) || isa<ForStmt>(Val: S) || isa<CompoundStmt>(Val: S) ||
1729 isa<CXXForRangeStmt>(Val: S) || isa<CXXTryStmt>(Val: S) ||
1730 isa<ObjCForCollectionStmt>(Val: S) || isa<ObjCAtTryStmt>(Val: S))
1731 return false;
1732
1733 if (isa<DeclStmt>(Val: S))
1734 return true;
1735
1736 for (const Stmt *SubStmt : S->children())
1737 if (mightAddDeclToScope(S: SubStmt))
1738 return true;
1739
1740 return false;
1741}
1742
1743/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1744/// to a constant, or if it does but contains a label, return false. If it
1745/// constant folds return true and set the boolean result in Result.
1746bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1747 bool &ResultBool,
1748 bool AllowLabels) {
1749 // If MC/DC is enabled, disable folding so that we can instrument all
1750 // conditions to yield complete test vectors. We still keep track of
1751 // folded conditions during region mapping and visualization.
1752 if (!AllowLabels && CGM.getCodeGenOpts().hasProfileClangInstr() &&
1753 CGM.getCodeGenOpts().MCDCCoverage)
1754 return false;
1755
1756 llvm::APSInt ResultInt;
1757 if (!ConstantFoldsToSimpleInteger(Cond, Result&: ResultInt, AllowLabels))
1758 return false;
1759
1760 ResultBool = ResultInt.getBoolValue();
1761 return true;
1762}
1763
1764/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1765/// to a constant, or if it does but contains a label, return false. If it
1766/// constant folds return true and set the folded value.
1767bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1768 llvm::APSInt &ResultInt,
1769 bool AllowLabels) {
1770 // FIXME: Rename and handle conversion of other evaluatable things
1771 // to bool.
1772 Expr::EvalResult Result;
1773 if (!Cond->EvaluateAsInt(Result, Ctx: getContext()))
1774 return false; // Not foldable, not integer or not fully evaluatable.
1775
1776 llvm::APSInt Int = Result.Val.getInt();
1777 if (!AllowLabels && CodeGenFunction::ContainsLabel(S: Cond))
1778 return false; // Contains a label.
1779
1780 PGO->markStmtMaybeUsed(S: Cond);
1781 ResultInt = Int;
1782 return true;
1783}
1784
1785/// Strip parentheses and simplistic logical-NOT operators.
1786const Expr *CodeGenFunction::stripCond(const Expr *C) {
1787 while (true) {
1788 const Expr *SC = IgnoreExprNodes(
1789 E: C, Fns&: IgnoreParensSingleStep, Fns&: IgnoreUOpLNotSingleStep,
1790 Fns&: IgnoreBuiltinExpectSingleStep, Fns&: IgnoreImplicitCastsSingleStep);
1791 if (C == SC)
1792 return SC;
1793 C = SC;
1794 }
1795}
1796
1797/// Determine whether the given condition is an instrumentable condition
1798/// (i.e. no "&&" or "||").
1799bool CodeGenFunction::isInstrumentedCondition(const Expr *C) {
1800 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: stripCond(C));
1801 return (!BOp || !BOp->isLogicalOp());
1802}
1803
1804/// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
1805/// increments a profile counter based on the semantics of the given logical
1806/// operator opcode. This is used to instrument branch condition coverage for
1807/// logical operators.
1808void CodeGenFunction::EmitBranchToCounterBlock(
1809 const Expr *Cond, BinaryOperator::Opcode LOp, llvm::BasicBlock *TrueBlock,
1810 llvm::BasicBlock *FalseBlock, uint64_t TrueCount /* = 0 */,
1811 Stmt::Likelihood LH /* =None */, const Expr *CntrIdx /* = nullptr */) {
1812 // If not instrumenting, just emit a branch.
1813 bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr();
1814 if (!InstrumentRegions || !isInstrumentedCondition(C: Cond))
1815 return EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount, LH);
1816
1817 const Stmt *CntrStmt = (CntrIdx ? CntrIdx : Cond);
1818
1819 llvm::BasicBlock *ThenBlock = nullptr;
1820 llvm::BasicBlock *ElseBlock = nullptr;
1821 llvm::BasicBlock *NextBlock = nullptr;
1822
1823 // Create the block we'll use to increment the appropriate counter.
1824 llvm::BasicBlock *CounterIncrBlock = createBasicBlock(name: "lop.rhscnt");
1825
1826 llvm::BasicBlock *SkipIncrBlock =
1827 (hasSkipCounter(S: CntrStmt) ? createBasicBlock(name: "lop.rhsskip") : nullptr);
1828 llvm::BasicBlock *SkipNextBlock = nullptr;
1829
1830 // Set block pointers according to Logical-AND (BO_LAnd) semantics. This
1831 // means we need to evaluate the condition and increment the counter on TRUE:
1832 //
1833 // if (Cond)
1834 // goto CounterIncrBlock;
1835 // else
1836 // goto FalseBlock;
1837 //
1838 // CounterIncrBlock:
1839 // Counter++;
1840 // goto TrueBlock;
1841
1842 if (LOp == BO_LAnd) {
1843 SkipNextBlock = FalseBlock;
1844 ThenBlock = CounterIncrBlock;
1845 ElseBlock = (SkipIncrBlock ? SkipIncrBlock : SkipNextBlock);
1846 NextBlock = TrueBlock;
1847 }
1848
1849 // Set block pointers according to Logical-OR (BO_LOr) semantics. This means
1850 // we need to evaluate the condition and increment the counter on FALSE:
1851 //
1852 // if (Cond)
1853 // goto TrueBlock;
1854 // else
1855 // goto CounterIncrBlock;
1856 //
1857 // CounterIncrBlock:
1858 // Counter++;
1859 // goto FalseBlock;
1860
1861 else if (LOp == BO_LOr) {
1862 SkipNextBlock = TrueBlock;
1863 ThenBlock = (SkipIncrBlock ? SkipIncrBlock : SkipNextBlock);
1864 ElseBlock = CounterIncrBlock;
1865 NextBlock = FalseBlock;
1866 } else {
1867 llvm_unreachable("Expected Opcode must be that of a Logical Operator");
1868 }
1869
1870 // Emit Branch based on condition.
1871 EmitBranchOnBoolExpr(Cond, TrueBlock: ThenBlock, FalseBlock: ElseBlock, TrueCount, LH);
1872
1873 if (SkipIncrBlock) {
1874 EmitBlock(BB: SkipIncrBlock);
1875 incrementProfileCounter(ExecSkip: UseSkipPath, S: CntrStmt);
1876 EmitBranch(Block: SkipNextBlock);
1877 }
1878
1879 // Emit the block containing the counter increment(s).
1880 EmitBlock(BB: CounterIncrBlock);
1881
1882 // Increment corresponding counter; if index not provided, use Cond as index.
1883 incrementProfileCounter(ExecSkip: UseExecPath, S: CntrStmt);
1884
1885 // Go to the next block.
1886 EmitBranch(Block: NextBlock);
1887}
1888
1889/// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an if
1890/// statement) to the specified blocks. Based on the condition, this might try
1891/// to simplify the codegen of the conditional based on the branch.
1892/// \param LH The value of the likelihood attribute on the True branch.
1893/// \param ConditionalOp Used by MC/DC code coverage to track the result of the
1894/// ConditionalOperator (ternary) through a recursive call for the operator's
1895/// LHS and RHS nodes.
1896void CodeGenFunction::EmitBranchOnBoolExpr(
1897 const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock,
1898 uint64_t TrueCount, Stmt::Likelihood LH, const Expr *ConditionalOp,
1899 const VarDecl *ConditionalDecl) {
1900 Cond = Cond->IgnoreParens();
1901
1902 if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Val: Cond)) {
1903 bool HasSkip = hasSkipCounter(S: CondBOp);
1904
1905 // Handle X && Y in a condition.
1906 if (CondBOp->getOpcode() == BO_LAnd) {
1907 // If we have "1 && X", simplify the code. "0 && X" would have constant
1908 // folded if the case was simple enough.
1909 bool ConstantBool = false;
1910 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getLHS(), ResultBool&: ConstantBool) &&
1911 ConstantBool) {
1912 // br(1 && X) -> br(X).
1913 incrementProfileCounter(S: CondBOp);
1914 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LAnd, TrueBlock,
1915 FalseBlock, TrueCount, LH);
1916 return;
1917 }
1918
1919 // If we have "X && 1", simplify the code to use an uncond branch.
1920 // "X && 0" would have been constant folded to 0.
1921 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getRHS(), ResultBool&: ConstantBool) &&
1922 ConstantBool) {
1923 // br(X && 1) -> br(X).
1924 EmitBranchToCounterBlock(Cond: CondBOp->getLHS(), LOp: BO_LAnd, TrueBlock,
1925 FalseBlock, TrueCount, LH, CntrIdx: CondBOp);
1926 return;
1927 }
1928
1929 // Emit the LHS as a conditional. If the LHS conditional is false, we
1930 // want to jump to the FalseBlock.
1931 llvm::BasicBlock *LHSTrue = createBasicBlock(name: "land.lhs.true");
1932 llvm::BasicBlock *LHSFalse =
1933 (HasSkip ? createBasicBlock(name: "land.lhsskip") : FalseBlock);
1934 // The counter tells us how often we evaluate RHS, and all of TrueCount
1935 // can be propagated to that branch.
1936 uint64_t RHSCount = getProfileCount(S: CondBOp->getRHS());
1937
1938 ConditionalEvaluation eval(*this);
1939 {
1940 ApplyDebugLocation DL(*this, Cond);
1941 // Propagate the likelihood attribute like __builtin_expect
1942 // __builtin_expect(X && Y, 1) -> X and Y are likely
1943 // __builtin_expect(X && Y, 0) -> only Y is unlikely
1944 EmitBranchOnBoolExpr(Cond: CondBOp->getLHS(), TrueBlock: LHSTrue, FalseBlock: LHSFalse, TrueCount: RHSCount,
1945 LH: LH == Stmt::LH_Unlikely ? Stmt::LH_None : LH);
1946 if (HasSkip) {
1947 EmitBlock(BB: LHSFalse);
1948 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondBOp);
1949 EmitBranch(Block: FalseBlock);
1950 }
1951 EmitBlock(BB: LHSTrue);
1952 }
1953
1954 incrementProfileCounter(ExecSkip: UseExecPath, S: CondBOp);
1955 setCurrentProfileCount(getProfileCount(S: CondBOp->getRHS()));
1956
1957 // Any temporaries created here are conditional.
1958 eval.begin(CGF&: *this);
1959 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LAnd, TrueBlock,
1960 FalseBlock, TrueCount, LH);
1961 eval.end(CGF&: *this);
1962 return;
1963 }
1964
1965 if (CondBOp->getOpcode() == BO_LOr) {
1966 // If we have "0 || X", simplify the code. "1 || X" would have constant
1967 // folded if the case was simple enough.
1968 bool ConstantBool = false;
1969 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getLHS(), ResultBool&: ConstantBool) &&
1970 !ConstantBool) {
1971 // br(0 || X) -> br(X).
1972 incrementProfileCounter(S: CondBOp);
1973 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LOr, TrueBlock,
1974 FalseBlock, TrueCount, LH);
1975 return;
1976 }
1977
1978 // If we have "X || 0", simplify the code to use an uncond branch.
1979 // "X || 1" would have been constant folded to 1.
1980 if (ConstantFoldsToSimpleInteger(Cond: CondBOp->getRHS(), ResultBool&: ConstantBool) &&
1981 !ConstantBool) {
1982 // br(X || 0) -> br(X).
1983 EmitBranchToCounterBlock(Cond: CondBOp->getLHS(), LOp: BO_LOr, TrueBlock,
1984 FalseBlock, TrueCount, LH, CntrIdx: CondBOp);
1985 return;
1986 }
1987 // Emit the LHS as a conditional. If the LHS conditional is true, we
1988 // want to jump to the TrueBlock.
1989 llvm::BasicBlock *LHSTrue =
1990 (HasSkip ? createBasicBlock(name: "lor.lhsskip") : TrueBlock);
1991 llvm::BasicBlock *LHSFalse = createBasicBlock(name: "lor.lhs.false");
1992 // We have the count for entry to the RHS and for the whole expression
1993 // being true, so we can divy up True count between the short circuit and
1994 // the RHS.
1995 uint64_t LHSCount =
1996 getCurrentProfileCount() - getProfileCount(S: CondBOp->getRHS());
1997 uint64_t RHSCount = TrueCount - LHSCount;
1998
1999 ConditionalEvaluation eval(*this);
2000 {
2001 // Propagate the likelihood attribute like __builtin_expect
2002 // __builtin_expect(X || Y, 1) -> only Y is likely
2003 // __builtin_expect(X || Y, 0) -> both X and Y are unlikely
2004 ApplyDebugLocation DL(*this, Cond);
2005 EmitBranchOnBoolExpr(Cond: CondBOp->getLHS(), TrueBlock: LHSTrue, FalseBlock: LHSFalse, TrueCount: LHSCount,
2006 LH: LH == Stmt::LH_Likely ? Stmt::LH_None : LH);
2007 if (HasSkip) {
2008 EmitBlock(BB: LHSTrue);
2009 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondBOp);
2010 EmitBranch(Block: TrueBlock);
2011 }
2012 EmitBlock(BB: LHSFalse);
2013 }
2014
2015 incrementProfileCounter(ExecSkip: UseExecPath, S: CondBOp);
2016 setCurrentProfileCount(getProfileCount(S: CondBOp->getRHS()));
2017
2018 // Any temporaries created here are conditional.
2019 eval.begin(CGF&: *this);
2020 EmitBranchToCounterBlock(Cond: CondBOp->getRHS(), LOp: BO_LOr, TrueBlock, FalseBlock,
2021 TrueCount: RHSCount, LH);
2022
2023 eval.end(CGF&: *this);
2024 return;
2025 }
2026 }
2027
2028 if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Val: Cond)) {
2029 // br(!x, t, f) -> br(x, f, t)
2030 // Avoid doing this optimization when instrumenting a condition for MC/DC.
2031 // LNot is taken as part of the condition for simplicity, and changing its
2032 // sense negatively impacts test vector tracking.
2033 bool MCDCCondition = CGM.getCodeGenOpts().hasProfileClangInstr() &&
2034 CGM.getCodeGenOpts().MCDCCoverage &&
2035 isInstrumentedCondition(C: Cond);
2036 if (CondUOp->getOpcode() == UO_LNot && !MCDCCondition) {
2037 // Negate the count.
2038 uint64_t FalseCount = getCurrentProfileCount() - TrueCount;
2039 // The values of the enum are chosen to make this negation possible.
2040 LH = static_cast<Stmt::Likelihood>(-LH);
2041 // Negate the condition and swap the destination blocks.
2042 return EmitBranchOnBoolExpr(Cond: CondUOp->getSubExpr(), TrueBlock: FalseBlock, FalseBlock: TrueBlock,
2043 TrueCount: FalseCount, LH);
2044 }
2045 }
2046
2047 if (const ConditionalOperator *CondOp = dyn_cast<ConditionalOperator>(Val: Cond)) {
2048 // br(c ? x : y, t, f) -> br(c, br(x, t, f), br(y, t, f))
2049 llvm::BasicBlock *LHSBlock = createBasicBlock(name: "cond.true");
2050 llvm::BasicBlock *RHSBlock = createBasicBlock(name: "cond.false");
2051
2052 // The ConditionalOperator itself has no likelihood information for its
2053 // true and false branches. This matches the behavior of __builtin_expect.
2054 ConditionalEvaluation cond(*this);
2055 EmitBranchOnBoolExpr(Cond: CondOp->getCond(), TrueBlock: LHSBlock, FalseBlock: RHSBlock,
2056 TrueCount: getProfileCount(S: CondOp), LH: Stmt::LH_None);
2057
2058 // When computing PGO branch weights, we only know the overall count for
2059 // the true block. This code is essentially doing tail duplication of the
2060 // naive code-gen, introducing new edges for which counts are not
2061 // available. Divide the counts proportionally between the LHS and RHS of
2062 // the conditional operator.
2063 uint64_t LHSScaledTrueCount = 0;
2064 if (TrueCount) {
2065 double LHSRatio =
2066 getProfileCount(S: CondOp) / (double)getCurrentProfileCount();
2067 LHSScaledTrueCount = TrueCount * LHSRatio;
2068 }
2069
2070 cond.begin(CGF&: *this);
2071 EmitBlock(BB: LHSBlock);
2072 incrementProfileCounter(ExecSkip: UseExecPath, S: CondOp);
2073 {
2074 ApplyDebugLocation DL(*this, Cond);
2075 EmitBranchOnBoolExpr(Cond: CondOp->getLHS(), TrueBlock, FalseBlock,
2076 TrueCount: LHSScaledTrueCount, LH, ConditionalOp: CondOp);
2077 }
2078 cond.end(CGF&: *this);
2079
2080 cond.begin(CGF&: *this);
2081 EmitBlock(BB: RHSBlock);
2082 incrementProfileCounter(ExecSkip: UseSkipPath, S: CondOp);
2083 EmitBranchOnBoolExpr(Cond: CondOp->getRHS(), TrueBlock, FalseBlock,
2084 TrueCount: TrueCount - LHSScaledTrueCount, LH, ConditionalOp: CondOp);
2085 cond.end(CGF&: *this);
2086
2087 return;
2088 }
2089
2090 if (const CXXThrowExpr *Throw = dyn_cast<CXXThrowExpr>(Val: Cond)) {
2091 // Conditional operator handling can give us a throw expression as a
2092 // condition for a case like:
2093 // br(c ? throw x : y, t, f) -> br(c, br(throw x, t, f), br(y, t, f)
2094 // Fold this to:
2095 // br(c, throw x, br(y, t, f))
2096 EmitCXXThrowExpr(E: Throw, /*KeepInsertionPoint*/false);
2097 return;
2098 }
2099
2100 // Emit the code with the fully general case.
2101 llvm::Value *CondV;
2102 {
2103 ApplyDebugLocation DL(*this, Cond);
2104 CondV = EvaluateExprAsBool(E: Cond);
2105 }
2106
2107 MaybeEmitDeferredVarDeclInit(var: ConditionalDecl);
2108
2109 // If not at the top of the logical operator nest, update MCDC temp with the
2110 // boolean result of the evaluated condition.
2111 {
2112 const Expr *MCDCBaseExpr = Cond;
2113 // When a nested ConditionalOperator (ternary) is encountered in a boolean
2114 // expression, MC/DC tracks the result of the ternary, and this is tied to
2115 // the ConditionalOperator expression and not the ternary's LHS or RHS. If
2116 // this is the case, the ConditionalOperator expression is passed through
2117 // the ConditionalOp parameter and then used as the MCDC base expression.
2118 if (ConditionalOp)
2119 MCDCBaseExpr = ConditionalOp;
2120
2121 if (isMCDCBranchExpr(E: stripCond(C: MCDCBaseExpr)) &&
2122 !isMCDCDecisionExpr(E: stripCond(C: Cond)))
2123 maybeUpdateMCDCCondBitmap(E: MCDCBaseExpr, Val: CondV);
2124 }
2125
2126 llvm::MDNode *Weights = nullptr;
2127 llvm::MDNode *Unpredictable = nullptr;
2128
2129 // If the branch has a condition wrapped by __builtin_unpredictable,
2130 // create metadata that specifies that the branch is unpredictable.
2131 // Don't bother if not optimizing because that metadata would not be used.
2132 auto *Call = dyn_cast<CallExpr>(Val: Cond->IgnoreImpCasts());
2133 if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
2134 auto *FD = dyn_cast_or_null<FunctionDecl>(Val: Call->getCalleeDecl());
2135 if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
2136 llvm::MDBuilder MDHelper(getLLVMContext());
2137 Unpredictable = MDHelper.createUnpredictable();
2138 }
2139 }
2140
2141 // If there is a Likelihood knowledge for the cond, lower it.
2142 // Note that if not optimizing this won't emit anything.
2143 llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(Cond: CondV, LH);
2144 if (CondV != NewCondV)
2145 CondV = NewCondV;
2146 else {
2147 // Otherwise, lower profile counts. Note that we do this even at -O0.
2148 uint64_t CurrentCount = std::max(a: getCurrentProfileCount(), b: TrueCount);
2149 Weights = createProfileWeights(TrueCount, FalseCount: CurrentCount - TrueCount);
2150 }
2151
2152 llvm::Instruction *BrInst = Builder.CreateCondBr(Cond: CondV, True: TrueBlock, False: FalseBlock,
2153 BranchWeights: Weights, Unpredictable);
2154 addInstToNewSourceAtom(KeyInstruction: BrInst, Backup: CondV);
2155
2156 switch (HLSLControlFlowAttr) {
2157 case HLSLControlFlowHintAttr::Microsoft_branch:
2158 case HLSLControlFlowHintAttr::Microsoft_flatten: {
2159 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2160
2161 llvm::ConstantInt *BranchHintConstant =
2162 HLSLControlFlowAttr ==
2163 HLSLControlFlowHintAttr::Spelling::Microsoft_branch
2164 ? llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 1)
2165 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 2);
2166
2167 SmallVector<llvm::Metadata *, 2> Vals(
2168 {MDHelper.createString(Str: "hlsl.controlflow.hint"),
2169 MDHelper.createConstant(C: BranchHintConstant)});
2170 BrInst->setMetadata(Kind: "hlsl.controlflow.hint",
2171 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Vals));
2172 break;
2173 }
2174 // This is required to avoid warnings during compilation
2175 case HLSLControlFlowHintAttr::SpellingNotCalculated:
2176 break;
2177 }
2178}
2179
2180llvm::Value *CodeGenFunction::EmitScalarOrConstFoldImmArg(unsigned ICEArguments,
2181 unsigned Idx,
2182 const CallExpr *E) {
2183 llvm::Value *Arg = nullptr;
2184 if ((ICEArguments & (1 << Idx)) == 0) {
2185 Arg = EmitScalarExpr(E: E->getArg(Arg: Idx));
2186 } else {
2187 // If this is required to be a constant, constant fold it so that we
2188 // know that the generated intrinsic gets a ConstantInt.
2189 std::optional<llvm::APSInt> Result =
2190 E->getArg(Arg: Idx)->getIntegerConstantExpr(Ctx: getContext());
2191 assert(Result && "Expected argument to be a constant");
2192 Arg = llvm::ConstantInt::get(Context&: getLLVMContext(), V: *Result);
2193 }
2194 return Arg;
2195}
2196
2197/// ErrorUnsupported - Print out an error that codegen doesn't support the
2198/// specified stmt yet.
2199void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) {
2200 CGM.ErrorUnsupported(S, Type);
2201}
2202
2203/// emitNonZeroVLAInit - Emit the "zero" initialization of a
2204/// variable-length array whose elements have a non-zero bit-pattern.
2205///
2206/// \param baseType the inner-most element type of the array
2207/// \param src - a char* pointing to the bit-pattern for a single
2208/// base element of the array
2209/// \param sizeInChars - the total size of the VLA, in chars
2210static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
2211 Address dest, Address src,
2212 llvm::Value *sizeInChars) {
2213 CGBuilderTy &Builder = CGF.Builder;
2214
2215 CharUnits baseSize = CGF.getContext().getTypeSizeInChars(T: baseType);
2216 llvm::Value *baseSizeInChars
2217 = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: baseSize.getQuantity());
2218
2219 Address begin = dest.withElementType(ElemTy: CGF.Int8Ty);
2220 llvm::Value *end = Builder.CreateInBoundsGEP(Ty: begin.getElementType(),
2221 Ptr: begin.emitRawPointer(CGF),
2222 IdxList: sizeInChars, Name: "vla.end");
2223
2224 llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
2225 llvm::BasicBlock *loopBB = CGF.createBasicBlock(name: "vla-init.loop");
2226 llvm::BasicBlock *contBB = CGF.createBasicBlock(name: "vla-init.cont");
2227
2228 // Make a loop over the VLA. C99 guarantees that the VLA element
2229 // count must be nonzero.
2230 CGF.EmitBlock(BB: loopBB);
2231
2232 llvm::PHINode *cur = Builder.CreatePHI(Ty: begin.getType(), NumReservedValues: 2, Name: "vla.cur");
2233 cur->addIncoming(V: begin.emitRawPointer(CGF), BB: originBB);
2234
2235 CharUnits curAlign =
2236 dest.getAlignment().alignmentOfArrayElement(elementSize: baseSize);
2237
2238 // memcpy the individual element bit-pattern.
2239 Builder.CreateMemCpy(Dest: Address(cur, CGF.Int8Ty, curAlign), Src: src, Size: baseSizeInChars,
2240 /*volatile*/ IsVolatile: false);
2241
2242 // Go to the next element.
2243 llvm::Value *next =
2244 Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: cur, IdxList: baseSizeInChars, Name: "vla.next");
2245
2246 // Leave if that's the end of the VLA.
2247 llvm::Value *done = Builder.CreateICmpEQ(LHS: next, RHS: end, Name: "vla-init.isdone");
2248 Builder.CreateCondBr(Cond: done, True: contBB, False: loopBB);
2249 cur->addIncoming(V: next, BB: loopBB);
2250
2251 CGF.EmitBlock(BB: contBB);
2252}
2253
2254void
2255CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
2256 // Ignore empty classes in C++.
2257 if (getLangOpts().CPlusPlus)
2258 if (const auto *RD = Ty->getAsCXXRecordDecl(); RD && RD->isEmpty())
2259 return;
2260
2261 if (DestPtr.getElementType() != Int8Ty)
2262 DestPtr = DestPtr.withElementType(ElemTy: Int8Ty);
2263
2264 // Get size and alignment info for this aggregate.
2265 CharUnits size = getContext().getTypeSizeInChars(T: Ty);
2266
2267 llvm::Value *SizeVal;
2268 const VariableArrayType *vla;
2269
2270 // Don't bother emitting a zero-byte memset.
2271 if (size.isZero()) {
2272 // But note that getTypeInfo returns 0 for a VLA.
2273 if (const VariableArrayType *vlaType =
2274 dyn_cast_or_null<VariableArrayType>(
2275 Val: getContext().getAsArrayType(T: Ty))) {
2276 auto VlaSize = getVLASize(vla: vlaType);
2277 SizeVal = VlaSize.NumElts;
2278 CharUnits eltSize = getContext().getTypeSizeInChars(T: VlaSize.Type);
2279 if (!eltSize.isOne())
2280 SizeVal = Builder.CreateNUWMul(LHS: SizeVal, RHS: CGM.getSize(numChars: eltSize));
2281 vla = vlaType;
2282 } else {
2283 return;
2284 }
2285 } else {
2286 SizeVal = CGM.getSize(numChars: size);
2287 vla = nullptr;
2288 }
2289
2290 // If the type contains a pointer to data member we can't memset it to zero.
2291 // Instead, create a null constant and copy it to the destination.
2292 // TODO: there are other patterns besides zero that we can usefully memset,
2293 // like -1, which happens to be the pattern used by member-pointers.
2294 if (!CGM.getTypes().isZeroInitializable(T: Ty)) {
2295 // For a VLA, emit a single element, then splat that over the VLA.
2296 if (vla) Ty = getContext().getBaseElementType(VAT: vla);
2297
2298 llvm::Constant *NullConstant = CGM.EmitNullConstant(T: Ty);
2299
2300 llvm::GlobalVariable *NullVariable =
2301 new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
2302 /*isConstant=*/true,
2303 llvm::GlobalVariable::PrivateLinkage,
2304 NullConstant, Twine());
2305 CharUnits NullAlign = DestPtr.getAlignment();
2306 NullVariable->setAlignment(NullAlign.getAsAlign());
2307 Address SrcPtr(NullVariable, Builder.getInt8Ty(), NullAlign);
2308
2309 if (vla) return emitNonZeroVLAInit(CGF&: *this, baseType: Ty, dest: DestPtr, src: SrcPtr, sizeInChars: SizeVal);
2310
2311 // Get and call the appropriate llvm.memcpy overload.
2312 Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: SizeVal, IsVolatile: false);
2313 return;
2314 }
2315
2316 // Otherwise, just memset the whole thing to zero. This is legal
2317 // because in LLVM, all default initializers (other than the ones we just
2318 // handled above) are guaranteed to have a bit pattern of all zeros.
2319 Builder.CreateMemSet(Dest: DestPtr, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
2320}
2321
2322llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
2323 // Make sure that there is a block for the indirect goto.
2324 if (!IndirectBranch)
2325 GetIndirectGotoBlock();
2326
2327 llvm::BasicBlock *BB = getJumpDestForLabel(S: L).getBlock();
2328
2329 // Make sure the indirect branch includes all of the address-taken blocks.
2330 IndirectBranch->addDestination(Dest: BB);
2331 return llvm::BlockAddress::get(Ty: CurFn->getType(), BB);
2332}
2333
2334llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {
2335 // If we already made the indirect branch for indirect goto, return its block.
2336 if (IndirectBranch) return IndirectBranch->getParent();
2337
2338 CGBuilderTy TmpBuilder(*this, createBasicBlock(name: "indirectgoto"));
2339
2340 // Create the PHI node that indirect gotos will add entries to.
2341 llvm::Value *DestVal = TmpBuilder.CreatePHI(Ty: Int8PtrTy, NumReservedValues: 0,
2342 Name: "indirect.goto.dest");
2343
2344 // Create the indirect branch instruction.
2345 IndirectBranch = TmpBuilder.CreateIndirectBr(Addr: DestVal);
2346 return IndirectBranch->getParent();
2347}
2348
2349/// Computes the length of an array in elements, as well as the base
2350/// element type and a properly-typed first element pointer.
2351llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
2352 QualType &baseType,
2353 Address &addr) {
2354 const ArrayType *arrayType = origArrayType;
2355
2356 // If it's a VLA, we have to load the stored size. Note that
2357 // this is the size of the VLA in bytes, not its size in elements.
2358 llvm::Value *numVLAElements = nullptr;
2359 if (isa<VariableArrayType>(Val: arrayType)) {
2360 numVLAElements = getVLASize(vla: cast<VariableArrayType>(Val: arrayType)).NumElts;
2361
2362 // Walk into all VLAs. This doesn't require changes to addr,
2363 // which has type T* where T is the first non-VLA element type.
2364 do {
2365 QualType elementType = arrayType->getElementType();
2366 arrayType = getContext().getAsArrayType(T: elementType);
2367
2368 // If we only have VLA components, 'addr' requires no adjustment.
2369 if (!arrayType) {
2370 baseType = elementType;
2371 return numVLAElements;
2372 }
2373 } while (isa<VariableArrayType>(Val: arrayType));
2374
2375 // We get out here only if we find a constant array type
2376 // inside the VLA.
2377 }
2378
2379 // We have some number of constant-length arrays, so addr should
2380 // have LLVM type [M x [N x [...]]]*. Build a GEP that walks
2381 // down to the first element of addr.
2382 SmallVector<llvm::Value*, 8> gepIndices;
2383
2384 // GEP down to the array type.
2385 llvm::ConstantInt *zero = Builder.getInt32(C: 0);
2386 gepIndices.push_back(Elt: zero);
2387
2388 uint64_t countFromCLAs = 1;
2389 QualType eltType;
2390
2391 llvm::ArrayType *llvmArrayType =
2392 dyn_cast<llvm::ArrayType>(Val: addr.getElementType());
2393 while (llvmArrayType) {
2394 assert(isa<ConstantArrayType>(arrayType));
2395 assert(cast<ConstantArrayType>(arrayType)->getZExtSize() ==
2396 llvmArrayType->getNumElements());
2397
2398 gepIndices.push_back(Elt: zero);
2399 countFromCLAs *= llvmArrayType->getNumElements();
2400 eltType = arrayType->getElementType();
2401
2402 llvmArrayType =
2403 dyn_cast<llvm::ArrayType>(Val: llvmArrayType->getElementType());
2404 arrayType = getContext().getAsArrayType(T: arrayType->getElementType());
2405 assert((!llvmArrayType || arrayType) &&
2406 "LLVM and Clang types are out-of-synch");
2407 }
2408
2409 if (arrayType) {
2410 // From this point onwards, the Clang array type has been emitted
2411 // as some other type (probably a packed struct). Compute the array
2412 // size, and just emit the 'begin' expression as a bitcast.
2413 while (arrayType) {
2414 countFromCLAs *= cast<ConstantArrayType>(Val: arrayType)->getZExtSize();
2415 eltType = arrayType->getElementType();
2416 arrayType = getContext().getAsArrayType(T: eltType);
2417 }
2418
2419 llvm::Type *baseType = ConvertType(T: eltType);
2420 addr = addr.withElementType(ElemTy: baseType);
2421 } else {
2422 // Create the actual GEP.
2423 addr = Address(Builder.CreateInBoundsGEP(Ty: addr.getElementType(),
2424 Ptr: addr.emitRawPointer(CGF&: *this),
2425 IdxList: gepIndices, Name: "array.begin"),
2426 ConvertTypeForMem(T: eltType), addr.getAlignment());
2427 }
2428
2429 baseType = eltType;
2430
2431 llvm::Value *numElements
2432 = llvm::ConstantInt::get(Ty: SizeTy, V: countFromCLAs);
2433
2434 // If we had any VLA dimensions, factor them in.
2435 if (numVLAElements)
2436 numElements = Builder.CreateNUWMul(LHS: numVLAElements, RHS: numElements);
2437
2438 return numElements;
2439}
2440
2441CodeGenFunction::VlaSizePair CodeGenFunction::getVLASize(QualType type) {
2442 const VariableArrayType *vla = getContext().getAsVariableArrayType(T: type);
2443 assert(vla && "type was not a variable array type!");
2444 return getVLASize(vla);
2445}
2446
2447CodeGenFunction::VlaSizePair
2448CodeGenFunction::getVLASize(const VariableArrayType *type) {
2449 // The number of elements so far; always size_t.
2450 llvm::Value *numElements = nullptr;
2451
2452 QualType elementType;
2453 do {
2454 elementType = type->getElementType();
2455 llvm::Value *vlaSize = VLASizeMap[type->getSizeExpr()];
2456 assert(vlaSize && "no size for VLA!");
2457 assert(vlaSize->getType() == SizeTy);
2458
2459 if (!numElements) {
2460 numElements = vlaSize;
2461 } else {
2462 // It's undefined behavior if this wraps around, so mark it that way.
2463 // FIXME: Teach -fsanitize=undefined to trap this.
2464 numElements = Builder.CreateNUWMul(LHS: numElements, RHS: vlaSize);
2465 }
2466 } while ((type = getContext().getAsVariableArrayType(T: elementType)));
2467
2468 return { numElements, elementType };
2469}
2470
2471CodeGenFunction::VlaSizePair
2472CodeGenFunction::getVLAElements1D(QualType type) {
2473 const VariableArrayType *vla = getContext().getAsVariableArrayType(T: type);
2474 assert(vla && "type was not a variable array type!");
2475 return getVLAElements1D(vla);
2476}
2477
2478CodeGenFunction::VlaSizePair
2479CodeGenFunction::getVLAElements1D(const VariableArrayType *Vla) {
2480 llvm::Value *VlaSize = VLASizeMap[Vla->getSizeExpr()];
2481 assert(VlaSize && "no size for VLA!");
2482 assert(VlaSize->getType() == SizeTy);
2483 return { VlaSize, Vla->getElementType() };
2484}
2485
2486void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
2487 assert(type->isVariablyModifiedType() &&
2488 "Must pass variably modified type to EmitVLASizes!");
2489
2490 EnsureInsertPoint();
2491
2492 // We're going to walk down into the type and look for VLA
2493 // expressions.
2494 do {
2495 assert(type->isVariablyModifiedType());
2496
2497 const Type *ty = type.getTypePtr();
2498 switch (ty->getTypeClass()) {
2499
2500#define TYPE(Class, Base)
2501#define ABSTRACT_TYPE(Class, Base)
2502#define NON_CANONICAL_TYPE(Class, Base)
2503#define DEPENDENT_TYPE(Class, Base) case Type::Class:
2504#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
2505#include "clang/AST/TypeNodes.inc"
2506 llvm_unreachable("unexpected dependent type!");
2507
2508 // These types are never variably-modified.
2509 case Type::Builtin:
2510 case Type::Complex:
2511 case Type::Vector:
2512 case Type::ExtVector:
2513 case Type::ConstantMatrix:
2514 case Type::Record:
2515 case Type::Enum:
2516 case Type::Using:
2517 case Type::TemplateSpecialization:
2518 case Type::ObjCTypeParam:
2519 case Type::ObjCObject:
2520 case Type::ObjCInterface:
2521 case Type::ObjCObjectPointer:
2522 case Type::BitInt:
2523 case Type::HLSLInlineSpirv:
2524 case Type::PredefinedSugar:
2525 llvm_unreachable("type class is never variably-modified!");
2526
2527 case Type::Adjusted:
2528 type = cast<AdjustedType>(Val: ty)->getAdjustedType();
2529 break;
2530
2531 case Type::Decayed:
2532 type = cast<DecayedType>(Val: ty)->getPointeeType();
2533 break;
2534
2535 case Type::Pointer:
2536 type = cast<PointerType>(Val: ty)->getPointeeType();
2537 break;
2538
2539 case Type::BlockPointer:
2540 type = cast<BlockPointerType>(Val: ty)->getPointeeType();
2541 break;
2542
2543 case Type::LValueReference:
2544 case Type::RValueReference:
2545 type = cast<ReferenceType>(Val: ty)->getPointeeType();
2546 break;
2547
2548 case Type::MemberPointer:
2549 type = cast<MemberPointerType>(Val: ty)->getPointeeType();
2550 break;
2551
2552 case Type::ArrayParameter:
2553 case Type::ConstantArray:
2554 case Type::IncompleteArray:
2555 // Losing element qualification here is fine.
2556 type = cast<ArrayType>(Val: ty)->getElementType();
2557 break;
2558
2559 case Type::VariableArray: {
2560 // Losing element qualification here is fine.
2561 const VariableArrayType *vat = cast<VariableArrayType>(Val: ty);
2562
2563 // Unknown size indication requires no size computation.
2564 // Otherwise, evaluate and record it.
2565 if (const Expr *sizeExpr = vat->getSizeExpr()) {
2566 // It's possible that we might have emitted this already,
2567 // e.g. with a typedef and a pointer to it.
2568 llvm::Value *&entry = VLASizeMap[sizeExpr];
2569 if (!entry) {
2570 llvm::Value *size = EmitScalarExpr(E: sizeExpr);
2571
2572 // C11 6.7.6.2p5:
2573 // If the size is an expression that is not an integer constant
2574 // expression [...] each time it is evaluated it shall have a value
2575 // greater than zero.
2576 if (SanOpts.has(K: SanitizerKind::VLABound)) {
2577 auto CheckOrdinal = SanitizerKind::SO_VLABound;
2578 auto CheckHandler = SanitizerHandler::VLABoundNotPositive;
2579 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2580 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: size->getType());
2581 clang::QualType SEType = sizeExpr->getType();
2582 llvm::Value *CheckCondition =
2583 SEType->isSignedIntegerType()
2584 ? Builder.CreateICmpSGT(LHS: size, RHS: Zero)
2585 : Builder.CreateICmpUGT(LHS: size, RHS: Zero);
2586 llvm::Constant *StaticArgs[] = {
2587 EmitCheckSourceLocation(Loc: sizeExpr->getBeginLoc()),
2588 EmitCheckTypeDescriptor(T: SEType)};
2589 EmitCheck(Checked: std::make_pair(x&: CheckCondition, y&: CheckOrdinal),
2590 Check: CheckHandler, StaticArgs, DynamicArgs: size);
2591 }
2592
2593 // Always zexting here would be wrong if it weren't
2594 // undefined behavior to have a negative bound.
2595 // FIXME: What about when size's type is larger than size_t?
2596 entry = Builder.CreateIntCast(V: size, DestTy: SizeTy, /*signed*/ isSigned: false);
2597 }
2598 }
2599 type = vat->getElementType();
2600 break;
2601 }
2602
2603 case Type::FunctionProto:
2604 case Type::FunctionNoProto:
2605 type = cast<FunctionType>(Val: ty)->getReturnType();
2606 break;
2607
2608 case Type::Paren:
2609 case Type::TypeOf:
2610 case Type::UnaryTransform:
2611 case Type::Attributed:
2612 case Type::BTFTagAttributed:
2613 case Type::HLSLAttributedResource:
2614 case Type::SubstTemplateTypeParm:
2615 case Type::MacroQualified:
2616 case Type::CountAttributed:
2617 // Keep walking after single level desugaring.
2618 type = type.getSingleStepDesugaredType(Context: getContext());
2619 break;
2620
2621 case Type::Typedef:
2622 case Type::Decltype:
2623 case Type::Auto:
2624 case Type::DeducedTemplateSpecialization:
2625 case Type::PackIndexing:
2626 // Stop walking: nothing to do.
2627 return;
2628
2629 case Type::TypeOfExpr:
2630 // Stop walking: emit typeof expression.
2631 EmitIgnoredExpr(E: cast<TypeOfExprType>(Val: ty)->getUnderlyingExpr());
2632 return;
2633
2634 case Type::Atomic:
2635 type = cast<AtomicType>(Val: ty)->getValueType();
2636 break;
2637
2638 case Type::Pipe:
2639 type = cast<PipeType>(Val: ty)->getElementType();
2640 break;
2641 }
2642 } while (type->isVariablyModifiedType());
2643}
2644
2645Address CodeGenFunction::EmitVAListRef(const Expr* E) {
2646 if (getContext().getBuiltinVaListType()->isArrayType())
2647 return EmitPointerWithAlignment(Addr: E);
2648 return EmitLValue(E).getAddress();
2649}
2650
2651Address CodeGenFunction::EmitMSVAListRef(const Expr *E) {
2652 return EmitLValue(E).getAddress();
2653}
2654
2655void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E,
2656 const APValue &Init) {
2657 assert(Init.hasValue() && "Invalid DeclRefExpr initializer!");
2658 if (CGDebugInfo *Dbg = getDebugInfo())
2659 if (CGM.getCodeGenOpts().hasReducedDebugInfo())
2660 Dbg->EmitGlobalVariable(VD: E->getDecl(), Init);
2661}
2662
2663CodeGenFunction::PeepholeProtection
2664CodeGenFunction::protectFromPeepholes(RValue rvalue) {
2665 // At the moment, the only aggressive peephole we do in IR gen
2666 // is trunc(zext) folding, but if we add more, we can easily
2667 // extend this protection.
2668
2669 if (!rvalue.isScalar()) return PeepholeProtection();
2670 llvm::Value *value = rvalue.getScalarVal();
2671 if (!isa<llvm::ZExtInst>(Val: value)) return PeepholeProtection();
2672
2673 // Just make an extra bitcast.
2674 assert(HaveInsertPoint());
2675 llvm::Instruction *inst = new llvm::BitCastInst(value, value->getType(), "",
2676 Builder.GetInsertBlock());
2677
2678 PeepholeProtection protection;
2679 protection.Inst = inst;
2680 return protection;
2681}
2682
2683void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {
2684 if (!protection.Inst) return;
2685
2686 // In theory, we could try to duplicate the peepholes now, but whatever.
2687 protection.Inst->eraseFromParent();
2688}
2689
2690void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2691 QualType Ty, SourceLocation Loc,
2692 SourceLocation AssumptionLoc,
2693 llvm::Value *Alignment,
2694 llvm::Value *OffsetValue) {
2695 if (Alignment->getType() != IntPtrTy)
2696 Alignment =
2697 Builder.CreateIntCast(V: Alignment, DestTy: IntPtrTy, isSigned: false, Name: "casted.align");
2698 if (OffsetValue && OffsetValue->getType() != IntPtrTy)
2699 OffsetValue =
2700 Builder.CreateIntCast(V: OffsetValue, DestTy: IntPtrTy, isSigned: true, Name: "casted.offset");
2701 llvm::Value *TheCheck = nullptr;
2702 if (SanOpts.has(K: SanitizerKind::Alignment)) {
2703 llvm::Value *PtrIntValue =
2704 Builder.CreatePtrToInt(V: PtrValue, DestTy: IntPtrTy, Name: "ptrint");
2705
2706 if (OffsetValue) {
2707 bool IsOffsetZero = false;
2708 if (const auto *CI = dyn_cast<llvm::ConstantInt>(Val: OffsetValue))
2709 IsOffsetZero = CI->isZero();
2710
2711 if (!IsOffsetZero)
2712 PtrIntValue = Builder.CreateSub(LHS: PtrIntValue, RHS: OffsetValue, Name: "offsetptr");
2713 }
2714
2715 llvm::Value *Zero = llvm::ConstantInt::get(Ty: IntPtrTy, V: 0);
2716 llvm::Value *Mask =
2717 Builder.CreateSub(LHS: Alignment, RHS: llvm::ConstantInt::get(Ty: IntPtrTy, V: 1));
2718 llvm::Value *MaskedPtr = Builder.CreateAnd(LHS: PtrIntValue, RHS: Mask, Name: "maskedptr");
2719 TheCheck = Builder.CreateICmpEQ(LHS: MaskedPtr, RHS: Zero, Name: "maskcond");
2720 }
2721 llvm::Instruction *Assumption = Builder.CreateAlignmentAssumption(
2722 DL: CGM.getDataLayout(), PtrValue, Alignment, OffsetValue);
2723
2724 if (!SanOpts.has(K: SanitizerKind::Alignment))
2725 return;
2726 emitAlignmentAssumptionCheck(Ptr: PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2727 OffsetValue, TheCheck, Assumption);
2728}
2729
2730void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2731 const Expr *E,
2732 SourceLocation AssumptionLoc,
2733 llvm::Value *Alignment,
2734 llvm::Value *OffsetValue) {
2735 QualType Ty = E->getType();
2736 SourceLocation Loc = E->getExprLoc();
2737
2738 emitAlignmentAssumption(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2739 OffsetValue);
2740}
2741
2742llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn,
2743 llvm::Value *AnnotatedVal,
2744 StringRef AnnotationStr,
2745 SourceLocation Location,
2746 const AnnotateAttr *Attr) {
2747 SmallVector<llvm::Value *, 5> Args = {
2748 AnnotatedVal,
2749 CGM.EmitAnnotationString(Str: AnnotationStr),
2750 CGM.EmitAnnotationUnit(Loc: Location),
2751 CGM.EmitAnnotationLineNo(L: Location),
2752 };
2753 if (Attr)
2754 Args.push_back(Elt: CGM.EmitAnnotationArgs(Attr));
2755 return Builder.CreateCall(Callee: AnnotationFn, Args);
2756}
2757
2758void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
2759 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2760 for (const auto *I : D->specific_attrs<AnnotateAttr>())
2761 EmitAnnotationCall(AnnotationFn: CGM.getIntrinsic(IID: llvm::Intrinsic::var_annotation,
2762 Tys: {V->getType(), CGM.ConstGlobalsPtrTy}),
2763 AnnotatedVal: V, AnnotationStr: I->getAnnotation(), Location: D->getLocation(), Attr: I);
2764}
2765
2766Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
2767 Address Addr) {
2768 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2769 llvm::Value *V = Addr.emitRawPointer(CGF&: *this);
2770 llvm::Type *VTy = V->getType();
2771 auto *PTy = dyn_cast<llvm::PointerType>(Val: VTy);
2772 unsigned AS = PTy ? PTy->getAddressSpace() : 0;
2773 llvm::PointerType *IntrinTy =
2774 llvm::PointerType::get(C&: CGM.getLLVMContext(), AddressSpace: AS);
2775 llvm::Function *F = CGM.getIntrinsic(IID: llvm::Intrinsic::ptr_annotation,
2776 Tys: {IntrinTy, CGM.ConstGlobalsPtrTy});
2777
2778 for (const auto *I : D->specific_attrs<AnnotateAttr>()) {
2779 // FIXME Always emit the cast inst so we can differentiate between
2780 // annotation on the first field of a struct and annotation on the struct
2781 // itself.
2782 if (VTy != IntrinTy)
2783 V = Builder.CreateBitCast(V, DestTy: IntrinTy);
2784 V = EmitAnnotationCall(AnnotationFn: F, AnnotatedVal: V, AnnotationStr: I->getAnnotation(), Location: D->getLocation(), Attr: I);
2785 V = Builder.CreateBitCast(V, DestTy: VTy);
2786 }
2787
2788 return Address(V, Addr.getElementType(), Addr.getAlignment());
2789}
2790
2791CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
2792
2793CodeGenFunction::SanitizerScope::SanitizerScope(CodeGenFunction *CGF)
2794 : CGF(CGF) {
2795 assert(!CGF->IsSanitizerScope);
2796 CGF->IsSanitizerScope = true;
2797}
2798
2799CodeGenFunction::SanitizerScope::~SanitizerScope() {
2800 CGF->IsSanitizerScope = false;
2801}
2802
2803void CodeGenFunction::InsertHelper(llvm::Instruction *I,
2804 const llvm::Twine &Name,
2805 llvm::BasicBlock::iterator InsertPt) const {
2806 LoopStack.InsertHelper(I);
2807 if (IsSanitizerScope)
2808 I->setNoSanitizeMetadata();
2809}
2810
2811void CGBuilderInserter::InsertHelper(
2812 llvm::Instruction *I, const llvm::Twine &Name,
2813 llvm::BasicBlock::iterator InsertPt) const {
2814 llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, InsertPt);
2815 if (CGF)
2816 CGF->InsertHelper(I, Name, InsertPt);
2817}
2818
2819// Emits an error if we don't have a valid set of target features for the
2820// called function.
2821void CodeGenFunction::checkTargetFeatures(const CallExpr *E,
2822 const FunctionDecl *TargetDecl) {
2823 // SemaChecking cannot handle below x86 builtins because they have different
2824 // parameter ranges with different TargetAttribute of caller.
2825 if (CGM.getContext().getTargetInfo().getTriple().isX86()) {
2826 unsigned BuiltinID = TargetDecl->getBuiltinID();
2827 if (BuiltinID == X86::BI__builtin_ia32_cmpps ||
2828 BuiltinID == X86::BI__builtin_ia32_cmpss ||
2829 BuiltinID == X86::BI__builtin_ia32_cmppd ||
2830 BuiltinID == X86::BI__builtin_ia32_cmpsd) {
2831 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
2832 llvm::StringMap<bool> TargetFetureMap;
2833 CGM.getContext().getFunctionFeatureMap(FeatureMap&: TargetFetureMap, FD);
2834 llvm::APSInt Result =
2835 *(E->getArg(Arg: 2)->getIntegerConstantExpr(Ctx: CGM.getContext()));
2836 if (Result.getSExtValue() > 7 && !TargetFetureMap.lookup(Key: "avx"))
2837 CGM.getDiags().Report(Loc: E->getBeginLoc(), DiagID: diag::err_builtin_needs_feature)
2838 << TargetDecl->getDeclName() << "avx";
2839 }
2840 }
2841 return checkTargetFeatures(Loc: E->getBeginLoc(), TargetDecl);
2842}
2843
2844// Emits an error if we don't have a valid set of target features for the
2845// called function.
2846void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
2847 const FunctionDecl *TargetDecl) {
2848 // Early exit if this is an indirect call.
2849 if (!TargetDecl)
2850 return;
2851
2852 // Get the current enclosing function if it exists. If it doesn't
2853 // we can't check the target features anyhow.
2854 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: CurCodeDecl);
2855 if (!FD)
2856 return;
2857
2858 bool IsAlwaysInline = TargetDecl->hasAttr<AlwaysInlineAttr>();
2859 bool IsFlatten = FD && FD->hasAttr<FlattenAttr>();
2860
2861 // Grab the required features for the call. For a builtin this is listed in
2862 // the td file with the default cpu, for an always_inline function this is any
2863 // listed cpu and any listed features.
2864 unsigned BuiltinID = TargetDecl->getBuiltinID();
2865 std::string MissingFeature;
2866 llvm::StringMap<bool> CallerFeatureMap;
2867 CGM.getContext().getFunctionFeatureMap(FeatureMap&: CallerFeatureMap, FD);
2868 // When compiling in HipStdPar mode we have to be conservative in rejecting
2869 // target specific features in the FE, and defer the possible error to the
2870 // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
2871 // referenced by an accelerator executable function, we emit an error.
2872 bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;
2873 if (BuiltinID) {
2874 StringRef FeatureList(CGM.getContext().BuiltinInfo.getRequiredFeatures(ID: BuiltinID));
2875 if (!Builtin::evaluateRequiredTargetFeatures(
2876 RequiredFatures: FeatureList, TargetFetureMap: CallerFeatureMap) && !IsHipStdPar) {
2877 CGM.getDiags().Report(Loc, DiagID: diag::err_builtin_needs_feature)
2878 << TargetDecl->getDeclName()
2879 << FeatureList;
2880 }
2881 } else if (!TargetDecl->isMultiVersion() &&
2882 TargetDecl->hasAttr<TargetAttr>()) {
2883 // Get the required features for the callee.
2884
2885 const TargetAttr *TD = TargetDecl->getAttr<TargetAttr>();
2886 ParsedTargetAttr ParsedAttr =
2887 CGM.getContext().filterFunctionTargetAttrs(TD);
2888
2889 SmallVector<StringRef, 1> ReqFeatures;
2890 llvm::StringMap<bool> CalleeFeatureMap;
2891 CGM.getContext().getFunctionFeatureMap(FeatureMap&: CalleeFeatureMap, TargetDecl);
2892
2893 for (const auto &F : ParsedAttr.Features) {
2894 if (F[0] == '+' && CalleeFeatureMap.lookup(Key: F.substr(pos: 1)))
2895 ReqFeatures.push_back(Elt: StringRef(F).substr(Start: 1));
2896 }
2897
2898 for (const auto &F : CalleeFeatureMap) {
2899 // Only positive features are "required".
2900 if (F.getValue())
2901 ReqFeatures.push_back(Elt: F.getKey());
2902 }
2903 if (!llvm::all_of(Range&: ReqFeatures,
2904 P: [&](StringRef Feature) {
2905 if (!CallerFeatureMap.lookup(Key: Feature)) {
2906 MissingFeature = Feature.str();
2907 return false;
2908 }
2909 return true;
2910 }) &&
2911 !IsHipStdPar) {
2912 if (IsAlwaysInline)
2913 CGM.getDiags().Report(Loc, DiagID: diag::err_function_needs_feature)
2914 << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
2915 else if (IsFlatten)
2916 CGM.getDiags().Report(Loc, DiagID: diag::err_flatten_function_needs_feature)
2917 << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
2918 }
2919
2920 } else if (!FD->isMultiVersion() && FD->hasAttr<TargetAttr>()) {
2921 llvm::StringMap<bool> CalleeFeatureMap;
2922 CGM.getContext().getFunctionFeatureMap(FeatureMap&: CalleeFeatureMap, TargetDecl);
2923
2924 for (const auto &F : CalleeFeatureMap) {
2925 if (F.getValue() &&
2926 (!CallerFeatureMap.lookup(Key: F.getKey()) ||
2927 !CallerFeatureMap.find(Key: F.getKey())->getValue()) &&
2928 !IsHipStdPar) {
2929 if (IsAlwaysInline)
2930 CGM.getDiags().Report(Loc, DiagID: diag::err_function_needs_feature)
2931 << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();
2932 else if (IsFlatten)
2933 CGM.getDiags().Report(Loc, DiagID: diag::err_flatten_function_needs_feature)
2934 << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();
2935 }
2936 }
2937 }
2938}
2939
2940void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
2941 if (!CGM.getCodeGenOpts().SanitizeStats)
2942 return;
2943
2944 llvm::IRBuilder<> IRB(Builder.GetInsertBlock(), Builder.GetInsertPoint());
2945 IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation());
2946 CGM.getSanStats().create(B&: IRB, SK: SSK);
2947}
2948
2949void CodeGenFunction::EmitKCFIOperandBundle(
2950 const CGCallee &Callee, SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
2951 const CGCalleeInfo &CI = Callee.getAbstractInfo();
2952 const FunctionProtoType *FP = CI.getCalleeFunctionProtoType();
2953 if (!FP)
2954 return;
2955
2956 StringRef Salt;
2957 if (const auto &Info = FP->getExtraAttributeInfo())
2958 Salt = Info.CFISalt;
2959
2960 Bundles.emplace_back(Args: "kcfi", Args: CGM.CreateKCFITypeId(T: FP->desugar(), Salt));
2961}
2962
2963llvm::Value *
2964CodeGenFunction::FormAArch64ResolverCondition(const FMVResolverOption &RO) {
2965 return RO.Features.empty() ? nullptr : EmitAArch64CpuSupports(FeatureStrs: RO.Features);
2966}
2967
2968llvm::Value *
2969CodeGenFunction::FormX86ResolverCondition(const FMVResolverOption &RO) {
2970 llvm::Value *Condition = nullptr;
2971
2972 if (RO.Architecture) {
2973 StringRef Arch = *RO.Architecture;
2974 // If arch= specifies an x86-64 micro-architecture level, test the feature
2975 // with __builtin_cpu_supports, otherwise use __builtin_cpu_is.
2976 if (Arch.starts_with(Prefix: "x86-64"))
2977 Condition = EmitX86CpuSupports(FeatureStrs: {Arch});
2978 else
2979 Condition = EmitX86CpuIs(CPUStr: Arch);
2980 }
2981
2982 if (!RO.Features.empty()) {
2983 llvm::Value *FeatureCond = EmitX86CpuSupports(FeatureStrs: RO.Features);
2984 Condition =
2985 Condition ? Builder.CreateAnd(LHS: Condition, RHS: FeatureCond) : FeatureCond;
2986 }
2987 return Condition;
2988}
2989
2990static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
2991 llvm::Function *Resolver,
2992 CGBuilderTy &Builder,
2993 llvm::Function *FuncToReturn,
2994 bool SupportsIFunc) {
2995 if (SupportsIFunc) {
2996 Builder.CreateRet(V: FuncToReturn);
2997 return;
2998 }
2999
3000 llvm::SmallVector<llvm::Value *, 10> Args(
3001 llvm::make_pointer_range(Range: Resolver->args()));
3002
3003 llvm::CallInst *Result = Builder.CreateCall(Callee: FuncToReturn, Args);
3004 Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
3005
3006 if (Resolver->getReturnType()->isVoidTy())
3007 Builder.CreateRetVoid();
3008 else
3009 Builder.CreateRet(V: Result);
3010}
3011
3012void CodeGenFunction::EmitMultiVersionResolver(
3013 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3014
3015 llvm::Triple::ArchType ArchType =
3016 getContext().getTargetInfo().getTriple().getArch();
3017
3018 switch (ArchType) {
3019 case llvm::Triple::x86:
3020 case llvm::Triple::x86_64:
3021 EmitX86MultiVersionResolver(Resolver, Options);
3022 return;
3023 case llvm::Triple::aarch64:
3024 EmitAArch64MultiVersionResolver(Resolver, Options);
3025 return;
3026 case llvm::Triple::riscv32:
3027 case llvm::Triple::riscv64:
3028 EmitRISCVMultiVersionResolver(Resolver, Options);
3029 return;
3030
3031 default:
3032 assert(false && "Only implemented for x86, AArch64 and RISC-V targets");
3033 }
3034}
3035
3036void CodeGenFunction::EmitRISCVMultiVersionResolver(
3037 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3038
3039 if (getContext().getTargetInfo().getTriple().getOS() !=
3040 llvm::Triple::OSType::Linux) {
3041 CGM.getDiags().Report(DiagID: diag::err_os_unsupport_riscv_fmv);
3042 return;
3043 }
3044
3045 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3046 Builder.SetInsertPoint(CurBlock);
3047 EmitRISCVCpuInit();
3048
3049 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3050 bool HasDefault = false;
3051 unsigned DefaultIndex = 0;
3052
3053 // Check the each candidate function.
3054 for (unsigned Index = 0; Index < Options.size(); Index++) {
3055
3056 if (Options[Index].Features.empty()) {
3057 HasDefault = true;
3058 DefaultIndex = Index;
3059 continue;
3060 }
3061
3062 Builder.SetInsertPoint(CurBlock);
3063
3064 // FeaturesCondition: The bitmask of the required extension has been
3065 // enabled by the runtime object.
3066 // (__riscv_feature_bits.features[i] & REQUIRED_BITMASK) ==
3067 // REQUIRED_BITMASK
3068 //
3069 // When condition is met, return this version of the function.
3070 // Otherwise, try the next version.
3071 //
3072 // if (FeaturesConditionVersion1)
3073 // return Version1;
3074 // else if (FeaturesConditionVersion2)
3075 // return Version2;
3076 // else if (FeaturesConditionVersion3)
3077 // return Version3;
3078 // ...
3079 // else
3080 // return DefaultVersion;
3081
3082 // TODO: Add a condition to check the length before accessing elements.
3083 // Without checking the length first, we may access an incorrect memory
3084 // address when using different versions.
3085 llvm::SmallVector<StringRef, 8> CurrTargetAttrFeats;
3086 llvm::SmallVector<std::string, 8> TargetAttrFeats;
3087
3088 for (StringRef Feat : Options[Index].Features) {
3089 std::vector<std::string> FeatStr =
3090 getContext().getTargetInfo().parseTargetAttr(Str: Feat).Features;
3091
3092 assert(FeatStr.size() == 1 && "Feature string not delimited");
3093
3094 std::string &CurrFeat = FeatStr.front();
3095 if (CurrFeat[0] == '+')
3096 TargetAttrFeats.push_back(Elt: CurrFeat.substr(pos: 1));
3097 }
3098
3099 if (TargetAttrFeats.empty())
3100 continue;
3101
3102 for (std::string &Feat : TargetAttrFeats)
3103 CurrTargetAttrFeats.push_back(Elt: Feat);
3104
3105 Builder.SetInsertPoint(CurBlock);
3106 llvm::Value *FeatsCondition = EmitRISCVCpuSupports(FeaturesStrs: CurrTargetAttrFeats);
3107
3108 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3109 CGBuilderTy RetBuilder(*this, RetBlock);
3110 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder,
3111 FuncToReturn: Options[Index].Function, SupportsIFunc);
3112 llvm::BasicBlock *ElseBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3113
3114 Builder.SetInsertPoint(CurBlock);
3115 Builder.CreateCondBr(Cond: FeatsCondition, True: RetBlock, False: ElseBlock);
3116
3117 CurBlock = ElseBlock;
3118 }
3119
3120 // Finally, emit the default one.
3121 if (HasDefault) {
3122 Builder.SetInsertPoint(CurBlock);
3123 CreateMultiVersionResolverReturn(
3124 CGM, Resolver, Builder, FuncToReturn: Options[DefaultIndex].Function, SupportsIFunc);
3125 return;
3126 }
3127
3128 // If no generic/default, emit an unreachable.
3129 Builder.SetInsertPoint(CurBlock);
3130 llvm::CallInst *TrapCall = EmitTrapCall(IntrID: llvm::Intrinsic::trap);
3131 TrapCall->setDoesNotReturn();
3132 TrapCall->setDoesNotThrow();
3133 Builder.CreateUnreachable();
3134 Builder.ClearInsertionPoint();
3135}
3136
3137void CodeGenFunction::EmitAArch64MultiVersionResolver(
3138 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3139 assert(!Options.empty() && "No multiversion resolver options found");
3140 assert(Options.back().Features.size() == 0 && "Default case must be last");
3141 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3142 assert(SupportsIFunc &&
3143 "Multiversion resolver requires target IFUNC support");
3144 bool AArch64CpuInitialized = false;
3145 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3146
3147 for (const FMVResolverOption &RO : Options) {
3148 Builder.SetInsertPoint(CurBlock);
3149 llvm::Value *Condition = FormAArch64ResolverCondition(RO);
3150
3151 // The 'default' or 'all features enabled' case.
3152 if (!Condition) {
3153 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, FuncToReturn: RO.Function,
3154 SupportsIFunc);
3155 return;
3156 }
3157
3158 if (!AArch64CpuInitialized) {
3159 Builder.SetInsertPoint(TheBB: CurBlock, IP: CurBlock->begin());
3160 EmitAArch64CpuInit();
3161 AArch64CpuInitialized = true;
3162 Builder.SetInsertPoint(CurBlock);
3163 }
3164
3165 // Skip unreachable versions.
3166 if (RO.Function == nullptr)
3167 continue;
3168
3169 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3170 CGBuilderTy RetBuilder(*this, RetBlock);
3171 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder, FuncToReturn: RO.Function,
3172 SupportsIFunc);
3173 CurBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3174 Builder.CreateCondBr(Cond: Condition, True: RetBlock, False: CurBlock);
3175 }
3176
3177 // If no default, emit an unreachable.
3178 Builder.SetInsertPoint(CurBlock);
3179 llvm::CallInst *TrapCall = EmitTrapCall(IntrID: llvm::Intrinsic::trap);
3180 TrapCall->setDoesNotReturn();
3181 TrapCall->setDoesNotThrow();
3182 Builder.CreateUnreachable();
3183 Builder.ClearInsertionPoint();
3184}
3185
3186void CodeGenFunction::EmitX86MultiVersionResolver(
3187 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
3188
3189 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
3190
3191 // Main function's basic block.
3192 llvm::BasicBlock *CurBlock = createBasicBlock(name: "resolver_entry", parent: Resolver);
3193 Builder.SetInsertPoint(CurBlock);
3194 EmitX86CpuInit();
3195
3196 for (const FMVResolverOption &RO : Options) {
3197 Builder.SetInsertPoint(CurBlock);
3198 llvm::Value *Condition = FormX86ResolverCondition(RO);
3199
3200 // The 'default' or 'generic' case.
3201 if (!Condition) {
3202 assert(&RO == Options.end() - 1 &&
3203 "Default or Generic case must be last");
3204 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, FuncToReturn: RO.Function,
3205 SupportsIFunc);
3206 return;
3207 }
3208
3209 llvm::BasicBlock *RetBlock = createBasicBlock(name: "resolver_return", parent: Resolver);
3210 CGBuilderTy RetBuilder(*this, RetBlock);
3211 CreateMultiVersionResolverReturn(CGM, Resolver, Builder&: RetBuilder, FuncToReturn: RO.Function,
3212 SupportsIFunc);
3213 CurBlock = createBasicBlock(name: "resolver_else", parent: Resolver);
3214 Builder.CreateCondBr(Cond: Condition, True: RetBlock, False: CurBlock);
3215 }
3216
3217 // If no generic/default, emit an unreachable.
3218 Builder.SetInsertPoint(CurBlock);
3219 llvm::CallInst *TrapCall = EmitTrapCall(IntrID: llvm::Intrinsic::trap);
3220 TrapCall->setDoesNotReturn();
3221 TrapCall->setDoesNotThrow();
3222 Builder.CreateUnreachable();
3223 Builder.ClearInsertionPoint();
3224}
3225
3226// Loc - where the diagnostic will point, where in the source code this
3227// alignment has failed.
3228// SecondaryLoc - if present (will be present if sufficiently different from
3229// Loc), the diagnostic will additionally point a "Note:" to this location.
3230// It should be the location where the __attribute__((assume_aligned))
3231// was written e.g.
3232void CodeGenFunction::emitAlignmentAssumptionCheck(
3233 llvm::Value *Ptr, QualType Ty, SourceLocation Loc,
3234 SourceLocation SecondaryLoc, llvm::Value *Alignment,
3235 llvm::Value *OffsetValue, llvm::Value *TheCheck,
3236 llvm::Instruction *Assumption) {
3237 assert(isa_and_nonnull<llvm::CallInst>(Assumption) &&
3238 cast<llvm::CallInst>(Assumption)->getCalledOperand() ==
3239 llvm::Intrinsic::getOrInsertDeclaration(
3240 Builder.GetInsertBlock()->getParent()->getParent(),
3241 llvm::Intrinsic::assume) &&
3242 "Assumption should be a call to llvm.assume().");
3243 assert(&(Builder.GetInsertBlock()->back()) == Assumption &&
3244 "Assumption should be the last instruction of the basic block, "
3245 "since the basic block is still being generated.");
3246
3247 if (!SanOpts.has(K: SanitizerKind::Alignment))
3248 return;
3249
3250 // Don't check pointers to volatile data. The behavior here is implementation-
3251 // defined.
3252 if (Ty->getPointeeType().isVolatileQualified())
3253 return;
3254
3255 // We need to temorairly remove the assumption so we can insert the
3256 // sanitizer check before it, else the check will be dropped by optimizations.
3257 Assumption->removeFromParent();
3258
3259 {
3260 auto CheckOrdinal = SanitizerKind::SO_Alignment;
3261 auto CheckHandler = SanitizerHandler::AlignmentAssumption;
3262 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
3263
3264 if (!OffsetValue)
3265 OffsetValue = Builder.getInt1(V: false); // no offset.
3266
3267 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc),
3268 EmitCheckSourceLocation(Loc: SecondaryLoc),
3269 EmitCheckTypeDescriptor(T: Ty)};
3270 llvm::Value *DynamicData[] = {Ptr, Alignment, OffsetValue};
3271 EmitCheck(Checked: {std::make_pair(x&: TheCheck, y&: CheckOrdinal)}, Check: CheckHandler,
3272 StaticArgs: StaticData, DynamicArgs: DynamicData);
3273 }
3274
3275 // We are now in the (new, empty) "cont" basic block.
3276 // Reintroduce the assumption.
3277 Builder.Insert(I: Assumption);
3278 // FIXME: Assumption still has it's original basic block as it's Parent.
3279}
3280
3281llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
3282 if (CGDebugInfo *DI = getDebugInfo())
3283 return DI->SourceLocToDebugLoc(Loc: Location);
3284
3285 return llvm::DebugLoc();
3286}
3287
3288llvm::Value *
3289CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
3290 Stmt::Likelihood LH) {
3291 switch (LH) {
3292 case Stmt::LH_None:
3293 return Cond;
3294 case Stmt::LH_Likely:
3295 case Stmt::LH_Unlikely:
3296 // Don't generate llvm.expect on -O0 as the backend won't use it for
3297 // anything.
3298 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3299 return Cond;
3300 llvm::Type *CondTy = Cond->getType();
3301 assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");
3302 llvm::Function *FnExpect =
3303 CGM.getIntrinsic(IID: llvm::Intrinsic::expect, Tys: CondTy);
3304 llvm::Value *ExpectedValueOfCond =
3305 llvm::ConstantInt::getBool(Ty: CondTy, V: LH == Stmt::LH_Likely);
3306 return Builder.CreateCall(Callee: FnExpect, Args: {Cond, ExpectedValueOfCond},
3307 Name: Cond->getName() + ".expval");
3308 }
3309 llvm_unreachable("Unknown Likelihood");
3310}
3311
3312llvm::Value *CodeGenFunction::emitBoolVecConversion(llvm::Value *SrcVec,
3313 unsigned NumElementsDst,
3314 const llvm::Twine &Name) {
3315 auto *SrcTy = cast<llvm::FixedVectorType>(Val: SrcVec->getType());
3316 unsigned NumElementsSrc = SrcTy->getNumElements();
3317 if (NumElementsSrc == NumElementsDst)
3318 return SrcVec;
3319
3320 std::vector<int> ShuffleMask(NumElementsDst, -1);
3321 for (unsigned MaskIdx = 0;
3322 MaskIdx < std::min<>(a: NumElementsDst, b: NumElementsSrc); ++MaskIdx)
3323 ShuffleMask[MaskIdx] = MaskIdx;
3324
3325 return Builder.CreateShuffleVector(V: SrcVec, Mask: ShuffleMask, Name);
3326}
3327
3328void CodeGenFunction::EmitPointerAuthOperandBundle(
3329 const CGPointerAuthInfo &PointerAuth,
3330 SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
3331 if (!PointerAuth.isSigned())
3332 return;
3333
3334 auto *Key = Builder.getInt32(C: PointerAuth.getKey());
3335
3336 llvm::Value *Discriminator = PointerAuth.getDiscriminator();
3337 if (!Discriminator)
3338 Discriminator = Builder.getSize(N: 0);
3339
3340 llvm::Value *Args[] = {Key, Discriminator};
3341 Bundles.emplace_back(Args: "ptrauth", Args);
3342}
3343
3344static llvm::Value *EmitPointerAuthCommon(CodeGenFunction &CGF,
3345 const CGPointerAuthInfo &PointerAuth,
3346 llvm::Value *Pointer,
3347 unsigned IntrinsicID) {
3348 if (!PointerAuth)
3349 return Pointer;
3350
3351 auto Key = CGF.Builder.getInt32(C: PointerAuth.getKey());
3352
3353 llvm::Value *Discriminator = PointerAuth.getDiscriminator();
3354 if (!Discriminator) {
3355 Discriminator = CGF.Builder.getSize(N: 0);
3356 }
3357
3358 // Convert the pointer to intptr_t before signing it.
3359 auto OrigType = Pointer->getType();
3360 Pointer = CGF.Builder.CreatePtrToInt(V: Pointer, DestTy: CGF.IntPtrTy);
3361
3362 // call i64 @llvm.ptrauth.sign.i64(i64 %pointer, i32 %key, i64 %discriminator)
3363 auto Intrinsic = CGF.CGM.getIntrinsic(IID: IntrinsicID);
3364 Pointer = CGF.EmitRuntimeCall(callee: Intrinsic, args: {Pointer, Key, Discriminator});
3365
3366 // Convert back to the original type.
3367 Pointer = CGF.Builder.CreateIntToPtr(V: Pointer, DestTy: OrigType);
3368 return Pointer;
3369}
3370
3371llvm::Value *
3372CodeGenFunction::EmitPointerAuthSign(const CGPointerAuthInfo &PointerAuth,
3373 llvm::Value *Pointer) {
3374 if (!PointerAuth.shouldSign())
3375 return Pointer;
3376 return EmitPointerAuthCommon(CGF&: *this, PointerAuth, Pointer,
3377 IntrinsicID: llvm::Intrinsic::ptrauth_sign);
3378}
3379
3380static llvm::Value *EmitStrip(CodeGenFunction &CGF,
3381 const CGPointerAuthInfo &PointerAuth,
3382 llvm::Value *Pointer) {
3383 auto StripIntrinsic = CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::ptrauth_strip);
3384
3385 auto Key = CGF.Builder.getInt32(C: PointerAuth.getKey());
3386 // Convert the pointer to intptr_t before signing it.
3387 auto OrigType = Pointer->getType();
3388 Pointer = CGF.EmitRuntimeCall(
3389 callee: StripIntrinsic, args: {CGF.Builder.CreatePtrToInt(V: Pointer, DestTy: CGF.IntPtrTy), Key});
3390 return CGF.Builder.CreateIntToPtr(V: Pointer, DestTy: OrigType);
3391}
3392
3393llvm::Value *
3394CodeGenFunction::EmitPointerAuthAuth(const CGPointerAuthInfo &PointerAuth,
3395 llvm::Value *Pointer) {
3396 if (PointerAuth.shouldStrip()) {
3397 return EmitStrip(CGF&: *this, PointerAuth, Pointer);
3398 }
3399 if (!PointerAuth.shouldAuth()) {
3400 return Pointer;
3401 }
3402
3403 return EmitPointerAuthCommon(CGF&: *this, PointerAuth, Pointer,
3404 IntrinsicID: llvm::Intrinsic::ptrauth_auth);
3405}
3406
3407void CodeGenFunction::addInstToCurrentSourceAtom(
3408 llvm::Instruction *KeyInstruction, llvm::Value *Backup) {
3409 if (CGDebugInfo *DI = getDebugInfo())
3410 DI->addInstToCurrentSourceAtom(KeyInstruction, Backup);
3411}
3412
3413void CodeGenFunction::addInstToSpecificSourceAtom(
3414 llvm::Instruction *KeyInstruction, llvm::Value *Backup, uint64_t Atom) {
3415 if (CGDebugInfo *DI = getDebugInfo())
3416 DI->addInstToSpecificSourceAtom(KeyInstruction, Backup, Atom);
3417}
3418
3419void CodeGenFunction::addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,
3420 llvm::Value *Backup) {
3421 if (CGDebugInfo *DI = getDebugInfo()) {
3422 ApplyAtomGroup Grp(getDebugInfo());
3423 DI->addInstToCurrentSourceAtom(KeyInstruction, Backup);
3424 }
3425}
3426