1//===-- CodeGenFunction.h - Per-Function state for LLVM CodeGen -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the internal per-function state used for llvm translation.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
14#define LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
15
16#include "CGBuilder.h"
17#include "CGLoopInfo.h"
18#include "CGValue.h"
19#include "CodeGenModule.h"
20#include "EHScopeStack.h"
21#include "SanitizerHandler.h"
22#include "VarBypassDetector.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/CharUnits.h"
25#include "clang/AST/CurrentSourceLocExprScope.h"
26#include "clang/AST/ExprCXX.h"
27#include "clang/AST/ExprObjC.h"
28#include "clang/AST/ExprOpenMP.h"
29#include "clang/AST/StmtOpenACC.h"
30#include "clang/AST/StmtOpenMP.h"
31#include "clang/AST/StmtSYCL.h"
32#include "clang/AST/Type.h"
33#include "clang/Basic/ABI.h"
34#include "clang/Basic/CapturedStmt.h"
35#include "clang/Basic/CodeGenOptions.h"
36#include "clang/Basic/OpenMPKinds.h"
37#include "clang/Basic/TargetInfo.h"
38#include "llvm/ADT/ArrayRef.h"
39#include "llvm/ADT/DenseMap.h"
40#include "llvm/ADT/MapVector.h"
41#include "llvm/ADT/SmallVector.h"
42#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
43#include "llvm/IR/Instructions.h"
44#include "llvm/IR/ValueHandle.h"
45#include "llvm/Support/Debug.h"
46#include "llvm/Transforms/Utils/SanitizerStats.h"
47#include <optional>
48
49namespace llvm {
50class BasicBlock;
51class ConvergenceControlInst;
52class LLVMContext;
53class MDNode;
54class SwitchInst;
55class Twine;
56class Value;
57class CanonicalLoopInfo;
58} // namespace llvm
59
60namespace clang {
61class ASTContext;
62class AsmConstraintsInfo;
63class CXXDestructorDecl;
64class CXXForRangeStmt;
65class CXXTryStmt;
66class Decl;
67class LabelDecl;
68class FunctionDecl;
69class FunctionProtoType;
70class LabelStmt;
71class ObjCContainerDecl;
72class ObjCInterfaceDecl;
73class ObjCIvarDecl;
74class ObjCMethodDecl;
75class ObjCImplementationDecl;
76class ObjCPropertyImplDecl;
77class TargetInfo;
78class VarDecl;
79class ObjCForCollectionStmt;
80class ObjCAtTryStmt;
81class ObjCAtThrowStmt;
82class ObjCAtSynchronizedStmt;
83class ObjCAutoreleasePoolStmt;
84class OMPUseDevicePtrClause;
85class OMPUseDeviceAddrClause;
86class SVETypeFlags;
87class OMPExecutableDirective;
88
89namespace analyze_os_log {
90class OSLogBufferLayout;
91}
92
93namespace CodeGen {
94class CodeGenTypes;
95class CodeGenPGO;
96class CGCallee;
97class CGFunctionInfo;
98class CGBlockInfo;
99class CGCXXABI;
100class BlockByrefHelpers;
101class BlockByrefInfo;
102class BlockFieldFlags;
103class RegionCodeGenTy;
104class TargetCodeGenInfo;
105struct OMPTaskDataTy;
106struct CGCoroData;
107
108// clang-format off
109/// The kind of evaluation to perform on values of a particular
110/// type. Basically, is the code in CGExprScalar, CGExprComplex, or
111/// CGExprAgg?
112///
113/// TODO: should vectors maybe be split out into their own thing?
114enum TypeEvaluationKind {
115 TEK_Scalar,
116 TEK_Complex,
117 TEK_Aggregate
118};
119// clang-format on
120
121/// Helper class with most of the code for saving a value for a
122/// conditional expression cleanup.
123struct DominatingLLVMValue {
124 struct saved_type {
125 llvm::Value *Value; // Original value if not saved, alloca if saved
126 llvm::Type *Type; // nullptr if not saved, element type if saved
127
128 saved_type() : Value(nullptr), Type(nullptr) {}
129 saved_type(llvm::Value *V) : Value(V), Type(nullptr) {}
130 saved_type(llvm::AllocaInst *Alloca, llvm::Type *Ty)
131 : Value(Alloca), Type(Ty) {}
132
133 bool isSaved() const { return Type != nullptr; }
134 };
135
136 /// Answer whether the given value needs extra work to be saved.
137 static bool needsSaving(llvm::Value *value) {
138 if (!value)
139 return false;
140
141 // If it's not an instruction, we don't need to save.
142 if (!isa<llvm::Instruction>(Val: value))
143 return false;
144
145 // If it's an instruction in the entry block, we don't need to save.
146 llvm::BasicBlock *block = cast<llvm::Instruction>(Val: value)->getParent();
147 return (block != &block->getParent()->getEntryBlock());
148 }
149
150 static saved_type save(CodeGenFunction &CGF, llvm::Value *value);
151 static llvm::Value *restore(CodeGenFunction &CGF, saved_type value);
152};
153
154/// A partial specialization of DominatingValue for llvm::Values that
155/// might be llvm::Instructions.
156template <class T> struct DominatingPointer<T, true> : DominatingLLVMValue {
157 typedef T *type;
158 static type restore(CodeGenFunction &CGF, saved_type value) {
159 return static_cast<T *>(DominatingLLVMValue::restore(CGF, value));
160 }
161};
162
163/// A specialization of DominatingValue for Address.
164template <> struct DominatingValue<Address> {
165 typedef Address type;
166
167 struct saved_type {
168 DominatingLLVMValue::saved_type BasePtr;
169 llvm::Type *ElementType;
170 CharUnits Alignment;
171 DominatingLLVMValue::saved_type Offset;
172 llvm::PointerType *EffectiveType;
173 };
174
175 static bool needsSaving(type value) {
176 if (DominatingLLVMValue::needsSaving(value: value.getBasePointer()) ||
177 DominatingLLVMValue::needsSaving(value: value.getOffset()))
178 return true;
179 return false;
180 }
181 static saved_type save(CodeGenFunction &CGF, type value) {
182 return {.BasePtr: DominatingLLVMValue::save(CGF, value: value.getBasePointer()),
183 .ElementType: value.getElementType(), .Alignment: value.getAlignment(),
184 .Offset: DominatingLLVMValue::save(CGF, value: value.getOffset()), .EffectiveType: value.getType()};
185 }
186 static type restore(CodeGenFunction &CGF, saved_type value) {
187 return Address(DominatingLLVMValue::restore(CGF, value: value.BasePtr),
188 value.ElementType, value.Alignment, CGPointerAuthInfo(),
189 DominatingLLVMValue::restore(CGF, value: value.Offset));
190 }
191};
192
193/// A specialization of DominatingValue for RValue.
194template <> struct DominatingValue<RValue> {
195 typedef RValue type;
196 class saved_type {
197 enum Kind {
198 ScalarLiteral,
199 ScalarAddress,
200 AggregateLiteral,
201 AggregateAddress,
202 ComplexAddress
203 };
204 union {
205 struct {
206 DominatingLLVMValue::saved_type first, second;
207 } Vals;
208 DominatingValue<Address>::saved_type AggregateAddr;
209 };
210 LLVM_PREFERRED_TYPE(Kind)
211 unsigned K : 3;
212
213 saved_type(DominatingLLVMValue::saved_type Val1, unsigned K)
214 : Vals{.first: Val1, .second: DominatingLLVMValue::saved_type()}, K(K) {}
215
216 saved_type(DominatingLLVMValue::saved_type Val1,
217 DominatingLLVMValue::saved_type Val2)
218 : Vals{.first: Val1, .second: Val2}, K(ComplexAddress) {}
219
220 saved_type(DominatingValue<Address>::saved_type AggregateAddr, unsigned K)
221 : AggregateAddr(AggregateAddr), K(K) {}
222
223 public:
224 static bool needsSaving(RValue value);
225 static saved_type save(CodeGenFunction &CGF, RValue value);
226 RValue restore(CodeGenFunction &CGF);
227
228 // implementations in CGCleanup.cpp
229 };
230
231 static bool needsSaving(type value) { return saved_type::needsSaving(value); }
232 static saved_type save(CodeGenFunction &CGF, type value) {
233 return saved_type::save(CGF, value);
234 }
235 static type restore(CodeGenFunction &CGF, saved_type value) {
236 return value.restore(CGF);
237 }
238};
239
240/// A scoped helper to set the current source atom group for
241/// CGDebugInfo::addInstToCurrentSourceAtom. A source atom is a source construct
242/// that is "interesting" for debug stepping purposes. We use an atom group
243/// number to track the instruction(s) that implement the functionality for the
244/// atom, plus backup instructions/source locations.
245class ApplyAtomGroup {
246 uint64_t OriginalAtom = 0;
247 CGDebugInfo *DI = nullptr;
248
249 ApplyAtomGroup(const ApplyAtomGroup &) = delete;
250 void operator=(const ApplyAtomGroup &) = delete;
251
252public:
253 ApplyAtomGroup(CGDebugInfo *DI);
254 ~ApplyAtomGroup();
255};
256
257/// CodeGenFunction - This class organizes the per-function state that is used
258/// while generating LLVM code.
259class CodeGenFunction : public CodeGenTypeCache {
260 CodeGenFunction(const CodeGenFunction &) = delete;
261 void operator=(const CodeGenFunction &) = delete;
262
263 friend class CGCXXABI;
264 friend class clang::AsmConstraintsInfo;
265
266public:
267 /// A jump destination is an abstract label, branching to which may
268 /// require a jump out through normal cleanups.
269 struct JumpDest {
270 JumpDest() : Block(nullptr), Index(0) {}
271 JumpDest(llvm::BasicBlock *Block, EHScopeStack::stable_iterator Depth,
272 unsigned Index)
273 : Block(Block), ScopeDepth(Depth), Index(Index) {}
274
275 bool isValid() const { return Block != nullptr; }
276 llvm::BasicBlock *getBlock() const { return Block; }
277 EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; }
278 unsigned getDestIndex() const { return Index; }
279
280 // This should be used cautiously.
281 void setScopeDepth(EHScopeStack::stable_iterator depth) {
282 ScopeDepth = depth;
283 }
284
285 private:
286 llvm::BasicBlock *Block;
287 EHScopeStack::stable_iterator ScopeDepth;
288 unsigned Index;
289 };
290
291 CodeGenModule &CGM; // Per-module state.
292 const TargetInfo &Target;
293
294 // For EH/SEH outlined funclets, this field points to parent's CGF
295 CodeGenFunction *ParentCGF = nullptr;
296
297 typedef std::pair<llvm::Value *, llvm::Value *> ComplexPairTy;
298 LoopInfoStack LoopStack;
299 CGBuilderTy Builder;
300
301 // Stores variables for which we can't generate correct lifetime markers
302 // because of jumps.
303 VarBypassDetector Bypasses;
304
305 /// List of recently emitted OMPCanonicalLoops.
306 ///
307 /// Since OMPCanonicalLoops are nested inside other statements (in particular
308 /// CapturedStmt generated by OMPExecutableDirective and non-perfectly nested
309 /// loops), we cannot directly call OMPEmitOMPCanonicalLoop and receive its
310 /// llvm::CanonicalLoopInfo. Instead, we call EmitStmt and any
311 /// OMPEmitOMPCanonicalLoop called by it will add its CanonicalLoopInfo to
312 /// this stack when done. Entering a new loop requires clearing this list; it
313 /// either means we start parsing a new loop nest (in which case the previous
314 /// loop nest goes out of scope) or a second loop in the same level in which
315 /// case it would be ambiguous into which of the two (or more) loops the loop
316 /// nest would extend.
317 SmallVector<llvm::CanonicalLoopInfo *, 4> OMPLoopNestStack;
318
319 /// Stack to track the controlled convergence tokens.
320 SmallVector<llvm::ConvergenceControlInst *, 4> ConvergenceTokenStack;
321
322 /// Number of nested loop to be consumed by the last surrounding
323 /// loop-associated directive.
324 int ExpectedOMPLoopDepth = 0;
325
326 // CodeGen lambda for loops and support for ordered clause
327 typedef llvm::function_ref<void(CodeGenFunction &, const OMPLoopDirective &,
328 JumpDest)>
329 CodeGenLoopTy;
330 typedef llvm::function_ref<void(CodeGenFunction &, SourceLocation,
331 const unsigned, const bool)>
332 CodeGenOrderedTy;
333
334 // Codegen lambda for loop bounds in worksharing loop constructs
335 typedef llvm::function_ref<std::pair<LValue, LValue>(
336 CodeGenFunction &, const OMPExecutableDirective &S)>
337 CodeGenLoopBoundsTy;
338
339 // Codegen lambda for loop bounds in dispatch-based loop implementation
340 typedef llvm::function_ref<std::pair<llvm::Value *, llvm::Value *>(
341 CodeGenFunction &, const OMPExecutableDirective &S, Address LB,
342 Address UB)>
343 CodeGenDispatchBoundsTy;
344
345 /// CGBuilder insert helper. This function is called after an
346 /// instruction is created using Builder.
347 void InsertHelper(llvm::Instruction *I, const llvm::Twine &Name,
348 llvm::BasicBlock::iterator InsertPt) const;
349
350 /// CurFuncDecl - Holds the Decl for the current outermost
351 /// non-closure context.
352 const Decl *CurFuncDecl = nullptr;
353 /// CurCodeDecl - This is the inner-most code context, which includes blocks.
354 const Decl *CurCodeDecl = nullptr;
355 const CGFunctionInfo *CurFnInfo = nullptr;
356 QualType FnRetTy;
357 llvm::Function *CurFn = nullptr;
358
359 /// If a cast expression is being visited, this holds the current cast's
360 /// expression.
361 const CastExpr *CurCast = nullptr;
362
363 /// Save Parameter Decl for coroutine.
364 llvm::SmallVector<const ParmVarDecl *, 4> FnArgs;
365
366 // Holds coroutine data if the current function is a coroutine. We use a
367 // wrapper to manage its lifetime, so that we don't have to define CGCoroData
368 // in this header.
369 struct CGCoroInfo {
370 std::unique_ptr<CGCoroData> Data;
371 bool InSuspendBlock = false;
372 CGCoroInfo();
373 ~CGCoroInfo();
374 };
375 CGCoroInfo CurCoro;
376
377 bool isCoroutine() const { return CurCoro.Data != nullptr; }
378
379 bool inSuspendBlock() const {
380 return isCoroutine() && CurCoro.InSuspendBlock;
381 }
382
383 // Holds FramePtr for await_suspend wrapper generation,
384 // so that __builtin_coro_frame call can be lowered
385 // directly to value of its second argument
386 struct AwaitSuspendWrapperInfo {
387 llvm::Value *FramePtr = nullptr;
388 };
389 AwaitSuspendWrapperInfo CurAwaitSuspendWrapper;
390
391 // Generates wrapper function for `llvm.coro.await.suspend.*` intrinisics.
392 // It encapsulates SuspendExpr in a function, to separate it's body
393 // from the main coroutine to avoid miscompilations. Intrinisic
394 // is lowered to this function call in CoroSplit pass
395 // Function signature is:
396 // <type> __await_suspend_wrapper_<name>(ptr %awaiter, ptr %hdl)
397 // where type is one of (void, i1, ptr)
398 llvm::Function *generateAwaitSuspendWrapper(Twine const &CoroName,
399 Twine const &SuspendPointName,
400 CoroutineSuspendExpr const &S);
401
402 /// CurGD - The GlobalDecl for the current function being compiled.
403 GlobalDecl CurGD;
404
405 /// PrologueCleanupDepth - The cleanup depth enclosing all the
406 /// cleanups associated with the parameters.
407 EHScopeStack::stable_iterator PrologueCleanupDepth;
408
409 /// ReturnBlock - Unified return block.
410 JumpDest ReturnBlock;
411
412 /// ReturnValue - The temporary alloca to hold the return
413 /// value. This is invalid iff the function has no return value.
414 Address ReturnValue = Address::invalid();
415
416 /// ReturnValuePointer - The temporary alloca to hold a pointer to sret.
417 /// This is invalid if sret is not in use.
418 Address ReturnValuePointer = Address::invalid();
419
420 /// If a return statement is being visited, this holds the return statment's
421 /// result expression.
422 const Expr *RetExpr = nullptr;
423
424 /// Return true if a label was seen in the current scope.
425 bool hasLabelBeenSeenInCurrentScope() const {
426 if (CurLexicalScope)
427 return CurLexicalScope->hasLabels();
428 return !LabelMap.empty();
429 }
430
431 /// AllocaInsertPoint - This is an instruction in the entry block before which
432 /// we prefer to insert allocas.
433 llvm::AssertingVH<llvm::Instruction> AllocaInsertPt;
434
435private:
436 /// PostAllocaInsertPt - This is a place in the prologue where code can be
437 /// inserted that will be dominated by all the static allocas. This helps
438 /// achieve two things:
439 /// 1. Contiguity of all static allocas (within the prologue) is maintained.
440 /// 2. All other prologue code (which are dominated by static allocas) do
441 /// appear in the source order immediately after all static allocas.
442 ///
443 /// PostAllocaInsertPt will be lazily created when it is *really* required.
444 llvm::AssertingVH<llvm::Instruction> PostAllocaInsertPt = nullptr;
445
446public:
447 /// Return PostAllocaInsertPt. If it is not yet created, then insert it
448 /// immediately after AllocaInsertPt.
449 llvm::Instruction *getPostAllocaInsertPoint() {
450 if (!PostAllocaInsertPt) {
451 assert(AllocaInsertPt &&
452 "Expected static alloca insertion point at function prologue");
453 assert(AllocaInsertPt->getParent()->isEntryBlock() &&
454 "EBB should be entry block of the current code gen function");
455 PostAllocaInsertPt = AllocaInsertPt->clone();
456 PostAllocaInsertPt->setName("postallocapt");
457 PostAllocaInsertPt->insertAfter(InsertPos: AllocaInsertPt->getIterator());
458 }
459
460 return PostAllocaInsertPt;
461 }
462
463 // Try to preserve the source's name to make IR more readable.
464 llvm::Value *performAddrSpaceCast(llvm::Value *Src, llvm::Type *DestTy) {
465 return Builder.CreateAddrSpaceCast(
466 V: Src, DestTy, Name: Src->hasName() ? Src->getName() + ".ascast" : "");
467 }
468
469 /// API for captured statement code generation.
470 class CGCapturedStmtInfo {
471 public:
472 explicit CGCapturedStmtInfo(CapturedRegionKind K = CR_Default)
473 : Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {}
474 explicit CGCapturedStmtInfo(const CapturedStmt &S,
475 CapturedRegionKind K = CR_Default)
476 : Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {
477
478 RecordDecl::field_iterator Field =
479 S.getCapturedRecordDecl()->field_begin();
480 for (CapturedStmt::const_capture_iterator I = S.capture_begin(),
481 E = S.capture_end();
482 I != E; ++I, ++Field) {
483 if (I->capturesThis())
484 CXXThisFieldDecl = *Field;
485 else if (I->capturesVariable())
486 CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
487 else if (I->capturesVariableByCopy())
488 CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
489 }
490 }
491
492 virtual ~CGCapturedStmtInfo();
493
494 CapturedRegionKind getKind() const { return Kind; }
495
496 virtual void setContextValue(llvm::Value *V) { ThisValue = V; }
497 // Retrieve the value of the context parameter.
498 virtual llvm::Value *getContextValue() const { return ThisValue; }
499
500 /// Lookup the captured field decl for a variable.
501 virtual const FieldDecl *lookup(const VarDecl *VD) const {
502 return CaptureFields.lookup(Val: VD->getCanonicalDecl());
503 }
504
505 bool isCXXThisExprCaptured() const { return getThisFieldDecl() != nullptr; }
506 virtual FieldDecl *getThisFieldDecl() const { return CXXThisFieldDecl; }
507
508 static bool classof(const CGCapturedStmtInfo *) { return true; }
509
510 /// Emit the captured statement body.
511 virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) {
512 CGF.incrementProfileCounter(S);
513 CGF.EmitStmt(S);
514 }
515
516 /// Get the name of the capture helper.
517 virtual StringRef getHelperName() const { return "__captured_stmt"; }
518
519 /// Get the CaptureFields
520 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> getCaptureFields() {
521 return CaptureFields;
522 }
523
524 private:
525 /// The kind of captured statement being generated.
526 CapturedRegionKind Kind;
527
528 /// Keep the map between VarDecl and FieldDecl.
529 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields;
530
531 /// The base address of the captured record, passed in as the first
532 /// argument of the parallel region function.
533 llvm::Value *ThisValue;
534
535 /// Captured 'this' type.
536 FieldDecl *CXXThisFieldDecl;
537 };
538 CGCapturedStmtInfo *CapturedStmtInfo = nullptr;
539
540 /// RAII for correct setting/restoring of CapturedStmtInfo.
541 class CGCapturedStmtRAII {
542 private:
543 CodeGenFunction &CGF;
544 CGCapturedStmtInfo *PrevCapturedStmtInfo;
545
546 public:
547 CGCapturedStmtRAII(CodeGenFunction &CGF,
548 CGCapturedStmtInfo *NewCapturedStmtInfo)
549 : CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo) {
550 CGF.CapturedStmtInfo = NewCapturedStmtInfo;
551 }
552 ~CGCapturedStmtRAII() { CGF.CapturedStmtInfo = PrevCapturedStmtInfo; }
553 };
554
555 /// An abstract representation of regular/ObjC call/message targets.
556 class AbstractCallee {
557 /// The function declaration of the callee.
558 const Decl *CalleeDecl;
559
560 public:
561 AbstractCallee() : CalleeDecl(nullptr) {}
562 AbstractCallee(const FunctionDecl *FD) : CalleeDecl(FD) {}
563 AbstractCallee(const ObjCMethodDecl *OMD) : CalleeDecl(OMD) {}
564 bool hasFunctionDecl() const {
565 return isa_and_nonnull<FunctionDecl>(Val: CalleeDecl);
566 }
567 const Decl *getDecl() const { return CalleeDecl; }
568 unsigned getNumParams() const {
569 if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
570 return FD->getNumParams();
571 return cast<ObjCMethodDecl>(Val: CalleeDecl)->param_size();
572 }
573 const ParmVarDecl *getParamDecl(unsigned I) const {
574 if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
575 return FD->getParamDecl(i: I);
576 return *(cast<ObjCMethodDecl>(Val: CalleeDecl)->param_begin() + I);
577 }
578 };
579
580 /// Sanitizers enabled for this function.
581 SanitizerSet SanOpts;
582
583 /// True if CodeGen currently emits code implementing sanitizer checks.
584 bool IsSanitizerScope = false;
585
586 /// RAII object to set/unset CodeGenFunction::IsSanitizerScope.
587 class SanitizerScope {
588 CodeGenFunction *CGF;
589
590 public:
591 SanitizerScope(CodeGenFunction *CGF);
592 ~SanitizerScope();
593 };
594
595 /// In C++, whether we are code generating a thunk. This controls whether we
596 /// should emit cleanups.
597 bool CurFuncIsThunk = false;
598
599 /// In ARC, whether we should autorelease the return value.
600 bool AutoreleaseResult = false;
601
602 /// Whether we processed a Microsoft-style asm block during CodeGen. These can
603 /// potentially set the return value.
604 bool SawAsmBlock = false;
605
606 GlobalDecl CurSEHParent;
607
608 /// True if the current function is an outlined SEH helper. This can be a
609 /// finally block or filter expression.
610 bool IsOutlinedSEHHelper = false;
611
612 /// True if CodeGen currently emits code inside presereved access index
613 /// region.
614 bool IsInPreservedAIRegion = false;
615
616 /// True if the current statement has nomerge attribute.
617 bool InNoMergeAttributedStmt = false;
618
619 /// True if the current statement has noinline attribute.
620 bool InNoInlineAttributedStmt = false;
621
622 /// True if the current statement has always_inline attribute.
623 bool InAlwaysInlineAttributedStmt = false;
624
625 /// True if the current statement has noconvergent attribute.
626 bool InNoConvergentAttributedStmt = false;
627
628 /// The mode string from the amdgpu_av attribute on the current statement,
629 /// or empty if the attribute is not present.
630 StringRef AMDGPUAvailableVisibleMode;
631
632 /// HLSL Branch attribute.
633 HLSLControlFlowHintAttr::Spelling HLSLControlFlowAttr =
634 HLSLControlFlowHintAttr::SpellingNotCalculated;
635
636 // The CallExpr within the current statement that the musttail attribute
637 // applies to. nullptr if there is no 'musttail' on the current statement.
638 const CallExpr *MustTailCall = nullptr;
639
640 /// Returns true if a function must make progress, which means the
641 /// mustprogress attribute can be added.
642 bool checkIfFunctionMustProgress() {
643 if (CGM.getCodeGenOpts().getFiniteLoops() ==
644 CodeGenOptions::FiniteLoopsKind::Never)
645 return false;
646
647 // C++11 and later guarantees that a thread eventually will do one of the
648 // following (C++11 [intro.multithread]p24 and C++17 [intro.progress]p1):
649 // - terminate,
650 // - make a call to a library I/O function,
651 // - perform an access through a volatile glvalue, or
652 // - perform a synchronization operation or an atomic operation.
653 //
654 // Hence each function is 'mustprogress' in C++11 or later.
655 return getLangOpts().CPlusPlus11;
656 }
657
658 /// Returns true if a loop must make progress, which means the mustprogress
659 /// attribute can be added. \p HasConstantCond indicates whether the branch
660 /// condition is a known constant.
661 bool checkIfLoopMustProgress(const Expr *, bool HasEmptyBody);
662
663 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
664 llvm::Value *BlockPointer = nullptr;
665
666 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
667 FieldDecl *LambdaThisCaptureField = nullptr;
668
669 /// A mapping from NRVO variables to the flags used to indicate
670 /// when the NRVO has been applied to this variable.
671 llvm::DenseMap<const VarDecl *, llvm::Value *> NRVOFlags;
672
673 EHScopeStack EHStack;
674 llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack;
675
676 // A stack of cleanups which were added to EHStack but have to be deactivated
677 // later before being popped or emitted. These are usually deactivated on
678 // exiting a `CleanupDeactivationScope` scope. For instance, after a
679 // full-expr.
680 //
681 // These are specially useful for correctly emitting cleanups while
682 // encountering branches out of expression (through stmt-expr or coroutine
683 // suspensions).
684 struct DeferredDeactivateCleanup {
685 EHScopeStack::stable_iterator Cleanup;
686 llvm::Instruction *DominatingIP;
687 };
688 llvm::SmallVector<DeferredDeactivateCleanup> DeferredDeactivationCleanupStack;
689
690 // Enters a new scope for capturing cleanups which are deferred to be
691 // deactivated, all of which will be deactivated once the scope is exited.
692 struct CleanupDeactivationScope {
693 CodeGenFunction &CGF;
694 size_t OldDeactivateCleanupStackSize;
695 bool Deactivated;
696 CleanupDeactivationScope(CodeGenFunction &CGF)
697 : CGF(CGF), OldDeactivateCleanupStackSize(
698 CGF.DeferredDeactivationCleanupStack.size()),
699 Deactivated(false) {}
700
701 void ForceDeactivate() {
702 assert(!Deactivated && "Deactivating already deactivated scope");
703 auto &Stack = CGF.DeferredDeactivationCleanupStack;
704 for (size_t I = Stack.size(); I > OldDeactivateCleanupStackSize; I--) {
705 CGF.DeactivateCleanupBlock(Cleanup: Stack[I - 1].Cleanup,
706 DominatingIP: Stack[I - 1].DominatingIP);
707 Stack[I - 1].DominatingIP->eraseFromParent();
708 }
709 Stack.resize(N: OldDeactivateCleanupStackSize);
710 Deactivated = true;
711 }
712
713 ~CleanupDeactivationScope() {
714 if (Deactivated)
715 return;
716 ForceDeactivate();
717 }
718 };
719
720 llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack;
721
722 llvm::Instruction *CurrentFuncletPad = nullptr;
723
724 class CallLifetimeEnd final : public EHScopeStack::Cleanup {
725 bool isRedundantBeforeReturn() override { return true; }
726
727 llvm::Value *Addr;
728
729 public:
730 CallLifetimeEnd(RawAddress addr) : Addr(addr.getPointer()) {}
731
732 void Emit(CodeGenFunction &CGF, Flags flags) override {
733 CGF.EmitLifetimeEnd(Addr);
734 }
735 };
736
737 // We are using objects of this 'cleanup' class to emit fake.use calls
738 // for -fextend-variable-liveness. They are placed at the end of a variable's
739 // scope analogous to lifetime markers.
740 class FakeUse final : public EHScopeStack::Cleanup {
741 Address Addr;
742
743 public:
744 FakeUse(Address addr) : Addr(addr) {}
745
746 void Emit(CodeGenFunction &CGF, Flags flags) override {
747 CGF.EmitFakeUse(Addr);
748 }
749 };
750
751 /// Header for data within LifetimeExtendedCleanupStack.
752 struct alignas(uint64_t) LifetimeExtendedCleanupHeader {
753 /// The size of the following cleanup object.
754 unsigned Size;
755 /// The kind of cleanup to push.
756 LLVM_PREFERRED_TYPE(CleanupKind)
757 unsigned Kind : 31;
758 /// Whether this is a conditional cleanup.
759 LLVM_PREFERRED_TYPE(bool)
760 unsigned IsConditional : 1;
761
762 size_t getSize() const { return Size; }
763 CleanupKind getKind() const { return (CleanupKind)Kind; }
764 bool isConditional() const { return IsConditional; }
765 };
766
767 /// i32s containing the indexes of the cleanup destinations.
768 RawAddress NormalCleanupDest = RawAddress::invalid();
769
770 unsigned NextCleanupDestIndex = 1;
771
772 /// EHResumeBlock - Unified block containing a call to llvm.eh.resume.
773 llvm::BasicBlock *EHResumeBlock = nullptr;
774
775 /// The exception slot. All landing pads write the current exception pointer
776 /// into this alloca.
777 llvm::Value *ExceptionSlot = nullptr;
778
779 /// The selector slot. Under the MandatoryCleanup model, all landing pads
780 /// write the current selector value into this alloca.
781 llvm::AllocaInst *EHSelectorSlot = nullptr;
782
783 /// A stack of exception code slots. Entering an __except block pushes a slot
784 /// on the stack and leaving pops one. The __exception_code() intrinsic loads
785 /// a value from the top of the stack.
786 SmallVector<Address, 1> SEHCodeSlotStack;
787
788 /// Value returned by __exception_info intrinsic.
789 llvm::Value *SEHInfo = nullptr;
790
791 /// Emits a landing pad for the current EH stack.
792 llvm::BasicBlock *EmitLandingPad();
793
794 llvm::BasicBlock *getInvokeDestImpl();
795
796 /// Parent loop-based directive for scan directive.
797 const OMPExecutableDirective *OMPParentLoopDirectiveForScan = nullptr;
798 llvm::BasicBlock *OMPBeforeScanBlock = nullptr;
799 llvm::BasicBlock *OMPAfterScanBlock = nullptr;
800 llvm::BasicBlock *OMPScanExitBlock = nullptr;
801 llvm::BasicBlock *OMPScanDispatch = nullptr;
802 bool OMPFirstScanLoop = false;
803
804 /// Manages parent directive for scan directives.
805 class ParentLoopDirectiveForScanRegion {
806 CodeGenFunction &CGF;
807 const OMPExecutableDirective *ParentLoopDirectiveForScan;
808
809 public:
810 ParentLoopDirectiveForScanRegion(
811 CodeGenFunction &CGF,
812 const OMPExecutableDirective &ParentLoopDirectiveForScan)
813 : CGF(CGF),
814 ParentLoopDirectiveForScan(CGF.OMPParentLoopDirectiveForScan) {
815 CGF.OMPParentLoopDirectiveForScan = &ParentLoopDirectiveForScan;
816 }
817 ~ParentLoopDirectiveForScanRegion() {
818 CGF.OMPParentLoopDirectiveForScan = ParentLoopDirectiveForScan;
819 }
820 };
821
822 template <class T>
823 typename DominatingValue<T>::saved_type saveValueInCond(T value) {
824 return DominatingValue<T>::save(*this, value);
825 }
826
827 class CGFPOptionsRAII {
828 public:
829 CGFPOptionsRAII(CodeGenFunction &CGF, FPOptions FPFeatures);
830 CGFPOptionsRAII(CodeGenFunction &CGF, const Expr *E);
831 ~CGFPOptionsRAII();
832
833 private:
834 void ConstructorHelper(FPOptions FPFeatures);
835 CodeGenFunction &CGF;
836 FPOptions OldFPFeatures;
837 llvm::fp::ExceptionBehavior OldExcept;
838 llvm::RoundingMode OldRounding;
839 std::optional<CGBuilderTy::FastMathFlagGuard> FMFGuard;
840 };
841 FPOptions CurFPFeatures;
842
843 class CGAtomicOptionsRAII {
844 public:
845 CGAtomicOptionsRAII(CodeGenModule &CGM_, AtomicOptions AO)
846 : CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {
847 CGM.setAtomicOpts(AO);
848 }
849 CGAtomicOptionsRAII(CodeGenModule &CGM_, const AtomicAttr *AA)
850 : CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {
851 if (!AA)
852 return;
853 AtomicOptions AO = SavedAtomicOpts;
854 for (auto Option : AA->atomicOptions()) {
855 switch (Option) {
856 case AtomicAttr::remote_memory:
857 AO.remote_memory = true;
858 break;
859 case AtomicAttr::no_remote_memory:
860 AO.remote_memory = false;
861 break;
862 case AtomicAttr::fine_grained_memory:
863 AO.fine_grained_memory = true;
864 break;
865 case AtomicAttr::no_fine_grained_memory:
866 AO.fine_grained_memory = false;
867 break;
868 case AtomicAttr::ignore_denormal_mode:
869 AO.ignore_denormal_mode = true;
870 break;
871 case AtomicAttr::no_ignore_denormal_mode:
872 AO.ignore_denormal_mode = false;
873 break;
874 }
875 }
876 CGM.setAtomicOpts(AO);
877 }
878
879 CGAtomicOptionsRAII(const CGAtomicOptionsRAII &) = delete;
880 CGAtomicOptionsRAII &operator=(const CGAtomicOptionsRAII &) = delete;
881 ~CGAtomicOptionsRAII() { CGM.setAtomicOpts(SavedAtomicOpts); }
882
883 private:
884 CodeGenModule &CGM;
885 AtomicOptions SavedAtomicOpts;
886 };
887
888public:
889 /// ObjCEHValueStack - Stack of Objective-C exception values, used for
890 /// rethrows.
891 SmallVector<llvm::Value *, 8> ObjCEHValueStack;
892
893 /// A class controlling the emission of a finally block.
894 class FinallyInfo {
895 /// Where the catchall's edge through the cleanup should go.
896 JumpDest RethrowDest;
897
898 /// A function to call to enter the catch.
899 llvm::FunctionCallee BeginCatchFn;
900
901 /// An i1 variable indicating whether or not the @finally is
902 /// running for an exception.
903 llvm::AllocaInst *ForEHVar = nullptr;
904
905 /// An i8* variable into which the exception pointer to rethrow
906 /// has been saved.
907 llvm::AllocaInst *SavedExnVar = nullptr;
908
909 public:
910 void enter(CodeGenFunction &CGF, const Stmt *Finally,
911 llvm::FunctionCallee beginCatchFn,
912 llvm::FunctionCallee endCatchFn, llvm::FunctionCallee rethrowFn);
913 void exit(CodeGenFunction &CGF);
914 };
915
916 /// Returns true inside SEH __try blocks.
917 bool isSEHTryScope() const { return !SEHTryEpilogueStack.empty(); }
918
919 /// Returns true while emitting a cleanuppad.
920 bool isCleanupPadScope() const {
921 return CurrentFuncletPad && isa<llvm::CleanupPadInst>(Val: CurrentFuncletPad);
922 }
923
924 /// pushFullExprCleanup - Push a cleanup to be run at the end of the
925 /// current full-expression. Safe against the possibility that
926 /// we're currently inside a conditionally-evaluated expression.
927 template <class T, class... As>
928 void pushFullExprCleanup(CleanupKind kind, As... A) {
929 // If we're not in a conditional branch, or if none of the
930 // arguments requires saving, then use the unconditional cleanup.
931 if (!isInConditionalBranch())
932 return EHStack.pushCleanup<T>(kind, A...);
933
934 // Stash values in a tuple so we can guarantee the order of saves.
935 typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
936 SavedTuple Saved{saveValueInCond(A)...};
937
938 typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
939 EHStack.pushCleanupTuple<CleanupType>(kind, Saved);
940 initFullExprCleanup();
941 }
942
943 /// Queue a cleanup to be pushed after finishing the current full-expression,
944 /// potentially with an active flag.
945 template <class T, class... As>
946 void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) {
947 if (!isInConditionalBranch())
948 return pushCleanupAfterFullExprWithActiveFlag<T>(
949 Kind, RawAddress::invalid(), A...);
950
951 RawAddress ActiveFlag = createCleanupActiveFlag();
952 assert(!DominatingValue<Address>::needsSaving(ActiveFlag) &&
953 "cleanup active flag should never need saving");
954
955 typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
956 SavedTuple Saved{saveValueInCond(A)...};
957
958 typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
959 pushCleanupAfterFullExprWithActiveFlag<CleanupType>(Kind, ActiveFlag,
960 Saved);
961 }
962
963 template <class T, class... As>
964 void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind,
965 RawAddress ActiveFlag, As... A) {
966 LifetimeExtendedCleanupHeader Header = {.Size: sizeof(T), .Kind: Kind,
967 .IsConditional: ActiveFlag.isValid()};
968
969 size_t OldSize = LifetimeExtendedCleanupStack.size();
970 LifetimeExtendedCleanupStack.resize(
971 N: LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size +
972 (Header.IsConditional ? sizeof(ActiveFlag) : 0));
973
974 static_assert((alignof(LifetimeExtendedCleanupHeader) == alignof(T)) &&
975 (alignof(T) == alignof(RawAddress)),
976 "Cleanup will be allocated on misaligned address");
977 char *Buffer = &LifetimeExtendedCleanupStack[OldSize];
978 new (Buffer) LifetimeExtendedCleanupHeader(Header);
979 new (Buffer + sizeof(Header)) T(A...);
980 if (Header.IsConditional)
981 new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag);
982 }
983
984 // Push a cleanup onto EHStack and deactivate it later. It is usually
985 // deactivated when exiting a `CleanupDeactivationScope` (for example: after a
986 // full expression).
987 template <class T, class... As>
988 void pushCleanupAndDeferDeactivation(CleanupKind Kind, As... A) {
989 // Placeholder dominating IP for this cleanup.
990 llvm::Instruction *DominatingIP =
991 Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
992 EHStack.pushCleanup<T>(Kind, A...);
993 DeferredDeactivationCleanupStack.push_back(
994 Elt: {.Cleanup: EHStack.stable_begin(), .DominatingIP: DominatingIP});
995 }
996
997 /// Set up the last cleanup that was pushed as a conditional
998 /// full-expression cleanup.
999 void initFullExprCleanup() {
1000 initFullExprCleanupWithFlag(ActiveFlag: createCleanupActiveFlag());
1001 }
1002
1003 void initFullExprCleanupWithFlag(RawAddress ActiveFlag);
1004 RawAddress createCleanupActiveFlag();
1005
1006 /// PushDestructorCleanup - Push a cleanup to call the
1007 /// complete-object destructor of an object of the given type at the
1008 /// given address. Does nothing if T is not a C++ class type with a
1009 /// non-trivial destructor.
1010 void PushDestructorCleanup(QualType T, Address Addr);
1011
1012 /// PushDestructorCleanup - Push a cleanup to call the
1013 /// complete-object variant of the given destructor on the object at
1014 /// the given address.
1015 void PushDestructorCleanup(const CXXDestructorDecl *Dtor, QualType T,
1016 Address Addr);
1017
1018 /// PopCleanupBlock - Will pop the cleanup entry on the stack and
1019 /// process all branch fixups.
1020 void PopCleanupBlock(bool FallThroughIsBranchThrough = false,
1021 bool ForDeactivation = false);
1022
1023 /// DeactivateCleanupBlock - Deactivates the given cleanup block.
1024 /// The block cannot be reactivated. Pops it if it's the top of the
1025 /// stack.
1026 ///
1027 /// \param DominatingIP - An instruction which is known to
1028 /// dominate the current IP (if set) and which lies along
1029 /// all paths of execution between the current IP and the
1030 /// the point at which the cleanup comes into scope.
1031 void DeactivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
1032 llvm::Instruction *DominatingIP);
1033
1034 /// ActivateCleanupBlock - Activates an initially-inactive cleanup.
1035 /// Cannot be used to resurrect a deactivated cleanup.
1036 ///
1037 /// \param DominatingIP - An instruction which is known to
1038 /// dominate the current IP (if set) and which lies along
1039 /// all paths of execution between the current IP and the
1040 /// the point at which the cleanup comes into scope.
1041 void ActivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
1042 llvm::Instruction *DominatingIP);
1043
1044 /// Enters a new scope for capturing cleanups, all of which
1045 /// will be executed once the scope is exited.
1046 class RunCleanupsScope {
1047 EHScopeStack::stable_iterator CleanupStackDepth, OldCleanupScopeDepth;
1048 size_t LifetimeExtendedCleanupStackSize;
1049 CleanupDeactivationScope DeactivateCleanups;
1050 bool OldDidCallStackSave;
1051
1052 protected:
1053 bool PerformCleanup;
1054
1055 private:
1056 RunCleanupsScope(const RunCleanupsScope &) = delete;
1057 void operator=(const RunCleanupsScope &) = delete;
1058
1059 protected:
1060 CodeGenFunction &CGF;
1061
1062 public:
1063 /// Enter a new cleanup scope.
1064 explicit RunCleanupsScope(CodeGenFunction &CGF)
1065 : DeactivateCleanups(CGF), PerformCleanup(true), CGF(CGF) {
1066 CleanupStackDepth = CGF.EHStack.stable_begin();
1067 LifetimeExtendedCleanupStackSize =
1068 CGF.LifetimeExtendedCleanupStack.size();
1069 OldDidCallStackSave = CGF.DidCallStackSave;
1070 CGF.DidCallStackSave = false;
1071 OldCleanupScopeDepth = CGF.CurrentCleanupScopeDepth;
1072 CGF.CurrentCleanupScopeDepth = CleanupStackDepth;
1073 }
1074
1075 /// Exit this cleanup scope, emitting any accumulated cleanups.
1076 ~RunCleanupsScope() {
1077 if (PerformCleanup)
1078 ForceCleanup();
1079 }
1080
1081 /// Determine whether this scope requires any cleanups.
1082 bool requiresCleanups() const {
1083 return CGF.EHStack.stable_begin() != CleanupStackDepth;
1084 }
1085
1086 /// Force the emission of cleanups now, instead of waiting
1087 /// until this object is destroyed.
1088 /// \param ValuesToReload - A list of values that need to be available at
1089 /// the insertion point after cleanup emission. If cleanup emission created
1090 /// a shared cleanup block, these value pointers will be rewritten.
1091 /// Otherwise, they not will be modified.
1092 void
1093 ForceCleanup(std::initializer_list<llvm::Value **> ValuesToReload = {}) {
1094 assert(PerformCleanup && "Already forced cleanup");
1095 CGF.DidCallStackSave = OldDidCallStackSave;
1096 DeactivateCleanups.ForceDeactivate();
1097 CGF.PopCleanupBlocks(OldCleanupStackSize: CleanupStackDepth, OldLifetimeExtendedStackSize: LifetimeExtendedCleanupStackSize,
1098 ValuesToReload);
1099 PerformCleanup = false;
1100 CGF.CurrentCleanupScopeDepth = OldCleanupScopeDepth;
1101 }
1102 };
1103
1104 // Cleanup stack depth of the RunCleanupsScope that was pushed most recently.
1105 EHScopeStack::stable_iterator CurrentCleanupScopeDepth =
1106 EHScopeStack::stable_end();
1107
1108 class LexicalScope : public RunCleanupsScope {
1109 SourceRange Range;
1110 SmallVector<const LabelDecl *, 4> Labels;
1111 LexicalScope *ParentScope;
1112
1113 LexicalScope(const LexicalScope &) = delete;
1114 void operator=(const LexicalScope &) = delete;
1115
1116 public:
1117 /// Enter a new cleanup scope.
1118 explicit LexicalScope(CodeGenFunction &CGF, SourceRange Range);
1119
1120 void addLabel(const LabelDecl *label) {
1121 assert(PerformCleanup && "adding label to dead scope?");
1122 Labels.push_back(Elt: label);
1123 }
1124
1125 /// Exit this cleanup scope, emitting any accumulated
1126 /// cleanups.
1127 ~LexicalScope();
1128
1129 /// Force the emission of cleanups now, instead of waiting
1130 /// until this object is destroyed.
1131 void ForceCleanup() {
1132 CGF.CurLexicalScope = ParentScope;
1133 RunCleanupsScope::ForceCleanup();
1134
1135 if (!Labels.empty())
1136 rescopeLabels();
1137 }
1138
1139 bool hasLabels() const { return !Labels.empty(); }
1140
1141 void rescopeLabels();
1142 };
1143
1144 typedef llvm::DenseMap<const Decl *, Address> DeclMapTy;
1145
1146 /// The class used to assign some variables some temporarily addresses.
1147 class OMPMapVars {
1148 DeclMapTy SavedLocals;
1149 DeclMapTy SavedTempAddresses;
1150 OMPMapVars(const OMPMapVars &) = delete;
1151 void operator=(const OMPMapVars &) = delete;
1152
1153 public:
1154 explicit OMPMapVars() = default;
1155 ~OMPMapVars() {
1156 assert(SavedLocals.empty() && "Did not restored original addresses.");
1157 };
1158
1159 /// Sets the address of the variable \p LocalVD to be \p TempAddr in
1160 /// function \p CGF.
1161 /// \return true if at least one variable was set already, false otherwise.
1162 bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD,
1163 Address TempAddr) {
1164 LocalVD = LocalVD->getCanonicalDecl();
1165 // Only save it once.
1166 if (SavedLocals.count(Val: LocalVD))
1167 return false;
1168
1169 // Copy the existing local entry to SavedLocals.
1170 auto it = CGF.LocalDeclMap.find(Val: LocalVD);
1171 if (it != CGF.LocalDeclMap.end())
1172 SavedLocals.try_emplace(Key: LocalVD, Args&: it->second);
1173 else
1174 SavedLocals.try_emplace(Key: LocalVD, Args: Address::invalid());
1175
1176 // Generate the private entry.
1177 QualType VarTy = LocalVD->getType();
1178 if (VarTy->isReferenceType()) {
1179 Address Temp = CGF.CreateMemTemp(T: VarTy);
1180 CGF.Builder.CreateStore(Val: TempAddr.emitRawPointer(CGF), Addr: Temp);
1181 TempAddr = Temp;
1182 }
1183 SavedTempAddresses.try_emplace(Key: LocalVD, Args&: TempAddr);
1184
1185 return true;
1186 }
1187
1188 /// Applies new addresses to the list of the variables.
1189 /// \return true if at least one variable is using new address, false
1190 /// otherwise.
1191 bool apply(CodeGenFunction &CGF) {
1192 copyInto(Src: SavedTempAddresses, Dest&: CGF.LocalDeclMap);
1193 SavedTempAddresses.clear();
1194 return !SavedLocals.empty();
1195 }
1196
1197 /// Restores original addresses of the variables.
1198 void restore(CodeGenFunction &CGF) {
1199 if (!SavedLocals.empty()) {
1200 copyInto(Src: SavedLocals, Dest&: CGF.LocalDeclMap);
1201 SavedLocals.clear();
1202 }
1203 }
1204
1205 private:
1206 /// Copy all the entries in the source map over the corresponding
1207 /// entries in the destination, which must exist.
1208 static void copyInto(const DeclMapTy &Src, DeclMapTy &Dest) {
1209 for (auto &[Decl, Addr] : Src) {
1210 if (!Addr.isValid())
1211 Dest.erase(Val: Decl);
1212 else
1213 Dest.insert_or_assign(Key: Decl, Val: Addr);
1214 }
1215 }
1216 };
1217
1218 /// The scope used to remap some variables as private in the OpenMP loop body
1219 /// (or other captured region emitted without outlining), and to restore old
1220 /// vars back on exit.
1221 class OMPPrivateScope : public RunCleanupsScope {
1222 OMPMapVars MappedVars;
1223 OMPPrivateScope(const OMPPrivateScope &) = delete;
1224 void operator=(const OMPPrivateScope &) = delete;
1225
1226 public:
1227 /// Enter a new OpenMP private scope.
1228 explicit OMPPrivateScope(CodeGenFunction &CGF) : RunCleanupsScope(CGF) {}
1229
1230 /// Registers \p LocalVD variable as a private with \p Addr as the address
1231 /// of the corresponding private variable. \p
1232 /// PrivateGen is the address of the generated private variable.
1233 /// \return true if the variable is registered as private, false if it has
1234 /// been privatized already.
1235 bool addPrivate(const VarDecl *LocalVD, Address Addr) {
1236 assert(PerformCleanup && "adding private to dead scope");
1237 return MappedVars.setVarAddr(CGF, LocalVD, TempAddr: Addr);
1238 }
1239
1240 /// Privatizes local variables previously registered as private.
1241 /// Registration is separate from the actual privatization to allow
1242 /// initializers use values of the original variables, not the private one.
1243 /// This is important, for example, if the private variable is a class
1244 /// variable initialized by a constructor that references other private
1245 /// variables. But at initialization original variables must be used, not
1246 /// private copies.
1247 /// \return true if at least one variable was privatized, false otherwise.
1248 bool Privatize() { return MappedVars.apply(CGF); }
1249
1250 void ForceCleanup() {
1251 RunCleanupsScope::ForceCleanup();
1252 restoreMap();
1253 }
1254
1255 /// Exit scope - all the mapped variables are restored.
1256 ~OMPPrivateScope() {
1257 if (PerformCleanup)
1258 ForceCleanup();
1259 }
1260
1261 /// Checks if the global variable is captured in current function.
1262 bool isGlobalVarCaptured(const VarDecl *VD) const {
1263 VD = VD->getCanonicalDecl();
1264 return !VD->isLocalVarDeclOrParm() && CGF.LocalDeclMap.count(Val: VD) > 0;
1265 }
1266
1267 /// Restore all mapped variables w/o clean up. This is usefully when we want
1268 /// to reference the original variables but don't want the clean up because
1269 /// that could emit lifetime end too early, causing backend issue #56913.
1270 void restoreMap() { MappedVars.restore(CGF); }
1271 };
1272
1273 /// Save/restore original map of previously emitted local vars in case when we
1274 /// need to duplicate emission of the same code several times in the same
1275 /// function for OpenMP code.
1276 class OMPLocalDeclMapRAII {
1277 CodeGenFunction &CGF;
1278 DeclMapTy SavedMap;
1279
1280 public:
1281 OMPLocalDeclMapRAII(CodeGenFunction &CGF)
1282 : CGF(CGF), SavedMap(CGF.LocalDeclMap) {}
1283 ~OMPLocalDeclMapRAII() { SavedMap.swap(RHS&: CGF.LocalDeclMap); }
1284 };
1285
1286 /// Takes the old cleanup stack size and emits the cleanup blocks
1287 /// that have been added.
1288 void
1289 PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1290 std::initializer_list<llvm::Value **> ValuesToReload = {});
1291
1292 /// Takes the old cleanup stack size and emits the cleanup blocks
1293 /// that have been added, then adds all lifetime-extended cleanups from
1294 /// the given position to the stack.
1295 void
1296 PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1297 size_t OldLifetimeExtendedStackSize,
1298 std::initializer_list<llvm::Value **> ValuesToReload = {});
1299
1300 void ResolveBranchFixups(llvm::BasicBlock *Target);
1301
1302 /// The given basic block lies in the current EH scope, but may be a
1303 /// target of a potentially scope-crossing jump; get a stable handle
1304 /// to which we can perform this jump later.
1305 JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target) {
1306 return JumpDest(Target, EHStack.getInnermostNormalCleanup(),
1307 NextCleanupDestIndex++);
1308 }
1309
1310 /// The given basic block lies in the current EH scope, but may be a
1311 /// target of a potentially scope-crossing jump; get a stable handle
1312 /// to which we can perform this jump later.
1313 JumpDest getJumpDestInCurrentScope(StringRef Name = StringRef()) {
1314 return getJumpDestInCurrentScope(Target: createBasicBlock(name: Name));
1315 }
1316
1317 /// EmitBranchThroughCleanup - Emit a branch from the current insert
1318 /// block through the normal cleanup handling code (if any) and then
1319 /// on to \arg Dest.
1320 void EmitBranchThroughCleanup(JumpDest Dest);
1321
1322 /// isObviouslyBranchWithoutCleanups - Return true if a branch to the
1323 /// specified destination obviously has no cleanups to run. 'false' is always
1324 /// a conservatively correct answer for this method.
1325 bool isObviouslyBranchWithoutCleanups(JumpDest Dest) const;
1326
1327 /// popCatchScope - Pops the catch scope at the top of the EHScope
1328 /// stack, emitting any required code (other than the catch handlers
1329 /// themselves).
1330 void popCatchScope();
1331
1332 // This function should be called after emitting all catch clauses and none
1333 // of them were 'catch-all' clauses.
1334 // Because in wasm we merge all catch clauses into one big catchpad, in case
1335 // none of the types in catch handlers matches after we test against each of
1336 // them, we should unwind to the next EH enclosing scope. We generate a call
1337 // to rethrow function here to do that.
1338 void WasmEmitFallthroughRethrow(llvm::BasicBlock *WasmCatchStartBlock);
1339
1340 llvm::BasicBlock *getEHResumeBlock(bool isCleanup);
1341 llvm::BasicBlock *getEHDispatchBlock(EHScopeStack::stable_iterator scope);
1342 llvm::BasicBlock *
1343 getFuncletEHDispatchBlock(EHScopeStack::stable_iterator scope);
1344
1345 /// An object to manage conditionally-evaluated expressions.
1346 class ConditionalEvaluation {
1347 llvm::BasicBlock *StartBB;
1348
1349 public:
1350 ConditionalEvaluation(CodeGenFunction &CGF)
1351 : StartBB(CGF.Builder.GetInsertBlock()) {}
1352
1353 void begin(CodeGenFunction &CGF) {
1354 assert(CGF.OutermostConditional != this);
1355 if (!CGF.OutermostConditional)
1356 CGF.OutermostConditional = this;
1357 }
1358
1359 void end(CodeGenFunction &CGF) {
1360 assert(CGF.OutermostConditional != nullptr);
1361 if (CGF.OutermostConditional == this)
1362 CGF.OutermostConditional = nullptr;
1363 }
1364
1365 /// Returns a block which will be executed prior to each
1366 /// evaluation of the conditional code.
1367 llvm::BasicBlock *getStartingBlock() const { return StartBB; }
1368 };
1369
1370 /// isInConditionalBranch - Return true if we're currently emitting
1371 /// one branch or the other of a conditional expression.
1372 bool isInConditionalBranch() const { return OutermostConditional != nullptr; }
1373
1374 void setBeforeOutermostConditional(llvm::Value *value, Address addr,
1375 CodeGenFunction &CGF) {
1376 assert(isInConditionalBranch());
1377 llvm::BasicBlock *block = OutermostConditional->getStartingBlock();
1378 auto store = new llvm::StoreInst(value, addr.emitRawPointer(CGF),
1379 block->back().getIterator());
1380 store->setAlignment(addr.getAlignment().getAsAlign());
1381 }
1382
1383 /// An RAII object to record that we're evaluating a statement
1384 /// expression.
1385 class StmtExprEvaluation {
1386 CodeGenFunction &CGF;
1387
1388 /// We have to save the outermost conditional: cleanups in a
1389 /// statement expression aren't conditional just because the
1390 /// StmtExpr is.
1391 ConditionalEvaluation *SavedOutermostConditional;
1392
1393 public:
1394 StmtExprEvaluation(CodeGenFunction &CGF)
1395 : CGF(CGF), SavedOutermostConditional(CGF.OutermostConditional) {
1396 CGF.OutermostConditional = nullptr;
1397 }
1398
1399 ~StmtExprEvaluation() {
1400 CGF.OutermostConditional = SavedOutermostConditional;
1401 CGF.EnsureInsertPoint();
1402 }
1403 };
1404
1405 /// An object which temporarily prevents a value from being
1406 /// destroyed by aggressive peephole optimizations that assume that
1407 /// all uses of a value have been realized in the IR.
1408 class PeepholeProtection {
1409 llvm::Instruction *Inst = nullptr;
1410 friend class CodeGenFunction;
1411
1412 public:
1413 PeepholeProtection() = default;
1414 };
1415
1416 /// A non-RAII class containing all the information about a bound
1417 /// opaque value. OpaqueValueMapping, below, is a RAII wrapper for
1418 /// this which makes individual mappings very simple; using this
1419 /// class directly is useful when you have a variable number of
1420 /// opaque values or don't want the RAII functionality for some
1421 /// reason.
1422 class OpaqueValueMappingData {
1423 const OpaqueValueExpr *OpaqueValue;
1424 bool BoundLValue;
1425 CodeGenFunction::PeepholeProtection Protection;
1426
1427 OpaqueValueMappingData(const OpaqueValueExpr *ov, bool boundLValue)
1428 : OpaqueValue(ov), BoundLValue(boundLValue) {}
1429
1430 public:
1431 OpaqueValueMappingData() : OpaqueValue(nullptr) {}
1432
1433 static bool shouldBindAsLValue(const Expr *expr) {
1434 // gl-values should be bound as l-values for obvious reasons.
1435 // Records should be bound as l-values because IR generation
1436 // always keeps them in memory. Expressions of function type
1437 // act exactly like l-values but are formally required to be
1438 // r-values in C.
1439 return expr->isGLValue() || expr->getType()->isFunctionType() ||
1440 hasAggregateEvaluationKind(T: expr->getType());
1441 }
1442
1443 static OpaqueValueMappingData
1444 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const Expr *e) {
1445 if (shouldBindAsLValue(expr: ov))
1446 return bind(CGF, ov, lv: CGF.EmitLValue(E: e));
1447 return bind(CGF, ov, rv: CGF.EmitAnyExpr(E: e));
1448 }
1449
1450 static OpaqueValueMappingData
1451 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const LValue &lv) {
1452 assert(shouldBindAsLValue(ov));
1453 CGF.OpaqueLValues.insert(KV: std::make_pair(x&: ov, y: lv));
1454 return OpaqueValueMappingData(ov, true);
1455 }
1456
1457 static OpaqueValueMappingData
1458 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const RValue &rv) {
1459 assert(!shouldBindAsLValue(ov));
1460 CGF.OpaqueRValues.insert(KV: std::make_pair(x&: ov, y: rv));
1461
1462 OpaqueValueMappingData data(ov, false);
1463
1464 // Work around an extremely aggressive peephole optimization in
1465 // EmitScalarConversion which assumes that all other uses of a
1466 // value are extant.
1467 data.Protection = CGF.protectFromPeepholes(rvalue: rv);
1468
1469 return data;
1470 }
1471
1472 bool isValid() const { return OpaqueValue != nullptr; }
1473 void clear() { OpaqueValue = nullptr; }
1474
1475 void unbind(CodeGenFunction &CGF) {
1476 assert(OpaqueValue && "no data to unbind!");
1477
1478 if (BoundLValue) {
1479 CGF.OpaqueLValues.erase(Val: OpaqueValue);
1480 } else {
1481 CGF.OpaqueRValues.erase(Val: OpaqueValue);
1482 CGF.unprotectFromPeepholes(protection: Protection);
1483 }
1484 }
1485 };
1486
1487 /// An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
1488 class OpaqueValueMapping {
1489 CodeGenFunction &CGF;
1490 OpaqueValueMappingData Data;
1491
1492 public:
1493 static bool shouldBindAsLValue(const Expr *expr) {
1494 return OpaqueValueMappingData::shouldBindAsLValue(expr);
1495 }
1496
1497 /// Build the opaque value mapping for the given conditional
1498 /// operator if it's the GNU ?: extension. This is a common
1499 /// enough pattern that the convenience operator is really
1500 /// helpful.
1501 ///
1502 OpaqueValueMapping(CodeGenFunction &CGF,
1503 const AbstractConditionalOperator *op)
1504 : CGF(CGF) {
1505 if (isa<ConditionalOperator>(Val: op))
1506 // Leave Data empty.
1507 return;
1508
1509 const BinaryConditionalOperator *e = cast<BinaryConditionalOperator>(Val: op);
1510 Data = OpaqueValueMappingData::bind(CGF, ov: e->getOpaqueValue(),
1511 e: e->getCommon());
1512 }
1513
1514 /// Build the opaque value mapping for an OpaqueValueExpr whose source
1515 /// expression is set to the expression the OVE represents.
1516 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *OV)
1517 : CGF(CGF) {
1518 if (OV) {
1519 assert(OV->getSourceExpr() && "wrong form of OpaqueValueMapping used "
1520 "for OVE with no source expression");
1521 Data = OpaqueValueMappingData::bind(CGF, ov: OV, e: OV->getSourceExpr());
1522 }
1523 }
1524
1525 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,
1526 LValue lvalue)
1527 : CGF(CGF),
1528 Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, lv: lvalue)) {}
1529
1530 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,
1531 RValue rvalue)
1532 : CGF(CGF),
1533 Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, rv: rvalue)) {}
1534
1535 void pop() {
1536 Data.unbind(CGF);
1537 Data.clear();
1538 }
1539
1540 ~OpaqueValueMapping() {
1541 if (Data.isValid())
1542 Data.unbind(CGF);
1543 }
1544 };
1545
1546private:
1547 CGDebugInfo *DebugInfo;
1548 /// Used to create unique names for artificial VLA size debug info variables.
1549 unsigned VLAExprCounter = 0;
1550 bool DisableDebugInfo = false;
1551
1552 /// DidCallStackSave - Whether llvm.stacksave has been called. Used to avoid
1553 /// calling llvm.stacksave for multiple VLAs in the same scope.
1554 bool DidCallStackSave = false;
1555
1556 /// IndirectBranch - The first time an indirect goto is seen we create a block
1557 /// with an indirect branch. Every time we see the address of a label taken,
1558 /// we add the label to the indirect goto. Every subsequent indirect goto is
1559 /// codegen'd as a jump to the IndirectBranch's basic block.
1560 llvm::IndirectBrInst *IndirectBranch = nullptr;
1561
1562 /// LocalDeclMap - This keeps track of the LLVM allocas or globals for local C
1563 /// decls.
1564 DeclMapTy LocalDeclMap;
1565
1566 // Keep track of the cleanups for callee-destructed parameters pushed to the
1567 // cleanup stack so that they can be deactivated later.
1568 llvm::DenseMap<const ParmVarDecl *, EHScopeStack::stable_iterator>
1569 CalleeDestructedParamCleanups;
1570
1571 /// SizeArguments - If a ParmVarDecl had the pass_object_size attribute, this
1572 /// will contain a mapping from said ParmVarDecl to its implicit "object_size"
1573 /// parameter.
1574 llvm::SmallDenseMap<const ParmVarDecl *, const ImplicitParamDecl *, 2>
1575 SizeArguments;
1576
1577 /// Track escaped local variables with auto storage. Used during SEH
1578 /// outlining to produce a call to llvm.localescape.
1579 llvm::DenseMap<llvm::AllocaInst *, int> EscapedLocals;
1580
1581 /// LabelMap - This keeps track of the LLVM basic block for each C label.
1582 llvm::DenseMap<const LabelDecl *, JumpDest> LabelMap;
1583
1584 // BreakContinueStack - This keeps track of where break and continue
1585 // statements should jump to.
1586 struct BreakContinue {
1587 BreakContinue(const Stmt &LoopOrSwitch, JumpDest Break, JumpDest Continue)
1588 : LoopOrSwitch(&LoopOrSwitch), BreakBlock(Break),
1589 ContinueBlock(Continue) {}
1590
1591 const Stmt *LoopOrSwitch;
1592 JumpDest BreakBlock;
1593 JumpDest ContinueBlock;
1594 };
1595 SmallVector<BreakContinue, 8> BreakContinueStack;
1596
1597 /// Handles cancellation exit points in OpenMP-related constructs.
1598 class OpenMPCancelExitStack {
1599 /// Tracks cancellation exit point and join point for cancel-related exit
1600 /// and normal exit.
1601 struct CancelExit {
1602 CancelExit() = default;
1603 CancelExit(OpenMPDirectiveKind Kind, JumpDest ExitBlock,
1604 JumpDest ContBlock)
1605 : Kind(Kind), ExitBlock(ExitBlock), ContBlock(ContBlock) {}
1606 OpenMPDirectiveKind Kind = llvm::omp::OMPD_unknown;
1607 /// true if the exit block has been emitted already by the special
1608 /// emitExit() call, false if the default codegen is used.
1609 bool HasBeenEmitted = false;
1610 JumpDest ExitBlock;
1611 JumpDest ContBlock;
1612 };
1613
1614 SmallVector<CancelExit, 8> Stack;
1615
1616 public:
1617 OpenMPCancelExitStack() : Stack(1) {}
1618 ~OpenMPCancelExitStack() = default;
1619 /// Fetches the exit block for the current OpenMP construct.
1620 JumpDest getExitBlock() const { return Stack.back().ExitBlock; }
1621 /// Emits exit block with special codegen procedure specific for the related
1622 /// OpenMP construct + emits code for normal construct cleanup.
1623 void emitExit(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
1624 const llvm::function_ref<void(CodeGenFunction &)> CodeGen) {
1625 if (Stack.back().Kind == Kind && getExitBlock().isValid()) {
1626 assert(CGF.getOMPCancelDestination(Kind).isValid());
1627 assert(CGF.HaveInsertPoint());
1628 assert(!Stack.back().HasBeenEmitted);
1629 auto IP = CGF.Builder.saveAndClearIP();
1630 CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1631 CodeGen(CGF);
1632 CGF.EmitBranch(Block: Stack.back().ContBlock.getBlock());
1633 CGF.Builder.restoreIP(IP);
1634 Stack.back().HasBeenEmitted = true;
1635 }
1636 CodeGen(CGF);
1637 }
1638 /// Enter the cancel supporting \a Kind construct.
1639 /// \param Kind OpenMP directive that supports cancel constructs.
1640 /// \param HasCancel true, if the construct has inner cancel directive,
1641 /// false otherwise.
1642 void enter(CodeGenFunction &CGF, OpenMPDirectiveKind Kind, bool HasCancel) {
1643 Stack.push_back(Elt: {Kind,
1644 HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.exit")
1645 : JumpDest(),
1646 HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.cont")
1647 : JumpDest()});
1648 }
1649 /// Emits default exit point for the cancel construct (if the special one
1650 /// has not be used) + join point for cancel/normal exits.
1651 void exit(CodeGenFunction &CGF) {
1652 if (getExitBlock().isValid()) {
1653 assert(CGF.getOMPCancelDestination(Stack.back().Kind).isValid());
1654 bool HaveIP = CGF.HaveInsertPoint();
1655 if (!Stack.back().HasBeenEmitted) {
1656 if (HaveIP)
1657 CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1658 CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1659 CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1660 }
1661 CGF.EmitBlock(BB: Stack.back().ContBlock.getBlock());
1662 if (!HaveIP) {
1663 CGF.Builder.CreateUnreachable();
1664 CGF.Builder.ClearInsertionPoint();
1665 }
1666 }
1667 Stack.pop_back();
1668 }
1669 };
1670 OpenMPCancelExitStack OMPCancelStack;
1671
1672 /// Lower the Likelihood knowledge about the \p Cond via llvm.expect intrin.
1673 llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
1674 Stmt::Likelihood LH);
1675
1676 std::unique_ptr<CodeGenPGO> PGO;
1677
1678 /// Calculate branch weights appropriate for PGO data
1679 llvm::MDNode *createProfileWeights(uint64_t TrueCount,
1680 uint64_t FalseCount) const;
1681 llvm::MDNode *createProfileWeights(ArrayRef<uint64_t> Weights) const;
1682 llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond,
1683 uint64_t LoopCount) const;
1684
1685public:
1686 bool hasSkipCounter(const Stmt *S) const;
1687
1688 void markStmtAsUsed(bool Skipped, const Stmt *S);
1689 void markStmtMaybeUsed(const Stmt *S);
1690
1691 /// Used to specify which counter in a pair shall be incremented.
1692 /// For non-binary counters, a skip counter is derived as (Parent - Exec).
1693 /// In contrast for binary counters, a skip counter cannot be computed from
1694 /// the Parent counter. In such cases, dedicated SkipPath counters must be
1695 /// allocated and marked (incremented as binary counters). (Parent can be
1696 /// synthesized with (Exec + Skip) in simple cases)
1697 enum CounterForIncrement {
1698 UseExecPath = 0, ///< Exec (true)
1699 UseSkipPath, ///< Skip (false)
1700 };
1701
1702 /// Increment the profiler's counter for the given statement by \p StepV.
1703 /// If \p StepV is null, the default increment is 1.
1704 void incrementProfileCounter(const Stmt *S, llvm::Value *StepV = nullptr) {
1705 incrementProfileCounter(ExecSkip: UseExecPath, S, UseBoth: false, StepV);
1706 }
1707
1708 /// Emit increment of Counter.
1709 /// \param ExecSkip Use `Skipped` Counter if UseSkipPath is specified.
1710 /// \param S The Stmt that Counter is associated.
1711 /// \param UseBoth Mark both Exec/Skip as used. (for verification)
1712 /// \param StepV The offset Value for adding to Counter.
1713 void incrementProfileCounter(CounterForIncrement ExecSkip, const Stmt *S,
1714 bool UseBoth = false,
1715 llvm::Value *StepV = nullptr);
1716
1717 bool isMCDCCoverageEnabled() const {
1718 return (CGM.getCodeGenOpts().hasProfileClangInstr() &&
1719 CGM.getCodeGenOpts().MCDCCoverage &&
1720 !CurFn->hasFnAttribute(Kind: llvm::Attribute::NoProfile));
1721 }
1722
1723 /// Allocate a temp value on the stack that MCDC can use to track condition
1724 /// results.
1725 void maybeCreateMCDCCondBitmap();
1726
1727 bool isBinaryLogicalOp(const Expr *E) const {
1728 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: E->IgnoreParens());
1729 return (BOp && BOp->isLogicalOp());
1730 }
1731
1732 bool isMCDCDecisionExpr(const Expr *E) const;
1733 bool isMCDCBranchExpr(const Expr *E) const;
1734
1735 /// Zero-init the MCDC temp value.
1736 void maybeResetMCDCCondBitmap(const Expr *E);
1737
1738 /// Increment the profiler's counter for the given expression by \p StepV.
1739 /// If \p StepV is null, the default increment is 1.
1740 void maybeUpdateMCDCTestVectorBitmap(const Expr *E);
1741
1742 /// Update the MCDC temp value with the condition's evaluated result.
1743 void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val);
1744
1745 /// Get the profiler's count for the given statement.
1746 uint64_t getProfileCount(const Stmt *S);
1747
1748 /// Set the profiler's current count.
1749 void setCurrentProfileCount(uint64_t Count);
1750
1751 /// Get the profiler's current count. This is generally the count for the most
1752 /// recently incremented counter.
1753 uint64_t getCurrentProfileCount();
1754
1755 /// See CGDebugInfo::addInstToCurrentSourceAtom.
1756 void addInstToCurrentSourceAtom(llvm::Instruction *KeyInstruction,
1757 llvm::Value *Backup);
1758
1759 /// See CGDebugInfo::addInstToSpecificSourceAtom.
1760 void addInstToSpecificSourceAtom(llvm::Instruction *KeyInstruction,
1761 llvm::Value *Backup, uint64_t Atom);
1762
1763 /// Add \p KeyInstruction and an optional \p Backup instruction to a new atom
1764 /// group (See ApplyAtomGroup for more info).
1765 void addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,
1766 llvm::Value *Backup);
1767
1768 /// Copy all PFP fields from SrcPtr to DestPtr while updating signatures,
1769 /// assuming that DestPtr was already memcpy'd from SrcPtr.
1770 void emitPFPPostCopyUpdates(Address DestPtr, Address SrcPtr, QualType Ty);
1771
1772private:
1773 /// SwitchInsn - This is nearest current switch instruction. It is null if
1774 /// current context is not in a switch.
1775 llvm::SwitchInst *SwitchInsn = nullptr;
1776 /// The branch weights of SwitchInsn when doing instrumentation based PGO.
1777 SmallVector<uint64_t, 16> *SwitchWeights = nullptr;
1778
1779 /// The likelihood attributes of the SwitchCase.
1780 SmallVector<Stmt::Likelihood, 16> *SwitchLikelihood = nullptr;
1781
1782 /// CaseRangeBlock - This block holds if condition check for last case
1783 /// statement range in current switch instruction.
1784 llvm::BasicBlock *CaseRangeBlock = nullptr;
1785
1786 /// OpaqueLValues - Keeps track of the current set of opaque value
1787 /// expressions.
1788 llvm::DenseMap<const OpaqueValueExpr *, LValue> OpaqueLValues;
1789 llvm::DenseMap<const OpaqueValueExpr *, RValue> OpaqueRValues;
1790
1791 // VLASizeMap - This keeps track of the associated size for each VLA type.
1792 // We track this by the size expression rather than the type itself because
1793 // in certain situations, like a const qualifier applied to an VLA typedef,
1794 // multiple VLA types can share the same size expression.
1795 // FIXME: Maybe this could be a stack of maps that is pushed/popped as we
1796 // enter/leave scopes.
1797 llvm::DenseMap<const Expr *, llvm::Value *> VLASizeMap;
1798
1799 /// A block containing a single 'unreachable' instruction. Created
1800 /// lazily by getUnreachableBlock().
1801 llvm::BasicBlock *UnreachableBlock = nullptr;
1802
1803 /// Counts of the number return expressions in the function.
1804 unsigned NumReturnExprs = 0;
1805
1806 /// Count the number of simple (constant) return expressions in the function.
1807 unsigned NumSimpleReturnExprs = 0;
1808
1809 /// The last regular (non-return) debug location (breakpoint) in the function.
1810 SourceLocation LastStopPoint;
1811
1812public:
1813 /// Source location information about the default argument or member
1814 /// initializer expression we're evaluating, if any.
1815 CurrentSourceLocExprScope CurSourceLocExprScope;
1816 using SourceLocExprScopeGuard =
1817 CurrentSourceLocExprScope::SourceLocExprScopeGuard;
1818
1819 /// A scope within which we are constructing the fields of an object which
1820 /// might use a CXXDefaultInitExpr. This stashes away a 'this' value to use
1821 /// if we need to evaluate a CXXDefaultInitExpr within the evaluation.
1822 class FieldConstructionScope {
1823 public:
1824 FieldConstructionScope(CodeGenFunction &CGF, Address This)
1825 : CGF(CGF), OldCXXDefaultInitExprThis(CGF.CXXDefaultInitExprThis) {
1826 CGF.CXXDefaultInitExprThis = This;
1827 }
1828 ~FieldConstructionScope() {
1829 CGF.CXXDefaultInitExprThis = OldCXXDefaultInitExprThis;
1830 }
1831
1832 private:
1833 CodeGenFunction &CGF;
1834 Address OldCXXDefaultInitExprThis;
1835 };
1836
1837 /// The scope of a CXXDefaultInitExpr. Within this scope, the value of 'this'
1838 /// is overridden to be the object under construction.
1839 class CXXDefaultInitExprScope {
1840 public:
1841 CXXDefaultInitExprScope(CodeGenFunction &CGF, const CXXDefaultInitExpr *E)
1842 : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue),
1843 OldCXXThisAlignment(CGF.CXXThisAlignment),
1844 SourceLocScope(E, CGF.CurSourceLocExprScope) {
1845 CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getBasePointer();
1846 CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment();
1847 }
1848 ~CXXDefaultInitExprScope() {
1849 CGF.CXXThisValue = OldCXXThisValue;
1850 CGF.CXXThisAlignment = OldCXXThisAlignment;
1851 }
1852
1853 public:
1854 CodeGenFunction &CGF;
1855 llvm::Value *OldCXXThisValue;
1856 CharUnits OldCXXThisAlignment;
1857 SourceLocExprScopeGuard SourceLocScope;
1858 };
1859
1860 struct CXXDefaultArgExprScope : SourceLocExprScopeGuard {
1861 CXXDefaultArgExprScope(CodeGenFunction &CGF, const CXXDefaultArgExpr *E)
1862 : SourceLocExprScopeGuard(E, CGF.CurSourceLocExprScope) {}
1863 };
1864
1865 /// The scope of an ArrayInitLoopExpr. Within this scope, the value of the
1866 /// current loop index is overridden.
1867 class ArrayInitLoopExprScope {
1868 public:
1869 ArrayInitLoopExprScope(CodeGenFunction &CGF, llvm::Value *Index)
1870 : CGF(CGF), OldArrayInitIndex(CGF.ArrayInitIndex) {
1871 CGF.ArrayInitIndex = Index;
1872 }
1873 ~ArrayInitLoopExprScope() { CGF.ArrayInitIndex = OldArrayInitIndex; }
1874
1875 private:
1876 CodeGenFunction &CGF;
1877 llvm::Value *OldArrayInitIndex;
1878 };
1879
1880 class InlinedInheritingConstructorScope {
1881 public:
1882 InlinedInheritingConstructorScope(CodeGenFunction &CGF, GlobalDecl GD)
1883 : CGF(CGF), OldCurGD(CGF.CurGD), OldCurFuncDecl(CGF.CurFuncDecl),
1884 OldCurCodeDecl(CGF.CurCodeDecl),
1885 OldCXXABIThisDecl(CGF.CXXABIThisDecl),
1886 OldCXXABIThisValue(CGF.CXXABIThisValue),
1887 OldCXXThisValue(CGF.CXXThisValue),
1888 OldCXXABIThisAlignment(CGF.CXXABIThisAlignment),
1889 OldCXXThisAlignment(CGF.CXXThisAlignment),
1890 OldReturnValue(CGF.ReturnValue), OldFnRetTy(CGF.FnRetTy),
1891 OldCXXInheritedCtorInitExprArgs(
1892 std::move(CGF.CXXInheritedCtorInitExprArgs)) {
1893 CGF.CurGD = GD;
1894 CGF.CurFuncDecl = CGF.CurCodeDecl =
1895 cast<CXXConstructorDecl>(Val: GD.getDecl());
1896 CGF.CXXABIThisDecl = nullptr;
1897 CGF.CXXABIThisValue = nullptr;
1898 CGF.CXXThisValue = nullptr;
1899 CGF.CXXABIThisAlignment = CharUnits();
1900 CGF.CXXThisAlignment = CharUnits();
1901 CGF.ReturnValue = Address::invalid();
1902 CGF.FnRetTy = QualType();
1903 CGF.CXXInheritedCtorInitExprArgs.clear();
1904 }
1905 ~InlinedInheritingConstructorScope() {
1906 CGF.CurGD = OldCurGD;
1907 CGF.CurFuncDecl = OldCurFuncDecl;
1908 CGF.CurCodeDecl = OldCurCodeDecl;
1909 CGF.CXXABIThisDecl = OldCXXABIThisDecl;
1910 CGF.CXXABIThisValue = OldCXXABIThisValue;
1911 CGF.CXXThisValue = OldCXXThisValue;
1912 CGF.CXXABIThisAlignment = OldCXXABIThisAlignment;
1913 CGF.CXXThisAlignment = OldCXXThisAlignment;
1914 CGF.ReturnValue = OldReturnValue;
1915 CGF.FnRetTy = OldFnRetTy;
1916 CGF.CXXInheritedCtorInitExprArgs =
1917 std::move(OldCXXInheritedCtorInitExprArgs);
1918 }
1919
1920 private:
1921 CodeGenFunction &CGF;
1922 GlobalDecl OldCurGD;
1923 const Decl *OldCurFuncDecl;
1924 const Decl *OldCurCodeDecl;
1925 ImplicitParamDecl *OldCXXABIThisDecl;
1926 llvm::Value *OldCXXABIThisValue;
1927 llvm::Value *OldCXXThisValue;
1928 CharUnits OldCXXABIThisAlignment;
1929 CharUnits OldCXXThisAlignment;
1930 Address OldReturnValue;
1931 QualType OldFnRetTy;
1932 CallArgList OldCXXInheritedCtorInitExprArgs;
1933 };
1934
1935 // Helper class for the OpenMP IR Builder. Allows reusability of code used for
1936 // region body, and finalization codegen callbacks. This will class will also
1937 // contain privatization functions used by the privatization call backs
1938 //
1939 // TODO: this is temporary class for things that are being moved out of
1940 // CGOpenMPRuntime, new versions of current CodeGenFunction methods, or
1941 // utility function for use with the OMPBuilder. Once that move to use the
1942 // OMPBuilder is done, everything here will either become part of CodeGenFunc.
1943 // directly, or a new helper class that will contain functions used by both
1944 // this and the OMPBuilder
1945
1946 struct OMPBuilderCBHelpers {
1947
1948 OMPBuilderCBHelpers() = delete;
1949 OMPBuilderCBHelpers(const OMPBuilderCBHelpers &) = delete;
1950 OMPBuilderCBHelpers &operator=(const OMPBuilderCBHelpers &) = delete;
1951
1952 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1953
1954 /// Cleanup action for allocate support.
1955 class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
1956
1957 private:
1958 llvm::CallInst *RTLFnCI;
1959
1960 public:
1961 OMPAllocateCleanupTy(llvm::CallInst *RLFnCI) : RTLFnCI(RLFnCI) {
1962 RLFnCI->removeFromParent();
1963 }
1964
1965 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
1966 if (!CGF.HaveInsertPoint())
1967 return;
1968 CGF.Builder.Insert(I: RTLFnCI);
1969 }
1970 };
1971
1972 /// Returns address of the threadprivate variable for the current
1973 /// thread. This Also create any necessary OMP runtime calls.
1974 ///
1975 /// \param VD VarDecl for Threadprivate variable.
1976 /// \param VDAddr Address of the Vardecl
1977 /// \param Loc The location where the barrier directive was encountered
1978 static Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
1979 const VarDecl *VD, Address VDAddr,
1980 SourceLocation Loc);
1981
1982 /// Gets the OpenMP-specific address of the local variable /p VD.
1983 static Address getAddressOfLocalVariable(CodeGenFunction &CGF,
1984 const VarDecl *VD);
1985 /// Get the platform-specific name separator.
1986 /// \param Parts different parts of the final name that needs separation
1987 /// \param FirstSeparator First separator used between the initial two
1988 /// parts of the name.
1989 /// \param Separator separator used between all of the rest consecutinve
1990 /// parts of the name
1991 static std::string getNameWithSeparators(ArrayRef<StringRef> Parts,
1992 StringRef FirstSeparator = ".",
1993 StringRef Separator = ".");
1994 /// Emit the Finalization for an OMP region
1995 /// \param CGF The Codegen function this belongs to
1996 /// \param IP Insertion point for generating the finalization code.
1997 static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP) {
1998 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1999 assert(IP.getBlock()->end() != IP.getPoint() &&
2000 "OpenMP IR Builder should cause terminated block!");
2001
2002 llvm::BasicBlock *IPBB = IP.getBlock();
2003 llvm::BasicBlock *DestBB = IPBB->getUniqueSuccessor();
2004 assert(DestBB && "Finalization block should have one successor!");
2005
2006 // erase and replace with cleanup branch.
2007 IPBB->getTerminator()->eraseFromParent();
2008 CGF.Builder.SetInsertPoint(IPBB);
2009 CodeGenFunction::JumpDest Dest = CGF.getJumpDestInCurrentScope(Target: DestBB);
2010 CGF.EmitBranchThroughCleanup(Dest);
2011 }
2012
2013 /// Emit the body of an OMP region
2014 /// \param CGF The Codegen function this belongs to
2015 /// \param RegionBodyStmt The body statement for the OpenMP region being
2016 /// generated
2017 /// \param AllocaIP Where to insert alloca instructions
2018 /// \param CodeGenIP Where to insert the region code
2019 /// \param RegionName Name to be used for new blocks
2020 static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF,
2021 const Stmt *RegionBodyStmt,
2022 InsertPointTy AllocaIP,
2023 InsertPointTy CodeGenIP,
2024 Twine RegionName);
2025
2026 static void EmitCaptureStmt(CodeGenFunction &CGF, InsertPointTy CodeGenIP,
2027 llvm::BasicBlock &FiniBB, llvm::Function *Fn,
2028 ArrayRef<llvm::Value *> Args) {
2029 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2030 if (llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminatorOrNull())
2031 CodeGenIPBBTI->eraseFromParent();
2032
2033 CGF.Builder.SetInsertPoint(CodeGenIPBB);
2034
2035 if (Fn->doesNotThrow())
2036 CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
2037 else
2038 CGF.EmitRuntimeCall(callee: Fn, args: Args);
2039
2040 if (CGF.Builder.saveIP().isSet())
2041 CGF.Builder.CreateBr(Dest: &FiniBB);
2042 }
2043
2044 /// Emit the body of an OMP region that will be outlined in
2045 /// OpenMPIRBuilder::finalize().
2046 /// \param CGF The Codegen function this belongs to
2047 /// \param RegionBodyStmt The body statement for the OpenMP region being
2048 /// generated
2049 /// \param AllocaIP Where to insert alloca instructions
2050 /// \param CodeGenIP Where to insert the region code
2051 /// \param RegionName Name to be used for new blocks
2052 static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF,
2053 const Stmt *RegionBodyStmt,
2054 InsertPointTy AllocaIP,
2055 InsertPointTy CodeGenIP,
2056 Twine RegionName);
2057
2058 /// RAII for preserving necessary info during Outlined region body codegen.
2059 class OutlinedRegionBodyRAII {
2060
2061 llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
2062 CodeGenFunction::JumpDest OldReturnBlock;
2063 CodeGenFunction &CGF;
2064
2065 public:
2066 OutlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
2067 llvm::BasicBlock &RetBB)
2068 : CGF(cgf) {
2069 assert(AllocaIP.isSet() &&
2070 "Must specify Insertion point for allocas of outlined function");
2071 OldAllocaIP = CGF.AllocaInsertPt;
2072 CGF.AllocaInsertPt = &*AllocaIP.getPoint();
2073
2074 OldReturnBlock = CGF.ReturnBlock;
2075 CGF.ReturnBlock = CGF.getJumpDestInCurrentScope(Target: &RetBB);
2076 }
2077
2078 ~OutlinedRegionBodyRAII() {
2079 CGF.AllocaInsertPt = OldAllocaIP;
2080 CGF.ReturnBlock = OldReturnBlock;
2081 }
2082 };
2083
2084 /// RAII for preserving necessary info during inlined region body codegen.
2085 class InlinedRegionBodyRAII {
2086
2087 llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
2088 CodeGenFunction &CGF;
2089
2090 public:
2091 InlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
2092 llvm::BasicBlock &FiniBB)
2093 : CGF(cgf) {
2094 // Alloca insertion block should be in the entry block of the containing
2095 // function so it expects an empty AllocaIP in which case will reuse the
2096 // old alloca insertion point, or a new AllocaIP in the same block as
2097 // the old one
2098 assert((!AllocaIP.isSet() ||
2099 CGF.AllocaInsertPt->getParent() == AllocaIP.getBlock()) &&
2100 "Insertion point should be in the entry block of containing "
2101 "function!");
2102 OldAllocaIP = CGF.AllocaInsertPt;
2103 if (AllocaIP.isSet())
2104 CGF.AllocaInsertPt = &*AllocaIP.getPoint();
2105
2106 // TODO: Remove the call, after making sure the counter is not used by
2107 // the EHStack.
2108 // Since this is an inlined region, it should not modify the
2109 // ReturnBlock, and should reuse the one for the enclosing outlined
2110 // region. So, the JumpDest being return by the function is discarded
2111 (void)CGF.getJumpDestInCurrentScope(Target: &FiniBB);
2112 }
2113
2114 ~InlinedRegionBodyRAII() { CGF.AllocaInsertPt = OldAllocaIP; }
2115 };
2116 };
2117
2118private:
2119 /// CXXThisDecl - When generating code for a C++ member function,
2120 /// this will hold the implicit 'this' declaration.
2121 ImplicitParamDecl *CXXABIThisDecl = nullptr;
2122 llvm::Value *CXXABIThisValue = nullptr;
2123 llvm::Value *CXXThisValue = nullptr;
2124 CharUnits CXXABIThisAlignment;
2125 CharUnits CXXThisAlignment;
2126
2127 /// The value of 'this' to use when evaluating CXXDefaultInitExprs within
2128 /// this expression.
2129 Address CXXDefaultInitExprThis = Address::invalid();
2130
2131 /// The current array initialization index when evaluating an
2132 /// ArrayInitIndexExpr within an ArrayInitLoopExpr.
2133 llvm::Value *ArrayInitIndex = nullptr;
2134
2135 /// The values of function arguments to use when evaluating
2136 /// CXXInheritedCtorInitExprs within this context.
2137 CallArgList CXXInheritedCtorInitExprArgs;
2138
2139 /// CXXStructorImplicitParamDecl - When generating code for a constructor or
2140 /// destructor, this will hold the implicit argument (e.g. VTT).
2141 ImplicitParamDecl *CXXStructorImplicitParamDecl = nullptr;
2142 llvm::Value *CXXStructorImplicitParamValue = nullptr;
2143
2144 /// OutermostConditional - Points to the outermost active
2145 /// conditional control. This is used so that we know if a
2146 /// temporary should be destroyed conditionally.
2147 ConditionalEvaluation *OutermostConditional = nullptr;
2148
2149 /// The current lexical scope.
2150 LexicalScope *CurLexicalScope = nullptr;
2151
2152 /// The current source location that should be used for exception
2153 /// handling code.
2154 SourceLocation CurEHLocation;
2155
2156 /// BlockByrefInfos - For each __block variable, contains
2157 /// information about the layout of the variable.
2158 llvm::DenseMap<const ValueDecl *, BlockByrefInfo> BlockByrefInfos;
2159
2160 /// Used by -fsanitize=nullability-return to determine whether the return
2161 /// value can be checked.
2162 llvm::Value *RetValNullabilityPrecondition = nullptr;
2163
2164 /// Check if -fsanitize=nullability-return instrumentation is required for
2165 /// this function.
2166 bool requiresReturnValueNullabilityCheck() const {
2167 return RetValNullabilityPrecondition;
2168 }
2169
2170 /// Used to store precise source locations for return statements by the
2171 /// runtime return value checks.
2172 Address ReturnLocation = Address::invalid();
2173
2174 /// Check if the return value of this function requires sanitization.
2175 bool requiresReturnValueCheck() const;
2176
2177 bool isInAllocaArgument(CGCXXABI &ABI, QualType Ty);
2178 bool hasInAllocaArg(const CXXMethodDecl *MD);
2179
2180 llvm::BasicBlock *TerminateLandingPad = nullptr;
2181 llvm::BasicBlock *TerminateHandler = nullptr;
2182 llvm::SmallVector<llvm::BasicBlock *, 2> TrapBBs;
2183
2184 /// Terminate funclets keyed by parent funclet pad.
2185 llvm::MapVector<llvm::Value *, llvm::BasicBlock *> TerminateFunclets;
2186
2187 /// Largest vector width used in ths function. Will be used to create a
2188 /// function attribute.
2189 unsigned LargestVectorWidth = 0;
2190
2191 /// True if we need emit the life-time markers. This is initially set in
2192 /// the constructor, but could be overwritten to true if this is a coroutine.
2193 bool ShouldEmitLifetimeMarkers;
2194
2195 /// Add OpenCL kernel arg metadata and the kernel attribute metadata to
2196 /// the function metadata.
2197 void EmitKernelMetadata(const FunctionDecl *FD, llvm::Function *Fn);
2198
2199public:
2200 CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext = false);
2201 ~CodeGenFunction();
2202
2203 CodeGenTypes &getTypes() const { return CGM.getTypes(); }
2204 ASTContext &getContext() const { return CGM.getContext(); }
2205 CGDebugInfo *getDebugInfo() {
2206 if (DisableDebugInfo)
2207 return nullptr;
2208 return DebugInfo;
2209 }
2210 void disableDebugInfo() { DisableDebugInfo = true; }
2211 void enableDebugInfo() { DisableDebugInfo = false; }
2212
2213 bool shouldUseFusedARCCalls() {
2214 return CGM.getCodeGenOpts().OptimizationLevel == 0;
2215 }
2216
2217 const LangOptions &getLangOpts() const { return CGM.getLangOpts(); }
2218
2219 /// Returns a pointer to the function's exception object and selector slot,
2220 /// which is assigned in every landing pad.
2221 Address getExceptionSlot();
2222 Address getEHSelectorSlot();
2223
2224 /// Returns the contents of the function's exception object and selector
2225 /// slots.
2226 llvm::Value *getExceptionFromSlot();
2227 llvm::Value *getSelectorFromSlot();
2228
2229 RawAddress getNormalCleanupDestSlot();
2230
2231 llvm::BasicBlock *getUnreachableBlock() {
2232 if (!UnreachableBlock) {
2233 UnreachableBlock = createBasicBlock(name: "unreachable");
2234 new llvm::UnreachableInst(getLLVMContext(), UnreachableBlock);
2235 }
2236 return UnreachableBlock;
2237 }
2238
2239 llvm::BasicBlock *getInvokeDest() {
2240 if (!EHStack.requiresLandingPad())
2241 return nullptr;
2242 return getInvokeDestImpl();
2243 }
2244
2245 bool currentFunctionUsesSEHTry() const { return !!CurSEHParent; }
2246
2247 const TargetInfo &getTarget() const { return Target; }
2248 llvm::LLVMContext &getLLVMContext() { return CGM.getLLVMContext(); }
2249 const TargetCodeGenInfo &getTargetHooks() const {
2250 return CGM.getTargetCodeGenInfo();
2251 }
2252 const FunctionDecl *getCurrentFunctionDecl() const;
2253
2254 //===--------------------------------------------------------------------===//
2255 // Cleanups
2256 //===--------------------------------------------------------------------===//
2257
2258 typedef void Destroyer(CodeGenFunction &CGF, Address addr, QualType ty);
2259
2260 void pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,
2261 Address arrayEndPointer,
2262 QualType elementType,
2263 CharUnits elementAlignment,
2264 Destroyer *destroyer);
2265 void pushRegularPartialArrayCleanup(llvm::Value *arrayBegin,
2266 llvm::Value *arrayEnd,
2267 QualType elementType,
2268 CharUnits elementAlignment,
2269 Destroyer *destroyer);
2270
2271 void pushDestroy(QualType::DestructionKind dtorKind, Address addr,
2272 QualType type);
2273 void pushEHDestroy(QualType::DestructionKind dtorKind, Address addr,
2274 QualType type);
2275 void pushDestroy(CleanupKind kind, Address addr, QualType type,
2276 Destroyer *destroyer, bool useEHCleanupForArray);
2277 void pushDestroyAndDeferDeactivation(QualType::DestructionKind dtorKind,
2278 Address addr, QualType type);
2279 void pushDestroyAndDeferDeactivation(CleanupKind cleanupKind, Address addr,
2280 QualType type, Destroyer *destroyer,
2281 bool useEHCleanupForArray);
2282 void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr,
2283 QualType type, Destroyer *destroyer,
2284 bool useEHCleanupForArray);
2285 void pushLifetimeExtendedDestroy(QualType::DestructionKind dtorKind,
2286 Address addr, QualType type);
2287 void pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete,
2288 llvm::Value *CompletePtr,
2289 QualType ElementType);
2290 void pushStackRestore(CleanupKind kind, Address SPMem);
2291 void pushKmpcAllocFree(CleanupKind Kind,
2292 std::pair<llvm::Value *, llvm::Value *> AddrSizePair);
2293 void emitDestroy(Address addr, QualType type, Destroyer *destroyer,
2294 bool useEHCleanupForArray);
2295 llvm::Function *generateDestroyHelper(Address addr, QualType type,
2296 Destroyer *destroyer,
2297 bool useEHCleanupForArray,
2298 const VarDecl *VD);
2299 void emitArrayDestroy(llvm::Value *begin, llvm::Value *end,
2300 QualType elementType, CharUnits elementAlign,
2301 Destroyer *destroyer, bool checkZeroLength,
2302 bool useEHCleanup);
2303
2304 Destroyer *getDestroyer(QualType::DestructionKind destructionKind);
2305
2306 /// Determines whether an EH cleanup is required to destroy a type
2307 /// with the given destruction kind.
2308 bool needsEHCleanup(QualType::DestructionKind kind) {
2309 switch (kind) {
2310 case QualType::DK_none:
2311 return false;
2312 case QualType::DK_cxx_destructor:
2313 case QualType::DK_objc_weak_lifetime:
2314 case QualType::DK_nontrivial_c_struct:
2315 return getLangOpts().Exceptions;
2316 case QualType::DK_objc_strong_lifetime:
2317 return getLangOpts().Exceptions &&
2318 CGM.getCodeGenOpts().ObjCAutoRefCountExceptions;
2319 }
2320 llvm_unreachable("bad destruction kind");
2321 }
2322
2323 CleanupKind getCleanupKind(QualType::DestructionKind kind) {
2324 return (needsEHCleanup(kind) ? NormalAndEHCleanup : NormalCleanup);
2325 }
2326
2327 //===--------------------------------------------------------------------===//
2328 // Objective-C
2329 //===--------------------------------------------------------------------===//
2330
2331 void GenerateObjCMethod(const ObjCMethodDecl *OMD);
2332
2333 void StartObjCMethod(const ObjCMethodDecl *MD, const ObjCContainerDecl *CD);
2334
2335 /// GenerateObjCGetter - Synthesize an Objective-C property getter function.
2336 void GenerateObjCGetter(ObjCImplementationDecl *IMP,
2337 const ObjCPropertyImplDecl *PID);
2338 void generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
2339 const ObjCPropertyImplDecl *propImpl,
2340 const ObjCMethodDecl *GetterMothodDecl,
2341 llvm::Constant *AtomicHelperFn);
2342
2343 void GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
2344 ObjCMethodDecl *MD, bool ctor);
2345
2346 /// GenerateObjCSetter - Synthesize an Objective-C property setter function
2347 /// for the given property.
2348 void GenerateObjCSetter(ObjCImplementationDecl *IMP,
2349 const ObjCPropertyImplDecl *PID);
2350 void generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
2351 const ObjCPropertyImplDecl *propImpl,
2352 llvm::Constant *AtomicHelperFn);
2353
2354 //===--------------------------------------------------------------------===//
2355 // Block Bits
2356 //===--------------------------------------------------------------------===//
2357
2358 /// Emit block literal.
2359 /// \return an LLVM value which is a pointer to a struct which contains
2360 /// information about the block, including the block invoke function, the
2361 /// captured variables, etc.
2362 llvm::Value *EmitBlockLiteral(const BlockExpr *);
2363
2364 llvm::Function *GenerateBlockFunction(GlobalDecl GD, const CGBlockInfo &Info,
2365 const DeclMapTy &ldm,
2366 bool IsLambdaConversionToBlock,
2367 bool BuildGlobalBlock);
2368
2369 /// Check if \p T is a C++ class that has a destructor that can throw.
2370 static bool cxxDestructorCanThrow(QualType T);
2371
2372 llvm::Constant *GenerateCopyHelperFunction(const CGBlockInfo &blockInfo);
2373 llvm::Constant *GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo);
2374 llvm::Constant *
2375 GenerateObjCAtomicSetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);
2376 llvm::Constant *
2377 GenerateObjCAtomicGetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);
2378 llvm::Value *EmitBlockCopyAndAutorelease(llvm::Value *Block, QualType Ty);
2379
2380 void BuildBlockRelease(llvm::Value *DeclPtr, BlockFieldFlags flags,
2381 bool CanThrow);
2382
2383 class AutoVarEmission;
2384
2385 void emitByrefStructureInit(const AutoVarEmission &emission);
2386
2387 /// Enter a cleanup to destroy a __block variable. Note that this
2388 /// cleanup should be a no-op if the variable hasn't left the stack
2389 /// yet; if a cleanup is required for the variable itself, that needs
2390 /// to be done externally.
2391 ///
2392 /// \param Kind Cleanup kind.
2393 ///
2394 /// \param Addr When \p LoadBlockVarAddr is false, the address of the __block
2395 /// structure that will be passed to _Block_object_dispose. When
2396 /// \p LoadBlockVarAddr is true, the address of the field of the block
2397 /// structure that holds the address of the __block structure.
2398 ///
2399 /// \param Flags The flag that will be passed to _Block_object_dispose.
2400 ///
2401 /// \param LoadBlockVarAddr Indicates whether we need to emit a load from
2402 /// \p Addr to get the address of the __block structure.
2403 void enterByrefCleanup(CleanupKind Kind, Address Addr, BlockFieldFlags Flags,
2404 bool LoadBlockVarAddr, bool CanThrow);
2405
2406 void setBlockContextParameter(const ImplicitParamDecl *D, unsigned argNum,
2407 llvm::Value *ptr);
2408
2409 Address LoadBlockStruct();
2410 Address GetAddrOfBlockDecl(const VarDecl *var);
2411
2412 /// BuildBlockByrefAddress - Computes the location of the
2413 /// data in a variable which is declared as __block.
2414 Address emitBlockByrefAddress(Address baseAddr, const VarDecl *V,
2415 bool followForward = true);
2416 Address emitBlockByrefAddress(Address baseAddr, const BlockByrefInfo &info,
2417 bool followForward, const llvm::Twine &name);
2418
2419 const BlockByrefInfo &getBlockByrefInfo(const VarDecl *var);
2420
2421 QualType BuildFunctionArgList(GlobalDecl GD, FunctionArgList &Args);
2422
2423 void GenerateCode(GlobalDecl GD, llvm::Function *Fn,
2424 const CGFunctionInfo &FnInfo);
2425
2426 /// Annotate the function with an attribute that disables TSan checking at
2427 /// runtime.
2428 void markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn);
2429
2430 /// Emit code for the start of a function.
2431 /// \param Loc The location to be associated with the function.
2432 /// \param StartLoc The location of the function body.
2433 void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn,
2434 const CGFunctionInfo &FnInfo, const FunctionArgList &Args,
2435 SourceLocation Loc = SourceLocation(),
2436 SourceLocation StartLoc = SourceLocation());
2437
2438 static bool IsConstructorDelegationValid(const CXXConstructorDecl *Ctor);
2439
2440 void EmitConstructorBody(FunctionArgList &Args);
2441 void EmitDestructorBody(FunctionArgList &Args);
2442 void emitImplicitAssignmentOperatorBody(FunctionArgList &Args);
2443 void EmitFunctionBody(const Stmt *Body);
2444 void EmitBlockWithFallThrough(llvm::BasicBlock *BB, const Stmt *S);
2445
2446 void EmitForwardingCallToLambda(const CXXMethodDecl *LambdaCallOperator,
2447 CallArgList &CallArgs,
2448 const CGFunctionInfo *CallOpFnInfo = nullptr,
2449 llvm::Constant *CallOpFn = nullptr);
2450 void EmitLambdaBlockInvokeBody();
2451 void EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD);
2452 void EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,
2453 CallArgList &CallArgs);
2454 void EmitLambdaInAllocaImplFn(const CXXMethodDecl *CallOp,
2455 const CGFunctionInfo **ImplFnInfo,
2456 llvm::Function **ImplFn);
2457 void EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD);
2458 void EmitLambdaVLACapture(const VariableArrayType *VAT, LValue LV) {
2459 EmitStoreThroughLValue(Src: RValue::get(V: VLASizeMap[VAT->getSizeExpr()]), Dst: LV);
2460 }
2461 void EmitAsanPrologueOrEpilogue(bool Prologue);
2462
2463 /// Emit the unified return block, trying to avoid its emission when
2464 /// possible.
2465 /// \return The debug location of the user written return statement if the
2466 /// return block is avoided.
2467 llvm::DebugLoc EmitReturnBlock();
2468
2469 /// FinishFunction - Complete IR generation of the current function. It is
2470 /// legal to call this function even if there is no current insertion point.
2471 void FinishFunction(SourceLocation EndLoc = SourceLocation());
2472
2473 void StartThunk(llvm::Function *Fn, GlobalDecl GD,
2474 const CGFunctionInfo &FnInfo, bool IsUnprototyped);
2475
2476 void EmitCallAndReturnForThunk(llvm::FunctionCallee Callee,
2477 const ThunkInfo *Thunk, bool IsUnprototyped);
2478
2479 void FinishThunk();
2480
2481 /// Start an Objective-C direct method thunk.
2482 void StartObjCDirectPreconditionThunk(const ObjCMethodDecl *OMD,
2483 llvm::Function *Fn,
2484 const CGFunctionInfo &FI);
2485
2486 /// Finish an Objective-C direct method thunk.
2487 void FinishObjCDirectPreconditionThunk();
2488
2489 /// Emit a musttail call for a thunk with a potentially adjusted this pointer.
2490 void EmitMustTailThunk(GlobalDecl GD, llvm::Value *AdjustedThisPtr,
2491 llvm::FunctionCallee Callee);
2492
2493 /// Generate a thunk for the given method.
2494 void generateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo,
2495 GlobalDecl GD, const ThunkInfo &Thunk,
2496 bool IsUnprototyped);
2497
2498 llvm::Function *GenerateVarArgsThunk(llvm::Function *Fn,
2499 const CGFunctionInfo &FnInfo,
2500 GlobalDecl GD, const ThunkInfo &Thunk);
2501
2502 void EmitCtorPrologue(const CXXConstructorDecl *CD, CXXCtorType Type,
2503 FunctionArgList &Args);
2504
2505 void EmitInitializerForField(FieldDecl *Field, LValue LHS, Expr *Init);
2506
2507 /// Struct with all information about dynamic [sub]class needed to set vptr.
2508 struct VPtr {
2509 BaseSubobject Base;
2510 const CXXRecordDecl *NearestVBase;
2511 CharUnits OffsetFromNearestVBase;
2512 const CXXRecordDecl *VTableClass;
2513 };
2514
2515 /// Initialize the vtable pointer of the given subobject.
2516 void InitializeVTablePointer(const VPtr &vptr);
2517
2518 typedef llvm::SmallVector<VPtr, 4> VPtrsVector;
2519
2520 typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
2521 VPtrsVector getVTablePointers(const CXXRecordDecl *VTableClass);
2522
2523 void getVTablePointers(BaseSubobject Base, const CXXRecordDecl *NearestVBase,
2524 CharUnits OffsetFromNearestVBase,
2525 bool BaseIsNonVirtualPrimaryBase,
2526 const CXXRecordDecl *VTableClass,
2527 VisitedVirtualBasesSetTy &VBases, VPtrsVector &vptrs);
2528
2529 void InitializeVTablePointers(const CXXRecordDecl *ClassDecl);
2530
2531 // VTableTrapMode - whether we guarantee that loading the
2532 // vtable is guaranteed to trap on authentication failure,
2533 // even if the resulting vtable pointer is unused.
2534 enum class VTableAuthMode {
2535 Authenticate,
2536 MustTrap,
2537 UnsafeUbsanStrip // Should only be used for Vptr UBSan check
2538 };
2539 /// GetVTablePtr - Return the Value of the vtable pointer member pointed
2540 /// to by This.
2541 llvm::Value *
2542 GetVTablePtr(Address This, llvm::Type *VTableTy,
2543 const CXXRecordDecl *VTableClass,
2544 VTableAuthMode AuthMode = VTableAuthMode::Authenticate);
2545
2546 enum CFITypeCheckKind {
2547 CFITCK_VCall,
2548 CFITCK_NVCall,
2549 CFITCK_DerivedCast,
2550 CFITCK_UnrelatedCast,
2551 CFITCK_ICall,
2552 CFITCK_NVMFCall,
2553 CFITCK_VMFCall,
2554 };
2555
2556 /// Derived is the presumed address of an object of type T after a
2557 /// cast. If T is a polymorphic class type, emit a check that the virtual
2558 /// table for Derived belongs to a class derived from T.
2559 void EmitVTablePtrCheckForCast(QualType T, Address Derived, bool MayBeNull,
2560 CFITypeCheckKind TCK, SourceLocation Loc);
2561
2562 /// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable.
2563 /// If vptr CFI is enabled, emit a check that VTable is valid.
2564 void EmitVTablePtrCheckForCall(const CXXRecordDecl *RD, llvm::Value *VTable,
2565 CFITypeCheckKind TCK, SourceLocation Loc);
2566
2567 /// EmitVTablePtrCheck - Emit a check that VTable is a valid virtual table for
2568 /// RD using llvm.type.test.
2569 void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable,
2570 CFITypeCheckKind TCK, SourceLocation Loc);
2571
2572 /// If whole-program virtual table optimization is enabled, emit an assumption
2573 /// that VTable is a member of RD's type identifier. Or, if vptr CFI is
2574 /// enabled, emit a check that VTable is a member of RD's type identifier.
2575 void EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2576 llvm::Value *VTable, SourceLocation Loc);
2577
2578 /// Returns whether we should perform a type checked load when loading a
2579 /// virtual function for virtual calls to members of RD. This is generally
2580 /// true when both vcall CFI and whole-program-vtables are enabled.
2581 bool ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD);
2582
2583 /// Emit a type checked load from the given vtable.
2584 llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD,
2585 llvm::Value *VTable,
2586 llvm::Type *VTableTy,
2587 uint64_t VTableByteOffset);
2588
2589 /// EnterDtorCleanups - Enter the cleanups necessary to complete the
2590 /// given phase of destruction for a destructor. The end result
2591 /// should call destructors on members and base classes in reverse
2592 /// order of their construction.
2593 void EnterDtorCleanups(const CXXDestructorDecl *Dtor, CXXDtorType Type);
2594
2595 /// ShouldInstrumentFunction - Return true if the current function should be
2596 /// instrumented with __cyg_profile_func_* calls
2597 bool ShouldInstrumentFunction();
2598
2599 /// ShouldSkipSanitizerInstrumentation - Return true if the current function
2600 /// should not be instrumented with sanitizers.
2601 bool ShouldSkipSanitizerInstrumentation();
2602
2603 /// ShouldXRayInstrument - Return true if the current function should be
2604 /// instrumented with XRay nop sleds.
2605 bool ShouldXRayInstrumentFunction() const;
2606
2607 /// AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit
2608 /// XRay custom event handling calls.
2609 bool AlwaysEmitXRayCustomEvents() const;
2610
2611 /// AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit
2612 /// XRay typed event handling calls.
2613 bool AlwaysEmitXRayTypedEvents() const;
2614
2615 /// Return a type hash constant for a function instrumented by
2616 /// -fsanitize=function.
2617 llvm::ConstantInt *getUBSanFunctionTypeHash(QualType T) const;
2618
2619 /// EmitFunctionProlog - Emit the target specific LLVM code to load the
2620 /// arguments for the given function. This is also responsible for naming the
2621 /// LLVM function arguments.
2622 void EmitFunctionProlog(const CGFunctionInfo &FI, llvm::Function *Fn,
2623 const FunctionArgList &Args);
2624
2625 /// EmitFunctionEpilog - Emit the target specific LLVM code to return the
2626 /// given temporary. Specify the source location atom group (Key Instructions
2627 /// debug info feature) for the `ret` using \p RetKeyInstructionsSourceAtom.
2628 /// If it's 0, the `ret` will get added to a new source atom group.
2629 void EmitFunctionEpilog(const CGFunctionInfo &FI, bool EmitRetDbgLoc,
2630 SourceLocation EndLoc,
2631 uint64_t RetKeyInstructionsSourceAtom);
2632
2633 /// Emit a test that checks if the return value \p RV is nonnull.
2634 void EmitReturnValueCheck(llvm::Value *RV);
2635
2636 /// EmitStartEHSpec - Emit the start of the exception spec.
2637 void EmitStartEHSpec(const Decl *D);
2638
2639 /// EmitEndEHSpec - Emit the end of the exception spec.
2640 void EmitEndEHSpec(const Decl *D);
2641
2642 /// getTerminateLandingPad - Return a landing pad that just calls terminate.
2643 llvm::BasicBlock *getTerminateLandingPad();
2644
2645 /// getTerminateLandingPad - Return a cleanup funclet that just calls
2646 /// terminate.
2647 llvm::BasicBlock *getTerminateFunclet();
2648
2649 /// getTerminateHandler - Return a handler (not a landing pad, just
2650 /// a catch handler) that just calls terminate. This is used when
2651 /// a terminate scope encloses a try.
2652 llvm::BasicBlock *getTerminateHandler();
2653
2654 llvm::Type *ConvertTypeForMem(QualType T);
2655 llvm::Type *ConvertType(QualType T);
2656 llvm::Type *convertTypeForLoadStore(QualType ASTTy,
2657 llvm::Type *LLVMTy = nullptr);
2658 llvm::Type *ConvertType(const TypeDecl *T) {
2659 return ConvertType(T: getContext().getTypeDeclType(Decl: T));
2660 }
2661
2662 /// LoadObjCSelf - Load the value of self. This function is only valid while
2663 /// generating code for an Objective-C method.
2664 llvm::Value *LoadObjCSelf();
2665
2666 /// TypeOfSelfObject - Return type of object that this self represents.
2667 QualType TypeOfSelfObject();
2668
2669 /// getEvaluationKind - Return the TypeEvaluationKind of QualType \c T.
2670 static TypeEvaluationKind getEvaluationKind(QualType T);
2671
2672 static bool hasScalarEvaluationKind(QualType T) {
2673 return getEvaluationKind(T) == TEK_Scalar;
2674 }
2675
2676 static bool hasAggregateEvaluationKind(QualType T) {
2677 return getEvaluationKind(T) == TEK_Aggregate;
2678 }
2679
2680 /// createBasicBlock - Create an LLVM basic block.
2681 llvm::BasicBlock *createBasicBlock(const Twine &name = "",
2682 llvm::Function *parent = nullptr,
2683 llvm::BasicBlock *before = nullptr) {
2684 return llvm::BasicBlock::Create(Context&: getLLVMContext(), Name: name, Parent: parent, InsertBefore: before);
2685 }
2686
2687 /// getBasicBlockForLabel - Return the LLVM basicblock that the specified
2688 /// label maps to.
2689 JumpDest getJumpDestForLabel(const LabelDecl *S);
2690
2691 /// SimplifyForwardingBlocks - If the given basic block is only a branch to
2692 /// another basic block, simplify it. This assumes that no other code could
2693 /// potentially reference the basic block.
2694 void SimplifyForwardingBlocks(llvm::BasicBlock *BB);
2695
2696 /// EmitBlock - Emit the given block \arg BB and set it as the insert point,
2697 /// adding a fall-through branch from the current insert block if
2698 /// necessary. It is legal to call this function even if there is no current
2699 /// insertion point.
2700 ///
2701 /// IsFinished - If true, indicates that the caller has finished emitting
2702 /// branches to the given block and does not expect to emit code into it. This
2703 /// means the block can be ignored if it is unreachable.
2704 void EmitBlock(llvm::BasicBlock *BB, bool IsFinished = false);
2705
2706 /// EmitBlockAfterUses - Emit the given block somewhere hopefully
2707 /// near its uses, and leave the insertion point in it.
2708 void EmitBlockAfterUses(llvm::BasicBlock *BB);
2709
2710 /// EmitBranch - Emit a branch to the specified basic block from the current
2711 /// insert block, taking care to avoid creation of branches from dummy
2712 /// blocks. It is legal to call this function even if there is no current
2713 /// insertion point.
2714 ///
2715 /// This function clears the current insertion point. The caller should follow
2716 /// calls to this function with calls to Emit*Block prior to generation new
2717 /// code.
2718 void EmitBranch(llvm::BasicBlock *Block);
2719
2720 /// HaveInsertPoint - True if an insertion point is defined. If not, this
2721 /// indicates that the current code being emitted is unreachable.
2722 bool HaveInsertPoint() const { return Builder.GetInsertBlock() != nullptr; }
2723
2724 /// EnsureInsertPoint - Ensure that an insertion point is defined so that
2725 /// emitted IR has a place to go. Note that by definition, if this function
2726 /// creates a block then that block is unreachable; callers may do better to
2727 /// detect when no insertion point is defined and simply skip IR generation.
2728 void EnsureInsertPoint() {
2729 if (!HaveInsertPoint())
2730 EmitBlock(BB: createBasicBlock());
2731 }
2732
2733 /// ErrorUnsupported - Print out an error that codegen doesn't support the
2734 /// specified stmt yet.
2735 void ErrorUnsupported(const Stmt *S, const char *Type);
2736
2737 //===--------------------------------------------------------------------===//
2738 // Helpers
2739 //===--------------------------------------------------------------------===//
2740
2741 Address mergeAddressesInConditionalExpr(Address LHS, Address RHS,
2742 llvm::BasicBlock *LHSBlock,
2743 llvm::BasicBlock *RHSBlock,
2744 llvm::BasicBlock *MergeBlock,
2745 QualType MergedType) {
2746 Builder.SetInsertPoint(MergeBlock);
2747 llvm::PHINode *PtrPhi = Builder.CreatePHI(Ty: LHS.getType(), NumReservedValues: 2, Name: "cond");
2748 PtrPhi->addIncoming(V: LHS.getBasePointer(), BB: LHSBlock);
2749 PtrPhi->addIncoming(V: RHS.getBasePointer(), BB: RHSBlock);
2750 LHS.replaceBasePointer(P: PtrPhi);
2751 LHS.setAlignment(std::min(a: LHS.getAlignment(), b: RHS.getAlignment()));
2752 return LHS;
2753 }
2754
2755 /// Construct an address with the natural alignment of T. If a pointer to T
2756 /// is expected to be signed, the pointer passed to this function must have
2757 /// been signed, and the returned Address will have the pointer authentication
2758 /// information needed to authenticate the signed pointer.
2759 Address makeNaturalAddressForPointer(
2760 llvm::Value *Ptr, QualType T, CharUnits Alignment = CharUnits::Zero(),
2761 bool ForPointeeType = false, LValueBaseInfo *BaseInfo = nullptr,
2762 TBAAAccessInfo *TBAAInfo = nullptr,
2763 KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
2764 if (Alignment.isZero())
2765 Alignment =
2766 CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, forPointeeType: ForPointeeType);
2767 return Address(Ptr, ConvertTypeForMem(T), Alignment,
2768 CGM.getPointerAuthInfoForPointeeType(type: T), /*Offset=*/nullptr,
2769 IsKnownNonNull);
2770 }
2771
2772 LValue MakeAddrLValue(Address Addr, QualType T,
2773 AlignmentSource Source = AlignmentSource::Type) {
2774 return MakeAddrLValue(Addr, T, BaseInfo: LValueBaseInfo(Source),
2775 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2776 }
2777
2778 LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo,
2779 TBAAAccessInfo TBAAInfo) {
2780 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo, TBAAInfo);
2781 }
2782
2783 LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2784 AlignmentSource Source = AlignmentSource::Type) {
2785 return MakeAddrLValue(Addr: makeNaturalAddressForPointer(Ptr: V, T, Alignment), T,
2786 BaseInfo: LValueBaseInfo(Source), TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2787 }
2788
2789 /// Same as MakeAddrLValue above except that the pointer is known to be
2790 /// unsigned.
2791 LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2792 AlignmentSource Source = AlignmentSource::Type) {
2793 Address Addr(V, ConvertTypeForMem(T), Alignment);
2794 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2795 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2796 }
2797
2798 LValue
2799 MakeAddrLValueWithoutTBAA(Address Addr, QualType T,
2800 AlignmentSource Source = AlignmentSource::Type) {
2801 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2802 TBAAInfo: TBAAAccessInfo());
2803 }
2804
2805 /// Given a value of type T* that may not be to a complete object, construct
2806 /// an l-value with the natural pointee alignment of T.
2807 LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T);
2808
2809 LValue
2810 MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
2811 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
2812
2813 /// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known
2814 /// to be unsigned.
2815 LValue MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, QualType T);
2816
2817 LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T);
2818
2819 Address EmitLoadOfReference(LValue RefLVal,
2820 LValueBaseInfo *PointeeBaseInfo = nullptr,
2821 TBAAAccessInfo *PointeeTBAAInfo = nullptr);
2822 LValue EmitLoadOfReferenceLValue(LValue RefLVal);
2823 LValue
2824 EmitLoadOfReferenceLValue(Address RefAddr, QualType RefTy,
2825 AlignmentSource Source = AlignmentSource::Type) {
2826 LValue RefLVal = MakeAddrLValue(Addr: RefAddr, T: RefTy, BaseInfo: LValueBaseInfo(Source),
2827 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: RefTy));
2828 return EmitLoadOfReferenceLValue(RefLVal);
2829 }
2830
2831 /// Load a pointer with type \p PtrTy stored at address \p Ptr.
2832 /// Note that \p PtrTy is the type of the loaded pointer, not the addresses
2833 /// it is loaded from.
2834 Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy,
2835 LValueBaseInfo *BaseInfo = nullptr,
2836 TBAAAccessInfo *TBAAInfo = nullptr);
2837 LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy);
2838
2839private:
2840 struct AllocaTracker {
2841 void Add(llvm::AllocaInst *I) { Allocas.push_back(Elt: I); }
2842 llvm::SmallVector<llvm::AllocaInst *> Take() { return std::move(Allocas); }
2843
2844 private:
2845 llvm::SmallVector<llvm::AllocaInst *> Allocas;
2846 };
2847 AllocaTracker *Allocas = nullptr;
2848
2849 /// CGDecl helper.
2850 void emitStoresForConstant(const VarDecl &D, Address Loc, bool isVolatile,
2851 llvm::Constant *constant, bool IsAutoInit);
2852 /// CGDecl helper.
2853 void emitStoresForZeroInit(const VarDecl &D, Address Loc, bool isVolatile);
2854 /// CGDecl helper.
2855 void emitStoresForPatternInit(const VarDecl &D, Address Loc, bool isVolatile);
2856 /// CGDecl helper.
2857 void emitStoresForInitAfterBZero(llvm::Constant *Init, Address Loc,
2858 bool isVolatile, bool IsAutoInit);
2859
2860public:
2861 // Captures all the allocas created during the scope of its RAII object.
2862 struct AllocaTrackerRAII {
2863 AllocaTrackerRAII(CodeGenFunction &CGF)
2864 : CGF(CGF), OldTracker(CGF.Allocas) {
2865 CGF.Allocas = &Tracker;
2866 }
2867 ~AllocaTrackerRAII() { CGF.Allocas = OldTracker; }
2868
2869 llvm::SmallVector<llvm::AllocaInst *> Take() { return Tracker.Take(); }
2870
2871 private:
2872 CodeGenFunction &CGF;
2873 AllocaTracker *OldTracker;
2874 AllocaTracker Tracker;
2875 };
2876
2877private:
2878 /// If \p Alloca is not in the same address space as \p DestLangAS, insert an
2879 /// address space cast and return a new RawAddress based on this value.
2880 RawAddress MaybeCastStackAddressSpace(RawAddress Alloca, LangAS DestLangAS,
2881 llvm::Value *ArraySize = nullptr);
2882
2883public:
2884 /// CreateTempAlloca - This creates an alloca and inserts it into the entry
2885 /// block if \p ArraySize is nullptr, otherwise inserts it at the current
2886 /// insertion point of the builder. The caller is responsible for setting an
2887 /// appropriate alignment on the alloca.
2888 ///
2889 /// \p ArraySize is the number of array elements to be allocated if it
2890 /// is not nullptr.
2891 ///
2892 /// LangAS::Default is the address space of pointers to local variables and
2893 /// temporaries, as exposed in the source language. In certain
2894 /// configurations, this is not the same as the alloca address space, and a
2895 /// cast is needed to lift the pointer from the alloca AS into
2896 /// LangAS::Default. This can happen when the target uses a restricted
2897 /// address space for the stack but the source language requires
2898 /// LangAS::Default to be a generic address space. The latter condition is
2899 /// common for most programming languages; OpenCL is an exception in that
2900 /// LangAS::Default is the private address space, which naturally maps
2901 /// to the stack.
2902 ///
2903 /// Because the address of a temporary is often exposed to the program in
2904 /// various ways, this function will perform the cast. The original alloca
2905 /// instruction is returned through \p Alloca if it is not nullptr.
2906 ///
2907 /// The cast is not performed in CreateTempAllocaWithoutCast. This is
2908 /// more efficient if the caller knows that the address will not be exposed.
2909 llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp",
2910 llvm::Value *ArraySize = nullptr);
2911
2912 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
2913 /// block. The alloca is casted to the address space of \p UseAddrSpace if
2914 /// necessary.
2915 RawAddress CreateTempAlloca(llvm::Type *Ty, LangAS UseAddrSpace,
2916 CharUnits align, const Twine &Name = "tmp",
2917 llvm::Value *ArraySize = nullptr,
2918 RawAddress *Alloca = nullptr);
2919
2920 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
2921 /// block. The alloca is casted to default address space if necessary.
2922 ///
2923 /// FIXME: This version should be removed, and context should provide the
2924 /// context use address space used instead of default.
2925 RawAddress CreateTempAlloca(llvm::Type *Ty, CharUnits align,
2926 const Twine &Name = "tmp",
2927 llvm::Value *ArraySize = nullptr,
2928 RawAddress *Alloca = nullptr) {
2929 return CreateTempAlloca(Ty, UseAddrSpace: LangAS::Default, align, Name, ArraySize,
2930 Alloca);
2931 }
2932
2933 RawAddress CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align,
2934 const Twine &Name = "tmp",
2935 llvm::Value *ArraySize = nullptr);
2936
2937 /// CreateDefaultAlignedTempAlloca - This creates an alloca with the
2938 /// default ABI alignment of the given LLVM type.
2939 ///
2940 /// IMPORTANT NOTE: This is *not* generally the right alignment for
2941 /// any given AST type that happens to have been lowered to the
2942 /// given IR type. This should only ever be used for function-local,
2943 /// IR-driven manipulations like saving and restoring a value. Do
2944 /// not hand this address off to arbitrary IRGen routines, and especially
2945 /// do not pass it as an argument to a function that might expect a
2946 /// properly ABI-aligned value.
2947 RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty,
2948 const Twine &Name = "tmp");
2949
2950 /// CreateIRTempWithoutCast - Create a temporary IR object of the given type,
2951 /// with appropriate alignment. This routine should only be used when an
2952 /// temporary value needs to be stored into an alloca (for example, to avoid
2953 /// explicit PHI construction), but the type is the IR type, not the type
2954 /// appropriate for storing in memory.
2955 ///
2956 /// That is, this is exactly equivalent to CreateMemTemp, but calling
2957 /// ConvertType instead of ConvertTypeForMem.
2958 RawAddress CreateIRTempWithoutCast(QualType T, const Twine &Name = "tmp");
2959
2960 /// CreateMemTemp - Create a temporary memory object of the given type, with
2961 /// appropriate alignmen and cast it to the default address space. Returns
2962 /// the original alloca instruction by \p Alloca if it is not nullptr.
2963 RawAddress CreateMemTemp(QualType T, const Twine &Name = "tmp",
2964 RawAddress *Alloca = nullptr);
2965 RawAddress CreateMemTemp(QualType T, CharUnits Align,
2966 const Twine &Name = "tmp",
2967 RawAddress *Alloca = nullptr);
2968
2969 /// CreateMemTemp - Create a temporary memory object of the given type, with
2970 /// appropriate alignmen without casting it to the default address space.
2971 RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp");
2972 RawAddress CreateMemTempWithoutCast(QualType T, CharUnits Align,
2973 const Twine &Name = "tmp");
2974
2975 /// CreateAggTemp - Create a temporary memory object for the given
2976 /// aggregate type.
2977 AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp",
2978 RawAddress *Alloca = nullptr) {
2979 RawAddress Addr = CreateMemTemp(T, Name, Alloca);
2980 return AggValueSlot::forAddr(
2981 addr: Addr, quals: T.getQualifiers(), isDestructed: AggValueSlot::IsNotDestructed,
2982 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2983 mayOverlap: AggValueSlot::DoesNotOverlap);
2984 }
2985
2986 /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
2987 /// expression and compare the result against zero, returning an Int1Ty value.
2988 llvm::Value *EvaluateExprAsBool(const Expr *E);
2989
2990 /// Retrieve the implicit cast expression of the rhs in a binary operator
2991 /// expression by passing pointers to Value and QualType
2992 /// This is used for implicit bitfield conversion checks, which
2993 /// must compare with the value before potential truncation.
2994 llvm::Value *EmitWithOriginalRHSBitfieldAssignment(const BinaryOperator *E,
2995 llvm::Value **Previous,
2996 QualType *SrcType);
2997
2998 /// Emit a check that an [implicit] conversion of a bitfield. It is not UB,
2999 /// so we use the value after conversion.
3000 void EmitBitfieldConversionCheck(llvm::Value *Src, QualType SrcType,
3001 llvm::Value *Dst, QualType DstType,
3002 const CGBitFieldInfo &Info,
3003 SourceLocation Loc);
3004
3005 /// EmitIgnoredExpr - Emit an expression in a context which ignores the
3006 /// result.
3007 void EmitIgnoredExpr(const Expr *E);
3008
3009 /// EmitAnyExpr - Emit code to compute the specified expression which can have
3010 /// any type. The result is returned as an RValue struct. If this is an
3011 /// aggregate expression, the aggloc/agglocvolatile arguments indicate where
3012 /// the result should be returned.
3013 ///
3014 /// \param ignoreResult True if the resulting value isn't used.
3015 RValue EmitAnyExpr(const Expr *E,
3016 AggValueSlot aggSlot = AggValueSlot::ignored(),
3017 bool ignoreResult = false);
3018
3019 // EmitVAListRef - Emit a "reference" to a va_list; this is either the address
3020 // or the value of the expression, depending on how va_list is defined.
3021 Address EmitVAListRef(const Expr *E);
3022
3023 /// Emit a "reference" to a __builtin_ms_va_list; this is
3024 /// always the value of the expression, because a __builtin_ms_va_list is a
3025 /// pointer to a char.
3026 Address EmitMSVAListRef(const Expr *E);
3027
3028 /// Emit a "reference" to a __builtin_zos_va_list; this is always the
3029 /// address of the expression, because a __builtin_zos_va_list is an
3030 /// array of pointer to a char.
3031 Address EmitZOSVAListRef(const Expr *E);
3032
3033 /// EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will
3034 /// always be accessible even if no aggregate location is provided.
3035 RValue EmitAnyExprToTemp(const Expr *E);
3036
3037 /// EmitAnyExprToMem - Emits the code necessary to evaluate an
3038 /// arbitrary expression into the given memory location.
3039 void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals,
3040 bool IsInitializer);
3041
3042 void EmitAnyExprToExn(const Expr *E, Address Addr);
3043
3044 /// EmitInitializationToLValue - Emit an initializer to an LValue.
3045 void EmitInitializationToLValue(
3046 const Expr *E, LValue LV,
3047 AggValueSlot::IsZeroed_t IsZeroed = AggValueSlot::IsNotZeroed);
3048
3049 /// EmitExprAsInit - Emits the code necessary to initialize a
3050 /// location in memory with the given initializer.
3051 void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue,
3052 bool capturedByInit);
3053
3054 /// hasVolatileMember - returns true if aggregate type has a volatile
3055 /// member.
3056 bool hasVolatileMember(QualType T) {
3057 if (const auto *RD = T->getAsRecordDecl())
3058 return RD->hasVolatileMember();
3059 return false;
3060 }
3061
3062 /// Determine whether a return value slot may overlap some other object.
3063 AggValueSlot::Overlap_t getOverlapForReturnValue() {
3064 // FIXME: Assuming no overlap here breaks guaranteed copy elision for base
3065 // class subobjects. These cases may need to be revisited depending on the
3066 // resolution of the relevant core issue.
3067 return AggValueSlot::DoesNotOverlap;
3068 }
3069
3070 /// Determine whether a field initialization may overlap some other object.
3071 AggValueSlot::Overlap_t getOverlapForFieldInit(const FieldDecl *FD);
3072
3073 /// Determine whether a base class initialization may overlap some other
3074 /// object.
3075 AggValueSlot::Overlap_t getOverlapForBaseInit(const CXXRecordDecl *RD,
3076 const CXXRecordDecl *BaseRD,
3077 bool IsVirtual);
3078
3079 /// Emit an aggregate assignment.
3080 void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy) {
3081 ApplyAtomGroup Grp(getDebugInfo());
3082 bool IsVolatile = hasVolatileMember(T: EltTy);
3083 EmitAggregateCopy(Dest, Src, EltTy, MayOverlap: AggValueSlot::MayOverlap, isVolatile: IsVolatile);
3084 }
3085
3086 void EmitAggregateCopyCtor(LValue Dest, LValue Src,
3087 AggValueSlot::Overlap_t MayOverlap) {
3088 EmitAggregateCopy(Dest, Src, EltTy: Src.getType(), MayOverlap);
3089 }
3090
3091 /// EmitAggregateCopy - Emit an aggregate copy.
3092 ///
3093 /// \param isVolatile \c true iff either the source or the destination is
3094 /// volatile.
3095 /// \param MayOverlap Whether the tail padding of the destination might be
3096 /// occupied by some other object. More efficient code can often be
3097 /// generated if not.
3098 void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy,
3099 AggValueSlot::Overlap_t MayOverlap,
3100 bool isVolatile = false);
3101
3102 /// GetAddrOfLocalVar - Return the address of a local variable.
3103 Address GetAddrOfLocalVar(const VarDecl *VD) {
3104 auto it = LocalDeclMap.find(Val: VD);
3105 assert(it != LocalDeclMap.end() &&
3106 "Invalid argument to GetAddrOfLocalVar(), no decl!");
3107 return it->second;
3108 }
3109
3110 /// Given an opaque value expression, return its LValue mapping if it exists,
3111 /// otherwise create one.
3112 LValue getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e);
3113
3114 /// Given an opaque value expression, return its RValue mapping if it exists,
3115 /// otherwise create one.
3116 RValue getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e);
3117
3118 /// isOpaqueValueEmitted - Return true if the opaque value expression has
3119 /// already been emitted.
3120 bool isOpaqueValueEmitted(const OpaqueValueExpr *E);
3121
3122 /// Get the index of the current ArrayInitLoopExpr, if any.
3123 llvm::Value *getArrayInitIndex() { return ArrayInitIndex; }
3124
3125 /// getAccessedFieldNo - Given an encoded value and a result number, return
3126 /// the input field number being accessed.
3127 static unsigned getAccessedFieldNo(unsigned Idx, const llvm::Constant *Elts);
3128
3129 llvm::BlockAddress *GetAddrOfLabel(const LabelDecl *L);
3130 llvm::BasicBlock *GetIndirectGotoBlock();
3131
3132 /// Check if \p E is a C++ "this" pointer wrapped in value-preserving casts.
3133 static bool IsWrappedCXXThis(const Expr *E);
3134
3135 /// EmitNullInitialization - Generate code to set a value of the given type to
3136 /// null, If the type contains data member pointers, they will be initialized
3137 /// to -1 in accordance with the Itanium C++ ABI.
3138 void EmitNullInitialization(Address DestPtr, QualType Ty);
3139
3140 /// Emits a call to an LLVM variable-argument intrinsic, either
3141 /// \c llvm.va_start or \c llvm.va_end.
3142 /// \param ArgValue A reference to the \c va_list as emitted by either
3143 /// \c EmitVAListRef or \c EmitMSVAListRef.
3144 /// \param IsStart If \c true, emits a call to \c llvm.va_start; otherwise,
3145 /// calls \c llvm.va_end.
3146 llvm::Value *EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart);
3147
3148 /// Generate code to get an argument from the passed in pointer
3149 /// and update it accordingly.
3150 /// \param VE The \c VAArgExpr for which to generate code.
3151 /// \param VAListAddr Receives a reference to the \c va_list as emitted by
3152 /// either \c EmitVAListRef or \c EmitMSVAListRef.
3153 /// \returns A pointer to the argument.
3154 // FIXME: We should be able to get rid of this method and use the va_arg
3155 // instruction in LLVM instead once it works well enough.
3156 RValue EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
3157 AggValueSlot Slot = AggValueSlot::ignored());
3158
3159 /// emitArrayLength - Compute the length of an array, even if it's a
3160 /// VLA, and drill down to the base element type.
3161 llvm::Value *emitArrayLength(const ArrayType *arrayType, QualType &baseType,
3162 Address &addr);
3163
3164 /// EmitVLASize - Capture all the sizes for the VLA expressions in
3165 /// the given variably-modified type and store them in the VLASizeMap.
3166 ///
3167 /// This function can be called with a null (unreachable) insert point.
3168 void EmitVariablyModifiedType(QualType Ty);
3169
3170 struct VlaSizePair {
3171 llvm::Value *NumElts;
3172 QualType Type;
3173
3174 VlaSizePair(llvm::Value *NE, QualType T) : NumElts(NE), Type(T) {}
3175 };
3176
3177 /// Return the number of elements for a single dimension
3178 /// for the given array type.
3179 VlaSizePair getVLAElements1D(const VariableArrayType *vla);
3180 VlaSizePair getVLAElements1D(QualType vla);
3181
3182 /// Returns an LLVM value that corresponds to the size,
3183 /// in non-variably-sized elements, of a variable length array type,
3184 /// plus that largest non-variably-sized element type. Assumes that
3185 /// the type has already been emitted with EmitVariablyModifiedType.
3186 VlaSizePair getVLASize(const VariableArrayType *vla);
3187 VlaSizePair getVLASize(QualType vla);
3188
3189 /// LoadCXXThis - Load the value of 'this'. This function is only valid while
3190 /// generating code for an C++ member function.
3191 llvm::Value *LoadCXXThis() {
3192 assert(CXXThisValue && "no 'this' value for this function");
3193 return CXXThisValue;
3194 }
3195 Address LoadCXXThisAddress();
3196
3197 /// LoadCXXVTT - Load the VTT parameter to base constructors/destructors have
3198 /// virtual bases.
3199 // FIXME: Every place that calls LoadCXXVTT is something
3200 // that needs to be abstracted properly.
3201 llvm::Value *LoadCXXVTT() {
3202 assert(CXXStructorImplicitParamValue && "no VTT value for this function");
3203 return CXXStructorImplicitParamValue;
3204 }
3205
3206 /// GetAddressOfBaseOfCompleteClass - Convert the given pointer to a
3207 /// complete class to the given direct base.
3208 Address GetAddressOfDirectBaseInCompleteClass(Address Value,
3209 const CXXRecordDecl *Derived,
3210 const CXXRecordDecl *Base,
3211 bool BaseIsVirtual);
3212
3213 static bool ShouldNullCheckClassCastValue(const CastExpr *Cast);
3214
3215 /// GetAddressOfBaseClass - This function will add the necessary delta to the
3216 /// load of 'this' and returns address of the base class.
3217 Address GetAddressOfBaseClass(Address Value, const CXXRecordDecl *Derived,
3218 CastExpr::path_const_iterator PathBegin,
3219 CastExpr::path_const_iterator PathEnd,
3220 bool NullCheckValue, SourceLocation Loc);
3221
3222 Address GetAddressOfDerivedClass(Address Value, const CXXRecordDecl *Derived,
3223 CastExpr::path_const_iterator PathBegin,
3224 CastExpr::path_const_iterator PathEnd,
3225 bool NullCheckValue);
3226
3227 /// GetVTTParameter - Return the VTT parameter that should be passed to a
3228 /// base constructor/destructor with virtual bases.
3229 /// FIXME: VTTs are Itanium ABI-specific, so the definition should move
3230 /// to ItaniumCXXABI.cpp together with all the references to VTT.
3231 llvm::Value *GetVTTParameter(GlobalDecl GD, bool ForVirtualBase,
3232 bool Delegating);
3233
3234 void EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
3235 CXXCtorType CtorType,
3236 const FunctionArgList &Args,
3237 SourceLocation Loc);
3238 // It's important not to confuse this and the previous function. Delegating
3239 // constructors are the C++0x feature. The constructor delegate optimization
3240 // is used to reduce duplication in the base and complete consturctors where
3241 // they are substantially the same.
3242 void EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3243 const FunctionArgList &Args);
3244
3245 /// Emit a call to an inheriting constructor (that is, one that invokes a
3246 /// constructor inherited from a base class) by inlining its definition. This
3247 /// is necessary if the ABI does not support forwarding the arguments to the
3248 /// base class constructor (because they're variadic or similar).
3249 void EmitInlinedInheritingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3250 CXXCtorType CtorType,
3251 bool ForVirtualBase,
3252 bool Delegating,
3253 CallArgList &Args);
3254
3255 /// Emit a call to a constructor inherited from a base class, passing the
3256 /// current constructor's arguments along unmodified (without even making
3257 /// a copy).
3258 void EmitInheritedCXXConstructorCall(const CXXConstructorDecl *D,
3259 bool ForVirtualBase, Address This,
3260 bool InheritedFromVBase,
3261 const CXXInheritedCtorInitExpr *E);
3262
3263 void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3264 bool ForVirtualBase, bool Delegating,
3265 AggValueSlot ThisAVS, const CXXConstructExpr *E);
3266
3267 void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3268 bool ForVirtualBase, bool Delegating,
3269 Address This, CallArgList &Args,
3270 AggValueSlot::Overlap_t Overlap,
3271 SourceLocation Loc, bool NewPointerIsChecked,
3272 llvm::CallBase **CallOrInvoke = nullptr);
3273
3274 /// Emit assumption load for all bases. Requires to be called only on
3275 /// most-derived class and not under construction of the object.
3276 void EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl, Address This);
3277
3278 /// Emit assumption that vptr load == global vtable.
3279 void EmitVTableAssumptionLoad(const VPtr &vptr, Address This);
3280
3281 void EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, Address This,
3282 Address Src, const CXXConstructExpr *E);
3283
3284 void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3285 const ArrayType *ArrayTy, Address ArrayPtr,
3286 const CXXConstructExpr *E,
3287 bool NewPointerIsChecked,
3288 bool ZeroInitialization = false);
3289
3290 void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3291 llvm::Value *NumElements, Address ArrayPtr,
3292 const CXXConstructExpr *E,
3293 bool NewPointerIsChecked,
3294 bool ZeroInitialization = false);
3295
3296 static Destroyer destroyCXXObject;
3297
3298 void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type,
3299 bool ForVirtualBase, bool Delegating, Address This,
3300 QualType ThisTy);
3301
3302 void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
3303 llvm::Type *ElementTy, Address NewPtr,
3304 llvm::Value *NumElements,
3305 llvm::Value *AllocSizeWithoutCookie);
3306
3307 void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
3308 Address Ptr);
3309
3310 void EmitSehCppScopeBegin();
3311 void EmitSehCppScopeEnd();
3312 void EmitSehTryScopeBegin();
3313 void EmitSehTryScopeEnd();
3314
3315 bool EmitLifetimeStart(llvm::Value *Addr);
3316 void EmitLifetimeEnd(llvm::Value *Addr);
3317
3318 llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
3319 void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
3320
3321 void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,
3322 QualType DeleteTy, llvm::Value *NumElements = nullptr,
3323 CharUnits CookieSize = CharUnits(),
3324 llvm::Constant *CalleeOverride = nullptr);
3325
3326 RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
3327 const CallExpr *TheCallExpr, bool IsDelete);
3328
3329 llvm::Value *EmitCXXTypeidExpr(const CXXTypeidExpr *E);
3330 llvm::Value *EmitDynamicCast(Address V, const CXXDynamicCastExpr *DCE);
3331 Address EmitCXXUuidofExpr(const CXXUuidofExpr *E);
3332
3333 /// Situations in which we might emit a check for the suitability of a
3334 /// pointer or glvalue. Needs to be kept in sync with ubsan_handlers.cpp in
3335 /// compiler-rt.
3336 enum TypeCheckKind {
3337 /// Checking the operand of a load. Must be suitably sized and aligned.
3338 TCK_Load,
3339 /// Checking the destination of a store. Must be suitably sized and aligned.
3340 TCK_Store,
3341 /// Checking the bound value in a reference binding. Must be suitably sized
3342 /// and aligned, but is not required to refer to an object (until the
3343 /// reference is used), per core issue 453.
3344 TCK_ReferenceBinding,
3345 /// Checking the object expression in a non-static data member access. Must
3346 /// be an object within its lifetime.
3347 TCK_MemberAccess,
3348 /// Checking the 'this' pointer for a call to a non-static member function.
3349 /// Must be an object within its lifetime.
3350 TCK_MemberCall,
3351 /// Checking the 'this' pointer for a constructor call.
3352 TCK_ConstructorCall,
3353 /// Checking the operand of a static_cast to a derived pointer type. Must be
3354 /// null or an object within its lifetime.
3355 TCK_DowncastPointer,
3356 /// Checking the operand of a static_cast to a derived reference type. Must
3357 /// be an object within its lifetime.
3358 TCK_DowncastReference,
3359 /// Checking the operand of a cast to a base object. Must be suitably sized
3360 /// and aligned.
3361 TCK_Upcast,
3362 /// Checking the operand of a cast to a virtual base object. Must be an
3363 /// object within its lifetime.
3364 TCK_UpcastToVirtualBase,
3365 /// Checking the value assigned to a _Nonnull pointer. Must not be null.
3366 TCK_NonnullAssign,
3367 /// Checking the operand of a dynamic_cast or a typeid expression. Must be
3368 /// null or an object within its lifetime.
3369 TCK_DynamicOperation
3370 };
3371
3372 /// Determine whether the pointer type check \p TCK permits null pointers.
3373 static bool isNullPointerAllowed(TypeCheckKind TCK);
3374
3375 /// Determine whether the pointer type check \p TCK requires a vptr check.
3376 static bool isVptrCheckRequired(TypeCheckKind TCK, QualType Ty);
3377
3378 /// Whether any type-checking sanitizers are enabled. If \c false,
3379 /// calls to EmitTypeCheck can be skipped.
3380 bool sanitizePerformTypeCheck() const;
3381
3382 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV,
3383 QualType Type, SanitizerSet SkippedChecks = SanitizerSet(),
3384 llvm::Value *ArraySize = nullptr) {
3385 if (!sanitizePerformTypeCheck())
3386 return;
3387 EmitTypeCheck(TCK, Loc, V: LV.emitRawPointer(CGF&: *this), Type, Alignment: LV.getAlignment(),
3388 SkippedChecks, ArraySize);
3389 }
3390
3391 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, Address Addr,
3392 QualType Type, CharUnits Alignment = CharUnits::Zero(),
3393 SanitizerSet SkippedChecks = SanitizerSet(),
3394 llvm::Value *ArraySize = nullptr) {
3395 if (!sanitizePerformTypeCheck())
3396 return;
3397 EmitTypeCheck(TCK, Loc, V: Addr.emitRawPointer(CGF&: *this), Type, Alignment,
3398 SkippedChecks, ArraySize);
3399 }
3400
3401 /// Emit a check that \p V is the address of storage of the
3402 /// appropriate size and alignment for an object of type \p Type
3403 /// (or if ArraySize is provided, for an array of that bound).
3404 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Value *V,
3405 QualType Type, CharUnits Alignment = CharUnits::Zero(),
3406 SanitizerSet SkippedChecks = SanitizerSet(),
3407 llvm::Value *ArraySize = nullptr);
3408
3409 /// Emit a check that \p Base points into an array object, which
3410 /// we can access at index \p Index. \p Accessed should be \c false if we
3411 /// this expression is used as an lvalue, for instance in "&Arr[Idx]".
3412 void EmitBoundsCheck(const Expr *ArrayExpr, const Expr *ArrayExprBase,
3413 llvm::Value *Index, QualType IndexType, bool Accessed);
3414 void EmitBoundsCheckImpl(const Expr *ArrayExpr, QualType ArrayBaseType,
3415 llvm::Value *IndexVal, QualType IndexType,
3416 llvm::Value *BoundsVal, QualType BoundsType,
3417 bool Accessed);
3418
3419 /// Returns debug info, with additional annotation if
3420 /// CGM.getCodeGenOpts().SanitizeAnnotateDebugInfo[Ordinal] is enabled for
3421 /// any of the ordinals.
3422 llvm::DILocation *
3423 SanitizerAnnotateDebugInfo(ArrayRef<SanitizerKind::SanitizerOrdinal> Ordinals,
3424 SanitizerHandler Handler);
3425
3426 /// Build metadata used by the AllocToken instrumentation.
3427 llvm::MDNode *buildAllocToken(QualType AllocType);
3428 /// Emit and set additional metadata used by the AllocToken instrumentation.
3429 void EmitAllocToken(llvm::CallBase *CB, QualType AllocType);
3430 /// Build additional metadata used by the AllocToken instrumentation,
3431 /// inferring the type from an allocation call expression.
3432 llvm::MDNode *buildAllocToken(const CallExpr *E);
3433 /// Emit and set additional metadata used by the AllocToken instrumentation,
3434 /// inferring the type from an allocation call expression.
3435 void EmitAllocToken(llvm::CallBase *CB, const CallExpr *E);
3436
3437 llvm::Value *GetCountedByFieldExprGEP(const Expr *Base, const FieldDecl *FD,
3438 const FieldDecl *CountDecl);
3439
3440 /// Build an expression accessing the "counted_by" field.
3441 llvm::Value *EmitLoadOfCountedByField(const Expr *Base, const FieldDecl *FD,
3442 const FieldDecl *CountDecl);
3443
3444 // Emit bounds checking for flexible array and pointer members with the
3445 // counted_by attribute.
3446 void EmitCountedByBoundsChecking(const Expr *ArrayExpr, QualType ArrayType,
3447 Address ArrayInst, QualType IndexType,
3448 llvm::Value *IndexVal, bool Accessed,
3449 bool FlexibleArray);
3450
3451 llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
3452 bool isInc, bool isPre);
3453 ComplexPairTy EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
3454 bool isInc, bool isPre);
3455
3456 /// Converts Location to a DebugLoc, if debug information is enabled.
3457 llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location);
3458
3459 /// Get the record field index as represented in debug info.
3460 unsigned getDebugInfoFIndex(const RecordDecl *Rec, unsigned FieldIndex);
3461
3462 //===--------------------------------------------------------------------===//
3463 // Declaration Emission
3464 //===--------------------------------------------------------------------===//
3465
3466 /// EmitDecl - Emit a declaration.
3467 ///
3468 /// This function can be called with a null (unreachable) insert point.
3469 void EmitDecl(const Decl &D, bool EvaluateConditionDecl = false);
3470
3471 /// EmitVarDecl - Emit a local variable declaration.
3472 ///
3473 /// This function can be called with a null (unreachable) insert point.
3474 void EmitVarDecl(const VarDecl &D);
3475
3476 void EmitScalarInit(const Expr *init, const ValueDecl *D, LValue lvalue,
3477 bool capturedByInit);
3478
3479 typedef void SpecialInitFn(CodeGenFunction &Init, const VarDecl &D,
3480 llvm::Value *Address);
3481
3482 /// Determine whether the given initializer is trivial in the sense
3483 /// that it requires no code to be generated.
3484 bool isTrivialInitializer(const Expr *Init);
3485
3486 /// EmitAutoVarDecl - Emit an auto variable declaration.
3487 ///
3488 /// This function can be called with a null (unreachable) insert point.
3489 void EmitAutoVarDecl(const VarDecl &D);
3490
3491 class AutoVarEmission {
3492 friend class CodeGenFunction;
3493
3494 const VarDecl *Variable;
3495
3496 /// The address of the alloca for languages with explicit address space
3497 /// (e.g. OpenCL) or alloca casted to generic pointer for address space
3498 /// agnostic languages (e.g. C++). Invalid if the variable was emitted
3499 /// as a global constant.
3500 Address Addr;
3501
3502 llvm::Value *NRVOFlag;
3503
3504 /// True if the variable is a __block variable that is captured by an
3505 /// escaping block.
3506 bool IsEscapingByRef;
3507
3508 /// True if the variable is of aggregate type and has a constant
3509 /// initializer.
3510 bool IsConstantAggregate;
3511
3512 /// True if lifetime markers should be used.
3513 bool UseLifetimeMarkers;
3514
3515 /// Address with original alloca instruction. Invalid if the variable was
3516 /// emitted as a global constant.
3517 RawAddress AllocaAddr;
3518
3519 struct Invalid {};
3520 AutoVarEmission(Invalid)
3521 : Variable(nullptr), Addr(Address::invalid()),
3522 AllocaAddr(RawAddress::invalid()) {}
3523
3524 AutoVarEmission(const VarDecl &variable)
3525 : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr),
3526 IsEscapingByRef(false), IsConstantAggregate(false),
3527 UseLifetimeMarkers(false), AllocaAddr(RawAddress::invalid()) {}
3528
3529 bool wasEmittedAsGlobal() const { return !Addr.isValid(); }
3530
3531 public:
3532 static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }
3533
3534 bool useLifetimeMarkers() const { return UseLifetimeMarkers; }
3535
3536 /// Returns the raw, allocated address, which is not necessarily
3537 /// the address of the object itself. It is casted to default
3538 /// address space for address space agnostic languages.
3539 Address getAllocatedAddress() const { return Addr; }
3540
3541 /// Returns the address for the original alloca instruction.
3542 RawAddress getOriginalAllocatedAddress() const { return AllocaAddr; }
3543
3544 /// Returns the address of the object within this declaration.
3545 /// Note that this does not chase the forwarding pointer for
3546 /// __block decls.
3547 Address getObjectAddress(CodeGenFunction &CGF) const {
3548 if (!IsEscapingByRef)
3549 return Addr;
3550
3551 return CGF.emitBlockByrefAddress(baseAddr: Addr, V: Variable, /*forward*/ followForward: false);
3552 }
3553 };
3554 AutoVarEmission EmitAutoVarAlloca(const VarDecl &var);
3555 void EmitAutoVarInit(const AutoVarEmission &emission);
3556 void EmitAutoVarCleanups(const AutoVarEmission &emission);
3557 void emitAutoVarTypeCleanup(const AutoVarEmission &emission,
3558 QualType::DestructionKind dtorKind);
3559
3560 void MaybeEmitDeferredVarDeclInit(const VarDecl *var);
3561
3562 /// Emits the alloca and debug information for the size expressions for each
3563 /// dimension of an array. It registers the association of its (1-dimensional)
3564 /// QualTypes and size expression's debug node, so that CGDebugInfo can
3565 /// reference this node when creating the DISubrange object to describe the
3566 /// array types.
3567 void EmitAndRegisterVariableArrayDimensions(CGDebugInfo *DI, const VarDecl &D,
3568 bool EmitDebugInfo);
3569
3570 void EmitStaticVarDecl(const VarDecl &D,
3571 llvm::GlobalValue::LinkageTypes Linkage);
3572
3573 class ParamValue {
3574 union {
3575 Address Addr;
3576 llvm::Value *Value;
3577 };
3578
3579 bool IsIndirect;
3580
3581 ParamValue(llvm::Value *V) : Value(V), IsIndirect(false) {}
3582 ParamValue(Address A) : Addr(A), IsIndirect(true) {}
3583
3584 public:
3585 static ParamValue forDirect(llvm::Value *value) {
3586 return ParamValue(value);
3587 }
3588 static ParamValue forIndirect(Address addr) {
3589 assert(!addr.getAlignment().isZero());
3590 return ParamValue(addr);
3591 }
3592
3593 bool isIndirect() const { return IsIndirect; }
3594 llvm::Value *getAnyValue() const {
3595 if (!isIndirect())
3596 return Value;
3597 assert(!Addr.hasOffset() && "unexpected offset");
3598 return Addr.getBasePointer();
3599 }
3600
3601 llvm::Value *getDirectValue() const {
3602 assert(!isIndirect());
3603 return Value;
3604 }
3605
3606 Address getIndirectAddress() const {
3607 assert(isIndirect());
3608 return Addr;
3609 }
3610 };
3611
3612 /// EmitParmDecl - Emit a ParmVarDecl or an ImplicitParamDecl.
3613 void EmitParmDecl(const VarDecl &D, ParamValue Arg, unsigned ArgNo);
3614
3615 /// protectFromPeepholes - Protect a value that we're intending to
3616 /// store to the side, but which will probably be used later, from
3617 /// aggressive peepholing optimizations that might delete it.
3618 ///
3619 /// Pass the result to unprotectFromPeepholes to declare that
3620 /// protection is no longer required.
3621 ///
3622 /// There's no particular reason why this shouldn't apply to
3623 /// l-values, it's just that no existing peepholes work on pointers.
3624 PeepholeProtection protectFromPeepholes(RValue rvalue);
3625 void unprotectFromPeepholes(PeepholeProtection protection);
3626
3627 void emitAlignmentAssumptionCheck(llvm::Value *Ptr, QualType Ty,
3628 SourceLocation Loc,
3629 SourceLocation AssumptionLoc,
3630 llvm::Value *Alignment,
3631 llvm::Value *OffsetValue,
3632 llvm::Value *TheCheck,
3633 llvm::Instruction *Assumption);
3634
3635 void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty,
3636 SourceLocation Loc, SourceLocation AssumptionLoc,
3637 llvm::Value *Alignment,
3638 llvm::Value *OffsetValue = nullptr);
3639
3640 void emitAlignmentAssumption(llvm::Value *PtrValue, const Expr *E,
3641 SourceLocation AssumptionLoc,
3642 llvm::Value *Alignment,
3643 llvm::Value *OffsetValue = nullptr);
3644
3645 //===--------------------------------------------------------------------===//
3646 // Statement Emission
3647 //===--------------------------------------------------------------------===//
3648
3649 /// EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
3650 void EmitStopPoint(const Stmt *S);
3651
3652 /// EmitStmt - Emit the code for the statement \arg S. It is legal to call
3653 /// this function even if there is no current insertion point.
3654 ///
3655 /// This function may clear the current insertion point; callers should use
3656 /// EnsureInsertPoint if they wish to subsequently generate code without first
3657 /// calling EmitBlock, EmitBranch, or EmitStmt.
3658 void EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs = {});
3659
3660 /// EmitSimpleStmt - Try to emit a "simple" statement which does not
3661 /// necessarily require an insertion point or debug information; typically
3662 /// because the statement amounts to a jump or a container of other
3663 /// statements.
3664 ///
3665 /// \return True if the statement was handled.
3666 bool EmitSimpleStmt(const Stmt *S, ArrayRef<const Attr *> Attrs);
3667
3668 Address EmitCompoundStmt(const CompoundStmt &S, bool GetLast = false,
3669 AggValueSlot AVS = AggValueSlot::ignored());
3670 Address
3671 EmitCompoundStmtWithoutScope(const CompoundStmt &S, bool GetLast = false,
3672 AggValueSlot AVS = AggValueSlot::ignored());
3673
3674 /// EmitLabel - Emit the block for the given label. It is legal to call this
3675 /// function even if there is no current insertion point.
3676 void EmitLabel(const LabelDecl *D); // helper for EmitLabelStmt.
3677
3678 void EmitLabelStmt(const LabelStmt &S);
3679 void EmitAttributedStmt(const AttributedStmt &S);
3680 void EmitGotoStmt(const GotoStmt &S);
3681 void EmitIndirectGotoStmt(const IndirectGotoStmt &S);
3682 void EmitIfStmt(const IfStmt &S);
3683
3684 void EmitWhileStmt(const WhileStmt &S, ArrayRef<const Attr *> Attrs = {});
3685 void EmitDoStmt(const DoStmt &S, ArrayRef<const Attr *> Attrs = {});
3686 void EmitForStmt(const ForStmt &S, ArrayRef<const Attr *> Attrs = {});
3687 void EmitReturnStmt(const ReturnStmt &S);
3688 void EmitDeclStmt(const DeclStmt &S);
3689 void EmitBreakStmt(const BreakStmt &S);
3690 void EmitContinueStmt(const ContinueStmt &S);
3691 void EmitSwitchStmt(const SwitchStmt &S);
3692 void EmitDefaultStmt(const DefaultStmt &S, ArrayRef<const Attr *> Attrs);
3693 void EmitCaseStmt(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3694 void EmitCaseStmtRange(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3695 void EmitDeferStmt(const DeferStmt &S);
3696 void EmitAsmStmt(const AsmStmt &S);
3697
3698 const BreakContinue *GetDestForLoopControlStmt(const LoopControlStmt &S);
3699
3700 void EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S);
3701 void EmitObjCAtTryStmt(const ObjCAtTryStmt &S);
3702 void EmitObjCAtThrowStmt(const ObjCAtThrowStmt &S);
3703 void EmitObjCAtSynchronizedStmt(const ObjCAtSynchronizedStmt &S);
3704 void EmitObjCAutoreleasePoolStmt(const ObjCAutoreleasePoolStmt &S);
3705
3706 void EmitCoroutineBody(const CoroutineBodyStmt &S);
3707 void EmitCoreturnStmt(const CoreturnStmt &S);
3708 RValue EmitCoawaitExpr(const CoawaitExpr &E,
3709 AggValueSlot aggSlot = AggValueSlot::ignored(),
3710 bool ignoreResult = false);
3711 LValue EmitCoawaitLValue(const CoawaitExpr *E);
3712 RValue EmitCoyieldExpr(const CoyieldExpr &E,
3713 AggValueSlot aggSlot = AggValueSlot::ignored(),
3714 bool ignoreResult = false);
3715 LValue EmitCoyieldLValue(const CoyieldExpr *E);
3716 RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID);
3717
3718 void EmitSYCLKernelCallStmt(const SYCLKernelCallStmt &S);
3719
3720 void EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3721 void ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3722
3723 void EmitCXXTryStmt(const CXXTryStmt &S);
3724 void EmitSEHTryStmt(const SEHTryStmt &S);
3725 void EmitSEHLeaveStmt(const SEHLeaveStmt &S);
3726 void EnterSEHTryStmt(const SEHTryStmt &S);
3727 void ExitSEHTryStmt(const SEHTryStmt &S);
3728 void VolatilizeTryBlocks(llvm::BasicBlock *BB,
3729 llvm::SmallPtrSet<llvm::BasicBlock *, 10> &V);
3730
3731 void pushSEHCleanup(CleanupKind kind, llvm::Function *FinallyFunc);
3732 void startOutlinedSEHHelper(CodeGenFunction &ParentCGF, bool IsFilter,
3733 const Stmt *OutlinedStmt);
3734
3735 llvm::Function *GenerateSEHFilterFunction(CodeGenFunction &ParentCGF,
3736 const SEHExceptStmt &Except);
3737
3738 llvm::Function *GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF,
3739 const SEHFinallyStmt &Finally);
3740
3741 void EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF,
3742 llvm::Value *ParentFP, llvm::Value *EntryEBP);
3743 llvm::Value *EmitSEHExceptionCode();
3744 llvm::Value *EmitSEHExceptionInfo();
3745 llvm::Value *EmitSEHAbnormalTermination();
3746
3747 /// Emit simple code for OpenMP directives in Simd-only mode.
3748 void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D);
3749
3750 /// Scan the outlined statement for captures from the parent function. For
3751 /// each capture, mark the capture as escaped and emit a call to
3752 /// llvm.localrecover. Insert the localrecover result into the LocalDeclMap.
3753 void EmitCapturedLocals(CodeGenFunction &ParentCGF, const Stmt *OutlinedStmt,
3754 bool IsFilter);
3755
3756 /// Recovers the address of a local in a parent function. ParentVar is the
3757 /// address of the variable used in the immediate parent function. It can
3758 /// either be an alloca or a call to llvm.localrecover if there are nested
3759 /// outlined functions. ParentFP is the frame pointer of the outermost parent
3760 /// frame.
3761 Address recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
3762 Address ParentVar, llvm::Value *ParentFP);
3763
3764 void EmitCXXForRangeStmt(const CXXForRangeStmt &S,
3765 ArrayRef<const Attr *> Attrs = {});
3766
3767 void
3768 EmitCXXExpansionStmtInstantiation(const CXXExpansionStmtInstantiation &S);
3769
3770 /// Controls insertion of cancellation exit blocks in worksharing constructs.
3771 class OMPCancelStackRAII {
3772 CodeGenFunction &CGF;
3773
3774 public:
3775 OMPCancelStackRAII(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
3776 bool HasCancel)
3777 : CGF(CGF) {
3778 CGF.OMPCancelStack.enter(CGF, Kind, HasCancel);
3779 }
3780 ~OMPCancelStackRAII() { CGF.OMPCancelStack.exit(CGF); }
3781 };
3782
3783 /// Returns calculated size of the specified type.
3784 llvm::Value *getTypeSize(QualType Ty);
3785 LValue InitCapturedStruct(const CapturedStmt &S);
3786 llvm::Function *EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K);
3787 llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S);
3788 Address GenerateCapturedStmtArgument(const CapturedStmt &S);
3789 llvm::Function *
3790 GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S,
3791 const OMPExecutableDirective &D);
3792 llvm::Function *
3793 GenerateOpenMPCapturedStmtFunctionAggregate(const CapturedStmt &S,
3794 const OMPExecutableDirective &D);
3795 void GenerateOpenMPCapturedVars(const CapturedStmt &S,
3796 SmallVectorImpl<llvm::Value *> &CapturedVars);
3797 void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy,
3798 SourceLocation Loc);
3799 /// Perform element by element copying of arrays with type \a
3800 /// OriginalType from \a SrcAddr to \a DestAddr using copying procedure
3801 /// generated by \a CopyGen.
3802 ///
3803 /// \param DestAddr Address of the destination array.
3804 /// \param SrcAddr Address of the source array.
3805 /// \param OriginalType Type of destination and source arrays.
3806 /// \param CopyGen Copying procedure that copies value of single array element
3807 /// to another single array element.
3808 void EmitOMPAggregateAssign(
3809 Address DestAddr, Address SrcAddr, QualType OriginalType,
3810 const llvm::function_ref<void(Address, Address)> CopyGen);
3811 /// Emit proper copying of data from one variable to another.
3812 ///
3813 /// \param OriginalType Original type of the copied variables.
3814 /// \param DestAddr Destination address.
3815 /// \param SrcAddr Source address.
3816 /// \param DestVD Destination variable used in \a CopyExpr (for arrays, has
3817 /// type of the base array element).
3818 /// \param SrcVD Source variable used in \a CopyExpr (for arrays, has type of
3819 /// the base array element).
3820 /// \param Copy Actual copygin expression for copying data from \a SrcVD to \a
3821 /// DestVD.
3822 void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr,
3823 const VarDecl *DestVD, const VarDecl *SrcVD,
3824 const Expr *Copy);
3825 /// Emit atomic update code for constructs: \a X = \a X \a BO \a E or
3826 /// \a X = \a E \a BO \a E.
3827 ///
3828 /// \param X Value to be updated.
3829 /// \param E Update value.
3830 /// \param BO Binary operation for update operation.
3831 /// \param IsXLHSInRHSPart true if \a X is LHS in RHS part of the update
3832 /// expression, false otherwise.
3833 /// \param AO Atomic ordering of the generated atomic instructions.
3834 /// \param CommonGen Code generator for complex expressions that cannot be
3835 /// expressed through atomicrmw instruction.
3836 /// \returns <true, OldAtomicValue> if simple 'atomicrmw' instruction was
3837 /// generated, <false, RValue::get(nullptr)> otherwise.
3838 std::pair<bool, RValue> EmitOMPAtomicSimpleUpdateExpr(
3839 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3840 llvm::AtomicOrdering AO, SourceLocation Loc,
3841 const llvm::function_ref<RValue(RValue)> CommonGen);
3842 bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
3843 OMPPrivateScope &PrivateScope);
3844 void EmitOMPPrivateClause(const OMPExecutableDirective &D,
3845 OMPPrivateScope &PrivateScope);
3846 void EmitOMPUseDevicePtrClause(
3847 const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
3848 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3849 CaptureDeviceAddrMap);
3850 void EmitOMPUseDeviceAddrClause(
3851 const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
3852 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3853 CaptureDeviceAddrMap);
3854 /// Emit code for copyin clause in \a D directive. The next code is
3855 /// generated at the start of outlined functions for directives:
3856 /// \code
3857 /// threadprivate_var1 = master_threadprivate_var1;
3858 /// operator=(threadprivate_var2, master_threadprivate_var2);
3859 /// ...
3860 /// __kmpc_barrier(&loc, global_tid);
3861 /// \endcode
3862 ///
3863 /// \param D OpenMP directive possibly with 'copyin' clause(s).
3864 /// \returns true if at least one copyin variable is found, false otherwise.
3865 bool EmitOMPCopyinClause(const OMPExecutableDirective &D);
3866 /// Emit initial code for lastprivate variables. If some variable is
3867 /// not also firstprivate, then the default initialization is used. Otherwise
3868 /// initialization of this variable is performed by EmitOMPFirstprivateClause
3869 /// method.
3870 ///
3871 /// \param D Directive that may have 'lastprivate' directives.
3872 /// \param PrivateScope Private scope for capturing lastprivate variables for
3873 /// proper codegen in internal captured statement.
3874 ///
3875 /// \returns true if there is at least one lastprivate variable, false
3876 /// otherwise.
3877 bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D,
3878 OMPPrivateScope &PrivateScope);
3879 /// Emit final copying of lastprivate values to original variables at
3880 /// the end of the worksharing or simd directive.
3881 ///
3882 /// \param D Directive that has at least one 'lastprivate' directives.
3883 /// \param IsLastIterCond Boolean condition that must be set to 'i1 true' if
3884 /// it is the last iteration of the loop code in associated directive, or to
3885 /// 'i1 false' otherwise. If this item is nullptr, no final check is required.
3886 void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D,
3887 bool NoFinals,
3888 llvm::Value *IsLastIterCond = nullptr);
3889 /// Emit initial code for linear clauses.
3890 void EmitOMPLinearClause(const OMPLoopDirective &D,
3891 CodeGenFunction::OMPPrivateScope &PrivateScope);
3892 /// Emit final code for linear clauses.
3893 /// \param CondGen Optional conditional code for final part of codegen for
3894 /// linear clause.
3895 void EmitOMPLinearClauseFinal(
3896 const OMPLoopDirective &D,
3897 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
3898 /// Emit initial code for reduction variables. Creates reduction copies
3899 /// and initializes them with the values according to OpenMP standard.
3900 ///
3901 /// \param D Directive (possibly) with the 'reduction' clause.
3902 /// \param PrivateScope Private scope for capturing reduction variables for
3903 /// proper codegen in internal captured statement.
3904 ///
3905 void EmitOMPReductionClauseInit(const OMPExecutableDirective &D,
3906 OMPPrivateScope &PrivateScope,
3907 bool ForInscan = false);
3908 /// Emit final update of reduction values to original variables at
3909 /// the end of the directive.
3910 ///
3911 /// \param D Directive that has at least one 'reduction' directives.
3912 /// \param ReductionKind The kind of reduction to perform.
3913 void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D,
3914 const OpenMPDirectiveKind ReductionKind);
3915 /// Emit initial code for linear variables. Creates private copies
3916 /// and initializes them with the values according to OpenMP standard.
3917 ///
3918 /// \param D Directive (possibly) with the 'linear' clause.
3919 /// \return true if at least one linear variable is found that should be
3920 /// initialized with the value of the original variable, false otherwise.
3921 bool EmitOMPLinearClauseInit(const OMPLoopDirective &D);
3922
3923 typedef const llvm::function_ref<void(CodeGenFunction & /*CGF*/,
3924 llvm::Function * /*OutlinedFn*/,
3925 const OMPTaskDataTy & /*Data*/)>
3926 TaskGenTy;
3927 void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S,
3928 const OpenMPDirectiveKind CapturedRegion,
3929 const RegionCodeGenTy &BodyGen,
3930 const TaskGenTy &TaskGen, OMPTaskDataTy &Data);
3931 struct OMPTargetDataInfo {
3932 Address BasePointersArray = Address::invalid();
3933 Address PointersArray = Address::invalid();
3934 Address SizesArray = Address::invalid();
3935 Address MappersArray = Address::invalid();
3936 unsigned NumberOfTargetItems = 0;
3937 explicit OMPTargetDataInfo() = default;
3938 OMPTargetDataInfo(Address BasePointersArray, Address PointersArray,
3939 Address SizesArray, Address MappersArray,
3940 unsigned NumberOfTargetItems)
3941 : BasePointersArray(BasePointersArray), PointersArray(PointersArray),
3942 SizesArray(SizesArray), MappersArray(MappersArray),
3943 NumberOfTargetItems(NumberOfTargetItems) {}
3944 };
3945 void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S,
3946 const RegionCodeGenTy &BodyGen,
3947 OMPTargetDataInfo &InputInfo);
3948 void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data,
3949 CodeGenFunction &CGF, const CapturedStmt *CS,
3950 OMPPrivateScope &Scope);
3951 void EmitOMPMetaDirective(const OMPMetaDirective &S);
3952 void EmitOMPParallelDirective(const OMPParallelDirective &S);
3953 void EmitOMPSimdDirective(const OMPSimdDirective &S);
3954 void EmitOMPTileDirective(const OMPTileDirective &S);
3955 void EmitOMPStripeDirective(const OMPStripeDirective &S);
3956 void EmitOMPUnrollDirective(const OMPUnrollDirective &S);
3957 void EmitOMPReverseDirective(const OMPReverseDirective &S);
3958 void EmitOMPSplitDirective(const OMPSplitDirective &S);
3959 void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
3960 void EmitOMPFuseDirective(const OMPFuseDirective &S);
3961 void EmitOMPForDirective(const OMPForDirective &S);
3962 void EmitOMPForSimdDirective(const OMPForSimdDirective &S);
3963 void EmitOMPScopeDirective(const OMPScopeDirective &S);
3964 void EmitOMPSectionsDirective(const OMPSectionsDirective &S);
3965 void EmitOMPSectionDirective(const OMPSectionDirective &S);
3966 void EmitOMPSingleDirective(const OMPSingleDirective &S);
3967 void EmitOMPMasterDirective(const OMPMasterDirective &S);
3968 void EmitOMPMaskedDirective(const OMPMaskedDirective &S);
3969 void EmitOMPCriticalDirective(const OMPCriticalDirective &S);
3970 void EmitOMPParallelForDirective(const OMPParallelForDirective &S);
3971 void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S);
3972 void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S);
3973 void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S);
3974 void EmitOMPTaskDirective(const OMPTaskDirective &S);
3975 void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S);
3976 void EmitOMPErrorDirective(const OMPErrorDirective &S);
3977 void EmitOMPBarrierDirective(const OMPBarrierDirective &S);
3978 void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S);
3979 void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S);
3980 void EmitOMPFlushDirective(const OMPFlushDirective &S);
3981 void EmitOMPDepobjDirective(const OMPDepobjDirective &S);
3982 void EmitOMPScanDirective(const OMPScanDirective &S);
3983 void
3984 EmitOMPOrderedStandaloneDirective(const OMPOrderedStandaloneDirective &S);
3985 void
3986 EmitOMPOrderedBlockAssocDirective(const OMPOrderedBlockAssocDirective &S);
3987 void EmitOMPAtomicDirective(const OMPAtomicDirective &S);
3988 void EmitOMPTargetDirective(const OMPTargetDirective &S);
3989 void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S);
3990 void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S);
3991 void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S);
3992 void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S);
3993 void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S);
3994 void
3995 EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S);
3996 void EmitOMPTeamsDirective(const OMPTeamsDirective &S);
3997 void
3998 EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S);
3999 void EmitOMPCancelDirective(const OMPCancelDirective &S);
4000 void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S);
4001 void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S);
4002 void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S);
4003 void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S);
4004 void EmitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &S);
4005 void
4006 EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S);
4007 void
4008 EmitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &S);
4009 void EmitOMPParallelMasterTaskLoopDirective(
4010 const OMPParallelMasterTaskLoopDirective &S);
4011 void EmitOMPParallelMaskedTaskLoopDirective(
4012 const OMPParallelMaskedTaskLoopDirective &S);
4013 void EmitOMPParallelMasterTaskLoopSimdDirective(
4014 const OMPParallelMasterTaskLoopSimdDirective &S);
4015 void EmitOMPParallelMaskedTaskLoopSimdDirective(
4016 const OMPParallelMaskedTaskLoopSimdDirective &S);
4017 void EmitOMPDistributeDirective(const OMPDistributeDirective &S);
4018 void EmitOMPDistributeParallelForDirective(
4019 const OMPDistributeParallelForDirective &S);
4020 void EmitOMPDistributeParallelForSimdDirective(
4021 const OMPDistributeParallelForSimdDirective &S);
4022 void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S);
4023 void EmitOMPTargetParallelForSimdDirective(
4024 const OMPTargetParallelForSimdDirective &S);
4025 void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S);
4026 void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S);
4027 void
4028 EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S);
4029 void EmitOMPTeamsDistributeParallelForSimdDirective(
4030 const OMPTeamsDistributeParallelForSimdDirective &S);
4031 void EmitOMPTeamsDistributeParallelForDirective(
4032 const OMPTeamsDistributeParallelForDirective &S);
4033 void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S);
4034 void EmitOMPTargetTeamsDistributeDirective(
4035 const OMPTargetTeamsDistributeDirective &S);
4036 void EmitOMPTargetTeamsDistributeParallelForDirective(
4037 const OMPTargetTeamsDistributeParallelForDirective &S);
4038 void EmitOMPTargetTeamsDistributeParallelForSimdDirective(
4039 const OMPTargetTeamsDistributeParallelForSimdDirective &S);
4040 void EmitOMPTargetTeamsDistributeSimdDirective(
4041 const OMPTargetTeamsDistributeSimdDirective &S);
4042 void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S);
4043 void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S);
4044 void EmitOMPTargetParallelGenericLoopDirective(
4045 const OMPTargetParallelGenericLoopDirective &S);
4046 void EmitOMPTargetTeamsGenericLoopDirective(
4047 const OMPTargetTeamsGenericLoopDirective &S);
4048 void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S);
4049 void EmitOMPInteropDirective(const OMPInteropDirective &S);
4050 void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S);
4051 void EmitOMPAssumeDirective(const OMPAssumeDirective &S);
4052
4053 /// Emit device code for the target directive.
4054 static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
4055 StringRef ParentName,
4056 const OMPTargetDirective &S);
4057 static void
4058 EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
4059 const OMPTargetParallelDirective &S);
4060 /// Emit device code for the target parallel for directive.
4061 static void EmitOMPTargetParallelForDeviceFunction(
4062 CodeGenModule &CGM, StringRef ParentName,
4063 const OMPTargetParallelForDirective &S);
4064 /// Emit device code for the target parallel for simd directive.
4065 static void EmitOMPTargetParallelForSimdDeviceFunction(
4066 CodeGenModule &CGM, StringRef ParentName,
4067 const OMPTargetParallelForSimdDirective &S);
4068 /// Emit device code for the target teams directive.
4069 static void
4070 EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
4071 const OMPTargetTeamsDirective &S);
4072 /// Emit device code for the target teams distribute directive.
4073 static void EmitOMPTargetTeamsDistributeDeviceFunction(
4074 CodeGenModule &CGM, StringRef ParentName,
4075 const OMPTargetTeamsDistributeDirective &S);
4076 /// Emit device code for the target teams distribute simd directive.
4077 static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(
4078 CodeGenModule &CGM, StringRef ParentName,
4079 const OMPTargetTeamsDistributeSimdDirective &S);
4080 /// Emit device code for the target simd directive.
4081 static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM,
4082 StringRef ParentName,
4083 const OMPTargetSimdDirective &S);
4084 /// Emit device code for the target teams distribute parallel for simd
4085 /// directive.
4086 static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
4087 CodeGenModule &CGM, StringRef ParentName,
4088 const OMPTargetTeamsDistributeParallelForSimdDirective &S);
4089
4090 /// Emit device code for the target teams loop directive.
4091 static void EmitOMPTargetTeamsGenericLoopDeviceFunction(
4092 CodeGenModule &CGM, StringRef ParentName,
4093 const OMPTargetTeamsGenericLoopDirective &S);
4094
4095 /// Emit device code for the target parallel loop directive.
4096 static void EmitOMPTargetParallelGenericLoopDeviceFunction(
4097 CodeGenModule &CGM, StringRef ParentName,
4098 const OMPTargetParallelGenericLoopDirective &S);
4099
4100 static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
4101 CodeGenModule &CGM, StringRef ParentName,
4102 const OMPTargetTeamsDistributeParallelForDirective &S);
4103
4104 /// Emit the Stmt \p S and return its topmost canonical loop, if any.
4105 /// TODO: The \p Depth paramter is not yet implemented and must be 1. In the
4106 /// future it is meant to be the number of loops expected in the loop nests
4107 /// (usually specified by the "collapse" clause) that are collapsed to a
4108 /// single loop by this function.
4109 llvm::CanonicalLoopInfo *EmitOMPCollapsedCanonicalLoopNest(const Stmt *S,
4110 int Depth);
4111
4112 /// Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
4113 void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S);
4114
4115 /// Emit inner loop of the worksharing/simd construct.
4116 ///
4117 /// \param S Directive, for which the inner loop must be emitted.
4118 /// \param RequiresCleanup true, if directive has some associated private
4119 /// variables.
4120 /// \param LoopCond Bollean condition for loop continuation.
4121 /// \param IncExpr Increment expression for loop control variable.
4122 /// \param BodyGen Generator for the inner body of the inner loop.
4123 /// \param PostIncGen Genrator for post-increment code (required for ordered
4124 /// loop directvies).
4125 void EmitOMPInnerLoop(
4126 const OMPExecutableDirective &S, bool RequiresCleanup,
4127 const Expr *LoopCond, const Expr *IncExpr,
4128 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
4129 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen);
4130
4131 JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind);
4132 /// Emit initial code for loop counters of loop-based directives.
4133 void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S,
4134 OMPPrivateScope &LoopScope);
4135
4136 /// Helper for the OpenMP loop directives.
4137 void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit);
4138
4139 /// Emit code for the worksharing loop-based directive.
4140 /// \return true, if this construct has any lastprivate clause, false -
4141 /// otherwise.
4142 bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB,
4143 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
4144 const CodeGenDispatchBoundsTy &CGDispatchBounds);
4145
4146 /// Emit code for the distribute loop-based directive.
4147 void EmitOMPDistributeLoop(const OMPLoopDirective &S,
4148 const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr);
4149
4150 /// Helpers for the OpenMP loop directives.
4151 void EmitOMPSimdInit(const OMPLoopDirective &D);
4152 void EmitOMPSimdFinal(
4153 const OMPLoopDirective &D,
4154 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
4155
4156 /// Emits the lvalue for the expression with possibly captured variable.
4157 LValue EmitOMPSharedLValue(const Expr *E);
4158
4159private:
4160 /// Helpers for blocks.
4161 llvm::Value *EmitBlockLiteral(const CGBlockInfo &Info);
4162
4163 /// struct with the values to be passed to the OpenMP loop-related functions
4164 struct OMPLoopArguments {
4165 /// loop lower bound
4166 Address LB = Address::invalid();
4167 /// loop upper bound
4168 Address UB = Address::invalid();
4169 /// loop stride
4170 Address ST = Address::invalid();
4171 /// isLastIteration argument for runtime functions
4172 Address IL = Address::invalid();
4173 /// Chunk value generated by sema
4174 llvm::Value *Chunk = nullptr;
4175 /// EnsureUpperBound
4176 Expr *EUB = nullptr;
4177 /// IncrementExpression
4178 Expr *IncExpr = nullptr;
4179 /// Loop initialization
4180 Expr *Init = nullptr;
4181 /// Loop exit condition
4182 Expr *Cond = nullptr;
4183 /// Update of LB after a whole chunk has been executed
4184 Expr *NextLB = nullptr;
4185 /// Update of UB after a whole chunk has been executed
4186 Expr *NextUB = nullptr;
4187 /// Distinguish between the for distribute and sections
4188 OpenMPDirectiveKind DKind = llvm::omp::OMPD_unknown;
4189 OMPLoopArguments() = default;
4190 OMPLoopArguments(Address LB, Address UB, Address ST, Address IL,
4191 llvm::Value *Chunk = nullptr, Expr *EUB = nullptr,
4192 Expr *IncExpr = nullptr, Expr *Init = nullptr,
4193 Expr *Cond = nullptr, Expr *NextLB = nullptr,
4194 Expr *NextUB = nullptr)
4195 : LB(LB), UB(UB), ST(ST), IL(IL), Chunk(Chunk), EUB(EUB),
4196 IncExpr(IncExpr), Init(Init), Cond(Cond), NextLB(NextLB),
4197 NextUB(NextUB) {}
4198 };
4199 void EmitOMPOuterLoop(bool DynamicOrOrdered, bool IsMonotonic,
4200 const OMPLoopDirective &S, OMPPrivateScope &LoopScope,
4201 const OMPLoopArguments &LoopArgs,
4202 const CodeGenLoopTy &CodeGenLoop,
4203 const CodeGenOrderedTy &CodeGenOrdered);
4204 void EmitOMPForOuterLoop(const OpenMPScheduleTy &ScheduleKind,
4205 bool IsMonotonic, const OMPLoopDirective &S,
4206 OMPPrivateScope &LoopScope, bool Ordered,
4207 const OMPLoopArguments &LoopArgs,
4208 const CodeGenDispatchBoundsTy &CGDispatchBounds);
4209 void EmitOMPDistributeOuterLoop(OpenMPDistScheduleClauseKind ScheduleKind,
4210 const OMPLoopDirective &S,
4211 OMPPrivateScope &LoopScope,
4212 const OMPLoopArguments &LoopArgs,
4213 const CodeGenLoopTy &CodeGenLoopContent);
4214 /// Emit code for sections directive.
4215 void EmitSections(const OMPExecutableDirective &S);
4216
4217public:
4218 //===--------------------------------------------------------------------===//
4219 // OpenACC Emission
4220 //===--------------------------------------------------------------------===//
4221 void EmitOpenACCComputeConstruct(const OpenACCComputeConstruct &S) {
4222 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4223 // simply emitting its structured block, but in the future we will implement
4224 // some sort of IR.
4225 if (S.getStructuredBlock())
4226 EmitStmt(S: S.getStructuredBlock());
4227 }
4228
4229 void EmitOpenACCLoopConstruct(const OpenACCLoopConstruct &S) {
4230 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4231 // simply emitting its loop, but in the future we will implement
4232 // some sort of IR.
4233 if (S.getLoop())
4234 EmitStmt(S: S.getLoop());
4235 }
4236
4237 void EmitOpenACCCombinedConstruct(const OpenACCCombinedConstruct &S) {
4238 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4239 // simply emitting its loop, but in the future we will implement
4240 // some sort of IR.
4241 if (S.getLoop())
4242 EmitStmt(S: S.getLoop());
4243 }
4244
4245 void EmitOpenACCDataConstruct(const OpenACCDataConstruct &S) {
4246 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4247 // simply emitting its structured block, but in the future we will implement
4248 // some sort of IR.
4249 if (S.getStructuredBlock())
4250 EmitStmt(S: S.getStructuredBlock());
4251 }
4252
4253 void EmitOpenACCEnterDataConstruct(const OpenACCEnterDataConstruct &S) {
4254 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4255 // but in the future we will implement some sort of IR.
4256 }
4257
4258 void EmitOpenACCExitDataConstruct(const OpenACCExitDataConstruct &S) {
4259 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4260 // but in the future we will implement some sort of IR.
4261 }
4262
4263 void EmitOpenACCHostDataConstruct(const OpenACCHostDataConstruct &S) {
4264 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4265 // simply emitting its structured block, but in the future we will implement
4266 // some sort of IR.
4267 if (S.getStructuredBlock())
4268 EmitStmt(S: S.getStructuredBlock());
4269 }
4270
4271 void EmitOpenACCWaitConstruct(const OpenACCWaitConstruct &S) {
4272 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4273 // but in the future we will implement some sort of IR.
4274 }
4275
4276 void EmitOpenACCInitConstruct(const OpenACCInitConstruct &S) {
4277 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4278 // but in the future we will implement some sort of IR.
4279 }
4280
4281 void EmitOpenACCShutdownConstruct(const OpenACCShutdownConstruct &S) {
4282 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4283 // but in the future we will implement some sort of IR.
4284 }
4285
4286 void EmitOpenACCSetConstruct(const OpenACCSetConstruct &S) {
4287 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4288 // but in the future we will implement some sort of IR.
4289 }
4290
4291 void EmitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &S) {
4292 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4293 // but in the future we will implement some sort of IR.
4294 }
4295
4296 void EmitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &S) {
4297 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4298 // simply emitting its associated stmt, but in the future we will implement
4299 // some sort of IR.
4300 if (S.getAssociatedStmt())
4301 EmitStmt(S: S.getAssociatedStmt());
4302 }
4303 void EmitOpenACCCacheConstruct(const OpenACCCacheConstruct &S) {
4304 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4305 // but in the future we will implement some sort of IR.
4306 }
4307
4308 //===--------------------------------------------------------------------===//
4309 // LValue Expression Emission
4310 //===--------------------------------------------------------------------===//
4311
4312 /// Create a check that a scalar RValue is non-null.
4313 llvm::Value *EmitNonNullRValueCheck(RValue RV, QualType T);
4314
4315 /// GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
4316 RValue GetUndefRValue(QualType Ty);
4317
4318 /// EmitUnsupportedRValue - Emit a dummy r-value using the type of E
4319 /// and issue an ErrorUnsupported style diagnostic (using the
4320 /// provided Name).
4321 RValue EmitUnsupportedRValue(const Expr *E, const char *Name);
4322
4323 /// EmitUnsupportedLValue - Emit a dummy l-value using the type of E and issue
4324 /// an ErrorUnsupported style diagnostic (using the provided Name).
4325 LValue EmitUnsupportedLValue(const Expr *E, const char *Name);
4326
4327 /// EmitLValue - Emit code to compute a designator that specifies the location
4328 /// of the expression.
4329 ///
4330 /// This can return one of two things: a simple address or a bitfield
4331 /// reference. In either case, the LLVM Value* in the LValue structure is
4332 /// guaranteed to be an LLVM pointer type.
4333 ///
4334 /// If this returns a bitfield reference, nothing about the pointee type of
4335 /// the LLVM value is known: For example, it may not be a pointer to an
4336 /// integer.
4337 ///
4338 /// If this returns a normal address, and if the lvalue's C type is fixed
4339 /// size, this method guarantees that the returned pointer type will point to
4340 /// an LLVM type of the same size of the lvalue's type. If the lvalue has a
4341 /// variable length type, this is not possible.
4342 ///
4343 LValue EmitLValue(const Expr *E,
4344 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
4345
4346private:
4347 LValue EmitLValueHelper(const Expr *E, KnownNonNull_t IsKnownNonNull);
4348
4349public:
4350 /// Same as EmitLValue but additionally we generate checking code to
4351 /// guard against undefined behavior. This is only suitable when we know
4352 /// that the address will be used to access the object.
4353 LValue EmitCheckedLValue(const Expr *E, TypeCheckKind TCK);
4354
4355 RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc);
4356
4357 void EmitAtomicInit(Expr *E, LValue lvalue);
4358
4359 bool LValueIsSuitableForInlineAtomic(LValue Src);
4360
4361 RValue EmitAtomicLoad(LValue LV, SourceLocation SL,
4362 AggValueSlot Slot = AggValueSlot::ignored());
4363
4364 RValue EmitAtomicLoad(LValue lvalue, SourceLocation loc,
4365 llvm::AtomicOrdering AO, bool IsVolatile = false,
4366 AggValueSlot slot = AggValueSlot::ignored());
4367
4368 void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit);
4369
4370 void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO,
4371 bool IsVolatile, bool isInit);
4372
4373 std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
4374 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
4375 llvm::AtomicOrdering Success =
4376 llvm::AtomicOrdering::SequentiallyConsistent,
4377 llvm::AtomicOrdering Failure =
4378 llvm::AtomicOrdering::SequentiallyConsistent,
4379 bool IsWeak = false, AggValueSlot Slot = AggValueSlot::ignored());
4380
4381 /// Emit an atomicrmw instruction, and applying relevant metadata when
4382 /// applicable.
4383 llvm::AtomicRMWInst *emitAtomicRMWInst(
4384 llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val,
4385 llvm::AtomicOrdering Order = llvm::AtomicOrdering::SequentiallyConsistent,
4386 llvm::SyncScope::ID SSID = llvm::SyncScope::System,
4387 const AtomicExpr *AE = nullptr);
4388
4389 /// Emit a fence instruction, applying relevant target-specific metadata when
4390 /// applicable.
4391 llvm::FenceInst *
4392 emitAtomicFence(llvm::AtomicOrdering Order,
4393 llvm::SyncScope::ID SSID = llvm::SyncScope::System);
4394
4395 void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO,
4396 const llvm::function_ref<RValue(RValue)> &UpdateOp,
4397 bool IsVolatile);
4398
4399 /// EmitToMemory - Change a scalar value from its value
4400 /// representation to its in-memory representation.
4401 llvm::Value *EmitToMemory(llvm::Value *Value, QualType Ty);
4402
4403 /// EmitFromMemory - Change a scalar value from its memory
4404 /// representation to its value representation.
4405 llvm::Value *EmitFromMemory(llvm::Value *Value, QualType Ty);
4406
4407 /// Check if the scalar \p Value is within the valid range for the given
4408 /// type \p Ty.
4409 ///
4410 /// Returns true if a check is needed (even if the range is unknown).
4411 bool EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
4412 SourceLocation Loc);
4413
4414 /// EmitLoadOfScalar - Load a scalar value from an address, taking
4415 /// care to appropriately convert from the memory representation to
4416 /// the LLVM value representation.
4417 llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4418 SourceLocation Loc,
4419 AlignmentSource Source = AlignmentSource::Type,
4420 bool isNontemporal = false) {
4421 return EmitLoadOfScalar(Addr, Volatile, Ty, Loc, BaseInfo: LValueBaseInfo(Source),
4422 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isNontemporal);
4423 }
4424
4425 llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4426 SourceLocation Loc, LValueBaseInfo BaseInfo,
4427 TBAAAccessInfo TBAAInfo,
4428 bool isNontemporal = false);
4429
4430 /// EmitLoadOfScalar - Load a scalar value from an address, taking
4431 /// care to appropriately convert from the memory representation to
4432 /// the LLVM value representation. The l-value must be a simple
4433 /// l-value.
4434 llvm::Value *EmitLoadOfScalar(LValue lvalue, SourceLocation Loc);
4435
4436 /// EmitStoreOfScalar - Store a scalar value to an address, taking
4437 /// care to appropriately convert from the memory representation to
4438 /// the LLVM value representation.
4439 void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,
4440 QualType Ty,
4441 AlignmentSource Source = AlignmentSource::Type,
4442 bool isInit = false, bool isNontemporal = false) {
4443 EmitStoreOfScalar(Value, Addr, Volatile, Ty, BaseInfo: LValueBaseInfo(Source),
4444 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isInit, isNontemporal);
4445 }
4446
4447 void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,
4448 QualType Ty, LValueBaseInfo BaseInfo,
4449 TBAAAccessInfo TBAAInfo, bool isInit = false,
4450 bool isNontemporal = false);
4451
4452 /// EmitStoreOfScalar - Store a scalar value to an address, taking
4453 /// care to appropriately convert from the memory representation to
4454 /// the LLVM value representation. The l-value must be a simple
4455 /// l-value. The isInit flag indicates whether this is an initialization.
4456 /// If so, atomic qualifiers are ignored and the store is always non-atomic.
4457 void EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
4458 bool isInit = false);
4459
4460 /// EmitLoadOfLValue - Given an expression that represents a value lvalue,
4461 /// this method emits the address of the lvalue, then loads the result as an
4462 /// rvalue, returning the rvalue.
4463 RValue EmitLoadOfLValue(LValue V, SourceLocation Loc);
4464 RValue EmitLoadOfExtVectorElementLValue(LValue V);
4465 RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc);
4466 RValue EmitLoadOfGlobalRegLValue(LValue LV);
4467
4468 /// Like EmitLoadOfLValue but also handles complex and aggregate types.
4469 RValue EmitLoadOfAnyValue(LValue V,
4470 AggValueSlot Slot = AggValueSlot::ignored(),
4471 SourceLocation Loc = {});
4472
4473 /// EmitStoreThroughLValue - Store the specified rvalue into the specified
4474 /// lvalue, where both are guaranteed to the have the same type, and that type
4475 /// is 'Ty'.
4476 void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit = false);
4477 void EmitStoreThroughExtVectorComponentLValue(RValue Src, LValue Dst);
4478 void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst);
4479
4480 /// EmitStoreThroughBitfieldLValue - Store Src into Dst with same constraints
4481 /// as EmitStoreThroughLValue.
4482 ///
4483 /// \param Result [out] - If non-null, this will be set to a Value* for the
4484 /// bit-field contents after the store, appropriate for use as the result of
4485 /// an assignment to the bit-field.
4486 void EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
4487 llvm::Value **Result = nullptr);
4488
4489 /// Emit an l-value for an assignment (simple or compound) of complex type.
4490 LValue EmitComplexAssignmentLValue(const BinaryOperator *E);
4491 LValue EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E);
4492 LValue EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E,
4493 llvm::Value *&Result);
4494
4495 // Note: only available for agg return types
4496 LValue EmitBinaryOperatorLValue(const BinaryOperator *E);
4497 LValue EmitCompoundAssignmentLValue(const CompoundAssignOperator *E);
4498 // Note: only available for agg return types
4499 LValue EmitCallExprLValue(const CallExpr *E,
4500 llvm::CallBase **CallOrInvoke = nullptr);
4501 // Note: only available for agg return types
4502 LValue EmitVAArgExprLValue(const VAArgExpr *E);
4503 LValue EmitDeclRefLValue(const DeclRefExpr *E);
4504 LValue EmitStringLiteralLValue(const StringLiteral *E);
4505 LValue EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E);
4506 LValue EmitPredefinedLValue(const PredefinedExpr *E);
4507 LValue EmitUnaryOpLValue(const UnaryOperator *E);
4508 LValue EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
4509 bool Accessed = false);
4510 llvm::Value *EmitMatrixIndexExpr(const Expr *E);
4511 LValue EmitMatrixSingleSubscriptExpr(const MatrixSingleSubscriptExpr *E);
4512 LValue EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E);
4513 LValue EmitArraySectionExpr(const ArraySectionExpr *E,
4514 bool IsLowerBound = true);
4515 LValue EmitExtVectorElementExpr(const ExtVectorElementExpr *E);
4516 LValue EmitMatrixElementExpr(const MatrixElementExpr *E);
4517 LValue EmitMemberExpr(const MemberExpr *E);
4518 LValue EmitObjCIsaExpr(const ObjCIsaExpr *E);
4519 LValue EmitCompoundLiteralLValue(const CompoundLiteralExpr *E);
4520 LValue EmitInitListLValue(const InitListExpr *E);
4521 void EmitIgnoredConditionalOperator(const AbstractConditionalOperator *E);
4522 LValue EmitConditionalOperatorLValue(const AbstractConditionalOperator *E);
4523 LValue EmitCastLValue(const CastExpr *E);
4524 LValue EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);
4525 LValue EmitOpaqueValueLValue(const OpaqueValueExpr *e);
4526 LValue EmitHLSLArrayAssignLValue(const BinaryOperator *E);
4527
4528 std::pair<LValue, LValue> EmitHLSLOutArgLValues(const HLSLOutArgExpr *E,
4529 QualType Ty);
4530 LValue EmitHLSLOutArgExpr(const HLSLOutArgExpr *E, CallArgList &Args,
4531 QualType Ty);
4532
4533 Address EmitExtVectorElementLValue(LValue V);
4534
4535 RValue EmitRValueForField(LValue LV, const FieldDecl *FD, SourceLocation Loc);
4536
4537 Address EmitArrayToPointerDecay(const Expr *Array,
4538 LValueBaseInfo *BaseInfo = nullptr,
4539 TBAAAccessInfo *TBAAInfo = nullptr);
4540
4541 class ConstantEmission {
4542 llvm::PointerIntPair<llvm::Constant *, 1, bool> ValueAndIsReference;
4543 ConstantEmission(llvm::Constant *C, bool isReference)
4544 : ValueAndIsReference(C, isReference) {}
4545
4546 public:
4547 ConstantEmission() {}
4548 static ConstantEmission forReference(llvm::Constant *C) {
4549 return ConstantEmission(C, true);
4550 }
4551 static ConstantEmission forValue(llvm::Constant *C) {
4552 return ConstantEmission(C, false);
4553 }
4554
4555 explicit operator bool() const {
4556 return ValueAndIsReference.getOpaqueValue() != nullptr;
4557 }
4558
4559 bool isReference() const { return ValueAndIsReference.getInt(); }
4560 LValue getReferenceLValue(CodeGenFunction &CGF, const Expr *RefExpr) const {
4561 assert(isReference());
4562 return CGF.MakeNaturalAlignAddrLValue(V: ValueAndIsReference.getPointer(),
4563 T: RefExpr->getType());
4564 }
4565
4566 llvm::Constant *getValue() const {
4567 assert(!isReference());
4568 return ValueAndIsReference.getPointer();
4569 }
4570 };
4571
4572 ConstantEmission tryEmitAsConstant(const DeclRefExpr *RefExpr);
4573 ConstantEmission tryEmitAsConstant(const MemberExpr *ME);
4574 llvm::Value *emitScalarConstant(const ConstantEmission &Constant, Expr *E);
4575
4576 RValue EmitPseudoObjectRValue(const PseudoObjectExpr *e,
4577 AggValueSlot slot = AggValueSlot::ignored());
4578 LValue EmitPseudoObjectLValue(const PseudoObjectExpr *e);
4579
4580 void FlattenAccessAndTypeLValue(LValue LVal,
4581 SmallVectorImpl<LValue> &AccessList);
4582
4583 llvm::Value *EmitIvarOffset(const ObjCInterfaceDecl *Interface,
4584 const ObjCIvarDecl *Ivar);
4585 llvm::Value *EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
4586 const ObjCIvarDecl *Ivar);
4587 LValue EmitLValueForField(LValue Base, const FieldDecl *Field,
4588 bool IsInBounds = true);
4589 LValue EmitLValueForLambdaField(const FieldDecl *Field);
4590 LValue EmitLValueForLambdaField(const FieldDecl *Field,
4591 llvm::Value *ThisValue);
4592
4593 /// EmitLValueForFieldInitialization - Like EmitLValueForField, except that
4594 /// if the Field is a reference, this will return the address of the reference
4595 /// and not the address of the value stored in the reference.
4596 LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field);
4597
4598 LValue EmitLValueForIvar(QualType ObjectTy, llvm::Value *Base,
4599 const ObjCIvarDecl *Ivar, unsigned CVRQualifiers);
4600
4601 LValue EmitCXXConstructLValue(const CXXConstructExpr *E);
4602 LValue EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E);
4603 LValue EmitCXXTypeidLValue(const CXXTypeidExpr *E);
4604 LValue EmitCXXUuidofLValue(const CXXUuidofExpr *E);
4605
4606 LValue EmitObjCMessageExprLValue(const ObjCMessageExpr *E);
4607 LValue EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E);
4608 LValue EmitStmtExprLValue(const StmtExpr *E);
4609 LValue EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E);
4610 LValue EmitObjCSelectorLValue(const ObjCSelectorExpr *E);
4611 void EmitDeclRefExprDbgValue(const DeclRefExpr *E, const APValue &Init);
4612
4613 //===--------------------------------------------------------------------===//
4614 // Scalar Expression Emission
4615 //===--------------------------------------------------------------------===//
4616
4617 /// EmitCall - Generate a call of the given function, expecting the given
4618 /// result type, and using the given argument list which specifies both the
4619 /// LLVM arguments and the types they were derived from.
4620 RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4621 ReturnValueSlot ReturnValue, const CallArgList &Args,
4622 llvm::CallBase **CallOrInvoke, bool IsMustTail,
4623 SourceLocation Loc,
4624 bool IsVirtualFunctionPointerThunk = false);
4625 RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4626 ReturnValueSlot ReturnValue, const CallArgList &Args,
4627 llvm::CallBase **CallOrInvoke = nullptr,
4628 bool IsMustTail = false) {
4629 return EmitCall(CallInfo, Callee, ReturnValue, Args, CallOrInvoke,
4630 IsMustTail, Loc: SourceLocation());
4631 }
4632 RValue EmitCall(QualType FnType, const CGCallee &Callee, const CallExpr *E,
4633 ReturnValueSlot ReturnValue, llvm::Value *Chain = nullptr,
4634 llvm::CallBase **CallOrInvoke = nullptr,
4635 CGFunctionInfo const **ResolvedFnInfo = nullptr);
4636
4637 // If a Call or Invoke instruction was emitted for this CallExpr, this method
4638 // writes the pointer to `CallOrInvoke` if it's not null.
4639 RValue EmitCallExpr(const CallExpr *E,
4640 ReturnValueSlot ReturnValue = ReturnValueSlot(),
4641 llvm::CallBase **CallOrInvoke = nullptr);
4642 RValue EmitSimpleCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,
4643 llvm::CallBase **CallOrInvoke = nullptr);
4644 CGCallee EmitCallee(const Expr *E);
4645
4646 void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl);
4647 void checkTargetFeatures(SourceLocation Loc, const FunctionDecl *TargetDecl);
4648
4649 llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4650 const Twine &name = "");
4651 llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4652 ArrayRef<llvm::Value *> args,
4653 const Twine &name = "");
4654 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4655 const Twine &Name = "");
4656 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4657 ArrayRef<llvm::Value *> Args,
4658 const Twine &Name = "");
4659 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4660 ArrayRef<llvm::Type *> Types,
4661 ArrayRef<llvm::Value *> Args,
4662 const Twine &Name = "");
4663 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4664 ArrayRef<llvm::Value *> Args,
4665 llvm::Type *RetTy, const Twine &Name = "");
4666 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4667 const Twine &name = "");
4668 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4669 ArrayRef<Address> args,
4670 const Twine &name = "");
4671 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4672 ArrayRef<llvm::Value *> args,
4673 const Twine &name = "");
4674
4675 SmallVector<llvm::OperandBundleDef, 1>
4676 getBundlesForFunclet(llvm::Value *Callee);
4677
4678 llvm::CallBase *EmitCallOrInvoke(llvm::FunctionCallee Callee,
4679 ArrayRef<llvm::Value *> Args,
4680 const Twine &Name = "");
4681 llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4682 ArrayRef<llvm::Value *> args,
4683 const Twine &name = "");
4684 llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4685 const Twine &name = "");
4686 void EmitNoreturnRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4687 ArrayRef<llvm::Value *> args);
4688
4689 CGCallee BuildAppleKextVirtualCall(const CXXMethodDecl *MD,
4690 NestedNameSpecifier Qual, llvm::Type *Ty);
4691
4692 CGCallee BuildAppleKextVirtualDestructorCall(const CXXDestructorDecl *DD,
4693 CXXDtorType Type,
4694 const CXXRecordDecl *RD);
4695
4696 bool isPointerKnownNonNull(const Expr *E);
4697 /// Check whether the underlying base pointer is a constant null.
4698 bool isUnderlyingBasePointerConstantNull(const Expr *E);
4699
4700 /// Create the discriminator from the storage address and the entity hash.
4701 llvm::Value *EmitPointerAuthBlendDiscriminator(llvm::Value *StorageAddress,
4702 llvm::Value *Discriminator);
4703 CGPointerAuthInfo EmitPointerAuthInfo(const PointerAuthSchema &Schema,
4704 llvm::Value *StorageAddress,
4705 llvm::ConstantInt *Discriminator);
4706 CGPointerAuthInfo EmitPointerAuthInfo(const PointerAuthSchema &Schema,
4707 llvm::Value *StorageAddress,
4708 GlobalDecl SchemaDecl,
4709 QualType SchemaType);
4710
4711 llvm::Value *EmitPointerAuthSign(const CGPointerAuthInfo &Info,
4712 llvm::Value *Pointer);
4713
4714 llvm::Value *EmitPointerAuthAuth(const CGPointerAuthInfo &Info,
4715 llvm::Value *Pointer);
4716
4717 llvm::Value *emitPointerAuthResign(llvm::Value *Pointer, QualType PointerType,
4718 const CGPointerAuthInfo &CurAuthInfo,
4719 const CGPointerAuthInfo &NewAuthInfo,
4720 bool IsKnownNonNull);
4721 llvm::Value *emitPointerAuthResignCall(llvm::Value *Pointer,
4722 const CGPointerAuthInfo &CurInfo,
4723 const CGPointerAuthInfo &NewInfo);
4724
4725 void EmitPointerAuthOperandBundle(
4726 const CGPointerAuthInfo &Info,
4727 SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
4728
4729 CGPointerAuthInfo EmitPointerAuthInfo(PointerAuthQualifier Qualifier,
4730 Address StorageAddress);
4731 llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
4732 llvm::Value *Pointer, QualType ValueType,
4733 Address StorageAddress,
4734 bool IsKnownNonNull);
4735 llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
4736 const Expr *PointerExpr,
4737 Address StorageAddress);
4738 llvm::Value *EmitPointerAuthUnqualify(PointerAuthQualifier Qualifier,
4739 llvm::Value *Pointer,
4740 QualType PointerType,
4741 Address StorageAddress,
4742 bool IsKnownNonNull);
4743 void EmitPointerAuthCopy(PointerAuthQualifier Qualifier, QualType Type,
4744 Address DestField, Address SrcField);
4745
4746 std::pair<llvm::Value *, CGPointerAuthInfo>
4747 EmitOrigPointerRValue(const Expr *E);
4748
4749 llvm::Value *authPointerToPointerCast(llvm::Value *ResultPtr,
4750 QualType SourceType, QualType DestType);
4751 Address authPointerToPointerCast(Address Ptr, QualType SourceType,
4752 QualType DestType);
4753
4754 Address getAsNaturalAddressOf(Address Addr, QualType PointeeTy);
4755
4756 llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) {
4757 return getAsNaturalAddressOf(Addr, PointeeTy: PointeeType).getBasePointer();
4758 }
4759
4760 // Return the copy constructor name with the prefix "__copy_constructor_"
4761 // removed.
4762 static std::string getNonTrivialCopyConstructorStr(QualType QT,
4763 CharUnits Alignment,
4764 bool IsVolatile,
4765 ASTContext &Ctx);
4766
4767 // Return the destructor name with the prefix "__destructor_" removed.
4768 static std::string getNonTrivialDestructorStr(QualType QT,
4769 CharUnits Alignment,
4770 bool IsVolatile,
4771 ASTContext &Ctx);
4772
4773 // These functions emit calls to the special functions of non-trivial C
4774 // structs.
4775 void defaultInitNonTrivialCStructVar(LValue Dst);
4776 void callCStructDefaultConstructor(LValue Dst);
4777 void callCStructDestructor(LValue Dst);
4778 void callCStructCopyConstructor(LValue Dst, LValue Src);
4779 void callCStructMoveConstructor(LValue Dst, LValue Src);
4780 void callCStructCopyAssignmentOperator(LValue Dst, LValue Src);
4781 void callCStructMoveAssignmentOperator(LValue Dst, LValue Src);
4782
4783 RValue EmitCXXMemberOrOperatorCall(
4784 const CXXMethodDecl *Method, const CGCallee &Callee,
4785 ReturnValueSlot ReturnValue, llvm::Value *This,
4786 llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *E,
4787 CallArgList *RtlArgs, llvm::CallBase **CallOrInvoke);
4788 RValue EmitCXXDestructorCall(GlobalDecl Dtor, const CGCallee &Callee,
4789 llvm::Value *This, QualType ThisTy,
4790 llvm::Value *ImplicitParam,
4791 QualType ImplicitParamTy, const CallExpr *E,
4792 llvm::CallBase **CallOrInvoke = nullptr);
4793 RValue EmitCXXMemberCallExpr(const CXXMemberCallExpr *E,
4794 ReturnValueSlot ReturnValue,
4795 llvm::CallBase **CallOrInvoke = nullptr);
4796 RValue EmitCXXMemberOrOperatorMemberCallExpr(
4797 const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue,
4798 bool HasQualifier, NestedNameSpecifier Qualifier, bool IsArrow,
4799 const Expr *Base, llvm::CallBase **CallOrInvoke);
4800 // Compute the object pointer.
4801 Address EmitCXXMemberDataPointerAddress(
4802 const Expr *E, Address base, llvm::Value *memberPtr,
4803 const MemberPointerType *memberPtrType, bool IsInBounds,
4804 LValueBaseInfo *BaseInfo = nullptr, TBAAAccessInfo *TBAAInfo = nullptr);
4805 RValue EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
4806 ReturnValueSlot ReturnValue,
4807 llvm::CallBase **CallOrInvoke);
4808
4809 RValue EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
4810 const CXXMethodDecl *MD,
4811 ReturnValueSlot ReturnValue,
4812 llvm::CallBase **CallOrInvoke);
4813 RValue EmitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *E);
4814
4815 RValue EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
4816 ReturnValueSlot ReturnValue,
4817 llvm::CallBase **CallOrInvoke);
4818
4819 RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E);
4820 RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E);
4821
4822 RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
4823 const CallExpr *E, ReturnValueSlot ReturnValue);
4824
4825 RValue emitRotate(const CallExpr *E, bool IsRotateRight);
4826
4827 RValue emitStdcCountIntrinsic(const CallExpr *E, llvm::Intrinsic::ID IntID,
4828 bool InvertArg, bool IsPop = false);
4829 RValue emitStdcBitWidthMinus(const CallExpr *E, llvm::Intrinsic::ID IntID,
4830 bool IsPop);
4831 RValue emitStdcFirstBit(const CallExpr *E, llvm::Intrinsic::ID IntID,
4832 bool InvertArg);
4833
4834 /// Emit IR for __builtin_os_log_format.
4835 RValue emitBuiltinOSLogFormat(const CallExpr &E);
4836
4837 /// Emit IR for __builtin_is_aligned.
4838 RValue EmitBuiltinIsAligned(const CallExpr *E);
4839 /// Emit IR for __builtin_align_up/__builtin_align_down.
4840 RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp);
4841
4842 llvm::Function *generateBuiltinOSLogHelperFunction(
4843 const analyze_os_log::OSLogBufferLayout &Layout,
4844 CharUnits BufferAlignment);
4845
4846 RValue EmitBlockCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,
4847 llvm::CallBase **CallOrInvoke);
4848
4849 /// EmitTargetBuiltinExpr - Emit the given builtin call. Returns 0 if the call
4850 /// is unhandled by the current target.
4851 llvm::Value *EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4852 ReturnValueSlot ReturnValue);
4853
4854 llvm::Value *
4855 EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty,
4856 const llvm::CmpInst::Predicate Pred,
4857 const llvm::Twine &Name = "");
4858 llvm::Value *EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4859 ReturnValueSlot ReturnValue,
4860 llvm::Triple::ArchType Arch);
4861 llvm::Value *EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4862 ReturnValueSlot ReturnValue,
4863 llvm::Triple::ArchType Arch);
4864 llvm::Value *EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4865 ReturnValueSlot ReturnValue,
4866 llvm::Triple::ArchType Arch);
4867 llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::IntegerType *ITy,
4868 QualType RTy);
4869 llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::ArrayType *ATy,
4870 QualType RTy);
4871
4872 llvm::Value *
4873 EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic,
4874 unsigned AltLLVMIntrinsic, const char *NameHint,
4875 unsigned Modifier, const CallExpr *E,
4876 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0,
4877 Address PtrOp1, llvm::Triple::ArchType Arch);
4878
4879 llvm::Function *LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
4880 unsigned Modifier, llvm::Type *ArgTy,
4881 const CallExpr *E);
4882 llvm::Value *EmitNeonCall(llvm::Function *F,
4883 SmallVectorImpl<llvm::Value *> &O, const char *name,
4884 unsigned shift = 0, bool rightshift = false);
4885 llvm::Value *EmitFP8NeonCall(unsigned IID, ArrayRef<llvm::Type *> Tys,
4886 SmallVectorImpl<llvm::Value *> &O,
4887 const CallExpr *E, const char *name);
4888 llvm::Value *EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,
4889 llvm::Type *Ty1, bool Extract,
4890 SmallVectorImpl<llvm::Value *> &Ops,
4891 const CallExpr *E, const char *name);
4892 llvm::Value *EmitFP8NeonFDOTCall(unsigned IID, bool ExtendLaneArg,
4893 llvm::Type *RetTy,
4894 SmallVectorImpl<llvm::Value *> &Ops,
4895 const CallExpr *E, const char *name);
4896 llvm::Value *EmitFP8NeonFMLACall(unsigned IID, bool ExtendLaneArg,
4897 llvm::Type *RetTy,
4898 SmallVectorImpl<llvm::Value *> &Ops,
4899 const CallExpr *E, const char *name);
4900 llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx,
4901 const llvm::ElementCount &Count);
4902 llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx);
4903 llvm::Value *EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty,
4904 bool negateForRightShift);
4905 llvm::Value *EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt,
4906 llvm::Type *Ty, bool usgn, const char *name);
4907 llvm::Value *vectorWrapScalar16(llvm::Value *Op);
4908 /// SVEBuiltinMemEltTy - Returns the memory element type for this memory
4909 /// access builtin. Only required if it can't be inferred from the base
4910 /// pointer operand.
4911 llvm::Type *SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags);
4912
4913 SmallVector<llvm::Type *, 2>
4914 getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType,
4915 ArrayRef<llvm::Value *> Ops);
4916 llvm::Type *getEltType(const SVETypeFlags &TypeFlags);
4917 llvm::ScalableVectorType *getSVEType(const SVETypeFlags &TypeFlags);
4918 llvm::ScalableVectorType *getSVEPredType(const SVETypeFlags &TypeFlags);
4919 llvm::Value *EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags,
4920 ArrayRef<llvm::Value *> Ops);
4921 llvm::Value *EmitSVETupleCreate(const SVETypeFlags &TypeFlags,
4922 llvm::Type *ReturnType,
4923 ArrayRef<llvm::Value *> Ops);
4924 llvm::Value *EmitSVEDupX(llvm::Value *Scalar);
4925 llvm::Value *EmitSVEDupX(llvm::Value *Scalar, llvm::Type *Ty);
4926 llvm::Value *EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty);
4927 llvm::Value *EmitSVEPMull(const SVETypeFlags &TypeFlags,
4928 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4929 unsigned BuiltinID);
4930 llvm::Value *EmitSVEMovl(const SVETypeFlags &TypeFlags,
4931 llvm::ArrayRef<llvm::Value *> Ops,
4932 unsigned BuiltinID);
4933 llvm::Value *EmitSVEPredicateCast(llvm::Value *Pred,
4934 llvm::ScalableVectorType *VTy);
4935 llvm::Value *EmitSVEPredicateTupleCast(llvm::Value *PredTuple,
4936 llvm::StructType *Ty);
4937 llvm::Value *EmitSVEGatherLoad(const SVETypeFlags &TypeFlags,
4938 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4939 unsigned IntID);
4940 llvm::Value *EmitSVEScatterStore(const SVETypeFlags &TypeFlags,
4941 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4942 unsigned IntID);
4943 llvm::Value *EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy,
4944 SmallVectorImpl<llvm::Value *> &Ops,
4945 unsigned BuiltinID, bool IsZExtReturn);
4946 llvm::Value *EmitSVEMaskedStore(const CallExpr *,
4947 SmallVectorImpl<llvm::Value *> &Ops,
4948 unsigned BuiltinID);
4949 llvm::Value *EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,
4950 SmallVectorImpl<llvm::Value *> &Ops,
4951 unsigned BuiltinID);
4952 llvm::Value *EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags,
4953 SmallVectorImpl<llvm::Value *> &Ops,
4954 unsigned IntID);
4955 llvm::Value *EmitSVEStructLoad(const SVETypeFlags &TypeFlags,
4956 SmallVectorImpl<llvm::Value *> &Ops,
4957 unsigned IntID);
4958 llvm::Value *EmitSVEStructStore(const SVETypeFlags &TypeFlags,
4959 SmallVectorImpl<llvm::Value *> &Ops,
4960 unsigned IntID);
4961 llvm::Value *EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4962
4963 llvm::Value *EmitSMELd1St1(const SVETypeFlags &TypeFlags,
4964 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4965 unsigned IntID);
4966 llvm::Value *EmitSMEReadWrite(const SVETypeFlags &TypeFlags,
4967 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4968 unsigned IntID);
4969 llvm::Value *EmitSMEZero(const SVETypeFlags &TypeFlags,
4970 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4971 unsigned IntID);
4972 llvm::Value *EmitSMELdrStr(const SVETypeFlags &TypeFlags,
4973 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4974 unsigned IntID);
4975
4976 void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E,
4977 SmallVectorImpl<llvm::Value *> &Ops,
4978 SVETypeFlags TypeFlags);
4979
4980 llvm::Value *EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4981
4982 llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4983 llvm::Triple::ArchType Arch);
4984 llvm::Value *EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4985
4986 llvm::Value *BuildVector(ArrayRef<llvm::Value *> Ops);
4987 llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4988 llvm::Value *EmitPPCBuiltinCpu(unsigned BuiltinID, llvm::Type *ReturnType,
4989 StringRef CPUStr);
4990 llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4991 llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4992 llvm::Value *EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4993 ReturnValueSlot ReturnValue);
4994
4995 // Returns a builtin function that the SPIR-V backend will expand into a spec
4996 // constant.
4997 llvm::Function *
4998 getSpecConstantFunction(const clang::QualType &SpecConstantType);
4999
5000 llvm::Value *EmitDirectXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5001 llvm::Value *EmitSPIRVBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5002 llvm::Value *EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx,
5003 const CallExpr *E);
5004 llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5005 llvm::Value *EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5006 llvm::Value *EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
5007 const CallExpr *E);
5008 llvm::Value *EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5009 llvm::Value *EmitAVRBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5010 llvm::Value *EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
5011 ReturnValueSlot ReturnValue);
5012
5013 llvm::Value *EmitRISCVCpuSupports(const CallExpr *E);
5014 llvm::Value *EmitRISCVCpuSupports(ArrayRef<StringRef> FeaturesStrs);
5015 llvm::Value *EmitRISCVCpuInit();
5016 llvm::Value *EmitRISCVCpuIs(const CallExpr *E);
5017 llvm::Value *EmitRISCVCpuIs(StringRef CPUStr);
5018
5019 void AddAMDGPUFenceAddressSpaceMMRA(llvm::Instruction *Inst,
5020 const CallExpr *E);
5021 /// Attach the AMDGPU availability/visibility MMRA to \p Inst when the
5022 /// amdgpu_av attribute is active on the current statement.
5023 void AddAMDGPUAvailableVisibleMMRA(llvm::Instruction *Inst);
5024 void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope,
5025 llvm::AtomicOrdering &AO,
5026 llvm::SyncScope::ID &SSID);
5027
5028 enum class MSVCIntrin;
5029 llvm::Value *EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E);
5030
5031 llvm::Value *EmitBuiltinAvailable(const VersionTuple &Version);
5032
5033 llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E);
5034 llvm::Value *EmitObjCStringLiteral(const ObjCStringLiteral *E);
5035 llvm::Value *EmitObjCBoxedExpr(const ObjCBoxedExpr *E);
5036 llvm::Value *EmitObjCArrayLiteral(const ObjCArrayLiteral *E);
5037 llvm::Value *EmitObjCDictionaryLiteral(const ObjCDictionaryLiteral *E);
5038 llvm::Value *
5039 EmitObjCCollectionLiteral(const Expr *E,
5040 const ObjCMethodDecl *MethodWithObjects);
5041 llvm::Value *EmitObjCSelectorExpr(const ObjCSelectorExpr *E);
5042 RValue EmitObjCMessageExpr(const ObjCMessageExpr *E,
5043 ReturnValueSlot Return = ReturnValueSlot());
5044
5045 /// Retrieves the default cleanup kind for an ARC cleanup.
5046 /// Except under -fobjc-arc-eh, ARC cleanups are normal-only.
5047 CleanupKind getARCCleanupKind() {
5048 return CGM.getCodeGenOpts().ObjCAutoRefCountExceptions ? NormalAndEHCleanup
5049 : NormalCleanup;
5050 }
5051
5052 // ARC primitives.
5053 void EmitARCInitWeak(Address addr, llvm::Value *value);
5054 void EmitARCDestroyWeak(Address addr);
5055 llvm::Value *EmitARCLoadWeak(Address addr);
5056 llvm::Value *EmitARCLoadWeakRetained(Address addr);
5057 llvm::Value *EmitARCStoreWeak(Address addr, llvm::Value *value, bool ignored);
5058 void emitARCCopyAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
5059 void emitARCMoveAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
5060 void EmitARCCopyWeak(Address dst, Address src);
5061 void EmitARCMoveWeak(Address dst, Address src);
5062 llvm::Value *EmitARCRetainAutorelease(QualType type, llvm::Value *value);
5063 llvm::Value *EmitARCRetainAutoreleaseNonBlock(llvm::Value *value);
5064 llvm::Value *EmitARCStoreStrong(LValue lvalue, llvm::Value *value,
5065 bool resultIgnored);
5066 llvm::Value *EmitARCStoreStrongCall(Address addr, llvm::Value *value,
5067 bool resultIgnored);
5068 llvm::Value *EmitARCRetain(QualType type, llvm::Value *value);
5069 llvm::Value *EmitARCRetainNonBlock(llvm::Value *value);
5070 llvm::Value *EmitARCRetainBlock(llvm::Value *value, bool mandatory);
5071 void EmitARCDestroyStrong(Address addr, ARCPreciseLifetime_t precise);
5072 void EmitARCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
5073 llvm::Value *EmitARCAutorelease(llvm::Value *value);
5074 llvm::Value *EmitARCAutoreleaseReturnValue(llvm::Value *value);
5075 llvm::Value *EmitARCRetainAutoreleaseReturnValue(llvm::Value *value);
5076 llvm::Value *EmitARCRetainAutoreleasedReturnValue(llvm::Value *value);
5077 llvm::Value *EmitARCUnsafeClaimAutoreleasedReturnValue(llvm::Value *value);
5078
5079 llvm::Value *EmitObjCAutorelease(llvm::Value *value, llvm::Type *returnType);
5080 llvm::Value *EmitObjCRetainNonBlock(llvm::Value *value,
5081 llvm::Type *returnType);
5082 void EmitObjCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
5083
5084 std::pair<LValue, llvm::Value *>
5085 EmitARCStoreAutoreleasing(const BinaryOperator *e);
5086 std::pair<LValue, llvm::Value *> EmitARCStoreStrong(const BinaryOperator *e,
5087 bool ignored);
5088 std::pair<LValue, llvm::Value *>
5089 EmitARCStoreUnsafeUnretained(const BinaryOperator *e, bool ignored);
5090
5091 llvm::Value *EmitObjCAlloc(llvm::Value *value, llvm::Type *returnType);
5092 llvm::Value *EmitObjCAllocWithZone(llvm::Value *value,
5093 llvm::Type *returnType);
5094 llvm::Value *EmitObjCAllocInit(llvm::Value *value, llvm::Type *resultType);
5095
5096 llvm::Value *EmitObjCThrowOperand(const Expr *expr);
5097 llvm::Value *EmitObjCConsumeObject(QualType T, llvm::Value *Ptr);
5098 llvm::Value *EmitObjCExtendObjectLifetime(QualType T, llvm::Value *Ptr);
5099
5100 llvm::Value *EmitARCExtendBlockObject(const Expr *expr);
5101 llvm::Value *EmitARCReclaimReturnedObject(const Expr *e,
5102 bool allowUnsafeClaim);
5103 llvm::Value *EmitARCRetainScalarExpr(const Expr *expr);
5104 llvm::Value *EmitARCRetainAutoreleaseScalarExpr(const Expr *expr);
5105 llvm::Value *EmitARCUnsafeUnretainedScalarExpr(const Expr *expr);
5106
5107 void EmitARCIntrinsicUse(ArrayRef<llvm::Value *> values);
5108
5109 void EmitARCNoopIntrinsicUse(ArrayRef<llvm::Value *> values);
5110
5111 static Destroyer destroyARCStrongImprecise;
5112 static Destroyer destroyARCStrongPrecise;
5113 static Destroyer destroyARCWeak;
5114 static Destroyer emitARCIntrinsicUse;
5115 static Destroyer destroyNonTrivialCStruct;
5116
5117 void EmitObjCAutoreleasePoolPop(llvm::Value *Ptr);
5118 llvm::Value *EmitObjCAutoreleasePoolPush();
5119 llvm::Value *EmitObjCMRRAutoreleasePoolPush();
5120 void EmitObjCAutoreleasePoolCleanup(llvm::Value *Ptr);
5121 void EmitObjCMRRAutoreleasePoolPop(llvm::Value *Ptr);
5122
5123 /// Emits a reference binding to the passed in expression.
5124 RValue EmitReferenceBindingToExpr(const Expr *E);
5125
5126 //===--------------------------------------------------------------------===//
5127 // Expression Emission
5128 //===--------------------------------------------------------------------===//
5129
5130 // Expressions are broken into three classes: scalar, complex, aggregate.
5131
5132 /// EmitScalarExpr - Emit the computation of the specified expression of LLVM
5133 /// scalar type, returning the result.
5134 llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
5135
5136 /// Emit a conversion from the specified type to the specified destination
5137 /// type, both of which are LLVM scalar types.
5138 llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
5139 QualType DstTy, SourceLocation Loc);
5140
5141 /// Emit a conversion from the specified complex type to the specified
5142 /// destination type, where the destination type is an LLVM scalar type.
5143 llvm::Value *EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy,
5144 QualType DstTy,
5145 SourceLocation Loc);
5146
5147 /// EmitAggExpr - Emit the computation of the specified expression
5148 /// of aggregate type. The result is computed into the given slot,
5149 /// which may be null to indicate that the value is not needed.
5150 void EmitAggExpr(const Expr *E, AggValueSlot AS);
5151
5152 /// EmitAggExprToLValue - Emit the computation of the specified expression of
5153 /// aggregate type into a temporary LValue.
5154 LValue EmitAggExprToLValue(const Expr *E);
5155
5156 enum ExprValueKind { EVK_RValue, EVK_NonRValue };
5157
5158 /// EmitAggFinalDestCopy - Emit copy of the specified aggregate into
5159 /// destination address.
5160 void EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest, const LValue &Src,
5161 ExprValueKind SrcKind);
5162
5163 /// Create a store to \arg DstPtr from \arg Src, truncating the stored value
5164 /// to at most \arg DstSize bytes.
5165 void CreateCoercedStore(llvm::Value *Src, QualType SrcFETy, Address Dst,
5166 llvm::TypeSize DstSize, bool DstIsVolatile);
5167
5168 /// EmitExtendGCLifetime - Given a pointer to an Objective-C object,
5169 /// make sure it survives garbage collection until this point.
5170 void EmitExtendGCLifetime(llvm::Value *object);
5171
5172 /// EmitComplexExpr - Emit the computation of the specified expression of
5173 /// complex type, returning the result.
5174 ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal = false,
5175 bool IgnoreImag = false);
5176
5177 /// EmitComplexExprIntoLValue - Emit the given expression of complex
5178 /// type and place its result into the specified l-value.
5179 void EmitComplexExprIntoLValue(const Expr *E, LValue dest, bool isInit);
5180
5181 /// EmitStoreOfComplex - Store a complex number into the specified l-value.
5182 void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit);
5183
5184 /// EmitLoadOfComplex - Load a complex number from the specified l-value.
5185 ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc);
5186
5187 ComplexPairTy EmitPromotedComplexExpr(const Expr *E, QualType PromotionType);
5188 llvm::Value *EmitPromotedScalarExpr(const Expr *E, QualType PromotionType);
5189 ComplexPairTy EmitPromotedValue(ComplexPairTy result, QualType PromotionType);
5190 ComplexPairTy EmitUnPromotedValue(ComplexPairTy result,
5191 QualType PromotionType);
5192
5193 Address emitAddrOfRealComponent(Address complex, QualType complexType);
5194 Address emitAddrOfImagComponent(Address complex, QualType complexType);
5195
5196 /// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the
5197 /// global variable that has already been created for it. If the initializer
5198 /// has a different type than GV does, this may free GV and return a different
5199 /// one. Otherwise it just returns GV.
5200 llvm::GlobalVariable *AddInitializerToStaticVarDecl(const VarDecl &D,
5201 llvm::GlobalVariable *GV);
5202
5203 // Emit an @llvm.invariant.start call for the given memory region.
5204 void EmitInvariantStart(llvm::Constant *Addr, CharUnits Size);
5205
5206 /// EmitCXXGlobalVarDeclInit - Create the initializer for a C++
5207 /// variable with global storage.
5208 void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::GlobalVariable *GV,
5209 bool PerformInit);
5210
5211 llvm::Constant *createAtExitStub(const VarDecl &VD, llvm::FunctionCallee Dtor,
5212 llvm::Constant *Addr);
5213
5214 llvm::Function *createTLSAtExitStub(const VarDecl &VD,
5215 llvm::FunctionCallee Dtor,
5216 llvm::Constant *Addr,
5217 llvm::FunctionCallee &AtExit);
5218
5219 /// Call atexit() with a function that passes the given argument to
5220 /// the given function.
5221 void registerGlobalDtorWithAtExit(const VarDecl &D, llvm::FunctionCallee fn,
5222 llvm::Constant *addr);
5223
5224 /// Registers the dtor using 'llvm.global_dtors' for platforms that do not
5225 /// support an 'atexit()' function.
5226 void registerGlobalDtorWithLLVM(const VarDecl &D, llvm::FunctionCallee fn,
5227 llvm::Constant *addr);
5228
5229 /// Call atexit() with function dtorStub.
5230 void registerGlobalDtorWithAtExit(llvm::Constant *dtorStub);
5231
5232 /// Call unatexit() with function dtorStub.
5233 llvm::Value *unregisterGlobalDtorWithUnAtExit(llvm::Constant *dtorStub);
5234
5235 /// Emit code in this function to perform a guarded variable
5236 /// initialization. Guarded initializations are used when it's not
5237 /// possible to prove that an initialization will be done exactly
5238 /// once, e.g. with a static local variable or a static data member
5239 /// of a class template.
5240 void EmitCXXGuardedInit(const VarDecl &D, llvm::GlobalVariable *DeclPtr,
5241 bool PerformInit);
5242
5243 enum class GuardKind { VariableGuard, TlsGuard };
5244
5245 /// Emit a branch to select whether or not to perform guarded initialization.
5246 void EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
5247 llvm::BasicBlock *InitBlock,
5248 llvm::BasicBlock *NoInitBlock, GuardKind Kind,
5249 const VarDecl *D);
5250
5251 /// GenerateCXXGlobalInitFunc - Generates code for initializing global
5252 /// variables.
5253 void
5254 GenerateCXXGlobalInitFunc(llvm::Function *Fn,
5255 ArrayRef<llvm::Function *> CXXThreadLocals,
5256 ConstantAddress Guard = ConstantAddress::invalid());
5257
5258 /// GenerateCXXGlobalCleanUpFunc - Generates code for cleaning up global
5259 /// variables.
5260 void GenerateCXXGlobalCleanUpFunc(
5261 llvm::Function *Fn,
5262 ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
5263 llvm::Constant *>>
5264 DtorsOrStermFinalizers);
5265
5266 void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn, const VarDecl *D,
5267 llvm::GlobalVariable *Addr,
5268 bool PerformInit);
5269
5270 void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest);
5271
5272 void EmitSynthesizedCXXCopyCtor(Address Dest, Address Src, const Expr *Exp);
5273
5274 void EmitCXXThrowExpr(const CXXThrowExpr *E, bool KeepInsertionPoint = true);
5275
5276 RValue EmitAtomicExpr(AtomicExpr *E);
5277
5278 void EmitFakeUse(Address Addr);
5279
5280 //===--------------------------------------------------------------------===//
5281 // Annotations Emission
5282 //===--------------------------------------------------------------------===//
5283
5284 /// Emit an annotation call (intrinsic).
5285 llvm::Value *EmitAnnotationCall(llvm::Function *AnnotationFn,
5286 llvm::Value *AnnotatedVal,
5287 StringRef AnnotationStr,
5288 SourceLocation Location,
5289 const AnnotateAttr *Attr);
5290
5291 /// Emit local annotations for the local variable V, declared by D.
5292 void EmitVarAnnotations(const VarDecl *D, llvm::Value *V);
5293
5294 /// Emit field annotations for the given field & value. Returns the
5295 /// annotation result.
5296 Address EmitFieldAnnotations(const FieldDecl *D, Address V);
5297
5298 //===--------------------------------------------------------------------===//
5299 // Internal Helpers
5300 //===--------------------------------------------------------------------===//
5301
5302 /// ContainsLabel - Return true if the statement contains a label in it. If
5303 /// this statement is not executed normally, it not containing a label means
5304 /// that we can just remove the code.
5305 static bool ContainsLabel(const Stmt *S, bool IgnoreCaseStmts = false);
5306
5307 /// containsBreak - Return true if the statement contains a break out of it.
5308 /// If the statement (recursively) contains a switch or loop with a break
5309 /// inside of it, this is fine.
5310 static bool containsBreak(const Stmt *S);
5311
5312 /// Determine if the given statement might introduce a declaration into the
5313 /// current scope, by being a (possibly-labelled) DeclStmt.
5314 static bool mightAddDeclToScope(const Stmt *S);
5315
5316 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
5317 /// to a constant, or if it does but contains a label, return false. If it
5318 /// constant folds return true and set the boolean result in Result.
5319 bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result,
5320 bool AllowLabels = false);
5321
5322 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
5323 /// to a constant, or if it does but contains a label, return false. If it
5324 /// constant folds return true and set the folded value.
5325 bool ConstantFoldsToSimpleInteger(const Expr *Cond, llvm::APSInt &Result,
5326 bool AllowLabels = false);
5327
5328 /// Ignore parentheses and logical-NOT to track conditions consistently.
5329 static const Expr *stripCond(const Expr *C);
5330
5331 /// isInstrumentedCondition - Determine whether the given condition is an
5332 /// instrumentable condition (i.e. no "&&" or "||").
5333 static bool isInstrumentedCondition(const Expr *C);
5334
5335 /// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
5336 /// increments a profile counter based on the semantics of the given logical
5337 /// operator opcode. This is used to instrument branch condition coverage
5338 /// for logical operators.
5339 void EmitBranchToCounterBlock(const Expr *Cond, BinaryOperator::Opcode LOp,
5340 llvm::BasicBlock *TrueBlock,
5341 llvm::BasicBlock *FalseBlock,
5342 uint64_t TrueCount = 0,
5343 Stmt::Likelihood LH = Stmt::LH_None,
5344 const Expr *CntrIdx = nullptr);
5345
5346 /// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an
5347 /// if statement) to the specified blocks. Based on the condition, this might
5348 /// try to simplify the codegen of the conditional based on the branch.
5349 /// TrueCount should be the number of times we expect the condition to
5350 /// evaluate to true based on PGO data.
5351 void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock,
5352 llvm::BasicBlock *FalseBlock, uint64_t TrueCount,
5353 Stmt::Likelihood LH = Stmt::LH_None,
5354 const Expr *ConditionalOp = nullptr,
5355 const VarDecl *ConditionalDecl = nullptr);
5356
5357 /// Given an assignment `*LHS = RHS`, emit a test that checks if \p RHS is
5358 /// nonnull, if \p LHS is marked _Nonnull.
5359 void EmitNullabilityCheck(LValue LHS, llvm::Value *RHS, SourceLocation Loc);
5360
5361 /// An enumeration which makes it easier to specify whether or not an
5362 /// operation is a subtraction.
5363 enum { NotSubtraction = false, IsSubtraction = true };
5364
5365 /// Emit pointer + index arithmetic.
5366 llvm::Value *EmitPointerArithmetic(const BinaryOperator *BO,
5367 Expr *pointerOperand, llvm::Value *pointer,
5368 Expr *indexOperand, llvm::Value *index,
5369 bool isSubtraction);
5370
5371 /// Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to
5372 /// detect undefined behavior when the pointer overflow sanitizer is enabled.
5373 /// \p SignedIndices indicates whether any of the GEP indices are signed.
5374 /// \p IsSubtraction indicates whether the expression used to form the GEP
5375 /// is a subtraction.
5376 llvm::Value *EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr,
5377 ArrayRef<llvm::Value *> IdxList,
5378 bool SignedIndices, bool IsSubtraction,
5379 SourceLocation Loc,
5380 const Twine &Name = "");
5381
5382 Address EmitCheckedInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
5383 llvm::Type *elementType, bool SignedIndices,
5384 bool IsSubtraction, SourceLocation Loc,
5385 CharUnits Align, const Twine &Name = "");
5386
5387 /// Specifies which type of sanitizer check to apply when handling a
5388 /// particular builtin.
5389 enum BuiltinCheckKind {
5390 BCK_CTZPassedZero,
5391 BCK_CLZPassedZero,
5392 BCK_AssumePassedFalse,
5393 };
5394
5395 /// Emits an argument for a call to a builtin. If the builtin sanitizer is
5396 /// enabled, a runtime check specified by \p Kind is also emitted.
5397 llvm::Value *EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind);
5398
5399 /// Emits an argument for a call to a `__builtin_assume`. If the builtin
5400 /// sanitizer is enabled, a runtime check is also emitted.
5401 llvm::Value *EmitCheckedArgForAssume(const Expr *E);
5402
5403 /// Emit a description of a type in a format suitable for passing to
5404 /// a runtime sanitizer handler.
5405 llvm::Constant *EmitCheckTypeDescriptor(QualType T);
5406
5407 /// Convert a value into a format suitable for passing to a runtime
5408 /// sanitizer handler.
5409 llvm::Value *EmitCheckValue(llvm::Value *V);
5410
5411 /// Emit a description of a source location in a format suitable for
5412 /// passing to a runtime sanitizer handler.
5413 llvm::Constant *EmitCheckSourceLocation(SourceLocation Loc);
5414
5415 void EmitKCFIOperandBundle(const CGCallee &Callee,
5416 SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
5417
5418 /// Create a basic block that will either trap or call a handler function in
5419 /// the UBSan runtime with the provided arguments, and create a conditional
5420 /// branch to it.
5421 void
5422 EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
5423 Checked,
5424 SanitizerHandler Check, ArrayRef<llvm::Constant *> StaticArgs,
5425 ArrayRef<llvm::Value *> DynamicArgs,
5426 const TrapReason *TR = nullptr);
5427
5428 /// Emit a slow path cross-DSO CFI check which calls __cfi_slowpath
5429 /// if Cond if false.
5430 void EmitCfiSlowPathCheck(SanitizerKind::SanitizerOrdinal Ordinal,
5431 llvm::Value *Cond, llvm::ConstantInt *TypeId,
5432 llvm::Value *Ptr,
5433 ArrayRef<llvm::Constant *> StaticArgs);
5434
5435 /// Emit a reached-unreachable diagnostic if \p Loc is valid and runtime
5436 /// checking is enabled. Otherwise, just emit an unreachable instruction.
5437 void EmitUnreachable(SourceLocation Loc);
5438
5439 /// Create a basic block that will call the trap intrinsic, and emit a
5440 /// conditional branch to it, for the -ftrapv checks.
5441 void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID,
5442 bool NoMerge = false, const TrapReason *TR = nullptr);
5443
5444 /// Emit a call to trap or debugtrap and attach function attribute
5445 /// "trap-func-name" if specified.
5446 llvm::CallInst *EmitTrapCall(llvm::Intrinsic::ID IntrID);
5447
5448 /// Emit a stub for the cross-DSO CFI check function.
5449 void EmitCfiCheckStub();
5450
5451 /// Emit a cross-DSO CFI failure handling function.
5452 void EmitCfiCheckFail();
5453
5454 /// Create a check for a function parameter that may potentially be
5455 /// declared as non-null.
5456 void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc,
5457 AbstractCallee AC, unsigned ParmNum);
5458
5459 void EmitNonNullArgCheck(Address Addr, QualType ArgType,
5460 SourceLocation ArgLoc, AbstractCallee AC,
5461 unsigned ParmNum);
5462
5463 /// EmitWriteback - Emit callbacks for function.
5464 void EmitWritebacks(const CallArgList &Args);
5465
5466 /// EmitCallArg - Emit a single call argument.
5467 void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType);
5468
5469 /// EmitDelegateCallArg - We are performing a delegate call; that
5470 /// is, the current function is delegating to another one. Produce
5471 /// a r-value suitable for passing the given parameter.
5472 void EmitDelegateCallArg(CallArgList &args, const VarDecl *param,
5473 SourceLocation loc);
5474
5475 /// SetFPAccuracy - Set the minimum required accuracy of the given floating
5476 /// point operation, expressed as the maximum relative error in ulp.
5477 void SetFPAccuracy(llvm::Value *Val, float Accuracy);
5478
5479 /// Set the minimum required accuracy of the given sqrt operation
5480 /// based on CodeGenOpts.
5481 void SetSqrtFPAccuracy(llvm::Value *Val);
5482
5483 /// Set the minimum required accuracy of the given sqrt operation based on
5484 /// CodeGenOpts.
5485 void SetDivFPAccuracy(llvm::Value *Val);
5486
5487 /// Set the codegen fast-math flags.
5488 void SetFastMathFlags(FPOptions FPFeatures);
5489
5490 // Truncate or extend a boolean vector to the requested number of elements.
5491 llvm::Value *emitBoolVecConversion(llvm::Value *SrcVec,
5492 unsigned NumElementsDst,
5493 const llvm::Twine &Name = "");
5494
5495 void maybeAttachRangeForLoad(llvm::LoadInst *Load, QualType Ty,
5496 SourceLocation Loc);
5497
5498 // Emits a convergence_loop instruction for the given |BB|, with |ParentToken|
5499 // as it's parent convergence instr.
5500 llvm::ConvergenceControlInst *emitConvergenceLoopToken(llvm::BasicBlock *BB);
5501
5502private:
5503 // Adds a convergence_ctrl token with |ParentToken| as parent convergence
5504 // instr to the call |Input|.
5505 llvm::CallBase *addConvergenceControlToken(llvm::CallBase *Input);
5506
5507 // Find the convergence_entry instruction |F|, or emits ones if none exists.
5508 // Returns the convergence instruction.
5509 llvm::ConvergenceControlInst *
5510 getOrEmitConvergenceEntryToken(llvm::Function *F);
5511
5512private:
5513 llvm::MDNode *getRangeForLoadFromType(QualType Ty);
5514 void EmitReturnOfRValue(RValue RV, QualType Ty);
5515
5516 void deferPlaceholderReplacement(llvm::Instruction *Old, llvm::Value *New);
5517
5518 llvm::SmallVector<std::pair<llvm::WeakTrackingVH, llvm::Value *>, 4>
5519 DeferredReplacements;
5520
5521 /// Set the address of a local variable.
5522 void setAddrOfLocalVar(const VarDecl *VD, Address Addr) {
5523 assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!");
5524 LocalDeclMap.insert(KV: {VD, Addr});
5525 }
5526
5527 /// ExpandTypeFromArgs - Reconstruct a structure of type \arg Ty
5528 /// from function arguments into \arg Dst. See ABIArgInfo::Expand.
5529 ///
5530 /// \param AI - The first function argument of the expansion.
5531 void ExpandTypeFromArgs(QualType Ty, LValue Dst,
5532 llvm::Function::arg_iterator &AI);
5533
5534 /// ExpandTypeToArgs - Expand an CallArg \arg Arg, with the LLVM type for \arg
5535 /// Ty, into individual arguments on the provided vector \arg IRCallArgs,
5536 /// starting at index \arg IRCallArgPos. See ABIArgInfo::Expand.
5537 void ExpandTypeToArgs(QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
5538 SmallVectorImpl<llvm::Value *> &IRCallArgs,
5539 unsigned &IRCallArgPos);
5540
5541 std::pair<llvm::Value *, llvm::Type *>
5542 EmitAsmInput(const TargetInfo::ConstraintInfo &Info, const Expr *InputExpr,
5543 std::string &ConstraintStr);
5544
5545 std::pair<llvm::Value *, llvm::Type *>
5546 EmitAsmInputLValue(const TargetInfo::ConstraintInfo &Info, LValue InputValue,
5547 QualType InputType, std::string &ConstraintStr,
5548 SourceLocation Loc);
5549
5550 /// Attempts to statically evaluate the object size of E. If that
5551 /// fails, emits code to figure the size of E out for us. This is
5552 /// pass_object_size aware.
5553 ///
5554 /// If EmittedExpr is non-null, this will use that instead of re-emitting E.
5555 llvm::Value *evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
5556 llvm::IntegerType *ResType,
5557 llvm::Value *EmittedE,
5558 bool IsDynamic);
5559
5560 /// Emits the size of E, as required by __builtin_object_size. This
5561 /// function is aware of pass_object_size parameters, and will act accordingly
5562 /// if E is a parameter with the pass_object_size attribute.
5563 llvm::Value *emitBuiltinObjectSize(const Expr *E, unsigned Type,
5564 llvm::IntegerType *ResType,
5565 llvm::Value *EmittedE, bool IsDynamic);
5566
5567 llvm::Value *emitCountedBySize(const Expr *E, llvm::Value *EmittedE,
5568 unsigned Type, llvm::IntegerType *ResType);
5569
5570 llvm::Value *emitCountedByMemberSize(const MemberExpr *E, const Expr *Idx,
5571 llvm::Value *EmittedE,
5572 QualType CastedArrayElementTy,
5573 unsigned Type,
5574 llvm::IntegerType *ResType);
5575
5576 llvm::Value *emitCountedByPointerSize(const ImplicitCastExpr *E,
5577 const Expr *Idx, llvm::Value *EmittedE,
5578 QualType CastedArrayElementTy,
5579 unsigned Type,
5580 llvm::IntegerType *ResType);
5581
5582 void emitZeroOrPatternForAutoVarInit(QualType type, const VarDecl &D,
5583 Address Loc);
5584
5585public:
5586 enum class EvaluationOrder {
5587 ///! No language constraints on evaluation order.
5588 Default,
5589 ///! Language semantics require left-to-right evaluation.
5590 ForceLeftToRight,
5591 ///! Language semantics require right-to-left evaluation.
5592 ForceRightToLeft
5593 };
5594
5595 // Wrapper for function prototype sources. Wraps either a FunctionProtoType or
5596 // an ObjCMethodDecl.
5597 struct PrototypeWrapper {
5598 llvm::PointerUnion<const FunctionProtoType *, const ObjCMethodDecl *> P;
5599
5600 PrototypeWrapper(const FunctionProtoType *FT) : P(FT) {}
5601 PrototypeWrapper(const ObjCMethodDecl *MD) : P(MD) {}
5602 };
5603
5604 void EmitCallArgs(CallArgList &Args, PrototypeWrapper Prototype,
5605 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
5606 AbstractCallee AC = AbstractCallee(),
5607 unsigned ParamsToSkip = 0,
5608 EvaluationOrder Order = EvaluationOrder::Default);
5609
5610 /// EmitPointerWithAlignment - Given an expression with a pointer type,
5611 /// emit the value and compute our best estimate of the alignment of the
5612 /// pointee.
5613 ///
5614 /// \param BaseInfo - If non-null, this will be initialized with
5615 /// information about the source of the alignment and the may-alias
5616 /// attribute. Note that this function will conservatively fall back on
5617 /// the type when it doesn't recognize the expression and may-alias will
5618 /// be set to false.
5619 ///
5620 /// One reasonable way to use this information is when there's a language
5621 /// guarantee that the pointer must be aligned to some stricter value, and
5622 /// we're simply trying to ensure that sufficiently obvious uses of under-
5623 /// aligned objects don't get miscompiled; for example, a placement new
5624 /// into the address of a local variable. In such a case, it's quite
5625 /// reasonable to just ignore the returned alignment when it isn't from an
5626 /// explicit source.
5627 Address
5628 EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo = nullptr,
5629 TBAAAccessInfo *TBAAInfo = nullptr,
5630 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
5631
5632 /// If \p E references a parameter with pass_object_size info or a constant
5633 /// array size modifier, emit the object size divided by the size of \p EltTy.
5634 /// Otherwise return null.
5635 llvm::Value *LoadPassedObjectSize(const Expr *E, QualType EltTy);
5636
5637 void EmitSanitizerStatReport(llvm::SanitizerStatKind SSK);
5638
5639 struct FMVResolverOption {
5640 llvm::Function *Function;
5641 llvm::SmallVector<StringRef, 8> Features;
5642 std::optional<StringRef> Architecture;
5643
5644 FMVResolverOption(llvm::Function *F, ArrayRef<StringRef> Feats,
5645 std::optional<StringRef> Arch = std::nullopt)
5646 : Function(F), Features(Feats), Architecture(Arch) {}
5647 };
5648
5649 // Emits the body of a multiversion function's resolver. Assumes that the
5650 // options are already sorted in the proper order, with the 'default' option
5651 // last (if it exists).
5652 void EmitMultiVersionResolver(llvm::Function *Resolver,
5653 ArrayRef<FMVResolverOption> Options);
5654 void EmitX86MultiVersionResolver(llvm::Function *Resolver,
5655 ArrayRef<FMVResolverOption> Options);
5656 void EmitAArch64MultiVersionResolver(llvm::Function *Resolver,
5657 ArrayRef<FMVResolverOption> Options);
5658 void EmitRISCVMultiVersionResolver(llvm::Function *Resolver,
5659 ArrayRef<FMVResolverOption> Options);
5660 void EmitPPCAIXMultiVersionResolver(llvm::Function *Resolver,
5661 ArrayRef<FMVResolverOption> Options);
5662
5663 Address EmitAddressOfPFPField(Address RecordPtr, const PFPField &Field);
5664 Address EmitAddressOfPFPField(Address RecordPtr, Address FieldPtr,
5665 const FieldDecl *Field);
5666
5667private:
5668 QualType getVarArgType(const Expr *Arg);
5669
5670 void EmitDeclMetadata();
5671
5672 BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType,
5673 const AutoVarEmission &emission);
5674
5675 void AddObjCARCExceptionMetadata(llvm::Instruction *Inst);
5676
5677 llvm::Value *GetValueForARMHint(unsigned BuiltinID);
5678 llvm::Value *EmitX86CpuIs(const CallExpr *E);
5679 llvm::Value *EmitX86CpuIs(StringRef CPUStr);
5680 llvm::Value *EmitX86CpuSupports(const CallExpr *E);
5681 llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs);
5682 llvm::Value *EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask);
5683 llvm::Value *EmitX86CpuInit();
5684 llvm::Value *FormX86ResolverCondition(const FMVResolverOption &RO);
5685 llvm::Value *EmitAArch64CpuInit();
5686 llvm::Value *FormAArch64ResolverCondition(const FMVResolverOption &RO);
5687 llvm::Value *EmitAArch64CpuSupports(const CallExpr *E);
5688 llvm::Value *EmitAArch64CpuSupports(ArrayRef<StringRef> FeatureStrs);
5689};
5690
5691inline DominatingLLVMValue::saved_type
5692DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) {
5693 if (!needsSaving(value))
5694 return saved_type(value);
5695
5696 // Otherwise, we need an alloca.
5697 auto align = CharUnits::fromQuantity(
5698 Quantity: CGF.CGM.getDataLayout().getPrefTypeAlign(Ty: value->getType()))
5699 .getAsAlign();
5700 llvm::AllocaInst *AI =
5701 CGF.CreateTempAlloca(Ty: value->getType(), Name: "cond-cleanup.save");
5702 AI->setAlignment(align);
5703 CGF.Builder.CreateAlignedStore(Val: value, Ptr: AI, Align: align);
5704
5705 return saved_type(AI, value->getType());
5706}
5707
5708inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF,
5709 saved_type value) {
5710 // If the value says it wasn't saved, trust that it's still dominating.
5711 if (!value.isSaved())
5712 return value.Value;
5713
5714 // Otherwise, it should be an alloca instruction, as set up in save().
5715 auto Alloca = cast<llvm::AllocaInst>(Val: value.Value);
5716 return CGF.Builder.CreateAlignedLoad(Ty: value.Type, Ptr: Alloca, Align: Alloca->getAlign());
5717}
5718
5719} // end namespace CodeGen
5720
5721// Map the LangOption for floating point exception behavior into
5722// the corresponding enum in the IR.
5723llvm::fp::ExceptionBehavior
5724ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind);
5725} // end namespace clang
5726
5727#endif
5728