1//===-- CodeGenFunction.h - Per-Function state for LLVM CodeGen -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the internal per-function state used for llvm translation.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
14#define LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
15
16#include "CGBuilder.h"
17#include "CGLoopInfo.h"
18#include "CGValue.h"
19#include "CodeGenModule.h"
20#include "EHScopeStack.h"
21#include "SanitizerHandler.h"
22#include "VarBypassDetector.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/CharUnits.h"
25#include "clang/AST/CurrentSourceLocExprScope.h"
26#include "clang/AST/ExprCXX.h"
27#include "clang/AST/ExprObjC.h"
28#include "clang/AST/ExprOpenMP.h"
29#include "clang/AST/StmtOpenACC.h"
30#include "clang/AST/StmtOpenMP.h"
31#include "clang/AST/StmtSYCL.h"
32#include "clang/AST/Type.h"
33#include "clang/Basic/ABI.h"
34#include "clang/Basic/CapturedStmt.h"
35#include "clang/Basic/CodeGenOptions.h"
36#include "clang/Basic/OpenMPKinds.h"
37#include "clang/Basic/TargetInfo.h"
38#include "llvm/ADT/ArrayRef.h"
39#include "llvm/ADT/DenseMap.h"
40#include "llvm/ADT/MapVector.h"
41#include "llvm/ADT/SmallVector.h"
42#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
43#include "llvm/IR/Instructions.h"
44#include "llvm/IR/ValueHandle.h"
45#include "llvm/Support/Debug.h"
46#include "llvm/Transforms/Utils/SanitizerStats.h"
47#include <optional>
48
49namespace llvm {
50class BasicBlock;
51class ConvergenceControlInst;
52class LLVMContext;
53class MDNode;
54class SwitchInst;
55class Twine;
56class Value;
57class CanonicalLoopInfo;
58} // namespace llvm
59
60namespace clang {
61class ASTContext;
62class AsmConstraintsInfo;
63class CXXDestructorDecl;
64class CXXForRangeStmt;
65class CXXTryStmt;
66class Decl;
67class LabelDecl;
68class FunctionDecl;
69class FunctionProtoType;
70class LabelStmt;
71class ObjCContainerDecl;
72class ObjCInterfaceDecl;
73class ObjCIvarDecl;
74class ObjCMethodDecl;
75class ObjCImplementationDecl;
76class ObjCPropertyImplDecl;
77class TargetInfo;
78class VarDecl;
79class ObjCForCollectionStmt;
80class ObjCAtTryStmt;
81class ObjCAtThrowStmt;
82class ObjCAtSynchronizedStmt;
83class ObjCAutoreleasePoolStmt;
84class OMPUseDevicePtrClause;
85class OMPUseDeviceAddrClause;
86class SVETypeFlags;
87class OMPExecutableDirective;
88
89namespace analyze_os_log {
90class OSLogBufferLayout;
91}
92
93namespace CodeGen {
94class CodeGenTypes;
95class CodeGenPGO;
96class CGCallee;
97class CGFunctionInfo;
98class CGBlockInfo;
99class CGCXXABI;
100class BlockByrefHelpers;
101class BlockByrefInfo;
102class BlockFieldFlags;
103class RegionCodeGenTy;
104class TargetCodeGenInfo;
105struct OMPTaskDataTy;
106struct CGCoroData;
107
108// clang-format off
109/// The kind of evaluation to perform on values of a particular
110/// type. Basically, is the code in CGExprScalar, CGExprComplex, or
111/// CGExprAgg?
112///
113/// TODO: should vectors maybe be split out into their own thing?
114enum TypeEvaluationKind {
115 TEK_Scalar,
116 TEK_Complex,
117 TEK_Aggregate
118};
119// clang-format on
120
121/// Helper class with most of the code for saving a value for a
122/// conditional expression cleanup.
123struct DominatingLLVMValue {
124 struct saved_type {
125 llvm::Value *Value; // Original value if not saved, alloca if saved
126 llvm::Type *Type; // nullptr if not saved, element type if saved
127
128 saved_type() : Value(nullptr), Type(nullptr) {}
129 saved_type(llvm::Value *V) : Value(V), Type(nullptr) {}
130 saved_type(llvm::AllocaInst *Alloca, llvm::Type *Ty)
131 : Value(Alloca), Type(Ty) {}
132
133 bool isSaved() const { return Type != nullptr; }
134 };
135
136 /// Answer whether the given value needs extra work to be saved.
137 static bool needsSaving(llvm::Value *value) {
138 if (!value)
139 return false;
140
141 // If it's not an instruction, we don't need to save.
142 if (!isa<llvm::Instruction>(Val: value))
143 return false;
144
145 // If it's an instruction in the entry block, we don't need to save.
146 llvm::BasicBlock *block = cast<llvm::Instruction>(Val: value)->getParent();
147 return (block != &block->getParent()->getEntryBlock());
148 }
149
150 static saved_type save(CodeGenFunction &CGF, llvm::Value *value);
151 static llvm::Value *restore(CodeGenFunction &CGF, saved_type value);
152};
153
154/// A partial specialization of DominatingValue for llvm::Values that
155/// might be llvm::Instructions.
156template <class T> struct DominatingPointer<T, true> : DominatingLLVMValue {
157 typedef T *type;
158 static type restore(CodeGenFunction &CGF, saved_type value) {
159 return static_cast<T *>(DominatingLLVMValue::restore(CGF, value));
160 }
161};
162
163/// A specialization of DominatingValue for Address.
164template <> struct DominatingValue<Address> {
165 typedef Address type;
166
167 struct saved_type {
168 DominatingLLVMValue::saved_type BasePtr;
169 llvm::Type *ElementType;
170 CharUnits Alignment;
171 DominatingLLVMValue::saved_type Offset;
172 llvm::PointerType *EffectiveType;
173 };
174
175 static bool needsSaving(type value) {
176 if (DominatingLLVMValue::needsSaving(value: value.getBasePointer()) ||
177 DominatingLLVMValue::needsSaving(value: value.getOffset()))
178 return true;
179 return false;
180 }
181 static saved_type save(CodeGenFunction &CGF, type value) {
182 return {.BasePtr: DominatingLLVMValue::save(CGF, value: value.getBasePointer()),
183 .ElementType: value.getElementType(), .Alignment: value.getAlignment(),
184 .Offset: DominatingLLVMValue::save(CGF, value: value.getOffset()), .EffectiveType: value.getType()};
185 }
186 static type restore(CodeGenFunction &CGF, saved_type value) {
187 return Address(DominatingLLVMValue::restore(CGF, value: value.BasePtr),
188 value.ElementType, value.Alignment, CGPointerAuthInfo(),
189 DominatingLLVMValue::restore(CGF, value: value.Offset));
190 }
191};
192
193/// A specialization of DominatingValue for RValue.
194template <> struct DominatingValue<RValue> {
195 typedef RValue type;
196 class saved_type {
197 enum Kind {
198 ScalarLiteral,
199 ScalarAddress,
200 AggregateLiteral,
201 AggregateAddress,
202 ComplexAddress
203 };
204 union {
205 struct {
206 DominatingLLVMValue::saved_type first, second;
207 } Vals;
208 DominatingValue<Address>::saved_type AggregateAddr;
209 };
210 LLVM_PREFERRED_TYPE(Kind)
211 unsigned K : 3;
212
213 saved_type(DominatingLLVMValue::saved_type Val1, unsigned K)
214 : Vals{.first: Val1, .second: DominatingLLVMValue::saved_type()}, K(K) {}
215
216 saved_type(DominatingLLVMValue::saved_type Val1,
217 DominatingLLVMValue::saved_type Val2)
218 : Vals{.first: Val1, .second: Val2}, K(ComplexAddress) {}
219
220 saved_type(DominatingValue<Address>::saved_type AggregateAddr, unsigned K)
221 : AggregateAddr(AggregateAddr), K(K) {}
222
223 public:
224 static bool needsSaving(RValue value);
225 static saved_type save(CodeGenFunction &CGF, RValue value);
226 RValue restore(CodeGenFunction &CGF);
227
228 // implementations in CGCleanup.cpp
229 };
230
231 static bool needsSaving(type value) { return saved_type::needsSaving(value); }
232 static saved_type save(CodeGenFunction &CGF, type value) {
233 return saved_type::save(CGF, value);
234 }
235 static type restore(CodeGenFunction &CGF, saved_type value) {
236 return value.restore(CGF);
237 }
238};
239
240/// A scoped helper to set the current source atom group for
241/// CGDebugInfo::addInstToCurrentSourceAtom. A source atom is a source construct
242/// that is "interesting" for debug stepping purposes. We use an atom group
243/// number to track the instruction(s) that implement the functionality for the
244/// atom, plus backup instructions/source locations.
245class ApplyAtomGroup {
246 uint64_t OriginalAtom = 0;
247 CGDebugInfo *DI = nullptr;
248
249 ApplyAtomGroup(const ApplyAtomGroup &) = delete;
250 void operator=(const ApplyAtomGroup &) = delete;
251
252public:
253 ApplyAtomGroup(CGDebugInfo *DI);
254 ~ApplyAtomGroup();
255};
256
257/// CodeGenFunction - This class organizes the per-function state that is used
258/// while generating LLVM code.
259class CodeGenFunction : public CodeGenTypeCache {
260 CodeGenFunction(const CodeGenFunction &) = delete;
261 void operator=(const CodeGenFunction &) = delete;
262
263 friend class CGCXXABI;
264 friend class clang::AsmConstraintsInfo;
265
266public:
267 /// A jump destination is an abstract label, branching to which may
268 /// require a jump out through normal cleanups.
269 struct JumpDest {
270 JumpDest() : Block(nullptr), Index(0) {}
271 JumpDest(llvm::BasicBlock *Block, EHScopeStack::stable_iterator Depth,
272 unsigned Index)
273 : Block(Block), ScopeDepth(Depth), Index(Index) {}
274
275 bool isValid() const { return Block != nullptr; }
276 llvm::BasicBlock *getBlock() const { return Block; }
277 EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; }
278 unsigned getDestIndex() const { return Index; }
279
280 // This should be used cautiously.
281 void setScopeDepth(EHScopeStack::stable_iterator depth) {
282 ScopeDepth = depth;
283 }
284
285 private:
286 llvm::BasicBlock *Block;
287 EHScopeStack::stable_iterator ScopeDepth;
288 unsigned Index;
289 };
290
291 CodeGenModule &CGM; // Per-module state.
292 const TargetInfo &Target;
293
294 // For EH/SEH outlined funclets, this field points to parent's CGF
295 CodeGenFunction *ParentCGF = nullptr;
296
297 typedef std::pair<llvm::Value *, llvm::Value *> ComplexPairTy;
298 LoopInfoStack LoopStack;
299 CGBuilderTy Builder;
300
301 // Stores variables for which we can't generate correct lifetime markers
302 // because of jumps.
303 VarBypassDetector Bypasses;
304
305 // Addresses of bypassed variables, for re-emitting their
306 // trivial-auto-var-init at a jump that re-enters their scope.
307 llvm::SmallDenseMap<const VarDecl *, Address, 4> BypassedVarInits;
308
309 // Jumps, like gotos or switches, that may bypass a declaration that has not
310 // been emitted yet. EmitAutoVarAlloca patches the init in before the jump
311 // once the alloca exists.
312 struct BypassingForwardJump {
313 llvm::AssertingVH<llvm::BasicBlock> Block;
314 const Stmt *Source;
315 };
316 llvm::SmallVector<BypassingForwardJump, 4> BypassingForwardJumps;
317
318 /// List of recently emitted OMPCanonicalLoops.
319 ///
320 /// Since OMPCanonicalLoops are nested inside other statements (in particular
321 /// CapturedStmt generated by OMPExecutableDirective and non-perfectly nested
322 /// loops), we cannot directly call OMPEmitOMPCanonicalLoop and receive its
323 /// llvm::CanonicalLoopInfo. Instead, we call EmitStmt and any
324 /// OMPEmitOMPCanonicalLoop called by it will add its CanonicalLoopInfo to
325 /// this stack when done. Entering a new loop requires clearing this list; it
326 /// either means we start parsing a new loop nest (in which case the previous
327 /// loop nest goes out of scope) or a second loop in the same level in which
328 /// case it would be ambiguous into which of the two (or more) loops the loop
329 /// nest would extend.
330 SmallVector<llvm::CanonicalLoopInfo *, 4> OMPLoopNestStack;
331
332 /// Stack to track the controlled convergence tokens.
333 SmallVector<llvm::ConvergenceControlInst *, 4> ConvergenceTokenStack;
334
335 /// Number of nested loop to be consumed by the last surrounding
336 /// loop-associated directive.
337 int ExpectedOMPLoopDepth = 0;
338
339 // CodeGen lambda for loops and support for ordered clause
340 typedef llvm::function_ref<void(CodeGenFunction &, const OMPLoopDirective &,
341 JumpDest)>
342 CodeGenLoopTy;
343 typedef llvm::function_ref<void(CodeGenFunction &, SourceLocation,
344 const unsigned, const bool)>
345 CodeGenOrderedTy;
346
347 // Codegen lambda for loop bounds in worksharing loop constructs
348 typedef llvm::function_ref<std::pair<LValue, LValue>(
349 CodeGenFunction &, const OMPExecutableDirective &S)>
350 CodeGenLoopBoundsTy;
351
352 // Codegen lambda for loop bounds in dispatch-based loop implementation
353 typedef llvm::function_ref<std::pair<llvm::Value *, llvm::Value *>(
354 CodeGenFunction &, const OMPExecutableDirective &S, Address LB,
355 Address UB)>
356 CodeGenDispatchBoundsTy;
357
358 /// CGBuilder insert helper. This function is called after an
359 /// instruction is created using Builder.
360 void InsertHelper(llvm::Instruction *I, const llvm::Twine &Name,
361 llvm::BasicBlock::iterator InsertPt) const;
362
363 /// CurFuncDecl - Holds the Decl for the current outermost
364 /// non-closure context.
365 const Decl *CurFuncDecl = nullptr;
366 /// CurCodeDecl - This is the inner-most code context, which includes blocks.
367 const Decl *CurCodeDecl = nullptr;
368 const CGFunctionInfo *CurFnInfo = nullptr;
369 QualType FnRetTy;
370 llvm::Function *CurFn = nullptr;
371
372 /// If a cast expression is being visited, this holds the current cast's
373 /// expression.
374 const CastExpr *CurCast = nullptr;
375
376 /// Save Parameter Decl for coroutine.
377 llvm::SmallVector<const ParmVarDecl *, 4> FnArgs;
378
379 // Holds coroutine data if the current function is a coroutine. We use a
380 // wrapper to manage its lifetime, so that we don't have to define CGCoroData
381 // in this header.
382 struct CGCoroInfo {
383 std::unique_ptr<CGCoroData> Data;
384 bool InSuspendBlock = false;
385 CGCoroInfo();
386 ~CGCoroInfo();
387 };
388 CGCoroInfo CurCoro;
389
390 bool isCoroutine() const { return CurCoro.Data != nullptr; }
391
392 bool inSuspendBlock() const {
393 return isCoroutine() && CurCoro.InSuspendBlock;
394 }
395
396 // Holds FramePtr for await_suspend wrapper generation,
397 // so that __builtin_coro_frame call can be lowered
398 // directly to value of its second argument
399 struct AwaitSuspendWrapperInfo {
400 llvm::Value *FramePtr = nullptr;
401 };
402 AwaitSuspendWrapperInfo CurAwaitSuspendWrapper;
403
404 // Generates wrapper function for `llvm.coro.await.suspend.*` intrinisics.
405 // It encapsulates SuspendExpr in a function, to separate it's body
406 // from the main coroutine to avoid miscompilations. Intrinisic
407 // is lowered to this function call in CoroSplit pass
408 // Function signature is:
409 // <type> __await_suspend_wrapper_<name>(ptr %awaiter, ptr %hdl)
410 // where type is one of (void, i1, ptr)
411 llvm::Function *generateAwaitSuspendWrapper(Twine const &CoroName,
412 Twine const &SuspendPointName,
413 CoroutineSuspendExpr const &S);
414
415 /// CurGD - The GlobalDecl for the current function being compiled.
416 GlobalDecl CurGD;
417
418 /// PrologueCleanupDepth - The cleanup depth enclosing all the
419 /// cleanups associated with the parameters.
420 EHScopeStack::stable_iterator PrologueCleanupDepth;
421
422 /// ReturnBlock - Unified return block.
423 JumpDest ReturnBlock;
424
425 /// ReturnValue - The temporary alloca to hold the return
426 /// value. This is invalid iff the function has no return value.
427 Address ReturnValue = Address::invalid();
428
429 /// ReturnValuePointer - The temporary alloca to hold a pointer to sret.
430 /// This is invalid if sret is not in use.
431 Address ReturnValuePointer = Address::invalid();
432
433 /// If a return statement is being visited, this holds the return statment's
434 /// result expression.
435 const Expr *RetExpr = nullptr;
436
437 /// Return true if a label was seen in the current scope.
438 bool hasLabelBeenSeenInCurrentScope() const {
439 if (CurLexicalScope)
440 return CurLexicalScope->hasLabels();
441 return !LabelMap.empty();
442 }
443
444 /// AllocaInsertPoint - This is an instruction in the entry block before which
445 /// we prefer to insert allocas.
446 llvm::AssertingVH<llvm::Instruction> AllocaInsertPt;
447
448private:
449 /// PostAllocaInsertPt - This is a place in the prologue where code can be
450 /// inserted that will be dominated by all the static allocas. This helps
451 /// achieve two things:
452 /// 1. Contiguity of all static allocas (within the prologue) is maintained.
453 /// 2. All other prologue code (which are dominated by static allocas) do
454 /// appear in the source order immediately after all static allocas.
455 ///
456 /// PostAllocaInsertPt will be lazily created when it is *really* required.
457 llvm::AssertingVH<llvm::Instruction> PostAllocaInsertPt = nullptr;
458
459public:
460 /// Return PostAllocaInsertPt. If it is not yet created, then insert it
461 /// immediately after AllocaInsertPt.
462 llvm::Instruction *getPostAllocaInsertPoint() {
463 if (!PostAllocaInsertPt) {
464 assert(AllocaInsertPt &&
465 "Expected static alloca insertion point at function prologue");
466 assert(AllocaInsertPt->getParent()->isEntryBlock() &&
467 "EBB should be entry block of the current code gen function");
468 PostAllocaInsertPt = AllocaInsertPt->clone();
469 PostAllocaInsertPt->setName("postallocapt");
470 PostAllocaInsertPt->insertAfter(InsertPos: AllocaInsertPt->getIterator());
471 }
472
473 return PostAllocaInsertPt;
474 }
475
476 // Try to preserve the source's name to make IR more readable.
477 llvm::Value *performAddrSpaceCast(llvm::Value *Src, llvm::Type *DestTy) {
478 return Builder.CreateAddrSpaceCast(
479 V: Src, DestTy, Name: Src->hasName() ? Src->getName() + ".ascast" : "");
480 }
481
482 /// API for captured statement code generation.
483 class CGCapturedStmtInfo {
484 public:
485 explicit CGCapturedStmtInfo(CapturedRegionKind K = CR_Default)
486 : Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {}
487 explicit CGCapturedStmtInfo(const CapturedStmt &S,
488 CapturedRegionKind K = CR_Default)
489 : Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {
490
491 RecordDecl::field_iterator Field =
492 S.getCapturedRecordDecl()->field_begin();
493 for (CapturedStmt::const_capture_iterator I = S.capture_begin(),
494 E = S.capture_end();
495 I != E; ++I, ++Field) {
496 if (I->capturesThis())
497 CXXThisFieldDecl = *Field;
498 else if (I->capturesVariable())
499 CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
500 else if (I->capturesVariableByCopy())
501 CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
502 }
503 }
504
505 virtual ~CGCapturedStmtInfo();
506
507 CapturedRegionKind getKind() const { return Kind; }
508
509 virtual void setContextValue(llvm::Value *V) { ThisValue = V; }
510 // Retrieve the value of the context parameter.
511 virtual llvm::Value *getContextValue() const { return ThisValue; }
512
513 /// Lookup the captured field decl for a variable.
514 virtual const FieldDecl *lookup(const VarDecl *VD) const {
515 return CaptureFields.lookup(Val: VD->getCanonicalDecl());
516 }
517
518 bool isCXXThisExprCaptured() const { return getThisFieldDecl() != nullptr; }
519 virtual FieldDecl *getThisFieldDecl() const { return CXXThisFieldDecl; }
520
521 static bool classof(const CGCapturedStmtInfo *) { return true; }
522
523 /// Emit the captured statement body.
524 virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) {
525 CGF.incrementProfileCounter(S);
526 CGF.EmitStmt(S);
527 }
528
529 /// Get the name of the capture helper.
530 virtual StringRef getHelperName() const { return "__captured_stmt"; }
531
532 /// Get the CaptureFields
533 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> getCaptureFields() {
534 return CaptureFields;
535 }
536
537 private:
538 /// The kind of captured statement being generated.
539 CapturedRegionKind Kind;
540
541 /// Keep the map between VarDecl and FieldDecl.
542 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields;
543
544 /// The base address of the captured record, passed in as the first
545 /// argument of the parallel region function.
546 llvm::Value *ThisValue;
547
548 /// Captured 'this' type.
549 FieldDecl *CXXThisFieldDecl;
550 };
551 CGCapturedStmtInfo *CapturedStmtInfo = nullptr;
552
553 /// RAII for correct setting/restoring of CapturedStmtInfo.
554 class CGCapturedStmtRAII {
555 private:
556 CodeGenFunction &CGF;
557 CGCapturedStmtInfo *PrevCapturedStmtInfo;
558
559 public:
560 CGCapturedStmtRAII(CodeGenFunction &CGF,
561 CGCapturedStmtInfo *NewCapturedStmtInfo)
562 : CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo) {
563 CGF.CapturedStmtInfo = NewCapturedStmtInfo;
564 }
565 ~CGCapturedStmtRAII() { CGF.CapturedStmtInfo = PrevCapturedStmtInfo; }
566 };
567
568 /// An abstract representation of regular/ObjC call/message targets.
569 class AbstractCallee {
570 /// The function declaration of the callee.
571 const Decl *CalleeDecl;
572
573 public:
574 AbstractCallee() : CalleeDecl(nullptr) {}
575 AbstractCallee(const FunctionDecl *FD) : CalleeDecl(FD) {}
576 AbstractCallee(const ObjCMethodDecl *OMD) : CalleeDecl(OMD) {}
577 bool hasFunctionDecl() const {
578 return isa_and_nonnull<FunctionDecl>(Val: CalleeDecl);
579 }
580 const Decl *getDecl() const { return CalleeDecl; }
581 unsigned getNumParams() const {
582 if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
583 return FD->getNumParams();
584 return cast<ObjCMethodDecl>(Val: CalleeDecl)->param_size();
585 }
586 const ParmVarDecl *getParamDecl(unsigned I) const {
587 if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
588 return FD->getParamDecl(i: I);
589 return *(cast<ObjCMethodDecl>(Val: CalleeDecl)->param_begin() + I);
590 }
591 };
592
593 /// Sanitizers enabled for this function.
594 SanitizerSet SanOpts;
595
596 /// True if CodeGen currently emits code implementing sanitizer checks.
597 bool IsSanitizerScope = false;
598
599 /// RAII object to set/unset CodeGenFunction::IsSanitizerScope.
600 class SanitizerScope {
601 CodeGenFunction *CGF;
602
603 public:
604 SanitizerScope(CodeGenFunction *CGF);
605 ~SanitizerScope();
606 };
607
608 /// In C++, whether we are code generating a thunk. This controls whether we
609 /// should emit cleanups.
610 bool CurFuncIsThunk = false;
611
612 /// In ARC, whether we should autorelease the return value.
613 bool AutoreleaseResult = false;
614
615 /// Whether we processed a Microsoft-style asm block during CodeGen. These can
616 /// potentially set the return value.
617 bool SawAsmBlock = false;
618
619 GlobalDecl CurSEHParent;
620
621 /// True if the current function is an outlined SEH helper. This can be a
622 /// finally block or filter expression.
623 bool IsOutlinedSEHHelper = false;
624
625 /// True if CodeGen currently emits code inside presereved access index
626 /// region.
627 bool IsInPreservedAIRegion = false;
628
629 /// True if the current statement has nomerge attribute.
630 bool InNoMergeAttributedStmt = false;
631
632 /// True if the current statement has noinline attribute.
633 bool InNoInlineAttributedStmt = false;
634
635 /// True if the current statement has always_inline attribute.
636 bool InAlwaysInlineAttributedStmt = false;
637
638 /// True if the current statement has noconvergent attribute.
639 bool InNoConvergentAttributedStmt = false;
640
641 /// The mode string from the amdgpu_av attribute on the current statement,
642 /// or empty if the attribute is not present.
643 StringRef AMDGPUAvailableVisibleMode;
644
645 /// HLSL Branch attribute.
646 HLSLControlFlowHintAttr::Spelling HLSLControlFlowAttr =
647 HLSLControlFlowHintAttr::SpellingNotCalculated;
648
649 // The CallExpr within the current statement that the musttail attribute
650 // applies to. nullptr if there is no 'musttail' on the current statement.
651 const CallExpr *MustTailCall = nullptr;
652
653 /// Returns true if a function must make progress, which means the
654 /// mustprogress attribute can be added.
655 bool checkIfFunctionMustProgress() {
656 if (CGM.getCodeGenOpts().getFiniteLoops() ==
657 CodeGenOptions::FiniteLoopsKind::Never)
658 return false;
659
660 // C++11 and later guarantees that a thread eventually will do one of the
661 // following (C++11 [intro.multithread]p24 and C++17 [intro.progress]p1):
662 // - terminate,
663 // - make a call to a library I/O function,
664 // - perform an access through a volatile glvalue, or
665 // - perform a synchronization operation or an atomic operation.
666 //
667 // Hence each function is 'mustprogress' in C++11 or later.
668 return getLangOpts().CPlusPlus11;
669 }
670
671 /// Returns true if a loop must make progress, which means the mustprogress
672 /// attribute can be added. \p HasConstantCond indicates whether the branch
673 /// condition is a known constant.
674 bool checkIfLoopMustProgress(const Expr *, bool HasEmptyBody);
675
676 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
677 llvm::Value *BlockPointer = nullptr;
678
679 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
680 FieldDecl *LambdaThisCaptureField = nullptr;
681
682 /// A mapping from NRVO variables to the flags used to indicate
683 /// when the NRVO has been applied to this variable.
684 llvm::DenseMap<const VarDecl *, llvm::Value *> NRVOFlags;
685
686 EHScopeStack EHStack;
687 llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack;
688
689 // A stack of cleanups which were added to EHStack but have to be deactivated
690 // later before being popped or emitted. These are usually deactivated on
691 // exiting a `CleanupDeactivationScope` scope. For instance, after a
692 // full-expr.
693 //
694 // These are specially useful for correctly emitting cleanups while
695 // encountering branches out of expression (through stmt-expr or coroutine
696 // suspensions).
697 struct DeferredDeactivateCleanup {
698 EHScopeStack::stable_iterator Cleanup;
699 llvm::Instruction *DominatingIP;
700 };
701 llvm::SmallVector<DeferredDeactivateCleanup> DeferredDeactivationCleanupStack;
702
703 // Enters a new scope for capturing cleanups which are deferred to be
704 // deactivated, all of which will be deactivated once the scope is exited.
705 struct CleanupDeactivationScope {
706 CodeGenFunction &CGF;
707 size_t OldDeactivateCleanupStackSize;
708 bool Deactivated;
709 CleanupDeactivationScope(CodeGenFunction &CGF)
710 : CGF(CGF), OldDeactivateCleanupStackSize(
711 CGF.DeferredDeactivationCleanupStack.size()),
712 Deactivated(false) {}
713
714 void ForceDeactivate() {
715 assert(!Deactivated && "Deactivating already deactivated scope");
716 auto &Stack = CGF.DeferredDeactivationCleanupStack;
717 for (size_t I = Stack.size(); I > OldDeactivateCleanupStackSize; I--) {
718 CGF.DeactivateCleanupBlock(Cleanup: Stack[I - 1].Cleanup,
719 DominatingIP: Stack[I - 1].DominatingIP);
720 Stack[I - 1].DominatingIP->eraseFromParent();
721 }
722 Stack.resize(N: OldDeactivateCleanupStackSize);
723 Deactivated = true;
724 }
725
726 ~CleanupDeactivationScope() {
727 if (Deactivated)
728 return;
729 ForceDeactivate();
730 }
731 };
732
733 llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack;
734
735 llvm::Instruction *CurrentFuncletPad = nullptr;
736
737 class CallLifetimeEnd final : public EHScopeStack::Cleanup {
738 llvm::Value *Addr;
739
740 public:
741 CallLifetimeEnd(RawAddress addr) : Addr(addr.getPointer()) {}
742
743 void Emit(CodeGenFunction &CGF, Flags flags) override {
744 CGF.EmitLifetimeEnd(Addr);
745 }
746 };
747
748 // We are using objects of this 'cleanup' class to emit fake.use calls
749 // for -fextend-variable-liveness. They are placed at the end of a variable's
750 // scope analogous to lifetime markers.
751 class FakeUse final : public EHScopeStack::Cleanup {
752 Address Addr;
753
754 public:
755 FakeUse(Address addr) : Addr(addr) {}
756
757 void Emit(CodeGenFunction &CGF, Flags flags) override {
758 CGF.EmitFakeUse(Addr);
759 }
760 };
761
762 /// Header for data within LifetimeExtendedCleanupStack.
763 struct alignas(uint64_t) LifetimeExtendedCleanupHeader {
764 /// The size of the following cleanup object.
765 unsigned Size;
766 /// The kind of cleanup to push.
767 LLVM_PREFERRED_TYPE(CleanupKind)
768 unsigned Kind : 31;
769 /// Whether this is a conditional cleanup.
770 LLVM_PREFERRED_TYPE(bool)
771 unsigned IsConditional : 1;
772
773 size_t getSize() const { return Size; }
774 CleanupKind getKind() const { return (CleanupKind)Kind; }
775 bool isConditional() const { return IsConditional; }
776 };
777
778 /// i32s containing the indexes of the cleanup destinations.
779 RawAddress NormalCleanupDest = RawAddress::invalid();
780
781 unsigned NextCleanupDestIndex = 1;
782
783 /// EHResumeBlock - Unified block containing a call to llvm.eh.resume.
784 llvm::BasicBlock *EHResumeBlock = nullptr;
785
786 /// The exception slot. All landing pads write the current exception pointer
787 /// into this alloca.
788 llvm::Value *ExceptionSlot = nullptr;
789
790 /// The selector slot. Under the MandatoryCleanup model, all landing pads
791 /// write the current selector value into this alloca.
792 llvm::AllocaInst *EHSelectorSlot = nullptr;
793
794 /// A stack of exception code slots. Entering an __except block pushes a slot
795 /// on the stack and leaving pops one. The __exception_code() intrinsic loads
796 /// a value from the top of the stack.
797 SmallVector<Address, 1> SEHCodeSlotStack;
798
799 /// Value returned by __exception_info intrinsic.
800 llvm::Value *SEHInfo = nullptr;
801
802 /// Emits a landing pad for the current EH stack.
803 llvm::BasicBlock *EmitLandingPad();
804
805 llvm::BasicBlock *getInvokeDestImpl();
806
807 /// Parent loop-based directive for scan directive.
808 const OMPExecutableDirective *OMPParentLoopDirectiveForScan = nullptr;
809 llvm::BasicBlock *OMPBeforeScanBlock = nullptr;
810 llvm::BasicBlock *OMPAfterScanBlock = nullptr;
811 llvm::BasicBlock *OMPScanExitBlock = nullptr;
812 llvm::BasicBlock *OMPScanDispatch = nullptr;
813 bool OMPFirstScanLoop = false;
814
815 /// Manages parent directive for scan directives.
816 class ParentLoopDirectiveForScanRegion {
817 CodeGenFunction &CGF;
818 const OMPExecutableDirective *ParentLoopDirectiveForScan;
819
820 public:
821 ParentLoopDirectiveForScanRegion(
822 CodeGenFunction &CGF,
823 const OMPExecutableDirective &ParentLoopDirectiveForScan)
824 : CGF(CGF),
825 ParentLoopDirectiveForScan(CGF.OMPParentLoopDirectiveForScan) {
826 CGF.OMPParentLoopDirectiveForScan = &ParentLoopDirectiveForScan;
827 }
828 ~ParentLoopDirectiveForScanRegion() {
829 CGF.OMPParentLoopDirectiveForScan = ParentLoopDirectiveForScan;
830 }
831 };
832
833 template <class T>
834 typename DominatingValue<T>::saved_type saveValueInCond(T value) {
835 return DominatingValue<T>::save(*this, value);
836 }
837
838 class CGFPOptionsRAII {
839 public:
840 CGFPOptionsRAII(CodeGenFunction &CGF, FPOptions FPFeatures);
841 CGFPOptionsRAII(CodeGenFunction &CGF, const Expr *E);
842 ~CGFPOptionsRAII();
843
844 private:
845 void ConstructorHelper(FPOptions FPFeatures);
846 CodeGenFunction &CGF;
847 FPOptions OldFPFeatures;
848 llvm::fp::ExceptionBehavior OldExcept;
849 llvm::RoundingMode OldRounding;
850 std::optional<CGBuilderTy::FastMathFlagGuard> FMFGuard;
851 };
852 FPOptions CurFPFeatures;
853
854 class CGAtomicOptionsRAII {
855 public:
856 CGAtomicOptionsRAII(CodeGenModule &CGM_, AtomicOptions AO)
857 : CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {
858 CGM.setAtomicOpts(AO);
859 }
860 CGAtomicOptionsRAII(CodeGenModule &CGM_, const AtomicAttr *AA)
861 : CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {
862 if (!AA)
863 return;
864 AtomicOptions AO = SavedAtomicOpts;
865 for (auto Option : AA->atomicOptions()) {
866 switch (Option) {
867 case AtomicAttr::remote_memory:
868 AO.remote_memory = true;
869 break;
870 case AtomicAttr::no_remote_memory:
871 AO.remote_memory = false;
872 break;
873 case AtomicAttr::fine_grained_memory:
874 AO.fine_grained_memory = true;
875 break;
876 case AtomicAttr::no_fine_grained_memory:
877 AO.fine_grained_memory = false;
878 break;
879 case AtomicAttr::ignore_denormal_mode:
880 AO.ignore_denormal_mode = true;
881 break;
882 case AtomicAttr::no_ignore_denormal_mode:
883 AO.ignore_denormal_mode = false;
884 break;
885 }
886 }
887 CGM.setAtomicOpts(AO);
888 }
889
890 CGAtomicOptionsRAII(const CGAtomicOptionsRAII &) = delete;
891 CGAtomicOptionsRAII &operator=(const CGAtomicOptionsRAII &) = delete;
892 ~CGAtomicOptionsRAII() { CGM.setAtomicOpts(SavedAtomicOpts); }
893
894 private:
895 CodeGenModule &CGM;
896 AtomicOptions SavedAtomicOpts;
897 };
898
899public:
900 /// ObjCEHValueStack - Stack of Objective-C exception values, used for
901 /// rethrows.
902 SmallVector<llvm::Value *, 8> ObjCEHValueStack;
903
904 /// A class controlling the emission of a finally block.
905 class FinallyInfo {
906 /// Where the catchall's edge through the cleanup should go.
907 JumpDest RethrowDest;
908
909 /// A function to call to enter the catch.
910 llvm::FunctionCallee BeginCatchFn;
911
912 /// An i1 variable indicating whether or not the @finally is
913 /// running for an exception.
914 llvm::AllocaInst *ForEHVar = nullptr;
915
916 /// An i8* variable into which the exception pointer to rethrow
917 /// has been saved.
918 llvm::AllocaInst *SavedExnVar = nullptr;
919
920 public:
921 void enter(CodeGenFunction &CGF, const Stmt *Finally,
922 llvm::FunctionCallee beginCatchFn,
923 llvm::FunctionCallee endCatchFn, llvm::FunctionCallee rethrowFn);
924 void exit(CodeGenFunction &CGF);
925 };
926
927 /// Returns true inside SEH __try blocks.
928 bool isSEHTryScope() const { return !SEHTryEpilogueStack.empty(); }
929
930 /// Returns true while emitting a cleanuppad.
931 bool isCleanupPadScope() const {
932 return CurrentFuncletPad && isa<llvm::CleanupPadInst>(Val: CurrentFuncletPad);
933 }
934
935 /// pushFullExprCleanup - Push a cleanup to be run at the end of the
936 /// current full-expression. Safe against the possibility that
937 /// we're currently inside a conditionally-evaluated expression.
938 template <class T, class... As>
939 void pushFullExprCleanup(CleanupKind kind, As... A) {
940 // If we're not in a conditional branch, or if none of the
941 // arguments requires saving, then use the unconditional cleanup.
942 if (!isInConditionalBranch())
943 return EHStack.pushCleanup<T>(kind, A...);
944
945 // Stash values in a tuple so we can guarantee the order of saves.
946 typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
947 SavedTuple Saved{saveValueInCond(A)...};
948
949 typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
950 EHStack.pushCleanupTuple<CleanupType>(kind, Saved);
951 initFullExprCleanup();
952 }
953
954 /// Queue a cleanup to be pushed after finishing the current full-expression,
955 /// potentially with an active flag.
956 template <class T, class... As>
957 void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) {
958 if (!isInConditionalBranch())
959 return pushCleanupAfterFullExprWithActiveFlag<T>(
960 Kind, RawAddress::invalid(), A...);
961
962 RawAddress ActiveFlag = createCleanupActiveFlag();
963 assert(!DominatingValue<Address>::needsSaving(ActiveFlag) &&
964 "cleanup active flag should never need saving");
965
966 typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
967 SavedTuple Saved{saveValueInCond(A)...};
968
969 typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
970 pushCleanupAfterFullExprWithActiveFlag<CleanupType>(Kind, ActiveFlag,
971 Saved);
972 }
973
974 template <class T, class... As>
975 void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind,
976 RawAddress ActiveFlag, As... A) {
977 LifetimeExtendedCleanupHeader Header = {.Size: sizeof(T), .Kind: Kind,
978 .IsConditional: ActiveFlag.isValid()};
979
980 size_t OldSize = LifetimeExtendedCleanupStack.size();
981 LifetimeExtendedCleanupStack.resize(
982 N: LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size +
983 (Header.IsConditional ? sizeof(ActiveFlag) : 0));
984
985 static_assert((alignof(LifetimeExtendedCleanupHeader) == alignof(T)) &&
986 (alignof(T) == alignof(RawAddress)),
987 "Cleanup will be allocated on misaligned address");
988 char *Buffer = &LifetimeExtendedCleanupStack[OldSize];
989 new (Buffer) LifetimeExtendedCleanupHeader(Header);
990 new (Buffer + sizeof(Header)) T(A...);
991 if (Header.IsConditional)
992 new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag);
993 }
994
995 // Push a cleanup onto EHStack and deactivate it later. It is usually
996 // deactivated when exiting a `CleanupDeactivationScope` (for example: after a
997 // full expression).
998 template <class T, class... As>
999 void pushCleanupAndDeferDeactivation(CleanupKind Kind, As... A) {
1000 // Placeholder dominating IP for this cleanup.
1001 llvm::Instruction *DominatingIP =
1002 Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
1003 EHStack.pushCleanup<T>(Kind, A...);
1004 DeferredDeactivationCleanupStack.push_back(
1005 Elt: {.Cleanup: EHStack.stable_begin(), .DominatingIP: DominatingIP});
1006 }
1007
1008 /// Set up the last cleanup that was pushed as a conditional
1009 /// full-expression cleanup.
1010 void initFullExprCleanup() {
1011 initFullExprCleanupWithFlag(ActiveFlag: createCleanupActiveFlag());
1012 }
1013
1014 void initFullExprCleanupWithFlag(RawAddress ActiveFlag);
1015 RawAddress createCleanupActiveFlag();
1016
1017 /// PushDestructorCleanup - Push a cleanup to call the
1018 /// complete-object destructor of an object of the given type at the
1019 /// given address. Does nothing if T is not a C++ class type with a
1020 /// non-trivial destructor.
1021 void PushDestructorCleanup(QualType T, Address Addr);
1022
1023 /// PushDestructorCleanup - Push a cleanup to call the
1024 /// complete-object variant of the given destructor on the object at
1025 /// the given address.
1026 void PushDestructorCleanup(const CXXDestructorDecl *Dtor, QualType T,
1027 Address Addr);
1028
1029 /// PopCleanupBlock - Will pop the cleanup entry on the stack and
1030 /// process all branch fixups.
1031 void PopCleanupBlock(bool FallThroughIsBranchThrough = false,
1032 bool ForDeactivation = false);
1033
1034 /// DeactivateCleanupBlock - Deactivates the given cleanup block.
1035 /// The block cannot be reactivated. Pops it if it's the top of the
1036 /// stack.
1037 ///
1038 /// \param DominatingIP - An instruction which is known to
1039 /// dominate the current IP (if set) and which lies along
1040 /// all paths of execution between the current IP and the
1041 /// the point at which the cleanup comes into scope.
1042 void DeactivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
1043 llvm::Instruction *DominatingIP);
1044
1045 /// ActivateCleanupBlock - Activates an initially-inactive cleanup.
1046 /// Cannot be used to resurrect a deactivated cleanup.
1047 ///
1048 /// \param DominatingIP - An instruction which is known to
1049 /// dominate the current IP (if set) and which lies along
1050 /// all paths of execution between the current IP and the
1051 /// the point at which the cleanup comes into scope.
1052 void ActivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
1053 llvm::Instruction *DominatingIP);
1054
1055 /// Enters a new scope for capturing cleanups, all of which
1056 /// will be executed once the scope is exited.
1057 class RunCleanupsScope {
1058 EHScopeStack::stable_iterator CleanupStackDepth, OldCleanupScopeDepth;
1059 size_t LifetimeExtendedCleanupStackSize;
1060 CleanupDeactivationScope DeactivateCleanups;
1061 bool OldDidCallStackSave;
1062
1063 protected:
1064 bool PerformCleanup;
1065
1066 private:
1067 RunCleanupsScope(const RunCleanupsScope &) = delete;
1068 void operator=(const RunCleanupsScope &) = delete;
1069
1070 protected:
1071 CodeGenFunction &CGF;
1072
1073 public:
1074 /// Enter a new cleanup scope.
1075 explicit RunCleanupsScope(CodeGenFunction &CGF)
1076 : DeactivateCleanups(CGF), PerformCleanup(true), CGF(CGF) {
1077 CleanupStackDepth = CGF.EHStack.stable_begin();
1078 LifetimeExtendedCleanupStackSize =
1079 CGF.LifetimeExtendedCleanupStack.size();
1080 OldDidCallStackSave = CGF.DidCallStackSave;
1081 CGF.DidCallStackSave = false;
1082 OldCleanupScopeDepth = CGF.CurrentCleanupScopeDepth;
1083 CGF.CurrentCleanupScopeDepth = CleanupStackDepth;
1084 }
1085
1086 /// Exit this cleanup scope, emitting any accumulated cleanups.
1087 ~RunCleanupsScope() {
1088 if (PerformCleanup)
1089 ForceCleanup();
1090 }
1091
1092 /// Determine whether this scope requires any cleanups.
1093 bool requiresCleanups() const {
1094 return CGF.EHStack.stable_begin() != CleanupStackDepth;
1095 }
1096
1097 /// Force the emission of cleanups now, instead of waiting
1098 /// until this object is destroyed.
1099 /// \param ValuesToReload - A list of values that need to be available at
1100 /// the insertion point after cleanup emission. If cleanup emission created
1101 /// a shared cleanup block, these value pointers will be rewritten.
1102 /// Otherwise, they not will be modified.
1103 void
1104 ForceCleanup(std::initializer_list<llvm::Value **> ValuesToReload = {}) {
1105 assert(PerformCleanup && "Already forced cleanup");
1106 CGF.DidCallStackSave = OldDidCallStackSave;
1107 DeactivateCleanups.ForceDeactivate();
1108 CGF.PopCleanupBlocks(OldCleanupStackSize: CleanupStackDepth, OldLifetimeExtendedStackSize: LifetimeExtendedCleanupStackSize,
1109 ValuesToReload);
1110 PerformCleanup = false;
1111 CGF.CurrentCleanupScopeDepth = OldCleanupScopeDepth;
1112 }
1113 };
1114
1115 // Cleanup stack depth of the RunCleanupsScope that was pushed most recently.
1116 EHScopeStack::stable_iterator CurrentCleanupScopeDepth =
1117 EHScopeStack::stable_end();
1118
1119 class LexicalScope : public RunCleanupsScope {
1120 SourceRange Range;
1121 SmallVector<const LabelDecl *, 4> Labels;
1122 LexicalScope *ParentScope;
1123
1124 LexicalScope(const LexicalScope &) = delete;
1125 void operator=(const LexicalScope &) = delete;
1126
1127 public:
1128 /// Enter a new cleanup scope.
1129 explicit LexicalScope(CodeGenFunction &CGF, SourceRange Range);
1130
1131 void addLabel(const LabelDecl *label) {
1132 assert(PerformCleanup && "adding label to dead scope?");
1133 Labels.push_back(Elt: label);
1134 }
1135
1136 /// Exit this cleanup scope, emitting any accumulated
1137 /// cleanups.
1138 ~LexicalScope();
1139
1140 /// Force the emission of cleanups now, instead of waiting
1141 /// until this object is destroyed.
1142 void ForceCleanup() {
1143 CGF.CurLexicalScope = ParentScope;
1144 RunCleanupsScope::ForceCleanup();
1145
1146 if (!Labels.empty())
1147 rescopeLabels();
1148 }
1149
1150 bool hasLabels() const { return !Labels.empty(); }
1151
1152 void rescopeLabels();
1153 };
1154
1155 typedef llvm::DenseMap<const Decl *, Address> DeclMapTy;
1156
1157 /// The class used to assign some variables some temporarily addresses.
1158 class OMPMapVars {
1159 DeclMapTy SavedLocals;
1160 DeclMapTy SavedTempAddresses;
1161 OMPMapVars(const OMPMapVars &) = delete;
1162 void operator=(const OMPMapVars &) = delete;
1163
1164 public:
1165 explicit OMPMapVars() = default;
1166 ~OMPMapVars() {
1167 assert(SavedLocals.empty() && "Did not restored original addresses.");
1168 };
1169
1170 /// Sets the address of the variable \p LocalVD to be \p TempAddr in
1171 /// function \p CGF.
1172 /// \return true if at least one variable was set already, false otherwise.
1173 bool setVarAddr(CodeGenFunction &CGF, const ValueDecl *LocalVD,
1174 Address TempAddr) {
1175 LocalVD = cast<ValueDecl>(Val: LocalVD->getCanonicalDecl());
1176
1177 // Only save it once.
1178 if (SavedLocals.count(Val: LocalVD))
1179 return false;
1180
1181 // Copy the existing local entry to SavedLocals.
1182 auto it = CGF.LocalDeclMap.find(Val: LocalVD);
1183 if (it != CGF.LocalDeclMap.end())
1184 SavedLocals.try_emplace(Key: LocalVD, Args&: it->second);
1185 else
1186 SavedLocals.try_emplace(Key: LocalVD, Args: Address::invalid());
1187
1188 // Generate the private entry.
1189 QualType VarTy = LocalVD->getType();
1190 if (VarTy->isReferenceType()) {
1191 Address Temp = CGF.CreateMemTemp(T: VarTy);
1192 CGF.Builder.CreateStore(Val: TempAddr.emitRawPointer(CGF), Addr: Temp);
1193 TempAddr = Temp;
1194 }
1195 if (const auto *BD = dyn_cast<BindingDecl>(Val: LocalVD))
1196 CGF.OMPPrivatizedBindings.insert_or_assign(Key: BD, Val&: TempAddr);
1197 SavedTempAddresses.try_emplace(Key: LocalVD, Args&: TempAddr);
1198
1199 return true;
1200 }
1201
1202 /// Applies new addresses to the list of the variables.
1203 /// \return true if at least one variable is using new address, false
1204 /// otherwise.
1205 bool apply(CodeGenFunction &CGF) {
1206 copyInto(Src: SavedTempAddresses, Dest&: CGF.LocalDeclMap);
1207 SavedTempAddresses.clear();
1208 return !SavedLocals.empty();
1209 }
1210
1211 /// Restores original addresses of the variables.
1212 void restore(CodeGenFunction &CGF) {
1213 if (!SavedLocals.empty()) {
1214 copyInto(Src: SavedLocals, Dest&: CGF.LocalDeclMap);
1215 SavedLocals.clear();
1216 }
1217 }
1218
1219 private:
1220 /// Copy all the entries in the source map over the corresponding
1221 /// entries in the destination, which must exist.
1222 static void copyInto(const DeclMapTy &Src, DeclMapTy &Dest) {
1223 for (auto &[Decl, Addr] : Src) {
1224 if (!Addr.isValid())
1225 Dest.erase(Val: Decl);
1226 else
1227 Dest.insert_or_assign(Key: Decl, Val: Addr);
1228 }
1229 }
1230 };
1231
1232 /// The scope used to remap some variables as private in the OpenMP loop body
1233 /// (or other captured region emitted without outlining), and to restore old
1234 /// vars back on exit.
1235 class OMPPrivateScope : public RunCleanupsScope {
1236 OMPMapVars MappedVars;
1237 OMPPrivateScope(const OMPPrivateScope &) = delete;
1238 void operator=(const OMPPrivateScope &) = delete;
1239 llvm::DenseMap<const BindingDecl *, Address> BindingChanges;
1240
1241 public:
1242 /// Enter a new OpenMP private scope.
1243 explicit OMPPrivateScope(CodeGenFunction &CGF) : RunCleanupsScope(CGF) {}
1244
1245 /// Registers \p LocalVD variable as a private with \p Addr as the address
1246 /// of the corresponding private variable. \p
1247 /// PrivateGen is the address of the generated private variable.
1248 /// \return true if the variable is registered as private, false if it has
1249 /// been privatized already.
1250 bool addPrivate(const ValueDecl *LocalVD, Address Addr) {
1251 assert(PerformCleanup && "adding private to dead scope");
1252 if (const auto *BD = dyn_cast<BindingDecl>(Val: LocalVD->getCanonicalDecl())) {
1253 auto It = CGF.OMPPrivatizedBindings.find(Val: BD);
1254 BindingChanges.insert(KV: {BD, It != CGF.OMPPrivatizedBindings.end()
1255 ? It->second
1256 : Address::invalid()});
1257 }
1258 return MappedVars.setVarAddr(CGF, LocalVD, TempAddr: Addr);
1259 }
1260
1261 /// Privatizes local variables previously registered as private.
1262 /// Registration is separate from the actual privatization to allow
1263 /// initializers use values of the original variables, not the private one.
1264 /// This is important, for example, if the private variable is a class
1265 /// variable initialized by a constructor that references other private
1266 /// variables. But at initialization original variables must be used, not
1267 /// private copies.
1268 /// \return true if at least one variable was privatized, false otherwise.
1269 bool Privatize() { return MappedVars.apply(CGF); }
1270
1271 void ForceCleanup() {
1272 RunCleanupsScope::ForceCleanup();
1273 restoreMap();
1274 }
1275
1276 /// Exit scope - all the mapped variables are restored.
1277 ~OMPPrivateScope() {
1278 if (PerformCleanup)
1279 ForceCleanup();
1280 for (auto &Change : BindingChanges) {
1281 if (Change.second.isValid()) {
1282 auto It = CGF.OMPPrivatizedBindings.find(Val: Change.first);
1283 if (It != CGF.OMPPrivatizedBindings.end())
1284 It->second = Change.second;
1285 else
1286 CGF.OMPPrivatizedBindings.insert(KV: {Change.first, Change.second});
1287 } else {
1288 CGF.OMPPrivatizedBindings.erase(Val: Change.first);
1289 }
1290 }
1291 }
1292
1293 /// Checks if the global variable is captured in current function.
1294 bool isGlobalVarCaptured(const VarDecl *VD) const {
1295 VD = VD->getCanonicalDecl();
1296 return !VD->isLocalVarDeclOrParm() && CGF.LocalDeclMap.count(Val: VD) > 0;
1297 }
1298
1299 /// Restore all mapped variables w/o clean up. This is usefully when we want
1300 /// to reference the original variables but don't want the clean up because
1301 /// that could emit lifetime end too early, causing backend issue #56913.
1302 void restoreMap() { MappedVars.restore(CGF); }
1303 };
1304
1305 /// Save/restore original map of previously emitted local vars in case when we
1306 /// need to duplicate emission of the same code several times in the same
1307 /// function for OpenMP code.
1308 class OMPLocalDeclMapRAII {
1309 CodeGenFunction &CGF;
1310 DeclMapTy SavedMap;
1311
1312 public:
1313 OMPLocalDeclMapRAII(CodeGenFunction &CGF)
1314 : CGF(CGF), SavedMap(CGF.LocalDeclMap) {}
1315 ~OMPLocalDeclMapRAII() { SavedMap.swap(RHS&: CGF.LocalDeclMap); }
1316 };
1317
1318 /// Takes the old cleanup stack size and emits the cleanup blocks
1319 /// that have been added.
1320 void
1321 PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1322 std::initializer_list<llvm::Value **> ValuesToReload = {});
1323
1324 /// Takes the old cleanup stack size and emits the cleanup blocks
1325 /// that have been added, then adds all lifetime-extended cleanups from
1326 /// the given position to the stack.
1327 void
1328 PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1329 size_t OldLifetimeExtendedStackSize,
1330 std::initializer_list<llvm::Value **> ValuesToReload = {});
1331
1332 void ResolveBranchFixups(llvm::BasicBlock *Target);
1333
1334 /// The given basic block lies in the current EH scope, but may be a
1335 /// target of a potentially scope-crossing jump; get a stable handle
1336 /// to which we can perform this jump later.
1337 JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target) {
1338 return JumpDest(Target, EHStack.getInnermostNormalCleanup(),
1339 NextCleanupDestIndex++);
1340 }
1341
1342 /// The given basic block lies in the current EH scope, but may be a
1343 /// target of a potentially scope-crossing jump; get a stable handle
1344 /// to which we can perform this jump later.
1345 JumpDest getJumpDestInCurrentScope(StringRef Name = StringRef()) {
1346 return getJumpDestInCurrentScope(Target: createBasicBlock(name: Name));
1347 }
1348
1349 /// EmitBranchThroughCleanup - Emit a branch from the current insert
1350 /// block through the normal cleanup handling code (if any) and then
1351 /// on to \arg Dest.
1352 void EmitBranchThroughCleanup(JumpDest Dest);
1353
1354 /// isObviouslyBranchWithoutCleanups - Return true if a branch to the
1355 /// specified destination obviously has no cleanups to run. 'false' is always
1356 /// a conservatively correct answer for this method.
1357 bool isObviouslyBranchWithoutCleanups(JumpDest Dest) const;
1358
1359 /// popCatchScope - Pops the catch scope at the top of the EHScope
1360 /// stack, emitting any required code (other than the catch handlers
1361 /// themselves).
1362 void popCatchScope();
1363
1364 // This function should be called after emitting all catch clauses and none
1365 // of them were 'catch-all' clauses.
1366 // Because in wasm we merge all catch clauses into one big catchpad, in case
1367 // none of the types in catch handlers matches after we test against each of
1368 // them, we should unwind to the next EH enclosing scope. We generate a call
1369 // to rethrow function here to do that.
1370 void WasmEmitFallthroughRethrow(llvm::BasicBlock *WasmCatchStartBlock);
1371
1372 llvm::BasicBlock *getEHResumeBlock(bool isCleanup);
1373 llvm::BasicBlock *getEHDispatchBlock(EHScopeStack::stable_iterator scope);
1374 llvm::BasicBlock *
1375 getFuncletEHDispatchBlock(EHScopeStack::stable_iterator scope);
1376
1377 /// An object to manage conditionally-evaluated expressions.
1378 class ConditionalEvaluation {
1379 llvm::BasicBlock *StartBB;
1380
1381 public:
1382 ConditionalEvaluation(CodeGenFunction &CGF)
1383 : StartBB(CGF.Builder.GetInsertBlock()) {}
1384
1385 void begin(CodeGenFunction &CGF) {
1386 assert(CGF.OutermostConditional != this);
1387 if (!CGF.OutermostConditional)
1388 CGF.OutermostConditional = this;
1389 }
1390
1391 void end(CodeGenFunction &CGF) {
1392 assert(CGF.OutermostConditional != nullptr);
1393 if (CGF.OutermostConditional == this)
1394 CGF.OutermostConditional = nullptr;
1395 }
1396
1397 /// Returns a block which will be executed prior to each
1398 /// evaluation of the conditional code.
1399 llvm::BasicBlock *getStartingBlock() const { return StartBB; }
1400 };
1401
1402 /// isInConditionalBranch - Return true if we're currently emitting
1403 /// one branch or the other of a conditional expression.
1404 bool isInConditionalBranch() const { return OutermostConditional != nullptr; }
1405
1406 void setBeforeOutermostConditional(llvm::Value *value, Address addr,
1407 CodeGenFunction &CGF) {
1408 assert(isInConditionalBranch());
1409 llvm::BasicBlock *block = OutermostConditional->getStartingBlock();
1410 auto store = new llvm::StoreInst(value, addr.emitRawPointer(CGF),
1411 block->back().getIterator());
1412 store->setAlignment(addr.getAlignment().getAsAlign());
1413 }
1414
1415 /// An RAII object to record that we're evaluating a statement
1416 /// expression.
1417 class StmtExprEvaluation {
1418 CodeGenFunction &CGF;
1419
1420 /// We have to save the outermost conditional: cleanups in a
1421 /// statement expression aren't conditional just because the
1422 /// StmtExpr is.
1423 ConditionalEvaluation *SavedOutermostConditional;
1424
1425 public:
1426 StmtExprEvaluation(CodeGenFunction &CGF)
1427 : CGF(CGF), SavedOutermostConditional(CGF.OutermostConditional) {
1428 CGF.OutermostConditional = nullptr;
1429 }
1430
1431 ~StmtExprEvaluation() {
1432 CGF.OutermostConditional = SavedOutermostConditional;
1433 CGF.EnsureInsertPoint();
1434 }
1435 };
1436
1437 /// An object which temporarily prevents a value from being
1438 /// destroyed by aggressive peephole optimizations that assume that
1439 /// all uses of a value have been realized in the IR.
1440 class PeepholeProtection {
1441 llvm::Instruction *Inst = nullptr;
1442 friend class CodeGenFunction;
1443
1444 public:
1445 PeepholeProtection() = default;
1446 };
1447
1448 /// A non-RAII class containing all the information about a bound
1449 /// opaque value. OpaqueValueMapping, below, is a RAII wrapper for
1450 /// this which makes individual mappings very simple; using this
1451 /// class directly is useful when you have a variable number of
1452 /// opaque values or don't want the RAII functionality for some
1453 /// reason.
1454 class OpaqueValueMappingData {
1455 const OpaqueValueExpr *OpaqueValue;
1456 bool BoundLValue;
1457 CodeGenFunction::PeepholeProtection Protection;
1458
1459 OpaqueValueMappingData(const OpaqueValueExpr *ov, bool boundLValue)
1460 : OpaqueValue(ov), BoundLValue(boundLValue) {}
1461
1462 public:
1463 OpaqueValueMappingData() : OpaqueValue(nullptr) {}
1464
1465 static bool shouldBindAsLValue(const Expr *expr) {
1466 // gl-values should be bound as l-values for obvious reasons.
1467 // Records should be bound as l-values because IR generation
1468 // always keeps them in memory. Expressions of function type
1469 // act exactly like l-values but are formally required to be
1470 // r-values in C.
1471 return expr->isGLValue() || expr->getType()->isFunctionType() ||
1472 hasAggregateEvaluationKind(T: expr->getType());
1473 }
1474
1475 static OpaqueValueMappingData
1476 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const Expr *e) {
1477 if (shouldBindAsLValue(expr: ov))
1478 return bind(CGF, ov, lv: CGF.EmitLValue(E: e));
1479 return bind(CGF, ov, rv: CGF.EmitAnyExpr(E: e));
1480 }
1481
1482 static OpaqueValueMappingData
1483 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const LValue &lv) {
1484 assert(shouldBindAsLValue(ov));
1485 CGF.OpaqueLValues.insert(KV: std::make_pair(x&: ov, y: lv));
1486 return OpaqueValueMappingData(ov, true);
1487 }
1488
1489 static OpaqueValueMappingData
1490 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const RValue &rv) {
1491 assert(!shouldBindAsLValue(ov));
1492 CGF.OpaqueRValues.insert(KV: std::make_pair(x&: ov, y: rv));
1493
1494 OpaqueValueMappingData data(ov, false);
1495
1496 // Work around an extremely aggressive peephole optimization in
1497 // EmitScalarConversion which assumes that all other uses of a
1498 // value are extant.
1499 data.Protection = CGF.protectFromPeepholes(rvalue: rv);
1500
1501 return data;
1502 }
1503
1504 bool isValid() const { return OpaqueValue != nullptr; }
1505 void clear() { OpaqueValue = nullptr; }
1506
1507 void unbind(CodeGenFunction &CGF) {
1508 assert(OpaqueValue && "no data to unbind!");
1509
1510 if (BoundLValue) {
1511 CGF.OpaqueLValues.erase(Val: OpaqueValue);
1512 } else {
1513 CGF.OpaqueRValues.erase(Val: OpaqueValue);
1514 CGF.unprotectFromPeepholes(protection: Protection);
1515 }
1516 }
1517 };
1518
1519 /// An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
1520 class OpaqueValueMapping {
1521 CodeGenFunction &CGF;
1522 OpaqueValueMappingData Data;
1523
1524 public:
1525 static bool shouldBindAsLValue(const Expr *expr) {
1526 return OpaqueValueMappingData::shouldBindAsLValue(expr);
1527 }
1528
1529 /// Build the opaque value mapping for the given conditional
1530 /// operator if it's the GNU ?: extension. This is a common
1531 /// enough pattern that the convenience operator is really
1532 /// helpful.
1533 ///
1534 OpaqueValueMapping(CodeGenFunction &CGF,
1535 const AbstractConditionalOperator *op)
1536 : CGF(CGF) {
1537 if (isa<ConditionalOperator>(Val: op))
1538 // Leave Data empty.
1539 return;
1540
1541 const BinaryConditionalOperator *e = cast<BinaryConditionalOperator>(Val: op);
1542 Data = OpaqueValueMappingData::bind(CGF, ov: e->getOpaqueValue(),
1543 e: e->getCommon());
1544 }
1545
1546 /// Build the opaque value mapping for an OpaqueValueExpr whose source
1547 /// expression is set to the expression the OVE represents.
1548 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *OV)
1549 : CGF(CGF) {
1550 if (OV) {
1551 assert(OV->getSourceExpr() && "wrong form of OpaqueValueMapping used "
1552 "for OVE with no source expression");
1553 Data = OpaqueValueMappingData::bind(CGF, ov: OV, e: OV->getSourceExpr());
1554 }
1555 }
1556
1557 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,
1558 LValue lvalue)
1559 : CGF(CGF),
1560 Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, lv: lvalue)) {}
1561
1562 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,
1563 RValue rvalue)
1564 : CGF(CGF),
1565 Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, rv: rvalue)) {}
1566
1567 void pop() {
1568 Data.unbind(CGF);
1569 Data.clear();
1570 }
1571
1572 ~OpaqueValueMapping() {
1573 if (Data.isValid())
1574 Data.unbind(CGF);
1575 }
1576 };
1577
1578private:
1579 CGDebugInfo *DebugInfo;
1580 /// Used to create unique names for artificial VLA size debug info variables.
1581 unsigned VLAExprCounter = 0;
1582 bool DisableDebugInfo = false;
1583
1584 /// DidCallStackSave - Whether llvm.stacksave has been called. Used to avoid
1585 /// calling llvm.stacksave for multiple VLAs in the same scope.
1586 bool DidCallStackSave = false;
1587
1588 /// IndirectBranch - The first time an indirect goto is seen we create a block
1589 /// with an indirect branch. Every time we see the address of a label taken,
1590 /// we add the label to the indirect goto. Every subsequent indirect goto is
1591 /// codegen'd as a jump to the IndirectBranch's basic block.
1592 llvm::IndirectBrInst *IndirectBranch = nullptr;
1593
1594 /// LocalDeclMap - This keeps track of the LLVM allocas or globals for local C
1595 /// decls.
1596 DeclMapTy LocalDeclMap;
1597
1598 /// Lookup map for privatized BindingDecls.
1599 /// Used when BindingDecls are remapped during OpenMP outlining, since the
1600 /// remapped BindingDecl has a different pointer than the original.
1601 llvm::SmallDenseMap<const BindingDecl *, Address> OMPPrivatizedBindings;
1602
1603 // Keep track of the cleanups for callee-destructed parameters pushed to the
1604 // cleanup stack so that they can be deactivated later.
1605 llvm::DenseMap<const ParmVarDecl *, EHScopeStack::stable_iterator>
1606 CalleeDestructedParamCleanups;
1607
1608 /// SizeArguments - If a ParmVarDecl had the pass_object_size attribute, this
1609 /// will contain a mapping from said ParmVarDecl to its implicit "object_size"
1610 /// parameter.
1611 llvm::SmallDenseMap<const ParmVarDecl *, const ImplicitParamDecl *, 2>
1612 SizeArguments;
1613
1614 /// Track escaped local variables with auto storage. Used during SEH
1615 /// outlining to produce a call to llvm.localescape.
1616 llvm::DenseMap<llvm::AllocaInst *, int> EscapedLocals;
1617
1618 /// LabelMap - This keeps track of the LLVM basic block for each C label.
1619 llvm::DenseMap<const LabelDecl *, JumpDest> LabelMap;
1620
1621 // BreakContinueStack - This keeps track of where break and continue
1622 // statements should jump to.
1623 struct BreakContinue {
1624 BreakContinue(const Stmt &LoopOrSwitch, JumpDest Break, JumpDest Continue)
1625 : LoopOrSwitch(&LoopOrSwitch), BreakBlock(Break),
1626 ContinueBlock(Continue) {}
1627
1628 const Stmt *LoopOrSwitch;
1629 JumpDest BreakBlock;
1630 JumpDest ContinueBlock;
1631 };
1632 SmallVector<BreakContinue, 8> BreakContinueStack;
1633
1634 /// Handles cancellation exit points in OpenMP-related constructs.
1635 class OpenMPCancelExitStack {
1636 /// Tracks cancellation exit point and join point for cancel-related exit
1637 /// and normal exit.
1638 struct CancelExit {
1639 CancelExit() = default;
1640 CancelExit(OpenMPDirectiveKind Kind, JumpDest ExitBlock,
1641 JumpDest ContBlock)
1642 : Kind(Kind), ExitBlock(ExitBlock), ContBlock(ContBlock) {}
1643 OpenMPDirectiveKind Kind = llvm::omp::OMPD_unknown;
1644 /// true if the exit block has been emitted already by the special
1645 /// emitExit() call, false if the default codegen is used.
1646 bool HasBeenEmitted = false;
1647 JumpDest ExitBlock;
1648 JumpDest ContBlock;
1649 };
1650
1651 SmallVector<CancelExit, 8> Stack;
1652
1653 public:
1654 OpenMPCancelExitStack() : Stack(1) {}
1655 ~OpenMPCancelExitStack() = default;
1656 /// Fetches the exit block for the current OpenMP construct.
1657 JumpDest getExitBlock() const { return Stack.back().ExitBlock; }
1658 /// Emits exit block with special codegen procedure specific for the related
1659 /// OpenMP construct + emits code for normal construct cleanup.
1660 void emitExit(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
1661 const llvm::function_ref<void(CodeGenFunction &)> CodeGen) {
1662 if (Stack.back().Kind == Kind && getExitBlock().isValid()) {
1663 assert(CGF.getOMPCancelDestination(Kind).isValid());
1664 assert(CGF.HaveInsertPoint());
1665 assert(!Stack.back().HasBeenEmitted);
1666 auto IP = CGF.Builder.saveAndClearIP();
1667 CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1668 CodeGen(CGF);
1669 CGF.EmitBranch(Block: Stack.back().ContBlock.getBlock());
1670 CGF.Builder.restoreIP(IP);
1671 Stack.back().HasBeenEmitted = true;
1672 }
1673 CodeGen(CGF);
1674 }
1675 /// Enter the cancel supporting \a Kind construct.
1676 /// \param Kind OpenMP directive that supports cancel constructs.
1677 /// \param HasCancel true, if the construct has inner cancel directive,
1678 /// false otherwise.
1679 void enter(CodeGenFunction &CGF, OpenMPDirectiveKind Kind, bool HasCancel) {
1680 Stack.push_back(Elt: {Kind,
1681 HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.exit")
1682 : JumpDest(),
1683 HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.cont")
1684 : JumpDest()});
1685 }
1686 /// Emits default exit point for the cancel construct (if the special one
1687 /// has not be used) + join point for cancel/normal exits.
1688 void exit(CodeGenFunction &CGF) {
1689 if (getExitBlock().isValid()) {
1690 assert(CGF.getOMPCancelDestination(Stack.back().Kind).isValid());
1691 bool HaveIP = CGF.HaveInsertPoint();
1692 if (!Stack.back().HasBeenEmitted) {
1693 if (HaveIP)
1694 CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1695 CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1696 CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1697 }
1698 CGF.EmitBlock(BB: Stack.back().ContBlock.getBlock());
1699 if (!HaveIP) {
1700 CGF.Builder.CreateUnreachable();
1701 CGF.Builder.ClearInsertionPoint();
1702 }
1703 }
1704 Stack.pop_back();
1705 }
1706 };
1707 OpenMPCancelExitStack OMPCancelStack;
1708
1709 /// Lower the Likelihood knowledge about the \p Cond via llvm.expect intrin.
1710 llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
1711 Stmt::Likelihood LH);
1712
1713 std::unique_ptr<CodeGenPGO> PGO;
1714
1715 /// Calculate branch weights appropriate for PGO data
1716 llvm::MDNode *createProfileWeights(uint64_t TrueCount,
1717 uint64_t FalseCount) const;
1718 llvm::MDNode *createProfileWeights(ArrayRef<uint64_t> Weights) const;
1719 llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond,
1720 uint64_t LoopCount) const;
1721
1722public:
1723 bool hasSkipCounter(const Stmt *S) const;
1724
1725 void markStmtAsUsed(bool Skipped, const Stmt *S);
1726 void markStmtMaybeUsed(const Stmt *S);
1727
1728 /// Used to specify which counter in a pair shall be incremented.
1729 /// For non-binary counters, a skip counter is derived as (Parent - Exec).
1730 /// In contrast for binary counters, a skip counter cannot be computed from
1731 /// the Parent counter. In such cases, dedicated SkipPath counters must be
1732 /// allocated and marked (incremented as binary counters). (Parent can be
1733 /// synthesized with (Exec + Skip) in simple cases)
1734 enum CounterForIncrement {
1735 UseExecPath = 0, ///< Exec (true)
1736 UseSkipPath, ///< Skip (false)
1737 };
1738
1739 /// Increment the profiler's counter for the given statement by \p StepV.
1740 /// If \p StepV is null, the default increment is 1.
1741 void incrementProfileCounter(const Stmt *S, llvm::Value *StepV = nullptr) {
1742 incrementProfileCounter(ExecSkip: UseExecPath, S, UseBoth: false, StepV);
1743 }
1744
1745 /// Emit increment of Counter.
1746 /// \param ExecSkip Use `Skipped` Counter if UseSkipPath is specified.
1747 /// \param S The Stmt that Counter is associated.
1748 /// \param UseBoth Mark both Exec/Skip as used. (for verification)
1749 /// \param StepV The offset Value for adding to Counter.
1750 void incrementProfileCounter(CounterForIncrement ExecSkip, const Stmt *S,
1751 bool UseBoth = false,
1752 llvm::Value *StepV = nullptr);
1753
1754 bool isMCDCCoverageEnabled() const {
1755 return (CGM.getCodeGenOpts().hasProfileClangInstr() &&
1756 CGM.getCodeGenOpts().MCDCCoverage &&
1757 !CurFn->hasFnAttribute(Kind: llvm::Attribute::NoProfile));
1758 }
1759
1760 /// Allocate a temp value on the stack that MCDC can use to track condition
1761 /// results.
1762 void maybeCreateMCDCCondBitmap();
1763
1764 bool isBinaryLogicalOp(const Expr *E) const {
1765 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: E->IgnoreParens());
1766 return (BOp && BOp->isLogicalOp());
1767 }
1768
1769 bool isMCDCDecisionExpr(const Expr *E) const;
1770 bool isMCDCBranchExpr(const Expr *E) const;
1771
1772 /// Zero-init the MCDC temp value.
1773 void maybeResetMCDCCondBitmap(const Expr *E);
1774
1775 /// Increment the profiler's counter for the given expression by \p StepV.
1776 /// If \p StepV is null, the default increment is 1.
1777 void maybeUpdateMCDCTestVectorBitmap(const Expr *E);
1778
1779 /// Update the MCDC temp value with the condition's evaluated result.
1780 void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val);
1781
1782 /// Get the profiler's count for the given statement.
1783 uint64_t getProfileCount(const Stmt *S);
1784
1785 /// Set the profiler's current count.
1786 void setCurrentProfileCount(uint64_t Count);
1787
1788 /// Get the profiler's current count. This is generally the count for the most
1789 /// recently incremented counter.
1790 uint64_t getCurrentProfileCount();
1791
1792 /// See CGDebugInfo::addInstToCurrentSourceAtom.
1793 void addInstToCurrentSourceAtom(llvm::Instruction *KeyInstruction,
1794 llvm::Value *Backup);
1795
1796 /// See CGDebugInfo::addInstToSpecificSourceAtom.
1797 void addInstToSpecificSourceAtom(llvm::Instruction *KeyInstruction,
1798 llvm::Value *Backup, uint64_t Atom);
1799
1800 /// Add \p KeyInstruction and an optional \p Backup instruction to a new atom
1801 /// group (See ApplyAtomGroup for more info).
1802 void addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,
1803 llvm::Value *Backup);
1804
1805 /// Copy all PFP fields from SrcPtr to DestPtr while updating signatures,
1806 /// assuming that DestPtr was already memcpy'd from SrcPtr.
1807 void emitPFPPostCopyUpdates(Address DestPtr, Address SrcPtr, QualType Ty);
1808
1809private:
1810 /// SwitchInsn - This is nearest current switch instruction. It is null if
1811 /// current context is not in a switch.
1812 llvm::SwitchInst *SwitchInsn = nullptr;
1813 /// The branch weights of SwitchInsn when doing instrumentation based PGO.
1814 SmallVector<uint64_t, 16> *SwitchWeights = nullptr;
1815
1816 /// The likelihood attributes of the SwitchCase.
1817 SmallVector<Stmt::Likelihood, 16> *SwitchLikelihood = nullptr;
1818
1819 /// CaseRangeBlock - This block holds if condition check for last case
1820 /// statement range in current switch instruction.
1821 llvm::BasicBlock *CaseRangeBlock = nullptr;
1822
1823 /// OpaqueLValues - Keeps track of the current set of opaque value
1824 /// expressions.
1825 llvm::DenseMap<const OpaqueValueExpr *, LValue> OpaqueLValues;
1826 llvm::DenseMap<const OpaqueValueExpr *, RValue> OpaqueRValues;
1827
1828 // VLASizeMap - This keeps track of the associated size for each VLA type.
1829 // We track this by the size expression rather than the type itself because
1830 // in certain situations, like a const qualifier applied to an VLA typedef,
1831 // multiple VLA types can share the same size expression.
1832 // FIXME: Maybe this could be a stack of maps that is pushed/popped as we
1833 // enter/leave scopes.
1834 llvm::DenseMap<const Expr *, llvm::Value *> VLASizeMap;
1835
1836 /// A block containing a single 'unreachable' instruction. Created
1837 /// lazily by getUnreachableBlock().
1838 llvm::BasicBlock *UnreachableBlock = nullptr;
1839
1840 /// Counts of the number return expressions in the function.
1841 unsigned NumReturnExprs = 0;
1842
1843 /// Count the number of simple (constant) return expressions in the function.
1844 unsigned NumSimpleReturnExprs = 0;
1845
1846 /// The last regular (non-return) debug location (breakpoint) in the function.
1847 SourceLocation LastStopPoint;
1848
1849public:
1850 /// Source location information about the default argument or member
1851 /// initializer expression we're evaluating, if any.
1852 CurrentSourceLocExprScope CurSourceLocExprScope;
1853 using SourceLocExprScopeGuard =
1854 CurrentSourceLocExprScope::SourceLocExprScopeGuard;
1855
1856 /// A scope within which we are constructing the fields of an object which
1857 /// might use a CXXDefaultInitExpr. This stashes away a 'this' value to use
1858 /// if we need to evaluate a CXXDefaultInitExpr within the evaluation.
1859 class FieldConstructionScope {
1860 public:
1861 FieldConstructionScope(CodeGenFunction &CGF, Address This)
1862 : CGF(CGF), OldCXXDefaultInitExprThis(CGF.CXXDefaultInitExprThis) {
1863 CGF.CXXDefaultInitExprThis = This;
1864 }
1865 ~FieldConstructionScope() {
1866 CGF.CXXDefaultInitExprThis = OldCXXDefaultInitExprThis;
1867 }
1868
1869 private:
1870 CodeGenFunction &CGF;
1871 Address OldCXXDefaultInitExprThis;
1872 };
1873
1874 /// The scope of a CXXDefaultInitExpr. Within this scope, the value of 'this'
1875 /// is overridden to be the object under construction.
1876 class CXXDefaultInitExprScope {
1877 public:
1878 CXXDefaultInitExprScope(CodeGenFunction &CGF, const CXXDefaultInitExpr *E)
1879 : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue),
1880 OldCXXThisAlignment(CGF.CXXThisAlignment),
1881 SourceLocScope(E, CGF.CurSourceLocExprScope) {
1882 CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getBasePointer();
1883 CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment();
1884 }
1885 ~CXXDefaultInitExprScope() {
1886 CGF.CXXThisValue = OldCXXThisValue;
1887 CGF.CXXThisAlignment = OldCXXThisAlignment;
1888 }
1889
1890 public:
1891 CodeGenFunction &CGF;
1892 llvm::Value *OldCXXThisValue;
1893 CharUnits OldCXXThisAlignment;
1894 SourceLocExprScopeGuard SourceLocScope;
1895 };
1896
1897 struct CXXDefaultArgExprScope : SourceLocExprScopeGuard {
1898 CXXDefaultArgExprScope(CodeGenFunction &CGF, const CXXDefaultArgExpr *E)
1899 : SourceLocExprScopeGuard(E, CGF.CurSourceLocExprScope) {}
1900 };
1901
1902 /// The scope of an ArrayInitLoopExpr. Within this scope, the value of the
1903 /// current loop index is overridden.
1904 class ArrayInitLoopExprScope {
1905 public:
1906 ArrayInitLoopExprScope(CodeGenFunction &CGF, llvm::Value *Index)
1907 : CGF(CGF), OldArrayInitIndex(CGF.ArrayInitIndex) {
1908 CGF.ArrayInitIndex = Index;
1909 }
1910 ~ArrayInitLoopExprScope() { CGF.ArrayInitIndex = OldArrayInitIndex; }
1911
1912 private:
1913 CodeGenFunction &CGF;
1914 llvm::Value *OldArrayInitIndex;
1915 };
1916
1917 class InlinedInheritingConstructorScope {
1918 public:
1919 InlinedInheritingConstructorScope(CodeGenFunction &CGF, GlobalDecl GD)
1920 : CGF(CGF), OldCurGD(CGF.CurGD), OldCurFuncDecl(CGF.CurFuncDecl),
1921 OldCurCodeDecl(CGF.CurCodeDecl),
1922 OldCXXABIThisDecl(CGF.CXXABIThisDecl),
1923 OldCXXABIThisValue(CGF.CXXABIThisValue),
1924 OldCXXThisValue(CGF.CXXThisValue),
1925 OldCXXABIThisAlignment(CGF.CXXABIThisAlignment),
1926 OldCXXThisAlignment(CGF.CXXThisAlignment),
1927 OldReturnValue(CGF.ReturnValue), OldFnRetTy(CGF.FnRetTy),
1928 OldCXXInheritedCtorInitExprArgs(
1929 std::move(CGF.CXXInheritedCtorInitExprArgs)) {
1930 CGF.CurGD = GD;
1931 CGF.CurFuncDecl = CGF.CurCodeDecl =
1932 cast<CXXConstructorDecl>(Val: GD.getDecl());
1933 CGF.CXXABIThisDecl = nullptr;
1934 CGF.CXXABIThisValue = nullptr;
1935 CGF.CXXThisValue = nullptr;
1936 CGF.CXXABIThisAlignment = CharUnits();
1937 CGF.CXXThisAlignment = CharUnits();
1938 CGF.ReturnValue = Address::invalid();
1939 CGF.FnRetTy = QualType();
1940 CGF.CXXInheritedCtorInitExprArgs.clear();
1941 }
1942 ~InlinedInheritingConstructorScope() {
1943 CGF.CurGD = OldCurGD;
1944 CGF.CurFuncDecl = OldCurFuncDecl;
1945 CGF.CurCodeDecl = OldCurCodeDecl;
1946 CGF.CXXABIThisDecl = OldCXXABIThisDecl;
1947 CGF.CXXABIThisValue = OldCXXABIThisValue;
1948 CGF.CXXThisValue = OldCXXThisValue;
1949 CGF.CXXABIThisAlignment = OldCXXABIThisAlignment;
1950 CGF.CXXThisAlignment = OldCXXThisAlignment;
1951 CGF.ReturnValue = OldReturnValue;
1952 CGF.FnRetTy = OldFnRetTy;
1953 CGF.CXXInheritedCtorInitExprArgs =
1954 std::move(OldCXXInheritedCtorInitExprArgs);
1955 }
1956
1957 private:
1958 CodeGenFunction &CGF;
1959 GlobalDecl OldCurGD;
1960 const Decl *OldCurFuncDecl;
1961 const Decl *OldCurCodeDecl;
1962 ImplicitParamDecl *OldCXXABIThisDecl;
1963 llvm::Value *OldCXXABIThisValue;
1964 llvm::Value *OldCXXThisValue;
1965 CharUnits OldCXXABIThisAlignment;
1966 CharUnits OldCXXThisAlignment;
1967 Address OldReturnValue;
1968 QualType OldFnRetTy;
1969 CallArgList OldCXXInheritedCtorInitExprArgs;
1970 };
1971
1972 // Helper class for the OpenMP IR Builder. Allows reusability of code used for
1973 // region body, and finalization codegen callbacks. This will class will also
1974 // contain privatization functions used by the privatization call backs
1975 //
1976 // TODO: this is temporary class for things that are being moved out of
1977 // CGOpenMPRuntime, new versions of current CodeGenFunction methods, or
1978 // utility function for use with the OMPBuilder. Once that move to use the
1979 // OMPBuilder is done, everything here will either become part of CodeGenFunc.
1980 // directly, or a new helper class that will contain functions used by both
1981 // this and the OMPBuilder
1982
1983 struct OMPBuilderCBHelpers {
1984
1985 OMPBuilderCBHelpers() = delete;
1986 OMPBuilderCBHelpers(const OMPBuilderCBHelpers &) = delete;
1987 OMPBuilderCBHelpers &operator=(const OMPBuilderCBHelpers &) = delete;
1988
1989 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1990
1991 /// Cleanup action for allocate support.
1992 class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
1993
1994 private:
1995 llvm::CallInst *RTLFnCI;
1996
1997 public:
1998 OMPAllocateCleanupTy(llvm::CallInst *RLFnCI) : RTLFnCI(RLFnCI) {
1999 RLFnCI->removeFromParent();
2000 }
2001
2002 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
2003 if (!CGF.HaveInsertPoint())
2004 return;
2005 CGF.Builder.Insert(I: RTLFnCI);
2006 }
2007 };
2008
2009 /// Returns address of the threadprivate variable for the current
2010 /// thread. This Also create any necessary OMP runtime calls.
2011 ///
2012 /// \param VD VarDecl for Threadprivate variable.
2013 /// \param VDAddr Address of the Vardecl
2014 /// \param Loc The location where the barrier directive was encountered
2015 static Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
2016 const VarDecl *VD, Address VDAddr,
2017 SourceLocation Loc);
2018
2019 /// Gets the OpenMP-specific address of the local variable /p VD.
2020 static Address getAddressOfLocalVariable(CodeGenFunction &CGF,
2021 const VarDecl *VD);
2022 /// Get the platform-specific name separator.
2023 /// \param Parts different parts of the final name that needs separation
2024 /// \param FirstSeparator First separator used between the initial two
2025 /// parts of the name.
2026 /// \param Separator separator used between all of the rest consecutinve
2027 /// parts of the name
2028 static std::string getNameWithSeparators(ArrayRef<StringRef> Parts,
2029 StringRef FirstSeparator = ".",
2030 StringRef Separator = ".");
2031 /// Emit the Finalization for an OMP region
2032 /// \param CGF The Codegen function this belongs to
2033 /// \param IP Insertion point for generating the finalization code.
2034 static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP) {
2035 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
2036 llvm::BasicBlock *IPBB = IP.getNodeParent();
2037 assert(IPBB->end() != IP &&
2038 "OpenMP IR Builder should cause terminated block!");
2039
2040 llvm::BasicBlock *DestBB = IPBB->getUniqueSuccessor();
2041 assert(DestBB && "Finalization block should have one successor!");
2042
2043 // erase and replace with cleanup branch.
2044 IPBB->getTerminator()->eraseFromParent();
2045 CGF.Builder.SetInsertPoint(IPBB);
2046 CodeGenFunction::JumpDest Dest = CGF.getJumpDestInCurrentScope(Target: DestBB);
2047 CGF.EmitBranchThroughCleanup(Dest);
2048 }
2049
2050 /// Emit the body of an OMP region
2051 /// \param CGF The Codegen function this belongs to
2052 /// \param RegionBodyStmt The body statement for the OpenMP region being
2053 /// generated
2054 /// \param AllocaIP Where to insert alloca instructions
2055 /// \param CodeGenIP Where to insert the region code
2056 /// \param RegionName Name to be used for new blocks
2057 static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF,
2058 const Stmt *RegionBodyStmt,
2059 InsertPointTy AllocaIP,
2060 InsertPointTy CodeGenIP,
2061 Twine RegionName);
2062
2063 static void EmitCaptureStmt(CodeGenFunction &CGF,
2064 llvm::BasicBlock *CodeGenIPBB,
2065 llvm::BasicBlock &FiniBB, llvm::Function *Fn,
2066 ArrayRef<llvm::Value *> Args) {
2067 if (llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminatorOrNull())
2068 CodeGenIPBBTI->eraseFromParent();
2069
2070 CGF.Builder.SetInsertPoint(CodeGenIPBB);
2071
2072 if (Fn->doesNotThrow())
2073 CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
2074 else
2075 CGF.EmitRuntimeCall(callee: Fn, args: Args);
2076
2077 if (CGF.Builder.saveIP().isValid())
2078 CGF.Builder.CreateBr(Dest: &FiniBB);
2079 }
2080
2081 /// Emit the body of an OMP region that will be outlined in
2082 /// OpenMPIRBuilder::finalize().
2083 /// \param CGF The Codegen function this belongs to
2084 /// \param RegionBodyStmt The body statement for the OpenMP region being
2085 /// generated
2086 /// \param AllocaIP Where to insert alloca instructions
2087 /// \param CodeGenIP Where to insert the region code
2088 /// \param RegionName Name to be used for new blocks
2089 static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF,
2090 const Stmt *RegionBodyStmt,
2091 InsertPointTy AllocaIP,
2092 InsertPointTy CodeGenIP,
2093 Twine RegionName);
2094
2095 /// RAII for preserving necessary info during Outlined region body codegen.
2096 class OutlinedRegionBodyRAII {
2097
2098 llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
2099 CodeGenFunction::JumpDest OldReturnBlock;
2100 CodeGenFunction &CGF;
2101
2102 public:
2103 OutlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
2104 llvm::BasicBlock &RetBB)
2105 : CGF(cgf) {
2106 assert(AllocaIP.isValid() &&
2107 "Must specify Insertion point for allocas of outlined function");
2108 OldAllocaIP = CGF.AllocaInsertPt;
2109 CGF.AllocaInsertPt = &*AllocaIP;
2110
2111 OldReturnBlock = CGF.ReturnBlock;
2112 CGF.ReturnBlock = CGF.getJumpDestInCurrentScope(Target: &RetBB);
2113 }
2114
2115 ~OutlinedRegionBodyRAII() {
2116 CGF.AllocaInsertPt = OldAllocaIP;
2117 CGF.ReturnBlock = OldReturnBlock;
2118 }
2119 };
2120
2121 /// RAII for preserving necessary info during inlined region body codegen.
2122 class InlinedRegionBodyRAII {
2123
2124 llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
2125 CodeGenFunction &CGF;
2126
2127 public:
2128 InlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
2129 llvm::BasicBlock &FiniBB)
2130 : CGF(cgf) {
2131 // Alloca insertion block should be in the entry block of the containing
2132 // function so it expects an empty AllocaIP in which case will reuse the
2133 // old alloca insertion point, or a new AllocaIP in the same block as
2134 // the old one
2135 assert((!AllocaIP.isValid() ||
2136 CGF.AllocaInsertPt->getParent() == AllocaIP.getNodeParent()) &&
2137 "Insertion point should be in the entry block of containing "
2138 "function!");
2139 OldAllocaIP = CGF.AllocaInsertPt;
2140 if (AllocaIP.isValid())
2141 CGF.AllocaInsertPt = &*AllocaIP;
2142
2143 // TODO: Remove the call, after making sure the counter is not used by
2144 // the EHStack.
2145 // Since this is an inlined region, it should not modify the
2146 // ReturnBlock, and should reuse the one for the enclosing outlined
2147 // region. So, the JumpDest being return by the function is discarded
2148 (void)CGF.getJumpDestInCurrentScope(Target: &FiniBB);
2149 }
2150
2151 ~InlinedRegionBodyRAII() { CGF.AllocaInsertPt = OldAllocaIP; }
2152 };
2153 };
2154
2155private:
2156 /// CXXThisDecl - When generating code for a C++ member function,
2157 /// this will hold the implicit 'this' declaration.
2158 ImplicitParamDecl *CXXABIThisDecl = nullptr;
2159 llvm::Value *CXXABIThisValue = nullptr;
2160 llvm::Value *CXXThisValue = nullptr;
2161 CharUnits CXXABIThisAlignment;
2162 CharUnits CXXThisAlignment;
2163
2164 /// The value of 'this' to use when evaluating CXXDefaultInitExprs within
2165 /// this expression.
2166 Address CXXDefaultInitExprThis = Address::invalid();
2167
2168 /// The current array initialization index when evaluating an
2169 /// ArrayInitIndexExpr within an ArrayInitLoopExpr.
2170 llvm::Value *ArrayInitIndex = nullptr;
2171
2172 /// The values of function arguments to use when evaluating
2173 /// CXXInheritedCtorInitExprs within this context.
2174 CallArgList CXXInheritedCtorInitExprArgs;
2175
2176 /// CXXStructorImplicitParamDecl - When generating code for a constructor or
2177 /// destructor, this will hold the implicit argument (e.g. VTT).
2178 ImplicitParamDecl *CXXStructorImplicitParamDecl = nullptr;
2179 llvm::Value *CXXStructorImplicitParamValue = nullptr;
2180
2181 /// OutermostConditional - Points to the outermost active
2182 /// conditional control. This is used so that we know if a
2183 /// temporary should be destroyed conditionally.
2184 ConditionalEvaluation *OutermostConditional = nullptr;
2185
2186 /// The current lexical scope.
2187 LexicalScope *CurLexicalScope = nullptr;
2188
2189 /// The current source location that should be used for exception
2190 /// handling code.
2191 SourceLocation CurEHLocation;
2192
2193 /// BlockByrefInfos - For each __block variable, contains
2194 /// information about the layout of the variable.
2195 llvm::DenseMap<const ValueDecl *, BlockByrefInfo> BlockByrefInfos;
2196
2197 /// Used by -fsanitize=nullability-return to determine whether the return
2198 /// value can be checked.
2199 llvm::Value *RetValNullabilityPrecondition = nullptr;
2200
2201 /// Check if -fsanitize=nullability-return instrumentation is required for
2202 /// this function.
2203 bool requiresReturnValueNullabilityCheck() const {
2204 return RetValNullabilityPrecondition;
2205 }
2206
2207 /// Used to store precise source locations for return statements by the
2208 /// runtime return value checks.
2209 Address ReturnLocation = Address::invalid();
2210
2211 /// Check if the return value of this function requires sanitization.
2212 bool requiresReturnValueCheck() const;
2213
2214 bool isInAllocaArgument(CGCXXABI &ABI, QualType Ty);
2215 bool hasInAllocaArg(const CXXMethodDecl *MD);
2216
2217 llvm::BasicBlock *TerminateLandingPad = nullptr;
2218 llvm::BasicBlock *TerminateHandler = nullptr;
2219 llvm::SmallVector<llvm::BasicBlock *, 2> TrapBBs;
2220
2221 /// Terminate funclets keyed by parent funclet pad.
2222 llvm::MapVector<llvm::Value *, llvm::BasicBlock *> TerminateFunclets;
2223
2224 /// Largest vector width used in ths function. Will be used to create a
2225 /// function attribute.
2226 unsigned LargestVectorWidth = 0;
2227
2228 /// True if we need emit the life-time markers. This is initially set in
2229 /// the constructor, but could be overwritten to true if this is a coroutine.
2230 bool ShouldEmitLifetimeMarkers;
2231
2232 /// Add OpenCL kernel arg metadata and the kernel attribute metadata to
2233 /// the function metadata.
2234 void EmitKernelMetadata(const FunctionDecl *FD, llvm::Function *Fn);
2235
2236public:
2237 CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext = false);
2238 ~CodeGenFunction();
2239
2240 CodeGenTypes &getTypes() const { return CGM.getTypes(); }
2241 ASTContext &getContext() const { return CGM.getContext(); }
2242 CGDebugInfo *getDebugInfo() {
2243 if (DisableDebugInfo)
2244 return nullptr;
2245 return DebugInfo;
2246 }
2247 void disableDebugInfo() { DisableDebugInfo = true; }
2248 void enableDebugInfo() { DisableDebugInfo = false; }
2249
2250 bool shouldUseFusedARCCalls() {
2251 return CGM.getCodeGenOpts().OptimizationLevel == 0;
2252 }
2253
2254 const LangOptions &getLangOpts() const { return CGM.getLangOpts(); }
2255
2256 /// Returns a pointer to the function's exception object and selector slot,
2257 /// which is assigned in every landing pad.
2258 Address getExceptionSlot();
2259 Address getEHSelectorSlot();
2260
2261 /// Returns the contents of the function's exception object and selector
2262 /// slots.
2263 llvm::Value *getExceptionFromSlot();
2264 llvm::Value *getSelectorFromSlot();
2265
2266 RawAddress getNormalCleanupDestSlot();
2267
2268 llvm::BasicBlock *getUnreachableBlock() {
2269 if (!UnreachableBlock) {
2270 UnreachableBlock = createBasicBlock(name: "unreachable");
2271 new llvm::UnreachableInst(getLLVMContext(), UnreachableBlock);
2272 }
2273 return UnreachableBlock;
2274 }
2275
2276 llvm::BasicBlock *getInvokeDest() {
2277 if (!EHStack.requiresLandingPad())
2278 return nullptr;
2279 return getInvokeDestImpl();
2280 }
2281
2282 bool currentFunctionUsesSEHTry() const { return !!CurSEHParent; }
2283
2284 const TargetInfo &getTarget() const { return Target; }
2285 llvm::LLVMContext &getLLVMContext() { return CGM.getLLVMContext(); }
2286
2287 /// Accessors for LocalDeclMap.
2288 DeclMapTy::iterator findLocalDecl(const Decl *D) {
2289 return LocalDeclMap.find(Val: D);
2290 }
2291 DeclMapTy::iterator localDeclMapEnd() { return LocalDeclMap.end(); }
2292 std::pair<DeclMapTy::iterator, bool> insertLocalDecl(const Decl *D,
2293 Address Addr) {
2294 return LocalDeclMap.insert(KV: {D, Addr});
2295 }
2296 void eraseLocalDecl(const Decl *D) { LocalDeclMap.erase(Val: D); }
2297 const TargetCodeGenInfo &getTargetHooks() const {
2298 return CGM.getTargetCodeGenInfo();
2299 }
2300 const FunctionDecl *getCurrentFunctionDecl() const;
2301
2302 //===--------------------------------------------------------------------===//
2303 // Cleanups
2304 //===--------------------------------------------------------------------===//
2305
2306 typedef void Destroyer(CodeGenFunction &CGF, Address addr, QualType ty);
2307
2308 void pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,
2309 Address arrayEndPointer,
2310 QualType elementType,
2311 CharUnits elementAlignment,
2312 Destroyer *destroyer);
2313 void pushRegularPartialArrayCleanup(llvm::Value *arrayBegin,
2314 llvm::Value *arrayEnd,
2315 QualType elementType,
2316 CharUnits elementAlignment,
2317 Destroyer *destroyer);
2318
2319 void pushDestroy(QualType::DestructionKind dtorKind, Address addr,
2320 QualType type);
2321 void pushEHDestroy(QualType::DestructionKind dtorKind, Address addr,
2322 QualType type);
2323 void pushDestroy(CleanupKind kind, Address addr, QualType type,
2324 Destroyer *destroyer, bool useEHCleanupForArray);
2325 void pushDestroyAndDeferDeactivation(QualType::DestructionKind dtorKind,
2326 Address addr, QualType type);
2327 void pushDestroyAndDeferDeactivation(CleanupKind cleanupKind, Address addr,
2328 QualType type, Destroyer *destroyer,
2329 bool useEHCleanupForArray);
2330 void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr,
2331 QualType type, Destroyer *destroyer,
2332 bool useEHCleanupForArray);
2333 void pushLifetimeExtendedDestroy(QualType::DestructionKind dtorKind,
2334 Address addr, QualType type);
2335 void pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete,
2336 llvm::Value *CompletePtr,
2337 QualType ElementType);
2338 void pushStackRestore(CleanupKind kind, Address SPMem);
2339 void pushKmpcAllocFree(CleanupKind Kind,
2340 std::pair<llvm::Value *, llvm::Value *> AddrSizePair);
2341 void emitDestroy(Address addr, QualType type, Destroyer *destroyer,
2342 bool useEHCleanupForArray);
2343 llvm::Function *generateDestroyHelper(Address addr, QualType type,
2344 Destroyer *destroyer,
2345 bool useEHCleanupForArray,
2346 const VarDecl *VD);
2347 void emitArrayDestroy(llvm::Value *begin, llvm::Value *end,
2348 QualType elementType, CharUnits elementAlign,
2349 Destroyer *destroyer, bool checkZeroLength,
2350 bool useEHCleanup);
2351
2352 Destroyer *getDestroyer(QualType::DestructionKind destructionKind);
2353
2354 /// Determines whether an EH cleanup is required to destroy a type
2355 /// with the given destruction kind.
2356 bool needsEHCleanup(QualType::DestructionKind kind) {
2357 switch (kind) {
2358 case QualType::DK_none:
2359 return false;
2360 case QualType::DK_cxx_destructor:
2361 case QualType::DK_objc_weak_lifetime:
2362 case QualType::DK_nontrivial_c_struct:
2363 return getLangOpts().Exceptions;
2364 case QualType::DK_objc_strong_lifetime:
2365 return getLangOpts().Exceptions &&
2366 CGM.getCodeGenOpts().ObjCAutoRefCountExceptions;
2367 }
2368 llvm_unreachable("bad destruction kind");
2369 }
2370
2371 CleanupKind getCleanupKind(QualType::DestructionKind kind) {
2372 return (needsEHCleanup(kind) ? NormalAndEHCleanup : NormalCleanup);
2373 }
2374
2375 //===--------------------------------------------------------------------===//
2376 // Objective-C
2377 //===--------------------------------------------------------------------===//
2378
2379 void GenerateObjCMethod(const ObjCMethodDecl *OMD);
2380
2381 void StartObjCMethod(const ObjCMethodDecl *MD, const ObjCContainerDecl *CD);
2382
2383 /// GenerateObjCGetter - Synthesize an Objective-C property getter function.
2384 void GenerateObjCGetter(ObjCImplementationDecl *IMP,
2385 const ObjCPropertyImplDecl *PID);
2386 void generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
2387 const ObjCPropertyImplDecl *propImpl,
2388 const ObjCMethodDecl *GetterMothodDecl,
2389 llvm::Constant *AtomicHelperFn);
2390
2391 void GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
2392 ObjCMethodDecl *MD, bool ctor);
2393
2394 /// GenerateObjCSetter - Synthesize an Objective-C property setter function
2395 /// for the given property.
2396 void GenerateObjCSetter(ObjCImplementationDecl *IMP,
2397 const ObjCPropertyImplDecl *PID);
2398 void generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
2399 const ObjCPropertyImplDecl *propImpl,
2400 llvm::Constant *AtomicHelperFn);
2401
2402 //===--------------------------------------------------------------------===//
2403 // Block Bits
2404 //===--------------------------------------------------------------------===//
2405
2406 /// Emit block literal.
2407 /// \return an LLVM value which is a pointer to a struct which contains
2408 /// information about the block, including the block invoke function, the
2409 /// captured variables, etc.
2410 llvm::Value *EmitBlockLiteral(const BlockExpr *);
2411
2412 llvm::Function *GenerateBlockFunction(GlobalDecl GD, const CGBlockInfo &Info,
2413 const DeclMapTy &ldm,
2414 bool IsLambdaConversionToBlock,
2415 bool BuildGlobalBlock);
2416
2417 /// Check if \p T is a C++ class that has a destructor that can throw.
2418 static bool cxxDestructorCanThrow(QualType T);
2419
2420 llvm::Constant *GenerateCopyHelperFunction(const CGBlockInfo &blockInfo);
2421 llvm::Constant *GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo);
2422 llvm::Constant *
2423 GenerateObjCAtomicSetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);
2424 llvm::Constant *
2425 GenerateObjCAtomicGetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);
2426 llvm::Value *EmitBlockCopyAndAutorelease(llvm::Value *Block, QualType Ty);
2427
2428 void BuildBlockRelease(llvm::Value *DeclPtr, BlockFieldFlags flags,
2429 bool CanThrow);
2430
2431 class AutoVarEmission;
2432
2433 void emitByrefStructureInit(const AutoVarEmission &emission);
2434
2435 /// Enter a cleanup to destroy a __block variable. Note that this
2436 /// cleanup should be a no-op if the variable hasn't left the stack
2437 /// yet; if a cleanup is required for the variable itself, that needs
2438 /// to be done externally.
2439 ///
2440 /// \param Kind Cleanup kind.
2441 ///
2442 /// \param Addr When \p LoadBlockVarAddr is false, the address of the __block
2443 /// structure that will be passed to _Block_object_dispose. When
2444 /// \p LoadBlockVarAddr is true, the address of the field of the block
2445 /// structure that holds the address of the __block structure.
2446 ///
2447 /// \param Flags The flag that will be passed to _Block_object_dispose.
2448 ///
2449 /// \param LoadBlockVarAddr Indicates whether we need to emit a load from
2450 /// \p Addr to get the address of the __block structure.
2451 void enterByrefCleanup(CleanupKind Kind, Address Addr, BlockFieldFlags Flags,
2452 bool LoadBlockVarAddr, bool CanThrow);
2453
2454 void setBlockContextParameter(const ImplicitParamDecl *D, unsigned argNum,
2455 llvm::Value *ptr);
2456
2457 Address LoadBlockStruct();
2458 Address GetAddrOfBlockDecl(const VarDecl *var);
2459
2460 /// BuildBlockByrefAddress - Computes the location of the
2461 /// data in a variable which is declared as __block.
2462 Address emitBlockByrefAddress(Address baseAddr, const VarDecl *V,
2463 bool followForward = true);
2464 Address emitBlockByrefAddress(Address baseAddr, const BlockByrefInfo &info,
2465 bool followForward, const llvm::Twine &name);
2466
2467 const BlockByrefInfo &getBlockByrefInfo(const VarDecl *var);
2468
2469 QualType BuildFunctionArgList(GlobalDecl GD, FunctionArgList &Args);
2470
2471 void GenerateCode(GlobalDecl GD, llvm::Function *Fn,
2472 const CGFunctionInfo &FnInfo);
2473
2474 /// Annotate the function with an attribute that disables TSan checking at
2475 /// runtime.
2476 void markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn);
2477
2478 /// Emit code for the start of a function.
2479 /// \param Loc The location to be associated with the function.
2480 /// \param StartLoc The location of the function body.
2481 void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn,
2482 const CGFunctionInfo &FnInfo, const FunctionArgList &Args,
2483 SourceLocation Loc = SourceLocation(),
2484 SourceLocation StartLoc = SourceLocation());
2485
2486 static bool IsConstructorDelegationValid(const CXXConstructorDecl *Ctor);
2487
2488 void EmitConstructorBody(FunctionArgList &Args);
2489 void EmitDestructorBody(FunctionArgList &Args);
2490 void emitImplicitAssignmentOperatorBody(FunctionArgList &Args);
2491 void EmitFunctionBody(const Stmt *Body);
2492 void EmitBlockWithFallThrough(llvm::BasicBlock *BB, const Stmt *S);
2493
2494 void EmitForwardingCallToLambda(const CXXMethodDecl *LambdaCallOperator,
2495 CallArgList &CallArgs,
2496 const CGFunctionInfo *CallOpFnInfo = nullptr,
2497 llvm::Constant *CallOpFn = nullptr);
2498 void EmitLambdaBlockInvokeBody();
2499 void EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD);
2500 void EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,
2501 CallArgList &CallArgs);
2502 void EmitLambdaInAllocaImplFn(const CXXMethodDecl *CallOp,
2503 const CGFunctionInfo **ImplFnInfo,
2504 llvm::Function **ImplFn);
2505 void EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD);
2506 void EmitLambdaVLACapture(const VariableArrayType *VAT, LValue LV) {
2507 EmitStoreThroughLValue(Src: RValue::get(V: VLASizeMap[VAT->getSizeExpr()]), Dst: LV);
2508 }
2509 void EmitAsanPrologueOrEpilogue(bool Prologue);
2510
2511 /// Emit the unified return block, trying to avoid its emission when
2512 /// possible.
2513 /// \return The debug location of the user written return statement if the
2514 /// return block is avoided.
2515 llvm::DebugLoc EmitReturnBlock();
2516
2517 /// FinishFunction - Complete IR generation of the current function. It is
2518 /// legal to call this function even if there is no current insertion point.
2519 void FinishFunction(SourceLocation EndLoc = SourceLocation());
2520
2521 void StartThunk(llvm::Function *Fn, GlobalDecl GD,
2522 const CGFunctionInfo &FnInfo, bool IsUnprototyped);
2523
2524 void EmitCallAndReturnForThunk(llvm::FunctionCallee Callee,
2525 const ThunkInfo *Thunk, bool IsUnprototyped);
2526
2527 void FinishThunk();
2528
2529 /// Start an Objective-C direct method thunk.
2530 void StartObjCDirectPreconditionThunk(const ObjCMethodDecl *OMD,
2531 llvm::Function *Fn,
2532 const CGFunctionInfo &FI);
2533
2534 /// Finish an Objective-C direct method thunk.
2535 void FinishObjCDirectPreconditionThunk();
2536
2537 /// Emit a musttail call for a thunk with a potentially adjusted this pointer.
2538 void EmitMustTailThunk(GlobalDecl GD, llvm::Value *AdjustedThisPtr,
2539 llvm::FunctionCallee Callee);
2540
2541 /// Generate a thunk for the given method.
2542 void generateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo,
2543 GlobalDecl GD, const ThunkInfo &Thunk,
2544 bool IsUnprototyped);
2545
2546 llvm::Function *GenerateVarArgsThunk(llvm::Function *Fn,
2547 const CGFunctionInfo &FnInfo,
2548 GlobalDecl GD, const ThunkInfo &Thunk);
2549
2550 void EmitCtorPrologue(const CXXConstructorDecl *CD, CXXCtorType Type,
2551 FunctionArgList &Args);
2552
2553 void EmitInitializerForField(FieldDecl *Field, LValue LHS, Expr *Init);
2554
2555 /// Struct with all information about dynamic [sub]class needed to set vptr.
2556 struct VPtr {
2557 BaseSubobject Base;
2558 const CXXRecordDecl *NearestVBase;
2559 CharUnits OffsetFromNearestVBase;
2560 const CXXRecordDecl *VTableClass;
2561 };
2562
2563 /// Initialize the vtable pointer of the given subobject.
2564 void InitializeVTablePointer(const VPtr &vptr);
2565
2566 typedef llvm::SmallVector<VPtr, 4> VPtrsVector;
2567
2568 typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
2569 VPtrsVector getVTablePointers(const CXXRecordDecl *VTableClass);
2570
2571 void getVTablePointers(BaseSubobject Base, const CXXRecordDecl *NearestVBase,
2572 CharUnits OffsetFromNearestVBase,
2573 bool BaseIsNonVirtualPrimaryBase,
2574 const CXXRecordDecl *VTableClass,
2575 VisitedVirtualBasesSetTy &VBases, VPtrsVector &vptrs);
2576
2577 void InitializeVTablePointers(const CXXRecordDecl *ClassDecl);
2578
2579 // VTableTrapMode - whether we guarantee that loading the
2580 // vtable is guaranteed to trap on authentication failure,
2581 // even if the resulting vtable pointer is unused.
2582 enum class VTableAuthMode {
2583 Authenticate,
2584 MustTrap,
2585 UnsafeUbsanStrip // Should only be used for Vptr UBSan check
2586 };
2587 /// GetVTablePtr - Return the Value of the vtable pointer member pointed
2588 /// to by This.
2589 llvm::Value *
2590 GetVTablePtr(Address This, llvm::Type *VTableTy,
2591 const CXXRecordDecl *VTableClass,
2592 VTableAuthMode AuthMode = VTableAuthMode::Authenticate);
2593
2594 enum CFITypeCheckKind {
2595 CFITCK_VCall,
2596 CFITCK_NVCall,
2597 CFITCK_DerivedCast,
2598 CFITCK_UnrelatedCast,
2599 CFITCK_ICall,
2600 CFITCK_NVMFCall,
2601 CFITCK_VMFCall,
2602 };
2603
2604 /// Derived is the presumed address of an object of type T after a
2605 /// cast. If T is a polymorphic class type, emit a check that the virtual
2606 /// table for Derived belongs to a class derived from T.
2607 void EmitVTablePtrCheckForCast(QualType T, Address Derived, bool MayBeNull,
2608 CFITypeCheckKind TCK, SourceLocation Loc);
2609
2610 /// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable.
2611 /// If vptr CFI is enabled, emit a check that VTable is valid.
2612 void EmitVTablePtrCheckForCall(const CXXRecordDecl *RD, llvm::Value *VTable,
2613 CFITypeCheckKind TCK, SourceLocation Loc);
2614
2615 /// EmitVTablePtrCheck - Emit a check that VTable is a valid virtual table for
2616 /// RD using llvm.type.test.
2617 void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable,
2618 CFITypeCheckKind TCK, SourceLocation Loc);
2619
2620 /// If whole-program virtual table optimization is enabled, emit an assumption
2621 /// that VTable is a member of RD's type identifier. Or, if vptr CFI is
2622 /// enabled, emit a check that VTable is a member of RD's type identifier.
2623 void EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2624 llvm::Value *VTable, SourceLocation Loc);
2625
2626 /// Returns whether we should perform a type checked load when loading a
2627 /// virtual function for virtual calls to members of RD. This is generally
2628 /// true when both vcall CFI and whole-program-vtables are enabled.
2629 bool ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD);
2630
2631 /// Emit a type checked load from the given vtable.
2632 llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD,
2633 llvm::Value *VTable,
2634 llvm::Type *VTableTy,
2635 uint64_t VTableByteOffset);
2636
2637 /// EnterDtorCleanups - Enter the cleanups necessary to complete the
2638 /// given phase of destruction for a destructor. The end result
2639 /// should call destructors on members and base classes in reverse
2640 /// order of their construction.
2641 void EnterDtorCleanups(const CXXDestructorDecl *Dtor, CXXDtorType Type);
2642
2643 /// ShouldInstrumentFunction - Return true if the current function should be
2644 /// instrumented with __cyg_profile_func_* calls
2645 bool ShouldInstrumentFunction();
2646
2647 /// ShouldSkipSanitizerInstrumentation - Return true if the current function
2648 /// should not be instrumented with sanitizers.
2649 bool ShouldSkipSanitizerInstrumentation();
2650
2651 /// ShouldXRayInstrument - Return true if the current function should be
2652 /// instrumented with XRay nop sleds.
2653 bool ShouldXRayInstrumentFunction() const;
2654
2655 /// AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit
2656 /// XRay custom event handling calls.
2657 bool AlwaysEmitXRayCustomEvents() const;
2658
2659 /// AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit
2660 /// XRay typed event handling calls.
2661 bool AlwaysEmitXRayTypedEvents() const;
2662
2663 /// Return a type hash constant for a function instrumented by
2664 /// -fsanitize=function.
2665 llvm::ConstantInt *getUBSanFunctionTypeHash(QualType T) const;
2666
2667 /// EmitFunctionProlog - Emit the target specific LLVM code to load the
2668 /// arguments for the given function. This is also responsible for naming the
2669 /// LLVM function arguments.
2670 void EmitFunctionProlog(const CGFunctionInfo &FI, llvm::Function *Fn,
2671 const FunctionArgList &Args);
2672
2673 /// EmitFunctionEpilog - Emit the target specific LLVM code to return the
2674 /// given temporary. Specify the source location atom group (Key Instructions
2675 /// debug info feature) for the `ret` using \p RetKeyInstructionsSourceAtom.
2676 /// If it's 0, the `ret` will get added to a new source atom group.
2677 void EmitFunctionEpilog(const CGFunctionInfo &FI, bool EmitRetDbgLoc,
2678 SourceLocation EndLoc,
2679 uint64_t RetKeyInstructionsSourceAtom);
2680
2681 /// Emit a test that checks if the return value \p RV is nonnull.
2682 void EmitReturnValueCheck(llvm::Value *RV);
2683
2684 /// EmitStartEHSpec - Emit the start of the exception spec.
2685 void EmitStartEHSpec(const Decl *D);
2686
2687 /// EmitEndEHSpec - Emit the end of the exception spec.
2688 void EmitEndEHSpec(const Decl *D);
2689
2690 /// getTerminateLandingPad - Return a landing pad that just calls terminate.
2691 llvm::BasicBlock *getTerminateLandingPad();
2692
2693 /// getTerminateLandingPad - Return a cleanup funclet that just calls
2694 /// terminate.
2695 llvm::BasicBlock *getTerminateFunclet();
2696
2697 /// getTerminateHandler - Return a handler (not a landing pad, just
2698 /// a catch handler) that just calls terminate. This is used when
2699 /// a terminate scope encloses a try.
2700 llvm::BasicBlock *getTerminateHandler();
2701
2702 llvm::Type *ConvertTypeForMem(QualType T);
2703 llvm::Type *ConvertType(QualType T);
2704 llvm::Type *convertTypeForLoadStore(QualType ASTTy,
2705 llvm::Type *LLVMTy = nullptr);
2706 llvm::Type *ConvertType(const TypeDecl *T) {
2707 return ConvertType(T: getContext().getTypeDeclType(Decl: T));
2708 }
2709
2710 /// LoadObjCSelf - Load the value of self. This function is only valid while
2711 /// generating code for an Objective-C method.
2712 llvm::Value *LoadObjCSelf();
2713
2714 /// TypeOfSelfObject - Return type of object that this self represents.
2715 QualType TypeOfSelfObject();
2716
2717 /// getEvaluationKind - Return the TypeEvaluationKind of QualType \c T.
2718 static TypeEvaluationKind getEvaluationKind(QualType T);
2719
2720 static bool hasScalarEvaluationKind(QualType T) {
2721 return getEvaluationKind(T) == TEK_Scalar;
2722 }
2723
2724 static bool hasAggregateEvaluationKind(QualType T) {
2725 return getEvaluationKind(T) == TEK_Aggregate;
2726 }
2727
2728 /// createBasicBlock - Create an LLVM basic block.
2729 llvm::BasicBlock *createBasicBlock(const Twine &name = "",
2730 llvm::Function *parent = nullptr,
2731 llvm::BasicBlock *before = nullptr) {
2732 return llvm::BasicBlock::Create(Context&: getLLVMContext(), Name: name, Parent: parent, InsertBefore: before);
2733 }
2734
2735 /// getBasicBlockForLabel - Return the LLVM basicblock that the specified
2736 /// label maps to.
2737 JumpDest getJumpDestForLabel(const LabelDecl *S);
2738
2739 /// SimplifyForwardingBlocks - If the given basic block is only a branch to
2740 /// another basic block, simplify it. This assumes that no other code could
2741 /// potentially reference the basic block.
2742 void SimplifyForwardingBlocks(llvm::BasicBlock *BB);
2743
2744 /// EmitBlock - Emit the given block \arg BB and set it as the insert point,
2745 /// adding a fall-through branch from the current insert block if
2746 /// necessary. It is legal to call this function even if there is no current
2747 /// insertion point.
2748 ///
2749 /// IsFinished - If true, indicates that the caller has finished emitting
2750 /// branches to the given block and does not expect to emit code into it. This
2751 /// means the block can be ignored if it is unreachable.
2752 void EmitBlock(llvm::BasicBlock *BB, bool IsFinished = false);
2753
2754 /// EmitBlockAfterUses - Emit the given block somewhere hopefully
2755 /// near its uses, and leave the insertion point in it.
2756 void EmitBlockAfterUses(llvm::BasicBlock *BB);
2757
2758 /// EmitBranch - Emit a branch to the specified basic block from the current
2759 /// insert block, taking care to avoid creation of branches from dummy
2760 /// blocks. It is legal to call this function even if there is no current
2761 /// insertion point.
2762 ///
2763 /// This function clears the current insertion point. The caller should follow
2764 /// calls to this function with calls to Emit*Block prior to generation new
2765 /// code.
2766 void EmitBranch(llvm::BasicBlock *Block);
2767
2768 /// HaveInsertPoint - True if an insertion point is defined. If not, this
2769 /// indicates that the current code being emitted is unreachable.
2770 bool HaveInsertPoint() const { return Builder.GetInsertBlock() != nullptr; }
2771
2772 /// EnsureInsertPoint - Ensure that an insertion point is defined so that
2773 /// emitted IR has a place to go. Note that by definition, if this function
2774 /// creates a block then that block is unreachable; callers may do better to
2775 /// detect when no insertion point is defined and simply skip IR generation.
2776 void EnsureInsertPoint() {
2777 if (!HaveInsertPoint())
2778 EmitBlock(BB: createBasicBlock());
2779 }
2780
2781 /// ErrorUnsupported - Print out an error that codegen doesn't support the
2782 /// specified stmt yet.
2783 void ErrorUnsupported(const Stmt *S, const char *Type);
2784
2785 //===--------------------------------------------------------------------===//
2786 // Helpers
2787 //===--------------------------------------------------------------------===//
2788
2789 Address mergeAddressesInConditionalExpr(Address LHS, Address RHS,
2790 llvm::BasicBlock *LHSBlock,
2791 llvm::BasicBlock *RHSBlock,
2792 llvm::BasicBlock *MergeBlock,
2793 QualType MergedType) {
2794 Builder.SetInsertPoint(MergeBlock);
2795 llvm::PHINode *PtrPhi = Builder.CreatePHI(Ty: LHS.getType(), NumReservedValues: 2, Name: "cond");
2796 PtrPhi->addIncoming(V: LHS.getBasePointer(), BB: LHSBlock);
2797 PtrPhi->addIncoming(V: RHS.getBasePointer(), BB: RHSBlock);
2798 LHS.replaceBasePointer(P: PtrPhi);
2799 LHS.setAlignment(std::min(a: LHS.getAlignment(), b: RHS.getAlignment()));
2800 return LHS;
2801 }
2802
2803 /// Construct an address with the natural alignment of T. If a pointer to T
2804 /// is expected to be signed, the pointer passed to this function must have
2805 /// been signed, and the returned Address will have the pointer authentication
2806 /// information needed to authenticate the signed pointer.
2807 Address makeNaturalAddressForPointer(
2808 llvm::Value *Ptr, QualType T, CharUnits Alignment = CharUnits::Zero(),
2809 bool ForPointeeType = false, LValueBaseInfo *BaseInfo = nullptr,
2810 TBAAAccessInfo *TBAAInfo = nullptr,
2811 KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
2812 if (Alignment.isZero())
2813 Alignment =
2814 CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, forPointeeType: ForPointeeType);
2815 return Address(Ptr, ConvertTypeForMem(T), Alignment,
2816 CGM.getPointerAuthInfoForPointeeType(type: T), /*Offset=*/nullptr,
2817 IsKnownNonNull);
2818 }
2819
2820 LValue MakeAddrLValue(Address Addr, QualType T,
2821 AlignmentSource Source = AlignmentSource::Type) {
2822 return MakeAddrLValue(Addr, T, BaseInfo: LValueBaseInfo(Source),
2823 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2824 }
2825
2826 LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo,
2827 TBAAAccessInfo TBAAInfo) {
2828 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo, TBAAInfo);
2829 }
2830
2831 LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2832 AlignmentSource Source = AlignmentSource::Type) {
2833 return MakeAddrLValue(Addr: makeNaturalAddressForPointer(Ptr: V, T, Alignment), T,
2834 BaseInfo: LValueBaseInfo(Source), TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2835 }
2836
2837 /// Same as MakeAddrLValue above except that the pointer is known to be
2838 /// unsigned.
2839 LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2840 AlignmentSource Source = AlignmentSource::Type) {
2841 Address Addr(V, ConvertTypeForMem(T), Alignment);
2842 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2843 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2844 }
2845
2846 LValue
2847 MakeAddrLValueWithoutTBAA(Address Addr, QualType T,
2848 AlignmentSource Source = AlignmentSource::Type) {
2849 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2850 TBAAInfo: TBAAAccessInfo());
2851 }
2852
2853 /// Given a value of type T* that may not be to a complete object, construct
2854 /// an l-value with the natural pointee alignment of T.
2855 LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T);
2856
2857 LValue
2858 MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
2859 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
2860
2861 /// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known
2862 /// to be unsigned.
2863 LValue MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, QualType T);
2864
2865 LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T);
2866
2867 Address EmitLoadOfReference(LValue RefLVal,
2868 LValueBaseInfo *PointeeBaseInfo = nullptr,
2869 TBAAAccessInfo *PointeeTBAAInfo = nullptr);
2870 LValue EmitLoadOfReferenceLValue(LValue RefLVal);
2871 LValue
2872 EmitLoadOfReferenceLValue(Address RefAddr, QualType RefTy,
2873 AlignmentSource Source = AlignmentSource::Type) {
2874 LValue RefLVal = MakeAddrLValue(Addr: RefAddr, T: RefTy, BaseInfo: LValueBaseInfo(Source),
2875 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: RefTy));
2876 return EmitLoadOfReferenceLValue(RefLVal);
2877 }
2878
2879 /// Load a pointer with type \p PtrTy stored at address \p Ptr.
2880 /// Note that \p PtrTy is the type of the loaded pointer, not the addresses
2881 /// it is loaded from.
2882 Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy,
2883 LValueBaseInfo *BaseInfo = nullptr,
2884 TBAAAccessInfo *TBAAInfo = nullptr);
2885 LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy);
2886
2887private:
2888 struct AllocaTracker {
2889 void Add(llvm::AllocaInst *I) { Allocas.push_back(Elt: I); }
2890 llvm::SmallVector<llvm::AllocaInst *> Take() { return std::move(Allocas); }
2891
2892 private:
2893 llvm::SmallVector<llvm::AllocaInst *> Allocas;
2894 };
2895 AllocaTracker *Allocas = nullptr;
2896
2897 /// CGDecl helper.
2898 void emitStoresForConstant(const VarDecl &D, Address Loc, bool isVolatile,
2899 llvm::Constant *constant, bool IsAutoInit);
2900 /// CGDecl helper.
2901 void emitStoresForZeroInit(const VarDecl &D, Address Loc, bool isVolatile);
2902 /// CGDecl helper.
2903 void emitStoresForPatternInit(const VarDecl &D, Address Loc, bool isVolatile);
2904 /// CGDecl helper.
2905 void emitStoresForInitAfterBZero(llvm::Constant *Init, Address Loc,
2906 bool isVolatile, bool IsAutoInit);
2907
2908public:
2909 // Captures all the allocas created during the scope of its RAII object.
2910 struct AllocaTrackerRAII {
2911 AllocaTrackerRAII(CodeGenFunction &CGF)
2912 : CGF(CGF), OldTracker(CGF.Allocas) {
2913 CGF.Allocas = &Tracker;
2914 }
2915 ~AllocaTrackerRAII() { CGF.Allocas = OldTracker; }
2916
2917 llvm::SmallVector<llvm::AllocaInst *> Take() { return Tracker.Take(); }
2918
2919 private:
2920 CodeGenFunction &CGF;
2921 AllocaTracker *OldTracker;
2922 AllocaTracker Tracker;
2923 };
2924
2925private:
2926 /// If \p Alloca is not in the same address space as \p DestLangAS, insert an
2927 /// address space cast and return a new RawAddress based on this value.
2928 RawAddress MaybeCastStackAddressSpace(RawAddress Alloca, LangAS DestLangAS,
2929 llvm::Value *ArraySize = nullptr);
2930
2931public:
2932 /// CreateTempAlloca - This creates an alloca and inserts it into the entry
2933 /// block if \p ArraySize is nullptr, otherwise inserts it at the current
2934 /// insertion point of the builder. The caller is responsible for setting an
2935 /// appropriate alignment on the alloca.
2936 ///
2937 /// \p ArraySize is the number of array elements to be allocated if it
2938 /// is not nullptr.
2939 ///
2940 /// LangAS::Default is the address space of pointers to local variables and
2941 /// temporaries, as exposed in the source language. In certain
2942 /// configurations, this is not the same as the alloca address space, and a
2943 /// cast is needed to lift the pointer from the alloca AS into
2944 /// LangAS::Default. This can happen when the target uses a restricted
2945 /// address space for the stack but the source language requires
2946 /// LangAS::Default to be a generic address space. The latter condition is
2947 /// common for most programming languages; OpenCL is an exception in that
2948 /// LangAS::Default is the private address space, which naturally maps
2949 /// to the stack.
2950 ///
2951 /// Because the address of a temporary is often exposed to the program in
2952 /// various ways, this function will perform the cast. The original alloca
2953 /// instruction is returned through \p Alloca if it is not nullptr.
2954 ///
2955 /// The cast is not performed in CreateTempAllocaWithoutCast. This is
2956 /// more efficient if the caller knows that the address will not be exposed.
2957 llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp",
2958 llvm::Value *ArraySize = nullptr);
2959
2960 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
2961 /// block. The alloca is casted to the address space of \p UseAddrSpace if
2962 /// necessary.
2963 RawAddress CreateTempAlloca(llvm::Type *Ty, LangAS UseAddrSpace,
2964 CharUnits align, const Twine &Name = "tmp",
2965 llvm::Value *ArraySize = nullptr,
2966 RawAddress *Alloca = nullptr);
2967
2968 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
2969 /// block. The alloca is casted to default address space if necessary.
2970 ///
2971 /// FIXME: This version should be removed, and context should provide the
2972 /// context use address space used instead of default.
2973 RawAddress CreateTempAlloca(llvm::Type *Ty, CharUnits align,
2974 const Twine &Name = "tmp",
2975 llvm::Value *ArraySize = nullptr,
2976 RawAddress *Alloca = nullptr) {
2977 return CreateTempAlloca(Ty, UseAddrSpace: LangAS::Default, align, Name, ArraySize,
2978 Alloca);
2979 }
2980
2981 RawAddress CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align,
2982 const Twine &Name = "tmp",
2983 llvm::Value *ArraySize = nullptr);
2984
2985 /// CreateDefaultAlignedTempAlloca - This creates an alloca with the
2986 /// default ABI alignment of the given LLVM type.
2987 ///
2988 /// IMPORTANT NOTE: This is *not* generally the right alignment for
2989 /// any given AST type that happens to have been lowered to the
2990 /// given IR type. This should only ever be used for function-local,
2991 /// IR-driven manipulations like saving and restoring a value. Do
2992 /// not hand this address off to arbitrary IRGen routines, and especially
2993 /// do not pass it as an argument to a function that might expect a
2994 /// properly ABI-aligned value.
2995 RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty,
2996 const Twine &Name = "tmp");
2997
2998 /// CreateIRTempWithoutCast - Create a temporary IR object of the given type,
2999 /// with appropriate alignment. This routine should only be used when an
3000 /// temporary value needs to be stored into an alloca (for example, to avoid
3001 /// explicit PHI construction), but the type is the IR type, not the type
3002 /// appropriate for storing in memory.
3003 ///
3004 /// That is, this is exactly equivalent to CreateMemTemp, but calling
3005 /// ConvertType instead of ConvertTypeForMem.
3006 RawAddress CreateIRTempWithoutCast(QualType T, const Twine &Name = "tmp");
3007
3008 /// CreateMemTemp - Create a temporary memory object of the given type, with
3009 /// appropriate alignmen and cast it to the default address space. Returns
3010 /// the original alloca instruction by \p Alloca if it is not nullptr.
3011 RawAddress CreateMemTemp(QualType T, const Twine &Name = "tmp",
3012 RawAddress *Alloca = nullptr);
3013 RawAddress CreateMemTemp(QualType T, CharUnits Align,
3014 const Twine &Name = "tmp",
3015 RawAddress *Alloca = nullptr);
3016
3017 /// CreateMemTemp - Create a temporary memory object of the given type, with
3018 /// appropriate alignmen without casting it to the default address space.
3019 RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp");
3020 RawAddress CreateMemTempWithoutCast(QualType T, CharUnits Align,
3021 const Twine &Name = "tmp");
3022
3023 /// CreateAggTemp - Create a temporary memory object for the given
3024 /// aggregate type.
3025 AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp",
3026 RawAddress *Alloca = nullptr) {
3027 RawAddress Addr = CreateMemTemp(T, Name, Alloca);
3028 return AggValueSlot::forAddr(
3029 addr: Addr, quals: T.getQualifiers(), isDestructed: AggValueSlot::IsNotDestructed,
3030 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
3031 mayOverlap: AggValueSlot::DoesNotOverlap);
3032 }
3033
3034 /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
3035 /// expression and compare the result against zero, returning an Int1Ty value.
3036 llvm::Value *EvaluateExprAsBool(const Expr *E);
3037
3038 /// Retrieve the implicit cast expression of the rhs in a binary operator
3039 /// expression by passing pointers to Value and QualType
3040 /// This is used for implicit bitfield conversion checks, which
3041 /// must compare with the value before potential truncation.
3042 llvm::Value *EmitWithOriginalRHSBitfieldAssignment(const BinaryOperator *E,
3043 llvm::Value **Previous,
3044 QualType *SrcType);
3045
3046 /// Emit a check that an [implicit] conversion of a bitfield. It is not UB,
3047 /// so we use the value after conversion.
3048 void EmitBitfieldConversionCheck(llvm::Value *Src, QualType SrcType,
3049 llvm::Value *Dst, QualType DstType,
3050 const CGBitFieldInfo &Info,
3051 SourceLocation Loc);
3052
3053 /// EmitIgnoredExpr - Emit an expression in a context which ignores the
3054 /// result.
3055 void EmitIgnoredExpr(const Expr *E);
3056
3057 /// EmitAnyExpr - Emit code to compute the specified expression which can have
3058 /// any type. The result is returned as an RValue struct. If this is an
3059 /// aggregate expression, the aggloc/agglocvolatile arguments indicate where
3060 /// the result should be returned.
3061 ///
3062 /// \param ignoreResult True if the resulting value isn't used.
3063 RValue EmitAnyExpr(const Expr *E,
3064 AggValueSlot aggSlot = AggValueSlot::ignored(),
3065 bool ignoreResult = false);
3066
3067 // EmitVAListRef - Emit a "reference" to a va_list; this is either the address
3068 // or the value of the expression, depending on how va_list is defined.
3069 Address EmitVAListRef(const Expr *E);
3070
3071 /// Emit a "reference" to a __builtin_ms_va_list; this is
3072 /// always the value of the expression, because a __builtin_ms_va_list is a
3073 /// pointer to a char.
3074 Address EmitMSVAListRef(const Expr *E);
3075
3076 /// Emit a "reference" to a __builtin_zos_va_list; this is always the
3077 /// address of the expression, because a __builtin_zos_va_list is an
3078 /// array of pointer to a char.
3079 Address EmitZOSVAListRef(const Expr *E);
3080
3081 /// EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will
3082 /// always be accessible even if no aggregate location is provided.
3083 RValue EmitAnyExprToTemp(const Expr *E);
3084
3085 /// EmitAnyExprToMem - Emits the code necessary to evaluate an
3086 /// arbitrary expression into the given memory location.
3087 void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals,
3088 bool IsInitializer);
3089
3090 void EmitAnyExprToExn(const Expr *E, Address Addr);
3091
3092 /// EmitInitializationToLValue - Emit an initializer to an LValue.
3093 void EmitInitializationToLValue(
3094 const Expr *E, LValue LV,
3095 AggValueSlot::IsZeroed_t IsZeroed = AggValueSlot::IsNotZeroed);
3096
3097 /// EmitExprAsInit - Emits the code necessary to initialize a
3098 /// location in memory with the given initializer.
3099 void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue,
3100 bool capturedByInit);
3101
3102 /// hasVolatileMember - returns true if aggregate type has a volatile
3103 /// member.
3104 bool hasVolatileMember(QualType T) {
3105 if (const auto *RD = T->getAsRecordDecl())
3106 return RD->hasVolatileMember();
3107 return false;
3108 }
3109
3110 /// Determine whether a return value slot may overlap some other object.
3111 AggValueSlot::Overlap_t getOverlapForReturnValue() {
3112 // FIXME: Assuming no overlap here breaks guaranteed copy elision for base
3113 // class subobjects. These cases may need to be revisited depending on the
3114 // resolution of the relevant core issue.
3115 return AggValueSlot::DoesNotOverlap;
3116 }
3117
3118 /// Determine whether a field initialization may overlap some other object.
3119 AggValueSlot::Overlap_t getOverlapForFieldInit(const FieldDecl *FD);
3120
3121 /// Determine whether a base class initialization may overlap some other
3122 /// object.
3123 AggValueSlot::Overlap_t getOverlapForBaseInit(const CXXRecordDecl *RD,
3124 const CXXRecordDecl *BaseRD,
3125 bool IsVirtual);
3126
3127 /// Emit an aggregate assignment.
3128 void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy) {
3129 ApplyAtomGroup Grp(getDebugInfo());
3130 bool IsVolatile = hasVolatileMember(T: EltTy);
3131 EmitAggregateCopy(Dest, Src, EltTy, MayOverlap: AggValueSlot::MayOverlap, isVolatile: IsVolatile);
3132 }
3133
3134 void EmitAggregateCopyCtor(LValue Dest, LValue Src,
3135 AggValueSlot::Overlap_t MayOverlap) {
3136 EmitAggregateCopy(Dest, Src, EltTy: Src.getType(), MayOverlap);
3137 }
3138
3139 /// EmitAggregateCopy - Emit an aggregate copy.
3140 ///
3141 /// \param isVolatile \c true iff either the source or the destination is
3142 /// volatile.
3143 /// \param MayOverlap Whether the tail padding of the destination might be
3144 /// occupied by some other object. More efficient code can often be
3145 /// generated if not.
3146 void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy,
3147 AggValueSlot::Overlap_t MayOverlap,
3148 bool isVolatile = false);
3149
3150 /// GetAddrOfLocalVar - Return the address of a local variable.
3151 Address GetAddrOfLocalVar(const VarDecl *VD) {
3152 auto it = LocalDeclMap.find(Val: VD);
3153 assert(it != LocalDeclMap.end() &&
3154 "Invalid argument to GetAddrOfLocalVar(), no decl!");
3155 return it->second;
3156 }
3157
3158 /// Given an opaque value expression, return its LValue mapping if it exists,
3159 /// otherwise create one.
3160 LValue getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e);
3161
3162 /// Given an opaque value expression, return its RValue mapping if it exists,
3163 /// otherwise create one.
3164 RValue getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e);
3165
3166 /// isOpaqueValueEmitted - Return true if the opaque value expression has
3167 /// already been emitted.
3168 bool isOpaqueValueEmitted(const OpaqueValueExpr *E);
3169
3170 /// Get the index of the current ArrayInitLoopExpr, if any.
3171 llvm::Value *getArrayInitIndex() { return ArrayInitIndex; }
3172
3173 /// getAccessedFieldNo - Given an encoded value and a result number, return
3174 /// the input field number being accessed.
3175 static unsigned getAccessedFieldNo(unsigned Idx, const llvm::Constant *Elts);
3176
3177 llvm::BlockAddress *GetAddrOfLabel(const LabelDecl *L);
3178 llvm::BasicBlock *GetIndirectGotoBlock();
3179
3180 /// Check if \p E is a C++ "this" pointer wrapped in value-preserving casts.
3181 static bool IsWrappedCXXThis(const Expr *E);
3182
3183 /// EmitNullInitialization - Generate code to set a value of the given type to
3184 /// null, If the type contains data member pointers, they will be initialized
3185 /// to -1 in accordance with the Itanium C++ ABI.
3186 void EmitNullInitialization(Address DestPtr, QualType Ty);
3187
3188 /// Emits a call to an LLVM variable-argument intrinsic, either
3189 /// \c llvm.va_start or \c llvm.va_end.
3190 /// \param ArgValue A reference to the \c va_list as emitted by either
3191 /// \c EmitVAListRef or \c EmitMSVAListRef.
3192 /// \param IsStart If \c true, emits a call to \c llvm.va_start; otherwise,
3193 /// calls \c llvm.va_end.
3194 llvm::Value *EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart);
3195
3196 /// Generate code to get an argument from the passed in pointer
3197 /// and update it accordingly.
3198 /// \param VE The \c VAArgExpr for which to generate code.
3199 /// \param VAListAddr Receives a reference to the \c va_list as emitted by
3200 /// either \c EmitVAListRef or \c EmitMSVAListRef.
3201 /// \returns A pointer to the argument.
3202 // FIXME: We should be able to get rid of this method and use the va_arg
3203 // instruction in LLVM instead once it works well enough.
3204 RValue EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
3205 AggValueSlot Slot = AggValueSlot::ignored());
3206
3207 /// emitArrayLength - Compute the length of an array, even if it's a
3208 /// VLA, and drill down to the base element type.
3209 llvm::Value *emitArrayLength(const ArrayType *arrayType, QualType &baseType,
3210 Address &addr);
3211
3212 /// EmitVLASize - Capture all the sizes for the VLA expressions in
3213 /// the given variably-modified type and store them in the VLASizeMap.
3214 ///
3215 /// This function can be called with a null (unreachable) insert point.
3216 void EmitVariablyModifiedType(QualType Ty);
3217
3218 struct VlaSizePair {
3219 llvm::Value *NumElts;
3220 QualType Type;
3221
3222 VlaSizePair(llvm::Value *NE, QualType T) : NumElts(NE), Type(T) {}
3223 };
3224
3225 /// Return the number of elements for a single dimension
3226 /// for the given array type.
3227 VlaSizePair getVLAElements1D(const VariableArrayType *vla);
3228 VlaSizePair getVLAElements1D(QualType vla);
3229
3230 /// Returns an LLVM value that corresponds to the size,
3231 /// in non-variably-sized elements, of a variable length array type,
3232 /// plus that largest non-variably-sized element type. Assumes that
3233 /// the type has already been emitted with EmitVariablyModifiedType.
3234 VlaSizePair getVLASize(const VariableArrayType *vla);
3235 VlaSizePair getVLASize(QualType vla);
3236
3237 /// LoadCXXThis - Load the value of 'this'. This function is only valid while
3238 /// generating code for an C++ member function.
3239 llvm::Value *LoadCXXThis() {
3240 assert(CXXThisValue && "no 'this' value for this function");
3241 return CXXThisValue;
3242 }
3243 Address LoadCXXThisAddress();
3244
3245 /// LoadCXXVTT - Load the VTT parameter to base constructors/destructors have
3246 /// virtual bases.
3247 // FIXME: Every place that calls LoadCXXVTT is something
3248 // that needs to be abstracted properly.
3249 llvm::Value *LoadCXXVTT() {
3250 assert(CXXStructorImplicitParamValue && "no VTT value for this function");
3251 return CXXStructorImplicitParamValue;
3252 }
3253
3254 /// GetAddressOfBaseOfCompleteClass - Convert the given pointer to a
3255 /// complete class to the given direct base.
3256 Address GetAddressOfDirectBaseInCompleteClass(Address Value,
3257 const CXXRecordDecl *Derived,
3258 const CXXRecordDecl *Base,
3259 bool BaseIsVirtual);
3260
3261 static bool ShouldNullCheckClassCastValue(const CastExpr *Cast);
3262
3263 /// GetAddressOfBaseClass - This function will add the necessary delta to the
3264 /// load of 'this' and returns address of the base class.
3265 Address GetAddressOfBaseClass(Address Value, const CXXRecordDecl *Derived,
3266 CastExpr::path_const_iterator PathBegin,
3267 CastExpr::path_const_iterator PathEnd,
3268 bool NullCheckValue, SourceLocation Loc);
3269
3270 Address GetAddressOfDerivedClass(Address Value, const CXXRecordDecl *Derived,
3271 CastExpr::path_const_iterator PathBegin,
3272 CastExpr::path_const_iterator PathEnd,
3273 bool NullCheckValue);
3274
3275 /// GetVTTParameter - Return the VTT parameter that should be passed to a
3276 /// base constructor/destructor with virtual bases.
3277 /// FIXME: VTTs are Itanium ABI-specific, so the definition should move
3278 /// to ItaniumCXXABI.cpp together with all the references to VTT.
3279 llvm::Value *GetVTTParameter(GlobalDecl GD, bool ForVirtualBase,
3280 bool Delegating);
3281
3282 void EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
3283 CXXCtorType CtorType,
3284 const FunctionArgList &Args,
3285 SourceLocation Loc);
3286 // It's important not to confuse this and the previous function. Delegating
3287 // constructors are the C++0x feature. The constructor delegate optimization
3288 // is used to reduce duplication in the base and complete consturctors where
3289 // they are substantially the same.
3290 void EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3291 const FunctionArgList &Args);
3292
3293 /// Emit a call to an inheriting constructor (that is, one that invokes a
3294 /// constructor inherited from a base class) by inlining its definition. This
3295 /// is necessary if the ABI does not support forwarding the arguments to the
3296 /// base class constructor (because they're variadic or similar).
3297 void EmitInlinedInheritingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3298 CXXCtorType CtorType,
3299 bool ForVirtualBase,
3300 bool Delegating,
3301 CallArgList &Args);
3302
3303 /// Emit a call to a constructor inherited from a base class, passing the
3304 /// current constructor's arguments along unmodified (without even making
3305 /// a copy).
3306 void EmitInheritedCXXConstructorCall(const CXXConstructorDecl *D,
3307 bool ForVirtualBase, Address This,
3308 bool InheritedFromVBase,
3309 const CXXInheritedCtorInitExpr *E);
3310
3311 void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3312 bool ForVirtualBase, bool Delegating,
3313 AggValueSlot ThisAVS, const CXXConstructExpr *E);
3314
3315 void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3316 bool ForVirtualBase, bool Delegating,
3317 Address This, CallArgList &Args,
3318 AggValueSlot::Overlap_t Overlap,
3319 SourceLocation Loc, bool NewPointerIsChecked,
3320 llvm::CallBase **CallOrInvoke = nullptr);
3321
3322 /// Emit assumption load for all bases. Requires to be called only on
3323 /// most-derived class and not under construction of the object.
3324 void EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl, Address This);
3325
3326 /// Emit assumption that vptr load == global vtable.
3327 void EmitVTableAssumptionLoad(const VPtr &vptr, Address This);
3328
3329 void EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, Address This,
3330 Address Src, const CXXConstructExpr *E);
3331
3332 void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3333 const ArrayType *ArrayTy, Address ArrayPtr,
3334 const CXXConstructExpr *E,
3335 bool NewPointerIsChecked,
3336 bool ZeroInitialization = false);
3337
3338 void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3339 llvm::Value *NumElements, Address ArrayPtr,
3340 const CXXConstructExpr *E,
3341 bool NewPointerIsChecked,
3342 bool ZeroInitialization = false);
3343
3344 static Destroyer destroyCXXObject;
3345
3346 void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type,
3347 bool ForVirtualBase, bool Delegating, Address This,
3348 QualType ThisTy);
3349
3350 void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
3351 llvm::Type *ElementTy, Address NewPtr,
3352 llvm::Value *NumElements,
3353 llvm::Value *AllocSizeWithoutCookie);
3354
3355 void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
3356 Address Ptr);
3357
3358 void EmitSehCppScopeBegin();
3359 void EmitSehCppScopeEnd();
3360 void EmitSehTryScopeBegin();
3361 void EmitSehTryScopeEnd();
3362
3363 bool EmitLifetimeStart(llvm::Value *Addr);
3364 void EmitLifetimeEnd(llvm::Value *Addr);
3365
3366 llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
3367 void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
3368
3369 void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,
3370 QualType DeleteTy, llvm::Value *NumElements = nullptr,
3371 CharUnits CookieSize = CharUnits(),
3372 llvm::Constant *CalleeOverride = nullptr);
3373
3374 RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
3375 const CallExpr *TheCallExpr, bool IsDelete);
3376
3377 llvm::Value *EmitCXXTypeidExpr(const CXXTypeidExpr *E);
3378 llvm::Value *EmitDynamicCast(Address V, const CXXDynamicCastExpr *DCE);
3379 Address EmitCXXUuidofExpr(const CXXUuidofExpr *E);
3380
3381 /// Situations in which we might emit a check for the suitability of a
3382 /// pointer or glvalue. Needs to be kept in sync with ubsan_handlers.cpp in
3383 /// compiler-rt.
3384 enum TypeCheckKind {
3385 /// Checking the operand of a load. Must be suitably sized and aligned.
3386 TCK_Load,
3387 /// Checking the destination of a store. Must be suitably sized and aligned.
3388 TCK_Store,
3389 /// Checking the bound value in a reference binding. Must be suitably sized
3390 /// and aligned, but is not required to refer to an object (until the
3391 /// reference is used), per core issue 453.
3392 TCK_ReferenceBinding,
3393 /// Checking the object expression in a non-static data member access. Must
3394 /// be an object within its lifetime.
3395 TCK_MemberAccess,
3396 /// Checking the 'this' pointer for a call to a non-static member function.
3397 /// Must be an object within its lifetime.
3398 TCK_MemberCall,
3399 /// Checking the 'this' pointer for a constructor call.
3400 TCK_ConstructorCall,
3401 /// Checking the operand of a static_cast to a derived pointer type. Must be
3402 /// null or an object within its lifetime.
3403 TCK_DowncastPointer,
3404 /// Checking the operand of a static_cast to a derived reference type. Must
3405 /// be an object within its lifetime.
3406 TCK_DowncastReference,
3407 /// Checking the operand of a cast to a base object. Must be suitably sized
3408 /// and aligned.
3409 TCK_Upcast,
3410 /// Checking the operand of a cast to a virtual base object. Must be an
3411 /// object within its lifetime.
3412 TCK_UpcastToVirtualBase,
3413 /// Checking the value assigned to a _Nonnull pointer. Must not be null.
3414 TCK_NonnullAssign,
3415 /// Checking the operand of a dynamic_cast or a typeid expression. Must be
3416 /// null or an object within its lifetime.
3417 TCK_DynamicOperation
3418 };
3419
3420 /// Determine whether the pointer type check \p TCK permits null pointers.
3421 static bool isNullPointerAllowed(TypeCheckKind TCK);
3422
3423 /// Determine whether the pointer type check \p TCK requires a vptr check.
3424 static bool isVptrCheckRequired(TypeCheckKind TCK, QualType Ty);
3425
3426 /// Whether any type-checking sanitizers are enabled. If \c false,
3427 /// calls to EmitTypeCheck can be skipped.
3428 bool sanitizePerformTypeCheck() const;
3429
3430 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV,
3431 QualType Type, SanitizerSet SkippedChecks = SanitizerSet(),
3432 llvm::Value *ArraySize = nullptr) {
3433 if (!sanitizePerformTypeCheck())
3434 return;
3435 EmitTypeCheck(TCK, Loc, V: LV.emitRawPointer(CGF&: *this), Type, Alignment: LV.getAlignment(),
3436 SkippedChecks, ArraySize);
3437 }
3438
3439 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, Address Addr,
3440 QualType Type, CharUnits Alignment = CharUnits::Zero(),
3441 SanitizerSet SkippedChecks = SanitizerSet(),
3442 llvm::Value *ArraySize = nullptr) {
3443 if (!sanitizePerformTypeCheck())
3444 return;
3445 EmitTypeCheck(TCK, Loc, V: Addr.emitRawPointer(CGF&: *this), Type, Alignment,
3446 SkippedChecks, ArraySize);
3447 }
3448
3449 /// Emit a check that \p V is the address of storage of the
3450 /// appropriate size and alignment for an object of type \p Type
3451 /// (or if ArraySize is provided, for an array of that bound).
3452 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Value *V,
3453 QualType Type, CharUnits Alignment = CharUnits::Zero(),
3454 SanitizerSet SkippedChecks = SanitizerSet(),
3455 llvm::Value *ArraySize = nullptr);
3456
3457 /// Emit a check that \p Base points into an array object, which
3458 /// we can access at index \p Index. \p Accessed should be \c false if we
3459 /// this expression is used as an lvalue, for instance in "&Arr[Idx]".
3460 void EmitBoundsCheck(const Expr *ArrayExpr, const Expr *ArrayExprBase,
3461 llvm::Value *Index, QualType IndexType, bool Accessed);
3462 void EmitBoundsCheckImpl(const Expr *ArrayExpr, QualType ArrayBaseType,
3463 llvm::Value *IndexVal, QualType IndexType,
3464 llvm::Value *BoundsVal, QualType BoundsType,
3465 bool Accessed);
3466
3467 /// Returns debug info, with additional annotation if
3468 /// CGM.getCodeGenOpts().SanitizeAnnotateDebugInfo[Ordinal] is enabled for
3469 /// any of the ordinals.
3470 llvm::DILocation *
3471 SanitizerAnnotateDebugInfo(ArrayRef<SanitizerKind::SanitizerOrdinal> Ordinals,
3472 SanitizerHandler Handler);
3473
3474 /// Build metadata used by the AllocToken instrumentation.
3475 llvm::MDNode *buildAllocToken(QualType AllocType);
3476 /// Emit and set additional metadata used by the AllocToken instrumentation.
3477 void EmitAllocToken(llvm::CallBase *CB, QualType AllocType);
3478 /// Build additional metadata used by the AllocToken instrumentation,
3479 /// inferring the type from an allocation call expression.
3480 llvm::MDNode *buildAllocToken(const CallExpr *E);
3481 /// Emit and set additional metadata used by the AllocToken instrumentation,
3482 /// inferring the type from an allocation call expression.
3483 void EmitAllocToken(llvm::CallBase *CB, const CallExpr *E);
3484
3485 llvm::Value *GetCountedByFieldExprGEP(const Expr *Base, const FieldDecl *FD,
3486 const FieldDecl *CountDecl);
3487
3488 /// Build an expression accessing the "counted_by" field.
3489 llvm::Value *EmitLoadOfCountedByField(const Expr *Base, const FieldDecl *FD,
3490 const FieldDecl *CountDecl);
3491
3492 // Emit bounds checking for flexible array and pointer members with the
3493 // counted_by attribute.
3494 void EmitCountedByBoundsChecking(const Expr *ArrayExpr, QualType ArrayType,
3495 Address ArrayInst, QualType IndexType,
3496 llvm::Value *IndexVal, bool Accessed,
3497 bool FlexibleArray);
3498
3499 llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
3500 bool isInc, bool isPre);
3501 ComplexPairTy EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
3502 bool isInc, bool isPre);
3503
3504 /// Converts Location to a DebugLoc, if debug information is enabled.
3505 llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location);
3506
3507 /// Get the record field index as represented in debug info.
3508 unsigned getDebugInfoFIndex(const RecordDecl *Rec, unsigned FieldIndex);
3509
3510 //===--------------------------------------------------------------------===//
3511 // Declaration Emission
3512 //===--------------------------------------------------------------------===//
3513
3514 /// EmitDecl - Emit a declaration.
3515 ///
3516 /// This function can be called with a null (unreachable) insert point.
3517 void EmitDecl(const Decl &D, bool EvaluateConditionDecl = false);
3518
3519 /// EmitVarDecl - Emit a local variable declaration.
3520 ///
3521 /// This function can be called with a null (unreachable) insert point.
3522 void EmitVarDecl(const VarDecl &D);
3523
3524 void EmitScalarInit(const Expr *init, const ValueDecl *D, LValue lvalue,
3525 bool capturedByInit);
3526
3527 typedef void SpecialInitFn(CodeGenFunction &Init, const VarDecl &D,
3528 llvm::Value *Address);
3529
3530 /// Determine whether the given initializer is trivial in the sense
3531 /// that it requires no code to be generated.
3532 bool isTrivialInitializer(const Expr *Init);
3533
3534 /// EmitAutoVarDecl - Emit an auto variable declaration.
3535 ///
3536 /// This function can be called with a null (unreachable) insert point.
3537 void EmitAutoVarDecl(const VarDecl &D);
3538
3539 class AutoVarEmission {
3540 friend class CodeGenFunction;
3541
3542 const VarDecl *Variable;
3543
3544 /// The address of the alloca for languages with explicit address space
3545 /// (e.g. OpenCL) or alloca casted to generic pointer for address space
3546 /// agnostic languages (e.g. C++). Invalid if the variable was emitted
3547 /// as a global constant.
3548 Address Addr;
3549
3550 llvm::Value *NRVOFlag;
3551
3552 /// True if the variable is a __block variable that is captured by an
3553 /// escaping block.
3554 bool IsEscapingByRef;
3555
3556 /// If the variable is of aggregate type and has a constant initializer,
3557 /// a constant representing that initializer.
3558 llvm::Constant *ConstantAggregateInitializer;
3559
3560 /// True if lifetime markers should be used.
3561 bool UseLifetimeMarkers;
3562
3563 /// Address with original alloca instruction. Invalid if the variable was
3564 /// emitted as a global constant.
3565 RawAddress AllocaAddr;
3566
3567 struct Invalid {};
3568 AutoVarEmission(Invalid)
3569 : Variable(nullptr), Addr(Address::invalid()),
3570 AllocaAddr(RawAddress::invalid()) {}
3571
3572 AutoVarEmission(const VarDecl &variable)
3573 : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr),
3574 IsEscapingByRef(false), ConstantAggregateInitializer(nullptr),
3575 UseLifetimeMarkers(false), AllocaAddr(RawAddress::invalid()) {}
3576
3577 bool wasEmittedAsGlobal() const { return !Addr.isValid(); }
3578
3579 public:
3580 static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }
3581
3582 bool useLifetimeMarkers() const { return UseLifetimeMarkers; }
3583
3584 /// Returns the raw, allocated address, which is not necessarily
3585 /// the address of the object itself. It is casted to default
3586 /// address space for address space agnostic languages.
3587 Address getAllocatedAddress() const { return Addr; }
3588
3589 /// Returns the address for the original alloca instruction.
3590 RawAddress getOriginalAllocatedAddress() const { return AllocaAddr; }
3591
3592 /// Returns the address of the object within this declaration.
3593 /// Note that this does not chase the forwarding pointer for
3594 /// __block decls.
3595 Address getObjectAddress(CodeGenFunction &CGF) const {
3596 if (!IsEscapingByRef)
3597 return Addr;
3598
3599 return CGF.emitBlockByrefAddress(baseAddr: Addr, V: Variable, /*forward*/ followForward: false);
3600 }
3601 };
3602 AutoVarEmission EmitAutoVarAlloca(const VarDecl &var);
3603 void EmitAutoVarInit(const AutoVarEmission &emission);
3604 void EmitAutoVarCleanups(const AutoVarEmission &emission);
3605 void emitAutoVarTypeCleanup(const AutoVarEmission &emission,
3606 QualType::DestructionKind dtorKind);
3607
3608 /// Re-emit trivial-auto-var-init stores for variables bypassed by the jump
3609 /// Source. No-op in a function containing a computed goto, where jump sources
3610 /// are unknown and a single function-scope init is used instead.
3611 void emitBypassedVarInitsForSource(const Stmt *Source);
3612
3613 void MaybeEmitDeferredVarDeclInit(const VarDecl *var);
3614
3615 /// Emits the alloca and debug information for the size expressions for each
3616 /// dimension of an array. It registers the association of its (1-dimensional)
3617 /// QualTypes and size expression's debug node, so that CGDebugInfo can
3618 /// reference this node when creating the DISubrange object to describe the
3619 /// array types.
3620 void EmitAndRegisterVariableArrayDimensions(CGDebugInfo *DI, const VarDecl &D,
3621 bool EmitDebugInfo);
3622
3623 void EmitStaticVarDecl(const VarDecl &D,
3624 llvm::GlobalValue::LinkageTypes Linkage);
3625
3626 class ParamValue {
3627 union {
3628 Address Addr;
3629 llvm::Value *Value;
3630 };
3631
3632 bool IsIndirect;
3633
3634 ParamValue(llvm::Value *V) : Value(V), IsIndirect(false) {}
3635 ParamValue(Address A) : Addr(A), IsIndirect(true) {}
3636
3637 public:
3638 static ParamValue forDirect(llvm::Value *value) {
3639 return ParamValue(value);
3640 }
3641 static ParamValue forIndirect(Address addr) {
3642 assert(!addr.getAlignment().isZero());
3643 return ParamValue(addr);
3644 }
3645
3646 bool isIndirect() const { return IsIndirect; }
3647 llvm::Value *getAnyValue() const {
3648 if (!isIndirect())
3649 return Value;
3650 assert(!Addr.hasOffset() && "unexpected offset");
3651 return Addr.getBasePointer();
3652 }
3653
3654 llvm::Value *getDirectValue() const {
3655 assert(!isIndirect());
3656 return Value;
3657 }
3658
3659 Address getIndirectAddress() const {
3660 assert(isIndirect());
3661 return Addr;
3662 }
3663 };
3664
3665 /// EmitParmDecl - Emit a ParmVarDecl or an ImplicitParamDecl.
3666 void EmitParmDecl(const VarDecl &D, ParamValue Arg, unsigned ArgNo);
3667
3668 /// protectFromPeepholes - Protect a value that we're intending to
3669 /// store to the side, but which will probably be used later, from
3670 /// aggressive peepholing optimizations that might delete it.
3671 ///
3672 /// Pass the result to unprotectFromPeepholes to declare that
3673 /// protection is no longer required.
3674 ///
3675 /// There's no particular reason why this shouldn't apply to
3676 /// l-values, it's just that no existing peepholes work on pointers.
3677 PeepholeProtection protectFromPeepholes(RValue rvalue);
3678 void unprotectFromPeepholes(PeepholeProtection protection);
3679
3680 void emitAlignmentAssumptionCheck(llvm::Value *Ptr, QualType Ty,
3681 SourceLocation Loc,
3682 SourceLocation AssumptionLoc,
3683 llvm::Value *Alignment,
3684 llvm::Value *OffsetValue,
3685 llvm::Value *TheCheck,
3686 llvm::Instruction *Assumption);
3687
3688 void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty,
3689 SourceLocation Loc, SourceLocation AssumptionLoc,
3690 llvm::Value *Alignment,
3691 llvm::Value *OffsetValue = nullptr);
3692
3693 void emitAlignmentAssumption(llvm::Value *PtrValue, const Expr *E,
3694 SourceLocation AssumptionLoc,
3695 llvm::Value *Alignment,
3696 llvm::Value *OffsetValue = nullptr);
3697
3698 //===--------------------------------------------------------------------===//
3699 // Statement Emission
3700 //===--------------------------------------------------------------------===//
3701
3702 /// EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
3703 void EmitStopPoint(const Stmt *S);
3704
3705 /// EmitStmt - Emit the code for the statement \arg S. It is legal to call
3706 /// this function even if there is no current insertion point.
3707 ///
3708 /// This function may clear the current insertion point; callers should use
3709 /// EnsureInsertPoint if they wish to subsequently generate code without first
3710 /// calling EmitBlock, EmitBranch, or EmitStmt.
3711 void EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs = {});
3712
3713 /// EmitSimpleStmt - Try to emit a "simple" statement which does not
3714 /// necessarily require an insertion point or debug information; typically
3715 /// because the statement amounts to a jump or a container of other
3716 /// statements.
3717 ///
3718 /// \return True if the statement was handled.
3719 bool EmitSimpleStmt(const Stmt *S, ArrayRef<const Attr *> Attrs);
3720
3721 Address EmitCompoundStmt(const CompoundStmt &S, bool GetLast = false,
3722 AggValueSlot AVS = AggValueSlot::ignored());
3723 Address
3724 EmitCompoundStmtWithoutScope(const CompoundStmt &S, bool GetLast = false,
3725 AggValueSlot AVS = AggValueSlot::ignored());
3726
3727 /// EmitLabel - Emit the block for the given label. It is legal to call this
3728 /// function even if there is no current insertion point.
3729 void EmitLabel(const LabelDecl *D); // helper for EmitLabelStmt.
3730
3731 void EmitLabelStmt(const LabelStmt &S);
3732 void EmitAttributedStmt(const AttributedStmt &S);
3733 void EmitGotoStmt(const GotoStmt &S);
3734 void EmitIndirectGotoStmt(const IndirectGotoStmt &S);
3735 void EmitIfStmt(const IfStmt &S);
3736
3737 void EmitWhileStmt(const WhileStmt &S, ArrayRef<const Attr *> Attrs = {});
3738 void EmitDoStmt(const DoStmt &S, ArrayRef<const Attr *> Attrs = {});
3739 void EmitForStmt(const ForStmt &S, ArrayRef<const Attr *> Attrs = {});
3740 void EmitReturnStmt(const ReturnStmt &S);
3741 void EmitDeclStmt(const DeclStmt &S);
3742 void EmitBreakStmt(const BreakStmt &S);
3743 void EmitContinueStmt(const ContinueStmt &S);
3744 void EmitSwitchStmt(const SwitchStmt &S);
3745 void EmitDefaultStmt(const DefaultStmt &S, ArrayRef<const Attr *> Attrs);
3746 void EmitCaseStmt(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3747 void EmitCaseStmtRange(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3748 void EmitDeferStmt(const DeferStmt &S);
3749 void EmitAsmStmt(const AsmStmt &S);
3750
3751 const BreakContinue *GetDestForLoopControlStmt(const LoopControlStmt &S);
3752
3753 void EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S);
3754 void EmitObjCAtTryStmt(const ObjCAtTryStmt &S);
3755 void EmitObjCAtThrowStmt(const ObjCAtThrowStmt &S);
3756 void EmitObjCAtSynchronizedStmt(const ObjCAtSynchronizedStmt &S);
3757 void EmitObjCAutoreleasePoolStmt(const ObjCAutoreleasePoolStmt &S);
3758
3759 void EmitCoroutineBody(const CoroutineBodyStmt &S);
3760 void EmitCoreturnStmt(const CoreturnStmt &S);
3761 RValue EmitCoawaitExpr(const CoawaitExpr &E,
3762 AggValueSlot aggSlot = AggValueSlot::ignored(),
3763 bool ignoreResult = false);
3764 LValue EmitCoawaitLValue(const CoawaitExpr *E);
3765 RValue EmitCoyieldExpr(const CoyieldExpr &E,
3766 AggValueSlot aggSlot = AggValueSlot::ignored(),
3767 bool ignoreResult = false);
3768 LValue EmitCoyieldLValue(const CoyieldExpr *E);
3769 RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID);
3770
3771 void EmitSYCLKernelCallStmt(const SYCLKernelCallStmt &S);
3772
3773 void EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3774 void ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3775
3776 void EmitCXXTryStmt(const CXXTryStmt &S);
3777 void EmitSEHTryStmt(const SEHTryStmt &S);
3778 void EmitSEHLeaveStmt(const SEHLeaveStmt &S);
3779 void EnterSEHTryStmt(const SEHTryStmt &S);
3780 void ExitSEHTryStmt(const SEHTryStmt &S);
3781 void VolatilizeTryBlocks(llvm::BasicBlock *BB,
3782 llvm::SmallPtrSet<llvm::BasicBlock *, 10> &V);
3783
3784 void pushSEHCleanup(CleanupKind kind, llvm::Function *FinallyFunc);
3785 void startOutlinedSEHHelper(CodeGenFunction &ParentCGF, bool IsFilter,
3786 const Stmt *OutlinedStmt);
3787
3788 llvm::Function *GenerateSEHFilterFunction(CodeGenFunction &ParentCGF,
3789 const SEHExceptStmt &Except);
3790
3791 llvm::Function *GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF,
3792 const SEHFinallyStmt &Finally);
3793
3794 void EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF,
3795 llvm::Value *ParentFP, llvm::Value *EntryEBP);
3796 llvm::Value *EmitSEHExceptionCode();
3797 llvm::Value *EmitSEHExceptionInfo();
3798 llvm::Value *EmitSEHAbnormalTermination();
3799
3800 /// Emit simple code for OpenMP directives in Simd-only mode.
3801 void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D);
3802
3803 /// Scan the outlined statement for captures from the parent function. For
3804 /// each capture, mark the capture as escaped and emit a call to
3805 /// llvm.localrecover. Insert the localrecover result into the LocalDeclMap.
3806 void EmitCapturedLocals(CodeGenFunction &ParentCGF, const Stmt *OutlinedStmt,
3807 bool IsFilter);
3808
3809 /// Recovers the address of a local in a parent function. ParentVar is the
3810 /// address of the variable used in the immediate parent function. It can
3811 /// either be an alloca or a call to llvm.localrecover if there are nested
3812 /// outlined functions. ParentFP is the frame pointer of the outermost parent
3813 /// frame.
3814 Address recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
3815 Address ParentVar, llvm::Value *ParentFP);
3816
3817 void EmitCXXForRangeStmt(const CXXForRangeStmt &S,
3818 ArrayRef<const Attr *> Attrs = {});
3819
3820 void
3821 EmitCXXExpansionStmtInstantiation(const CXXExpansionStmtInstantiation &S);
3822
3823 /// Controls insertion of cancellation exit blocks in worksharing constructs.
3824 class OMPCancelStackRAII {
3825 CodeGenFunction &CGF;
3826
3827 public:
3828 OMPCancelStackRAII(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
3829 bool HasCancel)
3830 : CGF(CGF) {
3831 CGF.OMPCancelStack.enter(CGF, Kind, HasCancel);
3832 }
3833 ~OMPCancelStackRAII() { CGF.OMPCancelStack.exit(CGF); }
3834 };
3835
3836 /// Returns calculated size of the specified type.
3837 llvm::Value *getTypeSize(QualType Ty);
3838 LValue InitCapturedStruct(const CapturedStmt &S);
3839 llvm::Function *EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K);
3840 llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S);
3841 Address GenerateCapturedStmtArgument(const CapturedStmt &S);
3842 llvm::Function *
3843 GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S,
3844 const OMPExecutableDirective &D);
3845 llvm::Function *
3846 GenerateOpenMPCapturedStmtFunctionAggregate(const CapturedStmt &S,
3847 const OMPExecutableDirective &D);
3848 void GenerateOpenMPCapturedVars(const CapturedStmt &S,
3849 SmallVectorImpl<llvm::Value *> &CapturedVars);
3850 void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy,
3851 SourceLocation Loc);
3852 /// Perform element by element copying of arrays with type \a
3853 /// OriginalType from \a SrcAddr to \a DestAddr using copying procedure
3854 /// generated by \a CopyGen.
3855 ///
3856 /// \param DestAddr Address of the destination array.
3857 /// \param SrcAddr Address of the source array.
3858 /// \param OriginalType Type of destination and source arrays.
3859 /// \param CopyGen Copying procedure that copies value of single array element
3860 /// to another single array element.
3861 void EmitOMPAggregateAssign(
3862 Address DestAddr, Address SrcAddr, QualType OriginalType,
3863 const llvm::function_ref<void(Address, Address)> CopyGen);
3864 /// Emit proper copying of data from one variable to another.
3865 ///
3866 /// \param OriginalType Original type of the copied variables.
3867 /// \param DestAddr Destination address.
3868 /// \param SrcAddr Source address.
3869 /// \param DestVD Destination variable used in \a CopyExpr (for arrays, has
3870 /// type of the base array element).
3871 /// \param SrcVD Source variable used in \a CopyExpr (for arrays, has type of
3872 /// the base array element).
3873 /// \param Copy Actual copygin expression for copying data from \a SrcVD to \a
3874 /// DestVD.
3875 void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr,
3876 const VarDecl *DestVD, const VarDecl *SrcVD,
3877 const Expr *Copy);
3878 /// Emit atomic update code for constructs: \a X = \a X \a BO \a E or
3879 /// \a X = \a E \a BO \a E.
3880 ///
3881 /// \param X Value to be updated.
3882 /// \param E Update value.
3883 /// \param BO Binary operation for update operation.
3884 /// \param IsXLHSInRHSPart true if \a X is LHS in RHS part of the update
3885 /// expression, false otherwise.
3886 /// \param AO Atomic ordering of the generated atomic instructions.
3887 /// \param CommonGen Code generator for complex expressions that cannot be
3888 /// expressed through atomicrmw instruction.
3889 /// \returns <true, OldAtomicValue> if simple 'atomicrmw' instruction was
3890 /// generated, <false, RValue::get(nullptr)> otherwise.
3891 std::pair<bool, RValue> EmitOMPAtomicSimpleUpdateExpr(
3892 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3893 llvm::AtomicOrdering AO, SourceLocation Loc,
3894 const llvm::function_ref<RValue(RValue)> CommonGen);
3895 bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
3896 OMPPrivateScope &PrivateScope);
3897 void EmitOMPPrivateClause(const OMPExecutableDirective &D,
3898 OMPPrivateScope &PrivateScope);
3899 void EmitOMPUseDevicePtrClause(
3900 const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
3901 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3902 CaptureDeviceAddrMap);
3903 void EmitOMPUseDeviceAddrClause(
3904 const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
3905 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3906 CaptureDeviceAddrMap);
3907 /// Emit code for copyin clause in \a D directive. The next code is
3908 /// generated at the start of outlined functions for directives:
3909 /// \code
3910 /// threadprivate_var1 = master_threadprivate_var1;
3911 /// operator=(threadprivate_var2, master_threadprivate_var2);
3912 /// ...
3913 /// __kmpc_barrier(&loc, global_tid);
3914 /// \endcode
3915 ///
3916 /// \param D OpenMP directive possibly with 'copyin' clause(s).
3917 /// \returns true if at least one copyin variable is found, false otherwise.
3918 bool EmitOMPCopyinClause(const OMPExecutableDirective &D);
3919 /// Emit initial code for lastprivate variables. If some variable is
3920 /// not also firstprivate, then the default initialization is used. Otherwise
3921 /// initialization of this variable is performed by EmitOMPFirstprivateClause
3922 /// method.
3923 ///
3924 /// \param D Directive that may have 'lastprivate' directives.
3925 /// \param PrivateScope Private scope for capturing lastprivate variables for
3926 /// proper codegen in internal captured statement.
3927 ///
3928 /// \returns true if there is at least one lastprivate variable, false
3929 /// otherwise.
3930 bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D,
3931 OMPPrivateScope &PrivateScope);
3932 /// Emit final copying of lastprivate values to original variables at
3933 /// the end of the worksharing or simd directive.
3934 ///
3935 /// \param D Directive that has at least one 'lastprivate' directives.
3936 /// \param IsLastIterCond Boolean condition that must be set to 'i1 true' if
3937 /// it is the last iteration of the loop code in associated directive, or to
3938 /// 'i1 false' otherwise. If this item is nullptr, no final check is required.
3939 void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D,
3940 bool NoFinals,
3941 llvm::Value *IsLastIterCond = nullptr);
3942 /// Emit initial code for linear clauses.
3943 void EmitOMPLinearClause(const OMPLoopDirective &D,
3944 CodeGenFunction::OMPPrivateScope &PrivateScope);
3945 /// Emit final code for linear clauses.
3946 /// \param CondGen Optional conditional code for final part of codegen for
3947 /// linear clause.
3948 void EmitOMPLinearClauseFinal(
3949 const OMPLoopDirective &D,
3950 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
3951 /// Emit initial code for reduction variables. Creates reduction copies
3952 /// and initializes them with the values according to OpenMP standard.
3953 ///
3954 /// \param D Directive (possibly) with the 'reduction' clause.
3955 /// \param PrivateScope Private scope for capturing reduction variables for
3956 /// proper codegen in internal captured statement.
3957 ///
3958 void EmitOMPReductionClauseInit(const OMPExecutableDirective &D,
3959 OMPPrivateScope &PrivateScope,
3960 bool ForInscan = false);
3961 /// Emit final update of reduction values to original variables at
3962 /// the end of the directive.
3963 ///
3964 /// \param D Directive that has at least one 'reduction' directives.
3965 /// \param ReductionKind The kind of reduction to perform.
3966 void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D,
3967 const OpenMPDirectiveKind ReductionKind);
3968 /// Emit initial code for linear variables. Creates private copies
3969 /// and initializes them with the values according to OpenMP standard.
3970 ///
3971 /// \param D Directive (possibly) with the 'linear' clause.
3972 /// \return true if at least one linear variable is found that should be
3973 /// initialized with the value of the original variable, false otherwise.
3974 bool EmitOMPLinearClauseInit(const OMPLoopDirective &D);
3975
3976 typedef const llvm::function_ref<void(CodeGenFunction & /*CGF*/,
3977 llvm::Function * /*OutlinedFn*/,
3978 const OMPTaskDataTy & /*Data*/)>
3979 TaskGenTy;
3980 void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S,
3981 const OpenMPDirectiveKind CapturedRegion,
3982 const RegionCodeGenTy &BodyGen,
3983 const TaskGenTy &TaskGen, OMPTaskDataTy &Data);
3984 struct OMPTargetDataInfo {
3985 Address BasePointersArray = Address::invalid();
3986 Address PointersArray = Address::invalid();
3987 Address SizesArray = Address::invalid();
3988 Address MappersArray = Address::invalid();
3989 unsigned NumberOfTargetItems = 0;
3990 explicit OMPTargetDataInfo() = default;
3991 OMPTargetDataInfo(Address BasePointersArray, Address PointersArray,
3992 Address SizesArray, Address MappersArray,
3993 unsigned NumberOfTargetItems)
3994 : BasePointersArray(BasePointersArray), PointersArray(PointersArray),
3995 SizesArray(SizesArray), MappersArray(MappersArray),
3996 NumberOfTargetItems(NumberOfTargetItems) {}
3997 };
3998 void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S,
3999 const RegionCodeGenTy &BodyGen,
4000 OMPTargetDataInfo &InputInfo);
4001 void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data,
4002 CodeGenFunction &CGF, const CapturedStmt *CS,
4003 OMPPrivateScope &Scope);
4004 void EmitOMPMetaDirective(const OMPMetaDirective &S);
4005 void EmitOMPParallelDirective(const OMPParallelDirective &S);
4006 void EmitOMPSimdDirective(const OMPSimdDirective &S);
4007 void EmitOMPTileDirective(const OMPTileDirective &S);
4008 void EmitOMPStripeDirective(const OMPStripeDirective &S);
4009 void EmitOMPUnrollDirective(const OMPUnrollDirective &S);
4010 void EmitOMPReverseDirective(const OMPReverseDirective &S);
4011 void EmitOMPSplitDirective(const OMPSplitDirective &S);
4012 void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
4013 void EmitOMPFlattenDirective(const OMPFlattenDirective &S);
4014 void EmitOMPFuseDirective(const OMPFuseDirective &S);
4015 void EmitOMPForDirective(const OMPForDirective &S);
4016 void EmitOMPForSimdDirective(const OMPForSimdDirective &S);
4017 void EmitOMPScopeDirective(const OMPScopeDirective &S);
4018 void EmitOMPSectionsDirective(const OMPSectionsDirective &S);
4019 void EmitOMPSectionDirective(const OMPSectionDirective &S);
4020 void EmitOMPSingleDirective(const OMPSingleDirective &S);
4021 void EmitOMPMasterDirective(const OMPMasterDirective &S);
4022 void EmitOMPMaskedDirective(const OMPMaskedDirective &S);
4023 void EmitOMPCriticalDirective(const OMPCriticalDirective &S);
4024 void EmitOMPParallelForDirective(const OMPParallelForDirective &S);
4025 void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S);
4026 void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S);
4027 void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S);
4028 void EmitOMPTaskDirective(const OMPTaskDirective &S);
4029 void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S);
4030 void EmitOMPErrorDirective(const OMPErrorDirective &S);
4031 void EmitOMPBarrierDirective(const OMPBarrierDirective &S);
4032 void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S);
4033 void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S);
4034 void EmitOMPFlushDirective(const OMPFlushDirective &S);
4035 void EmitOMPDepobjDirective(const OMPDepobjDirective &S);
4036 void EmitOMPScanDirective(const OMPScanDirective &S);
4037 void
4038 EmitOMPOrderedStandaloneDirective(const OMPOrderedStandaloneDirective &S);
4039 void
4040 EmitOMPOrderedBlockAssocDirective(const OMPOrderedBlockAssocDirective &S);
4041 void EmitOMPAtomicDirective(const OMPAtomicDirective &S);
4042 void EmitOMPTargetDirective(const OMPTargetDirective &S);
4043 void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S);
4044 void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S);
4045 void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S);
4046 void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S);
4047 void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S);
4048 void
4049 EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S);
4050 void EmitOMPTeamsDirective(const OMPTeamsDirective &S);
4051 void
4052 EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S);
4053 void EmitOMPCancelDirective(const OMPCancelDirective &S);
4054 void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S);
4055 void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S);
4056 void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S);
4057 void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S);
4058 void EmitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &S);
4059 void
4060 EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S);
4061 void
4062 EmitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &S);
4063 void EmitOMPParallelMasterTaskLoopDirective(
4064 const OMPParallelMasterTaskLoopDirective &S);
4065 void EmitOMPParallelMaskedTaskLoopDirective(
4066 const OMPParallelMaskedTaskLoopDirective &S);
4067 void EmitOMPParallelMasterTaskLoopSimdDirective(
4068 const OMPParallelMasterTaskLoopSimdDirective &S);
4069 void EmitOMPParallelMaskedTaskLoopSimdDirective(
4070 const OMPParallelMaskedTaskLoopSimdDirective &S);
4071 void EmitOMPDistributeDirective(const OMPDistributeDirective &S);
4072 void EmitOMPDistributeParallelForDirective(
4073 const OMPDistributeParallelForDirective &S);
4074 void EmitOMPDistributeParallelForSimdDirective(
4075 const OMPDistributeParallelForSimdDirective &S);
4076 void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S);
4077 void EmitOMPTargetParallelForSimdDirective(
4078 const OMPTargetParallelForSimdDirective &S);
4079 void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S);
4080 void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S);
4081 void
4082 EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S);
4083 void EmitOMPTeamsDistributeParallelForSimdDirective(
4084 const OMPTeamsDistributeParallelForSimdDirective &S);
4085 void EmitOMPTeamsDistributeParallelForDirective(
4086 const OMPTeamsDistributeParallelForDirective &S);
4087 void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S);
4088 void EmitOMPTargetTeamsDistributeDirective(
4089 const OMPTargetTeamsDistributeDirective &S);
4090 void EmitOMPTargetTeamsDistributeParallelForDirective(
4091 const OMPTargetTeamsDistributeParallelForDirective &S);
4092 void EmitOMPTargetTeamsDistributeParallelForSimdDirective(
4093 const OMPTargetTeamsDistributeParallelForSimdDirective &S);
4094 void EmitOMPTargetTeamsDistributeSimdDirective(
4095 const OMPTargetTeamsDistributeSimdDirective &S);
4096 void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S);
4097 void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S);
4098 void EmitOMPTargetParallelGenericLoopDirective(
4099 const OMPTargetParallelGenericLoopDirective &S);
4100 void EmitOMPTargetTeamsGenericLoopDirective(
4101 const OMPTargetTeamsGenericLoopDirective &S);
4102 void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S);
4103 void EmitOMPInteropDirective(const OMPInteropDirective &S);
4104 void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S);
4105 void EmitOMPAssumeDirective(const OMPAssumeDirective &S);
4106
4107 /// Emit device code for the target directive.
4108 static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
4109 StringRef ParentName,
4110 const OMPTargetDirective &S);
4111 static void
4112 EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
4113 const OMPTargetParallelDirective &S);
4114 /// Emit device code for the target parallel for directive.
4115 static void EmitOMPTargetParallelForDeviceFunction(
4116 CodeGenModule &CGM, StringRef ParentName,
4117 const OMPTargetParallelForDirective &S);
4118 /// Emit device code for the target parallel for simd directive.
4119 static void EmitOMPTargetParallelForSimdDeviceFunction(
4120 CodeGenModule &CGM, StringRef ParentName,
4121 const OMPTargetParallelForSimdDirective &S);
4122 /// Emit device code for the target teams directive.
4123 static void
4124 EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
4125 const OMPTargetTeamsDirective &S);
4126 /// Emit device code for the target teams distribute directive.
4127 static void EmitOMPTargetTeamsDistributeDeviceFunction(
4128 CodeGenModule &CGM, StringRef ParentName,
4129 const OMPTargetTeamsDistributeDirective &S);
4130 /// Emit device code for the target teams distribute simd directive.
4131 static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(
4132 CodeGenModule &CGM, StringRef ParentName,
4133 const OMPTargetTeamsDistributeSimdDirective &S);
4134 /// Emit device code for the target simd directive.
4135 static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM,
4136 StringRef ParentName,
4137 const OMPTargetSimdDirective &S);
4138 /// Emit device code for the target teams distribute parallel for simd
4139 /// directive.
4140 static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
4141 CodeGenModule &CGM, StringRef ParentName,
4142 const OMPTargetTeamsDistributeParallelForSimdDirective &S);
4143
4144 /// Emit device code for the target teams loop directive.
4145 static void EmitOMPTargetTeamsGenericLoopDeviceFunction(
4146 CodeGenModule &CGM, StringRef ParentName,
4147 const OMPTargetTeamsGenericLoopDirective &S);
4148
4149 /// Emit device code for the target parallel loop directive.
4150 static void EmitOMPTargetParallelGenericLoopDeviceFunction(
4151 CodeGenModule &CGM, StringRef ParentName,
4152 const OMPTargetParallelGenericLoopDirective &S);
4153
4154 static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
4155 CodeGenModule &CGM, StringRef ParentName,
4156 const OMPTargetTeamsDistributeParallelForDirective &S);
4157
4158 /// Emit the Stmt \p S and return its topmost canonical loop, if any.
4159 /// TODO: The \p Depth paramter is not yet implemented and must be 1. In the
4160 /// future it is meant to be the number of loops expected in the loop nests
4161 /// (usually specified by the "collapse" clause) that are collapsed to a
4162 /// single loop by this function.
4163 llvm::CanonicalLoopInfo *EmitOMPCollapsedCanonicalLoopNest(const Stmt *S,
4164 int Depth);
4165
4166 /// Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
4167 void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S);
4168
4169 /// Emit inner loop of the worksharing/simd construct.
4170 ///
4171 /// \param S Directive, for which the inner loop must be emitted.
4172 /// \param RequiresCleanup true, if directive has some associated private
4173 /// variables.
4174 /// \param LoopCond Bollean condition for loop continuation.
4175 /// \param IncExpr Increment expression for loop control variable.
4176 /// \param BodyGen Generator for the inner body of the inner loop.
4177 /// \param PostIncGen Genrator for post-increment code (required for ordered
4178 /// loop directvies).
4179 void EmitOMPInnerLoop(
4180 const OMPExecutableDirective &S, bool RequiresCleanup,
4181 const Expr *LoopCond, const Expr *IncExpr,
4182 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
4183 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen);
4184
4185 JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind);
4186 /// Emit initial code for loop counters of loop-based directives.
4187 void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S,
4188 OMPPrivateScope &LoopScope);
4189
4190 /// Helper for the OpenMP loop directives.
4191 void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit);
4192
4193 /// Emit code for the worksharing loop-based directive.
4194 /// \return true, if this construct has any lastprivate clause, false -
4195 /// otherwise.
4196 bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB,
4197 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
4198 const CodeGenDispatchBoundsTy &CGDispatchBounds);
4199
4200 /// Emit code for the distribute loop-based directive.
4201 void EmitOMPDistributeLoop(const OMPLoopDirective &S,
4202 const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr);
4203
4204 /// Helpers for the OpenMP loop directives.
4205 void EmitOMPSimdInit(const OMPLoopDirective &D);
4206 void EmitOMPSimdFinal(
4207 const OMPLoopDirective &D,
4208 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
4209
4210 /// Emits the lvalue for the expression with possibly captured variable.
4211 LValue EmitOMPSharedLValue(const Expr *E);
4212
4213 /// Emits the original address for a structured binding.
4214 Address EmitOMPBindingOriginalAddr(const BindingDecl *BD, SourceLocation Loc);
4215
4216private:
4217 /// Helpers for blocks.
4218 llvm::Value *EmitBlockLiteral(const CGBlockInfo &Info);
4219
4220 /// struct with the values to be passed to the OpenMP loop-related functions
4221 struct OMPLoopArguments {
4222 /// loop lower bound
4223 Address LB = Address::invalid();
4224 /// loop upper bound
4225 Address UB = Address::invalid();
4226 /// loop stride
4227 Address ST = Address::invalid();
4228 /// isLastIteration argument for runtime functions
4229 Address IL = Address::invalid();
4230 /// Chunk value generated by sema
4231 llvm::Value *Chunk = nullptr;
4232 /// EnsureUpperBound
4233 Expr *EUB = nullptr;
4234 /// IncrementExpression
4235 Expr *IncExpr = nullptr;
4236 /// Loop initialization
4237 Expr *Init = nullptr;
4238 /// Loop exit condition
4239 Expr *Cond = nullptr;
4240 /// Update of LB after a whole chunk has been executed
4241 Expr *NextLB = nullptr;
4242 /// Update of UB after a whole chunk has been executed
4243 Expr *NextUB = nullptr;
4244 /// Distinguish between the for distribute and sections
4245 OpenMPDirectiveKind DKind = llvm::omp::OMPD_unknown;
4246 OMPLoopArguments() = default;
4247 OMPLoopArguments(Address LB, Address UB, Address ST, Address IL,
4248 llvm::Value *Chunk = nullptr, Expr *EUB = nullptr,
4249 Expr *IncExpr = nullptr, Expr *Init = nullptr,
4250 Expr *Cond = nullptr, Expr *NextLB = nullptr,
4251 Expr *NextUB = nullptr)
4252 : LB(LB), UB(UB), ST(ST), IL(IL), Chunk(Chunk), EUB(EUB),
4253 IncExpr(IncExpr), Init(Init), Cond(Cond), NextLB(NextLB),
4254 NextUB(NextUB) {}
4255 };
4256 void EmitOMPOuterLoop(bool DynamicOrOrdered, bool IsMonotonic,
4257 const OMPLoopDirective &S, OMPPrivateScope &LoopScope,
4258 const OMPLoopArguments &LoopArgs,
4259 const CodeGenLoopTy &CodeGenLoop,
4260 const CodeGenOrderedTy &CodeGenOrdered);
4261 void EmitOMPForOuterLoop(const OpenMPScheduleTy &ScheduleKind,
4262 bool IsMonotonic, const OMPLoopDirective &S,
4263 OMPPrivateScope &LoopScope, bool Ordered,
4264 const OMPLoopArguments &LoopArgs,
4265 const CodeGenDispatchBoundsTy &CGDispatchBounds);
4266 void EmitOMPDistributeOuterLoop(OpenMPDistScheduleClauseKind ScheduleKind,
4267 const OMPLoopDirective &S,
4268 OMPPrivateScope &LoopScope,
4269 const OMPLoopArguments &LoopArgs,
4270 const CodeGenLoopTy &CodeGenLoopContent);
4271 /// Emit code for sections directive.
4272 void EmitSections(const OMPExecutableDirective &S);
4273
4274public:
4275 //===--------------------------------------------------------------------===//
4276 // OpenACC Emission
4277 //===--------------------------------------------------------------------===//
4278 void EmitOpenACCComputeConstruct(const OpenACCComputeConstruct &S) {
4279 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4280 // simply emitting its structured block, but in the future we will implement
4281 // some sort of IR.
4282 if (S.getStructuredBlock())
4283 EmitStmt(S: S.getStructuredBlock());
4284 }
4285
4286 void EmitOpenACCLoopConstruct(const OpenACCLoopConstruct &S) {
4287 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4288 // simply emitting its loop, but in the future we will implement
4289 // some sort of IR.
4290 if (S.getLoop())
4291 EmitStmt(S: S.getLoop());
4292 }
4293
4294 void EmitOpenACCCombinedConstruct(const OpenACCCombinedConstruct &S) {
4295 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4296 // simply emitting its loop, but in the future we will implement
4297 // some sort of IR.
4298 if (S.getLoop())
4299 EmitStmt(S: S.getLoop());
4300 }
4301
4302 void EmitOpenACCDataConstruct(const OpenACCDataConstruct &S) {
4303 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4304 // simply emitting its structured block, but in the future we will implement
4305 // some sort of IR.
4306 if (S.getStructuredBlock())
4307 EmitStmt(S: S.getStructuredBlock());
4308 }
4309
4310 void EmitOpenACCEnterDataConstruct(const OpenACCEnterDataConstruct &S) {
4311 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4312 // but in the future we will implement some sort of IR.
4313 }
4314
4315 void EmitOpenACCExitDataConstruct(const OpenACCExitDataConstruct &S) {
4316 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4317 // but in the future we will implement some sort of IR.
4318 }
4319
4320 void EmitOpenACCHostDataConstruct(const OpenACCHostDataConstruct &S) {
4321 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4322 // simply emitting its structured block, but in the future we will implement
4323 // some sort of IR.
4324 if (S.getStructuredBlock())
4325 EmitStmt(S: S.getStructuredBlock());
4326 }
4327
4328 void EmitOpenACCWaitConstruct(const OpenACCWaitConstruct &S) {
4329 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4330 // but in the future we will implement some sort of IR.
4331 }
4332
4333 void EmitOpenACCInitConstruct(const OpenACCInitConstruct &S) {
4334 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4335 // but in the future we will implement some sort of IR.
4336 }
4337
4338 void EmitOpenACCShutdownConstruct(const OpenACCShutdownConstruct &S) {
4339 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4340 // but in the future we will implement some sort of IR.
4341 }
4342
4343 void EmitOpenACCSetConstruct(const OpenACCSetConstruct &S) {
4344 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4345 // but in the future we will implement some sort of IR.
4346 }
4347
4348 void EmitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &S) {
4349 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4350 // but in the future we will implement some sort of IR.
4351 }
4352
4353 void EmitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &S) {
4354 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4355 // simply emitting its associated stmt, but in the future we will implement
4356 // some sort of IR.
4357 if (S.getAssociatedStmt())
4358 EmitStmt(S: S.getAssociatedStmt());
4359 }
4360 void EmitOpenACCCacheConstruct(const OpenACCCacheConstruct &S) {
4361 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4362 // but in the future we will implement some sort of IR.
4363 }
4364
4365 //===--------------------------------------------------------------------===//
4366 // LValue Expression Emission
4367 //===--------------------------------------------------------------------===//
4368
4369 /// Create a check that a scalar RValue is non-null.
4370 llvm::Value *EmitNonNullRValueCheck(RValue RV, QualType T);
4371
4372 /// GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
4373 RValue GetUndefRValue(QualType Ty);
4374
4375 /// EmitUnsupportedRValue - Emit a dummy r-value using the type of E
4376 /// and issue an ErrorUnsupported style diagnostic (using the
4377 /// provided Name).
4378 RValue EmitUnsupportedRValue(const Expr *E, const char *Name);
4379
4380 /// EmitUnsupportedLValue - Emit a dummy l-value using the type of E and issue
4381 /// an ErrorUnsupported style diagnostic (using the provided Name).
4382 LValue EmitUnsupportedLValue(const Expr *E, const char *Name);
4383
4384 /// EmitLValue - Emit code to compute a designator that specifies the location
4385 /// of the expression.
4386 ///
4387 /// This can return one of two things: a simple address or a bitfield
4388 /// reference. In either case, the LLVM Value* in the LValue structure is
4389 /// guaranteed to be an LLVM pointer type.
4390 ///
4391 /// If this returns a bitfield reference, nothing about the pointee type of
4392 /// the LLVM value is known: For example, it may not be a pointer to an
4393 /// integer.
4394 ///
4395 /// If this returns a normal address, and if the lvalue's C type is fixed
4396 /// size, this method guarantees that the returned pointer type will point to
4397 /// an LLVM type of the same size of the lvalue's type. If the lvalue has a
4398 /// variable length type, this is not possible.
4399 ///
4400 LValue EmitLValue(const Expr *E,
4401 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
4402
4403private:
4404 LValue EmitLValueHelper(const Expr *E, KnownNonNull_t IsKnownNonNull);
4405
4406public:
4407 /// Same as EmitLValue but additionally we generate checking code to
4408 /// guard against undefined behavior. This is only suitable when we know
4409 /// that the address will be used to access the object.
4410 LValue EmitCheckedLValue(const Expr *E, TypeCheckKind TCK);
4411
4412 RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc);
4413
4414 void EmitAtomicInit(Expr *E, LValue lvalue);
4415
4416 bool LValueIsSuitableForInlineAtomic(LValue Src);
4417
4418 RValue EmitAtomicLoad(LValue LV, SourceLocation SL,
4419 AggValueSlot Slot = AggValueSlot::ignored());
4420
4421 RValue EmitAtomicLoad(LValue lvalue, SourceLocation loc,
4422 llvm::AtomicOrdering AO, bool IsVolatile = false,
4423 AggValueSlot slot = AggValueSlot::ignored());
4424
4425 void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit);
4426
4427 void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO,
4428 bool IsVolatile, bool isInit);
4429
4430 std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
4431 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
4432 llvm::AtomicOrdering Success =
4433 llvm::AtomicOrdering::SequentiallyConsistent,
4434 llvm::AtomicOrdering Failure =
4435 llvm::AtomicOrdering::SequentiallyConsistent,
4436 bool IsWeak = false, AggValueSlot Slot = AggValueSlot::ignored());
4437
4438 /// Emit an atomicrmw instruction, and applying relevant metadata when
4439 /// applicable.
4440 llvm::AtomicRMWInst *emitAtomicRMWInst(
4441 llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val,
4442 llvm::AtomicOrdering Order = llvm::AtomicOrdering::SequentiallyConsistent,
4443 llvm::SyncScope::ID SSID = llvm::SyncScope::System,
4444 const AtomicExpr *AE = nullptr);
4445
4446 /// Emit a fence instruction, applying relevant target-specific metadata when
4447 /// applicable.
4448 llvm::FenceInst *
4449 emitAtomicFence(llvm::AtomicOrdering Order,
4450 llvm::SyncScope::ID SSID = llvm::SyncScope::System);
4451
4452 void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO,
4453 const llvm::function_ref<RValue(RValue)> &UpdateOp,
4454 bool IsVolatile);
4455
4456 /// EmitToMemory - Change a scalar value from its value
4457 /// representation to its in-memory representation.
4458 llvm::Value *EmitToMemory(llvm::Value *Value, QualType Ty);
4459
4460 /// EmitFromMemory - Change a scalar value from its memory
4461 /// representation to its value representation.
4462 llvm::Value *EmitFromMemory(llvm::Value *Value, QualType Ty);
4463
4464 /// Check if the scalar \p Value is within the valid range for the given
4465 /// type \p Ty.
4466 ///
4467 /// Returns true if a check is needed (even if the range is unknown).
4468 bool EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
4469 SourceLocation Loc);
4470
4471 /// EmitLoadOfScalar - Load a scalar value from an address, taking
4472 /// care to appropriately convert from the memory representation to
4473 /// the LLVM value representation.
4474 llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4475 SourceLocation Loc,
4476 AlignmentSource Source = AlignmentSource::Type,
4477 bool isNontemporal = false) {
4478 return EmitLoadOfScalar(Addr, Volatile, Ty, Loc, BaseInfo: LValueBaseInfo(Source),
4479 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isNontemporal);
4480 }
4481
4482 llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4483 SourceLocation Loc, LValueBaseInfo BaseInfo,
4484 TBAAAccessInfo TBAAInfo,
4485 bool isNontemporal = false);
4486
4487 /// EmitLoadOfScalar - Load a scalar value from an address, taking
4488 /// care to appropriately convert from the memory representation to
4489 /// the LLVM value representation. The l-value must be a simple
4490 /// l-value.
4491 llvm::Value *EmitLoadOfScalar(LValue lvalue, SourceLocation Loc);
4492
4493 /// EmitStoreOfScalar - Store a scalar value to an address, taking
4494 /// care to appropriately convert from the memory representation to
4495 /// the LLVM value representation.
4496 void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,
4497 QualType Ty,
4498 AlignmentSource Source = AlignmentSource::Type,
4499 bool isInit = false, bool isNontemporal = false) {
4500 EmitStoreOfScalar(Value, Addr, Volatile, Ty, BaseInfo: LValueBaseInfo(Source),
4501 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isInit, isNontemporal);
4502 }
4503
4504 void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,
4505 QualType Ty, LValueBaseInfo BaseInfo,
4506 TBAAAccessInfo TBAAInfo, bool isInit = false,
4507 bool isNontemporal = false);
4508
4509 /// EmitStoreOfScalar - Store a scalar value to an address, taking
4510 /// care to appropriately convert from the memory representation to
4511 /// the LLVM value representation. The l-value must be a simple
4512 /// l-value. The isInit flag indicates whether this is an initialization.
4513 /// If so, atomic qualifiers are ignored and the store is always non-atomic.
4514 void EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
4515 bool isInit = false);
4516
4517 /// EmitLoadOfLValue - Given an expression that represents a value lvalue,
4518 /// this method emits the address of the lvalue, then loads the result as an
4519 /// rvalue, returning the rvalue.
4520 RValue EmitLoadOfLValue(LValue V, SourceLocation Loc);
4521 RValue EmitLoadOfExtVectorElementLValue(LValue V);
4522 RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc);
4523 RValue EmitLoadOfGlobalRegLValue(LValue LV);
4524
4525 /// Like EmitLoadOfLValue but also handles complex and aggregate types.
4526 RValue EmitLoadOfAnyValue(LValue V,
4527 AggValueSlot Slot = AggValueSlot::ignored(),
4528 SourceLocation Loc = {});
4529
4530 /// EmitStoreThroughLValue - Store the specified rvalue into the specified
4531 /// lvalue, where both are guaranteed to the have the same type, and that type
4532 /// is 'Ty'.
4533 void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit = false);
4534 void EmitStoreThroughExtVectorComponentLValue(RValue Src, LValue Dst);
4535 void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst);
4536
4537 /// EmitStoreThroughBitfieldLValue - Store Src into Dst with same constraints
4538 /// as EmitStoreThroughLValue.
4539 ///
4540 /// \param Result [out] - If non-null, this will be set to a Value* for the
4541 /// bit-field contents after the store, appropriate for use as the result of
4542 /// an assignment to the bit-field.
4543 void EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
4544 llvm::Value **Result = nullptr);
4545
4546 /// Emit an l-value for an assignment (simple or compound) of complex type.
4547 LValue EmitComplexAssignmentLValue(const BinaryOperator *E);
4548 LValue EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E);
4549 LValue EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E,
4550 llvm::Value *&Result);
4551
4552 // Note: only available for agg return types
4553 LValue EmitBinaryOperatorLValue(const BinaryOperator *E);
4554 LValue EmitCompoundAssignmentLValue(const CompoundAssignOperator *E);
4555 // Note: only available for agg return types
4556 LValue EmitCallExprLValue(const CallExpr *E,
4557 llvm::CallBase **CallOrInvoke = nullptr);
4558 // Note: only available for agg return types
4559 LValue EmitVAArgExprLValue(const VAArgExpr *E);
4560 LValue EmitDeclRefLValue(const DeclRefExpr *E);
4561 LValue EmitOMPCapturedBindingLValue(const BindingDecl *BD);
4562 LValue EmitStringLiteralLValue(const StringLiteral *E);
4563 LValue EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E);
4564 LValue EmitPredefinedLValue(const PredefinedExpr *E);
4565 LValue EmitUnaryOpLValue(const UnaryOperator *E);
4566 LValue EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
4567 bool Accessed = false);
4568 llvm::Value *EmitMatrixIndexExpr(const Expr *E);
4569 LValue EmitMatrixSingleSubscriptExpr(const MatrixSingleSubscriptExpr *E);
4570 LValue EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E);
4571 LValue EmitArraySectionExpr(const ArraySectionExpr *E,
4572 bool IsLowerBound = true);
4573 LValue EmitExtVectorElementExpr(const ExtVectorElementExpr *E);
4574 LValue EmitMatrixElementExpr(const MatrixElementExpr *E);
4575 LValue EmitMemberExpr(const MemberExpr *E);
4576 LValue EmitObjCIsaExpr(const ObjCIsaExpr *E);
4577 LValue EmitCompoundLiteralLValue(const CompoundLiteralExpr *E);
4578 LValue EmitInitListLValue(const InitListExpr *E);
4579 void EmitIgnoredConditionalOperator(const AbstractConditionalOperator *E);
4580 LValue EmitConditionalOperatorLValue(const AbstractConditionalOperator *E);
4581 LValue EmitCastLValue(const CastExpr *E);
4582 LValue EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);
4583 LValue EmitOpaqueValueLValue(const OpaqueValueExpr *e);
4584 LValue EmitHLSLArrayAssignLValue(const BinaryOperator *E);
4585
4586 std::pair<LValue, LValue> EmitHLSLOutArgLValues(const HLSLOutArgExpr *E,
4587 QualType Ty);
4588 LValue EmitHLSLOutArgExpr(const HLSLOutArgExpr *E, CallArgList &Args,
4589 QualType Ty);
4590
4591 Address EmitExtVectorElementLValue(LValue V);
4592
4593 RValue EmitRValueForField(LValue LV, const FieldDecl *FD, SourceLocation Loc);
4594
4595 Address EmitArrayToPointerDecay(const Expr *Array,
4596 LValueBaseInfo *BaseInfo = nullptr,
4597 TBAAAccessInfo *TBAAInfo = nullptr);
4598
4599 class ConstantEmission {
4600 llvm::PointerIntPair<llvm::Constant *, 1, bool> ValueAndIsReference;
4601 ConstantEmission(llvm::Constant *C, bool isReference)
4602 : ValueAndIsReference(C, isReference) {}
4603
4604 public:
4605 ConstantEmission() {}
4606 static ConstantEmission forReference(llvm::Constant *C) {
4607 return ConstantEmission(C, true);
4608 }
4609 static ConstantEmission forValue(llvm::Constant *C) {
4610 return ConstantEmission(C, false);
4611 }
4612
4613 explicit operator bool() const {
4614 return ValueAndIsReference.getOpaqueValue() != nullptr;
4615 }
4616
4617 bool isReference() const { return ValueAndIsReference.getInt(); }
4618 LValue getReferenceLValue(CodeGenFunction &CGF, const Expr *RefExpr) const {
4619 assert(isReference());
4620 return CGF.MakeNaturalAlignAddrLValue(V: ValueAndIsReference.getPointer(),
4621 T: RefExpr->getType());
4622 }
4623
4624 llvm::Constant *getValue() const {
4625 assert(!isReference());
4626 return ValueAndIsReference.getPointer();
4627 }
4628 };
4629
4630 ConstantEmission tryEmitAsConstant(const DeclRefExpr *RefExpr);
4631 ConstantEmission tryEmitAsConstant(const MemberExpr *ME);
4632 llvm::Value *emitScalarConstant(const ConstantEmission &Constant, Expr *E);
4633
4634 RValue EmitPseudoObjectRValue(const PseudoObjectExpr *e,
4635 AggValueSlot slot = AggValueSlot::ignored());
4636 LValue EmitPseudoObjectLValue(const PseudoObjectExpr *e);
4637
4638 void FlattenAccessAndTypeLValue(LValue LVal,
4639 SmallVectorImpl<LValue> &AccessList);
4640
4641 llvm::Value *EmitIvarOffset(const ObjCInterfaceDecl *Interface,
4642 const ObjCIvarDecl *Ivar);
4643 llvm::Value *EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
4644 const ObjCIvarDecl *Ivar);
4645 LValue EmitLValueForField(LValue Base, const FieldDecl *Field,
4646 bool IsInBounds = true);
4647 LValue EmitLValueForLambdaField(const FieldDecl *Field);
4648 LValue EmitLValueForLambdaField(const FieldDecl *Field,
4649 llvm::Value *ThisValue);
4650
4651 /// EmitLValueForFieldInitialization - Like EmitLValueForField, except that
4652 /// if the Field is a reference, this will return the address of the reference
4653 /// and not the address of the value stored in the reference.
4654 LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field);
4655
4656 LValue EmitLValueForIvar(QualType ObjectTy, llvm::Value *Base,
4657 const ObjCIvarDecl *Ivar, unsigned CVRQualifiers);
4658
4659 LValue EmitCXXConstructLValue(const CXXConstructExpr *E);
4660 LValue EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E);
4661 LValue EmitCXXTypeidLValue(const CXXTypeidExpr *E);
4662 LValue EmitCXXUuidofLValue(const CXXUuidofExpr *E);
4663
4664 LValue EmitObjCMessageExprLValue(const ObjCMessageExpr *E);
4665 LValue EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E);
4666 LValue EmitStmtExprLValue(const StmtExpr *E);
4667 LValue EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E);
4668 LValue EmitObjCSelectorLValue(const ObjCSelectorExpr *E);
4669 void EmitDeclRefExprDbgValue(const DeclRefExpr *E, const APValue &Init);
4670
4671 //===--------------------------------------------------------------------===//
4672 // Scalar Expression Emission
4673 //===--------------------------------------------------------------------===//
4674
4675 /// EmitCall - Generate a call of the given function, expecting the given
4676 /// result type, and using the given argument list which specifies both the
4677 /// LLVM arguments and the types they were derived from.
4678 RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4679 ReturnValueSlot ReturnValue, const CallArgList &Args,
4680 llvm::CallBase **CallOrInvoke, bool IsMustTail,
4681 SourceLocation Loc,
4682 bool IsVirtualFunctionPointerThunk = false);
4683 RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4684 ReturnValueSlot ReturnValue, const CallArgList &Args,
4685 llvm::CallBase **CallOrInvoke = nullptr,
4686 bool IsMustTail = false) {
4687 return EmitCall(CallInfo, Callee, ReturnValue, Args, CallOrInvoke,
4688 IsMustTail, Loc: SourceLocation());
4689 }
4690 RValue EmitCall(QualType FnType, const CGCallee &Callee, const CallExpr *E,
4691 ReturnValueSlot ReturnValue, llvm::Value *Chain = nullptr,
4692 llvm::CallBase **CallOrInvoke = nullptr,
4693 CGFunctionInfo const **ResolvedFnInfo = nullptr);
4694
4695 // If a Call or Invoke instruction was emitted for this CallExpr, this method
4696 // writes the pointer to `CallOrInvoke` if it's not null.
4697 RValue EmitCallExpr(const CallExpr *E,
4698 ReturnValueSlot ReturnValue = ReturnValueSlot(),
4699 llvm::CallBase **CallOrInvoke = nullptr);
4700 RValue EmitSimpleCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,
4701 llvm::CallBase **CallOrInvoke = nullptr);
4702 CGCallee EmitCallee(const Expr *E);
4703
4704 void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl);
4705 void checkTargetFeatures(SourceLocation Loc, const FunctionDecl *TargetDecl);
4706
4707 llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4708 const Twine &name = "");
4709 llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4710 ArrayRef<llvm::Value *> args,
4711 const Twine &name = "");
4712 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4713 const Twine &Name = "");
4714 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4715 ArrayRef<llvm::Value *> Args,
4716 const Twine &Name = "");
4717 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4718 ArrayRef<llvm::Type *> Types,
4719 ArrayRef<llvm::Value *> Args,
4720 const Twine &Name = "");
4721 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4722 ArrayRef<llvm::Value *> Args,
4723 llvm::Type *RetTy, const Twine &Name = "");
4724 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4725 const Twine &name = "");
4726 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4727 ArrayRef<Address> args,
4728 const Twine &name = "");
4729 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4730 ArrayRef<llvm::Value *> args,
4731 const Twine &name = "");
4732
4733 SmallVector<llvm::OperandBundleDef, 1>
4734 getBundlesForFunclet(llvm::Value *Callee);
4735
4736 llvm::CallBase *EmitCallOrInvoke(llvm::FunctionCallee Callee,
4737 ArrayRef<llvm::Value *> Args,
4738 const Twine &Name = "");
4739 llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4740 ArrayRef<llvm::Value *> args,
4741 const Twine &name = "");
4742 llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4743 const Twine &name = "");
4744 void EmitNoreturnRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4745 ArrayRef<llvm::Value *> args);
4746
4747 CGCallee BuildAppleKextVirtualCall(const CXXMethodDecl *MD,
4748 NestedNameSpecifier Qual, llvm::Type *Ty);
4749
4750 CGCallee BuildAppleKextVirtualDestructorCall(const CXXDestructorDecl *DD,
4751 CXXDtorType Type,
4752 const CXXRecordDecl *RD);
4753
4754 bool isPointerKnownNonNull(const Expr *E);
4755 /// Check whether the underlying base pointer is a constant null.
4756 bool isUnderlyingBasePointerConstantNull(const Expr *E);
4757
4758 /// Create the discriminator from the storage address and the entity hash.
4759 llvm::Value *EmitPointerAuthBlendDiscriminator(llvm::Value *StorageAddress,
4760 llvm::Value *Discriminator);
4761 CGPointerAuthInfo EmitPointerAuthInfo(const PointerAuthSchema &Schema,
4762 llvm::Value *StorageAddress,
4763 llvm::ConstantInt *Discriminator);
4764 CGPointerAuthInfo EmitPointerAuthInfo(const PointerAuthSchema &Schema,
4765 llvm::Value *StorageAddress,
4766 GlobalDecl SchemaDecl,
4767 QualType SchemaType);
4768
4769 llvm::Value *EmitPointerAuthSign(const CGPointerAuthInfo &Info,
4770 llvm::Value *Pointer);
4771
4772 llvm::Value *EmitPointerAuthAuth(const CGPointerAuthInfo &Info,
4773 llvm::Value *Pointer);
4774
4775 llvm::Value *emitPointerAuthResign(llvm::Value *Pointer, QualType PointerType,
4776 const CGPointerAuthInfo &CurAuthInfo,
4777 const CGPointerAuthInfo &NewAuthInfo,
4778 bool IsKnownNonNull);
4779 llvm::Value *emitPointerAuthResignCall(llvm::Value *Pointer,
4780 const CGPointerAuthInfo &CurInfo,
4781 const CGPointerAuthInfo &NewInfo);
4782
4783 void EmitPointerAuthOperandBundle(
4784 const CGPointerAuthInfo &Info,
4785 SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
4786
4787 CGPointerAuthInfo EmitPointerAuthInfo(PointerAuthQualifier Qualifier,
4788 Address StorageAddress);
4789 llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
4790 llvm::Value *Pointer, QualType ValueType,
4791 Address StorageAddress,
4792 bool IsKnownNonNull);
4793 llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
4794 const Expr *PointerExpr,
4795 Address StorageAddress);
4796 llvm::Value *EmitPointerAuthUnqualify(PointerAuthQualifier Qualifier,
4797 llvm::Value *Pointer,
4798 QualType PointerType,
4799 Address StorageAddress,
4800 bool IsKnownNonNull);
4801 void EmitPointerAuthCopy(PointerAuthQualifier Qualifier, QualType Type,
4802 Address DestField, Address SrcField);
4803
4804 std::pair<llvm::Value *, CGPointerAuthInfo>
4805 EmitOrigPointerRValue(const Expr *E);
4806
4807 llvm::Value *authPointerToPointerCast(llvm::Value *ResultPtr,
4808 QualType SourceType, QualType DestType);
4809 Address authPointerToPointerCast(Address Ptr, QualType SourceType,
4810 QualType DestType);
4811
4812 Address getAsNaturalAddressOf(Address Addr, QualType PointeeTy);
4813
4814 llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) {
4815 return getAsNaturalAddressOf(Addr, PointeeTy: PointeeType).getBasePointer();
4816 }
4817
4818 // Return the copy constructor name with the prefix "__copy_constructor_"
4819 // removed.
4820 static std::string getNonTrivialCopyConstructorStr(QualType QT,
4821 CharUnits Alignment,
4822 bool IsVolatile,
4823 ASTContext &Ctx);
4824
4825 // Return the destructor name with the prefix "__destructor_" removed.
4826 static std::string getNonTrivialDestructorStr(QualType QT,
4827 CharUnits Alignment,
4828 bool IsVolatile,
4829 ASTContext &Ctx);
4830
4831 // These functions emit calls to the special functions of non-trivial C
4832 // structs.
4833 void defaultInitNonTrivialCStructVar(LValue Dst);
4834 void callCStructDefaultConstructor(LValue Dst);
4835 void callCStructDestructor(LValue Dst);
4836 void callCStructCopyConstructor(LValue Dst, LValue Src);
4837 void callCStructMoveConstructor(LValue Dst, LValue Src);
4838 void callCStructCopyAssignmentOperator(LValue Dst, LValue Src);
4839 void callCStructMoveAssignmentOperator(LValue Dst, LValue Src);
4840
4841 RValue EmitCXXMemberOrOperatorCall(
4842 const CXXMethodDecl *Method, const CGCallee &Callee,
4843 ReturnValueSlot ReturnValue, llvm::Value *This,
4844 llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *E,
4845 CallArgList *RtlArgs, llvm::CallBase **CallOrInvoke);
4846 RValue EmitCXXDestructorCall(GlobalDecl Dtor, const CGCallee &Callee,
4847 llvm::Value *This, QualType ThisTy,
4848 llvm::Value *ImplicitParam,
4849 QualType ImplicitParamTy, const CallExpr *E,
4850 llvm::CallBase **CallOrInvoke = nullptr);
4851 RValue EmitCXXMemberCallExpr(const CXXMemberCallExpr *E,
4852 ReturnValueSlot ReturnValue,
4853 llvm::CallBase **CallOrInvoke = nullptr);
4854 RValue EmitCXXMemberOrOperatorMemberCallExpr(
4855 const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue,
4856 bool HasQualifier, NestedNameSpecifier Qualifier, bool IsArrow,
4857 const Expr *Base, llvm::CallBase **CallOrInvoke);
4858 // Compute the object pointer.
4859 Address EmitCXXMemberDataPointerAddress(
4860 const Expr *E, Address base, llvm::Value *memberPtr,
4861 const MemberPointerType *memberPtrType, bool IsInBounds,
4862 LValueBaseInfo *BaseInfo = nullptr, TBAAAccessInfo *TBAAInfo = nullptr);
4863 RValue EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
4864 ReturnValueSlot ReturnValue,
4865 llvm::CallBase **CallOrInvoke);
4866
4867 RValue EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
4868 const CXXMethodDecl *MD,
4869 ReturnValueSlot ReturnValue,
4870 llvm::CallBase **CallOrInvoke);
4871 RValue EmitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *E);
4872
4873 RValue EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
4874 ReturnValueSlot ReturnValue,
4875 llvm::CallBase **CallOrInvoke);
4876
4877 RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E);
4878 RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E);
4879
4880 RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
4881 const CallExpr *E, ReturnValueSlot ReturnValue);
4882
4883 RValue emitRotate(const CallExpr *E, bool IsRotateRight);
4884
4885 RValue emitStdcCountIntrinsic(const CallExpr *E, llvm::Intrinsic::ID IntID,
4886 bool InvertArg, bool IsPop = false);
4887 RValue emitStdcBitWidthMinus(const CallExpr *E, llvm::Intrinsic::ID IntID,
4888 bool IsPop);
4889 RValue emitStdcFirstBit(const CallExpr *E, llvm::Intrinsic::ID IntID,
4890 bool InvertArg);
4891
4892 /// Emit IR for __builtin_os_log_format.
4893 RValue emitBuiltinOSLogFormat(const CallExpr &E);
4894
4895 /// Emit IR for __builtin_is_aligned.
4896 RValue EmitBuiltinIsAligned(const CallExpr *E);
4897 /// Emit IR for __builtin_align_up/__builtin_align_down.
4898 RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp);
4899
4900 llvm::Function *generateBuiltinOSLogHelperFunction(
4901 const analyze_os_log::OSLogBufferLayout &Layout,
4902 CharUnits BufferAlignment);
4903
4904 RValue EmitBlockCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,
4905 llvm::CallBase **CallOrInvoke);
4906
4907 /// EmitTargetBuiltinExpr - Emit the given builtin call. Returns 0 if the call
4908 /// is unhandled by the current target.
4909 llvm::Value *EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4910 ReturnValueSlot ReturnValue);
4911
4912 llvm::Value *
4913 EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty,
4914 const llvm::CmpInst::Predicate Pred,
4915 const llvm::Twine &Name = "");
4916 llvm::Value *EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4917 ReturnValueSlot ReturnValue,
4918 llvm::Triple::ArchType Arch);
4919 llvm::Value *EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4920 ReturnValueSlot ReturnValue,
4921 llvm::Triple::ArchType Arch);
4922 llvm::Value *EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4923 ReturnValueSlot ReturnValue,
4924 llvm::Triple::ArchType Arch);
4925 llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::IntegerType *ITy,
4926 QualType RTy);
4927 llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::ArrayType *ATy,
4928 QualType RTy);
4929
4930 llvm::Value *
4931 EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic,
4932 unsigned AltLLVMIntrinsic, const char *NameHint,
4933 unsigned Modifier, const CallExpr *E,
4934 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0,
4935 Address PtrOp1, llvm::Triple::ArchType Arch);
4936
4937 llvm::Function *LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
4938 unsigned Modifier, llvm::Type *ArgTy,
4939 const CallExpr *E);
4940 llvm::Value *EmitNeonCall(llvm::Function *F,
4941 SmallVectorImpl<llvm::Value *> &O, const char *name,
4942 unsigned shift = 0, bool rightshift = false);
4943 llvm::Value *EmitFP8NeonCall(unsigned IID, ArrayRef<llvm::Type *> Tys,
4944 SmallVectorImpl<llvm::Value *> &O,
4945 const CallExpr *E, const char *name);
4946 llvm::Value *EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,
4947 llvm::Type *Ty1, bool Extract,
4948 SmallVectorImpl<llvm::Value *> &Ops,
4949 const CallExpr *E, const char *name);
4950 llvm::Value *EmitFP8NeonFDOTCall(unsigned IID, bool ExtendLaneArg,
4951 llvm::Type *RetTy,
4952 SmallVectorImpl<llvm::Value *> &Ops,
4953 const CallExpr *E, const char *name);
4954 llvm::Value *EmitFP8NeonFMLACall(unsigned IID, bool ExtendLaneArg,
4955 llvm::Type *RetTy,
4956 SmallVectorImpl<llvm::Value *> &Ops,
4957 const CallExpr *E, const char *name);
4958 llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx,
4959 const llvm::ElementCount &Count);
4960 llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx);
4961 llvm::Value *EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty,
4962 bool negateForRightShift);
4963 llvm::Value *EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt,
4964 llvm::Type *Ty, bool usgn, const char *name);
4965 llvm::Value *vectorWrapScalar16(llvm::Value *Op);
4966 /// SVEBuiltinMemEltTy - Returns the memory element type for this memory
4967 /// access builtin. Only required if it can't be inferred from the base
4968 /// pointer operand.
4969 llvm::Type *SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags);
4970
4971 SmallVector<llvm::Type *, 2>
4972 getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType,
4973 ArrayRef<llvm::Value *> Ops);
4974 llvm::Type *getEltType(const SVETypeFlags &TypeFlags);
4975 llvm::ScalableVectorType *getSVEType(const SVETypeFlags &TypeFlags);
4976 llvm::ScalableVectorType *getSVEPredType(const SVETypeFlags &TypeFlags);
4977 llvm::Value *EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags,
4978 ArrayRef<llvm::Value *> Ops);
4979 llvm::Value *EmitSVETupleCreate(const SVETypeFlags &TypeFlags,
4980 llvm::Type *ReturnType,
4981 ArrayRef<llvm::Value *> Ops);
4982 llvm::Value *EmitSVEDupX(llvm::Value *Scalar);
4983 llvm::Value *EmitSVEDupX(llvm::Value *Scalar, llvm::Type *Ty);
4984 llvm::Value *EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty);
4985 llvm::Value *EmitSVEPMull(const SVETypeFlags &TypeFlags,
4986 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4987 unsigned BuiltinID);
4988 llvm::Value *EmitSVEMovl(const SVETypeFlags &TypeFlags,
4989 llvm::ArrayRef<llvm::Value *> Ops,
4990 unsigned BuiltinID);
4991 llvm::Value *EmitSVEPredicateCast(llvm::Value *Pred,
4992 llvm::ScalableVectorType *VTy);
4993 llvm::Value *EmitSVEPredicateTupleCast(llvm::Value *PredTuple,
4994 llvm::StructType *Ty);
4995 llvm::Value *EmitSVEGatherLoad(const SVETypeFlags &TypeFlags,
4996 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4997 unsigned IntID);
4998 llvm::Value *EmitSVEScatterStore(const SVETypeFlags &TypeFlags,
4999 llvm::SmallVectorImpl<llvm::Value *> &Ops,
5000 unsigned IntID);
5001 llvm::Value *EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy,
5002 SmallVectorImpl<llvm::Value *> &Ops,
5003 unsigned BuiltinID, bool IsZExtReturn);
5004 llvm::Value *EmitSVEMaskedStore(const CallExpr *,
5005 SmallVectorImpl<llvm::Value *> &Ops,
5006 unsigned BuiltinID);
5007 llvm::Value *EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,
5008 SmallVectorImpl<llvm::Value *> &Ops,
5009 unsigned BuiltinID);
5010 llvm::Value *EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags,
5011 SmallVectorImpl<llvm::Value *> &Ops,
5012 unsigned IntID);
5013 llvm::Value *EmitSVEStructLoad(const SVETypeFlags &TypeFlags,
5014 SmallVectorImpl<llvm::Value *> &Ops,
5015 unsigned IntID);
5016 llvm::Value *EmitSVEStructStore(const SVETypeFlags &TypeFlags,
5017 SmallVectorImpl<llvm::Value *> &Ops,
5018 unsigned IntID);
5019 llvm::Value *EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5020
5021 llvm::Value *EmitSMELd1St1(const SVETypeFlags &TypeFlags,
5022 llvm::SmallVectorImpl<llvm::Value *> &Ops,
5023 unsigned IntID);
5024 llvm::Value *EmitSMEReadWrite(const SVETypeFlags &TypeFlags,
5025 llvm::SmallVectorImpl<llvm::Value *> &Ops,
5026 unsigned IntID);
5027 llvm::Value *EmitSMEZero(const SVETypeFlags &TypeFlags,
5028 llvm::SmallVectorImpl<llvm::Value *> &Ops,
5029 unsigned IntID);
5030 llvm::Value *EmitSMELdrStr(const SVETypeFlags &TypeFlags,
5031 llvm::SmallVectorImpl<llvm::Value *> &Ops,
5032 unsigned IntID);
5033
5034 void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E,
5035 SmallVectorImpl<llvm::Value *> &Ops,
5036 SVETypeFlags TypeFlags);
5037
5038 llvm::Value *EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5039
5040 llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E,
5041 llvm::Triple::ArchType Arch);
5042 llvm::Value *EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5043
5044 llvm::Value *BuildVector(ArrayRef<llvm::Value *> Ops);
5045 llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5046 llvm::Value *EmitPPCBuiltinCpu(unsigned BuiltinID, llvm::Type *ReturnType,
5047 StringRef CPUStr);
5048 llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5049 llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5050 llvm::Value *EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
5051 ReturnValueSlot ReturnValue);
5052
5053 // Returns a builtin function that the SPIR-V backend will expand into a spec
5054 // constant.
5055 llvm::Function *
5056 getSpecConstantFunction(const clang::QualType &SpecConstantType);
5057
5058 llvm::Value *EmitDirectXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5059 llvm::Value *EmitSPIRVBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5060 llvm::Value *EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx,
5061 const CallExpr *E);
5062 llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5063 llvm::Value *EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5064 llvm::Value *EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
5065 const CallExpr *E);
5066 llvm::Value *EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5067 llvm::Value *EmitAVRBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5068 llvm::Value *EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
5069 ReturnValueSlot ReturnValue);
5070
5071 llvm::Value *EmitRISCVCpuSupports(const CallExpr *E);
5072 llvm::Value *EmitRISCVCpuSupports(ArrayRef<StringRef> FeaturesStrs);
5073 llvm::Value *EmitRISCVCpuInit();
5074 llvm::Value *EmitRISCVCpuIs(const CallExpr *E);
5075 llvm::Value *EmitRISCVCpuIs(StringRef CPUStr);
5076
5077 void AddAMDGPUFenceAddressSpaceMMRA(llvm::Instruction *Inst,
5078 const CallExpr *E);
5079 /// Attach the AMDGPU availability/visibility MMRA to \p Inst when the
5080 /// amdgpu_av attribute is active on the current statement.
5081 void AddAMDGPUAvailableVisibleMMRA(llvm::Instruction *Inst);
5082 void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope,
5083 llvm::AtomicOrdering &AO,
5084 llvm::SyncScope::ID &SSID);
5085
5086 enum class MSVCIntrin;
5087 llvm::Value *EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E);
5088
5089 llvm::Value *EmitBuiltinAvailable(const VersionTuple &Version);
5090
5091 llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E);
5092 llvm::Value *EmitObjCStringLiteral(const ObjCStringLiteral *E);
5093 llvm::Value *EmitObjCBoxedExpr(const ObjCBoxedExpr *E);
5094 llvm::Value *EmitObjCArrayLiteral(const ObjCArrayLiteral *E);
5095 llvm::Value *EmitObjCDictionaryLiteral(const ObjCDictionaryLiteral *E);
5096 llvm::Value *
5097 EmitObjCCollectionLiteral(const Expr *E,
5098 const ObjCMethodDecl *MethodWithObjects);
5099 llvm::Value *EmitObjCSelectorExpr(const ObjCSelectorExpr *E);
5100 RValue EmitObjCMessageExpr(const ObjCMessageExpr *E,
5101 ReturnValueSlot Return = ReturnValueSlot());
5102
5103 /// Retrieves the default cleanup kind for an ARC cleanup.
5104 /// Except under -fobjc-arc-eh, ARC cleanups are normal-only.
5105 CleanupKind getARCCleanupKind() {
5106 return CGM.getCodeGenOpts().ObjCAutoRefCountExceptions ? NormalAndEHCleanup
5107 : NormalCleanup;
5108 }
5109
5110 // ARC primitives.
5111 void EmitARCInitWeak(Address addr, llvm::Value *value);
5112 void EmitARCDestroyWeak(Address addr);
5113 llvm::Value *EmitARCLoadWeak(Address addr);
5114 llvm::Value *EmitARCLoadWeakRetained(Address addr);
5115 llvm::Value *EmitARCStoreWeak(Address addr, llvm::Value *value, bool ignored);
5116 void emitARCCopyAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
5117 void emitARCMoveAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
5118 void EmitARCCopyWeak(Address dst, Address src);
5119 void EmitARCMoveWeak(Address dst, Address src);
5120 llvm::Value *EmitARCRetainAutorelease(QualType type, llvm::Value *value);
5121 llvm::Value *EmitARCRetainAutoreleaseNonBlock(llvm::Value *value);
5122 llvm::Value *EmitARCStoreStrong(LValue lvalue, llvm::Value *value,
5123 bool resultIgnored);
5124 llvm::Value *EmitARCStoreStrongCall(Address addr, llvm::Value *value,
5125 bool resultIgnored);
5126 llvm::Value *EmitARCRetain(QualType type, llvm::Value *value);
5127 llvm::Value *EmitARCRetainNonBlock(llvm::Value *value);
5128 llvm::Value *EmitARCRetainBlock(llvm::Value *value, bool mandatory);
5129 void EmitARCDestroyStrong(Address addr, ARCPreciseLifetime_t precise);
5130 void EmitARCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
5131 llvm::Value *EmitARCAutorelease(llvm::Value *value);
5132 llvm::Value *EmitARCAutoreleaseReturnValue(llvm::Value *value);
5133 llvm::Value *EmitARCRetainAutoreleaseReturnValue(llvm::Value *value);
5134 llvm::Value *EmitARCRetainAutoreleasedReturnValue(llvm::Value *value);
5135 llvm::Value *EmitARCUnsafeClaimAutoreleasedReturnValue(llvm::Value *value);
5136
5137 llvm::Value *EmitObjCAutorelease(llvm::Value *value, llvm::Type *returnType);
5138 llvm::Value *EmitObjCRetainNonBlock(llvm::Value *value,
5139 llvm::Type *returnType);
5140 void EmitObjCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
5141
5142 std::pair<LValue, llvm::Value *>
5143 EmitARCStoreAutoreleasing(const BinaryOperator *e);
5144 std::pair<LValue, llvm::Value *> EmitARCStoreStrong(const BinaryOperator *e,
5145 bool ignored);
5146 std::pair<LValue, llvm::Value *>
5147 EmitARCStoreUnsafeUnretained(const BinaryOperator *e, bool ignored);
5148
5149 llvm::Value *EmitObjCAlloc(llvm::Value *value, llvm::Type *returnType);
5150 llvm::Value *EmitObjCAllocWithZone(llvm::Value *value,
5151 llvm::Type *returnType);
5152 llvm::Value *EmitObjCAllocInit(llvm::Value *value, llvm::Type *resultType);
5153
5154 llvm::Value *EmitObjCThrowOperand(const Expr *expr);
5155 llvm::Value *EmitObjCConsumeObject(QualType T, llvm::Value *Ptr);
5156 llvm::Value *EmitObjCExtendObjectLifetime(QualType T, llvm::Value *Ptr);
5157
5158 llvm::Value *EmitARCExtendBlockObject(const Expr *expr);
5159 llvm::Value *EmitARCReclaimReturnedObject(const Expr *e,
5160 bool allowUnsafeClaim);
5161 llvm::Value *EmitARCRetainScalarExpr(const Expr *expr);
5162 llvm::Value *EmitARCRetainAutoreleaseScalarExpr(const Expr *expr);
5163 llvm::Value *EmitARCUnsafeUnretainedScalarExpr(const Expr *expr);
5164
5165 void EmitARCIntrinsicUse(ArrayRef<llvm::Value *> values);
5166
5167 void EmitARCNoopIntrinsicUse(ArrayRef<llvm::Value *> values);
5168
5169 static Destroyer destroyARCStrongImprecise;
5170 static Destroyer destroyARCStrongPrecise;
5171 static Destroyer destroyARCWeak;
5172 static Destroyer emitARCIntrinsicUse;
5173 static Destroyer destroyNonTrivialCStruct;
5174
5175 void EmitObjCAutoreleasePoolPop(llvm::Value *Ptr);
5176 llvm::Value *EmitObjCAutoreleasePoolPush();
5177 llvm::Value *EmitObjCMRRAutoreleasePoolPush();
5178 void EmitObjCAutoreleasePoolCleanup(llvm::Value *Ptr);
5179 void EmitObjCMRRAutoreleasePoolPop(llvm::Value *Ptr);
5180
5181 /// Emits a reference binding to the passed in expression.
5182 RValue EmitReferenceBindingToExpr(const Expr *E);
5183
5184 //===--------------------------------------------------------------------===//
5185 // Expression Emission
5186 //===--------------------------------------------------------------------===//
5187
5188 // Expressions are broken into three classes: scalar, complex, aggregate.
5189
5190 /// EmitScalarExpr - Emit the computation of the specified expression of LLVM
5191 /// scalar type, returning the result.
5192 llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
5193
5194 /// Emit a conversion from the specified type to the specified destination
5195 /// type, both of which are LLVM scalar types.
5196 llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
5197 QualType DstTy, SourceLocation Loc);
5198
5199 /// Emit a conversion from the specified complex type to the specified
5200 /// destination type, where the destination type is an LLVM scalar type.
5201 llvm::Value *EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy,
5202 QualType DstTy,
5203 SourceLocation Loc);
5204
5205 /// EmitAggExpr - Emit the computation of the specified expression
5206 /// of aggregate type. The result is computed into the given slot,
5207 /// which may be null to indicate that the value is not needed.
5208 void EmitAggExpr(const Expr *E, AggValueSlot AS);
5209
5210 /// EmitAggExprToLValue - Emit the computation of the specified expression of
5211 /// aggregate type into a temporary LValue.
5212 LValue EmitAggExprToLValue(const Expr *E);
5213
5214 enum ExprValueKind { EVK_RValue, EVK_NonRValue };
5215
5216 /// EmitAggFinalDestCopy - Emit copy of the specified aggregate into
5217 /// destination address.
5218 void EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest, const LValue &Src,
5219 ExprValueKind SrcKind);
5220
5221 /// Create a store to \arg DstPtr from \arg Src, truncating the stored value
5222 /// to at most \arg DstSize bytes.
5223 void CreateCoercedStore(llvm::Value *Src, QualType SrcFETy, Address Dst,
5224 llvm::TypeSize DstSize, bool DstIsVolatile);
5225
5226 /// EmitExtendGCLifetime - Given a pointer to an Objective-C object,
5227 /// make sure it survives garbage collection until this point.
5228 void EmitExtendGCLifetime(llvm::Value *object);
5229
5230 /// EmitComplexExpr - Emit the computation of the specified expression of
5231 /// complex type, returning the result.
5232 ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal = false,
5233 bool IgnoreImag = false);
5234
5235 /// EmitComplexExprIntoLValue - Emit the given expression of complex
5236 /// type and place its result into the specified l-value.
5237 void EmitComplexExprIntoLValue(const Expr *E, LValue dest, bool isInit);
5238
5239 /// EmitStoreOfComplex - Store a complex number into the specified l-value.
5240 void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit);
5241
5242 /// EmitLoadOfComplex - Load a complex number from the specified l-value.
5243 ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc);
5244
5245 ComplexPairTy EmitPromotedComplexExpr(const Expr *E, QualType PromotionType);
5246 llvm::Value *EmitPromotedScalarExpr(const Expr *E, QualType PromotionType);
5247 ComplexPairTy EmitPromotedValue(ComplexPairTy result, QualType PromotionType);
5248 ComplexPairTy EmitUnPromotedValue(ComplexPairTy result,
5249 QualType PromotionType);
5250
5251 Address emitAddrOfRealComponent(Address complex, QualType complexType);
5252 Address emitAddrOfImagComponent(Address complex, QualType complexType);
5253
5254 /// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the
5255 /// global variable that has already been created for it. If the initializer
5256 /// has a different type than GV does, this may free GV and return a different
5257 /// one. Otherwise it just returns GV.
5258 llvm::GlobalVariable *AddInitializerToStaticVarDecl(const VarDecl &D,
5259 llvm::GlobalVariable *GV);
5260
5261 // Emit an @llvm.invariant.start call for the given memory region.
5262 void EmitInvariantStart(llvm::Constant *Addr, CharUnits Size);
5263
5264 /// EmitCXXGlobalVarDeclInit - Create the initializer for a C++
5265 /// variable with global storage.
5266 void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::GlobalVariable *GV,
5267 bool PerformInit);
5268
5269 llvm::Constant *createAtExitStub(const VarDecl &VD, llvm::FunctionCallee Dtor,
5270 llvm::Constant *Addr);
5271
5272 llvm::Function *createTLSAtExitStub(const VarDecl &VD,
5273 llvm::FunctionCallee Dtor,
5274 llvm::Constant *Addr,
5275 llvm::FunctionCallee &AtExit);
5276
5277 /// Call atexit() with a function that passes the given argument to
5278 /// the given function.
5279 void registerGlobalDtorWithAtExit(const VarDecl &D, llvm::FunctionCallee fn,
5280 llvm::Constant *addr);
5281
5282 /// Registers the dtor using 'llvm.global_dtors' for platforms that do not
5283 /// support an 'atexit()' function.
5284 void registerGlobalDtorWithLLVM(const VarDecl &D, llvm::FunctionCallee fn,
5285 llvm::Constant *addr);
5286
5287 /// Call atexit() with function dtorStub.
5288 void registerGlobalDtorWithAtExit(llvm::Constant *dtorStub);
5289
5290 /// Call unatexit() with function dtorStub.
5291 llvm::Value *unregisterGlobalDtorWithUnAtExit(llvm::Constant *dtorStub);
5292
5293 /// Emit code in this function to perform a guarded variable
5294 /// initialization. Guarded initializations are used when it's not
5295 /// possible to prove that an initialization will be done exactly
5296 /// once, e.g. with a static local variable or a static data member
5297 /// of a class template.
5298 void EmitCXXGuardedInit(const VarDecl &D, llvm::GlobalVariable *DeclPtr,
5299 bool PerformInit);
5300
5301 enum class GuardKind { VariableGuard, TlsGuard };
5302
5303 /// Emit a branch to select whether or not to perform guarded initialization.
5304 void EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
5305 llvm::BasicBlock *InitBlock,
5306 llvm::BasicBlock *NoInitBlock, GuardKind Kind,
5307 const VarDecl *D);
5308
5309 /// GenerateCXXGlobalInitFunc - Generates code for initializing global
5310 /// variables.
5311 void
5312 GenerateCXXGlobalInitFunc(llvm::Function *Fn,
5313 ArrayRef<llvm::Function *> CXXThreadLocals,
5314 ConstantAddress Guard = ConstantAddress::invalid());
5315
5316 /// GenerateCXXGlobalCleanUpFunc - Generates code for cleaning up global
5317 /// variables.
5318 void GenerateCXXGlobalCleanUpFunc(
5319 llvm::Function *Fn,
5320 ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
5321 llvm::Constant *>>
5322 DtorsOrStermFinalizers);
5323
5324 void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn, const VarDecl *D,
5325 llvm::GlobalVariable *Addr,
5326 bool PerformInit);
5327
5328 void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest);
5329
5330 void EmitSynthesizedCXXCopyCtor(Address Dest, Address Src, const Expr *Exp);
5331
5332 void EmitCXXThrowExpr(const CXXThrowExpr *E, bool KeepInsertionPoint = true);
5333
5334 RValue EmitAtomicExpr(AtomicExpr *E);
5335
5336 void EmitFakeUse(Address Addr);
5337
5338 //===--------------------------------------------------------------------===//
5339 // Annotations Emission
5340 //===--------------------------------------------------------------------===//
5341
5342 /// Emit an annotation call (intrinsic).
5343 llvm::Value *EmitAnnotationCall(llvm::Function *AnnotationFn,
5344 llvm::Value *AnnotatedVal,
5345 StringRef AnnotationStr,
5346 SourceLocation Location,
5347 const AnnotateAttr *Attr);
5348
5349 /// Emit local annotations for the local variable V, declared by D.
5350 void EmitVarAnnotations(const VarDecl *D, llvm::Value *V);
5351
5352 /// Emit field annotations for the given field & value. Returns the
5353 /// annotation result.
5354 Address EmitFieldAnnotations(const FieldDecl *D, Address V);
5355
5356 //===--------------------------------------------------------------------===//
5357 // Internal Helpers
5358 //===--------------------------------------------------------------------===//
5359
5360 /// ContainsLabel - Return true if the statement contains a label in it. If
5361 /// this statement is not executed normally, it not containing a label means
5362 /// that we can just remove the code.
5363 static bool ContainsLabel(const Stmt *S, bool IgnoreCaseStmts = false);
5364
5365 /// containsBreak - Return true if the statement contains a break out of it.
5366 /// If the statement (recursively) contains a switch or loop with a break
5367 /// inside of it, this is fine.
5368 static bool containsBreak(const Stmt *S);
5369
5370 /// Determine if the given statement might introduce a declaration into the
5371 /// current scope, by being a (possibly-labelled) DeclStmt.
5372 static bool mightAddDeclToScope(const Stmt *S);
5373
5374 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
5375 /// to a constant, or if it does but contains a label, return false. If it
5376 /// constant folds return true and set the boolean result in Result.
5377 bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result,
5378 bool AllowLabels = false);
5379
5380 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
5381 /// to a constant, or if it does but contains a label, return false. If it
5382 /// constant folds return true and set the folded value.
5383 bool ConstantFoldsToSimpleInteger(const Expr *Cond, llvm::APSInt &Result,
5384 bool AllowLabels = false);
5385
5386 /// Ignore parentheses and logical-NOT to track conditions consistently.
5387 static const Expr *stripCond(const Expr *C);
5388
5389 /// isInstrumentedCondition - Determine whether the given condition is an
5390 /// instrumentable condition (i.e. no "&&" or "||").
5391 static bool isInstrumentedCondition(const Expr *C);
5392
5393 /// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
5394 /// increments a profile counter based on the semantics of the given logical
5395 /// operator opcode. This is used to instrument branch condition coverage
5396 /// for logical operators.
5397 void EmitBranchToCounterBlock(const Expr *Cond, BinaryOperator::Opcode LOp,
5398 llvm::BasicBlock *TrueBlock,
5399 llvm::BasicBlock *FalseBlock,
5400 uint64_t TrueCount = 0,
5401 Stmt::Likelihood LH = Stmt::LH_None,
5402 const Expr *CntrIdx = nullptr);
5403
5404 /// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an
5405 /// if statement) to the specified blocks. Based on the condition, this might
5406 /// try to simplify the codegen of the conditional based on the branch.
5407 /// TrueCount should be the number of times we expect the condition to
5408 /// evaluate to true based on PGO data.
5409 void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock,
5410 llvm::BasicBlock *FalseBlock, uint64_t TrueCount,
5411 Stmt::Likelihood LH = Stmt::LH_None,
5412 const Expr *ConditionalOp = nullptr,
5413 const VarDecl *ConditionalDecl = nullptr);
5414
5415 /// Given an assignment `*LHS = RHS`, emit a test that checks if \p RHS is
5416 /// nonnull, if \p LHS is marked _Nonnull.
5417 void EmitNullabilityCheck(LValue LHS, llvm::Value *RHS, SourceLocation Loc);
5418
5419 /// An enumeration which makes it easier to specify whether or not an
5420 /// operation is a subtraction.
5421 enum { NotSubtraction = false, IsSubtraction = true };
5422
5423 /// Emit pointer + index arithmetic.
5424 llvm::Value *EmitPointerArithmetic(const BinaryOperator *BO,
5425 Expr *pointerOperand, llvm::Value *pointer,
5426 Expr *indexOperand, llvm::Value *index,
5427 bool isSubtraction);
5428
5429 /// Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to
5430 /// detect undefined behavior when the pointer overflow sanitizer is enabled.
5431 /// \p SignedIndices indicates whether any of the GEP indices are signed.
5432 /// \p IsSubtraction indicates whether the expression used to form the GEP
5433 /// is a subtraction.
5434 llvm::Value *EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr,
5435 ArrayRef<llvm::Value *> IdxList,
5436 bool SignedIndices, bool IsSubtraction,
5437 SourceLocation Loc,
5438 const Twine &Name = "");
5439
5440 Address EmitCheckedInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
5441 llvm::Type *elementType, bool SignedIndices,
5442 bool IsSubtraction, SourceLocation Loc,
5443 CharUnits Align, const Twine &Name = "");
5444
5445 /// Specifies which type of sanitizer check to apply when handling a
5446 /// particular builtin.
5447 enum BuiltinCheckKind {
5448 BCK_CTZPassedZero,
5449 BCK_CLZPassedZero,
5450 BCK_AssumePassedFalse,
5451 };
5452
5453 /// Emits an argument for a call to a builtin. If the builtin sanitizer is
5454 /// enabled, a runtime check specified by \p Kind is also emitted.
5455 llvm::Value *EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind);
5456
5457 /// Emits an argument for a call to a `__builtin_assume`. If the builtin
5458 /// sanitizer is enabled, a runtime check is also emitted.
5459 llvm::Value *EmitCheckedArgForAssume(const Expr *E);
5460
5461 /// Emit a description of a type in a format suitable for passing to
5462 /// a runtime sanitizer handler.
5463 llvm::Constant *EmitCheckTypeDescriptor(QualType T);
5464
5465 /// Convert a value into a format suitable for passing to a runtime
5466 /// sanitizer handler.
5467 llvm::Value *EmitCheckValue(llvm::Value *V);
5468
5469 /// Emit a description of a source location in a format suitable for
5470 /// passing to a runtime sanitizer handler.
5471 llvm::Constant *EmitCheckSourceLocation(SourceLocation Loc);
5472
5473 void EmitKCFIOperandBundle(const CGCallee &Callee,
5474 SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
5475
5476 /// Create a basic block that will either trap or call a handler function in
5477 /// the UBSan runtime with the provided arguments, and create a conditional
5478 /// branch to it.
5479 void
5480 EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
5481 Checked,
5482 SanitizerHandler Check, ArrayRef<llvm::Constant *> StaticArgs,
5483 ArrayRef<llvm::Value *> DynamicArgs,
5484 const TrapReason *TR = nullptr);
5485
5486 /// Emit a slow path cross-DSO CFI check which calls __cfi_slowpath
5487 /// if Cond if false.
5488 void EmitCfiSlowPathCheck(SanitizerKind::SanitizerOrdinal Ordinal,
5489 llvm::Value *Cond, llvm::ConstantInt *TypeId,
5490 llvm::Value *Ptr,
5491 ArrayRef<llvm::Constant *> StaticArgs);
5492
5493 /// Emit a reached-unreachable diagnostic if \p Loc is valid and runtime
5494 /// checking is enabled. Otherwise, just emit an unreachable instruction.
5495 void EmitUnreachable(SourceLocation Loc);
5496
5497 /// Create a basic block that will call the trap intrinsic, and emit a
5498 /// conditional branch to it, for the -ftrapv checks.
5499 void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID,
5500 bool NoMerge = false, const TrapReason *TR = nullptr);
5501
5502 /// Emit a call to trap or debugtrap. If 'EnsureInsertPoint' is false, the
5503 /// IR builder need not have a valid insert point after this returns.
5504 llvm::CallInst *EmitTrapCall(llvm::Intrinsic::ID IntrID,
5505 bool EnsureInsertPoint = true);
5506
5507 /// Emit a call to '\@llvm.trap()' and clear the current insert point.
5508 void EmitTrapCallAndMakeUnreachable();
5509
5510 /// Emit a stub for the cross-DSO CFI check function.
5511 void EmitCfiCheckStub();
5512
5513 /// Emit a cross-DSO CFI failure handling function.
5514 void EmitCfiCheckFail();
5515
5516 /// Create a check for a function parameter that may potentially be
5517 /// declared as non-null.
5518 void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc,
5519 AbstractCallee AC, unsigned ParmNum);
5520
5521 void EmitNonNullArgCheck(Address Addr, QualType ArgType,
5522 SourceLocation ArgLoc, AbstractCallee AC,
5523 unsigned ParmNum);
5524
5525 /// EmitWriteback - Emit callbacks for function.
5526 void EmitWritebacks(const CallArgList &Args);
5527
5528 /// EmitCallArg - Emit a single call argument.
5529 void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType);
5530
5531 /// EmitDelegateCallArg - We are performing a delegate call; that
5532 /// is, the current function is delegating to another one. Produce
5533 /// a r-value suitable for passing the given parameter.
5534 void EmitDelegateCallArg(CallArgList &args, const VarDecl *param,
5535 SourceLocation loc);
5536
5537 /// SetFPAccuracy - Set the minimum required accuracy of the given floating
5538 /// point operation, expressed as the maximum relative error in ulp.
5539 void SetFPAccuracy(llvm::Value *Val, float Accuracy);
5540
5541 /// Set the minimum required accuracy of the given sqrt operation
5542 /// based on CodeGenOpts.
5543 void SetSqrtFPAccuracy(llvm::Value *Val);
5544
5545 /// Set the minimum required accuracy of the given sqrt operation based on
5546 /// CodeGenOpts.
5547 void SetDivFPAccuracy(llvm::Value *Val);
5548
5549 /// Set the codegen fast-math flags.
5550 void SetFastMathFlags(FPOptions FPFeatures);
5551
5552 // Truncate or extend a boolean vector to the requested number of elements.
5553 llvm::Value *emitBoolVecConversion(llvm::Value *SrcVec,
5554 unsigned NumElementsDst,
5555 const llvm::Twine &Name = "");
5556
5557 void maybeAttachRangeForLoad(llvm::LoadInst *Load, QualType Ty,
5558 SourceLocation Loc);
5559
5560 // Emits a convergence_loop instruction for the given |BB|, with |ParentToken|
5561 // as it's parent convergence instr.
5562 llvm::ConvergenceControlInst *emitConvergenceLoopToken(llvm::BasicBlock *BB);
5563
5564private:
5565 // Adds a convergence_ctrl token with |ParentToken| as parent convergence
5566 // instr to the call |Input|.
5567 llvm::CallBase *addConvergenceControlToken(llvm::CallBase *Input);
5568
5569 // Find the convergence_entry instruction |F|, or emits ones if none exists.
5570 // Returns the convergence instruction.
5571 llvm::ConvergenceControlInst *
5572 getOrEmitConvergenceEntryToken(llvm::Function *F);
5573
5574private:
5575 llvm::MDNode *getRangeForLoadFromType(QualType Ty);
5576 void EmitReturnOfRValue(RValue RV, QualType Ty);
5577
5578 void deferPlaceholderReplacement(llvm::Instruction *Old, llvm::Value *New);
5579
5580 llvm::SmallVector<std::pair<llvm::WeakTrackingVH, llvm::Value *>, 4>
5581 DeferredReplacements;
5582
5583 /// Set the address of a local variable.
5584 void setAddrOfLocalVar(const VarDecl *VD, Address Addr) {
5585 assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!");
5586 LocalDeclMap.insert(KV: {VD, Addr});
5587 }
5588
5589 /// ExpandTypeFromArgs - Reconstruct a structure of type \arg Ty
5590 /// from function arguments into \arg Dst. See ABIArgInfo::Expand.
5591 ///
5592 /// \param AI - The first function argument of the expansion.
5593 void ExpandTypeFromArgs(QualType Ty, LValue Dst,
5594 llvm::Function::arg_iterator &AI);
5595
5596 /// ExpandTypeToArgs - Expand an CallArg \arg Arg, with the LLVM type for \arg
5597 /// Ty, into individual arguments on the provided vector \arg IRCallArgs,
5598 /// starting at index \arg IRCallArgPos. See ABIArgInfo::Expand.
5599 void ExpandTypeToArgs(QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
5600 SmallVectorImpl<llvm::Value *> &IRCallArgs,
5601 unsigned &IRCallArgPos);
5602
5603 std::pair<llvm::Value *, llvm::Type *>
5604 EmitAsmInput(const TargetInfo::ConstraintInfo &Info, const Expr *InputExpr,
5605 std::string &ConstraintStr);
5606
5607 std::pair<llvm::Value *, llvm::Type *>
5608 EmitAsmInputLValue(const TargetInfo::ConstraintInfo &Info, LValue InputValue,
5609 QualType InputType, std::string &ConstraintStr,
5610 SourceLocation Loc);
5611
5612 /// Attempts to statically evaluate the object size of E. If that
5613 /// fails, emits code to figure the size of E out for us. This is
5614 /// pass_object_size aware.
5615 ///
5616 /// If EmittedExpr is non-null, this will use that instead of re-emitting E.
5617 llvm::Value *evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
5618 llvm::IntegerType *ResType,
5619 llvm::Value *EmittedE,
5620 bool IsDynamic);
5621
5622 /// Emits the size of E, as required by __builtin_object_size. This
5623 /// function is aware of pass_object_size parameters, and will act accordingly
5624 /// if E is a parameter with the pass_object_size attribute.
5625 llvm::Value *emitBuiltinObjectSize(const Expr *E, unsigned Type,
5626 llvm::IntegerType *ResType,
5627 llvm::Value *EmittedE, bool IsDynamic);
5628
5629 llvm::Value *emitCountedBySize(const Expr *E, llvm::Value *EmittedE,
5630 unsigned Type, llvm::IntegerType *ResType);
5631
5632 llvm::Value *emitCountedByMemberSize(const MemberExpr *E, const Expr *Idx,
5633 llvm::Value *EmittedE,
5634 QualType CastedArrayElementTy,
5635 unsigned Type,
5636 llvm::IntegerType *ResType);
5637
5638 llvm::Value *emitCountedByPointerSize(const ImplicitCastExpr *E,
5639 const Expr *Idx, llvm::Value *EmittedE,
5640 QualType CastedArrayElementTy,
5641 unsigned Type,
5642 llvm::IntegerType *ResType);
5643
5644 void emitZeroOrPatternForAutoVarInit(QualType type, const VarDecl &D,
5645 Address Loc);
5646 LangOptions::TrivialAutoVarInitKind getAutoVarInitKind(QualType Ty,
5647 const VarDecl &D);
5648
5649public:
5650 enum class EvaluationOrder {
5651 ///! No language constraints on evaluation order.
5652 Default,
5653 ///! Language semantics require left-to-right evaluation.
5654 ForceLeftToRight,
5655 ///! Language semantics require right-to-left evaluation.
5656 ForceRightToLeft
5657 };
5658
5659 // Wrapper for function prototype sources. Wraps either a FunctionProtoType or
5660 // an ObjCMethodDecl.
5661 struct PrototypeWrapper {
5662 llvm::PointerUnion<const FunctionProtoType *, const ObjCMethodDecl *> P;
5663
5664 PrototypeWrapper(const FunctionProtoType *FT) : P(FT) {}
5665 PrototypeWrapper(const ObjCMethodDecl *MD) : P(MD) {}
5666 };
5667
5668 void EmitCallArgs(CallArgList &Args, PrototypeWrapper Prototype,
5669 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
5670 AbstractCallee AC = AbstractCallee(),
5671 unsigned ParamsToSkip = 0,
5672 EvaluationOrder Order = EvaluationOrder::Default);
5673
5674 /// EmitPointerWithAlignment - Given an expression with a pointer type,
5675 /// emit the value and compute our best estimate of the alignment of the
5676 /// pointee.
5677 ///
5678 /// \param BaseInfo - If non-null, this will be initialized with
5679 /// information about the source of the alignment and the may-alias
5680 /// attribute. Note that this function will conservatively fall back on
5681 /// the type when it doesn't recognize the expression and may-alias will
5682 /// be set to false.
5683 ///
5684 /// One reasonable way to use this information is when there's a language
5685 /// guarantee that the pointer must be aligned to some stricter value, and
5686 /// we're simply trying to ensure that sufficiently obvious uses of under-
5687 /// aligned objects don't get miscompiled; for example, a placement new
5688 /// into the address of a local variable. In such a case, it's quite
5689 /// reasonable to just ignore the returned alignment when it isn't from an
5690 /// explicit source.
5691 Address
5692 EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo = nullptr,
5693 TBAAAccessInfo *TBAAInfo = nullptr,
5694 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
5695
5696 /// If \p E references a parameter with pass_object_size info or a constant
5697 /// array size modifier, emit the object size divided by the size of \p EltTy.
5698 /// Otherwise return null.
5699 llvm::Value *LoadPassedObjectSize(const Expr *E, QualType EltTy);
5700
5701 void EmitSanitizerStatReport(llvm::SanitizerStatKind SSK);
5702
5703 struct FMVResolverOption {
5704 llvm::Function *Function;
5705 llvm::SmallVector<StringRef, 8> Features;
5706 std::optional<StringRef> Architecture;
5707
5708 FMVResolverOption(llvm::Function *F, ArrayRef<StringRef> Feats,
5709 std::optional<StringRef> Arch = std::nullopt)
5710 : Function(F), Features(Feats), Architecture(Arch) {}
5711 };
5712
5713 // Emits the body of a multiversion function's resolver. Assumes that the
5714 // options are already sorted in the proper order, with the 'default' option
5715 // last (if it exists).
5716 void EmitMultiVersionResolver(llvm::Function *Resolver,
5717 ArrayRef<FMVResolverOption> Options);
5718 void EmitX86MultiVersionResolver(llvm::Function *Resolver,
5719 ArrayRef<FMVResolverOption> Options);
5720 void EmitAArch64MultiVersionResolver(llvm::Function *Resolver,
5721 ArrayRef<FMVResolverOption> Options);
5722 void EmitRISCVMultiVersionResolver(llvm::Function *Resolver,
5723 ArrayRef<FMVResolverOption> Options);
5724 void EmitPPCAIXMultiVersionResolver(llvm::Function *Resolver,
5725 ArrayRef<FMVResolverOption> Options);
5726
5727 Address EmitAddressOfPFPField(Address RecordPtr, const PFPField &Field);
5728 Address EmitAddressOfPFPField(Address RecordPtr, Address FieldPtr,
5729 const FieldDecl *Field);
5730
5731private:
5732 QualType getVarArgType(const Expr *Arg);
5733
5734 void EmitDeclMetadata();
5735
5736 BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType,
5737 const AutoVarEmission &emission);
5738
5739 void AddObjCARCExceptionMetadata(llvm::Instruction *Inst);
5740
5741 llvm::Value *GetValueForARMHint(unsigned BuiltinID);
5742 llvm::Value *EmitX86CpuIs(const CallExpr *E);
5743 llvm::Value *EmitX86CpuIs(StringRef CPUStr);
5744 llvm::Value *EmitX86CpuSupports(const CallExpr *E);
5745 llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs);
5746 llvm::Value *EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask);
5747 llvm::Value *EmitX86CpuInit();
5748 llvm::Value *FormX86ResolverCondition(const FMVResolverOption &RO);
5749 llvm::Value *EmitAArch64CpuInit();
5750 llvm::Value *FormAArch64ResolverCondition(const FMVResolverOption &RO);
5751 llvm::Value *EmitAArch64CpuSupports(const CallExpr *E);
5752 llvm::Value *EmitAArch64CpuSupports(ArrayRef<StringRef> FeatureStrs);
5753};
5754
5755inline DominatingLLVMValue::saved_type
5756DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) {
5757 if (!needsSaving(value))
5758 return saved_type(value);
5759
5760 // Otherwise, we need an alloca.
5761 auto align = CharUnits::fromQuantity(
5762 Quantity: CGF.CGM.getDataLayout().getPrefTypeAlign(Ty: value->getType()))
5763 .getAsAlign();
5764 llvm::AllocaInst *AI =
5765 CGF.CreateTempAlloca(Ty: value->getType(), Name: "cond-cleanup.save");
5766 AI->setAlignment(align);
5767 CGF.Builder.CreateAlignedStore(Val: value, Ptr: AI, Align: align);
5768
5769 return saved_type(AI, value->getType());
5770}
5771
5772inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF,
5773 saved_type value) {
5774 // If the value says it wasn't saved, trust that it's still dominating.
5775 if (!value.isSaved())
5776 return value.Value;
5777
5778 // Otherwise, it should be an alloca instruction, as set up in save().
5779 auto Alloca = cast<llvm::AllocaInst>(Val: value.Value);
5780 return CGF.Builder.CreateAlignedLoad(Ty: value.Type, Ptr: Alloca, Align: Alloca->getAlign());
5781}
5782
5783} // end namespace CodeGen
5784
5785// Map the LangOption for floating point exception behavior into
5786// the corresponding enum in the IR.
5787llvm::fp::ExceptionBehavior
5788ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind);
5789} // end namespace clang
5790
5791#endif
5792