1//===-- CodeGenFunction.h - Per-Function state for LLVM CodeGen -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the internal per-function state used for llvm translation.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
14#define LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
15
16#include "CGBuilder.h"
17#include "CGLoopInfo.h"
18#include "CGValue.h"
19#include "CodeGenModule.h"
20#include "EHScopeStack.h"
21#include "SanitizerHandler.h"
22#include "VarBypassDetector.h"
23#include "clang/AST/CharUnits.h"
24#include "clang/AST/CurrentSourceLocExprScope.h"
25#include "clang/AST/ExprCXX.h"
26#include "clang/AST/ExprObjC.h"
27#include "clang/AST/ExprOpenMP.h"
28#include "clang/AST/StmtOpenACC.h"
29#include "clang/AST/StmtOpenMP.h"
30#include "clang/AST/StmtSYCL.h"
31#include "clang/AST/Type.h"
32#include "clang/Basic/ABI.h"
33#include "clang/Basic/CapturedStmt.h"
34#include "clang/Basic/CodeGenOptions.h"
35#include "clang/Basic/OpenMPKinds.h"
36#include "clang/Basic/TargetInfo.h"
37#include "llvm/ADT/ArrayRef.h"
38#include "llvm/ADT/DenseMap.h"
39#include "llvm/ADT/MapVector.h"
40#include "llvm/ADT/SmallVector.h"
41#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
42#include "llvm/IR/Instructions.h"
43#include "llvm/IR/ValueHandle.h"
44#include "llvm/Support/Debug.h"
45#include "llvm/Transforms/Utils/SanitizerStats.h"
46#include <optional>
47
48namespace llvm {
49class BasicBlock;
50class ConvergenceControlInst;
51class LLVMContext;
52class MDNode;
53class SwitchInst;
54class Twine;
55class Value;
56class CanonicalLoopInfo;
57} // namespace llvm
58
59namespace clang {
60class ASTContext;
61class AsmConstraintsInfo;
62class CXXDestructorDecl;
63class CXXForRangeStmt;
64class CXXTryStmt;
65class Decl;
66class LabelDecl;
67class FunctionDecl;
68class FunctionProtoType;
69class LabelStmt;
70class ObjCContainerDecl;
71class ObjCInterfaceDecl;
72class ObjCIvarDecl;
73class ObjCMethodDecl;
74class ObjCImplementationDecl;
75class ObjCPropertyImplDecl;
76class TargetInfo;
77class VarDecl;
78class ObjCForCollectionStmt;
79class ObjCAtTryStmt;
80class ObjCAtThrowStmt;
81class ObjCAtSynchronizedStmt;
82class ObjCAutoreleasePoolStmt;
83class OMPUseDevicePtrClause;
84class OMPUseDeviceAddrClause;
85class SVETypeFlags;
86class OMPExecutableDirective;
87
88namespace analyze_os_log {
89class OSLogBufferLayout;
90}
91
92namespace CodeGen {
93class CodeGenTypes;
94class CodeGenPGO;
95class CGCallee;
96class CGFunctionInfo;
97class CGBlockInfo;
98class CGCXXABI;
99class BlockByrefHelpers;
100class BlockByrefInfo;
101class BlockFieldFlags;
102class RegionCodeGenTy;
103class TargetCodeGenInfo;
104struct OMPTaskDataTy;
105struct CGCoroData;
106
107// clang-format off
108/// The kind of evaluation to perform on values of a particular
109/// type. Basically, is the code in CGExprScalar, CGExprComplex, or
110/// CGExprAgg?
111///
112/// TODO: should vectors maybe be split out into their own thing?
113enum TypeEvaluationKind {
114 TEK_Scalar,
115 TEK_Complex,
116 TEK_Aggregate
117};
118// clang-format on
119
120/// Helper class with most of the code for saving a value for a
121/// conditional expression cleanup.
122struct DominatingLLVMValue {
123 struct saved_type {
124 llvm::Value *Value; // Original value if not saved, alloca if saved
125 llvm::Type *Type; // nullptr if not saved, element type if saved
126
127 saved_type() : Value(nullptr), Type(nullptr) {}
128 saved_type(llvm::Value *V) : Value(V), Type(nullptr) {}
129 saved_type(llvm::AllocaInst *Alloca, llvm::Type *Ty)
130 : Value(Alloca), Type(Ty) {}
131
132 bool isSaved() const { return Type != nullptr; }
133 };
134
135 /// Answer whether the given value needs extra work to be saved.
136 static bool needsSaving(llvm::Value *value) {
137 if (!value)
138 return false;
139
140 // If it's not an instruction, we don't need to save.
141 if (!isa<llvm::Instruction>(Val: value))
142 return false;
143
144 // If it's an instruction in the entry block, we don't need to save.
145 llvm::BasicBlock *block = cast<llvm::Instruction>(Val: value)->getParent();
146 return (block != &block->getParent()->getEntryBlock());
147 }
148
149 static saved_type save(CodeGenFunction &CGF, llvm::Value *value);
150 static llvm::Value *restore(CodeGenFunction &CGF, saved_type value);
151};
152
153/// A partial specialization of DominatingValue for llvm::Values that
154/// might be llvm::Instructions.
155template <class T> struct DominatingPointer<T, true> : DominatingLLVMValue {
156 typedef T *type;
157 static type restore(CodeGenFunction &CGF, saved_type value) {
158 return static_cast<T *>(DominatingLLVMValue::restore(CGF, value));
159 }
160};
161
162/// A specialization of DominatingValue for Address.
163template <> struct DominatingValue<Address> {
164 typedef Address type;
165
166 struct saved_type {
167 DominatingLLVMValue::saved_type BasePtr;
168 llvm::Type *ElementType;
169 CharUnits Alignment;
170 DominatingLLVMValue::saved_type Offset;
171 llvm::PointerType *EffectiveType;
172 };
173
174 static bool needsSaving(type value) {
175 if (DominatingLLVMValue::needsSaving(value: value.getBasePointer()) ||
176 DominatingLLVMValue::needsSaving(value: value.getOffset()))
177 return true;
178 return false;
179 }
180 static saved_type save(CodeGenFunction &CGF, type value) {
181 return {.BasePtr: DominatingLLVMValue::save(CGF, value: value.getBasePointer()),
182 .ElementType: value.getElementType(), .Alignment: value.getAlignment(),
183 .Offset: DominatingLLVMValue::save(CGF, value: value.getOffset()), .EffectiveType: value.getType()};
184 }
185 static type restore(CodeGenFunction &CGF, saved_type value) {
186 return Address(DominatingLLVMValue::restore(CGF, value: value.BasePtr),
187 value.ElementType, value.Alignment, CGPointerAuthInfo(),
188 DominatingLLVMValue::restore(CGF, value: value.Offset));
189 }
190};
191
192/// A specialization of DominatingValue for RValue.
193template <> struct DominatingValue<RValue> {
194 typedef RValue type;
195 class saved_type {
196 enum Kind {
197 ScalarLiteral,
198 ScalarAddress,
199 AggregateLiteral,
200 AggregateAddress,
201 ComplexAddress
202 };
203 union {
204 struct {
205 DominatingLLVMValue::saved_type first, second;
206 } Vals;
207 DominatingValue<Address>::saved_type AggregateAddr;
208 };
209 LLVM_PREFERRED_TYPE(Kind)
210 unsigned K : 3;
211
212 saved_type(DominatingLLVMValue::saved_type Val1, unsigned K)
213 : Vals{.first: Val1, .second: DominatingLLVMValue::saved_type()}, K(K) {}
214
215 saved_type(DominatingLLVMValue::saved_type Val1,
216 DominatingLLVMValue::saved_type Val2)
217 : Vals{.first: Val1, .second: Val2}, K(ComplexAddress) {}
218
219 saved_type(DominatingValue<Address>::saved_type AggregateAddr, unsigned K)
220 : AggregateAddr(AggregateAddr), K(K) {}
221
222 public:
223 static bool needsSaving(RValue value);
224 static saved_type save(CodeGenFunction &CGF, RValue value);
225 RValue restore(CodeGenFunction &CGF);
226
227 // implementations in CGCleanup.cpp
228 };
229
230 static bool needsSaving(type value) { return saved_type::needsSaving(value); }
231 static saved_type save(CodeGenFunction &CGF, type value) {
232 return saved_type::save(CGF, value);
233 }
234 static type restore(CodeGenFunction &CGF, saved_type value) {
235 return value.restore(CGF);
236 }
237};
238
239/// A scoped helper to set the current source atom group for
240/// CGDebugInfo::addInstToCurrentSourceAtom. A source atom is a source construct
241/// that is "interesting" for debug stepping purposes. We use an atom group
242/// number to track the instruction(s) that implement the functionality for the
243/// atom, plus backup instructions/source locations.
244class ApplyAtomGroup {
245 uint64_t OriginalAtom = 0;
246 CGDebugInfo *DI = nullptr;
247
248 ApplyAtomGroup(const ApplyAtomGroup &) = delete;
249 void operator=(const ApplyAtomGroup &) = delete;
250
251public:
252 ApplyAtomGroup(CGDebugInfo *DI);
253 ~ApplyAtomGroup();
254};
255
256/// CodeGenFunction - This class organizes the per-function state that is used
257/// while generating LLVM code.
258class CodeGenFunction : public CodeGenTypeCache {
259 CodeGenFunction(const CodeGenFunction &) = delete;
260 void operator=(const CodeGenFunction &) = delete;
261
262 friend class CGCXXABI;
263 friend class clang::AsmConstraintsInfo;
264
265public:
266 /// A jump destination is an abstract label, branching to which may
267 /// require a jump out through normal cleanups.
268 struct JumpDest {
269 JumpDest() : Block(nullptr), Index(0) {}
270 JumpDest(llvm::BasicBlock *Block, EHScopeStack::stable_iterator Depth,
271 unsigned Index)
272 : Block(Block), ScopeDepth(Depth), Index(Index) {}
273
274 bool isValid() const { return Block != nullptr; }
275 llvm::BasicBlock *getBlock() const { return Block; }
276 EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; }
277 unsigned getDestIndex() const { return Index; }
278
279 // This should be used cautiously.
280 void setScopeDepth(EHScopeStack::stable_iterator depth) {
281 ScopeDepth = depth;
282 }
283
284 private:
285 llvm::BasicBlock *Block;
286 EHScopeStack::stable_iterator ScopeDepth;
287 unsigned Index;
288 };
289
290 CodeGenModule &CGM; // Per-module state.
291 const TargetInfo &Target;
292
293 // For EH/SEH outlined funclets, this field points to parent's CGF
294 CodeGenFunction *ParentCGF = nullptr;
295
296 typedef std::pair<llvm::Value *, llvm::Value *> ComplexPairTy;
297 LoopInfoStack LoopStack;
298 CGBuilderTy Builder;
299
300 // Stores variables for which we can't generate correct lifetime markers
301 // because of jumps.
302 VarBypassDetector Bypasses;
303
304 /// List of recently emitted OMPCanonicalLoops.
305 ///
306 /// Since OMPCanonicalLoops are nested inside other statements (in particular
307 /// CapturedStmt generated by OMPExecutableDirective and non-perfectly nested
308 /// loops), we cannot directly call OMPEmitOMPCanonicalLoop and receive its
309 /// llvm::CanonicalLoopInfo. Instead, we call EmitStmt and any
310 /// OMPEmitOMPCanonicalLoop called by it will add its CanonicalLoopInfo to
311 /// this stack when done. Entering a new loop requires clearing this list; it
312 /// either means we start parsing a new loop nest (in which case the previous
313 /// loop nest goes out of scope) or a second loop in the same level in which
314 /// case it would be ambiguous into which of the two (or more) loops the loop
315 /// nest would extend.
316 SmallVector<llvm::CanonicalLoopInfo *, 4> OMPLoopNestStack;
317
318 /// Stack to track the controlled convergence tokens.
319 SmallVector<llvm::ConvergenceControlInst *, 4> ConvergenceTokenStack;
320
321 /// Number of nested loop to be consumed by the last surrounding
322 /// loop-associated directive.
323 int ExpectedOMPLoopDepth = 0;
324
325 // CodeGen lambda for loops and support for ordered clause
326 typedef llvm::function_ref<void(CodeGenFunction &, const OMPLoopDirective &,
327 JumpDest)>
328 CodeGenLoopTy;
329 typedef llvm::function_ref<void(CodeGenFunction &, SourceLocation,
330 const unsigned, const bool)>
331 CodeGenOrderedTy;
332
333 // Codegen lambda for loop bounds in worksharing loop constructs
334 typedef llvm::function_ref<std::pair<LValue, LValue>(
335 CodeGenFunction &, const OMPExecutableDirective &S)>
336 CodeGenLoopBoundsTy;
337
338 // Codegen lambda for loop bounds in dispatch-based loop implementation
339 typedef llvm::function_ref<std::pair<llvm::Value *, llvm::Value *>(
340 CodeGenFunction &, const OMPExecutableDirective &S, Address LB,
341 Address UB)>
342 CodeGenDispatchBoundsTy;
343
344 /// CGBuilder insert helper. This function is called after an
345 /// instruction is created using Builder.
346 void InsertHelper(llvm::Instruction *I, const llvm::Twine &Name,
347 llvm::BasicBlock::iterator InsertPt) const;
348
349 /// CurFuncDecl - Holds the Decl for the current outermost
350 /// non-closure context.
351 const Decl *CurFuncDecl = nullptr;
352 /// CurCodeDecl - This is the inner-most code context, which includes blocks.
353 const Decl *CurCodeDecl = nullptr;
354 const CGFunctionInfo *CurFnInfo = nullptr;
355 QualType FnRetTy;
356 llvm::Function *CurFn = nullptr;
357
358 /// If a cast expression is being visited, this holds the current cast's
359 /// expression.
360 const CastExpr *CurCast = nullptr;
361
362 /// Save Parameter Decl for coroutine.
363 llvm::SmallVector<const ParmVarDecl *, 4> FnArgs;
364
365 // Holds coroutine data if the current function is a coroutine. We use a
366 // wrapper to manage its lifetime, so that we don't have to define CGCoroData
367 // in this header.
368 struct CGCoroInfo {
369 std::unique_ptr<CGCoroData> Data;
370 bool InSuspendBlock = false;
371 CGCoroInfo();
372 ~CGCoroInfo();
373 };
374 CGCoroInfo CurCoro;
375
376 bool isCoroutine() const { return CurCoro.Data != nullptr; }
377
378 bool inSuspendBlock() const {
379 return isCoroutine() && CurCoro.InSuspendBlock;
380 }
381
382 // Holds FramePtr for await_suspend wrapper generation,
383 // so that __builtin_coro_frame call can be lowered
384 // directly to value of its second argument
385 struct AwaitSuspendWrapperInfo {
386 llvm::Value *FramePtr = nullptr;
387 };
388 AwaitSuspendWrapperInfo CurAwaitSuspendWrapper;
389
390 // Generates wrapper function for `llvm.coro.await.suspend.*` intrinisics.
391 // It encapsulates SuspendExpr in a function, to separate it's body
392 // from the main coroutine to avoid miscompilations. Intrinisic
393 // is lowered to this function call in CoroSplit pass
394 // Function signature is:
395 // <type> __await_suspend_wrapper_<name>(ptr %awaiter, ptr %hdl)
396 // where type is one of (void, i1, ptr)
397 llvm::Function *generateAwaitSuspendWrapper(Twine const &CoroName,
398 Twine const &SuspendPointName,
399 CoroutineSuspendExpr const &S);
400
401 /// CurGD - The GlobalDecl for the current function being compiled.
402 GlobalDecl CurGD;
403
404 /// PrologueCleanupDepth - The cleanup depth enclosing all the
405 /// cleanups associated with the parameters.
406 EHScopeStack::stable_iterator PrologueCleanupDepth;
407
408 /// ReturnBlock - Unified return block.
409 JumpDest ReturnBlock;
410
411 /// ReturnValue - The temporary alloca to hold the return
412 /// value. This is invalid iff the function has no return value.
413 Address ReturnValue = Address::invalid();
414
415 /// ReturnValuePointer - The temporary alloca to hold a pointer to sret.
416 /// This is invalid if sret is not in use.
417 Address ReturnValuePointer = Address::invalid();
418
419 /// If a return statement is being visited, this holds the return statment's
420 /// result expression.
421 const Expr *RetExpr = nullptr;
422
423 /// Return true if a label was seen in the current scope.
424 bool hasLabelBeenSeenInCurrentScope() const {
425 if (CurLexicalScope)
426 return CurLexicalScope->hasLabels();
427 return !LabelMap.empty();
428 }
429
430 /// AllocaInsertPoint - This is an instruction in the entry block before which
431 /// we prefer to insert allocas.
432 llvm::AssertingVH<llvm::Instruction> AllocaInsertPt;
433
434private:
435 /// PostAllocaInsertPt - This is a place in the prologue where code can be
436 /// inserted that will be dominated by all the static allocas. This helps
437 /// achieve two things:
438 /// 1. Contiguity of all static allocas (within the prologue) is maintained.
439 /// 2. All other prologue code (which are dominated by static allocas) do
440 /// appear in the source order immediately after all static allocas.
441 ///
442 /// PostAllocaInsertPt will be lazily created when it is *really* required.
443 llvm::AssertingVH<llvm::Instruction> PostAllocaInsertPt = nullptr;
444
445public:
446 /// Return PostAllocaInsertPt. If it is not yet created, then insert it
447 /// immediately after AllocaInsertPt.
448 llvm::Instruction *getPostAllocaInsertPoint() {
449 if (!PostAllocaInsertPt) {
450 assert(AllocaInsertPt &&
451 "Expected static alloca insertion point at function prologue");
452 assert(AllocaInsertPt->getParent()->isEntryBlock() &&
453 "EBB should be entry block of the current code gen function");
454 PostAllocaInsertPt = AllocaInsertPt->clone();
455 PostAllocaInsertPt->setName("postallocapt");
456 PostAllocaInsertPt->insertAfter(InsertPos: AllocaInsertPt->getIterator());
457 }
458
459 return PostAllocaInsertPt;
460 }
461
462 // Try to preserve the source's name to make IR more readable.
463 llvm::Value *performAddrSpaceCast(llvm::Value *Src, llvm::Type *DestTy) {
464 return Builder.CreateAddrSpaceCast(
465 V: Src, DestTy, Name: Src->hasName() ? Src->getName() + ".ascast" : "");
466 }
467
468 /// API for captured statement code generation.
469 class CGCapturedStmtInfo {
470 public:
471 explicit CGCapturedStmtInfo(CapturedRegionKind K = CR_Default)
472 : Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {}
473 explicit CGCapturedStmtInfo(const CapturedStmt &S,
474 CapturedRegionKind K = CR_Default)
475 : Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {
476
477 RecordDecl::field_iterator Field =
478 S.getCapturedRecordDecl()->field_begin();
479 for (CapturedStmt::const_capture_iterator I = S.capture_begin(),
480 E = S.capture_end();
481 I != E; ++I, ++Field) {
482 if (I->capturesThis())
483 CXXThisFieldDecl = *Field;
484 else if (I->capturesVariable())
485 CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
486 else if (I->capturesVariableByCopy())
487 CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
488 }
489 }
490
491 virtual ~CGCapturedStmtInfo();
492
493 CapturedRegionKind getKind() const { return Kind; }
494
495 virtual void setContextValue(llvm::Value *V) { ThisValue = V; }
496 // Retrieve the value of the context parameter.
497 virtual llvm::Value *getContextValue() const { return ThisValue; }
498
499 /// Lookup the captured field decl for a variable.
500 virtual const FieldDecl *lookup(const VarDecl *VD) const {
501 return CaptureFields.lookup(Val: VD->getCanonicalDecl());
502 }
503
504 bool isCXXThisExprCaptured() const { return getThisFieldDecl() != nullptr; }
505 virtual FieldDecl *getThisFieldDecl() const { return CXXThisFieldDecl; }
506
507 static bool classof(const CGCapturedStmtInfo *) { return true; }
508
509 /// Emit the captured statement body.
510 virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) {
511 CGF.incrementProfileCounter(S);
512 CGF.EmitStmt(S);
513 }
514
515 /// Get the name of the capture helper.
516 virtual StringRef getHelperName() const { return "__captured_stmt"; }
517
518 /// Get the CaptureFields
519 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> getCaptureFields() {
520 return CaptureFields;
521 }
522
523 private:
524 /// The kind of captured statement being generated.
525 CapturedRegionKind Kind;
526
527 /// Keep the map between VarDecl and FieldDecl.
528 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields;
529
530 /// The base address of the captured record, passed in as the first
531 /// argument of the parallel region function.
532 llvm::Value *ThisValue;
533
534 /// Captured 'this' type.
535 FieldDecl *CXXThisFieldDecl;
536 };
537 CGCapturedStmtInfo *CapturedStmtInfo = nullptr;
538
539 /// RAII for correct setting/restoring of CapturedStmtInfo.
540 class CGCapturedStmtRAII {
541 private:
542 CodeGenFunction &CGF;
543 CGCapturedStmtInfo *PrevCapturedStmtInfo;
544
545 public:
546 CGCapturedStmtRAII(CodeGenFunction &CGF,
547 CGCapturedStmtInfo *NewCapturedStmtInfo)
548 : CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo) {
549 CGF.CapturedStmtInfo = NewCapturedStmtInfo;
550 }
551 ~CGCapturedStmtRAII() { CGF.CapturedStmtInfo = PrevCapturedStmtInfo; }
552 };
553
554 /// An abstract representation of regular/ObjC call/message targets.
555 class AbstractCallee {
556 /// The function declaration of the callee.
557 const Decl *CalleeDecl;
558
559 public:
560 AbstractCallee() : CalleeDecl(nullptr) {}
561 AbstractCallee(const FunctionDecl *FD) : CalleeDecl(FD) {}
562 AbstractCallee(const ObjCMethodDecl *OMD) : CalleeDecl(OMD) {}
563 bool hasFunctionDecl() const {
564 return isa_and_nonnull<FunctionDecl>(Val: CalleeDecl);
565 }
566 const Decl *getDecl() const { return CalleeDecl; }
567 unsigned getNumParams() const {
568 if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
569 return FD->getNumParams();
570 return cast<ObjCMethodDecl>(Val: CalleeDecl)->param_size();
571 }
572 const ParmVarDecl *getParamDecl(unsigned I) const {
573 if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
574 return FD->getParamDecl(i: I);
575 return *(cast<ObjCMethodDecl>(Val: CalleeDecl)->param_begin() + I);
576 }
577 };
578
579 /// Sanitizers enabled for this function.
580 SanitizerSet SanOpts;
581
582 /// True if CodeGen currently emits code implementing sanitizer checks.
583 bool IsSanitizerScope = false;
584
585 /// RAII object to set/unset CodeGenFunction::IsSanitizerScope.
586 class SanitizerScope {
587 CodeGenFunction *CGF;
588
589 public:
590 SanitizerScope(CodeGenFunction *CGF);
591 ~SanitizerScope();
592 };
593
594 /// In C++, whether we are code generating a thunk. This controls whether we
595 /// should emit cleanups.
596 bool CurFuncIsThunk = false;
597
598 /// In ARC, whether we should autorelease the return value.
599 bool AutoreleaseResult = false;
600
601 /// Whether we processed a Microsoft-style asm block during CodeGen. These can
602 /// potentially set the return value.
603 bool SawAsmBlock = false;
604
605 GlobalDecl CurSEHParent;
606
607 /// True if the current function is an outlined SEH helper. This can be a
608 /// finally block or filter expression.
609 bool IsOutlinedSEHHelper = false;
610
611 /// True if CodeGen currently emits code inside presereved access index
612 /// region.
613 bool IsInPreservedAIRegion = false;
614
615 /// True if the current statement has nomerge attribute.
616 bool InNoMergeAttributedStmt = false;
617
618 /// True if the current statement has noinline attribute.
619 bool InNoInlineAttributedStmt = false;
620
621 /// True if the current statement has always_inline attribute.
622 bool InAlwaysInlineAttributedStmt = false;
623
624 /// True if the current statement has noconvergent attribute.
625 bool InNoConvergentAttributedStmt = false;
626
627 /// HLSL Branch attribute.
628 HLSLControlFlowHintAttr::Spelling HLSLControlFlowAttr =
629 HLSLControlFlowHintAttr::SpellingNotCalculated;
630
631 // The CallExpr within the current statement that the musttail attribute
632 // applies to. nullptr if there is no 'musttail' on the current statement.
633 const CallExpr *MustTailCall = nullptr;
634
635 /// Returns true if a function must make progress, which means the
636 /// mustprogress attribute can be added.
637 bool checkIfFunctionMustProgress() {
638 if (CGM.getCodeGenOpts().getFiniteLoops() ==
639 CodeGenOptions::FiniteLoopsKind::Never)
640 return false;
641
642 // C++11 and later guarantees that a thread eventually will do one of the
643 // following (C++11 [intro.multithread]p24 and C++17 [intro.progress]p1):
644 // - terminate,
645 // - make a call to a library I/O function,
646 // - perform an access through a volatile glvalue, or
647 // - perform a synchronization operation or an atomic operation.
648 //
649 // Hence each function is 'mustprogress' in C++11 or later.
650 return getLangOpts().CPlusPlus11;
651 }
652
653 /// Returns true if a loop must make progress, which means the mustprogress
654 /// attribute can be added. \p HasConstantCond indicates whether the branch
655 /// condition is a known constant.
656 bool checkIfLoopMustProgress(const Expr *, bool HasEmptyBody);
657
658 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
659 llvm::Value *BlockPointer = nullptr;
660
661 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
662 FieldDecl *LambdaThisCaptureField = nullptr;
663
664 /// A mapping from NRVO variables to the flags used to indicate
665 /// when the NRVO has been applied to this variable.
666 llvm::DenseMap<const VarDecl *, llvm::Value *> NRVOFlags;
667
668 EHScopeStack EHStack;
669 llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack;
670
671 // A stack of cleanups which were added to EHStack but have to be deactivated
672 // later before being popped or emitted. These are usually deactivated on
673 // exiting a `CleanupDeactivationScope` scope. For instance, after a
674 // full-expr.
675 //
676 // These are specially useful for correctly emitting cleanups while
677 // encountering branches out of expression (through stmt-expr or coroutine
678 // suspensions).
679 struct DeferredDeactivateCleanup {
680 EHScopeStack::stable_iterator Cleanup;
681 llvm::Instruction *DominatingIP;
682 };
683 llvm::SmallVector<DeferredDeactivateCleanup> DeferredDeactivationCleanupStack;
684
685 // Enters a new scope for capturing cleanups which are deferred to be
686 // deactivated, all of which will be deactivated once the scope is exited.
687 struct CleanupDeactivationScope {
688 CodeGenFunction &CGF;
689 size_t OldDeactivateCleanupStackSize;
690 bool Deactivated;
691 CleanupDeactivationScope(CodeGenFunction &CGF)
692 : CGF(CGF), OldDeactivateCleanupStackSize(
693 CGF.DeferredDeactivationCleanupStack.size()),
694 Deactivated(false) {}
695
696 void ForceDeactivate() {
697 assert(!Deactivated && "Deactivating already deactivated scope");
698 auto &Stack = CGF.DeferredDeactivationCleanupStack;
699 for (size_t I = Stack.size(); I > OldDeactivateCleanupStackSize; I--) {
700 CGF.DeactivateCleanupBlock(Cleanup: Stack[I - 1].Cleanup,
701 DominatingIP: Stack[I - 1].DominatingIP);
702 Stack[I - 1].DominatingIP->eraseFromParent();
703 }
704 Stack.resize(N: OldDeactivateCleanupStackSize);
705 Deactivated = true;
706 }
707
708 ~CleanupDeactivationScope() {
709 if (Deactivated)
710 return;
711 ForceDeactivate();
712 }
713 };
714
715 llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack;
716
717 llvm::Instruction *CurrentFuncletPad = nullptr;
718
719 class CallLifetimeEnd final : public EHScopeStack::Cleanup {
720 bool isRedundantBeforeReturn() override { return true; }
721
722 llvm::Value *Addr;
723
724 public:
725 CallLifetimeEnd(RawAddress addr) : Addr(addr.getPointer()) {}
726
727 void Emit(CodeGenFunction &CGF, Flags flags) override {
728 CGF.EmitLifetimeEnd(Addr);
729 }
730 };
731
732 // We are using objects of this 'cleanup' class to emit fake.use calls
733 // for -fextend-variable-liveness. They are placed at the end of a variable's
734 // scope analogous to lifetime markers.
735 class FakeUse final : public EHScopeStack::Cleanup {
736 Address Addr;
737
738 public:
739 FakeUse(Address addr) : Addr(addr) {}
740
741 void Emit(CodeGenFunction &CGF, Flags flags) override {
742 CGF.EmitFakeUse(Addr);
743 }
744 };
745
746 /// Header for data within LifetimeExtendedCleanupStack.
747 struct alignas(uint64_t) LifetimeExtendedCleanupHeader {
748 /// The size of the following cleanup object.
749 unsigned Size;
750 /// The kind of cleanup to push.
751 LLVM_PREFERRED_TYPE(CleanupKind)
752 unsigned Kind : 31;
753 /// Whether this is a conditional cleanup.
754 LLVM_PREFERRED_TYPE(bool)
755 unsigned IsConditional : 1;
756
757 size_t getSize() const { return Size; }
758 CleanupKind getKind() const { return (CleanupKind)Kind; }
759 bool isConditional() const { return IsConditional; }
760 };
761
762 /// i32s containing the indexes of the cleanup destinations.
763 RawAddress NormalCleanupDest = RawAddress::invalid();
764
765 unsigned NextCleanupDestIndex = 1;
766
767 /// EHResumeBlock - Unified block containing a call to llvm.eh.resume.
768 llvm::BasicBlock *EHResumeBlock = nullptr;
769
770 /// The exception slot. All landing pads write the current exception pointer
771 /// into this alloca.
772 llvm::Value *ExceptionSlot = nullptr;
773
774 /// The selector slot. Under the MandatoryCleanup model, all landing pads
775 /// write the current selector value into this alloca.
776 llvm::AllocaInst *EHSelectorSlot = nullptr;
777
778 /// A stack of exception code slots. Entering an __except block pushes a slot
779 /// on the stack and leaving pops one. The __exception_code() intrinsic loads
780 /// a value from the top of the stack.
781 SmallVector<Address, 1> SEHCodeSlotStack;
782
783 /// Value returned by __exception_info intrinsic.
784 llvm::Value *SEHInfo = nullptr;
785
786 /// Emits a landing pad for the current EH stack.
787 llvm::BasicBlock *EmitLandingPad();
788
789 llvm::BasicBlock *getInvokeDestImpl();
790
791 /// Parent loop-based directive for scan directive.
792 const OMPExecutableDirective *OMPParentLoopDirectiveForScan = nullptr;
793 llvm::BasicBlock *OMPBeforeScanBlock = nullptr;
794 llvm::BasicBlock *OMPAfterScanBlock = nullptr;
795 llvm::BasicBlock *OMPScanExitBlock = nullptr;
796 llvm::BasicBlock *OMPScanDispatch = nullptr;
797 bool OMPFirstScanLoop = false;
798
799 /// Manages parent directive for scan directives.
800 class ParentLoopDirectiveForScanRegion {
801 CodeGenFunction &CGF;
802 const OMPExecutableDirective *ParentLoopDirectiveForScan;
803
804 public:
805 ParentLoopDirectiveForScanRegion(
806 CodeGenFunction &CGF,
807 const OMPExecutableDirective &ParentLoopDirectiveForScan)
808 : CGF(CGF),
809 ParentLoopDirectiveForScan(CGF.OMPParentLoopDirectiveForScan) {
810 CGF.OMPParentLoopDirectiveForScan = &ParentLoopDirectiveForScan;
811 }
812 ~ParentLoopDirectiveForScanRegion() {
813 CGF.OMPParentLoopDirectiveForScan = ParentLoopDirectiveForScan;
814 }
815 };
816
817 template <class T>
818 typename DominatingValue<T>::saved_type saveValueInCond(T value) {
819 return DominatingValue<T>::save(*this, value);
820 }
821
822 class CGFPOptionsRAII {
823 public:
824 CGFPOptionsRAII(CodeGenFunction &CGF, FPOptions FPFeatures);
825 CGFPOptionsRAII(CodeGenFunction &CGF, const Expr *E);
826 ~CGFPOptionsRAII();
827
828 private:
829 void ConstructorHelper(FPOptions FPFeatures);
830 CodeGenFunction &CGF;
831 FPOptions OldFPFeatures;
832 llvm::fp::ExceptionBehavior OldExcept;
833 llvm::RoundingMode OldRounding;
834 std::optional<CGBuilderTy::FastMathFlagGuard> FMFGuard;
835 };
836 FPOptions CurFPFeatures;
837
838 class CGAtomicOptionsRAII {
839 public:
840 CGAtomicOptionsRAII(CodeGenModule &CGM_, AtomicOptions AO)
841 : CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {
842 CGM.setAtomicOpts(AO);
843 }
844 CGAtomicOptionsRAII(CodeGenModule &CGM_, const AtomicAttr *AA)
845 : CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {
846 if (!AA)
847 return;
848 AtomicOptions AO = SavedAtomicOpts;
849 for (auto Option : AA->atomicOptions()) {
850 switch (Option) {
851 case AtomicAttr::remote_memory:
852 AO.remote_memory = true;
853 break;
854 case AtomicAttr::no_remote_memory:
855 AO.remote_memory = false;
856 break;
857 case AtomicAttr::fine_grained_memory:
858 AO.fine_grained_memory = true;
859 break;
860 case AtomicAttr::no_fine_grained_memory:
861 AO.fine_grained_memory = false;
862 break;
863 case AtomicAttr::ignore_denormal_mode:
864 AO.ignore_denormal_mode = true;
865 break;
866 case AtomicAttr::no_ignore_denormal_mode:
867 AO.ignore_denormal_mode = false;
868 break;
869 }
870 }
871 CGM.setAtomicOpts(AO);
872 }
873
874 CGAtomicOptionsRAII(const CGAtomicOptionsRAII &) = delete;
875 CGAtomicOptionsRAII &operator=(const CGAtomicOptionsRAII &) = delete;
876 ~CGAtomicOptionsRAII() { CGM.setAtomicOpts(SavedAtomicOpts); }
877
878 private:
879 CodeGenModule &CGM;
880 AtomicOptions SavedAtomicOpts;
881 };
882
883public:
884 /// ObjCEHValueStack - Stack of Objective-C exception values, used for
885 /// rethrows.
886 SmallVector<llvm::Value *, 8> ObjCEHValueStack;
887
888 /// A class controlling the emission of a finally block.
889 class FinallyInfo {
890 /// Where the catchall's edge through the cleanup should go.
891 JumpDest RethrowDest;
892
893 /// A function to call to enter the catch.
894 llvm::FunctionCallee BeginCatchFn;
895
896 /// An i1 variable indicating whether or not the @finally is
897 /// running for an exception.
898 llvm::AllocaInst *ForEHVar = nullptr;
899
900 /// An i8* variable into which the exception pointer to rethrow
901 /// has been saved.
902 llvm::AllocaInst *SavedExnVar = nullptr;
903
904 public:
905 void enter(CodeGenFunction &CGF, const Stmt *Finally,
906 llvm::FunctionCallee beginCatchFn,
907 llvm::FunctionCallee endCatchFn, llvm::FunctionCallee rethrowFn);
908 void exit(CodeGenFunction &CGF);
909 };
910
911 /// Returns true inside SEH __try blocks.
912 bool isSEHTryScope() const { return !SEHTryEpilogueStack.empty(); }
913
914 /// Returns true while emitting a cleanuppad.
915 bool isCleanupPadScope() const {
916 return CurrentFuncletPad && isa<llvm::CleanupPadInst>(Val: CurrentFuncletPad);
917 }
918
919 /// pushFullExprCleanup - Push a cleanup to be run at the end of the
920 /// current full-expression. Safe against the possibility that
921 /// we're currently inside a conditionally-evaluated expression.
922 template <class T, class... As>
923 void pushFullExprCleanup(CleanupKind kind, As... A) {
924 // If we're not in a conditional branch, or if none of the
925 // arguments requires saving, then use the unconditional cleanup.
926 if (!isInConditionalBranch())
927 return EHStack.pushCleanup<T>(kind, A...);
928
929 // Stash values in a tuple so we can guarantee the order of saves.
930 typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
931 SavedTuple Saved{saveValueInCond(A)...};
932
933 typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
934 EHStack.pushCleanupTuple<CleanupType>(kind, Saved);
935 initFullExprCleanup();
936 }
937
938 /// Queue a cleanup to be pushed after finishing the current full-expression,
939 /// potentially with an active flag.
940 template <class T, class... As>
941 void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) {
942 if (!isInConditionalBranch())
943 return pushCleanupAfterFullExprWithActiveFlag<T>(
944 Kind, RawAddress::invalid(), A...);
945
946 RawAddress ActiveFlag = createCleanupActiveFlag();
947 assert(!DominatingValue<Address>::needsSaving(ActiveFlag) &&
948 "cleanup active flag should never need saving");
949
950 typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
951 SavedTuple Saved{saveValueInCond(A)...};
952
953 typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
954 pushCleanupAfterFullExprWithActiveFlag<CleanupType>(Kind, ActiveFlag,
955 Saved);
956 }
957
958 template <class T, class... As>
959 void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind,
960 RawAddress ActiveFlag, As... A) {
961 LifetimeExtendedCleanupHeader Header = {.Size: sizeof(T), .Kind: Kind,
962 .IsConditional: ActiveFlag.isValid()};
963
964 size_t OldSize = LifetimeExtendedCleanupStack.size();
965 LifetimeExtendedCleanupStack.resize(
966 N: LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size +
967 (Header.IsConditional ? sizeof(ActiveFlag) : 0));
968
969 static_assert((alignof(LifetimeExtendedCleanupHeader) == alignof(T)) &&
970 (alignof(T) == alignof(RawAddress)),
971 "Cleanup will be allocated on misaligned address");
972 char *Buffer = &LifetimeExtendedCleanupStack[OldSize];
973 new (Buffer) LifetimeExtendedCleanupHeader(Header);
974 new (Buffer + sizeof(Header)) T(A...);
975 if (Header.IsConditional)
976 new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag);
977 }
978
979 // Push a cleanup onto EHStack and deactivate it later. It is usually
980 // deactivated when exiting a `CleanupDeactivationScope` (for example: after a
981 // full expression).
982 template <class T, class... As>
983 void pushCleanupAndDeferDeactivation(CleanupKind Kind, As... A) {
984 // Placeholder dominating IP for this cleanup.
985 llvm::Instruction *DominatingIP =
986 Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
987 EHStack.pushCleanup<T>(Kind, A...);
988 DeferredDeactivationCleanupStack.push_back(
989 Elt: {.Cleanup: EHStack.stable_begin(), .DominatingIP: DominatingIP});
990 }
991
992 /// Set up the last cleanup that was pushed as a conditional
993 /// full-expression cleanup.
994 void initFullExprCleanup() {
995 initFullExprCleanupWithFlag(ActiveFlag: createCleanupActiveFlag());
996 }
997
998 void initFullExprCleanupWithFlag(RawAddress ActiveFlag);
999 RawAddress createCleanupActiveFlag();
1000
1001 /// PushDestructorCleanup - Push a cleanup to call the
1002 /// complete-object destructor of an object of the given type at the
1003 /// given address. Does nothing if T is not a C++ class type with a
1004 /// non-trivial destructor.
1005 void PushDestructorCleanup(QualType T, Address Addr);
1006
1007 /// PushDestructorCleanup - Push a cleanup to call the
1008 /// complete-object variant of the given destructor on the object at
1009 /// the given address.
1010 void PushDestructorCleanup(const CXXDestructorDecl *Dtor, QualType T,
1011 Address Addr);
1012
1013 /// PopCleanupBlock - Will pop the cleanup entry on the stack and
1014 /// process all branch fixups.
1015 void PopCleanupBlock(bool FallThroughIsBranchThrough = false,
1016 bool ForDeactivation = false);
1017
1018 /// DeactivateCleanupBlock - Deactivates the given cleanup block.
1019 /// The block cannot be reactivated. Pops it if it's the top of the
1020 /// stack.
1021 ///
1022 /// \param DominatingIP - An instruction which is known to
1023 /// dominate the current IP (if set) and which lies along
1024 /// all paths of execution between the current IP and the
1025 /// the point at which the cleanup comes into scope.
1026 void DeactivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
1027 llvm::Instruction *DominatingIP);
1028
1029 /// ActivateCleanupBlock - Activates an initially-inactive cleanup.
1030 /// Cannot be used to resurrect a deactivated cleanup.
1031 ///
1032 /// \param DominatingIP - An instruction which is known to
1033 /// dominate the current IP (if set) and which lies along
1034 /// all paths of execution between the current IP and the
1035 /// the point at which the cleanup comes into scope.
1036 void ActivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
1037 llvm::Instruction *DominatingIP);
1038
1039 /// Enters a new scope for capturing cleanups, all of which
1040 /// will be executed once the scope is exited.
1041 class RunCleanupsScope {
1042 EHScopeStack::stable_iterator CleanupStackDepth, OldCleanupScopeDepth;
1043 size_t LifetimeExtendedCleanupStackSize;
1044 CleanupDeactivationScope DeactivateCleanups;
1045 bool OldDidCallStackSave;
1046
1047 protected:
1048 bool PerformCleanup;
1049
1050 private:
1051 RunCleanupsScope(const RunCleanupsScope &) = delete;
1052 void operator=(const RunCleanupsScope &) = delete;
1053
1054 protected:
1055 CodeGenFunction &CGF;
1056
1057 public:
1058 /// Enter a new cleanup scope.
1059 explicit RunCleanupsScope(CodeGenFunction &CGF)
1060 : DeactivateCleanups(CGF), PerformCleanup(true), CGF(CGF) {
1061 CleanupStackDepth = CGF.EHStack.stable_begin();
1062 LifetimeExtendedCleanupStackSize =
1063 CGF.LifetimeExtendedCleanupStack.size();
1064 OldDidCallStackSave = CGF.DidCallStackSave;
1065 CGF.DidCallStackSave = false;
1066 OldCleanupScopeDepth = CGF.CurrentCleanupScopeDepth;
1067 CGF.CurrentCleanupScopeDepth = CleanupStackDepth;
1068 }
1069
1070 /// Exit this cleanup scope, emitting any accumulated cleanups.
1071 ~RunCleanupsScope() {
1072 if (PerformCleanup)
1073 ForceCleanup();
1074 }
1075
1076 /// Determine whether this scope requires any cleanups.
1077 bool requiresCleanups() const {
1078 return CGF.EHStack.stable_begin() != CleanupStackDepth;
1079 }
1080
1081 /// Force the emission of cleanups now, instead of waiting
1082 /// until this object is destroyed.
1083 /// \param ValuesToReload - A list of values that need to be available at
1084 /// the insertion point after cleanup emission. If cleanup emission created
1085 /// a shared cleanup block, these value pointers will be rewritten.
1086 /// Otherwise, they not will be modified.
1087 void
1088 ForceCleanup(std::initializer_list<llvm::Value **> ValuesToReload = {}) {
1089 assert(PerformCleanup && "Already forced cleanup");
1090 CGF.DidCallStackSave = OldDidCallStackSave;
1091 DeactivateCleanups.ForceDeactivate();
1092 CGF.PopCleanupBlocks(OldCleanupStackSize: CleanupStackDepth, OldLifetimeExtendedStackSize: LifetimeExtendedCleanupStackSize,
1093 ValuesToReload);
1094 PerformCleanup = false;
1095 CGF.CurrentCleanupScopeDepth = OldCleanupScopeDepth;
1096 }
1097 };
1098
1099 // Cleanup stack depth of the RunCleanupsScope that was pushed most recently.
1100 EHScopeStack::stable_iterator CurrentCleanupScopeDepth =
1101 EHScopeStack::stable_end();
1102
1103 class LexicalScope : public RunCleanupsScope {
1104 SourceRange Range;
1105 SmallVector<const LabelDecl *, 4> Labels;
1106 LexicalScope *ParentScope;
1107
1108 LexicalScope(const LexicalScope &) = delete;
1109 void operator=(const LexicalScope &) = delete;
1110
1111 public:
1112 /// Enter a new cleanup scope.
1113 explicit LexicalScope(CodeGenFunction &CGF, SourceRange Range);
1114
1115 void addLabel(const LabelDecl *label) {
1116 assert(PerformCleanup && "adding label to dead scope?");
1117 Labels.push_back(Elt: label);
1118 }
1119
1120 /// Exit this cleanup scope, emitting any accumulated
1121 /// cleanups.
1122 ~LexicalScope();
1123
1124 /// Force the emission of cleanups now, instead of waiting
1125 /// until this object is destroyed.
1126 void ForceCleanup() {
1127 CGF.CurLexicalScope = ParentScope;
1128 RunCleanupsScope::ForceCleanup();
1129
1130 if (!Labels.empty())
1131 rescopeLabels();
1132 }
1133
1134 bool hasLabels() const { return !Labels.empty(); }
1135
1136 void rescopeLabels();
1137 };
1138
1139 typedef llvm::DenseMap<const Decl *, Address> DeclMapTy;
1140
1141 /// The class used to assign some variables some temporarily addresses.
1142 class OMPMapVars {
1143 DeclMapTy SavedLocals;
1144 DeclMapTy SavedTempAddresses;
1145 OMPMapVars(const OMPMapVars &) = delete;
1146 void operator=(const OMPMapVars &) = delete;
1147
1148 public:
1149 explicit OMPMapVars() = default;
1150 ~OMPMapVars() {
1151 assert(SavedLocals.empty() && "Did not restored original addresses.");
1152 };
1153
1154 /// Sets the address of the variable \p LocalVD to be \p TempAddr in
1155 /// function \p CGF.
1156 /// \return true if at least one variable was set already, false otherwise.
1157 bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD,
1158 Address TempAddr) {
1159 LocalVD = LocalVD->getCanonicalDecl();
1160 // Only save it once.
1161 if (SavedLocals.count(Val: LocalVD))
1162 return false;
1163
1164 // Copy the existing local entry to SavedLocals.
1165 auto it = CGF.LocalDeclMap.find(Val: LocalVD);
1166 if (it != CGF.LocalDeclMap.end())
1167 SavedLocals.try_emplace(Key: LocalVD, Args&: it->second);
1168 else
1169 SavedLocals.try_emplace(Key: LocalVD, Args: Address::invalid());
1170
1171 // Generate the private entry.
1172 QualType VarTy = LocalVD->getType();
1173 if (VarTy->isReferenceType()) {
1174 Address Temp = CGF.CreateMemTemp(T: VarTy);
1175 CGF.Builder.CreateStore(Val: TempAddr.emitRawPointer(CGF), Addr: Temp);
1176 TempAddr = Temp;
1177 }
1178 SavedTempAddresses.try_emplace(Key: LocalVD, Args&: TempAddr);
1179
1180 return true;
1181 }
1182
1183 /// Applies new addresses to the list of the variables.
1184 /// \return true if at least one variable is using new address, false
1185 /// otherwise.
1186 bool apply(CodeGenFunction &CGF) {
1187 copyInto(Src: SavedTempAddresses, Dest&: CGF.LocalDeclMap);
1188 SavedTempAddresses.clear();
1189 return !SavedLocals.empty();
1190 }
1191
1192 /// Restores original addresses of the variables.
1193 void restore(CodeGenFunction &CGF) {
1194 if (!SavedLocals.empty()) {
1195 copyInto(Src: SavedLocals, Dest&: CGF.LocalDeclMap);
1196 SavedLocals.clear();
1197 }
1198 }
1199
1200 private:
1201 /// Copy all the entries in the source map over the corresponding
1202 /// entries in the destination, which must exist.
1203 static void copyInto(const DeclMapTy &Src, DeclMapTy &Dest) {
1204 for (auto &[Decl, Addr] : Src) {
1205 if (!Addr.isValid())
1206 Dest.erase(Val: Decl);
1207 else
1208 Dest.insert_or_assign(Key: Decl, Val: Addr);
1209 }
1210 }
1211 };
1212
1213 /// The scope used to remap some variables as private in the OpenMP loop body
1214 /// (or other captured region emitted without outlining), and to restore old
1215 /// vars back on exit.
1216 class OMPPrivateScope : public RunCleanupsScope {
1217 OMPMapVars MappedVars;
1218 OMPPrivateScope(const OMPPrivateScope &) = delete;
1219 void operator=(const OMPPrivateScope &) = delete;
1220
1221 public:
1222 /// Enter a new OpenMP private scope.
1223 explicit OMPPrivateScope(CodeGenFunction &CGF) : RunCleanupsScope(CGF) {}
1224
1225 /// Registers \p LocalVD variable as a private with \p Addr as the address
1226 /// of the corresponding private variable. \p
1227 /// PrivateGen is the address of the generated private variable.
1228 /// \return true if the variable is registered as private, false if it has
1229 /// been privatized already.
1230 bool addPrivate(const VarDecl *LocalVD, Address Addr) {
1231 assert(PerformCleanup && "adding private to dead scope");
1232 return MappedVars.setVarAddr(CGF, LocalVD, TempAddr: Addr);
1233 }
1234
1235 /// Privatizes local variables previously registered as private.
1236 /// Registration is separate from the actual privatization to allow
1237 /// initializers use values of the original variables, not the private one.
1238 /// This is important, for example, if the private variable is a class
1239 /// variable initialized by a constructor that references other private
1240 /// variables. But at initialization original variables must be used, not
1241 /// private copies.
1242 /// \return true if at least one variable was privatized, false otherwise.
1243 bool Privatize() { return MappedVars.apply(CGF); }
1244
1245 void ForceCleanup() {
1246 RunCleanupsScope::ForceCleanup();
1247 restoreMap();
1248 }
1249
1250 /// Exit scope - all the mapped variables are restored.
1251 ~OMPPrivateScope() {
1252 if (PerformCleanup)
1253 ForceCleanup();
1254 }
1255
1256 /// Checks if the global variable is captured in current function.
1257 bool isGlobalVarCaptured(const VarDecl *VD) const {
1258 VD = VD->getCanonicalDecl();
1259 return !VD->isLocalVarDeclOrParm() && CGF.LocalDeclMap.count(Val: VD) > 0;
1260 }
1261
1262 /// Restore all mapped variables w/o clean up. This is usefully when we want
1263 /// to reference the original variables but don't want the clean up because
1264 /// that could emit lifetime end too early, causing backend issue #56913.
1265 void restoreMap() { MappedVars.restore(CGF); }
1266 };
1267
1268 /// Save/restore original map of previously emitted local vars in case when we
1269 /// need to duplicate emission of the same code several times in the same
1270 /// function for OpenMP code.
1271 class OMPLocalDeclMapRAII {
1272 CodeGenFunction &CGF;
1273 DeclMapTy SavedMap;
1274
1275 public:
1276 OMPLocalDeclMapRAII(CodeGenFunction &CGF)
1277 : CGF(CGF), SavedMap(CGF.LocalDeclMap) {}
1278 ~OMPLocalDeclMapRAII() { SavedMap.swap(RHS&: CGF.LocalDeclMap); }
1279 };
1280
1281 /// Takes the old cleanup stack size and emits the cleanup blocks
1282 /// that have been added.
1283 void
1284 PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1285 std::initializer_list<llvm::Value **> ValuesToReload = {});
1286
1287 /// Takes the old cleanup stack size and emits the cleanup blocks
1288 /// that have been added, then adds all lifetime-extended cleanups from
1289 /// the given position to the stack.
1290 void
1291 PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1292 size_t OldLifetimeExtendedStackSize,
1293 std::initializer_list<llvm::Value **> ValuesToReload = {});
1294
1295 void ResolveBranchFixups(llvm::BasicBlock *Target);
1296
1297 /// The given basic block lies in the current EH scope, but may be a
1298 /// target of a potentially scope-crossing jump; get a stable handle
1299 /// to which we can perform this jump later.
1300 JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target) {
1301 return JumpDest(Target, EHStack.getInnermostNormalCleanup(),
1302 NextCleanupDestIndex++);
1303 }
1304
1305 /// The given basic block lies in the current EH scope, but may be a
1306 /// target of a potentially scope-crossing jump; get a stable handle
1307 /// to which we can perform this jump later.
1308 JumpDest getJumpDestInCurrentScope(StringRef Name = StringRef()) {
1309 return getJumpDestInCurrentScope(Target: createBasicBlock(name: Name));
1310 }
1311
1312 /// EmitBranchThroughCleanup - Emit a branch from the current insert
1313 /// block through the normal cleanup handling code (if any) and then
1314 /// on to \arg Dest.
1315 void EmitBranchThroughCleanup(JumpDest Dest);
1316
1317 /// isObviouslyBranchWithoutCleanups - Return true if a branch to the
1318 /// specified destination obviously has no cleanups to run. 'false' is always
1319 /// a conservatively correct answer for this method.
1320 bool isObviouslyBranchWithoutCleanups(JumpDest Dest) const;
1321
1322 /// popCatchScope - Pops the catch scope at the top of the EHScope
1323 /// stack, emitting any required code (other than the catch handlers
1324 /// themselves).
1325 void popCatchScope();
1326
1327 llvm::BasicBlock *getEHResumeBlock(bool isCleanup);
1328 llvm::BasicBlock *getEHDispatchBlock(EHScopeStack::stable_iterator scope);
1329 llvm::BasicBlock *
1330 getFuncletEHDispatchBlock(EHScopeStack::stable_iterator scope);
1331
1332 /// An object to manage conditionally-evaluated expressions.
1333 class ConditionalEvaluation {
1334 llvm::BasicBlock *StartBB;
1335
1336 public:
1337 ConditionalEvaluation(CodeGenFunction &CGF)
1338 : StartBB(CGF.Builder.GetInsertBlock()) {}
1339
1340 void begin(CodeGenFunction &CGF) {
1341 assert(CGF.OutermostConditional != this);
1342 if (!CGF.OutermostConditional)
1343 CGF.OutermostConditional = this;
1344 }
1345
1346 void end(CodeGenFunction &CGF) {
1347 assert(CGF.OutermostConditional != nullptr);
1348 if (CGF.OutermostConditional == this)
1349 CGF.OutermostConditional = nullptr;
1350 }
1351
1352 /// Returns a block which will be executed prior to each
1353 /// evaluation of the conditional code.
1354 llvm::BasicBlock *getStartingBlock() const { return StartBB; }
1355 };
1356
1357 /// isInConditionalBranch - Return true if we're currently emitting
1358 /// one branch or the other of a conditional expression.
1359 bool isInConditionalBranch() const { return OutermostConditional != nullptr; }
1360
1361 void setBeforeOutermostConditional(llvm::Value *value, Address addr,
1362 CodeGenFunction &CGF) {
1363 assert(isInConditionalBranch());
1364 llvm::BasicBlock *block = OutermostConditional->getStartingBlock();
1365 auto store = new llvm::StoreInst(value, addr.emitRawPointer(CGF),
1366 block->back().getIterator());
1367 store->setAlignment(addr.getAlignment().getAsAlign());
1368 }
1369
1370 /// An RAII object to record that we're evaluating a statement
1371 /// expression.
1372 class StmtExprEvaluation {
1373 CodeGenFunction &CGF;
1374
1375 /// We have to save the outermost conditional: cleanups in a
1376 /// statement expression aren't conditional just because the
1377 /// StmtExpr is.
1378 ConditionalEvaluation *SavedOutermostConditional;
1379
1380 public:
1381 StmtExprEvaluation(CodeGenFunction &CGF)
1382 : CGF(CGF), SavedOutermostConditional(CGF.OutermostConditional) {
1383 CGF.OutermostConditional = nullptr;
1384 }
1385
1386 ~StmtExprEvaluation() {
1387 CGF.OutermostConditional = SavedOutermostConditional;
1388 CGF.EnsureInsertPoint();
1389 }
1390 };
1391
1392 /// An object which temporarily prevents a value from being
1393 /// destroyed by aggressive peephole optimizations that assume that
1394 /// all uses of a value have been realized in the IR.
1395 class PeepholeProtection {
1396 llvm::Instruction *Inst = nullptr;
1397 friend class CodeGenFunction;
1398
1399 public:
1400 PeepholeProtection() = default;
1401 };
1402
1403 /// A non-RAII class containing all the information about a bound
1404 /// opaque value. OpaqueValueMapping, below, is a RAII wrapper for
1405 /// this which makes individual mappings very simple; using this
1406 /// class directly is useful when you have a variable number of
1407 /// opaque values or don't want the RAII functionality for some
1408 /// reason.
1409 class OpaqueValueMappingData {
1410 const OpaqueValueExpr *OpaqueValue;
1411 bool BoundLValue;
1412 CodeGenFunction::PeepholeProtection Protection;
1413
1414 OpaqueValueMappingData(const OpaqueValueExpr *ov, bool boundLValue)
1415 : OpaqueValue(ov), BoundLValue(boundLValue) {}
1416
1417 public:
1418 OpaqueValueMappingData() : OpaqueValue(nullptr) {}
1419
1420 static bool shouldBindAsLValue(const Expr *expr) {
1421 // gl-values should be bound as l-values for obvious reasons.
1422 // Records should be bound as l-values because IR generation
1423 // always keeps them in memory. Expressions of function type
1424 // act exactly like l-values but are formally required to be
1425 // r-values in C.
1426 return expr->isGLValue() || expr->getType()->isFunctionType() ||
1427 hasAggregateEvaluationKind(T: expr->getType());
1428 }
1429
1430 static OpaqueValueMappingData
1431 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const Expr *e) {
1432 if (shouldBindAsLValue(expr: ov))
1433 return bind(CGF, ov, lv: CGF.EmitLValue(E: e));
1434 return bind(CGF, ov, rv: CGF.EmitAnyExpr(E: e));
1435 }
1436
1437 static OpaqueValueMappingData
1438 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const LValue &lv) {
1439 assert(shouldBindAsLValue(ov));
1440 CGF.OpaqueLValues.insert(KV: std::make_pair(x&: ov, y: lv));
1441 return OpaqueValueMappingData(ov, true);
1442 }
1443
1444 static OpaqueValueMappingData
1445 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const RValue &rv) {
1446 assert(!shouldBindAsLValue(ov));
1447 CGF.OpaqueRValues.insert(KV: std::make_pair(x&: ov, y: rv));
1448
1449 OpaqueValueMappingData data(ov, false);
1450
1451 // Work around an extremely aggressive peephole optimization in
1452 // EmitScalarConversion which assumes that all other uses of a
1453 // value are extant.
1454 data.Protection = CGF.protectFromPeepholes(rvalue: rv);
1455
1456 return data;
1457 }
1458
1459 bool isValid() const { return OpaqueValue != nullptr; }
1460 void clear() { OpaqueValue = nullptr; }
1461
1462 void unbind(CodeGenFunction &CGF) {
1463 assert(OpaqueValue && "no data to unbind!");
1464
1465 if (BoundLValue) {
1466 CGF.OpaqueLValues.erase(Val: OpaqueValue);
1467 } else {
1468 CGF.OpaqueRValues.erase(Val: OpaqueValue);
1469 CGF.unprotectFromPeepholes(protection: Protection);
1470 }
1471 }
1472 };
1473
1474 /// An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
1475 class OpaqueValueMapping {
1476 CodeGenFunction &CGF;
1477 OpaqueValueMappingData Data;
1478
1479 public:
1480 static bool shouldBindAsLValue(const Expr *expr) {
1481 return OpaqueValueMappingData::shouldBindAsLValue(expr);
1482 }
1483
1484 /// Build the opaque value mapping for the given conditional
1485 /// operator if it's the GNU ?: extension. This is a common
1486 /// enough pattern that the convenience operator is really
1487 /// helpful.
1488 ///
1489 OpaqueValueMapping(CodeGenFunction &CGF,
1490 const AbstractConditionalOperator *op)
1491 : CGF(CGF) {
1492 if (isa<ConditionalOperator>(Val: op))
1493 // Leave Data empty.
1494 return;
1495
1496 const BinaryConditionalOperator *e = cast<BinaryConditionalOperator>(Val: op);
1497 Data = OpaqueValueMappingData::bind(CGF, ov: e->getOpaqueValue(),
1498 e: e->getCommon());
1499 }
1500
1501 /// Build the opaque value mapping for an OpaqueValueExpr whose source
1502 /// expression is set to the expression the OVE represents.
1503 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *OV)
1504 : CGF(CGF) {
1505 if (OV) {
1506 assert(OV->getSourceExpr() && "wrong form of OpaqueValueMapping used "
1507 "for OVE with no source expression");
1508 Data = OpaqueValueMappingData::bind(CGF, ov: OV, e: OV->getSourceExpr());
1509 }
1510 }
1511
1512 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,
1513 LValue lvalue)
1514 : CGF(CGF),
1515 Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, lv: lvalue)) {}
1516
1517 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,
1518 RValue rvalue)
1519 : CGF(CGF),
1520 Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, rv: rvalue)) {}
1521
1522 void pop() {
1523 Data.unbind(CGF);
1524 Data.clear();
1525 }
1526
1527 ~OpaqueValueMapping() {
1528 if (Data.isValid())
1529 Data.unbind(CGF);
1530 }
1531 };
1532
1533private:
1534 CGDebugInfo *DebugInfo;
1535 /// Used to create unique names for artificial VLA size debug info variables.
1536 unsigned VLAExprCounter = 0;
1537 bool DisableDebugInfo = false;
1538
1539 /// DidCallStackSave - Whether llvm.stacksave has been called. Used to avoid
1540 /// calling llvm.stacksave for multiple VLAs in the same scope.
1541 bool DidCallStackSave = false;
1542
1543 /// IndirectBranch - The first time an indirect goto is seen we create a block
1544 /// with an indirect branch. Every time we see the address of a label taken,
1545 /// we add the label to the indirect goto. Every subsequent indirect goto is
1546 /// codegen'd as a jump to the IndirectBranch's basic block.
1547 llvm::IndirectBrInst *IndirectBranch = nullptr;
1548
1549 /// LocalDeclMap - This keeps track of the LLVM allocas or globals for local C
1550 /// decls.
1551 DeclMapTy LocalDeclMap;
1552
1553 // Keep track of the cleanups for callee-destructed parameters pushed to the
1554 // cleanup stack so that they can be deactivated later.
1555 llvm::DenseMap<const ParmVarDecl *, EHScopeStack::stable_iterator>
1556 CalleeDestructedParamCleanups;
1557
1558 /// SizeArguments - If a ParmVarDecl had the pass_object_size attribute, this
1559 /// will contain a mapping from said ParmVarDecl to its implicit "object_size"
1560 /// parameter.
1561 llvm::SmallDenseMap<const ParmVarDecl *, const ImplicitParamDecl *, 2>
1562 SizeArguments;
1563
1564 /// Track escaped local variables with auto storage. Used during SEH
1565 /// outlining to produce a call to llvm.localescape.
1566 llvm::DenseMap<llvm::AllocaInst *, int> EscapedLocals;
1567
1568 /// LabelMap - This keeps track of the LLVM basic block for each C label.
1569 llvm::DenseMap<const LabelDecl *, JumpDest> LabelMap;
1570
1571 // BreakContinueStack - This keeps track of where break and continue
1572 // statements should jump to.
1573 struct BreakContinue {
1574 BreakContinue(const Stmt &LoopOrSwitch, JumpDest Break, JumpDest Continue)
1575 : LoopOrSwitch(&LoopOrSwitch), BreakBlock(Break),
1576 ContinueBlock(Continue) {}
1577
1578 const Stmt *LoopOrSwitch;
1579 JumpDest BreakBlock;
1580 JumpDest ContinueBlock;
1581 };
1582 SmallVector<BreakContinue, 8> BreakContinueStack;
1583
1584 /// Handles cancellation exit points in OpenMP-related constructs.
1585 class OpenMPCancelExitStack {
1586 /// Tracks cancellation exit point and join point for cancel-related exit
1587 /// and normal exit.
1588 struct CancelExit {
1589 CancelExit() = default;
1590 CancelExit(OpenMPDirectiveKind Kind, JumpDest ExitBlock,
1591 JumpDest ContBlock)
1592 : Kind(Kind), ExitBlock(ExitBlock), ContBlock(ContBlock) {}
1593 OpenMPDirectiveKind Kind = llvm::omp::OMPD_unknown;
1594 /// true if the exit block has been emitted already by the special
1595 /// emitExit() call, false if the default codegen is used.
1596 bool HasBeenEmitted = false;
1597 JumpDest ExitBlock;
1598 JumpDest ContBlock;
1599 };
1600
1601 SmallVector<CancelExit, 8> Stack;
1602
1603 public:
1604 OpenMPCancelExitStack() : Stack(1) {}
1605 ~OpenMPCancelExitStack() = default;
1606 /// Fetches the exit block for the current OpenMP construct.
1607 JumpDest getExitBlock() const { return Stack.back().ExitBlock; }
1608 /// Emits exit block with special codegen procedure specific for the related
1609 /// OpenMP construct + emits code for normal construct cleanup.
1610 void emitExit(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
1611 const llvm::function_ref<void(CodeGenFunction &)> CodeGen) {
1612 if (Stack.back().Kind == Kind && getExitBlock().isValid()) {
1613 assert(CGF.getOMPCancelDestination(Kind).isValid());
1614 assert(CGF.HaveInsertPoint());
1615 assert(!Stack.back().HasBeenEmitted);
1616 auto IP = CGF.Builder.saveAndClearIP();
1617 CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1618 CodeGen(CGF);
1619 CGF.EmitBranch(Block: Stack.back().ContBlock.getBlock());
1620 CGF.Builder.restoreIP(IP);
1621 Stack.back().HasBeenEmitted = true;
1622 }
1623 CodeGen(CGF);
1624 }
1625 /// Enter the cancel supporting \a Kind construct.
1626 /// \param Kind OpenMP directive that supports cancel constructs.
1627 /// \param HasCancel true, if the construct has inner cancel directive,
1628 /// false otherwise.
1629 void enter(CodeGenFunction &CGF, OpenMPDirectiveKind Kind, bool HasCancel) {
1630 Stack.push_back(Elt: {Kind,
1631 HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.exit")
1632 : JumpDest(),
1633 HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.cont")
1634 : JumpDest()});
1635 }
1636 /// Emits default exit point for the cancel construct (if the special one
1637 /// has not be used) + join point for cancel/normal exits.
1638 void exit(CodeGenFunction &CGF) {
1639 if (getExitBlock().isValid()) {
1640 assert(CGF.getOMPCancelDestination(Stack.back().Kind).isValid());
1641 bool HaveIP = CGF.HaveInsertPoint();
1642 if (!Stack.back().HasBeenEmitted) {
1643 if (HaveIP)
1644 CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1645 CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1646 CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1647 }
1648 CGF.EmitBlock(BB: Stack.back().ContBlock.getBlock());
1649 if (!HaveIP) {
1650 CGF.Builder.CreateUnreachable();
1651 CGF.Builder.ClearInsertionPoint();
1652 }
1653 }
1654 Stack.pop_back();
1655 }
1656 };
1657 OpenMPCancelExitStack OMPCancelStack;
1658
1659 /// Lower the Likelihood knowledge about the \p Cond via llvm.expect intrin.
1660 llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
1661 Stmt::Likelihood LH);
1662
1663 std::unique_ptr<CodeGenPGO> PGO;
1664
1665 /// Calculate branch weights appropriate for PGO data
1666 llvm::MDNode *createProfileWeights(uint64_t TrueCount,
1667 uint64_t FalseCount) const;
1668 llvm::MDNode *createProfileWeights(ArrayRef<uint64_t> Weights) const;
1669 llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond,
1670 uint64_t LoopCount) const;
1671
1672public:
1673 bool hasSkipCounter(const Stmt *S) const;
1674
1675 void markStmtAsUsed(bool Skipped, const Stmt *S);
1676 void markStmtMaybeUsed(const Stmt *S);
1677
1678 /// Used to specify which counter in a pair shall be incremented.
1679 /// For non-binary counters, a skip counter is derived as (Parent - Exec).
1680 /// In contrast for binary counters, a skip counter cannot be computed from
1681 /// the Parent counter. In such cases, dedicated SkipPath counters must be
1682 /// allocated and marked (incremented as binary counters). (Parent can be
1683 /// synthesized with (Exec + Skip) in simple cases)
1684 enum CounterForIncrement {
1685 UseExecPath = 0, ///< Exec (true)
1686 UseSkipPath, ///< Skip (false)
1687 };
1688
1689 /// Increment the profiler's counter for the given statement by \p StepV.
1690 /// If \p StepV is null, the default increment is 1.
1691 void incrementProfileCounter(const Stmt *S, llvm::Value *StepV = nullptr) {
1692 incrementProfileCounter(ExecSkip: UseExecPath, S, UseBoth: false, StepV);
1693 }
1694
1695 /// Emit increment of Counter.
1696 /// \param ExecSkip Use `Skipped` Counter if UseSkipPath is specified.
1697 /// \param S The Stmt that Counter is associated.
1698 /// \param UseBoth Mark both Exec/Skip as used. (for verification)
1699 /// \param StepV The offset Value for adding to Counter.
1700 void incrementProfileCounter(CounterForIncrement ExecSkip, const Stmt *S,
1701 bool UseBoth = false,
1702 llvm::Value *StepV = nullptr);
1703
1704 bool isMCDCCoverageEnabled() const {
1705 return (CGM.getCodeGenOpts().hasProfileClangInstr() &&
1706 CGM.getCodeGenOpts().MCDCCoverage &&
1707 !CurFn->hasFnAttribute(Kind: llvm::Attribute::NoProfile));
1708 }
1709
1710 /// Allocate a temp value on the stack that MCDC can use to track condition
1711 /// results.
1712 void maybeCreateMCDCCondBitmap();
1713
1714 bool isBinaryLogicalOp(const Expr *E) const {
1715 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: E->IgnoreParens());
1716 return (BOp && BOp->isLogicalOp());
1717 }
1718
1719 bool isMCDCDecisionExpr(const Expr *E) const;
1720 bool isMCDCBranchExpr(const Expr *E) const;
1721
1722 /// Zero-init the MCDC temp value.
1723 void maybeResetMCDCCondBitmap(const Expr *E);
1724
1725 /// Increment the profiler's counter for the given expression by \p StepV.
1726 /// If \p StepV is null, the default increment is 1.
1727 void maybeUpdateMCDCTestVectorBitmap(const Expr *E);
1728
1729 /// Update the MCDC temp value with the condition's evaluated result.
1730 void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val);
1731
1732 /// Get the profiler's count for the given statement.
1733 uint64_t getProfileCount(const Stmt *S);
1734
1735 /// Set the profiler's current count.
1736 void setCurrentProfileCount(uint64_t Count);
1737
1738 /// Get the profiler's current count. This is generally the count for the most
1739 /// recently incremented counter.
1740 uint64_t getCurrentProfileCount();
1741
1742 /// See CGDebugInfo::addInstToCurrentSourceAtom.
1743 void addInstToCurrentSourceAtom(llvm::Instruction *KeyInstruction,
1744 llvm::Value *Backup);
1745
1746 /// See CGDebugInfo::addInstToSpecificSourceAtom.
1747 void addInstToSpecificSourceAtom(llvm::Instruction *KeyInstruction,
1748 llvm::Value *Backup, uint64_t Atom);
1749
1750 /// Add \p KeyInstruction and an optional \p Backup instruction to a new atom
1751 /// group (See ApplyAtomGroup for more info).
1752 void addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,
1753 llvm::Value *Backup);
1754
1755 /// Copy all PFP fields from SrcPtr to DestPtr while updating signatures,
1756 /// assuming that DestPtr was already memcpy'd from SrcPtr.
1757 void emitPFPPostCopyUpdates(Address DestPtr, Address SrcPtr, QualType Ty);
1758
1759private:
1760 /// SwitchInsn - This is nearest current switch instruction. It is null if
1761 /// current context is not in a switch.
1762 llvm::SwitchInst *SwitchInsn = nullptr;
1763 /// The branch weights of SwitchInsn when doing instrumentation based PGO.
1764 SmallVector<uint64_t, 16> *SwitchWeights = nullptr;
1765
1766 /// The likelihood attributes of the SwitchCase.
1767 SmallVector<Stmt::Likelihood, 16> *SwitchLikelihood = nullptr;
1768
1769 /// CaseRangeBlock - This block holds if condition check for last case
1770 /// statement range in current switch instruction.
1771 llvm::BasicBlock *CaseRangeBlock = nullptr;
1772
1773 /// OpaqueLValues - Keeps track of the current set of opaque value
1774 /// expressions.
1775 llvm::DenseMap<const OpaqueValueExpr *, LValue> OpaqueLValues;
1776 llvm::DenseMap<const OpaqueValueExpr *, RValue> OpaqueRValues;
1777
1778 // VLASizeMap - This keeps track of the associated size for each VLA type.
1779 // We track this by the size expression rather than the type itself because
1780 // in certain situations, like a const qualifier applied to an VLA typedef,
1781 // multiple VLA types can share the same size expression.
1782 // FIXME: Maybe this could be a stack of maps that is pushed/popped as we
1783 // enter/leave scopes.
1784 llvm::DenseMap<const Expr *, llvm::Value *> VLASizeMap;
1785
1786 /// A block containing a single 'unreachable' instruction. Created
1787 /// lazily by getUnreachableBlock().
1788 llvm::BasicBlock *UnreachableBlock = nullptr;
1789
1790 /// Counts of the number return expressions in the function.
1791 unsigned NumReturnExprs = 0;
1792
1793 /// Count the number of simple (constant) return expressions in the function.
1794 unsigned NumSimpleReturnExprs = 0;
1795
1796 /// The last regular (non-return) debug location (breakpoint) in the function.
1797 SourceLocation LastStopPoint;
1798
1799public:
1800 /// Source location information about the default argument or member
1801 /// initializer expression we're evaluating, if any.
1802 CurrentSourceLocExprScope CurSourceLocExprScope;
1803 using SourceLocExprScopeGuard =
1804 CurrentSourceLocExprScope::SourceLocExprScopeGuard;
1805
1806 /// A scope within which we are constructing the fields of an object which
1807 /// might use a CXXDefaultInitExpr. This stashes away a 'this' value to use
1808 /// if we need to evaluate a CXXDefaultInitExpr within the evaluation.
1809 class FieldConstructionScope {
1810 public:
1811 FieldConstructionScope(CodeGenFunction &CGF, Address This)
1812 : CGF(CGF), OldCXXDefaultInitExprThis(CGF.CXXDefaultInitExprThis) {
1813 CGF.CXXDefaultInitExprThis = This;
1814 }
1815 ~FieldConstructionScope() {
1816 CGF.CXXDefaultInitExprThis = OldCXXDefaultInitExprThis;
1817 }
1818
1819 private:
1820 CodeGenFunction &CGF;
1821 Address OldCXXDefaultInitExprThis;
1822 };
1823
1824 /// The scope of a CXXDefaultInitExpr. Within this scope, the value of 'this'
1825 /// is overridden to be the object under construction.
1826 class CXXDefaultInitExprScope {
1827 public:
1828 CXXDefaultInitExprScope(CodeGenFunction &CGF, const CXXDefaultInitExpr *E)
1829 : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue),
1830 OldCXXThisAlignment(CGF.CXXThisAlignment),
1831 SourceLocScope(E, CGF.CurSourceLocExprScope) {
1832 CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getBasePointer();
1833 CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment();
1834 }
1835 ~CXXDefaultInitExprScope() {
1836 CGF.CXXThisValue = OldCXXThisValue;
1837 CGF.CXXThisAlignment = OldCXXThisAlignment;
1838 }
1839
1840 public:
1841 CodeGenFunction &CGF;
1842 llvm::Value *OldCXXThisValue;
1843 CharUnits OldCXXThisAlignment;
1844 SourceLocExprScopeGuard SourceLocScope;
1845 };
1846
1847 struct CXXDefaultArgExprScope : SourceLocExprScopeGuard {
1848 CXXDefaultArgExprScope(CodeGenFunction &CGF, const CXXDefaultArgExpr *E)
1849 : SourceLocExprScopeGuard(E, CGF.CurSourceLocExprScope) {}
1850 };
1851
1852 /// The scope of an ArrayInitLoopExpr. Within this scope, the value of the
1853 /// current loop index is overridden.
1854 class ArrayInitLoopExprScope {
1855 public:
1856 ArrayInitLoopExprScope(CodeGenFunction &CGF, llvm::Value *Index)
1857 : CGF(CGF), OldArrayInitIndex(CGF.ArrayInitIndex) {
1858 CGF.ArrayInitIndex = Index;
1859 }
1860 ~ArrayInitLoopExprScope() { CGF.ArrayInitIndex = OldArrayInitIndex; }
1861
1862 private:
1863 CodeGenFunction &CGF;
1864 llvm::Value *OldArrayInitIndex;
1865 };
1866
1867 class InlinedInheritingConstructorScope {
1868 public:
1869 InlinedInheritingConstructorScope(CodeGenFunction &CGF, GlobalDecl GD)
1870 : CGF(CGF), OldCurGD(CGF.CurGD), OldCurFuncDecl(CGF.CurFuncDecl),
1871 OldCurCodeDecl(CGF.CurCodeDecl),
1872 OldCXXABIThisDecl(CGF.CXXABIThisDecl),
1873 OldCXXABIThisValue(CGF.CXXABIThisValue),
1874 OldCXXThisValue(CGF.CXXThisValue),
1875 OldCXXABIThisAlignment(CGF.CXXABIThisAlignment),
1876 OldCXXThisAlignment(CGF.CXXThisAlignment),
1877 OldReturnValue(CGF.ReturnValue), OldFnRetTy(CGF.FnRetTy),
1878 OldCXXInheritedCtorInitExprArgs(
1879 std::move(CGF.CXXInheritedCtorInitExprArgs)) {
1880 CGF.CurGD = GD;
1881 CGF.CurFuncDecl = CGF.CurCodeDecl =
1882 cast<CXXConstructorDecl>(Val: GD.getDecl());
1883 CGF.CXXABIThisDecl = nullptr;
1884 CGF.CXXABIThisValue = nullptr;
1885 CGF.CXXThisValue = nullptr;
1886 CGF.CXXABIThisAlignment = CharUnits();
1887 CGF.CXXThisAlignment = CharUnits();
1888 CGF.ReturnValue = Address::invalid();
1889 CGF.FnRetTy = QualType();
1890 CGF.CXXInheritedCtorInitExprArgs.clear();
1891 }
1892 ~InlinedInheritingConstructorScope() {
1893 CGF.CurGD = OldCurGD;
1894 CGF.CurFuncDecl = OldCurFuncDecl;
1895 CGF.CurCodeDecl = OldCurCodeDecl;
1896 CGF.CXXABIThisDecl = OldCXXABIThisDecl;
1897 CGF.CXXABIThisValue = OldCXXABIThisValue;
1898 CGF.CXXThisValue = OldCXXThisValue;
1899 CGF.CXXABIThisAlignment = OldCXXABIThisAlignment;
1900 CGF.CXXThisAlignment = OldCXXThisAlignment;
1901 CGF.ReturnValue = OldReturnValue;
1902 CGF.FnRetTy = OldFnRetTy;
1903 CGF.CXXInheritedCtorInitExprArgs =
1904 std::move(OldCXXInheritedCtorInitExprArgs);
1905 }
1906
1907 private:
1908 CodeGenFunction &CGF;
1909 GlobalDecl OldCurGD;
1910 const Decl *OldCurFuncDecl;
1911 const Decl *OldCurCodeDecl;
1912 ImplicitParamDecl *OldCXXABIThisDecl;
1913 llvm::Value *OldCXXABIThisValue;
1914 llvm::Value *OldCXXThisValue;
1915 CharUnits OldCXXABIThisAlignment;
1916 CharUnits OldCXXThisAlignment;
1917 Address OldReturnValue;
1918 QualType OldFnRetTy;
1919 CallArgList OldCXXInheritedCtorInitExprArgs;
1920 };
1921
1922 // Helper class for the OpenMP IR Builder. Allows reusability of code used for
1923 // region body, and finalization codegen callbacks. This will class will also
1924 // contain privatization functions used by the privatization call backs
1925 //
1926 // TODO: this is temporary class for things that are being moved out of
1927 // CGOpenMPRuntime, new versions of current CodeGenFunction methods, or
1928 // utility function for use with the OMPBuilder. Once that move to use the
1929 // OMPBuilder is done, everything here will either become part of CodeGenFunc.
1930 // directly, or a new helper class that will contain functions used by both
1931 // this and the OMPBuilder
1932
1933 struct OMPBuilderCBHelpers {
1934
1935 OMPBuilderCBHelpers() = delete;
1936 OMPBuilderCBHelpers(const OMPBuilderCBHelpers &) = delete;
1937 OMPBuilderCBHelpers &operator=(const OMPBuilderCBHelpers &) = delete;
1938
1939 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1940
1941 /// Cleanup action for allocate support.
1942 class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
1943
1944 private:
1945 llvm::CallInst *RTLFnCI;
1946
1947 public:
1948 OMPAllocateCleanupTy(llvm::CallInst *RLFnCI) : RTLFnCI(RLFnCI) {
1949 RLFnCI->removeFromParent();
1950 }
1951
1952 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
1953 if (!CGF.HaveInsertPoint())
1954 return;
1955 CGF.Builder.Insert(I: RTLFnCI);
1956 }
1957 };
1958
1959 /// Returns address of the threadprivate variable for the current
1960 /// thread. This Also create any necessary OMP runtime calls.
1961 ///
1962 /// \param VD VarDecl for Threadprivate variable.
1963 /// \param VDAddr Address of the Vardecl
1964 /// \param Loc The location where the barrier directive was encountered
1965 static Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
1966 const VarDecl *VD, Address VDAddr,
1967 SourceLocation Loc);
1968
1969 /// Gets the OpenMP-specific address of the local variable /p VD.
1970 static Address getAddressOfLocalVariable(CodeGenFunction &CGF,
1971 const VarDecl *VD);
1972 /// Get the platform-specific name separator.
1973 /// \param Parts different parts of the final name that needs separation
1974 /// \param FirstSeparator First separator used between the initial two
1975 /// parts of the name.
1976 /// \param Separator separator used between all of the rest consecutinve
1977 /// parts of the name
1978 static std::string getNameWithSeparators(ArrayRef<StringRef> Parts,
1979 StringRef FirstSeparator = ".",
1980 StringRef Separator = ".");
1981 /// Emit the Finalization for an OMP region
1982 /// \param CGF The Codegen function this belongs to
1983 /// \param IP Insertion point for generating the finalization code.
1984 static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP) {
1985 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1986 assert(IP.getBlock()->end() != IP.getPoint() &&
1987 "OpenMP IR Builder should cause terminated block!");
1988
1989 llvm::BasicBlock *IPBB = IP.getBlock();
1990 llvm::BasicBlock *DestBB = IPBB->getUniqueSuccessor();
1991 assert(DestBB && "Finalization block should have one successor!");
1992
1993 // erase and replace with cleanup branch.
1994 IPBB->getTerminator()->eraseFromParent();
1995 CGF.Builder.SetInsertPoint(IPBB);
1996 CodeGenFunction::JumpDest Dest = CGF.getJumpDestInCurrentScope(Target: DestBB);
1997 CGF.EmitBranchThroughCleanup(Dest);
1998 }
1999
2000 /// Emit the body of an OMP region
2001 /// \param CGF The Codegen function this belongs to
2002 /// \param RegionBodyStmt The body statement for the OpenMP region being
2003 /// generated
2004 /// \param AllocaIP Where to insert alloca instructions
2005 /// \param CodeGenIP Where to insert the region code
2006 /// \param RegionName Name to be used for new blocks
2007 static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF,
2008 const Stmt *RegionBodyStmt,
2009 InsertPointTy AllocaIP,
2010 InsertPointTy CodeGenIP,
2011 Twine RegionName);
2012
2013 static void EmitCaptureStmt(CodeGenFunction &CGF, InsertPointTy CodeGenIP,
2014 llvm::BasicBlock &FiniBB, llvm::Function *Fn,
2015 ArrayRef<llvm::Value *> Args) {
2016 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2017 if (llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminatorOrNull())
2018 CodeGenIPBBTI->eraseFromParent();
2019
2020 CGF.Builder.SetInsertPoint(CodeGenIPBB);
2021
2022 if (Fn->doesNotThrow())
2023 CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
2024 else
2025 CGF.EmitRuntimeCall(callee: Fn, args: Args);
2026
2027 if (CGF.Builder.saveIP().isSet())
2028 CGF.Builder.CreateBr(Dest: &FiniBB);
2029 }
2030
2031 /// Emit the body of an OMP region that will be outlined in
2032 /// OpenMPIRBuilder::finalize().
2033 /// \param CGF The Codegen function this belongs to
2034 /// \param RegionBodyStmt The body statement for the OpenMP region being
2035 /// generated
2036 /// \param AllocaIP Where to insert alloca instructions
2037 /// \param CodeGenIP Where to insert the region code
2038 /// \param RegionName Name to be used for new blocks
2039 static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF,
2040 const Stmt *RegionBodyStmt,
2041 InsertPointTy AllocaIP,
2042 InsertPointTy CodeGenIP,
2043 Twine RegionName);
2044
2045 /// RAII for preserving necessary info during Outlined region body codegen.
2046 class OutlinedRegionBodyRAII {
2047
2048 llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
2049 CodeGenFunction::JumpDest OldReturnBlock;
2050 CodeGenFunction &CGF;
2051
2052 public:
2053 OutlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
2054 llvm::BasicBlock &RetBB)
2055 : CGF(cgf) {
2056 assert(AllocaIP.isSet() &&
2057 "Must specify Insertion point for allocas of outlined function");
2058 OldAllocaIP = CGF.AllocaInsertPt;
2059 CGF.AllocaInsertPt = &*AllocaIP.getPoint();
2060
2061 OldReturnBlock = CGF.ReturnBlock;
2062 CGF.ReturnBlock = CGF.getJumpDestInCurrentScope(Target: &RetBB);
2063 }
2064
2065 ~OutlinedRegionBodyRAII() {
2066 CGF.AllocaInsertPt = OldAllocaIP;
2067 CGF.ReturnBlock = OldReturnBlock;
2068 }
2069 };
2070
2071 /// RAII for preserving necessary info during inlined region body codegen.
2072 class InlinedRegionBodyRAII {
2073
2074 llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
2075 CodeGenFunction &CGF;
2076
2077 public:
2078 InlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
2079 llvm::BasicBlock &FiniBB)
2080 : CGF(cgf) {
2081 // Alloca insertion block should be in the entry block of the containing
2082 // function so it expects an empty AllocaIP in which case will reuse the
2083 // old alloca insertion point, or a new AllocaIP in the same block as
2084 // the old one
2085 assert((!AllocaIP.isSet() ||
2086 CGF.AllocaInsertPt->getParent() == AllocaIP.getBlock()) &&
2087 "Insertion point should be in the entry block of containing "
2088 "function!");
2089 OldAllocaIP = CGF.AllocaInsertPt;
2090 if (AllocaIP.isSet())
2091 CGF.AllocaInsertPt = &*AllocaIP.getPoint();
2092
2093 // TODO: Remove the call, after making sure the counter is not used by
2094 // the EHStack.
2095 // Since this is an inlined region, it should not modify the
2096 // ReturnBlock, and should reuse the one for the enclosing outlined
2097 // region. So, the JumpDest being return by the function is discarded
2098 (void)CGF.getJumpDestInCurrentScope(Target: &FiniBB);
2099 }
2100
2101 ~InlinedRegionBodyRAII() { CGF.AllocaInsertPt = OldAllocaIP; }
2102 };
2103 };
2104
2105private:
2106 /// CXXThisDecl - When generating code for a C++ member function,
2107 /// this will hold the implicit 'this' declaration.
2108 ImplicitParamDecl *CXXABIThisDecl = nullptr;
2109 llvm::Value *CXXABIThisValue = nullptr;
2110 llvm::Value *CXXThisValue = nullptr;
2111 CharUnits CXXABIThisAlignment;
2112 CharUnits CXXThisAlignment;
2113
2114 /// The value of 'this' to use when evaluating CXXDefaultInitExprs within
2115 /// this expression.
2116 Address CXXDefaultInitExprThis = Address::invalid();
2117
2118 /// The current array initialization index when evaluating an
2119 /// ArrayInitIndexExpr within an ArrayInitLoopExpr.
2120 llvm::Value *ArrayInitIndex = nullptr;
2121
2122 /// The values of function arguments to use when evaluating
2123 /// CXXInheritedCtorInitExprs within this context.
2124 CallArgList CXXInheritedCtorInitExprArgs;
2125
2126 /// CXXStructorImplicitParamDecl - When generating code for a constructor or
2127 /// destructor, this will hold the implicit argument (e.g. VTT).
2128 ImplicitParamDecl *CXXStructorImplicitParamDecl = nullptr;
2129 llvm::Value *CXXStructorImplicitParamValue = nullptr;
2130
2131 /// OutermostConditional - Points to the outermost active
2132 /// conditional control. This is used so that we know if a
2133 /// temporary should be destroyed conditionally.
2134 ConditionalEvaluation *OutermostConditional = nullptr;
2135
2136 /// The current lexical scope.
2137 LexicalScope *CurLexicalScope = nullptr;
2138
2139 /// The current source location that should be used for exception
2140 /// handling code.
2141 SourceLocation CurEHLocation;
2142
2143 /// BlockByrefInfos - For each __block variable, contains
2144 /// information about the layout of the variable.
2145 llvm::DenseMap<const ValueDecl *, BlockByrefInfo> BlockByrefInfos;
2146
2147 /// Used by -fsanitize=nullability-return to determine whether the return
2148 /// value can be checked.
2149 llvm::Value *RetValNullabilityPrecondition = nullptr;
2150
2151 /// Check if -fsanitize=nullability-return instrumentation is required for
2152 /// this function.
2153 bool requiresReturnValueNullabilityCheck() const {
2154 return RetValNullabilityPrecondition;
2155 }
2156
2157 /// Used to store precise source locations for return statements by the
2158 /// runtime return value checks.
2159 Address ReturnLocation = Address::invalid();
2160
2161 /// Check if the return value of this function requires sanitization.
2162 bool requiresReturnValueCheck() const;
2163
2164 bool isInAllocaArgument(CGCXXABI &ABI, QualType Ty);
2165 bool hasInAllocaArg(const CXXMethodDecl *MD);
2166
2167 llvm::BasicBlock *TerminateLandingPad = nullptr;
2168 llvm::BasicBlock *TerminateHandler = nullptr;
2169 llvm::SmallVector<llvm::BasicBlock *, 2> TrapBBs;
2170
2171 /// Terminate funclets keyed by parent funclet pad.
2172 llvm::MapVector<llvm::Value *, llvm::BasicBlock *> TerminateFunclets;
2173
2174 /// Largest vector width used in ths function. Will be used to create a
2175 /// function attribute.
2176 unsigned LargestVectorWidth = 0;
2177
2178 /// True if we need emit the life-time markers. This is initially set in
2179 /// the constructor, but could be overwritten to true if this is a coroutine.
2180 bool ShouldEmitLifetimeMarkers;
2181
2182 /// Add OpenCL kernel arg metadata and the kernel attribute metadata to
2183 /// the function metadata.
2184 void EmitKernelMetadata(const FunctionDecl *FD, llvm::Function *Fn);
2185
2186public:
2187 CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext = false);
2188 ~CodeGenFunction();
2189
2190 CodeGenTypes &getTypes() const { return CGM.getTypes(); }
2191 ASTContext &getContext() const { return CGM.getContext(); }
2192 CGDebugInfo *getDebugInfo() {
2193 if (DisableDebugInfo)
2194 return nullptr;
2195 return DebugInfo;
2196 }
2197 void disableDebugInfo() { DisableDebugInfo = true; }
2198 void enableDebugInfo() { DisableDebugInfo = false; }
2199
2200 bool shouldUseFusedARCCalls() {
2201 return CGM.getCodeGenOpts().OptimizationLevel == 0;
2202 }
2203
2204 const LangOptions &getLangOpts() const { return CGM.getLangOpts(); }
2205
2206 /// Returns a pointer to the function's exception object and selector slot,
2207 /// which is assigned in every landing pad.
2208 Address getExceptionSlot();
2209 Address getEHSelectorSlot();
2210
2211 /// Returns the contents of the function's exception object and selector
2212 /// slots.
2213 llvm::Value *getExceptionFromSlot();
2214 llvm::Value *getSelectorFromSlot();
2215
2216 RawAddress getNormalCleanupDestSlot();
2217
2218 llvm::BasicBlock *getUnreachableBlock() {
2219 if (!UnreachableBlock) {
2220 UnreachableBlock = createBasicBlock(name: "unreachable");
2221 new llvm::UnreachableInst(getLLVMContext(), UnreachableBlock);
2222 }
2223 return UnreachableBlock;
2224 }
2225
2226 llvm::BasicBlock *getInvokeDest() {
2227 if (!EHStack.requiresLandingPad())
2228 return nullptr;
2229 return getInvokeDestImpl();
2230 }
2231
2232 bool currentFunctionUsesSEHTry() const { return !!CurSEHParent; }
2233
2234 const TargetInfo &getTarget() const { return Target; }
2235 llvm::LLVMContext &getLLVMContext() { return CGM.getLLVMContext(); }
2236 const TargetCodeGenInfo &getTargetHooks() const {
2237 return CGM.getTargetCodeGenInfo();
2238 }
2239 const FunctionDecl *getCurrentFunctionDecl() const;
2240
2241 //===--------------------------------------------------------------------===//
2242 // Cleanups
2243 //===--------------------------------------------------------------------===//
2244
2245 typedef void Destroyer(CodeGenFunction &CGF, Address addr, QualType ty);
2246
2247 void pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,
2248 Address arrayEndPointer,
2249 QualType elementType,
2250 CharUnits elementAlignment,
2251 Destroyer *destroyer);
2252 void pushRegularPartialArrayCleanup(llvm::Value *arrayBegin,
2253 llvm::Value *arrayEnd,
2254 QualType elementType,
2255 CharUnits elementAlignment,
2256 Destroyer *destroyer);
2257
2258 void pushDestroy(QualType::DestructionKind dtorKind, Address addr,
2259 QualType type);
2260 void pushEHDestroy(QualType::DestructionKind dtorKind, Address addr,
2261 QualType type);
2262 void pushDestroy(CleanupKind kind, Address addr, QualType type,
2263 Destroyer *destroyer, bool useEHCleanupForArray);
2264 void pushDestroyAndDeferDeactivation(QualType::DestructionKind dtorKind,
2265 Address addr, QualType type);
2266 void pushDestroyAndDeferDeactivation(CleanupKind cleanupKind, Address addr,
2267 QualType type, Destroyer *destroyer,
2268 bool useEHCleanupForArray);
2269 void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr,
2270 QualType type, Destroyer *destroyer,
2271 bool useEHCleanupForArray);
2272 void pushLifetimeExtendedDestroy(QualType::DestructionKind dtorKind,
2273 Address addr, QualType type);
2274 void pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete,
2275 llvm::Value *CompletePtr,
2276 QualType ElementType);
2277 void pushStackRestore(CleanupKind kind, Address SPMem);
2278 void pushKmpcAllocFree(CleanupKind Kind,
2279 std::pair<llvm::Value *, llvm::Value *> AddrSizePair);
2280 void emitDestroy(Address addr, QualType type, Destroyer *destroyer,
2281 bool useEHCleanupForArray);
2282 llvm::Function *generateDestroyHelper(Address addr, QualType type,
2283 Destroyer *destroyer,
2284 bool useEHCleanupForArray,
2285 const VarDecl *VD);
2286 void emitArrayDestroy(llvm::Value *begin, llvm::Value *end,
2287 QualType elementType, CharUnits elementAlign,
2288 Destroyer *destroyer, bool checkZeroLength,
2289 bool useEHCleanup);
2290
2291 Destroyer *getDestroyer(QualType::DestructionKind destructionKind);
2292
2293 /// Determines whether an EH cleanup is required to destroy a type
2294 /// with the given destruction kind.
2295 bool needsEHCleanup(QualType::DestructionKind kind) {
2296 switch (kind) {
2297 case QualType::DK_none:
2298 return false;
2299 case QualType::DK_cxx_destructor:
2300 case QualType::DK_objc_weak_lifetime:
2301 case QualType::DK_nontrivial_c_struct:
2302 return getLangOpts().Exceptions;
2303 case QualType::DK_objc_strong_lifetime:
2304 return getLangOpts().Exceptions &&
2305 CGM.getCodeGenOpts().ObjCAutoRefCountExceptions;
2306 }
2307 llvm_unreachable("bad destruction kind");
2308 }
2309
2310 CleanupKind getCleanupKind(QualType::DestructionKind kind) {
2311 return (needsEHCleanup(kind) ? NormalAndEHCleanup : NormalCleanup);
2312 }
2313
2314 //===--------------------------------------------------------------------===//
2315 // Objective-C
2316 //===--------------------------------------------------------------------===//
2317
2318 void GenerateObjCMethod(const ObjCMethodDecl *OMD);
2319
2320 void StartObjCMethod(const ObjCMethodDecl *MD, const ObjCContainerDecl *CD);
2321
2322 /// GenerateObjCGetter - Synthesize an Objective-C property getter function.
2323 void GenerateObjCGetter(ObjCImplementationDecl *IMP,
2324 const ObjCPropertyImplDecl *PID);
2325 void generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
2326 const ObjCPropertyImplDecl *propImpl,
2327 const ObjCMethodDecl *GetterMothodDecl,
2328 llvm::Constant *AtomicHelperFn);
2329
2330 void GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
2331 ObjCMethodDecl *MD, bool ctor);
2332
2333 /// GenerateObjCSetter - Synthesize an Objective-C property setter function
2334 /// for the given property.
2335 void GenerateObjCSetter(ObjCImplementationDecl *IMP,
2336 const ObjCPropertyImplDecl *PID);
2337 void generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
2338 const ObjCPropertyImplDecl *propImpl,
2339 llvm::Constant *AtomicHelperFn);
2340
2341 //===--------------------------------------------------------------------===//
2342 // Block Bits
2343 //===--------------------------------------------------------------------===//
2344
2345 /// Emit block literal.
2346 /// \return an LLVM value which is a pointer to a struct which contains
2347 /// information about the block, including the block invoke function, the
2348 /// captured variables, etc.
2349 llvm::Value *EmitBlockLiteral(const BlockExpr *);
2350
2351 llvm::Function *GenerateBlockFunction(GlobalDecl GD, const CGBlockInfo &Info,
2352 const DeclMapTy &ldm,
2353 bool IsLambdaConversionToBlock,
2354 bool BuildGlobalBlock);
2355
2356 /// Check if \p T is a C++ class that has a destructor that can throw.
2357 static bool cxxDestructorCanThrow(QualType T);
2358
2359 llvm::Constant *GenerateCopyHelperFunction(const CGBlockInfo &blockInfo);
2360 llvm::Constant *GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo);
2361 llvm::Constant *
2362 GenerateObjCAtomicSetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);
2363 llvm::Constant *
2364 GenerateObjCAtomicGetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);
2365 llvm::Value *EmitBlockCopyAndAutorelease(llvm::Value *Block, QualType Ty);
2366
2367 void BuildBlockRelease(llvm::Value *DeclPtr, BlockFieldFlags flags,
2368 bool CanThrow);
2369
2370 class AutoVarEmission;
2371
2372 void emitByrefStructureInit(const AutoVarEmission &emission);
2373
2374 /// Enter a cleanup to destroy a __block variable. Note that this
2375 /// cleanup should be a no-op if the variable hasn't left the stack
2376 /// yet; if a cleanup is required for the variable itself, that needs
2377 /// to be done externally.
2378 ///
2379 /// \param Kind Cleanup kind.
2380 ///
2381 /// \param Addr When \p LoadBlockVarAddr is false, the address of the __block
2382 /// structure that will be passed to _Block_object_dispose. When
2383 /// \p LoadBlockVarAddr is true, the address of the field of the block
2384 /// structure that holds the address of the __block structure.
2385 ///
2386 /// \param Flags The flag that will be passed to _Block_object_dispose.
2387 ///
2388 /// \param LoadBlockVarAddr Indicates whether we need to emit a load from
2389 /// \p Addr to get the address of the __block structure.
2390 void enterByrefCleanup(CleanupKind Kind, Address Addr, BlockFieldFlags Flags,
2391 bool LoadBlockVarAddr, bool CanThrow);
2392
2393 void setBlockContextParameter(const ImplicitParamDecl *D, unsigned argNum,
2394 llvm::Value *ptr);
2395
2396 Address LoadBlockStruct();
2397 Address GetAddrOfBlockDecl(const VarDecl *var);
2398
2399 /// BuildBlockByrefAddress - Computes the location of the
2400 /// data in a variable which is declared as __block.
2401 Address emitBlockByrefAddress(Address baseAddr, const VarDecl *V,
2402 bool followForward = true);
2403 Address emitBlockByrefAddress(Address baseAddr, const BlockByrefInfo &info,
2404 bool followForward, const llvm::Twine &name);
2405
2406 const BlockByrefInfo &getBlockByrefInfo(const VarDecl *var);
2407
2408 QualType BuildFunctionArgList(GlobalDecl GD, FunctionArgList &Args);
2409
2410 void GenerateCode(GlobalDecl GD, llvm::Function *Fn,
2411 const CGFunctionInfo &FnInfo);
2412
2413 /// Annotate the function with an attribute that disables TSan checking at
2414 /// runtime.
2415 void markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn);
2416
2417 /// Emit code for the start of a function.
2418 /// \param Loc The location to be associated with the function.
2419 /// \param StartLoc The location of the function body.
2420 void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn,
2421 const CGFunctionInfo &FnInfo, const FunctionArgList &Args,
2422 SourceLocation Loc = SourceLocation(),
2423 SourceLocation StartLoc = SourceLocation());
2424
2425 static bool IsConstructorDelegationValid(const CXXConstructorDecl *Ctor);
2426
2427 void EmitConstructorBody(FunctionArgList &Args);
2428 void EmitDestructorBody(FunctionArgList &Args);
2429 void emitImplicitAssignmentOperatorBody(FunctionArgList &Args);
2430 void EmitFunctionBody(const Stmt *Body);
2431 void EmitBlockWithFallThrough(llvm::BasicBlock *BB, const Stmt *S);
2432
2433 void EmitForwardingCallToLambda(const CXXMethodDecl *LambdaCallOperator,
2434 CallArgList &CallArgs,
2435 const CGFunctionInfo *CallOpFnInfo = nullptr,
2436 llvm::Constant *CallOpFn = nullptr);
2437 void EmitLambdaBlockInvokeBody();
2438 void EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD);
2439 void EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,
2440 CallArgList &CallArgs);
2441 void EmitLambdaInAllocaImplFn(const CXXMethodDecl *CallOp,
2442 const CGFunctionInfo **ImplFnInfo,
2443 llvm::Function **ImplFn);
2444 void EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD);
2445 void EmitLambdaVLACapture(const VariableArrayType *VAT, LValue LV) {
2446 EmitStoreThroughLValue(Src: RValue::get(V: VLASizeMap[VAT->getSizeExpr()]), Dst: LV);
2447 }
2448 void EmitAsanPrologueOrEpilogue(bool Prologue);
2449
2450 /// Emit the unified return block, trying to avoid its emission when
2451 /// possible.
2452 /// \return The debug location of the user written return statement if the
2453 /// return block is avoided.
2454 llvm::DebugLoc EmitReturnBlock();
2455
2456 /// FinishFunction - Complete IR generation of the current function. It is
2457 /// legal to call this function even if there is no current insertion point.
2458 void FinishFunction(SourceLocation EndLoc = SourceLocation());
2459
2460 void StartThunk(llvm::Function *Fn, GlobalDecl GD,
2461 const CGFunctionInfo &FnInfo, bool IsUnprototyped);
2462
2463 void EmitCallAndReturnForThunk(llvm::FunctionCallee Callee,
2464 const ThunkInfo *Thunk, bool IsUnprototyped);
2465
2466 void FinishThunk();
2467
2468 /// Start an Objective-C direct method thunk.
2469 void StartObjCDirectPreconditionThunk(const ObjCMethodDecl *OMD,
2470 llvm::Function *Fn,
2471 const CGFunctionInfo &FI);
2472
2473 /// Finish an Objective-C direct method thunk.
2474 void FinishObjCDirectPreconditionThunk();
2475
2476 /// Emit a musttail call for a thunk with a potentially adjusted this pointer.
2477 void EmitMustTailThunk(GlobalDecl GD, llvm::Value *AdjustedThisPtr,
2478 llvm::FunctionCallee Callee);
2479
2480 /// Generate a thunk for the given method.
2481 void generateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo,
2482 GlobalDecl GD, const ThunkInfo &Thunk,
2483 bool IsUnprototyped);
2484
2485 llvm::Function *GenerateVarArgsThunk(llvm::Function *Fn,
2486 const CGFunctionInfo &FnInfo,
2487 GlobalDecl GD, const ThunkInfo &Thunk);
2488
2489 void EmitCtorPrologue(const CXXConstructorDecl *CD, CXXCtorType Type,
2490 FunctionArgList &Args);
2491
2492 void EmitInitializerForField(FieldDecl *Field, LValue LHS, Expr *Init);
2493
2494 /// Struct with all information about dynamic [sub]class needed to set vptr.
2495 struct VPtr {
2496 BaseSubobject Base;
2497 const CXXRecordDecl *NearestVBase;
2498 CharUnits OffsetFromNearestVBase;
2499 const CXXRecordDecl *VTableClass;
2500 };
2501
2502 /// Initialize the vtable pointer of the given subobject.
2503 void InitializeVTablePointer(const VPtr &vptr);
2504
2505 typedef llvm::SmallVector<VPtr, 4> VPtrsVector;
2506
2507 typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
2508 VPtrsVector getVTablePointers(const CXXRecordDecl *VTableClass);
2509
2510 void getVTablePointers(BaseSubobject Base, const CXXRecordDecl *NearestVBase,
2511 CharUnits OffsetFromNearestVBase,
2512 bool BaseIsNonVirtualPrimaryBase,
2513 const CXXRecordDecl *VTableClass,
2514 VisitedVirtualBasesSetTy &VBases, VPtrsVector &vptrs);
2515
2516 void InitializeVTablePointers(const CXXRecordDecl *ClassDecl);
2517
2518 // VTableTrapMode - whether we guarantee that loading the
2519 // vtable is guaranteed to trap on authentication failure,
2520 // even if the resulting vtable pointer is unused.
2521 enum class VTableAuthMode {
2522 Authenticate,
2523 MustTrap,
2524 UnsafeUbsanStrip // Should only be used for Vptr UBSan check
2525 };
2526 /// GetVTablePtr - Return the Value of the vtable pointer member pointed
2527 /// to by This.
2528 llvm::Value *
2529 GetVTablePtr(Address This, llvm::Type *VTableTy,
2530 const CXXRecordDecl *VTableClass,
2531 VTableAuthMode AuthMode = VTableAuthMode::Authenticate);
2532
2533 enum CFITypeCheckKind {
2534 CFITCK_VCall,
2535 CFITCK_NVCall,
2536 CFITCK_DerivedCast,
2537 CFITCK_UnrelatedCast,
2538 CFITCK_ICall,
2539 CFITCK_NVMFCall,
2540 CFITCK_VMFCall,
2541 };
2542
2543 /// Derived is the presumed address of an object of type T after a
2544 /// cast. If T is a polymorphic class type, emit a check that the virtual
2545 /// table for Derived belongs to a class derived from T.
2546 void EmitVTablePtrCheckForCast(QualType T, Address Derived, bool MayBeNull,
2547 CFITypeCheckKind TCK, SourceLocation Loc);
2548
2549 /// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable.
2550 /// If vptr CFI is enabled, emit a check that VTable is valid.
2551 void EmitVTablePtrCheckForCall(const CXXRecordDecl *RD, llvm::Value *VTable,
2552 CFITypeCheckKind TCK, SourceLocation Loc);
2553
2554 /// EmitVTablePtrCheck - Emit a check that VTable is a valid virtual table for
2555 /// RD using llvm.type.test.
2556 void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable,
2557 CFITypeCheckKind TCK, SourceLocation Loc);
2558
2559 /// If whole-program virtual table optimization is enabled, emit an assumption
2560 /// that VTable is a member of RD's type identifier. Or, if vptr CFI is
2561 /// enabled, emit a check that VTable is a member of RD's type identifier.
2562 void EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2563 llvm::Value *VTable, SourceLocation Loc);
2564
2565 /// Returns whether we should perform a type checked load when loading a
2566 /// virtual function for virtual calls to members of RD. This is generally
2567 /// true when both vcall CFI and whole-program-vtables are enabled.
2568 bool ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD);
2569
2570 /// Emit a type checked load from the given vtable.
2571 llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD,
2572 llvm::Value *VTable,
2573 llvm::Type *VTableTy,
2574 uint64_t VTableByteOffset);
2575
2576 /// EnterDtorCleanups - Enter the cleanups necessary to complete the
2577 /// given phase of destruction for a destructor. The end result
2578 /// should call destructors on members and base classes in reverse
2579 /// order of their construction.
2580 void EnterDtorCleanups(const CXXDestructorDecl *Dtor, CXXDtorType Type);
2581
2582 /// ShouldInstrumentFunction - Return true if the current function should be
2583 /// instrumented with __cyg_profile_func_* calls
2584 bool ShouldInstrumentFunction();
2585
2586 /// ShouldSkipSanitizerInstrumentation - Return true if the current function
2587 /// should not be instrumented with sanitizers.
2588 bool ShouldSkipSanitizerInstrumentation();
2589
2590 /// ShouldXRayInstrument - Return true if the current function should be
2591 /// instrumented with XRay nop sleds.
2592 bool ShouldXRayInstrumentFunction() const;
2593
2594 /// AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit
2595 /// XRay custom event handling calls.
2596 bool AlwaysEmitXRayCustomEvents() const;
2597
2598 /// AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit
2599 /// XRay typed event handling calls.
2600 bool AlwaysEmitXRayTypedEvents() const;
2601
2602 /// Return a type hash constant for a function instrumented by
2603 /// -fsanitize=function.
2604 llvm::ConstantInt *getUBSanFunctionTypeHash(QualType T) const;
2605
2606 /// EmitFunctionProlog - Emit the target specific LLVM code to load the
2607 /// arguments for the given function. This is also responsible for naming the
2608 /// LLVM function arguments.
2609 void EmitFunctionProlog(const CGFunctionInfo &FI, llvm::Function *Fn,
2610 const FunctionArgList &Args);
2611
2612 /// EmitFunctionEpilog - Emit the target specific LLVM code to return the
2613 /// given temporary. Specify the source location atom group (Key Instructions
2614 /// debug info feature) for the `ret` using \p RetKeyInstructionsSourceAtom.
2615 /// If it's 0, the `ret` will get added to a new source atom group.
2616 void EmitFunctionEpilog(const CGFunctionInfo &FI, bool EmitRetDbgLoc,
2617 SourceLocation EndLoc,
2618 uint64_t RetKeyInstructionsSourceAtom);
2619
2620 /// Emit a test that checks if the return value \p RV is nonnull.
2621 void EmitReturnValueCheck(llvm::Value *RV);
2622
2623 /// EmitStartEHSpec - Emit the start of the exception spec.
2624 void EmitStartEHSpec(const Decl *D);
2625
2626 /// EmitEndEHSpec - Emit the end of the exception spec.
2627 void EmitEndEHSpec(const Decl *D);
2628
2629 /// getTerminateLandingPad - Return a landing pad that just calls terminate.
2630 llvm::BasicBlock *getTerminateLandingPad();
2631
2632 /// getTerminateLandingPad - Return a cleanup funclet that just calls
2633 /// terminate.
2634 llvm::BasicBlock *getTerminateFunclet();
2635
2636 /// getTerminateHandler - Return a handler (not a landing pad, just
2637 /// a catch handler) that just calls terminate. This is used when
2638 /// a terminate scope encloses a try.
2639 llvm::BasicBlock *getTerminateHandler();
2640
2641 llvm::Type *ConvertTypeForMem(QualType T);
2642 llvm::Type *ConvertType(QualType T);
2643 llvm::Type *convertTypeForLoadStore(QualType ASTTy,
2644 llvm::Type *LLVMTy = nullptr);
2645 llvm::Type *ConvertType(const TypeDecl *T) {
2646 return ConvertType(T: getContext().getTypeDeclType(Decl: T));
2647 }
2648
2649 /// LoadObjCSelf - Load the value of self. This function is only valid while
2650 /// generating code for an Objective-C method.
2651 llvm::Value *LoadObjCSelf();
2652
2653 /// TypeOfSelfObject - Return type of object that this self represents.
2654 QualType TypeOfSelfObject();
2655
2656 /// getEvaluationKind - Return the TypeEvaluationKind of QualType \c T.
2657 static TypeEvaluationKind getEvaluationKind(QualType T);
2658
2659 static bool hasScalarEvaluationKind(QualType T) {
2660 return getEvaluationKind(T) == TEK_Scalar;
2661 }
2662
2663 static bool hasAggregateEvaluationKind(QualType T) {
2664 return getEvaluationKind(T) == TEK_Aggregate;
2665 }
2666
2667 /// createBasicBlock - Create an LLVM basic block.
2668 llvm::BasicBlock *createBasicBlock(const Twine &name = "",
2669 llvm::Function *parent = nullptr,
2670 llvm::BasicBlock *before = nullptr) {
2671 return llvm::BasicBlock::Create(Context&: getLLVMContext(), Name: name, Parent: parent, InsertBefore: before);
2672 }
2673
2674 /// getBasicBlockForLabel - Return the LLVM basicblock that the specified
2675 /// label maps to.
2676 JumpDest getJumpDestForLabel(const LabelDecl *S);
2677
2678 /// SimplifyForwardingBlocks - If the given basic block is only a branch to
2679 /// another basic block, simplify it. This assumes that no other code could
2680 /// potentially reference the basic block.
2681 void SimplifyForwardingBlocks(llvm::BasicBlock *BB);
2682
2683 /// EmitBlock - Emit the given block \arg BB and set it as the insert point,
2684 /// adding a fall-through branch from the current insert block if
2685 /// necessary. It is legal to call this function even if there is no current
2686 /// insertion point.
2687 ///
2688 /// IsFinished - If true, indicates that the caller has finished emitting
2689 /// branches to the given block and does not expect to emit code into it. This
2690 /// means the block can be ignored if it is unreachable.
2691 void EmitBlock(llvm::BasicBlock *BB, bool IsFinished = false);
2692
2693 /// EmitBlockAfterUses - Emit the given block somewhere hopefully
2694 /// near its uses, and leave the insertion point in it.
2695 void EmitBlockAfterUses(llvm::BasicBlock *BB);
2696
2697 /// EmitBranch - Emit a branch to the specified basic block from the current
2698 /// insert block, taking care to avoid creation of branches from dummy
2699 /// blocks. It is legal to call this function even if there is no current
2700 /// insertion point.
2701 ///
2702 /// This function clears the current insertion point. The caller should follow
2703 /// calls to this function with calls to Emit*Block prior to generation new
2704 /// code.
2705 void EmitBranch(llvm::BasicBlock *Block);
2706
2707 /// HaveInsertPoint - True if an insertion point is defined. If not, this
2708 /// indicates that the current code being emitted is unreachable.
2709 bool HaveInsertPoint() const { return Builder.GetInsertBlock() != nullptr; }
2710
2711 /// EnsureInsertPoint - Ensure that an insertion point is defined so that
2712 /// emitted IR has a place to go. Note that by definition, if this function
2713 /// creates a block then that block is unreachable; callers may do better to
2714 /// detect when no insertion point is defined and simply skip IR generation.
2715 void EnsureInsertPoint() {
2716 if (!HaveInsertPoint())
2717 EmitBlock(BB: createBasicBlock());
2718 }
2719
2720 /// ErrorUnsupported - Print out an error that codegen doesn't support the
2721 /// specified stmt yet.
2722 void ErrorUnsupported(const Stmt *S, const char *Type);
2723
2724 //===--------------------------------------------------------------------===//
2725 // Helpers
2726 //===--------------------------------------------------------------------===//
2727
2728 Address mergeAddressesInConditionalExpr(Address LHS, Address RHS,
2729 llvm::BasicBlock *LHSBlock,
2730 llvm::BasicBlock *RHSBlock,
2731 llvm::BasicBlock *MergeBlock,
2732 QualType MergedType) {
2733 Builder.SetInsertPoint(MergeBlock);
2734 llvm::PHINode *PtrPhi = Builder.CreatePHI(Ty: LHS.getType(), NumReservedValues: 2, Name: "cond");
2735 PtrPhi->addIncoming(V: LHS.getBasePointer(), BB: LHSBlock);
2736 PtrPhi->addIncoming(V: RHS.getBasePointer(), BB: RHSBlock);
2737 LHS.replaceBasePointer(P: PtrPhi);
2738 LHS.setAlignment(std::min(a: LHS.getAlignment(), b: RHS.getAlignment()));
2739 return LHS;
2740 }
2741
2742 /// Construct an address with the natural alignment of T. If a pointer to T
2743 /// is expected to be signed, the pointer passed to this function must have
2744 /// been signed, and the returned Address will have the pointer authentication
2745 /// information needed to authenticate the signed pointer.
2746 Address makeNaturalAddressForPointer(
2747 llvm::Value *Ptr, QualType T, CharUnits Alignment = CharUnits::Zero(),
2748 bool ForPointeeType = false, LValueBaseInfo *BaseInfo = nullptr,
2749 TBAAAccessInfo *TBAAInfo = nullptr,
2750 KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
2751 if (Alignment.isZero())
2752 Alignment =
2753 CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, forPointeeType: ForPointeeType);
2754 return Address(Ptr, ConvertTypeForMem(T), Alignment,
2755 CGM.getPointerAuthInfoForPointeeType(type: T), /*Offset=*/nullptr,
2756 IsKnownNonNull);
2757 }
2758
2759 LValue MakeAddrLValue(Address Addr, QualType T,
2760 AlignmentSource Source = AlignmentSource::Type) {
2761 return MakeAddrLValue(Addr, T, BaseInfo: LValueBaseInfo(Source),
2762 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2763 }
2764
2765 LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo,
2766 TBAAAccessInfo TBAAInfo) {
2767 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo, TBAAInfo);
2768 }
2769
2770 LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2771 AlignmentSource Source = AlignmentSource::Type) {
2772 return MakeAddrLValue(Addr: makeNaturalAddressForPointer(Ptr: V, T, Alignment), T,
2773 BaseInfo: LValueBaseInfo(Source), TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2774 }
2775
2776 /// Same as MakeAddrLValue above except that the pointer is known to be
2777 /// unsigned.
2778 LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2779 AlignmentSource Source = AlignmentSource::Type) {
2780 Address Addr(V, ConvertTypeForMem(T), Alignment);
2781 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2782 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2783 }
2784
2785 LValue
2786 MakeAddrLValueWithoutTBAA(Address Addr, QualType T,
2787 AlignmentSource Source = AlignmentSource::Type) {
2788 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2789 TBAAInfo: TBAAAccessInfo());
2790 }
2791
2792 /// Given a value of type T* that may not be to a complete object, construct
2793 /// an l-value with the natural pointee alignment of T.
2794 LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T);
2795
2796 LValue
2797 MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
2798 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
2799
2800 /// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known
2801 /// to be unsigned.
2802 LValue MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, QualType T);
2803
2804 LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T);
2805
2806 Address EmitLoadOfReference(LValue RefLVal,
2807 LValueBaseInfo *PointeeBaseInfo = nullptr,
2808 TBAAAccessInfo *PointeeTBAAInfo = nullptr);
2809 LValue EmitLoadOfReferenceLValue(LValue RefLVal);
2810 LValue
2811 EmitLoadOfReferenceLValue(Address RefAddr, QualType RefTy,
2812 AlignmentSource Source = AlignmentSource::Type) {
2813 LValue RefLVal = MakeAddrLValue(Addr: RefAddr, T: RefTy, BaseInfo: LValueBaseInfo(Source),
2814 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: RefTy));
2815 return EmitLoadOfReferenceLValue(RefLVal);
2816 }
2817
2818 /// Load a pointer with type \p PtrTy stored at address \p Ptr.
2819 /// Note that \p PtrTy is the type of the loaded pointer, not the addresses
2820 /// it is loaded from.
2821 Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy,
2822 LValueBaseInfo *BaseInfo = nullptr,
2823 TBAAAccessInfo *TBAAInfo = nullptr);
2824 LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy);
2825
2826private:
2827 struct AllocaTracker {
2828 void Add(llvm::AllocaInst *I) { Allocas.push_back(Elt: I); }
2829 llvm::SmallVector<llvm::AllocaInst *> Take() { return std::move(Allocas); }
2830
2831 private:
2832 llvm::SmallVector<llvm::AllocaInst *> Allocas;
2833 };
2834 AllocaTracker *Allocas = nullptr;
2835
2836 /// CGDecl helper.
2837 void emitStoresForConstant(const VarDecl &D, Address Loc, bool isVolatile,
2838 llvm::Constant *constant, bool IsAutoInit);
2839 /// CGDecl helper.
2840 void emitStoresForZeroInit(const VarDecl &D, Address Loc, bool isVolatile);
2841 /// CGDecl helper.
2842 void emitStoresForPatternInit(const VarDecl &D, Address Loc, bool isVolatile);
2843 /// CGDecl helper.
2844 void emitStoresForInitAfterBZero(llvm::Constant *Init, Address Loc,
2845 bool isVolatile, bool IsAutoInit);
2846
2847public:
2848 // Captures all the allocas created during the scope of its RAII object.
2849 struct AllocaTrackerRAII {
2850 AllocaTrackerRAII(CodeGenFunction &CGF)
2851 : CGF(CGF), OldTracker(CGF.Allocas) {
2852 CGF.Allocas = &Tracker;
2853 }
2854 ~AllocaTrackerRAII() { CGF.Allocas = OldTracker; }
2855
2856 llvm::SmallVector<llvm::AllocaInst *> Take() { return Tracker.Take(); }
2857
2858 private:
2859 CodeGenFunction &CGF;
2860 AllocaTracker *OldTracker;
2861 AllocaTracker Tracker;
2862 };
2863
2864private:
2865 /// If \p Alloca is not in the same address space as \p DestLangAS, insert an
2866 /// address space cast and return a new RawAddress based on this value.
2867 RawAddress MaybeCastStackAddressSpace(RawAddress Alloca, LangAS DestLangAS,
2868 llvm::Value *ArraySize = nullptr);
2869
2870public:
2871 /// CreateTempAlloca - This creates an alloca and inserts it into the entry
2872 /// block if \p ArraySize is nullptr, otherwise inserts it at the current
2873 /// insertion point of the builder. The caller is responsible for setting an
2874 /// appropriate alignment on the alloca.
2875 ///
2876 /// \p ArraySize is the number of array elements to be allocated if it
2877 /// is not nullptr.
2878 ///
2879 /// LangAS::Default is the address space of pointers to local variables and
2880 /// temporaries, as exposed in the source language. In certain
2881 /// configurations, this is not the same as the alloca address space, and a
2882 /// cast is needed to lift the pointer from the alloca AS into
2883 /// LangAS::Default. This can happen when the target uses a restricted
2884 /// address space for the stack but the source language requires
2885 /// LangAS::Default to be a generic address space. The latter condition is
2886 /// common for most programming languages; OpenCL is an exception in that
2887 /// LangAS::Default is the private address space, which naturally maps
2888 /// to the stack.
2889 ///
2890 /// Because the address of a temporary is often exposed to the program in
2891 /// various ways, this function will perform the cast. The original alloca
2892 /// instruction is returned through \p Alloca if it is not nullptr.
2893 ///
2894 /// The cast is not performed in CreateTempAllocaWithoutCast. This is
2895 /// more efficient if the caller knows that the address will not be exposed.
2896 llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp",
2897 llvm::Value *ArraySize = nullptr);
2898
2899 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
2900 /// block. The alloca is casted to the address space of \p UseAddrSpace if
2901 /// necessary.
2902 RawAddress CreateTempAlloca(llvm::Type *Ty, LangAS UseAddrSpace,
2903 CharUnits align, const Twine &Name = "tmp",
2904 llvm::Value *ArraySize = nullptr,
2905 RawAddress *Alloca = nullptr);
2906
2907 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
2908 /// block. The alloca is casted to default address space if necessary.
2909 ///
2910 /// FIXME: This version should be removed, and context should provide the
2911 /// context use address space used instead of default.
2912 RawAddress CreateTempAlloca(llvm::Type *Ty, CharUnits align,
2913 const Twine &Name = "tmp",
2914 llvm::Value *ArraySize = nullptr,
2915 RawAddress *Alloca = nullptr) {
2916 return CreateTempAlloca(Ty, UseAddrSpace: LangAS::Default, align, Name, ArraySize,
2917 Alloca);
2918 }
2919
2920 RawAddress CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align,
2921 const Twine &Name = "tmp",
2922 llvm::Value *ArraySize = nullptr);
2923
2924 /// CreateDefaultAlignedTempAlloca - This creates an alloca with the
2925 /// default ABI alignment of the given LLVM type.
2926 ///
2927 /// IMPORTANT NOTE: This is *not* generally the right alignment for
2928 /// any given AST type that happens to have been lowered to the
2929 /// given IR type. This should only ever be used for function-local,
2930 /// IR-driven manipulations like saving and restoring a value. Do
2931 /// not hand this address off to arbitrary IRGen routines, and especially
2932 /// do not pass it as an argument to a function that might expect a
2933 /// properly ABI-aligned value.
2934 RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty,
2935 const Twine &Name = "tmp");
2936
2937 /// CreateIRTempWithoutCast - Create a temporary IR object of the given type,
2938 /// with appropriate alignment. This routine should only be used when an
2939 /// temporary value needs to be stored into an alloca (for example, to avoid
2940 /// explicit PHI construction), but the type is the IR type, not the type
2941 /// appropriate for storing in memory.
2942 ///
2943 /// That is, this is exactly equivalent to CreateMemTemp, but calling
2944 /// ConvertType instead of ConvertTypeForMem.
2945 RawAddress CreateIRTempWithoutCast(QualType T, const Twine &Name = "tmp");
2946
2947 /// CreateMemTemp - Create a temporary memory object of the given type, with
2948 /// appropriate alignmen and cast it to the default address space. Returns
2949 /// the original alloca instruction by \p Alloca if it is not nullptr.
2950 RawAddress CreateMemTemp(QualType T, const Twine &Name = "tmp",
2951 RawAddress *Alloca = nullptr);
2952 RawAddress CreateMemTemp(QualType T, CharUnits Align,
2953 const Twine &Name = "tmp",
2954 RawAddress *Alloca = nullptr);
2955
2956 /// CreateMemTemp - Create a temporary memory object of the given type, with
2957 /// appropriate alignmen without casting it to the default address space.
2958 RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp");
2959 RawAddress CreateMemTempWithoutCast(QualType T, CharUnits Align,
2960 const Twine &Name = "tmp");
2961
2962 /// CreateAggTemp - Create a temporary memory object for the given
2963 /// aggregate type.
2964 AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp",
2965 RawAddress *Alloca = nullptr) {
2966 RawAddress Addr = CreateMemTemp(T, Name, Alloca);
2967 return AggValueSlot::forAddr(
2968 addr: Addr, quals: T.getQualifiers(), isDestructed: AggValueSlot::IsNotDestructed,
2969 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2970 mayOverlap: AggValueSlot::DoesNotOverlap);
2971 }
2972
2973 /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
2974 /// expression and compare the result against zero, returning an Int1Ty value.
2975 llvm::Value *EvaluateExprAsBool(const Expr *E);
2976
2977 /// Retrieve the implicit cast expression of the rhs in a binary operator
2978 /// expression by passing pointers to Value and QualType
2979 /// This is used for implicit bitfield conversion checks, which
2980 /// must compare with the value before potential truncation.
2981 llvm::Value *EmitWithOriginalRHSBitfieldAssignment(const BinaryOperator *E,
2982 llvm::Value **Previous,
2983 QualType *SrcType);
2984
2985 /// Emit a check that an [implicit] conversion of a bitfield. It is not UB,
2986 /// so we use the value after conversion.
2987 void EmitBitfieldConversionCheck(llvm::Value *Src, QualType SrcType,
2988 llvm::Value *Dst, QualType DstType,
2989 const CGBitFieldInfo &Info,
2990 SourceLocation Loc);
2991
2992 /// EmitIgnoredExpr - Emit an expression in a context which ignores the
2993 /// result.
2994 void EmitIgnoredExpr(const Expr *E);
2995
2996 /// EmitAnyExpr - Emit code to compute the specified expression which can have
2997 /// any type. The result is returned as an RValue struct. If this is an
2998 /// aggregate expression, the aggloc/agglocvolatile arguments indicate where
2999 /// the result should be returned.
3000 ///
3001 /// \param ignoreResult True if the resulting value isn't used.
3002 RValue EmitAnyExpr(const Expr *E,
3003 AggValueSlot aggSlot = AggValueSlot::ignored(),
3004 bool ignoreResult = false);
3005
3006 // EmitVAListRef - Emit a "reference" to a va_list; this is either the address
3007 // or the value of the expression, depending on how va_list is defined.
3008 Address EmitVAListRef(const Expr *E);
3009
3010 /// Emit a "reference" to a __builtin_ms_va_list; this is
3011 /// always the value of the expression, because a __builtin_ms_va_list is a
3012 /// pointer to a char.
3013 Address EmitMSVAListRef(const Expr *E);
3014
3015 /// Emit a "reference" to a __builtin_zos_va_list; this is always the
3016 /// address of the expression, because a __builtin_zos_va_list is an
3017 /// array of pointer to a char.
3018 Address EmitZOSVAListRef(const Expr *E);
3019
3020 /// EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will
3021 /// always be accessible even if no aggregate location is provided.
3022 RValue EmitAnyExprToTemp(const Expr *E);
3023
3024 /// EmitAnyExprToMem - Emits the code necessary to evaluate an
3025 /// arbitrary expression into the given memory location.
3026 void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals,
3027 bool IsInitializer);
3028
3029 void EmitAnyExprToExn(const Expr *E, Address Addr);
3030
3031 /// EmitInitializationToLValue - Emit an initializer to an LValue.
3032 void EmitInitializationToLValue(
3033 const Expr *E, LValue LV,
3034 AggValueSlot::IsZeroed_t IsZeroed = AggValueSlot::IsNotZeroed);
3035
3036 /// EmitExprAsInit - Emits the code necessary to initialize a
3037 /// location in memory with the given initializer.
3038 void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue,
3039 bool capturedByInit);
3040
3041 /// hasVolatileMember - returns true if aggregate type has a volatile
3042 /// member.
3043 bool hasVolatileMember(QualType T) {
3044 if (const auto *RD = T->getAsRecordDecl())
3045 return RD->hasVolatileMember();
3046 return false;
3047 }
3048
3049 /// Determine whether a return value slot may overlap some other object.
3050 AggValueSlot::Overlap_t getOverlapForReturnValue() {
3051 // FIXME: Assuming no overlap here breaks guaranteed copy elision for base
3052 // class subobjects. These cases may need to be revisited depending on the
3053 // resolution of the relevant core issue.
3054 return AggValueSlot::DoesNotOverlap;
3055 }
3056
3057 /// Determine whether a field initialization may overlap some other object.
3058 AggValueSlot::Overlap_t getOverlapForFieldInit(const FieldDecl *FD);
3059
3060 /// Determine whether a base class initialization may overlap some other
3061 /// object.
3062 AggValueSlot::Overlap_t getOverlapForBaseInit(const CXXRecordDecl *RD,
3063 const CXXRecordDecl *BaseRD,
3064 bool IsVirtual);
3065
3066 /// Emit an aggregate assignment.
3067 void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy) {
3068 ApplyAtomGroup Grp(getDebugInfo());
3069 bool IsVolatile = hasVolatileMember(T: EltTy);
3070 EmitAggregateCopy(Dest, Src, EltTy, MayOverlap: AggValueSlot::MayOverlap, isVolatile: IsVolatile);
3071 }
3072
3073 void EmitAggregateCopyCtor(LValue Dest, LValue Src,
3074 AggValueSlot::Overlap_t MayOverlap) {
3075 EmitAggregateCopy(Dest, Src, EltTy: Src.getType(), MayOverlap);
3076 }
3077
3078 /// EmitAggregateCopy - Emit an aggregate copy.
3079 ///
3080 /// \param isVolatile \c true iff either the source or the destination is
3081 /// volatile.
3082 /// \param MayOverlap Whether the tail padding of the destination might be
3083 /// occupied by some other object. More efficient code can often be
3084 /// generated if not.
3085 void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy,
3086 AggValueSlot::Overlap_t MayOverlap,
3087 bool isVolatile = false);
3088
3089 /// GetAddrOfLocalVar - Return the address of a local variable.
3090 Address GetAddrOfLocalVar(const VarDecl *VD) {
3091 auto it = LocalDeclMap.find(Val: VD);
3092 assert(it != LocalDeclMap.end() &&
3093 "Invalid argument to GetAddrOfLocalVar(), no decl!");
3094 return it->second;
3095 }
3096
3097 /// Given an opaque value expression, return its LValue mapping if it exists,
3098 /// otherwise create one.
3099 LValue getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e);
3100
3101 /// Given an opaque value expression, return its RValue mapping if it exists,
3102 /// otherwise create one.
3103 RValue getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e);
3104
3105 /// isOpaqueValueEmitted - Return true if the opaque value expression has
3106 /// already been emitted.
3107 bool isOpaqueValueEmitted(const OpaqueValueExpr *E);
3108
3109 /// Get the index of the current ArrayInitLoopExpr, if any.
3110 llvm::Value *getArrayInitIndex() { return ArrayInitIndex; }
3111
3112 /// getAccessedFieldNo - Given an encoded value and a result number, return
3113 /// the input field number being accessed.
3114 static unsigned getAccessedFieldNo(unsigned Idx, const llvm::Constant *Elts);
3115
3116 llvm::BlockAddress *GetAddrOfLabel(const LabelDecl *L);
3117 llvm::BasicBlock *GetIndirectGotoBlock();
3118
3119 /// Check if \p E is a C++ "this" pointer wrapped in value-preserving casts.
3120 static bool IsWrappedCXXThis(const Expr *E);
3121
3122 /// EmitNullInitialization - Generate code to set a value of the given type to
3123 /// null, If the type contains data member pointers, they will be initialized
3124 /// to -1 in accordance with the Itanium C++ ABI.
3125 void EmitNullInitialization(Address DestPtr, QualType Ty);
3126
3127 /// Emits a call to an LLVM variable-argument intrinsic, either
3128 /// \c llvm.va_start or \c llvm.va_end.
3129 /// \param ArgValue A reference to the \c va_list as emitted by either
3130 /// \c EmitVAListRef or \c EmitMSVAListRef.
3131 /// \param IsStart If \c true, emits a call to \c llvm.va_start; otherwise,
3132 /// calls \c llvm.va_end.
3133 llvm::Value *EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart);
3134
3135 /// Generate code to get an argument from the passed in pointer
3136 /// and update it accordingly.
3137 /// \param VE The \c VAArgExpr for which to generate code.
3138 /// \param VAListAddr Receives a reference to the \c va_list as emitted by
3139 /// either \c EmitVAListRef or \c EmitMSVAListRef.
3140 /// \returns A pointer to the argument.
3141 // FIXME: We should be able to get rid of this method and use the va_arg
3142 // instruction in LLVM instead once it works well enough.
3143 RValue EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
3144 AggValueSlot Slot = AggValueSlot::ignored());
3145
3146 /// emitArrayLength - Compute the length of an array, even if it's a
3147 /// VLA, and drill down to the base element type.
3148 llvm::Value *emitArrayLength(const ArrayType *arrayType, QualType &baseType,
3149 Address &addr);
3150
3151 /// EmitVLASize - Capture all the sizes for the VLA expressions in
3152 /// the given variably-modified type and store them in the VLASizeMap.
3153 ///
3154 /// This function can be called with a null (unreachable) insert point.
3155 void EmitVariablyModifiedType(QualType Ty);
3156
3157 struct VlaSizePair {
3158 llvm::Value *NumElts;
3159 QualType Type;
3160
3161 VlaSizePair(llvm::Value *NE, QualType T) : NumElts(NE), Type(T) {}
3162 };
3163
3164 /// Return the number of elements for a single dimension
3165 /// for the given array type.
3166 VlaSizePair getVLAElements1D(const VariableArrayType *vla);
3167 VlaSizePair getVLAElements1D(QualType vla);
3168
3169 /// Returns an LLVM value that corresponds to the size,
3170 /// in non-variably-sized elements, of a variable length array type,
3171 /// plus that largest non-variably-sized element type. Assumes that
3172 /// the type has already been emitted with EmitVariablyModifiedType.
3173 VlaSizePair getVLASize(const VariableArrayType *vla);
3174 VlaSizePair getVLASize(QualType vla);
3175
3176 /// LoadCXXThis - Load the value of 'this'. This function is only valid while
3177 /// generating code for an C++ member function.
3178 llvm::Value *LoadCXXThis() {
3179 assert(CXXThisValue && "no 'this' value for this function");
3180 return CXXThisValue;
3181 }
3182 Address LoadCXXThisAddress();
3183
3184 /// LoadCXXVTT - Load the VTT parameter to base constructors/destructors have
3185 /// virtual bases.
3186 // FIXME: Every place that calls LoadCXXVTT is something
3187 // that needs to be abstracted properly.
3188 llvm::Value *LoadCXXVTT() {
3189 assert(CXXStructorImplicitParamValue && "no VTT value for this function");
3190 return CXXStructorImplicitParamValue;
3191 }
3192
3193 /// GetAddressOfBaseOfCompleteClass - Convert the given pointer to a
3194 /// complete class to the given direct base.
3195 Address GetAddressOfDirectBaseInCompleteClass(Address Value,
3196 const CXXRecordDecl *Derived,
3197 const CXXRecordDecl *Base,
3198 bool BaseIsVirtual);
3199
3200 static bool ShouldNullCheckClassCastValue(const CastExpr *Cast);
3201
3202 /// GetAddressOfBaseClass - This function will add the necessary delta to the
3203 /// load of 'this' and returns address of the base class.
3204 Address GetAddressOfBaseClass(Address Value, const CXXRecordDecl *Derived,
3205 CastExpr::path_const_iterator PathBegin,
3206 CastExpr::path_const_iterator PathEnd,
3207 bool NullCheckValue, SourceLocation Loc);
3208
3209 Address GetAddressOfDerivedClass(Address Value, const CXXRecordDecl *Derived,
3210 CastExpr::path_const_iterator PathBegin,
3211 CastExpr::path_const_iterator PathEnd,
3212 bool NullCheckValue);
3213
3214 /// GetVTTParameter - Return the VTT parameter that should be passed to a
3215 /// base constructor/destructor with virtual bases.
3216 /// FIXME: VTTs are Itanium ABI-specific, so the definition should move
3217 /// to ItaniumCXXABI.cpp together with all the references to VTT.
3218 llvm::Value *GetVTTParameter(GlobalDecl GD, bool ForVirtualBase,
3219 bool Delegating);
3220
3221 void EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
3222 CXXCtorType CtorType,
3223 const FunctionArgList &Args,
3224 SourceLocation Loc);
3225 // It's important not to confuse this and the previous function. Delegating
3226 // constructors are the C++0x feature. The constructor delegate optimization
3227 // is used to reduce duplication in the base and complete consturctors where
3228 // they are substantially the same.
3229 void EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3230 const FunctionArgList &Args);
3231
3232 /// Emit a call to an inheriting constructor (that is, one that invokes a
3233 /// constructor inherited from a base class) by inlining its definition. This
3234 /// is necessary if the ABI does not support forwarding the arguments to the
3235 /// base class constructor (because they're variadic or similar).
3236 void EmitInlinedInheritingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3237 CXXCtorType CtorType,
3238 bool ForVirtualBase,
3239 bool Delegating,
3240 CallArgList &Args);
3241
3242 /// Emit a call to a constructor inherited from a base class, passing the
3243 /// current constructor's arguments along unmodified (without even making
3244 /// a copy).
3245 void EmitInheritedCXXConstructorCall(const CXXConstructorDecl *D,
3246 bool ForVirtualBase, Address This,
3247 bool InheritedFromVBase,
3248 const CXXInheritedCtorInitExpr *E);
3249
3250 void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3251 bool ForVirtualBase, bool Delegating,
3252 AggValueSlot ThisAVS, const CXXConstructExpr *E);
3253
3254 void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3255 bool ForVirtualBase, bool Delegating,
3256 Address This, CallArgList &Args,
3257 AggValueSlot::Overlap_t Overlap,
3258 SourceLocation Loc, bool NewPointerIsChecked,
3259 llvm::CallBase **CallOrInvoke = nullptr);
3260
3261 /// Emit assumption load for all bases. Requires to be called only on
3262 /// most-derived class and not under construction of the object.
3263 void EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl, Address This);
3264
3265 /// Emit assumption that vptr load == global vtable.
3266 void EmitVTableAssumptionLoad(const VPtr &vptr, Address This);
3267
3268 void EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, Address This,
3269 Address Src, const CXXConstructExpr *E);
3270
3271 void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3272 const ArrayType *ArrayTy, Address ArrayPtr,
3273 const CXXConstructExpr *E,
3274 bool NewPointerIsChecked,
3275 bool ZeroInitialization = false);
3276
3277 void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3278 llvm::Value *NumElements, Address ArrayPtr,
3279 const CXXConstructExpr *E,
3280 bool NewPointerIsChecked,
3281 bool ZeroInitialization = false);
3282
3283 static Destroyer destroyCXXObject;
3284
3285 void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type,
3286 bool ForVirtualBase, bool Delegating, Address This,
3287 QualType ThisTy);
3288
3289 void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
3290 llvm::Type *ElementTy, Address NewPtr,
3291 llvm::Value *NumElements,
3292 llvm::Value *AllocSizeWithoutCookie);
3293
3294 void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
3295 Address Ptr);
3296
3297 void EmitSehCppScopeBegin();
3298 void EmitSehCppScopeEnd();
3299 void EmitSehTryScopeBegin();
3300 void EmitSehTryScopeEnd();
3301
3302 bool EmitLifetimeStart(llvm::Value *Addr);
3303 void EmitLifetimeEnd(llvm::Value *Addr);
3304
3305 llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
3306 void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
3307
3308 void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,
3309 QualType DeleteTy, llvm::Value *NumElements = nullptr,
3310 CharUnits CookieSize = CharUnits(),
3311 llvm::Constant *CalleeOverride = nullptr);
3312
3313 RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
3314 const CallExpr *TheCallExpr, bool IsDelete);
3315
3316 llvm::Value *EmitCXXTypeidExpr(const CXXTypeidExpr *E);
3317 llvm::Value *EmitDynamicCast(Address V, const CXXDynamicCastExpr *DCE);
3318 Address EmitCXXUuidofExpr(const CXXUuidofExpr *E);
3319
3320 /// Situations in which we might emit a check for the suitability of a
3321 /// pointer or glvalue. Needs to be kept in sync with ubsan_handlers.cpp in
3322 /// compiler-rt.
3323 enum TypeCheckKind {
3324 /// Checking the operand of a load. Must be suitably sized and aligned.
3325 TCK_Load,
3326 /// Checking the destination of a store. Must be suitably sized and aligned.
3327 TCK_Store,
3328 /// Checking the bound value in a reference binding. Must be suitably sized
3329 /// and aligned, but is not required to refer to an object (until the
3330 /// reference is used), per core issue 453.
3331 TCK_ReferenceBinding,
3332 /// Checking the object expression in a non-static data member access. Must
3333 /// be an object within its lifetime.
3334 TCK_MemberAccess,
3335 /// Checking the 'this' pointer for a call to a non-static member function.
3336 /// Must be an object within its lifetime.
3337 TCK_MemberCall,
3338 /// Checking the 'this' pointer for a constructor call.
3339 TCK_ConstructorCall,
3340 /// Checking the operand of a static_cast to a derived pointer type. Must be
3341 /// null or an object within its lifetime.
3342 TCK_DowncastPointer,
3343 /// Checking the operand of a static_cast to a derived reference type. Must
3344 /// be an object within its lifetime.
3345 TCK_DowncastReference,
3346 /// Checking the operand of a cast to a base object. Must be suitably sized
3347 /// and aligned.
3348 TCK_Upcast,
3349 /// Checking the operand of a cast to a virtual base object. Must be an
3350 /// object within its lifetime.
3351 TCK_UpcastToVirtualBase,
3352 /// Checking the value assigned to a _Nonnull pointer. Must not be null.
3353 TCK_NonnullAssign,
3354 /// Checking the operand of a dynamic_cast or a typeid expression. Must be
3355 /// null or an object within its lifetime.
3356 TCK_DynamicOperation
3357 };
3358
3359 /// Determine whether the pointer type check \p TCK permits null pointers.
3360 static bool isNullPointerAllowed(TypeCheckKind TCK);
3361
3362 /// Determine whether the pointer type check \p TCK requires a vptr check.
3363 static bool isVptrCheckRequired(TypeCheckKind TCK, QualType Ty);
3364
3365 /// Whether any type-checking sanitizers are enabled. If \c false,
3366 /// calls to EmitTypeCheck can be skipped.
3367 bool sanitizePerformTypeCheck() const;
3368
3369 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV,
3370 QualType Type, SanitizerSet SkippedChecks = SanitizerSet(),
3371 llvm::Value *ArraySize = nullptr) {
3372 if (!sanitizePerformTypeCheck())
3373 return;
3374 EmitTypeCheck(TCK, Loc, V: LV.emitRawPointer(CGF&: *this), Type, Alignment: LV.getAlignment(),
3375 SkippedChecks, ArraySize);
3376 }
3377
3378 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, Address Addr,
3379 QualType Type, CharUnits Alignment = CharUnits::Zero(),
3380 SanitizerSet SkippedChecks = SanitizerSet(),
3381 llvm::Value *ArraySize = nullptr) {
3382 if (!sanitizePerformTypeCheck())
3383 return;
3384 EmitTypeCheck(TCK, Loc, V: Addr.emitRawPointer(CGF&: *this), Type, Alignment,
3385 SkippedChecks, ArraySize);
3386 }
3387
3388 /// Emit a check that \p V is the address of storage of the
3389 /// appropriate size and alignment for an object of type \p Type
3390 /// (or if ArraySize is provided, for an array of that bound).
3391 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Value *V,
3392 QualType Type, CharUnits Alignment = CharUnits::Zero(),
3393 SanitizerSet SkippedChecks = SanitizerSet(),
3394 llvm::Value *ArraySize = nullptr);
3395
3396 /// Emit a check that \p Base points into an array object, which
3397 /// we can access at index \p Index. \p Accessed should be \c false if we
3398 /// this expression is used as an lvalue, for instance in "&Arr[Idx]".
3399 void EmitBoundsCheck(const Expr *ArrayExpr, const Expr *ArrayExprBase,
3400 llvm::Value *Index, QualType IndexType, bool Accessed);
3401 void EmitBoundsCheckImpl(const Expr *ArrayExpr, QualType ArrayBaseType,
3402 llvm::Value *IndexVal, QualType IndexType,
3403 llvm::Value *BoundsVal, QualType BoundsType,
3404 bool Accessed);
3405
3406 /// Returns debug info, with additional annotation if
3407 /// CGM.getCodeGenOpts().SanitizeAnnotateDebugInfo[Ordinal] is enabled for
3408 /// any of the ordinals.
3409 llvm::DILocation *
3410 SanitizerAnnotateDebugInfo(ArrayRef<SanitizerKind::SanitizerOrdinal> Ordinals,
3411 SanitizerHandler Handler);
3412
3413 /// Build metadata used by the AllocToken instrumentation.
3414 llvm::MDNode *buildAllocToken(QualType AllocType);
3415 /// Emit and set additional metadata used by the AllocToken instrumentation.
3416 void EmitAllocToken(llvm::CallBase *CB, QualType AllocType);
3417 /// Build additional metadata used by the AllocToken instrumentation,
3418 /// inferring the type from an allocation call expression.
3419 llvm::MDNode *buildAllocToken(const CallExpr *E);
3420 /// Emit and set additional metadata used by the AllocToken instrumentation,
3421 /// inferring the type from an allocation call expression.
3422 void EmitAllocToken(llvm::CallBase *CB, const CallExpr *E);
3423
3424 llvm::Value *GetCountedByFieldExprGEP(const Expr *Base, const FieldDecl *FD,
3425 const FieldDecl *CountDecl);
3426
3427 /// Build an expression accessing the "counted_by" field.
3428 llvm::Value *EmitLoadOfCountedByField(const Expr *Base, const FieldDecl *FD,
3429 const FieldDecl *CountDecl);
3430
3431 // Emit bounds checking for flexible array and pointer members with the
3432 // counted_by attribute.
3433 void EmitCountedByBoundsChecking(const Expr *ArrayExpr, QualType ArrayType,
3434 Address ArrayInst, QualType IndexType,
3435 llvm::Value *IndexVal, bool Accessed,
3436 bool FlexibleArray);
3437
3438 llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
3439 bool isInc, bool isPre);
3440 ComplexPairTy EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
3441 bool isInc, bool isPre);
3442
3443 /// Converts Location to a DebugLoc, if debug information is enabled.
3444 llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location);
3445
3446 /// Get the record field index as represented in debug info.
3447 unsigned getDebugInfoFIndex(const RecordDecl *Rec, unsigned FieldIndex);
3448
3449 //===--------------------------------------------------------------------===//
3450 // Declaration Emission
3451 //===--------------------------------------------------------------------===//
3452
3453 /// EmitDecl - Emit a declaration.
3454 ///
3455 /// This function can be called with a null (unreachable) insert point.
3456 void EmitDecl(const Decl &D, bool EvaluateConditionDecl = false);
3457
3458 /// EmitVarDecl - Emit a local variable declaration.
3459 ///
3460 /// This function can be called with a null (unreachable) insert point.
3461 void EmitVarDecl(const VarDecl &D);
3462
3463 void EmitScalarInit(const Expr *init, const ValueDecl *D, LValue lvalue,
3464 bool capturedByInit);
3465
3466 typedef void SpecialInitFn(CodeGenFunction &Init, const VarDecl &D,
3467 llvm::Value *Address);
3468
3469 /// Determine whether the given initializer is trivial in the sense
3470 /// that it requires no code to be generated.
3471 bool isTrivialInitializer(const Expr *Init);
3472
3473 /// EmitAutoVarDecl - Emit an auto variable declaration.
3474 ///
3475 /// This function can be called with a null (unreachable) insert point.
3476 void EmitAutoVarDecl(const VarDecl &D);
3477
3478 class AutoVarEmission {
3479 friend class CodeGenFunction;
3480
3481 const VarDecl *Variable;
3482
3483 /// The address of the alloca for languages with explicit address space
3484 /// (e.g. OpenCL) or alloca casted to generic pointer for address space
3485 /// agnostic languages (e.g. C++). Invalid if the variable was emitted
3486 /// as a global constant.
3487 Address Addr;
3488
3489 llvm::Value *NRVOFlag;
3490
3491 /// True if the variable is a __block variable that is captured by an
3492 /// escaping block.
3493 bool IsEscapingByRef;
3494
3495 /// True if the variable is of aggregate type and has a constant
3496 /// initializer.
3497 bool IsConstantAggregate;
3498
3499 /// True if lifetime markers should be used.
3500 bool UseLifetimeMarkers;
3501
3502 /// Address with original alloca instruction. Invalid if the variable was
3503 /// emitted as a global constant.
3504 RawAddress AllocaAddr;
3505
3506 struct Invalid {};
3507 AutoVarEmission(Invalid)
3508 : Variable(nullptr), Addr(Address::invalid()),
3509 AllocaAddr(RawAddress::invalid()) {}
3510
3511 AutoVarEmission(const VarDecl &variable)
3512 : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr),
3513 IsEscapingByRef(false), IsConstantAggregate(false),
3514 UseLifetimeMarkers(false), AllocaAddr(RawAddress::invalid()) {}
3515
3516 bool wasEmittedAsGlobal() const { return !Addr.isValid(); }
3517
3518 public:
3519 static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }
3520
3521 bool useLifetimeMarkers() const { return UseLifetimeMarkers; }
3522
3523 /// Returns the raw, allocated address, which is not necessarily
3524 /// the address of the object itself. It is casted to default
3525 /// address space for address space agnostic languages.
3526 Address getAllocatedAddress() const { return Addr; }
3527
3528 /// Returns the address for the original alloca instruction.
3529 RawAddress getOriginalAllocatedAddress() const { return AllocaAddr; }
3530
3531 /// Returns the address of the object within this declaration.
3532 /// Note that this does not chase the forwarding pointer for
3533 /// __block decls.
3534 Address getObjectAddress(CodeGenFunction &CGF) const {
3535 if (!IsEscapingByRef)
3536 return Addr;
3537
3538 return CGF.emitBlockByrefAddress(baseAddr: Addr, V: Variable, /*forward*/ followForward: false);
3539 }
3540 };
3541 AutoVarEmission EmitAutoVarAlloca(const VarDecl &var);
3542 void EmitAutoVarInit(const AutoVarEmission &emission);
3543 void EmitAutoVarCleanups(const AutoVarEmission &emission);
3544 void emitAutoVarTypeCleanup(const AutoVarEmission &emission,
3545 QualType::DestructionKind dtorKind);
3546
3547 void MaybeEmitDeferredVarDeclInit(const VarDecl *var);
3548
3549 /// Emits the alloca and debug information for the size expressions for each
3550 /// dimension of an array. It registers the association of its (1-dimensional)
3551 /// QualTypes and size expression's debug node, so that CGDebugInfo can
3552 /// reference this node when creating the DISubrange object to describe the
3553 /// array types.
3554 void EmitAndRegisterVariableArrayDimensions(CGDebugInfo *DI, const VarDecl &D,
3555 bool EmitDebugInfo);
3556
3557 void EmitStaticVarDecl(const VarDecl &D,
3558 llvm::GlobalValue::LinkageTypes Linkage);
3559
3560 class ParamValue {
3561 union {
3562 Address Addr;
3563 llvm::Value *Value;
3564 };
3565
3566 bool IsIndirect;
3567
3568 ParamValue(llvm::Value *V) : Value(V), IsIndirect(false) {}
3569 ParamValue(Address A) : Addr(A), IsIndirect(true) {}
3570
3571 public:
3572 static ParamValue forDirect(llvm::Value *value) {
3573 return ParamValue(value);
3574 }
3575 static ParamValue forIndirect(Address addr) {
3576 assert(!addr.getAlignment().isZero());
3577 return ParamValue(addr);
3578 }
3579
3580 bool isIndirect() const { return IsIndirect; }
3581 llvm::Value *getAnyValue() const {
3582 if (!isIndirect())
3583 return Value;
3584 assert(!Addr.hasOffset() && "unexpected offset");
3585 return Addr.getBasePointer();
3586 }
3587
3588 llvm::Value *getDirectValue() const {
3589 assert(!isIndirect());
3590 return Value;
3591 }
3592
3593 Address getIndirectAddress() const {
3594 assert(isIndirect());
3595 return Addr;
3596 }
3597 };
3598
3599 /// EmitParmDecl - Emit a ParmVarDecl or an ImplicitParamDecl.
3600 void EmitParmDecl(const VarDecl &D, ParamValue Arg, unsigned ArgNo);
3601
3602 /// protectFromPeepholes - Protect a value that we're intending to
3603 /// store to the side, but which will probably be used later, from
3604 /// aggressive peepholing optimizations that might delete it.
3605 ///
3606 /// Pass the result to unprotectFromPeepholes to declare that
3607 /// protection is no longer required.
3608 ///
3609 /// There's no particular reason why this shouldn't apply to
3610 /// l-values, it's just that no existing peepholes work on pointers.
3611 PeepholeProtection protectFromPeepholes(RValue rvalue);
3612 void unprotectFromPeepholes(PeepholeProtection protection);
3613
3614 void emitAlignmentAssumptionCheck(llvm::Value *Ptr, QualType Ty,
3615 SourceLocation Loc,
3616 SourceLocation AssumptionLoc,
3617 llvm::Value *Alignment,
3618 llvm::Value *OffsetValue,
3619 llvm::Value *TheCheck,
3620 llvm::Instruction *Assumption);
3621
3622 void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty,
3623 SourceLocation Loc, SourceLocation AssumptionLoc,
3624 llvm::Value *Alignment,
3625 llvm::Value *OffsetValue = nullptr);
3626
3627 void emitAlignmentAssumption(llvm::Value *PtrValue, const Expr *E,
3628 SourceLocation AssumptionLoc,
3629 llvm::Value *Alignment,
3630 llvm::Value *OffsetValue = nullptr);
3631
3632 //===--------------------------------------------------------------------===//
3633 // Statement Emission
3634 //===--------------------------------------------------------------------===//
3635
3636 /// EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
3637 void EmitStopPoint(const Stmt *S);
3638
3639 /// EmitStmt - Emit the code for the statement \arg S. It is legal to call
3640 /// this function even if there is no current insertion point.
3641 ///
3642 /// This function may clear the current insertion point; callers should use
3643 /// EnsureInsertPoint if they wish to subsequently generate code without first
3644 /// calling EmitBlock, EmitBranch, or EmitStmt.
3645 void EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs = {});
3646
3647 /// EmitSimpleStmt - Try to emit a "simple" statement which does not
3648 /// necessarily require an insertion point or debug information; typically
3649 /// because the statement amounts to a jump or a container of other
3650 /// statements.
3651 ///
3652 /// \return True if the statement was handled.
3653 bool EmitSimpleStmt(const Stmt *S, ArrayRef<const Attr *> Attrs);
3654
3655 Address EmitCompoundStmt(const CompoundStmt &S, bool GetLast = false,
3656 AggValueSlot AVS = AggValueSlot::ignored());
3657 Address
3658 EmitCompoundStmtWithoutScope(const CompoundStmt &S, bool GetLast = false,
3659 AggValueSlot AVS = AggValueSlot::ignored());
3660
3661 /// EmitLabel - Emit the block for the given label. It is legal to call this
3662 /// function even if there is no current insertion point.
3663 void EmitLabel(const LabelDecl *D); // helper for EmitLabelStmt.
3664
3665 void EmitLabelStmt(const LabelStmt &S);
3666 void EmitAttributedStmt(const AttributedStmt &S);
3667 void EmitGotoStmt(const GotoStmt &S);
3668 void EmitIndirectGotoStmt(const IndirectGotoStmt &S);
3669 void EmitIfStmt(const IfStmt &S);
3670
3671 void EmitWhileStmt(const WhileStmt &S, ArrayRef<const Attr *> Attrs = {});
3672 void EmitDoStmt(const DoStmt &S, ArrayRef<const Attr *> Attrs = {});
3673 void EmitForStmt(const ForStmt &S, ArrayRef<const Attr *> Attrs = {});
3674 void EmitReturnStmt(const ReturnStmt &S);
3675 void EmitDeclStmt(const DeclStmt &S);
3676 void EmitBreakStmt(const BreakStmt &S);
3677 void EmitContinueStmt(const ContinueStmt &S);
3678 void EmitSwitchStmt(const SwitchStmt &S);
3679 void EmitDefaultStmt(const DefaultStmt &S, ArrayRef<const Attr *> Attrs);
3680 void EmitCaseStmt(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3681 void EmitCaseStmtRange(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3682 void EmitDeferStmt(const DeferStmt &S);
3683 void EmitAsmStmt(const AsmStmt &S);
3684
3685 const BreakContinue *GetDestForLoopControlStmt(const LoopControlStmt &S);
3686
3687 void EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S);
3688 void EmitObjCAtTryStmt(const ObjCAtTryStmt &S);
3689 void EmitObjCAtThrowStmt(const ObjCAtThrowStmt &S);
3690 void EmitObjCAtSynchronizedStmt(const ObjCAtSynchronizedStmt &S);
3691 void EmitObjCAutoreleasePoolStmt(const ObjCAutoreleasePoolStmt &S);
3692
3693 void EmitCoroutineBody(const CoroutineBodyStmt &S);
3694 void EmitCoreturnStmt(const CoreturnStmt &S);
3695 RValue EmitCoawaitExpr(const CoawaitExpr &E,
3696 AggValueSlot aggSlot = AggValueSlot::ignored(),
3697 bool ignoreResult = false);
3698 LValue EmitCoawaitLValue(const CoawaitExpr *E);
3699 RValue EmitCoyieldExpr(const CoyieldExpr &E,
3700 AggValueSlot aggSlot = AggValueSlot::ignored(),
3701 bool ignoreResult = false);
3702 LValue EmitCoyieldLValue(const CoyieldExpr *E);
3703 RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID);
3704
3705 void EmitSYCLKernelCallStmt(const SYCLKernelCallStmt &S);
3706
3707 void EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3708 void ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3709
3710 void EmitCXXTryStmt(const CXXTryStmt &S);
3711 void EmitSEHTryStmt(const SEHTryStmt &S);
3712 void EmitSEHLeaveStmt(const SEHLeaveStmt &S);
3713 void EnterSEHTryStmt(const SEHTryStmt &S);
3714 void ExitSEHTryStmt(const SEHTryStmt &S);
3715 void VolatilizeTryBlocks(llvm::BasicBlock *BB,
3716 llvm::SmallPtrSet<llvm::BasicBlock *, 10> &V);
3717
3718 void pushSEHCleanup(CleanupKind kind, llvm::Function *FinallyFunc);
3719 void startOutlinedSEHHelper(CodeGenFunction &ParentCGF, bool IsFilter,
3720 const Stmt *OutlinedStmt);
3721
3722 llvm::Function *GenerateSEHFilterFunction(CodeGenFunction &ParentCGF,
3723 const SEHExceptStmt &Except);
3724
3725 llvm::Function *GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF,
3726 const SEHFinallyStmt &Finally);
3727
3728 void EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF,
3729 llvm::Value *ParentFP, llvm::Value *EntryEBP);
3730 llvm::Value *EmitSEHExceptionCode();
3731 llvm::Value *EmitSEHExceptionInfo();
3732 llvm::Value *EmitSEHAbnormalTermination();
3733
3734 /// Emit simple code for OpenMP directives in Simd-only mode.
3735 void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D);
3736
3737 /// Scan the outlined statement for captures from the parent function. For
3738 /// each capture, mark the capture as escaped and emit a call to
3739 /// llvm.localrecover. Insert the localrecover result into the LocalDeclMap.
3740 void EmitCapturedLocals(CodeGenFunction &ParentCGF, const Stmt *OutlinedStmt,
3741 bool IsFilter);
3742
3743 /// Recovers the address of a local in a parent function. ParentVar is the
3744 /// address of the variable used in the immediate parent function. It can
3745 /// either be an alloca or a call to llvm.localrecover if there are nested
3746 /// outlined functions. ParentFP is the frame pointer of the outermost parent
3747 /// frame.
3748 Address recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
3749 Address ParentVar, llvm::Value *ParentFP);
3750
3751 void EmitCXXForRangeStmt(const CXXForRangeStmt &S,
3752 ArrayRef<const Attr *> Attrs = {});
3753
3754 void
3755 EmitCXXExpansionStmtInstantiation(const CXXExpansionStmtInstantiation &S);
3756
3757 /// Controls insertion of cancellation exit blocks in worksharing constructs.
3758 class OMPCancelStackRAII {
3759 CodeGenFunction &CGF;
3760
3761 public:
3762 OMPCancelStackRAII(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
3763 bool HasCancel)
3764 : CGF(CGF) {
3765 CGF.OMPCancelStack.enter(CGF, Kind, HasCancel);
3766 }
3767 ~OMPCancelStackRAII() { CGF.OMPCancelStack.exit(CGF); }
3768 };
3769
3770 /// Returns calculated size of the specified type.
3771 llvm::Value *getTypeSize(QualType Ty);
3772 LValue InitCapturedStruct(const CapturedStmt &S);
3773 llvm::Function *EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K);
3774 llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S);
3775 Address GenerateCapturedStmtArgument(const CapturedStmt &S);
3776 llvm::Function *
3777 GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S,
3778 const OMPExecutableDirective &D);
3779 llvm::Function *
3780 GenerateOpenMPCapturedStmtFunctionAggregate(const CapturedStmt &S,
3781 const OMPExecutableDirective &D);
3782 void GenerateOpenMPCapturedVars(const CapturedStmt &S,
3783 SmallVectorImpl<llvm::Value *> &CapturedVars);
3784 void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy,
3785 SourceLocation Loc);
3786 /// Perform element by element copying of arrays with type \a
3787 /// OriginalType from \a SrcAddr to \a DestAddr using copying procedure
3788 /// generated by \a CopyGen.
3789 ///
3790 /// \param DestAddr Address of the destination array.
3791 /// \param SrcAddr Address of the source array.
3792 /// \param OriginalType Type of destination and source arrays.
3793 /// \param CopyGen Copying procedure that copies value of single array element
3794 /// to another single array element.
3795 void EmitOMPAggregateAssign(
3796 Address DestAddr, Address SrcAddr, QualType OriginalType,
3797 const llvm::function_ref<void(Address, Address)> CopyGen);
3798 /// Emit proper copying of data from one variable to another.
3799 ///
3800 /// \param OriginalType Original type of the copied variables.
3801 /// \param DestAddr Destination address.
3802 /// \param SrcAddr Source address.
3803 /// \param DestVD Destination variable used in \a CopyExpr (for arrays, has
3804 /// type of the base array element).
3805 /// \param SrcVD Source variable used in \a CopyExpr (for arrays, has type of
3806 /// the base array element).
3807 /// \param Copy Actual copygin expression for copying data from \a SrcVD to \a
3808 /// DestVD.
3809 void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr,
3810 const VarDecl *DestVD, const VarDecl *SrcVD,
3811 const Expr *Copy);
3812 /// Emit atomic update code for constructs: \a X = \a X \a BO \a E or
3813 /// \a X = \a E \a BO \a E.
3814 ///
3815 /// \param X Value to be updated.
3816 /// \param E Update value.
3817 /// \param BO Binary operation for update operation.
3818 /// \param IsXLHSInRHSPart true if \a X is LHS in RHS part of the update
3819 /// expression, false otherwise.
3820 /// \param AO Atomic ordering of the generated atomic instructions.
3821 /// \param CommonGen Code generator for complex expressions that cannot be
3822 /// expressed through atomicrmw instruction.
3823 /// \returns <true, OldAtomicValue> if simple 'atomicrmw' instruction was
3824 /// generated, <false, RValue::get(nullptr)> otherwise.
3825 std::pair<bool, RValue> EmitOMPAtomicSimpleUpdateExpr(
3826 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3827 llvm::AtomicOrdering AO, SourceLocation Loc,
3828 const llvm::function_ref<RValue(RValue)> CommonGen);
3829 bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
3830 OMPPrivateScope &PrivateScope);
3831 void EmitOMPPrivateClause(const OMPExecutableDirective &D,
3832 OMPPrivateScope &PrivateScope);
3833 void EmitOMPUseDevicePtrClause(
3834 const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
3835 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3836 CaptureDeviceAddrMap);
3837 void EmitOMPUseDeviceAddrClause(
3838 const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
3839 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3840 CaptureDeviceAddrMap);
3841 /// Emit code for copyin clause in \a D directive. The next code is
3842 /// generated at the start of outlined functions for directives:
3843 /// \code
3844 /// threadprivate_var1 = master_threadprivate_var1;
3845 /// operator=(threadprivate_var2, master_threadprivate_var2);
3846 /// ...
3847 /// __kmpc_barrier(&loc, global_tid);
3848 /// \endcode
3849 ///
3850 /// \param D OpenMP directive possibly with 'copyin' clause(s).
3851 /// \returns true if at least one copyin variable is found, false otherwise.
3852 bool EmitOMPCopyinClause(const OMPExecutableDirective &D);
3853 /// Emit initial code for lastprivate variables. If some variable is
3854 /// not also firstprivate, then the default initialization is used. Otherwise
3855 /// initialization of this variable is performed by EmitOMPFirstprivateClause
3856 /// method.
3857 ///
3858 /// \param D Directive that may have 'lastprivate' directives.
3859 /// \param PrivateScope Private scope for capturing lastprivate variables for
3860 /// proper codegen in internal captured statement.
3861 ///
3862 /// \returns true if there is at least one lastprivate variable, false
3863 /// otherwise.
3864 bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D,
3865 OMPPrivateScope &PrivateScope);
3866 /// Emit final copying of lastprivate values to original variables at
3867 /// the end of the worksharing or simd directive.
3868 ///
3869 /// \param D Directive that has at least one 'lastprivate' directives.
3870 /// \param IsLastIterCond Boolean condition that must be set to 'i1 true' if
3871 /// it is the last iteration of the loop code in associated directive, or to
3872 /// 'i1 false' otherwise. If this item is nullptr, no final check is required.
3873 void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D,
3874 bool NoFinals,
3875 llvm::Value *IsLastIterCond = nullptr);
3876 /// Emit initial code for linear clauses.
3877 void EmitOMPLinearClause(const OMPLoopDirective &D,
3878 CodeGenFunction::OMPPrivateScope &PrivateScope);
3879 /// Emit final code for linear clauses.
3880 /// \param CondGen Optional conditional code for final part of codegen for
3881 /// linear clause.
3882 void EmitOMPLinearClauseFinal(
3883 const OMPLoopDirective &D,
3884 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
3885 /// Emit initial code for reduction variables. Creates reduction copies
3886 /// and initializes them with the values according to OpenMP standard.
3887 ///
3888 /// \param D Directive (possibly) with the 'reduction' clause.
3889 /// \param PrivateScope Private scope for capturing reduction variables for
3890 /// proper codegen in internal captured statement.
3891 ///
3892 void EmitOMPReductionClauseInit(const OMPExecutableDirective &D,
3893 OMPPrivateScope &PrivateScope,
3894 bool ForInscan = false);
3895 /// Emit final update of reduction values to original variables at
3896 /// the end of the directive.
3897 ///
3898 /// \param D Directive that has at least one 'reduction' directives.
3899 /// \param ReductionKind The kind of reduction to perform.
3900 void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D,
3901 const OpenMPDirectiveKind ReductionKind);
3902 /// Emit initial code for linear variables. Creates private copies
3903 /// and initializes them with the values according to OpenMP standard.
3904 ///
3905 /// \param D Directive (possibly) with the 'linear' clause.
3906 /// \return true if at least one linear variable is found that should be
3907 /// initialized with the value of the original variable, false otherwise.
3908 bool EmitOMPLinearClauseInit(const OMPLoopDirective &D);
3909
3910 typedef const llvm::function_ref<void(CodeGenFunction & /*CGF*/,
3911 llvm::Function * /*OutlinedFn*/,
3912 const OMPTaskDataTy & /*Data*/)>
3913 TaskGenTy;
3914 void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S,
3915 const OpenMPDirectiveKind CapturedRegion,
3916 const RegionCodeGenTy &BodyGen,
3917 const TaskGenTy &TaskGen, OMPTaskDataTy &Data);
3918 struct OMPTargetDataInfo {
3919 Address BasePointersArray = Address::invalid();
3920 Address PointersArray = Address::invalid();
3921 Address SizesArray = Address::invalid();
3922 Address MappersArray = Address::invalid();
3923 unsigned NumberOfTargetItems = 0;
3924 explicit OMPTargetDataInfo() = default;
3925 OMPTargetDataInfo(Address BasePointersArray, Address PointersArray,
3926 Address SizesArray, Address MappersArray,
3927 unsigned NumberOfTargetItems)
3928 : BasePointersArray(BasePointersArray), PointersArray(PointersArray),
3929 SizesArray(SizesArray), MappersArray(MappersArray),
3930 NumberOfTargetItems(NumberOfTargetItems) {}
3931 };
3932 void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S,
3933 const RegionCodeGenTy &BodyGen,
3934 OMPTargetDataInfo &InputInfo);
3935 void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data,
3936 CodeGenFunction &CGF, const CapturedStmt *CS,
3937 OMPPrivateScope &Scope);
3938 void EmitOMPMetaDirective(const OMPMetaDirective &S);
3939 void EmitOMPParallelDirective(const OMPParallelDirective &S);
3940 void EmitOMPSimdDirective(const OMPSimdDirective &S);
3941 void EmitOMPTileDirective(const OMPTileDirective &S);
3942 void EmitOMPStripeDirective(const OMPStripeDirective &S);
3943 void EmitOMPUnrollDirective(const OMPUnrollDirective &S);
3944 void EmitOMPReverseDirective(const OMPReverseDirective &S);
3945 void EmitOMPSplitDirective(const OMPSplitDirective &S);
3946 void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
3947 void EmitOMPFuseDirective(const OMPFuseDirective &S);
3948 void EmitOMPForDirective(const OMPForDirective &S);
3949 void EmitOMPForSimdDirective(const OMPForSimdDirective &S);
3950 void EmitOMPScopeDirective(const OMPScopeDirective &S);
3951 void EmitOMPSectionsDirective(const OMPSectionsDirective &S);
3952 void EmitOMPSectionDirective(const OMPSectionDirective &S);
3953 void EmitOMPSingleDirective(const OMPSingleDirective &S);
3954 void EmitOMPMasterDirective(const OMPMasterDirective &S);
3955 void EmitOMPMaskedDirective(const OMPMaskedDirective &S);
3956 void EmitOMPCriticalDirective(const OMPCriticalDirective &S);
3957 void EmitOMPParallelForDirective(const OMPParallelForDirective &S);
3958 void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S);
3959 void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S);
3960 void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S);
3961 void EmitOMPTaskDirective(const OMPTaskDirective &S);
3962 void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S);
3963 void EmitOMPErrorDirective(const OMPErrorDirective &S);
3964 void EmitOMPBarrierDirective(const OMPBarrierDirective &S);
3965 void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S);
3966 void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S);
3967 void EmitOMPFlushDirective(const OMPFlushDirective &S);
3968 void EmitOMPDepobjDirective(const OMPDepobjDirective &S);
3969 void EmitOMPScanDirective(const OMPScanDirective &S);
3970 void EmitOMPOrderedDirective(const OMPOrderedDirective &S);
3971 void EmitOMPAtomicDirective(const OMPAtomicDirective &S);
3972 void EmitOMPTargetDirective(const OMPTargetDirective &S);
3973 void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S);
3974 void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S);
3975 void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S);
3976 void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S);
3977 void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S);
3978 void
3979 EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S);
3980 void EmitOMPTeamsDirective(const OMPTeamsDirective &S);
3981 void
3982 EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S);
3983 void EmitOMPCancelDirective(const OMPCancelDirective &S);
3984 void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S);
3985 void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S);
3986 void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S);
3987 void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S);
3988 void EmitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &S);
3989 void
3990 EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S);
3991 void
3992 EmitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &S);
3993 void EmitOMPParallelMasterTaskLoopDirective(
3994 const OMPParallelMasterTaskLoopDirective &S);
3995 void EmitOMPParallelMaskedTaskLoopDirective(
3996 const OMPParallelMaskedTaskLoopDirective &S);
3997 void EmitOMPParallelMasterTaskLoopSimdDirective(
3998 const OMPParallelMasterTaskLoopSimdDirective &S);
3999 void EmitOMPParallelMaskedTaskLoopSimdDirective(
4000 const OMPParallelMaskedTaskLoopSimdDirective &S);
4001 void EmitOMPDistributeDirective(const OMPDistributeDirective &S);
4002 void EmitOMPDistributeParallelForDirective(
4003 const OMPDistributeParallelForDirective &S);
4004 void EmitOMPDistributeParallelForSimdDirective(
4005 const OMPDistributeParallelForSimdDirective &S);
4006 void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S);
4007 void EmitOMPTargetParallelForSimdDirective(
4008 const OMPTargetParallelForSimdDirective &S);
4009 void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S);
4010 void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S);
4011 void
4012 EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S);
4013 void EmitOMPTeamsDistributeParallelForSimdDirective(
4014 const OMPTeamsDistributeParallelForSimdDirective &S);
4015 void EmitOMPTeamsDistributeParallelForDirective(
4016 const OMPTeamsDistributeParallelForDirective &S);
4017 void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S);
4018 void EmitOMPTargetTeamsDistributeDirective(
4019 const OMPTargetTeamsDistributeDirective &S);
4020 void EmitOMPTargetTeamsDistributeParallelForDirective(
4021 const OMPTargetTeamsDistributeParallelForDirective &S);
4022 void EmitOMPTargetTeamsDistributeParallelForSimdDirective(
4023 const OMPTargetTeamsDistributeParallelForSimdDirective &S);
4024 void EmitOMPTargetTeamsDistributeSimdDirective(
4025 const OMPTargetTeamsDistributeSimdDirective &S);
4026 void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S);
4027 void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S);
4028 void EmitOMPTargetParallelGenericLoopDirective(
4029 const OMPTargetParallelGenericLoopDirective &S);
4030 void EmitOMPTargetTeamsGenericLoopDirective(
4031 const OMPTargetTeamsGenericLoopDirective &S);
4032 void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S);
4033 void EmitOMPInteropDirective(const OMPInteropDirective &S);
4034 void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S);
4035 void EmitOMPAssumeDirective(const OMPAssumeDirective &S);
4036
4037 /// Emit device code for the target directive.
4038 static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
4039 StringRef ParentName,
4040 const OMPTargetDirective &S);
4041 static void
4042 EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
4043 const OMPTargetParallelDirective &S);
4044 /// Emit device code for the target parallel for directive.
4045 static void EmitOMPTargetParallelForDeviceFunction(
4046 CodeGenModule &CGM, StringRef ParentName,
4047 const OMPTargetParallelForDirective &S);
4048 /// Emit device code for the target parallel for simd directive.
4049 static void EmitOMPTargetParallelForSimdDeviceFunction(
4050 CodeGenModule &CGM, StringRef ParentName,
4051 const OMPTargetParallelForSimdDirective &S);
4052 /// Emit device code for the target teams directive.
4053 static void
4054 EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
4055 const OMPTargetTeamsDirective &S);
4056 /// Emit device code for the target teams distribute directive.
4057 static void EmitOMPTargetTeamsDistributeDeviceFunction(
4058 CodeGenModule &CGM, StringRef ParentName,
4059 const OMPTargetTeamsDistributeDirective &S);
4060 /// Emit device code for the target teams distribute simd directive.
4061 static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(
4062 CodeGenModule &CGM, StringRef ParentName,
4063 const OMPTargetTeamsDistributeSimdDirective &S);
4064 /// Emit device code for the target simd directive.
4065 static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM,
4066 StringRef ParentName,
4067 const OMPTargetSimdDirective &S);
4068 /// Emit device code for the target teams distribute parallel for simd
4069 /// directive.
4070 static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
4071 CodeGenModule &CGM, StringRef ParentName,
4072 const OMPTargetTeamsDistributeParallelForSimdDirective &S);
4073
4074 /// Emit device code for the target teams loop directive.
4075 static void EmitOMPTargetTeamsGenericLoopDeviceFunction(
4076 CodeGenModule &CGM, StringRef ParentName,
4077 const OMPTargetTeamsGenericLoopDirective &S);
4078
4079 /// Emit device code for the target parallel loop directive.
4080 static void EmitOMPTargetParallelGenericLoopDeviceFunction(
4081 CodeGenModule &CGM, StringRef ParentName,
4082 const OMPTargetParallelGenericLoopDirective &S);
4083
4084 static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
4085 CodeGenModule &CGM, StringRef ParentName,
4086 const OMPTargetTeamsDistributeParallelForDirective &S);
4087
4088 /// Emit the Stmt \p S and return its topmost canonical loop, if any.
4089 /// TODO: The \p Depth paramter is not yet implemented and must be 1. In the
4090 /// future it is meant to be the number of loops expected in the loop nests
4091 /// (usually specified by the "collapse" clause) that are collapsed to a
4092 /// single loop by this function.
4093 llvm::CanonicalLoopInfo *EmitOMPCollapsedCanonicalLoopNest(const Stmt *S,
4094 int Depth);
4095
4096 /// Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
4097 void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S);
4098
4099 /// Emit inner loop of the worksharing/simd construct.
4100 ///
4101 /// \param S Directive, for which the inner loop must be emitted.
4102 /// \param RequiresCleanup true, if directive has some associated private
4103 /// variables.
4104 /// \param LoopCond Bollean condition for loop continuation.
4105 /// \param IncExpr Increment expression for loop control variable.
4106 /// \param BodyGen Generator for the inner body of the inner loop.
4107 /// \param PostIncGen Genrator for post-increment code (required for ordered
4108 /// loop directvies).
4109 void EmitOMPInnerLoop(
4110 const OMPExecutableDirective &S, bool RequiresCleanup,
4111 const Expr *LoopCond, const Expr *IncExpr,
4112 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
4113 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen);
4114
4115 JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind);
4116 /// Emit initial code for loop counters of loop-based directives.
4117 void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S,
4118 OMPPrivateScope &LoopScope);
4119
4120 /// Helper for the OpenMP loop directives.
4121 void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit);
4122
4123 /// Emit code for the worksharing loop-based directive.
4124 /// \return true, if this construct has any lastprivate clause, false -
4125 /// otherwise.
4126 bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB,
4127 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
4128 const CodeGenDispatchBoundsTy &CGDispatchBounds);
4129
4130 /// Emit code for the distribute loop-based directive.
4131 void EmitOMPDistributeLoop(const OMPLoopDirective &S,
4132 const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr);
4133
4134 /// Helpers for the OpenMP loop directives.
4135 void EmitOMPSimdInit(const OMPLoopDirective &D);
4136 void EmitOMPSimdFinal(
4137 const OMPLoopDirective &D,
4138 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
4139
4140 /// Emits the lvalue for the expression with possibly captured variable.
4141 LValue EmitOMPSharedLValue(const Expr *E);
4142
4143private:
4144 /// Helpers for blocks.
4145 llvm::Value *EmitBlockLiteral(const CGBlockInfo &Info);
4146
4147 /// struct with the values to be passed to the OpenMP loop-related functions
4148 struct OMPLoopArguments {
4149 /// loop lower bound
4150 Address LB = Address::invalid();
4151 /// loop upper bound
4152 Address UB = Address::invalid();
4153 /// loop stride
4154 Address ST = Address::invalid();
4155 /// isLastIteration argument for runtime functions
4156 Address IL = Address::invalid();
4157 /// Chunk value generated by sema
4158 llvm::Value *Chunk = nullptr;
4159 /// EnsureUpperBound
4160 Expr *EUB = nullptr;
4161 /// IncrementExpression
4162 Expr *IncExpr = nullptr;
4163 /// Loop initialization
4164 Expr *Init = nullptr;
4165 /// Loop exit condition
4166 Expr *Cond = nullptr;
4167 /// Update of LB after a whole chunk has been executed
4168 Expr *NextLB = nullptr;
4169 /// Update of UB after a whole chunk has been executed
4170 Expr *NextUB = nullptr;
4171 /// Distinguish between the for distribute and sections
4172 OpenMPDirectiveKind DKind = llvm::omp::OMPD_unknown;
4173 OMPLoopArguments() = default;
4174 OMPLoopArguments(Address LB, Address UB, Address ST, Address IL,
4175 llvm::Value *Chunk = nullptr, Expr *EUB = nullptr,
4176 Expr *IncExpr = nullptr, Expr *Init = nullptr,
4177 Expr *Cond = nullptr, Expr *NextLB = nullptr,
4178 Expr *NextUB = nullptr)
4179 : LB(LB), UB(UB), ST(ST), IL(IL), Chunk(Chunk), EUB(EUB),
4180 IncExpr(IncExpr), Init(Init), Cond(Cond), NextLB(NextLB),
4181 NextUB(NextUB) {}
4182 };
4183 void EmitOMPOuterLoop(bool DynamicOrOrdered, bool IsMonotonic,
4184 const OMPLoopDirective &S, OMPPrivateScope &LoopScope,
4185 const OMPLoopArguments &LoopArgs,
4186 const CodeGenLoopTy &CodeGenLoop,
4187 const CodeGenOrderedTy &CodeGenOrdered);
4188 void EmitOMPForOuterLoop(const OpenMPScheduleTy &ScheduleKind,
4189 bool IsMonotonic, const OMPLoopDirective &S,
4190 OMPPrivateScope &LoopScope, bool Ordered,
4191 const OMPLoopArguments &LoopArgs,
4192 const CodeGenDispatchBoundsTy &CGDispatchBounds);
4193 void EmitOMPDistributeOuterLoop(OpenMPDistScheduleClauseKind ScheduleKind,
4194 const OMPLoopDirective &S,
4195 OMPPrivateScope &LoopScope,
4196 const OMPLoopArguments &LoopArgs,
4197 const CodeGenLoopTy &CodeGenLoopContent);
4198 /// Emit code for sections directive.
4199 void EmitSections(const OMPExecutableDirective &S);
4200
4201public:
4202 //===--------------------------------------------------------------------===//
4203 // OpenACC Emission
4204 //===--------------------------------------------------------------------===//
4205 void EmitOpenACCComputeConstruct(const OpenACCComputeConstruct &S) {
4206 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4207 // simply emitting its structured block, but in the future we will implement
4208 // some sort of IR.
4209 EmitStmt(S: S.getStructuredBlock());
4210 }
4211
4212 void EmitOpenACCLoopConstruct(const OpenACCLoopConstruct &S) {
4213 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4214 // simply emitting its loop, but in the future we will implement
4215 // some sort of IR.
4216 EmitStmt(S: S.getLoop());
4217 }
4218
4219 void EmitOpenACCCombinedConstruct(const OpenACCCombinedConstruct &S) {
4220 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4221 // simply emitting its loop, but in the future we will implement
4222 // some sort of IR.
4223 EmitStmt(S: S.getLoop());
4224 }
4225
4226 void EmitOpenACCDataConstruct(const OpenACCDataConstruct &S) {
4227 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4228 // simply emitting its structured block, but in the future we will implement
4229 // some sort of IR.
4230 EmitStmt(S: S.getStructuredBlock());
4231 }
4232
4233 void EmitOpenACCEnterDataConstruct(const OpenACCEnterDataConstruct &S) {
4234 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4235 // but in the future we will implement some sort of IR.
4236 }
4237
4238 void EmitOpenACCExitDataConstruct(const OpenACCExitDataConstruct &S) {
4239 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4240 // but in the future we will implement some sort of IR.
4241 }
4242
4243 void EmitOpenACCHostDataConstruct(const OpenACCHostDataConstruct &S) {
4244 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4245 // simply emitting its structured block, but in the future we will implement
4246 // some sort of IR.
4247 EmitStmt(S: S.getStructuredBlock());
4248 }
4249
4250 void EmitOpenACCWaitConstruct(const OpenACCWaitConstruct &S) {
4251 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4252 // but in the future we will implement some sort of IR.
4253 }
4254
4255 void EmitOpenACCInitConstruct(const OpenACCInitConstruct &S) {
4256 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4257 // but in the future we will implement some sort of IR.
4258 }
4259
4260 void EmitOpenACCShutdownConstruct(const OpenACCShutdownConstruct &S) {
4261 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4262 // but in the future we will implement some sort of IR.
4263 }
4264
4265 void EmitOpenACCSetConstruct(const OpenACCSetConstruct &S) {
4266 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4267 // but in the future we will implement some sort of IR.
4268 }
4269
4270 void EmitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &S) {
4271 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4272 // but in the future we will implement some sort of IR.
4273 }
4274
4275 void EmitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &S) {
4276 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4277 // simply emitting its associated stmt, but in the future we will implement
4278 // some sort of IR.
4279 EmitStmt(S: S.getAssociatedStmt());
4280 }
4281 void EmitOpenACCCacheConstruct(const OpenACCCacheConstruct &S) {
4282 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4283 // but in the future we will implement some sort of IR.
4284 }
4285
4286 //===--------------------------------------------------------------------===//
4287 // LValue Expression Emission
4288 //===--------------------------------------------------------------------===//
4289
4290 /// Create a check that a scalar RValue is non-null.
4291 llvm::Value *EmitNonNullRValueCheck(RValue RV, QualType T);
4292
4293 /// GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
4294 RValue GetUndefRValue(QualType Ty);
4295
4296 /// EmitUnsupportedRValue - Emit a dummy r-value using the type of E
4297 /// and issue an ErrorUnsupported style diagnostic (using the
4298 /// provided Name).
4299 RValue EmitUnsupportedRValue(const Expr *E, const char *Name);
4300
4301 /// EmitUnsupportedLValue - Emit a dummy l-value using the type of E and issue
4302 /// an ErrorUnsupported style diagnostic (using the provided Name).
4303 LValue EmitUnsupportedLValue(const Expr *E, const char *Name);
4304
4305 /// EmitLValue - Emit code to compute a designator that specifies the location
4306 /// of the expression.
4307 ///
4308 /// This can return one of two things: a simple address or a bitfield
4309 /// reference. In either case, the LLVM Value* in the LValue structure is
4310 /// guaranteed to be an LLVM pointer type.
4311 ///
4312 /// If this returns a bitfield reference, nothing about the pointee type of
4313 /// the LLVM value is known: For example, it may not be a pointer to an
4314 /// integer.
4315 ///
4316 /// If this returns a normal address, and if the lvalue's C type is fixed
4317 /// size, this method guarantees that the returned pointer type will point to
4318 /// an LLVM type of the same size of the lvalue's type. If the lvalue has a
4319 /// variable length type, this is not possible.
4320 ///
4321 LValue EmitLValue(const Expr *E,
4322 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
4323
4324private:
4325 LValue EmitLValueHelper(const Expr *E, KnownNonNull_t IsKnownNonNull);
4326
4327public:
4328 /// Same as EmitLValue but additionally we generate checking code to
4329 /// guard against undefined behavior. This is only suitable when we know
4330 /// that the address will be used to access the object.
4331 LValue EmitCheckedLValue(const Expr *E, TypeCheckKind TCK);
4332
4333 RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc);
4334
4335 void EmitAtomicInit(Expr *E, LValue lvalue);
4336
4337 bool LValueIsSuitableForInlineAtomic(LValue Src);
4338
4339 RValue EmitAtomicLoad(LValue LV, SourceLocation SL,
4340 AggValueSlot Slot = AggValueSlot::ignored());
4341
4342 RValue EmitAtomicLoad(LValue lvalue, SourceLocation loc,
4343 llvm::AtomicOrdering AO, bool IsVolatile = false,
4344 AggValueSlot slot = AggValueSlot::ignored());
4345
4346 void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit);
4347
4348 void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO,
4349 bool IsVolatile, bool isInit);
4350
4351 std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
4352 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
4353 llvm::AtomicOrdering Success =
4354 llvm::AtomicOrdering::SequentiallyConsistent,
4355 llvm::AtomicOrdering Failure =
4356 llvm::AtomicOrdering::SequentiallyConsistent,
4357 bool IsWeak = false, AggValueSlot Slot = AggValueSlot::ignored());
4358
4359 /// Emit an atomicrmw instruction, and applying relevant metadata when
4360 /// applicable.
4361 llvm::AtomicRMWInst *emitAtomicRMWInst(
4362 llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val,
4363 llvm::AtomicOrdering Order = llvm::AtomicOrdering::SequentiallyConsistent,
4364 llvm::SyncScope::ID SSID = llvm::SyncScope::System,
4365 const AtomicExpr *AE = nullptr);
4366
4367 void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO,
4368 const llvm::function_ref<RValue(RValue)> &UpdateOp,
4369 bool IsVolatile);
4370
4371 /// EmitToMemory - Change a scalar value from its value
4372 /// representation to its in-memory representation.
4373 llvm::Value *EmitToMemory(llvm::Value *Value, QualType Ty);
4374
4375 /// EmitFromMemory - Change a scalar value from its memory
4376 /// representation to its value representation.
4377 llvm::Value *EmitFromMemory(llvm::Value *Value, QualType Ty);
4378
4379 /// Check if the scalar \p Value is within the valid range for the given
4380 /// type \p Ty.
4381 ///
4382 /// Returns true if a check is needed (even if the range is unknown).
4383 bool EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
4384 SourceLocation Loc);
4385
4386 /// EmitLoadOfScalar - Load a scalar value from an address, taking
4387 /// care to appropriately convert from the memory representation to
4388 /// the LLVM value representation.
4389 llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4390 SourceLocation Loc,
4391 AlignmentSource Source = AlignmentSource::Type,
4392 bool isNontemporal = false) {
4393 return EmitLoadOfScalar(Addr, Volatile, Ty, Loc, BaseInfo: LValueBaseInfo(Source),
4394 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isNontemporal);
4395 }
4396
4397 llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4398 SourceLocation Loc, LValueBaseInfo BaseInfo,
4399 TBAAAccessInfo TBAAInfo,
4400 bool isNontemporal = false);
4401
4402 /// EmitLoadOfScalar - Load a scalar value from an address, taking
4403 /// care to appropriately convert from the memory representation to
4404 /// the LLVM value representation. The l-value must be a simple
4405 /// l-value.
4406 llvm::Value *EmitLoadOfScalar(LValue lvalue, SourceLocation Loc);
4407
4408 /// EmitStoreOfScalar - Store a scalar value to an address, taking
4409 /// care to appropriately convert from the memory representation to
4410 /// the LLVM value representation.
4411 void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,
4412 QualType Ty,
4413 AlignmentSource Source = AlignmentSource::Type,
4414 bool isInit = false, bool isNontemporal = false) {
4415 EmitStoreOfScalar(Value, Addr, Volatile, Ty, BaseInfo: LValueBaseInfo(Source),
4416 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isInit, isNontemporal);
4417 }
4418
4419 void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,
4420 QualType Ty, LValueBaseInfo BaseInfo,
4421 TBAAAccessInfo TBAAInfo, bool isInit = false,
4422 bool isNontemporal = false);
4423
4424 /// EmitStoreOfScalar - Store a scalar value to an address, taking
4425 /// care to appropriately convert from the memory representation to
4426 /// the LLVM value representation. The l-value must be a simple
4427 /// l-value. The isInit flag indicates whether this is an initialization.
4428 /// If so, atomic qualifiers are ignored and the store is always non-atomic.
4429 void EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
4430 bool isInit = false);
4431
4432 /// EmitLoadOfLValue - Given an expression that represents a value lvalue,
4433 /// this method emits the address of the lvalue, then loads the result as an
4434 /// rvalue, returning the rvalue.
4435 RValue EmitLoadOfLValue(LValue V, SourceLocation Loc);
4436 RValue EmitLoadOfExtVectorElementLValue(LValue V);
4437 RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc);
4438 RValue EmitLoadOfGlobalRegLValue(LValue LV);
4439
4440 /// Like EmitLoadOfLValue but also handles complex and aggregate types.
4441 RValue EmitLoadOfAnyValue(LValue V,
4442 AggValueSlot Slot = AggValueSlot::ignored(),
4443 SourceLocation Loc = {});
4444
4445 /// EmitStoreThroughLValue - Store the specified rvalue into the specified
4446 /// lvalue, where both are guaranteed to the have the same type, and that type
4447 /// is 'Ty'.
4448 void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit = false);
4449 void EmitStoreThroughExtVectorComponentLValue(RValue Src, LValue Dst);
4450 void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst);
4451
4452 /// EmitStoreThroughBitfieldLValue - Store Src into Dst with same constraints
4453 /// as EmitStoreThroughLValue.
4454 ///
4455 /// \param Result [out] - If non-null, this will be set to a Value* for the
4456 /// bit-field contents after the store, appropriate for use as the result of
4457 /// an assignment to the bit-field.
4458 void EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
4459 llvm::Value **Result = nullptr);
4460
4461 /// Emit an l-value for an assignment (simple or compound) of complex type.
4462 LValue EmitComplexAssignmentLValue(const BinaryOperator *E);
4463 LValue EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E);
4464 LValue EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E,
4465 llvm::Value *&Result);
4466
4467 // Note: only available for agg return types
4468 LValue EmitBinaryOperatorLValue(const BinaryOperator *E);
4469 LValue EmitCompoundAssignmentLValue(const CompoundAssignOperator *E);
4470 // Note: only available for agg return types
4471 LValue EmitCallExprLValue(const CallExpr *E,
4472 llvm::CallBase **CallOrInvoke = nullptr);
4473 // Note: only available for agg return types
4474 LValue EmitVAArgExprLValue(const VAArgExpr *E);
4475 LValue EmitDeclRefLValue(const DeclRefExpr *E);
4476 LValue EmitStringLiteralLValue(const StringLiteral *E);
4477 LValue EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E);
4478 LValue EmitPredefinedLValue(const PredefinedExpr *E);
4479 LValue EmitUnaryOpLValue(const UnaryOperator *E);
4480 LValue EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
4481 bool Accessed = false);
4482 llvm::Value *EmitMatrixIndexExpr(const Expr *E);
4483 LValue EmitMatrixSingleSubscriptExpr(const MatrixSingleSubscriptExpr *E);
4484 LValue EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E);
4485 LValue EmitArraySectionExpr(const ArraySectionExpr *E,
4486 bool IsLowerBound = true);
4487 LValue EmitExtVectorElementExpr(const ExtVectorElementExpr *E);
4488 LValue EmitMatrixElementExpr(const MatrixElementExpr *E);
4489 LValue EmitMemberExpr(const MemberExpr *E);
4490 LValue EmitObjCIsaExpr(const ObjCIsaExpr *E);
4491 LValue EmitCompoundLiteralLValue(const CompoundLiteralExpr *E);
4492 LValue EmitInitListLValue(const InitListExpr *E);
4493 void EmitIgnoredConditionalOperator(const AbstractConditionalOperator *E);
4494 LValue EmitConditionalOperatorLValue(const AbstractConditionalOperator *E);
4495 LValue EmitCastLValue(const CastExpr *E);
4496 LValue EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);
4497 LValue EmitOpaqueValueLValue(const OpaqueValueExpr *e);
4498 LValue EmitHLSLArrayAssignLValue(const BinaryOperator *E);
4499
4500 std::pair<LValue, LValue> EmitHLSLOutArgLValues(const HLSLOutArgExpr *E,
4501 QualType Ty);
4502 LValue EmitHLSLOutArgExpr(const HLSLOutArgExpr *E, CallArgList &Args,
4503 QualType Ty);
4504
4505 Address EmitExtVectorElementLValue(LValue V);
4506
4507 RValue EmitRValueForField(LValue LV, const FieldDecl *FD, SourceLocation Loc);
4508
4509 Address EmitArrayToPointerDecay(const Expr *Array,
4510 LValueBaseInfo *BaseInfo = nullptr,
4511 TBAAAccessInfo *TBAAInfo = nullptr);
4512
4513 class ConstantEmission {
4514 llvm::PointerIntPair<llvm::Constant *, 1, bool> ValueAndIsReference;
4515 ConstantEmission(llvm::Constant *C, bool isReference)
4516 : ValueAndIsReference(C, isReference) {}
4517
4518 public:
4519 ConstantEmission() {}
4520 static ConstantEmission forReference(llvm::Constant *C) {
4521 return ConstantEmission(C, true);
4522 }
4523 static ConstantEmission forValue(llvm::Constant *C) {
4524 return ConstantEmission(C, false);
4525 }
4526
4527 explicit operator bool() const {
4528 return ValueAndIsReference.getOpaqueValue() != nullptr;
4529 }
4530
4531 bool isReference() const { return ValueAndIsReference.getInt(); }
4532 LValue getReferenceLValue(CodeGenFunction &CGF, const Expr *RefExpr) const {
4533 assert(isReference());
4534 return CGF.MakeNaturalAlignAddrLValue(V: ValueAndIsReference.getPointer(),
4535 T: RefExpr->getType());
4536 }
4537
4538 llvm::Constant *getValue() const {
4539 assert(!isReference());
4540 return ValueAndIsReference.getPointer();
4541 }
4542 };
4543
4544 ConstantEmission tryEmitAsConstant(const DeclRefExpr *RefExpr);
4545 ConstantEmission tryEmitAsConstant(const MemberExpr *ME);
4546 llvm::Value *emitScalarConstant(const ConstantEmission &Constant, Expr *E);
4547
4548 RValue EmitPseudoObjectRValue(const PseudoObjectExpr *e,
4549 AggValueSlot slot = AggValueSlot::ignored());
4550 LValue EmitPseudoObjectLValue(const PseudoObjectExpr *e);
4551
4552 void FlattenAccessAndTypeLValue(LValue LVal,
4553 SmallVectorImpl<LValue> &AccessList);
4554
4555 llvm::Value *EmitIvarOffset(const ObjCInterfaceDecl *Interface,
4556 const ObjCIvarDecl *Ivar);
4557 llvm::Value *EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
4558 const ObjCIvarDecl *Ivar);
4559 LValue EmitLValueForField(LValue Base, const FieldDecl *Field,
4560 bool IsInBounds = true);
4561 LValue EmitLValueForLambdaField(const FieldDecl *Field);
4562 LValue EmitLValueForLambdaField(const FieldDecl *Field,
4563 llvm::Value *ThisValue);
4564
4565 /// EmitLValueForFieldInitialization - Like EmitLValueForField, except that
4566 /// if the Field is a reference, this will return the address of the reference
4567 /// and not the address of the value stored in the reference.
4568 LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field);
4569
4570 LValue EmitLValueForIvar(QualType ObjectTy, llvm::Value *Base,
4571 const ObjCIvarDecl *Ivar, unsigned CVRQualifiers);
4572
4573 LValue EmitCXXConstructLValue(const CXXConstructExpr *E);
4574 LValue EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E);
4575 LValue EmitCXXTypeidLValue(const CXXTypeidExpr *E);
4576 LValue EmitCXXUuidofLValue(const CXXUuidofExpr *E);
4577
4578 LValue EmitObjCMessageExprLValue(const ObjCMessageExpr *E);
4579 LValue EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E);
4580 LValue EmitStmtExprLValue(const StmtExpr *E);
4581 LValue EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E);
4582 LValue EmitObjCSelectorLValue(const ObjCSelectorExpr *E);
4583 void EmitDeclRefExprDbgValue(const DeclRefExpr *E, const APValue &Init);
4584
4585 //===--------------------------------------------------------------------===//
4586 // Scalar Expression Emission
4587 //===--------------------------------------------------------------------===//
4588
4589 /// EmitCall - Generate a call of the given function, expecting the given
4590 /// result type, and using the given argument list which specifies both the
4591 /// LLVM arguments and the types they were derived from.
4592 RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4593 ReturnValueSlot ReturnValue, const CallArgList &Args,
4594 llvm::CallBase **CallOrInvoke, bool IsMustTail,
4595 SourceLocation Loc,
4596 bool IsVirtualFunctionPointerThunk = false);
4597 RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4598 ReturnValueSlot ReturnValue, const CallArgList &Args,
4599 llvm::CallBase **CallOrInvoke = nullptr,
4600 bool IsMustTail = false) {
4601 return EmitCall(CallInfo, Callee, ReturnValue, Args, CallOrInvoke,
4602 IsMustTail, Loc: SourceLocation());
4603 }
4604 RValue EmitCall(QualType FnType, const CGCallee &Callee, const CallExpr *E,
4605 ReturnValueSlot ReturnValue, llvm::Value *Chain = nullptr,
4606 llvm::CallBase **CallOrInvoke = nullptr,
4607 CGFunctionInfo const **ResolvedFnInfo = nullptr);
4608
4609 // If a Call or Invoke instruction was emitted for this CallExpr, this method
4610 // writes the pointer to `CallOrInvoke` if it's not null.
4611 RValue EmitCallExpr(const CallExpr *E,
4612 ReturnValueSlot ReturnValue = ReturnValueSlot(),
4613 llvm::CallBase **CallOrInvoke = nullptr);
4614 RValue EmitSimpleCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,
4615 llvm::CallBase **CallOrInvoke = nullptr);
4616 CGCallee EmitCallee(const Expr *E);
4617
4618 void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl);
4619 void checkTargetFeatures(SourceLocation Loc, const FunctionDecl *TargetDecl);
4620
4621 llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4622 const Twine &name = "");
4623 llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4624 ArrayRef<llvm::Value *> args,
4625 const Twine &name = "");
4626 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4627 const Twine &Name = "");
4628 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4629 ArrayRef<llvm::Value *> Args,
4630 const Twine &Name = "");
4631 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4632 ArrayRef<llvm::Type *> Types,
4633 ArrayRef<llvm::Value *> Args,
4634 const Twine &Name = "");
4635 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4636 const Twine &name = "");
4637 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4638 ArrayRef<Address> args,
4639 const Twine &name = "");
4640 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4641 ArrayRef<llvm::Value *> args,
4642 const Twine &name = "");
4643
4644 SmallVector<llvm::OperandBundleDef, 1>
4645 getBundlesForFunclet(llvm::Value *Callee);
4646
4647 llvm::CallBase *EmitCallOrInvoke(llvm::FunctionCallee Callee,
4648 ArrayRef<llvm::Value *> Args,
4649 const Twine &Name = "");
4650 llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4651 ArrayRef<llvm::Value *> args,
4652 const Twine &name = "");
4653 llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4654 const Twine &name = "");
4655 void EmitNoreturnRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4656 ArrayRef<llvm::Value *> args);
4657
4658 CGCallee BuildAppleKextVirtualCall(const CXXMethodDecl *MD,
4659 NestedNameSpecifier Qual, llvm::Type *Ty);
4660
4661 CGCallee BuildAppleKextVirtualDestructorCall(const CXXDestructorDecl *DD,
4662 CXXDtorType Type,
4663 const CXXRecordDecl *RD);
4664
4665 bool isPointerKnownNonNull(const Expr *E);
4666 /// Check whether the underlying base pointer is a constant null.
4667 bool isUnderlyingBasePointerConstantNull(const Expr *E);
4668
4669 /// Create the discriminator from the storage address and the entity hash.
4670 llvm::Value *EmitPointerAuthBlendDiscriminator(llvm::Value *StorageAddress,
4671 llvm::Value *Discriminator);
4672 CGPointerAuthInfo EmitPointerAuthInfo(const PointerAuthSchema &Schema,
4673 llvm::Value *StorageAddress,
4674 GlobalDecl SchemaDecl,
4675 QualType SchemaType);
4676
4677 llvm::Value *EmitPointerAuthSign(const CGPointerAuthInfo &Info,
4678 llvm::Value *Pointer);
4679
4680 llvm::Value *EmitPointerAuthAuth(const CGPointerAuthInfo &Info,
4681 llvm::Value *Pointer);
4682
4683 llvm::Value *emitPointerAuthResign(llvm::Value *Pointer, QualType PointerType,
4684 const CGPointerAuthInfo &CurAuthInfo,
4685 const CGPointerAuthInfo &NewAuthInfo,
4686 bool IsKnownNonNull);
4687 llvm::Value *emitPointerAuthResignCall(llvm::Value *Pointer,
4688 const CGPointerAuthInfo &CurInfo,
4689 const CGPointerAuthInfo &NewInfo);
4690
4691 void EmitPointerAuthOperandBundle(
4692 const CGPointerAuthInfo &Info,
4693 SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
4694
4695 CGPointerAuthInfo EmitPointerAuthInfo(PointerAuthQualifier Qualifier,
4696 Address StorageAddress);
4697 llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
4698 llvm::Value *Pointer, QualType ValueType,
4699 Address StorageAddress,
4700 bool IsKnownNonNull);
4701 llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
4702 const Expr *PointerExpr,
4703 Address StorageAddress);
4704 llvm::Value *EmitPointerAuthUnqualify(PointerAuthQualifier Qualifier,
4705 llvm::Value *Pointer,
4706 QualType PointerType,
4707 Address StorageAddress,
4708 bool IsKnownNonNull);
4709 void EmitPointerAuthCopy(PointerAuthQualifier Qualifier, QualType Type,
4710 Address DestField, Address SrcField);
4711
4712 std::pair<llvm::Value *, CGPointerAuthInfo>
4713 EmitOrigPointerRValue(const Expr *E);
4714
4715 llvm::Value *authPointerToPointerCast(llvm::Value *ResultPtr,
4716 QualType SourceType, QualType DestType);
4717 Address authPointerToPointerCast(Address Ptr, QualType SourceType,
4718 QualType DestType);
4719
4720 Address getAsNaturalAddressOf(Address Addr, QualType PointeeTy);
4721
4722 llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) {
4723 return getAsNaturalAddressOf(Addr, PointeeTy: PointeeType).getBasePointer();
4724 }
4725
4726 // Return the copy constructor name with the prefix "__copy_constructor_"
4727 // removed.
4728 static std::string getNonTrivialCopyConstructorStr(QualType QT,
4729 CharUnits Alignment,
4730 bool IsVolatile,
4731 ASTContext &Ctx);
4732
4733 // Return the destructor name with the prefix "__destructor_" removed.
4734 static std::string getNonTrivialDestructorStr(QualType QT,
4735 CharUnits Alignment,
4736 bool IsVolatile,
4737 ASTContext &Ctx);
4738
4739 // These functions emit calls to the special functions of non-trivial C
4740 // structs.
4741 void defaultInitNonTrivialCStructVar(LValue Dst);
4742 void callCStructDefaultConstructor(LValue Dst);
4743 void callCStructDestructor(LValue Dst);
4744 void callCStructCopyConstructor(LValue Dst, LValue Src);
4745 void callCStructMoveConstructor(LValue Dst, LValue Src);
4746 void callCStructCopyAssignmentOperator(LValue Dst, LValue Src);
4747 void callCStructMoveAssignmentOperator(LValue Dst, LValue Src);
4748
4749 RValue EmitCXXMemberOrOperatorCall(
4750 const CXXMethodDecl *Method, const CGCallee &Callee,
4751 ReturnValueSlot ReturnValue, llvm::Value *This,
4752 llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *E,
4753 CallArgList *RtlArgs, llvm::CallBase **CallOrInvoke);
4754 RValue EmitCXXDestructorCall(GlobalDecl Dtor, const CGCallee &Callee,
4755 llvm::Value *This, QualType ThisTy,
4756 llvm::Value *ImplicitParam,
4757 QualType ImplicitParamTy, const CallExpr *E,
4758 llvm::CallBase **CallOrInvoke = nullptr);
4759 RValue EmitCXXMemberCallExpr(const CXXMemberCallExpr *E,
4760 ReturnValueSlot ReturnValue,
4761 llvm::CallBase **CallOrInvoke = nullptr);
4762 RValue EmitCXXMemberOrOperatorMemberCallExpr(
4763 const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue,
4764 bool HasQualifier, NestedNameSpecifier Qualifier, bool IsArrow,
4765 const Expr *Base, llvm::CallBase **CallOrInvoke);
4766 // Compute the object pointer.
4767 Address EmitCXXMemberDataPointerAddress(
4768 const Expr *E, Address base, llvm::Value *memberPtr,
4769 const MemberPointerType *memberPtrType, bool IsInBounds,
4770 LValueBaseInfo *BaseInfo = nullptr, TBAAAccessInfo *TBAAInfo = nullptr);
4771 RValue EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
4772 ReturnValueSlot ReturnValue,
4773 llvm::CallBase **CallOrInvoke);
4774
4775 RValue EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
4776 const CXXMethodDecl *MD,
4777 ReturnValueSlot ReturnValue,
4778 llvm::CallBase **CallOrInvoke);
4779 RValue EmitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *E);
4780
4781 RValue EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
4782 ReturnValueSlot ReturnValue,
4783 llvm::CallBase **CallOrInvoke);
4784
4785 RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E);
4786 RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E);
4787
4788 RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
4789 const CallExpr *E, ReturnValueSlot ReturnValue);
4790
4791 RValue emitRotate(const CallExpr *E, bool IsRotateRight);
4792
4793 RValue emitStdcCountIntrinsic(const CallExpr *E, llvm::Intrinsic::ID IntID,
4794 bool InvertArg, bool IsPop = false);
4795 RValue emitStdcBitWidthMinus(const CallExpr *E, llvm::Intrinsic::ID IntID,
4796 bool IsPop);
4797 RValue emitStdcFirstBit(const CallExpr *E, llvm::Intrinsic::ID IntID,
4798 bool InvertArg);
4799
4800 /// Emit IR for __builtin_os_log_format.
4801 RValue emitBuiltinOSLogFormat(const CallExpr &E);
4802
4803 /// Emit IR for __builtin_is_aligned.
4804 RValue EmitBuiltinIsAligned(const CallExpr *E);
4805 /// Emit IR for __builtin_align_up/__builtin_align_down.
4806 RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp);
4807
4808 llvm::Function *generateBuiltinOSLogHelperFunction(
4809 const analyze_os_log::OSLogBufferLayout &Layout,
4810 CharUnits BufferAlignment);
4811
4812 RValue EmitBlockCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,
4813 llvm::CallBase **CallOrInvoke);
4814
4815 /// EmitTargetBuiltinExpr - Emit the given builtin call. Returns 0 if the call
4816 /// is unhandled by the current target.
4817 llvm::Value *EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4818 ReturnValueSlot ReturnValue);
4819
4820 llvm::Value *
4821 EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty,
4822 const llvm::CmpInst::Predicate Pred,
4823 const llvm::Twine &Name = "");
4824 llvm::Value *EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4825 ReturnValueSlot ReturnValue,
4826 llvm::Triple::ArchType Arch);
4827 llvm::Value *EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4828 ReturnValueSlot ReturnValue,
4829 llvm::Triple::ArchType Arch);
4830 llvm::Value *EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4831 ReturnValueSlot ReturnValue,
4832 llvm::Triple::ArchType Arch);
4833 llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::IntegerType *ITy,
4834 QualType RTy);
4835 llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::ArrayType *ATy,
4836 QualType RTy);
4837
4838 llvm::Value *
4839 EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic,
4840 unsigned AltLLVMIntrinsic, const char *NameHint,
4841 unsigned Modifier, const CallExpr *E,
4842 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0,
4843 Address PtrOp1, llvm::Triple::ArchType Arch);
4844
4845 llvm::Function *LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
4846 unsigned Modifier, llvm::Type *ArgTy,
4847 const CallExpr *E);
4848 llvm::Value *EmitNeonCall(llvm::Function *F,
4849 SmallVectorImpl<llvm::Value *> &O, const char *name,
4850 unsigned shift = 0, bool rightshift = false);
4851 llvm::Value *EmitFP8NeonCall(unsigned IID, ArrayRef<llvm::Type *> Tys,
4852 SmallVectorImpl<llvm::Value *> &O,
4853 const CallExpr *E, const char *name);
4854 llvm::Value *EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,
4855 llvm::Type *Ty1, bool Extract,
4856 SmallVectorImpl<llvm::Value *> &Ops,
4857 const CallExpr *E, const char *name);
4858 llvm::Value *EmitFP8NeonFDOTCall(unsigned IID, bool ExtendLaneArg,
4859 llvm::Type *RetTy,
4860 SmallVectorImpl<llvm::Value *> &Ops,
4861 const CallExpr *E, const char *name);
4862 llvm::Value *EmitFP8NeonFMLACall(unsigned IID, bool ExtendLaneArg,
4863 llvm::Type *RetTy,
4864 SmallVectorImpl<llvm::Value *> &Ops,
4865 const CallExpr *E, const char *name);
4866 llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx,
4867 const llvm::ElementCount &Count);
4868 llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx);
4869 llvm::Value *EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty,
4870 bool negateForRightShift);
4871 llvm::Value *EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt,
4872 llvm::Type *Ty, bool usgn, const char *name);
4873 llvm::Value *vectorWrapScalar16(llvm::Value *Op);
4874 /// SVEBuiltinMemEltTy - Returns the memory element type for this memory
4875 /// access builtin. Only required if it can't be inferred from the base
4876 /// pointer operand.
4877 llvm::Type *SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags);
4878
4879 SmallVector<llvm::Type *, 2>
4880 getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType,
4881 ArrayRef<llvm::Value *> Ops);
4882 llvm::Type *getEltType(const SVETypeFlags &TypeFlags);
4883 llvm::ScalableVectorType *getSVEType(const SVETypeFlags &TypeFlags);
4884 llvm::ScalableVectorType *getSVEPredType(const SVETypeFlags &TypeFlags);
4885 llvm::Value *EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags,
4886 ArrayRef<llvm::Value *> Ops);
4887 llvm::Value *EmitSVETupleCreate(const SVETypeFlags &TypeFlags,
4888 llvm::Type *ReturnType,
4889 ArrayRef<llvm::Value *> Ops);
4890 llvm::Value *EmitSVEDupX(llvm::Value *Scalar);
4891 llvm::Value *EmitSVEDupX(llvm::Value *Scalar, llvm::Type *Ty);
4892 llvm::Value *EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty);
4893 llvm::Value *EmitSVEPMull(const SVETypeFlags &TypeFlags,
4894 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4895 unsigned BuiltinID);
4896 llvm::Value *EmitSVEMovl(const SVETypeFlags &TypeFlags,
4897 llvm::ArrayRef<llvm::Value *> Ops,
4898 unsigned BuiltinID);
4899 llvm::Value *EmitSVEPredicateCast(llvm::Value *Pred,
4900 llvm::ScalableVectorType *VTy);
4901 llvm::Value *EmitSVEPredicateTupleCast(llvm::Value *PredTuple,
4902 llvm::StructType *Ty);
4903 llvm::Value *EmitSVEGatherLoad(const SVETypeFlags &TypeFlags,
4904 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4905 unsigned IntID);
4906 llvm::Value *EmitSVEScatterStore(const SVETypeFlags &TypeFlags,
4907 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4908 unsigned IntID);
4909 llvm::Value *EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy,
4910 SmallVectorImpl<llvm::Value *> &Ops,
4911 unsigned BuiltinID, bool IsZExtReturn);
4912 llvm::Value *EmitSVEMaskedStore(const CallExpr *,
4913 SmallVectorImpl<llvm::Value *> &Ops,
4914 unsigned BuiltinID);
4915 llvm::Value *EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,
4916 SmallVectorImpl<llvm::Value *> &Ops,
4917 unsigned BuiltinID);
4918 llvm::Value *EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags,
4919 SmallVectorImpl<llvm::Value *> &Ops,
4920 unsigned IntID);
4921 llvm::Value *EmitSVEStructLoad(const SVETypeFlags &TypeFlags,
4922 SmallVectorImpl<llvm::Value *> &Ops,
4923 unsigned IntID);
4924 llvm::Value *EmitSVEStructStore(const SVETypeFlags &TypeFlags,
4925 SmallVectorImpl<llvm::Value *> &Ops,
4926 unsigned IntID);
4927 llvm::Value *EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4928
4929 llvm::Value *EmitSMELd1St1(const SVETypeFlags &TypeFlags,
4930 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4931 unsigned IntID);
4932 llvm::Value *EmitSMEReadWrite(const SVETypeFlags &TypeFlags,
4933 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4934 unsigned IntID);
4935 llvm::Value *EmitSMEZero(const SVETypeFlags &TypeFlags,
4936 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4937 unsigned IntID);
4938 llvm::Value *EmitSMELdrStr(const SVETypeFlags &TypeFlags,
4939 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4940 unsigned IntID);
4941
4942 void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E,
4943 SmallVectorImpl<llvm::Value *> &Ops,
4944 SVETypeFlags TypeFlags);
4945
4946 llvm::Value *EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4947
4948 llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4949 llvm::Triple::ArchType Arch);
4950 llvm::Value *EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4951
4952 llvm::Value *BuildVector(ArrayRef<llvm::Value *> Ops);
4953 llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4954 llvm::Value *EmitPPCBuiltinCpu(unsigned BuiltinID, llvm::Type *ReturnType,
4955 StringRef CPUStr);
4956 llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4957 llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4958 llvm::Value *EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4959 ReturnValueSlot ReturnValue);
4960
4961 // Returns a builtin function that the SPIR-V backend will expand into a spec
4962 // constant.
4963 llvm::Function *
4964 getSpecConstantFunction(const clang::QualType &SpecConstantType);
4965
4966 llvm::Value *EmitDirectXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4967 llvm::Value *EmitSPIRVBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4968 llvm::Value *EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx,
4969 const CallExpr *E);
4970 llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4971 llvm::Value *EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4972 llvm::Value *EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
4973 const CallExpr *E);
4974 llvm::Value *EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4975 llvm::Value *EmitAVRBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4976 llvm::Value *EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4977 ReturnValueSlot ReturnValue);
4978
4979 llvm::Value *EmitRISCVCpuSupports(const CallExpr *E);
4980 llvm::Value *EmitRISCVCpuSupports(ArrayRef<StringRef> FeaturesStrs);
4981 llvm::Value *EmitRISCVCpuInit();
4982 llvm::Value *EmitRISCVCpuIs(const CallExpr *E);
4983 llvm::Value *EmitRISCVCpuIs(StringRef CPUStr);
4984
4985 void AddAMDGPUFenceAddressSpaceMMRA(llvm::Instruction *Inst,
4986 const CallExpr *E);
4987 void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope,
4988 llvm::AtomicOrdering &AO,
4989 llvm::SyncScope::ID &SSID);
4990
4991 enum class MSVCIntrin;
4992 llvm::Value *EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E);
4993
4994 llvm::Value *EmitBuiltinAvailable(const VersionTuple &Version);
4995
4996 llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E);
4997 llvm::Value *EmitObjCStringLiteral(const ObjCStringLiteral *E);
4998 llvm::Value *EmitObjCBoxedExpr(const ObjCBoxedExpr *E);
4999 llvm::Value *EmitObjCArrayLiteral(const ObjCArrayLiteral *E);
5000 llvm::Value *EmitObjCDictionaryLiteral(const ObjCDictionaryLiteral *E);
5001 llvm::Value *
5002 EmitObjCCollectionLiteral(const Expr *E,
5003 const ObjCMethodDecl *MethodWithObjects);
5004 llvm::Value *EmitObjCSelectorExpr(const ObjCSelectorExpr *E);
5005 RValue EmitObjCMessageExpr(const ObjCMessageExpr *E,
5006 ReturnValueSlot Return = ReturnValueSlot());
5007
5008 /// Retrieves the default cleanup kind for an ARC cleanup.
5009 /// Except under -fobjc-arc-eh, ARC cleanups are normal-only.
5010 CleanupKind getARCCleanupKind() {
5011 return CGM.getCodeGenOpts().ObjCAutoRefCountExceptions ? NormalAndEHCleanup
5012 : NormalCleanup;
5013 }
5014
5015 // ARC primitives.
5016 void EmitARCInitWeak(Address addr, llvm::Value *value);
5017 void EmitARCDestroyWeak(Address addr);
5018 llvm::Value *EmitARCLoadWeak(Address addr);
5019 llvm::Value *EmitARCLoadWeakRetained(Address addr);
5020 llvm::Value *EmitARCStoreWeak(Address addr, llvm::Value *value, bool ignored);
5021 void emitARCCopyAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
5022 void emitARCMoveAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
5023 void EmitARCCopyWeak(Address dst, Address src);
5024 void EmitARCMoveWeak(Address dst, Address src);
5025 llvm::Value *EmitARCRetainAutorelease(QualType type, llvm::Value *value);
5026 llvm::Value *EmitARCRetainAutoreleaseNonBlock(llvm::Value *value);
5027 llvm::Value *EmitARCStoreStrong(LValue lvalue, llvm::Value *value,
5028 bool resultIgnored);
5029 llvm::Value *EmitARCStoreStrongCall(Address addr, llvm::Value *value,
5030 bool resultIgnored);
5031 llvm::Value *EmitARCRetain(QualType type, llvm::Value *value);
5032 llvm::Value *EmitARCRetainNonBlock(llvm::Value *value);
5033 llvm::Value *EmitARCRetainBlock(llvm::Value *value, bool mandatory);
5034 void EmitARCDestroyStrong(Address addr, ARCPreciseLifetime_t precise);
5035 void EmitARCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
5036 llvm::Value *EmitARCAutorelease(llvm::Value *value);
5037 llvm::Value *EmitARCAutoreleaseReturnValue(llvm::Value *value);
5038 llvm::Value *EmitARCRetainAutoreleaseReturnValue(llvm::Value *value);
5039 llvm::Value *EmitARCRetainAutoreleasedReturnValue(llvm::Value *value);
5040 llvm::Value *EmitARCUnsafeClaimAutoreleasedReturnValue(llvm::Value *value);
5041
5042 llvm::Value *EmitObjCAutorelease(llvm::Value *value, llvm::Type *returnType);
5043 llvm::Value *EmitObjCRetainNonBlock(llvm::Value *value,
5044 llvm::Type *returnType);
5045 void EmitObjCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
5046
5047 std::pair<LValue, llvm::Value *>
5048 EmitARCStoreAutoreleasing(const BinaryOperator *e);
5049 std::pair<LValue, llvm::Value *> EmitARCStoreStrong(const BinaryOperator *e,
5050 bool ignored);
5051 std::pair<LValue, llvm::Value *>
5052 EmitARCStoreUnsafeUnretained(const BinaryOperator *e, bool ignored);
5053
5054 llvm::Value *EmitObjCAlloc(llvm::Value *value, llvm::Type *returnType);
5055 llvm::Value *EmitObjCAllocWithZone(llvm::Value *value,
5056 llvm::Type *returnType);
5057 llvm::Value *EmitObjCAllocInit(llvm::Value *value, llvm::Type *resultType);
5058
5059 llvm::Value *EmitObjCThrowOperand(const Expr *expr);
5060 llvm::Value *EmitObjCConsumeObject(QualType T, llvm::Value *Ptr);
5061 llvm::Value *EmitObjCExtendObjectLifetime(QualType T, llvm::Value *Ptr);
5062
5063 llvm::Value *EmitARCExtendBlockObject(const Expr *expr);
5064 llvm::Value *EmitARCReclaimReturnedObject(const Expr *e,
5065 bool allowUnsafeClaim);
5066 llvm::Value *EmitARCRetainScalarExpr(const Expr *expr);
5067 llvm::Value *EmitARCRetainAutoreleaseScalarExpr(const Expr *expr);
5068 llvm::Value *EmitARCUnsafeUnretainedScalarExpr(const Expr *expr);
5069
5070 void EmitARCIntrinsicUse(ArrayRef<llvm::Value *> values);
5071
5072 void EmitARCNoopIntrinsicUse(ArrayRef<llvm::Value *> values);
5073
5074 static Destroyer destroyARCStrongImprecise;
5075 static Destroyer destroyARCStrongPrecise;
5076 static Destroyer destroyARCWeak;
5077 static Destroyer emitARCIntrinsicUse;
5078 static Destroyer destroyNonTrivialCStruct;
5079
5080 void EmitObjCAutoreleasePoolPop(llvm::Value *Ptr);
5081 llvm::Value *EmitObjCAutoreleasePoolPush();
5082 llvm::Value *EmitObjCMRRAutoreleasePoolPush();
5083 void EmitObjCAutoreleasePoolCleanup(llvm::Value *Ptr);
5084 void EmitObjCMRRAutoreleasePoolPop(llvm::Value *Ptr);
5085
5086 /// Emits a reference binding to the passed in expression.
5087 RValue EmitReferenceBindingToExpr(const Expr *E);
5088
5089 //===--------------------------------------------------------------------===//
5090 // Expression Emission
5091 //===--------------------------------------------------------------------===//
5092
5093 // Expressions are broken into three classes: scalar, complex, aggregate.
5094
5095 /// EmitScalarExpr - Emit the computation of the specified expression of LLVM
5096 /// scalar type, returning the result.
5097 llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
5098
5099 /// Emit a conversion from the specified type to the specified destination
5100 /// type, both of which are LLVM scalar types.
5101 llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
5102 QualType DstTy, SourceLocation Loc);
5103
5104 /// Emit a conversion from the specified complex type to the specified
5105 /// destination type, where the destination type is an LLVM scalar type.
5106 llvm::Value *EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy,
5107 QualType DstTy,
5108 SourceLocation Loc);
5109
5110 /// EmitAggExpr - Emit the computation of the specified expression
5111 /// of aggregate type. The result is computed into the given slot,
5112 /// which may be null to indicate that the value is not needed.
5113 void EmitAggExpr(const Expr *E, AggValueSlot AS);
5114
5115 /// EmitAggExprToLValue - Emit the computation of the specified expression of
5116 /// aggregate type into a temporary LValue.
5117 LValue EmitAggExprToLValue(const Expr *E);
5118
5119 enum ExprValueKind { EVK_RValue, EVK_NonRValue };
5120
5121 /// EmitAggFinalDestCopy - Emit copy of the specified aggregate into
5122 /// destination address.
5123 void EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest, const LValue &Src,
5124 ExprValueKind SrcKind);
5125
5126 /// Create a store to \arg DstPtr from \arg Src, truncating the stored value
5127 /// to at most \arg DstSize bytes.
5128 void CreateCoercedStore(llvm::Value *Src, QualType SrcFETy, Address Dst,
5129 llvm::TypeSize DstSize, bool DstIsVolatile);
5130
5131 /// EmitExtendGCLifetime - Given a pointer to an Objective-C object,
5132 /// make sure it survives garbage collection until this point.
5133 void EmitExtendGCLifetime(llvm::Value *object);
5134
5135 /// EmitComplexExpr - Emit the computation of the specified expression of
5136 /// complex type, returning the result.
5137 ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal = false,
5138 bool IgnoreImag = false);
5139
5140 /// EmitComplexExprIntoLValue - Emit the given expression of complex
5141 /// type and place its result into the specified l-value.
5142 void EmitComplexExprIntoLValue(const Expr *E, LValue dest, bool isInit);
5143
5144 /// EmitStoreOfComplex - Store a complex number into the specified l-value.
5145 void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit);
5146
5147 /// EmitLoadOfComplex - Load a complex number from the specified l-value.
5148 ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc);
5149
5150 ComplexPairTy EmitPromotedComplexExpr(const Expr *E, QualType PromotionType);
5151 llvm::Value *EmitPromotedScalarExpr(const Expr *E, QualType PromotionType);
5152 ComplexPairTy EmitPromotedValue(ComplexPairTy result, QualType PromotionType);
5153 ComplexPairTy EmitUnPromotedValue(ComplexPairTy result,
5154 QualType PromotionType);
5155
5156 Address emitAddrOfRealComponent(Address complex, QualType complexType);
5157 Address emitAddrOfImagComponent(Address complex, QualType complexType);
5158
5159 /// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the
5160 /// global variable that has already been created for it. If the initializer
5161 /// has a different type than GV does, this may free GV and return a different
5162 /// one. Otherwise it just returns GV.
5163 llvm::GlobalVariable *AddInitializerToStaticVarDecl(const VarDecl &D,
5164 llvm::GlobalVariable *GV);
5165
5166 // Emit an @llvm.invariant.start call for the given memory region.
5167 void EmitInvariantStart(llvm::Constant *Addr, CharUnits Size);
5168
5169 /// EmitCXXGlobalVarDeclInit - Create the initializer for a C++
5170 /// variable with global storage.
5171 void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::GlobalVariable *GV,
5172 bool PerformInit);
5173
5174 llvm::Constant *createAtExitStub(const VarDecl &VD, llvm::FunctionCallee Dtor,
5175 llvm::Constant *Addr);
5176
5177 llvm::Function *createTLSAtExitStub(const VarDecl &VD,
5178 llvm::FunctionCallee Dtor,
5179 llvm::Constant *Addr,
5180 llvm::FunctionCallee &AtExit);
5181
5182 /// Call atexit() with a function that passes the given argument to
5183 /// the given function.
5184 void registerGlobalDtorWithAtExit(const VarDecl &D, llvm::FunctionCallee fn,
5185 llvm::Constant *addr);
5186
5187 /// Registers the dtor using 'llvm.global_dtors' for platforms that do not
5188 /// support an 'atexit()' function.
5189 void registerGlobalDtorWithLLVM(const VarDecl &D, llvm::FunctionCallee fn,
5190 llvm::Constant *addr);
5191
5192 /// Call atexit() with function dtorStub.
5193 void registerGlobalDtorWithAtExit(llvm::Constant *dtorStub);
5194
5195 /// Call unatexit() with function dtorStub.
5196 llvm::Value *unregisterGlobalDtorWithUnAtExit(llvm::Constant *dtorStub);
5197
5198 /// Emit code in this function to perform a guarded variable
5199 /// initialization. Guarded initializations are used when it's not
5200 /// possible to prove that an initialization will be done exactly
5201 /// once, e.g. with a static local variable or a static data member
5202 /// of a class template.
5203 void EmitCXXGuardedInit(const VarDecl &D, llvm::GlobalVariable *DeclPtr,
5204 bool PerformInit);
5205
5206 enum class GuardKind { VariableGuard, TlsGuard };
5207
5208 /// Emit a branch to select whether or not to perform guarded initialization.
5209 void EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
5210 llvm::BasicBlock *InitBlock,
5211 llvm::BasicBlock *NoInitBlock, GuardKind Kind,
5212 const VarDecl *D);
5213
5214 /// GenerateCXXGlobalInitFunc - Generates code for initializing global
5215 /// variables.
5216 void
5217 GenerateCXXGlobalInitFunc(llvm::Function *Fn,
5218 ArrayRef<llvm::Function *> CXXThreadLocals,
5219 ConstantAddress Guard = ConstantAddress::invalid());
5220
5221 /// GenerateCXXGlobalCleanUpFunc - Generates code for cleaning up global
5222 /// variables.
5223 void GenerateCXXGlobalCleanUpFunc(
5224 llvm::Function *Fn,
5225 ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
5226 llvm::Constant *>>
5227 DtorsOrStermFinalizers);
5228
5229 void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn, const VarDecl *D,
5230 llvm::GlobalVariable *Addr,
5231 bool PerformInit);
5232
5233 void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest);
5234
5235 void EmitSynthesizedCXXCopyCtor(Address Dest, Address Src, const Expr *Exp);
5236
5237 void EmitCXXThrowExpr(const CXXThrowExpr *E, bool KeepInsertionPoint = true);
5238
5239 RValue EmitAtomicExpr(AtomicExpr *E);
5240
5241 void EmitFakeUse(Address Addr);
5242
5243 //===--------------------------------------------------------------------===//
5244 // Annotations Emission
5245 //===--------------------------------------------------------------------===//
5246
5247 /// Emit an annotation call (intrinsic).
5248 llvm::Value *EmitAnnotationCall(llvm::Function *AnnotationFn,
5249 llvm::Value *AnnotatedVal,
5250 StringRef AnnotationStr,
5251 SourceLocation Location,
5252 const AnnotateAttr *Attr);
5253
5254 /// Emit local annotations for the local variable V, declared by D.
5255 void EmitVarAnnotations(const VarDecl *D, llvm::Value *V);
5256
5257 /// Emit field annotations for the given field & value. Returns the
5258 /// annotation result.
5259 Address EmitFieldAnnotations(const FieldDecl *D, Address V);
5260
5261 //===--------------------------------------------------------------------===//
5262 // Internal Helpers
5263 //===--------------------------------------------------------------------===//
5264
5265 /// ContainsLabel - Return true if the statement contains a label in it. If
5266 /// this statement is not executed normally, it not containing a label means
5267 /// that we can just remove the code.
5268 static bool ContainsLabel(const Stmt *S, bool IgnoreCaseStmts = false);
5269
5270 /// containsBreak - Return true if the statement contains a break out of it.
5271 /// If the statement (recursively) contains a switch or loop with a break
5272 /// inside of it, this is fine.
5273 static bool containsBreak(const Stmt *S);
5274
5275 /// Determine if the given statement might introduce a declaration into the
5276 /// current scope, by being a (possibly-labelled) DeclStmt.
5277 static bool mightAddDeclToScope(const Stmt *S);
5278
5279 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
5280 /// to a constant, or if it does but contains a label, return false. If it
5281 /// constant folds return true and set the boolean result in Result.
5282 bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result,
5283 bool AllowLabels = false);
5284
5285 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
5286 /// to a constant, or if it does but contains a label, return false. If it
5287 /// constant folds return true and set the folded value.
5288 bool ConstantFoldsToSimpleInteger(const Expr *Cond, llvm::APSInt &Result,
5289 bool AllowLabels = false);
5290
5291 /// Ignore parentheses and logical-NOT to track conditions consistently.
5292 static const Expr *stripCond(const Expr *C);
5293
5294 /// isInstrumentedCondition - Determine whether the given condition is an
5295 /// instrumentable condition (i.e. no "&&" or "||").
5296 static bool isInstrumentedCondition(const Expr *C);
5297
5298 /// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
5299 /// increments a profile counter based on the semantics of the given logical
5300 /// operator opcode. This is used to instrument branch condition coverage
5301 /// for logical operators.
5302 void EmitBranchToCounterBlock(const Expr *Cond, BinaryOperator::Opcode LOp,
5303 llvm::BasicBlock *TrueBlock,
5304 llvm::BasicBlock *FalseBlock,
5305 uint64_t TrueCount = 0,
5306 Stmt::Likelihood LH = Stmt::LH_None,
5307 const Expr *CntrIdx = nullptr);
5308
5309 /// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an
5310 /// if statement) to the specified blocks. Based on the condition, this might
5311 /// try to simplify the codegen of the conditional based on the branch.
5312 /// TrueCount should be the number of times we expect the condition to
5313 /// evaluate to true based on PGO data.
5314 void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock,
5315 llvm::BasicBlock *FalseBlock, uint64_t TrueCount,
5316 Stmt::Likelihood LH = Stmt::LH_None,
5317 const Expr *ConditionalOp = nullptr,
5318 const VarDecl *ConditionalDecl = nullptr);
5319
5320 /// Given an assignment `*LHS = RHS`, emit a test that checks if \p RHS is
5321 /// nonnull, if \p LHS is marked _Nonnull.
5322 void EmitNullabilityCheck(LValue LHS, llvm::Value *RHS, SourceLocation Loc);
5323
5324 /// An enumeration which makes it easier to specify whether or not an
5325 /// operation is a subtraction.
5326 enum { NotSubtraction = false, IsSubtraction = true };
5327
5328 /// Emit pointer + index arithmetic.
5329 llvm::Value *EmitPointerArithmetic(const BinaryOperator *BO,
5330 Expr *pointerOperand, llvm::Value *pointer,
5331 Expr *indexOperand, llvm::Value *index,
5332 bool isSubtraction);
5333
5334 /// Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to
5335 /// detect undefined behavior when the pointer overflow sanitizer is enabled.
5336 /// \p SignedIndices indicates whether any of the GEP indices are signed.
5337 /// \p IsSubtraction indicates whether the expression used to form the GEP
5338 /// is a subtraction.
5339 llvm::Value *EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr,
5340 ArrayRef<llvm::Value *> IdxList,
5341 bool SignedIndices, bool IsSubtraction,
5342 SourceLocation Loc,
5343 const Twine &Name = "");
5344
5345 Address EmitCheckedInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
5346 llvm::Type *elementType, bool SignedIndices,
5347 bool IsSubtraction, SourceLocation Loc,
5348 CharUnits Align, const Twine &Name = "");
5349
5350 /// Specifies which type of sanitizer check to apply when handling a
5351 /// particular builtin.
5352 enum BuiltinCheckKind {
5353 BCK_CTZPassedZero,
5354 BCK_CLZPassedZero,
5355 BCK_AssumePassedFalse,
5356 };
5357
5358 /// Emits an argument for a call to a builtin. If the builtin sanitizer is
5359 /// enabled, a runtime check specified by \p Kind is also emitted.
5360 llvm::Value *EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind);
5361
5362 /// Emits an argument for a call to a `__builtin_assume`. If the builtin
5363 /// sanitizer is enabled, a runtime check is also emitted.
5364 llvm::Value *EmitCheckedArgForAssume(const Expr *E);
5365
5366 /// Emit a description of a type in a format suitable for passing to
5367 /// a runtime sanitizer handler.
5368 llvm::Constant *EmitCheckTypeDescriptor(QualType T);
5369
5370 /// Convert a value into a format suitable for passing to a runtime
5371 /// sanitizer handler.
5372 llvm::Value *EmitCheckValue(llvm::Value *V);
5373
5374 /// Emit a description of a source location in a format suitable for
5375 /// passing to a runtime sanitizer handler.
5376 llvm::Constant *EmitCheckSourceLocation(SourceLocation Loc);
5377
5378 void EmitKCFIOperandBundle(const CGCallee &Callee,
5379 SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
5380
5381 /// Create a basic block that will either trap or call a handler function in
5382 /// the UBSan runtime with the provided arguments, and create a conditional
5383 /// branch to it.
5384 void
5385 EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
5386 Checked,
5387 SanitizerHandler Check, ArrayRef<llvm::Constant *> StaticArgs,
5388 ArrayRef<llvm::Value *> DynamicArgs,
5389 const TrapReason *TR = nullptr);
5390
5391 /// Emit a slow path cross-DSO CFI check which calls __cfi_slowpath
5392 /// if Cond if false.
5393 void EmitCfiSlowPathCheck(SanitizerKind::SanitizerOrdinal Ordinal,
5394 llvm::Value *Cond, llvm::ConstantInt *TypeId,
5395 llvm::Value *Ptr,
5396 ArrayRef<llvm::Constant *> StaticArgs);
5397
5398 /// Emit a reached-unreachable diagnostic if \p Loc is valid and runtime
5399 /// checking is enabled. Otherwise, just emit an unreachable instruction.
5400 void EmitUnreachable(SourceLocation Loc);
5401
5402 /// Create a basic block that will call the trap intrinsic, and emit a
5403 /// conditional branch to it, for the -ftrapv checks.
5404 void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID,
5405 bool NoMerge = false, const TrapReason *TR = nullptr);
5406
5407 /// Emit a call to trap or debugtrap and attach function attribute
5408 /// "trap-func-name" if specified.
5409 llvm::CallInst *EmitTrapCall(llvm::Intrinsic::ID IntrID);
5410
5411 /// Emit a stub for the cross-DSO CFI check function.
5412 void EmitCfiCheckStub();
5413
5414 /// Emit a cross-DSO CFI failure handling function.
5415 void EmitCfiCheckFail();
5416
5417 /// Create a check for a function parameter that may potentially be
5418 /// declared as non-null.
5419 void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc,
5420 AbstractCallee AC, unsigned ParmNum);
5421
5422 void EmitNonNullArgCheck(Address Addr, QualType ArgType,
5423 SourceLocation ArgLoc, AbstractCallee AC,
5424 unsigned ParmNum);
5425
5426 /// EmitWriteback - Emit callbacks for function.
5427 void EmitWritebacks(const CallArgList &Args);
5428
5429 /// EmitCallArg - Emit a single call argument.
5430 void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType);
5431
5432 /// EmitDelegateCallArg - We are performing a delegate call; that
5433 /// is, the current function is delegating to another one. Produce
5434 /// a r-value suitable for passing the given parameter.
5435 void EmitDelegateCallArg(CallArgList &args, const VarDecl *param,
5436 SourceLocation loc);
5437
5438 /// SetFPAccuracy - Set the minimum required accuracy of the given floating
5439 /// point operation, expressed as the maximum relative error in ulp.
5440 void SetFPAccuracy(llvm::Value *Val, float Accuracy);
5441
5442 /// Set the minimum required accuracy of the given sqrt operation
5443 /// based on CodeGenOpts.
5444 void SetSqrtFPAccuracy(llvm::Value *Val);
5445
5446 /// Set the minimum required accuracy of the given sqrt operation based on
5447 /// CodeGenOpts.
5448 void SetDivFPAccuracy(llvm::Value *Val);
5449
5450 /// Set the codegen fast-math flags.
5451 void SetFastMathFlags(FPOptions FPFeatures);
5452
5453 // Truncate or extend a boolean vector to the requested number of elements.
5454 llvm::Value *emitBoolVecConversion(llvm::Value *SrcVec,
5455 unsigned NumElementsDst,
5456 const llvm::Twine &Name = "");
5457
5458 void maybeAttachRangeForLoad(llvm::LoadInst *Load, QualType Ty,
5459 SourceLocation Loc);
5460
5461 // Emits a convergence_loop instruction for the given |BB|, with |ParentToken|
5462 // as it's parent convergence instr.
5463 llvm::ConvergenceControlInst *emitConvergenceLoopToken(llvm::BasicBlock *BB);
5464
5465private:
5466 // Adds a convergence_ctrl token with |ParentToken| as parent convergence
5467 // instr to the call |Input|.
5468 llvm::CallBase *addConvergenceControlToken(llvm::CallBase *Input);
5469
5470 // Find the convergence_entry instruction |F|, or emits ones if none exists.
5471 // Returns the convergence instruction.
5472 llvm::ConvergenceControlInst *
5473 getOrEmitConvergenceEntryToken(llvm::Function *F);
5474
5475private:
5476 llvm::MDNode *getRangeForLoadFromType(QualType Ty);
5477 void EmitReturnOfRValue(RValue RV, QualType Ty);
5478
5479 void deferPlaceholderReplacement(llvm::Instruction *Old, llvm::Value *New);
5480
5481 llvm::SmallVector<std::pair<llvm::WeakTrackingVH, llvm::Value *>, 4>
5482 DeferredReplacements;
5483
5484 /// Set the address of a local variable.
5485 void setAddrOfLocalVar(const VarDecl *VD, Address Addr) {
5486 assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!");
5487 LocalDeclMap.insert(KV: {VD, Addr});
5488 }
5489
5490 /// ExpandTypeFromArgs - Reconstruct a structure of type \arg Ty
5491 /// from function arguments into \arg Dst. See ABIArgInfo::Expand.
5492 ///
5493 /// \param AI - The first function argument of the expansion.
5494 void ExpandTypeFromArgs(QualType Ty, LValue Dst,
5495 llvm::Function::arg_iterator &AI);
5496
5497 /// ExpandTypeToArgs - Expand an CallArg \arg Arg, with the LLVM type for \arg
5498 /// Ty, into individual arguments on the provided vector \arg IRCallArgs,
5499 /// starting at index \arg IRCallArgPos. See ABIArgInfo::Expand.
5500 void ExpandTypeToArgs(QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
5501 SmallVectorImpl<llvm::Value *> &IRCallArgs,
5502 unsigned &IRCallArgPos);
5503
5504 std::pair<llvm::Value *, llvm::Type *>
5505 EmitAsmInput(const TargetInfo::ConstraintInfo &Info, const Expr *InputExpr,
5506 std::string &ConstraintStr);
5507
5508 std::pair<llvm::Value *, llvm::Type *>
5509 EmitAsmInputLValue(const TargetInfo::ConstraintInfo &Info, LValue InputValue,
5510 QualType InputType, std::string &ConstraintStr,
5511 SourceLocation Loc);
5512
5513 /// Attempts to statically evaluate the object size of E. If that
5514 /// fails, emits code to figure the size of E out for us. This is
5515 /// pass_object_size aware.
5516 ///
5517 /// If EmittedExpr is non-null, this will use that instead of re-emitting E.
5518 llvm::Value *evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
5519 llvm::IntegerType *ResType,
5520 llvm::Value *EmittedE,
5521 bool IsDynamic);
5522
5523 /// Emits the size of E, as required by __builtin_object_size. This
5524 /// function is aware of pass_object_size parameters, and will act accordingly
5525 /// if E is a parameter with the pass_object_size attribute.
5526 llvm::Value *emitBuiltinObjectSize(const Expr *E, unsigned Type,
5527 llvm::IntegerType *ResType,
5528 llvm::Value *EmittedE, bool IsDynamic);
5529
5530 llvm::Value *emitCountedBySize(const Expr *E, llvm::Value *EmittedE,
5531 unsigned Type, llvm::IntegerType *ResType);
5532
5533 llvm::Value *emitCountedByMemberSize(const MemberExpr *E, const Expr *Idx,
5534 llvm::Value *EmittedE,
5535 QualType CastedArrayElementTy,
5536 unsigned Type,
5537 llvm::IntegerType *ResType);
5538
5539 llvm::Value *emitCountedByPointerSize(const ImplicitCastExpr *E,
5540 const Expr *Idx, llvm::Value *EmittedE,
5541 QualType CastedArrayElementTy,
5542 unsigned Type,
5543 llvm::IntegerType *ResType);
5544
5545 void emitZeroOrPatternForAutoVarInit(QualType type, const VarDecl &D,
5546 Address Loc);
5547
5548public:
5549 enum class EvaluationOrder {
5550 ///! No language constraints on evaluation order.
5551 Default,
5552 ///! Language semantics require left-to-right evaluation.
5553 ForceLeftToRight,
5554 ///! Language semantics require right-to-left evaluation.
5555 ForceRightToLeft
5556 };
5557
5558 // Wrapper for function prototype sources. Wraps either a FunctionProtoType or
5559 // an ObjCMethodDecl.
5560 struct PrototypeWrapper {
5561 llvm::PointerUnion<const FunctionProtoType *, const ObjCMethodDecl *> P;
5562
5563 PrototypeWrapper(const FunctionProtoType *FT) : P(FT) {}
5564 PrototypeWrapper(const ObjCMethodDecl *MD) : P(MD) {}
5565 };
5566
5567 void EmitCallArgs(CallArgList &Args, PrototypeWrapper Prototype,
5568 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
5569 AbstractCallee AC = AbstractCallee(),
5570 unsigned ParamsToSkip = 0,
5571 EvaluationOrder Order = EvaluationOrder::Default);
5572
5573 /// EmitPointerWithAlignment - Given an expression with a pointer type,
5574 /// emit the value and compute our best estimate of the alignment of the
5575 /// pointee.
5576 ///
5577 /// \param BaseInfo - If non-null, this will be initialized with
5578 /// information about the source of the alignment and the may-alias
5579 /// attribute. Note that this function will conservatively fall back on
5580 /// the type when it doesn't recognize the expression and may-alias will
5581 /// be set to false.
5582 ///
5583 /// One reasonable way to use this information is when there's a language
5584 /// guarantee that the pointer must be aligned to some stricter value, and
5585 /// we're simply trying to ensure that sufficiently obvious uses of under-
5586 /// aligned objects don't get miscompiled; for example, a placement new
5587 /// into the address of a local variable. In such a case, it's quite
5588 /// reasonable to just ignore the returned alignment when it isn't from an
5589 /// explicit source.
5590 Address
5591 EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo = nullptr,
5592 TBAAAccessInfo *TBAAInfo = nullptr,
5593 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
5594
5595 /// If \p E references a parameter with pass_object_size info or a constant
5596 /// array size modifier, emit the object size divided by the size of \p EltTy.
5597 /// Otherwise return null.
5598 llvm::Value *LoadPassedObjectSize(const Expr *E, QualType EltTy);
5599
5600 void EmitSanitizerStatReport(llvm::SanitizerStatKind SSK);
5601
5602 struct FMVResolverOption {
5603 llvm::Function *Function;
5604 llvm::SmallVector<StringRef, 8> Features;
5605 std::optional<StringRef> Architecture;
5606
5607 FMVResolverOption(llvm::Function *F, ArrayRef<StringRef> Feats,
5608 std::optional<StringRef> Arch = std::nullopt)
5609 : Function(F), Features(Feats), Architecture(Arch) {}
5610 };
5611
5612 // Emits the body of a multiversion function's resolver. Assumes that the
5613 // options are already sorted in the proper order, with the 'default' option
5614 // last (if it exists).
5615 void EmitMultiVersionResolver(llvm::Function *Resolver,
5616 ArrayRef<FMVResolverOption> Options);
5617 void EmitX86MultiVersionResolver(llvm::Function *Resolver,
5618 ArrayRef<FMVResolverOption> Options);
5619 void EmitAArch64MultiVersionResolver(llvm::Function *Resolver,
5620 ArrayRef<FMVResolverOption> Options);
5621 void EmitRISCVMultiVersionResolver(llvm::Function *Resolver,
5622 ArrayRef<FMVResolverOption> Options);
5623 void EmitPPCAIXMultiVersionResolver(llvm::Function *Resolver,
5624 ArrayRef<FMVResolverOption> Options);
5625
5626 Address EmitAddressOfPFPField(Address RecordPtr, const PFPField &Field);
5627 Address EmitAddressOfPFPField(Address RecordPtr, Address FieldPtr,
5628 const FieldDecl *Field);
5629
5630private:
5631 QualType getVarArgType(const Expr *Arg);
5632
5633 void EmitDeclMetadata();
5634
5635 BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType,
5636 const AutoVarEmission &emission);
5637
5638 void AddObjCARCExceptionMetadata(llvm::Instruction *Inst);
5639
5640 llvm::Value *GetValueForARMHint(unsigned BuiltinID);
5641 llvm::Value *EmitX86CpuIs(const CallExpr *E);
5642 llvm::Value *EmitX86CpuIs(StringRef CPUStr);
5643 llvm::Value *EmitX86CpuSupports(const CallExpr *E);
5644 llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs);
5645 llvm::Value *EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask);
5646 llvm::Value *EmitX86CpuInit();
5647 llvm::Value *FormX86ResolverCondition(const FMVResolverOption &RO);
5648 llvm::Value *EmitAArch64CpuInit();
5649 llvm::Value *FormAArch64ResolverCondition(const FMVResolverOption &RO);
5650 llvm::Value *EmitAArch64CpuSupports(const CallExpr *E);
5651 llvm::Value *EmitAArch64CpuSupports(ArrayRef<StringRef> FeatureStrs);
5652};
5653
5654inline DominatingLLVMValue::saved_type
5655DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) {
5656 if (!needsSaving(value))
5657 return saved_type(value);
5658
5659 // Otherwise, we need an alloca.
5660 auto align = CharUnits::fromQuantity(
5661 Quantity: CGF.CGM.getDataLayout().getPrefTypeAlign(Ty: value->getType()))
5662 .getAsAlign();
5663 llvm::AllocaInst *AI =
5664 CGF.CreateTempAlloca(Ty: value->getType(), Name: "cond-cleanup.save");
5665 AI->setAlignment(align);
5666 CGF.Builder.CreateAlignedStore(Val: value, Ptr: AI, Align: align);
5667
5668 return saved_type(AI, value->getType());
5669}
5670
5671inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF,
5672 saved_type value) {
5673 // If the value says it wasn't saved, trust that it's still dominating.
5674 if (!value.isSaved())
5675 return value.Value;
5676
5677 // Otherwise, it should be an alloca instruction, as set up in save().
5678 auto Alloca = cast<llvm::AllocaInst>(Val: value.Value);
5679 return CGF.Builder.CreateAlignedLoad(Ty: value.Type, Ptr: Alloca, Align: Alloca->getAlign());
5680}
5681
5682} // end namespace CodeGen
5683
5684// Map the LangOption for floating point exception behavior into
5685// the corresponding enum in the IR.
5686llvm::fp::ExceptionBehavior
5687ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind);
5688} // end namespace clang
5689
5690#endif
5691