1//===-- CodeGenFunction.h - Per-Function state for LLVM CodeGen -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the internal per-function state used for llvm translation.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
14#define LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
15
16#include "CGBuilder.h"
17#include "CGLoopInfo.h"
18#include "CGValue.h"
19#include "CodeGenModule.h"
20#include "EHScopeStack.h"
21#include "SanitizerHandler.h"
22#include "VarBypassDetector.h"
23#include "clang/AST/CharUnits.h"
24#include "clang/AST/CurrentSourceLocExprScope.h"
25#include "clang/AST/ExprCXX.h"
26#include "clang/AST/ExprObjC.h"
27#include "clang/AST/ExprOpenMP.h"
28#include "clang/AST/StmtOpenACC.h"
29#include "clang/AST/StmtOpenMP.h"
30#include "clang/AST/StmtSYCL.h"
31#include "clang/AST/Type.h"
32#include "clang/Basic/ABI.h"
33#include "clang/Basic/CapturedStmt.h"
34#include "clang/Basic/CodeGenOptions.h"
35#include "clang/Basic/OpenMPKinds.h"
36#include "clang/Basic/TargetInfo.h"
37#include "llvm/ADT/ArrayRef.h"
38#include "llvm/ADT/DenseMap.h"
39#include "llvm/ADT/MapVector.h"
40#include "llvm/ADT/SmallVector.h"
41#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
42#include "llvm/IR/Instructions.h"
43#include "llvm/IR/ValueHandle.h"
44#include "llvm/Support/Debug.h"
45#include "llvm/Transforms/Utils/SanitizerStats.h"
46#include <optional>
47
48namespace llvm {
49class BasicBlock;
50class ConvergenceControlInst;
51class LLVMContext;
52class MDNode;
53class SwitchInst;
54class Twine;
55class Value;
56class CanonicalLoopInfo;
57} // namespace llvm
58
59namespace clang {
60class ASTContext;
61class CXXDestructorDecl;
62class CXXForRangeStmt;
63class CXXTryStmt;
64class Decl;
65class LabelDecl;
66class FunctionDecl;
67class FunctionProtoType;
68class LabelStmt;
69class ObjCContainerDecl;
70class ObjCInterfaceDecl;
71class ObjCIvarDecl;
72class ObjCMethodDecl;
73class ObjCImplementationDecl;
74class ObjCPropertyImplDecl;
75class TargetInfo;
76class VarDecl;
77class ObjCForCollectionStmt;
78class ObjCAtTryStmt;
79class ObjCAtThrowStmt;
80class ObjCAtSynchronizedStmt;
81class ObjCAutoreleasePoolStmt;
82class OMPUseDevicePtrClause;
83class OMPUseDeviceAddrClause;
84class SVETypeFlags;
85class OMPExecutableDirective;
86
87namespace analyze_os_log {
88class OSLogBufferLayout;
89}
90
91namespace CodeGen {
92class CodeGenTypes;
93class CodeGenPGO;
94class CGCallee;
95class CGFunctionInfo;
96class CGBlockInfo;
97class CGCXXABI;
98class BlockByrefHelpers;
99class BlockByrefInfo;
100class BlockFieldFlags;
101class RegionCodeGenTy;
102class TargetCodeGenInfo;
103struct OMPTaskDataTy;
104struct CGCoroData;
105
106// clang-format off
107/// The kind of evaluation to perform on values of a particular
108/// type. Basically, is the code in CGExprScalar, CGExprComplex, or
109/// CGExprAgg?
110///
111/// TODO: should vectors maybe be split out into their own thing?
112enum TypeEvaluationKind {
113 TEK_Scalar,
114 TEK_Complex,
115 TEK_Aggregate
116};
117// clang-format on
118
119/// Helper class with most of the code for saving a value for a
120/// conditional expression cleanup.
121struct DominatingLLVMValue {
122 struct saved_type {
123 llvm::Value *Value; // Original value if not saved, alloca if saved
124 llvm::Type *Type; // nullptr if not saved, element type if saved
125
126 saved_type() : Value(nullptr), Type(nullptr) {}
127 saved_type(llvm::Value *V) : Value(V), Type(nullptr) {}
128 saved_type(llvm::AllocaInst *Alloca, llvm::Type *Ty)
129 : Value(Alloca), Type(Ty) {}
130
131 bool isSaved() const { return Type != nullptr; }
132 };
133
134 /// Answer whether the given value needs extra work to be saved.
135 static bool needsSaving(llvm::Value *value) {
136 if (!value)
137 return false;
138
139 // If it's not an instruction, we don't need to save.
140 if (!isa<llvm::Instruction>(Val: value))
141 return false;
142
143 // If it's an instruction in the entry block, we don't need to save.
144 llvm::BasicBlock *block = cast<llvm::Instruction>(Val: value)->getParent();
145 return (block != &block->getParent()->getEntryBlock());
146 }
147
148 static saved_type save(CodeGenFunction &CGF, llvm::Value *value);
149 static llvm::Value *restore(CodeGenFunction &CGF, saved_type value);
150};
151
152/// A partial specialization of DominatingValue for llvm::Values that
153/// might be llvm::Instructions.
154template <class T> struct DominatingPointer<T, true> : DominatingLLVMValue {
155 typedef T *type;
156 static type restore(CodeGenFunction &CGF, saved_type value) {
157 return static_cast<T *>(DominatingLLVMValue::restore(CGF, value));
158 }
159};
160
161/// A specialization of DominatingValue for Address.
162template <> struct DominatingValue<Address> {
163 typedef Address type;
164
165 struct saved_type {
166 DominatingLLVMValue::saved_type BasePtr;
167 llvm::Type *ElementType;
168 CharUnits Alignment;
169 DominatingLLVMValue::saved_type Offset;
170 llvm::PointerType *EffectiveType;
171 };
172
173 static bool needsSaving(type value) {
174 if (DominatingLLVMValue::needsSaving(value: value.getBasePointer()) ||
175 DominatingLLVMValue::needsSaving(value: value.getOffset()))
176 return true;
177 return false;
178 }
179 static saved_type save(CodeGenFunction &CGF, type value) {
180 return {.BasePtr: DominatingLLVMValue::save(CGF, value: value.getBasePointer()),
181 .ElementType: value.getElementType(), .Alignment: value.getAlignment(),
182 .Offset: DominatingLLVMValue::save(CGF, value: value.getOffset()), .EffectiveType: value.getType()};
183 }
184 static type restore(CodeGenFunction &CGF, saved_type value) {
185 return Address(DominatingLLVMValue::restore(CGF, value: value.BasePtr),
186 value.ElementType, value.Alignment, CGPointerAuthInfo(),
187 DominatingLLVMValue::restore(CGF, value: value.Offset));
188 }
189};
190
191/// A specialization of DominatingValue for RValue.
192template <> struct DominatingValue<RValue> {
193 typedef RValue type;
194 class saved_type {
195 enum Kind {
196 ScalarLiteral,
197 ScalarAddress,
198 AggregateLiteral,
199 AggregateAddress,
200 ComplexAddress
201 };
202 union {
203 struct {
204 DominatingLLVMValue::saved_type first, second;
205 } Vals;
206 DominatingValue<Address>::saved_type AggregateAddr;
207 };
208 LLVM_PREFERRED_TYPE(Kind)
209 unsigned K : 3;
210
211 saved_type(DominatingLLVMValue::saved_type Val1, unsigned K)
212 : Vals{.first: Val1, .second: DominatingLLVMValue::saved_type()}, K(K) {}
213
214 saved_type(DominatingLLVMValue::saved_type Val1,
215 DominatingLLVMValue::saved_type Val2)
216 : Vals{.first: Val1, .second: Val2}, K(ComplexAddress) {}
217
218 saved_type(DominatingValue<Address>::saved_type AggregateAddr, unsigned K)
219 : AggregateAddr(AggregateAddr), K(K) {}
220
221 public:
222 static bool needsSaving(RValue value);
223 static saved_type save(CodeGenFunction &CGF, RValue value);
224 RValue restore(CodeGenFunction &CGF);
225
226 // implementations in CGCleanup.cpp
227 };
228
229 static bool needsSaving(type value) { return saved_type::needsSaving(value); }
230 static saved_type save(CodeGenFunction &CGF, type value) {
231 return saved_type::save(CGF, value);
232 }
233 static type restore(CodeGenFunction &CGF, saved_type value) {
234 return value.restore(CGF);
235 }
236};
237
238/// A scoped helper to set the current source atom group for
239/// CGDebugInfo::addInstToCurrentSourceAtom. A source atom is a source construct
240/// that is "interesting" for debug stepping purposes. We use an atom group
241/// number to track the instruction(s) that implement the functionality for the
242/// atom, plus backup instructions/source locations.
243class ApplyAtomGroup {
244 uint64_t OriginalAtom = 0;
245 CGDebugInfo *DI = nullptr;
246
247 ApplyAtomGroup(const ApplyAtomGroup &) = delete;
248 void operator=(const ApplyAtomGroup &) = delete;
249
250public:
251 ApplyAtomGroup(CGDebugInfo *DI);
252 ~ApplyAtomGroup();
253};
254
255/// CodeGenFunction - This class organizes the per-function state that is used
256/// while generating LLVM code.
257class CodeGenFunction : public CodeGenTypeCache {
258 CodeGenFunction(const CodeGenFunction &) = delete;
259 void operator=(const CodeGenFunction &) = delete;
260
261 friend class CGCXXABI;
262
263public:
264 /// A jump destination is an abstract label, branching to which may
265 /// require a jump out through normal cleanups.
266 struct JumpDest {
267 JumpDest() : Block(nullptr), Index(0) {}
268 JumpDest(llvm::BasicBlock *Block, EHScopeStack::stable_iterator Depth,
269 unsigned Index)
270 : Block(Block), ScopeDepth(Depth), Index(Index) {}
271
272 bool isValid() const { return Block != nullptr; }
273 llvm::BasicBlock *getBlock() const { return Block; }
274 EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; }
275 unsigned getDestIndex() const { return Index; }
276
277 // This should be used cautiously.
278 void setScopeDepth(EHScopeStack::stable_iterator depth) {
279 ScopeDepth = depth;
280 }
281
282 private:
283 llvm::BasicBlock *Block;
284 EHScopeStack::stable_iterator ScopeDepth;
285 unsigned Index;
286 };
287
288 CodeGenModule &CGM; // Per-module state.
289 const TargetInfo &Target;
290
291 // For EH/SEH outlined funclets, this field points to parent's CGF
292 CodeGenFunction *ParentCGF = nullptr;
293
294 typedef std::pair<llvm::Value *, llvm::Value *> ComplexPairTy;
295 LoopInfoStack LoopStack;
296 CGBuilderTy Builder;
297
298 // Stores variables for which we can't generate correct lifetime markers
299 // because of jumps.
300 VarBypassDetector Bypasses;
301
302 /// List of recently emitted OMPCanonicalLoops.
303 ///
304 /// Since OMPCanonicalLoops are nested inside other statements (in particular
305 /// CapturedStmt generated by OMPExecutableDirective and non-perfectly nested
306 /// loops), we cannot directly call OMPEmitOMPCanonicalLoop and receive its
307 /// llvm::CanonicalLoopInfo. Instead, we call EmitStmt and any
308 /// OMPEmitOMPCanonicalLoop called by it will add its CanonicalLoopInfo to
309 /// this stack when done. Entering a new loop requires clearing this list; it
310 /// either means we start parsing a new loop nest (in which case the previous
311 /// loop nest goes out of scope) or a second loop in the same level in which
312 /// case it would be ambiguous into which of the two (or more) loops the loop
313 /// nest would extend.
314 SmallVector<llvm::CanonicalLoopInfo *, 4> OMPLoopNestStack;
315
316 /// Stack to track the controlled convergence tokens.
317 SmallVector<llvm::ConvergenceControlInst *, 4> ConvergenceTokenStack;
318
319 /// Number of nested loop to be consumed by the last surrounding
320 /// loop-associated directive.
321 int ExpectedOMPLoopDepth = 0;
322
323 // CodeGen lambda for loops and support for ordered clause
324 typedef llvm::function_ref<void(CodeGenFunction &, const OMPLoopDirective &,
325 JumpDest)>
326 CodeGenLoopTy;
327 typedef llvm::function_ref<void(CodeGenFunction &, SourceLocation,
328 const unsigned, const bool)>
329 CodeGenOrderedTy;
330
331 // Codegen lambda for loop bounds in worksharing loop constructs
332 typedef llvm::function_ref<std::pair<LValue, LValue>(
333 CodeGenFunction &, const OMPExecutableDirective &S)>
334 CodeGenLoopBoundsTy;
335
336 // Codegen lambda for loop bounds in dispatch-based loop implementation
337 typedef llvm::function_ref<std::pair<llvm::Value *, llvm::Value *>(
338 CodeGenFunction &, const OMPExecutableDirective &S, Address LB,
339 Address UB)>
340 CodeGenDispatchBoundsTy;
341
342 /// CGBuilder insert helper. This function is called after an
343 /// instruction is created using Builder.
344 void InsertHelper(llvm::Instruction *I, const llvm::Twine &Name,
345 llvm::BasicBlock::iterator InsertPt) const;
346
347 /// CurFuncDecl - Holds the Decl for the current outermost
348 /// non-closure context.
349 const Decl *CurFuncDecl = nullptr;
350 /// CurCodeDecl - This is the inner-most code context, which includes blocks.
351 const Decl *CurCodeDecl = nullptr;
352 const CGFunctionInfo *CurFnInfo = nullptr;
353 QualType FnRetTy;
354 llvm::Function *CurFn = nullptr;
355
356 /// If a cast expression is being visited, this holds the current cast's
357 /// expression.
358 const CastExpr *CurCast = nullptr;
359
360 /// Save Parameter Decl for coroutine.
361 llvm::SmallVector<const ParmVarDecl *, 4> FnArgs;
362
363 // Holds coroutine data if the current function is a coroutine. We use a
364 // wrapper to manage its lifetime, so that we don't have to define CGCoroData
365 // in this header.
366 struct CGCoroInfo {
367 std::unique_ptr<CGCoroData> Data;
368 bool InSuspendBlock = false;
369 CGCoroInfo();
370 ~CGCoroInfo();
371 };
372 CGCoroInfo CurCoro;
373
374 bool isCoroutine() const { return CurCoro.Data != nullptr; }
375
376 bool inSuspendBlock() const {
377 return isCoroutine() && CurCoro.InSuspendBlock;
378 }
379
380 // Holds FramePtr for await_suspend wrapper generation,
381 // so that __builtin_coro_frame call can be lowered
382 // directly to value of its second argument
383 struct AwaitSuspendWrapperInfo {
384 llvm::Value *FramePtr = nullptr;
385 };
386 AwaitSuspendWrapperInfo CurAwaitSuspendWrapper;
387
388 // Generates wrapper function for `llvm.coro.await.suspend.*` intrinisics.
389 // It encapsulates SuspendExpr in a function, to separate it's body
390 // from the main coroutine to avoid miscompilations. Intrinisic
391 // is lowered to this function call in CoroSplit pass
392 // Function signature is:
393 // <type> __await_suspend_wrapper_<name>(ptr %awaiter, ptr %hdl)
394 // where type is one of (void, i1, ptr)
395 llvm::Function *generateAwaitSuspendWrapper(Twine const &CoroName,
396 Twine const &SuspendPointName,
397 CoroutineSuspendExpr const &S);
398
399 /// CurGD - The GlobalDecl for the current function being compiled.
400 GlobalDecl CurGD;
401
402 /// PrologueCleanupDepth - The cleanup depth enclosing all the
403 /// cleanups associated with the parameters.
404 EHScopeStack::stable_iterator PrologueCleanupDepth;
405
406 /// ReturnBlock - Unified return block.
407 JumpDest ReturnBlock;
408
409 /// ReturnValue - The temporary alloca to hold the return
410 /// value. This is invalid iff the function has no return value.
411 Address ReturnValue = Address::invalid();
412
413 /// ReturnValuePointer - The temporary alloca to hold a pointer to sret.
414 /// This is invalid if sret is not in use.
415 Address ReturnValuePointer = Address::invalid();
416
417 /// If a return statement is being visited, this holds the return statment's
418 /// result expression.
419 const Expr *RetExpr = nullptr;
420
421 /// Return true if a label was seen in the current scope.
422 bool hasLabelBeenSeenInCurrentScope() const {
423 if (CurLexicalScope)
424 return CurLexicalScope->hasLabels();
425 return !LabelMap.empty();
426 }
427
428 /// AllocaInsertPoint - This is an instruction in the entry block before which
429 /// we prefer to insert allocas.
430 llvm::AssertingVH<llvm::Instruction> AllocaInsertPt;
431
432private:
433 /// PostAllocaInsertPt - This is a place in the prologue where code can be
434 /// inserted that will be dominated by all the static allocas. This helps
435 /// achieve two things:
436 /// 1. Contiguity of all static allocas (within the prologue) is maintained.
437 /// 2. All other prologue code (which are dominated by static allocas) do
438 /// appear in the source order immediately after all static allocas.
439 ///
440 /// PostAllocaInsertPt will be lazily created when it is *really* required.
441 llvm::AssertingVH<llvm::Instruction> PostAllocaInsertPt = nullptr;
442
443public:
444 /// Return PostAllocaInsertPt. If it is not yet created, then insert it
445 /// immediately after AllocaInsertPt.
446 llvm::Instruction *getPostAllocaInsertPoint() {
447 if (!PostAllocaInsertPt) {
448 assert(AllocaInsertPt &&
449 "Expected static alloca insertion point at function prologue");
450 assert(AllocaInsertPt->getParent()->isEntryBlock() &&
451 "EBB should be entry block of the current code gen function");
452 PostAllocaInsertPt = AllocaInsertPt->clone();
453 PostAllocaInsertPt->setName("postallocapt");
454 PostAllocaInsertPt->insertAfter(InsertPos: AllocaInsertPt->getIterator());
455 }
456
457 return PostAllocaInsertPt;
458 }
459
460 // Try to preserve the source's name to make IR more readable.
461 llvm::Value *performAddrSpaceCast(llvm::Value *Src, llvm::Type *DestTy) {
462 return Builder.CreateAddrSpaceCast(
463 V: Src, DestTy, Name: Src->hasName() ? Src->getName() + ".ascast" : "");
464 }
465
466 /// API for captured statement code generation.
467 class CGCapturedStmtInfo {
468 public:
469 explicit CGCapturedStmtInfo(CapturedRegionKind K = CR_Default)
470 : Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {}
471 explicit CGCapturedStmtInfo(const CapturedStmt &S,
472 CapturedRegionKind K = CR_Default)
473 : Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {
474
475 RecordDecl::field_iterator Field =
476 S.getCapturedRecordDecl()->field_begin();
477 for (CapturedStmt::const_capture_iterator I = S.capture_begin(),
478 E = S.capture_end();
479 I != E; ++I, ++Field) {
480 if (I->capturesThis())
481 CXXThisFieldDecl = *Field;
482 else if (I->capturesVariable())
483 CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
484 else if (I->capturesVariableByCopy())
485 CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
486 }
487 }
488
489 virtual ~CGCapturedStmtInfo();
490
491 CapturedRegionKind getKind() const { return Kind; }
492
493 virtual void setContextValue(llvm::Value *V) { ThisValue = V; }
494 // Retrieve the value of the context parameter.
495 virtual llvm::Value *getContextValue() const { return ThisValue; }
496
497 /// Lookup the captured field decl for a variable.
498 virtual const FieldDecl *lookup(const VarDecl *VD) const {
499 return CaptureFields.lookup(Val: VD->getCanonicalDecl());
500 }
501
502 bool isCXXThisExprCaptured() const { return getThisFieldDecl() != nullptr; }
503 virtual FieldDecl *getThisFieldDecl() const { return CXXThisFieldDecl; }
504
505 static bool classof(const CGCapturedStmtInfo *) { return true; }
506
507 /// Emit the captured statement body.
508 virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) {
509 CGF.incrementProfileCounter(S);
510 CGF.EmitStmt(S);
511 }
512
513 /// Get the name of the capture helper.
514 virtual StringRef getHelperName() const { return "__captured_stmt"; }
515
516 /// Get the CaptureFields
517 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> getCaptureFields() {
518 return CaptureFields;
519 }
520
521 private:
522 /// The kind of captured statement being generated.
523 CapturedRegionKind Kind;
524
525 /// Keep the map between VarDecl and FieldDecl.
526 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields;
527
528 /// The base address of the captured record, passed in as the first
529 /// argument of the parallel region function.
530 llvm::Value *ThisValue;
531
532 /// Captured 'this' type.
533 FieldDecl *CXXThisFieldDecl;
534 };
535 CGCapturedStmtInfo *CapturedStmtInfo = nullptr;
536
537 /// RAII for correct setting/restoring of CapturedStmtInfo.
538 class CGCapturedStmtRAII {
539 private:
540 CodeGenFunction &CGF;
541 CGCapturedStmtInfo *PrevCapturedStmtInfo;
542
543 public:
544 CGCapturedStmtRAII(CodeGenFunction &CGF,
545 CGCapturedStmtInfo *NewCapturedStmtInfo)
546 : CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo) {
547 CGF.CapturedStmtInfo = NewCapturedStmtInfo;
548 }
549 ~CGCapturedStmtRAII() { CGF.CapturedStmtInfo = PrevCapturedStmtInfo; }
550 };
551
552 /// An abstract representation of regular/ObjC call/message targets.
553 class AbstractCallee {
554 /// The function declaration of the callee.
555 const Decl *CalleeDecl;
556
557 public:
558 AbstractCallee() : CalleeDecl(nullptr) {}
559 AbstractCallee(const FunctionDecl *FD) : CalleeDecl(FD) {}
560 AbstractCallee(const ObjCMethodDecl *OMD) : CalleeDecl(OMD) {}
561 bool hasFunctionDecl() const {
562 return isa_and_nonnull<FunctionDecl>(Val: CalleeDecl);
563 }
564 const Decl *getDecl() const { return CalleeDecl; }
565 unsigned getNumParams() const {
566 if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
567 return FD->getNumParams();
568 return cast<ObjCMethodDecl>(Val: CalleeDecl)->param_size();
569 }
570 const ParmVarDecl *getParamDecl(unsigned I) const {
571 if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
572 return FD->getParamDecl(i: I);
573 return *(cast<ObjCMethodDecl>(Val: CalleeDecl)->param_begin() + I);
574 }
575 };
576
577 /// Sanitizers enabled for this function.
578 SanitizerSet SanOpts;
579
580 /// True if CodeGen currently emits code implementing sanitizer checks.
581 bool IsSanitizerScope = false;
582
583 /// RAII object to set/unset CodeGenFunction::IsSanitizerScope.
584 class SanitizerScope {
585 CodeGenFunction *CGF;
586
587 public:
588 SanitizerScope(CodeGenFunction *CGF);
589 ~SanitizerScope();
590 };
591
592 /// In C++, whether we are code generating a thunk. This controls whether we
593 /// should emit cleanups.
594 bool CurFuncIsThunk = false;
595
596 /// In ARC, whether we should autorelease the return value.
597 bool AutoreleaseResult = false;
598
599 /// Whether we processed a Microsoft-style asm block during CodeGen. These can
600 /// potentially set the return value.
601 bool SawAsmBlock = false;
602
603 GlobalDecl CurSEHParent;
604
605 /// True if the current function is an outlined SEH helper. This can be a
606 /// finally block or filter expression.
607 bool IsOutlinedSEHHelper = false;
608
609 /// True if CodeGen currently emits code inside presereved access index
610 /// region.
611 bool IsInPreservedAIRegion = false;
612
613 /// True if the current statement has nomerge attribute.
614 bool InNoMergeAttributedStmt = false;
615
616 /// True if the current statement has noinline attribute.
617 bool InNoInlineAttributedStmt = false;
618
619 /// True if the current statement has always_inline attribute.
620 bool InAlwaysInlineAttributedStmt = false;
621
622 /// True if the current statement has noconvergent attribute.
623 bool InNoConvergentAttributedStmt = false;
624
625 /// HLSL Branch attribute.
626 HLSLControlFlowHintAttr::Spelling HLSLControlFlowAttr =
627 HLSLControlFlowHintAttr::SpellingNotCalculated;
628
629 // The CallExpr within the current statement that the musttail attribute
630 // applies to. nullptr if there is no 'musttail' on the current statement.
631 const CallExpr *MustTailCall = nullptr;
632
633 /// Returns true if a function must make progress, which means the
634 /// mustprogress attribute can be added.
635 bool checkIfFunctionMustProgress() {
636 if (CGM.getCodeGenOpts().getFiniteLoops() ==
637 CodeGenOptions::FiniteLoopsKind::Never)
638 return false;
639
640 // C++11 and later guarantees that a thread eventually will do one of the
641 // following (C++11 [intro.multithread]p24 and C++17 [intro.progress]p1):
642 // - terminate,
643 // - make a call to a library I/O function,
644 // - perform an access through a volatile glvalue, or
645 // - perform a synchronization operation or an atomic operation.
646 //
647 // Hence each function is 'mustprogress' in C++11 or later.
648 return getLangOpts().CPlusPlus11;
649 }
650
651 /// Returns true if a loop must make progress, which means the mustprogress
652 /// attribute can be added. \p HasConstantCond indicates whether the branch
653 /// condition is a known constant.
654 bool checkIfLoopMustProgress(const Expr *, bool HasEmptyBody);
655
656 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
657 llvm::Value *BlockPointer = nullptr;
658
659 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
660 FieldDecl *LambdaThisCaptureField = nullptr;
661
662 /// A mapping from NRVO variables to the flags used to indicate
663 /// when the NRVO has been applied to this variable.
664 llvm::DenseMap<const VarDecl *, llvm::Value *> NRVOFlags;
665
666 EHScopeStack EHStack;
667 llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack;
668
669 // A stack of cleanups which were added to EHStack but have to be deactivated
670 // later before being popped or emitted. These are usually deactivated on
671 // exiting a `CleanupDeactivationScope` scope. For instance, after a
672 // full-expr.
673 //
674 // These are specially useful for correctly emitting cleanups while
675 // encountering branches out of expression (through stmt-expr or coroutine
676 // suspensions).
677 struct DeferredDeactivateCleanup {
678 EHScopeStack::stable_iterator Cleanup;
679 llvm::Instruction *DominatingIP;
680 };
681 llvm::SmallVector<DeferredDeactivateCleanup> DeferredDeactivationCleanupStack;
682
683 // Enters a new scope for capturing cleanups which are deferred to be
684 // deactivated, all of which will be deactivated once the scope is exited.
685 struct CleanupDeactivationScope {
686 CodeGenFunction &CGF;
687 size_t OldDeactivateCleanupStackSize;
688 bool Deactivated;
689 CleanupDeactivationScope(CodeGenFunction &CGF)
690 : CGF(CGF), OldDeactivateCleanupStackSize(
691 CGF.DeferredDeactivationCleanupStack.size()),
692 Deactivated(false) {}
693
694 void ForceDeactivate() {
695 assert(!Deactivated && "Deactivating already deactivated scope");
696 auto &Stack = CGF.DeferredDeactivationCleanupStack;
697 for (size_t I = Stack.size(); I > OldDeactivateCleanupStackSize; I--) {
698 CGF.DeactivateCleanupBlock(Cleanup: Stack[I - 1].Cleanup,
699 DominatingIP: Stack[I - 1].DominatingIP);
700 Stack[I - 1].DominatingIP->eraseFromParent();
701 }
702 Stack.resize(N: OldDeactivateCleanupStackSize);
703 Deactivated = true;
704 }
705
706 ~CleanupDeactivationScope() {
707 if (Deactivated)
708 return;
709 ForceDeactivate();
710 }
711 };
712
713 llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack;
714
715 llvm::Instruction *CurrentFuncletPad = nullptr;
716
717 class CallLifetimeEnd final : public EHScopeStack::Cleanup {
718 bool isRedundantBeforeReturn() override { return true; }
719
720 llvm::Value *Addr;
721
722 public:
723 CallLifetimeEnd(RawAddress addr) : Addr(addr.getPointer()) {}
724
725 void Emit(CodeGenFunction &CGF, Flags flags) override {
726 CGF.EmitLifetimeEnd(Addr);
727 }
728 };
729
730 // We are using objects of this 'cleanup' class to emit fake.use calls
731 // for -fextend-variable-liveness. They are placed at the end of a variable's
732 // scope analogous to lifetime markers.
733 class FakeUse final : public EHScopeStack::Cleanup {
734 Address Addr;
735
736 public:
737 FakeUse(Address addr) : Addr(addr) {}
738
739 void Emit(CodeGenFunction &CGF, Flags flags) override {
740 CGF.EmitFakeUse(Addr);
741 }
742 };
743
744 /// Header for data within LifetimeExtendedCleanupStack.
745 struct alignas(uint64_t) LifetimeExtendedCleanupHeader {
746 /// The size of the following cleanup object.
747 unsigned Size;
748 /// The kind of cleanup to push.
749 LLVM_PREFERRED_TYPE(CleanupKind)
750 unsigned Kind : 31;
751 /// Whether this is a conditional cleanup.
752 LLVM_PREFERRED_TYPE(bool)
753 unsigned IsConditional : 1;
754
755 size_t getSize() const { return Size; }
756 CleanupKind getKind() const { return (CleanupKind)Kind; }
757 bool isConditional() const { return IsConditional; }
758 };
759
760 /// i32s containing the indexes of the cleanup destinations.
761 RawAddress NormalCleanupDest = RawAddress::invalid();
762
763 unsigned NextCleanupDestIndex = 1;
764
765 /// EHResumeBlock - Unified block containing a call to llvm.eh.resume.
766 llvm::BasicBlock *EHResumeBlock = nullptr;
767
768 /// The exception slot. All landing pads write the current exception pointer
769 /// into this alloca.
770 llvm::Value *ExceptionSlot = nullptr;
771
772 /// The selector slot. Under the MandatoryCleanup model, all landing pads
773 /// write the current selector value into this alloca.
774 llvm::AllocaInst *EHSelectorSlot = nullptr;
775
776 /// A stack of exception code slots. Entering an __except block pushes a slot
777 /// on the stack and leaving pops one. The __exception_code() intrinsic loads
778 /// a value from the top of the stack.
779 SmallVector<Address, 1> SEHCodeSlotStack;
780
781 /// Value returned by __exception_info intrinsic.
782 llvm::Value *SEHInfo = nullptr;
783
784 /// Emits a landing pad for the current EH stack.
785 llvm::BasicBlock *EmitLandingPad();
786
787 llvm::BasicBlock *getInvokeDestImpl();
788
789 /// Parent loop-based directive for scan directive.
790 const OMPExecutableDirective *OMPParentLoopDirectiveForScan = nullptr;
791 llvm::BasicBlock *OMPBeforeScanBlock = nullptr;
792 llvm::BasicBlock *OMPAfterScanBlock = nullptr;
793 llvm::BasicBlock *OMPScanExitBlock = nullptr;
794 llvm::BasicBlock *OMPScanDispatch = nullptr;
795 bool OMPFirstScanLoop = false;
796
797 /// Manages parent directive for scan directives.
798 class ParentLoopDirectiveForScanRegion {
799 CodeGenFunction &CGF;
800 const OMPExecutableDirective *ParentLoopDirectiveForScan;
801
802 public:
803 ParentLoopDirectiveForScanRegion(
804 CodeGenFunction &CGF,
805 const OMPExecutableDirective &ParentLoopDirectiveForScan)
806 : CGF(CGF),
807 ParentLoopDirectiveForScan(CGF.OMPParentLoopDirectiveForScan) {
808 CGF.OMPParentLoopDirectiveForScan = &ParentLoopDirectiveForScan;
809 }
810 ~ParentLoopDirectiveForScanRegion() {
811 CGF.OMPParentLoopDirectiveForScan = ParentLoopDirectiveForScan;
812 }
813 };
814
815 template <class T>
816 typename DominatingValue<T>::saved_type saveValueInCond(T value) {
817 return DominatingValue<T>::save(*this, value);
818 }
819
820 class CGFPOptionsRAII {
821 public:
822 CGFPOptionsRAII(CodeGenFunction &CGF, FPOptions FPFeatures);
823 CGFPOptionsRAII(CodeGenFunction &CGF, const Expr *E);
824 ~CGFPOptionsRAII();
825
826 private:
827 void ConstructorHelper(FPOptions FPFeatures);
828 CodeGenFunction &CGF;
829 FPOptions OldFPFeatures;
830 llvm::fp::ExceptionBehavior OldExcept;
831 llvm::RoundingMode OldRounding;
832 std::optional<CGBuilderTy::FastMathFlagGuard> FMFGuard;
833 };
834 FPOptions CurFPFeatures;
835
836 class CGAtomicOptionsRAII {
837 public:
838 CGAtomicOptionsRAII(CodeGenModule &CGM_, AtomicOptions AO)
839 : CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {
840 CGM.setAtomicOpts(AO);
841 }
842 CGAtomicOptionsRAII(CodeGenModule &CGM_, const AtomicAttr *AA)
843 : CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {
844 if (!AA)
845 return;
846 AtomicOptions AO = SavedAtomicOpts;
847 for (auto Option : AA->atomicOptions()) {
848 switch (Option) {
849 case AtomicAttr::remote_memory:
850 AO.remote_memory = true;
851 break;
852 case AtomicAttr::no_remote_memory:
853 AO.remote_memory = false;
854 break;
855 case AtomicAttr::fine_grained_memory:
856 AO.fine_grained_memory = true;
857 break;
858 case AtomicAttr::no_fine_grained_memory:
859 AO.fine_grained_memory = false;
860 break;
861 case AtomicAttr::ignore_denormal_mode:
862 AO.ignore_denormal_mode = true;
863 break;
864 case AtomicAttr::no_ignore_denormal_mode:
865 AO.ignore_denormal_mode = false;
866 break;
867 }
868 }
869 CGM.setAtomicOpts(AO);
870 }
871
872 CGAtomicOptionsRAII(const CGAtomicOptionsRAII &) = delete;
873 CGAtomicOptionsRAII &operator=(const CGAtomicOptionsRAII &) = delete;
874 ~CGAtomicOptionsRAII() { CGM.setAtomicOpts(SavedAtomicOpts); }
875
876 private:
877 CodeGenModule &CGM;
878 AtomicOptions SavedAtomicOpts;
879 };
880
881public:
882 /// ObjCEHValueStack - Stack of Objective-C exception values, used for
883 /// rethrows.
884 SmallVector<llvm::Value *, 8> ObjCEHValueStack;
885
886 /// A class controlling the emission of a finally block.
887 class FinallyInfo {
888 /// Where the catchall's edge through the cleanup should go.
889 JumpDest RethrowDest;
890
891 /// A function to call to enter the catch.
892 llvm::FunctionCallee BeginCatchFn;
893
894 /// An i1 variable indicating whether or not the @finally is
895 /// running for an exception.
896 llvm::AllocaInst *ForEHVar = nullptr;
897
898 /// An i8* variable into which the exception pointer to rethrow
899 /// has been saved.
900 llvm::AllocaInst *SavedExnVar = nullptr;
901
902 public:
903 void enter(CodeGenFunction &CGF, const Stmt *Finally,
904 llvm::FunctionCallee beginCatchFn,
905 llvm::FunctionCallee endCatchFn, llvm::FunctionCallee rethrowFn);
906 void exit(CodeGenFunction &CGF);
907 };
908
909 /// Returns true inside SEH __try blocks.
910 bool isSEHTryScope() const { return !SEHTryEpilogueStack.empty(); }
911
912 /// Returns true while emitting a cleanuppad.
913 bool isCleanupPadScope() const {
914 return CurrentFuncletPad && isa<llvm::CleanupPadInst>(Val: CurrentFuncletPad);
915 }
916
917 /// pushFullExprCleanup - Push a cleanup to be run at the end of the
918 /// current full-expression. Safe against the possibility that
919 /// we're currently inside a conditionally-evaluated expression.
920 template <class T, class... As>
921 void pushFullExprCleanup(CleanupKind kind, As... A) {
922 // If we're not in a conditional branch, or if none of the
923 // arguments requires saving, then use the unconditional cleanup.
924 if (!isInConditionalBranch())
925 return EHStack.pushCleanup<T>(kind, A...);
926
927 // Stash values in a tuple so we can guarantee the order of saves.
928 typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
929 SavedTuple Saved{saveValueInCond(A)...};
930
931 typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
932 EHStack.pushCleanupTuple<CleanupType>(kind, Saved);
933 initFullExprCleanup();
934 }
935
936 /// Queue a cleanup to be pushed after finishing the current full-expression,
937 /// potentially with an active flag.
938 template <class T, class... As>
939 void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) {
940 if (!isInConditionalBranch())
941 return pushCleanupAfterFullExprWithActiveFlag<T>(
942 Kind, RawAddress::invalid(), A...);
943
944 RawAddress ActiveFlag = createCleanupActiveFlag();
945 assert(!DominatingValue<Address>::needsSaving(ActiveFlag) &&
946 "cleanup active flag should never need saving");
947
948 typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
949 SavedTuple Saved{saveValueInCond(A)...};
950
951 typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
952 pushCleanupAfterFullExprWithActiveFlag<CleanupType>(Kind, ActiveFlag,
953 Saved);
954 }
955
956 template <class T, class... As>
957 void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind,
958 RawAddress ActiveFlag, As... A) {
959 LifetimeExtendedCleanupHeader Header = {.Size: sizeof(T), .Kind: Kind,
960 .IsConditional: ActiveFlag.isValid()};
961
962 size_t OldSize = LifetimeExtendedCleanupStack.size();
963 LifetimeExtendedCleanupStack.resize(
964 N: LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size +
965 (Header.IsConditional ? sizeof(ActiveFlag) : 0));
966
967 static_assert((alignof(LifetimeExtendedCleanupHeader) == alignof(T)) &&
968 (alignof(T) == alignof(RawAddress)),
969 "Cleanup will be allocated on misaligned address");
970 char *Buffer = &LifetimeExtendedCleanupStack[OldSize];
971 new (Buffer) LifetimeExtendedCleanupHeader(Header);
972 new (Buffer + sizeof(Header)) T(A...);
973 if (Header.IsConditional)
974 new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag);
975 }
976
977 // Push a cleanup onto EHStack and deactivate it later. It is usually
978 // deactivated when exiting a `CleanupDeactivationScope` (for example: after a
979 // full expression).
980 template <class T, class... As>
981 void pushCleanupAndDeferDeactivation(CleanupKind Kind, As... A) {
982 // Placeholder dominating IP for this cleanup.
983 llvm::Instruction *DominatingIP =
984 Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
985 EHStack.pushCleanup<T>(Kind, A...);
986 DeferredDeactivationCleanupStack.push_back(
987 Elt: {.Cleanup: EHStack.stable_begin(), .DominatingIP: DominatingIP});
988 }
989
990 /// Set up the last cleanup that was pushed as a conditional
991 /// full-expression cleanup.
992 void initFullExprCleanup() {
993 initFullExprCleanupWithFlag(ActiveFlag: createCleanupActiveFlag());
994 }
995
996 void initFullExprCleanupWithFlag(RawAddress ActiveFlag);
997 RawAddress createCleanupActiveFlag();
998
999 /// PushDestructorCleanup - Push a cleanup to call the
1000 /// complete-object destructor of an object of the given type at the
1001 /// given address. Does nothing if T is not a C++ class type with a
1002 /// non-trivial destructor.
1003 void PushDestructorCleanup(QualType T, Address Addr);
1004
1005 /// PushDestructorCleanup - Push a cleanup to call the
1006 /// complete-object variant of the given destructor on the object at
1007 /// the given address.
1008 void PushDestructorCleanup(const CXXDestructorDecl *Dtor, QualType T,
1009 Address Addr);
1010
1011 /// PopCleanupBlock - Will pop the cleanup entry on the stack and
1012 /// process all branch fixups.
1013 void PopCleanupBlock(bool FallThroughIsBranchThrough = false,
1014 bool ForDeactivation = false);
1015
1016 /// DeactivateCleanupBlock - Deactivates the given cleanup block.
1017 /// The block cannot be reactivated. Pops it if it's the top of the
1018 /// stack.
1019 ///
1020 /// \param DominatingIP - An instruction which is known to
1021 /// dominate the current IP (if set) and which lies along
1022 /// all paths of execution between the current IP and the
1023 /// the point at which the cleanup comes into scope.
1024 void DeactivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
1025 llvm::Instruction *DominatingIP);
1026
1027 /// ActivateCleanupBlock - Activates an initially-inactive cleanup.
1028 /// Cannot be used to resurrect a deactivated cleanup.
1029 ///
1030 /// \param DominatingIP - An instruction which is known to
1031 /// dominate the current IP (if set) and which lies along
1032 /// all paths of execution between the current IP and the
1033 /// the point at which the cleanup comes into scope.
1034 void ActivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
1035 llvm::Instruction *DominatingIP);
1036
1037 /// Enters a new scope for capturing cleanups, all of which
1038 /// will be executed once the scope is exited.
1039 class RunCleanupsScope {
1040 EHScopeStack::stable_iterator CleanupStackDepth, OldCleanupScopeDepth;
1041 size_t LifetimeExtendedCleanupStackSize;
1042 CleanupDeactivationScope DeactivateCleanups;
1043 bool OldDidCallStackSave;
1044
1045 protected:
1046 bool PerformCleanup;
1047
1048 private:
1049 RunCleanupsScope(const RunCleanupsScope &) = delete;
1050 void operator=(const RunCleanupsScope &) = delete;
1051
1052 protected:
1053 CodeGenFunction &CGF;
1054
1055 public:
1056 /// Enter a new cleanup scope.
1057 explicit RunCleanupsScope(CodeGenFunction &CGF)
1058 : DeactivateCleanups(CGF), PerformCleanup(true), CGF(CGF) {
1059 CleanupStackDepth = CGF.EHStack.stable_begin();
1060 LifetimeExtendedCleanupStackSize =
1061 CGF.LifetimeExtendedCleanupStack.size();
1062 OldDidCallStackSave = CGF.DidCallStackSave;
1063 CGF.DidCallStackSave = false;
1064 OldCleanupScopeDepth = CGF.CurrentCleanupScopeDepth;
1065 CGF.CurrentCleanupScopeDepth = CleanupStackDepth;
1066 }
1067
1068 /// Exit this cleanup scope, emitting any accumulated cleanups.
1069 ~RunCleanupsScope() {
1070 if (PerformCleanup)
1071 ForceCleanup();
1072 }
1073
1074 /// Determine whether this scope requires any cleanups.
1075 bool requiresCleanups() const {
1076 return CGF.EHStack.stable_begin() != CleanupStackDepth;
1077 }
1078
1079 /// Force the emission of cleanups now, instead of waiting
1080 /// until this object is destroyed.
1081 /// \param ValuesToReload - A list of values that need to be available at
1082 /// the insertion point after cleanup emission. If cleanup emission created
1083 /// a shared cleanup block, these value pointers will be rewritten.
1084 /// Otherwise, they not will be modified.
1085 void
1086 ForceCleanup(std::initializer_list<llvm::Value **> ValuesToReload = {}) {
1087 assert(PerformCleanup && "Already forced cleanup");
1088 CGF.DidCallStackSave = OldDidCallStackSave;
1089 DeactivateCleanups.ForceDeactivate();
1090 CGF.PopCleanupBlocks(OldCleanupStackSize: CleanupStackDepth, OldLifetimeExtendedStackSize: LifetimeExtendedCleanupStackSize,
1091 ValuesToReload);
1092 PerformCleanup = false;
1093 CGF.CurrentCleanupScopeDepth = OldCleanupScopeDepth;
1094 }
1095 };
1096
1097 // Cleanup stack depth of the RunCleanupsScope that was pushed most recently.
1098 EHScopeStack::stable_iterator CurrentCleanupScopeDepth =
1099 EHScopeStack::stable_end();
1100
1101 class LexicalScope : public RunCleanupsScope {
1102 SourceRange Range;
1103 SmallVector<const LabelDecl *, 4> Labels;
1104 LexicalScope *ParentScope;
1105
1106 LexicalScope(const LexicalScope &) = delete;
1107 void operator=(const LexicalScope &) = delete;
1108
1109 public:
1110 /// Enter a new cleanup scope.
1111 explicit LexicalScope(CodeGenFunction &CGF, SourceRange Range);
1112
1113 void addLabel(const LabelDecl *label) {
1114 assert(PerformCleanup && "adding label to dead scope?");
1115 Labels.push_back(Elt: label);
1116 }
1117
1118 /// Exit this cleanup scope, emitting any accumulated
1119 /// cleanups.
1120 ~LexicalScope();
1121
1122 /// Force the emission of cleanups now, instead of waiting
1123 /// until this object is destroyed.
1124 void ForceCleanup() {
1125 CGF.CurLexicalScope = ParentScope;
1126 RunCleanupsScope::ForceCleanup();
1127
1128 if (!Labels.empty())
1129 rescopeLabels();
1130 }
1131
1132 bool hasLabels() const { return !Labels.empty(); }
1133
1134 void rescopeLabels();
1135 };
1136
1137 typedef llvm::DenseMap<const Decl *, Address> DeclMapTy;
1138
1139 /// The class used to assign some variables some temporarily addresses.
1140 class OMPMapVars {
1141 DeclMapTy SavedLocals;
1142 DeclMapTy SavedTempAddresses;
1143 OMPMapVars(const OMPMapVars &) = delete;
1144 void operator=(const OMPMapVars &) = delete;
1145
1146 public:
1147 explicit OMPMapVars() = default;
1148 ~OMPMapVars() {
1149 assert(SavedLocals.empty() && "Did not restored original addresses.");
1150 };
1151
1152 /// Sets the address of the variable \p LocalVD to be \p TempAddr in
1153 /// function \p CGF.
1154 /// \return true if at least one variable was set already, false otherwise.
1155 bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD,
1156 Address TempAddr) {
1157 LocalVD = LocalVD->getCanonicalDecl();
1158 // Only save it once.
1159 if (SavedLocals.count(Val: LocalVD))
1160 return false;
1161
1162 // Copy the existing local entry to SavedLocals.
1163 auto it = CGF.LocalDeclMap.find(Val: LocalVD);
1164 if (it != CGF.LocalDeclMap.end())
1165 SavedLocals.try_emplace(Key: LocalVD, Args&: it->second);
1166 else
1167 SavedLocals.try_emplace(Key: LocalVD, Args: Address::invalid());
1168
1169 // Generate the private entry.
1170 QualType VarTy = LocalVD->getType();
1171 if (VarTy->isReferenceType()) {
1172 Address Temp = CGF.CreateMemTemp(T: VarTy);
1173 CGF.Builder.CreateStore(Val: TempAddr.emitRawPointer(CGF), Addr: Temp);
1174 TempAddr = Temp;
1175 }
1176 SavedTempAddresses.try_emplace(Key: LocalVD, Args&: TempAddr);
1177
1178 return true;
1179 }
1180
1181 /// Applies new addresses to the list of the variables.
1182 /// \return true if at least one variable is using new address, false
1183 /// otherwise.
1184 bool apply(CodeGenFunction &CGF) {
1185 copyInto(Src: SavedTempAddresses, Dest&: CGF.LocalDeclMap);
1186 SavedTempAddresses.clear();
1187 return !SavedLocals.empty();
1188 }
1189
1190 /// Restores original addresses of the variables.
1191 void restore(CodeGenFunction &CGF) {
1192 if (!SavedLocals.empty()) {
1193 copyInto(Src: SavedLocals, Dest&: CGF.LocalDeclMap);
1194 SavedLocals.clear();
1195 }
1196 }
1197
1198 private:
1199 /// Copy all the entries in the source map over the corresponding
1200 /// entries in the destination, which must exist.
1201 static void copyInto(const DeclMapTy &Src, DeclMapTy &Dest) {
1202 for (auto &[Decl, Addr] : Src) {
1203 if (!Addr.isValid())
1204 Dest.erase(Val: Decl);
1205 else
1206 Dest.insert_or_assign(Key: Decl, Val: Addr);
1207 }
1208 }
1209 };
1210
1211 /// The scope used to remap some variables as private in the OpenMP loop body
1212 /// (or other captured region emitted without outlining), and to restore old
1213 /// vars back on exit.
1214 class OMPPrivateScope : public RunCleanupsScope {
1215 OMPMapVars MappedVars;
1216 OMPPrivateScope(const OMPPrivateScope &) = delete;
1217 void operator=(const OMPPrivateScope &) = delete;
1218
1219 public:
1220 /// Enter a new OpenMP private scope.
1221 explicit OMPPrivateScope(CodeGenFunction &CGF) : RunCleanupsScope(CGF) {}
1222
1223 /// Registers \p LocalVD variable as a private with \p Addr as the address
1224 /// of the corresponding private variable. \p
1225 /// PrivateGen is the address of the generated private variable.
1226 /// \return true if the variable is registered as private, false if it has
1227 /// been privatized already.
1228 bool addPrivate(const VarDecl *LocalVD, Address Addr) {
1229 assert(PerformCleanup && "adding private to dead scope");
1230 return MappedVars.setVarAddr(CGF, LocalVD, TempAddr: Addr);
1231 }
1232
1233 /// Privatizes local variables previously registered as private.
1234 /// Registration is separate from the actual privatization to allow
1235 /// initializers use values of the original variables, not the private one.
1236 /// This is important, for example, if the private variable is a class
1237 /// variable initialized by a constructor that references other private
1238 /// variables. But at initialization original variables must be used, not
1239 /// private copies.
1240 /// \return true if at least one variable was privatized, false otherwise.
1241 bool Privatize() { return MappedVars.apply(CGF); }
1242
1243 void ForceCleanup() {
1244 RunCleanupsScope::ForceCleanup();
1245 restoreMap();
1246 }
1247
1248 /// Exit scope - all the mapped variables are restored.
1249 ~OMPPrivateScope() {
1250 if (PerformCleanup)
1251 ForceCleanup();
1252 }
1253
1254 /// Checks if the global variable is captured in current function.
1255 bool isGlobalVarCaptured(const VarDecl *VD) const {
1256 VD = VD->getCanonicalDecl();
1257 return !VD->isLocalVarDeclOrParm() && CGF.LocalDeclMap.count(Val: VD) > 0;
1258 }
1259
1260 /// Restore all mapped variables w/o clean up. This is usefully when we want
1261 /// to reference the original variables but don't want the clean up because
1262 /// that could emit lifetime end too early, causing backend issue #56913.
1263 void restoreMap() { MappedVars.restore(CGF); }
1264 };
1265
1266 /// Save/restore original map of previously emitted local vars in case when we
1267 /// need to duplicate emission of the same code several times in the same
1268 /// function for OpenMP code.
1269 class OMPLocalDeclMapRAII {
1270 CodeGenFunction &CGF;
1271 DeclMapTy SavedMap;
1272
1273 public:
1274 OMPLocalDeclMapRAII(CodeGenFunction &CGF)
1275 : CGF(CGF), SavedMap(CGF.LocalDeclMap) {}
1276 ~OMPLocalDeclMapRAII() { SavedMap.swap(RHS&: CGF.LocalDeclMap); }
1277 };
1278
1279 /// Takes the old cleanup stack size and emits the cleanup blocks
1280 /// that have been added.
1281 void
1282 PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1283 std::initializer_list<llvm::Value **> ValuesToReload = {});
1284
1285 /// Takes the old cleanup stack size and emits the cleanup blocks
1286 /// that have been added, then adds all lifetime-extended cleanups from
1287 /// the given position to the stack.
1288 void
1289 PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1290 size_t OldLifetimeExtendedStackSize,
1291 std::initializer_list<llvm::Value **> ValuesToReload = {});
1292
1293 void ResolveBranchFixups(llvm::BasicBlock *Target);
1294
1295 /// The given basic block lies in the current EH scope, but may be a
1296 /// target of a potentially scope-crossing jump; get a stable handle
1297 /// to which we can perform this jump later.
1298 JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target) {
1299 return JumpDest(Target, EHStack.getInnermostNormalCleanup(),
1300 NextCleanupDestIndex++);
1301 }
1302
1303 /// The given basic block lies in the current EH scope, but may be a
1304 /// target of a potentially scope-crossing jump; get a stable handle
1305 /// to which we can perform this jump later.
1306 JumpDest getJumpDestInCurrentScope(StringRef Name = StringRef()) {
1307 return getJumpDestInCurrentScope(Target: createBasicBlock(name: Name));
1308 }
1309
1310 /// EmitBranchThroughCleanup - Emit a branch from the current insert
1311 /// block through the normal cleanup handling code (if any) and then
1312 /// on to \arg Dest.
1313 void EmitBranchThroughCleanup(JumpDest Dest);
1314
1315 /// isObviouslyBranchWithoutCleanups - Return true if a branch to the
1316 /// specified destination obviously has no cleanups to run. 'false' is always
1317 /// a conservatively correct answer for this method.
1318 bool isObviouslyBranchWithoutCleanups(JumpDest Dest) const;
1319
1320 /// popCatchScope - Pops the catch scope at the top of the EHScope
1321 /// stack, emitting any required code (other than the catch handlers
1322 /// themselves).
1323 void popCatchScope();
1324
1325 llvm::BasicBlock *getEHResumeBlock(bool isCleanup);
1326 llvm::BasicBlock *getEHDispatchBlock(EHScopeStack::stable_iterator scope);
1327 llvm::BasicBlock *
1328 getFuncletEHDispatchBlock(EHScopeStack::stable_iterator scope);
1329
1330 /// An object to manage conditionally-evaluated expressions.
1331 class ConditionalEvaluation {
1332 llvm::BasicBlock *StartBB;
1333
1334 public:
1335 ConditionalEvaluation(CodeGenFunction &CGF)
1336 : StartBB(CGF.Builder.GetInsertBlock()) {}
1337
1338 void begin(CodeGenFunction &CGF) {
1339 assert(CGF.OutermostConditional != this);
1340 if (!CGF.OutermostConditional)
1341 CGF.OutermostConditional = this;
1342 }
1343
1344 void end(CodeGenFunction &CGF) {
1345 assert(CGF.OutermostConditional != nullptr);
1346 if (CGF.OutermostConditional == this)
1347 CGF.OutermostConditional = nullptr;
1348 }
1349
1350 /// Returns a block which will be executed prior to each
1351 /// evaluation of the conditional code.
1352 llvm::BasicBlock *getStartingBlock() const { return StartBB; }
1353 };
1354
1355 /// isInConditionalBranch - Return true if we're currently emitting
1356 /// one branch or the other of a conditional expression.
1357 bool isInConditionalBranch() const { return OutermostConditional != nullptr; }
1358
1359 void setBeforeOutermostConditional(llvm::Value *value, Address addr,
1360 CodeGenFunction &CGF) {
1361 assert(isInConditionalBranch());
1362 llvm::BasicBlock *block = OutermostConditional->getStartingBlock();
1363 auto store = new llvm::StoreInst(value, addr.emitRawPointer(CGF),
1364 block->back().getIterator());
1365 store->setAlignment(addr.getAlignment().getAsAlign());
1366 }
1367
1368 /// An RAII object to record that we're evaluating a statement
1369 /// expression.
1370 class StmtExprEvaluation {
1371 CodeGenFunction &CGF;
1372
1373 /// We have to save the outermost conditional: cleanups in a
1374 /// statement expression aren't conditional just because the
1375 /// StmtExpr is.
1376 ConditionalEvaluation *SavedOutermostConditional;
1377
1378 public:
1379 StmtExprEvaluation(CodeGenFunction &CGF)
1380 : CGF(CGF), SavedOutermostConditional(CGF.OutermostConditional) {
1381 CGF.OutermostConditional = nullptr;
1382 }
1383
1384 ~StmtExprEvaluation() {
1385 CGF.OutermostConditional = SavedOutermostConditional;
1386 CGF.EnsureInsertPoint();
1387 }
1388 };
1389
1390 /// An object which temporarily prevents a value from being
1391 /// destroyed by aggressive peephole optimizations that assume that
1392 /// all uses of a value have been realized in the IR.
1393 class PeepholeProtection {
1394 llvm::Instruction *Inst = nullptr;
1395 friend class CodeGenFunction;
1396
1397 public:
1398 PeepholeProtection() = default;
1399 };
1400
1401 /// A non-RAII class containing all the information about a bound
1402 /// opaque value. OpaqueValueMapping, below, is a RAII wrapper for
1403 /// this which makes individual mappings very simple; using this
1404 /// class directly is useful when you have a variable number of
1405 /// opaque values or don't want the RAII functionality for some
1406 /// reason.
1407 class OpaqueValueMappingData {
1408 const OpaqueValueExpr *OpaqueValue;
1409 bool BoundLValue;
1410 CodeGenFunction::PeepholeProtection Protection;
1411
1412 OpaqueValueMappingData(const OpaqueValueExpr *ov, bool boundLValue)
1413 : OpaqueValue(ov), BoundLValue(boundLValue) {}
1414
1415 public:
1416 OpaqueValueMappingData() : OpaqueValue(nullptr) {}
1417
1418 static bool shouldBindAsLValue(const Expr *expr) {
1419 // gl-values should be bound as l-values for obvious reasons.
1420 // Records should be bound as l-values because IR generation
1421 // always keeps them in memory. Expressions of function type
1422 // act exactly like l-values but are formally required to be
1423 // r-values in C.
1424 return expr->isGLValue() || expr->getType()->isFunctionType() ||
1425 hasAggregateEvaluationKind(T: expr->getType());
1426 }
1427
1428 static OpaqueValueMappingData
1429 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const Expr *e) {
1430 if (shouldBindAsLValue(expr: ov))
1431 return bind(CGF, ov, lv: CGF.EmitLValue(E: e));
1432 return bind(CGF, ov, rv: CGF.EmitAnyExpr(E: e));
1433 }
1434
1435 static OpaqueValueMappingData
1436 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const LValue &lv) {
1437 assert(shouldBindAsLValue(ov));
1438 CGF.OpaqueLValues.insert(KV: std::make_pair(x&: ov, y: lv));
1439 return OpaqueValueMappingData(ov, true);
1440 }
1441
1442 static OpaqueValueMappingData
1443 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const RValue &rv) {
1444 assert(!shouldBindAsLValue(ov));
1445 CGF.OpaqueRValues.insert(KV: std::make_pair(x&: ov, y: rv));
1446
1447 OpaqueValueMappingData data(ov, false);
1448
1449 // Work around an extremely aggressive peephole optimization in
1450 // EmitScalarConversion which assumes that all other uses of a
1451 // value are extant.
1452 data.Protection = CGF.protectFromPeepholes(rvalue: rv);
1453
1454 return data;
1455 }
1456
1457 bool isValid() const { return OpaqueValue != nullptr; }
1458 void clear() { OpaqueValue = nullptr; }
1459
1460 void unbind(CodeGenFunction &CGF) {
1461 assert(OpaqueValue && "no data to unbind!");
1462
1463 if (BoundLValue) {
1464 CGF.OpaqueLValues.erase(Val: OpaqueValue);
1465 } else {
1466 CGF.OpaqueRValues.erase(Val: OpaqueValue);
1467 CGF.unprotectFromPeepholes(protection: Protection);
1468 }
1469 }
1470 };
1471
1472 /// An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
1473 class OpaqueValueMapping {
1474 CodeGenFunction &CGF;
1475 OpaqueValueMappingData Data;
1476
1477 public:
1478 static bool shouldBindAsLValue(const Expr *expr) {
1479 return OpaqueValueMappingData::shouldBindAsLValue(expr);
1480 }
1481
1482 /// Build the opaque value mapping for the given conditional
1483 /// operator if it's the GNU ?: extension. This is a common
1484 /// enough pattern that the convenience operator is really
1485 /// helpful.
1486 ///
1487 OpaqueValueMapping(CodeGenFunction &CGF,
1488 const AbstractConditionalOperator *op)
1489 : CGF(CGF) {
1490 if (isa<ConditionalOperator>(Val: op))
1491 // Leave Data empty.
1492 return;
1493
1494 const BinaryConditionalOperator *e = cast<BinaryConditionalOperator>(Val: op);
1495 Data = OpaqueValueMappingData::bind(CGF, ov: e->getOpaqueValue(),
1496 e: e->getCommon());
1497 }
1498
1499 /// Build the opaque value mapping for an OpaqueValueExpr whose source
1500 /// expression is set to the expression the OVE represents.
1501 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *OV)
1502 : CGF(CGF) {
1503 if (OV) {
1504 assert(OV->getSourceExpr() && "wrong form of OpaqueValueMapping used "
1505 "for OVE with no source expression");
1506 Data = OpaqueValueMappingData::bind(CGF, ov: OV, e: OV->getSourceExpr());
1507 }
1508 }
1509
1510 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,
1511 LValue lvalue)
1512 : CGF(CGF),
1513 Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, lv: lvalue)) {}
1514
1515 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,
1516 RValue rvalue)
1517 : CGF(CGF),
1518 Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, rv: rvalue)) {}
1519
1520 void pop() {
1521 Data.unbind(CGF);
1522 Data.clear();
1523 }
1524
1525 ~OpaqueValueMapping() {
1526 if (Data.isValid())
1527 Data.unbind(CGF);
1528 }
1529 };
1530
1531private:
1532 CGDebugInfo *DebugInfo;
1533 /// Used to create unique names for artificial VLA size debug info variables.
1534 unsigned VLAExprCounter = 0;
1535 bool DisableDebugInfo = false;
1536
1537 /// DidCallStackSave - Whether llvm.stacksave has been called. Used to avoid
1538 /// calling llvm.stacksave for multiple VLAs in the same scope.
1539 bool DidCallStackSave = false;
1540
1541 /// IndirectBranch - The first time an indirect goto is seen we create a block
1542 /// with an indirect branch. Every time we see the address of a label taken,
1543 /// we add the label to the indirect goto. Every subsequent indirect goto is
1544 /// codegen'd as a jump to the IndirectBranch's basic block.
1545 llvm::IndirectBrInst *IndirectBranch = nullptr;
1546
1547 /// LocalDeclMap - This keeps track of the LLVM allocas or globals for local C
1548 /// decls.
1549 DeclMapTy LocalDeclMap;
1550
1551 // Keep track of the cleanups for callee-destructed parameters pushed to the
1552 // cleanup stack so that they can be deactivated later.
1553 llvm::DenseMap<const ParmVarDecl *, EHScopeStack::stable_iterator>
1554 CalleeDestructedParamCleanups;
1555
1556 /// SizeArguments - If a ParmVarDecl had the pass_object_size attribute, this
1557 /// will contain a mapping from said ParmVarDecl to its implicit "object_size"
1558 /// parameter.
1559 llvm::SmallDenseMap<const ParmVarDecl *, const ImplicitParamDecl *, 2>
1560 SizeArguments;
1561
1562 /// Track escaped local variables with auto storage. Used during SEH
1563 /// outlining to produce a call to llvm.localescape.
1564 llvm::DenseMap<llvm::AllocaInst *, int> EscapedLocals;
1565
1566 /// LabelMap - This keeps track of the LLVM basic block for each C label.
1567 llvm::DenseMap<const LabelDecl *, JumpDest> LabelMap;
1568
1569 // BreakContinueStack - This keeps track of where break and continue
1570 // statements should jump to.
1571 struct BreakContinue {
1572 BreakContinue(const Stmt &LoopOrSwitch, JumpDest Break, JumpDest Continue)
1573 : LoopOrSwitch(&LoopOrSwitch), BreakBlock(Break),
1574 ContinueBlock(Continue) {}
1575
1576 const Stmt *LoopOrSwitch;
1577 JumpDest BreakBlock;
1578 JumpDest ContinueBlock;
1579 };
1580 SmallVector<BreakContinue, 8> BreakContinueStack;
1581
1582 /// Handles cancellation exit points in OpenMP-related constructs.
1583 class OpenMPCancelExitStack {
1584 /// Tracks cancellation exit point and join point for cancel-related exit
1585 /// and normal exit.
1586 struct CancelExit {
1587 CancelExit() = default;
1588 CancelExit(OpenMPDirectiveKind Kind, JumpDest ExitBlock,
1589 JumpDest ContBlock)
1590 : Kind(Kind), ExitBlock(ExitBlock), ContBlock(ContBlock) {}
1591 OpenMPDirectiveKind Kind = llvm::omp::OMPD_unknown;
1592 /// true if the exit block has been emitted already by the special
1593 /// emitExit() call, false if the default codegen is used.
1594 bool HasBeenEmitted = false;
1595 JumpDest ExitBlock;
1596 JumpDest ContBlock;
1597 };
1598
1599 SmallVector<CancelExit, 8> Stack;
1600
1601 public:
1602 OpenMPCancelExitStack() : Stack(1) {}
1603 ~OpenMPCancelExitStack() = default;
1604 /// Fetches the exit block for the current OpenMP construct.
1605 JumpDest getExitBlock() const { return Stack.back().ExitBlock; }
1606 /// Emits exit block with special codegen procedure specific for the related
1607 /// OpenMP construct + emits code for normal construct cleanup.
1608 void emitExit(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
1609 const llvm::function_ref<void(CodeGenFunction &)> CodeGen) {
1610 if (Stack.back().Kind == Kind && getExitBlock().isValid()) {
1611 assert(CGF.getOMPCancelDestination(Kind).isValid());
1612 assert(CGF.HaveInsertPoint());
1613 assert(!Stack.back().HasBeenEmitted);
1614 auto IP = CGF.Builder.saveAndClearIP();
1615 CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1616 CodeGen(CGF);
1617 CGF.EmitBranch(Block: Stack.back().ContBlock.getBlock());
1618 CGF.Builder.restoreIP(IP);
1619 Stack.back().HasBeenEmitted = true;
1620 }
1621 CodeGen(CGF);
1622 }
1623 /// Enter the cancel supporting \a Kind construct.
1624 /// \param Kind OpenMP directive that supports cancel constructs.
1625 /// \param HasCancel true, if the construct has inner cancel directive,
1626 /// false otherwise.
1627 void enter(CodeGenFunction &CGF, OpenMPDirectiveKind Kind, bool HasCancel) {
1628 Stack.push_back(Elt: {Kind,
1629 HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.exit")
1630 : JumpDest(),
1631 HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.cont")
1632 : JumpDest()});
1633 }
1634 /// Emits default exit point for the cancel construct (if the special one
1635 /// has not be used) + join point for cancel/normal exits.
1636 void exit(CodeGenFunction &CGF) {
1637 if (getExitBlock().isValid()) {
1638 assert(CGF.getOMPCancelDestination(Stack.back().Kind).isValid());
1639 bool HaveIP = CGF.HaveInsertPoint();
1640 if (!Stack.back().HasBeenEmitted) {
1641 if (HaveIP)
1642 CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1643 CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1644 CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1645 }
1646 CGF.EmitBlock(BB: Stack.back().ContBlock.getBlock());
1647 if (!HaveIP) {
1648 CGF.Builder.CreateUnreachable();
1649 CGF.Builder.ClearInsertionPoint();
1650 }
1651 }
1652 Stack.pop_back();
1653 }
1654 };
1655 OpenMPCancelExitStack OMPCancelStack;
1656
1657 /// Lower the Likelihood knowledge about the \p Cond via llvm.expect intrin.
1658 llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
1659 Stmt::Likelihood LH);
1660
1661 std::unique_ptr<CodeGenPGO> PGO;
1662
1663 /// Calculate branch weights appropriate for PGO data
1664 llvm::MDNode *createProfileWeights(uint64_t TrueCount,
1665 uint64_t FalseCount) const;
1666 llvm::MDNode *createProfileWeights(ArrayRef<uint64_t> Weights) const;
1667 llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond,
1668 uint64_t LoopCount) const;
1669
1670public:
1671 bool hasSkipCounter(const Stmt *S) const;
1672
1673 void markStmtAsUsed(bool Skipped, const Stmt *S);
1674 void markStmtMaybeUsed(const Stmt *S);
1675
1676 /// Used to specify which counter in a pair shall be incremented.
1677 /// For non-binary counters, a skip counter is derived as (Parent - Exec).
1678 /// In contrast for binary counters, a skip counter cannot be computed from
1679 /// the Parent counter. In such cases, dedicated SkipPath counters must be
1680 /// allocated and marked (incremented as binary counters). (Parent can be
1681 /// synthesized with (Exec + Skip) in simple cases)
1682 enum CounterForIncrement {
1683 UseExecPath = 0, ///< Exec (true)
1684 UseSkipPath, ///< Skip (false)
1685 };
1686
1687 /// Increment the profiler's counter for the given statement by \p StepV.
1688 /// If \p StepV is null, the default increment is 1.
1689 void incrementProfileCounter(const Stmt *S, llvm::Value *StepV = nullptr) {
1690 incrementProfileCounter(ExecSkip: UseExecPath, S, UseBoth: false, StepV);
1691 }
1692
1693 /// Emit increment of Counter.
1694 /// \param ExecSkip Use `Skipped` Counter if UseSkipPath is specified.
1695 /// \param S The Stmt that Counter is associated.
1696 /// \param UseBoth Mark both Exec/Skip as used. (for verification)
1697 /// \param StepV The offset Value for adding to Counter.
1698 void incrementProfileCounter(CounterForIncrement ExecSkip, const Stmt *S,
1699 bool UseBoth = false,
1700 llvm::Value *StepV = nullptr);
1701
1702 bool isMCDCCoverageEnabled() const {
1703 return (CGM.getCodeGenOpts().hasProfileClangInstr() &&
1704 CGM.getCodeGenOpts().MCDCCoverage &&
1705 !CurFn->hasFnAttribute(Kind: llvm::Attribute::NoProfile));
1706 }
1707
1708 /// Allocate a temp value on the stack that MCDC can use to track condition
1709 /// results.
1710 void maybeCreateMCDCCondBitmap();
1711
1712 bool isBinaryLogicalOp(const Expr *E) const {
1713 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: E->IgnoreParens());
1714 return (BOp && BOp->isLogicalOp());
1715 }
1716
1717 bool isMCDCDecisionExpr(const Expr *E) const;
1718 bool isMCDCBranchExpr(const Expr *E) const;
1719
1720 /// Zero-init the MCDC temp value.
1721 void maybeResetMCDCCondBitmap(const Expr *E);
1722
1723 /// Increment the profiler's counter for the given expression by \p StepV.
1724 /// If \p StepV is null, the default increment is 1.
1725 void maybeUpdateMCDCTestVectorBitmap(const Expr *E);
1726
1727 /// Update the MCDC temp value with the condition's evaluated result.
1728 void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val);
1729
1730 /// Get the profiler's count for the given statement.
1731 uint64_t getProfileCount(const Stmt *S);
1732
1733 /// Set the profiler's current count.
1734 void setCurrentProfileCount(uint64_t Count);
1735
1736 /// Get the profiler's current count. This is generally the count for the most
1737 /// recently incremented counter.
1738 uint64_t getCurrentProfileCount();
1739
1740 /// See CGDebugInfo::addInstToCurrentSourceAtom.
1741 void addInstToCurrentSourceAtom(llvm::Instruction *KeyInstruction,
1742 llvm::Value *Backup);
1743
1744 /// See CGDebugInfo::addInstToSpecificSourceAtom.
1745 void addInstToSpecificSourceAtom(llvm::Instruction *KeyInstruction,
1746 llvm::Value *Backup, uint64_t Atom);
1747
1748 /// Add \p KeyInstruction and an optional \p Backup instruction to a new atom
1749 /// group (See ApplyAtomGroup for more info).
1750 void addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,
1751 llvm::Value *Backup);
1752
1753 /// Copy all PFP fields from SrcPtr to DestPtr while updating signatures,
1754 /// assuming that DestPtr was already memcpy'd from SrcPtr.
1755 void emitPFPPostCopyUpdates(Address DestPtr, Address SrcPtr, QualType Ty);
1756
1757private:
1758 /// SwitchInsn - This is nearest current switch instruction. It is null if
1759 /// current context is not in a switch.
1760 llvm::SwitchInst *SwitchInsn = nullptr;
1761 /// The branch weights of SwitchInsn when doing instrumentation based PGO.
1762 SmallVector<uint64_t, 16> *SwitchWeights = nullptr;
1763
1764 /// The likelihood attributes of the SwitchCase.
1765 SmallVector<Stmt::Likelihood, 16> *SwitchLikelihood = nullptr;
1766
1767 /// CaseRangeBlock - This block holds if condition check for last case
1768 /// statement range in current switch instruction.
1769 llvm::BasicBlock *CaseRangeBlock = nullptr;
1770
1771 /// OpaqueLValues - Keeps track of the current set of opaque value
1772 /// expressions.
1773 llvm::DenseMap<const OpaqueValueExpr *, LValue> OpaqueLValues;
1774 llvm::DenseMap<const OpaqueValueExpr *, RValue> OpaqueRValues;
1775
1776 // VLASizeMap - This keeps track of the associated size for each VLA type.
1777 // We track this by the size expression rather than the type itself because
1778 // in certain situations, like a const qualifier applied to an VLA typedef,
1779 // multiple VLA types can share the same size expression.
1780 // FIXME: Maybe this could be a stack of maps that is pushed/popped as we
1781 // enter/leave scopes.
1782 llvm::DenseMap<const Expr *, llvm::Value *> VLASizeMap;
1783
1784 /// A block containing a single 'unreachable' instruction. Created
1785 /// lazily by getUnreachableBlock().
1786 llvm::BasicBlock *UnreachableBlock = nullptr;
1787
1788 /// Counts of the number return expressions in the function.
1789 unsigned NumReturnExprs = 0;
1790
1791 /// Count the number of simple (constant) return expressions in the function.
1792 unsigned NumSimpleReturnExprs = 0;
1793
1794 /// The last regular (non-return) debug location (breakpoint) in the function.
1795 SourceLocation LastStopPoint;
1796
1797public:
1798 /// Source location information about the default argument or member
1799 /// initializer expression we're evaluating, if any.
1800 CurrentSourceLocExprScope CurSourceLocExprScope;
1801 using SourceLocExprScopeGuard =
1802 CurrentSourceLocExprScope::SourceLocExprScopeGuard;
1803
1804 /// A scope within which we are constructing the fields of an object which
1805 /// might use a CXXDefaultInitExpr. This stashes away a 'this' value to use
1806 /// if we need to evaluate a CXXDefaultInitExpr within the evaluation.
1807 class FieldConstructionScope {
1808 public:
1809 FieldConstructionScope(CodeGenFunction &CGF, Address This)
1810 : CGF(CGF), OldCXXDefaultInitExprThis(CGF.CXXDefaultInitExprThis) {
1811 CGF.CXXDefaultInitExprThis = This;
1812 }
1813 ~FieldConstructionScope() {
1814 CGF.CXXDefaultInitExprThis = OldCXXDefaultInitExprThis;
1815 }
1816
1817 private:
1818 CodeGenFunction &CGF;
1819 Address OldCXXDefaultInitExprThis;
1820 };
1821
1822 /// The scope of a CXXDefaultInitExpr. Within this scope, the value of 'this'
1823 /// is overridden to be the object under construction.
1824 class CXXDefaultInitExprScope {
1825 public:
1826 CXXDefaultInitExprScope(CodeGenFunction &CGF, const CXXDefaultInitExpr *E)
1827 : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue),
1828 OldCXXThisAlignment(CGF.CXXThisAlignment),
1829 SourceLocScope(E, CGF.CurSourceLocExprScope) {
1830 CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getBasePointer();
1831 CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment();
1832 }
1833 ~CXXDefaultInitExprScope() {
1834 CGF.CXXThisValue = OldCXXThisValue;
1835 CGF.CXXThisAlignment = OldCXXThisAlignment;
1836 }
1837
1838 public:
1839 CodeGenFunction &CGF;
1840 llvm::Value *OldCXXThisValue;
1841 CharUnits OldCXXThisAlignment;
1842 SourceLocExprScopeGuard SourceLocScope;
1843 };
1844
1845 struct CXXDefaultArgExprScope : SourceLocExprScopeGuard {
1846 CXXDefaultArgExprScope(CodeGenFunction &CGF, const CXXDefaultArgExpr *E)
1847 : SourceLocExprScopeGuard(E, CGF.CurSourceLocExprScope) {}
1848 };
1849
1850 /// The scope of an ArrayInitLoopExpr. Within this scope, the value of the
1851 /// current loop index is overridden.
1852 class ArrayInitLoopExprScope {
1853 public:
1854 ArrayInitLoopExprScope(CodeGenFunction &CGF, llvm::Value *Index)
1855 : CGF(CGF), OldArrayInitIndex(CGF.ArrayInitIndex) {
1856 CGF.ArrayInitIndex = Index;
1857 }
1858 ~ArrayInitLoopExprScope() { CGF.ArrayInitIndex = OldArrayInitIndex; }
1859
1860 private:
1861 CodeGenFunction &CGF;
1862 llvm::Value *OldArrayInitIndex;
1863 };
1864
1865 class InlinedInheritingConstructorScope {
1866 public:
1867 InlinedInheritingConstructorScope(CodeGenFunction &CGF, GlobalDecl GD)
1868 : CGF(CGF), OldCurGD(CGF.CurGD), OldCurFuncDecl(CGF.CurFuncDecl),
1869 OldCurCodeDecl(CGF.CurCodeDecl),
1870 OldCXXABIThisDecl(CGF.CXXABIThisDecl),
1871 OldCXXABIThisValue(CGF.CXXABIThisValue),
1872 OldCXXThisValue(CGF.CXXThisValue),
1873 OldCXXABIThisAlignment(CGF.CXXABIThisAlignment),
1874 OldCXXThisAlignment(CGF.CXXThisAlignment),
1875 OldReturnValue(CGF.ReturnValue), OldFnRetTy(CGF.FnRetTy),
1876 OldCXXInheritedCtorInitExprArgs(
1877 std::move(CGF.CXXInheritedCtorInitExprArgs)) {
1878 CGF.CurGD = GD;
1879 CGF.CurFuncDecl = CGF.CurCodeDecl =
1880 cast<CXXConstructorDecl>(Val: GD.getDecl());
1881 CGF.CXXABIThisDecl = nullptr;
1882 CGF.CXXABIThisValue = nullptr;
1883 CGF.CXXThisValue = nullptr;
1884 CGF.CXXABIThisAlignment = CharUnits();
1885 CGF.CXXThisAlignment = CharUnits();
1886 CGF.ReturnValue = Address::invalid();
1887 CGF.FnRetTy = QualType();
1888 CGF.CXXInheritedCtorInitExprArgs.clear();
1889 }
1890 ~InlinedInheritingConstructorScope() {
1891 CGF.CurGD = OldCurGD;
1892 CGF.CurFuncDecl = OldCurFuncDecl;
1893 CGF.CurCodeDecl = OldCurCodeDecl;
1894 CGF.CXXABIThisDecl = OldCXXABIThisDecl;
1895 CGF.CXXABIThisValue = OldCXXABIThisValue;
1896 CGF.CXXThisValue = OldCXXThisValue;
1897 CGF.CXXABIThisAlignment = OldCXXABIThisAlignment;
1898 CGF.CXXThisAlignment = OldCXXThisAlignment;
1899 CGF.ReturnValue = OldReturnValue;
1900 CGF.FnRetTy = OldFnRetTy;
1901 CGF.CXXInheritedCtorInitExprArgs =
1902 std::move(OldCXXInheritedCtorInitExprArgs);
1903 }
1904
1905 private:
1906 CodeGenFunction &CGF;
1907 GlobalDecl OldCurGD;
1908 const Decl *OldCurFuncDecl;
1909 const Decl *OldCurCodeDecl;
1910 ImplicitParamDecl *OldCXXABIThisDecl;
1911 llvm::Value *OldCXXABIThisValue;
1912 llvm::Value *OldCXXThisValue;
1913 CharUnits OldCXXABIThisAlignment;
1914 CharUnits OldCXXThisAlignment;
1915 Address OldReturnValue;
1916 QualType OldFnRetTy;
1917 CallArgList OldCXXInheritedCtorInitExprArgs;
1918 };
1919
1920 // Helper class for the OpenMP IR Builder. Allows reusability of code used for
1921 // region body, and finalization codegen callbacks. This will class will also
1922 // contain privatization functions used by the privatization call backs
1923 //
1924 // TODO: this is temporary class for things that are being moved out of
1925 // CGOpenMPRuntime, new versions of current CodeGenFunction methods, or
1926 // utility function for use with the OMPBuilder. Once that move to use the
1927 // OMPBuilder is done, everything here will either become part of CodeGenFunc.
1928 // directly, or a new helper class that will contain functions used by both
1929 // this and the OMPBuilder
1930
1931 struct OMPBuilderCBHelpers {
1932
1933 OMPBuilderCBHelpers() = delete;
1934 OMPBuilderCBHelpers(const OMPBuilderCBHelpers &) = delete;
1935 OMPBuilderCBHelpers &operator=(const OMPBuilderCBHelpers &) = delete;
1936
1937 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1938
1939 /// Cleanup action for allocate support.
1940 class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
1941
1942 private:
1943 llvm::CallInst *RTLFnCI;
1944
1945 public:
1946 OMPAllocateCleanupTy(llvm::CallInst *RLFnCI) : RTLFnCI(RLFnCI) {
1947 RLFnCI->removeFromParent();
1948 }
1949
1950 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
1951 if (!CGF.HaveInsertPoint())
1952 return;
1953 CGF.Builder.Insert(I: RTLFnCI);
1954 }
1955 };
1956
1957 /// Returns address of the threadprivate variable for the current
1958 /// thread. This Also create any necessary OMP runtime calls.
1959 ///
1960 /// \param VD VarDecl for Threadprivate variable.
1961 /// \param VDAddr Address of the Vardecl
1962 /// \param Loc The location where the barrier directive was encountered
1963 static Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
1964 const VarDecl *VD, Address VDAddr,
1965 SourceLocation Loc);
1966
1967 /// Gets the OpenMP-specific address of the local variable /p VD.
1968 static Address getAddressOfLocalVariable(CodeGenFunction &CGF,
1969 const VarDecl *VD);
1970 /// Get the platform-specific name separator.
1971 /// \param Parts different parts of the final name that needs separation
1972 /// \param FirstSeparator First separator used between the initial two
1973 /// parts of the name.
1974 /// \param Separator separator used between all of the rest consecutinve
1975 /// parts of the name
1976 static std::string getNameWithSeparators(ArrayRef<StringRef> Parts,
1977 StringRef FirstSeparator = ".",
1978 StringRef Separator = ".");
1979 /// Emit the Finalization for an OMP region
1980 /// \param CGF The Codegen function this belongs to
1981 /// \param IP Insertion point for generating the finalization code.
1982 static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP) {
1983 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1984 assert(IP.getBlock()->end() != IP.getPoint() &&
1985 "OpenMP IR Builder should cause terminated block!");
1986
1987 llvm::BasicBlock *IPBB = IP.getBlock();
1988 llvm::BasicBlock *DestBB = IPBB->getUniqueSuccessor();
1989 assert(DestBB && "Finalization block should have one successor!");
1990
1991 // erase and replace with cleanup branch.
1992 IPBB->getTerminator()->eraseFromParent();
1993 CGF.Builder.SetInsertPoint(IPBB);
1994 CodeGenFunction::JumpDest Dest = CGF.getJumpDestInCurrentScope(Target: DestBB);
1995 CGF.EmitBranchThroughCleanup(Dest);
1996 }
1997
1998 /// Emit the body of an OMP region
1999 /// \param CGF The Codegen function this belongs to
2000 /// \param RegionBodyStmt The body statement for the OpenMP region being
2001 /// generated
2002 /// \param AllocaIP Where to insert alloca instructions
2003 /// \param CodeGenIP Where to insert the region code
2004 /// \param RegionName Name to be used for new blocks
2005 static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF,
2006 const Stmt *RegionBodyStmt,
2007 InsertPointTy AllocaIP,
2008 InsertPointTy CodeGenIP,
2009 Twine RegionName);
2010
2011 static void EmitCaptureStmt(CodeGenFunction &CGF, InsertPointTy CodeGenIP,
2012 llvm::BasicBlock &FiniBB, llvm::Function *Fn,
2013 ArrayRef<llvm::Value *> Args) {
2014 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2015 if (llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminatorOrNull())
2016 CodeGenIPBBTI->eraseFromParent();
2017
2018 CGF.Builder.SetInsertPoint(CodeGenIPBB);
2019
2020 if (Fn->doesNotThrow())
2021 CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
2022 else
2023 CGF.EmitRuntimeCall(callee: Fn, args: Args);
2024
2025 if (CGF.Builder.saveIP().isSet())
2026 CGF.Builder.CreateBr(Dest: &FiniBB);
2027 }
2028
2029 /// Emit the body of an OMP region that will be outlined in
2030 /// OpenMPIRBuilder::finalize().
2031 /// \param CGF The Codegen function this belongs to
2032 /// \param RegionBodyStmt The body statement for the OpenMP region being
2033 /// generated
2034 /// \param AllocaIP Where to insert alloca instructions
2035 /// \param CodeGenIP Where to insert the region code
2036 /// \param RegionName Name to be used for new blocks
2037 static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF,
2038 const Stmt *RegionBodyStmt,
2039 InsertPointTy AllocaIP,
2040 InsertPointTy CodeGenIP,
2041 Twine RegionName);
2042
2043 /// RAII for preserving necessary info during Outlined region body codegen.
2044 class OutlinedRegionBodyRAII {
2045
2046 llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
2047 CodeGenFunction::JumpDest OldReturnBlock;
2048 CodeGenFunction &CGF;
2049
2050 public:
2051 OutlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
2052 llvm::BasicBlock &RetBB)
2053 : CGF(cgf) {
2054 assert(AllocaIP.isSet() &&
2055 "Must specify Insertion point for allocas of outlined function");
2056 OldAllocaIP = CGF.AllocaInsertPt;
2057 CGF.AllocaInsertPt = &*AllocaIP.getPoint();
2058
2059 OldReturnBlock = CGF.ReturnBlock;
2060 CGF.ReturnBlock = CGF.getJumpDestInCurrentScope(Target: &RetBB);
2061 }
2062
2063 ~OutlinedRegionBodyRAII() {
2064 CGF.AllocaInsertPt = OldAllocaIP;
2065 CGF.ReturnBlock = OldReturnBlock;
2066 }
2067 };
2068
2069 /// RAII for preserving necessary info during inlined region body codegen.
2070 class InlinedRegionBodyRAII {
2071
2072 llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
2073 CodeGenFunction &CGF;
2074
2075 public:
2076 InlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
2077 llvm::BasicBlock &FiniBB)
2078 : CGF(cgf) {
2079 // Alloca insertion block should be in the entry block of the containing
2080 // function so it expects an empty AllocaIP in which case will reuse the
2081 // old alloca insertion point, or a new AllocaIP in the same block as
2082 // the old one
2083 assert((!AllocaIP.isSet() ||
2084 CGF.AllocaInsertPt->getParent() == AllocaIP.getBlock()) &&
2085 "Insertion point should be in the entry block of containing "
2086 "function!");
2087 OldAllocaIP = CGF.AllocaInsertPt;
2088 if (AllocaIP.isSet())
2089 CGF.AllocaInsertPt = &*AllocaIP.getPoint();
2090
2091 // TODO: Remove the call, after making sure the counter is not used by
2092 // the EHStack.
2093 // Since this is an inlined region, it should not modify the
2094 // ReturnBlock, and should reuse the one for the enclosing outlined
2095 // region. So, the JumpDest being return by the function is discarded
2096 (void)CGF.getJumpDestInCurrentScope(Target: &FiniBB);
2097 }
2098
2099 ~InlinedRegionBodyRAII() { CGF.AllocaInsertPt = OldAllocaIP; }
2100 };
2101 };
2102
2103private:
2104 /// CXXThisDecl - When generating code for a C++ member function,
2105 /// this will hold the implicit 'this' declaration.
2106 ImplicitParamDecl *CXXABIThisDecl = nullptr;
2107 llvm::Value *CXXABIThisValue = nullptr;
2108 llvm::Value *CXXThisValue = nullptr;
2109 CharUnits CXXABIThisAlignment;
2110 CharUnits CXXThisAlignment;
2111
2112 /// The value of 'this' to use when evaluating CXXDefaultInitExprs within
2113 /// this expression.
2114 Address CXXDefaultInitExprThis = Address::invalid();
2115
2116 /// The current array initialization index when evaluating an
2117 /// ArrayInitIndexExpr within an ArrayInitLoopExpr.
2118 llvm::Value *ArrayInitIndex = nullptr;
2119
2120 /// The values of function arguments to use when evaluating
2121 /// CXXInheritedCtorInitExprs within this context.
2122 CallArgList CXXInheritedCtorInitExprArgs;
2123
2124 /// CXXStructorImplicitParamDecl - When generating code for a constructor or
2125 /// destructor, this will hold the implicit argument (e.g. VTT).
2126 ImplicitParamDecl *CXXStructorImplicitParamDecl = nullptr;
2127 llvm::Value *CXXStructorImplicitParamValue = nullptr;
2128
2129 /// OutermostConditional - Points to the outermost active
2130 /// conditional control. This is used so that we know if a
2131 /// temporary should be destroyed conditionally.
2132 ConditionalEvaluation *OutermostConditional = nullptr;
2133
2134 /// The current lexical scope.
2135 LexicalScope *CurLexicalScope = nullptr;
2136
2137 /// The current source location that should be used for exception
2138 /// handling code.
2139 SourceLocation CurEHLocation;
2140
2141 /// BlockByrefInfos - For each __block variable, contains
2142 /// information about the layout of the variable.
2143 llvm::DenseMap<const ValueDecl *, BlockByrefInfo> BlockByrefInfos;
2144
2145 /// Used by -fsanitize=nullability-return to determine whether the return
2146 /// value can be checked.
2147 llvm::Value *RetValNullabilityPrecondition = nullptr;
2148
2149 /// Check if -fsanitize=nullability-return instrumentation is required for
2150 /// this function.
2151 bool requiresReturnValueNullabilityCheck() const {
2152 return RetValNullabilityPrecondition;
2153 }
2154
2155 /// Used to store precise source locations for return statements by the
2156 /// runtime return value checks.
2157 Address ReturnLocation = Address::invalid();
2158
2159 /// Check if the return value of this function requires sanitization.
2160 bool requiresReturnValueCheck() const;
2161
2162 bool isInAllocaArgument(CGCXXABI &ABI, QualType Ty);
2163 bool hasInAllocaArg(const CXXMethodDecl *MD);
2164
2165 llvm::BasicBlock *TerminateLandingPad = nullptr;
2166 llvm::BasicBlock *TerminateHandler = nullptr;
2167 llvm::SmallVector<llvm::BasicBlock *, 2> TrapBBs;
2168
2169 /// Terminate funclets keyed by parent funclet pad.
2170 llvm::MapVector<llvm::Value *, llvm::BasicBlock *> TerminateFunclets;
2171
2172 /// Largest vector width used in ths function. Will be used to create a
2173 /// function attribute.
2174 unsigned LargestVectorWidth = 0;
2175
2176 /// True if we need emit the life-time markers. This is initially set in
2177 /// the constructor, but could be overwritten to true if this is a coroutine.
2178 bool ShouldEmitLifetimeMarkers;
2179
2180 /// Add OpenCL kernel arg metadata and the kernel attribute metadata to
2181 /// the function metadata.
2182 void EmitKernelMetadata(const FunctionDecl *FD, llvm::Function *Fn);
2183
2184public:
2185 CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext = false);
2186 ~CodeGenFunction();
2187
2188 CodeGenTypes &getTypes() const { return CGM.getTypes(); }
2189 ASTContext &getContext() const { return CGM.getContext(); }
2190 CGDebugInfo *getDebugInfo() {
2191 if (DisableDebugInfo)
2192 return nullptr;
2193 return DebugInfo;
2194 }
2195 void disableDebugInfo() { DisableDebugInfo = true; }
2196 void enableDebugInfo() { DisableDebugInfo = false; }
2197
2198 bool shouldUseFusedARCCalls() {
2199 return CGM.getCodeGenOpts().OptimizationLevel == 0;
2200 }
2201
2202 const LangOptions &getLangOpts() const { return CGM.getLangOpts(); }
2203
2204 /// Returns a pointer to the function's exception object and selector slot,
2205 /// which is assigned in every landing pad.
2206 Address getExceptionSlot();
2207 Address getEHSelectorSlot();
2208
2209 /// Returns the contents of the function's exception object and selector
2210 /// slots.
2211 llvm::Value *getExceptionFromSlot();
2212 llvm::Value *getSelectorFromSlot();
2213
2214 RawAddress getNormalCleanupDestSlot();
2215
2216 llvm::BasicBlock *getUnreachableBlock() {
2217 if (!UnreachableBlock) {
2218 UnreachableBlock = createBasicBlock(name: "unreachable");
2219 new llvm::UnreachableInst(getLLVMContext(), UnreachableBlock);
2220 }
2221 return UnreachableBlock;
2222 }
2223
2224 llvm::BasicBlock *getInvokeDest() {
2225 if (!EHStack.requiresLandingPad())
2226 return nullptr;
2227 return getInvokeDestImpl();
2228 }
2229
2230 bool currentFunctionUsesSEHTry() const { return !!CurSEHParent; }
2231
2232 const TargetInfo &getTarget() const { return Target; }
2233 llvm::LLVMContext &getLLVMContext() { return CGM.getLLVMContext(); }
2234 const TargetCodeGenInfo &getTargetHooks() const {
2235 return CGM.getTargetCodeGenInfo();
2236 }
2237
2238 //===--------------------------------------------------------------------===//
2239 // Cleanups
2240 //===--------------------------------------------------------------------===//
2241
2242 typedef void Destroyer(CodeGenFunction &CGF, Address addr, QualType ty);
2243
2244 void pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,
2245 Address arrayEndPointer,
2246 QualType elementType,
2247 CharUnits elementAlignment,
2248 Destroyer *destroyer);
2249 void pushRegularPartialArrayCleanup(llvm::Value *arrayBegin,
2250 llvm::Value *arrayEnd,
2251 QualType elementType,
2252 CharUnits elementAlignment,
2253 Destroyer *destroyer);
2254
2255 void pushDestroy(QualType::DestructionKind dtorKind, Address addr,
2256 QualType type);
2257 void pushEHDestroy(QualType::DestructionKind dtorKind, Address addr,
2258 QualType type);
2259 void pushDestroy(CleanupKind kind, Address addr, QualType type,
2260 Destroyer *destroyer, bool useEHCleanupForArray);
2261 void pushDestroyAndDeferDeactivation(QualType::DestructionKind dtorKind,
2262 Address addr, QualType type);
2263 void pushDestroyAndDeferDeactivation(CleanupKind cleanupKind, Address addr,
2264 QualType type, Destroyer *destroyer,
2265 bool useEHCleanupForArray);
2266 void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr,
2267 QualType type, Destroyer *destroyer,
2268 bool useEHCleanupForArray);
2269 void pushLifetimeExtendedDestroy(QualType::DestructionKind dtorKind,
2270 Address addr, QualType type);
2271 void pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete,
2272 llvm::Value *CompletePtr,
2273 QualType ElementType);
2274 void pushStackRestore(CleanupKind kind, Address SPMem);
2275 void pushKmpcAllocFree(CleanupKind Kind,
2276 std::pair<llvm::Value *, llvm::Value *> AddrSizePair);
2277 void emitDestroy(Address addr, QualType type, Destroyer *destroyer,
2278 bool useEHCleanupForArray);
2279 llvm::Function *generateDestroyHelper(Address addr, QualType type,
2280 Destroyer *destroyer,
2281 bool useEHCleanupForArray,
2282 const VarDecl *VD);
2283 void emitArrayDestroy(llvm::Value *begin, llvm::Value *end,
2284 QualType elementType, CharUnits elementAlign,
2285 Destroyer *destroyer, bool checkZeroLength,
2286 bool useEHCleanup);
2287
2288 Destroyer *getDestroyer(QualType::DestructionKind destructionKind);
2289
2290 /// Determines whether an EH cleanup is required to destroy a type
2291 /// with the given destruction kind.
2292 bool needsEHCleanup(QualType::DestructionKind kind) {
2293 switch (kind) {
2294 case QualType::DK_none:
2295 return false;
2296 case QualType::DK_cxx_destructor:
2297 case QualType::DK_objc_weak_lifetime:
2298 case QualType::DK_nontrivial_c_struct:
2299 return getLangOpts().Exceptions;
2300 case QualType::DK_objc_strong_lifetime:
2301 return getLangOpts().Exceptions &&
2302 CGM.getCodeGenOpts().ObjCAutoRefCountExceptions;
2303 }
2304 llvm_unreachable("bad destruction kind");
2305 }
2306
2307 CleanupKind getCleanupKind(QualType::DestructionKind kind) {
2308 return (needsEHCleanup(kind) ? NormalAndEHCleanup : NormalCleanup);
2309 }
2310
2311 //===--------------------------------------------------------------------===//
2312 // Objective-C
2313 //===--------------------------------------------------------------------===//
2314
2315 void GenerateObjCMethod(const ObjCMethodDecl *OMD);
2316
2317 void StartObjCMethod(const ObjCMethodDecl *MD, const ObjCContainerDecl *CD);
2318
2319 /// GenerateObjCGetter - Synthesize an Objective-C property getter function.
2320 void GenerateObjCGetter(ObjCImplementationDecl *IMP,
2321 const ObjCPropertyImplDecl *PID);
2322 void generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
2323 const ObjCPropertyImplDecl *propImpl,
2324 const ObjCMethodDecl *GetterMothodDecl,
2325 llvm::Constant *AtomicHelperFn);
2326
2327 void GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
2328 ObjCMethodDecl *MD, bool ctor);
2329
2330 /// GenerateObjCSetter - Synthesize an Objective-C property setter function
2331 /// for the given property.
2332 void GenerateObjCSetter(ObjCImplementationDecl *IMP,
2333 const ObjCPropertyImplDecl *PID);
2334 void generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
2335 const ObjCPropertyImplDecl *propImpl,
2336 llvm::Constant *AtomicHelperFn);
2337
2338 //===--------------------------------------------------------------------===//
2339 // Block Bits
2340 //===--------------------------------------------------------------------===//
2341
2342 /// Emit block literal.
2343 /// \return an LLVM value which is a pointer to a struct which contains
2344 /// information about the block, including the block invoke function, the
2345 /// captured variables, etc.
2346 llvm::Value *EmitBlockLiteral(const BlockExpr *);
2347
2348 llvm::Function *GenerateBlockFunction(GlobalDecl GD, const CGBlockInfo &Info,
2349 const DeclMapTy &ldm,
2350 bool IsLambdaConversionToBlock,
2351 bool BuildGlobalBlock);
2352
2353 /// Check if \p T is a C++ class that has a destructor that can throw.
2354 static bool cxxDestructorCanThrow(QualType T);
2355
2356 llvm::Constant *GenerateCopyHelperFunction(const CGBlockInfo &blockInfo);
2357 llvm::Constant *GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo);
2358 llvm::Constant *
2359 GenerateObjCAtomicSetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);
2360 llvm::Constant *
2361 GenerateObjCAtomicGetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);
2362 llvm::Value *EmitBlockCopyAndAutorelease(llvm::Value *Block, QualType Ty);
2363
2364 void BuildBlockRelease(llvm::Value *DeclPtr, BlockFieldFlags flags,
2365 bool CanThrow);
2366
2367 class AutoVarEmission;
2368
2369 void emitByrefStructureInit(const AutoVarEmission &emission);
2370
2371 /// Enter a cleanup to destroy a __block variable. Note that this
2372 /// cleanup should be a no-op if the variable hasn't left the stack
2373 /// yet; if a cleanup is required for the variable itself, that needs
2374 /// to be done externally.
2375 ///
2376 /// \param Kind Cleanup kind.
2377 ///
2378 /// \param Addr When \p LoadBlockVarAddr is false, the address of the __block
2379 /// structure that will be passed to _Block_object_dispose. When
2380 /// \p LoadBlockVarAddr is true, the address of the field of the block
2381 /// structure that holds the address of the __block structure.
2382 ///
2383 /// \param Flags The flag that will be passed to _Block_object_dispose.
2384 ///
2385 /// \param LoadBlockVarAddr Indicates whether we need to emit a load from
2386 /// \p Addr to get the address of the __block structure.
2387 void enterByrefCleanup(CleanupKind Kind, Address Addr, BlockFieldFlags Flags,
2388 bool LoadBlockVarAddr, bool CanThrow);
2389
2390 void setBlockContextParameter(const ImplicitParamDecl *D, unsigned argNum,
2391 llvm::Value *ptr);
2392
2393 Address LoadBlockStruct();
2394 Address GetAddrOfBlockDecl(const VarDecl *var);
2395
2396 /// BuildBlockByrefAddress - Computes the location of the
2397 /// data in a variable which is declared as __block.
2398 Address emitBlockByrefAddress(Address baseAddr, const VarDecl *V,
2399 bool followForward = true);
2400 Address emitBlockByrefAddress(Address baseAddr, const BlockByrefInfo &info,
2401 bool followForward, const llvm::Twine &name);
2402
2403 const BlockByrefInfo &getBlockByrefInfo(const VarDecl *var);
2404
2405 QualType BuildFunctionArgList(GlobalDecl GD, FunctionArgList &Args);
2406
2407 void GenerateCode(GlobalDecl GD, llvm::Function *Fn,
2408 const CGFunctionInfo &FnInfo);
2409
2410 /// Annotate the function with an attribute that disables TSan checking at
2411 /// runtime.
2412 void markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn);
2413
2414 /// Emit code for the start of a function.
2415 /// \param Loc The location to be associated with the function.
2416 /// \param StartLoc The location of the function body.
2417 void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn,
2418 const CGFunctionInfo &FnInfo, const FunctionArgList &Args,
2419 SourceLocation Loc = SourceLocation(),
2420 SourceLocation StartLoc = SourceLocation());
2421
2422 static bool IsConstructorDelegationValid(const CXXConstructorDecl *Ctor);
2423
2424 void EmitConstructorBody(FunctionArgList &Args);
2425 void EmitDestructorBody(FunctionArgList &Args);
2426 void emitImplicitAssignmentOperatorBody(FunctionArgList &Args);
2427 void EmitFunctionBody(const Stmt *Body);
2428 void EmitBlockWithFallThrough(llvm::BasicBlock *BB, const Stmt *S);
2429
2430 void EmitForwardingCallToLambda(const CXXMethodDecl *LambdaCallOperator,
2431 CallArgList &CallArgs,
2432 const CGFunctionInfo *CallOpFnInfo = nullptr,
2433 llvm::Constant *CallOpFn = nullptr);
2434 void EmitLambdaBlockInvokeBody();
2435 void EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD);
2436 void EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,
2437 CallArgList &CallArgs);
2438 void EmitLambdaInAllocaImplFn(const CXXMethodDecl *CallOp,
2439 const CGFunctionInfo **ImplFnInfo,
2440 llvm::Function **ImplFn);
2441 void EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD);
2442 void EmitLambdaVLACapture(const VariableArrayType *VAT, LValue LV) {
2443 EmitStoreThroughLValue(Src: RValue::get(V: VLASizeMap[VAT->getSizeExpr()]), Dst: LV);
2444 }
2445 void EmitAsanPrologueOrEpilogue(bool Prologue);
2446
2447 /// Emit the unified return block, trying to avoid its emission when
2448 /// possible.
2449 /// \return The debug location of the user written return statement if the
2450 /// return block is avoided.
2451 llvm::DebugLoc EmitReturnBlock();
2452
2453 /// FinishFunction - Complete IR generation of the current function. It is
2454 /// legal to call this function even if there is no current insertion point.
2455 void FinishFunction(SourceLocation EndLoc = SourceLocation());
2456
2457 void StartThunk(llvm::Function *Fn, GlobalDecl GD,
2458 const CGFunctionInfo &FnInfo, bool IsUnprototyped);
2459
2460 void EmitCallAndReturnForThunk(llvm::FunctionCallee Callee,
2461 const ThunkInfo *Thunk, bool IsUnprototyped);
2462
2463 void FinishThunk();
2464
2465 /// Start an Objective-C direct method thunk.
2466 void StartObjCDirectPreconditionThunk(const ObjCMethodDecl *OMD,
2467 llvm::Function *Fn,
2468 const CGFunctionInfo &FI);
2469
2470 /// Finish an Objective-C direct method thunk.
2471 void FinishObjCDirectPreconditionThunk();
2472
2473 /// Emit a musttail call for a thunk with a potentially adjusted this pointer.
2474 void EmitMustTailThunk(GlobalDecl GD, llvm::Value *AdjustedThisPtr,
2475 llvm::FunctionCallee Callee);
2476
2477 /// Generate a thunk for the given method.
2478 void generateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo,
2479 GlobalDecl GD, const ThunkInfo &Thunk,
2480 bool IsUnprototyped);
2481
2482 llvm::Function *GenerateVarArgsThunk(llvm::Function *Fn,
2483 const CGFunctionInfo &FnInfo,
2484 GlobalDecl GD, const ThunkInfo &Thunk);
2485
2486 void EmitCtorPrologue(const CXXConstructorDecl *CD, CXXCtorType Type,
2487 FunctionArgList &Args);
2488
2489 void EmitInitializerForField(FieldDecl *Field, LValue LHS, Expr *Init);
2490
2491 /// Struct with all information about dynamic [sub]class needed to set vptr.
2492 struct VPtr {
2493 BaseSubobject Base;
2494 const CXXRecordDecl *NearestVBase;
2495 CharUnits OffsetFromNearestVBase;
2496 const CXXRecordDecl *VTableClass;
2497 };
2498
2499 /// Initialize the vtable pointer of the given subobject.
2500 void InitializeVTablePointer(const VPtr &vptr);
2501
2502 typedef llvm::SmallVector<VPtr, 4> VPtrsVector;
2503
2504 typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
2505 VPtrsVector getVTablePointers(const CXXRecordDecl *VTableClass);
2506
2507 void getVTablePointers(BaseSubobject Base, const CXXRecordDecl *NearestVBase,
2508 CharUnits OffsetFromNearestVBase,
2509 bool BaseIsNonVirtualPrimaryBase,
2510 const CXXRecordDecl *VTableClass,
2511 VisitedVirtualBasesSetTy &VBases, VPtrsVector &vptrs);
2512
2513 void InitializeVTablePointers(const CXXRecordDecl *ClassDecl);
2514
2515 // VTableTrapMode - whether we guarantee that loading the
2516 // vtable is guaranteed to trap on authentication failure,
2517 // even if the resulting vtable pointer is unused.
2518 enum class VTableAuthMode {
2519 Authenticate,
2520 MustTrap,
2521 UnsafeUbsanStrip // Should only be used for Vptr UBSan check
2522 };
2523 /// GetVTablePtr - Return the Value of the vtable pointer member pointed
2524 /// to by This.
2525 llvm::Value *
2526 GetVTablePtr(Address This, llvm::Type *VTableTy,
2527 const CXXRecordDecl *VTableClass,
2528 VTableAuthMode AuthMode = VTableAuthMode::Authenticate);
2529
2530 enum CFITypeCheckKind {
2531 CFITCK_VCall,
2532 CFITCK_NVCall,
2533 CFITCK_DerivedCast,
2534 CFITCK_UnrelatedCast,
2535 CFITCK_ICall,
2536 CFITCK_NVMFCall,
2537 CFITCK_VMFCall,
2538 };
2539
2540 /// Derived is the presumed address of an object of type T after a
2541 /// cast. If T is a polymorphic class type, emit a check that the virtual
2542 /// table for Derived belongs to a class derived from T.
2543 void EmitVTablePtrCheckForCast(QualType T, Address Derived, bool MayBeNull,
2544 CFITypeCheckKind TCK, SourceLocation Loc);
2545
2546 /// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable.
2547 /// If vptr CFI is enabled, emit a check that VTable is valid.
2548 void EmitVTablePtrCheckForCall(const CXXRecordDecl *RD, llvm::Value *VTable,
2549 CFITypeCheckKind TCK, SourceLocation Loc);
2550
2551 /// EmitVTablePtrCheck - Emit a check that VTable is a valid virtual table for
2552 /// RD using llvm.type.test.
2553 void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable,
2554 CFITypeCheckKind TCK, SourceLocation Loc);
2555
2556 /// If whole-program virtual table optimization is enabled, emit an assumption
2557 /// that VTable is a member of RD's type identifier. Or, if vptr CFI is
2558 /// enabled, emit a check that VTable is a member of RD's type identifier.
2559 void EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2560 llvm::Value *VTable, SourceLocation Loc);
2561
2562 /// Returns whether we should perform a type checked load when loading a
2563 /// virtual function for virtual calls to members of RD. This is generally
2564 /// true when both vcall CFI and whole-program-vtables are enabled.
2565 bool ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD);
2566
2567 /// Emit a type checked load from the given vtable.
2568 llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD,
2569 llvm::Value *VTable,
2570 llvm::Type *VTableTy,
2571 uint64_t VTableByteOffset);
2572
2573 /// EnterDtorCleanups - Enter the cleanups necessary to complete the
2574 /// given phase of destruction for a destructor. The end result
2575 /// should call destructors on members and base classes in reverse
2576 /// order of their construction.
2577 void EnterDtorCleanups(const CXXDestructorDecl *Dtor, CXXDtorType Type);
2578
2579 /// ShouldInstrumentFunction - Return true if the current function should be
2580 /// instrumented with __cyg_profile_func_* calls
2581 bool ShouldInstrumentFunction();
2582
2583 /// ShouldSkipSanitizerInstrumentation - Return true if the current function
2584 /// should not be instrumented with sanitizers.
2585 bool ShouldSkipSanitizerInstrumentation();
2586
2587 /// ShouldXRayInstrument - Return true if the current function should be
2588 /// instrumented with XRay nop sleds.
2589 bool ShouldXRayInstrumentFunction() const;
2590
2591 /// AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit
2592 /// XRay custom event handling calls.
2593 bool AlwaysEmitXRayCustomEvents() const;
2594
2595 /// AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit
2596 /// XRay typed event handling calls.
2597 bool AlwaysEmitXRayTypedEvents() const;
2598
2599 /// Return a type hash constant for a function instrumented by
2600 /// -fsanitize=function.
2601 llvm::ConstantInt *getUBSanFunctionTypeHash(QualType T) const;
2602
2603 /// EmitFunctionProlog - Emit the target specific LLVM code to load the
2604 /// arguments for the given function. This is also responsible for naming the
2605 /// LLVM function arguments.
2606 void EmitFunctionProlog(const CGFunctionInfo &FI, llvm::Function *Fn,
2607 const FunctionArgList &Args);
2608
2609 /// EmitFunctionEpilog - Emit the target specific LLVM code to return the
2610 /// given temporary. Specify the source location atom group (Key Instructions
2611 /// debug info feature) for the `ret` using \p RetKeyInstructionsSourceAtom.
2612 /// If it's 0, the `ret` will get added to a new source atom group.
2613 void EmitFunctionEpilog(const CGFunctionInfo &FI, bool EmitRetDbgLoc,
2614 SourceLocation EndLoc,
2615 uint64_t RetKeyInstructionsSourceAtom);
2616
2617 /// Emit a test that checks if the return value \p RV is nonnull.
2618 void EmitReturnValueCheck(llvm::Value *RV);
2619
2620 /// EmitStartEHSpec - Emit the start of the exception spec.
2621 void EmitStartEHSpec(const Decl *D);
2622
2623 /// EmitEndEHSpec - Emit the end of the exception spec.
2624 void EmitEndEHSpec(const Decl *D);
2625
2626 /// getTerminateLandingPad - Return a landing pad that just calls terminate.
2627 llvm::BasicBlock *getTerminateLandingPad();
2628
2629 /// getTerminateLandingPad - Return a cleanup funclet that just calls
2630 /// terminate.
2631 llvm::BasicBlock *getTerminateFunclet();
2632
2633 /// getTerminateHandler - Return a handler (not a landing pad, just
2634 /// a catch handler) that just calls terminate. This is used when
2635 /// a terminate scope encloses a try.
2636 llvm::BasicBlock *getTerminateHandler();
2637
2638 llvm::Type *ConvertTypeForMem(QualType T);
2639 llvm::Type *ConvertType(QualType T);
2640 llvm::Type *convertTypeForLoadStore(QualType ASTTy,
2641 llvm::Type *LLVMTy = nullptr);
2642 llvm::Type *ConvertType(const TypeDecl *T) {
2643 return ConvertType(T: getContext().getTypeDeclType(Decl: T));
2644 }
2645
2646 /// LoadObjCSelf - Load the value of self. This function is only valid while
2647 /// generating code for an Objective-C method.
2648 llvm::Value *LoadObjCSelf();
2649
2650 /// TypeOfSelfObject - Return type of object that this self represents.
2651 QualType TypeOfSelfObject();
2652
2653 /// getEvaluationKind - Return the TypeEvaluationKind of QualType \c T.
2654 static TypeEvaluationKind getEvaluationKind(QualType T);
2655
2656 static bool hasScalarEvaluationKind(QualType T) {
2657 return getEvaluationKind(T) == TEK_Scalar;
2658 }
2659
2660 static bool hasAggregateEvaluationKind(QualType T) {
2661 return getEvaluationKind(T) == TEK_Aggregate;
2662 }
2663
2664 /// createBasicBlock - Create an LLVM basic block.
2665 llvm::BasicBlock *createBasicBlock(const Twine &name = "",
2666 llvm::Function *parent = nullptr,
2667 llvm::BasicBlock *before = nullptr) {
2668 return llvm::BasicBlock::Create(Context&: getLLVMContext(), Name: name, Parent: parent, InsertBefore: before);
2669 }
2670
2671 /// getBasicBlockForLabel - Return the LLVM basicblock that the specified
2672 /// label maps to.
2673 JumpDest getJumpDestForLabel(const LabelDecl *S);
2674
2675 /// SimplifyForwardingBlocks - If the given basic block is only a branch to
2676 /// another basic block, simplify it. This assumes that no other code could
2677 /// potentially reference the basic block.
2678 void SimplifyForwardingBlocks(llvm::BasicBlock *BB);
2679
2680 /// EmitBlock - Emit the given block \arg BB and set it as the insert point,
2681 /// adding a fall-through branch from the current insert block if
2682 /// necessary. It is legal to call this function even if there is no current
2683 /// insertion point.
2684 ///
2685 /// IsFinished - If true, indicates that the caller has finished emitting
2686 /// branches to the given block and does not expect to emit code into it. This
2687 /// means the block can be ignored if it is unreachable.
2688 void EmitBlock(llvm::BasicBlock *BB, bool IsFinished = false);
2689
2690 /// EmitBlockAfterUses - Emit the given block somewhere hopefully
2691 /// near its uses, and leave the insertion point in it.
2692 void EmitBlockAfterUses(llvm::BasicBlock *BB);
2693
2694 /// EmitBranch - Emit a branch to the specified basic block from the current
2695 /// insert block, taking care to avoid creation of branches from dummy
2696 /// blocks. It is legal to call this function even if there is no current
2697 /// insertion point.
2698 ///
2699 /// This function clears the current insertion point. The caller should follow
2700 /// calls to this function with calls to Emit*Block prior to generation new
2701 /// code.
2702 void EmitBranch(llvm::BasicBlock *Block);
2703
2704 /// HaveInsertPoint - True if an insertion point is defined. If not, this
2705 /// indicates that the current code being emitted is unreachable.
2706 bool HaveInsertPoint() const { return Builder.GetInsertBlock() != nullptr; }
2707
2708 /// EnsureInsertPoint - Ensure that an insertion point is defined so that
2709 /// emitted IR has a place to go. Note that by definition, if this function
2710 /// creates a block then that block is unreachable; callers may do better to
2711 /// detect when no insertion point is defined and simply skip IR generation.
2712 void EnsureInsertPoint() {
2713 if (!HaveInsertPoint())
2714 EmitBlock(BB: createBasicBlock());
2715 }
2716
2717 /// ErrorUnsupported - Print out an error that codegen doesn't support the
2718 /// specified stmt yet.
2719 void ErrorUnsupported(const Stmt *S, const char *Type);
2720
2721 //===--------------------------------------------------------------------===//
2722 // Helpers
2723 //===--------------------------------------------------------------------===//
2724
2725 Address mergeAddressesInConditionalExpr(Address LHS, Address RHS,
2726 llvm::BasicBlock *LHSBlock,
2727 llvm::BasicBlock *RHSBlock,
2728 llvm::BasicBlock *MergeBlock,
2729 QualType MergedType) {
2730 Builder.SetInsertPoint(MergeBlock);
2731 llvm::PHINode *PtrPhi = Builder.CreatePHI(Ty: LHS.getType(), NumReservedValues: 2, Name: "cond");
2732 PtrPhi->addIncoming(V: LHS.getBasePointer(), BB: LHSBlock);
2733 PtrPhi->addIncoming(V: RHS.getBasePointer(), BB: RHSBlock);
2734 LHS.replaceBasePointer(P: PtrPhi);
2735 LHS.setAlignment(std::min(a: LHS.getAlignment(), b: RHS.getAlignment()));
2736 return LHS;
2737 }
2738
2739 /// Construct an address with the natural alignment of T. If a pointer to T
2740 /// is expected to be signed, the pointer passed to this function must have
2741 /// been signed, and the returned Address will have the pointer authentication
2742 /// information needed to authenticate the signed pointer.
2743 Address makeNaturalAddressForPointer(
2744 llvm::Value *Ptr, QualType T, CharUnits Alignment = CharUnits::Zero(),
2745 bool ForPointeeType = false, LValueBaseInfo *BaseInfo = nullptr,
2746 TBAAAccessInfo *TBAAInfo = nullptr,
2747 KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
2748 if (Alignment.isZero())
2749 Alignment =
2750 CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, forPointeeType: ForPointeeType);
2751 return Address(Ptr, ConvertTypeForMem(T), Alignment,
2752 CGM.getPointerAuthInfoForPointeeType(type: T), /*Offset=*/nullptr,
2753 IsKnownNonNull);
2754 }
2755
2756 LValue MakeAddrLValue(Address Addr, QualType T,
2757 AlignmentSource Source = AlignmentSource::Type) {
2758 return MakeAddrLValue(Addr, T, BaseInfo: LValueBaseInfo(Source),
2759 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2760 }
2761
2762 LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo,
2763 TBAAAccessInfo TBAAInfo) {
2764 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo, TBAAInfo);
2765 }
2766
2767 LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2768 AlignmentSource Source = AlignmentSource::Type) {
2769 return MakeAddrLValue(Addr: makeNaturalAddressForPointer(Ptr: V, T, Alignment), T,
2770 BaseInfo: LValueBaseInfo(Source), TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2771 }
2772
2773 /// Same as MakeAddrLValue above except that the pointer is known to be
2774 /// unsigned.
2775 LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2776 AlignmentSource Source = AlignmentSource::Type) {
2777 Address Addr(V, ConvertTypeForMem(T), Alignment);
2778 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2779 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2780 }
2781
2782 LValue
2783 MakeAddrLValueWithoutTBAA(Address Addr, QualType T,
2784 AlignmentSource Source = AlignmentSource::Type) {
2785 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2786 TBAAInfo: TBAAAccessInfo());
2787 }
2788
2789 /// Given a value of type T* that may not be to a complete object, construct
2790 /// an l-value with the natural pointee alignment of T.
2791 LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T);
2792
2793 LValue
2794 MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
2795 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
2796
2797 /// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known
2798 /// to be unsigned.
2799 LValue MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, QualType T);
2800
2801 LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T);
2802
2803 Address EmitLoadOfReference(LValue RefLVal,
2804 LValueBaseInfo *PointeeBaseInfo = nullptr,
2805 TBAAAccessInfo *PointeeTBAAInfo = nullptr);
2806 LValue EmitLoadOfReferenceLValue(LValue RefLVal);
2807 LValue
2808 EmitLoadOfReferenceLValue(Address RefAddr, QualType RefTy,
2809 AlignmentSource Source = AlignmentSource::Type) {
2810 LValue RefLVal = MakeAddrLValue(Addr: RefAddr, T: RefTy, BaseInfo: LValueBaseInfo(Source),
2811 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: RefTy));
2812 return EmitLoadOfReferenceLValue(RefLVal);
2813 }
2814
2815 /// Load a pointer with type \p PtrTy stored at address \p Ptr.
2816 /// Note that \p PtrTy is the type of the loaded pointer, not the addresses
2817 /// it is loaded from.
2818 Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy,
2819 LValueBaseInfo *BaseInfo = nullptr,
2820 TBAAAccessInfo *TBAAInfo = nullptr);
2821 LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy);
2822
2823private:
2824 struct AllocaTracker {
2825 void Add(llvm::AllocaInst *I) { Allocas.push_back(Elt: I); }
2826 llvm::SmallVector<llvm::AllocaInst *> Take() { return std::move(Allocas); }
2827
2828 private:
2829 llvm::SmallVector<llvm::AllocaInst *> Allocas;
2830 };
2831 AllocaTracker *Allocas = nullptr;
2832
2833 /// CGDecl helper.
2834 void emitStoresForConstant(const VarDecl &D, Address Loc, bool isVolatile,
2835 llvm::Constant *constant, bool IsAutoInit);
2836 /// CGDecl helper.
2837 void emitStoresForZeroInit(const VarDecl &D, Address Loc, bool isVolatile);
2838 /// CGDecl helper.
2839 void emitStoresForPatternInit(const VarDecl &D, Address Loc, bool isVolatile);
2840 /// CGDecl helper.
2841 void emitStoresForInitAfterBZero(llvm::Constant *Init, Address Loc,
2842 bool isVolatile, bool IsAutoInit);
2843
2844public:
2845 // Captures all the allocas created during the scope of its RAII object.
2846 struct AllocaTrackerRAII {
2847 AllocaTrackerRAII(CodeGenFunction &CGF)
2848 : CGF(CGF), OldTracker(CGF.Allocas) {
2849 CGF.Allocas = &Tracker;
2850 }
2851 ~AllocaTrackerRAII() { CGF.Allocas = OldTracker; }
2852
2853 llvm::SmallVector<llvm::AllocaInst *> Take() { return Tracker.Take(); }
2854
2855 private:
2856 CodeGenFunction &CGF;
2857 AllocaTracker *OldTracker;
2858 AllocaTracker Tracker;
2859 };
2860
2861private:
2862 /// If \p Alloca is not in the same address space as \p DestLangAS, insert an
2863 /// address space cast and return a new RawAddress based on this value.
2864 RawAddress MaybeCastStackAddressSpace(RawAddress Alloca, LangAS DestLangAS,
2865 llvm::Value *ArraySize = nullptr);
2866
2867public:
2868 /// CreateTempAlloca - This creates an alloca and inserts it into the entry
2869 /// block if \p ArraySize is nullptr, otherwise inserts it at the current
2870 /// insertion point of the builder. The caller is responsible for setting an
2871 /// appropriate alignment on
2872 /// the alloca.
2873 ///
2874 /// \p ArraySize is the number of array elements to be allocated if it
2875 /// is not nullptr.
2876 ///
2877 /// LangAS::Default is the address space of pointers to local variables and
2878 /// temporaries, as exposed in the source language. In certain
2879 /// configurations, this is not the same as the alloca address space, and a
2880 /// cast is needed to lift the pointer from the alloca AS into
2881 /// LangAS::Default. This can happen when the target uses a restricted
2882 /// address space for the stack but the source language requires
2883 /// LangAS::Default to be a generic address space. The latter condition is
2884 /// common for most programming languages; OpenCL is an exception in that
2885 /// LangAS::Default is the private address space, which naturally maps
2886 /// to the stack.
2887 ///
2888 /// Because the address of a temporary is often exposed to the program in
2889 /// various ways, this function will perform the cast. The original alloca
2890 /// instruction is returned through \p Alloca if it is not nullptr.
2891 ///
2892 /// The cast is not performaed in CreateTempAllocaWithoutCast. This is
2893 /// more efficient if the caller knows that the address will not be exposed.
2894 llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp",
2895 llvm::Value *ArraySize = nullptr);
2896
2897 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
2898 /// block. The alloca is casted to the address space of \p UseAddrSpace if
2899 /// necessary.
2900 RawAddress CreateTempAlloca(llvm::Type *Ty, LangAS UseAddrSpace,
2901 CharUnits align, const Twine &Name = "tmp",
2902 llvm::Value *ArraySize = nullptr,
2903 RawAddress *Alloca = nullptr);
2904
2905 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
2906 /// block. The alloca is casted to default address space if necessary.
2907 ///
2908 /// FIXME: This version should be removed, and context should provide the
2909 /// context use address space used instead of default.
2910 RawAddress CreateTempAlloca(llvm::Type *Ty, CharUnits align,
2911 const Twine &Name = "tmp",
2912 llvm::Value *ArraySize = nullptr,
2913 RawAddress *Alloca = nullptr) {
2914 return CreateTempAlloca(Ty, UseAddrSpace: LangAS::Default, align, Name, ArraySize,
2915 Alloca);
2916 }
2917
2918 RawAddress CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align,
2919 const Twine &Name = "tmp",
2920 llvm::Value *ArraySize = nullptr);
2921
2922 /// CreateDefaultAlignedTempAlloca - This creates an alloca with the
2923 /// default ABI alignment of the given LLVM type.
2924 ///
2925 /// IMPORTANT NOTE: This is *not* generally the right alignment for
2926 /// any given AST type that happens to have been lowered to the
2927 /// given IR type. This should only ever be used for function-local,
2928 /// IR-driven manipulations like saving and restoring a value. Do
2929 /// not hand this address off to arbitrary IRGen routines, and especially
2930 /// do not pass it as an argument to a function that might expect a
2931 /// properly ABI-aligned value.
2932 RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty,
2933 const Twine &Name = "tmp");
2934
2935 /// CreateIRTempWithoutCast - Create a temporary IR object of the given type,
2936 /// with appropriate alignment. This routine should only be used when an
2937 /// temporary value needs to be stored into an alloca (for example, to avoid
2938 /// explicit PHI construction), but the type is the IR type, not the type
2939 /// appropriate for storing in memory.
2940 ///
2941 /// That is, this is exactly equivalent to CreateMemTemp, but calling
2942 /// ConvertType instead of ConvertTypeForMem.
2943 RawAddress CreateIRTempWithoutCast(QualType T, const Twine &Name = "tmp");
2944
2945 /// CreateMemTemp - Create a temporary memory object of the given type, with
2946 /// appropriate alignmen and cast it to the default address space. Returns
2947 /// the original alloca instruction by \p Alloca if it is not nullptr.
2948 RawAddress CreateMemTemp(QualType T, const Twine &Name = "tmp",
2949 RawAddress *Alloca = nullptr);
2950 RawAddress CreateMemTemp(QualType T, CharUnits Align,
2951 const Twine &Name = "tmp",
2952 RawAddress *Alloca = nullptr);
2953
2954 /// CreateMemTemp - Create a temporary memory object of the given type, with
2955 /// appropriate alignmen without casting it to the default address space.
2956 RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp");
2957 RawAddress CreateMemTempWithoutCast(QualType T, CharUnits Align,
2958 const Twine &Name = "tmp");
2959
2960 /// CreateAggTemp - Create a temporary memory object for the given
2961 /// aggregate type.
2962 AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp",
2963 RawAddress *Alloca = nullptr) {
2964 return AggValueSlot::forAddr(
2965 addr: CreateMemTemp(T, Name, Alloca), quals: T.getQualifiers(),
2966 isDestructed: AggValueSlot::IsNotDestructed, needsGC: AggValueSlot::DoesNotNeedGCBarriers,
2967 isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap);
2968 }
2969
2970 /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
2971 /// expression and compare the result against zero, returning an Int1Ty value.
2972 llvm::Value *EvaluateExprAsBool(const Expr *E);
2973
2974 /// Retrieve the implicit cast expression of the rhs in a binary operator
2975 /// expression by passing pointers to Value and QualType
2976 /// This is used for implicit bitfield conversion checks, which
2977 /// must compare with the value before potential truncation.
2978 llvm::Value *EmitWithOriginalRHSBitfieldAssignment(const BinaryOperator *E,
2979 llvm::Value **Previous,
2980 QualType *SrcType);
2981
2982 /// Emit a check that an [implicit] conversion of a bitfield. It is not UB,
2983 /// so we use the value after conversion.
2984 void EmitBitfieldConversionCheck(llvm::Value *Src, QualType SrcType,
2985 llvm::Value *Dst, QualType DstType,
2986 const CGBitFieldInfo &Info,
2987 SourceLocation Loc);
2988
2989 /// EmitIgnoredExpr - Emit an expression in a context which ignores the
2990 /// result.
2991 void EmitIgnoredExpr(const Expr *E);
2992
2993 /// EmitAnyExpr - Emit code to compute the specified expression which can have
2994 /// any type. The result is returned as an RValue struct. If this is an
2995 /// aggregate expression, the aggloc/agglocvolatile arguments indicate where
2996 /// the result should be returned.
2997 ///
2998 /// \param ignoreResult True if the resulting value isn't used.
2999 RValue EmitAnyExpr(const Expr *E,
3000 AggValueSlot aggSlot = AggValueSlot::ignored(),
3001 bool ignoreResult = false);
3002
3003 // EmitVAListRef - Emit a "reference" to a va_list; this is either the address
3004 // or the value of the expression, depending on how va_list is defined.
3005 Address EmitVAListRef(const Expr *E);
3006
3007 /// Emit a "reference" to a __builtin_ms_va_list; this is
3008 /// always the value of the expression, because a __builtin_ms_va_list is a
3009 /// pointer to a char.
3010 Address EmitMSVAListRef(const Expr *E);
3011
3012 /// EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will
3013 /// always be accessible even if no aggregate location is provided.
3014 RValue EmitAnyExprToTemp(const Expr *E);
3015
3016 /// EmitAnyExprToMem - Emits the code necessary to evaluate an
3017 /// arbitrary expression into the given memory location.
3018 void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals,
3019 bool IsInitializer);
3020
3021 void EmitAnyExprToExn(const Expr *E, Address Addr);
3022
3023 /// EmitInitializationToLValue - Emit an initializer to an LValue.
3024 void EmitInitializationToLValue(
3025 const Expr *E, LValue LV,
3026 AggValueSlot::IsZeroed_t IsZeroed = AggValueSlot::IsNotZeroed);
3027
3028 /// EmitExprAsInit - Emits the code necessary to initialize a
3029 /// location in memory with the given initializer.
3030 void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue,
3031 bool capturedByInit);
3032
3033 /// hasVolatileMember - returns true if aggregate type has a volatile
3034 /// member.
3035 bool hasVolatileMember(QualType T) {
3036 if (const auto *RD = T->getAsRecordDecl())
3037 return RD->hasVolatileMember();
3038 return false;
3039 }
3040
3041 /// Determine whether a return value slot may overlap some other object.
3042 AggValueSlot::Overlap_t getOverlapForReturnValue() {
3043 // FIXME: Assuming no overlap here breaks guaranteed copy elision for base
3044 // class subobjects. These cases may need to be revisited depending on the
3045 // resolution of the relevant core issue.
3046 return AggValueSlot::DoesNotOverlap;
3047 }
3048
3049 /// Determine whether a field initialization may overlap some other object.
3050 AggValueSlot::Overlap_t getOverlapForFieldInit(const FieldDecl *FD);
3051
3052 /// Determine whether a base class initialization may overlap some other
3053 /// object.
3054 AggValueSlot::Overlap_t getOverlapForBaseInit(const CXXRecordDecl *RD,
3055 const CXXRecordDecl *BaseRD,
3056 bool IsVirtual);
3057
3058 /// Emit an aggregate assignment.
3059 void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy) {
3060 ApplyAtomGroup Grp(getDebugInfo());
3061 bool IsVolatile = hasVolatileMember(T: EltTy);
3062 EmitAggregateCopy(Dest, Src, EltTy, MayOverlap: AggValueSlot::MayOverlap, isVolatile: IsVolatile);
3063 }
3064
3065 void EmitAggregateCopyCtor(LValue Dest, LValue Src,
3066 AggValueSlot::Overlap_t MayOverlap) {
3067 EmitAggregateCopy(Dest, Src, EltTy: Src.getType(), MayOverlap);
3068 }
3069
3070 /// EmitAggregateCopy - Emit an aggregate copy.
3071 ///
3072 /// \param isVolatile \c true iff either the source or the destination is
3073 /// volatile.
3074 /// \param MayOverlap Whether the tail padding of the destination might be
3075 /// occupied by some other object. More efficient code can often be
3076 /// generated if not.
3077 void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy,
3078 AggValueSlot::Overlap_t MayOverlap,
3079 bool isVolatile = false);
3080
3081 /// GetAddrOfLocalVar - Return the address of a local variable.
3082 Address GetAddrOfLocalVar(const VarDecl *VD) {
3083 auto it = LocalDeclMap.find(Val: VD);
3084 assert(it != LocalDeclMap.end() &&
3085 "Invalid argument to GetAddrOfLocalVar(), no decl!");
3086 return it->second;
3087 }
3088
3089 /// Given an opaque value expression, return its LValue mapping if it exists,
3090 /// otherwise create one.
3091 LValue getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e);
3092
3093 /// Given an opaque value expression, return its RValue mapping if it exists,
3094 /// otherwise create one.
3095 RValue getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e);
3096
3097 /// isOpaqueValueEmitted - Return true if the opaque value expression has
3098 /// already been emitted.
3099 bool isOpaqueValueEmitted(const OpaqueValueExpr *E);
3100
3101 /// Get the index of the current ArrayInitLoopExpr, if any.
3102 llvm::Value *getArrayInitIndex() { return ArrayInitIndex; }
3103
3104 /// getAccessedFieldNo - Given an encoded value and a result number, return
3105 /// the input field number being accessed.
3106 static unsigned getAccessedFieldNo(unsigned Idx, const llvm::Constant *Elts);
3107
3108 llvm::BlockAddress *GetAddrOfLabel(const LabelDecl *L);
3109 llvm::BasicBlock *GetIndirectGotoBlock();
3110
3111 /// Check if \p E is a C++ "this" pointer wrapped in value-preserving casts.
3112 static bool IsWrappedCXXThis(const Expr *E);
3113
3114 /// EmitNullInitialization - Generate code to set a value of the given type to
3115 /// null, If the type contains data member pointers, they will be initialized
3116 /// to -1 in accordance with the Itanium C++ ABI.
3117 void EmitNullInitialization(Address DestPtr, QualType Ty);
3118
3119 /// Emits a call to an LLVM variable-argument intrinsic, either
3120 /// \c llvm.va_start or \c llvm.va_end.
3121 /// \param ArgValue A reference to the \c va_list as emitted by either
3122 /// \c EmitVAListRef or \c EmitMSVAListRef.
3123 /// \param IsStart If \c true, emits a call to \c llvm.va_start; otherwise,
3124 /// calls \c llvm.va_end.
3125 llvm::Value *EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart);
3126
3127 /// Generate code to get an argument from the passed in pointer
3128 /// and update it accordingly.
3129 /// \param VE The \c VAArgExpr for which to generate code.
3130 /// \param VAListAddr Receives a reference to the \c va_list as emitted by
3131 /// either \c EmitVAListRef or \c EmitMSVAListRef.
3132 /// \returns A pointer to the argument.
3133 // FIXME: We should be able to get rid of this method and use the va_arg
3134 // instruction in LLVM instead once it works well enough.
3135 RValue EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
3136 AggValueSlot Slot = AggValueSlot::ignored());
3137
3138 /// emitArrayLength - Compute the length of an array, even if it's a
3139 /// VLA, and drill down to the base element type.
3140 llvm::Value *emitArrayLength(const ArrayType *arrayType, QualType &baseType,
3141 Address &addr);
3142
3143 /// EmitVLASize - Capture all the sizes for the VLA expressions in
3144 /// the given variably-modified type and store them in the VLASizeMap.
3145 ///
3146 /// This function can be called with a null (unreachable) insert point.
3147 void EmitVariablyModifiedType(QualType Ty);
3148
3149 struct VlaSizePair {
3150 llvm::Value *NumElts;
3151 QualType Type;
3152
3153 VlaSizePair(llvm::Value *NE, QualType T) : NumElts(NE), Type(T) {}
3154 };
3155
3156 /// Return the number of elements for a single dimension
3157 /// for the given array type.
3158 VlaSizePair getVLAElements1D(const VariableArrayType *vla);
3159 VlaSizePair getVLAElements1D(QualType vla);
3160
3161 /// Returns an LLVM value that corresponds to the size,
3162 /// in non-variably-sized elements, of a variable length array type,
3163 /// plus that largest non-variably-sized element type. Assumes that
3164 /// the type has already been emitted with EmitVariablyModifiedType.
3165 VlaSizePair getVLASize(const VariableArrayType *vla);
3166 VlaSizePair getVLASize(QualType vla);
3167
3168 /// LoadCXXThis - Load the value of 'this'. This function is only valid while
3169 /// generating code for an C++ member function.
3170 llvm::Value *LoadCXXThis() {
3171 assert(CXXThisValue && "no 'this' value for this function");
3172 return CXXThisValue;
3173 }
3174 Address LoadCXXThisAddress();
3175
3176 /// LoadCXXVTT - Load the VTT parameter to base constructors/destructors have
3177 /// virtual bases.
3178 // FIXME: Every place that calls LoadCXXVTT is something
3179 // that needs to be abstracted properly.
3180 llvm::Value *LoadCXXVTT() {
3181 assert(CXXStructorImplicitParamValue && "no VTT value for this function");
3182 return CXXStructorImplicitParamValue;
3183 }
3184
3185 /// GetAddressOfBaseOfCompleteClass - Convert the given pointer to a
3186 /// complete class to the given direct base.
3187 Address GetAddressOfDirectBaseInCompleteClass(Address Value,
3188 const CXXRecordDecl *Derived,
3189 const CXXRecordDecl *Base,
3190 bool BaseIsVirtual);
3191
3192 static bool ShouldNullCheckClassCastValue(const CastExpr *Cast);
3193
3194 /// GetAddressOfBaseClass - This function will add the necessary delta to the
3195 /// load of 'this' and returns address of the base class.
3196 Address GetAddressOfBaseClass(Address Value, const CXXRecordDecl *Derived,
3197 CastExpr::path_const_iterator PathBegin,
3198 CastExpr::path_const_iterator PathEnd,
3199 bool NullCheckValue, SourceLocation Loc);
3200
3201 Address GetAddressOfDerivedClass(Address Value, const CXXRecordDecl *Derived,
3202 CastExpr::path_const_iterator PathBegin,
3203 CastExpr::path_const_iterator PathEnd,
3204 bool NullCheckValue);
3205
3206 /// GetVTTParameter - Return the VTT parameter that should be passed to a
3207 /// base constructor/destructor with virtual bases.
3208 /// FIXME: VTTs are Itanium ABI-specific, so the definition should move
3209 /// to ItaniumCXXABI.cpp together with all the references to VTT.
3210 llvm::Value *GetVTTParameter(GlobalDecl GD, bool ForVirtualBase,
3211 bool Delegating);
3212
3213 void EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
3214 CXXCtorType CtorType,
3215 const FunctionArgList &Args,
3216 SourceLocation Loc);
3217 // It's important not to confuse this and the previous function. Delegating
3218 // constructors are the C++0x feature. The constructor delegate optimization
3219 // is used to reduce duplication in the base and complete consturctors where
3220 // they are substantially the same.
3221 void EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3222 const FunctionArgList &Args);
3223
3224 /// Emit a call to an inheriting constructor (that is, one that invokes a
3225 /// constructor inherited from a base class) by inlining its definition. This
3226 /// is necessary if the ABI does not support forwarding the arguments to the
3227 /// base class constructor (because they're variadic or similar).
3228 void EmitInlinedInheritingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3229 CXXCtorType CtorType,
3230 bool ForVirtualBase,
3231 bool Delegating,
3232 CallArgList &Args);
3233
3234 /// Emit a call to a constructor inherited from a base class, passing the
3235 /// current constructor's arguments along unmodified (without even making
3236 /// a copy).
3237 void EmitInheritedCXXConstructorCall(const CXXConstructorDecl *D,
3238 bool ForVirtualBase, Address This,
3239 bool InheritedFromVBase,
3240 const CXXInheritedCtorInitExpr *E);
3241
3242 void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3243 bool ForVirtualBase, bool Delegating,
3244 AggValueSlot ThisAVS, const CXXConstructExpr *E);
3245
3246 void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3247 bool ForVirtualBase, bool Delegating,
3248 Address This, CallArgList &Args,
3249 AggValueSlot::Overlap_t Overlap,
3250 SourceLocation Loc, bool NewPointerIsChecked,
3251 llvm::CallBase **CallOrInvoke = nullptr);
3252
3253 /// Emit assumption load for all bases. Requires to be called only on
3254 /// most-derived class and not under construction of the object.
3255 void EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl, Address This);
3256
3257 /// Emit assumption that vptr load == global vtable.
3258 void EmitVTableAssumptionLoad(const VPtr &vptr, Address This);
3259
3260 void EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, Address This,
3261 Address Src, const CXXConstructExpr *E);
3262
3263 void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3264 const ArrayType *ArrayTy, Address ArrayPtr,
3265 const CXXConstructExpr *E,
3266 bool NewPointerIsChecked,
3267 bool ZeroInitialization = false);
3268
3269 void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3270 llvm::Value *NumElements, Address ArrayPtr,
3271 const CXXConstructExpr *E,
3272 bool NewPointerIsChecked,
3273 bool ZeroInitialization = false);
3274
3275 static Destroyer destroyCXXObject;
3276
3277 void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type,
3278 bool ForVirtualBase, bool Delegating, Address This,
3279 QualType ThisTy);
3280
3281 void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
3282 llvm::Type *ElementTy, Address NewPtr,
3283 llvm::Value *NumElements,
3284 llvm::Value *AllocSizeWithoutCookie);
3285
3286 void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
3287 Address Ptr);
3288
3289 void EmitSehCppScopeBegin();
3290 void EmitSehCppScopeEnd();
3291 void EmitSehTryScopeBegin();
3292 void EmitSehTryScopeEnd();
3293
3294 bool EmitLifetimeStart(llvm::Value *Addr);
3295 void EmitLifetimeEnd(llvm::Value *Addr);
3296
3297 llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
3298 void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
3299
3300 void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,
3301 QualType DeleteTy, llvm::Value *NumElements = nullptr,
3302 CharUnits CookieSize = CharUnits());
3303
3304 RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
3305 const CallExpr *TheCallExpr, bool IsDelete);
3306
3307 llvm::Value *EmitCXXTypeidExpr(const CXXTypeidExpr *E);
3308 llvm::Value *EmitDynamicCast(Address V, const CXXDynamicCastExpr *DCE);
3309 Address EmitCXXUuidofExpr(const CXXUuidofExpr *E);
3310
3311 /// Situations in which we might emit a check for the suitability of a
3312 /// pointer or glvalue. Needs to be kept in sync with ubsan_handlers.cpp in
3313 /// compiler-rt.
3314 enum TypeCheckKind {
3315 /// Checking the operand of a load. Must be suitably sized and aligned.
3316 TCK_Load,
3317 /// Checking the destination of a store. Must be suitably sized and aligned.
3318 TCK_Store,
3319 /// Checking the bound value in a reference binding. Must be suitably sized
3320 /// and aligned, but is not required to refer to an object (until the
3321 /// reference is used), per core issue 453.
3322 TCK_ReferenceBinding,
3323 /// Checking the object expression in a non-static data member access. Must
3324 /// be an object within its lifetime.
3325 TCK_MemberAccess,
3326 /// Checking the 'this' pointer for a call to a non-static member function.
3327 /// Must be an object within its lifetime.
3328 TCK_MemberCall,
3329 /// Checking the 'this' pointer for a constructor call.
3330 TCK_ConstructorCall,
3331 /// Checking the operand of a static_cast to a derived pointer type. Must be
3332 /// null or an object within its lifetime.
3333 TCK_DowncastPointer,
3334 /// Checking the operand of a static_cast to a derived reference type. Must
3335 /// be an object within its lifetime.
3336 TCK_DowncastReference,
3337 /// Checking the operand of a cast to a base object. Must be suitably sized
3338 /// and aligned.
3339 TCK_Upcast,
3340 /// Checking the operand of a cast to a virtual base object. Must be an
3341 /// object within its lifetime.
3342 TCK_UpcastToVirtualBase,
3343 /// Checking the value assigned to a _Nonnull pointer. Must not be null.
3344 TCK_NonnullAssign,
3345 /// Checking the operand of a dynamic_cast or a typeid expression. Must be
3346 /// null or an object within its lifetime.
3347 TCK_DynamicOperation
3348 };
3349
3350 /// Determine whether the pointer type check \p TCK permits null pointers.
3351 static bool isNullPointerAllowed(TypeCheckKind TCK);
3352
3353 /// Determine whether the pointer type check \p TCK requires a vptr check.
3354 static bool isVptrCheckRequired(TypeCheckKind TCK, QualType Ty);
3355
3356 /// Whether any type-checking sanitizers are enabled. If \c false,
3357 /// calls to EmitTypeCheck can be skipped.
3358 bool sanitizePerformTypeCheck() const;
3359
3360 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV,
3361 QualType Type, SanitizerSet SkippedChecks = SanitizerSet(),
3362 llvm::Value *ArraySize = nullptr) {
3363 if (!sanitizePerformTypeCheck())
3364 return;
3365 EmitTypeCheck(TCK, Loc, V: LV.emitRawPointer(CGF&: *this), Type, Alignment: LV.getAlignment(),
3366 SkippedChecks, ArraySize);
3367 }
3368
3369 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, Address Addr,
3370 QualType Type, CharUnits Alignment = CharUnits::Zero(),
3371 SanitizerSet SkippedChecks = SanitizerSet(),
3372 llvm::Value *ArraySize = nullptr) {
3373 if (!sanitizePerformTypeCheck())
3374 return;
3375 EmitTypeCheck(TCK, Loc, V: Addr.emitRawPointer(CGF&: *this), Type, Alignment,
3376 SkippedChecks, ArraySize);
3377 }
3378
3379 /// Emit a check that \p V is the address of storage of the
3380 /// appropriate size and alignment for an object of type \p Type
3381 /// (or if ArraySize is provided, for an array of that bound).
3382 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Value *V,
3383 QualType Type, CharUnits Alignment = CharUnits::Zero(),
3384 SanitizerSet SkippedChecks = SanitizerSet(),
3385 llvm::Value *ArraySize = nullptr);
3386
3387 /// Emit a check that \p Base points into an array object, which
3388 /// we can access at index \p Index. \p Accessed should be \c false if we
3389 /// this expression is used as an lvalue, for instance in "&Arr[Idx]".
3390 void EmitBoundsCheck(const Expr *ArrayExpr, const Expr *ArrayExprBase,
3391 llvm::Value *Index, QualType IndexType, bool Accessed);
3392 void EmitBoundsCheckImpl(const Expr *ArrayExpr, QualType ArrayBaseType,
3393 llvm::Value *IndexVal, QualType IndexType,
3394 llvm::Value *BoundsVal, QualType BoundsType,
3395 bool Accessed);
3396
3397 /// Returns debug info, with additional annotation if
3398 /// CGM.getCodeGenOpts().SanitizeAnnotateDebugInfo[Ordinal] is enabled for
3399 /// any of the ordinals.
3400 llvm::DILocation *
3401 SanitizerAnnotateDebugInfo(ArrayRef<SanitizerKind::SanitizerOrdinal> Ordinals,
3402 SanitizerHandler Handler);
3403
3404 /// Build metadata used by the AllocToken instrumentation.
3405 llvm::MDNode *buildAllocToken(QualType AllocType);
3406 /// Emit and set additional metadata used by the AllocToken instrumentation.
3407 void EmitAllocToken(llvm::CallBase *CB, QualType AllocType);
3408 /// Build additional metadata used by the AllocToken instrumentation,
3409 /// inferring the type from an allocation call expression.
3410 llvm::MDNode *buildAllocToken(const CallExpr *E);
3411 /// Emit and set additional metadata used by the AllocToken instrumentation,
3412 /// inferring the type from an allocation call expression.
3413 void EmitAllocToken(llvm::CallBase *CB, const CallExpr *E);
3414
3415 llvm::Value *GetCountedByFieldExprGEP(const Expr *Base, const FieldDecl *FD,
3416 const FieldDecl *CountDecl);
3417
3418 /// Build an expression accessing the "counted_by" field.
3419 llvm::Value *EmitLoadOfCountedByField(const Expr *Base, const FieldDecl *FD,
3420 const FieldDecl *CountDecl);
3421
3422 // Emit bounds checking for flexible array and pointer members with the
3423 // counted_by attribute.
3424 void EmitCountedByBoundsChecking(const Expr *ArrayExpr, QualType ArrayType,
3425 Address ArrayInst, QualType IndexType,
3426 llvm::Value *IndexVal, bool Accessed,
3427 bool FlexibleArray);
3428
3429 llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
3430 bool isInc, bool isPre);
3431 ComplexPairTy EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
3432 bool isInc, bool isPre);
3433
3434 /// Converts Location to a DebugLoc, if debug information is enabled.
3435 llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location);
3436
3437 /// Get the record field index as represented in debug info.
3438 unsigned getDebugInfoFIndex(const RecordDecl *Rec, unsigned FieldIndex);
3439
3440 //===--------------------------------------------------------------------===//
3441 // Declaration Emission
3442 //===--------------------------------------------------------------------===//
3443
3444 /// EmitDecl - Emit a declaration.
3445 ///
3446 /// This function can be called with a null (unreachable) insert point.
3447 void EmitDecl(const Decl &D, bool EvaluateConditionDecl = false);
3448
3449 /// EmitVarDecl - Emit a local variable declaration.
3450 ///
3451 /// This function can be called with a null (unreachable) insert point.
3452 void EmitVarDecl(const VarDecl &D);
3453
3454 void EmitScalarInit(const Expr *init, const ValueDecl *D, LValue lvalue,
3455 bool capturedByInit);
3456
3457 typedef void SpecialInitFn(CodeGenFunction &Init, const VarDecl &D,
3458 llvm::Value *Address);
3459
3460 /// Determine whether the given initializer is trivial in the sense
3461 /// that it requires no code to be generated.
3462 bool isTrivialInitializer(const Expr *Init);
3463
3464 /// EmitAutoVarDecl - Emit an auto variable declaration.
3465 ///
3466 /// This function can be called with a null (unreachable) insert point.
3467 void EmitAutoVarDecl(const VarDecl &D);
3468
3469 class AutoVarEmission {
3470 friend class CodeGenFunction;
3471
3472 const VarDecl *Variable;
3473
3474 /// The address of the alloca for languages with explicit address space
3475 /// (e.g. OpenCL) or alloca casted to generic pointer for address space
3476 /// agnostic languages (e.g. C++). Invalid if the variable was emitted
3477 /// as a global constant.
3478 Address Addr;
3479
3480 llvm::Value *NRVOFlag;
3481
3482 /// True if the variable is a __block variable that is captured by an
3483 /// escaping block.
3484 bool IsEscapingByRef;
3485
3486 /// True if the variable is of aggregate type and has a constant
3487 /// initializer.
3488 bool IsConstantAggregate;
3489
3490 /// True if lifetime markers should be used.
3491 bool UseLifetimeMarkers;
3492
3493 /// Address with original alloca instruction. Invalid if the variable was
3494 /// emitted as a global constant.
3495 RawAddress AllocaAddr;
3496
3497 struct Invalid {};
3498 AutoVarEmission(Invalid)
3499 : Variable(nullptr), Addr(Address::invalid()),
3500 AllocaAddr(RawAddress::invalid()) {}
3501
3502 AutoVarEmission(const VarDecl &variable)
3503 : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr),
3504 IsEscapingByRef(false), IsConstantAggregate(false),
3505 UseLifetimeMarkers(false), AllocaAddr(RawAddress::invalid()) {}
3506
3507 bool wasEmittedAsGlobal() const { return !Addr.isValid(); }
3508
3509 public:
3510 static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }
3511
3512 bool useLifetimeMarkers() const { return UseLifetimeMarkers; }
3513
3514 /// Returns the raw, allocated address, which is not necessarily
3515 /// the address of the object itself. It is casted to default
3516 /// address space for address space agnostic languages.
3517 Address getAllocatedAddress() const { return Addr; }
3518
3519 /// Returns the address for the original alloca instruction.
3520 RawAddress getOriginalAllocatedAddress() const { return AllocaAddr; }
3521
3522 /// Returns the address of the object within this declaration.
3523 /// Note that this does not chase the forwarding pointer for
3524 /// __block decls.
3525 Address getObjectAddress(CodeGenFunction &CGF) const {
3526 if (!IsEscapingByRef)
3527 return Addr;
3528
3529 return CGF.emitBlockByrefAddress(baseAddr: Addr, V: Variable, /*forward*/ followForward: false);
3530 }
3531 };
3532 AutoVarEmission EmitAutoVarAlloca(const VarDecl &var);
3533 void EmitAutoVarInit(const AutoVarEmission &emission);
3534 void EmitAutoVarCleanups(const AutoVarEmission &emission);
3535 void emitAutoVarTypeCleanup(const AutoVarEmission &emission,
3536 QualType::DestructionKind dtorKind);
3537
3538 void MaybeEmitDeferredVarDeclInit(const VarDecl *var);
3539
3540 /// Emits the alloca and debug information for the size expressions for each
3541 /// dimension of an array. It registers the association of its (1-dimensional)
3542 /// QualTypes and size expression's debug node, so that CGDebugInfo can
3543 /// reference this node when creating the DISubrange object to describe the
3544 /// array types.
3545 void EmitAndRegisterVariableArrayDimensions(CGDebugInfo *DI, const VarDecl &D,
3546 bool EmitDebugInfo);
3547
3548 void EmitStaticVarDecl(const VarDecl &D,
3549 llvm::GlobalValue::LinkageTypes Linkage);
3550
3551 class ParamValue {
3552 union {
3553 Address Addr;
3554 llvm::Value *Value;
3555 };
3556
3557 bool IsIndirect;
3558
3559 ParamValue(llvm::Value *V) : Value(V), IsIndirect(false) {}
3560 ParamValue(Address A) : Addr(A), IsIndirect(true) {}
3561
3562 public:
3563 static ParamValue forDirect(llvm::Value *value) {
3564 return ParamValue(value);
3565 }
3566 static ParamValue forIndirect(Address addr) {
3567 assert(!addr.getAlignment().isZero());
3568 return ParamValue(addr);
3569 }
3570
3571 bool isIndirect() const { return IsIndirect; }
3572 llvm::Value *getAnyValue() const {
3573 if (!isIndirect())
3574 return Value;
3575 assert(!Addr.hasOffset() && "unexpected offset");
3576 return Addr.getBasePointer();
3577 }
3578
3579 llvm::Value *getDirectValue() const {
3580 assert(!isIndirect());
3581 return Value;
3582 }
3583
3584 Address getIndirectAddress() const {
3585 assert(isIndirect());
3586 return Addr;
3587 }
3588 };
3589
3590 /// EmitParmDecl - Emit a ParmVarDecl or an ImplicitParamDecl.
3591 void EmitParmDecl(const VarDecl &D, ParamValue Arg, unsigned ArgNo);
3592
3593 /// protectFromPeepholes - Protect a value that we're intending to
3594 /// store to the side, but which will probably be used later, from
3595 /// aggressive peepholing optimizations that might delete it.
3596 ///
3597 /// Pass the result to unprotectFromPeepholes to declare that
3598 /// protection is no longer required.
3599 ///
3600 /// There's no particular reason why this shouldn't apply to
3601 /// l-values, it's just that no existing peepholes work on pointers.
3602 PeepholeProtection protectFromPeepholes(RValue rvalue);
3603 void unprotectFromPeepholes(PeepholeProtection protection);
3604
3605 void emitAlignmentAssumptionCheck(llvm::Value *Ptr, QualType Ty,
3606 SourceLocation Loc,
3607 SourceLocation AssumptionLoc,
3608 llvm::Value *Alignment,
3609 llvm::Value *OffsetValue,
3610 llvm::Value *TheCheck,
3611 llvm::Instruction *Assumption);
3612
3613 void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty,
3614 SourceLocation Loc, SourceLocation AssumptionLoc,
3615 llvm::Value *Alignment,
3616 llvm::Value *OffsetValue = nullptr);
3617
3618 void emitAlignmentAssumption(llvm::Value *PtrValue, const Expr *E,
3619 SourceLocation AssumptionLoc,
3620 llvm::Value *Alignment,
3621 llvm::Value *OffsetValue = nullptr);
3622
3623 //===--------------------------------------------------------------------===//
3624 // Statement Emission
3625 //===--------------------------------------------------------------------===//
3626
3627 /// EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
3628 void EmitStopPoint(const Stmt *S);
3629
3630 /// EmitStmt - Emit the code for the statement \arg S. It is legal to call
3631 /// this function even if there is no current insertion point.
3632 ///
3633 /// This function may clear the current insertion point; callers should use
3634 /// EnsureInsertPoint if they wish to subsequently generate code without first
3635 /// calling EmitBlock, EmitBranch, or EmitStmt.
3636 void EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs = {});
3637
3638 /// EmitSimpleStmt - Try to emit a "simple" statement which does not
3639 /// necessarily require an insertion point or debug information; typically
3640 /// because the statement amounts to a jump or a container of other
3641 /// statements.
3642 ///
3643 /// \return True if the statement was handled.
3644 bool EmitSimpleStmt(const Stmt *S, ArrayRef<const Attr *> Attrs);
3645
3646 Address EmitCompoundStmt(const CompoundStmt &S, bool GetLast = false,
3647 AggValueSlot AVS = AggValueSlot::ignored());
3648 Address
3649 EmitCompoundStmtWithoutScope(const CompoundStmt &S, bool GetLast = false,
3650 AggValueSlot AVS = AggValueSlot::ignored());
3651
3652 /// EmitLabel - Emit the block for the given label. It is legal to call this
3653 /// function even if there is no current insertion point.
3654 void EmitLabel(const LabelDecl *D); // helper for EmitLabelStmt.
3655
3656 void EmitLabelStmt(const LabelStmt &S);
3657 void EmitAttributedStmt(const AttributedStmt &S);
3658 void EmitGotoStmt(const GotoStmt &S);
3659 void EmitIndirectGotoStmt(const IndirectGotoStmt &S);
3660 void EmitIfStmt(const IfStmt &S);
3661
3662 void EmitWhileStmt(const WhileStmt &S, ArrayRef<const Attr *> Attrs = {});
3663 void EmitDoStmt(const DoStmt &S, ArrayRef<const Attr *> Attrs = {});
3664 void EmitForStmt(const ForStmt &S, ArrayRef<const Attr *> Attrs = {});
3665 void EmitReturnStmt(const ReturnStmt &S);
3666 void EmitDeclStmt(const DeclStmt &S);
3667 void EmitBreakStmt(const BreakStmt &S);
3668 void EmitContinueStmt(const ContinueStmt &S);
3669 void EmitSwitchStmt(const SwitchStmt &S);
3670 void EmitDefaultStmt(const DefaultStmt &S, ArrayRef<const Attr *> Attrs);
3671 void EmitCaseStmt(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3672 void EmitCaseStmtRange(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3673 void EmitDeferStmt(const DeferStmt &S);
3674 void EmitAsmStmt(const AsmStmt &S);
3675
3676 const BreakContinue *GetDestForLoopControlStmt(const LoopControlStmt &S);
3677
3678 void EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S);
3679 void EmitObjCAtTryStmt(const ObjCAtTryStmt &S);
3680 void EmitObjCAtThrowStmt(const ObjCAtThrowStmt &S);
3681 void EmitObjCAtSynchronizedStmt(const ObjCAtSynchronizedStmt &S);
3682 void EmitObjCAutoreleasePoolStmt(const ObjCAutoreleasePoolStmt &S);
3683
3684 void EmitCoroutineBody(const CoroutineBodyStmt &S);
3685 void EmitCoreturnStmt(const CoreturnStmt &S);
3686 RValue EmitCoawaitExpr(const CoawaitExpr &E,
3687 AggValueSlot aggSlot = AggValueSlot::ignored(),
3688 bool ignoreResult = false);
3689 LValue EmitCoawaitLValue(const CoawaitExpr *E);
3690 RValue EmitCoyieldExpr(const CoyieldExpr &E,
3691 AggValueSlot aggSlot = AggValueSlot::ignored(),
3692 bool ignoreResult = false);
3693 LValue EmitCoyieldLValue(const CoyieldExpr *E);
3694 RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID);
3695
3696 void EmitSYCLKernelCallStmt(const SYCLKernelCallStmt &S);
3697
3698 void EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3699 void ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3700
3701 void EmitCXXTryStmt(const CXXTryStmt &S);
3702 void EmitSEHTryStmt(const SEHTryStmt &S);
3703 void EmitSEHLeaveStmt(const SEHLeaveStmt &S);
3704 void EnterSEHTryStmt(const SEHTryStmt &S);
3705 void ExitSEHTryStmt(const SEHTryStmt &S);
3706 void VolatilizeTryBlocks(llvm::BasicBlock *BB,
3707 llvm::SmallPtrSet<llvm::BasicBlock *, 10> &V);
3708
3709 void pushSEHCleanup(CleanupKind kind, llvm::Function *FinallyFunc);
3710 void startOutlinedSEHHelper(CodeGenFunction &ParentCGF, bool IsFilter,
3711 const Stmt *OutlinedStmt);
3712
3713 llvm::Function *GenerateSEHFilterFunction(CodeGenFunction &ParentCGF,
3714 const SEHExceptStmt &Except);
3715
3716 llvm::Function *GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF,
3717 const SEHFinallyStmt &Finally);
3718
3719 void EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF,
3720 llvm::Value *ParentFP, llvm::Value *EntryEBP);
3721 llvm::Value *EmitSEHExceptionCode();
3722 llvm::Value *EmitSEHExceptionInfo();
3723 llvm::Value *EmitSEHAbnormalTermination();
3724
3725 /// Emit simple code for OpenMP directives in Simd-only mode.
3726 void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D);
3727
3728 /// Scan the outlined statement for captures from the parent function. For
3729 /// each capture, mark the capture as escaped and emit a call to
3730 /// llvm.localrecover. Insert the localrecover result into the LocalDeclMap.
3731 void EmitCapturedLocals(CodeGenFunction &ParentCGF, const Stmt *OutlinedStmt,
3732 bool IsFilter);
3733
3734 /// Recovers the address of a local in a parent function. ParentVar is the
3735 /// address of the variable used in the immediate parent function. It can
3736 /// either be an alloca or a call to llvm.localrecover if there are nested
3737 /// outlined functions. ParentFP is the frame pointer of the outermost parent
3738 /// frame.
3739 Address recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
3740 Address ParentVar, llvm::Value *ParentFP);
3741
3742 void EmitCXXForRangeStmt(const CXXForRangeStmt &S,
3743 ArrayRef<const Attr *> Attrs = {});
3744
3745 /// Controls insertion of cancellation exit blocks in worksharing constructs.
3746 class OMPCancelStackRAII {
3747 CodeGenFunction &CGF;
3748
3749 public:
3750 OMPCancelStackRAII(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
3751 bool HasCancel)
3752 : CGF(CGF) {
3753 CGF.OMPCancelStack.enter(CGF, Kind, HasCancel);
3754 }
3755 ~OMPCancelStackRAII() { CGF.OMPCancelStack.exit(CGF); }
3756 };
3757
3758 /// Returns calculated size of the specified type.
3759 llvm::Value *getTypeSize(QualType Ty);
3760 LValue InitCapturedStruct(const CapturedStmt &S);
3761 llvm::Function *EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K);
3762 llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S);
3763 Address GenerateCapturedStmtArgument(const CapturedStmt &S);
3764 llvm::Function *
3765 GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S,
3766 const OMPExecutableDirective &D);
3767 llvm::Function *
3768 GenerateOpenMPCapturedStmtFunctionAggregate(const CapturedStmt &S,
3769 const OMPExecutableDirective &D);
3770 void GenerateOpenMPCapturedVars(const CapturedStmt &S,
3771 SmallVectorImpl<llvm::Value *> &CapturedVars);
3772 void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy,
3773 SourceLocation Loc);
3774 /// Perform element by element copying of arrays with type \a
3775 /// OriginalType from \a SrcAddr to \a DestAddr using copying procedure
3776 /// generated by \a CopyGen.
3777 ///
3778 /// \param DestAddr Address of the destination array.
3779 /// \param SrcAddr Address of the source array.
3780 /// \param OriginalType Type of destination and source arrays.
3781 /// \param CopyGen Copying procedure that copies value of single array element
3782 /// to another single array element.
3783 void EmitOMPAggregateAssign(
3784 Address DestAddr, Address SrcAddr, QualType OriginalType,
3785 const llvm::function_ref<void(Address, Address)> CopyGen);
3786 /// Emit proper copying of data from one variable to another.
3787 ///
3788 /// \param OriginalType Original type of the copied variables.
3789 /// \param DestAddr Destination address.
3790 /// \param SrcAddr Source address.
3791 /// \param DestVD Destination variable used in \a CopyExpr (for arrays, has
3792 /// type of the base array element).
3793 /// \param SrcVD Source variable used in \a CopyExpr (for arrays, has type of
3794 /// the base array element).
3795 /// \param Copy Actual copygin expression for copying data from \a SrcVD to \a
3796 /// DestVD.
3797 void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr,
3798 const VarDecl *DestVD, const VarDecl *SrcVD,
3799 const Expr *Copy);
3800 /// Emit atomic update code for constructs: \a X = \a X \a BO \a E or
3801 /// \a X = \a E \a BO \a E.
3802 ///
3803 /// \param X Value to be updated.
3804 /// \param E Update value.
3805 /// \param BO Binary operation for update operation.
3806 /// \param IsXLHSInRHSPart true if \a X is LHS in RHS part of the update
3807 /// expression, false otherwise.
3808 /// \param AO Atomic ordering of the generated atomic instructions.
3809 /// \param CommonGen Code generator for complex expressions that cannot be
3810 /// expressed through atomicrmw instruction.
3811 /// \returns <true, OldAtomicValue> if simple 'atomicrmw' instruction was
3812 /// generated, <false, RValue::get(nullptr)> otherwise.
3813 std::pair<bool, RValue> EmitOMPAtomicSimpleUpdateExpr(
3814 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3815 llvm::AtomicOrdering AO, SourceLocation Loc,
3816 const llvm::function_ref<RValue(RValue)> CommonGen);
3817 bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
3818 OMPPrivateScope &PrivateScope);
3819 void EmitOMPPrivateClause(const OMPExecutableDirective &D,
3820 OMPPrivateScope &PrivateScope);
3821 void EmitOMPUseDevicePtrClause(
3822 const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
3823 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3824 CaptureDeviceAddrMap);
3825 void EmitOMPUseDeviceAddrClause(
3826 const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
3827 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3828 CaptureDeviceAddrMap);
3829 /// Emit code for copyin clause in \a D directive. The next code is
3830 /// generated at the start of outlined functions for directives:
3831 /// \code
3832 /// threadprivate_var1 = master_threadprivate_var1;
3833 /// operator=(threadprivate_var2, master_threadprivate_var2);
3834 /// ...
3835 /// __kmpc_barrier(&loc, global_tid);
3836 /// \endcode
3837 ///
3838 /// \param D OpenMP directive possibly with 'copyin' clause(s).
3839 /// \returns true if at least one copyin variable is found, false otherwise.
3840 bool EmitOMPCopyinClause(const OMPExecutableDirective &D);
3841 /// Emit initial code for lastprivate variables. If some variable is
3842 /// not also firstprivate, then the default initialization is used. Otherwise
3843 /// initialization of this variable is performed by EmitOMPFirstprivateClause
3844 /// method.
3845 ///
3846 /// \param D Directive that may have 'lastprivate' directives.
3847 /// \param PrivateScope Private scope for capturing lastprivate variables for
3848 /// proper codegen in internal captured statement.
3849 ///
3850 /// \returns true if there is at least one lastprivate variable, false
3851 /// otherwise.
3852 bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D,
3853 OMPPrivateScope &PrivateScope);
3854 /// Emit final copying of lastprivate values to original variables at
3855 /// the end of the worksharing or simd directive.
3856 ///
3857 /// \param D Directive that has at least one 'lastprivate' directives.
3858 /// \param IsLastIterCond Boolean condition that must be set to 'i1 true' if
3859 /// it is the last iteration of the loop code in associated directive, or to
3860 /// 'i1 false' otherwise. If this item is nullptr, no final check is required.
3861 void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D,
3862 bool NoFinals,
3863 llvm::Value *IsLastIterCond = nullptr);
3864 /// Emit initial code for linear clauses.
3865 void EmitOMPLinearClause(const OMPLoopDirective &D,
3866 CodeGenFunction::OMPPrivateScope &PrivateScope);
3867 /// Emit final code for linear clauses.
3868 /// \param CondGen Optional conditional code for final part of codegen for
3869 /// linear clause.
3870 void EmitOMPLinearClauseFinal(
3871 const OMPLoopDirective &D,
3872 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
3873 /// Emit initial code for reduction variables. Creates reduction copies
3874 /// and initializes them with the values according to OpenMP standard.
3875 ///
3876 /// \param D Directive (possibly) with the 'reduction' clause.
3877 /// \param PrivateScope Private scope for capturing reduction variables for
3878 /// proper codegen in internal captured statement.
3879 ///
3880 void EmitOMPReductionClauseInit(const OMPExecutableDirective &D,
3881 OMPPrivateScope &PrivateScope,
3882 bool ForInscan = false);
3883 /// Emit final update of reduction values to original variables at
3884 /// the end of the directive.
3885 ///
3886 /// \param D Directive that has at least one 'reduction' directives.
3887 /// \param ReductionKind The kind of reduction to perform.
3888 void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D,
3889 const OpenMPDirectiveKind ReductionKind);
3890 /// Emit initial code for linear variables. Creates private copies
3891 /// and initializes them with the values according to OpenMP standard.
3892 ///
3893 /// \param D Directive (possibly) with the 'linear' clause.
3894 /// \return true if at least one linear variable is found that should be
3895 /// initialized with the value of the original variable, false otherwise.
3896 bool EmitOMPLinearClauseInit(const OMPLoopDirective &D);
3897
3898 typedef const llvm::function_ref<void(CodeGenFunction & /*CGF*/,
3899 llvm::Function * /*OutlinedFn*/,
3900 const OMPTaskDataTy & /*Data*/)>
3901 TaskGenTy;
3902 void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S,
3903 const OpenMPDirectiveKind CapturedRegion,
3904 const RegionCodeGenTy &BodyGen,
3905 const TaskGenTy &TaskGen, OMPTaskDataTy &Data);
3906 struct OMPTargetDataInfo {
3907 Address BasePointersArray = Address::invalid();
3908 Address PointersArray = Address::invalid();
3909 Address SizesArray = Address::invalid();
3910 Address MappersArray = Address::invalid();
3911 unsigned NumberOfTargetItems = 0;
3912 explicit OMPTargetDataInfo() = default;
3913 OMPTargetDataInfo(Address BasePointersArray, Address PointersArray,
3914 Address SizesArray, Address MappersArray,
3915 unsigned NumberOfTargetItems)
3916 : BasePointersArray(BasePointersArray), PointersArray(PointersArray),
3917 SizesArray(SizesArray), MappersArray(MappersArray),
3918 NumberOfTargetItems(NumberOfTargetItems) {}
3919 };
3920 void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S,
3921 const RegionCodeGenTy &BodyGen,
3922 OMPTargetDataInfo &InputInfo);
3923 void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data,
3924 CodeGenFunction &CGF, const CapturedStmt *CS,
3925 OMPPrivateScope &Scope);
3926 void EmitOMPMetaDirective(const OMPMetaDirective &S);
3927 void EmitOMPParallelDirective(const OMPParallelDirective &S);
3928 void EmitOMPSimdDirective(const OMPSimdDirective &S);
3929 void EmitOMPTileDirective(const OMPTileDirective &S);
3930 void EmitOMPStripeDirective(const OMPStripeDirective &S);
3931 void EmitOMPUnrollDirective(const OMPUnrollDirective &S);
3932 void EmitOMPReverseDirective(const OMPReverseDirective &S);
3933 void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
3934 void EmitOMPFuseDirective(const OMPFuseDirective &S);
3935 void EmitOMPForDirective(const OMPForDirective &S);
3936 void EmitOMPForSimdDirective(const OMPForSimdDirective &S);
3937 void EmitOMPScopeDirective(const OMPScopeDirective &S);
3938 void EmitOMPSectionsDirective(const OMPSectionsDirective &S);
3939 void EmitOMPSectionDirective(const OMPSectionDirective &S);
3940 void EmitOMPSingleDirective(const OMPSingleDirective &S);
3941 void EmitOMPMasterDirective(const OMPMasterDirective &S);
3942 void EmitOMPMaskedDirective(const OMPMaskedDirective &S);
3943 void EmitOMPCriticalDirective(const OMPCriticalDirective &S);
3944 void EmitOMPParallelForDirective(const OMPParallelForDirective &S);
3945 void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S);
3946 void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S);
3947 void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S);
3948 void EmitOMPTaskDirective(const OMPTaskDirective &S);
3949 void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S);
3950 void EmitOMPErrorDirective(const OMPErrorDirective &S);
3951 void EmitOMPBarrierDirective(const OMPBarrierDirective &S);
3952 void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S);
3953 void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S);
3954 void EmitOMPFlushDirective(const OMPFlushDirective &S);
3955 void EmitOMPDepobjDirective(const OMPDepobjDirective &S);
3956 void EmitOMPScanDirective(const OMPScanDirective &S);
3957 void EmitOMPOrderedDirective(const OMPOrderedDirective &S);
3958 void EmitOMPAtomicDirective(const OMPAtomicDirective &S);
3959 void EmitOMPTargetDirective(const OMPTargetDirective &S);
3960 void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S);
3961 void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S);
3962 void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S);
3963 void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S);
3964 void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S);
3965 void
3966 EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S);
3967 void EmitOMPTeamsDirective(const OMPTeamsDirective &S);
3968 void
3969 EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S);
3970 void EmitOMPCancelDirective(const OMPCancelDirective &S);
3971 void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S);
3972 void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S);
3973 void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S);
3974 void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S);
3975 void EmitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &S);
3976 void
3977 EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S);
3978 void
3979 EmitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &S);
3980 void EmitOMPParallelMasterTaskLoopDirective(
3981 const OMPParallelMasterTaskLoopDirective &S);
3982 void EmitOMPParallelMaskedTaskLoopDirective(
3983 const OMPParallelMaskedTaskLoopDirective &S);
3984 void EmitOMPParallelMasterTaskLoopSimdDirective(
3985 const OMPParallelMasterTaskLoopSimdDirective &S);
3986 void EmitOMPParallelMaskedTaskLoopSimdDirective(
3987 const OMPParallelMaskedTaskLoopSimdDirective &S);
3988 void EmitOMPDistributeDirective(const OMPDistributeDirective &S);
3989 void EmitOMPDistributeParallelForDirective(
3990 const OMPDistributeParallelForDirective &S);
3991 void EmitOMPDistributeParallelForSimdDirective(
3992 const OMPDistributeParallelForSimdDirective &S);
3993 void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S);
3994 void EmitOMPTargetParallelForSimdDirective(
3995 const OMPTargetParallelForSimdDirective &S);
3996 void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S);
3997 void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S);
3998 void
3999 EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S);
4000 void EmitOMPTeamsDistributeParallelForSimdDirective(
4001 const OMPTeamsDistributeParallelForSimdDirective &S);
4002 void EmitOMPTeamsDistributeParallelForDirective(
4003 const OMPTeamsDistributeParallelForDirective &S);
4004 void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S);
4005 void EmitOMPTargetTeamsDistributeDirective(
4006 const OMPTargetTeamsDistributeDirective &S);
4007 void EmitOMPTargetTeamsDistributeParallelForDirective(
4008 const OMPTargetTeamsDistributeParallelForDirective &S);
4009 void EmitOMPTargetTeamsDistributeParallelForSimdDirective(
4010 const OMPTargetTeamsDistributeParallelForSimdDirective &S);
4011 void EmitOMPTargetTeamsDistributeSimdDirective(
4012 const OMPTargetTeamsDistributeSimdDirective &S);
4013 void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S);
4014 void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S);
4015 void EmitOMPTargetParallelGenericLoopDirective(
4016 const OMPTargetParallelGenericLoopDirective &S);
4017 void EmitOMPTargetTeamsGenericLoopDirective(
4018 const OMPTargetTeamsGenericLoopDirective &S);
4019 void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S);
4020 void EmitOMPInteropDirective(const OMPInteropDirective &S);
4021 void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S);
4022 void EmitOMPAssumeDirective(const OMPAssumeDirective &S);
4023
4024 /// Emit device code for the target directive.
4025 static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
4026 StringRef ParentName,
4027 const OMPTargetDirective &S);
4028 static void
4029 EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
4030 const OMPTargetParallelDirective &S);
4031 /// Emit device code for the target parallel for directive.
4032 static void EmitOMPTargetParallelForDeviceFunction(
4033 CodeGenModule &CGM, StringRef ParentName,
4034 const OMPTargetParallelForDirective &S);
4035 /// Emit device code for the target parallel for simd directive.
4036 static void EmitOMPTargetParallelForSimdDeviceFunction(
4037 CodeGenModule &CGM, StringRef ParentName,
4038 const OMPTargetParallelForSimdDirective &S);
4039 /// Emit device code for the target teams directive.
4040 static void
4041 EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
4042 const OMPTargetTeamsDirective &S);
4043 /// Emit device code for the target teams distribute directive.
4044 static void EmitOMPTargetTeamsDistributeDeviceFunction(
4045 CodeGenModule &CGM, StringRef ParentName,
4046 const OMPTargetTeamsDistributeDirective &S);
4047 /// Emit device code for the target teams distribute simd directive.
4048 static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(
4049 CodeGenModule &CGM, StringRef ParentName,
4050 const OMPTargetTeamsDistributeSimdDirective &S);
4051 /// Emit device code for the target simd directive.
4052 static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM,
4053 StringRef ParentName,
4054 const OMPTargetSimdDirective &S);
4055 /// Emit device code for the target teams distribute parallel for simd
4056 /// directive.
4057 static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
4058 CodeGenModule &CGM, StringRef ParentName,
4059 const OMPTargetTeamsDistributeParallelForSimdDirective &S);
4060
4061 /// Emit device code for the target teams loop directive.
4062 static void EmitOMPTargetTeamsGenericLoopDeviceFunction(
4063 CodeGenModule &CGM, StringRef ParentName,
4064 const OMPTargetTeamsGenericLoopDirective &S);
4065
4066 /// Emit device code for the target parallel loop directive.
4067 static void EmitOMPTargetParallelGenericLoopDeviceFunction(
4068 CodeGenModule &CGM, StringRef ParentName,
4069 const OMPTargetParallelGenericLoopDirective &S);
4070
4071 static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
4072 CodeGenModule &CGM, StringRef ParentName,
4073 const OMPTargetTeamsDistributeParallelForDirective &S);
4074
4075 /// Emit the Stmt \p S and return its topmost canonical loop, if any.
4076 /// TODO: The \p Depth paramter is not yet implemented and must be 1. In the
4077 /// future it is meant to be the number of loops expected in the loop nests
4078 /// (usually specified by the "collapse" clause) that are collapsed to a
4079 /// single loop by this function.
4080 llvm::CanonicalLoopInfo *EmitOMPCollapsedCanonicalLoopNest(const Stmt *S,
4081 int Depth);
4082
4083 /// Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
4084 void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S);
4085
4086 /// Emit inner loop of the worksharing/simd construct.
4087 ///
4088 /// \param S Directive, for which the inner loop must be emitted.
4089 /// \param RequiresCleanup true, if directive has some associated private
4090 /// variables.
4091 /// \param LoopCond Bollean condition for loop continuation.
4092 /// \param IncExpr Increment expression for loop control variable.
4093 /// \param BodyGen Generator for the inner body of the inner loop.
4094 /// \param PostIncGen Genrator for post-increment code (required for ordered
4095 /// loop directvies).
4096 void EmitOMPInnerLoop(
4097 const OMPExecutableDirective &S, bool RequiresCleanup,
4098 const Expr *LoopCond, const Expr *IncExpr,
4099 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
4100 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen);
4101
4102 JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind);
4103 /// Emit initial code for loop counters of loop-based directives.
4104 void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S,
4105 OMPPrivateScope &LoopScope);
4106
4107 /// Helper for the OpenMP loop directives.
4108 void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit);
4109
4110 /// Emit code for the worksharing loop-based directive.
4111 /// \return true, if this construct has any lastprivate clause, false -
4112 /// otherwise.
4113 bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB,
4114 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
4115 const CodeGenDispatchBoundsTy &CGDispatchBounds);
4116
4117 /// Emit code for the distribute loop-based directive.
4118 void EmitOMPDistributeLoop(const OMPLoopDirective &S,
4119 const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr);
4120
4121 /// Helpers for the OpenMP loop directives.
4122 void EmitOMPSimdInit(const OMPLoopDirective &D);
4123 void EmitOMPSimdFinal(
4124 const OMPLoopDirective &D,
4125 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
4126
4127 /// Emits the lvalue for the expression with possibly captured variable.
4128 LValue EmitOMPSharedLValue(const Expr *E);
4129
4130private:
4131 /// Helpers for blocks.
4132 llvm::Value *EmitBlockLiteral(const CGBlockInfo &Info);
4133
4134 /// struct with the values to be passed to the OpenMP loop-related functions
4135 struct OMPLoopArguments {
4136 /// loop lower bound
4137 Address LB = Address::invalid();
4138 /// loop upper bound
4139 Address UB = Address::invalid();
4140 /// loop stride
4141 Address ST = Address::invalid();
4142 /// isLastIteration argument for runtime functions
4143 Address IL = Address::invalid();
4144 /// Chunk value generated by sema
4145 llvm::Value *Chunk = nullptr;
4146 /// EnsureUpperBound
4147 Expr *EUB = nullptr;
4148 /// IncrementExpression
4149 Expr *IncExpr = nullptr;
4150 /// Loop initialization
4151 Expr *Init = nullptr;
4152 /// Loop exit condition
4153 Expr *Cond = nullptr;
4154 /// Update of LB after a whole chunk has been executed
4155 Expr *NextLB = nullptr;
4156 /// Update of UB after a whole chunk has been executed
4157 Expr *NextUB = nullptr;
4158 /// Distinguish between the for distribute and sections
4159 OpenMPDirectiveKind DKind = llvm::omp::OMPD_unknown;
4160 OMPLoopArguments() = default;
4161 OMPLoopArguments(Address LB, Address UB, Address ST, Address IL,
4162 llvm::Value *Chunk = nullptr, Expr *EUB = nullptr,
4163 Expr *IncExpr = nullptr, Expr *Init = nullptr,
4164 Expr *Cond = nullptr, Expr *NextLB = nullptr,
4165 Expr *NextUB = nullptr)
4166 : LB(LB), UB(UB), ST(ST), IL(IL), Chunk(Chunk), EUB(EUB),
4167 IncExpr(IncExpr), Init(Init), Cond(Cond), NextLB(NextLB),
4168 NextUB(NextUB) {}
4169 };
4170 void EmitOMPOuterLoop(bool DynamicOrOrdered, bool IsMonotonic,
4171 const OMPLoopDirective &S, OMPPrivateScope &LoopScope,
4172 const OMPLoopArguments &LoopArgs,
4173 const CodeGenLoopTy &CodeGenLoop,
4174 const CodeGenOrderedTy &CodeGenOrdered);
4175 void EmitOMPForOuterLoop(const OpenMPScheduleTy &ScheduleKind,
4176 bool IsMonotonic, const OMPLoopDirective &S,
4177 OMPPrivateScope &LoopScope, bool Ordered,
4178 const OMPLoopArguments &LoopArgs,
4179 const CodeGenDispatchBoundsTy &CGDispatchBounds);
4180 void EmitOMPDistributeOuterLoop(OpenMPDistScheduleClauseKind ScheduleKind,
4181 const OMPLoopDirective &S,
4182 OMPPrivateScope &LoopScope,
4183 const OMPLoopArguments &LoopArgs,
4184 const CodeGenLoopTy &CodeGenLoopContent);
4185 /// Emit code for sections directive.
4186 void EmitSections(const OMPExecutableDirective &S);
4187
4188public:
4189 //===--------------------------------------------------------------------===//
4190 // OpenACC Emission
4191 //===--------------------------------------------------------------------===//
4192 void EmitOpenACCComputeConstruct(const OpenACCComputeConstruct &S) {
4193 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4194 // simply emitting its structured block, but in the future we will implement
4195 // some sort of IR.
4196 EmitStmt(S: S.getStructuredBlock());
4197 }
4198
4199 void EmitOpenACCLoopConstruct(const OpenACCLoopConstruct &S) {
4200 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4201 // simply emitting its loop, but in the future we will implement
4202 // some sort of IR.
4203 EmitStmt(S: S.getLoop());
4204 }
4205
4206 void EmitOpenACCCombinedConstruct(const OpenACCCombinedConstruct &S) {
4207 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4208 // simply emitting its loop, but in the future we will implement
4209 // some sort of IR.
4210 EmitStmt(S: S.getLoop());
4211 }
4212
4213 void EmitOpenACCDataConstruct(const OpenACCDataConstruct &S) {
4214 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4215 // simply emitting its structured block, but in the future we will implement
4216 // some sort of IR.
4217 EmitStmt(S: S.getStructuredBlock());
4218 }
4219
4220 void EmitOpenACCEnterDataConstruct(const OpenACCEnterDataConstruct &S) {
4221 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4222 // but in the future we will implement some sort of IR.
4223 }
4224
4225 void EmitOpenACCExitDataConstruct(const OpenACCExitDataConstruct &S) {
4226 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4227 // but in the future we will implement some sort of IR.
4228 }
4229
4230 void EmitOpenACCHostDataConstruct(const OpenACCHostDataConstruct &S) {
4231 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4232 // simply emitting its structured block, but in the future we will implement
4233 // some sort of IR.
4234 EmitStmt(S: S.getStructuredBlock());
4235 }
4236
4237 void EmitOpenACCWaitConstruct(const OpenACCWaitConstruct &S) {
4238 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4239 // but in the future we will implement some sort of IR.
4240 }
4241
4242 void EmitOpenACCInitConstruct(const OpenACCInitConstruct &S) {
4243 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4244 // but in the future we will implement some sort of IR.
4245 }
4246
4247 void EmitOpenACCShutdownConstruct(const OpenACCShutdownConstruct &S) {
4248 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4249 // but in the future we will implement some sort of IR.
4250 }
4251
4252 void EmitOpenACCSetConstruct(const OpenACCSetConstruct &S) {
4253 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4254 // but in the future we will implement some sort of IR.
4255 }
4256
4257 void EmitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &S) {
4258 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4259 // but in the future we will implement some sort of IR.
4260 }
4261
4262 void EmitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &S) {
4263 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4264 // simply emitting its associated stmt, but in the future we will implement
4265 // some sort of IR.
4266 EmitStmt(S: S.getAssociatedStmt());
4267 }
4268 void EmitOpenACCCacheConstruct(const OpenACCCacheConstruct &S) {
4269 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4270 // but in the future we will implement some sort of IR.
4271 }
4272
4273 //===--------------------------------------------------------------------===//
4274 // LValue Expression Emission
4275 //===--------------------------------------------------------------------===//
4276
4277 /// Create a check that a scalar RValue is non-null.
4278 llvm::Value *EmitNonNullRValueCheck(RValue RV, QualType T);
4279
4280 /// GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
4281 RValue GetUndefRValue(QualType Ty);
4282
4283 /// EmitUnsupportedRValue - Emit a dummy r-value using the type of E
4284 /// and issue an ErrorUnsupported style diagnostic (using the
4285 /// provided Name).
4286 RValue EmitUnsupportedRValue(const Expr *E, const char *Name);
4287
4288 /// EmitUnsupportedLValue - Emit a dummy l-value using the type of E and issue
4289 /// an ErrorUnsupported style diagnostic (using the provided Name).
4290 LValue EmitUnsupportedLValue(const Expr *E, const char *Name);
4291
4292 /// EmitLValue - Emit code to compute a designator that specifies the location
4293 /// of the expression.
4294 ///
4295 /// This can return one of two things: a simple address or a bitfield
4296 /// reference. In either case, the LLVM Value* in the LValue structure is
4297 /// guaranteed to be an LLVM pointer type.
4298 ///
4299 /// If this returns a bitfield reference, nothing about the pointee type of
4300 /// the LLVM value is known: For example, it may not be a pointer to an
4301 /// integer.
4302 ///
4303 /// If this returns a normal address, and if the lvalue's C type is fixed
4304 /// size, this method guarantees that the returned pointer type will point to
4305 /// an LLVM type of the same size of the lvalue's type. If the lvalue has a
4306 /// variable length type, this is not possible.
4307 ///
4308 LValue EmitLValue(const Expr *E,
4309 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
4310
4311private:
4312 LValue EmitLValueHelper(const Expr *E, KnownNonNull_t IsKnownNonNull);
4313
4314public:
4315 /// Same as EmitLValue but additionally we generate checking code to
4316 /// guard against undefined behavior. This is only suitable when we know
4317 /// that the address will be used to access the object.
4318 LValue EmitCheckedLValue(const Expr *E, TypeCheckKind TCK);
4319
4320 RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc);
4321
4322 void EmitAtomicInit(Expr *E, LValue lvalue);
4323
4324 bool LValueIsSuitableForInlineAtomic(LValue Src);
4325
4326 RValue EmitAtomicLoad(LValue LV, SourceLocation SL,
4327 AggValueSlot Slot = AggValueSlot::ignored());
4328
4329 RValue EmitAtomicLoad(LValue lvalue, SourceLocation loc,
4330 llvm::AtomicOrdering AO, bool IsVolatile = false,
4331 AggValueSlot slot = AggValueSlot::ignored());
4332
4333 void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit);
4334
4335 void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO,
4336 bool IsVolatile, bool isInit);
4337
4338 std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
4339 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
4340 llvm::AtomicOrdering Success =
4341 llvm::AtomicOrdering::SequentiallyConsistent,
4342 llvm::AtomicOrdering Failure =
4343 llvm::AtomicOrdering::SequentiallyConsistent,
4344 bool IsWeak = false, AggValueSlot Slot = AggValueSlot::ignored());
4345
4346 /// Emit an atomicrmw instruction, and applying relevant metadata when
4347 /// applicable.
4348 llvm::AtomicRMWInst *emitAtomicRMWInst(
4349 llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val,
4350 llvm::AtomicOrdering Order = llvm::AtomicOrdering::SequentiallyConsistent,
4351 llvm::SyncScope::ID SSID = llvm::SyncScope::System,
4352 const AtomicExpr *AE = nullptr);
4353
4354 void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO,
4355 const llvm::function_ref<RValue(RValue)> &UpdateOp,
4356 bool IsVolatile);
4357
4358 /// EmitToMemory - Change a scalar value from its value
4359 /// representation to its in-memory representation.
4360 llvm::Value *EmitToMemory(llvm::Value *Value, QualType Ty);
4361
4362 /// EmitFromMemory - Change a scalar value from its memory
4363 /// representation to its value representation.
4364 llvm::Value *EmitFromMemory(llvm::Value *Value, QualType Ty);
4365
4366 /// Check if the scalar \p Value is within the valid range for the given
4367 /// type \p Ty.
4368 ///
4369 /// Returns true if a check is needed (even if the range is unknown).
4370 bool EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
4371 SourceLocation Loc);
4372
4373 /// EmitLoadOfScalar - Load a scalar value from an address, taking
4374 /// care to appropriately convert from the memory representation to
4375 /// the LLVM value representation.
4376 llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4377 SourceLocation Loc,
4378 AlignmentSource Source = AlignmentSource::Type,
4379 bool isNontemporal = false) {
4380 return EmitLoadOfScalar(Addr, Volatile, Ty, Loc, BaseInfo: LValueBaseInfo(Source),
4381 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isNontemporal);
4382 }
4383
4384 llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4385 SourceLocation Loc, LValueBaseInfo BaseInfo,
4386 TBAAAccessInfo TBAAInfo,
4387 bool isNontemporal = false);
4388
4389 /// EmitLoadOfScalar - Load a scalar value from an address, taking
4390 /// care to appropriately convert from the memory representation to
4391 /// the LLVM value representation. The l-value must be a simple
4392 /// l-value.
4393 llvm::Value *EmitLoadOfScalar(LValue lvalue, SourceLocation Loc);
4394
4395 /// EmitStoreOfScalar - Store a scalar value to an address, taking
4396 /// care to appropriately convert from the memory representation to
4397 /// the LLVM value representation.
4398 void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,
4399 QualType Ty,
4400 AlignmentSource Source = AlignmentSource::Type,
4401 bool isInit = false, bool isNontemporal = false) {
4402 EmitStoreOfScalar(Value, Addr, Volatile, Ty, BaseInfo: LValueBaseInfo(Source),
4403 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isInit, isNontemporal);
4404 }
4405
4406 void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,
4407 QualType Ty, LValueBaseInfo BaseInfo,
4408 TBAAAccessInfo TBAAInfo, bool isInit = false,
4409 bool isNontemporal = false);
4410
4411 /// EmitStoreOfScalar - Store a scalar value to an address, taking
4412 /// care to appropriately convert from the memory representation to
4413 /// the LLVM value representation. The l-value must be a simple
4414 /// l-value. The isInit flag indicates whether this is an initialization.
4415 /// If so, atomic qualifiers are ignored and the store is always non-atomic.
4416 void EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
4417 bool isInit = false);
4418
4419 /// EmitLoadOfLValue - Given an expression that represents a value lvalue,
4420 /// this method emits the address of the lvalue, then loads the result as an
4421 /// rvalue, returning the rvalue.
4422 RValue EmitLoadOfLValue(LValue V, SourceLocation Loc);
4423 RValue EmitLoadOfExtVectorElementLValue(LValue V);
4424 RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc);
4425 RValue EmitLoadOfGlobalRegLValue(LValue LV);
4426
4427 /// Like EmitLoadOfLValue but also handles complex and aggregate types.
4428 RValue EmitLoadOfAnyValue(LValue V,
4429 AggValueSlot Slot = AggValueSlot::ignored(),
4430 SourceLocation Loc = {});
4431
4432 /// EmitStoreThroughLValue - Store the specified rvalue into the specified
4433 /// lvalue, where both are guaranteed to the have the same type, and that type
4434 /// is 'Ty'.
4435 void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit = false);
4436 void EmitStoreThroughExtVectorComponentLValue(RValue Src, LValue Dst);
4437 void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst);
4438
4439 /// EmitStoreThroughBitfieldLValue - Store Src into Dst with same constraints
4440 /// as EmitStoreThroughLValue.
4441 ///
4442 /// \param Result [out] - If non-null, this will be set to a Value* for the
4443 /// bit-field contents after the store, appropriate for use as the result of
4444 /// an assignment to the bit-field.
4445 void EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
4446 llvm::Value **Result = nullptr);
4447
4448 /// Emit an l-value for an assignment (simple or compound) of complex type.
4449 LValue EmitComplexAssignmentLValue(const BinaryOperator *E);
4450 LValue EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E);
4451 LValue EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E,
4452 llvm::Value *&Result);
4453
4454 // Note: only available for agg return types
4455 LValue EmitBinaryOperatorLValue(const BinaryOperator *E);
4456 LValue EmitCompoundAssignmentLValue(const CompoundAssignOperator *E);
4457 // Note: only available for agg return types
4458 LValue EmitCallExprLValue(const CallExpr *E,
4459 llvm::CallBase **CallOrInvoke = nullptr);
4460 // Note: only available for agg return types
4461 LValue EmitVAArgExprLValue(const VAArgExpr *E);
4462 LValue EmitDeclRefLValue(const DeclRefExpr *E);
4463 LValue EmitStringLiteralLValue(const StringLiteral *E);
4464 LValue EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E);
4465 LValue EmitPredefinedLValue(const PredefinedExpr *E);
4466 LValue EmitUnaryOpLValue(const UnaryOperator *E);
4467 LValue EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
4468 bool Accessed = false);
4469 llvm::Value *EmitMatrixIndexExpr(const Expr *E);
4470 LValue EmitMatrixSingleSubscriptExpr(const MatrixSingleSubscriptExpr *E);
4471 LValue EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E);
4472 LValue EmitArraySectionExpr(const ArraySectionExpr *E,
4473 bool IsLowerBound = true);
4474 LValue EmitExtVectorElementExpr(const ExtVectorElementExpr *E);
4475 LValue EmitMatrixElementExpr(const MatrixElementExpr *E);
4476 LValue EmitMemberExpr(const MemberExpr *E);
4477 LValue EmitObjCIsaExpr(const ObjCIsaExpr *E);
4478 LValue EmitCompoundLiteralLValue(const CompoundLiteralExpr *E);
4479 LValue EmitInitListLValue(const InitListExpr *E);
4480 void EmitIgnoredConditionalOperator(const AbstractConditionalOperator *E);
4481 LValue EmitConditionalOperatorLValue(const AbstractConditionalOperator *E);
4482 LValue EmitCastLValue(const CastExpr *E);
4483 LValue EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);
4484 LValue EmitOpaqueValueLValue(const OpaqueValueExpr *e);
4485 LValue EmitHLSLArrayAssignLValue(const BinaryOperator *E);
4486
4487 std::pair<LValue, LValue> EmitHLSLOutArgLValues(const HLSLOutArgExpr *E,
4488 QualType Ty);
4489 LValue EmitHLSLOutArgExpr(const HLSLOutArgExpr *E, CallArgList &Args,
4490 QualType Ty);
4491
4492 Address EmitExtVectorElementLValue(LValue V);
4493
4494 RValue EmitRValueForField(LValue LV, const FieldDecl *FD, SourceLocation Loc);
4495
4496 Address EmitArrayToPointerDecay(const Expr *Array,
4497 LValueBaseInfo *BaseInfo = nullptr,
4498 TBAAAccessInfo *TBAAInfo = nullptr);
4499
4500 class ConstantEmission {
4501 llvm::PointerIntPair<llvm::Constant *, 1, bool> ValueAndIsReference;
4502 ConstantEmission(llvm::Constant *C, bool isReference)
4503 : ValueAndIsReference(C, isReference) {}
4504
4505 public:
4506 ConstantEmission() {}
4507 static ConstantEmission forReference(llvm::Constant *C) {
4508 return ConstantEmission(C, true);
4509 }
4510 static ConstantEmission forValue(llvm::Constant *C) {
4511 return ConstantEmission(C, false);
4512 }
4513
4514 explicit operator bool() const {
4515 return ValueAndIsReference.getOpaqueValue() != nullptr;
4516 }
4517
4518 bool isReference() const { return ValueAndIsReference.getInt(); }
4519 LValue getReferenceLValue(CodeGenFunction &CGF, const Expr *RefExpr) const {
4520 assert(isReference());
4521 return CGF.MakeNaturalAlignAddrLValue(V: ValueAndIsReference.getPointer(),
4522 T: RefExpr->getType());
4523 }
4524
4525 llvm::Constant *getValue() const {
4526 assert(!isReference());
4527 return ValueAndIsReference.getPointer();
4528 }
4529 };
4530
4531 ConstantEmission tryEmitAsConstant(const DeclRefExpr *RefExpr);
4532 ConstantEmission tryEmitAsConstant(const MemberExpr *ME);
4533 llvm::Value *emitScalarConstant(const ConstantEmission &Constant, Expr *E);
4534
4535 RValue EmitPseudoObjectRValue(const PseudoObjectExpr *e,
4536 AggValueSlot slot = AggValueSlot::ignored());
4537 LValue EmitPseudoObjectLValue(const PseudoObjectExpr *e);
4538
4539 void FlattenAccessAndTypeLValue(LValue LVal,
4540 SmallVectorImpl<LValue> &AccessList);
4541
4542 llvm::Value *EmitIvarOffset(const ObjCInterfaceDecl *Interface,
4543 const ObjCIvarDecl *Ivar);
4544 llvm::Value *EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
4545 const ObjCIvarDecl *Ivar);
4546 LValue EmitLValueForField(LValue Base, const FieldDecl *Field,
4547 bool IsInBounds = true);
4548 LValue EmitLValueForLambdaField(const FieldDecl *Field);
4549 LValue EmitLValueForLambdaField(const FieldDecl *Field,
4550 llvm::Value *ThisValue);
4551
4552 /// EmitLValueForFieldInitialization - Like EmitLValueForField, except that
4553 /// if the Field is a reference, this will return the address of the reference
4554 /// and not the address of the value stored in the reference.
4555 LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field);
4556
4557 LValue EmitLValueForIvar(QualType ObjectTy, llvm::Value *Base,
4558 const ObjCIvarDecl *Ivar, unsigned CVRQualifiers);
4559
4560 LValue EmitCXXConstructLValue(const CXXConstructExpr *E);
4561 LValue EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E);
4562 LValue EmitCXXTypeidLValue(const CXXTypeidExpr *E);
4563 LValue EmitCXXUuidofLValue(const CXXUuidofExpr *E);
4564
4565 LValue EmitObjCMessageExprLValue(const ObjCMessageExpr *E);
4566 LValue EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E);
4567 LValue EmitStmtExprLValue(const StmtExpr *E);
4568 LValue EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E);
4569 LValue EmitObjCSelectorLValue(const ObjCSelectorExpr *E);
4570 void EmitDeclRefExprDbgValue(const DeclRefExpr *E, const APValue &Init);
4571
4572 //===--------------------------------------------------------------------===//
4573 // Scalar Expression Emission
4574 //===--------------------------------------------------------------------===//
4575
4576 /// EmitCall - Generate a call of the given function, expecting the given
4577 /// result type, and using the given argument list which specifies both the
4578 /// LLVM arguments and the types they were derived from.
4579 RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4580 ReturnValueSlot ReturnValue, const CallArgList &Args,
4581 llvm::CallBase **CallOrInvoke, bool IsMustTail,
4582 SourceLocation Loc,
4583 bool IsVirtualFunctionPointerThunk = false);
4584 RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4585 ReturnValueSlot ReturnValue, const CallArgList &Args,
4586 llvm::CallBase **CallOrInvoke = nullptr,
4587 bool IsMustTail = false) {
4588 return EmitCall(CallInfo, Callee, ReturnValue, Args, CallOrInvoke,
4589 IsMustTail, Loc: SourceLocation());
4590 }
4591 RValue EmitCall(QualType FnType, const CGCallee &Callee, const CallExpr *E,
4592 ReturnValueSlot ReturnValue, llvm::Value *Chain = nullptr,
4593 llvm::CallBase **CallOrInvoke = nullptr,
4594 CGFunctionInfo const **ResolvedFnInfo = nullptr);
4595
4596 // If a Call or Invoke instruction was emitted for this CallExpr, this method
4597 // writes the pointer to `CallOrInvoke` if it's not null.
4598 RValue EmitCallExpr(const CallExpr *E,
4599 ReturnValueSlot ReturnValue = ReturnValueSlot(),
4600 llvm::CallBase **CallOrInvoke = nullptr);
4601 RValue EmitSimpleCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,
4602 llvm::CallBase **CallOrInvoke = nullptr);
4603 CGCallee EmitCallee(const Expr *E);
4604
4605 void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl);
4606 void checkTargetFeatures(SourceLocation Loc, const FunctionDecl *TargetDecl);
4607
4608 llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4609 const Twine &name = "");
4610 llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4611 ArrayRef<llvm::Value *> args,
4612 const Twine &name = "");
4613 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4614 const Twine &name = "");
4615 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4616 ArrayRef<Address> args,
4617 const Twine &name = "");
4618 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4619 ArrayRef<llvm::Value *> args,
4620 const Twine &name = "");
4621
4622 SmallVector<llvm::OperandBundleDef, 1>
4623 getBundlesForFunclet(llvm::Value *Callee);
4624
4625 llvm::CallBase *EmitCallOrInvoke(llvm::FunctionCallee Callee,
4626 ArrayRef<llvm::Value *> Args,
4627 const Twine &Name = "");
4628 llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4629 ArrayRef<llvm::Value *> args,
4630 const Twine &name = "");
4631 llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4632 const Twine &name = "");
4633 void EmitNoreturnRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4634 ArrayRef<llvm::Value *> args);
4635
4636 CGCallee BuildAppleKextVirtualCall(const CXXMethodDecl *MD,
4637 NestedNameSpecifier Qual, llvm::Type *Ty);
4638
4639 CGCallee BuildAppleKextVirtualDestructorCall(const CXXDestructorDecl *DD,
4640 CXXDtorType Type,
4641 const CXXRecordDecl *RD);
4642
4643 bool isPointerKnownNonNull(const Expr *E);
4644 /// Check whether the underlying base pointer is a constant null.
4645 bool isUnderlyingBasePointerConstantNull(const Expr *E);
4646
4647 /// Create the discriminator from the storage address and the entity hash.
4648 llvm::Value *EmitPointerAuthBlendDiscriminator(llvm::Value *StorageAddress,
4649 llvm::Value *Discriminator);
4650 CGPointerAuthInfo EmitPointerAuthInfo(const PointerAuthSchema &Schema,
4651 llvm::Value *StorageAddress,
4652 GlobalDecl SchemaDecl,
4653 QualType SchemaType);
4654
4655 llvm::Value *EmitPointerAuthSign(const CGPointerAuthInfo &Info,
4656 llvm::Value *Pointer);
4657
4658 llvm::Value *EmitPointerAuthAuth(const CGPointerAuthInfo &Info,
4659 llvm::Value *Pointer);
4660
4661 llvm::Value *emitPointerAuthResign(llvm::Value *Pointer, QualType PointerType,
4662 const CGPointerAuthInfo &CurAuthInfo,
4663 const CGPointerAuthInfo &NewAuthInfo,
4664 bool IsKnownNonNull);
4665 llvm::Value *emitPointerAuthResignCall(llvm::Value *Pointer,
4666 const CGPointerAuthInfo &CurInfo,
4667 const CGPointerAuthInfo &NewInfo);
4668
4669 void EmitPointerAuthOperandBundle(
4670 const CGPointerAuthInfo &Info,
4671 SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
4672
4673 CGPointerAuthInfo EmitPointerAuthInfo(PointerAuthQualifier Qualifier,
4674 Address StorageAddress);
4675 llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
4676 llvm::Value *Pointer, QualType ValueType,
4677 Address StorageAddress,
4678 bool IsKnownNonNull);
4679 llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
4680 const Expr *PointerExpr,
4681 Address StorageAddress);
4682 llvm::Value *EmitPointerAuthUnqualify(PointerAuthQualifier Qualifier,
4683 llvm::Value *Pointer,
4684 QualType PointerType,
4685 Address StorageAddress,
4686 bool IsKnownNonNull);
4687 void EmitPointerAuthCopy(PointerAuthQualifier Qualifier, QualType Type,
4688 Address DestField, Address SrcField);
4689
4690 std::pair<llvm::Value *, CGPointerAuthInfo>
4691 EmitOrigPointerRValue(const Expr *E);
4692
4693 llvm::Value *authPointerToPointerCast(llvm::Value *ResultPtr,
4694 QualType SourceType, QualType DestType);
4695 Address authPointerToPointerCast(Address Ptr, QualType SourceType,
4696 QualType DestType);
4697
4698 Address getAsNaturalAddressOf(Address Addr, QualType PointeeTy);
4699
4700 llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) {
4701 return getAsNaturalAddressOf(Addr, PointeeTy: PointeeType).getBasePointer();
4702 }
4703
4704 // Return the copy constructor name with the prefix "__copy_constructor_"
4705 // removed.
4706 static std::string getNonTrivialCopyConstructorStr(QualType QT,
4707 CharUnits Alignment,
4708 bool IsVolatile,
4709 ASTContext &Ctx);
4710
4711 // Return the destructor name with the prefix "__destructor_" removed.
4712 static std::string getNonTrivialDestructorStr(QualType QT,
4713 CharUnits Alignment,
4714 bool IsVolatile,
4715 ASTContext &Ctx);
4716
4717 // These functions emit calls to the special functions of non-trivial C
4718 // structs.
4719 void defaultInitNonTrivialCStructVar(LValue Dst);
4720 void callCStructDefaultConstructor(LValue Dst);
4721 void callCStructDestructor(LValue Dst);
4722 void callCStructCopyConstructor(LValue Dst, LValue Src);
4723 void callCStructMoveConstructor(LValue Dst, LValue Src);
4724 void callCStructCopyAssignmentOperator(LValue Dst, LValue Src);
4725 void callCStructMoveAssignmentOperator(LValue Dst, LValue Src);
4726
4727 RValue EmitCXXMemberOrOperatorCall(
4728 const CXXMethodDecl *Method, const CGCallee &Callee,
4729 ReturnValueSlot ReturnValue, llvm::Value *This,
4730 llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *E,
4731 CallArgList *RtlArgs, llvm::CallBase **CallOrInvoke);
4732 RValue EmitCXXDestructorCall(GlobalDecl Dtor, const CGCallee &Callee,
4733 llvm::Value *This, QualType ThisTy,
4734 llvm::Value *ImplicitParam,
4735 QualType ImplicitParamTy, const CallExpr *E,
4736 llvm::CallBase **CallOrInvoke = nullptr);
4737 RValue EmitCXXMemberCallExpr(const CXXMemberCallExpr *E,
4738 ReturnValueSlot ReturnValue,
4739 llvm::CallBase **CallOrInvoke = nullptr);
4740 RValue EmitCXXMemberOrOperatorMemberCallExpr(
4741 const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue,
4742 bool HasQualifier, NestedNameSpecifier Qualifier, bool IsArrow,
4743 const Expr *Base, llvm::CallBase **CallOrInvoke);
4744 // Compute the object pointer.
4745 Address EmitCXXMemberDataPointerAddress(
4746 const Expr *E, Address base, llvm::Value *memberPtr,
4747 const MemberPointerType *memberPtrType, bool IsInBounds,
4748 LValueBaseInfo *BaseInfo = nullptr, TBAAAccessInfo *TBAAInfo = nullptr);
4749 RValue EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
4750 ReturnValueSlot ReturnValue,
4751 llvm::CallBase **CallOrInvoke);
4752
4753 RValue EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
4754 const CXXMethodDecl *MD,
4755 ReturnValueSlot ReturnValue,
4756 llvm::CallBase **CallOrInvoke);
4757 RValue EmitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *E);
4758
4759 RValue EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
4760 ReturnValueSlot ReturnValue,
4761 llvm::CallBase **CallOrInvoke);
4762
4763 RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E);
4764 RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E);
4765
4766 RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
4767 const CallExpr *E, ReturnValueSlot ReturnValue);
4768
4769 RValue emitRotate(const CallExpr *E, bool IsRotateRight);
4770
4771 /// Emit IR for __builtin_os_log_format.
4772 RValue emitBuiltinOSLogFormat(const CallExpr &E);
4773
4774 /// Emit IR for __builtin_is_aligned.
4775 RValue EmitBuiltinIsAligned(const CallExpr *E);
4776 /// Emit IR for __builtin_align_up/__builtin_align_down.
4777 RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp);
4778
4779 llvm::Function *generateBuiltinOSLogHelperFunction(
4780 const analyze_os_log::OSLogBufferLayout &Layout,
4781 CharUnits BufferAlignment);
4782
4783 RValue EmitBlockCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,
4784 llvm::CallBase **CallOrInvoke);
4785
4786 /// EmitTargetBuiltinExpr - Emit the given builtin call. Returns 0 if the call
4787 /// is unhandled by the current target.
4788 llvm::Value *EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4789 ReturnValueSlot ReturnValue);
4790
4791 llvm::Value *
4792 EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty,
4793 const llvm::CmpInst::Predicate Pred,
4794 const llvm::Twine &Name = "");
4795 llvm::Value *EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4796 ReturnValueSlot ReturnValue,
4797 llvm::Triple::ArchType Arch);
4798 llvm::Value *EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4799 ReturnValueSlot ReturnValue,
4800 llvm::Triple::ArchType Arch);
4801 llvm::Value *EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4802 ReturnValueSlot ReturnValue,
4803 llvm::Triple::ArchType Arch);
4804 llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::IntegerType *ITy,
4805 QualType RTy);
4806 llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::ArrayType *ATy,
4807 QualType RTy);
4808
4809 llvm::Value *
4810 EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic,
4811 unsigned AltLLVMIntrinsic, const char *NameHint,
4812 unsigned Modifier, const CallExpr *E,
4813 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0,
4814 Address PtrOp1, llvm::Triple::ArchType Arch);
4815
4816 llvm::Function *LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
4817 unsigned Modifier, llvm::Type *ArgTy,
4818 const CallExpr *E);
4819 llvm::Value *EmitNeonCall(llvm::Function *F,
4820 SmallVectorImpl<llvm::Value *> &O, const char *name,
4821 unsigned shift = 0, bool rightshift = false);
4822 llvm::Value *EmitFP8NeonCall(unsigned IID, ArrayRef<llvm::Type *> Tys,
4823 SmallVectorImpl<llvm::Value *> &O,
4824 const CallExpr *E, const char *name);
4825 llvm::Value *EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,
4826 llvm::Type *Ty1, bool Extract,
4827 SmallVectorImpl<llvm::Value *> &Ops,
4828 const CallExpr *E, const char *name);
4829 llvm::Value *EmitFP8NeonFDOTCall(unsigned IID, bool ExtendLaneArg,
4830 llvm::Type *RetTy,
4831 SmallVectorImpl<llvm::Value *> &Ops,
4832 const CallExpr *E, const char *name);
4833 llvm::Value *EmitFP8NeonFMLACall(unsigned IID, bool ExtendLaneArg,
4834 llvm::Type *RetTy,
4835 SmallVectorImpl<llvm::Value *> &Ops,
4836 const CallExpr *E, const char *name);
4837 llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx,
4838 const llvm::ElementCount &Count);
4839 llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx);
4840 llvm::Value *EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty,
4841 bool negateForRightShift);
4842 llvm::Value *EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt,
4843 llvm::Type *Ty, bool usgn, const char *name);
4844 llvm::Value *vectorWrapScalar16(llvm::Value *Op);
4845 /// SVEBuiltinMemEltTy - Returns the memory element type for this memory
4846 /// access builtin. Only required if it can't be inferred from the base
4847 /// pointer operand.
4848 llvm::Type *SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags);
4849
4850 SmallVector<llvm::Type *, 2>
4851 getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType,
4852 ArrayRef<llvm::Value *> Ops);
4853 llvm::Type *getEltType(const SVETypeFlags &TypeFlags);
4854 llvm::ScalableVectorType *getSVEType(const SVETypeFlags &TypeFlags);
4855 llvm::ScalableVectorType *getSVEPredType(const SVETypeFlags &TypeFlags);
4856 llvm::Value *EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags,
4857 ArrayRef<llvm::Value *> Ops);
4858 llvm::Value *EmitSVETupleCreate(const SVETypeFlags &TypeFlags,
4859 llvm::Type *ReturnType,
4860 ArrayRef<llvm::Value *> Ops);
4861 llvm::Value *EmitSVEAllTruePred(const SVETypeFlags &TypeFlags);
4862 llvm::Value *EmitSVEDupX(llvm::Value *Scalar);
4863 llvm::Value *EmitSVEDupX(llvm::Value *Scalar, llvm::Type *Ty);
4864 llvm::Value *EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty);
4865 llvm::Value *EmitSVEPMull(const SVETypeFlags &TypeFlags,
4866 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4867 unsigned BuiltinID);
4868 llvm::Value *EmitSVEMovl(const SVETypeFlags &TypeFlags,
4869 llvm::ArrayRef<llvm::Value *> Ops,
4870 unsigned BuiltinID);
4871 llvm::Value *EmitSVEPredicateCast(llvm::Value *Pred,
4872 llvm::ScalableVectorType *VTy);
4873 llvm::Value *EmitSVEPredicateTupleCast(llvm::Value *PredTuple,
4874 llvm::StructType *Ty);
4875 llvm::Value *EmitSVEGatherLoad(const SVETypeFlags &TypeFlags,
4876 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4877 unsigned IntID);
4878 llvm::Value *EmitSVEScatterStore(const SVETypeFlags &TypeFlags,
4879 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4880 unsigned IntID);
4881 llvm::Value *EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy,
4882 SmallVectorImpl<llvm::Value *> &Ops,
4883 unsigned BuiltinID, bool IsZExtReturn);
4884 llvm::Value *EmitSVEMaskedStore(const CallExpr *,
4885 SmallVectorImpl<llvm::Value *> &Ops,
4886 unsigned BuiltinID);
4887 llvm::Value *EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,
4888 SmallVectorImpl<llvm::Value *> &Ops,
4889 unsigned BuiltinID);
4890 llvm::Value *EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags,
4891 SmallVectorImpl<llvm::Value *> &Ops,
4892 unsigned IntID);
4893 llvm::Value *EmitSVEStructLoad(const SVETypeFlags &TypeFlags,
4894 SmallVectorImpl<llvm::Value *> &Ops,
4895 unsigned IntID);
4896 llvm::Value *EmitSVEStructStore(const SVETypeFlags &TypeFlags,
4897 SmallVectorImpl<llvm::Value *> &Ops,
4898 unsigned IntID);
4899 llvm::Value *EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4900
4901 llvm::Value *EmitSMELd1St1(const SVETypeFlags &TypeFlags,
4902 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4903 unsigned IntID);
4904 llvm::Value *EmitSMEReadWrite(const SVETypeFlags &TypeFlags,
4905 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4906 unsigned IntID);
4907 llvm::Value *EmitSMEZero(const SVETypeFlags &TypeFlags,
4908 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4909 unsigned IntID);
4910 llvm::Value *EmitSMELdrStr(const SVETypeFlags &TypeFlags,
4911 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4912 unsigned IntID);
4913
4914 void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E,
4915 SmallVectorImpl<llvm::Value *> &Ops,
4916 SVETypeFlags TypeFlags);
4917
4918 llvm::Value *EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4919
4920 llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4921 llvm::Triple::ArchType Arch);
4922 llvm::Value *EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4923
4924 llvm::Value *BuildVector(ArrayRef<llvm::Value *> Ops);
4925 llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4926 llvm::Value *EmitPPCBuiltinCpu(unsigned BuiltinID, llvm::Type *ReturnType,
4927 StringRef CPUStr);
4928 llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4929 llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4930 llvm::Value *EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4931 ReturnValueSlot ReturnValue);
4932
4933 // Returns a builtin function that the SPIR-V backend will expand into a spec
4934 // constant.
4935 llvm::Function *
4936 getSpecConstantFunction(const clang::QualType &SpecConstantType);
4937
4938 llvm::Value *EmitDirectXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4939 llvm::Value *EmitSPIRVBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4940 llvm::Value *EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx,
4941 const CallExpr *E);
4942 llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4943 llvm::Value *EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4944 llvm::Value *EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
4945 const CallExpr *E);
4946 llvm::Value *EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
4947 llvm::Value *EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4948 ReturnValueSlot ReturnValue);
4949
4950 llvm::Value *EmitRISCVCpuSupports(const CallExpr *E);
4951 llvm::Value *EmitRISCVCpuSupports(ArrayRef<StringRef> FeaturesStrs);
4952 llvm::Value *EmitRISCVCpuInit();
4953 llvm::Value *EmitRISCVCpuIs(const CallExpr *E);
4954 llvm::Value *EmitRISCVCpuIs(StringRef CPUStr);
4955
4956 void AddAMDGPUFenceAddressSpaceMMRA(llvm::Instruction *Inst,
4957 const CallExpr *E);
4958 void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope,
4959 llvm::AtomicOrdering &AO,
4960 llvm::SyncScope::ID &SSID);
4961
4962 enum class MSVCIntrin;
4963 llvm::Value *EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E);
4964
4965 llvm::Value *EmitBuiltinAvailable(const VersionTuple &Version);
4966
4967 llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E);
4968 llvm::Value *EmitObjCStringLiteral(const ObjCStringLiteral *E);
4969 llvm::Value *EmitObjCBoxedExpr(const ObjCBoxedExpr *E);
4970 llvm::Value *EmitObjCArrayLiteral(const ObjCArrayLiteral *E);
4971 llvm::Value *EmitObjCDictionaryLiteral(const ObjCDictionaryLiteral *E);
4972 llvm::Value *
4973 EmitObjCCollectionLiteral(const Expr *E,
4974 const ObjCMethodDecl *MethodWithObjects);
4975 llvm::Value *EmitObjCSelectorExpr(const ObjCSelectorExpr *E);
4976 RValue EmitObjCMessageExpr(const ObjCMessageExpr *E,
4977 ReturnValueSlot Return = ReturnValueSlot());
4978
4979 /// Retrieves the default cleanup kind for an ARC cleanup.
4980 /// Except under -fobjc-arc-eh, ARC cleanups are normal-only.
4981 CleanupKind getARCCleanupKind() {
4982 return CGM.getCodeGenOpts().ObjCAutoRefCountExceptions ? NormalAndEHCleanup
4983 : NormalCleanup;
4984 }
4985
4986 // ARC primitives.
4987 void EmitARCInitWeak(Address addr, llvm::Value *value);
4988 void EmitARCDestroyWeak(Address addr);
4989 llvm::Value *EmitARCLoadWeak(Address addr);
4990 llvm::Value *EmitARCLoadWeakRetained(Address addr);
4991 llvm::Value *EmitARCStoreWeak(Address addr, llvm::Value *value, bool ignored);
4992 void emitARCCopyAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
4993 void emitARCMoveAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
4994 void EmitARCCopyWeak(Address dst, Address src);
4995 void EmitARCMoveWeak(Address dst, Address src);
4996 llvm::Value *EmitARCRetainAutorelease(QualType type, llvm::Value *value);
4997 llvm::Value *EmitARCRetainAutoreleaseNonBlock(llvm::Value *value);
4998 llvm::Value *EmitARCStoreStrong(LValue lvalue, llvm::Value *value,
4999 bool resultIgnored);
5000 llvm::Value *EmitARCStoreStrongCall(Address addr, llvm::Value *value,
5001 bool resultIgnored);
5002 llvm::Value *EmitARCRetain(QualType type, llvm::Value *value);
5003 llvm::Value *EmitARCRetainNonBlock(llvm::Value *value);
5004 llvm::Value *EmitARCRetainBlock(llvm::Value *value, bool mandatory);
5005 void EmitARCDestroyStrong(Address addr, ARCPreciseLifetime_t precise);
5006 void EmitARCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
5007 llvm::Value *EmitARCAutorelease(llvm::Value *value);
5008 llvm::Value *EmitARCAutoreleaseReturnValue(llvm::Value *value);
5009 llvm::Value *EmitARCRetainAutoreleaseReturnValue(llvm::Value *value);
5010 llvm::Value *EmitARCRetainAutoreleasedReturnValue(llvm::Value *value);
5011 llvm::Value *EmitARCUnsafeClaimAutoreleasedReturnValue(llvm::Value *value);
5012
5013 llvm::Value *EmitObjCAutorelease(llvm::Value *value, llvm::Type *returnType);
5014 llvm::Value *EmitObjCRetainNonBlock(llvm::Value *value,
5015 llvm::Type *returnType);
5016 void EmitObjCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
5017
5018 std::pair<LValue, llvm::Value *>
5019 EmitARCStoreAutoreleasing(const BinaryOperator *e);
5020 std::pair<LValue, llvm::Value *> EmitARCStoreStrong(const BinaryOperator *e,
5021 bool ignored);
5022 std::pair<LValue, llvm::Value *>
5023 EmitARCStoreUnsafeUnretained(const BinaryOperator *e, bool ignored);
5024
5025 llvm::Value *EmitObjCAlloc(llvm::Value *value, llvm::Type *returnType);
5026 llvm::Value *EmitObjCAllocWithZone(llvm::Value *value,
5027 llvm::Type *returnType);
5028 llvm::Value *EmitObjCAllocInit(llvm::Value *value, llvm::Type *resultType);
5029
5030 llvm::Value *EmitObjCThrowOperand(const Expr *expr);
5031 llvm::Value *EmitObjCConsumeObject(QualType T, llvm::Value *Ptr);
5032 llvm::Value *EmitObjCExtendObjectLifetime(QualType T, llvm::Value *Ptr);
5033
5034 llvm::Value *EmitARCExtendBlockObject(const Expr *expr);
5035 llvm::Value *EmitARCReclaimReturnedObject(const Expr *e,
5036 bool allowUnsafeClaim);
5037 llvm::Value *EmitARCRetainScalarExpr(const Expr *expr);
5038 llvm::Value *EmitARCRetainAutoreleaseScalarExpr(const Expr *expr);
5039 llvm::Value *EmitARCUnsafeUnretainedScalarExpr(const Expr *expr);
5040
5041 void EmitARCIntrinsicUse(ArrayRef<llvm::Value *> values);
5042
5043 void EmitARCNoopIntrinsicUse(ArrayRef<llvm::Value *> values);
5044
5045 static Destroyer destroyARCStrongImprecise;
5046 static Destroyer destroyARCStrongPrecise;
5047 static Destroyer destroyARCWeak;
5048 static Destroyer emitARCIntrinsicUse;
5049 static Destroyer destroyNonTrivialCStruct;
5050
5051 void EmitObjCAutoreleasePoolPop(llvm::Value *Ptr);
5052 llvm::Value *EmitObjCAutoreleasePoolPush();
5053 llvm::Value *EmitObjCMRRAutoreleasePoolPush();
5054 void EmitObjCAutoreleasePoolCleanup(llvm::Value *Ptr);
5055 void EmitObjCMRRAutoreleasePoolPop(llvm::Value *Ptr);
5056
5057 /// Emits a reference binding to the passed in expression.
5058 RValue EmitReferenceBindingToExpr(const Expr *E);
5059
5060 //===--------------------------------------------------------------------===//
5061 // Expression Emission
5062 //===--------------------------------------------------------------------===//
5063
5064 // Expressions are broken into three classes: scalar, complex, aggregate.
5065
5066 /// EmitScalarExpr - Emit the computation of the specified expression of LLVM
5067 /// scalar type, returning the result.
5068 llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
5069
5070 /// Emit a conversion from the specified type to the specified destination
5071 /// type, both of which are LLVM scalar types.
5072 llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
5073 QualType DstTy, SourceLocation Loc);
5074
5075 /// Emit a conversion from the specified complex type to the specified
5076 /// destination type, where the destination type is an LLVM scalar type.
5077 llvm::Value *EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy,
5078 QualType DstTy,
5079 SourceLocation Loc);
5080
5081 /// EmitAggExpr - Emit the computation of the specified expression
5082 /// of aggregate type. The result is computed into the given slot,
5083 /// which may be null to indicate that the value is not needed.
5084 void EmitAggExpr(const Expr *E, AggValueSlot AS);
5085
5086 /// EmitAggExprToLValue - Emit the computation of the specified expression of
5087 /// aggregate type into a temporary LValue.
5088 LValue EmitAggExprToLValue(const Expr *E);
5089
5090 enum ExprValueKind { EVK_RValue, EVK_NonRValue };
5091
5092 /// EmitAggFinalDestCopy - Emit copy of the specified aggregate into
5093 /// destination address.
5094 void EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest, const LValue &Src,
5095 ExprValueKind SrcKind);
5096
5097 /// Create a store to \arg DstPtr from \arg Src, truncating the stored value
5098 /// to at most \arg DstSize bytes.
5099 void CreateCoercedStore(llvm::Value *Src, QualType SrcFETy, Address Dst,
5100 llvm::TypeSize DstSize, bool DstIsVolatile);
5101
5102 /// EmitExtendGCLifetime - Given a pointer to an Objective-C object,
5103 /// make sure it survives garbage collection until this point.
5104 void EmitExtendGCLifetime(llvm::Value *object);
5105
5106 /// EmitComplexExpr - Emit the computation of the specified expression of
5107 /// complex type, returning the result.
5108 ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal = false,
5109 bool IgnoreImag = false);
5110
5111 /// EmitComplexExprIntoLValue - Emit the given expression of complex
5112 /// type and place its result into the specified l-value.
5113 void EmitComplexExprIntoLValue(const Expr *E, LValue dest, bool isInit);
5114
5115 /// EmitStoreOfComplex - Store a complex number into the specified l-value.
5116 void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit);
5117
5118 /// EmitLoadOfComplex - Load a complex number from the specified l-value.
5119 ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc);
5120
5121 ComplexPairTy EmitPromotedComplexExpr(const Expr *E, QualType PromotionType);
5122 llvm::Value *EmitPromotedScalarExpr(const Expr *E, QualType PromotionType);
5123 ComplexPairTy EmitPromotedValue(ComplexPairTy result, QualType PromotionType);
5124 ComplexPairTy EmitUnPromotedValue(ComplexPairTy result,
5125 QualType PromotionType);
5126
5127 Address emitAddrOfRealComponent(Address complex, QualType complexType);
5128 Address emitAddrOfImagComponent(Address complex, QualType complexType);
5129
5130 /// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the
5131 /// global variable that has already been created for it. If the initializer
5132 /// has a different type than GV does, this may free GV and return a different
5133 /// one. Otherwise it just returns GV.
5134 llvm::GlobalVariable *AddInitializerToStaticVarDecl(const VarDecl &D,
5135 llvm::GlobalVariable *GV);
5136
5137 // Emit an @llvm.invariant.start call for the given memory region.
5138 void EmitInvariantStart(llvm::Constant *Addr, CharUnits Size);
5139
5140 /// EmitCXXGlobalVarDeclInit - Create the initializer for a C++
5141 /// variable with global storage.
5142 void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::GlobalVariable *GV,
5143 bool PerformInit);
5144
5145 llvm::Constant *createAtExitStub(const VarDecl &VD, llvm::FunctionCallee Dtor,
5146 llvm::Constant *Addr);
5147
5148 llvm::Function *createTLSAtExitStub(const VarDecl &VD,
5149 llvm::FunctionCallee Dtor,
5150 llvm::Constant *Addr,
5151 llvm::FunctionCallee &AtExit);
5152
5153 /// Call atexit() with a function that passes the given argument to
5154 /// the given function.
5155 void registerGlobalDtorWithAtExit(const VarDecl &D, llvm::FunctionCallee fn,
5156 llvm::Constant *addr);
5157
5158 /// Registers the dtor using 'llvm.global_dtors' for platforms that do not
5159 /// support an 'atexit()' function.
5160 void registerGlobalDtorWithLLVM(const VarDecl &D, llvm::FunctionCallee fn,
5161 llvm::Constant *addr);
5162
5163 /// Call atexit() with function dtorStub.
5164 void registerGlobalDtorWithAtExit(llvm::Constant *dtorStub);
5165
5166 /// Call unatexit() with function dtorStub.
5167 llvm::Value *unregisterGlobalDtorWithUnAtExit(llvm::Constant *dtorStub);
5168
5169 /// Emit code in this function to perform a guarded variable
5170 /// initialization. Guarded initializations are used when it's not
5171 /// possible to prove that an initialization will be done exactly
5172 /// once, e.g. with a static local variable or a static data member
5173 /// of a class template.
5174 void EmitCXXGuardedInit(const VarDecl &D, llvm::GlobalVariable *DeclPtr,
5175 bool PerformInit);
5176
5177 enum class GuardKind { VariableGuard, TlsGuard };
5178
5179 /// Emit a branch to select whether or not to perform guarded initialization.
5180 void EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
5181 llvm::BasicBlock *InitBlock,
5182 llvm::BasicBlock *NoInitBlock, GuardKind Kind,
5183 const VarDecl *D);
5184
5185 /// GenerateCXXGlobalInitFunc - Generates code for initializing global
5186 /// variables.
5187 void
5188 GenerateCXXGlobalInitFunc(llvm::Function *Fn,
5189 ArrayRef<llvm::Function *> CXXThreadLocals,
5190 ConstantAddress Guard = ConstantAddress::invalid());
5191
5192 /// GenerateCXXGlobalCleanUpFunc - Generates code for cleaning up global
5193 /// variables.
5194 void GenerateCXXGlobalCleanUpFunc(
5195 llvm::Function *Fn,
5196 ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
5197 llvm::Constant *>>
5198 DtorsOrStermFinalizers);
5199
5200 void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn, const VarDecl *D,
5201 llvm::GlobalVariable *Addr,
5202 bool PerformInit);
5203
5204 void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest);
5205
5206 void EmitSynthesizedCXXCopyCtor(Address Dest, Address Src, const Expr *Exp);
5207
5208 void EmitCXXThrowExpr(const CXXThrowExpr *E, bool KeepInsertionPoint = true);
5209
5210 RValue EmitAtomicExpr(AtomicExpr *E);
5211
5212 void EmitFakeUse(Address Addr);
5213
5214 //===--------------------------------------------------------------------===//
5215 // Annotations Emission
5216 //===--------------------------------------------------------------------===//
5217
5218 /// Emit an annotation call (intrinsic).
5219 llvm::Value *EmitAnnotationCall(llvm::Function *AnnotationFn,
5220 llvm::Value *AnnotatedVal,
5221 StringRef AnnotationStr,
5222 SourceLocation Location,
5223 const AnnotateAttr *Attr);
5224
5225 /// Emit local annotations for the local variable V, declared by D.
5226 void EmitVarAnnotations(const VarDecl *D, llvm::Value *V);
5227
5228 /// Emit field annotations for the given field & value. Returns the
5229 /// annotation result.
5230 Address EmitFieldAnnotations(const FieldDecl *D, Address V);
5231
5232 //===--------------------------------------------------------------------===//
5233 // Internal Helpers
5234 //===--------------------------------------------------------------------===//
5235
5236 /// ContainsLabel - Return true if the statement contains a label in it. If
5237 /// this statement is not executed normally, it not containing a label means
5238 /// that we can just remove the code.
5239 static bool ContainsLabel(const Stmt *S, bool IgnoreCaseStmts = false);
5240
5241 /// containsBreak - Return true if the statement contains a break out of it.
5242 /// If the statement (recursively) contains a switch or loop with a break
5243 /// inside of it, this is fine.
5244 static bool containsBreak(const Stmt *S);
5245
5246 /// Determine if the given statement might introduce a declaration into the
5247 /// current scope, by being a (possibly-labelled) DeclStmt.
5248 static bool mightAddDeclToScope(const Stmt *S);
5249
5250 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
5251 /// to a constant, or if it does but contains a label, return false. If it
5252 /// constant folds return true and set the boolean result in Result.
5253 bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result,
5254 bool AllowLabels = false);
5255
5256 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
5257 /// to a constant, or if it does but contains a label, return false. If it
5258 /// constant folds return true and set the folded value.
5259 bool ConstantFoldsToSimpleInteger(const Expr *Cond, llvm::APSInt &Result,
5260 bool AllowLabels = false);
5261
5262 /// Ignore parentheses and logical-NOT to track conditions consistently.
5263 static const Expr *stripCond(const Expr *C);
5264
5265 /// isInstrumentedCondition - Determine whether the given condition is an
5266 /// instrumentable condition (i.e. no "&&" or "||").
5267 static bool isInstrumentedCondition(const Expr *C);
5268
5269 /// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
5270 /// increments a profile counter based on the semantics of the given logical
5271 /// operator opcode. This is used to instrument branch condition coverage
5272 /// for logical operators.
5273 void EmitBranchToCounterBlock(const Expr *Cond, BinaryOperator::Opcode LOp,
5274 llvm::BasicBlock *TrueBlock,
5275 llvm::BasicBlock *FalseBlock,
5276 uint64_t TrueCount = 0,
5277 Stmt::Likelihood LH = Stmt::LH_None,
5278 const Expr *CntrIdx = nullptr);
5279
5280 /// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an
5281 /// if statement) to the specified blocks. Based on the condition, this might
5282 /// try to simplify the codegen of the conditional based on the branch.
5283 /// TrueCount should be the number of times we expect the condition to
5284 /// evaluate to true based on PGO data.
5285 void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock,
5286 llvm::BasicBlock *FalseBlock, uint64_t TrueCount,
5287 Stmt::Likelihood LH = Stmt::LH_None,
5288 const Expr *ConditionalOp = nullptr,
5289 const VarDecl *ConditionalDecl = nullptr);
5290
5291 /// Given an assignment `*LHS = RHS`, emit a test that checks if \p RHS is
5292 /// nonnull, if \p LHS is marked _Nonnull.
5293 void EmitNullabilityCheck(LValue LHS, llvm::Value *RHS, SourceLocation Loc);
5294
5295 /// An enumeration which makes it easier to specify whether or not an
5296 /// operation is a subtraction.
5297 enum { NotSubtraction = false, IsSubtraction = true };
5298
5299 /// Emit pointer + index arithmetic.
5300 llvm::Value *EmitPointerArithmetic(const BinaryOperator *BO,
5301 Expr *pointerOperand, llvm::Value *pointer,
5302 Expr *indexOperand, llvm::Value *index,
5303 bool isSubtraction);
5304
5305 /// Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to
5306 /// detect undefined behavior when the pointer overflow sanitizer is enabled.
5307 /// \p SignedIndices indicates whether any of the GEP indices are signed.
5308 /// \p IsSubtraction indicates whether the expression used to form the GEP
5309 /// is a subtraction.
5310 llvm::Value *EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr,
5311 ArrayRef<llvm::Value *> IdxList,
5312 bool SignedIndices, bool IsSubtraction,
5313 SourceLocation Loc,
5314 const Twine &Name = "");
5315
5316 Address EmitCheckedInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
5317 llvm::Type *elementType, bool SignedIndices,
5318 bool IsSubtraction, SourceLocation Loc,
5319 CharUnits Align, const Twine &Name = "");
5320
5321 /// Specifies which type of sanitizer check to apply when handling a
5322 /// particular builtin.
5323 enum BuiltinCheckKind {
5324 BCK_CTZPassedZero,
5325 BCK_CLZPassedZero,
5326 BCK_AssumePassedFalse,
5327 };
5328
5329 /// Emits an argument for a call to a builtin. If the builtin sanitizer is
5330 /// enabled, a runtime check specified by \p Kind is also emitted.
5331 llvm::Value *EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind);
5332
5333 /// Emits an argument for a call to a `__builtin_assume`. If the builtin
5334 /// sanitizer is enabled, a runtime check is also emitted.
5335 llvm::Value *EmitCheckedArgForAssume(const Expr *E);
5336
5337 /// Emit a description of a type in a format suitable for passing to
5338 /// a runtime sanitizer handler.
5339 llvm::Constant *EmitCheckTypeDescriptor(QualType T);
5340
5341 /// Convert a value into a format suitable for passing to a runtime
5342 /// sanitizer handler.
5343 llvm::Value *EmitCheckValue(llvm::Value *V);
5344
5345 /// Emit a description of a source location in a format suitable for
5346 /// passing to a runtime sanitizer handler.
5347 llvm::Constant *EmitCheckSourceLocation(SourceLocation Loc);
5348
5349 void EmitKCFIOperandBundle(const CGCallee &Callee,
5350 SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
5351
5352 /// Create a basic block that will either trap or call a handler function in
5353 /// the UBSan runtime with the provided arguments, and create a conditional
5354 /// branch to it.
5355 void
5356 EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
5357 Checked,
5358 SanitizerHandler Check, ArrayRef<llvm::Constant *> StaticArgs,
5359 ArrayRef<llvm::Value *> DynamicArgs,
5360 const TrapReason *TR = nullptr);
5361
5362 /// Emit a slow path cross-DSO CFI check which calls __cfi_slowpath
5363 /// if Cond if false.
5364 void EmitCfiSlowPathCheck(SanitizerKind::SanitizerOrdinal Ordinal,
5365 llvm::Value *Cond, llvm::ConstantInt *TypeId,
5366 llvm::Value *Ptr,
5367 ArrayRef<llvm::Constant *> StaticArgs);
5368
5369 /// Emit a reached-unreachable diagnostic if \p Loc is valid and runtime
5370 /// checking is enabled. Otherwise, just emit an unreachable instruction.
5371 void EmitUnreachable(SourceLocation Loc);
5372
5373 /// Create a basic block that will call the trap intrinsic, and emit a
5374 /// conditional branch to it, for the -ftrapv checks.
5375 void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID,
5376 bool NoMerge = false, const TrapReason *TR = nullptr);
5377
5378 /// Emit a call to trap or debugtrap and attach function attribute
5379 /// "trap-func-name" if specified.
5380 llvm::CallInst *EmitTrapCall(llvm::Intrinsic::ID IntrID);
5381
5382 /// Emit a stub for the cross-DSO CFI check function.
5383 void EmitCfiCheckStub();
5384
5385 /// Emit a cross-DSO CFI failure handling function.
5386 void EmitCfiCheckFail();
5387
5388 /// Create a check for a function parameter that may potentially be
5389 /// declared as non-null.
5390 void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc,
5391 AbstractCallee AC, unsigned ParmNum);
5392
5393 void EmitNonNullArgCheck(Address Addr, QualType ArgType,
5394 SourceLocation ArgLoc, AbstractCallee AC,
5395 unsigned ParmNum);
5396
5397 /// EmitWriteback - Emit callbacks for function.
5398 void EmitWritebacks(const CallArgList &Args);
5399
5400 /// EmitCallArg - Emit a single call argument.
5401 void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType);
5402
5403 /// EmitDelegateCallArg - We are performing a delegate call; that
5404 /// is, the current function is delegating to another one. Produce
5405 /// a r-value suitable for passing the given parameter.
5406 void EmitDelegateCallArg(CallArgList &args, const VarDecl *param,
5407 SourceLocation loc);
5408
5409 /// SetFPAccuracy - Set the minimum required accuracy of the given floating
5410 /// point operation, expressed as the maximum relative error in ulp.
5411 void SetFPAccuracy(llvm::Value *Val, float Accuracy);
5412
5413 /// Set the minimum required accuracy of the given sqrt operation
5414 /// based on CodeGenOpts.
5415 void SetSqrtFPAccuracy(llvm::Value *Val);
5416
5417 /// Set the minimum required accuracy of the given sqrt operation based on
5418 /// CodeGenOpts.
5419 void SetDivFPAccuracy(llvm::Value *Val);
5420
5421 /// Set the codegen fast-math flags.
5422 void SetFastMathFlags(FPOptions FPFeatures);
5423
5424 // Truncate or extend a boolean vector to the requested number of elements.
5425 llvm::Value *emitBoolVecConversion(llvm::Value *SrcVec,
5426 unsigned NumElementsDst,
5427 const llvm::Twine &Name = "");
5428
5429 void maybeAttachRangeForLoad(llvm::LoadInst *Load, QualType Ty,
5430 SourceLocation Loc);
5431
5432private:
5433 // Emits a convergence_loop instruction for the given |BB|, with |ParentToken|
5434 // as it's parent convergence instr.
5435 llvm::ConvergenceControlInst *emitConvergenceLoopToken(llvm::BasicBlock *BB);
5436
5437 // Adds a convergence_ctrl token with |ParentToken| as parent convergence
5438 // instr to the call |Input|.
5439 llvm::CallBase *addConvergenceControlToken(llvm::CallBase *Input);
5440
5441 // Find the convergence_entry instruction |F|, or emits ones if none exists.
5442 // Returns the convergence instruction.
5443 llvm::ConvergenceControlInst *
5444 getOrEmitConvergenceEntryToken(llvm::Function *F);
5445
5446private:
5447 llvm::MDNode *getRangeForLoadFromType(QualType Ty);
5448 void EmitReturnOfRValue(RValue RV, QualType Ty);
5449
5450 void deferPlaceholderReplacement(llvm::Instruction *Old, llvm::Value *New);
5451
5452 llvm::SmallVector<std::pair<llvm::WeakTrackingVH, llvm::Value *>, 4>
5453 DeferredReplacements;
5454
5455 /// Set the address of a local variable.
5456 void setAddrOfLocalVar(const VarDecl *VD, Address Addr) {
5457 assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!");
5458 LocalDeclMap.insert(KV: {VD, Addr});
5459 }
5460
5461 /// ExpandTypeFromArgs - Reconstruct a structure of type \arg Ty
5462 /// from function arguments into \arg Dst. See ABIArgInfo::Expand.
5463 ///
5464 /// \param AI - The first function argument of the expansion.
5465 void ExpandTypeFromArgs(QualType Ty, LValue Dst,
5466 llvm::Function::arg_iterator &AI);
5467
5468 /// ExpandTypeToArgs - Expand an CallArg \arg Arg, with the LLVM type for \arg
5469 /// Ty, into individual arguments on the provided vector \arg IRCallArgs,
5470 /// starting at index \arg IRCallArgPos. See ABIArgInfo::Expand.
5471 void ExpandTypeToArgs(QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
5472 SmallVectorImpl<llvm::Value *> &IRCallArgs,
5473 unsigned &IRCallArgPos);
5474
5475 std::pair<llvm::Value *, llvm::Type *>
5476 EmitAsmInput(const TargetInfo::ConstraintInfo &Info, const Expr *InputExpr,
5477 std::string &ConstraintStr);
5478
5479 std::pair<llvm::Value *, llvm::Type *>
5480 EmitAsmInputLValue(const TargetInfo::ConstraintInfo &Info, LValue InputValue,
5481 QualType InputType, std::string &ConstraintStr,
5482 SourceLocation Loc);
5483
5484 /// Attempts to statically evaluate the object size of E. If that
5485 /// fails, emits code to figure the size of E out for us. This is
5486 /// pass_object_size aware.
5487 ///
5488 /// If EmittedExpr is non-null, this will use that instead of re-emitting E.
5489 llvm::Value *evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
5490 llvm::IntegerType *ResType,
5491 llvm::Value *EmittedE,
5492 bool IsDynamic);
5493
5494 /// Emits the size of E, as required by __builtin_object_size. This
5495 /// function is aware of pass_object_size parameters, and will act accordingly
5496 /// if E is a parameter with the pass_object_size attribute.
5497 llvm::Value *emitBuiltinObjectSize(const Expr *E, unsigned Type,
5498 llvm::IntegerType *ResType,
5499 llvm::Value *EmittedE, bool IsDynamic);
5500
5501 llvm::Value *emitCountedBySize(const Expr *E, llvm::Value *EmittedE,
5502 unsigned Type, llvm::IntegerType *ResType);
5503
5504 llvm::Value *emitCountedByMemberSize(const MemberExpr *E, const Expr *Idx,
5505 llvm::Value *EmittedE,
5506 QualType CastedArrayElementTy,
5507 unsigned Type,
5508 llvm::IntegerType *ResType);
5509
5510 llvm::Value *emitCountedByPointerSize(const ImplicitCastExpr *E,
5511 const Expr *Idx, llvm::Value *EmittedE,
5512 QualType CastedArrayElementTy,
5513 unsigned Type,
5514 llvm::IntegerType *ResType);
5515
5516 void emitZeroOrPatternForAutoVarInit(QualType type, const VarDecl &D,
5517 Address Loc);
5518
5519public:
5520 enum class EvaluationOrder {
5521 ///! No language constraints on evaluation order.
5522 Default,
5523 ///! Language semantics require left-to-right evaluation.
5524 ForceLeftToRight,
5525 ///! Language semantics require right-to-left evaluation.
5526 ForceRightToLeft
5527 };
5528
5529 // Wrapper for function prototype sources. Wraps either a FunctionProtoType or
5530 // an ObjCMethodDecl.
5531 struct PrototypeWrapper {
5532 llvm::PointerUnion<const FunctionProtoType *, const ObjCMethodDecl *> P;
5533
5534 PrototypeWrapper(const FunctionProtoType *FT) : P(FT) {}
5535 PrototypeWrapper(const ObjCMethodDecl *MD) : P(MD) {}
5536 };
5537
5538 void EmitCallArgs(CallArgList &Args, PrototypeWrapper Prototype,
5539 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
5540 AbstractCallee AC = AbstractCallee(),
5541 unsigned ParamsToSkip = 0,
5542 EvaluationOrder Order = EvaluationOrder::Default);
5543
5544 /// EmitPointerWithAlignment - Given an expression with a pointer type,
5545 /// emit the value and compute our best estimate of the alignment of the
5546 /// pointee.
5547 ///
5548 /// \param BaseInfo - If non-null, this will be initialized with
5549 /// information about the source of the alignment and the may-alias
5550 /// attribute. Note that this function will conservatively fall back on
5551 /// the type when it doesn't recognize the expression and may-alias will
5552 /// be set to false.
5553 ///
5554 /// One reasonable way to use this information is when there's a language
5555 /// guarantee that the pointer must be aligned to some stricter value, and
5556 /// we're simply trying to ensure that sufficiently obvious uses of under-
5557 /// aligned objects don't get miscompiled; for example, a placement new
5558 /// into the address of a local variable. In such a case, it's quite
5559 /// reasonable to just ignore the returned alignment when it isn't from an
5560 /// explicit source.
5561 Address
5562 EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo = nullptr,
5563 TBAAAccessInfo *TBAAInfo = nullptr,
5564 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
5565
5566 /// If \p E references a parameter with pass_object_size info or a constant
5567 /// array size modifier, emit the object size divided by the size of \p EltTy.
5568 /// Otherwise return null.
5569 llvm::Value *LoadPassedObjectSize(const Expr *E, QualType EltTy);
5570
5571 void EmitSanitizerStatReport(llvm::SanitizerStatKind SSK);
5572
5573 struct FMVResolverOption {
5574 llvm::Function *Function;
5575 llvm::SmallVector<StringRef, 8> Features;
5576 std::optional<StringRef> Architecture;
5577
5578 FMVResolverOption(llvm::Function *F, ArrayRef<StringRef> Feats,
5579 std::optional<StringRef> Arch = std::nullopt)
5580 : Function(F), Features(Feats), Architecture(Arch) {}
5581 };
5582
5583 // Emits the body of a multiversion function's resolver. Assumes that the
5584 // options are already sorted in the proper order, with the 'default' option
5585 // last (if it exists).
5586 void EmitMultiVersionResolver(llvm::Function *Resolver,
5587 ArrayRef<FMVResolverOption> Options);
5588 void EmitX86MultiVersionResolver(llvm::Function *Resolver,
5589 ArrayRef<FMVResolverOption> Options);
5590 void EmitAArch64MultiVersionResolver(llvm::Function *Resolver,
5591 ArrayRef<FMVResolverOption> Options);
5592 void EmitRISCVMultiVersionResolver(llvm::Function *Resolver,
5593 ArrayRef<FMVResolverOption> Options);
5594 void EmitPPCAIXMultiVersionResolver(llvm::Function *Resolver,
5595 ArrayRef<FMVResolverOption> Options);
5596
5597 Address EmitAddressOfPFPField(Address RecordPtr, const PFPField &Field);
5598 Address EmitAddressOfPFPField(Address RecordPtr, Address FieldPtr,
5599 const FieldDecl *Field);
5600
5601private:
5602 QualType getVarArgType(const Expr *Arg);
5603
5604 void EmitDeclMetadata();
5605
5606 BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType,
5607 const AutoVarEmission &emission);
5608
5609 void AddObjCARCExceptionMetadata(llvm::Instruction *Inst);
5610
5611 llvm::Value *GetValueForARMHint(unsigned BuiltinID);
5612 llvm::Value *EmitX86CpuIs(const CallExpr *E);
5613 llvm::Value *EmitX86CpuIs(StringRef CPUStr);
5614 llvm::Value *EmitX86CpuSupports(const CallExpr *E);
5615 llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs);
5616 llvm::Value *EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask);
5617 llvm::Value *EmitX86CpuInit();
5618 llvm::Value *FormX86ResolverCondition(const FMVResolverOption &RO);
5619 llvm::Value *EmitAArch64CpuInit();
5620 llvm::Value *FormAArch64ResolverCondition(const FMVResolverOption &RO);
5621 llvm::Value *EmitAArch64CpuSupports(const CallExpr *E);
5622 llvm::Value *EmitAArch64CpuSupports(ArrayRef<StringRef> FeatureStrs);
5623};
5624
5625inline DominatingLLVMValue::saved_type
5626DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) {
5627 if (!needsSaving(value))
5628 return saved_type(value);
5629
5630 // Otherwise, we need an alloca.
5631 auto align = CharUnits::fromQuantity(
5632 Quantity: CGF.CGM.getDataLayout().getPrefTypeAlign(Ty: value->getType()))
5633 .getAsAlign();
5634 llvm::AllocaInst *AI =
5635 CGF.CreateTempAlloca(Ty: value->getType(), Name: "cond-cleanup.save");
5636 AI->setAlignment(align);
5637 CGF.Builder.CreateAlignedStore(Val: value, Ptr: AI, Align: align);
5638
5639 return saved_type(AI, value->getType());
5640}
5641
5642inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF,
5643 saved_type value) {
5644 // If the value says it wasn't saved, trust that it's still dominating.
5645 if (!value.isSaved())
5646 return value.Value;
5647
5648 // Otherwise, it should be an alloca instruction, as set up in save().
5649 auto Alloca = cast<llvm::AllocaInst>(Val: value.Value);
5650 return CGF.Builder.CreateAlignedLoad(Ty: value.Type, Ptr: Alloca, Align: Alloca->getAlign());
5651}
5652
5653} // end namespace CodeGen
5654
5655// Map the LangOption for floating point exception behavior into
5656// the corresponding enum in the IR.
5657llvm::fp::ExceptionBehavior
5658ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind);
5659} // end namespace clang
5660
5661#endif
5662