1//===-- CodeGenFunction.h - Per-Function state for LLVM CodeGen -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the internal per-function state used for llvm translation.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
14#define LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
15
16#include "CGBuilder.h"
17#include "CGLoopInfo.h"
18#include "CGValue.h"
19#include "CodeGenModule.h"
20#include "EHScopeStack.h"
21#include "SanitizerHandler.h"
22#include "VarBypassDetector.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/CharUnits.h"
25#include "clang/AST/CurrentSourceLocExprScope.h"
26#include "clang/AST/ExprCXX.h"
27#include "clang/AST/ExprObjC.h"
28#include "clang/AST/ExprOpenMP.h"
29#include "clang/AST/StmtOpenACC.h"
30#include "clang/AST/StmtOpenMP.h"
31#include "clang/AST/StmtSYCL.h"
32#include "clang/AST/Type.h"
33#include "clang/Basic/ABI.h"
34#include "clang/Basic/CapturedStmt.h"
35#include "clang/Basic/CodeGenOptions.h"
36#include "clang/Basic/OpenMPKinds.h"
37#include "clang/Basic/TargetInfo.h"
38#include "clang/CodeGenUtils/CallUtils.h"
39#include "clang/CodeGenUtils/TargetUtils.h"
40#include "llvm/ADT/ArrayRef.h"
41#include "llvm/ADT/DenseMap.h"
42#include "llvm/ADT/MapVector.h"
43#include "llvm/ADT/SmallVector.h"
44#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
45#include "llvm/IR/Instructions.h"
46#include "llvm/IR/ValueHandle.h"
47#include "llvm/Support/Debug.h"
48#include "llvm/Transforms/Utils/SanitizerStats.h"
49#include <optional>
50
51namespace llvm {
52class BasicBlock;
53class ConvergenceControlInst;
54class LLVMContext;
55class MDNode;
56class SwitchInst;
57class Twine;
58class Value;
59class CanonicalLoopInfo;
60} // namespace llvm
61
62namespace clang {
63class ASTContext;
64class AsmConstraintsInfo;
65class CXXDestructorDecl;
66class CXXForRangeStmt;
67class CXXTryStmt;
68class Decl;
69class LabelDecl;
70class FunctionDecl;
71class FunctionProtoType;
72class LabelStmt;
73class ObjCContainerDecl;
74class ObjCInterfaceDecl;
75class ObjCIvarDecl;
76class ObjCMethodDecl;
77class ObjCImplementationDecl;
78class ObjCPropertyImplDecl;
79class TargetInfo;
80class VarDecl;
81class ObjCForCollectionStmt;
82class ObjCAtTryStmt;
83class ObjCAtThrowStmt;
84class ObjCAtSynchronizedStmt;
85class ObjCAutoreleasePoolStmt;
86class OMPUseDevicePtrClause;
87class OMPUseDeviceAddrClause;
88class SVETypeFlags;
89class OMPExecutableDirective;
90
91namespace analyze_os_log {
92class OSLogBufferLayout;
93}
94
95namespace CodeGen {
96class CodeGenTypes;
97class CodeGenPGO;
98class CGCallee;
99class CGFunctionInfo;
100class CGBlockInfo;
101class CGCXXABI;
102class BlockByrefHelpers;
103class BlockByrefInfo;
104class BlockFieldFlags;
105class RegionCodeGenTy;
106class TargetCodeGenInfo;
107struct OMPTaskDataTy;
108struct CGCoroData;
109
110// clang-format off
111/// The kind of evaluation to perform on values of a particular
112/// type. Basically, is the code in CGExprScalar, CGExprComplex, or
113/// CGExprAgg?
114///
115/// TODO: should vectors maybe be split out into their own thing?
116enum TypeEvaluationKind {
117 TEK_Scalar,
118 TEK_Complex,
119 TEK_Aggregate
120};
121// clang-format on
122
123/// Helper class with most of the code for saving a value for a
124/// conditional expression cleanup.
125struct DominatingLLVMValue {
126 struct saved_type {
127 llvm::Value *Value; // Original value if not saved, alloca if saved
128 llvm::Type *Type; // nullptr if not saved, element type if saved
129
130 saved_type() : Value(nullptr), Type(nullptr) {}
131 saved_type(llvm::Value *V) : Value(V), Type(nullptr) {}
132 saved_type(llvm::AllocaInst *Alloca, llvm::Type *Ty)
133 : Value(Alloca), Type(Ty) {}
134
135 bool isSaved() const { return Type != nullptr; }
136 };
137
138 /// Answer whether the given value needs extra work to be saved.
139 static bool needsSaving(llvm::Value *value) {
140 if (!value)
141 return false;
142
143 // If it's not an instruction, we don't need to save.
144 if (!isa<llvm::Instruction>(Val: value))
145 return false;
146
147 // If it's an instruction in the entry block, we don't need to save.
148 llvm::BasicBlock *block = cast<llvm::Instruction>(Val: value)->getParent();
149 return (block != &block->getParent()->getEntryBlock());
150 }
151
152 static saved_type save(CodeGenFunction &CGF, llvm::Value *value);
153 static llvm::Value *restore(CodeGenFunction &CGF, saved_type value);
154};
155
156/// A partial specialization of DominatingValue for llvm::Values that
157/// might be llvm::Instructions.
158template <class T> struct DominatingPointer<T, true> : DominatingLLVMValue {
159 typedef T *type;
160 static type restore(CodeGenFunction &CGF, saved_type value) {
161 return static_cast<T *>(DominatingLLVMValue::restore(CGF, value));
162 }
163};
164
165/// A specialization of DominatingValue for Address.
166template <> struct DominatingValue<Address> {
167 typedef Address type;
168
169 struct saved_type {
170 DominatingLLVMValue::saved_type BasePtr;
171 llvm::Type *ElementType;
172 CharUnits Alignment;
173 DominatingLLVMValue::saved_type Offset;
174 llvm::PointerType *EffectiveType;
175 };
176
177 static bool needsSaving(type value) {
178 if (DominatingLLVMValue::needsSaving(value: value.getBasePointer()) ||
179 DominatingLLVMValue::needsSaving(value: value.getOffset()))
180 return true;
181 return false;
182 }
183 static saved_type save(CodeGenFunction &CGF, type value) {
184 return {.BasePtr: DominatingLLVMValue::save(CGF, value: value.getBasePointer()),
185 .ElementType: value.getElementType(), .Alignment: value.getAlignment(),
186 .Offset: DominatingLLVMValue::save(CGF, value: value.getOffset()), .EffectiveType: value.getType()};
187 }
188 static type restore(CodeGenFunction &CGF, saved_type value) {
189 return Address(DominatingLLVMValue::restore(CGF, value: value.BasePtr),
190 value.ElementType, value.Alignment, CGPointerAuthInfo(),
191 DominatingLLVMValue::restore(CGF, value: value.Offset));
192 }
193};
194
195/// A specialization of DominatingValue for RValue.
196template <> struct DominatingValue<RValue> {
197 typedef RValue type;
198 class saved_type {
199 enum Kind {
200 ScalarLiteral,
201 ScalarAddress,
202 AggregateLiteral,
203 AggregateAddress,
204 ComplexAddress
205 };
206 union {
207 struct {
208 DominatingLLVMValue::saved_type first, second;
209 } Vals;
210 DominatingValue<Address>::saved_type AggregateAddr;
211 };
212 LLVM_PREFERRED_TYPE(Kind)
213 unsigned K : 3;
214
215 saved_type(DominatingLLVMValue::saved_type Val1, unsigned K)
216 : Vals{.first: Val1, .second: DominatingLLVMValue::saved_type()}, K(K) {}
217
218 saved_type(DominatingLLVMValue::saved_type Val1,
219 DominatingLLVMValue::saved_type Val2)
220 : Vals{.first: Val1, .second: Val2}, K(ComplexAddress) {}
221
222 saved_type(DominatingValue<Address>::saved_type AggregateAddr, unsigned K)
223 : AggregateAddr(AggregateAddr), K(K) {}
224
225 public:
226 static bool needsSaving(RValue value);
227 static saved_type save(CodeGenFunction &CGF, RValue value);
228 RValue restore(CodeGenFunction &CGF);
229
230 // implementations in CGCleanup.cpp
231 };
232
233 static bool needsSaving(type value) { return saved_type::needsSaving(value); }
234 static saved_type save(CodeGenFunction &CGF, type value) {
235 return saved_type::save(CGF, value);
236 }
237 static type restore(CodeGenFunction &CGF, saved_type value) {
238 return value.restore(CGF);
239 }
240};
241
242/// A scoped helper to set the current source atom group for
243/// CGDebugInfo::addInstToCurrentSourceAtom. A source atom is a source construct
244/// that is "interesting" for debug stepping purposes. We use an atom group
245/// number to track the instruction(s) that implement the functionality for the
246/// atom, plus backup instructions/source locations.
247class ApplyAtomGroup {
248 uint64_t OriginalAtom = 0;
249 CGDebugInfo *DI = nullptr;
250
251 ApplyAtomGroup(const ApplyAtomGroup &) = delete;
252 void operator=(const ApplyAtomGroup &) = delete;
253
254public:
255 ApplyAtomGroup(CGDebugInfo *DI);
256 ~ApplyAtomGroup();
257};
258
259/// CodeGenFunction - This class organizes the per-function state that is used
260/// while generating LLVM code.
261class CodeGenFunction : public CodeGenTypeCache {
262 CodeGenFunction(const CodeGenFunction &) = delete;
263 void operator=(const CodeGenFunction &) = delete;
264
265 friend class CGCXXABI;
266 friend class clang::AsmConstraintsInfo;
267
268public:
269 /// A jump destination is an abstract label, branching to which may
270 /// require a jump out through normal cleanups.
271 struct JumpDest {
272 JumpDest() : Block(nullptr), Index(0) {}
273 JumpDest(llvm::BasicBlock *Block, EHScopeStack::stable_iterator Depth,
274 unsigned Index)
275 : Block(Block), ScopeDepth(Depth), Index(Index) {}
276
277 bool isValid() const { return Block != nullptr; }
278 llvm::BasicBlock *getBlock() const { return Block; }
279 EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; }
280 unsigned getDestIndex() const { return Index; }
281
282 // This should be used cautiously.
283 void setScopeDepth(EHScopeStack::stable_iterator depth) {
284 ScopeDepth = depth;
285 }
286
287 private:
288 llvm::BasicBlock *Block;
289 EHScopeStack::stable_iterator ScopeDepth;
290 unsigned Index;
291 };
292
293 CodeGenModule &CGM; // Per-module state.
294 const TargetInfo &Target;
295
296 // For EH/SEH outlined funclets, this field points to parent's CGF
297 CodeGenFunction *ParentCGF = nullptr;
298
299 typedef std::pair<llvm::Value *, llvm::Value *> ComplexPairTy;
300 LoopInfoStack LoopStack;
301 CGBuilderTy Builder;
302
303 // Stores variables for which we can't generate correct lifetime markers
304 // because of jumps.
305 VarBypassDetector Bypasses;
306
307 // Addresses of bypassed variables, for re-emitting their
308 // trivial-auto-var-init at a jump that re-enters their scope.
309 llvm::SmallDenseMap<const VarDecl *, Address, 4> BypassedVarInits;
310
311 // Jumps, like gotos or switches, that may bypass a declaration that has not
312 // been emitted yet. EmitAutoVarAlloca patches the init in before the jump
313 // once the alloca exists.
314 struct BypassingForwardJump {
315 llvm::AssertingVH<llvm::BasicBlock> Block;
316 const Stmt *Source;
317 };
318 llvm::SmallVector<BypassingForwardJump, 4> BypassingForwardJumps;
319
320 /// List of recently emitted OMPCanonicalLoops.
321 ///
322 /// Since OMPCanonicalLoops are nested inside other statements (in particular
323 /// CapturedStmt generated by OMPExecutableDirective and non-perfectly nested
324 /// loops), we cannot directly call OMPEmitOMPCanonicalLoop and receive its
325 /// llvm::CanonicalLoopInfo. Instead, we call EmitStmt and any
326 /// OMPEmitOMPCanonicalLoop called by it will add its CanonicalLoopInfo to
327 /// this stack when done. Entering a new loop requires clearing this list; it
328 /// either means we start parsing a new loop nest (in which case the previous
329 /// loop nest goes out of scope) or a second loop in the same level in which
330 /// case it would be ambiguous into which of the two (or more) loops the loop
331 /// nest would extend.
332 SmallVector<llvm::CanonicalLoopInfo *, 4> OMPLoopNestStack;
333
334 /// Stack to track the controlled convergence tokens.
335 SmallVector<llvm::ConvergenceControlInst *, 4> ConvergenceTokenStack;
336
337 /// Number of nested loop to be consumed by the last surrounding
338 /// loop-associated directive.
339 int ExpectedOMPLoopDepth = 0;
340
341 // CodeGen lambda for loops and support for ordered clause
342 typedef llvm::function_ref<void(CodeGenFunction &, const OMPLoopDirective &,
343 JumpDest)>
344 CodeGenLoopTy;
345 typedef llvm::function_ref<void(CodeGenFunction &, SourceLocation,
346 const unsigned, const bool)>
347 CodeGenOrderedTy;
348
349 // Codegen lambda for loop bounds in worksharing loop constructs
350 typedef llvm::function_ref<std::pair<LValue, LValue>(
351 CodeGenFunction &, const OMPExecutableDirective &S)>
352 CodeGenLoopBoundsTy;
353
354 // Codegen lambda for loop bounds in dispatch-based loop implementation
355 typedef llvm::function_ref<std::pair<llvm::Value *, llvm::Value *>(
356 CodeGenFunction &, const OMPExecutableDirective &S, Address LB,
357 Address UB)>
358 CodeGenDispatchBoundsTy;
359
360 /// CGBuilder insert helper. This function is called after an
361 /// instruction is created using Builder.
362 void InsertHelper(llvm::Instruction *I, const llvm::Twine &Name,
363 llvm::BasicBlock::iterator InsertPt) const;
364
365 /// CurFuncDecl - Holds the Decl for the current outermost
366 /// non-closure context.
367 const Decl *CurFuncDecl = nullptr;
368 /// CurCodeDecl - This is the inner-most code context, which includes blocks.
369 const Decl *CurCodeDecl = nullptr;
370 const CGFunctionInfo *CurFnInfo = nullptr;
371 QualType FnRetTy;
372 llvm::Function *CurFn = nullptr;
373
374 /// If a cast expression is being visited, this holds the current cast's
375 /// expression.
376 const CastExpr *CurCast = nullptr;
377
378 /// Allocation calls of the current function body whose result is used as
379 /// placement-new storage, mapped to the constructed type; collected lazily
380 /// for -fsanitize=alloc-token type inference.
381 std::optional<llvm::DenseMap<const CallExpr *, QualType>>
382 AllocTokenPlacementTypes;
383
384 /// Save Parameter Decl for coroutine.
385 llvm::SmallVector<const ParmVarDecl *, 4> FnArgs;
386
387 // Holds coroutine data if the current function is a coroutine. We use a
388 // wrapper to manage its lifetime, so that we don't have to define CGCoroData
389 // in this header.
390 struct CGCoroInfo {
391 std::unique_ptr<CGCoroData> Data;
392 bool InSuspendBlock = false;
393 CGCoroInfo();
394 ~CGCoroInfo();
395 };
396 CGCoroInfo CurCoro;
397
398 bool isCoroutine() const { return CurCoro.Data != nullptr; }
399
400 bool inSuspendBlock() const {
401 return isCoroutine() && CurCoro.InSuspendBlock;
402 }
403
404 // Holds FramePtr for await_suspend wrapper generation,
405 // so that __builtin_coro_frame call can be lowered
406 // directly to value of its second argument
407 struct AwaitSuspendWrapperInfo {
408 llvm::Value *FramePtr = nullptr;
409 };
410 AwaitSuspendWrapperInfo CurAwaitSuspendWrapper;
411
412 // Generates wrapper function for `llvm.coro.await.suspend.*` intrinisics.
413 // It encapsulates SuspendExpr in a function, to separate it's body
414 // from the main coroutine to avoid miscompilations. Intrinisic
415 // is lowered to this function call in CoroSplit pass
416 // Function signature is:
417 // <type> __await_suspend_wrapper_<name>(ptr %awaiter, ptr %hdl)
418 // where type is one of (void, i1, ptr)
419 llvm::Function *generateAwaitSuspendWrapper(Twine const &CoroName,
420 Twine const &SuspendPointName,
421 CoroutineSuspendExpr const &S);
422
423 /// CurGD - The GlobalDecl for the current function being compiled.
424 GlobalDecl CurGD;
425
426 /// PrologueCleanupDepth - The cleanup depth enclosing all the
427 /// cleanups associated with the parameters.
428 EHScopeStack::stable_iterator PrologueCleanupDepth;
429
430 /// ReturnBlock - Unified return block.
431 JumpDest ReturnBlock;
432
433 /// ReturnValue - The temporary alloca to hold the return
434 /// value. This is invalid iff the function has no return value.
435 Address ReturnValue = Address::invalid();
436
437 /// ReturnValuePointer - The temporary alloca to hold a pointer to sret.
438 /// This is invalid if sret is not in use.
439 Address ReturnValuePointer = Address::invalid();
440
441 /// If a return statement is being visited, this holds the return statment's
442 /// result expression.
443 const Expr *RetExpr = nullptr;
444
445 /// Return true if a label was seen in the current scope.
446 bool hasLabelBeenSeenInCurrentScope() const {
447 if (CurLexicalScope)
448 return CurLexicalScope->hasLabels();
449 return !LabelMap.empty();
450 }
451
452 /// AllocaInsertPoint - This is an instruction in the entry block before which
453 /// we prefer to insert allocas.
454 llvm::AssertingVH<llvm::Instruction> AllocaInsertPt;
455
456private:
457 /// PostAllocaInsertPt - This is a place in the prologue where code can be
458 /// inserted that will be dominated by all the static allocas. This helps
459 /// achieve two things:
460 /// 1. Contiguity of all static allocas (within the prologue) is maintained.
461 /// 2. All other prologue code (which are dominated by static allocas) do
462 /// appear in the source order immediately after all static allocas.
463 ///
464 /// PostAllocaInsertPt will be lazily created when it is *really* required.
465 llvm::AssertingVH<llvm::Instruction> PostAllocaInsertPt = nullptr;
466
467public:
468 /// Return PostAllocaInsertPt. If it is not yet created, then insert it
469 /// immediately after AllocaInsertPt.
470 llvm::Instruction *getPostAllocaInsertPoint() {
471 if (!PostAllocaInsertPt) {
472 assert(AllocaInsertPt &&
473 "Expected static alloca insertion point at function prologue");
474 assert(AllocaInsertPt->getParent()->isEntryBlock() &&
475 "EBB should be entry block of the current code gen function");
476 PostAllocaInsertPt = AllocaInsertPt->clone();
477 PostAllocaInsertPt->setName("postallocapt");
478 PostAllocaInsertPt->insertAfter(InsertPos: AllocaInsertPt->getIterator());
479 }
480
481 return PostAllocaInsertPt;
482 }
483
484 // Try to preserve the source's name to make IR more readable.
485 llvm::Value *performAddrSpaceCast(llvm::Value *Src, llvm::Type *DestTy) {
486 return Builder.CreateAddrSpaceCast(
487 V: Src, DestTy, Name: Src->hasName() ? Src->getName() + ".ascast" : "");
488 }
489
490 /// API for captured statement code generation.
491 class CGCapturedStmtInfo {
492 public:
493 explicit CGCapturedStmtInfo(CapturedRegionKind K = CR_Default)
494 : Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {}
495 explicit CGCapturedStmtInfo(const CapturedStmt &S,
496 CapturedRegionKind K = CR_Default)
497 : Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {
498
499 RecordDecl::field_iterator Field =
500 S.getCapturedRecordDecl()->field_begin();
501 for (CapturedStmt::const_capture_iterator I = S.capture_begin(),
502 E = S.capture_end();
503 I != E; ++I, ++Field) {
504 if (I->capturesThis())
505 CXXThisFieldDecl = *Field;
506 else if (I->capturesVariable())
507 CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
508 else if (I->capturesVariableByCopy())
509 CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
510 }
511 }
512
513 virtual ~CGCapturedStmtInfo();
514
515 CapturedRegionKind getKind() const { return Kind; }
516
517 virtual void setContextValue(llvm::Value *V) { ThisValue = V; }
518 // Retrieve the value of the context parameter.
519 virtual llvm::Value *getContextValue() const { return ThisValue; }
520
521 /// Lookup the captured field decl for a variable.
522 virtual const FieldDecl *lookup(const VarDecl *VD) const {
523 return CaptureFields.lookup(Val: VD->getCanonicalDecl());
524 }
525
526 bool isCXXThisExprCaptured() const { return getThisFieldDecl() != nullptr; }
527 virtual FieldDecl *getThisFieldDecl() const { return CXXThisFieldDecl; }
528
529 static bool classof(const CGCapturedStmtInfo *) { return true; }
530
531 /// Emit the captured statement body.
532 virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) {
533 CGF.incrementProfileCounter(S);
534 CGF.EmitStmt(S);
535 }
536
537 /// Get the name of the capture helper.
538 virtual StringRef getHelperName() const { return "__captured_stmt"; }
539
540 /// Get the CaptureFields
541 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> getCaptureFields() {
542 return CaptureFields;
543 }
544
545 private:
546 /// The kind of captured statement being generated.
547 CapturedRegionKind Kind;
548
549 /// Keep the map between VarDecl and FieldDecl.
550 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields;
551
552 /// The base address of the captured record, passed in as the first
553 /// argument of the parallel region function.
554 llvm::Value *ThisValue;
555
556 /// Captured 'this' type.
557 FieldDecl *CXXThisFieldDecl;
558 };
559 CGCapturedStmtInfo *CapturedStmtInfo = nullptr;
560
561 /// RAII for correct setting/restoring of CapturedStmtInfo.
562 class CGCapturedStmtRAII {
563 private:
564 CodeGenFunction &CGF;
565 CGCapturedStmtInfo *PrevCapturedStmtInfo;
566
567 public:
568 CGCapturedStmtRAII(CodeGenFunction &CGF,
569 CGCapturedStmtInfo *NewCapturedStmtInfo)
570 : CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo) {
571 CGF.CapturedStmtInfo = NewCapturedStmtInfo;
572 }
573 ~CGCapturedStmtRAII() { CGF.CapturedStmtInfo = PrevCapturedStmtInfo; }
574 };
575
576 /// An abstract representation of regular/ObjC call/message targets.
577 class AbstractCallee {
578 /// The function declaration of the callee.
579 const Decl *CalleeDecl;
580
581 public:
582 AbstractCallee() : CalleeDecl(nullptr) {}
583 AbstractCallee(const FunctionDecl *FD) : CalleeDecl(FD) {}
584 AbstractCallee(const ObjCMethodDecl *OMD) : CalleeDecl(OMD) {}
585 bool hasFunctionDecl() const {
586 return isa_and_nonnull<FunctionDecl>(Val: CalleeDecl);
587 }
588 const Decl *getDecl() const { return CalleeDecl; }
589 unsigned getNumParams() const {
590 if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
591 return FD->getNumParams();
592 return cast<ObjCMethodDecl>(Val: CalleeDecl)->param_size();
593 }
594 const ParmVarDecl *getParamDecl(unsigned I) const {
595 if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
596 return FD->getParamDecl(i: I);
597 return *(cast<ObjCMethodDecl>(Val: CalleeDecl)->param_begin() + I);
598 }
599 };
600
601 /// Sanitizers enabled for this function.
602 SanitizerSet SanOpts;
603
604 /// True if CodeGen currently emits code implementing sanitizer checks.
605 bool IsSanitizerScope = false;
606
607 /// RAII object to set/unset CodeGenFunction::IsSanitizerScope.
608 class SanitizerScope {
609 CodeGenFunction *CGF;
610
611 public:
612 SanitizerScope(CodeGenFunction *CGF);
613 ~SanitizerScope();
614 };
615
616 /// In C++, whether we are code generating a thunk. This controls whether we
617 /// should emit cleanups.
618 bool CurFuncIsThunk = false;
619
620 /// In ARC, whether we should autorelease the return value.
621 bool AutoreleaseResult = false;
622
623 /// Whether we processed a Microsoft-style asm block during CodeGen. These can
624 /// potentially set the return value.
625 bool SawAsmBlock = false;
626
627 GlobalDecl CurSEHParent;
628
629 /// True if the current function is an outlined SEH helper. This can be a
630 /// finally block or filter expression.
631 bool IsOutlinedSEHHelper = false;
632
633 /// True if CodeGen currently emits code inside presereved access index
634 /// region.
635 bool IsInPreservedAIRegion = false;
636
637 /// True if CodeGen currently emits the argument of
638 /// __builtin_preserve_field_info.
639 bool IsInPreserveFieldInfo = false;
640
641 /// True if the current statement has nomerge attribute.
642 bool InNoMergeAttributedStmt = false;
643
644 /// True if the current statement has noinline attribute.
645 bool InNoInlineAttributedStmt = false;
646
647 /// True if the current statement has always_inline attribute.
648 bool InAlwaysInlineAttributedStmt = false;
649
650 /// True if the current statement has noconvergent attribute.
651 bool InNoConvergentAttributedStmt = false;
652
653 /// The mode string from the amdgpu_av attribute on the current statement,
654 /// or empty if the attribute is not present.
655 StringRef AMDGPUAvailableVisibleMode;
656
657 /// HLSL Branch attribute.
658 HLSLControlFlowHintAttr::Spelling HLSLControlFlowAttr =
659 HLSLControlFlowHintAttr::SpellingNotCalculated;
660
661 // The CallExpr within the current statement that the musttail attribute
662 // applies to. nullptr if there is no 'musttail' on the current statement.
663 const CallExpr *MustTailCall = nullptr;
664
665 /// Records the l-value that ScalarExprEmitter loads from when it emits a
666 /// specific expression. EmitArraySubscriptExpr sets this while emitting the
667 /// base of a 'counted_by' pointer subscript, so that the bounds check can
668 /// locate the count from the pointer field's address without re-emitting
669 /// (and so re-evaluating) the base. Capturing the l-value, instead of
670 /// emitting the load separately, keeps everything else the scalar load path
671 /// does (debug locations, 'align_value' assumptions, pseudo variables for
672 /// -fdebug-info-for-profiling, etc.).
673 struct LoadedLValueCapture {
674 /// The expression whose l-value is recorded.
675 const Expr *E;
676 /// The recorded l-value, set if \c E was loaded by ScalarExprEmitter.
677 std::optional<LValue> LV;
678 };
679 LoadedLValueCapture *CurLoadedLValueCapture = nullptr;
680
681 /// Returns true if a function must make progress, which means the
682 /// mustprogress attribute can be added.
683 bool checkIfFunctionMustProgress() {
684 if (CGM.getCodeGenOpts().getFiniteLoops() ==
685 CodeGenOptions::FiniteLoopsKind::Never)
686 return false;
687
688 // C++11 and later guarantees that a thread eventually will do one of the
689 // following (C++11 [intro.multithread]p24 and C++17 [intro.progress]p1):
690 // - terminate,
691 // - make a call to a library I/O function,
692 // - perform an access through a volatile glvalue, or
693 // - perform a synchronization operation or an atomic operation.
694 //
695 // Hence each function is 'mustprogress' in C++11 or later.
696 return getLangOpts().CPlusPlus11;
697 }
698
699 /// Returns true if a loop must make progress, which means the mustprogress
700 /// attribute can be added. \p HasConstantCond indicates whether the branch
701 /// condition is a known constant.
702 bool checkIfLoopMustProgress(const Expr *, bool HasEmptyBody);
703
704 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
705 llvm::Value *BlockPointer = nullptr;
706
707 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
708 FieldDecl *LambdaThisCaptureField = nullptr;
709
710 /// A mapping from NRVO variables to the flags used to indicate
711 /// when the NRVO has been applied to this variable.
712 llvm::DenseMap<const VarDecl *, llvm::Value *> NRVOFlags;
713
714 EHScopeStack EHStack;
715 llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack;
716
717 // A stack of cleanups which were added to EHStack but have to be deactivated
718 // later before being popped or emitted. These are usually deactivated on
719 // exiting a `CleanupDeactivationScope` scope. For instance, after a
720 // full-expr.
721 //
722 // These are specially useful for correctly emitting cleanups while
723 // encountering branches out of expression (through stmt-expr or coroutine
724 // suspensions).
725 struct DeferredDeactivateCleanup {
726 EHScopeStack::stable_iterator Cleanup;
727 llvm::Instruction *DominatingIP;
728 };
729 llvm::SmallVector<DeferredDeactivateCleanup> DeferredDeactivationCleanupStack;
730
731 // Enters a new scope for capturing cleanups which are deferred to be
732 // deactivated, all of which will be deactivated once the scope is exited.
733 struct CleanupDeactivationScope {
734 CodeGenFunction &CGF;
735 size_t OldDeactivateCleanupStackSize;
736 bool Deactivated;
737 CleanupDeactivationScope(CodeGenFunction &CGF)
738 : CGF(CGF), OldDeactivateCleanupStackSize(
739 CGF.DeferredDeactivationCleanupStack.size()),
740 Deactivated(false) {}
741
742 void ForceDeactivate() {
743 assert(!Deactivated && "Deactivating already deactivated scope");
744 auto &Stack = CGF.DeferredDeactivationCleanupStack;
745 for (size_t I = Stack.size(); I > OldDeactivateCleanupStackSize; I--) {
746 CGF.DeactivateCleanupBlock(Cleanup: Stack[I - 1].Cleanup,
747 DominatingIP: Stack[I - 1].DominatingIP);
748 Stack[I - 1].DominatingIP->eraseFromParent();
749 }
750 Stack.resize(N: OldDeactivateCleanupStackSize);
751 Deactivated = true;
752 }
753
754 ~CleanupDeactivationScope() {
755 if (Deactivated)
756 return;
757 ForceDeactivate();
758 }
759 };
760
761 llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack;
762
763 llvm::Instruction *CurrentFuncletPad = nullptr;
764
765 class CallLifetimeEnd final : public EHScopeStack::Cleanup {
766 llvm::Value *Addr;
767
768 public:
769 CallLifetimeEnd(RawAddress addr) : Addr(addr.getPointer()) {}
770
771 void Emit(CodeGenFunction &CGF, Flags flags) override {
772 CGF.EmitLifetimeEnd(Addr);
773 }
774 };
775
776 // We are using objects of this 'cleanup' class to emit fake.use calls
777 // for -fextend-variable-liveness. They are placed at the end of a variable's
778 // scope analogous to lifetime markers.
779 class FakeUse final : public EHScopeStack::Cleanup {
780 Address Addr;
781
782 public:
783 FakeUse(Address addr) : Addr(addr) {}
784
785 void Emit(CodeGenFunction &CGF, Flags flags) override {
786 CGF.EmitFakeUse(Addr);
787 }
788 };
789
790 /// Header for data within LifetimeExtendedCleanupStack.
791 struct alignas(uint64_t) LifetimeExtendedCleanupHeader {
792 /// The size of the following cleanup object.
793 unsigned Size;
794 /// The kind of cleanup to push.
795 LLVM_PREFERRED_TYPE(CleanupKind)
796 unsigned Kind : 31;
797 /// Whether this is a conditional cleanup.
798 LLVM_PREFERRED_TYPE(bool)
799 unsigned IsConditional : 1;
800
801 size_t getSize() const { return Size; }
802 CleanupKind getKind() const { return (CleanupKind)Kind; }
803 bool isConditional() const { return IsConditional; }
804 };
805
806 /// i32s containing the indexes of the cleanup destinations.
807 RawAddress NormalCleanupDest = RawAddress::invalid();
808
809 unsigned NextCleanupDestIndex = 1;
810
811 /// EHResumeBlock - Unified block containing a call to llvm.eh.resume.
812 llvm::BasicBlock *EHResumeBlock = nullptr;
813
814 /// The exception slot. All landing pads write the current exception pointer
815 /// into this alloca.
816 llvm::Value *ExceptionSlot = nullptr;
817
818 /// The selector slot. Under the MandatoryCleanup model, all landing pads
819 /// write the current selector value into this alloca.
820 llvm::AllocaInst *EHSelectorSlot = nullptr;
821
822 /// A stack of exception code slots. Entering an __except block pushes a slot
823 /// on the stack and leaving pops one. The __exception_code() intrinsic loads
824 /// a value from the top of the stack.
825 SmallVector<Address, 1> SEHCodeSlotStack;
826
827 /// Value returned by __exception_info intrinsic.
828 llvm::Value *SEHInfo = nullptr;
829
830 /// Emits a landing pad for the current EH stack.
831 llvm::BasicBlock *EmitLandingPad();
832
833 llvm::BasicBlock *getInvokeDestImpl();
834
835 /// Parent loop-based directive for scan directive.
836 const OMPExecutableDirective *OMPParentLoopDirectiveForScan = nullptr;
837 llvm::BasicBlock *OMPBeforeScanBlock = nullptr;
838 llvm::BasicBlock *OMPAfterScanBlock = nullptr;
839 llvm::BasicBlock *OMPScanExitBlock = nullptr;
840 llvm::BasicBlock *OMPScanDispatch = nullptr;
841 bool OMPFirstScanLoop = false;
842
843 /// Manages parent directive for scan directives.
844 class ParentLoopDirectiveForScanRegion {
845 CodeGenFunction &CGF;
846 const OMPExecutableDirective *ParentLoopDirectiveForScan;
847
848 public:
849 ParentLoopDirectiveForScanRegion(
850 CodeGenFunction &CGF,
851 const OMPExecutableDirective &ParentLoopDirectiveForScan)
852 : CGF(CGF),
853 ParentLoopDirectiveForScan(CGF.OMPParentLoopDirectiveForScan) {
854 CGF.OMPParentLoopDirectiveForScan = &ParentLoopDirectiveForScan;
855 }
856 ~ParentLoopDirectiveForScanRegion() {
857 CGF.OMPParentLoopDirectiveForScan = ParentLoopDirectiveForScan;
858 }
859 };
860
861 template <class T>
862 typename DominatingValue<T>::saved_type saveValueInCond(T value) {
863 return DominatingValue<T>::save(*this, value);
864 }
865
866 class CGFPOptionsRAII {
867 public:
868 CGFPOptionsRAII(CodeGenFunction &CGF, FPOptions FPFeatures);
869 CGFPOptionsRAII(CodeGenFunction &CGF, const Expr *E);
870 ~CGFPOptionsRAII();
871
872 private:
873 void ConstructorHelper(FPOptions FPFeatures);
874 CodeGenFunction &CGF;
875 FPOptions OldFPFeatures;
876 llvm::fp::ExceptionBehavior OldExcept;
877 llvm::RoundingMode OldRounding;
878 std::optional<CGBuilderTy::FastMathFlagGuard> FMFGuard;
879 };
880 FPOptions CurFPFeatures;
881
882 class CGAtomicOptionsRAII {
883 public:
884 CGAtomicOptionsRAII(CodeGenModule &CGM_, AtomicOptions AO)
885 : CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {
886 CGM.setAtomicOpts(AO);
887 }
888 CGAtomicOptionsRAII(CodeGenModule &CGM_, const AtomicAttr *AA)
889 : CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {
890 if (!AA)
891 return;
892 AtomicOptions AO = SavedAtomicOpts;
893 for (auto Option : AA->atomicOptions()) {
894 switch (Option) {
895 case AtomicAttr::remote_memory:
896 AO.remote_memory = true;
897 break;
898 case AtomicAttr::no_remote_memory:
899 AO.remote_memory = false;
900 break;
901 case AtomicAttr::fine_grained_memory:
902 AO.fine_grained_memory = true;
903 break;
904 case AtomicAttr::no_fine_grained_memory:
905 AO.fine_grained_memory = false;
906 break;
907 case AtomicAttr::ignore_denormal_mode:
908 AO.ignore_denormal_mode = true;
909 break;
910 case AtomicAttr::no_ignore_denormal_mode:
911 AO.ignore_denormal_mode = false;
912 break;
913 }
914 }
915 CGM.setAtomicOpts(AO);
916 }
917
918 CGAtomicOptionsRAII(const CGAtomicOptionsRAII &) = delete;
919 CGAtomicOptionsRAII &operator=(const CGAtomicOptionsRAII &) = delete;
920 ~CGAtomicOptionsRAII() { CGM.setAtomicOpts(SavedAtomicOpts); }
921
922 private:
923 CodeGenModule &CGM;
924 AtomicOptions SavedAtomicOpts;
925 };
926
927public:
928 /// ObjCEHValueStack - Stack of Objective-C exception values, used for
929 /// rethrows.
930 SmallVector<llvm::Value *, 8> ObjCEHValueStack;
931
932 /// A class controlling the emission of a finally block.
933 class FinallyInfo {
934 /// Where the catchall's edge through the cleanup should go.
935 JumpDest RethrowDest;
936
937 /// A function to call to enter the catch.
938 llvm::FunctionCallee BeginCatchFn;
939
940 /// An i1 variable indicating whether or not the @finally is
941 /// running for an exception.
942 llvm::AllocaInst *ForEHVar = nullptr;
943
944 /// An i8* variable into which the exception pointer to rethrow
945 /// has been saved.
946 llvm::AllocaInst *SavedExnVar = nullptr;
947
948 public:
949 void enter(CodeGenFunction &CGF, const Stmt *Finally,
950 llvm::FunctionCallee beginCatchFn,
951 llvm::FunctionCallee endCatchFn, llvm::FunctionCallee rethrowFn);
952 void exit(CodeGenFunction &CGF);
953 };
954
955 /// Returns true inside SEH __try blocks.
956 bool isSEHTryScope() const { return !SEHTryEpilogueStack.empty(); }
957
958 /// Returns true while emitting a cleanuppad.
959 bool isCleanupPadScope() const {
960 return CurrentFuncletPad && isa<llvm::CleanupPadInst>(Val: CurrentFuncletPad);
961 }
962
963 /// pushFullExprCleanup - Push a cleanup to be run at the end of the
964 /// current full-expression. Safe against the possibility that
965 /// we're currently inside a conditionally-evaluated expression.
966 template <class T, class... As>
967 void pushFullExprCleanup(CleanupKind kind, As... A) {
968 // If we're not in a conditional branch, or if none of the
969 // arguments requires saving, then use the unconditional cleanup.
970 if (!isInConditionalBranch())
971 return EHStack.pushCleanup<T>(kind, A...);
972
973 // Stash values in a tuple so we can guarantee the order of saves.
974 typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
975 SavedTuple Saved{saveValueInCond(A)...};
976
977 typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
978 EHStack.pushCleanupTuple<CleanupType>(kind, Saved);
979 initFullExprCleanup();
980 }
981
982 /// Queue a cleanup to be pushed after finishing the current full-expression,
983 /// potentially with an active flag.
984 template <class T, class... As>
985 void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) {
986 if (!isInConditionalBranch())
987 return pushCleanupAfterFullExprWithActiveFlag<T>(
988 Kind, RawAddress::invalid(), A...);
989
990 RawAddress ActiveFlag = createCleanupActiveFlag();
991 assert(!DominatingValue<Address>::needsSaving(ActiveFlag) &&
992 "cleanup active flag should never need saving");
993
994 typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
995 SavedTuple Saved{saveValueInCond(A)...};
996
997 typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
998 pushCleanupAfterFullExprWithActiveFlag<CleanupType>(Kind, ActiveFlag,
999 Saved);
1000 }
1001
1002 template <class T, class... As>
1003 void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind,
1004 RawAddress ActiveFlag, As... A) {
1005 LifetimeExtendedCleanupHeader Header = {.Size: sizeof(T), .Kind: Kind,
1006 .IsConditional: ActiveFlag.isValid()};
1007
1008 size_t OldSize = LifetimeExtendedCleanupStack.size();
1009 LifetimeExtendedCleanupStack.resize(
1010 N: LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size +
1011 (Header.IsConditional ? sizeof(ActiveFlag) : 0));
1012
1013 static_assert((alignof(LifetimeExtendedCleanupHeader) == alignof(T)) &&
1014 (alignof(T) == alignof(RawAddress)),
1015 "Cleanup will be allocated on misaligned address");
1016 char *Buffer = &LifetimeExtendedCleanupStack[OldSize];
1017 new (Buffer) LifetimeExtendedCleanupHeader(Header);
1018 new (Buffer + sizeof(Header)) T(A...);
1019 if (Header.IsConditional)
1020 new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag);
1021 }
1022
1023 // Push a cleanup onto EHStack and deactivate it later. It is usually
1024 // deactivated when exiting a `CleanupDeactivationScope` (for example: after a
1025 // full expression).
1026 template <class T, class... As>
1027 void pushCleanupAndDeferDeactivation(CleanupKind Kind, As... A) {
1028 // Placeholder dominating IP for this cleanup.
1029 llvm::Instruction *DominatingIP =
1030 Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
1031 EHStack.pushCleanup<T>(Kind, A...);
1032 DeferredDeactivationCleanupStack.push_back(
1033 Elt: {.Cleanup: EHStack.stable_begin(), .DominatingIP: DominatingIP});
1034 }
1035
1036 /// Set up the last cleanup that was pushed as a conditional
1037 /// full-expression cleanup.
1038 void initFullExprCleanup() {
1039 initFullExprCleanupWithFlag(ActiveFlag: createCleanupActiveFlag());
1040 }
1041
1042 void initFullExprCleanupWithFlag(RawAddress ActiveFlag);
1043 RawAddress createCleanupActiveFlag();
1044
1045 /// PushDestructorCleanup - Push a cleanup to call the
1046 /// complete-object destructor of an object of the given type at the
1047 /// given address. Does nothing if T is not a C++ class type with a
1048 /// non-trivial destructor.
1049 void PushDestructorCleanup(QualType T, Address Addr);
1050
1051 /// PushDestructorCleanup - Push a cleanup to call the
1052 /// complete-object variant of the given destructor on the object at
1053 /// the given address.
1054 void PushDestructorCleanup(const CXXDestructorDecl *Dtor, QualType T,
1055 Address Addr);
1056
1057 /// PopCleanupBlock - Will pop the cleanup entry on the stack and
1058 /// process all branch fixups.
1059 void PopCleanupBlock(bool FallThroughIsBranchThrough = false,
1060 bool ForDeactivation = false);
1061
1062 /// DeactivateCleanupBlock - Deactivates the given cleanup block.
1063 /// The block cannot be reactivated. Pops it if it's the top of the
1064 /// stack.
1065 ///
1066 /// \param DominatingIP - An instruction which is known to
1067 /// dominate the current IP (if set) and which lies along
1068 /// all paths of execution between the current IP and the
1069 /// the point at which the cleanup comes into scope.
1070 void DeactivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
1071 llvm::Instruction *DominatingIP);
1072
1073 /// ActivateCleanupBlock - Activates an initially-inactive cleanup.
1074 /// Cannot be used to resurrect a deactivated cleanup.
1075 ///
1076 /// \param DominatingIP - An instruction which is known to
1077 /// dominate the current IP (if set) and which lies along
1078 /// all paths of execution between the current IP and the
1079 /// the point at which the cleanup comes into scope.
1080 void ActivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
1081 llvm::Instruction *DominatingIP);
1082
1083 /// Enters a new scope for capturing cleanups, all of which
1084 /// will be executed once the scope is exited.
1085 class RunCleanupsScope {
1086 EHScopeStack::stable_iterator CleanupStackDepth, OldCleanupScopeDepth;
1087 size_t LifetimeExtendedCleanupStackSize;
1088 CleanupDeactivationScope DeactivateCleanups;
1089 bool OldDidCallStackSave;
1090
1091 protected:
1092 bool PerformCleanup;
1093
1094 private:
1095 RunCleanupsScope(const RunCleanupsScope &) = delete;
1096 void operator=(const RunCleanupsScope &) = delete;
1097
1098 protected:
1099 CodeGenFunction &CGF;
1100
1101 public:
1102 /// Enter a new cleanup scope.
1103 explicit RunCleanupsScope(CodeGenFunction &CGF)
1104 : DeactivateCleanups(CGF), PerformCleanup(true), CGF(CGF) {
1105 CleanupStackDepth = CGF.EHStack.stable_begin();
1106 LifetimeExtendedCleanupStackSize =
1107 CGF.LifetimeExtendedCleanupStack.size();
1108 OldDidCallStackSave = CGF.DidCallStackSave;
1109 CGF.DidCallStackSave = false;
1110 OldCleanupScopeDepth = CGF.CurrentCleanupScopeDepth;
1111 CGF.CurrentCleanupScopeDepth = CleanupStackDepth;
1112 }
1113
1114 /// Exit this cleanup scope, emitting any accumulated cleanups.
1115 ~RunCleanupsScope() {
1116 if (PerformCleanup)
1117 ForceCleanup();
1118 }
1119
1120 /// Determine whether this scope requires any cleanups.
1121 bool requiresCleanups() const {
1122 return CGF.EHStack.stable_begin() != CleanupStackDepth;
1123 }
1124
1125 /// Force the emission of cleanups now, instead of waiting
1126 /// until this object is destroyed.
1127 /// \param ValuesToReload - A list of values that need to be available at
1128 /// the insertion point after cleanup emission. If cleanup emission created
1129 /// a shared cleanup block, these value pointers will be rewritten.
1130 /// Otherwise, they not will be modified.
1131 void
1132 ForceCleanup(std::initializer_list<llvm::Value **> ValuesToReload = {}) {
1133 assert(PerformCleanup && "Already forced cleanup");
1134 CGF.DidCallStackSave = OldDidCallStackSave;
1135 DeactivateCleanups.ForceDeactivate();
1136 CGF.PopCleanupBlocks(OldCleanupStackSize: CleanupStackDepth, OldLifetimeExtendedStackSize: LifetimeExtendedCleanupStackSize,
1137 ValuesToReload);
1138 PerformCleanup = false;
1139 CGF.CurrentCleanupScopeDepth = OldCleanupScopeDepth;
1140 }
1141 };
1142
1143 // Cleanup stack depth of the RunCleanupsScope that was pushed most recently.
1144 EHScopeStack::stable_iterator CurrentCleanupScopeDepth =
1145 EHScopeStack::stable_end();
1146
1147 class LexicalScope : public RunCleanupsScope {
1148 SourceRange Range;
1149 SmallVector<const LabelDecl *, 4> Labels;
1150 LexicalScope *ParentScope;
1151
1152 LexicalScope(const LexicalScope &) = delete;
1153 void operator=(const LexicalScope &) = delete;
1154
1155 public:
1156 /// Enter a new cleanup scope.
1157 explicit LexicalScope(CodeGenFunction &CGF, SourceRange Range);
1158
1159 void addLabel(const LabelDecl *label) {
1160 assert(PerformCleanup && "adding label to dead scope?");
1161 Labels.push_back(Elt: label);
1162 }
1163
1164 /// Exit this cleanup scope, emitting any accumulated
1165 /// cleanups.
1166 ~LexicalScope();
1167
1168 /// Force the emission of cleanups now, instead of waiting
1169 /// until this object is destroyed.
1170 void ForceCleanup() {
1171 CGF.CurLexicalScope = ParentScope;
1172 RunCleanupsScope::ForceCleanup();
1173
1174 if (!Labels.empty())
1175 rescopeLabels();
1176 }
1177
1178 bool hasLabels() const { return !Labels.empty(); }
1179
1180 void rescopeLabels();
1181 };
1182
1183 typedef llvm::DenseMap<const Decl *, Address> DeclMapTy;
1184
1185 /// The class used to assign some variables some temporarily addresses.
1186 class OMPMapVars {
1187 DeclMapTy SavedLocals;
1188 DeclMapTy SavedTempAddresses;
1189 OMPMapVars(const OMPMapVars &) = delete;
1190 void operator=(const OMPMapVars &) = delete;
1191
1192 public:
1193 explicit OMPMapVars() = default;
1194 ~OMPMapVars() {
1195 assert(SavedLocals.empty() && "Did not restored original addresses.");
1196 };
1197
1198 /// Sets the address of the variable \p LocalVD to be \p TempAddr in
1199 /// function \p CGF.
1200 /// \return true if at least one variable was set already, false otherwise.
1201 bool setVarAddr(CodeGenFunction &CGF, const ValueDecl *LocalVD,
1202 Address TempAddr) {
1203 LocalVD = cast<ValueDecl>(Val: LocalVD->getCanonicalDecl());
1204
1205 // Only save it once.
1206 if (SavedLocals.count(Val: LocalVD))
1207 return false;
1208
1209 // Copy the existing local entry to SavedLocals.
1210 auto it = CGF.LocalDeclMap.find(Val: LocalVD);
1211 if (it != CGF.LocalDeclMap.end())
1212 SavedLocals.try_emplace(Key: LocalVD, Args&: it->second);
1213 else
1214 SavedLocals.try_emplace(Key: LocalVD, Args: Address::invalid());
1215
1216 // Generate the private entry.
1217 QualType VarTy = LocalVD->getType();
1218 if (VarTy->isReferenceType()) {
1219 Address Temp = CGF.CreateMemTemp(T: VarTy);
1220 CGF.Builder.CreateStore(Val: TempAddr.emitRawPointer(CGF), Addr: Temp);
1221 TempAddr = Temp;
1222 }
1223 if (const auto *BD = dyn_cast<BindingDecl>(Val: LocalVD))
1224 CGF.OMPPrivatizedBindings.insert_or_assign(Key: BD, Val&: TempAddr);
1225 SavedTempAddresses.try_emplace(Key: LocalVD, Args&: TempAddr);
1226
1227 return true;
1228 }
1229
1230 /// Applies new addresses to the list of the variables.
1231 /// \return true if at least one variable is using new address, false
1232 /// otherwise.
1233 bool apply(CodeGenFunction &CGF) {
1234 copyInto(Src: SavedTempAddresses, Dest&: CGF.LocalDeclMap);
1235 SavedTempAddresses.clear();
1236 return !SavedLocals.empty();
1237 }
1238
1239 /// Restores original addresses of the variables.
1240 void restore(CodeGenFunction &CGF) {
1241 if (!SavedLocals.empty()) {
1242 copyInto(Src: SavedLocals, Dest&: CGF.LocalDeclMap);
1243 SavedLocals.clear();
1244 }
1245 }
1246
1247 private:
1248 /// Copy all the entries in the source map over the corresponding
1249 /// entries in the destination, which must exist.
1250 static void copyInto(const DeclMapTy &Src, DeclMapTy &Dest) {
1251 for (auto &[Decl, Addr] : Src) {
1252 if (!Addr.isValid())
1253 Dest.erase(Val: Decl);
1254 else
1255 Dest.insert_or_assign(Key: Decl, Val: Addr);
1256 }
1257 }
1258 };
1259
1260 /// The scope used to remap some variables as private in the OpenMP loop body
1261 /// (or other captured region emitted without outlining), and to restore old
1262 /// vars back on exit.
1263 class OMPPrivateScope : public RunCleanupsScope {
1264 OMPMapVars MappedVars;
1265 OMPPrivateScope(const OMPPrivateScope &) = delete;
1266 void operator=(const OMPPrivateScope &) = delete;
1267 llvm::DenseMap<const BindingDecl *, Address> BindingChanges;
1268
1269 public:
1270 /// Enter a new OpenMP private scope.
1271 explicit OMPPrivateScope(CodeGenFunction &CGF) : RunCleanupsScope(CGF) {}
1272
1273 /// Registers \p LocalVD variable as a private with \p Addr as the address
1274 /// of the corresponding private variable. \p
1275 /// PrivateGen is the address of the generated private variable.
1276 /// \return true if the variable is registered as private, false if it has
1277 /// been privatized already.
1278 bool addPrivate(const ValueDecl *LocalVD, Address Addr) {
1279 assert(PerformCleanup && "adding private to dead scope");
1280 if (const auto *BD = dyn_cast<BindingDecl>(Val: LocalVD->getCanonicalDecl())) {
1281 auto It = CGF.OMPPrivatizedBindings.find(Val: BD);
1282 BindingChanges.insert(KV: {BD, It != CGF.OMPPrivatizedBindings.end()
1283 ? It->second
1284 : Address::invalid()});
1285 }
1286 return MappedVars.setVarAddr(CGF, LocalVD, TempAddr: Addr);
1287 }
1288
1289 /// Privatizes local variables previously registered as private.
1290 /// Registration is separate from the actual privatization to allow
1291 /// initializers use values of the original variables, not the private one.
1292 /// This is important, for example, if the private variable is a class
1293 /// variable initialized by a constructor that references other private
1294 /// variables. But at initialization original variables must be used, not
1295 /// private copies.
1296 /// \return true if at least one variable was privatized, false otherwise.
1297 bool Privatize() { return MappedVars.apply(CGF); }
1298
1299 void ForceCleanup() {
1300 RunCleanupsScope::ForceCleanup();
1301 restoreMap();
1302 }
1303
1304 /// Exit scope - all the mapped variables are restored.
1305 ~OMPPrivateScope() {
1306 if (PerformCleanup)
1307 ForceCleanup();
1308 for (auto &Change : BindingChanges) {
1309 if (Change.second.isValid()) {
1310 auto It = CGF.OMPPrivatizedBindings.find(Val: Change.first);
1311 if (It != CGF.OMPPrivatizedBindings.end())
1312 It->second = Change.second;
1313 else
1314 CGF.OMPPrivatizedBindings.insert(KV: {Change.first, Change.second});
1315 } else {
1316 CGF.OMPPrivatizedBindings.erase(Val: Change.first);
1317 }
1318 }
1319 }
1320
1321 /// Checks if the global variable is captured in current function.
1322 bool isGlobalVarCaptured(const VarDecl *VD) const {
1323 VD = VD->getCanonicalDecl();
1324 return !VD->isLocalVarDeclOrParm() && CGF.LocalDeclMap.count(Val: VD) > 0;
1325 }
1326
1327 /// Restore all mapped variables w/o clean up. This is usefully when we want
1328 /// to reference the original variables but don't want the clean up because
1329 /// that could emit lifetime end too early, causing backend issue #56913.
1330 void restoreMap() { MappedVars.restore(CGF); }
1331 };
1332
1333 /// Save/restore original map of previously emitted local vars in case when we
1334 /// need to duplicate emission of the same code several times in the same
1335 /// function for OpenMP code.
1336 class OMPLocalDeclMapRAII {
1337 CodeGenFunction &CGF;
1338 DeclMapTy SavedMap;
1339
1340 public:
1341 OMPLocalDeclMapRAII(CodeGenFunction &CGF)
1342 : CGF(CGF), SavedMap(CGF.LocalDeclMap) {}
1343 ~OMPLocalDeclMapRAII() { SavedMap.swap(RHS&: CGF.LocalDeclMap); }
1344 };
1345
1346 /// Takes the old cleanup stack size and emits the cleanup blocks
1347 /// that have been added.
1348 void
1349 PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1350 std::initializer_list<llvm::Value **> ValuesToReload = {});
1351
1352 /// Takes the old cleanup stack size and emits the cleanup blocks
1353 /// that have been added, then adds all lifetime-extended cleanups from
1354 /// the given position to the stack.
1355 void
1356 PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1357 size_t OldLifetimeExtendedStackSize,
1358 std::initializer_list<llvm::Value **> ValuesToReload = {});
1359
1360 void ResolveBranchFixups(llvm::BasicBlock *Target);
1361
1362 /// The given basic block lies in the current EH scope, but may be a
1363 /// target of a potentially scope-crossing jump; get a stable handle
1364 /// to which we can perform this jump later.
1365 JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target) {
1366 return JumpDest(Target, EHStack.getInnermostNormalCleanup(),
1367 NextCleanupDestIndex++);
1368 }
1369
1370 /// The given basic block lies in the current EH scope, but may be a
1371 /// target of a potentially scope-crossing jump; get a stable handle
1372 /// to which we can perform this jump later.
1373 JumpDest getJumpDestInCurrentScope(StringRef Name = StringRef()) {
1374 return getJumpDestInCurrentScope(Target: createBasicBlock(name: Name));
1375 }
1376
1377 /// EmitBranchThroughCleanup - Emit a branch from the current insert
1378 /// block through the normal cleanup handling code (if any) and then
1379 /// on to \arg Dest.
1380 void EmitBranchThroughCleanup(JumpDest Dest);
1381
1382 /// isObviouslyBranchWithoutCleanups - Return true if a branch to the
1383 /// specified destination obviously has no cleanups to run. 'false' is always
1384 /// a conservatively correct answer for this method.
1385 bool isObviouslyBranchWithoutCleanups(JumpDest Dest) const;
1386
1387 /// popCatchScope - Pops the catch scope at the top of the EHScope
1388 /// stack, emitting any required code (other than the catch handlers
1389 /// themselves).
1390 void popCatchScope();
1391
1392 // This function should be called after emitting all catch clauses and none
1393 // of them were 'catch-all' clauses.
1394 // Because in wasm we merge all catch clauses into one big catchpad, in case
1395 // none of the types in catch handlers matches after we test against each of
1396 // them, we should unwind to the next EH enclosing scope. We generate a call
1397 // to rethrow function here to do that.
1398 void WasmEmitFallthroughRethrow(llvm::BasicBlock *WasmCatchStartBlock);
1399
1400 llvm::BasicBlock *getEHResumeBlock(bool isCleanup);
1401 llvm::BasicBlock *getEHDispatchBlock(EHScopeStack::stable_iterator scope);
1402 llvm::BasicBlock *
1403 getFuncletEHDispatchBlock(EHScopeStack::stable_iterator scope);
1404
1405 /// An object to manage conditionally-evaluated expressions.
1406 class ConditionalEvaluation {
1407 llvm::BasicBlock *StartBB;
1408
1409 public:
1410 ConditionalEvaluation(CodeGenFunction &CGF)
1411 : StartBB(CGF.Builder.GetInsertBlock()) {}
1412
1413 void begin(CodeGenFunction &CGF) {
1414 assert(CGF.OutermostConditional != this);
1415 if (!CGF.OutermostConditional)
1416 CGF.OutermostConditional = this;
1417 }
1418
1419 void end(CodeGenFunction &CGF) {
1420 assert(CGF.OutermostConditional != nullptr);
1421 if (CGF.OutermostConditional == this)
1422 CGF.OutermostConditional = nullptr;
1423 }
1424
1425 /// Returns a block which will be executed prior to each
1426 /// evaluation of the conditional code.
1427 llvm::BasicBlock *getStartingBlock() const { return StartBB; }
1428 };
1429
1430 /// isInConditionalBranch - Return true if we're currently emitting
1431 /// one branch or the other of a conditional expression.
1432 bool isInConditionalBranch() const { return OutermostConditional != nullptr; }
1433
1434 void setBeforeOutermostConditional(llvm::Value *value, Address addr,
1435 CodeGenFunction &CGF) {
1436 assert(isInConditionalBranch());
1437 llvm::BasicBlock *block = OutermostConditional->getStartingBlock();
1438 auto store = new llvm::StoreInst(value, addr.emitRawPointer(CGF),
1439 block->back().getIterator());
1440 store->setAlignment(addr.getAlignment().getAsAlign());
1441 }
1442
1443 /// An RAII object to record that we're evaluating a statement
1444 /// expression.
1445 class StmtExprEvaluation {
1446 CodeGenFunction &CGF;
1447
1448 /// We have to save the outermost conditional: cleanups in a
1449 /// statement expression aren't conditional just because the
1450 /// StmtExpr is.
1451 ConditionalEvaluation *SavedOutermostConditional;
1452
1453 public:
1454 StmtExprEvaluation(CodeGenFunction &CGF)
1455 : CGF(CGF), SavedOutermostConditional(CGF.OutermostConditional) {
1456 CGF.OutermostConditional = nullptr;
1457 }
1458
1459 ~StmtExprEvaluation() {
1460 CGF.OutermostConditional = SavedOutermostConditional;
1461 CGF.EnsureInsertPoint();
1462 }
1463 };
1464
1465 /// An object which temporarily prevents a value from being
1466 /// destroyed by aggressive peephole optimizations that assume that
1467 /// all uses of a value have been realized in the IR.
1468 class PeepholeProtection {
1469 llvm::Instruction *Inst = nullptr;
1470 friend class CodeGenFunction;
1471
1472 public:
1473 PeepholeProtection() = default;
1474 };
1475
1476 /// A non-RAII class containing all the information about a bound
1477 /// opaque value. OpaqueValueMapping, below, is a RAII wrapper for
1478 /// this which makes individual mappings very simple; using this
1479 /// class directly is useful when you have a variable number of
1480 /// opaque values or don't want the RAII functionality for some
1481 /// reason.
1482 class OpaqueValueMappingData {
1483 const OpaqueValueExpr *OpaqueValue;
1484 bool BoundLValue;
1485 CodeGenFunction::PeepholeProtection Protection;
1486
1487 OpaqueValueMappingData(const OpaqueValueExpr *ov, bool boundLValue)
1488 : OpaqueValue(ov), BoundLValue(boundLValue) {}
1489
1490 public:
1491 OpaqueValueMappingData() : OpaqueValue(nullptr) {}
1492
1493 static bool shouldBindAsLValue(const Expr *expr) {
1494 // gl-values should be bound as l-values for obvious reasons.
1495 // Records should be bound as l-values because IR generation
1496 // always keeps them in memory. Expressions of function type
1497 // act exactly like l-values but are formally required to be
1498 // r-values in C.
1499 return expr->isGLValue() || expr->getType()->isFunctionType() ||
1500 hasAggregateEvaluationKind(T: expr->getType());
1501 }
1502
1503 static OpaqueValueMappingData
1504 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const Expr *e) {
1505 if (shouldBindAsLValue(expr: ov))
1506 return bind(CGF, ov, lv: CGF.EmitLValue(E: e));
1507 return bind(CGF, ov, rv: CGF.EmitAnyExpr(E: e));
1508 }
1509
1510 static OpaqueValueMappingData
1511 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const LValue &lv) {
1512 assert(shouldBindAsLValue(ov));
1513 CGF.OpaqueLValues.insert(KV: std::make_pair(x&: ov, y: lv));
1514 return OpaqueValueMappingData(ov, true);
1515 }
1516
1517 static OpaqueValueMappingData
1518 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const RValue &rv) {
1519 assert(!shouldBindAsLValue(ov));
1520 CGF.OpaqueRValues.insert(KV: std::make_pair(x&: ov, y: rv));
1521
1522 OpaqueValueMappingData data(ov, false);
1523
1524 // Work around an extremely aggressive peephole optimization in
1525 // EmitScalarConversion which assumes that all other uses of a
1526 // value are extant.
1527 data.Protection = CGF.protectFromPeepholes(rvalue: rv);
1528
1529 return data;
1530 }
1531
1532 bool isValid() const { return OpaqueValue != nullptr; }
1533 void clear() { OpaqueValue = nullptr; }
1534
1535 void unbind(CodeGenFunction &CGF) {
1536 assert(OpaqueValue && "no data to unbind!");
1537
1538 if (BoundLValue) {
1539 CGF.OpaqueLValues.erase(Val: OpaqueValue);
1540 } else {
1541 CGF.OpaqueRValues.erase(Val: OpaqueValue);
1542 CGF.unprotectFromPeepholes(protection: Protection);
1543 }
1544 }
1545 };
1546
1547 /// An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
1548 class OpaqueValueMapping {
1549 CodeGenFunction &CGF;
1550 OpaqueValueMappingData Data;
1551
1552 public:
1553 static bool shouldBindAsLValue(const Expr *expr) {
1554 return OpaqueValueMappingData::shouldBindAsLValue(expr);
1555 }
1556
1557 /// Build the opaque value mapping for the given conditional
1558 /// operator if it's the GNU ?: extension. This is a common
1559 /// enough pattern that the convenience operator is really
1560 /// helpful.
1561 ///
1562 OpaqueValueMapping(CodeGenFunction &CGF,
1563 const AbstractConditionalOperator *op)
1564 : CGF(CGF) {
1565 if (isa<ConditionalOperator>(Val: op))
1566 // Leave Data empty.
1567 return;
1568
1569 const BinaryConditionalOperator *e = cast<BinaryConditionalOperator>(Val: op);
1570 Data = OpaqueValueMappingData::bind(CGF, ov: e->getOpaqueValue(),
1571 e: e->getCommon());
1572 }
1573
1574 /// Build the opaque value mapping for an OpaqueValueExpr whose source
1575 /// expression is set to the expression the OVE represents.
1576 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *OV)
1577 : CGF(CGF) {
1578 if (OV) {
1579 assert(OV->getSourceExpr() && "wrong form of OpaqueValueMapping used "
1580 "for OVE with no source expression");
1581 Data = OpaqueValueMappingData::bind(CGF, ov: OV, e: OV->getSourceExpr());
1582 }
1583 }
1584
1585 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,
1586 LValue lvalue)
1587 : CGF(CGF),
1588 Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, lv: lvalue)) {}
1589
1590 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,
1591 RValue rvalue)
1592 : CGF(CGF),
1593 Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, rv: rvalue)) {}
1594
1595 void pop() {
1596 Data.unbind(CGF);
1597 Data.clear();
1598 }
1599
1600 ~OpaqueValueMapping() {
1601 if (Data.isValid())
1602 Data.unbind(CGF);
1603 }
1604 };
1605
1606private:
1607 CGDebugInfo *DebugInfo;
1608 /// Used to create unique names for artificial VLA size debug info variables.
1609 unsigned VLAExprCounter = 0;
1610 bool DisableDebugInfo = false;
1611
1612 /// DidCallStackSave - Whether llvm.stacksave has been called. Used to avoid
1613 /// calling llvm.stacksave for multiple VLAs in the same scope.
1614 bool DidCallStackSave = false;
1615
1616 /// IndirectBranch - The first time an indirect goto is seen we create a block
1617 /// with an indirect branch. Every time we see the address of a label taken,
1618 /// we add the label to the indirect goto. Every subsequent indirect goto is
1619 /// codegen'd as a jump to the IndirectBranch's basic block.
1620 llvm::IndirectBrInst *IndirectBranch = nullptr;
1621
1622 /// LocalDeclMap - This keeps track of the LLVM allocas or globals for local C
1623 /// decls.
1624 DeclMapTy LocalDeclMap;
1625
1626 /// Lookup map for privatized BindingDecls.
1627 /// Used when BindingDecls are remapped during OpenMP outlining, since the
1628 /// remapped BindingDecl has a different pointer than the original.
1629 llvm::SmallDenseMap<const BindingDecl *, Address> OMPPrivatizedBindings;
1630
1631 // Keep track of the cleanups for callee-destructed parameters pushed to the
1632 // cleanup stack so that they can be deactivated later.
1633 llvm::DenseMap<const ParmVarDecl *, EHScopeStack::stable_iterator>
1634 CalleeDestructedParamCleanups;
1635
1636 /// SizeArguments - If a ParmVarDecl had the pass_object_size attribute, this
1637 /// will contain a mapping from said ParmVarDecl to its implicit "object_size"
1638 /// parameter.
1639 llvm::SmallDenseMap<const ParmVarDecl *, const ImplicitParamDecl *, 2>
1640 SizeArguments;
1641
1642 /// Track escaped local variables with auto storage. Used during SEH
1643 /// outlining to produce a call to llvm.localescape.
1644 llvm::DenseMap<llvm::AllocaInst *, int> EscapedLocals;
1645
1646 /// LabelMap - This keeps track of the LLVM basic block for each C label.
1647 llvm::DenseMap<const LabelDecl *, JumpDest> LabelMap;
1648
1649 // BreakContinueStack - This keeps track of where break and continue
1650 // statements should jump to.
1651 struct BreakContinue {
1652 BreakContinue(const Stmt &LoopOrSwitch, JumpDest Break, JumpDest Continue)
1653 : LoopOrSwitch(&LoopOrSwitch), BreakBlock(Break),
1654 ContinueBlock(Continue) {}
1655
1656 const Stmt *LoopOrSwitch;
1657 JumpDest BreakBlock;
1658 JumpDest ContinueBlock;
1659 };
1660 SmallVector<BreakContinue, 8> BreakContinueStack;
1661
1662 /// Handles cancellation exit points in OpenMP-related constructs.
1663 class OpenMPCancelExitStack {
1664 /// Tracks cancellation exit point and join point for cancel-related exit
1665 /// and normal exit.
1666 struct CancelExit {
1667 CancelExit() = default;
1668 CancelExit(OpenMPDirectiveKind Kind, JumpDest ExitBlock,
1669 JumpDest ContBlock)
1670 : Kind(Kind), ExitBlock(ExitBlock), ContBlock(ContBlock) {}
1671 OpenMPDirectiveKind Kind = llvm::omp::OMPD_unknown;
1672 /// true if the exit block has been emitted already by the special
1673 /// emitExit() call, false if the default codegen is used.
1674 bool HasBeenEmitted = false;
1675 JumpDest ExitBlock;
1676 JumpDest ContBlock;
1677 };
1678
1679 SmallVector<CancelExit, 8> Stack;
1680
1681 public:
1682 OpenMPCancelExitStack() : Stack(1) {}
1683 ~OpenMPCancelExitStack() = default;
1684 /// Fetches the exit block for the current OpenMP construct.
1685 JumpDest getExitBlock() const { return Stack.back().ExitBlock; }
1686 /// Emits exit block with special codegen procedure specific for the related
1687 /// OpenMP construct + emits code for normal construct cleanup.
1688 void emitExit(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
1689 const llvm::function_ref<void(CodeGenFunction &)> CodeGen) {
1690 if (Stack.back().Kind == Kind && getExitBlock().isValid()) {
1691 assert(CGF.getOMPCancelDestination(Kind).isValid());
1692 assert(CGF.HaveInsertPoint());
1693 assert(!Stack.back().HasBeenEmitted);
1694 auto IP = CGF.Builder.saveAndClearIP();
1695 CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1696 CodeGen(CGF);
1697 CGF.EmitBranch(Block: Stack.back().ContBlock.getBlock());
1698 CGF.Builder.restoreIP(IP);
1699 Stack.back().HasBeenEmitted = true;
1700 }
1701 CodeGen(CGF);
1702 }
1703 /// Enter the cancel supporting \a Kind construct.
1704 /// \param Kind OpenMP directive that supports cancel constructs.
1705 /// \param HasCancel true, if the construct has inner cancel directive,
1706 /// false otherwise.
1707 void enter(CodeGenFunction &CGF, OpenMPDirectiveKind Kind, bool HasCancel) {
1708 Stack.push_back(Elt: {Kind,
1709 HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.exit")
1710 : JumpDest(),
1711 HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.cont")
1712 : JumpDest()});
1713 }
1714 /// Emits default exit point for the cancel construct (if the special one
1715 /// has not be used) + join point for cancel/normal exits.
1716 void exit(CodeGenFunction &CGF) {
1717 if (getExitBlock().isValid()) {
1718 assert(CGF.getOMPCancelDestination(Stack.back().Kind).isValid());
1719 bool HaveIP = CGF.HaveInsertPoint();
1720 if (!Stack.back().HasBeenEmitted) {
1721 if (HaveIP)
1722 CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1723 CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1724 CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1725 }
1726 CGF.EmitBlock(BB: Stack.back().ContBlock.getBlock());
1727 if (!HaveIP) {
1728 CGF.Builder.CreateUnreachable();
1729 CGF.Builder.ClearInsertionPoint();
1730 }
1731 }
1732 Stack.pop_back();
1733 }
1734 };
1735 OpenMPCancelExitStack OMPCancelStack;
1736
1737 /// Lower the Likelihood knowledge about the \p Cond via llvm.expect intrin.
1738 llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
1739 Stmt::Likelihood LH);
1740
1741 std::unique_ptr<CodeGenPGO> PGO;
1742
1743 /// Calculate branch weights appropriate for PGO data
1744 llvm::MDNode *createProfileWeights(uint64_t TrueCount,
1745 uint64_t FalseCount) const;
1746 llvm::MDNode *createProfileWeights(ArrayRef<uint64_t> Weights) const;
1747 llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond,
1748 uint64_t LoopCount) const;
1749
1750public:
1751 bool hasSkipCounter(const Stmt *S) const;
1752
1753 void markStmtAsUsed(bool Skipped, const Stmt *S);
1754 void markStmtMaybeUsed(const Stmt *S);
1755
1756 /// Used to specify which counter in a pair shall be incremented.
1757 /// For non-binary counters, a skip counter is derived as (Parent - Exec).
1758 /// In contrast for binary counters, a skip counter cannot be computed from
1759 /// the Parent counter. In such cases, dedicated SkipPath counters must be
1760 /// allocated and marked (incremented as binary counters). (Parent can be
1761 /// synthesized with (Exec + Skip) in simple cases)
1762 enum CounterForIncrement {
1763 UseExecPath = 0, ///< Exec (true)
1764 UseSkipPath, ///< Skip (false)
1765 };
1766
1767 /// Increment the profiler's counter for the given statement by \p StepV.
1768 /// If \p StepV is null, the default increment is 1.
1769 void incrementProfileCounter(const Stmt *S, llvm::Value *StepV = nullptr) {
1770 incrementProfileCounter(ExecSkip: UseExecPath, S, UseBoth: false, StepV);
1771 }
1772
1773 /// Emit increment of Counter.
1774 /// \param ExecSkip Use `Skipped` Counter if UseSkipPath is specified.
1775 /// \param S The Stmt that Counter is associated.
1776 /// \param UseBoth Mark both Exec/Skip as used. (for verification)
1777 /// \param StepV The offset Value for adding to Counter.
1778 void incrementProfileCounter(CounterForIncrement ExecSkip, const Stmt *S,
1779 bool UseBoth = false,
1780 llvm::Value *StepV = nullptr);
1781
1782 bool isMCDCCoverageEnabled() const {
1783 return (CGM.getCodeGenOpts().hasProfileClangInstr() &&
1784 CGM.getCodeGenOpts().MCDCCoverage &&
1785 !CurFn->hasFnAttribute(Kind: llvm::Attribute::NoProfile));
1786 }
1787
1788 /// Allocate a temp value on the stack that MCDC can use to track condition
1789 /// results.
1790 void maybeCreateMCDCCondBitmap();
1791
1792 bool isBinaryLogicalOp(const Expr *E) const {
1793 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: E->IgnoreParens());
1794 return (BOp && BOp->isLogicalOp());
1795 }
1796
1797 bool isMCDCDecisionExpr(const Expr *E) const;
1798 bool isMCDCBranchExpr(const Expr *E) const;
1799
1800 /// Zero-init the MCDC temp value.
1801 void maybeResetMCDCCondBitmap(const Expr *E);
1802
1803 /// Increment the profiler's counter for the given expression by \p StepV.
1804 /// If \p StepV is null, the default increment is 1.
1805 void maybeUpdateMCDCTestVectorBitmap(const Expr *E);
1806
1807 /// Update the MCDC temp value with the condition's evaluated result.
1808 void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val);
1809
1810 /// Get the profiler's count for the given statement.
1811 uint64_t getProfileCount(const Stmt *S);
1812
1813 /// Set the profiler's current count.
1814 void setCurrentProfileCount(uint64_t Count);
1815
1816 /// Get the profiler's current count. This is generally the count for the most
1817 /// recently incremented counter.
1818 uint64_t getCurrentProfileCount();
1819
1820 /// See CGDebugInfo::addInstToCurrentSourceAtom.
1821 void addInstToCurrentSourceAtom(llvm::Instruction *KeyInstruction,
1822 llvm::Value *Backup);
1823
1824 /// See CGDebugInfo::addInstToSpecificSourceAtom.
1825 void addInstToSpecificSourceAtom(llvm::Instruction *KeyInstruction,
1826 llvm::Value *Backup, uint64_t Atom);
1827
1828 /// Add \p KeyInstruction and an optional \p Backup instruction to a new atom
1829 /// group (See ApplyAtomGroup for more info).
1830 void addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,
1831 llvm::Value *Backup);
1832
1833 /// Copy all PFP fields from SrcPtr to DestPtr while updating signatures,
1834 /// assuming that DestPtr was already memcpy'd from SrcPtr.
1835 void emitPFPPostCopyUpdates(Address DestPtr, Address SrcPtr, QualType Ty);
1836
1837private:
1838 /// SwitchInsn - This is nearest current switch instruction. It is null if
1839 /// current context is not in a switch.
1840 llvm::SwitchInst *SwitchInsn = nullptr;
1841 /// The branch weights of SwitchInsn when doing instrumentation based PGO.
1842 SmallVector<uint64_t, 16> *SwitchWeights = nullptr;
1843
1844 /// The likelihood attributes of the SwitchCase.
1845 SmallVector<Stmt::Likelihood, 16> *SwitchLikelihood = nullptr;
1846
1847 /// CaseRangeBlock - This block holds if condition check for last case
1848 /// statement range in current switch instruction.
1849 llvm::BasicBlock *CaseRangeBlock = nullptr;
1850
1851 /// OpaqueLValues - Keeps track of the current set of opaque value
1852 /// expressions.
1853 llvm::DenseMap<const OpaqueValueExpr *, LValue> OpaqueLValues;
1854 llvm::DenseMap<const OpaqueValueExpr *, RValue> OpaqueRValues;
1855
1856 // VLASizeMap - This keeps track of the associated size for each VLA type.
1857 // We track this by the size expression rather than the type itself because
1858 // in certain situations, like a const qualifier applied to an VLA typedef,
1859 // multiple VLA types can share the same size expression.
1860 // FIXME: Maybe this could be a stack of maps that is pushed/popped as we
1861 // enter/leave scopes.
1862 llvm::DenseMap<const Expr *, llvm::Value *> VLASizeMap;
1863
1864 /// A block containing a single 'unreachable' instruction. Created
1865 /// lazily by getUnreachableBlock().
1866 llvm::BasicBlock *UnreachableBlock = nullptr;
1867
1868 /// Counts of the number return expressions in the function.
1869 unsigned NumReturnExprs = 0;
1870
1871 /// Count the number of simple (constant) return expressions in the function.
1872 unsigned NumSimpleReturnExprs = 0;
1873
1874 /// The last regular (non-return) debug location (breakpoint) in the function.
1875 SourceLocation LastStopPoint;
1876
1877public:
1878 /// Source location information about the default argument or member
1879 /// initializer expression we're evaluating, if any.
1880 CurrentSourceLocExprScope CurSourceLocExprScope;
1881 using SourceLocExprScopeGuard =
1882 CurrentSourceLocExprScope::SourceLocExprScopeGuard;
1883
1884 /// A scope within which we are constructing the fields of an object which
1885 /// might use a CXXDefaultInitExpr. This stashes away a 'this' value to use
1886 /// if we need to evaluate a CXXDefaultInitExpr within the evaluation.
1887 class FieldConstructionScope {
1888 public:
1889 FieldConstructionScope(CodeGenFunction &CGF, Address This)
1890 : CGF(CGF), OldCXXDefaultInitExprThis(CGF.CXXDefaultInitExprThis) {
1891 CGF.CXXDefaultInitExprThis = This;
1892 }
1893 ~FieldConstructionScope() {
1894 CGF.CXXDefaultInitExprThis = OldCXXDefaultInitExprThis;
1895 }
1896
1897 private:
1898 CodeGenFunction &CGF;
1899 Address OldCXXDefaultInitExprThis;
1900 };
1901
1902 /// The scope of a CXXDefaultInitExpr. Within this scope, the value of 'this'
1903 /// is overridden to be the object under construction.
1904 class CXXDefaultInitExprScope {
1905 public:
1906 CXXDefaultInitExprScope(CodeGenFunction &CGF, const CXXDefaultInitExpr *E)
1907 : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue),
1908 OldCXXThisAlignment(CGF.CXXThisAlignment),
1909 SourceLocScope(E, CGF.CurSourceLocExprScope) {
1910 CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getBasePointer();
1911 CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment();
1912 }
1913 ~CXXDefaultInitExprScope() {
1914 CGF.CXXThisValue = OldCXXThisValue;
1915 CGF.CXXThisAlignment = OldCXXThisAlignment;
1916 }
1917
1918 public:
1919 CodeGenFunction &CGF;
1920 llvm::Value *OldCXXThisValue;
1921 CharUnits OldCXXThisAlignment;
1922 SourceLocExprScopeGuard SourceLocScope;
1923 };
1924
1925 struct CXXDefaultArgExprScope : SourceLocExprScopeGuard {
1926 CXXDefaultArgExprScope(CodeGenFunction &CGF, const CXXDefaultArgExpr *E)
1927 : SourceLocExprScopeGuard(E, CGF.CurSourceLocExprScope) {}
1928 };
1929
1930 /// The scope of an ArrayInitLoopExpr. Within this scope, the value of the
1931 /// current loop index is overridden.
1932 class ArrayInitLoopExprScope {
1933 public:
1934 ArrayInitLoopExprScope(CodeGenFunction &CGF, llvm::Value *Index)
1935 : CGF(CGF), OldArrayInitIndex(CGF.ArrayInitIndex) {
1936 CGF.ArrayInitIndex = Index;
1937 }
1938 ~ArrayInitLoopExprScope() { CGF.ArrayInitIndex = OldArrayInitIndex; }
1939
1940 private:
1941 CodeGenFunction &CGF;
1942 llvm::Value *OldArrayInitIndex;
1943 };
1944
1945 class InlinedInheritingConstructorScope {
1946 public:
1947 InlinedInheritingConstructorScope(CodeGenFunction &CGF, GlobalDecl GD)
1948 : CGF(CGF), OldCurGD(CGF.CurGD), OldCurFuncDecl(CGF.CurFuncDecl),
1949 OldCurCodeDecl(CGF.CurCodeDecl),
1950 OldCXXABIThisDecl(CGF.CXXABIThisDecl),
1951 OldCXXABIThisValue(CGF.CXXABIThisValue),
1952 OldCXXThisValue(CGF.CXXThisValue),
1953 OldCXXABIThisAlignment(CGF.CXXABIThisAlignment),
1954 OldCXXThisAlignment(CGF.CXXThisAlignment),
1955 OldReturnValue(CGF.ReturnValue), OldFnRetTy(CGF.FnRetTy),
1956 OldCXXInheritedCtorInitExprArgs(
1957 std::move(CGF.CXXInheritedCtorInitExprArgs)) {
1958 CGF.CurGD = GD;
1959 CGF.CurFuncDecl = CGF.CurCodeDecl =
1960 cast<CXXConstructorDecl>(Val: GD.getDecl());
1961 CGF.CXXABIThisDecl = nullptr;
1962 CGF.CXXABIThisValue = nullptr;
1963 CGF.CXXThisValue = nullptr;
1964 CGF.CXXABIThisAlignment = CharUnits();
1965 CGF.CXXThisAlignment = CharUnits();
1966 CGF.ReturnValue = Address::invalid();
1967 CGF.FnRetTy = QualType();
1968 CGF.CXXInheritedCtorInitExprArgs.clear();
1969 }
1970 ~InlinedInheritingConstructorScope() {
1971 CGF.CurGD = OldCurGD;
1972 CGF.CurFuncDecl = OldCurFuncDecl;
1973 CGF.CurCodeDecl = OldCurCodeDecl;
1974 CGF.CXXABIThisDecl = OldCXXABIThisDecl;
1975 CGF.CXXABIThisValue = OldCXXABIThisValue;
1976 CGF.CXXThisValue = OldCXXThisValue;
1977 CGF.CXXABIThisAlignment = OldCXXABIThisAlignment;
1978 CGF.CXXThisAlignment = OldCXXThisAlignment;
1979 CGF.ReturnValue = OldReturnValue;
1980 CGF.FnRetTy = OldFnRetTy;
1981 CGF.CXXInheritedCtorInitExprArgs =
1982 std::move(OldCXXInheritedCtorInitExprArgs);
1983 }
1984
1985 private:
1986 CodeGenFunction &CGF;
1987 GlobalDecl OldCurGD;
1988 const Decl *OldCurFuncDecl;
1989 const Decl *OldCurCodeDecl;
1990 ImplicitParamDecl *OldCXXABIThisDecl;
1991 llvm::Value *OldCXXABIThisValue;
1992 llvm::Value *OldCXXThisValue;
1993 CharUnits OldCXXABIThisAlignment;
1994 CharUnits OldCXXThisAlignment;
1995 Address OldReturnValue;
1996 QualType OldFnRetTy;
1997 CallArgList OldCXXInheritedCtorInitExprArgs;
1998 };
1999
2000 // Helper class for the OpenMP IR Builder. Allows reusability of code used for
2001 // region body, and finalization codegen callbacks. This will class will also
2002 // contain privatization functions used by the privatization call backs
2003 //
2004 // TODO: this is temporary class for things that are being moved out of
2005 // CGOpenMPRuntime, new versions of current CodeGenFunction methods, or
2006 // utility function for use with the OMPBuilder. Once that move to use the
2007 // OMPBuilder is done, everything here will either become part of CodeGenFunc.
2008 // directly, or a new helper class that will contain functions used by both
2009 // this and the OMPBuilder
2010
2011 struct OMPBuilderCBHelpers {
2012
2013 OMPBuilderCBHelpers() = delete;
2014 OMPBuilderCBHelpers(const OMPBuilderCBHelpers &) = delete;
2015 OMPBuilderCBHelpers &operator=(const OMPBuilderCBHelpers &) = delete;
2016
2017 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
2018
2019 /// Cleanup action for allocate support.
2020 class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
2021
2022 private:
2023 llvm::CallInst *RTLFnCI;
2024
2025 public:
2026 OMPAllocateCleanupTy(llvm::CallInst *RLFnCI) : RTLFnCI(RLFnCI) {
2027 RLFnCI->removeFromParent();
2028 }
2029
2030 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
2031 if (!CGF.HaveInsertPoint())
2032 return;
2033 CGF.Builder.Insert(I: RTLFnCI);
2034 }
2035 };
2036
2037 /// Returns address of the threadprivate variable for the current
2038 /// thread. This Also create any necessary OMP runtime calls.
2039 ///
2040 /// \param VD VarDecl for Threadprivate variable.
2041 /// \param VDAddr Address of the Vardecl
2042 /// \param Loc The location where the barrier directive was encountered
2043 static Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
2044 const VarDecl *VD, Address VDAddr,
2045 SourceLocation Loc);
2046
2047 /// Gets the OpenMP-specific address of the local variable /p VD.
2048 static Address getAddressOfLocalVariable(CodeGenFunction &CGF,
2049 const VarDecl *VD);
2050 /// Get the platform-specific name separator.
2051 /// \param Parts different parts of the final name that needs separation
2052 /// \param FirstSeparator First separator used between the initial two
2053 /// parts of the name.
2054 /// \param Separator separator used between all of the rest consecutinve
2055 /// parts of the name
2056 static std::string getNameWithSeparators(ArrayRef<StringRef> Parts,
2057 StringRef FirstSeparator = ".",
2058 StringRef Separator = ".");
2059 /// Emit the Finalization for an OMP region
2060 /// \param CGF The Codegen function this belongs to
2061 /// \param IP Insertion point for generating the finalization code.
2062 static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP) {
2063 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
2064 llvm::BasicBlock *IPBB = IP.getNodeParent();
2065 assert(IPBB->end() != IP &&
2066 "OpenMP IR Builder should cause terminated block!");
2067
2068 llvm::BasicBlock *DestBB = IPBB->getUniqueSuccessor();
2069 assert(DestBB && "Finalization block should have one successor!");
2070
2071 // erase and replace with cleanup branch.
2072 IPBB->getTerminator()->eraseFromParent();
2073 CGF.Builder.SetInsertPoint(IPBB);
2074 CodeGenFunction::JumpDest Dest = CGF.getJumpDestInCurrentScope(Target: DestBB);
2075 CGF.EmitBranchThroughCleanup(Dest);
2076 }
2077
2078 /// Emit the body of an OMP region
2079 /// \param CGF The Codegen function this belongs to
2080 /// \param RegionBodyStmt The body statement for the OpenMP region being
2081 /// generated
2082 /// \param AllocaIP Where to insert alloca instructions
2083 /// \param CodeGenIP Where to insert the region code
2084 /// \param RegionName Name to be used for new blocks
2085 static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF,
2086 const Stmt *RegionBodyStmt,
2087 InsertPointTy AllocaIP,
2088 InsertPointTy CodeGenIP,
2089 Twine RegionName);
2090
2091 static void EmitCaptureStmt(CodeGenFunction &CGF,
2092 llvm::BasicBlock *CodeGenIPBB,
2093 llvm::BasicBlock &FiniBB, llvm::Function *Fn,
2094 ArrayRef<llvm::Value *> Args) {
2095 if (llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminatorOrNull())
2096 CodeGenIPBBTI->eraseFromParent();
2097
2098 CGF.Builder.SetInsertPoint(CodeGenIPBB);
2099
2100 if (Fn->doesNotThrow())
2101 CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
2102 else
2103 CGF.EmitRuntimeCall(callee: Fn, args: Args);
2104
2105 if (CGF.Builder.saveIP().isValid())
2106 CGF.Builder.CreateBr(Dest: &FiniBB);
2107 }
2108
2109 /// Emit the body of an OMP region that will be outlined in
2110 /// OpenMPIRBuilder::finalize().
2111 /// \param CGF The Codegen function this belongs to
2112 /// \param RegionBodyStmt The body statement for the OpenMP region being
2113 /// generated
2114 /// \param AllocaIP Where to insert alloca instructions
2115 /// \param CodeGenIP Where to insert the region code
2116 /// \param RegionName Name to be used for new blocks
2117 static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF,
2118 const Stmt *RegionBodyStmt,
2119 InsertPointTy AllocaIP,
2120 InsertPointTy CodeGenIP,
2121 Twine RegionName);
2122
2123 /// RAII for preserving necessary info during Outlined region body codegen.
2124 class OutlinedRegionBodyRAII {
2125
2126 llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
2127 CodeGenFunction::JumpDest OldReturnBlock;
2128 CodeGenFunction &CGF;
2129
2130 public:
2131 OutlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
2132 llvm::BasicBlock &RetBB)
2133 : CGF(cgf) {
2134 assert(AllocaIP.isValid() &&
2135 "Must specify Insertion point for allocas of outlined function");
2136 OldAllocaIP = CGF.AllocaInsertPt;
2137 CGF.AllocaInsertPt = &*AllocaIP;
2138
2139 OldReturnBlock = CGF.ReturnBlock;
2140 CGF.ReturnBlock = CGF.getJumpDestInCurrentScope(Target: &RetBB);
2141 }
2142
2143 ~OutlinedRegionBodyRAII() {
2144 CGF.AllocaInsertPt = OldAllocaIP;
2145 CGF.ReturnBlock = OldReturnBlock;
2146 }
2147 };
2148
2149 /// RAII for preserving necessary info during inlined region body codegen.
2150 class InlinedRegionBodyRAII {
2151
2152 llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
2153 CodeGenFunction &CGF;
2154
2155 public:
2156 InlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
2157 llvm::BasicBlock &FiniBB)
2158 : CGF(cgf) {
2159 // Alloca insertion block should be in the entry block of the containing
2160 // function so it expects an empty AllocaIP in which case will reuse the
2161 // old alloca insertion point, or a new AllocaIP in the same block as
2162 // the old one
2163 assert((!AllocaIP.isValid() ||
2164 CGF.AllocaInsertPt->getParent() == AllocaIP.getNodeParent()) &&
2165 "Insertion point should be in the entry block of containing "
2166 "function!");
2167 OldAllocaIP = CGF.AllocaInsertPt;
2168 if (AllocaIP.isValid())
2169 CGF.AllocaInsertPt = &*AllocaIP;
2170
2171 // TODO: Remove the call, after making sure the counter is not used by
2172 // the EHStack.
2173 // Since this is an inlined region, it should not modify the
2174 // ReturnBlock, and should reuse the one for the enclosing outlined
2175 // region. So, the JumpDest being return by the function is discarded
2176 (void)CGF.getJumpDestInCurrentScope(Target: &FiniBB);
2177 }
2178
2179 ~InlinedRegionBodyRAII() { CGF.AllocaInsertPt = OldAllocaIP; }
2180 };
2181 };
2182
2183private:
2184 /// CXXThisDecl - When generating code for a C++ member function,
2185 /// this will hold the implicit 'this' declaration.
2186 ImplicitParamDecl *CXXABIThisDecl = nullptr;
2187 llvm::Value *CXXABIThisValue = nullptr;
2188 llvm::Value *CXXThisValue = nullptr;
2189 CharUnits CXXABIThisAlignment;
2190 CharUnits CXXThisAlignment;
2191
2192 /// The value of 'this' to use when evaluating CXXDefaultInitExprs within
2193 /// this expression.
2194 Address CXXDefaultInitExprThis = Address::invalid();
2195
2196 /// The current array initialization index when evaluating an
2197 /// ArrayInitIndexExpr within an ArrayInitLoopExpr.
2198 llvm::Value *ArrayInitIndex = nullptr;
2199
2200 /// The values of function arguments to use when evaluating
2201 /// CXXInheritedCtorInitExprs within this context.
2202 CallArgList CXXInheritedCtorInitExprArgs;
2203
2204 /// CXXStructorImplicitParamDecl - When generating code for a constructor or
2205 /// destructor, this will hold the implicit argument (e.g. VTT).
2206 ImplicitParamDecl *CXXStructorImplicitParamDecl = nullptr;
2207 llvm::Value *CXXStructorImplicitParamValue = nullptr;
2208
2209 /// OutermostConditional - Points to the outermost active
2210 /// conditional control. This is used so that we know if a
2211 /// temporary should be destroyed conditionally.
2212 ConditionalEvaluation *OutermostConditional = nullptr;
2213
2214 /// The current lexical scope.
2215 LexicalScope *CurLexicalScope = nullptr;
2216
2217 /// The current source location that should be used for exception
2218 /// handling code.
2219 SourceLocation CurEHLocation;
2220
2221 /// BlockByrefInfos - For each __block variable, contains
2222 /// information about the layout of the variable.
2223 llvm::DenseMap<const ValueDecl *, BlockByrefInfo> BlockByrefInfos;
2224
2225 /// Used by -fsanitize=nullability-return to determine whether the return
2226 /// value can be checked.
2227 llvm::Value *RetValNullabilityPrecondition = nullptr;
2228
2229 /// Check if -fsanitize=nullability-return instrumentation is required for
2230 /// this function.
2231 bool requiresReturnValueNullabilityCheck() const {
2232 return RetValNullabilityPrecondition;
2233 }
2234
2235 /// Used to store precise source locations for return statements by the
2236 /// runtime return value checks.
2237 Address ReturnLocation = Address::invalid();
2238
2239 /// Check if the return value of this function requires sanitization.
2240 bool requiresReturnValueCheck() const;
2241
2242 bool isInAllocaArgument(CGCXXABI &ABI, QualType Ty);
2243 bool hasInAllocaArg(const CXXMethodDecl *MD);
2244
2245 llvm::BasicBlock *TerminateLandingPad = nullptr;
2246 llvm::BasicBlock *TerminateHandler = nullptr;
2247 llvm::SmallVector<llvm::BasicBlock *, 2> TrapBBs;
2248
2249 /// Terminate funclets keyed by parent funclet pad.
2250 llvm::MapVector<llvm::Value *, llvm::BasicBlock *> TerminateFunclets;
2251
2252 /// Largest vector width used in ths function. Will be used to create a
2253 /// function attribute.
2254 unsigned LargestVectorWidth = 0;
2255
2256 /// True if we need emit the life-time markers. This is initially set in
2257 /// the constructor, but could be overwritten to true if this is a coroutine.
2258 bool ShouldEmitLifetimeMarkers;
2259
2260 /// Add OpenCL kernel arg metadata and the kernel attribute metadata to
2261 /// the function metadata.
2262 void EmitKernelMetadata(const FunctionDecl *FD, llvm::Function *Fn);
2263
2264public:
2265 CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext = false);
2266 ~CodeGenFunction();
2267
2268 CodeGenTypes &getTypes() const { return CGM.getTypes(); }
2269 ASTContext &getContext() const { return CGM.getContext(); }
2270 CGDebugInfo *getDebugInfo() {
2271 if (DisableDebugInfo)
2272 return nullptr;
2273 return DebugInfo;
2274 }
2275 void disableDebugInfo() { DisableDebugInfo = true; }
2276 void enableDebugInfo() { DisableDebugInfo = false; }
2277
2278 bool shouldUseFusedARCCalls() {
2279 return CGM.getCodeGenOpts().OptimizationLevel == 0 &&
2280 !SanOpts.has(K: SanitizerKind::Thread);
2281 }
2282
2283 const LangOptions &getLangOpts() const { return CGM.getLangOpts(); }
2284
2285 /// Returns a pointer to the function's exception object and selector slot,
2286 /// which is assigned in every landing pad.
2287 Address getExceptionSlot();
2288 Address getEHSelectorSlot();
2289
2290 /// Returns the contents of the function's exception object and selector
2291 /// slots.
2292 llvm::Value *getExceptionFromSlot();
2293 llvm::Value *getSelectorFromSlot();
2294
2295 RawAddress getNormalCleanupDestSlot();
2296
2297 llvm::BasicBlock *getUnreachableBlock() {
2298 if (!UnreachableBlock) {
2299 UnreachableBlock = createBasicBlock(name: "unreachable");
2300 new llvm::UnreachableInst(getLLVMContext(), UnreachableBlock);
2301 }
2302 return UnreachableBlock;
2303 }
2304
2305 llvm::BasicBlock *getInvokeDest() {
2306 if (!EHStack.requiresLandingPad())
2307 return nullptr;
2308 return getInvokeDestImpl();
2309 }
2310
2311 bool currentFunctionUsesSEHTry() const { return !!CurSEHParent; }
2312
2313 const TargetInfo &getTarget() const { return Target; }
2314 llvm::LLVMContext &getLLVMContext() { return CGM.getLLVMContext(); }
2315
2316 /// Accessors for LocalDeclMap.
2317 DeclMapTy::iterator findLocalDecl(const Decl *D) {
2318 return LocalDeclMap.find(Val: D);
2319 }
2320 DeclMapTy::iterator localDeclMapEnd() { return LocalDeclMap.end(); }
2321 std::pair<DeclMapTy::iterator, bool> insertLocalDecl(const Decl *D,
2322 Address Addr) {
2323 return LocalDeclMap.insert(KV: {D, Addr});
2324 }
2325 void eraseLocalDecl(const Decl *D) { LocalDeclMap.erase(Val: D); }
2326 const TargetCodeGenInfo &getTargetHooks() const {
2327 return CGM.getTargetCodeGenInfo();
2328 }
2329 const FunctionDecl *getCurrentFunctionDecl() const;
2330
2331 //===--------------------------------------------------------------------===//
2332 // Cleanups
2333 //===--------------------------------------------------------------------===//
2334
2335 typedef void Destroyer(CodeGenFunction &CGF, Address addr, QualType ty);
2336
2337 void pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,
2338 Address arrayEndPointer,
2339 QualType elementType,
2340 CharUnits elementAlignment,
2341 Destroyer *destroyer);
2342 void pushRegularPartialArrayCleanup(llvm::Value *arrayBegin,
2343 llvm::Value *arrayEnd,
2344 QualType elementType,
2345 CharUnits elementAlignment,
2346 Destroyer *destroyer);
2347
2348 void pushDestroy(QualType::DestructionKind dtorKind, Address addr,
2349 QualType type);
2350 void pushEHDestroy(QualType::DestructionKind dtorKind, Address addr,
2351 QualType type);
2352 void pushDestroy(CleanupKind kind, Address addr, QualType type,
2353 Destroyer *destroyer, bool useEHCleanupForArray);
2354 void pushDestroyAndDeferDeactivation(QualType::DestructionKind dtorKind,
2355 Address addr, QualType type);
2356 void pushDestroyAndDeferDeactivation(CleanupKind cleanupKind, Address addr,
2357 QualType type, Destroyer *destroyer,
2358 bool useEHCleanupForArray);
2359 void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr,
2360 QualType type, Destroyer *destroyer,
2361 bool useEHCleanupForArray);
2362 void pushLifetimeExtendedDestroy(QualType::DestructionKind dtorKind,
2363 Address addr, QualType type);
2364 void pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete,
2365 llvm::Value *CompletePtr,
2366 QualType ElementType);
2367 void pushStackRestore(CleanupKind kind, Address SPMem);
2368 void pushKmpcAllocFree(CleanupKind Kind,
2369 std::pair<llvm::Value *, llvm::Value *> AddrSizePair);
2370 void emitDestroy(Address addr, QualType type, Destroyer *destroyer,
2371 bool useEHCleanupForArray);
2372 llvm::Function *generateDestroyHelper(Address addr, QualType type,
2373 Destroyer *destroyer,
2374 bool useEHCleanupForArray,
2375 const VarDecl *VD);
2376 void emitArrayDestroy(llvm::Value *begin, llvm::Value *end,
2377 QualType elementType, CharUnits elementAlign,
2378 Destroyer *destroyer, bool checkZeroLength,
2379 bool useEHCleanup);
2380
2381 Destroyer *getDestroyer(QualType::DestructionKind destructionKind);
2382
2383 /// Determines whether an EH cleanup is required to destroy a type
2384 /// with the given destruction kind.
2385 bool needsEHCleanup(QualType::DestructionKind kind) {
2386 switch (kind) {
2387 case QualType::DK_none:
2388 return false;
2389 case QualType::DK_cxx_destructor:
2390 case QualType::DK_objc_weak_lifetime:
2391 case QualType::DK_nontrivial_c_struct:
2392 return getLangOpts().Exceptions;
2393 case QualType::DK_objc_strong_lifetime:
2394 return getLangOpts().Exceptions &&
2395 CGM.getCodeGenOpts().ObjCAutoRefCountExceptions;
2396 }
2397 llvm_unreachable("bad destruction kind");
2398 }
2399
2400 CleanupKind getCleanupKind(QualType::DestructionKind kind) {
2401 return (needsEHCleanup(kind) ? NormalAndEHCleanup : NormalCleanup);
2402 }
2403
2404 //===--------------------------------------------------------------------===//
2405 // Objective-C
2406 //===--------------------------------------------------------------------===//
2407
2408 void GenerateObjCMethod(const ObjCMethodDecl *OMD);
2409
2410 void StartObjCMethod(const ObjCMethodDecl *MD, const ObjCContainerDecl *CD);
2411
2412 /// GenerateObjCGetter - Synthesize an Objective-C property getter function.
2413 void GenerateObjCGetter(ObjCImplementationDecl *IMP,
2414 const ObjCPropertyImplDecl *PID);
2415 void generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
2416 const ObjCPropertyImplDecl *propImpl,
2417 const ObjCMethodDecl *GetterMothodDecl,
2418 llvm::Constant *AtomicHelperFn);
2419
2420 void GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
2421 ObjCMethodDecl *MD, bool ctor);
2422
2423 /// GenerateObjCSetter - Synthesize an Objective-C property setter function
2424 /// for the given property.
2425 void GenerateObjCSetter(ObjCImplementationDecl *IMP,
2426 const ObjCPropertyImplDecl *PID);
2427 void generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
2428 const ObjCPropertyImplDecl *propImpl,
2429 llvm::Constant *AtomicHelperFn);
2430
2431 //===--------------------------------------------------------------------===//
2432 // Block Bits
2433 //===--------------------------------------------------------------------===//
2434
2435 /// Emit block literal.
2436 /// \return an LLVM value which is a pointer to a struct which contains
2437 /// information about the block, including the block invoke function, the
2438 /// captured variables, etc.
2439 llvm::Value *EmitBlockLiteral(const BlockExpr *);
2440
2441 llvm::Function *GenerateBlockFunction(GlobalDecl GD, const CGBlockInfo &Info,
2442 const DeclMapTy &ldm,
2443 bool IsLambdaConversionToBlock,
2444 bool BuildGlobalBlock);
2445
2446 /// Check if \p T is a C++ class that has a destructor that can throw.
2447 static bool cxxDestructorCanThrow(QualType T);
2448
2449 llvm::Constant *GenerateCopyHelperFunction(const CGBlockInfo &blockInfo);
2450 llvm::Constant *GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo);
2451 llvm::Constant *
2452 GenerateObjCAtomicSetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);
2453 llvm::Constant *
2454 GenerateObjCAtomicGetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);
2455 llvm::Value *EmitBlockCopyAndAutorelease(llvm::Value *Block, QualType Ty);
2456
2457 void BuildBlockRelease(llvm::Value *DeclPtr, BlockFieldFlags flags,
2458 bool CanThrow);
2459
2460 class AutoVarEmission;
2461
2462 void emitByrefStructureInit(const AutoVarEmission &emission);
2463
2464 /// Enter a cleanup to destroy a __block variable. Note that this
2465 /// cleanup should be a no-op if the variable hasn't left the stack
2466 /// yet; if a cleanup is required for the variable itself, that needs
2467 /// to be done externally.
2468 ///
2469 /// \param Kind Cleanup kind.
2470 ///
2471 /// \param Addr When \p LoadBlockVarAddr is false, the address of the __block
2472 /// structure that will be passed to _Block_object_dispose. When
2473 /// \p LoadBlockVarAddr is true, the address of the field of the block
2474 /// structure that holds the address of the __block structure.
2475 ///
2476 /// \param Flags The flag that will be passed to _Block_object_dispose.
2477 ///
2478 /// \param LoadBlockVarAddr Indicates whether we need to emit a load from
2479 /// \p Addr to get the address of the __block structure.
2480 void enterByrefCleanup(CleanupKind Kind, Address Addr, BlockFieldFlags Flags,
2481 bool LoadBlockVarAddr, bool CanThrow);
2482
2483 void setBlockContextParameter(const ImplicitParamDecl *D, unsigned argNum,
2484 llvm::Value *ptr);
2485
2486 Address LoadBlockStruct();
2487 Address GetAddrOfBlockDecl(const VarDecl *var);
2488
2489 /// BuildBlockByrefAddress - Computes the location of the
2490 /// data in a variable which is declared as __block.
2491 Address emitBlockByrefAddress(Address baseAddr, const VarDecl *V,
2492 bool followForward = true);
2493 Address emitBlockByrefAddress(Address baseAddr, const BlockByrefInfo &info,
2494 bool followForward, const llvm::Twine &name);
2495
2496 const BlockByrefInfo &getBlockByrefInfo(const VarDecl *var);
2497
2498 QualType BuildFunctionArgList(GlobalDecl GD, FunctionArgList &Args);
2499
2500 void GenerateCode(GlobalDecl GD, llvm::Function *Fn,
2501 const CGFunctionInfo &FnInfo);
2502
2503 /// Annotate the function with an attribute that disables TSan checking at
2504 /// runtime.
2505 void markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn);
2506
2507 /// Emit code for the start of a function.
2508 /// \param Loc The location to be associated with the function.
2509 /// \param StartLoc The location of the function body.
2510 void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn,
2511 const CGFunctionInfo &FnInfo, const FunctionArgList &Args,
2512 SourceLocation Loc = SourceLocation(),
2513 SourceLocation StartLoc = SourceLocation());
2514
2515 static bool IsConstructorDelegationValid(const CXXConstructorDecl *Ctor);
2516
2517 void EmitConstructorBody(FunctionArgList &Args);
2518 void EmitDestructorBody(FunctionArgList &Args);
2519 void emitImplicitAssignmentOperatorBody(FunctionArgList &Args);
2520 void EmitFunctionBody(const Stmt *Body);
2521 void EmitBlockWithFallThrough(llvm::BasicBlock *BB, const Stmt *S);
2522
2523 void EmitForwardingCallToLambda(const CXXMethodDecl *LambdaCallOperator,
2524 CallArgList &CallArgs,
2525 const CGFunctionInfo *CallOpFnInfo = nullptr,
2526 llvm::Constant *CallOpFn = nullptr);
2527 void EmitLambdaBlockInvokeBody();
2528 void EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD);
2529 void EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,
2530 CallArgList &CallArgs);
2531 void EmitLambdaInAllocaImplFn(const CXXMethodDecl *CallOp,
2532 const CGFunctionInfo **ImplFnInfo,
2533 llvm::Function **ImplFn);
2534 void EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD);
2535 void EmitLambdaVLACapture(const VariableArrayType *VAT, LValue LV) {
2536 EmitStoreThroughLValue(Src: RValue::get(V: VLASizeMap[VAT->getSizeExpr()]), Dst: LV);
2537 }
2538 void EmitAsanPrologueOrEpilogue(bool Prologue);
2539
2540 /// Emit the unified return block, trying to avoid its emission when
2541 /// possible.
2542 /// \return The debug location of the user written return statement if the
2543 /// return block is avoided.
2544 llvm::DebugLoc EmitReturnBlock();
2545
2546 /// FinishFunction - Complete IR generation of the current function. It is
2547 /// legal to call this function even if there is no current insertion point.
2548 void FinishFunction(SourceLocation EndLoc = SourceLocation());
2549
2550 void StartThunk(llvm::Function *Fn, GlobalDecl GD,
2551 const CGFunctionInfo &FnInfo, bool IsUnprototyped);
2552
2553 void EmitCallAndReturnForThunk(llvm::FunctionCallee Callee,
2554 const ThunkInfo *Thunk, bool IsUnprototyped);
2555
2556 void FinishThunk();
2557
2558 /// Start an Objective-C direct method thunk.
2559 void StartObjCDirectPreconditionThunk(const ObjCMethodDecl *OMD,
2560 llvm::Function *Fn,
2561 const CGFunctionInfo &FI);
2562
2563 /// Finish an Objective-C direct method thunk.
2564 void FinishObjCDirectPreconditionThunk();
2565
2566 /// Emit a musttail call for a thunk with a potentially adjusted this pointer.
2567 void EmitMustTailThunk(GlobalDecl GD, llvm::Value *AdjustedThisPtr,
2568 llvm::FunctionCallee Callee);
2569
2570 /// Generate a thunk for the given method.
2571 void generateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo,
2572 GlobalDecl GD, const ThunkInfo &Thunk,
2573 bool IsUnprototyped);
2574
2575 llvm::Function *GenerateVarArgsThunk(llvm::Function *Fn,
2576 const CGFunctionInfo &FnInfo,
2577 GlobalDecl GD, const ThunkInfo &Thunk);
2578
2579 void EmitCtorPrologue(const CXXConstructorDecl *CD, CXXCtorType Type,
2580 FunctionArgList &Args);
2581
2582 void EmitInitializerForField(FieldDecl *Field, LValue LHS, Expr *Init);
2583
2584 /// Struct with all information about dynamic [sub]class needed to set vptr.
2585 struct VPtr {
2586 BaseSubobject Base;
2587 const CXXRecordDecl *NearestVBase;
2588 CharUnits OffsetFromNearestVBase;
2589 const CXXRecordDecl *VTableClass;
2590 };
2591
2592 /// Initialize the vtable pointer of the given subobject.
2593 void InitializeVTablePointer(const VPtr &vptr);
2594
2595 typedef llvm::SmallVector<VPtr, 4> VPtrsVector;
2596
2597 typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
2598 VPtrsVector getVTablePointers(const CXXRecordDecl *VTableClass);
2599
2600 void getVTablePointers(BaseSubobject Base, const CXXRecordDecl *NearestVBase,
2601 CharUnits OffsetFromNearestVBase,
2602 bool BaseIsNonVirtualPrimaryBase,
2603 const CXXRecordDecl *VTableClass,
2604 VisitedVirtualBasesSetTy &VBases, VPtrsVector &vptrs);
2605
2606 void InitializeVTablePointers(const CXXRecordDecl *ClassDecl);
2607
2608 // VTableTrapMode - whether we guarantee that loading the
2609 // vtable is guaranteed to trap on authentication failure,
2610 // even if the resulting vtable pointer is unused.
2611 enum class VTableAuthMode {
2612 Authenticate,
2613 MustTrap,
2614 UnsafeUbsanStrip // Should only be used for Vptr UBSan check
2615 };
2616 /// GetVTablePtr - Return the Value of the vtable pointer member pointed
2617 /// to by This.
2618 llvm::Value *
2619 GetVTablePtr(Address This, llvm::Type *VTableTy,
2620 const CXXRecordDecl *VTableClass,
2621 VTableAuthMode AuthMode = VTableAuthMode::Authenticate);
2622
2623 enum CFITypeCheckKind {
2624 CFITCK_VCall,
2625 CFITCK_NVCall,
2626 CFITCK_DerivedCast,
2627 CFITCK_UnrelatedCast,
2628 CFITCK_ICall,
2629 CFITCK_NVMFCall,
2630 CFITCK_VMFCall,
2631 };
2632
2633 /// Derived is the presumed address of an object of type T after a
2634 /// cast. If T is a polymorphic class type, emit a check that the virtual
2635 /// table for Derived belongs to a class derived from T.
2636 void EmitVTablePtrCheckForCast(QualType T, Address Derived, bool MayBeNull,
2637 CFITypeCheckKind TCK, SourceLocation Loc);
2638
2639 /// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable.
2640 /// If vptr CFI is enabled, emit a check that VTable is valid.
2641 void EmitVTablePtrCheckForCall(const CXXRecordDecl *RD, llvm::Value *VTable,
2642 CFITypeCheckKind TCK, SourceLocation Loc);
2643
2644 /// EmitVTablePtrCheck - Emit a check that VTable is a valid virtual table for
2645 /// RD using llvm.type.test.
2646 void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable,
2647 CFITypeCheckKind TCK, SourceLocation Loc);
2648
2649 /// Emit a CFI check failure handler or trap for a vtable type test result.
2650 void EmitVTableCheck(CFITypeCheckKind TCK, CanQualType T, llvm::Metadata *MD,
2651 llvm::Value *VTable, llvm::Value *TypeTest,
2652 SourceLocation Loc, bool UseCrossDsoSlowPath);
2653
2654 /// If whole-program virtual table optimization is enabled, emit an assumption
2655 /// that VTable is a member of RD's type identifier. Or, if vptr CFI is
2656 /// enabled, emit a check that VTable is a member of RD's type identifier.
2657 void EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2658 llvm::Value *VTable, SourceLocation Loc);
2659
2660 /// Returns whether we should perform a type checked load when loading a
2661 /// virtual function for virtual calls to members of RD. This is generally
2662 /// true when both vcall CFI and whole-program-vtables are enabled.
2663 bool ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD);
2664
2665 /// Emit a type checked load from the given vtable.
2666 llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD,
2667 llvm::Value *VTable,
2668 llvm::Type *VTableTy,
2669 uint64_t VTableByteOffset,
2670 SourceLocation Loc);
2671
2672 /// EnterDtorCleanups - Enter the cleanups necessary to complete the
2673 /// given phase of destruction for a destructor. The end result
2674 /// should call destructors on members and base classes in reverse
2675 /// order of their construction.
2676 void EnterDtorCleanups(const CXXDestructorDecl *Dtor, CXXDtorType Type);
2677
2678 /// ShouldInstrumentFunction - Return true if the current function should be
2679 /// instrumented with __cyg_profile_func_* calls
2680 bool ShouldInstrumentFunction();
2681
2682 /// ShouldSkipSanitizerInstrumentation - Return true if the current function
2683 /// should not be instrumented with sanitizers.
2684 bool ShouldSkipSanitizerInstrumentation();
2685
2686 /// ShouldXRayInstrument - Return true if the current function should be
2687 /// instrumented with XRay nop sleds.
2688 bool ShouldXRayInstrumentFunction() const;
2689
2690 /// AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit
2691 /// XRay custom event handling calls.
2692 bool AlwaysEmitXRayCustomEvents() const;
2693
2694 /// AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit
2695 /// XRay typed event handling calls.
2696 bool AlwaysEmitXRayTypedEvents() const;
2697
2698 /// Return a type hash constant for a function instrumented by
2699 /// -fsanitize=function.
2700 llvm::ConstantInt *getUBSanFunctionTypeHash(QualType T) const;
2701
2702 /// EmitFunctionProlog - Emit the target specific LLVM code to load the
2703 /// arguments for the given function. This is also responsible for naming the
2704 /// LLVM function arguments.
2705 void EmitFunctionProlog(const CGFunctionInfo &FI, llvm::Function *Fn,
2706 const FunctionArgList &Args);
2707
2708 /// EmitFunctionEpilog - Emit the target specific LLVM code to return the
2709 /// given temporary. Specify the source location atom group (Key Instructions
2710 /// debug info feature) for the `ret` using \p RetKeyInstructionsSourceAtom.
2711 /// If it's 0, the `ret` will get added to a new source atom group.
2712 void EmitFunctionEpilog(const CGFunctionInfo &FI, bool EmitRetDbgLoc,
2713 SourceLocation EndLoc,
2714 uint64_t RetKeyInstructionsSourceAtom);
2715
2716 /// Emit a test that checks if the return value \p RV is nonnull.
2717 void EmitReturnValueCheck(llvm::Value *RV);
2718
2719 /// EmitStartEHSpec - Emit the start of the exception spec.
2720 void EmitStartEHSpec(const Decl *D);
2721
2722 /// EmitEndEHSpec - Emit the end of the exception spec.
2723 void EmitEndEHSpec(const Decl *D);
2724
2725 /// getTerminateLandingPad - Return a landing pad that just calls terminate.
2726 llvm::BasicBlock *getTerminateLandingPad();
2727
2728 /// getTerminateLandingPad - Return a cleanup funclet that just calls
2729 /// terminate.
2730 llvm::BasicBlock *getTerminateFunclet();
2731
2732 /// getTerminateHandler - Return a handler (not a landing pad, just
2733 /// a catch handler) that just calls terminate. This is used when
2734 /// a terminate scope encloses a try.
2735 llvm::BasicBlock *getTerminateHandler();
2736
2737 llvm::Type *ConvertTypeForMem(QualType T);
2738 llvm::Type *ConvertType(QualType T);
2739 llvm::Type *convertTypeForLoadStore(QualType ASTTy,
2740 llvm::Type *LLVMTy = nullptr);
2741 llvm::Type *ConvertType(const TypeDecl *T) {
2742 return ConvertType(T: getContext().getTypeDeclType(Decl: T));
2743 }
2744
2745 /// LoadObjCSelf - Load the value of self. This function is only valid while
2746 /// generating code for an Objective-C method.
2747 llvm::Value *LoadObjCSelf();
2748
2749 /// TypeOfSelfObject - Return type of object that this self represents.
2750 QualType TypeOfSelfObject();
2751
2752 /// getEvaluationKind - Return the TypeEvaluationKind of QualType \c T.
2753 static TypeEvaluationKind getEvaluationKind(QualType T);
2754
2755 static bool hasScalarEvaluationKind(QualType T) {
2756 return getEvaluationKind(T) == TEK_Scalar;
2757 }
2758
2759 static bool hasAggregateEvaluationKind(QualType T) {
2760 return getEvaluationKind(T) == TEK_Aggregate;
2761 }
2762
2763 /// createBasicBlock - Create an LLVM basic block.
2764 llvm::BasicBlock *createBasicBlock(const Twine &name = "",
2765 llvm::Function *parent = nullptr,
2766 llvm::BasicBlock *before = nullptr) {
2767 return llvm::BasicBlock::Create(Context&: getLLVMContext(), Name: name, Parent: parent, InsertBefore: before);
2768 }
2769
2770 /// getBasicBlockForLabel - Return the LLVM basicblock that the specified
2771 /// label maps to.
2772 JumpDest getJumpDestForLabel(const LabelDecl *S);
2773
2774 /// SimplifyForwardingBlocks - If the given basic block is only a branch to
2775 /// another basic block, simplify it. This assumes that no other code could
2776 /// potentially reference the basic block.
2777 void SimplifyForwardingBlocks(llvm::BasicBlock *BB);
2778
2779 /// EmitBlock - Emit the given block \arg BB and set it as the insert point,
2780 /// adding a fall-through branch from the current insert block if
2781 /// necessary. It is legal to call this function even if there is no current
2782 /// insertion point.
2783 ///
2784 /// IsFinished - If true, indicates that the caller has finished emitting
2785 /// branches to the given block and does not expect to emit code into it. This
2786 /// means the block can be ignored if it is unreachable.
2787 void EmitBlock(llvm::BasicBlock *BB, bool IsFinished = false);
2788
2789 /// EmitBlockAfterUses - Emit the given block somewhere hopefully
2790 /// near its uses, and leave the insertion point in it.
2791 void EmitBlockAfterUses(llvm::BasicBlock *BB);
2792
2793 /// EmitBranch - Emit a branch to the specified basic block from the current
2794 /// insert block, taking care to avoid creation of branches from dummy
2795 /// blocks. It is legal to call this function even if there is no current
2796 /// insertion point.
2797 ///
2798 /// This function clears the current insertion point. The caller should follow
2799 /// calls to this function with calls to Emit*Block prior to generation new
2800 /// code.
2801 void EmitBranch(llvm::BasicBlock *Block);
2802
2803 /// HaveInsertPoint - True if an insertion point is defined. If not, this
2804 /// indicates that the current code being emitted is unreachable.
2805 bool HaveInsertPoint() const { return Builder.GetInsertBlock() != nullptr; }
2806
2807 /// EnsureInsertPoint - Ensure that an insertion point is defined so that
2808 /// emitted IR has a place to go. Note that by definition, if this function
2809 /// creates a block then that block is unreachable; callers may do better to
2810 /// detect when no insertion point is defined and simply skip IR generation.
2811 void EnsureInsertPoint() {
2812 if (!HaveInsertPoint())
2813 EmitBlock(BB: createBasicBlock());
2814 }
2815
2816 /// ErrorUnsupported - Print out an error that codegen doesn't support the
2817 /// specified stmt yet.
2818 void ErrorUnsupported(const Stmt *S, const char *Type);
2819
2820 //===--------------------------------------------------------------------===//
2821 // Helpers
2822 //===--------------------------------------------------------------------===//
2823
2824 Address mergeAddressesInConditionalExpr(Address LHS, Address RHS,
2825 llvm::BasicBlock *LHSBlock,
2826 llvm::BasicBlock *RHSBlock,
2827 llvm::BasicBlock *MergeBlock,
2828 QualType MergedType) {
2829 Builder.SetInsertPoint(MergeBlock);
2830 llvm::PHINode *PtrPhi = Builder.CreatePHI(Ty: LHS.getType(), NumReservedValues: 2, Name: "cond");
2831 PtrPhi->addIncoming(V: LHS.getBasePointer(), BB: LHSBlock);
2832 PtrPhi->addIncoming(V: RHS.getBasePointer(), BB: RHSBlock);
2833 LHS.replaceBasePointer(P: PtrPhi);
2834 LHS.setAlignment(std::min(a: LHS.getAlignment(), b: RHS.getAlignment()));
2835 return LHS;
2836 }
2837
2838 /// Construct an address with the natural alignment of T. If a pointer to T
2839 /// is expected to be signed, the pointer passed to this function must have
2840 /// been signed, and the returned Address will have the pointer authentication
2841 /// information needed to authenticate the signed pointer.
2842 Address makeNaturalAddressForPointer(
2843 llvm::Value *Ptr, QualType T, CharUnits Alignment = CharUnits::Zero(),
2844 bool ForPointeeType = false, LValueBaseInfo *BaseInfo = nullptr,
2845 TBAAAccessInfo *TBAAInfo = nullptr,
2846 KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
2847 if (Alignment.isZero())
2848 Alignment =
2849 CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, forPointeeType: ForPointeeType);
2850 return Address(Ptr, ConvertTypeForMem(T), Alignment,
2851 CGM.getPointerAuthInfoForPointeeType(type: T), /*Offset=*/nullptr,
2852 IsKnownNonNull);
2853 }
2854
2855 LValue MakeAddrLValue(Address Addr, QualType T,
2856 AlignmentSource Source = AlignmentSource::Type) {
2857 return MakeAddrLValue(Addr, T, BaseInfo: LValueBaseInfo(Source),
2858 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2859 }
2860
2861 LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo,
2862 TBAAAccessInfo TBAAInfo) {
2863 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo, TBAAInfo);
2864 }
2865
2866 LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2867 AlignmentSource Source = AlignmentSource::Type) {
2868 return MakeAddrLValue(Addr: makeNaturalAddressForPointer(Ptr: V, T, Alignment), T,
2869 BaseInfo: LValueBaseInfo(Source), TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2870 }
2871
2872 /// Same as MakeAddrLValue above except that the pointer is known to be
2873 /// unsigned.
2874 LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2875 AlignmentSource Source = AlignmentSource::Type) {
2876 Address Addr(V, ConvertTypeForMem(T), Alignment);
2877 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2878 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2879 }
2880
2881 LValue
2882 MakeAddrLValueWithoutTBAA(Address Addr, QualType T,
2883 AlignmentSource Source = AlignmentSource::Type) {
2884 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2885 TBAAInfo: TBAAAccessInfo());
2886 }
2887
2888 /// Given a value of type T* that may not be to a complete object, construct
2889 /// an l-value with the natural pointee alignment of T.
2890 LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T);
2891
2892 LValue
2893 MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
2894 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
2895
2896 /// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known
2897 /// to be unsigned.
2898 LValue MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, QualType T);
2899
2900 LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T);
2901
2902 Address EmitLoadOfReference(LValue RefLVal,
2903 LValueBaseInfo *PointeeBaseInfo = nullptr,
2904 TBAAAccessInfo *PointeeTBAAInfo = nullptr);
2905 LValue EmitLoadOfReferenceLValue(LValue RefLVal);
2906 LValue
2907 EmitLoadOfReferenceLValue(Address RefAddr, QualType RefTy,
2908 AlignmentSource Source = AlignmentSource::Type) {
2909 LValue RefLVal = MakeAddrLValue(Addr: RefAddr, T: RefTy, BaseInfo: LValueBaseInfo(Source),
2910 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: RefTy));
2911 return EmitLoadOfReferenceLValue(RefLVal);
2912 }
2913
2914 /// Load a pointer with type \p PtrTy stored at address \p Ptr.
2915 /// Note that \p PtrTy is the type of the loaded pointer, not the addresses
2916 /// it is loaded from.
2917 Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy,
2918 LValueBaseInfo *BaseInfo = nullptr,
2919 TBAAAccessInfo *TBAAInfo = nullptr);
2920 LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy);
2921
2922private:
2923 struct AllocaTracker {
2924 void Add(llvm::AllocaInst *I) { Allocas.push_back(Elt: I); }
2925 llvm::SmallVector<llvm::AllocaInst *> Take() { return std::move(Allocas); }
2926
2927 private:
2928 llvm::SmallVector<llvm::AllocaInst *> Allocas;
2929 };
2930 AllocaTracker *Allocas = nullptr;
2931
2932 /// CGDecl helper.
2933 void emitStoresForConstant(const VarDecl &D, Address Loc, bool isVolatile,
2934 llvm::Constant *constant, bool IsAutoInit);
2935 /// CGDecl helper.
2936 void emitStoresForZeroInit(const VarDecl &D, Address Loc, bool isVolatile);
2937 /// CGDecl helper.
2938 void emitStoresForPatternInit(const VarDecl &D, Address Loc, bool isVolatile);
2939 /// CGDecl helper.
2940 void emitStoresForInitAfterBZero(llvm::Constant *Init, Address Loc,
2941 bool isVolatile, bool IsAutoInit);
2942
2943public:
2944 // Captures all the allocas created during the scope of its RAII object.
2945 struct AllocaTrackerRAII {
2946 AllocaTrackerRAII(CodeGenFunction &CGF)
2947 : CGF(CGF), OldTracker(CGF.Allocas) {
2948 CGF.Allocas = &Tracker;
2949 }
2950 ~AllocaTrackerRAII() { CGF.Allocas = OldTracker; }
2951
2952 llvm::SmallVector<llvm::AllocaInst *> Take() { return Tracker.Take(); }
2953
2954 private:
2955 CodeGenFunction &CGF;
2956 AllocaTracker *OldTracker;
2957 AllocaTracker Tracker;
2958 };
2959
2960private:
2961 /// If \p Alloca is not in the same address space as \p DestLangAS, insert an
2962 /// address space cast and return a new RawAddress based on this value.
2963 RawAddress MaybeCastStackAddressSpace(RawAddress Alloca, LangAS DestLangAS,
2964 llvm::Value *ArraySize = nullptr);
2965
2966public:
2967 /// CreateTempAlloca - This creates an alloca and inserts it into the entry
2968 /// block if \p ArraySize is nullptr, otherwise inserts it at the current
2969 /// insertion point of the builder. The caller is responsible for setting an
2970 /// appropriate alignment on the alloca.
2971 ///
2972 /// \p ArraySize is the number of array elements to be allocated if it
2973 /// is not nullptr.
2974 ///
2975 /// LangAS::Default is the address space of pointers to local variables and
2976 /// temporaries, as exposed in the source language. In certain
2977 /// configurations, this is not the same as the alloca address space, and a
2978 /// cast is needed to lift the pointer from the alloca AS into
2979 /// LangAS::Default. This can happen when the target uses a restricted
2980 /// address space for the stack but the source language requires
2981 /// LangAS::Default to be a generic address space. The latter condition is
2982 /// common for most programming languages; OpenCL is an exception in that
2983 /// LangAS::Default is the private address space, which naturally maps
2984 /// to the stack.
2985 ///
2986 /// Because the address of a temporary is often exposed to the program in
2987 /// various ways, this function will perform the cast. The original alloca
2988 /// instruction is returned through \p Alloca if it is not nullptr.
2989 ///
2990 /// The cast is not performed in CreateTempAllocaWithoutCast. This is
2991 /// more efficient if the caller knows that the address will not be exposed.
2992 llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp",
2993 llvm::Value *ArraySize = nullptr);
2994
2995 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
2996 /// block. The alloca is casted to the address space of \p UseAddrSpace if
2997 /// necessary.
2998 RawAddress CreateTempAlloca(llvm::Type *Ty, LangAS UseAddrSpace,
2999 CharUnits align, const Twine &Name = "tmp",
3000 llvm::Value *ArraySize = nullptr,
3001 RawAddress *Alloca = nullptr);
3002
3003 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
3004 /// block. The alloca is casted to default address space if necessary.
3005 ///
3006 /// FIXME: This version should be removed, and context should provide the
3007 /// context use address space used instead of default.
3008 RawAddress CreateTempAlloca(llvm::Type *Ty, CharUnits align,
3009 const Twine &Name = "tmp",
3010 llvm::Value *ArraySize = nullptr,
3011 RawAddress *Alloca = nullptr) {
3012 return CreateTempAlloca(Ty, UseAddrSpace: LangAS::Default, align, Name, ArraySize,
3013 Alloca);
3014 }
3015
3016 RawAddress CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align,
3017 const Twine &Name = "tmp",
3018 llvm::Value *ArraySize = nullptr);
3019
3020 /// CreateDefaultAlignedTempAlloca - This creates an alloca with the
3021 /// default ABI alignment of the given LLVM type.
3022 ///
3023 /// IMPORTANT NOTE: This is *not* generally the right alignment for
3024 /// any given AST type that happens to have been lowered to the
3025 /// given IR type. This should only ever be used for function-local,
3026 /// IR-driven manipulations like saving and restoring a value. Do
3027 /// not hand this address off to arbitrary IRGen routines, and especially
3028 /// do not pass it as an argument to a function that might expect a
3029 /// properly ABI-aligned value.
3030 RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty,
3031 const Twine &Name = "tmp");
3032
3033 /// CreateIRTempWithoutCast - Create a temporary IR object of the given type,
3034 /// with appropriate alignment. This routine should only be used when an
3035 /// temporary value needs to be stored into an alloca (for example, to avoid
3036 /// explicit PHI construction), but the type is the IR type, not the type
3037 /// appropriate for storing in memory.
3038 ///
3039 /// That is, this is exactly equivalent to CreateMemTemp, but calling
3040 /// ConvertType instead of ConvertTypeForMem.
3041 RawAddress CreateIRTempWithoutCast(QualType T, const Twine &Name = "tmp");
3042
3043 /// CreateMemTemp - Create a temporary memory object of the given type, with
3044 /// appropriate alignmen and cast it to the default address space. Returns
3045 /// the original alloca instruction by \p Alloca if it is not nullptr.
3046 RawAddress CreateMemTemp(QualType T, const Twine &Name = "tmp",
3047 RawAddress *Alloca = nullptr);
3048 RawAddress CreateMemTemp(QualType T, CharUnits Align,
3049 const Twine &Name = "tmp",
3050 RawAddress *Alloca = nullptr);
3051
3052 /// CreateMemTemp - Create a temporary memory object of the given type, with
3053 /// appropriate alignmen without casting it to the default address space.
3054 RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp");
3055 RawAddress CreateMemTempWithoutCast(QualType T, CharUnits Align,
3056 const Twine &Name = "tmp");
3057
3058 /// CreateAggTemp - Create a temporary memory object for the given
3059 /// aggregate type.
3060 AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp",
3061 RawAddress *Alloca = nullptr) {
3062 RawAddress Addr = CreateMemTemp(T, Name, Alloca);
3063 return AggValueSlot::forAddr(
3064 addr: Addr, quals: T.getQualifiers(), isDestructed: AggValueSlot::IsNotDestructed,
3065 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
3066 mayOverlap: AggValueSlot::DoesNotOverlap);
3067 }
3068
3069 /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
3070 /// expression and compare the result against zero, returning an Int1Ty value.
3071 llvm::Value *EvaluateExprAsBool(const Expr *E);
3072
3073 /// Retrieve the implicit cast expression of the rhs in a binary operator
3074 /// expression by passing pointers to Value and QualType
3075 /// This is used for implicit bitfield conversion checks, which
3076 /// must compare with the value before potential truncation.
3077 llvm::Value *EmitWithOriginalRHSBitfieldAssignment(const BinaryOperator *E,
3078 llvm::Value **Previous,
3079 QualType *SrcType);
3080
3081 /// Emit a check that an [implicit] conversion of a bitfield. It is not UB,
3082 /// so we use the value after conversion.
3083 void EmitBitfieldConversionCheck(llvm::Value *Src, QualType SrcType,
3084 llvm::Value *Dst, QualType DstType,
3085 const CGBitFieldInfo &Info,
3086 SourceLocation Loc);
3087
3088 /// EmitIgnoredExpr - Emit an expression in a context which ignores the
3089 /// result.
3090 void EmitIgnoredExpr(const Expr *E);
3091
3092 /// EmitAnyExpr - Emit code to compute the specified expression which can have
3093 /// any type. The result is returned as an RValue struct. If this is an
3094 /// aggregate expression, the aggloc/agglocvolatile arguments indicate where
3095 /// the result should be returned.
3096 ///
3097 /// \param ignoreResult True if the resulting value isn't used.
3098 RValue EmitAnyExpr(const Expr *E,
3099 AggValueSlot aggSlot = AggValueSlot::ignored(),
3100 bool ignoreResult = false);
3101
3102 // EmitVAListRef - Emit a "reference" to a va_list; this is either the address
3103 // or the value of the expression, depending on how va_list is defined.
3104 Address EmitVAListRef(const Expr *E);
3105
3106 /// Emit a "reference" to a __builtin_ms_va_list; this is
3107 /// always the value of the expression, because a __builtin_ms_va_list is a
3108 /// pointer to a char.
3109 Address EmitMSVAListRef(const Expr *E);
3110
3111 /// Emit a "reference" to a __builtin_zos_va_list; this is always the
3112 /// address of the expression, because a __builtin_zos_va_list is an
3113 /// array of pointer to a char.
3114 Address EmitZOSVAListRef(const Expr *E);
3115
3116 /// EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will
3117 /// always be accessible even if no aggregate location is provided.
3118 RValue EmitAnyExprToTemp(const Expr *E);
3119
3120 /// EmitAnyExprToMem - Emits the code necessary to evaluate an
3121 /// arbitrary expression into the given memory location.
3122 void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals,
3123 bool IsInitializer);
3124
3125 void EmitAnyExprToExn(const Expr *E, Address Addr);
3126
3127 /// EmitInitializationToLValue - Emit an initializer to an LValue.
3128 void EmitInitializationToLValue(
3129 const Expr *E, LValue LV,
3130 AggValueSlot::IsZeroed_t IsZeroed = AggValueSlot::IsNotZeroed);
3131
3132 /// EmitExprAsInit - Emits the code necessary to initialize a
3133 /// location in memory with the given initializer.
3134 void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue,
3135 bool capturedByInit);
3136
3137 /// hasVolatileMember - returns true if aggregate type has a volatile
3138 /// member.
3139 bool hasVolatileMember(QualType T) {
3140 if (const auto *RD = T->getAsRecordDecl())
3141 return RD->hasVolatileMember();
3142 return false;
3143 }
3144
3145 /// Determine whether a return value slot may overlap some other object.
3146 AggValueSlot::Overlap_t getOverlapForReturnValue() {
3147 // FIXME: Assuming no overlap here breaks guaranteed copy elision for base
3148 // class subobjects. These cases may need to be revisited depending on the
3149 // resolution of the relevant core issue.
3150 return AggValueSlot::DoesNotOverlap;
3151 }
3152
3153 /// Determine whether a field initialization may overlap some other object.
3154 AggValueSlot::Overlap_t getOverlapForFieldInit(const FieldDecl *FD);
3155
3156 /// Determine whether a base class initialization may overlap some other
3157 /// object.
3158 AggValueSlot::Overlap_t getOverlapForBaseInit(const CXXRecordDecl *RD,
3159 const CXXRecordDecl *BaseRD,
3160 bool IsVirtual);
3161
3162 /// Emit an aggregate assignment.
3163 void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy) {
3164 ApplyAtomGroup Grp(getDebugInfo());
3165 bool IsVolatile = hasVolatileMember(T: EltTy);
3166 EmitAggregateCopy(Dest, Src, EltTy, MayOverlap: AggValueSlot::MayOverlap, isVolatile: IsVolatile);
3167 }
3168
3169 void EmitAggregateCopyCtor(LValue Dest, LValue Src,
3170 AggValueSlot::Overlap_t MayOverlap) {
3171 EmitAggregateCopy(Dest, Src, EltTy: Src.getType(), MayOverlap);
3172 }
3173
3174 /// EmitAggregateCopy - Emit an aggregate copy.
3175 ///
3176 /// \param isVolatile \c true iff either the source or the destination is
3177 /// volatile.
3178 /// \param MayOverlap Whether the tail padding of the destination might be
3179 /// occupied by some other object. More efficient code can often be
3180 /// generated if not.
3181 void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy,
3182 AggValueSlot::Overlap_t MayOverlap,
3183 bool isVolatile = false);
3184
3185 /// GetAddrOfLocalVar - Return the address of a local variable.
3186 Address GetAddrOfLocalVar(const VarDecl *VD) {
3187 auto it = LocalDeclMap.find(Val: VD);
3188 assert(it != LocalDeclMap.end() &&
3189 "Invalid argument to GetAddrOfLocalVar(), no decl!");
3190 return it->second;
3191 }
3192
3193 /// Given an opaque value expression, return its LValue mapping if it exists,
3194 /// otherwise create one.
3195 LValue getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e);
3196
3197 /// Given an opaque value expression, return its RValue mapping if it exists,
3198 /// otherwise create one.
3199 RValue getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e);
3200
3201 /// isOpaqueValueEmitted - Return true if the opaque value expression has
3202 /// already been emitted.
3203 bool isOpaqueValueEmitted(const OpaqueValueExpr *E);
3204
3205 /// Get the index of the current ArrayInitLoopExpr, if any.
3206 llvm::Value *getArrayInitIndex() { return ArrayInitIndex; }
3207
3208 /// getAccessedFieldNo - Given an encoded value and a result number, return
3209 /// the input field number being accessed.
3210 static unsigned getAccessedFieldNo(unsigned Idx, const llvm::Constant *Elts);
3211
3212 llvm::BlockAddress *GetAddrOfLabel(const LabelDecl *L);
3213 llvm::BasicBlock *GetIndirectGotoBlock();
3214
3215 /// Check if \p E is a C++ "this" pointer wrapped in value-preserving casts.
3216 static bool IsWrappedCXXThis(const Expr *E);
3217
3218 /// EmitNullInitialization - Generate code to set a value of the given type to
3219 /// null, If the type contains data member pointers, they will be initialized
3220 /// to -1 in accordance with the Itanium C++ ABI.
3221 void EmitNullInitialization(Address DestPtr, QualType Ty);
3222
3223 /// Emits a call to an LLVM variable-argument intrinsic, either
3224 /// \c llvm.va_start or \c llvm.va_end.
3225 /// \param ArgValue A reference to the \c va_list as emitted by either
3226 /// \c EmitVAListRef or \c EmitMSVAListRef.
3227 /// \param IsStart If \c true, emits a call to \c llvm.va_start; otherwise,
3228 /// calls \c llvm.va_end.
3229 llvm::Value *EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart);
3230
3231 /// Generate code to get an argument from the passed in pointer
3232 /// and update it accordingly.
3233 /// \param VE The \c VAArgExpr for which to generate code.
3234 /// \param VAListAddr Receives a reference to the \c va_list as emitted by
3235 /// either \c EmitVAListRef or \c EmitMSVAListRef.
3236 /// \returns A pointer to the argument.
3237 // FIXME: We should be able to get rid of this method and use the va_arg
3238 // instruction in LLVM instead once it works well enough.
3239 RValue EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
3240 AggValueSlot Slot = AggValueSlot::ignored());
3241
3242 /// emitArrayLength - Compute the length of an array, even if it's a
3243 /// VLA, and drill down to the base element type.
3244 llvm::Value *emitArrayLength(const ArrayType *arrayType, QualType &baseType,
3245 Address &addr);
3246
3247 /// EmitVLASize - Capture all the sizes for the VLA expressions in
3248 /// the given variably-modified type and store them in the VLASizeMap.
3249 ///
3250 /// This function can be called with a null (unreachable) insert point.
3251 void EmitVariablyModifiedType(QualType Ty);
3252
3253 struct VlaSizePair {
3254 llvm::Value *NumElts;
3255 QualType Type;
3256
3257 VlaSizePair(llvm::Value *NE, QualType T) : NumElts(NE), Type(T) {}
3258 };
3259
3260 /// Return the number of elements for a single dimension
3261 /// for the given array type.
3262 VlaSizePair getVLAElements1D(const VariableArrayType *vla);
3263 VlaSizePair getVLAElements1D(QualType vla);
3264
3265 /// Returns an LLVM value that corresponds to the size,
3266 /// in non-variably-sized elements, of a variable length array type,
3267 /// plus that largest non-variably-sized element type. Assumes that
3268 /// the type has already been emitted with EmitVariablyModifiedType.
3269 VlaSizePair getVLASize(const VariableArrayType *vla);
3270 VlaSizePair getVLASize(QualType vla);
3271
3272 /// LoadCXXThis - Load the value of 'this'. This function is only valid while
3273 /// generating code for an C++ member function.
3274 llvm::Value *LoadCXXThis() {
3275 assert(CXXThisValue && "no 'this' value for this function");
3276 return CXXThisValue;
3277 }
3278 Address LoadCXXThisAddress();
3279
3280 /// LoadCXXVTT - Load the VTT parameter to base constructors/destructors have
3281 /// virtual bases.
3282 // FIXME: Every place that calls LoadCXXVTT is something
3283 // that needs to be abstracted properly.
3284 llvm::Value *LoadCXXVTT() {
3285 assert(CXXStructorImplicitParamValue && "no VTT value for this function");
3286 return CXXStructorImplicitParamValue;
3287 }
3288
3289 /// GetAddressOfBaseOfCompleteClass - Convert the given pointer to a
3290 /// complete class to the given direct base.
3291 Address GetAddressOfDirectBaseInCompleteClass(Address Value,
3292 const CXXRecordDecl *Derived,
3293 const CXXRecordDecl *Base,
3294 bool BaseIsVirtual);
3295
3296 static bool ShouldNullCheckClassCastValue(const CastExpr *Cast);
3297
3298 /// GetAddressOfBaseClass - This function will add the necessary delta to the
3299 /// load of 'this' and returns address of the base class.
3300 Address GetAddressOfBaseClass(Address Value, const CXXRecordDecl *Derived,
3301 CastExpr::path_const_iterator PathBegin,
3302 CastExpr::path_const_iterator PathEnd,
3303 bool NullCheckValue, SourceLocation Loc);
3304
3305 Address GetAddressOfDerivedClass(Address Value, const CXXRecordDecl *Derived,
3306 CastExpr::path_const_iterator PathBegin,
3307 CastExpr::path_const_iterator PathEnd,
3308 bool NullCheckValue);
3309
3310 /// GetVTTParameter - Return the VTT parameter that should be passed to a
3311 /// base constructor/destructor with virtual bases.
3312 /// FIXME: VTTs are Itanium ABI-specific, so the definition should move
3313 /// to ItaniumCXXABI.cpp together with all the references to VTT.
3314 llvm::Value *GetVTTParameter(GlobalDecl GD, bool ForVirtualBase,
3315 bool Delegating);
3316
3317 void EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
3318 CXXCtorType CtorType,
3319 const FunctionArgList &Args,
3320 SourceLocation Loc);
3321 // It's important not to confuse this and the previous function. Delegating
3322 // constructors are the C++0x feature. The constructor delegate optimization
3323 // is used to reduce duplication in the base and complete consturctors where
3324 // they are substantially the same.
3325 void EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3326 const FunctionArgList &Args);
3327
3328 /// Emit a call to an inheriting constructor (that is, one that invokes a
3329 /// constructor inherited from a base class) by inlining its definition. This
3330 /// is necessary if the ABI does not support forwarding the arguments to the
3331 /// base class constructor (because they're variadic or similar).
3332 void EmitInlinedInheritingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3333 CXXCtorType CtorType,
3334 bool ForVirtualBase,
3335 bool Delegating,
3336 CallArgList &Args);
3337
3338 /// Emit a call to a constructor inherited from a base class, passing the
3339 /// current constructor's arguments along unmodified (without even making
3340 /// a copy).
3341 void EmitInheritedCXXConstructorCall(const CXXConstructorDecl *D,
3342 bool ForVirtualBase, Address This,
3343 bool InheritedFromVBase,
3344 const CXXInheritedCtorInitExpr *E);
3345
3346 void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3347 bool ForVirtualBase, bool Delegating,
3348 AggValueSlot ThisAVS, const CXXConstructExpr *E);
3349
3350 void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3351 bool ForVirtualBase, bool Delegating,
3352 Address This, CallArgList &Args,
3353 AggValueSlot::Overlap_t Overlap,
3354 SourceLocation Loc, bool NewPointerIsChecked,
3355 llvm::CallBase **CallOrInvoke = nullptr);
3356
3357 /// Emit assumption load for all bases. Requires to be called only on
3358 /// most-derived class and not under construction of the object.
3359 void EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl, Address This);
3360
3361 /// Emit assumption that vptr load == global vtable.
3362 void EmitVTableAssumptionLoad(const VPtr &vptr, Address This);
3363
3364 void EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, Address This,
3365 Address Src, const CXXConstructExpr *E);
3366
3367 void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3368 const ArrayType *ArrayTy, Address ArrayPtr,
3369 const CXXConstructExpr *E,
3370 bool NewPointerIsChecked,
3371 bool ZeroInitialization = false);
3372
3373 void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3374 llvm::Value *NumElements, Address ArrayPtr,
3375 const CXXConstructExpr *E,
3376 bool NewPointerIsChecked,
3377 bool ZeroInitialization = false);
3378
3379 static Destroyer destroyCXXObject;
3380
3381 void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type,
3382 bool ForVirtualBase, bool Delegating, Address This,
3383 QualType ThisTy);
3384
3385 void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
3386 llvm::Type *ElementTy, Address NewPtr,
3387 llvm::Value *NumElements,
3388 llvm::Value *AllocSizeWithoutCookie);
3389
3390 void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
3391 Address Ptr);
3392
3393 void EmitSehCppScopeBegin();
3394 void EmitSehCppScopeEnd();
3395 void EmitSehTryScopeBegin();
3396 void EmitSehTryScopeEnd();
3397
3398 bool EmitLifetimeStart(llvm::Value *Addr);
3399 void EmitLifetimeEnd(llvm::Value *Addr);
3400
3401 llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
3402 void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
3403
3404 void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,
3405 QualType DeleteTy, llvm::Value *NumElements = nullptr,
3406 CharUnits CookieSize = CharUnits(),
3407 llvm::Constant *CalleeOverride = nullptr);
3408
3409 RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
3410 const CallExpr *TheCallExpr, bool IsDelete);
3411
3412 llvm::Value *EmitCXXTypeidExpr(const CXXTypeidExpr *E);
3413 llvm::Value *EmitDynamicCast(Address V, const CXXDynamicCastExpr *DCE);
3414 Address EmitCXXUuidofExpr(const CXXUuidofExpr *E);
3415
3416 /// Situations in which we might emit a check for the suitability of a
3417 /// pointer or glvalue. Needs to be kept in sync with ubsan_handlers.cpp in
3418 /// compiler-rt.
3419 enum TypeCheckKind {
3420 /// Checking the operand of a load. Must be suitably sized and aligned.
3421 TCK_Load,
3422 /// Checking the destination of a store. Must be suitably sized and aligned.
3423 TCK_Store,
3424 /// Checking the bound value in a reference binding. Must be suitably sized
3425 /// and aligned, but is not required to refer to an object (until the
3426 /// reference is used), per core issue 453.
3427 TCK_ReferenceBinding,
3428 /// Checking the object expression in a non-static data member access. Must
3429 /// be an object within its lifetime.
3430 TCK_MemberAccess,
3431 /// Checking the 'this' pointer for a call to a non-static member function.
3432 /// Must be an object within its lifetime.
3433 TCK_MemberCall,
3434 /// Checking the 'this' pointer for a constructor call.
3435 TCK_ConstructorCall,
3436 /// Checking the operand of a static_cast to a derived pointer type. Must be
3437 /// null or an object within its lifetime.
3438 TCK_DowncastPointer,
3439 /// Checking the operand of a static_cast to a derived reference type. Must
3440 /// be an object within its lifetime.
3441 TCK_DowncastReference,
3442 /// Checking the operand of a cast to a base object. Must be suitably sized
3443 /// and aligned.
3444 TCK_Upcast,
3445 /// Checking the operand of a cast to a virtual base object. Must be an
3446 /// object within its lifetime.
3447 TCK_UpcastToVirtualBase,
3448 /// Checking the value assigned to a _Nonnull pointer. Must not be null.
3449 TCK_NonnullAssign,
3450 /// Checking the operand of a dynamic_cast or a typeid expression. Must be
3451 /// null or an object within its lifetime.
3452 TCK_DynamicOperation
3453 };
3454
3455 /// Determine whether the pointer type check \p TCK permits null pointers.
3456 static bool isNullPointerAllowed(TypeCheckKind TCK);
3457
3458 /// Determine whether the pointer type check \p TCK requires a vptr check.
3459 static bool isVptrCheckRequired(TypeCheckKind TCK, QualType Ty);
3460
3461 /// Whether any type-checking sanitizers are enabled. If \c false,
3462 /// calls to EmitTypeCheck can be skipped.
3463 bool sanitizePerformTypeCheck() const;
3464
3465 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV,
3466 QualType Type, SanitizerSet SkippedChecks = SanitizerSet(),
3467 llvm::Value *ArraySize = nullptr) {
3468 if (!sanitizePerformTypeCheck())
3469 return;
3470 EmitTypeCheck(TCK, Loc, V: LV.emitRawPointer(CGF&: *this), Type, Alignment: LV.getAlignment(),
3471 SkippedChecks, ArraySize);
3472 }
3473
3474 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, Address Addr,
3475 QualType Type, CharUnits Alignment = CharUnits::Zero(),
3476 SanitizerSet SkippedChecks = SanitizerSet(),
3477 llvm::Value *ArraySize = nullptr) {
3478 if (!sanitizePerformTypeCheck())
3479 return;
3480 EmitTypeCheck(TCK, Loc, V: Addr.emitRawPointer(CGF&: *this), Type, Alignment,
3481 SkippedChecks, ArraySize);
3482 }
3483
3484 /// Emit a check that \p V is the address of storage of the
3485 /// appropriate size and alignment for an object of type \p Type
3486 /// (or if ArraySize is provided, for an array of that bound).
3487 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Value *V,
3488 QualType Type, CharUnits Alignment = CharUnits::Zero(),
3489 SanitizerSet SkippedChecks = SanitizerSet(),
3490 llvm::Value *ArraySize = nullptr);
3491
3492 /// Emit a check that \p Base points into an array object, which
3493 /// we can access at index \p Index. \p Accessed should be \c false if we
3494 /// this expression is used as an lvalue, for instance in "&Arr[Idx]".
3495 void EmitBoundsCheck(const Expr *ArrayExpr, const Expr *ArrayExprBase,
3496 llvm::Value *Index, QualType IndexType, bool Accessed);
3497 void EmitBoundsCheckImpl(const Expr *ArrayExpr, QualType ArrayBaseType,
3498 llvm::Value *IndexVal, QualType IndexType,
3499 llvm::Value *BoundsVal, QualType BoundsType,
3500 bool Accessed);
3501
3502 /// Returns debug info, with additional annotation if
3503 /// CGM.getCodeGenOpts().SanitizeAnnotateDebugInfo[Ordinal] is enabled for
3504 /// any of the ordinals.
3505 llvm::DILocation *
3506 SanitizerAnnotateDebugInfo(ArrayRef<SanitizerKind::SanitizerOrdinal> Ordinals,
3507 SanitizerHandler Handler);
3508
3509 /// Build metadata used by the AllocToken instrumentation.
3510 llvm::MDNode *buildAllocToken(QualType AllocType);
3511 /// Emit and set additional metadata used by the AllocToken instrumentation.
3512 void EmitAllocToken(llvm::CallBase *CB, QualType AllocType);
3513 /// Build additional metadata used by the AllocToken instrumentation,
3514 /// inferring the type from an allocation call expression.
3515 llvm::MDNode *buildAllocToken(const CallExpr *E);
3516 /// Emit and set additional metadata used by the AllocToken instrumentation,
3517 /// inferring the type from an allocation call expression.
3518 void EmitAllocToken(llvm::CallBase *CB, const CallExpr *E);
3519
3520 llvm::Value *GetCountedByFieldExprGEP(const Expr *Base, const FieldDecl *FD,
3521 const FieldDecl *CountDecl);
3522
3523 /// Build an expression accessing the "counted_by" field.
3524 llvm::Value *EmitLoadOfCountedByField(const Expr *Base, const FieldDecl *FD,
3525 const FieldDecl *CountDecl);
3526
3527 // Emit bounds checking for flexible array and pointer members with the
3528 // counted_by attribute.
3529 void EmitCountedByBoundsChecking(const Expr *ArrayExpr, QualType ArrayType,
3530 Address ArrayInst, QualType IndexType,
3531 llvm::Value *IndexVal, bool Accessed,
3532 bool FlexibleArray);
3533
3534 llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
3535 bool isInc, bool isPre);
3536 ComplexPairTy EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
3537 bool isInc, bool isPre);
3538
3539 /// Converts Location to a DebugLoc, if debug information is enabled.
3540 llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location);
3541
3542 //===--------------------------------------------------------------------===//
3543 // Declaration Emission
3544 //===--------------------------------------------------------------------===//
3545
3546 /// EmitDecl - Emit a declaration.
3547 ///
3548 /// This function can be called with a null (unreachable) insert point.
3549 void EmitDecl(const Decl &D, bool EvaluateConditionDecl = false);
3550
3551 /// EmitVarDecl - Emit a local variable declaration.
3552 ///
3553 /// This function can be called with a null (unreachable) insert point.
3554 void EmitVarDecl(const VarDecl &D);
3555
3556 void EmitScalarInit(const Expr *init, const ValueDecl *D, LValue lvalue,
3557 bool capturedByInit);
3558
3559 typedef void SpecialInitFn(CodeGenFunction &Init, const VarDecl &D,
3560 llvm::Value *Address);
3561
3562 /// Determine whether the given initializer is trivial in the sense
3563 /// that it requires no code to be generated.
3564 bool isTrivialInitializer(const Expr *Init);
3565
3566 /// EmitAutoVarDecl - Emit an auto variable declaration.
3567 ///
3568 /// This function can be called with a null (unreachable) insert point.
3569 void EmitAutoVarDecl(const VarDecl &D);
3570
3571 class AutoVarEmission {
3572 friend class CodeGenFunction;
3573
3574 const VarDecl *Variable;
3575
3576 /// The address of the alloca for languages with explicit address space
3577 /// (e.g. OpenCL) or alloca casted to generic pointer for address space
3578 /// agnostic languages (e.g. C++). Invalid if the variable was emitted
3579 /// as a global constant.
3580 Address Addr;
3581
3582 llvm::Value *NRVOFlag;
3583
3584 /// True if the variable is a __block variable that is captured by an
3585 /// escaping block.
3586 bool IsEscapingByRef;
3587
3588 /// If the variable is of aggregate type and has a constant initializer,
3589 /// a constant representing that initializer.
3590 llvm::Constant *ConstantAggregateInitializer;
3591
3592 /// True if lifetime markers should be used.
3593 bool UseLifetimeMarkers;
3594
3595 /// Address with original alloca instruction. Invalid if the variable was
3596 /// emitted as a global constant.
3597 RawAddress AllocaAddr;
3598
3599 struct Invalid {};
3600 AutoVarEmission(Invalid)
3601 : Variable(nullptr), Addr(Address::invalid()),
3602 AllocaAddr(RawAddress::invalid()) {}
3603
3604 AutoVarEmission(const VarDecl &variable)
3605 : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr),
3606 IsEscapingByRef(false), ConstantAggregateInitializer(nullptr),
3607 UseLifetimeMarkers(false), AllocaAddr(RawAddress::invalid()) {}
3608
3609 bool wasEmittedAsGlobal() const { return !Addr.isValid(); }
3610
3611 public:
3612 static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }
3613
3614 bool useLifetimeMarkers() const { return UseLifetimeMarkers; }
3615
3616 /// Returns the raw, allocated address, which is not necessarily
3617 /// the address of the object itself. It is casted to default
3618 /// address space for address space agnostic languages.
3619 Address getAllocatedAddress() const { return Addr; }
3620
3621 /// Returns the address for the original alloca instruction.
3622 RawAddress getOriginalAllocatedAddress() const { return AllocaAddr; }
3623
3624 /// Returns the address of the object within this declaration.
3625 /// Note that this does not chase the forwarding pointer for
3626 /// __block decls.
3627 Address getObjectAddress(CodeGenFunction &CGF) const {
3628 if (!IsEscapingByRef)
3629 return Addr;
3630
3631 return CGF.emitBlockByrefAddress(baseAddr: Addr, V: Variable, /*forward*/ followForward: false);
3632 }
3633 };
3634 AutoVarEmission EmitAutoVarAlloca(const VarDecl &var);
3635 void EmitAutoVarInit(const AutoVarEmission &emission);
3636 void EmitAutoVarCleanups(const AutoVarEmission &emission);
3637 void emitAutoVarTypeCleanup(const AutoVarEmission &emission,
3638 QualType::DestructionKind dtorKind);
3639
3640 /// Re-emit trivial-auto-var-init stores for variables bypassed by the jump
3641 /// Source. No-op in a function containing a computed goto, where jump sources
3642 /// are unknown and a single function-scope init is used instead.
3643 void emitBypassedVarInitsForSource(const Stmt *Source);
3644
3645 void MaybeEmitDeferredVarDeclInit(const VarDecl *var);
3646
3647 /// Emits the alloca and debug information for the size expressions for each
3648 /// dimension of an array. It registers the association of its (1-dimensional)
3649 /// QualTypes and size expression's debug node, so that CGDebugInfo can
3650 /// reference this node when creating the DISubrange object to describe the
3651 /// array types.
3652 void EmitAndRegisterVariableArrayDimensions(CGDebugInfo *DI, const VarDecl &D,
3653 bool EmitDebugInfo);
3654
3655 void EmitStaticVarDecl(const VarDecl &D,
3656 llvm::GlobalValue::LinkageTypes Linkage);
3657
3658 class ParamValue {
3659 union {
3660 Address Addr;
3661 llvm::Value *Value;
3662 };
3663
3664 bool IsIndirect;
3665
3666 ParamValue(llvm::Value *V) : Value(V), IsIndirect(false) {}
3667 ParamValue(Address A) : Addr(A), IsIndirect(true) {}
3668
3669 public:
3670 static ParamValue forDirect(llvm::Value *value) {
3671 return ParamValue(value);
3672 }
3673 static ParamValue forIndirect(Address addr) {
3674 assert(!addr.getAlignment().isZero());
3675 return ParamValue(addr);
3676 }
3677
3678 bool isIndirect() const { return IsIndirect; }
3679 llvm::Value *getAnyValue() const {
3680 if (!isIndirect())
3681 return Value;
3682 assert(!Addr.hasOffset() && "unexpected offset");
3683 return Addr.getBasePointer();
3684 }
3685
3686 llvm::Value *getDirectValue() const {
3687 assert(!isIndirect());
3688 return Value;
3689 }
3690
3691 Address getIndirectAddress() const {
3692 assert(isIndirect());
3693 return Addr;
3694 }
3695 };
3696
3697 /// EmitParmDecl - Emit a ParmVarDecl or an ImplicitParamDecl.
3698 void EmitParmDecl(const VarDecl &D, ParamValue Arg, unsigned ArgNo);
3699
3700 /// protectFromPeepholes - Protect a value that we're intending to
3701 /// store to the side, but which will probably be used later, from
3702 /// aggressive peepholing optimizations that might delete it.
3703 ///
3704 /// Pass the result to unprotectFromPeepholes to declare that
3705 /// protection is no longer required.
3706 ///
3707 /// There's no particular reason why this shouldn't apply to
3708 /// l-values, it's just that no existing peepholes work on pointers.
3709 PeepholeProtection protectFromPeepholes(RValue rvalue);
3710 void unprotectFromPeepholes(PeepholeProtection protection);
3711
3712 void emitAlignmentAssumptionCheck(llvm::Value *Ptr, QualType Ty,
3713 SourceLocation Loc,
3714 SourceLocation AssumptionLoc,
3715 llvm::Value *Alignment,
3716 llvm::Value *OffsetValue = nullptr);
3717
3718 void emitReturnAlignmentAssumptionCheck(llvm::Value *Ptr, QualType Ty,
3719 SourceLocation AssumptionLoc,
3720 llvm::Value *Alignment,
3721 llvm::Value *OffsetValue = nullptr);
3722
3723 void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty,
3724 SourceLocation Loc, SourceLocation AssumptionLoc,
3725 llvm::Value *Alignment,
3726 llvm::Value *OffsetValue = nullptr);
3727
3728 void emitAlignmentAssumption(llvm::Value *PtrValue, const Expr *E,
3729 SourceLocation AssumptionLoc,
3730 llvm::Value *Alignment,
3731 llvm::Value *OffsetValue = nullptr);
3732
3733 //===--------------------------------------------------------------------===//
3734 // Statement Emission
3735 //===--------------------------------------------------------------------===//
3736
3737 /// EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
3738 void EmitStopPoint(const Stmt *S);
3739
3740 /// EmitStmt - Emit the code for the statement \arg S. It is legal to call
3741 /// this function even if there is no current insertion point.
3742 ///
3743 /// This function may clear the current insertion point; callers should use
3744 /// EnsureInsertPoint if they wish to subsequently generate code without first
3745 /// calling EmitBlock, EmitBranch, or EmitStmt.
3746 void EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs = {});
3747
3748 /// EmitSimpleStmt - Try to emit a "simple" statement which does not
3749 /// necessarily require an insertion point or debug information; typically
3750 /// because the statement amounts to a jump or a container of other
3751 /// statements.
3752 ///
3753 /// \return True if the statement was handled.
3754 bool EmitSimpleStmt(const Stmt *S, ArrayRef<const Attr *> Attrs);
3755
3756 Address EmitCompoundStmt(const CompoundStmt &S, bool GetLast = false,
3757 AggValueSlot AVS = AggValueSlot::ignored());
3758 Address
3759 EmitCompoundStmtWithoutScope(const CompoundStmt &S, bool GetLast = false,
3760 AggValueSlot AVS = AggValueSlot::ignored());
3761
3762 /// EmitLabel - Emit the block for the given label. It is legal to call this
3763 /// function even if there is no current insertion point.
3764 void EmitLabel(const LabelDecl *D); // helper for EmitLabelStmt.
3765
3766 void EmitLabelStmt(const LabelStmt &S);
3767 void EmitAttributedStmt(const AttributedStmt &S);
3768 void EmitGotoStmt(const GotoStmt &S);
3769 void EmitIndirectGotoStmt(const IndirectGotoStmt &S);
3770 void EmitIfStmt(const IfStmt &S);
3771
3772 void EmitWhileStmt(const WhileStmt &S, ArrayRef<const Attr *> Attrs = {});
3773 void EmitDoStmt(const DoStmt &S, ArrayRef<const Attr *> Attrs = {});
3774 void EmitForStmt(const ForStmt &S, ArrayRef<const Attr *> Attrs = {});
3775 void EmitReturnStmt(const ReturnStmt &S);
3776 void EmitDeclStmt(const DeclStmt &S);
3777 void EmitBreakStmt(const BreakStmt &S);
3778 void EmitContinueStmt(const ContinueStmt &S);
3779 void EmitSwitchStmt(const SwitchStmt &S);
3780 void EmitDefaultStmt(const DefaultStmt &S, ArrayRef<const Attr *> Attrs);
3781 void EmitCaseStmt(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3782 void EmitCaseStmtRange(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3783 void EmitDeferStmt(const DeferStmt &S);
3784 void EmitAsmStmt(const AsmStmt &S);
3785
3786 const BreakContinue *GetDestForLoopControlStmt(const LoopControlStmt &S);
3787
3788 void EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S);
3789 void EmitObjCAtTryStmt(const ObjCAtTryStmt &S);
3790 void EmitObjCAtThrowStmt(const ObjCAtThrowStmt &S);
3791 void EmitObjCAtSynchronizedStmt(const ObjCAtSynchronizedStmt &S);
3792 void EmitObjCAutoreleasePoolStmt(const ObjCAutoreleasePoolStmt &S);
3793
3794 void EmitCoroutineBody(const CoroutineBodyStmt &S);
3795 void EmitCoreturnStmt(const CoreturnStmt &S);
3796 RValue EmitCoawaitExpr(const CoawaitExpr &E,
3797 AggValueSlot aggSlot = AggValueSlot::ignored(),
3798 bool ignoreResult = false);
3799 LValue EmitCoawaitLValue(const CoawaitExpr *E);
3800 RValue EmitCoyieldExpr(const CoyieldExpr &E,
3801 AggValueSlot aggSlot = AggValueSlot::ignored(),
3802 bool ignoreResult = false);
3803 LValue EmitCoyieldLValue(const CoyieldExpr *E);
3804 RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID);
3805
3806 void EmitSYCLKernelCallStmt(const SYCLKernelCallStmt &S);
3807
3808 void EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3809 void ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3810
3811 void EmitCXXTryStmt(const CXXTryStmt &S);
3812 void EmitSEHTryStmt(const SEHTryStmt &S);
3813 void EmitSEHLeaveStmt(const SEHLeaveStmt &S);
3814 void EnterSEHTryStmt(const SEHTryStmt &S);
3815 void ExitSEHTryStmt(const SEHTryStmt &S);
3816 void VolatilizeTryBlocks(llvm::BasicBlock *BB,
3817 llvm::SmallPtrSet<llvm::BasicBlock *, 10> &V);
3818
3819 void pushSEHCleanup(CleanupKind kind, llvm::Function *FinallyFunc);
3820 void startOutlinedSEHHelper(CodeGenFunction &ParentCGF, bool IsFilter,
3821 const Stmt *OutlinedStmt);
3822
3823 llvm::Function *GenerateSEHFilterFunction(CodeGenFunction &ParentCGF,
3824 const SEHExceptStmt &Except);
3825
3826 llvm::Function *GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF,
3827 const SEHFinallyStmt &Finally);
3828
3829 void EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF,
3830 llvm::Value *ParentFP, llvm::Value *EntryEBP);
3831 llvm::Value *EmitSEHExceptionCode();
3832 llvm::Value *EmitSEHExceptionInfo();
3833 llvm::Value *EmitSEHAbnormalTermination();
3834
3835 /// Emit simple code for OpenMP directives in Simd-only mode.
3836 void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D);
3837
3838 /// Scan the outlined statement for captures from the parent function. For
3839 /// each capture, mark the capture as escaped and emit a call to
3840 /// llvm.localrecover. Insert the localrecover result into the LocalDeclMap.
3841 void EmitCapturedLocals(CodeGenFunction &ParentCGF, const Stmt *OutlinedStmt,
3842 bool IsFilter);
3843
3844 /// Recovers the address of a local in a parent function. ParentVar is the
3845 /// address of the variable used in the immediate parent function. It can
3846 /// either be an alloca or a call to llvm.localrecover if there are nested
3847 /// outlined functions. ParentFP is the frame pointer of the outermost parent
3848 /// frame.
3849 Address recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
3850 Address ParentVar, llvm::Value *ParentFP);
3851
3852 void EmitCXXForRangeStmt(const CXXForRangeStmt &S,
3853 ArrayRef<const Attr *> Attrs = {});
3854
3855 void
3856 EmitCXXExpansionStmtInstantiation(const CXXExpansionStmtInstantiation &S);
3857
3858 /// Controls insertion of cancellation exit blocks in worksharing constructs.
3859 class OMPCancelStackRAII {
3860 CodeGenFunction &CGF;
3861
3862 public:
3863 OMPCancelStackRAII(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
3864 bool HasCancel)
3865 : CGF(CGF) {
3866 CGF.OMPCancelStack.enter(CGF, Kind, HasCancel);
3867 }
3868 ~OMPCancelStackRAII() { CGF.OMPCancelStack.exit(CGF); }
3869 };
3870
3871 /// Returns calculated size of the specified type.
3872 llvm::Value *getTypeSize(QualType Ty);
3873 LValue InitCapturedStruct(const CapturedStmt &S);
3874 llvm::Function *EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K);
3875 llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S);
3876 Address GenerateCapturedStmtArgument(const CapturedStmt &S);
3877 llvm::Function *
3878 GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S,
3879 const OMPExecutableDirective &D);
3880 llvm::Function *
3881 GenerateOpenMPCapturedStmtFunctionAggregate(const CapturedStmt &S,
3882 const OMPExecutableDirective &D);
3883 void GenerateOpenMPCapturedVars(const CapturedStmt &S,
3884 SmallVectorImpl<llvm::Value *> &CapturedVars);
3885 void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy,
3886 SourceLocation Loc);
3887 /// Perform element by element copying of arrays with type \a
3888 /// OriginalType from \a SrcAddr to \a DestAddr using copying procedure
3889 /// generated by \a CopyGen.
3890 ///
3891 /// \param DestAddr Address of the destination array.
3892 /// \param SrcAddr Address of the source array.
3893 /// \param OriginalType Type of destination and source arrays.
3894 /// \param CopyGen Copying procedure that copies value of single array element
3895 /// to another single array element.
3896 void EmitOMPAggregateAssign(
3897 Address DestAddr, Address SrcAddr, QualType OriginalType,
3898 const llvm::function_ref<void(Address, Address)> CopyGen);
3899 /// Emit proper copying of data from one variable to another.
3900 ///
3901 /// \param OriginalType Original type of the copied variables.
3902 /// \param DestAddr Destination address.
3903 /// \param SrcAddr Source address.
3904 /// \param DestVD Destination variable used in \a CopyExpr (for arrays, has
3905 /// type of the base array element).
3906 /// \param SrcVD Source variable used in \a CopyExpr (for arrays, has type of
3907 /// the base array element).
3908 /// \param Copy Actual copygin expression for copying data from \a SrcVD to \a
3909 /// DestVD.
3910 void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr,
3911 const VarDecl *DestVD, const VarDecl *SrcVD,
3912 const Expr *Copy);
3913 /// Emit atomic update code for constructs: \a X = \a X \a BO \a E or
3914 /// \a X = \a E \a BO \a E.
3915 ///
3916 /// \param X Value to be updated.
3917 /// \param E Update value.
3918 /// \param BO Binary operation for update operation.
3919 /// \param IsXLHSInRHSPart true if \a X is LHS in RHS part of the update
3920 /// expression, false otherwise.
3921 /// \param AO Atomic ordering of the generated atomic instructions.
3922 /// \param CommonGen Code generator for complex expressions that cannot be
3923 /// expressed through atomicrmw instruction.
3924 /// \returns <true, OldAtomicValue> if simple 'atomicrmw' instruction was
3925 /// generated, <false, RValue::get(nullptr)> otherwise.
3926 std::pair<bool, RValue> EmitOMPAtomicSimpleUpdateExpr(
3927 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3928 llvm::AtomicOrdering AO, SourceLocation Loc,
3929 const llvm::function_ref<RValue(RValue)> CommonGen);
3930 bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
3931 OMPPrivateScope &PrivateScope);
3932 void EmitOMPPrivateClause(const OMPExecutableDirective &D,
3933 OMPPrivateScope &PrivateScope);
3934 void EmitOMPUseDevicePtrClause(
3935 const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
3936 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3937 CaptureDeviceAddrMap);
3938 void EmitOMPUseDeviceAddrClause(
3939 const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
3940 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3941 CaptureDeviceAddrMap);
3942 /// Emit code for copyin clause in \a D directive. The next code is
3943 /// generated at the start of outlined functions for directives:
3944 /// \code
3945 /// threadprivate_var1 = master_threadprivate_var1;
3946 /// operator=(threadprivate_var2, master_threadprivate_var2);
3947 /// ...
3948 /// __kmpc_barrier(&loc, global_tid);
3949 /// \endcode
3950 ///
3951 /// \param D OpenMP directive possibly with 'copyin' clause(s).
3952 /// \returns true if at least one copyin variable is found, false otherwise.
3953 bool EmitOMPCopyinClause(const OMPExecutableDirective &D);
3954 /// Emit initial code for lastprivate variables. If some variable is
3955 /// not also firstprivate, then the default initialization is used. Otherwise
3956 /// initialization of this variable is performed by EmitOMPFirstprivateClause
3957 /// method.
3958 ///
3959 /// \param D Directive that may have 'lastprivate' directives.
3960 /// \param PrivateScope Private scope for capturing lastprivate variables for
3961 /// proper codegen in internal captured statement.
3962 ///
3963 /// \returns true if there is at least one lastprivate variable, false
3964 /// otherwise.
3965 bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D,
3966 OMPPrivateScope &PrivateScope);
3967 /// Emit final copying of lastprivate values to original variables at
3968 /// the end of the worksharing or simd directive.
3969 ///
3970 /// \param D Directive that has at least one 'lastprivate' directives.
3971 /// \param IsLastIterCond Boolean condition that must be set to 'i1 true' if
3972 /// it is the last iteration of the loop code in associated directive, or to
3973 /// 'i1 false' otherwise. If this item is nullptr, no final check is required.
3974 void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D,
3975 bool NoFinals,
3976 llvm::Value *IsLastIterCond = nullptr);
3977 /// Emit initial code for linear clauses.
3978 void EmitOMPLinearClause(const OMPLoopDirective &D,
3979 CodeGenFunction::OMPPrivateScope &PrivateScope);
3980 /// Emit final code for linear clauses.
3981 /// \param CondGen Optional conditional code for final part of codegen for
3982 /// linear clause.
3983 void EmitOMPLinearClauseFinal(
3984 const OMPLoopDirective &D,
3985 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
3986 /// Emit initial code for reduction variables. Creates reduction copies
3987 /// and initializes them with the values according to OpenMP standard.
3988 ///
3989 /// \param D Directive (possibly) with the 'reduction' clause.
3990 /// \param PrivateScope Private scope for capturing reduction variables for
3991 /// proper codegen in internal captured statement.
3992 ///
3993 void EmitOMPReductionClauseInit(const OMPExecutableDirective &D,
3994 OMPPrivateScope &PrivateScope,
3995 bool ForInscan = false);
3996 /// Emit final update of reduction values to original variables at
3997 /// the end of the directive.
3998 ///
3999 /// \param D Directive that has at least one 'reduction' directives.
4000 /// \param ReductionKind The kind of reduction to perform.
4001 void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D,
4002 const OpenMPDirectiveKind ReductionKind);
4003 /// Emit initial code for linear variables. Creates private copies
4004 /// and initializes them with the values according to OpenMP standard.
4005 ///
4006 /// \param D Directive (possibly) with the 'linear' clause.
4007 /// \return true if at least one linear variable is found that should be
4008 /// initialized with the value of the original variable, false otherwise.
4009 bool EmitOMPLinearClauseInit(const OMPLoopDirective &D);
4010
4011 typedef const llvm::function_ref<void(CodeGenFunction & /*CGF*/,
4012 llvm::Function * /*OutlinedFn*/,
4013 const OMPTaskDataTy & /*Data*/)>
4014 TaskGenTy;
4015 void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S,
4016 const OpenMPDirectiveKind CapturedRegion,
4017 const RegionCodeGenTy &BodyGen,
4018 const TaskGenTy &TaskGen, OMPTaskDataTy &Data);
4019 struct OMPTargetDataInfo {
4020 Address BasePointersArray = Address::invalid();
4021 Address PointersArray = Address::invalid();
4022 Address SizesArray = Address::invalid();
4023 Address MappersArray = Address::invalid();
4024 unsigned NumberOfTargetItems = 0;
4025 explicit OMPTargetDataInfo() = default;
4026 OMPTargetDataInfo(Address BasePointersArray, Address PointersArray,
4027 Address SizesArray, Address MappersArray,
4028 unsigned NumberOfTargetItems)
4029 : BasePointersArray(BasePointersArray), PointersArray(PointersArray),
4030 SizesArray(SizesArray), MappersArray(MappersArray),
4031 NumberOfTargetItems(NumberOfTargetItems) {}
4032 };
4033 void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S,
4034 const RegionCodeGenTy &BodyGen,
4035 OMPTargetDataInfo &InputInfo);
4036 void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data,
4037 CodeGenFunction &CGF, const CapturedStmt *CS,
4038 OMPPrivateScope &Scope);
4039 void EmitOMPMetaDirective(const OMPMetaDirective &S);
4040 void EmitOMPParallelDirective(const OMPParallelDirective &S);
4041 void EmitOMPSimdDirective(const OMPSimdDirective &S);
4042 void EmitOMPTileDirective(const OMPTileDirective &S);
4043 void EmitOMPStripeDirective(const OMPStripeDirective &S);
4044 void EmitOMPUnrollDirective(const OMPUnrollDirective &S);
4045 void EmitOMPReverseDirective(const OMPReverseDirective &S);
4046 void EmitOMPSplitDirective(const OMPSplitDirective &S);
4047 void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
4048 void EmitOMPFlattenDirective(const OMPFlattenDirective &S);
4049 void EmitOMPFuseDirective(const OMPFuseDirective &S);
4050 void EmitOMPForDirective(const OMPForDirective &S);
4051 void EmitOMPForSimdDirective(const OMPForSimdDirective &S);
4052 void EmitOMPScopeDirective(const OMPScopeDirective &S);
4053 void EmitOMPSectionsDirective(const OMPSectionsDirective &S);
4054 void EmitOMPSectionDirective(const OMPSectionDirective &S);
4055 void EmitOMPSingleDirective(const OMPSingleDirective &S);
4056 void EmitOMPMasterDirective(const OMPMasterDirective &S);
4057 void EmitOMPMaskedDirective(const OMPMaskedDirective &S);
4058 void EmitOMPCriticalDirective(const OMPCriticalDirective &S);
4059 void EmitOMPParallelForDirective(const OMPParallelForDirective &S);
4060 void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S);
4061 void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S);
4062 void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S);
4063 void EmitOMPTaskDirective(const OMPTaskDirective &S);
4064 void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S);
4065 void EmitOMPErrorDirective(const OMPErrorDirective &S);
4066 void EmitOMPBarrierDirective(const OMPBarrierDirective &S);
4067 void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S);
4068 void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S);
4069 void EmitOMPFlushDirective(const OMPFlushDirective &S);
4070 void EmitOMPDepobjDirective(const OMPDepobjDirective &S);
4071 void EmitOMPScanDirective(const OMPScanDirective &S);
4072 void
4073 EmitOMPOrderedStandaloneDirective(const OMPOrderedStandaloneDirective &S);
4074 void
4075 EmitOMPOrderedBlockAssocDirective(const OMPOrderedBlockAssocDirective &S);
4076 void EmitOMPAtomicDirective(const OMPAtomicDirective &S);
4077 void EmitOMPTargetDirective(const OMPTargetDirective &S);
4078 void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S);
4079 void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S);
4080 void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S);
4081 void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S);
4082 void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S);
4083 void
4084 EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S);
4085 void EmitOMPTeamsDirective(const OMPTeamsDirective &S);
4086 void
4087 EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S);
4088 void EmitOMPCancelDirective(const OMPCancelDirective &S);
4089 void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S);
4090 void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S);
4091 void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S);
4092 void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S);
4093 void EmitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &S);
4094 void
4095 EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S);
4096 void
4097 EmitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &S);
4098 void EmitOMPParallelMasterTaskLoopDirective(
4099 const OMPParallelMasterTaskLoopDirective &S);
4100 void EmitOMPParallelMaskedTaskLoopDirective(
4101 const OMPParallelMaskedTaskLoopDirective &S);
4102 void EmitOMPParallelMasterTaskLoopSimdDirective(
4103 const OMPParallelMasterTaskLoopSimdDirective &S);
4104 void EmitOMPParallelMaskedTaskLoopSimdDirective(
4105 const OMPParallelMaskedTaskLoopSimdDirective &S);
4106 void EmitOMPDistributeDirective(const OMPDistributeDirective &S);
4107 void EmitOMPDistributeParallelForDirective(
4108 const OMPDistributeParallelForDirective &S);
4109 void EmitOMPDistributeParallelForSimdDirective(
4110 const OMPDistributeParallelForSimdDirective &S);
4111 void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S);
4112 void EmitOMPTargetParallelForSimdDirective(
4113 const OMPTargetParallelForSimdDirective &S);
4114 /// Also emits 'target loop'.
4115 void EmitOMPTargetSimdDirective(const OMPLoopDirective &S);
4116 void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S);
4117 void
4118 EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S);
4119 void EmitOMPTeamsDistributeParallelForSimdDirective(
4120 const OMPTeamsDistributeParallelForSimdDirective &S);
4121 void EmitOMPTeamsDistributeParallelForDirective(
4122 const OMPTeamsDistributeParallelForDirective &S);
4123 void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S);
4124 void EmitOMPTargetTeamsDistributeDirective(
4125 const OMPTargetTeamsDistributeDirective &S);
4126 void EmitOMPTargetTeamsDistributeParallelForDirective(
4127 const OMPTargetTeamsDistributeParallelForDirective &S);
4128 void EmitOMPTargetTeamsDistributeParallelForSimdDirective(
4129 const OMPTargetTeamsDistributeParallelForSimdDirective &S);
4130 void EmitOMPTargetTeamsDistributeSimdDirective(
4131 const OMPTargetTeamsDistributeSimdDirective &S);
4132 void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S);
4133 void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S);
4134 void EmitOMPTargetParallelGenericLoopDirective(
4135 const OMPTargetParallelGenericLoopDirective &S);
4136 void EmitOMPTargetTeamsGenericLoopDirective(
4137 const OMPTargetTeamsGenericLoopDirective &S);
4138 void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S);
4139 void EmitOMPInteropDirective(const OMPInteropDirective &S);
4140 void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S);
4141 void EmitOMPAssumeDirective(const OMPAssumeDirective &S);
4142
4143 /// Emit device code for the target directive.
4144 static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
4145 StringRef ParentName,
4146 const OMPTargetDirective &S);
4147 static void
4148 EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
4149 const OMPTargetParallelDirective &S);
4150 /// Emit device code for the target parallel for directive.
4151 static void EmitOMPTargetParallelForDeviceFunction(
4152 CodeGenModule &CGM, StringRef ParentName,
4153 const OMPTargetParallelForDirective &S);
4154 /// Emit device code for the target parallel for simd directive.
4155 static void EmitOMPTargetParallelForSimdDeviceFunction(
4156 CodeGenModule &CGM, StringRef ParentName,
4157 const OMPTargetParallelForSimdDirective &S);
4158 /// Emit device code for the target teams directive.
4159 static void
4160 EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
4161 const OMPTargetTeamsDirective &S);
4162 /// Emit device code for the target teams distribute directive.
4163 static void EmitOMPTargetTeamsDistributeDeviceFunction(
4164 CodeGenModule &CGM, StringRef ParentName,
4165 const OMPTargetTeamsDistributeDirective &S);
4166 /// Emit device code for the target teams distribute simd directive.
4167 static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(
4168 CodeGenModule &CGM, StringRef ParentName,
4169 const OMPTargetTeamsDistributeSimdDirective &S);
4170 /// Emit device code for the target simd and target loop directives.
4171 static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM,
4172 StringRef ParentName,
4173 const OMPLoopDirective &S);
4174 /// Emit device code for the target teams distribute parallel for simd
4175 /// directive.
4176 static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
4177 CodeGenModule &CGM, StringRef ParentName,
4178 const OMPTargetTeamsDistributeParallelForSimdDirective &S);
4179
4180 /// Emit device code for the target teams loop directive.
4181 static void EmitOMPTargetTeamsGenericLoopDeviceFunction(
4182 CodeGenModule &CGM, StringRef ParentName,
4183 const OMPTargetTeamsGenericLoopDirective &S);
4184
4185 /// Emit device code for the target parallel loop directive.
4186 static void EmitOMPTargetParallelGenericLoopDeviceFunction(
4187 CodeGenModule &CGM, StringRef ParentName,
4188 const OMPTargetParallelGenericLoopDirective &S);
4189
4190 static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
4191 CodeGenModule &CGM, StringRef ParentName,
4192 const OMPTargetTeamsDistributeParallelForDirective &S);
4193
4194 /// Emit the Stmt \p S and return its topmost canonical loop, if any.
4195 /// TODO: The \p Depth paramter is not yet implemented and must be 1. In the
4196 /// future it is meant to be the number of loops expected in the loop nests
4197 /// (usually specified by the "collapse" clause) that are collapsed to a
4198 /// single loop by this function.
4199 llvm::CanonicalLoopInfo *EmitOMPCollapsedCanonicalLoopNest(const Stmt *S,
4200 int Depth);
4201
4202 /// Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
4203 void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S);
4204
4205 /// Emit inner loop of the worksharing/simd construct.
4206 ///
4207 /// \param S Directive, for which the inner loop must be emitted.
4208 /// \param RequiresCleanup true, if directive has some associated private
4209 /// variables.
4210 /// \param LoopCond Bollean condition for loop continuation.
4211 /// \param IncExpr Increment expression for loop control variable.
4212 /// \param BodyGen Generator for the inner body of the inner loop.
4213 /// \param PostIncGen Genrator for post-increment code (required for ordered
4214 /// loop directvies).
4215 void EmitOMPInnerLoop(
4216 const OMPExecutableDirective &S, bool RequiresCleanup,
4217 const Expr *LoopCond, const Expr *IncExpr,
4218 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
4219 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen);
4220
4221 JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind);
4222 /// Emit initial code for loop counters of loop-based directives.
4223 void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S,
4224 OMPPrivateScope &LoopScope);
4225
4226 /// Helper for the OpenMP loop directives.
4227 void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit);
4228
4229 /// Emit code for the worksharing loop-based directive.
4230 /// \return true, if this construct has any lastprivate clause, false -
4231 /// otherwise.
4232 bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB,
4233 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
4234 const CodeGenDispatchBoundsTy &CGDispatchBounds);
4235
4236 /// Emit code for the distribute loop-based directive.
4237 void EmitOMPDistributeLoop(const OMPLoopDirective &S,
4238 const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr);
4239
4240 /// Helpers for the OpenMP loop directives.
4241 void EmitOMPSimdInit(const OMPLoopDirective &D);
4242 void EmitOMPSimdFinal(
4243 const OMPLoopDirective &D,
4244 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
4245
4246 /// Emits the lvalue for the expression with possibly captured variable.
4247 LValue EmitOMPSharedLValue(const Expr *E);
4248
4249 /// Emits the original address for a structured binding.
4250 Address EmitOMPBindingOriginalAddr(const BindingDecl *BD, SourceLocation Loc);
4251
4252private:
4253 /// Helpers for blocks.
4254 llvm::Value *EmitBlockLiteral(const CGBlockInfo &Info);
4255
4256 /// struct with the values to be passed to the OpenMP loop-related functions
4257 struct OMPLoopArguments {
4258 /// loop lower bound
4259 Address LB = Address::invalid();
4260 /// loop upper bound
4261 Address UB = Address::invalid();
4262 /// loop stride
4263 Address ST = Address::invalid();
4264 /// isLastIteration argument for runtime functions
4265 Address IL = Address::invalid();
4266 /// Chunk value generated by sema
4267 llvm::Value *Chunk = nullptr;
4268 /// EnsureUpperBound
4269 Expr *EUB = nullptr;
4270 /// IncrementExpression
4271 Expr *IncExpr = nullptr;
4272 /// Loop initialization
4273 Expr *Init = nullptr;
4274 /// Loop exit condition
4275 Expr *Cond = nullptr;
4276 /// Update of LB after a whole chunk has been executed
4277 Expr *NextLB = nullptr;
4278 /// Update of UB after a whole chunk has been executed
4279 Expr *NextUB = nullptr;
4280 /// Distinguish between the for distribute and sections
4281 OpenMPDirectiveKind DKind = llvm::omp::OMPD_unknown;
4282 OMPLoopArguments() = default;
4283 OMPLoopArguments(Address LB, Address UB, Address ST, Address IL,
4284 llvm::Value *Chunk = nullptr, Expr *EUB = nullptr,
4285 Expr *IncExpr = nullptr, Expr *Init = nullptr,
4286 Expr *Cond = nullptr, Expr *NextLB = nullptr,
4287 Expr *NextUB = nullptr)
4288 : LB(LB), UB(UB), ST(ST), IL(IL), Chunk(Chunk), EUB(EUB),
4289 IncExpr(IncExpr), Init(Init), Cond(Cond), NextLB(NextLB),
4290 NextUB(NextUB) {}
4291 };
4292 void EmitOMPOuterLoop(bool DynamicOrOrdered, bool IsMonotonic,
4293 const OMPLoopDirective &S, OMPPrivateScope &LoopScope,
4294 const OMPLoopArguments &LoopArgs,
4295 const CodeGenLoopTy &CodeGenLoop,
4296 const CodeGenOrderedTy &CodeGenOrdered);
4297 void EmitOMPForOuterLoop(const OpenMPScheduleTy &ScheduleKind,
4298 bool IsMonotonic, const OMPLoopDirective &S,
4299 OMPPrivateScope &LoopScope, bool Ordered,
4300 const OMPLoopArguments &LoopArgs,
4301 const CodeGenDispatchBoundsTy &CGDispatchBounds);
4302 void EmitOMPDistributeOuterLoop(OpenMPDistScheduleClauseKind ScheduleKind,
4303 const OMPLoopDirective &S,
4304 OMPPrivateScope &LoopScope,
4305 const OMPLoopArguments &LoopArgs,
4306 const CodeGenLoopTy &CodeGenLoopContent);
4307 /// Emit code for sections directive.
4308 void EmitSections(const OMPExecutableDirective &S);
4309
4310public:
4311 //===--------------------------------------------------------------------===//
4312 // OpenACC Emission
4313 //===--------------------------------------------------------------------===//
4314 void EmitOpenACCComputeConstruct(const OpenACCComputeConstruct &S) {
4315 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4316 // simply emitting its structured block, but in the future we will implement
4317 // some sort of IR.
4318 if (S.getStructuredBlock())
4319 EmitStmt(S: S.getStructuredBlock());
4320 }
4321
4322 void EmitOpenACCLoopConstruct(const OpenACCLoopConstruct &S) {
4323 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4324 // simply emitting its loop, but in the future we will implement
4325 // some sort of IR.
4326 if (S.getLoop())
4327 EmitStmt(S: S.getLoop());
4328 }
4329
4330 void EmitOpenACCCombinedConstruct(const OpenACCCombinedConstruct &S) {
4331 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4332 // simply emitting its loop, but in the future we will implement
4333 // some sort of IR.
4334 if (S.getLoop())
4335 EmitStmt(S: S.getLoop());
4336 }
4337
4338 void EmitOpenACCDataConstruct(const OpenACCDataConstruct &S) {
4339 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4340 // simply emitting its structured block, but in the future we will implement
4341 // some sort of IR.
4342 if (S.getStructuredBlock())
4343 EmitStmt(S: S.getStructuredBlock());
4344 }
4345
4346 void EmitOpenACCEnterDataConstruct(const OpenACCEnterDataConstruct &S) {
4347 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4348 // but in the future we will implement some sort of IR.
4349 }
4350
4351 void EmitOpenACCExitDataConstruct(const OpenACCExitDataConstruct &S) {
4352 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4353 // but in the future we will implement some sort of IR.
4354 }
4355
4356 void EmitOpenACCHostDataConstruct(const OpenACCHostDataConstruct &S) {
4357 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4358 // simply emitting its structured block, but in the future we will implement
4359 // some sort of IR.
4360 if (S.getStructuredBlock())
4361 EmitStmt(S: S.getStructuredBlock());
4362 }
4363
4364 void EmitOpenACCWaitConstruct(const OpenACCWaitConstruct &S) {
4365 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4366 // but in the future we will implement some sort of IR.
4367 }
4368
4369 void EmitOpenACCInitConstruct(const OpenACCInitConstruct &S) {
4370 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4371 // but in the future we will implement some sort of IR.
4372 }
4373
4374 void EmitOpenACCShutdownConstruct(const OpenACCShutdownConstruct &S) {
4375 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4376 // but in the future we will implement some sort of IR.
4377 }
4378
4379 void EmitOpenACCSetConstruct(const OpenACCSetConstruct &S) {
4380 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4381 // but in the future we will implement some sort of IR.
4382 }
4383
4384 void EmitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &S) {
4385 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4386 // but in the future we will implement some sort of IR.
4387 }
4388
4389 void EmitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &S) {
4390 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4391 // simply emitting its associated stmt, but in the future we will implement
4392 // some sort of IR.
4393 if (S.getAssociatedStmt())
4394 EmitStmt(S: S.getAssociatedStmt());
4395 }
4396 void EmitOpenACCCacheConstruct(const OpenACCCacheConstruct &S) {
4397 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4398 // but in the future we will implement some sort of IR.
4399 }
4400
4401 //===--------------------------------------------------------------------===//
4402 // LValue Expression Emission
4403 //===--------------------------------------------------------------------===//
4404
4405 /// Create a check that a scalar RValue is non-null.
4406 llvm::Value *EmitNonNullRValueCheck(RValue RV, QualType T);
4407
4408 /// GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
4409 RValue GetUndefRValue(QualType Ty);
4410
4411 /// EmitUnsupportedRValue - Emit a dummy r-value using the type of E
4412 /// and issue an ErrorUnsupported style diagnostic (using the
4413 /// provided Name).
4414 RValue EmitUnsupportedRValue(const Expr *E, const char *Name);
4415
4416 /// EmitUnsupportedLValue - Emit a dummy l-value using the type of E and issue
4417 /// an ErrorUnsupported style diagnostic (using the provided Name).
4418 LValue EmitUnsupportedLValue(const Expr *E, const char *Name);
4419
4420 /// EmitLValue - Emit code to compute a designator that specifies the location
4421 /// of the expression.
4422 ///
4423 /// This can return one of two things: a simple address or a bitfield
4424 /// reference. In either case, the LLVM Value* in the LValue structure is
4425 /// guaranteed to be an LLVM pointer type.
4426 ///
4427 /// If this returns a bitfield reference, nothing about the pointee type of
4428 /// the LLVM value is known: For example, it may not be a pointer to an
4429 /// integer.
4430 ///
4431 /// If this returns a normal address, and if the lvalue's C type is fixed
4432 /// size, this method guarantees that the returned pointer type will point to
4433 /// an LLVM type of the same size of the lvalue's type. If the lvalue has a
4434 /// variable length type, this is not possible.
4435 ///
4436 LValue EmitLValue(const Expr *E,
4437 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
4438
4439private:
4440 LValue EmitLValueHelper(const Expr *E, KnownNonNull_t IsKnownNonNull);
4441
4442public:
4443 /// Same as EmitLValue but additionally we generate checking code to
4444 /// guard against undefined behavior. This is only suitable when we know
4445 /// that the address will be used to access the object.
4446 LValue EmitCheckedLValue(const Expr *E, TypeCheckKind TCK);
4447
4448 RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc);
4449
4450 void EmitAtomicInit(Expr *E, LValue lvalue);
4451
4452 bool LValueIsSuitableForInlineAtomic(LValue Src);
4453
4454 RValue EmitAtomicLoad(LValue LV, SourceLocation SL,
4455 AggValueSlot Slot = AggValueSlot::ignored());
4456
4457 RValue EmitAtomicLoad(LValue lvalue, SourceLocation loc,
4458 llvm::AtomicOrdering AO, bool IsVolatile = false,
4459 AggValueSlot slot = AggValueSlot::ignored());
4460
4461 void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit);
4462
4463 void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO,
4464 bool IsVolatile, bool isInit);
4465
4466 std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
4467 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
4468 llvm::AtomicOrdering Success =
4469 llvm::AtomicOrdering::SequentiallyConsistent,
4470 llvm::AtomicOrdering Failure =
4471 llvm::AtomicOrdering::SequentiallyConsistent,
4472 bool IsWeak = false, AggValueSlot Slot = AggValueSlot::ignored());
4473
4474 /// Emit an atomicrmw instruction, and applying relevant metadata when
4475 /// applicable.
4476 llvm::AtomicRMWInst *emitAtomicRMWInst(
4477 llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val,
4478 llvm::AtomicOrdering Order = llvm::AtomicOrdering::SequentiallyConsistent,
4479 llvm::SyncScope::ID SSID = llvm::SyncScope::System,
4480 const AtomicExpr *AE = nullptr);
4481
4482 /// Emit a fence instruction, applying relevant target-specific metadata when
4483 /// applicable.
4484 llvm::FenceInst *
4485 emitAtomicFence(llvm::AtomicOrdering Order,
4486 llvm::SyncScope::ID SSID = llvm::SyncScope::System);
4487
4488 void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO,
4489 const llvm::function_ref<RValue(RValue)> &UpdateOp,
4490 bool IsVolatile);
4491
4492 /// EmitToMemory - Change a scalar value from its value
4493 /// representation to its in-memory representation.
4494 llvm::Value *EmitToMemory(llvm::Value *Value, QualType Ty);
4495
4496 /// EmitFromMemory - Change a scalar value from its memory
4497 /// representation to its value representation.
4498 llvm::Value *EmitFromMemory(llvm::Value *Value, QualType Ty);
4499
4500 /// Check if the scalar \p Value is within the valid range for the given
4501 /// type \p Ty.
4502 ///
4503 /// Returns true if a check is needed (even if the range is unknown).
4504 bool EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
4505 SourceLocation Loc);
4506
4507 /// EmitLoadOfScalar - Load a scalar value from an address, taking
4508 /// care to appropriately convert from the memory representation to
4509 /// the LLVM value representation.
4510 llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4511 SourceLocation Loc,
4512 AlignmentSource Source = AlignmentSource::Type,
4513 bool isNontemporal = false) {
4514 return EmitLoadOfScalar(Addr, Volatile, Ty, Loc, BaseInfo: LValueBaseInfo(Source),
4515 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isNontemporal);
4516 }
4517
4518 llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4519 SourceLocation Loc, LValueBaseInfo BaseInfo,
4520 TBAAAccessInfo TBAAInfo,
4521 bool isNontemporal = false);
4522
4523 /// EmitLoadOfScalar - Load a scalar value from an address, taking
4524 /// care to appropriately convert from the memory representation to
4525 /// the LLVM value representation. The l-value must be a simple
4526 /// l-value.
4527 llvm::Value *EmitLoadOfScalar(LValue lvalue, SourceLocation Loc);
4528
4529 /// EmitStoreOfScalar - Store a scalar value to an address, taking
4530 /// care to appropriately convert from the memory representation to
4531 /// the LLVM value representation.
4532 void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,
4533 QualType Ty,
4534 AlignmentSource Source = AlignmentSource::Type,
4535 bool isInit = false, bool isNontemporal = false) {
4536 EmitStoreOfScalar(Value, Addr, Volatile, Ty, BaseInfo: LValueBaseInfo(Source),
4537 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isInit, isNontemporal);
4538 }
4539
4540 void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,
4541 QualType Ty, LValueBaseInfo BaseInfo,
4542 TBAAAccessInfo TBAAInfo, bool isInit = false,
4543 bool isNontemporal = false);
4544
4545 /// EmitStoreOfScalar - Store a scalar value to an address, taking
4546 /// care to appropriately convert from the memory representation to
4547 /// the LLVM value representation. The l-value must be a simple
4548 /// l-value. The isInit flag indicates whether this is an initialization.
4549 /// If so, atomic qualifiers are ignored and the store is always non-atomic.
4550 void EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
4551 bool isInit = false);
4552
4553 /// EmitLoadOfLValue - Given an expression that represents a value lvalue,
4554 /// this method emits the address of the lvalue, then loads the result as an
4555 /// rvalue, returning the rvalue.
4556 RValue EmitLoadOfLValue(LValue V, SourceLocation Loc);
4557 RValue EmitLoadOfExtVectorElementLValue(LValue V);
4558 RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc);
4559 RValue EmitLoadOfGlobalRegLValue(LValue LV);
4560
4561 /// Like EmitLoadOfLValue but also handles complex and aggregate types.
4562 RValue EmitLoadOfAnyValue(LValue V,
4563 AggValueSlot Slot = AggValueSlot::ignored(),
4564 SourceLocation Loc = {});
4565
4566 /// EmitStoreThroughLValue - Store the specified rvalue into the specified
4567 /// lvalue, where both are guaranteed to the have the same type, and that type
4568 /// is 'Ty'.
4569 void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit = false);
4570 void EmitStoreThroughExtVectorComponentLValue(RValue Src, LValue Dst);
4571 void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst);
4572
4573 /// EmitStoreThroughBitfieldLValue - Store Src into Dst with same constraints
4574 /// as EmitStoreThroughLValue.
4575 ///
4576 /// \param Result [out] - If non-null, this will be set to a Value* for the
4577 /// bit-field contents after the store, appropriate for use as the result of
4578 /// an assignment to the bit-field.
4579 void EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
4580 llvm::Value **Result = nullptr);
4581
4582 /// Emit an l-value for an assignment (simple or compound) of complex type.
4583 LValue EmitComplexAssignmentLValue(const BinaryOperator *E);
4584 LValue EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E);
4585 LValue EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E,
4586 llvm::Value *&Result);
4587
4588 // Note: only available for agg return types
4589 LValue EmitBinaryOperatorLValue(const BinaryOperator *E);
4590 LValue EmitCompoundAssignmentLValue(const CompoundAssignOperator *E);
4591 // Note: only available for agg return types
4592 LValue EmitCallExprLValue(const CallExpr *E,
4593 llvm::CallBase **CallOrInvoke = nullptr);
4594 // Note: only available for agg return types
4595 LValue EmitVAArgExprLValue(const VAArgExpr *E);
4596 LValue EmitDeclRefLValue(const DeclRefExpr *E);
4597 LValue EmitOMPCapturedBindingLValue(const BindingDecl *BD);
4598 LValue EmitStringLiteralLValue(const StringLiteral *E);
4599 LValue EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E);
4600 LValue EmitPredefinedLValue(const PredefinedExpr *E);
4601 LValue EmitUnaryOpLValue(const UnaryOperator *E);
4602 LValue EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
4603 bool Accessed = false);
4604 llvm::Value *EmitMatrixIndexExpr(const Expr *E);
4605 LValue EmitMatrixSingleSubscriptExpr(const MatrixSingleSubscriptExpr *E);
4606 LValue EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E);
4607 LValue EmitArraySectionExpr(const ArraySectionExpr *E,
4608 bool IsLowerBound = true);
4609 LValue EmitExtVectorElementExpr(const ExtVectorElementExpr *E);
4610 LValue EmitMatrixElementExpr(const MatrixElementExpr *E);
4611 LValue EmitMemberExpr(const MemberExpr *E);
4612 LValue EmitObjCIsaExpr(const ObjCIsaExpr *E);
4613 LValue EmitCompoundLiteralLValue(const CompoundLiteralExpr *E);
4614 LValue EmitInitListLValue(const InitListExpr *E);
4615 void EmitIgnoredConditionalOperator(const AbstractConditionalOperator *E);
4616 LValue EmitConditionalOperatorLValue(const AbstractConditionalOperator *E);
4617 LValue EmitCastLValue(const CastExpr *E);
4618 LValue EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);
4619 LValue EmitOpaqueValueLValue(const OpaqueValueExpr *e);
4620 LValue EmitHLSLArrayAssignLValue(const BinaryOperator *E);
4621
4622 std::pair<LValue, LValue> EmitHLSLOutArgLValues(const HLSLOutArgExpr *E,
4623 QualType Ty);
4624 LValue EmitHLSLOutArgExpr(const HLSLOutArgExpr *E, CallArgList &Args,
4625 QualType Ty, bool IsImplicitObject = false);
4626
4627 Address EmitExtVectorElementLValue(LValue V);
4628
4629 RValue EmitRValueForField(LValue LV, const FieldDecl *FD, SourceLocation Loc);
4630
4631 Address EmitArrayToPointerDecay(const Expr *Array,
4632 LValueBaseInfo *BaseInfo = nullptr,
4633 TBAAAccessInfo *TBAAInfo = nullptr);
4634
4635 class ConstantEmission {
4636 llvm::PointerIntPair<llvm::Constant *, 1, bool> ValueAndIsReference;
4637 ConstantEmission(llvm::Constant *C, bool isReference)
4638 : ValueAndIsReference(C, isReference) {}
4639
4640 public:
4641 ConstantEmission() {}
4642 static ConstantEmission forReference(llvm::Constant *C) {
4643 return ConstantEmission(C, true);
4644 }
4645 static ConstantEmission forValue(llvm::Constant *C) {
4646 return ConstantEmission(C, false);
4647 }
4648
4649 explicit operator bool() const {
4650 return ValueAndIsReference.getOpaqueValue() != nullptr;
4651 }
4652
4653 bool isReference() const { return ValueAndIsReference.getInt(); }
4654 LValue getReferenceLValue(CodeGenFunction &CGF, const Expr *RefExpr) const {
4655 assert(isReference());
4656 return CGF.MakeNaturalAlignAddrLValue(V: ValueAndIsReference.getPointer(),
4657 T: RefExpr->getType());
4658 }
4659
4660 llvm::Constant *getValue() const {
4661 assert(!isReference());
4662 return ValueAndIsReference.getPointer();
4663 }
4664 };
4665
4666 ConstantEmission tryEmitAsConstant(const DeclRefExpr *RefExpr);
4667 ConstantEmission tryEmitAsConstant(const MemberExpr *ME);
4668 llvm::Value *emitScalarConstant(const ConstantEmission &Constant, Expr *E);
4669
4670 RValue EmitPseudoObjectRValue(const PseudoObjectExpr *e,
4671 AggValueSlot slot = AggValueSlot::ignored());
4672 LValue EmitPseudoObjectLValue(const PseudoObjectExpr *e);
4673
4674 void FlattenAccessAndTypeLValue(LValue LVal,
4675 SmallVectorImpl<LValue> &AccessList);
4676
4677 llvm::Value *EmitIvarOffset(const ObjCInterfaceDecl *Interface,
4678 const ObjCIvarDecl *Ivar);
4679 llvm::Value *EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
4680 const ObjCIvarDecl *Ivar);
4681 LValue EmitLValueForField(LValue Base, const FieldDecl *Field,
4682 bool IsInBounds = true);
4683 LValue EmitLValueForLambdaField(const FieldDecl *Field);
4684 LValue EmitLValueForLambdaField(const FieldDecl *Field,
4685 llvm::Value *ThisValue);
4686
4687 /// EmitLValueForFieldInitialization - Like EmitLValueForField, except that
4688 /// if the Field is a reference, this will return the address of the reference
4689 /// and not the address of the value stored in the reference.
4690 LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field);
4691
4692 LValue EmitLValueForIvar(QualType ObjectTy, llvm::Value *Base,
4693 const ObjCIvarDecl *Ivar, unsigned CVRQualifiers);
4694
4695 LValue EmitCXXConstructLValue(const CXXConstructExpr *E);
4696 LValue EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E);
4697 LValue EmitCXXTypeidLValue(const CXXTypeidExpr *E);
4698 LValue EmitCXXUuidofLValue(const CXXUuidofExpr *E);
4699
4700 LValue EmitObjCMessageExprLValue(const ObjCMessageExpr *E);
4701 LValue EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E);
4702 LValue EmitStmtExprLValue(const StmtExpr *E);
4703 LValue EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E);
4704 LValue EmitObjCSelectorLValue(const ObjCSelectorExpr *E);
4705 void EmitDeclRefExprDbgValue(const DeclRefExpr *E, const APValue &Init);
4706
4707 //===--------------------------------------------------------------------===//
4708 // Scalar Expression Emission
4709 //===--------------------------------------------------------------------===//
4710
4711 /// EmitCall - Generate a call of the given function, expecting the given
4712 /// result type, and using the given argument list which specifies both the
4713 /// LLVM arguments and the types they were derived from.
4714 RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4715 ReturnValueSlot ReturnValue, const CallArgList &Args,
4716 llvm::CallBase **CallOrInvoke, bool IsMustTail,
4717 SourceLocation Loc,
4718 bool IsVirtualFunctionPointerThunk = false);
4719 RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4720 ReturnValueSlot ReturnValue, const CallArgList &Args,
4721 llvm::CallBase **CallOrInvoke = nullptr,
4722 bool IsMustTail = false) {
4723 return EmitCall(CallInfo, Callee, ReturnValue, Args, CallOrInvoke,
4724 IsMustTail, Loc: SourceLocation());
4725 }
4726 RValue EmitCall(QualType FnType, const CGCallee &Callee, const CallExpr *E,
4727 ReturnValueSlot ReturnValue, llvm::Value *Chain = nullptr,
4728 llvm::CallBase **CallOrInvoke = nullptr,
4729 CGFunctionInfo const **ResolvedFnInfo = nullptr);
4730
4731 // If a Call or Invoke instruction was emitted for this CallExpr, this method
4732 // writes the pointer to `CallOrInvoke` if it's not null.
4733 RValue EmitCallExpr(const CallExpr *E,
4734 ReturnValueSlot ReturnValue = ReturnValueSlot(),
4735 llvm::CallBase **CallOrInvoke = nullptr);
4736 RValue EmitSimpleCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,
4737 llvm::CallBase **CallOrInvoke = nullptr);
4738 CGCallee EmitCallee(const Expr *E);
4739
4740 void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl);
4741 void checkTargetFeatures(SourceLocation Loc, const FunctionDecl *TargetDecl);
4742
4743 llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4744 const Twine &name = "");
4745 llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4746 ArrayRef<llvm::Value *> args,
4747 const Twine &name = "",
4748 bool IsConvergent = false);
4749 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4750 const Twine &Name = "");
4751 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4752 ArrayRef<llvm::Value *> Args,
4753 const Twine &Name = "");
4754 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4755 ArrayRef<llvm::Type *> Types,
4756 ArrayRef<llvm::Value *> Args,
4757 const Twine &Name = "");
4758 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4759 ArrayRef<llvm::Value *> Args,
4760 llvm::Type *RetTy, const Twine &Name = "");
4761 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4762 const Twine &name = "");
4763 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4764 ArrayRef<Address> args,
4765 const Twine &name = "");
4766 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4767 ArrayRef<llvm::Value *> args,
4768 const Twine &name = "");
4769
4770 SmallVector<llvm::OperandBundleDef, 1>
4771 getBundlesForFunclet(llvm::Value *Callee);
4772
4773 llvm::CallBase *EmitCallOrInvoke(llvm::FunctionCallee Callee,
4774 ArrayRef<llvm::Value *> Args,
4775 const Twine &Name = "");
4776 llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4777 ArrayRef<llvm::Value *> args,
4778 const Twine &name = "");
4779 llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4780 const Twine &name = "");
4781 void EmitNoreturnRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4782 ArrayRef<llvm::Value *> args);
4783
4784 CGCallee BuildAppleKextVirtualCall(const CXXMethodDecl *MD,
4785 NestedNameSpecifier Qual, llvm::Type *Ty);
4786
4787 CGCallee BuildAppleKextVirtualDestructorCall(const CXXDestructorDecl *DD,
4788 CXXDtorType Type,
4789 const CXXRecordDecl *RD);
4790
4791 bool isPointerKnownNonNull(const Expr *E);
4792 /// Check whether the underlying base pointer is a constant null.
4793 bool isUnderlyingBasePointerConstantNull(const Expr *E);
4794
4795 /// Create the discriminator from the storage address and the entity hash.
4796 llvm::Value *EmitPointerAuthBlendDiscriminator(llvm::Value *StorageAddress,
4797 llvm::Value *Discriminator);
4798 CGPointerAuthInfo EmitPointerAuthInfo(const PointerAuthSchema &Schema,
4799 llvm::Value *StorageAddress,
4800 GlobalDecl SchemaDecl,
4801 QualType SchemaType);
4802
4803 llvm::Value *EmitPointerAuthSign(const CGPointerAuthInfo &Info,
4804 llvm::Value *Pointer);
4805
4806 llvm::Value *EmitPointerAuthAuth(const CGPointerAuthInfo &Info,
4807 llvm::Value *Pointer);
4808
4809 llvm::Value *emitPointerAuthResign(llvm::Value *Pointer, QualType PointerType,
4810 const CGPointerAuthInfo &CurAuthInfo,
4811 const CGPointerAuthInfo &NewAuthInfo,
4812 bool IsKnownNonNull);
4813 llvm::Value *emitPointerAuthResignCall(llvm::Value *Pointer,
4814 const CGPointerAuthInfo &CurInfo,
4815 const CGPointerAuthInfo &NewInfo);
4816
4817 void EmitPointerAuthOperandBundle(
4818 const CGPointerAuthInfo &Info,
4819 SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
4820
4821 CGPointerAuthInfo EmitPointerAuthInfo(PointerAuthQualifier Qualifier,
4822 Address StorageAddress);
4823 llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
4824 llvm::Value *Pointer, QualType ValueType,
4825 Address StorageAddress,
4826 bool IsKnownNonNull);
4827 llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
4828 const Expr *PointerExpr,
4829 Address StorageAddress);
4830 llvm::Value *EmitPointerAuthUnqualify(PointerAuthQualifier Qualifier,
4831 llvm::Value *Pointer,
4832 QualType PointerType,
4833 Address StorageAddress,
4834 bool IsKnownNonNull);
4835 void EmitPointerAuthCopy(PointerAuthQualifier Qualifier, QualType Type,
4836 Address DestField, Address SrcField);
4837
4838 std::pair<llvm::Value *, CGPointerAuthInfo>
4839 EmitOrigPointerRValue(const Expr *E);
4840
4841 llvm::Value *authPointerToPointerCast(llvm::Value *ResultPtr,
4842 QualType SourceType, QualType DestType);
4843 Address authPointerToPointerCast(Address Ptr, QualType SourceType,
4844 QualType DestType);
4845
4846 Address getAsNaturalAddressOf(Address Addr, QualType PointeeTy);
4847
4848 llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) {
4849 return getAsNaturalAddressOf(Addr, PointeeTy: PointeeType).getBasePointer();
4850 }
4851
4852 // Return the copy constructor name with the prefix "__copy_constructor_"
4853 // removed.
4854 static std::string getNonTrivialCopyConstructorStr(QualType QT,
4855 CharUnits Alignment,
4856 bool IsVolatile,
4857 ASTContext &Ctx);
4858
4859 // Return the destructor name with the prefix "__destructor_" removed.
4860 static std::string getNonTrivialDestructorStr(QualType QT,
4861 CharUnits Alignment,
4862 bool IsVolatile,
4863 ASTContext &Ctx);
4864
4865 // These functions emit calls to the special functions of non-trivial C
4866 // structs.
4867 void defaultInitNonTrivialCStructVar(LValue Dst);
4868 void callCStructDefaultConstructor(LValue Dst);
4869 void callCStructDestructor(LValue Dst);
4870 void callCStructCopyConstructor(LValue Dst, LValue Src);
4871 void callCStructMoveConstructor(LValue Dst, LValue Src);
4872 void callCStructCopyAssignmentOperator(LValue Dst, LValue Src);
4873 void callCStructMoveAssignmentOperator(LValue Dst, LValue Src);
4874
4875 RValue EmitCXXMemberOrOperatorCall(
4876 const CXXMethodDecl *Method, const CGCallee &Callee,
4877 ReturnValueSlot ReturnValue, llvm::Value *This,
4878 llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *E,
4879 CallArgList *RtlArgs, llvm::CallBase **CallOrInvoke);
4880 RValue EmitCXXDestructorCall(GlobalDecl Dtor, const CGCallee &Callee,
4881 llvm::Value *This, QualType ThisTy,
4882 llvm::Value *ImplicitParam,
4883 QualType ImplicitParamTy, const CallExpr *E,
4884 llvm::CallBase **CallOrInvoke = nullptr);
4885 RValue EmitCXXMemberCallExpr(const CXXMemberCallExpr *E,
4886 ReturnValueSlot ReturnValue,
4887 llvm::CallBase **CallOrInvoke = nullptr);
4888 RValue EmitCXXMemberOrOperatorMemberCallExpr(
4889 const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue,
4890 bool HasQualifier, NestedNameSpecifier Qualifier, bool IsArrow,
4891 const Expr *Base, llvm::CallBase **CallOrInvoke);
4892 // Compute the object pointer.
4893 Address EmitCXXMemberDataPointerAddress(
4894 const Expr *E, Address base, llvm::Value *memberPtr,
4895 const MemberPointerType *memberPtrType, bool IsInBounds,
4896 LValueBaseInfo *BaseInfo = nullptr, TBAAAccessInfo *TBAAInfo = nullptr);
4897 RValue EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
4898 ReturnValueSlot ReturnValue,
4899 llvm::CallBase **CallOrInvoke);
4900
4901 RValue EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
4902 const CXXMethodDecl *MD,
4903 ReturnValueSlot ReturnValue,
4904 llvm::CallBase **CallOrInvoke);
4905 RValue EmitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *E);
4906
4907 RValue EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
4908 ReturnValueSlot ReturnValue,
4909 llvm::CallBase **CallOrInvoke);
4910
4911 RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E);
4912 RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E);
4913
4914 RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
4915 const CallExpr *E, ReturnValueSlot ReturnValue);
4916
4917 RValue emitRotate(const CallExpr *E, bool IsRotateRight);
4918
4919 RValue emitStdcCountIntrinsic(const CallExpr *E, llvm::Intrinsic::ID IntID,
4920 bool InvertArg, bool IsPop = false);
4921 RValue emitStdcBitWidthMinus(const CallExpr *E, llvm::Intrinsic::ID IntID,
4922 bool IsPop);
4923 RValue emitStdcFirstBit(const CallExpr *E, llvm::Intrinsic::ID IntID,
4924 bool InvertArg);
4925
4926 /// Emit IR for __builtin_os_log_format.
4927 RValue emitBuiltinOSLogFormat(const CallExpr &E);
4928
4929 /// Emit IR for __builtin_is_aligned.
4930 RValue EmitBuiltinIsAligned(const CallExpr *E);
4931 /// Emit IR for __builtin_align_up/__builtin_align_down.
4932 RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp);
4933
4934 llvm::Function *generateBuiltinOSLogHelperFunction(
4935 const analyze_os_log::OSLogBufferLayout &Layout,
4936 CharUnits BufferAlignment);
4937
4938 RValue EmitBlockCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,
4939 llvm::CallBase **CallOrInvoke);
4940
4941 /// EmitTargetBuiltinExpr - Emit the given builtin call. Returns 0 if the call
4942 /// is unhandled by the current target.
4943 llvm::Value *EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4944 ReturnValueSlot ReturnValue);
4945
4946 llvm::Value *
4947 EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty,
4948 const llvm::CmpInst::Predicate Pred,
4949 const llvm::Twine &Name = "");
4950 llvm::Value *EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4951 ReturnValueSlot ReturnValue,
4952 llvm::Triple::ArchType Arch);
4953 llvm::Value *EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4954 ReturnValueSlot ReturnValue,
4955 llvm::Triple::ArchType Arch);
4956 llvm::Value *EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4957 ReturnValueSlot ReturnValue,
4958 llvm::Triple::ArchType Arch);
4959 llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::IntegerType *ITy,
4960 QualType RTy);
4961 llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::ArrayType *ATy,
4962 QualType RTy);
4963
4964 llvm::Value *
4965 EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic,
4966 unsigned AltLLVMIntrinsic, const char *NameHint,
4967 unsigned Modifier, const CallExpr *E,
4968 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0,
4969 Address PtrOp1, llvm::Triple::ArchType Arch);
4970
4971 llvm::Function *LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
4972 unsigned Modifier, llvm::Type *ArgTy,
4973 const CallExpr *E);
4974 llvm::Value *EmitNeonCall(llvm::Function *F,
4975 SmallVectorImpl<llvm::Value *> &O, const char *name,
4976 unsigned shift = 0, bool rightshift = false);
4977 llvm::Value *EmitFP8NeonCall(unsigned IID, ArrayRef<llvm::Type *> Tys,
4978 SmallVectorImpl<llvm::Value *> &O,
4979 const CallExpr *E, const char *name);
4980 llvm::Value *EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,
4981 llvm::Type *Ty1, bool Extract,
4982 SmallVectorImpl<llvm::Value *> &Ops,
4983 const CallExpr *E, const char *name);
4984 llvm::Value *EmitFP8NeonFDOTCall(unsigned IID, bool ExtendLaneArg,
4985 llvm::Type *RetTy,
4986 SmallVectorImpl<llvm::Value *> &Ops,
4987 const CallExpr *E, const char *name);
4988 llvm::Value *EmitFP8NeonFMLACall(unsigned IID, bool ExtendLaneArg,
4989 llvm::Type *RetTy,
4990 SmallVectorImpl<llvm::Value *> &Ops,
4991 const CallExpr *E, const char *name);
4992 llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx,
4993 const llvm::ElementCount &Count);
4994 llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx);
4995 llvm::Value *EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty,
4996 bool negateForRightShift);
4997 llvm::Value *EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt,
4998 llvm::Type *Ty, bool usgn, const char *name);
4999 llvm::Value *vectorWrapScalar16(llvm::Value *Op);
5000 /// SVEBuiltinMemEltTy - Returns the memory element type for this memory
5001 /// access builtin. Only required if it can't be inferred from the base
5002 /// pointer operand.
5003 llvm::Type *SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags);
5004
5005 SmallVector<llvm::Type *, 2>
5006 getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType,
5007 ArrayRef<llvm::Value *> Ops);
5008 llvm::Type *getEltType(const SVETypeFlags &TypeFlags);
5009 llvm::ScalableVectorType *getSVEType(const SVETypeFlags &TypeFlags);
5010 llvm::ScalableVectorType *getSVEPredType(const SVETypeFlags &TypeFlags);
5011 llvm::Value *EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags,
5012 ArrayRef<llvm::Value *> Ops);
5013 llvm::Value *EmitSVETupleCreate(const SVETypeFlags &TypeFlags,
5014 llvm::Type *ReturnType,
5015 ArrayRef<llvm::Value *> Ops);
5016 llvm::Value *EmitSVEDupX(llvm::Value *Scalar);
5017 llvm::Value *EmitSVEDupX(llvm::Value *Scalar, llvm::Type *Ty);
5018 llvm::Value *EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty);
5019 llvm::Value *EmitSVEPMull(const SVETypeFlags &TypeFlags,
5020 llvm::SmallVectorImpl<llvm::Value *> &Ops,
5021 unsigned BuiltinID);
5022 llvm::Value *EmitSVEMovl(const SVETypeFlags &TypeFlags,
5023 llvm::ArrayRef<llvm::Value *> Ops,
5024 unsigned BuiltinID);
5025 llvm::Value *EmitSVEPredicateCast(llvm::Value *Pred,
5026 llvm::ScalableVectorType *VTy);
5027 llvm::Value *EmitSVEPredicateTupleCast(llvm::Value *PredTuple,
5028 llvm::StructType *Ty);
5029 llvm::Value *EmitSVEGatherLoad(const SVETypeFlags &TypeFlags,
5030 llvm::SmallVectorImpl<llvm::Value *> &Ops,
5031 unsigned IntID);
5032 llvm::Value *EmitSVEScatterStore(const SVETypeFlags &TypeFlags,
5033 llvm::SmallVectorImpl<llvm::Value *> &Ops,
5034 unsigned IntID);
5035 llvm::Value *EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy,
5036 SmallVectorImpl<llvm::Value *> &Ops,
5037 unsigned BuiltinID, bool IsZExtReturn);
5038 llvm::Value *EmitSVEMaskedStore(const CallExpr *,
5039 SmallVectorImpl<llvm::Value *> &Ops,
5040 unsigned BuiltinID);
5041 llvm::Value *EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,
5042 SmallVectorImpl<llvm::Value *> &Ops,
5043 unsigned BuiltinID);
5044 llvm::Value *EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags,
5045 SmallVectorImpl<llvm::Value *> &Ops,
5046 unsigned IntID);
5047 llvm::Value *EmitSVEStructLoad(const SVETypeFlags &TypeFlags,
5048 SmallVectorImpl<llvm::Value *> &Ops,
5049 unsigned IntID);
5050 llvm::Value *EmitSVEStructStore(const SVETypeFlags &TypeFlags,
5051 SmallVectorImpl<llvm::Value *> &Ops,
5052 unsigned IntID);
5053 llvm::Value *EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5054
5055 llvm::Value *EmitSMELd1St1(const SVETypeFlags &TypeFlags,
5056 llvm::SmallVectorImpl<llvm::Value *> &Ops,
5057 unsigned IntID);
5058 llvm::Value *EmitSMEReadWrite(const SVETypeFlags &TypeFlags,
5059 llvm::SmallVectorImpl<llvm::Value *> &Ops,
5060 unsigned IntID);
5061 llvm::Value *EmitSMEZero(const SVETypeFlags &TypeFlags,
5062 llvm::SmallVectorImpl<llvm::Value *> &Ops,
5063 unsigned IntID);
5064 llvm::Value *EmitSMELdrStr(const SVETypeFlags &TypeFlags,
5065 llvm::SmallVectorImpl<llvm::Value *> &Ops,
5066 unsigned IntID);
5067
5068 void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E,
5069 SmallVectorImpl<llvm::Value *> &Ops,
5070 SVETypeFlags TypeFlags);
5071
5072 llvm::Value *EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5073
5074 llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E,
5075 llvm::Triple::ArchType Arch);
5076 llvm::Value *EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5077
5078 llvm::Value *BuildVector(ArrayRef<llvm::Value *> Ops);
5079 llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5080 llvm::Value *EmitPPCBuiltinCpu(unsigned BuiltinID, llvm::Type *ReturnType,
5081 StringRef CPUStr);
5082 llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5083 llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5084 llvm::Value *EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
5085 ReturnValueSlot ReturnValue);
5086
5087 // Returns a builtin function that the SPIR-V backend will expand into a spec
5088 // constant.
5089 llvm::Function *
5090 getSpecConstantFunction(const clang::QualType &SpecConstantType);
5091
5092 llvm::Value *EmitDirectXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5093 llvm::Value *EmitSPIRVBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5094 llvm::Value *EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx,
5095 const CallExpr *E);
5096 llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5097 llvm::Value *EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5098 llvm::Value *EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
5099 const CallExpr *E);
5100 llvm::Value *EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5101 llvm::Value *EmitAVRBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5102 llvm::Value *EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
5103 ReturnValueSlot ReturnValue);
5104
5105 llvm::Value *EmitRISCVCpuSupports(const CallExpr *E);
5106 llvm::Value *EmitRISCVCpuSupports(ArrayRef<StringRef> FeaturesStrs);
5107 llvm::Value *EmitRISCVCpuInit();
5108 llvm::Value *EmitRISCVCpuIs(const CallExpr *E);
5109 llvm::Value *EmitRISCVCpuIs(StringRef CPUStr);
5110
5111 void AddAMDGPUFenceAddressSpaceMMRA(llvm::Instruction *Inst,
5112 const CallExpr *E);
5113 /// Attach the AMDGPU availability/visibility MMRA to \p Inst when the
5114 /// amdgpu_av attribute is active on the current statement.
5115 void AddAMDGPUAvailableVisibleMMRA(llvm::Instruction *Inst);
5116 void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope,
5117 llvm::AtomicOrdering &AO,
5118 llvm::SyncScope::ID &SSID);
5119
5120 using MSVCIntrin = CodeGenUtils::MSVCIntrin;
5121 llvm::Value *EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E);
5122
5123 llvm::Value *EmitBuiltinAvailable(const VersionTuple &Version);
5124
5125 llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E);
5126 llvm::Value *EmitObjCStringLiteral(const ObjCStringLiteral *E);
5127 llvm::Value *EmitObjCBoxedExpr(const ObjCBoxedExpr *E);
5128 llvm::Value *EmitObjCArrayLiteral(const ObjCArrayLiteral *E);
5129 llvm::Value *EmitObjCDictionaryLiteral(const ObjCDictionaryLiteral *E);
5130 llvm::Value *
5131 EmitObjCCollectionLiteral(const Expr *E,
5132 const ObjCMethodDecl *MethodWithObjects);
5133 llvm::Value *EmitObjCSelectorExpr(const ObjCSelectorExpr *E);
5134 RValue EmitObjCMessageExpr(const ObjCMessageExpr *E,
5135 ReturnValueSlot Return = ReturnValueSlot());
5136
5137 /// Retrieves the default cleanup kind for an ARC cleanup.
5138 /// Except under -fobjc-arc-eh, ARC cleanups are normal-only.
5139 CleanupKind getARCCleanupKind() {
5140 return CGM.getCodeGenOpts().ObjCAutoRefCountExceptions ? NormalAndEHCleanup
5141 : NormalCleanup;
5142 }
5143
5144 // ARC primitives.
5145 void EmitARCInitWeak(Address addr, llvm::Value *value);
5146 void EmitARCDestroyWeak(Address addr);
5147 llvm::Value *EmitARCLoadWeak(Address addr);
5148 llvm::Value *EmitARCLoadWeakRetained(Address addr);
5149 llvm::Value *EmitARCStoreWeak(Address addr, llvm::Value *value, bool ignored);
5150 void emitARCCopyAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
5151 void emitARCMoveAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
5152 void EmitARCCopyWeak(Address dst, Address src);
5153 void EmitARCMoveWeak(Address dst, Address src);
5154 llvm::Value *EmitARCRetainAutorelease(QualType type, llvm::Value *value);
5155 llvm::Value *EmitARCRetainAutoreleaseNonBlock(llvm::Value *value);
5156 llvm::Value *EmitARCStoreStrong(LValue lvalue, llvm::Value *value,
5157 bool resultIgnored);
5158 llvm::Value *EmitARCStoreStrongCall(Address addr, llvm::Value *value,
5159 bool resultIgnored);
5160 llvm::Value *EmitARCRetain(QualType type, llvm::Value *value);
5161 llvm::Value *EmitARCRetainNonBlock(llvm::Value *value);
5162 llvm::Value *EmitARCRetainBlock(llvm::Value *value, bool mandatory);
5163 void EmitARCDestroyStrong(Address addr, ARCPreciseLifetime_t precise);
5164 void EmitARCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
5165 llvm::Value *EmitARCAutorelease(llvm::Value *value);
5166 llvm::Value *EmitARCAutoreleaseReturnValue(llvm::Value *value);
5167 llvm::Value *EmitARCRetainAutoreleaseReturnValue(llvm::Value *value);
5168 llvm::Value *EmitARCRetainAutoreleasedReturnValue(llvm::Value *value);
5169 llvm::Value *EmitARCUnsafeClaimAutoreleasedReturnValue(llvm::Value *value);
5170
5171 llvm::Value *EmitObjCAutorelease(llvm::Value *value, llvm::Type *returnType);
5172 llvm::Value *EmitObjCRetainNonBlock(llvm::Value *value,
5173 llvm::Type *returnType);
5174 void EmitObjCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
5175
5176 std::pair<LValue, llvm::Value *>
5177 EmitARCStoreAutoreleasing(const BinaryOperator *e);
5178 std::pair<LValue, llvm::Value *> EmitARCStoreStrong(const BinaryOperator *e,
5179 bool ignored);
5180 std::pair<LValue, llvm::Value *>
5181 EmitARCStoreUnsafeUnretained(const BinaryOperator *e, bool ignored);
5182
5183 llvm::Value *EmitObjCAlloc(llvm::Value *value, llvm::Type *returnType);
5184 llvm::Value *EmitObjCAllocWithZone(llvm::Value *value,
5185 llvm::Type *returnType);
5186 llvm::Value *EmitObjCAllocInit(llvm::Value *value, llvm::Type *resultType);
5187
5188 llvm::Value *EmitObjCThrowOperand(const Expr *expr);
5189 llvm::Value *EmitObjCConsumeObject(QualType T, llvm::Value *Ptr);
5190 llvm::Value *EmitObjCExtendObjectLifetime(QualType T, llvm::Value *Ptr);
5191
5192 llvm::Value *EmitARCExtendBlockObject(const Expr *expr);
5193 llvm::Value *EmitARCReclaimReturnedObject(const Expr *e,
5194 bool allowUnsafeClaim);
5195 llvm::Value *EmitARCRetainScalarExpr(const Expr *expr);
5196 llvm::Value *EmitARCRetainAutoreleaseScalarExpr(const Expr *expr);
5197 llvm::Value *EmitARCUnsafeUnretainedScalarExpr(const Expr *expr);
5198
5199 void EmitARCIntrinsicUse(ArrayRef<llvm::Value *> values);
5200
5201 void EmitARCNoopIntrinsicUse(ArrayRef<llvm::Value *> values);
5202
5203 static Destroyer destroyARCStrongImprecise;
5204 static Destroyer destroyARCStrongPrecise;
5205 static Destroyer destroyARCWeak;
5206 static Destroyer emitARCIntrinsicUse;
5207 static Destroyer destroyNonTrivialCStruct;
5208
5209 void EmitObjCAutoreleasePoolPop(llvm::Value *Ptr);
5210 llvm::Value *EmitObjCAutoreleasePoolPush();
5211 llvm::Value *EmitObjCMRRAutoreleasePoolPush();
5212 void EmitObjCAutoreleasePoolCleanup(llvm::Value *Ptr);
5213 void EmitObjCMRRAutoreleasePoolPop(llvm::Value *Ptr);
5214
5215 /// Emits a reference binding to the passed in expression.
5216 RValue EmitReferenceBindingToExpr(const Expr *E);
5217
5218 //===--------------------------------------------------------------------===//
5219 // Expression Emission
5220 //===--------------------------------------------------------------------===//
5221
5222 // Expressions are broken into three classes: scalar, complex, aggregate.
5223
5224 /// EmitScalarExpr - Emit the computation of the specified expression of LLVM
5225 /// scalar type, returning the result.
5226 llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
5227
5228 /// Emit a conversion from the specified type to the specified destination
5229 /// type, both of which are LLVM scalar types.
5230 llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
5231 QualType DstTy, SourceLocation Loc);
5232
5233 /// Emit a conversion from the specified complex type to the specified
5234 /// destination type, where the destination type is an LLVM scalar type.
5235 llvm::Value *EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy,
5236 QualType DstTy,
5237 SourceLocation Loc);
5238
5239 /// EmitAggExpr - Emit the computation of the specified expression
5240 /// of aggregate type. The result is computed into the given slot,
5241 /// which may be null to indicate that the value is not needed.
5242 void EmitAggExpr(const Expr *E, AggValueSlot AS);
5243
5244 /// EmitAggExprToLValue - Emit the computation of the specified expression of
5245 /// aggregate type into a temporary LValue.
5246 LValue EmitAggExprToLValue(const Expr *E);
5247
5248 enum ExprValueKind { EVK_RValue, EVK_NonRValue };
5249
5250 /// EmitAggFinalDestCopy - Emit copy of the specified aggregate into
5251 /// destination address.
5252 void EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest, const LValue &Src,
5253 ExprValueKind SrcKind);
5254
5255 /// Create a store to \arg DstPtr from \arg Src, truncating the stored value
5256 /// to at most \arg DstSize bytes.
5257 void CreateCoercedStore(llvm::Value *Src, QualType SrcFETy, Address Dst,
5258 llvm::TypeSize DstSize, bool DstIsVolatile);
5259
5260 /// EmitExtendGCLifetime - Given a pointer to an Objective-C object,
5261 /// make sure it survives garbage collection until this point.
5262 void EmitExtendGCLifetime(llvm::Value *object);
5263
5264 /// EmitComplexExpr - Emit the computation of the specified expression of
5265 /// complex type, returning the result.
5266 ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal = false,
5267 bool IgnoreImag = false);
5268
5269 /// EmitComplexExprIntoLValue - Emit the given expression of complex
5270 /// type and place its result into the specified l-value.
5271 void EmitComplexExprIntoLValue(const Expr *E, LValue dest, bool isInit);
5272
5273 /// EmitStoreOfComplex - Store a complex number into the specified l-value.
5274 void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit);
5275
5276 /// EmitLoadOfComplex - Load a complex number from the specified l-value.
5277 ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc);
5278
5279 ComplexPairTy EmitPromotedComplexExpr(const Expr *E, QualType PromotionType);
5280 llvm::Value *EmitPromotedScalarExpr(const Expr *E, QualType PromotionType);
5281 ComplexPairTy EmitPromotedValue(ComplexPairTy result, QualType PromotionType);
5282 ComplexPairTy EmitUnPromotedValue(ComplexPairTy result,
5283 QualType PromotionType);
5284
5285 Address emitAddrOfRealComponent(Address complex, QualType complexType);
5286 Address emitAddrOfImagComponent(Address complex, QualType complexType);
5287
5288 /// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the
5289 /// global variable that has already been created for it. If the initializer
5290 /// has a different type than GV does, this may free GV and return a different
5291 /// one. Otherwise it just returns GV.
5292 llvm::GlobalVariable *AddInitializerToStaticVarDecl(const VarDecl &D,
5293 llvm::GlobalVariable *GV);
5294
5295 // Emit an @llvm.invariant.start call for the given memory region.
5296 void EmitInvariantStart(llvm::Constant *Addr, CharUnits Size);
5297
5298 /// EmitCXXGlobalVarDeclInit - Create the initializer for a C++
5299 /// variable with global storage.
5300 void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::GlobalVariable *GV,
5301 bool PerformInit);
5302
5303 llvm::Constant *createAtExitStub(const VarDecl &VD, llvm::FunctionCallee Dtor,
5304 llvm::Constant *Addr);
5305
5306 llvm::Function *createTLSAtExitStub(const VarDecl &VD,
5307 llvm::FunctionCallee Dtor,
5308 llvm::Constant *Addr,
5309 llvm::FunctionCallee &AtExit);
5310
5311 /// Call atexit() with a function that passes the given argument to
5312 /// the given function.
5313 void registerGlobalDtorWithAtExit(const VarDecl &D, llvm::FunctionCallee fn,
5314 llvm::Constant *addr);
5315
5316 /// Registers the dtor using 'llvm.global_dtors' for platforms that do not
5317 /// support an 'atexit()' function.
5318 void registerGlobalDtorWithLLVM(const VarDecl &D, llvm::FunctionCallee fn,
5319 llvm::Constant *addr);
5320
5321 /// Call atexit() with function dtorStub.
5322 void registerGlobalDtorWithAtExit(llvm::Constant *dtorStub);
5323
5324 /// Call unatexit() with function dtorStub.
5325 llvm::Value *unregisterGlobalDtorWithUnAtExit(llvm::Constant *dtorStub);
5326
5327 /// Emit code in this function to perform a guarded variable
5328 /// initialization. Guarded initializations are used when it's not
5329 /// possible to prove that an initialization will be done exactly
5330 /// once, e.g. with a static local variable or a static data member
5331 /// of a class template.
5332 void EmitCXXGuardedInit(const VarDecl &D, llvm::GlobalVariable *DeclPtr,
5333 bool PerformInit);
5334
5335 enum class GuardKind { VariableGuard, TlsGuard };
5336
5337 /// Emit a branch to select whether or not to perform guarded initialization.
5338 void EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
5339 llvm::BasicBlock *InitBlock,
5340 llvm::BasicBlock *NoInitBlock, GuardKind Kind,
5341 const VarDecl *D);
5342
5343 /// GenerateCXXGlobalInitFunc - Generates code for initializing global
5344 /// variables.
5345 void
5346 GenerateCXXGlobalInitFunc(llvm::Function *Fn,
5347 ArrayRef<llvm::Function *> CXXThreadLocals,
5348 ConstantAddress Guard = ConstantAddress::invalid());
5349
5350 /// GenerateCXXGlobalCleanUpFunc - Generates code for cleaning up global
5351 /// variables.
5352 void GenerateCXXGlobalCleanUpFunc(
5353 llvm::Function *Fn,
5354 ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
5355 llvm::Constant *>>
5356 DtorsOrStermFinalizers);
5357
5358 void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn, const VarDecl *D,
5359 llvm::GlobalVariable *Addr,
5360 bool PerformInit);
5361
5362 void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest);
5363
5364 void EmitSynthesizedCXXCopyCtor(Address Dest, Address Src, const Expr *Exp);
5365
5366 void EmitCXXThrowExpr(const CXXThrowExpr *E, bool KeepInsertionPoint = true);
5367
5368 RValue EmitAtomicExpr(AtomicExpr *E);
5369
5370 void EmitFakeUse(Address Addr);
5371
5372 //===--------------------------------------------------------------------===//
5373 // Annotations Emission
5374 //===--------------------------------------------------------------------===//
5375
5376 /// Emit an annotation call (intrinsic).
5377 llvm::Value *EmitAnnotationCall(llvm::Function *AnnotationFn,
5378 llvm::Value *AnnotatedVal,
5379 StringRef AnnotationStr,
5380 SourceLocation Location,
5381 const AnnotateAttr *Attr);
5382
5383 /// Emit local annotations for the local variable V, declared by D.
5384 void EmitVarAnnotations(const VarDecl *D, llvm::Value *V);
5385
5386 /// Emit field annotations for the given field & value. Returns the
5387 /// annotation result.
5388 Address EmitFieldAnnotations(const FieldDecl *D, Address V);
5389
5390 //===--------------------------------------------------------------------===//
5391 // Internal Helpers
5392 //===--------------------------------------------------------------------===//
5393
5394 /// ContainsLabel - Return true if the statement contains a label in it. If
5395 /// this statement is not executed normally, it not containing a label means
5396 /// that we can just remove the code.
5397 static bool ContainsLabel(const Stmt *S, bool IgnoreCaseStmts = false);
5398
5399 /// containsBreak - Return true if the statement contains a break out of it.
5400 /// If the statement (recursively) contains a switch or loop with a break
5401 /// inside of it, this is fine.
5402 static bool containsBreak(const Stmt *S);
5403
5404 /// Determine if the given statement might introduce a declaration into the
5405 /// current scope, by being a (possibly-labelled) DeclStmt.
5406 static bool mightAddDeclToScope(const Stmt *S);
5407
5408 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
5409 /// to a constant, or if it does but contains a label, return false. If it
5410 /// constant folds return true and set the boolean result in Result.
5411 bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result,
5412 bool AllowLabels = false);
5413
5414 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
5415 /// to a constant, or if it does but contains a label, return false. If it
5416 /// constant folds return true and set the folded value.
5417 bool ConstantFoldsToSimpleInteger(const Expr *Cond, llvm::APSInt &Result,
5418 bool AllowLabels = false);
5419
5420 /// Ignore parentheses and logical-NOT to track conditions consistently.
5421 static const Expr *stripCond(const Expr *C);
5422
5423 /// isInstrumentedCondition - Determine whether the given condition is an
5424 /// instrumentable condition (i.e. no "&&" or "||").
5425 static bool isInstrumentedCondition(const Expr *C);
5426
5427 /// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
5428 /// increments a profile counter based on the semantics of the given logical
5429 /// operator opcode. This is used to instrument branch condition coverage
5430 /// for logical operators.
5431 void EmitBranchToCounterBlock(const Expr *Cond, BinaryOperator::Opcode LOp,
5432 llvm::BasicBlock *TrueBlock,
5433 llvm::BasicBlock *FalseBlock,
5434 uint64_t TrueCount = 0,
5435 Stmt::Likelihood LH = Stmt::LH_None,
5436 const Expr *CntrIdx = nullptr);
5437
5438 /// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an
5439 /// if statement) to the specified blocks. Based on the condition, this might
5440 /// try to simplify the codegen of the conditional based on the branch.
5441 /// TrueCount should be the number of times we expect the condition to
5442 /// evaluate to true based on PGO data.
5443 void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock,
5444 llvm::BasicBlock *FalseBlock, uint64_t TrueCount,
5445 Stmt::Likelihood LH = Stmt::LH_None,
5446 const Expr *ConditionalOp = nullptr,
5447 const VarDecl *ConditionalDecl = nullptr);
5448
5449 /// Given an assignment `*LHS = RHS`, emit a test that checks if \p RHS is
5450 /// nonnull, if \p LHS is marked _Nonnull.
5451 void EmitNullabilityCheck(LValue LHS, llvm::Value *RHS, SourceLocation Loc);
5452
5453 /// An enumeration which makes it easier to specify whether or not an
5454 /// operation is a subtraction.
5455 enum { NotSubtraction = false, IsSubtraction = true };
5456
5457 /// Emit pointer + index arithmetic.
5458 llvm::Value *EmitPointerArithmetic(const BinaryOperator *BO,
5459 Expr *pointerOperand, llvm::Value *pointer,
5460 Expr *indexOperand, llvm::Value *index,
5461 bool isSubtraction);
5462
5463 /// Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to
5464 /// detect undefined behavior when the pointer overflow sanitizer is enabled.
5465 /// \p SignedIndices indicates whether any of the GEP indices are signed.
5466 /// \p IsSubtraction indicates whether the expression used to form the GEP
5467 /// is a subtraction.
5468 llvm::Value *EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr,
5469 ArrayRef<llvm::Value *> IdxList,
5470 bool SignedIndices, bool IsSubtraction,
5471 SourceLocation Loc,
5472 const Twine &Name = "");
5473
5474 Address EmitCheckedInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
5475 llvm::Type *elementType, bool SignedIndices,
5476 bool IsSubtraction, SourceLocation Loc,
5477 CharUnits Align, const Twine &Name = "");
5478
5479 /// Specifies which type of sanitizer check to apply when handling a
5480 /// particular builtin.
5481 enum BuiltinCheckKind {
5482 BCK_CTZPassedZero,
5483 BCK_CLZPassedZero,
5484 BCK_AssumePassedFalse,
5485 BCK_NonUniformLane,
5486 BCK_InactiveLane,
5487 };
5488
5489 /// Emits an argument for a call to a builtin. If the builtin sanitizer is
5490 /// enabled, a runtime check specified by \p Kind is also emitted.
5491 llvm::Value *EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind);
5492
5493 /// Emits an argument for a call to a `__builtin_assume`. If the builtin
5494 /// sanitizer is enabled, a runtime check is also emitted.
5495 llvm::Value *EmitCheckedArgForAssume(const Expr *E);
5496
5497 /// Emit a description of a type in a format suitable for passing to
5498 /// a runtime sanitizer handler.
5499 llvm::Constant *EmitCheckTypeDescriptor(QualType T);
5500
5501 /// Convert a value into a format suitable for passing to a runtime
5502 /// sanitizer handler.
5503 llvm::Value *EmitCheckValue(llvm::Value *V);
5504
5505 /// Emit a description of a source location in a format suitable for
5506 /// passing to a runtime sanitizer handler.
5507 llvm::Constant *EmitCheckSourceLocation(SourceLocation Loc);
5508
5509 void EmitKCFIOperandBundle(const CGCallee &Callee,
5510 SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
5511
5512 /// Create a basic block that will either trap or call a handler function in
5513 /// the UBSan runtime with the provided arguments, and create a conditional
5514 /// branch to it.
5515 void
5516 EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
5517 Checked,
5518 SanitizerHandler Check, ArrayRef<llvm::Constant *> StaticArgs,
5519 ArrayRef<llvm::Value *> DynamicArgs,
5520 const TrapReason *TR = nullptr);
5521
5522 /// Emit a slow path cross-DSO CFI check which calls __cfi_slowpath
5523 /// if Cond if false.
5524 void EmitCfiSlowPathCheck(SanitizerKind::SanitizerOrdinal Ordinal,
5525 llvm::Value *Cond, llvm::ConstantInt *TypeId,
5526 llvm::Value *Ptr,
5527 ArrayRef<llvm::Constant *> StaticArgs);
5528
5529 /// Emit a reached-unreachable diagnostic if \p Loc is valid and runtime
5530 /// checking is enabled. Otherwise, just emit an unreachable instruction.
5531 void EmitUnreachable(SourceLocation Loc);
5532
5533 /// Create a basic block that will call the trap intrinsic, and emit a
5534 /// conditional branch to it, for the -ftrapv checks.
5535 void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID,
5536 bool NoMerge = false, const TrapReason *TR = nullptr);
5537
5538 /// Emit a call to trap or debugtrap. If 'EnsureInsertPoint' is false, the
5539 /// IR builder need not have a valid insert point after this returns.
5540 llvm::CallInst *EmitTrapCall(llvm::Intrinsic::ID IntrID,
5541 bool EnsureInsertPoint = true);
5542
5543 /// Emit a call to '\@llvm.trap()' and clear the current insert point.
5544 void EmitTrapCallAndMakeUnreachable();
5545
5546 /// Emit a stub for the cross-DSO CFI check function.
5547 void EmitCfiCheckStub();
5548
5549 /// Emit a cross-DSO CFI failure handling function.
5550 void EmitCfiCheckFail();
5551
5552 /// Create a check for a function parameter that may potentially be
5553 /// declared as non-null.
5554 void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc,
5555 AbstractCallee AC, unsigned ParmNum);
5556
5557 void EmitNonNullArgCheck(Address Addr, QualType ArgType,
5558 SourceLocation ArgLoc, AbstractCallee AC,
5559 unsigned ParmNum);
5560
5561 /// EmitWriteback - Emit callbacks for function.
5562 void EmitWritebacks(const CallArgList &Args);
5563
5564 /// EmitCallArg - Emit a single call argument.
5565 void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType);
5566
5567 /// EmitDelegateCallArg - We are performing a delegate call; that
5568 /// is, the current function is delegating to another one. Produce
5569 /// a r-value suitable for passing the given parameter.
5570 void EmitDelegateCallArg(CallArgList &args, const VarDecl *param,
5571 SourceLocation loc);
5572
5573 /// SetFPAccuracy - Set the minimum required accuracy of the given floating
5574 /// point operation, expressed as the maximum relative error in ulp.
5575 void SetFPAccuracy(llvm::Value *Val, float Accuracy);
5576
5577 /// Set the minimum required accuracy of the given sqrt operation
5578 /// based on CodeGenOpts.
5579 void SetSqrtFPAccuracy(llvm::Value *Val);
5580
5581 /// Set the minimum required accuracy of the given sqrt operation based on
5582 /// CodeGenOpts.
5583 void SetDivFPAccuracy(llvm::Value *Val);
5584
5585 /// Set the codegen fast-math flags.
5586 void SetFastMathFlags(FPOptions FPFeatures);
5587
5588 // Truncate or extend a boolean vector to the requested number of elements.
5589 llvm::Value *emitBoolVecConversion(llvm::Value *SrcVec,
5590 unsigned NumElementsDst,
5591 const llvm::Twine &Name = "");
5592
5593 void maybeAttachRangeForLoad(llvm::LoadInst *Load, QualType Ty,
5594 SourceLocation Loc);
5595
5596 // Emits a convergence_loop instruction for the given |BB|, with |ParentToken|
5597 // as it's parent convergence instr.
5598 llvm::ConvergenceControlInst *emitConvergenceLoopToken(llvm::BasicBlock *BB);
5599
5600private:
5601 // Adds a convergencectrl bundle with the current convergence token to
5602 // |Bundles|, if convergence tokens are used.
5603 void
5604 addConvergenceControlBundle(SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
5605
5606 // Marks |Call| convergent if |IsConvergent| and its callee does not already
5607 // imply it.
5608 void setConvergentCallSite(llvm::CallBase *Call, bool IsConvergent);
5609
5610 // Find the convergence_entry instruction |F|, or emits ones if none exists.
5611 // Returns the convergence instruction.
5612 llvm::ConvergenceControlInst *
5613 getOrEmitConvergenceEntryToken(llvm::Function *F);
5614
5615private:
5616 llvm::MDNode *getRangeForLoadFromType(QualType Ty);
5617 void EmitReturnOfRValue(RValue RV, QualType Ty);
5618
5619 void deferPlaceholderReplacement(llvm::Instruction *Old, llvm::Value *New);
5620
5621 llvm::SmallVector<std::pair<llvm::WeakTrackingVH, llvm::Value *>, 4>
5622 DeferredReplacements;
5623
5624 /// Set the address of a local variable.
5625 void setAddrOfLocalVar(const VarDecl *VD, Address Addr) {
5626 assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!");
5627 LocalDeclMap.insert(KV: {VD, Addr});
5628 }
5629
5630 /// ExpandTypeFromArgs - Reconstruct a structure of type \arg Ty
5631 /// from function arguments into \arg Dst. See ABIArgInfo::Expand.
5632 ///
5633 /// \param AI - The first function argument of the expansion.
5634 void ExpandTypeFromArgs(QualType Ty, LValue Dst,
5635 llvm::Function::arg_iterator &AI);
5636
5637 /// ExpandTypeToArgs - Expand an CallArg \arg Arg, with the LLVM type for \arg
5638 /// Ty, into individual arguments on the provided vector \arg IRCallArgs,
5639 /// starting at index \arg IRCallArgPos. See ABIArgInfo::Expand.
5640 void ExpandTypeToArgs(QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
5641 SmallVectorImpl<llvm::Value *> &IRCallArgs,
5642 unsigned &IRCallArgPos);
5643
5644 std::pair<llvm::Value *, llvm::Type *>
5645 EmitAsmInput(const TargetInfo::ConstraintInfo &Info, const Expr *InputExpr,
5646 std::string &ConstraintStr);
5647
5648 std::pair<llvm::Value *, llvm::Type *>
5649 EmitAsmInputLValue(const TargetInfo::ConstraintInfo &Info, LValue InputValue,
5650 QualType InputType, std::string &ConstraintStr,
5651 SourceLocation Loc);
5652
5653 /// Attempts to statically evaluate the object size of E. If that
5654 /// fails, emits code to figure the size of E out for us. This is
5655 /// pass_object_size aware.
5656 ///
5657 /// If EmittedExpr is non-null, this will use that instead of re-emitting E.
5658 llvm::Value *evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
5659 llvm::IntegerType *ResType,
5660 llvm::Value *EmittedE,
5661 bool IsDynamic);
5662
5663 /// Emits the size of E, as required by __builtin_object_size. This
5664 /// function is aware of pass_object_size parameters, and will act accordingly
5665 /// if E is a parameter with the pass_object_size attribute.
5666 llvm::Value *emitBuiltinObjectSize(const Expr *E, unsigned Type,
5667 llvm::IntegerType *ResType,
5668 llvm::Value *EmittedE, bool IsDynamic);
5669
5670 llvm::Value *emitCountedBySize(const Expr *E, llvm::Value *EmittedE,
5671 unsigned Type, llvm::IntegerType *ResType);
5672
5673 llvm::Value *emitCountedByMemberSize(const MemberExpr *E, const Expr *Idx,
5674 llvm::Value *EmittedE,
5675 QualType CastedArrayElementTy,
5676 unsigned Type,
5677 llvm::IntegerType *ResType);
5678
5679 llvm::Value *emitCountedByPointerSize(const ImplicitCastExpr *E,
5680 const Expr *Idx, llvm::Value *EmittedE,
5681 QualType CastedArrayElementTy,
5682 unsigned Type,
5683 llvm::IntegerType *ResType);
5684
5685 void emitZeroOrPatternForAutoVarInit(QualType type, const VarDecl &D,
5686 Address Loc);
5687 LangOptions::TrivialAutoVarInitKind getAutoVarInitKind(QualType Ty,
5688 const VarDecl &D);
5689
5690public:
5691 using EvaluationOrder = CodeGenUtils::EvaluationOrder;
5692
5693 // Wrapper for function prototype sources. Wraps either a FunctionProtoType or
5694 // an ObjCMethodDecl.
5695 struct PrototypeWrapper {
5696 llvm::PointerUnion<const FunctionProtoType *, const ObjCMethodDecl *> P;
5697
5698 PrototypeWrapper(const FunctionProtoType *FT) : P(FT) {}
5699 PrototypeWrapper(const ObjCMethodDecl *MD) : P(MD) {}
5700 };
5701
5702 void EmitCallArgs(CallArgList &Args, PrototypeWrapper Prototype,
5703 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
5704 AbstractCallee AC = AbstractCallee(),
5705 unsigned ParamsToSkip = 0,
5706 EvaluationOrder Order = EvaluationOrder::Default);
5707
5708 /// EmitPointerWithAlignment - Given an expression with a pointer type,
5709 /// emit the value and compute our best estimate of the alignment of the
5710 /// pointee.
5711 ///
5712 /// \param BaseInfo - If non-null, this will be initialized with
5713 /// information about the source of the alignment and the may-alias
5714 /// attribute. Note that this function will conservatively fall back on
5715 /// the type when it doesn't recognize the expression and may-alias will
5716 /// be set to false.
5717 ///
5718 /// One reasonable way to use this information is when there's a language
5719 /// guarantee that the pointer must be aligned to some stricter value, and
5720 /// we're simply trying to ensure that sufficiently obvious uses of under-
5721 /// aligned objects don't get miscompiled; for example, a placement new
5722 /// into the address of a local variable. In such a case, it's quite
5723 /// reasonable to just ignore the returned alignment when it isn't from an
5724 /// explicit source.
5725 Address
5726 EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo = nullptr,
5727 TBAAAccessInfo *TBAAInfo = nullptr,
5728 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
5729
5730 /// If \p E references a parameter with pass_object_size info or a constant
5731 /// array size modifier, emit the object size divided by the size of \p EltTy.
5732 /// Otherwise return null.
5733 llvm::Value *LoadPassedObjectSize(const Expr *E, QualType EltTy);
5734
5735 void EmitSanitizerStatReport(llvm::SanitizerStatKind SSK);
5736
5737 struct FMVResolverOption {
5738 llvm::Function *Function;
5739 llvm::SmallVector<StringRef, 8> Features;
5740 std::optional<StringRef> Architecture;
5741
5742 FMVResolverOption(llvm::Function *F, ArrayRef<StringRef> Feats,
5743 std::optional<StringRef> Arch = std::nullopt)
5744 : Function(F), Features(Feats), Architecture(Arch) {}
5745 };
5746
5747 // Emits the body of a multiversion function's resolver. Assumes that the
5748 // options are already sorted in the proper order, with the 'default' option
5749 // last (if it exists).
5750 void EmitMultiVersionResolver(llvm::Function *Resolver,
5751 ArrayRef<FMVResolverOption> Options);
5752 void EmitX86MultiVersionResolver(llvm::Function *Resolver,
5753 ArrayRef<FMVResolverOption> Options);
5754 void EmitAArch64MultiVersionResolver(llvm::Function *Resolver,
5755 ArrayRef<FMVResolverOption> Options);
5756 void EmitRISCVMultiVersionResolver(llvm::Function *Resolver,
5757 ArrayRef<FMVResolverOption> Options);
5758 void EmitPPCAIXMultiVersionResolver(llvm::Function *Resolver,
5759 ArrayRef<FMVResolverOption> Options);
5760
5761 Address EmitAddressOfPFPField(Address RecordPtr, const PFPField &Field);
5762 Address EmitAddressOfPFPField(Address RecordPtr, Address FieldPtr,
5763 const FieldDecl *Field);
5764
5765private:
5766 QualType getVarArgType(const Expr *Arg);
5767
5768 void EmitDeclMetadata();
5769
5770 BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType,
5771 const AutoVarEmission &emission);
5772
5773 void AddObjCARCExceptionMetadata(llvm::Instruction *Inst);
5774
5775 llvm::Value *GetValueForARMHint(unsigned BuiltinID);
5776 llvm::Value *EmitX86CpuIs(const CallExpr *E);
5777 llvm::Value *EmitX86CpuIs(StringRef CPUStr);
5778 llvm::Value *EmitX86CpuSupports(const CallExpr *E);
5779 llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs);
5780 llvm::Value *EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask);
5781 llvm::Value *EmitX86CpuInit();
5782 llvm::Value *FormX86ResolverCondition(const FMVResolverOption &RO);
5783 llvm::Value *EmitAArch64CpuInit();
5784 llvm::Value *FormAArch64ResolverCondition(const FMVResolverOption &RO);
5785 llvm::Value *EmitAArch64CpuSupports(const CallExpr *E);
5786 llvm::Value *EmitAArch64CpuSupports(ArrayRef<StringRef> FeatureStrs);
5787};
5788
5789inline DominatingLLVMValue::saved_type
5790DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) {
5791 if (!needsSaving(value))
5792 return saved_type(value);
5793
5794 // Otherwise, we need an alloca.
5795 auto align = CharUnits::fromQuantity(
5796 Quantity: CGF.CGM.getDataLayout().getPrefTypeAlign(Ty: value->getType()))
5797 .getAsAlign();
5798 llvm::AllocaInst *AI =
5799 CGF.CreateTempAlloca(Ty: value->getType(), Name: "cond-cleanup.save");
5800 AI->setAlignment(align);
5801 CGF.Builder.CreateAlignedStore(Val: value, Ptr: AI, Align: align);
5802
5803 return saved_type(AI, value->getType());
5804}
5805
5806inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF,
5807 saved_type value) {
5808 // If the value says it wasn't saved, trust that it's still dominating.
5809 if (!value.isSaved())
5810 return value.Value;
5811
5812 // Otherwise, it should be an alloca instruction, as set up in save().
5813 auto Alloca = cast<llvm::AllocaInst>(Val: value.Value);
5814 return CGF.Builder.CreateAlignedLoad(Ty: value.Type, Ptr: Alloca, Align: Alloca->getAlign());
5815}
5816
5817} // end namespace CodeGen
5818
5819// Map the LangOption for floating point exception behavior into
5820// the corresponding enum in the IR.
5821llvm::fp::ExceptionBehavior
5822ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind);
5823} // end namespace clang
5824
5825#endif
5826