1//===-- CodeGenFunction.h - Per-Function state for LLVM CodeGen -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the internal per-function state used for llvm translation.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
14#define LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H
15
16#include "CGBuilder.h"
17#include "CGLoopInfo.h"
18#include "CGValue.h"
19#include "CodeGenModule.h"
20#include "EHScopeStack.h"
21#include "SanitizerHandler.h"
22#include "VarBypassDetector.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/CharUnits.h"
25#include "clang/AST/CurrentSourceLocExprScope.h"
26#include "clang/AST/ExprCXX.h"
27#include "clang/AST/ExprObjC.h"
28#include "clang/AST/ExprOpenMP.h"
29#include "clang/AST/StmtOpenACC.h"
30#include "clang/AST/StmtOpenMP.h"
31#include "clang/AST/StmtSYCL.h"
32#include "clang/AST/Type.h"
33#include "clang/Basic/ABI.h"
34#include "clang/Basic/CapturedStmt.h"
35#include "clang/Basic/CodeGenOptions.h"
36#include "clang/Basic/OpenMPKinds.h"
37#include "clang/Basic/TargetInfo.h"
38#include "clang/CodeGenUtils/CallUtils.h"
39#include "llvm/ADT/ArrayRef.h"
40#include "llvm/ADT/DenseMap.h"
41#include "llvm/ADT/MapVector.h"
42#include "llvm/ADT/SmallVector.h"
43#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
44#include "llvm/IR/Instructions.h"
45#include "llvm/IR/ValueHandle.h"
46#include "llvm/Support/Debug.h"
47#include "llvm/Transforms/Utils/SanitizerStats.h"
48#include <optional>
49
50namespace llvm {
51class BasicBlock;
52class ConvergenceControlInst;
53class LLVMContext;
54class MDNode;
55class SwitchInst;
56class Twine;
57class Value;
58class CanonicalLoopInfo;
59} // namespace llvm
60
61namespace clang {
62class ASTContext;
63class AsmConstraintsInfo;
64class CXXDestructorDecl;
65class CXXForRangeStmt;
66class CXXTryStmt;
67class Decl;
68class LabelDecl;
69class FunctionDecl;
70class FunctionProtoType;
71class LabelStmt;
72class ObjCContainerDecl;
73class ObjCInterfaceDecl;
74class ObjCIvarDecl;
75class ObjCMethodDecl;
76class ObjCImplementationDecl;
77class ObjCPropertyImplDecl;
78class TargetInfo;
79class VarDecl;
80class ObjCForCollectionStmt;
81class ObjCAtTryStmt;
82class ObjCAtThrowStmt;
83class ObjCAtSynchronizedStmt;
84class ObjCAutoreleasePoolStmt;
85class OMPUseDevicePtrClause;
86class OMPUseDeviceAddrClause;
87class SVETypeFlags;
88class OMPExecutableDirective;
89
90namespace analyze_os_log {
91class OSLogBufferLayout;
92}
93
94namespace CodeGen {
95class CodeGenTypes;
96class CodeGenPGO;
97class CGCallee;
98class CGFunctionInfo;
99class CGBlockInfo;
100class CGCXXABI;
101class BlockByrefHelpers;
102class BlockByrefInfo;
103class BlockFieldFlags;
104class RegionCodeGenTy;
105class TargetCodeGenInfo;
106struct OMPTaskDataTy;
107struct CGCoroData;
108
109// clang-format off
110/// The kind of evaluation to perform on values of a particular
111/// type. Basically, is the code in CGExprScalar, CGExprComplex, or
112/// CGExprAgg?
113///
114/// TODO: should vectors maybe be split out into their own thing?
115enum TypeEvaluationKind {
116 TEK_Scalar,
117 TEK_Complex,
118 TEK_Aggregate
119};
120// clang-format on
121
122/// Helper class with most of the code for saving a value for a
123/// conditional expression cleanup.
124struct DominatingLLVMValue {
125 struct saved_type {
126 llvm::Value *Value; // Original value if not saved, alloca if saved
127 llvm::Type *Type; // nullptr if not saved, element type if saved
128
129 saved_type() : Value(nullptr), Type(nullptr) {}
130 saved_type(llvm::Value *V) : Value(V), Type(nullptr) {}
131 saved_type(llvm::AllocaInst *Alloca, llvm::Type *Ty)
132 : Value(Alloca), Type(Ty) {}
133
134 bool isSaved() const { return Type != nullptr; }
135 };
136
137 /// Answer whether the given value needs extra work to be saved.
138 static bool needsSaving(llvm::Value *value) {
139 if (!value)
140 return false;
141
142 // If it's not an instruction, we don't need to save.
143 if (!isa<llvm::Instruction>(Val: value))
144 return false;
145
146 // If it's an instruction in the entry block, we don't need to save.
147 llvm::BasicBlock *block = cast<llvm::Instruction>(Val: value)->getParent();
148 return (block != &block->getParent()->getEntryBlock());
149 }
150
151 static saved_type save(CodeGenFunction &CGF, llvm::Value *value);
152 static llvm::Value *restore(CodeGenFunction &CGF, saved_type value);
153};
154
155/// A partial specialization of DominatingValue for llvm::Values that
156/// might be llvm::Instructions.
157template <class T> struct DominatingPointer<T, true> : DominatingLLVMValue {
158 typedef T *type;
159 static type restore(CodeGenFunction &CGF, saved_type value) {
160 return static_cast<T *>(DominatingLLVMValue::restore(CGF, value));
161 }
162};
163
164/// A specialization of DominatingValue for Address.
165template <> struct DominatingValue<Address> {
166 typedef Address type;
167
168 struct saved_type {
169 DominatingLLVMValue::saved_type BasePtr;
170 llvm::Type *ElementType;
171 CharUnits Alignment;
172 DominatingLLVMValue::saved_type Offset;
173 llvm::PointerType *EffectiveType;
174 };
175
176 static bool needsSaving(type value) {
177 if (DominatingLLVMValue::needsSaving(value: value.getBasePointer()) ||
178 DominatingLLVMValue::needsSaving(value: value.getOffset()))
179 return true;
180 return false;
181 }
182 static saved_type save(CodeGenFunction &CGF, type value) {
183 return {.BasePtr: DominatingLLVMValue::save(CGF, value: value.getBasePointer()),
184 .ElementType: value.getElementType(), .Alignment: value.getAlignment(),
185 .Offset: DominatingLLVMValue::save(CGF, value: value.getOffset()), .EffectiveType: value.getType()};
186 }
187 static type restore(CodeGenFunction &CGF, saved_type value) {
188 return Address(DominatingLLVMValue::restore(CGF, value: value.BasePtr),
189 value.ElementType, value.Alignment, CGPointerAuthInfo(),
190 DominatingLLVMValue::restore(CGF, value: value.Offset));
191 }
192};
193
194/// A specialization of DominatingValue for RValue.
195template <> struct DominatingValue<RValue> {
196 typedef RValue type;
197 class saved_type {
198 enum Kind {
199 ScalarLiteral,
200 ScalarAddress,
201 AggregateLiteral,
202 AggregateAddress,
203 ComplexAddress
204 };
205 union {
206 struct {
207 DominatingLLVMValue::saved_type first, second;
208 } Vals;
209 DominatingValue<Address>::saved_type AggregateAddr;
210 };
211 LLVM_PREFERRED_TYPE(Kind)
212 unsigned K : 3;
213
214 saved_type(DominatingLLVMValue::saved_type Val1, unsigned K)
215 : Vals{.first: Val1, .second: DominatingLLVMValue::saved_type()}, K(K) {}
216
217 saved_type(DominatingLLVMValue::saved_type Val1,
218 DominatingLLVMValue::saved_type Val2)
219 : Vals{.first: Val1, .second: Val2}, K(ComplexAddress) {}
220
221 saved_type(DominatingValue<Address>::saved_type AggregateAddr, unsigned K)
222 : AggregateAddr(AggregateAddr), K(K) {}
223
224 public:
225 static bool needsSaving(RValue value);
226 static saved_type save(CodeGenFunction &CGF, RValue value);
227 RValue restore(CodeGenFunction &CGF);
228
229 // implementations in CGCleanup.cpp
230 };
231
232 static bool needsSaving(type value) { return saved_type::needsSaving(value); }
233 static saved_type save(CodeGenFunction &CGF, type value) {
234 return saved_type::save(CGF, value);
235 }
236 static type restore(CodeGenFunction &CGF, saved_type value) {
237 return value.restore(CGF);
238 }
239};
240
241/// A scoped helper to set the current source atom group for
242/// CGDebugInfo::addInstToCurrentSourceAtom. A source atom is a source construct
243/// that is "interesting" for debug stepping purposes. We use an atom group
244/// number to track the instruction(s) that implement the functionality for the
245/// atom, plus backup instructions/source locations.
246class ApplyAtomGroup {
247 uint64_t OriginalAtom = 0;
248 CGDebugInfo *DI = nullptr;
249
250 ApplyAtomGroup(const ApplyAtomGroup &) = delete;
251 void operator=(const ApplyAtomGroup &) = delete;
252
253public:
254 ApplyAtomGroup(CGDebugInfo *DI);
255 ~ApplyAtomGroup();
256};
257
258/// CodeGenFunction - This class organizes the per-function state that is used
259/// while generating LLVM code.
260class CodeGenFunction : public CodeGenTypeCache {
261 CodeGenFunction(const CodeGenFunction &) = delete;
262 void operator=(const CodeGenFunction &) = delete;
263
264 friend class CGCXXABI;
265 friend class clang::AsmConstraintsInfo;
266
267public:
268 /// A jump destination is an abstract label, branching to which may
269 /// require a jump out through normal cleanups.
270 struct JumpDest {
271 JumpDest() : Block(nullptr), Index(0) {}
272 JumpDest(llvm::BasicBlock *Block, EHScopeStack::stable_iterator Depth,
273 unsigned Index)
274 : Block(Block), ScopeDepth(Depth), Index(Index) {}
275
276 bool isValid() const { return Block != nullptr; }
277 llvm::BasicBlock *getBlock() const { return Block; }
278 EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; }
279 unsigned getDestIndex() const { return Index; }
280
281 // This should be used cautiously.
282 void setScopeDepth(EHScopeStack::stable_iterator depth) {
283 ScopeDepth = depth;
284 }
285
286 private:
287 llvm::BasicBlock *Block;
288 EHScopeStack::stable_iterator ScopeDepth;
289 unsigned Index;
290 };
291
292 CodeGenModule &CGM; // Per-module state.
293 const TargetInfo &Target;
294
295 // For EH/SEH outlined funclets, this field points to parent's CGF
296 CodeGenFunction *ParentCGF = nullptr;
297
298 typedef std::pair<llvm::Value *, llvm::Value *> ComplexPairTy;
299 LoopInfoStack LoopStack;
300 CGBuilderTy Builder;
301
302 // Stores variables for which we can't generate correct lifetime markers
303 // because of jumps.
304 VarBypassDetector Bypasses;
305
306 // Addresses of bypassed variables, for re-emitting their
307 // trivial-auto-var-init at a jump that re-enters their scope.
308 llvm::SmallDenseMap<const VarDecl *, Address, 4> BypassedVarInits;
309
310 // Jumps, like gotos or switches, that may bypass a declaration that has not
311 // been emitted yet. EmitAutoVarAlloca patches the init in before the jump
312 // once the alloca exists.
313 struct BypassingForwardJump {
314 llvm::AssertingVH<llvm::BasicBlock> Block;
315 const Stmt *Source;
316 };
317 llvm::SmallVector<BypassingForwardJump, 4> BypassingForwardJumps;
318
319 /// List of recently emitted OMPCanonicalLoops.
320 ///
321 /// Since OMPCanonicalLoops are nested inside other statements (in particular
322 /// CapturedStmt generated by OMPExecutableDirective and non-perfectly nested
323 /// loops), we cannot directly call OMPEmitOMPCanonicalLoop and receive its
324 /// llvm::CanonicalLoopInfo. Instead, we call EmitStmt and any
325 /// OMPEmitOMPCanonicalLoop called by it will add its CanonicalLoopInfo to
326 /// this stack when done. Entering a new loop requires clearing this list; it
327 /// either means we start parsing a new loop nest (in which case the previous
328 /// loop nest goes out of scope) or a second loop in the same level in which
329 /// case it would be ambiguous into which of the two (or more) loops the loop
330 /// nest would extend.
331 SmallVector<llvm::CanonicalLoopInfo *, 4> OMPLoopNestStack;
332
333 /// Stack to track the controlled convergence tokens.
334 SmallVector<llvm::ConvergenceControlInst *, 4> ConvergenceTokenStack;
335
336 /// Number of nested loop to be consumed by the last surrounding
337 /// loop-associated directive.
338 int ExpectedOMPLoopDepth = 0;
339
340 // CodeGen lambda for loops and support for ordered clause
341 typedef llvm::function_ref<void(CodeGenFunction &, const OMPLoopDirective &,
342 JumpDest)>
343 CodeGenLoopTy;
344 typedef llvm::function_ref<void(CodeGenFunction &, SourceLocation,
345 const unsigned, const bool)>
346 CodeGenOrderedTy;
347
348 // Codegen lambda for loop bounds in worksharing loop constructs
349 typedef llvm::function_ref<std::pair<LValue, LValue>(
350 CodeGenFunction &, const OMPExecutableDirective &S)>
351 CodeGenLoopBoundsTy;
352
353 // Codegen lambda for loop bounds in dispatch-based loop implementation
354 typedef llvm::function_ref<std::pair<llvm::Value *, llvm::Value *>(
355 CodeGenFunction &, const OMPExecutableDirective &S, Address LB,
356 Address UB)>
357 CodeGenDispatchBoundsTy;
358
359 /// CGBuilder insert helper. This function is called after an
360 /// instruction is created using Builder.
361 void InsertHelper(llvm::Instruction *I, const llvm::Twine &Name,
362 llvm::BasicBlock::iterator InsertPt) const;
363
364 /// CurFuncDecl - Holds the Decl for the current outermost
365 /// non-closure context.
366 const Decl *CurFuncDecl = nullptr;
367 /// CurCodeDecl - This is the inner-most code context, which includes blocks.
368 const Decl *CurCodeDecl = nullptr;
369 const CGFunctionInfo *CurFnInfo = nullptr;
370 QualType FnRetTy;
371 llvm::Function *CurFn = nullptr;
372
373 /// If a cast expression is being visited, this holds the current cast's
374 /// expression.
375 const CastExpr *CurCast = nullptr;
376
377 /// Save Parameter Decl for coroutine.
378 llvm::SmallVector<const ParmVarDecl *, 4> FnArgs;
379
380 // Holds coroutine data if the current function is a coroutine. We use a
381 // wrapper to manage its lifetime, so that we don't have to define CGCoroData
382 // in this header.
383 struct CGCoroInfo {
384 std::unique_ptr<CGCoroData> Data;
385 bool InSuspendBlock = false;
386 CGCoroInfo();
387 ~CGCoroInfo();
388 };
389 CGCoroInfo CurCoro;
390
391 bool isCoroutine() const { return CurCoro.Data != nullptr; }
392
393 bool inSuspendBlock() const {
394 return isCoroutine() && CurCoro.InSuspendBlock;
395 }
396
397 // Holds FramePtr for await_suspend wrapper generation,
398 // so that __builtin_coro_frame call can be lowered
399 // directly to value of its second argument
400 struct AwaitSuspendWrapperInfo {
401 llvm::Value *FramePtr = nullptr;
402 };
403 AwaitSuspendWrapperInfo CurAwaitSuspendWrapper;
404
405 // Generates wrapper function for `llvm.coro.await.suspend.*` intrinisics.
406 // It encapsulates SuspendExpr in a function, to separate it's body
407 // from the main coroutine to avoid miscompilations. Intrinisic
408 // is lowered to this function call in CoroSplit pass
409 // Function signature is:
410 // <type> __await_suspend_wrapper_<name>(ptr %awaiter, ptr %hdl)
411 // where type is one of (void, i1, ptr)
412 llvm::Function *generateAwaitSuspendWrapper(Twine const &CoroName,
413 Twine const &SuspendPointName,
414 CoroutineSuspendExpr const &S);
415
416 /// CurGD - The GlobalDecl for the current function being compiled.
417 GlobalDecl CurGD;
418
419 /// PrologueCleanupDepth - The cleanup depth enclosing all the
420 /// cleanups associated with the parameters.
421 EHScopeStack::stable_iterator PrologueCleanupDepth;
422
423 /// ReturnBlock - Unified return block.
424 JumpDest ReturnBlock;
425
426 /// ReturnValue - The temporary alloca to hold the return
427 /// value. This is invalid iff the function has no return value.
428 Address ReturnValue = Address::invalid();
429
430 /// ReturnValuePointer - The temporary alloca to hold a pointer to sret.
431 /// This is invalid if sret is not in use.
432 Address ReturnValuePointer = Address::invalid();
433
434 /// If a return statement is being visited, this holds the return statment's
435 /// result expression.
436 const Expr *RetExpr = nullptr;
437
438 /// Return true if a label was seen in the current scope.
439 bool hasLabelBeenSeenInCurrentScope() const {
440 if (CurLexicalScope)
441 return CurLexicalScope->hasLabels();
442 return !LabelMap.empty();
443 }
444
445 /// AllocaInsertPoint - This is an instruction in the entry block before which
446 /// we prefer to insert allocas.
447 llvm::AssertingVH<llvm::Instruction> AllocaInsertPt;
448
449private:
450 /// PostAllocaInsertPt - This is a place in the prologue where code can be
451 /// inserted that will be dominated by all the static allocas. This helps
452 /// achieve two things:
453 /// 1. Contiguity of all static allocas (within the prologue) is maintained.
454 /// 2. All other prologue code (which are dominated by static allocas) do
455 /// appear in the source order immediately after all static allocas.
456 ///
457 /// PostAllocaInsertPt will be lazily created when it is *really* required.
458 llvm::AssertingVH<llvm::Instruction> PostAllocaInsertPt = nullptr;
459
460public:
461 /// Return PostAllocaInsertPt. If it is not yet created, then insert it
462 /// immediately after AllocaInsertPt.
463 llvm::Instruction *getPostAllocaInsertPoint() {
464 if (!PostAllocaInsertPt) {
465 assert(AllocaInsertPt &&
466 "Expected static alloca insertion point at function prologue");
467 assert(AllocaInsertPt->getParent()->isEntryBlock() &&
468 "EBB should be entry block of the current code gen function");
469 PostAllocaInsertPt = AllocaInsertPt->clone();
470 PostAllocaInsertPt->setName("postallocapt");
471 PostAllocaInsertPt->insertAfter(InsertPos: AllocaInsertPt->getIterator());
472 }
473
474 return PostAllocaInsertPt;
475 }
476
477 // Try to preserve the source's name to make IR more readable.
478 llvm::Value *performAddrSpaceCast(llvm::Value *Src, llvm::Type *DestTy) {
479 return Builder.CreateAddrSpaceCast(
480 V: Src, DestTy, Name: Src->hasName() ? Src->getName() + ".ascast" : "");
481 }
482
483 /// API for captured statement code generation.
484 class CGCapturedStmtInfo {
485 public:
486 explicit CGCapturedStmtInfo(CapturedRegionKind K = CR_Default)
487 : Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {}
488 explicit CGCapturedStmtInfo(const CapturedStmt &S,
489 CapturedRegionKind K = CR_Default)
490 : Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {
491
492 RecordDecl::field_iterator Field =
493 S.getCapturedRecordDecl()->field_begin();
494 for (CapturedStmt::const_capture_iterator I = S.capture_begin(),
495 E = S.capture_end();
496 I != E; ++I, ++Field) {
497 if (I->capturesThis())
498 CXXThisFieldDecl = *Field;
499 else if (I->capturesVariable())
500 CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
501 else if (I->capturesVariableByCopy())
502 CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;
503 }
504 }
505
506 virtual ~CGCapturedStmtInfo();
507
508 CapturedRegionKind getKind() const { return Kind; }
509
510 virtual void setContextValue(llvm::Value *V) { ThisValue = V; }
511 // Retrieve the value of the context parameter.
512 virtual llvm::Value *getContextValue() const { return ThisValue; }
513
514 /// Lookup the captured field decl for a variable.
515 virtual const FieldDecl *lookup(const VarDecl *VD) const {
516 return CaptureFields.lookup(Val: VD->getCanonicalDecl());
517 }
518
519 bool isCXXThisExprCaptured() const { return getThisFieldDecl() != nullptr; }
520 virtual FieldDecl *getThisFieldDecl() const { return CXXThisFieldDecl; }
521
522 static bool classof(const CGCapturedStmtInfo *) { return true; }
523
524 /// Emit the captured statement body.
525 virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) {
526 CGF.incrementProfileCounter(S);
527 CGF.EmitStmt(S);
528 }
529
530 /// Get the name of the capture helper.
531 virtual StringRef getHelperName() const { return "__captured_stmt"; }
532
533 /// Get the CaptureFields
534 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> getCaptureFields() {
535 return CaptureFields;
536 }
537
538 private:
539 /// The kind of captured statement being generated.
540 CapturedRegionKind Kind;
541
542 /// Keep the map between VarDecl and FieldDecl.
543 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields;
544
545 /// The base address of the captured record, passed in as the first
546 /// argument of the parallel region function.
547 llvm::Value *ThisValue;
548
549 /// Captured 'this' type.
550 FieldDecl *CXXThisFieldDecl;
551 };
552 CGCapturedStmtInfo *CapturedStmtInfo = nullptr;
553
554 /// RAII for correct setting/restoring of CapturedStmtInfo.
555 class CGCapturedStmtRAII {
556 private:
557 CodeGenFunction &CGF;
558 CGCapturedStmtInfo *PrevCapturedStmtInfo;
559
560 public:
561 CGCapturedStmtRAII(CodeGenFunction &CGF,
562 CGCapturedStmtInfo *NewCapturedStmtInfo)
563 : CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo) {
564 CGF.CapturedStmtInfo = NewCapturedStmtInfo;
565 }
566 ~CGCapturedStmtRAII() { CGF.CapturedStmtInfo = PrevCapturedStmtInfo; }
567 };
568
569 /// An abstract representation of regular/ObjC call/message targets.
570 class AbstractCallee {
571 /// The function declaration of the callee.
572 const Decl *CalleeDecl;
573
574 public:
575 AbstractCallee() : CalleeDecl(nullptr) {}
576 AbstractCallee(const FunctionDecl *FD) : CalleeDecl(FD) {}
577 AbstractCallee(const ObjCMethodDecl *OMD) : CalleeDecl(OMD) {}
578 bool hasFunctionDecl() const {
579 return isa_and_nonnull<FunctionDecl>(Val: CalleeDecl);
580 }
581 const Decl *getDecl() const { return CalleeDecl; }
582 unsigned getNumParams() const {
583 if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
584 return FD->getNumParams();
585 return cast<ObjCMethodDecl>(Val: CalleeDecl)->param_size();
586 }
587 const ParmVarDecl *getParamDecl(unsigned I) const {
588 if (const auto *FD = dyn_cast<FunctionDecl>(Val: CalleeDecl))
589 return FD->getParamDecl(i: I);
590 return *(cast<ObjCMethodDecl>(Val: CalleeDecl)->param_begin() + I);
591 }
592 };
593
594 /// Sanitizers enabled for this function.
595 SanitizerSet SanOpts;
596
597 /// True if CodeGen currently emits code implementing sanitizer checks.
598 bool IsSanitizerScope = false;
599
600 /// RAII object to set/unset CodeGenFunction::IsSanitizerScope.
601 class SanitizerScope {
602 CodeGenFunction *CGF;
603
604 public:
605 SanitizerScope(CodeGenFunction *CGF);
606 ~SanitizerScope();
607 };
608
609 /// In C++, whether we are code generating a thunk. This controls whether we
610 /// should emit cleanups.
611 bool CurFuncIsThunk = false;
612
613 /// In ARC, whether we should autorelease the return value.
614 bool AutoreleaseResult = false;
615
616 /// Whether we processed a Microsoft-style asm block during CodeGen. These can
617 /// potentially set the return value.
618 bool SawAsmBlock = false;
619
620 GlobalDecl CurSEHParent;
621
622 /// True if the current function is an outlined SEH helper. This can be a
623 /// finally block or filter expression.
624 bool IsOutlinedSEHHelper = false;
625
626 /// True if CodeGen currently emits code inside presereved access index
627 /// region.
628 bool IsInPreservedAIRegion = false;
629
630 /// True if the current statement has nomerge attribute.
631 bool InNoMergeAttributedStmt = false;
632
633 /// True if the current statement has noinline attribute.
634 bool InNoInlineAttributedStmt = false;
635
636 /// True if the current statement has always_inline attribute.
637 bool InAlwaysInlineAttributedStmt = false;
638
639 /// True if the current statement has noconvergent attribute.
640 bool InNoConvergentAttributedStmt = false;
641
642 /// The mode string from the amdgpu_av attribute on the current statement,
643 /// or empty if the attribute is not present.
644 StringRef AMDGPUAvailableVisibleMode;
645
646 /// HLSL Branch attribute.
647 HLSLControlFlowHintAttr::Spelling HLSLControlFlowAttr =
648 HLSLControlFlowHintAttr::SpellingNotCalculated;
649
650 // The CallExpr within the current statement that the musttail attribute
651 // applies to. nullptr if there is no 'musttail' on the current statement.
652 const CallExpr *MustTailCall = nullptr;
653
654 /// Returns true if a function must make progress, which means the
655 /// mustprogress attribute can be added.
656 bool checkIfFunctionMustProgress() {
657 if (CGM.getCodeGenOpts().getFiniteLoops() ==
658 CodeGenOptions::FiniteLoopsKind::Never)
659 return false;
660
661 // C++11 and later guarantees that a thread eventually will do one of the
662 // following (C++11 [intro.multithread]p24 and C++17 [intro.progress]p1):
663 // - terminate,
664 // - make a call to a library I/O function,
665 // - perform an access through a volatile glvalue, or
666 // - perform a synchronization operation or an atomic operation.
667 //
668 // Hence each function is 'mustprogress' in C++11 or later.
669 return getLangOpts().CPlusPlus11;
670 }
671
672 /// Returns true if a loop must make progress, which means the mustprogress
673 /// attribute can be added. \p HasConstantCond indicates whether the branch
674 /// condition is a known constant.
675 bool checkIfLoopMustProgress(const Expr *, bool HasEmptyBody);
676
677 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
678 llvm::Value *BlockPointer = nullptr;
679
680 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
681 FieldDecl *LambdaThisCaptureField = nullptr;
682
683 /// A mapping from NRVO variables to the flags used to indicate
684 /// when the NRVO has been applied to this variable.
685 llvm::DenseMap<const VarDecl *, llvm::Value *> NRVOFlags;
686
687 EHScopeStack EHStack;
688 llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack;
689
690 // A stack of cleanups which were added to EHStack but have to be deactivated
691 // later before being popped or emitted. These are usually deactivated on
692 // exiting a `CleanupDeactivationScope` scope. For instance, after a
693 // full-expr.
694 //
695 // These are specially useful for correctly emitting cleanups while
696 // encountering branches out of expression (through stmt-expr or coroutine
697 // suspensions).
698 struct DeferredDeactivateCleanup {
699 EHScopeStack::stable_iterator Cleanup;
700 llvm::Instruction *DominatingIP;
701 };
702 llvm::SmallVector<DeferredDeactivateCleanup> DeferredDeactivationCleanupStack;
703
704 // Enters a new scope for capturing cleanups which are deferred to be
705 // deactivated, all of which will be deactivated once the scope is exited.
706 struct CleanupDeactivationScope {
707 CodeGenFunction &CGF;
708 size_t OldDeactivateCleanupStackSize;
709 bool Deactivated;
710 CleanupDeactivationScope(CodeGenFunction &CGF)
711 : CGF(CGF), OldDeactivateCleanupStackSize(
712 CGF.DeferredDeactivationCleanupStack.size()),
713 Deactivated(false) {}
714
715 void ForceDeactivate() {
716 assert(!Deactivated && "Deactivating already deactivated scope");
717 auto &Stack = CGF.DeferredDeactivationCleanupStack;
718 for (size_t I = Stack.size(); I > OldDeactivateCleanupStackSize; I--) {
719 CGF.DeactivateCleanupBlock(Cleanup: Stack[I - 1].Cleanup,
720 DominatingIP: Stack[I - 1].DominatingIP);
721 Stack[I - 1].DominatingIP->eraseFromParent();
722 }
723 Stack.resize(N: OldDeactivateCleanupStackSize);
724 Deactivated = true;
725 }
726
727 ~CleanupDeactivationScope() {
728 if (Deactivated)
729 return;
730 ForceDeactivate();
731 }
732 };
733
734 llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack;
735
736 llvm::Instruction *CurrentFuncletPad = nullptr;
737
738 class CallLifetimeEnd final : public EHScopeStack::Cleanup {
739 llvm::Value *Addr;
740
741 public:
742 CallLifetimeEnd(RawAddress addr) : Addr(addr.getPointer()) {}
743
744 void Emit(CodeGenFunction &CGF, Flags flags) override {
745 CGF.EmitLifetimeEnd(Addr);
746 }
747 };
748
749 // We are using objects of this 'cleanup' class to emit fake.use calls
750 // for -fextend-variable-liveness. They are placed at the end of a variable's
751 // scope analogous to lifetime markers.
752 class FakeUse final : public EHScopeStack::Cleanup {
753 Address Addr;
754
755 public:
756 FakeUse(Address addr) : Addr(addr) {}
757
758 void Emit(CodeGenFunction &CGF, Flags flags) override {
759 CGF.EmitFakeUse(Addr);
760 }
761 };
762
763 /// Header for data within LifetimeExtendedCleanupStack.
764 struct alignas(uint64_t) LifetimeExtendedCleanupHeader {
765 /// The size of the following cleanup object.
766 unsigned Size;
767 /// The kind of cleanup to push.
768 LLVM_PREFERRED_TYPE(CleanupKind)
769 unsigned Kind : 31;
770 /// Whether this is a conditional cleanup.
771 LLVM_PREFERRED_TYPE(bool)
772 unsigned IsConditional : 1;
773
774 size_t getSize() const { return Size; }
775 CleanupKind getKind() const { return (CleanupKind)Kind; }
776 bool isConditional() const { return IsConditional; }
777 };
778
779 /// i32s containing the indexes of the cleanup destinations.
780 RawAddress NormalCleanupDest = RawAddress::invalid();
781
782 unsigned NextCleanupDestIndex = 1;
783
784 /// EHResumeBlock - Unified block containing a call to llvm.eh.resume.
785 llvm::BasicBlock *EHResumeBlock = nullptr;
786
787 /// The exception slot. All landing pads write the current exception pointer
788 /// into this alloca.
789 llvm::Value *ExceptionSlot = nullptr;
790
791 /// The selector slot. Under the MandatoryCleanup model, all landing pads
792 /// write the current selector value into this alloca.
793 llvm::AllocaInst *EHSelectorSlot = nullptr;
794
795 /// A stack of exception code slots. Entering an __except block pushes a slot
796 /// on the stack and leaving pops one. The __exception_code() intrinsic loads
797 /// a value from the top of the stack.
798 SmallVector<Address, 1> SEHCodeSlotStack;
799
800 /// Value returned by __exception_info intrinsic.
801 llvm::Value *SEHInfo = nullptr;
802
803 /// Emits a landing pad for the current EH stack.
804 llvm::BasicBlock *EmitLandingPad();
805
806 llvm::BasicBlock *getInvokeDestImpl();
807
808 /// Parent loop-based directive for scan directive.
809 const OMPExecutableDirective *OMPParentLoopDirectiveForScan = nullptr;
810 llvm::BasicBlock *OMPBeforeScanBlock = nullptr;
811 llvm::BasicBlock *OMPAfterScanBlock = nullptr;
812 llvm::BasicBlock *OMPScanExitBlock = nullptr;
813 llvm::BasicBlock *OMPScanDispatch = nullptr;
814 bool OMPFirstScanLoop = false;
815
816 /// Manages parent directive for scan directives.
817 class ParentLoopDirectiveForScanRegion {
818 CodeGenFunction &CGF;
819 const OMPExecutableDirective *ParentLoopDirectiveForScan;
820
821 public:
822 ParentLoopDirectiveForScanRegion(
823 CodeGenFunction &CGF,
824 const OMPExecutableDirective &ParentLoopDirectiveForScan)
825 : CGF(CGF),
826 ParentLoopDirectiveForScan(CGF.OMPParentLoopDirectiveForScan) {
827 CGF.OMPParentLoopDirectiveForScan = &ParentLoopDirectiveForScan;
828 }
829 ~ParentLoopDirectiveForScanRegion() {
830 CGF.OMPParentLoopDirectiveForScan = ParentLoopDirectiveForScan;
831 }
832 };
833
834 template <class T>
835 typename DominatingValue<T>::saved_type saveValueInCond(T value) {
836 return DominatingValue<T>::save(*this, value);
837 }
838
839 class CGFPOptionsRAII {
840 public:
841 CGFPOptionsRAII(CodeGenFunction &CGF, FPOptions FPFeatures);
842 CGFPOptionsRAII(CodeGenFunction &CGF, const Expr *E);
843 ~CGFPOptionsRAII();
844
845 private:
846 void ConstructorHelper(FPOptions FPFeatures);
847 CodeGenFunction &CGF;
848 FPOptions OldFPFeatures;
849 llvm::fp::ExceptionBehavior OldExcept;
850 llvm::RoundingMode OldRounding;
851 std::optional<CGBuilderTy::FastMathFlagGuard> FMFGuard;
852 };
853 FPOptions CurFPFeatures;
854
855 class CGAtomicOptionsRAII {
856 public:
857 CGAtomicOptionsRAII(CodeGenModule &CGM_, AtomicOptions AO)
858 : CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {
859 CGM.setAtomicOpts(AO);
860 }
861 CGAtomicOptionsRAII(CodeGenModule &CGM_, const AtomicAttr *AA)
862 : CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {
863 if (!AA)
864 return;
865 AtomicOptions AO = SavedAtomicOpts;
866 for (auto Option : AA->atomicOptions()) {
867 switch (Option) {
868 case AtomicAttr::remote_memory:
869 AO.remote_memory = true;
870 break;
871 case AtomicAttr::no_remote_memory:
872 AO.remote_memory = false;
873 break;
874 case AtomicAttr::fine_grained_memory:
875 AO.fine_grained_memory = true;
876 break;
877 case AtomicAttr::no_fine_grained_memory:
878 AO.fine_grained_memory = false;
879 break;
880 case AtomicAttr::ignore_denormal_mode:
881 AO.ignore_denormal_mode = true;
882 break;
883 case AtomicAttr::no_ignore_denormal_mode:
884 AO.ignore_denormal_mode = false;
885 break;
886 }
887 }
888 CGM.setAtomicOpts(AO);
889 }
890
891 CGAtomicOptionsRAII(const CGAtomicOptionsRAII &) = delete;
892 CGAtomicOptionsRAII &operator=(const CGAtomicOptionsRAII &) = delete;
893 ~CGAtomicOptionsRAII() { CGM.setAtomicOpts(SavedAtomicOpts); }
894
895 private:
896 CodeGenModule &CGM;
897 AtomicOptions SavedAtomicOpts;
898 };
899
900public:
901 /// ObjCEHValueStack - Stack of Objective-C exception values, used for
902 /// rethrows.
903 SmallVector<llvm::Value *, 8> ObjCEHValueStack;
904
905 /// A class controlling the emission of a finally block.
906 class FinallyInfo {
907 /// Where the catchall's edge through the cleanup should go.
908 JumpDest RethrowDest;
909
910 /// A function to call to enter the catch.
911 llvm::FunctionCallee BeginCatchFn;
912
913 /// An i1 variable indicating whether or not the @finally is
914 /// running for an exception.
915 llvm::AllocaInst *ForEHVar = nullptr;
916
917 /// An i8* variable into which the exception pointer to rethrow
918 /// has been saved.
919 llvm::AllocaInst *SavedExnVar = nullptr;
920
921 public:
922 void enter(CodeGenFunction &CGF, const Stmt *Finally,
923 llvm::FunctionCallee beginCatchFn,
924 llvm::FunctionCallee endCatchFn, llvm::FunctionCallee rethrowFn);
925 void exit(CodeGenFunction &CGF);
926 };
927
928 /// Returns true inside SEH __try blocks.
929 bool isSEHTryScope() const { return !SEHTryEpilogueStack.empty(); }
930
931 /// Returns true while emitting a cleanuppad.
932 bool isCleanupPadScope() const {
933 return CurrentFuncletPad && isa<llvm::CleanupPadInst>(Val: CurrentFuncletPad);
934 }
935
936 /// pushFullExprCleanup - Push a cleanup to be run at the end of the
937 /// current full-expression. Safe against the possibility that
938 /// we're currently inside a conditionally-evaluated expression.
939 template <class T, class... As>
940 void pushFullExprCleanup(CleanupKind kind, As... A) {
941 // If we're not in a conditional branch, or if none of the
942 // arguments requires saving, then use the unconditional cleanup.
943 if (!isInConditionalBranch())
944 return EHStack.pushCleanup<T>(kind, A...);
945
946 // Stash values in a tuple so we can guarantee the order of saves.
947 typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
948 SavedTuple Saved{saveValueInCond(A)...};
949
950 typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
951 EHStack.pushCleanupTuple<CleanupType>(kind, Saved);
952 initFullExprCleanup();
953 }
954
955 /// Queue a cleanup to be pushed after finishing the current full-expression,
956 /// potentially with an active flag.
957 template <class T, class... As>
958 void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) {
959 if (!isInConditionalBranch())
960 return pushCleanupAfterFullExprWithActiveFlag<T>(
961 Kind, RawAddress::invalid(), A...);
962
963 RawAddress ActiveFlag = createCleanupActiveFlag();
964 assert(!DominatingValue<Address>::needsSaving(ActiveFlag) &&
965 "cleanup active flag should never need saving");
966
967 typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;
968 SavedTuple Saved{saveValueInCond(A)...};
969
970 typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;
971 pushCleanupAfterFullExprWithActiveFlag<CleanupType>(Kind, ActiveFlag,
972 Saved);
973 }
974
975 template <class T, class... As>
976 void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind,
977 RawAddress ActiveFlag, As... A) {
978 LifetimeExtendedCleanupHeader Header = {.Size: sizeof(T), .Kind: Kind,
979 .IsConditional: ActiveFlag.isValid()};
980
981 size_t OldSize = LifetimeExtendedCleanupStack.size();
982 LifetimeExtendedCleanupStack.resize(
983 N: LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size +
984 (Header.IsConditional ? sizeof(ActiveFlag) : 0));
985
986 static_assert((alignof(LifetimeExtendedCleanupHeader) == alignof(T)) &&
987 (alignof(T) == alignof(RawAddress)),
988 "Cleanup will be allocated on misaligned address");
989 char *Buffer = &LifetimeExtendedCleanupStack[OldSize];
990 new (Buffer) LifetimeExtendedCleanupHeader(Header);
991 new (Buffer + sizeof(Header)) T(A...);
992 if (Header.IsConditional)
993 new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag);
994 }
995
996 // Push a cleanup onto EHStack and deactivate it later. It is usually
997 // deactivated when exiting a `CleanupDeactivationScope` (for example: after a
998 // full expression).
999 template <class T, class... As>
1000 void pushCleanupAndDeferDeactivation(CleanupKind Kind, As... A) {
1001 // Placeholder dominating IP for this cleanup.
1002 llvm::Instruction *DominatingIP =
1003 Builder.CreateFlagLoad(Addr: llvm::Constant::getNullValue(Ty: Int8PtrTy));
1004 EHStack.pushCleanup<T>(Kind, A...);
1005 DeferredDeactivationCleanupStack.push_back(
1006 Elt: {.Cleanup: EHStack.stable_begin(), .DominatingIP: DominatingIP});
1007 }
1008
1009 /// Set up the last cleanup that was pushed as a conditional
1010 /// full-expression cleanup.
1011 void initFullExprCleanup() {
1012 initFullExprCleanupWithFlag(ActiveFlag: createCleanupActiveFlag());
1013 }
1014
1015 void initFullExprCleanupWithFlag(RawAddress ActiveFlag);
1016 RawAddress createCleanupActiveFlag();
1017
1018 /// PushDestructorCleanup - Push a cleanup to call the
1019 /// complete-object destructor of an object of the given type at the
1020 /// given address. Does nothing if T is not a C++ class type with a
1021 /// non-trivial destructor.
1022 void PushDestructorCleanup(QualType T, Address Addr);
1023
1024 /// PushDestructorCleanup - Push a cleanup to call the
1025 /// complete-object variant of the given destructor on the object at
1026 /// the given address.
1027 void PushDestructorCleanup(const CXXDestructorDecl *Dtor, QualType T,
1028 Address Addr);
1029
1030 /// PopCleanupBlock - Will pop the cleanup entry on the stack and
1031 /// process all branch fixups.
1032 void PopCleanupBlock(bool FallThroughIsBranchThrough = false,
1033 bool ForDeactivation = false);
1034
1035 /// DeactivateCleanupBlock - Deactivates the given cleanup block.
1036 /// The block cannot be reactivated. Pops it if it's the top of the
1037 /// stack.
1038 ///
1039 /// \param DominatingIP - An instruction which is known to
1040 /// dominate the current IP (if set) and which lies along
1041 /// all paths of execution between the current IP and the
1042 /// the point at which the cleanup comes into scope.
1043 void DeactivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
1044 llvm::Instruction *DominatingIP);
1045
1046 /// ActivateCleanupBlock - Activates an initially-inactive cleanup.
1047 /// Cannot be used to resurrect a deactivated cleanup.
1048 ///
1049 /// \param DominatingIP - An instruction which is known to
1050 /// dominate the current IP (if set) and which lies along
1051 /// all paths of execution between the current IP and the
1052 /// the point at which the cleanup comes into scope.
1053 void ActivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,
1054 llvm::Instruction *DominatingIP);
1055
1056 /// Enters a new scope for capturing cleanups, all of which
1057 /// will be executed once the scope is exited.
1058 class RunCleanupsScope {
1059 EHScopeStack::stable_iterator CleanupStackDepth, OldCleanupScopeDepth;
1060 size_t LifetimeExtendedCleanupStackSize;
1061 CleanupDeactivationScope DeactivateCleanups;
1062 bool OldDidCallStackSave;
1063
1064 protected:
1065 bool PerformCleanup;
1066
1067 private:
1068 RunCleanupsScope(const RunCleanupsScope &) = delete;
1069 void operator=(const RunCleanupsScope &) = delete;
1070
1071 protected:
1072 CodeGenFunction &CGF;
1073
1074 public:
1075 /// Enter a new cleanup scope.
1076 explicit RunCleanupsScope(CodeGenFunction &CGF)
1077 : DeactivateCleanups(CGF), PerformCleanup(true), CGF(CGF) {
1078 CleanupStackDepth = CGF.EHStack.stable_begin();
1079 LifetimeExtendedCleanupStackSize =
1080 CGF.LifetimeExtendedCleanupStack.size();
1081 OldDidCallStackSave = CGF.DidCallStackSave;
1082 CGF.DidCallStackSave = false;
1083 OldCleanupScopeDepth = CGF.CurrentCleanupScopeDepth;
1084 CGF.CurrentCleanupScopeDepth = CleanupStackDepth;
1085 }
1086
1087 /// Exit this cleanup scope, emitting any accumulated cleanups.
1088 ~RunCleanupsScope() {
1089 if (PerformCleanup)
1090 ForceCleanup();
1091 }
1092
1093 /// Determine whether this scope requires any cleanups.
1094 bool requiresCleanups() const {
1095 return CGF.EHStack.stable_begin() != CleanupStackDepth;
1096 }
1097
1098 /// Force the emission of cleanups now, instead of waiting
1099 /// until this object is destroyed.
1100 /// \param ValuesToReload - A list of values that need to be available at
1101 /// the insertion point after cleanup emission. If cleanup emission created
1102 /// a shared cleanup block, these value pointers will be rewritten.
1103 /// Otherwise, they not will be modified.
1104 void
1105 ForceCleanup(std::initializer_list<llvm::Value **> ValuesToReload = {}) {
1106 assert(PerformCleanup && "Already forced cleanup");
1107 CGF.DidCallStackSave = OldDidCallStackSave;
1108 DeactivateCleanups.ForceDeactivate();
1109 CGF.PopCleanupBlocks(OldCleanupStackSize: CleanupStackDepth, OldLifetimeExtendedStackSize: LifetimeExtendedCleanupStackSize,
1110 ValuesToReload);
1111 PerformCleanup = false;
1112 CGF.CurrentCleanupScopeDepth = OldCleanupScopeDepth;
1113 }
1114 };
1115
1116 // Cleanup stack depth of the RunCleanupsScope that was pushed most recently.
1117 EHScopeStack::stable_iterator CurrentCleanupScopeDepth =
1118 EHScopeStack::stable_end();
1119
1120 class LexicalScope : public RunCleanupsScope {
1121 SourceRange Range;
1122 SmallVector<const LabelDecl *, 4> Labels;
1123 LexicalScope *ParentScope;
1124
1125 LexicalScope(const LexicalScope &) = delete;
1126 void operator=(const LexicalScope &) = delete;
1127
1128 public:
1129 /// Enter a new cleanup scope.
1130 explicit LexicalScope(CodeGenFunction &CGF, SourceRange Range);
1131
1132 void addLabel(const LabelDecl *label) {
1133 assert(PerformCleanup && "adding label to dead scope?");
1134 Labels.push_back(Elt: label);
1135 }
1136
1137 /// Exit this cleanup scope, emitting any accumulated
1138 /// cleanups.
1139 ~LexicalScope();
1140
1141 /// Force the emission of cleanups now, instead of waiting
1142 /// until this object is destroyed.
1143 void ForceCleanup() {
1144 CGF.CurLexicalScope = ParentScope;
1145 RunCleanupsScope::ForceCleanup();
1146
1147 if (!Labels.empty())
1148 rescopeLabels();
1149 }
1150
1151 bool hasLabels() const { return !Labels.empty(); }
1152
1153 void rescopeLabels();
1154 };
1155
1156 typedef llvm::DenseMap<const Decl *, Address> DeclMapTy;
1157
1158 /// The class used to assign some variables some temporarily addresses.
1159 class OMPMapVars {
1160 DeclMapTy SavedLocals;
1161 DeclMapTy SavedTempAddresses;
1162 OMPMapVars(const OMPMapVars &) = delete;
1163 void operator=(const OMPMapVars &) = delete;
1164
1165 public:
1166 explicit OMPMapVars() = default;
1167 ~OMPMapVars() {
1168 assert(SavedLocals.empty() && "Did not restored original addresses.");
1169 };
1170
1171 /// Sets the address of the variable \p LocalVD to be \p TempAddr in
1172 /// function \p CGF.
1173 /// \return true if at least one variable was set already, false otherwise.
1174 bool setVarAddr(CodeGenFunction &CGF, const ValueDecl *LocalVD,
1175 Address TempAddr) {
1176 LocalVD = cast<ValueDecl>(Val: LocalVD->getCanonicalDecl());
1177
1178 // Only save it once.
1179 if (SavedLocals.count(Val: LocalVD))
1180 return false;
1181
1182 // Copy the existing local entry to SavedLocals.
1183 auto it = CGF.LocalDeclMap.find(Val: LocalVD);
1184 if (it != CGF.LocalDeclMap.end())
1185 SavedLocals.try_emplace(Key: LocalVD, Args&: it->second);
1186 else
1187 SavedLocals.try_emplace(Key: LocalVD, Args: Address::invalid());
1188
1189 // Generate the private entry.
1190 QualType VarTy = LocalVD->getType();
1191 if (VarTy->isReferenceType()) {
1192 Address Temp = CGF.CreateMemTemp(T: VarTy);
1193 CGF.Builder.CreateStore(Val: TempAddr.emitRawPointer(CGF), Addr: Temp);
1194 TempAddr = Temp;
1195 }
1196 if (const auto *BD = dyn_cast<BindingDecl>(Val: LocalVD))
1197 CGF.OMPPrivatizedBindings.insert_or_assign(Key: BD, Val&: TempAddr);
1198 SavedTempAddresses.try_emplace(Key: LocalVD, Args&: TempAddr);
1199
1200 return true;
1201 }
1202
1203 /// Applies new addresses to the list of the variables.
1204 /// \return true if at least one variable is using new address, false
1205 /// otherwise.
1206 bool apply(CodeGenFunction &CGF) {
1207 copyInto(Src: SavedTempAddresses, Dest&: CGF.LocalDeclMap);
1208 SavedTempAddresses.clear();
1209 return !SavedLocals.empty();
1210 }
1211
1212 /// Restores original addresses of the variables.
1213 void restore(CodeGenFunction &CGF) {
1214 if (!SavedLocals.empty()) {
1215 copyInto(Src: SavedLocals, Dest&: CGF.LocalDeclMap);
1216 SavedLocals.clear();
1217 }
1218 }
1219
1220 private:
1221 /// Copy all the entries in the source map over the corresponding
1222 /// entries in the destination, which must exist.
1223 static void copyInto(const DeclMapTy &Src, DeclMapTy &Dest) {
1224 for (auto &[Decl, Addr] : Src) {
1225 if (!Addr.isValid())
1226 Dest.erase(Val: Decl);
1227 else
1228 Dest.insert_or_assign(Key: Decl, Val: Addr);
1229 }
1230 }
1231 };
1232
1233 /// The scope used to remap some variables as private in the OpenMP loop body
1234 /// (or other captured region emitted without outlining), and to restore old
1235 /// vars back on exit.
1236 class OMPPrivateScope : public RunCleanupsScope {
1237 OMPMapVars MappedVars;
1238 OMPPrivateScope(const OMPPrivateScope &) = delete;
1239 void operator=(const OMPPrivateScope &) = delete;
1240 llvm::DenseMap<const BindingDecl *, Address> BindingChanges;
1241
1242 public:
1243 /// Enter a new OpenMP private scope.
1244 explicit OMPPrivateScope(CodeGenFunction &CGF) : RunCleanupsScope(CGF) {}
1245
1246 /// Registers \p LocalVD variable as a private with \p Addr as the address
1247 /// of the corresponding private variable. \p
1248 /// PrivateGen is the address of the generated private variable.
1249 /// \return true if the variable is registered as private, false if it has
1250 /// been privatized already.
1251 bool addPrivate(const ValueDecl *LocalVD, Address Addr) {
1252 assert(PerformCleanup && "adding private to dead scope");
1253 if (const auto *BD = dyn_cast<BindingDecl>(Val: LocalVD->getCanonicalDecl())) {
1254 auto It = CGF.OMPPrivatizedBindings.find(Val: BD);
1255 BindingChanges.insert(KV: {BD, It != CGF.OMPPrivatizedBindings.end()
1256 ? It->second
1257 : Address::invalid()});
1258 }
1259 return MappedVars.setVarAddr(CGF, LocalVD, TempAddr: Addr);
1260 }
1261
1262 /// Privatizes local variables previously registered as private.
1263 /// Registration is separate from the actual privatization to allow
1264 /// initializers use values of the original variables, not the private one.
1265 /// This is important, for example, if the private variable is a class
1266 /// variable initialized by a constructor that references other private
1267 /// variables. But at initialization original variables must be used, not
1268 /// private copies.
1269 /// \return true if at least one variable was privatized, false otherwise.
1270 bool Privatize() { return MappedVars.apply(CGF); }
1271
1272 void ForceCleanup() {
1273 RunCleanupsScope::ForceCleanup();
1274 restoreMap();
1275 }
1276
1277 /// Exit scope - all the mapped variables are restored.
1278 ~OMPPrivateScope() {
1279 if (PerformCleanup)
1280 ForceCleanup();
1281 for (auto &Change : BindingChanges) {
1282 if (Change.second.isValid()) {
1283 auto It = CGF.OMPPrivatizedBindings.find(Val: Change.first);
1284 if (It != CGF.OMPPrivatizedBindings.end())
1285 It->second = Change.second;
1286 else
1287 CGF.OMPPrivatizedBindings.insert(KV: {Change.first, Change.second});
1288 } else {
1289 CGF.OMPPrivatizedBindings.erase(Val: Change.first);
1290 }
1291 }
1292 }
1293
1294 /// Checks if the global variable is captured in current function.
1295 bool isGlobalVarCaptured(const VarDecl *VD) const {
1296 VD = VD->getCanonicalDecl();
1297 return !VD->isLocalVarDeclOrParm() && CGF.LocalDeclMap.count(Val: VD) > 0;
1298 }
1299
1300 /// Restore all mapped variables w/o clean up. This is usefully when we want
1301 /// to reference the original variables but don't want the clean up because
1302 /// that could emit lifetime end too early, causing backend issue #56913.
1303 void restoreMap() { MappedVars.restore(CGF); }
1304 };
1305
1306 /// Save/restore original map of previously emitted local vars in case when we
1307 /// need to duplicate emission of the same code several times in the same
1308 /// function for OpenMP code.
1309 class OMPLocalDeclMapRAII {
1310 CodeGenFunction &CGF;
1311 DeclMapTy SavedMap;
1312
1313 public:
1314 OMPLocalDeclMapRAII(CodeGenFunction &CGF)
1315 : CGF(CGF), SavedMap(CGF.LocalDeclMap) {}
1316 ~OMPLocalDeclMapRAII() { SavedMap.swap(RHS&: CGF.LocalDeclMap); }
1317 };
1318
1319 /// Takes the old cleanup stack size and emits the cleanup blocks
1320 /// that have been added.
1321 void
1322 PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1323 std::initializer_list<llvm::Value **> ValuesToReload = {});
1324
1325 /// Takes the old cleanup stack size and emits the cleanup blocks
1326 /// that have been added, then adds all lifetime-extended cleanups from
1327 /// the given position to the stack.
1328 void
1329 PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,
1330 size_t OldLifetimeExtendedStackSize,
1331 std::initializer_list<llvm::Value **> ValuesToReload = {});
1332
1333 void ResolveBranchFixups(llvm::BasicBlock *Target);
1334
1335 /// The given basic block lies in the current EH scope, but may be a
1336 /// target of a potentially scope-crossing jump; get a stable handle
1337 /// to which we can perform this jump later.
1338 JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target) {
1339 return JumpDest(Target, EHStack.getInnermostNormalCleanup(),
1340 NextCleanupDestIndex++);
1341 }
1342
1343 /// The given basic block lies in the current EH scope, but may be a
1344 /// target of a potentially scope-crossing jump; get a stable handle
1345 /// to which we can perform this jump later.
1346 JumpDest getJumpDestInCurrentScope(StringRef Name = StringRef()) {
1347 return getJumpDestInCurrentScope(Target: createBasicBlock(name: Name));
1348 }
1349
1350 /// EmitBranchThroughCleanup - Emit a branch from the current insert
1351 /// block through the normal cleanup handling code (if any) and then
1352 /// on to \arg Dest.
1353 void EmitBranchThroughCleanup(JumpDest Dest);
1354
1355 /// isObviouslyBranchWithoutCleanups - Return true if a branch to the
1356 /// specified destination obviously has no cleanups to run. 'false' is always
1357 /// a conservatively correct answer for this method.
1358 bool isObviouslyBranchWithoutCleanups(JumpDest Dest) const;
1359
1360 /// popCatchScope - Pops the catch scope at the top of the EHScope
1361 /// stack, emitting any required code (other than the catch handlers
1362 /// themselves).
1363 void popCatchScope();
1364
1365 // This function should be called after emitting all catch clauses and none
1366 // of them were 'catch-all' clauses.
1367 // Because in wasm we merge all catch clauses into one big catchpad, in case
1368 // none of the types in catch handlers matches after we test against each of
1369 // them, we should unwind to the next EH enclosing scope. We generate a call
1370 // to rethrow function here to do that.
1371 void WasmEmitFallthroughRethrow(llvm::BasicBlock *WasmCatchStartBlock);
1372
1373 llvm::BasicBlock *getEHResumeBlock(bool isCleanup);
1374 llvm::BasicBlock *getEHDispatchBlock(EHScopeStack::stable_iterator scope);
1375 llvm::BasicBlock *
1376 getFuncletEHDispatchBlock(EHScopeStack::stable_iterator scope);
1377
1378 /// An object to manage conditionally-evaluated expressions.
1379 class ConditionalEvaluation {
1380 llvm::BasicBlock *StartBB;
1381
1382 public:
1383 ConditionalEvaluation(CodeGenFunction &CGF)
1384 : StartBB(CGF.Builder.GetInsertBlock()) {}
1385
1386 void begin(CodeGenFunction &CGF) {
1387 assert(CGF.OutermostConditional != this);
1388 if (!CGF.OutermostConditional)
1389 CGF.OutermostConditional = this;
1390 }
1391
1392 void end(CodeGenFunction &CGF) {
1393 assert(CGF.OutermostConditional != nullptr);
1394 if (CGF.OutermostConditional == this)
1395 CGF.OutermostConditional = nullptr;
1396 }
1397
1398 /// Returns a block which will be executed prior to each
1399 /// evaluation of the conditional code.
1400 llvm::BasicBlock *getStartingBlock() const { return StartBB; }
1401 };
1402
1403 /// isInConditionalBranch - Return true if we're currently emitting
1404 /// one branch or the other of a conditional expression.
1405 bool isInConditionalBranch() const { return OutermostConditional != nullptr; }
1406
1407 void setBeforeOutermostConditional(llvm::Value *value, Address addr,
1408 CodeGenFunction &CGF) {
1409 assert(isInConditionalBranch());
1410 llvm::BasicBlock *block = OutermostConditional->getStartingBlock();
1411 auto store = new llvm::StoreInst(value, addr.emitRawPointer(CGF),
1412 block->back().getIterator());
1413 store->setAlignment(addr.getAlignment().getAsAlign());
1414 }
1415
1416 /// An RAII object to record that we're evaluating a statement
1417 /// expression.
1418 class StmtExprEvaluation {
1419 CodeGenFunction &CGF;
1420
1421 /// We have to save the outermost conditional: cleanups in a
1422 /// statement expression aren't conditional just because the
1423 /// StmtExpr is.
1424 ConditionalEvaluation *SavedOutermostConditional;
1425
1426 public:
1427 StmtExprEvaluation(CodeGenFunction &CGF)
1428 : CGF(CGF), SavedOutermostConditional(CGF.OutermostConditional) {
1429 CGF.OutermostConditional = nullptr;
1430 }
1431
1432 ~StmtExprEvaluation() {
1433 CGF.OutermostConditional = SavedOutermostConditional;
1434 CGF.EnsureInsertPoint();
1435 }
1436 };
1437
1438 /// An object which temporarily prevents a value from being
1439 /// destroyed by aggressive peephole optimizations that assume that
1440 /// all uses of a value have been realized in the IR.
1441 class PeepholeProtection {
1442 llvm::Instruction *Inst = nullptr;
1443 friend class CodeGenFunction;
1444
1445 public:
1446 PeepholeProtection() = default;
1447 };
1448
1449 /// A non-RAII class containing all the information about a bound
1450 /// opaque value. OpaqueValueMapping, below, is a RAII wrapper for
1451 /// this which makes individual mappings very simple; using this
1452 /// class directly is useful when you have a variable number of
1453 /// opaque values or don't want the RAII functionality for some
1454 /// reason.
1455 class OpaqueValueMappingData {
1456 const OpaqueValueExpr *OpaqueValue;
1457 bool BoundLValue;
1458 CodeGenFunction::PeepholeProtection Protection;
1459
1460 OpaqueValueMappingData(const OpaqueValueExpr *ov, bool boundLValue)
1461 : OpaqueValue(ov), BoundLValue(boundLValue) {}
1462
1463 public:
1464 OpaqueValueMappingData() : OpaqueValue(nullptr) {}
1465
1466 static bool shouldBindAsLValue(const Expr *expr) {
1467 // gl-values should be bound as l-values for obvious reasons.
1468 // Records should be bound as l-values because IR generation
1469 // always keeps them in memory. Expressions of function type
1470 // act exactly like l-values but are formally required to be
1471 // r-values in C.
1472 return expr->isGLValue() || expr->getType()->isFunctionType() ||
1473 hasAggregateEvaluationKind(T: expr->getType());
1474 }
1475
1476 static OpaqueValueMappingData
1477 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const Expr *e) {
1478 if (shouldBindAsLValue(expr: ov))
1479 return bind(CGF, ov, lv: CGF.EmitLValue(E: e));
1480 return bind(CGF, ov, rv: CGF.EmitAnyExpr(E: e));
1481 }
1482
1483 static OpaqueValueMappingData
1484 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const LValue &lv) {
1485 assert(shouldBindAsLValue(ov));
1486 CGF.OpaqueLValues.insert(KV: std::make_pair(x&: ov, y: lv));
1487 return OpaqueValueMappingData(ov, true);
1488 }
1489
1490 static OpaqueValueMappingData
1491 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const RValue &rv) {
1492 assert(!shouldBindAsLValue(ov));
1493 CGF.OpaqueRValues.insert(KV: std::make_pair(x&: ov, y: rv));
1494
1495 OpaqueValueMappingData data(ov, false);
1496
1497 // Work around an extremely aggressive peephole optimization in
1498 // EmitScalarConversion which assumes that all other uses of a
1499 // value are extant.
1500 data.Protection = CGF.protectFromPeepholes(rvalue: rv);
1501
1502 return data;
1503 }
1504
1505 bool isValid() const { return OpaqueValue != nullptr; }
1506 void clear() { OpaqueValue = nullptr; }
1507
1508 void unbind(CodeGenFunction &CGF) {
1509 assert(OpaqueValue && "no data to unbind!");
1510
1511 if (BoundLValue) {
1512 CGF.OpaqueLValues.erase(Val: OpaqueValue);
1513 } else {
1514 CGF.OpaqueRValues.erase(Val: OpaqueValue);
1515 CGF.unprotectFromPeepholes(protection: Protection);
1516 }
1517 }
1518 };
1519
1520 /// An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
1521 class OpaqueValueMapping {
1522 CodeGenFunction &CGF;
1523 OpaqueValueMappingData Data;
1524
1525 public:
1526 static bool shouldBindAsLValue(const Expr *expr) {
1527 return OpaqueValueMappingData::shouldBindAsLValue(expr);
1528 }
1529
1530 /// Build the opaque value mapping for the given conditional
1531 /// operator if it's the GNU ?: extension. This is a common
1532 /// enough pattern that the convenience operator is really
1533 /// helpful.
1534 ///
1535 OpaqueValueMapping(CodeGenFunction &CGF,
1536 const AbstractConditionalOperator *op)
1537 : CGF(CGF) {
1538 if (isa<ConditionalOperator>(Val: op))
1539 // Leave Data empty.
1540 return;
1541
1542 const BinaryConditionalOperator *e = cast<BinaryConditionalOperator>(Val: op);
1543 Data = OpaqueValueMappingData::bind(CGF, ov: e->getOpaqueValue(),
1544 e: e->getCommon());
1545 }
1546
1547 /// Build the opaque value mapping for an OpaqueValueExpr whose source
1548 /// expression is set to the expression the OVE represents.
1549 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *OV)
1550 : CGF(CGF) {
1551 if (OV) {
1552 assert(OV->getSourceExpr() && "wrong form of OpaqueValueMapping used "
1553 "for OVE with no source expression");
1554 Data = OpaqueValueMappingData::bind(CGF, ov: OV, e: OV->getSourceExpr());
1555 }
1556 }
1557
1558 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,
1559 LValue lvalue)
1560 : CGF(CGF),
1561 Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, lv: lvalue)) {}
1562
1563 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,
1564 RValue rvalue)
1565 : CGF(CGF),
1566 Data(OpaqueValueMappingData::bind(CGF, ov: opaqueValue, rv: rvalue)) {}
1567
1568 void pop() {
1569 Data.unbind(CGF);
1570 Data.clear();
1571 }
1572
1573 ~OpaqueValueMapping() {
1574 if (Data.isValid())
1575 Data.unbind(CGF);
1576 }
1577 };
1578
1579private:
1580 CGDebugInfo *DebugInfo;
1581 /// Used to create unique names for artificial VLA size debug info variables.
1582 unsigned VLAExprCounter = 0;
1583 bool DisableDebugInfo = false;
1584
1585 /// DidCallStackSave - Whether llvm.stacksave has been called. Used to avoid
1586 /// calling llvm.stacksave for multiple VLAs in the same scope.
1587 bool DidCallStackSave = false;
1588
1589 /// IndirectBranch - The first time an indirect goto is seen we create a block
1590 /// with an indirect branch. Every time we see the address of a label taken,
1591 /// we add the label to the indirect goto. Every subsequent indirect goto is
1592 /// codegen'd as a jump to the IndirectBranch's basic block.
1593 llvm::IndirectBrInst *IndirectBranch = nullptr;
1594
1595 /// LocalDeclMap - This keeps track of the LLVM allocas or globals for local C
1596 /// decls.
1597 DeclMapTy LocalDeclMap;
1598
1599 /// Lookup map for privatized BindingDecls.
1600 /// Used when BindingDecls are remapped during OpenMP outlining, since the
1601 /// remapped BindingDecl has a different pointer than the original.
1602 llvm::SmallDenseMap<const BindingDecl *, Address> OMPPrivatizedBindings;
1603
1604 // Keep track of the cleanups for callee-destructed parameters pushed to the
1605 // cleanup stack so that they can be deactivated later.
1606 llvm::DenseMap<const ParmVarDecl *, EHScopeStack::stable_iterator>
1607 CalleeDestructedParamCleanups;
1608
1609 /// SizeArguments - If a ParmVarDecl had the pass_object_size attribute, this
1610 /// will contain a mapping from said ParmVarDecl to its implicit "object_size"
1611 /// parameter.
1612 llvm::SmallDenseMap<const ParmVarDecl *, const ImplicitParamDecl *, 2>
1613 SizeArguments;
1614
1615 /// Track escaped local variables with auto storage. Used during SEH
1616 /// outlining to produce a call to llvm.localescape.
1617 llvm::DenseMap<llvm::AllocaInst *, int> EscapedLocals;
1618
1619 /// LabelMap - This keeps track of the LLVM basic block for each C label.
1620 llvm::DenseMap<const LabelDecl *, JumpDest> LabelMap;
1621
1622 // BreakContinueStack - This keeps track of where break and continue
1623 // statements should jump to.
1624 struct BreakContinue {
1625 BreakContinue(const Stmt &LoopOrSwitch, JumpDest Break, JumpDest Continue)
1626 : LoopOrSwitch(&LoopOrSwitch), BreakBlock(Break),
1627 ContinueBlock(Continue) {}
1628
1629 const Stmt *LoopOrSwitch;
1630 JumpDest BreakBlock;
1631 JumpDest ContinueBlock;
1632 };
1633 SmallVector<BreakContinue, 8> BreakContinueStack;
1634
1635 /// Handles cancellation exit points in OpenMP-related constructs.
1636 class OpenMPCancelExitStack {
1637 /// Tracks cancellation exit point and join point for cancel-related exit
1638 /// and normal exit.
1639 struct CancelExit {
1640 CancelExit() = default;
1641 CancelExit(OpenMPDirectiveKind Kind, JumpDest ExitBlock,
1642 JumpDest ContBlock)
1643 : Kind(Kind), ExitBlock(ExitBlock), ContBlock(ContBlock) {}
1644 OpenMPDirectiveKind Kind = llvm::omp::OMPD_unknown;
1645 /// true if the exit block has been emitted already by the special
1646 /// emitExit() call, false if the default codegen is used.
1647 bool HasBeenEmitted = false;
1648 JumpDest ExitBlock;
1649 JumpDest ContBlock;
1650 };
1651
1652 SmallVector<CancelExit, 8> Stack;
1653
1654 public:
1655 OpenMPCancelExitStack() : Stack(1) {}
1656 ~OpenMPCancelExitStack() = default;
1657 /// Fetches the exit block for the current OpenMP construct.
1658 JumpDest getExitBlock() const { return Stack.back().ExitBlock; }
1659 /// Emits exit block with special codegen procedure specific for the related
1660 /// OpenMP construct + emits code for normal construct cleanup.
1661 void emitExit(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
1662 const llvm::function_ref<void(CodeGenFunction &)> CodeGen) {
1663 if (Stack.back().Kind == Kind && getExitBlock().isValid()) {
1664 assert(CGF.getOMPCancelDestination(Kind).isValid());
1665 assert(CGF.HaveInsertPoint());
1666 assert(!Stack.back().HasBeenEmitted);
1667 auto IP = CGF.Builder.saveAndClearIP();
1668 CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1669 CodeGen(CGF);
1670 CGF.EmitBranch(Block: Stack.back().ContBlock.getBlock());
1671 CGF.Builder.restoreIP(IP);
1672 Stack.back().HasBeenEmitted = true;
1673 }
1674 CodeGen(CGF);
1675 }
1676 /// Enter the cancel supporting \a Kind construct.
1677 /// \param Kind OpenMP directive that supports cancel constructs.
1678 /// \param HasCancel true, if the construct has inner cancel directive,
1679 /// false otherwise.
1680 void enter(CodeGenFunction &CGF, OpenMPDirectiveKind Kind, bool HasCancel) {
1681 Stack.push_back(Elt: {Kind,
1682 HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.exit")
1683 : JumpDest(),
1684 HasCancel ? CGF.getJumpDestInCurrentScope(Name: "cancel.cont")
1685 : JumpDest()});
1686 }
1687 /// Emits default exit point for the cancel construct (if the special one
1688 /// has not be used) + join point for cancel/normal exits.
1689 void exit(CodeGenFunction &CGF) {
1690 if (getExitBlock().isValid()) {
1691 assert(CGF.getOMPCancelDestination(Stack.back().Kind).isValid());
1692 bool HaveIP = CGF.HaveInsertPoint();
1693 if (!Stack.back().HasBeenEmitted) {
1694 if (HaveIP)
1695 CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1696 CGF.EmitBlock(BB: Stack.back().ExitBlock.getBlock());
1697 CGF.EmitBranchThroughCleanup(Dest: Stack.back().ContBlock);
1698 }
1699 CGF.EmitBlock(BB: Stack.back().ContBlock.getBlock());
1700 if (!HaveIP) {
1701 CGF.Builder.CreateUnreachable();
1702 CGF.Builder.ClearInsertionPoint();
1703 }
1704 }
1705 Stack.pop_back();
1706 }
1707 };
1708 OpenMPCancelExitStack OMPCancelStack;
1709
1710 /// Lower the Likelihood knowledge about the \p Cond via llvm.expect intrin.
1711 llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
1712 Stmt::Likelihood LH);
1713
1714 std::unique_ptr<CodeGenPGO> PGO;
1715
1716 /// Calculate branch weights appropriate for PGO data
1717 llvm::MDNode *createProfileWeights(uint64_t TrueCount,
1718 uint64_t FalseCount) const;
1719 llvm::MDNode *createProfileWeights(ArrayRef<uint64_t> Weights) const;
1720 llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond,
1721 uint64_t LoopCount) const;
1722
1723public:
1724 bool hasSkipCounter(const Stmt *S) const;
1725
1726 void markStmtAsUsed(bool Skipped, const Stmt *S);
1727 void markStmtMaybeUsed(const Stmt *S);
1728
1729 /// Used to specify which counter in a pair shall be incremented.
1730 /// For non-binary counters, a skip counter is derived as (Parent - Exec).
1731 /// In contrast for binary counters, a skip counter cannot be computed from
1732 /// the Parent counter. In such cases, dedicated SkipPath counters must be
1733 /// allocated and marked (incremented as binary counters). (Parent can be
1734 /// synthesized with (Exec + Skip) in simple cases)
1735 enum CounterForIncrement {
1736 UseExecPath = 0, ///< Exec (true)
1737 UseSkipPath, ///< Skip (false)
1738 };
1739
1740 /// Increment the profiler's counter for the given statement by \p StepV.
1741 /// If \p StepV is null, the default increment is 1.
1742 void incrementProfileCounter(const Stmt *S, llvm::Value *StepV = nullptr) {
1743 incrementProfileCounter(ExecSkip: UseExecPath, S, UseBoth: false, StepV);
1744 }
1745
1746 /// Emit increment of Counter.
1747 /// \param ExecSkip Use `Skipped` Counter if UseSkipPath is specified.
1748 /// \param S The Stmt that Counter is associated.
1749 /// \param UseBoth Mark both Exec/Skip as used. (for verification)
1750 /// \param StepV The offset Value for adding to Counter.
1751 void incrementProfileCounter(CounterForIncrement ExecSkip, const Stmt *S,
1752 bool UseBoth = false,
1753 llvm::Value *StepV = nullptr);
1754
1755 bool isMCDCCoverageEnabled() const {
1756 return (CGM.getCodeGenOpts().hasProfileClangInstr() &&
1757 CGM.getCodeGenOpts().MCDCCoverage &&
1758 !CurFn->hasFnAttribute(Kind: llvm::Attribute::NoProfile));
1759 }
1760
1761 /// Allocate a temp value on the stack that MCDC can use to track condition
1762 /// results.
1763 void maybeCreateMCDCCondBitmap();
1764
1765 bool isBinaryLogicalOp(const Expr *E) const {
1766 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(Val: E->IgnoreParens());
1767 return (BOp && BOp->isLogicalOp());
1768 }
1769
1770 bool isMCDCDecisionExpr(const Expr *E) const;
1771 bool isMCDCBranchExpr(const Expr *E) const;
1772
1773 /// Zero-init the MCDC temp value.
1774 void maybeResetMCDCCondBitmap(const Expr *E);
1775
1776 /// Increment the profiler's counter for the given expression by \p StepV.
1777 /// If \p StepV is null, the default increment is 1.
1778 void maybeUpdateMCDCTestVectorBitmap(const Expr *E);
1779
1780 /// Update the MCDC temp value with the condition's evaluated result.
1781 void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val);
1782
1783 /// Get the profiler's count for the given statement.
1784 uint64_t getProfileCount(const Stmt *S);
1785
1786 /// Set the profiler's current count.
1787 void setCurrentProfileCount(uint64_t Count);
1788
1789 /// Get the profiler's current count. This is generally the count for the most
1790 /// recently incremented counter.
1791 uint64_t getCurrentProfileCount();
1792
1793 /// See CGDebugInfo::addInstToCurrentSourceAtom.
1794 void addInstToCurrentSourceAtom(llvm::Instruction *KeyInstruction,
1795 llvm::Value *Backup);
1796
1797 /// See CGDebugInfo::addInstToSpecificSourceAtom.
1798 void addInstToSpecificSourceAtom(llvm::Instruction *KeyInstruction,
1799 llvm::Value *Backup, uint64_t Atom);
1800
1801 /// Add \p KeyInstruction and an optional \p Backup instruction to a new atom
1802 /// group (See ApplyAtomGroup for more info).
1803 void addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,
1804 llvm::Value *Backup);
1805
1806 /// Copy all PFP fields from SrcPtr to DestPtr while updating signatures,
1807 /// assuming that DestPtr was already memcpy'd from SrcPtr.
1808 void emitPFPPostCopyUpdates(Address DestPtr, Address SrcPtr, QualType Ty);
1809
1810private:
1811 /// SwitchInsn - This is nearest current switch instruction. It is null if
1812 /// current context is not in a switch.
1813 llvm::SwitchInst *SwitchInsn = nullptr;
1814 /// The branch weights of SwitchInsn when doing instrumentation based PGO.
1815 SmallVector<uint64_t, 16> *SwitchWeights = nullptr;
1816
1817 /// The likelihood attributes of the SwitchCase.
1818 SmallVector<Stmt::Likelihood, 16> *SwitchLikelihood = nullptr;
1819
1820 /// CaseRangeBlock - This block holds if condition check for last case
1821 /// statement range in current switch instruction.
1822 llvm::BasicBlock *CaseRangeBlock = nullptr;
1823
1824 /// OpaqueLValues - Keeps track of the current set of opaque value
1825 /// expressions.
1826 llvm::DenseMap<const OpaqueValueExpr *, LValue> OpaqueLValues;
1827 llvm::DenseMap<const OpaqueValueExpr *, RValue> OpaqueRValues;
1828
1829 // VLASizeMap - This keeps track of the associated size for each VLA type.
1830 // We track this by the size expression rather than the type itself because
1831 // in certain situations, like a const qualifier applied to an VLA typedef,
1832 // multiple VLA types can share the same size expression.
1833 // FIXME: Maybe this could be a stack of maps that is pushed/popped as we
1834 // enter/leave scopes.
1835 llvm::DenseMap<const Expr *, llvm::Value *> VLASizeMap;
1836
1837 /// A block containing a single 'unreachable' instruction. Created
1838 /// lazily by getUnreachableBlock().
1839 llvm::BasicBlock *UnreachableBlock = nullptr;
1840
1841 /// Counts of the number return expressions in the function.
1842 unsigned NumReturnExprs = 0;
1843
1844 /// Count the number of simple (constant) return expressions in the function.
1845 unsigned NumSimpleReturnExprs = 0;
1846
1847 /// The last regular (non-return) debug location (breakpoint) in the function.
1848 SourceLocation LastStopPoint;
1849
1850public:
1851 /// Source location information about the default argument or member
1852 /// initializer expression we're evaluating, if any.
1853 CurrentSourceLocExprScope CurSourceLocExprScope;
1854 using SourceLocExprScopeGuard =
1855 CurrentSourceLocExprScope::SourceLocExprScopeGuard;
1856
1857 /// A scope within which we are constructing the fields of an object which
1858 /// might use a CXXDefaultInitExpr. This stashes away a 'this' value to use
1859 /// if we need to evaluate a CXXDefaultInitExpr within the evaluation.
1860 class FieldConstructionScope {
1861 public:
1862 FieldConstructionScope(CodeGenFunction &CGF, Address This)
1863 : CGF(CGF), OldCXXDefaultInitExprThis(CGF.CXXDefaultInitExprThis) {
1864 CGF.CXXDefaultInitExprThis = This;
1865 }
1866 ~FieldConstructionScope() {
1867 CGF.CXXDefaultInitExprThis = OldCXXDefaultInitExprThis;
1868 }
1869
1870 private:
1871 CodeGenFunction &CGF;
1872 Address OldCXXDefaultInitExprThis;
1873 };
1874
1875 /// The scope of a CXXDefaultInitExpr. Within this scope, the value of 'this'
1876 /// is overridden to be the object under construction.
1877 class CXXDefaultInitExprScope {
1878 public:
1879 CXXDefaultInitExprScope(CodeGenFunction &CGF, const CXXDefaultInitExpr *E)
1880 : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue),
1881 OldCXXThisAlignment(CGF.CXXThisAlignment),
1882 SourceLocScope(E, CGF.CurSourceLocExprScope) {
1883 CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getBasePointer();
1884 CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment();
1885 }
1886 ~CXXDefaultInitExprScope() {
1887 CGF.CXXThisValue = OldCXXThisValue;
1888 CGF.CXXThisAlignment = OldCXXThisAlignment;
1889 }
1890
1891 public:
1892 CodeGenFunction &CGF;
1893 llvm::Value *OldCXXThisValue;
1894 CharUnits OldCXXThisAlignment;
1895 SourceLocExprScopeGuard SourceLocScope;
1896 };
1897
1898 struct CXXDefaultArgExprScope : SourceLocExprScopeGuard {
1899 CXXDefaultArgExprScope(CodeGenFunction &CGF, const CXXDefaultArgExpr *E)
1900 : SourceLocExprScopeGuard(E, CGF.CurSourceLocExprScope) {}
1901 };
1902
1903 /// The scope of an ArrayInitLoopExpr. Within this scope, the value of the
1904 /// current loop index is overridden.
1905 class ArrayInitLoopExprScope {
1906 public:
1907 ArrayInitLoopExprScope(CodeGenFunction &CGF, llvm::Value *Index)
1908 : CGF(CGF), OldArrayInitIndex(CGF.ArrayInitIndex) {
1909 CGF.ArrayInitIndex = Index;
1910 }
1911 ~ArrayInitLoopExprScope() { CGF.ArrayInitIndex = OldArrayInitIndex; }
1912
1913 private:
1914 CodeGenFunction &CGF;
1915 llvm::Value *OldArrayInitIndex;
1916 };
1917
1918 class InlinedInheritingConstructorScope {
1919 public:
1920 InlinedInheritingConstructorScope(CodeGenFunction &CGF, GlobalDecl GD)
1921 : CGF(CGF), OldCurGD(CGF.CurGD), OldCurFuncDecl(CGF.CurFuncDecl),
1922 OldCurCodeDecl(CGF.CurCodeDecl),
1923 OldCXXABIThisDecl(CGF.CXXABIThisDecl),
1924 OldCXXABIThisValue(CGF.CXXABIThisValue),
1925 OldCXXThisValue(CGF.CXXThisValue),
1926 OldCXXABIThisAlignment(CGF.CXXABIThisAlignment),
1927 OldCXXThisAlignment(CGF.CXXThisAlignment),
1928 OldReturnValue(CGF.ReturnValue), OldFnRetTy(CGF.FnRetTy),
1929 OldCXXInheritedCtorInitExprArgs(
1930 std::move(CGF.CXXInheritedCtorInitExprArgs)) {
1931 CGF.CurGD = GD;
1932 CGF.CurFuncDecl = CGF.CurCodeDecl =
1933 cast<CXXConstructorDecl>(Val: GD.getDecl());
1934 CGF.CXXABIThisDecl = nullptr;
1935 CGF.CXXABIThisValue = nullptr;
1936 CGF.CXXThisValue = nullptr;
1937 CGF.CXXABIThisAlignment = CharUnits();
1938 CGF.CXXThisAlignment = CharUnits();
1939 CGF.ReturnValue = Address::invalid();
1940 CGF.FnRetTy = QualType();
1941 CGF.CXXInheritedCtorInitExprArgs.clear();
1942 }
1943 ~InlinedInheritingConstructorScope() {
1944 CGF.CurGD = OldCurGD;
1945 CGF.CurFuncDecl = OldCurFuncDecl;
1946 CGF.CurCodeDecl = OldCurCodeDecl;
1947 CGF.CXXABIThisDecl = OldCXXABIThisDecl;
1948 CGF.CXXABIThisValue = OldCXXABIThisValue;
1949 CGF.CXXThisValue = OldCXXThisValue;
1950 CGF.CXXABIThisAlignment = OldCXXABIThisAlignment;
1951 CGF.CXXThisAlignment = OldCXXThisAlignment;
1952 CGF.ReturnValue = OldReturnValue;
1953 CGF.FnRetTy = OldFnRetTy;
1954 CGF.CXXInheritedCtorInitExprArgs =
1955 std::move(OldCXXInheritedCtorInitExprArgs);
1956 }
1957
1958 private:
1959 CodeGenFunction &CGF;
1960 GlobalDecl OldCurGD;
1961 const Decl *OldCurFuncDecl;
1962 const Decl *OldCurCodeDecl;
1963 ImplicitParamDecl *OldCXXABIThisDecl;
1964 llvm::Value *OldCXXABIThisValue;
1965 llvm::Value *OldCXXThisValue;
1966 CharUnits OldCXXABIThisAlignment;
1967 CharUnits OldCXXThisAlignment;
1968 Address OldReturnValue;
1969 QualType OldFnRetTy;
1970 CallArgList OldCXXInheritedCtorInitExprArgs;
1971 };
1972
1973 // Helper class for the OpenMP IR Builder. Allows reusability of code used for
1974 // region body, and finalization codegen callbacks. This will class will also
1975 // contain privatization functions used by the privatization call backs
1976 //
1977 // TODO: this is temporary class for things that are being moved out of
1978 // CGOpenMPRuntime, new versions of current CodeGenFunction methods, or
1979 // utility function for use with the OMPBuilder. Once that move to use the
1980 // OMPBuilder is done, everything here will either become part of CodeGenFunc.
1981 // directly, or a new helper class that will contain functions used by both
1982 // this and the OMPBuilder
1983
1984 struct OMPBuilderCBHelpers {
1985
1986 OMPBuilderCBHelpers() = delete;
1987 OMPBuilderCBHelpers(const OMPBuilderCBHelpers &) = delete;
1988 OMPBuilderCBHelpers &operator=(const OMPBuilderCBHelpers &) = delete;
1989
1990 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1991
1992 /// Cleanup action for allocate support.
1993 class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
1994
1995 private:
1996 llvm::CallInst *RTLFnCI;
1997
1998 public:
1999 OMPAllocateCleanupTy(llvm::CallInst *RLFnCI) : RTLFnCI(RLFnCI) {
2000 RLFnCI->removeFromParent();
2001 }
2002
2003 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
2004 if (!CGF.HaveInsertPoint())
2005 return;
2006 CGF.Builder.Insert(I: RTLFnCI);
2007 }
2008 };
2009
2010 /// Returns address of the threadprivate variable for the current
2011 /// thread. This Also create any necessary OMP runtime calls.
2012 ///
2013 /// \param VD VarDecl for Threadprivate variable.
2014 /// \param VDAddr Address of the Vardecl
2015 /// \param Loc The location where the barrier directive was encountered
2016 static Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
2017 const VarDecl *VD, Address VDAddr,
2018 SourceLocation Loc);
2019
2020 /// Gets the OpenMP-specific address of the local variable /p VD.
2021 static Address getAddressOfLocalVariable(CodeGenFunction &CGF,
2022 const VarDecl *VD);
2023 /// Get the platform-specific name separator.
2024 /// \param Parts different parts of the final name that needs separation
2025 /// \param FirstSeparator First separator used between the initial two
2026 /// parts of the name.
2027 /// \param Separator separator used between all of the rest consecutinve
2028 /// parts of the name
2029 static std::string getNameWithSeparators(ArrayRef<StringRef> Parts,
2030 StringRef FirstSeparator = ".",
2031 StringRef Separator = ".");
2032 /// Emit the Finalization for an OMP region
2033 /// \param CGF The Codegen function this belongs to
2034 /// \param IP Insertion point for generating the finalization code.
2035 static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP) {
2036 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
2037 llvm::BasicBlock *IPBB = IP.getNodeParent();
2038 assert(IPBB->end() != IP &&
2039 "OpenMP IR Builder should cause terminated block!");
2040
2041 llvm::BasicBlock *DestBB = IPBB->getUniqueSuccessor();
2042 assert(DestBB && "Finalization block should have one successor!");
2043
2044 // erase and replace with cleanup branch.
2045 IPBB->getTerminator()->eraseFromParent();
2046 CGF.Builder.SetInsertPoint(IPBB);
2047 CodeGenFunction::JumpDest Dest = CGF.getJumpDestInCurrentScope(Target: DestBB);
2048 CGF.EmitBranchThroughCleanup(Dest);
2049 }
2050
2051 /// Emit the body of an OMP region
2052 /// \param CGF The Codegen function this belongs to
2053 /// \param RegionBodyStmt The body statement for the OpenMP region being
2054 /// generated
2055 /// \param AllocaIP Where to insert alloca instructions
2056 /// \param CodeGenIP Where to insert the region code
2057 /// \param RegionName Name to be used for new blocks
2058 static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF,
2059 const Stmt *RegionBodyStmt,
2060 InsertPointTy AllocaIP,
2061 InsertPointTy CodeGenIP,
2062 Twine RegionName);
2063
2064 static void EmitCaptureStmt(CodeGenFunction &CGF,
2065 llvm::BasicBlock *CodeGenIPBB,
2066 llvm::BasicBlock &FiniBB, llvm::Function *Fn,
2067 ArrayRef<llvm::Value *> Args) {
2068 if (llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminatorOrNull())
2069 CodeGenIPBBTI->eraseFromParent();
2070
2071 CGF.Builder.SetInsertPoint(CodeGenIPBB);
2072
2073 if (Fn->doesNotThrow())
2074 CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
2075 else
2076 CGF.EmitRuntimeCall(callee: Fn, args: Args);
2077
2078 if (CGF.Builder.saveIP().isValid())
2079 CGF.Builder.CreateBr(Dest: &FiniBB);
2080 }
2081
2082 /// Emit the body of an OMP region that will be outlined in
2083 /// OpenMPIRBuilder::finalize().
2084 /// \param CGF The Codegen function this belongs to
2085 /// \param RegionBodyStmt The body statement for the OpenMP region being
2086 /// generated
2087 /// \param AllocaIP Where to insert alloca instructions
2088 /// \param CodeGenIP Where to insert the region code
2089 /// \param RegionName Name to be used for new blocks
2090 static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF,
2091 const Stmt *RegionBodyStmt,
2092 InsertPointTy AllocaIP,
2093 InsertPointTy CodeGenIP,
2094 Twine RegionName);
2095
2096 /// RAII for preserving necessary info during Outlined region body codegen.
2097 class OutlinedRegionBodyRAII {
2098
2099 llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
2100 CodeGenFunction::JumpDest OldReturnBlock;
2101 CodeGenFunction &CGF;
2102
2103 public:
2104 OutlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
2105 llvm::BasicBlock &RetBB)
2106 : CGF(cgf) {
2107 assert(AllocaIP.isValid() &&
2108 "Must specify Insertion point for allocas of outlined function");
2109 OldAllocaIP = CGF.AllocaInsertPt;
2110 CGF.AllocaInsertPt = &*AllocaIP;
2111
2112 OldReturnBlock = CGF.ReturnBlock;
2113 CGF.ReturnBlock = CGF.getJumpDestInCurrentScope(Target: &RetBB);
2114 }
2115
2116 ~OutlinedRegionBodyRAII() {
2117 CGF.AllocaInsertPt = OldAllocaIP;
2118 CGF.ReturnBlock = OldReturnBlock;
2119 }
2120 };
2121
2122 /// RAII for preserving necessary info during inlined region body codegen.
2123 class InlinedRegionBodyRAII {
2124
2125 llvm::AssertingVH<llvm::Instruction> OldAllocaIP;
2126 CodeGenFunction &CGF;
2127
2128 public:
2129 InlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,
2130 llvm::BasicBlock &FiniBB)
2131 : CGF(cgf) {
2132 // Alloca insertion block should be in the entry block of the containing
2133 // function so it expects an empty AllocaIP in which case will reuse the
2134 // old alloca insertion point, or a new AllocaIP in the same block as
2135 // the old one
2136 assert((!AllocaIP.isValid() ||
2137 CGF.AllocaInsertPt->getParent() == AllocaIP.getNodeParent()) &&
2138 "Insertion point should be in the entry block of containing "
2139 "function!");
2140 OldAllocaIP = CGF.AllocaInsertPt;
2141 if (AllocaIP.isValid())
2142 CGF.AllocaInsertPt = &*AllocaIP;
2143
2144 // TODO: Remove the call, after making sure the counter is not used by
2145 // the EHStack.
2146 // Since this is an inlined region, it should not modify the
2147 // ReturnBlock, and should reuse the one for the enclosing outlined
2148 // region. So, the JumpDest being return by the function is discarded
2149 (void)CGF.getJumpDestInCurrentScope(Target: &FiniBB);
2150 }
2151
2152 ~InlinedRegionBodyRAII() { CGF.AllocaInsertPt = OldAllocaIP; }
2153 };
2154 };
2155
2156private:
2157 /// CXXThisDecl - When generating code for a C++ member function,
2158 /// this will hold the implicit 'this' declaration.
2159 ImplicitParamDecl *CXXABIThisDecl = nullptr;
2160 llvm::Value *CXXABIThisValue = nullptr;
2161 llvm::Value *CXXThisValue = nullptr;
2162 CharUnits CXXABIThisAlignment;
2163 CharUnits CXXThisAlignment;
2164
2165 /// The value of 'this' to use when evaluating CXXDefaultInitExprs within
2166 /// this expression.
2167 Address CXXDefaultInitExprThis = Address::invalid();
2168
2169 /// The current array initialization index when evaluating an
2170 /// ArrayInitIndexExpr within an ArrayInitLoopExpr.
2171 llvm::Value *ArrayInitIndex = nullptr;
2172
2173 /// The values of function arguments to use when evaluating
2174 /// CXXInheritedCtorInitExprs within this context.
2175 CallArgList CXXInheritedCtorInitExprArgs;
2176
2177 /// CXXStructorImplicitParamDecl - When generating code for a constructor or
2178 /// destructor, this will hold the implicit argument (e.g. VTT).
2179 ImplicitParamDecl *CXXStructorImplicitParamDecl = nullptr;
2180 llvm::Value *CXXStructorImplicitParamValue = nullptr;
2181
2182 /// OutermostConditional - Points to the outermost active
2183 /// conditional control. This is used so that we know if a
2184 /// temporary should be destroyed conditionally.
2185 ConditionalEvaluation *OutermostConditional = nullptr;
2186
2187 /// The current lexical scope.
2188 LexicalScope *CurLexicalScope = nullptr;
2189
2190 /// The current source location that should be used for exception
2191 /// handling code.
2192 SourceLocation CurEHLocation;
2193
2194 /// BlockByrefInfos - For each __block variable, contains
2195 /// information about the layout of the variable.
2196 llvm::DenseMap<const ValueDecl *, BlockByrefInfo> BlockByrefInfos;
2197
2198 /// Used by -fsanitize=nullability-return to determine whether the return
2199 /// value can be checked.
2200 llvm::Value *RetValNullabilityPrecondition = nullptr;
2201
2202 /// Check if -fsanitize=nullability-return instrumentation is required for
2203 /// this function.
2204 bool requiresReturnValueNullabilityCheck() const {
2205 return RetValNullabilityPrecondition;
2206 }
2207
2208 /// Used to store precise source locations for return statements by the
2209 /// runtime return value checks.
2210 Address ReturnLocation = Address::invalid();
2211
2212 /// Check if the return value of this function requires sanitization.
2213 bool requiresReturnValueCheck() const;
2214
2215 bool isInAllocaArgument(CGCXXABI &ABI, QualType Ty);
2216 bool hasInAllocaArg(const CXXMethodDecl *MD);
2217
2218 llvm::BasicBlock *TerminateLandingPad = nullptr;
2219 llvm::BasicBlock *TerminateHandler = nullptr;
2220 llvm::SmallVector<llvm::BasicBlock *, 2> TrapBBs;
2221
2222 /// Terminate funclets keyed by parent funclet pad.
2223 llvm::MapVector<llvm::Value *, llvm::BasicBlock *> TerminateFunclets;
2224
2225 /// Largest vector width used in ths function. Will be used to create a
2226 /// function attribute.
2227 unsigned LargestVectorWidth = 0;
2228
2229 /// True if we need emit the life-time markers. This is initially set in
2230 /// the constructor, but could be overwritten to true if this is a coroutine.
2231 bool ShouldEmitLifetimeMarkers;
2232
2233 /// Add OpenCL kernel arg metadata and the kernel attribute metadata to
2234 /// the function metadata.
2235 void EmitKernelMetadata(const FunctionDecl *FD, llvm::Function *Fn);
2236
2237public:
2238 CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext = false);
2239 ~CodeGenFunction();
2240
2241 CodeGenTypes &getTypes() const { return CGM.getTypes(); }
2242 ASTContext &getContext() const { return CGM.getContext(); }
2243 CGDebugInfo *getDebugInfo() {
2244 if (DisableDebugInfo)
2245 return nullptr;
2246 return DebugInfo;
2247 }
2248 void disableDebugInfo() { DisableDebugInfo = true; }
2249 void enableDebugInfo() { DisableDebugInfo = false; }
2250
2251 bool shouldUseFusedARCCalls() {
2252 return CGM.getCodeGenOpts().OptimizationLevel == 0;
2253 }
2254
2255 const LangOptions &getLangOpts() const { return CGM.getLangOpts(); }
2256
2257 /// Returns a pointer to the function's exception object and selector slot,
2258 /// which is assigned in every landing pad.
2259 Address getExceptionSlot();
2260 Address getEHSelectorSlot();
2261
2262 /// Returns the contents of the function's exception object and selector
2263 /// slots.
2264 llvm::Value *getExceptionFromSlot();
2265 llvm::Value *getSelectorFromSlot();
2266
2267 RawAddress getNormalCleanupDestSlot();
2268
2269 llvm::BasicBlock *getUnreachableBlock() {
2270 if (!UnreachableBlock) {
2271 UnreachableBlock = createBasicBlock(name: "unreachable");
2272 new llvm::UnreachableInst(getLLVMContext(), UnreachableBlock);
2273 }
2274 return UnreachableBlock;
2275 }
2276
2277 llvm::BasicBlock *getInvokeDest() {
2278 if (!EHStack.requiresLandingPad())
2279 return nullptr;
2280 return getInvokeDestImpl();
2281 }
2282
2283 bool currentFunctionUsesSEHTry() const { return !!CurSEHParent; }
2284
2285 const TargetInfo &getTarget() const { return Target; }
2286 llvm::LLVMContext &getLLVMContext() { return CGM.getLLVMContext(); }
2287
2288 /// Accessors for LocalDeclMap.
2289 DeclMapTy::iterator findLocalDecl(const Decl *D) {
2290 return LocalDeclMap.find(Val: D);
2291 }
2292 DeclMapTy::iterator localDeclMapEnd() { return LocalDeclMap.end(); }
2293 std::pair<DeclMapTy::iterator, bool> insertLocalDecl(const Decl *D,
2294 Address Addr) {
2295 return LocalDeclMap.insert(KV: {D, Addr});
2296 }
2297 void eraseLocalDecl(const Decl *D) { LocalDeclMap.erase(Val: D); }
2298 const TargetCodeGenInfo &getTargetHooks() const {
2299 return CGM.getTargetCodeGenInfo();
2300 }
2301 const FunctionDecl *getCurrentFunctionDecl() const;
2302
2303 //===--------------------------------------------------------------------===//
2304 // Cleanups
2305 //===--------------------------------------------------------------------===//
2306
2307 typedef void Destroyer(CodeGenFunction &CGF, Address addr, QualType ty);
2308
2309 void pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,
2310 Address arrayEndPointer,
2311 QualType elementType,
2312 CharUnits elementAlignment,
2313 Destroyer *destroyer);
2314 void pushRegularPartialArrayCleanup(llvm::Value *arrayBegin,
2315 llvm::Value *arrayEnd,
2316 QualType elementType,
2317 CharUnits elementAlignment,
2318 Destroyer *destroyer);
2319
2320 void pushDestroy(QualType::DestructionKind dtorKind, Address addr,
2321 QualType type);
2322 void pushEHDestroy(QualType::DestructionKind dtorKind, Address addr,
2323 QualType type);
2324 void pushDestroy(CleanupKind kind, Address addr, QualType type,
2325 Destroyer *destroyer, bool useEHCleanupForArray);
2326 void pushDestroyAndDeferDeactivation(QualType::DestructionKind dtorKind,
2327 Address addr, QualType type);
2328 void pushDestroyAndDeferDeactivation(CleanupKind cleanupKind, Address addr,
2329 QualType type, Destroyer *destroyer,
2330 bool useEHCleanupForArray);
2331 void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr,
2332 QualType type, Destroyer *destroyer,
2333 bool useEHCleanupForArray);
2334 void pushLifetimeExtendedDestroy(QualType::DestructionKind dtorKind,
2335 Address addr, QualType type);
2336 void pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete,
2337 llvm::Value *CompletePtr,
2338 QualType ElementType);
2339 void pushStackRestore(CleanupKind kind, Address SPMem);
2340 void pushKmpcAllocFree(CleanupKind Kind,
2341 std::pair<llvm::Value *, llvm::Value *> AddrSizePair);
2342 void emitDestroy(Address addr, QualType type, Destroyer *destroyer,
2343 bool useEHCleanupForArray);
2344 llvm::Function *generateDestroyHelper(Address addr, QualType type,
2345 Destroyer *destroyer,
2346 bool useEHCleanupForArray,
2347 const VarDecl *VD);
2348 void emitArrayDestroy(llvm::Value *begin, llvm::Value *end,
2349 QualType elementType, CharUnits elementAlign,
2350 Destroyer *destroyer, bool checkZeroLength,
2351 bool useEHCleanup);
2352
2353 Destroyer *getDestroyer(QualType::DestructionKind destructionKind);
2354
2355 /// Determines whether an EH cleanup is required to destroy a type
2356 /// with the given destruction kind.
2357 bool needsEHCleanup(QualType::DestructionKind kind) {
2358 switch (kind) {
2359 case QualType::DK_none:
2360 return false;
2361 case QualType::DK_cxx_destructor:
2362 case QualType::DK_objc_weak_lifetime:
2363 case QualType::DK_nontrivial_c_struct:
2364 return getLangOpts().Exceptions;
2365 case QualType::DK_objc_strong_lifetime:
2366 return getLangOpts().Exceptions &&
2367 CGM.getCodeGenOpts().ObjCAutoRefCountExceptions;
2368 }
2369 llvm_unreachable("bad destruction kind");
2370 }
2371
2372 CleanupKind getCleanupKind(QualType::DestructionKind kind) {
2373 return (needsEHCleanup(kind) ? NormalAndEHCleanup : NormalCleanup);
2374 }
2375
2376 //===--------------------------------------------------------------------===//
2377 // Objective-C
2378 //===--------------------------------------------------------------------===//
2379
2380 void GenerateObjCMethod(const ObjCMethodDecl *OMD);
2381
2382 void StartObjCMethod(const ObjCMethodDecl *MD, const ObjCContainerDecl *CD);
2383
2384 /// GenerateObjCGetter - Synthesize an Objective-C property getter function.
2385 void GenerateObjCGetter(ObjCImplementationDecl *IMP,
2386 const ObjCPropertyImplDecl *PID);
2387 void generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
2388 const ObjCPropertyImplDecl *propImpl,
2389 const ObjCMethodDecl *GetterMothodDecl,
2390 llvm::Constant *AtomicHelperFn);
2391
2392 void GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
2393 ObjCMethodDecl *MD, bool ctor);
2394
2395 /// GenerateObjCSetter - Synthesize an Objective-C property setter function
2396 /// for the given property.
2397 void GenerateObjCSetter(ObjCImplementationDecl *IMP,
2398 const ObjCPropertyImplDecl *PID);
2399 void generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
2400 const ObjCPropertyImplDecl *propImpl,
2401 llvm::Constant *AtomicHelperFn);
2402
2403 //===--------------------------------------------------------------------===//
2404 // Block Bits
2405 //===--------------------------------------------------------------------===//
2406
2407 /// Emit block literal.
2408 /// \return an LLVM value which is a pointer to a struct which contains
2409 /// information about the block, including the block invoke function, the
2410 /// captured variables, etc.
2411 llvm::Value *EmitBlockLiteral(const BlockExpr *);
2412
2413 llvm::Function *GenerateBlockFunction(GlobalDecl GD, const CGBlockInfo &Info,
2414 const DeclMapTy &ldm,
2415 bool IsLambdaConversionToBlock,
2416 bool BuildGlobalBlock);
2417
2418 /// Check if \p T is a C++ class that has a destructor that can throw.
2419 static bool cxxDestructorCanThrow(QualType T);
2420
2421 llvm::Constant *GenerateCopyHelperFunction(const CGBlockInfo &blockInfo);
2422 llvm::Constant *GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo);
2423 llvm::Constant *
2424 GenerateObjCAtomicSetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);
2425 llvm::Constant *
2426 GenerateObjCAtomicGetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);
2427 llvm::Value *EmitBlockCopyAndAutorelease(llvm::Value *Block, QualType Ty);
2428
2429 void BuildBlockRelease(llvm::Value *DeclPtr, BlockFieldFlags flags,
2430 bool CanThrow);
2431
2432 class AutoVarEmission;
2433
2434 void emitByrefStructureInit(const AutoVarEmission &emission);
2435
2436 /// Enter a cleanup to destroy a __block variable. Note that this
2437 /// cleanup should be a no-op if the variable hasn't left the stack
2438 /// yet; if a cleanup is required for the variable itself, that needs
2439 /// to be done externally.
2440 ///
2441 /// \param Kind Cleanup kind.
2442 ///
2443 /// \param Addr When \p LoadBlockVarAddr is false, the address of the __block
2444 /// structure that will be passed to _Block_object_dispose. When
2445 /// \p LoadBlockVarAddr is true, the address of the field of the block
2446 /// structure that holds the address of the __block structure.
2447 ///
2448 /// \param Flags The flag that will be passed to _Block_object_dispose.
2449 ///
2450 /// \param LoadBlockVarAddr Indicates whether we need to emit a load from
2451 /// \p Addr to get the address of the __block structure.
2452 void enterByrefCleanup(CleanupKind Kind, Address Addr, BlockFieldFlags Flags,
2453 bool LoadBlockVarAddr, bool CanThrow);
2454
2455 void setBlockContextParameter(const ImplicitParamDecl *D, unsigned argNum,
2456 llvm::Value *ptr);
2457
2458 Address LoadBlockStruct();
2459 Address GetAddrOfBlockDecl(const VarDecl *var);
2460
2461 /// BuildBlockByrefAddress - Computes the location of the
2462 /// data in a variable which is declared as __block.
2463 Address emitBlockByrefAddress(Address baseAddr, const VarDecl *V,
2464 bool followForward = true);
2465 Address emitBlockByrefAddress(Address baseAddr, const BlockByrefInfo &info,
2466 bool followForward, const llvm::Twine &name);
2467
2468 const BlockByrefInfo &getBlockByrefInfo(const VarDecl *var);
2469
2470 QualType BuildFunctionArgList(GlobalDecl GD, FunctionArgList &Args);
2471
2472 void GenerateCode(GlobalDecl GD, llvm::Function *Fn,
2473 const CGFunctionInfo &FnInfo);
2474
2475 /// Annotate the function with an attribute that disables TSan checking at
2476 /// runtime.
2477 void markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn);
2478
2479 /// Emit code for the start of a function.
2480 /// \param Loc The location to be associated with the function.
2481 /// \param StartLoc The location of the function body.
2482 void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn,
2483 const CGFunctionInfo &FnInfo, const FunctionArgList &Args,
2484 SourceLocation Loc = SourceLocation(),
2485 SourceLocation StartLoc = SourceLocation());
2486
2487 static bool IsConstructorDelegationValid(const CXXConstructorDecl *Ctor);
2488
2489 void EmitConstructorBody(FunctionArgList &Args);
2490 void EmitDestructorBody(FunctionArgList &Args);
2491 void emitImplicitAssignmentOperatorBody(FunctionArgList &Args);
2492 void EmitFunctionBody(const Stmt *Body);
2493 void EmitBlockWithFallThrough(llvm::BasicBlock *BB, const Stmt *S);
2494
2495 void EmitForwardingCallToLambda(const CXXMethodDecl *LambdaCallOperator,
2496 CallArgList &CallArgs,
2497 const CGFunctionInfo *CallOpFnInfo = nullptr,
2498 llvm::Constant *CallOpFn = nullptr);
2499 void EmitLambdaBlockInvokeBody();
2500 void EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD);
2501 void EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,
2502 CallArgList &CallArgs);
2503 void EmitLambdaInAllocaImplFn(const CXXMethodDecl *CallOp,
2504 const CGFunctionInfo **ImplFnInfo,
2505 llvm::Function **ImplFn);
2506 void EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD);
2507 void EmitLambdaVLACapture(const VariableArrayType *VAT, LValue LV) {
2508 EmitStoreThroughLValue(Src: RValue::get(V: VLASizeMap[VAT->getSizeExpr()]), Dst: LV);
2509 }
2510 void EmitAsanPrologueOrEpilogue(bool Prologue);
2511
2512 /// Emit the unified return block, trying to avoid its emission when
2513 /// possible.
2514 /// \return The debug location of the user written return statement if the
2515 /// return block is avoided.
2516 llvm::DebugLoc EmitReturnBlock();
2517
2518 /// FinishFunction - Complete IR generation of the current function. It is
2519 /// legal to call this function even if there is no current insertion point.
2520 void FinishFunction(SourceLocation EndLoc = SourceLocation());
2521
2522 void StartThunk(llvm::Function *Fn, GlobalDecl GD,
2523 const CGFunctionInfo &FnInfo, bool IsUnprototyped);
2524
2525 void EmitCallAndReturnForThunk(llvm::FunctionCallee Callee,
2526 const ThunkInfo *Thunk, bool IsUnprototyped);
2527
2528 void FinishThunk();
2529
2530 /// Start an Objective-C direct method thunk.
2531 void StartObjCDirectPreconditionThunk(const ObjCMethodDecl *OMD,
2532 llvm::Function *Fn,
2533 const CGFunctionInfo &FI);
2534
2535 /// Finish an Objective-C direct method thunk.
2536 void FinishObjCDirectPreconditionThunk();
2537
2538 /// Emit a musttail call for a thunk with a potentially adjusted this pointer.
2539 void EmitMustTailThunk(GlobalDecl GD, llvm::Value *AdjustedThisPtr,
2540 llvm::FunctionCallee Callee);
2541
2542 /// Generate a thunk for the given method.
2543 void generateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo,
2544 GlobalDecl GD, const ThunkInfo &Thunk,
2545 bool IsUnprototyped);
2546
2547 llvm::Function *GenerateVarArgsThunk(llvm::Function *Fn,
2548 const CGFunctionInfo &FnInfo,
2549 GlobalDecl GD, const ThunkInfo &Thunk);
2550
2551 void EmitCtorPrologue(const CXXConstructorDecl *CD, CXXCtorType Type,
2552 FunctionArgList &Args);
2553
2554 void EmitInitializerForField(FieldDecl *Field, LValue LHS, Expr *Init);
2555
2556 /// Struct with all information about dynamic [sub]class needed to set vptr.
2557 struct VPtr {
2558 BaseSubobject Base;
2559 const CXXRecordDecl *NearestVBase;
2560 CharUnits OffsetFromNearestVBase;
2561 const CXXRecordDecl *VTableClass;
2562 };
2563
2564 /// Initialize the vtable pointer of the given subobject.
2565 void InitializeVTablePointer(const VPtr &vptr);
2566
2567 typedef llvm::SmallVector<VPtr, 4> VPtrsVector;
2568
2569 typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
2570 VPtrsVector getVTablePointers(const CXXRecordDecl *VTableClass);
2571
2572 void getVTablePointers(BaseSubobject Base, const CXXRecordDecl *NearestVBase,
2573 CharUnits OffsetFromNearestVBase,
2574 bool BaseIsNonVirtualPrimaryBase,
2575 const CXXRecordDecl *VTableClass,
2576 VisitedVirtualBasesSetTy &VBases, VPtrsVector &vptrs);
2577
2578 void InitializeVTablePointers(const CXXRecordDecl *ClassDecl);
2579
2580 // VTableTrapMode - whether we guarantee that loading the
2581 // vtable is guaranteed to trap on authentication failure,
2582 // even if the resulting vtable pointer is unused.
2583 enum class VTableAuthMode {
2584 Authenticate,
2585 MustTrap,
2586 UnsafeUbsanStrip // Should only be used for Vptr UBSan check
2587 };
2588 /// GetVTablePtr - Return the Value of the vtable pointer member pointed
2589 /// to by This.
2590 llvm::Value *
2591 GetVTablePtr(Address This, llvm::Type *VTableTy,
2592 const CXXRecordDecl *VTableClass,
2593 VTableAuthMode AuthMode = VTableAuthMode::Authenticate);
2594
2595 enum CFITypeCheckKind {
2596 CFITCK_VCall,
2597 CFITCK_NVCall,
2598 CFITCK_DerivedCast,
2599 CFITCK_UnrelatedCast,
2600 CFITCK_ICall,
2601 CFITCK_NVMFCall,
2602 CFITCK_VMFCall,
2603 };
2604
2605 /// Derived is the presumed address of an object of type T after a
2606 /// cast. If T is a polymorphic class type, emit a check that the virtual
2607 /// table for Derived belongs to a class derived from T.
2608 void EmitVTablePtrCheckForCast(QualType T, Address Derived, bool MayBeNull,
2609 CFITypeCheckKind TCK, SourceLocation Loc);
2610
2611 /// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable.
2612 /// If vptr CFI is enabled, emit a check that VTable is valid.
2613 void EmitVTablePtrCheckForCall(const CXXRecordDecl *RD, llvm::Value *VTable,
2614 CFITypeCheckKind TCK, SourceLocation Loc);
2615
2616 /// EmitVTablePtrCheck - Emit a check that VTable is a valid virtual table for
2617 /// RD using llvm.type.test.
2618 void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable,
2619 CFITypeCheckKind TCK, SourceLocation Loc);
2620
2621 /// If whole-program virtual table optimization is enabled, emit an assumption
2622 /// that VTable is a member of RD's type identifier. Or, if vptr CFI is
2623 /// enabled, emit a check that VTable is a member of RD's type identifier.
2624 void EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2625 llvm::Value *VTable, SourceLocation Loc);
2626
2627 /// Returns whether we should perform a type checked load when loading a
2628 /// virtual function for virtual calls to members of RD. This is generally
2629 /// true when both vcall CFI and whole-program-vtables are enabled.
2630 bool ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD);
2631
2632 /// Emit a type checked load from the given vtable.
2633 llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD,
2634 llvm::Value *VTable,
2635 llvm::Type *VTableTy,
2636 uint64_t VTableByteOffset);
2637
2638 /// EnterDtorCleanups - Enter the cleanups necessary to complete the
2639 /// given phase of destruction for a destructor. The end result
2640 /// should call destructors on members and base classes in reverse
2641 /// order of their construction.
2642 void EnterDtorCleanups(const CXXDestructorDecl *Dtor, CXXDtorType Type);
2643
2644 /// ShouldInstrumentFunction - Return true if the current function should be
2645 /// instrumented with __cyg_profile_func_* calls
2646 bool ShouldInstrumentFunction();
2647
2648 /// ShouldSkipSanitizerInstrumentation - Return true if the current function
2649 /// should not be instrumented with sanitizers.
2650 bool ShouldSkipSanitizerInstrumentation();
2651
2652 /// ShouldXRayInstrument - Return true if the current function should be
2653 /// instrumented with XRay nop sleds.
2654 bool ShouldXRayInstrumentFunction() const;
2655
2656 /// AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit
2657 /// XRay custom event handling calls.
2658 bool AlwaysEmitXRayCustomEvents() const;
2659
2660 /// AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit
2661 /// XRay typed event handling calls.
2662 bool AlwaysEmitXRayTypedEvents() const;
2663
2664 /// Return a type hash constant for a function instrumented by
2665 /// -fsanitize=function.
2666 llvm::ConstantInt *getUBSanFunctionTypeHash(QualType T) const;
2667
2668 /// EmitFunctionProlog - Emit the target specific LLVM code to load the
2669 /// arguments for the given function. This is also responsible for naming the
2670 /// LLVM function arguments.
2671 void EmitFunctionProlog(const CGFunctionInfo &FI, llvm::Function *Fn,
2672 const FunctionArgList &Args);
2673
2674 /// EmitFunctionEpilog - Emit the target specific LLVM code to return the
2675 /// given temporary. Specify the source location atom group (Key Instructions
2676 /// debug info feature) for the `ret` using \p RetKeyInstructionsSourceAtom.
2677 /// If it's 0, the `ret` will get added to a new source atom group.
2678 void EmitFunctionEpilog(const CGFunctionInfo &FI, bool EmitRetDbgLoc,
2679 SourceLocation EndLoc,
2680 uint64_t RetKeyInstructionsSourceAtom);
2681
2682 /// Emit a test that checks if the return value \p RV is nonnull.
2683 void EmitReturnValueCheck(llvm::Value *RV);
2684
2685 /// EmitStartEHSpec - Emit the start of the exception spec.
2686 void EmitStartEHSpec(const Decl *D);
2687
2688 /// EmitEndEHSpec - Emit the end of the exception spec.
2689 void EmitEndEHSpec(const Decl *D);
2690
2691 /// getTerminateLandingPad - Return a landing pad that just calls terminate.
2692 llvm::BasicBlock *getTerminateLandingPad();
2693
2694 /// getTerminateLandingPad - Return a cleanup funclet that just calls
2695 /// terminate.
2696 llvm::BasicBlock *getTerminateFunclet();
2697
2698 /// getTerminateHandler - Return a handler (not a landing pad, just
2699 /// a catch handler) that just calls terminate. This is used when
2700 /// a terminate scope encloses a try.
2701 llvm::BasicBlock *getTerminateHandler();
2702
2703 llvm::Type *ConvertTypeForMem(QualType T);
2704 llvm::Type *ConvertType(QualType T);
2705 llvm::Type *convertTypeForLoadStore(QualType ASTTy,
2706 llvm::Type *LLVMTy = nullptr);
2707 llvm::Type *ConvertType(const TypeDecl *T) {
2708 return ConvertType(T: getContext().getTypeDeclType(Decl: T));
2709 }
2710
2711 /// LoadObjCSelf - Load the value of self. This function is only valid while
2712 /// generating code for an Objective-C method.
2713 llvm::Value *LoadObjCSelf();
2714
2715 /// TypeOfSelfObject - Return type of object that this self represents.
2716 QualType TypeOfSelfObject();
2717
2718 /// getEvaluationKind - Return the TypeEvaluationKind of QualType \c T.
2719 static TypeEvaluationKind getEvaluationKind(QualType T);
2720
2721 static bool hasScalarEvaluationKind(QualType T) {
2722 return getEvaluationKind(T) == TEK_Scalar;
2723 }
2724
2725 static bool hasAggregateEvaluationKind(QualType T) {
2726 return getEvaluationKind(T) == TEK_Aggregate;
2727 }
2728
2729 /// createBasicBlock - Create an LLVM basic block.
2730 llvm::BasicBlock *createBasicBlock(const Twine &name = "",
2731 llvm::Function *parent = nullptr,
2732 llvm::BasicBlock *before = nullptr) {
2733 return llvm::BasicBlock::Create(Context&: getLLVMContext(), Name: name, Parent: parent, InsertBefore: before);
2734 }
2735
2736 /// getBasicBlockForLabel - Return the LLVM basicblock that the specified
2737 /// label maps to.
2738 JumpDest getJumpDestForLabel(const LabelDecl *S);
2739
2740 /// SimplifyForwardingBlocks - If the given basic block is only a branch to
2741 /// another basic block, simplify it. This assumes that no other code could
2742 /// potentially reference the basic block.
2743 void SimplifyForwardingBlocks(llvm::BasicBlock *BB);
2744
2745 /// EmitBlock - Emit the given block \arg BB and set it as the insert point,
2746 /// adding a fall-through branch from the current insert block if
2747 /// necessary. It is legal to call this function even if there is no current
2748 /// insertion point.
2749 ///
2750 /// IsFinished - If true, indicates that the caller has finished emitting
2751 /// branches to the given block and does not expect to emit code into it. This
2752 /// means the block can be ignored if it is unreachable.
2753 void EmitBlock(llvm::BasicBlock *BB, bool IsFinished = false);
2754
2755 /// EmitBlockAfterUses - Emit the given block somewhere hopefully
2756 /// near its uses, and leave the insertion point in it.
2757 void EmitBlockAfterUses(llvm::BasicBlock *BB);
2758
2759 /// EmitBranch - Emit a branch to the specified basic block from the current
2760 /// insert block, taking care to avoid creation of branches from dummy
2761 /// blocks. It is legal to call this function even if there is no current
2762 /// insertion point.
2763 ///
2764 /// This function clears the current insertion point. The caller should follow
2765 /// calls to this function with calls to Emit*Block prior to generation new
2766 /// code.
2767 void EmitBranch(llvm::BasicBlock *Block);
2768
2769 /// HaveInsertPoint - True if an insertion point is defined. If not, this
2770 /// indicates that the current code being emitted is unreachable.
2771 bool HaveInsertPoint() const { return Builder.GetInsertBlock() != nullptr; }
2772
2773 /// EnsureInsertPoint - Ensure that an insertion point is defined so that
2774 /// emitted IR has a place to go. Note that by definition, if this function
2775 /// creates a block then that block is unreachable; callers may do better to
2776 /// detect when no insertion point is defined and simply skip IR generation.
2777 void EnsureInsertPoint() {
2778 if (!HaveInsertPoint())
2779 EmitBlock(BB: createBasicBlock());
2780 }
2781
2782 /// ErrorUnsupported - Print out an error that codegen doesn't support the
2783 /// specified stmt yet.
2784 void ErrorUnsupported(const Stmt *S, const char *Type);
2785
2786 //===--------------------------------------------------------------------===//
2787 // Helpers
2788 //===--------------------------------------------------------------------===//
2789
2790 Address mergeAddressesInConditionalExpr(Address LHS, Address RHS,
2791 llvm::BasicBlock *LHSBlock,
2792 llvm::BasicBlock *RHSBlock,
2793 llvm::BasicBlock *MergeBlock,
2794 QualType MergedType) {
2795 Builder.SetInsertPoint(MergeBlock);
2796 llvm::PHINode *PtrPhi = Builder.CreatePHI(Ty: LHS.getType(), NumReservedValues: 2, Name: "cond");
2797 PtrPhi->addIncoming(V: LHS.getBasePointer(), BB: LHSBlock);
2798 PtrPhi->addIncoming(V: RHS.getBasePointer(), BB: RHSBlock);
2799 LHS.replaceBasePointer(P: PtrPhi);
2800 LHS.setAlignment(std::min(a: LHS.getAlignment(), b: RHS.getAlignment()));
2801 return LHS;
2802 }
2803
2804 /// Construct an address with the natural alignment of T. If a pointer to T
2805 /// is expected to be signed, the pointer passed to this function must have
2806 /// been signed, and the returned Address will have the pointer authentication
2807 /// information needed to authenticate the signed pointer.
2808 Address makeNaturalAddressForPointer(
2809 llvm::Value *Ptr, QualType T, CharUnits Alignment = CharUnits::Zero(),
2810 bool ForPointeeType = false, LValueBaseInfo *BaseInfo = nullptr,
2811 TBAAAccessInfo *TBAAInfo = nullptr,
2812 KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
2813 if (Alignment.isZero())
2814 Alignment =
2815 CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, forPointeeType: ForPointeeType);
2816 return Address(Ptr, ConvertTypeForMem(T), Alignment,
2817 CGM.getPointerAuthInfoForPointeeType(type: T), /*Offset=*/nullptr,
2818 IsKnownNonNull);
2819 }
2820
2821 LValue MakeAddrLValue(Address Addr, QualType T,
2822 AlignmentSource Source = AlignmentSource::Type) {
2823 return MakeAddrLValue(Addr, T, BaseInfo: LValueBaseInfo(Source),
2824 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2825 }
2826
2827 LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo,
2828 TBAAAccessInfo TBAAInfo) {
2829 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo, TBAAInfo);
2830 }
2831
2832 LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2833 AlignmentSource Source = AlignmentSource::Type) {
2834 return MakeAddrLValue(Addr: makeNaturalAddressForPointer(Ptr: V, T, Alignment), T,
2835 BaseInfo: LValueBaseInfo(Source), TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2836 }
2837
2838 /// Same as MakeAddrLValue above except that the pointer is known to be
2839 /// unsigned.
2840 LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
2841 AlignmentSource Source = AlignmentSource::Type) {
2842 Address Addr(V, ConvertTypeForMem(T), Alignment);
2843 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2844 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: T));
2845 }
2846
2847 LValue
2848 MakeAddrLValueWithoutTBAA(Address Addr, QualType T,
2849 AlignmentSource Source = AlignmentSource::Type) {
2850 return LValue::MakeAddr(Addr, type: T, Context&: getContext(), BaseInfo: LValueBaseInfo(Source),
2851 TBAAInfo: TBAAAccessInfo());
2852 }
2853
2854 /// Given a value of type T* that may not be to a complete object, construct
2855 /// an l-value with the natural pointee alignment of T.
2856 LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T);
2857
2858 LValue
2859 MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
2860 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
2861
2862 /// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known
2863 /// to be unsigned.
2864 LValue MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, QualType T);
2865
2866 LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T);
2867
2868 Address EmitLoadOfReference(LValue RefLVal,
2869 LValueBaseInfo *PointeeBaseInfo = nullptr,
2870 TBAAAccessInfo *PointeeTBAAInfo = nullptr);
2871 LValue EmitLoadOfReferenceLValue(LValue RefLVal);
2872 LValue
2873 EmitLoadOfReferenceLValue(Address RefAddr, QualType RefTy,
2874 AlignmentSource Source = AlignmentSource::Type) {
2875 LValue RefLVal = MakeAddrLValue(Addr: RefAddr, T: RefTy, BaseInfo: LValueBaseInfo(Source),
2876 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: RefTy));
2877 return EmitLoadOfReferenceLValue(RefLVal);
2878 }
2879
2880 /// Load a pointer with type \p PtrTy stored at address \p Ptr.
2881 /// Note that \p PtrTy is the type of the loaded pointer, not the addresses
2882 /// it is loaded from.
2883 Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy,
2884 LValueBaseInfo *BaseInfo = nullptr,
2885 TBAAAccessInfo *TBAAInfo = nullptr);
2886 LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy);
2887
2888private:
2889 struct AllocaTracker {
2890 void Add(llvm::AllocaInst *I) { Allocas.push_back(Elt: I); }
2891 llvm::SmallVector<llvm::AllocaInst *> Take() { return std::move(Allocas); }
2892
2893 private:
2894 llvm::SmallVector<llvm::AllocaInst *> Allocas;
2895 };
2896 AllocaTracker *Allocas = nullptr;
2897
2898 /// CGDecl helper.
2899 void emitStoresForConstant(const VarDecl &D, Address Loc, bool isVolatile,
2900 llvm::Constant *constant, bool IsAutoInit);
2901 /// CGDecl helper.
2902 void emitStoresForZeroInit(const VarDecl &D, Address Loc, bool isVolatile);
2903 /// CGDecl helper.
2904 void emitStoresForPatternInit(const VarDecl &D, Address Loc, bool isVolatile);
2905 /// CGDecl helper.
2906 void emitStoresForInitAfterBZero(llvm::Constant *Init, Address Loc,
2907 bool isVolatile, bool IsAutoInit);
2908
2909public:
2910 // Captures all the allocas created during the scope of its RAII object.
2911 struct AllocaTrackerRAII {
2912 AllocaTrackerRAII(CodeGenFunction &CGF)
2913 : CGF(CGF), OldTracker(CGF.Allocas) {
2914 CGF.Allocas = &Tracker;
2915 }
2916 ~AllocaTrackerRAII() { CGF.Allocas = OldTracker; }
2917
2918 llvm::SmallVector<llvm::AllocaInst *> Take() { return Tracker.Take(); }
2919
2920 private:
2921 CodeGenFunction &CGF;
2922 AllocaTracker *OldTracker;
2923 AllocaTracker Tracker;
2924 };
2925
2926private:
2927 /// If \p Alloca is not in the same address space as \p DestLangAS, insert an
2928 /// address space cast and return a new RawAddress based on this value.
2929 RawAddress MaybeCastStackAddressSpace(RawAddress Alloca, LangAS DestLangAS,
2930 llvm::Value *ArraySize = nullptr);
2931
2932public:
2933 /// CreateTempAlloca - This creates an alloca and inserts it into the entry
2934 /// block if \p ArraySize is nullptr, otherwise inserts it at the current
2935 /// insertion point of the builder. The caller is responsible for setting an
2936 /// appropriate alignment on the alloca.
2937 ///
2938 /// \p ArraySize is the number of array elements to be allocated if it
2939 /// is not nullptr.
2940 ///
2941 /// LangAS::Default is the address space of pointers to local variables and
2942 /// temporaries, as exposed in the source language. In certain
2943 /// configurations, this is not the same as the alloca address space, and a
2944 /// cast is needed to lift the pointer from the alloca AS into
2945 /// LangAS::Default. This can happen when the target uses a restricted
2946 /// address space for the stack but the source language requires
2947 /// LangAS::Default to be a generic address space. The latter condition is
2948 /// common for most programming languages; OpenCL is an exception in that
2949 /// LangAS::Default is the private address space, which naturally maps
2950 /// to the stack.
2951 ///
2952 /// Because the address of a temporary is often exposed to the program in
2953 /// various ways, this function will perform the cast. The original alloca
2954 /// instruction is returned through \p Alloca if it is not nullptr.
2955 ///
2956 /// The cast is not performed in CreateTempAllocaWithoutCast. This is
2957 /// more efficient if the caller knows that the address will not be exposed.
2958 llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp",
2959 llvm::Value *ArraySize = nullptr);
2960
2961 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
2962 /// block. The alloca is casted to the address space of \p UseAddrSpace if
2963 /// necessary.
2964 RawAddress CreateTempAlloca(llvm::Type *Ty, LangAS UseAddrSpace,
2965 CharUnits align, const Twine &Name = "tmp",
2966 llvm::Value *ArraySize = nullptr,
2967 RawAddress *Alloca = nullptr);
2968
2969 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
2970 /// block. The alloca is casted to default address space if necessary.
2971 ///
2972 /// FIXME: This version should be removed, and context should provide the
2973 /// context use address space used instead of default.
2974 RawAddress CreateTempAlloca(llvm::Type *Ty, CharUnits align,
2975 const Twine &Name = "tmp",
2976 llvm::Value *ArraySize = nullptr,
2977 RawAddress *Alloca = nullptr) {
2978 return CreateTempAlloca(Ty, UseAddrSpace: LangAS::Default, align, Name, ArraySize,
2979 Alloca);
2980 }
2981
2982 RawAddress CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align,
2983 const Twine &Name = "tmp",
2984 llvm::Value *ArraySize = nullptr);
2985
2986 /// CreateDefaultAlignedTempAlloca - This creates an alloca with the
2987 /// default ABI alignment of the given LLVM type.
2988 ///
2989 /// IMPORTANT NOTE: This is *not* generally the right alignment for
2990 /// any given AST type that happens to have been lowered to the
2991 /// given IR type. This should only ever be used for function-local,
2992 /// IR-driven manipulations like saving and restoring a value. Do
2993 /// not hand this address off to arbitrary IRGen routines, and especially
2994 /// do not pass it as an argument to a function that might expect a
2995 /// properly ABI-aligned value.
2996 RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty,
2997 const Twine &Name = "tmp");
2998
2999 /// CreateIRTempWithoutCast - Create a temporary IR object of the given type,
3000 /// with appropriate alignment. This routine should only be used when an
3001 /// temporary value needs to be stored into an alloca (for example, to avoid
3002 /// explicit PHI construction), but the type is the IR type, not the type
3003 /// appropriate for storing in memory.
3004 ///
3005 /// That is, this is exactly equivalent to CreateMemTemp, but calling
3006 /// ConvertType instead of ConvertTypeForMem.
3007 RawAddress CreateIRTempWithoutCast(QualType T, const Twine &Name = "tmp");
3008
3009 /// CreateMemTemp - Create a temporary memory object of the given type, with
3010 /// appropriate alignmen and cast it to the default address space. Returns
3011 /// the original alloca instruction by \p Alloca if it is not nullptr.
3012 RawAddress CreateMemTemp(QualType T, const Twine &Name = "tmp",
3013 RawAddress *Alloca = nullptr);
3014 RawAddress CreateMemTemp(QualType T, CharUnits Align,
3015 const Twine &Name = "tmp",
3016 RawAddress *Alloca = nullptr);
3017
3018 /// CreateMemTemp - Create a temporary memory object of the given type, with
3019 /// appropriate alignmen without casting it to the default address space.
3020 RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp");
3021 RawAddress CreateMemTempWithoutCast(QualType T, CharUnits Align,
3022 const Twine &Name = "tmp");
3023
3024 /// CreateAggTemp - Create a temporary memory object for the given
3025 /// aggregate type.
3026 AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp",
3027 RawAddress *Alloca = nullptr) {
3028 RawAddress Addr = CreateMemTemp(T, Name, Alloca);
3029 return AggValueSlot::forAddr(
3030 addr: Addr, quals: T.getQualifiers(), isDestructed: AggValueSlot::IsNotDestructed,
3031 needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
3032 mayOverlap: AggValueSlot::DoesNotOverlap);
3033 }
3034
3035 /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
3036 /// expression and compare the result against zero, returning an Int1Ty value.
3037 llvm::Value *EvaluateExprAsBool(const Expr *E);
3038
3039 /// Retrieve the implicit cast expression of the rhs in a binary operator
3040 /// expression by passing pointers to Value and QualType
3041 /// This is used for implicit bitfield conversion checks, which
3042 /// must compare with the value before potential truncation.
3043 llvm::Value *EmitWithOriginalRHSBitfieldAssignment(const BinaryOperator *E,
3044 llvm::Value **Previous,
3045 QualType *SrcType);
3046
3047 /// Emit a check that an [implicit] conversion of a bitfield. It is not UB,
3048 /// so we use the value after conversion.
3049 void EmitBitfieldConversionCheck(llvm::Value *Src, QualType SrcType,
3050 llvm::Value *Dst, QualType DstType,
3051 const CGBitFieldInfo &Info,
3052 SourceLocation Loc);
3053
3054 /// EmitIgnoredExpr - Emit an expression in a context which ignores the
3055 /// result.
3056 void EmitIgnoredExpr(const Expr *E);
3057
3058 /// EmitAnyExpr - Emit code to compute the specified expression which can have
3059 /// any type. The result is returned as an RValue struct. If this is an
3060 /// aggregate expression, the aggloc/agglocvolatile arguments indicate where
3061 /// the result should be returned.
3062 ///
3063 /// \param ignoreResult True if the resulting value isn't used.
3064 RValue EmitAnyExpr(const Expr *E,
3065 AggValueSlot aggSlot = AggValueSlot::ignored(),
3066 bool ignoreResult = false);
3067
3068 // EmitVAListRef - Emit a "reference" to a va_list; this is either the address
3069 // or the value of the expression, depending on how va_list is defined.
3070 Address EmitVAListRef(const Expr *E);
3071
3072 /// Emit a "reference" to a __builtin_ms_va_list; this is
3073 /// always the value of the expression, because a __builtin_ms_va_list is a
3074 /// pointer to a char.
3075 Address EmitMSVAListRef(const Expr *E);
3076
3077 /// Emit a "reference" to a __builtin_zos_va_list; this is always the
3078 /// address of the expression, because a __builtin_zos_va_list is an
3079 /// array of pointer to a char.
3080 Address EmitZOSVAListRef(const Expr *E);
3081
3082 /// EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will
3083 /// always be accessible even if no aggregate location is provided.
3084 RValue EmitAnyExprToTemp(const Expr *E);
3085
3086 /// EmitAnyExprToMem - Emits the code necessary to evaluate an
3087 /// arbitrary expression into the given memory location.
3088 void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals,
3089 bool IsInitializer);
3090
3091 void EmitAnyExprToExn(const Expr *E, Address Addr);
3092
3093 /// EmitInitializationToLValue - Emit an initializer to an LValue.
3094 void EmitInitializationToLValue(
3095 const Expr *E, LValue LV,
3096 AggValueSlot::IsZeroed_t IsZeroed = AggValueSlot::IsNotZeroed);
3097
3098 /// EmitExprAsInit - Emits the code necessary to initialize a
3099 /// location in memory with the given initializer.
3100 void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue,
3101 bool capturedByInit);
3102
3103 /// hasVolatileMember - returns true if aggregate type has a volatile
3104 /// member.
3105 bool hasVolatileMember(QualType T) {
3106 if (const auto *RD = T->getAsRecordDecl())
3107 return RD->hasVolatileMember();
3108 return false;
3109 }
3110
3111 /// Determine whether a return value slot may overlap some other object.
3112 AggValueSlot::Overlap_t getOverlapForReturnValue() {
3113 // FIXME: Assuming no overlap here breaks guaranteed copy elision for base
3114 // class subobjects. These cases may need to be revisited depending on the
3115 // resolution of the relevant core issue.
3116 return AggValueSlot::DoesNotOverlap;
3117 }
3118
3119 /// Determine whether a field initialization may overlap some other object.
3120 AggValueSlot::Overlap_t getOverlapForFieldInit(const FieldDecl *FD);
3121
3122 /// Determine whether a base class initialization may overlap some other
3123 /// object.
3124 AggValueSlot::Overlap_t getOverlapForBaseInit(const CXXRecordDecl *RD,
3125 const CXXRecordDecl *BaseRD,
3126 bool IsVirtual);
3127
3128 /// Emit an aggregate assignment.
3129 void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy) {
3130 ApplyAtomGroup Grp(getDebugInfo());
3131 bool IsVolatile = hasVolatileMember(T: EltTy);
3132 EmitAggregateCopy(Dest, Src, EltTy, MayOverlap: AggValueSlot::MayOverlap, isVolatile: IsVolatile);
3133 }
3134
3135 void EmitAggregateCopyCtor(LValue Dest, LValue Src,
3136 AggValueSlot::Overlap_t MayOverlap) {
3137 EmitAggregateCopy(Dest, Src, EltTy: Src.getType(), MayOverlap);
3138 }
3139
3140 /// EmitAggregateCopy - Emit an aggregate copy.
3141 ///
3142 /// \param isVolatile \c true iff either the source or the destination is
3143 /// volatile.
3144 /// \param MayOverlap Whether the tail padding of the destination might be
3145 /// occupied by some other object. More efficient code can often be
3146 /// generated if not.
3147 void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy,
3148 AggValueSlot::Overlap_t MayOverlap,
3149 bool isVolatile = false);
3150
3151 /// GetAddrOfLocalVar - Return the address of a local variable.
3152 Address GetAddrOfLocalVar(const VarDecl *VD) {
3153 auto it = LocalDeclMap.find(Val: VD);
3154 assert(it != LocalDeclMap.end() &&
3155 "Invalid argument to GetAddrOfLocalVar(), no decl!");
3156 return it->second;
3157 }
3158
3159 /// Given an opaque value expression, return its LValue mapping if it exists,
3160 /// otherwise create one.
3161 LValue getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e);
3162
3163 /// Given an opaque value expression, return its RValue mapping if it exists,
3164 /// otherwise create one.
3165 RValue getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e);
3166
3167 /// isOpaqueValueEmitted - Return true if the opaque value expression has
3168 /// already been emitted.
3169 bool isOpaqueValueEmitted(const OpaqueValueExpr *E);
3170
3171 /// Get the index of the current ArrayInitLoopExpr, if any.
3172 llvm::Value *getArrayInitIndex() { return ArrayInitIndex; }
3173
3174 /// getAccessedFieldNo - Given an encoded value and a result number, return
3175 /// the input field number being accessed.
3176 static unsigned getAccessedFieldNo(unsigned Idx, const llvm::Constant *Elts);
3177
3178 llvm::BlockAddress *GetAddrOfLabel(const LabelDecl *L);
3179 llvm::BasicBlock *GetIndirectGotoBlock();
3180
3181 /// Check if \p E is a C++ "this" pointer wrapped in value-preserving casts.
3182 static bool IsWrappedCXXThis(const Expr *E);
3183
3184 /// EmitNullInitialization - Generate code to set a value of the given type to
3185 /// null, If the type contains data member pointers, they will be initialized
3186 /// to -1 in accordance with the Itanium C++ ABI.
3187 void EmitNullInitialization(Address DestPtr, QualType Ty);
3188
3189 /// Emits a call to an LLVM variable-argument intrinsic, either
3190 /// \c llvm.va_start or \c llvm.va_end.
3191 /// \param ArgValue A reference to the \c va_list as emitted by either
3192 /// \c EmitVAListRef or \c EmitMSVAListRef.
3193 /// \param IsStart If \c true, emits a call to \c llvm.va_start; otherwise,
3194 /// calls \c llvm.va_end.
3195 llvm::Value *EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart);
3196
3197 /// Generate code to get an argument from the passed in pointer
3198 /// and update it accordingly.
3199 /// \param VE The \c VAArgExpr for which to generate code.
3200 /// \param VAListAddr Receives a reference to the \c va_list as emitted by
3201 /// either \c EmitVAListRef or \c EmitMSVAListRef.
3202 /// \returns A pointer to the argument.
3203 // FIXME: We should be able to get rid of this method and use the va_arg
3204 // instruction in LLVM instead once it works well enough.
3205 RValue EmitVAArg(VAArgExpr *VE, Address &VAListAddr,
3206 AggValueSlot Slot = AggValueSlot::ignored());
3207
3208 /// emitArrayLength - Compute the length of an array, even if it's a
3209 /// VLA, and drill down to the base element type.
3210 llvm::Value *emitArrayLength(const ArrayType *arrayType, QualType &baseType,
3211 Address &addr);
3212
3213 /// EmitVLASize - Capture all the sizes for the VLA expressions in
3214 /// the given variably-modified type and store them in the VLASizeMap.
3215 ///
3216 /// This function can be called with a null (unreachable) insert point.
3217 void EmitVariablyModifiedType(QualType Ty);
3218
3219 struct VlaSizePair {
3220 llvm::Value *NumElts;
3221 QualType Type;
3222
3223 VlaSizePair(llvm::Value *NE, QualType T) : NumElts(NE), Type(T) {}
3224 };
3225
3226 /// Return the number of elements for a single dimension
3227 /// for the given array type.
3228 VlaSizePair getVLAElements1D(const VariableArrayType *vla);
3229 VlaSizePair getVLAElements1D(QualType vla);
3230
3231 /// Returns an LLVM value that corresponds to the size,
3232 /// in non-variably-sized elements, of a variable length array type,
3233 /// plus that largest non-variably-sized element type. Assumes that
3234 /// the type has already been emitted with EmitVariablyModifiedType.
3235 VlaSizePair getVLASize(const VariableArrayType *vla);
3236 VlaSizePair getVLASize(QualType vla);
3237
3238 /// LoadCXXThis - Load the value of 'this'. This function is only valid while
3239 /// generating code for an C++ member function.
3240 llvm::Value *LoadCXXThis() {
3241 assert(CXXThisValue && "no 'this' value for this function");
3242 return CXXThisValue;
3243 }
3244 Address LoadCXXThisAddress();
3245
3246 /// LoadCXXVTT - Load the VTT parameter to base constructors/destructors have
3247 /// virtual bases.
3248 // FIXME: Every place that calls LoadCXXVTT is something
3249 // that needs to be abstracted properly.
3250 llvm::Value *LoadCXXVTT() {
3251 assert(CXXStructorImplicitParamValue && "no VTT value for this function");
3252 return CXXStructorImplicitParamValue;
3253 }
3254
3255 /// GetAddressOfBaseOfCompleteClass - Convert the given pointer to a
3256 /// complete class to the given direct base.
3257 Address GetAddressOfDirectBaseInCompleteClass(Address Value,
3258 const CXXRecordDecl *Derived,
3259 const CXXRecordDecl *Base,
3260 bool BaseIsVirtual);
3261
3262 static bool ShouldNullCheckClassCastValue(const CastExpr *Cast);
3263
3264 /// GetAddressOfBaseClass - This function will add the necessary delta to the
3265 /// load of 'this' and returns address of the base class.
3266 Address GetAddressOfBaseClass(Address Value, const CXXRecordDecl *Derived,
3267 CastExpr::path_const_iterator PathBegin,
3268 CastExpr::path_const_iterator PathEnd,
3269 bool NullCheckValue, SourceLocation Loc);
3270
3271 Address GetAddressOfDerivedClass(Address Value, const CXXRecordDecl *Derived,
3272 CastExpr::path_const_iterator PathBegin,
3273 CastExpr::path_const_iterator PathEnd,
3274 bool NullCheckValue);
3275
3276 /// GetVTTParameter - Return the VTT parameter that should be passed to a
3277 /// base constructor/destructor with virtual bases.
3278 /// FIXME: VTTs are Itanium ABI-specific, so the definition should move
3279 /// to ItaniumCXXABI.cpp together with all the references to VTT.
3280 llvm::Value *GetVTTParameter(GlobalDecl GD, bool ForVirtualBase,
3281 bool Delegating);
3282
3283 void EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
3284 CXXCtorType CtorType,
3285 const FunctionArgList &Args,
3286 SourceLocation Loc);
3287 // It's important not to confuse this and the previous function. Delegating
3288 // constructors are the C++0x feature. The constructor delegate optimization
3289 // is used to reduce duplication in the base and complete consturctors where
3290 // they are substantially the same.
3291 void EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3292 const FunctionArgList &Args);
3293
3294 /// Emit a call to an inheriting constructor (that is, one that invokes a
3295 /// constructor inherited from a base class) by inlining its definition. This
3296 /// is necessary if the ABI does not support forwarding the arguments to the
3297 /// base class constructor (because they're variadic or similar).
3298 void EmitInlinedInheritingCXXConstructorCall(const CXXConstructorDecl *Ctor,
3299 CXXCtorType CtorType,
3300 bool ForVirtualBase,
3301 bool Delegating,
3302 CallArgList &Args);
3303
3304 /// Emit a call to a constructor inherited from a base class, passing the
3305 /// current constructor's arguments along unmodified (without even making
3306 /// a copy).
3307 void EmitInheritedCXXConstructorCall(const CXXConstructorDecl *D,
3308 bool ForVirtualBase, Address This,
3309 bool InheritedFromVBase,
3310 const CXXInheritedCtorInitExpr *E);
3311
3312 void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3313 bool ForVirtualBase, bool Delegating,
3314 AggValueSlot ThisAVS, const CXXConstructExpr *E);
3315
3316 void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,
3317 bool ForVirtualBase, bool Delegating,
3318 Address This, CallArgList &Args,
3319 AggValueSlot::Overlap_t Overlap,
3320 SourceLocation Loc, bool NewPointerIsChecked,
3321 llvm::CallBase **CallOrInvoke = nullptr);
3322
3323 /// Emit assumption load for all bases. Requires to be called only on
3324 /// most-derived class and not under construction of the object.
3325 void EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl, Address This);
3326
3327 /// Emit assumption that vptr load == global vtable.
3328 void EmitVTableAssumptionLoad(const VPtr &vptr, Address This);
3329
3330 void EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, Address This,
3331 Address Src, const CXXConstructExpr *E);
3332
3333 void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3334 const ArrayType *ArrayTy, Address ArrayPtr,
3335 const CXXConstructExpr *E,
3336 bool NewPointerIsChecked,
3337 bool ZeroInitialization = false);
3338
3339 void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
3340 llvm::Value *NumElements, Address ArrayPtr,
3341 const CXXConstructExpr *E,
3342 bool NewPointerIsChecked,
3343 bool ZeroInitialization = false);
3344
3345 static Destroyer destroyCXXObject;
3346
3347 void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type,
3348 bool ForVirtualBase, bool Delegating, Address This,
3349 QualType ThisTy);
3350
3351 void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,
3352 llvm::Type *ElementTy, Address NewPtr,
3353 llvm::Value *NumElements,
3354 llvm::Value *AllocSizeWithoutCookie);
3355
3356 void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,
3357 Address Ptr);
3358
3359 void EmitSehCppScopeBegin();
3360 void EmitSehCppScopeEnd();
3361 void EmitSehTryScopeBegin();
3362 void EmitSehTryScopeEnd();
3363
3364 bool EmitLifetimeStart(llvm::Value *Addr);
3365 void EmitLifetimeEnd(llvm::Value *Addr);
3366
3367 llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
3368 void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
3369
3370 void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,
3371 QualType DeleteTy, llvm::Value *NumElements = nullptr,
3372 CharUnits CookieSize = CharUnits(),
3373 llvm::Constant *CalleeOverride = nullptr);
3374
3375 RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
3376 const CallExpr *TheCallExpr, bool IsDelete);
3377
3378 llvm::Value *EmitCXXTypeidExpr(const CXXTypeidExpr *E);
3379 llvm::Value *EmitDynamicCast(Address V, const CXXDynamicCastExpr *DCE);
3380 Address EmitCXXUuidofExpr(const CXXUuidofExpr *E);
3381
3382 /// Situations in which we might emit a check for the suitability of a
3383 /// pointer or glvalue. Needs to be kept in sync with ubsan_handlers.cpp in
3384 /// compiler-rt.
3385 enum TypeCheckKind {
3386 /// Checking the operand of a load. Must be suitably sized and aligned.
3387 TCK_Load,
3388 /// Checking the destination of a store. Must be suitably sized and aligned.
3389 TCK_Store,
3390 /// Checking the bound value in a reference binding. Must be suitably sized
3391 /// and aligned, but is not required to refer to an object (until the
3392 /// reference is used), per core issue 453.
3393 TCK_ReferenceBinding,
3394 /// Checking the object expression in a non-static data member access. Must
3395 /// be an object within its lifetime.
3396 TCK_MemberAccess,
3397 /// Checking the 'this' pointer for a call to a non-static member function.
3398 /// Must be an object within its lifetime.
3399 TCK_MemberCall,
3400 /// Checking the 'this' pointer for a constructor call.
3401 TCK_ConstructorCall,
3402 /// Checking the operand of a static_cast to a derived pointer type. Must be
3403 /// null or an object within its lifetime.
3404 TCK_DowncastPointer,
3405 /// Checking the operand of a static_cast to a derived reference type. Must
3406 /// be an object within its lifetime.
3407 TCK_DowncastReference,
3408 /// Checking the operand of a cast to a base object. Must be suitably sized
3409 /// and aligned.
3410 TCK_Upcast,
3411 /// Checking the operand of a cast to a virtual base object. Must be an
3412 /// object within its lifetime.
3413 TCK_UpcastToVirtualBase,
3414 /// Checking the value assigned to a _Nonnull pointer. Must not be null.
3415 TCK_NonnullAssign,
3416 /// Checking the operand of a dynamic_cast or a typeid expression. Must be
3417 /// null or an object within its lifetime.
3418 TCK_DynamicOperation
3419 };
3420
3421 /// Determine whether the pointer type check \p TCK permits null pointers.
3422 static bool isNullPointerAllowed(TypeCheckKind TCK);
3423
3424 /// Determine whether the pointer type check \p TCK requires a vptr check.
3425 static bool isVptrCheckRequired(TypeCheckKind TCK, QualType Ty);
3426
3427 /// Whether any type-checking sanitizers are enabled. If \c false,
3428 /// calls to EmitTypeCheck can be skipped.
3429 bool sanitizePerformTypeCheck() const;
3430
3431 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV,
3432 QualType Type, SanitizerSet SkippedChecks = SanitizerSet(),
3433 llvm::Value *ArraySize = nullptr) {
3434 if (!sanitizePerformTypeCheck())
3435 return;
3436 EmitTypeCheck(TCK, Loc, V: LV.emitRawPointer(CGF&: *this), Type, Alignment: LV.getAlignment(),
3437 SkippedChecks, ArraySize);
3438 }
3439
3440 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, Address Addr,
3441 QualType Type, CharUnits Alignment = CharUnits::Zero(),
3442 SanitizerSet SkippedChecks = SanitizerSet(),
3443 llvm::Value *ArraySize = nullptr) {
3444 if (!sanitizePerformTypeCheck())
3445 return;
3446 EmitTypeCheck(TCK, Loc, V: Addr.emitRawPointer(CGF&: *this), Type, Alignment,
3447 SkippedChecks, ArraySize);
3448 }
3449
3450 /// Emit a check that \p V is the address of storage of the
3451 /// appropriate size and alignment for an object of type \p Type
3452 /// (or if ArraySize is provided, for an array of that bound).
3453 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Value *V,
3454 QualType Type, CharUnits Alignment = CharUnits::Zero(),
3455 SanitizerSet SkippedChecks = SanitizerSet(),
3456 llvm::Value *ArraySize = nullptr);
3457
3458 /// Emit a check that \p Base points into an array object, which
3459 /// we can access at index \p Index. \p Accessed should be \c false if we
3460 /// this expression is used as an lvalue, for instance in "&Arr[Idx]".
3461 void EmitBoundsCheck(const Expr *ArrayExpr, const Expr *ArrayExprBase,
3462 llvm::Value *Index, QualType IndexType, bool Accessed);
3463 void EmitBoundsCheckImpl(const Expr *ArrayExpr, QualType ArrayBaseType,
3464 llvm::Value *IndexVal, QualType IndexType,
3465 llvm::Value *BoundsVal, QualType BoundsType,
3466 bool Accessed);
3467
3468 /// Returns debug info, with additional annotation if
3469 /// CGM.getCodeGenOpts().SanitizeAnnotateDebugInfo[Ordinal] is enabled for
3470 /// any of the ordinals.
3471 llvm::DILocation *
3472 SanitizerAnnotateDebugInfo(ArrayRef<SanitizerKind::SanitizerOrdinal> Ordinals,
3473 SanitizerHandler Handler);
3474
3475 /// Build metadata used by the AllocToken instrumentation.
3476 llvm::MDNode *buildAllocToken(QualType AllocType);
3477 /// Emit and set additional metadata used by the AllocToken instrumentation.
3478 void EmitAllocToken(llvm::CallBase *CB, QualType AllocType);
3479 /// Build additional metadata used by the AllocToken instrumentation,
3480 /// inferring the type from an allocation call expression.
3481 llvm::MDNode *buildAllocToken(const CallExpr *E);
3482 /// Emit and set additional metadata used by the AllocToken instrumentation,
3483 /// inferring the type from an allocation call expression.
3484 void EmitAllocToken(llvm::CallBase *CB, const CallExpr *E);
3485
3486 llvm::Value *GetCountedByFieldExprGEP(const Expr *Base, const FieldDecl *FD,
3487 const FieldDecl *CountDecl);
3488
3489 /// Build an expression accessing the "counted_by" field.
3490 llvm::Value *EmitLoadOfCountedByField(const Expr *Base, const FieldDecl *FD,
3491 const FieldDecl *CountDecl);
3492
3493 // Emit bounds checking for flexible array and pointer members with the
3494 // counted_by attribute.
3495 void EmitCountedByBoundsChecking(const Expr *ArrayExpr, QualType ArrayType,
3496 Address ArrayInst, QualType IndexType,
3497 llvm::Value *IndexVal, bool Accessed,
3498 bool FlexibleArray);
3499
3500 llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
3501 bool isInc, bool isPre);
3502 ComplexPairTy EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
3503 bool isInc, bool isPre);
3504
3505 /// Converts Location to a DebugLoc, if debug information is enabled.
3506 llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location);
3507
3508 //===--------------------------------------------------------------------===//
3509 // Declaration Emission
3510 //===--------------------------------------------------------------------===//
3511
3512 /// EmitDecl - Emit a declaration.
3513 ///
3514 /// This function can be called with a null (unreachable) insert point.
3515 void EmitDecl(const Decl &D, bool EvaluateConditionDecl = false);
3516
3517 /// EmitVarDecl - Emit a local variable declaration.
3518 ///
3519 /// This function can be called with a null (unreachable) insert point.
3520 void EmitVarDecl(const VarDecl &D);
3521
3522 void EmitScalarInit(const Expr *init, const ValueDecl *D, LValue lvalue,
3523 bool capturedByInit);
3524
3525 typedef void SpecialInitFn(CodeGenFunction &Init, const VarDecl &D,
3526 llvm::Value *Address);
3527
3528 /// Determine whether the given initializer is trivial in the sense
3529 /// that it requires no code to be generated.
3530 bool isTrivialInitializer(const Expr *Init);
3531
3532 /// EmitAutoVarDecl - Emit an auto variable declaration.
3533 ///
3534 /// This function can be called with a null (unreachable) insert point.
3535 void EmitAutoVarDecl(const VarDecl &D);
3536
3537 class AutoVarEmission {
3538 friend class CodeGenFunction;
3539
3540 const VarDecl *Variable;
3541
3542 /// The address of the alloca for languages with explicit address space
3543 /// (e.g. OpenCL) or alloca casted to generic pointer for address space
3544 /// agnostic languages (e.g. C++). Invalid if the variable was emitted
3545 /// as a global constant.
3546 Address Addr;
3547
3548 llvm::Value *NRVOFlag;
3549
3550 /// True if the variable is a __block variable that is captured by an
3551 /// escaping block.
3552 bool IsEscapingByRef;
3553
3554 /// If the variable is of aggregate type and has a constant initializer,
3555 /// a constant representing that initializer.
3556 llvm::Constant *ConstantAggregateInitializer;
3557
3558 /// True if lifetime markers should be used.
3559 bool UseLifetimeMarkers;
3560
3561 /// Address with original alloca instruction. Invalid if the variable was
3562 /// emitted as a global constant.
3563 RawAddress AllocaAddr;
3564
3565 struct Invalid {};
3566 AutoVarEmission(Invalid)
3567 : Variable(nullptr), Addr(Address::invalid()),
3568 AllocaAddr(RawAddress::invalid()) {}
3569
3570 AutoVarEmission(const VarDecl &variable)
3571 : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr),
3572 IsEscapingByRef(false), ConstantAggregateInitializer(nullptr),
3573 UseLifetimeMarkers(false), AllocaAddr(RawAddress::invalid()) {}
3574
3575 bool wasEmittedAsGlobal() const { return !Addr.isValid(); }
3576
3577 public:
3578 static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }
3579
3580 bool useLifetimeMarkers() const { return UseLifetimeMarkers; }
3581
3582 /// Returns the raw, allocated address, which is not necessarily
3583 /// the address of the object itself. It is casted to default
3584 /// address space for address space agnostic languages.
3585 Address getAllocatedAddress() const { return Addr; }
3586
3587 /// Returns the address for the original alloca instruction.
3588 RawAddress getOriginalAllocatedAddress() const { return AllocaAddr; }
3589
3590 /// Returns the address of the object within this declaration.
3591 /// Note that this does not chase the forwarding pointer for
3592 /// __block decls.
3593 Address getObjectAddress(CodeGenFunction &CGF) const {
3594 if (!IsEscapingByRef)
3595 return Addr;
3596
3597 return CGF.emitBlockByrefAddress(baseAddr: Addr, V: Variable, /*forward*/ followForward: false);
3598 }
3599 };
3600 AutoVarEmission EmitAutoVarAlloca(const VarDecl &var);
3601 void EmitAutoVarInit(const AutoVarEmission &emission);
3602 void EmitAutoVarCleanups(const AutoVarEmission &emission);
3603 void emitAutoVarTypeCleanup(const AutoVarEmission &emission,
3604 QualType::DestructionKind dtorKind);
3605
3606 /// Re-emit trivial-auto-var-init stores for variables bypassed by the jump
3607 /// Source. No-op in a function containing a computed goto, where jump sources
3608 /// are unknown and a single function-scope init is used instead.
3609 void emitBypassedVarInitsForSource(const Stmt *Source);
3610
3611 void MaybeEmitDeferredVarDeclInit(const VarDecl *var);
3612
3613 /// Emits the alloca and debug information for the size expressions for each
3614 /// dimension of an array. It registers the association of its (1-dimensional)
3615 /// QualTypes and size expression's debug node, so that CGDebugInfo can
3616 /// reference this node when creating the DISubrange object to describe the
3617 /// array types.
3618 void EmitAndRegisterVariableArrayDimensions(CGDebugInfo *DI, const VarDecl &D,
3619 bool EmitDebugInfo);
3620
3621 void EmitStaticVarDecl(const VarDecl &D,
3622 llvm::GlobalValue::LinkageTypes Linkage);
3623
3624 class ParamValue {
3625 union {
3626 Address Addr;
3627 llvm::Value *Value;
3628 };
3629
3630 bool IsIndirect;
3631
3632 ParamValue(llvm::Value *V) : Value(V), IsIndirect(false) {}
3633 ParamValue(Address A) : Addr(A), IsIndirect(true) {}
3634
3635 public:
3636 static ParamValue forDirect(llvm::Value *value) {
3637 return ParamValue(value);
3638 }
3639 static ParamValue forIndirect(Address addr) {
3640 assert(!addr.getAlignment().isZero());
3641 return ParamValue(addr);
3642 }
3643
3644 bool isIndirect() const { return IsIndirect; }
3645 llvm::Value *getAnyValue() const {
3646 if (!isIndirect())
3647 return Value;
3648 assert(!Addr.hasOffset() && "unexpected offset");
3649 return Addr.getBasePointer();
3650 }
3651
3652 llvm::Value *getDirectValue() const {
3653 assert(!isIndirect());
3654 return Value;
3655 }
3656
3657 Address getIndirectAddress() const {
3658 assert(isIndirect());
3659 return Addr;
3660 }
3661 };
3662
3663 /// EmitParmDecl - Emit a ParmVarDecl or an ImplicitParamDecl.
3664 void EmitParmDecl(const VarDecl &D, ParamValue Arg, unsigned ArgNo);
3665
3666 /// protectFromPeepholes - Protect a value that we're intending to
3667 /// store to the side, but which will probably be used later, from
3668 /// aggressive peepholing optimizations that might delete it.
3669 ///
3670 /// Pass the result to unprotectFromPeepholes to declare that
3671 /// protection is no longer required.
3672 ///
3673 /// There's no particular reason why this shouldn't apply to
3674 /// l-values, it's just that no existing peepholes work on pointers.
3675 PeepholeProtection protectFromPeepholes(RValue rvalue);
3676 void unprotectFromPeepholes(PeepholeProtection protection);
3677
3678 void emitAlignmentAssumptionCheck(llvm::Value *Ptr, QualType Ty,
3679 SourceLocation Loc,
3680 SourceLocation AssumptionLoc,
3681 llvm::Value *Alignment,
3682 llvm::Value *OffsetValue,
3683 llvm::Value *TheCheck,
3684 llvm::Instruction *Assumption);
3685
3686 void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty,
3687 SourceLocation Loc, SourceLocation AssumptionLoc,
3688 llvm::Value *Alignment,
3689 llvm::Value *OffsetValue = nullptr);
3690
3691 void emitAlignmentAssumption(llvm::Value *PtrValue, const Expr *E,
3692 SourceLocation AssumptionLoc,
3693 llvm::Value *Alignment,
3694 llvm::Value *OffsetValue = nullptr);
3695
3696 //===--------------------------------------------------------------------===//
3697 // Statement Emission
3698 //===--------------------------------------------------------------------===//
3699
3700 /// EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
3701 void EmitStopPoint(const Stmt *S);
3702
3703 /// EmitStmt - Emit the code for the statement \arg S. It is legal to call
3704 /// this function even if there is no current insertion point.
3705 ///
3706 /// This function may clear the current insertion point; callers should use
3707 /// EnsureInsertPoint if they wish to subsequently generate code without first
3708 /// calling EmitBlock, EmitBranch, or EmitStmt.
3709 void EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs = {});
3710
3711 /// EmitSimpleStmt - Try to emit a "simple" statement which does not
3712 /// necessarily require an insertion point or debug information; typically
3713 /// because the statement amounts to a jump or a container of other
3714 /// statements.
3715 ///
3716 /// \return True if the statement was handled.
3717 bool EmitSimpleStmt(const Stmt *S, ArrayRef<const Attr *> Attrs);
3718
3719 Address EmitCompoundStmt(const CompoundStmt &S, bool GetLast = false,
3720 AggValueSlot AVS = AggValueSlot::ignored());
3721 Address
3722 EmitCompoundStmtWithoutScope(const CompoundStmt &S, bool GetLast = false,
3723 AggValueSlot AVS = AggValueSlot::ignored());
3724
3725 /// EmitLabel - Emit the block for the given label. It is legal to call this
3726 /// function even if there is no current insertion point.
3727 void EmitLabel(const LabelDecl *D); // helper for EmitLabelStmt.
3728
3729 void EmitLabelStmt(const LabelStmt &S);
3730 void EmitAttributedStmt(const AttributedStmt &S);
3731 void EmitGotoStmt(const GotoStmt &S);
3732 void EmitIndirectGotoStmt(const IndirectGotoStmt &S);
3733 void EmitIfStmt(const IfStmt &S);
3734
3735 void EmitWhileStmt(const WhileStmt &S, ArrayRef<const Attr *> Attrs = {});
3736 void EmitDoStmt(const DoStmt &S, ArrayRef<const Attr *> Attrs = {});
3737 void EmitForStmt(const ForStmt &S, ArrayRef<const Attr *> Attrs = {});
3738 void EmitReturnStmt(const ReturnStmt &S);
3739 void EmitDeclStmt(const DeclStmt &S);
3740 void EmitBreakStmt(const BreakStmt &S);
3741 void EmitContinueStmt(const ContinueStmt &S);
3742 void EmitSwitchStmt(const SwitchStmt &S);
3743 void EmitDefaultStmt(const DefaultStmt &S, ArrayRef<const Attr *> Attrs);
3744 void EmitCaseStmt(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3745 void EmitCaseStmtRange(const CaseStmt &S, ArrayRef<const Attr *> Attrs);
3746 void EmitDeferStmt(const DeferStmt &S);
3747 void EmitAsmStmt(const AsmStmt &S);
3748
3749 const BreakContinue *GetDestForLoopControlStmt(const LoopControlStmt &S);
3750
3751 void EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S);
3752 void EmitObjCAtTryStmt(const ObjCAtTryStmt &S);
3753 void EmitObjCAtThrowStmt(const ObjCAtThrowStmt &S);
3754 void EmitObjCAtSynchronizedStmt(const ObjCAtSynchronizedStmt &S);
3755 void EmitObjCAutoreleasePoolStmt(const ObjCAutoreleasePoolStmt &S);
3756
3757 void EmitCoroutineBody(const CoroutineBodyStmt &S);
3758 void EmitCoreturnStmt(const CoreturnStmt &S);
3759 RValue EmitCoawaitExpr(const CoawaitExpr &E,
3760 AggValueSlot aggSlot = AggValueSlot::ignored(),
3761 bool ignoreResult = false);
3762 LValue EmitCoawaitLValue(const CoawaitExpr *E);
3763 RValue EmitCoyieldExpr(const CoyieldExpr &E,
3764 AggValueSlot aggSlot = AggValueSlot::ignored(),
3765 bool ignoreResult = false);
3766 LValue EmitCoyieldLValue(const CoyieldExpr *E);
3767 RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID);
3768
3769 void EmitSYCLKernelCallStmt(const SYCLKernelCallStmt &S);
3770
3771 void EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3772 void ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
3773
3774 void EmitCXXTryStmt(const CXXTryStmt &S);
3775 void EmitSEHTryStmt(const SEHTryStmt &S);
3776 void EmitSEHLeaveStmt(const SEHLeaveStmt &S);
3777 void EnterSEHTryStmt(const SEHTryStmt &S);
3778 void ExitSEHTryStmt(const SEHTryStmt &S);
3779 void VolatilizeTryBlocks(llvm::BasicBlock *BB,
3780 llvm::SmallPtrSet<llvm::BasicBlock *, 10> &V);
3781
3782 void pushSEHCleanup(CleanupKind kind, llvm::Function *FinallyFunc);
3783 void startOutlinedSEHHelper(CodeGenFunction &ParentCGF, bool IsFilter,
3784 const Stmt *OutlinedStmt);
3785
3786 llvm::Function *GenerateSEHFilterFunction(CodeGenFunction &ParentCGF,
3787 const SEHExceptStmt &Except);
3788
3789 llvm::Function *GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF,
3790 const SEHFinallyStmt &Finally);
3791
3792 void EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF,
3793 llvm::Value *ParentFP, llvm::Value *EntryEBP);
3794 llvm::Value *EmitSEHExceptionCode();
3795 llvm::Value *EmitSEHExceptionInfo();
3796 llvm::Value *EmitSEHAbnormalTermination();
3797
3798 /// Emit simple code for OpenMP directives in Simd-only mode.
3799 void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D);
3800
3801 /// Scan the outlined statement for captures from the parent function. For
3802 /// each capture, mark the capture as escaped and emit a call to
3803 /// llvm.localrecover. Insert the localrecover result into the LocalDeclMap.
3804 void EmitCapturedLocals(CodeGenFunction &ParentCGF, const Stmt *OutlinedStmt,
3805 bool IsFilter);
3806
3807 /// Recovers the address of a local in a parent function. ParentVar is the
3808 /// address of the variable used in the immediate parent function. It can
3809 /// either be an alloca or a call to llvm.localrecover if there are nested
3810 /// outlined functions. ParentFP is the frame pointer of the outermost parent
3811 /// frame.
3812 Address recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
3813 Address ParentVar, llvm::Value *ParentFP);
3814
3815 void EmitCXXForRangeStmt(const CXXForRangeStmt &S,
3816 ArrayRef<const Attr *> Attrs = {});
3817
3818 void
3819 EmitCXXExpansionStmtInstantiation(const CXXExpansionStmtInstantiation &S);
3820
3821 /// Controls insertion of cancellation exit blocks in worksharing constructs.
3822 class OMPCancelStackRAII {
3823 CodeGenFunction &CGF;
3824
3825 public:
3826 OMPCancelStackRAII(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,
3827 bool HasCancel)
3828 : CGF(CGF) {
3829 CGF.OMPCancelStack.enter(CGF, Kind, HasCancel);
3830 }
3831 ~OMPCancelStackRAII() { CGF.OMPCancelStack.exit(CGF); }
3832 };
3833
3834 /// Returns calculated size of the specified type.
3835 llvm::Value *getTypeSize(QualType Ty);
3836 LValue InitCapturedStruct(const CapturedStmt &S);
3837 llvm::Function *EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K);
3838 llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S);
3839 Address GenerateCapturedStmtArgument(const CapturedStmt &S);
3840 llvm::Function *
3841 GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S,
3842 const OMPExecutableDirective &D);
3843 llvm::Function *
3844 GenerateOpenMPCapturedStmtFunctionAggregate(const CapturedStmt &S,
3845 const OMPExecutableDirective &D);
3846 void GenerateOpenMPCapturedVars(const CapturedStmt &S,
3847 SmallVectorImpl<llvm::Value *> &CapturedVars);
3848 void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy,
3849 SourceLocation Loc);
3850 /// Perform element by element copying of arrays with type \a
3851 /// OriginalType from \a SrcAddr to \a DestAddr using copying procedure
3852 /// generated by \a CopyGen.
3853 ///
3854 /// \param DestAddr Address of the destination array.
3855 /// \param SrcAddr Address of the source array.
3856 /// \param OriginalType Type of destination and source arrays.
3857 /// \param CopyGen Copying procedure that copies value of single array element
3858 /// to another single array element.
3859 void EmitOMPAggregateAssign(
3860 Address DestAddr, Address SrcAddr, QualType OriginalType,
3861 const llvm::function_ref<void(Address, Address)> CopyGen);
3862 /// Emit proper copying of data from one variable to another.
3863 ///
3864 /// \param OriginalType Original type of the copied variables.
3865 /// \param DestAddr Destination address.
3866 /// \param SrcAddr Source address.
3867 /// \param DestVD Destination variable used in \a CopyExpr (for arrays, has
3868 /// type of the base array element).
3869 /// \param SrcVD Source variable used in \a CopyExpr (for arrays, has type of
3870 /// the base array element).
3871 /// \param Copy Actual copygin expression for copying data from \a SrcVD to \a
3872 /// DestVD.
3873 void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr,
3874 const VarDecl *DestVD, const VarDecl *SrcVD,
3875 const Expr *Copy);
3876 /// Emit atomic update code for constructs: \a X = \a X \a BO \a E or
3877 /// \a X = \a E \a BO \a E.
3878 ///
3879 /// \param X Value to be updated.
3880 /// \param E Update value.
3881 /// \param BO Binary operation for update operation.
3882 /// \param IsXLHSInRHSPart true if \a X is LHS in RHS part of the update
3883 /// expression, false otherwise.
3884 /// \param AO Atomic ordering of the generated atomic instructions.
3885 /// \param CommonGen Code generator for complex expressions that cannot be
3886 /// expressed through atomicrmw instruction.
3887 /// \returns <true, OldAtomicValue> if simple 'atomicrmw' instruction was
3888 /// generated, <false, RValue::get(nullptr)> otherwise.
3889 std::pair<bool, RValue> EmitOMPAtomicSimpleUpdateExpr(
3890 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3891 llvm::AtomicOrdering AO, SourceLocation Loc,
3892 const llvm::function_ref<RValue(RValue)> CommonGen);
3893 bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
3894 OMPPrivateScope &PrivateScope);
3895 void EmitOMPPrivateClause(const OMPExecutableDirective &D,
3896 OMPPrivateScope &PrivateScope);
3897 void EmitOMPUseDevicePtrClause(
3898 const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
3899 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3900 CaptureDeviceAddrMap);
3901 void EmitOMPUseDeviceAddrClause(
3902 const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
3903 const llvm::DenseMap<const ValueDecl *, llvm::Value *>
3904 CaptureDeviceAddrMap);
3905 /// Emit code for copyin clause in \a D directive. The next code is
3906 /// generated at the start of outlined functions for directives:
3907 /// \code
3908 /// threadprivate_var1 = master_threadprivate_var1;
3909 /// operator=(threadprivate_var2, master_threadprivate_var2);
3910 /// ...
3911 /// __kmpc_barrier(&loc, global_tid);
3912 /// \endcode
3913 ///
3914 /// \param D OpenMP directive possibly with 'copyin' clause(s).
3915 /// \returns true if at least one copyin variable is found, false otherwise.
3916 bool EmitOMPCopyinClause(const OMPExecutableDirective &D);
3917 /// Emit initial code for lastprivate variables. If some variable is
3918 /// not also firstprivate, then the default initialization is used. Otherwise
3919 /// initialization of this variable is performed by EmitOMPFirstprivateClause
3920 /// method.
3921 ///
3922 /// \param D Directive that may have 'lastprivate' directives.
3923 /// \param PrivateScope Private scope for capturing lastprivate variables for
3924 /// proper codegen in internal captured statement.
3925 ///
3926 /// \returns true if there is at least one lastprivate variable, false
3927 /// otherwise.
3928 bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D,
3929 OMPPrivateScope &PrivateScope);
3930 /// Emit final copying of lastprivate values to original variables at
3931 /// the end of the worksharing or simd directive.
3932 ///
3933 /// \param D Directive that has at least one 'lastprivate' directives.
3934 /// \param IsLastIterCond Boolean condition that must be set to 'i1 true' if
3935 /// it is the last iteration of the loop code in associated directive, or to
3936 /// 'i1 false' otherwise. If this item is nullptr, no final check is required.
3937 void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D,
3938 bool NoFinals,
3939 llvm::Value *IsLastIterCond = nullptr);
3940 /// Emit initial code for linear clauses.
3941 void EmitOMPLinearClause(const OMPLoopDirective &D,
3942 CodeGenFunction::OMPPrivateScope &PrivateScope);
3943 /// Emit final code for linear clauses.
3944 /// \param CondGen Optional conditional code for final part of codegen for
3945 /// linear clause.
3946 void EmitOMPLinearClauseFinal(
3947 const OMPLoopDirective &D,
3948 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
3949 /// Emit initial code for reduction variables. Creates reduction copies
3950 /// and initializes them with the values according to OpenMP standard.
3951 ///
3952 /// \param D Directive (possibly) with the 'reduction' clause.
3953 /// \param PrivateScope Private scope for capturing reduction variables for
3954 /// proper codegen in internal captured statement.
3955 ///
3956 void EmitOMPReductionClauseInit(const OMPExecutableDirective &D,
3957 OMPPrivateScope &PrivateScope,
3958 bool ForInscan = false);
3959 /// Emit final update of reduction values to original variables at
3960 /// the end of the directive.
3961 ///
3962 /// \param D Directive that has at least one 'reduction' directives.
3963 /// \param ReductionKind The kind of reduction to perform.
3964 void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D,
3965 const OpenMPDirectiveKind ReductionKind);
3966 /// Emit initial code for linear variables. Creates private copies
3967 /// and initializes them with the values according to OpenMP standard.
3968 ///
3969 /// \param D Directive (possibly) with the 'linear' clause.
3970 /// \return true if at least one linear variable is found that should be
3971 /// initialized with the value of the original variable, false otherwise.
3972 bool EmitOMPLinearClauseInit(const OMPLoopDirective &D);
3973
3974 typedef const llvm::function_ref<void(CodeGenFunction & /*CGF*/,
3975 llvm::Function * /*OutlinedFn*/,
3976 const OMPTaskDataTy & /*Data*/)>
3977 TaskGenTy;
3978 void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S,
3979 const OpenMPDirectiveKind CapturedRegion,
3980 const RegionCodeGenTy &BodyGen,
3981 const TaskGenTy &TaskGen, OMPTaskDataTy &Data);
3982 struct OMPTargetDataInfo {
3983 Address BasePointersArray = Address::invalid();
3984 Address PointersArray = Address::invalid();
3985 Address SizesArray = Address::invalid();
3986 Address MappersArray = Address::invalid();
3987 unsigned NumberOfTargetItems = 0;
3988 explicit OMPTargetDataInfo() = default;
3989 OMPTargetDataInfo(Address BasePointersArray, Address PointersArray,
3990 Address SizesArray, Address MappersArray,
3991 unsigned NumberOfTargetItems)
3992 : BasePointersArray(BasePointersArray), PointersArray(PointersArray),
3993 SizesArray(SizesArray), MappersArray(MappersArray),
3994 NumberOfTargetItems(NumberOfTargetItems) {}
3995 };
3996 void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S,
3997 const RegionCodeGenTy &BodyGen,
3998 OMPTargetDataInfo &InputInfo);
3999 void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data,
4000 CodeGenFunction &CGF, const CapturedStmt *CS,
4001 OMPPrivateScope &Scope);
4002 void EmitOMPMetaDirective(const OMPMetaDirective &S);
4003 void EmitOMPParallelDirective(const OMPParallelDirective &S);
4004 void EmitOMPSimdDirective(const OMPSimdDirective &S);
4005 void EmitOMPTileDirective(const OMPTileDirective &S);
4006 void EmitOMPStripeDirective(const OMPStripeDirective &S);
4007 void EmitOMPUnrollDirective(const OMPUnrollDirective &S);
4008 void EmitOMPReverseDirective(const OMPReverseDirective &S);
4009 void EmitOMPSplitDirective(const OMPSplitDirective &S);
4010 void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
4011 void EmitOMPFlattenDirective(const OMPFlattenDirective &S);
4012 void EmitOMPFuseDirective(const OMPFuseDirective &S);
4013 void EmitOMPForDirective(const OMPForDirective &S);
4014 void EmitOMPForSimdDirective(const OMPForSimdDirective &S);
4015 void EmitOMPScopeDirective(const OMPScopeDirective &S);
4016 void EmitOMPSectionsDirective(const OMPSectionsDirective &S);
4017 void EmitOMPSectionDirective(const OMPSectionDirective &S);
4018 void EmitOMPSingleDirective(const OMPSingleDirective &S);
4019 void EmitOMPMasterDirective(const OMPMasterDirective &S);
4020 void EmitOMPMaskedDirective(const OMPMaskedDirective &S);
4021 void EmitOMPCriticalDirective(const OMPCriticalDirective &S);
4022 void EmitOMPParallelForDirective(const OMPParallelForDirective &S);
4023 void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S);
4024 void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S);
4025 void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S);
4026 void EmitOMPTaskDirective(const OMPTaskDirective &S);
4027 void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S);
4028 void EmitOMPErrorDirective(const OMPErrorDirective &S);
4029 void EmitOMPBarrierDirective(const OMPBarrierDirective &S);
4030 void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S);
4031 void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S);
4032 void EmitOMPFlushDirective(const OMPFlushDirective &S);
4033 void EmitOMPDepobjDirective(const OMPDepobjDirective &S);
4034 void EmitOMPScanDirective(const OMPScanDirective &S);
4035 void
4036 EmitOMPOrderedStandaloneDirective(const OMPOrderedStandaloneDirective &S);
4037 void
4038 EmitOMPOrderedBlockAssocDirective(const OMPOrderedBlockAssocDirective &S);
4039 void EmitOMPAtomicDirective(const OMPAtomicDirective &S);
4040 void EmitOMPTargetDirective(const OMPTargetDirective &S);
4041 void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S);
4042 void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S);
4043 void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S);
4044 void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S);
4045 void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S);
4046 void
4047 EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S);
4048 void EmitOMPTeamsDirective(const OMPTeamsDirective &S);
4049 void
4050 EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S);
4051 void EmitOMPCancelDirective(const OMPCancelDirective &S);
4052 void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S);
4053 void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S);
4054 void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S);
4055 void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S);
4056 void EmitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &S);
4057 void
4058 EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S);
4059 void
4060 EmitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &S);
4061 void EmitOMPParallelMasterTaskLoopDirective(
4062 const OMPParallelMasterTaskLoopDirective &S);
4063 void EmitOMPParallelMaskedTaskLoopDirective(
4064 const OMPParallelMaskedTaskLoopDirective &S);
4065 void EmitOMPParallelMasterTaskLoopSimdDirective(
4066 const OMPParallelMasterTaskLoopSimdDirective &S);
4067 void EmitOMPParallelMaskedTaskLoopSimdDirective(
4068 const OMPParallelMaskedTaskLoopSimdDirective &S);
4069 void EmitOMPDistributeDirective(const OMPDistributeDirective &S);
4070 void EmitOMPDistributeParallelForDirective(
4071 const OMPDistributeParallelForDirective &S);
4072 void EmitOMPDistributeParallelForSimdDirective(
4073 const OMPDistributeParallelForSimdDirective &S);
4074 void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S);
4075 void EmitOMPTargetParallelForSimdDirective(
4076 const OMPTargetParallelForSimdDirective &S);
4077 void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S);
4078 void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S);
4079 void
4080 EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S);
4081 void EmitOMPTeamsDistributeParallelForSimdDirective(
4082 const OMPTeamsDistributeParallelForSimdDirective &S);
4083 void EmitOMPTeamsDistributeParallelForDirective(
4084 const OMPTeamsDistributeParallelForDirective &S);
4085 void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S);
4086 void EmitOMPTargetTeamsDistributeDirective(
4087 const OMPTargetTeamsDistributeDirective &S);
4088 void EmitOMPTargetTeamsDistributeParallelForDirective(
4089 const OMPTargetTeamsDistributeParallelForDirective &S);
4090 void EmitOMPTargetTeamsDistributeParallelForSimdDirective(
4091 const OMPTargetTeamsDistributeParallelForSimdDirective &S);
4092 void EmitOMPTargetTeamsDistributeSimdDirective(
4093 const OMPTargetTeamsDistributeSimdDirective &S);
4094 void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S);
4095 void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S);
4096 void EmitOMPTargetParallelGenericLoopDirective(
4097 const OMPTargetParallelGenericLoopDirective &S);
4098 void EmitOMPTargetTeamsGenericLoopDirective(
4099 const OMPTargetTeamsGenericLoopDirective &S);
4100 void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S);
4101 void EmitOMPInteropDirective(const OMPInteropDirective &S);
4102 void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S);
4103 void EmitOMPAssumeDirective(const OMPAssumeDirective &S);
4104
4105 /// Emit device code for the target directive.
4106 static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
4107 StringRef ParentName,
4108 const OMPTargetDirective &S);
4109 static void
4110 EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
4111 const OMPTargetParallelDirective &S);
4112 /// Emit device code for the target parallel for directive.
4113 static void EmitOMPTargetParallelForDeviceFunction(
4114 CodeGenModule &CGM, StringRef ParentName,
4115 const OMPTargetParallelForDirective &S);
4116 /// Emit device code for the target parallel for simd directive.
4117 static void EmitOMPTargetParallelForSimdDeviceFunction(
4118 CodeGenModule &CGM, StringRef ParentName,
4119 const OMPTargetParallelForSimdDirective &S);
4120 /// Emit device code for the target teams directive.
4121 static void
4122 EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName,
4123 const OMPTargetTeamsDirective &S);
4124 /// Emit device code for the target teams distribute directive.
4125 static void EmitOMPTargetTeamsDistributeDeviceFunction(
4126 CodeGenModule &CGM, StringRef ParentName,
4127 const OMPTargetTeamsDistributeDirective &S);
4128 /// Emit device code for the target teams distribute simd directive.
4129 static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(
4130 CodeGenModule &CGM, StringRef ParentName,
4131 const OMPTargetTeamsDistributeSimdDirective &S);
4132 /// Emit device code for the target simd directive.
4133 static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM,
4134 StringRef ParentName,
4135 const OMPTargetSimdDirective &S);
4136 /// Emit device code for the target teams distribute parallel for simd
4137 /// directive.
4138 static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
4139 CodeGenModule &CGM, StringRef ParentName,
4140 const OMPTargetTeamsDistributeParallelForSimdDirective &S);
4141
4142 /// Emit device code for the target teams loop directive.
4143 static void EmitOMPTargetTeamsGenericLoopDeviceFunction(
4144 CodeGenModule &CGM, StringRef ParentName,
4145 const OMPTargetTeamsGenericLoopDirective &S);
4146
4147 /// Emit device code for the target parallel loop directive.
4148 static void EmitOMPTargetParallelGenericLoopDeviceFunction(
4149 CodeGenModule &CGM, StringRef ParentName,
4150 const OMPTargetParallelGenericLoopDirective &S);
4151
4152 static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
4153 CodeGenModule &CGM, StringRef ParentName,
4154 const OMPTargetTeamsDistributeParallelForDirective &S);
4155
4156 /// Emit the Stmt \p S and return its topmost canonical loop, if any.
4157 /// TODO: The \p Depth paramter is not yet implemented and must be 1. In the
4158 /// future it is meant to be the number of loops expected in the loop nests
4159 /// (usually specified by the "collapse" clause) that are collapsed to a
4160 /// single loop by this function.
4161 llvm::CanonicalLoopInfo *EmitOMPCollapsedCanonicalLoopNest(const Stmt *S,
4162 int Depth);
4163
4164 /// Emit an OMPCanonicalLoop using the OpenMPIRBuilder.
4165 void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S);
4166
4167 /// Emit inner loop of the worksharing/simd construct.
4168 ///
4169 /// \param S Directive, for which the inner loop must be emitted.
4170 /// \param RequiresCleanup true, if directive has some associated private
4171 /// variables.
4172 /// \param LoopCond Bollean condition for loop continuation.
4173 /// \param IncExpr Increment expression for loop control variable.
4174 /// \param BodyGen Generator for the inner body of the inner loop.
4175 /// \param PostIncGen Genrator for post-increment code (required for ordered
4176 /// loop directvies).
4177 void EmitOMPInnerLoop(
4178 const OMPExecutableDirective &S, bool RequiresCleanup,
4179 const Expr *LoopCond, const Expr *IncExpr,
4180 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
4181 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen);
4182
4183 JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind);
4184 /// Emit initial code for loop counters of loop-based directives.
4185 void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S,
4186 OMPPrivateScope &LoopScope);
4187
4188 /// Helper for the OpenMP loop directives.
4189 void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit);
4190
4191 /// Emit code for the worksharing loop-based directive.
4192 /// \return true, if this construct has any lastprivate clause, false -
4193 /// otherwise.
4194 bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB,
4195 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
4196 const CodeGenDispatchBoundsTy &CGDispatchBounds);
4197
4198 /// Emit code for the distribute loop-based directive.
4199 void EmitOMPDistributeLoop(const OMPLoopDirective &S,
4200 const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr);
4201
4202 /// Helpers for the OpenMP loop directives.
4203 void EmitOMPSimdInit(const OMPLoopDirective &D);
4204 void EmitOMPSimdFinal(
4205 const OMPLoopDirective &D,
4206 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);
4207
4208 /// Emits the lvalue for the expression with possibly captured variable.
4209 LValue EmitOMPSharedLValue(const Expr *E);
4210
4211 /// Emits the original address for a structured binding.
4212 Address EmitOMPBindingOriginalAddr(const BindingDecl *BD, SourceLocation Loc);
4213
4214private:
4215 /// Helpers for blocks.
4216 llvm::Value *EmitBlockLiteral(const CGBlockInfo &Info);
4217
4218 /// struct with the values to be passed to the OpenMP loop-related functions
4219 struct OMPLoopArguments {
4220 /// loop lower bound
4221 Address LB = Address::invalid();
4222 /// loop upper bound
4223 Address UB = Address::invalid();
4224 /// loop stride
4225 Address ST = Address::invalid();
4226 /// isLastIteration argument for runtime functions
4227 Address IL = Address::invalid();
4228 /// Chunk value generated by sema
4229 llvm::Value *Chunk = nullptr;
4230 /// EnsureUpperBound
4231 Expr *EUB = nullptr;
4232 /// IncrementExpression
4233 Expr *IncExpr = nullptr;
4234 /// Loop initialization
4235 Expr *Init = nullptr;
4236 /// Loop exit condition
4237 Expr *Cond = nullptr;
4238 /// Update of LB after a whole chunk has been executed
4239 Expr *NextLB = nullptr;
4240 /// Update of UB after a whole chunk has been executed
4241 Expr *NextUB = nullptr;
4242 /// Distinguish between the for distribute and sections
4243 OpenMPDirectiveKind DKind = llvm::omp::OMPD_unknown;
4244 OMPLoopArguments() = default;
4245 OMPLoopArguments(Address LB, Address UB, Address ST, Address IL,
4246 llvm::Value *Chunk = nullptr, Expr *EUB = nullptr,
4247 Expr *IncExpr = nullptr, Expr *Init = nullptr,
4248 Expr *Cond = nullptr, Expr *NextLB = nullptr,
4249 Expr *NextUB = nullptr)
4250 : LB(LB), UB(UB), ST(ST), IL(IL), Chunk(Chunk), EUB(EUB),
4251 IncExpr(IncExpr), Init(Init), Cond(Cond), NextLB(NextLB),
4252 NextUB(NextUB) {}
4253 };
4254 void EmitOMPOuterLoop(bool DynamicOrOrdered, bool IsMonotonic,
4255 const OMPLoopDirective &S, OMPPrivateScope &LoopScope,
4256 const OMPLoopArguments &LoopArgs,
4257 const CodeGenLoopTy &CodeGenLoop,
4258 const CodeGenOrderedTy &CodeGenOrdered);
4259 void EmitOMPForOuterLoop(const OpenMPScheduleTy &ScheduleKind,
4260 bool IsMonotonic, const OMPLoopDirective &S,
4261 OMPPrivateScope &LoopScope, bool Ordered,
4262 const OMPLoopArguments &LoopArgs,
4263 const CodeGenDispatchBoundsTy &CGDispatchBounds);
4264 void EmitOMPDistributeOuterLoop(OpenMPDistScheduleClauseKind ScheduleKind,
4265 const OMPLoopDirective &S,
4266 OMPPrivateScope &LoopScope,
4267 const OMPLoopArguments &LoopArgs,
4268 const CodeGenLoopTy &CodeGenLoopContent);
4269 /// Emit code for sections directive.
4270 void EmitSections(const OMPExecutableDirective &S);
4271
4272public:
4273 //===--------------------------------------------------------------------===//
4274 // OpenACC Emission
4275 //===--------------------------------------------------------------------===//
4276 void EmitOpenACCComputeConstruct(const OpenACCComputeConstruct &S) {
4277 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4278 // simply emitting its structured block, but in the future we will implement
4279 // some sort of IR.
4280 if (S.getStructuredBlock())
4281 EmitStmt(S: S.getStructuredBlock());
4282 }
4283
4284 void EmitOpenACCLoopConstruct(const OpenACCLoopConstruct &S) {
4285 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4286 // simply emitting its loop, but in the future we will implement
4287 // some sort of IR.
4288 if (S.getLoop())
4289 EmitStmt(S: S.getLoop());
4290 }
4291
4292 void EmitOpenACCCombinedConstruct(const OpenACCCombinedConstruct &S) {
4293 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4294 // simply emitting its loop, but in the future we will implement
4295 // some sort of IR.
4296 if (S.getLoop())
4297 EmitStmt(S: S.getLoop());
4298 }
4299
4300 void EmitOpenACCDataConstruct(const OpenACCDataConstruct &S) {
4301 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4302 // simply emitting its structured block, but in the future we will implement
4303 // some sort of IR.
4304 if (S.getStructuredBlock())
4305 EmitStmt(S: S.getStructuredBlock());
4306 }
4307
4308 void EmitOpenACCEnterDataConstruct(const OpenACCEnterDataConstruct &S) {
4309 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4310 // but in the future we will implement some sort of IR.
4311 }
4312
4313 void EmitOpenACCExitDataConstruct(const OpenACCExitDataConstruct &S) {
4314 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4315 // but in the future we will implement some sort of IR.
4316 }
4317
4318 void EmitOpenACCHostDataConstruct(const OpenACCHostDataConstruct &S) {
4319 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4320 // simply emitting its structured block, but in the future we will implement
4321 // some sort of IR.
4322 if (S.getStructuredBlock())
4323 EmitStmt(S: S.getStructuredBlock());
4324 }
4325
4326 void EmitOpenACCWaitConstruct(const OpenACCWaitConstruct &S) {
4327 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4328 // but in the future we will implement some sort of IR.
4329 }
4330
4331 void EmitOpenACCInitConstruct(const OpenACCInitConstruct &S) {
4332 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4333 // but in the future we will implement some sort of IR.
4334 }
4335
4336 void EmitOpenACCShutdownConstruct(const OpenACCShutdownConstruct &S) {
4337 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4338 // but in the future we will implement some sort of IR.
4339 }
4340
4341 void EmitOpenACCSetConstruct(const OpenACCSetConstruct &S) {
4342 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4343 // but in the future we will implement some sort of IR.
4344 }
4345
4346 void EmitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &S) {
4347 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4348 // but in the future we will implement some sort of IR.
4349 }
4350
4351 void EmitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &S) {
4352 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4353 // simply emitting its associated stmt, but in the future we will implement
4354 // some sort of IR.
4355 if (S.getAssociatedStmt())
4356 EmitStmt(S: S.getAssociatedStmt());
4357 }
4358 void EmitOpenACCCacheConstruct(const OpenACCCacheConstruct &S) {
4359 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',
4360 // but in the future we will implement some sort of IR.
4361 }
4362
4363 //===--------------------------------------------------------------------===//
4364 // LValue Expression Emission
4365 //===--------------------------------------------------------------------===//
4366
4367 /// Create a check that a scalar RValue is non-null.
4368 llvm::Value *EmitNonNullRValueCheck(RValue RV, QualType T);
4369
4370 /// GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
4371 RValue GetUndefRValue(QualType Ty);
4372
4373 /// EmitUnsupportedRValue - Emit a dummy r-value using the type of E
4374 /// and issue an ErrorUnsupported style diagnostic (using the
4375 /// provided Name).
4376 RValue EmitUnsupportedRValue(const Expr *E, const char *Name);
4377
4378 /// EmitUnsupportedLValue - Emit a dummy l-value using the type of E and issue
4379 /// an ErrorUnsupported style diagnostic (using the provided Name).
4380 LValue EmitUnsupportedLValue(const Expr *E, const char *Name);
4381
4382 /// EmitLValue - Emit code to compute a designator that specifies the location
4383 /// of the expression.
4384 ///
4385 /// This can return one of two things: a simple address or a bitfield
4386 /// reference. In either case, the LLVM Value* in the LValue structure is
4387 /// guaranteed to be an LLVM pointer type.
4388 ///
4389 /// If this returns a bitfield reference, nothing about the pointee type of
4390 /// the LLVM value is known: For example, it may not be a pointer to an
4391 /// integer.
4392 ///
4393 /// If this returns a normal address, and if the lvalue's C type is fixed
4394 /// size, this method guarantees that the returned pointer type will point to
4395 /// an LLVM type of the same size of the lvalue's type. If the lvalue has a
4396 /// variable length type, this is not possible.
4397 ///
4398 LValue EmitLValue(const Expr *E,
4399 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
4400
4401private:
4402 LValue EmitLValueHelper(const Expr *E, KnownNonNull_t IsKnownNonNull);
4403
4404public:
4405 /// Same as EmitLValue but additionally we generate checking code to
4406 /// guard against undefined behavior. This is only suitable when we know
4407 /// that the address will be used to access the object.
4408 LValue EmitCheckedLValue(const Expr *E, TypeCheckKind TCK);
4409
4410 RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc);
4411
4412 void EmitAtomicInit(Expr *E, LValue lvalue);
4413
4414 bool LValueIsSuitableForInlineAtomic(LValue Src);
4415
4416 RValue EmitAtomicLoad(LValue LV, SourceLocation SL,
4417 AggValueSlot Slot = AggValueSlot::ignored());
4418
4419 RValue EmitAtomicLoad(LValue lvalue, SourceLocation loc,
4420 llvm::AtomicOrdering AO, bool IsVolatile = false,
4421 AggValueSlot slot = AggValueSlot::ignored());
4422
4423 void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit);
4424
4425 void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO,
4426 bool IsVolatile, bool isInit);
4427
4428 std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
4429 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
4430 llvm::AtomicOrdering Success =
4431 llvm::AtomicOrdering::SequentiallyConsistent,
4432 llvm::AtomicOrdering Failure =
4433 llvm::AtomicOrdering::SequentiallyConsistent,
4434 bool IsWeak = false, AggValueSlot Slot = AggValueSlot::ignored());
4435
4436 /// Emit an atomicrmw instruction, and applying relevant metadata when
4437 /// applicable.
4438 llvm::AtomicRMWInst *emitAtomicRMWInst(
4439 llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val,
4440 llvm::AtomicOrdering Order = llvm::AtomicOrdering::SequentiallyConsistent,
4441 llvm::SyncScope::ID SSID = llvm::SyncScope::System,
4442 const AtomicExpr *AE = nullptr);
4443
4444 /// Emit a fence instruction, applying relevant target-specific metadata when
4445 /// applicable.
4446 llvm::FenceInst *
4447 emitAtomicFence(llvm::AtomicOrdering Order,
4448 llvm::SyncScope::ID SSID = llvm::SyncScope::System);
4449
4450 void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO,
4451 const llvm::function_ref<RValue(RValue)> &UpdateOp,
4452 bool IsVolatile);
4453
4454 /// EmitToMemory - Change a scalar value from its value
4455 /// representation to its in-memory representation.
4456 llvm::Value *EmitToMemory(llvm::Value *Value, QualType Ty);
4457
4458 /// EmitFromMemory - Change a scalar value from its memory
4459 /// representation to its value representation.
4460 llvm::Value *EmitFromMemory(llvm::Value *Value, QualType Ty);
4461
4462 /// Check if the scalar \p Value is within the valid range for the given
4463 /// type \p Ty.
4464 ///
4465 /// Returns true if a check is needed (even if the range is unknown).
4466 bool EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
4467 SourceLocation Loc);
4468
4469 /// EmitLoadOfScalar - Load a scalar value from an address, taking
4470 /// care to appropriately convert from the memory representation to
4471 /// the LLVM value representation.
4472 llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4473 SourceLocation Loc,
4474 AlignmentSource Source = AlignmentSource::Type,
4475 bool isNontemporal = false) {
4476 return EmitLoadOfScalar(Addr, Volatile, Ty, Loc, BaseInfo: LValueBaseInfo(Source),
4477 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isNontemporal);
4478 }
4479
4480 llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,
4481 SourceLocation Loc, LValueBaseInfo BaseInfo,
4482 TBAAAccessInfo TBAAInfo,
4483 bool isNontemporal = false);
4484
4485 /// EmitLoadOfScalar - Load a scalar value from an address, taking
4486 /// care to appropriately convert from the memory representation to
4487 /// the LLVM value representation. The l-value must be a simple
4488 /// l-value.
4489 llvm::Value *EmitLoadOfScalar(LValue lvalue, SourceLocation Loc);
4490
4491 /// EmitStoreOfScalar - Store a scalar value to an address, taking
4492 /// care to appropriately convert from the memory representation to
4493 /// the LLVM value representation.
4494 void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,
4495 QualType Ty,
4496 AlignmentSource Source = AlignmentSource::Type,
4497 bool isInit = false, bool isNontemporal = false) {
4498 EmitStoreOfScalar(Value, Addr, Volatile, Ty, BaseInfo: LValueBaseInfo(Source),
4499 TBAAInfo: CGM.getTBAAAccessInfo(AccessType: Ty), isInit, isNontemporal);
4500 }
4501
4502 void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,
4503 QualType Ty, LValueBaseInfo BaseInfo,
4504 TBAAAccessInfo TBAAInfo, bool isInit = false,
4505 bool isNontemporal = false);
4506
4507 /// EmitStoreOfScalar - Store a scalar value to an address, taking
4508 /// care to appropriately convert from the memory representation to
4509 /// the LLVM value representation. The l-value must be a simple
4510 /// l-value. The isInit flag indicates whether this is an initialization.
4511 /// If so, atomic qualifiers are ignored and the store is always non-atomic.
4512 void EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
4513 bool isInit = false);
4514
4515 /// EmitLoadOfLValue - Given an expression that represents a value lvalue,
4516 /// this method emits the address of the lvalue, then loads the result as an
4517 /// rvalue, returning the rvalue.
4518 RValue EmitLoadOfLValue(LValue V, SourceLocation Loc);
4519 RValue EmitLoadOfExtVectorElementLValue(LValue V);
4520 RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc);
4521 RValue EmitLoadOfGlobalRegLValue(LValue LV);
4522
4523 /// Like EmitLoadOfLValue but also handles complex and aggregate types.
4524 RValue EmitLoadOfAnyValue(LValue V,
4525 AggValueSlot Slot = AggValueSlot::ignored(),
4526 SourceLocation Loc = {});
4527
4528 /// EmitStoreThroughLValue - Store the specified rvalue into the specified
4529 /// lvalue, where both are guaranteed to the have the same type, and that type
4530 /// is 'Ty'.
4531 void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit = false);
4532 void EmitStoreThroughExtVectorComponentLValue(RValue Src, LValue Dst);
4533 void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst);
4534
4535 /// EmitStoreThroughBitfieldLValue - Store Src into Dst with same constraints
4536 /// as EmitStoreThroughLValue.
4537 ///
4538 /// \param Result [out] - If non-null, this will be set to a Value* for the
4539 /// bit-field contents after the store, appropriate for use as the result of
4540 /// an assignment to the bit-field.
4541 void EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
4542 llvm::Value **Result = nullptr);
4543
4544 /// Emit an l-value for an assignment (simple or compound) of complex type.
4545 LValue EmitComplexAssignmentLValue(const BinaryOperator *E);
4546 LValue EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E);
4547 LValue EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E,
4548 llvm::Value *&Result);
4549
4550 // Note: only available for agg return types
4551 LValue EmitBinaryOperatorLValue(const BinaryOperator *E);
4552 LValue EmitCompoundAssignmentLValue(const CompoundAssignOperator *E);
4553 // Note: only available for agg return types
4554 LValue EmitCallExprLValue(const CallExpr *E,
4555 llvm::CallBase **CallOrInvoke = nullptr);
4556 // Note: only available for agg return types
4557 LValue EmitVAArgExprLValue(const VAArgExpr *E);
4558 LValue EmitDeclRefLValue(const DeclRefExpr *E);
4559 LValue EmitOMPCapturedBindingLValue(const BindingDecl *BD);
4560 LValue EmitStringLiteralLValue(const StringLiteral *E);
4561 LValue EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E);
4562 LValue EmitPredefinedLValue(const PredefinedExpr *E);
4563 LValue EmitUnaryOpLValue(const UnaryOperator *E);
4564 LValue EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
4565 bool Accessed = false);
4566 llvm::Value *EmitMatrixIndexExpr(const Expr *E);
4567 LValue EmitMatrixSingleSubscriptExpr(const MatrixSingleSubscriptExpr *E);
4568 LValue EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E);
4569 LValue EmitArraySectionExpr(const ArraySectionExpr *E,
4570 bool IsLowerBound = true);
4571 LValue EmitExtVectorElementExpr(const ExtVectorElementExpr *E);
4572 LValue EmitMatrixElementExpr(const MatrixElementExpr *E);
4573 LValue EmitMemberExpr(const MemberExpr *E);
4574 LValue EmitObjCIsaExpr(const ObjCIsaExpr *E);
4575 LValue EmitCompoundLiteralLValue(const CompoundLiteralExpr *E);
4576 LValue EmitInitListLValue(const InitListExpr *E);
4577 void EmitIgnoredConditionalOperator(const AbstractConditionalOperator *E);
4578 LValue EmitConditionalOperatorLValue(const AbstractConditionalOperator *E);
4579 LValue EmitCastLValue(const CastExpr *E);
4580 LValue EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);
4581 LValue EmitOpaqueValueLValue(const OpaqueValueExpr *e);
4582 LValue EmitHLSLArrayAssignLValue(const BinaryOperator *E);
4583
4584 std::pair<LValue, LValue> EmitHLSLOutArgLValues(const HLSLOutArgExpr *E,
4585 QualType Ty);
4586 LValue EmitHLSLOutArgExpr(const HLSLOutArgExpr *E, CallArgList &Args,
4587 QualType Ty);
4588
4589 Address EmitExtVectorElementLValue(LValue V);
4590
4591 RValue EmitRValueForField(LValue LV, const FieldDecl *FD, SourceLocation Loc);
4592
4593 Address EmitArrayToPointerDecay(const Expr *Array,
4594 LValueBaseInfo *BaseInfo = nullptr,
4595 TBAAAccessInfo *TBAAInfo = nullptr);
4596
4597 class ConstantEmission {
4598 llvm::PointerIntPair<llvm::Constant *, 1, bool> ValueAndIsReference;
4599 ConstantEmission(llvm::Constant *C, bool isReference)
4600 : ValueAndIsReference(C, isReference) {}
4601
4602 public:
4603 ConstantEmission() {}
4604 static ConstantEmission forReference(llvm::Constant *C) {
4605 return ConstantEmission(C, true);
4606 }
4607 static ConstantEmission forValue(llvm::Constant *C) {
4608 return ConstantEmission(C, false);
4609 }
4610
4611 explicit operator bool() const {
4612 return ValueAndIsReference.getOpaqueValue() != nullptr;
4613 }
4614
4615 bool isReference() const { return ValueAndIsReference.getInt(); }
4616 LValue getReferenceLValue(CodeGenFunction &CGF, const Expr *RefExpr) const {
4617 assert(isReference());
4618 return CGF.MakeNaturalAlignAddrLValue(V: ValueAndIsReference.getPointer(),
4619 T: RefExpr->getType());
4620 }
4621
4622 llvm::Constant *getValue() const {
4623 assert(!isReference());
4624 return ValueAndIsReference.getPointer();
4625 }
4626 };
4627
4628 ConstantEmission tryEmitAsConstant(const DeclRefExpr *RefExpr);
4629 ConstantEmission tryEmitAsConstant(const MemberExpr *ME);
4630 llvm::Value *emitScalarConstant(const ConstantEmission &Constant, Expr *E);
4631
4632 RValue EmitPseudoObjectRValue(const PseudoObjectExpr *e,
4633 AggValueSlot slot = AggValueSlot::ignored());
4634 LValue EmitPseudoObjectLValue(const PseudoObjectExpr *e);
4635
4636 void FlattenAccessAndTypeLValue(LValue LVal,
4637 SmallVectorImpl<LValue> &AccessList);
4638
4639 llvm::Value *EmitIvarOffset(const ObjCInterfaceDecl *Interface,
4640 const ObjCIvarDecl *Ivar);
4641 llvm::Value *EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
4642 const ObjCIvarDecl *Ivar);
4643 LValue EmitLValueForField(LValue Base, const FieldDecl *Field,
4644 bool IsInBounds = true);
4645 LValue EmitLValueForLambdaField(const FieldDecl *Field);
4646 LValue EmitLValueForLambdaField(const FieldDecl *Field,
4647 llvm::Value *ThisValue);
4648
4649 /// EmitLValueForFieldInitialization - Like EmitLValueForField, except that
4650 /// if the Field is a reference, this will return the address of the reference
4651 /// and not the address of the value stored in the reference.
4652 LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field);
4653
4654 LValue EmitLValueForIvar(QualType ObjectTy, llvm::Value *Base,
4655 const ObjCIvarDecl *Ivar, unsigned CVRQualifiers);
4656
4657 LValue EmitCXXConstructLValue(const CXXConstructExpr *E);
4658 LValue EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E);
4659 LValue EmitCXXTypeidLValue(const CXXTypeidExpr *E);
4660 LValue EmitCXXUuidofLValue(const CXXUuidofExpr *E);
4661
4662 LValue EmitObjCMessageExprLValue(const ObjCMessageExpr *E);
4663 LValue EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E);
4664 LValue EmitStmtExprLValue(const StmtExpr *E);
4665 LValue EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E);
4666 LValue EmitObjCSelectorLValue(const ObjCSelectorExpr *E);
4667 void EmitDeclRefExprDbgValue(const DeclRefExpr *E, const APValue &Init);
4668
4669 //===--------------------------------------------------------------------===//
4670 // Scalar Expression Emission
4671 //===--------------------------------------------------------------------===//
4672
4673 /// EmitCall - Generate a call of the given function, expecting the given
4674 /// result type, and using the given argument list which specifies both the
4675 /// LLVM arguments and the types they were derived from.
4676 RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4677 ReturnValueSlot ReturnValue, const CallArgList &Args,
4678 llvm::CallBase **CallOrInvoke, bool IsMustTail,
4679 SourceLocation Loc,
4680 bool IsVirtualFunctionPointerThunk = false);
4681 RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,
4682 ReturnValueSlot ReturnValue, const CallArgList &Args,
4683 llvm::CallBase **CallOrInvoke = nullptr,
4684 bool IsMustTail = false) {
4685 return EmitCall(CallInfo, Callee, ReturnValue, Args, CallOrInvoke,
4686 IsMustTail, Loc: SourceLocation());
4687 }
4688 RValue EmitCall(QualType FnType, const CGCallee &Callee, const CallExpr *E,
4689 ReturnValueSlot ReturnValue, llvm::Value *Chain = nullptr,
4690 llvm::CallBase **CallOrInvoke = nullptr,
4691 CGFunctionInfo const **ResolvedFnInfo = nullptr);
4692
4693 // If a Call or Invoke instruction was emitted for this CallExpr, this method
4694 // writes the pointer to `CallOrInvoke` if it's not null.
4695 RValue EmitCallExpr(const CallExpr *E,
4696 ReturnValueSlot ReturnValue = ReturnValueSlot(),
4697 llvm::CallBase **CallOrInvoke = nullptr);
4698 RValue EmitSimpleCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,
4699 llvm::CallBase **CallOrInvoke = nullptr);
4700 CGCallee EmitCallee(const Expr *E);
4701
4702 void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl);
4703 void checkTargetFeatures(SourceLocation Loc, const FunctionDecl *TargetDecl);
4704
4705 llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4706 const Twine &name = "");
4707 llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,
4708 ArrayRef<llvm::Value *> args,
4709 const Twine &name = "",
4710 bool IsConvergent = false);
4711 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4712 const Twine &Name = "");
4713 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4714 ArrayRef<llvm::Value *> Args,
4715 const Twine &Name = "");
4716 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4717 ArrayRef<llvm::Type *> Types,
4718 ArrayRef<llvm::Value *> Args,
4719 const Twine &Name = "");
4720 llvm::CallInst *EmitIntrinsicCall(llvm::Intrinsic::ID ID,
4721 ArrayRef<llvm::Value *> Args,
4722 llvm::Type *RetTy, const Twine &Name = "");
4723 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4724 const Twine &name = "");
4725 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4726 ArrayRef<Address> args,
4727 const Twine &name = "");
4728 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4729 ArrayRef<llvm::Value *> args,
4730 const Twine &name = "");
4731
4732 SmallVector<llvm::OperandBundleDef, 1>
4733 getBundlesForFunclet(llvm::Value *Callee);
4734
4735 llvm::CallBase *EmitCallOrInvoke(llvm::FunctionCallee Callee,
4736 ArrayRef<llvm::Value *> Args,
4737 const Twine &Name = "");
4738 llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4739 ArrayRef<llvm::Value *> args,
4740 const Twine &name = "");
4741 llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4742 const Twine &name = "");
4743 void EmitNoreturnRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4744 ArrayRef<llvm::Value *> args);
4745
4746 CGCallee BuildAppleKextVirtualCall(const CXXMethodDecl *MD,
4747 NestedNameSpecifier Qual, llvm::Type *Ty);
4748
4749 CGCallee BuildAppleKextVirtualDestructorCall(const CXXDestructorDecl *DD,
4750 CXXDtorType Type,
4751 const CXXRecordDecl *RD);
4752
4753 bool isPointerKnownNonNull(const Expr *E);
4754 /// Check whether the underlying base pointer is a constant null.
4755 bool isUnderlyingBasePointerConstantNull(const Expr *E);
4756
4757 /// Create the discriminator from the storage address and the entity hash.
4758 llvm::Value *EmitPointerAuthBlendDiscriminator(llvm::Value *StorageAddress,
4759 llvm::Value *Discriminator);
4760 CGPointerAuthInfo EmitPointerAuthInfo(const PointerAuthSchema &Schema,
4761 llvm::Value *StorageAddress,
4762 llvm::ConstantInt *Discriminator);
4763 CGPointerAuthInfo EmitPointerAuthInfo(const PointerAuthSchema &Schema,
4764 llvm::Value *StorageAddress,
4765 GlobalDecl SchemaDecl,
4766 QualType SchemaType);
4767
4768 llvm::Value *EmitPointerAuthSign(const CGPointerAuthInfo &Info,
4769 llvm::Value *Pointer);
4770
4771 llvm::Value *EmitPointerAuthAuth(const CGPointerAuthInfo &Info,
4772 llvm::Value *Pointer);
4773
4774 llvm::Value *emitPointerAuthResign(llvm::Value *Pointer, QualType PointerType,
4775 const CGPointerAuthInfo &CurAuthInfo,
4776 const CGPointerAuthInfo &NewAuthInfo,
4777 bool IsKnownNonNull);
4778 llvm::Value *emitPointerAuthResignCall(llvm::Value *Pointer,
4779 const CGPointerAuthInfo &CurInfo,
4780 const CGPointerAuthInfo &NewInfo);
4781
4782 void EmitPointerAuthOperandBundle(
4783 const CGPointerAuthInfo &Info,
4784 SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
4785
4786 CGPointerAuthInfo EmitPointerAuthInfo(PointerAuthQualifier Qualifier,
4787 Address StorageAddress);
4788 llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
4789 llvm::Value *Pointer, QualType ValueType,
4790 Address StorageAddress,
4791 bool IsKnownNonNull);
4792 llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,
4793 const Expr *PointerExpr,
4794 Address StorageAddress);
4795 llvm::Value *EmitPointerAuthUnqualify(PointerAuthQualifier Qualifier,
4796 llvm::Value *Pointer,
4797 QualType PointerType,
4798 Address StorageAddress,
4799 bool IsKnownNonNull);
4800 void EmitPointerAuthCopy(PointerAuthQualifier Qualifier, QualType Type,
4801 Address DestField, Address SrcField);
4802
4803 std::pair<llvm::Value *, CGPointerAuthInfo>
4804 EmitOrigPointerRValue(const Expr *E);
4805
4806 llvm::Value *authPointerToPointerCast(llvm::Value *ResultPtr,
4807 QualType SourceType, QualType DestType);
4808 Address authPointerToPointerCast(Address Ptr, QualType SourceType,
4809 QualType DestType);
4810
4811 Address getAsNaturalAddressOf(Address Addr, QualType PointeeTy);
4812
4813 llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) {
4814 return getAsNaturalAddressOf(Addr, PointeeTy: PointeeType).getBasePointer();
4815 }
4816
4817 // Return the copy constructor name with the prefix "__copy_constructor_"
4818 // removed.
4819 static std::string getNonTrivialCopyConstructorStr(QualType QT,
4820 CharUnits Alignment,
4821 bool IsVolatile,
4822 ASTContext &Ctx);
4823
4824 // Return the destructor name with the prefix "__destructor_" removed.
4825 static std::string getNonTrivialDestructorStr(QualType QT,
4826 CharUnits Alignment,
4827 bool IsVolatile,
4828 ASTContext &Ctx);
4829
4830 // These functions emit calls to the special functions of non-trivial C
4831 // structs.
4832 void defaultInitNonTrivialCStructVar(LValue Dst);
4833 void callCStructDefaultConstructor(LValue Dst);
4834 void callCStructDestructor(LValue Dst);
4835 void callCStructCopyConstructor(LValue Dst, LValue Src);
4836 void callCStructMoveConstructor(LValue Dst, LValue Src);
4837 void callCStructCopyAssignmentOperator(LValue Dst, LValue Src);
4838 void callCStructMoveAssignmentOperator(LValue Dst, LValue Src);
4839
4840 RValue EmitCXXMemberOrOperatorCall(
4841 const CXXMethodDecl *Method, const CGCallee &Callee,
4842 ReturnValueSlot ReturnValue, llvm::Value *This,
4843 llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *E,
4844 CallArgList *RtlArgs, llvm::CallBase **CallOrInvoke);
4845 RValue EmitCXXDestructorCall(GlobalDecl Dtor, const CGCallee &Callee,
4846 llvm::Value *This, QualType ThisTy,
4847 llvm::Value *ImplicitParam,
4848 QualType ImplicitParamTy, const CallExpr *E,
4849 llvm::CallBase **CallOrInvoke = nullptr);
4850 RValue EmitCXXMemberCallExpr(const CXXMemberCallExpr *E,
4851 ReturnValueSlot ReturnValue,
4852 llvm::CallBase **CallOrInvoke = nullptr);
4853 RValue EmitCXXMemberOrOperatorMemberCallExpr(
4854 const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue,
4855 bool HasQualifier, NestedNameSpecifier Qualifier, bool IsArrow,
4856 const Expr *Base, llvm::CallBase **CallOrInvoke);
4857 // Compute the object pointer.
4858 Address EmitCXXMemberDataPointerAddress(
4859 const Expr *E, Address base, llvm::Value *memberPtr,
4860 const MemberPointerType *memberPtrType, bool IsInBounds,
4861 LValueBaseInfo *BaseInfo = nullptr, TBAAAccessInfo *TBAAInfo = nullptr);
4862 RValue EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
4863 ReturnValueSlot ReturnValue,
4864 llvm::CallBase **CallOrInvoke);
4865
4866 RValue EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
4867 const CXXMethodDecl *MD,
4868 ReturnValueSlot ReturnValue,
4869 llvm::CallBase **CallOrInvoke);
4870 RValue EmitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *E);
4871
4872 RValue EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
4873 ReturnValueSlot ReturnValue,
4874 llvm::CallBase **CallOrInvoke);
4875
4876 RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E);
4877 RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E);
4878
4879 RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
4880 const CallExpr *E, ReturnValueSlot ReturnValue);
4881
4882 RValue emitRotate(const CallExpr *E, bool IsRotateRight);
4883
4884 RValue emitStdcCountIntrinsic(const CallExpr *E, llvm::Intrinsic::ID IntID,
4885 bool InvertArg, bool IsPop = false);
4886 RValue emitStdcBitWidthMinus(const CallExpr *E, llvm::Intrinsic::ID IntID,
4887 bool IsPop);
4888 RValue emitStdcFirstBit(const CallExpr *E, llvm::Intrinsic::ID IntID,
4889 bool InvertArg);
4890
4891 /// Emit IR for __builtin_os_log_format.
4892 RValue emitBuiltinOSLogFormat(const CallExpr &E);
4893
4894 /// Emit IR for __builtin_is_aligned.
4895 RValue EmitBuiltinIsAligned(const CallExpr *E);
4896 /// Emit IR for __builtin_align_up/__builtin_align_down.
4897 RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp);
4898
4899 llvm::Function *generateBuiltinOSLogHelperFunction(
4900 const analyze_os_log::OSLogBufferLayout &Layout,
4901 CharUnits BufferAlignment);
4902
4903 RValue EmitBlockCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,
4904 llvm::CallBase **CallOrInvoke);
4905
4906 /// EmitTargetBuiltinExpr - Emit the given builtin call. Returns 0 if the call
4907 /// is unhandled by the current target.
4908 llvm::Value *EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4909 ReturnValueSlot ReturnValue);
4910
4911 llvm::Value *
4912 EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty,
4913 const llvm::CmpInst::Predicate Pred,
4914 const llvm::Twine &Name = "");
4915 llvm::Value *EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4916 ReturnValueSlot ReturnValue,
4917 llvm::Triple::ArchType Arch);
4918 llvm::Value *EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4919 ReturnValueSlot ReturnValue,
4920 llvm::Triple::ArchType Arch);
4921 llvm::Value *EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
4922 ReturnValueSlot ReturnValue,
4923 llvm::Triple::ArchType Arch);
4924 llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::IntegerType *ITy,
4925 QualType RTy);
4926 llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::ArrayType *ATy,
4927 QualType RTy);
4928
4929 llvm::Value *
4930 EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic,
4931 unsigned AltLLVMIntrinsic, const char *NameHint,
4932 unsigned Modifier, const CallExpr *E,
4933 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0,
4934 Address PtrOp1, llvm::Triple::ArchType Arch);
4935
4936 llvm::Function *LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
4937 unsigned Modifier, llvm::Type *ArgTy,
4938 const CallExpr *E);
4939 llvm::Value *EmitNeonCall(llvm::Function *F,
4940 SmallVectorImpl<llvm::Value *> &O, const char *name,
4941 unsigned shift = 0, bool rightshift = false);
4942 llvm::Value *EmitFP8NeonCall(unsigned IID, ArrayRef<llvm::Type *> Tys,
4943 SmallVectorImpl<llvm::Value *> &O,
4944 const CallExpr *E, const char *name);
4945 llvm::Value *EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,
4946 llvm::Type *Ty1, bool Extract,
4947 SmallVectorImpl<llvm::Value *> &Ops,
4948 const CallExpr *E, const char *name);
4949 llvm::Value *EmitFP8NeonFDOTCall(unsigned IID, bool ExtendLaneArg,
4950 llvm::Type *RetTy,
4951 SmallVectorImpl<llvm::Value *> &Ops,
4952 const CallExpr *E, const char *name);
4953 llvm::Value *EmitFP8NeonFMLACall(unsigned IID, bool ExtendLaneArg,
4954 llvm::Type *RetTy,
4955 SmallVectorImpl<llvm::Value *> &Ops,
4956 const CallExpr *E, const char *name);
4957 llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx,
4958 const llvm::ElementCount &Count);
4959 llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx);
4960 llvm::Value *EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty,
4961 bool negateForRightShift);
4962 llvm::Value *EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt,
4963 llvm::Type *Ty, bool usgn, const char *name);
4964 llvm::Value *vectorWrapScalar16(llvm::Value *Op);
4965 /// SVEBuiltinMemEltTy - Returns the memory element type for this memory
4966 /// access builtin. Only required if it can't be inferred from the base
4967 /// pointer operand.
4968 llvm::Type *SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags);
4969
4970 SmallVector<llvm::Type *, 2>
4971 getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType,
4972 ArrayRef<llvm::Value *> Ops);
4973 llvm::Type *getEltType(const SVETypeFlags &TypeFlags);
4974 llvm::ScalableVectorType *getSVEType(const SVETypeFlags &TypeFlags);
4975 llvm::ScalableVectorType *getSVEPredType(const SVETypeFlags &TypeFlags);
4976 llvm::Value *EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags,
4977 ArrayRef<llvm::Value *> Ops);
4978 llvm::Value *EmitSVETupleCreate(const SVETypeFlags &TypeFlags,
4979 llvm::Type *ReturnType,
4980 ArrayRef<llvm::Value *> Ops);
4981 llvm::Value *EmitSVEDupX(llvm::Value *Scalar);
4982 llvm::Value *EmitSVEDupX(llvm::Value *Scalar, llvm::Type *Ty);
4983 llvm::Value *EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty);
4984 llvm::Value *EmitSVEPMull(const SVETypeFlags &TypeFlags,
4985 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4986 unsigned BuiltinID);
4987 llvm::Value *EmitSVEMovl(const SVETypeFlags &TypeFlags,
4988 llvm::ArrayRef<llvm::Value *> Ops,
4989 unsigned BuiltinID);
4990 llvm::Value *EmitSVEPredicateCast(llvm::Value *Pred,
4991 llvm::ScalableVectorType *VTy);
4992 llvm::Value *EmitSVEPredicateTupleCast(llvm::Value *PredTuple,
4993 llvm::StructType *Ty);
4994 llvm::Value *EmitSVEGatherLoad(const SVETypeFlags &TypeFlags,
4995 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4996 unsigned IntID);
4997 llvm::Value *EmitSVEScatterStore(const SVETypeFlags &TypeFlags,
4998 llvm::SmallVectorImpl<llvm::Value *> &Ops,
4999 unsigned IntID);
5000 llvm::Value *EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy,
5001 SmallVectorImpl<llvm::Value *> &Ops,
5002 unsigned BuiltinID, bool IsZExtReturn);
5003 llvm::Value *EmitSVEMaskedStore(const CallExpr *,
5004 SmallVectorImpl<llvm::Value *> &Ops,
5005 unsigned BuiltinID);
5006 llvm::Value *EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,
5007 SmallVectorImpl<llvm::Value *> &Ops,
5008 unsigned BuiltinID);
5009 llvm::Value *EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags,
5010 SmallVectorImpl<llvm::Value *> &Ops,
5011 unsigned IntID);
5012 llvm::Value *EmitSVEStructLoad(const SVETypeFlags &TypeFlags,
5013 SmallVectorImpl<llvm::Value *> &Ops,
5014 unsigned IntID);
5015 llvm::Value *EmitSVEStructStore(const SVETypeFlags &TypeFlags,
5016 SmallVectorImpl<llvm::Value *> &Ops,
5017 unsigned IntID);
5018 llvm::Value *EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5019
5020 llvm::Value *EmitSMELd1St1(const SVETypeFlags &TypeFlags,
5021 llvm::SmallVectorImpl<llvm::Value *> &Ops,
5022 unsigned IntID);
5023 llvm::Value *EmitSMEReadWrite(const SVETypeFlags &TypeFlags,
5024 llvm::SmallVectorImpl<llvm::Value *> &Ops,
5025 unsigned IntID);
5026 llvm::Value *EmitSMEZero(const SVETypeFlags &TypeFlags,
5027 llvm::SmallVectorImpl<llvm::Value *> &Ops,
5028 unsigned IntID);
5029 llvm::Value *EmitSMELdrStr(const SVETypeFlags &TypeFlags,
5030 llvm::SmallVectorImpl<llvm::Value *> &Ops,
5031 unsigned IntID);
5032
5033 void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E,
5034 SmallVectorImpl<llvm::Value *> &Ops,
5035 SVETypeFlags TypeFlags);
5036
5037 llvm::Value *EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5038
5039 llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E,
5040 llvm::Triple::ArchType Arch);
5041 llvm::Value *EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5042
5043 llvm::Value *BuildVector(ArrayRef<llvm::Value *> Ops);
5044 llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5045 llvm::Value *EmitPPCBuiltinCpu(unsigned BuiltinID, llvm::Type *ReturnType,
5046 StringRef CPUStr);
5047 llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5048 llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5049 llvm::Value *EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
5050 ReturnValueSlot ReturnValue);
5051
5052 // Returns a builtin function that the SPIR-V backend will expand into a spec
5053 // constant.
5054 llvm::Function *
5055 getSpecConstantFunction(const clang::QualType &SpecConstantType);
5056
5057 llvm::Value *EmitDirectXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5058 llvm::Value *EmitSPIRVBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5059 llvm::Value *EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx,
5060 const CallExpr *E);
5061 llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5062 llvm::Value *EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5063 llvm::Value *EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
5064 const CallExpr *E);
5065 llvm::Value *EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5066 llvm::Value *EmitAVRBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
5067 llvm::Value *EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
5068 ReturnValueSlot ReturnValue);
5069
5070 llvm::Value *EmitRISCVCpuSupports(const CallExpr *E);
5071 llvm::Value *EmitRISCVCpuSupports(ArrayRef<StringRef> FeaturesStrs);
5072 llvm::Value *EmitRISCVCpuInit();
5073 llvm::Value *EmitRISCVCpuIs(const CallExpr *E);
5074 llvm::Value *EmitRISCVCpuIs(StringRef CPUStr);
5075
5076 void AddAMDGPUFenceAddressSpaceMMRA(llvm::Instruction *Inst,
5077 const CallExpr *E);
5078 /// Attach the AMDGPU availability/visibility MMRA to \p Inst when the
5079 /// amdgpu_av attribute is active on the current statement.
5080 void AddAMDGPUAvailableVisibleMMRA(llvm::Instruction *Inst);
5081 void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope,
5082 llvm::AtomicOrdering &AO,
5083 llvm::SyncScope::ID &SSID);
5084
5085 enum class MSVCIntrin;
5086 llvm::Value *EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E);
5087
5088 llvm::Value *EmitBuiltinAvailable(const VersionTuple &Version);
5089
5090 llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E);
5091 llvm::Value *EmitObjCStringLiteral(const ObjCStringLiteral *E);
5092 llvm::Value *EmitObjCBoxedExpr(const ObjCBoxedExpr *E);
5093 llvm::Value *EmitObjCArrayLiteral(const ObjCArrayLiteral *E);
5094 llvm::Value *EmitObjCDictionaryLiteral(const ObjCDictionaryLiteral *E);
5095 llvm::Value *
5096 EmitObjCCollectionLiteral(const Expr *E,
5097 const ObjCMethodDecl *MethodWithObjects);
5098 llvm::Value *EmitObjCSelectorExpr(const ObjCSelectorExpr *E);
5099 RValue EmitObjCMessageExpr(const ObjCMessageExpr *E,
5100 ReturnValueSlot Return = ReturnValueSlot());
5101
5102 /// Retrieves the default cleanup kind for an ARC cleanup.
5103 /// Except under -fobjc-arc-eh, ARC cleanups are normal-only.
5104 CleanupKind getARCCleanupKind() {
5105 return CGM.getCodeGenOpts().ObjCAutoRefCountExceptions ? NormalAndEHCleanup
5106 : NormalCleanup;
5107 }
5108
5109 // ARC primitives.
5110 void EmitARCInitWeak(Address addr, llvm::Value *value);
5111 void EmitARCDestroyWeak(Address addr);
5112 llvm::Value *EmitARCLoadWeak(Address addr);
5113 llvm::Value *EmitARCLoadWeakRetained(Address addr);
5114 llvm::Value *EmitARCStoreWeak(Address addr, llvm::Value *value, bool ignored);
5115 void emitARCCopyAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
5116 void emitARCMoveAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);
5117 void EmitARCCopyWeak(Address dst, Address src);
5118 void EmitARCMoveWeak(Address dst, Address src);
5119 llvm::Value *EmitARCRetainAutorelease(QualType type, llvm::Value *value);
5120 llvm::Value *EmitARCRetainAutoreleaseNonBlock(llvm::Value *value);
5121 llvm::Value *EmitARCStoreStrong(LValue lvalue, llvm::Value *value,
5122 bool resultIgnored);
5123 llvm::Value *EmitARCStoreStrongCall(Address addr, llvm::Value *value,
5124 bool resultIgnored);
5125 llvm::Value *EmitARCRetain(QualType type, llvm::Value *value);
5126 llvm::Value *EmitARCRetainNonBlock(llvm::Value *value);
5127 llvm::Value *EmitARCRetainBlock(llvm::Value *value, bool mandatory);
5128 void EmitARCDestroyStrong(Address addr, ARCPreciseLifetime_t precise);
5129 void EmitARCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
5130 llvm::Value *EmitARCAutorelease(llvm::Value *value);
5131 llvm::Value *EmitARCAutoreleaseReturnValue(llvm::Value *value);
5132 llvm::Value *EmitARCRetainAutoreleaseReturnValue(llvm::Value *value);
5133 llvm::Value *EmitARCRetainAutoreleasedReturnValue(llvm::Value *value);
5134 llvm::Value *EmitARCUnsafeClaimAutoreleasedReturnValue(llvm::Value *value);
5135
5136 llvm::Value *EmitObjCAutorelease(llvm::Value *value, llvm::Type *returnType);
5137 llvm::Value *EmitObjCRetainNonBlock(llvm::Value *value,
5138 llvm::Type *returnType);
5139 void EmitObjCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);
5140
5141 std::pair<LValue, llvm::Value *>
5142 EmitARCStoreAutoreleasing(const BinaryOperator *e);
5143 std::pair<LValue, llvm::Value *> EmitARCStoreStrong(const BinaryOperator *e,
5144 bool ignored);
5145 std::pair<LValue, llvm::Value *>
5146 EmitARCStoreUnsafeUnretained(const BinaryOperator *e, bool ignored);
5147
5148 llvm::Value *EmitObjCAlloc(llvm::Value *value, llvm::Type *returnType);
5149 llvm::Value *EmitObjCAllocWithZone(llvm::Value *value,
5150 llvm::Type *returnType);
5151 llvm::Value *EmitObjCAllocInit(llvm::Value *value, llvm::Type *resultType);
5152
5153 llvm::Value *EmitObjCThrowOperand(const Expr *expr);
5154 llvm::Value *EmitObjCConsumeObject(QualType T, llvm::Value *Ptr);
5155 llvm::Value *EmitObjCExtendObjectLifetime(QualType T, llvm::Value *Ptr);
5156
5157 llvm::Value *EmitARCExtendBlockObject(const Expr *expr);
5158 llvm::Value *EmitARCReclaimReturnedObject(const Expr *e,
5159 bool allowUnsafeClaim);
5160 llvm::Value *EmitARCRetainScalarExpr(const Expr *expr);
5161 llvm::Value *EmitARCRetainAutoreleaseScalarExpr(const Expr *expr);
5162 llvm::Value *EmitARCUnsafeUnretainedScalarExpr(const Expr *expr);
5163
5164 void EmitARCIntrinsicUse(ArrayRef<llvm::Value *> values);
5165
5166 void EmitARCNoopIntrinsicUse(ArrayRef<llvm::Value *> values);
5167
5168 static Destroyer destroyARCStrongImprecise;
5169 static Destroyer destroyARCStrongPrecise;
5170 static Destroyer destroyARCWeak;
5171 static Destroyer emitARCIntrinsicUse;
5172 static Destroyer destroyNonTrivialCStruct;
5173
5174 void EmitObjCAutoreleasePoolPop(llvm::Value *Ptr);
5175 llvm::Value *EmitObjCAutoreleasePoolPush();
5176 llvm::Value *EmitObjCMRRAutoreleasePoolPush();
5177 void EmitObjCAutoreleasePoolCleanup(llvm::Value *Ptr);
5178 void EmitObjCMRRAutoreleasePoolPop(llvm::Value *Ptr);
5179
5180 /// Emits a reference binding to the passed in expression.
5181 RValue EmitReferenceBindingToExpr(const Expr *E);
5182
5183 //===--------------------------------------------------------------------===//
5184 // Expression Emission
5185 //===--------------------------------------------------------------------===//
5186
5187 // Expressions are broken into three classes: scalar, complex, aggregate.
5188
5189 /// EmitScalarExpr - Emit the computation of the specified expression of LLVM
5190 /// scalar type, returning the result.
5191 llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);
5192
5193 /// Emit a conversion from the specified type to the specified destination
5194 /// type, both of which are LLVM scalar types.
5195 llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,
5196 QualType DstTy, SourceLocation Loc);
5197
5198 /// Emit a conversion from the specified complex type to the specified
5199 /// destination type, where the destination type is an LLVM scalar type.
5200 llvm::Value *EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy,
5201 QualType DstTy,
5202 SourceLocation Loc);
5203
5204 /// EmitAggExpr - Emit the computation of the specified expression
5205 /// of aggregate type. The result is computed into the given slot,
5206 /// which may be null to indicate that the value is not needed.
5207 void EmitAggExpr(const Expr *E, AggValueSlot AS);
5208
5209 /// EmitAggExprToLValue - Emit the computation of the specified expression of
5210 /// aggregate type into a temporary LValue.
5211 LValue EmitAggExprToLValue(const Expr *E);
5212
5213 enum ExprValueKind { EVK_RValue, EVK_NonRValue };
5214
5215 /// EmitAggFinalDestCopy - Emit copy of the specified aggregate into
5216 /// destination address.
5217 void EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest, const LValue &Src,
5218 ExprValueKind SrcKind);
5219
5220 /// Create a store to \arg DstPtr from \arg Src, truncating the stored value
5221 /// to at most \arg DstSize bytes.
5222 void CreateCoercedStore(llvm::Value *Src, QualType SrcFETy, Address Dst,
5223 llvm::TypeSize DstSize, bool DstIsVolatile);
5224
5225 /// EmitExtendGCLifetime - Given a pointer to an Objective-C object,
5226 /// make sure it survives garbage collection until this point.
5227 void EmitExtendGCLifetime(llvm::Value *object);
5228
5229 /// EmitComplexExpr - Emit the computation of the specified expression of
5230 /// complex type, returning the result.
5231 ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal = false,
5232 bool IgnoreImag = false);
5233
5234 /// EmitComplexExprIntoLValue - Emit the given expression of complex
5235 /// type and place its result into the specified l-value.
5236 void EmitComplexExprIntoLValue(const Expr *E, LValue dest, bool isInit);
5237
5238 /// EmitStoreOfComplex - Store a complex number into the specified l-value.
5239 void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit);
5240
5241 /// EmitLoadOfComplex - Load a complex number from the specified l-value.
5242 ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc);
5243
5244 ComplexPairTy EmitPromotedComplexExpr(const Expr *E, QualType PromotionType);
5245 llvm::Value *EmitPromotedScalarExpr(const Expr *E, QualType PromotionType);
5246 ComplexPairTy EmitPromotedValue(ComplexPairTy result, QualType PromotionType);
5247 ComplexPairTy EmitUnPromotedValue(ComplexPairTy result,
5248 QualType PromotionType);
5249
5250 Address emitAddrOfRealComponent(Address complex, QualType complexType);
5251 Address emitAddrOfImagComponent(Address complex, QualType complexType);
5252
5253 /// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the
5254 /// global variable that has already been created for it. If the initializer
5255 /// has a different type than GV does, this may free GV and return a different
5256 /// one. Otherwise it just returns GV.
5257 llvm::GlobalVariable *AddInitializerToStaticVarDecl(const VarDecl &D,
5258 llvm::GlobalVariable *GV);
5259
5260 // Emit an @llvm.invariant.start call for the given memory region.
5261 void EmitInvariantStart(llvm::Constant *Addr, CharUnits Size);
5262
5263 /// EmitCXXGlobalVarDeclInit - Create the initializer for a C++
5264 /// variable with global storage.
5265 void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::GlobalVariable *GV,
5266 bool PerformInit);
5267
5268 llvm::Constant *createAtExitStub(const VarDecl &VD, llvm::FunctionCallee Dtor,
5269 llvm::Constant *Addr);
5270
5271 llvm::Function *createTLSAtExitStub(const VarDecl &VD,
5272 llvm::FunctionCallee Dtor,
5273 llvm::Constant *Addr,
5274 llvm::FunctionCallee &AtExit);
5275
5276 /// Call atexit() with a function that passes the given argument to
5277 /// the given function.
5278 void registerGlobalDtorWithAtExit(const VarDecl &D, llvm::FunctionCallee fn,
5279 llvm::Constant *addr);
5280
5281 /// Registers the dtor using 'llvm.global_dtors' for platforms that do not
5282 /// support an 'atexit()' function.
5283 void registerGlobalDtorWithLLVM(const VarDecl &D, llvm::FunctionCallee fn,
5284 llvm::Constant *addr);
5285
5286 /// Call atexit() with function dtorStub.
5287 void registerGlobalDtorWithAtExit(llvm::Constant *dtorStub);
5288
5289 /// Call unatexit() with function dtorStub.
5290 llvm::Value *unregisterGlobalDtorWithUnAtExit(llvm::Constant *dtorStub);
5291
5292 /// Emit code in this function to perform a guarded variable
5293 /// initialization. Guarded initializations are used when it's not
5294 /// possible to prove that an initialization will be done exactly
5295 /// once, e.g. with a static local variable or a static data member
5296 /// of a class template.
5297 void EmitCXXGuardedInit(const VarDecl &D, llvm::GlobalVariable *DeclPtr,
5298 bool PerformInit);
5299
5300 enum class GuardKind { VariableGuard, TlsGuard };
5301
5302 /// Emit a branch to select whether or not to perform guarded initialization.
5303 void EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
5304 llvm::BasicBlock *InitBlock,
5305 llvm::BasicBlock *NoInitBlock, GuardKind Kind,
5306 const VarDecl *D);
5307
5308 /// GenerateCXXGlobalInitFunc - Generates code for initializing global
5309 /// variables.
5310 void
5311 GenerateCXXGlobalInitFunc(llvm::Function *Fn,
5312 ArrayRef<llvm::Function *> CXXThreadLocals,
5313 ConstantAddress Guard = ConstantAddress::invalid());
5314
5315 /// GenerateCXXGlobalCleanUpFunc - Generates code for cleaning up global
5316 /// variables.
5317 void GenerateCXXGlobalCleanUpFunc(
5318 llvm::Function *Fn,
5319 ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
5320 llvm::Constant *>>
5321 DtorsOrStermFinalizers);
5322
5323 void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn, const VarDecl *D,
5324 llvm::GlobalVariable *Addr,
5325 bool PerformInit);
5326
5327 void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest);
5328
5329 void EmitSynthesizedCXXCopyCtor(Address Dest, Address Src, const Expr *Exp);
5330
5331 void EmitCXXThrowExpr(const CXXThrowExpr *E, bool KeepInsertionPoint = true);
5332
5333 RValue EmitAtomicExpr(AtomicExpr *E);
5334
5335 void EmitFakeUse(Address Addr);
5336
5337 //===--------------------------------------------------------------------===//
5338 // Annotations Emission
5339 //===--------------------------------------------------------------------===//
5340
5341 /// Emit an annotation call (intrinsic).
5342 llvm::Value *EmitAnnotationCall(llvm::Function *AnnotationFn,
5343 llvm::Value *AnnotatedVal,
5344 StringRef AnnotationStr,
5345 SourceLocation Location,
5346 const AnnotateAttr *Attr);
5347
5348 /// Emit local annotations for the local variable V, declared by D.
5349 void EmitVarAnnotations(const VarDecl *D, llvm::Value *V);
5350
5351 /// Emit field annotations for the given field & value. Returns the
5352 /// annotation result.
5353 Address EmitFieldAnnotations(const FieldDecl *D, Address V);
5354
5355 //===--------------------------------------------------------------------===//
5356 // Internal Helpers
5357 //===--------------------------------------------------------------------===//
5358
5359 /// ContainsLabel - Return true if the statement contains a label in it. If
5360 /// this statement is not executed normally, it not containing a label means
5361 /// that we can just remove the code.
5362 static bool ContainsLabel(const Stmt *S, bool IgnoreCaseStmts = false);
5363
5364 /// containsBreak - Return true if the statement contains a break out of it.
5365 /// If the statement (recursively) contains a switch or loop with a break
5366 /// inside of it, this is fine.
5367 static bool containsBreak(const Stmt *S);
5368
5369 /// Determine if the given statement might introduce a declaration into the
5370 /// current scope, by being a (possibly-labelled) DeclStmt.
5371 static bool mightAddDeclToScope(const Stmt *S);
5372
5373 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
5374 /// to a constant, or if it does but contains a label, return false. If it
5375 /// constant folds return true and set the boolean result in Result.
5376 bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result,
5377 bool AllowLabels = false);
5378
5379 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
5380 /// to a constant, or if it does but contains a label, return false. If it
5381 /// constant folds return true and set the folded value.
5382 bool ConstantFoldsToSimpleInteger(const Expr *Cond, llvm::APSInt &Result,
5383 bool AllowLabels = false);
5384
5385 /// Ignore parentheses and logical-NOT to track conditions consistently.
5386 static const Expr *stripCond(const Expr *C);
5387
5388 /// isInstrumentedCondition - Determine whether the given condition is an
5389 /// instrumentable condition (i.e. no "&&" or "||").
5390 static bool isInstrumentedCondition(const Expr *C);
5391
5392 /// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
5393 /// increments a profile counter based on the semantics of the given logical
5394 /// operator opcode. This is used to instrument branch condition coverage
5395 /// for logical operators.
5396 void EmitBranchToCounterBlock(const Expr *Cond, BinaryOperator::Opcode LOp,
5397 llvm::BasicBlock *TrueBlock,
5398 llvm::BasicBlock *FalseBlock,
5399 uint64_t TrueCount = 0,
5400 Stmt::Likelihood LH = Stmt::LH_None,
5401 const Expr *CntrIdx = nullptr);
5402
5403 /// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an
5404 /// if statement) to the specified blocks. Based on the condition, this might
5405 /// try to simplify the codegen of the conditional based on the branch.
5406 /// TrueCount should be the number of times we expect the condition to
5407 /// evaluate to true based on PGO data.
5408 void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock,
5409 llvm::BasicBlock *FalseBlock, uint64_t TrueCount,
5410 Stmt::Likelihood LH = Stmt::LH_None,
5411 const Expr *ConditionalOp = nullptr,
5412 const VarDecl *ConditionalDecl = nullptr);
5413
5414 /// Given an assignment `*LHS = RHS`, emit a test that checks if \p RHS is
5415 /// nonnull, if \p LHS is marked _Nonnull.
5416 void EmitNullabilityCheck(LValue LHS, llvm::Value *RHS, SourceLocation Loc);
5417
5418 /// An enumeration which makes it easier to specify whether or not an
5419 /// operation is a subtraction.
5420 enum { NotSubtraction = false, IsSubtraction = true };
5421
5422 /// Emit pointer + index arithmetic.
5423 llvm::Value *EmitPointerArithmetic(const BinaryOperator *BO,
5424 Expr *pointerOperand, llvm::Value *pointer,
5425 Expr *indexOperand, llvm::Value *index,
5426 bool isSubtraction);
5427
5428 /// Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to
5429 /// detect undefined behavior when the pointer overflow sanitizer is enabled.
5430 /// \p SignedIndices indicates whether any of the GEP indices are signed.
5431 /// \p IsSubtraction indicates whether the expression used to form the GEP
5432 /// is a subtraction.
5433 llvm::Value *EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr,
5434 ArrayRef<llvm::Value *> IdxList,
5435 bool SignedIndices, bool IsSubtraction,
5436 SourceLocation Loc,
5437 const Twine &Name = "");
5438
5439 Address EmitCheckedInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
5440 llvm::Type *elementType, bool SignedIndices,
5441 bool IsSubtraction, SourceLocation Loc,
5442 CharUnits Align, const Twine &Name = "");
5443
5444 /// Specifies which type of sanitizer check to apply when handling a
5445 /// particular builtin.
5446 enum BuiltinCheckKind {
5447 BCK_CTZPassedZero,
5448 BCK_CLZPassedZero,
5449 BCK_AssumePassedFalse,
5450 };
5451
5452 /// Emits an argument for a call to a builtin. If the builtin sanitizer is
5453 /// enabled, a runtime check specified by \p Kind is also emitted.
5454 llvm::Value *EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind);
5455
5456 /// Emits an argument for a call to a `__builtin_assume`. If the builtin
5457 /// sanitizer is enabled, a runtime check is also emitted.
5458 llvm::Value *EmitCheckedArgForAssume(const Expr *E);
5459
5460 /// Emit a description of a type in a format suitable for passing to
5461 /// a runtime sanitizer handler.
5462 llvm::Constant *EmitCheckTypeDescriptor(QualType T);
5463
5464 /// Convert a value into a format suitable for passing to a runtime
5465 /// sanitizer handler.
5466 llvm::Value *EmitCheckValue(llvm::Value *V);
5467
5468 /// Emit a description of a source location in a format suitable for
5469 /// passing to a runtime sanitizer handler.
5470 llvm::Constant *EmitCheckSourceLocation(SourceLocation Loc);
5471
5472 void EmitKCFIOperandBundle(const CGCallee &Callee,
5473 SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
5474
5475 /// Create a basic block that will either trap or call a handler function in
5476 /// the UBSan runtime with the provided arguments, and create a conditional
5477 /// branch to it.
5478 void
5479 EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
5480 Checked,
5481 SanitizerHandler Check, ArrayRef<llvm::Constant *> StaticArgs,
5482 ArrayRef<llvm::Value *> DynamicArgs,
5483 const TrapReason *TR = nullptr);
5484
5485 /// Emit a slow path cross-DSO CFI check which calls __cfi_slowpath
5486 /// if Cond if false.
5487 void EmitCfiSlowPathCheck(SanitizerKind::SanitizerOrdinal Ordinal,
5488 llvm::Value *Cond, llvm::ConstantInt *TypeId,
5489 llvm::Value *Ptr,
5490 ArrayRef<llvm::Constant *> StaticArgs);
5491
5492 /// Emit a reached-unreachable diagnostic if \p Loc is valid and runtime
5493 /// checking is enabled. Otherwise, just emit an unreachable instruction.
5494 void EmitUnreachable(SourceLocation Loc);
5495
5496 /// Create a basic block that will call the trap intrinsic, and emit a
5497 /// conditional branch to it, for the -ftrapv checks.
5498 void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID,
5499 bool NoMerge = false, const TrapReason *TR = nullptr);
5500
5501 /// Emit a call to trap or debugtrap. If 'EnsureInsertPoint' is false, the
5502 /// IR builder need not have a valid insert point after this returns.
5503 llvm::CallInst *EmitTrapCall(llvm::Intrinsic::ID IntrID,
5504 bool EnsureInsertPoint = true);
5505
5506 /// Emit a call to '\@llvm.trap()' and clear the current insert point.
5507 void EmitTrapCallAndMakeUnreachable();
5508
5509 /// Emit a stub for the cross-DSO CFI check function.
5510 void EmitCfiCheckStub();
5511
5512 /// Emit a cross-DSO CFI failure handling function.
5513 void EmitCfiCheckFail();
5514
5515 /// Create a check for a function parameter that may potentially be
5516 /// declared as non-null.
5517 void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc,
5518 AbstractCallee AC, unsigned ParmNum);
5519
5520 void EmitNonNullArgCheck(Address Addr, QualType ArgType,
5521 SourceLocation ArgLoc, AbstractCallee AC,
5522 unsigned ParmNum);
5523
5524 /// EmitWriteback - Emit callbacks for function.
5525 void EmitWritebacks(const CallArgList &Args);
5526
5527 /// EmitCallArg - Emit a single call argument.
5528 void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType);
5529
5530 /// EmitDelegateCallArg - We are performing a delegate call; that
5531 /// is, the current function is delegating to another one. Produce
5532 /// a r-value suitable for passing the given parameter.
5533 void EmitDelegateCallArg(CallArgList &args, const VarDecl *param,
5534 SourceLocation loc);
5535
5536 /// SetFPAccuracy - Set the minimum required accuracy of the given floating
5537 /// point operation, expressed as the maximum relative error in ulp.
5538 void SetFPAccuracy(llvm::Value *Val, float Accuracy);
5539
5540 /// Set the minimum required accuracy of the given sqrt operation
5541 /// based on CodeGenOpts.
5542 void SetSqrtFPAccuracy(llvm::Value *Val);
5543
5544 /// Set the minimum required accuracy of the given sqrt operation based on
5545 /// CodeGenOpts.
5546 void SetDivFPAccuracy(llvm::Value *Val);
5547
5548 /// Set the codegen fast-math flags.
5549 void SetFastMathFlags(FPOptions FPFeatures);
5550
5551 // Truncate or extend a boolean vector to the requested number of elements.
5552 llvm::Value *emitBoolVecConversion(llvm::Value *SrcVec,
5553 unsigned NumElementsDst,
5554 const llvm::Twine &Name = "");
5555
5556 void maybeAttachRangeForLoad(llvm::LoadInst *Load, QualType Ty,
5557 SourceLocation Loc);
5558
5559 // Emits a convergence_loop instruction for the given |BB|, with |ParentToken|
5560 // as it's parent convergence instr.
5561 llvm::ConvergenceControlInst *emitConvergenceLoopToken(llvm::BasicBlock *BB);
5562
5563private:
5564 // Adds a convergencectrl bundle with the current convergence token to
5565 // |Bundles|, if convergence tokens are used.
5566 void
5567 addConvergenceControlBundle(SmallVectorImpl<llvm::OperandBundleDef> &Bundles);
5568
5569 // Marks |Call| convergent if |IsConvergent| and its callee does not already
5570 // imply it.
5571 void setConvergentCallSite(llvm::CallBase *Call, bool IsConvergent);
5572
5573 // Find the convergence_entry instruction |F|, or emits ones if none exists.
5574 // Returns the convergence instruction.
5575 llvm::ConvergenceControlInst *
5576 getOrEmitConvergenceEntryToken(llvm::Function *F);
5577
5578private:
5579 llvm::MDNode *getRangeForLoadFromType(QualType Ty);
5580 void EmitReturnOfRValue(RValue RV, QualType Ty);
5581
5582 void deferPlaceholderReplacement(llvm::Instruction *Old, llvm::Value *New);
5583
5584 llvm::SmallVector<std::pair<llvm::WeakTrackingVH, llvm::Value *>, 4>
5585 DeferredReplacements;
5586
5587 /// Set the address of a local variable.
5588 void setAddrOfLocalVar(const VarDecl *VD, Address Addr) {
5589 assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!");
5590 LocalDeclMap.insert(KV: {VD, Addr});
5591 }
5592
5593 /// ExpandTypeFromArgs - Reconstruct a structure of type \arg Ty
5594 /// from function arguments into \arg Dst. See ABIArgInfo::Expand.
5595 ///
5596 /// \param AI - The first function argument of the expansion.
5597 void ExpandTypeFromArgs(QualType Ty, LValue Dst,
5598 llvm::Function::arg_iterator &AI);
5599
5600 /// ExpandTypeToArgs - Expand an CallArg \arg Arg, with the LLVM type for \arg
5601 /// Ty, into individual arguments on the provided vector \arg IRCallArgs,
5602 /// starting at index \arg IRCallArgPos. See ABIArgInfo::Expand.
5603 void ExpandTypeToArgs(QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
5604 SmallVectorImpl<llvm::Value *> &IRCallArgs,
5605 unsigned &IRCallArgPos);
5606
5607 std::pair<llvm::Value *, llvm::Type *>
5608 EmitAsmInput(const TargetInfo::ConstraintInfo &Info, const Expr *InputExpr,
5609 std::string &ConstraintStr);
5610
5611 std::pair<llvm::Value *, llvm::Type *>
5612 EmitAsmInputLValue(const TargetInfo::ConstraintInfo &Info, LValue InputValue,
5613 QualType InputType, std::string &ConstraintStr,
5614 SourceLocation Loc);
5615
5616 /// Attempts to statically evaluate the object size of E. If that
5617 /// fails, emits code to figure the size of E out for us. This is
5618 /// pass_object_size aware.
5619 ///
5620 /// If EmittedExpr is non-null, this will use that instead of re-emitting E.
5621 llvm::Value *evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
5622 llvm::IntegerType *ResType,
5623 llvm::Value *EmittedE,
5624 bool IsDynamic);
5625
5626 /// Emits the size of E, as required by __builtin_object_size. This
5627 /// function is aware of pass_object_size parameters, and will act accordingly
5628 /// if E is a parameter with the pass_object_size attribute.
5629 llvm::Value *emitBuiltinObjectSize(const Expr *E, unsigned Type,
5630 llvm::IntegerType *ResType,
5631 llvm::Value *EmittedE, bool IsDynamic);
5632
5633 llvm::Value *emitCountedBySize(const Expr *E, llvm::Value *EmittedE,
5634 unsigned Type, llvm::IntegerType *ResType);
5635
5636 llvm::Value *emitCountedByMemberSize(const MemberExpr *E, const Expr *Idx,
5637 llvm::Value *EmittedE,
5638 QualType CastedArrayElementTy,
5639 unsigned Type,
5640 llvm::IntegerType *ResType);
5641
5642 llvm::Value *emitCountedByPointerSize(const ImplicitCastExpr *E,
5643 const Expr *Idx, llvm::Value *EmittedE,
5644 QualType CastedArrayElementTy,
5645 unsigned Type,
5646 llvm::IntegerType *ResType);
5647
5648 void emitZeroOrPatternForAutoVarInit(QualType type, const VarDecl &D,
5649 Address Loc);
5650 LangOptions::TrivialAutoVarInitKind getAutoVarInitKind(QualType Ty,
5651 const VarDecl &D);
5652
5653public:
5654 using EvaluationOrder = CodeGenUtils::EvaluationOrder;
5655
5656 // Wrapper for function prototype sources. Wraps either a FunctionProtoType or
5657 // an ObjCMethodDecl.
5658 struct PrototypeWrapper {
5659 llvm::PointerUnion<const FunctionProtoType *, const ObjCMethodDecl *> P;
5660
5661 PrototypeWrapper(const FunctionProtoType *FT) : P(FT) {}
5662 PrototypeWrapper(const ObjCMethodDecl *MD) : P(MD) {}
5663 };
5664
5665 void EmitCallArgs(CallArgList &Args, PrototypeWrapper Prototype,
5666 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
5667 AbstractCallee AC = AbstractCallee(),
5668 unsigned ParamsToSkip = 0,
5669 EvaluationOrder Order = EvaluationOrder::Default);
5670
5671 /// EmitPointerWithAlignment - Given an expression with a pointer type,
5672 /// emit the value and compute our best estimate of the alignment of the
5673 /// pointee.
5674 ///
5675 /// \param BaseInfo - If non-null, this will be initialized with
5676 /// information about the source of the alignment and the may-alias
5677 /// attribute. Note that this function will conservatively fall back on
5678 /// the type when it doesn't recognize the expression and may-alias will
5679 /// be set to false.
5680 ///
5681 /// One reasonable way to use this information is when there's a language
5682 /// guarantee that the pointer must be aligned to some stricter value, and
5683 /// we're simply trying to ensure that sufficiently obvious uses of under-
5684 /// aligned objects don't get miscompiled; for example, a placement new
5685 /// into the address of a local variable. In such a case, it's quite
5686 /// reasonable to just ignore the returned alignment when it isn't from an
5687 /// explicit source.
5688 Address
5689 EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo = nullptr,
5690 TBAAAccessInfo *TBAAInfo = nullptr,
5691 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);
5692
5693 /// If \p E references a parameter with pass_object_size info or a constant
5694 /// array size modifier, emit the object size divided by the size of \p EltTy.
5695 /// Otherwise return null.
5696 llvm::Value *LoadPassedObjectSize(const Expr *E, QualType EltTy);
5697
5698 void EmitSanitizerStatReport(llvm::SanitizerStatKind SSK);
5699
5700 struct FMVResolverOption {
5701 llvm::Function *Function;
5702 llvm::SmallVector<StringRef, 8> Features;
5703 std::optional<StringRef> Architecture;
5704
5705 FMVResolverOption(llvm::Function *F, ArrayRef<StringRef> Feats,
5706 std::optional<StringRef> Arch = std::nullopt)
5707 : Function(F), Features(Feats), Architecture(Arch) {}
5708 };
5709
5710 // Emits the body of a multiversion function's resolver. Assumes that the
5711 // options are already sorted in the proper order, with the 'default' option
5712 // last (if it exists).
5713 void EmitMultiVersionResolver(llvm::Function *Resolver,
5714 ArrayRef<FMVResolverOption> Options);
5715 void EmitX86MultiVersionResolver(llvm::Function *Resolver,
5716 ArrayRef<FMVResolverOption> Options);
5717 void EmitAArch64MultiVersionResolver(llvm::Function *Resolver,
5718 ArrayRef<FMVResolverOption> Options);
5719 void EmitRISCVMultiVersionResolver(llvm::Function *Resolver,
5720 ArrayRef<FMVResolverOption> Options);
5721 void EmitPPCAIXMultiVersionResolver(llvm::Function *Resolver,
5722 ArrayRef<FMVResolverOption> Options);
5723
5724 Address EmitAddressOfPFPField(Address RecordPtr, const PFPField &Field);
5725 Address EmitAddressOfPFPField(Address RecordPtr, Address FieldPtr,
5726 const FieldDecl *Field);
5727
5728private:
5729 QualType getVarArgType(const Expr *Arg);
5730
5731 void EmitDeclMetadata();
5732
5733 BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType,
5734 const AutoVarEmission &emission);
5735
5736 void AddObjCARCExceptionMetadata(llvm::Instruction *Inst);
5737
5738 llvm::Value *GetValueForARMHint(unsigned BuiltinID);
5739 llvm::Value *EmitX86CpuIs(const CallExpr *E);
5740 llvm::Value *EmitX86CpuIs(StringRef CPUStr);
5741 llvm::Value *EmitX86CpuSupports(const CallExpr *E);
5742 llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs);
5743 llvm::Value *EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask);
5744 llvm::Value *EmitX86CpuInit();
5745 llvm::Value *FormX86ResolverCondition(const FMVResolverOption &RO);
5746 llvm::Value *EmitAArch64CpuInit();
5747 llvm::Value *FormAArch64ResolverCondition(const FMVResolverOption &RO);
5748 llvm::Value *EmitAArch64CpuSupports(const CallExpr *E);
5749 llvm::Value *EmitAArch64CpuSupports(ArrayRef<StringRef> FeatureStrs);
5750};
5751
5752inline DominatingLLVMValue::saved_type
5753DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) {
5754 if (!needsSaving(value))
5755 return saved_type(value);
5756
5757 // Otherwise, we need an alloca.
5758 auto align = CharUnits::fromQuantity(
5759 Quantity: CGF.CGM.getDataLayout().getPrefTypeAlign(Ty: value->getType()))
5760 .getAsAlign();
5761 llvm::AllocaInst *AI =
5762 CGF.CreateTempAlloca(Ty: value->getType(), Name: "cond-cleanup.save");
5763 AI->setAlignment(align);
5764 CGF.Builder.CreateAlignedStore(Val: value, Ptr: AI, Align: align);
5765
5766 return saved_type(AI, value->getType());
5767}
5768
5769inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF,
5770 saved_type value) {
5771 // If the value says it wasn't saved, trust that it's still dominating.
5772 if (!value.isSaved())
5773 return value.Value;
5774
5775 // Otherwise, it should be an alloca instruction, as set up in save().
5776 auto Alloca = cast<llvm::AllocaInst>(Val: value.Value);
5777 return CGF.Builder.CreateAlignedLoad(Ty: value.Type, Ptr: Alloca, Align: Alloca->getAlign());
5778}
5779
5780} // end namespace CodeGen
5781
5782// Map the LangOption for floating point exception behavior into
5783// the corresponding enum in the IR.
5784llvm::fp::ExceptionBehavior
5785ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind);
5786} // end namespace clang
5787
5788#endif
5789