1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Internal per-function state used for AST-to-ClangIR code gen
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef CLANG_LIB_CIR_CODEGEN_CIRGENFUNCTION_H
14#define CLANG_LIB_CIR_CODEGEN_CIRGENFUNCTION_H
15
16#include "CIRGenBuilder.h"
17#include "CIRGenCall.h"
18#include "CIRGenModule.h"
19#include "CIRGenTypeCache.h"
20#include "CIRGenValue.h"
21#include "EHScopeStack.h"
22
23#include "Address.h"
24
25#include "clang/AST/ASTContext.h"
26#include "clang/AST/BaseSubobject.h"
27#include "clang/AST/CharUnits.h"
28#include "clang/AST/CurrentSourceLocExprScope.h"
29#include "clang/AST/Decl.h"
30#include "clang/AST/ExprCXX.h"
31#include "clang/AST/Stmt.h"
32#include "clang/AST/Type.h"
33#include "clang/Basic/OpenACCKinds.h"
34#include "clang/Basic/OperatorKinds.h"
35#include "clang/Basic/TargetBuiltins.h"
36#include "clang/CIR/Dialect/IR/CIRDialect.h"
37#include "clang/CIR/MissingFeatures.h"
38#include "clang/CIR/TypeEvaluationKind.h"
39#include "clang/CodeGenUtils/CallUtils.h"
40#include "clang/CodeGenUtils/StmtUtils.h"
41#include "clang/CodeGenUtils/TargetUtils.h"
42#include "llvm/ADT/ScopedHashTable.h"
43#include "llvm/IR/Instructions.h"
44#include "llvm/IR/Intrinsics.h"
45
46namespace mlir {
47namespace acc {
48class LoopOp;
49} // namespace acc
50} // namespace mlir
51
52namespace clang {
53class OutlinedFunctionDecl;
54class SYCLKernelCallStmt;
55} // namespace clang
56
57namespace clang::CIRGen {
58
59struct CGCoroData;
60
61class CIRGenFunction : public CIRGenTypeCache {
62public:
63 CIRGenModule &cgm;
64
65private:
66 /// The builder is a helper class to create IR inside a function. The
67 /// builder is stateful, in particular it keeps an "insertion point": this
68 /// is where the next operations will be introduced.
69 CIRGenBuilderTy &builder;
70
71 /// Saves the builder's constrained floating-point configuration on
72 /// construction and restores it on destruction. The builder is shared
73 /// across all functions in the module, so its constrained-FP state must be
74 /// scoped to each function's emission.
75 ///
76 /// Note that the similarly named CIRGenFunction::CIRGenFPOptionsRAII
77 /// intentionally avoids restoring the "isFPConstrained" state because that
78 /// state is function-wide, but this object does restore the state so that
79 /// any CIRGenFunction instance created while we are in the process of
80 /// emitting another function cannot corrupt the prior functions state.
81 struct ConstrainedFPRAII {
82 CIRGenBuilderTy &builder;
83 bool savedIsFPConstrained;
84 LangOptions::FPExceptionModeKind savedExcept;
85 llvm::RoundingMode savedRounding;
86 cir::FastMathFlags savedFastMathFlags;
87
88 explicit ConstrainedFPRAII(CIRGenBuilderTy &builder)
89 : builder(builder), savedIsFPConstrained(builder.getIsFPConstrained()),
90 savedExcept(builder.getDefaultConstrainedExcept()),
91 savedRounding(builder.getDefaultConstrainedRounding()),
92 savedFastMathFlags(builder.getFastMathFlags()) {}
93 ~ConstrainedFPRAII() {
94 builder.setIsFPConstrained(savedIsFPConstrained);
95 builder.setDefaultConstrainedExcept(savedExcept);
96 builder.setDefaultConstrainedRounding(savedRounding);
97 builder.setFastMathFlags(savedFastMathFlags);
98 }
99 } constrainedFPState{builder};
100
101public:
102 /// The GlobalDecl for the current function being compiled or the global
103 /// variable currently being initialized.
104 clang::GlobalDecl curGD;
105
106 unsigned nextCleanupDestIndex = 1;
107
108 /// The compiler-generated variable that holds the return value.
109 std::optional<mlir::Value> fnRetAlloca;
110
111 // Holds coroutine data if the current function is a coroutine. We use a
112 // wrapper to manage its lifetime, so that we don't have to define CGCoroData
113 // in this header.
114 struct CGCoroInfo {
115 std::unique_ptr<CGCoroData> data;
116 CGCoroInfo();
117 ~CGCoroInfo();
118 };
119 CGCoroInfo curCoro;
120
121 bool isCoroutine() const { return curCoro.data != nullptr; }
122
123 /// The temporary alloca to hold the return value. This is
124 /// invalid iff the function has no return value.
125 Address returnValue = Address::invalid();
126
127 /// Tracks function scope overall cleanup handling.
128 EHScopeStack ehStack;
129
130 typedef void Destroyer(CIRGenFunction &cgf, Address addr, QualType ty);
131
132 class CapturedCleanups;
133
134 /// A cleanup whose destructor call is not emitted where the cleanup is
135 /// registered. Used by the lifetime-extended cleanup stack, whose entries
136 /// are later promoted onto the EH scope stack, and by the deferred
137 /// conditional cleanup stack, whose entries are emitted directly into a
138 /// conditional scope's cleanup region.
139 ///
140 /// A valid \c activeFlag means the cleanup is conditional. The flag tracks
141 /// at run time whether the object has been constructed, and guards the
142 /// destructor call.
143 ///
144 /// Currently only DestroyObject cleanups use this. When other cleanup types
145 /// are needed (e.g., CallLifetimeEnd), this struct can be extended with a
146 /// std::variant of cleanup data types.
147 struct PendingCleanupEntry {
148 CleanupKind kind;
149 Address addr;
150 QualType type;
151 Destroyer *destroyer;
152 Address activeFlag = Address::invalid();
153 /// If set, the cleanup is captured by this owner instead of opening a
154 /// cir.cleanup.scope when it is pushed onto the EH stack.
155 CapturedCleanups *capturedBy = nullptr;
156 };
157
158 llvm::SmallVector<PendingCleanupEntry> lifetimeExtendedCleanupStack;
159
160 /// Cleanups for temporaries constructed inside a conditional.
161 ///
162 /// Temporaries are destroyed in reverse order of construction at the end of
163 /// the full expression. A cleanup lives in the cleanup region of a
164 /// cir.cleanup.scope, and those scopes nest, so nesting is what encodes the
165 /// order.
166 ///
167 /// For an unconditionally constructed temporary, EHScopeStack::pushCleanup
168 /// opens a scope where the temporary is constructed and leaves the builder in
169 /// its body, so each later temporary nests one level deeper and is destroyed
170 /// first.
171 ///
172 /// For a temporary constructed inside a conditional, pushFullExprCleanup
173 /// records the cleanup on this stack, paired with an active flag that is
174 /// cleared ahead of the conditional and set once the object is constructed.
175 /// The destructor call is emitted guarded by that flag, so it runs only on
176 /// the paths that constructed the object.
177 ///
178 /// ConditionalEvaluation opens the region holding those calls where the
179 /// conditional begins, nesting it inside the scopes of earlier temporaries
180 /// and outside later ones. Its body stays open for the rest of the full
181 /// expression, so its cleanup region runs at the end of it.
182 /// FullExprCleanupScope::exit closes these innermost first, then pops the
183 /// enclosing EH-stack cleanups.
184 llvm::SmallVector<PendingCleanupEntry> deferredConditionalCleanupStack;
185
186 /// One record per cir.cleanup.scope opened by a ConditionalEvaluation, as
187 /// described above. These are bookkeeping owned by CIRGenFunction, not RAII
188 /// objects. The scope outlives the ConditionalEvaluation that opened it,
189 /// and the enclosing FullExprCleanupScope is what closes it.
190 struct ConditionalCleanupScope {
191 cir::CleanupScopeOp scope;
192
193 /// The size of deferredConditionalCleanupStack when this scope was
194 /// opened. Every open conditional scope shares that one stack, so the
195 /// entries from this index onward are the ones deferred from inside this
196 /// conditional. They are emitted into this scope's cleanup region, in
197 /// reverse, when it is closed.
198 size_t deferredCleanupStackSize;
199 };
200 llvm::SmallVector<ConditionalCleanupScope> conditionalCleanupScopes;
201
202 /// The cir.cleanup.scope of the innermost FullExprCleanupScope that
203 /// materialized one, or null. ConditionalEvaluation opens a conditional
204 /// cleanup scope only while this is set, since a FullExprCleanupScope is
205 /// what closes those scopes.
206 cir::CleanupScopeOp currentFullExprCleanupScope = nullptr;
207
208 /// Set by emitAutoVarDeclWithCapturedCleanups while it emits the initializer
209 /// of a variable, such as a loop condition variable, that lifetime-extends
210 /// temporaries with a destructor.
211 /// The cleanups of the temporaries that \c extendingDecl extends are
212 /// captured by \c cleanups, e.g. for the loop's cleanup region, and guarded
213 /// by \c activeFlag, instead of opening cir.cleanup.scope operations around
214 /// the code that follows the declaration.
215 struct LifetimeExtensionCapture {
216 const clang::ValueDecl *extendingDecl;
217 CapturedCleanups *cleanups;
218 Address activeFlag;
219 };
220 const LifetimeExtensionCapture *lifetimeExtensionCapture = nullptr;
221
222 /// A cleanup that was pushed to the EH stack but whose deactivation is
223 /// deferred until the enclosing CleanupDeactivationScope exits. Used to
224 /// protect partially-constructed aggregates (e.g. lambda captures) so that
225 /// already-initialized sub-objects are destroyed if a later initializer
226 /// throws, while avoiding double-destruction after full construction.
227 struct DeferredDeactivateCleanup {
228 EHScopeStack::stable_iterator cleanup;
229 mlir::Operation *dominatingIP;
230 };
231 llvm::SmallVector<DeferredDeactivateCleanup> deferredDeactivationCleanupStack;
232
233 /// Scope that deactivates all enclosed deferred cleanups on exit.
234 /// Mirrors CodeGenFunction::CleanupDeactivationScope in classic codegen.
235 struct CleanupDeactivationScope {
236 CIRGenFunction &cgf;
237 size_t oldDeactivateCleanupStackSize;
238 bool deactivated = false;
239
240 CleanupDeactivationScope(CIRGenFunction &cgf)
241 : cgf(cgf), oldDeactivateCleanupStackSize(
242 cgf.deferredDeactivationCleanupStack.size()) {}
243
244 void forceDeactivate() {
245 assert(!deactivated && "Deactivating already deactivated scope");
246 auto &stack = cgf.deferredDeactivationCleanupStack;
247 for (size_t i = stack.size(); i > oldDeactivateCleanupStackSize; i--) {
248 cgf.deactivateCleanupBlock(cleanup: stack[i - 1].cleanup,
249 dominatingIP: stack[i - 1].dominatingIP);
250 stack[i - 1].dominatingIP->erase();
251 }
252 stack.resize(N: oldDeactivateCleanupStackSize);
253 deactivated = true;
254 }
255
256 ~CleanupDeactivationScope() {
257 if (!deactivated)
258 forceDeactivate();
259 }
260 };
261
262 GlobalDecl curSEHParent;
263
264 /// If a ParmVarDecl had the pass_object_size attribute, this will contain a
265 /// mapping from said ParmVarDecl to its implicit "object_size" parameter.
266 llvm::SmallDenseMap<const ParmVarDecl *, const ImplicitParamDecl *>
267 sizeArguments;
268
269 /// A mapping from NRVO variables to the flags used to indicate
270 /// when the NRVO has been applied to this variable.
271 llvm::DenseMap<const VarDecl *, mlir::Value> nrvoFlags;
272
273 llvm::DenseMap<const clang::ValueDecl *, clang::FieldDecl *>
274 lambdaCaptureFields;
275 clang::FieldDecl *lambdaThisCaptureField = nullptr;
276
277 /// CXXThisDecl - When generating code for a C++ member function,
278 /// this will hold the implicit 'this' declaration.
279 ImplicitParamDecl *cxxabiThisDecl = nullptr;
280 mlir::Value cxxabiThisValue = nullptr;
281 mlir::Value cxxThisValue = nullptr;
282 clang::CharUnits cxxabiThisAlignment;
283 clang::CharUnits cxxThisAlignment;
284
285 /// When generating code for a constructor or destructor, this will hold the
286 /// implicit argument (e.g. VTT).
287 ImplicitParamDecl *cxxStructorImplicitParamDecl{};
288 mlir::Value cxxStructorImplicitParamValue{};
289
290 /// The value of 'this' to sue when evaluating CXXDefaultInitExprs within this
291 /// expression.
292 Address cxxDefaultInitExprThis = Address::invalid();
293
294 /// The values of function arguments to use when evaluating
295 /// CXXInheritedCtorInitExprs within this context.
296 CallArgList cxxInheritedCtorInitExprArgs;
297
298 /// The current array initialization index when evaluating an
299 /// ArrayInitIndexExpr within an ArrayInitLoopExpr.
300 mlir::Value arrayInitIndex = nullptr;
301
302 // Holds the Decl for the current outermost non-closure context
303 const clang::Decl *curFuncDecl = nullptr;
304 /// This is the inner-most code context, which includes blocks.
305 const clang::Decl *curCodeDecl = nullptr;
306 const CIRGenFunctionInfo *curFnInfo = nullptr;
307 QualType fnRetTy;
308
309 /// The current function or global initializer that is generated code for.
310 /// This is usually a cir::FuncOp, but it can also be a cir::GlobalOp for
311 /// global initializers.
312 mlir::Operation *curFn = nullptr;
313
314 /// Save Parameter Decl for coroutine.
315 llvm::SmallVector<const ParmVarDecl *> fnArgs;
316
317 using DeclMapTy = llvm::DenseMap<const clang::Decl *, Address>;
318 /// This keeps track of the CIR allocas or globals for local C
319 /// declarations.
320 DeclMapTy localDeclMap;
321
322 /// The type of the condition for the emitting switch statement.
323 llvm::SmallVector<mlir::Type, 2> condTypeStack;
324
325 clang::ASTContext &getContext() const { return cgm.getASTContext(); }
326
327 CIRGenBuilderTy &getBuilder() { return builder; }
328
329 CIRGenModule &getCIRGenModule() { return cgm; }
330 const CIRGenModule &getCIRGenModule() const { return cgm; }
331
332 mlir::Block *getCurFunctionEntryBlock() {
333 // We currently assume this isn't called for a global initializer.
334 auto fn = mlir::cast<cir::FuncOp>(Val: curFn);
335 return &fn.getRegion().front();
336 }
337
338 /// Sanitizers enabled for this function.
339 clang::SanitizerSet sanOpts;
340
341 class CIRGenFPOptionsRAII {
342 public:
343 CIRGenFPOptionsRAII(CIRGenFunction &cgf, FPOptions FPFeatures);
344 CIRGenFPOptionsRAII(CIRGenFunction &cgf, const clang::Expr *E);
345 ~CIRGenFPOptionsRAII();
346
347 private:
348 void ConstructorHelper(clang::FPOptions FPFeatures);
349 CIRGenFunction &cgf;
350 clang::FPOptions oldFPFeatures;
351 LangOptions::FPExceptionModeKind oldExcept;
352 llvm::RoundingMode oldRounding;
353 std::optional<cir::FastMathFlags> oldFastMathFlags;
354 };
355 clang::FPOptions curFPFeatures;
356
357 void setFastMathFlags(FPOptions fpFeatures);
358
359 /// Convert the active Clang floating-point options to CIR fast-math flags.
360 cir::FastMathFlags getCurrentFastMathFlags() const;
361
362 /// Create a CIR fast-math attribute from the specified flags.
363 cir::FastMathFlagsAttr getFastMathFlagsAttr(cir::FastMathFlags fastMathFlags);
364
365 /// The symbol table maps a variable name to a value in the current scope.
366 /// Entering a function creates a new scope, and the function arguments are
367 /// added to the mapping. When the processing of a function is terminated,
368 /// the scope is destroyed and the mappings created in this scope are
369 /// dropped.
370 using SymTableTy = llvm::ScopedHashTable<const clang::Decl *, mlir::Value>;
371 SymTableTy symbolTable;
372
373 /// Whether a cir.stacksave operation has been added. Used to avoid
374 /// inserting cir.stacksave for multiple VLAs in the same scope.
375 bool didCallStackSave = false;
376
377 /// Whether or not a Microsoft-style asm block has been processed within
378 /// this fuction. These can potentially set the return value.
379 bool sawAsmBlock = false;
380
381 /// In C++, whether we are code generating a thunk. This controls whether we
382 /// should emit cleanups.
383 bool curFuncIsThunk = false;
384
385 mlir::Type convertTypeForMem(QualType t);
386
387 mlir::Type convertType(clang::QualType t);
388 mlir::Type convertType(const TypeDecl *t) {
389 return convertType(t: getContext().getTypeDeclType(Decl: t));
390 }
391
392 /// Get integer from a mlir::Value that is an int constant or a constant op.
393 static int64_t getSExtIntValueFromConstOp(mlir::Value val) {
394 auto constOp = val.getDefiningOp<cir::ConstantOp>();
395 assert(constOp && "getSExtIntValueFromConstOp call with non ConstantOp");
396 return constOp.getIntValue().getSExtValue();
397 }
398
399 /// Get zero-extended integer from a mlir::Value that is an int constant or a
400 /// constant op.
401 static int64_t getZExtIntValueFromConstOp(mlir::Value val) {
402 auto constOp = val.getDefiningOp<cir::ConstantOp>();
403 assert(constOp && "getZExtIntValueFromConstOp call with non ConstantOp");
404 return constOp.getIntValue().getZExtValue();
405 }
406
407 /// Return the cir::TypeEvaluationKind of QualType \c type.
408 static cir::TypeEvaluationKind getEvaluationKind(clang::QualType type);
409
410 static bool hasScalarEvaluationKind(clang::QualType type) {
411 return getEvaluationKind(type) == cir::TEK_Scalar;
412 }
413
414 static bool hasAggregateEvaluationKind(clang::QualType type) {
415 return getEvaluationKind(type) == cir::TEK_Aggregate;
416 }
417
418 CIRGenFunction(CIRGenModule &cgm, CIRGenBuilderTy &builder,
419 bool suppressNewContext = false);
420 ~CIRGenFunction();
421
422 CIRGenTypes &getTypes() const { return cgm.getTypes(); }
423
424 const TargetInfo &getTarget() const { return cgm.getTarget(); }
425 mlir::MLIRContext &getMLIRContext() { return cgm.getMLIRContext(); }
426
427 const TargetCIRGenInfo &getTargetHooks() const {
428 return cgm.getTargetCIRGenInfo();
429 }
430
431 // ---------------------
432 // Opaque value handling
433 // ---------------------
434
435 /// Keeps track of the current set of opaque value expressions.
436 llvm::DenseMap<const OpaqueValueExpr *, LValue> opaqueLValues;
437 llvm::DenseMap<const OpaqueValueExpr *, RValue> opaqueRValues;
438
439 // This keeps track of the associated size for each VLA type.
440 // We track this by the size expression rather than the type itself because
441 // in certain situations, like a const qualifier applied to an VLA typedef,
442 // multiple VLA types can share the same size expression.
443 // FIXME: Maybe this could be a stack of maps that is pushed/popped as we
444 // enter/leave scopes.
445 llvm::DenseMap<const Expr *, mlir::Value> vlaSizeMap;
446
447public:
448 /// A non-RAII class containing all the information about a bound
449 /// opaque value. OpaqueValueMapping, below, is a RAII wrapper for
450 /// this which makes individual mappings very simple; using this
451 /// class directly is useful when you have a variable number of
452 /// opaque values or don't want the RAII functionality for some
453 /// reason.
454 class OpaqueValueMappingData {
455 const OpaqueValueExpr *opaqueValue;
456 bool boundLValue;
457
458 OpaqueValueMappingData(const OpaqueValueExpr *ov, bool boundLValue)
459 : opaqueValue(ov), boundLValue(boundLValue) {}
460
461 public:
462 OpaqueValueMappingData() : opaqueValue(nullptr) {}
463
464 static bool shouldBindAsLValue(const Expr *expr) {
465 // gl-values should be bound as l-values for obvious reasons.
466 // Records should be bound as l-values because IR generation
467 // always keeps them in memory. Expressions of function type
468 // act exactly like l-values but are formally required to be
469 // r-values in C.
470 return expr->isGLValue() || expr->getType()->isFunctionType() ||
471 hasAggregateEvaluationKind(type: expr->getType());
472 }
473
474 static OpaqueValueMappingData
475 bind(CIRGenFunction &cgf, const OpaqueValueExpr *ov, const Expr *e) {
476 if (shouldBindAsLValue(expr: ov))
477 return bind(cgf, ov, lv: cgf.emitLValue(e));
478 return bind(cgf, ov, rv: cgf.emitAnyExpr(e));
479 }
480
481 static OpaqueValueMappingData
482 bind(CIRGenFunction &cgf, const OpaqueValueExpr *ov, const LValue &lv) {
483 assert(shouldBindAsLValue(ov));
484 cgf.opaqueLValues.insert(KV: std::make_pair(x&: ov, y: lv));
485 return OpaqueValueMappingData(ov, true);
486 }
487
488 static OpaqueValueMappingData
489 bind(CIRGenFunction &cgf, const OpaqueValueExpr *ov, const RValue &rv) {
490 assert(!shouldBindAsLValue(ov));
491 cgf.opaqueRValues.insert(KV: std::make_pair(x&: ov, y: rv));
492
493 OpaqueValueMappingData data(ov, false);
494
495 // Work around an extremely aggressive peephole optimization in
496 // EmitScalarConversion which assumes that all other uses of a
497 // value are extant.
498 assert(!cir::MissingFeatures::peepholeProtection() && "NYI");
499 return data;
500 }
501
502 bool isValid() const { return opaqueValue != nullptr; }
503 void clear() { opaqueValue = nullptr; }
504
505 void unbind(CIRGenFunction &cgf) {
506 assert(opaqueValue && "no data to unbind!");
507
508 if (boundLValue) {
509 cgf.opaqueLValues.erase(Val: opaqueValue);
510 } else {
511 cgf.opaqueRValues.erase(Val: opaqueValue);
512 assert(!cir::MissingFeatures::peepholeProtection() && "NYI");
513 }
514 }
515 };
516
517 /// An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
518 class OpaqueValueMapping {
519 CIRGenFunction &cgf;
520 OpaqueValueMappingData data;
521
522 public:
523 static bool shouldBindAsLValue(const Expr *expr) {
524 return OpaqueValueMappingData::shouldBindAsLValue(expr);
525 }
526
527 /// Build the opaque value mapping for the given conditional
528 /// operator if it's the GNU ?: extension. This is a common
529 /// enough pattern that the convenience operator is really
530 /// helpful.
531 ///
532 OpaqueValueMapping(CIRGenFunction &cgf,
533 const AbstractConditionalOperator *op)
534 : cgf(cgf) {
535 if (mlir::isa<ConditionalOperator>(Val: op))
536 // Leave Data empty.
537 return;
538
539 const BinaryConditionalOperator *e =
540 mlir::cast<BinaryConditionalOperator>(Val: op);
541 data = OpaqueValueMappingData::bind(cgf, ov: e->getOpaqueValue(),
542 e: e->getCommon());
543 }
544
545 /// Build the opaque value mapping for an OpaqueValueExpr whose source
546 /// expression is set to the expression the OVE represents.
547 OpaqueValueMapping(CIRGenFunction &cgf, const OpaqueValueExpr *ov)
548 : cgf(cgf) {
549 if (ov) {
550 assert(ov->getSourceExpr() && "wrong form of OpaqueValueMapping used "
551 "for OVE with no source expression");
552 data = OpaqueValueMappingData::bind(cgf, ov, e: ov->getSourceExpr());
553 }
554 }
555
556 OpaqueValueMapping(CIRGenFunction &cgf, const OpaqueValueExpr *opaqueValue,
557 LValue lvalue)
558 : cgf(cgf),
559 data(OpaqueValueMappingData::bind(cgf, ov: opaqueValue, lv: lvalue)) {}
560
561 OpaqueValueMapping(CIRGenFunction &cgf, const OpaqueValueExpr *opaqueValue,
562 RValue rvalue)
563 : cgf(cgf),
564 data(OpaqueValueMappingData::bind(cgf, ov: opaqueValue, rv: rvalue)) {}
565
566 void pop() {
567 data.unbind(cgf);
568 data.clear();
569 }
570
571 ~OpaqueValueMapping() {
572 if (data.isValid())
573 data.unbind(cgf);
574 }
575 };
576
577private:
578 /// Declare a variable in the current scope, return success if the variable
579 /// wasn't declared yet.
580 void declare(mlir::Value addrVal, const clang::Decl *var, clang::QualType ty,
581 mlir::Location loc, clang::CharUnits alignment,
582 bool isParam = false);
583
584public:
585 mlir::Value createDummyValue(mlir::Location loc, clang::QualType qt);
586
587 void emitNullInitialization(mlir::Location loc, Address destPtr, QualType ty);
588
589private:
590 // Track current variable initialization (if there's one)
591 const clang::VarDecl *currVarDecl = nullptr;
592 class VarDeclContext {
593 CIRGenFunction &p;
594 const clang::VarDecl *oldVal = nullptr;
595
596 public:
597 VarDeclContext(CIRGenFunction &p, const VarDecl *value) : p(p) {
598 if (p.currVarDecl)
599 oldVal = p.currVarDecl;
600 p.currVarDecl = value;
601 }
602
603 /// Can be used to restore the state early, before the dtor
604 /// is run.
605 void restore() { p.currVarDecl = oldVal; }
606 ~VarDeclContext() { restore(); }
607 };
608
609public:
610 /// Use to track source locations across nested visitor traversals.
611 /// Always use a `SourceLocRAIIObject` to change currSrcLoc.
612 std::optional<SourceRange> currSrcLoc;
613
614 class SourceLocRAIIObject {
615 CIRGenFunction &cgf;
616 std::optional<SourceRange> oldLoc;
617
618 public:
619 SourceLocRAIIObject(CIRGenFunction &cgf, SourceRange value) : cgf(cgf) {
620 if (cgf.currSrcLoc)
621 oldLoc = cgf.currSrcLoc;
622 cgf.currSrcLoc = value;
623 }
624
625 /// Can be used to restore the state early, before the dtor
626 /// is run.
627 void restore() { cgf.currSrcLoc = oldLoc; }
628 ~SourceLocRAIIObject() { restore(); }
629 };
630
631 using SymTableScopeTy =
632 llvm::ScopedHashTableScope<const clang::Decl *, mlir::Value>;
633
634 /// Hold counters for incrementally naming temporaries
635 unsigned counterRefTmp = 0;
636 unsigned counterAggTmp = 0;
637 std::string getCounterRefTmpAsString();
638 std::string getCounterAggTmpAsString();
639
640 /// Helpers to convert Clang's SourceLocation to a MLIR Location.
641 mlir::Location getLoc(clang::SourceLocation srcLoc);
642 mlir::Location getLoc(clang::SourceRange srcLoc);
643 mlir::Location getLoc(mlir::Location lhs, mlir::Location rhs);
644
645 const clang::LangOptions &getLangOpts() const { return cgm.getLangOpts(); }
646
647 bool checkIfFunctionMustProgress() {
648 if (cgm.getCodeGenOpts().getFiniteLoops() ==
649 clang::CodeGenOptions::FiniteLoopsKind::Never)
650 return false;
651
652 // C++11 and later guarantees that a thread eventually will do one of the
653 // following (C++11 [intro.multithread]p24 and C++17 [intro.progress]p1):
654 // - terminate,
655 // - make a call to a library I/O function,
656 // - perform an access through a volatile glvalue, or
657 // - perform a synchronization operation or an atomic operation.
658 //
659 // Hence each function is 'mustprogress' in C++11 or later.
660 return getLangOpts().CPlusPlus11;
661 }
662
663 // [C++26][intro.progress] (DR)
664 // The implementation may assume that any thread will eventually do one of
665 // the following:
666 // [...]
667 // - continue execution of a trivial infinite loop ([stmt.iter.general]).
668 //
669 // A trivial infinite loop with no side effects (e.g. `while (true) {}`)
670 // is therefore exempt from the forward-progress guarantee. If such a loop
671 // is found in a function that was speculatively marked 'mustprogress',
672 // that attribute must be removed: otherwise the optimizer would be
673 // licensed to assume the loop terminates and could delete it.
674 bool checkIfLoopMustProgress(const clang::Expr *controllingExpression,
675 bool hasEmptyBody) {
676 return clang::CodeGenUtils::checkIfLoopMustProgress(
677 LangOpts: getLangOpts(), CGOpts: cgm.getCodeGenOpts(), Ctx&: getContext(),
678 ControllingExpression: controllingExpression, HasEmptyBody: hasEmptyBody, RemoveMustProgress: [this] {
679 curFn->removeAttr(name: cir::CIRDialect::getMustProgressAttrName());
680 });
681 }
682
683 /// True if an insertion point is defined. If not, this indicates that the
684 /// current code being emitted is unreachable.
685 /// FIXME(cir): we need to inspect this and perhaps use a cleaner mechanism
686 /// since we don't yet force null insertion point to designate behavior (like
687 /// LLVM's codegen does) and we probably shouldn't.
688 bool haveInsertPoint() const {
689 return builder.getInsertionBlock() != nullptr;
690 }
691
692 /// True if code emitted at the builder's insertion point can be reached.
693 /// After emitting a terminator CIRGen opens a fresh block to continue in, so
694 /// the insertion block holds unreachable code whenever it is neither its
695 /// region's entry block nor the target of a branch.
696 bool insertionPointIsReachable() const {
697 mlir::Block *block = builder.getInsertionBlock();
698 return block && (block->isEntryBlock() || !block->hasNoPredecessors());
699 }
700
701 // Wrapper for function prototype sources. Wraps either a FunctionProtoType or
702 // an ObjCMethodDecl.
703 struct PrototypeWrapper {
704 llvm::PointerUnion<const clang::FunctionProtoType *,
705 const clang::ObjCMethodDecl *>
706 p;
707
708 PrototypeWrapper(const clang::FunctionProtoType *ft) : p(ft) {}
709 PrototypeWrapper(const clang::ObjCMethodDecl *md) : p(md) {}
710 };
711
712 using EvaluationOrder = CodeGenUtils::EvaluationOrder;
713
714 bool isLValueSuitableForInlineAtomic(LValue lv);
715
716 RValue emitAtomicLoad(LValue lvalue, SourceLocation loc,
717 AggValueSlot slot = AggValueSlot::ignored());
718 RValue emitAtomicLoad(LValue lvalue, SourceLocation loc, cir::MemOrder order,
719 bool isVolatile = false,
720 AggValueSlot slot = AggValueSlot::ignored());
721
722 /// An abstract representation of regular/ObjC call/message targets.
723 class AbstractCallee {
724 /// The function declaration of the callee.
725 [[maybe_unused]] const clang::Decl *calleeDecl;
726
727 public:
728 AbstractCallee() : calleeDecl(nullptr) {}
729 AbstractCallee(const clang::FunctionDecl *fd) : calleeDecl(fd) {}
730
731 bool hasFunctionDecl() const {
732 return llvm::isa_and_nonnull<clang::FunctionDecl>(Val: calleeDecl);
733 }
734
735 const clang::Decl *getDecl() const { return calleeDecl; }
736
737 unsigned getNumParams() const {
738 if (const auto *fd = llvm::dyn_cast<clang::FunctionDecl>(Val: calleeDecl))
739 return fd->getNumParams();
740 return llvm::cast<clang::ObjCMethodDecl>(Val: calleeDecl)->param_size();
741 }
742
743 const clang::ParmVarDecl *getParamDecl(unsigned I) const {
744 if (const auto *fd = llvm::dyn_cast<clang::FunctionDecl>(Val: calleeDecl))
745 return fd->getParamDecl(i: I);
746 return *(llvm::cast<clang::ObjCMethodDecl>(Val: calleeDecl)->param_begin() +
747 I);
748 }
749 };
750
751 /// True if the current statement has noinline attribute.
752 bool inNoInlineAttributedStmt = false;
753
754 /// True if the current statement has always_inline attribute.
755 bool inAlwaysInlineAttributedStmt = false;
756
757 // The CallExpr within the current statement that the musttail attribute
758 // applies to. nullptr if there is no 'musttail' on the current statement.
759 const CallExpr *mustTailCall = nullptr;
760
761 struct VlaSizePair {
762 mlir::Value numElts;
763 QualType type;
764
765 VlaSizePair(mlir::Value num, QualType ty) : numElts(num), type(ty) {}
766 };
767
768 /// Return the number of elements for a single dimension
769 /// for the given array type.
770 VlaSizePair getVLAElements1D(const VariableArrayType *vla);
771
772 /// Returns an MLIR::Value+QualType pair that corresponds to the size,
773 /// in non-variably-sized elements, of a variable length array type,
774 /// plus that largest non-variably-sized element type. Assumes that
775 /// the type has already been emitted with emitVariablyModifiedType.
776 VlaSizePair getVLASize(const VariableArrayType *type);
777 VlaSizePair getVLASize(QualType type);
778
779 Address getAsNaturalAddressOf(Address addr, QualType pointeeTy);
780
781 mlir::Value getAsNaturalPointerTo(Address addr, QualType pointeeType) {
782 return getAsNaturalAddressOf(addr, pointeeTy: pointeeType).getBasePointer();
783 }
784
785 void finishFunction(SourceLocation endLoc);
786
787 /// Determine whether the given initializer is trivial in the sense
788 /// that it requires no code to be generated.
789 bool isTrivialInitializer(const Expr *init);
790
791 /// If the specified expression does not fold to a constant, or if it does but
792 /// contains a label, return false. If it constant folds return true and set
793 /// the boolean result in Result.
794 bool constantFoldsToBool(const clang::Expr *cond, bool &resultBool,
795 bool allowLabels = false);
796 bool constantFoldsToSimpleInteger(const clang::Expr *cond,
797 llvm::APSInt &resultInt,
798 bool allowLabels = false);
799
800 /// Return true if the statement contains a label in it. If
801 /// this statement is not executed normally, it not containing a label means
802 /// that we can just remove the code.
803 bool containsLabel(const clang::Stmt *s, bool ignoreCaseStmts = false);
804
805 Address emitExtVectorElementLValue(LValue lv, mlir::Location loc);
806
807 class ConstantEmission {
808 // Cannot use mlir::TypedAttr directly here because of bit availability.
809 llvm::PointerIntPair<mlir::Attribute, 1, bool> valueAndIsReference;
810 ConstantEmission(mlir::TypedAttr c, bool isReference)
811 : valueAndIsReference(c, isReference) {}
812
813 public:
814 ConstantEmission() {}
815 static ConstantEmission forReference(mlir::TypedAttr c) {
816 return ConstantEmission(c, true);
817 }
818 static ConstantEmission forValue(mlir::TypedAttr c) {
819 return ConstantEmission(c, false);
820 }
821
822 explicit operator bool() const {
823 return valueAndIsReference.getOpaqueValue() != nullptr;
824 }
825
826 bool isReference() const { return valueAndIsReference.getInt(); }
827 LValue getReferenceLValue(CIRGenFunction &cgf, const Expr *refExpr) const {
828 assert(isReference());
829 mlir::Value addr = cgf.getBuilder().getConstant(
830 loc: cgf.getLoc(srcLoc: refExpr->getSourceRange()),
831 attr: mlir::cast<mlir::TypedAttr>(Val: valueAndIsReference.getPointer()));
832 return cgf.makeNaturalAlignAddrLValue(val: addr, ty: refExpr->getType());
833 }
834
835 mlir::TypedAttr getValue() const {
836 assert(!isReference());
837 return mlir::cast<mlir::TypedAttr>(Val: valueAndIsReference.getPointer());
838 }
839 };
840
841 ConstantEmission tryEmitAsConstant(const DeclRefExpr *refExpr);
842 ConstantEmission tryEmitAsConstant(const MemberExpr *me);
843
844 struct AutoVarEmission {
845 const clang::VarDecl *variable;
846 /// The address of the alloca for languages with explicit address space
847 /// (e.g. OpenCL) or alloca casted to generic pointer for address space
848 /// agnostic languages (e.g. C++). Invalid if the variable was emitted
849 /// as a global constant.
850 Address addr;
851
852 /// True if the variable is of aggregate type and has a constant
853 /// initializer.
854 bool isConstantAggregate = false;
855
856 /// True if the variable is a __block variable that is captured by an
857 /// escaping block.
858 bool isEscapingByRef = false;
859
860 /// True if the variable was emitted as an offload recipe, and thus doesn't
861 /// have the same sort of alloca initialization.
862 bool emittedAsOffload = false;
863
864 /// True if lifetime op should be used.
865 bool useLifetimeMarkers = false;
866
867 mlir::Value nrvoFlag{};
868
869 struct Invalid {};
870 AutoVarEmission(Invalid) : variable(nullptr), addr(Address::invalid()) {}
871
872 AutoVarEmission(const clang::VarDecl &variable)
873 : variable(&variable), addr(Address::invalid()) {}
874
875 static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }
876
877 bool wasEmittedAsGlobal() const { return !addr.isValid(); }
878
879 bool wasEmittedAsOffloadClause() const { return emittedAsOffload; }
880
881 /// Returns the raw, allocated address, which is not necessarily
882 /// the address of the object itself. It is casted to default
883 /// address space for address space agnostic languages.
884 Address getAllocatedAddress() const { return addr; }
885
886 // Changes the stored address for the emission. This function should only
887 // be used in extreme cases, and isn't required to model normal AST
888 // initialization/variables.
889 void setAllocatedAddress(Address a) { addr = a; }
890
891 /// Returns the address of the object within this declaration.
892 /// Note that this does not chase the forwarding pointer for
893 /// __block decls.
894 Address getObjectAddress(CIRGenFunction &cgf) const {
895 if (!isEscapingByRef)
896 return addr;
897
898 assert(!cir::MissingFeatures::opAllocaEscapeByReference());
899 return Address::invalid();
900 }
901 };
902
903 /// Perform the usual unary conversions on the specified expression and
904 /// compare the result against zero, returning an Int1Ty value.
905 mlir::Value evaluateExprAsBool(const clang::Expr *e);
906
907 cir::GlobalOp addInitializerToStaticVarDecl(const VarDecl &d,
908 cir::GlobalOp gv,
909 cir::GetGlobalOp gvAddr);
910
911 /// Enter the cleanups necessary to complete the given phase of destruction
912 /// for a destructor. The end result should call destructors on members and
913 /// base classes in reverse order of their construction.
914 void enterDtorCleanups(const CXXDestructorDecl *dtor, CXXDtorType type);
915
916 /// Return the pointer to pass to the operator delete of the given
917 /// destructor, converted to the type of its first parameter if needed.
918 mlir::Value loadThisForDtorDelete(const CXXDestructorDecl *dd);
919
920 /// Determines whether an EH cleanup is required to destroy a type
921 /// with the given destruction kind.
922 /// TODO(cir): could be shared with Clang LLVM codegen
923 bool needsEHCleanup(QualType::DestructionKind kind) {
924 switch (kind) {
925 case QualType::DK_none:
926 return false;
927 case QualType::DK_cxx_destructor:
928 case QualType::DK_objc_weak_lifetime:
929 case QualType::DK_nontrivial_c_struct:
930 return getLangOpts().Exceptions;
931 case QualType::DK_objc_strong_lifetime:
932 return getLangOpts().Exceptions &&
933 cgm.getCodeGenOpts().ObjCAutoRefCountExceptions;
934 }
935 llvm_unreachable("bad destruction kind");
936 }
937
938 CleanupKind getCleanupKind(QualType::DestructionKind kind) {
939 return needsEHCleanup(kind) ? NormalAndEHCleanup : NormalCleanup;
940 }
941
942 void pushStackRestore(CleanupKind kind, Address spMem);
943
944 /// Set the address of a local variable.
945 void setAddrOfLocalVar(const clang::VarDecl *vd, Address addr) {
946 assert(!localDeclMap.count(vd) && "Decl already exists in LocalDeclMap!");
947 localDeclMap.insert(KV: {vd, addr});
948
949 // Add to the symbol table if not there already.
950 if (symbolTable.count(Key: vd))
951 return;
952 symbolTable.insert(Key: vd, Val: addr.getPointer());
953 }
954
955 // Replaces the address of the local variable, if it exists. Else does the
956 // same thing as setAddrOfLocalVar.
957 void replaceAddrOfLocalVar(const clang::VarDecl *vd, Address addr) {
958 localDeclMap.insert_or_assign(Key: vd, Val&: addr);
959 }
960
961 // A class to allow reverting changes to a var-decl's registration to the
962 // localDeclMap. This is used in cases where things are being inserted into
963 // the variable list but don't follow normal lookup/search rules, like in
964 // OpenACC recipe generation.
965 class DeclMapRevertingRAII {
966 CIRGenFunction &cgf;
967 const VarDecl *vd;
968 bool shouldDelete = false;
969 Address oldAddr = Address::invalid();
970
971 public:
972 DeclMapRevertingRAII(CIRGenFunction &cgf, const VarDecl *vd)
973 : cgf(cgf), vd(vd) {
974 auto mapItr = cgf.localDeclMap.find(Val: vd);
975
976 if (mapItr != cgf.localDeclMap.end())
977 oldAddr = mapItr->second;
978 else
979 shouldDelete = true;
980 }
981
982 ~DeclMapRevertingRAII() {
983 if (shouldDelete)
984 cgf.localDeclMap.erase(Val: vd);
985 else
986 cgf.localDeclMap.insert_or_assign(Key: vd, Val&: oldAddr);
987 }
988 };
989
990 bool shouldNullCheckClassCastValue(const CastExpr *ce);
991
992 RValue convertTempToRValue(Address addr, clang::QualType type,
993 clang::SourceLocation loc);
994
995 static bool
996 isConstructorDelegationValid(const clang::CXXConstructorDecl *ctor);
997
998 struct VPtr {
999 clang::BaseSubobject base;
1000 const clang::CXXRecordDecl *nearestVBase;
1001 clang::CharUnits offsetFromNearestVBase;
1002 const clang::CXXRecordDecl *vtableClass;
1003 };
1004
1005 using VisitedVirtualBasesSetTy =
1006 llvm::SmallPtrSet<const clang::CXXRecordDecl *, 4>;
1007
1008 using VPtrsVector = llvm::SmallVector<VPtr, 4>;
1009 VPtrsVector getVTablePointers(const clang::CXXRecordDecl *vtableClass);
1010 void getVTablePointers(clang::BaseSubobject base,
1011 const clang::CXXRecordDecl *nearestVBase,
1012 clang::CharUnits offsetFromNearestVBase,
1013 bool baseIsNonVirtualPrimaryBase,
1014 const clang::CXXRecordDecl *vtableClass,
1015 VisitedVirtualBasesSetTy &vbases, VPtrsVector &vptrs);
1016 /// Return the Value of the vtable pointer member pointed to by thisAddr.
1017 mlir::Value getVTablePtr(mlir::Location loc, Address thisAddr,
1018 const clang::CXXRecordDecl *vtableClass);
1019
1020 /// Returns whether we should perform a type checked load when loading a
1021 /// virtual function for virtual calls to members of RD. This is generally
1022 /// true when both vcall CFI and whole-program-vtables are enabled.
1023 bool shouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *rd);
1024
1025 /// Source location information about the default argument or member
1026 /// initializer expression we're evaluating, if any.
1027 clang::CurrentSourceLocExprScope curSourceLocExprScope;
1028 using SourceLocExprScopeGuard =
1029 clang::CurrentSourceLocExprScope::SourceLocExprScopeGuard;
1030
1031 /// A scope within which we are constructing the fields of an object which
1032 /// might use a CXXDefaultInitExpr. This stashes away a 'this' value to use if
1033 /// we need to evaluate the CXXDefaultInitExpr within the evaluation.
1034 class FieldConstructionScope {
1035 public:
1036 FieldConstructionScope(CIRGenFunction &cgf, Address thisAddr)
1037 : cgf(cgf), oldCXXDefaultInitExprThis(cgf.cxxDefaultInitExprThis) {
1038 cgf.cxxDefaultInitExprThis = thisAddr;
1039 }
1040 ~FieldConstructionScope() {
1041 cgf.cxxDefaultInitExprThis = oldCXXDefaultInitExprThis;
1042 }
1043
1044 private:
1045 CIRGenFunction &cgf;
1046 Address oldCXXDefaultInitExprThis;
1047 };
1048
1049 /// The scope of a CXXDefaultInitExpr. Within this scope, the value of 'this'
1050 /// is overridden to be the object under construction.
1051 class CXXDefaultInitExprScope {
1052 public:
1053 CXXDefaultInitExprScope(CIRGenFunction &cgf, const CXXDefaultInitExpr *e)
1054 : cgf{cgf}, oldCXXThisValue(cgf.cxxThisValue),
1055 oldCXXThisAlignment(cgf.cxxThisAlignment),
1056 sourceLocScope(e, cgf.curSourceLocExprScope) {
1057 cgf.cxxThisValue = cgf.cxxDefaultInitExprThis.getPointer();
1058 cgf.cxxThisAlignment = cgf.cxxDefaultInitExprThis.getAlignment();
1059 }
1060 ~CXXDefaultInitExprScope() {
1061 cgf.cxxThisValue = oldCXXThisValue;
1062 cgf.cxxThisAlignment = oldCXXThisAlignment;
1063 }
1064
1065 public:
1066 CIRGenFunction &cgf;
1067 mlir::Value oldCXXThisValue;
1068 clang::CharUnits oldCXXThisAlignment;
1069 SourceLocExprScopeGuard sourceLocScope;
1070 };
1071
1072 struct CXXDefaultArgExprScope : SourceLocExprScopeGuard {
1073 CXXDefaultArgExprScope(CIRGenFunction &cfg, const CXXDefaultArgExpr *e)
1074 : SourceLocExprScopeGuard(e, cfg.curSourceLocExprScope) {}
1075 };
1076
1077 /// The scope of an ArrayInitLoopExpr. Within this scope, the value of the
1078 /// current loop index is overridden. In order to encourage re-use of existing
1079 /// array initialization, this uses a flag to determine if it is a 'no-op' or
1080 /// not.
1081 class ArrayInitLoopExprScope {
1082 public:
1083 ArrayInitLoopExprScope(CIRGenFunction &cgf, bool setIdx, mlir::Value index)
1084 : cgf(cgf),
1085 oldArrayInitIndex(setIdx
1086 ? std::optional<mlir::Value>(cgf.arrayInitIndex)
1087 : std::nullopt) {
1088 if (setIdx)
1089 cgf.arrayInitIndex = index;
1090 }
1091 ~ArrayInitLoopExprScope() {
1092 if (oldArrayInitIndex.has_value())
1093 cgf.arrayInitIndex = *oldArrayInitIndex;
1094 }
1095
1096 private:
1097 CIRGenFunction &cgf;
1098 std::optional<mlir::Value> oldArrayInitIndex;
1099 };
1100
1101 /// Get the index of the current ArrayInitLoopExpr, if any.
1102 mlir::Value getArrayInitIndex() { return arrayInitIndex; }
1103
1104 LValue makeNaturalAlignPointeeAddrLValue(mlir::Value v, clang::QualType t);
1105 LValue makeNaturalAlignAddrLValue(mlir::Value val, QualType ty);
1106
1107 /// Construct an address with the natural alignment of T. If a pointer to T
1108 /// is expected to be signed, the pointer passed to this function must have
1109 /// been signed, and the returned Address will have the pointer authentication
1110 /// information needed to authenticate the signed pointer.
1111 Address makeNaturalAddressForPointer(mlir::Value ptr, QualType t,
1112 CharUnits alignment,
1113 bool forPointeeType = false,
1114 LValueBaseInfo *baseInfo = nullptr) {
1115 if (alignment.isZero())
1116 alignment = cgm.getNaturalTypeAlignment(t, baseInfo);
1117 return Address(ptr, convertTypeForMem(t), alignment);
1118 }
1119
1120 Address getAddressOfBaseClass(
1121 Address value, const CXXRecordDecl *derived,
1122 llvm::iterator_range<CastExpr::path_const_iterator> path,
1123 bool nullCheckValue, SourceLocation loc);
1124
1125 Address getAddressOfDerivedClass(
1126 mlir::Location loc, Address baseAddr, const CXXRecordDecl *derived,
1127 llvm::iterator_range<CastExpr::path_const_iterator> path,
1128 bool nullCheckValue);
1129
1130 /// Return the VTT parameter that should be passed to a base
1131 /// constructor/destructor with virtual bases.
1132 /// FIXME: VTTs are Itanium ABI-specific, so the definition should move
1133 /// to ItaniumCXXABI.cpp together with all the references to VTT.
1134 mlir::Value getVTTParameter(GlobalDecl gd, bool forVirtualBase,
1135 bool delegating);
1136
1137 LValue makeAddrLValue(Address addr, QualType ty,
1138 AlignmentSource source = AlignmentSource::Type) {
1139 return makeAddrLValue(addr, ty, baseInfo: LValueBaseInfo(source));
1140 }
1141
1142 LValue makeAddrLValue(Address addr, QualType ty, LValueBaseInfo baseInfo) {
1143 return LValue::makeAddr(address: addr, t: ty, baseInfo);
1144 }
1145
1146 void initializeVTablePointers(mlir::Location loc,
1147 const clang::CXXRecordDecl *rd);
1148 void initializeVTablePointer(mlir::Location loc, const VPtr &vptr);
1149
1150 AggValueSlot::Overlap_t getOverlapForFieldInit(const FieldDecl *fd);
1151
1152 /// Return the address of a local variable.
1153 Address getAddrOfLocalVar(const clang::VarDecl *vd) {
1154 auto it = localDeclMap.find(Val: vd);
1155 assert(it != localDeclMap.end() &&
1156 "Invalid argument to getAddrOfLocalVar(), no decl!");
1157 return it->second;
1158 }
1159
1160 Address getAddrOfBitFieldStorage(LValue base, const clang::FieldDecl *field,
1161 mlir::Type fieldType, unsigned index);
1162
1163 /// Given an opaque value expression, return its LValue mapping if it exists,
1164 /// otherwise create one.
1165 LValue getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e);
1166
1167 /// Given an opaque value expression, return its RValue mapping if it exists,
1168 /// otherwise create one.
1169 RValue getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e);
1170
1171 /// Load the value for 'this'. This function is only valid while generating
1172 /// code for an C++ member function.
1173 /// FIXME(cir): this should return a mlir::Value!
1174 mlir::Value loadCXXThis() {
1175 assert(cxxThisValue && "no 'this' value for this function");
1176 return cxxThisValue;
1177 }
1178 Address loadCXXThisAddress();
1179
1180 /// Load the VTT parameter to base constructors/destructors have virtual
1181 /// bases. FIXME: Every place that calls LoadCXXVTT is something that needs to
1182 /// be abstracted properly.
1183 mlir::Value loadCXXVTT() {
1184 assert(cxxStructorImplicitParamValue && "no VTT value for this function");
1185 return cxxStructorImplicitParamValue;
1186 }
1187
1188 /// Convert the given pointer to a complete class to the given direct base.
1189 Address getAddressOfDirectBaseInCompleteClass(mlir::Location loc,
1190 Address value,
1191 const CXXRecordDecl *derived,
1192 const CXXRecordDecl *base,
1193 bool baseIsVirtual);
1194
1195 /// Determine whether a return value slot may overlap some other object.
1196 AggValueSlot::Overlap_t getOverlapForReturnValue() {
1197 // FIXME: Assuming no overlap here breaks guaranteed copy elision for base
1198 // class subobjects. These cases may need to be revisited depending on the
1199 // resolution of the relevant core issue.
1200 return AggValueSlot::DoesNotOverlap;
1201 }
1202
1203 /// Determine whether a base class initialization may overlap some other
1204 /// object.
1205 AggValueSlot::Overlap_t getOverlapForBaseInit(const CXXRecordDecl *rd,
1206 const CXXRecordDecl *baseRD,
1207 bool isVirtual);
1208
1209 /// Return a CIR constant for an undefined value of \p cirTy.
1210 mlir::Value getUndefConstant(mlir::Location loc, mlir::Type cirTy);
1211
1212 /// Get an appropriate 'undef' rvalue for the given type.
1213 RValue getUndefRValue(clang::QualType ty);
1214
1215 cir::FuncOp generateCode(clang::GlobalDecl gd, cir::FuncOp fn,
1216 cir::FuncType funcType);
1217
1218 clang::QualType buildFunctionArgList(clang::GlobalDecl gd,
1219 FunctionArgList &args);
1220
1221 /// Emit the function prologue: initialize the return value of an
1222 /// implicit-return-zero function and declare function arguments in the
1223 /// symbol table.
1224 void emitFunctionProlog(const FunctionArgList &args, mlir::Block *entryBB,
1225 const FunctionDecl *fd, SourceLocation bodyBeginLoc);
1226
1227 /// Emit code for the start of a function.
1228 /// \param loc The location to be associated with the function.
1229 /// \param startLoc The location of the function body.
1230 void startFunction(clang::GlobalDecl gd, clang::QualType returnType,
1231 cir::FuncOp fn, cir::FuncType funcType,
1232 FunctionArgList args, clang::SourceLocation loc,
1233 clang::SourceLocation startLoc);
1234
1235 /// Wrap the function body in a `cir.try` that enforces the exception
1236 /// specification of \p d: a filter handler for a dynamic specification
1237 /// (`throw(T...)` or pre-C++17 `throw()`), or a terminate handler for a
1238 /// specification that permits nothing to escape.
1239 void emitStartEHSpec(const clang::Decl *d);
1240
1241 /// Close the `cir.try` opened by emitStartEHSpec.
1242 void emitEndEHSpec(const clang::Decl *d);
1243
1244 /// returns true if aggregate type has a volatile member.
1245 bool hasVolatileMember(QualType t) {
1246 if (const auto *rd = t->getAsRecordDecl())
1247 return rd->hasVolatileMember();
1248 return false;
1249 }
1250
1251 void addCatchHandlerAttr(const CXXCatchStmt *catchStmt,
1252 SmallVector<mlir::Attribute> &handlerAttrs);
1253
1254 /// The cleanup depth enclosing all the cleanups associated with the
1255 /// parameters.
1256 EHScopeStack::stable_iterator prologueCleanupDepth;
1257
1258 /// The `cir.try` wrapping a function whose exception specification has to be
1259 /// enforced. Null when the current function needs no such wrapper.
1260 cir::TryOp ehSpecTryOp;
1261
1262 /// Whether the wrapper opened by emitStartEHSpec is a terminate scope, whose
1263 /// handler calls std::terminate() for any escaping exception, rather than the
1264 /// filter of a dynamic exception specification.
1265 bool inEHSpecTerminateScope() {
1266 return ehSpecTryOp &&
1267 mlir::isa<cir::CatchAllAttr>(Val: ehSpecTryOp.getHandlerTypes()[0]);
1268 }
1269
1270 bool isCatchOrCleanupRequired();
1271
1272 /// Takes the old cleanup stack size and emits the cleanup blocks
1273 /// that have been added.
1274 void popCleanupBlocks(EHScopeStack::stable_iterator oldCleanupStackDepth,
1275 ArrayRef<mlir::Value *> valuesToReload = {});
1276
1277 /// Pops cleanup blocks until the given savepoint is reached, then adds the
1278 /// cleanups from the given savepoint in the lifetime-extended cleanups stack.
1279 void popCleanupBlocks(EHScopeStack::stable_iterator oldCleanupStackDepth,
1280 size_t oldLifetimeExtendedSize,
1281 ArrayRef<mlir::Value *> valuesToReload = {});
1282 void popCleanupBlock(bool forDeactivation = false);
1283
1284 /// Emit the captured cleanups (those pushed above \p depth while EHScopeStack
1285 /// was capturing cleanups) at the current insertion point, which must be
1286 /// inside the cleanup region of the op that owns them, and pop them off the
1287 /// EH stack. Every cleanup above \p depth must have been captured.
1288 void emitCapturedCleanups(EHScopeStack::stable_iterator depth,
1289 mlir::Location loc);
1290
1291 /// Like emitCapturedCleanups, but push the captured cleanups again as
1292 /// ordinary cleanups, the first captured one outermost, and pop them, so that
1293 /// each one is emitted as a cir.cleanup.scope around the ones captured after
1294 /// it. An exception from one of them then still runs the ones captured
1295 /// before it. None of them may have an active flag.
1296 void emitCapturedCleanupsAsScopes(EHScopeStack::stable_iterator depth,
1297 mlir::Location loc);
1298
1299 void terminateStructuredRegionBody(mlir::Region &r, mlir::Location loc);
1300
1301 /// Deactivates the given cleanup block. The block cannot be reactivated. Pops
1302 /// it if it's the top of the stack.
1303 ///
1304 /// \param DominatingIP - An instruction which is known to
1305 /// dominate the current IP (if set) and which lies along
1306 /// all paths of execution between the current IP and the
1307 /// the point at which the cleanup comes into scope.
1308 void deactivateCleanupBlock(EHScopeStack::stable_iterator cleanup,
1309 mlir::Operation *dominatingIP);
1310
1311 /// Create an active flag variable for use with conditional cleanups. The
1312 /// flag is initialized to false before the outermost conditional and set to
1313 /// true at the current insertion point (inside the conditional branch).
1314 Address createCleanupActiveFlag();
1315
1316 /// Set up the last cleanup that was pushed as a conditional
1317 /// full-expression cleanup.
1318 void initFullExprCleanup();
1319 void initFullExprCleanupWithFlag(Address activeFlag);
1320
1321 /// Promote a single pending cleanup entry onto the EH scope stack. If the
1322 /// entry has a valid activeFlag, the cleanup is configured as conditional.
1323 /// Defined in CIRGenDecl.cpp where the concrete cleanup types are visible.
1324 void pushPendingCleanupToEHStack(const PendingCleanupEntry &entry);
1325
1326 /// Push a cleanup to be run at the end of the current full-expression. Safe
1327 /// against the possibility that we're currently inside a
1328 /// conditionally-evaluated expression.
1329 template <class T, class... As>
1330 void pushFullExprCleanup(CleanupKind kind, As... a) {
1331 // Outside a conditional the EH stack opens a scope here.
1332 if (!isInConditionalBranch())
1333 return ehStack.pushCleanup<T>(kind, a...);
1334
1335 // Inside a conditional, record the cleanup for the scope that
1336 // ConditionalEvaluation opened where the conditional began, guarded by a
1337 // flag set only where the object is constructed.
1338 Address activeFlag = createCleanupActiveFlag();
1339 deferredConditionalCleanupStack.push_back(
1340 Elt: PendingCleanupEntry{kind, a..., activeFlag});
1341 }
1342
1343 /// Push a cleanup and record it for deferred deactivation. The cleanup will
1344 /// be deactivated when the enclosing CleanupDeactivationScope exits.
1345 template <class T, class... As>
1346 void pushCleanupAndDeferDeactivation(CleanupKind kind, As... a) {
1347 mlir::Location loc = builder.getUnknownLoc();
1348 mlir::Operation *dominatingIP = builder.getBool(state: false, loc).getOperation();
1349 ehStack.pushCleanup<T>(kind, a...);
1350 deferredDeactivationCleanupStack.push_back(
1351 Elt: {.cleanup: ehStack.stable_begin(), .dominatingIP: dominatingIP});
1352 }
1353
1354 void pushDestroyAndDeferDeactivation(QualType::DestructionKind dtorKind,
1355 Address addr, QualType type);
1356 void pushDestroyAndDeferDeactivation(CleanupKind cleanupKind, Address addr,
1357 QualType type, Destroyer *destroyer,
1358 bool useEHCleanupForArray);
1359
1360 /// Queue a cleanup to be pushed after finishing the current full-expression.
1361 /// When the enclosing RunCleanupsScope exits, popCleanupBlocks promotes these
1362 /// entries onto the EH scope stack for the enclosing scope.
1363 void pushCleanupAfterFullExpr(CleanupKind kind, Address addr, QualType type,
1364 Destroyer *destroyer) {
1365 lifetimeExtendedCleanupStack.push_back(Elt: {.kind: kind, .addr: addr, .type: type, .destroyer: destroyer});
1366 }
1367
1368 /// Enters a new scope for capturing cleanups, all of which
1369 /// will be executed once the scope is exited.
1370 class RunCleanupsScope {
1371 EHScopeStack::stable_iterator cleanupStackDepth, oldCleanupStackDepth;
1372 size_t lifetimeExtendedCleanupStackSize;
1373 CleanupDeactivationScope deactivateCleanups;
1374 bool deferLifetimeExtended = false;
1375
1376 size_t oldLifetimeExtendedSize() const {
1377 return deferLifetimeExtended ? cgf.lifetimeExtendedCleanupStack.size()
1378 : lifetimeExtendedCleanupStackSize;
1379 }
1380
1381 protected:
1382 bool performCleanup;
1383 bool oldDidCallStackSave;
1384
1385 private:
1386 RunCleanupsScope(const RunCleanupsScope &) = delete;
1387 void operator=(const RunCleanupsScope &) = delete;
1388
1389 protected:
1390 CIRGenFunction &cgf;
1391
1392 public:
1393 /// Enter a new cleanup scope.
1394 explicit RunCleanupsScope(CIRGenFunction &cgf)
1395 : deactivateCleanups(cgf), performCleanup(true), cgf(cgf) {
1396 cleanupStackDepth = cgf.ehStack.stable_begin();
1397 lifetimeExtendedCleanupStackSize =
1398 cgf.lifetimeExtendedCleanupStack.size();
1399 oldDidCallStackSave = cgf.didCallStackSave;
1400 cgf.didCallStackSave = false;
1401 oldCleanupStackDepth = cgf.currentCleanupStackDepth;
1402 cgf.currentCleanupStackDepth = cleanupStackDepth;
1403 }
1404
1405 /// Exit this cleanup scope, emitting any accumulated cleanups.
1406 ~RunCleanupsScope() {
1407 if (performCleanup)
1408 forceCleanup();
1409 }
1410
1411 /// Force the emission of cleanups now, instead of waiting
1412 /// until this object is destroyed.
1413 void forceCleanup(ArrayRef<mlir::Value *> valuesToReload = {}) {
1414 assert(performCleanup && "Already forced cleanup");
1415 cgf.didCallStackSave = oldDidCallStackSave;
1416
1417 // forceDeactivate() can pop cleanup scopes that were pushed with
1418 // deferred deactivation, which moves the insertion point out of the
1419 // cleanup body region. Any caller value defined inside such a body
1420 // would no longer dominate uses past the scope. The downstream
1421 // popCleanupBlocks() handles the spill for any cleanups it pops
1422 // itself, but it cannot help with cleanups that forceDeactivate has
1423 // already popped. Spill those values here, while the insertion point
1424 // is still inside the body, so we can reload them after all popping
1425 // is done. We only spill values whose defining op lives inside a
1426 // cir.cleanup.scope, since values defined outside any cleanup scope
1427 // (e.g. allocas in the entry block) already dominate the post-scope
1428 // insertion point.
1429 const bool hasPendingDeactivations =
1430 cgf.deferredDeactivationCleanupStack.size() >
1431 deactivateCleanups.oldDeactivateCleanupStackSize;
1432
1433 llvm::SmallVector<Address> tempAllocas;
1434 bool didSpillAny = false;
1435 if (hasPendingDeactivations) {
1436 tempAllocas.reserve(N: valuesToReload.size());
1437 for (mlir::Value *valPtr : valuesToReload) {
1438 mlir::Value val = *valPtr;
1439 if (!val || !val.getDefiningOp() ||
1440 !val.getDefiningOp()->getParentOfType<cir::CleanupScopeOp>()) {
1441 tempAllocas.push_back(Elt: Address::invalid());
1442 continue;
1443 }
1444 Address temp = cgf.createDefaultAlignTempAlloca(
1445 ty: val.getType(), loc: val.getLoc(), name: "tmp.exprcleanup");
1446 tempAllocas.push_back(Elt: temp);
1447 cgf.builder.createStore(loc: val.getLoc(), val, dst: temp);
1448 didSpillAny = true;
1449 }
1450 }
1451
1452 deactivateCleanups.forceDeactivate();
1453 // If we already spilled some of the caller's values, don't ask
1454 // popCleanupBlocks to spill them again. Values we did not pre-spill
1455 // are not inside any cir.cleanup.scope, so they cannot be invalidated
1456 // by either forceDeactivate's or popCleanupBlocks's pops (both only
1457 // pop cir.cleanup.scope ops); they already dominate the post-scope
1458 // insertion point on their own.
1459 if (didSpillAny) {
1460 cgf.popCleanupBlocks(oldCleanupStackDepth: cleanupStackDepth, oldLifetimeExtendedSize: oldLifetimeExtendedSize());
1461
1462 // Reload the spilled values now that all cleanup popping (and
1463 // promotion of any lifetime-extended cleanups onto the EH stack) is
1464 // done.
1465 for (auto [addr, valPtr] : llvm::zip(t&: tempAllocas, u&: valuesToReload)) {
1466 if (!addr.isValid())
1467 continue;
1468 *valPtr = cgf.builder.createLoad(loc: valPtr->getLoc(), addr);
1469 }
1470 } else {
1471 cgf.popCleanupBlocks(oldCleanupStackDepth: cleanupStackDepth, oldLifetimeExtendedSize: oldLifetimeExtendedSize(),
1472 valuesToReload);
1473 }
1474
1475 performCleanup = false;
1476 cgf.currentCleanupStackDepth = oldCleanupStackDepth;
1477 }
1478
1479 /// Force the emission of EH cleanups now, but defer promoting any
1480 /// lifetime-extended cleanup entries onto the EH scope stack. The caller
1481 /// must subsequently call forceLifetimeExtendedCleanups() to finalize the
1482 /// scope.
1483 void forceCleanupExceptLifetimeExtended() {
1484 assert(performCleanup && "Already forced cleanup");
1485 cgf.didCallStackSave = oldDidCallStackSave;
1486 deactivateCleanups.forceDeactivate();
1487 cgf.popCleanupBlocks(oldCleanupStackDepth: cleanupStackDepth);
1488 }
1489
1490 /// Promote any pending lifetime-extended cleanup entries onto the EH scope
1491 /// stack at the current insertion point and finalize this scope. This must
1492 /// be paired with a prior call to forceCleanupExceptLifetimeExtended().
1493 void forceLifetimeExtendedCleanups() {
1494 assert(performCleanup && "Already forced cleanup");
1495 assert(deactivateCleanups.deactivated &&
1496 "forceCleanupExceptLifetimeExtended() must be called first");
1497 cgf.popCleanupBlocks(oldCleanupStackDepth: cleanupStackDepth, oldLifetimeExtendedSize: oldLifetimeExtendedSize());
1498 performCleanup = false;
1499 cgf.currentCleanupStackDepth = oldCleanupStackDepth;
1500 }
1501
1502 /// Whether there are any pending cleanups that have been pushed since
1503 /// this scope was entered.
1504 bool hasPendingCleanups() const {
1505 return cgf.ehStack.stable_begin() != cleanupStackDepth;
1506 }
1507
1508 /// Leave the cleanups of lifetime-extended temporaries created in this
1509 /// scope to the enclosing scope instead of pushing them onto the EH stack
1510 /// when this scope ends. This is for a scope that ends in the body of a
1511 /// loop, such as the construction of an array element: the cleanup scope
1512 /// of such a temporary ends with that of the variable that extends its
1513 /// lifetime, after the loop.
1514 void deferLifetimeExtendedCleanups() { deferLifetimeExtended = true; }
1515 };
1516
1517 // Cleanup stack depth of the RunCleanupsScope that was pushed most recently.
1518 EHScopeStack::stable_iterator currentCleanupStackDepth = ehStack.stable_end();
1519
1520 class ConditionalEvaluation;
1521
1522 class FullExprCleanupScope {
1523 CIRGenFunction &cgf;
1524 RunCleanupsScope cleanups;
1525 cir::CleanupScopeOp scope;
1526 cir::CleanupScopeOp oldFullExprCleanupScope;
1527 ConditionalEvaluation *oldOutermostConditional;
1528 ConditionalEvaluation *oldHiddenOutermostConditional;
1529 size_t deferredCleanupStackSize;
1530 size_t conditionalScopeDepth;
1531 bool exited = false;
1532
1533 public:
1534 /// \p s is the AST root for one cleanup boundary: a full-expression, an
1535 /// asm statement, or the construction of one array element.
1536 FullExprCleanupScope(CIRGenFunction &cgf, const Stmt *s);
1537
1538 void exit(ArrayRef<mlir::Value *> valuesToReload = {});
1539
1540 void deferLifetimeExtendedCleanups() {
1541 cleanups.deferLifetimeExtendedCleanups();
1542 }
1543
1544 ~FullExprCleanupScope() {
1545 if (!exited)
1546 exit();
1547 }
1548
1549 private:
1550 FullExprCleanupScope(const FullExprCleanupScope &) = delete;
1551 void operator=(const FullExprCleanupScope &) = delete;
1552 };
1553
1554 /// Whether the resume expression of \p s contains a construct that defers
1555 /// cleanups to the end of the enclosing full-expression: a temporary with a
1556 /// destructor, other than the result of await_resume(), or a conditional.
1557 static bool resumeExprDefersCleanups(const CoroutineSuspendExpr &s);
1558
1559 /// Captures cleanups for variables whose lifetime ends in a region owned by
1560 /// an enclosing op rather than at the end of a cir.cleanup.scope, so that
1561 /// they can be emitted into that region later. Examples for uses are:
1562 /// * A loop condition variable's cleanups in the loop op's per-iteration
1563 /// cleanup region.
1564 /// * Coroutine parameters and promises, which are cleaned up in a separate
1565 /// destroy region
1566 class CapturedCleanups {
1567 CIRGenFunction &cgf;
1568 EHScopeStack::stable_iterator depth;
1569 bool active;
1570
1571 public:
1572 CapturedCleanups(CIRGenFunction &cgf, bool active)
1573 : cgf(cgf), depth(cgf.ehStack.stable_begin()), active(active) {}
1574
1575 /// An RAII class that suppresses cir.cleanup.scope creation for cleanups
1576 /// pushed onto the EH stack while such a variable is being emitted and
1577 /// instead captures these cleanups so that they can be emitted into the
1578 /// owning op's cleanup region later.
1579 class CaptureScope {
1580 EHScopeStack &ehStack;
1581
1582 public:
1583 explicit CaptureScope(CapturedCleanups &scope)
1584 : ehStack(scope.cgf.ehStack) {
1585 // Capture scopes deliberately wrap individual cleanup-producing
1586 // operations, so they must never nest.
1587 assert(!ehStack.isCapturingCleanups() &&
1588 "cleanup capturing should not nest");
1589 if (scope.active)
1590 ehStack.setCapturingCleanups(true);
1591 }
1592 ~CaptureScope() { ehStack.setCapturingCleanups(false); }
1593
1594 CaptureScope(const CaptureScope &) = delete;
1595 void operator=(const CaptureScope &) = delete;
1596 };
1597
1598 /// Emit the captured cleanups into the current insertion point, which must
1599 /// be inside the owning op's cleanup region.
1600 void emitIntoCleanupRegion(mlir::Location loc) {
1601 if (active)
1602 cgf.emitCapturedCleanups(depth, loc);
1603 }
1604
1605 /// Emit the captured cleanups at the current insertion point as nested
1606 /// cir.cleanup.scope operations, so that an exception from one of them
1607 /// still runs the ones captured before it.
1608 void emitAsNestedScopes(mlir::Location loc) {
1609 if (active)
1610 cgf.emitCapturedCleanupsAsScopes(depth, loc);
1611 }
1612
1613 private:
1614 CapturedCleanups(const CapturedCleanups &) = delete;
1615 void operator=(const CapturedCleanups &) = delete;
1616 };
1617
1618public:
1619 /// Represents a scope, including function bodies, compound statements, and
1620 /// the substatements of if/while/do/for/switch/try statements. This class
1621 /// handles any automatic cleanup, along with the return value.
1622 struct LexicalScope : public RunCleanupsScope {
1623 private:
1624 // Points to the scope entry block. This is useful, for instance, for
1625 // helping to insert allocas before finalizing any recursive CodeGen from
1626 // switches.
1627 mlir::Block *entryBlock;
1628
1629 LexicalScope *parentScope = nullptr;
1630
1631 // Holds the actual value for ScopeKind::Try
1632 cir::TryOp tryOp = nullptr;
1633
1634 // On a coroutine body, the OnFallthrough sub stmt holds the handler
1635 // (CoreturnStmt) for control flow falling off the body. Keep track
1636 // of a co_return emitted in this scope and not followed by a label, and
1637 // allow OnFallthrough to be skipped.
1638 bool hasCoreturnSinceLastLabel = false;
1639
1640 // Only Regular is used at the moment. Support for other kinds will be
1641 // added as the relevant statements/expressions are upstreamed.
1642 enum Kind {
1643 Regular, // cir.if, cir.scope, if_regions
1644 Ternary, // cir.ternary
1645 Switch, // cir.switch
1646 Try, // cir.try
1647 GlobalInit // cir.global initialization code
1648 };
1649 Kind scopeKind = Kind::Regular;
1650
1651 // The scope return value.
1652 mlir::Value retVal = nullptr;
1653
1654 mlir::Location beginLoc;
1655 mlir::Location endLoc;
1656
1657 public:
1658 unsigned depth = 0;
1659
1660 LexicalScope(CIRGenFunction &cgf, mlir::Location loc, mlir::Block *eb)
1661 : RunCleanupsScope(cgf), entryBlock(eb), parentScope(cgf.curLexScope),
1662 beginLoc(loc), endLoc(loc) {
1663
1664 assert(entryBlock && "LexicalScope requires an entry block");
1665 cgf.curLexScope = this;
1666 if (parentScope)
1667 ++depth;
1668
1669 if (const auto fusedLoc = mlir::dyn_cast<mlir::FusedLoc>(Val&: loc)) {
1670 assert(fusedLoc.getLocations().size() == 2 && "too many locations");
1671 beginLoc = fusedLoc.getLocations()[0];
1672 endLoc = fusedLoc.getLocations()[1];
1673 }
1674 }
1675
1676 void setRetVal(mlir::Value v) { retVal = v; }
1677
1678 void cleanup();
1679 void restore() { cgf.curLexScope = parentScope; }
1680
1681 ~LexicalScope() {
1682 assert(!cir::MissingFeatures::generateDebugInfo());
1683 cleanup();
1684 restore();
1685 }
1686
1687 // ---
1688 // Coroutine tracking
1689 // ---
1690 bool hasCoreturn() const { return hasCoreturnSinceLastLabel; }
1691 // Record an emitted co_return for fallthrough tracking.
1692 // A later label clears this state.
1693 void markCoreturnEmitted() { hasCoreturnSinceLastLabel = true; }
1694 // A goto can reach a label after a co_return, and the code after it, so
1695 // the end of this scope and of the enclosing ones may be reachable again.
1696 void clearCoreturnInEnclosingScopes() {
1697 for (LexicalScope *s = this; s; s = s->parentScope)
1698 s->hasCoreturnSinceLastLabel = false;
1699 }
1700
1701 // ---
1702 // Kind
1703 // ---
1704 bool isGlobalInit() { return scopeKind == Kind::GlobalInit; }
1705 bool isRegular() { return scopeKind == Kind::Regular; }
1706 bool isSwitch() { return scopeKind == Kind::Switch; }
1707 bool isTernary() { return scopeKind == Kind::Ternary; }
1708 bool isTry() { return scopeKind == Kind::Try; }
1709 cir::TryOp getClosestTryParent();
1710 void setAsGlobalInit() { scopeKind = Kind::GlobalInit; }
1711 void setAsSwitch() { scopeKind = Kind::Switch; }
1712 void setAsTernary() { scopeKind = Kind::Ternary; }
1713 void setAsTry(cir::TryOp op) {
1714 scopeKind = Kind::Try;
1715 tryOp = op;
1716 }
1717
1718 cir::TryOp getTry() {
1719 assert(isTry());
1720 return tryOp;
1721 }
1722
1723 // ---
1724 // Return handling.
1725 // ---
1726
1727 private:
1728 // On switches we need one return block per region, since cases don't
1729 // have their own scopes but are distinct regions nonetheless.
1730
1731 // TODO: This implementation should change once we have support for early
1732 // exits in MLIR structured control flow (llvm-project#161575)
1733 llvm::SmallVector<mlir::Block *> retBlocks;
1734 llvm::DenseMap<mlir::Block *, mlir::Location> retLocs;
1735 llvm::DenseMap<cir::CaseOp, unsigned> retBlockInCaseIndex;
1736 std::optional<unsigned> normalRetBlockIndex;
1737
1738 // There's usually only one ret block per scope, but this needs to be
1739 // get or create because of potential unreachable return statements, note
1740 // that for those, all source location maps to the first one found.
1741 mlir::Block *createRetBlock(CIRGenFunction &cgf, mlir::Location loc) {
1742 assert((isa_and_nonnull<cir::CaseOp>(
1743 cgf.builder.getBlock()->getParentOp()) ||
1744 retBlocks.size() == 0) &&
1745 "only switches can hold more than one ret block");
1746
1747 // Create the return block but don't hook it up just yet.
1748 mlir::OpBuilder::InsertionGuard guard(cgf.builder);
1749 auto *b = cgf.builder.createBlock(parent: cgf.builder.getBlock()->getParent());
1750 retBlocks.push_back(Elt: b);
1751 updateRetLoc(b, loc);
1752 return b;
1753 }
1754
1755 cir::ReturnOp emitReturn(mlir::Location loc);
1756 void emitImplicitReturn();
1757
1758 public:
1759 llvm::ArrayRef<mlir::Block *> getRetBlocks() { return retBlocks; }
1760 mlir::Location getRetLoc(mlir::Block *b) { return retLocs.at(Val: b); }
1761 void updateRetLoc(mlir::Block *b, mlir::Location loc) {
1762 retLocs.insert_or_assign(Key: b, Val&: loc);
1763 }
1764
1765 mlir::Block *getOrCreateRetBlock(CIRGenFunction &cgf, mlir::Location loc) {
1766 // Check if we're inside a case region
1767 if (auto caseOp = mlir::dyn_cast_if_present<cir::CaseOp>(
1768 Val: cgf.builder.getBlock()->getParentOp())) {
1769 auto iter = retBlockInCaseIndex.find(Val: caseOp);
1770 if (iter != retBlockInCaseIndex.end()) {
1771 // Reuse existing return block
1772 mlir::Block *ret = retBlocks[iter->second];
1773 updateRetLoc(b: ret, loc);
1774 return ret;
1775 }
1776 // Create new return block
1777 mlir::Block *ret = createRetBlock(cgf, loc);
1778 retBlockInCaseIndex[caseOp] = retBlocks.size() - 1;
1779 return ret;
1780 }
1781
1782 if (normalRetBlockIndex) {
1783 mlir::Block *ret = retBlocks[*normalRetBlockIndex];
1784 updateRetLoc(b: ret, loc);
1785 return ret;
1786 }
1787
1788 mlir::Block *ret = createRetBlock(cgf, loc);
1789 normalRetBlockIndex = retBlocks.size() - 1;
1790 return ret;
1791 }
1792
1793 mlir::Block *getEntryBlock() { return entryBlock; }
1794 };
1795
1796 LexicalScope *curLexScope = nullptr;
1797
1798 static Destroyer destroyCXXObject;
1799
1800 void pushEHDestroyIfNeeded(QualType::DestructionKind dtorKind, Address addr,
1801 QualType type);
1802
1803 void pushDestroy(QualType::DestructionKind dtorKind, Address addr,
1804 QualType type);
1805
1806 void pushDestroy(CleanupKind kind, Address addr, QualType type,
1807 Destroyer *destroyer);
1808
1809 void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr,
1810 QualType type, Destroyer *destroyer,
1811 bool useEHCleanupForArray,
1812 const clang::ValueDecl *extendingDecl);
1813
1814 Destroyer *getDestroyer(clang::QualType::DestructionKind kind);
1815
1816 void pushIrregularPartialArrayCleanup(mlir::Value arrayBegin,
1817 Address arrayEndPointer,
1818 QualType elementType,
1819 CharUnits elementAlign,
1820 Destroyer *destroyer);
1821
1822 /// Start generating a thunk function.
1823 void startThunk(cir::FuncOp fn, GlobalDecl gd,
1824 const CIRGenFunctionInfo &fnInfo, bool isUnprototyped);
1825
1826 /// Finish generating a thunk function.
1827 void finishThunk();
1828
1829 /// Generate code for a thunk function.
1830 void generateThunk(cir::FuncOp fn, SourceRange fnLoc,
1831 const CIRGenFunctionInfo &fnInfo, GlobalDecl gd,
1832 const ThunkInfo &thunk, bool isUnprototyped);
1833
1834 /// ----------------------
1835 /// CIR emit functions
1836 /// ----------------------
1837public:
1838 bool getAArch64SVEProcessedOperands(unsigned builtinID, const CallExpr *expr,
1839 SmallVectorImpl<mlir::Value> &ops,
1840 clang::SVETypeFlags typeFlags);
1841 mlir::Value emitSVEPredicateCast(mlir::Value pred, unsigned minNumElts,
1842 mlir::Location loc);
1843 std::optional<mlir::Value>
1844 emitAArch64BuiltinExpr(unsigned builtinID, const CallExpr *expr,
1845 ReturnValueSlot returnValue,
1846 llvm::Triple::ArchType arch);
1847 std::optional<mlir::Value> emitAArch64SMEBuiltinExpr(unsigned builtinID,
1848 const CallExpr *expr);
1849 std::optional<mlir::Value> emitAArch64SVEBuiltinExpr(unsigned builtinID,
1850 const CallExpr *expr);
1851 cir::VectorType getSVEType(const SVETypeFlags &typeFlags);
1852
1853 mlir::Value emitAlignmentAssumption(mlir::Value ptrValue, QualType ty,
1854 SourceLocation loc,
1855 SourceLocation assumptionLoc,
1856 int64_t alignment,
1857 mlir::Value offsetValue = nullptr);
1858
1859 mlir::Value emitAlignmentAssumption(mlir::Value ptrValue, const Expr *expr,
1860 SourceLocation assumptionLoc,
1861 int64_t alignment,
1862 mlir::Value offsetValue = nullptr);
1863
1864 bool emitLifetimeStartOp(mlir::Location loc, mlir::Value addr);
1865 void emitLifetimeEndOp(mlir::Location loc, mlir::Value addr);
1866
1867private:
1868 void emitAndUpdateRetAlloca(clang::QualType type, mlir::Location loc,
1869 clang::CharUnits alignment);
1870
1871 CIRGenCallee emitDirectCallee(const GlobalDecl &gd);
1872
1873public:
1874 Address emitAddrOfFieldStorage(Address base, const FieldDecl *field,
1875 llvm::StringRef fieldName,
1876 unsigned fieldIndex);
1877
1878 mlir::Value emitAlloca(llvm::StringRef name, mlir::Type ty,
1879 mlir::Location loc, clang::CharUnits alignment,
1880 bool insertIntoFnEntryBlock,
1881 mlir::Value arraySize = nullptr);
1882 mlir::Value emitAlloca(llvm::StringRef name, mlir::Type ty,
1883 mlir::Location loc, clang::CharUnits alignment,
1884 mlir::OpBuilder::InsertPoint ip,
1885 mlir::Value arraySize = nullptr);
1886
1887 void emitAggregateStore(mlir::Value value, Address dest);
1888
1889 void emitAggExpr(const clang::Expr *e, AggValueSlot slot);
1890
1891 enum ExprValueKind { EVK_RValue, EVK_NonRValue };
1892
1893 LValue emitAggExprToLValue(const Expr *e);
1894
1895 /// Emit an aggregate copy.
1896 ///
1897 /// \param isVolatile \c true iff either the source or the destination is
1898 /// volatile.
1899 /// \param MayOverlap Whether the tail padding of the destination might be
1900 /// occupied by some other object. More efficient code can often be
1901 /// generated if not.
1902 void emitAggregateCopy(LValue dest, LValue src, QualType eltTy,
1903 AggValueSlot::Overlap_t mayOverlap,
1904 bool isVolatile = false);
1905
1906 /// Emit code to compute the specified expression which can have any type. The
1907 /// result is returned as an RValue struct. If this is an aggregate
1908 /// expression, the aggloc/agglocvolatile arguments indicate where the result
1909 /// should be returned.
1910 RValue emitAnyExpr(const clang::Expr *e,
1911 AggValueSlot aggSlot = AggValueSlot::ignored(),
1912 bool ignoreResult = false);
1913
1914 /// Emits the code necessary to evaluate an arbitrary expression into the
1915 /// given memory location.
1916 void emitAnyExprToMem(const Expr *e, Address location, Qualifiers quals,
1917 bool isInitializer);
1918
1919 /// Similarly to emitAnyExpr(), however, the result will always be accessible
1920 /// even if no aggregate location is provided.
1921 RValue emitAnyExprToTemp(const clang::Expr *e);
1922
1923 void emitAnyExprToExn(const Expr *e, Address addr);
1924
1925 void emitArrayDestroy(mlir::Value begin, mlir::Value numElements,
1926 QualType elementType, CharUnits elementAlign,
1927 Destroyer *destroyer);
1928
1929 mlir::Value emitArrayLength(const clang::ArrayType *arrayType,
1930 QualType &baseType, Address &addr);
1931 LValue emitArraySubscriptExpr(const clang::ArraySubscriptExpr *e);
1932 LValue emitInitListLValue(const InitListExpr *e);
1933
1934 LValue emitExtVectorElementExpr(const ExtVectorElementExpr *e);
1935
1936 LValue emitMatrixSubscriptExpr(const MatrixSubscriptExpr *e);
1937
1938 Address emitArrayToPointerDecay(const Expr *e,
1939 LValueBaseInfo *baseInfo = nullptr);
1940
1941 std::pair<mlir::Value, mlir::Type>
1942 emitAsmInputLValue(const TargetInfo::ConstraintInfo &info, LValue inputValue,
1943 QualType inputType, std::string &constraintString,
1944 SourceLocation loc);
1945 std::pair<mlir::Value, mlir::Type>
1946 emitAsmInput(const TargetInfo::ConstraintInfo &info, const Expr *inputExpr,
1947 std::string &constraintString);
1948 mlir::LogicalResult emitAsmStmt(const clang::AsmStmt &s);
1949
1950 RValue emitAtomicExpr(AtomicExpr *e);
1951 void emitAtomicInit(Expr *init, LValue dest);
1952 void emitAtomicStore(RValue rvalue, LValue dest, bool isInit);
1953 void emitAtomicStore(RValue rvalue, LValue dest, cir::MemOrder order,
1954 bool isVolatile, bool isInit);
1955 void emitAtomicExprWithMemOrder(
1956 const Expr *memOrder, bool isStore, bool isLoad, bool isFence,
1957 llvm::function_ref<void(cir::MemOrder)> emitAtomicOp);
1958
1959 mlir::Value makeBinaryAtomicValue(
1960 cir::AtomicFetchKind kind, const clang::CallExpr *expr,
1961 mlir::Type *originalArgType = nullptr,
1962 mlir::Value *emittedArgValue = nullptr,
1963 cir::MemOrder ordering = cir::MemOrder::SequentiallyConsistent);
1964
1965 /// Emit `cir.atomic.cmpxchg`. Returns the old value, or the success flag
1966 /// when `returnBool` is true.
1967 mlir::Value emitAtomicCmpXchg(
1968 const clang::CallExpr *expr, bool returnBool,
1969 cir::MemOrder successOrder = cir::MemOrder::SequentiallyConsistent,
1970 cir::MemOrder failureOrder = cir::MemOrder::SequentiallyConsistent,
1971 cir::SyncScopeKind scope = cir::SyncScopeKind::System);
1972
1973 mlir::LogicalResult emitAttributedStmt(const AttributedStmt &s);
1974
1975 AutoVarEmission emitAutoVarAlloca(const clang::VarDecl &d,
1976 mlir::OpBuilder::InsertPoint ip = {});
1977
1978 RValue emitPseudoObjectRValue(const PseudoObjectExpr *e,
1979 AggValueSlot slot = AggValueSlot::ignored());
1980 LValue emitPseudoObjectLValue(const PseudoObjectExpr *E);
1981
1982 /// Emit code and set up symbol table for a variable declaration with auto,
1983 /// register, or no storage class specifier. These turn into simple stack
1984 /// objects, globals depending on target.
1985 void emitAutoVarDecl(const clang::VarDecl &d);
1986
1987 void emitAutoVarCleanups(const AutoVarEmission &emission);
1988
1989 /// Emit a loop's condition-variable declaration. This needs special handling
1990 /// so that we can manage per-iteration cleanups for the loop condition.
1991 /// \p extendedTemporaries are the types of the temporaries with a destructor
1992 /// that its initializer lifetime-extends (see
1993 /// getLifetimeExtendedTemporariesWithDestructor).
1994 void emitLoopConditionVariable(const clang::VarDecl &d,
1995 CapturedCleanups &condCleanup,
1996 llvm::ArrayRef<QualType> extendedTemporaries);
1997
1998 /// The types of the temporaries with a destructor that the initializer of
1999 /// the variable \p d, which has automatic storage duration, lifetime-extends.
2000 static llvm::SmallVector<QualType, 2>
2001 getLifetimeExtendedTemporariesWithDestructor(const clang::VarDecl &d);
2002
2003 /// Whether destroying an object of type \p type, or its elements if it is an
2004 /// array, may throw an exception.
2005 static bool destructorMayThrow(QualType type);
2006
2007 /// Emit \p d with captured cleanups. Set \p guardDestruction if the cleanup
2008 /// region covers initialization. Otherwise it must be reached only after
2009 /// initialization. The cleanups of \p extendedTemporaries (see
2010 /// emitLoopConditionVariable) are captured too; they need
2011 /// \p guardDestruction.
2012 void emitAutoVarDeclWithCapturedCleanups(
2013 const clang::VarDecl &d, CapturedCleanups &cleanups,
2014 bool guardDestruction = false,
2015 llvm::ArrayRef<QualType> extendedTemporaries = {});
2016
2017 /// Emit the initializer for an allocated variable. If this call is not
2018 /// associated with the call to emitAutoVarAlloca (as the address of the
2019 /// emission is not directly an alloca), the allocatedSeparately parameter can
2020 /// be used to suppress the assertions. However, this should only be used in
2021 /// extreme cases, as it doesn't properly reflect the language/AST.
2022 void emitAutoVarInit(const AutoVarEmission &emission);
2023 void emitAutoVarTypeCleanup(const AutoVarEmission &emission,
2024 clang::QualType::DestructionKind dtorKind);
2025
2026 void maybeEmitDeferredVarDeclInit(const VarDecl *vd);
2027
2028 void emitBaseInitializer(mlir::Location loc, const CXXRecordDecl *classDecl,
2029 CXXCtorInitializer *baseInit);
2030
2031 LValue emitBinaryOperatorLValue(const BinaryOperator *e);
2032
2033 mlir::LogicalResult emitBreakStmt(const clang::BreakStmt &s);
2034
2035 RValue emitBuiltinExpr(const clang::GlobalDecl &gd, unsigned builtinID,
2036 const clang::CallExpr *e, ReturnValueSlot returnValue);
2037
2038 /// Returns a Value corresponding to the size of the given expression by
2039 /// emitting a `cir.objsize` operation.
2040 ///
2041 /// \param e The expression whose object size to compute
2042 /// \param type Determines the semantics of the object size computation.
2043 /// The type parameter is a 2-bit value where:
2044 /// bit 0 (type & 1): 0 = whole object, 1 = closest subobject
2045 /// bit 1 (type & 2): 0 = maximum size, 2 = minimum size
2046 /// \param resType The result type for the size value
2047 /// \param emittedE Optional pre-emitted pointer value. If non-null, we'll
2048 /// call `cir.objsize` on this value rather than emitting e.
2049 /// \param isDynamic If true, allows runtime evaluation via dynamic mode
2050 mlir::Value emitBuiltinObjectSize(const clang::Expr *e, unsigned type,
2051 cir::IntType resType, mlir::Value emittedE,
2052 bool isDynamic);
2053
2054 mlir::Value evaluateOrEmitBuiltinObjectSize(const clang::Expr *e,
2055 unsigned type,
2056 cir::IntType resType,
2057 mlir::Value emittedE,
2058 bool isDynamic);
2059
2060 int64_t getAccessedFieldNo(unsigned idx, mlir::ArrayAttr elts);
2061
2062 /// Return the CIR signature of the LLVM intrinsic \p id, resolving its
2063 /// overloaded types to \p overloadTys. Integer types are returned signed.
2064 cir::FuncType getIntrinsicType(llvm::Intrinsic::ID id,
2065 llvm::ArrayRef<mlir::Type> overloadTys = {});
2066
2067 /// Emit a simple LLVM intrinsic that takes N scalar arguments. The intrinsic
2068 /// name is used verbatim; any overload mangling (e.g. `.f32`, `.p1`) must be
2069 /// baked into \p intrinName by the caller. The result type defaults to the
2070 /// type of the first argument; pass \p resultType for intrinsics whose result
2071 /// differs from the operand, such as a vector reduction that returns the
2072 /// element type. Unlike classic CodeGen, CIR has no intrinsic registry to
2073 /// derive the result type from the operand, so it must be supplied here.
2074 template <unsigned N>
2075 [[maybe_unused]] RValue
2076 emitBuiltinWithOneOverloadedType(const CallExpr *e,
2077 llvm::StringRef intrinName,
2078 mlir::Type resultType = {}) {
2079 static_assert(N, "expect non-empty argument");
2080 mlir::Type cirTy =
2081 resultType ? resultType : convertType(t: e->getArg(Arg: 0)->getType());
2082 SmallVector<mlir::Value, N> args;
2083 for (unsigned i = 0; i < N; ++i)
2084 args.push_back(emitScalarExpr(e: e->getArg(Arg: i)));
2085 const auto call = cir::LLVMIntrinsicCallOp::create(
2086 builder, getLoc(srcLoc: e->getExprLoc()), builder.getStringAttr(bytes: intrinName),
2087 cirTy, args);
2088 return RValue::get(v: call->getResult(0));
2089 }
2090
2091 RValue emitCall(const CIRGenFunctionInfo &funcInfo,
2092 const CIRGenCallee &callee, ReturnValueSlot returnValue,
2093 const CallArgList &args, cir::CIRCallOpInterface *callOp,
2094 bool isMustTail, SourceRange clangLoc);
2095 RValue emitCall(const CIRGenFunctionInfo &funcInfo,
2096 const CIRGenCallee &callee, ReturnValueSlot returnValue,
2097 const CallArgList &args, bool isMustTail,
2098 cir::CIRCallOpInterface *callOrTryCall = nullptr) {
2099 assert(currSrcLoc && "source location must have been set");
2100 return emitCall(funcInfo, callee, returnValue, args, callOp: callOrTryCall,
2101 isMustTail, clangLoc: *currSrcLoc);
2102 }
2103
2104 RValue emitCall(clang::QualType calleeTy, const CIRGenCallee &callee,
2105 const clang::CallExpr *e, ReturnValueSlot returnValue);
2106
2107 /// Emit the call and return for a thunk function.
2108 void emitCallAndReturnForThunk(cir::FuncOp callee, SourceRange fnLoc,
2109 const ThunkInfo *thunk, bool isUnprototyped);
2110
2111 void emitCallArg(CallArgList &args, const clang::Expr *e,
2112 clang::QualType argType);
2113 void emitCallArgs(
2114 CallArgList &args, PrototypeWrapper prototype,
2115 llvm::iterator_range<clang::CallExpr::const_arg_iterator> argRange,
2116 AbstractCallee callee = AbstractCallee(), unsigned paramsToSkip = 0,
2117 EvaluationOrder order = EvaluationOrder::Default);
2118 RValue emitCallExpr(const clang::CallExpr *e,
2119 ReturnValueSlot returnValue = ReturnValueSlot());
2120 LValue emitCallExprLValue(const clang::CallExpr *e);
2121
2122 LValue emitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *e);
2123 LValue emitCXXConstructLValue(const CXXConstructExpr *e);
2124 CIRGenCallee emitCallee(const clang::Expr *e);
2125
2126 template <typename T>
2127 mlir::LogicalResult emitCaseDefaultCascade(const T *stmt, mlir::Type condType,
2128 mlir::ArrayAttr value,
2129 cir::CaseOpKind kind,
2130 bool buildingTopLevelCase);
2131
2132 LValue emitCXXTypeidLValue(const CXXTypeidExpr *e);
2133
2134 mlir::LogicalResult emitCaseStmt(const clang::CaseStmt &s,
2135 mlir::Type condType,
2136 bool buildingTopLevelCase);
2137
2138 LValue emitCastLValue(const CastExpr *e);
2139
2140 /// Emits an argument for a call to a `__builtin_assume`. If the builtin
2141 /// sanitizer is enabled, a runtime check is also emitted.
2142 mlir::Value emitCheckedArgForAssume(const Expr *e);
2143
2144 /// Emit a conversion from the specified complex type to the specified
2145 /// destination type, where the destination type is an LLVM scalar type.
2146 mlir::Value emitComplexToScalarConversion(mlir::Value src, QualType srcTy,
2147 QualType dstTy, SourceLocation loc);
2148
2149 LValue emitCompoundAssignmentLValue(const clang::CompoundAssignOperator *e);
2150 LValue emitCompoundLiteralLValue(const CompoundLiteralExpr *e);
2151
2152 void emitConstructorBody(FunctionArgList &args);
2153
2154 mlir::LogicalResult emitCoroutineBody(const CoroutineBodyStmt &s);
2155 cir::CoroEndOp emitCoroEndBuiltinCall(const CallExpr *e);
2156 cir::CoroIdOp emitCoroIDBuiltinCall(const CallExpr *e);
2157 cir::CoroAllocOp emitCoroAllocBuiltinCall(const CallExpr *e);
2158 cir::CoroBeginOp emitCoroBeginBuiltinCall(const CallExpr *e);
2159 cir::CoroPromiseOp emitCoroPromiseBuiltinCall(const CallExpr *e);
2160 cir::CoroDoneOp emitCoroDoneBuiltinCall(const CallExpr *e);
2161 cir::CoroResumeOp emitCoroResumeBuiltinCall(const CallExpr *e);
2162 cir::CoroDestroyOp emitCoroDestroyBuiltinCall(const CallExpr *e);
2163 cir::CoroNoopOp emitCoroNoopBuiltinCall(const CallExpr *e);
2164 cir::CoroSuspendOp emitCoroSuspendBuiltinCall(const CallExpr *e);
2165
2166 cir::CoroSizeOp emitCoroSizeBuiltinCall(const CallExpr *e);
2167 cir::CoroAlignOp emitCoroAlignBuiltinCall(const CallExpr *e);
2168 cir::CoroFreeOp emitCoroFreeBuiltin(const CallExpr *e);
2169 RValue emitCoroutineFrame();
2170
2171 void emitDestroy(Address addr, QualType type, Destroyer *destroyer);
2172
2173 void emitDestructorBody(FunctionArgList &args);
2174
2175 mlir::LogicalResult emitContinueStmt(const clang::ContinueStmt &s);
2176
2177 mlir::LogicalResult emitCoreturnStmt(const CoreturnStmt &s);
2178
2179 void emitCXXConstructExpr(const clang::CXXConstructExpr *e,
2180 AggValueSlot dest);
2181
2182 void emitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
2183 const clang::ArrayType *arrayType,
2184 Address arrayBegin, const CXXConstructExpr *e,
2185 bool newPointerIsChecked,
2186 bool zeroInitialize = false);
2187 void emitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
2188 mlir::Value numElements, Address arrayBase,
2189 const CXXConstructExpr *e,
2190 bool newPointerIsChecked, bool zeroInitialize,
2191 Address endOfInit);
2192 void emitCXXConstructorCall(const clang::CXXConstructorDecl *d,
2193 clang::CXXCtorType type, bool forVirtualBase,
2194 bool delegating, AggValueSlot thisAVS,
2195 const clang::CXXConstructExpr *e);
2196
2197 void emitCXXConstructorCall(const clang::CXXConstructorDecl *d,
2198 clang::CXXCtorType type, bool forVirtualBase,
2199 bool delegating, Address thisAddr,
2200 CallArgList &args, clang::SourceLocation loc);
2201
2202 void emitInheritedCXXConstructorCall(const CXXConstructorDecl *d,
2203 bool forVirtualBase, Address thisAddr,
2204 bool inheritedFromVBase,
2205 const CXXInheritedCtorInitExpr *e);
2206
2207 void emitInlinedInheritingCXXConstructorCall(
2208 SourceLocation loc, const CXXConstructorDecl *d, CXXCtorType ctorType,
2209 bool forVirtualBase, bool delegating, CallArgList &args);
2210
2211 void emitCXXDeleteExpr(const CXXDeleteExpr *e);
2212
2213 void emitCXXDestructorCall(const CXXDestructorDecl *dd, CXXDtorType type,
2214 bool forVirtualBase, bool delegating,
2215 Address thisAddr, QualType thisTy);
2216
2217 RValue emitCXXDestructorCall(GlobalDecl dtor, const CIRGenCallee &callee,
2218 mlir::Value thisVal, QualType thisTy,
2219 mlir::Value implicitParam,
2220 QualType implicitParamTy, const CallExpr *e);
2221
2222 mlir::LogicalResult emitCXXForRangeStmt(const CXXForRangeStmt &s,
2223 llvm::ArrayRef<const Attr *> attrs);
2224
2225 RValue emitCXXMemberCallExpr(const clang::CXXMemberCallExpr *e,
2226 ReturnValueSlot returnValue);
2227
2228 Address emitCXXMemberDataPointerAddress(
2229 const Expr *e, Address base, mlir::Value memberPtr,
2230 const MemberPointerType *memberPtrType, LValueBaseInfo *baseInfo);
2231
2232 RValue emitCXXMemberOrOperatorCall(
2233 const clang::CXXMethodDecl *md, const CIRGenCallee &callee,
2234 ReturnValueSlot returnValue, mlir::Value thisPtr,
2235 mlir::Value implicitParam, clang::QualType implicitParamTy,
2236 const clang::CallExpr *ce, CallArgList *rtlArgs);
2237
2238 RValue emitCXXMemberOrOperatorMemberCallExpr(
2239 const clang::CallExpr *ce, const clang::CXXMethodDecl *md,
2240 ReturnValueSlot returnValue, bool hasQualifier,
2241 clang::NestedNameSpecifier qualifier, bool isArrow,
2242 const clang::Expr *base);
2243
2244 RValue emitCXXMemberPointerCallExpr(const CXXMemberCallExpr *ce,
2245 ReturnValueSlot returnValue);
2246
2247 mlir::Value emitCXXNewExpr(const CXXNewExpr *e);
2248
2249 void emitNewArrayInitializer(const CXXNewExpr *e, QualType elementType,
2250 mlir::Type elementTy, Address beginPtr,
2251 mlir::Value numElements,
2252 mlir::Value allocSizeWithoutCookie);
2253
2254 /// Create a check for a function parameter that may potentially be
2255 /// declared as non-null.
2256 void emitNonNullArgCheck(RValue rv, QualType argType, SourceLocation argLoc,
2257 AbstractCallee ac, unsigned paramNum);
2258
2259 RValue emitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *e,
2260 const CXXMethodDecl *md,
2261 ReturnValueSlot returnValue);
2262
2263 RValue emitCUDAKernelCallExpr(const CUDAKernelCallExpr *expr,
2264 ReturnValueSlot returnValue);
2265
2266 RValue emitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *expr);
2267
2268 RValue emitNewOrDeleteBuiltinCall(const FunctionProtoType *type,
2269 const CallExpr *callExpr,
2270 OverloadedOperatorKind op);
2271
2272 void emitCXXTemporary(const CXXTemporary *temporary, QualType tempType,
2273 Address ptr);
2274
2275 void emitCXXThrowExpr(const CXXThrowExpr *e);
2276
2277 struct cxxTryBodyEmitter {
2278 virtual mlir::LogicalResult operator()(CIRGenFunction &cgf) = 0;
2279 virtual ~cxxTryBodyEmitter() = default;
2280 };
2281
2282 void emitBeginCatch(const CXXCatchStmt *catchStmt, mlir::Value ehToken);
2283
2284 mlir::LogicalResult emitCXXTryStmt(const clang::CXXTryStmt &s,
2285 cxxTryBodyEmitter &bodyCallback,
2286 bool isFnTryBlock = false);
2287 mlir::LogicalResult emitCXXTryStmt(const clang::CXXTryStmt &s);
2288
2289 void emitCtorPrologue(const clang::CXXConstructorDecl *ctor,
2290 clang::CXXCtorType ctorType, FunctionArgList &args);
2291
2292 // It's important not to confuse this and emitDelegateCXXConstructorCall.
2293 // Delegating constructors are the C++11 feature. The constructor delegate
2294 // optimization is used to reduce duplication in the base and complete
2295 // constructors where they are substantially the same.
2296 void emitDelegatingCXXConstructorCall(const CXXConstructorDecl *ctor,
2297 const FunctionArgList &args);
2298
2299 void emitDeleteCall(const FunctionDecl *deleteFD, mlir::Value ptr,
2300 QualType deleteTy);
2301
2302 mlir::LogicalResult emitDoStmt(const clang::DoStmt &s,
2303 llvm::ArrayRef<const Attr *> attrs = {});
2304
2305 mlir::Value emitCXXTypeidExpr(const CXXTypeidExpr *e);
2306 mlir::Value emitDynamicCast(Address thisAddr, const CXXDynamicCastExpr *dce);
2307
2308 /// Emit an expression as an initializer for an object (variable, field, etc.)
2309 /// at the given location. The expression is not necessarily the normal
2310 /// initializer for the object, and the address is not necessarily
2311 /// its normal location.
2312 ///
2313 /// \param init the initializing expression
2314 /// \param d the object to act as if we're initializing
2315 /// \param lvalue the lvalue to initialize
2316 /// \param capturedByInit true if \p d is a __block variable whose address is
2317 /// potentially changed by the initializer
2318 void emitExprAsInit(const clang::Expr *init, const clang::ValueDecl *d,
2319 LValue lvalue, bool capturedByInit = false);
2320
2321 mlir::LogicalResult emitFunctionBody(const clang::Stmt *body);
2322
2323 mlir::LogicalResult emitGotoStmt(const clang::GotoStmt &s);
2324
2325 mlir::LogicalResult emitIndirectGotoStmt(const IndirectGotoStmt &s);
2326
2327 void emitImplicitAssignmentOperatorBody(FunctionArgList &args);
2328
2329 void emitInitializerForField(clang::FieldDecl *field, LValue lhs,
2330 clang::Expr *init);
2331
2332 LValue emitPredefinedLValue(const PredefinedExpr *e);
2333
2334 mlir::Value emitPromotedComplexExpr(const Expr *e, QualType promotionType);
2335
2336 mlir::Value emitPromotedScalarExpr(const Expr *e, QualType promotionType);
2337
2338 mlir::Value emitPromotedValue(mlir::Value result, QualType promotionType);
2339
2340 void emitReturnOfRValue(mlir::Location loc, RValue rv, QualType ty);
2341
2342 mlir::Value emitRuntimeCall(mlir::Location loc, cir::FuncOp callee,
2343 llvm::ArrayRef<mlir::Value> args = {},
2344 mlir::NamedAttrList attrs = {});
2345
2346 void emitInvariantStart(CharUnits size, mlir::Value addr, mlir::Location loc);
2347
2348 /// Emit the computation of the specified expression of scalar type.
2349 mlir::Value emitScalarExpr(const clang::Expr *e,
2350 bool ignoreResultAssign = false);
2351
2352 mlir::Value emitScalarPrePostIncDec(const UnaryOperator *e, LValue lv);
2353
2354 /// Build a debug stoppoint if we are emitting debug info.
2355 void emitStopPoint(const Stmt *s);
2356
2357 // Build CIR for a statement. useCurrentScope should be true if no
2358 // new scopes need be created when finding a compound statement.
2359 mlir::LogicalResult emitStmt(const clang::Stmt *s, bool useCurrentScope,
2360 llvm::ArrayRef<const Attr *> attrs = {});
2361
2362 mlir::LogicalResult emitSimpleStmt(const clang::Stmt *s,
2363 bool useCurrentScope);
2364
2365 mlir::LogicalResult emitForStmt(const clang::ForStmt &s,
2366 llvm::ArrayRef<const Attr *> attrs = {});
2367
2368 void emitForwardingCallToLambda(const CXXMethodDecl *lambdaCallOperator,
2369 CallArgList &callArgs);
2370
2371 RValue emitCoawaitExpr(const CoawaitExpr &e,
2372 AggValueSlot aggSlot = AggValueSlot::ignored(),
2373 bool ignoreResult = false);
2374 LValue emitCoawaitLValue(const CoawaitExpr *e);
2375
2376 RValue emitCoyieldExpr(const CoyieldExpr &e,
2377 AggValueSlot aggSlot = AggValueSlot::ignored(),
2378 bool ignoreResult = false);
2379 LValue emitCoyieldLValue(const CoyieldExpr *e);
2380 /// Emit the computation of the specified expression of complex type,
2381 /// returning the result.
2382 mlir::Value emitComplexExpr(const Expr *e);
2383
2384 void emitComplexExprIntoLValue(const Expr *e, LValue dest, bool isInit);
2385
2386 mlir::Value emitComplexPrePostIncDec(const UnaryOperator *e, LValue lv);
2387
2388 LValue emitComplexAssignmentLValue(const BinaryOperator *e);
2389 LValue emitComplexCompoundAssignmentLValue(const CompoundAssignOperator *e);
2390 LValue emitScalarCompoundAssignWithComplex(const CompoundAssignOperator *e,
2391 mlir::Value &result);
2392
2393 mlir::LogicalResult
2394 emitCompoundStmt(const clang::CompoundStmt &s, Address *lastValue = nullptr,
2395 AggValueSlot slot = AggValueSlot::ignored());
2396
2397 mlir::LogicalResult
2398 emitCompoundStmtWithoutScope(const clang::CompoundStmt &s,
2399 Address *lastValue = nullptr,
2400 AggValueSlot slot = AggValueSlot::ignored());
2401
2402 void emitDecl(const clang::Decl &d, bool evaluateConditionDecl = false);
2403 mlir::LogicalResult emitDeclStmt(const clang::DeclStmt &s);
2404 LValue emitDeclRefLValue(const clang::DeclRefExpr *e);
2405
2406 mlir::LogicalResult emitDefaultStmt(const clang::DefaultStmt &s,
2407 mlir::Type condType,
2408 bool buildingTopLevelCase);
2409
2410 void emitDelegateCXXConstructorCall(const clang::CXXConstructorDecl *ctor,
2411 clang::CXXCtorType ctorType,
2412 const FunctionArgList &args,
2413 clang::SourceLocation loc);
2414
2415 /// We are performing a delegate call; that is, the current function is
2416 /// delegating to another one. Produce a r-value suitable for passing the
2417 /// given parameter.
2418 void emitDelegateCallArg(CallArgList &args, const clang::VarDecl *param,
2419 clang::SourceLocation loc);
2420
2421 /// Emit an `if` on a boolean condition to the specified blocks.
2422 /// FIXME: Based on the condition, this might try to simplify the codegen of
2423 /// the conditional based on the branch.
2424 /// In the future, we may apply code generation simplifications here,
2425 /// similar to those used in classic LLVM codegen
2426 /// See `EmitBranchOnBoolExpr` for inspiration.
2427 mlir::LogicalResult emitIfOnBoolExpr(const clang::Expr *cond,
2428 const clang::Stmt *thenS,
2429 const clang::Stmt *elseS);
2430 cir::IfOp emitIfOnBoolExpr(const clang::Expr *cond,
2431 BuilderCallbackRef thenBuilder,
2432 mlir::Location thenLoc,
2433 BuilderCallbackRef elseBuilder,
2434 std::optional<mlir::Location> elseLoc = {});
2435
2436 /// Build the cir.if for an already-emitted condition value.
2437 cir::IfOp emitIfOnBoolValue(mlir::Value condV, mlir::Location loc,
2438 BuilderCallbackRef thenBuilder,
2439 mlir::Location thenLoc,
2440 BuilderCallbackRef elseBuilder,
2441 std::optional<mlir::Location> elseLoc = {});
2442
2443 mlir::Value emitOpOnBoolExpr(mlir::Location loc, const clang::Expr *cond);
2444
2445 LValue emitPointerToDataMemberBinaryExpr(const BinaryOperator *e);
2446
2447 mlir::LogicalResult emitLabel(const clang::LabelDecl &d);
2448 mlir::LogicalResult emitLabelStmt(const clang::LabelStmt &s);
2449
2450 void emitLambdaDelegatingInvokeBody(const CXXMethodDecl *md);
2451 void emitLambdaStaticInvokeBody(const CXXMethodDecl *md);
2452
2453 mlir::LogicalResult emitIfStmt(const clang::IfStmt &s);
2454
2455 /// Emit code to compute the specified expression,
2456 /// ignoring the result.
2457 void emitIgnoredExpr(const clang::Expr *e);
2458
2459 RValue emitLoadOfBitfieldLValue(LValue lv, SourceLocation loc);
2460
2461 /// Load a complex number from the specified l-value.
2462 mlir::Value emitLoadOfComplex(LValue src, SourceLocation loc);
2463
2464 RValue emitLoadOfExtVectorElementLValue(LValue lv);
2465
2466 /// Given an expression that represents a value lvalue, this method emits
2467 /// the address of the lvalue, then loads the result as an rvalue,
2468 /// returning the rvalue.
2469 RValue emitLoadOfLValue(LValue lv, SourceLocation loc);
2470
2471 Address emitLoadOfReference(LValue refLVal, mlir::Location loc,
2472 LValueBaseInfo *pointeeBaseInfo);
2473 LValue emitLoadOfReferenceLValue(Address refAddr, mlir::Location loc,
2474 QualType refTy, AlignmentSource source);
2475
2476 /// EmitLoadOfScalar - Load a scalar value from an address, taking
2477 /// care to appropriately convert from the memory representation to
2478 /// the LLVM value representation. The l-value must be a simple
2479 /// l-value.
2480 mlir::Value emitLoadOfScalar(LValue lvalue, SourceLocation loc);
2481 mlir::Value emitLoadOfScalar(Address addr, bool isVolatile, QualType ty,
2482 SourceLocation loc, LValueBaseInfo baseInfo,
2483 bool isNontemporal = false);
2484
2485 /// Emit code to compute a designator that specifies the location
2486 /// of the expression.
2487 /// FIXME: document this function better.
2488 LValue emitLValue(const clang::Expr *e);
2489 LValue emitLValueForBitField(LValue base, const FieldDecl *field);
2490 LValue emitLValueForField(LValue base, const clang::FieldDecl *field);
2491
2492 /// Annotate the address of `field`; accesses must use the result.
2493 Address emitFieldAnnotations(const clang::FieldDecl *field, Address addr);
2494
2495 /// Annotate the storage of `d` at `addr`.
2496 void emitVarAnnotations(const clang::VarDecl *d, mlir::Value addr);
2497
2498 LValue emitLValueForLambdaField(const FieldDecl *field);
2499 LValue emitLValueForLambdaField(const FieldDecl *field,
2500 mlir::Value thisValue);
2501
2502 void emitLambdaVLACapture(const VariableArrayType *vat, LValue lv) {
2503 mlir::Value size = vlaSizeMap[vat->getSizeExpr()];
2504 emitStoreThroughLValue(src: RValue::get(v: size), dst: lv);
2505 }
2506
2507 /// Like emitLValueForField, excpet that if the Field is a reference, this
2508 /// will return the address of the reference and not the address of the value
2509 /// stored in the reference.
2510 LValue emitLValueForFieldInitialization(LValue base,
2511 const clang::FieldDecl *field,
2512 llvm::StringRef fieldName);
2513
2514 LValue emitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *e);
2515
2516 LValue emitMatrixElementExpr(const MatrixElementExpr *e);
2517 LValue emitMemberExpr(const MemberExpr *e);
2518
2519 /// Emit a musttail call for a thunk with a potentially different ABI.
2520 void emitMustTailThunk(GlobalDecl gd, mlir::Value adjustedThisPtr,
2521 cir::FuncOp callee);
2522
2523 /// Emit a call to an AMDGPU builtin function.
2524 std::optional<mlir::Value> emitAMDGPUBuiltinExpr(unsigned builtinID,
2525 const CallExpr *expr);
2526
2527 /// Emit a call to an NVPTX builtin function.
2528 std::optional<mlir::Value> emitNVPTXBuiltinExpr(unsigned builtinID,
2529 const CallExpr *expr);
2530
2531 /// Emit a call to a SPIR-V builtin function.
2532 std::optional<mlir::Value> emitSPIRVBuiltinExpr(unsigned builtinID,
2533 const CallExpr *expr);
2534
2535 /// Emit a device-side printf call for GPU targets.
2536 mlir::Value emitDevicePrintfCallExpr(const CallExpr *expr);
2537
2538 LValue emitOpaqueValueLValue(const OpaqueValueExpr *e);
2539
2540 LValue emitConditionalOperatorLValue(const AbstractConditionalOperator *expr);
2541
2542 /// Given an expression with a pointer type, emit the value and compute our
2543 /// best estimate of the alignment of the pointee.
2544 ///
2545 /// One reasonable way to use this information is when there's a language
2546 /// guarantee that the pointer must be aligned to some stricter value, and
2547 /// we're simply trying to ensure that sufficiently obvious uses of under-
2548 /// aligned objects don't get miscompiled; for example, a placement new
2549 /// into the address of a local variable. In such a case, it's quite
2550 /// reasonable to just ignore the returned alignment when it isn't from an
2551 /// explicit source.
2552 Address emitPointerWithAlignment(const clang::Expr *expr,
2553 LValueBaseInfo *baseInfo = nullptr);
2554
2555 /// Emits a reference binding to the passed in expression.
2556 RValue emitReferenceBindingToExpr(const Expr *e);
2557
2558 mlir::LogicalResult emitReturnStmt(const clang::ReturnStmt &s);
2559
2560 RValue emitRotate(const CallExpr *e, bool isRotateLeft);
2561
2562 mlir::Value emitScalarConstant(const ConstantEmission &constant, Expr *e);
2563
2564 /// Emit a conversion from the specified type to the specified destination
2565 /// type, both of which are CIR scalar types.
2566 mlir::Value emitScalarConversion(mlir::Value src, clang::QualType srcType,
2567 clang::QualType dstType,
2568 clang::SourceLocation loc);
2569
2570 void emitScalarInit(const clang::Expr *init, LValue lvalue,
2571 bool capturedByInit = false);
2572
2573 mlir::Value emitScalarOrConstFoldImmArg(unsigned iceArguments, unsigned idx,
2574 const Expr *argExpr);
2575
2576 void emitStaticVarDecl(const VarDecl &d, cir::GlobalLinkageKind linkage);
2577
2578 /// Emit a guarded initializer for a static local variable.
2579 void emitCXXGuardedInit(const VarDecl &varDecl, cir::GlobalOp globalOp,
2580 bool performInit);
2581
2582 void emitStoreOfComplex(mlir::Location loc, mlir::Value v, LValue dest,
2583 bool isInit);
2584
2585 void emitStoreOfScalar(mlir::Value value, Address addr, bool isVolatile,
2586 clang::QualType ty, LValueBaseInfo baseInfo,
2587 bool isInit = false, bool isNontemporal = false);
2588 void emitStoreOfScalar(mlir::Value value, LValue lvalue, bool isInit);
2589
2590 void emitStoreThroughExtVectorComponentLValue(RValue src, LValue dst);
2591
2592 /// Store the specified rvalue into the specified
2593 /// lvalue, where both are guaranteed to the have the same type, and that
2594 /// type is 'Ty'.
2595 void emitStoreThroughLValue(RValue src, LValue dst, bool isInit = false);
2596
2597 mlir::Value emitStoreThroughBitfieldLValue(RValue src, LValue dstresult);
2598
2599 LValue emitStringLiteralLValue(const StringLiteral *e,
2600 llvm::StringRef name = ".str");
2601
2602 mlir::LogicalResult emitSwitchBody(const clang::Stmt *s);
2603 mlir::LogicalResult emitSwitchCase(const clang::SwitchCase &s,
2604 bool buildingTopLevelCase);
2605 mlir::LogicalResult emitSwitchStmt(const clang::SwitchStmt &s);
2606
2607 mlir::LogicalResult emitSYCLKernelCallStmt(const SYCLKernelCallStmt &s);
2608
2609 void emitSYCLKernelCaller(const clang::OutlinedFunctionDecl *outlinedFnDecl,
2610 cir::FuncOp funcOp, cir::FuncType funcType,
2611 FunctionArgList &args);
2612
2613 /// Remove leftover empty and unreachable blocks from an emitted function.
2614 static void eraseEmptyAndUnusedBlocks(cir::FuncOp func);
2615
2616 std::optional<mlir::Value>
2617 emitTargetBuiltinExpr(unsigned builtinID, const clang::CallExpr *e,
2618 ReturnValueSlot &returnValue);
2619
2620 /// Emit a diagnostic if the target features required by \p targetDecl are
2621 /// not available in the calling function. Mirrors CodeGenFunction behavior.
2622 void checkTargetFeatures(const clang::CallExpr *e,
2623 const clang::FunctionDecl *targetDecl);
2624 void checkTargetFeatures(clang::SourceLocation loc,
2625 const clang::FunctionDecl *targetDecl);
2626
2627 /// Given a value and its clang type, returns the value casted to its memory
2628 /// representation.
2629 /// Note: CIR defers most of the special casting to the final lowering passes
2630 /// to conserve the high level information.
2631 mlir::Value emitToMemory(mlir::Value value, clang::QualType ty);
2632
2633 /// EmitFromMemory - Change a scalar value from its memory
2634 /// representation to its value representation.
2635 mlir::Value emitFromMemory(mlir::Value value, clang::QualType ty);
2636
2637 /// Emit a trap instruction, which is used to abort the program in an abnormal
2638 /// way, usually for debugging purposes.
2639 /// \p createNewBlock indicates whether to create a new block for the IR
2640 /// builder. Since the `cir.trap` operation is a terminator, operations that
2641 /// follow a trap cannot be emitted after `cir.trap` in the same block. To
2642 /// ensure these operations get emitted successfully, you need to create a new
2643 /// dummy block and set the insertion point there before continuing from the
2644 /// trap operation.
2645 void emitTrap(mlir::Location loc, bool createNewBlock);
2646
2647 LValue emitUnaryOpLValue(const clang::UnaryOperator *e);
2648
2649 mlir::Value emitUnPromotedValue(mlir::Value result, QualType unPromotionType);
2650
2651 /// Emit a reached-unreachable diagnostic if \p loc is valid and runtime
2652 /// checking is enabled. Otherwise, just emit an unreachable instruction.
2653 /// \p createNewBlock indicates whether to create a new block for the IR
2654 /// builder. Since the `cir.unreachable` operation is a terminator, operations
2655 /// that follow an unreachable point cannot be emitted after `cir.unreachable`
2656 /// in the same block. To ensure these operations get emitted successfully,
2657 /// you need to create a dummy block and set the insertion point there before
2658 /// continuing from the unreachable point.
2659 void emitUnreachable(clang::SourceLocation loc, bool createNewBlock);
2660
2661 /// This method handles emission of any variable declaration
2662 /// inside a function, including static vars etc.
2663 void emitVarDecl(const clang::VarDecl &d);
2664
2665 void emitVariablyModifiedType(QualType ty);
2666
2667 mlir::LogicalResult emitWhileStmt(const clang::WhileStmt &s,
2668 llvm::ArrayRef<const Attr *> attrs = {});
2669
2670 /// Build the hints that get attached to a loop, or a null attribute
2671 /// if there is nothing to say about it.
2672 cir::LoopHintAttr getLoopHints(llvm::ArrayRef<const Attr *> attrs,
2673 bool mustProgress);
2674
2675 std::optional<mlir::Value> emitRISCVBuiltinExpr(unsigned builtinID,
2676 const CallExpr *expr);
2677 cir::GetGlobalOp createGetCpuModel(mlir::Location loc);
2678 cir::GetGlobalOp createGetCpuFeatures2(mlir::Location loc);
2679 mlir::Value emitX86CpuIs(const CallExpr *expr);
2680 mlir::Value emitX86CpuIs(mlir::Location loc, StringRef cpuStr);
2681 mlir::Value emitX86CpuSupports(const CallExpr *expr);
2682 mlir::Value emitX86CpuSupports(mlir::Location loc,
2683 ArrayRef<StringRef> FeatureStrs);
2684 mlir::Value emitX86CpuSupports(mlir::Location loc,
2685 std::array<uint32_t, 4> FeatureMask);
2686 mlir::Value emitX86CpuInit(mlir::Location loc);
2687 using MSVCIntrin = CodeGenUtils::MSVCIntrin;
2688 mlir::Value emitX86MSVCIntrin(MSVCIntrin builtinID, const CallExpr *E);
2689 std::optional<mlir::Value> emitX86BuiltinExpr(unsigned builtinID,
2690 const CallExpr *expr);
2691
2692 /// Given an assignment `*lhs = rhs`, emit a test that checks if \p rhs is
2693 /// nonnull, if 1\p LHS is marked _Nonnull.
2694 void emitNullabilityCheck(LValue lhs, mlir::Value rhs,
2695 clang::SourceLocation loc);
2696
2697 /// An object to manage conditionally-evaluated expressions.
2698 class ConditionalEvaluation {
2699 CIRGenFunction &cgf;
2700
2701 /// The insertion point that precedes the conditional, stored as the
2702 /// enclosing block and the operation immediately before that point. Later
2703 /// operations (the condition, cleanup scopes) can be appended without
2704 /// moving this point. A null \c anchorAfter means the insertion point is
2705 /// the start of \c anchorBlock.
2706 mlir::Block *anchorBlock;
2707 mlir::Operation *anchorAfter = nullptr;
2708
2709 public:
2710 /// \p loc is the location of the conditional expression. It is used for
2711 /// the cleanup scope this may open.
2712 ConditionalEvaluation(CIRGenFunction &cgf, mlir::Location loc) : cgf(cgf) {
2713 // Open the cleanup scope that hosts cleanups deferred from inside this
2714 // conditional (see deferredConditionalCleanupStack). Only the outermost
2715 // conditional opens one, and only while a FullExprCleanupScope is
2716 // active to close it. When nothing is deferred into it the cleanup
2717 // region stays trivial and canonicalization inlines the scope away.
2718 if (cgf.currentFullExprCleanupScope && !cgf.isInConditionalBranch()) {
2719 cir::CleanupKind cleanupKind = cgf.getLangOpts().Exceptions
2720 ? cir::CleanupKind::All
2721 : cir::CleanupKind::Normal;
2722 cir::CleanupScopeOp scope = cir::CleanupScopeOp::create(
2723 builder&: cgf.builder, location: loc, cleanupKind,
2724 /*bodyBuilder=*/bodyBuilder: [](mlir::OpBuilder &, mlir::Location) {},
2725 /*cleanupBuilder=*/cleanupBuilder: [](mlir::OpBuilder &, mlir::Location) {});
2726 cgf.conditionalCleanupScopes.push_back(
2727 Elt: {.scope: scope, .deferredCleanupStackSize: cgf.deferredConditionalCleanupStack.size()});
2728 cgf.builder.setInsertionPointToEnd(&scope.getBodyRegion().front());
2729 }
2730
2731 anchorBlock = cgf.builder.getInsertionBlock();
2732 assert(anchorBlock && "conditional evaluation needs an insertion point");
2733 mlir::Block::iterator ip = cgf.builder.getInsertionPoint();
2734 if (ip != anchorBlock->begin())
2735 anchorAfter = &*std::prev(x: ip);
2736 }
2737
2738 void beginEvaluation() {
2739 assert(cgf.outermostConditional != this);
2740 if (!cgf.outermostConditional)
2741 cgf.outermostConditional = this;
2742 }
2743
2744 void endEvaluation() {
2745 assert(cgf.outermostConditional != nullptr);
2746 if (cgf.outermostConditional == this)
2747 cgf.outermostConditional = nullptr;
2748 }
2749
2750 /// Records \p op as the last operation emitted at the pre-conditional
2751 /// insertion point, so that a later emission lands after it rather than
2752 /// ahead of it.
2753 void advanceInsertPoint(mlir::Operation *op) { anchorAfter = op; }
2754
2755 /// Returns the insertion point which will be executed prior to each
2756 /// evaluation of the conditional code. In LLVM OG, this method
2757 /// is called getStartingBlock.
2758 mlir::OpBuilder::InsertPoint getInsertPoint() const {
2759 if (!anchorAfter)
2760 return mlir::OpBuilder::InsertPoint(anchorBlock, anchorBlock->begin());
2761 return mlir::OpBuilder::InsertPoint(
2762 anchorAfter->getBlock(), std::next(x: anchorAfter->getIterator()));
2763 }
2764 };
2765
2766 struct ConditionalInfo {
2767 std::optional<LValue> lhs{}, rhs{};
2768 mlir::Value result{};
2769 };
2770
2771 // Return true if we're currently emitting one branch or the other of a
2772 // conditional expression.
2773 bool isInConditionalBranch() const { return outermostConditional != nullptr; }
2774
2775 void setBeforeOutermostConditional(mlir::Value value, Address addr) {
2776 assert(isInConditionalBranch());
2777 {
2778 mlir::OpBuilder::InsertionGuard guard(builder);
2779 builder.restoreInsertionPoint(ip: outermostConditional->getInsertPoint());
2780 cir::StoreOp store = builder.createStore(
2781 loc: value.getLoc(), val: value, dst: addr, /*isVolatile=*/isVolatile: false,
2782 /*isNontemporal=*/isNontemporal: false,
2783 align: mlir::IntegerAttr::get(
2784 type: mlir::IntegerType::get(context: value.getContext(), width: 64),
2785 value: (uint64_t)addr.getAlignment().getAsAlign().value()));
2786 outermostConditional->advanceInsertPoint(op: store);
2787 }
2788 }
2789
2790 // Points to the outermost active conditional control. This is used so that
2791 // we know if a temporary should be destroyed conditionally.
2792 ConditionalEvaluation *outermostConditional = nullptr;
2793
2794 // The outermost conditional that a FullExprCleanupScope in a conditional
2795 // branch hides from isInConditionalBranch(), if any. The cleanups of that
2796 // full-expression run at its end, so they are not conditional, but a
2797 // temporary whose lifetime is extended beyond it is still created
2798 // conditionally.
2799 ConditionalEvaluation *hiddenOutermostConditional = nullptr;
2800
2801 /// An RAII object to record that we're evaluating a statement
2802 /// expression.
2803 class StmtExprEvaluation {
2804 CIRGenFunction &cgf;
2805
2806 /// We have to save the outermost conditional: cleanups in a
2807 /// statement expression aren't conditional just because the
2808 /// StmtExpr is.
2809 ConditionalEvaluation *savedOutermostConditional;
2810 ConditionalEvaluation *savedHiddenOutermostConditional;
2811
2812 public:
2813 StmtExprEvaluation(CIRGenFunction &cgf)
2814 : cgf(cgf), savedOutermostConditional(cgf.outermostConditional),
2815 savedHiddenOutermostConditional(cgf.hiddenOutermostConditional) {
2816 cgf.outermostConditional = nullptr;
2817 cgf.hiddenOutermostConditional = nullptr;
2818 }
2819
2820 ~StmtExprEvaluation() {
2821 cgf.outermostConditional = savedOutermostConditional;
2822 cgf.hiddenOutermostConditional = savedHiddenOutermostConditional;
2823 }
2824 };
2825
2826 template <typename FuncTy>
2827 ConditionalInfo emitConditionalBlocks(const AbstractConditionalOperator *e,
2828 const FuncTy &branchGenFunc);
2829
2830 /// Build a "reference" to a va_list; this is either the address or the value
2831 /// of the expression, depending on how va_list is defined.
2832 Address emitVAListRef(const Expr *e);
2833
2834 /// Emits the start of a CIR variable-argument operation (`cir.va_start`)
2835 ///
2836 /// \param vaList A reference to the \c va_list as emitted by either
2837 /// \c emitVAListRef or \c emitMSVAListRef.
2838 void emitVAStart(mlir::Value vaList);
2839
2840 /// Emits the end of a CIR variable-argument operation (`cir.va_start`)
2841 ///
2842 /// \param vaList A reference to the \c va_list as emitted by either
2843 /// \c emitVAListRef or \c emitMSVAListRef.
2844 void emitVAEnd(mlir::Value vaList);
2845
2846 /// Generate code to get an argument from the passed in pointer
2847 /// and update it accordingly.
2848 ///
2849 /// \param ve The \c VAArgExpr for which to generate code.
2850 ///
2851 /// \param vaListAddr Receives a reference to the \c va_list as emitted by
2852 /// either \c emitVAListRef or \c emitMSVAListRef.
2853 ///
2854 /// \returns SSA value with the argument.
2855 mlir::Value emitVAArg(VAArgExpr *ve);
2856
2857 /// ----------------------
2858 /// CIR build helpers
2859 /// -----------------
2860public:
2861 cir::AllocaOp createTempAlloca(mlir::Type ty, mlir::Location loc,
2862 const Twine &name = "tmp",
2863 mlir::Value arraySize = nullptr,
2864 bool insertIntoFnEntryBlock = false);
2865 cir::AllocaOp createTempAlloca(mlir::Type ty, mlir::Location loc,
2866 const Twine &name = "tmp",
2867 mlir::OpBuilder::InsertPoint ip = {},
2868 mlir::Value arraySize = nullptr);
2869 Address createTempAlloca(mlir::Type ty, CharUnits align, mlir::Location loc,
2870 const Twine &name = "tmp",
2871 mlir::Value arraySize = nullptr,
2872 Address *alloca = nullptr,
2873 mlir::OpBuilder::InsertPoint ip = {});
2874 Address createTempAlloca(mlir::Type ty,
2875 mlir::ptr::MemorySpaceAttrInterface destAddrSpace,
2876 CharUnits align, mlir::Location loc,
2877 const Twine &name = "tmp",
2878 mlir::Value arraySize = nullptr,
2879 Address *alloca = nullptr,
2880 mlir::OpBuilder::InsertPoint ip = {});
2881 Address createTempAllocaWithoutCast(mlir::Type ty, CharUnits align,
2882 mlir::Location loc,
2883 const Twine &name = "tmp",
2884 mlir::Value arraySize = nullptr,
2885 mlir::OpBuilder::InsertPoint ip = {});
2886 Address
2887 maybeCastStackAddressSpace(Address alloca,
2888 mlir::ptr::MemorySpaceAttrInterface destAddrSpace,
2889 mlir::Value arraySize = nullptr);
2890 Address createDefaultAlignTempAlloca(mlir::Type ty, mlir::Location loc,
2891 const Twine &name);
2892
2893 /// Create a temporary memory object of the given type, with
2894 /// appropriate alignmen and cast it to the default address space. Returns
2895 /// the original alloca instruction by \p Alloca if it is not nullptr.
2896 Address createMemTemp(QualType t, mlir::Location loc,
2897 const Twine &name = "tmp", Address *alloca = nullptr,
2898 mlir::OpBuilder::InsertPoint ip = {});
2899 Address createMemTemp(QualType t, CharUnits align, mlir::Location loc,
2900 const Twine &name = "tmp", Address *alloca = nullptr,
2901 mlir::OpBuilder::InsertPoint ip = {});
2902 Address createMemTempWithoutCast(QualType t, mlir::Location loc,
2903 const Twine &name = "tmp");
2904
2905 mlir::Value performAddrSpaceCast(mlir::Value v, mlir::Type destTy) const {
2906 if (cir::GlobalOp globalOp = v.getDefiningOp<cir::GlobalOp>())
2907 cgm.errorNYI(feature: "Global op addrspace cast");
2908 return builder.createAddrSpaceCast(src: v, newTy: destTy);
2909 }
2910
2911 //===--------------------------------------------------------------------===//
2912 // OpenMP Emission
2913 //===--------------------------------------------------------------------===//
2914public:
2915 /// Emits a Sema-synthesized OpenMP helper variable (loop counter, iteration
2916 /// variable or pre-init capture) without lifetime markers. These variables
2917 /// are emitted between nested combined/composite OpenMP ops, which must be
2918 /// direct children of each other's regions, so they must not open a
2919 /// lifetime cleanup scope.
2920 void emitOMPNoLifetimeVarDecl(const VarDecl &d);
2921
2922 mlir::LogicalResult emitOMPScopeDirective(const OMPScopeDirective &s);
2923 mlir::LogicalResult emitOMPErrorDirective(const OMPErrorDirective &s);
2924 mlir::LogicalResult emitOMPParallelDirective(const OMPParallelDirective &s);
2925 mlir::LogicalResult emitOMPTaskwaitDirective(const OMPTaskwaitDirective &s);
2926 mlir::LogicalResult emitOMPTaskyieldDirective(const OMPTaskyieldDirective &s);
2927 mlir::LogicalResult emitOMPBarrierDirective(const OMPBarrierDirective &s);
2928 mlir::LogicalResult emitOMPMetaDirective(const OMPMetaDirective &s);
2929 mlir::LogicalResult emitOMPCanonicalLoop(const OMPCanonicalLoop &s);
2930 mlir::LogicalResult emitOMPSimdDirective(const OMPSimdDirective &s);
2931 mlir::LogicalResult emitOMPTileDirective(const OMPTileDirective &s);
2932 mlir::LogicalResult emitOMPUnrollDirective(const OMPUnrollDirective &s);
2933 mlir::LogicalResult emitOMPFuseDirective(const OMPFuseDirective &s);
2934 mlir::LogicalResult emitOMPForDirective(const OMPForDirective &s);
2935 mlir::LogicalResult emitOMPForSimdDirective(const OMPForSimdDirective &s);
2936 mlir::LogicalResult emitOMPSectionsDirective(const OMPSectionsDirective &s);
2937 mlir::LogicalResult emitOMPSectionDirective(const OMPSectionDirective &s);
2938 mlir::LogicalResult emitOMPSingleDirective(const OMPSingleDirective &s);
2939 mlir::LogicalResult emitOMPMasterDirective(const OMPMasterDirective &s);
2940 mlir::LogicalResult emitOMPCriticalDirective(const OMPCriticalDirective &s);
2941 mlir::LogicalResult
2942 emitOMPParallelForDirective(const OMPParallelForDirective &s);
2943 mlir::LogicalResult
2944 emitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &s);
2945 mlir::LogicalResult
2946 emitOMPParallelMasterDirective(const OMPParallelMasterDirective &s);
2947 mlir::LogicalResult
2948 emitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &s);
2949 mlir::LogicalResult emitOMPTaskDirective(const OMPTaskDirective &s);
2950 mlir::LogicalResult emitOMPTaskgroupDirective(const OMPTaskgroupDirective &s);
2951 mlir::LogicalResult emitOMPFlushDirective(const OMPFlushDirective &s);
2952 mlir::LogicalResult emitOMPDepobjDirective(const OMPDepobjDirective &s);
2953 mlir::LogicalResult emitOMPScanDirective(const OMPScanDirective &s);
2954 mlir::LogicalResult
2955 emitOMPOrderedStandaloneDirective(const OMPOrderedStandaloneDirective &s);
2956 mlir::LogicalResult
2957 emitOMPOrderedBlockAssocDirective(const OMPOrderedBlockAssocDirective &s);
2958 mlir::LogicalResult emitOMPAtomicDirective(const OMPAtomicDirective &s);
2959 mlir::LogicalResult emitOMPTargetDirective(const OMPTargetDirective &s);
2960 mlir::LogicalResult emitOMPTeamsDirective(const OMPTeamsDirective &s);
2961 mlir::LogicalResult
2962 emitOMPCancellationPointDirective(const OMPCancellationPointDirective &s);
2963 mlir::LogicalResult emitOMPCancelDirective(const OMPCancelDirective &s);
2964 mlir::LogicalResult
2965 emitOMPTargetDataDirective(const OMPTargetDataDirective &s);
2966 mlir::LogicalResult
2967 emitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &s);
2968 mlir::LogicalResult
2969 emitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &s);
2970 mlir::LogicalResult
2971 emitOMPTargetParallelDirective(const OMPTargetParallelDirective &s);
2972 mlir::LogicalResult
2973 emitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &s);
2974 mlir::LogicalResult emitOMPTaskLoopDirective(const OMPTaskLoopDirective &s);
2975 mlir::LogicalResult
2976 emitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &s);
2977 mlir::LogicalResult
2978 emitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &s);
2979 mlir::LogicalResult
2980 emitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &s);
2981 mlir::LogicalResult
2982 emitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &s);
2983 mlir::LogicalResult
2984 emitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &s);
2985 mlir::LogicalResult
2986 emitOMPParallelGenericLoopDirective(const OMPParallelGenericLoopDirective &s);
2987 mlir::LogicalResult
2988 emitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &s);
2989 mlir::LogicalResult emitOMPParallelMaskedTaskLoopDirective(
2990 const OMPParallelMaskedTaskLoopDirective &s);
2991 mlir::LogicalResult emitOMPParallelMaskedTaskLoopSimdDirective(
2992 const OMPParallelMaskedTaskLoopSimdDirective &s);
2993 mlir::LogicalResult emitOMPParallelMasterTaskLoopDirective(
2994 const OMPParallelMasterTaskLoopDirective &s);
2995 mlir::LogicalResult emitOMPParallelMasterTaskLoopSimdDirective(
2996 const OMPParallelMasterTaskLoopSimdDirective &s);
2997 mlir::LogicalResult
2998 emitOMPDistributeDirective(const OMPDistributeDirective &s);
2999 mlir::LogicalResult emitOMPDistributeParallelForDirective(
3000 const OMPDistributeParallelForDirective &s);
3001 mlir::LogicalResult emitOMPDistributeParallelForSimdDirective(
3002 const OMPDistributeParallelForSimdDirective &s);
3003 mlir::LogicalResult
3004 emitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &s);
3005 mlir::LogicalResult emitOMPTargetParallelGenericLoopDirective(
3006 const OMPTargetParallelGenericLoopDirective &s);
3007 mlir::LogicalResult
3008 emitOMPTargetGenericLoopDirective(const OMPTargetGenericLoopDirective &s);
3009 mlir::LogicalResult emitOMPTargetParallelForSimdDirective(
3010 const OMPTargetParallelForSimdDirective &s);
3011 mlir::LogicalResult
3012 emitOMPTargetSimdDirective(const OMPTargetSimdDirective &s);
3013 mlir::LogicalResult emitOMPTargetTeamsGenericLoopDirective(
3014 const OMPTargetTeamsGenericLoopDirective &s);
3015 mlir::LogicalResult
3016 emitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &s);
3017 mlir::LogicalResult
3018 emitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &s);
3019 mlir::LogicalResult
3020 emitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &s);
3021 mlir::LogicalResult emitOMPTeamsDistributeParallelForSimdDirective(
3022 const OMPTeamsDistributeParallelForSimdDirective &s);
3023 mlir::LogicalResult emitOMPTeamsDistributeParallelForDirective(
3024 const OMPTeamsDistributeParallelForDirective &s);
3025 mlir::LogicalResult
3026 emitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &s);
3027 mlir::LogicalResult
3028 emitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &s);
3029 mlir::LogicalResult emitOMPTargetTeamsDistributeDirective(
3030 const OMPTargetTeamsDistributeDirective &s);
3031 mlir::LogicalResult emitOMPTargetTeamsDistributeParallelForDirective(
3032 const OMPTargetTeamsDistributeParallelForDirective &s);
3033 mlir::LogicalResult emitOMPTargetTeamsDistributeParallelForSimdDirective(
3034 const OMPTargetTeamsDistributeParallelForSimdDirective &s);
3035 mlir::LogicalResult emitOMPTargetTeamsDistributeSimdDirective(
3036 const OMPTargetTeamsDistributeSimdDirective &s);
3037 mlir::LogicalResult emitOMPInteropDirective(const OMPInteropDirective &s);
3038 mlir::LogicalResult emitOMPDispatchDirective(const OMPDispatchDirective &s);
3039 mlir::LogicalResult
3040 emitOMPGenericLoopDirective(const OMPGenericLoopDirective &s);
3041 mlir::LogicalResult emitOMPReverseDirective(const OMPReverseDirective &s);
3042 mlir::LogicalResult emitOMPSplitDirective(const OMPSplitDirective &s);
3043 mlir::LogicalResult
3044 emitOMPInterchangeDirective(const OMPInterchangeDirective &s);
3045 mlir::LogicalResult emitOMPFlattenDirective(const OMPFlattenDirective &s);
3046 mlir::LogicalResult emitOMPAssumeDirective(const OMPAssumeDirective &s);
3047 mlir::LogicalResult emitOMPMaskedDirective(const OMPMaskedDirective &s);
3048 mlir::LogicalResult emitOMPStripeDirective(const OMPStripeDirective &s);
3049
3050 void emitOMPThreadPrivateDecl(const OMPThreadPrivateDecl &d);
3051 void emitOMPGroupPrivateDecl(const OMPGroupPrivateDecl &d);
3052 void emitOMPCapturedExpr(const OMPCapturedExprDecl &d);
3053 void emitOMPAllocateDecl(const OMPAllocateDecl &d);
3054 void emitOMPDeclareReduction(const OMPDeclareReductionDecl &d);
3055 void emitOMPDeclareMapper(const OMPDeclareMapperDecl &d);
3056 void emitOMPRequiresDecl(const OMPRequiresDecl &d);
3057
3058 //===--------------------------------------------------------------------===//
3059 // OpenACC Emission
3060 //===--------------------------------------------------------------------===//
3061private:
3062 template <typename Op>
3063 Op emitOpenACCOp(mlir::Location start, OpenACCDirectiveKind dirKind,
3064 llvm::ArrayRef<const OpenACCClause *> clauses);
3065 // Function to do the basic implementation of an operation with an Associated
3066 // Statement. Models AssociatedStmtConstruct.
3067 template <typename Op, typename TermOp>
3068 mlir::LogicalResult
3069 emitOpenACCOpAssociatedStmt(mlir::Location start, mlir::Location end,
3070 OpenACCDirectiveKind dirKind,
3071 llvm::ArrayRef<const OpenACCClause *> clauses,
3072 const Stmt *associatedStmt);
3073
3074 template <typename Op, typename TermOp>
3075 mlir::LogicalResult emitOpenACCOpCombinedConstruct(
3076 mlir::Location start, mlir::Location end, OpenACCDirectiveKind dirKind,
3077 llvm::ArrayRef<const OpenACCClause *> clauses, const Stmt *loopStmt);
3078
3079 template <typename Op>
3080 void emitOpenACCClauses(Op &op, OpenACCDirectiveKind dirKind,
3081 ArrayRef<const OpenACCClause *> clauses);
3082 // The second template argument doesn't need to be a template, since it should
3083 // always be an mlir::acc::LoopOp, but as this is a template anyway, we make
3084 // it a template argument as this way we can avoid including the OpenACC MLIR
3085 // headers here. We will count on linker failures/explicit instantiation to
3086 // ensure we don't mess this up, but it is only called from 1 place, and
3087 // instantiated 3x.
3088 template <typename ComputeOp, typename LoopOp>
3089 void emitOpenACCClauses(ComputeOp &op, LoopOp &loopOp,
3090 OpenACCDirectiveKind dirKind,
3091 ArrayRef<const OpenACCClause *> clauses);
3092
3093 // The OpenACC LoopOp requires that we have auto, seq, or independent on all
3094 // LoopOp operations for the 'none' device type case. This function checks if
3095 // the LoopOp has one, else it updates it to have one.
3096 void updateLoopOpParallelism(mlir::acc::LoopOp &op, bool isOrphan,
3097 OpenACCDirectiveKind dk);
3098
3099 // The OpenACC 'cache' construct actually applies to the 'loop' if present. So
3100 // keep track of the 'loop' so that we can add the cache vars to it correctly.
3101 mlir::acc::LoopOp *activeLoopOp = nullptr;
3102
3103 struct ActiveOpenACCLoopRAII {
3104 CIRGenFunction &cgf;
3105 mlir::acc::LoopOp *oldLoopOp;
3106
3107 ActiveOpenACCLoopRAII(CIRGenFunction &cgf, mlir::acc::LoopOp *newOp)
3108 : cgf(cgf), oldLoopOp(cgf.activeLoopOp) {
3109 cgf.activeLoopOp = newOp;
3110 }
3111 ~ActiveOpenACCLoopRAII() { cgf.activeLoopOp = oldLoopOp; }
3112 };
3113
3114 // Keep track of the last place we inserted a 'recipe' so that we can insert
3115 // the next one in lexical order.
3116 mlir::OpBuilder::InsertPoint lastRecipeLocation;
3117
3118public:
3119 // Helper type used to store the list of important information for a 'data'
3120 // clause variable, or a 'cache' variable reference.
3121 struct OpenACCDataOperandInfo {
3122 mlir::Location beginLoc;
3123 mlir::Value varValue;
3124 std::string name;
3125 // The type of the original variable reference: that is, after 'bounds' have
3126 // removed pointers/array types/etc. So in the case of int arr[5], and a
3127 // private(arr[1]), 'origType' is 'int', but 'baseType' is 'int[5]'.
3128 QualType origType;
3129 QualType baseType;
3130 llvm::SmallVector<mlir::Value> bounds;
3131 // The list of types that we found when going through the bounds, which we
3132 // can use to properly set the alloca section.
3133 llvm::SmallVector<QualType> boundTypes;
3134 };
3135
3136 // Gets the collection of info required to lower and OpenACC clause or cache
3137 // construct variable reference.
3138 OpenACCDataOperandInfo getOpenACCDataOperandInfo(const Expr *e);
3139 // Helper function to emit the integer expressions as required by an OpenACC
3140 // clause/construct.
3141 mlir::Value emitOpenACCIntExpr(const Expr *intExpr);
3142 // Helper function to emit an integer constant as an mlir int type, used for
3143 // constants in OpenACC constructs/clauses.
3144 mlir::Value createOpenACCConstantInt(mlir::Location loc, unsigned width,
3145 int64_t value);
3146
3147 mlir::LogicalResult
3148 emitOpenACCComputeConstruct(const OpenACCComputeConstruct &s);
3149 mlir::LogicalResult emitOpenACCLoopConstruct(const OpenACCLoopConstruct &s);
3150 mlir::LogicalResult
3151 emitOpenACCCombinedConstruct(const OpenACCCombinedConstruct &s);
3152 mlir::LogicalResult emitOpenACCDataConstruct(const OpenACCDataConstruct &s);
3153 mlir::LogicalResult
3154 emitOpenACCEnterDataConstruct(const OpenACCEnterDataConstruct &s);
3155 mlir::LogicalResult
3156 emitOpenACCExitDataConstruct(const OpenACCExitDataConstruct &s);
3157 mlir::LogicalResult
3158 emitOpenACCHostDataConstruct(const OpenACCHostDataConstruct &s);
3159 mlir::LogicalResult emitOpenACCWaitConstruct(const OpenACCWaitConstruct &s);
3160 mlir::LogicalResult emitOpenACCInitConstruct(const OpenACCInitConstruct &s);
3161 mlir::LogicalResult
3162 emitOpenACCShutdownConstruct(const OpenACCShutdownConstruct &s);
3163 mlir::LogicalResult emitOpenACCSetConstruct(const OpenACCSetConstruct &s);
3164 mlir::LogicalResult
3165 emitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &s);
3166 mlir::LogicalResult
3167 emitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &s);
3168 mlir::LogicalResult emitOpenACCCacheConstruct(const OpenACCCacheConstruct &s);
3169
3170 void emitOpenACCDeclare(const OpenACCDeclareDecl &d);
3171 void emitOpenACCRoutine(const OpenACCRoutineDecl &d);
3172
3173 /// Create a temporary memory object for the given aggregate type.
3174 AggValueSlot createAggTemp(QualType ty, mlir::Location loc,
3175 const Twine &name = "tmp",
3176 Address *alloca = nullptr) {
3177 assert(!cir::MissingFeatures::aggValueSlot());
3178 return AggValueSlot::forAddr(
3179 addr: createMemTemp(t: ty, loc, name, alloca), quals: ty.getQualifiers(),
3180 isDestructed: AggValueSlot::IsNotDestructed, isAliased: AggValueSlot::IsNotAliased,
3181 mayOverlap: AggValueSlot::DoesNotOverlap);
3182 }
3183
3184private:
3185 QualType getVarArgType(const Expr *arg);
3186
3187 bool shouldEmitLifetimeMarkers = false;
3188 /// Set when the current function has a goto/switch that may bypass a local's
3189 /// init; lifetime markers are then suppressed. See functionMightHaveBypass.
3190 bool fnHasBypassStmt = false;
3191
3192 bool shouldEmitLifetimeMarkersForAutoVar() const {
3193 return shouldEmitLifetimeMarkers && !fnHasBypassStmt;
3194 }
3195
3196 class InlinedInheritingConstructorScope {
3197 public:
3198 InlinedInheritingConstructorScope(CIRGenFunction &cgf, GlobalDecl gd)
3199 : cgf(cgf), oldCurGD(cgf.curGD), oldCurFuncDecl(cgf.curFuncDecl),
3200 oldCurCodeDecl(cgf.curCodeDecl),
3201 oldCxxabiThisDecl(cgf.cxxabiThisDecl),
3202 oldCxxThisValue(cgf.cxxThisValue),
3203 oldCxxabiThisAlignment(cgf.cxxabiThisAlignment),
3204 oldCxxThisAlignment(cgf.cxxThisAlignment),
3205 oldReturnValue(cgf.returnValue), oldFnRetTy(cgf.fnRetTy),
3206 oldCxxInheritedCtorInitExprArgs(
3207 std::move(cgf.cxxInheritedCtorInitExprArgs)) {
3208 cgf.curGD = gd;
3209 cgf.curFuncDecl = cast<CXXConstructorDecl>(Val: gd.getDecl());
3210 cgf.curCodeDecl = cgf.curFuncDecl;
3211 cgf.cxxabiThisDecl = nullptr;
3212 cgf.cxxabiThisValue = nullptr;
3213 cgf.cxxThisValue = nullptr;
3214 cgf.cxxThisAlignment = CharUnits();
3215 cgf.cxxabiThisAlignment = CharUnits();
3216 cgf.returnValue = Address::invalid();
3217 cgf.fnRetTy = QualType();
3218 cgf.cxxInheritedCtorInitExprArgs.clear();
3219 // FIXME: at one point when we want to call one of these, we'll need
3220 // CXXInheritedCtorInitExprArgs here too.
3221 }
3222 ~InlinedInheritingConstructorScope() {
3223 cgf.curGD = oldCurGD;
3224 cgf.curFuncDecl = oldCurFuncDecl;
3225 cgf.curCodeDecl = oldCurCodeDecl;
3226 cgf.cxxabiThisDecl = oldCxxabiThisDecl;
3227 cgf.cxxabiThisValue = oldCxxabiThisValue;
3228 cgf.cxxThisValue = oldCxxThisValue;
3229 cgf.cxxThisAlignment = oldCxxThisAlignment;
3230 cgf.cxxabiThisAlignment = oldCxxabiThisAlignment;
3231 cgf.returnValue = oldReturnValue;
3232 cgf.fnRetTy = oldFnRetTy;
3233 cgf.cxxInheritedCtorInitExprArgs =
3234 std::move(oldCxxInheritedCtorInitExprArgs);
3235 }
3236
3237 private:
3238 CIRGenFunction &cgf;
3239 GlobalDecl oldCurGD;
3240 const Decl *oldCurFuncDecl;
3241 const Decl *oldCurCodeDecl;
3242 ImplicitParamDecl *oldCxxabiThisDecl;
3243 mlir::Value oldCxxabiThisValue;
3244 mlir::Value oldCxxThisValue;
3245 clang::CharUnits oldCxxabiThisAlignment;
3246 clang::CharUnits oldCxxThisAlignment;
3247 Address oldReturnValue;
3248 QualType oldFnRetTy;
3249 CallArgList oldCxxInheritedCtorInitExprArgs;
3250 };
3251};
3252
3253} // namespace clang::CIRGen
3254
3255#endif
3256