| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Internal per-function state used for AST-to-ClangIR code gen |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #ifndef CLANG_LIB_CIR_CODEGEN_CIRGENFUNCTION_H |
| 14 | #define CLANG_LIB_CIR_CODEGEN_CIRGENFUNCTION_H |
| 15 | |
| 16 | #include "CIRGenBuilder.h" |
| 17 | #include "CIRGenCall.h" |
| 18 | #include "CIRGenModule.h" |
| 19 | #include "CIRGenTypeCache.h" |
| 20 | #include "CIRGenValue.h" |
| 21 | #include "EHScopeStack.h" |
| 22 | |
| 23 | #include "Address.h" |
| 24 | |
| 25 | #include "clang/AST/ASTContext.h" |
| 26 | #include "clang/AST/BaseSubobject.h" |
| 27 | #include "clang/AST/CharUnits.h" |
| 28 | #include "clang/AST/CurrentSourceLocExprScope.h" |
| 29 | #include "clang/AST/Decl.h" |
| 30 | #include "clang/AST/ExprCXX.h" |
| 31 | #include "clang/AST/Stmt.h" |
| 32 | #include "clang/AST/Type.h" |
| 33 | #include "clang/Basic/OpenACCKinds.h" |
| 34 | #include "clang/Basic/OperatorKinds.h" |
| 35 | #include "clang/Basic/TargetBuiltins.h" |
| 36 | #include "clang/CIR/Dialect/IR/CIRDialect.h" |
| 37 | #include "clang/CIR/MissingFeatures.h" |
| 38 | #include "clang/CIR/TypeEvaluationKind.h" |
| 39 | #include "clang/CodeGenUtils/CallUtils.h" |
| 40 | #include "clang/CodeGenUtils/StmtUtils.h" |
| 41 | #include "clang/CodeGenUtils/TargetUtils.h" |
| 42 | #include "llvm/ADT/ScopedHashTable.h" |
| 43 | #include "llvm/IR/Instructions.h" |
| 44 | #include "llvm/IR/Intrinsics.h" |
| 45 | |
| 46 | namespace mlir { |
| 47 | namespace acc { |
| 48 | class LoopOp; |
| 49 | } // namespace acc |
| 50 | } // namespace mlir |
| 51 | |
| 52 | namespace clang { |
| 53 | class OutlinedFunctionDecl; |
| 54 | class SYCLKernelCallStmt; |
| 55 | } // namespace clang |
| 56 | |
| 57 | namespace clang::CIRGen { |
| 58 | |
| 59 | struct CGCoroData; |
| 60 | |
| 61 | class CIRGenFunction : public CIRGenTypeCache { |
| 62 | public: |
| 63 | CIRGenModule &cgm; |
| 64 | |
| 65 | private: |
| 66 | /// The builder is a helper class to create IR inside a function. The |
| 67 | /// builder is stateful, in particular it keeps an "insertion point": this |
| 68 | /// is where the next operations will be introduced. |
| 69 | CIRGenBuilderTy &builder; |
| 70 | |
| 71 | /// Saves the builder's constrained floating-point configuration on |
| 72 | /// construction and restores it on destruction. The builder is shared |
| 73 | /// across all functions in the module, so its constrained-FP state must be |
| 74 | /// scoped to each function's emission. |
| 75 | /// |
| 76 | /// Note that the similarly named CIRGenFunction::CIRGenFPOptionsRAII |
| 77 | /// intentionally avoids restoring the "isFPConstrained" state because that |
| 78 | /// state is function-wide, but this object does restore the state so that |
| 79 | /// any CIRGenFunction instance created while we are in the process of |
| 80 | /// emitting another function cannot corrupt the prior functions state. |
| 81 | struct ConstrainedFPRAII { |
| 82 | CIRGenBuilderTy &builder; |
| 83 | bool savedIsFPConstrained; |
| 84 | LangOptions::FPExceptionModeKind savedExcept; |
| 85 | llvm::RoundingMode savedRounding; |
| 86 | cir::FastMathFlags savedFastMathFlags; |
| 87 | |
| 88 | explicit ConstrainedFPRAII(CIRGenBuilderTy &builder) |
| 89 | : builder(builder), savedIsFPConstrained(builder.getIsFPConstrained()), |
| 90 | savedExcept(builder.getDefaultConstrainedExcept()), |
| 91 | savedRounding(builder.getDefaultConstrainedRounding()), |
| 92 | savedFastMathFlags(builder.getFastMathFlags()) {} |
| 93 | ~ConstrainedFPRAII() { |
| 94 | builder.setIsFPConstrained(savedIsFPConstrained); |
| 95 | builder.setDefaultConstrainedExcept(savedExcept); |
| 96 | builder.setDefaultConstrainedRounding(savedRounding); |
| 97 | builder.setFastMathFlags(savedFastMathFlags); |
| 98 | } |
| 99 | } constrainedFPState{builder}; |
| 100 | |
| 101 | public: |
| 102 | /// The GlobalDecl for the current function being compiled or the global |
| 103 | /// variable currently being initialized. |
| 104 | clang::GlobalDecl curGD; |
| 105 | |
| 106 | unsigned nextCleanupDestIndex = 1; |
| 107 | |
| 108 | /// The compiler-generated variable that holds the return value. |
| 109 | std::optional<mlir::Value> fnRetAlloca; |
| 110 | |
| 111 | // Holds coroutine data if the current function is a coroutine. We use a |
| 112 | // wrapper to manage its lifetime, so that we don't have to define CGCoroData |
| 113 | // in this header. |
| 114 | struct CGCoroInfo { |
| 115 | std::unique_ptr<CGCoroData> data; |
| 116 | CGCoroInfo(); |
| 117 | ~CGCoroInfo(); |
| 118 | }; |
| 119 | CGCoroInfo curCoro; |
| 120 | |
| 121 | bool isCoroutine() const { return curCoro.data != nullptr; } |
| 122 | |
| 123 | /// The temporary alloca to hold the return value. This is |
| 124 | /// invalid iff the function has no return value. |
| 125 | Address returnValue = Address::invalid(); |
| 126 | |
| 127 | /// Tracks function scope overall cleanup handling. |
| 128 | EHScopeStack ehStack; |
| 129 | |
| 130 | typedef void Destroyer(CIRGenFunction &cgf, Address addr, QualType ty); |
| 131 | |
| 132 | class CapturedCleanups; |
| 133 | |
| 134 | /// A cleanup whose destructor call is not emitted where the cleanup is |
| 135 | /// registered. Used by the lifetime-extended cleanup stack, whose entries |
| 136 | /// are later promoted onto the EH scope stack, and by the deferred |
| 137 | /// conditional cleanup stack, whose entries are emitted directly into a |
| 138 | /// conditional scope's cleanup region. |
| 139 | /// |
| 140 | /// A valid \c activeFlag means the cleanup is conditional. The flag tracks |
| 141 | /// at run time whether the object has been constructed, and guards the |
| 142 | /// destructor call. |
| 143 | /// |
| 144 | /// Currently only DestroyObject cleanups use this. When other cleanup types |
| 145 | /// are needed (e.g., CallLifetimeEnd), this struct can be extended with a |
| 146 | /// std::variant of cleanup data types. |
| 147 | struct PendingCleanupEntry { |
| 148 | CleanupKind kind; |
| 149 | Address addr; |
| 150 | QualType type; |
| 151 | Destroyer *destroyer; |
| 152 | Address activeFlag = Address::invalid(); |
| 153 | /// If set, the cleanup is captured by this owner instead of opening a |
| 154 | /// cir.cleanup.scope when it is pushed onto the EH stack. |
| 155 | CapturedCleanups *capturedBy = nullptr; |
| 156 | }; |
| 157 | |
| 158 | llvm::SmallVector<PendingCleanupEntry> lifetimeExtendedCleanupStack; |
| 159 | |
| 160 | /// Cleanups for temporaries constructed inside a conditional. |
| 161 | /// |
| 162 | /// Temporaries are destroyed in reverse order of construction at the end of |
| 163 | /// the full expression. A cleanup lives in the cleanup region of a |
| 164 | /// cir.cleanup.scope, and those scopes nest, so nesting is what encodes the |
| 165 | /// order. |
| 166 | /// |
| 167 | /// For an unconditionally constructed temporary, EHScopeStack::pushCleanup |
| 168 | /// opens a scope where the temporary is constructed and leaves the builder in |
| 169 | /// its body, so each later temporary nests one level deeper and is destroyed |
| 170 | /// first. |
| 171 | /// |
| 172 | /// For a temporary constructed inside a conditional, pushFullExprCleanup |
| 173 | /// records the cleanup on this stack, paired with an active flag that is |
| 174 | /// cleared ahead of the conditional and set once the object is constructed. |
| 175 | /// The destructor call is emitted guarded by that flag, so it runs only on |
| 176 | /// the paths that constructed the object. |
| 177 | /// |
| 178 | /// ConditionalEvaluation opens the region holding those calls where the |
| 179 | /// conditional begins, nesting it inside the scopes of earlier temporaries |
| 180 | /// and outside later ones. Its body stays open for the rest of the full |
| 181 | /// expression, so its cleanup region runs at the end of it. |
| 182 | /// FullExprCleanupScope::exit closes these innermost first, then pops the |
| 183 | /// enclosing EH-stack cleanups. |
| 184 | llvm::SmallVector<PendingCleanupEntry> deferredConditionalCleanupStack; |
| 185 | |
| 186 | /// One record per cir.cleanup.scope opened by a ConditionalEvaluation, as |
| 187 | /// described above. These are bookkeeping owned by CIRGenFunction, not RAII |
| 188 | /// objects. The scope outlives the ConditionalEvaluation that opened it, |
| 189 | /// and the enclosing FullExprCleanupScope is what closes it. |
| 190 | struct ConditionalCleanupScope { |
| 191 | cir::CleanupScopeOp scope; |
| 192 | |
| 193 | /// The size of deferredConditionalCleanupStack when this scope was |
| 194 | /// opened. Every open conditional scope shares that one stack, so the |
| 195 | /// entries from this index onward are the ones deferred from inside this |
| 196 | /// conditional. They are emitted into this scope's cleanup region, in |
| 197 | /// reverse, when it is closed. |
| 198 | size_t deferredCleanupStackSize; |
| 199 | }; |
| 200 | llvm::SmallVector<ConditionalCleanupScope> conditionalCleanupScopes; |
| 201 | |
| 202 | /// The cir.cleanup.scope of the innermost FullExprCleanupScope that |
| 203 | /// materialized one, or null. ConditionalEvaluation opens a conditional |
| 204 | /// cleanup scope only while this is set, since a FullExprCleanupScope is |
| 205 | /// what closes those scopes. |
| 206 | cir::CleanupScopeOp currentFullExprCleanupScope = nullptr; |
| 207 | |
| 208 | /// Set by emitAutoVarDeclWithCapturedCleanups while it emits the initializer |
| 209 | /// of a variable, such as a loop condition variable, that lifetime-extends |
| 210 | /// temporaries with a destructor. |
| 211 | /// The cleanups of the temporaries that \c extendingDecl extends are |
| 212 | /// captured by \c cleanups, e.g. for the loop's cleanup region, and guarded |
| 213 | /// by \c activeFlag, instead of opening cir.cleanup.scope operations around |
| 214 | /// the code that follows the declaration. |
| 215 | struct LifetimeExtensionCapture { |
| 216 | const clang::ValueDecl *extendingDecl; |
| 217 | CapturedCleanups *cleanups; |
| 218 | Address activeFlag; |
| 219 | }; |
| 220 | const LifetimeExtensionCapture *lifetimeExtensionCapture = nullptr; |
| 221 | |
| 222 | /// A cleanup that was pushed to the EH stack but whose deactivation is |
| 223 | /// deferred until the enclosing CleanupDeactivationScope exits. Used to |
| 224 | /// protect partially-constructed aggregates (e.g. lambda captures) so that |
| 225 | /// already-initialized sub-objects are destroyed if a later initializer |
| 226 | /// throws, while avoiding double-destruction after full construction. |
| 227 | struct DeferredDeactivateCleanup { |
| 228 | EHScopeStack::stable_iterator cleanup; |
| 229 | mlir::Operation *dominatingIP; |
| 230 | }; |
| 231 | llvm::SmallVector<DeferredDeactivateCleanup> deferredDeactivationCleanupStack; |
| 232 | |
| 233 | /// Scope that deactivates all enclosed deferred cleanups on exit. |
| 234 | /// Mirrors CodeGenFunction::CleanupDeactivationScope in classic codegen. |
| 235 | struct CleanupDeactivationScope { |
| 236 | CIRGenFunction &cgf; |
| 237 | size_t oldDeactivateCleanupStackSize; |
| 238 | bool deactivated = false; |
| 239 | |
| 240 | CleanupDeactivationScope(CIRGenFunction &cgf) |
| 241 | : cgf(cgf), oldDeactivateCleanupStackSize( |
| 242 | cgf.deferredDeactivationCleanupStack.size()) {} |
| 243 | |
| 244 | void forceDeactivate() { |
| 245 | assert(!deactivated && "Deactivating already deactivated scope" ); |
| 246 | auto &stack = cgf.deferredDeactivationCleanupStack; |
| 247 | for (size_t i = stack.size(); i > oldDeactivateCleanupStackSize; i--) { |
| 248 | cgf.deactivateCleanupBlock(cleanup: stack[i - 1].cleanup, |
| 249 | dominatingIP: stack[i - 1].dominatingIP); |
| 250 | stack[i - 1].dominatingIP->erase(); |
| 251 | } |
| 252 | stack.resize(N: oldDeactivateCleanupStackSize); |
| 253 | deactivated = true; |
| 254 | } |
| 255 | |
| 256 | ~CleanupDeactivationScope() { |
| 257 | if (!deactivated) |
| 258 | forceDeactivate(); |
| 259 | } |
| 260 | }; |
| 261 | |
| 262 | GlobalDecl curSEHParent; |
| 263 | |
| 264 | /// If a ParmVarDecl had the pass_object_size attribute, this will contain a |
| 265 | /// mapping from said ParmVarDecl to its implicit "object_size" parameter. |
| 266 | llvm::SmallDenseMap<const ParmVarDecl *, const ImplicitParamDecl *> |
| 267 | sizeArguments; |
| 268 | |
| 269 | /// A mapping from NRVO variables to the flags used to indicate |
| 270 | /// when the NRVO has been applied to this variable. |
| 271 | llvm::DenseMap<const VarDecl *, mlir::Value> nrvoFlags; |
| 272 | |
| 273 | llvm::DenseMap<const clang::ValueDecl *, clang::FieldDecl *> |
| 274 | lambdaCaptureFields; |
| 275 | clang::FieldDecl *lambdaThisCaptureField = nullptr; |
| 276 | |
| 277 | /// CXXThisDecl - When generating code for a C++ member function, |
| 278 | /// this will hold the implicit 'this' declaration. |
| 279 | ImplicitParamDecl *cxxabiThisDecl = nullptr; |
| 280 | mlir::Value cxxabiThisValue = nullptr; |
| 281 | mlir::Value cxxThisValue = nullptr; |
| 282 | clang::CharUnits cxxabiThisAlignment; |
| 283 | clang::CharUnits cxxThisAlignment; |
| 284 | |
| 285 | /// When generating code for a constructor or destructor, this will hold the |
| 286 | /// implicit argument (e.g. VTT). |
| 287 | ImplicitParamDecl *cxxStructorImplicitParamDecl{}; |
| 288 | mlir::Value cxxStructorImplicitParamValue{}; |
| 289 | |
| 290 | /// The value of 'this' to sue when evaluating CXXDefaultInitExprs within this |
| 291 | /// expression. |
| 292 | Address cxxDefaultInitExprThis = Address::invalid(); |
| 293 | |
| 294 | /// The values of function arguments to use when evaluating |
| 295 | /// CXXInheritedCtorInitExprs within this context. |
| 296 | CallArgList cxxInheritedCtorInitExprArgs; |
| 297 | |
| 298 | /// The current array initialization index when evaluating an |
| 299 | /// ArrayInitIndexExpr within an ArrayInitLoopExpr. |
| 300 | mlir::Value arrayInitIndex = nullptr; |
| 301 | |
| 302 | // Holds the Decl for the current outermost non-closure context |
| 303 | const clang::Decl *curFuncDecl = nullptr; |
| 304 | /// This is the inner-most code context, which includes blocks. |
| 305 | const clang::Decl *curCodeDecl = nullptr; |
| 306 | const CIRGenFunctionInfo *curFnInfo = nullptr; |
| 307 | QualType fnRetTy; |
| 308 | |
| 309 | /// The current function or global initializer that is generated code for. |
| 310 | /// This is usually a cir::FuncOp, but it can also be a cir::GlobalOp for |
| 311 | /// global initializers. |
| 312 | mlir::Operation *curFn = nullptr; |
| 313 | |
| 314 | /// Save Parameter Decl for coroutine. |
| 315 | llvm::SmallVector<const ParmVarDecl *> fnArgs; |
| 316 | |
| 317 | using DeclMapTy = llvm::DenseMap<const clang::Decl *, Address>; |
| 318 | /// This keeps track of the CIR allocas or globals for local C |
| 319 | /// declarations. |
| 320 | DeclMapTy localDeclMap; |
| 321 | |
| 322 | /// The type of the condition for the emitting switch statement. |
| 323 | llvm::SmallVector<mlir::Type, 2> condTypeStack; |
| 324 | |
| 325 | clang::ASTContext &getContext() const { return cgm.getASTContext(); } |
| 326 | |
| 327 | CIRGenBuilderTy &getBuilder() { return builder; } |
| 328 | |
| 329 | CIRGenModule &getCIRGenModule() { return cgm; } |
| 330 | const CIRGenModule &getCIRGenModule() const { return cgm; } |
| 331 | |
| 332 | mlir::Block *getCurFunctionEntryBlock() { |
| 333 | // We currently assume this isn't called for a global initializer. |
| 334 | auto fn = mlir::cast<cir::FuncOp>(Val: curFn); |
| 335 | return &fn.getRegion().front(); |
| 336 | } |
| 337 | |
| 338 | /// Sanitizers enabled for this function. |
| 339 | clang::SanitizerSet sanOpts; |
| 340 | |
| 341 | class CIRGenFPOptionsRAII { |
| 342 | public: |
| 343 | CIRGenFPOptionsRAII(CIRGenFunction &cgf, FPOptions FPFeatures); |
| 344 | CIRGenFPOptionsRAII(CIRGenFunction &cgf, const clang::Expr *E); |
| 345 | ~CIRGenFPOptionsRAII(); |
| 346 | |
| 347 | private: |
| 348 | void ConstructorHelper(clang::FPOptions FPFeatures); |
| 349 | CIRGenFunction &cgf; |
| 350 | clang::FPOptions oldFPFeatures; |
| 351 | LangOptions::FPExceptionModeKind oldExcept; |
| 352 | llvm::RoundingMode oldRounding; |
| 353 | std::optional<cir::FastMathFlags> oldFastMathFlags; |
| 354 | }; |
| 355 | clang::FPOptions curFPFeatures; |
| 356 | |
| 357 | void setFastMathFlags(FPOptions fpFeatures); |
| 358 | |
| 359 | /// Convert the active Clang floating-point options to CIR fast-math flags. |
| 360 | cir::FastMathFlags getCurrentFastMathFlags() const; |
| 361 | |
| 362 | /// Create a CIR fast-math attribute from the specified flags. |
| 363 | cir::FastMathFlagsAttr getFastMathFlagsAttr(cir::FastMathFlags fastMathFlags); |
| 364 | |
| 365 | /// The symbol table maps a variable name to a value in the current scope. |
| 366 | /// Entering a function creates a new scope, and the function arguments are |
| 367 | /// added to the mapping. When the processing of a function is terminated, |
| 368 | /// the scope is destroyed and the mappings created in this scope are |
| 369 | /// dropped. |
| 370 | using SymTableTy = llvm::ScopedHashTable<const clang::Decl *, mlir::Value>; |
| 371 | SymTableTy symbolTable; |
| 372 | |
| 373 | /// Whether a cir.stacksave operation has been added. Used to avoid |
| 374 | /// inserting cir.stacksave for multiple VLAs in the same scope. |
| 375 | bool didCallStackSave = false; |
| 376 | |
| 377 | /// Whether or not a Microsoft-style asm block has been processed within |
| 378 | /// this fuction. These can potentially set the return value. |
| 379 | bool sawAsmBlock = false; |
| 380 | |
| 381 | /// In C++, whether we are code generating a thunk. This controls whether we |
| 382 | /// should emit cleanups. |
| 383 | bool curFuncIsThunk = false; |
| 384 | |
| 385 | mlir::Type convertTypeForMem(QualType t); |
| 386 | |
| 387 | mlir::Type convertType(clang::QualType t); |
| 388 | mlir::Type convertType(const TypeDecl *t) { |
| 389 | return convertType(t: getContext().getTypeDeclType(Decl: t)); |
| 390 | } |
| 391 | |
| 392 | /// Get integer from a mlir::Value that is an int constant or a constant op. |
| 393 | static int64_t getSExtIntValueFromConstOp(mlir::Value val) { |
| 394 | auto constOp = val.getDefiningOp<cir::ConstantOp>(); |
| 395 | assert(constOp && "getSExtIntValueFromConstOp call with non ConstantOp" ); |
| 396 | return constOp.getIntValue().getSExtValue(); |
| 397 | } |
| 398 | |
| 399 | /// Get zero-extended integer from a mlir::Value that is an int constant or a |
| 400 | /// constant op. |
| 401 | static int64_t getZExtIntValueFromConstOp(mlir::Value val) { |
| 402 | auto constOp = val.getDefiningOp<cir::ConstantOp>(); |
| 403 | assert(constOp && "getZExtIntValueFromConstOp call with non ConstantOp" ); |
| 404 | return constOp.getIntValue().getZExtValue(); |
| 405 | } |
| 406 | |
| 407 | /// Return the cir::TypeEvaluationKind of QualType \c type. |
| 408 | static cir::TypeEvaluationKind getEvaluationKind(clang::QualType type); |
| 409 | |
| 410 | static bool hasScalarEvaluationKind(clang::QualType type) { |
| 411 | return getEvaluationKind(type) == cir::TEK_Scalar; |
| 412 | } |
| 413 | |
| 414 | static bool hasAggregateEvaluationKind(clang::QualType type) { |
| 415 | return getEvaluationKind(type) == cir::TEK_Aggregate; |
| 416 | } |
| 417 | |
| 418 | CIRGenFunction(CIRGenModule &cgm, CIRGenBuilderTy &builder, |
| 419 | bool suppressNewContext = false); |
| 420 | ~CIRGenFunction(); |
| 421 | |
| 422 | CIRGenTypes &getTypes() const { return cgm.getTypes(); } |
| 423 | |
| 424 | const TargetInfo &getTarget() const { return cgm.getTarget(); } |
| 425 | mlir::MLIRContext &getMLIRContext() { return cgm.getMLIRContext(); } |
| 426 | |
| 427 | const TargetCIRGenInfo &getTargetHooks() const { |
| 428 | return cgm.getTargetCIRGenInfo(); |
| 429 | } |
| 430 | |
| 431 | // --------------------- |
| 432 | // Opaque value handling |
| 433 | // --------------------- |
| 434 | |
| 435 | /// Keeps track of the current set of opaque value expressions. |
| 436 | llvm::DenseMap<const OpaqueValueExpr *, LValue> opaqueLValues; |
| 437 | llvm::DenseMap<const OpaqueValueExpr *, RValue> opaqueRValues; |
| 438 | |
| 439 | // This keeps track of the associated size for each VLA type. |
| 440 | // We track this by the size expression rather than the type itself because |
| 441 | // in certain situations, like a const qualifier applied to an VLA typedef, |
| 442 | // multiple VLA types can share the same size expression. |
| 443 | // FIXME: Maybe this could be a stack of maps that is pushed/popped as we |
| 444 | // enter/leave scopes. |
| 445 | llvm::DenseMap<const Expr *, mlir::Value> vlaSizeMap; |
| 446 | |
| 447 | public: |
| 448 | /// A non-RAII class containing all the information about a bound |
| 449 | /// opaque value. OpaqueValueMapping, below, is a RAII wrapper for |
| 450 | /// this which makes individual mappings very simple; using this |
| 451 | /// class directly is useful when you have a variable number of |
| 452 | /// opaque values or don't want the RAII functionality for some |
| 453 | /// reason. |
| 454 | class OpaqueValueMappingData { |
| 455 | const OpaqueValueExpr *opaqueValue; |
| 456 | bool boundLValue; |
| 457 | |
| 458 | OpaqueValueMappingData(const OpaqueValueExpr *ov, bool boundLValue) |
| 459 | : opaqueValue(ov), boundLValue(boundLValue) {} |
| 460 | |
| 461 | public: |
| 462 | OpaqueValueMappingData() : opaqueValue(nullptr) {} |
| 463 | |
| 464 | static bool shouldBindAsLValue(const Expr *expr) { |
| 465 | // gl-values should be bound as l-values for obvious reasons. |
| 466 | // Records should be bound as l-values because IR generation |
| 467 | // always keeps them in memory. Expressions of function type |
| 468 | // act exactly like l-values but are formally required to be |
| 469 | // r-values in C. |
| 470 | return expr->isGLValue() || expr->getType()->isFunctionType() || |
| 471 | hasAggregateEvaluationKind(type: expr->getType()); |
| 472 | } |
| 473 | |
| 474 | static OpaqueValueMappingData |
| 475 | bind(CIRGenFunction &cgf, const OpaqueValueExpr *ov, const Expr *e) { |
| 476 | if (shouldBindAsLValue(expr: ov)) |
| 477 | return bind(cgf, ov, lv: cgf.emitLValue(e)); |
| 478 | return bind(cgf, ov, rv: cgf.emitAnyExpr(e)); |
| 479 | } |
| 480 | |
| 481 | static OpaqueValueMappingData |
| 482 | bind(CIRGenFunction &cgf, const OpaqueValueExpr *ov, const LValue &lv) { |
| 483 | assert(shouldBindAsLValue(ov)); |
| 484 | cgf.opaqueLValues.insert(KV: std::make_pair(x&: ov, y: lv)); |
| 485 | return OpaqueValueMappingData(ov, true); |
| 486 | } |
| 487 | |
| 488 | static OpaqueValueMappingData |
| 489 | bind(CIRGenFunction &cgf, const OpaqueValueExpr *ov, const RValue &rv) { |
| 490 | assert(!shouldBindAsLValue(ov)); |
| 491 | cgf.opaqueRValues.insert(KV: std::make_pair(x&: ov, y: rv)); |
| 492 | |
| 493 | OpaqueValueMappingData data(ov, false); |
| 494 | |
| 495 | // Work around an extremely aggressive peephole optimization in |
| 496 | // EmitScalarConversion which assumes that all other uses of a |
| 497 | // value are extant. |
| 498 | assert(!cir::MissingFeatures::peepholeProtection() && "NYI" ); |
| 499 | return data; |
| 500 | } |
| 501 | |
| 502 | bool isValid() const { return opaqueValue != nullptr; } |
| 503 | void clear() { opaqueValue = nullptr; } |
| 504 | |
| 505 | void unbind(CIRGenFunction &cgf) { |
| 506 | assert(opaqueValue && "no data to unbind!" ); |
| 507 | |
| 508 | if (boundLValue) { |
| 509 | cgf.opaqueLValues.erase(Val: opaqueValue); |
| 510 | } else { |
| 511 | cgf.opaqueRValues.erase(Val: opaqueValue); |
| 512 | assert(!cir::MissingFeatures::peepholeProtection() && "NYI" ); |
| 513 | } |
| 514 | } |
| 515 | }; |
| 516 | |
| 517 | /// An RAII object to set (and then clear) a mapping for an OpaqueValueExpr. |
| 518 | class OpaqueValueMapping { |
| 519 | CIRGenFunction &cgf; |
| 520 | OpaqueValueMappingData data; |
| 521 | |
| 522 | public: |
| 523 | static bool shouldBindAsLValue(const Expr *expr) { |
| 524 | return OpaqueValueMappingData::shouldBindAsLValue(expr); |
| 525 | } |
| 526 | |
| 527 | /// Build the opaque value mapping for the given conditional |
| 528 | /// operator if it's the GNU ?: extension. This is a common |
| 529 | /// enough pattern that the convenience operator is really |
| 530 | /// helpful. |
| 531 | /// |
| 532 | OpaqueValueMapping(CIRGenFunction &cgf, |
| 533 | const AbstractConditionalOperator *op) |
| 534 | : cgf(cgf) { |
| 535 | if (mlir::isa<ConditionalOperator>(Val: op)) |
| 536 | // Leave Data empty. |
| 537 | return; |
| 538 | |
| 539 | const BinaryConditionalOperator *e = |
| 540 | mlir::cast<BinaryConditionalOperator>(Val: op); |
| 541 | data = OpaqueValueMappingData::bind(cgf, ov: e->getOpaqueValue(), |
| 542 | e: e->getCommon()); |
| 543 | } |
| 544 | |
| 545 | /// Build the opaque value mapping for an OpaqueValueExpr whose source |
| 546 | /// expression is set to the expression the OVE represents. |
| 547 | OpaqueValueMapping(CIRGenFunction &cgf, const OpaqueValueExpr *ov) |
| 548 | : cgf(cgf) { |
| 549 | if (ov) { |
| 550 | assert(ov->getSourceExpr() && "wrong form of OpaqueValueMapping used " |
| 551 | "for OVE with no source expression" ); |
| 552 | data = OpaqueValueMappingData::bind(cgf, ov, e: ov->getSourceExpr()); |
| 553 | } |
| 554 | } |
| 555 | |
| 556 | OpaqueValueMapping(CIRGenFunction &cgf, const OpaqueValueExpr *opaqueValue, |
| 557 | LValue lvalue) |
| 558 | : cgf(cgf), |
| 559 | data(OpaqueValueMappingData::bind(cgf, ov: opaqueValue, lv: lvalue)) {} |
| 560 | |
| 561 | OpaqueValueMapping(CIRGenFunction &cgf, const OpaqueValueExpr *opaqueValue, |
| 562 | RValue rvalue) |
| 563 | : cgf(cgf), |
| 564 | data(OpaqueValueMappingData::bind(cgf, ov: opaqueValue, rv: rvalue)) {} |
| 565 | |
| 566 | void pop() { |
| 567 | data.unbind(cgf); |
| 568 | data.clear(); |
| 569 | } |
| 570 | |
| 571 | ~OpaqueValueMapping() { |
| 572 | if (data.isValid()) |
| 573 | data.unbind(cgf); |
| 574 | } |
| 575 | }; |
| 576 | |
| 577 | private: |
| 578 | /// Declare a variable in the current scope, return success if the variable |
| 579 | /// wasn't declared yet. |
| 580 | void declare(mlir::Value addrVal, const clang::Decl *var, clang::QualType ty, |
| 581 | mlir::Location loc, clang::CharUnits alignment, |
| 582 | bool isParam = false); |
| 583 | |
| 584 | public: |
| 585 | mlir::Value createDummyValue(mlir::Location loc, clang::QualType qt); |
| 586 | |
| 587 | void emitNullInitialization(mlir::Location loc, Address destPtr, QualType ty); |
| 588 | |
| 589 | private: |
| 590 | // Track current variable initialization (if there's one) |
| 591 | const clang::VarDecl *currVarDecl = nullptr; |
| 592 | class VarDeclContext { |
| 593 | CIRGenFunction &p; |
| 594 | const clang::VarDecl *oldVal = nullptr; |
| 595 | |
| 596 | public: |
| 597 | VarDeclContext(CIRGenFunction &p, const VarDecl *value) : p(p) { |
| 598 | if (p.currVarDecl) |
| 599 | oldVal = p.currVarDecl; |
| 600 | p.currVarDecl = value; |
| 601 | } |
| 602 | |
| 603 | /// Can be used to restore the state early, before the dtor |
| 604 | /// is run. |
| 605 | void restore() { p.currVarDecl = oldVal; } |
| 606 | ~VarDeclContext() { restore(); } |
| 607 | }; |
| 608 | |
| 609 | public: |
| 610 | /// Use to track source locations across nested visitor traversals. |
| 611 | /// Always use a `SourceLocRAIIObject` to change currSrcLoc. |
| 612 | std::optional<SourceRange> currSrcLoc; |
| 613 | |
| 614 | class SourceLocRAIIObject { |
| 615 | CIRGenFunction &cgf; |
| 616 | std::optional<SourceRange> oldLoc; |
| 617 | |
| 618 | public: |
| 619 | SourceLocRAIIObject(CIRGenFunction &cgf, SourceRange value) : cgf(cgf) { |
| 620 | if (cgf.currSrcLoc) |
| 621 | oldLoc = cgf.currSrcLoc; |
| 622 | cgf.currSrcLoc = value; |
| 623 | } |
| 624 | |
| 625 | /// Can be used to restore the state early, before the dtor |
| 626 | /// is run. |
| 627 | void restore() { cgf.currSrcLoc = oldLoc; } |
| 628 | ~SourceLocRAIIObject() { restore(); } |
| 629 | }; |
| 630 | |
| 631 | using SymTableScopeTy = |
| 632 | llvm::ScopedHashTableScope<const clang::Decl *, mlir::Value>; |
| 633 | |
| 634 | /// Hold counters for incrementally naming temporaries |
| 635 | unsigned counterRefTmp = 0; |
| 636 | unsigned counterAggTmp = 0; |
| 637 | std::string getCounterRefTmpAsString(); |
| 638 | std::string getCounterAggTmpAsString(); |
| 639 | |
| 640 | /// Helpers to convert Clang's SourceLocation to a MLIR Location. |
| 641 | mlir::Location getLoc(clang::SourceLocation srcLoc); |
| 642 | mlir::Location getLoc(clang::SourceRange srcLoc); |
| 643 | mlir::Location getLoc(mlir::Location lhs, mlir::Location rhs); |
| 644 | |
| 645 | const clang::LangOptions &getLangOpts() const { return cgm.getLangOpts(); } |
| 646 | |
| 647 | bool checkIfFunctionMustProgress() { |
| 648 | if (cgm.getCodeGenOpts().getFiniteLoops() == |
| 649 | clang::CodeGenOptions::FiniteLoopsKind::Never) |
| 650 | return false; |
| 651 | |
| 652 | // C++11 and later guarantees that a thread eventually will do one of the |
| 653 | // following (C++11 [intro.multithread]p24 and C++17 [intro.progress]p1): |
| 654 | // - terminate, |
| 655 | // - make a call to a library I/O function, |
| 656 | // - perform an access through a volatile glvalue, or |
| 657 | // - perform a synchronization operation or an atomic operation. |
| 658 | // |
| 659 | // Hence each function is 'mustprogress' in C++11 or later. |
| 660 | return getLangOpts().CPlusPlus11; |
| 661 | } |
| 662 | |
| 663 | // [C++26][intro.progress] (DR) |
| 664 | // The implementation may assume that any thread will eventually do one of |
| 665 | // the following: |
| 666 | // [...] |
| 667 | // - continue execution of a trivial infinite loop ([stmt.iter.general]). |
| 668 | // |
| 669 | // A trivial infinite loop with no side effects (e.g. `while (true) {}`) |
| 670 | // is therefore exempt from the forward-progress guarantee. If such a loop |
| 671 | // is found in a function that was speculatively marked 'mustprogress', |
| 672 | // that attribute must be removed: otherwise the optimizer would be |
| 673 | // licensed to assume the loop terminates and could delete it. |
| 674 | bool checkIfLoopMustProgress(const clang::Expr *controllingExpression, |
| 675 | bool hasEmptyBody) { |
| 676 | return clang::CodeGenUtils::checkIfLoopMustProgress( |
| 677 | LangOpts: getLangOpts(), CGOpts: cgm.getCodeGenOpts(), Ctx&: getContext(), |
| 678 | ControllingExpression: controllingExpression, HasEmptyBody: hasEmptyBody, RemoveMustProgress: [this] { |
| 679 | curFn->removeAttr(name: cir::CIRDialect::getMustProgressAttrName()); |
| 680 | }); |
| 681 | } |
| 682 | |
| 683 | /// True if an insertion point is defined. If not, this indicates that the |
| 684 | /// current code being emitted is unreachable. |
| 685 | /// FIXME(cir): we need to inspect this and perhaps use a cleaner mechanism |
| 686 | /// since we don't yet force null insertion point to designate behavior (like |
| 687 | /// LLVM's codegen does) and we probably shouldn't. |
| 688 | bool haveInsertPoint() const { |
| 689 | return builder.getInsertionBlock() != nullptr; |
| 690 | } |
| 691 | |
| 692 | /// True if code emitted at the builder's insertion point can be reached. |
| 693 | /// After emitting a terminator CIRGen opens a fresh block to continue in, so |
| 694 | /// the insertion block holds unreachable code whenever it is neither its |
| 695 | /// region's entry block nor the target of a branch. |
| 696 | bool insertionPointIsReachable() const { |
| 697 | mlir::Block *block = builder.getInsertionBlock(); |
| 698 | return block && (block->isEntryBlock() || !block->hasNoPredecessors()); |
| 699 | } |
| 700 | |
| 701 | // Wrapper for function prototype sources. Wraps either a FunctionProtoType or |
| 702 | // an ObjCMethodDecl. |
| 703 | struct PrototypeWrapper { |
| 704 | llvm::PointerUnion<const clang::FunctionProtoType *, |
| 705 | const clang::ObjCMethodDecl *> |
| 706 | p; |
| 707 | |
| 708 | PrototypeWrapper(const clang::FunctionProtoType *ft) : p(ft) {} |
| 709 | PrototypeWrapper(const clang::ObjCMethodDecl *md) : p(md) {} |
| 710 | }; |
| 711 | |
| 712 | using EvaluationOrder = CodeGenUtils::EvaluationOrder; |
| 713 | |
| 714 | bool isLValueSuitableForInlineAtomic(LValue lv); |
| 715 | |
| 716 | RValue emitAtomicLoad(LValue lvalue, SourceLocation loc, |
| 717 | AggValueSlot slot = AggValueSlot::ignored()); |
| 718 | RValue emitAtomicLoad(LValue lvalue, SourceLocation loc, cir::MemOrder order, |
| 719 | bool isVolatile = false, |
| 720 | AggValueSlot slot = AggValueSlot::ignored()); |
| 721 | |
| 722 | /// An abstract representation of regular/ObjC call/message targets. |
| 723 | class AbstractCallee { |
| 724 | /// The function declaration of the callee. |
| 725 | [[maybe_unused]] const clang::Decl *calleeDecl; |
| 726 | |
| 727 | public: |
| 728 | AbstractCallee() : calleeDecl(nullptr) {} |
| 729 | AbstractCallee(const clang::FunctionDecl *fd) : calleeDecl(fd) {} |
| 730 | |
| 731 | bool hasFunctionDecl() const { |
| 732 | return llvm::isa_and_nonnull<clang::FunctionDecl>(Val: calleeDecl); |
| 733 | } |
| 734 | |
| 735 | const clang::Decl *getDecl() const { return calleeDecl; } |
| 736 | |
| 737 | unsigned getNumParams() const { |
| 738 | if (const auto *fd = llvm::dyn_cast<clang::FunctionDecl>(Val: calleeDecl)) |
| 739 | return fd->getNumParams(); |
| 740 | return llvm::cast<clang::ObjCMethodDecl>(Val: calleeDecl)->param_size(); |
| 741 | } |
| 742 | |
| 743 | const clang::ParmVarDecl *getParamDecl(unsigned I) const { |
| 744 | if (const auto *fd = llvm::dyn_cast<clang::FunctionDecl>(Val: calleeDecl)) |
| 745 | return fd->getParamDecl(i: I); |
| 746 | return *(llvm::cast<clang::ObjCMethodDecl>(Val: calleeDecl)->param_begin() + |
| 747 | I); |
| 748 | } |
| 749 | }; |
| 750 | |
| 751 | /// True if the current statement has noinline attribute. |
| 752 | bool inNoInlineAttributedStmt = false; |
| 753 | |
| 754 | /// True if the current statement has always_inline attribute. |
| 755 | bool inAlwaysInlineAttributedStmt = false; |
| 756 | |
| 757 | // The CallExpr within the current statement that the musttail attribute |
| 758 | // applies to. nullptr if there is no 'musttail' on the current statement. |
| 759 | const CallExpr *mustTailCall = nullptr; |
| 760 | |
| 761 | struct VlaSizePair { |
| 762 | mlir::Value numElts; |
| 763 | QualType type; |
| 764 | |
| 765 | VlaSizePair(mlir::Value num, QualType ty) : numElts(num), type(ty) {} |
| 766 | }; |
| 767 | |
| 768 | /// Return the number of elements for a single dimension |
| 769 | /// for the given array type. |
| 770 | VlaSizePair getVLAElements1D(const VariableArrayType *vla); |
| 771 | |
| 772 | /// Returns an MLIR::Value+QualType pair that corresponds to the size, |
| 773 | /// in non-variably-sized elements, of a variable length array type, |
| 774 | /// plus that largest non-variably-sized element type. Assumes that |
| 775 | /// the type has already been emitted with emitVariablyModifiedType. |
| 776 | VlaSizePair getVLASize(const VariableArrayType *type); |
| 777 | VlaSizePair getVLASize(QualType type); |
| 778 | |
| 779 | Address getAsNaturalAddressOf(Address addr, QualType pointeeTy); |
| 780 | |
| 781 | mlir::Value getAsNaturalPointerTo(Address addr, QualType pointeeType) { |
| 782 | return getAsNaturalAddressOf(addr, pointeeTy: pointeeType).getBasePointer(); |
| 783 | } |
| 784 | |
| 785 | void finishFunction(SourceLocation endLoc); |
| 786 | |
| 787 | /// Determine whether the given initializer is trivial in the sense |
| 788 | /// that it requires no code to be generated. |
| 789 | bool isTrivialInitializer(const Expr *init); |
| 790 | |
| 791 | /// If the specified expression does not fold to a constant, or if it does but |
| 792 | /// contains a label, return false. If it constant folds return true and set |
| 793 | /// the boolean result in Result. |
| 794 | bool constantFoldsToBool(const clang::Expr *cond, bool &resultBool, |
| 795 | bool allowLabels = false); |
| 796 | bool constantFoldsToSimpleInteger(const clang::Expr *cond, |
| 797 | llvm::APSInt &resultInt, |
| 798 | bool allowLabels = false); |
| 799 | |
| 800 | /// Return true if the statement contains a label in it. If |
| 801 | /// this statement is not executed normally, it not containing a label means |
| 802 | /// that we can just remove the code. |
| 803 | bool containsLabel(const clang::Stmt *s, bool ignoreCaseStmts = false); |
| 804 | |
| 805 | Address emitExtVectorElementLValue(LValue lv, mlir::Location loc); |
| 806 | |
| 807 | class ConstantEmission { |
| 808 | // Cannot use mlir::TypedAttr directly here because of bit availability. |
| 809 | llvm::PointerIntPair<mlir::Attribute, 1, bool> valueAndIsReference; |
| 810 | ConstantEmission(mlir::TypedAttr c, bool isReference) |
| 811 | : valueAndIsReference(c, isReference) {} |
| 812 | |
| 813 | public: |
| 814 | ConstantEmission() {} |
| 815 | static ConstantEmission forReference(mlir::TypedAttr c) { |
| 816 | return ConstantEmission(c, true); |
| 817 | } |
| 818 | static ConstantEmission forValue(mlir::TypedAttr c) { |
| 819 | return ConstantEmission(c, false); |
| 820 | } |
| 821 | |
| 822 | explicit operator bool() const { |
| 823 | return valueAndIsReference.getOpaqueValue() != nullptr; |
| 824 | } |
| 825 | |
| 826 | bool isReference() const { return valueAndIsReference.getInt(); } |
| 827 | LValue getReferenceLValue(CIRGenFunction &cgf, const Expr *refExpr) const { |
| 828 | assert(isReference()); |
| 829 | mlir::Value addr = cgf.getBuilder().getConstant( |
| 830 | loc: cgf.getLoc(srcLoc: refExpr->getSourceRange()), |
| 831 | attr: mlir::cast<mlir::TypedAttr>(Val: valueAndIsReference.getPointer())); |
| 832 | return cgf.makeNaturalAlignAddrLValue(val: addr, ty: refExpr->getType()); |
| 833 | } |
| 834 | |
| 835 | mlir::TypedAttr getValue() const { |
| 836 | assert(!isReference()); |
| 837 | return mlir::cast<mlir::TypedAttr>(Val: valueAndIsReference.getPointer()); |
| 838 | } |
| 839 | }; |
| 840 | |
| 841 | ConstantEmission tryEmitAsConstant(const DeclRefExpr *refExpr); |
| 842 | ConstantEmission tryEmitAsConstant(const MemberExpr *me); |
| 843 | |
| 844 | struct AutoVarEmission { |
| 845 | const clang::VarDecl *variable; |
| 846 | /// The address of the alloca for languages with explicit address space |
| 847 | /// (e.g. OpenCL) or alloca casted to generic pointer for address space |
| 848 | /// agnostic languages (e.g. C++). Invalid if the variable was emitted |
| 849 | /// as a global constant. |
| 850 | Address addr; |
| 851 | |
| 852 | /// True if the variable is of aggregate type and has a constant |
| 853 | /// initializer. |
| 854 | bool isConstantAggregate = false; |
| 855 | |
| 856 | /// True if the variable is a __block variable that is captured by an |
| 857 | /// escaping block. |
| 858 | bool isEscapingByRef = false; |
| 859 | |
| 860 | /// True if the variable was emitted as an offload recipe, and thus doesn't |
| 861 | /// have the same sort of alloca initialization. |
| 862 | bool emittedAsOffload = false; |
| 863 | |
| 864 | /// True if lifetime op should be used. |
| 865 | bool useLifetimeMarkers = false; |
| 866 | |
| 867 | mlir::Value nrvoFlag{}; |
| 868 | |
| 869 | struct Invalid {}; |
| 870 | AutoVarEmission(Invalid) : variable(nullptr), addr(Address::invalid()) {} |
| 871 | |
| 872 | AutoVarEmission(const clang::VarDecl &variable) |
| 873 | : variable(&variable), addr(Address::invalid()) {} |
| 874 | |
| 875 | static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); } |
| 876 | |
| 877 | bool wasEmittedAsGlobal() const { return !addr.isValid(); } |
| 878 | |
| 879 | bool wasEmittedAsOffloadClause() const { return emittedAsOffload; } |
| 880 | |
| 881 | /// Returns the raw, allocated address, which is not necessarily |
| 882 | /// the address of the object itself. It is casted to default |
| 883 | /// address space for address space agnostic languages. |
| 884 | Address getAllocatedAddress() const { return addr; } |
| 885 | |
| 886 | // Changes the stored address for the emission. This function should only |
| 887 | // be used in extreme cases, and isn't required to model normal AST |
| 888 | // initialization/variables. |
| 889 | void setAllocatedAddress(Address a) { addr = a; } |
| 890 | |
| 891 | /// Returns the address of the object within this declaration. |
| 892 | /// Note that this does not chase the forwarding pointer for |
| 893 | /// __block decls. |
| 894 | Address getObjectAddress(CIRGenFunction &cgf) const { |
| 895 | if (!isEscapingByRef) |
| 896 | return addr; |
| 897 | |
| 898 | assert(!cir::MissingFeatures::opAllocaEscapeByReference()); |
| 899 | return Address::invalid(); |
| 900 | } |
| 901 | }; |
| 902 | |
| 903 | /// Perform the usual unary conversions on the specified expression and |
| 904 | /// compare the result against zero, returning an Int1Ty value. |
| 905 | mlir::Value evaluateExprAsBool(const clang::Expr *e); |
| 906 | |
| 907 | cir::GlobalOp addInitializerToStaticVarDecl(const VarDecl &d, |
| 908 | cir::GlobalOp gv, |
| 909 | cir::GetGlobalOp gvAddr); |
| 910 | |
| 911 | /// Enter the cleanups necessary to complete the given phase of destruction |
| 912 | /// for a destructor. The end result should call destructors on members and |
| 913 | /// base classes in reverse order of their construction. |
| 914 | void enterDtorCleanups(const CXXDestructorDecl *dtor, CXXDtorType type); |
| 915 | |
| 916 | /// Return the pointer to pass to the operator delete of the given |
| 917 | /// destructor, converted to the type of its first parameter if needed. |
| 918 | mlir::Value loadThisForDtorDelete(const CXXDestructorDecl *dd); |
| 919 | |
| 920 | /// Determines whether an EH cleanup is required to destroy a type |
| 921 | /// with the given destruction kind. |
| 922 | /// TODO(cir): could be shared with Clang LLVM codegen |
| 923 | bool needsEHCleanup(QualType::DestructionKind kind) { |
| 924 | switch (kind) { |
| 925 | case QualType::DK_none: |
| 926 | return false; |
| 927 | case QualType::DK_cxx_destructor: |
| 928 | case QualType::DK_objc_weak_lifetime: |
| 929 | case QualType::DK_nontrivial_c_struct: |
| 930 | return getLangOpts().Exceptions; |
| 931 | case QualType::DK_objc_strong_lifetime: |
| 932 | return getLangOpts().Exceptions && |
| 933 | cgm.getCodeGenOpts().ObjCAutoRefCountExceptions; |
| 934 | } |
| 935 | llvm_unreachable("bad destruction kind" ); |
| 936 | } |
| 937 | |
| 938 | CleanupKind getCleanupKind(QualType::DestructionKind kind) { |
| 939 | return needsEHCleanup(kind) ? NormalAndEHCleanup : NormalCleanup; |
| 940 | } |
| 941 | |
| 942 | void pushStackRestore(CleanupKind kind, Address spMem); |
| 943 | |
| 944 | /// Set the address of a local variable. |
| 945 | void setAddrOfLocalVar(const clang::VarDecl *vd, Address addr) { |
| 946 | assert(!localDeclMap.count(vd) && "Decl already exists in LocalDeclMap!" ); |
| 947 | localDeclMap.insert(KV: {vd, addr}); |
| 948 | |
| 949 | // Add to the symbol table if not there already. |
| 950 | if (symbolTable.count(Key: vd)) |
| 951 | return; |
| 952 | symbolTable.insert(Key: vd, Val: addr.getPointer()); |
| 953 | } |
| 954 | |
| 955 | // Replaces the address of the local variable, if it exists. Else does the |
| 956 | // same thing as setAddrOfLocalVar. |
| 957 | void replaceAddrOfLocalVar(const clang::VarDecl *vd, Address addr) { |
| 958 | localDeclMap.insert_or_assign(Key: vd, Val&: addr); |
| 959 | } |
| 960 | |
| 961 | // A class to allow reverting changes to a var-decl's registration to the |
| 962 | // localDeclMap. This is used in cases where things are being inserted into |
| 963 | // the variable list but don't follow normal lookup/search rules, like in |
| 964 | // OpenACC recipe generation. |
| 965 | class DeclMapRevertingRAII { |
| 966 | CIRGenFunction &cgf; |
| 967 | const VarDecl *vd; |
| 968 | bool shouldDelete = false; |
| 969 | Address oldAddr = Address::invalid(); |
| 970 | |
| 971 | public: |
| 972 | DeclMapRevertingRAII(CIRGenFunction &cgf, const VarDecl *vd) |
| 973 | : cgf(cgf), vd(vd) { |
| 974 | auto mapItr = cgf.localDeclMap.find(Val: vd); |
| 975 | |
| 976 | if (mapItr != cgf.localDeclMap.end()) |
| 977 | oldAddr = mapItr->second; |
| 978 | else |
| 979 | shouldDelete = true; |
| 980 | } |
| 981 | |
| 982 | ~DeclMapRevertingRAII() { |
| 983 | if (shouldDelete) |
| 984 | cgf.localDeclMap.erase(Val: vd); |
| 985 | else |
| 986 | cgf.localDeclMap.insert_or_assign(Key: vd, Val&: oldAddr); |
| 987 | } |
| 988 | }; |
| 989 | |
| 990 | bool shouldNullCheckClassCastValue(const CastExpr *ce); |
| 991 | |
| 992 | RValue convertTempToRValue(Address addr, clang::QualType type, |
| 993 | clang::SourceLocation loc); |
| 994 | |
| 995 | static bool |
| 996 | isConstructorDelegationValid(const clang::CXXConstructorDecl *ctor); |
| 997 | |
| 998 | struct VPtr { |
| 999 | clang::BaseSubobject base; |
| 1000 | const clang::CXXRecordDecl *nearestVBase; |
| 1001 | clang::CharUnits offsetFromNearestVBase; |
| 1002 | const clang::CXXRecordDecl *vtableClass; |
| 1003 | }; |
| 1004 | |
| 1005 | using VisitedVirtualBasesSetTy = |
| 1006 | llvm::SmallPtrSet<const clang::CXXRecordDecl *, 4>; |
| 1007 | |
| 1008 | using VPtrsVector = llvm::SmallVector<VPtr, 4>; |
| 1009 | VPtrsVector getVTablePointers(const clang::CXXRecordDecl *vtableClass); |
| 1010 | void getVTablePointers(clang::BaseSubobject base, |
| 1011 | const clang::CXXRecordDecl *nearestVBase, |
| 1012 | clang::CharUnits offsetFromNearestVBase, |
| 1013 | bool baseIsNonVirtualPrimaryBase, |
| 1014 | const clang::CXXRecordDecl *vtableClass, |
| 1015 | VisitedVirtualBasesSetTy &vbases, VPtrsVector &vptrs); |
| 1016 | /// Return the Value of the vtable pointer member pointed to by thisAddr. |
| 1017 | mlir::Value getVTablePtr(mlir::Location loc, Address thisAddr, |
| 1018 | const clang::CXXRecordDecl *vtableClass); |
| 1019 | |
| 1020 | /// Returns whether we should perform a type checked load when loading a |
| 1021 | /// virtual function for virtual calls to members of RD. This is generally |
| 1022 | /// true when both vcall CFI and whole-program-vtables are enabled. |
| 1023 | bool shouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *rd); |
| 1024 | |
| 1025 | /// Source location information about the default argument or member |
| 1026 | /// initializer expression we're evaluating, if any. |
| 1027 | clang::CurrentSourceLocExprScope curSourceLocExprScope; |
| 1028 | using SourceLocExprScopeGuard = |
| 1029 | clang::CurrentSourceLocExprScope::SourceLocExprScopeGuard; |
| 1030 | |
| 1031 | /// A scope within which we are constructing the fields of an object which |
| 1032 | /// might use a CXXDefaultInitExpr. This stashes away a 'this' value to use if |
| 1033 | /// we need to evaluate the CXXDefaultInitExpr within the evaluation. |
| 1034 | class FieldConstructionScope { |
| 1035 | public: |
| 1036 | FieldConstructionScope(CIRGenFunction &cgf, Address thisAddr) |
| 1037 | : cgf(cgf), oldCXXDefaultInitExprThis(cgf.cxxDefaultInitExprThis) { |
| 1038 | cgf.cxxDefaultInitExprThis = thisAddr; |
| 1039 | } |
| 1040 | ~FieldConstructionScope() { |
| 1041 | cgf.cxxDefaultInitExprThis = oldCXXDefaultInitExprThis; |
| 1042 | } |
| 1043 | |
| 1044 | private: |
| 1045 | CIRGenFunction &cgf; |
| 1046 | Address oldCXXDefaultInitExprThis; |
| 1047 | }; |
| 1048 | |
| 1049 | /// The scope of a CXXDefaultInitExpr. Within this scope, the value of 'this' |
| 1050 | /// is overridden to be the object under construction. |
| 1051 | class CXXDefaultInitExprScope { |
| 1052 | public: |
| 1053 | CXXDefaultInitExprScope(CIRGenFunction &cgf, const CXXDefaultInitExpr *e) |
| 1054 | : cgf{cgf}, oldCXXThisValue(cgf.cxxThisValue), |
| 1055 | oldCXXThisAlignment(cgf.cxxThisAlignment), |
| 1056 | sourceLocScope(e, cgf.curSourceLocExprScope) { |
| 1057 | cgf.cxxThisValue = cgf.cxxDefaultInitExprThis.getPointer(); |
| 1058 | cgf.cxxThisAlignment = cgf.cxxDefaultInitExprThis.getAlignment(); |
| 1059 | } |
| 1060 | ~CXXDefaultInitExprScope() { |
| 1061 | cgf.cxxThisValue = oldCXXThisValue; |
| 1062 | cgf.cxxThisAlignment = oldCXXThisAlignment; |
| 1063 | } |
| 1064 | |
| 1065 | public: |
| 1066 | CIRGenFunction &cgf; |
| 1067 | mlir::Value oldCXXThisValue; |
| 1068 | clang::CharUnits oldCXXThisAlignment; |
| 1069 | SourceLocExprScopeGuard sourceLocScope; |
| 1070 | }; |
| 1071 | |
| 1072 | struct CXXDefaultArgExprScope : SourceLocExprScopeGuard { |
| 1073 | CXXDefaultArgExprScope(CIRGenFunction &cfg, const CXXDefaultArgExpr *e) |
| 1074 | : SourceLocExprScopeGuard(e, cfg.curSourceLocExprScope) {} |
| 1075 | }; |
| 1076 | |
| 1077 | /// The scope of an ArrayInitLoopExpr. Within this scope, the value of the |
| 1078 | /// current loop index is overridden. In order to encourage re-use of existing |
| 1079 | /// array initialization, this uses a flag to determine if it is a 'no-op' or |
| 1080 | /// not. |
| 1081 | class ArrayInitLoopExprScope { |
| 1082 | public: |
| 1083 | ArrayInitLoopExprScope(CIRGenFunction &cgf, bool setIdx, mlir::Value index) |
| 1084 | : cgf(cgf), |
| 1085 | oldArrayInitIndex(setIdx |
| 1086 | ? std::optional<mlir::Value>(cgf.arrayInitIndex) |
| 1087 | : std::nullopt) { |
| 1088 | if (setIdx) |
| 1089 | cgf.arrayInitIndex = index; |
| 1090 | } |
| 1091 | ~ArrayInitLoopExprScope() { |
| 1092 | if (oldArrayInitIndex.has_value()) |
| 1093 | cgf.arrayInitIndex = *oldArrayInitIndex; |
| 1094 | } |
| 1095 | |
| 1096 | private: |
| 1097 | CIRGenFunction &cgf; |
| 1098 | std::optional<mlir::Value> oldArrayInitIndex; |
| 1099 | }; |
| 1100 | |
| 1101 | /// Get the index of the current ArrayInitLoopExpr, if any. |
| 1102 | mlir::Value getArrayInitIndex() { return arrayInitIndex; } |
| 1103 | |
| 1104 | LValue makeNaturalAlignPointeeAddrLValue(mlir::Value v, clang::QualType t); |
| 1105 | LValue makeNaturalAlignAddrLValue(mlir::Value val, QualType ty); |
| 1106 | |
| 1107 | /// Construct an address with the natural alignment of T. If a pointer to T |
| 1108 | /// is expected to be signed, the pointer passed to this function must have |
| 1109 | /// been signed, and the returned Address will have the pointer authentication |
| 1110 | /// information needed to authenticate the signed pointer. |
| 1111 | Address makeNaturalAddressForPointer(mlir::Value ptr, QualType t, |
| 1112 | CharUnits alignment, |
| 1113 | bool forPointeeType = false, |
| 1114 | LValueBaseInfo *baseInfo = nullptr) { |
| 1115 | if (alignment.isZero()) |
| 1116 | alignment = cgm.getNaturalTypeAlignment(t, baseInfo); |
| 1117 | return Address(ptr, convertTypeForMem(t), alignment); |
| 1118 | } |
| 1119 | |
| 1120 | Address getAddressOfBaseClass( |
| 1121 | Address value, const CXXRecordDecl *derived, |
| 1122 | llvm::iterator_range<CastExpr::path_const_iterator> path, |
| 1123 | bool nullCheckValue, SourceLocation loc); |
| 1124 | |
| 1125 | Address getAddressOfDerivedClass( |
| 1126 | mlir::Location loc, Address baseAddr, const CXXRecordDecl *derived, |
| 1127 | llvm::iterator_range<CastExpr::path_const_iterator> path, |
| 1128 | bool nullCheckValue); |
| 1129 | |
| 1130 | /// Return the VTT parameter that should be passed to a base |
| 1131 | /// constructor/destructor with virtual bases. |
| 1132 | /// FIXME: VTTs are Itanium ABI-specific, so the definition should move |
| 1133 | /// to ItaniumCXXABI.cpp together with all the references to VTT. |
| 1134 | mlir::Value getVTTParameter(GlobalDecl gd, bool forVirtualBase, |
| 1135 | bool delegating); |
| 1136 | |
| 1137 | LValue makeAddrLValue(Address addr, QualType ty, |
| 1138 | AlignmentSource source = AlignmentSource::Type) { |
| 1139 | return makeAddrLValue(addr, ty, baseInfo: LValueBaseInfo(source)); |
| 1140 | } |
| 1141 | |
| 1142 | LValue makeAddrLValue(Address addr, QualType ty, LValueBaseInfo baseInfo) { |
| 1143 | return LValue::makeAddr(address: addr, t: ty, baseInfo); |
| 1144 | } |
| 1145 | |
| 1146 | void initializeVTablePointers(mlir::Location loc, |
| 1147 | const clang::CXXRecordDecl *rd); |
| 1148 | void initializeVTablePointer(mlir::Location loc, const VPtr &vptr); |
| 1149 | |
| 1150 | AggValueSlot::Overlap_t getOverlapForFieldInit(const FieldDecl *fd); |
| 1151 | |
| 1152 | /// Return the address of a local variable. |
| 1153 | Address getAddrOfLocalVar(const clang::VarDecl *vd) { |
| 1154 | auto it = localDeclMap.find(Val: vd); |
| 1155 | assert(it != localDeclMap.end() && |
| 1156 | "Invalid argument to getAddrOfLocalVar(), no decl!" ); |
| 1157 | return it->second; |
| 1158 | } |
| 1159 | |
| 1160 | Address getAddrOfBitFieldStorage(LValue base, const clang::FieldDecl *field, |
| 1161 | mlir::Type fieldType, unsigned index); |
| 1162 | |
| 1163 | /// Given an opaque value expression, return its LValue mapping if it exists, |
| 1164 | /// otherwise create one. |
| 1165 | LValue getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e); |
| 1166 | |
| 1167 | /// Given an opaque value expression, return its RValue mapping if it exists, |
| 1168 | /// otherwise create one. |
| 1169 | RValue getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e); |
| 1170 | |
| 1171 | /// Load the value for 'this'. This function is only valid while generating |
| 1172 | /// code for an C++ member function. |
| 1173 | /// FIXME(cir): this should return a mlir::Value! |
| 1174 | mlir::Value loadCXXThis() { |
| 1175 | assert(cxxThisValue && "no 'this' value for this function" ); |
| 1176 | return cxxThisValue; |
| 1177 | } |
| 1178 | Address loadCXXThisAddress(); |
| 1179 | |
| 1180 | /// Load the VTT parameter to base constructors/destructors have virtual |
| 1181 | /// bases. FIXME: Every place that calls LoadCXXVTT is something that needs to |
| 1182 | /// be abstracted properly. |
| 1183 | mlir::Value loadCXXVTT() { |
| 1184 | assert(cxxStructorImplicitParamValue && "no VTT value for this function" ); |
| 1185 | return cxxStructorImplicitParamValue; |
| 1186 | } |
| 1187 | |
| 1188 | /// Convert the given pointer to a complete class to the given direct base. |
| 1189 | Address getAddressOfDirectBaseInCompleteClass(mlir::Location loc, |
| 1190 | Address value, |
| 1191 | const CXXRecordDecl *derived, |
| 1192 | const CXXRecordDecl *base, |
| 1193 | bool baseIsVirtual); |
| 1194 | |
| 1195 | /// Determine whether a return value slot may overlap some other object. |
| 1196 | AggValueSlot::Overlap_t getOverlapForReturnValue() { |
| 1197 | // FIXME: Assuming no overlap here breaks guaranteed copy elision for base |
| 1198 | // class subobjects. These cases may need to be revisited depending on the |
| 1199 | // resolution of the relevant core issue. |
| 1200 | return AggValueSlot::DoesNotOverlap; |
| 1201 | } |
| 1202 | |
| 1203 | /// Determine whether a base class initialization may overlap some other |
| 1204 | /// object. |
| 1205 | AggValueSlot::Overlap_t getOverlapForBaseInit(const CXXRecordDecl *rd, |
| 1206 | const CXXRecordDecl *baseRD, |
| 1207 | bool isVirtual); |
| 1208 | |
| 1209 | /// Return a CIR constant for an undefined value of \p cirTy. |
| 1210 | mlir::Value getUndefConstant(mlir::Location loc, mlir::Type cirTy); |
| 1211 | |
| 1212 | /// Get an appropriate 'undef' rvalue for the given type. |
| 1213 | RValue getUndefRValue(clang::QualType ty); |
| 1214 | |
| 1215 | cir::FuncOp generateCode(clang::GlobalDecl gd, cir::FuncOp fn, |
| 1216 | cir::FuncType funcType); |
| 1217 | |
| 1218 | clang::QualType buildFunctionArgList(clang::GlobalDecl gd, |
| 1219 | FunctionArgList &args); |
| 1220 | |
| 1221 | /// Emit the function prologue: initialize the return value of an |
| 1222 | /// implicit-return-zero function and declare function arguments in the |
| 1223 | /// symbol table. |
| 1224 | void emitFunctionProlog(const FunctionArgList &args, mlir::Block *entryBB, |
| 1225 | const FunctionDecl *fd, SourceLocation bodyBeginLoc); |
| 1226 | |
| 1227 | /// Emit code for the start of a function. |
| 1228 | /// \param loc The location to be associated with the function. |
| 1229 | /// \param startLoc The location of the function body. |
| 1230 | void startFunction(clang::GlobalDecl gd, clang::QualType returnType, |
| 1231 | cir::FuncOp fn, cir::FuncType funcType, |
| 1232 | FunctionArgList args, clang::SourceLocation loc, |
| 1233 | clang::SourceLocation startLoc); |
| 1234 | |
| 1235 | /// Wrap the function body in a `cir.try` that enforces the exception |
| 1236 | /// specification of \p d: a filter handler for a dynamic specification |
| 1237 | /// (`throw(T...)` or pre-C++17 `throw()`), or a terminate handler for a |
| 1238 | /// specification that permits nothing to escape. |
| 1239 | void emitStartEHSpec(const clang::Decl *d); |
| 1240 | |
| 1241 | /// Close the `cir.try` opened by emitStartEHSpec. |
| 1242 | void emitEndEHSpec(const clang::Decl *d); |
| 1243 | |
| 1244 | /// returns true if aggregate type has a volatile member. |
| 1245 | bool hasVolatileMember(QualType t) { |
| 1246 | if (const auto *rd = t->getAsRecordDecl()) |
| 1247 | return rd->hasVolatileMember(); |
| 1248 | return false; |
| 1249 | } |
| 1250 | |
| 1251 | void addCatchHandlerAttr(const CXXCatchStmt *catchStmt, |
| 1252 | SmallVector<mlir::Attribute> &handlerAttrs); |
| 1253 | |
| 1254 | /// The cleanup depth enclosing all the cleanups associated with the |
| 1255 | /// parameters. |
| 1256 | EHScopeStack::stable_iterator prologueCleanupDepth; |
| 1257 | |
| 1258 | /// The `cir.try` wrapping a function whose exception specification has to be |
| 1259 | /// enforced. Null when the current function needs no such wrapper. |
| 1260 | cir::TryOp ehSpecTryOp; |
| 1261 | |
| 1262 | /// Whether the wrapper opened by emitStartEHSpec is a terminate scope, whose |
| 1263 | /// handler calls std::terminate() for any escaping exception, rather than the |
| 1264 | /// filter of a dynamic exception specification. |
| 1265 | bool inEHSpecTerminateScope() { |
| 1266 | return ehSpecTryOp && |
| 1267 | mlir::isa<cir::CatchAllAttr>(Val: ehSpecTryOp.getHandlerTypes()[0]); |
| 1268 | } |
| 1269 | |
| 1270 | bool isCatchOrCleanupRequired(); |
| 1271 | |
| 1272 | /// Takes the old cleanup stack size and emits the cleanup blocks |
| 1273 | /// that have been added. |
| 1274 | void popCleanupBlocks(EHScopeStack::stable_iterator oldCleanupStackDepth, |
| 1275 | ArrayRef<mlir::Value *> valuesToReload = {}); |
| 1276 | |
| 1277 | /// Pops cleanup blocks until the given savepoint is reached, then adds the |
| 1278 | /// cleanups from the given savepoint in the lifetime-extended cleanups stack. |
| 1279 | void popCleanupBlocks(EHScopeStack::stable_iterator oldCleanupStackDepth, |
| 1280 | size_t oldLifetimeExtendedSize, |
| 1281 | ArrayRef<mlir::Value *> valuesToReload = {}); |
| 1282 | void popCleanupBlock(bool forDeactivation = false); |
| 1283 | |
| 1284 | /// Emit the captured cleanups (those pushed above \p depth while EHScopeStack |
| 1285 | /// was capturing cleanups) at the current insertion point, which must be |
| 1286 | /// inside the cleanup region of the op that owns them, and pop them off the |
| 1287 | /// EH stack. Every cleanup above \p depth must have been captured. |
| 1288 | void emitCapturedCleanups(EHScopeStack::stable_iterator depth, |
| 1289 | mlir::Location loc); |
| 1290 | |
| 1291 | /// Like emitCapturedCleanups, but push the captured cleanups again as |
| 1292 | /// ordinary cleanups, the first captured one outermost, and pop them, so that |
| 1293 | /// each one is emitted as a cir.cleanup.scope around the ones captured after |
| 1294 | /// it. An exception from one of them then still runs the ones captured |
| 1295 | /// before it. None of them may have an active flag. |
| 1296 | void emitCapturedCleanupsAsScopes(EHScopeStack::stable_iterator depth, |
| 1297 | mlir::Location loc); |
| 1298 | |
| 1299 | void terminateStructuredRegionBody(mlir::Region &r, mlir::Location loc); |
| 1300 | |
| 1301 | /// Deactivates the given cleanup block. The block cannot be reactivated. Pops |
| 1302 | /// it if it's the top of the stack. |
| 1303 | /// |
| 1304 | /// \param DominatingIP - An instruction which is known to |
| 1305 | /// dominate the current IP (if set) and which lies along |
| 1306 | /// all paths of execution between the current IP and the |
| 1307 | /// the point at which the cleanup comes into scope. |
| 1308 | void deactivateCleanupBlock(EHScopeStack::stable_iterator cleanup, |
| 1309 | mlir::Operation *dominatingIP); |
| 1310 | |
| 1311 | /// Create an active flag variable for use with conditional cleanups. The |
| 1312 | /// flag is initialized to false before the outermost conditional and set to |
| 1313 | /// true at the current insertion point (inside the conditional branch). |
| 1314 | Address createCleanupActiveFlag(); |
| 1315 | |
| 1316 | /// Set up the last cleanup that was pushed as a conditional |
| 1317 | /// full-expression cleanup. |
| 1318 | void initFullExprCleanup(); |
| 1319 | void initFullExprCleanupWithFlag(Address activeFlag); |
| 1320 | |
| 1321 | /// Promote a single pending cleanup entry onto the EH scope stack. If the |
| 1322 | /// entry has a valid activeFlag, the cleanup is configured as conditional. |
| 1323 | /// Defined in CIRGenDecl.cpp where the concrete cleanup types are visible. |
| 1324 | void pushPendingCleanupToEHStack(const PendingCleanupEntry &entry); |
| 1325 | |
| 1326 | /// Push a cleanup to be run at the end of the current full-expression. Safe |
| 1327 | /// against the possibility that we're currently inside a |
| 1328 | /// conditionally-evaluated expression. |
| 1329 | template <class T, class... As> |
| 1330 | void pushFullExprCleanup(CleanupKind kind, As... a) { |
| 1331 | // Outside a conditional the EH stack opens a scope here. |
| 1332 | if (!isInConditionalBranch()) |
| 1333 | return ehStack.pushCleanup<T>(kind, a...); |
| 1334 | |
| 1335 | // Inside a conditional, record the cleanup for the scope that |
| 1336 | // ConditionalEvaluation opened where the conditional began, guarded by a |
| 1337 | // flag set only where the object is constructed. |
| 1338 | Address activeFlag = createCleanupActiveFlag(); |
| 1339 | deferredConditionalCleanupStack.push_back( |
| 1340 | Elt: PendingCleanupEntry{kind, a..., activeFlag}); |
| 1341 | } |
| 1342 | |
| 1343 | /// Push a cleanup and record it for deferred deactivation. The cleanup will |
| 1344 | /// be deactivated when the enclosing CleanupDeactivationScope exits. |
| 1345 | template <class T, class... As> |
| 1346 | void pushCleanupAndDeferDeactivation(CleanupKind kind, As... a) { |
| 1347 | mlir::Location loc = builder.getUnknownLoc(); |
| 1348 | mlir::Operation *dominatingIP = builder.getBool(state: false, loc).getOperation(); |
| 1349 | ehStack.pushCleanup<T>(kind, a...); |
| 1350 | deferredDeactivationCleanupStack.push_back( |
| 1351 | Elt: {.cleanup: ehStack.stable_begin(), .dominatingIP: dominatingIP}); |
| 1352 | } |
| 1353 | |
| 1354 | void pushDestroyAndDeferDeactivation(QualType::DestructionKind dtorKind, |
| 1355 | Address addr, QualType type); |
| 1356 | void pushDestroyAndDeferDeactivation(CleanupKind cleanupKind, Address addr, |
| 1357 | QualType type, Destroyer *destroyer, |
| 1358 | bool useEHCleanupForArray); |
| 1359 | |
| 1360 | /// Queue a cleanup to be pushed after finishing the current full-expression. |
| 1361 | /// When the enclosing RunCleanupsScope exits, popCleanupBlocks promotes these |
| 1362 | /// entries onto the EH scope stack for the enclosing scope. |
| 1363 | void pushCleanupAfterFullExpr(CleanupKind kind, Address addr, QualType type, |
| 1364 | Destroyer *destroyer) { |
| 1365 | lifetimeExtendedCleanupStack.push_back(Elt: {.kind: kind, .addr: addr, .type: type, .destroyer: destroyer}); |
| 1366 | } |
| 1367 | |
| 1368 | /// Enters a new scope for capturing cleanups, all of which |
| 1369 | /// will be executed once the scope is exited. |
| 1370 | class RunCleanupsScope { |
| 1371 | EHScopeStack::stable_iterator cleanupStackDepth, oldCleanupStackDepth; |
| 1372 | size_t lifetimeExtendedCleanupStackSize; |
| 1373 | CleanupDeactivationScope deactivateCleanups; |
| 1374 | bool deferLifetimeExtended = false; |
| 1375 | |
| 1376 | size_t oldLifetimeExtendedSize() const { |
| 1377 | return deferLifetimeExtended ? cgf.lifetimeExtendedCleanupStack.size() |
| 1378 | : lifetimeExtendedCleanupStackSize; |
| 1379 | } |
| 1380 | |
| 1381 | protected: |
| 1382 | bool performCleanup; |
| 1383 | bool oldDidCallStackSave; |
| 1384 | |
| 1385 | private: |
| 1386 | RunCleanupsScope(const RunCleanupsScope &) = delete; |
| 1387 | void operator=(const RunCleanupsScope &) = delete; |
| 1388 | |
| 1389 | protected: |
| 1390 | CIRGenFunction &cgf; |
| 1391 | |
| 1392 | public: |
| 1393 | /// Enter a new cleanup scope. |
| 1394 | explicit RunCleanupsScope(CIRGenFunction &cgf) |
| 1395 | : deactivateCleanups(cgf), performCleanup(true), cgf(cgf) { |
| 1396 | cleanupStackDepth = cgf.ehStack.stable_begin(); |
| 1397 | lifetimeExtendedCleanupStackSize = |
| 1398 | cgf.lifetimeExtendedCleanupStack.size(); |
| 1399 | oldDidCallStackSave = cgf.didCallStackSave; |
| 1400 | cgf.didCallStackSave = false; |
| 1401 | oldCleanupStackDepth = cgf.currentCleanupStackDepth; |
| 1402 | cgf.currentCleanupStackDepth = cleanupStackDepth; |
| 1403 | } |
| 1404 | |
| 1405 | /// Exit this cleanup scope, emitting any accumulated cleanups. |
| 1406 | ~RunCleanupsScope() { |
| 1407 | if (performCleanup) |
| 1408 | forceCleanup(); |
| 1409 | } |
| 1410 | |
| 1411 | /// Force the emission of cleanups now, instead of waiting |
| 1412 | /// until this object is destroyed. |
| 1413 | void forceCleanup(ArrayRef<mlir::Value *> valuesToReload = {}) { |
| 1414 | assert(performCleanup && "Already forced cleanup" ); |
| 1415 | cgf.didCallStackSave = oldDidCallStackSave; |
| 1416 | |
| 1417 | // forceDeactivate() can pop cleanup scopes that were pushed with |
| 1418 | // deferred deactivation, which moves the insertion point out of the |
| 1419 | // cleanup body region. Any caller value defined inside such a body |
| 1420 | // would no longer dominate uses past the scope. The downstream |
| 1421 | // popCleanupBlocks() handles the spill for any cleanups it pops |
| 1422 | // itself, but it cannot help with cleanups that forceDeactivate has |
| 1423 | // already popped. Spill those values here, while the insertion point |
| 1424 | // is still inside the body, so we can reload them after all popping |
| 1425 | // is done. We only spill values whose defining op lives inside a |
| 1426 | // cir.cleanup.scope, since values defined outside any cleanup scope |
| 1427 | // (e.g. allocas in the entry block) already dominate the post-scope |
| 1428 | // insertion point. |
| 1429 | const bool hasPendingDeactivations = |
| 1430 | cgf.deferredDeactivationCleanupStack.size() > |
| 1431 | deactivateCleanups.oldDeactivateCleanupStackSize; |
| 1432 | |
| 1433 | llvm::SmallVector<Address> tempAllocas; |
| 1434 | bool didSpillAny = false; |
| 1435 | if (hasPendingDeactivations) { |
| 1436 | tempAllocas.reserve(N: valuesToReload.size()); |
| 1437 | for (mlir::Value *valPtr : valuesToReload) { |
| 1438 | mlir::Value val = *valPtr; |
| 1439 | if (!val || !val.getDefiningOp() || |
| 1440 | !val.getDefiningOp()->getParentOfType<cir::CleanupScopeOp>()) { |
| 1441 | tempAllocas.push_back(Elt: Address::invalid()); |
| 1442 | continue; |
| 1443 | } |
| 1444 | Address temp = cgf.createDefaultAlignTempAlloca( |
| 1445 | ty: val.getType(), loc: val.getLoc(), name: "tmp.exprcleanup" ); |
| 1446 | tempAllocas.push_back(Elt: temp); |
| 1447 | cgf.builder.createStore(loc: val.getLoc(), val, dst: temp); |
| 1448 | didSpillAny = true; |
| 1449 | } |
| 1450 | } |
| 1451 | |
| 1452 | deactivateCleanups.forceDeactivate(); |
| 1453 | // If we already spilled some of the caller's values, don't ask |
| 1454 | // popCleanupBlocks to spill them again. Values we did not pre-spill |
| 1455 | // are not inside any cir.cleanup.scope, so they cannot be invalidated |
| 1456 | // by either forceDeactivate's or popCleanupBlocks's pops (both only |
| 1457 | // pop cir.cleanup.scope ops); they already dominate the post-scope |
| 1458 | // insertion point on their own. |
| 1459 | if (didSpillAny) { |
| 1460 | cgf.popCleanupBlocks(oldCleanupStackDepth: cleanupStackDepth, oldLifetimeExtendedSize: oldLifetimeExtendedSize()); |
| 1461 | |
| 1462 | // Reload the spilled values now that all cleanup popping (and |
| 1463 | // promotion of any lifetime-extended cleanups onto the EH stack) is |
| 1464 | // done. |
| 1465 | for (auto [addr, valPtr] : llvm::zip(t&: tempAllocas, u&: valuesToReload)) { |
| 1466 | if (!addr.isValid()) |
| 1467 | continue; |
| 1468 | *valPtr = cgf.builder.createLoad(loc: valPtr->getLoc(), addr); |
| 1469 | } |
| 1470 | } else { |
| 1471 | cgf.popCleanupBlocks(oldCleanupStackDepth: cleanupStackDepth, oldLifetimeExtendedSize: oldLifetimeExtendedSize(), |
| 1472 | valuesToReload); |
| 1473 | } |
| 1474 | |
| 1475 | performCleanup = false; |
| 1476 | cgf.currentCleanupStackDepth = oldCleanupStackDepth; |
| 1477 | } |
| 1478 | |
| 1479 | /// Force the emission of EH cleanups now, but defer promoting any |
| 1480 | /// lifetime-extended cleanup entries onto the EH scope stack. The caller |
| 1481 | /// must subsequently call forceLifetimeExtendedCleanups() to finalize the |
| 1482 | /// scope. |
| 1483 | void forceCleanupExceptLifetimeExtended() { |
| 1484 | assert(performCleanup && "Already forced cleanup" ); |
| 1485 | cgf.didCallStackSave = oldDidCallStackSave; |
| 1486 | deactivateCleanups.forceDeactivate(); |
| 1487 | cgf.popCleanupBlocks(oldCleanupStackDepth: cleanupStackDepth); |
| 1488 | } |
| 1489 | |
| 1490 | /// Promote any pending lifetime-extended cleanup entries onto the EH scope |
| 1491 | /// stack at the current insertion point and finalize this scope. This must |
| 1492 | /// be paired with a prior call to forceCleanupExceptLifetimeExtended(). |
| 1493 | void forceLifetimeExtendedCleanups() { |
| 1494 | assert(performCleanup && "Already forced cleanup" ); |
| 1495 | assert(deactivateCleanups.deactivated && |
| 1496 | "forceCleanupExceptLifetimeExtended() must be called first" ); |
| 1497 | cgf.popCleanupBlocks(oldCleanupStackDepth: cleanupStackDepth, oldLifetimeExtendedSize: oldLifetimeExtendedSize()); |
| 1498 | performCleanup = false; |
| 1499 | cgf.currentCleanupStackDepth = oldCleanupStackDepth; |
| 1500 | } |
| 1501 | |
| 1502 | /// Whether there are any pending cleanups that have been pushed since |
| 1503 | /// this scope was entered. |
| 1504 | bool hasPendingCleanups() const { |
| 1505 | return cgf.ehStack.stable_begin() != cleanupStackDepth; |
| 1506 | } |
| 1507 | |
| 1508 | /// Leave the cleanups of lifetime-extended temporaries created in this |
| 1509 | /// scope to the enclosing scope instead of pushing them onto the EH stack |
| 1510 | /// when this scope ends. This is for a scope that ends in the body of a |
| 1511 | /// loop, such as the construction of an array element: the cleanup scope |
| 1512 | /// of such a temporary ends with that of the variable that extends its |
| 1513 | /// lifetime, after the loop. |
| 1514 | void deferLifetimeExtendedCleanups() { deferLifetimeExtended = true; } |
| 1515 | }; |
| 1516 | |
| 1517 | // Cleanup stack depth of the RunCleanupsScope that was pushed most recently. |
| 1518 | EHScopeStack::stable_iterator currentCleanupStackDepth = ehStack.stable_end(); |
| 1519 | |
| 1520 | class ConditionalEvaluation; |
| 1521 | |
| 1522 | class FullExprCleanupScope { |
| 1523 | CIRGenFunction &cgf; |
| 1524 | RunCleanupsScope cleanups; |
| 1525 | cir::CleanupScopeOp scope; |
| 1526 | cir::CleanupScopeOp oldFullExprCleanupScope; |
| 1527 | ConditionalEvaluation *oldOutermostConditional; |
| 1528 | ConditionalEvaluation *oldHiddenOutermostConditional; |
| 1529 | size_t deferredCleanupStackSize; |
| 1530 | size_t conditionalScopeDepth; |
| 1531 | bool exited = false; |
| 1532 | |
| 1533 | public: |
| 1534 | /// \p s is the AST root for one cleanup boundary: a full-expression, an |
| 1535 | /// asm statement, or the construction of one array element. |
| 1536 | FullExprCleanupScope(CIRGenFunction &cgf, const Stmt *s); |
| 1537 | |
| 1538 | void exit(ArrayRef<mlir::Value *> valuesToReload = {}); |
| 1539 | |
| 1540 | void deferLifetimeExtendedCleanups() { |
| 1541 | cleanups.deferLifetimeExtendedCleanups(); |
| 1542 | } |
| 1543 | |
| 1544 | ~FullExprCleanupScope() { |
| 1545 | if (!exited) |
| 1546 | exit(); |
| 1547 | } |
| 1548 | |
| 1549 | private: |
| 1550 | FullExprCleanupScope(const FullExprCleanupScope &) = delete; |
| 1551 | void operator=(const FullExprCleanupScope &) = delete; |
| 1552 | }; |
| 1553 | |
| 1554 | /// Whether the resume expression of \p s contains a construct that defers |
| 1555 | /// cleanups to the end of the enclosing full-expression: a temporary with a |
| 1556 | /// destructor, other than the result of await_resume(), or a conditional. |
| 1557 | static bool resumeExprDefersCleanups(const CoroutineSuspendExpr &s); |
| 1558 | |
| 1559 | /// Captures cleanups for variables whose lifetime ends in a region owned by |
| 1560 | /// an enclosing op rather than at the end of a cir.cleanup.scope, so that |
| 1561 | /// they can be emitted into that region later. Examples for uses are: |
| 1562 | /// * A loop condition variable's cleanups in the loop op's per-iteration |
| 1563 | /// cleanup region. |
| 1564 | /// * Coroutine parameters and promises, which are cleaned up in a separate |
| 1565 | /// destroy region |
| 1566 | class CapturedCleanups { |
| 1567 | CIRGenFunction &cgf; |
| 1568 | EHScopeStack::stable_iterator depth; |
| 1569 | bool active; |
| 1570 | |
| 1571 | public: |
| 1572 | CapturedCleanups(CIRGenFunction &cgf, bool active) |
| 1573 | : cgf(cgf), depth(cgf.ehStack.stable_begin()), active(active) {} |
| 1574 | |
| 1575 | /// An RAII class that suppresses cir.cleanup.scope creation for cleanups |
| 1576 | /// pushed onto the EH stack while such a variable is being emitted and |
| 1577 | /// instead captures these cleanups so that they can be emitted into the |
| 1578 | /// owning op's cleanup region later. |
| 1579 | class CaptureScope { |
| 1580 | EHScopeStack &ehStack; |
| 1581 | |
| 1582 | public: |
| 1583 | explicit CaptureScope(CapturedCleanups &scope) |
| 1584 | : ehStack(scope.cgf.ehStack) { |
| 1585 | // Capture scopes deliberately wrap individual cleanup-producing |
| 1586 | // operations, so they must never nest. |
| 1587 | assert(!ehStack.isCapturingCleanups() && |
| 1588 | "cleanup capturing should not nest" ); |
| 1589 | if (scope.active) |
| 1590 | ehStack.setCapturingCleanups(true); |
| 1591 | } |
| 1592 | ~CaptureScope() { ehStack.setCapturingCleanups(false); } |
| 1593 | |
| 1594 | CaptureScope(const CaptureScope &) = delete; |
| 1595 | void operator=(const CaptureScope &) = delete; |
| 1596 | }; |
| 1597 | |
| 1598 | /// Emit the captured cleanups into the current insertion point, which must |
| 1599 | /// be inside the owning op's cleanup region. |
| 1600 | void emitIntoCleanupRegion(mlir::Location loc) { |
| 1601 | if (active) |
| 1602 | cgf.emitCapturedCleanups(depth, loc); |
| 1603 | } |
| 1604 | |
| 1605 | /// Emit the captured cleanups at the current insertion point as nested |
| 1606 | /// cir.cleanup.scope operations, so that an exception from one of them |
| 1607 | /// still runs the ones captured before it. |
| 1608 | void emitAsNestedScopes(mlir::Location loc) { |
| 1609 | if (active) |
| 1610 | cgf.emitCapturedCleanupsAsScopes(depth, loc); |
| 1611 | } |
| 1612 | |
| 1613 | private: |
| 1614 | CapturedCleanups(const CapturedCleanups &) = delete; |
| 1615 | void operator=(const CapturedCleanups &) = delete; |
| 1616 | }; |
| 1617 | |
| 1618 | public: |
| 1619 | /// Represents a scope, including function bodies, compound statements, and |
| 1620 | /// the substatements of if/while/do/for/switch/try statements. This class |
| 1621 | /// handles any automatic cleanup, along with the return value. |
| 1622 | struct LexicalScope : public RunCleanupsScope { |
| 1623 | private: |
| 1624 | // Points to the scope entry block. This is useful, for instance, for |
| 1625 | // helping to insert allocas before finalizing any recursive CodeGen from |
| 1626 | // switches. |
| 1627 | mlir::Block *entryBlock; |
| 1628 | |
| 1629 | LexicalScope *parentScope = nullptr; |
| 1630 | |
| 1631 | // Holds the actual value for ScopeKind::Try |
| 1632 | cir::TryOp tryOp = nullptr; |
| 1633 | |
| 1634 | // On a coroutine body, the OnFallthrough sub stmt holds the handler |
| 1635 | // (CoreturnStmt) for control flow falling off the body. Keep track |
| 1636 | // of a co_return emitted in this scope and not followed by a label, and |
| 1637 | // allow OnFallthrough to be skipped. |
| 1638 | bool hasCoreturnSinceLastLabel = false; |
| 1639 | |
| 1640 | // Only Regular is used at the moment. Support for other kinds will be |
| 1641 | // added as the relevant statements/expressions are upstreamed. |
| 1642 | enum Kind { |
| 1643 | Regular, // cir.if, cir.scope, if_regions |
| 1644 | Ternary, // cir.ternary |
| 1645 | Switch, // cir.switch |
| 1646 | Try, // cir.try |
| 1647 | GlobalInit // cir.global initialization code |
| 1648 | }; |
| 1649 | Kind scopeKind = Kind::Regular; |
| 1650 | |
| 1651 | // The scope return value. |
| 1652 | mlir::Value retVal = nullptr; |
| 1653 | |
| 1654 | mlir::Location beginLoc; |
| 1655 | mlir::Location endLoc; |
| 1656 | |
| 1657 | public: |
| 1658 | unsigned depth = 0; |
| 1659 | |
| 1660 | LexicalScope(CIRGenFunction &cgf, mlir::Location loc, mlir::Block *eb) |
| 1661 | : RunCleanupsScope(cgf), entryBlock(eb), parentScope(cgf.curLexScope), |
| 1662 | beginLoc(loc), endLoc(loc) { |
| 1663 | |
| 1664 | assert(entryBlock && "LexicalScope requires an entry block" ); |
| 1665 | cgf.curLexScope = this; |
| 1666 | if (parentScope) |
| 1667 | ++depth; |
| 1668 | |
| 1669 | if (const auto fusedLoc = mlir::dyn_cast<mlir::FusedLoc>(Val&: loc)) { |
| 1670 | assert(fusedLoc.getLocations().size() == 2 && "too many locations" ); |
| 1671 | beginLoc = fusedLoc.getLocations()[0]; |
| 1672 | endLoc = fusedLoc.getLocations()[1]; |
| 1673 | } |
| 1674 | } |
| 1675 | |
| 1676 | void setRetVal(mlir::Value v) { retVal = v; } |
| 1677 | |
| 1678 | void cleanup(); |
| 1679 | void restore() { cgf.curLexScope = parentScope; } |
| 1680 | |
| 1681 | ~LexicalScope() { |
| 1682 | assert(!cir::MissingFeatures::generateDebugInfo()); |
| 1683 | cleanup(); |
| 1684 | restore(); |
| 1685 | } |
| 1686 | |
| 1687 | // --- |
| 1688 | // Coroutine tracking |
| 1689 | // --- |
| 1690 | bool hasCoreturn() const { return hasCoreturnSinceLastLabel; } |
| 1691 | // Record an emitted co_return for fallthrough tracking. |
| 1692 | // A later label clears this state. |
| 1693 | void markCoreturnEmitted() { hasCoreturnSinceLastLabel = true; } |
| 1694 | // A goto can reach a label after a co_return, and the code after it, so |
| 1695 | // the end of this scope and of the enclosing ones may be reachable again. |
| 1696 | void clearCoreturnInEnclosingScopes() { |
| 1697 | for (LexicalScope *s = this; s; s = s->parentScope) |
| 1698 | s->hasCoreturnSinceLastLabel = false; |
| 1699 | } |
| 1700 | |
| 1701 | // --- |
| 1702 | // Kind |
| 1703 | // --- |
| 1704 | bool isGlobalInit() { return scopeKind == Kind::GlobalInit; } |
| 1705 | bool isRegular() { return scopeKind == Kind::Regular; } |
| 1706 | bool isSwitch() { return scopeKind == Kind::Switch; } |
| 1707 | bool isTernary() { return scopeKind == Kind::Ternary; } |
| 1708 | bool isTry() { return scopeKind == Kind::Try; } |
| 1709 | cir::TryOp getClosestTryParent(); |
| 1710 | void setAsGlobalInit() { scopeKind = Kind::GlobalInit; } |
| 1711 | void setAsSwitch() { scopeKind = Kind::Switch; } |
| 1712 | void setAsTernary() { scopeKind = Kind::Ternary; } |
| 1713 | void setAsTry(cir::TryOp op) { |
| 1714 | scopeKind = Kind::Try; |
| 1715 | tryOp = op; |
| 1716 | } |
| 1717 | |
| 1718 | cir::TryOp getTry() { |
| 1719 | assert(isTry()); |
| 1720 | return tryOp; |
| 1721 | } |
| 1722 | |
| 1723 | // --- |
| 1724 | // Return handling. |
| 1725 | // --- |
| 1726 | |
| 1727 | private: |
| 1728 | // On switches we need one return block per region, since cases don't |
| 1729 | // have their own scopes but are distinct regions nonetheless. |
| 1730 | |
| 1731 | // TODO: This implementation should change once we have support for early |
| 1732 | // exits in MLIR structured control flow (llvm-project#161575) |
| 1733 | llvm::SmallVector<mlir::Block *> retBlocks; |
| 1734 | llvm::DenseMap<mlir::Block *, mlir::Location> retLocs; |
| 1735 | llvm::DenseMap<cir::CaseOp, unsigned> retBlockInCaseIndex; |
| 1736 | std::optional<unsigned> normalRetBlockIndex; |
| 1737 | |
| 1738 | // There's usually only one ret block per scope, but this needs to be |
| 1739 | // get or create because of potential unreachable return statements, note |
| 1740 | // that for those, all source location maps to the first one found. |
| 1741 | mlir::Block *createRetBlock(CIRGenFunction &cgf, mlir::Location loc) { |
| 1742 | assert((isa_and_nonnull<cir::CaseOp>( |
| 1743 | cgf.builder.getBlock()->getParentOp()) || |
| 1744 | retBlocks.size() == 0) && |
| 1745 | "only switches can hold more than one ret block" ); |
| 1746 | |
| 1747 | // Create the return block but don't hook it up just yet. |
| 1748 | mlir::OpBuilder::InsertionGuard guard(cgf.builder); |
| 1749 | auto *b = cgf.builder.createBlock(parent: cgf.builder.getBlock()->getParent()); |
| 1750 | retBlocks.push_back(Elt: b); |
| 1751 | updateRetLoc(b, loc); |
| 1752 | return b; |
| 1753 | } |
| 1754 | |
| 1755 | cir::ReturnOp emitReturn(mlir::Location loc); |
| 1756 | void emitImplicitReturn(); |
| 1757 | |
| 1758 | public: |
| 1759 | llvm::ArrayRef<mlir::Block *> getRetBlocks() { return retBlocks; } |
| 1760 | mlir::Location getRetLoc(mlir::Block *b) { return retLocs.at(Val: b); } |
| 1761 | void updateRetLoc(mlir::Block *b, mlir::Location loc) { |
| 1762 | retLocs.insert_or_assign(Key: b, Val&: loc); |
| 1763 | } |
| 1764 | |
| 1765 | mlir::Block *getOrCreateRetBlock(CIRGenFunction &cgf, mlir::Location loc) { |
| 1766 | // Check if we're inside a case region |
| 1767 | if (auto caseOp = mlir::dyn_cast_if_present<cir::CaseOp>( |
| 1768 | Val: cgf.builder.getBlock()->getParentOp())) { |
| 1769 | auto iter = retBlockInCaseIndex.find(Val: caseOp); |
| 1770 | if (iter != retBlockInCaseIndex.end()) { |
| 1771 | // Reuse existing return block |
| 1772 | mlir::Block *ret = retBlocks[iter->second]; |
| 1773 | updateRetLoc(b: ret, loc); |
| 1774 | return ret; |
| 1775 | } |
| 1776 | // Create new return block |
| 1777 | mlir::Block *ret = createRetBlock(cgf, loc); |
| 1778 | retBlockInCaseIndex[caseOp] = retBlocks.size() - 1; |
| 1779 | return ret; |
| 1780 | } |
| 1781 | |
| 1782 | if (normalRetBlockIndex) { |
| 1783 | mlir::Block *ret = retBlocks[*normalRetBlockIndex]; |
| 1784 | updateRetLoc(b: ret, loc); |
| 1785 | return ret; |
| 1786 | } |
| 1787 | |
| 1788 | mlir::Block *ret = createRetBlock(cgf, loc); |
| 1789 | normalRetBlockIndex = retBlocks.size() - 1; |
| 1790 | return ret; |
| 1791 | } |
| 1792 | |
| 1793 | mlir::Block *getEntryBlock() { return entryBlock; } |
| 1794 | }; |
| 1795 | |
| 1796 | LexicalScope *curLexScope = nullptr; |
| 1797 | |
| 1798 | static Destroyer destroyCXXObject; |
| 1799 | |
| 1800 | void pushEHDestroyIfNeeded(QualType::DestructionKind dtorKind, Address addr, |
| 1801 | QualType type); |
| 1802 | |
| 1803 | void pushDestroy(QualType::DestructionKind dtorKind, Address addr, |
| 1804 | QualType type); |
| 1805 | |
| 1806 | void pushDestroy(CleanupKind kind, Address addr, QualType type, |
| 1807 | Destroyer *destroyer); |
| 1808 | |
| 1809 | void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr, |
| 1810 | QualType type, Destroyer *destroyer, |
| 1811 | bool useEHCleanupForArray, |
| 1812 | const clang::ValueDecl *extendingDecl); |
| 1813 | |
| 1814 | Destroyer *getDestroyer(clang::QualType::DestructionKind kind); |
| 1815 | |
| 1816 | void pushIrregularPartialArrayCleanup(mlir::Value arrayBegin, |
| 1817 | Address arrayEndPointer, |
| 1818 | QualType elementType, |
| 1819 | CharUnits elementAlign, |
| 1820 | Destroyer *destroyer); |
| 1821 | |
| 1822 | /// Start generating a thunk function. |
| 1823 | void startThunk(cir::FuncOp fn, GlobalDecl gd, |
| 1824 | const CIRGenFunctionInfo &fnInfo, bool isUnprototyped); |
| 1825 | |
| 1826 | /// Finish generating a thunk function. |
| 1827 | void finishThunk(); |
| 1828 | |
| 1829 | /// Generate code for a thunk function. |
| 1830 | void generateThunk(cir::FuncOp fn, SourceRange fnLoc, |
| 1831 | const CIRGenFunctionInfo &fnInfo, GlobalDecl gd, |
| 1832 | const ThunkInfo &thunk, bool isUnprototyped); |
| 1833 | |
| 1834 | /// ---------------------- |
| 1835 | /// CIR emit functions |
| 1836 | /// ---------------------- |
| 1837 | public: |
| 1838 | bool getAArch64SVEProcessedOperands(unsigned builtinID, const CallExpr *expr, |
| 1839 | SmallVectorImpl<mlir::Value> &ops, |
| 1840 | clang::SVETypeFlags typeFlags); |
| 1841 | mlir::Value emitSVEPredicateCast(mlir::Value pred, unsigned minNumElts, |
| 1842 | mlir::Location loc); |
| 1843 | std::optional<mlir::Value> |
| 1844 | emitAArch64BuiltinExpr(unsigned builtinID, const CallExpr *expr, |
| 1845 | ReturnValueSlot returnValue, |
| 1846 | llvm::Triple::ArchType arch); |
| 1847 | std::optional<mlir::Value> emitAArch64SMEBuiltinExpr(unsigned builtinID, |
| 1848 | const CallExpr *expr); |
| 1849 | std::optional<mlir::Value> emitAArch64SVEBuiltinExpr(unsigned builtinID, |
| 1850 | const CallExpr *expr); |
| 1851 | cir::VectorType getSVEType(const SVETypeFlags &typeFlags); |
| 1852 | |
| 1853 | mlir::Value emitAlignmentAssumption(mlir::Value ptrValue, QualType ty, |
| 1854 | SourceLocation loc, |
| 1855 | SourceLocation assumptionLoc, |
| 1856 | int64_t alignment, |
| 1857 | mlir::Value offsetValue = nullptr); |
| 1858 | |
| 1859 | mlir::Value emitAlignmentAssumption(mlir::Value ptrValue, const Expr *expr, |
| 1860 | SourceLocation assumptionLoc, |
| 1861 | int64_t alignment, |
| 1862 | mlir::Value offsetValue = nullptr); |
| 1863 | |
| 1864 | bool emitLifetimeStartOp(mlir::Location loc, mlir::Value addr); |
| 1865 | void emitLifetimeEndOp(mlir::Location loc, mlir::Value addr); |
| 1866 | |
| 1867 | private: |
| 1868 | void emitAndUpdateRetAlloca(clang::QualType type, mlir::Location loc, |
| 1869 | clang::CharUnits alignment); |
| 1870 | |
| 1871 | CIRGenCallee emitDirectCallee(const GlobalDecl &gd); |
| 1872 | |
| 1873 | public: |
| 1874 | Address emitAddrOfFieldStorage(Address base, const FieldDecl *field, |
| 1875 | llvm::StringRef fieldName, |
| 1876 | unsigned fieldIndex); |
| 1877 | |
| 1878 | mlir::Value emitAlloca(llvm::StringRef name, mlir::Type ty, |
| 1879 | mlir::Location loc, clang::CharUnits alignment, |
| 1880 | bool insertIntoFnEntryBlock, |
| 1881 | mlir::Value arraySize = nullptr); |
| 1882 | mlir::Value emitAlloca(llvm::StringRef name, mlir::Type ty, |
| 1883 | mlir::Location loc, clang::CharUnits alignment, |
| 1884 | mlir::OpBuilder::InsertPoint ip, |
| 1885 | mlir::Value arraySize = nullptr); |
| 1886 | |
| 1887 | void emitAggregateStore(mlir::Value value, Address dest); |
| 1888 | |
| 1889 | void emitAggExpr(const clang::Expr *e, AggValueSlot slot); |
| 1890 | |
| 1891 | enum ExprValueKind { EVK_RValue, EVK_NonRValue }; |
| 1892 | |
| 1893 | LValue emitAggExprToLValue(const Expr *e); |
| 1894 | |
| 1895 | /// Emit an aggregate copy. |
| 1896 | /// |
| 1897 | /// \param isVolatile \c true iff either the source or the destination is |
| 1898 | /// volatile. |
| 1899 | /// \param MayOverlap Whether the tail padding of the destination might be |
| 1900 | /// occupied by some other object. More efficient code can often be |
| 1901 | /// generated if not. |
| 1902 | void emitAggregateCopy(LValue dest, LValue src, QualType eltTy, |
| 1903 | AggValueSlot::Overlap_t mayOverlap, |
| 1904 | bool isVolatile = false); |
| 1905 | |
| 1906 | /// Emit code to compute the specified expression which can have any type. The |
| 1907 | /// result is returned as an RValue struct. If this is an aggregate |
| 1908 | /// expression, the aggloc/agglocvolatile arguments indicate where the result |
| 1909 | /// should be returned. |
| 1910 | RValue emitAnyExpr(const clang::Expr *e, |
| 1911 | AggValueSlot aggSlot = AggValueSlot::ignored(), |
| 1912 | bool ignoreResult = false); |
| 1913 | |
| 1914 | /// Emits the code necessary to evaluate an arbitrary expression into the |
| 1915 | /// given memory location. |
| 1916 | void emitAnyExprToMem(const Expr *e, Address location, Qualifiers quals, |
| 1917 | bool isInitializer); |
| 1918 | |
| 1919 | /// Similarly to emitAnyExpr(), however, the result will always be accessible |
| 1920 | /// even if no aggregate location is provided. |
| 1921 | RValue emitAnyExprToTemp(const clang::Expr *e); |
| 1922 | |
| 1923 | void emitAnyExprToExn(const Expr *e, Address addr); |
| 1924 | |
| 1925 | void emitArrayDestroy(mlir::Value begin, mlir::Value numElements, |
| 1926 | QualType elementType, CharUnits elementAlign, |
| 1927 | Destroyer *destroyer); |
| 1928 | |
| 1929 | mlir::Value emitArrayLength(const clang::ArrayType *arrayType, |
| 1930 | QualType &baseType, Address &addr); |
| 1931 | LValue emitArraySubscriptExpr(const clang::ArraySubscriptExpr *e); |
| 1932 | LValue emitInitListLValue(const InitListExpr *e); |
| 1933 | |
| 1934 | LValue emitExtVectorElementExpr(const ExtVectorElementExpr *e); |
| 1935 | |
| 1936 | LValue emitMatrixSubscriptExpr(const MatrixSubscriptExpr *e); |
| 1937 | |
| 1938 | Address emitArrayToPointerDecay(const Expr *e, |
| 1939 | LValueBaseInfo *baseInfo = nullptr); |
| 1940 | |
| 1941 | std::pair<mlir::Value, mlir::Type> |
| 1942 | emitAsmInputLValue(const TargetInfo::ConstraintInfo &info, LValue inputValue, |
| 1943 | QualType inputType, std::string &constraintString, |
| 1944 | SourceLocation loc); |
| 1945 | std::pair<mlir::Value, mlir::Type> |
| 1946 | emitAsmInput(const TargetInfo::ConstraintInfo &info, const Expr *inputExpr, |
| 1947 | std::string &constraintString); |
| 1948 | mlir::LogicalResult emitAsmStmt(const clang::AsmStmt &s); |
| 1949 | |
| 1950 | RValue emitAtomicExpr(AtomicExpr *e); |
| 1951 | void emitAtomicInit(Expr *init, LValue dest); |
| 1952 | void emitAtomicStore(RValue rvalue, LValue dest, bool isInit); |
| 1953 | void emitAtomicStore(RValue rvalue, LValue dest, cir::MemOrder order, |
| 1954 | bool isVolatile, bool isInit); |
| 1955 | void emitAtomicExprWithMemOrder( |
| 1956 | const Expr *memOrder, bool isStore, bool isLoad, bool isFence, |
| 1957 | llvm::function_ref<void(cir::MemOrder)> emitAtomicOp); |
| 1958 | |
| 1959 | mlir::Value makeBinaryAtomicValue( |
| 1960 | cir::AtomicFetchKind kind, const clang::CallExpr *expr, |
| 1961 | mlir::Type *originalArgType = nullptr, |
| 1962 | mlir::Value *emittedArgValue = nullptr, |
| 1963 | cir::MemOrder ordering = cir::MemOrder::SequentiallyConsistent); |
| 1964 | |
| 1965 | /// Emit `cir.atomic.cmpxchg`. Returns the old value, or the success flag |
| 1966 | /// when `returnBool` is true. |
| 1967 | mlir::Value emitAtomicCmpXchg( |
| 1968 | const clang::CallExpr *expr, bool returnBool, |
| 1969 | cir::MemOrder successOrder = cir::MemOrder::SequentiallyConsistent, |
| 1970 | cir::MemOrder failureOrder = cir::MemOrder::SequentiallyConsistent, |
| 1971 | cir::SyncScopeKind scope = cir::SyncScopeKind::System); |
| 1972 | |
| 1973 | mlir::LogicalResult emitAttributedStmt(const AttributedStmt &s); |
| 1974 | |
| 1975 | AutoVarEmission emitAutoVarAlloca(const clang::VarDecl &d, |
| 1976 | mlir::OpBuilder::InsertPoint ip = {}); |
| 1977 | |
| 1978 | RValue emitPseudoObjectRValue(const PseudoObjectExpr *e, |
| 1979 | AggValueSlot slot = AggValueSlot::ignored()); |
| 1980 | LValue emitPseudoObjectLValue(const PseudoObjectExpr *E); |
| 1981 | |
| 1982 | /// Emit code and set up symbol table for a variable declaration with auto, |
| 1983 | /// register, or no storage class specifier. These turn into simple stack |
| 1984 | /// objects, globals depending on target. |
| 1985 | void emitAutoVarDecl(const clang::VarDecl &d); |
| 1986 | |
| 1987 | void emitAutoVarCleanups(const AutoVarEmission &emission); |
| 1988 | |
| 1989 | /// Emit a loop's condition-variable declaration. This needs special handling |
| 1990 | /// so that we can manage per-iteration cleanups for the loop condition. |
| 1991 | /// \p extendedTemporaries are the types of the temporaries with a destructor |
| 1992 | /// that its initializer lifetime-extends (see |
| 1993 | /// getLifetimeExtendedTemporariesWithDestructor). |
| 1994 | void emitLoopConditionVariable(const clang::VarDecl &d, |
| 1995 | CapturedCleanups &condCleanup, |
| 1996 | llvm::ArrayRef<QualType> extendedTemporaries); |
| 1997 | |
| 1998 | /// The types of the temporaries with a destructor that the initializer of |
| 1999 | /// the variable \p d, which has automatic storage duration, lifetime-extends. |
| 2000 | static llvm::SmallVector<QualType, 2> |
| 2001 | getLifetimeExtendedTemporariesWithDestructor(const clang::VarDecl &d); |
| 2002 | |
| 2003 | /// Whether destroying an object of type \p type, or its elements if it is an |
| 2004 | /// array, may throw an exception. |
| 2005 | static bool destructorMayThrow(QualType type); |
| 2006 | |
| 2007 | /// Emit \p d with captured cleanups. Set \p guardDestruction if the cleanup |
| 2008 | /// region covers initialization. Otherwise it must be reached only after |
| 2009 | /// initialization. The cleanups of \p extendedTemporaries (see |
| 2010 | /// emitLoopConditionVariable) are captured too; they need |
| 2011 | /// \p guardDestruction. |
| 2012 | void emitAutoVarDeclWithCapturedCleanups( |
| 2013 | const clang::VarDecl &d, CapturedCleanups &cleanups, |
| 2014 | bool guardDestruction = false, |
| 2015 | llvm::ArrayRef<QualType> extendedTemporaries = {}); |
| 2016 | |
| 2017 | /// Emit the initializer for an allocated variable. If this call is not |
| 2018 | /// associated with the call to emitAutoVarAlloca (as the address of the |
| 2019 | /// emission is not directly an alloca), the allocatedSeparately parameter can |
| 2020 | /// be used to suppress the assertions. However, this should only be used in |
| 2021 | /// extreme cases, as it doesn't properly reflect the language/AST. |
| 2022 | void emitAutoVarInit(const AutoVarEmission &emission); |
| 2023 | void emitAutoVarTypeCleanup(const AutoVarEmission &emission, |
| 2024 | clang::QualType::DestructionKind dtorKind); |
| 2025 | |
| 2026 | void maybeEmitDeferredVarDeclInit(const VarDecl *vd); |
| 2027 | |
| 2028 | void emitBaseInitializer(mlir::Location loc, const CXXRecordDecl *classDecl, |
| 2029 | CXXCtorInitializer *baseInit); |
| 2030 | |
| 2031 | LValue emitBinaryOperatorLValue(const BinaryOperator *e); |
| 2032 | |
| 2033 | mlir::LogicalResult emitBreakStmt(const clang::BreakStmt &s); |
| 2034 | |
| 2035 | RValue emitBuiltinExpr(const clang::GlobalDecl &gd, unsigned builtinID, |
| 2036 | const clang::CallExpr *e, ReturnValueSlot returnValue); |
| 2037 | |
| 2038 | /// Returns a Value corresponding to the size of the given expression by |
| 2039 | /// emitting a `cir.objsize` operation. |
| 2040 | /// |
| 2041 | /// \param e The expression whose object size to compute |
| 2042 | /// \param type Determines the semantics of the object size computation. |
| 2043 | /// The type parameter is a 2-bit value where: |
| 2044 | /// bit 0 (type & 1): 0 = whole object, 1 = closest subobject |
| 2045 | /// bit 1 (type & 2): 0 = maximum size, 2 = minimum size |
| 2046 | /// \param resType The result type for the size value |
| 2047 | /// \param emittedE Optional pre-emitted pointer value. If non-null, we'll |
| 2048 | /// call `cir.objsize` on this value rather than emitting e. |
| 2049 | /// \param isDynamic If true, allows runtime evaluation via dynamic mode |
| 2050 | mlir::Value emitBuiltinObjectSize(const clang::Expr *e, unsigned type, |
| 2051 | cir::IntType resType, mlir::Value emittedE, |
| 2052 | bool isDynamic); |
| 2053 | |
| 2054 | mlir::Value evaluateOrEmitBuiltinObjectSize(const clang::Expr *e, |
| 2055 | unsigned type, |
| 2056 | cir::IntType resType, |
| 2057 | mlir::Value emittedE, |
| 2058 | bool isDynamic); |
| 2059 | |
| 2060 | int64_t getAccessedFieldNo(unsigned idx, mlir::ArrayAttr elts); |
| 2061 | |
| 2062 | /// Return the CIR signature of the LLVM intrinsic \p id, resolving its |
| 2063 | /// overloaded types to \p overloadTys. Integer types are returned signed. |
| 2064 | cir::FuncType getIntrinsicType(llvm::Intrinsic::ID id, |
| 2065 | llvm::ArrayRef<mlir::Type> overloadTys = {}); |
| 2066 | |
| 2067 | /// Emit a simple LLVM intrinsic that takes N scalar arguments. The intrinsic |
| 2068 | /// name is used verbatim; any overload mangling (e.g. `.f32`, `.p1`) must be |
| 2069 | /// baked into \p intrinName by the caller. The result type defaults to the |
| 2070 | /// type of the first argument; pass \p resultType for intrinsics whose result |
| 2071 | /// differs from the operand, such as a vector reduction that returns the |
| 2072 | /// element type. Unlike classic CodeGen, CIR has no intrinsic registry to |
| 2073 | /// derive the result type from the operand, so it must be supplied here. |
| 2074 | template <unsigned N> |
| 2075 | [[maybe_unused]] RValue |
| 2076 | emitBuiltinWithOneOverloadedType(const CallExpr *e, |
| 2077 | llvm::StringRef intrinName, |
| 2078 | mlir::Type resultType = {}) { |
| 2079 | static_assert(N, "expect non-empty argument" ); |
| 2080 | mlir::Type cirTy = |
| 2081 | resultType ? resultType : convertType(t: e->getArg(Arg: 0)->getType()); |
| 2082 | SmallVector<mlir::Value, N> args; |
| 2083 | for (unsigned i = 0; i < N; ++i) |
| 2084 | args.push_back(emitScalarExpr(e: e->getArg(Arg: i))); |
| 2085 | const auto call = cir::LLVMIntrinsicCallOp::create( |
| 2086 | builder, getLoc(srcLoc: e->getExprLoc()), builder.getStringAttr(bytes: intrinName), |
| 2087 | cirTy, args); |
| 2088 | return RValue::get(v: call->getResult(0)); |
| 2089 | } |
| 2090 | |
| 2091 | RValue emitCall(const CIRGenFunctionInfo &funcInfo, |
| 2092 | const CIRGenCallee &callee, ReturnValueSlot returnValue, |
| 2093 | const CallArgList &args, cir::CIRCallOpInterface *callOp, |
| 2094 | bool isMustTail, SourceRange clangLoc); |
| 2095 | RValue emitCall(const CIRGenFunctionInfo &funcInfo, |
| 2096 | const CIRGenCallee &callee, ReturnValueSlot returnValue, |
| 2097 | const CallArgList &args, bool isMustTail, |
| 2098 | cir::CIRCallOpInterface *callOrTryCall = nullptr) { |
| 2099 | assert(currSrcLoc && "source location must have been set" ); |
| 2100 | return emitCall(funcInfo, callee, returnValue, args, callOp: callOrTryCall, |
| 2101 | isMustTail, clangLoc: *currSrcLoc); |
| 2102 | } |
| 2103 | |
| 2104 | RValue emitCall(clang::QualType calleeTy, const CIRGenCallee &callee, |
| 2105 | const clang::CallExpr *e, ReturnValueSlot returnValue); |
| 2106 | |
| 2107 | /// Emit the call and return for a thunk function. |
| 2108 | void emitCallAndReturnForThunk(cir::FuncOp callee, SourceRange fnLoc, |
| 2109 | const ThunkInfo *thunk, bool isUnprototyped); |
| 2110 | |
| 2111 | void emitCallArg(CallArgList &args, const clang::Expr *e, |
| 2112 | clang::QualType argType); |
| 2113 | void emitCallArgs( |
| 2114 | CallArgList &args, PrototypeWrapper prototype, |
| 2115 | llvm::iterator_range<clang::CallExpr::const_arg_iterator> argRange, |
| 2116 | AbstractCallee callee = AbstractCallee(), unsigned paramsToSkip = 0, |
| 2117 | EvaluationOrder order = EvaluationOrder::Default); |
| 2118 | RValue emitCallExpr(const clang::CallExpr *e, |
| 2119 | ReturnValueSlot returnValue = ReturnValueSlot()); |
| 2120 | LValue emitCallExprLValue(const clang::CallExpr *e); |
| 2121 | |
| 2122 | LValue emitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *e); |
| 2123 | LValue emitCXXConstructLValue(const CXXConstructExpr *e); |
| 2124 | CIRGenCallee emitCallee(const clang::Expr *e); |
| 2125 | |
| 2126 | template <typename T> |
| 2127 | mlir::LogicalResult emitCaseDefaultCascade(const T *stmt, mlir::Type condType, |
| 2128 | mlir::ArrayAttr value, |
| 2129 | cir::CaseOpKind kind, |
| 2130 | bool buildingTopLevelCase); |
| 2131 | |
| 2132 | LValue emitCXXTypeidLValue(const CXXTypeidExpr *e); |
| 2133 | |
| 2134 | mlir::LogicalResult emitCaseStmt(const clang::CaseStmt &s, |
| 2135 | mlir::Type condType, |
| 2136 | bool buildingTopLevelCase); |
| 2137 | |
| 2138 | LValue emitCastLValue(const CastExpr *e); |
| 2139 | |
| 2140 | /// Emits an argument for a call to a `__builtin_assume`. If the builtin |
| 2141 | /// sanitizer is enabled, a runtime check is also emitted. |
| 2142 | mlir::Value emitCheckedArgForAssume(const Expr *e); |
| 2143 | |
| 2144 | /// Emit a conversion from the specified complex type to the specified |
| 2145 | /// destination type, where the destination type is an LLVM scalar type. |
| 2146 | mlir::Value emitComplexToScalarConversion(mlir::Value src, QualType srcTy, |
| 2147 | QualType dstTy, SourceLocation loc); |
| 2148 | |
| 2149 | LValue emitCompoundAssignmentLValue(const clang::CompoundAssignOperator *e); |
| 2150 | LValue emitCompoundLiteralLValue(const CompoundLiteralExpr *e); |
| 2151 | |
| 2152 | void emitConstructorBody(FunctionArgList &args); |
| 2153 | |
| 2154 | mlir::LogicalResult emitCoroutineBody(const CoroutineBodyStmt &s); |
| 2155 | cir::CoroEndOp emitCoroEndBuiltinCall(const CallExpr *e); |
| 2156 | cir::CoroIdOp emitCoroIDBuiltinCall(const CallExpr *e); |
| 2157 | cir::CoroAllocOp emitCoroAllocBuiltinCall(const CallExpr *e); |
| 2158 | cir::CoroBeginOp emitCoroBeginBuiltinCall(const CallExpr *e); |
| 2159 | cir::CoroPromiseOp emitCoroPromiseBuiltinCall(const CallExpr *e); |
| 2160 | cir::CoroDoneOp emitCoroDoneBuiltinCall(const CallExpr *e); |
| 2161 | cir::CoroResumeOp emitCoroResumeBuiltinCall(const CallExpr *e); |
| 2162 | cir::CoroDestroyOp emitCoroDestroyBuiltinCall(const CallExpr *e); |
| 2163 | cir::CoroNoopOp emitCoroNoopBuiltinCall(const CallExpr *e); |
| 2164 | cir::CoroSuspendOp emitCoroSuspendBuiltinCall(const CallExpr *e); |
| 2165 | |
| 2166 | cir::CoroSizeOp emitCoroSizeBuiltinCall(const CallExpr *e); |
| 2167 | cir::CoroAlignOp emitCoroAlignBuiltinCall(const CallExpr *e); |
| 2168 | cir::CoroFreeOp emitCoroFreeBuiltin(const CallExpr *e); |
| 2169 | RValue emitCoroutineFrame(); |
| 2170 | |
| 2171 | void emitDestroy(Address addr, QualType type, Destroyer *destroyer); |
| 2172 | |
| 2173 | void emitDestructorBody(FunctionArgList &args); |
| 2174 | |
| 2175 | mlir::LogicalResult emitContinueStmt(const clang::ContinueStmt &s); |
| 2176 | |
| 2177 | mlir::LogicalResult emitCoreturnStmt(const CoreturnStmt &s); |
| 2178 | |
| 2179 | void emitCXXConstructExpr(const clang::CXXConstructExpr *e, |
| 2180 | AggValueSlot dest); |
| 2181 | |
| 2182 | void emitCXXAggrConstructorCall(const CXXConstructorDecl *ctor, |
| 2183 | const clang::ArrayType *arrayType, |
| 2184 | Address arrayBegin, const CXXConstructExpr *e, |
| 2185 | bool newPointerIsChecked, |
| 2186 | bool zeroInitialize = false); |
| 2187 | void emitCXXAggrConstructorCall(const CXXConstructorDecl *ctor, |
| 2188 | mlir::Value numElements, Address arrayBase, |
| 2189 | const CXXConstructExpr *e, |
| 2190 | bool newPointerIsChecked, bool zeroInitialize, |
| 2191 | Address endOfInit); |
| 2192 | void emitCXXConstructorCall(const clang::CXXConstructorDecl *d, |
| 2193 | clang::CXXCtorType type, bool forVirtualBase, |
| 2194 | bool delegating, AggValueSlot thisAVS, |
| 2195 | const clang::CXXConstructExpr *e); |
| 2196 | |
| 2197 | void emitCXXConstructorCall(const clang::CXXConstructorDecl *d, |
| 2198 | clang::CXXCtorType type, bool forVirtualBase, |
| 2199 | bool delegating, Address thisAddr, |
| 2200 | CallArgList &args, clang::SourceLocation loc); |
| 2201 | |
| 2202 | void emitInheritedCXXConstructorCall(const CXXConstructorDecl *d, |
| 2203 | bool forVirtualBase, Address thisAddr, |
| 2204 | bool inheritedFromVBase, |
| 2205 | const CXXInheritedCtorInitExpr *e); |
| 2206 | |
| 2207 | void emitInlinedInheritingCXXConstructorCall( |
| 2208 | SourceLocation loc, const CXXConstructorDecl *d, CXXCtorType ctorType, |
| 2209 | bool forVirtualBase, bool delegating, CallArgList &args); |
| 2210 | |
| 2211 | void emitCXXDeleteExpr(const CXXDeleteExpr *e); |
| 2212 | |
| 2213 | void emitCXXDestructorCall(const CXXDestructorDecl *dd, CXXDtorType type, |
| 2214 | bool forVirtualBase, bool delegating, |
| 2215 | Address thisAddr, QualType thisTy); |
| 2216 | |
| 2217 | RValue emitCXXDestructorCall(GlobalDecl dtor, const CIRGenCallee &callee, |
| 2218 | mlir::Value thisVal, QualType thisTy, |
| 2219 | mlir::Value implicitParam, |
| 2220 | QualType implicitParamTy, const CallExpr *e); |
| 2221 | |
| 2222 | mlir::LogicalResult emitCXXForRangeStmt(const CXXForRangeStmt &s, |
| 2223 | llvm::ArrayRef<const Attr *> attrs); |
| 2224 | |
| 2225 | RValue emitCXXMemberCallExpr(const clang::CXXMemberCallExpr *e, |
| 2226 | ReturnValueSlot returnValue); |
| 2227 | |
| 2228 | Address emitCXXMemberDataPointerAddress( |
| 2229 | const Expr *e, Address base, mlir::Value memberPtr, |
| 2230 | const MemberPointerType *memberPtrType, LValueBaseInfo *baseInfo); |
| 2231 | |
| 2232 | RValue emitCXXMemberOrOperatorCall( |
| 2233 | const clang::CXXMethodDecl *md, const CIRGenCallee &callee, |
| 2234 | ReturnValueSlot returnValue, mlir::Value thisPtr, |
| 2235 | mlir::Value implicitParam, clang::QualType implicitParamTy, |
| 2236 | const clang::CallExpr *ce, CallArgList *rtlArgs); |
| 2237 | |
| 2238 | RValue emitCXXMemberOrOperatorMemberCallExpr( |
| 2239 | const clang::CallExpr *ce, const clang::CXXMethodDecl *md, |
| 2240 | ReturnValueSlot returnValue, bool hasQualifier, |
| 2241 | clang::NestedNameSpecifier qualifier, bool isArrow, |
| 2242 | const clang::Expr *base); |
| 2243 | |
| 2244 | RValue emitCXXMemberPointerCallExpr(const CXXMemberCallExpr *ce, |
| 2245 | ReturnValueSlot returnValue); |
| 2246 | |
| 2247 | mlir::Value emitCXXNewExpr(const CXXNewExpr *e); |
| 2248 | |
| 2249 | void emitNewArrayInitializer(const CXXNewExpr *e, QualType elementType, |
| 2250 | mlir::Type elementTy, Address beginPtr, |
| 2251 | mlir::Value numElements, |
| 2252 | mlir::Value allocSizeWithoutCookie); |
| 2253 | |
| 2254 | /// Create a check for a function parameter that may potentially be |
| 2255 | /// declared as non-null. |
| 2256 | void emitNonNullArgCheck(RValue rv, QualType argType, SourceLocation argLoc, |
| 2257 | AbstractCallee ac, unsigned paramNum); |
| 2258 | |
| 2259 | RValue emitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *e, |
| 2260 | const CXXMethodDecl *md, |
| 2261 | ReturnValueSlot returnValue); |
| 2262 | |
| 2263 | RValue emitCUDAKernelCallExpr(const CUDAKernelCallExpr *expr, |
| 2264 | ReturnValueSlot returnValue); |
| 2265 | |
| 2266 | RValue emitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *expr); |
| 2267 | |
| 2268 | RValue emitNewOrDeleteBuiltinCall(const FunctionProtoType *type, |
| 2269 | const CallExpr *callExpr, |
| 2270 | OverloadedOperatorKind op); |
| 2271 | |
| 2272 | void emitCXXTemporary(const CXXTemporary *temporary, QualType tempType, |
| 2273 | Address ptr); |
| 2274 | |
| 2275 | void emitCXXThrowExpr(const CXXThrowExpr *e); |
| 2276 | |
| 2277 | struct cxxTryBodyEmitter { |
| 2278 | virtual mlir::LogicalResult operator()(CIRGenFunction &cgf) = 0; |
| 2279 | virtual ~cxxTryBodyEmitter() = default; |
| 2280 | }; |
| 2281 | |
| 2282 | void emitBeginCatch(const CXXCatchStmt *catchStmt, mlir::Value ehToken); |
| 2283 | |
| 2284 | mlir::LogicalResult emitCXXTryStmt(const clang::CXXTryStmt &s, |
| 2285 | cxxTryBodyEmitter &bodyCallback, |
| 2286 | bool isFnTryBlock = false); |
| 2287 | mlir::LogicalResult emitCXXTryStmt(const clang::CXXTryStmt &s); |
| 2288 | |
| 2289 | void emitCtorPrologue(const clang::CXXConstructorDecl *ctor, |
| 2290 | clang::CXXCtorType ctorType, FunctionArgList &args); |
| 2291 | |
| 2292 | // It's important not to confuse this and emitDelegateCXXConstructorCall. |
| 2293 | // Delegating constructors are the C++11 feature. The constructor delegate |
| 2294 | // optimization is used to reduce duplication in the base and complete |
| 2295 | // constructors where they are substantially the same. |
| 2296 | void emitDelegatingCXXConstructorCall(const CXXConstructorDecl *ctor, |
| 2297 | const FunctionArgList &args); |
| 2298 | |
| 2299 | void emitDeleteCall(const FunctionDecl *deleteFD, mlir::Value ptr, |
| 2300 | QualType deleteTy); |
| 2301 | |
| 2302 | mlir::LogicalResult emitDoStmt(const clang::DoStmt &s, |
| 2303 | llvm::ArrayRef<const Attr *> attrs = {}); |
| 2304 | |
| 2305 | mlir::Value emitCXXTypeidExpr(const CXXTypeidExpr *e); |
| 2306 | mlir::Value emitDynamicCast(Address thisAddr, const CXXDynamicCastExpr *dce); |
| 2307 | |
| 2308 | /// Emit an expression as an initializer for an object (variable, field, etc.) |
| 2309 | /// at the given location. The expression is not necessarily the normal |
| 2310 | /// initializer for the object, and the address is not necessarily |
| 2311 | /// its normal location. |
| 2312 | /// |
| 2313 | /// \param init the initializing expression |
| 2314 | /// \param d the object to act as if we're initializing |
| 2315 | /// \param lvalue the lvalue to initialize |
| 2316 | /// \param capturedByInit true if \p d is a __block variable whose address is |
| 2317 | /// potentially changed by the initializer |
| 2318 | void emitExprAsInit(const clang::Expr *init, const clang::ValueDecl *d, |
| 2319 | LValue lvalue, bool capturedByInit = false); |
| 2320 | |
| 2321 | mlir::LogicalResult emitFunctionBody(const clang::Stmt *body); |
| 2322 | |
| 2323 | mlir::LogicalResult emitGotoStmt(const clang::GotoStmt &s); |
| 2324 | |
| 2325 | mlir::LogicalResult emitIndirectGotoStmt(const IndirectGotoStmt &s); |
| 2326 | |
| 2327 | void emitImplicitAssignmentOperatorBody(FunctionArgList &args); |
| 2328 | |
| 2329 | void emitInitializerForField(clang::FieldDecl *field, LValue lhs, |
| 2330 | clang::Expr *init); |
| 2331 | |
| 2332 | LValue emitPredefinedLValue(const PredefinedExpr *e); |
| 2333 | |
| 2334 | mlir::Value emitPromotedComplexExpr(const Expr *e, QualType promotionType); |
| 2335 | |
| 2336 | mlir::Value emitPromotedScalarExpr(const Expr *e, QualType promotionType); |
| 2337 | |
| 2338 | mlir::Value emitPromotedValue(mlir::Value result, QualType promotionType); |
| 2339 | |
| 2340 | void emitReturnOfRValue(mlir::Location loc, RValue rv, QualType ty); |
| 2341 | |
| 2342 | mlir::Value emitRuntimeCall(mlir::Location loc, cir::FuncOp callee, |
| 2343 | llvm::ArrayRef<mlir::Value> args = {}, |
| 2344 | mlir::NamedAttrList attrs = {}); |
| 2345 | |
| 2346 | void emitInvariantStart(CharUnits size, mlir::Value addr, mlir::Location loc); |
| 2347 | |
| 2348 | /// Emit the computation of the specified expression of scalar type. |
| 2349 | mlir::Value emitScalarExpr(const clang::Expr *e, |
| 2350 | bool ignoreResultAssign = false); |
| 2351 | |
| 2352 | mlir::Value emitScalarPrePostIncDec(const UnaryOperator *e, LValue lv); |
| 2353 | |
| 2354 | /// Build a debug stoppoint if we are emitting debug info. |
| 2355 | void emitStopPoint(const Stmt *s); |
| 2356 | |
| 2357 | // Build CIR for a statement. useCurrentScope should be true if no |
| 2358 | // new scopes need be created when finding a compound statement. |
| 2359 | mlir::LogicalResult emitStmt(const clang::Stmt *s, bool useCurrentScope, |
| 2360 | llvm::ArrayRef<const Attr *> attrs = {}); |
| 2361 | |
| 2362 | mlir::LogicalResult emitSimpleStmt(const clang::Stmt *s, |
| 2363 | bool useCurrentScope); |
| 2364 | |
| 2365 | mlir::LogicalResult emitForStmt(const clang::ForStmt &s, |
| 2366 | llvm::ArrayRef<const Attr *> attrs = {}); |
| 2367 | |
| 2368 | void emitForwardingCallToLambda(const CXXMethodDecl *lambdaCallOperator, |
| 2369 | CallArgList &callArgs); |
| 2370 | |
| 2371 | RValue emitCoawaitExpr(const CoawaitExpr &e, |
| 2372 | AggValueSlot aggSlot = AggValueSlot::ignored(), |
| 2373 | bool ignoreResult = false); |
| 2374 | LValue emitCoawaitLValue(const CoawaitExpr *e); |
| 2375 | |
| 2376 | RValue emitCoyieldExpr(const CoyieldExpr &e, |
| 2377 | AggValueSlot aggSlot = AggValueSlot::ignored(), |
| 2378 | bool ignoreResult = false); |
| 2379 | LValue emitCoyieldLValue(const CoyieldExpr *e); |
| 2380 | /// Emit the computation of the specified expression of complex type, |
| 2381 | /// returning the result. |
| 2382 | mlir::Value emitComplexExpr(const Expr *e); |
| 2383 | |
| 2384 | void emitComplexExprIntoLValue(const Expr *e, LValue dest, bool isInit); |
| 2385 | |
| 2386 | mlir::Value emitComplexPrePostIncDec(const UnaryOperator *e, LValue lv); |
| 2387 | |
| 2388 | LValue emitComplexAssignmentLValue(const BinaryOperator *e); |
| 2389 | LValue emitComplexCompoundAssignmentLValue(const CompoundAssignOperator *e); |
| 2390 | LValue emitScalarCompoundAssignWithComplex(const CompoundAssignOperator *e, |
| 2391 | mlir::Value &result); |
| 2392 | |
| 2393 | mlir::LogicalResult |
| 2394 | emitCompoundStmt(const clang::CompoundStmt &s, Address *lastValue = nullptr, |
| 2395 | AggValueSlot slot = AggValueSlot::ignored()); |
| 2396 | |
| 2397 | mlir::LogicalResult |
| 2398 | emitCompoundStmtWithoutScope(const clang::CompoundStmt &s, |
| 2399 | Address *lastValue = nullptr, |
| 2400 | AggValueSlot slot = AggValueSlot::ignored()); |
| 2401 | |
| 2402 | void emitDecl(const clang::Decl &d, bool evaluateConditionDecl = false); |
| 2403 | mlir::LogicalResult emitDeclStmt(const clang::DeclStmt &s); |
| 2404 | LValue emitDeclRefLValue(const clang::DeclRefExpr *e); |
| 2405 | |
| 2406 | mlir::LogicalResult emitDefaultStmt(const clang::DefaultStmt &s, |
| 2407 | mlir::Type condType, |
| 2408 | bool buildingTopLevelCase); |
| 2409 | |
| 2410 | void emitDelegateCXXConstructorCall(const clang::CXXConstructorDecl *ctor, |
| 2411 | clang::CXXCtorType ctorType, |
| 2412 | const FunctionArgList &args, |
| 2413 | clang::SourceLocation loc); |
| 2414 | |
| 2415 | /// We are performing a delegate call; that is, the current function is |
| 2416 | /// delegating to another one. Produce a r-value suitable for passing the |
| 2417 | /// given parameter. |
| 2418 | void emitDelegateCallArg(CallArgList &args, const clang::VarDecl *param, |
| 2419 | clang::SourceLocation loc); |
| 2420 | |
| 2421 | /// Emit an `if` on a boolean condition to the specified blocks. |
| 2422 | /// FIXME: Based on the condition, this might try to simplify the codegen of |
| 2423 | /// the conditional based on the branch. |
| 2424 | /// In the future, we may apply code generation simplifications here, |
| 2425 | /// similar to those used in classic LLVM codegen |
| 2426 | /// See `EmitBranchOnBoolExpr` for inspiration. |
| 2427 | mlir::LogicalResult emitIfOnBoolExpr(const clang::Expr *cond, |
| 2428 | const clang::Stmt *thenS, |
| 2429 | const clang::Stmt *elseS); |
| 2430 | cir::IfOp emitIfOnBoolExpr(const clang::Expr *cond, |
| 2431 | BuilderCallbackRef thenBuilder, |
| 2432 | mlir::Location thenLoc, |
| 2433 | BuilderCallbackRef elseBuilder, |
| 2434 | std::optional<mlir::Location> elseLoc = {}); |
| 2435 | |
| 2436 | /// Build the cir.if for an already-emitted condition value. |
| 2437 | cir::IfOp emitIfOnBoolValue(mlir::Value condV, mlir::Location loc, |
| 2438 | BuilderCallbackRef thenBuilder, |
| 2439 | mlir::Location thenLoc, |
| 2440 | BuilderCallbackRef elseBuilder, |
| 2441 | std::optional<mlir::Location> elseLoc = {}); |
| 2442 | |
| 2443 | mlir::Value emitOpOnBoolExpr(mlir::Location loc, const clang::Expr *cond); |
| 2444 | |
| 2445 | LValue emitPointerToDataMemberBinaryExpr(const BinaryOperator *e); |
| 2446 | |
| 2447 | mlir::LogicalResult emitLabel(const clang::LabelDecl &d); |
| 2448 | mlir::LogicalResult emitLabelStmt(const clang::LabelStmt &s); |
| 2449 | |
| 2450 | void emitLambdaDelegatingInvokeBody(const CXXMethodDecl *md); |
| 2451 | void emitLambdaStaticInvokeBody(const CXXMethodDecl *md); |
| 2452 | |
| 2453 | mlir::LogicalResult emitIfStmt(const clang::IfStmt &s); |
| 2454 | |
| 2455 | /// Emit code to compute the specified expression, |
| 2456 | /// ignoring the result. |
| 2457 | void emitIgnoredExpr(const clang::Expr *e); |
| 2458 | |
| 2459 | RValue emitLoadOfBitfieldLValue(LValue lv, SourceLocation loc); |
| 2460 | |
| 2461 | /// Load a complex number from the specified l-value. |
| 2462 | mlir::Value emitLoadOfComplex(LValue src, SourceLocation loc); |
| 2463 | |
| 2464 | RValue emitLoadOfExtVectorElementLValue(LValue lv); |
| 2465 | |
| 2466 | /// Given an expression that represents a value lvalue, this method emits |
| 2467 | /// the address of the lvalue, then loads the result as an rvalue, |
| 2468 | /// returning the rvalue. |
| 2469 | RValue emitLoadOfLValue(LValue lv, SourceLocation loc); |
| 2470 | |
| 2471 | Address emitLoadOfReference(LValue refLVal, mlir::Location loc, |
| 2472 | LValueBaseInfo *pointeeBaseInfo); |
| 2473 | LValue emitLoadOfReferenceLValue(Address refAddr, mlir::Location loc, |
| 2474 | QualType refTy, AlignmentSource source); |
| 2475 | |
| 2476 | /// EmitLoadOfScalar - Load a scalar value from an address, taking |
| 2477 | /// care to appropriately convert from the memory representation to |
| 2478 | /// the LLVM value representation. The l-value must be a simple |
| 2479 | /// l-value. |
| 2480 | mlir::Value emitLoadOfScalar(LValue lvalue, SourceLocation loc); |
| 2481 | mlir::Value emitLoadOfScalar(Address addr, bool isVolatile, QualType ty, |
| 2482 | SourceLocation loc, LValueBaseInfo baseInfo, |
| 2483 | bool isNontemporal = false); |
| 2484 | |
| 2485 | /// Emit code to compute a designator that specifies the location |
| 2486 | /// of the expression. |
| 2487 | /// FIXME: document this function better. |
| 2488 | LValue emitLValue(const clang::Expr *e); |
| 2489 | LValue emitLValueForBitField(LValue base, const FieldDecl *field); |
| 2490 | LValue emitLValueForField(LValue base, const clang::FieldDecl *field); |
| 2491 | |
| 2492 | /// Annotate the address of `field`; accesses must use the result. |
| 2493 | Address emitFieldAnnotations(const clang::FieldDecl *field, Address addr); |
| 2494 | |
| 2495 | /// Annotate the storage of `d` at `addr`. |
| 2496 | void emitVarAnnotations(const clang::VarDecl *d, mlir::Value addr); |
| 2497 | |
| 2498 | LValue emitLValueForLambdaField(const FieldDecl *field); |
| 2499 | LValue emitLValueForLambdaField(const FieldDecl *field, |
| 2500 | mlir::Value thisValue); |
| 2501 | |
| 2502 | void emitLambdaVLACapture(const VariableArrayType *vat, LValue lv) { |
| 2503 | mlir::Value size = vlaSizeMap[vat->getSizeExpr()]; |
| 2504 | emitStoreThroughLValue(src: RValue::get(v: size), dst: lv); |
| 2505 | } |
| 2506 | |
| 2507 | /// Like emitLValueForField, excpet that if the Field is a reference, this |
| 2508 | /// will return the address of the reference and not the address of the value |
| 2509 | /// stored in the reference. |
| 2510 | LValue emitLValueForFieldInitialization(LValue base, |
| 2511 | const clang::FieldDecl *field, |
| 2512 | llvm::StringRef fieldName); |
| 2513 | |
| 2514 | LValue emitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *e); |
| 2515 | |
| 2516 | LValue emitMatrixElementExpr(const MatrixElementExpr *e); |
| 2517 | LValue emitMemberExpr(const MemberExpr *e); |
| 2518 | |
| 2519 | /// Emit a musttail call for a thunk with a potentially different ABI. |
| 2520 | void emitMustTailThunk(GlobalDecl gd, mlir::Value adjustedThisPtr, |
| 2521 | cir::FuncOp callee); |
| 2522 | |
| 2523 | /// Emit a call to an AMDGPU builtin function. |
| 2524 | std::optional<mlir::Value> emitAMDGPUBuiltinExpr(unsigned builtinID, |
| 2525 | const CallExpr *expr); |
| 2526 | |
| 2527 | /// Emit a call to an NVPTX builtin function. |
| 2528 | std::optional<mlir::Value> emitNVPTXBuiltinExpr(unsigned builtinID, |
| 2529 | const CallExpr *expr); |
| 2530 | |
| 2531 | /// Emit a call to a SPIR-V builtin function. |
| 2532 | std::optional<mlir::Value> emitSPIRVBuiltinExpr(unsigned builtinID, |
| 2533 | const CallExpr *expr); |
| 2534 | |
| 2535 | /// Emit a device-side printf call for GPU targets. |
| 2536 | mlir::Value emitDevicePrintfCallExpr(const CallExpr *expr); |
| 2537 | |
| 2538 | LValue emitOpaqueValueLValue(const OpaqueValueExpr *e); |
| 2539 | |
| 2540 | LValue emitConditionalOperatorLValue(const AbstractConditionalOperator *expr); |
| 2541 | |
| 2542 | /// Given an expression with a pointer type, emit the value and compute our |
| 2543 | /// best estimate of the alignment of the pointee. |
| 2544 | /// |
| 2545 | /// One reasonable way to use this information is when there's a language |
| 2546 | /// guarantee that the pointer must be aligned to some stricter value, and |
| 2547 | /// we're simply trying to ensure that sufficiently obvious uses of under- |
| 2548 | /// aligned objects don't get miscompiled; for example, a placement new |
| 2549 | /// into the address of a local variable. In such a case, it's quite |
| 2550 | /// reasonable to just ignore the returned alignment when it isn't from an |
| 2551 | /// explicit source. |
| 2552 | Address emitPointerWithAlignment(const clang::Expr *expr, |
| 2553 | LValueBaseInfo *baseInfo = nullptr); |
| 2554 | |
| 2555 | /// Emits a reference binding to the passed in expression. |
| 2556 | RValue emitReferenceBindingToExpr(const Expr *e); |
| 2557 | |
| 2558 | mlir::LogicalResult emitReturnStmt(const clang::ReturnStmt &s); |
| 2559 | |
| 2560 | RValue emitRotate(const CallExpr *e, bool isRotateLeft); |
| 2561 | |
| 2562 | mlir::Value emitScalarConstant(const ConstantEmission &constant, Expr *e); |
| 2563 | |
| 2564 | /// Emit a conversion from the specified type to the specified destination |
| 2565 | /// type, both of which are CIR scalar types. |
| 2566 | mlir::Value emitScalarConversion(mlir::Value src, clang::QualType srcType, |
| 2567 | clang::QualType dstType, |
| 2568 | clang::SourceLocation loc); |
| 2569 | |
| 2570 | void emitScalarInit(const clang::Expr *init, LValue lvalue, |
| 2571 | bool capturedByInit = false); |
| 2572 | |
| 2573 | mlir::Value emitScalarOrConstFoldImmArg(unsigned iceArguments, unsigned idx, |
| 2574 | const Expr *argExpr); |
| 2575 | |
| 2576 | void emitStaticVarDecl(const VarDecl &d, cir::GlobalLinkageKind linkage); |
| 2577 | |
| 2578 | /// Emit a guarded initializer for a static local variable. |
| 2579 | void emitCXXGuardedInit(const VarDecl &varDecl, cir::GlobalOp globalOp, |
| 2580 | bool performInit); |
| 2581 | |
| 2582 | void emitStoreOfComplex(mlir::Location loc, mlir::Value v, LValue dest, |
| 2583 | bool isInit); |
| 2584 | |
| 2585 | void emitStoreOfScalar(mlir::Value value, Address addr, bool isVolatile, |
| 2586 | clang::QualType ty, LValueBaseInfo baseInfo, |
| 2587 | bool isInit = false, bool isNontemporal = false); |
| 2588 | void emitStoreOfScalar(mlir::Value value, LValue lvalue, bool isInit); |
| 2589 | |
| 2590 | void emitStoreThroughExtVectorComponentLValue(RValue src, LValue dst); |
| 2591 | |
| 2592 | /// Store the specified rvalue into the specified |
| 2593 | /// lvalue, where both are guaranteed to the have the same type, and that |
| 2594 | /// type is 'Ty'. |
| 2595 | void emitStoreThroughLValue(RValue src, LValue dst, bool isInit = false); |
| 2596 | |
| 2597 | mlir::Value emitStoreThroughBitfieldLValue(RValue src, LValue dstresult); |
| 2598 | |
| 2599 | LValue emitStringLiteralLValue(const StringLiteral *e, |
| 2600 | llvm::StringRef name = ".str" ); |
| 2601 | |
| 2602 | mlir::LogicalResult emitSwitchBody(const clang::Stmt *s); |
| 2603 | mlir::LogicalResult emitSwitchCase(const clang::SwitchCase &s, |
| 2604 | bool buildingTopLevelCase); |
| 2605 | mlir::LogicalResult emitSwitchStmt(const clang::SwitchStmt &s); |
| 2606 | |
| 2607 | mlir::LogicalResult emitSYCLKernelCallStmt(const SYCLKernelCallStmt &s); |
| 2608 | |
| 2609 | void emitSYCLKernelCaller(const clang::OutlinedFunctionDecl *outlinedFnDecl, |
| 2610 | cir::FuncOp funcOp, cir::FuncType funcType, |
| 2611 | FunctionArgList &args); |
| 2612 | |
| 2613 | /// Remove leftover empty and unreachable blocks from an emitted function. |
| 2614 | static void eraseEmptyAndUnusedBlocks(cir::FuncOp func); |
| 2615 | |
| 2616 | std::optional<mlir::Value> |
| 2617 | emitTargetBuiltinExpr(unsigned builtinID, const clang::CallExpr *e, |
| 2618 | ReturnValueSlot &returnValue); |
| 2619 | |
| 2620 | /// Emit a diagnostic if the target features required by \p targetDecl are |
| 2621 | /// not available in the calling function. Mirrors CodeGenFunction behavior. |
| 2622 | void checkTargetFeatures(const clang::CallExpr *e, |
| 2623 | const clang::FunctionDecl *targetDecl); |
| 2624 | void checkTargetFeatures(clang::SourceLocation loc, |
| 2625 | const clang::FunctionDecl *targetDecl); |
| 2626 | |
| 2627 | /// Given a value and its clang type, returns the value casted to its memory |
| 2628 | /// representation. |
| 2629 | /// Note: CIR defers most of the special casting to the final lowering passes |
| 2630 | /// to conserve the high level information. |
| 2631 | mlir::Value emitToMemory(mlir::Value value, clang::QualType ty); |
| 2632 | |
| 2633 | /// EmitFromMemory - Change a scalar value from its memory |
| 2634 | /// representation to its value representation. |
| 2635 | mlir::Value emitFromMemory(mlir::Value value, clang::QualType ty); |
| 2636 | |
| 2637 | /// Emit a trap instruction, which is used to abort the program in an abnormal |
| 2638 | /// way, usually for debugging purposes. |
| 2639 | /// \p createNewBlock indicates whether to create a new block for the IR |
| 2640 | /// builder. Since the `cir.trap` operation is a terminator, operations that |
| 2641 | /// follow a trap cannot be emitted after `cir.trap` in the same block. To |
| 2642 | /// ensure these operations get emitted successfully, you need to create a new |
| 2643 | /// dummy block and set the insertion point there before continuing from the |
| 2644 | /// trap operation. |
| 2645 | void emitTrap(mlir::Location loc, bool createNewBlock); |
| 2646 | |
| 2647 | LValue emitUnaryOpLValue(const clang::UnaryOperator *e); |
| 2648 | |
| 2649 | mlir::Value emitUnPromotedValue(mlir::Value result, QualType unPromotionType); |
| 2650 | |
| 2651 | /// Emit a reached-unreachable diagnostic if \p loc is valid and runtime |
| 2652 | /// checking is enabled. Otherwise, just emit an unreachable instruction. |
| 2653 | /// \p createNewBlock indicates whether to create a new block for the IR |
| 2654 | /// builder. Since the `cir.unreachable` operation is a terminator, operations |
| 2655 | /// that follow an unreachable point cannot be emitted after `cir.unreachable` |
| 2656 | /// in the same block. To ensure these operations get emitted successfully, |
| 2657 | /// you need to create a dummy block and set the insertion point there before |
| 2658 | /// continuing from the unreachable point. |
| 2659 | void emitUnreachable(clang::SourceLocation loc, bool createNewBlock); |
| 2660 | |
| 2661 | /// This method handles emission of any variable declaration |
| 2662 | /// inside a function, including static vars etc. |
| 2663 | void emitVarDecl(const clang::VarDecl &d); |
| 2664 | |
| 2665 | void emitVariablyModifiedType(QualType ty); |
| 2666 | |
| 2667 | mlir::LogicalResult emitWhileStmt(const clang::WhileStmt &s, |
| 2668 | llvm::ArrayRef<const Attr *> attrs = {}); |
| 2669 | |
| 2670 | /// Build the hints that get attached to a loop, or a null attribute |
| 2671 | /// if there is nothing to say about it. |
| 2672 | cir::LoopHintAttr getLoopHints(llvm::ArrayRef<const Attr *> attrs, |
| 2673 | bool mustProgress); |
| 2674 | |
| 2675 | std::optional<mlir::Value> emitRISCVBuiltinExpr(unsigned builtinID, |
| 2676 | const CallExpr *expr); |
| 2677 | cir::GetGlobalOp createGetCpuModel(mlir::Location loc); |
| 2678 | cir::GetGlobalOp createGetCpuFeatures2(mlir::Location loc); |
| 2679 | mlir::Value emitX86CpuIs(const CallExpr *expr); |
| 2680 | mlir::Value emitX86CpuIs(mlir::Location loc, StringRef cpuStr); |
| 2681 | mlir::Value emitX86CpuSupports(const CallExpr *expr); |
| 2682 | mlir::Value emitX86CpuSupports(mlir::Location loc, |
| 2683 | ArrayRef<StringRef> FeatureStrs); |
| 2684 | mlir::Value emitX86CpuSupports(mlir::Location loc, |
| 2685 | std::array<uint32_t, 4> FeatureMask); |
| 2686 | mlir::Value emitX86CpuInit(mlir::Location loc); |
| 2687 | using MSVCIntrin = CodeGenUtils::MSVCIntrin; |
| 2688 | mlir::Value emitX86MSVCIntrin(MSVCIntrin builtinID, const CallExpr *E); |
| 2689 | std::optional<mlir::Value> emitX86BuiltinExpr(unsigned builtinID, |
| 2690 | const CallExpr *expr); |
| 2691 | |
| 2692 | /// Given an assignment `*lhs = rhs`, emit a test that checks if \p rhs is |
| 2693 | /// nonnull, if 1\p LHS is marked _Nonnull. |
| 2694 | void emitNullabilityCheck(LValue lhs, mlir::Value rhs, |
| 2695 | clang::SourceLocation loc); |
| 2696 | |
| 2697 | /// An object to manage conditionally-evaluated expressions. |
| 2698 | class ConditionalEvaluation { |
| 2699 | CIRGenFunction &cgf; |
| 2700 | |
| 2701 | /// The insertion point that precedes the conditional, stored as the |
| 2702 | /// enclosing block and the operation immediately before that point. Later |
| 2703 | /// operations (the condition, cleanup scopes) can be appended without |
| 2704 | /// moving this point. A null \c anchorAfter means the insertion point is |
| 2705 | /// the start of \c anchorBlock. |
| 2706 | mlir::Block *anchorBlock; |
| 2707 | mlir::Operation *anchorAfter = nullptr; |
| 2708 | |
| 2709 | public: |
| 2710 | /// \p loc is the location of the conditional expression. It is used for |
| 2711 | /// the cleanup scope this may open. |
| 2712 | ConditionalEvaluation(CIRGenFunction &cgf, mlir::Location loc) : cgf(cgf) { |
| 2713 | // Open the cleanup scope that hosts cleanups deferred from inside this |
| 2714 | // conditional (see deferredConditionalCleanupStack). Only the outermost |
| 2715 | // conditional opens one, and only while a FullExprCleanupScope is |
| 2716 | // active to close it. When nothing is deferred into it the cleanup |
| 2717 | // region stays trivial and canonicalization inlines the scope away. |
| 2718 | if (cgf.currentFullExprCleanupScope && !cgf.isInConditionalBranch()) { |
| 2719 | cir::CleanupKind cleanupKind = cgf.getLangOpts().Exceptions |
| 2720 | ? cir::CleanupKind::All |
| 2721 | : cir::CleanupKind::Normal; |
| 2722 | cir::CleanupScopeOp scope = cir::CleanupScopeOp::create( |
| 2723 | builder&: cgf.builder, location: loc, cleanupKind, |
| 2724 | /*bodyBuilder=*/bodyBuilder: [](mlir::OpBuilder &, mlir::Location) {}, |
| 2725 | /*cleanupBuilder=*/cleanupBuilder: [](mlir::OpBuilder &, mlir::Location) {}); |
| 2726 | cgf.conditionalCleanupScopes.push_back( |
| 2727 | Elt: {.scope: scope, .deferredCleanupStackSize: cgf.deferredConditionalCleanupStack.size()}); |
| 2728 | cgf.builder.setInsertionPointToEnd(&scope.getBodyRegion().front()); |
| 2729 | } |
| 2730 | |
| 2731 | anchorBlock = cgf.builder.getInsertionBlock(); |
| 2732 | assert(anchorBlock && "conditional evaluation needs an insertion point" ); |
| 2733 | mlir::Block::iterator ip = cgf.builder.getInsertionPoint(); |
| 2734 | if (ip != anchorBlock->begin()) |
| 2735 | anchorAfter = &*std::prev(x: ip); |
| 2736 | } |
| 2737 | |
| 2738 | void beginEvaluation() { |
| 2739 | assert(cgf.outermostConditional != this); |
| 2740 | if (!cgf.outermostConditional) |
| 2741 | cgf.outermostConditional = this; |
| 2742 | } |
| 2743 | |
| 2744 | void endEvaluation() { |
| 2745 | assert(cgf.outermostConditional != nullptr); |
| 2746 | if (cgf.outermostConditional == this) |
| 2747 | cgf.outermostConditional = nullptr; |
| 2748 | } |
| 2749 | |
| 2750 | /// Records \p op as the last operation emitted at the pre-conditional |
| 2751 | /// insertion point, so that a later emission lands after it rather than |
| 2752 | /// ahead of it. |
| 2753 | void advanceInsertPoint(mlir::Operation *op) { anchorAfter = op; } |
| 2754 | |
| 2755 | /// Returns the insertion point which will be executed prior to each |
| 2756 | /// evaluation of the conditional code. In LLVM OG, this method |
| 2757 | /// is called getStartingBlock. |
| 2758 | mlir::OpBuilder::InsertPoint getInsertPoint() const { |
| 2759 | if (!anchorAfter) |
| 2760 | return mlir::OpBuilder::InsertPoint(anchorBlock, anchorBlock->begin()); |
| 2761 | return mlir::OpBuilder::InsertPoint( |
| 2762 | anchorAfter->getBlock(), std::next(x: anchorAfter->getIterator())); |
| 2763 | } |
| 2764 | }; |
| 2765 | |
| 2766 | struct ConditionalInfo { |
| 2767 | std::optional<LValue> lhs{}, rhs{}; |
| 2768 | mlir::Value result{}; |
| 2769 | }; |
| 2770 | |
| 2771 | // Return true if we're currently emitting one branch or the other of a |
| 2772 | // conditional expression. |
| 2773 | bool isInConditionalBranch() const { return outermostConditional != nullptr; } |
| 2774 | |
| 2775 | void setBeforeOutermostConditional(mlir::Value value, Address addr) { |
| 2776 | assert(isInConditionalBranch()); |
| 2777 | { |
| 2778 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 2779 | builder.restoreInsertionPoint(ip: outermostConditional->getInsertPoint()); |
| 2780 | cir::StoreOp store = builder.createStore( |
| 2781 | loc: value.getLoc(), val: value, dst: addr, /*isVolatile=*/isVolatile: false, |
| 2782 | /*isNontemporal=*/isNontemporal: false, |
| 2783 | align: mlir::IntegerAttr::get( |
| 2784 | type: mlir::IntegerType::get(context: value.getContext(), width: 64), |
| 2785 | value: (uint64_t)addr.getAlignment().getAsAlign().value())); |
| 2786 | outermostConditional->advanceInsertPoint(op: store); |
| 2787 | } |
| 2788 | } |
| 2789 | |
| 2790 | // Points to the outermost active conditional control. This is used so that |
| 2791 | // we know if a temporary should be destroyed conditionally. |
| 2792 | ConditionalEvaluation *outermostConditional = nullptr; |
| 2793 | |
| 2794 | // The outermost conditional that a FullExprCleanupScope in a conditional |
| 2795 | // branch hides from isInConditionalBranch(), if any. The cleanups of that |
| 2796 | // full-expression run at its end, so they are not conditional, but a |
| 2797 | // temporary whose lifetime is extended beyond it is still created |
| 2798 | // conditionally. |
| 2799 | ConditionalEvaluation *hiddenOutermostConditional = nullptr; |
| 2800 | |
| 2801 | /// An RAII object to record that we're evaluating a statement |
| 2802 | /// expression. |
| 2803 | class StmtExprEvaluation { |
| 2804 | CIRGenFunction &cgf; |
| 2805 | |
| 2806 | /// We have to save the outermost conditional: cleanups in a |
| 2807 | /// statement expression aren't conditional just because the |
| 2808 | /// StmtExpr is. |
| 2809 | ConditionalEvaluation *savedOutermostConditional; |
| 2810 | ConditionalEvaluation *savedHiddenOutermostConditional; |
| 2811 | |
| 2812 | public: |
| 2813 | StmtExprEvaluation(CIRGenFunction &cgf) |
| 2814 | : cgf(cgf), savedOutermostConditional(cgf.outermostConditional), |
| 2815 | savedHiddenOutermostConditional(cgf.hiddenOutermostConditional) { |
| 2816 | cgf.outermostConditional = nullptr; |
| 2817 | cgf.hiddenOutermostConditional = nullptr; |
| 2818 | } |
| 2819 | |
| 2820 | ~StmtExprEvaluation() { |
| 2821 | cgf.outermostConditional = savedOutermostConditional; |
| 2822 | cgf.hiddenOutermostConditional = savedHiddenOutermostConditional; |
| 2823 | } |
| 2824 | }; |
| 2825 | |
| 2826 | template <typename FuncTy> |
| 2827 | ConditionalInfo emitConditionalBlocks(const AbstractConditionalOperator *e, |
| 2828 | const FuncTy &branchGenFunc); |
| 2829 | |
| 2830 | /// Build a "reference" to a va_list; this is either the address or the value |
| 2831 | /// of the expression, depending on how va_list is defined. |
| 2832 | Address emitVAListRef(const Expr *e); |
| 2833 | |
| 2834 | /// Emits the start of a CIR variable-argument operation (`cir.va_start`) |
| 2835 | /// |
| 2836 | /// \param vaList A reference to the \c va_list as emitted by either |
| 2837 | /// \c emitVAListRef or \c emitMSVAListRef. |
| 2838 | void emitVAStart(mlir::Value vaList); |
| 2839 | |
| 2840 | /// Emits the end of a CIR variable-argument operation (`cir.va_start`) |
| 2841 | /// |
| 2842 | /// \param vaList A reference to the \c va_list as emitted by either |
| 2843 | /// \c emitVAListRef or \c emitMSVAListRef. |
| 2844 | void emitVAEnd(mlir::Value vaList); |
| 2845 | |
| 2846 | /// Generate code to get an argument from the passed in pointer |
| 2847 | /// and update it accordingly. |
| 2848 | /// |
| 2849 | /// \param ve The \c VAArgExpr for which to generate code. |
| 2850 | /// |
| 2851 | /// \param vaListAddr Receives a reference to the \c va_list as emitted by |
| 2852 | /// either \c emitVAListRef or \c emitMSVAListRef. |
| 2853 | /// |
| 2854 | /// \returns SSA value with the argument. |
| 2855 | mlir::Value emitVAArg(VAArgExpr *ve); |
| 2856 | |
| 2857 | /// ---------------------- |
| 2858 | /// CIR build helpers |
| 2859 | /// ----------------- |
| 2860 | public: |
| 2861 | cir::AllocaOp createTempAlloca(mlir::Type ty, mlir::Location loc, |
| 2862 | const Twine &name = "tmp" , |
| 2863 | mlir::Value arraySize = nullptr, |
| 2864 | bool insertIntoFnEntryBlock = false); |
| 2865 | cir::AllocaOp createTempAlloca(mlir::Type ty, mlir::Location loc, |
| 2866 | const Twine &name = "tmp" , |
| 2867 | mlir::OpBuilder::InsertPoint ip = {}, |
| 2868 | mlir::Value arraySize = nullptr); |
| 2869 | Address createTempAlloca(mlir::Type ty, CharUnits align, mlir::Location loc, |
| 2870 | const Twine &name = "tmp" , |
| 2871 | mlir::Value arraySize = nullptr, |
| 2872 | Address *alloca = nullptr, |
| 2873 | mlir::OpBuilder::InsertPoint ip = {}); |
| 2874 | Address createTempAlloca(mlir::Type ty, |
| 2875 | mlir::ptr::MemorySpaceAttrInterface destAddrSpace, |
| 2876 | CharUnits align, mlir::Location loc, |
| 2877 | const Twine &name = "tmp" , |
| 2878 | mlir::Value arraySize = nullptr, |
| 2879 | Address *alloca = nullptr, |
| 2880 | mlir::OpBuilder::InsertPoint ip = {}); |
| 2881 | Address createTempAllocaWithoutCast(mlir::Type ty, CharUnits align, |
| 2882 | mlir::Location loc, |
| 2883 | const Twine &name = "tmp" , |
| 2884 | mlir::Value arraySize = nullptr, |
| 2885 | mlir::OpBuilder::InsertPoint ip = {}); |
| 2886 | Address |
| 2887 | maybeCastStackAddressSpace(Address alloca, |
| 2888 | mlir::ptr::MemorySpaceAttrInterface destAddrSpace, |
| 2889 | mlir::Value arraySize = nullptr); |
| 2890 | Address createDefaultAlignTempAlloca(mlir::Type ty, mlir::Location loc, |
| 2891 | const Twine &name); |
| 2892 | |
| 2893 | /// Create a temporary memory object of the given type, with |
| 2894 | /// appropriate alignmen and cast it to the default address space. Returns |
| 2895 | /// the original alloca instruction by \p Alloca if it is not nullptr. |
| 2896 | Address createMemTemp(QualType t, mlir::Location loc, |
| 2897 | const Twine &name = "tmp" , Address *alloca = nullptr, |
| 2898 | mlir::OpBuilder::InsertPoint ip = {}); |
| 2899 | Address createMemTemp(QualType t, CharUnits align, mlir::Location loc, |
| 2900 | const Twine &name = "tmp" , Address *alloca = nullptr, |
| 2901 | mlir::OpBuilder::InsertPoint ip = {}); |
| 2902 | Address createMemTempWithoutCast(QualType t, mlir::Location loc, |
| 2903 | const Twine &name = "tmp" ); |
| 2904 | |
| 2905 | mlir::Value performAddrSpaceCast(mlir::Value v, mlir::Type destTy) const { |
| 2906 | if (cir::GlobalOp globalOp = v.getDefiningOp<cir::GlobalOp>()) |
| 2907 | cgm.errorNYI(feature: "Global op addrspace cast" ); |
| 2908 | return builder.createAddrSpaceCast(src: v, newTy: destTy); |
| 2909 | } |
| 2910 | |
| 2911 | //===--------------------------------------------------------------------===// |
| 2912 | // OpenMP Emission |
| 2913 | //===--------------------------------------------------------------------===// |
| 2914 | public: |
| 2915 | /// Emits a Sema-synthesized OpenMP helper variable (loop counter, iteration |
| 2916 | /// variable or pre-init capture) without lifetime markers. These variables |
| 2917 | /// are emitted between nested combined/composite OpenMP ops, which must be |
| 2918 | /// direct children of each other's regions, so they must not open a |
| 2919 | /// lifetime cleanup scope. |
| 2920 | void emitOMPNoLifetimeVarDecl(const VarDecl &d); |
| 2921 | |
| 2922 | mlir::LogicalResult emitOMPScopeDirective(const OMPScopeDirective &s); |
| 2923 | mlir::LogicalResult emitOMPErrorDirective(const OMPErrorDirective &s); |
| 2924 | mlir::LogicalResult emitOMPParallelDirective(const OMPParallelDirective &s); |
| 2925 | mlir::LogicalResult emitOMPTaskwaitDirective(const OMPTaskwaitDirective &s); |
| 2926 | mlir::LogicalResult emitOMPTaskyieldDirective(const OMPTaskyieldDirective &s); |
| 2927 | mlir::LogicalResult emitOMPBarrierDirective(const OMPBarrierDirective &s); |
| 2928 | mlir::LogicalResult emitOMPMetaDirective(const OMPMetaDirective &s); |
| 2929 | mlir::LogicalResult emitOMPCanonicalLoop(const OMPCanonicalLoop &s); |
| 2930 | mlir::LogicalResult emitOMPSimdDirective(const OMPSimdDirective &s); |
| 2931 | mlir::LogicalResult emitOMPTileDirective(const OMPTileDirective &s); |
| 2932 | mlir::LogicalResult emitOMPUnrollDirective(const OMPUnrollDirective &s); |
| 2933 | mlir::LogicalResult emitOMPFuseDirective(const OMPFuseDirective &s); |
| 2934 | mlir::LogicalResult emitOMPForDirective(const OMPForDirective &s); |
| 2935 | mlir::LogicalResult emitOMPForSimdDirective(const OMPForSimdDirective &s); |
| 2936 | mlir::LogicalResult emitOMPSectionsDirective(const OMPSectionsDirective &s); |
| 2937 | mlir::LogicalResult emitOMPSectionDirective(const OMPSectionDirective &s); |
| 2938 | mlir::LogicalResult emitOMPSingleDirective(const OMPSingleDirective &s); |
| 2939 | mlir::LogicalResult emitOMPMasterDirective(const OMPMasterDirective &s); |
| 2940 | mlir::LogicalResult emitOMPCriticalDirective(const OMPCriticalDirective &s); |
| 2941 | mlir::LogicalResult |
| 2942 | emitOMPParallelForDirective(const OMPParallelForDirective &s); |
| 2943 | mlir::LogicalResult |
| 2944 | emitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &s); |
| 2945 | mlir::LogicalResult |
| 2946 | emitOMPParallelMasterDirective(const OMPParallelMasterDirective &s); |
| 2947 | mlir::LogicalResult |
| 2948 | emitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &s); |
| 2949 | mlir::LogicalResult emitOMPTaskDirective(const OMPTaskDirective &s); |
| 2950 | mlir::LogicalResult emitOMPTaskgroupDirective(const OMPTaskgroupDirective &s); |
| 2951 | mlir::LogicalResult emitOMPFlushDirective(const OMPFlushDirective &s); |
| 2952 | mlir::LogicalResult emitOMPDepobjDirective(const OMPDepobjDirective &s); |
| 2953 | mlir::LogicalResult emitOMPScanDirective(const OMPScanDirective &s); |
| 2954 | mlir::LogicalResult |
| 2955 | emitOMPOrderedStandaloneDirective(const OMPOrderedStandaloneDirective &s); |
| 2956 | mlir::LogicalResult |
| 2957 | emitOMPOrderedBlockAssocDirective(const OMPOrderedBlockAssocDirective &s); |
| 2958 | mlir::LogicalResult emitOMPAtomicDirective(const OMPAtomicDirective &s); |
| 2959 | mlir::LogicalResult emitOMPTargetDirective(const OMPTargetDirective &s); |
| 2960 | mlir::LogicalResult emitOMPTeamsDirective(const OMPTeamsDirective &s); |
| 2961 | mlir::LogicalResult |
| 2962 | emitOMPCancellationPointDirective(const OMPCancellationPointDirective &s); |
| 2963 | mlir::LogicalResult emitOMPCancelDirective(const OMPCancelDirective &s); |
| 2964 | mlir::LogicalResult |
| 2965 | emitOMPTargetDataDirective(const OMPTargetDataDirective &s); |
| 2966 | mlir::LogicalResult |
| 2967 | emitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &s); |
| 2968 | mlir::LogicalResult |
| 2969 | emitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &s); |
| 2970 | mlir::LogicalResult |
| 2971 | emitOMPTargetParallelDirective(const OMPTargetParallelDirective &s); |
| 2972 | mlir::LogicalResult |
| 2973 | emitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &s); |
| 2974 | mlir::LogicalResult emitOMPTaskLoopDirective(const OMPTaskLoopDirective &s); |
| 2975 | mlir::LogicalResult |
| 2976 | emitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &s); |
| 2977 | mlir::LogicalResult |
| 2978 | emitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &s); |
| 2979 | mlir::LogicalResult |
| 2980 | emitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &s); |
| 2981 | mlir::LogicalResult |
| 2982 | emitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &s); |
| 2983 | mlir::LogicalResult |
| 2984 | emitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &s); |
| 2985 | mlir::LogicalResult |
| 2986 | emitOMPParallelGenericLoopDirective(const OMPParallelGenericLoopDirective &s); |
| 2987 | mlir::LogicalResult |
| 2988 | emitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &s); |
| 2989 | mlir::LogicalResult emitOMPParallelMaskedTaskLoopDirective( |
| 2990 | const OMPParallelMaskedTaskLoopDirective &s); |
| 2991 | mlir::LogicalResult emitOMPParallelMaskedTaskLoopSimdDirective( |
| 2992 | const OMPParallelMaskedTaskLoopSimdDirective &s); |
| 2993 | mlir::LogicalResult emitOMPParallelMasterTaskLoopDirective( |
| 2994 | const OMPParallelMasterTaskLoopDirective &s); |
| 2995 | mlir::LogicalResult emitOMPParallelMasterTaskLoopSimdDirective( |
| 2996 | const OMPParallelMasterTaskLoopSimdDirective &s); |
| 2997 | mlir::LogicalResult |
| 2998 | emitOMPDistributeDirective(const OMPDistributeDirective &s); |
| 2999 | mlir::LogicalResult emitOMPDistributeParallelForDirective( |
| 3000 | const OMPDistributeParallelForDirective &s); |
| 3001 | mlir::LogicalResult emitOMPDistributeParallelForSimdDirective( |
| 3002 | const OMPDistributeParallelForSimdDirective &s); |
| 3003 | mlir::LogicalResult |
| 3004 | emitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &s); |
| 3005 | mlir::LogicalResult emitOMPTargetParallelGenericLoopDirective( |
| 3006 | const OMPTargetParallelGenericLoopDirective &s); |
| 3007 | mlir::LogicalResult |
| 3008 | emitOMPTargetGenericLoopDirective(const OMPTargetGenericLoopDirective &s); |
| 3009 | mlir::LogicalResult emitOMPTargetParallelForSimdDirective( |
| 3010 | const OMPTargetParallelForSimdDirective &s); |
| 3011 | mlir::LogicalResult |
| 3012 | emitOMPTargetSimdDirective(const OMPTargetSimdDirective &s); |
| 3013 | mlir::LogicalResult emitOMPTargetTeamsGenericLoopDirective( |
| 3014 | const OMPTargetTeamsGenericLoopDirective &s); |
| 3015 | mlir::LogicalResult |
| 3016 | emitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &s); |
| 3017 | mlir::LogicalResult |
| 3018 | emitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &s); |
| 3019 | mlir::LogicalResult |
| 3020 | emitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &s); |
| 3021 | mlir::LogicalResult emitOMPTeamsDistributeParallelForSimdDirective( |
| 3022 | const OMPTeamsDistributeParallelForSimdDirective &s); |
| 3023 | mlir::LogicalResult emitOMPTeamsDistributeParallelForDirective( |
| 3024 | const OMPTeamsDistributeParallelForDirective &s); |
| 3025 | mlir::LogicalResult |
| 3026 | emitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &s); |
| 3027 | mlir::LogicalResult |
| 3028 | emitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &s); |
| 3029 | mlir::LogicalResult emitOMPTargetTeamsDistributeDirective( |
| 3030 | const OMPTargetTeamsDistributeDirective &s); |
| 3031 | mlir::LogicalResult emitOMPTargetTeamsDistributeParallelForDirective( |
| 3032 | const OMPTargetTeamsDistributeParallelForDirective &s); |
| 3033 | mlir::LogicalResult emitOMPTargetTeamsDistributeParallelForSimdDirective( |
| 3034 | const OMPTargetTeamsDistributeParallelForSimdDirective &s); |
| 3035 | mlir::LogicalResult emitOMPTargetTeamsDistributeSimdDirective( |
| 3036 | const OMPTargetTeamsDistributeSimdDirective &s); |
| 3037 | mlir::LogicalResult emitOMPInteropDirective(const OMPInteropDirective &s); |
| 3038 | mlir::LogicalResult emitOMPDispatchDirective(const OMPDispatchDirective &s); |
| 3039 | mlir::LogicalResult |
| 3040 | emitOMPGenericLoopDirective(const OMPGenericLoopDirective &s); |
| 3041 | mlir::LogicalResult emitOMPReverseDirective(const OMPReverseDirective &s); |
| 3042 | mlir::LogicalResult emitOMPSplitDirective(const OMPSplitDirective &s); |
| 3043 | mlir::LogicalResult |
| 3044 | emitOMPInterchangeDirective(const OMPInterchangeDirective &s); |
| 3045 | mlir::LogicalResult emitOMPFlattenDirective(const OMPFlattenDirective &s); |
| 3046 | mlir::LogicalResult emitOMPAssumeDirective(const OMPAssumeDirective &s); |
| 3047 | mlir::LogicalResult emitOMPMaskedDirective(const OMPMaskedDirective &s); |
| 3048 | mlir::LogicalResult emitOMPStripeDirective(const OMPStripeDirective &s); |
| 3049 | |
| 3050 | void emitOMPThreadPrivateDecl(const OMPThreadPrivateDecl &d); |
| 3051 | void emitOMPGroupPrivateDecl(const OMPGroupPrivateDecl &d); |
| 3052 | void emitOMPCapturedExpr(const OMPCapturedExprDecl &d); |
| 3053 | void emitOMPAllocateDecl(const OMPAllocateDecl &d); |
| 3054 | void emitOMPDeclareReduction(const OMPDeclareReductionDecl &d); |
| 3055 | void emitOMPDeclareMapper(const OMPDeclareMapperDecl &d); |
| 3056 | void emitOMPRequiresDecl(const OMPRequiresDecl &d); |
| 3057 | |
| 3058 | //===--------------------------------------------------------------------===// |
| 3059 | // OpenACC Emission |
| 3060 | //===--------------------------------------------------------------------===// |
| 3061 | private: |
| 3062 | template <typename Op> |
| 3063 | Op emitOpenACCOp(mlir::Location start, OpenACCDirectiveKind dirKind, |
| 3064 | llvm::ArrayRef<const OpenACCClause *> clauses); |
| 3065 | // Function to do the basic implementation of an operation with an Associated |
| 3066 | // Statement. Models AssociatedStmtConstruct. |
| 3067 | template <typename Op, typename TermOp> |
| 3068 | mlir::LogicalResult |
| 3069 | emitOpenACCOpAssociatedStmt(mlir::Location start, mlir::Location end, |
| 3070 | OpenACCDirectiveKind dirKind, |
| 3071 | llvm::ArrayRef<const OpenACCClause *> clauses, |
| 3072 | const Stmt *associatedStmt); |
| 3073 | |
| 3074 | template <typename Op, typename TermOp> |
| 3075 | mlir::LogicalResult emitOpenACCOpCombinedConstruct( |
| 3076 | mlir::Location start, mlir::Location end, OpenACCDirectiveKind dirKind, |
| 3077 | llvm::ArrayRef<const OpenACCClause *> clauses, const Stmt *loopStmt); |
| 3078 | |
| 3079 | template <typename Op> |
| 3080 | void emitOpenACCClauses(Op &op, OpenACCDirectiveKind dirKind, |
| 3081 | ArrayRef<const OpenACCClause *> clauses); |
| 3082 | // The second template argument doesn't need to be a template, since it should |
| 3083 | // always be an mlir::acc::LoopOp, but as this is a template anyway, we make |
| 3084 | // it a template argument as this way we can avoid including the OpenACC MLIR |
| 3085 | // headers here. We will count on linker failures/explicit instantiation to |
| 3086 | // ensure we don't mess this up, but it is only called from 1 place, and |
| 3087 | // instantiated 3x. |
| 3088 | template <typename ComputeOp, typename LoopOp> |
| 3089 | void emitOpenACCClauses(ComputeOp &op, LoopOp &loopOp, |
| 3090 | OpenACCDirectiveKind dirKind, |
| 3091 | ArrayRef<const OpenACCClause *> clauses); |
| 3092 | |
| 3093 | // The OpenACC LoopOp requires that we have auto, seq, or independent on all |
| 3094 | // LoopOp operations for the 'none' device type case. This function checks if |
| 3095 | // the LoopOp has one, else it updates it to have one. |
| 3096 | void updateLoopOpParallelism(mlir::acc::LoopOp &op, bool isOrphan, |
| 3097 | OpenACCDirectiveKind dk); |
| 3098 | |
| 3099 | // The OpenACC 'cache' construct actually applies to the 'loop' if present. So |
| 3100 | // keep track of the 'loop' so that we can add the cache vars to it correctly. |
| 3101 | mlir::acc::LoopOp *activeLoopOp = nullptr; |
| 3102 | |
| 3103 | struct ActiveOpenACCLoopRAII { |
| 3104 | CIRGenFunction &cgf; |
| 3105 | mlir::acc::LoopOp *oldLoopOp; |
| 3106 | |
| 3107 | ActiveOpenACCLoopRAII(CIRGenFunction &cgf, mlir::acc::LoopOp *newOp) |
| 3108 | : cgf(cgf), oldLoopOp(cgf.activeLoopOp) { |
| 3109 | cgf.activeLoopOp = newOp; |
| 3110 | } |
| 3111 | ~ActiveOpenACCLoopRAII() { cgf.activeLoopOp = oldLoopOp; } |
| 3112 | }; |
| 3113 | |
| 3114 | // Keep track of the last place we inserted a 'recipe' so that we can insert |
| 3115 | // the next one in lexical order. |
| 3116 | mlir::OpBuilder::InsertPoint lastRecipeLocation; |
| 3117 | |
| 3118 | public: |
| 3119 | // Helper type used to store the list of important information for a 'data' |
| 3120 | // clause variable, or a 'cache' variable reference. |
| 3121 | struct OpenACCDataOperandInfo { |
| 3122 | mlir::Location beginLoc; |
| 3123 | mlir::Value varValue; |
| 3124 | std::string name; |
| 3125 | // The type of the original variable reference: that is, after 'bounds' have |
| 3126 | // removed pointers/array types/etc. So in the case of int arr[5], and a |
| 3127 | // private(arr[1]), 'origType' is 'int', but 'baseType' is 'int[5]'. |
| 3128 | QualType origType; |
| 3129 | QualType baseType; |
| 3130 | llvm::SmallVector<mlir::Value> bounds; |
| 3131 | // The list of types that we found when going through the bounds, which we |
| 3132 | // can use to properly set the alloca section. |
| 3133 | llvm::SmallVector<QualType> boundTypes; |
| 3134 | }; |
| 3135 | |
| 3136 | // Gets the collection of info required to lower and OpenACC clause or cache |
| 3137 | // construct variable reference. |
| 3138 | OpenACCDataOperandInfo getOpenACCDataOperandInfo(const Expr *e); |
| 3139 | // Helper function to emit the integer expressions as required by an OpenACC |
| 3140 | // clause/construct. |
| 3141 | mlir::Value emitOpenACCIntExpr(const Expr *intExpr); |
| 3142 | // Helper function to emit an integer constant as an mlir int type, used for |
| 3143 | // constants in OpenACC constructs/clauses. |
| 3144 | mlir::Value createOpenACCConstantInt(mlir::Location loc, unsigned width, |
| 3145 | int64_t value); |
| 3146 | |
| 3147 | mlir::LogicalResult |
| 3148 | emitOpenACCComputeConstruct(const OpenACCComputeConstruct &s); |
| 3149 | mlir::LogicalResult emitOpenACCLoopConstruct(const OpenACCLoopConstruct &s); |
| 3150 | mlir::LogicalResult |
| 3151 | emitOpenACCCombinedConstruct(const OpenACCCombinedConstruct &s); |
| 3152 | mlir::LogicalResult emitOpenACCDataConstruct(const OpenACCDataConstruct &s); |
| 3153 | mlir::LogicalResult |
| 3154 | emitOpenACCEnterDataConstruct(const OpenACCEnterDataConstruct &s); |
| 3155 | mlir::LogicalResult |
| 3156 | emitOpenACCExitDataConstruct(const OpenACCExitDataConstruct &s); |
| 3157 | mlir::LogicalResult |
| 3158 | emitOpenACCHostDataConstruct(const OpenACCHostDataConstruct &s); |
| 3159 | mlir::LogicalResult emitOpenACCWaitConstruct(const OpenACCWaitConstruct &s); |
| 3160 | mlir::LogicalResult emitOpenACCInitConstruct(const OpenACCInitConstruct &s); |
| 3161 | mlir::LogicalResult |
| 3162 | emitOpenACCShutdownConstruct(const OpenACCShutdownConstruct &s); |
| 3163 | mlir::LogicalResult emitOpenACCSetConstruct(const OpenACCSetConstruct &s); |
| 3164 | mlir::LogicalResult |
| 3165 | emitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &s); |
| 3166 | mlir::LogicalResult |
| 3167 | emitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &s); |
| 3168 | mlir::LogicalResult emitOpenACCCacheConstruct(const OpenACCCacheConstruct &s); |
| 3169 | |
| 3170 | void emitOpenACCDeclare(const OpenACCDeclareDecl &d); |
| 3171 | void emitOpenACCRoutine(const OpenACCRoutineDecl &d); |
| 3172 | |
| 3173 | /// Create a temporary memory object for the given aggregate type. |
| 3174 | AggValueSlot createAggTemp(QualType ty, mlir::Location loc, |
| 3175 | const Twine &name = "tmp" , |
| 3176 | Address *alloca = nullptr) { |
| 3177 | assert(!cir::MissingFeatures::aggValueSlot()); |
| 3178 | return AggValueSlot::forAddr( |
| 3179 | addr: createMemTemp(t: ty, loc, name, alloca), quals: ty.getQualifiers(), |
| 3180 | isDestructed: AggValueSlot::IsNotDestructed, isAliased: AggValueSlot::IsNotAliased, |
| 3181 | mayOverlap: AggValueSlot::DoesNotOverlap); |
| 3182 | } |
| 3183 | |
| 3184 | private: |
| 3185 | QualType getVarArgType(const Expr *arg); |
| 3186 | |
| 3187 | bool shouldEmitLifetimeMarkers = false; |
| 3188 | /// Set when the current function has a goto/switch that may bypass a local's |
| 3189 | /// init; lifetime markers are then suppressed. See functionMightHaveBypass. |
| 3190 | bool fnHasBypassStmt = false; |
| 3191 | |
| 3192 | bool shouldEmitLifetimeMarkersForAutoVar() const { |
| 3193 | return shouldEmitLifetimeMarkers && !fnHasBypassStmt; |
| 3194 | } |
| 3195 | |
| 3196 | class InlinedInheritingConstructorScope { |
| 3197 | public: |
| 3198 | InlinedInheritingConstructorScope(CIRGenFunction &cgf, GlobalDecl gd) |
| 3199 | : cgf(cgf), oldCurGD(cgf.curGD), oldCurFuncDecl(cgf.curFuncDecl), |
| 3200 | oldCurCodeDecl(cgf.curCodeDecl), |
| 3201 | oldCxxabiThisDecl(cgf.cxxabiThisDecl), |
| 3202 | oldCxxThisValue(cgf.cxxThisValue), |
| 3203 | oldCxxabiThisAlignment(cgf.cxxabiThisAlignment), |
| 3204 | oldCxxThisAlignment(cgf.cxxThisAlignment), |
| 3205 | oldReturnValue(cgf.returnValue), oldFnRetTy(cgf.fnRetTy), |
| 3206 | oldCxxInheritedCtorInitExprArgs( |
| 3207 | std::move(cgf.cxxInheritedCtorInitExprArgs)) { |
| 3208 | cgf.curGD = gd; |
| 3209 | cgf.curFuncDecl = cast<CXXConstructorDecl>(Val: gd.getDecl()); |
| 3210 | cgf.curCodeDecl = cgf.curFuncDecl; |
| 3211 | cgf.cxxabiThisDecl = nullptr; |
| 3212 | cgf.cxxabiThisValue = nullptr; |
| 3213 | cgf.cxxThisValue = nullptr; |
| 3214 | cgf.cxxThisAlignment = CharUnits(); |
| 3215 | cgf.cxxabiThisAlignment = CharUnits(); |
| 3216 | cgf.returnValue = Address::invalid(); |
| 3217 | cgf.fnRetTy = QualType(); |
| 3218 | cgf.cxxInheritedCtorInitExprArgs.clear(); |
| 3219 | // FIXME: at one point when we want to call one of these, we'll need |
| 3220 | // CXXInheritedCtorInitExprArgs here too. |
| 3221 | } |
| 3222 | ~InlinedInheritingConstructorScope() { |
| 3223 | cgf.curGD = oldCurGD; |
| 3224 | cgf.curFuncDecl = oldCurFuncDecl; |
| 3225 | cgf.curCodeDecl = oldCurCodeDecl; |
| 3226 | cgf.cxxabiThisDecl = oldCxxabiThisDecl; |
| 3227 | cgf.cxxabiThisValue = oldCxxabiThisValue; |
| 3228 | cgf.cxxThisValue = oldCxxThisValue; |
| 3229 | cgf.cxxThisAlignment = oldCxxThisAlignment; |
| 3230 | cgf.cxxabiThisAlignment = oldCxxabiThisAlignment; |
| 3231 | cgf.returnValue = oldReturnValue; |
| 3232 | cgf.fnRetTy = oldFnRetTy; |
| 3233 | cgf.cxxInheritedCtorInitExprArgs = |
| 3234 | std::move(oldCxxInheritedCtorInitExprArgs); |
| 3235 | } |
| 3236 | |
| 3237 | private: |
| 3238 | CIRGenFunction &cgf; |
| 3239 | GlobalDecl oldCurGD; |
| 3240 | const Decl *oldCurFuncDecl; |
| 3241 | const Decl *oldCurCodeDecl; |
| 3242 | ImplicitParamDecl *oldCxxabiThisDecl; |
| 3243 | mlir::Value oldCxxabiThisValue; |
| 3244 | mlir::Value oldCxxThisValue; |
| 3245 | clang::CharUnits oldCxxabiThisAlignment; |
| 3246 | clang::CharUnits oldCxxThisAlignment; |
| 3247 | Address oldReturnValue; |
| 3248 | QualType oldFnRetTy; |
| 3249 | CallArgList oldCxxInheritedCtorInitExprArgs; |
| 3250 | }; |
| 3251 | }; |
| 3252 | |
| 3253 | } // namespace clang::CIRGen |
| 3254 | |
| 3255 | #endif |
| 3256 | |