| 1 | //===- CIRDialect.cpp - MLIR CIR ops implementation -----------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements the CIR dialect and its operations. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "clang/CIR/Dialect/IR/CIRDialect.h" |
| 14 | |
| 15 | #include "clang/CIR/Dialect/IR/CIRAttrs.h" |
| 16 | #include "clang/CIR/Dialect/IR/CIRDataLayout.h" |
| 17 | #include "clang/CIR/Dialect/IR/CIROpsEnums.h" |
| 18 | #include "clang/CIR/Dialect/IR/CIRTypes.h" |
| 19 | |
| 20 | #include "mlir/IR/Attributes.h" |
| 21 | #include "mlir/IR/BuiltinOps.h" |
| 22 | #include "mlir/IR/BuiltinTypes.h" |
| 23 | #include "mlir/IR/DialectImplementation.h" |
| 24 | #include "mlir/IR/PatternMatch.h" |
| 25 | #include "mlir/IR/Value.h" |
| 26 | #include "mlir/Interfaces/ControlFlowInterfaces.h" |
| 27 | #include "mlir/Interfaces/FunctionImplementation.h" |
| 28 | #include "mlir/Support/LLVM.h" |
| 29 | |
| 30 | #include "clang/Basic/TargetCXXABI.h" |
| 31 | #include "clang/CIR/Dialect/IR/CIROpsDialect.cpp.inc" |
| 32 | #include "clang/CIR/Dialect/IR/CIROpsEnums.cpp.inc" |
| 33 | #include "clang/CIR/MissingFeatures.h" |
| 34 | #include "llvm/ADT/SetOperations.h" |
| 35 | #include "llvm/ADT/SmallSet.h" |
| 36 | #include "llvm/ADT/TypeSwitch.h" |
| 37 | #include "llvm/Support/Casting.h" |
| 38 | #include "llvm/Support/LogicalResult.h" |
| 39 | |
| 40 | #include "CIRDialectBytecode.h" |
| 41 | #include "CIROpDefinitions.h" |
| 42 | |
| 43 | using namespace mlir; |
| 44 | using namespace cir; |
| 45 | |
| 46 | //===----------------------------------------------------------------------===// |
| 47 | // CIR Dialect |
| 48 | //===----------------------------------------------------------------------===// |
| 49 | namespace { |
| 50 | struct CIROpAsmDialectInterface : public OpAsmDialectInterface { |
| 51 | using OpAsmDialectInterface::OpAsmDialectInterface; |
| 52 | |
| 53 | AliasResult getAlias(Type type, raw_ostream &os) const final { |
| 54 | if (auto recordType = dyn_cast<cir::RecordType>(Val&: type)) { |
| 55 | StringAttr nameAttr = recordType.getName(); |
| 56 | if (!nameAttr) |
| 57 | os << "rec_anon_" << recordType.getKindAsStr(); |
| 58 | else |
| 59 | os << "rec_" << nameAttr.getValue(); |
| 60 | return AliasResult::OverridableAlias; |
| 61 | } |
| 62 | if (auto intType = dyn_cast<cir::IntType>(Val&: type)) { |
| 63 | // We only provide alias for standard integer types (i.e. integer types |
| 64 | // whose width is a power of 2 and at least 8). |
| 65 | unsigned width = intType.getWidth(); |
| 66 | if (width < 8 || !llvm::isPowerOf2_32(Value: width)) |
| 67 | return AliasResult::NoAlias; |
| 68 | os << intType.getAlias(); |
| 69 | return AliasResult::OverridableAlias; |
| 70 | } |
| 71 | if (auto voidType = dyn_cast<cir::VoidType>(Val&: type)) { |
| 72 | os << voidType.getAlias(); |
| 73 | return AliasResult::OverridableAlias; |
| 74 | } |
| 75 | |
| 76 | return AliasResult::NoAlias; |
| 77 | } |
| 78 | |
| 79 | AliasResult getAlias(Attribute attr, raw_ostream &os) const final { |
| 80 | if (auto boolAttr = mlir::dyn_cast<cir::BoolAttr>(Val&: attr)) { |
| 81 | os << (boolAttr.getValue() ? "true" : "false" ); |
| 82 | return AliasResult::FinalAlias; |
| 83 | } |
| 84 | if (auto bitfield = mlir::dyn_cast<cir::BitfieldInfoAttr>(Val&: attr)) { |
| 85 | os << "bfi_" << bitfield.getName().str(); |
| 86 | return AliasResult::FinalAlias; |
| 87 | } |
| 88 | if (auto dynCastInfoAttr = mlir::dyn_cast<cir::DynamicCastInfoAttr>(Val&: attr)) { |
| 89 | os << dynCastInfoAttr.getAlias(); |
| 90 | return AliasResult::FinalAlias; |
| 91 | } |
| 92 | if (auto cmpThreeWayInfoAttr = |
| 93 | mlir::dyn_cast<cir::CmpThreeWayInfoAttr>(Val&: attr)) { |
| 94 | os << cmpThreeWayInfoAttr.getAlias(); |
| 95 | return AliasResult::FinalAlias; |
| 96 | } |
| 97 | return AliasResult::NoAlias; |
| 98 | } |
| 99 | }; |
| 100 | } // namespace |
| 101 | |
| 102 | void cir::CIRDialect::initialize() { |
| 103 | registerTypes(); |
| 104 | registerAttributes(); |
| 105 | registerCIRDialectOperations(dialect: this); |
| 106 | addInterfaces<CIROpAsmDialectInterface>(); |
| 107 | detail::addBytecodeInterface(dialect: this); |
| 108 | } |
| 109 | |
| 110 | Operation *cir::CIRDialect::materializeConstant(mlir::OpBuilder &builder, |
| 111 | mlir::Attribute value, |
| 112 | mlir::Type type, |
| 113 | mlir::Location loc) { |
| 114 | return cir::ConstantOp::create(builder, location: loc, res: type, |
| 115 | value: mlir::cast<mlir::TypedAttr>(Val&: value)); |
| 116 | } |
| 117 | |
| 118 | static bool isOpenCLVersionAttrName(StringRef attrName) { |
| 119 | return attrName == CIRDialect::getOpenCLVersionAttrName() || |
| 120 | attrName == CIRDialect::getOpenCLCXXVersionAttrName() || |
| 121 | attrName == CIRDialect::getOpenCLSPIRVersionAttrName(); |
| 122 | } |
| 123 | |
| 124 | static LogicalResult verifyOpenCLVersionAttrPlacement(Operation *op, |
| 125 | NamedAttribute attr) { |
| 126 | StringRef attrName = attr.getName().getValue(); |
| 127 | if (isa<ModuleOp>(Val: op)) |
| 128 | return success(); |
| 129 | |
| 130 | return op->emitError() << attrName |
| 131 | << " attribute must be attached to a module" ; |
| 132 | } |
| 133 | |
| 134 | static bool areOpenCLVersionsCompatible(cir::OpenCLVersionAttr openCLVersion, |
| 135 | cir::OpenCLVersionAttr cxxVersion) { |
| 136 | return (openCLVersion.getMajorVersion() == 2 && |
| 137 | openCLVersion.getMinorVersion() == 0 && |
| 138 | cxxVersion.getMajorVersion() == 1 && |
| 139 | cxxVersion.getMinorVersion() == 0) || |
| 140 | (openCLVersion.getMajorVersion() == 3 && |
| 141 | openCLVersion.getMinorVersion() == 0 && |
| 142 | cxxVersion.getMajorVersion() == 2021 && |
| 143 | cxxVersion.getMinorVersion() == 0); |
| 144 | } |
| 145 | |
| 146 | static LogicalResult verifyOpenCLCXXVersion(ModuleOp module, |
| 147 | cir::OpenCLVersionAttr cxxVersion) { |
| 148 | Attribute openCLAttr = |
| 149 | module->getAttr(name: CIRDialect::getOpenCLVersionAttrName()); |
| 150 | if (!openCLAttr) |
| 151 | return module.emitError() |
| 152 | << "module attribute '" << CIRDialect::getOpenCLCXXVersionAttrName() |
| 153 | << "' requires the companion attribute '" |
| 154 | << CIRDialect::getOpenCLVersionAttrName() << "'" ; |
| 155 | |
| 156 | auto openCLVersion = dyn_cast<cir::OpenCLVersionAttr>(Val&: openCLAttr); |
| 157 | if (!openCLVersion) |
| 158 | return success(); |
| 159 | |
| 160 | if (!areOpenCLVersionsCompatible(openCLVersion, cxxVersion)) |
| 161 | return module.emitError(message: "incompatible OpenCL and C++ for OpenCL versions" ); |
| 162 | |
| 163 | return success(); |
| 164 | } |
| 165 | |
| 166 | static LogicalResult verifyOpenCLVersionAttr(Operation *op, |
| 167 | NamedAttribute attr) { |
| 168 | if (failed(Result: verifyOpenCLVersionAttrPlacement(op, attr))) |
| 169 | return failure(); |
| 170 | |
| 171 | StringRef attrName = attr.getName().getValue(); |
| 172 | auto version = dyn_cast<cir::OpenCLVersionAttr>(Val: attr.getValue()); |
| 173 | if (!version) { |
| 174 | return op->emitError() << "expected " << attrName |
| 175 | << " to be #cir.cl.version" ; |
| 176 | } |
| 177 | |
| 178 | if (attrName == CIRDialect::getOpenCLCXXVersionAttrName()) |
| 179 | return verifyOpenCLCXXVersion(module: cast<ModuleOp>(Val: op), cxxVersion: version); |
| 180 | |
| 181 | return success(); |
| 182 | } |
| 183 | |
| 184 | LogicalResult cir::CIRDialect::verifyRegionArgAttribute( |
| 185 | Operation *op, unsigned /*regionIndex*/, unsigned /*argIndex*/, |
| 186 | NamedAttribute attr) { |
| 187 | if (!isOpenCLVersionAttrName(attrName: attr.getName().getValue())) |
| 188 | return success(); |
| 189 | |
| 190 | return verifyOpenCLVersionAttrPlacement(op, attr); |
| 191 | } |
| 192 | |
| 193 | LogicalResult cir::CIRDialect::verifyRegionResultAttribute( |
| 194 | Operation *op, unsigned /*regionIndex*/, unsigned /*resultIndex*/, |
| 195 | NamedAttribute attr) { |
| 196 | if (!isOpenCLVersionAttrName(attrName: attr.getName().getValue())) |
| 197 | return success(); |
| 198 | |
| 199 | return verifyOpenCLVersionAttrPlacement(op, attr); |
| 200 | } |
| 201 | |
| 202 | //===----------------------------------------------------------------------===// |
| 203 | // Dialect attribute verification |
| 204 | //===----------------------------------------------------------------------===// |
| 205 | |
| 206 | static LogicalResult verifyOffloadKind(mlir::ModuleOp module, |
| 207 | cir::OffloadKind expected) { |
| 208 | auto attr = module->getAttrOfType<cir::OffloadKindAttr>( |
| 209 | name: cir::CIRDialect::getOffloadKindAttrName()); |
| 210 | if (!attr) |
| 211 | return module.emitOpError() |
| 212 | << "expects '" << cir::CIRDialect::getOffloadKindAttrName() |
| 213 | << "' offload kind attribute" ; |
| 214 | if (attr.getValue() != expected) |
| 215 | return module.emitOpError() |
| 216 | << "expects '" << cir::CIRDialect::getOffloadKindAttrName() |
| 217 | << "' value '" << cir::stringifyOffloadKind(val: expected) << "'" ; |
| 218 | return success(); |
| 219 | } |
| 220 | |
| 221 | // A module marked with `cir.offload.container` holds the host module followed |
| 222 | // by one or more device modules, each tagged with `cir.offload.kind`. Keeping |
| 223 | // the host module first gives later offload passes a simple convention for |
| 224 | // finding the host side while iterating the remaining device modules. |
| 225 | static LogicalResult verifyOffloadContainer(mlir::Operation *op) { |
| 226 | auto container = mlir::dyn_cast<mlir::ModuleOp>(Val: op); |
| 227 | if (!container) |
| 228 | return op->emitOpError() |
| 229 | << "expects '" << cir::CIRDialect::getOffloadContainerAttrName() |
| 230 | << "' attribute to be attached to '" |
| 231 | << mlir::ModuleOp::getOperationName() << "'" ; |
| 232 | |
| 233 | mlir::Block &body = *container.getBody(); |
| 234 | if (body.empty()) |
| 235 | return container.emitOpError() |
| 236 | << "expects host module as the first nested op" ; |
| 237 | |
| 238 | auto host = mlir::dyn_cast<mlir::ModuleOp>(Val&: body.front()); |
| 239 | if (!host) |
| 240 | return container.emitOpError() |
| 241 | << "expects host module as the first nested op" ; |
| 242 | if (failed(Result: verifyOffloadKind(module: host, expected: cir::OffloadKind::Host))) |
| 243 | return failure(); |
| 244 | |
| 245 | // At least one device module is required after the host module. |
| 246 | if (std::next(x: body.begin()) == body.end()) |
| 247 | return container.emitOpError() << "expects at least one device module" ; |
| 248 | |
| 249 | for (auto &op : llvm::drop_begin(RangeOrContainer&: body)) { |
| 250 | auto module = mlir::dyn_cast<mlir::ModuleOp>(Val&: op); |
| 251 | if (!module) |
| 252 | return container.emitOpError() |
| 253 | << "expects only nested builtin.module ops" ; |
| 254 | if (failed(Result: verifyOffloadKind(module, expected: cir::OffloadKind::Device))) |
| 255 | return failure(); |
| 256 | } |
| 257 | return success(); |
| 258 | } |
| 259 | |
| 260 | LogicalResult |
| 261 | cir::CIRDialect::verifyOperationAttribute(mlir::Operation *op, |
| 262 | mlir::NamedAttribute attr) { |
| 263 | llvm::StringRef attrName = attr.getName().getValue(); |
| 264 | if (isOpenCLVersionAttrName(attrName)) |
| 265 | return verifyOpenCLVersionAttr(op, attr); |
| 266 | |
| 267 | if (attrName == getCXXABIAttrName()) { |
| 268 | if (!mlir::isa<mlir::ModuleOp>(Val: op)) |
| 269 | return op->emitOpError() << "expects '" << getCXXABIAttrName() |
| 270 | << "' attribute to be attached to '" |
| 271 | << mlir::ModuleOp::getOperationName() << "'" ; |
| 272 | auto abi = mlir::dyn_cast<mlir::StringAttr>(Val: attr.getValue()); |
| 273 | if (!abi) |
| 274 | return op->emitOpError() |
| 275 | << "expects '" << getCXXABIAttrName() << "' to be a string" ; |
| 276 | |
| 277 | StringRef abival = abi.getValue(); |
| 278 | if (!clang::TargetCXXABI::isABI(Name: abival)) |
| 279 | return op->emitOpError() << "unknown C++ ABI '" << abi.getValue() << "'" ; |
| 280 | |
| 281 | clang::TargetCXXABI::Kind kind = clang::TargetCXXABI::getKind(Name: abival); |
| 282 | if (auto triple = op->getAttrOfType<mlir::StringAttr>(name: getTripleAttrName())) |
| 283 | if (!clang::TargetCXXABI::isSupportedCXXABI( |
| 284 | T: llvm::Triple(triple.getValue()), Kind: kind)) |
| 285 | return op->emitOpError() |
| 286 | << "C++ ABI '" << abi.getValue() |
| 287 | << "' is not supported on target '" << triple.getValue() << "'" ; |
| 288 | |
| 289 | return success(); |
| 290 | } |
| 291 | |
| 292 | if (attrName == getOffloadContainerAttrName()) { |
| 293 | if (!mlir::isa<mlir::UnitAttr>(Val: attr.getValue())) |
| 294 | return op->emitOpError() << "expects '" << getOffloadContainerAttrName() |
| 295 | << "' to be a unit attribute" ; |
| 296 | return verifyOffloadContainer(op); |
| 297 | } |
| 298 | |
| 299 | // The container verifier owns the structural contract between a container |
| 300 | // and the modules it holds. All this can add is that the kind attribute |
| 301 | // never lands on something that is not a module. |
| 302 | if (attrName == getOffloadKindAttrName() && !mlir::isa<mlir::ModuleOp>(Val: op)) |
| 303 | return op->emitOpError() << "expects '" << getOffloadKindAttrName() |
| 304 | << "' attribute to be attached to '" |
| 305 | << mlir::ModuleOp::getOperationName() << "'" ; |
| 306 | |
| 307 | if (attrName == getModuleFramePointerAttrName()) { |
| 308 | if (!mlir::isa<mlir::ModuleOp>(Val: op)) |
| 309 | return op->emitOpError() << "expects '" << getModuleFramePointerAttrName() |
| 310 | << "' attribute to be attached to '" |
| 311 | << mlir::ModuleOp::getOperationName() << "'" ; |
| 312 | if (!mlir::isa<cir::FramePointerKindAttr>(Val: attr.getValue())) |
| 313 | return op->emitOpError() << "expects '" << getModuleFramePointerAttrName() |
| 314 | << "' to be a #cir.frame_pointer attribute" ; |
| 315 | return success(); |
| 316 | } |
| 317 | |
| 318 | // LoweringPrepare uses this attribute directly as the fatbin global's |
| 319 | // initializer, so it must be a valid #cir.const_array payload for its type. |
| 320 | if (attrName == getCUDADeviceBinaryAttrName()) { |
| 321 | auto bytes = mlir::dyn_cast<mlir::StringAttr>(Val: attr.getValue()); |
| 322 | auto arrayTy = |
| 323 | bytes ? mlir::dyn_cast<cir::ArrayType>(Val: bytes.getType()) : nullptr; |
| 324 | if (!arrayTy || arrayTy.getSize() != bytes.size()) |
| 325 | return op->emitOpError() |
| 326 | << "expects '" << getCUDADeviceBinaryAttrName() |
| 327 | << "' to be a string typed as an array of its length" ; |
| 328 | return cir::ConstArrayAttr::verify(emitError: [&] { return op->emitOpError(); }, |
| 329 | type: arrayTy, elts: bytes, |
| 330 | /*trailingZerosNum=*/0); |
| 331 | } |
| 332 | |
| 333 | if (attrName == getTripleAttrName()) { |
| 334 | if (!mlir::isa<mlir::ModuleOp>(Val: op)) |
| 335 | return op->emitOpError() |
| 336 | << "expects '" << getTripleAttrName() << "' attribute to be " |
| 337 | << "attached to '" << mlir::ModuleOp::getOperationName() << "'" ; |
| 338 | |
| 339 | auto tripleStr = mlir::dyn_cast<mlir::StringAttr>(Val: attr.getValue()); |
| 340 | if (!tripleStr) |
| 341 | return op->emitOpError() << "expects '" << getTripleAttrName() << "' to " |
| 342 | << "be a string" ; |
| 343 | llvm::Triple triple(tripleStr.getValue()); |
| 344 | // "Validate" our triple by checking if it has a known architecture; |
| 345 | // although not all triples have a corresponding clang TargetInfo. |
| 346 | if (triple.getArch() == llvm::Triple::UnknownArch) |
| 347 | return op->emitOpError() |
| 348 | << "unknown triple '" << tripleStr.getValue() << "' in '" |
| 349 | << getTripleAttrName() << "' attribute" ; |
| 350 | } |
| 351 | |
| 352 | if (attrName == getOptInfoAttrName()) { |
| 353 | if (!mlir::isa<mlir::ModuleOp>(Val: op)) |
| 354 | return op->emitOpError() |
| 355 | << "expects '" << getOptInfoAttrName() << "' attribute to be " |
| 356 | << "attached to '" << mlir::ModuleOp::getOperationName() << "'" ; |
| 357 | |
| 358 | auto optInfo = mlir::dyn_cast<cir::OptInfoAttr>(Val: attr.getValue()); |
| 359 | // Checks whether or not OptInfoAttr is even parseable; opt info range |
| 360 | // sanity checks are done in OptInfoAttr::verify. |
| 361 | if (!optInfo) |
| 362 | return op->emitOpError() |
| 363 | << "malformed " << cir::CIRDialect::getOptInfoAttrName() |
| 364 | << " attribute" ; |
| 365 | } |
| 366 | |
| 367 | return success(); |
| 368 | } |
| 369 | |
| 370 | //===----------------------------------------------------------------------===// |
| 371 | // Helpers |
| 372 | //===----------------------------------------------------------------------===// |
| 373 | |
| 374 | // Parses one of the keywords provided in the list `keywords` and returns the |
| 375 | // position of the parsed keyword in the list. If none of the keywords from the |
| 376 | // list is parsed, returns -1. |
| 377 | static int parseOptionalKeywordAlternative(AsmParser &parser, |
| 378 | ArrayRef<llvm::StringRef> keywords) { |
| 379 | for (auto en : llvm::enumerate(First&: keywords)) { |
| 380 | if (succeeded(Result: parser.parseOptionalKeyword(keyword: en.value()))) |
| 381 | return en.index(); |
| 382 | } |
| 383 | return -1; |
| 384 | } |
| 385 | |
| 386 | namespace { |
| 387 | template <typename Ty> struct EnumTraits {}; |
| 388 | |
| 389 | #define REGISTER_ENUM_TYPE(Ty) \ |
| 390 | template <> struct EnumTraits<cir::Ty> { \ |
| 391 | static llvm::StringRef stringify(cir::Ty value) { \ |
| 392 | return stringify##Ty(value); \ |
| 393 | } \ |
| 394 | static unsigned getMaxEnumVal() { return cir::getMaxEnumValFor##Ty(); } \ |
| 395 | } |
| 396 | |
| 397 | REGISTER_ENUM_TYPE(GlobalLinkageKind); |
| 398 | REGISTER_ENUM_TYPE(VisibilityKind); |
| 399 | REGISTER_ENUM_TYPE(CallingConv); |
| 400 | } // namespace |
| 401 | |
| 402 | /// Parse an enum from the keyword, or default to the provided default value. |
| 403 | /// The return type is the enum type by default, unless overriden with the |
| 404 | /// second template argument. |
| 405 | template <typename EnumTy, typename RetTy = EnumTy> |
| 406 | static RetTy parseOptionalCIRKeyword(AsmParser &parser, EnumTy defaultValue) { |
| 407 | llvm::SmallVector<llvm::StringRef, 10> names; |
| 408 | for (unsigned i = 0, e = EnumTraits<EnumTy>::getMaxEnumVal(); i <= e; ++i) |
| 409 | names.push_back(Elt: EnumTraits<EnumTy>::stringify(static_cast<EnumTy>(i))); |
| 410 | |
| 411 | int index = parseOptionalKeywordAlternative(parser, keywords: names); |
| 412 | if (index == -1) |
| 413 | return static_cast<RetTy>(defaultValue); |
| 414 | return static_cast<RetTy>(index); |
| 415 | } |
| 416 | |
| 417 | /// Parse an enum from the keyword, return failure if the keyword is not found. |
| 418 | template <typename EnumTy, typename RetTy = EnumTy> |
| 419 | static ParseResult parseCIRKeyword(AsmParser &parser, RetTy &result) { |
| 420 | llvm::SmallVector<llvm::StringRef, 10> names; |
| 421 | for (unsigned i = 0, e = EnumTraits<EnumTy>::getMaxEnumVal(); i <= e; ++i) |
| 422 | names.push_back(Elt: EnumTraits<EnumTy>::stringify(static_cast<EnumTy>(i))); |
| 423 | |
| 424 | int index = parseOptionalKeywordAlternative(parser, keywords: names); |
| 425 | if (index == -1) |
| 426 | return failure(); |
| 427 | result = static_cast<RetTy>(index); |
| 428 | return success(); |
| 429 | } |
| 430 | |
| 431 | // Check if a region's termination omission is valid and, if so, creates and |
| 432 | // inserts the omitted terminator into the region. |
| 433 | static LogicalResult ensureRegionTerm(OpAsmParser &parser, Region ®ion, |
| 434 | SMLoc errLoc) { |
| 435 | Location eLoc = parser.getEncodedSourceLoc(loc: parser.getCurrentLocation()); |
| 436 | OpBuilder builder(parser.getBuilder().getContext()); |
| 437 | |
| 438 | // Insert empty block in case the region is empty to ensure the terminator |
| 439 | // will be inserted |
| 440 | if (region.empty()) |
| 441 | builder.createBlock(parent: ®ion); |
| 442 | |
| 443 | Block &block = region.back(); |
| 444 | // Region is properly terminated: nothing to do. |
| 445 | if (!block.empty() && block.back().hasTrait<OpTrait::IsTerminator>()) |
| 446 | return success(); |
| 447 | |
| 448 | // Check for invalid terminator omissions. |
| 449 | if (!region.hasOneBlock()) |
| 450 | return parser.emitError(loc: errLoc, |
| 451 | message: "multi-block region must not omit terminator" ); |
| 452 | |
| 453 | // Terminator was omitted correctly: recreate it. |
| 454 | builder.setInsertionPointToEnd(&block); |
| 455 | cir::YieldOp::create(builder, location: eLoc); |
| 456 | return success(); |
| 457 | } |
| 458 | |
| 459 | // True if the region's terminator should be omitted. |
| 460 | static bool omitRegionTerm(mlir::Region &r) { |
| 461 | const auto singleNonEmptyBlock = r.hasOneBlock() && !r.back().empty(); |
| 462 | const auto yieldsNothing = [&r]() { |
| 463 | auto y = dyn_cast<cir::YieldOp>(Val: r.back().getTerminator()); |
| 464 | return y && y.getArgs().empty(); |
| 465 | }; |
| 466 | return singleNonEmptyBlock && yieldsNothing(); |
| 467 | } |
| 468 | |
| 469 | // Verifies that the given operand is produced by an operation of type |
| 470 | // ExpectedProducerOp. |
| 471 | template <typename ExpectedProducerOp> |
| 472 | static LogicalResult verifyProducedBy(Operation *op, Value operand, |
| 473 | StringRef operandName) { |
| 474 | Operation *producer = operand.getDefiningOp(); |
| 475 | if (!producer || !isa<ExpectedProducerOp>(producer)) |
| 476 | return op->emitOpError() |
| 477 | << "operand '" << operandName << "' must be produced by '" |
| 478 | << ExpectedProducerOp::getOperationName() << "'" ; |
| 479 | return success(); |
| 480 | } |
| 481 | |
| 482 | std::optional<llvm::Triple> cir::getTripleFromModule(mlir::ModuleOp mod) { |
| 483 | auto tripleAttr = |
| 484 | mod->getAttrOfType<mlir::StringAttr>(name: CIRDialect::getTripleAttrName()); |
| 485 | if (!tripleAttr) |
| 486 | return std::nullopt; |
| 487 | |
| 488 | // Verifying the validity of the triple attribute happens in |
| 489 | // cir::CIRDialect::verifyOperationAttribute; verifyOperationAttribute |
| 490 | // should error first if we have an invalid attribute. |
| 491 | return llvm::Triple{tripleAttr.getValue()}; |
| 492 | } |
| 493 | |
| 494 | //===----------------------------------------------------------------------===// |
| 495 | // InlineKindAttr (FIXME: remove once FuncOp uses assembly format) |
| 496 | //===----------------------------------------------------------------------===// |
| 497 | |
| 498 | ParseResult parseInlineKindAttr(OpAsmParser &parser, |
| 499 | cir::InlineKindAttr &inlineKindAttr) { |
| 500 | // Static list of possible inline kind keywords |
| 501 | static constexpr llvm::StringRef keywords[] = {"no_inline" , "always_inline" , |
| 502 | "inline_hint" }; |
| 503 | |
| 504 | // Parse the inline kind keyword (optional) |
| 505 | llvm::StringRef keyword; |
| 506 | if (parser.parseOptionalKeyword(keyword: &keyword, allowedValues: keywords).failed()) { |
| 507 | // Not an inline kind keyword, leave inlineKindAttr empty |
| 508 | return success(); |
| 509 | } |
| 510 | |
| 511 | // Parse the enum value from the keyword |
| 512 | auto inlineKindResult = ::cir::symbolizeEnum<::cir::InlineKind>(str: keyword); |
| 513 | if (!inlineKindResult) { |
| 514 | return parser.emitError(loc: parser.getCurrentLocation(), message: "expected one of [" ) |
| 515 | << llvm::join(R: llvm::ArrayRef(keywords), Separator: ", " ) |
| 516 | << "] for inlineKind, got: " << keyword; |
| 517 | } |
| 518 | |
| 519 | inlineKindAttr = |
| 520 | ::cir::InlineKindAttr::get(context: parser.getContext(), value: *inlineKindResult); |
| 521 | return success(); |
| 522 | } |
| 523 | |
| 524 | void printInlineKindAttr(OpAsmPrinter &p, cir::InlineKindAttr inlineKindAttr) { |
| 525 | if (inlineKindAttr) { |
| 526 | p << " " << stringifyInlineKind(val: inlineKindAttr.getValue()); |
| 527 | } |
| 528 | } |
| 529 | |
| 530 | //===----------------------------------------------------------------------===// |
| 531 | // CIR Custom Parsers/Printers |
| 532 | //===----------------------------------------------------------------------===// |
| 533 | |
| 534 | mlir::ParseResult cir::parseOmittedTerminatorRegion(mlir::OpAsmParser &parser, |
| 535 | mlir::Region ®ion) { |
| 536 | auto regionLoc = parser.getCurrentLocation(); |
| 537 | if (parser.parseRegion(region)) |
| 538 | return failure(); |
| 539 | if (ensureRegionTerm(parser, region, errLoc: regionLoc).failed()) |
| 540 | return failure(); |
| 541 | return success(); |
| 542 | } |
| 543 | |
| 544 | void cir::printOmittedTerminatorRegion(mlir::OpAsmPrinter &printer, |
| 545 | cir::ScopeOp &op, mlir::Region ®ion) { |
| 546 | printer.printRegion(blocks&: region, |
| 547 | /*printEntryBlockArgs=*/false, |
| 548 | /*printBlockTerminators=*/!omitRegionTerm(r&: region)); |
| 549 | } |
| 550 | |
| 551 | //===----------------------------------------------------------------------===// |
| 552 | // AllocaOp |
| 553 | //===----------------------------------------------------------------------===// |
| 554 | |
| 555 | void cir::AllocaOp::build(mlir::OpBuilder &odsBuilder, |
| 556 | mlir::OperationState &odsState, mlir::Type addr, |
| 557 | llvm::StringRef name, mlir::IntegerAttr alignment) { |
| 558 | odsState.addAttribute(name: getNameAttrName(name: odsState.name), |
| 559 | attr: odsBuilder.getStringAttr(bytes: name)); |
| 560 | if (alignment) { |
| 561 | odsState.addAttribute(name: getAlignmentAttrName(name: odsState.name), attr: alignment); |
| 562 | } |
| 563 | odsState.addTypes(newTypes: addr); |
| 564 | } |
| 565 | |
| 566 | bool cir::usesRelativeVTables(mlir::ModuleOp mod) { |
| 567 | auto loweringOpts = mod->getAttrOfType<cir::LoweringOptionsAttr>( |
| 568 | name: cir::CIRDialect::getLoweringOptionsAttrName()); |
| 569 | return loweringOpts && loweringOpts.getRelativeCxxAbiVtables(); |
| 570 | } |
| 571 | |
| 572 | uint64_t cir::getVTableEntrySize(mlir::ModuleOp mod, |
| 573 | const mlir::DataLayout &dataLayout) { |
| 574 | // Relative layout entries are 32-bit offsets. |
| 575 | if (usesRelativeVTables(mod)) |
| 576 | return 4; |
| 577 | return dataLayout.getTypeSize( |
| 578 | t: cir::PointerType::get(pointee: cir::VoidType::get(ctx: mod.getContext()))); |
| 579 | } |
| 580 | |
| 581 | cir::AllocaOp cir::getUnderlyingAlloca(mlir::Value addr) { |
| 582 | mlir::Value ptr = addr; |
| 583 | while (cir::CastOp castOp = ptr.getDefiningOp<cir::CastOp>()) { |
| 584 | if (!castOp.isAllocaPreservingCast()) |
| 585 | break; |
| 586 | ptr = castOp.getSrc(); |
| 587 | } |
| 588 | return ptr.getDefiningOp<cir::AllocaOp>(); |
| 589 | } |
| 590 | |
| 591 | //===----------------------------------------------------------------------===// |
| 592 | // ArrayCtor & ArrayDtor |
| 593 | //===----------------------------------------------------------------------===// |
| 594 | |
| 595 | template <typename Op> static LogicalResult verifyArrayCtorDtor(Op op) { |
| 596 | auto ptrTy = mlir::cast<cir::PointerType>(op.getAddr().getType()); |
| 597 | mlir::Type pointeeTy = ptrTy.getPointee(); |
| 598 | |
| 599 | mlir::Block &body = op.getBody().front(); |
| 600 | if (body.getNumArguments() != 1) |
| 601 | return op.emitOpError("body must have exactly one block argument" ); |
| 602 | |
| 603 | auto expectedEltPtrTy = |
| 604 | mlir::dyn_cast<cir::PointerType>(Val: body.getArgument(i: 0).getType()); |
| 605 | if (!expectedEltPtrTy) |
| 606 | return op.emitOpError("block argument must be a !cir.ptr type" ); |
| 607 | |
| 608 | if (op.getNumElements()) { |
| 609 | auto recTy = mlir::dyn_cast<cir::RecordType>(Val&: pointeeTy); |
| 610 | if (!recTy) |
| 611 | return op.emitOpError( |
| 612 | "when 'num_elements' is present, 'addr' must be a pointer to a " |
| 613 | "!cir.struct or !cir.union type" ); |
| 614 | |
| 615 | if (expectedEltPtrTy != ptrTy) |
| 616 | return op.emitOpError("when 'num_elements' is present, 'addr' type must " |
| 617 | "match the block argument type" ); |
| 618 | } else { |
| 619 | auto arrayTy = mlir::dyn_cast<cir::ArrayType>(Val&: pointeeTy); |
| 620 | if (!arrayTy) |
| 621 | return op.emitOpError( |
| 622 | "when 'num_elements' is absent, 'addr' must be a pointer to a " |
| 623 | "!cir.array type" ); |
| 624 | |
| 625 | mlir::Type innerEltTy = arrayTy.getElementType(); |
| 626 | while (auto nested = mlir::dyn_cast<cir::ArrayType>(Val&: innerEltTy)) |
| 627 | innerEltTy = nested.getElementType(); |
| 628 | |
| 629 | auto recTy = mlir::dyn_cast<cir::RecordType>(Val&: innerEltTy); |
| 630 | if (!recTy) |
| 631 | return op.emitOpError("the block argument type must be a pointer to a " |
| 632 | "!cir.struct or !cir.union type" ); |
| 633 | |
| 634 | if (expectedEltPtrTy.getPointee() != innerEltTy) |
| 635 | return op.emitOpError( |
| 636 | "block argument pointee type must match the innermost array " |
| 637 | "element type" ); |
| 638 | |
| 639 | if (expectedEltPtrTy.getAddrSpace() != ptrTy.getAddrSpace()) |
| 640 | return op.emitOpError( |
| 641 | "block argument address space must match the 'addr' address space" ); |
| 642 | } |
| 643 | |
| 644 | return success(); |
| 645 | } |
| 646 | |
| 647 | LogicalResult cir::ArrayCtor::verify() { |
| 648 | if (failed(Result: verifyArrayCtorDtor(op: *this))) |
| 649 | return failure(); |
| 650 | |
| 651 | mlir::Region &partialDtor = getPartialDtor(); |
| 652 | if (!partialDtor.empty()) { |
| 653 | mlir::Block &dtorBlock = partialDtor.front(); |
| 654 | if (dtorBlock.getNumArguments() != 1) |
| 655 | return emitOpError(message: "partial_dtor must have exactly one block argument" ); |
| 656 | |
| 657 | auto bodyArgTy = getBody().front().getArgument(i: 0).getType(); |
| 658 | if (dtorBlock.getArgument(i: 0).getType() != bodyArgTy) |
| 659 | return emitOpError(message: "partial_dtor block argument type must match " |
| 660 | "the body block argument type" ); |
| 661 | } |
| 662 | return success(); |
| 663 | } |
| 664 | LogicalResult cir::ArrayDtor::verify() { return verifyArrayCtorDtor(op: *this); } |
| 665 | |
| 666 | //===----------------------------------------------------------------------===// |
| 667 | // DeleteArrayOp |
| 668 | //===----------------------------------------------------------------------===// |
| 669 | |
| 670 | LogicalResult cir::DeleteArrayOp::verify() { |
| 671 | if (getDtorMayThrow() && !getElementDtorAttr()) |
| 672 | return emitOpError( |
| 673 | message: "'dtor_may_throw' requires an 'element_dtor' to be present" ); |
| 674 | return success(); |
| 675 | } |
| 676 | |
| 677 | //===----------------------------------------------------------------------===// |
| 678 | // AssumeOp |
| 679 | //===----------------------------------------------------------------------===// |
| 680 | |
| 681 | void cir::printAssumeBundle(OpAsmPrinter &p, cir::AssumeOp op, |
| 682 | cir::AssumeBundleKindAttr kindAttr, |
| 683 | OperandRange bundleArgs, TypeRange bundleArgTypes) { |
| 684 | cir::AssumeBundleKind kind = kindAttr.getValue(); |
| 685 | if (kind == cir::AssumeBundleKind::None) |
| 686 | return; |
| 687 | |
| 688 | p << " " << cir::stringifyAssumeBundleKind(val: kind); |
| 689 | if (bundleArgs.empty()) |
| 690 | return; |
| 691 | |
| 692 | p << "(" ; |
| 693 | p.printOperands(operands: bundleArgs); |
| 694 | p << " : " ; |
| 695 | llvm::interleaveComma(c: bundleArgTypes, os&: p); |
| 696 | p << ")" ; |
| 697 | } |
| 698 | |
| 699 | ParseResult cir::parseAssumeBundle( |
| 700 | OpAsmParser &p, cir::AssumeBundleKindAttr &bundleKindAttr, |
| 701 | llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand, 4> &bundleArgs, |
| 702 | llvm::SmallVector<mlir::Type, 1> &bundleArgTypes) { |
| 703 | StringRef keyword; |
| 704 | auto loc = p.getCurrentLocation(); |
| 705 | if (failed(Result: p.parseOptionalKeyword(keyword: &keyword))) { |
| 706 | bundleKindAttr = cir::AssumeBundleKindAttr::get( |
| 707 | context: p.getContext(), value: cir::AssumeBundleKind::None); |
| 708 | return success(); |
| 709 | } |
| 710 | |
| 711 | std::optional<cir::AssumeBundleKind> parsedKind = |
| 712 | cir::symbolizeAssumeBundleKind(str: keyword); |
| 713 | if (!parsedKind) |
| 714 | return p.emitError(loc, message: "unknown assume bundle kind '" ) << keyword << "'" ; |
| 715 | |
| 716 | bundleKindAttr = cir::AssumeBundleKindAttr::get(context: p.getContext(), value: *parsedKind); |
| 717 | |
| 718 | if (p.parseOptionalLParen()) |
| 719 | return success(); |
| 720 | |
| 721 | if (p.parseOperandList(result&: bundleArgs) || p.parseColon() || |
| 722 | p.parseTypeList(result&: bundleArgTypes) || p.parseRParen()) |
| 723 | return failure(); |
| 724 | |
| 725 | return success(); |
| 726 | } |
| 727 | |
| 728 | LogicalResult cir::AssumeOp::verify() { |
| 729 | cir::AssumeBundleKind kind = getBundleKind(); |
| 730 | size_t numArgs = getBundleArgs().size(); |
| 731 | |
| 732 | if (kind == cir::AssumeBundleKind::None) { |
| 733 | if (numArgs != 0) |
| 734 | return emitOpError(message: "unexpected bundle operands for kind 'none'" ); |
| 735 | return success(); |
| 736 | } |
| 737 | |
| 738 | if (numArgs == 0) |
| 739 | return emitOpError(message: "expected bundle operands for kind '" ) |
| 740 | << cir::stringifyAssumeBundleKind(val: kind) << "'" ; |
| 741 | |
| 742 | switch (kind) { |
| 743 | case cir::AssumeBundleKind::Align: |
| 744 | if (numArgs != 2 && numArgs != 3) |
| 745 | return emitOpError(message: "align bundle expects 2 or 3 operands" ); |
| 746 | break; |
| 747 | case cir::AssumeBundleKind::SeparateStorage: |
| 748 | if (numArgs != 2) |
| 749 | return emitOpError(message: "separate_storage bundle expects 2 operands" ); |
| 750 | break; |
| 751 | case cir::AssumeBundleKind::Dereferenceable: |
| 752 | if (numArgs != 2) |
| 753 | return emitOpError(message: "dereferenceable bundle expects 2 operands" ); |
| 754 | break; |
| 755 | default: |
| 756 | break; |
| 757 | } |
| 758 | return success(); |
| 759 | } |
| 760 | |
| 761 | //===----------------------------------------------------------------------===// |
| 762 | // AnnotationOp |
| 763 | //===----------------------------------------------------------------------===// |
| 764 | |
| 765 | LogicalResult cir::AnnotationOp::verify() { |
| 766 | if (mlir::ArrayAttr args = getAnnotation().getArgs(); args && !args.empty()) |
| 767 | return emitOpError(message: "annotation must not have arguments" ); |
| 768 | return mlir::success(); |
| 769 | } |
| 770 | |
| 771 | //===----------------------------------------------------------------------===// |
| 772 | // LocalInitOp |
| 773 | //===----------------------------------------------------------------------===// |
| 774 | |
| 775 | LogicalResult |
| 776 | cir::LocalInitOp::verifySymbolUses(SymbolTableCollection &symbolTable) { |
| 777 | cir::GlobalOp global = getReferencedGlobal(tables&: symbolTable); |
| 778 | if (!global) |
| 779 | return emitOpError(message: "'" ) |
| 780 | << getGlobalName() << "' does not reference a valid cir.global" ; |
| 781 | |
| 782 | if (getTls() && !global.getTlsModel()) |
| 783 | return emitOpError(message: "access to global not marked thread local" ); |
| 784 | |
| 785 | if (!global.getDynamicInitGuard().has_value()) |
| 786 | return emitOpError(message: "static_local attribute mismatch" ); |
| 787 | |
| 788 | return success(); |
| 789 | } |
| 790 | |
| 791 | //===----------------------------------------------------------------------===// |
| 792 | // RegisterExitDtorOp |
| 793 | //===----------------------------------------------------------------------===// |
| 794 | |
| 795 | /// Returns true if \p op is nested in the ctor region of a cir.global or |
| 796 | /// cir.local_init without crossing a cir.func or a cir.register_exit_dtor. |
| 797 | static bool isInVarCtorRegion(mlir::Operation *op) { |
| 798 | for (mlir::Operation *parent = op->getParentOp(); parent; |
| 799 | parent = parent->getParentOp()) { |
| 800 | mlir::Region *ctorRegion = nullptr; |
| 801 | if (auto global = mlir::dyn_cast<cir::GlobalOp>(Val: parent)) |
| 802 | ctorRegion = &global.getCtorRegion(); |
| 803 | else if (auto localInit = mlir::dyn_cast<cir::LocalInitOp>(Val: parent)) |
| 804 | ctorRegion = &localInit.getCtorRegion(); |
| 805 | else if (mlir::isa<cir::FuncOp, cir::RegisterExitDtorOp>(Val: parent)) |
| 806 | return false; |
| 807 | if (ctorRegion) |
| 808 | return ctorRegion->isAncestor(other: op->getParentRegion()); |
| 809 | } |
| 810 | return false; |
| 811 | } |
| 812 | |
| 813 | LogicalResult cir::RegisterExitDtorOp::verify() { |
| 814 | mlir::Block &body = getBody().front(); |
| 815 | if (body.without_terminator().empty()) |
| 816 | return emitOpError(message: "body must destroy the object" ); |
| 817 | if (body.getTerminator()->getNumOperands()) |
| 818 | return emitOpError(message: "body must not yield a value" ); |
| 819 | if (!isInVarCtorRegion(op: *this)) |
| 820 | return emitOpError(message: "must be in the ctor region of a cir.global or " |
| 821 | "cir.local_init" ); |
| 822 | return success(); |
| 823 | } |
| 824 | |
| 825 | LogicalResult |
| 826 | cir::RegisterExitDtorOp::verifySymbolUses(SymbolTableCollection &symbolTable) { |
| 827 | auto global = symbolTable.lookupNearestSymbolFrom<cir::GlobalOp>( |
| 828 | from: *this, symbol: getObjectAttr()); |
| 829 | if (!global) |
| 830 | return emitOpError(message: "'" ) |
| 831 | << getObject() << "' does not reference a valid cir.global" ; |
| 832 | if (!global.getCtorRegion().empty() || !global.getDtorRegion().empty()) |
| 833 | return emitOpError(message: "object must not have ctor or dtor regions" ); |
| 834 | return success(); |
| 835 | } |
| 836 | |
| 837 | //===----------------------------------------------------------------------===// |
| 838 | // ConditionOp |
| 839 | //===----------------------------------------------------------------------===// |
| 840 | |
| 841 | //===---------------------------------- |
| 842 | // BranchOpTerminatorInterface Methods |
| 843 | //===---------------------------------- |
| 844 | |
| 845 | /// Return true if a cir.break may leave `loop`. A break nested in an inner |
| 846 | /// loop or switch leaves that operation instead. |
| 847 | static bool mayBreakOutOf(cir::LoopOpInterface loop) { |
| 848 | mlir::Operation *loopOp = loop.getOperation(); |
| 849 | return loopOp |
| 850 | ->walk<mlir::WalkOrder::PreOrder>(callback: [&](mlir::Operation *op) { |
| 851 | if (op != loopOp && isa<cir::LoopOpInterface, cir::SwitchOp>(Val: op)) |
| 852 | return mlir::WalkResult::skip(); |
| 853 | if (isa<cir::BreakOp>(Val: op)) |
| 854 | return mlir::WalkResult::interrupt(); |
| 855 | return mlir::WalkResult::advance(); |
| 856 | }) |
| 857 | .wasInterrupted(); |
| 858 | } |
| 859 | |
| 860 | void cir::ConditionOp::getSuccessorRegions( |
| 861 | ArrayRef<Attribute> operands, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 862 | // A constant condition selects only one of the successors below. |
| 863 | FoldAdaptor adaptor(operands, *this); |
| 864 | auto cond = mlir::dyn_cast_if_present<cir::BoolAttr>(Val: adaptor.getCondition()); |
| 865 | bool mayBeTrue = !cond || cond.getValue(); |
| 866 | bool mayBeFalse = !cond || !cond.getValue(); |
| 867 | |
| 868 | // Parent is a loop: condition may branch to the body, or on the false edge |
| 869 | // to the per-iteration cleanup region if present, otherwise to the parent op. |
| 870 | // cir.break is not a region branch terminator (see the FIXME in |
| 871 | // getLoopOpSuccessorRegions), so the false edge also stands for the loop |
| 872 | // exit through a break and is kept when the loop may break. |
| 873 | if (auto loopOp = dyn_cast<LoopOpInterface>(Val: getOperation()->getParentOp())) { |
| 874 | if (mayBeTrue) |
| 875 | regions.emplace_back(Args: &loopOp.getBody()); |
| 876 | if (mayBeFalse || mayBreakOutOf(loop: loopOp)) { |
| 877 | if (mlir::Region *cleanup = loopOp.maybeGetCleanup()) |
| 878 | regions.emplace_back(Args&: cleanup); |
| 879 | else |
| 880 | regions.emplace_back(Args: loopOp.getOperation()); |
| 881 | } |
| 882 | return; |
| 883 | } |
| 884 | |
| 885 | // Parent is an await: condition may branch to resume (true) or suspend |
| 886 | // (false) regions. |
| 887 | auto await = cast<AwaitOp>(Val: getOperation()->getParentOp()); |
| 888 | if (mayBeTrue) |
| 889 | regions.emplace_back(Args: &await.getResume()); |
| 890 | if (mayBeFalse) |
| 891 | regions.emplace_back(Args: &await.getSuspend()); |
| 892 | } |
| 893 | |
| 894 | MutableOperandRange |
| 895 | cir::ConditionOp::getMutableSuccessorOperands(RegionSuccessor point) { |
| 896 | // No values are yielded to the successor region. |
| 897 | return MutableOperandRange(getOperation(), 0, 0); |
| 898 | } |
| 899 | |
| 900 | MutableOperandRange |
| 901 | cir::ResumeOp::getMutableSuccessorOperands(RegionSuccessor point) { |
| 902 | // The eh_token operand is not modeled as forwarded to any successor. |
| 903 | return MutableOperandRange(getOperation(), 0, 0); |
| 904 | } |
| 905 | |
| 906 | MutableOperandRange |
| 907 | cir::CoroSuspendPoint::getMutableSuccessorOperands(RegionSuccessor point) { |
| 908 | // The condition operand is not forwarded to successor regions. |
| 909 | return MutableOperandRange(getOperation(), 0, 0); |
| 910 | } |
| 911 | |
| 912 | LogicalResult cir::ConditionOp::verify() { |
| 913 | if (!isa<LoopOpInterface, AwaitOp>(Val: getOperation()->getParentOp())) |
| 914 | return emitOpError(message: "condition must be within a conditional region" ); |
| 915 | return success(); |
| 916 | } |
| 917 | |
| 918 | //===----------------------------------------------------------------------===// |
| 919 | // WhileOp & ForOp |
| 920 | //===----------------------------------------------------------------------===// |
| 921 | |
| 922 | template <typename LoopOpTy> |
| 923 | static LogicalResult verifyLoopCleanup(LoopOpTy op) { |
| 924 | std::optional<cir::CleanupKind> cleanupKind = op.getCleanupKind(); |
| 925 | |
| 926 | // The cleanup kind attribute must be present exactly when a (non-empty) |
| 927 | // cleanup region is present. |
| 928 | if (cleanupKind.has_value() == op.getCleanup().empty()) |
| 929 | return op.emitOpError("cleanup kind must be present if and only if the " |
| 930 | "cleanup region is non-empty" ); |
| 931 | |
| 932 | // A loop's per-iteration cleanup runs on every normal exit edge (loop exit, |
| 933 | // end of iteration, break/continue), so an EH-only cleanup is meaningless. |
| 934 | // Only 'normal' (exceptions disabled) and 'all' (normal + EH unwind) apply. |
| 935 | if (cleanupKind == cir::CleanupKind::EH) |
| 936 | return op.emitOpError("loop cleanup kind must be 'normal' or 'all', " |
| 937 | "not 'eh'" ); |
| 938 | |
| 939 | return success(); |
| 940 | } |
| 941 | |
| 942 | LogicalResult cir::WhileOp::verify() { return verifyLoopCleanup(op: *this); } |
| 943 | |
| 944 | LogicalResult cir::ForOp::verify() { return verifyLoopCleanup(op: *this); } |
| 945 | |
| 946 | //===----------------------------------------------------------------------===// |
| 947 | // ConstantOp |
| 948 | //===----------------------------------------------------------------------===// |
| 949 | |
| 950 | static LogicalResult checkConstantTypes(mlir::Operation *op, mlir::Type opType, |
| 951 | mlir::Attribute attrType) { |
| 952 | if (isa<cir::ConstPtrAttr>(Val: attrType)) { |
| 953 | if (!mlir::isa<cir::PointerType>(Val: opType)) |
| 954 | return op->emitOpError( |
| 955 | message: "pointer constant initializing a non-pointer type" ); |
| 956 | return success(); |
| 957 | } |
| 958 | |
| 959 | if (isa<cir::DataMemberAttr, cir::DataMemberOffsetAttr, cir::MethodAttr>( |
| 960 | Val: attrType)) { |
| 961 | // More detailed type verifications are already done in |
| 962 | // DataMemberAttr::verify or MethodAttr::verify. Don't need to repeat here. |
| 963 | return success(); |
| 964 | } |
| 965 | |
| 966 | if (isa<cir::ZeroAttr>(Val: attrType)) { |
| 967 | if (isa<cir::RecordType, cir::ArrayType, cir::MatrixType, cir::VectorType, |
| 968 | cir::ComplexType>(Val: opType)) |
| 969 | return success(); |
| 970 | return op->emitOpError( |
| 971 | message: "zero expects struct, array, vector, or complex type" ); |
| 972 | } |
| 973 | |
| 974 | if (mlir::isa<cir::UndefAttr>(Val: attrType)) { |
| 975 | if (!mlir::isa<cir::VoidType>(Val: opType)) |
| 976 | return success(); |
| 977 | return op->emitOpError(message: "undef expects non-void type" ); |
| 978 | } |
| 979 | |
| 980 | if (mlir::isa<cir::BoolAttr>(Val: attrType)) { |
| 981 | if (!mlir::isa<cir::BoolType>(Val: opType)) |
| 982 | return op->emitOpError(message: "result type (" ) |
| 983 | << opType << ") must be '!cir.bool' for '" << attrType << "'" ; |
| 984 | return success(); |
| 985 | } |
| 986 | |
| 987 | if (mlir::isa<cir::IntAttr, cir::FPAttr>(Val: attrType)) { |
| 988 | auto at = cast<TypedAttr>(Val&: attrType); |
| 989 | if (at.getType() != opType) { |
| 990 | return op->emitOpError(message: "result type (" ) |
| 991 | << opType << ") does not match value type (" << at.getType() |
| 992 | << ")" ; |
| 993 | } |
| 994 | return success(); |
| 995 | } |
| 996 | |
| 997 | if (mlir::isa<cir::BlockAddrDiffAttr, cir::BlockAddrInfoAttr, |
| 998 | cir::ConstArrayAttr, cir::ConstVectorAttr, |
| 999 | cir::ConstComplexAttr, cir::ConstRecordAttr, |
| 1000 | cir::GlobalOffsetAttr, cir::GlobalViewAttr, cir::PoisonAttr, |
| 1001 | cir::TypeInfoAttr, cir::VTableAttr>(Val: attrType)) |
| 1002 | return success(); |
| 1003 | |
| 1004 | assert(isa<TypedAttr>(attrType) && "What else could we be looking at here?" ); |
| 1005 | return op->emitOpError(message: "global with type " ) |
| 1006 | << cast<TypedAttr>(Val&: attrType).getType() << " not yet supported" ; |
| 1007 | } |
| 1008 | |
| 1009 | LogicalResult cir::ConstantOp::verify() { |
| 1010 | // ODS already generates checks to make sure the result type is valid. We just |
| 1011 | // need to additionally check that the value's attribute type is consistent |
| 1012 | // with the result type. |
| 1013 | return checkConstantTypes(op: getOperation(), opType: getType(), attrType: getValue()); |
| 1014 | } |
| 1015 | |
| 1016 | OpFoldResult cir::ConstantOp::fold(FoldAdaptor /*adaptor*/) { |
| 1017 | return getValue(); |
| 1018 | } |
| 1019 | |
| 1020 | //===----------------------------------------------------------------------===// |
| 1021 | // CastOp |
| 1022 | //===----------------------------------------------------------------------===// |
| 1023 | |
| 1024 | static bool isFloatingPointCastKind(cir::CastKind kind) { |
| 1025 | switch (kind) { |
| 1026 | case cir::CastKind::floating: |
| 1027 | case cir::CastKind::int_to_float: |
| 1028 | case cir::CastKind::float_to_int: |
| 1029 | case cir::CastKind::float_to_bool: |
| 1030 | case cir::CastKind::bool_to_float: |
| 1031 | case cir::CastKind::float_to_complex: |
| 1032 | case cir::CastKind::float_complex_to_real: |
| 1033 | case cir::CastKind::float_complex_to_bool: |
| 1034 | case cir::CastKind::float_complex: |
| 1035 | case cir::CastKind::float_complex_to_int_complex: |
| 1036 | case cir::CastKind::int_complex_to_float_complex: |
| 1037 | return true; |
| 1038 | default: |
| 1039 | return false; |
| 1040 | } |
| 1041 | } |
| 1042 | |
| 1043 | LogicalResult cir::CastOp::verify() { |
| 1044 | mlir::Type resType = getType(); |
| 1045 | mlir::Type srcType = getSrc().getType(); |
| 1046 | cir::CastKind kind = getKind(); |
| 1047 | if (getFenvAttr() && !isFloatingPointCastKind(kind)) |
| 1048 | return emitOpError() |
| 1049 | << "'fenv' is only valid for floating-point cast kinds" ; |
| 1050 | |
| 1051 | // Verify address space casts for pointer types. given that |
| 1052 | // casts for within a different address space are illegal. |
| 1053 | auto srcPtrTy = mlir::dyn_cast<cir::PointerType>(Val&: srcType); |
| 1054 | auto resPtrTy = mlir::dyn_cast<cir::PointerType>(Val&: resType); |
| 1055 | if (srcPtrTy && resPtrTy && (kind != cir::CastKind::address_space)) |
| 1056 | if (srcPtrTy.getAddrSpace() != resPtrTy.getAddrSpace()) { |
| 1057 | return emitOpError() << "result type address space does not match the " |
| 1058 | "address space of the operand" ; |
| 1059 | } |
| 1060 | |
| 1061 | auto srcVTy = mlir::dyn_cast<cir::VectorType>(Val&: srcType); |
| 1062 | auto resVTy = mlir::dyn_cast<cir::VectorType>(Val&: resType); |
| 1063 | if (srcVTy && resVTy) { |
| 1064 | if (kind != cir::CastKind::bitcast && |
| 1065 | srcVTy.getSize() != resVTy.getSize()) { |
| 1066 | return emitOpError() << "vector " << cir::stringifyCastKind(val: kind) |
| 1067 | << " cast require source and destination vectors to " |
| 1068 | "have the same number of " |
| 1069 | "elements" ; |
| 1070 | } |
| 1071 | // Use the element type of the vector to verify the cast kind. (Except for |
| 1072 | // bitcast, see below.) |
| 1073 | srcType = srcVTy.getElementType(); |
| 1074 | resType = resVTy.getElementType(); |
| 1075 | } |
| 1076 | |
| 1077 | auto srcMatTy = mlir::dyn_cast<cir::MatrixType>(Val&: srcType); |
| 1078 | auto resMatTy = mlir::dyn_cast<cir::MatrixType>(Val&: resType); |
| 1079 | if (srcMatTy && resMatTy) { |
| 1080 | if (kind != cir::CastKind::bitcast && |
| 1081 | (srcMatTy.getNumRows() != resMatTy.getNumRows() || |
| 1082 | srcMatTy.getNumColumns() != resMatTy.getNumColumns())) { |
| 1083 | return emitOpError() << "matrix " << cir::stringifyCastKind(val: kind) |
| 1084 | << " cast require source and destination matrices " |
| 1085 | "to have the same number of " |
| 1086 | "rows and columns" ; |
| 1087 | } |
| 1088 | |
| 1089 | srcType = srcMatTy.getElementType(); |
| 1090 | resType = resMatTy.getElementType(); |
| 1091 | } |
| 1092 | |
| 1093 | switch (kind) { |
| 1094 | case cir::CastKind::int_to_bool: { |
| 1095 | if (!mlir::isa<cir::BoolType>(Val: resType)) |
| 1096 | return emitOpError() << "requires !cir.bool type for result" ; |
| 1097 | if (!mlir::isa<cir::IntType>(Val: srcType)) |
| 1098 | return emitOpError() << "requires !cir.int type for source" ; |
| 1099 | return success(); |
| 1100 | } |
| 1101 | case cir::CastKind::ptr_to_bool: { |
| 1102 | if (!mlir::isa<cir::BoolType>(Val: resType)) |
| 1103 | return emitOpError() << "requires !cir.bool type for result" ; |
| 1104 | if (!mlir::isa<cir::PointerType>(Val: srcType)) |
| 1105 | return emitOpError() << "requires !cir.ptr type for source" ; |
| 1106 | return success(); |
| 1107 | } |
| 1108 | case cir::CastKind::integral: { |
| 1109 | if (!mlir::isa<cir::IntType>(Val: resType)) |
| 1110 | return emitOpError() << "requires !cir.int type for result" ; |
| 1111 | if (!mlir::isa<cir::IntType>(Val: srcType)) |
| 1112 | return emitOpError() << "requires !cir.int type for source" ; |
| 1113 | return success(); |
| 1114 | } |
| 1115 | case cir::CastKind::array_to_ptrdecay: { |
| 1116 | const auto arrayPtrTy = mlir::dyn_cast<cir::PointerType>(Val&: srcType); |
| 1117 | const auto flatPtrTy = mlir::dyn_cast<cir::PointerType>(Val&: resType); |
| 1118 | if (!arrayPtrTy || !flatPtrTy) |
| 1119 | return emitOpError() << "requires !cir.ptr type for source and result" ; |
| 1120 | |
| 1121 | // TODO(CIR): Make sure the AddrSpace of both types are equals |
| 1122 | return success(); |
| 1123 | } |
| 1124 | case cir::CastKind::bitcast: { |
| 1125 | // Handle the pointer types first. |
| 1126 | auto srcPtrTy = mlir::dyn_cast<cir::PointerType>(Val&: srcType); |
| 1127 | auto resPtrTy = mlir::dyn_cast<cir::PointerType>(Val&: resType); |
| 1128 | |
| 1129 | if (srcPtrTy && resPtrTy) { |
| 1130 | return success(); |
| 1131 | } |
| 1132 | |
| 1133 | return success(); |
| 1134 | } |
| 1135 | case cir::CastKind::floating: { |
| 1136 | if (!mlir::isa<cir::FPTypeInterface>(Val: srcType) || |
| 1137 | !mlir::isa<cir::FPTypeInterface>(Val: resType)) |
| 1138 | return emitOpError() << "requires !cir.float type for source and result" ; |
| 1139 | return success(); |
| 1140 | } |
| 1141 | case cir::CastKind::float_to_int: { |
| 1142 | if (!mlir::isa<cir::FPTypeInterface>(Val: srcType)) |
| 1143 | return emitOpError() << "requires !cir.float type for source" ; |
| 1144 | if (!mlir::dyn_cast<cir::IntType>(Val&: resType)) |
| 1145 | return emitOpError() << "requires !cir.int type for result" ; |
| 1146 | return success(); |
| 1147 | } |
| 1148 | case cir::CastKind::int_to_ptr: { |
| 1149 | if (!mlir::dyn_cast<cir::IntType>(Val&: srcType)) |
| 1150 | return emitOpError() << "requires !cir.int type for source" ; |
| 1151 | if (!mlir::dyn_cast<cir::PointerType>(Val&: resType)) |
| 1152 | return emitOpError() << "requires !cir.ptr type for result" ; |
| 1153 | return success(); |
| 1154 | } |
| 1155 | case cir::CastKind::ptr_to_int: { |
| 1156 | if (!mlir::dyn_cast<cir::PointerType>(Val&: srcType)) |
| 1157 | return emitOpError() << "requires !cir.ptr type for source" ; |
| 1158 | if (!mlir::dyn_cast<cir::IntType>(Val&: resType)) |
| 1159 | return emitOpError() << "requires !cir.int type for result" ; |
| 1160 | return success(); |
| 1161 | } |
| 1162 | case cir::CastKind::float_to_bool: { |
| 1163 | if (!mlir::isa<cir::FPTypeInterface>(Val: srcType)) |
| 1164 | return emitOpError() << "requires !cir.float type for source" ; |
| 1165 | if (!mlir::isa<cir::BoolType>(Val: resType)) |
| 1166 | return emitOpError() << "requires !cir.bool type for result" ; |
| 1167 | return success(); |
| 1168 | } |
| 1169 | case cir::CastKind::bool_to_int: { |
| 1170 | if (!mlir::isa<cir::BoolType>(Val: srcType)) |
| 1171 | return emitOpError() << "requires !cir.bool type for source" ; |
| 1172 | if (!mlir::isa<cir::IntType>(Val: resType)) |
| 1173 | return emitOpError() << "requires !cir.int type for result" ; |
| 1174 | return success(); |
| 1175 | } |
| 1176 | case cir::CastKind::int_to_float: { |
| 1177 | if (!mlir::isa<cir::IntType>(Val: srcType)) |
| 1178 | return emitOpError() << "requires !cir.int type for source" ; |
| 1179 | if (!mlir::isa<cir::FPTypeInterface>(Val: resType)) |
| 1180 | return emitOpError() << "requires !cir.float type for result" ; |
| 1181 | return success(); |
| 1182 | } |
| 1183 | case cir::CastKind::bool_to_float: { |
| 1184 | if (!mlir::isa<cir::BoolType>(Val: srcType)) |
| 1185 | return emitOpError() << "requires !cir.bool type for source" ; |
| 1186 | if (!mlir::isa<cir::FPTypeInterface>(Val: resType)) |
| 1187 | return emitOpError() << "requires !cir.float type for result" ; |
| 1188 | return success(); |
| 1189 | } |
| 1190 | case cir::CastKind::address_space: { |
| 1191 | auto srcPtrTy = mlir::dyn_cast<cir::PointerType>(Val&: srcType); |
| 1192 | auto resPtrTy = mlir::dyn_cast<cir::PointerType>(Val&: resType); |
| 1193 | if (!srcPtrTy || !resPtrTy) |
| 1194 | return emitOpError() << "requires !cir.ptr type for source and result" ; |
| 1195 | if (srcPtrTy.getPointee() != resPtrTy.getPointee()) |
| 1196 | return emitOpError() << "requires two types differ in addrspace only" ; |
| 1197 | return success(); |
| 1198 | } |
| 1199 | case cir::CastKind::float_to_complex: { |
| 1200 | if (!mlir::isa<cir::FPTypeInterface>(Val: srcType)) |
| 1201 | return emitOpError() << "requires !cir.float type for source" ; |
| 1202 | auto resComplexTy = mlir::dyn_cast<cir::ComplexType>(Val&: resType); |
| 1203 | if (!resComplexTy) |
| 1204 | return emitOpError() << "requires !cir.complex type for result" ; |
| 1205 | if (srcType != resComplexTy.getElementType()) |
| 1206 | return emitOpError() << "requires source type match result element type" ; |
| 1207 | return success(); |
| 1208 | } |
| 1209 | case cir::CastKind::int_to_complex: { |
| 1210 | if (!mlir::isa<cir::IntType>(Val: srcType)) |
| 1211 | return emitOpError() << "requires !cir.int type for source" ; |
| 1212 | auto resComplexTy = mlir::dyn_cast<cir::ComplexType>(Val&: resType); |
| 1213 | if (!resComplexTy) |
| 1214 | return emitOpError() << "requires !cir.complex type for result" ; |
| 1215 | if (srcType != resComplexTy.getElementType()) |
| 1216 | return emitOpError() << "requires source type match result element type" ; |
| 1217 | return success(); |
| 1218 | } |
| 1219 | case cir::CastKind::float_complex_to_real: { |
| 1220 | auto srcComplexTy = mlir::dyn_cast<cir::ComplexType>(Val&: srcType); |
| 1221 | if (!srcComplexTy) |
| 1222 | return emitOpError() << "requires !cir.complex type for source" ; |
| 1223 | if (!mlir::isa<cir::FPTypeInterface>(Val: resType)) |
| 1224 | return emitOpError() << "requires !cir.float type for result" ; |
| 1225 | if (srcComplexTy.getElementType() != resType) |
| 1226 | return emitOpError() << "requires source element type match result type" ; |
| 1227 | return success(); |
| 1228 | } |
| 1229 | case cir::CastKind::int_complex_to_real: { |
| 1230 | auto srcComplexTy = mlir::dyn_cast<cir::ComplexType>(Val&: srcType); |
| 1231 | if (!srcComplexTy) |
| 1232 | return emitOpError() << "requires !cir.complex type for source" ; |
| 1233 | if (!mlir::isa<cir::IntType>(Val: resType)) |
| 1234 | return emitOpError() << "requires !cir.int type for result" ; |
| 1235 | if (srcComplexTy.getElementType() != resType) |
| 1236 | return emitOpError() << "requires source element type match result type" ; |
| 1237 | return success(); |
| 1238 | } |
| 1239 | case cir::CastKind::float_complex_to_bool: { |
| 1240 | auto srcComplexTy = mlir::dyn_cast<cir::ComplexType>(Val&: srcType); |
| 1241 | if (!srcComplexTy || !srcComplexTy.isFloatingPointComplex()) |
| 1242 | return emitOpError() |
| 1243 | << "requires floating point !cir.complex type for source" ; |
| 1244 | if (!mlir::isa<cir::BoolType>(Val: resType)) |
| 1245 | return emitOpError() << "requires !cir.bool type for result" ; |
| 1246 | return success(); |
| 1247 | } |
| 1248 | case cir::CastKind::int_complex_to_bool: { |
| 1249 | auto srcComplexTy = mlir::dyn_cast<cir::ComplexType>(Val&: srcType); |
| 1250 | if (!srcComplexTy || !srcComplexTy.isIntegerComplex()) |
| 1251 | return emitOpError() |
| 1252 | << "requires floating point !cir.complex type for source" ; |
| 1253 | if (!mlir::isa<cir::BoolType>(Val: resType)) |
| 1254 | return emitOpError() << "requires !cir.bool type for result" ; |
| 1255 | return success(); |
| 1256 | } |
| 1257 | case cir::CastKind::float_complex: { |
| 1258 | auto srcComplexTy = mlir::dyn_cast<cir::ComplexType>(Val&: srcType); |
| 1259 | if (!srcComplexTy || !srcComplexTy.isFloatingPointComplex()) |
| 1260 | return emitOpError() |
| 1261 | << "requires floating point !cir.complex type for source" ; |
| 1262 | auto resComplexTy = mlir::dyn_cast<cir::ComplexType>(Val&: resType); |
| 1263 | if (!resComplexTy || !resComplexTy.isFloatingPointComplex()) |
| 1264 | return emitOpError() |
| 1265 | << "requires floating point !cir.complex type for result" ; |
| 1266 | return success(); |
| 1267 | } |
| 1268 | case cir::CastKind::float_complex_to_int_complex: { |
| 1269 | auto srcComplexTy = mlir::dyn_cast<cir::ComplexType>(Val&: srcType); |
| 1270 | if (!srcComplexTy || !srcComplexTy.isFloatingPointComplex()) |
| 1271 | return emitOpError() |
| 1272 | << "requires floating point !cir.complex type for source" ; |
| 1273 | auto resComplexTy = mlir::dyn_cast<cir::ComplexType>(Val&: resType); |
| 1274 | if (!resComplexTy || !resComplexTy.isIntegerComplex()) |
| 1275 | return emitOpError() << "requires integer !cir.complex type for result" ; |
| 1276 | return success(); |
| 1277 | } |
| 1278 | case cir::CastKind::int_complex: { |
| 1279 | auto srcComplexTy = mlir::dyn_cast<cir::ComplexType>(Val&: srcType); |
| 1280 | if (!srcComplexTy || !srcComplexTy.isIntegerComplex()) |
| 1281 | return emitOpError() << "requires integer !cir.complex type for source" ; |
| 1282 | auto resComplexTy = mlir::dyn_cast<cir::ComplexType>(Val&: resType); |
| 1283 | if (!resComplexTy || !resComplexTy.isIntegerComplex()) |
| 1284 | return emitOpError() << "requires integer !cir.complex type for result" ; |
| 1285 | return success(); |
| 1286 | } |
| 1287 | case cir::CastKind::int_complex_to_float_complex: { |
| 1288 | auto srcComplexTy = mlir::dyn_cast<cir::ComplexType>(Val&: srcType); |
| 1289 | if (!srcComplexTy || !srcComplexTy.isIntegerComplex()) |
| 1290 | return emitOpError() << "requires integer !cir.complex type for source" ; |
| 1291 | auto resComplexTy = mlir::dyn_cast<cir::ComplexType>(Val&: resType); |
| 1292 | if (!resComplexTy || !resComplexTy.isFloatingPointComplex()) |
| 1293 | return emitOpError() |
| 1294 | << "requires floating point !cir.complex type for result" ; |
| 1295 | return success(); |
| 1296 | } |
| 1297 | case cir::CastKind::member_ptr_to_bool: { |
| 1298 | if (!mlir::isa<cir::DataMemberType, cir::MethodType>(Val: srcType)) |
| 1299 | return emitOpError() |
| 1300 | << "requires !cir.data_member or !cir.method type for source" ; |
| 1301 | if (!mlir::isa<cir::BoolType>(Val: resType)) |
| 1302 | return emitOpError() << "requires !cir.bool type for result" ; |
| 1303 | return success(); |
| 1304 | } |
| 1305 | } |
| 1306 | llvm_unreachable("Unknown CastOp kind?" ); |
| 1307 | } |
| 1308 | |
| 1309 | static bool isIntOrBoolCast(cir::CastOp op) { |
| 1310 | auto kind = op.getKind(); |
| 1311 | return kind == cir::CastKind::bool_to_int || |
| 1312 | kind == cir::CastKind::int_to_bool || kind == cir::CastKind::integral; |
| 1313 | } |
| 1314 | |
| 1315 | static bool isCirFunctionPointerType(mlir::Type ty) { |
| 1316 | const auto ptrTy = mlir::dyn_cast<cir::PointerType>(Val&: ty); |
| 1317 | return ptrTy && mlir::isa<cir::FuncType>(Val: ptrTy.getPointee()); |
| 1318 | } |
| 1319 | |
| 1320 | static Value tryFoldCastChain(cir::CastOp op) { |
| 1321 | cir::CastOp head = op, tail = op; |
| 1322 | |
| 1323 | while (op) { |
| 1324 | if (!isIntOrBoolCast(op)) |
| 1325 | break; |
| 1326 | head = op; |
| 1327 | op = head.getSrc().getDefiningOp<cir::CastOp>(); |
| 1328 | } |
| 1329 | |
| 1330 | if (head != tail) { |
| 1331 | // if bool_to_int -> ... -> int_to_bool: take the bool |
| 1332 | // as we had it was before all casts |
| 1333 | if (head.getKind() == cir::CastKind::bool_to_int && |
| 1334 | tail.getKind() == cir::CastKind::int_to_bool) |
| 1335 | return head.getSrc(); |
| 1336 | |
| 1337 | // if int_to_bool -> ... -> int_to_bool: take the result |
| 1338 | // of the first one, as no other casts (and ext casts as well) |
| 1339 | // don't change the first result |
| 1340 | if (head.getKind() == cir::CastKind::int_to_bool && |
| 1341 | tail.getKind() == cir::CastKind::int_to_bool) |
| 1342 | return head.getResult(); |
| 1343 | |
| 1344 | return {}; |
| 1345 | } |
| 1346 | |
| 1347 | // Bitcast round-trip on function pointers: T0 -> T1 -> T0 (e.g. no-proto |
| 1348 | // redeclaration vs. actual prototype). Restrict to function pointers so |
| 1349 | // other pointer bitcast chains are unchanged. |
| 1350 | if (tail.getKind() == cir::CastKind::bitcast) { |
| 1351 | auto *inner = tail.getSrc().getDefiningOp(); |
| 1352 | if (inner && isCirFunctionPointerType(ty: tail.getType())) { |
| 1353 | auto innerCast = mlir::dyn_cast<cir::CastOp>(Val: inner); |
| 1354 | if (innerCast && innerCast.getKind() == cir::CastKind::bitcast && |
| 1355 | innerCast.getSrc().getType() == tail.getType() && |
| 1356 | innerCast.getType() == tail.getSrc().getType()) { |
| 1357 | return innerCast.getSrc(); |
| 1358 | } |
| 1359 | } |
| 1360 | } |
| 1361 | |
| 1362 | return {}; |
| 1363 | } |
| 1364 | |
| 1365 | OpFoldResult cir::CastOp::fold(FoldAdaptor adaptor) { |
| 1366 | if (mlir::isa_and_present<cir::PoisonAttr>(Val: adaptor.getSrc())) { |
| 1367 | // Propagate poison value |
| 1368 | return cir::PoisonAttr::get(context: getContext(), type: getType()); |
| 1369 | } |
| 1370 | |
| 1371 | // Propagate Undef value |
| 1372 | if (mlir::isa_and_present<cir::UndefAttr>(Val: adaptor.getSrc())) |
| 1373 | return cir::UndefAttr::get(type: getType()); |
| 1374 | |
| 1375 | if (getSrc().getType() == getType()) { |
| 1376 | switch (getKind()) { |
| 1377 | case cir::CastKind::integral: |
| 1378 | case cir::CastKind::bitcast: |
| 1379 | case cir::CastKind::address_space: |
| 1380 | case cir::CastKind::float_complex: |
| 1381 | case cir::CastKind::int_complex: { |
| 1382 | return getSrc(); |
| 1383 | } |
| 1384 | default: |
| 1385 | return {}; |
| 1386 | } |
| 1387 | } |
| 1388 | |
| 1389 | // Handle cases where a chain of casts cancel out. |
| 1390 | Value result = tryFoldCastChain(op: *this); |
| 1391 | if (result) |
| 1392 | return result; |
| 1393 | |
| 1394 | // Handle simple constant casts. |
| 1395 | if (mlir::Attribute src = adaptor.getSrc()) { |
| 1396 | switch (getKind()) { |
| 1397 | case cir::CastKind::integral: { |
| 1398 | mlir::Type srcTy = getSrc().getType(); |
| 1399 | // Don't try to fold vector casts for now. |
| 1400 | assert(mlir::isa<cir::VectorType>(srcTy) == |
| 1401 | mlir::isa<cir::VectorType>(getType())); |
| 1402 | if (mlir::isa<cir::VectorType>(Val: srcTy)) |
| 1403 | break; |
| 1404 | |
| 1405 | auto srcIntTy = mlir::cast<cir::IntType>(Val&: srcTy); |
| 1406 | auto dstIntTy = mlir::cast<cir::IntType>(Val: getType()); |
| 1407 | auto constIntAttr = mlir::dyn_cast<cir::IntAttr>(Val&: src); |
| 1408 | if (!constIntAttr) |
| 1409 | return {}; |
| 1410 | |
| 1411 | APInt srcValue = constIntAttr.getValue(); |
| 1412 | APInt newVal = srcIntTy.isSigned() |
| 1413 | ? srcValue.sextOrTrunc(width: dstIntTy.getWidth()) |
| 1414 | : srcValue.zextOrTrunc(width: dstIntTy.getWidth()); |
| 1415 | return cir::IntAttr::get(type: dstIntTy, value: newVal); |
| 1416 | } |
| 1417 | case cir::CastKind::int_to_bool: |
| 1418 | if (auto intAttr = mlir::dyn_cast<cir::IntAttr>(Val&: src)) |
| 1419 | return cir::BoolAttr::get(context: getContext(), value: !intAttr.getValue().isZero()); |
| 1420 | break; |
| 1421 | case cir::CastKind::bool_to_int: |
| 1422 | // Zero-extend the bit, which also fits a signed one-bit integer. |
| 1423 | if (auto boolAttr = mlir::dyn_cast<cir::BoolAttr>(Val&: src)) |
| 1424 | return cir::IntAttr::get( |
| 1425 | type: getType(), |
| 1426 | value: llvm::APInt(mlir::cast<cir::IntType>(Val: getType()).getWidth(), |
| 1427 | boolAttr.getValue())); |
| 1428 | break; |
| 1429 | default: |
| 1430 | break; |
| 1431 | } |
| 1432 | } |
| 1433 | return {}; |
| 1434 | } |
| 1435 | |
| 1436 | //===----------------------------------------------------------------------===// |
| 1437 | // BuiltinIntCastOp |
| 1438 | //===----------------------------------------------------------------------===// |
| 1439 | |
| 1440 | LogicalResult cir::BuiltinIntCastOp::verify() { |
| 1441 | mlir::Type srcType = getSrc().getType(); |
| 1442 | mlir::Type resType = getType(); |
| 1443 | |
| 1444 | auto srcCirInt = mlir::dyn_cast<cir::IntType>(Val&: srcType); |
| 1445 | auto resCirInt = mlir::dyn_cast<cir::IntType>(Val&: resType); |
| 1446 | |
| 1447 | // One side must be a CIR integer the other must be a builtin |
| 1448 | // integer or index type. |
| 1449 | if (static_cast<bool>(srcCirInt) == static_cast<bool>(resCirInt)) |
| 1450 | return emitOpError() |
| 1451 | << "requires exactly one '!cir.int' operand or result; the other " |
| 1452 | "must be a builtin integer or 'index' type" ; |
| 1453 | |
| 1454 | mlir::Type builtinType = srcCirInt ? resType : srcType; |
| 1455 | if (!mlir::isa<mlir::IntegerType, mlir::IndexType>(Val: builtinType)) |
| 1456 | return emitOpError() << "requires a builtin integer or 'index' type on the " |
| 1457 | "non-CIR side" ; |
| 1458 | |
| 1459 | // The cast preserves bit width. 'index' has no fixed width, so only check |
| 1460 | // when the builtin side is a fixed-width integer. |
| 1461 | if (auto builtinInt = mlir::dyn_cast<mlir::IntegerType>(Val&: builtinType)) { |
| 1462 | cir::IntType cirInt = srcCirInt ? srcCirInt : resCirInt; |
| 1463 | if (cirInt.getWidth() != builtinInt.getWidth()) |
| 1464 | return emitOpError() |
| 1465 | << "requires the CIR and builtin integer types to have the same " |
| 1466 | "width; use 'cir.cast' for width conversions" ; |
| 1467 | } |
| 1468 | |
| 1469 | return success(); |
| 1470 | } |
| 1471 | |
| 1472 | OpFoldResult cir::BuiltinIntCastOp::fold(FoldAdaptor adaptor) { |
| 1473 | // Fold: builtin_int_cast(builtin_int_cast(x)) -> x |
| 1474 | // Inner source type must match the cast's result type. |
| 1475 | if (auto inner = getSrc().getDefiningOp<cir::BuiltinIntCastOp>()) |
| 1476 | if (inner.getSrc().getType() == getType()) |
| 1477 | return inner.getSrc(); |
| 1478 | return {}; |
| 1479 | } |
| 1480 | |
| 1481 | //===----------------------------------------------------------------------===// |
| 1482 | // CallOp |
| 1483 | //===----------------------------------------------------------------------===// |
| 1484 | |
| 1485 | mlir::OperandRange cir::CallOp::getArgOperands() { |
| 1486 | if (isIndirect()) |
| 1487 | return getArgs().drop_front(n: 1); |
| 1488 | return getArgs(); |
| 1489 | } |
| 1490 | |
| 1491 | mlir::MutableOperandRange cir::CallOp::getArgOperandsMutable() { |
| 1492 | mlir::MutableOperandRange args = getArgsMutable(); |
| 1493 | if (isIndirect()) |
| 1494 | return args.slice(subStart: 1, subLen: args.size() - 1); |
| 1495 | return args; |
| 1496 | } |
| 1497 | |
| 1498 | mlir::Value cir::CallOp::getIndirectCall() { |
| 1499 | assert(isIndirect()); |
| 1500 | return getOperand(i: 0); |
| 1501 | } |
| 1502 | |
| 1503 | /// Return the operand at index 'i'. |
| 1504 | Value cir::CallOp::getArgOperand(unsigned i) { |
| 1505 | if (isIndirect()) |
| 1506 | ++i; |
| 1507 | return getOperand(i); |
| 1508 | } |
| 1509 | |
| 1510 | /// Return the number of operands. |
| 1511 | unsigned cir::CallOp::getNumArgOperands() { |
| 1512 | if (isIndirect()) |
| 1513 | return this->getOperation()->getNumOperands() - 1; |
| 1514 | return this->getOperation()->getNumOperands(); |
| 1515 | } |
| 1516 | |
| 1517 | static mlir::ParseResult |
| 1518 | parseTryCallDestinations(mlir::OpAsmParser &parser, |
| 1519 | mlir::OperationState &result) { |
| 1520 | mlir::Block *normalDestSuccessor; |
| 1521 | if (parser.parseSuccessor(dest&: normalDestSuccessor)) |
| 1522 | return mlir::failure(); |
| 1523 | |
| 1524 | if (parser.parseComma()) |
| 1525 | return mlir::failure(); |
| 1526 | |
| 1527 | mlir::Block *unwindDestSuccessor; |
| 1528 | if (parser.parseSuccessor(dest&: unwindDestSuccessor)) |
| 1529 | return mlir::failure(); |
| 1530 | |
| 1531 | result.addSuccessors(successor: normalDestSuccessor); |
| 1532 | result.addSuccessors(successor: unwindDestSuccessor); |
| 1533 | return mlir::success(); |
| 1534 | } |
| 1535 | |
| 1536 | /// Reject an attribute in an explicit attribute dictionary whose value is not |
| 1537 | /// the attribute class expected for its name. Where an op declares the |
| 1538 | /// attribute it is stored as a property, so a value of the wrong class would |
| 1539 | /// be dropped without a diagnostic. |
| 1540 | static ParseResult checkEffectAttrKinds(mlir::OpAsmParser &parser, |
| 1541 | llvm::SMLoc loc, |
| 1542 | const mlir::NamedAttrList &attrs) { |
| 1543 | if (mlir::Attribute effects = |
| 1544 | attrs.get(name: CIRDialect::getMemoryEffectsAttrName())) |
| 1545 | if (!mlir::isa<cir::MemoryEffectsAttr>(Val: effects)) |
| 1546 | return parser.emitError(loc, message: "attribute '" ) |
| 1547 | << CIRDialect::getMemoryEffectsAttrName() |
| 1548 | << "' must be a #cir.memory_effects attribute" ; |
| 1549 | |
| 1550 | if (mlir::Attribute uwtable = attrs.get(name: CIRDialect::getUwtableAttrName())) |
| 1551 | if (!mlir::isa<cir::UnwindTableKindAttr>(Val: uwtable)) |
| 1552 | return parser.emitError(loc, message: "attribute '" ) |
| 1553 | << CIRDialect::getUwtableAttrName() |
| 1554 | << "' must be a #cir.uwtable attribute" ; |
| 1555 | |
| 1556 | if (mlir::Attribute framePointer = |
| 1557 | attrs.get(name: CIRDialect::getFramePointerAttrName())) |
| 1558 | if (!mlir::isa<cir::FramePointerKindAttr>(Val: framePointer)) |
| 1559 | return parser.emitError(loc, message: "attribute '" ) |
| 1560 | << CIRDialect::getFramePointerAttrName() |
| 1561 | << "' must be a #cir.frame_pointer attribute" ; |
| 1562 | |
| 1563 | for (llvm::StringRef name : |
| 1564 | {CIRDialect::getNoUnwindAttrName(), CIRDialect::getWillReturnAttrName(), |
| 1565 | CIRDialect::getMustProgressAttrName()}) |
| 1566 | if (mlir::Attribute flag = attrs.get(name)) |
| 1567 | if (!mlir::isa<mlir::UnitAttr>(Val: flag)) |
| 1568 | return parser.emitError(loc, message: "attribute '" ) |
| 1569 | << name << "' must be a unit attribute" ; |
| 1570 | |
| 1571 | return mlir::success(); |
| 1572 | } |
| 1573 | |
| 1574 | static mlir::ParseResult parseCallCommon(mlir::OpAsmParser &parser, |
| 1575 | mlir::OperationState &result, |
| 1576 | bool hasDestinationBlocks = false) { |
| 1577 | llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand, 4> ops; |
| 1578 | llvm::SMLoc opsLoc; |
| 1579 | mlir::FlatSymbolRefAttr calleeAttr; |
| 1580 | |
| 1581 | // If we cannot parse a string callee, it means this is an indirect call. |
| 1582 | if (!parser |
| 1583 | .parseOptionalAttribute(result&: calleeAttr, attrName: CIRDialect::getCalleeAttrName(), |
| 1584 | attrs&: result.attributes) |
| 1585 | .has_value()) { |
| 1586 | OpAsmParser::UnresolvedOperand indirectVal; |
| 1587 | // Do not resolve right now, since we need to figure out the type |
| 1588 | if (parser.parseOperand(result&: indirectVal).failed()) |
| 1589 | return failure(); |
| 1590 | ops.push_back(Elt: indirectVal); |
| 1591 | } |
| 1592 | |
| 1593 | if (parser.parseLParen()) |
| 1594 | return mlir::failure(); |
| 1595 | |
| 1596 | opsLoc = parser.getCurrentLocation(); |
| 1597 | if (parser.parseOperandList(result&: ops)) |
| 1598 | return mlir::failure(); |
| 1599 | if (parser.parseRParen()) |
| 1600 | return mlir::failure(); |
| 1601 | |
| 1602 | if (hasDestinationBlocks && |
| 1603 | parseTryCallDestinations(parser, result).failed()) { |
| 1604 | return ::mlir::failure(); |
| 1605 | } |
| 1606 | |
| 1607 | if (parser.parseOptionalKeyword(keyword: "cc" ).succeeded()) { |
| 1608 | cir::CallingConv callingConv; |
| 1609 | if (parser.parseLParen().failed()) |
| 1610 | return failure(); |
| 1611 | if (parseCIRKeyword<cir::CallingConv>(parser, result&: callingConv).failed()) |
| 1612 | return parser.emitError(loc: parser.getCurrentLocation(), |
| 1613 | message: "unknown calling convention" ); |
| 1614 | if (parser.parseRParen().failed()) |
| 1615 | return failure(); |
| 1616 | result.addAttribute( |
| 1617 | name: CIRDialect::getCallingConvAttrName(), |
| 1618 | attr: cir::CallingConvAttr::get(context: parser.getContext(), value: callingConv)); |
| 1619 | } |
| 1620 | |
| 1621 | if (parser.parseOptionalKeyword(keyword: "musttail" ).succeeded()) |
| 1622 | result.addAttribute(name: CIRDialect::getMustTailAttrName(), |
| 1623 | attr: mlir::UnitAttr::get(context: parser.getContext())); |
| 1624 | |
| 1625 | if (parser.parseOptionalKeyword(keyword: "nothrow" ).succeeded()) |
| 1626 | result.addAttribute(name: CIRDialect::getNoThrowAttrName(), |
| 1627 | attr: mlir::UnitAttr::get(context: parser.getContext())); |
| 1628 | |
| 1629 | if (parser.parseOptionalKeyword(keyword: "nounwind" ).succeeded()) |
| 1630 | result.addAttribute(name: CIRDialect::getNoUnwindAttrName(), |
| 1631 | attr: mlir::UnitAttr::get(context: parser.getContext())); |
| 1632 | |
| 1633 | if (parser.parseOptionalKeyword(keyword: "willreturn" ).succeeded()) |
| 1634 | result.addAttribute(name: CIRDialect::getWillReturnAttrName(), |
| 1635 | attr: mlir::UnitAttr::get(context: parser.getContext())); |
| 1636 | |
| 1637 | llvm::SMLoc attrsLoc = parser.getCurrentLocation(); |
| 1638 | if (parser.parseOptionalAttrDict(result&: result.attributes)) |
| 1639 | return ::mlir::failure(); |
| 1640 | |
| 1641 | if (checkEffectAttrKinds(parser, loc: attrsLoc, attrs: result.attributes).failed()) |
| 1642 | return ::mlir::failure(); |
| 1643 | |
| 1644 | if (parser.parseColon()) |
| 1645 | return ::mlir::failure(); |
| 1646 | |
| 1647 | SmallVector<Type> argTypes; |
| 1648 | SmallVector<DictionaryAttr> argAttrs; |
| 1649 | SmallVector<Type> resultTypes; |
| 1650 | SmallVector<DictionaryAttr> resultAttrs; |
| 1651 | if (call_interface_impl::parseFunctionSignature(parser, argTypes, argAttrs, |
| 1652 | resultTypes, resultAttrs)) |
| 1653 | return mlir::failure(); |
| 1654 | |
| 1655 | if (resultTypes.size() > 1 || resultAttrs.size() > 1) |
| 1656 | return parser.emitError( |
| 1657 | loc: parser.getCurrentLocation(), |
| 1658 | message: "functions with multiple return types are not supported" ); |
| 1659 | |
| 1660 | result.addTypes(newTypes: resultTypes); |
| 1661 | |
| 1662 | if (parser.resolveOperands(operands&: ops, types&: argTypes, loc: opsLoc, result&: result.operands)) |
| 1663 | return mlir::failure(); |
| 1664 | |
| 1665 | if (!resultAttrs.empty() && resultAttrs[0]) |
| 1666 | result.addAttribute( |
| 1667 | name: CIRDialect::getResAttrsAttrName(), |
| 1668 | attr: mlir::ArrayAttr::get(context: parser.getContext(), value: {resultAttrs[0]})); |
| 1669 | |
| 1670 | // ArrayAttr requires a vector of 'Attribute', so we have to do the conversion |
| 1671 | // here into a separate collection. |
| 1672 | llvm::SmallVector<Attribute> convertedArgAttrs; |
| 1673 | bool argAttrsEmpty = true; |
| 1674 | |
| 1675 | llvm::transform(Range&: argAttrs, d_first: std::back_inserter(x&: convertedArgAttrs), |
| 1676 | F: [&](DictionaryAttr da) -> mlir::Attribute { |
| 1677 | if (da) |
| 1678 | argAttrsEmpty = false; |
| 1679 | return da; |
| 1680 | }); |
| 1681 | |
| 1682 | if (!argAttrsEmpty) { |
| 1683 | llvm::ArrayRef argAttrsRef = convertedArgAttrs; |
| 1684 | if (!calleeAttr) { |
| 1685 | // Fixup for indirect calls, which get an extra entry in the 'args' for |
| 1686 | // the indirect type, which doesn't get attributes. |
| 1687 | argAttrsRef = argAttrsRef.drop_front(); |
| 1688 | } |
| 1689 | result.addAttribute(name: CIRDialect::getArgAttrsAttrName(), |
| 1690 | attr: mlir::ArrayAttr::get(context: parser.getContext(), value: argAttrsRef)); |
| 1691 | } |
| 1692 | |
| 1693 | return mlir::success(); |
| 1694 | } |
| 1695 | |
| 1696 | static void printCallCommon(mlir::Operation *op, |
| 1697 | mlir::FlatSymbolRefAttr calleeSym, |
| 1698 | mlir::Value indirectCallee, |
| 1699 | mlir::OpAsmPrinter &printer, bool isNothrow, |
| 1700 | ArrayAttr argAttrs, ArrayAttr resAttrs, |
| 1701 | mlir::Block *normalDest = nullptr, |
| 1702 | mlir::Block *unwindDest = nullptr) { |
| 1703 | printer << ' '; |
| 1704 | |
| 1705 | auto callLikeOp = mlir::cast<cir::CIRCallOpInterface>(Val: op); |
| 1706 | auto ops = callLikeOp.getArgOperands(); |
| 1707 | |
| 1708 | if (calleeSym) { |
| 1709 | // Direct calls |
| 1710 | printer.printAttributeWithoutType(attr: calleeSym); |
| 1711 | } else { |
| 1712 | // Indirect calls |
| 1713 | assert(indirectCallee); |
| 1714 | printer << indirectCallee; |
| 1715 | } |
| 1716 | |
| 1717 | printer << "(" << ops << ")" ; |
| 1718 | |
| 1719 | if (normalDest) { |
| 1720 | assert(unwindDest && "expected two successors" ); |
| 1721 | auto tryCall = cast<cir::TryCallOp>(Val: op); |
| 1722 | printer << ' ' << tryCall.getNormalDest(); |
| 1723 | printer << "," ; |
| 1724 | printer << ' '; |
| 1725 | printer << tryCall.getUnwindDest(); |
| 1726 | } |
| 1727 | |
| 1728 | cir::CallingConv callingConv = callLikeOp.getCallingConv(); |
| 1729 | if (callingConv != cir::CallingConv::C) |
| 1730 | printer << " cc(" << stringifyCallingConv(val: callingConv) << ")" ; |
| 1731 | |
| 1732 | if (op->hasAttr(name: CIRDialect::getMustTailAttrName())) |
| 1733 | printer << " musttail" ; |
| 1734 | |
| 1735 | if (isNothrow) |
| 1736 | printer << " nothrow" ; |
| 1737 | |
| 1738 | if (op->hasAttr(name: CIRDialect::getNoUnwindAttrName())) |
| 1739 | printer << " nounwind" ; |
| 1740 | |
| 1741 | if (op->hasAttr(name: CIRDialect::getWillReturnAttrName())) |
| 1742 | printer << " willreturn" ; |
| 1743 | |
| 1744 | llvm::StringRef elidedAttrs[] = { |
| 1745 | CIRDialect::getCalleeAttrName(), |
| 1746 | CIRDialect::getCallingConvAttrName(), |
| 1747 | CIRDialect::getMustTailAttrName(), |
| 1748 | CIRDialect::getNoThrowAttrName(), |
| 1749 | CIRDialect::getNoUnwindAttrName(), |
| 1750 | CIRDialect::getWillReturnAttrName(), |
| 1751 | CIRDialect::getOperandSegmentSizesAttrName(), |
| 1752 | "res_attrs" , |
| 1753 | "arg_attrs" , |
| 1754 | }; |
| 1755 | // TODO: Split inherent and discardable attribute printing instead of |
| 1756 | // materializing a single dictionary that mixes the two storage classes. |
| 1757 | llvm::SmallVector<mlir::NamedAttribute> attrs; |
| 1758 | for (mlir::NamedAttribute attr : op->getDiscardableAttrs()) |
| 1759 | if (!llvm::is_contained(Range&: elidedAttrs, Element: attr.getName())) |
| 1760 | attrs.push_back(Elt: attr); |
| 1761 | op->getName().walkInherentAttrs(op, visitor: [&](llvm::StringRef name, |
| 1762 | mlir::Attribute &attr) { |
| 1763 | if (!llvm::is_contained(Range&: elidedAttrs, Element: name)) |
| 1764 | attrs.emplace_back(Args: mlir::StringAttr::get(context: op->getContext(), bytes: name), Args&: attr); |
| 1765 | }); |
| 1766 | llvm::sort(C&: attrs, Comp: [](mlir::NamedAttribute lhs, mlir::NamedAttribute rhs) { |
| 1767 | return lhs.getName().strref() < rhs.getName().strref(); |
| 1768 | }); |
| 1769 | printer.printOptionalAttrDict(attrs); |
| 1770 | printer << " : " ; |
| 1771 | if (calleeSym || !argAttrs) { |
| 1772 | call_interface_impl::printFunctionSignature( |
| 1773 | p&: printer, argTypes: op->getOperands().getTypes(), argAttrs, |
| 1774 | /*isVariadic=*/false, resultTypes: op->getResultTypes(), resultAttrs: resAttrs); |
| 1775 | } else { |
| 1776 | // indirect function calls use an 'arg' type for the type of its indirect |
| 1777 | // argument. However, we don't store a similar attribute collection. In |
| 1778 | // order to make `printFunctionSignature` have the attributes line up, we |
| 1779 | // have to make a 'shimmed' copy of the attributes that have a blank set of |
| 1780 | // attributes for the indirect argument. |
| 1781 | llvm::SmallVector<Attribute> shimmedArgAttrs; |
| 1782 | shimmedArgAttrs.push_back(Elt: mlir::DictionaryAttr::get(context: op->getContext(), value: {})); |
| 1783 | shimmedArgAttrs.append(in_start: argAttrs.begin(), in_end: argAttrs.end()); |
| 1784 | call_interface_impl::printFunctionSignature( |
| 1785 | p&: printer, argTypes: op->getOperands().getTypes(), |
| 1786 | argAttrs: mlir::ArrayAttr::get(context: op->getContext(), value: shimmedArgAttrs), |
| 1787 | /*isVariadic=*/false, resultTypes: op->getResultTypes(), resultAttrs: resAttrs); |
| 1788 | } |
| 1789 | } |
| 1790 | |
| 1791 | mlir::ParseResult cir::CallOp::parse(mlir::OpAsmParser &parser, |
| 1792 | mlir::OperationState &result) { |
| 1793 | return parseCallCommon(parser, result); |
| 1794 | } |
| 1795 | |
| 1796 | void cir::CallOp::print(mlir::OpAsmPrinter &p) { |
| 1797 | mlir::Value indirectCallee = isIndirect() ? getIndirectCall() : nullptr; |
| 1798 | printCallCommon(op: *this, calleeSym: getCalleeAttr(), indirectCallee, printer&: p, isNothrow: getNothrow(), |
| 1799 | argAttrs: getArgAttrsAttr(), resAttrs: getResAttrsAttr()); |
| 1800 | } |
| 1801 | |
| 1802 | static LogicalResult |
| 1803 | verifyCallCommInSymbolUses(mlir::Operation *op, |
| 1804 | SymbolTableCollection &symbolTable) { |
| 1805 | auto callIf = cast<cir::CIRCallOpInterface>(Val: op); |
| 1806 | |
| 1807 | // An indirect call is checked against the function type its callee pointer |
| 1808 | // points to, and a direct call against its callee's declaration. |
| 1809 | cir::FuncType fnType; |
| 1810 | bool hasPrototype = true; |
| 1811 | auto fnAttr = |
| 1812 | op->getAttrOfType<FlatSymbolRefAttr>(name: CIRDialect::getCalleeAttrName()); |
| 1813 | if (!fnAttr) { |
| 1814 | if (op->getNumOperands() == 0) |
| 1815 | return op->emitOpError(message: "indirect call requires a callee operand" ); |
| 1816 | mlir::Type calleeTy = callIf.getIndirectCall().getType(); |
| 1817 | if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(Val&: calleeTy)) |
| 1818 | fnType = mlir::dyn_cast<cir::FuncType>(Val: ptrTy.getPointee()); |
| 1819 | if (!fnType) |
| 1820 | return op->emitOpError() |
| 1821 | << "indirect callee must be a pointer to a function, but has type " |
| 1822 | << calleeTy; |
| 1823 | } else { |
| 1824 | auto fn = symbolTable.lookupNearestSymbolFrom<cir::FuncOp>(from: op, symbol: fnAttr); |
| 1825 | if (!fn) |
| 1826 | return op->emitOpError() << "'" << fnAttr.getValue() |
| 1827 | << "' does not reference a valid function" ; |
| 1828 | fnType = fn.getFunctionType(); |
| 1829 | hasPrototype = !fn.getNoProto(); |
| 1830 | } |
| 1831 | |
| 1832 | // Verify that the operand and result types match the callee. Operands are |
| 1833 | // not checked for a direct call to a function declared no_proto. |
| 1834 | if (hasPrototype) { |
| 1835 | unsigned numCallOperands = callIf.getNumArgOperands(); |
| 1836 | unsigned numFnOpOperands = fnType.getNumInputs(); |
| 1837 | |
| 1838 | if (!fnType.isVarArg() && numCallOperands != numFnOpOperands) |
| 1839 | return op->emitOpError(message: "incorrect number of operands for callee" ); |
| 1840 | if (fnType.isVarArg() && numCallOperands < numFnOpOperands) |
| 1841 | return op->emitOpError(message: "too few operands for callee" ); |
| 1842 | |
| 1843 | for (unsigned i = 0, e = numFnOpOperands; i != e; ++i) |
| 1844 | if (callIf.getArgOperand(i).getType() != fnType.getInput(i)) |
| 1845 | return op->emitOpError(message: "operand type mismatch: expected operand type " ) |
| 1846 | << fnType.getInput(i) << ", but provided " |
| 1847 | << callIf.getArgOperand(i).getType() << " for operand number " |
| 1848 | << i; |
| 1849 | } |
| 1850 | |
| 1851 | assert(!cir::MissingFeatures::opCallCallConv()); |
| 1852 | |
| 1853 | // Void function must not return any results. |
| 1854 | if (fnType.hasVoidReturn() && op->getNumResults() != 0) |
| 1855 | return op->emitOpError(message: "callee returns void but call has results" ); |
| 1856 | |
| 1857 | // Non-void function calls must return exactly one result. |
| 1858 | if (!fnType.hasVoidReturn() && op->getNumResults() != 1) |
| 1859 | return op->emitOpError(message: "incorrect number of results for callee" ); |
| 1860 | |
| 1861 | // Parent function and return value types must match. |
| 1862 | if (!fnType.hasVoidReturn() && |
| 1863 | op->getResultTypes().front() != fnType.getReturnType()) { |
| 1864 | return op->emitOpError(message: "result type mismatch: expected " ) |
| 1865 | << fnType.getReturnType() << ", but provided " |
| 1866 | << op->getResult(idx: 0).getType(); |
| 1867 | } |
| 1868 | |
| 1869 | return mlir::success(); |
| 1870 | } |
| 1871 | |
| 1872 | LogicalResult |
| 1873 | cir::CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) { |
| 1874 | return verifyCallCommInSymbolUses(op: *this, symbolTable); |
| 1875 | } |
| 1876 | |
| 1877 | //===----------------------------------------------------------------------===// |
| 1878 | // TryCallOp |
| 1879 | //===----------------------------------------------------------------------===// |
| 1880 | |
| 1881 | mlir::OperandRange cir::TryCallOp::getArgOperands() { |
| 1882 | if (isIndirect()) |
| 1883 | return getArgs().drop_front(n: 1); |
| 1884 | return getArgs(); |
| 1885 | } |
| 1886 | |
| 1887 | mlir::MutableOperandRange cir::TryCallOp::getArgOperandsMutable() { |
| 1888 | mlir::MutableOperandRange args = getArgsMutable(); |
| 1889 | if (isIndirect()) |
| 1890 | return args.slice(subStart: 1, subLen: args.size() - 1); |
| 1891 | return args; |
| 1892 | } |
| 1893 | |
| 1894 | mlir::Value cir::TryCallOp::getIndirectCall() { |
| 1895 | assert(isIndirect()); |
| 1896 | return getOperand(i: 0); |
| 1897 | } |
| 1898 | |
| 1899 | /// Return the operand at index 'i'. |
| 1900 | Value cir::TryCallOp::getArgOperand(unsigned i) { |
| 1901 | if (isIndirect()) |
| 1902 | ++i; |
| 1903 | return getOperand(i); |
| 1904 | } |
| 1905 | |
| 1906 | /// Return the number of operands. |
| 1907 | unsigned cir::TryCallOp::getNumArgOperands() { |
| 1908 | if (isIndirect()) |
| 1909 | return this->getOperation()->getNumOperands() - 1; |
| 1910 | return this->getOperation()->getNumOperands(); |
| 1911 | } |
| 1912 | |
| 1913 | LogicalResult |
| 1914 | cir::TryCallOp::verifySymbolUses(SymbolTableCollection &symbolTable) { |
| 1915 | return verifyCallCommInSymbolUses(op: *this, symbolTable); |
| 1916 | } |
| 1917 | |
| 1918 | mlir::ParseResult cir::TryCallOp::parse(mlir::OpAsmParser &parser, |
| 1919 | mlir::OperationState &result) { |
| 1920 | return parseCallCommon(parser, result, /*hasDestinationBlocks=*/true); |
| 1921 | } |
| 1922 | |
| 1923 | void cir::TryCallOp::print(::mlir::OpAsmPrinter &p) { |
| 1924 | mlir::Value indirectCallee = isIndirect() ? getIndirectCall() : nullptr; |
| 1925 | printCallCommon(op: *this, calleeSym: getCalleeAttr(), indirectCallee, printer&: p, isNothrow: getNothrow(), |
| 1926 | argAttrs: getArgAttrsAttr(), resAttrs: getResAttrsAttr(), normalDest: getNormalDest(), |
| 1927 | unwindDest: getUnwindDest()); |
| 1928 | } |
| 1929 | |
| 1930 | //===----------------------------------------------------------------------===// |
| 1931 | // ReturnOp |
| 1932 | //===----------------------------------------------------------------------===// |
| 1933 | |
| 1934 | static mlir::LogicalResult checkReturnAndFunction(cir::ReturnOp op, |
| 1935 | cir::FuncOp function) { |
| 1936 | // ReturnOps currently only have a single optional operand. |
| 1937 | if (op.getNumOperands() > 1) |
| 1938 | return op.emitOpError() << "expects at most 1 return operand" ; |
| 1939 | |
| 1940 | // Ensure returned type matches the function signature. |
| 1941 | auto expectedTy = function.getFunctionType().getReturnType(); |
| 1942 | auto actualTy = |
| 1943 | (op.getNumOperands() == 0 ? cir::VoidType::get(ctx: op.getContext()) |
| 1944 | : op.getOperand(i: 0).getType()); |
| 1945 | if (actualTy != expectedTy) |
| 1946 | return op.emitOpError() << "returns " << actualTy |
| 1947 | << " but enclosing function returns " << expectedTy; |
| 1948 | |
| 1949 | return mlir::success(); |
| 1950 | } |
| 1951 | |
| 1952 | mlir::LogicalResult cir::ReturnOp::verify() { |
| 1953 | // Returns can be present in multiple different scopes, get the |
| 1954 | // wrapping function and start from there. |
| 1955 | auto *fnOp = getOperation()->getParentOp(); |
| 1956 | while (!isa<cir::FuncOp>(Val: fnOp)) |
| 1957 | fnOp = fnOp->getParentOp(); |
| 1958 | |
| 1959 | // Make sure return types match function return type. |
| 1960 | if (checkReturnAndFunction(op: *this, function: cast<cir::FuncOp>(Val: fnOp)).failed()) |
| 1961 | return failure(); |
| 1962 | |
| 1963 | return success(); |
| 1964 | } |
| 1965 | |
| 1966 | //===----------------------------------------------------------------------===// |
| 1967 | // IfOp |
| 1968 | //===----------------------------------------------------------------------===// |
| 1969 | |
| 1970 | ParseResult cir::IfOp::parse(OpAsmParser &parser, OperationState &result) { |
| 1971 | // create the regions for 'then'. |
| 1972 | result.regions.reserve(N: 2); |
| 1973 | Region *thenRegion = result.addRegion(); |
| 1974 | Region *elseRegion = result.addRegion(); |
| 1975 | |
| 1976 | mlir::Builder &builder = parser.getBuilder(); |
| 1977 | OpAsmParser::UnresolvedOperand cond; |
| 1978 | Type boolType = cir::BoolType::get(ctx: builder.getContext()); |
| 1979 | |
| 1980 | if (parser.parseOperand(result&: cond) || |
| 1981 | parser.resolveOperand(operand: cond, type: boolType, result&: result.operands)) |
| 1982 | return failure(); |
| 1983 | |
| 1984 | // Parse 'then' region. |
| 1985 | mlir::SMLoc parseThenLoc = parser.getCurrentLocation(); |
| 1986 | if (parser.parseRegion(region&: *thenRegion, /*arguments=*/{}, /*argTypes=*/enableNameShadowing: {})) |
| 1987 | return failure(); |
| 1988 | |
| 1989 | if (ensureRegionTerm(parser, region&: *thenRegion, errLoc: parseThenLoc).failed()) |
| 1990 | return failure(); |
| 1991 | |
| 1992 | // If we find an 'else' keyword, parse the 'else' region. |
| 1993 | if (!parser.parseOptionalKeyword(keyword: "else" )) { |
| 1994 | mlir::SMLoc parseElseLoc = parser.getCurrentLocation(); |
| 1995 | if (parser.parseRegion(region&: *elseRegion, /*arguments=*/{}, /*argTypes=*/enableNameShadowing: {})) |
| 1996 | return failure(); |
| 1997 | if (ensureRegionTerm(parser, region&: *elseRegion, errLoc: parseElseLoc).failed()) |
| 1998 | return failure(); |
| 1999 | } |
| 2000 | |
| 2001 | // Parse the optional attribute list. |
| 2002 | if (parser.parseOptionalAttrDict(result&: result.attributes)) |
| 2003 | return failure(); |
| 2004 | return success(); |
| 2005 | } |
| 2006 | |
| 2007 | void cir::IfOp::print(OpAsmPrinter &p) { |
| 2008 | p << " " << getCondition() << " " ; |
| 2009 | mlir::Region &thenRegion = this->getThenRegion(); |
| 2010 | p.printRegion(blocks&: thenRegion, |
| 2011 | /*printEntryBlockArgs=*/false, |
| 2012 | /*printBlockTerminators=*/!omitRegionTerm(r&: thenRegion)); |
| 2013 | |
| 2014 | // Print the 'else' regions if it exists and has a block. |
| 2015 | mlir::Region &elseRegion = this->getElseRegion(); |
| 2016 | if (!elseRegion.empty()) { |
| 2017 | p << " else " ; |
| 2018 | p.printRegion(blocks&: elseRegion, |
| 2019 | /*printEntryBlockArgs=*/false, |
| 2020 | /*printBlockTerminators=*/!omitRegionTerm(r&: elseRegion)); |
| 2021 | } |
| 2022 | |
| 2023 | p.printOptionalAttrDict( |
| 2024 | attrs: getOperation()->getDiscardableAttrDictionary().getValue()); |
| 2025 | } |
| 2026 | |
| 2027 | /// Default callback for IfOp builders. |
| 2028 | void cir::buildTerminatedBody(OpBuilder &builder, Location loc) { |
| 2029 | // add cir.yield to end of the block |
| 2030 | cir::YieldOp::create(builder, location: loc); |
| 2031 | } |
| 2032 | |
| 2033 | /// Given the region at `index`, or the parent operation if `index` is None, |
| 2034 | /// return the successor regions. These are the regions that may be selected |
| 2035 | /// during the flow of control. `operands` is a set of optional attributes that |
| 2036 | /// correspond to a constant value for each operand, or null if that operand is |
| 2037 | /// not a constant. |
| 2038 | void cir::IfOp::getSuccessorRegions(mlir::RegionBranchPoint point, |
| 2039 | SmallVectorImpl<RegionSuccessor> ®ions) { |
| 2040 | // The `then` and the `else` region branch back to the parent operation. |
| 2041 | if (!point.isParent()) { |
| 2042 | regions.emplace_back(Args: getOperation()); |
| 2043 | return; |
| 2044 | } |
| 2045 | |
| 2046 | // Don't consider the else region if it is empty. |
| 2047 | Region *elseRegion = &this->getElseRegion(); |
| 2048 | if (elseRegion->empty()) |
| 2049 | elseRegion = nullptr; |
| 2050 | |
| 2051 | // If the condition isn't constant, both regions may be executed. |
| 2052 | regions.push_back(Elt: RegionSuccessor(&getThenRegion())); |
| 2053 | if (elseRegion) |
| 2054 | regions.push_back(Elt: RegionSuccessor(elseRegion)); |
| 2055 | else |
| 2056 | regions.emplace_back(Args: getOperation()); |
| 2057 | } |
| 2058 | |
| 2059 | void cir::IfOp::getEntrySuccessorRegions( |
| 2060 | ArrayRef<Attribute> operands, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 2061 | FoldAdaptor adaptor(operands, *this); |
| 2062 | auto cond = mlir::dyn_cast_if_present<cir::BoolAttr>(Val: adaptor.getCondition()); |
| 2063 | if (!cond) { |
| 2064 | getSuccessorRegions(point: mlir::RegionBranchPoint::parent(), regions); |
| 2065 | return; |
| 2066 | } |
| 2067 | |
| 2068 | // A constant condition selects one region. Without an else region, a false |
| 2069 | // condition skips straight to the parent operation, the same parent edge |
| 2070 | // getSuccessorRegions reports for an unknown condition. |
| 2071 | if (cond.getValue()) |
| 2072 | regions.emplace_back(Args: &getThenRegion()); |
| 2073 | else if (!getElseRegion().empty()) |
| 2074 | regions.emplace_back(Args: &getElseRegion()); |
| 2075 | else |
| 2076 | regions.emplace_back(Args: getOperation()); |
| 2077 | } |
| 2078 | |
| 2079 | void cir::IfOp::build(OpBuilder &builder, OperationState &result, Value cond, |
| 2080 | bool withElseRegion, BuilderCallbackRef thenBuilder, |
| 2081 | BuilderCallbackRef elseBuilder) { |
| 2082 | assert(thenBuilder && "the builder callback for 'then' must be present" ); |
| 2083 | result.addOperands(newOperands: cond); |
| 2084 | |
| 2085 | OpBuilder::InsertionGuard guard(builder); |
| 2086 | Region *thenRegion = result.addRegion(); |
| 2087 | builder.createBlock(parent: thenRegion); |
| 2088 | thenBuilder(builder, result.location); |
| 2089 | |
| 2090 | Region *elseRegion = result.addRegion(); |
| 2091 | if (!withElseRegion) |
| 2092 | return; |
| 2093 | |
| 2094 | builder.createBlock(parent: elseRegion); |
| 2095 | elseBuilder(builder, result.location); |
| 2096 | } |
| 2097 | |
| 2098 | //===----------------------------------------------------------------------===// |
| 2099 | // ScopeOp |
| 2100 | //===----------------------------------------------------------------------===// |
| 2101 | |
| 2102 | void cir::ScopeOp::build( |
| 2103 | OpBuilder &builder, OperationState &result, |
| 2104 | function_ref<void(OpBuilder &, Type &, Location)> scopeBuilder) { |
| 2105 | assert(scopeBuilder && "the builder callback for 'then' must be present" ); |
| 2106 | |
| 2107 | OpBuilder::InsertionGuard guard(builder); |
| 2108 | Region *scopeRegion = result.addRegion(); |
| 2109 | builder.createBlock(parent: scopeRegion); |
| 2110 | assert(!cir::MissingFeatures::opScopeCleanupRegion()); |
| 2111 | |
| 2112 | mlir::Type yieldTy; |
| 2113 | scopeBuilder(builder, yieldTy, result.location); |
| 2114 | |
| 2115 | if (yieldTy) |
| 2116 | result.addTypes(newTypes: TypeRange{yieldTy}); |
| 2117 | } |
| 2118 | |
| 2119 | void cir::ScopeOp::build( |
| 2120 | OpBuilder &builder, OperationState &result, |
| 2121 | function_ref<void(OpBuilder &, Location)> scopeBuilder) { |
| 2122 | assert(scopeBuilder && "the builder callback for 'then' must be present" ); |
| 2123 | OpBuilder::InsertionGuard guard(builder); |
| 2124 | Region *scopeRegion = result.addRegion(); |
| 2125 | builder.createBlock(parent: scopeRegion); |
| 2126 | assert(!cir::MissingFeatures::opScopeCleanupRegion()); |
| 2127 | scopeBuilder(builder, result.location); |
| 2128 | } |
| 2129 | |
| 2130 | LogicalResult cir::ScopeOp::verify() { |
| 2131 | if (getRegion().empty()) { |
| 2132 | return emitOpError() << "cir.scope must not be empty since it should " |
| 2133 | "include at least an implicit cir.yield " ; |
| 2134 | } |
| 2135 | |
| 2136 | mlir::Block &lastBlock = getRegion().back(); |
| 2137 | if (lastBlock.empty() || !lastBlock.mightHaveTerminator() || |
| 2138 | !lastBlock.getTerminator()->hasTrait<OpTrait::IsTerminator>()) |
| 2139 | return emitOpError() << "last block of cir.scope must be terminated" ; |
| 2140 | return success(); |
| 2141 | } |
| 2142 | |
| 2143 | LogicalResult cir::ScopeOp::fold(FoldAdaptor /*adaptor*/, |
| 2144 | SmallVectorImpl<OpFoldResult> &results) { |
| 2145 | // Only fold "trivial" scopes: a single block containing only a `cir.yield`. |
| 2146 | if (!getRegion().hasOneBlock()) |
| 2147 | return failure(); |
| 2148 | Block &block = getRegion().front(); |
| 2149 | if (block.getOperations().size() != 1) |
| 2150 | return failure(); |
| 2151 | |
| 2152 | auto yield = dyn_cast<cir::YieldOp>(Val&: block.front()); |
| 2153 | if (!yield) |
| 2154 | return failure(); |
| 2155 | |
| 2156 | // Only fold when the scope produces a value. |
| 2157 | if (getNumResults() != 1 || yield.getNumOperands() != 1) |
| 2158 | return failure(); |
| 2159 | |
| 2160 | results.push_back(Elt: yield.getOperand(i: 0)); |
| 2161 | return success(); |
| 2162 | } |
| 2163 | |
| 2164 | //===----------------------------------------------------------------------===// |
| 2165 | // CleanupScopeOp |
| 2166 | //===----------------------------------------------------------------------===// |
| 2167 | |
| 2168 | static bool isRedundantBeforeReturn(mlir::Region &cleanupRegion) { |
| 2169 | for (mlir::Block &block : cleanupRegion) { |
| 2170 | for (mlir::Operation &op : block) { |
| 2171 | if (isa<cir::YieldOp, cir::LifetimeEndOp, cir::StackRestoreOp>(Val: op)) |
| 2172 | continue; |
| 2173 | // The stack restore reloads the pointer saved before the VLA. |
| 2174 | auto loadOp = dyn_cast<cir::LoadOp>(Val&: op); |
| 2175 | if (loadOp && loadOp.getResult().hasOneUse() && |
| 2176 | isa<cir::StackRestoreOp>(Val: *loadOp.getResult().getUsers().begin())) |
| 2177 | continue; |
| 2178 | return false; |
| 2179 | } |
| 2180 | } |
| 2181 | return true; |
| 2182 | } |
| 2183 | |
| 2184 | LogicalResult cir::CleanupScopeOp::verify() { |
| 2185 | // If the cleanup contains a musttail call, it must be a cleanup that can be |
| 2186 | // skipped on return (such as a lifetime end or a stack restore). Other |
| 2187 | // cleanups must never contain musttail calls. |
| 2188 | cir::CallOp mustTailCall; |
| 2189 | getBodyRegion().walk(callback: [&](cir::CallOp callOp) { |
| 2190 | if (!callOp.getMusttail()) |
| 2191 | return WalkResult::advance(); |
| 2192 | mustTailCall = callOp; |
| 2193 | return WalkResult::interrupt(); |
| 2194 | }); |
| 2195 | if (!mustTailCall) |
| 2196 | return success(); |
| 2197 | |
| 2198 | if (isRedundantBeforeReturn(cleanupRegion&: getCleanupRegion())) |
| 2199 | return success(); |
| 2200 | |
| 2201 | InFlightDiagnostic diag = |
| 2202 | emitOpError(message: "cleanup is not redundant before a return, so it cannot be " |
| 2203 | "skipped by a musttail call" ); |
| 2204 | diag.attachNote(noteLoc: mustTailCall.getLoc()) << "musttail call is here" ; |
| 2205 | return diag; |
| 2206 | } |
| 2207 | |
| 2208 | void cir::CleanupScopeOp::getSuccessorRegions( |
| 2209 | mlir::RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 2210 | // The cleanup can also run from the middle of the body, on an unwind out of |
| 2211 | // a call or on a cir.return, cir.break, cir.continue or cir.goto. None of |
| 2212 | // these is a branch point, so the cleanup region is a successor of the |
| 2213 | // parent and of every body exit to stand in for them. |
| 2214 | if (point.isParent()) { |
| 2215 | regions.reserve(N: regions.size() + 2); |
| 2216 | regions.emplace_back(Args: &getBodyRegion()); |
| 2217 | regions.emplace_back(Args: &getCleanupRegion()); |
| 2218 | return; |
| 2219 | } |
| 2220 | |
| 2221 | mlir::Operation *term = point.getTerminatorPredecessorOrNull(); |
| 2222 | if (term->getParentRegion() == &getCleanupRegion()) { |
| 2223 | regions.emplace_back(Args: getOperation()); |
| 2224 | return; |
| 2225 | } |
| 2226 | |
| 2227 | regions.emplace_back(Args: &getCleanupRegion()); |
| 2228 | |
| 2229 | // cir.resume unwinds out of the body, and every other terminator is a |
| 2230 | // normal exit. An exit that the cleanup kind does not cover can also leave |
| 2231 | // the operation directly. |
| 2232 | cir::CleanupKindAttr kind = getCleanupKindAttr(); |
| 2233 | if (isa<cir::ResumeOp>(Val: term) ? !kind.isEH() : !kind.isNormal()) |
| 2234 | regions.emplace_back(Args: getOperation()); |
| 2235 | } |
| 2236 | |
| 2237 | LogicalResult cir::CleanupScopeOp::canonicalize(CleanupScopeOp op, |
| 2238 | PatternRewriter &rewriter) { |
| 2239 | auto isRegionTrivial = [](Region ®ion) { |
| 2240 | assert(!region.empty() && "CleanupScopeOp regions must not be empty" ); |
| 2241 | if (!region.hasOneBlock()) |
| 2242 | return false; |
| 2243 | Block &block = llvm::getSingleElement(C&: region); |
| 2244 | return llvm::hasSingleElement(C&: block) && |
| 2245 | isa<cir::YieldOp>(Val: llvm::getSingleElement(C&: block)); |
| 2246 | }; |
| 2247 | |
| 2248 | Region &body = op.getBodyRegion(); |
| 2249 | Region &cleanup = op.getCleanupRegion(); |
| 2250 | |
| 2251 | // An EH-only cleanup scope with an empty body can never trigger its cleanup |
| 2252 | // region — there are no operations in the body that could throw. Erase it. |
| 2253 | if (op.getCleanupKind() == CleanupKind::EH && isRegionTrivial(body)) { |
| 2254 | rewriter.eraseOp(op); |
| 2255 | return success(); |
| 2256 | } |
| 2257 | |
| 2258 | // A cleanup scope with a trivial cleanup region has no cleanup to perform. |
| 2259 | // Inline the body into the parent block and erase the scope. |
| 2260 | if (!isRegionTrivial(cleanup) || !body.hasOneBlock()) |
| 2261 | return failure(); |
| 2262 | |
| 2263 | Block &bodyBlock = body.front(); |
| 2264 | if (!isa<cir::YieldOp>(Val: bodyBlock.getTerminator())) |
| 2265 | return failure(); |
| 2266 | |
| 2267 | Operation *yield = bodyBlock.getTerminator(); |
| 2268 | rewriter.inlineBlockBefore(source: &bodyBlock, op); |
| 2269 | rewriter.eraseOp(op: yield); |
| 2270 | rewriter.eraseOp(op); |
| 2271 | return success(); |
| 2272 | } |
| 2273 | |
| 2274 | void cir::CleanupScopeOp::build( |
| 2275 | OpBuilder &builder, OperationState &result, CleanupKind cleanupKind, |
| 2276 | function_ref<void(OpBuilder &, Location)> bodyBuilder, |
| 2277 | function_ref<void(OpBuilder &, Location)> cleanupBuilder) { |
| 2278 | result.addAttribute(name: getCleanupKindAttrName(name: result.name), |
| 2279 | attr: CleanupKindAttr::get(context: builder.getContext(), value: cleanupKind)); |
| 2280 | |
| 2281 | OpBuilder::InsertionGuard guard(builder); |
| 2282 | |
| 2283 | // Build body region. |
| 2284 | Region *bodyRegion = result.addRegion(); |
| 2285 | builder.createBlock(parent: bodyRegion); |
| 2286 | if (bodyBuilder) |
| 2287 | bodyBuilder(builder, result.location); |
| 2288 | |
| 2289 | // Build cleanup region. |
| 2290 | Region *cleanupRegion = result.addRegion(); |
| 2291 | builder.createBlock(parent: cleanupRegion); |
| 2292 | if (cleanupBuilder) |
| 2293 | cleanupBuilder(builder, result.location); |
| 2294 | } |
| 2295 | |
| 2296 | //===----------------------------------------------------------------------===// |
| 2297 | // BrOp |
| 2298 | //===----------------------------------------------------------------------===// |
| 2299 | |
| 2300 | /// Merges blocks connected by a unique unconditional branch. |
| 2301 | /// |
| 2302 | /// ^bb0: ^bb0: |
| 2303 | /// ... ... |
| 2304 | /// cir.br ^bb1 => ... |
| 2305 | /// ^bb1: cir.return |
| 2306 | /// ... |
| 2307 | /// cir.return |
| 2308 | LogicalResult cir::BrOp::canonicalize(BrOp op, PatternRewriter &rewriter) { |
| 2309 | Block *src = op->getBlock(); |
| 2310 | Block *dst = op.getDest(); |
| 2311 | |
| 2312 | // Do not fold self-loops. |
| 2313 | if (src == dst) |
| 2314 | return failure(); |
| 2315 | |
| 2316 | // Only merge when this is the unique edge between the blocks. |
| 2317 | if (src->getNumSuccessors() != 1 || dst->getSinglePredecessor() != src) |
| 2318 | return failure(); |
| 2319 | |
| 2320 | // Don't merge blocks that start with LabelOp or IndirectBrOp. |
| 2321 | // This is to avoid merging blocks that have an indirect predecessor. |
| 2322 | if (isa<cir::LabelOp, cir::IndirectBrOp>(Val: dst->front())) |
| 2323 | return failure(); |
| 2324 | |
| 2325 | // An operand that is an argument of the destination itself (possible in an |
| 2326 | // unreachable cycle) would be replaced by itself, leaving its other uses |
| 2327 | // dangling once the destination is erased. |
| 2328 | if (llvm::any_of(Range: op.getDestOperands(), P: [&](Value operand) { |
| 2329 | auto arg = dyn_cast<BlockArgument>(Val&: operand); |
| 2330 | return arg && arg.getOwner() == dst; |
| 2331 | })) |
| 2332 | return failure(); |
| 2333 | |
| 2334 | // Copy the operands out: erasing the branch frees its operand storage. |
| 2335 | SmallVector<Value> operands(op.getDestOperands()); |
| 2336 | rewriter.eraseOp(op); |
| 2337 | rewriter.mergeBlocks(source: dst, dest: src, argValues: operands); |
| 2338 | return success(); |
| 2339 | } |
| 2340 | |
| 2341 | mlir::SuccessorOperands cir::BrOp::getSuccessorOperands(unsigned index) { |
| 2342 | assert(index == 0 && "invalid successor index" ); |
| 2343 | return mlir::SuccessorOperands(getDestOperandsMutable()); |
| 2344 | } |
| 2345 | |
| 2346 | Block *cir::BrOp::getSuccessorForOperands(ArrayRef<Attribute>) { |
| 2347 | return getDest(); |
| 2348 | } |
| 2349 | |
| 2350 | //===----------------------------------------------------------------------===// |
| 2351 | // IndirectBrCondOp |
| 2352 | //===----------------------------------------------------------------------===// |
| 2353 | |
| 2354 | mlir::SuccessorOperands |
| 2355 | cir::IndirectBrOp::getSuccessorOperands(unsigned index) { |
| 2356 | assert(index < getNumSuccessors() && "invalid successor index" ); |
| 2357 | return mlir::SuccessorOperands(getSuccOperandsMutable()[index]); |
| 2358 | } |
| 2359 | |
| 2360 | ParseResult cir::parseIndirectBrOpSucessors( |
| 2361 | OpAsmParser &parser, Type &flagType, |
| 2362 | SmallVectorImpl<Block *> &succOperandBlocks, |
| 2363 | SmallVectorImpl<SmallVector<OpAsmParser::UnresolvedOperand>> &succOperands, |
| 2364 | SmallVectorImpl<SmallVector<Type>> &succOperandsTypes) { |
| 2365 | if (failed(Result: parser.parseCommaSeparatedList( |
| 2366 | delimiter: OpAsmParser::Delimiter::Square, |
| 2367 | parseElementFn: [&]() { |
| 2368 | Block *destination = nullptr; |
| 2369 | SmallVector<OpAsmParser::UnresolvedOperand> operands; |
| 2370 | SmallVector<Type> operandTypes; |
| 2371 | |
| 2372 | if (parser.parseSuccessor(dest&: destination).failed()) |
| 2373 | return failure(); |
| 2374 | |
| 2375 | if (succeeded(Result: parser.parseOptionalLParen())) { |
| 2376 | if (failed(Result: parser.parseOperandList( |
| 2377 | result&: operands, delimiter: OpAsmParser::Delimiter::None)) || |
| 2378 | failed(Result: parser.parseColonTypeList(result&: operandTypes)) || |
| 2379 | failed(Result: parser.parseRParen())) |
| 2380 | return failure(); |
| 2381 | } |
| 2382 | succOperandBlocks.push_back(Elt: destination); |
| 2383 | succOperands.emplace_back(Args&: operands); |
| 2384 | succOperandsTypes.emplace_back(Args&: operandTypes); |
| 2385 | return success(); |
| 2386 | }, |
| 2387 | contextMessage: "successor blocks" ))) |
| 2388 | return failure(); |
| 2389 | return success(); |
| 2390 | } |
| 2391 | |
| 2392 | void cir::printIndirectBrOpSucessors(OpAsmPrinter &p, cir::IndirectBrOp op, |
| 2393 | Type flagType, SuccessorRange succs, |
| 2394 | OperandRangeRange succOperands, |
| 2395 | const TypeRangeRange &succOperandsTypes) { |
| 2396 | p << "[" ; |
| 2397 | llvm::interleave( |
| 2398 | c: llvm::zip(t&: succs, u&: succOperands), |
| 2399 | each_fn: [&](auto i) { |
| 2400 | p.printNewline(); |
| 2401 | p.printSuccessorAndUseList(successor: std::get<0>(i), succOperands: std::get<1>(i)); |
| 2402 | }, |
| 2403 | between_fn: [&] { p << ','; }); |
| 2404 | if (!succOperands.empty()) |
| 2405 | p.printNewline(); |
| 2406 | p << "]" ; |
| 2407 | } |
| 2408 | |
| 2409 | //===----------------------------------------------------------------------===// |
| 2410 | // BrCondOp |
| 2411 | //===----------------------------------------------------------------------===// |
| 2412 | |
| 2413 | mlir::SuccessorOperands cir::BrCondOp::getSuccessorOperands(unsigned index) { |
| 2414 | assert(index < getNumSuccessors() && "invalid successor index" ); |
| 2415 | return SuccessorOperands(index == 0 ? getDestOperandsTrueMutable() |
| 2416 | : getDestOperandsFalseMutable()); |
| 2417 | } |
| 2418 | |
| 2419 | Block *cir::BrCondOp::getSuccessorForOperands(ArrayRef<Attribute> operands) { |
| 2420 | if (IntegerAttr condAttr = dyn_cast_if_present<IntegerAttr>(Val: operands.front())) |
| 2421 | return condAttr.getValue().isOne() ? getDestTrue() : getDestFalse(); |
| 2422 | return nullptr; |
| 2423 | } |
| 2424 | |
| 2425 | //===----------------------------------------------------------------------===// |
| 2426 | // CaseOp |
| 2427 | //===----------------------------------------------------------------------===// |
| 2428 | |
| 2429 | void cir::CaseOp::build(OpBuilder &builder, OperationState &result, |
| 2430 | ArrayAttr value, CaseOpKind kind, |
| 2431 | OpBuilder::InsertPoint &insertPoint) { |
| 2432 | OpBuilder::InsertionGuard guardSwitch(builder); |
| 2433 | result.addAttribute(name: "value" , attr: value); |
| 2434 | result.getOrAddProperties<Properties>().kind = |
| 2435 | cir::CaseOpKindAttr::get(context: builder.getContext(), value: kind); |
| 2436 | Region *caseRegion = result.addRegion(); |
| 2437 | builder.createBlock(parent: caseRegion); |
| 2438 | |
| 2439 | insertPoint = builder.saveInsertionPoint(); |
| 2440 | } |
| 2441 | |
| 2442 | //===----------------------------------------------------------------------===// |
| 2443 | // SwitchOp |
| 2444 | //===----------------------------------------------------------------------===// |
| 2445 | |
| 2446 | void cir::SwitchOp::build(OpBuilder &builder, OperationState &result, |
| 2447 | Value cond, BuilderOpStateCallbackRef switchBuilder) { |
| 2448 | assert(switchBuilder && "the builder callback for regions must be present" ); |
| 2449 | OpBuilder::InsertionGuard guardSwitch(builder); |
| 2450 | Region *switchRegion = result.addRegion(); |
| 2451 | builder.createBlock(parent: switchRegion); |
| 2452 | result.addOperands(newOperands: {cond}); |
| 2453 | switchBuilder(builder, result.location, result); |
| 2454 | } |
| 2455 | |
| 2456 | void cir::SwitchOp::collectCases(llvm::SmallVectorImpl<CaseOp> &cases) { |
| 2457 | walk<mlir::WalkOrder::PreOrder>(callback: [&](mlir::Operation *op) { |
| 2458 | // Don't walk in nested switch op. |
| 2459 | if (isa<cir::SwitchOp>(Val: op) && op != *this) |
| 2460 | return WalkResult::skip(); |
| 2461 | |
| 2462 | if (auto caseOp = dyn_cast<cir::CaseOp>(Val: op)) |
| 2463 | cases.push_back(Elt: caseOp); |
| 2464 | |
| 2465 | return WalkResult::advance(); |
| 2466 | }); |
| 2467 | } |
| 2468 | |
| 2469 | bool cir::SwitchOp::isSimpleForm(llvm::SmallVectorImpl<CaseOp> &cases) { |
| 2470 | collectCases(cases); |
| 2471 | |
| 2472 | if (getBody().empty()) |
| 2473 | return false; |
| 2474 | |
| 2475 | if (!isa<YieldOp>(Val: getBody().front().back())) |
| 2476 | return false; |
| 2477 | |
| 2478 | if (!llvm::all_of(Range&: getBody().front(), |
| 2479 | P: [](Operation &op) { return isa<CaseOp, YieldOp>(Val: op); })) |
| 2480 | return false; |
| 2481 | |
| 2482 | return llvm::all_of(Range&: cases, P: [this](CaseOp op) { |
| 2483 | return op->getParentOfType<SwitchOp>() == *this; |
| 2484 | }); |
| 2485 | } |
| 2486 | |
| 2487 | //===----------------------------------------------------------------------===// |
| 2488 | // SwitchFlatOp |
| 2489 | //===----------------------------------------------------------------------===// |
| 2490 | |
| 2491 | void cir::SwitchFlatOp::build(OpBuilder &builder, OperationState &result, |
| 2492 | Value value, Block *defaultDestination, |
| 2493 | ValueRange defaultOperands, |
| 2494 | ArrayRef<APInt> caseValues, |
| 2495 | BlockRange caseDestinations, |
| 2496 | ArrayRef<ValueRange> caseOperands) { |
| 2497 | |
| 2498 | std::vector<mlir::Attribute> caseValuesAttrs; |
| 2499 | for (const APInt &val : caseValues) |
| 2500 | caseValuesAttrs.push_back(x: cir::IntAttr::get(type: value.getType(), value: val)); |
| 2501 | mlir::ArrayAttr attrs = ArrayAttr::get(context: builder.getContext(), value: caseValuesAttrs); |
| 2502 | |
| 2503 | build(odsBuilder&: builder, odsState&: result, condition: value, defaultOperands, caseOperands, caseValues: attrs, |
| 2504 | defaultDestination, caseDestinations); |
| 2505 | } |
| 2506 | |
| 2507 | SuccessorOperands cir::SwitchFlatOp::getSuccessorOperands(unsigned index) { |
| 2508 | assert(index < getNumSuccessors() && "invalid successor index" ); |
| 2509 | if (index == 0) |
| 2510 | return SuccessorOperands(getDefaultOperandsMutable()); |
| 2511 | return SuccessorOperands(getCaseOperandsMutable()[index - 1]); |
| 2512 | } |
| 2513 | |
| 2514 | Block * |
| 2515 | cir::SwitchFlatOp::getSuccessorForOperands(ArrayRef<Attribute> operands) { |
| 2516 | auto cond = dyn_cast_if_present<cir::IntAttr>(Val: operands.front()); |
| 2517 | if (!cond) |
| 2518 | return nullptr; |
| 2519 | for (auto [value, dest] : llvm::zip(t: getCaseValues(), u: getCaseDestinations())) { |
| 2520 | const APInt &caseValue = cast<cir::IntAttr>(Val: value).getValue(); |
| 2521 | if (caseValue == cond.getValue()) |
| 2522 | return dest; |
| 2523 | } |
| 2524 | return getDefaultDestination(); |
| 2525 | } |
| 2526 | |
| 2527 | /// <cases> ::= `[` (case (`,` case )* )? `]` |
| 2528 | /// <case> ::= integer `:` bb-id (`(` ssa-use-and-type-list `)`)? |
| 2529 | ParseResult cir::parseSwitchFlatOpCases( |
| 2530 | OpAsmParser &parser, Type flagType, mlir::ArrayAttr &caseValues, |
| 2531 | SmallVectorImpl<Block *> &caseDestinations, |
| 2532 | SmallVectorImpl<llvm::SmallVector<OpAsmParser::UnresolvedOperand>> |
| 2533 | &caseOperands, |
| 2534 | SmallVectorImpl<llvm::SmallVector<Type>> &caseOperandTypes) { |
| 2535 | if (failed(Result: parser.parseLSquare())) |
| 2536 | return failure(); |
| 2537 | if (succeeded(Result: parser.parseOptionalRSquare())) |
| 2538 | return success(); |
| 2539 | llvm::SmallVector<mlir::Attribute> values; |
| 2540 | |
| 2541 | auto parseCase = [&]() { |
| 2542 | int64_t value = 0; |
| 2543 | if (failed(Result: parser.parseInteger(result&: value))) |
| 2544 | return failure(); |
| 2545 | |
| 2546 | values.push_back(Elt: cir::IntAttr::get(type: flagType, value)); |
| 2547 | |
| 2548 | Block *destination; |
| 2549 | llvm::SmallVector<OpAsmParser::UnresolvedOperand> operands; |
| 2550 | llvm::SmallVector<Type> operandTypes; |
| 2551 | if (parser.parseColon() || parser.parseSuccessor(dest&: destination)) |
| 2552 | return failure(); |
| 2553 | if (!parser.parseOptionalLParen()) { |
| 2554 | if (parser.parseOperandList(result&: operands, delimiter: OpAsmParser::Delimiter::None, |
| 2555 | /*allowResultNumber=*/false) || |
| 2556 | parser.parseColonTypeList(result&: operandTypes) || parser.parseRParen()) |
| 2557 | return failure(); |
| 2558 | } |
| 2559 | caseDestinations.push_back(Elt: destination); |
| 2560 | caseOperands.emplace_back(Args&: operands); |
| 2561 | caseOperandTypes.emplace_back(Args&: operandTypes); |
| 2562 | return success(); |
| 2563 | }; |
| 2564 | if (failed(Result: parser.parseCommaSeparatedList(parseElementFn: parseCase))) |
| 2565 | return failure(); |
| 2566 | |
| 2567 | caseValues = ArrayAttr::get(context: flagType.getContext(), value: values); |
| 2568 | |
| 2569 | return parser.parseRSquare(); |
| 2570 | } |
| 2571 | |
| 2572 | void cir::printSwitchFlatOpCases(OpAsmPrinter &p, cir::SwitchFlatOp op, |
| 2573 | Type flagType, mlir::ArrayAttr caseValues, |
| 2574 | SuccessorRange caseDestinations, |
| 2575 | OperandRangeRange caseOperands, |
| 2576 | const TypeRangeRange &caseOperandTypes) { |
| 2577 | p << '['; |
| 2578 | p.printNewline(); |
| 2579 | if (!caseValues) { |
| 2580 | p << ']'; |
| 2581 | return; |
| 2582 | } |
| 2583 | |
| 2584 | size_t index = 0; |
| 2585 | llvm::interleave( |
| 2586 | c: llvm::zip(t&: caseValues, u&: caseDestinations), |
| 2587 | each_fn: [&](auto i) { |
| 2588 | p << " " ; |
| 2589 | mlir::Attribute a = std::get<0>(i); |
| 2590 | p << mlir::cast<cir::IntAttr>(Val&: a).getValue(); |
| 2591 | p << ": " ; |
| 2592 | p.printSuccessorAndUseList(successor: std::get<1>(i), succOperands: caseOperands[index++]); |
| 2593 | }, |
| 2594 | between_fn: [&] { |
| 2595 | p << ','; |
| 2596 | p.printNewline(); |
| 2597 | }); |
| 2598 | p.printNewline(); |
| 2599 | p << ']'; |
| 2600 | } |
| 2601 | |
| 2602 | //===----------------------------------------------------------------------===// |
| 2603 | // GlobalOp |
| 2604 | //===----------------------------------------------------------------------===// |
| 2605 | |
| 2606 | static ParseResult parseConstantValue(OpAsmParser &parser, |
| 2607 | mlir::Attribute &valueAttr) { |
| 2608 | NamedAttrList attr; |
| 2609 | return parser.parseAttribute(result&: valueAttr, attrName: "value" , attrs&: attr); |
| 2610 | } |
| 2611 | |
| 2612 | static void printConstant(OpAsmPrinter &p, Attribute value) { |
| 2613 | p.printAttribute(attr: value); |
| 2614 | } |
| 2615 | |
| 2616 | mlir::LogicalResult cir::GlobalOp::verify() { |
| 2617 | // A function is not an object, so it cannot be the type of a global. A |
| 2618 | // global that holds a function's address carries a pointer type instead. |
| 2619 | if (mlir::isa<cir::FuncType>(Val: getSymType())) |
| 2620 | return emitOpError(message: "global type cannot be a function type" ); |
| 2621 | |
| 2622 | // Verify that the initial value, if present, is either a unit attribute or |
| 2623 | // an attribute CIR supports. |
| 2624 | if (getInitialValue().has_value()) { |
| 2625 | if (checkConstantTypes(op: getOperation(), opType: getSymType(), attrType: *getInitialValue()) |
| 2626 | .failed()) |
| 2627 | return failure(); |
| 2628 | } |
| 2629 | |
| 2630 | // The initial value can't be recomputed from the type once the ctor is |
| 2631 | // lowered (e.g. a null member pointer becomes -1 after CXXABILowering). |
| 2632 | if (!getCtorRegion().empty() && !getInitialValue().has_value()) |
| 2633 | return emitOpError( |
| 2634 | message: "cannot have a constructor region without an initial value" ); |
| 2635 | |
| 2636 | std::optional<DynamicInitInfoAttr> info = getDynamicInitInfo(); |
| 2637 | |
| 2638 | if ((getDynamicInitGuard().has_value()) && (!info || info->getLocal()) && |
| 2639 | (!getCtorRegion().empty() || !getDtorRegion().empty())) |
| 2640 | return emitOpError( |
| 2641 | message: "Cannot have a static-local global-op with a constructor or " |
| 2642 | "destructor, they require in-function initialization via LocalInitOp" ); |
| 2643 | |
| 2644 | // CIRGen emits 'dynamic_init_guard' and 'dynamic_init_info' together and |
| 2645 | // they are only meaningful together: the guard drives lowering, which reads |
| 2646 | // the info. Require both or neither so malformed .cir can carry neither a |
| 2647 | // guard without info nor a dangling info nothing will read. |
| 2648 | if (getDynamicInitGuard().has_value() != getDynamicInitInfo().has_value()) |
| 2649 | return emitOpError(message: "'dynamic_init_guard' and 'dynamic_init_info' must be " |
| 2650 | "present together" ); |
| 2651 | |
| 2652 | if (getTlsRefs()) { |
| 2653 | // 'Unordered' TLS globals (variable template instantiations) legitimately |
| 2654 | // carry both: they can't use the shared __tls_init guard, so CIRGen also |
| 2655 | // gives them their own dynamic-init guard, mangled the same as |
| 2656 | // 'tls_refs's own guard name. 'Ordered' TLS globals share __tls_init and |
| 2657 | // have no per-variable guard name, so a dynamic-init guard there would be |
| 2658 | // a mismatch. |
| 2659 | if (getDynamicInitGuard().has_value() && !getTlsRefs()->getGuardName()) |
| 2660 | return emitOpError(message: "cannot have a dynamic-init guard combined with " |
| 2661 | "ordered tls references" ); |
| 2662 | if (!getTlsModel()) |
| 2663 | return emitOpError(message: "'tls_refs' only valid for tls" ); |
| 2664 | } |
| 2665 | |
| 2666 | if (getAliasee().has_value()) { |
| 2667 | if (getInitialValue().has_value() || !getCtorRegion().empty() || |
| 2668 | !getDtorRegion().empty()) |
| 2669 | return emitOpError(message: "global alias shall not have an initializer or " |
| 2670 | "constructor/destructor regions" ); |
| 2671 | } |
| 2672 | |
| 2673 | // TODO(CIR): Many other checks for properties that haven't been upstreamed |
| 2674 | // yet. |
| 2675 | |
| 2676 | return success(); |
| 2677 | } |
| 2678 | |
| 2679 | void cir::GlobalOp::build( |
| 2680 | OpBuilder &odsBuilder, OperationState &odsState, llvm::StringRef sym_name, |
| 2681 | mlir::Type sym_type, bool isConstant, |
| 2682 | mlir::ptr::MemorySpaceAttrInterface addrSpace, |
| 2683 | cir::GlobalLinkageKind linkage, |
| 2684 | function_ref<void(OpBuilder &, Location)> ctorBuilder, |
| 2685 | function_ref<void(OpBuilder &, Location)> dtorBuilder) { |
| 2686 | odsState.addAttribute(name: getSymNameAttrName(name: odsState.name), |
| 2687 | attr: odsBuilder.getStringAttr(bytes: sym_name)); |
| 2688 | odsState.addAttribute(name: getSymTypeAttrName(name: odsState.name), |
| 2689 | attr: mlir::TypeAttr::get(type: sym_type)); |
| 2690 | auto &properties = odsState.getOrAddProperties<cir::GlobalOp::Properties>(); |
| 2691 | properties.setConstant(isConstant); |
| 2692 | |
| 2693 | addrSpace = normalizeDefaultAddressSpace(addrSpace); |
| 2694 | if (addrSpace) |
| 2695 | odsState.addAttribute(name: getAddrSpaceAttrName(name: odsState.name), attr: addrSpace); |
| 2696 | |
| 2697 | cir::GlobalLinkageKindAttr linkageAttr = |
| 2698 | cir::GlobalLinkageKindAttr::get(context: odsBuilder.getContext(), value: linkage); |
| 2699 | odsState.addAttribute(name: getLinkageAttrName(name: odsState.name), attr: linkageAttr); |
| 2700 | |
| 2701 | Region *ctorRegion = odsState.addRegion(); |
| 2702 | if (ctorBuilder) { |
| 2703 | odsBuilder.createBlock(parent: ctorRegion); |
| 2704 | ctorBuilder(odsBuilder, odsState.location); |
| 2705 | } |
| 2706 | |
| 2707 | Region *dtorRegion = odsState.addRegion(); |
| 2708 | if (dtorBuilder) { |
| 2709 | odsBuilder.createBlock(parent: dtorRegion); |
| 2710 | dtorBuilder(odsBuilder, odsState.location); |
| 2711 | } |
| 2712 | } |
| 2713 | |
| 2714 | /// Given the region at `index`, or the parent operation if `index` is None, |
| 2715 | /// return the successor regions. These are the regions that may be selected |
| 2716 | /// during the flow of control. `operands` is a set of optional attributes that |
| 2717 | /// correspond to a constant value for each operand, or null if that operand is |
| 2718 | /// not a constant. |
| 2719 | void cir::GlobalOp::getSuccessorRegions( |
| 2720 | mlir::RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 2721 | // The `ctor` and `dtor` regions always branch back to the parent operation. |
| 2722 | if (!point.isParent()) { |
| 2723 | regions.emplace_back(Args: getOperation()); |
| 2724 | return; |
| 2725 | } |
| 2726 | |
| 2727 | // Don't consider the ctor region if it is empty. |
| 2728 | Region *ctorRegion = &this->getCtorRegion(); |
| 2729 | if (ctorRegion->empty()) |
| 2730 | ctorRegion = nullptr; |
| 2731 | |
| 2732 | // Don't consider the dtor region if it is empty. |
| 2733 | Region *dtorRegion = &this->getDtorRegion(); |
| 2734 | if (dtorRegion->empty()) |
| 2735 | dtorRegion = nullptr; |
| 2736 | |
| 2737 | // If the condition isn't constant, both regions may be executed. |
| 2738 | if (ctorRegion) |
| 2739 | regions.push_back(Elt: RegionSuccessor(ctorRegion)); |
| 2740 | if (dtorRegion) |
| 2741 | regions.push_back(Elt: RegionSuccessor(dtorRegion)); |
| 2742 | } |
| 2743 | |
| 2744 | static void printComdatName(OpAsmPrinter &p, StringAttr comdat) { |
| 2745 | if (!comdat) |
| 2746 | return; |
| 2747 | p << "comdat" ; |
| 2748 | if (!comdat.getValue().empty()) |
| 2749 | p << "(\"" << comdat.getValue() << "\")" ; |
| 2750 | } |
| 2751 | |
| 2752 | void cir::printComdatName(OpAsmPrinter &p, cir::GlobalOp op, |
| 2753 | StringAttr comdat) { |
| 2754 | ::printComdatName(p, comdat); |
| 2755 | } |
| 2756 | |
| 2757 | ParseResult cir::parseComdatName(OpAsmParser &parser, StringAttr &comdatAttr) { |
| 2758 | if (parser.parseOptionalKeyword(keyword: "comdat" ).failed()) |
| 2759 | return success(); |
| 2760 | std::string comdatKey; |
| 2761 | if (succeeded(Result: parser.parseOptionalLParen())) { |
| 2762 | if (parser.parseString(string: &comdatKey).failed() || |
| 2763 | parser.parseRParen().failed()) |
| 2764 | return failure(); |
| 2765 | } |
| 2766 | comdatAttr = parser.getBuilder().getStringAttr(bytes: comdatKey); |
| 2767 | return success(); |
| 2768 | } |
| 2769 | |
| 2770 | void cir::printGlobalOpTypeAndInitialValue(OpAsmPrinter &p, cir::GlobalOp op, |
| 2771 | TypeAttr type, Attribute initAttr, |
| 2772 | mlir::Region &ctorRegion, |
| 2773 | mlir::Region &dtorRegion) { |
| 2774 | auto printType = [&]() { p << ": " << type; }; |
| 2775 | // Aliases are definitions but they have no initial value or ctor/dtor; the |
| 2776 | // assembly prints them like declarations (`: type`). |
| 2777 | if (op.isDeclaration() || op.getAliasee()) { |
| 2778 | printType(); |
| 2779 | return; |
| 2780 | } |
| 2781 | |
| 2782 | p << "= " ; |
| 2783 | if (initAttr) |
| 2784 | printConstant(p, value: initAttr); |
| 2785 | |
| 2786 | if (!ctorRegion.empty()) { |
| 2787 | p << " ctor " ; |
| 2788 | p.printRegion(blocks&: ctorRegion, |
| 2789 | /*printEntryBlockArgs=*/false, |
| 2790 | /*printBlockTerminators=*/false); |
| 2791 | } |
| 2792 | |
| 2793 | if (!dtorRegion.empty()) { |
| 2794 | p << " dtor " ; |
| 2795 | p.printRegion(blocks&: dtorRegion, |
| 2796 | /*printEntryBlockArgs=*/false, |
| 2797 | /*printBlockTerminators=*/false); |
| 2798 | } |
| 2799 | } |
| 2800 | |
| 2801 | ParseResult cir::parseGlobalOpTypeAndInitialValue(OpAsmParser &parser, |
| 2802 | TypeAttr &typeAttr, |
| 2803 | Attribute &initialValueAttr, |
| 2804 | mlir::Region &ctorRegion, |
| 2805 | mlir::Region &dtorRegion) { |
| 2806 | mlir::Type opTy; |
| 2807 | if (parser.parseOptionalEqual().failed()) { |
| 2808 | // Absence of equal means a declaration, so we need to parse the type. |
| 2809 | // cir.global @a : !cir.int<s, 32> |
| 2810 | if (parser.parseColonType(result&: opTy)) |
| 2811 | return failure(); |
| 2812 | } else { |
| 2813 | // Parse constant with initializer, examples: |
| 2814 | // cir.global @y = 3.400000e+00 : f32 |
| 2815 | // cir.global @rgb = #cir.const_array<[...] : !cir.array<!s8i x 3>> |
| 2816 | if (parseConstantValue(parser, valueAttr&: initialValueAttr).failed()) |
| 2817 | return failure(); |
| 2818 | |
| 2819 | assert(mlir::isa<mlir::TypedAttr>(initialValueAttr) && |
| 2820 | "Non-typed attrs shouldn't appear here." ); |
| 2821 | opTy = mlir::cast<mlir::TypedAttr>(Val&: initialValueAttr).getType(); |
| 2822 | |
| 2823 | // Parse constructor, example: |
| 2824 | // cir.global @rgb = #cir.zero : type ctor { ... } |
| 2825 | if (!parser.parseOptionalKeyword(keyword: "ctor" )) { |
| 2826 | auto parseLoc = parser.getCurrentLocation(); |
| 2827 | if (parser.parseRegion(region&: ctorRegion, /*arguments=*/{}, /*argTypes=*/enableNameShadowing: {})) |
| 2828 | return failure(); |
| 2829 | if (ensureRegionTerm(parser, region&: ctorRegion, errLoc: parseLoc).failed()) |
| 2830 | return failure(); |
| 2831 | } |
| 2832 | |
| 2833 | // Parse destructor, example: |
| 2834 | // dtor { ... } |
| 2835 | if (!parser.parseOptionalKeyword(keyword: "dtor" )) { |
| 2836 | auto parseLoc = parser.getCurrentLocation(); |
| 2837 | if (parser.parseRegion(region&: dtorRegion, /*arguments=*/{}, /*argTypes=*/enableNameShadowing: {})) |
| 2838 | return failure(); |
| 2839 | if (ensureRegionTerm(parser, region&: dtorRegion, errLoc: parseLoc).failed()) |
| 2840 | return failure(); |
| 2841 | } |
| 2842 | } |
| 2843 | |
| 2844 | typeAttr = TypeAttr::get(type: opTy); |
| 2845 | return success(); |
| 2846 | } |
| 2847 | |
| 2848 | //===----------------------------------------------------------------------===// |
| 2849 | // GetGlobalOp |
| 2850 | //===----------------------------------------------------------------------===// |
| 2851 | |
| 2852 | LogicalResult |
| 2853 | cir::GetGlobalOp::verifySymbolUses(SymbolTableCollection &symbolTable) { |
| 2854 | // Verify that the result type underlying pointer type matches the type of |
| 2855 | // the referenced cir.global or cir.func op. |
| 2856 | mlir::Operation *op = |
| 2857 | symbolTable.lookupNearestSymbolFrom(from: *this, symbol: getNameAttr()); |
| 2858 | if (op == nullptr || !(isa<GlobalOp>(Val: op) || isa<FuncOp>(Val: op))) |
| 2859 | return emitOpError(message: "'" ) |
| 2860 | << getName() |
| 2861 | << "' does not reference a valid cir.global or cir.func" ; |
| 2862 | |
| 2863 | mlir::Type symTy; |
| 2864 | mlir::ptr::MemorySpaceAttrInterface symAddrSpaceAttr{}; |
| 2865 | if (auto g = dyn_cast<GlobalOp>(Val: op)) { |
| 2866 | symTy = g.getSymType(); |
| 2867 | symAddrSpaceAttr = g.getAddrSpaceAttr(); |
| 2868 | // Verify that for thread local global access, the global needs to |
| 2869 | // be marked with tls bits. |
| 2870 | if (getTls() && !g.getTlsModel()) |
| 2871 | return emitOpError(message: "access to global not marked thread local" ); |
| 2872 | |
| 2873 | // Verify that the static_local attribute on GetGlobalOp matches the |
| 2874 | // dynamic_init_guard attribute on GlobalOp. GetGlobalOp uses a UnitAttr, |
| 2875 | // GlobalOp uses DynamicInitGuardAttr. Both should be present, or neither. |
| 2876 | bool getGlobalIsStaticLocal = getStaticLocal(); |
| 2877 | bool globalIsStaticLocal = |
| 2878 | g.getDynamicInitGuard().has_value() && |
| 2879 | (!g.getDynamicInitInfo() || g.getDynamicInitInfo()->getLocal()); |
| 2880 | |
| 2881 | if (getGlobalIsStaticLocal != globalIsStaticLocal && |
| 2882 | !getOperation()->getParentOfType<cir::GlobalOp>()) |
| 2883 | return emitOpError(message: "static_local attribute mismatch" ); |
| 2884 | } else if (auto f = dyn_cast<FuncOp>(Val: op)) { |
| 2885 | symTy = f.getFunctionType(); |
| 2886 | } else { |
| 2887 | llvm_unreachable("Unexpected operation for GetGlobalOp" ); |
| 2888 | } |
| 2889 | |
| 2890 | auto resultType = dyn_cast<PointerType>(Val: getAddr().getType()); |
| 2891 | if (!resultType || symTy != resultType.getPointee()) |
| 2892 | return emitOpError(message: "result type pointee type '" ) |
| 2893 | << resultType.getPointee() << "' does not match type " << symTy |
| 2894 | << " of the global @" << getName(); |
| 2895 | |
| 2896 | // A function address may also be taken in the program address space. |
| 2897 | auto targetAS = |
| 2898 | dyn_cast_if_present<TargetAddressSpaceAttr>(Val: resultType.getAddrSpace()); |
| 2899 | bool isProgramAS = |
| 2900 | isa<FuncOp>(Val: op) && targetAS && |
| 2901 | targetAS.getValue() == |
| 2902 | CIRDataLayout(getOperation()->getParentOfType<ModuleOp>()) |
| 2903 | .getProgramAddressSpace(); |
| 2904 | if (!isProgramAS && symAddrSpaceAttr != resultType.getAddrSpace()) { |
| 2905 | return emitOpError() |
| 2906 | << "result type address space does not match the address " |
| 2907 | "space of the global @" |
| 2908 | << getName(); |
| 2909 | } |
| 2910 | |
| 2911 | return success(); |
| 2912 | } |
| 2913 | |
| 2914 | //===----------------------------------------------------------------------===// |
| 2915 | // VTableAddrPointOp |
| 2916 | //===----------------------------------------------------------------------===// |
| 2917 | |
| 2918 | LogicalResult |
| 2919 | cir::VTableAddrPointOp::verifySymbolUses(SymbolTableCollection &symbolTable) { |
| 2920 | StringRef name = getName(); |
| 2921 | |
| 2922 | // Verify that the result type underlying pointer type matches the type of |
| 2923 | // the referenced cir.global. |
| 2924 | auto op = |
| 2925 | symbolTable.lookupNearestSymbolFrom<cir::GlobalOp>(from: *this, symbol: getNameAttr()); |
| 2926 | if (!op) |
| 2927 | return emitOpError(message: "'" ) |
| 2928 | << name << "' does not reference a valid cir.global" ; |
| 2929 | std::optional<mlir::Attribute> init = op.getInitialValue(); |
| 2930 | if (!init) |
| 2931 | return success(); |
| 2932 | auto vtable = dyn_cast<cir::VTableAttr>(Val&: *init); |
| 2933 | if (!vtable) |
| 2934 | return emitOpError(message: "Expected #cir.vtable in initializer for global '" ) |
| 2935 | << name << "'" ; |
| 2936 | |
| 2937 | // The address point has to be the one that the vtable records for the |
| 2938 | // array that it points into. |
| 2939 | cir::AddressPointAttr addressPoint = getAddressPoint(); |
| 2940 | mlir::ArrayAttr addressPoints = vtable.getAddressPoints(); |
| 2941 | if (addressPoint.getIndex() < 0 || |
| 2942 | static_cast<size_t>(addressPoint.getIndex()) >= addressPoints.size()) |
| 2943 | return emitOpError(message: "address point refers to vtable " ) |
| 2944 | << addressPoint.getIndex() << ", but '" << name << "' has " |
| 2945 | << addressPoints.size() << " vtables" ; |
| 2946 | if (addressPoint != addressPoints[addressPoint.getIndex()]) |
| 2947 | return emitOpError(message: "address point " ) |
| 2948 | << addressPoint << " does not match the address point " |
| 2949 | << addressPoints[addressPoint.getIndex()] << " of '" << name << "'" ; |
| 2950 | return success(); |
| 2951 | } |
| 2952 | |
| 2953 | //===----------------------------------------------------------------------===// |
| 2954 | // VTTAddrPointOp |
| 2955 | //===----------------------------------------------------------------------===// |
| 2956 | |
| 2957 | LogicalResult |
| 2958 | cir::VTTAddrPointOp::verifySymbolUses(SymbolTableCollection &symbolTable) { |
| 2959 | // VTT ptr is not coming from a symbol. |
| 2960 | if (!getName()) |
| 2961 | return success(); |
| 2962 | StringRef name = *getName(); |
| 2963 | |
| 2964 | // Verify that the result type underlying pointer type matches the type of |
| 2965 | // the referenced cir.global op. |
| 2966 | auto op = |
| 2967 | symbolTable.lookupNearestSymbolFrom<cir::GlobalOp>(from: *this, symbol: getNameAttr()); |
| 2968 | if (!op) |
| 2969 | return emitOpError(message: "'" ) |
| 2970 | << name << "' does not reference a valid cir.global" ; |
| 2971 | auto resultType = mlir::cast<cir::PointerType>(Val: getAddr().getType()); |
| 2972 | if (op.getAddrSpaceAttr() != resultType.getAddrSpace()) |
| 2973 | return emitOpError() |
| 2974 | << "result type address space does not match the address " |
| 2975 | "space of the global @" |
| 2976 | << name; |
| 2977 | std::optional<mlir::Attribute> init = op.getInitialValue(); |
| 2978 | if (!init) |
| 2979 | return success(); |
| 2980 | if (!isa<cir::ConstArrayAttr>(Val: *init)) |
| 2981 | return emitOpError( |
| 2982 | message: "Expected constant array in initializer for global VTT '" ) |
| 2983 | << name << "'" ; |
| 2984 | return success(); |
| 2985 | } |
| 2986 | |
| 2987 | LogicalResult cir::VTTAddrPointOp::verify() { |
| 2988 | // The operation uses either a symbol or a value to operate, but not both |
| 2989 | if (getName() && getSymAddr()) |
| 2990 | return emitOpError(message: "should use either a symbol or value, but not both" ); |
| 2991 | |
| 2992 | // If not a symbol, stick with the concrete type used for getSymAddr. |
| 2993 | if (getSymAddr()) |
| 2994 | return success(); |
| 2995 | |
| 2996 | // Both pointer levels may carry an address space. |
| 2997 | auto resultType = mlir::cast<cir::PointerType>(Val: getAddr().getType()); |
| 2998 | auto eltTy = mlir::dyn_cast<cir::PointerType>(Val: resultType.getPointee()); |
| 2999 | mlir::Type resTy = cir::PointerType::get( |
| 3000 | pointee: cir::PointerType::get(pointee: cir::VoidType::get(ctx: getContext()), |
| 3001 | addrSpace: eltTy ? eltTy.getAddrSpace() : nullptr), |
| 3002 | addrSpace: resultType.getAddrSpace()); |
| 3003 | |
| 3004 | if (resultType != resTy) |
| 3005 | return emitOpError(message: "result type must be " ) |
| 3006 | << resTy << ", but provided result type is " << resultType; |
| 3007 | return success(); |
| 3008 | } |
| 3009 | |
| 3010 | //===----------------------------------------------------------------------===// |
| 3011 | // FuncOp |
| 3012 | //===----------------------------------------------------------------------===// |
| 3013 | |
| 3014 | /// Returns the name used for the linkage attribute. This *must* correspond to |
| 3015 | /// the name of the attribute in ODS. |
| 3016 | static llvm::StringRef getLinkageAttrNameString() { return "linkage" ; } |
| 3017 | |
| 3018 | /// Declared cir.func attributes with no dedicated syntax, which print in and |
| 3019 | /// parse from the attribute dictionary instead. |
| 3020 | static bool isFuncAttrInDict(llvm::StringRef name) { |
| 3021 | return name == CIRDialect::getMemoryEffectsAttrName() || |
| 3022 | name == CIRDialect::getUwtableAttrName() || |
| 3023 | name == CIRDialect::getFramePointerAttrName(); |
| 3024 | } |
| 3025 | |
| 3026 | void cir::FuncOp::build(OpBuilder &builder, OperationState &result, |
| 3027 | StringRef name, FuncType type, |
| 3028 | GlobalLinkageKind linkage, CallingConv callingConv) { |
| 3029 | result.addRegion(); |
| 3030 | result.addAttribute(name: getSymNameAttrName(name: result.name), |
| 3031 | attr: builder.getStringAttr(bytes: name)); |
| 3032 | result.addAttribute(name: getFunctionTypeAttrName(name: result.name), |
| 3033 | attr: TypeAttr::get(type)); |
| 3034 | result.addAttribute( |
| 3035 | name: getLinkageAttrNameString(), |
| 3036 | attr: GlobalLinkageKindAttr::get(context: builder.getContext(), value: linkage)); |
| 3037 | result.addAttribute(name: getCallingConvAttrName(name: result.name), |
| 3038 | attr: CallingConvAttr::get(context: builder.getContext(), value: callingConv)); |
| 3039 | // A function starts out written in source-level types, and CallConvLowering |
| 3040 | // rewrites it to the target ABI later. |
| 3041 | result.addAttribute(name: getRequiresCallConvLoweringAttrName(name: result.name), |
| 3042 | attr: builder.getUnitAttr()); |
| 3043 | } |
| 3044 | |
| 3045 | //===----------------------------------------------------------------------===// |
| 3046 | // AnnotationAttr |
| 3047 | //===----------------------------------------------------------------------===// |
| 3048 | |
| 3049 | LogicalResult |
| 3050 | cir::AnnotationAttr::verify(function_ref<InFlightDiagnostic()> emitError, |
| 3051 | mlir::StringAttr name, mlir::ArrayAttr args) { |
| 3052 | if (!args) |
| 3053 | return success(); |
| 3054 | for (mlir::Attribute arg : args) { |
| 3055 | if (!isa<mlir::StringAttr, mlir::IntegerAttr>(Val: arg)) |
| 3056 | return emitError() << "annotation args must be StringAttr or IntegerAttr," |
| 3057 | << " got " << arg; |
| 3058 | } |
| 3059 | return success(); |
| 3060 | } |
| 3061 | |
| 3062 | ParseResult cir::FuncOp::parse(OpAsmParser &parser, OperationState &state) { |
| 3063 | llvm::SMLoc loc = parser.getCurrentLocation(); |
| 3064 | mlir::Builder &builder = parser.getBuilder(); |
| 3065 | |
| 3066 | mlir::StringAttr builtinNameAttr = getBuiltinAttrName(name: state.name); |
| 3067 | mlir::StringAttr coroutineKindNameAttr = getCoroutineKindAttrName(name: state.name); |
| 3068 | mlir::StringAttr inlineKindNameAttr = getInlineKindAttrName(name: state.name); |
| 3069 | mlir::StringAttr lambdaNameAttr = getLambdaAttrName(name: state.name); |
| 3070 | mlir::StringAttr noProtoNameAttr = getNoProtoAttrName(name: state.name); |
| 3071 | mlir::StringAttr requiresCallConvLoweringNameAttr = |
| 3072 | getRequiresCallConvLoweringAttrName(name: state.name); |
| 3073 | mlir::StringAttr comdatNameAttr = getComdatAttrName(name: state.name); |
| 3074 | mlir::StringAttr alignmentNameAttr = getAlignmentAttrName(name: state.name); |
| 3075 | mlir::StringAttr preferredAlignmentNameAttr = |
| 3076 | getPreferredAlignmentAttrName(name: state.name); |
| 3077 | mlir::StringAttr visNameAttr = getSymVisibilityAttrName(name: state.name); |
| 3078 | mlir::StringAttr dsoLocalNameAttr = getDsoLocalAttrName(name: state.name); |
| 3079 | mlir::StringAttr unnamedAddrNameAttr = getUnnamedAddrAttrName(name: state.name); |
| 3080 | mlir::StringAttr funcInfoNameAttr = getFuncInfoAttrName(name: state.name); |
| 3081 | |
| 3082 | if (::mlir::succeeded(Result: parser.parseOptionalKeyword(keyword: builtinNameAttr.strref()))) |
| 3083 | state.addAttribute(name: builtinNameAttr, attr: parser.getBuilder().getUnitAttr()); |
| 3084 | |
| 3085 | // Parse optional coroutine kind attribute |
| 3086 | { |
| 3087 | static constexpr llvm::StringRef keywords[] = {"coroutine" , |
| 3088 | "flattened_coroutine" }; |
| 3089 | llvm::StringRef keyword; |
| 3090 | if (parser.parseOptionalKeyword(keyword: &keyword, allowedValues: keywords).succeeded()) |
| 3091 | state.addAttribute( |
| 3092 | name: coroutineKindNameAttr, |
| 3093 | attr: cir::CoroutineKindAttr::get(context: builder.getContext(), |
| 3094 | value: *cir::symbolizeCoroutineKind(str: keyword))); |
| 3095 | } |
| 3096 | |
| 3097 | // Parse optional inline kind attribute |
| 3098 | cir::InlineKindAttr inlineKindAttr; |
| 3099 | if (failed(Result: parseInlineKindAttr(parser, inlineKindAttr))) |
| 3100 | return failure(); |
| 3101 | if (inlineKindAttr) |
| 3102 | state.addAttribute(name: inlineKindNameAttr, attr: inlineKindAttr); |
| 3103 | |
| 3104 | if (::mlir::succeeded(Result: parser.parseOptionalKeyword(keyword: lambdaNameAttr.strref()))) |
| 3105 | state.addAttribute(name: lambdaNameAttr, attr: parser.getBuilder().getUnitAttr()); |
| 3106 | if (parser.parseOptionalKeyword(keyword: noProtoNameAttr).succeeded()) |
| 3107 | state.addAttribute(name: noProtoNameAttr, attr: parser.getBuilder().getUnitAttr()); |
| 3108 | |
| 3109 | { |
| 3110 | StringAttr comdatAttr; |
| 3111 | if (parseComdatName(parser, comdatAttr).failed()) |
| 3112 | return failure(); |
| 3113 | if (comdatAttr) |
| 3114 | state.addAttribute(name: comdatNameAttr, attr: comdatAttr); |
| 3115 | } |
| 3116 | |
| 3117 | auto parseAlignmentBody = [&](int64_t &value) { |
| 3118 | if (parser.parseLParen().failed() || parser.parseInteger(result&: value).failed() || |
| 3119 | parser.parseRParen().failed()) |
| 3120 | return failure(); |
| 3121 | |
| 3122 | if (value <= 0) |
| 3123 | return static_cast<LogicalResult>(parser.emitError( |
| 3124 | loc, message: "function alignment must be a positive integer" )); |
| 3125 | |
| 3126 | return success(); |
| 3127 | }; |
| 3128 | |
| 3129 | if (parser.parseOptionalKeyword(keyword: alignmentNameAttr).succeeded()) { |
| 3130 | int64_t value; |
| 3131 | if (parseAlignmentBody(value).failed()) |
| 3132 | return failure(); |
| 3133 | state.addAttribute(name: alignmentNameAttr, attr: builder.getI64IntegerAttr(value)); |
| 3134 | } |
| 3135 | |
| 3136 | if (parser.parseOptionalKeyword(keyword: preferredAlignmentNameAttr).succeeded()) { |
| 3137 | int64_t value; |
| 3138 | if (parseAlignmentBody(value).failed()) |
| 3139 | return failure(); |
| 3140 | state.addAttribute(name: preferredAlignmentNameAttr, |
| 3141 | attr: builder.getI64IntegerAttr(value)); |
| 3142 | } |
| 3143 | |
| 3144 | // Default to external linkage if no keyword is provided. |
| 3145 | state.addAttribute(name: getLinkageAttrNameString(), |
| 3146 | attr: GlobalLinkageKindAttr::get( |
| 3147 | context: parser.getContext(), |
| 3148 | value: parseOptionalCIRKeyword<GlobalLinkageKind>( |
| 3149 | parser, defaultValue: GlobalLinkageKind::ExternalLinkage))); |
| 3150 | |
| 3151 | ::llvm::StringRef visAttrStr; |
| 3152 | if (parser.parseOptionalKeyword(keyword: &visAttrStr, allowedValues: {"private" , "public" , "nested" }) |
| 3153 | .succeeded()) { |
| 3154 | state.addAttribute(name: visNameAttr, |
| 3155 | attr: parser.getBuilder().getStringAttr(bytes: visAttrStr)); |
| 3156 | } |
| 3157 | |
| 3158 | state.getOrAddProperties<cir::FuncOp::Properties>().global_visibility = |
| 3159 | parseOptionalCIRKeyword(parser, defaultValue: cir::VisibilityKind::Default); |
| 3160 | |
| 3161 | if (parser.parseOptionalKeyword(keyword: dsoLocalNameAttr).succeeded()) |
| 3162 | state.addAttribute(name: dsoLocalNameAttr, attr: parser.getBuilder().getUnitAttr()); |
| 3163 | |
| 3164 | if (parser.parseOptionalKeyword(keyword: unnamedAddrNameAttr).succeeded()) |
| 3165 | state.addAttribute(name: unnamedAddrNameAttr, attr: parser.getBuilder().getUnitAttr()); |
| 3166 | |
| 3167 | StringAttr nameAttr; |
| 3168 | if (parser.parseSymbolName(result&: nameAttr, attrName: getSymNameAttrName(name: state.name), |
| 3169 | attrs&: state.attributes)) |
| 3170 | return failure(); |
| 3171 | llvm::SmallVector<OpAsmParser::Argument, 8> arguments; |
| 3172 | llvm::SmallVector<mlir::Type> resultTypes; |
| 3173 | llvm::SmallVector<DictionaryAttr> resultAttrs; |
| 3174 | bool isVariadic = false; |
| 3175 | if (function_interface_impl::parseFunctionSignatureWithArguments( |
| 3176 | parser, /*allowVariadic=*/true, arguments, isVariadic, resultTypes, |
| 3177 | resultAttrs)) |
| 3178 | return failure(); |
| 3179 | llvm::SmallVector<mlir::Type> argTypes; |
| 3180 | llvm::SmallVector<mlir::Attribute> argAttrs; |
| 3181 | bool argAttrsEmpty = true; |
| 3182 | for (OpAsmParser::Argument &arg : arguments) { |
| 3183 | argTypes.push_back(Elt: arg.type); |
| 3184 | // Add the 'empty' attribute anyway to make sure the arity matches, but we |
| 3185 | // only want to 'set' the attribute at the top level if there is SOME data |
| 3186 | // along the way. |
| 3187 | argAttrs.push_back(Elt: arg.attrs); |
| 3188 | if (arg.attrs) |
| 3189 | argAttrsEmpty = false; |
| 3190 | } |
| 3191 | |
| 3192 | // These should be in sync anyway, but test both of them anyway. |
| 3193 | if (resultTypes.size() > 1 || resultAttrs.size() > 1) |
| 3194 | return parser.emitError( |
| 3195 | loc, message: "functions with multiple return types are not supported" ); |
| 3196 | |
| 3197 | mlir::Type returnType = |
| 3198 | (resultTypes.empty() ? cir::VoidType::get(ctx: builder.getContext()) |
| 3199 | : resultTypes.front()); |
| 3200 | |
| 3201 | cir::FuncType fnType = |
| 3202 | cir::FuncType::getChecked(emitError: [&]() { return parser.emitError(loc); }, |
| 3203 | inputs: argTypes, returnType, isVarArg: isVariadic); |
| 3204 | if (!fnType) |
| 3205 | return failure(); |
| 3206 | |
| 3207 | state.addAttribute(name: getFunctionTypeAttrName(name: state.name), |
| 3208 | attr: TypeAttr::get(type: fnType)); |
| 3209 | |
| 3210 | if (!resultAttrs.empty() && resultAttrs[0]) |
| 3211 | state.addAttribute( |
| 3212 | name: getResAttrsAttrName(name: state.name), |
| 3213 | attr: mlir::ArrayAttr::get(context: parser.getContext(), value: {resultAttrs[0]})); |
| 3214 | |
| 3215 | if (!argAttrsEmpty) |
| 3216 | state.addAttribute(name: getArgAttrsAttrName(name: state.name), |
| 3217 | attr: mlir::ArrayAttr::get(context: parser.getContext(), value: argAttrs)); |
| 3218 | |
| 3219 | bool hasAlias = false; |
| 3220 | mlir::StringAttr aliaseeNameAttr = getAliaseeAttrName(name: state.name); |
| 3221 | if (parser.parseOptionalKeyword(keyword: "alias" ).succeeded()) { |
| 3222 | if (parser.parseLParen().failed()) |
| 3223 | return failure(); |
| 3224 | mlir::StringAttr aliaseeAttr; |
| 3225 | if (parser.parseOptionalSymbolName(result&: aliaseeAttr).failed()) |
| 3226 | return failure(); |
| 3227 | state.addAttribute(name: aliaseeNameAttr, attr: FlatSymbolRefAttr::get(value: aliaseeAttr)); |
| 3228 | if (parser.parseRParen().failed()) |
| 3229 | return failure(); |
| 3230 | hasAlias = true; |
| 3231 | } |
| 3232 | |
| 3233 | mlir::StringAttr personalityNameAttr = getPersonalityAttrName(name: state.name); |
| 3234 | if (parser.parseOptionalKeyword(keyword: "personality" ).succeeded()) { |
| 3235 | if (parser.parseLParen().failed()) |
| 3236 | return failure(); |
| 3237 | mlir::StringAttr personalityAttr; |
| 3238 | if (parser.parseOptionalSymbolName(result&: personalityAttr).failed()) |
| 3239 | return failure(); |
| 3240 | state.addAttribute(name: personalityNameAttr, |
| 3241 | attr: FlatSymbolRefAttr::get(value: personalityAttr)); |
| 3242 | if (parser.parseRParen().failed()) |
| 3243 | return failure(); |
| 3244 | } |
| 3245 | |
| 3246 | // Default to C calling convention if no keyword is provided. |
| 3247 | mlir::StringAttr callConvNameAttr = getCallingConvAttrName(name: state.name); |
| 3248 | cir::CallingConv callConv = cir::CallingConv::C; |
| 3249 | if (parser.parseOptionalKeyword(keyword: "cc" ).succeeded()) { |
| 3250 | if (parser.parseLParen().failed()) |
| 3251 | return failure(); |
| 3252 | if (parseCIRKeyword<cir::CallingConv>(parser, result&: callConv).failed()) |
| 3253 | return parser.emitError(loc) << "unknown calling convention" ; |
| 3254 | if (parser.parseRParen().failed()) |
| 3255 | return failure(); |
| 3256 | } |
| 3257 | state.addAttribute(name: callConvNameAttr, |
| 3258 | attr: cir::CallingConvAttr::get(context: parser.getContext(), value: callConv)); |
| 3259 | |
| 3260 | auto parseGlobalDtorCtor = |
| 3261 | [&](StringRef keyword, |
| 3262 | llvm::function_ref<void(std::optional<int> prio)> createAttr) |
| 3263 | -> mlir::LogicalResult { |
| 3264 | if (mlir::succeeded(Result: parser.parseOptionalKeyword(keyword))) { |
| 3265 | std::optional<int> priority; |
| 3266 | if (mlir::succeeded(Result: parser.parseOptionalLParen())) { |
| 3267 | auto parsedPriority = mlir::FieldParser<int>::parse(parser); |
| 3268 | if (mlir::failed(Result: parsedPriority)) |
| 3269 | return parser.emitError(loc: parser.getCurrentLocation(), |
| 3270 | message: "failed to parse 'priority', of type 'int'" ); |
| 3271 | priority = parsedPriority.value_or(u: int()); |
| 3272 | // Parse literal ')' |
| 3273 | if (parser.parseRParen()) |
| 3274 | return failure(); |
| 3275 | } |
| 3276 | createAttr(priority); |
| 3277 | } |
| 3278 | return success(); |
| 3279 | }; |
| 3280 | |
| 3281 | // Parse the func_info attribute |
| 3282 | if (parser.parseOptionalKeyword(keyword: "func_info" ).succeeded()) { |
| 3283 | if (parser.parseLess().failed()) |
| 3284 | return failure(); |
| 3285 | |
| 3286 | llvm::SMLoc attrLoc = parser.getCurrentLocation(); |
| 3287 | mlir::Attribute attr; |
| 3288 | if (parser.parseAttribute(result&: attr).failed()) |
| 3289 | return failure(); |
| 3290 | if (!mlir::isa<cir::CXXCtorAttr, cir::CXXDtorAttr, cir::CXXAssignAttr, |
| 3291 | cir::FuncIdentityAttr>(Val: attr)) |
| 3292 | return parser.emitError(loc: attrLoc, |
| 3293 | message: "expected a function info attribute, got " ) |
| 3294 | << attr; |
| 3295 | state.addAttribute(name: funcInfoNameAttr, attr); |
| 3296 | |
| 3297 | if (parser.parseGreater().failed()) |
| 3298 | return failure(); |
| 3299 | } |
| 3300 | |
| 3301 | if (parseGlobalDtorCtor("global_ctor" , [&](std::optional<int> priority) { |
| 3302 | mlir::IntegerAttr globalCtorPriorityAttr = |
| 3303 | builder.getI32IntegerAttr(value: priority.value_or(u: 65535)); |
| 3304 | state.addAttribute(name: getGlobalCtorPriorityAttrName(name: state.name), |
| 3305 | attr: globalCtorPriorityAttr); |
| 3306 | }).failed()) |
| 3307 | return failure(); |
| 3308 | |
| 3309 | if (parseGlobalDtorCtor("global_dtor" , [&](std::optional<int> priority) { |
| 3310 | mlir::IntegerAttr globalDtorPriorityAttr = |
| 3311 | builder.getI32IntegerAttr(value: priority.value_or(u: 65535)); |
| 3312 | state.addAttribute(name: getGlobalDtorPriorityAttrName(name: state.name), |
| 3313 | attr: globalDtorPriorityAttr); |
| 3314 | }).failed()) |
| 3315 | return failure(); |
| 3316 | |
| 3317 | // Parse optional annotations attribute (an ArrayAttr of AnnotationAttr). |
| 3318 | mlir::StringAttr annotationsNameAttr = getAnnotationsAttrName(name: state.name); |
| 3319 | mlir::ArrayAttr annotationsAttr; |
| 3320 | if (parser.parseOptionalAttribute(result&: annotationsAttr).has_value() && |
| 3321 | annotationsAttr) |
| 3322 | state.addAttribute(name: annotationsNameAttr, attr: annotationsAttr); |
| 3323 | |
| 3324 | if (parser.parseOptionalKeyword(keyword: requiresCallConvLoweringNameAttr).succeeded()) |
| 3325 | state.addAttribute(name: requiresCallConvLoweringNameAttr, |
| 3326 | attr: parser.getBuilder().getUnitAttr()); |
| 3327 | |
| 3328 | // Parse the rest of the attributes. |
| 3329 | NamedAttrList parsedAttrs; |
| 3330 | llvm::SMLoc attrsLoc = parser.getCurrentLocation(); |
| 3331 | if (parser.parseOptionalAttrDictWithKeyword(result&: parsedAttrs)) |
| 3332 | return failure(); |
| 3333 | |
| 3334 | // Every other declared attribute has dedicated syntax above. |
| 3335 | for (StringRef disallowed : cir::FuncOp::getAttributeNames()) { |
| 3336 | if (isFuncAttrInDict(name: disallowed)) |
| 3337 | continue; |
| 3338 | if (parsedAttrs.get(name: disallowed)) |
| 3339 | return parser.emitError(loc, message: "attribute '" ) |
| 3340 | << disallowed |
| 3341 | << "' should not be specified in the explicit attribute list" ; |
| 3342 | } |
| 3343 | |
| 3344 | if (checkEffectAttrKinds(parser, loc: attrsLoc, attrs: parsedAttrs).failed()) |
| 3345 | return failure(); |
| 3346 | |
| 3347 | state.attributes.append(newAttributes&: parsedAttrs); |
| 3348 | |
| 3349 | // Parse the optional function body. |
| 3350 | auto *body = state.addRegion(); |
| 3351 | OptionalParseResult parseResult = parser.parseOptionalRegion( |
| 3352 | region&: *body, arguments, /*enableNameShadowing=*/false); |
| 3353 | if (parseResult.has_value()) { |
| 3354 | if (hasAlias) |
| 3355 | return parser.emitError(loc, message: "function alias shall not have a body" ); |
| 3356 | if (failed(Result: *parseResult)) |
| 3357 | return failure(); |
| 3358 | // Function body was parsed, make sure its not empty. |
| 3359 | if (body->empty()) |
| 3360 | return parser.emitError(loc, message: "expected non-empty function body" ); |
| 3361 | } |
| 3362 | |
| 3363 | return success(); |
| 3364 | } |
| 3365 | |
| 3366 | // This function corresponds to `llvm::GlobalValue::isDeclaration` and should |
| 3367 | // have a similar implementation. We don't currently ifuncs or materializable |
| 3368 | // functions, but those should be handled here as they are implemented. |
| 3369 | bool cir::FuncOp::isDeclaration() { |
| 3370 | assert(!cir::MissingFeatures::supportIFuncAttr()); |
| 3371 | |
| 3372 | std::optional<StringRef> aliasee = getAliasee(); |
| 3373 | if (!aliasee) |
| 3374 | return getFunctionBody().empty(); |
| 3375 | |
| 3376 | // Aliases are always definitions. |
| 3377 | return false; |
| 3378 | } |
| 3379 | |
| 3380 | bool cir::FuncOp::isCXXSpecialMemberFunction() { |
| 3381 | // The func_info union can grow forms that are not special members, so the |
| 3382 | // check names the concrete forms rather than testing for presence. |
| 3383 | mlir::Attribute attr = getFuncInfoAttr(); |
| 3384 | return attr && mlir::isa<CXXCtorAttr, CXXDtorAttr, CXXAssignAttr>(Val: attr); |
| 3385 | } |
| 3386 | |
| 3387 | bool cir::FuncOp::isCxxConstructor() { |
| 3388 | auto attr = getFuncInfoAttr(); |
| 3389 | return attr && dyn_cast<CXXCtorAttr>(Val&: attr); |
| 3390 | } |
| 3391 | |
| 3392 | bool cir::FuncOp::isCxxDestructor() { |
| 3393 | auto attr = getFuncInfoAttr(); |
| 3394 | return attr && dyn_cast<CXXDtorAttr>(Val&: attr); |
| 3395 | } |
| 3396 | |
| 3397 | bool cir::FuncOp::isCxxSpecialAssignment() { |
| 3398 | auto attr = getFuncInfoAttr(); |
| 3399 | return attr && dyn_cast<CXXAssignAttr>(Val&: attr); |
| 3400 | } |
| 3401 | |
| 3402 | std::optional<CtorKind> cir::FuncOp::getCxxConstructorKind() { |
| 3403 | mlir::Attribute attr = getFuncInfoAttr(); |
| 3404 | if (attr) { |
| 3405 | if (auto ctor = dyn_cast<CXXCtorAttr>(Val&: attr)) |
| 3406 | return ctor.getCtorKind(); |
| 3407 | } |
| 3408 | return std::nullopt; |
| 3409 | } |
| 3410 | |
| 3411 | std::optional<AssignKind> cir::FuncOp::getCxxSpecialAssignKind() { |
| 3412 | mlir::Attribute attr = getFuncInfoAttr(); |
| 3413 | if (attr) { |
| 3414 | if (auto assign = dyn_cast<CXXAssignAttr>(Val&: attr)) |
| 3415 | return assign.getAssignKind(); |
| 3416 | } |
| 3417 | return std::nullopt; |
| 3418 | } |
| 3419 | |
| 3420 | bool cir::FuncOp::isCxxTrivialMemberFunction() { |
| 3421 | mlir::Attribute attr = getFuncInfoAttr(); |
| 3422 | if (attr) { |
| 3423 | if (auto ctor = dyn_cast<CXXCtorAttr>(Val&: attr)) |
| 3424 | return ctor.getIsTrivial(); |
| 3425 | if (auto dtor = dyn_cast<CXXDtorAttr>(Val&: attr)) |
| 3426 | return dtor.getIsTrivial(); |
| 3427 | if (auto assign = dyn_cast<CXXAssignAttr>(Val&: attr)) |
| 3428 | return assign.getIsTrivial(); |
| 3429 | } |
| 3430 | return false; |
| 3431 | } |
| 3432 | |
| 3433 | mlir::Region *cir::FuncOp::getCallableRegion() { |
| 3434 | // Declarations and aliases have no body to analyze or inline. Returning the |
| 3435 | // empty region would make interprocedural analyses (e.g. SCCP) treat calls |
| 3436 | // to them as having no returns, leaving their results uninitialized. |
| 3437 | if (getBody().empty()) |
| 3438 | return nullptr; |
| 3439 | return &getBody(); |
| 3440 | } |
| 3441 | |
| 3442 | void cir::FuncOp::print(OpAsmPrinter &p) { |
| 3443 | if (getBuiltin()) |
| 3444 | p << " builtin" ; |
| 3445 | |
| 3446 | if (std::optional<cir::CoroutineKind> coroKind = getCoroutineKind()) |
| 3447 | p << ' ' << cir::stringifyCoroutineKind(val: *coroKind); |
| 3448 | |
| 3449 | printInlineKindAttr(p, inlineKindAttr: getInlineKindAttr()); |
| 3450 | |
| 3451 | if (getLambda()) |
| 3452 | p << " lambda" ; |
| 3453 | |
| 3454 | if (getNoProto()) |
| 3455 | p << " no_proto" ; |
| 3456 | |
| 3457 | if (getComdatAttr()) { |
| 3458 | p << ' '; |
| 3459 | ::printComdatName(p, comdat: getComdatAttr()); |
| 3460 | } |
| 3461 | |
| 3462 | if (getAlignment()) |
| 3463 | p << " alignment(" << *getAlignment() << ')'; |
| 3464 | |
| 3465 | if (getPreferredAlignment()) |
| 3466 | p << " preferred_alignment(" << *getPreferredAlignment() << ')'; |
| 3467 | |
| 3468 | if (getLinkage() != GlobalLinkageKind::ExternalLinkage) |
| 3469 | p << ' ' << stringifyGlobalLinkageKind(val: getLinkage()); |
| 3470 | |
| 3471 | mlir::SymbolTable::Visibility vis = getVisibility(); |
| 3472 | if (vis != mlir::SymbolTable::Visibility::Public) |
| 3473 | p << ' ' << vis; |
| 3474 | |
| 3475 | if (getGlobalVisibility() != cir::VisibilityKind::Default) |
| 3476 | p << ' ' << stringifyVisibilityKind(val: getGlobalVisibility()); |
| 3477 | |
| 3478 | if (getDsoLocal()) |
| 3479 | p << " dso_local" ; |
| 3480 | |
| 3481 | if (getUnnamedAddr()) |
| 3482 | p << " unnamed_addr" ; |
| 3483 | |
| 3484 | p << ' '; |
| 3485 | p.printSymbolName(symbolRef: getSymName()); |
| 3486 | cir::FuncType fnType = getFunctionType(); |
| 3487 | function_interface_impl::printFunctionSignature( |
| 3488 | p, op: *this, argTypes: fnType.getInputs(), isVariadic: fnType.isVarArg(), resultTypes: fnType.getReturnTypes()); |
| 3489 | |
| 3490 | if (std::optional<StringRef> aliaseeName = getAliasee()) { |
| 3491 | p << " alias(" ; |
| 3492 | p.printSymbolName(symbolRef: *aliaseeName); |
| 3493 | p << ")" ; |
| 3494 | } |
| 3495 | |
| 3496 | if (getCallingConv() != cir::CallingConv::C) { |
| 3497 | p << " cc(" ; |
| 3498 | p << stringifyCallingConv(val: getCallingConv()); |
| 3499 | p << ")" ; |
| 3500 | } |
| 3501 | |
| 3502 | if (std::optional<StringRef> personalityName = getPersonality()) { |
| 3503 | p << " personality(" ; |
| 3504 | p.printSymbolName(symbolRef: *personalityName); |
| 3505 | p << ")" ; |
| 3506 | } |
| 3507 | |
| 3508 | if (mlir::Attribute funcInfo = getFuncInfoAttr()) { |
| 3509 | p << " func_info<" ; |
| 3510 | p.printAttribute(attr: funcInfo); |
| 3511 | p << '>'; |
| 3512 | } |
| 3513 | |
| 3514 | if (auto globalCtorPriority = getGlobalCtorPriority()) { |
| 3515 | p << " global_ctor" ; |
| 3516 | if (globalCtorPriority.value() != 65535) |
| 3517 | p << "(" << globalCtorPriority.value() << ")" ; |
| 3518 | } |
| 3519 | |
| 3520 | if (auto globalDtorPriority = getGlobalDtorPriority()) { |
| 3521 | p << " global_dtor" ; |
| 3522 | if (globalDtorPriority.value() != 65535) |
| 3523 | p << "(" << globalDtorPriority.value() << ")" ; |
| 3524 | } |
| 3525 | |
| 3526 | if (mlir::ArrayAttr annotations = getAnnotationsAttr()) { |
| 3527 | p << ' '; |
| 3528 | p.printAttribute(attr: annotations); |
| 3529 | } |
| 3530 | |
| 3531 | if (getRequiresCallConvLowering()) |
| 3532 | p << ' ' << getRequiresCallConvLoweringAttrName().getValue(); |
| 3533 | |
| 3534 | // Every other declared attribute is printed by the syntax above. |
| 3535 | llvm::SmallVector<llvm::StringRef> elidedAttrs; |
| 3536 | for (llvm::StringRef name : cir::FuncOp::getAttributeNames()) |
| 3537 | if (!isFuncAttrInDict(name)) |
| 3538 | elidedAttrs.push_back(Elt: name); |
| 3539 | function_interface_impl::printFunctionAttributes(p, op: *this, elided: elidedAttrs); |
| 3540 | |
| 3541 | // Print the body if this is not an external function. |
| 3542 | Region &body = getOperation()->getRegion(index: 0); |
| 3543 | if (!body.empty()) { |
| 3544 | p << ' '; |
| 3545 | p.printRegion(blocks&: body, /*printEntryBlockArgs=*/false, |
| 3546 | /*printBlockTerminators=*/true); |
| 3547 | } |
| 3548 | } |
| 3549 | |
| 3550 | mlir::LogicalResult cir::FuncOp::verify() { |
| 3551 | |
| 3552 | if (!isDeclaration() && getCoroutineKind() == cir::CoroutineKind::Coroutine) { |
| 3553 | int coroutineOpCount = 0; |
| 3554 | this->walk(callback: [&](Operation *op) { |
| 3555 | if (isa<CoroutineOp>(Val: op)) { |
| 3556 | coroutineOpCount++; |
| 3557 | if (coroutineOpCount > 1) { |
| 3558 | return mlir::WalkResult::interrupt(); |
| 3559 | } |
| 3560 | } |
| 3561 | return mlir::WalkResult::advance(); |
| 3562 | }); |
| 3563 | if (coroutineOpCount != 1) |
| 3564 | return emitOpError() |
| 3565 | << "coroutine function must have exactly one cir.coroutine op" ; |
| 3566 | } |
| 3567 | |
| 3568 | llvm::SmallSet<llvm::StringRef, 16> labels; |
| 3569 | llvm::SmallSet<llvm::StringRef, 16> gotos; |
| 3570 | llvm::SmallSet<llvm::StringRef, 16> blockAddresses; |
| 3571 | bool invalidBlockAddress = false; |
| 3572 | getOperation()->walk(callback: [&](mlir::Operation *op) { |
| 3573 | if (auto lab = dyn_cast<cir::LabelOp>(Val: op)) { |
| 3574 | labels.insert(V: lab.getLabel()); |
| 3575 | } else if (auto goTo = dyn_cast<cir::GotoOp>(Val: op)) { |
| 3576 | gotos.insert(V: goTo.getLabel()); |
| 3577 | } else if (auto blkAdd = dyn_cast<cir::BlockAddressOp>(Val: op)) { |
| 3578 | if (blkAdd.getBlockAddrInfoAttr().getFunc().getAttr() != getSymName()) { |
| 3579 | // Stop the walk early, no need to continue |
| 3580 | invalidBlockAddress = true; |
| 3581 | return mlir::WalkResult::interrupt(); |
| 3582 | } |
| 3583 | blockAddresses.insert(V: blkAdd.getBlockAddrInfoAttr().getLabel()); |
| 3584 | } |
| 3585 | return mlir::WalkResult::advance(); |
| 3586 | }); |
| 3587 | |
| 3588 | if (invalidBlockAddress) |
| 3589 | return emitOpError() << "blockaddress references a different function" ; |
| 3590 | |
| 3591 | llvm::SmallSet<llvm::StringRef, 16> mismatched; |
| 3592 | if (!labels.empty() || !gotos.empty()) { |
| 3593 | mismatched = llvm::set_difference(S1: gotos, S2: labels); |
| 3594 | |
| 3595 | if (!mismatched.empty()) |
| 3596 | return emitOpError() << "goto/label mismatch" ; |
| 3597 | } |
| 3598 | |
| 3599 | mismatched.clear(); |
| 3600 | |
| 3601 | if (!labels.empty() || !blockAddresses.empty()) { |
| 3602 | mismatched = llvm::set_difference(S1: blockAddresses, S2: labels); |
| 3603 | |
| 3604 | if (!mismatched.empty()) |
| 3605 | return emitOpError() |
| 3606 | << "expects an existing label target in the referenced function" ; |
| 3607 | } |
| 3608 | |
| 3609 | return success(); |
| 3610 | } |
| 3611 | |
| 3612 | //===----------------------------------------------------------------------===// |
| 3613 | // TernaryOp |
| 3614 | //===----------------------------------------------------------------------===// |
| 3615 | |
| 3616 | void cir::TernaryOp::build( |
| 3617 | OpBuilder &builder, OperationState &result, Value cond, |
| 3618 | function_ref<void(OpBuilder &, Location)> trueBuilder, |
| 3619 | function_ref<void(OpBuilder &, Location)> falseBuilder) { |
| 3620 | result.addOperands(newOperands: cond); |
| 3621 | OpBuilder::InsertionGuard guard(builder); |
| 3622 | Region *trueRegion = result.addRegion(); |
| 3623 | builder.createBlock(parent: trueRegion); |
| 3624 | trueBuilder(builder, result.location); |
| 3625 | Region *falseRegion = result.addRegion(); |
| 3626 | builder.createBlock(parent: falseRegion); |
| 3627 | falseBuilder(builder, result.location); |
| 3628 | |
| 3629 | // Get result type from whichever branch has a yield (the other may have |
| 3630 | // unreachable from a throw expression) |
| 3631 | cir::YieldOp yield; |
| 3632 | if (trueRegion->back().mightHaveTerminator()) |
| 3633 | yield = dyn_cast_or_null<cir::YieldOp>(Val: trueRegion->back().getTerminator()); |
| 3634 | if (!yield && falseRegion->back().mightHaveTerminator()) |
| 3635 | yield = dyn_cast_or_null<cir::YieldOp>(Val: falseRegion->back().getTerminator()); |
| 3636 | |
| 3637 | assert((!yield || yield.getNumOperands() <= 1) && |
| 3638 | "expected zero or one result type" ); |
| 3639 | if (yield && yield.getNumOperands() == 1) |
| 3640 | result.addTypes(newTypes: TypeRange{yield.getOperandTypes().front()}); |
| 3641 | } |
| 3642 | |
| 3643 | void cir::TernaryOp::getEntrySuccessorRegions( |
| 3644 | ArrayRef<Attribute> operands, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 3645 | // A constant condition selects one region. |
| 3646 | FoldAdaptor adaptor(operands, *this); |
| 3647 | auto cond = mlir::dyn_cast_if_present<cir::BoolAttr>(Val: adaptor.getCond()); |
| 3648 | if (!cond || cond.getValue()) |
| 3649 | regions.emplace_back(Args: &getTrueRegion()); |
| 3650 | if (!cond || !cond.getValue()) |
| 3651 | regions.emplace_back(Args: &getFalseRegion()); |
| 3652 | } |
| 3653 | |
| 3654 | //===----------------------------------------------------------------------===// |
| 3655 | // SelectOp |
| 3656 | //===----------------------------------------------------------------------===// |
| 3657 | |
| 3658 | OpFoldResult cir::SelectOp::fold(FoldAdaptor adaptor) { |
| 3659 | if (auto condition = |
| 3660 | mlir::dyn_cast_if_present<cir::BoolAttr>(Val: adaptor.getCondition())) |
| 3661 | return condition.getValue() ? getTrueValue() : getFalseValue(); |
| 3662 | |
| 3663 | // cir.select if %0 then x else x -> x |
| 3664 | mlir::Attribute trueValue = adaptor.getTrueValue(); |
| 3665 | mlir::Attribute falseValue = adaptor.getFalseValue(); |
| 3666 | if (trueValue == falseValue) |
| 3667 | return trueValue; |
| 3668 | if (getTrueValue() == getFalseValue()) |
| 3669 | return getTrueValue(); |
| 3670 | |
| 3671 | return {}; |
| 3672 | } |
| 3673 | |
| 3674 | LogicalResult cir::SelectOp::verify() { |
| 3675 | // AllTypesMatch already guarantees trueVal and falseVal have matching types. |
| 3676 | auto condTy = dyn_cast<cir::VectorType>(Val: getCondition().getType()); |
| 3677 | |
| 3678 | // If condition is not a vector, no further checks are needed. |
| 3679 | if (!condTy) |
| 3680 | return success(); |
| 3681 | |
| 3682 | // When condition is a vector, both other operands must also be vectors. |
| 3683 | if (!isa<cir::VectorType>(Val: getTrueValue().getType()) || |
| 3684 | !isa<cir::VectorType>(Val: getFalseValue().getType())) { |
| 3685 | return emitOpError() |
| 3686 | << "expected both true and false operands to be vector types " |
| 3687 | "when the condition is a vector boolean type" ; |
| 3688 | } |
| 3689 | |
| 3690 | return success(); |
| 3691 | } |
| 3692 | |
| 3693 | //===----------------------------------------------------------------------===// |
| 3694 | // ShiftOp |
| 3695 | //===----------------------------------------------------------------------===// |
| 3696 | LogicalResult cir::ShiftOp::verify() { |
| 3697 | mlir::Operation *op = getOperation(); |
| 3698 | auto op0VecTy = mlir::dyn_cast<cir::VectorType>(Val: op->getOperand(idx: 0).getType()); |
| 3699 | auto op1VecTy = mlir::dyn_cast<cir::VectorType>(Val: op->getOperand(idx: 1).getType()); |
| 3700 | if (!op0VecTy ^ !op1VecTy) |
| 3701 | return emitOpError() << "input types cannot be one vector and one scalar" ; |
| 3702 | |
| 3703 | if (op0VecTy) { |
| 3704 | if (op0VecTy.getSize() != op1VecTy.getSize()) |
| 3705 | return emitOpError() << "input vector types must have the same size" ; |
| 3706 | |
| 3707 | auto opResultTy = mlir::dyn_cast<cir::VectorType>(Val: getType()); |
| 3708 | if (!opResultTy) |
| 3709 | return emitOpError() << "the type of the result must be a vector " |
| 3710 | << "if it is vector shift" ; |
| 3711 | |
| 3712 | auto op0VecEleTy = mlir::cast<cir::IntType>(Val: op0VecTy.getElementType()); |
| 3713 | auto op1VecEleTy = mlir::cast<cir::IntType>(Val: op1VecTy.getElementType()); |
| 3714 | if (op0VecEleTy.getWidth() != op1VecEleTy.getWidth()) |
| 3715 | return emitOpError() |
| 3716 | << "vector operands do not have the same elements sizes" ; |
| 3717 | |
| 3718 | auto resVecEleTy = mlir::cast<cir::IntType>(Val: opResultTy.getElementType()); |
| 3719 | if (op0VecEleTy.getWidth() != resVecEleTy.getWidth()) |
| 3720 | return emitOpError() << "vector operands and result type do not have the " |
| 3721 | "same elements sizes" ; |
| 3722 | } |
| 3723 | |
| 3724 | return mlir::success(); |
| 3725 | } |
| 3726 | |
| 3727 | //===----------------------------------------------------------------------===// |
| 3728 | // LabelOp Definitions |
| 3729 | //===----------------------------------------------------------------------===// |
| 3730 | |
| 3731 | LogicalResult cir::LabelOp::verify() { |
| 3732 | mlir::Operation *op = getOperation(); |
| 3733 | mlir::Block *blk = op->getBlock(); |
| 3734 | if (&blk->front() != op) |
| 3735 | return emitError() << "must be the first operation in a block" ; |
| 3736 | |
| 3737 | return mlir::success(); |
| 3738 | } |
| 3739 | |
| 3740 | //===----------------------------------------------------------------------===// |
| 3741 | // IncOp |
| 3742 | //===----------------------------------------------------------------------===// |
| 3743 | |
| 3744 | OpFoldResult cir::IncOp::fold(FoldAdaptor adaptor) { |
| 3745 | if (mlir::isa_and_present<cir::PoisonAttr>(Val: adaptor.getInput())) |
| 3746 | return adaptor.getInput(); |
| 3747 | return {}; |
| 3748 | } |
| 3749 | |
| 3750 | //===----------------------------------------------------------------------===// |
| 3751 | // DecOp |
| 3752 | //===----------------------------------------------------------------------===// |
| 3753 | |
| 3754 | OpFoldResult cir::DecOp::fold(FoldAdaptor adaptor) { |
| 3755 | if (mlir::isa_and_present<cir::PoisonAttr>(Val: adaptor.getInput())) |
| 3756 | return adaptor.getInput(); |
| 3757 | return {}; |
| 3758 | } |
| 3759 | |
| 3760 | //===----------------------------------------------------------------------===// |
| 3761 | // MinusOp |
| 3762 | //===----------------------------------------------------------------------===// |
| 3763 | |
| 3764 | OpFoldResult cir::MinusOp::fold(FoldAdaptor adaptor) { |
| 3765 | if (mlir::isa_and_present<cir::PoisonAttr>(Val: adaptor.getInput())) |
| 3766 | return adaptor.getInput(); |
| 3767 | |
| 3768 | // Fold with constant inputs. Floating-point negation is handled by |
| 3769 | // cir::FNegOp. |
| 3770 | if (auto intAttr = |
| 3771 | mlir::dyn_cast_if_present<cir::IntAttr>(Val: adaptor.getInput())) { |
| 3772 | APInt val = intAttr.getValue(); |
| 3773 | val.negate(); |
| 3774 | return cir::IntAttr::get(type: getType(), value: val); |
| 3775 | } |
| 3776 | |
| 3777 | return {}; |
| 3778 | } |
| 3779 | |
| 3780 | //===----------------------------------------------------------------------===// |
| 3781 | // FNegOp |
| 3782 | //===----------------------------------------------------------------------===// |
| 3783 | |
| 3784 | OpFoldResult cir::FNegOp::fold(FoldAdaptor adaptor) { |
| 3785 | if (mlir::isa_and_present<cir::PoisonAttr>(Val: adaptor.getInput())) |
| 3786 | return adaptor.getInput(); |
| 3787 | |
| 3788 | // Fold with constant inputs. |
| 3789 | if (auto fpAttr = |
| 3790 | mlir::dyn_cast_if_present<cir::FPAttr>(Val: adaptor.getInput())) { |
| 3791 | APFloat val = fpAttr.getValue(); |
| 3792 | val.changeSign(); |
| 3793 | return cir::FPAttr::get(type: getType(), value: val); |
| 3794 | } |
| 3795 | |
| 3796 | return {}; |
| 3797 | } |
| 3798 | |
| 3799 | //===----------------------------------------------------------------------===// |
| 3800 | // NotOp |
| 3801 | //===----------------------------------------------------------------------===// |
| 3802 | |
| 3803 | OpFoldResult cir::NotOp::fold(FoldAdaptor adaptor) { |
| 3804 | if (mlir::isa_and_present<cir::PoisonAttr>(Val: adaptor.getInput())) |
| 3805 | return adaptor.getInput(); |
| 3806 | |
| 3807 | // not(not(x)) -> x is handled by the Involution trait. |
| 3808 | |
| 3809 | // Fold with constant inputs. |
| 3810 | if (mlir::Attribute attr = adaptor.getInput()) { |
| 3811 | if (auto intAttr = mlir::dyn_cast<cir::IntAttr>(Val&: attr)) { |
| 3812 | APInt val = intAttr.getValue(); |
| 3813 | val.flipAllBits(); |
| 3814 | return cir::IntAttr::get(type: getType(), value: val); |
| 3815 | } |
| 3816 | if (auto boolAttr = mlir::dyn_cast<cir::BoolAttr>(Val&: attr)) |
| 3817 | return cir::BoolAttr::get(context: getContext(), value: !boolAttr.getValue()); |
| 3818 | } |
| 3819 | |
| 3820 | return {}; |
| 3821 | } |
| 3822 | |
| 3823 | //===----------------------------------------------------------------------===// |
| 3824 | // BaseClassAddrOp & DerivedClassAddrOp |
| 3825 | //===----------------------------------------------------------------------===// |
| 3826 | |
| 3827 | LogicalResult cir::BaseClassAddrOp::verify() { |
| 3828 | if (getSrcAddr().getType().getAddrSpace() != getType().getAddrSpace()) |
| 3829 | return emitOpError() << "address space mismatch" ; |
| 3830 | return mlir::success(); |
| 3831 | } |
| 3832 | |
| 3833 | LogicalResult cir::DerivedClassAddrOp::verify() { |
| 3834 | if (getSrcAddr().getType().getAddrSpace() != getType().getAddrSpace()) |
| 3835 | return emitOpError() << "address space mismatch" ; |
| 3836 | return mlir::success(); |
| 3837 | } |
| 3838 | |
| 3839 | //===----------------------------------------------------------------------===// |
| 3840 | // BaseDataMemberOp & DerivedDataMemberOp |
| 3841 | //===----------------------------------------------------------------------===// |
| 3842 | |
| 3843 | static LogicalResult verifyMemberPtrCast(Operation *op, mlir::Value src, |
| 3844 | mlir::Type resultTy) { |
| 3845 | // Let the operand type be T1 C1::*, let the result type be T2 C2::*. |
| 3846 | // Verify that T1 and T2 are the same type. |
| 3847 | mlir::Type inputMemberTy; |
| 3848 | mlir::Type resultMemberTy; |
| 3849 | if (mlir::isa<cir::DataMemberType>(Val: src.getType())) { |
| 3850 | inputMemberTy = |
| 3851 | mlir::cast<cir::DataMemberType>(Val: src.getType()).getMemberTy(); |
| 3852 | resultMemberTy = mlir::cast<cir::DataMemberType>(Val&: resultTy).getMemberTy(); |
| 3853 | } |
| 3854 | assert(!cir::MissingFeatures::memberFuncPtrCast()); |
| 3855 | if (inputMemberTy != resultMemberTy) |
| 3856 | return op->emitOpError() |
| 3857 | << "member types of the operand and the result do not match" ; |
| 3858 | |
| 3859 | return mlir::success(); |
| 3860 | } |
| 3861 | |
| 3862 | LogicalResult cir::BaseDataMemberOp::verify() { |
| 3863 | return verifyMemberPtrCast(op: getOperation(), src: getSrc(), resultTy: getType()); |
| 3864 | } |
| 3865 | |
| 3866 | LogicalResult cir::DerivedDataMemberOp::verify() { |
| 3867 | return verifyMemberPtrCast(op: getOperation(), src: getSrc(), resultTy: getType()); |
| 3868 | } |
| 3869 | |
| 3870 | //===----------------------------------------------------------------------===// |
| 3871 | // BaseMethodOp & DerivedMethodOp |
| 3872 | //===----------------------------------------------------------------------===// |
| 3873 | |
| 3874 | LogicalResult cir::BaseMethodOp::verify() { |
| 3875 | return verifyMemberPtrCast(op: getOperation(), src: getSrc(), resultTy: getType()); |
| 3876 | } |
| 3877 | |
| 3878 | LogicalResult cir::DerivedMethodOp::verify() { |
| 3879 | return verifyMemberPtrCast(op: getOperation(), src: getSrc(), resultTy: getType()); |
| 3880 | } |
| 3881 | |
| 3882 | //===----------------------------------------------------------------------===// |
| 3883 | // AwaitOp |
| 3884 | //===----------------------------------------------------------------------===// |
| 3885 | |
| 3886 | void cir::AwaitOp::build(OpBuilder &builder, OperationState &result, |
| 3887 | cir::AwaitKind kind, BuilderCallbackRef readyBuilder, |
| 3888 | BuilderCallbackRef suspendBuilder, |
| 3889 | BuilderCallbackRef resumeBuilder) { |
| 3890 | result.addAttribute(name: getKindAttrName(name: result.name), |
| 3891 | attr: cir::AwaitKindAttr::get(context: builder.getContext(), value: kind)); |
| 3892 | { |
| 3893 | OpBuilder::InsertionGuard guard(builder); |
| 3894 | Region *readyRegion = result.addRegion(); |
| 3895 | builder.createBlock(parent: readyRegion); |
| 3896 | readyBuilder(builder, result.location); |
| 3897 | } |
| 3898 | |
| 3899 | { |
| 3900 | OpBuilder::InsertionGuard guard(builder); |
| 3901 | Region *suspendRegion = result.addRegion(); |
| 3902 | builder.createBlock(parent: suspendRegion); |
| 3903 | suspendBuilder(builder, result.location); |
| 3904 | } |
| 3905 | |
| 3906 | { |
| 3907 | OpBuilder::InsertionGuard guard(builder); |
| 3908 | Region *resumeRegion = result.addRegion(); |
| 3909 | builder.createBlock(parent: resumeRegion); |
| 3910 | resumeBuilder(builder, result.location); |
| 3911 | } |
| 3912 | } |
| 3913 | |
| 3914 | void cir::AwaitOp::getSuccessorRegions( |
| 3915 | mlir::RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 3916 | assert(point.isParent() || point.getTerminatorPredecessorOrNull()); |
| 3917 | |
| 3918 | // Execution always starts in the ready region. |
| 3919 | if (point.isParent()) { |
| 3920 | regions.emplace_back(Args: &getReady()); |
| 3921 | return; |
| 3922 | } |
| 3923 | |
| 3924 | mlir::Region *parentRegion = |
| 3925 | point.getTerminatorPredecessorOrNull()->getParentRegion(); |
| 3926 | |
| 3927 | // Branching from ready: the cir.condition terminating it selects between |
| 3928 | // suspending and resuming. Keep in sync with |
| 3929 | // ConditionOp::getSuccessorRegions. |
| 3930 | // |
| 3931 | // TODO: retrieve information from the promise and only push the |
| 3932 | // necessary ones. Example: `std::suspend_never` on initial or final |
| 3933 | // await's might allow suspend region to be skipped. |
| 3934 | if (&getReady() == parentRegion) { |
| 3935 | regions.emplace_back(Args: &getResume()); |
| 3936 | regions.emplace_back(Args: &getSuspend()); |
| 3937 | return; |
| 3938 | } |
| 3939 | |
| 3940 | // Branching from suspend: continue in resume once the coroutine is resumed |
| 3941 | // (or immediately if suspension is vetoed), or exit to the parent operation |
| 3942 | // if it stays suspended or is destroyed. |
| 3943 | if (&getSuspend() == parentRegion) { |
| 3944 | regions.emplace_back(Args: &getResume()); |
| 3945 | regions.emplace_back(Args: getOperation()); |
| 3946 | return; |
| 3947 | } |
| 3948 | |
| 3949 | // Branching from resume: exit to the parent operation. |
| 3950 | regions.emplace_back(Args: getOperation()); |
| 3951 | } |
| 3952 | |
| 3953 | LogicalResult cir::AwaitOp::verify() { |
| 3954 | if (!isa<ConditionOp>(Val: this->getReady().back().getTerminator())) |
| 3955 | return emitOpError(message: "ready region must end with cir.condition" ); |
| 3956 | if (this->getSuspend().empty()) |
| 3957 | return emitOpError(message: "suspend region must not be empty" ); |
| 3958 | if (!isa<CoroSuspendPoint>(Val: this->getSuspend().back().getTerminator())) |
| 3959 | return emitOpError(message: "suspend region must end with cir.coro.suspend_point" ); |
| 3960 | return success(); |
| 3961 | } |
| 3962 | |
| 3963 | //===----------------------------------------------------------------------===// |
| 3964 | // CoReturnOp |
| 3965 | //===----------------------------------------------------------------------===// |
| 3966 | |
| 3967 | LogicalResult cir::CoReturnOp::verify() { |
| 3968 | mlir::Operation *coRet = getOperation(); |
| 3969 | auto coroutine = coRet->getParentOfType<CoroutineOp>(); |
| 3970 | if (!coroutine.getBody().isAncestor(other: getOperation()->getParentRegion())) |
| 3971 | return emitOpError(message: "must be inside the cir.coroutine body region" ); |
| 3972 | return success(); |
| 3973 | } |
| 3974 | |
| 3975 | //===----------------------------------------------------------------------===// |
| 3976 | // CoroutineOp |
| 3977 | //===----------------------------------------------------------------------===// |
| 3978 | |
| 3979 | void cir::CoroutineOp::build(OpBuilder &builder, OperationState &result, |
| 3980 | BuilderCallbackRef initialSuspendBuilder, |
| 3981 | BuilderCallbackRef bodyBuilder, |
| 3982 | BuilderCallbackRef finalSuspendBuilder, |
| 3983 | BuilderCallbackRef destroyBuilder, |
| 3984 | BuilderCallbackRef exitBuilder) { |
| 3985 | { |
| 3986 | OpBuilder::InsertionGuard guard(builder); |
| 3987 | Region *initialRegion = result.addRegion(); |
| 3988 | builder.createBlock(parent: initialRegion); |
| 3989 | initialSuspendBuilder(builder, result.location); |
| 3990 | } |
| 3991 | |
| 3992 | { |
| 3993 | OpBuilder::InsertionGuard guard(builder); |
| 3994 | Region *bodyRegion = result.addRegion(); |
| 3995 | builder.createBlock(parent: bodyRegion); |
| 3996 | bodyBuilder(builder, result.location); |
| 3997 | } |
| 3998 | |
| 3999 | { |
| 4000 | OpBuilder::InsertionGuard guard(builder); |
| 4001 | Region *finalRegion = result.addRegion(); |
| 4002 | builder.createBlock(parent: finalRegion); |
| 4003 | finalSuspendBuilder(builder, result.location); |
| 4004 | } |
| 4005 | |
| 4006 | { |
| 4007 | OpBuilder::InsertionGuard guard(builder); |
| 4008 | Region *destroyRegion = result.addRegion(); |
| 4009 | builder.createBlock(parent: destroyRegion); |
| 4010 | destroyBuilder(builder, result.location); |
| 4011 | } |
| 4012 | |
| 4013 | { |
| 4014 | OpBuilder::InsertionGuard guard(builder); |
| 4015 | Region *exitRegion = result.addRegion(); |
| 4016 | builder.createBlock(parent: exitRegion); |
| 4017 | exitBuilder(builder, result.location); |
| 4018 | } |
| 4019 | } |
| 4020 | |
| 4021 | void cir::CoroutineOp::getSuccessorRegions( |
| 4022 | mlir::RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> ®ions) { |
| 4023 | // The parent op always enters through initial_suspend get_return_object() |
| 4024 | // (if present) and the init await always run first. |
| 4025 | if (point.isParent()) { |
| 4026 | regions.emplace_back(Args: &getInitialSuspend()); |
| 4027 | return; |
| 4028 | } |
| 4029 | |
| 4030 | mlir::Region *parent = |
| 4031 | point.getTerminatorPredecessorOrNull()->getParentRegion(); |
| 4032 | |
| 4033 | // initial_suspend either falls into body (resumed, or never actually |
| 4034 | // suspended because await_ready() was true), exits directly, or reaches |
| 4035 | // destroy when the coroutine is destroyed while suspended at its initial |
| 4036 | // suspend point. |
| 4037 | if (parent == &getInitialSuspend()) { |
| 4038 | regions.emplace_back(Args: &getBody()); |
| 4039 | regions.emplace_back(Args: &getExit()); |
| 4040 | regions.emplace_back(Args: &getDestroy()); |
| 4041 | return; |
| 4042 | } |
| 4043 | |
| 4044 | // body can fall through into final_suspend (co_return), exit directly |
| 4045 | // from any suspend_point inside it that just suspends, or reach destroy |
| 4046 | // from a suspend_point being destroyed. An exception that escapes body, for |
| 4047 | // example one that unhandled_exception() rethrows, will leave through the |
| 4048 | // unwind region once it is implemented. |
| 4049 | if (parent == &getBody()) { |
| 4050 | assert(!cir::MissingFeatures::coroUnwindRegion()); |
| 4051 | regions.emplace_back(Args: &getFinalSuspend()); |
| 4052 | regions.emplace_back(Args: &getExit()); // any suspend_point inside body |
| 4053 | regions.emplace_back(Args: &getDestroy()); // destroy() on any suspend inside body |
| 4054 | return; |
| 4055 | } |
| 4056 | |
| 4057 | // final_suspend's only live edge in valid programs is destroy. Resuming |
| 4058 | // past the final suspend is UB. The ready-immediately edge to exit is |
| 4059 | // kept for structural symmetry with the other two suspend regions even |
| 4060 | // though it's effectively dead. |
| 4061 | if (parent == &getFinalSuspend()) { |
| 4062 | regions.emplace_back( |
| 4063 | Args: &getDestroy()); // the only real exit from final_suspend |
| 4064 | regions.emplace_back(Args: &getExit()); // ready==true edge, rarely taken |
| 4065 | return; |
| 4066 | } |
| 4067 | |
| 4068 | // destroy falls through to exit (normal return). |
| 4069 | if (parent == &getDestroy()) { |
| 4070 | regions.emplace_back(Args: &getExit()); |
| 4071 | return; |
| 4072 | } |
| 4073 | // exit always terminates the op there's no region it loops back into. |
| 4074 | if (parent == &getExit()) { |
| 4075 | regions.emplace_back(Args: getOperation()); |
| 4076 | return; |
| 4077 | } |
| 4078 | } |
| 4079 | |
| 4080 | mlir::ValueRange |
| 4081 | cir::CoroutineOp::getSuccessorInputs(RegionSuccessor successor) { |
| 4082 | return ValueRange(); |
| 4083 | } |
| 4084 | |
| 4085 | LogicalResult cir::CoroutineOp::verify() { |
| 4086 | mlir::Region &initialSuspend = getInitialSuspend(); |
| 4087 | mlir::Region &body = getBody(); |
| 4088 | mlir::Region &finalSuspend = getFinalSuspend(); |
| 4089 | mlir::Region &destroy = getDestroy(); |
| 4090 | mlir::Region &exit = getExit(); |
| 4091 | |
| 4092 | // initial_suspend must contain exactly one 'init'-kind cir.await an |
| 4093 | // optional prelude of ordinary statements (get_return_object() and |
| 4094 | // similar) is allowed before it, but no *other* await may appear. |
| 4095 | int initAwaitCount = 0; |
| 4096 | bool hasInvalidAwait = false; |
| 4097 | |
| 4098 | initialSuspend.walk(callback: [&](Operation *op) { |
| 4099 | auto awaitOp = mlir::dyn_cast<AwaitOp>(Val: op); |
| 4100 | if (!awaitOp) |
| 4101 | return mlir::WalkResult::advance(); |
| 4102 | |
| 4103 | if (awaitOp.getKind() == cir::AwaitKind::Init) |
| 4104 | ++initAwaitCount; |
| 4105 | else |
| 4106 | hasInvalidAwait = true; |
| 4107 | |
| 4108 | if (initAwaitCount > 1 || hasInvalidAwait) |
| 4109 | return mlir::WalkResult::interrupt(); |
| 4110 | |
| 4111 | return mlir::WalkResult::advance(); |
| 4112 | }); |
| 4113 | |
| 4114 | if (hasInvalidAwait) |
| 4115 | return emitOpError(message: "'initial_suspend' must not contain any cir.await other " |
| 4116 | "than the 'init' one" ); |
| 4117 | if (initAwaitCount != 1) |
| 4118 | return emitOpError( |
| 4119 | message: "must have exactly one 'init' cir.await in 'initial_suspend'" ); |
| 4120 | |
| 4121 | // final_suspend must contain only one or zero 'final'-kind cir.await |
| 4122 | int finalAwaitCount = 0; |
| 4123 | hasInvalidAwait = false; |
| 4124 | |
| 4125 | finalSuspend.walk(callback: [&](Operation *op) { |
| 4126 | auto awaitOp = mlir::dyn_cast<AwaitOp>(Val: op); |
| 4127 | if (!awaitOp) |
| 4128 | return mlir::WalkResult::advance(); |
| 4129 | |
| 4130 | if (awaitOp.getKind() == cir::AwaitKind::Final) |
| 4131 | ++finalAwaitCount; |
| 4132 | else |
| 4133 | hasInvalidAwait = true; |
| 4134 | |
| 4135 | if (finalAwaitCount > 1 || hasInvalidAwait) |
| 4136 | return mlir::WalkResult::interrupt(); |
| 4137 | |
| 4138 | return mlir::WalkResult::advance(); |
| 4139 | }); |
| 4140 | |
| 4141 | if (hasInvalidAwait) |
| 4142 | return emitOpError(message: "'final_suspend' must not contain any cir.await other " |
| 4143 | "than the 'final' one" ); |
| 4144 | if (finalAwaitCount > 1) |
| 4145 | return emitOpError( |
| 4146 | message: "must have one or zero 'final' cir.await in 'final_suspend'" ); |
| 4147 | |
| 4148 | // Each region must end with the terminator its role requires: |
| 4149 | // initial_suspend/final_suspend/destroy yield back into cir.coroutine's |
| 4150 | // own control flow, body ends in a cir.co_return (or a plain yield if |
| 4151 | // some path never reaches one), and exit actually returns from the |
| 4152 | // function. |
| 4153 | if (initialSuspend.empty() || initialSuspend.back().empty()) |
| 4154 | return emitOpError(message: "initial_suspend region must not be empty" ); |
| 4155 | if (!isa<YieldOp>(Val: initialSuspend.back().back())) |
| 4156 | return emitOpError(message: "'initial_suspend' must end with cir.yield" ); |
| 4157 | |
| 4158 | if (body.empty() || body.back().empty()) |
| 4159 | return emitOpError(message: "'body' region must not be empty" ); |
| 4160 | if (!isa<YieldOp, CoReturnOp>(Val: body.back().back())) |
| 4161 | return emitOpError(message: "'body' must end with cir.yield or cir.co_return" ); |
| 4162 | |
| 4163 | if (finalSuspend.empty() || finalSuspend.back().empty()) |
| 4164 | return emitOpError(message: "'final_suspend' region must not be empty" ); |
| 4165 | if (!isa<YieldOp>(Val: finalSuspend.back().back())) |
| 4166 | return emitOpError(message: "'final_suspend' must end with cir.yield" ); |
| 4167 | |
| 4168 | if (destroy.empty() || destroy.back().empty()) |
| 4169 | return emitOpError(message: "'destroy' region must not be empty" ); |
| 4170 | if (!isa<YieldOp>(Val: destroy.back().back())) |
| 4171 | return emitOpError(message: "'destroy' must end with cir.yield" ); |
| 4172 | |
| 4173 | if (exit.empty() || exit.back().empty()) |
| 4174 | return emitOpError(message: "'exit' region must not be empty" ); |
| 4175 | if (!isa<ReturnOp>(Val: exit.back().back())) |
| 4176 | return emitOpError(message: "'exit' must end with cir.return" ); |
| 4177 | |
| 4178 | return mlir::success(); |
| 4179 | } |
| 4180 | |
| 4181 | //===----------------------------------------------------------------------===// |
| 4182 | // CopyOp Definitions |
| 4183 | //===----------------------------------------------------------------------===// |
| 4184 | |
| 4185 | // Prints the pointer type(s) for a `cir.copy`. `src` and `dst` share the same |
| 4186 | // pointee type but may differ in address space; a single type is printed when |
| 4187 | // they match, otherwise both are printed (`src` first). |
| 4188 | void cir::printCopyTypes(mlir::OpAsmPrinter &printer, mlir::Operation *, |
| 4189 | mlir::Type srcType, mlir::Type dstType) { |
| 4190 | printer.printType(type: srcType); |
| 4191 | if (srcType != dstType) { |
| 4192 | printer << ", " ; |
| 4193 | printer.printType(type: dstType); |
| 4194 | } |
| 4195 | } |
| 4196 | |
| 4197 | mlir::ParseResult cir::parseCopyTypes(mlir::OpAsmParser &parser, |
| 4198 | mlir::Type &srcType, |
| 4199 | mlir::Type &dstType) { |
| 4200 | if (parser.parseType(result&: srcType)) |
| 4201 | return mlir::failure(); |
| 4202 | if (parser.parseOptionalComma().succeeded()) { |
| 4203 | if (parser.parseType(result&: dstType)) |
| 4204 | return mlir::failure(); |
| 4205 | } else { |
| 4206 | dstType = srcType; |
| 4207 | } |
| 4208 | return mlir::success(); |
| 4209 | } |
| 4210 | |
| 4211 | LogicalResult cir::CopyOp::verify() { |
| 4212 | // The pointee types of `src` and `dst` are guaranteed to match by the |
| 4213 | // SameOperandsPointeeType trait; they may still differ in address space. |
| 4214 | |
| 4215 | // A data layout is required for us to know the number of bytes to be copied. |
| 4216 | if (!getType().getPointee().hasTrait<DataLayoutTypeInterface::Trait>()) |
| 4217 | return emitError() << "missing data layout for pointee type" ; |
| 4218 | |
| 4219 | if (getSkipTailPadding() && |
| 4220 | !mlir::isa<cir::RecordType>(Val: getType().getPointee())) |
| 4221 | return emitError() |
| 4222 | << "skip_tail_padding is only valid for record pointee types" ; |
| 4223 | |
| 4224 | return mlir::success(); |
| 4225 | } |
| 4226 | |
| 4227 | //===----------------------------------------------------------------------===// |
| 4228 | // PtrMaskOp Definitions |
| 4229 | //===----------------------------------------------------------------------===// |
| 4230 | |
| 4231 | LogicalResult cir::PtrMaskOp::verify() { |
| 4232 | mlir::DataLayout layout = mlir::DataLayout::closest(op: *this); |
| 4233 | std::optional<uint64_t> indexWidth = |
| 4234 | layout.getTypeIndexBitwidth(t: getPtr().getType()); |
| 4235 | if (!indexWidth) |
| 4236 | return emitOpError() << "pointer has no index width" ; |
| 4237 | |
| 4238 | uint64_t maskWidth = getMask().getType().getWidth(); |
| 4239 | if (maskWidth != *indexWidth) |
| 4240 | return emitOpError() << "mask width " << maskWidth |
| 4241 | << " must equal the pointer index width " |
| 4242 | << *indexWidth; |
| 4243 | |
| 4244 | return mlir::success(); |
| 4245 | } |
| 4246 | |
| 4247 | //===----------------------------------------------------------------------===// |
| 4248 | // GetRuntimeMemberOp Definitions |
| 4249 | //===----------------------------------------------------------------------===// |
| 4250 | |
| 4251 | LogicalResult cir::GetRuntimeMemberOp::verify() { |
| 4252 | cir::DataMemberType memberPtrTy = getMember().getType(); |
| 4253 | |
| 4254 | if (getAddr().getType().getPointee() != memberPtrTy.getClassTy()) |
| 4255 | return emitError() << "record type does not match the member pointer type" ; |
| 4256 | if (getType().getPointee() != memberPtrTy.getMemberTy()) |
| 4257 | return emitError() << "result type does not match the member pointer type" ; |
| 4258 | if (getType().getAddrSpace() != getAddr().getType().getAddrSpace()) |
| 4259 | return emitError() |
| 4260 | << "result address space does not match the object address space" ; |
| 4261 | return mlir::success(); |
| 4262 | } |
| 4263 | |
| 4264 | //===----------------------------------------------------------------------===// |
| 4265 | // GetMethodOp Definitions |
| 4266 | //===----------------------------------------------------------------------===// |
| 4267 | |
| 4268 | LogicalResult cir::GetMethodOp::verify() { |
| 4269 | cir::MethodType methodTy = getMethod().getType(); |
| 4270 | |
| 4271 | // Assume objectTy is !cir.ptr<!T> |
| 4272 | cir::PointerType objectPtrTy = getObject().getType(); |
| 4273 | mlir::Type objectTy = objectPtrTy.getPointee(); |
| 4274 | |
| 4275 | if (methodTy.getClassTy() != objectTy) |
| 4276 | return emitError() << "method class type and object type do not match" ; |
| 4277 | |
| 4278 | // Assume methodFuncTy is !cir.func<!Ret (!Args)> |
| 4279 | auto calleeTy = mlir::cast<cir::FuncType>(Val: getCallee().getType().getPointee()); |
| 4280 | cir::FuncType methodFuncTy = methodTy.getMemberFuncTy(); |
| 4281 | |
| 4282 | // We verify at here that calleeTy is !cir.func<!Ret (!cir.ptr<!void>, !Args)> |
| 4283 | // Note that the first parameter type of the callee is !cir.ptr<!void> instead |
| 4284 | // of !cir.ptr<!T> because the "this" pointer may be adjusted before calling |
| 4285 | // the callee. |
| 4286 | |
| 4287 | if (methodFuncTy.getReturnType() != calleeTy.getReturnType()) |
| 4288 | return emitError() |
| 4289 | << "method return type and callee return type do not match" ; |
| 4290 | |
| 4291 | llvm::ArrayRef<mlir::Type> calleeArgsTy = calleeTy.getInputs(); |
| 4292 | llvm::ArrayRef<mlir::Type> methodFuncArgsTy = methodFuncTy.getInputs(); |
| 4293 | |
| 4294 | if (calleeArgsTy.empty()) |
| 4295 | return emitError() << "callee parameter list lacks receiver object ptr" ; |
| 4296 | |
| 4297 | auto calleeThisArgPtrTy = mlir::dyn_cast<cir::PointerType>(Val: calleeArgsTy[0]); |
| 4298 | if (!calleeThisArgPtrTy || |
| 4299 | !mlir::isa<cir::VoidType>(Val: calleeThisArgPtrTy.getPointee())) { |
| 4300 | return emitError() |
| 4301 | << "the first parameter of callee must be a void pointer" ; |
| 4302 | } |
| 4303 | |
| 4304 | if (calleeArgsTy.size() != methodFuncArgsTy.size()) |
| 4305 | return emitError() << "callee and method parameter counts do not match" ; |
| 4306 | |
| 4307 | if (calleeArgsTy.size() > 1 && |
| 4308 | calleeArgsTy.slice(N: 1) != methodFuncArgsTy.slice(N: 1)) |
| 4309 | return emitError() |
| 4310 | << "callee parameters and method parameters do not match" ; |
| 4311 | |
| 4312 | return mlir::success(); |
| 4313 | } |
| 4314 | |
| 4315 | //===----------------------------------------------------------------------===// |
| 4316 | // GetMemberOp Definitions |
| 4317 | //===----------------------------------------------------------------------===// |
| 4318 | |
| 4319 | static mlir::Type memberPointeeType(cir::RecordType recordTy, unsigned idx) { |
| 4320 | return cir::memberStorageType(memberTy: recordTy.getMembers()[idx]); |
| 4321 | } |
| 4322 | |
| 4323 | LogicalResult cir::GetMemberOp::verify() { |
| 4324 | const auto recordTy = dyn_cast<RecordType>(Val: getAddrTy().getPointee()); |
| 4325 | if (!recordTy) |
| 4326 | return emitError() << "expected pointer to a record type" ; |
| 4327 | |
| 4328 | if (recordTy.getMembers().size() <= getIndex()) |
| 4329 | return emitError() << "member index out of bounds" ; |
| 4330 | |
| 4331 | mlir::Type pointeeTy = memberPointeeType(recordTy, idx: getIndex()); |
| 4332 | if (!pointeeTy) |
| 4333 | return emitError() << "member owns no storage to point at" ; |
| 4334 | |
| 4335 | if (pointeeTy != getType().getPointee()) |
| 4336 | return emitError() << "member type mismatch" ; |
| 4337 | |
| 4338 | if (getAddrTy().getAddrSpace() != getType().getAddrSpace()) |
| 4339 | return emitError() << "address space mismatch" ; |
| 4340 | |
| 4341 | return mlir::success(); |
| 4342 | } |
| 4343 | |
| 4344 | //===----------------------------------------------------------------------===// |
| 4345 | // ExtractMemberOp Definitions |
| 4346 | //===----------------------------------------------------------------------===// |
| 4347 | |
| 4348 | LogicalResult cir::ExtractMemberOp::() { |
| 4349 | if (mlir::isa<cir::UnionType>(Val: getRecord().getType())) |
| 4350 | return emitError() |
| 4351 | << "cir.extract_member currently does not support unions" ; |
| 4352 | auto structTy = mlir::cast<cir::StructType>(Val: getRecord().getType()); |
| 4353 | if (structTy.getMembers().size() <= getIndex()) |
| 4354 | return emitError() << "member index out of bounds" ; |
| 4355 | mlir::Type memberTy = structTy.getMembers()[getIndex()]; |
| 4356 | if (mlir::isa<cir::BitFieldType>(Val: memberTy)) |
| 4357 | return emitError() << "cir.extract_member does not support bit-fields" ; |
| 4358 | if (memberTy != getType()) |
| 4359 | return emitError() << "member type mismatch" ; |
| 4360 | return mlir::success(); |
| 4361 | } |
| 4362 | |
| 4363 | //===----------------------------------------------------------------------===// |
| 4364 | // InsertMemberOp Definitions |
| 4365 | //===----------------------------------------------------------------------===// |
| 4366 | |
| 4367 | LogicalResult cir::InsertMemberOp::verify() { |
| 4368 | if (mlir::isa<cir::UnionType>(Val: getRecord().getType())) |
| 4369 | return emitError() << "cir.insert_member currently does not support unions" ; |
| 4370 | auto structTy = mlir::cast<cir::StructType>(Val: getRecord().getType()); |
| 4371 | if (structTy.getMembers().size() <= getIndex()) |
| 4372 | return emitError() << "member index out of bounds" ; |
| 4373 | mlir::Type memberTy = structTy.getMembers()[getIndex()]; |
| 4374 | if (mlir::isa<cir::BitFieldType>(Val: memberTy)) |
| 4375 | return emitError() << "cir.insert_member does not support bit-fields" ; |
| 4376 | if (memberTy != getValue().getType()) |
| 4377 | return emitError() << "member type mismatch" ; |
| 4378 | // The op trait already checks that the types of $result and $record match. |
| 4379 | return mlir::success(); |
| 4380 | } |
| 4381 | |
| 4382 | //===----------------------------------------------------------------------===// |
| 4383 | // VecCreateOp |
| 4384 | //===----------------------------------------------------------------------===// |
| 4385 | |
| 4386 | OpFoldResult cir::VecCreateOp::fold(FoldAdaptor adaptor) { |
| 4387 | if (llvm::any_of(Range: getElements(), P: [](mlir::Value value) { |
| 4388 | return !value.getDefiningOp<cir::ConstantOp>(); |
| 4389 | })) |
| 4390 | return {}; |
| 4391 | |
| 4392 | return cir::ConstVectorAttr::get( |
| 4393 | type: getType(), elts: mlir::ArrayAttr::get(context: getContext(), value: adaptor.getElements())); |
| 4394 | } |
| 4395 | |
| 4396 | LogicalResult cir::VecCreateOp::verify() { |
| 4397 | // Verify that the number of arguments matches the number of elements in the |
| 4398 | // vector, and that the type of all the arguments matches the type of the |
| 4399 | // elements in the vector. |
| 4400 | const cir::VectorType vecTy = getType(); |
| 4401 | if (vecTy.isScalable()) |
| 4402 | return emitOpError() << "scalable vectors are not supported" ; |
| 4403 | |
| 4404 | if (getElements().size() != vecTy.getFixedSize()) { |
| 4405 | return emitOpError() << "operand count of " << getElements().size() |
| 4406 | << " doesn't match vector type " << vecTy |
| 4407 | << " element count of " << vecTy.getFixedSize(); |
| 4408 | } |
| 4409 | |
| 4410 | const mlir::Type elementType = vecTy.getElementType(); |
| 4411 | for (const mlir::Value element : getElements()) { |
| 4412 | if (element.getType() != elementType) { |
| 4413 | return emitOpError() << "operand type " << element.getType() |
| 4414 | << " doesn't match vector element type " |
| 4415 | << elementType; |
| 4416 | } |
| 4417 | } |
| 4418 | |
| 4419 | return success(); |
| 4420 | } |
| 4421 | |
| 4422 | //===----------------------------------------------------------------------===// |
| 4423 | // VecExtractOp |
| 4424 | //===----------------------------------------------------------------------===// |
| 4425 | |
| 4426 | OpFoldResult cir::VecExtractOp::(FoldAdaptor adaptor) { |
| 4427 | const auto vectorAttr = |
| 4428 | llvm::dyn_cast_if_present<cir::ConstVectorAttr>(Val: adaptor.getVec()); |
| 4429 | if (!vectorAttr) |
| 4430 | return {}; |
| 4431 | |
| 4432 | const auto indexAttr = |
| 4433 | llvm::dyn_cast_if_present<cir::IntAttr>(Val: adaptor.getIndex()); |
| 4434 | if (!indexAttr) |
| 4435 | return {}; |
| 4436 | |
| 4437 | const mlir::ArrayAttr elements = vectorAttr.getElts(); |
| 4438 | const uint64_t index = indexAttr.getUInt(); |
| 4439 | if (index >= elements.size()) |
| 4440 | return {}; |
| 4441 | |
| 4442 | return elements[index]; |
| 4443 | } |
| 4444 | |
| 4445 | //===----------------------------------------------------------------------===// |
| 4446 | // CmpOp |
| 4447 | //===----------------------------------------------------------------------===// |
| 4448 | |
| 4449 | LogicalResult cir::CmpOp::verify() { |
| 4450 | if (getFenvAttr() && !cir::isAnyFloatingPointType(type: getLhs().getType())) |
| 4451 | return emitOpError() |
| 4452 | << "'fenv' is only valid for floating-point comparisons" ; |
| 4453 | return success(); |
| 4454 | } |
| 4455 | |
| 4456 | //===----------------------------------------------------------------------===// |
| 4457 | // VecCmpOp |
| 4458 | //===----------------------------------------------------------------------===// |
| 4459 | |
| 4460 | OpFoldResult cir::VecCmpOp::fold(FoldAdaptor adaptor) { |
| 4461 | auto lhsVecAttr = |
| 4462 | mlir::dyn_cast_if_present<cir::ConstVectorAttr>(Val: adaptor.getLhs()); |
| 4463 | auto rhsVecAttr = |
| 4464 | mlir::dyn_cast_if_present<cir::ConstVectorAttr>(Val: adaptor.getRhs()); |
| 4465 | if (!lhsVecAttr || !rhsVecAttr) |
| 4466 | return {}; |
| 4467 | |
| 4468 | mlir::Type inputElemTy = |
| 4469 | mlir::cast<cir::VectorType>(Val: lhsVecAttr.getType()).getElementType(); |
| 4470 | auto IntTy = mlir::dyn_cast<cir::IntType>(Val&: inputElemTy); |
| 4471 | // FIXME: Fold vectors of bool too? |
| 4472 | if (!IntTy) |
| 4473 | return {}; |
| 4474 | |
| 4475 | cir::CmpOpKind opKind = adaptor.getKind(); |
| 4476 | mlir::ArrayAttr lhsVecElhs = lhsVecAttr.getElts(); |
| 4477 | mlir::ArrayAttr rhsVecElhs = rhsVecAttr.getElts(); |
| 4478 | uint64_t vecSize = lhsVecElhs.size(); |
| 4479 | |
| 4480 | SmallVector<mlir::Attribute, 16> elements(vecSize); |
| 4481 | bool isUnsignedInt = IntTy.isUnsigned(); |
| 4482 | for (uint64_t i = 0; i < vecSize; i++) { |
| 4483 | mlir::Attribute lhsAttr = lhsVecElhs[i]; |
| 4484 | mlir::Attribute rhsAttr = rhsVecElhs[i]; |
| 4485 | bool cmpResult = false; |
| 4486 | switch (opKind) { |
| 4487 | case cir::CmpOpKind::lt: { |
| 4488 | if (isUnsignedInt) |
| 4489 | cmpResult = mlir::cast<cir::IntAttr>(Val&: lhsAttr).getUInt() < |
| 4490 | mlir::cast<cir::IntAttr>(Val&: rhsAttr).getUInt(); |
| 4491 | else |
| 4492 | cmpResult = mlir::cast<cir::IntAttr>(Val&: lhsAttr).getSInt() < |
| 4493 | mlir::cast<cir::IntAttr>(Val&: rhsAttr).getSInt(); |
| 4494 | break; |
| 4495 | } |
| 4496 | case cir::CmpOpKind::le: { |
| 4497 | if (isUnsignedInt) |
| 4498 | cmpResult = mlir::cast<cir::IntAttr>(Val&: lhsAttr).getUInt() <= |
| 4499 | mlir::cast<cir::IntAttr>(Val&: rhsAttr).getUInt(); |
| 4500 | else |
| 4501 | cmpResult = mlir::cast<cir::IntAttr>(Val&: lhsAttr).getSInt() <= |
| 4502 | mlir::cast<cir::IntAttr>(Val&: rhsAttr).getSInt(); |
| 4503 | break; |
| 4504 | } |
| 4505 | case cir::CmpOpKind::gt: { |
| 4506 | if (isUnsignedInt) |
| 4507 | cmpResult = mlir::cast<cir::IntAttr>(Val&: lhsAttr).getUInt() > |
| 4508 | mlir::cast<cir::IntAttr>(Val&: rhsAttr).getUInt(); |
| 4509 | else |
| 4510 | cmpResult = mlir::cast<cir::IntAttr>(Val&: lhsAttr).getSInt() > |
| 4511 | mlir::cast<cir::IntAttr>(Val&: rhsAttr).getSInt(); |
| 4512 | break; |
| 4513 | } |
| 4514 | case cir::CmpOpKind::ge: { |
| 4515 | if (isUnsignedInt) |
| 4516 | cmpResult = mlir::cast<cir::IntAttr>(Val&: lhsAttr).getUInt() >= |
| 4517 | mlir::cast<cir::IntAttr>(Val&: rhsAttr).getUInt(); |
| 4518 | else |
| 4519 | cmpResult = mlir::cast<cir::IntAttr>(Val&: lhsAttr).getSInt() >= |
| 4520 | mlir::cast<cir::IntAttr>(Val&: rhsAttr).getSInt(); |
| 4521 | break; |
| 4522 | } |
| 4523 | case cir::CmpOpKind::eq: |
| 4524 | cmpResult = mlir::cast<cir::IntAttr>(Val&: lhsAttr).getValue() == |
| 4525 | mlir::cast<cir::IntAttr>(Val&: rhsAttr).getValue(); |
| 4526 | break; |
| 4527 | case cir::CmpOpKind::ne: |
| 4528 | cmpResult = mlir::cast<cir::IntAttr>(Val&: lhsAttr).getValue() != |
| 4529 | mlir::cast<cir::IntAttr>(Val&: rhsAttr).getValue(); |
| 4530 | break; |
| 4531 | case cir::CmpOpKind::one: |
| 4532 | case cir::CmpOpKind::uno: { |
| 4533 | llvm_unreachable("Should not reach fp-only comparison kinds here" ); |
| 4534 | } |
| 4535 | } |
| 4536 | |
| 4537 | // A true result is all bits set (-1 in two's complement), and a false |
| 4538 | // result is all bits clear. For a 1-bit element type these are the same |
| 4539 | // bit pattern as 1 and 0, respectively. |
| 4540 | elements[i] = |
| 4541 | cir::IntAttr::get(type: getType().getElementType(), value: cmpResult ? -1LL : 0LL); |
| 4542 | } |
| 4543 | |
| 4544 | return cir::ConstVectorAttr::get( |
| 4545 | type: getType(), elts: mlir::ArrayAttr::get(context: getContext(), value: elements)); |
| 4546 | } |
| 4547 | |
| 4548 | //===----------------------------------------------------------------------===// |
| 4549 | // VecFCmpOp |
| 4550 | //===----------------------------------------------------------------------===// |
| 4551 | |
| 4552 | OpFoldResult cir::VecFCmpOp::fold(FoldAdaptor adaptor) { |
| 4553 | // Do not fold when fenv is present. |
| 4554 | // TODO(cir): This is overly conservative. We could fold comparisons as long |
| 4555 | // as we can prove that the operation wouldn't raise exceptions or |
| 4556 | // when the fenv attribute does not require strict exception |
| 4557 | // semantics. |
| 4558 | if (getFenvAttr()) |
| 4559 | return {}; |
| 4560 | |
| 4561 | cir::FCmpOpKind kind = getKind(); |
| 4562 | if (kind == cir::FCmpOpKind::_true || kind == cir::FCmpOpKind::_false) { |
| 4563 | mlir::Attribute elt = |
| 4564 | cir::IntAttr::get(type: getType().getElementType(), |
| 4565 | value: (kind == cir::FCmpOpKind::_true) ? -1LL : 0LL); |
| 4566 | SmallVector<mlir::Attribute, 16> elements(getType().getFixedSize(), elt); |
| 4567 | return cir::ConstVectorAttr::get( |
| 4568 | type: getType(), elts: mlir::ArrayAttr::get(context: getContext(), value: elements)); |
| 4569 | } |
| 4570 | |
| 4571 | auto lhsVecAttr = |
| 4572 | mlir::dyn_cast_if_present<cir::ConstVectorAttr>(Val: adaptor.getLhs()); |
| 4573 | auto rhsVecAttr = |
| 4574 | mlir::dyn_cast_if_present<cir::ConstVectorAttr>(Val: adaptor.getRhs()); |
| 4575 | if (!lhsVecAttr || !rhsVecAttr) |
| 4576 | return {}; |
| 4577 | |
| 4578 | mlir::ArrayAttr lhsElts = lhsVecAttr.getElts(); |
| 4579 | mlir::ArrayAttr rhsElts = rhsVecAttr.getElts(); |
| 4580 | uint64_t vecSize = lhsElts.size(); |
| 4581 | SmallVector<mlir::Attribute, 16> elements; |
| 4582 | elements.reserve(N: vecSize); |
| 4583 | for (auto [lhsElt, rhsElt] : llvm::zip_equal(t&: lhsElts, u&: rhsElts)) { |
| 4584 | auto lhsFP = mlir::cast<cir::FPAttr>(Val: lhsElt); |
| 4585 | auto rhsFP = mlir::cast<cir::FPAttr>(Val: rhsElt); |
| 4586 | |
| 4587 | llvm::APFloat::cmpResult cr = lhsFP.getValue().compare(RHS: rhsFP.getValue()); |
| 4588 | bool unordered = cr == llvm::APFloat::cmpUnordered; |
| 4589 | bool eq = cr == llvm::APFloat::cmpEqual; |
| 4590 | bool lt = cr == llvm::APFloat::cmpLessThan; |
| 4591 | bool gt = cr == llvm::APFloat::cmpGreaterThan; |
| 4592 | bool cmpResult = false; |
| 4593 | switch (kind) { |
| 4594 | case cir::FCmpOpKind::oeq: |
| 4595 | cmpResult = eq; |
| 4596 | break; |
| 4597 | case cir::FCmpOpKind::ogt: |
| 4598 | cmpResult = gt; |
| 4599 | break; |
| 4600 | case cir::FCmpOpKind::oge: |
| 4601 | cmpResult = gt || eq; |
| 4602 | break; |
| 4603 | case cir::FCmpOpKind::olt: |
| 4604 | cmpResult = lt; |
| 4605 | break; |
| 4606 | case cir::FCmpOpKind::ole: |
| 4607 | cmpResult = lt || eq; |
| 4608 | break; |
| 4609 | case cir::FCmpOpKind::one: |
| 4610 | cmpResult = lt || gt; |
| 4611 | break; |
| 4612 | case cir::FCmpOpKind::ord: |
| 4613 | cmpResult = !unordered; |
| 4614 | break; |
| 4615 | case cir::FCmpOpKind::ueq: |
| 4616 | cmpResult = unordered || eq; |
| 4617 | break; |
| 4618 | case cir::FCmpOpKind::ugt: |
| 4619 | cmpResult = unordered || gt; |
| 4620 | break; |
| 4621 | case cir::FCmpOpKind::uge: |
| 4622 | cmpResult = unordered || gt || eq; |
| 4623 | break; |
| 4624 | case cir::FCmpOpKind::ult: |
| 4625 | cmpResult = unordered || lt; |
| 4626 | break; |
| 4627 | case cir::FCmpOpKind::ule: |
| 4628 | cmpResult = unordered || lt || eq; |
| 4629 | break; |
| 4630 | case cir::FCmpOpKind::une: |
| 4631 | cmpResult = unordered || lt || gt; |
| 4632 | break; |
| 4633 | case cir::FCmpOpKind::uno: |
| 4634 | cmpResult = unordered; |
| 4635 | break; |
| 4636 | case cir::FCmpOpKind::_true: |
| 4637 | case cir::FCmpOpKind::_false: |
| 4638 | llvm_unreachable("Op kind have been handled already" ); |
| 4639 | } |
| 4640 | |
| 4641 | // A true result is all bits set (-1 in two's complement), and a false |
| 4642 | // result is all bits clear. For a 1-bit element type these are the same |
| 4643 | // bit pattern as 1 and 0, respectively. |
| 4644 | elements.push_back( |
| 4645 | Elt: cir::IntAttr::get(type: getType().getElementType(), value: cmpResult ? -1LL : 0LL)); |
| 4646 | } |
| 4647 | |
| 4648 | return cir::ConstVectorAttr::get( |
| 4649 | type: getType(), elts: mlir::ArrayAttr::get(context: getContext(), value: elements)); |
| 4650 | } |
| 4651 | |
| 4652 | //===----------------------------------------------------------------------===// |
| 4653 | // VecShuffleOp |
| 4654 | //===----------------------------------------------------------------------===// |
| 4655 | |
| 4656 | OpFoldResult cir::VecShuffleOp::fold(FoldAdaptor adaptor) { |
| 4657 | auto vec1Attr = |
| 4658 | mlir::dyn_cast_if_present<cir::ConstVectorAttr>(Val: adaptor.getVec1()); |
| 4659 | auto vec2Attr = |
| 4660 | mlir::dyn_cast_if_present<cir::ConstVectorAttr>(Val: adaptor.getVec2()); |
| 4661 | if (!vec1Attr || !vec2Attr) |
| 4662 | return {}; |
| 4663 | |
| 4664 | mlir::Type vec1ElemTy = |
| 4665 | mlir::cast<cir::VectorType>(Val: vec1Attr.getType()).getElementType(); |
| 4666 | |
| 4667 | mlir::ArrayAttr vec1Elts = vec1Attr.getElts(); |
| 4668 | mlir::ArrayAttr vec2Elts = vec2Attr.getElts(); |
| 4669 | mlir::ArrayAttr indicesElts = adaptor.getIndices(); |
| 4670 | |
| 4671 | SmallVector<mlir::Attribute, 16> elements; |
| 4672 | elements.reserve(N: indicesElts.size()); |
| 4673 | |
| 4674 | uint64_t vec1Size = vec1Elts.size(); |
| 4675 | for (const auto &idxAttr : indicesElts.getAsRange<cir::IntAttr>()) { |
| 4676 | if (idxAttr.getSInt() == -1) { |
| 4677 | elements.push_back(Elt: cir::UndefAttr::get(type: vec1ElemTy)); |
| 4678 | continue; |
| 4679 | } |
| 4680 | |
| 4681 | uint64_t idxValue = idxAttr.getUInt(); |
| 4682 | elements.push_back(Elt: idxValue < vec1Size ? vec1Elts[idxValue] |
| 4683 | : vec2Elts[idxValue - vec1Size]); |
| 4684 | } |
| 4685 | |
| 4686 | return cir::ConstVectorAttr::get( |
| 4687 | type: mlir::cast<cir::VectorType>(Val: getType()), |
| 4688 | elts: mlir::ArrayAttr::get(context: getContext(), value: elements)); |
| 4689 | } |
| 4690 | |
| 4691 | LogicalResult cir::VecShuffleOp::verify() { |
| 4692 | if (!mlir::isa<cir::VectorType>(Val: getType())) |
| 4693 | return success(); |
| 4694 | |
| 4695 | auto vec1Ty = mlir::cast<cir::VectorType>(Val: getVec1().getType()); |
| 4696 | auto vec2Ty = mlir::cast<cir::VectorType>(Val: getVec2().getType()); |
| 4697 | auto resultTy = mlir::cast<cir::VectorType>(Val: getResult().getType()); |
| 4698 | if (vec1Ty.isScalable() || vec2Ty.isScalable() || resultTy.isScalable()) |
| 4699 | return emitOpError() << "scalable vectors are not supported" ; |
| 4700 | |
| 4701 | // The number of elements in the indices array must match the number of |
| 4702 | // elements in the result type. |
| 4703 | if (getIndices().size() != resultTy.getFixedSize()) { |
| 4704 | return emitOpError() << ": the number of elements in " << getIndices() |
| 4705 | << " and " << resultTy << " don't match" ; |
| 4706 | } |
| 4707 | |
| 4708 | // The element types of the two input vectors and of the result type must |
| 4709 | // match. |
| 4710 | if (vec1Ty.getElementType() != resultTy.getElementType()) { |
| 4711 | return emitOpError() << ": element types of " << getVec1().getType() |
| 4712 | << " and " << resultTy << " don't match" ; |
| 4713 | } |
| 4714 | |
| 4715 | const uint64_t maxValidIndex = |
| 4716 | vec1Ty.getFixedSize() + vec2Ty.getFixedSize() - 1; |
| 4717 | if (llvm::any_of( |
| 4718 | Range: getIndices().getAsRange<cir::IntAttr>(), P: [&](cir::IntAttr idxAttr) { |
| 4719 | return idxAttr.getSInt() != -1 && idxAttr.getUInt() > maxValidIndex; |
| 4720 | })) { |
| 4721 | return emitOpError() << ": index for __builtin_shufflevector must be " |
| 4722 | "less than the total number of vector elements" ; |
| 4723 | } |
| 4724 | return success(); |
| 4725 | } |
| 4726 | |
| 4727 | //===----------------------------------------------------------------------===// |
| 4728 | // VecShuffleDynamicOp |
| 4729 | //===----------------------------------------------------------------------===// |
| 4730 | |
| 4731 | OpFoldResult cir::VecShuffleDynamicOp::fold(FoldAdaptor adaptor) { |
| 4732 | mlir::Attribute vec = adaptor.getVec(); |
| 4733 | mlir::Attribute indices = adaptor.getIndices(); |
| 4734 | if (mlir::isa_and_nonnull<cir::ConstVectorAttr>(Val: vec) && |
| 4735 | mlir::isa_and_nonnull<cir::ConstVectorAttr>(Val: indices)) { |
| 4736 | auto vecAttr = mlir::cast<cir::ConstVectorAttr>(Val&: vec); |
| 4737 | auto indicesAttr = mlir::cast<cir::ConstVectorAttr>(Val&: indices); |
| 4738 | |
| 4739 | mlir::ArrayAttr vecElts = vecAttr.getElts(); |
| 4740 | mlir::ArrayAttr indicesElts = indicesAttr.getElts(); |
| 4741 | |
| 4742 | const uint64_t numElements = vecElts.size(); |
| 4743 | |
| 4744 | SmallVector<mlir::Attribute, 16> elements; |
| 4745 | elements.reserve(N: numElements); |
| 4746 | |
| 4747 | const uint64_t maskBits = llvm::NextPowerOf2(A: numElements - 1) - 1; |
| 4748 | for (const auto &idxAttr : indicesElts.getAsRange<cir::IntAttr>()) { |
| 4749 | uint64_t idxValue = idxAttr.getUInt(); |
| 4750 | uint64_t newIdx = idxValue & maskBits; |
| 4751 | elements.push_back(Elt: vecElts[newIdx]); |
| 4752 | } |
| 4753 | |
| 4754 | return cir::ConstVectorAttr::get( |
| 4755 | type: getType(), elts: mlir::ArrayAttr::get(context: getContext(), value: elements)); |
| 4756 | } |
| 4757 | |
| 4758 | return {}; |
| 4759 | } |
| 4760 | |
| 4761 | LogicalResult cir::VecShuffleDynamicOp::verify() { |
| 4762 | // The number of elements in the two input vectors must match. |
| 4763 | if (getVec().getType().getSize() != |
| 4764 | mlir::cast<cir::VectorType>(Val: getIndices().getType()).getSize()) { |
| 4765 | return emitOpError() << ": the number of elements in " << getVec().getType() |
| 4766 | << " and " << getIndices().getType() << " don't match" ; |
| 4767 | } |
| 4768 | return success(); |
| 4769 | } |
| 4770 | |
| 4771 | //===----------------------------------------------------------------------===// |
| 4772 | // VecTernaryOp |
| 4773 | //===----------------------------------------------------------------------===// |
| 4774 | |
| 4775 | LogicalResult cir::VecTernaryOp::verify() { |
| 4776 | // Verify that the condition operand has the same number of elements as the |
| 4777 | // other operands. (The automatic verification already checked that all |
| 4778 | // operands are vector types and that the second and third operands are the |
| 4779 | // same type.) |
| 4780 | if (getCond().getType().getSize() != getLhs().getType().getSize()) { |
| 4781 | return emitOpError() << ": the number of elements in " |
| 4782 | << getCond().getType() << " and " << getLhs().getType() |
| 4783 | << " don't match" ; |
| 4784 | } |
| 4785 | return success(); |
| 4786 | } |
| 4787 | |
| 4788 | OpFoldResult cir::VecTernaryOp::fold(FoldAdaptor adaptor) { |
| 4789 | mlir::Attribute cond = adaptor.getCond(); |
| 4790 | mlir::Attribute lhs = adaptor.getLhs(); |
| 4791 | mlir::Attribute rhs = adaptor.getRhs(); |
| 4792 | |
| 4793 | if (!mlir::isa_and_nonnull<cir::ConstVectorAttr>(Val: cond) || |
| 4794 | !mlir::isa_and_nonnull<cir::ConstVectorAttr>(Val: lhs) || |
| 4795 | !mlir::isa_and_nonnull<cir::ConstVectorAttr>(Val: rhs)) |
| 4796 | return {}; |
| 4797 | auto condVec = mlir::cast<cir::ConstVectorAttr>(Val&: cond); |
| 4798 | auto lhsVec = mlir::cast<cir::ConstVectorAttr>(Val&: lhs); |
| 4799 | auto rhsVec = mlir::cast<cir::ConstVectorAttr>(Val&: rhs); |
| 4800 | |
| 4801 | mlir::ArrayAttr condElts = condVec.getElts(); |
| 4802 | |
| 4803 | SmallVector<mlir::Attribute, 16> elements; |
| 4804 | elements.reserve(N: condElts.size()); |
| 4805 | |
| 4806 | for (const auto &[idx, condAttr] : |
| 4807 | llvm::enumerate(First: condElts.getAsRange<cir::IntAttr>())) { |
| 4808 | if (condAttr.getSInt()) { |
| 4809 | elements.push_back(Elt: lhsVec.getElts()[idx]); |
| 4810 | } else { |
| 4811 | elements.push_back(Elt: rhsVec.getElts()[idx]); |
| 4812 | } |
| 4813 | } |
| 4814 | |
| 4815 | cir::VectorType vecTy = getLhs().getType(); |
| 4816 | return cir::ConstVectorAttr::get( |
| 4817 | type: vecTy, elts: mlir::ArrayAttr::get(context: getContext(), value: elements)); |
| 4818 | } |
| 4819 | |
| 4820 | //===----------------------------------------------------------------------===// |
| 4821 | // LdexpOp |
| 4822 | //===----------------------------------------------------------------------===// |
| 4823 | |
| 4824 | LogicalResult cir::LdexpOp::verify() { |
| 4825 | auto srcVecTy = mlir::dyn_cast<cir::VectorType>(Val: getSrc().getType()); |
| 4826 | auto expVecTy = mlir::dyn_cast<cir::VectorType>(Val: getExp().getType()); |
| 4827 | if (!srcVecTy != !expVecTy || |
| 4828 | (srcVecTy && srcVecTy.getSize() != expVecTy.getSize())) |
| 4829 | return emitOpError() |
| 4830 | << "exponent must have the same number of elements as the value" ; |
| 4831 | return success(); |
| 4832 | } |
| 4833 | |
| 4834 | //===----------------------------------------------------------------------===// |
| 4835 | // LoadOp, GetBitfieldOp and MatrixColumnMajorLoadOp |
| 4836 | //===----------------------------------------------------------------------===// |
| 4837 | |
| 4838 | /// Adds the effects of a load from `addr`. A volatile or atomic load also has |
| 4839 | /// unknown read and write effects, as in the LLVM dialect, so that it is never |
| 4840 | /// erased when its result is unused. |
| 4841 | static void |
| 4842 | getLoadEffects(OpOperand &addr, bool isVolatileOrAtomic, |
| 4843 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 4844 | effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &addr); |
| 4845 | if (isVolatileOrAtomic) { |
| 4846 | effects.emplace_back(Args: MemoryEffects::Write::get()); |
| 4847 | effects.emplace_back(Args: MemoryEffects::Read::get()); |
| 4848 | } |
| 4849 | } |
| 4850 | |
| 4851 | void cir::LoadOp::getEffects( |
| 4852 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 4853 | getLoadEffects(addr&: getAddrMutable(), isVolatileOrAtomic: isVolatileOrAtomic(), effects); |
| 4854 | } |
| 4855 | |
| 4856 | void cir::GetBitfieldOp::getEffects( |
| 4857 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 4858 | getLoadEffects(addr&: getAddrMutable(), isVolatileOrAtomic: getIsVolatile(), effects); |
| 4859 | } |
| 4860 | |
| 4861 | void cir::MatrixColumnMajorLoadOp::getEffects( |
| 4862 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 4863 | getLoadEffects(addr&: getValueMutable(), isVolatileOrAtomic: getIsVolatile(), effects); |
| 4864 | } |
| 4865 | |
| 4866 | //===----------------------------------------------------------------------===// |
| 4867 | // MatrixTransposeOp |
| 4868 | //===----------------------------------------------------------------------===// |
| 4869 | |
| 4870 | LogicalResult cir::MatrixTransposeOp::verify() { |
| 4871 | cir::MatrixType valueTy = getValue().getType(); |
| 4872 | cir::MatrixType resultTy = getResult().getType(); |
| 4873 | |
| 4874 | if ((valueTy.getElementType() != resultTy.getElementType()) || |
| 4875 | (valueTy.getNumRows() != resultTy.getNumColumns()) || |
| 4876 | (valueTy.getNumColumns() != resultTy.getNumRows())) { |
| 4877 | auto expectedTy = |
| 4878 | cir::MatrixType::get(element_type: valueTy.getElementType(), num_rows: valueTy.getNumColumns(), |
| 4879 | num_columns: valueTy.getNumRows()); |
| 4880 | emitOpError() << "operand type " << valueTy << " expects result type of " |
| 4881 | << expectedTy << " but got " << resultTy; |
| 4882 | return failure(); |
| 4883 | } |
| 4884 | |
| 4885 | return success(); |
| 4886 | } |
| 4887 | |
| 4888 | //===----------------------------------------------------------------------===// |
| 4889 | // ComplexCreateOp |
| 4890 | //===----------------------------------------------------------------------===// |
| 4891 | |
| 4892 | LogicalResult cir::ComplexCreateOp::verify() { |
| 4893 | if (getType().getElementType() != getReal().getType()) { |
| 4894 | emitOpError() |
| 4895 | << "operand type of cir.complex.create does not match its result type" ; |
| 4896 | return failure(); |
| 4897 | } |
| 4898 | |
| 4899 | return success(); |
| 4900 | } |
| 4901 | |
| 4902 | OpFoldResult cir::ComplexCreateOp::fold(FoldAdaptor adaptor) { |
| 4903 | mlir::Attribute real = adaptor.getReal(); |
| 4904 | mlir::Attribute imag = adaptor.getImag(); |
| 4905 | if (!real || !imag) |
| 4906 | return {}; |
| 4907 | |
| 4908 | // When both of real and imag are constants, we can fold the operation into an |
| 4909 | // `#cir.const_complex` operation. |
| 4910 | auto realAttr = mlir::cast<mlir::TypedAttr>(Val&: real); |
| 4911 | auto imagAttr = mlir::cast<mlir::TypedAttr>(Val&: imag); |
| 4912 | return cir::ConstComplexAttr::get(real: realAttr, imag: imagAttr); |
| 4913 | } |
| 4914 | |
| 4915 | //===----------------------------------------------------------------------===// |
| 4916 | // ComplexRealOp |
| 4917 | //===----------------------------------------------------------------------===// |
| 4918 | |
| 4919 | LogicalResult cir::ComplexRealOp::verify() { |
| 4920 | mlir::Type operandTy = getOperand().getType(); |
| 4921 | if (auto complexOperandTy = mlir::dyn_cast<cir::ComplexType>(Val&: operandTy)) |
| 4922 | operandTy = complexOperandTy.getElementType(); |
| 4923 | |
| 4924 | if (getType() != operandTy) { |
| 4925 | emitOpError() << ": result type does not match operand type" ; |
| 4926 | return failure(); |
| 4927 | } |
| 4928 | |
| 4929 | return success(); |
| 4930 | } |
| 4931 | |
| 4932 | OpFoldResult cir::ComplexRealOp::fold(FoldAdaptor adaptor) { |
| 4933 | if (!mlir::isa<cir::ComplexType>(Val: getOperand().getType())) |
| 4934 | return nullptr; |
| 4935 | |
| 4936 | if (auto complexCreateOp = getOperand().getDefiningOp<cir::ComplexCreateOp>()) |
| 4937 | return complexCreateOp.getOperand(i: 0); |
| 4938 | |
| 4939 | auto complex = |
| 4940 | mlir::cast_if_present<cir::ConstComplexAttr>(Val: adaptor.getOperand()); |
| 4941 | return complex ? complex.getReal() : nullptr; |
| 4942 | } |
| 4943 | |
| 4944 | //===----------------------------------------------------------------------===// |
| 4945 | // ComplexImagOp |
| 4946 | //===----------------------------------------------------------------------===// |
| 4947 | |
| 4948 | LogicalResult cir::ComplexImagOp::verify() { |
| 4949 | mlir::Type operandTy = getOperand().getType(); |
| 4950 | if (auto complexOperandTy = mlir::dyn_cast<cir::ComplexType>(Val&: operandTy)) |
| 4951 | operandTy = complexOperandTy.getElementType(); |
| 4952 | |
| 4953 | if (getType() != operandTy) { |
| 4954 | emitOpError() << ": result type does not match operand type" ; |
| 4955 | return failure(); |
| 4956 | } |
| 4957 | |
| 4958 | return success(); |
| 4959 | } |
| 4960 | |
| 4961 | OpFoldResult cir::ComplexImagOp::fold(FoldAdaptor adaptor) { |
| 4962 | if (!mlir::isa<cir::ComplexType>(Val: getOperand().getType())) |
| 4963 | return nullptr; |
| 4964 | |
| 4965 | if (auto complexCreateOp = getOperand().getDefiningOp<cir::ComplexCreateOp>()) |
| 4966 | return complexCreateOp.getOperand(i: 1); |
| 4967 | |
| 4968 | auto complex = |
| 4969 | mlir::cast_if_present<cir::ConstComplexAttr>(Val: adaptor.getOperand()); |
| 4970 | return complex ? complex.getImag() : nullptr; |
| 4971 | } |
| 4972 | |
| 4973 | //===----------------------------------------------------------------------===// |
| 4974 | // ComplexRealPtrOp |
| 4975 | //===----------------------------------------------------------------------===// |
| 4976 | |
| 4977 | LogicalResult cir::ComplexRealPtrOp::verify() { |
| 4978 | mlir::Type resultPointeeTy = getType().getPointee(); |
| 4979 | cir::PointerType operandPtrTy = getOperand().getType(); |
| 4980 | auto operandPointeeTy = |
| 4981 | mlir::cast<cir::ComplexType>(Val: operandPtrTy.getPointee()); |
| 4982 | |
| 4983 | if (resultPointeeTy != operandPointeeTy.getElementType()) { |
| 4984 | return emitOpError() << ": result type does not match operand type" ; |
| 4985 | } |
| 4986 | |
| 4987 | return success(); |
| 4988 | } |
| 4989 | |
| 4990 | //===----------------------------------------------------------------------===// |
| 4991 | // ComplexImagPtrOp |
| 4992 | //===----------------------------------------------------------------------===// |
| 4993 | |
| 4994 | LogicalResult cir::ComplexImagPtrOp::verify() { |
| 4995 | mlir::Type resultPointeeTy = getType().getPointee(); |
| 4996 | cir::PointerType operandPtrTy = getOperand().getType(); |
| 4997 | auto operandPointeeTy = |
| 4998 | mlir::cast<cir::ComplexType>(Val: operandPtrTy.getPointee()); |
| 4999 | |
| 5000 | if (resultPointeeTy != operandPointeeTy.getElementType()) { |
| 5001 | return emitOpError() |
| 5002 | << "cir.complex.imag_ptr result type does not match operand type" ; |
| 5003 | } |
| 5004 | return success(); |
| 5005 | } |
| 5006 | |
| 5007 | //===----------------------------------------------------------------------===// |
| 5008 | // Bit manipulation operations |
| 5009 | //===----------------------------------------------------------------------===// |
| 5010 | |
| 5011 | static OpFoldResult |
| 5012 | foldUnaryBitOp(mlir::Attribute inputAttr, |
| 5013 | llvm::function_ref<llvm::APInt(const llvm::APInt &)> func, |
| 5014 | bool poisonZero = false) { |
| 5015 | if (mlir::isa_and_present<cir::PoisonAttr>(Val: inputAttr)) { |
| 5016 | // Propagate poison value |
| 5017 | return inputAttr; |
| 5018 | } |
| 5019 | |
| 5020 | auto input = mlir::dyn_cast_if_present<IntAttr>(Val&: inputAttr); |
| 5021 | if (!input) |
| 5022 | return nullptr; |
| 5023 | |
| 5024 | llvm::APInt inputValue = input.getValue(); |
| 5025 | if (poisonZero && inputValue.isZero()) |
| 5026 | return cir::PoisonAttr::get(type: input.getType()); |
| 5027 | |
| 5028 | llvm::APInt resultValue = func(inputValue); |
| 5029 | return IntAttr::get(type: input.getType(), value: resultValue); |
| 5030 | } |
| 5031 | |
| 5032 | OpFoldResult BitClrsbOp::fold(FoldAdaptor adaptor) { |
| 5033 | return foldUnaryBitOp(inputAttr: adaptor.getInput(), func: [](const llvm::APInt &inputValue) { |
| 5034 | unsigned resultValue = |
| 5035 | inputValue.getBitWidth() - inputValue.getSignificantBits(); |
| 5036 | return llvm::APInt(inputValue.getBitWidth(), resultValue); |
| 5037 | }); |
| 5038 | } |
| 5039 | |
| 5040 | OpFoldResult BitClzOp::fold(FoldAdaptor adaptor) { |
| 5041 | return foldUnaryBitOp( |
| 5042 | inputAttr: adaptor.getInput(), |
| 5043 | func: [](const llvm::APInt &inputValue) { |
| 5044 | unsigned resultValue = inputValue.countLeadingZeros(); |
| 5045 | return llvm::APInt(inputValue.getBitWidth(), resultValue); |
| 5046 | }, |
| 5047 | poisonZero: getPoisonZero()); |
| 5048 | } |
| 5049 | |
| 5050 | OpFoldResult BitCtzOp::fold(FoldAdaptor adaptor) { |
| 5051 | return foldUnaryBitOp( |
| 5052 | inputAttr: adaptor.getInput(), |
| 5053 | func: [](const llvm::APInt &inputValue) { |
| 5054 | return llvm::APInt(inputValue.getBitWidth(), |
| 5055 | inputValue.countTrailingZeros()); |
| 5056 | }, |
| 5057 | poisonZero: getPoisonZero()); |
| 5058 | } |
| 5059 | |
| 5060 | OpFoldResult BitFfsOp::fold(FoldAdaptor adaptor) { |
| 5061 | return foldUnaryBitOp(inputAttr: adaptor.getInput(), func: [](const llvm::APInt &inputValue) { |
| 5062 | unsigned trailingZeros = inputValue.countTrailingZeros(); |
| 5063 | unsigned result = |
| 5064 | trailingZeros == inputValue.getBitWidth() ? 0 : trailingZeros + 1; |
| 5065 | return llvm::APInt(inputValue.getBitWidth(), result); |
| 5066 | }); |
| 5067 | } |
| 5068 | |
| 5069 | OpFoldResult BitParityOp::fold(FoldAdaptor adaptor) { |
| 5070 | return foldUnaryBitOp(inputAttr: adaptor.getInput(), func: [](const llvm::APInt &inputValue) { |
| 5071 | return llvm::APInt(inputValue.getBitWidth(), inputValue.popcount() % 2); |
| 5072 | }); |
| 5073 | } |
| 5074 | |
| 5075 | OpFoldResult BitPopcountOp::fold(FoldAdaptor adaptor) { |
| 5076 | return foldUnaryBitOp(inputAttr: adaptor.getInput(), func: [](const llvm::APInt &inputValue) { |
| 5077 | return llvm::APInt(inputValue.getBitWidth(), inputValue.popcount()); |
| 5078 | }); |
| 5079 | } |
| 5080 | |
| 5081 | OpFoldResult BitReverseOp::fold(FoldAdaptor adaptor) { |
| 5082 | return foldUnaryBitOp(inputAttr: adaptor.getInput(), func: [](const llvm::APInt &inputValue) { |
| 5083 | return inputValue.reverseBits(); |
| 5084 | }); |
| 5085 | } |
| 5086 | |
| 5087 | OpFoldResult ByteSwapOp::fold(FoldAdaptor adaptor) { |
| 5088 | return foldUnaryBitOp(inputAttr: adaptor.getInput(), func: [](const llvm::APInt &inputValue) { |
| 5089 | return inputValue.byteSwap(); |
| 5090 | }); |
| 5091 | } |
| 5092 | |
| 5093 | OpFoldResult RotateOp::fold(FoldAdaptor adaptor) { |
| 5094 | if (mlir::isa_and_present<cir::PoisonAttr>(Val: adaptor.getInput()) || |
| 5095 | mlir::isa_and_present<cir::PoisonAttr>(Val: adaptor.getAmount())) { |
| 5096 | // Propagate poison values |
| 5097 | return cir::PoisonAttr::get(type: getType()); |
| 5098 | } |
| 5099 | |
| 5100 | auto input = mlir::dyn_cast_if_present<IntAttr>(Val: adaptor.getInput()); |
| 5101 | auto amount = mlir::dyn_cast_if_present<IntAttr>(Val: adaptor.getAmount()); |
| 5102 | if (!input && !amount) |
| 5103 | return nullptr; |
| 5104 | |
| 5105 | // We could fold cir.rotate even if one of its two operands is not a constant: |
| 5106 | // - `cir.rotate left/right %0, 0` could be folded into just %0 even if %0 |
| 5107 | // is not a constant. |
| 5108 | // - `cir.rotate left/right 0/0b111...111, %0` could be folded into 0 or |
| 5109 | // 0b111...111 even if %0 is not a constant. |
| 5110 | |
| 5111 | llvm::APInt inputValue; |
| 5112 | if (input) { |
| 5113 | inputValue = input.getValue(); |
| 5114 | if (inputValue.isZero() || inputValue.isAllOnes()) { |
| 5115 | // An input value of all 0s or all 1s will not change after rotation |
| 5116 | return input; |
| 5117 | } |
| 5118 | } |
| 5119 | |
| 5120 | uint64_t amountValue; |
| 5121 | if (amount) { |
| 5122 | amountValue = amount.getValue().urem(RHS: getInput().getType().getWidth()); |
| 5123 | if (amountValue == 0) { |
| 5124 | // A shift amount of 0 will not change the input value |
| 5125 | return getInput(); |
| 5126 | } |
| 5127 | } |
| 5128 | |
| 5129 | if (!input || !amount) |
| 5130 | return nullptr; |
| 5131 | |
| 5132 | assert(inputValue.getBitWidth() == getInput().getType().getWidth() && |
| 5133 | "input value must have the same bit width as the input type" ); |
| 5134 | |
| 5135 | llvm::APInt resultValue; |
| 5136 | if (isRotateLeft()) |
| 5137 | resultValue = inputValue.rotl(rotateAmt: amountValue); |
| 5138 | else |
| 5139 | resultValue = inputValue.rotr(rotateAmt: amountValue); |
| 5140 | |
| 5141 | return IntAttr::get(context: input.getContext(), type: input.getType(), value: resultValue); |
| 5142 | } |
| 5143 | |
| 5144 | //===----------------------------------------------------------------------===// |
| 5145 | // InlineAsmOp |
| 5146 | //===----------------------------------------------------------------------===// |
| 5147 | |
| 5148 | void cir::InlineAsmOp::print(OpAsmPrinter &p) { |
| 5149 | p << '(' << getAsmFlavor() << ", " ; |
| 5150 | p.increaseIndent(); |
| 5151 | p.printNewline(); |
| 5152 | |
| 5153 | llvm::SmallVector<std::string, 3> names{"out" , "in" , "in_out" }; |
| 5154 | mlir::MutableOperandRangeRange groups = getAsmOperandsMutable(); |
| 5155 | |
| 5156 | for (auto [index, name] : llvm::enumerate(First&: names)) { |
| 5157 | p << name << " = " ; |
| 5158 | |
| 5159 | p << '['; |
| 5160 | llvm::interleaveComma(c: groups[index], os&: p, each_fn: [&](OpOperand &operand) { |
| 5161 | p.printOperand(value: operand.get()); |
| 5162 | p << " : " << operand.get().getType(); |
| 5163 | if (isMemoryOperand(index: operand.getOperandNumber())) |
| 5164 | p << " (maybe_memory)" ; |
| 5165 | }); |
| 5166 | p << "]," ; |
| 5167 | p.printNewline(); |
| 5168 | } |
| 5169 | |
| 5170 | p << "{" ; |
| 5171 | p.printString(string: getAsmString()); |
| 5172 | p << " " ; |
| 5173 | p.printString(string: getConstraints()); |
| 5174 | p << "}" ; |
| 5175 | p.decreaseIndent(); |
| 5176 | p << ')'; |
| 5177 | if (getSideEffects()) |
| 5178 | p << " side_effects" ; |
| 5179 | if (getConvergent()) |
| 5180 | p << " convergent" ; |
| 5181 | |
| 5182 | p.printOptionalAttrDict( |
| 5183 | attrs: getOperation()->getDiscardableAttrDictionary().getValue()); |
| 5184 | |
| 5185 | if (auto v = getRes()) |
| 5186 | p << " -> " << v.getType(); |
| 5187 | } |
| 5188 | |
| 5189 | void cir::InlineAsmOp::build(OpBuilder &odsBuilder, OperationState &odsState, |
| 5190 | ArrayRef<ValueRange> asmOperands, |
| 5191 | StringRef asmString, StringRef constraints, |
| 5192 | bool sideEffects, bool convergent, |
| 5193 | cir::AsmFlavor asmFlavor, |
| 5194 | ArrayRef<Attribute> operandAttrs) { |
| 5195 | // Set up the operands_segments for VariadicOfVariadic |
| 5196 | SmallVector<int32_t> segments; |
| 5197 | for (auto operandRange : asmOperands) { |
| 5198 | segments.push_back(Elt: operandRange.size()); |
| 5199 | odsState.addOperands(newOperands: operandRange); |
| 5200 | } |
| 5201 | |
| 5202 | odsState.addAttribute( |
| 5203 | name: "operands_segments" , |
| 5204 | attr: DenseI32ArrayAttr::get(context: odsBuilder.getContext(), content: segments)); |
| 5205 | odsState.addAttribute(name: "asm_string" , attr: odsBuilder.getStringAttr(bytes: asmString)); |
| 5206 | odsState.addAttribute(name: "constraints" , attr: odsBuilder.getStringAttr(bytes: constraints)); |
| 5207 | odsState.addAttribute(name: "asm_flavor" , |
| 5208 | attr: AsmFlavorAttr::get(context: odsBuilder.getContext(), value: asmFlavor)); |
| 5209 | |
| 5210 | if (sideEffects) |
| 5211 | odsState.addAttribute(name: "side_effects" , attr: odsBuilder.getUnitAttr()); |
| 5212 | |
| 5213 | if (convergent) |
| 5214 | odsState.addAttribute(name: "convergent" , attr: odsBuilder.getUnitAttr()); |
| 5215 | |
| 5216 | odsState.addAttribute(name: "operand_attrs" , attr: odsBuilder.getArrayAttr(value: operandAttrs)); |
| 5217 | } |
| 5218 | |
| 5219 | ParseResult cir::InlineAsmOp::parse(OpAsmParser &parser, |
| 5220 | OperationState &result) { |
| 5221 | llvm::SmallVector<mlir::Attribute> operandAttrs; |
| 5222 | llvm::SmallVector<int32_t> operandsGroupSizes; |
| 5223 | std::string asmString, constraints; |
| 5224 | Type resType; |
| 5225 | MLIRContext *ctxt = parser.getBuilder().getContext(); |
| 5226 | |
| 5227 | auto error = [&](const Twine &msg) -> LogicalResult { |
| 5228 | return parser.emitError(loc: parser.getCurrentLocation(), message: msg); |
| 5229 | }; |
| 5230 | |
| 5231 | auto expected = [&](const std::string &c) { |
| 5232 | return error("expected '" + c + "'" ); |
| 5233 | }; |
| 5234 | |
| 5235 | if (parser.parseLParen().failed()) |
| 5236 | return expected("(" ); |
| 5237 | |
| 5238 | auto flavor = FieldParser<AsmFlavor, AsmFlavor>::parse(parser); |
| 5239 | if (failed(Result: flavor)) |
| 5240 | return error("Unknown AsmFlavor" ); |
| 5241 | |
| 5242 | if (parser.parseComma().failed()) |
| 5243 | return expected("," ); |
| 5244 | |
| 5245 | auto parseValue = [&](Value &v) { |
| 5246 | OpAsmParser::UnresolvedOperand op; |
| 5247 | |
| 5248 | if (parser.parseOperand(result&: op) || parser.parseColon()) |
| 5249 | return error("can't parse operand" ); |
| 5250 | |
| 5251 | Type typ; |
| 5252 | if (parser.parseType(result&: typ).failed()) |
| 5253 | return error("can't parse operand type" ); |
| 5254 | llvm::SmallVector<mlir::Value> tmp; |
| 5255 | if (parser.resolveOperand(operand: op, type: typ, result&: tmp)) |
| 5256 | return error("can't resolve operand" ); |
| 5257 | v = tmp[0]; |
| 5258 | return mlir::success(); |
| 5259 | }; |
| 5260 | |
| 5261 | auto parseOperands = [&](llvm::StringRef name) { |
| 5262 | if (parser.parseKeyword(keyword: name).failed()) |
| 5263 | return error("expected " + name + " operands here" ); |
| 5264 | if (parser.parseEqual().failed()) |
| 5265 | return expected("=" ); |
| 5266 | if (parser.parseLSquare().failed()) |
| 5267 | return expected("[" ); |
| 5268 | |
| 5269 | int size = 0; |
| 5270 | if (parser.parseOptionalRSquare().succeeded()) { |
| 5271 | operandsGroupSizes.push_back(Elt: size); |
| 5272 | if (parser.parseComma()) |
| 5273 | return expected("," ); |
| 5274 | return mlir::success(); |
| 5275 | } |
| 5276 | |
| 5277 | auto parseOperand = [&]() { |
| 5278 | Value val; |
| 5279 | if (parseValue(val).succeeded()) { |
| 5280 | result.operands.push_back(Elt: val); |
| 5281 | size++; |
| 5282 | |
| 5283 | if (parser.parseOptionalLParen().failed()) { |
| 5284 | operandAttrs.push_back(Elt: mlir::DictionaryAttr::get(context: ctxt)); |
| 5285 | return mlir::success(); |
| 5286 | } |
| 5287 | |
| 5288 | if (parser.parseKeyword(keyword: "maybe_memory" ).succeeded()) { |
| 5289 | operandAttrs.push_back(Elt: mlir::UnitAttr::get(context: ctxt)); |
| 5290 | if (parser.parseRParen()) |
| 5291 | return expected(")" ); |
| 5292 | return mlir::success(); |
| 5293 | } else { |
| 5294 | return expected("maybe_memory" ); |
| 5295 | } |
| 5296 | } |
| 5297 | return mlir::failure(); |
| 5298 | }; |
| 5299 | |
| 5300 | if (parser.parseCommaSeparatedList(parseElementFn: parseOperand).failed()) |
| 5301 | return mlir::failure(); |
| 5302 | |
| 5303 | if (parser.parseRSquare().failed() || parser.parseComma().failed()) |
| 5304 | return expected("]" ); |
| 5305 | operandsGroupSizes.push_back(Elt: size); |
| 5306 | return mlir::success(); |
| 5307 | }; |
| 5308 | |
| 5309 | if (parseOperands("out" ).failed() || parseOperands("in" ).failed() || |
| 5310 | parseOperands("in_out" ).failed()) |
| 5311 | return error("failed to parse operands" ); |
| 5312 | |
| 5313 | if (parser.parseLBrace()) |
| 5314 | return expected("{" ); |
| 5315 | if (parser.parseString(string: &asmString)) |
| 5316 | return error("asm string parsing failed" ); |
| 5317 | if (parser.parseString(string: &constraints)) |
| 5318 | return error("constraints string parsing failed" ); |
| 5319 | if (parser.parseRBrace()) |
| 5320 | return expected("}" ); |
| 5321 | if (parser.parseRParen()) |
| 5322 | return expected(")" ); |
| 5323 | |
| 5324 | if (parser.parseOptionalKeyword(keyword: "side_effects" ).succeeded()) |
| 5325 | result.attributes.set(name: "side_effects" , value: UnitAttr::get(context: ctxt)); |
| 5326 | |
| 5327 | if (parser.parseOptionalKeyword(keyword: "convergent" ).succeeded()) |
| 5328 | result.attributes.set(name: "convergent" , value: UnitAttr::get(context: ctxt)); |
| 5329 | |
| 5330 | if (parser.parseOptionalAttrDict(result&: result.attributes).failed()) |
| 5331 | return mlir::failure(); |
| 5332 | |
| 5333 | if (parser.parseOptionalArrow().succeeded() && |
| 5334 | parser.parseType(result&: resType).failed()) |
| 5335 | return mlir::failure(); |
| 5336 | |
| 5337 | result.attributes.set(name: "asm_flavor" , value: AsmFlavorAttr::get(context: ctxt, value: *flavor)); |
| 5338 | result.attributes.set(name: "asm_string" , value: StringAttr::get(context: ctxt, bytes: asmString)); |
| 5339 | result.attributes.set(name: "constraints" , value: StringAttr::get(context: ctxt, bytes: constraints)); |
| 5340 | result.attributes.set(name: "operand_attrs" , value: ArrayAttr::get(context: ctxt, value: operandAttrs)); |
| 5341 | result.getOrAddProperties<InlineAsmOp::Properties>().operands_segments = |
| 5342 | parser.getBuilder().getDenseI32ArrayAttr(values: operandsGroupSizes); |
| 5343 | if (resType) |
| 5344 | result.addTypes(newTypes: TypeRange{resType}); |
| 5345 | |
| 5346 | return mlir::success(); |
| 5347 | } |
| 5348 | |
| 5349 | LogicalResult cir::InlineAsmOp::verify() { |
| 5350 | if (getOperandAttrs().size() != getNumOperands()) |
| 5351 | return emitOpError() << "expects " << getNumOperands() |
| 5352 | << " operand attributes, but got " |
| 5353 | << getOperandAttrs().size(); |
| 5354 | for (auto [index, operand] : llvm::enumerate(First: getOperands())) |
| 5355 | if (isMemoryOperand(index) && !isa<cir::PointerType>(Val: operand.getType())) |
| 5356 | return emitOpError() << "memory operand #" << index |
| 5357 | << " must be a pointer, but got " |
| 5358 | << operand.getType(); |
| 5359 | return success(); |
| 5360 | } |
| 5361 | |
| 5362 | void InlineAsmOp::getEffects( |
| 5363 | llvm::SmallVectorImpl<mlir::MemoryEffects::EffectInstance> &effects) { |
| 5364 | // If we have any side effects (that is, we're volatile asm), add a read and |
| 5365 | // write memory effect. We do this the same as the llvm dialect InlineAsmOp. |
| 5366 | if (getSideEffects()) { |
| 5367 | effects.emplace_back(Args: mlir::MemoryEffects::Read::get()); |
| 5368 | effects.emplace_back(Args: mlir::MemoryEffects::Write::get()); |
| 5369 | } |
| 5370 | // Memory operands are read and written through. |
| 5371 | for (OpOperand &operand : getOperation()->getOpOperands()) { |
| 5372 | if (!isMemoryOperand(index: operand.getOperandNumber())) |
| 5373 | continue; |
| 5374 | effects.emplace_back(Args: mlir::MemoryEffects::Read::get(), Args: &operand); |
| 5375 | effects.emplace_back(Args: mlir::MemoryEffects::Write::get(), Args: &operand); |
| 5376 | } |
| 5377 | } |
| 5378 | |
| 5379 | //===----------------------------------------------------------------------===// |
| 5380 | // ThrowOp / TryThrowOp |
| 5381 | //===----------------------------------------------------------------------===// |
| 5382 | |
| 5383 | template <typename ThrowOpTy> |
| 5384 | static mlir::LogicalResult verifyThrowOpImpl(ThrowOpTy op) { |
| 5385 | if (op.rethrows()) |
| 5386 | return mlir::success(); |
| 5387 | |
| 5388 | if (op.getNumOperands() != 0) { |
| 5389 | if (op.getTypeInfo()) |
| 5390 | return mlir::success(); |
| 5391 | return op.emitOpError() << "'type_info' symbol attribute missing" ; |
| 5392 | } |
| 5393 | |
| 5394 | return mlir::failure(); |
| 5395 | } |
| 5396 | |
| 5397 | mlir::LogicalResult cir::ThrowOp::verify() { return verifyThrowOpImpl(op: *this); } |
| 5398 | |
| 5399 | mlir::LogicalResult cir::TryThrowOp::verify() { |
| 5400 | return verifyThrowOpImpl(op: *this); |
| 5401 | } |
| 5402 | |
| 5403 | //===----------------------------------------------------------------------===// |
| 5404 | // AtomicFetchOp |
| 5405 | //===----------------------------------------------------------------------===// |
| 5406 | |
| 5407 | LogicalResult cir::AtomicFetchOp::verify() { |
| 5408 | if (getBinop() != cir::AtomicFetchKind::Add && |
| 5409 | getBinop() != cir::AtomicFetchKind::Sub && |
| 5410 | getBinop() != cir::AtomicFetchKind::Max && |
| 5411 | getBinop() != cir::AtomicFetchKind::Min && |
| 5412 | getBinop() != cir::AtomicFetchKind::Maximum && |
| 5413 | getBinop() != cir::AtomicFetchKind::Minimum && |
| 5414 | getBinop() != cir::AtomicFetchKind::MaximumNum && |
| 5415 | getBinop() != cir::AtomicFetchKind::MinimumNum && |
| 5416 | !mlir::isa<cir::IntType>(Val: getVal().getType())) |
| 5417 | return emitError(message: "only atomic add, sub, max, min, maximum, minimum, " |
| 5418 | "maximum_num, and minimum_num operation could operate on " |
| 5419 | "floating-point values" ); |
| 5420 | |
| 5421 | if ((getBinop() == cir::AtomicFetchKind::Maximum || |
| 5422 | getBinop() == cir::AtomicFetchKind::Minimum || |
| 5423 | getBinop() == cir::AtomicFetchKind::MaximumNum || |
| 5424 | getBinop() == cir::AtomicFetchKind::MinimumNum) && |
| 5425 | !cir::isFPOrVectorOfFPType(type: getVal().getType())) |
| 5426 | return emitError(message: "atomic maximum, minimum, maximum_num, and minimum_num " |
| 5427 | "operation could only operate on floating-point values" ); |
| 5428 | |
| 5429 | return success(); |
| 5430 | } |
| 5431 | |
| 5432 | //===----------------------------------------------------------------------===// |
| 5433 | // TypeInfoAttr |
| 5434 | //===----------------------------------------------------------------------===// |
| 5435 | |
| 5436 | LogicalResult cir::TypeInfoAttr::verify( |
| 5437 | ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError, |
| 5438 | ::mlir::Type type, ::mlir::ArrayAttr typeInfoData) { |
| 5439 | |
| 5440 | if (cir::ConstRecordAttr::verify(emitError, type, members: typeInfoData).failed()) |
| 5441 | return failure(); |
| 5442 | |
| 5443 | return success(); |
| 5444 | } |
| 5445 | |
| 5446 | //===----------------------------------------------------------------------===// |
| 5447 | // TryOp |
| 5448 | //===----------------------------------------------------------------------===// |
| 5449 | |
| 5450 | void cir::TryOp::getSuccessorRegions( |
| 5451 | mlir::RegionBranchPoint point, |
| 5452 | llvm::SmallVectorImpl<mlir::RegionSuccessor> ®ions) { |
| 5453 | if (point.isParent()) { |
| 5454 | regions.emplace_back(Args: &getTryRegion()); |
| 5455 | } else { |
| 5456 | // A handler exit leaves the operation. |
| 5457 | mlir::Operation *term = point.getTerminatorPredecessorOrNull(); |
| 5458 | if (term->getParentRegion() != &getTryRegion()) { |
| 5459 | regions.emplace_back(Args: getOperation()); |
| 5460 | return; |
| 5461 | } |
| 5462 | // A cir.resume ending the try region unwinds into a handler, so it leaves |
| 5463 | // the operation only when there is none. Every other exit can leave it. |
| 5464 | if (!isa<cir::ResumeOp>(Val: term) || getHandlerRegions().empty()) |
| 5465 | regions.emplace_back(Args: getOperation()); |
| 5466 | } |
| 5467 | |
| 5468 | // Most handlers are entered by an unwind out of a call in the try region, |
| 5469 | // which is not a branch point, so the handlers are successors of the parent |
| 5470 | // and of every exit from the try region to stand in for it. |
| 5471 | // |
| 5472 | // TODO(CIR): If we know a target function never throws a specific type, we |
| 5473 | // can remove the catch handler. |
| 5474 | regions.reserve(N: regions.size() + getHandlerRegions().size()); |
| 5475 | for (mlir::Region &handlerRegion : getHandlerRegions()) |
| 5476 | regions.emplace_back(Args: &handlerRegion); |
| 5477 | } |
| 5478 | |
| 5479 | /// Verify that each cir.init_catch_param in \p handlerRegion, outside any |
| 5480 | /// nested cir.try, agrees with the handler's leading cir.construct_catch_param |
| 5481 | /// \p constructOp, which may be null. A cir.init_catch_param of kind |
| 5482 | /// reference_to_pointer or non_trivial_copy needs \p constructOp on its |
| 5483 | /// parameter, and a cir.init_catch_param on the parameter of \p constructOp |
| 5484 | /// must have the kind of \p constructOp. |
| 5485 | static LogicalResult |
| 5486 | verifyCatchParamPairing(mlir::Region &handlerRegion, |
| 5487 | cir::ConstructCatchParamOp constructOp) { |
| 5488 | mlir::WalkResult result = |
| 5489 | handlerRegion.walk<mlir::WalkOrder::PreOrder>(callback: [&](mlir::Operation *op) { |
| 5490 | if (mlir::isa<cir::TryOp>(Val: op)) |
| 5491 | return mlir::WalkResult::skip(); |
| 5492 | auto initOp = mlir::dyn_cast<cir::InitCatchParamOp>(Val: op); |
| 5493 | if (!initOp) |
| 5494 | return mlir::WalkResult::advance(); |
| 5495 | |
| 5496 | cir::InitCatchKind kind = initOp.getKind(); |
| 5497 | bool sameParam = |
| 5498 | constructOp && constructOp.getParamAddr() == initOp.getParamAddr(); |
| 5499 | if (sameParam && constructOp.getKind() != kind) { |
| 5500 | initOp.emitOpError(message: "kind '" ) |
| 5501 | << cir::stringifyInitCatchKind(val: kind) |
| 5502 | << "' does not match the kind '" |
| 5503 | << cir::stringifyInitCatchKind(val: constructOp.getKind()) |
| 5504 | << "' of the preceding 'cir.construct_catch_param'" ; |
| 5505 | return mlir::WalkResult::interrupt(); |
| 5506 | } |
| 5507 | if ((kind == cir::InitCatchKind::ReferenceToPointer || |
| 5508 | kind == cir::InitCatchKind::NonTrivialCopy) && |
| 5509 | !sameParam) { |
| 5510 | initOp.emitOpError(message: "'" ) |
| 5511 | << cir::stringifyInitCatchKind(val: kind) |
| 5512 | << "' requires a preceding 'cir.construct_catch_param' of the " |
| 5513 | "same kind on the same parameter" ; |
| 5514 | return mlir::WalkResult::interrupt(); |
| 5515 | } |
| 5516 | return mlir::WalkResult::advance(); |
| 5517 | }); |
| 5518 | return failure(IsFailure: result.wasInterrupted()); |
| 5519 | } |
| 5520 | |
| 5521 | LogicalResult cir::TryOp::verifyRegions() { |
| 5522 | mlir::ArrayAttr handlerTypes = getHandlerTypes(); |
| 5523 | if (!handlerTypes) { |
| 5524 | if (!getHandlerRegions().empty()) |
| 5525 | return emitOpError( |
| 5526 | message: "handler regions must be empty when no handler types are present" ); |
| 5527 | return success(); |
| 5528 | } |
| 5529 | |
| 5530 | mlir::MutableArrayRef<mlir::Region> handlerRegions = getHandlerRegions(); |
| 5531 | |
| 5532 | // The parser and builder won't allow this to happen, but the loop below |
| 5533 | // relies on the sizes being the same, so we check it here. |
| 5534 | if (handlerRegions.size() != handlerTypes.size()) |
| 5535 | return emitOpError( |
| 5536 | message: "number of handler regions and handler types must match" ); |
| 5537 | |
| 5538 | // A filter handler and an unexpected handler together implement a dynamic |
| 5539 | // exception specification. The filter try operation wraps the whole function |
| 5540 | // body and exists only to check the specification, so the two must appear |
| 5541 | // together and must stand alone. A catch-all in the same list would make the |
| 5542 | // filter unreachable, and the filter handler already provides the |
| 5543 | // continue-unwinding path that an unwind handler would supply. |
| 5544 | if (llvm::any_of(Range&: handlerTypes, P: [](mlir::Attribute typeAttr) { |
| 5545 | return mlir::isa<cir::EhFilterAttr, cir::EhUnexpectedAttr>(Val: typeAttr); |
| 5546 | })) { |
| 5547 | if (handlerTypes.size() != 2 || |
| 5548 | !mlir::isa<cir::EhFilterAttr>(Val: handlerTypes[0]) || |
| 5549 | !mlir::isa<cir::EhUnexpectedAttr>(Val: handlerTypes[1])) |
| 5550 | return emitOpError(message: "a filter handler must be followed by an unexpected " |
| 5551 | "handler, and the two must be the only handlers" ); |
| 5552 | } |
| 5553 | |
| 5554 | for (const auto &[typeAttr, handlerRegion] : |
| 5555 | llvm::zip(t&: handlerTypes, u&: handlerRegions)) { |
| 5556 | // Verify that handler regions have a !cir.eh_token block argument. |
| 5557 | mlir::Block &entryBlock = handlerRegion.front(); |
| 5558 | if (entryBlock.getNumArguments() != 1 || |
| 5559 | !mlir::isa<cir::EhTokenType>(Val: entryBlock.getArgument(i: 0).getType())) |
| 5560 | return emitOpError( |
| 5561 | message: "handler region must have a single '!cir.eh_token' argument" ); |
| 5562 | |
| 5563 | // The unwind, filter and unexpected regions do not require a |
| 5564 | // cir.begin_catch. None of them catches the exception. |
| 5565 | if (mlir::isa<cir::UnwindAttr, cir::EhFilterAttr, cir::EhUnexpectedAttr>( |
| 5566 | Val: typeAttr)) |
| 5567 | continue; |
| 5568 | |
| 5569 | if (entryBlock.empty()) |
| 5570 | return emitOpError(message: "catch handler region must not be empty" ); |
| 5571 | |
| 5572 | // A terminate scope, which wraps the body of a function that cannot throw, |
| 5573 | // is a catch-all handler that only terminates the program. The exception is |
| 5574 | // caught by the runtime helper that cir.eh.terminate lowers to, so the |
| 5575 | // handler region has no cir.begin_catch of its own. |
| 5576 | if (mlir::isa<cir::EhTerminateOp>(Val: entryBlock.front())) { |
| 5577 | if (!mlir::isa<cir::CatchAllAttr>(Val: typeAttr)) |
| 5578 | return emitOpError(message: "'cir.eh.terminate' is only allowed in a catch-all " |
| 5579 | "handler region" ); |
| 5580 | continue; |
| 5581 | } |
| 5582 | |
| 5583 | // Nothing may run in a catch handler before cir.begin_catch, so it has to |
| 5584 | // be the handler region's first operation, with two exceptions. |
| 5585 | // |
| 5586 | // When lifetime markers are enabled, the catch parameter's storage can be |
| 5587 | // marked by a cir.lifetime.start. That parameter is the only variable |
| 5588 | // whose lifetime begins before the catch is entered, so there is at most |
| 5589 | // one such marker. Its lifetime-end cleanup has to run after the catch |
| 5590 | // handler is finished (or exited by an exception unwind), so if there is a |
| 5591 | // lifetime begin marker, it is followed by a cir.cleanup.scope that |
| 5592 | // encloses the the rest of the handler with a cir.lifetime.end in its |
| 5593 | // cleanup region. |
| 5594 | // |
| 5595 | // A cir.construct_catch_param may also precede cir.begin_catch, to |
| 5596 | // perform any pre-begin_catch initialization of the catch parameter. |
| 5597 | mlir::Operation *firstOp = &entryBlock.front(); |
| 5598 | if (mlir::isa<cir::LifetimeStartOp>(Val: firstOp)) { |
| 5599 | mlir::Operation *next = firstOp->getNextNode(); |
| 5600 | auto lifetimeScope = mlir::dyn_cast_if_present<cir::CleanupScopeOp>(Val: next); |
| 5601 | if (!lifetimeScope) |
| 5602 | return emitOpError(message: "'cir.lifetime.start' in a catch handler region " |
| 5603 | "must be followed by the 'cir.cleanup.scope' of " |
| 5604 | "its lifetime-end cleanup" ); |
| 5605 | if (lifetimeScope.getBodyRegion().empty()) |
| 5606 | return emitOpError( |
| 5607 | message: "'cir.lifetime.start' in a catch handler region must be " |
| 5608 | "followed by the 'cir.cleanup.scope' of its lifetime-end " |
| 5609 | "cleanup" ); |
| 5610 | mlir::Block &scopeBody = lifetimeScope.getBodyRegion().front(); |
| 5611 | firstOp = scopeBody.empty() ? nullptr : &scopeBody.front(); |
| 5612 | } |
| 5613 | |
| 5614 | auto constructOp = |
| 5615 | mlir::dyn_cast_if_present<cir::ConstructCatchParamOp>(Val: firstOp); |
| 5616 | if (constructOp) |
| 5617 | firstOp = firstOp->getNextNode(); |
| 5618 | if (!mlir::isa_and_present<cir::BeginCatchOp>(Val: firstOp)) |
| 5619 | return emitOpError( |
| 5620 | message: "catch handler region must start with 'cir.begin_catch'" ); |
| 5621 | |
| 5622 | if (failed(Result: verifyCatchParamPairing(handlerRegion, constructOp))) |
| 5623 | return failure(); |
| 5624 | } |
| 5625 | |
| 5626 | return success(); |
| 5627 | } |
| 5628 | |
| 5629 | void cir::printTryHandlerRegions( |
| 5630 | mlir::OpAsmPrinter &printer, cir::TryOp op, |
| 5631 | mlir::MutableArrayRef<mlir::Region> handlerRegions, |
| 5632 | mlir::ArrayAttr handlerTypes) { |
| 5633 | if (!handlerTypes) |
| 5634 | return; |
| 5635 | |
| 5636 | for (const auto [typeIdx, typeAttr] : llvm::enumerate(First&: handlerTypes)) { |
| 5637 | if (typeIdx) |
| 5638 | printer << " " ; |
| 5639 | |
| 5640 | if (mlir::isa<cir::CatchAllAttr>(Val: typeAttr)) { |
| 5641 | printer << "catch all " ; |
| 5642 | } else if (mlir::isa<cir::UnwindAttr>(Val: typeAttr)) { |
| 5643 | printer << "unwind " ; |
| 5644 | } else if (auto filterAttr = mlir::dyn_cast<cir::EhFilterAttr>(Val: typeAttr)) { |
| 5645 | printer << "filter [" ; |
| 5646 | llvm::interleaveComma( |
| 5647 | c: filterAttr.getPermittedTypes(), os&: printer, |
| 5648 | each_fn: [&](mlir::Attribute sym) { printer.printAttribute(attr: sym); }); |
| 5649 | printer << "] " ; |
| 5650 | } else if (mlir::isa<cir::EhUnexpectedAttr>(Val: typeAttr)) { |
| 5651 | printer << "unexpected " ; |
| 5652 | } else { |
| 5653 | printer << "catch [type " ; |
| 5654 | printer.printAttribute(attr: typeAttr); |
| 5655 | printer << "] " ; |
| 5656 | } |
| 5657 | |
| 5658 | // Print the handler region's !cir.eh_token block argument. |
| 5659 | mlir::Region ®ion = handlerRegions[typeIdx]; |
| 5660 | if (!region.empty() && region.front().getNumArguments() > 0) { |
| 5661 | printer << "(" ; |
| 5662 | printer.printRegionArgument(arg: region.front().getArgument(i: 0)); |
| 5663 | printer << ") " ; |
| 5664 | } |
| 5665 | |
| 5666 | printer.printRegion(blocks&: region, |
| 5667 | /*printEntryBLockArgs=*/printEntryBlockArgs: false, |
| 5668 | /*printBlockTerminators=*/true); |
| 5669 | } |
| 5670 | } |
| 5671 | |
| 5672 | mlir::ParseResult cir::parseTryHandlerRegions( |
| 5673 | mlir::OpAsmParser &parser, |
| 5674 | llvm::SmallVectorImpl<std::unique_ptr<mlir::Region>> &handlerRegions, |
| 5675 | mlir::ArrayAttr &handlerTypes) { |
| 5676 | |
| 5677 | auto parseCheckedCatcherRegion = [&]() -> mlir::ParseResult { |
| 5678 | handlerRegions.emplace_back(Args: new mlir::Region); |
| 5679 | |
| 5680 | mlir::Region &currRegion = *handlerRegions.back(); |
| 5681 | |
| 5682 | // Parse the required region argument: (%eh_token : !cir.eh_token) |
| 5683 | llvm::SmallVector<mlir::OpAsmParser::Argument> regionArgs; |
| 5684 | if (parser.parseLParen()) |
| 5685 | return failure(); |
| 5686 | mlir::OpAsmParser::Argument arg; |
| 5687 | if (parser.parseArgument(result&: arg, /*allowType=*/true)) |
| 5688 | return failure(); |
| 5689 | regionArgs.push_back(Elt: arg); |
| 5690 | if (parser.parseRParen()) |
| 5691 | return failure(); |
| 5692 | |
| 5693 | mlir::SMLoc regionLoc = parser.getCurrentLocation(); |
| 5694 | if (parser.parseRegion(region&: currRegion, arguments: regionArgs)) { |
| 5695 | handlerRegions.clear(); |
| 5696 | return failure(); |
| 5697 | } |
| 5698 | |
| 5699 | if (currRegion.empty()) |
| 5700 | return parser.emitError(loc: regionLoc, message: "handler region shall not be empty" ); |
| 5701 | |
| 5702 | if (!(currRegion.back().mightHaveTerminator() && |
| 5703 | currRegion.back().getTerminator())) |
| 5704 | return parser.emitError( |
| 5705 | loc: regionLoc, message: "blocks are expected to be explicitly terminated" ); |
| 5706 | |
| 5707 | return success(); |
| 5708 | }; |
| 5709 | |
| 5710 | bool hasCatchAll = false; |
| 5711 | llvm::SmallVector<mlir::Attribute, 4> catcherAttrs; |
| 5712 | while (parser.parseOptionalKeyword(keyword: "catch" ).succeeded()) { |
| 5713 | bool hasLSquare = parser.parseOptionalLSquare().succeeded(); |
| 5714 | |
| 5715 | llvm::StringRef attrStr; |
| 5716 | if (parser.parseOptionalKeyword(keyword: &attrStr, allowedValues: {"all" , "type" }).failed()) |
| 5717 | return parser.emitError(loc: parser.getCurrentLocation(), |
| 5718 | message: "expected 'all' or 'type' keyword" ); |
| 5719 | |
| 5720 | bool isCatchAll = attrStr == "all" ; |
| 5721 | if (isCatchAll) { |
| 5722 | if (hasCatchAll) |
| 5723 | return parser.emitError(loc: parser.getCurrentLocation(), |
| 5724 | message: "can't have more than one catch all" ); |
| 5725 | hasCatchAll = true; |
| 5726 | } |
| 5727 | |
| 5728 | mlir::Attribute exceptionRTTIAttr; |
| 5729 | if (!isCatchAll && parser.parseAttribute(result&: exceptionRTTIAttr).failed()) |
| 5730 | return parser.emitError(loc: parser.getCurrentLocation(), |
| 5731 | message: "expected valid RTTI info attribute" ); |
| 5732 | |
| 5733 | catcherAttrs.push_back(Elt: isCatchAll |
| 5734 | ? cir::CatchAllAttr::get(ctx: parser.getContext()) |
| 5735 | : exceptionRTTIAttr); |
| 5736 | |
| 5737 | if (hasLSquare && isCatchAll) |
| 5738 | return parser.emitError(loc: parser.getCurrentLocation(), |
| 5739 | message: "catch all dosen't need RTTI info attribute" ); |
| 5740 | |
| 5741 | if (hasLSquare && parser.parseRSquare().failed()) |
| 5742 | return parser.emitError(loc: parser.getCurrentLocation(), |
| 5743 | message: "expected `]` after RTTI info attribute" ); |
| 5744 | |
| 5745 | if (parseCheckedCatcherRegion().failed()) |
| 5746 | return mlir::failure(); |
| 5747 | } |
| 5748 | |
| 5749 | // A filter handler carries the type info symbols permitted by the enclosing |
| 5750 | // function's dynamic exception specification. TryOp::verifyRegions enforces |
| 5751 | // that it is paired with an unexpected handler and that the two stand alone. |
| 5752 | if (parser.parseOptionalKeyword(keyword: "filter" ).succeeded()) { |
| 5753 | mlir::SMLoc filterLoc = parser.getCurrentLocation(); |
| 5754 | llvm::SmallVector<mlir::Attribute, 4> permittedTypes; |
| 5755 | auto parsePermittedType = [&]() -> mlir::ParseResult { |
| 5756 | mlir::SMLoc typeLoc = parser.getCurrentLocation(); |
| 5757 | mlir::Attribute rtti; |
| 5758 | if (parser.parseAttribute(result&: rtti).failed()) |
| 5759 | return mlir::failure(); |
| 5760 | if (!mlir::isa<cir::GlobalViewAttr>(Val: rtti)) |
| 5761 | return parser.emitError(loc: typeLoc, message: "expected a type info symbol naming a " |
| 5762 | "permitted exception type" ); |
| 5763 | permittedTypes.push_back(Elt: rtti); |
| 5764 | return mlir::success(); |
| 5765 | }; |
| 5766 | if (parser |
| 5767 | .parseCommaSeparatedList(delimiter: mlir::OpAsmParser::Delimiter::Square, |
| 5768 | parseElementFn: parsePermittedType) |
| 5769 | .failed()) |
| 5770 | return mlir::failure(); |
| 5771 | |
| 5772 | auto filterAttr = cir::EhFilterAttr::getChecked( |
| 5773 | emitError: [&]() { return parser.emitError(loc: filterLoc); }, context: parser.getContext(), |
| 5774 | permitted_types: parser.getBuilder().getArrayAttr(value: permittedTypes)); |
| 5775 | if (!filterAttr) |
| 5776 | return mlir::failure(); |
| 5777 | catcherAttrs.push_back(Elt: filterAttr); |
| 5778 | if (parseCheckedCatcherRegion().failed()) |
| 5779 | return mlir::failure(); |
| 5780 | } |
| 5781 | |
| 5782 | if (parser.parseOptionalKeyword(keyword: "unexpected" ).succeeded()) { |
| 5783 | catcherAttrs.push_back(Elt: cir::EhUnexpectedAttr::get(ctx: parser.getContext())); |
| 5784 | if (parseCheckedCatcherRegion().failed()) |
| 5785 | return mlir::failure(); |
| 5786 | } |
| 5787 | |
| 5788 | if (parser.parseOptionalKeyword(keyword: "unwind" ).succeeded()) { |
| 5789 | if (hasCatchAll) |
| 5790 | return parser.emitError(loc: parser.getCurrentLocation(), |
| 5791 | message: "unwind can't be used with catch all" ); |
| 5792 | |
| 5793 | catcherAttrs.push_back(Elt: cir::UnwindAttr::get(ctx: parser.getContext())); |
| 5794 | if (parseCheckedCatcherRegion().failed()) |
| 5795 | return mlir::failure(); |
| 5796 | } |
| 5797 | |
| 5798 | handlerTypes = parser.getBuilder().getArrayAttr(value: catcherAttrs); |
| 5799 | return mlir::success(); |
| 5800 | } |
| 5801 | |
| 5802 | //===----------------------------------------------------------------------===// |
| 5803 | // EhTypeIdOp |
| 5804 | //===----------------------------------------------------------------------===// |
| 5805 | |
| 5806 | LogicalResult |
| 5807 | cir::EhTypeIdOp::verifySymbolUses(SymbolTableCollection &symbolTable) { |
| 5808 | Operation *op = symbolTable.lookupNearestSymbolFrom(from: *this, symbol: getTypeSymAttr()); |
| 5809 | if (!isa_and_nonnull<GlobalOp>(Val: op)) |
| 5810 | return emitOpError(message: "'" ) |
| 5811 | << getTypeSym() << "' does not reference a valid cir.global" ; |
| 5812 | return success(); |
| 5813 | } |
| 5814 | |
| 5815 | //===----------------------------------------------------------------------===// |
| 5816 | // LifetimeStartOp & LifetimeEndOp |
| 5817 | //===----------------------------------------------------------------------===// |
| 5818 | |
| 5819 | LogicalResult cir::LifetimeStartOp::verify() { |
| 5820 | return verifyProducedBy<cir::AllocaOp>(op: *this, operand: getPtr(), operandName: "ptr" ); |
| 5821 | } |
| 5822 | |
| 5823 | LogicalResult cir::LifetimeEndOp::verify() { |
| 5824 | return verifyProducedBy<cir::AllocaOp>(op: *this, operand: getPtr(), operandName: "ptr" ); |
| 5825 | } |
| 5826 | |
| 5827 | //===----------------------------------------------------------------------===// |
| 5828 | // MemChrOp |
| 5829 | //===----------------------------------------------------------------------===// |
| 5830 | |
| 5831 | /// Reads a fundamental integer width from a signless i32 attribute. |
| 5832 | static std::optional<unsigned> getRecordedIntegerWidth(mlir::Attribute attr) { |
| 5833 | auto intAttr = mlir::dyn_cast<mlir::IntegerAttr>(Val&: attr); |
| 5834 | if (!intAttr || !intAttr.getType().isSignlessInteger(width: 32)) |
| 5835 | return std::nullopt; |
| 5836 | int64_t width = intAttr.getInt(); |
| 5837 | if (width < 0 || |
| 5838 | !cir::isValidFundamentalIntWidth(width: static_cast<unsigned>(width))) |
| 5839 | return std::nullopt; |
| 5840 | return static_cast<unsigned>(width); |
| 5841 | } |
| 5842 | |
| 5843 | LogicalResult cir::MemChrOp::verify() { |
| 5844 | auto moduleOp = (*this)->getParentOfType<mlir::ModuleOp>(); |
| 5845 | if (!moduleOp) |
| 5846 | return emitOpError(message: "expects an enclosing module" ); |
| 5847 | |
| 5848 | // libc memchr uses pointers in the target's default address space. |
| 5849 | if (mlir::cast<cir::PointerType>(Val: getSrc().getType()).getAddrSpace()) |
| 5850 | return emitOpError(message: "src must be in the default address space" ); |
| 5851 | |
| 5852 | auto checkWidth = [&](cir::IntType type, llvm::StringRef operandName, |
| 5853 | llvm::StringRef attrName) -> LogicalResult { |
| 5854 | mlir::Attribute attr = moduleOp->getAttr(name: attrName); |
| 5855 | if (!attr) |
| 5856 | return emitOpError(message: "expects the module to record " ) << attrName; |
| 5857 | std::optional<unsigned> width = getRecordedIntegerWidth(attr); |
| 5858 | if (!width) |
| 5859 | return emitOpError(message: "requires " ) |
| 5860 | << attrName |
| 5861 | << " to be a signless i32 holding a fundamental integer width" ; |
| 5862 | if (type.getWidth() != *width) |
| 5863 | return emitOpError() << operandName << " must have the width recorded in " |
| 5864 | << attrName; |
| 5865 | return success(); |
| 5866 | }; |
| 5867 | |
| 5868 | if (failed(Result: checkWidth(getPattern().getType(), "pattern" , |
| 5869 | cir::CIRDialect::getIntTypeWidthAttrName()))) |
| 5870 | return failure(); |
| 5871 | return checkWidth(getLen().getType(), "len" , |
| 5872 | cir::CIRDialect::getSizeTypeWidthAttrName()); |
| 5873 | } |
| 5874 | |
| 5875 | //===----------------------------------------------------------------------===// |
| 5876 | // InitCatchParamOp |
| 5877 | //===----------------------------------------------------------------------===// |
| 5878 | |
| 5879 | /// For the reference_to_pointer and reference_to_record_pointer kinds, emit an |
| 5880 | /// error on \p op unless \p paramAddrType is the type of the address of a |
| 5881 | /// reference to a pointer. Any other \p kind is accepted. |
| 5882 | static LogicalResult verifyCatchParamAddr(mlir::Operation *op, |
| 5883 | cir::InitCatchKind kind, |
| 5884 | cir::PointerType paramAddrType) { |
| 5885 | if (kind != cir::InitCatchKind::ReferenceToPointer && |
| 5886 | kind != cir::InitCatchKind::ReferenceToRecordPointer) |
| 5887 | return success(); |
| 5888 | |
| 5889 | auto refType = mlir::dyn_cast<cir::PointerType>(Val: paramAddrType.getPointee()); |
| 5890 | if (!refType || !mlir::isa<cir::PointerType>(Val: refType.getPointee())) |
| 5891 | return op->emitOpError(message: "'" ) |
| 5892 | << cir::stringifyInitCatchKind(val: kind) |
| 5893 | << "' requires 'param_addr' to be the address of a reference to a " |
| 5894 | "pointer" ; |
| 5895 | return success(); |
| 5896 | } |
| 5897 | |
| 5898 | LogicalResult cir::InitCatchParamOp::verify() { |
| 5899 | return verifyCatchParamAddr(op: *this, kind: getKind(), paramAddrType: getParamAddr().getType()); |
| 5900 | } |
| 5901 | |
| 5902 | //===----------------------------------------------------------------------===// |
| 5903 | // ConstructCatchParamOp |
| 5904 | //===----------------------------------------------------------------------===// |
| 5905 | |
| 5906 | LogicalResult cir::ConstructCatchParamOp::verify() { |
| 5907 | return verifyCatchParamAddr(op: *this, kind: getKind(), paramAddrType: getParamAddr().getType()); |
| 5908 | } |
| 5909 | |
| 5910 | LogicalResult cir::ConstructCatchParamOp::verifySymbolUses( |
| 5911 | SymbolTableCollection &symbolTable) { |
| 5912 | auto copyFnAttr = getCopyFnAttr(); |
| 5913 | if (!copyFnAttr) |
| 5914 | return success(); |
| 5915 | auto fn = |
| 5916 | symbolTable.lookupNearestSymbolFrom<cir::FuncOp>(from: *this, symbol: getCopyFnAttr()); |
| 5917 | if (!fn) |
| 5918 | return emitOpError(message: "'" ) |
| 5919 | << *getCopyFn() << "' does not reference a valid cir.func" ; |
| 5920 | |
| 5921 | if (!fn->hasAttr(name: cir::CIRDialect::getCatchCopyThunkAttrName())) |
| 5922 | return emitOpError(message: "catch-init copy_fn must be tagged with the " ) |
| 5923 | << cir::CIRDialect::getCatchCopyThunkAttrName() << " attribute" ; |
| 5924 | |
| 5925 | cir::FuncType fnType = fn.getFunctionType(); |
| 5926 | if (fnType.getNumInputs() != 2 || !fnType.hasVoidReturn()) |
| 5927 | return emitOpError(message: "catch-init copy_fn must take two pointer arguments and " |
| 5928 | "return void" ); |
| 5929 | |
| 5930 | if (fnType.getInput(i: 0) != getParamAddr().getType()) |
| 5931 | return emitOpError(message: "first argument of catch-init copy_fn must match the " |
| 5932 | "type of 'param_addr'" ); |
| 5933 | |
| 5934 | if (fnType.getInput(i: 1) != getParamAddr().getType()) |
| 5935 | return emitOpError( |
| 5936 | message: "second argument of catch-init copy_fn must be a pointer " |
| 5937 | "to the catch type" ); |
| 5938 | |
| 5939 | return success(); |
| 5940 | } |
| 5941 | |
| 5942 | //===----------------------------------------------------------------------===// |
| 5943 | // EhDispatchOp |
| 5944 | //===----------------------------------------------------------------------===// |
| 5945 | |
| 5946 | LogicalResult cir::EhDispatchOp::verify() { |
| 5947 | mlir::ArrayAttr handlerTypes = getCatchTypesAttr(); |
| 5948 | if (!handlerTypes) |
| 5949 | return success(); |
| 5950 | |
| 5951 | bool hasFilter = false; |
| 5952 | for (mlir::Attribute typeAttr : handlerTypes) { |
| 5953 | if (!mlir::isa<cir::EhFilterAttr>(Val: typeAttr)) |
| 5954 | continue; |
| 5955 | if (hasFilter) |
| 5956 | return emitOpError(message: "can't have more than one 'filter' handler" ); |
| 5957 | hasFilter = true; |
| 5958 | } |
| 5959 | |
| 5960 | // Unlike the other handlers, a filter does not take the place of the default |
| 5961 | // destination. Its destination is taken when the exception is not permitted |
| 5962 | // by the specification, and the default 'unwind' destination is taken when |
| 5963 | // it is. A catch-all default would leave the filter unreachable. |
| 5964 | if (hasFilter && getDefaultIsCatchAll()) |
| 5965 | return emitOpError( |
| 5966 | message: "'filter' handler requires an 'unwind' default destination" ); |
| 5967 | |
| 5968 | return success(); |
| 5969 | } |
| 5970 | |
| 5971 | ParseResult cir::parseEhDispatchDestinations( |
| 5972 | OpAsmParser &parser, mlir::ArrayAttr &catchTypes, |
| 5973 | SmallVectorImpl<Block *> &catchDestinations, Block *&defaultDestination, |
| 5974 | mlir::UnitAttr &defaultIsCatchAll) { |
| 5975 | // Parse: [ ... ] |
| 5976 | if (parser.parseLSquare()) |
| 5977 | return failure(); |
| 5978 | |
| 5979 | SmallVector<Attribute> handlerTypes; |
| 5980 | bool hasCatchAll = false; |
| 5981 | bool hasUnwind = false; |
| 5982 | |
| 5983 | // Parse handler list. |
| 5984 | auto parseHandler = [&]() -> ParseResult { |
| 5985 | // Check for 'catch_all' or 'unwind' keywords. |
| 5986 | if (succeeded(Result: parser.parseOptionalKeyword(keyword: "catch_all" ))) { |
| 5987 | if (hasCatchAll) |
| 5988 | return parser.emitError(loc: parser.getCurrentLocation(), |
| 5989 | message: "duplicate 'catch_all' handler" ); |
| 5990 | if (hasUnwind) |
| 5991 | return parser.emitError(loc: parser.getCurrentLocation(), |
| 5992 | message: "cannot have both 'catch_all' and 'unwind'" ); |
| 5993 | hasCatchAll = true; |
| 5994 | |
| 5995 | if (parser.parseColon().failed()) |
| 5996 | return failure(); |
| 5997 | |
| 5998 | if (parser.parseSuccessor(dest&: defaultDestination).failed()) |
| 5999 | return failure(); |
| 6000 | |
| 6001 | return success(); |
| 6002 | } |
| 6003 | |
| 6004 | if (succeeded(Result: parser.parseOptionalKeyword(keyword: "unwind" ))) { |
| 6005 | if (hasUnwind) |
| 6006 | return parser.emitError(loc: parser.getCurrentLocation(), |
| 6007 | message: "duplicate 'unwind' handler" ); |
| 6008 | if (hasCatchAll) |
| 6009 | return parser.emitError(loc: parser.getCurrentLocation(), |
| 6010 | message: "cannot have both 'catch_all' and 'unwind'" ); |
| 6011 | hasUnwind = true; |
| 6012 | |
| 6013 | if (parser.parseColon().failed()) |
| 6014 | return failure(); |
| 6015 | |
| 6016 | if (parser.parseSuccessor(dest&: defaultDestination).failed()) |
| 6017 | return failure(); |
| 6018 | return success(); |
| 6019 | } |
| 6020 | |
| 6021 | // A filter handler carries the type info symbols permitted by the |
| 6022 | // enclosing function's dynamic exception specification. Its destination is |
| 6023 | // taken when the exception is *not* one of those types. |
| 6024 | if (succeeded(Result: parser.parseOptionalKeyword(keyword: "filter" ))) { |
| 6025 | SMLoc filterLoc = parser.getCurrentLocation(); |
| 6026 | SmallVector<Attribute, 4> permittedTypes; |
| 6027 | auto parsePermittedType = [&]() -> ParseResult { |
| 6028 | mlir::SMLoc typeLoc = parser.getCurrentLocation(); |
| 6029 | mlir::Attribute rtti; |
| 6030 | if (parser.parseAttribute(result&: rtti).failed()) |
| 6031 | return failure(); |
| 6032 | if (!mlir::isa<cir::GlobalViewAttr>(Val: rtti)) |
| 6033 | return parser.emitError(loc: typeLoc, |
| 6034 | message: "expected a type info symbol naming a " |
| 6035 | "permitted exception type" ); |
| 6036 | permittedTypes.push_back(Elt: rtti); |
| 6037 | return success(); |
| 6038 | }; |
| 6039 | if (parser |
| 6040 | .parseCommaSeparatedList(delimiter: OpAsmParser::Delimiter::Paren, |
| 6041 | parseElementFn: parsePermittedType) |
| 6042 | .failed()) |
| 6043 | return failure(); |
| 6044 | |
| 6045 | auto filterAttr = cir::EhFilterAttr::getChecked( |
| 6046 | emitError: [&]() { return parser.emitError(loc: filterLoc); }, context: parser.getContext(), |
| 6047 | permitted_types: parser.getBuilder().getArrayAttr(value: permittedTypes)); |
| 6048 | if (!filterAttr) |
| 6049 | return failure(); |
| 6050 | handlerTypes.push_back(Elt: filterAttr); |
| 6051 | |
| 6052 | if (parser.parseColon().failed()) |
| 6053 | return failure(); |
| 6054 | |
| 6055 | Block *dest; |
| 6056 | if (parser.parseSuccessor(dest).failed()) |
| 6057 | return failure(); |
| 6058 | catchDestinations.push_back(Elt: dest); |
| 6059 | return success(); |
| 6060 | } |
| 6061 | |
| 6062 | // Otherwise, expect 'catch(<attr> : <type>) : ^block'. |
| 6063 | // The 'catch(...)' wrapper allows the attribute to include its type |
| 6064 | // without conflicting with the ':' used for the block destination. |
| 6065 | if (parser.parseKeyword(keyword: "catch" ).failed()) |
| 6066 | return failure(); |
| 6067 | |
| 6068 | if (parser.parseLParen().failed()) |
| 6069 | return failure(); |
| 6070 | |
| 6071 | mlir::Attribute catchTypeAttr; |
| 6072 | if (parser.parseAttribute(result&: catchTypeAttr).failed()) |
| 6073 | return failure(); |
| 6074 | handlerTypes.push_back(Elt: catchTypeAttr); |
| 6075 | |
| 6076 | if (parser.parseRParen().failed()) |
| 6077 | return failure(); |
| 6078 | |
| 6079 | if (parser.parseColon().failed()) |
| 6080 | return failure(); |
| 6081 | |
| 6082 | Block *dest; |
| 6083 | if (parser.parseSuccessor(dest).failed()) |
| 6084 | return failure(); |
| 6085 | catchDestinations.push_back(Elt: dest); |
| 6086 | return success(); |
| 6087 | }; |
| 6088 | |
| 6089 | if (parser.parseCommaSeparatedList(parseElementFn: parseHandler).failed()) |
| 6090 | return failure(); |
| 6091 | |
| 6092 | if (parser.parseRSquare().failed()) |
| 6093 | return failure(); |
| 6094 | |
| 6095 | // Verify we have catch_all or unwind. |
| 6096 | if (!hasCatchAll && !hasUnwind) |
| 6097 | return parser.emitError(loc: parser.getCurrentLocation(), |
| 6098 | message: "must have either 'catch_all' or 'unwind' handler" ); |
| 6099 | |
| 6100 | // Add attributes and successors. |
| 6101 | if (!handlerTypes.empty()) |
| 6102 | catchTypes = parser.getBuilder().getArrayAttr(value: handlerTypes); |
| 6103 | |
| 6104 | if (hasCatchAll) |
| 6105 | defaultIsCatchAll = parser.getBuilder().getUnitAttr(); |
| 6106 | |
| 6107 | return success(); |
| 6108 | } |
| 6109 | |
| 6110 | void cir::printEhDispatchDestinations(OpAsmPrinter &p, cir::EhDispatchOp op, |
| 6111 | mlir::ArrayAttr catchTypes, |
| 6112 | SuccessorRange catchDestinations, |
| 6113 | Block *defaultDestination, |
| 6114 | mlir::UnitAttr defaultIsCatchAll) { |
| 6115 | p << " [" ; |
| 6116 | p.printNewline(); |
| 6117 | |
| 6118 | // If we have at least one catch type, print them. |
| 6119 | if (catchTypes) { |
| 6120 | // Print type handlers using 'catch(<attr>) : ^block' syntax. |
| 6121 | llvm::interleave( |
| 6122 | c: llvm::zip(t&: catchTypes, u&: catchDestinations), |
| 6123 | each_fn: [&](auto i) { |
| 6124 | mlir::Attribute typeAttr = std::get<0>(i); |
| 6125 | if (auto filterAttr = mlir::dyn_cast<cir::EhFilterAttr>(Val&: typeAttr)) { |
| 6126 | p << " filter(" ; |
| 6127 | llvm::interleaveComma( |
| 6128 | filterAttr.getPermittedTypes(), p, |
| 6129 | [&](mlir::Attribute sym) { p.printAttribute(attr: sym); }); |
| 6130 | } else { |
| 6131 | p << " catch(" ; |
| 6132 | p.printAttribute(attr: typeAttr); |
| 6133 | } |
| 6134 | p << ") : " ; |
| 6135 | p.printSuccessor(successor: std::get<1>(i)); |
| 6136 | }, |
| 6137 | between_fn: [&] { |
| 6138 | p << ','; |
| 6139 | p.printNewline(); |
| 6140 | }); |
| 6141 | |
| 6142 | p << ", " ; |
| 6143 | p.printNewline(); |
| 6144 | } |
| 6145 | |
| 6146 | // Print catch_all or unwind handler. |
| 6147 | if (defaultIsCatchAll) |
| 6148 | p << " catch_all : " ; |
| 6149 | else |
| 6150 | p << " unwind : " ; |
| 6151 | p.printSuccessor(successor: defaultDestination); |
| 6152 | p.printNewline(); |
| 6153 | |
| 6154 | p << "]" ; |
| 6155 | } |
| 6156 | |
| 6157 | //===----------------------------------------------------------------------===// |
| 6158 | // Standard library op signature matching |
| 6159 | //===----------------------------------------------------------------------===// |
| 6160 | |
| 6161 | bool cir::StdFindOp::signatureMatches(mlir::TypeRange operands, |
| 6162 | mlir::TypeRange results) { |
| 6163 | if (operands.size() != getNumArgs() || results.size() != 1) |
| 6164 | return false; |
| 6165 | mlir::Type iterTy = operands[0]; |
| 6166 | return iterTy == operands[1] && iterTy == operands[2] && iterTy == results[0]; |
| 6167 | } |
| 6168 | |