| 1 | //===- Operation.cpp - Operation support code -----------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/IR/Operation.h" |
| 10 | #include "mlir/IR/Attributes.h" |
| 11 | #include "mlir/IR/BuiltinAttributes.h" |
| 12 | #include "mlir/IR/BuiltinTypes.h" |
| 13 | #include "mlir/IR/Dialect.h" |
| 14 | #include "mlir/IR/IRMapping.h" |
| 15 | #include "mlir/IR/Matchers.h" |
| 16 | #include "mlir/IR/OpImplementation.h" |
| 17 | #include "mlir/IR/OperationSupport.h" |
| 18 | #include "mlir/IR/PatternMatch.h" |
| 19 | #include "mlir/IR/TypeUtilities.h" |
| 20 | #include "mlir/Interfaces/FoldInterfaces.h" |
| 21 | #include "llvm/ADT/STLExtras.h" |
| 22 | #include "llvm/ADT/SmallVector.h" |
| 23 | #include "llvm/Support/ErrorHandling.h" |
| 24 | #include <optional> |
| 25 | |
| 26 | using namespace mlir; |
| 27 | |
| 28 | //===----------------------------------------------------------------------===// |
| 29 | // Operation |
| 30 | //===----------------------------------------------------------------------===// |
| 31 | |
| 32 | /// Create a new Operation from operation state. |
| 33 | Operation *Operation::create(const OperationState &state) { |
| 34 | Operation *op = |
| 35 | create(location: state.location, name: state.name, resultTypes: state.types, operands: state.operands, |
| 36 | attributes: state.attributes.getDictionary(context: state.getContext()), |
| 37 | properties: state.properties, successors: state.successors, regions: state.regions); |
| 38 | if (LLVM_UNLIKELY(state.propertiesAttr)) { |
| 39 | assert(!state.properties); |
| 40 | LogicalResult result = |
| 41 | op->setPropertiesFromAttribute(attr: state.propertiesAttr, |
| 42 | /*emitError=*/nullptr); |
| 43 | assert(result.succeeded() && "invalid properties in op creation" ); |
| 44 | (void)result; |
| 45 | } |
| 46 | return op; |
| 47 | } |
| 48 | |
| 49 | /// Create a new Operation with the specific fields. |
| 50 | Operation *Operation::create(Location location, OperationName name, |
| 51 | TypeRange resultTypes, ValueRange operands, |
| 52 | NamedAttrList &&attributes, PropertyRef properties, |
| 53 | BlockRange successors, RegionRange regions) { |
| 54 | unsigned numRegions = regions.size(); |
| 55 | Operation *op = |
| 56 | create(location, name, resultTypes, operands, attributes: std::move(attributes), |
| 57 | properties, successors, numRegions); |
| 58 | for (unsigned i = 0; i < numRegions; ++i) |
| 59 | if (regions[i]) |
| 60 | op->getRegion(index: i).takeBody(other&: *regions[i]); |
| 61 | return op; |
| 62 | } |
| 63 | |
| 64 | /// Create a new Operation with the specific fields. |
| 65 | Operation *Operation::create(Location location, OperationName name, |
| 66 | TypeRange resultTypes, ValueRange operands, |
| 67 | NamedAttrList &&attributes, PropertyRef properties, |
| 68 | BlockRange successors, unsigned numRegions) { |
| 69 | // Populate default attributes. |
| 70 | name.populateDefaultAttrs(attrs&: attributes); |
| 71 | |
| 72 | return create(location, name, resultTypes, operands, |
| 73 | attributes: attributes.getDictionary(context: location.getContext()), properties, |
| 74 | successors, numRegions); |
| 75 | } |
| 76 | |
| 77 | /// Overload of create that takes an existing DictionaryAttr to avoid |
| 78 | /// unnecessarily uniquing a list of attributes. |
| 79 | Operation *Operation::create(Location location, OperationName name, |
| 80 | TypeRange resultTypes, ValueRange operands, |
| 81 | DictionaryAttr attributes, PropertyRef properties, |
| 82 | BlockRange successors, unsigned numRegions) { |
| 83 | assert(llvm::all_of(resultTypes, [](Type t) { return t; }) && |
| 84 | "unexpected null result type" ); |
| 85 | |
| 86 | // We only need to allocate additional memory for a subset of results. |
| 87 | unsigned numTrailingResults = OpResult::getNumTrailing(numResults: resultTypes.size()); |
| 88 | unsigned numInlineResults = OpResult::getNumInline(numResults: resultTypes.size()); |
| 89 | unsigned numSuccessors = successors.size(); |
| 90 | unsigned numOperands = operands.size(); |
| 91 | unsigned numResults = resultTypes.size(); |
| 92 | int opPropertiesAllocSize = llvm::alignTo<8>(Value: name.getOpPropertyByteSize()); |
| 93 | |
| 94 | // If the operation is known to have no operands, don't allocate an operand |
| 95 | // storage. |
| 96 | bool needsOperandStorage = |
| 97 | operands.empty() ? !name.hasTrait<OpTrait::ZeroOperands>() : true; |
| 98 | |
| 99 | // Compute the byte size for the operation and the operand storage. This takes |
| 100 | // into account the size of the operation, its trailing objects, and its |
| 101 | // prefixed objects. |
| 102 | size_t byteSize = |
| 103 | totalSizeToAlloc<detail::OperandStorage, detail::OpProperties, |
| 104 | BlockOperand, Region, OpOperand>( |
| 105 | Counts: needsOperandStorage ? 1 : 0, Counts: opPropertiesAllocSize, Counts: numSuccessors, |
| 106 | Counts: numRegions, Counts: numOperands); |
| 107 | size_t prefixByteSize = llvm::alignTo( |
| 108 | Value: Operation::prefixAllocSize(numOutOfLineResults: numTrailingResults, numInlineResults), |
| 109 | Align: alignof(Operation)); |
| 110 | char *mallocMem = reinterpret_cast<char *>(malloc(size: byteSize + prefixByteSize)); |
| 111 | void *rawMem = mallocMem + prefixByteSize; |
| 112 | |
| 113 | // Create the new Operation. |
| 114 | Operation *op = ::new (rawMem) Operation( |
| 115 | location, name, numResults, numSuccessors, numRegions, |
| 116 | opPropertiesAllocSize, attributes, properties, needsOperandStorage); |
| 117 | |
| 118 | assert((numSuccessors == 0 || op->mightHaveTrait<OpTrait::IsTerminator>()) && |
| 119 | "unexpected successors in a non-terminator operation" ); |
| 120 | |
| 121 | // Initialize the results. |
| 122 | auto resultTypeIt = resultTypes.begin(); |
| 123 | for (unsigned i = 0; i < numInlineResults; ++i, ++resultTypeIt) |
| 124 | new (op->getInlineOpResult(resultNumber: i)) detail::InlineOpResult(*resultTypeIt, i); |
| 125 | for (unsigned i = 0; i < numTrailingResults; ++i, ++resultTypeIt) { |
| 126 | new (op->getOutOfLineOpResult(resultNumber: i)) |
| 127 | detail::OutOfLineOpResult(*resultTypeIt, i); |
| 128 | } |
| 129 | |
| 130 | // Initialize the regions. |
| 131 | for (unsigned i = 0; i != numRegions; ++i) |
| 132 | new (&op->getRegion(index: i)) Region(op); |
| 133 | |
| 134 | // Initialize the operands. |
| 135 | if (needsOperandStorage) { |
| 136 | new (&op->getOperandStorage()) detail::OperandStorage( |
| 137 | op, op->getTrailingObjects<OpOperand>(), operands); |
| 138 | } |
| 139 | |
| 140 | // Initialize the successors. |
| 141 | auto blockOperands = op->getBlockOperands(); |
| 142 | for (unsigned i = 0; i != numSuccessors; ++i) |
| 143 | new (&blockOperands[i]) BlockOperand(op, successors[i]); |
| 144 | |
| 145 | // This must be done after properties are initialized. |
| 146 | op->setAttrs(attributes); |
| 147 | |
| 148 | return op; |
| 149 | } |
| 150 | |
| 151 | Operation::Operation(Location location, OperationName name, unsigned numResults, |
| 152 | unsigned numSuccessors, unsigned numRegions, |
| 153 | int fullPropertiesStorageSize, DictionaryAttr attributes, |
| 154 | PropertyRef properties, bool hasOperandStorage) |
| 155 | : location(location), numResults(numResults), numSuccs(numSuccessors), |
| 156 | numRegions(numRegions), hasOperandStorage(hasOperandStorage), |
| 157 | propertiesStorageSize((fullPropertiesStorageSize + 7) / 8), name(name) { |
| 158 | assert(attributes && "unexpected null attribute dictionary" ); |
| 159 | assert(fullPropertiesStorageSize <= propertiesCapacity && |
| 160 | "Properties size overflow" ); |
| 161 | #ifndef NDEBUG |
| 162 | if (!getDialect() && !getContext()->allowsUnregisteredDialects()) |
| 163 | llvm::report_fatal_error( |
| 164 | name.getStringRef() + |
| 165 | " created with unregistered dialect. If this is intended, please call " |
| 166 | "allowUnregisteredDialects() on the MLIRContext, or use " |
| 167 | "-allow-unregistered-dialect with the MLIR tool used." ); |
| 168 | #endif |
| 169 | if (fullPropertiesStorageSize) |
| 170 | name.initOpProperties(storage: getPropertiesStorage(), init: properties); |
| 171 | } |
| 172 | |
| 173 | // Operations are deleted through the destroy() member because they are |
| 174 | // allocated via malloc. |
| 175 | Operation::~Operation() { |
| 176 | assert(block == nullptr && "operation destroyed but still in a block" ); |
| 177 | #ifndef NDEBUG |
| 178 | if (!use_empty()) { |
| 179 | { |
| 180 | InFlightDiagnostic diag = |
| 181 | emitOpError("operation destroyed but still has uses" ); |
| 182 | for (Operation *user : getUsers()) |
| 183 | diag.attachNote(user->getLoc()) << "- use: " << *user << "\n" ; |
| 184 | } |
| 185 | llvm::report_fatal_error("operation destroyed but still has uses" ); |
| 186 | } |
| 187 | #endif |
| 188 | // Explicitly run the destructors for the operands. |
| 189 | if (hasOperandStorage) |
| 190 | getOperandStorage().~OperandStorage(); |
| 191 | |
| 192 | // Explicitly run the destructors for the successors. |
| 193 | for (auto &successor : getBlockOperands()) |
| 194 | successor.~BlockOperand(); |
| 195 | |
| 196 | // Explicitly destroy the regions. |
| 197 | for (auto ®ion : getRegions()) |
| 198 | region.~Region(); |
| 199 | if (propertiesStorageSize) |
| 200 | name.destroyOpProperties(properties: getPropertiesStorage()); |
| 201 | } |
| 202 | |
| 203 | /// Destroy this operation or one of its subclasses. |
| 204 | void Operation::destroy() { |
| 205 | // Operations may have additional prefixed allocation, which needs to be |
| 206 | // accounted for here when computing the address to free. |
| 207 | char *rawMem = reinterpret_cast<char *>(this) - |
| 208 | llvm::alignTo(Value: prefixAllocSize(), Align: alignof(Operation)); |
| 209 | this->~Operation(); |
| 210 | free(ptr: rawMem); |
| 211 | } |
| 212 | |
| 213 | /// Return true if this operation is a proper ancestor of the `other` |
| 214 | /// operation. |
| 215 | bool Operation::isProperAncestor(Operation *other) { |
| 216 | while ((other = other->getParentOp())) |
| 217 | if (this == other) |
| 218 | return true; |
| 219 | return false; |
| 220 | } |
| 221 | |
| 222 | /// Replace any uses of 'from' with 'to' within this operation. |
| 223 | void Operation::replaceUsesOfWith(Value from, Value to) { |
| 224 | if (from == to) |
| 225 | return; |
| 226 | for (auto &operand : getOpOperands()) |
| 227 | if (operand.get() == from) |
| 228 | operand.set(to); |
| 229 | } |
| 230 | |
| 231 | /// Replace the current operands of this operation with the ones provided in |
| 232 | /// 'operands'. |
| 233 | void Operation::setOperands(ValueRange operands) { |
| 234 | if (LLVM_LIKELY(hasOperandStorage)) |
| 235 | return getOperandStorage().setOperands(owner: this, values: operands); |
| 236 | assert(operands.empty() && "setting operands without an operand storage" ); |
| 237 | } |
| 238 | |
| 239 | /// Replace the operands beginning at 'start' and ending at 'start' + 'length' |
| 240 | /// with the ones provided in 'operands'. 'operands' may be smaller or larger |
| 241 | /// than the range pointed to by 'start'+'length'. |
| 242 | void Operation::setOperands(unsigned start, unsigned length, |
| 243 | ValueRange operands) { |
| 244 | assert((start + length) <= getNumOperands() && |
| 245 | "invalid operand range specified" ); |
| 246 | if (LLVM_LIKELY(hasOperandStorage)) |
| 247 | return getOperandStorage().setOperands(owner: this, start, length, operands); |
| 248 | assert(operands.empty() && "setting operands without an operand storage" ); |
| 249 | } |
| 250 | |
| 251 | /// Insert the given operands into the operand list at the given 'index'. |
| 252 | void Operation::insertOperands(unsigned index, ValueRange operands) { |
| 253 | if (LLVM_LIKELY(hasOperandStorage)) |
| 254 | return setOperands(start: index, /*length=*/0, operands); |
| 255 | assert(operands.empty() && "inserting operands without an operand storage" ); |
| 256 | } |
| 257 | |
| 258 | //===----------------------------------------------------------------------===// |
| 259 | // Diagnostics |
| 260 | //===----------------------------------------------------------------------===// |
| 261 | |
| 262 | /// Emit an error about fatal conditions with this operation, reporting up to |
| 263 | /// any diagnostic handlers that may be listening. |
| 264 | InFlightDiagnostic Operation::emitError(const Twine &message) { |
| 265 | InFlightDiagnostic diag = mlir::emitError(loc: getLoc(), message); |
| 266 | if (getContext()->shouldPrintOpOnDiagnostic()) { |
| 267 | diag.attachNote(noteLoc: getLoc()) |
| 268 | .append(arg: "see current operation: " ) |
| 269 | .appendOp(op&: *this, flags: OpPrintingFlags().printGenericOpForm()); |
| 270 | } |
| 271 | return diag; |
| 272 | } |
| 273 | |
| 274 | /// Emit a warning about this operation, reporting up to any diagnostic |
| 275 | /// handlers that may be listening. |
| 276 | InFlightDiagnostic Operation::emitWarning(const Twine &message) { |
| 277 | InFlightDiagnostic diag = mlir::emitWarning(loc: getLoc(), message); |
| 278 | if (getContext()->shouldPrintOpOnDiagnostic()) |
| 279 | diag.attachNote(noteLoc: getLoc()) << "see current operation: " << *this; |
| 280 | return diag; |
| 281 | } |
| 282 | |
| 283 | /// Emit a remark about this operation, reporting up to any diagnostic |
| 284 | /// handlers that may be listening. |
| 285 | InFlightDiagnostic Operation::(const Twine &message) { |
| 286 | InFlightDiagnostic diag = mlir::emitRemark(loc: getLoc(), message); |
| 287 | if (getContext()->shouldPrintOpOnDiagnostic()) |
| 288 | diag.attachNote(noteLoc: getLoc()) << "see current operation: " << *this; |
| 289 | return diag; |
| 290 | } |
| 291 | |
| 292 | DictionaryAttr Operation::getAttrDictionary() { |
| 293 | if (getPropertiesStorageSize()) { |
| 294 | NamedAttrList attrsList = attrs; |
| 295 | getName().populateInherentAttrs(op: this, attrs&: attrsList); |
| 296 | return attrsList.getDictionary(context: getContext()); |
| 297 | } |
| 298 | return attrs; |
| 299 | } |
| 300 | |
| 301 | void Operation::setAttrs(DictionaryAttr newAttrs) { |
| 302 | assert(newAttrs && "expected valid attribute dictionary" ); |
| 303 | if (getPropertiesStorageSize()) { |
| 304 | // We're spliting the providing DictionaryAttr by removing the inherentAttr |
| 305 | // which will be stored in the properties. |
| 306 | SmallVector<NamedAttribute> discardableAttrs; |
| 307 | discardableAttrs.reserve(N: newAttrs.size()); |
| 308 | for (NamedAttribute attr : newAttrs) { |
| 309 | if (getInherentAttr(name: attr.getName())) |
| 310 | setInherentAttr(name: attr.getName(), value: attr.getValue()); |
| 311 | else |
| 312 | discardableAttrs.push_back(Elt: attr); |
| 313 | } |
| 314 | if (discardableAttrs.size() != newAttrs.size()) |
| 315 | newAttrs = DictionaryAttr::get(context: getContext(), value: discardableAttrs); |
| 316 | } |
| 317 | attrs = newAttrs; |
| 318 | } |
| 319 | void Operation::setAttrs(ArrayRef<NamedAttribute> newAttrs) { |
| 320 | if (getPropertiesStorageSize()) { |
| 321 | // We're spliting the providing array of attributes by removing the |
| 322 | // inherentAttr which will be stored in the properties. |
| 323 | SmallVector<NamedAttribute> discardableAttrs; |
| 324 | discardableAttrs.reserve(N: newAttrs.size()); |
| 325 | for (NamedAttribute attr : newAttrs) { |
| 326 | if (getInherentAttr(name: attr.getName())) |
| 327 | setInherentAttr(name: attr.getName(), value: attr.getValue()); |
| 328 | else |
| 329 | discardableAttrs.push_back(Elt: attr); |
| 330 | } |
| 331 | attrs = DictionaryAttr::get(context: getContext(), value: discardableAttrs); |
| 332 | return; |
| 333 | } |
| 334 | attrs = DictionaryAttr::get(context: getContext(), value: newAttrs); |
| 335 | } |
| 336 | |
| 337 | std::optional<Attribute> Operation::getInherentAttr(StringRef name) { |
| 338 | return getName().getInherentAttr(op: this, name); |
| 339 | } |
| 340 | |
| 341 | void Operation::setInherentAttr(StringAttr name, Attribute value) { |
| 342 | getName().setInherentAttr(op: this, name, value); |
| 343 | } |
| 344 | |
| 345 | Attribute Operation::getPropertiesAsAttribute() { |
| 346 | std::optional<RegisteredOperationName> info = getRegisteredInfo(); |
| 347 | if (LLVM_UNLIKELY(!info)) |
| 348 | return *getPropertiesStorage().as<Attribute *>(); |
| 349 | return info->getOpPropertiesAsAttribute(op: this); |
| 350 | } |
| 351 | LogicalResult Operation::setPropertiesFromAttribute( |
| 352 | Attribute attr, function_ref<InFlightDiagnostic()> emitError) { |
| 353 | std::optional<RegisteredOperationName> info = getRegisteredInfo(); |
| 354 | if (LLVM_UNLIKELY(!info)) { |
| 355 | *getPropertiesStorage().as<Attribute *>() = attr; |
| 356 | return success(); |
| 357 | } |
| 358 | return info->setOpPropertiesFromAttribute( |
| 359 | opName: this->getName(), properties: this->getPropertiesStorage(), attr, emitError); |
| 360 | } |
| 361 | |
| 362 | void Operation::copyProperties(PropertyRef rhs) { |
| 363 | name.copyOpProperties(lhs: getPropertiesStorage(), rhs); |
| 364 | } |
| 365 | |
| 366 | llvm::hash_code Operation::hashProperties() { |
| 367 | return name.hashOpProperties(properties: getPropertiesStorage()); |
| 368 | } |
| 369 | |
| 370 | //===----------------------------------------------------------------------===// |
| 371 | // Operation Ordering |
| 372 | //===----------------------------------------------------------------------===// |
| 373 | |
| 374 | /// Given an operation 'other' that is within the same parent block, return |
| 375 | /// whether the current operation is before 'other' in the operation list |
| 376 | /// of the parent block. |
| 377 | /// Note: This function has an average complexity of O(1), but worst case may |
| 378 | /// take O(N) where N is the number of operations within the parent block. |
| 379 | bool Operation::isBeforeInBlock(Operation *other) { |
| 380 | assert(block && "Operations without parent blocks have no order." ); |
| 381 | assert(other && other->block == block && |
| 382 | "Expected other operation to have the same parent block." ); |
| 383 | // If the order of the block is already invalid, directly recompute the |
| 384 | // parent. |
| 385 | if (!block->isOpOrderValid()) { |
| 386 | block->recomputeOpOrder(); |
| 387 | } else { |
| 388 | // Update the order either operation if necessary. |
| 389 | updateOrderIfNecessary(); |
| 390 | other->updateOrderIfNecessary(); |
| 391 | } |
| 392 | |
| 393 | return orderIndex < other->orderIndex; |
| 394 | } |
| 395 | |
| 396 | /// Update the order index of this operation of this operation if necessary, |
| 397 | /// potentially recomputing the order of the parent block. |
| 398 | void Operation::updateOrderIfNecessary() { |
| 399 | assert(block && "expected valid parent" ); |
| 400 | |
| 401 | // If the order is valid for this operation there is nothing to do. |
| 402 | if (hasValidOrder() || llvm::hasSingleElement(C&: *block)) |
| 403 | return; |
| 404 | Operation *blockFront = &block->front(); |
| 405 | Operation *blockBack = &block->back(); |
| 406 | |
| 407 | // This method is expected to only be invoked on blocks with more than one |
| 408 | // operation. |
| 409 | assert(blockFront != blockBack && "expected more than one operation" ); |
| 410 | |
| 411 | // If the operation is at the end of the block. |
| 412 | if (this == blockBack) { |
| 413 | Operation *prevNode = getPrevNode(); |
| 414 | if (!prevNode->hasValidOrder()) |
| 415 | return block->recomputeOpOrder(); |
| 416 | |
| 417 | // Add the stride to the previous operation. |
| 418 | orderIndex = prevNode->orderIndex + kOrderStride; |
| 419 | return; |
| 420 | } |
| 421 | |
| 422 | // If this is the first operation try to use the next operation to compute the |
| 423 | // ordering. |
| 424 | if (this == blockFront) { |
| 425 | Operation *nextNode = getNextNode(); |
| 426 | if (!nextNode->hasValidOrder()) |
| 427 | return block->recomputeOpOrder(); |
| 428 | // There is no order to give this operation. |
| 429 | if (nextNode->orderIndex == 0) |
| 430 | return block->recomputeOpOrder(); |
| 431 | |
| 432 | // If we can't use the stride, just take the middle value left. This is safe |
| 433 | // because we know there is at least one valid index to assign to. |
| 434 | if (nextNode->orderIndex <= kOrderStride) |
| 435 | orderIndex = (nextNode->orderIndex / 2); |
| 436 | else |
| 437 | orderIndex = kOrderStride; |
| 438 | return; |
| 439 | } |
| 440 | |
| 441 | // Otherwise, this operation is between two others. Place this operation in |
| 442 | // the middle of the previous and next if possible. |
| 443 | Operation *prevNode = getPrevNode(), *nextNode = getNextNode(); |
| 444 | if (!prevNode->hasValidOrder() || !nextNode->hasValidOrder()) |
| 445 | return block->recomputeOpOrder(); |
| 446 | unsigned prevOrder = prevNode->orderIndex, nextOrder = nextNode->orderIndex; |
| 447 | |
| 448 | // Check to see if there is a valid order between the two. |
| 449 | if (prevOrder + 1 == nextOrder) |
| 450 | return block->recomputeOpOrder(); |
| 451 | orderIndex = prevOrder + ((nextOrder - prevOrder) / 2); |
| 452 | } |
| 453 | |
| 454 | //===----------------------------------------------------------------------===// |
| 455 | // ilist_traits for Operation |
| 456 | //===----------------------------------------------------------------------===// |
| 457 | |
| 458 | auto llvm::ilist_detail::SpecificNodeAccess< |
| 459 | llvm::ilist_detail::compute_node_options<::mlir::Operation>::type>:: |
| 460 | getNodePtr(pointer n) -> node_type * { |
| 461 | return NodeAccess::getNodePtr<OptionsT>(N: n); |
| 462 | } |
| 463 | |
| 464 | auto llvm::ilist_detail::SpecificNodeAccess< |
| 465 | llvm::ilist_detail::compute_node_options<::mlir::Operation>::type>:: |
| 466 | getNodePtr(const_pointer n) -> const node_type * { |
| 467 | return NodeAccess::getNodePtr<OptionsT>(N: n); |
| 468 | } |
| 469 | |
| 470 | auto llvm::ilist_detail::SpecificNodeAccess< |
| 471 | llvm::ilist_detail::compute_node_options<::mlir::Operation>::type>:: |
| 472 | getValuePtr(node_type *n) -> pointer { |
| 473 | return NodeAccess::getValuePtr<OptionsT>(N: n); |
| 474 | } |
| 475 | |
| 476 | auto llvm::ilist_detail::SpecificNodeAccess< |
| 477 | llvm::ilist_detail::compute_node_options<::mlir::Operation>::type>:: |
| 478 | getValuePtr(const node_type *n) -> const_pointer { |
| 479 | return NodeAccess::getValuePtr<OptionsT>(N: n); |
| 480 | } |
| 481 | |
| 482 | void llvm::ilist_traits<::mlir::Operation>::deleteNode(Operation *op) { |
| 483 | op->destroy(); |
| 484 | } |
| 485 | |
| 486 | Block *llvm::ilist_traits<::mlir::Operation>::getContainingBlock() { |
| 487 | size_t offset(size_t(&((Block *)nullptr->*Block::getSublistAccess(nullptr)))); |
| 488 | iplist<Operation> *anchor(static_cast<iplist<Operation> *>(this)); |
| 489 | return reinterpret_cast<Block *>(reinterpret_cast<char *>(anchor) - offset); |
| 490 | } |
| 491 | |
| 492 | /// This is a trait method invoked when an operation is added to a block. We |
| 493 | /// keep the block pointer up to date. |
| 494 | void llvm::ilist_traits<::mlir::Operation>::addNodeToList(Operation *op) { |
| 495 | assert(!op->getBlock() && "already in an operation block!" ); |
| 496 | op->block = getContainingBlock(); |
| 497 | |
| 498 | // Invalidate the order on the operation. |
| 499 | op->orderIndex = Operation::kInvalidOrderIdx; |
| 500 | } |
| 501 | |
| 502 | /// This is a trait method invoked when an operation is removed from a block. |
| 503 | /// We keep the block pointer up to date. |
| 504 | void llvm::ilist_traits<::mlir::Operation>::removeNodeFromList(Operation *op) { |
| 505 | assert(op->block && "not already in an operation block!" ); |
| 506 | op->block = nullptr; |
| 507 | } |
| 508 | |
| 509 | /// This is a trait method invoked when an operation is moved from one block |
| 510 | /// to another. We keep the block pointer up to date. |
| 511 | void llvm::ilist_traits<::mlir::Operation>::transferNodesFromList( |
| 512 | ilist_traits<Operation> &otherList, op_iterator first, op_iterator last) { |
| 513 | Block *curParent = getContainingBlock(); |
| 514 | |
| 515 | // Invalidate the ordering of the parent block. |
| 516 | curParent->invalidateOpOrder(); |
| 517 | |
| 518 | // If we are transferring operations within the same block, the block |
| 519 | // pointer doesn't need to be updated. |
| 520 | if (curParent == otherList.getContainingBlock()) |
| 521 | return; |
| 522 | |
| 523 | // Update the 'block' member of each operation. |
| 524 | for (; first != last; ++first) |
| 525 | first->block = curParent; |
| 526 | } |
| 527 | |
| 528 | /// Remove this operation (and its descendants) from its Block and delete |
| 529 | /// all of them. |
| 530 | void Operation::erase() { |
| 531 | if (auto *parent = getBlock()) |
| 532 | parent->getOperations().erase(IT: this); |
| 533 | else |
| 534 | destroy(); |
| 535 | } |
| 536 | |
| 537 | /// Remove the operation from its parent block, but don't delete it. |
| 538 | void Operation::remove() { |
| 539 | if (Block *parent = getBlock()) |
| 540 | parent->getOperations().remove(IT: this); |
| 541 | } |
| 542 | |
| 543 | /// Unlink this operation from its current block and insert it right before |
| 544 | /// `existingOp` which may be in the same or another block in the same |
| 545 | /// function. |
| 546 | void Operation::moveBefore(Operation *existingOp) { |
| 547 | moveBefore(block: existingOp->getBlock(), iterator: existingOp->getIterator()); |
| 548 | } |
| 549 | |
| 550 | /// Unlink this operation from its current basic block and insert it right |
| 551 | /// before `iterator` in the specified basic block. |
| 552 | void Operation::moveBefore(Block *block, |
| 553 | llvm::iplist<Operation>::iterator iterator) { |
| 554 | assert(getBlock() && |
| 555 | "cannot move an operation that isn't contained in a block" ); |
| 556 | block->getOperations().splice(where: iterator, L2&: getBlock()->getOperations(), |
| 557 | first: getIterator()); |
| 558 | } |
| 559 | |
| 560 | /// Unlink this operation from its current block and insert it right after |
| 561 | /// `existingOp` which may be in the same or another block in the same function. |
| 562 | void Operation::moveAfter(Operation *existingOp) { |
| 563 | moveAfter(block: existingOp->getBlock(), iterator: existingOp->getIterator()); |
| 564 | } |
| 565 | |
| 566 | /// Unlink this operation from its current block and insert it right after |
| 567 | /// `iterator` in the specified block. |
| 568 | void Operation::moveAfter(Block *block, |
| 569 | llvm::iplist<Operation>::iterator iterator) { |
| 570 | assert(iterator != block->end() && "cannot move after end of block" ); |
| 571 | moveBefore(block, iterator: std::next(x: iterator)); |
| 572 | } |
| 573 | |
| 574 | /// This drops all operand uses from this operation, which is an essential |
| 575 | /// step in breaking cyclic dependences between references when they are to |
| 576 | /// be deleted. |
| 577 | void Operation::dropAllReferences() { |
| 578 | for (auto &op : getOpOperands()) |
| 579 | op.drop(); |
| 580 | |
| 581 | for (auto ®ion : getRegions()) |
| 582 | region.dropAllReferences(); |
| 583 | |
| 584 | for (auto &dest : getBlockOperands()) |
| 585 | dest.drop(); |
| 586 | } |
| 587 | |
| 588 | /// This drops all uses of any values defined by this operation or its nested |
| 589 | /// regions, wherever they are located. |
| 590 | void Operation::dropAllDefinedValueUses() { |
| 591 | dropAllUses(); |
| 592 | |
| 593 | for (auto ®ion : getRegions()) |
| 594 | for (auto &block : region) |
| 595 | block.dropAllDefinedValueUses(); |
| 596 | } |
| 597 | |
| 598 | void Operation::setSuccessor(Block *block, unsigned index) { |
| 599 | assert(index < getNumSuccessors()); |
| 600 | getBlockOperands()[index].set(block); |
| 601 | } |
| 602 | |
| 603 | #ifndef NDEBUG |
| 604 | /// Assert that the folded results (in case of values) have the same type as |
| 605 | /// the results of the given op. |
| 606 | static void checkFoldResultTypes(Operation *op, |
| 607 | SmallVectorImpl<OpFoldResult> &results) { |
| 608 | if (results.empty()) |
| 609 | return; |
| 610 | |
| 611 | for (auto [ofr, opResult] : llvm::zip_equal(results, op->getResults())) { |
| 612 | if (auto value = dyn_cast<Value>(ofr)) { |
| 613 | if (value.getType() != opResult.getType()) { |
| 614 | op->emitOpError() << "folder produced a value of incorrect type: " |
| 615 | << value.getType() |
| 616 | << ", expected: " << opResult.getType(); |
| 617 | assert(false && "incorrect fold result type" ); |
| 618 | } |
| 619 | } |
| 620 | } |
| 621 | } |
| 622 | #endif // NDEBUG |
| 623 | |
| 624 | /// Attempt to fold this operation using the Op's registered foldHook. |
| 625 | LogicalResult Operation::fold(ArrayRef<Attribute> operands, |
| 626 | SmallVectorImpl<OpFoldResult> &results) { |
| 627 | // If we have a registered operation definition matching this one, use it to |
| 628 | // try to constant fold the operation. |
| 629 | if (succeeded(Result: name.foldHook(op: this, operands, results))) { |
| 630 | #ifndef NDEBUG |
| 631 | checkFoldResultTypes(this, results); |
| 632 | #endif // NDEBUG |
| 633 | return success(); |
| 634 | } |
| 635 | |
| 636 | // Otherwise, fall back on the dialect hook to handle it. |
| 637 | Dialect *dialect = getDialect(); |
| 638 | if (!dialect) |
| 639 | return failure(); |
| 640 | |
| 641 | auto *interface = dyn_cast<DialectFoldInterface>(Val: dialect); |
| 642 | if (!interface) |
| 643 | return failure(); |
| 644 | |
| 645 | LogicalResult status = interface->fold(op: this, operands, results); |
| 646 | #ifndef NDEBUG |
| 647 | if (succeeded(status)) |
| 648 | checkFoldResultTypes(this, results); |
| 649 | #endif // NDEBUG |
| 650 | return status; |
| 651 | } |
| 652 | |
| 653 | LogicalResult Operation::fold(SmallVectorImpl<OpFoldResult> &results) { |
| 654 | // Check if any operands are constants. |
| 655 | SmallVector<Attribute> constants; |
| 656 | constants.assign(NumElts: getNumOperands(), Elt: Attribute()); |
| 657 | for (unsigned i = 0, e = getNumOperands(); i != e; ++i) |
| 658 | matchPattern(value: getOperand(idx: i), pattern: m_Constant(bind_value: &constants[i])); |
| 659 | return fold(operands: constants, results); |
| 660 | } |
| 661 | |
| 662 | /// Emit an error with the op name prefixed, like "'dim' op " which is |
| 663 | /// convenient for verifiers. |
| 664 | InFlightDiagnostic Operation::emitOpError(const Twine &message) { |
| 665 | return emitError() << "'" << getName() << "' op " << message; |
| 666 | } |
| 667 | |
| 668 | //===----------------------------------------------------------------------===// |
| 669 | // Operation Cloning |
| 670 | //===----------------------------------------------------------------------===// |
| 671 | |
| 672 | Operation::CloneOptions::CloneOptions() |
| 673 | : cloneRegionsFlag(false), cloneOperandsFlag(false), |
| 674 | resultTypes(std::nullopt) {} |
| 675 | |
| 676 | Operation::CloneOptions::CloneOptions( |
| 677 | bool cloneRegions, bool cloneOperands, |
| 678 | std::optional<SmallVector<Type>> resultTypes) |
| 679 | : cloneRegionsFlag(cloneRegions), cloneOperandsFlag(cloneOperands), |
| 680 | resultTypes(std::move(resultTypes)) {} |
| 681 | |
| 682 | Operation::CloneOptions Operation::CloneOptions::all() { |
| 683 | return CloneOptions().cloneRegions().cloneOperands().withResultTypes( |
| 684 | resultTypes: std::nullopt); |
| 685 | } |
| 686 | |
| 687 | Operation::CloneOptions &Operation::CloneOptions::cloneRegions(bool enable) { |
| 688 | cloneRegionsFlag = enable; |
| 689 | return *this; |
| 690 | } |
| 691 | |
| 692 | Operation::CloneOptions &Operation::CloneOptions::cloneOperands(bool enable) { |
| 693 | cloneOperandsFlag = enable; |
| 694 | return *this; |
| 695 | } |
| 696 | |
| 697 | Operation::CloneOptions &Operation::CloneOptions::withResultTypes( |
| 698 | std::optional<SmallVector<Type>> resultTypes) { |
| 699 | this->resultTypes = std::move(resultTypes); |
| 700 | return *this; |
| 701 | } |
| 702 | |
| 703 | /// Create a deep copy of this operation but keep the operation regions empty. |
| 704 | /// Operands are remapped using `mapper` (if present), and `mapper` is updated |
| 705 | /// to contain the results. The `mapResults` flag specifies whether the results |
| 706 | /// of the cloned operation should be added to the map. |
| 707 | Operation *Operation::cloneWithoutRegions(IRMapping &mapper) { |
| 708 | return clone(mapper, options: CloneOptions::all().cloneRegions(enable: false)); |
| 709 | } |
| 710 | |
| 711 | Operation *Operation::cloneWithoutRegions() { |
| 712 | IRMapping mapper; |
| 713 | return cloneWithoutRegions(mapper); |
| 714 | } |
| 715 | |
| 716 | /// Create a deep copy of this operation, remapping any operands that use |
| 717 | /// values outside of the operation using the map that is provided (leaving |
| 718 | /// them alone if no entry is present). Replaces references to cloned |
| 719 | /// sub-operations to the corresponding operation that is copied, and adds |
| 720 | /// those mappings to the map. |
| 721 | Operation *Operation::clone(IRMapping &mapper, const CloneOptions &options) { |
| 722 | SmallVector<Value, 8> operands; |
| 723 | SmallVector<Block *, 2> successors; |
| 724 | |
| 725 | // Remap the operands. |
| 726 | if (options.shouldCloneOperands()) { |
| 727 | operands.reserve(N: getNumOperands()); |
| 728 | for (auto opValue : getOperands()) |
| 729 | operands.push_back(Elt: mapper.lookupOrDefault(from: opValue)); |
| 730 | } |
| 731 | |
| 732 | // Remap the successors. |
| 733 | successors.reserve(N: getNumSuccessors()); |
| 734 | for (Block *successor : getSuccessors()) |
| 735 | successors.push_back(Elt: mapper.lookupOrDefault(from: successor)); |
| 736 | |
| 737 | // Create the new operation. |
| 738 | auto *newOp = create(location: getLoc(), name: getName(), |
| 739 | resultTypes: options.resultTypesOr(defaultResultTypes: getResultTypes()), operands, attributes: attrs, |
| 740 | properties: getPropertiesStorage(), successors, numRegions: getNumRegions()); |
| 741 | mapper.map(from: this, to: newOp); |
| 742 | |
| 743 | // Clone the regions. |
| 744 | if (options.shouldCloneRegions()) { |
| 745 | for (unsigned i = 0; i != numRegions; ++i) |
| 746 | getRegion(index: i).cloneInto(dest: &newOp->getRegion(index: i), mapper); |
| 747 | } |
| 748 | |
| 749 | // Remember the mapping of any results. |
| 750 | if (options.shouldCloneResults()) |
| 751 | for (unsigned i = 0, e = getNumResults(); i != e; ++i) |
| 752 | mapper.map(from: getResult(idx: i), to: newOp->getResult(idx: i)); |
| 753 | |
| 754 | return newOp; |
| 755 | } |
| 756 | |
| 757 | Operation *Operation::clone(const CloneOptions &options) { |
| 758 | IRMapping mapper; |
| 759 | return clone(mapper, options); |
| 760 | } |
| 761 | |
| 762 | //===----------------------------------------------------------------------===// |
| 763 | // OpState trait class. |
| 764 | //===----------------------------------------------------------------------===// |
| 765 | |
| 766 | // The fallback for the parser is to try for a dialect operation parser. |
| 767 | // Otherwise, reject the custom assembly form. |
| 768 | ParseResult OpState::parse(OpAsmParser &parser, OperationState &result) { |
| 769 | if (auto parseFn = result.name.getDialect()->getParseOperationHook( |
| 770 | opName: result.name.getStringRef())) |
| 771 | return (*parseFn)(parser, result); |
| 772 | return parser.emitError(loc: parser.getNameLoc(), message: "has no custom assembly form" ); |
| 773 | } |
| 774 | |
| 775 | // The fallback for the printer is to try for a dialect operation printer. |
| 776 | // Otherwise, it prints the generic form. |
| 777 | void OpState::print(Operation *op, OpAsmPrinter &p, StringRef defaultDialect) { |
| 778 | if (auto printFn = op->getDialect()->getOperationPrinter(op)) { |
| 779 | printOpName(op, p, defaultDialect); |
| 780 | printFn(op, p); |
| 781 | } else { |
| 782 | p.printGenericOp(op); |
| 783 | } |
| 784 | } |
| 785 | |
| 786 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) |
| 787 | void OpState::dump() { state->dump(); } |
| 788 | #endif |
| 789 | |
| 790 | /// Print an operation name, eliding the dialect prefix if necessary and doesn't |
| 791 | /// lead to ambiguities. |
| 792 | void OpState::printOpName(Operation *op, OpAsmPrinter &p, |
| 793 | StringRef defaultDialect) { |
| 794 | StringRef name = op->getName().getStringRef(); |
| 795 | if (name.starts_with(Prefix: (defaultDialect + "." ).str()) && name.count(C: '.') == 1) |
| 796 | name = name.drop_front(N: defaultDialect.size() + 1); |
| 797 | p.getStream() << name; |
| 798 | } |
| 799 | |
| 800 | /// Parse properties as a Attribute. |
| 801 | ParseResult OpState::genericParseProperties(OpAsmParser &parser, |
| 802 | Attribute &result) { |
| 803 | if (succeeded(Result: parser.parseOptionalLess())) { // The less is optional. |
| 804 | if (parser.parseAttribute(result) || parser.parseGreater()) |
| 805 | return failure(); |
| 806 | } |
| 807 | return success(); |
| 808 | } |
| 809 | |
| 810 | /// Print the properties as a Attribute with names not included within |
| 811 | /// 'elidedProps'. A leading space is printed as part of the properties, so |
| 812 | /// that nothing is emitted at all when every property is elided. |
| 813 | void OpState::genericPrintProperties(OpAsmPrinter &p, Attribute properties, |
| 814 | ArrayRef<StringRef> elidedProps) { |
| 815 | if (!properties) |
| 816 | return; |
| 817 | auto dictAttr = dyn_cast_or_null<::mlir::DictionaryAttr>(Val&: properties); |
| 818 | if (dictAttr && !elidedProps.empty()) { |
| 819 | ArrayRef<NamedAttribute> attrs = dictAttr.getValue(); |
| 820 | llvm::SmallDenseSet<StringRef> (elidedProps.begin(), |
| 821 | elidedProps.end()); |
| 822 | auto filteredAttrs = |
| 823 | llvm::make_filter_range(Range&: attrs, Pred: [&](NamedAttribute attr) { |
| 824 | return !elidedAttrsSet.contains(V: attr.getName().strref()); |
| 825 | }); |
| 826 | if (!filteredAttrs.empty()) { |
| 827 | p << " <{" ; |
| 828 | interleaveComma(c: filteredAttrs, os&: p, each_fn: [&](NamedAttribute attr) { |
| 829 | p.printNamedAttribute(attr); |
| 830 | }); |
| 831 | p << "}>" ; |
| 832 | } |
| 833 | } else { |
| 834 | p << " <" << properties << ">" ; |
| 835 | } |
| 836 | } |
| 837 | |
| 838 | /// Emit an error about fatal conditions with this operation, reporting up to |
| 839 | /// any diagnostic handlers that may be listening. |
| 840 | InFlightDiagnostic OpState::emitError(const Twine &message) { |
| 841 | return getOperation()->emitError(message); |
| 842 | } |
| 843 | |
| 844 | /// Emit an error with the op name prefixed, like "'dim' op " which is |
| 845 | /// convenient for verifiers. |
| 846 | InFlightDiagnostic OpState::emitOpError(const Twine &message) { |
| 847 | return getOperation()->emitOpError(message); |
| 848 | } |
| 849 | |
| 850 | /// Emit a warning about this operation, reporting up to any diagnostic |
| 851 | /// handlers that may be listening. |
| 852 | InFlightDiagnostic OpState::emitWarning(const Twine &message) { |
| 853 | return getOperation()->emitWarning(message); |
| 854 | } |
| 855 | |
| 856 | /// Emit a remark about this operation, reporting up to any diagnostic |
| 857 | /// handlers that may be listening. |
| 858 | InFlightDiagnostic OpState::(const Twine &message) { |
| 859 | return getOperation()->emitRemark(message); |
| 860 | } |
| 861 | |
| 862 | //===----------------------------------------------------------------------===// |
| 863 | // Op Trait implementations |
| 864 | //===----------------------------------------------------------------------===// |
| 865 | |
| 866 | LogicalResult |
| 867 | OpTrait::impl::foldCommutative(Operation *op, ArrayRef<Attribute> operands, |
| 868 | SmallVectorImpl<OpFoldResult> &results) { |
| 869 | // Nothing to fold if there are not at least 2 operands. |
| 870 | if (op->getNumOperands() < 2) |
| 871 | return failure(); |
| 872 | // Move all constant operands to the end. |
| 873 | OpOperand *operandsBegin = op->getOpOperands().begin(); |
| 874 | auto isNonConstant = [&](OpOperand &o) { |
| 875 | return !static_cast<bool>(operands[std::distance(first: operandsBegin, last: &o)]); |
| 876 | }; |
| 877 | auto *firstConstantIt = llvm::find_if_not(Range: op->getOpOperands(), P: isNonConstant); |
| 878 | auto *newConstantIt = std::stable_partition( |
| 879 | first: firstConstantIt, last: op->getOpOperands().end(), pred: isNonConstant); |
| 880 | // Return success if the op was modified. |
| 881 | return success(IsSuccess: firstConstantIt != newConstantIt); |
| 882 | } |
| 883 | |
| 884 | OpFoldResult OpTrait::impl::foldIdempotent(Operation *op) { |
| 885 | if (op->getNumOperands() == 1) { |
| 886 | auto *argumentOp = op->getOperand(idx: 0).getDefiningOp(); |
| 887 | if (argumentOp && op->getName() == argumentOp->getName()) { |
| 888 | // Replace the outer operation output with the inner operation. |
| 889 | return op->getOperand(idx: 0); |
| 890 | } |
| 891 | } else if (op->getOperand(idx: 0) == op->getOperand(idx: 1)) { |
| 892 | return op->getOperand(idx: 0); |
| 893 | } |
| 894 | |
| 895 | return {}; |
| 896 | } |
| 897 | |
| 898 | OpFoldResult OpTrait::impl::foldInvolution(Operation *op) { |
| 899 | auto *argumentOp = op->getOperand(idx: 0).getDefiningOp(); |
| 900 | if (argumentOp && op->getName() == argumentOp->getName()) { |
| 901 | // Replace the outer involutions output with inner's input. |
| 902 | return argumentOp->getOperand(idx: 0); |
| 903 | } |
| 904 | |
| 905 | return {}; |
| 906 | } |
| 907 | |
| 908 | LogicalResult OpTrait::impl::verifyZeroOperands(Operation *op) { |
| 909 | if (op->getNumOperands() != 0) |
| 910 | return op->emitOpError() << "requires zero operands" ; |
| 911 | return success(); |
| 912 | } |
| 913 | |
| 914 | LogicalResult OpTrait::impl::verifyOneOperand(Operation *op) { |
| 915 | if (op->getNumOperands() != 1) |
| 916 | return op->emitOpError() << "requires a single operand" ; |
| 917 | return success(); |
| 918 | } |
| 919 | |
| 920 | LogicalResult OpTrait::impl::verifyNOperands(Operation *op, |
| 921 | unsigned numOperands) { |
| 922 | if (op->getNumOperands() != numOperands) { |
| 923 | return op->emitOpError() << "expected " << numOperands |
| 924 | << " operands, but found " << op->getNumOperands(); |
| 925 | } |
| 926 | return success(); |
| 927 | } |
| 928 | |
| 929 | LogicalResult OpTrait::impl::verifyAtLeastNOperands(Operation *op, |
| 930 | unsigned numOperands) { |
| 931 | if (op->getNumOperands() < numOperands) |
| 932 | return op->emitOpError() |
| 933 | << "expected " << numOperands << " or more operands, but found " |
| 934 | << op->getNumOperands(); |
| 935 | return success(); |
| 936 | } |
| 937 | |
| 938 | /// If this is a vector type, or a tensor type, return the scalar element type |
| 939 | /// that it is built around, otherwise return the type unmodified. |
| 940 | static Type getTensorOrVectorElementType(Type type) { |
| 941 | if (auto vec = llvm::dyn_cast<VectorType>(Val&: type)) |
| 942 | return vec.getElementType(); |
| 943 | |
| 944 | // Look through tensor<vector<...>> to find the underlying element type. |
| 945 | if (auto tensor = llvm::dyn_cast<TensorType>(Val&: type)) |
| 946 | return getTensorOrVectorElementType(type: tensor.getElementType()); |
| 947 | return type; |
| 948 | } |
| 949 | |
| 950 | LogicalResult OpTrait::impl::verifyIsIdempotent(Operation *op) { |
| 951 | // FIXME: Add back check for no side effects on operation. |
| 952 | // Currently adding it would cause the shared library build |
| 953 | // to fail since there would be a dependency of IR on SideEffectInterfaces |
| 954 | // which is cyclical. |
| 955 | return success(); |
| 956 | } |
| 957 | |
| 958 | LogicalResult OpTrait::impl::verifyIsInvolution(Operation *op) { |
| 959 | // FIXME: Add back check for no side effects on operation. |
| 960 | // Currently adding it would cause the shared library build |
| 961 | // to fail since there would be a dependency of IR on SideEffectInterfaces |
| 962 | // which is cyclical. |
| 963 | return success(); |
| 964 | } |
| 965 | |
| 966 | LogicalResult |
| 967 | OpTrait::impl::verifyOperandsAreSignlessIntegerLike(Operation *op) { |
| 968 | for (auto opType : op->getOperandTypes()) { |
| 969 | auto type = getTensorOrVectorElementType(type: opType); |
| 970 | if (!type.isSignlessIntOrIndex()) |
| 971 | return op->emitOpError() << "requires an integer or index type" ; |
| 972 | } |
| 973 | return success(); |
| 974 | } |
| 975 | |
| 976 | LogicalResult OpTrait::impl::verifyOperandsAreFloatLike(Operation *op) { |
| 977 | for (auto opType : op->getOperandTypes()) { |
| 978 | auto type = getTensorOrVectorElementType(type: opType); |
| 979 | if (!llvm::isa<FloatType>(Val: type)) |
| 980 | return op->emitOpError(message: "requires a float type" ); |
| 981 | } |
| 982 | return success(); |
| 983 | } |
| 984 | |
| 985 | LogicalResult OpTrait::impl::verifySameTypeOperands(Operation *op) { |
| 986 | // Zero or one operand always have the "same" type. |
| 987 | unsigned nOperands = op->getNumOperands(); |
| 988 | if (nOperands < 2) |
| 989 | return success(); |
| 990 | |
| 991 | auto type = op->getOperand(idx: 0).getType(); |
| 992 | for (auto opType : llvm::drop_begin(RangeOrContainer: op->getOperandTypes(), N: 1)) |
| 993 | if (opType != type) |
| 994 | return op->emitOpError() << "requires all operands to have the same type" ; |
| 995 | return success(); |
| 996 | } |
| 997 | |
| 998 | LogicalResult OpTrait::impl::verifyZeroRegions(Operation *op) { |
| 999 | if (op->getNumRegions() != 0) |
| 1000 | return op->emitOpError() << "requires zero regions" ; |
| 1001 | return success(); |
| 1002 | } |
| 1003 | |
| 1004 | LogicalResult OpTrait::impl::verifyOneRegion(Operation *op) { |
| 1005 | if (op->getNumRegions() != 1) |
| 1006 | return op->emitOpError() << "requires one region" ; |
| 1007 | return success(); |
| 1008 | } |
| 1009 | |
| 1010 | LogicalResult OpTrait::impl::verifyNRegions(Operation *op, |
| 1011 | unsigned numRegions) { |
| 1012 | if (op->getNumRegions() != numRegions) |
| 1013 | return op->emitOpError() << "expected " << numRegions << " regions" ; |
| 1014 | return success(); |
| 1015 | } |
| 1016 | |
| 1017 | LogicalResult OpTrait::impl::verifyAtLeastNRegions(Operation *op, |
| 1018 | unsigned numRegions) { |
| 1019 | if (op->getNumRegions() < numRegions) |
| 1020 | return op->emitOpError() << "expected " << numRegions << " or more regions" ; |
| 1021 | return success(); |
| 1022 | } |
| 1023 | |
| 1024 | LogicalResult OpTrait::impl::verifyZeroResults(Operation *op) { |
| 1025 | if (op->getNumResults() != 0) |
| 1026 | return op->emitOpError() << "requires zero results" ; |
| 1027 | return success(); |
| 1028 | } |
| 1029 | |
| 1030 | LogicalResult OpTrait::impl::verifyOneResult(Operation *op) { |
| 1031 | if (op->getNumResults() != 1) |
| 1032 | return op->emitOpError() << "requires one result" ; |
| 1033 | return success(); |
| 1034 | } |
| 1035 | |
| 1036 | LogicalResult OpTrait::impl::verifyNResults(Operation *op, |
| 1037 | unsigned numOperands) { |
| 1038 | if (op->getNumResults() != numOperands) |
| 1039 | return op->emitOpError() << "expected " << numOperands << " results" ; |
| 1040 | return success(); |
| 1041 | } |
| 1042 | |
| 1043 | LogicalResult OpTrait::impl::verifyAtLeastNResults(Operation *op, |
| 1044 | unsigned numOperands) { |
| 1045 | if (op->getNumResults() < numOperands) |
| 1046 | return op->emitOpError() |
| 1047 | << "expected " << numOperands << " or more results" ; |
| 1048 | return success(); |
| 1049 | } |
| 1050 | |
| 1051 | LogicalResult OpTrait::impl::verifySameOperandsShape(Operation *op) { |
| 1052 | if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1))) |
| 1053 | return failure(); |
| 1054 | |
| 1055 | if (failed(Result: verifyCompatibleShapes(types: op->getOperandTypes()))) |
| 1056 | return op->emitOpError() << "requires the same shape for all operands" ; |
| 1057 | |
| 1058 | return success(); |
| 1059 | } |
| 1060 | |
| 1061 | LogicalResult OpTrait::impl::verifySameOperandsAndResultShape(Operation *op) { |
| 1062 | if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)) || |
| 1063 | failed(Result: verifyAtLeastNResults(op, numOperands: 1))) |
| 1064 | return failure(); |
| 1065 | |
| 1066 | SmallVector<Type, 8> types(op->getOperandTypes()); |
| 1067 | types.append(RHS: llvm::to_vector<4>(Range: op->getResultTypes())); |
| 1068 | |
| 1069 | if (failed(Result: verifyCompatibleShapes(types))) |
| 1070 | return op->emitOpError() |
| 1071 | << "requires the same shape for all operands and results" ; |
| 1072 | |
| 1073 | return success(); |
| 1074 | } |
| 1075 | |
| 1076 | LogicalResult OpTrait::impl::verifySameOperandsElementType(Operation *op) { |
| 1077 | if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1))) |
| 1078 | return failure(); |
| 1079 | auto elementType = getElementTypeOrSelf(val: op->getOperand(idx: 0)); |
| 1080 | |
| 1081 | for (auto operand : llvm::drop_begin(RangeOrContainer: op->getOperands(), N: 1)) { |
| 1082 | if (getElementTypeOrSelf(val: operand) != elementType) |
| 1083 | return op->emitOpError(message: "requires the same element type for all operands" ); |
| 1084 | } |
| 1085 | |
| 1086 | return success(); |
| 1087 | } |
| 1088 | |
| 1089 | LogicalResult |
| 1090 | OpTrait::impl::verifySameOperandsAndResultElementType(Operation *op) { |
| 1091 | if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)) || |
| 1092 | failed(Result: verifyAtLeastNResults(op, numOperands: 1))) |
| 1093 | return failure(); |
| 1094 | |
| 1095 | auto elementType = getElementTypeOrSelf(val: op->getResult(idx: 0)); |
| 1096 | |
| 1097 | // Verify result element type matches first result's element type. |
| 1098 | for (auto result : llvm::drop_begin(RangeOrContainer: op->getResults(), N: 1)) { |
| 1099 | if (getElementTypeOrSelf(val: result) != elementType) |
| 1100 | return op->emitOpError( |
| 1101 | message: "requires the same element type for all operands and results" ); |
| 1102 | } |
| 1103 | |
| 1104 | // Verify operand's element type matches first result's element type. |
| 1105 | for (auto operand : op->getOperands()) { |
| 1106 | if (getElementTypeOrSelf(val: operand) != elementType) |
| 1107 | return op->emitOpError( |
| 1108 | message: "requires the same element type for all operands and results" ); |
| 1109 | } |
| 1110 | |
| 1111 | return success(); |
| 1112 | } |
| 1113 | |
| 1114 | LogicalResult OpTrait::impl::verifySameOperandsAndResultType(Operation *op) { |
| 1115 | if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1)) || |
| 1116 | failed(Result: verifyAtLeastNResults(op, numOperands: 1))) |
| 1117 | return failure(); |
| 1118 | |
| 1119 | auto type = op->getResult(idx: 0).getType(); |
| 1120 | auto elementType = getElementTypeOrSelf(type); |
| 1121 | Attribute encoding = nullptr; |
| 1122 | if (auto rankedType = dyn_cast<RankedTensorType>(Val&: type)) |
| 1123 | encoding = rankedType.getEncoding(); |
| 1124 | for (auto resultType : llvm::drop_begin(RangeOrContainer: op->getResultTypes())) { |
| 1125 | if (getElementTypeOrSelf(type: resultType) != elementType || |
| 1126 | failed(Result: verifyCompatibleShape(type1: resultType, type2: type))) |
| 1127 | return op->emitOpError() |
| 1128 | << "requires the same type for all operands and results" ; |
| 1129 | if (encoding) |
| 1130 | if (auto rankedType = dyn_cast<RankedTensorType>(Val&: resultType); |
| 1131 | encoding != rankedType.getEncoding()) |
| 1132 | return op->emitOpError() |
| 1133 | << "requires the same encoding for all operands and results" ; |
| 1134 | } |
| 1135 | for (auto opType : op->getOperandTypes()) { |
| 1136 | if (getElementTypeOrSelf(type: opType) != elementType || |
| 1137 | failed(Result: verifyCompatibleShape(type1: opType, type2: type))) |
| 1138 | return op->emitOpError() |
| 1139 | << "requires the same type for all operands and results" ; |
| 1140 | if (encoding) |
| 1141 | if (auto rankedType = dyn_cast<RankedTensorType>(Val&: opType); |
| 1142 | encoding != rankedType.getEncoding()) |
| 1143 | return op->emitOpError() |
| 1144 | << "requires the same encoding for all operands and results" ; |
| 1145 | } |
| 1146 | return success(); |
| 1147 | } |
| 1148 | |
| 1149 | LogicalResult OpTrait::impl::verifySameOperandsAndResultRank(Operation *op) { |
| 1150 | if (failed(Result: verifyAtLeastNOperands(op, numOperands: 1))) |
| 1151 | return failure(); |
| 1152 | |
| 1153 | // delegate function that returns true if type is a shaped type with known |
| 1154 | // rank |
| 1155 | auto hasRank = [](const Type type) { |
| 1156 | if (auto shapedType = dyn_cast<ShapedType>(Val: type)) |
| 1157 | return shapedType.hasRank(); |
| 1158 | |
| 1159 | return false; |
| 1160 | }; |
| 1161 | |
| 1162 | auto rankedOperandTypes = |
| 1163 | llvm::make_filter_range(Range: op->getOperandTypes(), Pred: hasRank); |
| 1164 | auto rankedResultTypes = |
| 1165 | llvm::make_filter_range(Range: op->getResultTypes(), Pred: hasRank); |
| 1166 | |
| 1167 | // If all operands and results are unranked, then no further verification. |
| 1168 | if (rankedOperandTypes.empty() && rankedResultTypes.empty()) |
| 1169 | return success(); |
| 1170 | |
| 1171 | // delegate function that returns rank of shaped type with known rank |
| 1172 | auto getRank = [](const Type type) { |
| 1173 | return cast<ShapedType>(Val: type).getRank(); |
| 1174 | }; |
| 1175 | |
| 1176 | auto rank = !rankedOperandTypes.empty() ? getRank(*rankedOperandTypes.begin()) |
| 1177 | : getRank(*rankedResultTypes.begin()); |
| 1178 | |
| 1179 | for (const auto type : rankedOperandTypes) { |
| 1180 | if (rank != getRank(type)) { |
| 1181 | return op->emitOpError(message: "operands don't have matching ranks" ); |
| 1182 | } |
| 1183 | } |
| 1184 | |
| 1185 | for (const auto type : rankedResultTypes) { |
| 1186 | if (rank != getRank(type)) { |
| 1187 | return op->emitOpError(message: "result type has different rank than operands" ); |
| 1188 | } |
| 1189 | } |
| 1190 | |
| 1191 | return success(); |
| 1192 | } |
| 1193 | |
| 1194 | LogicalResult OpTrait::impl::verifyIsTerminator(Operation *op) { |
| 1195 | Block *block = op->getBlock(); |
| 1196 | // Verify that the operation is at the end of the respective parent block. |
| 1197 | if (!block || &block->back() != op) |
| 1198 | return op->emitOpError(message: "must be the last operation in the parent block" ); |
| 1199 | return success(); |
| 1200 | } |
| 1201 | |
| 1202 | static LogicalResult verifyTerminatorSuccessors(Operation *op) { |
| 1203 | auto *parent = op->getParentRegion(); |
| 1204 | |
| 1205 | // Verify that the operands lines up with the BB arguments in the successor. |
| 1206 | for (Block *succ : op->getSuccessors()) |
| 1207 | if (succ->getParent() != parent) |
| 1208 | return op->emitError(message: "reference to block defined in another region" ); |
| 1209 | return success(); |
| 1210 | } |
| 1211 | |
| 1212 | LogicalResult OpTrait::impl::verifyZeroSuccessors(Operation *op) { |
| 1213 | if (op->getNumSuccessors() != 0) { |
| 1214 | return op->emitOpError(message: "requires 0 successors but found " ) |
| 1215 | << op->getNumSuccessors(); |
| 1216 | } |
| 1217 | return success(); |
| 1218 | } |
| 1219 | |
| 1220 | LogicalResult OpTrait::impl::verifyOneSuccessor(Operation *op) { |
| 1221 | if (op->getNumSuccessors() != 1) { |
| 1222 | return op->emitOpError(message: "requires 1 successor but found " ) |
| 1223 | << op->getNumSuccessors(); |
| 1224 | } |
| 1225 | return verifyTerminatorSuccessors(op); |
| 1226 | } |
| 1227 | LogicalResult OpTrait::impl::verifyNSuccessors(Operation *op, |
| 1228 | unsigned numSuccessors) { |
| 1229 | if (op->getNumSuccessors() != numSuccessors) { |
| 1230 | return op->emitOpError(message: "requires " ) |
| 1231 | << numSuccessors << " successors but found " |
| 1232 | << op->getNumSuccessors(); |
| 1233 | } |
| 1234 | return verifyTerminatorSuccessors(op); |
| 1235 | } |
| 1236 | LogicalResult OpTrait::impl::verifyAtLeastNSuccessors(Operation *op, |
| 1237 | unsigned numSuccessors) { |
| 1238 | if (op->getNumSuccessors() < numSuccessors) { |
| 1239 | return op->emitOpError(message: "requires at least " ) |
| 1240 | << numSuccessors << " successors but found " |
| 1241 | << op->getNumSuccessors(); |
| 1242 | } |
| 1243 | return verifyTerminatorSuccessors(op); |
| 1244 | } |
| 1245 | |
| 1246 | LogicalResult OpTrait::impl::verifyResultsAreBoolLike(Operation *op) { |
| 1247 | for (auto resultType : op->getResultTypes()) { |
| 1248 | auto elementType = getTensorOrVectorElementType(type: resultType); |
| 1249 | bool isBoolType = elementType.isInteger(width: 1); |
| 1250 | if (!isBoolType) |
| 1251 | return op->emitOpError() << "requires a bool result type" ; |
| 1252 | } |
| 1253 | |
| 1254 | return success(); |
| 1255 | } |
| 1256 | |
| 1257 | LogicalResult OpTrait::impl::verifyResultsAreFloatLike(Operation *op) { |
| 1258 | for (auto resultType : op->getResultTypes()) |
| 1259 | if (!llvm::isa<FloatType>(Val: getTensorOrVectorElementType(type: resultType))) |
| 1260 | return op->emitOpError() << "requires a floating point type" ; |
| 1261 | |
| 1262 | return success(); |
| 1263 | } |
| 1264 | |
| 1265 | LogicalResult |
| 1266 | OpTrait::impl::verifyResultsAreSignlessIntegerLike(Operation *op) { |
| 1267 | for (auto resultType : op->getResultTypes()) |
| 1268 | if (!getTensorOrVectorElementType(type: resultType).isSignlessIntOrIndex()) |
| 1269 | return op->emitOpError() << "requires an integer or index type" ; |
| 1270 | return success(); |
| 1271 | } |
| 1272 | |
| 1273 | LogicalResult OpTrait::impl::verifyValueSizeAttr(Operation *op, |
| 1274 | StringRef attrName, |
| 1275 | StringRef valueGroupName, |
| 1276 | size_t expectedCount) { |
| 1277 | auto sizeAttr = dyn_cast_or_null<DenseI32ArrayAttr>( |
| 1278 | Val: op->getInherentAttr(name: attrName).value_or(u: Attribute{})); |
| 1279 | if (!sizeAttr) |
| 1280 | return op->emitOpError(message: "requires dense i32 array attribute '" ) |
| 1281 | << attrName << "'" ; |
| 1282 | |
| 1283 | ArrayRef<int32_t> sizes = sizeAttr.asArrayRef(); |
| 1284 | if (llvm::any_of(Range&: sizes, P: [](int32_t element) { return element < 0; })) |
| 1285 | return op->emitOpError(message: "'" ) |
| 1286 | << attrName << "' attribute cannot have negative elements" ; |
| 1287 | |
| 1288 | size_t totalCount = llvm::sum_of(Range&: sizes, Init: size_t(0)); |
| 1289 | if (totalCount != expectedCount) |
| 1290 | return op->emitOpError() |
| 1291 | << valueGroupName << " count (" << expectedCount |
| 1292 | << ") does not match with the total size (" << totalCount |
| 1293 | << ") specified in attribute '" << attrName << "'" ; |
| 1294 | return success(); |
| 1295 | } |
| 1296 | |
| 1297 | LogicalResult OpTrait::impl::verifyOperandSizeAttr(Operation *op, |
| 1298 | StringRef attrName) { |
| 1299 | return verifyValueSizeAttr(op, attrName, valueGroupName: "operand" , expectedCount: op->getNumOperands()); |
| 1300 | } |
| 1301 | |
| 1302 | LogicalResult OpTrait::impl::verifyResultSizeAttr(Operation *op, |
| 1303 | StringRef attrName) { |
| 1304 | return verifyValueSizeAttr(op, attrName, valueGroupName: "result" , expectedCount: op->getNumResults()); |
| 1305 | } |
| 1306 | |
| 1307 | LogicalResult OpTrait::impl::verifyNoRegionArguments(Operation *op) { |
| 1308 | for (Region ®ion : op->getRegions()) { |
| 1309 | if (region.empty()) |
| 1310 | continue; |
| 1311 | |
| 1312 | if (region.getNumArguments() != 0) { |
| 1313 | if (op->getNumRegions() > 1) |
| 1314 | return op->emitOpError(message: "region #" ) |
| 1315 | << region.getRegionNumber() << " should have no arguments" ; |
| 1316 | return op->emitOpError(message: "region should have no arguments" ); |
| 1317 | } |
| 1318 | } |
| 1319 | return success(); |
| 1320 | } |
| 1321 | |
| 1322 | LogicalResult OpTrait::impl::verifyElementwise(Operation *op) { |
| 1323 | auto isMappableType = llvm::IsaPred<VectorType, TensorType>; |
| 1324 | auto resultMappableTypes = |
| 1325 | llvm::filter_to_vector<1>(C: op->getResultTypes(), Pred&: isMappableType); |
| 1326 | auto operandMappableTypes = |
| 1327 | llvm::filter_to_vector<2>(C: op->getOperandTypes(), Pred&: isMappableType); |
| 1328 | |
| 1329 | // If the op only has scalar operand/result types, then we have nothing to |
| 1330 | // check. |
| 1331 | if (resultMappableTypes.empty() && operandMappableTypes.empty()) |
| 1332 | return success(); |
| 1333 | |
| 1334 | if (!resultMappableTypes.empty() && operandMappableTypes.empty()) |
| 1335 | return op->emitOpError(message: "if a result is non-scalar, then at least one " |
| 1336 | "operand must be non-scalar" ); |
| 1337 | |
| 1338 | assert(!operandMappableTypes.empty()); |
| 1339 | |
| 1340 | if (resultMappableTypes.empty()) |
| 1341 | return op->emitOpError(message: "if an operand is non-scalar, then there must be at " |
| 1342 | "least one non-scalar result" ); |
| 1343 | |
| 1344 | if (resultMappableTypes.size() != op->getNumResults()) |
| 1345 | return op->emitOpError( |
| 1346 | message: "if an operand is non-scalar, then all results must be non-scalar" ); |
| 1347 | |
| 1348 | SmallVector<Type, 4> types = llvm::to_vector<2>( |
| 1349 | Range: llvm::concat<Type>(Ranges&: operandMappableTypes, Ranges&: resultMappableTypes)); |
| 1350 | TypeID expectedBaseTy = types.front().getTypeID(); |
| 1351 | if (!llvm::all_of(Range&: types, |
| 1352 | P: [&](Type t) { return t.getTypeID() == expectedBaseTy; }) || |
| 1353 | failed(Result: verifyCompatibleShapes(types))) { |
| 1354 | return op->emitOpError() << "all non-scalar operands/results must have the " |
| 1355 | "same shape and base type" ; |
| 1356 | } |
| 1357 | |
| 1358 | return success(); |
| 1359 | } |
| 1360 | |
| 1361 | /// Check for any values used by operations regions attached to the |
| 1362 | /// specified "IsIsolatedFromAbove" operation defined outside of it. |
| 1363 | LogicalResult OpTrait::impl::verifyIsIsolatedFromAbove(Operation *isolatedOp) { |
| 1364 | assert(isolatedOp->hasTrait<OpTrait::IsIsolatedFromAbove>() && |
| 1365 | "Intended to check IsolatedFromAbove ops" ); |
| 1366 | |
| 1367 | // List of regions to analyze. Each region is processed independently, with |
| 1368 | // respect to the common `limit` region, so we can look at them in any order. |
| 1369 | // Therefore, use a simple vector and push/pop back the current region. |
| 1370 | SmallVector<Region *, 8> pendingRegions; |
| 1371 | for (auto ®ion : isolatedOp->getRegions()) { |
| 1372 | pendingRegions.push_back(Elt: ®ion); |
| 1373 | |
| 1374 | // Traverse all operations in the region. |
| 1375 | while (!pendingRegions.empty()) { |
| 1376 | for (Operation &op : pendingRegions.pop_back_val()->getOps()) { |
| 1377 | for (Value operand : op.getOperands()) { |
| 1378 | // Check that any value that is used by an operation is defined in the |
| 1379 | // same region as either an operation result. |
| 1380 | auto *operandRegion = operand.getParentRegion(); |
| 1381 | if (!operandRegion) |
| 1382 | return op.emitError(message: "operation's operand is unlinked" ); |
| 1383 | if (!region.isAncestor(other: operandRegion)) { |
| 1384 | return op.emitOpError(message: "using value defined outside the region" ) |
| 1385 | .attachNote(noteLoc: isolatedOp->getLoc()) |
| 1386 | << "required by region isolation constraints" ; |
| 1387 | } |
| 1388 | } |
| 1389 | |
| 1390 | // Schedule any regions in the operation for further checking. Don't |
| 1391 | // recurse into other IsolatedFromAbove ops, because they will check |
| 1392 | // themselves. |
| 1393 | if (op.getNumRegions() && |
| 1394 | !op.hasTrait<OpTrait::IsIsolatedFromAbove>()) { |
| 1395 | for (Region &subRegion : op.getRegions()) |
| 1396 | pendingRegions.push_back(Elt: &subRegion); |
| 1397 | } |
| 1398 | } |
| 1399 | } |
| 1400 | } |
| 1401 | |
| 1402 | return success(); |
| 1403 | } |
| 1404 | |
| 1405 | bool OpTrait::hasElementwiseMappableTraits(Operation *op) { |
| 1406 | return op->hasTrait<Elementwise>() && op->hasTrait<Scalarizable>() && |
| 1407 | op->hasTrait<Vectorizable>() && op->hasTrait<Tensorizable>(); |
| 1408 | } |
| 1409 | |
| 1410 | //===----------------------------------------------------------------------===// |
| 1411 | // Misc. utils |
| 1412 | //===----------------------------------------------------------------------===// |
| 1413 | |
| 1414 | /// Insert an operation, generated by `buildTerminatorOp`, at the end of the |
| 1415 | /// region's only block if it does not have a terminator already. If the region |
| 1416 | /// is empty, insert a new block first. `buildTerminatorOp` should return the |
| 1417 | /// terminator operation to insert. |
| 1418 | void impl::ensureRegionTerminator( |
| 1419 | Region ®ion, OpBuilder &builder, Location loc, |
| 1420 | function_ref<Operation *(OpBuilder &, Location)> buildTerminatorOp) { |
| 1421 | OpBuilder::InsertionGuard guard(builder); |
| 1422 | if (region.empty()) |
| 1423 | builder.createBlock(parent: ®ion); |
| 1424 | |
| 1425 | Block &block = region.back(); |
| 1426 | if (!block.empty() && block.back().hasTrait<OpTrait::IsTerminator>()) |
| 1427 | return; |
| 1428 | |
| 1429 | builder.setInsertionPointToEnd(&block); |
| 1430 | builder.insert(op: buildTerminatorOp(builder, loc)); |
| 1431 | } |
| 1432 | |
| 1433 | /// Create a simple OpBuilder and forward to the OpBuilder version of this |
| 1434 | /// function. |
| 1435 | void impl::ensureRegionTerminator( |
| 1436 | Region ®ion, Builder &builder, Location loc, |
| 1437 | function_ref<Operation *(OpBuilder &, Location)> buildTerminatorOp) { |
| 1438 | OpBuilder opBuilder(builder.getContext()); |
| 1439 | ensureRegionTerminator(region, builder&: opBuilder, loc, buildTerminatorOp); |
| 1440 | } |
| 1441 | |