| 1 | //===- TransformInterfaces.cpp - Transform Dialect Interfaces -------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include <utility> |
| 10 | |
| 11 | #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" |
| 12 | |
| 13 | #include "mlir/IR/Diagnostics.h" |
| 14 | #include "mlir/IR/Operation.h" |
| 15 | #include "mlir/IR/PatternMatch.h" |
| 16 | #include "mlir/Interfaces/CastInterfaces.h" |
| 17 | #include "llvm/ADT/STLExtras.h" |
| 18 | #include "llvm/ADT/ScopeExit.h" |
| 19 | #include "llvm/ADT/iterator.h" |
| 20 | #include "llvm/Support/Debug.h" |
| 21 | #include "llvm/Support/DebugLog.h" |
| 22 | #include "llvm/Support/ErrorHandling.h" |
| 23 | #include "llvm/Support/InterleavedRange.h" |
| 24 | |
| 25 | #define DEBUG_TYPE "transform-dialect" |
| 26 | #define DEBUG_PRINT_AFTER_ALL "transform-dialect-print-top-level-after-all" |
| 27 | #define FULL_LDBG() LDBG(4) |
| 28 | |
| 29 | using namespace mlir; |
| 30 | |
| 31 | //===----------------------------------------------------------------------===// |
| 32 | // Helper functions |
| 33 | //===----------------------------------------------------------------------===// |
| 34 | |
| 35 | /// Return true if `a` happens before `b`, i.e., `a` or one of its ancestors |
| 36 | /// properly dominates `b` and `b` is not inside `a`. |
| 37 | static bool happensBefore(Operation *a, Operation *b) { |
| 38 | do { |
| 39 | if (a->isProperAncestor(other: b)) |
| 40 | return false; |
| 41 | if (Operation *bAncestor = a->getBlock()->findAncestorOpInBlock(op&: *b)) { |
| 42 | return a->isBeforeInBlock(other: bAncestor); |
| 43 | } |
| 44 | } while ((a = a->getParentOp())); |
| 45 | return false; |
| 46 | } |
| 47 | |
| 48 | //===----------------------------------------------------------------------===// |
| 49 | // TransformState |
| 50 | //===----------------------------------------------------------------------===// |
| 51 | |
| 52 | transform::TransformState::TransformState( |
| 53 | Region *region, Operation *payloadRoot, |
| 54 | const RaggedArray<MappedValue> &, |
| 55 | const TransformOptions &options) |
| 56 | : topLevel(payloadRoot), options(options) { |
| 57 | topLevelMappedValues.reserve(size: extraMappings.size()); |
| 58 | for (ArrayRef<MappedValue> mapping : extraMappings) |
| 59 | topLevelMappedValues.push_back(elements&: mapping); |
| 60 | if (region) { |
| 61 | RegionScope *scope = new RegionScope(*this, *region); |
| 62 | topLevelRegionScope.reset(p: scope); |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | Operation *transform::TransformState::getTopLevel() const { return topLevel; } |
| 67 | |
| 68 | ArrayRef<Operation *> |
| 69 | transform::TransformState::getPayloadOpsView(Value value) const { |
| 70 | const TransformOpMapping &operationMapping = getMapping(value).direct; |
| 71 | auto iter = operationMapping.find(Val: value); |
| 72 | assert(iter != operationMapping.end() && |
| 73 | "cannot find mapping for payload handle (param/value handle " |
| 74 | "provided?)" ); |
| 75 | return iter->getSecond(); |
| 76 | } |
| 77 | |
| 78 | ArrayRef<Attribute> transform::TransformState::getParams(Value value) const { |
| 79 | const ParamMapping &mapping = getMapping(value).params; |
| 80 | auto iter = mapping.find(Val: value); |
| 81 | assert(iter != mapping.end() && "cannot find mapping for param handle " |
| 82 | "(operation/value handle provided?)" ); |
| 83 | return iter->getSecond(); |
| 84 | } |
| 85 | |
| 86 | ArrayRef<Value> |
| 87 | transform::TransformState::getPayloadValuesView(Value handleValue) const { |
| 88 | const ValueMapping &mapping = getMapping(value: handleValue).values; |
| 89 | auto iter = mapping.find(Val: handleValue); |
| 90 | assert(iter != mapping.end() && "cannot find mapping for value handle " |
| 91 | "(param/operation handle provided?)" ); |
| 92 | return iter->getSecond(); |
| 93 | } |
| 94 | |
| 95 | LogicalResult transform::TransformState::getHandlesForPayloadOp( |
| 96 | Operation *op, SmallVectorImpl<Value> &handles, |
| 97 | bool includeOutOfScope) const { |
| 98 | bool found = false; |
| 99 | for (const auto &[region, mapping] : llvm::reverse(C: mappings)) { |
| 100 | auto iterator = mapping->reverse.find(Val: op); |
| 101 | if (iterator != mapping->reverse.end()) { |
| 102 | llvm::append_range(C&: handles, R&: iterator->getSecond()); |
| 103 | found = true; |
| 104 | } |
| 105 | // Stop looking when reaching a region that is isolated from above. |
| 106 | if (!includeOutOfScope && |
| 107 | region->getParentOp()->hasTrait<OpTrait::IsIsolatedFromAbove>()) |
| 108 | break; |
| 109 | } |
| 110 | |
| 111 | return success(IsSuccess: found); |
| 112 | } |
| 113 | |
| 114 | LogicalResult transform::TransformState::getHandlesForPayloadValue( |
| 115 | Value payloadValue, SmallVectorImpl<Value> &handles, |
| 116 | bool includeOutOfScope) const { |
| 117 | bool found = false; |
| 118 | for (const auto &[region, mapping] : llvm::reverse(C: mappings)) { |
| 119 | auto iterator = mapping->reverseValues.find(Val: payloadValue); |
| 120 | if (iterator != mapping->reverseValues.end()) { |
| 121 | llvm::append_range(C&: handles, R&: iterator->getSecond()); |
| 122 | found = true; |
| 123 | } |
| 124 | // Stop looking when reaching a region that is isolated from above. |
| 125 | if (!includeOutOfScope && |
| 126 | region->getParentOp()->hasTrait<OpTrait::IsIsolatedFromAbove>()) |
| 127 | break; |
| 128 | } |
| 129 | |
| 130 | return success(IsSuccess: found); |
| 131 | } |
| 132 | |
| 133 | /// Given a list of MappedValues, cast them to the value kind implied by the |
| 134 | /// interface of the handle type, and dispatch to one of the callbacks. |
| 135 | static DiagnosedSilenceableFailure dispatchMappedValues( |
| 136 | Value handle, ArrayRef<transform::MappedValue> values, |
| 137 | function_ref<LogicalResult(ArrayRef<Operation *>)> operationsFn, |
| 138 | function_ref<LogicalResult(ArrayRef<transform::Param>)> paramsFn, |
| 139 | function_ref<LogicalResult(ValueRange)> valuesFn) { |
| 140 | if (llvm::isa<transform::TransformHandleTypeInterface>(Val: handle.getType())) { |
| 141 | SmallVector<Operation *> operations; |
| 142 | operations.reserve(N: values.size()); |
| 143 | for (transform::MappedValue value : values) { |
| 144 | if (auto *op = llvm::dyn_cast_if_present<Operation *>(Val&: value)) { |
| 145 | operations.push_back(Elt: op); |
| 146 | continue; |
| 147 | } |
| 148 | return emitSilenceableFailure(loc: handle.getLoc()) |
| 149 | << "wrong kind of value provided for top-level operation handle" ; |
| 150 | } |
| 151 | if (failed(Result: operationsFn(operations))) |
| 152 | return DiagnosedSilenceableFailure::definiteFailure(); |
| 153 | return DiagnosedSilenceableFailure::success(); |
| 154 | } |
| 155 | |
| 156 | if (llvm::isa<transform::TransformValueHandleTypeInterface>( |
| 157 | Val: handle.getType())) { |
| 158 | SmallVector<Value> payloadValues; |
| 159 | payloadValues.reserve(N: values.size()); |
| 160 | for (transform::MappedValue value : values) { |
| 161 | if (auto v = llvm::dyn_cast_if_present<Value>(Val&: value)) { |
| 162 | payloadValues.push_back(Elt: v); |
| 163 | continue; |
| 164 | } |
| 165 | return emitSilenceableFailure(loc: handle.getLoc()) |
| 166 | << "wrong kind of value provided for the top-level value handle" ; |
| 167 | } |
| 168 | if (failed(Result: valuesFn(payloadValues))) |
| 169 | return DiagnosedSilenceableFailure::definiteFailure(); |
| 170 | return DiagnosedSilenceableFailure::success(); |
| 171 | } |
| 172 | |
| 173 | assert(llvm::isa<transform::TransformParamTypeInterface>(handle.getType()) && |
| 174 | "unsupported kind of block argument" ); |
| 175 | SmallVector<transform::Param> parameters; |
| 176 | parameters.reserve(N: values.size()); |
| 177 | for (transform::MappedValue value : values) { |
| 178 | if (auto attr = llvm::dyn_cast_if_present<Attribute>(Val&: value)) { |
| 179 | parameters.push_back(Elt: attr); |
| 180 | continue; |
| 181 | } |
| 182 | return emitSilenceableFailure(loc: handle.getLoc()) |
| 183 | << "wrong kind of value provided for top-level parameter" ; |
| 184 | } |
| 185 | if (failed(Result: paramsFn(parameters))) |
| 186 | return DiagnosedSilenceableFailure::definiteFailure(); |
| 187 | return DiagnosedSilenceableFailure::success(); |
| 188 | } |
| 189 | |
| 190 | LogicalResult |
| 191 | transform::TransformState::mapBlockArgument(BlockArgument argument, |
| 192 | ArrayRef<MappedValue> values) { |
| 193 | return dispatchMappedValues( |
| 194 | handle: argument, values, |
| 195 | operationsFn: [&](ArrayRef<Operation *> operations) { |
| 196 | return setPayloadOps(value: argument, targets: operations); |
| 197 | }, |
| 198 | paramsFn: [&](ArrayRef<Param> params) { |
| 199 | return setParams(value: argument, params); |
| 200 | }, |
| 201 | valuesFn: [&](ValueRange payloadValues) { |
| 202 | return setPayloadValues(handle: argument, payloadValues); |
| 203 | }) |
| 204 | .checkAndReport(); |
| 205 | } |
| 206 | |
| 207 | LogicalResult transform::TransformState::mapBlockArguments( |
| 208 | Block::BlockArgListType arguments, |
| 209 | ArrayRef<SmallVector<MappedValue>> mapping) { |
| 210 | for (auto &&[argument, values] : llvm::zip_equal(t&: arguments, u&: mapping)) |
| 211 | if (failed(Result: mapBlockArgument(argument, values))) |
| 212 | return failure(); |
| 213 | return success(); |
| 214 | } |
| 215 | |
| 216 | LogicalResult |
| 217 | transform::TransformState::setPayloadOps(Value value, |
| 218 | ArrayRef<Operation *> targets) { |
| 219 | assert(value != kTopLevelValue && |
| 220 | "attempting to reset the transformation root" ); |
| 221 | assert(llvm::isa<TransformHandleTypeInterface>(value.getType()) && |
| 222 | "wrong handle type" ); |
| 223 | |
| 224 | for (Operation *target : targets) { |
| 225 | if (target) |
| 226 | continue; |
| 227 | return emitError(loc: value.getLoc()) |
| 228 | << "attempting to assign a null payload op to this transform value" ; |
| 229 | } |
| 230 | |
| 231 | auto iface = llvm::cast<TransformHandleTypeInterface>(Val: value.getType()); |
| 232 | DiagnosedSilenceableFailure result = |
| 233 | iface.checkPayload(loc: value.getLoc(), payload: targets); |
| 234 | if (failed(Result: result.checkAndReport())) |
| 235 | return failure(); |
| 236 | |
| 237 | // Setting new payload for the value without cleaning it first is a misuse of |
| 238 | // the API, assert here. |
| 239 | SmallVector<Operation *> storedTargets(targets); |
| 240 | Mappings &mappings = getMapping(value); |
| 241 | bool inserted = |
| 242 | mappings.direct.insert(KV: {value, std::move(storedTargets)}).second; |
| 243 | assert(inserted && "value is already associated with another list" ); |
| 244 | (void)inserted; |
| 245 | |
| 246 | for (Operation *op : targets) |
| 247 | mappings.reverse[op].push_back(Elt: value); |
| 248 | |
| 249 | return success(); |
| 250 | } |
| 251 | |
| 252 | LogicalResult |
| 253 | transform::TransformState::setPayloadValues(Value handle, |
| 254 | ValueRange payloadValues) { |
| 255 | assert(handle != nullptr && "attempting to set params for a null value" ); |
| 256 | assert(llvm::isa<TransformValueHandleTypeInterface>(handle.getType()) && |
| 257 | "wrong handle type" ); |
| 258 | |
| 259 | for (Value payload : payloadValues) { |
| 260 | if (payload) |
| 261 | continue; |
| 262 | return emitError(loc: handle.getLoc()) << "attempting to assign a null payload " |
| 263 | "value to this transform handle" ; |
| 264 | } |
| 265 | |
| 266 | auto iface = llvm::cast<TransformValueHandleTypeInterface>(Val: handle.getType()); |
| 267 | SmallVector<Value> payloadValueVector = llvm::to_vector(Range&: payloadValues); |
| 268 | DiagnosedSilenceableFailure result = |
| 269 | iface.checkPayload(loc: handle.getLoc(), payload: payloadValueVector); |
| 270 | if (failed(Result: result.checkAndReport())) |
| 271 | return failure(); |
| 272 | |
| 273 | Mappings &mappings = getMapping(value: handle); |
| 274 | bool inserted = |
| 275 | mappings.values.insert(KV: {handle, std::move(payloadValueVector)}).second; |
| 276 | assert( |
| 277 | inserted && |
| 278 | "value handle is already associated with another list of payload values" ); |
| 279 | (void)inserted; |
| 280 | |
| 281 | for (Value payload : payloadValues) |
| 282 | mappings.reverseValues[payload].push_back(Elt: handle); |
| 283 | |
| 284 | return success(); |
| 285 | } |
| 286 | |
| 287 | LogicalResult transform::TransformState::setParams(Value value, |
| 288 | ArrayRef<Param> params) { |
| 289 | assert(value != nullptr && "attempting to set params for a null value" ); |
| 290 | |
| 291 | for (Attribute attr : params) { |
| 292 | if (attr) |
| 293 | continue; |
| 294 | return emitError(loc: value.getLoc()) |
| 295 | << "attempting to assign a null parameter to this transform value" ; |
| 296 | } |
| 297 | |
| 298 | auto valueType = llvm::dyn_cast<TransformParamTypeInterface>(Val: value.getType()); |
| 299 | assert(value && |
| 300 | "cannot associate parameter with a value of non-parameter type" ); |
| 301 | DiagnosedSilenceableFailure result = |
| 302 | valueType.checkPayload(loc: value.getLoc(), payload: params); |
| 303 | if (failed(Result: result.checkAndReport())) |
| 304 | return failure(); |
| 305 | |
| 306 | Mappings &mappings = getMapping(value); |
| 307 | bool inserted = |
| 308 | mappings.params.insert(KV: {value, llvm::to_vector(Range&: params)}).second; |
| 309 | assert(inserted && "value is already associated with another list of params" ); |
| 310 | (void)inserted; |
| 311 | return success(); |
| 312 | } |
| 313 | |
| 314 | template <typename Mapping, typename Key, typename Mapped> |
| 315 | static void dropMappingEntry(Mapping &mapping, Key key, Mapped mapped) { |
| 316 | auto it = mapping.find(key); |
| 317 | if (it == mapping.end()) |
| 318 | return; |
| 319 | |
| 320 | llvm::erase(it->getSecond(), mapped); |
| 321 | if (it->getSecond().empty()) |
| 322 | mapping.erase(it); |
| 323 | } |
| 324 | |
| 325 | void transform::TransformState::forgetMapping(Value opHandle, |
| 326 | ValueRange origOpFlatResults, |
| 327 | bool allowOutOfScope) { |
| 328 | Mappings &mappings = getMapping(value: opHandle, allowOutOfScope); |
| 329 | for (Operation *op : mappings.direct[opHandle]) |
| 330 | dropMappingEntry(mapping&: mappings.reverse, key: op, mapped: opHandle); |
| 331 | mappings.direct.erase(Val: opHandle); |
| 332 | #if LLVM_ENABLE_ABI_BREAKING_CHECKS |
| 333 | // Payload IR is removed from the mapping. This invalidates the respective |
| 334 | // iterators. |
| 335 | mappings.incrementTimestamp(opHandle); |
| 336 | #endif // LLVM_ENABLE_ABI_BREAKING_CHECKS |
| 337 | |
| 338 | for (Value opResult : origOpFlatResults) { |
| 339 | SmallVector<Value> resultHandles; |
| 340 | (void)getHandlesForPayloadValue(payloadValue: opResult, handles&: resultHandles); |
| 341 | for (Value resultHandle : resultHandles) { |
| 342 | Mappings &localMappings = getMapping(value: resultHandle); |
| 343 | dropMappingEntry(mapping&: localMappings.values, key: resultHandle, mapped: opResult); |
| 344 | #if LLVM_ENABLE_ABI_BREAKING_CHECKS |
| 345 | // Payload IR is removed from the mapping. This invalidates the respective |
| 346 | // iterators. |
| 347 | mappings.incrementTimestamp(resultHandle); |
| 348 | #endif // LLVM_ENABLE_ABI_BREAKING_CHECKS |
| 349 | dropMappingEntry(mapping&: localMappings.reverseValues, key: opResult, mapped: resultHandle); |
| 350 | } |
| 351 | } |
| 352 | } |
| 353 | |
| 354 | void transform::TransformState::forgetValueMapping( |
| 355 | Value valueHandle, ArrayRef<Operation *> payloadOperations) { |
| 356 | Mappings &mappings = getMapping(value: valueHandle); |
| 357 | for (Value payloadValue : mappings.reverseValues[valueHandle]) |
| 358 | dropMappingEntry(mapping&: mappings.reverseValues, key: payloadValue, mapped: valueHandle); |
| 359 | mappings.values.erase(Val: valueHandle); |
| 360 | #if LLVM_ENABLE_ABI_BREAKING_CHECKS |
| 361 | // Payload IR is removed from the mapping. This invalidates the respective |
| 362 | // iterators. |
| 363 | mappings.incrementTimestamp(valueHandle); |
| 364 | #endif // LLVM_ENABLE_ABI_BREAKING_CHECKS |
| 365 | |
| 366 | for (Operation *payloadOp : payloadOperations) { |
| 367 | SmallVector<Value> opHandles; |
| 368 | (void)getHandlesForPayloadOp(op: payloadOp, handles&: opHandles); |
| 369 | for (Value opHandle : opHandles) { |
| 370 | Mappings &localMappings = getMapping(value: opHandle); |
| 371 | dropMappingEntry(mapping&: localMappings.direct, key: opHandle, mapped: payloadOp); |
| 372 | dropMappingEntry(mapping&: localMappings.reverse, key: payloadOp, mapped: opHandle); |
| 373 | |
| 374 | #if LLVM_ENABLE_ABI_BREAKING_CHECKS |
| 375 | // Payload IR is removed from the mapping. This invalidates the respective |
| 376 | // iterators. |
| 377 | localMappings.incrementTimestamp(opHandle); |
| 378 | #endif // LLVM_ENABLE_ABI_BREAKING_CHECKS |
| 379 | } |
| 380 | } |
| 381 | } |
| 382 | |
| 383 | LogicalResult |
| 384 | transform::TransformState::replacePayloadOp(Operation *op, |
| 385 | Operation *replacement) { |
| 386 | // TODO: consider invalidating the handles to nested objects here. |
| 387 | |
| 388 | #ifndef NDEBUG |
| 389 | for (Value opResult : op->getResults()) { |
| 390 | SmallVector<Value> valueHandles; |
| 391 | (void)getHandlesForPayloadValue(opResult, valueHandles, |
| 392 | /*includeOutOfScope=*/true); |
| 393 | assert(valueHandles.empty() && "expected no mapping to old results" ); |
| 394 | } |
| 395 | #endif // NDEBUG |
| 396 | |
| 397 | // Drop the mapping between the op and all handles that point to it. Fail if |
| 398 | // there are no handles. |
| 399 | SmallVector<Value> opHandles; |
| 400 | if (failed(Result: getHandlesForPayloadOp(op, handles&: opHandles, /*includeOutOfScope=*/true))) |
| 401 | return failure(); |
| 402 | for (Value handle : opHandles) { |
| 403 | Mappings &mappings = getMapping(value: handle, /*allowOutOfScope=*/true); |
| 404 | dropMappingEntry(mapping&: mappings.reverse, key: op, mapped: handle); |
| 405 | } |
| 406 | |
| 407 | // Replace the pointed-to object of all handles with the replacement object. |
| 408 | // In case a payload op was erased (replacement object is nullptr), a nullptr |
| 409 | // is stored in the mapping. These nullptrs are removed after each transform. |
| 410 | // Furthermore, nullptrs are not enumerated by payload op iterators. The |
| 411 | // relative order of ops is preserved. |
| 412 | // |
| 413 | // Removing an op from the mapping would be problematic because removing an |
| 414 | // element from an array invalidates iterators; merely changing the value of |
| 415 | // elements does not. |
| 416 | for (Value handle : opHandles) { |
| 417 | Mappings &mappings = getMapping(value: handle, /*allowOutOfScope=*/true); |
| 418 | auto it = mappings.direct.find(Val: handle); |
| 419 | if (it == mappings.direct.end()) |
| 420 | continue; |
| 421 | |
| 422 | SmallVector<Operation *, 2> &association = it->getSecond(); |
| 423 | // Note that an operation may be associated with the handle more than once. |
| 424 | for (Operation *&mapped : association) { |
| 425 | if (mapped == op) |
| 426 | mapped = replacement; |
| 427 | } |
| 428 | |
| 429 | if (replacement) { |
| 430 | mappings.reverse[replacement].push_back(Elt: handle); |
| 431 | } else { |
| 432 | opHandlesToCompact.insert(V: handle); |
| 433 | } |
| 434 | } |
| 435 | |
| 436 | return success(); |
| 437 | } |
| 438 | |
| 439 | LogicalResult |
| 440 | transform::TransformState::replacePayloadValue(Value value, Value replacement) { |
| 441 | SmallVector<Value> valueHandles; |
| 442 | if (failed(Result: getHandlesForPayloadValue(payloadValue: value, handles&: valueHandles, |
| 443 | /*includeOutOfScope=*/true))) |
| 444 | return failure(); |
| 445 | |
| 446 | for (Value handle : valueHandles) { |
| 447 | Mappings &mappings = getMapping(value: handle, /*allowOutOfScope=*/true); |
| 448 | dropMappingEntry(mapping&: mappings.reverseValues, key: value, mapped: handle); |
| 449 | |
| 450 | // If replacing with null, that is erasing the mapping, drop the mapping |
| 451 | // between the handles and the IR objects |
| 452 | if (!replacement) { |
| 453 | dropMappingEntry(mapping&: mappings.values, key: handle, mapped: value); |
| 454 | #if LLVM_ENABLE_ABI_BREAKING_CHECKS |
| 455 | // Payload IR is removed from the mapping. This invalidates the respective |
| 456 | // iterators. |
| 457 | mappings.incrementTimestamp(handle); |
| 458 | #endif // LLVM_ENABLE_ABI_BREAKING_CHECKS |
| 459 | } else { |
| 460 | auto it = mappings.values.find(Val: handle); |
| 461 | if (it == mappings.values.end()) |
| 462 | continue; |
| 463 | |
| 464 | SmallVector<Value> &association = it->getSecond(); |
| 465 | for (Value &mapped : association) { |
| 466 | if (mapped == value) |
| 467 | mapped = replacement; |
| 468 | } |
| 469 | mappings.reverseValues[replacement].push_back(Elt: handle); |
| 470 | } |
| 471 | } |
| 472 | |
| 473 | return success(); |
| 474 | } |
| 475 | |
| 476 | void transform::TransformState::recordOpHandleInvalidationOne( |
| 477 | OpOperand &consumingHandle, ArrayRef<Operation *> potentialAncestors, |
| 478 | Operation *payloadOp, Value otherHandle, Value throughValue, |
| 479 | transform::TransformState::InvalidatedHandleMap &newlyInvalidated) const { |
| 480 | // If the op is associated with invalidated handle, skip the check as it |
| 481 | // may be reading invalid IR. This also ensures we report the first |
| 482 | // invalidation and not the last one. |
| 483 | if (invalidatedHandles.count(Val: otherHandle) || |
| 484 | newlyInvalidated.count(Val: otherHandle)) |
| 485 | return; |
| 486 | |
| 487 | FULL_LDBG() << "--recordOpHandleInvalidationOne" ; |
| 488 | FULL_LDBG() << "--ancestors: " |
| 489 | << llvm::interleaved( |
| 490 | R: llvm::make_pointee_range(Range&: potentialAncestors)); |
| 491 | |
| 492 | Operation *owner = consumingHandle.getOwner(); |
| 493 | unsigned operandNo = consumingHandle.getOperandNumber(); |
| 494 | for (Operation *ancestor : potentialAncestors) { |
| 495 | // clang-format off |
| 496 | FULL_LDBG() << "----handle one ancestor: " << *ancestor;; |
| 497 | |
| 498 | FULL_LDBG() << "----of payload with name: " |
| 499 | << payloadOp->getName().getIdentifier(); |
| 500 | FULL_LDBG() << "----of payload: " << *payloadOp; |
| 501 | // clang-format on |
| 502 | if (!ancestor->isAncestor(other: payloadOp)) |
| 503 | continue; |
| 504 | |
| 505 | // Make sure the error-reporting lambda doesn't capture anything |
| 506 | // by-reference because it will go out of scope. Additionally, extract |
| 507 | // location from Payload IR ops because the ops themselves may be |
| 508 | // deleted before the lambda gets called. |
| 509 | Location ancestorLoc = ancestor->getLoc(); |
| 510 | Location opLoc = payloadOp->getLoc(); |
| 511 | std::optional<Location> throughValueLoc = |
| 512 | throughValue ? std::make_optional(t: throughValue.getLoc()) : std::nullopt; |
| 513 | newlyInvalidated[otherHandle] = [ancestorLoc, opLoc, owner, operandNo, |
| 514 | otherHandle, |
| 515 | throughValueLoc](Location currentLoc) { |
| 516 | InFlightDiagnostic diag = emitError(loc: currentLoc) |
| 517 | << "op uses a handle invalidated by a " |
| 518 | "previously executed transform op" ; |
| 519 | diag.attachNote(noteLoc: otherHandle.getLoc()) << "handle to invalidated ops" ; |
| 520 | diag.attachNote(noteLoc: owner->getLoc()) |
| 521 | << "invalidated by this transform op that consumes its operand #" |
| 522 | << operandNo |
| 523 | << " and invalidates all handles to payload IR entities associated " |
| 524 | "with this operand and entities nested in them" ; |
| 525 | diag.attachNote(noteLoc: ancestorLoc) << "ancestor payload op" ; |
| 526 | diag.attachNote(noteLoc: opLoc) << "nested payload op" ; |
| 527 | if (throughValueLoc) { |
| 528 | diag.attachNote(noteLoc: *throughValueLoc) |
| 529 | << "consumed handle points to this payload value" ; |
| 530 | } |
| 531 | }; |
| 532 | } |
| 533 | } |
| 534 | |
| 535 | void transform::TransformState::recordValueHandleInvalidationByOpHandleOne( |
| 536 | OpOperand &opHandle, ArrayRef<Operation *> potentialAncestors, |
| 537 | Value payloadValue, Value valueHandle, |
| 538 | transform::TransformState::InvalidatedHandleMap &newlyInvalidated) const { |
| 539 | // If the op is associated with invalidated handle, skip the check as it |
| 540 | // may be reading invalid IR. This also ensures we report the first |
| 541 | // invalidation and not the last one. |
| 542 | if (invalidatedHandles.count(Val: valueHandle) || |
| 543 | newlyInvalidated.count(Val: valueHandle)) |
| 544 | return; |
| 545 | |
| 546 | for (Operation *ancestor : potentialAncestors) { |
| 547 | Operation *definingOp; |
| 548 | std::optional<unsigned> resultNo; |
| 549 | unsigned argumentNo = std::numeric_limits<unsigned>::max(); |
| 550 | unsigned blockNo = std::numeric_limits<unsigned>::max(); |
| 551 | unsigned regionNo = std::numeric_limits<unsigned>::max(); |
| 552 | if (auto opResult = llvm::dyn_cast<OpResult>(Val&: payloadValue)) { |
| 553 | definingOp = opResult.getOwner(); |
| 554 | resultNo = opResult.getResultNumber(); |
| 555 | } else { |
| 556 | auto arg = llvm::cast<BlockArgument>(Val&: payloadValue); |
| 557 | definingOp = arg.getParentBlock()->getParentOp(); |
| 558 | argumentNo = arg.getArgNumber(); |
| 559 | blockNo = arg.getOwner()->computeBlockNumber(); |
| 560 | regionNo = arg.getOwner()->getParent()->getRegionNumber(); |
| 561 | } |
| 562 | assert(definingOp && "expected the value to be defined by an op as result " |
| 563 | "or block argument" ); |
| 564 | if (!ancestor->isAncestor(other: definingOp)) |
| 565 | continue; |
| 566 | |
| 567 | Operation *owner = opHandle.getOwner(); |
| 568 | unsigned operandNo = opHandle.getOperandNumber(); |
| 569 | Location ancestorLoc = ancestor->getLoc(); |
| 570 | Location opLoc = definingOp->getLoc(); |
| 571 | Location valueLoc = payloadValue.getLoc(); |
| 572 | newlyInvalidated[valueHandle] = [valueHandle, owner, operandNo, resultNo, |
| 573 | argumentNo, blockNo, regionNo, ancestorLoc, |
| 574 | opLoc, valueLoc](Location currentLoc) { |
| 575 | InFlightDiagnostic diag = emitError(loc: currentLoc) |
| 576 | << "op uses a handle invalidated by a " |
| 577 | "previously executed transform op" ; |
| 578 | diag.attachNote(noteLoc: valueHandle.getLoc()) << "invalidated handle" ; |
| 579 | diag.attachNote(noteLoc: owner->getLoc()) |
| 580 | << "invalidated by this transform op that consumes its operand #" |
| 581 | << operandNo |
| 582 | << " and invalidates all handles to payload IR entities " |
| 583 | "associated with this operand and entities nested in them" ; |
| 584 | diag.attachNote(noteLoc: ancestorLoc) |
| 585 | << "ancestor op associated with the consumed handle" ; |
| 586 | if (resultNo) { |
| 587 | diag.attachNote(noteLoc: opLoc) |
| 588 | << "op defining the value as result #" << *resultNo; |
| 589 | } else { |
| 590 | diag.attachNote(noteLoc: opLoc) |
| 591 | << "op defining the value as block argument #" << argumentNo |
| 592 | << " of block #" << blockNo << " in region #" << regionNo; |
| 593 | } |
| 594 | diag.attachNote(noteLoc: valueLoc) << "payload value" ; |
| 595 | }; |
| 596 | } |
| 597 | } |
| 598 | |
| 599 | void transform::TransformState::recordOpHandleInvalidation( |
| 600 | OpOperand &handle, ArrayRef<Operation *> potentialAncestors, |
| 601 | Value throughValue, |
| 602 | transform::TransformState::InvalidatedHandleMap &newlyInvalidated) const { |
| 603 | |
| 604 | if (potentialAncestors.empty()) { |
| 605 | FULL_LDBG() << "----recording invalidation for empty handle: " |
| 606 | << handle.get(); |
| 607 | |
| 608 | Operation *owner = handle.getOwner(); |
| 609 | unsigned operandNo = handle.getOperandNumber(); |
| 610 | newlyInvalidated[handle.get()] = [owner, operandNo](Location currentLoc) { |
| 611 | InFlightDiagnostic diag = emitError(loc: currentLoc) |
| 612 | << "op uses a handle associated with empty " |
| 613 | "payload and invalidated by a " |
| 614 | "previously executed transform op" ; |
| 615 | diag.attachNote(noteLoc: owner->getLoc()) |
| 616 | << "invalidated by this transform op that consumes its operand #" |
| 617 | << operandNo; |
| 618 | }; |
| 619 | return; |
| 620 | } |
| 621 | |
| 622 | // Iterate over the mapping and invalidate aliasing handles. This is quite |
| 623 | // expensive and only necessary for error reporting in case of transform |
| 624 | // dialect misuse with dangling handles. Iteration over the handles is based |
| 625 | // on the assumption that the number of handles is significantly less than the |
| 626 | // number of IR objects (operations and values). Alternatively, we could walk |
| 627 | // the IR nested in each payload op associated with the given handle and look |
| 628 | // for handles associated with each operation and value. |
| 629 | for (const auto &[region, mapping] : llvm::reverse(C: mappings)) { |
| 630 | // Go over all op handle mappings and mark as invalidated any handle |
| 631 | // pointing to any of the payload ops associated with the given handle or |
| 632 | // any op nested in them. |
| 633 | for (const auto &[payloadOp, otherHandles] : mapping->reverse) { |
| 634 | for (Value otherHandle : otherHandles) |
| 635 | recordOpHandleInvalidationOne(consumingHandle&: handle, potentialAncestors, payloadOp, |
| 636 | otherHandle, throughValue, |
| 637 | newlyInvalidated); |
| 638 | } |
| 639 | // Go over all value handle mappings and mark as invalidated any handle |
| 640 | // pointing to any result of the payload op associated with the given handle |
| 641 | // or any op nested in them. Similarly invalidate handles to argument of |
| 642 | // blocks belonging to any region of any payload op associated with the |
| 643 | // given handle or any op nested in them. |
| 644 | for (const auto &[payloadValue, valueHandles] : mapping->reverseValues) { |
| 645 | for (Value valueHandle : valueHandles) |
| 646 | recordValueHandleInvalidationByOpHandleOne(opHandle&: handle, potentialAncestors, |
| 647 | payloadValue, valueHandle, |
| 648 | newlyInvalidated); |
| 649 | } |
| 650 | |
| 651 | // Stop lookup when reaching a region that is isolated from above. |
| 652 | if (region->getParentOp()->hasTrait<OpTrait::IsIsolatedFromAbove>()) |
| 653 | break; |
| 654 | } |
| 655 | } |
| 656 | |
| 657 | void transform::TransformState::recordValueHandleInvalidation( |
| 658 | OpOperand &valueHandle, |
| 659 | transform::TransformState::InvalidatedHandleMap &newlyInvalidated) const { |
| 660 | // Invalidate other handles to the same value. |
| 661 | for (Value payloadValue : getPayloadValuesView(handleValue: valueHandle.get())) { |
| 662 | SmallVector<Value> otherValueHandles; |
| 663 | (void)getHandlesForPayloadValue(payloadValue, handles&: otherValueHandles); |
| 664 | for (Value otherHandle : otherValueHandles) { |
| 665 | Operation *owner = valueHandle.getOwner(); |
| 666 | unsigned operandNo = valueHandle.getOperandNumber(); |
| 667 | Location valueLoc = payloadValue.getLoc(); |
| 668 | newlyInvalidated[otherHandle] = [otherHandle, owner, operandNo, |
| 669 | valueLoc](Location currentLoc) { |
| 670 | InFlightDiagnostic diag = emitError(loc: currentLoc) |
| 671 | << "op uses a handle invalidated by a " |
| 672 | "previously executed transform op" ; |
| 673 | diag.attachNote(noteLoc: otherHandle.getLoc()) << "invalidated handle" ; |
| 674 | diag.attachNote(noteLoc: owner->getLoc()) |
| 675 | << "invalidated by this transform op that consumes its operand #" |
| 676 | << operandNo |
| 677 | << " and invalidates handles to the same values as associated with " |
| 678 | "it" ; |
| 679 | diag.attachNote(noteLoc: valueLoc) << "payload value" ; |
| 680 | }; |
| 681 | } |
| 682 | |
| 683 | if (auto opResult = llvm::dyn_cast<OpResult>(Val&: payloadValue)) { |
| 684 | Operation *payloadOp = opResult.getOwner(); |
| 685 | recordOpHandleInvalidation(handle&: valueHandle, potentialAncestors: payloadOp, throughValue: payloadValue, |
| 686 | newlyInvalidated); |
| 687 | } else { |
| 688 | auto arg = llvm::dyn_cast<BlockArgument>(Val&: payloadValue); |
| 689 | for (Operation &payloadOp : *arg.getOwner()) |
| 690 | recordOpHandleInvalidation(handle&: valueHandle, potentialAncestors: &payloadOp, throughValue: payloadValue, |
| 691 | newlyInvalidated); |
| 692 | } |
| 693 | } |
| 694 | } |
| 695 | |
| 696 | /// Checks that the operation does not use invalidated handles as operands. |
| 697 | /// Reports errors and returns failure if it does. Otherwise, invalidates the |
| 698 | /// handles consumed by the operation as well as any handles pointing to payload |
| 699 | /// IR operations nested in the operations associated with the consumed handles. |
| 700 | LogicalResult transform::TransformState::checkAndRecordHandleInvalidationImpl( |
| 701 | transform::TransformOpInterface transform, |
| 702 | transform::TransformState::InvalidatedHandleMap &newlyInvalidated) const { |
| 703 | FULL_LDBG() << "--Start checkAndRecordHandleInvalidation" ; |
| 704 | auto memoryEffectsIface = |
| 705 | cast<MemoryEffectOpInterface>(Val: transform.getOperation()); |
| 706 | SmallVector<MemoryEffects::EffectInstance> effects; |
| 707 | memoryEffectsIface.getEffectsOnResource( |
| 708 | resource: transform::TransformMappingResource::get(), effects); |
| 709 | |
| 710 | for (OpOperand &target : transform->getOpOperands()) { |
| 711 | FULL_LDBG() << "----iterate on handle: " << target.get(); |
| 712 | // If the operand uses an invalidated handle, report it. If the operation |
| 713 | // allows handles to point to repeated payload operations, only report |
| 714 | // pre-existing invalidation errors. Otherwise, also report invalidations |
| 715 | // caused by the current transform operation affecting its other operands. |
| 716 | auto it = invalidatedHandles.find(Val: target.get()); |
| 717 | auto nit = newlyInvalidated.find(Val: target.get()); |
| 718 | if (it != invalidatedHandles.end()) { |
| 719 | FULL_LDBG() << "--End checkAndRecordHandleInvalidation, found already " |
| 720 | "invalidated -> FAILURE" ; |
| 721 | return it->getSecond()(transform->getLoc()), failure(); |
| 722 | } |
| 723 | if (!transform.allowsRepeatedHandleOperands() && |
| 724 | nit != newlyInvalidated.end()) { |
| 725 | FULL_LDBG() << "--End checkAndRecordHandleInvalidation, found newly " |
| 726 | "invalidated (by this op) -> FAILURE" ; |
| 727 | return nit->getSecond()(transform->getLoc()), failure(); |
| 728 | } |
| 729 | |
| 730 | // Invalidate handles pointing to the operations nested in the operation |
| 731 | // associated with the handle consumed by this operation. |
| 732 | auto consumesTarget = [&](const MemoryEffects::EffectInstance &effect) { |
| 733 | return isa<MemoryEffects::Free>(Val: effect.getEffect()) && |
| 734 | effect.getValue() == target.get(); |
| 735 | }; |
| 736 | if (llvm::any_of(Range&: effects, P: consumesTarget)) { |
| 737 | FULL_LDBG() << "----found consume effect" ; |
| 738 | if (llvm::isa<transform::TransformHandleTypeInterface>( |
| 739 | Val: target.get().getType())) { |
| 740 | FULL_LDBG() << "----recordOpHandleInvalidation" ; |
| 741 | SmallVector<Operation *> payloadOps = |
| 742 | llvm::to_vector(Range: getPayloadOps(value: target.get())); |
| 743 | recordOpHandleInvalidation(handle&: target, potentialAncestors: payloadOps, throughValue: nullptr, |
| 744 | newlyInvalidated); |
| 745 | } else if (llvm::isa<transform::TransformValueHandleTypeInterface>( |
| 746 | Val: target.get().getType())) { |
| 747 | FULL_LDBG() << "----recordValueHandleInvalidation" ; |
| 748 | recordValueHandleInvalidation(valueHandle&: target, newlyInvalidated); |
| 749 | } else { |
| 750 | FULL_LDBG() |
| 751 | << "----not a TransformHandle -> SKIP AND DROP ON THE FLOOR" ; |
| 752 | } |
| 753 | } else { |
| 754 | FULL_LDBG() << "----no consume effect -> SKIP" ; |
| 755 | } |
| 756 | } |
| 757 | |
| 758 | FULL_LDBG() << "--End checkAndRecordHandleInvalidation -> SUCCESS" ; |
| 759 | return success(); |
| 760 | } |
| 761 | |
| 762 | LogicalResult transform::TransformState::checkAndRecordHandleInvalidation( |
| 763 | transform::TransformOpInterface transform) { |
| 764 | InvalidatedHandleMap newlyInvalidated; |
| 765 | LogicalResult checkResult = |
| 766 | checkAndRecordHandleInvalidationImpl(transform, newlyInvalidated); |
| 767 | invalidatedHandles.insert(I: std::make_move_iterator(i: newlyInvalidated.begin()), |
| 768 | E: std::make_move_iterator(i: newlyInvalidated.end())); |
| 769 | return checkResult; |
| 770 | } |
| 771 | |
| 772 | template <typename T> |
| 773 | static DiagnosedSilenceableFailure |
| 774 | checkRepeatedConsumptionInOperand(ArrayRef<T> payload, |
| 775 | transform::TransformOpInterface transform, |
| 776 | unsigned operandNumber) { |
| 777 | DenseSet<T> seen; |
| 778 | for (T p : payload) { |
| 779 | if (!seen.insert(p).second) { |
| 780 | DiagnosedSilenceableFailure diag = |
| 781 | transform.emitSilenceableError() |
| 782 | << "a handle passed as operand #" << operandNumber |
| 783 | << " and consumed by this operation points to a payload " |
| 784 | "entity more than once" ; |
| 785 | if constexpr (std::is_pointer_v<T>) |
| 786 | diag.attachNote(loc: p->getLoc()) << "repeated target op" ; |
| 787 | else |
| 788 | diag.attachNote(loc: p.getLoc()) << "repeated target value" ; |
| 789 | return diag; |
| 790 | } |
| 791 | } |
| 792 | return DiagnosedSilenceableFailure::success(); |
| 793 | } |
| 794 | |
| 795 | void transform::TransformState::compactOpHandles() { |
| 796 | for (Value handle : opHandlesToCompact) { |
| 797 | Mappings &mappings = getMapping(value: handle, /*allowOutOfScope=*/true); |
| 798 | #if LLVM_ENABLE_ABI_BREAKING_CHECKS |
| 799 | if (llvm::is_contained(mappings.direct[handle], nullptr)) |
| 800 | // Payload IR is removed from the mapping. This invalidates the respective |
| 801 | // iterators. |
| 802 | mappings.incrementTimestamp(handle); |
| 803 | #endif // LLVM_ENABLE_ABI_BREAKING_CHECKS |
| 804 | llvm::erase(C&: mappings.direct[handle], V: nullptr); |
| 805 | } |
| 806 | opHandlesToCompact.clear(); |
| 807 | } |
| 808 | |
| 809 | DiagnosedSilenceableFailure |
| 810 | transform::TransformState::applyTransform(TransformOpInterface transform) { |
| 811 | LDBG() << "applying: " |
| 812 | << OpWithFlags(transform, OpPrintingFlags().skipRegions()); |
| 813 | FULL_LDBG() << "Top-level payload before application:\n" << *getTopLevel(); |
| 814 | llvm::scope_exit printOnFailureRAII([this] { |
| 815 | (void)this; |
| 816 | LDBG() << "Failing Top-level payload:\n" |
| 817 | << OpWithFlags(getTopLevel(), |
| 818 | OpPrintingFlags().printGenericOpForm()); |
| 819 | }); |
| 820 | |
| 821 | // Set current transform op. |
| 822 | regionStack.back()->currentTransform = transform; |
| 823 | |
| 824 | // Expensive checks to detect invalid transform IR. |
| 825 | if (options.getExpensiveChecksEnabled()) { |
| 826 | FULL_LDBG() << "ExpensiveChecksEnabled" ; |
| 827 | if (failed(Result: checkAndRecordHandleInvalidation(transform))) |
| 828 | return DiagnosedSilenceableFailure::definiteFailure(); |
| 829 | |
| 830 | for (OpOperand &operand : transform->getOpOperands()) { |
| 831 | FULL_LDBG() << "iterate on handle: " << operand.get(); |
| 832 | if (!isHandleConsumed(handle: operand.get(), transform)) { |
| 833 | FULL_LDBG() << "--handle not consumed -> SKIP" ; |
| 834 | continue; |
| 835 | } |
| 836 | if (transform.allowsRepeatedHandleOperands()) { |
| 837 | FULL_LDBG() << "--op allows repeated handles -> SKIP" ; |
| 838 | continue; |
| 839 | } |
| 840 | FULL_LDBG() << "--handle is consumed" ; |
| 841 | |
| 842 | Type operandType = operand.get().getType(); |
| 843 | if (llvm::isa<TransformHandleTypeInterface>(Val: operandType)) { |
| 844 | FULL_LDBG() << "--checkRepeatedConsumptionInOperand for Operation*" ; |
| 845 | DiagnosedSilenceableFailure check = |
| 846 | checkRepeatedConsumptionInOperand<Operation *>( |
| 847 | payload: getPayloadOpsView(value: operand.get()), transform, |
| 848 | operandNumber: operand.getOperandNumber()); |
| 849 | if (!check.succeeded()) { |
| 850 | FULL_LDBG() << "----FAILED" ; |
| 851 | return check; |
| 852 | } |
| 853 | } else if (llvm::isa<TransformValueHandleTypeInterface>(Val: operandType)) { |
| 854 | FULL_LDBG() << "--checkRepeatedConsumptionInOperand For Value" ; |
| 855 | DiagnosedSilenceableFailure check = |
| 856 | checkRepeatedConsumptionInOperand<Value>( |
| 857 | payload: getPayloadValuesView(handleValue: operand.get()), transform, |
| 858 | operandNumber: operand.getOperandNumber()); |
| 859 | if (!check.succeeded()) { |
| 860 | FULL_LDBG() << "----FAILED" ; |
| 861 | return check; |
| 862 | } |
| 863 | } else { |
| 864 | FULL_LDBG() << "--not a TransformHandle -> SKIP AND DROP ON THE FLOOR" ; |
| 865 | } |
| 866 | } |
| 867 | } |
| 868 | |
| 869 | // Find which operands are consumed. |
| 870 | SmallVector<OpOperand *> consumedOperands = |
| 871 | transform.getConsumedHandleOpOperands(); |
| 872 | |
| 873 | // Remember the results of the payload ops associated with the consumed |
| 874 | // op handles or the ops defining the value handles so we can drop the |
| 875 | // association with them later. This must happen here because the |
| 876 | // transformation may destroy or mutate them so we cannot traverse the payload |
| 877 | // IR after that. |
| 878 | SmallVector<Value> origOpFlatResults; |
| 879 | SmallVector<Operation *> origAssociatedOps; |
| 880 | for (OpOperand *opOperand : consumedOperands) { |
| 881 | Value operand = opOperand->get(); |
| 882 | if (llvm::isa<TransformHandleTypeInterface>(Val: operand.getType())) { |
| 883 | for (Operation *payloadOp : getPayloadOps(value: operand)) { |
| 884 | llvm::append_range(C&: origOpFlatResults, R: payloadOp->getResults()); |
| 885 | } |
| 886 | continue; |
| 887 | } |
| 888 | if (llvm::isa<TransformValueHandleTypeInterface>(Val: operand.getType())) { |
| 889 | for (Value payloadValue : getPayloadValuesView(handleValue: operand)) { |
| 890 | if (llvm::isa<OpResult>(Val: payloadValue)) { |
| 891 | origAssociatedOps.push_back(Elt: payloadValue.getDefiningOp()); |
| 892 | continue; |
| 893 | } |
| 894 | llvm::append_range( |
| 895 | C&: origAssociatedOps, |
| 896 | R: llvm::map_range(C&: *llvm::cast<BlockArgument>(Val&: payloadValue).getOwner(), |
| 897 | F: [](Operation &op) { return &op; })); |
| 898 | } |
| 899 | continue; |
| 900 | } |
| 901 | DiagnosedDefiniteFailure diag = |
| 902 | emitDefiniteFailure(loc: transform->getLoc()) |
| 903 | << "unexpectedly consumed a value that is not a handle as operand #" |
| 904 | << opOperand->getOperandNumber(); |
| 905 | diag.attachNote(loc: operand.getLoc()) |
| 906 | << "value defined here with type " << operand.getType(); |
| 907 | return diag; |
| 908 | } |
| 909 | |
| 910 | // Prepare rewriter and listener. |
| 911 | TrackingListenerConfig config; |
| 912 | config.skipHandleFn = [&](Value handle) { |
| 913 | // Skip handle if it is dead. |
| 914 | auto scopeIt = |
| 915 | llvm::find_if(Range: llvm::reverse(C&: regionStack), P: [&](RegionScope *scope) { |
| 916 | return handle.getParentRegion() == scope->region; |
| 917 | }); |
| 918 | assert(scopeIt != regionStack.rend() && |
| 919 | "could not find region scope for handle" ); |
| 920 | RegionScope *scope = *scopeIt; |
| 921 | return llvm::all_of(Range: handle.getUsers(), P: [&](Operation *user) { |
| 922 | return user == scope->currentTransform || |
| 923 | happensBefore(a: user, b: scope->currentTransform); |
| 924 | }); |
| 925 | }; |
| 926 | transform::ErrorCheckingTrackingListener trackingListener(*this, transform, |
| 927 | config); |
| 928 | transform::TransformRewriter rewriter(transform->getContext(), |
| 929 | &trackingListener); |
| 930 | |
| 931 | // Compute the result but do not short-circuit the silenceable failure case as |
| 932 | // we still want the handles to propagate properly so the "suppress" mode can |
| 933 | // proceed on a best effort basis. |
| 934 | transform::TransformResults results(transform->getNumResults()); |
| 935 | DiagnosedSilenceableFailure result(transform.apply(rewriter, transformResults&: results, state&: *this)); |
| 936 | compactOpHandles(); |
| 937 | |
| 938 | // Error handling: fail if transform or listener failed. |
| 939 | DiagnosedSilenceableFailure trackingFailure = |
| 940 | trackingListener.checkAndResetError(); |
| 941 | if (!transform->hasTrait<ReportTrackingListenerFailuresOpTrait>() || |
| 942 | transform->hasAttr(name: FindPayloadReplacementOpInterface:: |
| 943 | kSilenceTrackingFailuresAttrName)) { |
| 944 | // Only report failures for ReportTrackingListenerFailuresOpTrait ops. Also |
| 945 | // do not report failures if the above mentioned attribute is set. |
| 946 | if (trackingFailure.isSilenceableFailure()) |
| 947 | (void)trackingFailure.silence(); |
| 948 | trackingFailure = DiagnosedSilenceableFailure::success(); |
| 949 | } |
| 950 | if (!trackingFailure.succeeded()) { |
| 951 | if (result.succeeded()) { |
| 952 | result = std::move(trackingFailure); |
| 953 | } else { |
| 954 | // Transform op errors have precedence, report those first. |
| 955 | if (result.isSilenceableFailure()) |
| 956 | result.attachNote() << "tracking listener also failed: " |
| 957 | << trackingFailure.getMessage(); |
| 958 | (void)trackingFailure.silence(); |
| 959 | } |
| 960 | } |
| 961 | if (result.isDefiniteFailure()) |
| 962 | return result; |
| 963 | |
| 964 | // If a silenceable failure was produced, some results may be unset, set them |
| 965 | // to empty lists. |
| 966 | if (result.isSilenceableFailure()) |
| 967 | results.setRemainingToEmpty(transform); |
| 968 | |
| 969 | // Remove the mapping for the operand if it is consumed by the operation. This |
| 970 | // allows us to catch use-after-free with assertions later on. |
| 971 | for (OpOperand *opOperand : consumedOperands) { |
| 972 | Value operand = opOperand->get(); |
| 973 | if (llvm::isa<TransformHandleTypeInterface>(Val: operand.getType())) { |
| 974 | forgetMapping(opHandle: operand, origOpFlatResults); |
| 975 | } else if (llvm::isa<TransformValueHandleTypeInterface>( |
| 976 | Val: operand.getType())) { |
| 977 | forgetValueMapping(valueHandle: operand, payloadOperations: origAssociatedOps); |
| 978 | } |
| 979 | } |
| 980 | |
| 981 | if (failed(Result: updateStateFromResults(results, opResults: transform->getResults()))) |
| 982 | return DiagnosedSilenceableFailure::definiteFailure(); |
| 983 | |
| 984 | printOnFailureRAII.release(); |
| 985 | DEBUG_WITH_TYPE(DEBUG_PRINT_AFTER_ALL, { |
| 986 | LDBG() << "Top-level payload:\n" << *getTopLevel(); |
| 987 | }); |
| 988 | return result; |
| 989 | } |
| 990 | |
| 991 | LogicalResult transform::TransformState::updateStateFromResults( |
| 992 | const TransformResults &results, ResultRange opResults) { |
| 993 | for (OpResult result : opResults) { |
| 994 | if (llvm::isa<TransformParamTypeInterface>(Val: result.getType())) { |
| 995 | assert(results.isParam(result.getResultNumber()) && |
| 996 | "expected parameters for the parameter-typed result" ); |
| 997 | if (failed( |
| 998 | Result: setParams(value: result, params: results.getParams(resultNumber: result.getResultNumber())))) { |
| 999 | return failure(); |
| 1000 | } |
| 1001 | } else if (llvm::isa<TransformValueHandleTypeInterface>(Val: result.getType())) { |
| 1002 | assert(results.isValue(result.getResultNumber()) && |
| 1003 | "expected values for value-type-result" ); |
| 1004 | if (failed(Result: setPayloadValues( |
| 1005 | handle: result, payloadValues: results.getValues(resultNumber: result.getResultNumber())))) { |
| 1006 | return failure(); |
| 1007 | } |
| 1008 | } else { |
| 1009 | assert(!results.isParam(result.getResultNumber()) && |
| 1010 | "expected payload ops for the non-parameter typed result" ); |
| 1011 | if (failed( |
| 1012 | Result: setPayloadOps(value: result, targets: results.get(resultNumber: result.getResultNumber())))) { |
| 1013 | return failure(); |
| 1014 | } |
| 1015 | } |
| 1016 | } |
| 1017 | return success(); |
| 1018 | } |
| 1019 | |
| 1020 | //===----------------------------------------------------------------------===// |
| 1021 | // TransformState::Extension |
| 1022 | //===----------------------------------------------------------------------===// |
| 1023 | |
| 1024 | transform::TransformState::Extension::~Extension() = default; |
| 1025 | |
| 1026 | LogicalResult |
| 1027 | transform::TransformState::Extension::replacePayloadOp(Operation *op, |
| 1028 | Operation *replacement) { |
| 1029 | // TODO: we may need to invalidate handles to operations and values nested in |
| 1030 | // the operation being replaced. |
| 1031 | return state.replacePayloadOp(op, replacement); |
| 1032 | } |
| 1033 | |
| 1034 | LogicalResult |
| 1035 | transform::TransformState::Extension::replacePayloadValue(Value value, |
| 1036 | Value replacement) { |
| 1037 | return state.replacePayloadValue(value, replacement); |
| 1038 | } |
| 1039 | |
| 1040 | //===----------------------------------------------------------------------===// |
| 1041 | // TransformState::RegionScope |
| 1042 | //===----------------------------------------------------------------------===// |
| 1043 | |
| 1044 | transform::TransformState::RegionScope::~RegionScope() { |
| 1045 | // Remove handle invalidation notices as handles are going out of scope. |
| 1046 | // The same region may be re-entered leading to incorrect invalidation |
| 1047 | // errors. |
| 1048 | for (Block &block : *region) { |
| 1049 | for (Value handle : block.getArguments()) { |
| 1050 | state.invalidatedHandles.erase(Val: handle); |
| 1051 | } |
| 1052 | for (Operation &op : block) { |
| 1053 | for (Value handle : op.getResults()) { |
| 1054 | state.invalidatedHandles.erase(Val: handle); |
| 1055 | } |
| 1056 | } |
| 1057 | } |
| 1058 | |
| 1059 | #if LLVM_ENABLE_ABI_BREAKING_CHECKS |
| 1060 | // Remember pointers to payload ops referenced by the handles going out of |
| 1061 | // scope. |
| 1062 | SmallVector<Operation *> referencedOps = |
| 1063 | llvm::to_vector(llvm::make_first_range(state.mappings[region]->reverse)); |
| 1064 | #endif // LLVM_ENABLE_ABI_BREAKING_CHECKS |
| 1065 | |
| 1066 | state.mappings.erase(Key: region); |
| 1067 | state.regionStack.pop_back(); |
| 1068 | } |
| 1069 | |
| 1070 | //===----------------------------------------------------------------------===// |
| 1071 | // TransformResults |
| 1072 | //===----------------------------------------------------------------------===// |
| 1073 | |
| 1074 | transform::TransformResults::TransformResults(unsigned numSegments) { |
| 1075 | operations.appendEmptyRows(num: numSegments); |
| 1076 | params.appendEmptyRows(num: numSegments); |
| 1077 | values.appendEmptyRows(num: numSegments); |
| 1078 | } |
| 1079 | |
| 1080 | void transform::TransformResults::setParams( |
| 1081 | OpResult value, ArrayRef<transform::TransformState::Param> params) { |
| 1082 | int64_t position = value.getResultNumber(); |
| 1083 | assert(position < static_cast<int64_t>(this->params.size()) && |
| 1084 | "setting params for a non-existent handle" ); |
| 1085 | assert(this->params[position].data() == nullptr && "params already set" ); |
| 1086 | assert(operations[position].data() == nullptr && |
| 1087 | "another kind of results already set" ); |
| 1088 | assert(values[position].data() == nullptr && |
| 1089 | "another kind of results already set" ); |
| 1090 | this->params.replace(pos: position, elements&: params); |
| 1091 | } |
| 1092 | |
| 1093 | void transform::TransformResults::setMappedValues( |
| 1094 | OpResult handle, ArrayRef<MappedValue> values) { |
| 1095 | DiagnosedSilenceableFailure diag = dispatchMappedValues( |
| 1096 | handle, values, |
| 1097 | operationsFn: [&](ArrayRef<Operation *> operations) { |
| 1098 | return set(value: handle, ops&: operations), success(); |
| 1099 | }, |
| 1100 | paramsFn: [&](ArrayRef<Param> params) { |
| 1101 | return setParams(value: handle, params), success(); |
| 1102 | }, |
| 1103 | valuesFn: [&](ValueRange payloadValues) { |
| 1104 | return setValues(handle, values&: payloadValues), success(); |
| 1105 | }); |
| 1106 | #ifndef NDEBUG |
| 1107 | if (!diag.succeeded()) |
| 1108 | llvm::dbgs() << diag.getStatusString() << "\n" ; |
| 1109 | assert(diag.succeeded() && "incorrect mapping" ); |
| 1110 | #endif // NDEBUG |
| 1111 | (void)diag.silence(); |
| 1112 | } |
| 1113 | |
| 1114 | void transform::TransformResults::setRemainingToEmpty( |
| 1115 | transform::TransformOpInterface transform) { |
| 1116 | for (OpResult opResult : transform->getResults()) { |
| 1117 | if (!isSet(resultNumber: opResult.getResultNumber())) |
| 1118 | setMappedValues(handle: opResult, values: {}); |
| 1119 | } |
| 1120 | } |
| 1121 | |
| 1122 | ArrayRef<Operation *> |
| 1123 | transform::TransformResults::get(unsigned resultNumber) const { |
| 1124 | assert(resultNumber < operations.size() && |
| 1125 | "querying results for a non-existent handle" ); |
| 1126 | assert(operations[resultNumber].data() != nullptr && |
| 1127 | "querying unset results (values or params expected?)" ); |
| 1128 | return operations[resultNumber]; |
| 1129 | } |
| 1130 | |
| 1131 | ArrayRef<transform::TransformState::Param> |
| 1132 | transform::TransformResults::getParams(unsigned resultNumber) const { |
| 1133 | assert(resultNumber < params.size() && |
| 1134 | "querying params for a non-existent handle" ); |
| 1135 | assert(params[resultNumber].data() != nullptr && |
| 1136 | "querying unset params (ops or values expected?)" ); |
| 1137 | return params[resultNumber]; |
| 1138 | } |
| 1139 | |
| 1140 | ArrayRef<Value> |
| 1141 | transform::TransformResults::getValues(unsigned resultNumber) const { |
| 1142 | assert(resultNumber < values.size() && |
| 1143 | "querying values for a non-existent handle" ); |
| 1144 | assert(values[resultNumber].data() != nullptr && |
| 1145 | "querying unset values (ops or params expected?)" ); |
| 1146 | return values[resultNumber]; |
| 1147 | } |
| 1148 | |
| 1149 | bool transform::TransformResults::isParam(unsigned resultNumber) const { |
| 1150 | assert(resultNumber < params.size() && |
| 1151 | "querying association for a non-existent handle" ); |
| 1152 | return params[resultNumber].data() != nullptr; |
| 1153 | } |
| 1154 | |
| 1155 | bool transform::TransformResults::isValue(unsigned resultNumber) const { |
| 1156 | assert(resultNumber < values.size() && |
| 1157 | "querying association for a non-existent handle" ); |
| 1158 | return values[resultNumber].data() != nullptr; |
| 1159 | } |
| 1160 | |
| 1161 | bool transform::TransformResults::isSet(unsigned resultNumber) const { |
| 1162 | assert(resultNumber < params.size() && |
| 1163 | "querying association for a non-existent handle" ); |
| 1164 | return params[resultNumber].data() != nullptr || |
| 1165 | operations[resultNumber].data() != nullptr || |
| 1166 | values[resultNumber].data() != nullptr; |
| 1167 | } |
| 1168 | |
| 1169 | //===----------------------------------------------------------------------===// |
| 1170 | // TrackingListener |
| 1171 | //===----------------------------------------------------------------------===// |
| 1172 | |
| 1173 | transform::TrackingListener::TrackingListener(TransformState &state, |
| 1174 | TransformOpInterface op, |
| 1175 | TrackingListenerConfig config) |
| 1176 | : TransformState::Extension(state), transformOp(op), |
| 1177 | config(std::move(config)) { |
| 1178 | if (op) { |
| 1179 | for (OpOperand *opOperand : transformOp.getConsumedHandleOpOperands()) { |
| 1180 | consumedHandles.insert(V: opOperand->get()); |
| 1181 | } |
| 1182 | } |
| 1183 | } |
| 1184 | |
| 1185 | Operation *transform::TrackingListener::getCommonDefiningOp(ValueRange values) { |
| 1186 | Operation *defOp = nullptr; |
| 1187 | for (Value v : values) { |
| 1188 | // Skip empty values. |
| 1189 | if (!v) |
| 1190 | continue; |
| 1191 | if (!defOp) { |
| 1192 | defOp = v.getDefiningOp(); |
| 1193 | continue; |
| 1194 | } |
| 1195 | if (defOp != v.getDefiningOp()) |
| 1196 | return nullptr; |
| 1197 | } |
| 1198 | return defOp; |
| 1199 | } |
| 1200 | |
| 1201 | DiagnosedSilenceableFailure transform::TrackingListener::findReplacementOp( |
| 1202 | Operation *&result, Operation *op, ValueRange newValues) const { |
| 1203 | assert(op->getNumResults() == newValues.size() && |
| 1204 | "invalid number of replacement values" ); |
| 1205 | SmallVector<Value> values(newValues.begin(), newValues.end()); |
| 1206 | |
| 1207 | DiagnosedSilenceableFailure diag = emitSilenceableFailure( |
| 1208 | op: getTransformOp(), message: "tracking listener failed to find replacement op " |
| 1209 | "during application of this transform op" ); |
| 1210 | |
| 1211 | do { |
| 1212 | // If the replacement values belong to different ops, drop the mapping. |
| 1213 | Operation *defOp = getCommonDefiningOp(values); |
| 1214 | if (!defOp) { |
| 1215 | diag.attachNote() << "replacement values belong to different ops" ; |
| 1216 | return diag; |
| 1217 | } |
| 1218 | |
| 1219 | // Skip through ops that implement CastOpInterface. |
| 1220 | if (config.skipCastOps && isa<CastOpInterface>(Val: defOp)) { |
| 1221 | values.clear(); |
| 1222 | values.assign(in_start: defOp->getOperands().begin(), in_end: defOp->getOperands().end()); |
| 1223 | diag.attachNote(loc: defOp->getLoc()) |
| 1224 | << "using output of 'CastOpInterface' op" ; |
| 1225 | continue; |
| 1226 | } |
| 1227 | |
| 1228 | // If the defining op has the same name or we do not care about the name of |
| 1229 | // op replacements at all, we take it as a replacement. |
| 1230 | if (!config.requireMatchingReplacementOpName || |
| 1231 | op->getName() == defOp->getName()) { |
| 1232 | result = defOp; |
| 1233 | return DiagnosedSilenceableFailure::success(); |
| 1234 | } |
| 1235 | |
| 1236 | // Replacing an op with a constant-like equivalent is a common |
| 1237 | // canonicalization. |
| 1238 | if (defOp->hasTrait<OpTrait::ConstantLike>()) { |
| 1239 | result = defOp; |
| 1240 | return DiagnosedSilenceableFailure::success(); |
| 1241 | } |
| 1242 | |
| 1243 | values.clear(); |
| 1244 | |
| 1245 | // Skip through ops that implement FindPayloadReplacementOpInterface. |
| 1246 | if (auto findReplacementOpInterface = |
| 1247 | dyn_cast<FindPayloadReplacementOpInterface>(Val: defOp)) { |
| 1248 | values.assign(RHS: findReplacementOpInterface.getNextOperands()); |
| 1249 | diag.attachNote(loc: defOp->getLoc()) << "using operands provided by " |
| 1250 | "'FindPayloadReplacementOpInterface'" ; |
| 1251 | continue; |
| 1252 | } |
| 1253 | } while (!values.empty()); |
| 1254 | |
| 1255 | diag.attachNote() << "ran out of suitable replacement values" ; |
| 1256 | return diag; |
| 1257 | } |
| 1258 | |
| 1259 | void transform::TrackingListener::notifyMatchFailure( |
| 1260 | Location loc, function_ref<void(Diagnostic &)> reasonCallback) { |
| 1261 | LLVM_DEBUG({ |
| 1262 | Diagnostic diag(loc, DiagnosticSeverity::Remark); |
| 1263 | reasonCallback(diag); |
| 1264 | LDBG() << "Match Failure : " << diag.str(); |
| 1265 | }); |
| 1266 | } |
| 1267 | |
| 1268 | void transform::TrackingListener::notifyOperationErased(Operation *op) { |
| 1269 | // Remove mappings for result values. |
| 1270 | for (OpResult value : op->getResults()) |
| 1271 | (void)replacePayloadValue(value, replacement: nullptr); |
| 1272 | // Remove mapping for op. |
| 1273 | (void)replacePayloadOp(op, replacement: nullptr); |
| 1274 | } |
| 1275 | |
| 1276 | void transform::TrackingListener::notifyOperationReplaced( |
| 1277 | Operation *op, ValueRange newValues) { |
| 1278 | assert(op->getNumResults() == newValues.size() && |
| 1279 | "invalid number of replacement values" ); |
| 1280 | |
| 1281 | // Replace value handles. |
| 1282 | for (auto [oldValue, newValue] : llvm::zip(t: op->getResults(), u&: newValues)) |
| 1283 | (void)replacePayloadValue(value: oldValue, replacement: newValue); |
| 1284 | |
| 1285 | // Replace op handle. |
| 1286 | SmallVector<Value> opHandles; |
| 1287 | if (failed(Result: getTransformState().getHandlesForPayloadOp( |
| 1288 | op, handles&: opHandles, /*includeOutOfScope=*/true))) { |
| 1289 | // Op is not tracked. |
| 1290 | return; |
| 1291 | } |
| 1292 | |
| 1293 | // Helper function to check if the current transform op consumes any handle |
| 1294 | // that is mapped to `op`. |
| 1295 | // |
| 1296 | // Note: If a handle was consumed, there shouldn't be any alive users, so it |
| 1297 | // is not really necessary to check for consumed handles. However, in case |
| 1298 | // there are indeed alive handles that were consumed (which is undefined |
| 1299 | // behavior) and a replacement op could not be found, we want to fail with a |
| 1300 | // nicer error message: "op uses a handle invalidated..." instead of "could |
| 1301 | // not find replacement op". This nicer error is produced later. |
| 1302 | auto handleWasConsumed = [&] { |
| 1303 | return llvm::any_of(Range&: opHandles, |
| 1304 | P: [&](Value h) { return consumedHandles.contains(V: h); }); |
| 1305 | }; |
| 1306 | |
| 1307 | // Check if there are any handles that must be updated. |
| 1308 | Value aliveHandle; |
| 1309 | if (config.skipHandleFn) { |
| 1310 | auto *it = llvm::find_if(Range&: opHandles, |
| 1311 | P: [&](Value v) { return !config.skipHandleFn(v); }); |
| 1312 | if (it != opHandles.end()) |
| 1313 | aliveHandle = *it; |
| 1314 | } else if (!opHandles.empty()) { |
| 1315 | aliveHandle = opHandles.front(); |
| 1316 | } |
| 1317 | if (!aliveHandle || handleWasConsumed()) { |
| 1318 | // The op is tracked but the corresponding handles are dead or were |
| 1319 | // consumed. Drop the op form the mapping. |
| 1320 | (void)replacePayloadOp(op, replacement: nullptr); |
| 1321 | return; |
| 1322 | } |
| 1323 | |
| 1324 | Operation *replacement; |
| 1325 | DiagnosedSilenceableFailure diag = |
| 1326 | findReplacementOp(result&: replacement, op, newValues); |
| 1327 | // If the op is tracked but no replacement op was found, send a |
| 1328 | // notification. |
| 1329 | if (!diag.succeeded()) { |
| 1330 | diag.attachNote(loc: aliveHandle.getLoc()) |
| 1331 | << "replacement is required because this handle must be updated" ; |
| 1332 | notifyPayloadReplacementNotFound(op, values: newValues, diag: std::move(diag)); |
| 1333 | (void)replacePayloadOp(op, replacement: nullptr); |
| 1334 | return; |
| 1335 | } |
| 1336 | |
| 1337 | (void)replacePayloadOp(op, replacement); |
| 1338 | } |
| 1339 | |
| 1340 | transform::ErrorCheckingTrackingListener::~ErrorCheckingTrackingListener() { |
| 1341 | // The state of the ErrorCheckingTrackingListener must be checked and reset |
| 1342 | // if there was an error. This is to prevent errors from accidentally being |
| 1343 | // missed. |
| 1344 | assert(status.succeeded() && "listener state was not checked" ); |
| 1345 | } |
| 1346 | |
| 1347 | DiagnosedSilenceableFailure |
| 1348 | transform::ErrorCheckingTrackingListener::checkAndResetError() { |
| 1349 | DiagnosedSilenceableFailure s = std::move(status); |
| 1350 | status = DiagnosedSilenceableFailure::success(); |
| 1351 | errorCounter = 0; |
| 1352 | return s; |
| 1353 | } |
| 1354 | |
| 1355 | bool transform::ErrorCheckingTrackingListener::failed() const { |
| 1356 | return !status.succeeded(); |
| 1357 | } |
| 1358 | |
| 1359 | void transform::ErrorCheckingTrackingListener::notifyPayloadReplacementNotFound( |
| 1360 | Operation *op, ValueRange values, DiagnosedSilenceableFailure &&diag) { |
| 1361 | |
| 1362 | // Merge potentially existing diags and store the result in the listener. |
| 1363 | SmallVector<Diagnostic> diags; |
| 1364 | diag.takeDiagnostics(diags); |
| 1365 | if (!status.succeeded()) |
| 1366 | status.takeDiagnostics(diags); |
| 1367 | status = DiagnosedSilenceableFailure::silenceableFailure(diag: std::move(diags)); |
| 1368 | |
| 1369 | // Report more details. |
| 1370 | status.attachNote(loc: op->getLoc()) << "[" << errorCounter << "] replaced op" ; |
| 1371 | for (auto &&[index, value] : llvm::enumerate(First&: values)) |
| 1372 | status.attachNote(loc: value.getLoc()) |
| 1373 | << "[" << errorCounter << "] replacement value " << index; |
| 1374 | ++errorCounter; |
| 1375 | } |
| 1376 | |
| 1377 | std::string |
| 1378 | transform::ErrorCheckingTrackingListener::getLatestMatchFailureMessage() { |
| 1379 | if (!matchFailure) { |
| 1380 | return "" ; |
| 1381 | } |
| 1382 | return matchFailure->str(); |
| 1383 | } |
| 1384 | |
| 1385 | void transform::ErrorCheckingTrackingListener::notifyMatchFailure( |
| 1386 | Location loc, function_ref<void(Diagnostic &)> reasonCallback) { |
| 1387 | Diagnostic diag(loc, DiagnosticSeverity::Remark); |
| 1388 | reasonCallback(diag); |
| 1389 | matchFailure = std::move(diag); |
| 1390 | } |
| 1391 | |
| 1392 | //===----------------------------------------------------------------------===// |
| 1393 | // TransformRewriter |
| 1394 | //===----------------------------------------------------------------------===// |
| 1395 | |
| 1396 | transform::TransformRewriter::TransformRewriter( |
| 1397 | MLIRContext *ctx, ErrorCheckingTrackingListener *listener) |
| 1398 | : RewriterBase(ctx), listener(listener) { |
| 1399 | setListener(listener); |
| 1400 | } |
| 1401 | |
| 1402 | bool transform::TransformRewriter::hasTrackingFailures() const { |
| 1403 | return listener->failed(); |
| 1404 | } |
| 1405 | |
| 1406 | /// Silence all tracking failures that have been encountered so far. |
| 1407 | void transform::TransformRewriter::silenceTrackingFailure() { |
| 1408 | if (hasTrackingFailures()) { |
| 1409 | DiagnosedSilenceableFailure status = listener->checkAndResetError(); |
| 1410 | (void)status.silence(); |
| 1411 | } |
| 1412 | } |
| 1413 | |
| 1414 | LogicalResult transform::TransformRewriter::notifyPayloadOperationReplaced( |
| 1415 | Operation *op, Operation *replacement) { |
| 1416 | return listener->replacePayloadOp(op, replacement); |
| 1417 | } |
| 1418 | |
| 1419 | //===----------------------------------------------------------------------===// |
| 1420 | // Utilities for TransformEachOpTrait. |
| 1421 | //===----------------------------------------------------------------------===// |
| 1422 | |
| 1423 | LogicalResult |
| 1424 | transform::detail::checkNestedConsumption(Location loc, |
| 1425 | ArrayRef<Operation *> targets) { |
| 1426 | for (auto &&[position, parent] : llvm::enumerate(First&: targets)) { |
| 1427 | for (Operation *child : targets.drop_front(N: position + 1)) { |
| 1428 | if (parent->isAncestor(other: child)) { |
| 1429 | InFlightDiagnostic diag = |
| 1430 | emitError(loc) |
| 1431 | << "transform operation consumes a handle pointing to an ancestor " |
| 1432 | "payload operation before its descendant" ; |
| 1433 | diag.attachNote() |
| 1434 | << "the ancestor is likely erased or rewritten before the " |
| 1435 | "descendant is accessed, leading to undefined behavior" ; |
| 1436 | diag.attachNote(noteLoc: parent->getLoc()) << "ancestor payload op" ; |
| 1437 | diag.attachNote(noteLoc: child->getLoc()) << "descendant payload op" ; |
| 1438 | return diag; |
| 1439 | } |
| 1440 | } |
| 1441 | } |
| 1442 | return success(); |
| 1443 | } |
| 1444 | |
| 1445 | LogicalResult |
| 1446 | transform::detail::checkApplyToOne(Operation *transformOp, |
| 1447 | Location payloadOpLoc, |
| 1448 | const ApplyToEachResultList &partialResult) { |
| 1449 | Location transformOpLoc = transformOp->getLoc(); |
| 1450 | StringRef transformOpName = transformOp->getName().getStringRef(); |
| 1451 | unsigned expectedNumResults = transformOp->getNumResults(); |
| 1452 | |
| 1453 | // Reuse the emission of the diagnostic note. |
| 1454 | auto emitDiag = [&]() { |
| 1455 | auto diag = mlir::emitError(loc: transformOpLoc); |
| 1456 | diag.attachNote(noteLoc: payloadOpLoc) << "when applied to this op" ; |
| 1457 | return diag; |
| 1458 | }; |
| 1459 | |
| 1460 | if (partialResult.size() != expectedNumResults) { |
| 1461 | auto diag = emitDiag() << "application of " << transformOpName |
| 1462 | << " expected to produce " << expectedNumResults |
| 1463 | << " results (actually produced " |
| 1464 | << partialResult.size() << ")." ; |
| 1465 | diag.attachNote(noteLoc: transformOpLoc) |
| 1466 | << "if you need variadic results, consider a generic `apply` " |
| 1467 | << "instead of the specialized `applyToOne`." ; |
| 1468 | return failure(); |
| 1469 | } |
| 1470 | |
| 1471 | // Check that the right kind of value was produced. |
| 1472 | for (const auto &[ptr, res] : |
| 1473 | llvm::zip(t: partialResult, u: transformOp->getResults())) { |
| 1474 | if (ptr.isNull()) |
| 1475 | continue; |
| 1476 | if (llvm::isa<TransformHandleTypeInterface>(Val: res.getType()) && |
| 1477 | !isa<Operation *>(Val: ptr)) { |
| 1478 | return emitDiag() << "application of " << transformOpName |
| 1479 | << " expected to produce an Operation * for result #" |
| 1480 | << res.getResultNumber(); |
| 1481 | } |
| 1482 | if (llvm::isa<TransformParamTypeInterface>(Val: res.getType()) && |
| 1483 | !isa<Attribute>(Val: ptr)) { |
| 1484 | return emitDiag() << "application of " << transformOpName |
| 1485 | << " expected to produce an Attribute for result #" |
| 1486 | << res.getResultNumber(); |
| 1487 | } |
| 1488 | if (llvm::isa<TransformValueHandleTypeInterface>(Val: res.getType()) && |
| 1489 | !isa<Value>(Val: ptr)) { |
| 1490 | return emitDiag() << "application of " << transformOpName |
| 1491 | << " expected to produce a Value for result #" |
| 1492 | << res.getResultNumber(); |
| 1493 | } |
| 1494 | } |
| 1495 | return success(); |
| 1496 | } |
| 1497 | |
| 1498 | template <typename T> |
| 1499 | static SmallVector<T> castVector(ArrayRef<transform::MappedValue> range) { |
| 1500 | return llvm::map_to_vector(range, llvm::CastTo<T>); |
| 1501 | } |
| 1502 | |
| 1503 | void transform::detail::setApplyToOneResults( |
| 1504 | Operation *transformOp, TransformResults &transformResults, |
| 1505 | ArrayRef<ApplyToEachResultList> results) { |
| 1506 | SmallVector<SmallVector<MappedValue>> transposed; |
| 1507 | transposed.resize(N: transformOp->getNumResults()); |
| 1508 | for (const ApplyToEachResultList &partialResults : results) { |
| 1509 | if (llvm::any_of(Range: partialResults, |
| 1510 | P: [](MappedValue value) { return value.isNull(); })) |
| 1511 | continue; |
| 1512 | assert(transformOp->getNumResults() == partialResults.size() && |
| 1513 | "expected as many partial results as op as results" ); |
| 1514 | for (auto [i, value] : llvm::enumerate(First: partialResults)) |
| 1515 | transposed[i].push_back(Elt: value); |
| 1516 | } |
| 1517 | |
| 1518 | for (OpResult r : transformOp->getResults()) { |
| 1519 | unsigned position = r.getResultNumber(); |
| 1520 | if (llvm::isa<TransformParamTypeInterface>(Val: r.getType())) { |
| 1521 | transformResults.setParams(value: r, |
| 1522 | params: castVector<Attribute>(range: transposed[position])); |
| 1523 | } else if (llvm::isa<TransformValueHandleTypeInterface>(Val: r.getType())) { |
| 1524 | transformResults.setValues(handle: r, values: castVector<Value>(range: transposed[position])); |
| 1525 | } else { |
| 1526 | transformResults.set(value: r, ops: castVector<Operation *>(range: transposed[position])); |
| 1527 | } |
| 1528 | } |
| 1529 | } |
| 1530 | |
| 1531 | //===----------------------------------------------------------------------===// |
| 1532 | // Utilities for implementing transform ops with regions. |
| 1533 | //===----------------------------------------------------------------------===// |
| 1534 | |
| 1535 | LogicalResult transform::detail::appendValueMappings( |
| 1536 | MutableArrayRef<SmallVector<transform::MappedValue>> mappings, |
| 1537 | ValueRange values, const transform::TransformState &state, bool flatten) { |
| 1538 | assert(mappings.size() == values.size() && "mismatching number of mappings" ); |
| 1539 | for (auto &&[operand, mapped] : llvm::zip_equal(t&: values, u&: mappings)) { |
| 1540 | size_t mappedSize = mapped.size(); |
| 1541 | if (llvm::isa<TransformHandleTypeInterface>(Val: operand.getType())) { |
| 1542 | llvm::append_range(C&: mapped, R: state.getPayloadOps(value: operand)); |
| 1543 | } else if (llvm::isa<TransformValueHandleTypeInterface>( |
| 1544 | Val: operand.getType())) { |
| 1545 | llvm::append_range(C&: mapped, R: state.getPayloadValues(handleValue: operand)); |
| 1546 | } else { |
| 1547 | assert(llvm::isa<TransformParamTypeInterface>(operand.getType()) && |
| 1548 | "unsupported kind of transform dialect value" ); |
| 1549 | llvm::append_range(C&: mapped, R: state.getParams(value: operand)); |
| 1550 | } |
| 1551 | |
| 1552 | if (mapped.size() - mappedSize != 1 && !flatten) |
| 1553 | return failure(); |
| 1554 | } |
| 1555 | return success(); |
| 1556 | } |
| 1557 | |
| 1558 | void transform::detail::prepareValueMappings( |
| 1559 | SmallVectorImpl<SmallVector<transform::MappedValue>> &mappings, |
| 1560 | ValueRange values, const transform::TransformState &state) { |
| 1561 | mappings.resize(N: mappings.size() + values.size()); |
| 1562 | (void)appendValueMappings( |
| 1563 | mappings: MutableArrayRef<SmallVector<transform::MappedValue>>(mappings).take_back( |
| 1564 | N: values.size()), |
| 1565 | values, state); |
| 1566 | } |
| 1567 | |
| 1568 | void transform::detail::forwardTerminatorOperands( |
| 1569 | Block *block, transform::TransformState &state, |
| 1570 | transform::TransformResults &results) { |
| 1571 | for (auto &&[terminatorOperand, result] : |
| 1572 | llvm::zip(t: block->getTerminator()->getOperands(), |
| 1573 | u: block->getParentOp()->getOpResults())) { |
| 1574 | if (llvm::isa<transform::TransformHandleTypeInterface>(Val: result.getType())) { |
| 1575 | results.set(value: result, ops: state.getPayloadOps(value: terminatorOperand)); |
| 1576 | } else if (llvm::isa<transform::TransformValueHandleTypeInterface>( |
| 1577 | Val: result.getType())) { |
| 1578 | results.setValues(handle: result, values: state.getPayloadValues(handleValue: terminatorOperand)); |
| 1579 | } else { |
| 1580 | assert( |
| 1581 | llvm::isa<transform::TransformParamTypeInterface>(result.getType()) && |
| 1582 | "unhandled transform type interface" ); |
| 1583 | results.setParams(value: result, params: state.getParams(value: terminatorOperand)); |
| 1584 | } |
| 1585 | } |
| 1586 | } |
| 1587 | |
| 1588 | transform::TransformState |
| 1589 | transform::detail::makeTransformStateForTesting(Region *region, |
| 1590 | Operation *payloadRoot) { |
| 1591 | return TransformState(region, payloadRoot); |
| 1592 | } |
| 1593 | |
| 1594 | //===----------------------------------------------------------------------===// |
| 1595 | // Utilities for PossibleTopLevelTransformOpTrait. |
| 1596 | //===----------------------------------------------------------------------===// |
| 1597 | |
| 1598 | /// Appends to `effects` the memory effect instances on `target` with the same |
| 1599 | /// resource and effect as the ones the operation `iface` having on `source`. |
| 1600 | static void |
| 1601 | remapEffects(MemoryEffectOpInterface iface, BlockArgument source, |
| 1602 | OpOperand *target, |
| 1603 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 1604 | SmallVector<MemoryEffects::EffectInstance> nestedEffects; |
| 1605 | iface.getEffectsOnValue(value: source, effects&: nestedEffects); |
| 1606 | for (const auto &effect : nestedEffects) |
| 1607 | effects.emplace_back(Args: effect.getEffect(), Args&: target, Args: effect.getResource()); |
| 1608 | } |
| 1609 | |
| 1610 | /// Appends to `effects` the same effects as the operations of `block` have on |
| 1611 | /// block arguments but associated with `operands.` |
| 1612 | static void |
| 1613 | remapArgumentEffects(Block &block, MutableArrayRef<OpOperand> operands, |
| 1614 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 1615 | for (Operation &op : block) { |
| 1616 | auto iface = dyn_cast<MemoryEffectOpInterface>(Val: &op); |
| 1617 | if (!iface) |
| 1618 | continue; |
| 1619 | |
| 1620 | for (auto &&[source, target] : llvm::zip(t: block.getArguments(), u&: operands)) { |
| 1621 | remapEffects(iface, source, target: &target, effects); |
| 1622 | } |
| 1623 | |
| 1624 | SmallVector<MemoryEffects::EffectInstance> nestedEffects; |
| 1625 | iface.getEffectsOnResource(resource: transform::PayloadIRResource::get(), |
| 1626 | effects&: nestedEffects); |
| 1627 | llvm::append_range(C&: effects, R&: nestedEffects); |
| 1628 | } |
| 1629 | } |
| 1630 | |
| 1631 | void transform::detail::getPotentialTopLevelEffects( |
| 1632 | Operation *operation, Value root, Block &body, |
| 1633 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 1634 | transform::onlyReadsHandle(handles: operation->getOpOperands(), effects); |
| 1635 | transform::producesHandle(handles: operation->getOpResults(), effects); |
| 1636 | |
| 1637 | if (!root) { |
| 1638 | for (Operation &op : body) { |
| 1639 | auto iface = dyn_cast<MemoryEffectOpInterface>(Val: &op); |
| 1640 | if (!iface) |
| 1641 | continue; |
| 1642 | |
| 1643 | iface.getEffects(effects); |
| 1644 | } |
| 1645 | return; |
| 1646 | } |
| 1647 | |
| 1648 | // Carry over all effects on arguments of the entry block as those on the |
| 1649 | // operands, this is the same value just remapped. |
| 1650 | remapArgumentEffects(block&: body, operands: operation->getOpOperands(), effects); |
| 1651 | } |
| 1652 | |
| 1653 | LogicalResult transform::detail::mapPossibleTopLevelTransformOpBlockArguments( |
| 1654 | TransformState &state, Operation *op, Region ®ion) { |
| 1655 | SmallVector<Operation *> targets; |
| 1656 | SmallVector<SmallVector<MappedValue>> ; |
| 1657 | if (op->getNumOperands() != 0) { |
| 1658 | llvm::append_range(C&: targets, R: state.getPayloadOps(value: op->getOperand(idx: 0))); |
| 1659 | prepareValueMappings(mappings&: extraMappings, values: op->getOperands().drop_front(), state); |
| 1660 | } else { |
| 1661 | if (state.getNumTopLevelMappings() != |
| 1662 | region.front().getNumArguments() - 1) { |
| 1663 | return emitError(loc: op->getLoc()) |
| 1664 | << "operation expects " << region.front().getNumArguments() - 1 |
| 1665 | << " extra value bindings, but " << state.getNumTopLevelMappings() |
| 1666 | << " were provided to the interpreter" ; |
| 1667 | } |
| 1668 | |
| 1669 | targets.push_back(Elt: state.getTopLevel()); |
| 1670 | |
| 1671 | for (unsigned i = 0, e = state.getNumTopLevelMappings(); i < e; ++i) |
| 1672 | extraMappings.push_back(Elt: llvm::to_vector(Range: state.getTopLevelMapping(position: i))); |
| 1673 | } |
| 1674 | |
| 1675 | if (failed(Result: state.mapBlockArguments(argument: region.front().getArgument(i: 0), operations: targets))) |
| 1676 | return failure(); |
| 1677 | |
| 1678 | for (BlockArgument argument : region.front().getArguments().drop_front()) { |
| 1679 | if (failed(Result: state.mapBlockArgument( |
| 1680 | argument, values: extraMappings[argument.getArgNumber() - 1]))) |
| 1681 | return failure(); |
| 1682 | } |
| 1683 | |
| 1684 | return success(); |
| 1685 | } |
| 1686 | |
| 1687 | LogicalResult |
| 1688 | transform::detail::verifyPossibleTopLevelTransformOpTrait(Operation *op) { |
| 1689 | // Attaching this trait without the interface is a misuse of the API, but it |
| 1690 | // cannot be caught via a static_assert because interface registration is |
| 1691 | // dynamic. |
| 1692 | assert(isa<TransformOpInterface>(op) && |
| 1693 | "should implement TransformOpInterface to have " |
| 1694 | "PossibleTopLevelTransformOpTrait" ); |
| 1695 | |
| 1696 | if (op->getNumRegions() < 1) |
| 1697 | return op->emitOpError() << "expects at least one region" ; |
| 1698 | |
| 1699 | Region *bodyRegion = &op->getRegion(index: 0); |
| 1700 | if (!llvm::hasNItems(C&: *bodyRegion, N: 1)) |
| 1701 | return op->emitOpError() << "expects a single-block region" ; |
| 1702 | |
| 1703 | Block *body = &bodyRegion->front(); |
| 1704 | if (body->getNumArguments() == 0) { |
| 1705 | return op->emitOpError() |
| 1706 | << "expects the entry block to have at least one argument" ; |
| 1707 | } |
| 1708 | if (!llvm::isa<TransformHandleTypeInterface>( |
| 1709 | Val: body->getArgument(i: 0).getType())) { |
| 1710 | return op->emitOpError() |
| 1711 | << "expects the first entry block argument to be of type " |
| 1712 | "implementing TransformHandleTypeInterface" ; |
| 1713 | } |
| 1714 | BlockArgument arg = body->getArgument(i: 0); |
| 1715 | if (op->getNumOperands() != 0) { |
| 1716 | if (arg.getType() != op->getOperand(idx: 0).getType()) { |
| 1717 | return op->emitOpError() |
| 1718 | << "expects the type of the block argument to match " |
| 1719 | "the type of the operand" ; |
| 1720 | } |
| 1721 | } |
| 1722 | for (BlockArgument arg : body->getArguments().drop_front()) { |
| 1723 | if (llvm::isa<TransformHandleTypeInterface, TransformParamTypeInterface, |
| 1724 | TransformValueHandleTypeInterface>(Val: arg.getType())) |
| 1725 | continue; |
| 1726 | |
| 1727 | InFlightDiagnostic diag = |
| 1728 | op->emitOpError() |
| 1729 | << "expects trailing entry block arguments to be of type implementing " |
| 1730 | "TransformHandleTypeInterface, TransformValueHandleTypeInterface or " |
| 1731 | "TransformParamTypeInterface" ; |
| 1732 | diag.attachNote() << "argument #" << arg.getArgNumber() << " does not" ; |
| 1733 | return diag; |
| 1734 | } |
| 1735 | |
| 1736 | if (auto *parent = |
| 1737 | op->getParentWithTrait<PossibleTopLevelTransformOpTrait>()) { |
| 1738 | if (op->getNumOperands() != body->getNumArguments()) { |
| 1739 | InFlightDiagnostic diag = |
| 1740 | op->emitOpError() |
| 1741 | << "expects operands to be provided for a nested op" ; |
| 1742 | diag.attachNote(noteLoc: parent->getLoc()) |
| 1743 | << "nested in another possible top-level op" ; |
| 1744 | return diag; |
| 1745 | } |
| 1746 | } |
| 1747 | |
| 1748 | return success(); |
| 1749 | } |
| 1750 | |
| 1751 | //===----------------------------------------------------------------------===// |
| 1752 | // Utilities for ParamProducedTransformOpTrait. |
| 1753 | //===----------------------------------------------------------------------===// |
| 1754 | |
| 1755 | void transform::detail::getParamProducerTransformOpTraitEffects( |
| 1756 | Operation *op, SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 1757 | producesHandle(handles: op->getResults(), effects); |
| 1758 | bool hasPayloadOperands = false; |
| 1759 | for (OpOperand &operand : op->getOpOperands()) { |
| 1760 | onlyReadsHandle(handles: operand, effects); |
| 1761 | if (llvm::isa<TransformHandleTypeInterface, |
| 1762 | TransformValueHandleTypeInterface>(Val: operand.get().getType())) |
| 1763 | hasPayloadOperands = true; |
| 1764 | } |
| 1765 | if (hasPayloadOperands) |
| 1766 | onlyReadsPayload(effects); |
| 1767 | } |
| 1768 | |
| 1769 | LogicalResult |
| 1770 | transform::detail::verifyParamProducerTransformOpTrait(Operation *op) { |
| 1771 | // Interfaces can be attached dynamically, so this cannot be a static |
| 1772 | // assert. |
| 1773 | if (!op->getName().getInterface<MemoryEffectOpInterface>()) { |
| 1774 | llvm::report_fatal_error( |
| 1775 | reason: Twine("ParamProducerTransformOpTrait must be attached to an op that " |
| 1776 | "implements MemoryEffectsOpInterface, found on " ) + |
| 1777 | op->getName().getStringRef()); |
| 1778 | } |
| 1779 | for (Value result : op->getResults()) { |
| 1780 | if (llvm::isa<TransformParamTypeInterface>(Val: result.getType())) |
| 1781 | continue; |
| 1782 | return op->emitOpError() |
| 1783 | << "ParamProducerTransformOpTrait attached to this op expects " |
| 1784 | "result types to implement TransformParamTypeInterface" ; |
| 1785 | } |
| 1786 | return success(); |
| 1787 | } |
| 1788 | |
| 1789 | //===----------------------------------------------------------------------===// |
| 1790 | // Memory effects. |
| 1791 | //===----------------------------------------------------------------------===// |
| 1792 | |
| 1793 | void transform::consumesHandle( |
| 1794 | MutableArrayRef<OpOperand> handles, |
| 1795 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 1796 | for (OpOperand &handle : handles) { |
| 1797 | effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &handle, |
| 1798 | Args: TransformMappingResource::get()); |
| 1799 | effects.emplace_back(Args: MemoryEffects::Free::get(), Args: &handle, |
| 1800 | Args: TransformMappingResource::get()); |
| 1801 | } |
| 1802 | } |
| 1803 | |
| 1804 | /// Returns `true` if the given list of effects instances contains an instance |
| 1805 | /// with the effect type specified as template parameter. |
| 1806 | template <typename EffectTy, typename ResourceTy, typename Range> |
| 1807 | static bool hasEffect(Range &&effects) { |
| 1808 | return llvm::any_of(effects, [](const MemoryEffects::EffectInstance &effect) { |
| 1809 | return isa<EffectTy>(effect.getEffect()) && |
| 1810 | isa<ResourceTy>(effect.getResource()); |
| 1811 | }); |
| 1812 | } |
| 1813 | |
| 1814 | bool transform::isHandleConsumed(Value handle, |
| 1815 | transform::TransformOpInterface transform) { |
| 1816 | auto iface = cast<MemoryEffectOpInterface>(Val: transform.getOperation()); |
| 1817 | SmallVector<MemoryEffects::EffectInstance> effects; |
| 1818 | iface.getEffectsOnValue(value: handle, effects); |
| 1819 | return ::hasEffect<MemoryEffects::Read, TransformMappingResource>(effects) && |
| 1820 | ::hasEffect<MemoryEffects::Free, TransformMappingResource>(effects); |
| 1821 | } |
| 1822 | |
| 1823 | void transform::producesHandle( |
| 1824 | ResultRange handles, |
| 1825 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 1826 | for (OpResult handle : handles) { |
| 1827 | effects.emplace_back(Args: MemoryEffects::Allocate::get(), Args&: handle, |
| 1828 | Args: TransformMappingResource::get()); |
| 1829 | effects.emplace_back(Args: MemoryEffects::Write::get(), Args&: handle, |
| 1830 | Args: TransformMappingResource::get()); |
| 1831 | } |
| 1832 | } |
| 1833 | |
| 1834 | void transform::producesHandle( |
| 1835 | MutableArrayRef<BlockArgument> handles, |
| 1836 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 1837 | for (BlockArgument handle : handles) { |
| 1838 | effects.emplace_back(Args: MemoryEffects::Allocate::get(), Args&: handle, |
| 1839 | Args: TransformMappingResource::get()); |
| 1840 | effects.emplace_back(Args: MemoryEffects::Write::get(), Args&: handle, |
| 1841 | Args: TransformMappingResource::get()); |
| 1842 | } |
| 1843 | } |
| 1844 | |
| 1845 | void transform::onlyReadsHandle( |
| 1846 | MutableArrayRef<OpOperand> handles, |
| 1847 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 1848 | for (OpOperand &handle : handles) { |
| 1849 | effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &handle, |
| 1850 | Args: TransformMappingResource::get()); |
| 1851 | } |
| 1852 | } |
| 1853 | |
| 1854 | void transform::modifiesPayload( |
| 1855 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 1856 | effects.emplace_back(Args: MemoryEffects::Read::get(), Args: PayloadIRResource::get()); |
| 1857 | effects.emplace_back(Args: MemoryEffects::Write::get(), Args: PayloadIRResource::get()); |
| 1858 | } |
| 1859 | |
| 1860 | void transform::onlyReadsPayload( |
| 1861 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 1862 | effects.emplace_back(Args: MemoryEffects::Read::get(), Args: PayloadIRResource::get()); |
| 1863 | } |
| 1864 | |
| 1865 | bool transform::doesModifyPayload(transform::TransformOpInterface transform) { |
| 1866 | auto iface = cast<MemoryEffectOpInterface>(Val: transform.getOperation()); |
| 1867 | SmallVector<MemoryEffects::EffectInstance> effects; |
| 1868 | iface.getEffects(effects); |
| 1869 | return ::hasEffect<MemoryEffects::Write, PayloadIRResource>(effects); |
| 1870 | } |
| 1871 | |
| 1872 | bool transform::doesReadPayload(transform::TransformOpInterface transform) { |
| 1873 | auto iface = cast<MemoryEffectOpInterface>(Val: transform.getOperation()); |
| 1874 | SmallVector<MemoryEffects::EffectInstance> effects; |
| 1875 | iface.getEffects(effects); |
| 1876 | return ::hasEffect<MemoryEffects::Read, PayloadIRResource>(effects); |
| 1877 | } |
| 1878 | |
| 1879 | void transform::getConsumedBlockArguments( |
| 1880 | Block &block, llvm::SmallDenseSet<unsigned int> &consumedArguments) { |
| 1881 | SmallVector<MemoryEffects::EffectInstance> effects; |
| 1882 | for (Operation &nested : block) { |
| 1883 | auto iface = dyn_cast<MemoryEffectOpInterface>(Val&: nested); |
| 1884 | if (!iface) |
| 1885 | continue; |
| 1886 | |
| 1887 | effects.clear(); |
| 1888 | iface.getEffects(effects); |
| 1889 | for (const MemoryEffects::EffectInstance &effect : effects) { |
| 1890 | BlockArgument argument = |
| 1891 | dyn_cast_or_null<BlockArgument>(Val: effect.getValue()); |
| 1892 | if (!argument || argument.getOwner() != &block || |
| 1893 | !isa<MemoryEffects::Free>(Val: effect.getEffect()) || |
| 1894 | effect.getResource() != transform::TransformMappingResource::get()) { |
| 1895 | continue; |
| 1896 | } |
| 1897 | consumedArguments.insert(V: argument.getArgNumber()); |
| 1898 | } |
| 1899 | } |
| 1900 | } |
| 1901 | |
| 1902 | //===----------------------------------------------------------------------===// |
| 1903 | // Utilities for TransformOpInterface. |
| 1904 | //===----------------------------------------------------------------------===// |
| 1905 | |
| 1906 | SmallVector<OpOperand *> transform::detail::getConsumedHandleOpOperands( |
| 1907 | TransformOpInterface transformOp) { |
| 1908 | SmallVector<OpOperand *> consumedOperands; |
| 1909 | consumedOperands.reserve(N: transformOp->getNumOperands()); |
| 1910 | auto memEffectInterface = |
| 1911 | cast<MemoryEffectOpInterface>(Val: transformOp.getOperation()); |
| 1912 | SmallVector<MemoryEffects::EffectInstance, 2> effects; |
| 1913 | for (OpOperand &target : transformOp->getOpOperands()) { |
| 1914 | effects.clear(); |
| 1915 | memEffectInterface.getEffectsOnValue(value: target.get(), effects); |
| 1916 | if (llvm::any_of(Range&: effects, P: [](const MemoryEffects::EffectInstance &effect) { |
| 1917 | return isa<transform::TransformMappingResource>( |
| 1918 | Val: effect.getResource()) && |
| 1919 | isa<MemoryEffects::Free>(Val: effect.getEffect()); |
| 1920 | })) { |
| 1921 | consumedOperands.push_back(Elt: &target); |
| 1922 | } |
| 1923 | } |
| 1924 | return consumedOperands; |
| 1925 | } |
| 1926 | |
| 1927 | LogicalResult transform::detail::verifyTransformOpInterface(Operation *op) { |
| 1928 | auto iface = cast<MemoryEffectOpInterface>(Val: op); |
| 1929 | SmallVector<MemoryEffects::EffectInstance> effects; |
| 1930 | iface.getEffects(effects); |
| 1931 | |
| 1932 | auto effectsOn = [&](Value value) { |
| 1933 | return llvm::make_filter_range( |
| 1934 | Range&: effects, Pred: [value](const MemoryEffects::EffectInstance &instance) { |
| 1935 | return instance.getValue() == value; |
| 1936 | }); |
| 1937 | }; |
| 1938 | |
| 1939 | std::optional<unsigned> firstConsumedOperand; |
| 1940 | for (OpOperand &operand : op->getOpOperands()) { |
| 1941 | auto range = effectsOn(operand.get()); |
| 1942 | if (range.empty()) { |
| 1943 | InFlightDiagnostic diag = |
| 1944 | op->emitError() << "TransformOpInterface requires memory effects " |
| 1945 | "on operands to be specified" ; |
| 1946 | diag.attachNote() << "no effects specified for operand #" |
| 1947 | << operand.getOperandNumber(); |
| 1948 | return diag; |
| 1949 | } |
| 1950 | if (::hasEffect<MemoryEffects::Allocate, TransformMappingResource>(effects&: range)) { |
| 1951 | InFlightDiagnostic diag = op->emitError() |
| 1952 | << "TransformOpInterface did not expect " |
| 1953 | "'allocate' memory effect on an operand" ; |
| 1954 | diag.attachNote() << "specified for operand #" |
| 1955 | << operand.getOperandNumber(); |
| 1956 | return diag; |
| 1957 | } |
| 1958 | if (!firstConsumedOperand && |
| 1959 | ::hasEffect<MemoryEffects::Free, TransformMappingResource>(effects&: range)) { |
| 1960 | firstConsumedOperand = operand.getOperandNumber(); |
| 1961 | } |
| 1962 | } |
| 1963 | |
| 1964 | if (firstConsumedOperand && |
| 1965 | !::hasEffect<MemoryEffects::Write, PayloadIRResource>(effects)) { |
| 1966 | InFlightDiagnostic diag = |
| 1967 | op->emitError() |
| 1968 | << "TransformOpInterface expects ops consuming operands to have a " |
| 1969 | "'write' effect on the payload resource" ; |
| 1970 | diag.attachNote() << "consumes operand #" << *firstConsumedOperand; |
| 1971 | return diag; |
| 1972 | } |
| 1973 | |
| 1974 | for (OpResult result : op->getResults()) { |
| 1975 | auto range = effectsOn(result); |
| 1976 | if (!::hasEffect<MemoryEffects::Allocate, TransformMappingResource>( |
| 1977 | effects&: range)) { |
| 1978 | InFlightDiagnostic diag = |
| 1979 | op->emitError() << "TransformOpInterface requires 'allocate' memory " |
| 1980 | "effect to be specified for results" ; |
| 1981 | diag.attachNote() << "no 'allocate' effect specified for result #" |
| 1982 | << result.getResultNumber(); |
| 1983 | return diag; |
| 1984 | } |
| 1985 | } |
| 1986 | |
| 1987 | return success(); |
| 1988 | } |
| 1989 | |
| 1990 | //===----------------------------------------------------------------------===// |
| 1991 | // Normal form utilities. |
| 1992 | //===----------------------------------------------------------------------===// |
| 1993 | |
| 1994 | DiagnosedSilenceableFailure transform::detail::checkNormalForms( |
| 1995 | ArrayRef<NormalFormAttrInterface> normalForms, |
| 1996 | ArrayRef<Operation *> payload) { |
| 1997 | // Return any definite failure or the first silenceable failure. |
| 1998 | auto overallResult = DiagnosedSilenceableFailure::success(); |
| 1999 | for (Operation *op : payload) { |
| 2000 | for (NormalFormAttrInterface normalForm : normalForms) { |
| 2001 | DiagnosedSilenceableFailure result = normalForm.checkOperation(op); |
| 2002 | if (result.isDefiniteFailure()) |
| 2003 | return result; |
| 2004 | if (result.isSilenceableFailure() && overallResult.succeeded()) |
| 2005 | overallResult = std::move(result); |
| 2006 | } |
| 2007 | } |
| 2008 | return overallResult; |
| 2009 | } |
| 2010 | |
| 2011 | LogicalResult transform::detail::verifyNormalFormList( |
| 2012 | function_ref<InFlightDiagnostic()> emitError, |
| 2013 | ArrayRef<NormalFormAttrInterface> normalForms) { |
| 2014 | llvm::DenseMap<TypeID, NormalFormAttrInterface> seen; |
| 2015 | for (NormalFormAttrInterface normalForm : normalForms) { |
| 2016 | auto [previous, inserted] = |
| 2017 | seen.try_emplace(Key: normalForm.getTypeID(), Args&: normalForm); |
| 2018 | if (!inserted) { |
| 2019 | InFlightDiagnostic diag = emitError() |
| 2020 | << "duplicate normal form: " << normalForm; |
| 2021 | diag.attachNote() << "previous instance: " << previous->second; |
| 2022 | return diag; |
| 2023 | } |
| 2024 | } |
| 2025 | |
| 2026 | return success(); |
| 2027 | } |
| 2028 | |
| 2029 | //===----------------------------------------------------------------------===// |
| 2030 | // Entry point. |
| 2031 | //===----------------------------------------------------------------------===// |
| 2032 | |
| 2033 | LogicalResult transform::applyTransforms( |
| 2034 | Operation *payloadRoot, TransformOpInterface transform, |
| 2035 | const RaggedArray<MappedValue> &, |
| 2036 | const TransformOptions &options, bool enforceToplevelTransformOp, |
| 2037 | function_ref<void(TransformState &)> stateInitializer, |
| 2038 | function_ref<LogicalResult(TransformState &)> stateExporter) { |
| 2039 | if (enforceToplevelTransformOp) { |
| 2040 | if (!transform->hasTrait<PossibleTopLevelTransformOpTrait>() || |
| 2041 | transform->getNumOperands() != 0) { |
| 2042 | return transform->emitError() |
| 2043 | << "expected transform to start at the top-level transform op" ; |
| 2044 | } |
| 2045 | } else if (failed( |
| 2046 | Result: detail::verifyPossibleTopLevelTransformOpTrait(op: transform))) { |
| 2047 | return failure(); |
| 2048 | } |
| 2049 | |
| 2050 | TransformState state(transform->getParentRegion(), payloadRoot, extraMapping, |
| 2051 | options); |
| 2052 | if (stateInitializer) |
| 2053 | stateInitializer(state); |
| 2054 | if (state.applyTransform(transform).checkAndReport().failed()) |
| 2055 | return failure(); |
| 2056 | if (stateExporter) |
| 2057 | return stateExporter(state); |
| 2058 | return success(); |
| 2059 | } |
| 2060 | |
| 2061 | //===----------------------------------------------------------------------===// |
| 2062 | // Generated interface implementation. |
| 2063 | //===----------------------------------------------------------------------===// |
| 2064 | |
| 2065 | #include "mlir/Dialect/Transform/Interfaces/TransformAttrInterfaces.cpp.inc" |
| 2066 | #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.cpp.inc" |
| 2067 | #include "mlir/Dialect/Transform/Interfaces/TransformTypeInterfaces.cpp.inc" |
| 2068 | |