1//===- TransformOps.cpp - Transform dialect operations --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Transform/IR/TransformOps.h"
10
11#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
12#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
13#include "mlir/Dialect/Transform/IR/TransformAttrs.h"
14#include "mlir/Dialect/Transform/IR/TransformDialect.h"
15#include "mlir/Dialect/Transform/IR/TransformTypes.h"
16#include "mlir/Dialect/Transform/Interfaces/MatchInterfaces.h"
17#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
18#include "mlir/IR/BuiltinAttributes.h"
19#include "mlir/IR/Diagnostics.h"
20#include "mlir/IR/Dominance.h"
21#include "mlir/IR/OpImplementation.h"
22#include "mlir/IR/OperationSupport.h"
23#include "mlir/IR/PatternMatch.h"
24#include "mlir/IR/Verifier.h"
25#include "mlir/Interfaces/ControlFlowInterfaces.h"
26#include "mlir/Interfaces/FunctionImplementation.h"
27#include "mlir/Interfaces/FunctionInterfaces.h"
28#include "mlir/Pass/PassManager.h"
29#include "mlir/Pass/PassRegistry.h"
30#include "mlir/Transforms/CSE.h"
31#include "mlir/Transforms/DialectConversion.h"
32#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
33#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
34#include "mlir/Transforms/RegionUtils.h"
35#include "llvm/ADT/DenseSet.h"
36#include "llvm/ADT/STLExtras.h"
37#include "llvm/ADT/ScopeExit.h"
38#include "llvm/ADT/SmallPtrSet.h"
39#include "llvm/ADT/SmallVectorExtras.h"
40#include "llvm/ADT/TypeSwitch.h"
41#include "llvm/Support/Debug.h"
42#include "llvm/Support/DebugLog.h"
43#include "llvm/Support/ErrorHandling.h"
44#include "llvm/Support/InterleavedRange.h"
45#include <optional>
46
47#define DEBUG_TYPE "transform-dialect"
48#define DEBUG_TYPE_MATCHER "transform-matcher"
49
50using namespace mlir;
51
52static ParseResult parseApplyRegisteredPassOptions(
53 OpAsmParser &parser, DictionaryAttr &options,
54 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &dynamicOptions);
55static void printApplyRegisteredPassOptions(OpAsmPrinter &printer,
56 Operation *op,
57 DictionaryAttr options,
58 ValueRange dynamicOptions);
59static ParseResult parseSequenceOpOperands(
60 OpAsmParser &parser, std::optional<OpAsmParser::UnresolvedOperand> &root,
61 Type &rootType,
62 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &extraBindings,
63 SmallVectorImpl<Type> &extraBindingTypes);
64static void printSequenceOpOperands(OpAsmPrinter &printer, Operation *op,
65 Value root, Type rootType,
66 ValueRange extraBindings,
67 TypeRange extraBindingTypes);
68static void printForeachMatchSymbols(OpAsmPrinter &printer, Operation *op,
69 ArrayAttr matchers, ArrayAttr actions);
70static ParseResult parseForeachMatchSymbols(OpAsmParser &parser,
71 ArrayAttr &matchers,
72 ArrayAttr &actions);
73
74/// Helper function to check if the given transform op is contained in (or
75/// equal to) the given payload target op. In that case, an error is returned.
76/// Transforming transform IR that is currently executing is generally unsafe.
77static DiagnosedSilenceableFailure
78ensurePayloadIsSeparateFromTransform(transform::TransformOpInterface transform,
79 Operation *payload) {
80 Operation *transformAncestor = transform.getOperation();
81 while (transformAncestor) {
82 if (transformAncestor == payload) {
83 DiagnosedDefiniteFailure diag =
84 transform.emitDefiniteFailure()
85 << "cannot apply transform to itself (or one of its ancestors)";
86 diag.attachNote(loc: payload->getLoc()) << "target payload op";
87 return diag;
88 }
89 transformAncestor = transformAncestor->getParentOp();
90 }
91 return DiagnosedSilenceableFailure::success();
92}
93
94#define GET_OP_CLASSES
95#include "mlir/Dialect/Transform/IR/TransformOps.cpp.inc"
96
97//===----------------------------------------------------------------------===//
98// AlternativesOp
99//===----------------------------------------------------------------------===//
100
101OperandRange transform::AlternativesOp::getEntrySuccessorOperands(
102 RegionSuccessor successor) {
103 if (!successor.isOperation() && getOperation()->getNumOperands() == 1)
104 return getOperation()->getOperands();
105 return OperandRange(getOperation()->operand_end(),
106 getOperation()->operand_end());
107}
108
109void transform::AlternativesOp::getSuccessorRegions(
110 RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &regions) {
111 for (Region &alternative : llvm::drop_begin(
112 RangeOrContainer: getAlternatives(), N: point.isParent()
113 ? 0
114 : point.getTerminatorPredecessorOrNull()
115 ->getParentRegion()
116 ->getRegionNumber() +
117 1)) {
118 regions.emplace_back(Args: &alternative);
119 }
120 if (!point.isParent())
121 regions.push_back(Elt: RegionSuccessor(getOperation()));
122}
123
124ValueRange
125transform::AlternativesOp::getSuccessorInputs(RegionSuccessor successor) {
126 if (successor.isOperation())
127 return getOperation()->getResults();
128 return successor.getSuccessor()->getArguments();
129}
130
131void transform::AlternativesOp::getRegionInvocationBounds(
132 ArrayRef<Attribute> operands, SmallVectorImpl<InvocationBounds> &bounds) {
133 (void)operands;
134 // The region corresponding to the first alternative is always executed, the
135 // remaining may or may not be executed.
136 bounds.reserve(N: getNumRegions());
137 bounds.emplace_back(Args: 1, Args: 1);
138 bounds.resize(N: getNumRegions(), NV: InvocationBounds(0, 1));
139}
140
141static void forwardEmptyOperands(Block *block, transform::TransformState &state,
142 transform::TransformResults &results) {
143 for (const auto &res : block->getParentOp()->getOpResults())
144 results.set(value: res, ops: {});
145}
146
147DiagnosedSilenceableFailure
148transform::AlternativesOp::apply(transform::TransformRewriter &rewriter,
149 transform::TransformResults &results,
150 transform::TransformState &state) {
151 SmallVector<Operation *> originals;
152 if (Value scopeHandle = getScope())
153 llvm::append_range(C&: originals, R: state.getPayloadOps(value: scopeHandle));
154 else
155 originals.push_back(Elt: state.getTopLevel());
156
157 for (Operation *original : originals) {
158 if (original->isAncestor(other: getOperation())) {
159 auto diag = emitDefiniteFailure()
160 << "scope must not contain the transforms being applied";
161 diag.attachNote(loc: original->getLoc()) << "scope";
162 return diag;
163 }
164 if (!original->hasTrait<OpTrait::IsIsolatedFromAbove>()) {
165 auto diag = emitDefiniteFailure()
166 << "only isolated-from-above ops can be alternative scopes";
167 diag.attachNote(loc: original->getLoc()) << "scope";
168 return diag;
169 }
170 }
171
172 for (Region &reg : getAlternatives()) {
173 // Clone the scope operations and make the transforms in this alternative
174 // region apply to them by virtue of mapping the block argument (the only
175 // visible handle) to the cloned scope operations. This effectively prevents
176 // the transformation from accessing any IR outside the scope.
177 auto scope = state.make_region_scope(region&: reg);
178 auto clones = llvm::map_to_vector(
179 C&: originals, F: [](Operation *op) { return op->clone(); });
180 llvm::scope_exit deleteClones([&] {
181 for (Operation *clone : clones)
182 clone->erase();
183 });
184 if (failed(Result: state.mapBlockArguments(argument: reg.front().getArgument(i: 0), operations: clones)))
185 return DiagnosedSilenceableFailure::definiteFailure();
186
187 bool failed = false;
188 for (Operation &transform : reg.front().without_terminator()) {
189 DiagnosedSilenceableFailure result =
190 state.applyTransform(transform: cast<TransformOpInterface>(Val&: transform));
191 if (result.isSilenceableFailure()) {
192 LDBG() << "alternative failed: " << result.getMessage();
193 failed = true;
194 break;
195 }
196
197 if (::mlir::failed(Result: result.silence()))
198 return DiagnosedSilenceableFailure::definiteFailure();
199 }
200
201 // If all operations in the given alternative succeeded, no need to consider
202 // the rest. Replace the original scoping operation with the clone on which
203 // the transformations were performed.
204 if (!failed) {
205 // We will be using the clones, so cancel their scheduled deletion.
206 deleteClones.release();
207 TrackingListener listener(state, *this);
208 IRRewriter rewriter(getContext(), &listener);
209 for (const auto &kvp : llvm::zip(t&: originals, u&: clones)) {
210 Operation *original = std::get<0>(t: kvp);
211 Operation *clone = std::get<1>(t: kvp);
212 original->getBlock()->getOperations().insert(where: original->getIterator(),
213 New: clone);
214 rewriter.replaceOp(op: original, newValues: clone->getResults());
215 }
216 detail::forwardTerminatorOperands(block: &reg.front(), state, results);
217 return DiagnosedSilenceableFailure::success();
218 }
219 }
220 return emitSilenceableError() << "all alternatives failed";
221}
222
223void transform::AlternativesOp::getEffects(
224 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
225 consumesHandle(handles: getOperation()->getOpOperands(), effects);
226 producesHandle(handles: getOperation()->getOpResults(), effects);
227 for (Region *region : getRegions()) {
228 if (!region->empty())
229 producesHandle(handles: region->front().getArguments(), effects);
230 }
231 modifiesPayload(effects);
232}
233
234LogicalResult transform::AlternativesOp::verify() {
235 for (Region &alternative : getAlternatives()) {
236 Block &block = alternative.front();
237 Operation *terminator = block.getTerminator();
238 if (terminator->getOperands().getTypes() != getResults().getTypes()) {
239 InFlightDiagnostic diag = emitOpError()
240 << "expects terminator operands to have the "
241 "same type as results of the operation";
242 diag.attachNote(noteLoc: terminator->getLoc()) << "terminator";
243 return diag;
244 }
245 }
246
247 return success();
248}
249
250//===----------------------------------------------------------------------===//
251// AnnotateOp
252//===----------------------------------------------------------------------===//
253
254DiagnosedSilenceableFailure
255transform::AnnotateOp::apply(transform::TransformRewriter &rewriter,
256 transform::TransformResults &results,
257 transform::TransformState &state) {
258 SmallVector<Operation *> targets =
259 llvm::to_vector(Range: state.getPayloadOps(value: getTarget()));
260
261 Attribute attr = UnitAttr::get(context: getContext());
262 if (auto paramH = getParam()) {
263 ArrayRef<Attribute> params = state.getParams(value: paramH);
264 if (params.size() != 1) {
265 if (targets.size() != params.size()) {
266 return emitSilenceableError()
267 << "parameter and target have different payload lengths ("
268 << params.size() << " vs " << targets.size() << ")";
269 }
270 for (auto &&[target, attr] : llvm::zip_equal(t&: targets, u&: params))
271 target->setAttr(name: getName(), value: attr);
272 return DiagnosedSilenceableFailure::success();
273 }
274 attr = params[0];
275 }
276 for (auto *target : targets)
277 target->setAttr(name: getName(), value: attr);
278 return DiagnosedSilenceableFailure::success();
279}
280
281void transform::AnnotateOp::getEffects(
282 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
283 onlyReadsHandle(handles: getTargetMutable(), effects);
284 onlyReadsHandle(handles: getParamMutable(), effects);
285 modifiesPayload(effects);
286}
287
288//===----------------------------------------------------------------------===//
289// ApplyCommonSubexpressionEliminationOp
290//===----------------------------------------------------------------------===//
291
292DiagnosedSilenceableFailure
293transform::ApplyCommonSubexpressionEliminationOp::applyToOne(
294 transform::TransformRewriter &rewriter, Operation *target,
295 ApplyToEachResultList &results, transform::TransformState &state) {
296 // Make sure that this transform is not applied to itself. Modifying the
297 // transform IR while it is being interpreted is generally dangerous.
298 DiagnosedSilenceableFailure payloadCheck =
299 ensurePayloadIsSeparateFromTransform(transform: *this, payload: target);
300 if (!payloadCheck.succeeded())
301 return payloadCheck;
302
303 DominanceInfo domInfo;
304 mlir::eliminateCommonSubExpressions(rewriter, domInfo, op: target);
305 return DiagnosedSilenceableFailure::success();
306}
307
308void transform::ApplyCommonSubexpressionEliminationOp::getEffects(
309 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
310 transform::onlyReadsHandle(handles: getTargetMutable(), effects);
311 transform::modifiesPayload(effects);
312}
313
314//===----------------------------------------------------------------------===//
315// ApplyDeadCodeEliminationOp
316//===----------------------------------------------------------------------===//
317
318DiagnosedSilenceableFailure transform::ApplyDeadCodeEliminationOp::applyToOne(
319 transform::TransformRewriter &rewriter, Operation *target,
320 ApplyToEachResultList &results, transform::TransformState &state) {
321 // Make sure that this transform is not applied to itself. Modifying the
322 // transform IR while it is being interpreted is generally dangerous.
323 DiagnosedSilenceableFailure payloadCheck =
324 ensurePayloadIsSeparateFromTransform(transform: *this, payload: target);
325 if (!payloadCheck.succeeded())
326 return payloadCheck;
327
328 for (Region &region : target->getRegions())
329 eliminateTriviallyDeadOps(rewriter, region);
330
331 return DiagnosedSilenceableFailure::success();
332}
333
334void transform::ApplyDeadCodeEliminationOp::getEffects(
335 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
336 transform::onlyReadsHandle(handles: getTargetMutable(), effects);
337 transform::modifiesPayload(effects);
338}
339
340//===----------------------------------------------------------------------===//
341// ApplyPatternsOp
342//===----------------------------------------------------------------------===//
343
344DiagnosedSilenceableFailure transform::ApplyPatternsOp::applyToOne(
345 transform::TransformRewriter &rewriter, Operation *target,
346 ApplyToEachResultList &results, transform::TransformState &state) {
347 // Make sure that this transform is not applied to itself. Modifying the
348 // transform IR while it is being interpreted is generally dangerous. Even
349 // more so for the ApplyPatternsOp because the GreedyPatternRewriteDriver
350 // performs many additional simplifications such as dead code elimination.
351 DiagnosedSilenceableFailure payloadCheck =
352 ensurePayloadIsSeparateFromTransform(transform: *this, payload: target);
353 if (!payloadCheck.succeeded())
354 return payloadCheck;
355
356 // Gather all specified patterns.
357 MLIRContext *ctx = target->getContext();
358 RewritePatternSet patterns(ctx);
359 if (!getRegion().empty()) {
360 for (Operation &op : getRegion().front()) {
361 cast<transform::PatternDescriptorOpInterface>(Val: &op)
362 .populatePatternsWithState(patterns, state);
363 }
364 }
365
366 // Configure the GreedyPatternRewriteDriver.
367 GreedyRewriteConfig config;
368 config.setListener(
369 static_cast<RewriterBase::Listener *>(rewriter.getListener()));
370 FrozenRewritePatternSet frozenPatterns(std::move(patterns));
371
372 config.setMaxIterations(getMaxIterations() == static_cast<uint64_t>(-1)
373 ? GreedyRewriteConfig::kNoLimit
374 : getMaxIterations());
375 config.setMaxNumRewrites(getMaxNumRewrites() == static_cast<uint64_t>(-1)
376 ? GreedyRewriteConfig::kNoLimit
377 : getMaxNumRewrites());
378
379 if (target->hasTrait<OpTrait::IsIsolatedFromAbove>()) {
380 // Op is isolated from above. The greedy driver iterates to a fixpoint
381 // internally and optionally runs full CSE between iterations.
382 config.enableCSEBetweenIterations(enable: getApplyCse());
383 if (failed(Result: applyPatternsGreedily(op: target, patterns: frozenPatterns, config))) {
384 return emitSilenceableFailure(op: target)
385 << "greedy pattern application failed";
386 }
387 return DiagnosedSilenceableFailure::success();
388 }
389
390 // Non-isolated case: gather the ops manually because the op-list
391 // GreedyPatternRewriteDriver overload only performs a single iteration and
392 // does not simplify regions. CSE is driven externally to reach a fixpoint.
393
394 // One or two iterations should be sufficient. Stop iterating after a certain
395 // threshold to make debugging easier.
396 static const int64_t kNumMaxIterations = 50;
397 int64_t iteration = 0;
398 bool cseChanged = false;
399 do {
400 SmallVector<Operation *> ops;
401 target->walk(callback: [&](Operation *nestedOp) {
402 if (target != nestedOp)
403 ops.push_back(Elt: nestedOp);
404 });
405
406 if (failed(Result: applyOpPatternsGreedily(ops, patterns: frozenPatterns, config))) {
407 return emitSilenceableFailure(op: target)
408 << "greedy pattern application failed";
409 }
410
411 if (getApplyCse()) {
412 DominanceInfo domInfo;
413 mlir::eliminateCommonSubExpressions(rewriter, domInfo, op: target,
414 changed: &cseChanged);
415 }
416 } while (cseChanged && ++iteration < kNumMaxIterations);
417
418 if (iteration == kNumMaxIterations)
419 return emitDefiniteFailure() << "fixpoint iteration did not converge";
420
421 return DiagnosedSilenceableFailure::success();
422}
423
424LogicalResult transform::ApplyPatternsOp::verify() {
425 if (!getRegion().empty()) {
426 for (Operation &op : getRegion().front()) {
427 if (!isa<transform::PatternDescriptorOpInterface>(Val: &op)) {
428 InFlightDiagnostic diag = emitOpError()
429 << "expected children ops to implement "
430 "PatternDescriptorOpInterface";
431 diag.attachNote(noteLoc: op.getLoc()) << "op without interface";
432 return diag;
433 }
434 }
435 }
436 return success();
437}
438
439void transform::ApplyPatternsOp::getEffects(
440 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
441 transform::onlyReadsHandle(handles: getTargetMutable(), effects);
442 transform::modifiesPayload(effects);
443}
444
445void transform::ApplyPatternsOp::build(
446 OpBuilder &builder, OperationState &result, Value target,
447 function_ref<void(OpBuilder &, Location)> bodyBuilder) {
448 result.addOperands(newOperands: target);
449
450 OpBuilder::InsertionGuard g(builder);
451 Region *region = result.addRegion();
452 builder.createBlock(parent: region);
453 if (bodyBuilder)
454 bodyBuilder(builder, result.location);
455}
456
457//===----------------------------------------------------------------------===//
458// ApplyCanonicalizationPatternsOp
459//===----------------------------------------------------------------------===//
460
461void transform::ApplyCanonicalizationPatternsOp::populatePatterns(
462 RewritePatternSet &patterns) {
463 MLIRContext *ctx = patterns.getContext();
464 for (Dialect *dialect : ctx->getLoadedDialects())
465 dialect->getCanonicalizationPatterns(results&: patterns);
466 for (RegisteredOperationName op : ctx->getRegisteredOperations())
467 op.getCanonicalizationPatterns(results&: patterns, context: ctx);
468}
469
470//===----------------------------------------------------------------------===//
471// ApplyConversionPatternsOp
472//===----------------------------------------------------------------------===//
473
474DiagnosedSilenceableFailure transform::ApplyConversionPatternsOp::apply(
475 transform::TransformRewriter &rewriter,
476 transform::TransformResults &results, transform::TransformState &state) {
477 MLIRContext *ctx = getContext();
478
479 // Instantiate the default type converter if a type converter builder is
480 // specified.
481 std::unique_ptr<TypeConverter> defaultTypeConverter;
482 transform::TypeConverterBuilderOpInterface typeConverterBuilder =
483 getDefaultTypeConverter();
484 if (typeConverterBuilder)
485 defaultTypeConverter = typeConverterBuilder.getTypeConverter();
486
487 // Configure conversion target.
488 ConversionTarget conversionTarget(*getContext());
489 if (getLegalOps())
490 for (Attribute attr : cast<ArrayAttr>(Val: *getLegalOps()))
491 conversionTarget.addLegalOp(
492 op: OperationName(cast<StringAttr>(Val&: attr).getValue(), ctx));
493 if (getIllegalOps())
494 for (Attribute attr : cast<ArrayAttr>(Val: *getIllegalOps()))
495 conversionTarget.addIllegalOp(
496 op: OperationName(cast<StringAttr>(Val&: attr).getValue(), ctx));
497 if (getLegalDialects())
498 for (Attribute attr : cast<ArrayAttr>(Val: *getLegalDialects()))
499 conversionTarget.addLegalDialect(name: cast<StringAttr>(Val&: attr).getValue());
500 if (getIllegalDialects())
501 for (Attribute attr : cast<ArrayAttr>(Val: *getIllegalDialects()))
502 conversionTarget.addIllegalDialect(name: cast<StringAttr>(Val&: attr).getValue());
503
504 // Gather all specified patterns.
505 RewritePatternSet patterns(ctx);
506 // Need to keep the converters alive until after pattern application because
507 // the patterns take a reference to an object that would otherwise get out of
508 // scope.
509 SmallVector<std::unique_ptr<TypeConverter>> keepAliveConverters;
510 if (!getPatterns().empty()) {
511 for (Operation &op : getPatterns().front()) {
512 auto descriptor =
513 cast<transform::ConversionPatternDescriptorOpInterface>(Val: &op);
514
515 // Check if this pattern set specifies a type converter.
516 std::unique_ptr<TypeConverter> typeConverter =
517 descriptor.getTypeConverter();
518 TypeConverter *converter = nullptr;
519 if (typeConverter) {
520 keepAliveConverters.emplace_back(Args: std::move(typeConverter));
521 converter = keepAliveConverters.back().get();
522 } else {
523 // No type converter specified: Use the default type converter.
524 if (!defaultTypeConverter) {
525 auto diag = emitDefiniteFailure()
526 << "pattern descriptor does not specify type "
527 "converter and apply_conversion_patterns op has "
528 "no default type converter";
529 diag.attachNote(loc: op.getLoc()) << "pattern descriptor op";
530 return diag;
531 }
532 converter = defaultTypeConverter.get();
533 }
534
535 // Add descriptor-specific updates to the conversion target, which may
536 // depend on the final type converter. In structural converters, the
537 // legality of types dictates the dynamic legality of an operation.
538 descriptor.populateConversionTargetRules(typeConverter: *converter, conversionTarget);
539
540 descriptor.populatePatterns(typeConverter&: *converter, patterns);
541 }
542 }
543
544 // Attach a tracking listener if handles should be preserved. We configure the
545 // listener to allow op replacements with different names, as conversion
546 // patterns typically replace ops with replacement ops that have a different
547 // name.
548 TrackingListenerConfig trackingConfig;
549 trackingConfig.requireMatchingReplacementOpName = false;
550 ErrorCheckingTrackingListener trackingListener(state, *this, trackingConfig);
551 ConversionConfig conversionConfig;
552 if (getPreserveHandles())
553 conversionConfig.listener = &trackingListener;
554
555 FrozenRewritePatternSet frozenPatterns(std::move(patterns));
556 for (Operation *target : state.getPayloadOps(value: getTarget())) {
557 // Make sure that this transform is not applied to itself. Modifying the
558 // transform IR while it is being interpreted is generally dangerous.
559 DiagnosedSilenceableFailure payloadCheck =
560 ensurePayloadIsSeparateFromTransform(transform: *this, payload: target);
561 if (!payloadCheck.succeeded())
562 return payloadCheck;
563
564 LogicalResult status = failure();
565 if (getPartialConversion()) {
566 status = applyPartialConversion(op: target, target: conversionTarget, patterns: frozenPatterns,
567 config: conversionConfig);
568 } else {
569 status = applyFullConversion(op: target, target: conversionTarget, patterns: frozenPatterns,
570 config: conversionConfig);
571 }
572
573 // Check dialect conversion state.
574 DiagnosedSilenceableFailure diag = DiagnosedSilenceableFailure::success();
575 if (failed(Result: status)) {
576 diag = emitSilenceableError() << "dialect conversion failed";
577 diag.attachNote(loc: target->getLoc()) << "target op";
578 }
579
580 // Check tracking listener error state.
581 DiagnosedSilenceableFailure trackingFailure =
582 trackingListener.checkAndResetError();
583 if (!trackingFailure.succeeded()) {
584 if (diag.succeeded()) {
585 // Tracking failure is the only failure.
586 return trackingFailure;
587 }
588 diag.attachNote() << "tracking listener also failed: "
589 << trackingFailure.getMessage();
590 (void)trackingFailure.silence();
591 }
592
593 if (!diag.succeeded())
594 return diag;
595 }
596
597 return DiagnosedSilenceableFailure::success();
598}
599
600LogicalResult transform::ApplyConversionPatternsOp::verify() {
601 if (getNumRegions() != 1 && getNumRegions() != 2)
602 return emitOpError() << "expected 1 or 2 regions";
603 if (!getPatterns().empty()) {
604 for (Operation &op : getPatterns().front()) {
605 if (!isa<transform::ConversionPatternDescriptorOpInterface>(Val: &op)) {
606 InFlightDiagnostic diag =
607 emitOpError() << "expected pattern children ops to implement "
608 "ConversionPatternDescriptorOpInterface";
609 diag.attachNote(noteLoc: op.getLoc()) << "op without interface";
610 return diag;
611 }
612 }
613 }
614 if (getNumRegions() == 2) {
615 Region &typeConverterRegion = getRegion(i: 1);
616 if (typeConverterRegion.empty() ||
617 !llvm::hasSingleElement(C&: typeConverterRegion.front()))
618 return emitOpError()
619 << "expected exactly one op in default type converter region";
620 Operation *maybeTypeConverter = &typeConverterRegion.front().front();
621 auto typeConverterOp = dyn_cast<transform::TypeConverterBuilderOpInterface>(
622 Val: maybeTypeConverter);
623 if (!typeConverterOp) {
624 InFlightDiagnostic diag = emitOpError()
625 << "expected default converter child op to "
626 "implement TypeConverterBuilderOpInterface";
627 diag.attachNote(noteLoc: maybeTypeConverter->getLoc()) << "op without interface";
628 return diag;
629 }
630 // Check default type converter type.
631 if (!getPatterns().empty()) {
632 for (Operation &op : getPatterns().front()) {
633 auto descriptor =
634 cast<transform::ConversionPatternDescriptorOpInterface>(Val: &op);
635 if (failed(Result: descriptor.verifyTypeConverter(builder: typeConverterOp)))
636 return failure();
637 }
638 }
639 }
640 return success();
641}
642
643void transform::ApplyConversionPatternsOp::getEffects(
644 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
645 if (!getPreserveHandles()) {
646 transform::consumesHandle(handles: getTargetMutable(), effects);
647 } else {
648 transform::onlyReadsHandle(handles: getTargetMutable(), effects);
649 }
650 transform::modifiesPayload(effects);
651}
652
653void transform::ApplyConversionPatternsOp::build(
654 OpBuilder &builder, OperationState &result, Value target,
655 function_ref<void(OpBuilder &, Location)> patternsBodyBuilder,
656 function_ref<void(OpBuilder &, Location)> typeConverterBodyBuilder) {
657 result.addOperands(newOperands: target);
658
659 {
660 OpBuilder::InsertionGuard g(builder);
661 Region *region1 = result.addRegion();
662 builder.createBlock(parent: region1);
663 if (patternsBodyBuilder)
664 patternsBodyBuilder(builder, result.location);
665 }
666 {
667 OpBuilder::InsertionGuard g(builder);
668 Region *region2 = result.addRegion();
669 builder.createBlock(parent: region2);
670 if (typeConverterBodyBuilder)
671 typeConverterBodyBuilder(builder, result.location);
672 }
673}
674
675//===----------------------------------------------------------------------===//
676// ApplyToLLVMConversionPatternsOp
677//===----------------------------------------------------------------------===//
678
679void transform::ApplyToLLVMConversionPatternsOp::populatePatterns(
680 TypeConverter &typeConverter, RewritePatternSet &patterns) {
681 Dialect *dialect = getContext()->getLoadedDialect(name: getDialectName());
682 assert(dialect && "expected that dialect is loaded");
683 auto *iface = cast<ConvertToLLVMPatternInterface>(Val: dialect);
684 // ConversionTarget is currently ignored because the enclosing
685 // apply_conversion_patterns op sets up its own ConversionTarget.
686 ConversionTarget target(*getContext());
687 iface->populateConvertToLLVMConversionPatterns(
688 target, typeConverter&: static_cast<LLVMTypeConverter &>(typeConverter), patterns);
689}
690
691LogicalResult transform::ApplyToLLVMConversionPatternsOp::verifyTypeConverter(
692 transform::TypeConverterBuilderOpInterface builder) {
693 if (builder.getTypeConverterType() != "LLVMTypeConverter")
694 return emitOpError(message: "expected LLVMTypeConverter");
695 return success();
696}
697
698LogicalResult transform::ApplyToLLVMConversionPatternsOp::verify() {
699 Dialect *dialect = getContext()->getLoadedDialect(name: getDialectName());
700 if (!dialect)
701 return emitOpError(message: "unknown dialect or dialect not loaded: ")
702 << getDialectName();
703 auto *iface = dyn_cast<ConvertToLLVMPatternInterface>(Val: dialect);
704 if (!iface)
705 return emitOpError(
706 message: "dialect does not implement ConvertToLLVMPatternInterface or "
707 "extension was not loaded: ")
708 << getDialectName();
709 return success();
710}
711
712//===----------------------------------------------------------------------===//
713// ApplyLoopInvariantCodeMotionOp
714//===----------------------------------------------------------------------===//
715
716DiagnosedSilenceableFailure
717transform::ApplyLoopInvariantCodeMotionOp::applyToOne(
718 transform::TransformRewriter &rewriter, LoopLikeOpInterface target,
719 transform::ApplyToEachResultList &results,
720 transform::TransformState &state) {
721 // Currently, LICM does not remove operations, so we don't need tracking.
722 // If this ever changes, add a LICM entry point that takes a rewriter.
723 moveLoopInvariantCode(loopLike: target);
724 return DiagnosedSilenceableFailure::success();
725}
726
727void transform::ApplyLoopInvariantCodeMotionOp::getEffects(
728 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
729 transform::onlyReadsHandle(handles: getTargetMutable(), effects);
730 transform::modifiesPayload(effects);
731}
732
733//===----------------------------------------------------------------------===//
734// ApplyRegisteredPassOp
735//===----------------------------------------------------------------------===//
736
737void transform::ApplyRegisteredPassOp::getEffects(
738 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
739 consumesHandle(handles: getTargetMutable(), effects);
740 onlyReadsHandle(handles: getDynamicOptionsMutable(), effects);
741 producesHandle(handles: getOperation()->getOpResults(), effects);
742 modifiesPayload(effects);
743}
744
745DiagnosedSilenceableFailure
746transform::ApplyRegisteredPassOp::apply(transform::TransformRewriter &rewriter,
747 transform::TransformResults &results,
748 transform::TransformState &state) {
749 // Obtain a single options-string to pass to the pass(-pipeline) from options
750 // passed in as a dictionary of keys mapping to values which are either
751 // attributes or param-operands pointing to attributes.
752 OperandRange dynamicOptions = getDynamicOptions();
753
754 std::string options;
755 llvm::raw_string_ostream optionsStream(options); // For "printing" attrs.
756
757 // A helper to convert an option's attribute value into a corresponding
758 // string representation, with the ability to obtain the attr(s) from a param.
759 std::function<void(Attribute)> appendValueAttr = [&](Attribute valueAttr) {
760 if (auto paramOperand = dyn_cast<transform::ParamOperandAttr>(Val&: valueAttr)) {
761 // The corresponding value attribute(s) is/are passed in via a param.
762 // Obtain the param-operand via its specified index.
763 int64_t dynamicOptionIdx = paramOperand.getIndex().getInt();
764 assert(dynamicOptionIdx < static_cast<int64_t>(dynamicOptions.size()) &&
765 "the number of ParamOperandAttrs in the options DictionaryAttr"
766 "should be the same as the number of options passed as params");
767 ArrayRef<Attribute> attrsAssociatedToParam =
768 state.getParams(value: dynamicOptions[dynamicOptionIdx]);
769 // Recursive so as to append all attrs associated to the param.
770 llvm::interleave(c: attrsAssociatedToParam, os&: optionsStream, each_fn: appendValueAttr,
771 separator: ",");
772 } else if (auto arrayAttr = dyn_cast<ArrayAttr>(Val&: valueAttr)) {
773 // Recursive so as to append all nested attrs of the array.
774 llvm::interleave(c: arrayAttr, os&: optionsStream, each_fn: appendValueAttr, separator: ",");
775 } else if (auto strAttr = dyn_cast<StringAttr>(Val&: valueAttr)) {
776 // Convert to unquoted string.
777 optionsStream << strAttr.getValue().str();
778 } else {
779 // For all other attributes, ask the attr to print itself (without type).
780 valueAttr.print(os&: optionsStream, /*elideType=*/true);
781 }
782 };
783
784 // Convert the options DictionaryAttr into a single string.
785 llvm::interleave(
786 c: getOptions(), os&: optionsStream,
787 each_fn: [&](auto namedAttribute) {
788 optionsStream << namedAttribute.getName().str(); // Append the key.
789 optionsStream << "="; // And the key-value separator.
790 appendValueAttr(namedAttribute.getValue()); // And the attr's str repr.
791 },
792 separator: " ");
793 optionsStream.flush();
794
795 // Get pass or pass pipeline from registry.
796 const PassRegistryEntry *info = PassPipelineInfo::lookup(pipelineArg: getPassName());
797 if (!info)
798 info = PassInfo::lookup(passArg: getPassName());
799 if (!info)
800 return emitDefiniteFailure()
801 << "unknown pass or pass pipeline: " << getPassName();
802
803 // Create pass manager and add the pass or pass pipeline.
804 PassManager pm(getContext());
805 if (failed(Result: info->addToPipeline(pm, options, errorHandler: [&](const Twine &msg) {
806 emitError(message: msg);
807 return failure();
808 }))) {
809 return emitDefiniteFailure()
810 << "failed to add pass or pass pipeline to pipeline: "
811 << getPassName();
812 }
813
814 auto targets = SmallVector<Operation *>(state.getPayloadOps(value: getTarget()));
815 for (Operation *target : targets) {
816 // Make sure that this transform is not applied to itself. Modifying the
817 // transform IR while it is being interpreted is generally dangerous. Even
818 // more so when applying passes because they may perform a wide range of IR
819 // modifications.
820 DiagnosedSilenceableFailure payloadCheck =
821 ensurePayloadIsSeparateFromTransform(transform: *this, payload: target);
822 if (!payloadCheck.succeeded())
823 return payloadCheck;
824
825 // Run the pass or pass pipeline on the current target operation.
826 if (failed(Result: pm.run(op: target))) {
827 auto diag = emitSilenceableError() << "pass pipeline failed";
828 diag.attachNote(loc: target->getLoc()) << "target op";
829 return diag;
830 }
831 }
832
833 // The applied pass will have directly modified the payload IR(s).
834 results.set(value: llvm::cast<OpResult>(Val: getResult()), ops&: targets);
835 return DiagnosedSilenceableFailure::success();
836}
837
838static ParseResult parseApplyRegisteredPassOptions(
839 OpAsmParser &parser, DictionaryAttr &options,
840 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &dynamicOptions) {
841 // Construct the options DictionaryAttr per a `{ key = value, ... }` syntax.
842 SmallVector<NamedAttribute> keyValuePairs;
843 size_t dynamicOptionsIdx = 0;
844
845 // Helper for allowing parsing of option values which can be of the form:
846 // - a normal attribute
847 // - an operand (which would be converted to an attr referring to the operand)
848 // - ArrayAttrs containing the foregoing (in correspondence with ListOptions)
849 std::function<ParseResult(Attribute &)> parseValue =
850 [&](Attribute &valueAttr) -> ParseResult {
851 // Allow for array syntax, e.g. `[0 : i64, %param, true, %other_param]`:
852 if (succeeded(Result: parser.parseOptionalLSquare())) {
853 SmallVector<Attribute> attrs;
854
855 // Recursively parse the array's elements, which might be operands.
856 if (parser.parseCommaSeparatedList(
857 delimiter: AsmParser::Delimiter::None,
858 parseElementFn: [&]() -> ParseResult { return parseValue(attrs.emplace_back()); },
859 contextMessage: " in options dictionary") ||
860 parser.parseRSquare())
861 return failure(); // NB: Attempted parse should've output error message.
862
863 valueAttr = ArrayAttr::get(context: parser.getContext(), value: attrs);
864
865 return success();
866 }
867
868 // Parse the value, which can be either an attribute or an operand.
869 OptionalParseResult parsedValueAttr =
870 parser.parseOptionalAttribute(result&: valueAttr);
871 if (!parsedValueAttr.has_value()) {
872 OpAsmParser::UnresolvedOperand operand;
873 ParseResult parsedOperand = parser.parseOperand(result&: operand);
874 if (failed(Result: parsedOperand))
875 return failure(); // NB: Attempted parse should've output error message.
876 // To make use of the operand, we need to store it in the options dict.
877 // As SSA-values cannot occur in attributes, what we do instead is store
878 // an attribute in its place that contains the index of the param-operand,
879 // so that an attr-value associated to the param can be resolved later on.
880 dynamicOptions.push_back(Elt: operand);
881 auto wrappedIndex = IntegerAttr::get(
882 type: IntegerType::get(context: parser.getContext(), width: 64), value: dynamicOptionsIdx++);
883 valueAttr =
884 transform::ParamOperandAttr::get(context: parser.getContext(), index: wrappedIndex);
885 } else if (failed(Result: parsedValueAttr.value())) {
886 return failure(); // NB: Attempted parse should have output error message.
887 } else if (isa<transform::ParamOperandAttr>(Val: valueAttr)) {
888 return parser.emitError(loc: parser.getCurrentLocation())
889 << "the param_operand attribute is a marker reserved for "
890 << "indicating a value will be passed via params and is only used "
891 << "in the generic print format";
892 }
893
894 return success();
895 };
896
897 // Helper for `key = value`-pair parsing where `key` is a bare identifier or a
898 // string and `value` looks like either an attribute or an operand-in-an-attr.
899 std::function<ParseResult()> parseKeyValuePair = [&]() -> ParseResult {
900 std::string key;
901 Attribute valueAttr;
902
903 if (failed(Result: parser.parseOptionalKeywordOrString(result: &key)) || key.empty())
904 return parser.emitError(loc: parser.getCurrentLocation())
905 << "expected key to either be an identifier or a string";
906
907 if (failed(Result: parser.parseEqual()))
908 return parser.emitError(loc: parser.getCurrentLocation())
909 << "expected '=' after key in key-value pair";
910
911 if (failed(Result: parseValue(valueAttr)))
912 return parser.emitError(loc: parser.getCurrentLocation())
913 << "expected a valid attribute or operand as value associated "
914 << "to key '" << key << "'";
915
916 keyValuePairs.push_back(Elt: NamedAttribute(key, valueAttr));
917
918 return success();
919 };
920
921 if (parser.parseCommaSeparatedList(delimiter: AsmParser::Delimiter::Braces,
922 parseElementFn: parseKeyValuePair,
923 contextMessage: " in options dictionary"))
924 return failure(); // NB: Attempted parse should have output error message.
925
926 if (DictionaryAttr::findDuplicate(
927 array&: keyValuePairs, /*isSorted=*/false) // Also sorts the keyValuePairs.
928 .has_value())
929 return parser.emitError(loc: parser.getCurrentLocation())
930 << "duplicate keys found in options dictionary";
931
932 options = DictionaryAttr::getWithSorted(context: parser.getContext(), value: keyValuePairs);
933
934 return success();
935}
936
937static void printApplyRegisteredPassOptions(OpAsmPrinter &printer,
938 Operation *op,
939 DictionaryAttr options,
940 ValueRange dynamicOptions) {
941 if (options.empty())
942 return;
943
944 std::function<void(Attribute)> printOptionValue = [&](Attribute valueAttr) {
945 if (auto paramOperandAttr =
946 dyn_cast<transform::ParamOperandAttr>(Val&: valueAttr)) {
947 // Resolve index of param-operand to its actual SSA-value and print that.
948 printer.printOperand(
949 value: dynamicOptions[paramOperandAttr.getIndex().getInt()]);
950 } else if (auto arrayAttr = dyn_cast<ArrayAttr>(Val&: valueAttr)) {
951 // This case is so that ArrayAttr-contained operands are pretty-printed.
952 printer << "[";
953 llvm::interleaveComma(c: arrayAttr, os&: printer, each_fn: printOptionValue);
954 printer << "]";
955 } else {
956 printer.printAttribute(attr: valueAttr);
957 }
958 };
959
960 printer << "{";
961 llvm::interleaveComma(c: options, os&: printer, each_fn: [&](NamedAttribute namedAttribute) {
962 printer << namedAttribute.getName();
963 printer << " = ";
964 printOptionValue(namedAttribute.getValue());
965 });
966 printer << "}";
967}
968
969LogicalResult transform::ApplyRegisteredPassOp::verify() {
970 // Check that there is a one-to-one correspondence between param operands
971 // and references to dynamic options in the options dictionary.
972
973 auto dynamicOptions = SmallVector<Value>(getDynamicOptions());
974
975 // Helper for option values to mark seen operands as having been seen (once).
976 std::function<LogicalResult(Attribute)> checkOptionValue =
977 [&](Attribute valueAttr) -> LogicalResult {
978 if (auto paramOperand = dyn_cast<transform::ParamOperandAttr>(Val&: valueAttr)) {
979 int64_t dynamicOptionIdx = paramOperand.getIndex().getInt();
980 if (dynamicOptionIdx < 0 ||
981 dynamicOptionIdx >= static_cast<int64_t>(dynamicOptions.size()))
982 return emitOpError()
983 << "dynamic option index " << dynamicOptionIdx
984 << " is out of bounds for the number of dynamic options: "
985 << dynamicOptions.size();
986 if (dynamicOptions[dynamicOptionIdx] == nullptr)
987 return emitOpError() << "dynamic option index " << dynamicOptionIdx
988 << " is already used in options";
989 dynamicOptions[dynamicOptionIdx] = nullptr; // Mark this option as used.
990 } else if (auto arrayAttr = dyn_cast<ArrayAttr>(Val&: valueAttr)) {
991 // Recurse into ArrayAttrs as they may contain references to operands.
992 for (auto eltAttr : arrayAttr)
993 if (failed(Result: checkOptionValue(eltAttr)))
994 return failure();
995 }
996 return success();
997 };
998
999 for (NamedAttribute namedAttr : getOptions())
1000 if (failed(Result: checkOptionValue(namedAttr.getValue())))
1001 return failure();
1002
1003 // All dynamicOptions-params seen in the dict will have been set to null.
1004 for (Value dynamicOption : dynamicOptions)
1005 if (dynamicOption)
1006 return emitOpError() << "a param operand does not have a corresponding "
1007 << "param_operand attr in the options dict";
1008
1009 return success();
1010}
1011
1012//===----------------------------------------------------------------------===//
1013// CastOp
1014//===----------------------------------------------------------------------===//
1015
1016DiagnosedSilenceableFailure
1017transform::CastOp::applyToOne(transform::TransformRewriter &rewriter,
1018 Operation *target, ApplyToEachResultList &results,
1019 transform::TransformState &state) {
1020 results.push_back(op: target);
1021 return DiagnosedSilenceableFailure::success();
1022}
1023
1024void transform::CastOp::getEffects(
1025 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
1026 onlyReadsPayload(effects);
1027 onlyReadsHandle(handles: getInputMutable(), effects);
1028 producesHandle(handles: getOperation()->getOpResults(), effects);
1029}
1030
1031bool transform::CastOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
1032 assert(inputs.size() == 1 && "expected one input");
1033 assert(outputs.size() == 1 && "expected one output");
1034 return llvm::all_of(
1035 Range: std::initializer_list<Type>{inputs.front(), outputs.front()},
1036 P: llvm::IsaPred<transform::TransformHandleTypeInterface>);
1037}
1038
1039//===----------------------------------------------------------------------===//
1040// CollectMatchingOp
1041//===----------------------------------------------------------------------===//
1042
1043/// Applies matcher operations from the given `block` using
1044/// `blockArgumentMapping` to initialize block arguments. Updates `state`
1045/// accordingly. If any of the matcher produces a silenceable failure, discards
1046/// it (printing the content to the debug output stream) and returns failure. If
1047/// any of the matchers produces a definite failure, reports it and returns
1048/// failure. If all matchers in the block succeed, populates `mappings` with the
1049/// payload entities associated with the block terminator operands. Note that
1050/// `mappings` will be cleared before that.
1051static DiagnosedSilenceableFailure
1052matchBlock(Block &block,
1053 ArrayRef<SmallVector<transform::MappedValue>> blockArgumentMapping,
1054 transform::TransformState &state,
1055 SmallVectorImpl<SmallVector<transform::MappedValue>> &mappings) {
1056 assert(block.getParent() && "cannot match using a detached block");
1057 auto matchScope = state.make_region_scope(region&: *block.getParent());
1058 if (failed(
1059 Result: state.mapBlockArguments(arguments: block.getArguments(), mapping: blockArgumentMapping)))
1060 return DiagnosedSilenceableFailure::definiteFailure();
1061
1062 for (Operation &match : block.without_terminator()) {
1063 if (!isa<transform::MatchOpInterface>(Val: match)) {
1064 return emitDefiniteFailure(loc: match.getLoc())
1065 << "expected operations in the match part to "
1066 "implement MatchOpInterface";
1067 }
1068 DiagnosedSilenceableFailure diag =
1069 state.applyTransform(transform: cast<transform::TransformOpInterface>(Val&: match));
1070 if (diag.succeeded())
1071 continue;
1072
1073 return diag;
1074 }
1075
1076 // Remember the values mapped to the terminator operands so we can
1077 // forward them to the action.
1078 ValueRange yieldedValues = block.getTerminator()->getOperands();
1079 // Our contract with the caller is that the mappings will contain only the
1080 // newly mapped values, clear the rest.
1081 mappings.clear();
1082 transform::detail::prepareValueMappings(mappings, values: yieldedValues, state);
1083 return DiagnosedSilenceableFailure::success();
1084}
1085
1086/// Returns `true` if both types implement one of the interfaces provided as
1087/// template parameters.
1088template <typename... Tys>
1089static bool implementSameInterface(Type t1, Type t2) {
1090 return ((isa<Tys>(t1) && isa<Tys>(t2)) || ... || false);
1091}
1092
1093/// Returns `true` if both types implement one of the transform dialect
1094/// interfaces.
1095static bool implementSameTransformInterface(Type t1, Type t2) {
1096 return implementSameInterface<transform::TransformHandleTypeInterface,
1097 transform::TransformParamTypeInterface,
1098 transform::TransformValueHandleTypeInterface>(
1099 t1, t2);
1100}
1101
1102//===----------------------------------------------------------------------===//
1103// CollectMatchingOp
1104//===----------------------------------------------------------------------===//
1105
1106DiagnosedSilenceableFailure
1107transform::CollectMatchingOp::apply(transform::TransformRewriter &rewriter,
1108 transform::TransformResults &results,
1109 transform::TransformState &state) {
1110 auto matcher = SymbolTable::lookupNearestSymbolFrom<FunctionOpInterface>(
1111 from: getOperation(), symbol: getMatcher());
1112 if (matcher.isExternal()) {
1113 return emitDefiniteFailure()
1114 << "unresolved external symbol " << getMatcher();
1115 }
1116
1117 SmallVector<SmallVector<MappedValue>, 2> rawResults;
1118 rawResults.resize(N: getOperation()->getNumResults());
1119 std::optional<DiagnosedSilenceableFailure> maybeFailure;
1120 for (Operation *root : state.getPayloadOps(value: getRoot())) {
1121 WalkResult walkResult = root->walk(callback: [&](Operation *op) {
1122 LDBG(DEBUG_TYPE_MATCHER, 1)
1123 << "matching "
1124 << OpWithFlags(op, OpPrintingFlags().assumeVerified().skipRegions())
1125 << " @" << op;
1126
1127 // Try matching.
1128 SmallVector<SmallVector<MappedValue>> mappings;
1129 SmallVector<transform::MappedValue> inputMapping({op});
1130 DiagnosedSilenceableFailure diag = matchBlock(
1131 block&: matcher.getFunctionBody().front(),
1132 blockArgumentMapping: ArrayRef<SmallVector<transform::MappedValue>>(inputMapping), state,
1133 mappings);
1134 if (diag.isDefiniteFailure())
1135 return WalkResult::interrupt();
1136 if (diag.isSilenceableFailure()) {
1137 LDBG(DEBUG_TYPE_MATCHER, 1) << "matcher " << matcher.getName()
1138 << " failed: " << diag.getMessage();
1139 return WalkResult::advance();
1140 }
1141
1142 // If succeeded, collect results.
1143 for (auto &&[i, mapping] : llvm::enumerate(First&: mappings)) {
1144 if (mapping.size() != 1) {
1145 maybeFailure.emplace(args: emitSilenceableError()
1146 << "result #" << i << ", associated with "
1147 << mapping.size()
1148 << " payload objects, expected 1");
1149 return WalkResult::interrupt();
1150 }
1151 rawResults[i].push_back(Elt: mapping[0]);
1152 }
1153 return WalkResult::advance();
1154 });
1155 if (walkResult.wasInterrupted())
1156 return std::move(*maybeFailure);
1157 assert(!maybeFailure && "failure set but the walk was not interrupted");
1158
1159 for (auto &&[opResult, rawResult] :
1160 llvm::zip_equal(t: getOperation()->getResults(), u&: rawResults)) {
1161 results.setMappedValues(handle: opResult, values: rawResult);
1162 }
1163 }
1164 return DiagnosedSilenceableFailure::success();
1165}
1166
1167void transform::CollectMatchingOp::getEffects(
1168 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
1169 onlyReadsHandle(handles: getRootMutable(), effects);
1170 producesHandle(handles: getOperation()->getOpResults(), effects);
1171 onlyReadsPayload(effects);
1172}
1173
1174LogicalResult transform::CollectMatchingOp::verifySymbolUses(
1175 SymbolTableCollection &symbolTable) {
1176 auto matcherSymbol = dyn_cast_or_null<FunctionOpInterface>(
1177 Val: symbolTable.lookupNearestSymbolFrom(from: getOperation(), symbol: getMatcher()));
1178 if (!matcherSymbol ||
1179 !isa<TransformOpInterface>(Val: matcherSymbol.getOperation()))
1180 return emitError() << "unresolved matcher symbol " << getMatcher();
1181
1182 ArrayRef<Type> argumentTypes = matcherSymbol.getArgumentTypes();
1183 if (argumentTypes.size() != 1 ||
1184 !isa<TransformHandleTypeInterface>(Val: argumentTypes[0])) {
1185 return emitError()
1186 << "expected the matcher to take one operation handle argument";
1187 }
1188 if (!matcherSymbol.getArgAttr(
1189 index: 0, name: transform::TransformDialect::kArgReadOnlyAttrName)) {
1190 return emitError() << "expected the matcher argument to be marked readonly";
1191 }
1192
1193 ArrayRef<Type> resultTypes = matcherSymbol.getResultTypes();
1194 if (resultTypes.size() != getOperation()->getNumResults()) {
1195 return emitError()
1196 << "expected the matcher to yield as many values as op has results ("
1197 << getOperation()->getNumResults() << "), got "
1198 << resultTypes.size();
1199 }
1200
1201 for (auto &&[i, matcherType, resultType] :
1202 llvm::enumerate(First&: resultTypes, Rest: getOperation()->getResultTypes())) {
1203 if (implementSameTransformInterface(t1: matcherType, t2: resultType))
1204 continue;
1205
1206 return emitError()
1207 << "mismatching type interfaces for matcher result and op result #"
1208 << i;
1209 }
1210
1211 return success();
1212}
1213
1214//===----------------------------------------------------------------------===//
1215// ForeachMatchOp
1216//===----------------------------------------------------------------------===//
1217
1218// This is fine because nothing is actually consumed by this op.
1219bool transform::ForeachMatchOp::allowsRepeatedHandleOperands() { return true; }
1220
1221DiagnosedSilenceableFailure
1222transform::ForeachMatchOp::apply(transform::TransformRewriter &rewriter,
1223 transform::TransformResults &results,
1224 transform::TransformState &state) {
1225 SmallVector<std::pair<FunctionOpInterface, FunctionOpInterface>>
1226 matchActionPairs;
1227 matchActionPairs.reserve(N: getMatchers().size());
1228 SymbolTableCollection symbolTable;
1229 for (auto &&[matcher, action] :
1230 llvm::zip_equal(t: getMatchers(), u: getActions())) {
1231 auto matcherSymbol =
1232 symbolTable.lookupNearestSymbolFrom<FunctionOpInterface>(
1233 from: getOperation(), symbol: cast<SymbolRefAttr>(Val: matcher));
1234 auto actionSymbol =
1235 symbolTable.lookupNearestSymbolFrom<FunctionOpInterface>(
1236 from: getOperation(), symbol: cast<SymbolRefAttr>(Val: action));
1237 assert(matcherSymbol && actionSymbol &&
1238 "unresolved symbols not caught by the verifier");
1239
1240 if (matcherSymbol.isExternal())
1241 return emitDefiniteFailure() << "unresolved external symbol " << matcher;
1242 if (actionSymbol.isExternal())
1243 return emitDefiniteFailure() << "unresolved external symbol " << action;
1244
1245 matchActionPairs.emplace_back(Args&: matcherSymbol, Args&: actionSymbol);
1246 }
1247
1248 DiagnosedSilenceableFailure overallDiag =
1249 DiagnosedSilenceableFailure::success();
1250
1251 SmallVector<SmallVector<MappedValue>> matchInputMapping;
1252 SmallVector<SmallVector<MappedValue>> matchOutputMapping;
1253 SmallVector<SmallVector<MappedValue>> actionResultMapping;
1254 // Explicitly add the mapping for the first block argument (the op being
1255 // matched).
1256 matchInputMapping.emplace_back();
1257 transform::detail::prepareValueMappings(mappings&: matchInputMapping,
1258 values: getForwardedInputs(), state);
1259 SmallVector<MappedValue> &firstMatchArgument = matchInputMapping.front();
1260 actionResultMapping.resize(N: getForwardedOutputs().size());
1261
1262 for (Operation *root : state.getPayloadOps(value: getRoot())) {
1263 WalkResult walkResult = root->walk(callback: [&](Operation *op) {
1264 // If getRestrictRoot is not present, skip over the root op itself so we
1265 // don't invalidate it.
1266 if (!getRestrictRoot() && op == root)
1267 return WalkResult::advance();
1268
1269 LDBG(DEBUG_TYPE_MATCHER, 1)
1270 << "matching "
1271 << OpWithFlags(op, OpPrintingFlags().assumeVerified().skipRegions())
1272 << " @" << op;
1273
1274 firstMatchArgument.clear();
1275 firstMatchArgument.push_back(Elt: op);
1276
1277 // Try all the match/action pairs until the first successful match.
1278 for (auto [matcher, action] : matchActionPairs) {
1279 DiagnosedSilenceableFailure diag =
1280 matchBlock(block&: matcher.getFunctionBody().front(), blockArgumentMapping: matchInputMapping,
1281 state, mappings&: matchOutputMapping);
1282 if (diag.isDefiniteFailure())
1283 return WalkResult::interrupt();
1284 if (diag.isSilenceableFailure()) {
1285 LDBG(DEBUG_TYPE_MATCHER, 1) << "matcher " << matcher.getName()
1286 << " failed: " << diag.getMessage();
1287 continue;
1288 }
1289
1290 auto scope = state.make_region_scope(region&: action.getFunctionBody());
1291 if (failed(Result: state.mapBlockArguments(
1292 arguments: action.getFunctionBody().front().getArguments(),
1293 mapping: matchOutputMapping))) {
1294 return WalkResult::interrupt();
1295 }
1296
1297 for (Operation &transform :
1298 action.getFunctionBody().front().without_terminator()) {
1299 DiagnosedSilenceableFailure result =
1300 state.applyTransform(transform: cast<TransformOpInterface>(Val&: transform));
1301 if (result.isDefiniteFailure())
1302 return WalkResult::interrupt();
1303 if (result.isSilenceableFailure()) {
1304 if (overallDiag.succeeded()) {
1305 overallDiag = emitSilenceableError() << "actions failed";
1306 }
1307 overallDiag.attachNote(loc: action->getLoc())
1308 << "failed action: " << result.getMessage();
1309 overallDiag.attachNote(loc: op->getLoc())
1310 << "when applied to this matching payload";
1311 (void)result.silence();
1312 continue;
1313 }
1314 }
1315 if (failed(Result: detail::appendValueMappings(
1316 mappings: MutableArrayRef<SmallVector<MappedValue>>(actionResultMapping),
1317 values: action.getFunctionBody().front().getTerminator()->getOperands(),
1318 state, flatten: getFlattenResults()))) {
1319 emitDefiniteFailure()
1320 << "action @" << action.getName()
1321 << " has results associated with multiple payload entities, "
1322 "but flattening was not requested";
1323 return WalkResult::interrupt();
1324 }
1325 break;
1326 }
1327 return WalkResult::advance();
1328 });
1329 if (walkResult.wasInterrupted())
1330 return DiagnosedSilenceableFailure::definiteFailure();
1331 }
1332
1333 // The root operation should not have been affected, so we can just reassign
1334 // the payload to the result. Note that we need to consume the root handle to
1335 // make sure any handles to operations inside, that could have been affected
1336 // by actions, are invalidated.
1337 results.set(value: llvm::cast<OpResult>(Val: getUpdated()),
1338 ops: state.getPayloadOps(value: getRoot()));
1339 for (auto &&[result, mapping] :
1340 llvm::zip_equal(t: getForwardedOutputs(), u&: actionResultMapping)) {
1341 results.setMappedValues(handle: result, values: mapping);
1342 }
1343 return overallDiag;
1344}
1345
1346void transform::ForeachMatchOp::getAsmResultNames(
1347 OpAsmSetValueNameFn setNameFn) {
1348 setNameFn(getUpdated(), "updated_root");
1349 for (Value v : getForwardedOutputs()) {
1350 setNameFn(v, "yielded");
1351 }
1352}
1353
1354void transform::ForeachMatchOp::getEffects(
1355 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
1356 // Bail if invalid.
1357 if (getOperation()->getNumOperands() < 1 ||
1358 getOperation()->getNumResults() < 1) {
1359 return modifiesPayload(effects);
1360 }
1361
1362 consumesHandle(handles: getRootMutable(), effects);
1363 onlyReadsHandle(handles: getForwardedInputsMutable(), effects);
1364 producesHandle(handles: getOperation()->getOpResults(), effects);
1365 modifiesPayload(effects);
1366}
1367
1368/// Parses the comma-separated list of symbol reference pairs of the format
1369/// `@matcher -> @action`.
1370static ParseResult parseForeachMatchSymbols(OpAsmParser &parser,
1371 ArrayAttr &matchers,
1372 ArrayAttr &actions) {
1373 StringAttr matcher;
1374 StringAttr action;
1375 SmallVector<Attribute> matcherList;
1376 SmallVector<Attribute> actionList;
1377 do {
1378 if (parser.parseSymbolName(result&: matcher) || parser.parseArrow() ||
1379 parser.parseSymbolName(result&: action)) {
1380 return failure();
1381 }
1382 matcherList.push_back(Elt: SymbolRefAttr::get(value: matcher));
1383 actionList.push_back(Elt: SymbolRefAttr::get(value: action));
1384 } while (parser.parseOptionalComma().succeeded());
1385
1386 matchers = parser.getBuilder().getArrayAttr(value: matcherList);
1387 actions = parser.getBuilder().getArrayAttr(value: actionList);
1388 return success();
1389}
1390
1391/// Prints the comma-separated list of symbol reference pairs of the format
1392/// `@matcher -> @action`.
1393static void printForeachMatchSymbols(OpAsmPrinter &printer, Operation *op,
1394 ArrayAttr matchers, ArrayAttr actions) {
1395 printer.increaseIndent();
1396 printer.increaseIndent();
1397 for (auto &&[matcher, action, idx] : llvm::zip_equal(
1398 t&: matchers, u&: actions, args: llvm::seq<unsigned>(Begin: 0, End: matchers.size()))) {
1399 printer.printNewline();
1400 printer << cast<SymbolRefAttr>(Val: matcher) << " -> "
1401 << cast<SymbolRefAttr>(Val: action);
1402 if (idx != matchers.size() - 1)
1403 printer << ", ";
1404 }
1405 printer.decreaseIndent();
1406 printer.decreaseIndent();
1407}
1408
1409LogicalResult transform::ForeachMatchOp::verify() {
1410 if (getMatchers().size() != getActions().size())
1411 return emitOpError() << "expected the same number of matchers and actions";
1412 if (getMatchers().empty())
1413 return emitOpError() << "expected at least one match/action pair";
1414
1415 llvm::SmallPtrSet<Attribute, 8> matcherNames;
1416 for (Attribute name : getMatchers()) {
1417 if (matcherNames.insert(Ptr: name).second)
1418 continue;
1419 emitWarning() << "matcher " << name
1420 << " is used more than once, only the first match will apply";
1421 }
1422
1423 return success();
1424}
1425
1426/// Checks that the attributes of the function-like operation have correct
1427/// consumption effect annotations. If `alsoVerifyInternal`, checks for
1428/// annotations being present even if they can be inferred from the body.
1429static DiagnosedSilenceableFailure
1430verifyFunctionLikeConsumeAnnotations(FunctionOpInterface op, bool emitWarnings,
1431 bool alsoVerifyInternal = false) {
1432 auto transformOp = cast<transform::TransformOpInterface>(Val: op.getOperation());
1433 llvm::SmallDenseSet<unsigned> consumedArguments;
1434 if (!op.isExternal()) {
1435 transform::getConsumedBlockArguments(block&: op.getFunctionBody().front(),
1436 consumedArguments);
1437 }
1438 for (unsigned i = 0, e = op.getNumArguments(); i < e; ++i) {
1439 bool isConsumed =
1440 op.getArgAttr(index: i, name: transform::TransformDialect::kArgConsumedAttrName) !=
1441 nullptr;
1442 bool isReadOnly =
1443 op.getArgAttr(index: i, name: transform::TransformDialect::kArgReadOnlyAttrName) !=
1444 nullptr;
1445 if (isConsumed && isReadOnly) {
1446 return transformOp.emitSilenceableError()
1447 << "argument #" << i << " cannot be both readonly and consumed";
1448 }
1449 if ((op.isExternal() || alsoVerifyInternal) && !isConsumed && !isReadOnly) {
1450 return transformOp.emitSilenceableError()
1451 << "must provide consumed/readonly status for arguments of "
1452 "external or called ops";
1453 }
1454 if (op.isExternal())
1455 continue;
1456
1457 if (consumedArguments.contains(V: i) && !isConsumed && isReadOnly) {
1458 return transformOp.emitSilenceableError()
1459 << "argument #" << i
1460 << " is consumed in the body but is not marked as such";
1461 }
1462 if (emitWarnings && !consumedArguments.contains(V: i) && isConsumed) {
1463 // Cannot use op.emitWarning() here as it would attempt to verify the op
1464 // before printing, resulting in infinite recursion.
1465 emitWarning(loc: op->getLoc())
1466 << "op argument #" << i
1467 << " is not consumed in the body but is marked as consumed";
1468 }
1469 }
1470 return DiagnosedSilenceableFailure::success();
1471}
1472
1473LogicalResult transform::ForeachMatchOp::verifySymbolUses(
1474 SymbolTableCollection &symbolTable) {
1475 assert(getMatchers().size() == getActions().size());
1476 auto consumedAttr =
1477 StringAttr::get(context: getContext(), bytes: TransformDialect::kArgConsumedAttrName);
1478 for (auto &&[matcher, action] :
1479 llvm::zip_equal(t: getMatchers(), u: getActions())) {
1480 // Presence and typing.
1481 auto matcherSymbol = dyn_cast_or_null<FunctionOpInterface>(
1482 Val: symbolTable.lookupNearestSymbolFrom(from: getOperation(),
1483 symbol: cast<SymbolRefAttr>(Val: matcher)));
1484 auto actionSymbol = dyn_cast_or_null<FunctionOpInterface>(
1485 Val: symbolTable.lookupNearestSymbolFrom(from: getOperation(),
1486 symbol: cast<SymbolRefAttr>(Val: action)));
1487 if (!matcherSymbol ||
1488 !isa<TransformOpInterface>(Val: matcherSymbol.getOperation()))
1489 return emitError() << "unresolved matcher symbol " << matcher;
1490 if (!actionSymbol ||
1491 !isa<TransformOpInterface>(Val: actionSymbol.getOperation()))
1492 return emitError() << "unresolved action symbol " << action;
1493
1494 if (failed(Result: verifyFunctionLikeConsumeAnnotations(op: matcherSymbol,
1495 /*emitWarnings=*/false,
1496 /*alsoVerifyInternal=*/true)
1497 .checkAndReport())) {
1498 return failure();
1499 }
1500 if (failed(Result: verifyFunctionLikeConsumeAnnotations(op: actionSymbol,
1501 /*emitWarnings=*/false,
1502 /*alsoVerifyInternal=*/true)
1503 .checkAndReport())) {
1504 return failure();
1505 }
1506
1507 // Input -> matcher forwarding.
1508 TypeRange operandTypes = getOperandTypes();
1509 TypeRange matcherArguments = matcherSymbol.getArgumentTypes();
1510 if (operandTypes.size() != matcherArguments.size()) {
1511 InFlightDiagnostic diag =
1512 emitError() << "the number of operands (" << operandTypes.size()
1513 << ") doesn't match the number of matcher arguments ("
1514 << matcherArguments.size() << ") for " << matcher;
1515 diag.attachNote(noteLoc: matcherSymbol->getLoc()) << "symbol declaration";
1516 return diag;
1517 }
1518 for (auto &&[i, operand, argument] :
1519 llvm::enumerate(First&: operandTypes, Rest&: matcherArguments)) {
1520 if (matcherSymbol.getArgAttr(index: i, name: consumedAttr)) {
1521 InFlightDiagnostic diag =
1522 emitOpError()
1523 << "does not expect matcher symbol to consume its operand #" << i;
1524 diag.attachNote(noteLoc: matcherSymbol->getLoc()) << "symbol declaration";
1525 return diag;
1526 }
1527
1528 if (implementSameTransformInterface(t1: operand, t2: argument))
1529 continue;
1530
1531 InFlightDiagnostic diag =
1532 emitError()
1533 << "mismatching type interfaces for operand and matcher argument #"
1534 << i << " of matcher " << matcher;
1535 diag.attachNote(noteLoc: matcherSymbol->getLoc()) << "symbol declaration";
1536 return diag;
1537 }
1538
1539 // Matcher -> action forwarding.
1540 TypeRange matcherResults = matcherSymbol.getResultTypes();
1541 TypeRange actionArguments = actionSymbol.getArgumentTypes();
1542 if (matcherResults.size() != actionArguments.size()) {
1543 return emitError() << "mismatching number of matcher results and "
1544 "action arguments between "
1545 << matcher << " (" << matcherResults.size() << ") and "
1546 << action << " (" << actionArguments.size() << ")";
1547 }
1548 for (auto &&[i, matcherType, actionType] :
1549 llvm::enumerate(First&: matcherResults, Rest&: actionArguments)) {
1550 if (implementSameTransformInterface(t1: matcherType, t2: actionType))
1551 continue;
1552
1553 return emitError() << "mismatching type interfaces for matcher result "
1554 "and action argument #"
1555 << i << "of matcher " << matcher << " and action "
1556 << action;
1557 }
1558
1559 // Action -> result forwarding.
1560 TypeRange actionResults = actionSymbol.getResultTypes();
1561 auto resultTypes = TypeRange(getResultTypes()).drop_front();
1562 if (actionResults.size() != resultTypes.size()) {
1563 InFlightDiagnostic diag =
1564 emitError() << "the number of action results ("
1565 << actionResults.size() << ") for " << action
1566 << " doesn't match the number of extra op results ("
1567 << resultTypes.size() << ")";
1568 diag.attachNote(noteLoc: actionSymbol->getLoc()) << "symbol declaration";
1569 return diag;
1570 }
1571 for (auto &&[i, resultType, actionType] :
1572 llvm::enumerate(First&: resultTypes, Rest&: actionResults)) {
1573 if (implementSameTransformInterface(t1: resultType, t2: actionType))
1574 continue;
1575
1576 InFlightDiagnostic diag =
1577 emitError() << "mismatching type interfaces for action result #" << i
1578 << " of action " << action << " and op result";
1579 diag.attachNote(noteLoc: actionSymbol->getLoc()) << "symbol declaration";
1580 return diag;
1581 }
1582 }
1583 return success();
1584}
1585
1586//===----------------------------------------------------------------------===//
1587// ForeachOp
1588//===----------------------------------------------------------------------===//
1589
1590DiagnosedSilenceableFailure
1591transform::ForeachOp::apply(transform::TransformRewriter &rewriter,
1592 transform::TransformResults &results,
1593 transform::TransformState &state) {
1594 // We store the payloads before executing the body as ops may be removed from
1595 // the mapping by the TrackingRewriter while iteration is in progress.
1596 SmallVector<SmallVector<MappedValue>> payloads;
1597 detail::prepareValueMappings(mappings&: payloads, values: getTargets(), state);
1598 size_t numIterations = payloads.empty() ? 0 : payloads.front().size();
1599 bool withZipShortest = getWithZipShortest();
1600
1601 // In case of `zip_shortest`, set the number of iterations to the
1602 // smallest payload in the targets.
1603 if (withZipShortest) {
1604 numIterations =
1605 llvm::min_element(Range&: payloads, C: [&](const SmallVector<MappedValue> &a,
1606 const SmallVector<MappedValue> &b) {
1607 return a.size() < b.size();
1608 })->size();
1609
1610 for (auto &payload : payloads)
1611 payload.resize(N: numIterations);
1612 }
1613
1614 // As we will be "zipping" over them, check all payloads have the same size.
1615 // `zip_shortest` adjusts all payloads to the same size, so skip this check
1616 // when true.
1617 for (size_t argIdx = 1; !withZipShortest && argIdx < payloads.size();
1618 argIdx++) {
1619 if (payloads[argIdx].size() != numIterations) {
1620 return emitSilenceableError()
1621 << "prior targets' payload size (" << numIterations
1622 << ") differs from payload size (" << payloads[argIdx].size()
1623 << ") of target " << getTargets()[argIdx];
1624 }
1625 }
1626
1627 // Start iterating, indexing into payloads to obtain the right arguments to
1628 // call the body with - each slice of payloads at the same argument index
1629 // corresponding to a tuple to use as the body's block arguments.
1630 ArrayRef<BlockArgument> blockArguments = getBody().front().getArguments();
1631 SmallVector<SmallVector<MappedValue>> zippedResults(getNumResults(), {});
1632 for (size_t iterIdx = 0; iterIdx < numIterations; iterIdx++) {
1633 auto scope = state.make_region_scope(region&: getBody());
1634 // Set up arguments to the region's block.
1635 for (auto &&[argIdx, blockArg] : llvm::enumerate(First&: blockArguments)) {
1636 MappedValue argument = payloads[argIdx][iterIdx];
1637 // Note that each blockArg's handle gets associated with just a single
1638 // element from the corresponding target's payload.
1639 if (failed(Result: state.mapBlockArgument(argument: blockArg, values: {argument})))
1640 return DiagnosedSilenceableFailure::definiteFailure();
1641 }
1642
1643 // Execute loop body.
1644 for (Operation &transform : getBody().front().without_terminator()) {
1645 DiagnosedSilenceableFailure result = state.applyTransform(
1646 transform: llvm::cast<transform::TransformOpInterface>(Val&: transform));
1647 if (!result.succeeded())
1648 return result;
1649 }
1650
1651 // Append yielded payloads to corresponding results from prior iterations.
1652 OperandRange yieldOperands = getYieldOp().getOperands();
1653 for (auto &&[result, yieldOperand, resTuple] :
1654 llvm::zip_equal(t: getResults(), u&: yieldOperands, args&: zippedResults))
1655 // NB: each iteration we add any number of ops/vals/params to a result.
1656 if (isa<TransformHandleTypeInterface>(Val: result.getType()))
1657 llvm::append_range(C&: resTuple, R: state.getPayloadOps(value: yieldOperand));
1658 else if (isa<TransformValueHandleTypeInterface>(Val: result.getType()))
1659 llvm::append_range(C&: resTuple, R: state.getPayloadValues(handleValue: yieldOperand));
1660 else if (isa<TransformParamTypeInterface>(Val: result.getType()))
1661 llvm::append_range(C&: resTuple, R: state.getParams(value: yieldOperand));
1662 else
1663 assert(false && "unhandled handle type");
1664 }
1665
1666 // Associate the accumulated result payloads to the op's actual results.
1667 for (auto &&[result, resPayload] : zip_equal(t: getResults(), u&: zippedResults))
1668 results.setMappedValues(handle: llvm::cast<OpResult>(Val&: result), values: resPayload);
1669
1670 return DiagnosedSilenceableFailure::success();
1671}
1672
1673void transform::ForeachOp::getEffects(
1674 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
1675 // NB: this `zip` should be `zip_equal` - while this op's verifier catches
1676 // arity errors, this method might get called before/in absence of `verify()`.
1677 for (auto &&[target, blockArg] :
1678 llvm::zip(t: getTargetsMutable(), u: getBody().front().getArguments())) {
1679 BlockArgument blockArgument = blockArg;
1680 if (any_of(Range: getBody().front().without_terminator(), P: [&](Operation &op) {
1681 return isHandleConsumed(handle: blockArgument,
1682 transform: cast<TransformOpInterface>(Val: &op));
1683 })) {
1684 consumesHandle(handles: target, effects);
1685 } else {
1686 onlyReadsHandle(handles: target, effects);
1687 }
1688 }
1689
1690 if (any_of(Range: getBody().front().without_terminator(), P: [&](Operation &op) {
1691 return doesModifyPayload(transform: cast<TransformOpInterface>(Val: &op));
1692 })) {
1693 modifiesPayload(effects);
1694 } else if (any_of(Range: getBody().front().without_terminator(), P: [&](Operation &op) {
1695 return doesReadPayload(transform: cast<TransformOpInterface>(Val: &op));
1696 })) {
1697 onlyReadsPayload(effects);
1698 }
1699
1700 producesHandle(handles: getOperation()->getOpResults(), effects);
1701}
1702
1703void transform::ForeachOp::getSuccessorRegions(
1704 RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &regions) {
1705 Region *bodyRegion = &getBody();
1706 if (point.isParent()) {
1707 regions.emplace_back(Args&: bodyRegion);
1708 return;
1709 }
1710
1711 // Branch back to the region or the parent.
1712 assert(point.getTerminatorPredecessorOrNull()->getParentRegion() ==
1713 &getBody() &&
1714 "unexpected region index");
1715 regions.emplace_back(Args&: bodyRegion);
1716 regions.push_back(Elt: RegionSuccessor(getOperation()));
1717}
1718
1719ValueRange transform::ForeachOp::getSuccessorInputs(RegionSuccessor successor) {
1720 return successor.isOperation() ? ValueRange(getResults())
1721 : ValueRange(getBody().getArguments());
1722}
1723
1724OperandRange
1725transform::ForeachOp::getEntrySuccessorOperands(RegionSuccessor successor) {
1726 // Each block argument handle is mapped to a subset (one op to be precise)
1727 // of the payload of the corresponding `targets` operand of ForeachOp.
1728 assert(successor.getSuccessor() == &getBody() && "unexpected region index");
1729 return getOperation()->getOperands();
1730}
1731
1732transform::YieldOp transform::ForeachOp::getYieldOp() {
1733 return cast<transform::YieldOp>(Val: getBody().front().getTerminator());
1734}
1735
1736LogicalResult transform::ForeachOp::verify() {
1737 for (auto [targetOpt, bodyArgOpt] :
1738 llvm::zip_longest(t: getTargets(), u: getBody().front().getArguments())) {
1739 if (!targetOpt || !bodyArgOpt)
1740 return emitOpError() << "expects the same number of targets as the body "
1741 "has block arguments";
1742 if (targetOpt.value().getType() != bodyArgOpt.value().getType())
1743 return emitOpError(
1744 message: "expects co-indexed targets and the body's "
1745 "block arguments to have the same op/value/param type");
1746 }
1747
1748 for (auto [resultOpt, yieldOperandOpt] :
1749 llvm::zip_longest(t: getResults(), u: getYieldOp().getOperands())) {
1750 if (!resultOpt || !yieldOperandOpt)
1751 return emitOpError() << "expects the same number of results as the "
1752 "yield terminator has operands";
1753 if (resultOpt.value().getType() != yieldOperandOpt.value().getType())
1754 return emitOpError(message: "expects co-indexed results and yield "
1755 "operands to have the same op/value/param type");
1756 }
1757
1758 return success();
1759}
1760
1761//===----------------------------------------------------------------------===//
1762// GetParentOp
1763//===----------------------------------------------------------------------===//
1764
1765DiagnosedSilenceableFailure
1766transform::GetParentOp::apply(transform::TransformRewriter &rewriter,
1767 transform::TransformResults &results,
1768 transform::TransformState &state) {
1769 SmallVector<Operation *> parents;
1770 DenseSet<Operation *> resultSet;
1771 for (Operation *target : state.getPayloadOps(value: getTarget())) {
1772 Operation *parent = target;
1773 for (int64_t i = 0, e = getNthParent(); i < e; ++i) {
1774 parent = parent->getParentOp();
1775 while (parent) {
1776 bool checkIsolatedFromAbove =
1777 !getIsolatedFromAbove() ||
1778 parent->hasTrait<OpTrait::IsIsolatedFromAbove>();
1779 bool checkOpName = !getOpName().has_value() ||
1780 parent->getName().getStringRef() == *getOpName();
1781 if (checkIsolatedFromAbove && checkOpName)
1782 break;
1783 parent = parent->getParentOp();
1784 }
1785 if (!parent) {
1786 if (getAllowEmptyResults()) {
1787 results.set(value: llvm::cast<OpResult>(Val: getResult()), ops&: parents);
1788 return DiagnosedSilenceableFailure::success();
1789 }
1790 DiagnosedSilenceableFailure diag =
1791 emitSilenceableError()
1792 << "could not find a parent op that matches all requirements";
1793 diag.attachNote(loc: target->getLoc()) << "target op";
1794 return diag;
1795 }
1796 }
1797 if (getDeduplicate()) {
1798 if (resultSet.insert(V: parent).second)
1799 parents.push_back(Elt: parent);
1800 } else {
1801 parents.push_back(Elt: parent);
1802 }
1803 }
1804 results.set(value: llvm::cast<OpResult>(Val: getResult()), ops&: parents);
1805 return DiagnosedSilenceableFailure::success();
1806}
1807
1808//===----------------------------------------------------------------------===//
1809// GetConsumersOfResult
1810//===----------------------------------------------------------------------===//
1811
1812DiagnosedSilenceableFailure
1813transform::GetConsumersOfResult::apply(transform::TransformRewriter &rewriter,
1814 transform::TransformResults &results,
1815 transform::TransformState &state) {
1816 int64_t resultNumber = getResultNumber();
1817 auto payloadOps = state.getPayloadOps(value: getTarget());
1818 if (std::empty(cont: payloadOps)) {
1819 results.set(value: cast<OpResult>(Val: getResult()), ops: {});
1820 return DiagnosedSilenceableFailure::success();
1821 }
1822 if (!llvm::hasSingleElement(C&: payloadOps))
1823 return emitDefiniteFailure()
1824 << "handle must be mapped to exactly one payload op";
1825
1826 Operation *target = *payloadOps.begin();
1827 if (target->getNumResults() <= resultNumber)
1828 return emitDefiniteFailure() << "result number overflow";
1829 results.set(value: llvm::cast<OpResult>(Val: getResult()),
1830 ops: llvm::to_vector(Range: target->getResult(idx: resultNumber).getUsers()));
1831 return DiagnosedSilenceableFailure::success();
1832}
1833
1834//===----------------------------------------------------------------------===//
1835// GetDefiningOp
1836//===----------------------------------------------------------------------===//
1837
1838DiagnosedSilenceableFailure
1839transform::GetDefiningOp::apply(transform::TransformRewriter &rewriter,
1840 transform::TransformResults &results,
1841 transform::TransformState &state) {
1842 SmallVector<Operation *> definingOps;
1843 for (Value v : state.getPayloadValues(handleValue: getTarget())) {
1844 if (llvm::isa<BlockArgument>(Val: v)) {
1845 DiagnosedSilenceableFailure diag =
1846 emitSilenceableError() << "cannot get defining op of block argument";
1847 diag.attachNote(loc: v.getLoc()) << "target value";
1848 return diag;
1849 }
1850 definingOps.push_back(Elt: v.getDefiningOp());
1851 }
1852 results.set(value: llvm::cast<OpResult>(Val: getResult()), ops&: definingOps);
1853 return DiagnosedSilenceableFailure::success();
1854}
1855
1856//===----------------------------------------------------------------------===//
1857// GetProducerOfOperand
1858//===----------------------------------------------------------------------===//
1859
1860DiagnosedSilenceableFailure
1861transform::GetProducerOfOperand::apply(transform::TransformRewriter &rewriter,
1862 transform::TransformResults &results,
1863 transform::TransformState &state) {
1864 int64_t operandNumber = getOperandNumber();
1865 SmallVector<Operation *> producers;
1866 for (Operation *target : state.getPayloadOps(value: getTarget())) {
1867 Operation *producer =
1868 target->getNumOperands() <= operandNumber
1869 ? nullptr
1870 : target->getOperand(idx: operandNumber).getDefiningOp();
1871 if (!producer) {
1872 DiagnosedSilenceableFailure diag =
1873 emitSilenceableError()
1874 << "could not find a producer for operand number: " << operandNumber
1875 << " of " << *target;
1876 diag.attachNote(loc: target->getLoc()) << "target op";
1877 return diag;
1878 }
1879 producers.push_back(Elt: producer);
1880 }
1881 results.set(value: llvm::cast<OpResult>(Val: getResult()), ops&: producers);
1882 return DiagnosedSilenceableFailure::success();
1883}
1884
1885//===----------------------------------------------------------------------===//
1886// GetOperandOp
1887//===----------------------------------------------------------------------===//
1888
1889DiagnosedSilenceableFailure
1890transform::GetOperandOp::apply(transform::TransformRewriter &rewriter,
1891 transform::TransformResults &results,
1892 transform::TransformState &state) {
1893 SmallVector<Value> operands;
1894 for (Operation *target : state.getPayloadOps(value: getTarget())) {
1895 SmallVector<int64_t> operandPositions;
1896 DiagnosedSilenceableFailure diag = expandTargetSpecification(
1897 loc: getLoc(), isAll: getIsAll(), isInverted: getIsInverted(), rawList: getRawPositionList(),
1898 maxNumber: target->getNumOperands(), result&: operandPositions);
1899 if (diag.isSilenceableFailure()) {
1900 diag.attachNote(loc: target->getLoc())
1901 << "while considering positions of this payload operation";
1902 return diag;
1903 }
1904 llvm::append_range(C&: operands,
1905 R: llvm::map_range(C&: operandPositions, F: [&](int64_t pos) {
1906 return target->getOperand(idx: pos);
1907 }));
1908 }
1909 results.setValues(handle: cast<OpResult>(Val: getResult()), values&: operands);
1910 return DiagnosedSilenceableFailure::success();
1911}
1912
1913LogicalResult transform::GetOperandOp::verify() {
1914 return verifyTransformMatchDimsOp(op: getOperation(), raw: getRawPositionList(),
1915 inverted: getIsInverted(), all: getIsAll());
1916}
1917
1918//===----------------------------------------------------------------------===//
1919// GetResultOp
1920//===----------------------------------------------------------------------===//
1921
1922DiagnosedSilenceableFailure
1923transform::GetResultOp::apply(transform::TransformRewriter &rewriter,
1924 transform::TransformResults &results,
1925 transform::TransformState &state) {
1926 SmallVector<Value> opResults;
1927 for (Operation *target : state.getPayloadOps(value: getTarget())) {
1928 SmallVector<int64_t> resultPositions;
1929 DiagnosedSilenceableFailure diag = expandTargetSpecification(
1930 loc: getLoc(), isAll: getIsAll(), isInverted: getIsInverted(), rawList: getRawPositionList(),
1931 maxNumber: target->getNumResults(), result&: resultPositions);
1932 if (diag.isSilenceableFailure()) {
1933 diag.attachNote(loc: target->getLoc())
1934 << "while considering positions of this payload operation";
1935 return diag;
1936 }
1937 llvm::append_range(C&: opResults,
1938 R: llvm::map_range(C&: resultPositions, F: [&](int64_t pos) {
1939 return target->getResult(idx: pos);
1940 }));
1941 }
1942 results.setValues(handle: cast<OpResult>(Val: getResult()), values&: opResults);
1943 return DiagnosedSilenceableFailure::success();
1944}
1945
1946LogicalResult transform::GetResultOp::verify() {
1947 return verifyTransformMatchDimsOp(op: getOperation(), raw: getRawPositionList(),
1948 inverted: getIsInverted(), all: getIsAll());
1949}
1950
1951//===----------------------------------------------------------------------===//
1952// GetTypeOp
1953//===----------------------------------------------------------------------===//
1954
1955void transform::GetTypeOp::getEffects(
1956 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
1957 onlyReadsHandle(handles: getValueMutable(), effects);
1958 producesHandle(handles: getOperation()->getOpResults(), effects);
1959 onlyReadsPayload(effects);
1960}
1961
1962DiagnosedSilenceableFailure
1963transform::GetTypeOp::apply(transform::TransformRewriter &rewriter,
1964 transform::TransformResults &results,
1965 transform::TransformState &state) {
1966 SmallVector<Attribute> params;
1967 for (Value value : state.getPayloadValues(handleValue: getValue())) {
1968 Type type = value.getType();
1969 if (getElemental()) {
1970 if (auto shaped = dyn_cast<ShapedType>(Val&: type)) {
1971 type = shaped.getElementType();
1972 }
1973 }
1974 params.push_back(Elt: TypeAttr::get(type));
1975 }
1976 results.setParams(value: cast<OpResult>(Val: getResult()), params);
1977 return DiagnosedSilenceableFailure::success();
1978}
1979
1980//===----------------------------------------------------------------------===//
1981// IncludeOp
1982//===----------------------------------------------------------------------===//
1983
1984/// Applies the transform ops contained in `block`. Maps `results` to the same
1985/// values as the operands of the block terminator.
1986static DiagnosedSilenceableFailure
1987applySequenceBlock(Block &block, transform::FailurePropagationMode mode,
1988 transform::TransformState &state,
1989 transform::TransformResults &results) {
1990 // Apply the sequenced ops one by one.
1991 for (Operation &transform : block.without_terminator()) {
1992 DiagnosedSilenceableFailure result =
1993 state.applyTransform(transform: cast<transform::TransformOpInterface>(Val&: transform));
1994 if (result.isDefiniteFailure())
1995 return result;
1996
1997 if (result.isSilenceableFailure()) {
1998 if (mode == transform::FailurePropagationMode::Propagate) {
1999 // Propagate empty results in case of early exit.
2000 forwardEmptyOperands(block: &block, state, results);
2001 return result;
2002 }
2003 (void)result.silence();
2004 }
2005 }
2006
2007 // Forward the operation mapping for values yielded from the sequence to the
2008 // values produced by the sequence op.
2009 transform::detail::forwardTerminatorOperands(block: &block, state, results);
2010 return DiagnosedSilenceableFailure::success();
2011}
2012
2013DiagnosedSilenceableFailure
2014transform::IncludeOp::apply(transform::TransformRewriter &rewriter,
2015 transform::TransformResults &results,
2016 transform::TransformState &state) {
2017 auto callee = SymbolTable::lookupNearestSymbolFrom<NamedSequenceOp>(
2018 from: getOperation(), symbol: getTarget());
2019 assert(callee && "unverified reference to unknown symbol");
2020
2021 if (callee.isExternal())
2022 return emitDefiniteFailure() << "unresolved external named sequence";
2023
2024 // Map operands to block arguments.
2025 SmallVector<SmallVector<MappedValue>> mappings;
2026 detail::prepareValueMappings(mappings, values: getOperands(), state);
2027 auto scope = state.make_region_scope(region&: callee.getBody());
2028 for (auto &&[arg, map] :
2029 llvm::zip_equal(t: callee.getBody().front().getArguments(), u&: mappings)) {
2030 if (failed(Result: state.mapBlockArgument(argument: arg, values: map)))
2031 return DiagnosedSilenceableFailure::definiteFailure();
2032 }
2033
2034 DiagnosedSilenceableFailure result = applySequenceBlock(
2035 block&: callee.getBody().front(), mode: getFailurePropagationMode(), state, results);
2036
2037 if (!result.succeeded())
2038 return result;
2039
2040 mappings.clear();
2041 detail::prepareValueMappings(
2042 mappings, values: callee.getBody().front().getTerminator()->getOperands(), state);
2043 for (auto &&[result, mapping] : llvm::zip_equal(t: getResults(), u&: mappings))
2044 results.setMappedValues(handle: result, values: mapping);
2045 return result;
2046}
2047
2048static DiagnosedSilenceableFailure
2049verifyNamedSequenceOp(transform::NamedSequenceOp op, bool emitWarnings);
2050
2051void transform::IncludeOp::getEffects(
2052 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
2053 // Always mark as modifying the payload.
2054 // TODO: a mechanism to annotate effects on payload. Even when all handles are
2055 // only read, the payload may still be modified, so we currently stay on the
2056 // conservative side and always indicate modification. This may prevent some
2057 // code reordering.
2058 modifiesPayload(effects);
2059
2060 // Results are always produced.
2061 producesHandle(handles: getOperation()->getOpResults(), effects);
2062
2063 // Adds default effects to operands and results. This will be added if
2064 // preconditions fail so the trait verifier doesn't complain about missing
2065 // effects and the real precondition failure is reported later on.
2066 auto defaultEffects = [&] {
2067 onlyReadsHandle(handles: getOperation()->getOpOperands(), effects);
2068 };
2069
2070 // Bail if the callee is unknown. This may run as part of the verification
2071 // process before we verified the validity of the callee or of this op.
2072 auto target =
2073 getOperation()->getAttrOfType<SymbolRefAttr>(name: getTargetAttrName());
2074 if (!target)
2075 return defaultEffects();
2076 auto callee = SymbolTable::lookupNearestSymbolFrom<NamedSequenceOp>(
2077 from: getOperation(), symbol: getTarget());
2078 if (!callee)
2079 return defaultEffects();
2080
2081 for (unsigned i = 0, e = getNumOperands(); i < e; ++i) {
2082 if (callee.getArgAttr(index: i, name: TransformDialect::kArgConsumedAttrName))
2083 consumesHandle(handles: getOperation()->getOpOperand(idx: i), effects);
2084 else if (callee.getArgAttr(index: i, name: TransformDialect::kArgReadOnlyAttrName))
2085 onlyReadsHandle(handles: getOperation()->getOpOperand(idx: i), effects);
2086 }
2087}
2088
2089LogicalResult
2090transform::IncludeOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2091 // Access through indirection and do additional checking because this may be
2092 // running before the main op verifier.
2093 auto targetAttr = getOperation()->getAttrOfType<SymbolRefAttr>(name: "target");
2094 if (!targetAttr)
2095 return emitOpError() << "expects a 'target' symbol reference attribute";
2096
2097 auto target = symbolTable.lookupNearestSymbolFrom<transform::NamedSequenceOp>(
2098 from: *this, symbol: targetAttr);
2099 if (!target)
2100 return emitOpError() << "does not reference a named transform sequence";
2101
2102 FunctionType fnType = target.getFunctionType();
2103 if (fnType.getNumInputs() != getNumOperands())
2104 return emitError(message: "incorrect number of operands for callee");
2105
2106 for (unsigned i = 0, e = fnType.getNumInputs(); i != e; ++i) {
2107 if (getOperand(i).getType() != fnType.getInput(i)) {
2108 return emitOpError(message: "operand type mismatch: expected operand type ")
2109 << fnType.getInput(i) << ", but provided "
2110 << getOperand(i).getType() << " for operand number " << i;
2111 }
2112 }
2113
2114 if (fnType.getNumResults() != getNumResults())
2115 return emitError(message: "incorrect number of results for callee");
2116
2117 for (unsigned i = 0, e = fnType.getNumResults(); i != e; ++i) {
2118 Type resultType = getResult(i).getType();
2119 Type funcType = fnType.getResult(i);
2120 if (!implementSameTransformInterface(t1: resultType, t2: funcType)) {
2121 return emitOpError() << "type of result #" << i
2122 << " must implement the same transform dialect "
2123 "interface as the corresponding callee result";
2124 }
2125 }
2126
2127 return verifyFunctionLikeConsumeAnnotations(
2128 op: cast<FunctionOpInterface>(Val&: *target), /*emitWarnings=*/false,
2129 /*alsoVerifyInternal=*/true)
2130 .checkAndReport();
2131}
2132
2133//===----------------------------------------------------------------------===//
2134// MatchOperationEmptyOp
2135//===----------------------------------------------------------------------===//
2136
2137DiagnosedSilenceableFailure transform::MatchOperationEmptyOp::matchOperation(
2138 ::std::optional<::mlir::Operation *> maybeCurrent,
2139 transform::TransformResults &results, transform::TransformState &state) {
2140 if (!maybeCurrent.has_value()) {
2141 LDBG(DEBUG_TYPE_MATCHER, 1) << "MatchOperationEmptyOp success";
2142 return DiagnosedSilenceableFailure::success();
2143 }
2144 LDBG(DEBUG_TYPE_MATCHER, 1) << "MatchOperationEmptyOp failure";
2145 return emitSilenceableError() << "operation is not empty";
2146}
2147
2148//===----------------------------------------------------------------------===//
2149// MatchOperationNameOp
2150//===----------------------------------------------------------------------===//
2151
2152DiagnosedSilenceableFailure transform::MatchOperationNameOp::matchOperation(
2153 Operation *current, transform::TransformResults &results,
2154 transform::TransformState &state) {
2155 StringRef currentOpName = current->getName().getStringRef();
2156 for (auto acceptedAttr : getOpNames().getAsRange<StringAttr>()) {
2157 if (acceptedAttr.getValue() == currentOpName)
2158 return DiagnosedSilenceableFailure::success();
2159 }
2160 return emitSilenceableError() << "wrong operation name";
2161}
2162
2163//===----------------------------------------------------------------------===//
2164// MatchParamCmpIOp
2165//===----------------------------------------------------------------------===//
2166
2167DiagnosedSilenceableFailure
2168transform::MatchParamCmpIOp::apply(transform::TransformRewriter &rewriter,
2169 transform::TransformResults &results,
2170 transform::TransformState &state) {
2171 auto signedAPIntAsString = [&](const APInt &value) {
2172 std::string str;
2173 llvm::raw_string_ostream os(str);
2174 value.print(OS&: os, /*isSigned=*/true);
2175 return str;
2176 };
2177
2178 ArrayRef<Attribute> params = state.getParams(value: getParam());
2179 ArrayRef<Attribute> references = state.getParams(value: getReference());
2180
2181 if (params.size() != references.size()) {
2182 return emitSilenceableError()
2183 << "parameters have different payload lengths (" << params.size()
2184 << " vs " << references.size() << ")";
2185 }
2186
2187 for (auto &&[i, param, reference] : llvm::enumerate(First&: params, Rest&: references)) {
2188 auto intAttr = llvm::dyn_cast<IntegerAttr>(Val: param);
2189 auto refAttr = llvm::dyn_cast<IntegerAttr>(Val: reference);
2190 if (!intAttr || !refAttr) {
2191 return emitDefiniteFailure()
2192 << "non-integer parameter value not expected";
2193 }
2194 if (intAttr.getType() != refAttr.getType()) {
2195 return emitDefiniteFailure()
2196 << "mismatching integer attribute types in parameter #" << i;
2197 }
2198 APInt value = intAttr.getValue();
2199 APInt refValue = refAttr.getValue();
2200
2201 // TODO: this copy will not be necessary in C++20.
2202 int64_t position = i;
2203 auto reportError = [&](StringRef direction) {
2204 DiagnosedSilenceableFailure diag =
2205 emitSilenceableError() << "expected parameter to be " << direction
2206 << " " << signedAPIntAsString(refValue)
2207 << ", got " << signedAPIntAsString(value);
2208 diag.attachNote(loc: getParam().getLoc())
2209 << "value # " << position
2210 << " associated with the parameter defined here";
2211 return diag;
2212 };
2213
2214 switch (getPredicate()) {
2215 case MatchCmpIPredicate::eq:
2216 if (value.eq(RHS: refValue))
2217 break;
2218 return reportError("equal to");
2219 case MatchCmpIPredicate::ne:
2220 if (value.ne(RHS: refValue))
2221 break;
2222 return reportError("not equal to");
2223 case MatchCmpIPredicate::lt:
2224 if (value.slt(RHS: refValue))
2225 break;
2226 return reportError("less than");
2227 case MatchCmpIPredicate::le:
2228 if (value.sle(RHS: refValue))
2229 break;
2230 return reportError("less than or equal to");
2231 case MatchCmpIPredicate::gt:
2232 if (value.sgt(RHS: refValue))
2233 break;
2234 return reportError("greater than");
2235 case MatchCmpIPredicate::ge:
2236 if (value.sge(RHS: refValue))
2237 break;
2238 return reportError("greater than or equal to");
2239 }
2240 }
2241 return DiagnosedSilenceableFailure::success();
2242}
2243
2244void transform::MatchParamCmpIOp::getEffects(
2245 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
2246 onlyReadsHandle(handles: getParamMutable(), effects);
2247 onlyReadsHandle(handles: getReferenceMutable(), effects);
2248}
2249
2250//===----------------------------------------------------------------------===//
2251// ParamConstantOp
2252//===----------------------------------------------------------------------===//
2253
2254DiagnosedSilenceableFailure
2255transform::ParamConstantOp::apply(transform::TransformRewriter &rewriter,
2256 transform::TransformResults &results,
2257 transform::TransformState &state) {
2258 results.setParams(value: cast<OpResult>(Val: getParam()), params: {getValue()});
2259 return DiagnosedSilenceableFailure::success();
2260}
2261
2262//===----------------------------------------------------------------------===//
2263// MergeHandlesOp
2264//===----------------------------------------------------------------------===//
2265
2266DiagnosedSilenceableFailure
2267transform::MergeHandlesOp::apply(transform::TransformRewriter &rewriter,
2268 transform::TransformResults &results,
2269 transform::TransformState &state) {
2270 ValueRange handles = getHandles();
2271 if (isa<TransformHandleTypeInterface>(Val: handles.front().getType())) {
2272 SmallVector<Operation *> operations;
2273 for (Value operand : handles)
2274 llvm::append_range(C&: operations, R: state.getPayloadOps(value: operand));
2275 if (!getDeduplicate()) {
2276 results.set(value: llvm::cast<OpResult>(Val: getResult()), ops&: operations);
2277 return DiagnosedSilenceableFailure::success();
2278 }
2279
2280 SetVector<Operation *> uniqued(llvm::from_range, operations);
2281 results.set(value: llvm::cast<OpResult>(Val: getResult()), ops: uniqued.getArrayRef());
2282 return DiagnosedSilenceableFailure::success();
2283 }
2284
2285 if (llvm::isa<TransformParamTypeInterface>(Val: handles.front().getType())) {
2286 SmallVector<Attribute> attrs;
2287 for (Value attribute : handles)
2288 llvm::append_range(C&: attrs, R: state.getParams(value: attribute));
2289 if (!getDeduplicate()) {
2290 results.setParams(value: cast<OpResult>(Val: getResult()), params: attrs);
2291 return DiagnosedSilenceableFailure::success();
2292 }
2293
2294 SetVector<Attribute> uniqued(llvm::from_range, attrs);
2295 results.setParams(value: cast<OpResult>(Val: getResult()), params: uniqued.getArrayRef());
2296 return DiagnosedSilenceableFailure::success();
2297 }
2298
2299 assert(
2300 llvm::isa<TransformValueHandleTypeInterface>(handles.front().getType()) &&
2301 "expected value handle type");
2302 SmallVector<Value> payloadValues;
2303 for (Value value : handles)
2304 llvm::append_range(C&: payloadValues, R: state.getPayloadValues(handleValue: value));
2305 if (!getDeduplicate()) {
2306 results.setValues(handle: cast<OpResult>(Val: getResult()), values&: payloadValues);
2307 return DiagnosedSilenceableFailure::success();
2308 }
2309
2310 SetVector<Value> uniqued(llvm::from_range, payloadValues);
2311 results.setValues(handle: cast<OpResult>(Val: getResult()), values: uniqued.getArrayRef());
2312 return DiagnosedSilenceableFailure::success();
2313}
2314
2315bool transform::MergeHandlesOp::allowsRepeatedHandleOperands() {
2316 // Handles may be the same if deduplicating is enabled.
2317 return getDeduplicate();
2318}
2319
2320void transform::MergeHandlesOp::getEffects(
2321 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
2322 onlyReadsHandle(handles: getHandlesMutable(), effects);
2323 producesHandle(handles: getOperation()->getOpResults(), effects);
2324
2325 // There are no effects on the Payload IR as this is only a handle
2326 // manipulation.
2327}
2328
2329OpFoldResult transform::MergeHandlesOp::fold(FoldAdaptor adaptor) {
2330 if (getDeduplicate() || getHandles().size() != 1)
2331 return {};
2332
2333 // If deduplication is not required and there is only one operand, it can be
2334 // used directly instead of merging.
2335 return getHandles().front();
2336}
2337
2338//===----------------------------------------------------------------------===//
2339// NamedSequenceOp
2340//===----------------------------------------------------------------------===//
2341
2342DiagnosedSilenceableFailure
2343transform::NamedSequenceOp::apply(transform::TransformRewriter &rewriter,
2344 transform::TransformResults &results,
2345 transform::TransformState &state) {
2346 if (isExternal())
2347 return emitDefiniteFailure() << "unresolved external named sequence";
2348
2349 // Map the entry block argument to the list of operations.
2350 // Note: this is the same implementation as PossibleTopLevelTransformOp but
2351 // without attaching the interface / trait since that is tailored to a
2352 // dangling top-level op that does not get "called".
2353 auto scope = state.make_region_scope(region&: getBody());
2354 if (failed(Result: detail::mapPossibleTopLevelTransformOpBlockArguments(
2355 state, op: this->getOperation(), region&: getBody())))
2356 return DiagnosedSilenceableFailure::definiteFailure();
2357
2358 return applySequenceBlock(block&: getBody().front(),
2359 mode: FailurePropagationMode::Propagate, state, results);
2360}
2361
2362void transform::NamedSequenceOp::getEffects(
2363 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {}
2364
2365ParseResult transform::NamedSequenceOp::parse(OpAsmParser &parser,
2366 OperationState &result) {
2367 return function_interface_impl::parseFunctionOp(
2368 parser, result, /*allowVariadic=*/false,
2369 typeAttrName: getFunctionTypeAttrName(name: result.name),
2370 funcTypeBuilder: [](Builder &builder, ArrayRef<Type> inputs, ArrayRef<Type> results,
2371 function_interface_impl::VariadicFlag,
2372 std::string &) { return builder.getFunctionType(inputs, results); },
2373 argAttrsName: getArgAttrsAttrName(name: result.name), resAttrsName: getResAttrsAttrName(name: result.name));
2374}
2375
2376void transform::NamedSequenceOp::print(OpAsmPrinter &printer) {
2377 function_interface_impl::printFunctionOp(
2378 p&: printer, op: cast<FunctionOpInterface>(Val: getOperation()), /*isVariadic=*/false,
2379 typeAttrName: getFunctionTypeAttrName().getValue(), argAttrsName: getArgAttrsAttrName(),
2380 resAttrsName: getResAttrsAttrName());
2381}
2382
2383/// Verifies that a symbol function-like transform dialect operation has the
2384/// signature and the terminator that have conforming types, i.e., types
2385/// implementing the same transform dialect type interface. If `allowExternal`
2386/// is set, allow external symbols (declarations) and don't check the terminator
2387/// as it may not exist.
2388static DiagnosedSilenceableFailure
2389verifyYieldingSingleBlockOp(FunctionOpInterface op, bool allowExternal) {
2390 if (auto parent = op->getParentOfType<transform::TransformOpInterface>()) {
2391 DiagnosedSilenceableFailure diag =
2392 emitSilenceableFailure(op)
2393 << "cannot be defined inside another transform op";
2394 diag.attachNote(loc: parent.getLoc()) << "ancestor transform op";
2395 return diag;
2396 }
2397
2398 if (op.isExternal() || op.getFunctionBody().empty()) {
2399 if (allowExternal)
2400 return DiagnosedSilenceableFailure::success();
2401
2402 return emitSilenceableFailure(op) << "cannot be external";
2403 }
2404
2405 if (op.getFunctionBody().front().empty())
2406 return emitSilenceableFailure(op) << "expected a non-empty body block";
2407
2408 Operation *terminator = &op.getFunctionBody().front().back();
2409 if (!isa<transform::YieldOp>(Val: terminator)) {
2410 DiagnosedSilenceableFailure diag = emitSilenceableFailure(op)
2411 << "expected '"
2412 << transform::YieldOp::getOperationName()
2413 << "' as terminator";
2414 diag.attachNote(loc: terminator->getLoc()) << "terminator";
2415 return diag;
2416 }
2417
2418 if (terminator->getNumOperands() != op.getResultTypes().size()) {
2419 return emitSilenceableFailure(op: terminator)
2420 << "expected terminator to have as many operands as the parent op "
2421 "has results";
2422 }
2423 for (auto [i, operandType, resultType] : llvm::zip_equal(
2424 t: llvm::seq<unsigned>(Begin: 0, End: terminator->getNumOperands()),
2425 u: terminator->getOperands().getType(), args: op.getResultTypes())) {
2426 if (operandType == resultType)
2427 continue;
2428 return emitSilenceableFailure(op: terminator)
2429 << "the type of the terminator operand #" << i
2430 << " must match the type of the corresponding parent op result ("
2431 << operandType << " vs " << resultType << ")";
2432 }
2433
2434 return DiagnosedSilenceableFailure::success();
2435}
2436
2437/// Verification of a NamedSequenceOp. This does not report the error
2438/// immediately, so it can be used to check for op's well-formedness before the
2439/// verifier runs, e.g., during trait verification.
2440static DiagnosedSilenceableFailure
2441verifyNamedSequenceOp(transform::NamedSequenceOp op, bool emitWarnings) {
2442 if (Operation *parent = op->getParentWithTrait<OpTrait::SymbolTable>()) {
2443 if (!parent->getAttr(
2444 name: transform::TransformDialect::kWithNamedSequenceAttrName)) {
2445 DiagnosedSilenceableFailure diag =
2446 emitSilenceableFailure(op)
2447 << "expects the parent symbol table to have the '"
2448 << transform::TransformDialect::kWithNamedSequenceAttrName
2449 << "' attribute";
2450 diag.attachNote(loc: parent->getLoc()) << "symbol table operation";
2451 return diag;
2452 }
2453 }
2454
2455 if (auto parent = op->getParentOfType<transform::TransformOpInterface>()) {
2456 DiagnosedSilenceableFailure diag =
2457 emitSilenceableFailure(op)
2458 << "cannot be defined inside another transform op";
2459 diag.attachNote(loc: parent.getLoc()) << "ancestor transform op";
2460 return diag;
2461 }
2462
2463 if (op.isExternal() || op.getBody().empty())
2464 return verifyFunctionLikeConsumeAnnotations(op: cast<FunctionOpInterface>(Val&: *op),
2465 emitWarnings);
2466
2467 if (op.getBody().front().empty())
2468 return emitSilenceableFailure(op) << "expected a non-empty body block";
2469
2470 // Check that all operations in the body implement TransformOpInterface
2471 for (Operation &child : op.getBody().front().without_terminator()) {
2472 if (!isa<transform::TransformOpInterface>(Val: child)) {
2473 DiagnosedSilenceableFailure diag =
2474 emitSilenceableFailure(op: &child)
2475 << "expected children ops to implement TransformOpInterface";
2476 diag.attachNote(loc: child.getLoc()) << "op without interface";
2477 return diag;
2478 }
2479 }
2480
2481 Operation *terminator = &op.getBody().front().back();
2482 if (!isa<transform::YieldOp>(Val: terminator)) {
2483 DiagnosedSilenceableFailure diag = emitSilenceableFailure(op)
2484 << "expected '"
2485 << transform::YieldOp::getOperationName()
2486 << "' as terminator";
2487 diag.attachNote(loc: terminator->getLoc()) << "terminator";
2488 return diag;
2489 }
2490
2491 if (terminator->getNumOperands() != op.getFunctionType().getNumResults()) {
2492 return emitSilenceableFailure(op: terminator)
2493 << "expected terminator to have as many operands as the parent op "
2494 "has results";
2495 }
2496 for (auto [i, operandType, resultType] :
2497 llvm::zip_equal(t: llvm::seq<unsigned>(Begin: 0, End: terminator->getNumOperands()),
2498 u: terminator->getOperands().getType(),
2499 args: op.getFunctionType().getResults())) {
2500 if (operandType == resultType)
2501 continue;
2502 return emitSilenceableFailure(op: terminator)
2503 << "the type of the terminator operand #" << i
2504 << " must match the type of the corresponding parent op result ("
2505 << operandType << " vs " << resultType << ")";
2506 }
2507
2508 auto funcOp = cast<FunctionOpInterface>(Val&: *op);
2509 DiagnosedSilenceableFailure diag =
2510 verifyFunctionLikeConsumeAnnotations(op: funcOp, emitWarnings);
2511 if (!diag.succeeded())
2512 return diag;
2513
2514 return verifyYieldingSingleBlockOp(op: funcOp,
2515 /*allowExternal=*/true);
2516}
2517
2518LogicalResult transform::NamedSequenceOp::verify() {
2519 // Actual verification happens in a separate function for reusability.
2520 return verifyNamedSequenceOp(op: *this, /*emitWarnings=*/true).checkAndReport();
2521}
2522
2523template <typename FnTy>
2524static void buildSequenceBody(OpBuilder &builder, OperationState &state,
2525 Type bbArgType, TypeRange extraBindingTypes,
2526 FnTy bodyBuilder) {
2527 SmallVector<Type> types;
2528 types.reserve(N: 1 + extraBindingTypes.size());
2529 types.push_back(Elt: bbArgType);
2530 llvm::append_range(C&: types, R&: extraBindingTypes);
2531
2532 OpBuilder::InsertionGuard guard(builder);
2533 Region *region = state.regions.back().get();
2534 Block *bodyBlock =
2535 builder.createBlock(parent: region, insertPt: region->begin(), argTypes: types,
2536 locs: SmallVector<Location>(types.size(), state.location));
2537
2538 // Populate body.
2539 builder.setInsertionPointToStart(bodyBlock);
2540 if constexpr (llvm::function_traits<FnTy>::num_args == 3) {
2541 bodyBuilder(builder, state.location, bodyBlock->getArgument(i: 0));
2542 } else {
2543 bodyBuilder(builder, state.location, bodyBlock->getArgument(i: 0),
2544 bodyBlock->getArguments().drop_front());
2545 }
2546}
2547
2548void transform::NamedSequenceOp::build(OpBuilder &builder,
2549 OperationState &state, StringRef symName,
2550 Type rootType, TypeRange resultTypes,
2551 SequenceBodyBuilderFn bodyBuilder,
2552 ArrayRef<NamedAttribute> attrs,
2553 ArrayRef<DictionaryAttr> argAttrs) {
2554 state.getOrAddProperties<Properties>().sym_name =
2555 builder.getStringAttr(bytes: symName);
2556 state.addAttribute(name: getFunctionTypeAttrName(name: state.name),
2557 attr: TypeAttr::get(type: FunctionType::get(context: builder.getContext(),
2558 inputs: rootType, results: resultTypes)));
2559 state.attributes.append(inStart: attrs.begin(), inEnd: attrs.end());
2560 state.addRegion();
2561
2562 buildSequenceBody(builder, state, bbArgType: rootType,
2563 /*extraBindingTypes=*/TypeRange(), bodyBuilder);
2564}
2565
2566//===----------------------------------------------------------------------===//
2567// NumAssociationsOp
2568//===----------------------------------------------------------------------===//
2569
2570DiagnosedSilenceableFailure
2571transform::NumAssociationsOp::apply(transform::TransformRewriter &rewriter,
2572 transform::TransformResults &results,
2573 transform::TransformState &state) {
2574 size_t numAssociations =
2575 llvm::TypeSwitch<Type, size_t>(getHandle().getType())
2576 .Case(caseFn: [&](TransformHandleTypeInterface opHandle) {
2577 return llvm::range_size(Range: state.getPayloadOps(value: getHandle()));
2578 })
2579 .Case(caseFn: [&](TransformValueHandleTypeInterface valueHandle) {
2580 return llvm::range_size(Range: state.getPayloadValues(handleValue: getHandle()));
2581 })
2582 .Case(caseFn: [&](TransformParamTypeInterface param) {
2583 return llvm::range_size(Range: state.getParams(value: getHandle()));
2584 })
2585 .DefaultUnreachable(message: "unknown kind of transform dialect type");
2586 results.setParams(value: cast<OpResult>(Val: getNum()),
2587 params: rewriter.getI64IntegerAttr(value: numAssociations));
2588 return DiagnosedSilenceableFailure::success();
2589}
2590
2591LogicalResult transform::NumAssociationsOp::verify() {
2592 // Verify that the result type accepts an i64 attribute as payload.
2593 auto resultType = cast<TransformParamTypeInterface>(Val: getNum().getType());
2594 return resultType
2595 .checkPayload(loc: getLoc(), payload: {Builder(getContext()).getI64IntegerAttr(value: 0)})
2596 .checkAndReport();
2597}
2598
2599//===----------------------------------------------------------------------===//
2600// SelectOp
2601//===----------------------------------------------------------------------===//
2602
2603DiagnosedSilenceableFailure
2604transform::SelectOp::apply(transform::TransformRewriter &rewriter,
2605 transform::TransformResults &results,
2606 transform::TransformState &state) {
2607 SmallVector<Operation *> result;
2608 auto payloadOps = state.getPayloadOps(value: getTarget());
2609 for (Operation *op : payloadOps) {
2610 if (op->getName().getStringRef() == getOpName())
2611 result.push_back(Elt: op);
2612 }
2613 results.set(value: cast<OpResult>(Val: getResult()), ops&: result);
2614 return DiagnosedSilenceableFailure::success();
2615}
2616
2617//===----------------------------------------------------------------------===//
2618// SplitHandleOp
2619//===----------------------------------------------------------------------===//
2620
2621void transform::SplitHandleOp::build(OpBuilder &builder, OperationState &result,
2622 Value target, int64_t numResultHandles) {
2623 result.addOperands(newOperands: target);
2624 result.addTypes(newTypes: SmallVector<Type>(numResultHandles, target.getType()));
2625}
2626
2627DiagnosedSilenceableFailure
2628transform::SplitHandleOp::apply(transform::TransformRewriter &rewriter,
2629 transform::TransformResults &results,
2630 transform::TransformState &state) {
2631 int64_t numPayloads =
2632 llvm::TypeSwitch<Type, int64_t>(getHandle().getType())
2633 .Case(caseFn: [&](TransformHandleTypeInterface x) {
2634 return llvm::range_size(Range: state.getPayloadOps(value: getHandle()));
2635 })
2636 .Case(caseFn: [&](TransformValueHandleTypeInterface x) {
2637 return llvm::range_size(Range: state.getPayloadValues(handleValue: getHandle()));
2638 })
2639 .Case(caseFn: [&](TransformParamTypeInterface x) {
2640 return llvm::range_size(Range: state.getParams(value: getHandle()));
2641 })
2642 .DefaultUnreachable(message: "unknown transform dialect type interface");
2643
2644 auto produceNumOpsError = [&]() {
2645 return emitSilenceableError()
2646 << getHandle() << " expected to contain " << this->getNumResults()
2647 << " payloads but it contains " << numPayloads << " payloads";
2648 };
2649
2650 // Fail if there are more payload ops than results and no overflow result was
2651 // specified.
2652 if (numPayloads > getNumResults() && !getOverflowResult().has_value())
2653 return produceNumOpsError();
2654
2655 // Fail if there are more results than payload ops. Unless:
2656 // - "fail_on_payload_too_small" is set to "false", or
2657 // - "pass_through_empty_handle" is set to "true" and there are 0 payload ops.
2658 if (numPayloads < getNumResults() && getFailOnPayloadTooSmall() &&
2659 (numPayloads != 0 || !getPassThroughEmptyHandle()))
2660 return produceNumOpsError();
2661
2662 // Distribute payloads.
2663 SmallVector<SmallVector<MappedValue, 1>> resultHandles(getNumResults(), {});
2664 if (getOverflowResult())
2665 resultHandles[*getOverflowResult()].reserve(N: numPayloads - getNumResults());
2666
2667 auto container = [&]() {
2668 if (isa<TransformHandleTypeInterface>(Val: getHandle().getType())) {
2669 return llvm::map_to_vector(
2670 C: state.getPayloadOps(value: getHandle()),
2671 F: [](Operation *op) -> MappedValue { return op; });
2672 }
2673 if (isa<TransformValueHandleTypeInterface>(Val: getHandle().getType())) {
2674 return llvm::map_to_vector(C: state.getPayloadValues(handleValue: getHandle()),
2675 F: [](Value v) -> MappedValue { return v; });
2676 }
2677 assert(isa<TransformParamTypeInterface>(getHandle().getType()) &&
2678 "unsupported kind of transform dialect type");
2679 return llvm::map_to_vector(C: state.getParams(value: getHandle()),
2680 F: [](Attribute a) -> MappedValue { return a; });
2681 }();
2682
2683 for (auto &&en : llvm::enumerate(First&: container)) {
2684 int64_t resultNum = en.index();
2685 if (resultNum >= getNumResults())
2686 resultNum = *getOverflowResult();
2687 resultHandles[resultNum].push_back(Elt: en.value());
2688 }
2689
2690 // Set transform op results.
2691 for (auto &&it : llvm::enumerate(First&: resultHandles))
2692 results.setMappedValues(handle: llvm::cast<OpResult>(Val: getResult(i: it.index())),
2693 values: it.value());
2694
2695 return DiagnosedSilenceableFailure::success();
2696}
2697
2698void transform::SplitHandleOp::getEffects(
2699 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
2700 onlyReadsHandle(handles: getHandleMutable(), effects);
2701 producesHandle(handles: getOperation()->getOpResults(), effects);
2702 // There are no effects on the Payload IR as this is only a handle
2703 // manipulation.
2704}
2705
2706LogicalResult transform::SplitHandleOp::verify() {
2707 if (getOverflowResult().has_value() &&
2708 !(*getOverflowResult() < getNumResults()))
2709 return emitOpError(message: "overflow_result is not a valid result index");
2710
2711 for (Type resultType : getResultTypes()) {
2712 if (implementSameTransformInterface(t1: getHandle().getType(), t2: resultType))
2713 continue;
2714
2715 return emitOpError(message: "expects result types to implement the same transform "
2716 "interface as the operand type");
2717 }
2718
2719 return success();
2720}
2721
2722//===----------------------------------------------------------------------===//
2723// PayloadOp
2724//===----------------------------------------------------------------------===//
2725
2726void transform::PayloadOp::getCheckedNormalForms(
2727 SmallVectorImpl<NormalFormAttrInterface> &normalForms) {
2728 llvm::append_range(C&: normalForms,
2729 R: getNormalForms().getAsRange<NormalFormAttrInterface>());
2730}
2731
2732//===----------------------------------------------------------------------===//
2733// ReplicateOp
2734//===----------------------------------------------------------------------===//
2735
2736DiagnosedSilenceableFailure
2737transform::ReplicateOp::apply(transform::TransformRewriter &rewriter,
2738 transform::TransformResults &results,
2739 transform::TransformState &state) {
2740 unsigned numRepetitions = llvm::range_size(Range: state.getPayloadOps(value: getPattern()));
2741 for (const auto &en : llvm::enumerate(First: getHandles())) {
2742 Value handle = en.value();
2743 if (isa<TransformHandleTypeInterface>(Val: handle.getType())) {
2744 SmallVector<Operation *> current =
2745 llvm::to_vector(Range: state.getPayloadOps(value: handle));
2746 SmallVector<Operation *> payload;
2747 payload.reserve(N: numRepetitions * current.size());
2748 for (unsigned i = 0; i < numRepetitions; ++i)
2749 llvm::append_range(C&: payload, R&: current);
2750 results.set(value: llvm::cast<OpResult>(Val: getReplicated()[en.index()]), ops&: payload);
2751 } else {
2752 assert(llvm::isa<TransformParamTypeInterface>(handle.getType()) &&
2753 "expected param type");
2754 ArrayRef<Attribute> current = state.getParams(value: handle);
2755 SmallVector<Attribute> params;
2756 params.reserve(N: numRepetitions * current.size());
2757 for (unsigned i = 0; i < numRepetitions; ++i)
2758 llvm::append_range(C&: params, R&: current);
2759 results.setParams(value: llvm::cast<OpResult>(Val: getReplicated()[en.index()]),
2760 params);
2761 }
2762 }
2763 return DiagnosedSilenceableFailure::success();
2764}
2765
2766void transform::ReplicateOp::getEffects(
2767 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
2768 onlyReadsHandle(handles: getPatternMutable(), effects);
2769 onlyReadsHandle(handles: getHandlesMutable(), effects);
2770 producesHandle(handles: getOperation()->getOpResults(), effects);
2771}
2772
2773//===----------------------------------------------------------------------===//
2774// SequenceOp
2775//===----------------------------------------------------------------------===//
2776
2777DiagnosedSilenceableFailure
2778transform::SequenceOp::apply(transform::TransformRewriter &rewriter,
2779 transform::TransformResults &results,
2780 transform::TransformState &state) {
2781 // Map the entry block argument to the list of operations.
2782 auto scope = state.make_region_scope(region&: *getBodyBlock()->getParent());
2783 if (failed(Result: mapBlockArguments(state)))
2784 return DiagnosedSilenceableFailure::definiteFailure();
2785
2786 return applySequenceBlock(block&: *getBodyBlock(), mode: getFailurePropagationMode(), state,
2787 results);
2788}
2789
2790static ParseResult parseSequenceOpOperands(
2791 OpAsmParser &parser, std::optional<OpAsmParser::UnresolvedOperand> &root,
2792 Type &rootType,
2793 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &extraBindings,
2794 SmallVectorImpl<Type> &extraBindingTypes) {
2795 OpAsmParser::UnresolvedOperand rootOperand;
2796 OptionalParseResult hasRoot = parser.parseOptionalOperand(result&: rootOperand);
2797 if (!hasRoot.has_value()) {
2798 root = std::nullopt;
2799 return success();
2800 }
2801 if (failed(Result: hasRoot.value()))
2802 return failure();
2803 root = rootOperand;
2804
2805 if (succeeded(Result: parser.parseOptionalComma())) {
2806 if (failed(Result: parser.parseOperandList(result&: extraBindings)))
2807 return failure();
2808 }
2809 if (failed(Result: parser.parseColon()))
2810 return failure();
2811
2812 // The paren is truly optional.
2813 (void)parser.parseOptionalLParen();
2814
2815 if (failed(Result: parser.parseType(result&: rootType))) {
2816 return failure();
2817 }
2818
2819 if (!extraBindings.empty()) {
2820 if (parser.parseComma() || parser.parseTypeList(result&: extraBindingTypes))
2821 return failure();
2822 }
2823
2824 if (extraBindingTypes.size() != extraBindings.size()) {
2825 return parser.emitError(loc: parser.getNameLoc(),
2826 message: "expected types to be provided for all operands");
2827 }
2828
2829 // The paren is truly optional.
2830 (void)parser.parseOptionalRParen();
2831 return success();
2832}
2833
2834static void printSequenceOpOperands(OpAsmPrinter &printer, Operation *op,
2835 Value root, Type rootType,
2836 ValueRange extraBindings,
2837 TypeRange extraBindingTypes) {
2838 if (!root)
2839 return;
2840
2841 printer << root;
2842 bool hasExtras = !extraBindings.empty();
2843 if (hasExtras) {
2844 printer << ", ";
2845 printer.printOperands(container: extraBindings);
2846 }
2847
2848 printer << " : ";
2849 if (hasExtras)
2850 printer << "(";
2851
2852 printer << rootType;
2853 if (hasExtras)
2854 printer << ", " << llvm::interleaved(R: extraBindingTypes) << ')';
2855}
2856
2857/// Returns `true` if the given op operand may be consuming the handle value in
2858/// the Transform IR. That is, if it may have a Free effect on it.
2859static bool isValueUsePotentialConsumer(OpOperand &use) {
2860 // Conservatively assume the effect being present in absence of the interface.
2861 auto iface = dyn_cast<transform::TransformOpInterface>(Val: use.getOwner());
2862 if (!iface)
2863 return true;
2864
2865 return isHandleConsumed(handle: use.get(), transform: iface);
2866}
2867
2868static LogicalResult
2869checkDoubleConsume(Value value,
2870 function_ref<InFlightDiagnostic()> reportError) {
2871 OpOperand *potentialConsumer = nullptr;
2872 for (OpOperand &use : value.getUses()) {
2873 if (!isValueUsePotentialConsumer(use))
2874 continue;
2875
2876 if (!potentialConsumer) {
2877 potentialConsumer = &use;
2878 continue;
2879 }
2880
2881 InFlightDiagnostic diag = reportError()
2882 << " has more than one potential consumer";
2883 diag.attachNote(noteLoc: potentialConsumer->getOwner()->getLoc())
2884 << "used here as operand #" << potentialConsumer->getOperandNumber();
2885 diag.attachNote(noteLoc: use.getOwner()->getLoc())
2886 << "used here as operand #" << use.getOperandNumber();
2887 return diag;
2888 }
2889
2890 return success();
2891}
2892
2893LogicalResult transform::SequenceOp::verify() {
2894 assert(getBodyBlock()->getNumArguments() >= 1 &&
2895 "the number of arguments must have been verified to be more than 1 by "
2896 "PossibleTopLevelTransformOpTrait");
2897
2898 if (!getRoot() && !getExtraBindings().empty()) {
2899 return emitOpError()
2900 << "does not expect extra operands when used as top-level";
2901 }
2902
2903 // Check if a block argument has more than one consuming use.
2904 for (BlockArgument arg : getBodyBlock()->getArguments()) {
2905 if (failed(Result: checkDoubleConsume(value: arg, reportError: [this, arg]() {
2906 return (emitOpError() << "block argument #" << arg.getArgNumber());
2907 }))) {
2908 return failure();
2909 }
2910 }
2911
2912 // Check properties of the nested operations they cannot check themselves.
2913 for (Operation &child : *getBodyBlock()) {
2914 if (!isa<TransformOpInterface>(Val: child) &&
2915 &child != &getBodyBlock()->back()) {
2916 InFlightDiagnostic diag =
2917 emitOpError()
2918 << "expected children ops to implement TransformOpInterface";
2919 diag.attachNote(noteLoc: child.getLoc()) << "op without interface";
2920 return diag;
2921 }
2922
2923 for (OpResult result : child.getResults()) {
2924 auto report = [&]() {
2925 return (child.emitError() << "result #" << result.getResultNumber());
2926 };
2927 if (failed(Result: checkDoubleConsume(value: result, reportError: report)))
2928 return failure();
2929 }
2930 }
2931
2932 if (!getBodyBlock()->mightHaveTerminator())
2933 return emitOpError() << "expects to have a terminator in the body";
2934
2935 if (getBodyBlock()->getTerminator()->getOperandTypes() !=
2936 getOperation()->getResultTypes()) {
2937 InFlightDiagnostic diag = emitOpError()
2938 << "expects the types of the terminator operands "
2939 "to match the types of the result";
2940 diag.attachNote(noteLoc: getBodyBlock()->getTerminator()->getLoc()) << "terminator";
2941 return diag;
2942 }
2943 return success();
2944}
2945
2946void transform::SequenceOp::getEffects(
2947 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
2948 getPotentialTopLevelEffects(effects);
2949}
2950
2951OperandRange
2952transform::SequenceOp::getEntrySuccessorOperands(RegionSuccessor successor) {
2953 assert(successor.getSuccessor() == &getBody() && "unexpected region index");
2954 if (getOperation()->getNumOperands() > 0)
2955 return getOperation()->getOperands();
2956 return OperandRange(getOperation()->operand_end(),
2957 getOperation()->operand_end());
2958}
2959
2960void transform::SequenceOp::getSuccessorRegions(
2961 RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &regions) {
2962 if (point.isParent()) {
2963 Region *bodyRegion = &getBody();
2964 regions.emplace_back(Args&: bodyRegion);
2965 return;
2966 }
2967
2968 assert(point.getTerminatorPredecessorOrNull()->getParentRegion() ==
2969 &getBody() &&
2970 "unexpected region index");
2971 regions.push_back(Elt: RegionSuccessor(getOperation()));
2972}
2973
2974ValueRange
2975transform::SequenceOp::getSuccessorInputs(RegionSuccessor successor) {
2976 if (getNumOperands() == 0)
2977 return ValueRange();
2978 if (successor.isOperation())
2979 return getResults();
2980 return getBody().getArguments();
2981}
2982
2983void transform::SequenceOp::getRegionInvocationBounds(
2984 ArrayRef<Attribute> operands, SmallVectorImpl<InvocationBounds> &bounds) {
2985 (void)operands;
2986 bounds.emplace_back(Args: 1, Args: 1);
2987}
2988
2989void transform::SequenceOp::build(OpBuilder &builder, OperationState &state,
2990 TypeRange resultTypes,
2991 FailurePropagationMode failurePropagationMode,
2992 Value root,
2993 SequenceBodyBuilderFn bodyBuilder) {
2994 build(odsBuilder&: builder, odsState&: state, results: resultTypes, failure_propagation_mode: failurePropagationMode, root,
2995 /*extra_bindings=*/ValueRange());
2996 Type bbArgType = root.getType();
2997 buildSequenceBody(builder, state, bbArgType,
2998 /*extraBindingTypes=*/TypeRange(), bodyBuilder);
2999}
3000
3001void transform::SequenceOp::build(OpBuilder &builder, OperationState &state,
3002 TypeRange resultTypes,
3003 FailurePropagationMode failurePropagationMode,
3004 Value root, ValueRange extraBindings,
3005 SequenceBodyBuilderArgsFn bodyBuilder) {
3006 build(odsBuilder&: builder, odsState&: state, results: resultTypes, failure_propagation_mode: failurePropagationMode, root,
3007 extra_bindings: extraBindings);
3008 buildSequenceBody(builder, state, bbArgType: root.getType(), extraBindingTypes: extraBindings.getTypes(),
3009 bodyBuilder);
3010}
3011
3012void transform::SequenceOp::build(OpBuilder &builder, OperationState &state,
3013 TypeRange resultTypes,
3014 FailurePropagationMode failurePropagationMode,
3015 Type bbArgType,
3016 SequenceBodyBuilderFn bodyBuilder) {
3017 build(odsBuilder&: builder, odsState&: state, results: resultTypes, failure_propagation_mode: failurePropagationMode, /*root=*/Value(),
3018 /*extra_bindings=*/ValueRange());
3019 buildSequenceBody(builder, state, bbArgType,
3020 /*extraBindingTypes=*/TypeRange(), bodyBuilder);
3021}
3022
3023void transform::SequenceOp::build(OpBuilder &builder, OperationState &state,
3024 TypeRange resultTypes,
3025 FailurePropagationMode failurePropagationMode,
3026 Type bbArgType, TypeRange extraBindingTypes,
3027 SequenceBodyBuilderArgsFn bodyBuilder) {
3028 build(odsBuilder&: builder, odsState&: state, results: resultTypes, failure_propagation_mode: failurePropagationMode, /*root=*/Value(),
3029 /*extra_bindings=*/ValueRange());
3030 buildSequenceBody(builder, state, bbArgType, extraBindingTypes, bodyBuilder);
3031}
3032
3033//===----------------------------------------------------------------------===//
3034// PrintOp
3035//===----------------------------------------------------------------------===//
3036
3037void transform::PrintOp::build(OpBuilder &builder, OperationState &result,
3038 StringRef name) {
3039 if (!name.empty())
3040 result.getOrAddProperties<Properties>().name = builder.getStringAttr(bytes: name);
3041}
3042
3043void transform::PrintOp::build(OpBuilder &builder, OperationState &result,
3044 Value target, StringRef name) {
3045 result.addOperands(newOperands: {target});
3046 build(builder, result, name);
3047}
3048
3049DiagnosedSilenceableFailure
3050transform::PrintOp::apply(transform::TransformRewriter &rewriter,
3051 transform::TransformResults &results,
3052 transform::TransformState &state) {
3053 llvm::outs() << "[[[ IR printer: ";
3054 if (getName().has_value())
3055 llvm::outs() << *getName() << " ";
3056
3057 OpPrintingFlags printFlags;
3058 if (getAssumeVerified().value_or(u: false))
3059 printFlags.assumeVerified();
3060 if (getUseLocalScope().value_or(u: false))
3061 printFlags.useLocalScope();
3062 if (getSkipRegions().value_or(u: false))
3063 printFlags.skipRegions();
3064
3065 if (!getTarget()) {
3066 llvm::outs() << "top-level ]]]\n";
3067 state.getTopLevel()->print(os&: llvm::outs(), flags: printFlags);
3068 llvm::outs() << "\n";
3069 llvm::outs().flush();
3070 return DiagnosedSilenceableFailure::success();
3071 }
3072
3073 llvm::outs() << "]]]\n";
3074 for (Operation *target : state.getPayloadOps(value: getTarget())) {
3075 target->print(os&: llvm::outs(), flags: printFlags);
3076 llvm::outs() << "\n";
3077 }
3078
3079 llvm::outs().flush();
3080 return DiagnosedSilenceableFailure::success();
3081}
3082
3083void transform::PrintOp::getEffects(
3084 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
3085 // We don't really care about mutability here, but `getTarget` now
3086 // unconditionally casts to a specific type before verification could run
3087 // here.
3088 if (!getTargetMutable().empty())
3089 onlyReadsHandle(handles: getTargetMutable()[0], effects);
3090 onlyReadsPayload(effects);
3091
3092 // There is no resource for stderr file descriptor, so just declare print
3093 // writes into the default resource.
3094 effects.emplace_back(Args: MemoryEffects::Write::get());
3095}
3096
3097//===----------------------------------------------------------------------===//
3098// VerifyOp
3099//===----------------------------------------------------------------------===//
3100
3101DiagnosedSilenceableFailure
3102transform::VerifyOp::applyToOne(transform::TransformRewriter &rewriter,
3103 Operation *target,
3104 transform::ApplyToEachResultList &results,
3105 transform::TransformState &state) {
3106 if (failed(Result: ::mlir::verify(op: target))) {
3107 DiagnosedDefiniteFailure diag = emitDefiniteFailure()
3108 << "failed to verify payload op";
3109 diag.attachNote(loc: target->getLoc()) << "payload op";
3110 return diag;
3111 }
3112 return DiagnosedSilenceableFailure::success();
3113}
3114
3115void transform::VerifyOp::getEffects(
3116 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
3117 transform::onlyReadsHandle(handles: getTargetMutable(), effects);
3118}
3119
3120//===----------------------------------------------------------------------===//
3121// YieldOp
3122//===----------------------------------------------------------------------===//
3123
3124void transform::YieldOp::getEffects(
3125 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
3126 onlyReadsHandle(handles: getOperandsMutable(), effects);
3127}
3128