1//===- LinalgTransformOps.cpp - Implementation of Linalg match ops --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Linalg/TransformOps/LinalgMatchOps.h"
10#include "mlir/Analysis/SliceAnalysis.h"
11#include "mlir/Dialect/Linalg/IR/Linalg.h"
12#include "mlir/Dialect/Linalg/IR/LinalgInterfaces.h"
13#include "mlir/Dialect/Linalg/TransformOps/Syntax.h"
14#include "mlir/Dialect/Linalg/Utils/Utils.h"
15#include "mlir/Dialect/Transform/IR/TransformTypes.h"
16#include "mlir/Dialect/Transform/Interfaces/MatchInterfaces.h"
17#include "mlir/IR/BuiltinAttributes.h"
18#include "llvm/ADT/SmallVectorExtras.h"
19#include "llvm/Support/DebugLog.h"
20#include "llvm/Support/FormatVariadic.h"
21#include "llvm/Support/InterleavedRange.h"
22
23using namespace mlir;
24
25#define DEBUG_TYPE "linalg-transforms"
26
27//===----------------------------------------------------------------------===//
28// StructuredMatchOp
29//===----------------------------------------------------------------------===//
30
31DiagnosedSilenceableFailure transform::MatchStructuredOp::matchOperation(
32 Operation *current, transform::TransformResults &results,
33 transform::TransformState &state) {
34 // First, check if the payload operation is a structured Linalg operation.
35 if (!isa<linalg::LinalgOp>(Val: current)) {
36 if (getFailurePropagationMode().value_or(
37 u: FailurePropagationMode::Propagate) ==
38 FailurePropagationMode::Propagate) {
39 return emitSilenceableError() << "expected a Linalg op";
40 }
41 // If errors are suppressed, succeed and set all results to empty lists.
42 LDBG() << "optional nested matcher expected a Linalg op";
43 results.setRemainingToEmpty(cast<TransformOpInterface>(Val: getOperation()));
44 return DiagnosedSilenceableFailure::success();
45 }
46
47 // Bind `current` to the block argument.
48 auto scope = state.make_region_scope(region&: getBodyRegion());
49 if (failed(Result: state.mapBlockArgument(argument: getBody()->getArgument(i: 0),
50 values: MappedValue(current)))) {
51 return DiagnosedSilenceableFailure::definiteFailure();
52 }
53
54 for (Operation &nested : getBody()->without_terminator()) {
55 DiagnosedSilenceableFailure diag =
56 state.applyTransform(transform: cast<TransformOpInterface>(Val&: nested));
57 if (diag.isDefiniteFailure())
58 return diag;
59 if (diag.succeeded())
60 continue;
61
62 // If propagating errors, do this immediately.
63 assert(diag.isSilenceableFailure());
64 if (getFailurePropagationMode().value_or(
65 u: FailurePropagationMode::Propagate) ==
66 FailurePropagationMode::Propagate) {
67 return diag;
68 }
69
70 // If suppressing errors, print the message into the debug stream before
71 // silencing it. Then set all results value that are already known.
72 // Results come from the terminator operands, which may be defined in the
73 // (single) block of this operation or above it. When they are defined
74 // above, they are known to be mapped at this point per SSA dominance.
75 // When they are defined in this block, we additionally check if we have
76 // already applied the operation that defines them. If not, the
77 // corresponding results will be set to empty lists.
78 LDBG() << "optional nested matcher failed: " << diag.getMessage();
79 (void)diag.silence();
80 SmallVector<OpOperand *> undefinedOperands;
81 for (OpOperand &terminatorOperand :
82 getBody()->getTerminator()->getOpOperands()) {
83 Operation *definingOp = terminatorOperand.get().getDefiningOp();
84 if (!definingOp)
85 continue;
86 if (definingOp->getBlock() != getBody())
87 continue;
88 if (definingOp->isBeforeInBlock(other: &nested))
89 continue;
90
91 undefinedOperands.push_back(Elt: &terminatorOperand);
92 }
93
94 SmallVector<SmallVector<transform::MappedValue>> mappings;
95 auto filtered = llvm::make_filter_range(
96 Range: getBody()->getTerminator()->getOpOperands(), Pred: [&](OpOperand &opOperand) {
97 return !llvm::is_contained(Range&: undefinedOperands, Element: &opOperand);
98 });
99 SmallVector<Value> definedOperands = llvm::map_to_vector(
100 C&: filtered, F: [](OpOperand &opOperand) { return opOperand.get(); });
101 detail::prepareValueMappings(mappings, values: definedOperands, state);
102 for (auto &&[operand, mapping] : llvm::zip_equal(t&: filtered, u&: mappings)) {
103 results.setMappedValues(handle: getResults()[operand.getOperandNumber()],
104 values: mapping);
105 }
106 results.setRemainingToEmpty(cast<TransformOpInterface>(Val: getOperation()));
107 return DiagnosedSilenceableFailure::success();
108 }
109
110 // Set the results.
111 detail::forwardTerminatorOperands(block: getBody(), state, results);
112 return DiagnosedSilenceableFailure::success();
113}
114
115void transform::MatchStructuredOp::getEffects(
116 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
117 onlyReadsHandle(handles: getCurrentMutable(), effects);
118 onlyReadsPayload(effects);
119 producesHandle(handles: getOperation()->getOpResults(), effects);
120}
121
122LogicalResult transform::MatchStructuredOp::verify() {
123 if (getBody()->getNumArguments() != 1)
124 return emitOpError() << "expected one body argument";
125 if (!isa<TransformHandleTypeInterface>(Val: getBody()->getArgument(i: 0).getType())) {
126 return emitOpError() << "expected body argument to implement "
127 "TransformHandleTypeInterface";
128 }
129 for (Operation &nested : getBody()->without_terminator()) {
130 if (isa<MatchOpInterface>(Val: nested))
131 continue;
132 InFlightDiagnostic diag =
133 emitOpError()
134 << "expects nested operations to implement MatchOpInterface";
135 diag.attachNote(noteLoc: nested.getLoc()) << "offending operation";
136 return diag;
137 }
138 return success();
139}
140
141//===----------------------------------------------------------------------===//
142// StructuredOpPredicateOpTrait
143//===----------------------------------------------------------------------===//
144
145LogicalResult transform::detail::verifyStructuredOpPredicateOpTrait(
146 Operation *op, Value structuredOpHandle) {
147 if (!isa_and_nonnull<MatchStructuredOp>(Val: op->getParentOp())) {
148 return op->emitOpError() << "expects parent op to be '"
149 << MatchStructuredOp::getOperationName() << "'";
150 }
151
152 // Bail out here, let the verifier of the parent complain.
153 Operation *parent = op->getParentOp();
154 if (parent->getNumRegions() < 1 || parent->getRegion(index: 0).empty() ||
155 parent->getRegion(index: 0).front().getNumArguments() < 1)
156 return success();
157
158 if (structuredOpHandle != parent->getRegion(index: 0).front().getArgument(i: 0)) {
159 return op->emitOpError()
160 << "expected predicate to apply to the surrounding structured op";
161 }
162 return success();
163}
164
165//===----------------------------------------------------------------------===//
166// MatchStructuredBodyOp
167//===----------------------------------------------------------------------===//
168
169DiagnosedSilenceableFailure transform::MatchStructuredBodyOp::matchOperation(
170 Operation *current, transform::TransformResults &results,
171 transform::TransformState &state) {
172 auto linalgOp = cast<linalg::LinalgOp>(Val: current);
173 if (std::optional<uint64_t> position = getReductionPosition()) {
174 SmallVector<Operation *> combinerOps;
175 if (!matchReduction(iterCarriedArgs: linalgOp.getRegionOutputArgs(), redPos: *position,
176 combinerOps)) {
177 return emitSilenceableError() << "could not match reduction";
178 }
179 if (combinerOps.size() != 1) {
180 return emitSilenceableError() << "reduction combiner is not a single op";
181 }
182 return DiagnosedSilenceableFailure::success();
183 }
184 if (getPassthrough()) {
185 Block &body = linalgOp->getRegion(index: 0).front();
186 if (body.getTerminator()->getOperands() != linalgOp.getRegionInputArgs()) {
187 return emitSilenceableError() << "not a passthrough";
188 }
189 return DiagnosedSilenceableFailure::success();
190 }
191 if (getElementwise()) {
192 if (!isElementwise(op: linalgOp))
193 return emitSilenceableError() << "not elementwise";
194 return DiagnosedSilenceableFailure::success();
195 }
196 if (std::optional<ArrayAttr> contractionOps = getContraction()) {
197 Block &body = linalgOp->getRegion(index: 0).front();
198 std::string message;
199 llvm::raw_string_ostream os(message);
200 bool result = linalg::detail::isContractionBody(
201 block&: body,
202 isaPair: [&](Operation *elem, Operation *red) {
203 return elem->getName().getStringRef() ==
204 cast<StringAttr>(Val: (*contractionOps)[0]).getValue() &&
205 red->getName().getStringRef() ==
206 cast<StringAttr>(Val: (*contractionOps)[1]).getValue();
207 },
208 errs&: os);
209 if (result)
210 return DiagnosedSilenceableFailure::success();
211 return emitSilenceableError() << "contraction: " << message;
212 }
213 return emitDefiniteFailure() << "unknown body condition";
214}
215
216LogicalResult transform::MatchStructuredBodyOp::verify() {
217 int64_t numOptions = getReductionPosition().has_value() + getPassthrough() +
218 getElementwise() + getContraction().has_value();
219
220 if (numOptions > 1) {
221 StringAttr attributeNames[] = {
222 getReductionPositionAttrName(), getPassthroughAttrName(),
223 getElementwiseAttrName(), getContractionAttrName()};
224 return emitOpError() << "only one of {" << llvm::interleaved(R: attributeNames)
225 << "} is allowed";
226 }
227
228 if (std::optional<ArrayAttr> contractionAttr = getContraction()) {
229 if (contractionAttr->size() != 2) {
230 return emitOpError() << "expects " << getContractionAttrName()
231 << " to contain two elements";
232 }
233 }
234 return success();
235}
236
237//===----------------------------------------------------------------------===//
238// MatchStructuredClassifyContractionDimsOp
239//===----------------------------------------------------------------------===//
240
241DiagnosedSilenceableFailure
242transform::MatchStructuredClassifyContractionDimsOp::matchOperation(
243 Operation *current, transform::TransformResults &results,
244 transform::TransformState &state) {
245 FailureOr<linalg::ContractionDimensions> contractionDims =
246 linalg::inferContractionDims(linalgOp: cast<linalg::LinalgOp>(Val: current));
247 if (failed(Result: contractionDims))
248 return emitSilenceableError() << "could not infer contraction dimensions";
249
250 MLIRContext *context = current->getContext();
251 Builder builder(context);
252 auto makeI64Attrs = [&](ArrayRef<unsigned> values) {
253 return llvm::map_to_vector(C&: values, F: [&](unsigned value) -> Attribute {
254 return builder.getI64IntegerAttr(value);
255 });
256 };
257 results.setParams(value: cast<OpResult>(Val: getBatch()),
258 params: makeI64Attrs(contractionDims->batch));
259 results.setParams(value: cast<OpResult>(Val: getM()), params: makeI64Attrs(contractionDims->m));
260 results.setParams(value: cast<OpResult>(Val: getN()), params: makeI64Attrs(contractionDims->n));
261 results.setParams(value: cast<OpResult>(Val: getK()), params: makeI64Attrs(contractionDims->k));
262 return DiagnosedSilenceableFailure::success();
263}
264
265//===----------------------------------------------------------------------===//
266// MatchStructuredClassifyConvolutionDimsOp
267//===----------------------------------------------------------------------===//
268
269DiagnosedSilenceableFailure
270transform::MatchStructuredClassifyConvolutionDimsOp::matchOperation(
271 Operation *current, transform::TransformResults &results,
272 transform::TransformState &state) {
273 FailureOr<linalg::ConvolutionDimensions> convolutionDims =
274 linalg::inferConvolutionDims(linalgOp: cast<linalg::LinalgOp>(Val: current));
275 if (failed(Result: convolutionDims))
276 return emitSilenceableError() << "could not infer convolution dimensions";
277
278 MLIRContext *context = current->getContext();
279 Builder builder(context);
280 auto makeI64Attrs = [&](ArrayRef<unsigned> values) {
281 return llvm::map_to_vector(C&: values, F: [&](unsigned value) -> Attribute {
282 return builder.getI64IntegerAttr(value);
283 });
284 };
285 results.setParams(value: cast<OpResult>(Val: getBatch()),
286 params: makeI64Attrs(convolutionDims->batch));
287 results.setParams(value: cast<OpResult>(Val: getOutputImage()),
288 params: makeI64Attrs(convolutionDims->outputImage));
289 results.setParams(value: cast<OpResult>(Val: getOutputChannel()),
290 params: makeI64Attrs(convolutionDims->outputChannel));
291 results.setParams(value: cast<OpResult>(Val: getFilterLoop()),
292 params: makeI64Attrs(convolutionDims->filterLoop));
293 results.setParams(value: cast<OpResult>(Val: getInputChannel()),
294 params: makeI64Attrs(convolutionDims->inputChannel));
295 results.setParams(value: cast<OpResult>(Val: getDepth()),
296 params: makeI64Attrs(convolutionDims->depth));
297
298 auto makeI64AttrsFromI64 = [&](ArrayRef<int64_t> values) {
299 return llvm::map_to_vector(C&: values, F: [&](int64_t value) -> Attribute {
300 return builder.getI64IntegerAttr(value);
301 });
302 };
303 results.setParams(value: cast<OpResult>(Val: getStrides()),
304 params: makeI64AttrsFromI64(convolutionDims->strides));
305 results.setParams(value: cast<OpResult>(Val: getDilations()),
306 params: makeI64AttrsFromI64(convolutionDims->dilations));
307 return DiagnosedSilenceableFailure::success();
308}
309
310//===----------------------------------------------------------------------===//
311// Utilities for structured match predicates.
312//===----------------------------------------------------------------------===//
313
314/// Checks if all values from `list` are also contained in `reference`. Returns
315/// a silenceable error with the given message at the given location when it is
316/// not the case. The error message must contain the "{0}" placeholder that
317/// will be substituted with the value from `list` that is not contained in
318/// `reference`.
319static DiagnosedSilenceableFailure containsAll(ArrayRef<unsigned> reference,
320 ArrayRef<int64_t> list,
321 Location loc,
322 const char *message) {
323 for (int64_t value : list) {
324 if (llvm::any_of(Range&: reference, P: [&](unsigned ref) {
325 return static_cast<int64_t>(ref) == value;
326 })) {
327 continue;
328 }
329 return emitSilenceableFailure(loc) << llvm::formatv(Fmt: message, Vals&: value);
330 }
331 return DiagnosedSilenceableFailure::success();
332}
333
334//===----------------------------------------------------------------------===//
335// MatchStructuredDimOp
336//===----------------------------------------------------------------------===//
337
338DiagnosedSilenceableFailure transform::MatchStructuredDimOp::matchOperation(
339 Operation *current, transform::TransformResults &results,
340 transform::TransformState &state) {
341 auto linalgOp = cast<linalg::LinalgOp>(Val: current);
342 SmallVector<int64_t> dimensions;
343 DiagnosedSilenceableFailure diag = getDimensionsFor(op: linalgOp, dims&: dimensions);
344 if (!diag.succeeded())
345 return diag;
346
347 // If asked to check for the kind of dimension, perform the check.
348 if (getParallel() || getReduction()) {
349 SmallVector<unsigned> reference;
350 if (getParallel())
351 linalgOp.getParallelDims(res&: reference);
352 else if (getReduction())
353 linalgOp.getReductionDims(res&: reference);
354
355 DiagnosedSilenceableFailure diag =
356 containsAll(reference, list: dimensions, loc: getLoc(),
357 message: getParallel() ? "expects dimension #{0} to be parallel"
358 : "expects dimension #{0} to be reduction");
359 if (!diag.succeeded())
360 return diag;
361 }
362
363 // If not capturing, we are done here.
364 if (!getResult())
365 return diag;
366
367 SmallVector<int64_t, 4> ranges = linalgOp.getStaticLoopRanges();
368 Builder builder(current);
369 SmallVector<Attribute> captured =
370 llvm::map_to_vector(C&: dimensions, F: [&](int64_t dim) -> Attribute {
371 return builder.getI64IntegerAttr(value: ranges[dim]);
372 });
373 results.setParams(value: cast<OpResult>(Val: getResult()), params: captured);
374 return DiagnosedSilenceableFailure::success();
375}
376
377DiagnosedSilenceableFailure transform::MatchStructuredDimOp::getDimensionsFor(
378 linalg::LinalgOp op, SmallVectorImpl<int64_t> &dims) {
379 DiagnosedSilenceableFailure diag =
380 expandTargetSpecification(loc: getLoc(), isAll: getIsAll(), isInverted: getIsInverted(),
381 rawList: getRawDimList(), maxNumber: op.getNumLoops(), result&: dims);
382 if (diag.isSilenceableFailure()) {
383 diag.attachNote(loc: op->getLoc())
384 << "while considering dimensions of this payload operation";
385 }
386 return diag;
387}
388
389LogicalResult transform::MatchStructuredDimOp::verify() {
390 if (getParallel() && getReduction()) {
391 return emitOpError() << "cannot request the same dimension to be both "
392 "parallel and reduction";
393 }
394 return verifyTransformMatchDimsOp(op: getOperation(), raw: getRawDimList(),
395 inverted: getIsInverted(), all: getIsAll());
396}
397
398//===----------------------------------------------------------------------===//
399// MatchStructuredElementalBitwidthOp
400//===----------------------------------------------------------------------===//
401
402DiagnosedSilenceableFailure
403transform::MatchStructuredElementalBitwidthOp::matchValue(
404 Value current, transform::TransformResults &results,
405 transform::TransformState &state) {
406 auto setupResult = [&](int64_t bitwidth) {
407 Attribute attr = Builder(current.getContext()).getI64IntegerAttr(value: bitwidth);
408 results.setParams(value: cast<OpResult>(Val: getResult()), params: {attr});
409 return DiagnosedSilenceableFailure::success();
410 };
411
412 Type type = current.getType();
413 if (type.isIntOrFloat())
414 return setupResult(type.getIntOrFloatBitWidth());
415
416 if (auto shapedType = dyn_cast<ShapedType>(Val&: type)) {
417 if (shapedType.getElementType().isIntOrFloat())
418 return setupResult(shapedType.getElementTypeBitWidth());
419 }
420 return emitSilenceableError()
421 << "unsupported type for bitwidth extraction: " << type;
422}
423
424//===----------------------------------------------------------------------===//
425// MatchStructuredInputOp
426//===----------------------------------------------------------------------===//
427
428DiagnosedSilenceableFailure transform::MatchStructuredInputOp::matchOperation(
429 Operation *current, transform::TransformResults &results,
430 transform::TransformState &state) {
431 auto linalgOp = cast<linalg::LinalgOp>(Val: current);
432 SmallVector<int64_t> positions;
433 DiagnosedSilenceableFailure diag = getPositionsFor(op: linalgOp, positions);
434 if (!diag.succeeded())
435 return diag;
436
437 SmallVector<MappedValue> operandMapping;
438 operandMapping.reserve(N: positions.size());
439 for (int64_t position : positions) {
440 AffineMap indexingMap =
441 linalgOp.getMatchingIndexingMap(opOperand: linalgOp.getDpsInputOperand(i: position));
442 if (getPermutation() && !indexingMap.isPermutation()) {
443 return emitSilenceableError() << "the indexing map for input #"
444 << position << " is not a permutation";
445 }
446 if (getProjectedPermutation() && !indexingMap.isProjectedPermutation()) {
447 return emitSilenceableError()
448 << "the indexing map for input #" << position
449 << " is not a projected permutation";
450 }
451
452 // If capture not requested, skip it.
453 if (!getResult())
454 continue;
455
456 if (isa<AffineMapParamType>(Val: getResult().getType())) {
457 operandMapping.emplace_back(Args: AffineMapAttr::get(value: indexingMap));
458 continue;
459 }
460
461 Value operand = linalgOp.getDpsInputOperand(i: position)->get();
462 if (isa<TransformValueHandleTypeInterface>(Val: getResult().getType())) {
463 operandMapping.emplace_back(Args&: operand);
464 continue;
465 }
466
467 Operation *operandProducer = operand.getDefiningOp();
468 if (!operandProducer) {
469 return emitSilenceableError()
470 << "input #" << position << " is not produced by an operation";
471 }
472 operandMapping.emplace_back(Args&: operandProducer);
473 }
474 if (getResult())
475 results.setMappedValues(handle: cast<OpResult>(Val: getResult()), values: operandMapping);
476 return DiagnosedSilenceableFailure::success();
477}
478
479DiagnosedSilenceableFailure transform::MatchStructuredInputOp::getPositionsFor(
480 linalg::LinalgOp op, SmallVectorImpl<int64_t> &positions) {
481 DiagnosedSilenceableFailure diag = expandTargetSpecification(
482 loc: getLoc(), isAll: getIsAll(), isInverted: getIsInverted(), rawList: getRawPositionList(),
483 maxNumber: op.getNumDpsInputs(), result&: positions);
484 if (diag.isSilenceableFailure()) {
485 diag.attachNote(loc: op->getLoc())
486 << "while considering DPS inputs of this payload operation";
487 }
488 return diag;
489}
490
491/// Verifies a matcher op for structured input or output, specifically the
492/// attributes specifying the operand positions.
493template <typename OpTy>
494LogicalResult verifyStructuredOperandOp(OpTy op) {
495 if (op.getPermutation() && op.getProjectedPermutation()) {
496 return op.emitOpError()
497 << op.getPermutationAttrName() << " and "
498 << op.getProjectedPermutationAttrName() << " are mutually exclusive";
499 }
500 if (op.getRawPositionList().size() > 1 && op.getResult()) {
501 return op.emitOpError()
502 << "cannot bind multiple inputs/inits to the same value";
503 }
504
505 return success();
506}
507
508LogicalResult transform::MatchStructuredInputOp::verify() {
509 if (failed(Result: verifyStructuredOperandOp(op: *this)))
510 return failure();
511 return verifyTransformMatchDimsOp(op: getOperation(), raw: getRawPositionList(),
512 inverted: getIsInverted(), all: getIsAll());
513}
514
515//===----------------------------------------------------------------------===//
516// MatchStructuredInitOp
517//===----------------------------------------------------------------------===//
518
519DiagnosedSilenceableFailure transform::MatchStructuredInitOp::matchOperation(
520 Operation *current, transform::TransformResults &results,
521 transform::TransformState &state) {
522 auto linalgOp = cast<linalg::LinalgOp>(Val: current);
523 SmallVector<int64_t> positions;
524 DiagnosedSilenceableFailure diag = getPositionsFor(op: linalgOp, positions);
525 if (!diag.succeeded())
526 return diag;
527
528 SmallVector<MappedValue> operandMapping;
529 operandMapping.reserve(N: positions.size());
530 for (int64_t position : positions) {
531 AffineMap indexingMap =
532 linalgOp.getMatchingIndexingMap(opOperand: linalgOp.getDpsInitOperand(i: position));
533 if (getPermutation() && !indexingMap.isPermutation()) {
534 return emitSilenceableError() << "the indexing map for output(init) #"
535 << position << " is not a permutation";
536 }
537 if (getProjectedPermutation() && !indexingMap.isProjectedPermutation()) {
538 return emitSilenceableError() << "the indexing map for output(init) #"
539 << position << " is not a permutation";
540 }
541
542 // If capture not requested, skip it.
543 if (!getResult())
544 continue;
545
546 if (isa<AffineMapParamType>(Val: getResult().getType())) {
547 operandMapping.emplace_back(Args: AffineMapAttr::get(value: indexingMap));
548 continue;
549 }
550
551 Value operand = linalgOp.getDpsInitOperand(i: position)->get();
552 if (isa<TransformValueHandleTypeInterface>(Val: getResult().getType())) {
553 operandMapping.emplace_back(Args&: operand);
554 continue;
555 }
556
557 Operation *operandProducer = operand.getDefiningOp();
558 if (!operandProducer) {
559 return emitSilenceableError() << "output(init) #" << position
560 << " is not produced by an operation";
561 }
562 operandMapping.emplace_back(Args&: operandProducer);
563 }
564 if (getResult())
565 results.setMappedValues(handle: cast<OpResult>(Val: getResult()), values: operandMapping);
566 return DiagnosedSilenceableFailure::success();
567}
568
569DiagnosedSilenceableFailure transform::MatchStructuredInitOp::getPositionsFor(
570 linalg::LinalgOp op, SmallVectorImpl<int64_t> &positions) {
571 DiagnosedSilenceableFailure diag = expandTargetSpecification(
572 loc: getLoc(), isAll: getIsAll(), isInverted: getIsInverted(), rawList: getRawPositionList(),
573 maxNumber: op.getNumDpsInits(), result&: positions);
574 if (diag.isSilenceableFailure()) {
575 diag.attachNote(loc: op->getLoc())
576 << "while considering DPS inits (outputs) of this payload operation";
577 }
578 return diag;
579}
580
581LogicalResult transform::MatchStructuredInitOp::verify() {
582 if (failed(Result: verifyStructuredOperandOp(op: *this)))
583 return failure();
584 return verifyTransformMatchDimsOp(op: getOperation(), raw: getRawPositionList(),
585 inverted: getIsInverted(), all: getIsAll());
586}
587
588//===----------------------------------------------------------------------===//
589// MatchStructuredNumInputsOp
590//===----------------------------------------------------------------------===//
591
592DiagnosedSilenceableFailure
593transform::MatchStructuredNumInputsOp::matchOperation(
594 Operation *current, transform::TransformResults &results,
595 transform::TransformState &state) {
596 auto linalgOp = cast<linalg::LinalgOp>(Val: current);
597 Attribute attr =
598 Builder(current).getI64IntegerAttr(value: linalgOp.getNumDpsInputs());
599 results.setParams(value: cast<OpResult>(Val: getResult()), params: {attr});
600 return DiagnosedSilenceableFailure::success();
601}
602
603//===----------------------------------------------------------------------===//
604// MatchStructuredNumInitsOp
605//===----------------------------------------------------------------------===//
606
607DiagnosedSilenceableFailure
608transform::MatchStructuredNumInitsOp::matchOperation(
609 Operation *current, transform::TransformResults &results,
610 transform::TransformState &state) {
611 auto linalgOp = cast<linalg::LinalgOp>(Val: current);
612 Attribute attr =
613 Builder(current).getI64IntegerAttr(value: linalgOp.getNumDpsInits());
614 results.setParams(value: cast<OpResult>(Val: getResult()), params: {attr});
615 return DiagnosedSilenceableFailure::success();
616}
617
618//===----------------------------------------------------------------------===//
619// MatchStructuredRankOp
620//===----------------------------------------------------------------------===//
621
622DiagnosedSilenceableFailure transform::MatchStructuredRankOp::matchOperation(
623 Operation *current, transform::TransformResults &results,
624 transform::TransformState &state) {
625 auto linalgOp = cast<linalg::LinalgOp>(Val: current);
626 int64_t numLoops = linalgOp.getNumLoops();
627 Attribute attr = Builder(linalgOp->getContext()).getI64IntegerAttr(value: numLoops);
628 results.setParams(value: cast<OpResult>(Val: getRank()), params: {attr});
629 return DiagnosedSilenceableFailure::success();
630}
631
632//===----------------------------------------------------------------------===//
633// MatchStructuredResultOp
634//===----------------------------------------------------------------------===//
635
636DiagnosedSilenceableFailure transform::MatchStructuredResultOp::matchOperation(
637 Operation *op, transform::TransformResults &results,
638 transform::TransformState &state) {
639 auto linalgOp = cast<linalg::LinalgOp>(Val: op);
640 int64_t position;
641 DiagnosedSilenceableFailure diag = getPositionFor(op: linalgOp, position);
642 if (!diag.succeeded())
643 return diag;
644
645 Value result = linalgOp.getTiedOpResult(opOperand: linalgOp.getDpsInitOperand(i: position));
646 if (isa<TransformValueHandleTypeInterface>(Val: getResult().getType())) {
647 results.setValues(handle: cast<OpResult>(Val: getResult()), values: {result});
648 return DiagnosedSilenceableFailure::success();
649 }
650
651 if (result.getUsers().empty()) {
652 return emitSilenceableError()
653 << "no users of the result #" << getPosition();
654 }
655 Operation *firstUser = *result.getUsers().begin();
656 if (getAny()) {
657 results.set(value: cast<OpResult>(Val: getResult()), ops: {firstUser});
658 return DiagnosedSilenceableFailure::success();
659 }
660 if (getSingle()) {
661 if (!llvm::hasSingleElement(C: result.getUsers())) {
662 return emitSilenceableError()
663 << "more than one result user with single user requested";
664 }
665 results.set(value: cast<OpResult>(Val: getResult()), ops: {firstUser});
666 return DiagnosedSilenceableFailure::success();
667 }
668
669 return emitDefiniteFailure() << "unknown sub-predicate";
670}
671
672DiagnosedSilenceableFailure
673transform::MatchStructuredResultOp::getPositionFor(linalg::LinalgOp op,
674 int64_t &position) {
675 auto rawPosition = static_cast<int64_t>(getPosition());
676 position = rawPosition < 0 ? op.getNumDpsInits() + rawPosition : rawPosition;
677 if (position >= op.getNumDpsInits() || position < 0) {
678 return emitSilenceableError()
679 << "position " << rawPosition
680 << " overflows the number of results(ints) of the payload operation";
681 }
682 return DiagnosedSilenceableFailure::success();
683}
684
685LogicalResult transform::MatchStructuredResultOp::verify() {
686 if ((getAny() || getSingle()) ^
687 isa<TransformHandleTypeInterface>(Val: getResult().getType())) {
688 return emitOpError() << "expects either the any/single keyword or the type "
689 "value handle result type";
690 }
691 if (getAny() && getSingle()) {
692 return emitOpError() << "'any' and 'single' are mutually exclusive";
693 }
694 return success();
695}
696
697//===----------------------------------------------------------------------===//
698// MatchStructuredYieldOp
699//===----------------------------------------------------------------------===//
700
701void transform::MatchStructuredYieldOp::getEffects(
702 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
703 onlyReadsHandle(handles: getHandlesMutable(), effects);
704 onlyReadsPayload(effects);
705}
706
707void transform::MatchStructuredYieldOp::build(OpBuilder &builder,
708 OperationState &state) {
709 build(odsBuilder&: builder, odsState&: state, handles: ValueRange());
710}
711
712#define GET_OP_CLASSES
713#include "mlir/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp.inc"
714