1//===- LinalgTransformOps.cpp - Implementation of Linalg transform ops ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h"
10
11#include "mlir/AsmParser/AsmParser.h"
12#include "mlir/Dialect/Affine/IR/AffineOps.h"
13#include "mlir/Dialect/Arith/IR/Arith.h"
14#include "mlir/Dialect/Arith/Utils/Utils.h"
15#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
16#include "mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h"
17#include "mlir/Dialect/CommonFolders.h"
18#include "mlir/Dialect/GPU/IR/GPUDialect.h"
19#include "mlir/Dialect/Linalg/IR/Linalg.h"
20#include "mlir/Dialect/Linalg/TransformOps/GPUHeuristics.h"
21#include "mlir/Dialect/Linalg/TransformOps/Syntax.h"
22#include "mlir/Dialect/Linalg/Transforms/Hoisting.h"
23#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
24#include "mlir/Dialect/Linalg/Utils/Utils.h"
25#include "mlir/Dialect/SCF/Transforms/TileUsingInterface.h"
26#include "mlir/Dialect/Tensor/IR/Tensor.h"
27#include "mlir/Dialect/Transform/IR/TransformTypes.h"
28#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
29#include "mlir/Dialect/Transform/Utils/Utils.h"
30#include "mlir/Dialect/UB/IR/UBMatchers.h"
31#include "mlir/Dialect/Utils/IndexingUtils.h"
32#include "mlir/Dialect/Utils/StaticValueUtils.h"
33#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
34#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
35#include "mlir/IR/BuiltinTypeInterfaces.h"
36#include "mlir/IR/PatternMatch.h"
37#include "mlir/IR/TypeUtilities.h"
38#include "mlir/Interfaces/ParallelCombiningOpInterface.h"
39#include "mlir/Interfaces/TilingInterface.h"
40#include "mlir/Support/LLVM.h"
41#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
42#include "llvm/ADT/STLExtras.h"
43#include "llvm/ADT/ScopeExit.h"
44#include "llvm/ADT/SmallPtrSet.h"
45#include "llvm/ADT/SmallVectorExtras.h"
46#include "llvm/ADT/TypeSwitch.h"
47#include "llvm/Support/DebugLog.h"
48#include "llvm/Support/LogicalResult.h"
49#include <type_traits>
50
51using namespace mlir;
52using namespace mlir::linalg;
53using namespace mlir::transform;
54
55#define DEBUG_TYPE "linalg-transforms"
56
57/// Assuming that `ofr` is an index attr or a param of index type
58/// or a transform dialect handle mapped to exactly one op
59/// with one index result, return that value.
60static DiagnosedSilenceableFailure unpackSingleIndexResultPayloadOperations(
61 transform::TransformState &state, TransformOpInterface transformOp,
62 SmallVector<OpFoldResult> &result, ArrayRef<OpFoldResult> ofrs) {
63 for (OpFoldResult ofr : ofrs) {
64 if (auto attr = dyn_cast<Attribute>(Val&: ofr)) {
65 if (!isa<IntegerAttr>(Val: attr))
66 return transformOp.emitDefiniteFailure() << "expected IntegerAttr";
67 result.push_back(Elt: ofr);
68 continue;
69 }
70
71 Value transformValue = cast<Value>(Val&: ofr);
72 if (isa<TransformParamTypeInterface>(Val: transformValue.getType())) {
73 ArrayRef<Attribute> params = state.getParams(value: transformValue);
74 if (params.size() != 1)
75 return transformOp.emitDefiniteFailure()
76 << "requires exactly one parameter associated";
77 result.push_back(Elt: params[0]);
78 continue;
79 }
80
81 auto payloadOps = state.getPayloadOps(value: transformValue);
82 if (!llvm::hasSingleElement(C&: payloadOps)) {
83 DiagnosedSilenceableFailure diag =
84 transformOp.emitSilenceableError()
85 << "handle must be mapped to exactly one payload op";
86 diag.attachNote(loc: transformValue.getLoc())
87 << "mapped to " << llvm::range_size(Range&: payloadOps) << " payload ops";
88 return diag;
89 }
90
91 Operation *op = *payloadOps.begin();
92 if (op->getNumResults() != 1 || !op->getResult(idx: 0).getType().isIndex()) {
93 DiagnosedSilenceableFailure diag =
94 transformOp.emitSilenceableError()
95 << "payload op must have exactly 1 index result";
96 diag.attachNote(loc: op->getLoc())
97 << "has " << op->getNumResults() << " results";
98 return diag;
99 }
100 result.push_back(Elt: op->getResult(idx: 0));
101 }
102
103 return DiagnosedSilenceableFailure::success();
104}
105
106// Given a list of params that are index attrs or a list of OpFoldResults
107// that are either index attrs or op handles, return a list of OpFoldResults
108// of index attrs or a list of OpFoldResults where all op handles are
109// replaced with the first (and only) OpResult of that payload op.
110// (There must be exactly one parameter associated with the AnyParamType or
111// one mapped payload op which must have exactly one index result.)
112static DiagnosedSilenceableFailure unpackSingleIndexResultPayloadOperations(
113 transform::TransformState &state, TransformOpInterface transformOp,
114 SmallVector<OpFoldResult> &result, Value packedHandle) {
115 if (isa<TransformParamTypeInterface>(Val: packedHandle.getType())) {
116 ArrayRef<Attribute> params = state.getParams(value: packedHandle);
117 for (auto param : params) {
118 if (!isa<IntegerAttr>(Val: param))
119 return transformOp.emitDefiniteFailure()
120 << "expected the parameter to be associated with an integer "
121 "attribute";
122 result.push_back(Elt: param);
123 }
124 return DiagnosedSilenceableFailure::success();
125 }
126
127 for (Operation *op : state.getPayloadOps(value: packedHandle)) {
128 if (op->getNumResults() != 1 || !op->getResult(idx: 0).getType().isIndex()) {
129 DiagnosedSilenceableFailure diag =
130 transformOp.emitSilenceableError()
131 << "payload op must have exactly 1 index result";
132 diag.attachNote(loc: op->getLoc())
133 << "has " << op->getNumResults() << " results";
134 return diag;
135 }
136 result.push_back(Elt: op->getResult(idx: 0));
137 }
138
139 return DiagnosedSilenceableFailure::success();
140}
141
142/// When possible, converts each `OpFoldResult` in `mixedResult` to
143/// an integer if the value can be statically inferred. If a result
144/// is a `Value` then it must be either a `ParamType` or a handle
145/// to an a constant like op.
146static DiagnosedSilenceableFailure reifyMixedParamAndHandleResults(
147 TransformState &state, TransformOpInterface &transformOp,
148 ArrayRef<OpFoldResult> mixedResults, SmallVectorImpl<int64_t> &reified) {
149 for (OpFoldResult paramOrHandle : mixedResults) {
150 if (auto attr = dyn_cast<Attribute>(Val&: paramOrHandle)) {
151 reified.push_back(Elt: cast<IntegerAttr>(Val&: attr).getInt());
152 continue;
153 }
154 if (isa<TransformParamTypeInterface>(
155 Val: cast<Value>(Val&: paramOrHandle).getType())) {
156 ArrayRef<Attribute> params = state.getParams(value: cast<Value>(Val&: paramOrHandle));
157 if (params.size() != 1)
158 return transformOp.emitSilenceableError() << "expected a single param";
159 reified.push_back(
160 Elt: cast<IntegerAttr>(Val: params.front()).getValue().getSExtValue());
161 continue;
162 }
163
164 Value handle = cast<Value>(Val&: paramOrHandle);
165 if (!isa<TransformHandleTypeInterface>(Val: handle.getType()))
166 return transformOp.emitSilenceableError() << "unexpected value handle";
167 auto payload = state.getPayloadOps(value: handle);
168 if (!llvm::hasSingleElement(C&: payload))
169 return transformOp.emitSilenceableError()
170 << "requires param or handle that is mapped to 1 payload op";
171
172 Operation *paramOrHandlePayloadOp = *payload.begin();
173 if (paramOrHandlePayloadOp->getNumResults() != 1 ||
174 !paramOrHandlePayloadOp->getResult(idx: 0).getType().isIndex()) {
175 return transformOp.emitSilenceableError()
176 << "requires param or handle to be result of op with 1 index "
177 "result";
178 }
179
180 IntegerAttr attr;
181 if (!matchPattern(value: paramOrHandlePayloadOp->getResult(idx: 0), pattern: m_Constant(bind_value: &attr)))
182 return transformOp.emitSilenceableError()
183 << "requires param or handle to be the result of a constant like "
184 "op";
185
186 reified.push_back(Elt: attr.getInt());
187 }
188 return DiagnosedSilenceableFailure::success();
189}
190
191//===----------------------------------------------------------------------===//
192// Apply...PatternsOp
193//===----------------------------------------------------------------------===//
194
195void transform::ApplyEraseUnnecessaryInputsPatternsOp::populatePatterns(
196 RewritePatternSet &patterns) {
197 linalg::populateEraseUnnecessaryInputsPatterns(patterns);
198}
199
200void transform::ApplyDecomposeTensorPackUnpackPatternsOp::populatePatterns(
201 RewritePatternSet &patterns) {
202 linalg::populateDecomposePackUnpackPatterns(patterns);
203}
204
205void transform::ApplyDecomposeTensorPadPatternsOp::populatePatterns(
206 RewritePatternSet &patterns) {
207 linalg::populateDecomposePadPatterns(patterns);
208}
209
210void transform::ApplyFoldUnitExtentDimsViaReshapesPatternsOp::populatePatterns(
211 RewritePatternSet &patterns) {
212 linalg::ControlDropUnitDims options;
213 linalg::populateFoldUnitExtentDimsPatterns(patterns, options);
214}
215
216void transform::ApplyFoldUnitExtentDimsViaSlicesPatternsOp::populatePatterns(
217 RewritePatternSet &patterns) {
218 linalg::ControlDropUnitDims options;
219 options.rankReductionStrategy =
220 linalg::ControlDropUnitDims::RankReductionStrategy::ExtractInsertSlice;
221 linalg::populateFoldUnitExtentDimsPatterns(patterns, options);
222}
223
224void transform::ApplyTilingCanonicalizationPatternsOp::populatePatterns(
225 RewritePatternSet &patterns) {
226 linalg::populateLinalgTilingCanonicalizationPatterns(patterns);
227}
228
229void transform::ApplyFoldAddIntoDestPatternsOp::populatePatterns(
230 RewritePatternSet &patterns) {
231 linalg::populateFoldAddIntoDestPatterns(patterns);
232}
233
234void transform::ApplyPadVectorizationPatternsOp::populatePatterns(
235 RewritePatternSet &patterns) {
236 linalg::populatePadOpVectorizationPatterns(patterns);
237}
238
239void transform::ApplyFoldIntoPackAndUnpackPatternsOp::populatePatterns(
240 RewritePatternSet &patterns) {
241 linalg::populateFoldIntoPackAndUnpackPatterns(patterns);
242}
243
244void transform::ApplyFoldPackUnpackIntoEmptyPatternsOp::populatePatterns(
245 RewritePatternSet &patterns) {
246 linalg::populateFoldPackUnpackIntoTensorEmptyPatterns(patterns);
247}
248
249void transform::ApplyDataLayoutPropagationPatternsOp::populatePatterns(
250 RewritePatternSet &patterns) {
251 linalg::ControlPropagationFn defaultControlFn = [](OpOperand *operand) {
252 return true;
253 };
254 linalg::populateDataLayoutPropagationPatterns(patterns, controlPackUnPackPropagation: defaultControlFn,
255 PoisonPaddingOk: getPoisonPadding());
256}
257
258void transform::ApplyExtractSliceSinkingPatternsOp::populatePatterns(
259 RewritePatternSet &patterns) {
260 linalg::ControlPropagationFn defaultControlFn =
261 [](OpOperand *opOperand) -> bool {
262 Operation *producer = opOperand->get().getDefiningOp();
263 Operation *consumer = opOperand->getOwner();
264 return consumer->getBlock() == producer->getBlock();
265 };
266 linalg::populateExtractSliceSinkingPatterns(patterns, controlPackUnPackPropagation: defaultControlFn);
267}
268
269void transform::ApplySwapExtractSliceWithFillPatternsOp::populatePatterns(
270 RewritePatternSet &patterns) {
271 linalg::populateSwapExtractSliceWithFillPatterns(patterns);
272}
273
274void transform::ApplyEraseUnusedOperandsAndResultsPatternsOp::populatePatterns(
275 RewritePatternSet &patterns) {
276 linalg::populateEraseUnusedOperandsAndResultsPatterns(patterns);
277}
278
279void transform::ApplyFoldReshapeOpsByExpansionPatternsOp::populatePatterns(
280 RewritePatternSet &patterns) {
281 linalg::ControlFusionFn controlFn =
282 [foldSingleUseOnly = getFoldSingleUseOnly()](OpOperand *fusedOperand) {
283 if (!foldSingleUseOnly)
284 return true;
285 Operation *producer = fusedOperand->get().getDefiningOp();
286 return producer && producer->hasOneUse();
287 };
288 linalg::populateFoldReshapeOpsByExpansionPatterns(patterns, controlFoldingReshapes: controlFn);
289}
290
291//===----------------------------------------------------------------------===//
292// BufferizeToAllocationOp
293//===----------------------------------------------------------------------===//
294
295namespace {
296class NewOpsListener : public RewriterBase::ForwardingListener {
297public:
298 using RewriterBase::ForwardingListener::ForwardingListener;
299
300 SmallVector<Operation *> getNewOps() const {
301 return SmallVector<Operation *>(newOps.begin(), newOps.end());
302 }
303
304private:
305 void notifyOperationInserted(Operation *op,
306 OpBuilder::InsertPoint previous) override {
307 ForwardingListener::notifyOperationInserted(op, previous);
308 // We only care about newly created ops.
309 if (previous.isSet())
310 return;
311 auto inserted = newOps.insert(V: op);
312 (void)inserted;
313 assert(inserted.second && "expected newly created op");
314 }
315
316 void notifyOperationErased(Operation *op) override {
317 ForwardingListener::notifyOperationErased(op);
318 op->walk(callback: [&](Operation *op) { newOps.erase(V: op); });
319 }
320
321 DenseSet<Operation *> newOps;
322};
323} // namespace
324
325DiagnosedSilenceableFailure transform::BufferizeToAllocationOp::apply(
326 transform::TransformRewriter &rewriter,
327 transform::TransformResults &results, transform::TransformState &state) {
328 // Attach listener to keep track of newly created ops.
329 OpBuilder::Listener *previousListener = rewriter.getListener();
330 llvm::scope_exit resetListener(
331 [&]() { rewriter.setListener(previousListener); });
332 NewOpsListener newOpsListener(previousListener);
333 rewriter.setListener(&newOpsListener);
334
335 linalg::BufferizeToAllocationOptions options;
336 if (getMemcpyOp() == "bufferization.materialize_in_destination") {
337 options.memcpyOp = linalg::BufferizeToAllocationOptions::MemcpyOp::
338 MaterializeInDestination;
339 } else if (getMemcpyOp() == "memref.copy") {
340 options.memcpyOp =
341 linalg::BufferizeToAllocationOptions::MemcpyOp::MemrefCopy;
342 } else if (getMemcpyOp() == "linalg.copy") {
343 options.memcpyOp =
344 linalg::BufferizeToAllocationOptions::MemcpyOp::LinalgCopy;
345 } else {
346 llvm_unreachable("invalid memcpy op");
347 }
348 if (getAllocOp() == "memref.alloc") {
349 options.allocOp =
350 linalg::BufferizeToAllocationOptions::AllocOp::MemrefAlloc;
351 } else if (getAllocOp() == "memref.alloca") {
352 options.allocOp =
353 linalg::BufferizeToAllocationOptions::AllocOp::MemrefAlloca;
354 } else {
355 llvm_unreachable("invalid alloc op");
356 }
357 options.bufferizeDestinationOnly = getBufferizeDestinationOnly();
358 options.emitDealloc = getEmitDealloc();
359
360 // Bufferize ops.
361 Attribute memorySpace =
362 getMemorySpace().has_value() ? getMemorySpace().value() : Attribute();
363 SmallVector<Value> allocatedBuffers;
364 for (Operation *op : state.getPayloadOps(value: getTarget())) {
365 Value buffer =
366 linalg::bufferizeToAllocation(rewriter, options, op, memorySpace);
367 if (!buffer) {
368 DiagnosedSilenceableFailure diag = emitSilenceableError()
369 << "failed to bufferize operation";
370 diag.attachNote(loc: op->getLoc()) << "target payload op";
371 return diag;
372 }
373 allocatedBuffers.push_back(Elt: buffer);
374 }
375
376 // Set results.
377 results.setValues(handle: cast<OpResult>(Val: getAllocatedBuffer()), values&: allocatedBuffers);
378 results.set(value: cast<OpResult>(Val: getNewOps()), ops: newOpsListener.getNewOps());
379 return DiagnosedSilenceableFailure::success();
380}
381
382void transform::BufferizeToAllocationOp::getEffects(
383 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
384 if (getBufferizeDestinationOnly()) {
385 // The destination is replaced with a newly allocated buffer, but the op
386 // itself remains in place.
387 onlyReadsHandle(handles: getTargetMutable(), effects);
388 } else {
389 consumesHandle(handles: getTargetMutable(), effects);
390 }
391 producesHandle(handles: getOperation()->getOpResults(), effects);
392 modifiesPayload(effects);
393}
394
395LogicalResult transform::BufferizeToAllocationOp::verify() {
396 if (getMemcpyOp() != "bufferization.materialize_in_destination" &&
397 getMemcpyOp() != "memref.copy" && getMemcpyOp() != "linalg.copy")
398 return emitOpError() << "unsupported memcpy op";
399 if (getAllocOp() != "memref.alloc" && getAllocOp() != "memref.alloca")
400 return emitOpError() << "unsupported alloc op";
401 return success();
402}
403
404//===----------------------------------------------------------------------===//
405// PromoteTensorOp
406//===----------------------------------------------------------------------===//
407
408/// Return true if the operand may be read from by its owner. This is currently
409/// very conservative and only looks inside linalg operations to prevent
410/// unintentional data loss.
411static bool mayBeRead(OpOperand &operand) {
412 auto linalgOp = dyn_cast<linalg::LinalgOp>(Val: operand.getOwner());
413
414 // Be conservative about ops we cannot analyze deeper.
415 if (!linalgOp)
416 return true;
417
418 // Look inside linalg ops.
419 Value blockArgument = linalgOp.getMatchingBlockArgument(opOperand: &operand);
420 return !blockArgument.use_empty();
421}
422
423/// Return true if the value may be read through any of its uses.
424static bool mayBeRead(Value value) {
425 // If the value has a reference semantics, it
426 // may be read through any alias...
427 if (!isa<TensorType, FloatType, IntegerType>(Val: value.getType()))
428 return true;
429 return llvm::any_of(Range: value.getUses(),
430 P: static_cast<bool (&)(OpOperand &)>(mayBeRead));
431}
432
433DiagnosedSilenceableFailure
434transform::PromoteTensorOp::apply(transform::TransformRewriter &rewriter,
435 transform::TransformResults &results,
436 transform::TransformState &state) {
437 SmallVector<Value> promoted;
438 for (Value tensor : state.getPayloadValues(handleValue: getTensor())) {
439 auto type = dyn_cast<RankedTensorType>(Val: tensor.getType());
440 if (!type) {
441 return emitSilenceableError() << "non-tensor type: " << tensor;
442 }
443
444 Operation *definingOp = tensor.getDefiningOp();
445 if (definingOp)
446 rewriter.setInsertionPointAfter(definingOp);
447 else
448 rewriter.setInsertionPointToStart(cast<BlockArgument>(Val&: tensor).getOwner());
449
450 // Check this before we emit operations using this value.
451 bool needsMaterialization = mayBeRead(value: tensor);
452
453 SmallVector<Value> dynamicDims;
454 llvm::SmallPtrSet<Operation *, 4> preservedOps;
455 for (auto [pos, dim] : llvm::enumerate(First: type.getShape())) {
456 if (!ShapedType::isDynamic(dValue: dim))
457 continue;
458 Value cst =
459 arith::ConstantIndexOp::create(builder&: rewriter, location: tensor.getLoc(), value: pos);
460 auto dimOp =
461 tensor::DimOp::create(builder&: rewriter, location: tensor.getLoc(), source: tensor, index: cst);
462 preservedOps.insert(Ptr: dimOp);
463 dynamicDims.push_back(Elt: dimOp);
464 }
465 auto allocation = bufferization::AllocTensorOp::create(
466 builder&: rewriter, location: tensor.getLoc(), type, dynamicSizes: dynamicDims);
467 // Set memory space if provided.
468 if (getMemorySpaceAttr())
469 allocation.setMemorySpaceAttr(getMemorySpaceAttr());
470 Value allocated = allocation;
471
472 // Only insert a materialization (typically bufferizes to a copy) when the
473 // value may be read from.
474 if (needsMaterialization) {
475 auto copy = bufferization::MaterializeInDestinationOp::create(
476 builder&: rewriter, location: tensor.getLoc(), source: tensor, dest: allocated);
477 preservedOps.insert(Ptr: copy);
478 promoted.push_back(Elt: copy.getResult());
479 } else {
480 promoted.push_back(Elt: allocated);
481 }
482 rewriter.replaceAllUsesExcept(from: tensor, to: promoted.back(), preservedUsers: preservedOps);
483 }
484 results.setValues(handle: cast<OpResult>(Val: getPromoted()), values&: promoted);
485 return DiagnosedSilenceableFailure::success();
486}
487
488void transform::PromoteTensorOp::getEffects(
489 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
490 transform::onlyReadsHandle(handles: getTensorMutable(), effects);
491 transform::producesHandle(handles: getOperation()->getOpResults(), effects);
492 transform::modifiesPayload(effects);
493}
494
495//===----------------------------------------------------------------------===//
496// DecomposeOp
497//===----------------------------------------------------------------------===//
498
499DiagnosedSilenceableFailure
500transform::DecomposeOp::applyToOne(transform::TransformRewriter &rewriter,
501 LinalgOp target,
502 transform::ApplyToEachResultList &results,
503 transform::TransformState &state) {
504 FailureOr<linalg::LinalgOp> res =
505 downscaleSizeOneWindowedConvolution(rewriter, op: target);
506 if (succeeded(Result: res)) {
507 results.push_back(op: *res);
508 return DiagnosedSilenceableFailure::success();
509 }
510 return emitDefaultSilenceableFailure(target);
511}
512
513//===----------------------------------------------------------------------===//
514// DecomposeInterfaceOp
515//===----------------------------------------------------------------------===//
516
517// Decompose the target operation if it implements the AggregatedOpInterface.
518// Push the decomposed operations (the ones that replaces the values produced by
519// \p target) in the `results`.
520DiagnosedSilenceableFailure transform::DecomposeInterfaceOp::applyToOne(
521 transform::TransformRewriter &rewriter, Operation *target,
522 transform::ApplyToEachResultList &results,
523 transform::TransformState &state) {
524 auto decomposableOp = dyn_cast<AggregatedOpInterface>(Val: target);
525 if (!decomposableOp) {
526 (void)rewriter.notifyMatchFailure(arg&: target,
527 msg: "payload is not a decomposable op");
528 return emitDefaultSilenceableFailure(target);
529 }
530
531 FailureOr<SmallVector<Value>> maybeNewResults =
532 decomposableOp.decomposeOperation(b&: rewriter);
533 if (failed(Result: maybeNewResults))
534 return emitDefaultSilenceableFailure(target);
535
536 rewriter.replaceOp(op: decomposableOp, newValues: *maybeNewResults);
537 for (Value val : *maybeNewResults) {
538 Operation *definition = val.getDefiningOp();
539 if (definition)
540 results.push_back(op: definition);
541 }
542 return DiagnosedSilenceableFailure::success();
543}
544
545//===----------------------------------------------------------------------===//
546// EliminateLinalgOpAnchoredEmptyTensorsOp
547//===----------------------------------------------------------------------===//
548
549void transform::EliminateLinalgOpAnchoredEmptyTensorsOp::getEffects(
550 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
551 onlyReadsHandle(handles: getTargetMutable(), effects);
552 modifiesPayload(effects);
553}
554
555DiagnosedSilenceableFailure
556transform::EliminateLinalgOpAnchoredEmptyTensorsOp::apply(
557 transform::TransformRewriter &rewriter, TransformResults &transformResults,
558 TransformState &state) {
559 bufferization::OneShotBufferizationOptions options;
560 options.allowReturnAllocsFromLoops = true;
561
562 for (Operation *target : state.getPayloadOps(value: getTarget())) {
563 bufferization::OneShotAnalysisState state(target, options);
564 if (failed(Result: analyzeOp(op: target, state)))
565 return mlir::emitSilenceableFailure(loc: target->getLoc())
566 << "failed to analyze op";
567 if (failed(Result: linalg::linalgOpAnchoredEmptyTensorEliminationStep(
568 rewriter, op: target, state)))
569 return mlir::emitSilenceableFailure(loc: target->getLoc())
570 << "failed to eliminate LinalgOp anchored tensor.empty ops";
571 }
572 return DiagnosedSilenceableFailure::success();
573}
574
575//===----------------------------------------------------------------------===//
576// FuseOp
577//===----------------------------------------------------------------------===//
578
579void transform::FuseOp::build(OpBuilder &builder, OperationState &result,
580 TypeRange loopTypes, Value target,
581 ArrayRef<int64_t> staticTileSizes,
582 ArrayRef<int64_t> staticTileInterchange,
583 bool applyCleanup, bool useForall) {
584 return build(
585 odsBuilder&: builder, odsState&: result, loopTypes,
586 /*target=*/target,
587 /*mixedTileSizes=*/
588 getAsOpFoldResult(arrayAttr: builder.getI64ArrayAttr(values: staticTileSizes)),
589 /*mixedTileInterchange=*/
590 getAsOpFoldResult(arrayAttr: builder.getI64ArrayAttr(values: staticTileInterchange)),
591 applyCleanup, useForall);
592}
593
594void transform::FuseOp::build(OpBuilder &builder, OperationState &result,
595 Value target, ArrayRef<int64_t> staticTileSizes,
596 ArrayRef<int64_t> staticTileInterchange,
597 bool applyCleanup, bool useForall) {
598 return build(
599 odsBuilder&: builder, odsState&: result,
600 /*target=*/target,
601 /*mixedTileSizes=*/
602 getAsOpFoldResult(arrayAttr: builder.getI64ArrayAttr(values: staticTileSizes)),
603 /*mixedTileInterchange=*/
604 getAsOpFoldResult(arrayAttr: builder.getI64ArrayAttr(values: staticTileInterchange)),
605 applyCleanup, useForall);
606}
607
608void transform::FuseOp::build(OpBuilder &builder, OperationState &result,
609 Value target,
610 ArrayRef<OpFoldResult> mixedTileSizes,
611 ArrayRef<OpFoldResult> mixedTileInterchange,
612 bool applyCleanup, bool useForall) {
613 // Loop types are automaticaly splat by the callee, setting up one is
614 // enough.
615 SmallVector<Type> loopTypes(1, builder.getType<transform::AnyOpType>());
616 build(odsBuilder&: builder, odsState&: result, loopTypes, target, mixedTileSizes,
617 mixedTileInterchange, applyCleanup, useForall);
618}
619
620void transform::FuseOp::build(OpBuilder &builder, OperationState &result,
621 TypeRange loopTypes, Value target,
622 ArrayRef<OpFoldResult> mixedTileSizes,
623 ArrayRef<OpFoldResult> mixedTileInterchange,
624 bool applyCleanup, bool useForall) {
625 SmallVector<int64_t> staticTileSizes;
626 SmallVector<Value> dynamicTileSizes;
627 dispatchIndexOpFoldResults(ofrs: mixedTileSizes, dynamicVec&: dynamicTileSizes, staticVec&: staticTileSizes);
628 SmallVector<int64_t> staticTileInterchange;
629 SmallVector<Value> dynamicTileInterchange;
630 dispatchIndexOpFoldResults(ofrs: mixedTileInterchange, dynamicVec&: dynamicTileInterchange,
631 staticVec&: staticTileInterchange);
632 // Call the default builder which sets up the proper operands segment sizes
633 // attributes for multiple variadic operands. In the absence of this,
634 // horrible bugs ensue.
635 auto staticTileSizesAttr = builder.getDenseI64ArrayAttr(values: staticTileSizes);
636 auto staticTileInterchangeAttr =
637 builder.getDenseI64ArrayAttr(values: staticTileInterchange);
638 unsigned numExpectedLoops =
639 useForall ? 1 : staticTileSizes.size() - llvm::count(Range&: staticTileSizes, Element: 0);
640 SmallVector<Type> resultTypes;
641 resultTypes.reserve(N: numExpectedLoops);
642 assert((loopTypes.size() == 1 || loopTypes.size() == numExpectedLoops) &&
643 "expected one loop type or as many as loops");
644 if (loopTypes.size() == 1)
645 resultTypes.append(NumInputs: numExpectedLoops, Elt: loopTypes[0]);
646 else
647 llvm::append_range(C&: resultTypes, R&: loopTypes);
648 build(odsBuilder&: builder, odsState&: result, /*transformed=*/target.getType(),
649 /*loops=*/resultTypes,
650 /*target=*/target,
651 /*tile_sizes=*/dynamicTileSizes,
652 /*tile_interchange=*/dynamicTileInterchange,
653 /*packed_tile_sizes=*/Value(),
654 /*static_tile_sizes=*/staticTileSizesAttr,
655 /*static_tile_interchange=*/staticTileInterchangeAttr,
656 /*inner_tile_alignments=*/ArrayRef<int64_t>{},
657 /*apply_cleanup=*/applyCleanup,
658 /*use_forall=*/useForall);
659}
660
661/// Apply a tiling transformation to all payload ops and store both the
662/// tiled operation as well as the created tile loops.
663template <typename Range>
664static LogicalResult applyTilingToAll(
665 RewriterBase &rewriter, Operation *transformOp, Range &&payloadOps,
666 unsigned numLoops, transform::TransformResults &transformResults,
667 bool packedResults,
668 function_ref<FailureOr<scf::SCFTileAndFuseResult>(TilingInterface)>
669 applyFn) {
670 SmallVector<Operation *> tiledLinalgOps;
671 SmallVector<SmallVector<Operation *>> loopOps(numLoops);
672 size_t numTargets = llvm::range_size(payloadOps);
673
674 for (Operation *target : payloadOps) {
675 auto tilingInterfaceOp = dyn_cast<TilingInterface>(Val: target);
676 if (!tilingInterfaceOp)
677 return transformOp->emitError(message: "only TilingInterface ops are supported");
678
679 rewriter.setInsertionPoint(target);
680 FailureOr<scf::SCFTileAndFuseResult> tiledResults =
681 applyFn(tilingInterfaceOp);
682 if (failed(Result: tiledResults))
683 return failure();
684
685 // Perform the replacement of tiled and fused values.
686 SmallVector<Operation *> opsToReplace{target};
687 llvm::append_range(C&: opsToReplace, R&: tiledResults->fusedProducers);
688 for (Operation *toReplace : opsToReplace) {
689 for (OpResult res : toReplace->getResults())
690 if (auto replacement = tiledResults->replacements.lookup(Val: res))
691 rewriter.replaceAllUsesWith(from: res, to: replacement);
692 if (toReplace->use_empty()) {
693 rewriter.eraseOp(op: toReplace);
694 }
695 }
696
697 // Report back the relevant handles to the transform op.
698 tiledLinalgOps.push_back(Elt: tiledResults->tiledAndFusedOps.front());
699 assert(tiledResults->loops.size() == numLoops &&
700 "Mismatched number of loops, tile and fuse transform should have "
701 "failed");
702 for (unsigned int i = 0; i < numLoops; ++i)
703 loopOps[i].push_back(Elt: tiledResults->loops[i]);
704 }
705
706 transformResults.set(value: transformOp->getOpResult(idx: 0), ops&: tiledLinalgOps);
707 if (packedResults) {
708 // In case of packed results, all created loops are assigned to a single
709 // handle. Loops are returned in order of targets such as:
710 // %loops_handle = {
711 // target0:loop0, ..., target0:loopN,
712 // target1:loop0, ..., target1:loopN,
713 // ... }
714 SmallVector<Operation *> flattenedLoopOps;
715 for (unsigned int idx = 0; idx < numTargets; ++idx)
716 for (unsigned int i = 0; i < numLoops; ++i)
717 flattenedLoopOps.push_back(Elt: loopOps[i][idx]);
718 transformResults.set(value: transformOp->getOpResult(idx: 1), ops&: flattenedLoopOps);
719 } else {
720 for (unsigned int i = 0; i < numLoops; ++i)
721 transformResults.set(value: transformOp->getOpResult(idx: i + 1), ops&: loopOps[i]);
722 }
723
724 return success();
725}
726
727DiagnosedSilenceableFailure
728transform::FuseOp::apply(transform::TransformRewriter &rewriter,
729 mlir::transform::TransformResults &transformResults,
730 mlir::transform::TransformState &state) {
731 auto transformOp = cast<TransformOpInterface>(Val: getOperation());
732
733 SmallVector<OpFoldResult> mixedTileSizes;
734 DiagnosedSilenceableFailure status =
735 getPackedTileSizes()
736 ? unpackSingleIndexResultPayloadOperations(
737 state, transformOp, result&: mixedTileSizes, packedHandle: getPackedTileSizes())
738 : unpackSingleIndexResultPayloadOperations(
739 state, transformOp, result&: mixedTileSizes, ofrs: getMixedTileSizes());
740 if (!status.succeeded())
741 return status;
742 SmallVector<int64_t> tileInterchange;
743 status = reifyMixedParamAndHandleResults(
744 state, transformOp, mixedResults: getMixedTileInterchange(), reified&: tileInterchange);
745 if (!status.succeeded())
746 return status;
747
748 scf::SCFTilingOptions tilingOptions;
749 tilingOptions.interchangeVector = std::move(tileInterchange);
750 bool useForall = getUseForall();
751 tilingOptions.setLoopType(useForall
752 ? scf::SCFTilingOptions::LoopType::ForallOp
753 : scf::SCFTilingOptions::LoopType::ForOp);
754 tilingOptions.setTileSizes(mixedTileSizes);
755 scf::SCFTileAndFuseOptions tileAndFuseOptions;
756 tileAndFuseOptions.tilingOptions = std::move(tilingOptions);
757 // Optional caller-asserted pack/unpack inner-tile alignment (see
758 // InnerTileAlignment).
759 tileAndFuseOptions.tilingOptions.setInnerTileAlignments(
760 convertInnerTileAlignments(alignments: getInnerTileAlignments()));
761
762 if (getApplyCleanup()) {
763 MLIRContext *context = rewriter.getContext();
764 RewritePatternSet patterns(context);
765 tensor::ExtractSliceOp::getCanonicalizationPatterns(results&: patterns, context);
766 tensor::populateMergeConsecutiveInsertExtractSlicePatterns(patterns);
767 tensor::populateBubbleUpExtractSliceOpPatterns(patterns);
768 tileAndFuseOptions.cleanupPatterns = std::move(patterns);
769 }
770
771 size_t numLoops;
772 if (useForall) {
773 numLoops = 1;
774 } else {
775 numLoops = llvm::count_if(Range&: mixedTileSizes, P: [](OpFoldResult ofr) {
776 auto attr = dyn_cast<Attribute>(Val&: ofr);
777 if (!attr)
778 return true;
779 return cast<IntegerAttr>(Val&: attr).getInt() != 0;
780 });
781 }
782 LogicalResult result = applyTilingToAll(
783 rewriter, transformOp: getOperation(), payloadOps: state.getPayloadOps(value: getTarget()), numLoops,
784 transformResults, /*packedResults=*/getPackedTileSizes() != nullptr,
785 applyFn: [&](TilingInterface tilingInterfaceOp)
786 -> FailureOr<scf::SCFTileAndFuseResult> {
787 return tileConsumerAndFuseProducersUsingSCF(rewriter, consumer: tilingInterfaceOp,
788 options: tileAndFuseOptions);
789 });
790 return failed(Result: result) ? DiagnosedSilenceableFailure::definiteFailure()
791 : DiagnosedSilenceableFailure::success();
792}
793
794LogicalResult transform::FuseOp::verify() {
795 bool hasPackedTiles = getPackedTileSizes() != nullptr;
796 if (!getMixedTileSizes().empty() && hasPackedTiles)
797 return emitOpError(
798 message: "tile_sizes and packed_tile_sizes are mutually exclusive");
799
800 auto iterspace_rank = getStaticTileSizes().size();
801 ArrayRef<int64_t> permutation = getStaticTileInterchange();
802 if (permutation.size() > iterspace_rank)
803 return emitOpError()
804 << "interchange length exceeds iteration space dimensions ("
805 << iterspace_rank << "), found " << getTileInterchange();
806 SmallVector<bool> seen(iterspace_rank, false);
807 for (int64_t v : permutation) {
808 if (!ShapedType::isDynamic(dValue: v)) {
809 if (v < 0 || v >= static_cast<int64_t>(iterspace_rank))
810 return emitOpError() << "expects interchange values to be in range [0, "
811 << iterspace_rank << "), found: " << v;
812 if (seen[v])
813 return emitOpError() << "found duplicate interchange value: " << v;
814 seen[v] = true;
815 }
816 }
817
818 ArrayRef<int64_t> sizes = getStaticTileSizes();
819 size_t numExpectedLoops = getUseForall() || hasPackedTiles
820 ? 1
821 : sizes.size() - llvm::count(Range&: sizes, Element: 0);
822 if (numExpectedLoops != getNumResults() - 1)
823 return emitOpError() << "expects " << numExpectedLoops << " loop results";
824
825 return verifyInnerTileAlignments(op: getOperation(), alignments: getInnerTileAlignments());
826}
827
828SmallVector<OpFoldResult> transform::FuseOp::getMixedTileSizes() {
829 return getMixedValues(staticValues: getStaticTileSizes(), dynamicValues: getTileSizes(), context: getContext());
830}
831
832SmallVector<OpFoldResult> transform::FuseOp::getMixedTileInterchange() {
833 return getMixedValues(staticValues: getStaticTileInterchange(), dynamicValues: getTileInterchange(),
834 context: getContext());
835}
836
837void transform::FuseOp::getEffects(
838 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
839 consumesHandle(handles: getTargetMutable(), effects);
840 onlyReadsHandle(handles: getTileSizesMutable(), effects);
841 onlyReadsHandle(handles: getPackedTileSizesMutable(), effects);
842 onlyReadsHandle(handles: getTileInterchangeMutable(), effects);
843 producesHandle(handles: getOperation()->getOpResults(), effects);
844 modifiesPayload(effects);
845}
846
847//===----------------------------------------------------------------------===//
848// FuseIntoContainingOp
849//===----------------------------------------------------------------------===//
850
851void transform::FuseIntoContainingOp::build(OpBuilder &builder,
852 OperationState &result,
853 Value producerOp,
854 Value containingOp) {
855 result.addOperands(newOperands: {producerOp, containingOp});
856 auto resultType = transform::AnyOpType::get(ctx: builder.getContext());
857 result.addTypes(newTypes: {resultType, resultType});
858}
859
860/// Add new operands to the forall op for users of the producerOp
861/// that are dominated by the containing scf.forall op.
862static Operation *replaceForAllWithNewSignature(
863 RewriterBase &rewriter, Diagnostic &diag, Operation *producerOp,
864 Operation *containingOp, TilingResult &tileAndFuseResult,
865 int64_t resultNumber, SmallVector<OpFoldResult> &offsets,
866 SmallVector<OpFoldResult> &sizes) {
867
868 // Count number of users not including the containing op
869 SetVector<Operation *> dominatedUsers;
870 DominanceInfo domInfo(containingOp);
871 for (Operation *user : producerOp->getResult(idx: resultNumber).getUsers()) {
872 if (!containingOp->isAncestor(other: user) &&
873 (domInfo.dominates(a: containingOp, b: user))) {
874 dominatedUsers.insert(X: user);
875 }
876 }
877 if (dominatedUsers.empty())
878 return nullptr;
879
880 // Create new scf.forall op
881 auto forallOp = cast<scf::ForallOp>(Val: containingOp);
882 OpBuilder::InsertionGuard g(rewriter);
883 rewriter.setInsertionPoint(forallOp);
884
885 // Get new output
886 Location loc = forallOp.getLoc();
887 auto genericOp = dyn_cast<linalg::GenericOp>(Val: producerOp);
888 if (!genericOp)
889 return nullptr;
890 SmallVector<Value> outputs = genericOp.getOutputs();
891 SmallVector<Value> newOuts(forallOp.getOutputs());
892 newOuts.push_back(Elt: outputs[resultNumber]);
893
894 // Create new scf.forall op
895 auto newforallOp = scf::ForallOp::create(
896 builder&: rewriter, location: loc, lbs: forallOp.getMixedLowerBound(),
897 ubs: forallOp.getMixedUpperBound(), steps: forallOp.getMixedStep(), outputs: newOuts,
898 mapping: forallOp.getMapping());
899 rewriter.eraseBlock(block: newforallOp.getBody());
900 newforallOp.getRegion().takeBody(other&: forallOp.getRegion());
901
902 // Add additional block argument for new value being returned
903 // and replaces all uses of the new output with corresponding bbArg
904 // inside the scf.forall to enable fusion into this new scf.forall.
905 newforallOp.getBody()->addArgument(type: newOuts.back().getType(),
906 loc: newOuts.back().getLoc());
907 auto bbArgs = newforallOp.getBody()->getArguments();
908 rewriter.replaceUsesWithIf(from: newOuts.back(), to: bbArgs.back(),
909 functor: [&](OpOperand &use) {
910 Operation *op = use.getOwner();
911 return newforallOp->isProperAncestor(other: op);
912 });
913
914 // Fix terminator
915 scf::InParallelOp terminatorOp = newforallOp.getTerminator();
916 SmallVector<Operation *> yieldingOps = llvm::map_to_vector<4>(
917 C: terminatorOp.getYieldingOps(), F: [](Operation &op) { return &op; });
918 Operation *firstYieldOp = yieldingOps.front();
919 rewriter.setInsertionPoint(firstYieldOp);
920 Value src = tileAndFuseResult.tiledValues[0];
921 Value dst = newforallOp.getRegionIterArgs().back();
922 SmallVector<OpFoldResult> strides(offsets.size(), rewriter.getIndexAttr(value: 1));
923 tensor::ParallelInsertSliceOp::create(builder&: rewriter, location: firstYieldOp->getLoc(), source: src,
924 dest: dst, offsets, sizes, strides);
925
926 for (auto result : llvm::enumerate(First: forallOp.getResults())) {
927 rewriter.replaceAllUsesWith(from: result.value(),
928 to: newforallOp->getResult(idx: result.index()));
929 }
930 rewriter.replaceUsesWithIf(from: producerOp->getResult(idx: resultNumber),
931 to: newforallOp->getResults().back(),
932 functor: [&](OpOperand &use) {
933 Operation *user = use.getOwner();
934 return dominatedUsers.contains(key: user);
935 });
936 return newforallOp;
937}
938
939/// Given two operands coming from a loop iter arg, 'src' and 'dst', return true
940/// if the operand 'src' is equal to 'dst' or equal to a iter arg present in a
941/// outer loop. To determine the second condition, this function iterates
942/// using a worklist over the enclosing loops, trying to find 'src' in any of
943/// the parent loop's iter args.
944static bool sameOrEquivalentIterArg(Value src, Value dst) {
945 // Stack like vector containing possible iterArgs candidates. The first one
946 // is dst, and we will transverse the IR from there.
947 SmallVector<Value> destWorklist;
948 destWorklist.push_back(Elt: dst);
949
950 while (!destWorklist.empty()) {
951 Value currentDst = destWorklist.pop_back_val();
952
953 // We have found the same operand in some iter arg in the loop structure,
954 // so src and dst are equivalent.
955 if (src == currentDst)
956 return true;
957
958 // The operands are not equivalent, look for enclosing loops over
959 // currentDst.
960 auto bbArg = dyn_cast<BlockArgument>(Val&: currentDst);
961 if (!bbArg)
962 continue;
963
964 Block *parentBlock = bbArg.getOwner();
965 assert(parentBlock && "unlinked block argument");
966
967 Operation *parentOp = parentBlock->getParentOp();
968 assert(parentOp && "expected block argument with parent operation");
969
970 // Check if parent is loop-like. If it's not, do not add it to the worklist.
971 auto parentLoop = dyn_cast<LoopLikeOpInterface>(Val: parentOp);
972 if (!parentLoop)
973 continue;
974
975 for (auto innerIterArg : parentLoop.getRegionIterArgs()) {
976 // No need to check for null as innerIterArg is tied to parentLoop.
977 OpOperand *operand = parentLoop.getTiedLoopInit(bbArg: innerIterArg);
978 Value loopBlockArgument =
979 parentLoop->getOperand(idx: operand->getOperandNumber());
980 destWorklist.push_back(Elt: loopBlockArgument);
981 }
982 }
983
984 return false;
985}
986
987/// Find the first "extract" user of `producerOp` and tile it right before its
988/// use. The tiled op is fused under the `containingOp`.
989/// Return this fused op on success or nullptr if anything fails.
990/// If tiled op has uses that are dominated by `containingOp`, return
991/// a new `containingOp` with results of the fused op appended to
992/// results of the `containingOp` or nullptr if there are no dominated uses.
993static std::tuple<SmallVector<Operation *>, Operation *>
994tileAndFuseFirstExtractUse(RewriterBase &rewriter, Diagnostic &diag,
995 Operation *producerOp, Operation *containingOp,
996 ArrayRef<InnerTileAlignment> innerTileAlignments) {
997 LDBG() << "Try to fuse a direct extract use";
998 auto tileableProducer = dyn_cast<TilingInterface>(Val: producerOp);
999 if (!tileableProducer) {
1000 diag.attachNote(noteLoc: producerOp->getLoc())
1001 << "producer is not a TileableInterface: " << *producerOp;
1002 return {};
1003 }
1004
1005 // Search the producer slices accessed within the containing operation.
1006 // TODO: Generalize to more extract/insert/parallel_insert triples, maybe
1007 // evolve into an interface.
1008 auto it = llvm::find_if(Range: tileableProducer->getUsers(), P: [&](Operation *user) {
1009 auto sliceOp = dyn_cast<tensor::ExtractSliceOp>(Val: user);
1010 return sliceOp && containingOp->isProperAncestor(other: sliceOp);
1011 });
1012
1013 // Find a fusion opportunity.
1014 if (it == tileableProducer->getUsers().end()) {
1015 diag.attachNote(noteLoc: tileableProducer->getLoc())
1016 << "could not find fusion opportunity for: " << *tileableProducer;
1017 return {};
1018 }
1019 auto sliceOpToTile = cast<tensor::ExtractSliceOp>(Val: *it);
1020
1021 // Try to fuse the producer in-place.
1022 OpBuilder::InsertionGuard guard(rewriter);
1023 rewriter.setInsertionPoint(sliceOpToTile);
1024
1025 // Clone the producer inside the consumer and try to update the producer init
1026 // operands using the loop bbArgs if applicable. More precisely, if the bbArg
1027 // of the container loop points to a value that it is used by the consumer op,
1028 // then, instead of using such value on the consumer, use the value coming
1029 // from the bbArg instead. This allows to reuse the output tensor (instead of
1030 // creating a new one) of the container when both producer and container write
1031 // to the same output.
1032 if (LoopLikeOpInterface containerLoop =
1033 dyn_cast<LoopLikeOpInterface>(Val: sliceOpToTile->getParentOp())) {
1034 Operation *clone = rewriter.clone(op&: *producerOp);
1035 rewriter.modifyOpInPlace(root: clone, callable: [&]() {
1036 // Iterate over the outputs of the producer and over the loop bbArgs and
1037 // check if any bbArg points to the same value as the producer output. In
1038 // such case, make the producer output point to the bbArg directly.
1039 auto dpsInterface = dyn_cast<DestinationStyleOpInterface>(Val: clone);
1040 if (!dpsInterface)
1041 return;
1042
1043 for (OpOperand &initOperandPtr : dpsInterface.getDpsInitsMutable()) {
1044 Value producerOperand =
1045 clone->getOperand(idx: initOperandPtr.getOperandNumber());
1046 for (BlockArgument containerIterArg :
1047 containerLoop.getRegionIterArgs()) {
1048 OpOperand *bbArg = containerLoop.getTiedLoopInit(bbArg: containerIterArg);
1049 Value consumerOperand =
1050 containerLoop->getOperand(idx: bbArg->getOperandNumber());
1051 // The producer has the same init as the loop bbArg, use it.
1052 if (sameOrEquivalentIterArg(src: producerOperand, dst: consumerOperand)) {
1053 initOperandPtr.set(containerIterArg);
1054 }
1055 }
1056 }
1057 });
1058
1059 tileableProducer = dyn_cast<TilingInterface>(Val: clone);
1060 }
1061
1062 // Tile the producer.
1063 int64_t resultNumber =
1064 cast<OpResult>(Val: sliceOpToTile.getSource()).getResultNumber();
1065 LDBG() << "resultNumber: " << resultNumber;
1066
1067 SmallVector<OpFoldResult> offsets = sliceOpToTile.getMixedOffsets();
1068 SmallVector<OpFoldResult> sizes = sliceOpToTile.getMixedSizes();
1069
1070 FailureOr<TilingResult> tileAndFuseResult =
1071 tileableProducer.generateResultTileValue(b&: rewriter, resultNumber, offsets,
1072 sizes, innerTileAlignments);
1073
1074 if (failed(Result: tileAndFuseResult)) {
1075 diag.attachNote(noteLoc: tileableProducer->getLoc())
1076 << "failed to tile producer op: " << *tileableProducer;
1077 return {};
1078 }
1079
1080#ifndef NDEBUG
1081 for (auto *tiledOp : tileAndFuseResult->tiledOps) {
1082 LDBG() << "tiledProducer: " << *tiledOp;
1083 }
1084#endif
1085
1086 // Replace the extract op.
1087 auto maybeRankReduced = tensor::ExtractSliceOp::rankReduceIfNeeded(
1088 b&: rewriter, loc: sliceOpToTile->getLoc(), value: tileAndFuseResult->tiledValues[0],
1089 desiredShape: cast<RankedTensorType>(Val: sliceOpToTile->getResult(idx: 0).getType()).getShape());
1090 if (failed(Result: maybeRankReduced)) {
1091 diag.attachNote(noteLoc: producerOp->getLoc())
1092 << "shape types don't match (missing canonicalization?):\nTiledOp: "
1093 << tileAndFuseResult->tiledValues[0]
1094 << "\nSliceOp: " << sliceOpToTile.getOperation() << '\n';
1095 return {};
1096 }
1097 rewriter.replaceOp(op: sliceOpToTile, newValues: *maybeRankReduced);
1098
1099 // Add new outputs to containing op, if required
1100 Operation *newContainingOp = replaceForAllWithNewSignature(
1101 rewriter, diag, producerOp, containingOp, tileAndFuseResult&: *tileAndFuseResult,
1102 resultNumber, offsets, sizes);
1103
1104 // Cleanup clone.
1105 if (isa<LoopLikeOpInterface>(Val: containingOp))
1106 rewriter.eraseOp(op: tileableProducer);
1107
1108 return std::make_tuple(args&: tileAndFuseResult->tiledOps, args&: newContainingOp);
1109}
1110
1111/// First, find the first "scf::ForallOp" user of `producerOp` and ensure
1112/// it is exactly the `containingOp`, otherwise bail.
1113/// Then, find the first "extract" user of the tied block argument and tile it
1114/// right before its "extract" use. The tiled op is fused under the
1115/// `containingOp`.
1116/// Return this fused op on success or nullptr if anything fails.
1117static SmallVector<Operation *>
1118tileAndFuseFirstExtractUseThroughContainingOpBlockArgument(
1119 RewriterBase &rewriter, Diagnostic &diag, Operation *producerOp,
1120 Operation *containingOp, ArrayRef<InnerTileAlignment> innerTileAlignments) {
1121 LDBG() << "Try to fuse an extract use through block argument";
1122
1123 auto tileableProducer = dyn_cast<TilingInterface>(Val: producerOp);
1124 if (!tileableProducer) {
1125 diag.attachNote(noteLoc: producerOp->getLoc())
1126 << "producer is not a TileableInterface: " << *producerOp;
1127 return {};
1128 }
1129
1130 // Search the first use by a "scf::ForallOp" user.
1131 scf::ForallOp forallOp;
1132 auto itProducerUses =
1133 llvm::find_if(Range: tileableProducer->getUses(), P: [&](OpOperand &use) {
1134 forallOp = dyn_cast<scf::ForallOp>(Val: use.getOwner());
1135 return forallOp;
1136 });
1137 // If it's not from the containing op, return.
1138 if (!forallOp || forallOp != containingOp) {
1139 diag.attachNote(noteLoc: tileableProducer->getLoc())
1140 << "could not find a use by the containing op: " << *tileableProducer;
1141 return {};
1142 }
1143
1144 // Search the producer slices accessed within the containing
1145 // operation.
1146 // TODO: Generalize to more extract/insert/parallel_insert triples.
1147 // Maybe evolve into an interface.
1148 OpOperand *pUse = &(*itProducerUses);
1149 BlockArgument bbArg = forallOp.getTiedBlockArgument(opOperand: pUse);
1150
1151 // Search the producer slices accessed within the containing operation.
1152 // TODO: Generalize to more extract/insert/parallel_insert triples, maybe
1153 // evolve into an interface.
1154 auto itBBArgUsers = llvm::find_if(Range: bbArg.getUsers(), P: [&](Operation *user) {
1155 auto sliceOp = dyn_cast<tensor::ExtractSliceOp>(Val: user);
1156 return sliceOp && containingOp->isProperAncestor(other: sliceOp);
1157 });
1158
1159 // Find a fusion opportunity.
1160 if (itBBArgUsers == bbArg.getUsers().end()) {
1161 diag.attachNote(noteLoc: containingOp->getLoc())
1162 << "could not find fusion opportunity for bbArg: " << bbArg;
1163 return {};
1164 }
1165 auto sliceOpToTile = cast<tensor::ExtractSliceOp>(Val: *itBBArgUsers);
1166
1167 // Try to fuse the producer in-place.
1168 OpBuilder::InsertionGuard guard(rewriter);
1169 rewriter.setInsertionPoint(sliceOpToTile);
1170
1171 // Replace the use in the tileableProducer before tiling: clone, replace and
1172 // then tile.
1173 int64_t resultNumber = cast<OpResult>(Val: pUse->get()).getResultNumber();
1174 LDBG() << "resultNumber: " << resultNumber;
1175
1176 // Gather destination tensors.
1177 SmallVector<Value> destinationTensors;
1178 if (failed(Result: tensor::getOrCreateDestinations(
1179 b&: rewriter, loc: tileableProducer->getLoc(), op: tileableProducer,
1180 result&: destinationTensors))) {
1181 diag.attachNote(noteLoc: tileableProducer->getLoc())
1182 << "failed to get destination tensors for: " << *tileableProducer;
1183 return {};
1184 }
1185
1186 IRMapping bvm;
1187 bvm.map(from: destinationTensors[resultNumber], to: bbArg);
1188 auto tileableProducerClone =
1189 cast<TilingInterface>(Val: rewriter.clone(op&: *tileableProducer, mapper&: bvm));
1190 llvm::scope_exit scopeGuard(
1191 [&]() { rewriter.eraseOp(op: tileableProducerClone); });
1192
1193 // Tile the producer.
1194 FailureOr<TilingResult> tileAndFuseResult =
1195 tileableProducerClone.generateResultTileValue(
1196 b&: rewriter, resultNumber, offsets: sliceOpToTile.getMixedOffsets(),
1197 sizes: sliceOpToTile.getMixedSizes(), innerTileAlignments);
1198 if (failed(Result: tileAndFuseResult)) {
1199 diag.attachNote(noteLoc: tileableProducer->getLoc())
1200 << "failed to tile producer op: " << *tileableProducer;
1201 return {};
1202 }
1203
1204 // Replace the extract op.
1205 auto maybeRankReduced = tensor::ExtractSliceOp::rankReduceIfNeeded(
1206 b&: rewriter, loc: sliceOpToTile->getLoc(), value: tileAndFuseResult->tiledValues[0],
1207 desiredShape: cast<RankedTensorType>(Val: sliceOpToTile->getResult(idx: 0).getType()).getShape());
1208 assert(succeeded(maybeRankReduced) && "unexpected shape");
1209 rewriter.replaceOp(op: sliceOpToTile, newValues: *maybeRankReduced);
1210
1211 // Replace the use in containingOp.
1212 rewriter.modifyOpInPlace(root: containingOp, callable: [&]() {
1213 containingOp->setOperand(idx: pUse->getOperandNumber(),
1214 value: destinationTensors.front());
1215 });
1216
1217 return tileAndFuseResult->tiledOps;
1218}
1219
1220static Operation *cloneAndFuseFirstUse(RewriterBase &rewriter, Diagnostic &diag,
1221 Operation *producerOp,
1222 Operation *containingOp) {
1223 LDBG() << "Try to fuse an use by cloning";
1224
1225 // Gather all uses inside the containing op.
1226 SmallVector<OpOperand *> uses;
1227 for (OpResult result : producerOp->getOpResults()) {
1228 for (OpOperand &use : result.getUses()) {
1229 if (containingOp->isProperAncestor(other: use.getOwner())) {
1230 uses.push_back(Elt: &use);
1231 continue;
1232 }
1233 // Cannot clone and fuse if the use is by the containing op itself: fail
1234 // immediately.
1235 if (containingOp == use.getOwner()) {
1236 diag.attachNote(noteLoc: producerOp->getLoc())
1237 << "producer op use by containing op cannot be fused by cloning";
1238 return nullptr;
1239 }
1240 }
1241 }
1242
1243 // Check for a non-empty list of fusion opportunities.
1244 if (uses.empty()) {
1245 diag.attachNote(noteLoc: producerOp->getLoc()) << "no fusion opportunity by cloning";
1246 return nullptr;
1247 }
1248
1249 // Clone and fuse inside the containing op.
1250 Operation *fusedOp = nullptr;
1251 OpOperand *use = uses.front();
1252 // Parallel insert slice is not a valid clone destination.
1253 // TODO: Generalize to other type of ops.
1254 assert(!isa<tensor::ParallelInsertSliceOp>(use->getOwner()) &&
1255 "Parallel insert slice is not a valid clone destination");
1256 unsigned resultNumber = cast<OpResult>(Val: use->get()).getResultNumber();
1257 LDBG() << "resultNumber: " << resultNumber;
1258
1259 OpBuilder::InsertionGuard guard(rewriter);
1260 rewriter.setInsertionPoint(use->getOwner());
1261 fusedOp = rewriter.clone(op&: *producerOp);
1262 rewriter.modifyOpInPlace(
1263 root: use->getOwner(), callable: [&] { use->set(fusedOp->getOpResult(idx: resultNumber)); });
1264
1265 return fusedOp;
1266}
1267
1268bool transform::FuseIntoContainingOp::allowsRepeatedHandleOperands() {
1269 // Allow repeated handles since we are fusing everything anyway.
1270 return true;
1271}
1272
1273LogicalResult transform::FuseIntoContainingOp::verify() {
1274 return verifyInnerTileAlignments(op: getOperation(), alignments: getInnerTileAlignments());
1275}
1276
1277DiagnosedSilenceableFailure
1278transform::FuseIntoContainingOp::apply(transform::TransformRewriter &rewriter,
1279 transform::TransformResults &results,
1280 transform::TransformState &state) {
1281 SmallVector<Operation *> fusedOps;
1282 auto producerOps = state.getPayloadOps(value: getProducerOp());
1283 auto containingOps = state.getPayloadOps(value: getContainingOp());
1284 if (!llvm::hasSingleElement(C&: containingOps)) {
1285 return emitDefiniteFailure()
1286 << "requires exactly one containing_op handle (got "
1287 << llvm::range_size(Range&: containingOps) << ")";
1288 }
1289 Operation *containingOp = *containingOps.begin();
1290
1291 // Forward the optional, caller-asserted alignment of a fused pack/unpack op's
1292 // inner tiles relative to the loop tile sizes (see InnerTileAlignment) to
1293 // each fused producer.
1294 SmallVector<InnerTileAlignment> innerTileAlignments =
1295 convertInnerTileAlignments(alignments: getInnerTileAlignments());
1296
1297 // If nothing to fuse, propagate success.
1298 if (std::empty(cont: producerOps)) {
1299 results.set(value: cast<OpResult>(Val: getFusedOp()), ops: SmallVector<mlir::Operation *>{});
1300 results.set(value: cast<OpResult>(Val: getNewContainingOp()), ops: {containingOp});
1301 return DiagnosedSilenceableFailure::success();
1302 }
1303
1304 // Helper function to find the next producer that should be fused. Take any
1305 // producer that has a use inside the containing op.
1306 SetVector<Operation *> remainingProducers(llvm::from_range, producerOps);
1307 auto getNextProducer = [&]() -> FailureOr<Operation *> {
1308 for (const auto &it : enumerate(First&: remainingProducers)) {
1309 Operation *producerOp = it.value();
1310 // The containing op may be a user of producerOp: use isAncestor.
1311 int64_t numUsesInContainingOp =
1312 llvm::count_if(Range: producerOp->getUsers(), P: [&](Operation *op) {
1313 return containingOp->isAncestor(other: op);
1314 });
1315 // TODO: When resolving the TODO below (no duplicate ops), take an op
1316 // that has no use among the remaining producers. This is a topological
1317 // sorting.
1318 if (numUsesInContainingOp > 0) {
1319 if (numUsesInContainingOp == 1)
1320 remainingProducers.erase(I: remainingProducers.begin() + it.index());
1321 return producerOp;
1322 }
1323 }
1324 return failure();
1325 };
1326
1327 while (!remainingProducers.empty()) {
1328 auto nextProducer = getNextProducer();
1329 if (failed(Result: nextProducer)) {
1330 auto diag = mlir::emitSilenceableFailure(loc: getLoc())
1331 << "could not find next producer to fuse into container";
1332 diag.attachNote(loc: containingOp->getLoc()) << "containing op";
1333 return diag;
1334 }
1335
1336 Operation *producerOp = *nextProducer;
1337
1338 // Default diagnostic, to be complemented with more failure information.
1339 Diagnostic diag(producerOp->getLoc(), DiagnosticSeverity::Remark);
1340 diag << "could not fuse " << *producerOp << " into " << *containingOp;
1341
1342 // TODO: If there are multiple uses of the producer in the containing op,
1343 // we currently tile/clone the op multiple times (once per use). In some
1344 // cases, we can tile/clone once and reuse the value for each use.
1345 // Futhermore, producers should then be traversed according to a
1346 // topological sorting.
1347 auto [tiledOps, newContainingOp] = tileAndFuseFirstExtractUse(
1348 rewriter, diag, producerOp, containingOp, innerTileAlignments);
1349 if (!tiledOps.empty()) {
1350 LDBG() << "\nFused a direct extract use\n" << *containingOp;
1351 fusedOps.append(RHS: tiledOps);
1352 if (newContainingOp) {
1353 // Update handles associated with the containing op so we don't need to
1354 // invalidate them. This is a hack to support better composability
1355 // between tiling and fusion while a proper mechanism is being
1356 // investigated.
1357 //
1358 // DO NOT replicate this elsewhere unless you understand what you are
1359 // doing.
1360 LogicalResult replacementStatus =
1361 rewriter.notifyPayloadOperationReplaced(op: containingOp,
1362 replacement: newContainingOp);
1363 (void)replacementStatus;
1364 assert(succeeded(replacementStatus) &&
1365 "unable to update transform state mapping");
1366 rewriter.eraseOp(op: containingOp);
1367 containingOp = newContainingOp;
1368 }
1369 continue;
1370 }
1371
1372 SmallVector<Operation *> tiledContainingOpOperand =
1373 tileAndFuseFirstExtractUseThroughContainingOpBlockArgument(
1374 rewriter, diag, producerOp, containingOp, innerTileAlignments);
1375 if (!tiledContainingOpOperand.empty()) {
1376 LDBG() << "\nFused an extract use through block argument\n"
1377 << *containingOp;
1378 fusedOps.append(RHS: tiledContainingOpOperand);
1379 continue;
1380 }
1381
1382 Operation *cloned =
1383 cloneAndFuseFirstUse(rewriter, diag, producerOp, containingOp);
1384 if (cloned) {
1385 LDBG() << "\nFused an use by cloning\n" << *containingOp;
1386 fusedOps.push_back(Elt: cloned);
1387 continue;
1388 }
1389 return DiagnosedSilenceableFailure::silenceableFailure(diag: std::move(diag));
1390 }
1391
1392 results.set(value: cast<OpResult>(Val: getFusedOp()), ops&: fusedOps);
1393 results.set(value: cast<OpResult>(Val: getNewContainingOp()), ops: {containingOp});
1394 return DiagnosedSilenceableFailure::success();
1395}
1396
1397void transform::FuseIntoContainingOp::getEffects(
1398 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
1399 consumesHandle(handles: getProducerOpMutable(), effects);
1400 onlyReadsHandle(handles: getContainingOpMutable(), effects);
1401 producesHandle(handles: getOperation()->getOpResults(), effects);
1402 modifiesPayload(effects);
1403}
1404
1405//===----------------------------------------------------------------------===//
1406// GeneralizeOp
1407//===----------------------------------------------------------------------===//
1408
1409DiagnosedSilenceableFailure
1410transform::GeneralizeOp::applyToOne(transform::TransformRewriter &rewriter,
1411 LinalgOp target,
1412 transform::ApplyToEachResultList &results,
1413 transform::TransformState &state) {
1414 // Exit early if no transformation is needed.
1415 if (isa<GenericOp>(Val: target)) {
1416 results.push_back(op: target);
1417 return DiagnosedSilenceableFailure::success();
1418 }
1419 rewriter.setInsertionPoint(target);
1420 FailureOr<LinalgOp> generic = generalizeNamedOp(rewriter, linalgOp: target);
1421 if (succeeded(Result: generic)) {
1422 results.push_back(op: generic->getOperation());
1423 return DiagnosedSilenceableFailure::success();
1424 }
1425 return emitDefaultSilenceableFailure(target);
1426}
1427
1428//===----------------------------------------------------------------------===//
1429// SpecializeOp
1430//===----------------------------------------------------------------------===/
1431
1432DiagnosedSilenceableFailure
1433transform::SpecializeOp::applyToOne(transform::TransformRewriter &rewriter,
1434 LinalgOp target,
1435 transform::ApplyToEachResultList &results,
1436 transform::TransformState &state) {
1437 // Exit early if the operation is not a generic.
1438 if (!isa<GenericOp>(Val: target)) {
1439 results.push_back(op: target);
1440 return DiagnosedSilenceableFailure::success();
1441 }
1442 rewriter.setInsertionPoint(target);
1443 FailureOr<LinalgOp> named =
1444 specializeGenericOp(rewriter, genericOp: cast<GenericOp>(Val&: target), emitCategoryOps: getEmitCategory());
1445 if (succeeded(Result: named)) {
1446 results.push_back(op: named->getOperation());
1447 return DiagnosedSilenceableFailure::success();
1448 }
1449 return emitDefaultSilenceableFailure(target);
1450}
1451
1452//===----------------------------------------------------------------------===//
1453// InterchangeOp
1454//===----------------------------------------------------------------------===//
1455
1456DiagnosedSilenceableFailure
1457transform::InterchangeOp::applyToOne(transform::TransformRewriter &rewriter,
1458 GenericOp target,
1459 transform::ApplyToEachResultList &results,
1460 transform::TransformState &state) {
1461 ArrayRef<int64_t> interchangeVector = getIteratorInterchange();
1462 // Exit early if no transformation is needed.
1463 if (interchangeVector.empty()) {
1464 results.push_back(op: target);
1465 return DiagnosedSilenceableFailure::success();
1466 }
1467
1468 unsigned numLoops = cast<LinalgOp>(Val: target.getOperation()).getNumLoops();
1469 if (interchangeVector.size() != numLoops) {
1470 return emitSilenceableError()
1471 << getIteratorInterchangeAttrName() << " has length ("
1472 << interchangeVector.size()
1473 << ") different from the number of loops in the target operation ("
1474 << numLoops << ")";
1475 }
1476 FailureOr<GenericOp> res = interchangeGenericOp(
1477 rewriter, genericOp: target, interchangeVector: SmallVector<unsigned>(interchangeVector));
1478 if (failed(Result: res))
1479 return emitDefiniteFailure() << "failed to apply";
1480 results.push_back(op: res->getOperation());
1481 return DiagnosedSilenceableFailure::success();
1482}
1483
1484LogicalResult transform::InterchangeOp::verify() {
1485 ArrayRef<int64_t> permutation = getIteratorInterchange();
1486 auto sequence = llvm::to_vector(Range: llvm::seq<int64_t>(Begin: 0, End: permutation.size()));
1487 if (!std::is_permutation(first1: sequence.begin(), last1: sequence.end(),
1488 first2: permutation.begin(), last2: permutation.end())) {
1489 return emitOpError()
1490 << "expects iterator_interchange to be a permutation, found "
1491 << getIteratorInterchange();
1492 }
1493 return success();
1494}
1495
1496//===----------------------------------------------------------------------===//
1497// LinalgCopyToMemrefOp
1498//===----------------------------------------------------------------------===//
1499
1500DiagnosedSilenceableFailure transform::LinalgCopyToMemrefOp::applyToOne(
1501 transform::TransformRewriter &rewriter, Operation *targetOp,
1502 transform::ApplyToEachResultList &results,
1503 transform::TransformState &state) {
1504
1505 // Check if the target can be converted.
1506 if (!isa<linalg::CopyOp>(Val: targetOp)) {
1507 DiagnosedSilenceableFailure diag =
1508 emitSilenceableError() << "only linalg.copy target ops are supported";
1509 diag.attachNote(loc: targetOp->getLoc()) << "target op";
1510 return diag;
1511 }
1512
1513 auto copyOp = dyn_cast<linalg::CopyOp>(Val: targetOp);
1514 if (!copyOp.hasPureBufferSemantics()) {
1515 DiagnosedSilenceableFailure diag =
1516 emitSilenceableError()
1517 << "cannot transform a linalg.copy on tensors into a memref.copy";
1518 diag.attachNote(loc: targetOp->getLoc()) << "target op";
1519 return diag;
1520 }
1521
1522 SmallVector<Value> inputs = copyOp.getInputs();
1523 SmallVector<Value> outputs = copyOp.getOutputs();
1524 assert(inputs.size() == 1 && "expected linalg copy op with one input");
1525 assert(outputs.size() == 1 && "expected memref copy op with one output");
1526 Value input = inputs.front();
1527 Value output = outputs.front();
1528
1529 // linalg.copy supports different element types on source/dest whereas
1530 // memref.copy does not, so we must check that the source and dest types can
1531 // be handled by memref.copy and otherwise reject the transformation.
1532 if (!isa<ShapedType>(Val: input.getType())) {
1533 DiagnosedSilenceableFailure diag =
1534 emitSilenceableError()
1535 << "cannot transform a linalg.copy which input has no shape";
1536 diag.attachNote(loc: targetOp->getLoc()) << "target op";
1537 return diag;
1538 }
1539
1540 // linalg.copy destination must be a shaped type.
1541 assert(isa<ShapedType>(output.getType()));
1542
1543 if (cast<ShapedType>(Val: input.getType()).getElementType() !=
1544 cast<ShapedType>(Val: output.getType()).getElementType()) {
1545 DiagnosedSilenceableFailure diag =
1546 emitSilenceableError()
1547 << "cannot transform a linalg.copy with different source and "
1548 "destination element types ";
1549 diag.attachNote(loc: targetOp->getLoc()) << "target op";
1550 return diag;
1551 }
1552
1553 // Target can be converted, do it.
1554 auto memrefCopyOp =
1555 rewriter.replaceOpWithNewOp<memref::CopyOp>(op: targetOp, args&: input, args&: output);
1556
1557 results.push_back(op: memrefCopyOp);
1558 return DiagnosedSilenceableFailure::success();
1559}
1560
1561//===----------------------------------------------------------------------===//
1562// LowerPackOp
1563//===----------------------------------------------------------------------===//
1564
1565DiagnosedSilenceableFailure transform::LowerPackOp::applyToOne(
1566 transform::TransformRewriter &rewriter, linalg::PackOp target,
1567 transform::ApplyToEachResultList &transformResults,
1568 transform::TransformState &state) {
1569 if (!target.hasPureTensorSemantics()) {
1570 return mlir::emitSilenceableFailure(loc: target->getLoc())
1571 << "lower_pack only supports tensor semantics. The target has "
1572 "memref operands";
1573 }
1574 rewriter.setInsertionPoint(target);
1575 bool lowerPadLikeWithInsertSlice = getLowerPadLikeWithInsertSlice();
1576 FailureOr<LowerPackResult> res =
1577 lowerPack(rewriter, packOp: target, lowerPadLikeWithInsertSlice);
1578 if (failed(Result: res)) {
1579 return mlir::emitSilenceableFailure(loc: target->getLoc())
1580 << "cannot lower to pad + expand + transpose";
1581 }
1582 transformResults.push_back(op: res->padOp);
1583 transformResults.push_back(op: res->expandShapeOp);
1584 transformResults.push_back(op: res->transposeOp);
1585 return DiagnosedSilenceableFailure::success();
1586}
1587
1588//===----------------------------------------------------------------------===//
1589// LowerUnPackOp
1590//===----------------------------------------------------------------------===//
1591
1592DiagnosedSilenceableFailure transform::LowerUnPackOp::applyToOne(
1593 transform::TransformRewriter &rewriter, linalg::UnPackOp target,
1594 transform::ApplyToEachResultList &transformResults,
1595 transform::TransformState &state) {
1596 if (!target.hasPureTensorSemantics()) {
1597 DiagnosedSilenceableFailure diag =
1598 emitSilenceableError() << "lower_unpack only supports tensor "
1599 "semantics. The target has memref operands";
1600 diag.attachNote(loc: target->getLoc()) << "target payload op";
1601 return diag;
1602 }
1603 rewriter.setInsertionPoint(target);
1604 bool lowerUnpadLikeWithExtractSlice = getLowerUnpadLikeWithExtractSlice();
1605 FailureOr<LowerUnPackOpResult> res =
1606 lowerUnPack(rewriter, unPackOp: target, lowerUnpadLikeWithExtractSlice);
1607 if (failed(Result: res)) {
1608 DiagnosedSilenceableFailure diag =
1609 emitSilenceableError()
1610 << "cannot lower to transpose + collapse + extract";
1611 diag.attachNote(loc: target->getLoc()) << "target payload op";
1612 return diag;
1613 }
1614 transformResults.push_back(op: res->emptyOp);
1615 transformResults.push_back(op: res->transposeOp);
1616 transformResults.push_back(op: res->collapseShapeOp);
1617 transformResults.push_back(op: res->extractSliceOp);
1618 transformResults.push_back(op: res->copyOp);
1619 return DiagnosedSilenceableFailure::success();
1620}
1621
1622//===---------------------------------------------------------------------===//
1623// MatchOp
1624//===---------------------------------------------------------------------===//
1625
1626void transform::MatchOp::build(OpBuilder &builder, OperationState &result,
1627 Value target, ArrayRef<StringRef> opNames) {
1628 result.addOperands(newOperands: target);
1629 result.addAttribute(name: MatchOp::getOpsAttrName(name: result.name),
1630 attr: builder.getStrArrayAttr(values: opNames));
1631 result.addTypes(newTypes: transform::AnyOpType::get(ctx: builder.getContext()));
1632}
1633
1634void transform::MatchOp::build(OpBuilder &builder, OperationState &result,
1635 TypeRange resultTypes, Value target,
1636 ArrayRef<StringRef> opNames) {
1637 result.addOperands(newOperands: target);
1638 result.addAttribute(name: MatchOp::getOpsAttrName(name: result.name),
1639 attr: builder.getStrArrayAttr(values: opNames));
1640 result.addTypes(newTypes&: resultTypes);
1641}
1642
1643DiagnosedSilenceableFailure
1644transform::MatchOp::apply(transform::TransformRewriter &rewriter,
1645 transform::TransformResults &results,
1646 transform::TransformState &state) {
1647 llvm::StringSet<> strs;
1648 if (getOps().has_value())
1649 strs.insert_range(R: getOps()->getAsValueRange<StringAttr>());
1650
1651 auto payloadOps = state.getPayloadOps(value: getTarget());
1652 if (!llvm::hasSingleElement(C&: payloadOps)) {
1653 return emitDefiniteFailure(message: "requires exactly one target handle");
1654 }
1655
1656 SmallVector<Operation *> res;
1657 bool incorrectNumOperandTypes = false;
1658 auto matchFun = [&](Operation *op) {
1659 if (getOps().has_value() && !strs.contains(key: op->getName().getStringRef()))
1660 return;
1661
1662 // Interfaces cannot be matched by name, just by ID.
1663 // So we specifically encode the interfaces we care about for this op.
1664 if (getInterface().has_value()) {
1665 auto iface = getInterface().value();
1666 if (iface == transform::MatchInterfaceEnum::LinalgOp &&
1667 !isa<LinalgOp>(Val: op))
1668 return;
1669 if (iface == transform::MatchInterfaceEnum::TilingInterface &&
1670 !isa<TilingInterface>(Val: op))
1671 return;
1672 if (iface == transform::MatchInterfaceEnum::LoopLikeInterface &&
1673 !isa<LoopLikeOpInterface>(Val: op))
1674 return;
1675 }
1676
1677 // Check if all specified attributes match.
1678 if (getOpAttrs().has_value()) {
1679 DictionaryAttr opAttrs = getOpAttrs().value();
1680 for (NamedAttribute attr : opAttrs) {
1681 if (attr.getName() == getInterfaceAttrName() ||
1682 attr.getName() == getOpsAttrName())
1683 continue;
1684 std::optional<Attribute> inherent = op->getInherentAttr(name: attr.getName());
1685 Attribute actual = inherent.has_value()
1686 ? *inherent
1687 : op->getDiscardableAttr(name: attr.getName());
1688 if (!actual)
1689 return;
1690 if (actual != attr.getValue())
1691 return;
1692 }
1693 }
1694
1695 if (getFilterResultType().has_value()) {
1696 Type t = getFilterResultType().value();
1697 if (op->getNumResults() != 1 || op->getResultTypes().front() != t)
1698 return;
1699 }
1700
1701 if (getFilterOperandTypes().has_value()) {
1702 mlir::ArrayAttr types = getFilterOperandTypes().value();
1703 auto operandTypes = op->getOperandTypes();
1704
1705 if (types.size() == 1) {
1706 // All the operands must must be equal to the specified type
1707 auto typeattr =
1708 dyn_cast<mlir::TypeAttr>(Val: getFilterOperandTypes().value()[0]);
1709 Type t = cast<::mlir::Type>(Val: typeattr.getValue());
1710 if (!llvm::all_of(Range: op->getOperandTypes(),
1711 P: [&](Type operandType) { return operandType == t; }))
1712 return;
1713 } else {
1714 // The operand types must match all the types in the list (in the same
1715 // order in with they are specified)
1716 if (types.size() != operandTypes.size()) {
1717 incorrectNumOperandTypes = true;
1718 return;
1719 }
1720
1721 for (auto [attr, operandType] :
1722 llvm::zip_equal(t: getFilterOperandTypes().value(), u&: operandTypes)) {
1723 auto typeattr = cast<mlir::TypeAttr>(Val: attr);
1724 Type type = cast<::mlir::Type>(Val: typeattr.getValue());
1725
1726 if (type != operandType)
1727 return;
1728 }
1729 }
1730 }
1731
1732 // All constraints are satisfied.
1733 res.push_back(Elt: op);
1734 return;
1735 };
1736
1737 (*payloadOps.begin())->walk(callback&: matchFun);
1738 if (incorrectNumOperandTypes)
1739 return emitDefiniteFailure(message: "If filter_operand_types contains more than a "
1740 "type, then it must contain as much types as "
1741 "the number of operands in the target ops");
1742 results.set(value: cast<OpResult>(Val: getResult()), ops&: res);
1743 return DiagnosedSilenceableFailure::success();
1744}
1745
1746//===---------------------------------------------------------------------===//
1747// MultiTileSizesOp
1748//===---------------------------------------------------------------------===//
1749
1750static void printMultitileSizesTypes(OpAsmPrinter &printer, Operation *op,
1751 Type targetType, Type lowSizeType, Type,
1752 Type) {
1753 printer.printFunctionalType(inputs: TypeRange{targetType}, results: TypeRange{lowSizeType});
1754}
1755
1756static ParseResult parseMultitileSizesTypes(OpAsmParser &parser,
1757 Type &targetType, Type &lowSizeType,
1758 Type &highSizeType,
1759 Type &splitPointType) {
1760 FunctionType funcType;
1761 llvm::SMLoc typeLoc = parser.getCurrentLocation();
1762 if (failed(Result: parser.parseType<FunctionType>(result&: funcType)))
1763 return failure();
1764
1765 if (funcType.getNumInputs() != 1 || funcType.getNumResults() != 1) {
1766 parser.emitError(loc: typeLoc) << "expects a trailing functional type with one "
1767 "argument and one result";
1768 }
1769 targetType = funcType.getInput(i: 0);
1770 lowSizeType = highSizeType = splitPointType = funcType.getResult(i: 0);
1771
1772 return success();
1773}
1774
1775DiagnosedSilenceableFailure transform::MultiTileSizesOp::applyToOne(
1776 transform::TransformRewriter &rewriter, LinalgOp target,
1777 transform::ApplyToEachResultList &results, TransformState &state) {
1778 if (isa<TransformParamTypeInterface>(Val: getLowSize().getType())) {
1779 if (target.hasDynamicShape()) {
1780 auto diag = emitSilenceableError()
1781 << "cannot compute parametric tile sizes for dynamically "
1782 "shaped payload op";
1783 diag.attachNote(loc: target->getLoc()) << "payload op";
1784 return diag;
1785 }
1786
1787 FailureOr<StaticMultiSizeSpecification> spec = computeStaticMultiTileSizes(
1788 op: target, dimension: getDimension(), targetSize: getTargetSize(), divisor: getDivisor());
1789 if (failed(Result: spec)) {
1790 return emitSilenceableError()
1791 << "failed to compute multi-size tiling sizes";
1792 }
1793
1794 Builder builder(target.getContext());
1795 results.assign(range: llvm::map_range(
1796 C: ArrayRef<int64_t>({spec->lowTileSize, spec->highTileSize,
1797 spec->lowTileSize * spec->lowTripCount}),
1798 F: [&builder, this](int64_t value) {
1799 return builder.getIntegerAttr(
1800 type: cast<ParamType>(Val: getLowSize().getType()).getType(), value);
1801 }));
1802 return DiagnosedSilenceableFailure::success();
1803 }
1804
1805 OpBuilder builder(target.getContext());
1806 builder.setInsertionPoint(target);
1807 OpFoldResult targetSize = builder.getIndexAttr(value: getTargetSize());
1808 OpFoldResult divisor = builder.getIndexAttr(value: getDivisor());
1809 FailureOr<MultiSizeSpecification> spec = computeMultiTileSizes(
1810 builder, op: target, dimension: getDimension(), targetSize, divisor);
1811 if (failed(Result: spec)) {
1812 return emitSilenceableError() << "could not generate tile size computation";
1813 }
1814
1815 AffineExpr s0 = builder.getAffineSymbolExpr(position: 0);
1816 AffineExpr s1 = builder.getAffineSymbolExpr(position: 1);
1817 Operation *splitPoint =
1818 affine::makeComposedAffineApply(b&: builder, loc: target.getLoc(), e: s0 * s1,
1819 operands: {spec->lowTileSize, spec->lowTripCount});
1820 Operation *lowTileSize = spec->lowTileSize.getDefiningOp();
1821 Operation *highTileSize = spec->highTileSize.getDefiningOp();
1822 assert(lowTileSize && highTileSize && splitPoint &&
1823 "tile sizes are not produced by operations");
1824 results.reserve(size: results.size() + 3);
1825 results.push_back(op: lowTileSize);
1826 results.push_back(op: highTileSize);
1827 results.push_back(op: splitPoint);
1828 return DiagnosedSilenceableFailure::success();
1829}
1830
1831void transform::MultiTileSizesOp::getEffects(
1832 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
1833 onlyReadsHandle(handles: getTargetMutable(), effects);
1834 producesHandle(handles: getOperation()->getOpResults(), effects);
1835 if (isa<TransformParamTypeInterface>(Val: getLowSize().getType()))
1836 onlyReadsPayload(effects);
1837 else
1838 modifiesPayload(effects);
1839}
1840
1841LogicalResult transform::MultiTileSizesOp::verify() {
1842 if (getLowSize().getType() != getHighSize().getType() ||
1843 getLowSize().getType() != getSplitPoint().getType()) {
1844 return emitOpError() << "expects all results type to be the same";
1845 }
1846 return success();
1847}
1848
1849//===---------------------------------------------------------------------===//
1850// PackOp
1851//===---------------------------------------------------------------------===//
1852
1853void transform::PackOp::build(OpBuilder &builder, OperationState &result,
1854 Value target,
1855 ArrayRef<OpFoldResult> mixedPackedSizes) {
1856 SmallVector<int64_t> staticPackedSizes;
1857 SmallVector<Value> dynamicPackedSizes;
1858 dispatchIndexOpFoldResults(ofrs: mixedPackedSizes, dynamicVec&: dynamicPackedSizes,
1859 staticVec&: staticPackedSizes);
1860 // Call the default builder which sets up the proper operands segment sizes
1861 // attributes for multiple variadic operands. In the absence of this, horrible
1862 // bugs ensue.
1863 Type linalgOpHType = transform::OperationType::get(
1864 context: builder.getContext(), operation_name: GenericOp::getOperationName());
1865 build(odsBuilder&: builder, odsState&: result,
1866 /*resultType=*/packed_op: linalgOpHType,
1867 /*target=*/target,
1868 /*dynamic_sizes=*/packed_sizes: dynamicPackedSizes,
1869 /*static_sizes=*/static_packed_sizes: builder.getDenseI64ArrayAttr(values: staticPackedSizes));
1870}
1871
1872SmallVector<OpFoldResult> transform::PackOp::getMixedPackedSizes() {
1873 Builder b(getContext());
1874 return getMixedValues(staticValues: getStaticPackedSizes(), dynamicValues: getPackedSizes(), b);
1875}
1876
1877DiagnosedSilenceableFailure
1878transform::PackOp::apply(transform::TransformRewriter &rewriter,
1879 transform::TransformResults &transformResults,
1880 transform::TransformState &state) {
1881 auto targetOps = state.getPayloadOps(value: getTarget());
1882 // If nothing to pack, propagate success.
1883 if (std::empty(cont: targetOps)) {
1884 transformResults.set(value: cast<OpResult>(Val: getPackedOp()),
1885 ops: ArrayRef<Operation *>({}));
1886 return DiagnosedSilenceableFailure::success();
1887 }
1888 // Fail on multi-op handles.
1889 auto linalgOp = dyn_cast<LinalgOp>(Val: *targetOps.begin());
1890 if (!llvm::hasSingleElement(C&: targetOps) || !linalgOp) {
1891 return emitSilenceableError()
1892 << "requires target to map to exactly 1 LinalgOp (got "
1893 << llvm::range_size(Range&: targetOps) << ")";
1894 }
1895 // Fail on memref operands: pack only supports tensor semantics.
1896 if (!linalgOp.hasPureTensorSemantics()) {
1897 return emitSilenceableError()
1898 << "structured.pack only supports tensor semantics. The target has "
1899 "memref operands";
1900 }
1901 // Fail on mismatched number of pack sizes.
1902 if (getMixedPackedSizes().size() != linalgOp.getNumLoops()) {
1903 return emitSilenceableError()
1904 << "requires number of packed sizes match the number of loops ("
1905 << getMixedPackedSizes().size() << " vs " << linalgOp.getNumLoops()
1906 << ")";
1907 }
1908
1909 // Unpack handles to constants or actual SSA index values.
1910 SmallVector<OpFoldResult> packedSizes;
1911 DiagnosedSilenceableFailure status = unpackSingleIndexResultPayloadOperations(
1912 state, transformOp: *this, result&: packedSizes, ofrs: getMixedPackedSizes());
1913
1914 rewriter.setInsertionPoint(linalgOp);
1915 FailureOr<PackResult> maybeResult = pack(rewriter, linalgOp, packedSizes);
1916 if (failed(Result: maybeResult))
1917 return emitDefiniteFailure(message: "data tiling failed");
1918
1919 transformResults.set(value: cast<OpResult>(Val: getPackedOp()),
1920 ops: {maybeResult->packedLinalgOp.getOperation()});
1921 return DiagnosedSilenceableFailure::success();
1922}
1923
1924void transform::PackOp::getEffects(
1925 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
1926 transform::consumesHandle(handles: getTargetMutable(), effects);
1927 transform::onlyReadsHandle(handles: getPackedSizesMutable(), effects);
1928 transform::producesHandle(handles: getOperation()->getOpResults(), effects);
1929 transform::modifiesPayload(effects);
1930}
1931
1932//===---------------------------------------------------------------------===//
1933// PackGreedilyOp.
1934//===---------------------------------------------------------------------===//
1935
1936LogicalResult transform::PackGreedilyOp::verify() {
1937 if (!isPermutationVector(interchange: getMatmulInnerDimsOrder())) {
1938 return emitOpError() << getMatmulInnerDimsOrderAttrName()
1939 << " is not a valid permutation";
1940 }
1941 // TODO: relax to allow empty once we have another strategy than just matmul.
1942 if (!getMatmulPaddedSizesNextMultipleOf().empty()) {
1943 for (auto [s, nmo] :
1944 llvm::zip_equal(t: getMixedMatmulPackedSizes(),
1945 u: getMatmulPaddedSizesNextMultipleOf())) {
1946 std::optional<int64_t> maybeStaticPackedSize = getConstantIntValue(ofr: s);
1947 if (nmo != 0 &&
1948 (!maybeStaticPackedSize.has_value() || *maybeStaticPackedSize != 0)) {
1949 return emitOpError() << "at most one of the packed_size and the "
1950 "padded_sizes_next_multiple_of can be nonzero "
1951 "for the matmul strategy";
1952 }
1953 }
1954 }
1955 return success();
1956}
1957
1958DiagnosedSilenceableFailure
1959PackGreedilyOp::apply(transform::TransformRewriter &rewriter,
1960 transform::TransformResults &transformResults,
1961 transform::TransformState &state) {
1962 SmallVector<Operation *> results;
1963 for (auto linalgOp :
1964 llvm::make_isa_range<LinalgOp>(Range: state.getPayloadOps(value: getTarget()))) {
1965 // linalgOp will be replaced and the insertion point may be invalidated if
1966 // we set it before -> set it after.
1967 rewriter.setInsertionPointAfter(linalgOp);
1968 // Failing to pack greedily is perfectly fine.
1969 // In the future we will want to order packings according to some metric.
1970 FailureOr<PackResult> packResult = packMatmulGreedily(
1971 /*rewriter=*/rewriter,
1972 /*linalgOp=*/linalgOp,
1973 /*mnkPackedSizes=*/getMixedMatmulPackedSizes(),
1974 /*mnkPaddedSizesNextMultipleOf=*/
1975 getMatmulPaddedSizesNextMultipleOf(),
1976 /*mnkOrder=*/getMatmulInnerDimsOrder());
1977 if (succeeded(Result: packResult)) {
1978 results.push_back(Elt: packResult->packedLinalgOp);
1979 continue;
1980 }
1981 results.push_back(Elt: linalgOp);
1982 }
1983 transformResults.set(value: cast<OpResult>(Val: getPackedOp()), ops&: results);
1984 return DiagnosedSilenceableFailure::success();
1985}
1986
1987SmallVector<OpFoldResult> PackGreedilyOp::getMixedMatmulPackedSizes() {
1988 Builder b(getContext());
1989 return getMixedValues(staticValues: getStaticMatmulPackedSizes(), dynamicValues: getMatmulPackedSizes(),
1990 b);
1991}
1992
1993void transform::PackGreedilyOp::getEffects(
1994 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
1995 transform::consumesHandle(handles: getTargetMutable(), effects);
1996 transform::onlyReadsHandle(handles: getMatmulPackedSizesMutable(), effects);
1997 transform::producesHandle(handles: getOperation()->getOpResults(), effects);
1998 transform::modifiesPayload(effects);
1999}
2000
2001//===---------------------------------------------------------------------===//
2002// PackTransposeOp
2003//===---------------------------------------------------------------------===//
2004
2005LogicalResult transform::PackTransposeOp::verify() {
2006 if (!isPermutationVector(interchange: getInnerPerm())) {
2007 return emitOpError() << getInnerPermAttrName()
2008 << " is not a valid permutation";
2009 }
2010 if (!isPermutationVector(interchange: getOuterPerm())) {
2011 return emitOpError() << getOuterPermAttrName()
2012 << " is not a valid permutation";
2013 }
2014 if (getInnerPerm().empty() && getOuterPerm().empty()) {
2015 return emitOpError() << " at least one of " << getInnerPermAttrName()
2016 << " or " << getOuterPermAttrName()
2017 << " must be specified";
2018 }
2019 return success();
2020}
2021
2022namespace {
2023enum class OuterOrInnerPerm { Outer = 0, Inner = 1 };
2024} // namespace
2025
2026/// Return true if `permutation` is a valid permutation of the
2027/// `outer_dims_perm` (case OuterOrInnerPerm::Outer) or `inner_dims_pos`
2028/// (OuterOrInnerPerm::Inner) of the `tensor.pack` or `tensor.unpack` `op.
2029/// This is the case when the `permutation` rank matches the rank expected by
2030/// `op` and `permutation` is itself a permutation vector.
2031/// Return true if either `op` or `permutation` are empty to allow a simpler
2032/// polymorphic implementation.
2033template <typename RelayoutOpTy>
2034static bool isValidPackingPermutation(
2035 RelayoutOpTy op, ArrayRef<int64_t> permutation,
2036 OuterOrInnerPerm outerOrInnerPerm = OuterOrInnerPerm::Outer) {
2037 static_assert(
2038 llvm::is_one_of<RelayoutOpTy, linalg::PackOp, linalg::UnPackOp>::value,
2039 "applies to only pack or unpack operations");
2040 if (!op || permutation.empty())
2041 return true;
2042 size_t innerRank = op.getInnerDimsPos().size();
2043 if (outerOrInnerPerm == OuterOrInnerPerm::Inner)
2044 return permutation.size() == innerRank && isPermutationVector(interchange: permutation);
2045 // op.getOuterDimsPerm() may be empty, in which case it is identity.
2046 // Don't rely on it.
2047 if (std::is_same<RelayoutOpTy, linalg::PackOp>::value) {
2048 return permutation.size() == op.getSourceRank() &&
2049 isPermutationVector(interchange: permutation);
2050 }
2051 return permutation.size() == op.getDestRank() &&
2052 isPermutationVector(interchange: permutation);
2053}
2054
2055DiagnosedSilenceableFailure
2056transform::PackTransposeOp::apply(transform::TransformRewriter &rewriter,
2057 transform::TransformResults &transformResults,
2058 transform::TransformState &state) {
2059 auto packOrUnpackOps = state.getPayloadOps(value: getTargetPackOrUnPackOp());
2060 auto linalgOps = state.getPayloadOps(value: getTargetLinalgOp());
2061 // Step 1. If nothing to pack, propagate success.
2062 if (std::empty(cont: packOrUnpackOps)) {
2063 transformResults.set(value: cast<OpResult>(Val: getPackedOp()), ops: {});
2064 transformResults.set(value: cast<OpResult>(Val: getPackOp()), ops: {});
2065 transformResults.set(value: cast<OpResult>(Val: getUnPackOp()), ops: {});
2066 return DiagnosedSilenceableFailure::success();
2067 }
2068
2069 // Step 2. Bunch of runtime sanity check and error messages.
2070 // Step 2.1. Fail on multi-op handles.
2071 if (!llvm::hasSingleElement(C&: packOrUnpackOps) ||
2072 !llvm::hasSingleElement(C&: linalgOps)) {
2073 return emitSilenceableError()
2074 << "requires target to map to exactly 1 "
2075 "packing op and 1 packed op ("
2076 << "got " << llvm::range_size(Range&: packOrUnpackOps) << " and "
2077 << llvm::range_size(Range&: linalgOps) << ")";
2078 }
2079
2080 // Step 2.2. Fail on wrong type.
2081 auto packOp = dyn_cast<linalg::PackOp>(Val: *packOrUnpackOps.begin());
2082 auto unPackOp = dyn_cast<linalg::UnPackOp>(Val: *packOrUnpackOps.begin());
2083 if ((!packOp && !unPackOp)) {
2084 return emitSilenceableError() << "requires target to map to a "
2085 "linalg.pack or linalg.unpack";
2086 }
2087 LinalgOp linalgOpTarget = dyn_cast<LinalgOp>(Val: *linalgOps.begin());
2088 if (!linalgOpTarget)
2089 return emitSilenceableError() << "requires a LinalgOp target";
2090
2091 // Step 2.3. Fail if we can't get the producer / consumer Linalg op.
2092 LinalgOp linalgOp;
2093 if (packOp && packOp.getResult().hasOneUse())
2094 linalgOp = dyn_cast<LinalgOp>(Val: *(packOp.getResult().getUsers().begin()));
2095 else if (unPackOp)
2096 linalgOp = unPackOp.getSource().getDefiningOp<LinalgOp>();
2097 if (linalgOp != linalgOpTarget) {
2098 auto errorMsg =
2099 packOp ? StringLiteral{"not a single use by the LinalgOp target"}
2100 : StringLiteral{"not produced by the LinalgOp target"};
2101 return emitSilenceableError() << errorMsg;
2102 }
2103
2104 // Step 2.4. If we have an UnPackOp, we need to fetch the symmetrical
2105 // PackOp.
2106 if (unPackOp) {
2107 assert(!packOp && "packOp must be null on entry when unPackOp is not null");
2108 OpOperand *packUse = linalgOp.getDpsInitOperand(
2109 i: cast<OpResult>(Val: unPackOp.getSource()).getResultNumber());
2110 packOp = packUse->get().getDefiningOp<linalg::PackOp>();
2111 if (!packOp || !packOp.getResult().hasOneUse())
2112 return emitSilenceableError() << "could not find matching pack op";
2113 }
2114
2115 // Step 2.5. Fail if any permutation does not validate.
2116 for (auto permType : {OuterOrInnerPerm::Outer, OuterOrInnerPerm::Inner}) {
2117 ArrayRef<int64_t> perm =
2118 (permType == OuterOrInnerPerm::Outer) ? getOuterPerm() : getInnerPerm();
2119 auto errorMsg = (permType == OuterOrInnerPerm::Outer)
2120 ? StringLiteral{"invalid outer_perm"}
2121 : StringLiteral{"invalid inner_perm"};
2122 if (!isValidPackingPermutation(op: packOp, permutation: perm, outerOrInnerPerm: permType) ||
2123 !isValidPackingPermutation(op: unPackOp, permutation: perm, outerOrInnerPerm: permType)) {
2124 Operation *packOrUnpackOp =
2125 unPackOp ? unPackOp.getOperation() : packOp.getOperation();
2126 return emitSilenceableError() << errorMsg << ": " << *packOrUnpackOp;
2127 }
2128 }
2129
2130 // From here on, packOp and linalgOp are always present, unPackOp may or may
2131 // not be present.
2132 assert(packOp && linalgOp && "unexpected null op");
2133
2134 // Step 3. Actually transpose the ops.
2135 FailureOr<PackTransposeResult> res = packTranspose(
2136 rewriter, packOp, linalgOp, maybeUnPackOp: unPackOp, outerPerm: getOuterPerm(), innerPerm: getInnerPerm());
2137 // Preconditions have been checked, it is an error to fail here.
2138 assert(succeeded(res) && "unexpected packTranspose failure");
2139
2140 // Step 4. Return results.
2141 transformResults.set(value: cast<OpResult>(Val: getPackOp()), ops: {res->transposedPackOp});
2142 transformResults.set(value: cast<OpResult>(Val: getPackedOp()),
2143 ops: {res->transposedLinalgOp});
2144 if (unPackOp) {
2145 transformResults.set(value: cast<OpResult>(Val: getUnPackOp()),
2146 ops: {res->transposedUnPackOp});
2147 } else {
2148 transformResults.set(value: cast<OpResult>(Val: getUnPackOp()), ops: {});
2149 }
2150
2151 return DiagnosedSilenceableFailure::success();
2152}
2153
2154//===---------------------------------------------------------------------===//
2155// PadOp
2156//===---------------------------------------------------------------------===//
2157
2158void transform::PadOp::build(OpBuilder &b, OperationState &result, Value target,
2159 ArrayRef<int64_t> paddingDimensions,
2160 ArrayRef<int64_t> padToMultipleOf,
2161 ArrayRef<int64_t> nofoldFlags,
2162 ArrayRef<Attribute> transposePaddings,
2163 StringRef copyBackOp,
2164 bool usePrescribedTensorShapes) {
2165 auto resultType = transform::AnyOpType::get(ctx: b.getContext());
2166 return build(/*odsBuilder=*/b,
2167 /*result=*/odsState&: result,
2168 /*types=*/resultTypes: TypeRange{resultType, resultType},
2169 /*target=*/target,
2170 /*padding_values=*/ArrayAttr(), // let inference handle this
2171 /*padding_dimensions=*/b.getI64ArrayAttr(values: paddingDimensions),
2172 /*pad_to_multiple_of=*/ValueRange{},
2173 /*padToMultipleOf=*/
2174 static_pad_to_multiple_of: (padToMultipleOf.empty()
2175 ? DenseI64ArrayAttr()
2176 : b.getDenseI64ArrayAttr(values: padToMultipleOf)),
2177 /*nofold_flags=*/b.getI64ArrayAttr(values: nofoldFlags),
2178 /*transpose_paddings=*/b.getArrayAttr(value: transposePaddings),
2179 /*copy_back_op=*/b.getStringAttr(bytes: copyBackOp),
2180 /*use_prescribed_tensor_shapes=*/
2181 usePrescribedTensorShapes ? b.getUnitAttr() : nullptr);
2182}
2183
2184void transform::PadOp::build(OpBuilder &b, OperationState &result, Value target,
2185 ArrayRef<int64_t> paddingDimensions,
2186 ArrayRef<OpFoldResult> mixedPadToMultipleOf,
2187 ArrayRef<int64_t> nofoldFlags,
2188 ArrayRef<Attribute> transposePaddings,
2189 StringRef copyBackOp,
2190 bool usePrescribedTensorShapes) {
2191 auto resultType = transform::AnyOpType::get(ctx: b.getContext());
2192 SmallVector<int64_t> staticPadToMultipleOf;
2193 SmallVector<Value> dynamicPadToMultipleOf;
2194 dispatchIndexOpFoldResults(ofrs: mixedPadToMultipleOf, dynamicVec&: dynamicPadToMultipleOf,
2195 staticVec&: staticPadToMultipleOf);
2196 return build(/*odsBuilder=*/b,
2197 /*result=*/odsState&: result,
2198 /*types=*/resultTypes: TypeRange{resultType, resultType},
2199 /*target=*/target,
2200 /*padding_values=*/ArrayAttr(), // let inference handle this
2201 /*padding_dimensions=*/b.getI64ArrayAttr(values: paddingDimensions),
2202 /*pad_to_multiple_of=*/dynamicPadToMultipleOf,
2203 /*padToMultipleOf=*/static_pad_to_multiple_of: staticPadToMultipleOf,
2204 /*nofold_flags=*/b.getI64ArrayAttr(values: nofoldFlags),
2205 /*transpose_paddings=*/b.getArrayAttr(value: transposePaddings),
2206 /*copy_back_op=*/copyBackOp,
2207 /*use_prescribed_tensor_shapes=*/usePrescribedTensorShapes);
2208}
2209
2210void PadOp::getEffects(
2211 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
2212 consumesHandle(handles: getTargetMutable(), effects);
2213 onlyReadsHandle(handles: getPadToMultipleOfMutable(), effects);
2214 producesHandle(handles: getOperation()->getOpResults(), effects);
2215 modifiesPayload(effects);
2216}
2217
2218SmallVector<OpFoldResult> PadOp::getMixedPadToMultipleOf() {
2219 Builder b(getContext());
2220 return getMixedValues(staticValues: getStaticPadToMultipleOf(), dynamicValues: getPadToMultipleOf(), b);
2221}
2222
2223DiagnosedSilenceableFailure
2224transform::PadOp::apply(transform::TransformRewriter &rewriter,
2225 transform::TransformResults &results,
2226 transform::TransformState &state) {
2227 auto transformOp = cast<TransformOpInterface>(Val: getOperation());
2228 SmallVector<Operation *> paddedOps, padOps, copyBackOps;
2229
2230 for (Operation *target : state.getPayloadOps(value: getTarget())) {
2231 auto linalgTarget = dyn_cast<LinalgOp>(Val: target);
2232 if (!linalgTarget) {
2233 auto diag = emitSilenceableError() << "expected LinalgOp target";
2234 diag.attachNote(loc: target->getLoc()) << "target op";
2235 return diag;
2236 }
2237
2238 // Convert the integer packing flags to booleans.
2239 SmallVector<bool> nofoldFlags;
2240 for (int64_t packPadding :
2241 extractFromIntegerArrayAttr<int64_t>(attr: getNofoldFlags()))
2242 nofoldFlags.push_back(Elt: static_cast<bool>(packPadding));
2243
2244 // Convert the padding values to attributes.
2245 SmallVector<Attribute> paddingValues;
2246 for (auto const &[untypedAttr, elementOrTensorType] :
2247 llvm::zip(t: getPaddingValues(), u: linalgTarget->getOperandTypes())) {
2248
2249 if (matchPattern(attr: untypedAttr, pattern: ub::m_Poison())) {
2250 paddingValues.push_back(Elt: untypedAttr);
2251 continue;
2252 }
2253 auto attr = dyn_cast<TypedAttr>(Val: untypedAttr);
2254 if (!attr) {
2255 emitOpError(message: "expects padding values to be typed attributes or poison");
2256 return DiagnosedSilenceableFailure::definiteFailure();
2257 }
2258 Type elementType = getElementTypeOrSelf(type: elementOrTensorType);
2259 // Try to parse string attributes to obtain an attribute of element type.
2260 if (auto stringAttr = dyn_cast<StringAttr>(Val&: attr)) {
2261 auto parsedAttr = dyn_cast_if_present<TypedAttr>(Val: parseAttribute(
2262 attrStr: stringAttr, context: getContext(), type: elementType,
2263 /*numRead=*/nullptr, /*isKnownNullTerminated=*/true));
2264 if (!parsedAttr || parsedAttr.getType() != elementType) {
2265 auto diag = this->emitOpError(message: "expects a padding that parses to ")
2266 << elementType << ", got " << untypedAttr;
2267 diag.attachNote(noteLoc: linalgTarget.getLoc()) << "when applied to this op";
2268 return DiagnosedSilenceableFailure::definiteFailure();
2269 }
2270 paddingValues.push_back(Elt: parsedAttr);
2271 continue;
2272 }
2273 // Otherwise, add the attribute directly.
2274 if (attr.getType() != elementType) {
2275 auto diag = this->emitOpError(message: "expects a padding value of type ")
2276 << elementType << ", got " << attr;
2277 diag.attachNote(noteLoc: linalgTarget.getLoc()) << "when applied to this op";
2278 return DiagnosedSilenceableFailure::definiteFailure();
2279 }
2280 paddingValues.push_back(Elt: attr);
2281 }
2282
2283 // Extract the transpose vectors.
2284 SmallVector<SmallVector<int64_t>> transposePaddings;
2285 for (Attribute transposeVector : cast<ArrayAttr>(Val: getTransposePaddings()))
2286 transposePaddings.push_back(Elt: extractFromIntegerArrayAttr<int64_t>(
2287 attr: cast<ArrayAttr>(Val&: transposeVector)));
2288
2289 LinalgOp paddedOp;
2290 LinalgPaddingOptions options;
2291 options.paddingDimensions =
2292 extractFromIntegerArrayAttr<int64_t>(attr: getPaddingDimensions());
2293
2294 SmallVector<int64_t> padToMultipleOf;
2295 DiagnosedSilenceableFailure status = reifyMixedParamAndHandleResults(
2296 state, transformOp, mixedResults: getMixedPadToMultipleOf(), reified&: padToMultipleOf);
2297 if (!status.succeeded())
2298 return status;
2299 if (padToMultipleOf.empty())
2300 padToMultipleOf =
2301 SmallVector<int64_t>(options.paddingDimensions.size(), 1);
2302
2303 options.padToMultipleOf = std::move(padToMultipleOf);
2304 options.paddingValues = std::move(paddingValues);
2305 options.nofoldFlags = std::move(nofoldFlags);
2306 if (getCopyBackOp() ==
2307 bufferization::MaterializeInDestinationOp::getOperationName()) {
2308 options.copyBackOp = LinalgPaddingOptions::CopyBackOp::
2309 BufferizationMaterializeInDestination;
2310 } else if (getCopyBackOp() == linalg::CopyOp::getOperationName()) {
2311 options.copyBackOp = LinalgPaddingOptions::CopyBackOp::LinalgCopy;
2312 } else if (getCopyBackOp() == kCopyOpNone) {
2313 options.copyBackOp = LinalgPaddingOptions::CopyBackOp::None;
2314 } else {
2315 llvm_unreachable("unsupported copy_back op");
2316 }
2317 // Populate `sizeToPadTo` with the dynamic tensor sizes for each operand.
2318 bool irChanged = false;
2319 if (getUsePrescribedTensorShapes() &&
2320 linalgTarget.hasPureTensorSemantics()) {
2321 OpBuilder::InsertionGuard g(rewriter);
2322 rewriter.setInsertionPoint(linalgTarget);
2323 for (OpOperand &operand : linalgTarget->getOpOperands()) {
2324 for (auto [i, dim] : llvm::enumerate(First: linalgTarget.getShape(opOperand: &operand))) {
2325 if (ShapedType::isStatic(dValue: dim))
2326 continue;
2327 options.setSizeToPadTo(operandIndex: operand.getOperandNumber(), dimIndex: i,
2328 size: tensor::getMixedSize(builder&: rewriter,
2329 loc: operand.get().getLoc(),
2330 value: operand.get(), dim: i));
2331 irChanged = true;
2332 }
2333 }
2334 }
2335
2336 SmallVector<Value> replacements;
2337 SmallVector<tensor::PadOp> newPadOps;
2338 if (failed(Result: rewriteAsPaddedOp(rewriter, opToPad: linalgTarget, options, paddedOp,
2339 replacements, padOps&: newPadOps))) {
2340 if (irChanged) {
2341 auto diag = emitDefiniteFailure() << "failed to pad op";
2342 diag.attachNote(loc: target->getLoc()) << "target op";
2343 return diag;
2344 }
2345 auto diag = emitSilenceableError() << "failed to pad op";
2346 diag.attachNote(loc: target->getLoc()) << "target op";
2347 return diag;
2348 }
2349
2350 // We need to perform our own replacement here because this API is still
2351 // used in patterns that "pad and hoist", for which the replacement values
2352 // need to be different.
2353 // TODO: clean this up and stop "pad and hoist" behavior more globally now
2354 // that we have more composable abstractions.
2355 rewriter.replaceOp(op: linalgTarget, newValues: replacements);
2356 paddedOps.push_back(Elt: paddedOp);
2357 padOps.append(in_start: newPadOps.begin(), in_end: newPadOps.end());
2358 if (options.copyBackOp != LinalgPaddingOptions::CopyBackOp::None) {
2359 for (Value v : replacements) {
2360 Operation *copyBackOp = v.getDefiningOp();
2361 if (!llvm::is_contained(Range&: copyBackOps, Element: copyBackOp))
2362 copyBackOps.push_back(Elt: copyBackOp);
2363 }
2364 }
2365 }
2366
2367 results.set(value: cast<OpResult>(Val: getPadded()), ops&: paddedOps);
2368 results.set(value: cast<OpResult>(Val: getPad()), ops&: padOps);
2369 results.set(value: cast<OpResult>(Val: getCopy()), ops&: copyBackOps);
2370 return DiagnosedSilenceableFailure::success();
2371}
2372
2373LogicalResult transform::PadOp::verify() {
2374 SmallVector<int64_t> nofoldFlags =
2375 extractFromIntegerArrayAttr<int64_t>(attr: getNofoldFlags());
2376 if (any_of(Range&: nofoldFlags, P: [](int64_t packPadding) {
2377 return packPadding != 0 && packPadding != 1;
2378 })) {
2379 return emitOpError()
2380 << "expects nofold_flags to contain booleans (0/1), found "
2381 << getNofoldFlags();
2382 }
2383
2384 SmallVector<int64_t> paddingDimensions =
2385 extractFromIntegerArrayAttr<int64_t>(attr: getPaddingDimensions());
2386 if (any_of(Range&: paddingDimensions,
2387 P: [](int64_t paddingDimension) { return paddingDimension < 0; })) {
2388 return emitOpError() << "expects padding_dimensions to contain positive "
2389 "integers, found "
2390 << getPaddingDimensions();
2391 }
2392 if (!getMixedPadToMultipleOf().empty()) {
2393 if (getMixedPadToMultipleOf().size() != paddingDimensions.size()) {
2394 return emitOpError() << "expects as many multiples as padding_dimensions";
2395 }
2396 }
2397 ArrayAttr transposes = getTransposePaddings();
2398 for (Attribute attr : transposes) {
2399 SmallVector<int64_t> transpose = extractFromIntegerArrayAttr<int64_t>(attr);
2400 auto sequence = llvm::to_vector(Range: llvm::seq<int64_t>(Begin: 0, End: transpose.size()));
2401 if (!std::is_permutation(first1: sequence.begin(), last1: sequence.end(),
2402 first2: transpose.begin(), last2: transpose.end())) {
2403 return emitOpError()
2404 << "expects transpose_paddings to be a permutation, found "
2405 << attr;
2406 }
2407 }
2408 if (getCopyBackOp() !=
2409 bufferization::MaterializeInDestinationOp::getOperationName() &&
2410 getCopyBackOp() != linalg::CopyOp::getOperationName() &&
2411 getCopyBackOp() != kCopyOpNone)
2412 return emitOpError() << "invalid copy_back_op";
2413 return success();
2414}
2415
2416//===---------------------------------------------------------------------===//
2417// PadTilingInterfaceOp
2418//===---------------------------------------------------------------------===//
2419
2420void transform::PadTilingInterfaceOp::build(OpBuilder &b,
2421 OperationState &result,
2422 Value target,
2423 ArrayRef<int64_t> paddingSizes,
2424 bool padToMultipleOf) {
2425 auto resultType = transform::AnyOpType::get(ctx: b.getContext());
2426 return build(/*odsBuilder=*/b,
2427 /*result=*/odsState&: result,
2428 /*types=*/resultTypes: TypeRange{resultType, resultType},
2429 /*target=*/target,
2430 /*padding_values=*/ArrayAttr(), // let inference handle this
2431 /*padding_sizes=*/ValueRange{},
2432 /*paddingSizes=*/
2433 static_padding_sizes: (paddingSizes.empty() ? DenseI64ArrayAttr()
2434 : b.getDenseI64ArrayAttr(values: paddingSizes)),
2435 /*pad_to_multiple_of=*/
2436 padToMultipleOf ? b.getUnitAttr() : nullptr);
2437}
2438
2439void transform::PadTilingInterfaceOp::build(
2440 OpBuilder &b, OperationState &result, Value target,
2441 ArrayRef<OpFoldResult> mixedPaddingSizes, bool padToMultipleOf) {
2442 auto resultType = transform::AnyOpType::get(ctx: b.getContext());
2443 SmallVector<int64_t> staticPaddingSizes;
2444 SmallVector<Value> dynamicPaddingSizes;
2445 dispatchIndexOpFoldResults(ofrs: mixedPaddingSizes, dynamicVec&: dynamicPaddingSizes,
2446 staticVec&: staticPaddingSizes);
2447 return build(/*odsBuilder=*/b,
2448 /*result=*/odsState&: result,
2449 /*types=*/resultTypes: TypeRange{resultType, resultType},
2450 /*target=*/target,
2451 /*padding_values=*/ArrayAttr(), // let inference handle this
2452 /*padding_sizes=*/dynamicPaddingSizes,
2453 /*paddingSizes=*/static_padding_sizes: staticPaddingSizes,
2454 /*usePrescribedTensorShapes=*/pad_to_multiple_of: padToMultipleOf);
2455}
2456
2457void transform::PadTilingInterfaceOp::getEffects(
2458 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
2459 consumesHandle(handles: getTargetMutable(), effects);
2460 onlyReadsHandle(handles: getPaddingSizesMutable(), effects);
2461 producesHandle(handles: getOperation()->getOpResults(), effects);
2462 modifiesPayload(effects);
2463}
2464
2465SmallVector<OpFoldResult>
2466transform::PadTilingInterfaceOp::getMixedPaddingSizes() {
2467 Builder b(getContext());
2468 return getMixedValues(staticValues: getStaticPaddingSizes(), dynamicValues: getPaddingSizes(), b);
2469}
2470
2471DiagnosedSilenceableFailure
2472transform::PadTilingInterfaceOp::apply(transform::TransformRewriter &rewriter,
2473 transform::TransformResults &results,
2474 transform::TransformState &state) {
2475 SmallVector<Operation *> paddedOps, padOps;
2476
2477 for (Operation *target : state.getPayloadOps(value: getTarget())) {
2478 auto targetOp = dyn_cast<TilingInterface>(Val: target);
2479 if (!targetOp) {
2480 auto diag = emitSilenceableError() << "expected TilingInterface target";
2481 diag.attachNote(loc: target->getLoc()) << "target op";
2482 return diag;
2483 }
2484
2485 // Only IndexingMapOpInterface ops for now, until TilingInterface exposes a
2486 // loopsToOperand map / C++ APIs to compute the effect of padding on
2487 // operands.
2488 if (!isa<IndexingMapOpInterface>(Val: targetOp.getOperation())) {
2489 auto diag = emitSilenceableError() << "only IndexingMapOpInterface ops "
2490 "supported atm";
2491 diag.attachNote(loc: target->getLoc()) << "target op";
2492 return diag;
2493 }
2494
2495 // Convert the padding values to attributes.
2496 SmallVector<Attribute> paddingValues;
2497 for (auto const &[untypedAttr, elementOrTensorType] :
2498 llvm::zip(t: getPaddingValues(), u: targetOp->getOperandTypes())) {
2499 auto attr = dyn_cast<TypedAttr>(Val: untypedAttr);
2500 Type elementType = getElementTypeOrSelf(type: elementOrTensorType);
2501
2502 if (matchPattern(attr: untypedAttr, pattern: ub::m_Poison())) {
2503 paddingValues.push_back(Elt: untypedAttr);
2504 continue;
2505 }
2506 if (!attr) {
2507 emitOpError(message: "expects padding values to be typed attributes or poison");
2508 return DiagnosedSilenceableFailure::definiteFailure();
2509 }
2510 // Try to parse string attributes to obtain an attribute of element type.
2511 if (auto stringAttr = dyn_cast<StringAttr>(Val&: attr)) {
2512 auto parsedAttr = dyn_cast_if_present<TypedAttr>(Val: parseAttribute(
2513 attrStr: stringAttr, context: getContext(), type: elementType,
2514 /*numRead=*/nullptr, /*isKnownNullTerminated=*/true));
2515 if (!parsedAttr || parsedAttr.getType() != elementType) {
2516 auto diag = this->emitOpError(message: "expects a padding that parses to ")
2517 << elementType << ", got " << attr;
2518 diag.attachNote(noteLoc: targetOp.getLoc()) << "when applied to this op";
2519 return DiagnosedSilenceableFailure::definiteFailure();
2520 }
2521 paddingValues.push_back(Elt: parsedAttr);
2522 continue;
2523 }
2524 // Otherwise, add the attribute directly.
2525 if (attr.getType() != elementType) {
2526 auto diag = this->emitOpError(message: "expects a padding value of type ")
2527 << elementType << ", got " << attr;
2528 diag.attachNote(noteLoc: targetOp.getLoc()) << "when applied to this op";
2529 return DiagnosedSilenceableFailure::definiteFailure();
2530 }
2531 paddingValues.push_back(Elt: attr);
2532 }
2533
2534 // Set options.
2535 PadTilingInterfaceOptions options;
2536 options.paddingValues = std::move(paddingValues);
2537 options.setPaddingSizes(getMixedPaddingSizes())
2538 .setPadToMultipleOf(getPadToMultipleOf());
2539
2540 OpBuilder::InsertionGuard g(rewriter);
2541 rewriter.setInsertionPointAfter(targetOp);
2542 auto maybePadOps = rewriteAsPaddedOp(
2543 rewriter, toPad: cast<TilingInterface>(Val: targetOp.getOperation()),
2544 options: std::move(options));
2545 if (failed(Result: maybePadOps)) {
2546 auto diag = emitSilenceableError() << "failed to pad op";
2547 diag.attachNote(loc: target->getLoc()) << "target op";
2548 return diag;
2549 }
2550 const auto &[paddedOperands, paddedOp, slicedResults] = maybePadOps.value();
2551
2552 // Set transform results.
2553 paddedOps.push_back(Elt: paddedOp);
2554 padOps.append(in_start: paddedOperands.begin(), in_end: paddedOperands.end());
2555 rewriter.replaceOp(op: targetOp.getOperation(), newValues: slicedResults);
2556 }
2557
2558 results.set(value: cast<OpResult>(Val: getPadded()), ops&: paddedOps);
2559 results.set(value: cast<OpResult>(Val: getPad()), ops&: padOps);
2560 return DiagnosedSilenceableFailure::success();
2561}
2562
2563LogicalResult transform::PadTilingInterfaceOp::verify() { return success(); }
2564
2565//===---------------------------------------------------------------------===//
2566// HoistPadOp
2567//===---------------------------------------------------------------------===//
2568
2569DiagnosedSilenceableFailure transform::HoistPadBuildPackingLoopNestOp::apply(
2570 transform::TransformRewriter &rewriter,
2571 transform::TransformResults &transformResults,
2572 transform::TransformState &state) {
2573 auto targetOps = state.getPayloadOps(value: getTarget());
2574 auto loopOps = state.getPayloadOps(value: getLoop());
2575 if (!llvm::hasSingleElement(C&: targetOps) || !llvm::hasSingleElement(C&: loopOps)) {
2576 return emitDefiniteFailure()
2577 << "requires exactly one target and one loop handle (got "
2578 << llvm::range_size(Range&: targetOps) << " and "
2579 << llvm::range_size(Range&: loopOps) << ")";
2580 }
2581
2582 auto padOp = dyn_cast_or_null<tensor::PadOp>(Val: *targetOps.begin());
2583 auto loopOp = dyn_cast_or_null<scf::ForOp>(Val: *loopOps.begin());
2584 if (!padOp || !loopOp)
2585 return emitDefiniteFailure() << "requires exactly 2 non-null handles";
2586
2587 FailureOr<linalg::detail::PackingResult> result =
2588 linalg::detail::buildPackingLoopNest(rewriter, opToHoist: padOp, outermostEnclosingForOp: loopOp,
2589 transposeVector: getTranspose());
2590 if (failed(Result: result))
2591 return emitDefiniteFailure() << "could not build packing loop nest";
2592
2593 if (result->clonedLoopIvs.empty()) {
2594 transformResults.set(value: cast<OpResult>(Val: getPackingLoop()),
2595 ops: {result->hoistedPadOp.getOperation()});
2596 return DiagnosedSilenceableFailure::success();
2597 }
2598 auto outerPackedLoop =
2599 scf::getForInductionVarOwner(val: result->clonedLoopIvs.front());
2600 transformResults.set(value: cast<OpResult>(Val: getPackingLoop()),
2601 ops: {outerPackedLoop.getOperation()});
2602 return DiagnosedSilenceableFailure::success();
2603}
2604
2605LogicalResult transform::HoistPadBuildPackingLoopNestOp::verify() {
2606 ArrayRef<int64_t> transpose = getTranspose();
2607 auto sequence = llvm::to_vector(Range: llvm::seq<int64_t>(Begin: 0, End: transpose.size()));
2608 if (!std::is_permutation(first1: sequence.begin(), last1: sequence.end(), first2: transpose.begin(),
2609 last2: transpose.end())) {
2610 return emitOpError() << "expects transpose to be a permutation, found "
2611 << getTranspose();
2612 }
2613 return success();
2614}
2615
2616void transform::HoistPadBuildPackingLoopNestOp::getEffects(
2617 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
2618 transform::onlyReadsHandle(handles: getTargetMutable(), effects);
2619 transform::onlyReadsHandle(handles: getLoopMutable(), effects);
2620 transform::producesHandle(handles: getOperation()->getOpResults(), effects);
2621 transform::modifiesPayload(effects);
2622}
2623
2624DiagnosedSilenceableFailure
2625transform::HoistPadOp::applyToOne(transform::TransformRewriter &rewriter,
2626 tensor::PadOp target,
2627 transform::ApplyToEachResultList &results,
2628 transform::TransformState &state) {
2629 tensor::PadOp hoistedPadOp;
2630 SmallVector<TransposeOp> transposeOps;
2631 FailureOr<Value> result =
2632 hoistPaddingOnTensors(rewriter, opToHoist: target, numLoops: getNumLoops(), transposeVector: getTranspose(),
2633 hoistedOp&: hoistedPadOp, transposeOps);
2634 if (succeeded(Result: result)) {
2635 // We need to perform our own replacement here because this API is still
2636 // used in patterns that "pad and hoist", for which the replacement values
2637 // need to be different.
2638 // TODO: clean this up and stop "pad and hoist" behavior more globally now
2639 // that we have more composable abstractions.
2640 rewriter.replaceOp(op: target, newValues: *result);
2641 results.push_back(op: hoistedPadOp);
2642 return DiagnosedSilenceableFailure::success();
2643 }
2644 return emitDefaultSilenceableFailure(target);
2645}
2646
2647LogicalResult transform::HoistPadOp::verify() {
2648 ArrayRef<int64_t> transpose = getTranspose();
2649 auto sequence = llvm::to_vector(Range: llvm::seq<int64_t>(Begin: 0, End: transpose.size()));
2650 if (!std::is_permutation(first1: sequence.begin(), last1: sequence.end(), first2: transpose.begin(),
2651 last2: transpose.end())) {
2652 return emitOpError() << "expects transpose to be a permutation, found "
2653 << getTranspose();
2654 }
2655 return success();
2656}
2657
2658//===----------------------------------------------------------------------===//
2659// PromoteOp
2660//===----------------------------------------------------------------------===//
2661
2662DiagnosedSilenceableFailure
2663transform::PromoteOp::applyToOne(transform::TransformRewriter &rewriter,
2664 LinalgOp target,
2665 transform::ApplyToEachResultList &results,
2666 transform::TransformState &state) {
2667 LinalgPromotionOptions promotionOptions;
2668 if (!getOperandsToPromote().empty())
2669 promotionOptions = promotionOptions.setOperandsToPromote(
2670 extractFromIntegerArrayAttr<int64_t>(attr: getOperandsToPromote()));
2671 if (getUseFullTilesByDefault())
2672 promotionOptions = promotionOptions.setUseFullTileBuffersByDefault(
2673 getUseFullTilesByDefault());
2674 if (getUseOriginalSubviewSize())
2675 promotionOptions =
2676 promotionOptions.setUseOriginalSubviewSize(getUseOriginalSubviewSize());
2677 if (getUseAlloca())
2678 promotionOptions = promotionOptions.setUseAlloca(getUseAlloca());
2679 if (!getUseFullTileBuffers().empty())
2680 promotionOptions = promotionOptions.setUseFullTileBuffers(
2681 llvm::to_vector(Range: getUseFullTileBuffers().getAsValueRange<BoolAttr>()));
2682 if (getAlignment().has_value())
2683 promotionOptions = promotionOptions.setAlignment(*getAlignment());
2684 if (getMemorySpace().has_value())
2685 promotionOptions = promotionOptions.setMemorySpace(*getMemorySpace());
2686
2687 if (getMapping().has_value()) {
2688 // The mapping should only contain an element
2689 auto mapping = *getMapping();
2690 if (mapping.size() > 1)
2691 return emitDefaultDefiniteFailure(target);
2692
2693 auto addressSpace = cast<mlir::gpu::GPUMemorySpaceMappingAttr>(Val: mapping[0]);
2694
2695 if (addressSpace.getAddressSpace() ==
2696 mlir::gpu::GPUDialect::getWorkgroupAddressSpace()) {
2697 promotionOptions =
2698 promotionOptions
2699 .setAllocationDeallocationFns(allocFn: allocateWorkgroupMemory,
2700 deallocFn: deallocateWorkgroupMemory)
2701 .setCopyInOutFns(copyIn: copyToWorkgroupMemory, copyOut: copyToWorkgroupMemory)
2702 .setUseFullTileBuffers({false, false});
2703 } else if (addressSpace.getAddressSpace() ==
2704 mlir::gpu::GPUDialect::getPrivateAddressSpace()) {
2705 promotionOptions =
2706 promotionOptions
2707 .setAllocationDeallocationFns(allocFn: allocateGPUPrivateMemory,
2708 deallocFn: deallocateGPUPrivateMemory)
2709 .setCopyInOutFns(copyIn: copyToGPUPrivateMemory, copyOut: copyToGPUPrivateMemory)
2710 .setUseFullTileBuffers({false, false});
2711 } else {
2712 return emitDefaultDefiniteFailure(target);
2713 }
2714 }
2715
2716 if (failed(Result: promoteSubviewsPrecondition(op: target, options: promotionOptions)))
2717 return emitDefaultDefiniteFailure(target);
2718
2719 rewriter.setInsertionPoint(target);
2720 FailureOr<LinalgOp> res = promoteSubViews(b&: rewriter, op: target, options: promotionOptions);
2721 if (failed(Result: res))
2722 return emitDefaultDefiniteFailure(target);
2723 results.push_back(op: target);
2724 return DiagnosedSilenceableFailure::success();
2725}
2726
2727//===----------------------------------------------------------------------===//
2728// ReplaceOp
2729//===----------------------------------------------------------------------===//
2730
2731DiagnosedSilenceableFailure
2732transform::ReplaceOp::apply(transform::TransformRewriter &rewriter,
2733 TransformResults &transformResults,
2734 TransformState &state) {
2735 auto payload = state.getPayloadOps(value: getTarget());
2736
2737 // Check for invalid targets.
2738 for (Operation *target : payload) {
2739 if (target->getNumOperands() > 0)
2740 return emitDefiniteFailure() << "expected target without operands";
2741 if (!target->hasTrait<OpTrait::IsIsolatedFromAbove>() &&
2742 target->getNumRegions() > 0)
2743 return emitDefiniteFailure()
2744 << "expected target that is isolated from above";
2745 }
2746
2747 // Clone and replace.
2748 Operation *pattern = &getBodyRegion().front().front();
2749 SmallVector<Operation *> replacements;
2750 for (Operation *target : payload) {
2751 if (getOperation()->isAncestor(other: target))
2752 continue;
2753 rewriter.setInsertionPoint(target);
2754 Operation *replacement = rewriter.clone(op&: *pattern);
2755 rewriter.replaceOp(op: target, newValues: replacement->getResults());
2756 replacements.push_back(Elt: replacement);
2757 }
2758 transformResults.set(value: cast<OpResult>(Val: getReplacement()), ops&: replacements);
2759 return DiagnosedSilenceableFailure::success();
2760}
2761
2762void transform::ReplaceOp::getEffects(
2763 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
2764 consumesHandle(handles: getTargetMutable(), effects);
2765 producesHandle(handles: getOperation()->getOpResults(), effects);
2766 modifiesPayload(effects);
2767}
2768
2769LogicalResult transform::ReplaceOp::verify() {
2770 if (!getBodyRegion().hasOneBlock())
2771 return emitOpError() << "expected one block";
2772 if (std::distance(first: getBodyRegion().front().begin(),
2773 last: getBodyRegion().front().end()) != 1)
2774 return emitOpError() << "expected one operation in block";
2775 Operation *replacement = &getBodyRegion().front().front();
2776 if (replacement->getNumOperands() > 0)
2777 return replacement->emitOpError()
2778 << "expected replacement without operands";
2779 if (!replacement->hasTrait<OpTrait::IsIsolatedFromAbove>() &&
2780 replacement->getNumRegions() > 0)
2781 return replacement->emitOpError()
2782 << "expect op that is isolated from above";
2783 return success();
2784}
2785
2786//===----------------------------------------------------------------------===//
2787// ScalarizeOp
2788//===----------------------------------------------------------------------===//
2789
2790DiagnosedSilenceableFailure
2791transform::ScalarizeOp::applyToOne(transform::TransformRewriter &rewriter,
2792 LinalgOp target,
2793 transform::ApplyToEachResultList &results,
2794 transform::TransformState &state) {
2795 scf::SCFTilingOptions tilingOptions;
2796 tilingOptions.setTileSizeComputationFunction([&](OpBuilder &b, Operation *) {
2797 SmallVector<OpFoldResult> tileSizes;
2798 Location loc = target.getLoc();
2799 SmallVector<OpFoldResult> allShapeSizes =
2800 target.createFlatListOfOperandDims(b, loc);
2801 AffineMap map = target.getShapesToLoopsMap();
2802 if (!map)
2803 return tileSizes;
2804 SmallVector<OpFoldResult> shapeSizes =
2805 affine::makeComposedFoldedMultiResultAffineApply(b&: rewriter, loc, map,
2806 operands: allShapeSizes);
2807 // If the shape size is dynamic, tile by 1.
2808 // Otherwise, do not tile (i.e. tile size 0).
2809 for (OpFoldResult shapeSize : shapeSizes) {
2810 tileSizes.push_back(Elt: getConstantIntValue(ofr: shapeSize) ? b.getIndexAttr(value: 0)
2811 : b.getIndexAttr(value: 1));
2812 }
2813 return tileSizes;
2814 });
2815 rewriter.setInsertionPoint(target);
2816 FailureOr<scf::SCFTilingResult> maybeTilingResult = tileUsingSCF(
2817 rewriter, op: cast<TilingInterface>(Val: target.getOperation()), options: tilingOptions);
2818 if (failed(Result: maybeTilingResult))
2819 return emitDefaultDefiniteFailure(target);
2820
2821 if (target->getNumResults())
2822 rewriter.replaceOp(op: target, newValues: maybeTilingResult->replacements);
2823 else
2824 rewriter.eraseOp(op: target);
2825
2826 results.reserve(size: maybeTilingResult->tiledOps.size());
2827 for (Operation *tiled : maybeTilingResult->tiledOps)
2828 results.push_back(op: tiled);
2829 return DiagnosedSilenceableFailure::success();
2830}
2831
2832//===----------------------------------------------------------------------===//
2833// ConvertToLoopsOp
2834//===----------------------------------------------------------------------===//
2835
2836DiagnosedSilenceableFailure
2837transform::ConvertToLoopsOp::apply(transform::TransformRewriter &rewriter,
2838 transform::TransformResults &results,
2839 transform::TransformState &state) {
2840 SmallVector<Operation *> loops;
2841 for (Operation *target : state.getPayloadOps(value: getTarget())) {
2842 auto tilingOp = dyn_cast<TilingInterface>(Val&: *target);
2843 if (!tilingOp) {
2844 DiagnosedSilenceableFailure diag =
2845 emitSilenceableError()
2846 << "expected the payload to implement TilingInterface";
2847 diag.attachNote(loc: target->getLoc()) << "payload op";
2848 return diag;
2849 }
2850 rewriter.setInsertionPoint(target);
2851 FailureOr<SmallVector<scf::ForOp>> generatedLoops =
2852 scf::lowerToLoopsUsingSCFForOp(rewriter, op: tilingOp);
2853 if (failed(Result: generatedLoops))
2854 return emitDefaultDefiniteFailure(target);
2855 for (scf::ForOp &loop : *generatedLoops) {
2856 loops.push_back(Elt: loop.getOperation());
2857 }
2858 rewriter.eraseOp(op: target);
2859 }
2860 results.set(value: cast<OpResult>(Val: getResult()), ops&: loops);
2861 return DiagnosedSilenceableFailure::success();
2862}
2863
2864//===----------------------------------------------------------------------===//
2865// RewriteInDestinationPassingStyleOp
2866//===----------------------------------------------------------------------===//
2867
2868DiagnosedSilenceableFailure
2869transform::RewriteInDestinationPassingStyleOp::applyToOne(
2870 transform::TransformRewriter &rewriter, Operation *target,
2871 transform::ApplyToEachResultList &results,
2872 transform::TransformState &state) {
2873 rewriter.setInsertionPoint(target);
2874 FailureOr<Operation *> maybeResult =
2875 TypeSwitch<Operation *, FailureOr<Operation *>>(target)
2876 .Case<DestinationStyleOpInterface>(caseFn: [](auto op) { return op; })
2877 .Case<tensor::FromElementsOp, tensor::GenerateOp, tensor::PadOp>(
2878 caseFn: [&rewriter](auto op) {
2879 return rewriteInDestinationPassingStyle(rewriter, op);
2880 });
2881 if (failed(Result: maybeResult))
2882 return emitDefaultSilenceableFailure(target);
2883 results.push_back(op: *maybeResult);
2884 return DiagnosedSilenceableFailure::success();
2885}
2886
2887//===----------------------------------------------------------------------===//
2888// SplitOp
2889//===----------------------------------------------------------------------===//
2890
2891DiagnosedSilenceableFailure
2892SplitOp::apply(transform::TransformRewriter &rewriter,
2893 TransformResults &results, TransformState &state) {
2894 // Collect the dynamic split points if provided.
2895 SmallVector<Operation *> payload =
2896 llvm::to_vector(Range: state.getPayloadOps(value: getTarget()));
2897
2898 bool isMultiwaySplit = getMultiway();
2899
2900 if (isMultiwaySplit && !llvm::hasSingleElement(C&: payload)) {
2901 return mlir::emitSilenceableFailure(loc: getLoc())
2902 << "requires exactly one target when "
2903 "multiway split is enabled (got "
2904 << llvm::range_size(Range&: payload) << ")";
2905 }
2906
2907 SmallVector<OpFoldResult> chunkSizes;
2908
2909 if (!isMultiwaySplit)
2910 chunkSizes.reserve(N: payload.size());
2911
2912 if (getDynamicChunkSizes()) {
2913 auto diag = DiagnosedSilenceableFailure::success();
2914 if (isa<TransformHandleTypeInterface>(Val: getDynamicChunkSizes().getType())) {
2915 chunkSizes = llvm::map_to_vector(
2916 C: state.getPayloadOps(value: getDynamicChunkSizes()), F: [&](Operation *op) {
2917 if (op->getNumResults() != 1 ||
2918 !op->getResult(idx: 0).getType().isIndex()) {
2919 diag = emitSilenceableError()
2920 << "expected dynamic split point handle to point to a "
2921 "single-result index-typed op";
2922 diag.attachNote(loc: op->getLoc()) << "dynamic split point";
2923 }
2924 return OpFoldResult(op->getResult(idx: 0));
2925 });
2926 } else {
2927 chunkSizes = llvm::map_to_vector(
2928 C: state.getParams(value: getDynamicChunkSizes()),
2929 F: [](Attribute attr) { return OpFoldResult(attr); });
2930 }
2931 if (diag.isSilenceableFailure())
2932 return diag;
2933
2934 // For multiway split, a single payload is expected to have multiple
2935 // split points.
2936 if (!isMultiwaySplit && chunkSizes.size() != payload.size()) {
2937 return emitDefiniteFailure()
2938 << "expected the dynamic split point handle to point to as "
2939 "many operations ("
2940 << chunkSizes.size() << ") as the target handle ("
2941 << payload.size() << ")";
2942 }
2943 } else {
2944 chunkSizes.resize(N: payload.size(),
2945 NV: rewriter.getIndexAttr(value: getStaticChunkSizes()));
2946 }
2947
2948 auto checkStructuredOpAndDimensions =
2949 [&](LinalgOp linalgOp, Location loc) -> DiagnosedSilenceableFailure {
2950 if (!linalgOp) {
2951 auto diag = emitSilenceableError() << "only applies to structured ops";
2952 diag.attachNote(loc) << "target op";
2953 return diag;
2954 }
2955
2956 if (getDimension() >= linalgOp.getNumLoops()) {
2957 auto diag = emitSilenceableError() << "dimension " << getDimension()
2958 << " does not exist in target op";
2959 diag.attachNote(loc) << "target op";
2960 return diag;
2961 }
2962 return DiagnosedSilenceableFailure::success();
2963 };
2964
2965 auto checkFailureInSplitting =
2966 [&](bool hasFailed, Location loc) -> DiagnosedSilenceableFailure {
2967 if (hasFailed) {
2968 auto diag = emitDefiniteFailure() << "internal failure in splitting";
2969 diag.attachNote(loc) << "target op";
2970 return diag;
2971 }
2972 return DiagnosedSilenceableFailure::success();
2973 };
2974
2975 SmallVector<Operation *> opList;
2976 if (isMultiwaySplit) {
2977
2978 // Split a single target operation at multiple points.
2979 TilingInterface head, tail;
2980 Operation *target = payload.front();
2981
2982 LinalgOp linalgOp = dyn_cast<LinalgOp>(Val: target);
2983
2984 // Check that the target is a valid LinalgOp with correct dimensions.
2985 DiagnosedSilenceableFailure diag =
2986 checkStructuredOpAndDimensions(linalgOp, target->getLoc());
2987 if (diag.isSilenceableFailure())
2988 return diag;
2989
2990 for (auto &&[idx, chunkSize] : llvm::enumerate(First&: chunkSizes)) {
2991
2992 if (idx > 0)
2993 target = tail.getOperation();
2994
2995 if (!target)
2996 break;
2997
2998 linalgOp = cast<LinalgOp>(Val: target);
2999 Location loc = target->getLoc();
3000
3001 rewriter.setInsertionPoint(linalgOp);
3002 std::tie(args&: head, args&: tail) = linalg::splitOp(
3003 rewriter, op: cast<TilingInterface>(Val: linalgOp.getOperation()),
3004 dimension: getDimension(), splitPoint: chunkSize);
3005
3006 // Propagate errors.
3007 DiagnosedSilenceableFailure diag =
3008 checkFailureInSplitting(!head && !tail, loc);
3009 if (diag.isDefiniteFailure())
3010 return diag;
3011
3012 opList.push_back(Elt: head.getOperation());
3013 }
3014
3015 // Append any leftover parts to the end of the result list.
3016 if (tail)
3017 opList.push_back(Elt: tail.getOperation());
3018
3019 } else {
3020 // Split each target operation.
3021 SmallVector<Operation *> first, second;
3022 Operation *noSecondPart = nullptr;
3023 for (const auto &pair : llvm::zip(t&: payload, u&: chunkSizes)) {
3024 Operation *target = std::get<0>(t: pair);
3025 Location loc = target->getLoc();
3026 LinalgOp linalgOp = dyn_cast<LinalgOp>(Val: target);
3027 DiagnosedSilenceableFailure diag =
3028 checkStructuredOpAndDimensions(linalgOp, target->getLoc());
3029
3030 if (diag.isSilenceableFailure())
3031 return diag;
3032
3033 rewriter.setInsertionPoint(linalgOp);
3034 std::tie(args&: first.emplace_back(), args&: second.emplace_back()) = linalg::splitOp(
3035 rewriter, op: cast<TilingInterface>(Val: linalgOp.getOperation()),
3036 dimension: getDimension(), splitPoint: std::get<1>(t: pair));
3037
3038 // Propagate errors.
3039 DiagnosedSilenceableFailure diagSplit =
3040 checkFailureInSplitting(!first.back() && !second.back(), loc);
3041 if (diagSplit.isDefiniteFailure())
3042 return diag;
3043
3044 // Do not add null second parts.
3045 if (!second.back()) {
3046 noSecondPart = target;
3047 second.pop_back();
3048 }
3049 }
3050
3051 if (second.size() != first.size() && !second.empty()) {
3052 auto diag = emitSilenceableError()
3053 << "splitting does not produce the second part for a subset "
3054 "of targets";
3055 diag.attachNote()
3056 << "expected splitting to produce the second part of all "
3057 "or none of the targets";
3058 diag.attachNote(loc: noSecondPart->getLoc())
3059 << "first target with no second part";
3060 return diag;
3061 }
3062
3063 opList.append(RHS: first);
3064 if (!second.empty())
3065 opList.append(RHS: second);
3066 }
3067 results.set(value: cast<OpResult>(Val: getSplitList()), ops&: opList);
3068 return DiagnosedSilenceableFailure::success();
3069}
3070
3071void SplitOp::getEffects(
3072 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
3073 consumesHandle(handles: getTargetMutable(), effects);
3074 if (getDynamicChunkSizes())
3075 onlyReadsHandle(handles: getDynamicChunkSizesMutable(), effects);
3076 producesHandle(handles: getOperation()->getOpResults(), effects);
3077 modifiesPayload(effects);
3078}
3079
3080ParseResult SplitOp::parse(OpAsmParser &parser, OperationState &result) {
3081 OpAsmParser::UnresolvedOperand target, dynamicChunkSizes;
3082 IntegerAttr staticChunkSizes;
3083 if (parser.parseOperand(result&: target) || parser.parseKeyword(keyword: "after"))
3084 return failure();
3085
3086 OptionalParseResult dynamicPointParseResult =
3087 parser.parseOptionalOperand(result&: dynamicChunkSizes);
3088 if (!dynamicPointParseResult.has_value()) {
3089 int64_t staticChunkSizesValue;
3090 if (failed(Result: parser.parseInteger(result&: staticChunkSizesValue)))
3091 return failure();
3092
3093 staticChunkSizes =
3094 parser.getBuilder().getI64IntegerAttr(value: staticChunkSizesValue);
3095 }
3096
3097 Type targetType;
3098 if (parser.parseOptionalAttrDict(result&: result.attributes) ||
3099 parser.parseColonType(result&: targetType) ||
3100 parser.resolveOperand(operand: target, type: targetType, result&: result.operands)) {
3101 return failure();
3102 }
3103 if (dynamicPointParseResult.has_value()) {
3104 Type chunkSizesType;
3105 if (failed(Result: *dynamicPointParseResult) || parser.parseComma() ||
3106 parser.parseType(result&: chunkSizesType) ||
3107 parser.resolveOperand(operand: dynamicChunkSizes, type: chunkSizesType,
3108 result&: result.operands)) {
3109 return failure();
3110 }
3111
3112 staticChunkSizes =
3113 parser.getBuilder().getI64IntegerAttr(value: ShapedType::kDynamic);
3114 }
3115
3116 result.addAttribute(
3117 name: SplitOp::getStaticChunkSizesAttrName(name: result.name).getValue(),
3118 attr: staticChunkSizes);
3119 result.addTypes(newTypes: targetType);
3120 return success();
3121}
3122
3123void SplitOp::print(OpAsmPrinter &printer) {
3124 printer << " " << getTarget() << " after ";
3125 int64_t staticChunkSize = static_cast<int64_t>(getStaticChunkSizes());
3126 if (staticChunkSize != ShapedType::kDynamic)
3127 printer << staticChunkSize;
3128 else
3129 printer << getDynamicChunkSizes();
3130 printer << " ";
3131 NamedAttrList attrs(getOperation()->getDiscardableAttrDictionary());
3132 attrs.append(name: getDimensionAttrName(), attr: getDimensionAttr());
3133 if (UnitAttr multiway = getMultiwayAttr())
3134 attrs.append(name: getMultiwayAttrName(), attr: multiway);
3135 printer.printOptionalAttrDict(attrs, elidedAttrs: {getStaticChunkSizesAttrName()});
3136 printer << " : " << getTarget().getType();
3137 if (staticChunkSize == ShapedType::kDynamic)
3138 printer << ", " << getDynamicChunkSizes().getType();
3139}
3140
3141LogicalResult SplitOp::verify() {
3142 if ((static_cast<int64_t>(getStaticChunkSizes()) != ShapedType::kDynamic) ^
3143 (getDynamicChunkSizes() == nullptr)) {
3144 return emitOpError() << "expects either a dynamic or a static split "
3145 "point to be provided";
3146 }
3147 return success();
3148}
3149
3150//===----------------------------------------------------------------------===//
3151// SplitReductionOp
3152//===----------------------------------------------------------------------===//
3153
3154void transform::SplitReductionOp::build(
3155 OpBuilder &builder, OperationState &result, Value target,
3156 int64_t splitFactor, int64_t insertSplitDimension, bool innerParallel,
3157 bool useScalingAlgorithm, bool useAlloc) {
3158 MLIRContext *ctx = builder.getContext();
3159 result.addOperands(newOperands: target);
3160 result.addAttribute(name: SplitReductionOp::getSplitFactorAttrName(name: result.name),
3161 attr: builder.getI64IntegerAttr(value: splitFactor));
3162 result.addAttribute(
3163 name: SplitReductionOp::getInsertSplitDimensionAttrName(name: result.name),
3164 attr: builder.getI64IntegerAttr(value: insertSplitDimension));
3165 if (innerParallel) {
3166 result.addAttribute(name: SplitReductionOp::getInnerParallelAttrName(name: result.name),
3167 attr: builder.getUnitAttr());
3168 }
3169 if (useScalingAlgorithm) {
3170 result.addAttribute(
3171 name: SplitReductionOp::getUseScalingAlgorithmAttrName(name: result.name),
3172 attr: builder.getUnitAttr());
3173 }
3174 if (useAlloc) {
3175 result.addAttribute(name: SplitReductionOp::getUseAllocAttrName(name: result.name),
3176 attr: builder.getUnitAttr());
3177 }
3178 auto resultType = transform::AnyOpType::get(ctx);
3179 result.addTypes(newTypes: {resultType, resultType, resultType, resultType});
3180}
3181
3182DiagnosedSilenceableFailure transform::SplitReductionOp::applyToOne(
3183 transform::TransformRewriter &rewriter, LinalgOp target,
3184 transform::ApplyToEachResultList &results,
3185 transform::TransformState &state) {
3186 ControlSplitReductionFn splitFn = [&](LinalgOp) {
3187 return linalg::SplitReductionOptions{.ratio: int64_t(getSplitFactor()),
3188 .index: unsigned(getInsertSplitDimension()),
3189 .innerParallel: bool(getInnerParallel())};
3190 };
3191 rewriter.setInsertionPoint(target);
3192 FailureOr<SplitReductionResult> splitResult =
3193 (getUseScalingAlgorithm())
3194 ? splitReductionByScaling(b&: rewriter, op: target, controlSplitReductionFn: splitFn, useAlloc: getUseAlloc())
3195 : splitReduction(b&: rewriter, op: target, controlSplitReductionFn: splitFn, useAlloc: getUseAlloc());
3196 if (failed(Result: splitResult))
3197 return emitDefaultDefiniteFailure(target);
3198
3199 results.push_back(op: splitResult->initOrAlloc);
3200 results.push_back(op: splitResult->fillOp);
3201 results.push_back(op: splitResult->splitLinalgOp);
3202 results.push_back(op: splitResult->resultCombiningLinalgOp);
3203 return DiagnosedSilenceableFailure::success();
3204}
3205
3206//===----------------------------------------------------------------------===//
3207// TileReductionUsingForOp
3208//===----------------------------------------------------------------------===//
3209
3210void transform::TileReductionUsingForOp::build(
3211 OpBuilder &builder, OperationState &result, Value target,
3212 ArrayRef<int64_t> staticTileSizes) {
3213 // Call the default builder.
3214 // This is future-proof re mixed static-dynamic and setting up the proper
3215 // operands segment sizes attributes for multiple variadic operands.
3216 // In the absence of this, horrible bugs ensue.
3217 // TODO: support mixed static-dynamic (see TileUsingForallOp).
3218 MLIRContext *ctx = builder.getContext();
3219 auto opTy = transform::AnyOpType::get(ctx);
3220 auto staticTileSizesAttr = builder.getI64ArrayAttr(values: staticTileSizes);
3221 build(odsBuilder&: builder, odsState&: result,
3222 /*resultTypes=*/TypeRange{opTy, opTy, opTy, opTy},
3223 /*target=*/target,
3224 /*reduction_dims=*/nullptr,
3225 /*tile_sizes=*/staticTileSizesAttr);
3226}
3227
3228DiagnosedSilenceableFailure transform::TileReductionUsingForOp::applyToOne(
3229 transform::TransformRewriter &rewriter, Operation *target,
3230 transform::ApplyToEachResultList &results,
3231 transform::TransformState &state) {
3232 rewriter.setInsertionPoint(target);
3233
3234 auto partialReductionOp = dyn_cast<PartialReductionOpInterface>(Val: target);
3235 if (!partialReductionOp) {
3236 return emitSilenceableFailure(
3237 loc: target->getLoc(),
3238 message: "Operation should implement PartialReductionOpInterface");
3239 }
3240
3241 SmallVector<unsigned> reductionDims =
3242 extractFromIntegerArrayAttr<unsigned>(attr: getReductionDims());
3243 if (reductionDims.empty()) {
3244 for (auto [idx, iteratorType] :
3245 llvm::enumerate(First: partialReductionOp.getLoopIteratorTypes())) {
3246 if (iteratorType == utils::IteratorType::reduction)
3247 reductionDims.push_back(Elt: idx);
3248 }
3249 }
3250
3251 scf::SCFTilingOptions options;
3252 options.setLoopType(scf::SCFTilingOptions::LoopType::ForOp);
3253 options.setReductionTilingStrategy(
3254 ReductionTilingStrategy::PartialReductionOuterReduction);
3255 options.setTileSizes(getAsOpFoldResult(arrayAttr: getTileSizesAttr()));
3256 options.setReductionDims(reductionDims);
3257 FailureOr<scf::SCFTilingResult> result =
3258 scf::tileUsingSCF(rewriter, op: partialReductionOp, options);
3259
3260 if (failed(Result: result)) {
3261 return emitSilenceableFailure(loc: getLoc(),
3262 message: "failed to tile using partial reduction");
3263 }
3264 rewriter.replaceOp(op: target, newValues: result->replacements);
3265 for (Value initValue : result->initialValues)
3266 results.push_back(op: initValue.getDefiningOp());
3267 for (auto *parallelTiledOp : result->tiledOps)
3268 results.push_back(op: parallelTiledOp);
3269 for (auto *mergeOp : result->mergeOps)
3270 results.push_back(op: mergeOp);
3271 results.push_back(op: result->loops.front());
3272 return DiagnosedSilenceableFailure::success();
3273}
3274
3275//===----------------------------------------------------------------------===//
3276// TileReductionUsingForallOp
3277//===----------------------------------------------------------------------===//
3278
3279void transform::TileReductionUsingForallOp::build(
3280 OpBuilder &builder, OperationState &result, Value target,
3281 ArrayRef<int64_t> staticNumThreads, ArrayRef<int64_t> staticTileSizes,
3282 ArrayAttr mapping) {
3283 // Call the default builder.
3284 // This is future-proof re mixed static-dynamic and setting up the proper
3285 // operands segment sizes attributes for multiple variadic operands.
3286 // In the absence of this, horrible bugs ensue.
3287 // TODO: support mixed static-dynamic (see TileUsingForallOp).
3288 MLIRContext *ctx = builder.getContext();
3289 auto opTy = transform::AnyOpType::get(ctx);
3290 auto staticNumThreadsAttr = builder.getDenseI64ArrayAttr(values: staticNumThreads);
3291 auto staticTileSizesAttr = builder.getDenseI64ArrayAttr(values: staticTileSizes);
3292 build(odsBuilder&: builder, odsState&: result,
3293 /*resultTypes=*/TypeRange{opTy, opTy, opTy, opTy},
3294 /*target=*/target,
3295 /*reduction_dims=*/{},
3296 /*num_threads=*/staticNumThreadsAttr,
3297 /*tile_sizes=*/staticTileSizesAttr,
3298 /*mapping=*/mapping);
3299}
3300
3301DiagnosedSilenceableFailure transform::TileReductionUsingForallOp::applyToOne(
3302 transform::TransformRewriter &rewriter, Operation *target,
3303 transform::ApplyToEachResultList &results,
3304 transform::TransformState &state) {
3305 rewriter.setInsertionPoint(target);
3306
3307 auto partialReductionOp = dyn_cast<PartialReductionOpInterface>(Val: target);
3308 if (!partialReductionOp) {
3309 return emitSilenceableFailure(
3310 loc: target->getLoc(),
3311 message: "Operation should implement PartialReductionOpInterface");
3312 }
3313 SmallVector<OpFoldResult> numThreads =
3314 getAsOpFoldResult(arrayAttr: rewriter.getI64ArrayAttr(values: getNumThreads()));
3315 SmallVector<OpFoldResult> tileSizes =
3316 getAsOpFoldResult(arrayAttr: rewriter.getI64ArrayAttr(values: getTileSizes()));
3317
3318 scf::SCFTilingOptions options;
3319 options.setLoopType(scf::SCFTilingOptions::LoopType::ForallOp);
3320 options.setReductionTilingStrategy(
3321 ReductionTilingStrategy::PartialReductionOuterParallel);
3322 if (!getNumThreads().empty()) {
3323 options.setNumThreads(numThreads);
3324 } else {
3325 options.setTileSizes(tileSizes);
3326 }
3327 if (auto mapping = getMapping()) {
3328 options.setMapping(mapping.value().getValue());
3329 }
3330 SmallVector<unsigned> reductionDims =
3331 extractFromIntegerArrayAttr<unsigned>(attr: getReductionDims());
3332 if (reductionDims.empty()) {
3333 for (auto [idx, iteratorType] :
3334 llvm::enumerate(First: partialReductionOp.getLoopIteratorTypes())) {
3335 if (iteratorType == utils::IteratorType::reduction)
3336 reductionDims.push_back(Elt: idx);
3337 }
3338 }
3339 options.setReductionDims(reductionDims);
3340 FailureOr<scf::SCFTilingResult> result =
3341 scf::tileUsingSCF(rewriter, op: partialReductionOp, options);
3342
3343 if (failed(Result: result)) {
3344 auto diag = emitSilenceableError() << "could not tile reduction";
3345 return diag;
3346 }
3347 rewriter.replaceOp(op: target, newValues: result->replacements);
3348
3349 for (Value initValue : result->initialValues)
3350 results.push_back(op: initValue.getDefiningOp());
3351 for (auto *parallelTiledOp : result->tiledOps)
3352 results.push_back(op: parallelTiledOp);
3353 for (auto *mergeOp : result->mergeOps)
3354 results.push_back(op: mergeOp);
3355 results.push_back(op: result->loops.front());
3356 return DiagnosedSilenceableFailure::success();
3357}
3358
3359//===----------------------------------------------------------------------===//
3360// ContinuousTileSizesOp
3361//===----------------------------------------------------------------------===//
3362
3363DiagnosedSilenceableFailure
3364transform::ContinuousTileSizesOp::apply(transform::TransformRewriter &rewriter,
3365 TransformResults &transformResults,
3366 TransformState &state) {
3367
3368 SmallVector<Operation *> targetOps =
3369 llvm::to_vector(Range: state.getPayloadOps(value: getTarget()));
3370
3371 if (!llvm::hasSingleElement(C&: targetOps)) {
3372 return mlir::emitSilenceableFailure(loc: getLoc())
3373 << "requires exactly one target (got " << llvm::range_size(Range&: targetOps)
3374 << ")";
3375 }
3376
3377 Operation *target = *targetOps.begin();
3378 auto linalgOp = dyn_cast<LinalgOp>(Val: target);
3379 auto tileableOp = dyn_cast<TilingInterface>(Val: target);
3380
3381 if (!linalgOp)
3382 return emitDefiniteFailure() << "expected Linalg Op";
3383
3384 OpBuilder builder(linalgOp.getContext());
3385
3386 if (isa<TransformParamTypeInterface>(Val: getChunkSizes().getType())) {
3387 if (linalgOp.hasDynamicShape()) {
3388 auto diag = emitSilenceableError()
3389 << "cannot compute parametric tile sizes for dynamically "
3390 "shaped payload op";
3391 diag.attachNote(loc: linalgOp->getLoc()) << "payload op";
3392 return diag;
3393 }
3394
3395 FailureOr<StaticContinuousTileSizeSpecification> spec =
3396 computeStaticContinuousTileSizes(op: linalgOp, dimension: getDimension(),
3397 targetSize: getTargetSize());
3398 if (failed(Result: spec)) {
3399 return emitSilenceableError()
3400 << "failed to compute multi-size tiling sizes";
3401 }
3402
3403 SmallVector<int64_t> chunkSizes;
3404
3405 for (auto &&[tileSize, tripCount] :
3406 llvm::zip_equal(t&: spec->tileSizes, u&: spec->tripCounts))
3407 chunkSizes.push_back(Elt: tileSize * tripCount);
3408
3409 auto getI64AttrsFromI64 = [&](ArrayRef<int64_t> values) {
3410 return llvm::map_to_vector(C&: values, F: [&](int64_t value) -> Attribute {
3411 return builder.getI64IntegerAttr(value);
3412 });
3413 };
3414 transformResults.setParams(value: cast<OpResult>(Val: getTileSizes()),
3415 params: getI64AttrsFromI64(spec->tileSizes));
3416 transformResults.setParams(value: cast<OpResult>(Val: getChunkSizes()),
3417 params: getI64AttrsFromI64(chunkSizes));
3418
3419 return DiagnosedSilenceableFailure::success();
3420 }
3421
3422 builder.setInsertionPoint(linalgOp);
3423
3424 OpFoldResult targetSize = builder.getIndexAttr(value: getTargetSize());
3425 unsigned dimension = getDimension();
3426
3427 FailureOr<ContinuousTileSizeSpecification> spec = computeContinuousTileSizes(
3428 builder, op: tileableOp, dimension, targetSize, emitAssertions: true);
3429 if (failed(Result: spec)) {
3430 return emitSilenceableError() << "could not generate tile size computation";
3431 }
3432
3433 AffineExpr s0 = builder.getAffineSymbolExpr(position: 0);
3434 AffineExpr s1 = builder.getAffineSymbolExpr(position: 1);
3435 auto apply = [&](AffineExpr expr, ArrayRef<OpFoldResult> ofrs) -> Value {
3436 return affine::makeComposedAffineApply(b&: builder, loc: linalgOp->getLoc(), e: expr,
3437 operands: ofrs);
3438 };
3439
3440 SmallVector<Value> chunkSizes;
3441 Value splitPoint;
3442 for (auto &&[tileSize, tripCount] :
3443 llvm::zip_equal(t&: spec->tileSizes, u&: spec->tripCounts)) {
3444 splitPoint = apply(s0 * s1, {tileSize, tripCount});
3445 chunkSizes.push_back(Elt: splitPoint);
3446 }
3447
3448 auto getDefiningOps = [&](ArrayRef<Value> values) {
3449 return llvm::map_to_vector(C&: values, F: [&](Value value) -> Operation * {
3450 return value.getDefiningOp();
3451 });
3452 };
3453
3454 transformResults.set(value: cast<OpResult>(Val: getTileSizes()),
3455 ops: getDefiningOps(spec->tileSizes));
3456 transformResults.set(value: cast<OpResult>(Val: getChunkSizes()),
3457 ops: getDefiningOps(chunkSizes));
3458
3459 return DiagnosedSilenceableFailure::success();
3460}
3461
3462LogicalResult transform::ContinuousTileSizesOp::verify() {
3463
3464 if (getTileSizes().getType() != getChunkSizes().getType()) {
3465 return emitOpError() << "expects all results type to be the same";
3466 }
3467
3468 return success();
3469}
3470
3471void transform::ContinuousTileSizesOp::getEffects(
3472 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
3473 if (isa<TransformParamTypeInterface>(Val: getTileSizes().getType()))
3474 onlyReadsPayload(effects);
3475 else
3476 modifiesPayload(effects);
3477 onlyReadsHandle(handles: getTargetMutable(), effects);
3478 producesHandle(handles: getOperation()->getOpResults(), effects);
3479}
3480
3481static void printContinuousTileSizeTypes(OpAsmPrinter &printer, Operation *op,
3482 Type targetType, Type tileSizes,
3483 Type) {
3484 printer.printFunctionalType(inputs: TypeRange{targetType}, results: TypeRange{tileSizes});
3485}
3486
3487static ParseResult parseContinuousTileSizeTypes(OpAsmParser &parser,
3488 Type &targetType,
3489 Type &tileSizesType,
3490 Type &chunkSizesType) {
3491 FunctionType funcType;
3492 llvm::SMLoc typeLoc = parser.getCurrentLocation();
3493 if (failed(Result: parser.parseType<FunctionType>(result&: funcType)))
3494 return failure();
3495
3496 if (funcType.getNumInputs() != 1 || funcType.getNumResults() != 1) {
3497 parser.emitError(loc: typeLoc) << "expects a trailing functional type with one "
3498 "argument and one result";
3499 }
3500 targetType = funcType.getInput(i: 0);
3501 tileSizesType = chunkSizesType = funcType.getResult(i: 0);
3502
3503 return success();
3504}
3505
3506//===----------------------------------------------------------------------===//
3507// TileUsingForOp
3508//===----------------------------------------------------------------------===//
3509
3510void transform::TileUsingForOp::build(
3511 OpBuilder &builder, OperationState &result, TypeRange loopTypes,
3512 Value target, ArrayRef<int64_t> staticTileSizes,
3513 ArrayRef<int64_t> interchange,
3514 std::optional<ArrayRef<bool>> scalableSizes) {
3515 return build(odsBuilder&: builder, odsState&: result, loopTypes,
3516 /*target=*/target,
3517 /*mixedTileSizes=*/
3518 getAsOpFoldResult(arrayAttr: builder.getI64ArrayAttr(values: staticTileSizes)),
3519 interchange, scalableSizes);
3520}
3521
3522void transform::TileUsingForOp::build(
3523 OpBuilder &builder, OperationState &result, Value target,
3524 ArrayRef<int64_t> staticTileSizes, ArrayRef<int64_t> interchange,
3525 std::optional<ArrayRef<bool>> scalableSizes) {
3526 build(odsBuilder&: builder, odsState&: result, target,
3527 mixedTileSizes: getAsOpFoldResult(arrayAttr: builder.getI64ArrayAttr(values: staticTileSizes)),
3528 interchange, scalableSizes);
3529}
3530
3531void transform::TileUsingForOp::build(
3532 OpBuilder &builder, OperationState &result, Value target,
3533 ArrayRef<OpFoldResult> mixedTileSizes, ArrayRef<int64_t> interchange,
3534 std::optional<ArrayRef<bool>> scalableSizes) {
3535 // Loop types are automaticaly splat by the callee, setting up one is
3536 // enough.
3537 SmallVector<Type> loopTypes(1, builder.getType<transform::AnyOpType>());
3538 build(odsBuilder&: builder, odsState&: result, loopTypes, target, mixedTileSizes, interchange,
3539 scalableSizes);
3540}
3541
3542void transform::TileUsingForOp::build(
3543 OpBuilder &builder, OperationState &result, Value target,
3544 ArrayRef<OpFoldResult> mixedTileSizes,
3545 ArrayRef<OpFoldResult> mixedInterchange,
3546 std::optional<ArrayRef<bool>> scalableSizes) {
3547 // Loop types are automaticaly splat by the callee, setting up one is
3548 // enough.
3549 SmallVector<Type> loopTypes(1, builder.getType<transform::AnyOpType>());
3550 build(odsBuilder&: builder, odsState&: result, loopTypes, target, mixedTileSizes, mixedInterchange,
3551 scalableSizes);
3552}
3553
3554void transform::TileUsingForOp::build(
3555 OpBuilder &builder, OperationState &result, TypeRange loopTypes,
3556 Value target, ArrayRef<OpFoldResult> mixedTileSizes,
3557 ArrayRef<int64_t> interchange,
3558 std::optional<ArrayRef<bool>> scalableSizes) {
3559 SmallVector<OpFoldResult> mixedInterchange =
3560 getAsOpFoldResult(arrayAttr: builder.getI64ArrayAttr(values: interchange));
3561 build(odsBuilder&: builder, odsState&: result, loopTypes, target, mixedTileSizes, mixedInterchange,
3562 scalableSizes);
3563}
3564
3565void transform::TileUsingForOp::build(
3566 OpBuilder &builder, OperationState &result, TypeRange loopTypes,
3567 Value target, ArrayRef<OpFoldResult> mixedTileSizes,
3568 ArrayRef<OpFoldResult> mixedInterchange,
3569 std::optional<ArrayRef<bool>> scalableSizes) {
3570 SmallVector<int64_t> staticTileSizes;
3571 SmallVector<Value> dynamicTileSizes;
3572 SmallVector<int64_t> staticInterchange;
3573 SmallVector<Value> dynamicInterchange;
3574 dispatchIndexOpFoldResults(ofrs: mixedTileSizes, dynamicVec&: dynamicTileSizes, staticVec&: staticTileSizes);
3575 dispatchIndexOpFoldResults(ofrs: mixedInterchange, dynamicVec&: dynamicInterchange,
3576 staticVec&: staticInterchange);
3577 // Call the default builder which sets up the proper operands segment sizes
3578 // attributes for multiple variadic operands. In the absence of this,
3579 // horrible bugs ensue.
3580 auto staticTileSizesAttr = builder.getDenseI64ArrayAttr(values: staticTileSizes);
3581 auto staticInterchangeAttr = builder.getDenseI64ArrayAttr(values: staticInterchange);
3582 unsigned numExpectedLoops =
3583 staticTileSizes.size() - llvm::count(Range&: staticTileSizes, Element: 0);
3584 SmallVector<Type> resultTypes;
3585 resultTypes.reserve(N: numExpectedLoops);
3586 assert((loopTypes.size() == 1 || loopTypes.size() == numExpectedLoops) &&
3587 "expected one loop type or as many as loops");
3588 if (loopTypes.size() == 1)
3589 resultTypes.append(NumInputs: numExpectedLoops, Elt: loopTypes[0]);
3590 else
3591 llvm::append_range(C&: resultTypes, R&: loopTypes);
3592 SmallVector<bool> expandedScalableSizes(mixedTileSizes.size(), false);
3593 if (scalableSizes.has_value())
3594 expandedScalableSizes.assign(in_start: scalableSizes->begin(), in_end: scalableSizes->end());
3595 Value packedTileSizes;
3596 build(odsBuilder&: builder, odsState&: result, /*tiled_linalg_op=*/target.getType(),
3597 /*loops=*/resultTypes,
3598 /*target=*/target,
3599 /*dynamic_sizes=*/dynamicTileSizes,
3600 /*interchange=*/dynamicInterchange,
3601 /*packed_tile_sizes=*/packedTileSizes,
3602 /*packed_interchange=*/Value(),
3603 /*static_sizes=*/staticTileSizesAttr,
3604 /*static_interchange=*/staticInterchangeAttr,
3605 /*scalable_sizes=*/expandedScalableSizes);
3606}
3607
3608LogicalResult transform::TileUsingForOp::verify() {
3609 bool hasPackedTiles = getPackedTileSizes() != Value();
3610 bool hasPackedInterchange = getPackedInterchange() != Value();
3611 if (!getMixedSizes().empty() && hasPackedTiles)
3612 return emitOpError(
3613 message: "tile_sizes and packed_tile_sizes are mutually exclusive");
3614 if (!getMixedInterchange().empty() && hasPackedInterchange)
3615 return emitOpError(
3616 message: "interchange and packed_interchange are mutually exclusive");
3617 if (hasPackedTiles && !getScalableSizes().empty())
3618 return emitOpError(
3619 message: "scalable tile_sizes are not supported with packed_tile_sizes");
3620
3621 if (getMixedSizes().size() != getScalableSizes().size())
3622 return emitOpError(message: "expected same number of sizes (")
3623 << getMixedSizes().size() << ") and scalable sizes ("
3624 << getScalableSizes().size() << ")";
3625
3626 auto iterspaceRank = getStaticSizes().size();
3627 ArrayRef<int64_t> permutation = getStaticInterchange();
3628 if (permutation.size() > iterspaceRank)
3629 return emitOpError()
3630 << "interchange length exceeds iteration space dimensions ("
3631 << iterspaceRank << "), found " << getInterchange();
3632 SmallVector<bool> seen(iterspaceRank, false);
3633 for (int64_t v : permutation) {
3634 if (!ShapedType::isDynamic(dValue: v)) {
3635 if (v < 0 || v >= static_cast<int64_t>(iterspaceRank))
3636 return emitOpError() << "expects interchange values to be in range [0, "
3637 << iterspaceRank << "), found: " << v;
3638 if (seen[v])
3639 return emitOpError() << "found duplicate interchange value: " << v;
3640 seen[v] = true;
3641 }
3642 }
3643
3644 ArrayRef<int64_t> staticSizes = getStaticSizes();
3645 unsigned numExpectedLoops =
3646 hasPackedTiles ? 1 : staticSizes.size() - llvm::count(Range&: staticSizes, Element: 0);
3647 if (getLoops().size() != numExpectedLoops)
3648 return emitOpError(message: "expected number of loops to tile (")
3649 << numExpectedLoops << ") to match number of `loops` results ("
3650 << getLoops().size() << ")";
3651 return verifyInnerTileAlignments(op: getOperation(), alignments: getInnerTileAlignments());
3652}
3653
3654DiagnosedSilenceableFailure
3655transform::TileUsingForOp::apply(transform::TransformRewriter &rewriter,
3656 TransformResults &transformResults,
3657 TransformState &state) {
3658 ArrayRef<int64_t> tileSizes = getStaticSizes();
3659 bool hasPackedTiles = getPackedTileSizes() != Value();
3660 bool hasPackedInterchange = getPackedInterchange() != Value();
3661 auto transformOp = cast<TransformOpInterface>(Val: getOperation());
3662
3663 SmallVector<OpFoldResult> mixedInterchange;
3664 if (hasPackedInterchange) {
3665 DiagnosedSilenceableFailure status =
3666 unpackSingleIndexResultPayloadOperations(
3667 state, transformOp, result&: mixedInterchange, packedHandle: getPackedInterchange());
3668 if (!status.succeeded())
3669 return status;
3670 } else {
3671 mixedInterchange = getMixedInterchange();
3672 }
3673 SmallVector<int64_t> tileInterchange;
3674 DiagnosedSilenceableFailure status = reifyMixedParamAndHandleResults(
3675 state, transformOp, mixedResults: mixedInterchange, reified&: tileInterchange);
3676 if (!status.succeeded())
3677 return status;
3678
3679 SmallVector<Operation *> targets =
3680 llvm::to_vector(Range: state.getPayloadOps(value: getTarget()));
3681 SmallVector<SmallVector<Operation *>> dynamicSizeProducers;
3682 SmallVector<SmallVector<int64_t>> paramSizes;
3683 SmallVector<OpFoldResult> mixedTileSizes;
3684 if (hasPackedTiles) {
3685 status = unpackSingleIndexResultPayloadOperations(
3686 state, transformOp, result&: mixedTileSizes, packedHandle: getPackedTileSizes());
3687 if (!status.succeeded())
3688 return status;
3689 tileSizes = {};
3690 } else {
3691 dynamicSizeProducers.reserve(N: getDynamicSizes().size());
3692 paramSizes.reserve(N: getDynamicSizes().size());
3693 for (Value transformValue : getDynamicSizes()) {
3694 if (isa<TransformParamTypeInterface>(Val: transformValue.getType())) {
3695 dynamicSizeProducers.push_back(Elt: {});
3696 ArrayRef<Attribute> params = state.getParams(value: transformValue);
3697 paramSizes.push_back(Elt: llvm::map_to_vector(C&: params, F: [](Attribute attr) {
3698 return cast<IntegerAttr>(Val&: attr).getValue().getSExtValue();
3699 }));
3700
3701 if (paramSizes.back().size() != targets.size()) {
3702 DiagnosedSilenceableFailure diag =
3703 emitSilenceableError()
3704 << "expected as many parameter values ("
3705 << dynamicSizeProducers.back().size() << ") as target ops ("
3706 << targets.size() << ")";
3707 diag.attachNote(loc: transformValue.getLoc()) << "for this parameter";
3708 return diag;
3709 }
3710
3711 continue;
3712 }
3713 paramSizes.push_back(Elt: {});
3714 dynamicSizeProducers.push_back(
3715 Elt: llvm::to_vector(Range: state.getPayloadOps(value: transformValue)));
3716
3717 if (dynamicSizeProducers.back().size() != targets.size()) {
3718 DiagnosedSilenceableFailure diag =
3719 emitSilenceableError()
3720 << "expected as many dynamic size-producing operations ("
3721 << dynamicSizeProducers.back().size() << ") as target ops ("
3722 << targets.size() << ")";
3723 diag.attachNote(loc: transformValue.getLoc()) << "for this handle";
3724 return diag;
3725 }
3726
3727 for (Operation *op : dynamicSizeProducers.back()) {
3728 if (op->getNumResults() == 1 &&
3729 isa<IndexType>(Val: op->getResult(idx: 0).getType())) {
3730 continue;
3731 }
3732
3733 DiagnosedSilenceableFailure diag =
3734 emitSilenceableError() << "expected sizes to be produced by ops "
3735 "with a single index-type result";
3736 diag.attachNote(loc: op->getLoc()) << "size producer op";
3737 diag.attachNote(loc: transformValue.getLoc()) << "for this handle";
3738 return diag;
3739 }
3740 }
3741 }
3742
3743 SmallVector<Operation *> tiled;
3744 SmallVector<SmallVector<Operation *, 4>, 4> loops;
3745 size_t numLoops =
3746 hasPackedTiles
3747 ? llvm::count_if(Range&: mixedTileSizes,
3748 P: [](OpFoldResult ofr) {
3749 if (auto attr = dyn_cast<Attribute>(Val&: ofr))
3750 return cast<IntegerAttr>(Val&: attr).getInt() != 0;
3751 return true;
3752 })
3753 : getLoops().size();
3754 loops.resize(N: numLoops);
3755 auto scalableSizes = getScalableSizes();
3756 for (auto [i, op] : llvm::enumerate(First&: targets)) {
3757 auto tilingInterface = dyn_cast<TilingInterface>(Val: op);
3758 if (!tilingInterface) {
3759 DiagnosedSilenceableFailure diag =
3760 emitSilenceableError()
3761 << "only ops implementing TilingInterface are supported";
3762 diag.attachNote(loc: op->getLoc()) << "target op";
3763 return diag;
3764 }
3765 if (isa<linalg::RelayoutOpInterface>(Val: op) &&
3766 !cast<DestinationStyleOpInterface>(Val: op).hasPureTensorSemantics()) {
3767 DiagnosedSilenceableFailure diag =
3768 emitSilenceableError()
3769 << "tiling only supports tensor semantics for linalg.pack / "
3770 "linalg.unpack. The target has memref operands";
3771 diag.attachNote(loc: op->getLoc()) << "target op";
3772 return diag;
3773 }
3774
3775 int64_t iterspaceRank = tilingInterface.getLoopIteratorTypes().size();
3776 if (tileInterchange.size() > static_cast<size_t>(iterspaceRank)) {
3777 return emitSilenceableError()
3778 << "interchange length exceeds iteration space dimensions ("
3779 << iterspaceRank << ")";
3780 }
3781 SmallVector<bool> seen(iterspaceRank, false);
3782 for (int64_t v : tileInterchange) {
3783 if (v < 0 || v >= iterspaceRank) {
3784 return emitSilenceableError()
3785 << "expects interchange values to be in range [0, "
3786 << iterspaceRank << "), found: " << v;
3787 }
3788 if (seen[v]) {
3789 return emitSilenceableError()
3790 << "found duplicate interchange value: " << v;
3791 }
3792 seen[v] = true;
3793 }
3794
3795 if (tileSizes.size() > tilingInterface.getLoopIteratorTypes().size()) {
3796 DiagnosedSilenceableFailure diag =
3797 emitSilenceableError()
3798 << "too many tiles provided, expected at most "
3799 << tilingInterface.getLoopIteratorTypes().size() << " found "
3800 << tileSizes.size();
3801 diag.attachNote(loc: op->getLoc()) << "target op";
3802 return diag;
3803 }
3804
3805 scf::SCFTilingOptions tilingOptions;
3806 if (!hasPackedTiles && tileSizes.empty()) {
3807 tilingOptions.setTileSizeComputationFunction(
3808 [](OpBuilder &, Operation *) -> SmallVector<OpFoldResult> {
3809 return {};
3810 });
3811 } else if (hasPackedTiles) {
3812 tilingOptions.setTileSizes(mixedTileSizes);
3813 } else {
3814 tilingOptions.setTileSizeComputationFunction([&, index = i](OpBuilder &b,
3815 Operation *) {
3816 SmallVector<OpFoldResult> sizes;
3817 sizes.reserve(N: tileSizes.size());
3818 unsigned dynamicIdx = 0;
3819
3820 for (auto [ofrIdx, ofr] : llvm::enumerate(First: getMixedSizes())) {
3821 if (auto attr = llvm::dyn_cast_if_present<Attribute>(Val&: ofr)) {
3822 if (scalableSizes[ofrIdx]) {
3823 auto val = arith::ConstantIndexOp::create(
3824 builder&: b, location: getLoc(), value: cast<IntegerAttr>(Val&: attr).getInt());
3825 Value vscale =
3826 vector::VectorScaleOp::create(builder&: b, location: getLoc(), result: b.getIndexType());
3827 sizes.push_back(
3828 Elt: arith::MulIOp::create(builder&: b, location: getLoc(), lhs: val, rhs: vscale).getResult());
3829 } else {
3830 sizes.push_back(Elt: attr);
3831 }
3832 continue;
3833 }
3834 ArrayRef<Operation *> dynamicSizes = dynamicSizeProducers[dynamicIdx];
3835 ArrayRef<int64_t> params = paramSizes[dynamicIdx];
3836 ++dynamicIdx;
3837 assert((dynamicSizes.empty() ^ params.empty()) &&
3838 "expected either dynamic sizes or parameters");
3839 if (!params.empty()) {
3840 sizes.push_back(Elt: b.getIndexAttr(value: params[index]));
3841 } else {
3842 sizes.push_back(Elt: dynamicSizes[index]->getResult(idx: 0));
3843 }
3844 }
3845 return sizes;
3846 });
3847 }
3848
3849 tilingOptions.setInterchange(tileInterchange);
3850 tilingOptions.setInnerTileAlignments(
3851 convertInnerTileAlignments(alignments: getInnerTileAlignments()));
3852 FailureOr<scf::SCFTilingResult> maybeTilingResult =
3853 tileUsingSCF(rewriter, op: tilingInterface, options: tilingOptions);
3854 if (failed(Result: maybeTilingResult))
3855 return DiagnosedSilenceableFailure::definiteFailure();
3856
3857 rewriter.replaceOp(op, newValues: maybeTilingResult->replacements);
3858
3859 tiled.append(RHS: maybeTilingResult->tiledOps);
3860 for (const auto &en2 : llvm::enumerate(First&: maybeTilingResult->loops))
3861 loops[en2.index()].push_back(Elt: en2.value());
3862 }
3863
3864 transformResults.set(value: cast<OpResult>(Val: getTiledLinalgOp()), ops&: tiled);
3865 if (hasPackedTiles) {
3866 // For packed sizes all created loops are assigned to a single handle.
3867 SmallVector<Operation *> flattenedLoops;
3868 for (auto [targetIdx, _] : llvm::enumerate(First&: targets))
3869 for (auto [loopIdx, __] : llvm::enumerate(First&: loops))
3870 flattenedLoops.push_back(Elt: loops[loopIdx][targetIdx]);
3871 transformResults.set(value: cast<OpResult>(Val: getLoops().front()), ops&: flattenedLoops);
3872 } else {
3873 for (const auto &en : llvm::enumerate(First&: loops))
3874 transformResults.set(value: cast<OpResult>(Val: getLoops()[en.index()]), ops&: en.value());
3875 }
3876
3877 return DiagnosedSilenceableFailure::success();
3878}
3879
3880SmallVector<OpFoldResult> transform::TileUsingForOp::getMixedSizes() {
3881 ValueRange dynamic = getDynamicSizes();
3882 ArrayRef<int64_t> tileSizes = getStaticSizes();
3883 SmallVector<OpFoldResult> results;
3884 results.reserve(N: tileSizes.size());
3885 unsigned dynamicPos = 0;
3886 Builder builder(getContext());
3887 for (int64_t size : tileSizes) {
3888 if (size == ShapedType::kDynamic) {
3889 results.push_back(Elt: dynamic[dynamicPos++]);
3890 } else {
3891 results.push_back(Elt: builder.getIndexAttr(value: size));
3892 }
3893 }
3894 return results;
3895}
3896
3897SmallVector<OpFoldResult> transform::TileUsingForOp::getMixedInterchange() {
3898 return getMixedValues(staticValues: getStaticInterchange(), dynamicValues: getInterchange(), context: getContext());
3899}
3900
3901void transform::TileUsingForOp::getEffects(
3902 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
3903 consumesHandle(handles: getTargetMutable(), effects);
3904 onlyReadsHandle(handles: getDynamicSizesMutable(), effects);
3905 onlyReadsHandle(handles: getInterchangeMutable(), effects);
3906 onlyReadsHandle(handles: getPackedTileSizesMutable(), effects);
3907 onlyReadsHandle(handles: getPackedInterchangeMutable(), effects);
3908 producesHandle(handles: getOperation()->getOpResults(), effects);
3909 modifiesPayload(effects);
3910}
3911
3912//===----------------------------------------------------------------------===//
3913// TileUsingForallOp
3914//===----------------------------------------------------------------------===//
3915
3916void transform::TileUsingForallOp::build(OpBuilder &builder,
3917 OperationState &result, Value target,
3918 ArrayRef<int64_t> staticTileSizes,
3919 transform::TileSizesSpec,
3920 ArrayAttr mapping) {
3921 return build(odsBuilder&: builder, odsState&: result,
3922 /*target=*/target,
3923 /*mixedTileSizes=*/
3924 getAsOpFoldResult(arrayAttr: builder.getI64ArrayAttr(values: staticTileSizes)),
3925 /*_=*/odsArg2: TileSizesSpec(),
3926 /*mapping=*/mapping);
3927}
3928
3929void transform::TileUsingForallOp::build(OpBuilder &builder,
3930 OperationState &result, Value target,
3931 ArrayRef<OpFoldResult> mixedTileSizes,
3932 transform::TileSizesSpec,
3933 ArrayAttr mapping) {
3934 SmallVector<int64_t> staticTileSizes;
3935 SmallVector<Value> dynamicTileSizes;
3936 dispatchIndexOpFoldResults(ofrs: mixedTileSizes, dynamicVec&: dynamicTileSizes, staticVec&: staticTileSizes);
3937 // Call the default builder which sets up the proper operands segment sizes
3938 // attributes for multiple variadic operands. In the absence of this,
3939 // horrible bugs ensue.
3940 MLIRContext *ctx = builder.getContext();
3941 auto operationType = transform::AnyOpType::get(ctx);
3942 auto staticTileSizesAttr = builder.getDenseI64ArrayAttr(values: staticTileSizes);
3943 build(odsBuilder&: builder, odsState&: result,
3944 /*resultTypes=*/TypeRange{operationType, operationType},
3945 /*target=*/target,
3946 /*num_threads=*/ValueRange{},
3947 /*tile_sizes=*/dynamicTileSizes,
3948 /*packed_num_threads=*/Value(),
3949 /*packed_tile_sizes=*/Value(),
3950 /*static_num_threads=*/builder.getDenseI64ArrayAttr(values: {}),
3951 /*static_tile_sizes=*/staticTileSizesAttr,
3952 /*mapping=*/mapping);
3953}
3954
3955void transform::TileUsingForallOp::build(OpBuilder &builder,
3956 OperationState &result, Value target,
3957 ArrayRef<int64_t> staticNumThreads,
3958 transform::NumThreadsSpec,
3959 ArrayAttr mapping) {
3960 return build(odsBuilder&: builder, odsState&: result, target,
3961 mixedNumThreads: getAsOpFoldResult(arrayAttr: builder.getI64ArrayAttr(values: staticNumThreads)),
3962 odsArg2: NumThreadsSpec(), mapping);
3963}
3964
3965void transform::TileUsingForallOp::build(OpBuilder &builder,
3966 OperationState &result, Value target,
3967 ArrayRef<OpFoldResult> mixedNumThreads,
3968 transform::NumThreadsSpec,
3969 ArrayAttr mapping) {
3970 SmallVector<int64_t> staticNumThreads;
3971 SmallVector<Value> dynamicNumThreads;
3972 dispatchIndexOpFoldResults(ofrs: mixedNumThreads, dynamicVec&: dynamicNumThreads,
3973 staticVec&: staticNumThreads);
3974 // Call the default builder which sets up the proper operands segment sizes
3975 // attributes for multiple variadic operands. In the absence of this,
3976 // horrible bugs ensue.
3977 MLIRContext *ctx = builder.getContext();
3978 auto operationType = transform::AnyOpType::get(ctx);
3979 auto staticNumThreadsAttr = builder.getDenseI64ArrayAttr(values: staticNumThreads);
3980 build(odsBuilder&: builder, odsState&: result,
3981 /*resultTypes=*/TypeRange{operationType, operationType},
3982 /*target=*/target,
3983 /*num_threads=*/dynamicNumThreads,
3984 /*tile_sizes=*/ValueRange{},
3985 /*packed_num_threads=*/Value(),
3986 /*packed_tile_sizes=*/Value(),
3987 /*static_num_threads=*/staticNumThreadsAttr,
3988 /*static_tile_sizes=*/builder.getDenseI64ArrayAttr(values: {}),
3989 /*mapping=*/mapping);
3990}
3991
3992/// Given `lbs`, `ubs` and `steps` of loops, return (for each loop), the
3993/// normalized upper bound.
3994static SmallVector<OpFoldResult>
3995normalizeUpperBounds(RewriterBase &rewriter, Location loc,
3996 ArrayRef<OpFoldResult> lbs, ArrayRef<OpFoldResult> ubs,
3997 ArrayRef<OpFoldResult> steps) {
3998 AffineExpr s0, s1, s2;
3999 bindSymbols(ctx: rewriter.getContext(), exprs&: s0, exprs&: s1, exprs&: s2);
4000 AffineExpr normalizedUbExpr = (s1 - s0).ceilDiv(other: s2);
4001 SmallVector<OpFoldResult> normalizedUbs;
4002 for (auto [lb, ub, step] : llvm::zip_equal(t&: lbs, u&: ubs, args&: steps)) {
4003 OpFoldResult normalizedUb = affine::makeComposedFoldedAffineApply(
4004 b&: rewriter, loc, expr: normalizedUbExpr, operands: {lb, ub, step});
4005 normalizedUbs.push_back(Elt: normalizedUb);
4006 }
4007 return normalizedUbs;
4008}
4009
4010/// When a loop is normalized, the uses of the induction variable within the
4011/// loop need to replaced with `original_lb + old_iv * original_step`.
4012static SmallVector<Value> denormalizeIndVar(RewriterBase &rewriter,
4013 Location loc, ValueRange ivs,
4014 ArrayRef<OpFoldResult> lbs,
4015 ArrayRef<OpFoldResult> steps) {
4016 AffineExpr s0, s1;
4017 AffineExpr d0;
4018 bindSymbols(ctx: rewriter.getContext(), exprs&: s0, exprs&: s1);
4019 bindDims(ctx: rewriter.getContext(), exprs&: d0);
4020 AffineExpr denormExpr = s0 + d0 * s1;
4021 SmallVector<Value> denormalizedIvs;
4022
4023 for (auto [iv, lb, step] : llvm::zip_equal(t&: ivs, u&: lbs, args&: steps)) {
4024 OpFoldResult denormValue = affine::makeComposedFoldedAffineApply(
4025 b&: rewriter, loc, expr: denormExpr, operands: ArrayRef<OpFoldResult>{iv, lb, step});
4026 denormalizedIvs.push_back(
4027 Elt: getValueOrCreateConstantIndexOp(b&: rewriter, loc, ofr: denormValue));
4028 }
4029 return denormalizedIvs;
4030}
4031
4032/// Given a `scf.forall` loop return a loop op with the loop bounds
4033/// normalized.
4034/// TODO: Replace this with a general utility to normalize `scf.forall`.
4035/// At the time of writing, this wasnt done since adding this to `scf`
4036/// dialect would disallow using of `affine.apply` operations due
4037/// to cyclic dependencies. To avoid churn in lit tests
4038/// with the change this was added with, defer that to a follow up.
4039static scf::ForallOp normalizeForallLoopOp(RewriterBase &rewriter,
4040 scf::ForallOp loop) {
4041 SmallVector<OpFoldResult> lbs = loop.getMixedLowerBound();
4042 SmallVector<OpFoldResult> ubs = loop.getMixedUpperBound();
4043 SmallVector<OpFoldResult> steps = loop.getMixedStep();
4044
4045 if (llvm::all_of(Range&: lbs, P: isZeroInteger) && llvm::all_of(Range&: steps, P: isOneInteger)) {
4046 return loop;
4047 }
4048
4049 Location loc = loop.getLoc();
4050 SmallVector<OpFoldResult> normalizedUbs =
4051 normalizeUpperBounds(rewriter, loc, lbs, ubs, steps);
4052 SmallVector<OpFoldResult> normalizedLbs(normalizedUbs.size(),
4053 rewriter.getIndexAttr(value: 0));
4054 SmallVector<OpFoldResult> normalizedSteps(normalizedUbs.size(),
4055 rewriter.getIndexAttr(value: 1));
4056
4057 auto normalizedForallOp = scf::ForallOp::create(
4058 builder&: rewriter, location: loc, lbs: normalizedLbs, ubs: normalizedUbs, steps: normalizedSteps,
4059 outputs: loop.getOutputs(), mapping: loop.getMapping(),
4060 bodyBuilderFn: [](OpBuilder &, Location, ValueRange) {});
4061
4062 auto normalizedLoopIvs = normalizedForallOp.getInductionVars();
4063 OpBuilder::InsertionGuard g(rewriter);
4064 Block *normalizedLoopBlock = normalizedForallOp.getBody();
4065 rewriter.setInsertionPointToStart(normalizedLoopBlock);
4066
4067 SmallVector<Value> argValues =
4068 denormalizeIndVar(rewriter, loc, ivs: normalizedLoopIvs, lbs, steps);
4069 argValues.append(in_start: normalizedForallOp.getRegionIterArgs().begin(),
4070 in_end: normalizedForallOp.getRegionIterArgs().end());
4071 Block *origLoopBlock = loop.getBody();
4072 rewriter.mergeBlocks(source: origLoopBlock, dest: normalizedLoopBlock, argValues);
4073
4074 rewriter.replaceOp(op: loop, newOp: normalizedForallOp);
4075 return normalizedForallOp;
4076}
4077
4078DiagnosedSilenceableFailure transform::tileToForallOpImpl(
4079 RewriterBase &rewriter, transform::TransformState &state,
4080 TransformOpInterface transformOp, Operation *target,
4081 ArrayRef<OpFoldResult> mixedNumThreads,
4082 ArrayRef<OpFoldResult> mixedTileSizes, std::optional<ArrayAttr> mapping,
4083 scf::SCFTilingResult &tilingResult) {
4084 // Transform all targets one by one.
4085 auto tileableOp = dyn_cast<TilingInterface>(Val: target);
4086 if (!tileableOp) {
4087 DiagnosedSilenceableFailure diag =
4088 transformOp.emitSilenceableError()
4089 << "only TilingInterface ops are supported";
4090 diag.attachNote(loc: target->getLoc()) << "target op";
4091 return diag;
4092 }
4093 if (isa<linalg::RelayoutOpInterface>(Val: target) &&
4094 !cast<DestinationStyleOpInterface>(Val: target).hasPureTensorSemantics()) {
4095 DiagnosedSilenceableFailure diag =
4096 transformOp.emitSilenceableError()
4097 << "tiling only supports tensor semantics for linalg.pack / "
4098 "linalg.unpack. The target has memref operands";
4099 diag.attachNote(loc: target->getLoc()) << "target op";
4100 return diag;
4101 }
4102 rewriter.setInsertionPoint(tileableOp);
4103 scf::SCFTilingOptions options;
4104 options.setLoopType(scf::SCFTilingOptions::LoopType::ForallOp);
4105 if (!mixedNumThreads.empty()) {
4106 options.setNumThreads(mixedNumThreads);
4107 } else {
4108 options.setTileSizes(mixedTileSizes);
4109 }
4110 if (mapping) {
4111 options.setMapping(mapping.value().getValue());
4112 }
4113 FailureOr<scf::SCFTilingResult> maybeTilingResult =
4114 scf::tileUsingSCF(rewriter, op: tileableOp, options);
4115
4116 if (failed(Result: maybeTilingResult))
4117 return transformOp.emitDefaultSilenceableFailure(target: tileableOp);
4118
4119 rewriter.replaceOp(op: tileableOp, newValues: maybeTilingResult->replacements);
4120
4121 tilingResult = *maybeTilingResult;
4122
4123 // Rank-0 ops produce no loops; skip normalization when there is nothing
4124 // to normalize.
4125 if (mixedNumThreads.empty() && !tilingResult.loops.empty()) {
4126 auto generatedForallOp = cast<scf::ForallOp>(Val&: tilingResult.loops.front());
4127 OpBuilder::InsertionGuard g(rewriter);
4128 rewriter.setInsertionPoint(generatedForallOp);
4129 scf::ForallOp normalizedForallOp =
4130 normalizeForallLoopOp(rewriter, loop: generatedForallOp);
4131 tilingResult.loops.front() = normalizedForallOp;
4132 }
4133
4134 return DiagnosedSilenceableFailure::success();
4135}
4136
4137DiagnosedSilenceableFailure transform::TileUsingForallOp::apply(
4138 transform::TransformRewriter &rewriter,
4139 transform::TransformResults &transformResults,
4140 transform::TransformState &state) {
4141 auto transformOp = cast<TransformOpInterface>(Val: getOperation());
4142
4143 // Result payload ops.
4144 SmallVector<Operation *> tileOps;
4145 SmallVector<Operation *> tiledOps;
4146
4147 // Unpack handles.
4148 SmallVector<OpFoldResult> mixedNumThreads;
4149 DiagnosedSilenceableFailure status =
4150 getPackedNumThreads()
4151 ? unpackSingleIndexResultPayloadOperations(
4152 state, transformOp, result&: mixedNumThreads, packedHandle: getPackedNumThreads())
4153 : unpackSingleIndexResultPayloadOperations(
4154 state, transformOp, result&: mixedNumThreads, ofrs: getMixedNumThreads());
4155 if (!status.succeeded())
4156 return status;
4157 SmallVector<OpFoldResult> mixedTileSizes;
4158 status = getPackedTileSizes()
4159 ? unpackSingleIndexResultPayloadOperations(
4160 state, transformOp, result&: mixedTileSizes, packedHandle: getPackedTileSizes())
4161 : unpackSingleIndexResultPayloadOperations(
4162 state, transformOp, result&: mixedTileSizes, ofrs: getMixedTileSizes());
4163 if (!status.succeeded())
4164 return status;
4165
4166 for (Operation *target : state.getPayloadOps(value: getTarget())) {
4167 scf::SCFTilingResult tilingResult;
4168 DiagnosedSilenceableFailure diag = tileToForallOpImpl(
4169 rewriter, state, transformOp, target, mixedNumThreads, mixedTileSizes,
4170 mapping: getMapping(), tilingResult);
4171 if (!diag.succeeded())
4172 return diag;
4173 if (!tilingResult.loops.empty())
4174 tileOps.push_back(Elt: tilingResult.loops.front());
4175 tiledOps.append(RHS: tilingResult.tiledOps);
4176 }
4177
4178 transformResults.set(value: cast<OpResult>(Val: getForallOp()), ops&: tileOps);
4179 transformResults.set(value: cast<OpResult>(Val: getTiledOp()), ops&: tiledOps);
4180
4181 return DiagnosedSilenceableFailure::success();
4182}
4183
4184void transform::TileUsingForallOp::getEffects(
4185 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
4186 consumesHandle(handles: getTargetMutable(), effects);
4187 onlyReadsHandle(handles: getTileSizesMutable(), effects);
4188 onlyReadsHandle(handles: getNumThreadsMutable(), effects);
4189 onlyReadsHandle(handles: getPackedNumThreadsMutable(), effects);
4190 onlyReadsHandle(handles: getPackedTileSizesMutable(), effects);
4191 producesHandle(handles: getOperation()->getOpResults(), effects);
4192 modifiesPayload(effects);
4193}
4194
4195SmallVector<OpFoldResult> TileUsingForallOp::getMixedNumThreads() {
4196 Builder b(getContext());
4197 return getMixedValues(staticValues: getStaticNumThreads(), dynamicValues: getNumThreads(), b);
4198}
4199
4200SmallVector<OpFoldResult> TileUsingForallOp::getMixedTileSizes() {
4201 Builder b(getContext());
4202 return getMixedValues(staticValues: getStaticTileSizes(), dynamicValues: getTileSizes(), b);
4203}
4204
4205LogicalResult TileUsingForallOp::verify() {
4206 int numThreadsSpec = static_cast<int>(!getMixedNumThreads().empty()) +
4207 static_cast<int>(getPackedNumThreads() != Value());
4208 if (numThreadsSpec > 1)
4209 return emitOpError(
4210 message: "num_threads and packed_num_threads are mutually exclusive");
4211 int tileSizesSpec = static_cast<int>(!getMixedTileSizes().empty()) +
4212 static_cast<int>(getPackedTileSizes() != Value());
4213 if (tileSizesSpec > 1)
4214 return emitOpError(
4215 message: "tile_sizes and packed_tile_sizes are mutually exclusive");
4216 if (numThreadsSpec == 0 && tileSizesSpec == 0)
4217 return emitOpError(message: "either (packed_)num_threads or (packed_)tile_sizes "
4218 "must be specified");
4219 return success();
4220}
4221
4222//===----------------------------------------------------------------------===//
4223// VectorizeChildrenAndApplyPatternsOp
4224//===----------------------------------------------------------------------===//
4225
4226void transform::VectorizeChildrenAndApplyPatternsOp::build(
4227 OpBuilder &builder, OperationState &result, Value target,
4228 bool foldTypeExtensionsIntoContract, bool vectorizePadding,
4229 bool vectorizeExtract, bool flatten1DDepthwiseConv) {
4230 result.addOperands(newOperands: target);
4231 if (foldTypeExtensionsIntoContract) {
4232 result.addAttribute(
4233 name: VectorizeChildrenAndApplyPatternsOp::
4234 getFoldTypeExtensionsIntoContractAttrName(name: result.name),
4235 attr: builder.getUnitAttr());
4236 }
4237 if (vectorizePadding) {
4238 result.addAttribute(
4239 name: VectorizeChildrenAndApplyPatternsOp::getVectorizePaddingAttrName(
4240 name: result.name),
4241 attr: builder.getUnitAttr());
4242 }
4243 if (vectorizeExtract) {
4244 result.addAttribute(
4245 name: VectorizeChildrenAndApplyPatternsOp::getVectorizeNdExtractAttrName(
4246 name: result.name),
4247 attr: builder.getUnitAttr());
4248 }
4249 if (flatten1DDepthwiseConv) {
4250 result.addAttribute(
4251 name: VectorizeChildrenAndApplyPatternsOp::getFlatten_1dDepthwiseConvAttrName(
4252 name: result.name),
4253 attr: builder.getUnitAttr());
4254 }
4255 result.addTypes(newTypes: transform::AnyOpType::get(ctx: builder.getContext()));
4256}
4257
4258namespace {
4259/// This is an helper only to call vectorize via a pattern inside of
4260/// VectorizeChildrenAndApplyPatternsOp::applyToOne.
4261struct VectorizationPattern : public RewritePattern {
4262 explicit VectorizationPattern(MLIRContext *context,
4263 bool vectorizeExtract = false,
4264 bool flattenConv = false)
4265 : RewritePattern(MatchAnyOpTypeTag(), /*benefit=*/1, context),
4266 vectorizeNDExtract(vectorizeExtract),
4267 flatten1DDepthwiseConv(flattenConv) {}
4268 LogicalResult matchAndRewrite(Operation *op,
4269 PatternRewriter &rewriter) const override {
4270 if (!linalg::hasVectorizationImpl(op))
4271 return rewriter.notifyMatchFailure(arg&: op,
4272 msg: "Unsupported Op, cannot vectorize");
4273 FailureOr<VectorizationResult> vectorResults =
4274 vectorize(rewriter, op, /*inputVectorSizes=*/{},
4275 /*inputScalableVecDims=*/{}, vectorizeNDExtract,
4276 flatten1DDepthwiseConv);
4277 if (failed(Result: vectorResults))
4278 return failure();
4279 rewriter.replaceOp(op, newValues: vectorResults->replacements);
4280 return success();
4281 }
4282
4283private:
4284 /// Controls whether to vectorize `tensor.extract` when the input tensor is
4285 /// rank >= 2.
4286 bool vectorizeNDExtract = false;
4287 /// Controls whether to "flatten" the channel dimension when vectorising 1D
4288 /// depthwise convolutions. This should lead to bette vectorization for
4289 /// tensors with a low number of channel dimensions.
4290 bool flatten1DDepthwiseConv = false;
4291};
4292} // namespace
4293
4294DiagnosedSilenceableFailure
4295transform::VectorizeChildrenAndApplyPatternsOp::applyToOne(
4296 transform::TransformRewriter &rewriter, Operation *target,
4297 transform::ApplyToEachResultList &results,
4298 transform::TransformState &state) {
4299 if (!target->hasTrait<OpTrait::IsIsolatedFromAbove>()) {
4300 auto diag = this->emitOpError(message: "requires isolated-from-above targets");
4301 diag.attachNote(noteLoc: target->getLoc()) << "non-isolated target";
4302 return DiagnosedSilenceableFailure::definiteFailure();
4303 }
4304
4305 MLIRContext *ctx = getContext();
4306 RewritePatternSet patterns(ctx);
4307 patterns.add<VectorizationPattern>(arg&: ctx, args: getVectorizeNdExtract(),
4308 args: getFlatten_1dDepthwiseConv());
4309
4310 if (!getDisableTransferPermutationMapLoweringPatterns())
4311 vector::populateVectorTransferPermutationMapLoweringPatterns(patterns);
4312
4313 if (!getDisableMultiReductionToContractPatterns())
4314 vector::populateVectorReductionToContractPatterns(patterns);
4315
4316 vector::populateSinkVectorOpsPatterns(patterns);
4317
4318 patterns.add<linalg::LinalgCopyVTRForwardingPattern,
4319 linalg::LinalgCopyVTWForwardingPattern>(arg&: ctx,
4320 /*benefit=*/args: 2);
4321 vector::TransferReadOp::getCanonicalizationPatterns(results&: patterns, context: ctx);
4322 vector::TransferWriteOp::getCanonicalizationPatterns(results&: patterns, context: ctx);
4323 tensor::populateFoldTensorSubsetIntoVectorTransferPatterns(patterns);
4324
4325 patterns.add<CopyVectorizationPattern>(arg&: ctx);
4326
4327 if (getFoldTypeExtensionsIntoContract())
4328 vector::populateFoldArithExtensionPatterns(patterns);
4329
4330 if (getVectorizePadding()) {
4331 linalg::populatePadOpVectorizationPatterns(patterns);
4332 // This creates an alternative path for lowering tensor.pad - by
4333 // decomposing it into e.g. linalg.fill.
4334 linalg::populateDecomposePadPatterns(patterns);
4335 }
4336 vector::populateVectorStepLoweringPatterns(patterns);
4337
4338 TrackingListener listener(state, *this);
4339 if (failed(
4340 Result: applyPatternsGreedily(op: target, patterns: std::move(patterns),
4341 config: GreedyRewriteConfig().setListener(&listener))))
4342 return emitDefaultDefiniteFailure(target);
4343
4344 results.push_back(op: target);
4345 return DiagnosedSilenceableFailure::success();
4346}
4347
4348//===----------------------------------------------------------------------===//
4349// VectorizeOp
4350//===----------------------------------------------------------------------===//
4351
4352DiagnosedSilenceableFailure transform::VectorizeOp::apply(
4353 transform::TransformRewriter &rewriter,
4354 mlir::transform::TransformResults &transformResults,
4355 mlir::transform::TransformState &state) {
4356 auto targets = state.getPayloadOps(value: getTarget());
4357 if (std::empty(cont: targets))
4358 return DiagnosedSilenceableFailure::success();
4359 auto transformOp = cast<TransformOpInterface>(Val: getOperation());
4360 SmallVector<int64_t> vectorSizes;
4361 DiagnosedSilenceableFailure status = reifyMixedParamAndHandleResults(
4362 state, transformOp, mixedResults: getMixedVectorSizes(), reified&: vectorSizes);
4363 if (!status.succeeded())
4364 return status;
4365
4366 // TODO: Check that the correct number of vectorSizes was provided.
4367 for (Operation *target : targets) {
4368 if (!linalg::hasVectorizationImpl(target)) {
4369 return mlir::emitSilenceableFailure(loc: target->getLoc())
4370 << "Unsupported Op, cannot vectorize";
4371 }
4372 FailureOr<VectorizationResult> vectorResults =
4373 linalg::vectorize(rewriter, op: target, inputVectorSizes: vectorSizes, inputScalableVecDims: getScalableSizes(),
4374 vectorizeNDExtract: getVectorizeNdExtract().value_or(u: false),
4375 /*flatten1DDepthwiseConv=*/false,
4376 assumeDynamicDimsMatchVecSizes: getAssumeDynamicDimsMatchVecSizes().value_or(u: false),
4377 createNamedContraction: getCreateNamedContraction().value_or(u: false));
4378 if (failed(Result: vectorResults)) {
4379 return mlir::emitSilenceableFailure(loc: target->getLoc())
4380 << "Attempted to vectorize, but failed";
4381 }
4382 rewriter.replaceOp(op: target, newValues: vectorResults->replacements);
4383 }
4384
4385 return DiagnosedSilenceableFailure::success();
4386}
4387
4388void transform::VectorizeOp::getEffects(
4389 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
4390 consumesHandle(handles: getTargetMutable(), effects);
4391 onlyReadsHandle(handles: getVectorSizesMutable(), effects);
4392 modifiesPayload(effects);
4393}
4394
4395SmallVector<OpFoldResult> VectorizeOp::getMixedVectorSizes() {
4396 OpBuilder b(getContext());
4397 return getMixedValues(staticValues: getStaticVectorSizes(), dynamicValues: getVectorSizes(), b);
4398}
4399
4400LogicalResult transform::VectorizeOp::verify() {
4401 if (getStaticVectorSizes().size() != getScalableSizes().size())
4402 return emitOpError(message: "expected same number of vector sizes (")
4403 << getStaticVectorSizes().size() << ") and scalable sizes ("
4404 << getScalableSizes().size() << ")";
4405 return success();
4406}
4407
4408//===----------------------------------------------------------------------===//
4409// HoistRedundantVectorTransfersOp
4410//===----------------------------------------------------------------------===//
4411
4412DiagnosedSilenceableFailure
4413transform::HoistRedundantVectorTransfersOp::applyToOne(
4414 transform::TransformRewriter &rewriter, func::FuncOp target,
4415 transform::ApplyToEachResultList &results,
4416 transform::TransformState &state) {
4417 // WARNING: This hoisting does not model parallelism and is generally
4418 // incorrect when used on distributed loops with memref semantics!
4419 // TODO: obsolete and should be retired.
4420 linalg::hoistRedundantVectorTransfers(root: target, verifyNonZeroTrip: getVerifyNonZeroTrip());
4421 results.push_back(op: target);
4422 return DiagnosedSilenceableFailure::success();
4423}
4424
4425//===----------------------------------------------------------------------===//
4426// HoistRedundantVectorBroadcastsOp
4427//===----------------------------------------------------------------------===//
4428
4429DiagnosedSilenceableFailure
4430transform::HoistRedundantVectorBroadcastsOp::applyToOne(
4431 transform::TransformRewriter &rewriter, mlir::Operation *target,
4432 transform::ApplyToEachResultList &results,
4433 transform::TransformState &state) {
4434 rewriter.setInsertionPoint(target);
4435 linalg::hoistRedundantVectorBroadcasts(rewriter, root: target);
4436 results.push_back(op: target);
4437 return DiagnosedSilenceableFailure::success();
4438}
4439
4440//===----------------------------------------------------------------------===//
4441// ConvertConv2DToImg2ColOp.
4442//===----------------------------------------------------------------------===//
4443
4444DiagnosedSilenceableFailure transform::ConvertConv2DToImg2ColOp::applyToOne(
4445 transform::TransformRewriter &rewriter, linalg::LinalgOp target,
4446 transform::ApplyToEachResultList &results,
4447 transform::TransformState &state) {
4448 rewriter.setInsertionPoint(target);
4449 auto maybeTransformed =
4450 TypeSwitch<Operation *, FailureOr<std::pair<Operation *, Operation *>>>(
4451 target)
4452 .Case(caseFn: [&](linalg::Conv2DNhwcHwcfOp op) {
4453 return rewriteInIm2Col(rewriter, convOp: op);
4454 })
4455 .Case(caseFn: [&](linalg::Conv2DNhwcFhwcOp op) {
4456 return rewriteInIm2Col(rewriter, convOp: op);
4457 })
4458 .Case(caseFn: [&](linalg::DepthwiseConv2DNhwcHwcOp op) {
4459 return rewriteInIm2Col(rewriter, convOp: op);
4460 })
4461 .Case(caseFn: [&](linalg::Conv2DNchwFchwOp op) {
4462 return rewriteInIm2Col(rewriter, convOp: op);
4463 })
4464 .Default(defaultFn: [&](Operation *op) {
4465 return rewriter.notifyMatchFailure(arg&: op, msg: "not supported");
4466 });
4467 if (failed(Result: maybeTransformed))
4468 return emitDefaultSilenceableFailure(target);
4469 // Handle to the operation producing the img2col tensor.
4470 results.push_back(op: maybeTransformed->first);
4471 // Handle to the operation that replaces the original convolution.
4472 results.push_back(op: maybeTransformed->second);
4473 return DiagnosedSilenceableFailure::success();
4474}
4475
4476//===----------------------------------------------------------------------===//
4477// FlattenElementwiseLinalgOp.
4478//===----------------------------------------------------------------------===//
4479
4480DiagnosedSilenceableFailure transform::FlattenElementwiseLinalgOp::applyToOne(
4481 transform::TransformRewriter &rewriter, linalg::LinalgOp target,
4482 transform::ApplyToEachResultList &results,
4483 transform::TransformState &state) {
4484 rewriter.setInsertionPoint(target);
4485 if (!isElementwise(op: target))
4486 return mlir::emitSilenceableFailure(loc: target->getLoc())
4487 << "only elementwise flattening is supported";
4488
4489 if (!llvm::all_of(Range: target.getIndexingMapsArray(), P: [](AffineMap m) {
4490 return m.isPermutation() || m.getNumResults() == 0;
4491 })) {
4492 results.push_back(op: target);
4493 return mlir::emitSilenceableFailure(loc: target->getLoc())
4494 << "broadcasting of non scalar operands is not supported";
4495 }
4496
4497 // If rank <= 1, do nothing
4498 if (target.getNumLoops() <= 1) {
4499 results.push_back(op: target);
4500 return DiagnosedSilenceableFailure::success();
4501 }
4502
4503 // Only broadcasts with a 0-D input are handled; leave anything else
4504 // unchanged.
4505 if (auto broadcastOp = dyn_cast<linalg::BroadcastOp>(Val: target.getOperation());
4506 broadcastOp && broadcastOp.getInput().getType().getRank() != 0) {
4507 results.push_back(op: target);
4508 return DiagnosedSilenceableFailure::success();
4509 }
4510
4511 // Attempt to flatten all dims to one.
4512 ReassociationIndices reassociation(target.getNumLoops());
4513 std::iota(first: reassociation.begin(), last: reassociation.end(), value: 0);
4514 auto maybeFlattened =
4515 collapseOpIterationDims(op: target, foldedIterationDims: reassociation, rewriter);
4516 if (failed(Result: maybeFlattened))
4517 return mlir::emitSilenceableFailure(loc: target->getLoc())
4518 << "attempted to flatten, but failed";
4519 results.push_back(op: maybeFlattened->collapsedOp);
4520 rewriter.replaceOp(op: target, newValues: maybeFlattened->results);
4521 return DiagnosedSilenceableFailure::success();
4522}
4523
4524//===----------------------------------------------------------------------===//
4525// TransposeConv2DOp
4526//===----------------------------------------------------------------------===//
4527
4528DiagnosedSilenceableFailure transform::TransposeConv2DOp::applyToOne(
4529 transform::TransformRewriter &rewriter, linalg::LinalgOp target,
4530 transform::ApplyToEachResultList &results,
4531 transform::TransformState &state) {
4532 rewriter.setInsertionPoint(target);
4533 auto maybeTransformed =
4534 TypeSwitch<Operation *, FailureOr<Operation *>>(target)
4535 .Case(caseFn: [&](linalg::Conv2DNhwcFhwcOp op) {
4536 return transposeConv2D(rewriter, op);
4537 })
4538 .Case(caseFn: [&](linalg::Conv2DNhwcFhwcQOp op) {
4539 return transposeConv2D(rewriter, op);
4540 })
4541 .Default(defaultFn: [&](Operation *op) {
4542 return rewriter.notifyMatchFailure(arg&: op, msg: "not supported");
4543 });
4544 if (failed(Result: maybeTransformed))
4545 return emitDefaultSilenceableFailure(target);
4546 // Handle to the new Conv2D operation with transposed filters
4547 results.push_back(op: *maybeTransformed);
4548 return DiagnosedSilenceableFailure::success();
4549}
4550
4551//===----------------------------------------------------------------------===//
4552// TransposeMatmulOp
4553//===----------------------------------------------------------------------===//
4554
4555DiagnosedSilenceableFailure transform::TransposeMatmulOp::applyToOne(
4556 transform::TransformRewriter &rewriter, linalg::LinalgOp target,
4557 transform::ApplyToEachResultList &results,
4558 transform::TransformState &state) {
4559 rewriter.setInsertionPoint(target);
4560 bool transposeLHS = getInputToTranspose() == TransposeMatmulInput::lhs;
4561 auto maybeTransformed =
4562 TypeSwitch<Operation *, FailureOr<Operation *>>(target)
4563 .Case(caseFn: [&](linalg::MatmulOp op) {
4564 return transposeMatmul(rewriter, op, transposeLHS);
4565 })
4566 .Case(caseFn: [&](linalg::BatchMatmulOp op) {
4567 return transposeBatchMatmul(rewriter, op, transposeLHS);
4568 })
4569 .Default(result: failure());
4570 if (failed(Result: maybeTransformed))
4571 return emitSilenceableFailure(loc: target->getLoc()) << "not supported";
4572 // Handle to the new Matmul operation with transposed filters
4573 results.push_back(op: *maybeTransformed);
4574 return DiagnosedSilenceableFailure::success();
4575}
4576
4577//===----------------------------------------------------------------------===//
4578// InsertSliceToCopyOp
4579//===----------------------------------------------------------------------===//
4580template <typename OpTy>
4581static DiagnosedSilenceableFailure
4582doit(RewriterBase &rewriter, OpTy target,
4583 transform::ApplyToEachResultList &results,
4584 transform::TransformState &state) {
4585 static_assert(llvm::is_one_of<OpTy, tensor::InsertSliceOp,
4586 tensor::ParallelInsertSliceOp>() &&
4587 "wrong op type");
4588
4589 if (auto copySource =
4590 target.getSource().template getDefiningOp<linalg::CopyOp>()) {
4591 results.push_back(copySource);
4592 return DiagnosedSilenceableFailure::success();
4593 }
4594
4595 // If we are inside a `ParallelCombiningOp` region, temporarily set the
4596 // insertion point outside: only ops implementing ParallelCombiningOpInterface
4597 // are allowed in there.
4598 if (isa<mlir::ParallelCombiningOpInterface>(target.getOperation()))
4599 rewriter.setInsertionPoint(target->getParentOp());
4600
4601 Value extracted = tensor::ExtractSliceOp::create(
4602 rewriter, target.getLoc(), target.getDest(), target.getMixedOffsets(),
4603 target.getMixedSizes(), target.getMixedStrides());
4604 Value copied = linalg::CopyOp::create(rewriter, target.getLoc(),
4605 target.getSource(), extracted)
4606 .getResult(0);
4607 // Reset the insertion point.
4608 rewriter.setInsertionPoint(target);
4609 rewriter.replaceOpWithNewOp<OpTy>(
4610 target, copied, target.getDest(), target.getMixedOffsets(),
4611 target.getMixedSizes(), target.getMixedStrides());
4612
4613 results.push_back(op: copied.getDefiningOp());
4614 return DiagnosedSilenceableFailure::success();
4615}
4616
4617DiagnosedSilenceableFailure transform::InsertSliceToCopyOp::applyToOne(
4618 transform::TransformRewriter &rewriter, Operation *targetOp,
4619 transform::ApplyToEachResultList &results,
4620 transform::TransformState &state) {
4621
4622 rewriter.setInsertionPoint(targetOp);
4623 if (auto target = dyn_cast<tensor::InsertSliceOp>(Val: targetOp))
4624 return doit(rewriter, target, results, state);
4625 if (auto target = dyn_cast<tensor::ParallelInsertSliceOp>(Val: targetOp))
4626 return doit(rewriter, target, results, state);
4627
4628 DiagnosedSilenceableFailure diag =
4629 emitSilenceableError()
4630 << "only InsertSliceOp and ParallelInsertSliceOp ops are supported";
4631 diag.attachNote(loc: targetOp->getLoc()) << "target op";
4632 return diag;
4633}
4634
4635//===----------------------------------------------------------------------===//
4636// MapCopyToThreadsOp
4637//===----------------------------------------------------------------------===//
4638
4639DiagnosedSilenceableFailure transform::MapCopyToThreadsOp::applyToOne(
4640 transform::TransformRewriter &rewriter, Operation *target,
4641 transform::ApplyToEachResultList &results,
4642 transform::TransformState &state) {
4643 // Check if the op is supported.
4644 if (!isa<linalg::CopyOp, tensor::PadOp>(Val: target)) {
4645 DiagnosedSilenceableFailure diag =
4646 emitSilenceableError()
4647 << "only linalg.copy and tensor.pad target ops are supported";
4648 diag.attachNote(loc: target->getLoc()) << "target op";
4649 return diag;
4650 }
4651 assert(target->getNumResults() == 1 && "expected single result");
4652 auto resultShapedType = cast<ShapedType>(Val: target->getResult(idx: 0).getType());
4653 if (!resultShapedType.hasStaticShape()) {
4654 DiagnosedSilenceableFailure diag =
4655 emitSilenceableError()
4656 << "only statically sized ops of rank <= 3 are supported";
4657 diag.attachNote(loc: target->getLoc()) << "target op";
4658 return diag;
4659 }
4660
4661 // Conservatively set the minimum viable desired bitwidth alignment.
4662 int64_t desiredBitAlignment = getDesiredBitAlignment();
4663 int64_t eltBitwidth =
4664 resultShapedType.getElementType().getIntOrFloatBitWidth();
4665 if (desiredBitAlignment % eltBitwidth != 0) {
4666 desiredBitAlignment = eltBitwidth;
4667 }
4668
4669 gpu::CopyMappingInfo mapping(
4670 /*ctx=*/getContext(),
4671 /*totalNumThreads=*/getTotalNumThreads(),
4672 /*alignment=*/desiredBitAlignment,
4673 /*sizes=*/resultShapedType.getShape(),
4674 /*favorPredication=*/false,
4675 /*elementalBitwidth=*/
4676 resultShapedType.getElementType().getIntOrFloatBitWidth());
4677 if (mapping.status == gpu::CopyMappingInfo::Status::Invalid) {
4678 DiagnosedSilenceableFailure diag =
4679 emitSilenceableError()
4680 << "too few threads to map copy op to threads on the most minor "
4681 "dimension, given alignment and vector size constraints, try "
4682 "smaller tile size of mapping to more threads";
4683 diag.attachNote(loc: target->getLoc()) << "target op";
4684 return diag;
4685 }
4686
4687 // OpBuilder only used to compute attributes.
4688 OpBuilder b(getContext());
4689 scf::SCFTilingResult tilingResult;
4690 DiagnosedSilenceableFailure diag = tileToForallOpImpl(
4691 /*rewriter=*/rewriter,
4692 /*state=*/state,
4693 /*transformOp=*/*this,
4694 /*target=*/target,
4695 /*mixedNumThreads=*/getMixedValues(staticValues: mapping.numThreads, dynamicValues: {}, b),
4696 /*mixedTileSizes=*/ArrayRef<OpFoldResult>{},
4697 /*mapping=*/b.getArrayAttr(value: mapping.threadMapping),
4698 /*tilingResult=*/tilingResult);
4699 if (!diag.succeeded())
4700 return diag;
4701
4702 results.push_back(op: tilingResult.loops.front());
4703 for (auto *op : tilingResult.tiledOps)
4704 results.push_back(op);
4705 return DiagnosedSilenceableFailure::success();
4706}
4707
4708//===----------------------------------------------------------------------===//
4709// WinogradConv2DOp
4710//===----------------------------------------------------------------------===//
4711
4712DiagnosedSilenceableFailure transform::WinogradConv2DOp::applyToOne(
4713 transform::TransformRewriter &rewriter, linalg::LinalgOp target,
4714 transform::ApplyToEachResultList &results,
4715 transform::TransformState &state) {
4716 rewriter.setInsertionPoint(target);
4717 FailureOr<Operation *> maybeTransformed = failure();
4718 bool supported = TypeSwitch<Operation *, bool>(target)
4719 .Case(caseFn: [&](linalg::Conv2DNhwcFhwcOp op) {
4720 maybeTransformed =
4721 winogradConv2D(rewriter, op, fmr: getFmr());
4722 return true;
4723 })
4724 .Default(defaultFn: [&](Operation *op) { return false; });
4725
4726 if (!supported) {
4727 return emitSilenceableError()
4728 << "this operation is not supported to convert to Winograd Conv2D";
4729 }
4730
4731 if (failed(Result: maybeTransformed)) {
4732 return emitSilenceableError() << "apply Winograd Conv2D failed";
4733 }
4734
4735 results.push_back(op: *maybeTransformed);
4736 return DiagnosedSilenceableFailure::success();
4737}
4738
4739DiagnosedSilenceableFailure transform::DecomposeWinogradOp::applyToOne(
4740 transform::TransformRewriter &rewriter, Operation *target,
4741 transform::ApplyToEachResultList &results,
4742 transform::TransformState &state) {
4743 rewriter.setInsertionPoint(target);
4744 FailureOr<Operation *> maybeTransformed = failure();
4745 bool supported =
4746 TypeSwitch<Operation *, bool>(target)
4747 .Case(caseFn: [&](linalg::WinogradFilterTransformOp op) {
4748 maybeTransformed = decomposeWinogradFilterTransformOp(rewriter, op);
4749 return true;
4750 })
4751 .Case(caseFn: [&](linalg::WinogradInputTransformOp op) {
4752 maybeTransformed = decomposeWinogradInputTransformOp(rewriter, op);
4753 return true;
4754 })
4755 .Case(caseFn: [&](linalg::WinogradOutputTransformOp op) {
4756 maybeTransformed = decomposeWinogradOutputTransformOp(rewriter, op);
4757 return true;
4758 })
4759 .Default(defaultResult: false);
4760
4761 if (!supported) {
4762 DiagnosedSilenceableFailure diag =
4763 emitSilenceableError()
4764 << "this operation is not supported to decompose into other operations";
4765 diag.attachNote(loc: target->getLoc()) << "target op";
4766 return diag;
4767 }
4768
4769 if (failed(Result: maybeTransformed)) {
4770 DiagnosedSilenceableFailure diag =
4771 emitSilenceableError() << "decompose Winograd operations failed";
4772 diag.attachNote(loc: target->getLoc()) << "target op";
4773 return diag;
4774 }
4775
4776 results.push_back(op: *maybeTransformed);
4777 return DiagnosedSilenceableFailure::success();
4778}
4779
4780#include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOpsEnums.cpp.inc"
4781
4782#define GET_OP_CLASSES
4783#include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp.inc"
4784