1//===- TestTilingInterfaceTransformOps.cpp - Test `TilingInterface` ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines transform dialect operations used for testing
10// TilingInterface
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/Affine/IR/AffineOps.h"
15#include "mlir/Dialect/Index/IR/IndexDialect.h"
16#include "mlir/Dialect/SCF/IR/SCF.h"
17#include "mlir/Dialect/SCF/Transforms/TileUsingInterface.h"
18#include "mlir/Dialect/Tensor/IR/Tensor.h"
19#include "mlir/Dialect/Transform/IR/TransformAttrs.h"
20#include "mlir/Dialect/Transform/IR/TransformDialect.h"
21#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
22#include "mlir/Dialect/Utils/StaticValueUtils.h"
23#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
24#include "mlir/IR/Dominance.h"
25#include "mlir/Interfaces/TilingInterface.h"
26#include "llvm/Support/Debug.h"
27
28#include <utility>
29
30#define DEBUG_TYPE "test-tiling-interface"
31
32#define GET_OP_CLASSES
33#include "TestTilingInterfaceTransformOps.h.inc"
34
35using namespace mlir;
36using namespace mlir::transform;
37
38//===----------------------------------------------------------------------===//
39// TestFuseAndYieldOp
40//===----------------------------------------------------------------------===//
41
42static llvm::SmallDenseSet<Operation *> collectTiledAndFusedOps(Operation *op) {
43 SmallVector<Operation *> worklist;
44 llvm::SmallDenseSet<Operation *> producers;
45 worklist.push_back(Elt: op);
46 producers.insert(op);
47 while (!worklist.empty()) {
48 Operation *current = worklist.pop_back_val();
49 for (OpOperand &operand : current->getOpOperands()) {
50 Operation *producer = operand.get().getDefiningOp();
51 if (!producer || !isa<TilingInterface>(producer) ||
52 producers.contains(V: producer))
53 continue;
54 worklist.push_back(Elt: producer);
55 producers.insert(producer);
56 }
57 }
58 return producers;
59}
60
61/// Apply a tile and fuse transformation to all payload ops and store both the
62/// tiled operation as well as the created tile loops.
63template <typename Range>
64static LogicalResult
65applyTileAndFuseToAll(RewriterBase &rewriter, Operation *transformOp,
66 Range &&payloadOps, unsigned numLoops,
67 const scf::SCFTilingOptions &tilingOptions,
68 TransformResults &transformResults) {
69 SmallVector<Operation *> tiledOps;
70 SmallVector<SmallVector<Operation *>> loopOps(numLoops);
71
72 for (Operation *target : payloadOps) {
73 auto tilingInterfaceOp = dyn_cast<TilingInterface>(Val: target);
74 if (!tilingInterfaceOp)
75 return transformOp->emitError(message: "only TilingInterface ops are supported");
76 DominanceInfo dominanceInfo(tilingInterfaceOp);
77
78 llvm::SmallDenseSet<Operation *> tiledAndFusedOps =
79 collectTiledAndFusedOps(op: tilingInterfaceOp);
80 llvm::DenseSet<Operation *> yieldReplacementsFor;
81 for (auto *op : tiledAndFusedOps) {
82 if (llvm::any_of(op->getUsers(), [&](Operation *user) {
83 return dominanceInfo.properlyDominates(a: tilingInterfaceOp, b: user);
84 })) {
85 yieldReplacementsFor.insert(V: op);
86 }
87 }
88
89 scf::SCFTileAndFuseOptions tileAndFuseOptions;
90 tileAndFuseOptions.setTilingOptions(tilingOptions);
91
92 scf::SCFTileAndFuseOptions::ControlFnTy controlFn =
93 [&](tensor::ExtractSliceOp candidateSliceOp, OpResult originalProducer,
94 bool isDestinationOperand)
95 -> std::optional<scf::SCFTileAndFuseOptions::ControlFnResult> {
96 Operation *owner = originalProducer.getOwner();
97 bool yieldProducerReplacement = yieldReplacementsFor.contains(V: owner);
98 return scf::SCFTileAndFuseOptions::ControlFnResult{
99 .yieldProducerReplacement: yieldProducerReplacement};
100 };
101 tileAndFuseOptions.setFusionControlFn(std::move(controlFn));
102
103 rewriter.setInsertionPoint(target);
104 FailureOr<scf::SCFTileAndFuseResult> tiledResults =
105 scf::tileConsumerAndFuseProducersUsingSCF(rewriter, consumer: tilingInterfaceOp,
106 options: tileAndFuseOptions);
107 if (failed(Result: tiledResults))
108 return failure();
109
110 // Perform the replacement of tiled and fused values.
111 SmallVector<Operation *> opsToReplace{target};
112 llvm::append_range(C&: opsToReplace, R&: tiledResults->fusedProducers);
113 for (Operation *toReplace : opsToReplace) {
114 for (OpResult res : toReplace->getResults())
115 if (auto replacement = tiledResults->replacements.lookup(Val: res)) {
116 Operation *replacementOp = replacement.getDefiningOp();
117 rewriter.replaceUsesWithIf(res, replacement, [&](OpOperand &use) {
118 Operation *user = use.getOwner();
119 return dominanceInfo.properlyDominates(a: replacementOp, b: user) &&
120 user->getParentOp() == replacementOp->getParentOp();
121 });
122 }
123
124 if (toReplace->use_empty()) {
125 rewriter.eraseOp(op: toReplace);
126 }
127 }
128
129 // Report back the relevant handles to the transform op.
130 tiledOps.push_back(Elt: tiledResults->tiledAndFusedOps.front());
131 assert(tiledResults->loops.size() == numLoops &&
132 "Mismatched number of loops, tile and fuse transform should have "
133 "failed");
134 for (unsigned int i = 0; i < numLoops; ++i)
135 loopOps[i].push_back(Elt: tiledResults->loops[i]);
136 }
137
138 transformResults.set(value: transformOp->getOpResult(idx: 0), ops&: tiledOps);
139 for (unsigned int i = 0; i < numLoops; ++i)
140 transformResults.set(value: transformOp->getOpResult(idx: i + 1), ops&: loopOps[i]);
141
142 return success();
143}
144
145DiagnosedSilenceableFailure
146transform::TestFuseAndYieldOp::apply(TransformRewriter &rewriter,
147 TransformResults &transformResults,
148 TransformState &state) {
149 SmallVector<int64_t> tileSizes =
150 extractFromIntegerArrayAttr<int64_t>(getTileSizes());
151 SmallVector<int64_t> tileInterchange =
152 extractFromIntegerArrayAttr<int64_t>(getTileInterchange());
153
154 SmallVector<OpFoldResult> tileSizesOfr =
155 getAsIndexOpFoldResult(rewriter.getContext(), tileSizes);
156
157 scf::SCFTilingOptions tilingOptions;
158 tilingOptions.setTileSizes(tileSizesOfr);
159 tilingOptions.interchangeVector = std::move(tileInterchange);
160 if (getUseForall()) {
161 tilingOptions.setLoopType(scf::SCFTilingOptions::LoopType::ForallOp);
162 }
163
164 LogicalResult result = applyTileAndFuseToAll(
165 rewriter, getOperation(), state.getPayloadOps(getTarget()),
166 tileSizes.size() - llvm::count(tileSizes, 0), tilingOptions,
167 transformResults);
168 return failed(result) ? DiagnosedSilenceableFailure::definiteFailure()
169 : DiagnosedSilenceableFailure::success();
170}
171
172//===----------------------------------------------------------------------===//
173// TestFuseConsumerOp
174//===----------------------------------------------------------------------===//
175
176/// Fuse the consumer and store both the original consumer operation as well as
177/// the fused consumer operation.
178static LogicalResult
179applyFuseConsumer(RewriterBase &rewriter, Operation *transformOp,
180 Operation *consumer,
181 MutableArrayRef<LoopLikeOpInterface> loops,
182 ArrayRef<InnerTileAlignment> innerTileAlignments,
183 TransformResults &transformResults) {
184 SmallVector<Operation *> fusedConsumerOps;
185 rewriter.setInsertionPoint(consumer);
186
187 // Drive the SCF helper with a constant control function returning the fixed
188 // `inner_tile_alignments` array for every op (consulted only by pack/unpack).
189 scf::InnerTileAlignmentFnTy alignmentFn =
190 [alignments = llvm::to_vector(Range&: innerTileAlignments)](
191 TilingInterface, ArrayRef<OpFoldResult>, ArrayRef<Operation *>) {
192 return alignments;
193 };
194 FailureOr<scf::SCFFuseConsumerOfSliceResult> fuseConsumerResults =
195 scf::tileAndFuseConsumer(rewriter, consumer, loops, fn: alignmentFn);
196 if (failed(Result: fuseConsumerResults))
197 return consumer->emitOpError(message: "failed to fuse consumer of slice");
198
199 // Report back the relevant handles to the transform op.
200 for (OpOperand *tiledAndFusedConsumerOperand :
201 fuseConsumerResults->tiledAndFusedConsumerOperands) {
202 fusedConsumerOps.push_back(Elt: tiledAndFusedConsumerOperand->getOwner());
203 }
204 transformResults.set(value: transformOp->getOpResult(idx: 0), ops&: fusedConsumerOps);
205 for (auto [index, loop] : llvm::enumerate(First&: loops)) {
206 transformResults.set(value: transformOp->getOpResult(idx: index + 1), ops: {loop});
207 }
208 return success();
209}
210
211LogicalResult transform::TestFuseConsumerOp::verify() {
212 return verifyInnerTileAlignments(getOperation(), getInnerTileAlignments());
213}
214
215DiagnosedSilenceableFailure
216transform::TestFuseConsumerOp::apply(TransformRewriter &rewriter,
217 TransformResults &transformResults,
218 TransformState &state) {
219 Operation *consumer = *state.getPayloadOps(getConsumer()).begin();
220
221 SmallVector<LoopLikeOpInterface> loops;
222 // Since the matcher works inside-out, we need to iterate the loops in
223 // reverse.
224 for (auto loop : llvm::reverse(getLoops())) {
225 auto loopLikeOp =
226 dyn_cast<LoopLikeOpInterface>(*state.getPayloadOps(loop).begin());
227 if (!loopLikeOp) {
228 return DiagnosedSilenceableFailure::definiteFailure();
229 }
230 loops.push_back(loopLikeOp);
231 }
232 SmallVector<InnerTileAlignment> innerTileAlignments =
233 convertInnerTileAlignments(getInnerTileAlignments());
234 LogicalResult result =
235 applyFuseConsumer(rewriter, getOperation(), consumer, loops,
236 innerTileAlignments, transformResults);
237 return failed(result) ? DiagnosedSilenceableFailure::definiteFailure()
238 : DiagnosedSilenceableFailure::success();
239}
240
241void transform::TestFuseConsumerOp::getEffects(
242 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
243 consumesHandle(getConsumerMutable(), effects);
244 consumesHandle(getLoopsMutable(), effects);
245 producesHandle(getOperation()->getOpResults(), effects);
246 modifiesPayload(effects);
247}
248
249//===----------------------------------------------------------------------===//
250// TestFuseConsumerUsingSliceOp
251//===----------------------------------------------------------------------===//
252
253/// Apply fusing of consumer transformation to all payload ops and store both
254/// the original consumer operation as well as the fused consumer operation.
255static LogicalResult applyFuseConsumerUsingSlices(
256 RewriterBase &rewriter, Operation *transformOp,
257 ArrayRef<Operation *> slices, MutableArrayRef<LoopLikeOpInterface> loops,
258 uint32_t numConsumerToFuse, TransformResults &transformResults) {
259 SmallVector<Operation *> originalConsumerOps;
260 SmallVector<Operation *> fusedConsumerOps;
261
262 rewriter.setInsertionPoint(slices.front());
263
264 while (numConsumerToFuse--) {
265 FailureOr<scf::SCFFuseConsumerOfSliceResult> fuseConsumerResults =
266 scf::tileAndFuseConsumerOfSlices(rewriter, candidateSlices: slices, loops);
267
268 if (failed(Result: fuseConsumerResults))
269 return slices.front()->emitOpError(message: "failed to fuse consumer of slice");
270
271 // Report back the relevant handles to the transform op.
272 for (OpOperand *origConsumerOperand :
273 fuseConsumerResults->origConsumerOperands) {
274 originalConsumerOps.push_back(Elt: origConsumerOperand->getOwner());
275 }
276 for (OpOperand *tiledAndFusedConsumerOperand :
277 fuseConsumerResults->tiledAndFusedConsumerOperands) {
278 fusedConsumerOps.push_back(Elt: tiledAndFusedConsumerOperand->getOwner());
279 }
280 }
281
282 transformResults.set(value: transformOp->getOpResult(idx: 0), ops&: originalConsumerOps);
283 transformResults.set(value: transformOp->getOpResult(idx: 1), ops&: fusedConsumerOps);
284 return success();
285}
286
287DiagnosedSilenceableFailure transform::TestFuseConsumerUsingSliceOp::apply(
288 TransformRewriter &rewriter, TransformResults &transformResults,
289 TransformState &state) {
290 SmallVector<Operation *> slices;
291 for (auto op : getTargets()) {
292 auto *sliceOp = *state.getPayloadOps(op).begin();
293 slices.push_back(sliceOp);
294 }
295
296 SmallVector<LoopLikeOpInterface> loops;
297 for (auto op : llvm::reverse(getLoops())) {
298 auto loopLikeOp =
299 dyn_cast<LoopLikeOpInterface>(*state.getPayloadOps(op).begin());
300 if (!loopLikeOp) {
301 return DiagnosedSilenceableFailure::definiteFailure();
302 }
303 loops.push_back(loopLikeOp);
304 }
305 LogicalResult result =
306 applyFuseConsumerUsingSlices(rewriter, getOperation(), slices, loops,
307 getNumConsumerToFuse(), transformResults);
308 return failed(result) ? DiagnosedSilenceableFailure::definiteFailure()
309 : DiagnosedSilenceableFailure::success();
310}
311
312void transform::TestFuseConsumerUsingSliceOp::getEffects(
313 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
314 consumesHandle(getTargetsMutable(), effects);
315 consumesHandle(getLoopsMutable(), effects);
316 producesHandle(getOperation()->getOpResults(), effects);
317 modifiesPayload(effects);
318}
319
320//===----------------------------------------------------------------------===//
321// TestTileUsingForallOp
322//===----------------------------------------------------------------------===//
323
324/// Apply a tiling transformation to all payload ops and store both the
325/// tiled operation as well as the created tile loops.
326template <typename Range>
327static LogicalResult
328applyTileToAll(RewriterBase &rewriter, Operation *transformOp,
329 Range &&payloadOps, ArrayRef<OpFoldResult> tileSizes,
330 ArrayRef<int64_t> interchange, std::optional<ArrayAttr> mapping,
331 TransformResults &transformResults) {
332 SmallVector<Operation *> tiledOps;
333 SmallVector<Operation *> loopOps;
334
335 for (Operation *target : payloadOps) {
336 auto tilingInterfaceOp = dyn_cast<TilingInterface>(Val: target);
337 if (!tilingInterfaceOp)
338 return transformOp->emitError(message: "only TilingInterface ops are supported");
339 scf::SCFTilingOptions tilingOptions;
340 tilingOptions.setTileSizes(tileSizes).setInterchange(interchange);
341 if (mapping) {
342 tilingOptions.setMapping(mapping.value().getValue());
343 }
344 tilingOptions.setLoopType(scf::SCFTilingOptions::LoopType::ForallOp);
345
346 rewriter.setInsertionPoint(target);
347 FailureOr<scf::SCFTilingResult> tiledResults =
348 scf::tileUsingSCF(rewriter, op: tilingInterfaceOp, options: tilingOptions);
349 if (failed(Result: tiledResults))
350 return failure();
351
352 // Perform the replacement of tiled and fused values.
353 rewriter.replaceOp(op: tilingInterfaceOp, newValues: tiledResults->replacements);
354
355 // Report back the relevant handles to the transform op.
356 tiledOps.push_back(Elt: tiledResults->tiledOps.front());
357 for (Operation *loop : tiledResults->loops)
358 loopOps.push_back(Elt: loop);
359 }
360
361 transformResults.set(value: transformOp->getOpResult(idx: 0), ops&: tiledOps);
362 for (auto [index, loop] : llvm::enumerate(First&: loopOps))
363 transformResults.set(value: transformOp->getOpResult(idx: index + 1), ops: {loop});
364
365 return success();
366}
367
368DiagnosedSilenceableFailure
369transform::TestTileUsingForallOp::apply(TransformRewriter &rewriter,
370 TransformResults &transformResults,
371 TransformState &state) {
372 SmallVector<int64_t> tileSizes =
373 extractFromIntegerArrayAttr<int64_t>(getTileSizes());
374 SmallVector<int64_t> interchange =
375 extractFromIntegerArrayAttr<int64_t>(getInterchange());
376 SmallVector<OpFoldResult> tileSizesOfr =
377 getAsIndexOpFoldResult(rewriter.getContext(), tileSizes);
378
379 LogicalResult result =
380 applyTileToAll(rewriter, getOperation(), state.getPayloadOps(getTarget()),
381 tileSizesOfr, interchange, getMapping(), transformResults);
382 return failed(result) ? DiagnosedSilenceableFailure::definiteFailure()
383 : DiagnosedSilenceableFailure::success();
384}
385
386void transform::TestTileUsingForallOp::getEffects(
387 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
388 consumesHandle(getTargetMutable(), effects);
389 producesHandle(getOperation()->getOpResults(), effects);
390 modifiesPayload(effects);
391}
392
393//===----------------------------------------------------------------------===//
394// TestFuseUsingForallOp
395//===----------------------------------------------------------------------===//
396
397/// Apply a tiling transformation to all payload ops and store both the
398/// tiled operation as well as the created tile loops.
399template <typename Range>
400static LogicalResult applyTilingToAll(
401 RewriterBase &rewriter, Operation *transformOp, Range &&payloadOps,
402 unsigned numLoops, TransformResults &transformResults,
403 function_ref<FailureOr<scf::SCFTileAndFuseResult>(TilingInterface)>
404 applyFn) {
405 SmallVector<Operation *> tiledLinalgOps;
406 SmallVector<SmallVector<Operation *>> loopOps(1);
407
408 for (Operation *target : payloadOps) {
409 auto tilingInterfaceOp = dyn_cast<TilingInterface>(Val: target);
410 if (!tilingInterfaceOp)
411 return transformOp->emitError(message: "only TilingInterface ops are supported");
412
413 rewriter.setInsertionPoint(target);
414 FailureOr<scf::SCFTileAndFuseResult> tiledResults =
415 applyFn(tilingInterfaceOp);
416 if (failed(Result: tiledResults))
417 return failure();
418
419 // Perform the replacement of tiled and fused values.
420 SmallVector<Operation *> opsToReplace{target};
421 llvm::append_range(C&: opsToReplace, R&: tiledResults->fusedProducers);
422 for (Operation *toReplace : opsToReplace) {
423 for (OpResult res : toReplace->getResults())
424 if (auto replacement = tiledResults->replacements.lookup(Val: res))
425 rewriter.replaceAllUsesWith(from: res, to: replacement);
426 if (toReplace->use_empty())
427 rewriter.eraseOp(op: toReplace);
428 }
429
430 // Report back the relevant handles to the transform op.
431 tiledLinalgOps.push_back(Elt: tiledResults->tiledAndFusedOps.front());
432 assert(tiledResults->loops.size() == 1 &&
433 cast<scf::ForallOp>(tiledResults->loops[0]).getRank() == numLoops &&
434 "Mismatched number of loops, tile and fuse transform should have "
435 "failed");
436 loopOps[0] = {tiledResults->loops[0]};
437 }
438
439 transformResults.set(value: transformOp->getOpResult(idx: 0), ops&: tiledLinalgOps);
440 if (!loopOps.empty())
441 transformResults.set(value: transformOp->getOpResult(idx: 1), ops&: loopOps[0]);
442
443 return success();
444}
445
446DiagnosedSilenceableFailure
447transform::TestFuseUsingForallOp::apply(TransformRewriter &rewriter,
448 TransformResults &transformResults,
449 TransformState &state) {
450 SmallVector<int64_t> tileSizes =
451 extractFromIntegerArrayAttr<int64_t>(getTileSizes());
452 SmallVector<int64_t> tileInterchange =
453 extractFromIntegerArrayAttr<int64_t>(getInterchange());
454
455 scf::SCFTilingOptions tilingOptions;
456 tilingOptions.interchangeVector = std::move(tileInterchange);
457 SmallVector<OpFoldResult> tileSizesOfr =
458 getAsIndexOpFoldResult(rewriter.getContext(), tileSizes);
459 tilingOptions.setTileSizes(tileSizesOfr);
460 tilingOptions.setLoopType(scf::SCFTilingOptions::LoopType::ForallOp);
461 scf::SCFTileAndFuseOptions tileAndFuseOptions;
462 tileAndFuseOptions.tilingOptions = std::move(tilingOptions);
463 LogicalResult result = applyTilingToAll(
464 rewriter, getOperation(), state.getPayloadOps(getRootOp()),
465 tileSizes.size() - llvm::count(tileSizes, 0), transformResults,
466 [&](TilingInterface tilingInterfaceOp)
467 -> FailureOr<scf::SCFTileAndFuseResult> {
468 return tileConsumerAndFuseProducersUsingSCF(rewriter, tilingInterfaceOp,
469 tileAndFuseOptions);
470 });
471 return failed(result) ? DiagnosedSilenceableFailure::definiteFailure()
472 : DiagnosedSilenceableFailure::success();
473}
474
475void transform::TestFuseUsingForallOp::getEffects(
476 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
477 consumesHandle(getRootOpMutable(), effects);
478 producesHandle(getOperation()->getOpResults(), effects);
479 modifiesPayload(effects);
480}
481
482//===----------------------------------------------------------------------===//
483// TestTileAndFuseOuterParallelPartialReduction
484//===----------------------------------------------------------------------===//
485
486DiagnosedSilenceableFailure
487transform::TestTileAndFuseOuterParallelPartialReductionOp::apply(
488 TransformRewriter &rewriter, TransformResults &transformResults,
489 TransformState &state) {
490 auto target =
491 dyn_cast<TilingInterface>(*state.getPayloadOps(getRootOp()).begin());
492 if (!target) {
493 emitOpError("expected root operation to implement `TilingInterface`");
494 return DiagnosedSilenceableFailure::definiteFailure();
495 }
496
497 SmallVector<unsigned> reductionDims =
498 extractFromIntegerArrayAttr<unsigned>(getReductionDims());
499 if (reductionDims.empty()) {
500 for (auto [index, iterator] :
501 llvm::enumerate(target.getLoopIteratorTypes()))
502 if (iterator == utils::IteratorType::reduction)
503 reductionDims.push_back(index);
504 }
505
506 if (reductionDims.empty()) {
507 emitOpError(
508 "no reduction dimension specified or found in the target operation");
509 return DiagnosedSilenceableFailure::definiteFailure();
510 }
511
512 SmallVector<int64_t> reductionTileSizes =
513 extractFromIntegerArrayAttr<int64_t>(getTileSizes());
514 if (reductionTileSizes.size() != reductionDims.size()) {
515 emitOpError(
516 "missing tile sizes for reduction dimensions that are to be tiled");
517 return DiagnosedSilenceableFailure::definiteFailure();
518 }
519
520 // Adjust tile sizes so that it corresponds to the reduction iterator types.
521 SmallVector<OpFoldResult> tileSizes;
522 int reductionTileSizeNum = 0;
523 OpFoldResult zero = rewriter.getIndexAttr(0);
524 for (auto iterator : target.getLoopIteratorTypes()) {
525 if (iterator == utils::IteratorType::parallel) {
526 tileSizes.push_back(zero);
527 continue;
528 }
529 tileSizes.push_back(
530 rewriter.getIndexAttr(reductionTileSizes[reductionTileSizeNum++]));
531 }
532
533 scf::SCFTilingOptions tilingOptions;
534 tilingOptions.setTileSizes(tileSizes)
535 .setLoopType(scf::SCFTilingOptions::LoopType::ForallOp)
536 .setReductionTilingStrategy(
537 ReductionTilingStrategy::PartialReductionOuterParallel)
538 .setReductionDims(reductionDims);
539 if (auto mapping = getMapping()) {
540 tilingOptions.setMapping(getMapping().value());
541 }
542
543 LogicalResult result = applyTileAndFuseToAll(
544 rewriter, getOperation(), state.getPayloadOps(getRootOp()),
545 /*numLoops =*/1, tilingOptions, transformResults);
546
547 return failed(result) ? DiagnosedSilenceableFailure::definiteFailure()
548 : DiagnosedSilenceableFailure::success();
549}
550
551//===----------------------------------------------------------------------===//
552// TestTileAndFuseOuterParallelPartialReduction
553//===----------------------------------------------------------------------===//
554
555DiagnosedSilenceableFailure transform::TestTileUsingCustomLoopOp::apply(
556 TransformRewriter &transformRewriter, TransformResults &transformResults,
557 TransformState &state) {
558 auto target =
559 dyn_cast<TilingInterface>(*state.getPayloadOps(getRootOp()).begin());
560 if (!target) {
561 emitOpError("expected root operation to implement `TilingInterface`");
562 return DiagnosedSilenceableFailure::definiteFailure();
563 }
564
565 OpFoldResult oneOfr = transformRewriter.getIndexAttr(1);
566
567 scf::SCFTilingOptions::GenerateLoopHeaderFn loopHeaderFn =
568 [&](RewriterBase &rewriter, Location loc, ArrayRef<Range> loopRanges,
569 ArrayRef<OpFoldResult> givenTileSizes,
570 ValueRange outerDestinationTensors)
571 -> FailureOr<scf::SCFTilingOptions::CustomLoopHeaderInfo> {
572 // Check that the strides are all 1 (to make it easier in the test).
573 if (llvm::any_of(loopRanges, [](Range r) {
574 return !isConstantIntValue(r.stride, 1);
575 })) {
576 return emitOpError("unable to handle loop ranges with strides != 1");
577 }
578 // Check number of tile sizes is equal to loop dimensions.
579 if (loopRanges.size() != givenTileSizes.size()) {
580 return emitOpError("expected number of tile sizes to be same as the "
581 "number of loops in the operation");
582 }
583 // For testing disallow any of the tile sizes being 0.
584 if (llvm::any_of(givenTileSizes, isZeroInteger)) {
585 return emitOpError("unhandled case of zero tile size");
586 }
587 // For testing, only handle tensor tiling.
588 if (outerDestinationTensors.empty()) {
589 return emitOpError("expected destination tensors");
590 }
591
592 // Compute the number of iterations for each of the loops.
593 AffineExpr s0, s1, s2;
594 bindSymbols(rewriter.getContext(), s0, s1, s2);
595 AffineExpr numItersExpr = (s1 - s0).ceilDiv(s2); // (ub - lb) / tileSize
596
597 SmallVector<OpFoldResult> allNumIters;
598 allNumIters.reserve(loopRanges.size());
599 for (auto [loopRange, tileSize] :
600 llvm::zip_equal(loopRanges, givenTileSizes)) {
601 OpFoldResult numIters = affine::makeComposedFoldedAffineApply(
602 rewriter, loc, numItersExpr,
603 {loopRange.offset, loopRange.size, tileSize});
604 allNumIters.push_back(numIters);
605 }
606 if (allNumIters.empty()) {
607 return emitOpError("invalid empty tile sizes and loop ranges");
608 }
609
610 AffineExpr mulExpr = s0 * s1;
611 OpFoldResult cumulative = oneOfr;
612 for (auto numIters : allNumIters) {
613 cumulative = affine::makeComposedFoldedAffineApply(
614 rewriter, loc, mulExpr, {cumulative, numIters});
615 }
616
617 Value zeroVal = arith::ConstantIndexOp::create(rewriter, loc, 0);
618 Value oneVal = arith::ConstantIndexOp::create(rewriter, loc, 1);
619 Value ub = getValueOrCreateConstantIndexOp(rewriter, loc, cumulative);
620
621 SmallVector<OpFoldResult> offsets;
622 SmallVector<OpFoldResult> sizes;
623 SmallVector<Value> innerDestinationTensors;
624 offsets.reserve(loopRanges.size());
625 sizes.reserve(loopRanges.size());
626
627 AffineExpr d0;
628 bindDims(rewriter.getContext(), d0);
629 AffineExpr offsetExpr = s0 + d0 * s1; // lb + iv * tileSize
630 AffineMap minMap =
631 AffineMap::get(1, 2, {s0 - d0, s1},
632 rewriter.getContext()); // min(ub - offset, tileSize)
633 auto forOp = scf::ForOp::create(
634 rewriter, loc, zeroVal, ub, oneVal, outerDestinationTensors,
635 [&](OpBuilder &b, Location bodyLoc, Value linearizedIv,
636 ValueRange destinations) {
637 auto delinearizeOp = affine::AffineDelinearizeIndexOp::create(
638 b, bodyLoc, linearizedIv, allNumIters);
639 for (auto [normalizedIv, range, tileSize] : llvm::zip_equal(
640 delinearizeOp.getResults(), loopRanges, givenTileSizes)) {
641
642 OpFoldResult normalizedIvOfr = getAsOpFoldResult(normalizedIv);
643 OpFoldResult offset = affine::makeComposedFoldedAffineApply(
644 b, bodyLoc, offsetExpr,
645 {normalizedIvOfr, range.offset, tileSize});
646 offsets.push_back(offset);
647
648 OpFoldResult size = affine::makeComposedFoldedAffineMin(
649 b, bodyLoc, minMap, {offset, range.size, tileSize});
650 sizes.push_back(size);
651 }
652 innerDestinationTensors = llvm::to_vector(destinations);
653 });
654 rewriter.setInsertionPointToEnd(forOp.getBody());
655 return scf::SCFTilingOptions::CustomLoopHeaderInfo{
656 {cast<LoopLikeOpInterface>(forOp.getOperation())},
657 std::move(offsets),
658 std::move(sizes),
659 std::move(innerDestinationTensors)};
660 };
661
662 scf::SCFTilingOptions::GenerateLoopTerminatorFn terminatorFn =
663 [&](RewriterBase &rewriter, Location loc,
664 ArrayRef<LoopLikeOpInterface> loops, ValueRange tiledResults,
665 ArrayRef<SmallVector<OpFoldResult>> resultOffsets,
666 ArrayRef<SmallVector<OpFoldResult>> resultSizes,
667 ValueRange destinationTensors) -> LogicalResult {
668 SmallVector<Value> yieldValues;
669 yieldValues.reserve(destinationTensors.size());
670 for (auto [tiledResult, offsets, sizes, destination] : llvm::zip_equal(
671 tiledResults, resultOffsets, resultSizes, destinationTensors)) {
672 SmallVector<OpFoldResult> strides(offsets.size(), oneOfr);
673 Value insertedVal = tensor::InsertSliceOp::create(
674 rewriter, loc, tiledResult, destination, offsets, sizes, strides);
675 yieldValues.push_back(insertedVal);
676 }
677 scf::YieldOp::create(rewriter, loc, yieldValues);
678 return success();
679 };
680
681 scf::SCFTilingOptions tilingOptions;
682 SmallVector<int64_t> staticTileSizes =
683 extractFromIntegerArrayAttr<int64_t>(getTileSizes());
684 SmallVector<OpFoldResult> tileSizes =
685 getAsIndexOpFoldResult(transformRewriter.getContext(), staticTileSizes);
686 tilingOptions.setTileSizes(tileSizes)
687 .setLoopType(scf::SCFTilingOptions::LoopType::CustomOp)
688 .setCustomLoopGenerationFns(std::move(loopHeaderFn),
689 std::move(terminatorFn));
690
691 OpBuilder::InsertionGuard g(transformRewriter);
692 transformRewriter.setInsertionPoint(target);
693 FailureOr<scf::SCFTilingResult> tiledResults =
694 scf::tileUsingSCF(transformRewriter, target, tilingOptions);
695 if (failed(tiledResults)) {
696 return DiagnosedSilenceableFailure::definiteFailure();
697 }
698 transformRewriter.replaceOp(target, tiledResults->replacements);
699 transformResults.set(getOperation()->getResult(0), tiledResults->tiledOps);
700 transformResults.set(getOperation()->getResult(1), tiledResults->loops);
701
702 return DiagnosedSilenceableFailure::success();
703}
704
705//===----------------------------------------------------------------------===//
706// TestQueryProducerFusability
707//===----------------------------------------------------------------------===//
708
709DiagnosedSilenceableFailure transform::TestQueryProducerFusability::apply(
710 TransformRewriter &rewriter, TransformResults &transformResults,
711 TransformState &state) {
712 for (Operation *target : state.getPayloadOps(getTarget())) {
713 auto tilingInterfaceOp = dyn_cast<TilingInterface>(target);
714 if (!tilingInterfaceOp) {
715 return emitSilenceableError()
716 << "target operation does not implement TilingInterface";
717 }
718
719 // Collect operand numbers and their corresponding producer insert_slice
720 // offsets and sizes.
721 SmallVector<unsigned> operandNumbers;
722 SmallVector<SmallVector<OpFoldResult>> allOffsets;
723 SmallVector<SmallVector<OpFoldResult>> allSizes;
724
725 for (OpOperand &operand : target->getOpOperands()) {
726 Value operandValue = operand.get();
727 Operation *definingOp = operandValue.getDefiningOp();
728
729 // Look for a producer tensor.insert_slice. This is only for testing
730 // purposes and otherwise is not a useful transformation.
731 if (auto insertSliceOp =
732 dyn_cast_or_null<tensor::InsertSliceOp>(definingOp)) {
733 operandNumbers.push_back(operand.getOperandNumber());
734 allOffsets.push_back(insertSliceOp.getMixedOffsets());
735 allSizes.push_back(insertSliceOp.getMixedSizes());
736 }
737 }
738
739 if (!operandNumbers.empty()) {
740 bool isFusable = tilingInterfaceOp.isOpFusableWithProducerSlices(
741 operandNumbers, allOffsets, allSizes);
742
743 if (isFusable) {
744 target->emitRemark()
745 << "can be fused with producer tensor.insert_slice ops";
746 } else {
747 target->emitRemark()
748 << "cannot be fused with producer tensor.insert_slice ops";
749 }
750 }
751 }
752
753 return DiagnosedSilenceableFailure::success();
754}
755
756void transform::TestQueryProducerFusability::getEffects(
757 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
758 onlyReadsHandle(getTargetMutable(), effects);
759 onlyReadsPayload(effects);
760}
761
762//===----------------------------------------------------------------------===//
763// TestQueryConsumerFusability
764//===----------------------------------------------------------------------===//
765
766DiagnosedSilenceableFailure transform::TestQueryConsumerFusability::apply(
767 TransformRewriter &rewriter, TransformResults &transformResults,
768 TransformState &state) {
769 for (Operation *target : state.getPayloadOps(getTarget())) {
770 auto tilingInterfaceOp = dyn_cast<TilingInterface>(target);
771 if (!tilingInterfaceOp) {
772 return emitSilenceableError()
773 << "target operation does not implement TilingInterface";
774 }
775
776 // Look for tensor.extract_slice ops that consume results of the tilable op.
777 for (OpResult result : target->getResults()) {
778 for (OpOperand &use : result.getUses()) {
779 Operation *user = use.getOwner();
780
781 // Look for a consumer tensor.extract_slice. This is only for testing
782 // purposes and otherwise is not a useful transformation.
783 if (auto extractSliceOp = dyn_cast<tensor::ExtractSliceOp>(user)) {
784 bool isFusable = tilingInterfaceOp.isOpFusableWithConsumerSlice(
785 result.getResultNumber(), extractSliceOp.getMixedOffsets(),
786 extractSliceOp.getMixedSizes());
787
788 if (isFusable) {
789 target->emitRemark()
790 << "can be fused with consumer tensor.extract_slice op";
791 } else {
792 target->emitRemark()
793 << "cannot be fused with consumer tensor.extract_slice op";
794 }
795 }
796 }
797 }
798 }
799
800 return DiagnosedSilenceableFailure::success();
801}
802
803void transform::TestQueryConsumerFusability::getEffects(
804 SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
805 onlyReadsHandle(getTargetMutable(), effects);
806 onlyReadsPayload(effects);
807}
808
809#define GET_OP_CLASSES
810#include "TestTilingInterfaceTransformOps.cpp.inc"
811
812namespace {
813class TestTilingInterfaceDialectExtension
814 : public transform::TransformDialectExtension<
815 TestTilingInterfaceDialectExtension> {
816public:
817 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
818 TestTilingInterfaceDialectExtension)
819
820 using Base::Base;
821
822 void init() {
823 declareDependentDialect<affine::AffineDialect>();
824 declareDependentDialect<index::IndexDialect>();
825 declareDependentDialect<scf::SCFDialect>();
826 declareDependentDialect<tensor::TensorDialect>();
827
828 registerTransformOps<
829#define GET_OP_LIST
830#include "TestTilingInterfaceTransformOps.cpp.inc"
831 >();
832 }
833};
834} // namespace
835
836namespace test {
837void registerTestTilingInterfaceTransformDialectExtension(
838 DialectRegistry &registry) {
839 registry.addExtensions<TestTilingInterfaceDialectExtension>();
840}
841} // namespace test
842