1//===- TestTensorTransforms.cpp - Test Tensor transformation patterns -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements logic for testing Tensor transformations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/Arith/IR/Arith.h"
14#include "mlir/Dialect/Linalg/IR/LinalgDialect.h"
15#include "mlir/Dialect/SCF/IR/SCF.h"
16#include "mlir/Dialect/Tensor/IR/Tensor.h"
17#include "mlir/Dialect/Tensor/Transforms/TransformUtils.h"
18#include "mlir/Dialect/Tensor/Transforms/Transforms.h"
19#include "mlir/Dialect/Transform/IR/TransformOps.h"
20#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
21#include "mlir/Pass/Pass.h"
22#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
23
24using namespace mlir;
25
26namespace {
27struct TestTensorTransforms
28 : public PassWrapper<TestTensorTransforms, OperationPass<>> {
29 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestTensorTransforms)
30
31 TestTensorTransforms() = default;
32 TestTensorTransforms(const TestTensorTransforms &pass) : PassWrapper(pass) {}
33
34 void getDependentDialects(DialectRegistry &registry) const override {
35 registry.insert<arith::ArithDialect, scf::SCFDialect, linalg::LinalgDialect,
36 transform::TransformDialect>();
37 }
38
39 StringRef getArgument() const final {
40 return "test-tensor-transform-patterns";
41 }
42 StringRef getDescription() const final {
43 return "Test Tensor transformation patterns by applying them greedily.";
44 }
45
46 void runOnOperation() override;
47
48 Option<bool> testFoldConstantExtractSlice{
49 *this, "test-fold-constant-extract-slice",
50 llvm::cl::desc("Test folding arith.constant and tensor.extract_slice"),
51 llvm::cl::init(Val: false)};
52
53 Option<bool> testFoldConsecutiveInsertExtractSlice{
54 *this, "test-fold-consecutive-insert-extract-slice",
55 llvm::cl::desc(
56 "Test folding consecutive tensor.insert_slice/tensor.extract_slice"),
57 llvm::cl::init(Val: false)};
58
59 Option<bool> testRewriteExtractSliceWithTiledCollapseShape{
60 *this, "test-rewrite-extract-slice-from-collapse-shape",
61 llvm::cl::desc("Test swapping tensor.extract_slice of a collapse_shape "
62 "with loop nest"),
63 llvm::cl::init(Val: false)};
64
65 Option<bool> testDropRedundantInsertSliceRankExpansion{
66 *this, "test-drop-redundant-insert-slice-rank-expansion",
67 llvm::cl::desc("Test dropping redundant insert_slice rank expansions"),
68 llvm::cl::init(Val: false)};
69
70 Option<bool> testReassociativeReshapeFolding{
71 *this, "test-reassociative-reshape-folding",
72 llvm::cl::desc("Test folding of expand_shape/collapse_shape"),
73 llvm::cl::init(Val: false)};
74
75 Option<bool> testBubbleUpExpandShapePatterns{
76 *this, "test-expand-shape-bubbling",
77 llvm::cl::desc("Test folding of expand_shape/collapse_shape"),
78 llvm::cl::init(Val: false)};
79
80 Option<bool> testFoldExtractFromCollapseShape{
81 *this, "test-fold-extract-from-collapse-shape",
82 llvm::cl::desc("Test folding of extract from collapse_shape"),
83 llvm::cl::init(Val: false)};
84
85 Option<bool> useForeach{
86 *this, "use-foreach",
87 llvm::cl::desc(
88 "Use the scf.forall operation when generating loop nests for "
89 "the extract_slice of collapse_shape pattern"),
90 llvm::cl::init(Val: false)};
91
92 Option<bool> testTrackingListener{
93 *this, "test-tracking-listener",
94 llvm::cl::desc("Test tensor TrackingListener for the transform dialect"),
95 llvm::cl::init(Val: false)};
96};
97} // namespace
98
99static void applyReassociativeReshapeFoldingPatterns(Operation *rootOp) {
100 RewritePatternSet patterns(rootOp->getContext());
101 tensor::populateReassociativeReshapeFoldingPatterns(patterns);
102 (void)applyPatternsGreedily(op: rootOp, patterns: std::move(patterns));
103}
104
105static void applyBubbleUpExpandShapePatterns(Operation *rootOp) {
106 RewritePatternSet patterns(rootOp->getContext());
107 tensor::populateBubbleUpExpandShapePatterns(patterns);
108 (void)applyPatternsGreedily(op: rootOp, patterns: std::move(patterns));
109}
110
111static void applyFoldConstantExtractSlicePatterns(Operation *rootOp) {
112 RewritePatternSet patterns(rootOp->getContext());
113 tensor::ControlConstantExtractSliceFusionFn controlFn =
114 [](tensor::ExtractSliceOp op) {
115 if (!op.getSource().hasOneUse())
116 return false;
117
118 auto resultType = cast<ShapedType>(Val: op.getResult().getType());
119 constexpr int64_t kConstantFoldingMaxNumElements = 1024;
120 return resultType.getNumElements() <= kConstantFoldingMaxNumElements;
121 };
122
123 tensor::populateFoldConstantExtractSlicePatterns(patterns, controlFn);
124 (void)applyPatternsGreedily(op: rootOp, patterns: std::move(patterns));
125}
126
127static void applyFoldConsecutiveInsertExtractSlicePatterns(Operation *rootOp) {
128 RewritePatternSet patterns(rootOp->getContext());
129 tensor::populateMergeConsecutiveInsertExtractSlicePatterns(patterns);
130 (void)applyPatternsGreedily(op: rootOp, patterns: std::move(patterns));
131}
132
133static void
134applyDropRedundantInsertSliceRankExpansionPatterns(Operation *rootOp) {
135 RewritePatternSet patterns(rootOp->getContext());
136 tensor::populateDropRedundantInsertSliceRankExpansionPatterns(patterns);
137 (void)applyPatternsGreedily(op: rootOp, patterns: std::move(patterns));
138}
139
140static void applyFoldExtractFromCollapseShapePatterns(Operation *rootOp) {
141 RewritePatternSet patterns(rootOp->getContext());
142 tensor::populateFoldCollapseExtractPatterns(patterns);
143 (void)applyPatternsGreedily(op: rootOp, patterns: std::move(patterns));
144}
145
146namespace {
147/// Base pattern to rewrite a `tensor.collapse_shape -> tensor.extract_slice`.
148/// The `tensor.extract_slice` is replaced by a loop or gather operation that
149/// stitches together the desired tile from slices of the source of the collapse
150/// shape op.
151struct RewriteExtractSliceFromCollapseShapeBase
152 : public OpRewritePattern<tensor::ExtractSliceOp> {
153 RewriteExtractSliceFromCollapseShapeBase(MLIRContext *context)
154 : mlir::OpRewritePattern<tensor::ExtractSliceOp>(context) {}
155
156 /// Emit a loop or gather operation that uses `helper` to take each point in
157 /// the parallel iteration space bounds, extract a slice from the source
158 /// tensor and insert it into `dest`. For examples, see below for `scf.for`
159 /// and `scf.foreach`.
160 virtual LogicalResult
161 emitReplacement(tensor::ExtractSliceOp op, Value dest,
162 tensor::ExtractSliceFromCollapseHelper &helper,
163 PatternRewriter &rewriter) const = 0;
164
165 LogicalResult matchAndRewrite(tensor::ExtractSliceOp op,
166 PatternRewriter &rewriter) const override {
167 auto collapseOp = op.getSource().getDefiningOp<tensor::CollapseShapeOp>();
168 if (!collapseOp)
169 return rewriter.notifyMatchFailure(
170 arg&: op, msg: "producer is not a tensor.collapse_shape op");
171
172 // Try to simplify the collapse shape using a rank-reducing slice, if
173 // possible.
174 FailureOr<Operation *> simplifiedCollapseShapeResult =
175 tensor::simplifyCollapseShapeWithRankReducingExtractSlice(op: collapseOp,
176 rewriter);
177 if (succeeded(Result: simplifiedCollapseShapeResult)) {
178 auto newCollapseOp =
179 dyn_cast<tensor::CollapseShapeOp>(Val: *simplifiedCollapseShapeResult);
180 // The collapse shape op might have been simplified away, so we can just
181 // return.
182 if (!newCollapseOp)
183 return success();
184 collapseOp = newCollapseOp;
185 }
186
187 // Materialize the output shape values of the slice operation.
188 ReifiedRankedShapedTypeDims reifiedShapes;
189 if (failed(Result: reifyResultShapes(b&: rewriter, op, reifiedReturnShapes&: reifiedShapes)))
190 return rewriter.notifyMatchFailure(arg&: op, msg: "failed to reify result shapes");
191
192 // Create the destination tensor using the above values.
193 Type elementType = op.getSourceType().getElementType();
194 SmallVector<OpFoldResult> outputShape = reifiedShapes[0];
195 Value dest = tensor::EmptyOp::create(builder&: rewriter, location: op->getLoc(), sizes: outputShape,
196 elementType);
197
198 // Calculate the parameters for the tile loop nest.
199 FailureOr<tensor::ExtractSliceFromCollapseHelper> params =
200 tensor::ExtractSliceFromCollapseHelper::create(b&: rewriter, collapseOp,
201 extractOp: op);
202 if (failed(Result: params))
203 return rewriter.notifyMatchFailure(
204 arg&: op, msg: "could not calculate tiling parameters");
205 return emitReplacement(op, dest, helper&: *params, rewriter);
206 }
207};
208
209struct RewriteExtractSliceFromCollapseShapeUsingScfFor
210 : public RewriteExtractSliceFromCollapseShapeBase {
211 RewriteExtractSliceFromCollapseShapeUsingScfFor(MLIRContext *context)
212 : RewriteExtractSliceFromCollapseShapeBase(context) {}
213 LogicalResult emitReplacement(tensor::ExtractSliceOp op, Value dest,
214 tensor::ExtractSliceFromCollapseHelper &helper,
215 PatternRewriter &rewriter) const override {
216 Location loc = op.getLoc();
217 const unsigned numTiledDims = helper.getIterationSpaceSizes().size();
218 auto zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
219 auto one = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1);
220 SmallVector<Value> lbs(numTiledDims, zero);
221 SmallVector<Value> steps(numTiledDims, one);
222
223 scf::LoopNest nest = scf::buildLoopNest(
224 builder&: rewriter, loc, lbs, ubs: helper.getIterationSpaceSizes(), steps, iterArgs: dest,
225 bodyBuilder: [&](OpBuilder &nestedBuilder, Location loc, ValueRange outputIvs,
226 ValueRange iterArgs) -> scf::ValueVector {
227 auto [tile, insertParams] =
228 helper.emitLoopNestBody(builder&: nestedBuilder, loc, tileInductionVars: outputIvs);
229
230 // Insert the slice into the destination.
231 return {tensor::InsertSliceOp::create(builder&: nestedBuilder, location: loc, source: tile,
232 dest: iterArgs[0], ranges: insertParams)};
233 });
234 rewriter.replaceOp(op, newValues: nest.results);
235
236 return success();
237 }
238};
239
240struct RewriteExtractSliceFromCollapseShapeUsingScfForeach
241 : public RewriteExtractSliceFromCollapseShapeBase {
242 RewriteExtractSliceFromCollapseShapeUsingScfForeach(MLIRContext *context)
243 : RewriteExtractSliceFromCollapseShapeBase(context) {}
244 LogicalResult emitReplacement(tensor::ExtractSliceOp op, Value dest,
245 tensor::ExtractSliceFromCollapseHelper &helper,
246 PatternRewriter &rewriter) const override {
247 Location loc = op.getLoc();
248 auto forallOp = scf::ForallOp::create(
249 builder&: rewriter, location: loc,
250 /*numThreads=*/ubs: getAsOpFoldResult(values: helper.getIterationSpaceSizes()),
251 /*outputs=*/dest,
252 /*mapping=*/std::nullopt,
253 bodyBuilderFn: [&](OpBuilder &nestedBuilder, Location loc, ValueRange regionArgs) {
254 unsigned numThreadIdRegionArgs =
255 helper.getIterationSpaceSizes().size();
256 unsigned numOutputRegionArgs =
257 regionArgs.size() - numThreadIdRegionArgs;
258 ValueRange outputIvs = regionArgs.take_front(n: numThreadIdRegionArgs);
259 ValueRange outputArgs = regionArgs.take_back(n: numOutputRegionArgs);
260 assert(outputArgs.size() == 1 &&
261 "there should only be one output region argument");
262 auto [tile, insertParams] =
263 helper.emitLoopNestBody(builder&: nestedBuilder, loc, tileInductionVars: outputIvs);
264 // Insert the slice into the destination.
265 auto term = scf::InParallelOp::create(builder&: nestedBuilder, location: loc);
266 nestedBuilder.setInsertionPointToStart(term.getBody());
267 tensor::ParallelInsertSliceOp::create(builder&: nestedBuilder, location: loc, source: tile,
268 dest: outputArgs[0], ranges: insertParams);
269 });
270 rewriter.replaceOp(op, newValues: forallOp->getResult(idx: 0));
271 return success();
272 }
273};
274} // namespace
275
276static LogicalResult
277applyRewriteExtractFromCollapseShapePatterns(Operation *rootOp,
278 bool useForeach) {
279 RewritePatternSet patterns(rootOp->getContext());
280 if (useForeach)
281 patterns.add<RewriteExtractSliceFromCollapseShapeUsingScfForeach>(
282 arg: rootOp->getContext());
283 else
284 patterns.add<RewriteExtractSliceFromCollapseShapeUsingScfFor>(
285 arg: rootOp->getContext());
286 return applyPatternsGreedily(op: rootOp, patterns: std::move(patterns));
287}
288
289namespace {
290class DummyTrackingListener : public transform::TrackingListener {
291public:
292 using transform::TrackingListener::TrackingListener;
293
294 // Expose `findReplacementOp` as a public function, so that it can be tested.
295 Operation *getReplacementOp(Operation *op, ValueRange newValues) const {
296 Operation *replacementOp;
297 if (!findReplacementOp(result&: replacementOp, op, newValues).succeeded())
298 return nullptr;
299 return replacementOp;
300 }
301};
302} // namespace
303
304static LogicalResult testTrackingListenerReplacements(Operation *rootOp) {
305 // Find replaced op.
306 Operation *replaced = nullptr;
307 WalkResult status = rootOp->walk(callback: [&](Operation *op) {
308 if (op->hasAttr(name: "replaced")) {
309 if (replaced) {
310 op->emitError(message: "only one 'replaced' op is allowed per test case");
311 replaced->emitRemark(message: "other 'replaced' op");
312 return WalkResult::interrupt();
313 }
314 replaced = op;
315 }
316 return WalkResult::advance();
317 });
318 if (status.wasInterrupted())
319 return failure();
320 if (!replaced) {
321 rootOp->emitError(message: "could not find 'replaced' op");
322 return failure();
323 }
324
325 // Find replacements.
326 SmallVector<Value> replacements(replaced->getNumResults(), Value());
327 status = rootOp->walk(callback: [&](Operation *op) {
328 for (int64_t i = 0; i < replaced->getNumResults(); ++i) {
329 if (auto attr = op->getAttrOfType<IntegerAttr>(name: "replacement_" +
330 std::to_string(val: i))) {
331 if (replacements[i]) {
332 op->emitError(message: "only one 'replacement_" + std::to_string(val: i) +
333 "' is allowed per test case");
334 replacements[i].getDefiningOp()->emitRemark(message: "other 'replacement_" +
335 std::to_string(val: i) + "'");
336 return WalkResult::interrupt();
337 }
338 replacements[i] = op->getResult(idx: attr.getInt());
339 }
340 }
341 return WalkResult::advance();
342 });
343 if (status.wasInterrupted())
344 return failure();
345
346 if (!llvm::all_of(Range&: replacements,
347 P: [](Value v) { return static_cast<bool>(v); })) {
348 replaced->emitError(message: "insufficient replacement values");
349 return failure();
350 }
351
352 // Find the replacement op (if any) and emit a remark/error.
353 transform::TransformState transformState =
354 transform::detail::makeTransformStateForTesting(/*region=*/nullptr,
355 /*payloadRoot=*/nullptr);
356 MLIRContext *context = rootOp->getContext();
357 OpBuilder builder(context);
358 OwningOpRef<transform::NamedSequenceOp> transformOp =
359 transform::NamedSequenceOp::create(
360 builder, location: rootOp->getLoc(),
361 /*sym_name=*/"test_sequence",
362 /*function_type=*/
363 TypeAttr::get(type: FunctionType::get(context, inputs: TypeRange{}, results: TypeRange{})),
364 /*sym_visibility*/ StringAttr::get(context, bytes: "public"),
365 /*arg_attrs=*/ArrayAttr::get(context, value: ArrayRef<Attribute>()),
366 /*res_attrs=*/ArrayAttr::get(context, value: ArrayRef<Attribute>()));
367 DummyTrackingListener listener(transformState, transformOp.get());
368 Operation *replacement = listener.getReplacementOp(op: replaced, newValues: replacements);
369 if (!replacement) {
370 replaced->emitError(message: "listener could not find replacement op");
371 return failure();
372 }
373
374 replacement->emitRemark(message: "replacement found");
375 return success();
376}
377
378void TestTensorTransforms::runOnOperation() {
379 Operation *rootOp = getOperation();
380 if (testFoldConstantExtractSlice)
381 applyFoldConstantExtractSlicePatterns(rootOp);
382 if (testFoldConsecutiveInsertExtractSlice)
383 applyFoldConsecutiveInsertExtractSlicePatterns(rootOp);
384 if (testDropRedundantInsertSliceRankExpansion)
385 applyDropRedundantInsertSliceRankExpansionPatterns(rootOp);
386 if (testReassociativeReshapeFolding)
387 applyReassociativeReshapeFoldingPatterns(rootOp);
388 if (testBubbleUpExpandShapePatterns)
389 applyBubbleUpExpandShapePatterns(rootOp);
390 if (testRewriteExtractSliceWithTiledCollapseShape) {
391 if (failed(
392 Result: applyRewriteExtractFromCollapseShapePatterns(rootOp, useForeach)))
393 return signalPassFailure();
394 }
395 if (testFoldExtractFromCollapseShape)
396 applyFoldExtractFromCollapseShapePatterns(rootOp);
397 if (testTrackingListener)
398 if (failed(Result: testTrackingListenerReplacements(rootOp)))
399 return signalPassFailure();
400}
401
402namespace mlir {
403namespace test {
404void registerTestTensorTransforms() {
405 PassRegistration<TestTensorTransforms>();
406}
407} // namespace test
408} // namespace mlir
409