1//===- TestPatterns.cpp - Test dialect pattern driver ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "../../IR/DumpNotifications.h"
10#include "TestDialect.h"
11#include "TestOps.h"
12#include "TestTypes.h"
13#include "mlir/Dialect/Arith/IR/Arith.h"
14#include "mlir/Dialect/CommonFolders.h"
15#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
16#include "mlir/Dialect/ControlFlow/Transforms/StructuralTypeConversions.h"
17#include "mlir/Dialect/Func/IR/FuncOps.h"
18#include "mlir/Dialect/Func/Transforms/FuncConversions.h"
19#include "mlir/Dialect/SCF/Transforms/Patterns.h"
20#include "mlir/Dialect/Tensor/IR/Tensor.h"
21#include "mlir/IR/BuiltinAttributes.h"
22#include "mlir/IR/Matchers.h"
23#include "mlir/IR/PatternMatch.h"
24#include "mlir/IR/Visitors.h"
25#include "mlir/Pass/Pass.h"
26#include "mlir/Transforms/DialectConversion.h"
27#include "mlir/Transforms/FoldUtils.h"
28#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
29#include "mlir/Transforms/WalkPatternRewriteDriver.h"
30#include "llvm/ADT/ScopeExit.h"
31#include <cstdint>
32
33using namespace mlir;
34using namespace test;
35
36// Native function for testing NativeCodeCall
37static Value chooseOperand(Value input1, Value input2, BoolAttr choice) {
38 return choice.getValue() ? input1 : input2;
39}
40
41static void createOpI(PatternRewriter &rewriter, Location loc, Value input) {
42 OpI::create(rewriter, loc, input);
43}
44
45static void handleNoResultOp(PatternRewriter &rewriter,
46 OpSymbolBindingNoResult op) {
47 // Turn the no result op to a one-result op.
48 OpSymbolBindingB::create(rewriter, op.getLoc(), op.getOperand().getType(),
49 op.getOperand());
50}
51
52static bool getFirstI32Result(Operation *op, Value &value) {
53 if (!Type(op->getResult(idx: 0).getType()).isSignlessInteger(width: 32))
54 return false;
55 value = op->getResult(idx: 0);
56 return true;
57}
58
59static Value bindNativeCodeCallResult(Value value) { return value; }
60
61static SmallVector<Value, 2> bindMultipleNativeCodeCallResult(Value input1,
62 Value input2) {
63 return SmallVector<Value, 2>({input2, input1});
64}
65
66// Test that natives calls are only called once during rewrites.
67// OpM_Test will return Pi, increased by 1 for each subsequent calls.
68// This let us check the number of times OpM_Test was called by inspecting
69// the returned value in the MLIR output.
70static int64_t opMIncreasingValue = 314159265;
71static Attribute opMTest(PatternRewriter &rewriter, Value val) {
72 int64_t i = opMIncreasingValue++;
73 return rewriter.getIntegerAttr(type: rewriter.getIntegerType(width: 32), value: i);
74}
75
76static bool assertBinOpEqualArgsAndReturnTrue(Value v) {
77 Operation *operation = v.getDefiningOp();
78 if (operation->getOperand(idx: 0) != operation->getOperand(idx: 1)) {
79 // Name binding equality check must happen before user-defined constraints,
80 // thus this must not be triggered.
81 llvm::report_fatal_error(reason: "Arguments are not equal");
82 }
83 return true;
84}
85
86namespace {
87#include "TestPatterns.inc"
88} // namespace
89
90//===----------------------------------------------------------------------===//
91// Test Reduce Pattern Interface
92//===----------------------------------------------------------------------===//
93
94void test::populateTestReductionPatterns(RewritePatternSet &patterns) {
95 populateWithGenerated(patterns);
96}
97
98//===----------------------------------------------------------------------===//
99// Canonicalizer Driver.
100//===----------------------------------------------------------------------===//
101
102namespace {
103struct FoldingPattern : public RewritePattern {
104public:
105 FoldingPattern(MLIRContext *context)
106 : RewritePattern(TestOpInPlaceFoldAnchor::getOperationName(),
107 /*benefit=*/1, context) {}
108
109 LogicalResult matchAndRewrite(Operation *op,
110 PatternRewriter &rewriter) const override {
111 // Exercise createOrFold API for a single-result operation that is folded
112 // upon construction. The operation being created has an in-place folder,
113 // and it should be still present in the output. Furthermore, the folder
114 // should not crash when attempting to recover the (unchanged) operation
115 // result.
116 Value result = rewriter.createOrFold<TestOpInPlaceFold>(
117 op->getLoc(), rewriter.getIntegerType(32), op->getOperand(0),
118 IntegerAttr{});
119 assert(result);
120 rewriter.replaceOp(op, newValues: result);
121 return success();
122 }
123};
124
125/// This pattern creates a foldable operation at the entry point of the block.
126/// This tests the situation where the operation folder will need to replace an
127/// operation with a previously created constant that does not initially
128/// dominate the operation to replace.
129struct FolderInsertBeforePreviouslyFoldedConstantPattern
130 : public OpRewritePattern<TestCastOp> {
131public:
132 static_assert(std::is_same_v<Base, OpRewritePattern<TestCastOp>>);
133 using Base::Base;
134
135 LogicalResult matchAndRewrite(TestCastOp op,
136 PatternRewriter &rewriter) const override {
137 if (!op->hasDiscardableAttr("test_fold_before_previously_folded_op"))
138 return failure();
139 rewriter.setInsertionPointToStart(op->getBlock());
140
141 auto constOp = arith::ConstantOp::create(rewriter, op.getLoc(),
142 rewriter.getBoolAttr(value: true));
143 rewriter.replaceOpWithNewOp<TestCastOp>(op, rewriter.getI32Type(),
144 Value(constOp));
145 return success();
146 }
147};
148
149/// This pattern matches test.op_commutative2 with the first operand being
150/// another test.op_commutative2 with a constant on the right side and fold it
151/// away by propagating it as its result. This is intend to check that patterns
152/// are applied after the commutative property moves constant to the right.
153struct FolderCommutativeOp2WithConstant
154 : public OpRewritePattern<TestCommutative2Op> {
155public:
156 using OpRewritePattern<TestCommutative2Op>::OpRewritePattern;
157
158 LogicalResult matchAndRewrite(TestCommutative2Op op,
159 PatternRewriter &rewriter) const override {
160 auto operand = op->getOperand(0).getDefiningOp<TestCommutative2Op>();
161 if (!operand)
162 return failure();
163 Attribute constInput;
164 if (!matchPattern(operand->getOperand(1), m_Constant(bind_value: &constInput)))
165 return failure();
166 rewriter.replaceOp(op, operand->getOperand(1));
167 return success();
168 }
169};
170
171/// This pattern matches test.any_attr_of_i32_str ops. In case of an integer
172/// attribute with value smaller than MaxVal, it increments the value by 1.
173template <int MaxVal>
174struct IncrementIntAttribute : public OpRewritePattern<AnyAttrOfOp> {
175 using OpRewritePattern<AnyAttrOfOp>::OpRewritePattern;
176
177 LogicalResult matchAndRewrite(AnyAttrOfOp op,
178 PatternRewriter &rewriter) const override {
179 auto intAttr = dyn_cast<IntegerAttr>(op.getAttr());
180 if (!intAttr)
181 return failure();
182 int64_t val = intAttr.getInt();
183 if (val >= MaxVal)
184 return failure();
185 rewriter.modifyOpInPlace(
186 op, [&]() { op.setAttrAttr(rewriter.getI32IntegerAttr(value: val + 1)); });
187 return success();
188 }
189};
190
191/// This patterns adds an "eligible" attribute to "foo.maybe_eligible_op".
192struct MakeOpEligible : public RewritePattern {
193 MakeOpEligible(MLIRContext *context)
194 : RewritePattern("foo.maybe_eligible_op", /*benefit=*/1, context) {}
195
196 LogicalResult matchAndRewrite(Operation *op,
197 PatternRewriter &rewriter) const override {
198 if (op->hasDiscardableAttr(name: "eligible"))
199 return failure();
200 rewriter.modifyOpInPlace(root: op, callable: [&]() {
201 op->setDiscardableAttr(name: "eligible", value: rewriter.getUnitAttr());
202 });
203 return success();
204 }
205};
206
207/// This pattern hoists eligible ops out of a "test.one_region_op".
208struct HoistEligibleOps : public OpRewritePattern<test::OneRegionOp> {
209 using OpRewritePattern<test::OneRegionOp>::OpRewritePattern;
210
211 LogicalResult matchAndRewrite(test::OneRegionOp op,
212 PatternRewriter &rewriter) const override {
213 Operation *terminator = op.getRegion().front().getTerminator();
214 Operation *toBeHoisted = terminator->getOperands()[0].getDefiningOp();
215 if (toBeHoisted->getParentOp() != op)
216 return failure();
217 if (!toBeHoisted->hasDiscardableAttr(name: "eligible"))
218 return failure();
219 rewriter.moveOpBefore(toBeHoisted, op);
220 return success();
221 }
222};
223
224struct FoldSignOpF32ToSI32 : public OpRewritePattern<test::SignOp> {
225 using OpRewritePattern<test::SignOp>::OpRewritePattern;
226
227 LogicalResult matchAndRewrite(test::SignOp op,
228 PatternRewriter &rewriter) const override {
229 if (op->getNumOperands() != 1 || op->getNumResults() != 1)
230 return failure();
231
232 TypedAttr operandAttr;
233 matchPattern(op->getOperand(0), m_Constant(bind_value: &operandAttr));
234 if (!operandAttr)
235 return failure();
236
237 TypedAttr res = cast_or_null<TypedAttr>(
238 constFoldUnaryOp<FloatAttr, FloatAttr::ValueType, void, IntegerAttr>(
239 operandAttr, op.getType(), [](APFloat operand) -> APSInt {
240 static const APFloat zero(0.0f);
241 int operandSign = 0;
242 if (operand != zero)
243 operandSign = (operand < zero) ? -1 : +1;
244 return APSInt(APInt(32, operandSign), false);
245 }));
246 if (!res)
247 return failure();
248
249 rewriter.replaceOpWithNewOp<arith::ConstantOp>(op, res);
250 return success();
251 }
252};
253
254struct FoldLessThanOpF32ToI1 : public OpRewritePattern<test::LessThanOp> {
255 using OpRewritePattern<test::LessThanOp>::OpRewritePattern;
256
257 LogicalResult matchAndRewrite(test::LessThanOp op,
258 PatternRewriter &rewriter) const override {
259 if (op->getNumOperands() != 2 || op->getNumResults() != 1)
260 return failure();
261
262 TypedAttr lhsAttr;
263 TypedAttr rhsAttr;
264 matchPattern(op->getOperand(0), m_Constant(bind_value: &lhsAttr));
265 matchPattern(op->getOperand(1), m_Constant(bind_value: &rhsAttr));
266
267 if (!lhsAttr || !rhsAttr)
268 return failure();
269
270 Attribute operandAttrs[2] = {lhsAttr, rhsAttr};
271 TypedAttr res = cast_or_null<TypedAttr>(
272 constFoldBinaryOp<FloatAttr, FloatAttr, FloatAttr::ValueType,
273 FloatAttr::ValueType, void, IntegerAttr>(
274 operandAttrs, op.getType(), [](APFloat lhs, APFloat rhs) -> APInt {
275 return APInt(1, lhs < rhs);
276 }));
277 if (!res)
278 return failure();
279
280 rewriter.replaceOpWithNewOp<arith::ConstantOp>(op, res);
281 return success();
282 }
283};
284
285/// This pattern moves "test.move_before_parent_op" before the parent op.
286struct MoveBeforeParentOp : public RewritePattern {
287 MoveBeforeParentOp(MLIRContext *context)
288 : RewritePattern("test.move_before_parent_op", /*benefit=*/1, context) {}
289
290 LogicalResult matchAndRewrite(Operation *op,
291 PatternRewriter &rewriter) const override {
292 // Do not hoist past functions.
293 if (isa<FunctionOpInterface>(Val: op->getParentOp()))
294 return failure();
295 rewriter.moveOpBefore(op, existingOp: op->getParentOp());
296 return success();
297 }
298};
299
300/// This pattern moves "test.move_after_parent_op" after the parent op.
301struct MoveAfterParentOp : public RewritePattern {
302 MoveAfterParentOp(MLIRContext *context)
303 : RewritePattern("test.move_after_parent_op", /*benefit=*/1, context) {}
304
305 LogicalResult matchAndRewrite(Operation *op,
306 PatternRewriter &rewriter) const override {
307 // Do not hoist past functions.
308 if (isa<FunctionOpInterface>(Val: op->getParentOp()))
309 return failure();
310
311 int64_t moveForwardBy = 0;
312 if (auto advanceBy = op->getDiscardableAttrOfType<IntegerAttr>(name: "advance"))
313 moveForwardBy = advanceBy.getInt();
314
315 Operation *moveAfter = op->getParentOp();
316 for (int64_t i = 0; i < moveForwardBy; ++i)
317 moveAfter = moveAfter->getNextNode();
318
319 rewriter.moveOpAfter(op, existingOp: moveAfter);
320 return success();
321 }
322};
323
324/// This pattern inlines blocks that are nested in
325/// "test.inline_blocks_into_parent" into the parent block.
326struct InlineBlocksIntoParent : public RewritePattern {
327 InlineBlocksIntoParent(MLIRContext *context)
328 : RewritePattern("test.inline_blocks_into_parent", /*benefit=*/1,
329 context) {}
330
331 LogicalResult matchAndRewrite(Operation *op,
332 PatternRewriter &rewriter) const override {
333 bool changed = false;
334 for (Region &r : op->getRegions()) {
335 while (!r.empty()) {
336 rewriter.inlineBlockBefore(source: &r.front(), op);
337 changed = true;
338 }
339 }
340 return success(IsSuccess: changed);
341 }
342};
343
344/// This pattern splits blocks at "test.split_block_here" and replaces the op
345/// with a new op (to prevent an infinite loop of block splitting).
346struct SplitBlockHere : public RewritePattern {
347 SplitBlockHere(MLIRContext *context)
348 : RewritePattern("test.split_block_here", /*benefit=*/1, context) {}
349
350 LogicalResult matchAndRewrite(Operation *op,
351 PatternRewriter &rewriter) const override {
352 rewriter.splitBlock(block: op->getBlock(), before: op->getIterator());
353 Operation *newOp = rewriter.create(
354 loc: op->getLoc(),
355 opName: OperationName("test.new_op", op->getContext()).getIdentifier(),
356 operands: op->getOperands(), types: op->getResultTypes());
357 rewriter.replaceOp(op, newOp);
358 return success();
359 }
360};
361
362/// This pattern clones "test.clone_me" ops.
363struct CloneOp : public RewritePattern {
364 CloneOp(MLIRContext *context)
365 : RewritePattern("test.clone_me", /*benefit=*/1, context) {}
366
367 LogicalResult matchAndRewrite(Operation *op,
368 PatternRewriter &rewriter) const override {
369 // Do not clone already cloned ops to avoid going into an infinite loop.
370 if (op->hasDiscardableAttr(name: "was_cloned"))
371 return failure();
372 Operation *cloned = rewriter.clone(op&: *op);
373 cloned->setDiscardableAttr(name: "was_cloned", value: rewriter.getUnitAttr());
374 return success();
375 }
376};
377
378/// This pattern clones regions of "test.clone_region_before" ops before the
379/// parent block.
380struct CloneRegionBeforeOp : public RewritePattern {
381 CloneRegionBeforeOp(MLIRContext *context)
382 : RewritePattern("test.clone_region_before", /*benefit=*/1, context) {}
383
384 LogicalResult matchAndRewrite(Operation *op,
385 PatternRewriter &rewriter) const override {
386 // Do not clone already cloned ops to avoid going into an infinite loop.
387 if (op->hasDiscardableAttr(name: "was_cloned"))
388 return failure();
389 for (Region &r : op->getRegions())
390 rewriter.cloneRegionBefore(region&: r, before: op->getBlock());
391 rewriter.modifyOpInPlace(root: op, callable: [&]() {
392 op->setDiscardableAttr(name: "was_cloned", value: rewriter.getUnitAttr());
393 });
394 return success();
395 }
396};
397
398/// Replace an operation may introduce the re-visiting of its users.
399class ReplaceWithNewOp : public RewritePattern {
400public:
401 ReplaceWithNewOp(MLIRContext *context)
402 : RewritePattern("test.replace_with_new_op", /*benefit=*/1, context) {}
403
404 LogicalResult matchAndRewrite(Operation *op,
405 PatternRewriter &rewriter) const override {
406 Operation *newOp;
407 if (op->hasDiscardableAttr(name: "create_erase_op")) {
408 newOp = rewriter.create(
409 loc: op->getLoc(),
410 opName: OperationName("test.erase_op", op->getContext()).getIdentifier(),
411 operands: ValueRange(), types: TypeRange());
412 } else {
413 newOp = rewriter.create(
414 loc: op->getLoc(),
415 opName: OperationName("test.new_op", op->getContext()).getIdentifier(),
416 operands: op->getOperands(), types: op->getResultTypes());
417 }
418 // "replaceOp" could be used instead of "replaceAllOpUsesWith"+"eraseOp".
419 // A "notifyOperationReplaced" callback is triggered in either case.
420 rewriter.replaceAllOpUsesWith(from: op, to: newOp->getResults());
421 rewriter.eraseOp(op);
422 return success();
423 }
424};
425
426/// Erases the first child block of the matched "test.erase_first_block"
427/// operation.
428class EraseFirstBlock : public RewritePattern {
429public:
430 EraseFirstBlock(MLIRContext *context)
431 : RewritePattern("test.erase_first_block", /*benefit=*/1, context) {}
432
433 LogicalResult matchAndRewrite(Operation *op,
434 PatternRewriter &rewriter) const override {
435 for (Region &r : op->getRegions()) {
436 for (Block &b : r.getBlocks()) {
437 rewriter.eraseBlock(block: &b);
438 return success();
439 }
440 }
441
442 return failure();
443 }
444};
445
446/// Creates and immediately erases an operation and a block.
447class CreateAndEraseOpAndBlock : public RewritePattern {
448public:
449 CreateAndEraseOpAndBlock(MLIRContext *context)
450 : RewritePattern("test.create_and_erase_op_and_block", /*benefit=*/1,
451 context) {}
452
453 LogicalResult matchAndRewrite(Operation *op,
454 PatternRewriter &rewriter) const override {
455 if (op->hasDiscardableAttr(name: "was_rewritten"))
456 return failure();
457
458 Operation *newOp = rewriter.create(
459 loc: op->getLoc(),
460 opName: OperationName("test.transient_op", op->getContext()).getIdentifier(),
461 operands: ValueRange(), types: TypeRange());
462 rewriter.eraseOp(op: newOp);
463
464 Block *newBlock = rewriter.createBlock(parent: op->getParentRegion());
465 rewriter.eraseBlock(block: newBlock);
466
467 rewriter.modifyOpInPlace(root: op, callable: [&]() {
468 op->setDiscardableAttr(name: "was_rewritten", value: rewriter.getUnitAttr());
469 });
470 return success();
471 }
472};
473
474/// Perform the CFG change selected by the `mode` of a `test.greedy_cfg_rewrite`
475/// op while a greedy rewrite is processing its worklist. The modes are
476/// documented on the op.
477class GreedyCfgRewritePattern
478 : public OpRewritePattern<TestGreedyCfgRewriteOp> {
479public:
480 using OpRewritePattern::OpRewritePattern;
481
482 LogicalResult matchAndRewrite(TestGreedyCfgRewriteOp op,
483 PatternRewriter &rewriter) const override {
484 Region *region = op->getParentRegion();
485 Location loc = op.getLoc();
486 // The unconditional branch that terminates the block of `op`.
487 auto getBranch = [&] {
488 return cast<cf::BranchOp>(op->getBlock()->getTerminator());
489 };
490 switch (op.getMode()) {
491 case GreedyCfgRewriteMode::Observe:
492 op.emitRemark("processed reachable block");
493 break;
494 case GreedyCfgRewriteMode::AppendUnreachable:
495 case GreedyCfgRewriteMode::AppendReachable: {
496 Block *newBlock = rewriter.createBlock(parent: region);
497 TestGreedyCfgRewriteOp::create(rewriter, loc,
498 GreedyCfgRewriteMode::Observe,
499 /*cond=*/Value());
500 func::ReturnOp::create(builder&: rewriter, location: loc);
501 if (op.getMode() == GreedyCfgRewriteMode::AppendReachable) {
502 cf::BranchOp branch = getBranch();
503 rewriter.setInsertionPoint(branch);
504 cf::CondBranchOp::create(rewriter, loc, op.getCond(), branch.getDest(),
505 ValueRange(), newBlock, ValueRange());
506 rewriter.eraseOp(op: branch);
507 }
508 break;
509 }
510 case GreedyCfgRewriteMode::InsertEntry:
511 rewriter.createBlock(parent: region, insertPt: region->begin());
512 func::ReturnOp::create(builder&: rewriter, location: loc);
513 break;
514 case GreedyCfgRewriteMode::MoveEntry: {
515 // The new entry must not have predecessors after the rewrite.
516 cf::BranchOp branch = getBranch();
517 rewriter.setInsertionPoint(branch);
518 rewriter.replaceOpWithNewOp<func::ReturnOp>(op: branch);
519 rewriter.moveBlockBefore(block: &region->back(), region, iterator: region->begin());
520 break;
521 }
522 case GreedyCfgRewriteMode::MoveEntryAway: {
523 Block *entry = &region->front();
524 rewriter.createBlock(parent: region, insertPt: std::next(x: region->begin()));
525 func::ReturnOp::create(builder&: rewriter, location: loc);
526 rewriter.moveBlockBefore(block: entry, region, iterator: region->end());
527 break;
528 }
529 case GreedyCfgRewriteMode::EraseEntry:
530 rewriter.eraseBlock(block: &region->front());
531 break;
532 case GreedyCfgRewriteMode::Redirect:
533 case GreedyCfgRewriteMode::DoubleRedirect: {
534 if (auto conditional =
535 dyn_cast<cf::CondBranchOp>(op->getBlock()->getTerminator())) {
536 assert(op.getMode() == GreedyCfgRewriteMode::DoubleRedirect);
537 rewriter.setInsertionPoint(conditional);
538 cf::BranchOp::create(rewriter, loc, conditional.getFalseDest());
539 rewriter.eraseOp(op: conditional);
540 } else {
541 cf::BranchOp branch = getBranch();
542 rewriter.modifyOpInPlace(root: branch,
543 callable: [&] { branch.setDest(&region->back()); });
544 }
545 if (op.getMode() == GreedyCfgRewriteMode::DoubleRedirect) {
546 Operation *entryBranch = region->front().getTerminator();
547 rewriter.setInsertionPoint(entryBranch);
548 func::ReturnOp::create(builder&: rewriter, location: loc);
549 rewriter.eraseOp(op: entryBranch);
550 }
551 break;
552 }
553 case GreedyCfgRewriteMode::Reconnect: {
554 cf::BranchOp branch = getBranch();
555 rewriter.setInsertionPoint(branch);
556 cf::CondBranchOp::create(
557 rewriter, loc, op.getCond(), branch.getDest(), ValueRange(),
558 &*std::next(op->getBlock()->getIterator()), ValueRange());
559 rewriter.eraseOp(op: branch);
560 break;
561 }
562 case GreedyCfgRewriteMode::DisconnectParent: {
563 Region *parentRegion = region->getParentOp()->getParentRegion();
564 auto branch = cast<cf::BranchOp>(Val: parentRegion->front().getTerminator());
565 rewriter.modifyOpInPlace(branch,
566 [&] { branch.setDest(&parentRegion->back()); });
567 break;
568 }
569 case GreedyCfgRewriteMode::MoveParent: {
570 Operation *outer = region->getParentOp()->getParentOp();
571 Block *newBlock = rewriter.createBlock(parent: outer->getParentRegion());
572 func::ReturnOp::create(builder&: rewriter, location: loc);
573 rewriter.moveOpBefore(op: outer, block: newBlock, iterator: newBlock->begin());
574 break;
575 }
576 case GreedyCfgRewriteMode::MoveErase: {
577 cf::BranchOp branch = getBranch();
578 Block *moved = branch.getDest();
579 rewriter.modifyOpInPlace(root: branch,
580 callable: [&] { branch.setDest(&region->back()); });
581 // The block is erased while it belongs to another region. The source
582 // region keeps its entry and several blocks, so its cached state must
583 // not retain a pointer to the erased block.
584 Region temporary;
585 rewriter.moveBlockBefore(block: moved, region: &temporary, iterator: temporary.end());
586 rewriter.eraseBlock(block: moved);
587 break;
588 }
589 case GreedyCfgRewriteMode::RedirectRetarget: {
590 // Two terminators change in one rewrite. The old destination has no
591 // predecessors left, so its retargeted branch must not make the new
592 // block reachable, whichever block the cache processes first.
593 cf::BranchOp branch = getBranch();
594 auto lostBranch = cast<cf::BranchOp>(Val: branch.getDest()->getTerminator());
595 Block *newBlock = rewriter.createBlock(parent: region);
596 TestGreedyCfgRewriteOp::create(rewriter, loc,
597 GreedyCfgRewriteMode::Observe,
598 /*cond=*/Value());
599 func::ReturnOp::create(builder&: rewriter, location: loc);
600 rewriter.modifyOpInPlace(root: branch,
601 callable: [&] { branch.setDest(lostBranch.getDest()); });
602 rewriter.modifyOpInPlace(lostBranch,
603 [&] { lostBranch.setDest(newBlock); });
604 break;
605 }
606 case GreedyCfgRewriteMode::Merge:
607 case GreedyCfgRewriteMode::MergeDisconnect: {
608 Block *dest = op->getBlock();
609 cf::BranchOp branch = getBranch();
610 Block *source = branch.getDest();
611 rewriter.eraseOp(op: branch);
612 rewriter.mergeBlocks(source, dest);
613 if (op.getMode() == GreedyCfgRewriteMode::MergeDisconnect) {
614 auto conditional = cast<cf::CondBranchOp>(Val: dest->getTerminator());
615 rewriter.setInsertionPoint(conditional);
616 cf::BranchOp::create(rewriter, loc, conditional.getTrueDest());
617 rewriter.eraseOp(op: conditional);
618 }
619 break;
620 }
621 }
622 rewriter.eraseOp(op: op);
623 return success();
624 }
625};
626
627struct TestGreedyPatternDriver
628 : public PassWrapper<TestGreedyPatternDriver, OperationPass<>> {
629 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestGreedyPatternDriver)
630
631 TestGreedyPatternDriver() = default;
632 TestGreedyPatternDriver(const TestGreedyPatternDriver &other)
633 : PassWrapper(other) {}
634
635 StringRef getArgument() const final { return "test-greedy-patterns"; }
636 StringRef getDescription() const final { return "Run test dialect patterns"; }
637 void runOnOperation() override {
638 mlir::RewritePatternSet patterns(&getContext());
639 populateWithGenerated(patterns);
640
641 // Verify named pattern is generated with expected name.
642 patterns.add<FoldingPattern, TestNamedPatternRule,
643 FolderInsertBeforePreviouslyFoldedConstantPattern,
644 FolderCommutativeOp2WithConstant, HoistEligibleOps,
645 MakeOpEligible, GreedyCfgRewritePattern>(&getContext());
646
647 // Additional patterns for testing the GreedyPatternRewriteDriver.
648 patterns.insert<IncrementIntAttribute<3>>(arg: &getContext());
649
650 DumpNotifications listener;
651 GreedyRewriteConfig config;
652 config.setListener(dumpNotifications ? &listener : nullptr);
653 config.setUseTopDownTraversal(useTopDownTraversal)
654 .setMaxIterations(this->maxIterations)
655 .enableFolding(enable: this->fold)
656 .enableConstantCSE(enable: this->cseConstants);
657 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns), config);
658 }
659
660 Option<bool> dumpNotifications{
661 *this, "dump-notifications",
662 llvm::cl::desc("Print rewrite listener notifications"),
663 llvm::cl::init(Val: false)};
664 Option<bool> useTopDownTraversal{
665 *this, "top-down",
666 llvm::cl::desc("Seed the worklist in general top-down order"),
667 llvm::cl::init(Val: GreedyRewriteConfig().getUseTopDownTraversal())};
668 Option<int> maxIterations{
669 *this, "max-iterations",
670 llvm::cl::desc("Max. iterations in the GreedyRewriteConfig"),
671 llvm::cl::init(Val: GreedyRewriteConfig().getMaxIterations())};
672 Option<bool> fold{*this, "fold", llvm::cl::desc("Whether to fold"),
673 llvm::cl::init(Val: GreedyRewriteConfig().isFoldingEnabled())};
674 Option<bool> cseConstants{
675 *this, "cse-constants", llvm::cl::desc("Whether to CSE constants"),
676 llvm::cl::init(Val: GreedyRewriteConfig().isConstantCSEEnabled())};
677};
678
679struct TestStrictPatternDriver
680 : public PassWrapper<TestStrictPatternDriver, OperationPass<func::FuncOp>> {
681public:
682 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestStrictPatternDriver)
683
684 TestStrictPatternDriver() = default;
685 TestStrictPatternDriver(const TestStrictPatternDriver &other)
686 : PassWrapper(other) {
687 strictMode = other.strictMode;
688 }
689
690 StringRef getArgument() const final { return "test-strict-pattern-driver"; }
691 StringRef getDescription() const final {
692 return "Test strict mode of pattern driver";
693 }
694
695 void runOnOperation() override {
696 MLIRContext *ctx = &getContext();
697 mlir::RewritePatternSet patterns(ctx);
698 patterns.add<
699 // clang-format off
700 ChangeBlockOp,
701 GreedyCfgRewritePattern,
702 CloneOp,
703 CloneRegionBeforeOp,
704 EraseOp,
705 ImplicitChangeOp,
706 InlineBlocksIntoParent,
707 InsertSameOp,
708 MoveBeforeParentOp,
709 ReplaceWithNewOp,
710 SplitBlockHere
711 // clang-format on
712 >(arg&: ctx);
713 SmallVector<Operation *> ops;
714 getOperation()->walk(callback: [&](Operation *op) {
715 StringRef opName = op->getName().getStringRef();
716 if (opName == "test.insert_same_op" || opName == "test.change_block_op" ||
717 opName == "test.replace_with_new_op" || opName == "test.erase_op" ||
718 opName == "test.move_before_parent_op" ||
719 opName == "test.inline_blocks_into_parent" ||
720 opName == "test.split_block_here" || opName == "test.clone_me" ||
721 opName == "test.clone_region_before" ||
722 opName == "test.greedy_cfg_rewrite") {
723 ops.push_back(Elt: op);
724 }
725 });
726
727 DumpNotifications dumpNotifications;
728 GreedyRewriteConfig config;
729 config.setListener(&dumpNotifications);
730 if (strictMode == "AnyOp") {
731 config.setStrictness(GreedyRewriteStrictness::AnyOp);
732 } else if (strictMode == "ExistingAndNewOps") {
733 config.setStrictness(GreedyRewriteStrictness::ExistingAndNewOps);
734 } else if (strictMode == "ExistingOps") {
735 config.setStrictness(GreedyRewriteStrictness::ExistingOps);
736 } else {
737 llvm_unreachable("invalid strictness option");
738 }
739
740 // Check if these transformations introduce visiting of operations that
741 // are not in the `ops` set (The new created ops are valid). An invalid
742 // operation will trigger the assertion while processing.
743 bool changed = false;
744 bool allErased = false;
745 if (failed(Result: applyOpPatternsGreedily(ops: ArrayRef(ops), patterns: std::move(patterns),
746 config, changed: &changed, allErased: &allErased)))
747 return signalPassFailure();
748 Builder b(ctx);
749 getOperation()->setDiscardableAttr(name: "pattern_driver_changed",
750 value: b.getBoolAttr(value: changed));
751 getOperation()->setDiscardableAttr(name: "pattern_driver_all_erased",
752 value: b.getBoolAttr(value: allErased));
753 }
754
755 Option<std::string> strictMode{
756 *this, "strictness",
757 llvm::cl::desc("Can be {AnyOp, ExistingAndNewOps, ExistingOps}"),
758 llvm::cl::init(Val: "AnyOp")};
759
760private:
761 // New inserted operation is valid for further transformation.
762 class InsertSameOp : public RewritePattern {
763 public:
764 InsertSameOp(MLIRContext *context)
765 : RewritePattern("test.insert_same_op", /*benefit=*/1, context) {}
766
767 LogicalResult matchAndRewrite(Operation *op,
768 PatternRewriter &rewriter) const override {
769 if (op->hasDiscardableAttr(name: "skip"))
770 return failure();
771
772 Operation *newOp =
773 rewriter.create(loc: op->getLoc(), opName: op->getName().getIdentifier(),
774 operands: op->getOperands(), types: op->getResultTypes());
775 rewriter.modifyOpInPlace(root: op, callable: [&]() {
776 op->setDiscardableAttr(name: "skip", value: rewriter.getBoolAttr(value: true));
777 });
778 newOp->setDiscardableAttr(name: "skip", value: rewriter.getBoolAttr(value: true));
779
780 return success();
781 }
782 };
783
784 // Remove an operation may introduce the re-visiting of its operands.
785 class EraseOp : public RewritePattern {
786 public:
787 EraseOp(MLIRContext *context)
788 : RewritePattern("test.erase_op", /*benefit=*/1, context) {}
789 LogicalResult matchAndRewrite(Operation *op,
790 PatternRewriter &rewriter) const override {
791 rewriter.eraseOp(op);
792 return success();
793 }
794 };
795
796 // The following two patterns test RewriterBase::replaceAllUsesWith.
797 //
798 // That function replaces all usages of a Block (or a Value) with another one
799 // *and tracks these changes in the rewriter.* The GreedyPatternRewriteDriver
800 // with GreedyRewriteStrictness::AnyOp uses that tracking to construct its
801 // worklist: when an op is modified, it is added to the worklist. The two
802 // patterns below make the tracking observable: ChangeBlockOp replaces all
803 // usages of a block and that pattern is applied because the corresponding ops
804 // are put on the initial worklist (see above). ImplicitChangeOp does an
805 // unrelated change but ops of the corresponding type are *not* on the initial
806 // worklist, so the effect of the second pattern is only visible if the
807 // tracking and subsequent adding to the worklist actually works.
808
809 // Replace all usages of the first successor with the second successor.
810 class ChangeBlockOp : public RewritePattern {
811 public:
812 ChangeBlockOp(MLIRContext *context)
813 : RewritePattern("test.change_block_op", /*benefit=*/1, context) {}
814 LogicalResult matchAndRewrite(Operation *op,
815 PatternRewriter &rewriter) const override {
816 if (op->getNumSuccessors() < 2)
817 return failure();
818 Block *firstSuccessor = op->getSuccessor(index: 0);
819 Block *secondSuccessor = op->getSuccessor(index: 1);
820 if (firstSuccessor == secondSuccessor)
821 return failure();
822 // This is the function being tested:
823 rewriter.replaceAllUsesWith(from: firstSuccessor, to: secondSuccessor);
824 // Using the following line instead would make the test fail:
825 // firstSuccessor->replaceAllUsesWith(secondSuccessor);
826 return success();
827 }
828 };
829
830 // Changes the successor to the parent block.
831 class ImplicitChangeOp : public RewritePattern {
832 public:
833 ImplicitChangeOp(MLIRContext *context)
834 : RewritePattern("test.implicit_change_op", /*benefit=*/1, context) {}
835 LogicalResult matchAndRewrite(Operation *op,
836 PatternRewriter &rewriter) const override {
837 if (op->getNumSuccessors() < 1 || op->getSuccessor(index: 0) == op->getBlock())
838 return failure();
839 rewriter.modifyOpInPlace(root: op,
840 callable: [&]() { op->setSuccessor(block: op->getBlock(), index: 0); });
841 return success();
842 }
843 };
844};
845
846struct TestWalkPatternDriver final
847 : PassWrapper<TestWalkPatternDriver, OperationPass<>> {
848 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestWalkPatternDriver)
849
850 TestWalkPatternDriver() = default;
851 TestWalkPatternDriver(const TestWalkPatternDriver &other)
852 : PassWrapper(other) {}
853
854 StringRef getArgument() const override {
855 return "test-walk-pattern-rewrite-driver";
856 }
857 StringRef getDescription() const override {
858 return "Run test walk pattern rewrite driver";
859 }
860 void runOnOperation() override {
861 mlir::RewritePatternSet patterns(&getContext());
862
863 // Patterns for testing the WalkPatternRewriteDriver.
864 patterns.add<IncrementIntAttribute<3>, MoveBeforeParentOp,
865 MoveAfterParentOp, CloneOp, ReplaceWithNewOp, EraseFirstBlock,
866 CreateAndEraseOpAndBlock>(arg: &getContext());
867
868 DumpNotifications dumpListener;
869 walkAndApplyPatterns(op: getOperation(), patterns: std::move(patterns),
870 listener: dumpNotifications ? &dumpListener : nullptr);
871 }
872
873 Option<bool> dumpNotifications{
874 *this, "dump-notifications",
875 llvm::cl::desc("Print rewrite listener notifications"),
876 llvm::cl::init(Val: false)};
877};
878
879} // namespace
880
881//===----------------------------------------------------------------------===//
882// ReturnType Driver.
883//===----------------------------------------------------------------------===//
884
885namespace {
886// Generate ops for each instance where the type can be successfully inferred.
887template <typename OpTy>
888static void invokeCreateWithInferredReturnType(Operation *op) {
889 auto *context = op->getContext();
890 auto fop = op->getParentOfType<func::FuncOp>();
891 auto location = UnknownLoc::get(context);
892 OpBuilder b(op);
893 b.setInsertionPointAfter(op);
894
895 // Use permutations of 2 args as operands.
896 assert(fop.getNumArguments() >= 2);
897 for (int i = 0, e = fop.getNumArguments(); i < e; ++i) {
898 for (int j = 0; j < e; ++j) {
899 std::array<Value, 2> values = {._M_elems: {fop.getArgument(idx: i), fop.getArgument(idx: j)}};
900 SmallVector<Type, 2> inferredReturnTypes;
901 // Only pass properties if the op's properties type matches OpTy's.
902 PropertyRef properties = op->getPropertiesStorage();
903 if (properties.getTypeID() != TypeID::get<typename OpTy::Properties>())
904 properties = PropertyRef();
905 if (succeeded(OpTy::inferReturnTypes(
906 context, std::nullopt, values, op->getDiscardableAttrDictionary(),
907 properties, op->getRegions(), inferredReturnTypes))) {
908 OperationState state(location, OpTy::getOperationName());
909 // TODO: Expand to regions.
910 OpTy::build(b, state, values,
911 op->getDiscardableAttrDictionary().getValue());
912 (void)b.create(state);
913 }
914 }
915 }
916}
917
918static void reifyReturnShape(Operation *op) {
919 OpBuilder b(op);
920
921 // Use permutations of 2 args as operands.
922 auto shapedOp = cast<OpWithShapedTypeInferTypeInterfaceOp>(op);
923 SmallVector<Value, 2> shapes;
924 if (failed(shapedOp.reifyReturnTypeShapes(b, op->getOperands(), shapes)) ||
925 !llvm::hasSingleElement(C&: shapes))
926 return;
927 for (const auto &it : llvm::enumerate(First&: shapes)) {
928 op->emitRemark() << "value " << it.index() << ": "
929 << it.value().getDefiningOp();
930 }
931}
932
933struct TestReturnTypeDriver
934 : public PassWrapper<TestReturnTypeDriver, OperationPass<func::FuncOp>> {
935 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestReturnTypeDriver)
936
937 void getDependentDialects(DialectRegistry &registry) const override {
938 registry.insert<tensor::TensorDialect>();
939 }
940 StringRef getArgument() const final { return "test-return-type"; }
941 StringRef getDescription() const final { return "Run return type functions"; }
942
943 void runOnOperation() override {
944 if (getOperation().getName() == "testCreateFunctions") {
945 std::vector<Operation *> ops;
946 // Collect ops to avoid triggering on inserted ops.
947 for (auto &op : getOperation().getBody().front())
948 ops.push_back(x: &op);
949 // Generate test patterns for each, but skip terminator.
950 for (auto *op : llvm::ArrayRef(ops).drop_back()) {
951 // Test create method of each of the Op classes below. The resultant
952 // output would be in reverse order underneath `op` from which
953 // the attributes and regions are used.
954 invokeCreateWithInferredReturnType<OpWithInferTypeInterfaceOp>(op);
955 invokeCreateWithInferredReturnType<OpWithInferTypeAdaptorInterfaceOp>(
956 op);
957 invokeCreateWithInferredReturnType<
958 OpWithShapedTypeInferTypeInterfaceOp>(op);
959 };
960 return;
961 }
962 if (getOperation().getName() == "testReifyFunctions") {
963 std::vector<Operation *> ops;
964 // Collect ops to avoid triggering on inserted ops.
965 for (auto &op : getOperation().getBody().front())
966 if (isa<OpWithShapedTypeInferTypeInterfaceOp>(op))
967 ops.push_back(x: &op);
968 // Generate test patterns for each, but skip terminator.
969 for (auto *op : ops)
970 reifyReturnShape(op);
971 }
972 }
973};
974} // namespace
975
976namespace {
977struct TestDerivedAttributeDriver
978 : public PassWrapper<TestDerivedAttributeDriver,
979 OperationPass<func::FuncOp>> {
980 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestDerivedAttributeDriver)
981
982 StringRef getArgument() const final { return "test-derived-attr"; }
983 StringRef getDescription() const final {
984 return "Run test derived attributes";
985 }
986 void runOnOperation() override;
987};
988} // namespace
989
990void TestDerivedAttributeDriver::runOnOperation() {
991 getOperation().walk(callback: [](DerivedAttributeOpInterface dOp) {
992 auto dAttr = dOp.materializeDerivedAttributes();
993 if (!dAttr)
994 return;
995 for (auto d : dAttr)
996 dOp.emitRemark() << d.getName().getValue() << " = " << d.getValue();
997 });
998}
999
1000//===----------------------------------------------------------------------===//
1001// Legalization Driver.
1002//===----------------------------------------------------------------------===//
1003
1004namespace {
1005//===----------------------------------------------------------------------===//
1006// Region-Block Rewrite Testing
1007//===----------------------------------------------------------------------===//
1008
1009/// This pattern applies a signature conversion to a block inside a detached
1010/// region.
1011struct TestDetachedSignatureConversion : public ConversionPattern {
1012 TestDetachedSignatureConversion(MLIRContext *ctx)
1013 : ConversionPattern("test.detached_signature_conversion", /*benefit=*/1,
1014 ctx) {}
1015
1016 LogicalResult
1017 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1018 ConversionPatternRewriter &rewriter) const final {
1019 if (op->getNumRegions() != 1)
1020 return failure();
1021 OperationState state(op->getLoc(), "test.legal_op", operands,
1022 op->getResultTypes(), {}, BlockRange());
1023 Region *newRegion = state.addRegion();
1024 rewriter.inlineRegionBefore(region&: op->getRegion(index: 0), parent&: *newRegion,
1025 before: newRegion->begin());
1026 TypeConverter::SignatureConversion result(newRegion->getNumArguments());
1027 for (unsigned i = 0, e = newRegion->getNumArguments(); i < e; ++i)
1028 result.addInputs(origInputNo: i, types: rewriter.getF64Type());
1029 rewriter.applySignatureConversion(block: &newRegion->front(), conversion&: result);
1030 Operation *newOp = rewriter.create(state);
1031 rewriter.replaceOp(op, newValues: newOp->getResults());
1032 return success();
1033 }
1034};
1035
1036/// This pattern is a simple pattern that inlines the first region of a given
1037/// operation into the parent region.
1038struct TestRegionRewriteBlockMovement : public ConversionPattern {
1039 TestRegionRewriteBlockMovement(MLIRContext *ctx)
1040 : ConversionPattern("test.region", 1, ctx) {}
1041
1042 LogicalResult
1043 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1044 ConversionPatternRewriter &rewriter) const final {
1045 // Inline this region into the parent region.
1046 auto &parentRegion = *op->getParentRegion();
1047 auto &opRegion = op->getRegion(index: 0);
1048 if (op->getDiscardableAttr(name: "legalizer.should_clone"))
1049 rewriter.cloneRegionBefore(region&: opRegion, parent&: parentRegion, before: parentRegion.end());
1050 else
1051 rewriter.inlineRegionBefore(region&: opRegion, parent&: parentRegion, before: parentRegion.end());
1052
1053 if (op->getDiscardableAttr(name: "legalizer.erase_old_blocks")) {
1054 while (!opRegion.empty())
1055 rewriter.eraseBlock(block: &opRegion.front());
1056 }
1057
1058 // Drop this operation.
1059 rewriter.eraseOp(op);
1060 return success();
1061 }
1062};
1063/// This pattern is a simple pattern that generates a region containing an
1064/// illegal operation.
1065struct TestRegionRewriteUndo : public RewritePattern {
1066 TestRegionRewriteUndo(MLIRContext *ctx)
1067 : RewritePattern("test.region_builder", 1, ctx) {}
1068
1069 LogicalResult matchAndRewrite(Operation *op,
1070 PatternRewriter &rewriter) const final {
1071 // Create the region operation with an entry block containing arguments.
1072 OperationState newRegion(op->getLoc(), "test.region");
1073 newRegion.addRegion();
1074 auto *regionOp = rewriter.create(state: newRegion);
1075 auto *entryBlock = rewriter.createBlock(parent: &regionOp->getRegion(index: 0));
1076 entryBlock->addArgument(type: rewriter.getIntegerType(width: 64),
1077 loc: rewriter.getUnknownLoc());
1078
1079 // Add an explicitly illegal operation to ensure the conversion fails.
1080 ILLegalOpF::create(rewriter, op->getLoc(), rewriter.getIntegerType(32));
1081 TestValidOp::create(rewriter, op->getLoc(), ArrayRef<Value>());
1082
1083 // Drop this operation.
1084 rewriter.eraseOp(op);
1085 return success();
1086 }
1087};
1088/// A simple pattern that creates a block at the end of the parent region of the
1089/// matched operation.
1090struct TestCreateBlock : public RewritePattern {
1091 TestCreateBlock(MLIRContext *ctx)
1092 : RewritePattern("test.create_block", /*benefit=*/1, ctx) {}
1093
1094 LogicalResult matchAndRewrite(Operation *op,
1095 PatternRewriter &rewriter) const final {
1096 Region &region = *op->getParentRegion();
1097 Type i32Type = rewriter.getIntegerType(width: 32);
1098 Location loc = op->getLoc();
1099 rewriter.createBlock(parent: &region, insertPt: region.end(), argTypes: {i32Type, i32Type}, locs: {loc, loc});
1100 TerminatorOp::create(rewriter, loc);
1101 rewriter.eraseOp(op);
1102 return success();
1103 }
1104};
1105
1106/// A simple pattern that creates a block containing an invalid operation in
1107/// order to trigger the block creation undo mechanism.
1108struct TestCreateIllegalBlock : public RewritePattern {
1109 TestCreateIllegalBlock(MLIRContext *ctx)
1110 : RewritePattern("test.create_illegal_block", /*benefit=*/1, ctx) {}
1111
1112 LogicalResult matchAndRewrite(Operation *op,
1113 PatternRewriter &rewriter) const final {
1114 Region &region = *op->getParentRegion();
1115 Type i32Type = rewriter.getIntegerType(width: 32);
1116 Location loc = op->getLoc();
1117 rewriter.createBlock(parent: &region, insertPt: region.end(), argTypes: {i32Type, i32Type}, locs: {loc, loc});
1118 // Create an illegal op to ensure the conversion fails.
1119 ILLegalOpF::create(rewriter, loc, i32Type);
1120 TerminatorOp::create(rewriter, loc);
1121 rewriter.eraseOp(op);
1122 return success();
1123 }
1124};
1125
1126/// A simple pattern that tests the "replaceAllUsesWith" API.
1127struct TestValueReplace : public ConversionPattern {
1128 TestValueReplace(MLIRContext *ctx, const TypeConverter &converter)
1129 : ConversionPattern(converter, "test.value_replace", /*benefit=*/1, ctx) {
1130 }
1131
1132 LogicalResult
1133 matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
1134 ConversionPatternRewriter &rewriter) const final {
1135 // Replace the first operand with 2x the second operand.
1136 Value from = op->getOperand(idx: 0);
1137 Value repl = op->getOperand(idx: 1);
1138 if (op->hasDiscardableAttr(name: "conditional")) {
1139 rewriter.replaceUsesWithIf(from, to: {repl, repl}, functor: [=](OpOperand &use) {
1140 return use.getOwner()->hasDiscardableAttr(name: "replace_uses");
1141 });
1142 } else {
1143 rewriter.replaceAllUsesWith(from, to: {repl, repl});
1144 }
1145 rewriter.modifyOpInPlace(root: op, callable: [&] {
1146 // If the "trigger_rollback" attribute is set, keep the op illegal, so
1147 // that a rollback is triggered.
1148 if (!op->hasDiscardableAttr(name: "trigger_rollback"))
1149 op->setDiscardableAttr(name: "is_legal", value: rewriter.getUnitAttr());
1150 });
1151 return success();
1152 }
1153};
1154
1155/// This pattern hoists ops out of a "test.hoist_me" and then fails conversion.
1156/// This is to test the rollback logic.
1157struct TestUndoMoveOpBefore : public ConversionPattern {
1158 TestUndoMoveOpBefore(MLIRContext *ctx)
1159 : ConversionPattern("test.hoist_me", /*benefit=*/1, ctx) {}
1160
1161 LogicalResult
1162 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1163 ConversionPatternRewriter &rewriter) const override {
1164 rewriter.moveOpBefore(op, existingOp: op->getParentOp());
1165 // Replace with an illegal op to ensure the conversion fails.
1166 rewriter.replaceOpWithNewOp<ILLegalOpF>(op, rewriter.getF32Type());
1167 return success();
1168 }
1169};
1170
1171/// A rewrite pattern that tests the undo mechanism when erasing a block.
1172struct TestUndoBlockErase : public ConversionPattern {
1173 TestUndoBlockErase(MLIRContext *ctx)
1174 : ConversionPattern("test.undo_block_erase", /*benefit=*/1, ctx) {}
1175
1176 LogicalResult
1177 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1178 ConversionPatternRewriter &rewriter) const final {
1179 Block *secondBlock = &*std::next(x: op->getRegion(index: 0).begin());
1180 rewriter.setInsertionPointToStart(secondBlock);
1181 ILLegalOpF::create(rewriter, op->getLoc(), rewriter.getF32Type());
1182 rewriter.eraseBlock(block: secondBlock);
1183 rewriter.modifyOpInPlace(root: op, callable: [] {});
1184 return success();
1185 }
1186};
1187
1188/// A pattern that modifies a property in-place, but keeps the op illegal.
1189struct TestUndoPropertiesModification : public ConversionPattern {
1190 TestUndoPropertiesModification(MLIRContext *ctx)
1191 : ConversionPattern("test.with_properties", /*benefit=*/1, ctx) {}
1192 LogicalResult
1193 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1194 ConversionPatternRewriter &rewriter) const final {
1195 if (!op->hasDiscardableAttr(name: "modify_inplace"))
1196 return failure();
1197 rewriter.modifyOpInPlace(
1198 root: op, callable: [&]() { cast<TestOpWithProperties>(op).getProperties().setA(42); });
1199 return success();
1200 }
1201};
1202
1203/// A pattern that tests the undo mechanism for a block move if the block was
1204/// moved to a detached region. The block is first moved to a detached region
1205/// and then a new operation is created with that region. During rollback, first
1206/// the `CreateOperationRewrite` is rolled back, causing the block to be
1207/// orphaned, i.e., removed from the region. Only then the `MoveBlockRewrite` is
1208/// rolled back, which now can't access the region anymore. The test ensures
1209/// that the rollback still works and doesn't try to access the orphaned block's
1210/// containing region, leading to segfault.
1211struct TestUndoMoveDetachedBlock : public ConversionPattern {
1212 TestUndoMoveDetachedBlock(MLIRContext *ctx)
1213 : ConversionPattern("test.undo_detached_block_move", /*benefit=*/1, ctx) {
1214 }
1215
1216 LogicalResult
1217 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1218 ConversionPatternRewriter &rewriter) const final {
1219 if (op->getNumRegions() != 1)
1220 return failure();
1221 // Create an illegal operation to trigger rollback.
1222 OperationState state(op->getLoc(), "test.illegal_op_created_after_move",
1223 operands, op->getResultTypes(), {}, BlockRange());
1224 // Create detached region.
1225 Region *newRegion = state.addRegion();
1226 // Move blocks to the still detached region
1227 rewriter.inlineRegionBefore(region&: op->getRegion(index: 0), parent&: *newRegion,
1228 before: newRegion->begin());
1229 Operation *newOp = rewriter.create(state);
1230 rewriter.replaceOp(op, newValues: newOp->getResults());
1231 return success();
1232 }
1233};
1234
1235//===----------------------------------------------------------------------===//
1236// Type-Conversion Rewrite Testing
1237//===----------------------------------------------------------------------===//
1238
1239/// This patterns erases a region operation that has had a type conversion.
1240struct TestDropOpSignatureConversion : public ConversionPattern {
1241 TestDropOpSignatureConversion(MLIRContext *ctx,
1242 const TypeConverter &converter)
1243 : ConversionPattern(converter, "test.drop_region_op", 1, ctx) {}
1244 LogicalResult
1245 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1246 ConversionPatternRewriter &rewriter) const override {
1247 Region &region = op->getRegion(index: 0);
1248 Block *entry = &region.front();
1249
1250 // Convert the original entry arguments.
1251 const TypeConverter &converter = *getTypeConverter();
1252 TypeConverter::SignatureConversion result(entry->getNumArguments());
1253 if (failed(Result: converter.convertSignatureArgs(types: entry->getArgumentTypes(),
1254 result)) ||
1255 failed(Result: rewriter.convertRegionTypes(region: &region, converter, entryConversion: &result)))
1256 return failure();
1257
1258 // Convert the region signature and just drop the operation.
1259 rewriter.eraseOp(op);
1260 return success();
1261 }
1262};
1263/// This pattern simply updates the operands of the given operation.
1264struct TestPassthroughInvalidOp : public ConversionPattern {
1265 TestPassthroughInvalidOp(MLIRContext *ctx, const TypeConverter &converter)
1266 : ConversionPattern(converter, "test.invalid", 1, ctx) {}
1267 LogicalResult
1268 matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
1269 ConversionPatternRewriter &rewriter) const final {
1270 SmallVector<Value> flattened;
1271 for (auto it : llvm::enumerate(First&: operands)) {
1272 ValueRange range = it.value();
1273 if (range.size() == 1) {
1274 flattened.push_back(Elt: range.front());
1275 continue;
1276 }
1277
1278 // This is a 1:N replacement. Insert a test.cast op. (That's what the
1279 // argument materialization used to do.)
1280 flattened.push_back(
1281 TestCastOp::create(rewriter, op->getLoc(),
1282 op->getOperand(it.index()).getType(), range)
1283 .getResult());
1284 }
1285 rewriter.replaceOpWithNewOp<TestValidOp>(op, TypeRange(), flattened,
1286 ArrayRef<NamedAttribute>());
1287 return success();
1288 }
1289};
1290/// Replace with valid op, but simply drop the operands. This is used in a
1291/// regression where we used to generate circular unrealized_conversion_cast
1292/// ops.
1293struct TestDropAndReplaceInvalidOp : public ConversionPattern {
1294 TestDropAndReplaceInvalidOp(MLIRContext *ctx, const TypeConverter &converter)
1295 : ConversionPattern(converter,
1296 "test.drop_operands_and_replace_with_valid", 1, ctx) {
1297 }
1298 LogicalResult
1299 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1300 ConversionPatternRewriter &rewriter) const final {
1301 rewriter.replaceOpWithNewOp<TestValidOp>(op, TypeRange(), ValueRange(),
1302 ArrayRef<NamedAttribute>());
1303 return success();
1304 }
1305};
1306/// This pattern handles the case of a split return value.
1307struct TestSplitReturnType : public ConversionPattern {
1308 TestSplitReturnType(MLIRContext *ctx)
1309 : ConversionPattern("test.return", 1, ctx) {}
1310 LogicalResult
1311 matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
1312 ConversionPatternRewriter &rewriter) const final {
1313 // Check for a return of F32.
1314 if (op->getNumOperands() != 1 || !op->getOperand(idx: 0).getType().isF32())
1315 return failure();
1316 rewriter.replaceOpWithNewOp<TestReturnOp>(op, operands[0]);
1317 return success();
1318 }
1319};
1320
1321//===----------------------------------------------------------------------===//
1322// Multi-Level Type-Conversion Rewrite Testing
1323struct TestChangeProducerTypeI32ToF32 : public ConversionPattern {
1324 TestChangeProducerTypeI32ToF32(MLIRContext *ctx)
1325 : ConversionPattern("test.type_producer", 1, ctx) {}
1326 LogicalResult
1327 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1328 ConversionPatternRewriter &rewriter) const final {
1329 // If the type is I32, change the type to F32.
1330 if (!Type(*op->result_type_begin()).isSignlessInteger(width: 32))
1331 return failure();
1332 rewriter.replaceOpWithNewOp<TestTypeProducerOp>(op, rewriter.getF32Type());
1333 return success();
1334 }
1335};
1336struct TestChangeProducerTypeF32ToF64 : public ConversionPattern {
1337 TestChangeProducerTypeF32ToF64(MLIRContext *ctx)
1338 : ConversionPattern("test.type_producer", 1, ctx) {}
1339 LogicalResult
1340 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1341 ConversionPatternRewriter &rewriter) const final {
1342 // If the type is F32, change the type to F64.
1343 if (!Type(*op->result_type_begin()).isF32())
1344 return rewriter.notifyMatchFailure(arg&: op, msg: "expected single f32 operand");
1345 rewriter.replaceOpWithNewOp<TestTypeProducerOp>(op, rewriter.getF64Type());
1346 return success();
1347 }
1348};
1349struct TestChangeProducerTypeF32ToInvalid : public ConversionPattern {
1350 TestChangeProducerTypeF32ToInvalid(MLIRContext *ctx)
1351 : ConversionPattern("test.type_producer", 10, ctx) {}
1352 LogicalResult
1353 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1354 ConversionPatternRewriter &rewriter) const final {
1355 // Always convert to B16, even though it is not a legal type. This tests
1356 // that values are unmapped correctly.
1357 rewriter.replaceOpWithNewOp<TestTypeProducerOp>(op, rewriter.getBF16Type());
1358 return success();
1359 }
1360};
1361struct TestUpdateConsumerType : public ConversionPattern {
1362 TestUpdateConsumerType(MLIRContext *ctx)
1363 : ConversionPattern("test.type_consumer", 1, ctx) {}
1364 LogicalResult
1365 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1366 ConversionPatternRewriter &rewriter) const final {
1367 // Verify that the incoming operand has been successfully remapped to F64.
1368 if (!operands[0].getType().isF64())
1369 return failure();
1370 rewriter.replaceOpWithNewOp<TestTypeConsumerOp>(op, operands[0]);
1371 return success();
1372 }
1373};
1374
1375//===----------------------------------------------------------------------===//
1376// Non-Root Replacement Rewrite Testing
1377/// This pattern generates an invalid operation, but replaces it before the
1378/// pattern is finished. This checks that we don't need to legalize the
1379/// temporary op.
1380struct TestNonRootReplacement : public RewritePattern {
1381 TestNonRootReplacement(MLIRContext *ctx)
1382 : RewritePattern("test.replace_non_root", 1, ctx) {}
1383
1384 LogicalResult matchAndRewrite(Operation *op,
1385 PatternRewriter &rewriter) const final {
1386 auto resultType = *op->result_type_begin();
1387 auto illegalOp = ILLegalOpF::create(rewriter, op->getLoc(), resultType);
1388 auto legalOp = LegalOpB::create(rewriter, op->getLoc(), resultType);
1389
1390 rewriter.replaceOp(op, illegalOp);
1391 rewriter.replaceOp(illegalOp, legalOp);
1392 return success();
1393 }
1394};
1395
1396//===----------------------------------------------------------------------===//
1397// Recursive Rewrite Testing
1398/// This pattern is applied to the same operation multiple times, but has a
1399/// bounded recursion.
1400struct TestBoundedRecursiveRewrite
1401 : public OpRewritePattern<TestRecursiveRewriteOp> {
1402 using OpRewritePattern<TestRecursiveRewriteOp>::OpRewritePattern;
1403
1404 void initialize() {
1405 // The conversion target handles bounding the recursion of this pattern.
1406 setHasBoundedRewriteRecursion();
1407 }
1408
1409 LogicalResult matchAndRewrite(TestRecursiveRewriteOp op,
1410 PatternRewriter &rewriter) const final {
1411 // Decrement the depth of the op in-place.
1412 rewriter.modifyOpInPlace(op, [&] { op.setDepth(op.getDepth() - 1); });
1413 return success();
1414 }
1415};
1416
1417struct TestNestedOpCreationUndoRewrite
1418 : public OpRewritePattern<IllegalOpWithRegionAnchor> {
1419 using OpRewritePattern<IllegalOpWithRegionAnchor>::OpRewritePattern;
1420
1421 LogicalResult matchAndRewrite(IllegalOpWithRegionAnchor op,
1422 PatternRewriter &rewriter) const final {
1423 // rewriter.replaceOpWithNewOp<IllegalOpWithRegion>(op);
1424 rewriter.replaceOpWithNewOp<IllegalOpWithRegion>(op);
1425 return success();
1426 };
1427};
1428
1429// This pattern matches `test.blackhole` and delete this op and its producer.
1430struct TestReplaceEraseOp : public OpRewritePattern<BlackHoleOp> {
1431 using OpRewritePattern<BlackHoleOp>::OpRewritePattern;
1432
1433 LogicalResult matchAndRewrite(BlackHoleOp op,
1434 PatternRewriter &rewriter) const final {
1435 Operation *producer = op.getOperand().getDefiningOp();
1436 // Always erase the user before the producer, the framework should handle
1437 // this correctly.
1438 rewriter.eraseOp(op: op);
1439 rewriter.eraseOp(op: producer);
1440 return success();
1441 };
1442};
1443
1444// This pattern replaces explicitly illegal op with explicitly legal op,
1445// but in addition creates unregistered operation.
1446struct TestCreateUnregisteredOp : public OpRewritePattern<ILLegalOpG> {
1447 using OpRewritePattern<ILLegalOpG>::OpRewritePattern;
1448
1449 LogicalResult matchAndRewrite(ILLegalOpG op,
1450 PatternRewriter &rewriter) const final {
1451 IntegerAttr attr = rewriter.getI32IntegerAttr(value: 0);
1452 Value val = arith::ConstantOp::create(rewriter, op->getLoc(), attr);
1453 rewriter.replaceOpWithNewOp<LegalOpC>(op, val);
1454 return success();
1455 };
1456};
1457
1458class TestEraseOp : public ConversionPattern {
1459public:
1460 TestEraseOp(MLIRContext *ctx) : ConversionPattern("test.erase_op", 1, ctx) {}
1461 LogicalResult
1462 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1463 ConversionPatternRewriter &rewriter) const final {
1464 // Erase op without replacements.
1465 rewriter.eraseOp(op);
1466 return success();
1467 }
1468};
1469
1470/// Pattern that replaces test.replace_with_valid_producer with
1471/// test.valid_producer and the specified type.
1472class TestReplaceWithValidProducer : public ConversionPattern {
1473public:
1474 TestReplaceWithValidProducer(MLIRContext *ctx)
1475 : ConversionPattern("test.replace_with_valid_producer", 1, ctx) {}
1476 LogicalResult
1477 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1478 ConversionPatternRewriter &rewriter) const final {
1479 auto attr = op->getDiscardableAttrOfType<TypeAttr>(name: "type");
1480 if (!attr)
1481 return failure();
1482 rewriter.replaceOpWithNewOp<TestValidProducerOp>(op, attr.getValue());
1483 return success();
1484 }
1485};
1486
1487/// Pattern that replaces test.replace_with_valid_consumer with
1488/// test.valid_consumer. Can be used with and without a type converter.
1489class TestReplaceWithValidConsumer : public ConversionPattern {
1490public:
1491 TestReplaceWithValidConsumer(MLIRContext *ctx, const TypeConverter &converter)
1492 : ConversionPattern(converter, "test.replace_with_valid_consumer", 1,
1493 ctx) {}
1494 TestReplaceWithValidConsumer(MLIRContext *ctx)
1495 : ConversionPattern("test.replace_with_valid_consumer", 1, ctx) {}
1496
1497 LogicalResult
1498 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
1499 ConversionPatternRewriter &rewriter) const final {
1500 // with_converter present: pattern must have been initialized with a type
1501 // converter.
1502 // with_converter absent: pattern must have been initialized without a type
1503 // converter.
1504 if (op->hasDiscardableAttr(name: "with_converter") !=
1505 static_cast<bool>(getTypeConverter()))
1506 return failure();
1507 rewriter.replaceOpWithNewOp<TestValidConsumerOp>(op, operands[0]);
1508 return success();
1509 }
1510};
1511
1512/// This pattern matches a test.convert_block_args op. It either:
1513/// a) Duplicates all block arguments,
1514/// b) or: drops all block arguments and replaces each with 2x the first
1515/// operand.
1516class TestConvertBlockArgs : public OpConversionPattern<ConvertBlockArgsOp> {
1517 static_assert(std::is_same_v<Base, OpConversionPattern<ConvertBlockArgsOp>>);
1518 using Base::Base;
1519
1520 LogicalResult
1521 matchAndRewrite(ConvertBlockArgsOp op, OpAdaptor adaptor,
1522 ConversionPatternRewriter &rewriter) const override {
1523 if (op.getIsLegal())
1524 return failure();
1525 Block *body = &op.getBody().front();
1526 TypeConverter::SignatureConversion result(body->getNumArguments());
1527 for (auto it : llvm::enumerate(body->getArgumentTypes())) {
1528 if (op.getReplaceWithOperand()) {
1529 result.remapInput(it.index(), {adaptor.getVal(), adaptor.getVal()});
1530 } else if (op.getDuplicate()) {
1531 result.addInputs(it.index(), {it.value(), it.value()});
1532 } else {
1533 // No action specified. Pattern does not apply.
1534 return failure();
1535 }
1536 }
1537 rewriter.startOpModification(op: op);
1538 rewriter.applySignatureConversion(block: body, conversion&: result, converter: getTypeConverter());
1539 op.setIsLegal(true);
1540 rewriter.finalizeOpModification(op: op);
1541 return success();
1542 }
1543};
1544
1545/// This pattern replaces test.repetitive_1_to_n_consumer ops with a test.valid
1546/// op. The pattern supports 1:N replacements and forwards the replacement
1547/// values of the single operand as test.valid operands.
1548class TestRepetitive1ToNConsumer : public ConversionPattern {
1549public:
1550 TestRepetitive1ToNConsumer(MLIRContext *ctx)
1551 : ConversionPattern("test.repetitive_1_to_n_consumer", 1, ctx) {}
1552 LogicalResult
1553 matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
1554 ConversionPatternRewriter &rewriter) const final {
1555 // A single operand is expected.
1556 if (op->getNumOperands() != 1)
1557 return failure();
1558 rewriter.replaceOpWithNewOp<TestValidOp>(op, operands.front());
1559 return success();
1560 }
1561};
1562
1563/// A pattern that tests two back-to-back 1 -> 2 op replacements.
1564class TestMultiple1ToNReplacement : public ConversionPattern {
1565public:
1566 TestMultiple1ToNReplacement(MLIRContext *ctx, const TypeConverter &converter)
1567 : ConversionPattern(converter, "test.multiple_1_to_n_replacement", 1,
1568 ctx) {}
1569 LogicalResult
1570 matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
1571 ConversionPatternRewriter &rewriter) const final {
1572 // Helper function that replaces the given op with a new op of the given
1573 // name and doubles each result (1 -> 2 replacement of each result).
1574 auto replaceWithDoubleResults = [&](Operation *op, StringRef name) {
1575 rewriter.setInsertionPointAfter(op);
1576 SmallVector<Type> types;
1577 for (Type t : op->getResultTypes()) {
1578 types.push_back(Elt: t);
1579 types.push_back(Elt: t);
1580 }
1581 OperationState state(op->getLoc(), name,
1582 /*operands=*/{}, types,
1583 op->getDiscardableAttrDictionary().getValue());
1584 auto *newOp = rewriter.create(state);
1585 SmallVector<ValueRange> repls;
1586 for (size_t i = 0, e = op->getNumResults(); i < e; ++i)
1587 repls.push_back(Elt: newOp->getResults().slice(n: 2 * i, m: 2));
1588 rewriter.replaceOpWithMultiple(op, newValues&: repls);
1589 return newOp;
1590 };
1591
1592 // Replace test.multiple_1_to_n_replacement with test.step_1.
1593 Operation *repl1 = replaceWithDoubleResults(op, "test.step_1");
1594 // Now replace test.step_1 with test.legal_op.
1595 replaceWithDoubleResults(repl1, "test.legal_op");
1596 return success();
1597 }
1598};
1599
1600/// Pattern that erases 'test.type_consumers' iff the input operand is the
1601/// result of a 1:1 type conversion.
1602/// Used to test correct skipping of 1:1 patterns in the 1:N case.
1603class TestTypeConsumerOpPattern
1604 : public OpConversionPattern<TestTypeConsumerOp> {
1605public:
1606 TestTypeConsumerOpPattern(MLIRContext *ctx, const TypeConverter &converter)
1607 : OpConversionPattern<TestTypeConsumerOp>(converter, ctx) {}
1608
1609 LogicalResult
1610 matchAndRewrite(TestTypeConsumerOp op, OpAdaptor operands,
1611 ConversionPatternRewriter &rewriter) const final {
1612 rewriter.eraseOp(op: op);
1613 return success();
1614 }
1615};
1616
1617class TestPostOrderLegalization : public ConversionPattern {
1618public:
1619 TestPostOrderLegalization(MLIRContext *ctx, const TypeConverter &converter)
1620 : ConversionPattern(converter, "test.post_order_legalization", 1, ctx) {}
1621 LogicalResult
1622 matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
1623 ConversionPatternRewriter &rewriter) const final {
1624 for (Region &r : op->getRegions())
1625 if (failed(Result: rewriter.legalize(r: &r)))
1626 return failure();
1627 rewriter.modifyOpInPlace(root: op, callable: [&]() {
1628 op->setDiscardableAttr(name: "is_legal", value: rewriter.getUnitAttr());
1629 });
1630 return success();
1631 }
1632};
1633
1634/// Test unambiguous overload resolution of replaceOpWithMultiple. This
1635/// function is just to trigger compiler errors. It is never executed.
1636[[maybe_unused]] void testReplaceOpWithMultipleOverloads(
1637 ConversionPatternRewriter &rewriter, Operation *op, ArrayRef<ValueRange> r1,
1638 SmallVector<ValueRange> r2, ArrayRef<SmallVector<Value>> r3,
1639 SmallVector<SmallVector<Value>> r4, ArrayRef<ArrayRef<Value>> r5,
1640 SmallVector<ArrayRef<Value>> r6, SmallVector<SmallVector<Value>> &&r7,
1641 Value v, ValueRange vr, ArrayRef<Value> ar) {
1642 rewriter.replaceOpWithMultiple(op, newValues: r1);
1643 rewriter.replaceOpWithMultiple(op, newValues&: r2);
1644 rewriter.replaceOpWithMultiple(op, newValues: r3);
1645 rewriter.replaceOpWithMultiple(op, newValues&: r4);
1646 rewriter.replaceOpWithMultiple(op, newValues: r5);
1647 rewriter.replaceOpWithMultiple(op, newValues&: r6);
1648 rewriter.replaceOpWithMultiple(op, newValues: std::move(r7));
1649 rewriter.replaceOpWithMultiple(op, newValues: {vr});
1650 rewriter.replaceOpWithMultiple(op, newValues: {ar});
1651 rewriter.replaceOpWithMultiple(op, newValues: {{v}});
1652 rewriter.replaceOpWithMultiple(op, newValues: {{v, v}});
1653 rewriter.replaceOpWithMultiple(op, newValues: {{v, v}, vr});
1654 rewriter.replaceOpWithMultiple(op, newValues: {{v, v}, ar});
1655 rewriter.replaceOpWithMultiple(op, newValues: {ar, {v, v}, vr});
1656}
1657} // namespace
1658
1659namespace {
1660struct TestTypeConverter : public TypeConverter {
1661 static_assert(std::is_same_v<Base, TypeConverter>);
1662 using Base::Base;
1663
1664 TestTypeConverter() {
1665 addConversion(callback&: convertType);
1666 addSourceMaterialization(callback&: materializeCast);
1667 addTargetMaterialization(callback&: materializeCast);
1668 }
1669
1670 static LogicalResult convertType(Type t, SmallVectorImpl<Type> &results) {
1671 // Drop I16 types.
1672 if (t.isSignlessInteger(width: 16))
1673 return success();
1674
1675 // Convert I64 to F64.
1676 if (t.isSignlessInteger(width: 64)) {
1677 results.push_back(Elt: Float64Type::get(context: t.getContext()));
1678 return success();
1679 }
1680
1681 // Convert I42 to I43.
1682 if (t.isInteger(width: 42)) {
1683 results.push_back(Elt: IntegerType::get(context: t.getContext(), width: 43));
1684 return success();
1685 }
1686
1687 // Split F32 into F16,F16.
1688 if (t.isF32()) {
1689 results.append(NumInputs: 2, Elt: Float16Type::get(context: t.getContext()));
1690 return success();
1691 }
1692
1693 // Drop I24 types.
1694 if (t.isInteger(width: 24)) {
1695 return success();
1696 }
1697
1698 // Otherwise, convert the type directly.
1699 results.push_back(Elt: t);
1700 return success();
1701 }
1702
1703 /// Hook for materializing a conversion. This is necessary because we generate
1704 /// 1->N type mappings.
1705 static Value materializeCast(OpBuilder &builder, Type resultType,
1706 ValueRange inputs, Location loc) {
1707 return TestCastOp::create(builder, loc, resultType, inputs).getResult();
1708 }
1709};
1710
1711struct TestLegalizePatternDriver
1712 : public PassWrapper<TestLegalizePatternDriver, OperationPass<>> {
1713 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestLegalizePatternDriver)
1714
1715 TestLegalizePatternDriver() = default;
1716 TestLegalizePatternDriver(const TestLegalizePatternDriver &other)
1717 : PassWrapper(other) {}
1718
1719 StringRef getArgument() const final { return "test-legalize-patterns"; }
1720 StringRef getDescription() const final {
1721 return "Run test dialect legalization patterns";
1722 }
1723 /// The mode of conversion to use with the driver.
1724 enum class ConversionMode { Analysis, Full, Partial };
1725
1726 void getDependentDialects(DialectRegistry &registry) const override {
1727 registry.insert<func::FuncDialect, test::TestDialect>();
1728 }
1729
1730 void runOnOperation() override {
1731 TestTypeConverter converter;
1732 mlir::RewritePatternSet patterns(&getContext());
1733 populateWithGenerated(patterns);
1734 patterns.add<
1735 TestRegionRewriteBlockMovement, TestDetachedSignatureConversion,
1736 TestRegionRewriteUndo, TestCreateBlock, TestCreateIllegalBlock,
1737 TestUndoBlockErase, TestSplitReturnType, TestChangeProducerTypeI32ToF32,
1738 TestChangeProducerTypeF32ToF64, TestChangeProducerTypeF32ToInvalid,
1739 TestUpdateConsumerType, TestNonRootReplacement,
1740 TestBoundedRecursiveRewrite, TestNestedOpCreationUndoRewrite,
1741 TestReplaceEraseOp, TestCreateUnregisteredOp, TestUndoMoveOpBefore,
1742 TestUndoPropertiesModification, TestUndoMoveDetachedBlock, TestEraseOp,
1743 TestReplaceWithValidProducer, TestReplaceWithValidConsumer,
1744 TestRepetitive1ToNConsumer>(arg: &getContext());
1745 patterns.add<TestDropOpSignatureConversion, TestDropAndReplaceInvalidOp,
1746 TestPassthroughInvalidOp, TestMultiple1ToNReplacement,
1747 TestValueReplace, TestReplaceWithValidConsumer,
1748 TestTypeConsumerOpPattern, TestPostOrderLegalization>(
1749 arg: &getContext(), args&: converter);
1750 patterns.add<TestConvertBlockArgs>(arg&: converter, args: &getContext());
1751 mlir::populateAnyFunctionOpInterfaceTypeConversionPattern(patterns,
1752 converter);
1753 mlir::populateCallOpTypeConversionPattern(patterns, converter);
1754
1755 // Define the conversion target used for the test.
1756 ConversionTarget target(getContext());
1757 target.addLegalOp<ModuleOp>();
1758 target.addLegalOp<LegalOpA, LegalOpB, LegalOpC, TestCastOp, TestValidOp,
1759 TerminatorOp, TestOpConstant, OneRegionOp,
1760 TestValidProducerOp, TestValidConsumerOp>();
1761 target.addLegalOp(op: OperationName("test.legal_op", &getContext()));
1762 target
1763 .addIllegalOp<ILLegalOpF, TestRegionBuilderOp, TestOpWithRegionFold>();
1764 target.addDynamicallyLegalOp<TestReturnOp>([](TestReturnOp op) {
1765 // Don't allow F32 operands.
1766 return llvm::none_of(op.getOperandTypes(),
1767 [](Type type) { return type.isF32(); });
1768 });
1769 target.addDynamicallyLegalOp<func::FuncOp>(callback: [&](func::FuncOp op) {
1770 return converter.isSignatureLegal(ty: op.getFunctionType());
1771 });
1772 target.addDynamicallyLegalOp<func::CallOp>(
1773 callback: [&](func::CallOp op) { return converter.isLegal(op); });
1774 target.addDynamicallyLegalOp(
1775 op: OperationName("test.value_replace", &getContext()),
1776 callback: [](Operation *op) { return op->hasDiscardableAttr(name: "is_legal"); });
1777 target.addDynamicallyLegalOp(
1778 op: OperationName("test.post_order_legalization", &getContext()),
1779 callback: [](Operation *op) { return op->hasDiscardableAttr(name: "is_legal"); });
1780
1781 // TestCreateUnregisteredOp creates `arith.constant` operation,
1782 // which was not added to target intentionally to test
1783 // correct error code from conversion driver.
1784 target.addDynamicallyLegalOp<ILLegalOpG>([](ILLegalOpG) { return false; });
1785
1786 // Expect the type_producer/type_consumer operations to only operate on f64.
1787 target.addDynamicallyLegalOp<TestTypeProducerOp>(
1788 [](TestTypeProducerOp op) { return op.getType().isF64(); });
1789 target.addDynamicallyLegalOp<TestTypeConsumerOp>([](TestTypeConsumerOp op) {
1790 return op.getOperand().getType().isF64();
1791 });
1792
1793 // Check support for marking certain operations as recursively legal.
1794 target.markOpRecursivelyLegal<func::FuncOp, ModuleOp>(callback: [](Operation *op) {
1795 return static_cast<bool>(
1796 op->getDiscardableAttrOfType<UnitAttr>(name: "test.recursively_legal"));
1797 });
1798
1799 // Mark the bound recursion operation as dynamically legal.
1800 target.addDynamicallyLegalOp<TestRecursiveRewriteOp>(
1801 [](TestRecursiveRewriteOp op) { return op.getDepth() == 0; });
1802
1803 // Create a dynamically legal rule that can only be legalized by folding it.
1804 target.addDynamicallyLegalOp<TestOpInPlaceSelfFold>(
1805 [](TestOpInPlaceSelfFold op) { return op.getFolded(); });
1806
1807 target.addDynamicallyLegalOp<ConvertBlockArgsOp>(
1808 [](ConvertBlockArgsOp op) { return op.getIsLegal(); });
1809
1810 // Set up configuration.
1811 ConversionConfig config;
1812 config.allowPatternRollback = allowPatternRollback;
1813 config.foldingMode = foldingMode;
1814 config.buildMaterializations = buildMaterializations;
1815 config.attachDebugMaterializationKind = attachDebugMaterializationKind;
1816 DumpNotifications dumpNotifications;
1817 config.listener = &dumpNotifications;
1818
1819 // Handle a partial conversion.
1820 if (mode == ConversionMode::Partial) {
1821 DenseSet<Operation *> unlegalizedOps;
1822 config.unlegalizedOps = &unlegalizedOps;
1823 if (failed(Result: applyPartialConversion(op: getOperation(), target,
1824 patterns: std::move(patterns), config))) {
1825 getOperation()->emitRemark() << "applyPartialConversion failed";
1826 }
1827 // Emit remarks for each legalizable operation.
1828 for (auto *op : unlegalizedOps)
1829 op->emitRemark() << "op '" << op->getName() << "' is not legalizable";
1830 return;
1831 }
1832
1833 // Handle a full conversion.
1834 if (mode == ConversionMode::Full) {
1835 // Check support for marking unknown operations as dynamically legal.
1836 target.markUnknownOpDynamicallyLegal(fn: [](Operation *op) {
1837 return (bool)op->getDiscardableAttrOfType<UnitAttr>(
1838 name: "test.dynamically_legal");
1839 });
1840
1841 if (failed(Result: applyFullConversion(op: getOperation(), target,
1842 patterns: std::move(patterns), config))) {
1843 getOperation()->emitRemark() << "applyFullConversion failed";
1844 }
1845 return;
1846 }
1847
1848 // Otherwise, handle an analysis conversion.
1849 assert(mode == ConversionMode::Analysis);
1850
1851 // Analyze the convertible operations.
1852 DenseSet<Operation *> legalizedOps;
1853 config.legalizableOps = &legalizedOps;
1854 if (failed(Result: applyAnalysisConversion(op: getOperation(), target,
1855 patterns: std::move(patterns), config)))
1856 return signalPassFailure();
1857
1858 // Emit remarks for each legalizable operation.
1859 for (auto *op : legalizedOps)
1860 op->emitRemark() << "op '" << op->getName() << "' is legalizable";
1861 }
1862
1863 Option<ConversionMode> mode{
1864 *this, "test-legalize-mode",
1865 llvm::cl::desc("The legalization mode to use with the test driver"),
1866 llvm::cl::init(Val: ConversionMode::Partial),
1867 llvm::cl::values(
1868 clEnumValN(ConversionMode::Analysis, "analysis",
1869 "Perform an analysis conversion"),
1870 clEnumValN(ConversionMode::Full, "full", "Perform a full conversion"),
1871 clEnumValN(ConversionMode::Partial, "partial",
1872 "Perform a partial conversion"))};
1873
1874 Option<DialectConversionFoldingMode> foldingMode{
1875 *this, "test-legalize-folding-mode",
1876 llvm::cl::desc("The folding mode to use with the test driver"),
1877 llvm::cl::init(Val: DialectConversionFoldingMode::BeforePatterns),
1878 llvm::cl::values(clEnumValN(DialectConversionFoldingMode::Never, "never",
1879 "Never attempt to fold"),
1880 clEnumValN(DialectConversionFoldingMode::BeforePatterns,
1881 "before-patterns",
1882 "Only attempt to fold not legal operations "
1883 "before applying patterns"),
1884 clEnumValN(DialectConversionFoldingMode::AfterPatterns,
1885 "after-patterns",
1886 "Only attempt to fold not legal operations "
1887 "after applying patterns"))};
1888 Option<bool> allowPatternRollback{*this, "allow-pattern-rollback",
1889 llvm::cl::desc("Allow pattern rollback"),
1890 llvm::cl::init(Val: true)};
1891 Option<bool> attachDebugMaterializationKind{
1892 *this, "attach-debug-materialization-kind",
1893 llvm::cl::desc(
1894 "Attach materialization kind to unrealized_conversion_cast ops"),
1895 llvm::cl::init(Val: false)};
1896 Option<bool> buildMaterializations{
1897 *this, "build-materializations",
1898 llvm::cl::desc(
1899 "If set to 'false', leave unrealized_conversion_cast ops in place"),
1900 llvm::cl::init(Val: true)};
1901};
1902} // namespace
1903
1904//===----------------------------------------------------------------------===//
1905// ConversionPatternRewriter::getRemappedValue testing. This method is used
1906// to get the remapped value of an original value that was replaced using
1907// ConversionPatternRewriter.
1908namespace {
1909struct TestRemapValueTypeConverter : public TypeConverter {
1910 using Base::Base;
1911
1912 TestRemapValueTypeConverter() {
1913 addConversion(
1914 callback: [](Float32Type type) { return Float64Type::get(context: type.getContext()); });
1915 addConversion(callback: [](Type type) { return type; });
1916 }
1917};
1918
1919/// Converter that replaces a one-result one-operand OneVResOneVOperandOp1 with
1920/// a one-operand two-result OneVResOneVOperandOp1 by replicating its original
1921/// operand twice.
1922///
1923/// Example:
1924/// %1 = test.one_variadic_out_one_variadic_in1"(%0)
1925/// is replaced with:
1926/// %1 = test.one_variadic_out_one_variadic_in1"(%0, %0)
1927struct OneVResOneVOperandOp1Converter
1928 : public OpConversionPattern<OneVResOneVOperandOp1> {
1929 using OpConversionPattern<OneVResOneVOperandOp1>::OpConversionPattern;
1930
1931 LogicalResult
1932 matchAndRewrite(OneVResOneVOperandOp1 op, OpAdaptor adaptor,
1933 ConversionPatternRewriter &rewriter) const override {
1934 auto origOps = op.getOperands();
1935 assert(std::distance(origOps.begin(), origOps.end()) == 1 &&
1936 "One operand expected");
1937 Value origOp = *origOps.begin();
1938 SmallVector<Value, 2> remappedOperands;
1939 // Replicate the remapped original operand twice. Note that we don't used
1940 // the remapped 'operand' since the goal is testing 'getRemappedValue'.
1941 remappedOperands.push_back(Elt: rewriter.getRemappedValue(key: origOp));
1942 remappedOperands.push_back(Elt: rewriter.getRemappedValue(key: origOp));
1943
1944 rewriter.replaceOpWithNewOp<OneVResOneVOperandOp1>(op, op.getResultTypes(),
1945 remappedOperands);
1946 return success();
1947 }
1948};
1949
1950/// A rewriter pattern that tests that blocks can be merged.
1951struct TestRemapValueInRegion
1952 : public OpConversionPattern<TestRemappedValueRegionOp> {
1953 using OpConversionPattern<TestRemappedValueRegionOp>::OpConversionPattern;
1954
1955 LogicalResult
1956 matchAndRewrite(TestRemappedValueRegionOp op, OpAdaptor adaptor,
1957 ConversionPatternRewriter &rewriter) const final {
1958 Block &block = op.getBody().front();
1959 Operation *terminator = block.getTerminator();
1960
1961 // Merge the block into the parent region.
1962 Block *parentBlock = op->getBlock();
1963 Block *finalBlock = rewriter.splitBlock(block: parentBlock, before: op->getIterator());
1964 rewriter.mergeBlocks(source: &block, dest: parentBlock, argValues: ValueRange());
1965 rewriter.mergeBlocks(source: finalBlock, dest: parentBlock, argValues: ValueRange());
1966
1967 // Replace the results of this operation with the remapped terminator
1968 // values.
1969 SmallVector<Value> terminatorOperands;
1970 if (failed(Result: rewriter.getRemappedValues(keys: terminator->getOperands(),
1971 results&: terminatorOperands)))
1972 return failure();
1973
1974 rewriter.eraseOp(op: terminator);
1975 rewriter.replaceOp(op, terminatorOperands);
1976 return success();
1977 }
1978};
1979
1980struct TestRemappedValue
1981 : public mlir::PassWrapper<TestRemappedValue, OperationPass<>> {
1982 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestRemappedValue)
1983
1984 StringRef getArgument() const final { return "test-remapped-value"; }
1985 StringRef getDescription() const final {
1986 return "Test public remapped value mechanism in ConversionPatternRewriter";
1987 }
1988 void runOnOperation() override {
1989 TestRemapValueTypeConverter typeConverter;
1990
1991 mlir::RewritePatternSet patterns(&getContext());
1992 patterns.add<OneVResOneVOperandOp1Converter>(arg: &getContext());
1993 patterns.add<TestChangeProducerTypeF32ToF64, TestUpdateConsumerType>(
1994 arg: &getContext());
1995 patterns.add<TestRemapValueInRegion>(arg&: typeConverter, args: &getContext());
1996
1997 mlir::ConversionTarget target(getContext());
1998 target.addLegalOp<ModuleOp, func::FuncOp, TestReturnOp>();
1999
2000 // Expect the type_producer/type_consumer operations to only operate on f64.
2001 target.addDynamicallyLegalOp<TestTypeProducerOp>(
2002 [](TestTypeProducerOp op) { return op.getType().isF64(); });
2003 target.addDynamicallyLegalOp<TestTypeConsumerOp>([](TestTypeConsumerOp op) {
2004 return op.getOperand().getType().isF64();
2005 });
2006
2007 // We make OneVResOneVOperandOp1 legal only when it has more that one
2008 // operand. This will trigger the conversion that will replace one-operand
2009 // OneVResOneVOperandOp1 with two-operand OneVResOneVOperandOp1.
2010 target.addDynamicallyLegalOp<OneVResOneVOperandOp1>(
2011 [](Operation *op) { return op->getNumOperands() > 1; });
2012
2013 if (failed(Result: mlir::applyFullConversion(op: getOperation(), target,
2014 patterns: std::move(patterns)))) {
2015 signalPassFailure();
2016 }
2017 }
2018};
2019} // namespace
2020
2021//===----------------------------------------------------------------------===//
2022// Test patterns without a specific root operation kind
2023//===----------------------------------------------------------------------===//
2024
2025namespace {
2026/// This pattern matches and removes any operation in the test dialect.
2027struct RemoveTestDialectOps : public RewritePattern {
2028 RemoveTestDialectOps(MLIRContext *context)
2029 : RewritePattern(MatchAnyOpTypeTag(), /*benefit=*/1, context) {}
2030
2031 LogicalResult matchAndRewrite(Operation *op,
2032 PatternRewriter &rewriter) const override {
2033 if (!isa<TestDialect>(Val: op->getDialect()))
2034 return failure();
2035 rewriter.eraseOp(op);
2036 return success();
2037 }
2038};
2039
2040struct TestUnknownRootOpDriver
2041 : public mlir::PassWrapper<TestUnknownRootOpDriver, OperationPass<>> {
2042 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestUnknownRootOpDriver)
2043
2044 StringRef getArgument() const final {
2045 return "test-legalize-unknown-root-patterns";
2046 }
2047 StringRef getDescription() const final {
2048 return "Test public remapped value mechanism in ConversionPatternRewriter";
2049 }
2050 void runOnOperation() override {
2051 mlir::RewritePatternSet patterns(&getContext());
2052 patterns.add<RemoveTestDialectOps>(arg: &getContext());
2053
2054 mlir::ConversionTarget target(getContext());
2055 target.addIllegalDialect<TestDialect>();
2056 if (failed(Result: applyPartialConversion(op: getOperation(), target,
2057 patterns: std::move(patterns))))
2058 signalPassFailure();
2059 }
2060};
2061} // namespace
2062
2063//===----------------------------------------------------------------------===//
2064// Test patterns that uses operations and types defined at runtime
2065//===----------------------------------------------------------------------===//
2066
2067namespace {
2068/// This pattern matches dynamic operations 'test.one_operand_two_results' and
2069/// replace them with dynamic operations 'test.generic_dynamic_op'.
2070struct RewriteDynamicOp : public RewritePattern {
2071 RewriteDynamicOp(MLIRContext *context)
2072 : RewritePattern("test.dynamic_one_operand_two_results", /*benefit=*/1,
2073 context) {}
2074
2075 LogicalResult matchAndRewrite(Operation *op,
2076 PatternRewriter &rewriter) const override {
2077 assert(op->getName().getStringRef() ==
2078 "test.dynamic_one_operand_two_results" &&
2079 "rewrite pattern should only match operations with the right name");
2080
2081 OperationState state(op->getLoc(), "test.dynamic_generic",
2082 op->getOperands(), op->getResultTypes(),
2083 op->getDiscardableAttrDictionary().getValue());
2084 auto *newOp = rewriter.create(state);
2085 rewriter.replaceOp(op, newValues: newOp->getResults());
2086 return success();
2087 }
2088};
2089
2090struct TestRewriteDynamicOpDriver
2091 : public PassWrapper<TestRewriteDynamicOpDriver, OperationPass<>> {
2092 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestRewriteDynamicOpDriver)
2093
2094 void getDependentDialects(DialectRegistry &registry) const override {
2095 registry.insert<TestDialect>();
2096 }
2097 StringRef getArgument() const final { return "test-rewrite-dynamic-op"; }
2098 StringRef getDescription() const final {
2099 return "Test rewritting on dynamic operations";
2100 }
2101 void runOnOperation() override {
2102 RewritePatternSet patterns(&getContext());
2103 patterns.add<RewriteDynamicOp>(arg: &getContext());
2104
2105 ConversionTarget target(getContext());
2106 target.addIllegalOp(
2107 op: OperationName("test.dynamic_one_operand_two_results", &getContext()));
2108 target.addLegalOp(op: OperationName("test.dynamic_generic", &getContext()));
2109 if (failed(Result: applyPartialConversion(op: getOperation(), target,
2110 patterns: std::move(patterns))))
2111 signalPassFailure();
2112 }
2113};
2114} // end anonymous namespace
2115
2116//===----------------------------------------------------------------------===//
2117// Test type conversions
2118//===----------------------------------------------------------------------===//
2119
2120namespace {
2121struct TestTypeConversionProducer
2122 : public OpConversionPattern<TestTypeProducerOp> {
2123 using OpConversionPattern<TestTypeProducerOp>::OpConversionPattern;
2124 LogicalResult
2125 matchAndRewrite(TestTypeProducerOp op, OpAdaptor adaptor,
2126 ConversionPatternRewriter &rewriter) const final {
2127 Type resultType = op.getType();
2128 Type convertedType = getTypeConverter()
2129 ? getTypeConverter()->convertType(resultType)
2130 : resultType;
2131 if (isa<FloatType>(Val: resultType))
2132 resultType = rewriter.getF64Type();
2133 else if (resultType.isInteger(width: 16))
2134 resultType = rewriter.getIntegerType(width: 64);
2135 else if (isa<test::TestRecursiveType>(Val: resultType) &&
2136 convertedType != resultType)
2137 resultType = convertedType;
2138 else
2139 return failure();
2140
2141 rewriter.replaceOpWithNewOp<TestTypeProducerOp>(op, resultType);
2142 return success();
2143 }
2144};
2145
2146/// Call signature conversion and then fail the rewrite to trigger the undo
2147/// mechanism.
2148struct TestSignatureConversionUndo
2149 : public OpConversionPattern<TestSignatureConversionUndoOp> {
2150 using OpConversionPattern<TestSignatureConversionUndoOp>::OpConversionPattern;
2151
2152 LogicalResult
2153 matchAndRewrite(TestSignatureConversionUndoOp op, OpAdaptor adaptor,
2154 ConversionPatternRewriter &rewriter) const final {
2155 (void)rewriter.convertRegionTypes(region: &op->getRegion(0), converter: *getTypeConverter());
2156 return failure();
2157 }
2158};
2159
2160/// Call signature conversion without providing a type converter to handle
2161/// materializations.
2162struct TestTestSignatureConversionNoConverter
2163 : public OpConversionPattern<TestSignatureConversionNoConverterOp> {
2164 TestTestSignatureConversionNoConverter(const TypeConverter &converter,
2165 MLIRContext *context)
2166 : OpConversionPattern<TestSignatureConversionNoConverterOp>(context),
2167 converter(converter) {}
2168
2169 LogicalResult
2170 matchAndRewrite(TestSignatureConversionNoConverterOp op, OpAdaptor adaptor,
2171 ConversionPatternRewriter &rewriter) const final {
2172 Region &region = op->getRegion(0);
2173 Block *entry = &region.front();
2174
2175 // Convert the original entry arguments.
2176 TypeConverter::SignatureConversion result(entry->getNumArguments());
2177 if (failed(
2178 Result: converter.convertSignatureArgs(types: entry->getArgumentTypes(), result)))
2179 return failure();
2180 rewriter.modifyOpInPlace(op, [&] {
2181 rewriter.applySignatureConversion(block: &region.front(), conversion&: result);
2182 });
2183 return success();
2184 }
2185
2186 const TypeConverter &converter;
2187};
2188
2189/// Just forward the operands to the root op. This is essentially a no-op
2190/// pattern that is used to trigger target materialization.
2191struct TestTypeConsumerForward
2192 : public OpConversionPattern<TestTypeConsumerOp> {
2193 using OpConversionPattern<TestTypeConsumerOp>::OpConversionPattern;
2194
2195 LogicalResult
2196 matchAndRewrite(TestTypeConsumerOp op, OpAdaptor adaptor,
2197 ConversionPatternRewriter &rewriter) const final {
2198 rewriter.modifyOpInPlace(op,
2199 [&] { op->setOperands(adaptor.getOperands()); });
2200 return success();
2201 }
2202};
2203
2204struct TestTypeConversionAnotherProducer
2205 : public OpRewritePattern<TestAnotherTypeProducerOp> {
2206 using OpRewritePattern<TestAnotherTypeProducerOp>::OpRewritePattern;
2207
2208 LogicalResult matchAndRewrite(TestAnotherTypeProducerOp op,
2209 PatternRewriter &rewriter) const final {
2210 rewriter.replaceOpWithNewOp<TestTypeProducerOp>(op, op.getType());
2211 return success();
2212 }
2213};
2214
2215struct TestReplaceWithLegalOp : public ConversionPattern {
2216 TestReplaceWithLegalOp(const TypeConverter &converter, MLIRContext *ctx)
2217 : ConversionPattern(converter, "test.replace_with_legal_op",
2218 /*benefit=*/1, ctx) {}
2219 LogicalResult
2220 matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
2221 ConversionPatternRewriter &rewriter) const final {
2222 rewriter.replaceOpWithNewOp<LegalOpD>(op, operands[0].front());
2223 return success();
2224 }
2225};
2226
2227struct TestTypeConversionDriver
2228 : public PassWrapper<TestTypeConversionDriver, OperationPass<>> {
2229 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestTypeConversionDriver)
2230
2231 TestTypeConversionDriver() = default;
2232 TestTypeConversionDriver(const TestTypeConversionDriver &other)
2233 : PassWrapper(other) {}
2234
2235 void getDependentDialects(DialectRegistry &registry) const override {
2236 registry.insert<TestDialect>();
2237 }
2238 StringRef getArgument() const final {
2239 return "test-legalize-type-conversion";
2240 }
2241 StringRef getDescription() const final {
2242 return "Test various type conversion functionalities in DialectConversion";
2243 }
2244
2245 void runOnOperation() override {
2246 // Initialize the type converter.
2247 SmallVector<Type, 2> conversionCallStack;
2248 TypeConverter converter;
2249
2250 /// Add the legal set of type conversions.
2251 converter.addConversion(callback: [](Type type) -> Type {
2252 // Treat F64 as legal.
2253 if (type.isF64())
2254 return type;
2255 // Allow converting BF16/F16/F32 to F64.
2256 if (type.isBF16() || type.isF16() || type.isF32())
2257 return Float64Type::get(context: type.getContext());
2258 // Otherwise, the type is illegal.
2259 return nullptr;
2260 });
2261 converter.addConversion(callback: [](IndexType type) { return type; });
2262 converter.addConversion(callback: [](IntegerType type, SmallVectorImpl<Type> &types) {
2263 if (type.isInteger(width: 1)) {
2264 // i1 is legal.
2265 types.push_back(Elt: type);
2266 }
2267 if (type.isInteger(width: 38)) {
2268 // i38 is legal.
2269 types.push_back(Elt: type);
2270 }
2271 // Drop all other integer types.
2272 return success();
2273 });
2274 converter.addConversion(
2275 // Convert a recursive self-referring type into a non-self-referring
2276 // type named "outer_converted_type" that contains a SimpleAType.
2277 callback: [&](test::TestRecursiveType type,
2278 SmallVectorImpl<Type> &results) -> std::optional<LogicalResult> {
2279 // If the type is already converted, return it to indicate that it is
2280 // legal.
2281 if (type.getName() == "outer_converted_type") {
2282 results.push_back(type);
2283 return success();
2284 }
2285
2286 conversionCallStack.push_back(type);
2287 llvm::scope_exit popConversionCallStack(
2288 [&conversionCallStack]() { conversionCallStack.pop_back(); });
2289
2290 // If the type is on the call stack more than once (it is there at
2291 // least once because of the _current_ call, which is always the last
2292 // element on the stack), we've hit the recursive case. Just return
2293 // SimpleAType here to create a non-recursive type as a result.
2294 if (llvm::is_contained(Range: ArrayRef(conversionCallStack).drop_back(),
2295 Element: type)) {
2296 results.push_back(test::SimpleAType::get(type.getContext()));
2297 return success();
2298 }
2299
2300 // Convert the body recursively.
2301 auto result = test::TestRecursiveType::get(ctx: type.getContext(),
2302 name: "outer_converted_type");
2303 if (failed(result.setBody(converter.convertType(t: type.getBody()))))
2304 return failure();
2305 results.push_back(Elt: result);
2306 return success();
2307 });
2308 converter.addConversion(callback: [](Value v) -> std::optional<Type> {
2309 // Context-aware type conversion rule that converts i37 to
2310 // i(37 + increment). The increment is taken from the enclosing
2311 // function.
2312 auto intType = dyn_cast<IntegerType>(Val: v.getType());
2313 if (!intType || intType.getWidth() != 37)
2314 return std::nullopt;
2315 Region *r = v.getParentRegion();
2316 if (!r) {
2317 // No enclosing region found. This can happen when running with
2318 // allow-pattern-rollback = true. Context-aware type conversions are
2319 // not fully supported when running in rollback mode.
2320 return Type();
2321 }
2322 Operation *op = r->getParentOp();
2323 if (!op)
2324 return Type();
2325 if (!isa<FunctionOpInterface>(Val: op))
2326 op = op->getParentOfType<FunctionOpInterface>();
2327 if (!op)
2328 return Type();
2329 auto incrementAttr =
2330 op->getDiscardableAttrOfType<IntegerAttr>(name: "increment");
2331 if (!incrementAttr)
2332 return Type();
2333 return IntegerType::get(context: v.getContext(),
2334 width: intType.getWidth() + incrementAttr.getInt());
2335 });
2336 converter.addConversion(
2337 callback: [](Value v, SmallVectorImpl<Type> &) -> std::optional<LogicalResult> {
2338 // Test dropping an i1 value used as an SCF condition.
2339 Operation *definingOp = v.getDefiningOp();
2340 if (!definingOp || !v.getType().isInteger(width: 1) ||
2341 !definingOp->hasAttr(name: "drop_i1"))
2342 return std::nullopt;
2343 return success();
2344 });
2345
2346 /// Add the legal set of type materializations.
2347 converter.addSourceMaterialization(callback: [](OpBuilder &builder, Type resultType,
2348 ValueRange inputs,
2349 Location loc) -> Value {
2350 // Allow casting from F64 back to F32.
2351 if (!resultType.isF16() && inputs.size() == 1 &&
2352 inputs[0].getType().isF64())
2353 return TestCastOp::create(builder, loc, resultType, inputs).getResult();
2354 // Allow producing an i32 or i64 from nothing.
2355 if ((resultType.isInteger(32) || resultType.isInteger(64)) &&
2356 inputs.empty())
2357 return TestTypeProducerOp::create(builder, loc, resultType);
2358 // Allow producing an i64 from an integer.
2359 if (isa<IntegerType>(resultType) && inputs.size() == 1 &&
2360 isa<IntegerType>(inputs[0].getType()))
2361 return TestCastOp::create(builder, loc, resultType, inputs).getResult();
2362 // Otherwise, fail.
2363 return nullptr;
2364 });
2365 // Materialize i37 to any desired type with unrealized_conversion_cast.
2366 converter.addTargetMaterialization(callback: [](OpBuilder &builder, Type type,
2367 ValueRange inputs,
2368 Location loc) -> Value {
2369 if (inputs.size() != 1 || !inputs[0].getType().isInteger(width: 37))
2370 return Value();
2371 return UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs)
2372 .getResult(i: 0);
2373 });
2374
2375 // Initialize the conversion target.
2376 mlir::ConversionTarget target(getContext());
2377 target.addLegalOp(op: OperationName("test.context_op", &getContext()));
2378 target.addLegalOp<LegalOpD>();
2379 target.addDynamicallyLegalOp<TestTypeProducerOp>([](TestTypeProducerOp op) {
2380 auto recursiveType = dyn_cast<test::TestRecursiveType>(op.getType());
2381 return op.getType().isF64() || op.getType().isInteger(64) ||
2382 (recursiveType &&
2383 recursiveType.getName() == "outer_converted_type");
2384 });
2385 target.addDynamicallyLegalOp<func::FuncOp>(callback: [&](func::FuncOp op) {
2386 return converter.isSignatureLegal(ty: op.getFunctionType());
2387 });
2388 target.addDynamicallyLegalOp<TestCastOp>([&](TestCastOp op) {
2389 // Allow casts from F64 to F32.
2390 return (*op.operand_type_begin()).isF64() && op.getType().isF32();
2391 });
2392 target.addDynamicallyLegalOp<TestSignatureConversionNoConverterOp>(
2393 [&](TestSignatureConversionNoConverterOp op) {
2394 return converter.isLegal(op.getRegion().front().getArgumentTypes());
2395 });
2396
2397 // Initialize the set of rewrite patterns.
2398 RewritePatternSet patterns(&getContext());
2399 patterns
2400 .add<TestTypeConsumerForward, TestTypeConversionProducer,
2401 TestSignatureConversionUndo,
2402 TestTestSignatureConversionNoConverter, TestReplaceWithLegalOp>(
2403 arg&: converter, args: &getContext());
2404 patterns.add<TestTypeConversionAnotherProducer>(arg: &getContext());
2405 mlir::populateAnyFunctionOpInterfaceTypeConversionPattern(patterns,
2406 converter);
2407 mlir::scf::populateSCFStructuralTypeConversionsAndLegality(
2408 typeConverter: converter, patterns, target);
2409 mlir::cf::populateCFStructuralTypeConversionsAndLegality(typeConverter: converter,
2410 patterns, target);
2411
2412 ConversionConfig config;
2413 config.allowPatternRollback = allowPatternRollback;
2414 if (failed(Result: applyPartialConversion(op: getOperation(), target,
2415 patterns: std::move(patterns), config)))
2416 signalPassFailure();
2417 }
2418
2419 Option<bool> allowPatternRollback{*this, "allow-pattern-rollback",
2420 llvm::cl::desc("Allow pattern rollback"),
2421 llvm::cl::init(Val: true)};
2422};
2423} // namespace
2424
2425//===----------------------------------------------------------------------===//
2426// Test Target Materialization With No Uses
2427//===----------------------------------------------------------------------===//
2428
2429namespace {
2430struct ForwardOperandPattern : public OpConversionPattern<TestTypeChangerOp> {
2431 using OpConversionPattern<TestTypeChangerOp>::OpConversionPattern;
2432
2433 LogicalResult
2434 matchAndRewrite(TestTypeChangerOp op, OpAdaptor adaptor,
2435 ConversionPatternRewriter &rewriter) const final {
2436 rewriter.replaceOp(op, adaptor.getOperands());
2437 return success();
2438 }
2439};
2440
2441struct TestTargetMaterializationWithNoUses
2442 : public PassWrapper<TestTargetMaterializationWithNoUses, OperationPass<>> {
2443 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
2444 TestTargetMaterializationWithNoUses)
2445
2446 StringRef getArgument() const final {
2447 return "test-target-materialization-with-no-uses";
2448 }
2449 StringRef getDescription() const final {
2450 return "Test a special case of target materialization in DialectConversion";
2451 }
2452
2453 void runOnOperation() override {
2454 TypeConverter converter;
2455 converter.addConversion(callback: [](Type t) { return t; });
2456 converter.addConversion(callback: [](IntegerType intTy) -> Type {
2457 if (intTy.getWidth() == 16)
2458 return IntegerType::get(context: intTy.getContext(), width: 64);
2459 return intTy;
2460 });
2461 converter.addTargetMaterialization(
2462 callback: [](OpBuilder &builder, Type type, ValueRange inputs, Location loc) {
2463 return TestCastOp::create(builder, loc, type, inputs).getResult();
2464 });
2465
2466 ConversionTarget target(getContext());
2467 target.addIllegalOp<TestTypeChangerOp>();
2468
2469 RewritePatternSet patterns(&getContext());
2470 patterns.add<ForwardOperandPattern>(arg&: converter, args: &getContext());
2471
2472 if (failed(Result: applyPartialConversion(op: getOperation(), target,
2473 patterns: std::move(patterns))))
2474 signalPassFailure();
2475 }
2476};
2477} // namespace
2478
2479//===----------------------------------------------------------------------===//
2480// Test Block Merging
2481//===----------------------------------------------------------------------===//
2482
2483namespace {
2484/// A rewriter pattern that tests that blocks can be merged.
2485struct TestMergeBlock : public OpConversionPattern<TestMergeBlocksOp> {
2486 using OpConversionPattern<TestMergeBlocksOp>::OpConversionPattern;
2487
2488 LogicalResult
2489 matchAndRewrite(TestMergeBlocksOp op, OpAdaptor adaptor,
2490 ConversionPatternRewriter &rewriter) const final {
2491 Block &firstBlock = op.getBody().front();
2492 Operation *branchOp = firstBlock.getTerminator();
2493 Block *secondBlock = &*(std::next(op.getBody().begin()));
2494 auto succOperands = branchOp->getOperands();
2495 SmallVector<Value, 2> replacements(succOperands);
2496 rewriter.eraseOp(op: branchOp);
2497 rewriter.mergeBlocks(source: secondBlock, dest: &firstBlock, argValues: replacements);
2498 rewriter.modifyOpInPlace(op, [] {});
2499 return success();
2500 }
2501};
2502
2503/// A rewrite pattern to tests the undo mechanism of blocks being merged.
2504struct TestUndoBlocksMerge : public ConversionPattern {
2505 TestUndoBlocksMerge(MLIRContext *ctx)
2506 : ConversionPattern("test.undo_blocks_merge", /*benefit=*/1, ctx) {}
2507 LogicalResult
2508 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
2509 ConversionPatternRewriter &rewriter) const final {
2510 Block &firstBlock = op->getRegion(index: 0).front();
2511 Operation *branchOp = firstBlock.getTerminator();
2512 Block *secondBlock = &*(std::next(x: op->getRegion(index: 0).begin()));
2513 rewriter.setInsertionPointToStart(secondBlock);
2514 ILLegalOpF::create(rewriter, op->getLoc(), rewriter.getF32Type());
2515 auto succOperands = branchOp->getOperands();
2516 SmallVector<Value, 2> replacements(succOperands);
2517 rewriter.eraseOp(op: branchOp);
2518 rewriter.mergeBlocks(source: secondBlock, dest: &firstBlock, argValues: replacements);
2519 rewriter.modifyOpInPlace(root: op, callable: [] {});
2520 return success();
2521 }
2522};
2523
2524/// A rewrite mechanism to inline the body of the op into its parent, when both
2525/// ops can have a single block.
2526struct TestMergeSingleBlockOps
2527 : public OpConversionPattern<SingleBlockImplicitTerminatorOp> {
2528 using OpConversionPattern<
2529 SingleBlockImplicitTerminatorOp>::OpConversionPattern;
2530
2531 LogicalResult
2532 matchAndRewrite(SingleBlockImplicitTerminatorOp op, OpAdaptor adaptor,
2533 ConversionPatternRewriter &rewriter) const final {
2534 SingleBlockImplicitTerminatorOp parentOp =
2535 op->getParentOfType<SingleBlockImplicitTerminatorOp>();
2536 if (!parentOp)
2537 return failure();
2538 Block &innerBlock = op.getRegion().front();
2539 TerminatorOp innerTerminator =
2540 cast<TerminatorOp>(innerBlock.getTerminator());
2541 rewriter.inlineBlockBefore(&innerBlock, op);
2542 rewriter.eraseOp(op: innerTerminator);
2543 rewriter.eraseOp(op: op);
2544 return success();
2545 }
2546};
2547
2548struct TestMergeBlocksPatternDriver
2549 : public PassWrapper<TestMergeBlocksPatternDriver, OperationPass<>> {
2550 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestMergeBlocksPatternDriver)
2551
2552 StringRef getArgument() const final { return "test-merge-blocks"; }
2553 StringRef getDescription() const final {
2554 return "Test Merging operation in ConversionPatternRewriter";
2555 }
2556 void runOnOperation() override {
2557 MLIRContext *context = &getContext();
2558 mlir::RewritePatternSet patterns(context);
2559 patterns.add<TestMergeBlock, TestUndoBlocksMerge, TestMergeSingleBlockOps>(
2560 arg&: context);
2561 ConversionTarget target(*context);
2562 target.addLegalOp<func::FuncOp, ModuleOp, TerminatorOp, TestBranchOp,
2563 TestTypeConsumerOp, TestTypeProducerOp, TestReturnOp>();
2564 target.addIllegalOp<ILLegalOpF>();
2565
2566 /// Expect the op to have a single block after legalization.
2567 target.addDynamicallyLegalOp<TestMergeBlocksOp>(
2568 [&](TestMergeBlocksOp op) -> bool {
2569 return op.getBody().hasOneBlock();
2570 });
2571
2572 /// Only allow `test.br` within test.merge_blocks op.
2573 target.addDynamicallyLegalOp<TestBranchOp>([&](TestBranchOp op) -> bool {
2574 return op->getParentOfType<TestMergeBlocksOp>();
2575 });
2576
2577 /// Expect that all nested test.SingleBlockImplicitTerminator ops are
2578 /// inlined.
2579 target.addDynamicallyLegalOp<SingleBlockImplicitTerminatorOp>(
2580 [&](SingleBlockImplicitTerminatorOp op) -> bool {
2581 return !op->getParentOfType<SingleBlockImplicitTerminatorOp>();
2582 });
2583
2584 DenseSet<Operation *> unlegalizedOps;
2585 ConversionConfig config;
2586 config.unlegalizedOps = &unlegalizedOps;
2587 (void)applyPartialConversion(op: getOperation(), target, patterns: std::move(patterns),
2588 config);
2589 for (auto *op : unlegalizedOps)
2590 op->emitRemark() << "op '" << op->getName() << "' is not legalizable";
2591 }
2592};
2593} // namespace
2594
2595//===----------------------------------------------------------------------===//
2596// Test Selective Replacement
2597//===----------------------------------------------------------------------===//
2598
2599namespace {
2600/// A rewrite mechanism to inline the body of the op into its parent, when both
2601/// ops can have a single block.
2602struct TestSelectiveOpReplacementPattern : public OpRewritePattern<TestCastOp> {
2603 using OpRewritePattern<TestCastOp>::OpRewritePattern;
2604
2605 LogicalResult matchAndRewrite(TestCastOp op,
2606 PatternRewriter &rewriter) const final {
2607 if (op.getNumOperands() != 2)
2608 return failure();
2609 OperandRange operands = op.getOperands();
2610
2611 // Replace non-terminator uses with the first operand.
2612 rewriter.replaceUsesWithIf(op, operands[0], [](OpOperand &operand) {
2613 return operand.getOwner()->hasTrait<OpTrait::IsTerminator>();
2614 });
2615 // Replace everything else with the second operand if the operation isn't
2616 // dead.
2617 rewriter.replaceOp(op, op.getOperand(1));
2618 return success();
2619 }
2620};
2621
2622struct TestSelectiveReplacementPatternDriver
2623 : public PassWrapper<TestSelectiveReplacementPatternDriver,
2624 OperationPass<>> {
2625 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
2626 TestSelectiveReplacementPatternDriver)
2627
2628 StringRef getArgument() const final {
2629 return "test-pattern-selective-replacement";
2630 }
2631 StringRef getDescription() const final {
2632 return "Test selective replacement in the PatternRewriter";
2633 }
2634 void runOnOperation() override {
2635 MLIRContext *context = &getContext();
2636 mlir::RewritePatternSet patterns(context);
2637 patterns.add<TestSelectiveOpReplacementPattern>(arg&: context);
2638 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
2639 }
2640};
2641
2642struct TestFoldTypeConvertingOp
2643 : public PassWrapper<TestFoldTypeConvertingOp, OperationPass<>> {
2644 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestFoldTypeConvertingOp)
2645
2646 StringRef getArgument() const final { return "test-fold-type-converting-op"; }
2647 StringRef getDescription() const final {
2648 return "Test helper functions for folding ops whose input and output types "
2649 "differ, e.g. float comparisons of the form `(f32, f32) -> i1`.";
2650 }
2651 void runOnOperation() override {
2652 MLIRContext *context = &getContext();
2653 mlir::RewritePatternSet patterns(context);
2654 patterns.add<FoldSignOpF32ToSI32, FoldLessThanOpF32ToI1>(arg&: context);
2655 if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns))))
2656 signalPassFailure();
2657 }
2658};
2659} // namespace
2660
2661//===----------------------------------------------------------------------===//
2662// PassRegistration
2663//===----------------------------------------------------------------------===//
2664
2665namespace mlir {
2666namespace test {
2667void registerPatternsTestPass() {
2668 PassRegistration<TestReturnTypeDriver>();
2669
2670 PassRegistration<TestDerivedAttributeDriver>();
2671
2672 PassRegistration<TestGreedyPatternDriver>();
2673 PassRegistration<TestStrictPatternDriver>();
2674 PassRegistration<TestWalkPatternDriver>();
2675
2676 PassRegistration<TestLegalizePatternDriver>();
2677
2678 PassRegistration<TestRemappedValue>();
2679
2680 PassRegistration<TestUnknownRootOpDriver>();
2681
2682 PassRegistration<TestTypeConversionDriver>();
2683 PassRegistration<TestTargetMaterializationWithNoUses>();
2684
2685 PassRegistration<TestRewriteDynamicOpDriver>();
2686
2687 PassRegistration<TestMergeBlocksPatternDriver>();
2688 PassRegistration<TestSelectiveReplacementPatternDriver>();
2689
2690 PassRegistration<TestFoldTypeConvertingOp>();
2691}
2692} // namespace test
2693} // namespace mlir
2694