| 1 | //===- WalkPatternRewriteDriver.cpp - A fast walk-based rewriter ---------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Implements mlir::walkAndApplyPatterns. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Transforms/WalkPatternRewriteDriver.h" |
| 14 | |
| 15 | #include "mlir/IR/MLIRContext.h" |
| 16 | #include "mlir/IR/Operation.h" |
| 17 | #include "mlir/IR/OperationSupport.h" |
| 18 | #include "mlir/IR/PatternMatch.h" |
| 19 | #include "mlir/IR/Verifier.h" |
| 20 | #include "mlir/IR/Visitors.h" |
| 21 | #include "mlir/Rewrite/PatternApplicator.h" |
| 22 | #include "llvm/ADT/DenseSet.h" |
| 23 | #include "llvm/ADT/STLExtras.h" |
| 24 | #include "llvm/Support/DebugLog.h" |
| 25 | #include "llvm/Support/ErrorHandling.h" |
| 26 | |
| 27 | #define DEBUG_TYPE "walk-rewriter" |
| 28 | |
| 29 | namespace mlir { |
| 30 | |
| 31 | // Find all reachable blocks in the region and add them to the visitedBlocks |
| 32 | // set. |
| 33 | static void findReachableBlocks(Region ®ion, |
| 34 | DenseSet<Block *> &reachableBlocks) { |
| 35 | Block *entryBlock = ®ion.front(); |
| 36 | reachableBlocks.insert(V: entryBlock); |
| 37 | // Traverse the CFG and add all reachable blocks to the blockList. |
| 38 | SmallVector<Block *> worklist({entryBlock}); |
| 39 | while (!worklist.empty()) { |
| 40 | Block *block = worklist.pop_back_val(); |
| 41 | Operation *terminator = &block->back(); |
| 42 | for (Block *successor : terminator->getSuccessors()) { |
| 43 | if (reachableBlocks.contains(V: successor)) |
| 44 | continue; |
| 45 | worklist.push_back(Elt: successor); |
| 46 | reachableBlocks.insert(V: successor); |
| 47 | } |
| 48 | } |
| 49 | } |
| 50 | |
| 51 | namespace { |
| 52 | struct WalkAndApplyPatternsAction final |
| 53 | : tracing::ActionImpl<WalkAndApplyPatternsAction> { |
| 54 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(WalkAndApplyPatternsAction) |
| 55 | using ActionImpl::ActionImpl; |
| 56 | static constexpr StringLiteral tag = "walk-and-apply-patterns" ; |
| 57 | void print(raw_ostream &os) const override { os << tag; } |
| 58 | }; |
| 59 | |
| 60 | #if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS |
| 61 | // Forwarding listener to guard against unsupported erasures of non-descendant |
| 62 | // ops/blocks. Because we use walk-based pattern application, erasing the |
| 63 | // op/block from the *next* iteration (e.g., a user of the visited op) is not |
| 64 | // valid. Note that this is only used with expensive pattern API checks. |
| 65 | // |
| 66 | // Ops and blocks that were *created* during the current pattern application are |
| 67 | // exempt: they were not in the walk schedule before the pattern ran, so erasing |
| 68 | // them cannot invalidate the current walk iterator. |
| 69 | struct ErasedOpsListener final : RewriterBase::ForwardingListener { |
| 70 | using RewriterBase::ForwardingListener::ForwardingListener; |
| 71 | |
| 72 | void notifyOperationInserted(Operation *op, |
| 73 | OpBuilder::InsertPoint previous) override { |
| 74 | if (visitedOp) |
| 75 | newlyCreatedOps.insert(op); |
| 76 | ForwardingListener::notifyOperationInserted(op, previous); |
| 77 | } |
| 78 | |
| 79 | void notifyBlockInserted(Block *block, Region *previous, |
| 80 | Region::iterator previousIt) override { |
| 81 | if (visitedOp) |
| 82 | newlyCreatedBlocks.insert(block); |
| 83 | ForwardingListener::notifyBlockInserted(block, previous, previousIt); |
| 84 | } |
| 85 | |
| 86 | void notifyOperationErased(Operation *op) override { |
| 87 | if (!newlyCreatedOps.contains(op)) |
| 88 | checkErasure(op); |
| 89 | newlyCreatedOps.erase(op); |
| 90 | ForwardingListener::notifyOperationErased(op); |
| 91 | } |
| 92 | |
| 93 | void notifyBlockErased(Block *block) override { |
| 94 | if (!newlyCreatedBlocks.contains(block)) |
| 95 | checkErasure(block->getParentOp()); |
| 96 | newlyCreatedBlocks.erase(block); |
| 97 | ForwardingListener::notifyBlockErased(block); |
| 98 | } |
| 99 | |
| 100 | void checkErasure(Operation *op) const { |
| 101 | Operation *ancestorOp = op; |
| 102 | while (ancestorOp && ancestorOp != visitedOp) |
| 103 | ancestorOp = ancestorOp->getParentOp(); |
| 104 | |
| 105 | if (ancestorOp != visitedOp) |
| 106 | llvm::report_fatal_error( |
| 107 | "unsupported erasure in WalkPatternRewriter; " |
| 108 | "erasure is only supported for matched ops and their descendants" ); |
| 109 | } |
| 110 | |
| 111 | Operation *visitedOp = nullptr; |
| 112 | // Ops and blocks inserted since visitedOp was last set; may be freely erased. |
| 113 | DenseSet<Operation *> newlyCreatedOps; |
| 114 | DenseSet<Block *> newlyCreatedBlocks; |
| 115 | }; |
| 116 | #endif // MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS |
| 117 | } // namespace |
| 118 | |
| 119 | void walkAndApplyPatterns(Operation *op, |
| 120 | const FrozenRewritePatternSet &patterns, |
| 121 | RewriterBase::Listener *listener) { |
| 122 | #if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS |
| 123 | if (failed(verify(op))) |
| 124 | llvm::report_fatal_error("walk pattern rewriter input IR failed to verify" ); |
| 125 | #endif // MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS |
| 126 | |
| 127 | MLIRContext *ctx = op->getContext(); |
| 128 | PatternRewriter rewriter(ctx); |
| 129 | #if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS |
| 130 | ErasedOpsListener erasedListener(listener); |
| 131 | rewriter.setListener(&erasedListener); |
| 132 | #else |
| 133 | rewriter.setListener(listener); |
| 134 | #endif // MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS |
| 135 | |
| 136 | PatternApplicator applicator(patterns); |
| 137 | applicator.applyDefaultCostModel(); |
| 138 | |
| 139 | // Iterator on all reachable operations in the region. |
| 140 | // Also keep track if we visited the nested regions of the current op |
| 141 | // already to drive the post-order traversal. |
| 142 | struct RegionReachableOpIterator { |
| 143 | RegionReachableOpIterator(Region *region) : region(region) { |
| 144 | regionIt = region->begin(); |
| 145 | if (regionIt != region->end()) |
| 146 | blockIt = regionIt->begin(); |
| 147 | if (!llvm::hasSingleElement(C&: *region)) |
| 148 | findReachableBlocks(region&: *region, reachableBlocks); |
| 149 | } |
| 150 | // Advance the iterator to the next reachable operation. |
| 151 | void advance() { |
| 152 | assert(regionIt != region->end()); |
| 153 | hasVisitedRegions = false; |
| 154 | if (blockIt == regionIt->end()) { |
| 155 | ++regionIt; |
| 156 | while (regionIt != region->end() && |
| 157 | !reachableBlocks.contains(V: &*regionIt)) |
| 158 | ++regionIt; |
| 159 | if (regionIt != region->end()) |
| 160 | blockIt = regionIt->begin(); |
| 161 | return; |
| 162 | } |
| 163 | ++blockIt; |
| 164 | if (blockIt != regionIt->end()) { |
| 165 | LDBG() << "Incrementing block iterator, next op: " |
| 166 | << OpWithFlags(&*blockIt, OpPrintingFlags().skipRegions()); |
| 167 | } |
| 168 | } |
| 169 | // The region we're iterating over. |
| 170 | Region *region; |
| 171 | // The Block currently being iterated over. |
| 172 | Region::iterator regionIt; |
| 173 | // The Operation currently being iterated over. |
| 174 | Block::iterator blockIt; |
| 175 | // The set of blocks that are reachable in the current region. |
| 176 | DenseSet<Block *> reachableBlocks; |
| 177 | // Whether we've visited the nested regions of the current op already. |
| 178 | bool hasVisitedRegions = false; |
| 179 | }; |
| 180 | |
| 181 | // Worklist of regions to visit to drive the post-order traversal. |
| 182 | SmallVector<RegionReachableOpIterator> worklist; |
| 183 | |
| 184 | LDBG() << "Starting walk-based pattern rewrite driver" ; |
| 185 | ctx->executeAction<WalkAndApplyPatternsAction>( |
| 186 | actionFn: [&] { |
| 187 | // Perform a post-order traversal of the regions, visiting each |
| 188 | // reachable operation. |
| 189 | for (Region ®ion : op->getRegions()) { |
| 190 | assert(worklist.empty()); |
| 191 | if (region.empty()) |
| 192 | continue; |
| 193 | |
| 194 | // Prime the worklist with the entry block of this region. |
| 195 | worklist.push_back(Elt: {®ion}); |
| 196 | while (!worklist.empty()) { |
| 197 | RegionReachableOpIterator &it = worklist.back(); |
| 198 | if (it.regionIt == it.region->end()) { |
| 199 | // We're done with this region. |
| 200 | worklist.pop_back(); |
| 201 | continue; |
| 202 | } |
| 203 | if (it.blockIt == it.regionIt->end()) { |
| 204 | // We're done with this block. |
| 205 | it.advance(); |
| 206 | continue; |
| 207 | } |
| 208 | Operation *op = &*it.blockIt; |
| 209 | // If we haven't visited the nested regions of this op yet, |
| 210 | // enqueue them. |
| 211 | if (!it.hasVisitedRegions) { |
| 212 | it.hasVisitedRegions = true; |
| 213 | for (Region &nestedRegion : llvm::reverse(C: op->getRegions())) { |
| 214 | if (nestedRegion.empty()) |
| 215 | continue; |
| 216 | worklist.push_back(Elt: {&nestedRegion}); |
| 217 | } |
| 218 | } |
| 219 | // If we're not at the back of the worklist, we've enqueued some |
| 220 | // nested region for processing. We'll come back to this op later |
| 221 | // (post-order) |
| 222 | if (&it != &worklist.back()) |
| 223 | continue; |
| 224 | |
| 225 | // Preemptively increment the iterator, in case the current op |
| 226 | // would be erased. |
| 227 | it.advance(); |
| 228 | |
| 229 | LDBG() << "Visiting op: " |
| 230 | << OpWithFlags(op, OpPrintingFlags().skipRegions()); |
| 231 | #if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS |
| 232 | erasedListener.visitedOp = op; |
| 233 | erasedListener.newlyCreatedOps.clear(); |
| 234 | erasedListener.newlyCreatedBlocks.clear(); |
| 235 | #endif // MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS |
| 236 | if (succeeded(Result: applicator.matchAndRewrite(op, rewriter))) |
| 237 | LDBG() << "\tOp matched and rewritten" ; |
| 238 | } |
| 239 | } |
| 240 | }, |
| 241 | irUnits: {op}); |
| 242 | |
| 243 | #if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS |
| 244 | if (failed(verify(op))) |
| 245 | llvm::report_fatal_error( |
| 246 | "walk pattern rewriter result IR failed to verify" ); |
| 247 | #endif // MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS |
| 248 | } |
| 249 | |
| 250 | } // namespace mlir |
| 251 | |