| 1 | //===- ACCLoopTiling.cpp - Tile ACC Loops ---------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This pass implements the OpenACC loop tiling transformation for acc.loop |
| 10 | // operations that have the tile clause (OpenACC 3.4 spec, section 2.9.8). |
| 11 | // |
| 12 | // Overview: |
| 13 | // --------- |
| 14 | // The tile clause specifies that the iterations of the associated loops should |
| 15 | // be divided into tiles (rectangular blocks). This pass transforms an acc.loop |
| 16 | // with a tile clause into a "tile group" loop (iterating over tiles) containing |
| 17 | // an "element group" loop (iterating within a tile). Each of them carries all |
| 18 | // the induction variables of the original loop. |
| 19 | // |
| 20 | // The tile clause lists its sizes innermost loop first, so the first size |
| 21 | // applies to the last induction variable. For example, tiling a 2-level loop |
| 22 | // nest with tile(T1, T2) produces: |
| 23 | // |
| 24 | // // Before tiling: |
| 25 | // acc.loop tile(T1, T2) control(%i, %j) = (lb1, lb2) to (ub1, ub2) |
| 26 | // step (s1, s2) |
| 27 | // |
| 28 | // // After tiling: |
| 29 | // acc.loop control(%i, %j) = (lb1, lb2) to (ub1, ub2) |
| 30 | // step (s1*T2, s2*T1) { // tile group |
| 31 | // acc.loop control(%ii, %jj) = (%i, %j) |
| 32 | // to (min(ub1, %i+s1*T2), min(ub2, %j+s2*T1)) |
| 33 | // step (s1, s2) { // element group |
| 34 | // // loop body using %ii, %jj |
| 35 | // } |
| 36 | // } |
| 37 | // |
| 38 | // The example element loop bounds above assume ascending steps. For an |
| 39 | // acc.loop with negative steps, a descending dimension would use |
| 40 | // max(ub, %i+s*T) instead. |
| 41 | // |
| 42 | // Gang/worker/vector attributes are distributed as follows: |
| 43 | // - gang: applied to tile loops |
| 44 | // - vector: applied to element loops |
| 45 | // - worker: removed from inner loops |
| 46 | // |
| 47 | // Unknown Tile Sizes: |
| 48 | // ------------------- |
| 49 | // The OpenACC tile(*) syntax indicates an implementation-defined tile size. |
| 50 | // In the IR, this is represented as -1. The pass resolves these to the |
| 51 | // default tile size (configurable via pass option). |
| 52 | // |
| 53 | // Requirements: |
| 54 | // ------------- |
| 55 | // 1. The pass uses the OpenACCSupport analysis for remark and NYI (not yet |
| 56 | // implemented) emission. Custom implementations can be registered via |
| 57 | // setImplementation() to provide pipeline-specific handling. |
| 58 | // |
| 59 | //===----------------------------------------------------------------------===// |
| 60 | |
| 61 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 62 | #include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h" |
| 63 | #include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h" |
| 64 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 65 | #include "mlir/Dialect/OpenACC/OpenACCUtilsTiling.h" |
| 66 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h" |
| 67 | #include "mlir/IR/BuiltinAttributes.h" |
| 68 | #include "mlir/IR/PatternMatch.h" |
| 69 | #include "mlir/Support/LLVM.h" |
| 70 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 71 | #include "llvm/ADT/StringExtras.h" |
| 72 | #include "llvm/Support/Debug.h" |
| 73 | |
| 74 | namespace mlir { |
| 75 | namespace acc { |
| 76 | #define GEN_PASS_DEF_ACCLOOPTILING |
| 77 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc" |
| 78 | } // namespace acc |
| 79 | } // namespace mlir |
| 80 | |
| 81 | #define DEBUG_TYPE "acc-loop-tile" |
| 82 | |
| 83 | namespace { |
| 84 | using namespace mlir; |
| 85 | |
| 86 | struct ACCLoopTilingImpl : public OpRewritePattern<acc::LoopOp> { |
| 87 | ACCLoopTilingImpl(MLIRContext *context, int32_t defaultTileSize, |
| 88 | acc::OpenACCSupport &accSupport) |
| 89 | : OpRewritePattern<acc::LoopOp>(context), |
| 90 | defaultTileSize(defaultTileSize), accSupport(accSupport) {} |
| 91 | |
| 92 | // Check that tile size types are not narrower than IV types. |
| 93 | // We only check when both types are IntegerType. For IndexType, the width |
| 94 | // is target-dependent and the casting utility will handle it correctly. |
| 95 | // The tile sizes are in clause order, so the last one belongs to the |
| 96 | // outermost IV. |
| 97 | LogicalResult checkTileSizeTypes(acc::LoopOp loop, |
| 98 | ArrayRef<Value> tileSizes) const { |
| 99 | auto ivTypes = loop.getBody().getArgumentTypes(); |
| 100 | for (auto [tileSize, ivType] : |
| 101 | llvm::zip(t: llvm::reverse(C&: tileSizes), u&: ivTypes)) { |
| 102 | Type tileType = tileSize.getType(); |
| 103 | |
| 104 | // Skip unknown tile sizes (will be created with correct type) |
| 105 | auto constVal = getConstantIntValue(ofr: tileSize); |
| 106 | if (constVal && *constVal < 0) |
| 107 | continue; |
| 108 | |
| 109 | // Only compare when both are integer types (not index) |
| 110 | auto tileIntType = dyn_cast<IntegerType>(Val&: tileType); |
| 111 | auto ivIntType = dyn_cast<IntegerType>(Val&: ivType); |
| 112 | if (tileIntType && ivIntType) { |
| 113 | if (tileIntType.getWidth() > ivIntType.getWidth()) { |
| 114 | accSupport.emitNYI(loc: loop.getLoc(), |
| 115 | message: "tile size type (i" + |
| 116 | std::to_string(val: tileIntType.getWidth()) + |
| 117 | ") is wider than loop IV type (i" + |
| 118 | std::to_string(val: ivIntType.getWidth()) + ")" ); |
| 119 | return failure(); |
| 120 | } |
| 121 | } |
| 122 | } |
| 123 | return success(); |
| 124 | } |
| 125 | |
| 126 | void (acc::LoopOp loop, ArrayRef<Value> tileSizes) const { |
| 127 | // Emit remarks for loop tiling |
| 128 | accSupport.emitRemark( |
| 129 | op: loop, |
| 130 | messageFn: [&]() { |
| 131 | auto getTileSizeStr = [&](Value v) -> std::string { |
| 132 | std::string name = accSupport.getVariableName(v); |
| 133 | // Use "*" for unknown tile sizes (represented as -1 or empty) |
| 134 | if (name.empty() || name == "-1" ) |
| 135 | return "*" ; |
| 136 | return name; |
| 137 | }; |
| 138 | SmallVector<std::string> tileStrs; |
| 139 | for (Value v : tileSizes) |
| 140 | tileStrs.push_back(Elt: getTileSizeStr(v)); |
| 141 | return "Tiling " + std::to_string(val: tileSizes.size()) + |
| 142 | "-level loop nest with tile(" + llvm::join(R&: tileStrs, Separator: "," ) + |
| 143 | ")" ; |
| 144 | }, |
| 145 | DEBUG_TYPE); |
| 146 | |
| 147 | // Emit remarks for unknown tile sizes that will be resolved to default |
| 148 | // TODO: Need to base the default tile size on some heuristics. |
| 149 | for (Value tileSize : tileSizes) { |
| 150 | std::optional<int64_t> val = getConstantIntValue(ofr: tileSize); |
| 151 | if (val && *val < 0) { |
| 152 | accSupport.emitRemark( |
| 153 | op: loop, |
| 154 | messageFn: [&]() { |
| 155 | return "Picking default tile size " + |
| 156 | std::to_string(val: defaultTileSize) + |
| 157 | " for unknown tile size '*'" ; |
| 158 | }, |
| 159 | DEBUG_TYPE); |
| 160 | } |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | static void clearTileOperands(acc::LoopOp loop, PatternRewriter &rewriter) { |
| 165 | rewriter.modifyOpInPlace(root: loop, callable: [&]() { |
| 166 | loop.getTileOperandsMutable().clear(); |
| 167 | loop.removeTileOperandsSegmentsAttr(); |
| 168 | loop.removeTileOperandsDeviceTypeAttr(); |
| 169 | }); |
| 170 | } |
| 171 | |
| 172 | LogicalResult matchAndRewrite(acc::LoopOp origLoop, |
| 173 | PatternRewriter &rewriter) const override { |
| 174 | |
| 175 | if (origLoop.getTileValues().empty()) |
| 176 | return failure(); |
| 177 | |
| 178 | // The iterations of an unstructured loop are implemented by the control |
| 179 | // flow in its body, and it has no induction variables to tile. The tile |
| 180 | // clause does not require tiling, so drop it and keep the loop as is. |
| 181 | if (origLoop.getUnstructured()) { |
| 182 | clearTileOperands(loop: origLoop, rewriter); |
| 183 | return success(); |
| 184 | } |
| 185 | |
| 186 | SmallVector<Value> tileSizes(origLoop.getTileValues().begin(), |
| 187 | origLoop.getTileValues().end()); |
| 188 | |
| 189 | // A tile clause and a collapse clause on the same loop are not supported: |
| 190 | // they give conflicting descriptions of how many loops the construct |
| 191 | // associates. |
| 192 | if (origLoop.getCollapseAttr()) { |
| 193 | accSupport.emitNYI(loc: origLoop.getLoc(), |
| 194 | message: "a tile clause combined with a collapse clause on the " |
| 195 | "same loop" ); |
| 196 | return failure(); |
| 197 | } |
| 198 | |
| 199 | // Sanity check tile size types |
| 200 | if (failed(Result: checkTileSizeTypes(loop: origLoop, tileSizes))) |
| 201 | return failure(); |
| 202 | |
| 203 | // Emit remarks for loop tiling. This is emitted before the original loop |
| 204 | // is modified. However, it assumes that tiling will not fail. |
| 205 | emitTilingRemarks(loop: origLoop, tileSizes); |
| 206 | |
| 207 | LLVM_DEBUG(llvm::dbgs() << "\nBefore tiling:\n" << *origLoop << "\n" ); |
| 208 | |
| 209 | clearTileOperands(loop: origLoop, rewriter); |
| 210 | |
| 211 | // Tile the single fused loop in place: the tile iterations become one |
| 212 | // multi-IV "tile group" loop wrapping one multi-IV "element group" loop. |
| 213 | // The utility handles unknown tile sizes (*) by using `defaultTileSize`. |
| 214 | acc::tileACCLoops(tileLoop: origLoop, tileSizes, defaultTileSize, rewriter); |
| 215 | |
| 216 | LLVM_DEBUG(llvm::dbgs() << "\nAfter tiling:\n " << *origLoop << "\n" ); |
| 217 | return success(); |
| 218 | } |
| 219 | |
| 220 | private: |
| 221 | int32_t defaultTileSize; |
| 222 | acc::OpenACCSupport &accSupport; |
| 223 | }; |
| 224 | |
| 225 | class ACCLoopTiling : public acc::impl::ACCLoopTilingBase<ACCLoopTiling> { |
| 226 | public: |
| 227 | using ACCLoopTilingBase<ACCLoopTiling>::ACCLoopTilingBase; |
| 228 | |
| 229 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 230 | ACCLoopTilingBase<ACCLoopTiling>::getDependentDialects(registry); |
| 231 | acc::registerAllExtensions(registry); |
| 232 | } |
| 233 | |
| 234 | void runOnOperation() override { |
| 235 | func::FuncOp funcOp = getOperation(); |
| 236 | MLIRContext *context = funcOp.getContext(); |
| 237 | acc::OpenACCSupport &accSupport = getAnalysis<acc::OpenACCSupport>(); |
| 238 | |
| 239 | RewritePatternSet patterns(context); |
| 240 | patterns.insert<ACCLoopTilingImpl>(arg&: context, args&: defaultTileSize, args&: accSupport); |
| 241 | GreedyRewriteConfig grc; |
| 242 | grc.setUseTopDownTraversal(true); |
| 243 | grc.setMaxIterations(1); |
| 244 | (void)applyPatternsGreedily(op: funcOp, patterns: std::move(patterns), config: grc); |
| 245 | } |
| 246 | }; |
| 247 | } // namespace |
| 248 | |