| 1 | //===- OpenACCUtilsTiling.cpp - OpenACC Loop Tiling Utilities -------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file contains utility functions for tiling OpenACC loops. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Dialect/OpenACC/OpenACCUtilsTiling.h" |
| 14 | |
| 15 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 16 | #include "mlir/Dialect/Arith/Utils/Utils.h" |
| 17 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 18 | #include "mlir/Dialect/Utils/StaticValueUtils.h" |
| 19 | #include "mlir/Transforms/RegionUtils.h" |
| 20 | #include "llvm/ADT/STLExtras.h" |
| 21 | |
| 22 | // Resolve unknown tile sizes (represented as -1 for tile(*)) to the default. |
| 23 | // Returns a value with the same type as targetType. |
| 24 | static mlir::Value resolveAndCastTileSize(mlir::Value tileSize, |
| 25 | int32_t defaultTileSize, |
| 26 | mlir::Type targetType, |
| 27 | mlir::RewriterBase &rewriter, |
| 28 | mlir::Location loc) { |
| 29 | auto constVal = mlir::getConstantIntValue(ofr: tileSize); |
| 30 | if (constVal && *constVal < 0) { |
| 31 | // Create constant with the target type directly |
| 32 | return mlir::arith::ConstantOp::create( |
| 33 | builder&: rewriter, location: loc, result: targetType, |
| 34 | value: rewriter.getIntegerAttr(type: targetType, value: defaultTileSize)); |
| 35 | } |
| 36 | return mlir::getValueOrCreateCastToIndexLike(b&: rewriter, loc, targetType, |
| 37 | value: tileSize); |
| 38 | } |
| 39 | |
| 40 | // Remove vector/worker attributes from loop |
| 41 | static void removeWorkerVectorFromLoop(mlir::acc::LoopOp loop) { |
| 42 | if (loop.hasVector() || loop.getVectorValue()) { |
| 43 | loop.removeVectorAttr(); |
| 44 | loop.removeVectorOperandsDeviceTypeAttr(); |
| 45 | } else if (loop.hasWorker() || loop.getWorkerValue()) { |
| 46 | loop.removeWorkerAttr(); |
| 47 | loop.removeWorkerNumOperandsDeviceTypeAttr(); |
| 48 | } |
| 49 | } |
| 50 | |
| 51 | // Create a new ACC loop with new steps, lb, ub from original loop |
| 52 | static mlir::acc::LoopOp createACCLoopFromOriginal( |
| 53 | mlir::acc::LoopOp origLoop, mlir::RewriterBase &rewriter, |
| 54 | mlir::ValueRange lb, mlir::ValueRange ub, mlir::ValueRange step, |
| 55 | mlir::DenseBoolArrayAttr inclusiveUBAttr, |
| 56 | mlir::acc::CombinedConstructsTypeAttr combinedAttr, mlir::Location loc) { |
| 57 | mlir::ArrayAttr collapseAttr = mlir::ArrayAttr{}; |
| 58 | mlir::ArrayAttr collapseDeviceTypeAttr = mlir::ArrayAttr{}; |
| 59 | auto newLoop = mlir::acc::LoopOp::create( |
| 60 | builder&: rewriter, location: loc, results: origLoop->getResultTypes(), lowerbound: lb, upperbound: ub, step, inclusiveUpperbound: inclusiveUBAttr, |
| 61 | collapse: collapseAttr, collapseDeviceType: collapseDeviceTypeAttr, gangOperands: origLoop.getGangOperands(), |
| 62 | gangOperandsArgType: origLoop.getGangOperandsArgTypeAttr(), |
| 63 | gangOperandsSegments: origLoop.getGangOperandsSegmentsAttr(), |
| 64 | gangOperandsDeviceType: origLoop.getGangOperandsDeviceTypeAttr(), workerNumOperands: origLoop.getWorkerNumOperands(), |
| 65 | workerNumOperandsDeviceType: origLoop.getWorkerNumOperandsDeviceTypeAttr(), |
| 66 | vectorOperands: origLoop.getVectorOperands(), vectorOperandsDeviceType: origLoop.getVectorOperandsDeviceTypeAttr(), |
| 67 | seq: origLoop.getSeqAttr(), independent: origLoop.getIndependentAttr(), |
| 68 | auto_: origLoop.getAuto_Attr(), gang: origLoop.getGangAttr(), worker: origLoop.getWorkerAttr(), |
| 69 | vector: origLoop.getVectorAttr(), tileOperands: mlir::ValueRange{}, tileOperandsSegments: mlir::DenseI32ArrayAttr{}, |
| 70 | tileOperandsDeviceType: mlir::ArrayAttr{}, cacheOperands: origLoop.getCacheOperands(), |
| 71 | privateOperands: origLoop.getPrivateOperands(), firstprivateOperands: origLoop.getFirstprivateOperands(), |
| 72 | reductionOperands: origLoop.getReductionOperands(), combined: combinedAttr); |
| 73 | return newLoop; |
| 74 | } |
| 75 | |
| 76 | // Move ops from source to target Loop and replace uses of IVs |
| 77 | static void moveOpsAndReplaceIVs(mlir::acc::LoopOp sourceLoop, |
| 78 | mlir::acc::LoopOp targetLoop, |
| 79 | llvm::ArrayRef<mlir::Value> newIVs, |
| 80 | llvm::ArrayRef<mlir::Value> origIVs, |
| 81 | size_t nOps, mlir::RewriterBase &rewriter) { |
| 82 | // nOps includes the terminator; move all ops except the terminator: |
| 83 | // [begin, begin + nOps - 1) |
| 84 | mlir::Block::iterator begin = sourceLoop.getBody().begin(); |
| 85 | mlir::Block::iterator end = std::next(x: begin, n: nOps - 1); |
| 86 | |
| 87 | // Notify the rewriter about all ops being moved (and their nested ops). |
| 88 | // Directly moved ops have their parent block changed (rewriter fingerprint |
| 89 | // tracking invalidated). Nested ops may have operands replaced by |
| 90 | // replaceAllUsesInRegionWith below. |
| 91 | llvm::SmallVector<mlir::Operation *> movedOps; |
| 92 | for (mlir::Block::iterator it = begin; it != end; ++it) |
| 93 | it->walk(callback: [&](mlir::Operation *op) { |
| 94 | movedOps.push_back(Elt: op); |
| 95 | rewriter.startOpModification(op); |
| 96 | }); |
| 97 | |
| 98 | targetLoop.getBody().getOperations().splice( |
| 99 | where: targetLoop.getBody().getOperations().begin(), |
| 100 | L2&: sourceLoop.getBody().getOperations(), first: begin, last: end); |
| 101 | |
| 102 | // Replace uses of origIV with newIV |
| 103 | for (auto [i, newIV] : llvm::enumerate(First&: newIVs)) |
| 104 | mlir::replaceAllUsesInRegionWith(orig: origIVs[i], replacement: newIV, region&: targetLoop.getRegion()); |
| 105 | |
| 106 | for (mlir::Operation *op : movedOps) |
| 107 | rewriter.finalizeOpModification(op); |
| 108 | } |
| 109 | |
| 110 | // Create a single "element group" loop nested in `tileLoop`, carrying |
| 111 | // `ivTypes.size()` induction variables so the whole element space is one |
| 112 | // multi-IV loop. The element group carries vector or worker but not gang. |
| 113 | static mlir::acc::LoopOp |
| 114 | createElementGroupLoop(mlir::acc::LoopOp tileLoop, mlir::RewriterBase &rewriter, |
| 115 | mlir::ValueRange lbs, mlir::ValueRange ubs, |
| 116 | mlir::ValueRange steps, |
| 117 | mlir::DenseBoolArrayAttr inclusiveUBAttr, |
| 118 | llvm::ArrayRef<mlir::Type> ivTypes, mlir::Location loc) { |
| 119 | mlir::acc::LoopOp elementLoop = createACCLoopFromOriginal( |
| 120 | origLoop: tileLoop, rewriter, lb: lbs, ub: ubs, step: steps, inclusiveUBAttr, |
| 121 | combinedAttr: mlir::acc::CombinedConstructsTypeAttr{}, loc); |
| 122 | |
| 123 | // Drop gang from the element group, keeping vector/worker. The operand |
| 124 | // values must be cleared too, not just the attributes. |
| 125 | rewriter.startOpModification(op: elementLoop); |
| 126 | if (tileLoop.hasGang() || |
| 127 | tileLoop.getGangValue(gangArgType: mlir::acc::GangArgType::Num) || |
| 128 | tileLoop.getGangValue(gangArgType: mlir::acc::GangArgType::Dim) || |
| 129 | tileLoop.getGangValue(gangArgType: mlir::acc::GangArgType::Static)) { |
| 130 | elementLoop.removeGangAttr(); |
| 131 | elementLoop.removeGangOperandsArgTypeAttr(); |
| 132 | elementLoop.removeGangOperandsSegmentsAttr(); |
| 133 | elementLoop.removeGangOperandsDeviceTypeAttr(); |
| 134 | elementLoop.getGangOperandsMutable().clear(); |
| 135 | } |
| 136 | if (tileLoop.hasVector() || tileLoop.getVectorValue()) { |
| 137 | elementLoop.removeWorkerAttr(); |
| 138 | elementLoop.removeWorkerNumOperandsDeviceTypeAttr(); |
| 139 | elementLoop.getWorkerNumOperandsMutable().clear(); |
| 140 | } |
| 141 | rewriter.finalizeOpModification(op: elementLoop); |
| 142 | |
| 143 | // Create the element loop body: one block argument per IV plus a terminator. |
| 144 | mlir::Block *blk = rewriter.createBlock(parent: &elementLoop.getRegion(), |
| 145 | insertPt: elementLoop.getRegion().begin()); |
| 146 | rewriter.setInsertionPointToEnd(blk); |
| 147 | mlir::acc::YieldOp::create(builder&: rewriter, location: loc); |
| 148 | for (mlir::Type ivType : ivTypes) |
| 149 | elementLoop.getBody().addArgument(type: ivType, loc); |
| 150 | |
| 151 | return elementLoop; |
| 152 | } |
| 153 | |
| 154 | mlir::acc::LoopOp |
| 155 | mlir::acc::tileACCLoops(mlir::acc::LoopOp tileLoop, |
| 156 | const llvm::SmallVector<mlir::Value> &tileSizes, |
| 157 | int32_t defaultTileSize, mlir::RewriterBase &rewriter) { |
| 158 | // Tile a single fused acc.loop that carries all associated induction |
| 159 | // variables. This keeps the tile iterations as one multi-IV "tile group" |
| 160 | // loop and the in-tile iterations as one multi-IV "element group" loop, each |
| 161 | // spanning all of its induction variables. |
| 162 | const mlir::Location loc = tileLoop.getLoc(); |
| 163 | const unsigned tileCount = tileSizes.size(); |
| 164 | |
| 165 | llvm::SmallVector<mlir::Value, 3> origIVs(tileLoop.getBody().getArguments()); |
| 166 | llvm::SmallVector<mlir::Value, 3> origUBs(tileLoop.getUpperbound()); |
| 167 | llvm::SmallVector<mlir::Value, 3> origSteps(tileLoop.getStep()); |
| 168 | const unsigned numIVs = origIVs.size(); |
| 169 | const size_t nOps = tileLoop.getBody().getOperations().size(); |
| 170 | |
| 171 | // Original inclusive-UB flags (default false when the attribute is absent). |
| 172 | llvm::SmallVector<bool> inclusiveUBs; |
| 173 | for (unsigned i = 0; i < numIVs; ++i) { |
| 174 | if (tileLoop.getInclusiveUpperboundAttr()) |
| 175 | inclusiveUBs.push_back( |
| 176 | Elt: tileLoop.getInclusiveUpperboundAttr().asArrayRef()[i]); |
| 177 | else |
| 178 | inclusiveUBs.push_back(Elt: false); |
| 179 | } |
| 180 | |
| 181 | // Scale each tiled dimension's step by its tile size to form the tile group |
| 182 | // loop steps. The tile sizes are in clause order, so the first one belongs |
| 183 | // to the innermost tiled IV. |
| 184 | rewriter.setInsertionPoint(tileLoop); |
| 185 | llvm::SmallVector<mlir::Value, 3> scaledSteps; |
| 186 | llvm::SmallVector<mlir::Value, 3> tileLoopSteps; |
| 187 | for (unsigned i = 0; i < numIVs; ++i) { |
| 188 | if (i < tileCount) { |
| 189 | mlir::Value tileSize = |
| 190 | resolveAndCastTileSize(tileSize: tileSizes[tileCount - 1 - i], defaultTileSize, |
| 191 | targetType: origSteps[i].getType(), rewriter, loc); |
| 192 | mlir::Value scaled = |
| 193 | mlir::arith::MulIOp::create(builder&: rewriter, location: loc, lhs: origSteps[i], rhs: tileSize); |
| 194 | scaledSteps.push_back(Elt: scaled); |
| 195 | tileLoopSteps.push_back(Elt: scaled); |
| 196 | } else { |
| 197 | tileLoopSteps.push_back(Elt: origSteps[i]); |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | // Compute each element-loop upper bound by clamping the tile's far edge, |
| 202 | // origIV + scaledStep, to origUB. An ascending loop takes |
| 203 | // min(origUB, edge - 1), a descending one max(origUB, edge + 1). The -1 and |
| 204 | // +1 apply in the presence of inclusive bounds and are omitted otherwise. |
| 205 | rewriter.setInsertionPoint(tileLoop.getBody().getTerminator()); |
| 206 | llvm::SmallVector<mlir::Value, 3> elemLBs, elemUBs, elemSteps; |
| 207 | llvm::SmallVector<mlir::Type, 3> elemIVTypes; |
| 208 | llvm::SmallVector<bool> elemInclusiveUBs; |
| 209 | for (unsigned i = 0; i < tileCount; ++i) { |
| 210 | mlir::Value stepped = |
| 211 | mlir::arith::AddIOp::create(builder&: rewriter, location: loc, lhs: origIVs[i], rhs: scaledSteps[i]); |
| 212 | mlir::Type stepTy = scaledSteps[i].getType(); |
| 213 | std::optional<int64_t> constStep = mlir::getConstantIntValue(ofr: origSteps[i]); |
| 214 | |
| 215 | // The inclusive edge is one iteration short of the next tile's start, in |
| 216 | // whichever direction the loop runs. |
| 217 | auto inclusiveEdge = [&](bool descending) -> mlir::Value { |
| 218 | if (!inclusiveUBs[i]) |
| 219 | return stepped; |
| 220 | mlir::Value c1 = mlir::arith::ConstantOp::create( |
| 221 | builder&: rewriter, location: loc, result: stepTy, value: rewriter.getIntegerAttr(type: stepTy, value: 1)); |
| 222 | if (descending) |
| 223 | return mlir::arith::AddIOp::create(builder&: rewriter, location: loc, lhs: stepped, rhs: c1); |
| 224 | return mlir::arith::SubIOp::create(builder&: rewriter, location: loc, lhs: stepped, rhs: c1); |
| 225 | }; |
| 226 | |
| 227 | mlir::Value elemUB; |
| 228 | if (constStep && *constStep < 0) { |
| 229 | elemUB = mlir::arith::MaxSIOp::create(builder&: rewriter, location: loc, lhs: origUBs[i], |
| 230 | rhs: inclusiveEdge(/*descending=*/true)); |
| 231 | } else if (constStep) { |
| 232 | elemUB = mlir::arith::MinSIOp::create( |
| 233 | builder&: rewriter, location: loc, lhs: origUBs[i], rhs: inclusiveEdge(/*descending=*/false)); |
| 234 | } else { |
| 235 | // The sign is only known at run time, so compute both and select. |
| 236 | mlir::Type origStepTy = origSteps[i].getType(); |
| 237 | mlir::Value zero = mlir::arith::ConstantOp::create( |
| 238 | builder&: rewriter, location: loc, result: origStepTy, value: rewriter.getIntegerAttr(type: origStepTy, value: 0)); |
| 239 | mlir::Value isDescending = mlir::arith::CmpIOp::create( |
| 240 | builder&: rewriter, location: loc, predicate: mlir::arith::CmpIPredicate::slt, lhs: origSteps[i], rhs: zero); |
| 241 | mlir::Value descUB = mlir::arith::MaxSIOp::create( |
| 242 | builder&: rewriter, location: loc, lhs: origUBs[i], rhs: inclusiveEdge(/*descending=*/true)); |
| 243 | mlir::Value ascUB = mlir::arith::MinSIOp::create( |
| 244 | builder&: rewriter, location: loc, lhs: origUBs[i], rhs: inclusiveEdge(/*descending=*/false)); |
| 245 | elemUB = mlir::arith::SelectOp::create(builder&: rewriter, location: loc, condition: isDescending, |
| 246 | true_value: descUB, false_value: ascUB); |
| 247 | } |
| 248 | elemUBs.push_back(Elt: elemUB); |
| 249 | elemLBs.push_back(Elt: origIVs[i]); |
| 250 | elemSteps.push_back(Elt: origSteps[i]); |
| 251 | elemIVTypes.push_back(Elt: origIVs[i].getType()); |
| 252 | elemInclusiveUBs.push_back(Elt: inclusiveUBs[i]); |
| 253 | } |
| 254 | |
| 255 | // Only attach an inclusiveUpperbound attribute if at least one element |
| 256 | // dimension is inclusive. |
| 257 | mlir::DenseBoolArrayAttr elemInclAttr = mlir::DenseBoolArrayAttr{}; |
| 258 | if (llvm::is_contained(Range&: elemInclusiveUBs, Element: true)) |
| 259 | elemInclAttr = rewriter.getDenseBoolArrayAttr(values: elemInclusiveUBs); |
| 260 | |
| 261 | // Create the element group loop from the unmodified tile loop. |
| 262 | mlir::acc::LoopOp elementLoop = |
| 263 | createElementGroupLoop(tileLoop, rewriter, lbs: elemLBs, ubs: elemUBs, steps: elemSteps, |
| 264 | inclusiveUBAttr: elemInclAttr, ivTypes: elemIVTypes, loc); |
| 265 | |
| 266 | // Move the original body into the element loop and remap the tiled IVs to the |
| 267 | // element IVs. |
| 268 | llvm::SmallVector<mlir::Value, 3> newIVs( |
| 269 | elementLoop.getBody().getArguments()); |
| 270 | llvm::SmallVector<mlir::Value, 3> tiledOrigIVs(origIVs.begin(), |
| 271 | origIVs.begin() + tileCount); |
| 272 | moveOpsAndReplaceIVs(sourceLoop: tileLoop, targetLoop: elementLoop, newIVs, origIVs: tiledOrigIVs, nOps, |
| 273 | rewriter); |
| 274 | |
| 275 | // Turn the fused loop into the tile group: scaled steps, gang only. |
| 276 | rewriter.startOpModification(op: tileLoop); |
| 277 | tileLoop.getStepMutable().clear(); |
| 278 | tileLoop.getStepMutable().append(values: tileLoopSteps); |
| 279 | removeWorkerVectorFromLoop(loop: tileLoop); |
| 280 | rewriter.finalizeOpModification(op: tileLoop); |
| 281 | |
| 282 | return tileLoop; |
| 283 | } |
| 284 | |