| 1 | //===- ACCComputeLowering.cpp - Lower ACC compute to compute_region -------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This pass decomposes OpenACC compute constructs into a representation that |
| 10 | // separates the data environment from the compute portion and prepares for |
| 11 | // parallelism assignment and privatization at the appropriate level. |
| 12 | // |
| 13 | // Overview: |
| 14 | // --------- |
| 15 | // Each compute construct (`acc.parallel`, `acc.serial`, `acc.kernels`) is |
| 16 | // lowered to (1) `acc.kernel_environment`, which captures the data environment |
| 17 | // and (2) `acc.compute_region`, which holds the compute body. Inside the |
| 18 | // compute region, acc.loop is converted to SCF loops (`scf.parallel` or |
| 19 | // `scf.for`) with any predetermined parallelism expressed as `par_dims`. This |
| 20 | // decomposition allows later phases to assign parallelism and handle |
| 21 | // privatization at the right granularity. |
| 22 | // |
| 23 | // Transformations: |
| 24 | // ---------------- |
| 25 | // 1. Compute constructs: acc.parallel, acc.serial, and acc.kernels are |
| 26 | // replaced by acc.kernel_environment containing a single acc.compute_region. |
| 27 | // For acc.parallel / acc.kernels, launch arguments (num_gangs, num_workers, |
| 28 | // vector_length) become acc.par_width ops (each result is `index`) and are |
| 29 | // passed as compute_region launch operands. Compute regions with |
| 30 | // num_gangs(1), num_workers(1), and vector_length(1) and acc serial use a |
| 31 | // single sequential acc.par_width launch operand. |
| 32 | // |
| 33 | // 2. acc.loop: Converted according to context and attributes: |
| 34 | // - Unstructured: body wrapped in scf.execute_region. |
| 35 | // - Sequential (serial region, seq clause, or compute region with |
| 36 | // num_gangs(1), num_workers(1), and vector_length(1)): |
| 37 | // scf.parallel with par_dims = sequential. |
| 38 | // - Auto (in parallel/kernels): scf.for with collapse when |
| 39 | // multi-dimensional. |
| 40 | // - Orphan (not inside a compute construct): scf.for, no collapse. |
| 41 | // - Independent (in parallel/kernels): scf.parallel with par_dims from |
| 42 | // gang/worker/vector mapping (e.g. block_x). |
| 43 | // |
| 44 | //===----------------------------------------------------------------------===// |
| 45 | |
| 46 | #include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h" |
| 47 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h" |
| 48 | |
| 49 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 50 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 51 | #include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h" |
| 52 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 53 | #include "mlir/Dialect/OpenACC/OpenACCParMapping.h" |
| 54 | #include "mlir/Dialect/OpenACC/OpenACCUtils.h" |
| 55 | #include "mlir/Dialect/OpenACC/OpenACCUtilsCG.h" |
| 56 | #include "mlir/Dialect/OpenACC/OpenACCUtilsLoop.h" |
| 57 | #include "mlir/Dialect/Utils/StaticValueUtils.h" |
| 58 | #include "mlir/IR/IRMapping.h" |
| 59 | #include "mlir/IR/Matchers.h" |
| 60 | #include "mlir/Interfaces/FunctionInterfaces.h" |
| 61 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 62 | #include "mlir/Transforms/RegionUtils.h" |
| 63 | #include "llvm/ADT/DenseMap.h" |
| 64 | #include "llvm/ADT/STLExtras.h" |
| 65 | |
| 66 | namespace mlir { |
| 67 | namespace acc { |
| 68 | #define GEN_PASS_DEF_ACCCOMPUTELOWERING |
| 69 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc" |
| 70 | } // namespace acc |
| 71 | } // namespace mlir |
| 72 | |
| 73 | #define DEBUG_TYPE "acc-compute-lowering" |
| 74 | |
| 75 | using namespace mlir; |
| 76 | using namespace mlir::acc; |
| 77 | |
| 78 | namespace { |
| 79 | //===----------------------------------------------------------------------===// |
| 80 | // Helper functions |
| 81 | //===----------------------------------------------------------------------===// |
| 82 | |
| 83 | static bool isOpInComputeRegion(Operation *op) { |
| 84 | Region *region = op->getBlock()->getParent(); |
| 85 | return getEnclosingComputeOp(region&: *region) != nullptr; |
| 86 | } |
| 87 | |
| 88 | static bool isOpInSerialRegion(Operation *op) { |
| 89 | if (auto parallelOp = op->getParentOfType<ParallelOp>()) |
| 90 | return parallelOp.isEffectivelySerial(); |
| 91 | if (auto kernelsOp = op->getParentOfType<KernelsOp>()) |
| 92 | return kernelsOp.isEffectivelySerial(); |
| 93 | if (op->getParentOfType<SerialOp>()) |
| 94 | return true; |
| 95 | if (auto computeRegion = op->getParentOfType<ComputeRegionOp>()) |
| 96 | return computeRegion.isEffectivelySerial(); |
| 97 | if (auto funcOp = op->getParentOfType<FunctionOpInterface>()) { |
| 98 | if (isSpecializedAccRoutine(op: funcOp)) { |
| 99 | auto attr = funcOp->getDiscardableAttrOfType<SpecializedRoutineAttr>( |
| 100 | name: getSpecializedRoutineAttrName()); |
| 101 | if (attr && attr.getLevel().getValue() == ParLevel::seq) |
| 102 | return true; |
| 103 | } |
| 104 | } |
| 105 | return false; |
| 106 | } |
| 107 | |
| 108 | /// Clone defining ops of constant live-in values into `region`, rewrite uses |
| 109 | /// inside the region to the clones, and remove those values from |
| 110 | /// `liveInValues` so they are not threaded through `acc.compute_region` ins. |
| 111 | static void materializeConstantLiveInsIntoRegion(Region ®ion, |
| 112 | SetVector<Value> &liveInValues, |
| 113 | RewriterBase &rewriter) { |
| 114 | SmallVector<Value> constantLiveIns; |
| 115 | for (Value v : liveInValues) { |
| 116 | Operation *defOp = v.getDefiningOp(); |
| 117 | if (defOp && matchPattern(op: defOp, pattern: m_Constant())) { |
| 118 | // As per the definition of ConstantLike trait, constants must have a |
| 119 | // single result. |
| 120 | assert(defOp->getNumResults() == 1 && |
| 121 | "constants must have a single result" ); |
| 122 | constantLiveIns.push_back(Elt: v); |
| 123 | } |
| 124 | } |
| 125 | if (constantLiveIns.empty()) |
| 126 | return; |
| 127 | |
| 128 | OpBuilder::InsertionGuard guard(rewriter); |
| 129 | rewriter.setInsertionPointToStart(®ion.front()); |
| 130 | |
| 131 | for (Value v : constantLiveIns) { |
| 132 | Value newV = rewriter.clone(op&: *v.getDefiningOp())->getResult(idx: 0); |
| 133 | replaceAllUsesInRegionWith(orig: v, replacement: newV, region); |
| 134 | liveInValues.remove(X: v); |
| 135 | } |
| 136 | } |
| 137 | |
| 138 | /// Return the device type from which gang/worker/vector clauses should be read. |
| 139 | /// If the requested device type has any such clauses, use that exclusively; |
| 140 | /// otherwise fall back to the default (DeviceType::None). |
| 141 | static DeviceType getGangWorkerVectorDeviceType(LoopOp loopOp, |
| 142 | DeviceType deviceType) { |
| 143 | if (deviceType != DeviceType::None && |
| 144 | loopOp.hasAnyGangWorkerVector(deviceType)) |
| 145 | return deviceType; |
| 146 | return DeviceType::None; |
| 147 | } |
| 148 | |
| 149 | template <typename ComputeConstructT> |
| 150 | static DeviceType getParDimsDeviceType(ComputeConstructT computeOp, |
| 151 | DeviceType deviceType) { |
| 152 | if (deviceType != DeviceType::None && |
| 153 | computeOp.hasAnyGangWorkerVector(deviceType)) |
| 154 | return deviceType; |
| 155 | return DeviceType::None; |
| 156 | } |
| 157 | |
| 158 | /// Constant sized gang/worker/vector clauses collected per compute construct. |
| 159 | struct SizedLevel { |
| 160 | ParLevel level; |
| 161 | int64_t size; |
| 162 | }; |
| 163 | using SizedLevelMap = DenseMap<Operation *, SmallVector<SizedLevel>>; |
| 164 | |
| 165 | /// Record a sized clause if `size` is a constant. A non-constant vector or |
| 166 | /// worker clause is ignored. Other non-constant sizes are NYI. |
| 167 | static LogicalResult tryAddSizedLevel(SizedLevelMap &sizedLevelMap, |
| 168 | Operation *computeOp, LoopOp loopOp, |
| 169 | ParLevel level, Value size, |
| 170 | OpenACCSupport &accSupport) { |
| 171 | if (!size) |
| 172 | return success(); |
| 173 | std::optional<int64_t> constSize = getConstantIntValue(ofr: size); |
| 174 | if (!constSize) { |
| 175 | if (level == ParLevel::vector || level == ParLevel::worker) { |
| 176 | accSupport.emitRemark(op: loopOp, |
| 177 | message: level == ParLevel::vector |
| 178 | ? "ignoring non-constant vector clause" |
| 179 | : "ignoring non-constant worker clause" , |
| 180 | DEBUG_TYPE); |
| 181 | return success(); |
| 182 | } |
| 183 | accSupport.emitNYI(loc: loopOp.getLoc(), |
| 184 | message: "non-constant sized parallelism clause" ); |
| 185 | return failure(); |
| 186 | } |
| 187 | sizedLevelMap[computeOp].push_back(Elt: {.level: level, .size: *constSize}); |
| 188 | return success(); |
| 189 | } |
| 190 | |
| 191 | /// Collect constant sized levels from loops in `acc.kernels` regions. |
| 192 | static LogicalResult fillSizedLevelMap(Operation *op, DeviceType deviceType, |
| 193 | SizedLevelMap &sizedLevelMap, |
| 194 | OpenACCSupport &accSupport) { |
| 195 | WalkResult result = op->walk(callback: [&](LoopOp loopOp) { |
| 196 | Operation *computeOp = |
| 197 | getEnclosingComputeOp(region&: *loopOp->getBlock()->getParent()); |
| 198 | if (!computeOp || !isa<KernelsOp>(Val: computeOp)) |
| 199 | return WalkResult::advance(); |
| 200 | DeviceType loopDeviceType = |
| 201 | getGangWorkerVectorDeviceType(loopOp, deviceType); |
| 202 | if (failed(Result: tryAddSizedLevel( |
| 203 | sizedLevelMap, computeOp, loopOp, level: ParLevel::vector, |
| 204 | size: loopOp.getVectorValue(deviceType: loopDeviceType), accSupport)) || |
| 205 | failed(Result: tryAddSizedLevel( |
| 206 | sizedLevelMap, computeOp, loopOp, level: ParLevel::worker, |
| 207 | size: loopOp.getWorkerValue(deviceType: loopDeviceType), accSupport)) || |
| 208 | failed(Result: tryAddSizedLevel( |
| 209 | sizedLevelMap, computeOp, loopOp, level: ParLevel::gang_dim1, |
| 210 | size: loopOp.getGangValue(gangArgType: GangArgType::Num, deviceType: loopDeviceType), accSupport))) |
| 211 | return WalkResult::interrupt(); |
| 212 | return WalkResult::advance(); |
| 213 | }); |
| 214 | return failure(IsFailure: result.wasInterrupted()); |
| 215 | } |
| 216 | |
| 217 | /// Map loop parallelism clauses (gang/worker/vector) to GPU parallel |
| 218 | /// dimensions using the given mapping policy. Sized clauses (e.g. vector(n)) |
| 219 | /// count as the corresponding level. |
| 220 | static SmallVector<GPUParallelDimAttr> |
| 221 | getParallelDimensions(LoopOp loopOp, const ACCToGPUMappingPolicy &policy, |
| 222 | DeviceType deviceType) { |
| 223 | deviceType = getGangWorkerVectorDeviceType(loopOp, deviceType); |
| 224 | SmallVector<GPUParallelDimAttr> parDims; |
| 225 | auto *ctx = loopOp->getContext(); |
| 226 | |
| 227 | if (loopOp.hasVector(deviceType) || loopOp.getVectorValue(deviceType)) |
| 228 | insertParDim(parDims, parDim: policy.vectorDim(ctx)); |
| 229 | if (loopOp.hasWorker(deviceType) || loopOp.getWorkerValue(deviceType)) |
| 230 | insertParDim(parDims, parDim: policy.workerDim(ctx)); |
| 231 | if (auto gangDimValue = loopOp.getGangValue(gangArgType: GangArgType::Dim, deviceType)) { |
| 232 | if (auto gangDimDefOp = |
| 233 | gangDimValue.getDefiningOp<arith::ConstantIntOp>()) { |
| 234 | auto gangLevel = getGangParLevel(gangDimValue: gangDimDefOp.value()); |
| 235 | insertParDim(parDims, parDim: policy.gangDim(ctx, level: gangLevel)); |
| 236 | } |
| 237 | } else if (loopOp.hasGang(deviceType) || |
| 238 | loopOp.getGangValue(gangArgType: GangArgType::Num, deviceType) || |
| 239 | loopOp.getGangValue(gangArgType: GangArgType::Static, deviceType)) { |
| 240 | insertParDim(parDims, parDim: policy.gangDim(ctx, level: ParLevel::gang_dim1)); |
| 241 | } |
| 242 | return parDims; |
| 243 | } |
| 244 | |
| 245 | /// If \p loopOp has a `gang(static:)` value, return the corresponding |
| 246 | /// `acc.chunk_size` attribute. A constant size is stored directly; a |
| 247 | /// non-constant size is recorded as -1, the same encoding as `static:*`. |
| 248 | /// Returns a null attribute when the clause is absent. |
| 249 | static ChunkSizeAttr getChunkSizeFromStatic(LoopOp loopOp, |
| 250 | DeviceType deviceType) { |
| 251 | DeviceType loopDeviceType = getGangWorkerVectorDeviceType(loopOp, deviceType); |
| 252 | Value gangStatic = loopOp.getGangValue(gangArgType: GangArgType::Static, deviceType: loopDeviceType); |
| 253 | if (!gangStatic) |
| 254 | return {}; |
| 255 | std::optional<int64_t> chunkSize = getConstantIntValue(ofr: gangStatic); |
| 256 | // Non-constant static is treated like `static:*` (implementation-chosen). |
| 257 | return ChunkSizeAttr::get(context: loopOp->getContext(), chunk_size: chunkSize.value_or(u: -1)); |
| 258 | } |
| 259 | |
| 260 | /// Build `acc.compute_region` launch operands: one sequential `acc.par_width` |
| 261 | /// for `acc.serial`, for `acc.parallel` / `acc.kernels` when every num_gangs |
| 262 | /// operand and num_workers / vector_length are the constant 1, and otherwise |
| 263 | /// `acc.par_width` from gang/worker/vector (device-type operands first, then |
| 264 | /// default DeviceType::None). |
| 265 | template <typename ComputeConstructT> |
| 266 | static SmallVector<Value> assignKnownLaunchArgs( |
| 267 | ComputeConstructT computeOp, DeviceType deviceType, RewriterBase &rewriter, |
| 268 | const ACCToGPUMappingPolicy &policy, const SizedLevelMap &sizedLevelMap) { |
| 269 | auto *ctx = rewriter.getContext(); |
| 270 | auto loc = computeOp->getLoc(); |
| 271 | |
| 272 | if constexpr (std::is_same_v<ComputeConstructT, SerialOp>) { |
| 273 | return {ParWidthOp::create(rewriter, loc, Value(), policy.seqDim(ctx))}; |
| 274 | } else if constexpr (llvm::is_one_of<ComputeConstructT, ParallelOp, |
| 275 | KernelsOp>::value) { |
| 276 | if (computeOp.isEffectivelySerial()) |
| 277 | return {ParWidthOp::create(rewriter, loc, Value(), policy.seqDim(ctx))}; |
| 278 | |
| 279 | deviceType = getParDimsDeviceType(computeOp, deviceType); |
| 280 | |
| 281 | SmallVector<Value> values; |
| 282 | auto indexTy = rewriter.getIndexType(); |
| 283 | |
| 284 | auto numGangs = computeOp.getNumGangsValues(deviceType); |
| 285 | for (auto [gangDimIdx, gangSize] : llvm::enumerate(numGangs)) { |
| 286 | auto gangLevel = getGangParLevel(gangDimIdx + 1); |
| 287 | values.push_back(Elt: ParWidthOp::create( |
| 288 | rewriter, loc, |
| 289 | getValueOrCreateCastToIndexLike(rewriter, gangSize.getLoc(), indexTy, |
| 290 | gangSize), |
| 291 | policy.gangDim(ctx, level: gangLevel))); |
| 292 | } |
| 293 | |
| 294 | Value numWorkers = computeOp.getNumWorkersValue(deviceType); |
| 295 | if (numWorkers) { |
| 296 | values.push_back(Elt: ParWidthOp::create( |
| 297 | rewriter, loc, |
| 298 | getValueOrCreateCastToIndexLike(b&: rewriter, loc: numWorkers.getLoc(), |
| 299 | targetType: indexTy, value: numWorkers), |
| 300 | policy.workerDim(ctx))); |
| 301 | } |
| 302 | |
| 303 | Value vectorLength = computeOp.getVectorLengthValue(deviceType); |
| 304 | if (vectorLength) { |
| 305 | values.push_back(Elt: ParWidthOp::create( |
| 306 | rewriter, loc, |
| 307 | getValueOrCreateCastToIndexLike(b&: rewriter, loc: vectorLength.getLoc(), |
| 308 | targetType: indexTy, value: vectorLength), |
| 309 | policy.vectorDim(ctx))); |
| 310 | } |
| 311 | |
| 312 | // Loop-level sized clauses. Skip a dim already set on the construct. |
| 313 | // Rematerialize the constant here so it dominates the compute region. |
| 314 | auto sizedLevels = sizedLevelMap.find(computeOp.getOperation()); |
| 315 | if (sizedLevels != sizedLevelMap.end()) { |
| 316 | for (const SizedLevel &sizedLevel : sizedLevels->second) { |
| 317 | GPUParallelDimAttr dim = policy.map(ctx, level: sizedLevel.level); |
| 318 | bool exists = llvm::any_of(values, [&](Value v) { |
| 319 | auto parWidth = v.getDefiningOp<ParWidthOp>(); |
| 320 | return parWidth && parWidth.getParDim() == dim; |
| 321 | }); |
| 322 | if (exists) |
| 323 | continue; |
| 324 | Value sizeVal = |
| 325 | arith::ConstantIndexOp::create(rewriter, loc, sizedLevel.size); |
| 326 | values.push_back(Elt: ParWidthOp::create(rewriter, loc, sizeVal, dim)); |
| 327 | } |
| 328 | } |
| 329 | return values; |
| 330 | } else { |
| 331 | llvm_unreachable("assignKnownLaunchArgs: expected parallel, kernels, or " |
| 332 | "serial" ); |
| 333 | } |
| 334 | } |
| 335 | |
| 336 | //===----------------------------------------------------------------------===// |
| 337 | // Loop conversion pattern |
| 338 | //===----------------------------------------------------------------------===// |
| 339 | |
| 340 | class ACCLoopConversion : public OpRewritePattern<LoopOp> { |
| 341 | public: |
| 342 | ACCLoopConversion(MLIRContext *ctx, const ACCToGPUMappingPolicy &policy, |
| 343 | DeviceType deviceType) |
| 344 | : OpRewritePattern<LoopOp>(ctx), policy(policy), deviceType(deviceType) {} |
| 345 | |
| 346 | LogicalResult matchAndRewrite(LoopOp loopOp, |
| 347 | PatternRewriter &rewriter) const override { |
| 348 | if (loopOp.getUnstructured()) { |
| 349 | // An independent loop is asserted parallel but cannot be parallelized |
| 350 | // once it is lowered as a region of blocks. Record that, so the loss |
| 351 | // can be reported instead of silently running the loop sequentially. |
| 352 | bool lostIndependence = |
| 353 | loopOp.getDefaultOrDeviceTypeParallelism(deviceType) == |
| 354 | LoopParMode::loop_independent && |
| 355 | !isOpInSerialRegion(op: loopOp) && |
| 356 | (isOpInComputeRegion(op: loopOp) || |
| 357 | isSpecializedAccRoutine( |
| 358 | op: loopOp->getParentOfType<FunctionOpInterface>())); |
| 359 | auto executeRegion = |
| 360 | convertUnstructuredACCLoopToSCFExecuteRegion(loopOp, rewriter); |
| 361 | if (!executeRegion) |
| 362 | return failure(); |
| 363 | if (lostIndependence) |
| 364 | executeRegion->setDiscardableAttr( |
| 365 | name: getUnstructuredIndependentLoopAttrName(), value: rewriter.getUnitAttr()); |
| 366 | rewriter.replaceOp(op: loopOp, newOp: executeRegion); |
| 367 | return success(); |
| 368 | } |
| 369 | |
| 370 | LoopParMode parMode = loopOp.getDefaultOrDeviceTypeParallelism(deviceType); |
| 371 | |
| 372 | if (parMode == LoopParMode::loop_seq || isOpInSerialRegion(op: loopOp)) { |
| 373 | // Use scf.for with sequential loops, because the loop's parallelism is |
| 374 | // already determined. |
| 375 | auto forOp = |
| 376 | convertACCLoopToSCFFor(loopOp, rewriter, /*enableCollapse=*/true); |
| 377 | if (!forOp) |
| 378 | return failure(); |
| 379 | setParDimsAttr(op: forOp, attr: GPUParallelDimsAttr::seq(loopOp->getContext())); |
| 380 | rewriter.replaceOp(op: loopOp, newOp: forOp); |
| 381 | } else if (parMode == LoopParMode::loop_auto) { |
| 382 | // All loops in serial regions should have already been handled. |
| 383 | assert(!isOpInSerialRegion(loopOp) && |
| 384 | "Expected loop to be in non-serial region" ); |
| 385 | // Mark as scf.for to allow auto-parallelization analysis later. |
| 386 | auto forOp = |
| 387 | convertACCLoopToSCFFor(loopOp, rewriter, /*enableCollapse=*/true); |
| 388 | if (!forOp) |
| 389 | return failure(); |
| 390 | SmallVector<GPUParallelDimAttr> parDims = |
| 391 | getParallelDimensions(loopOp, policy, deviceType); |
| 392 | if (!parDims.empty()) { |
| 393 | auto parDimsAttr = |
| 394 | GPUParallelDimsAttr::get(context: loopOp->getContext(), array: parDims); |
| 395 | setParDimsAttr(op: forOp, attr: parDimsAttr); |
| 396 | if (ChunkSizeAttr chunkSize = |
| 397 | getChunkSizeFromStatic(loopOp, deviceType)) |
| 398 | setChunkSizeAttr(op: forOp, attr: chunkSize); |
| 399 | } |
| 400 | rewriter.replaceOp(op: loopOp, newOp: forOp); |
| 401 | } else if (!isOpInComputeRegion(op: loopOp) && |
| 402 | !isSpecializedAccRoutine( |
| 403 | op: loopOp->getParentOfType<FunctionOpInterface>())) { |
| 404 | // This loop is an orphan `acc loop` but it is not in any sort |
| 405 | // of compute region. Thus it is just a sequential non-accelerator loop. |
| 406 | auto forOp = |
| 407 | convertACCLoopToSCFFor(loopOp, rewriter, /*enableCollapse=*/false); |
| 408 | if (!forOp) |
| 409 | return failure(); |
| 410 | rewriter.replaceOp(op: loopOp, newOp: forOp); |
| 411 | } else { |
| 412 | assert(parMode == LoopParMode::loop_independent && |
| 413 | "Expected loop to be independent" ); |
| 414 | auto parallelOp = convertACCLoopToSCFParallel(loopOp, rewriter); |
| 415 | if (!parallelOp) |
| 416 | return failure(); |
| 417 | |
| 418 | SmallVector<GPUParallelDimAttr> parDims = |
| 419 | getParallelDimensions(loopOp, policy, deviceType); |
| 420 | if (!parDims.empty()) { |
| 421 | auto parDimsAttr = |
| 422 | GPUParallelDimsAttr::get(context: loopOp->getContext(), array: parDims); |
| 423 | setParDimsAttr(op: parallelOp, attr: parDimsAttr); |
| 424 | if (ChunkSizeAttr chunkSize = |
| 425 | getChunkSizeFromStatic(loopOp, deviceType)) |
| 426 | setChunkSizeAttr(op: parallelOp, attr: chunkSize); |
| 427 | } |
| 428 | rewriter.replaceOp(op: loopOp, newOp: parallelOp); |
| 429 | } |
| 430 | return success(); |
| 431 | } |
| 432 | |
| 433 | private: |
| 434 | const ACCToGPUMappingPolicy &policy; |
| 435 | DeviceType deviceType; |
| 436 | }; |
| 437 | |
| 438 | //===----------------------------------------------------------------------===// |
| 439 | // Compute construct conversion pattern |
| 440 | //===----------------------------------------------------------------------===// |
| 441 | |
| 442 | template <typename ComputeConstructT> |
| 443 | class ComputeOpConversion : public OpRewritePattern<ComputeConstructT> { |
| 444 | public: |
| 445 | ComputeOpConversion(MLIRContext *ctx, const ACCToGPUMappingPolicy &policy, |
| 446 | DeviceType deviceType, const SizedLevelMap &sizedLevelMap) |
| 447 | : OpRewritePattern<ComputeConstructT>(ctx), policy(policy), |
| 448 | deviceType(deviceType), sizedLevelMap(sizedLevelMap) {} |
| 449 | |
| 450 | LogicalResult matchAndRewrite(ComputeConstructT computeOp, |
| 451 | PatternRewriter &rewriter) const override { |
| 452 | rewriter.setInsertionPoint(computeOp); |
| 453 | auto kernelEnv = |
| 454 | KernelEnvironmentOp::createAndPopulate(computeOp, deviceType, rewriter); |
| 455 | auto launchArgs = assignKnownLaunchArgs(computeOp, deviceType, rewriter, |
| 456 | policy, sizedLevelMap); |
| 457 | Region ®ion = computeOp.getRegion(); |
| 458 | SetVector<Value> liveInValues; |
| 459 | getUsedValuesDefinedAbove(region, limit&: region, values&: liveInValues); |
| 460 | materializeConstantLiveInsIntoRegion(region, liveInValues, rewriter); |
| 461 | IRMapping mapping; |
| 462 | auto computeRegion = buildComputeRegion( |
| 463 | computeOp->getLoc(), launchArgs, liveInValues.getArrayRef(), |
| 464 | ComputeConstructT::getOperationName(), region, rewriter, mapping); |
| 465 | if (!computeRegion) { |
| 466 | rewriter.eraseOp(op: kernelEnv); |
| 467 | return failure(); |
| 468 | } |
| 469 | rewriter.eraseOp(op: computeOp); |
| 470 | return success(); |
| 471 | } |
| 472 | |
| 473 | private: |
| 474 | const ACCToGPUMappingPolicy &policy; |
| 475 | DeviceType deviceType; |
| 476 | const SizedLevelMap &sizedLevelMap; |
| 477 | }; |
| 478 | |
| 479 | //===----------------------------------------------------------------------===// |
| 480 | // Pass implementation |
| 481 | //===----------------------------------------------------------------------===// |
| 482 | |
| 483 | class ACCComputeLowering |
| 484 | : public acc::impl::ACCComputeLoweringBase<ACCComputeLowering> { |
| 485 | public: |
| 486 | using ACCComputeLoweringBase::ACCComputeLoweringBase; |
| 487 | |
| 488 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 489 | ACCComputeLoweringBase::getDependentDialects(registry); |
| 490 | acc::registerAllExtensions(registry); |
| 491 | } |
| 492 | |
| 493 | void runOnOperation() override { |
| 494 | auto op = getOperation(); |
| 495 | auto *context = op.getContext(); |
| 496 | |
| 497 | DefaultACCToGPUMappingPolicy policy; |
| 498 | // Collect loop sized levels before loops are rewritten away. |
| 499 | SizedLevelMap sizedLevelMap; |
| 500 | OpenACCSupport &accSupport = getAnalysis<OpenACCSupport>(); |
| 501 | if (failed(Result: fillSizedLevelMap(op, deviceType, sizedLevelMap, accSupport))) |
| 502 | return signalPassFailure(); |
| 503 | |
| 504 | // Part 1: Convert acc.loop to scf.parallel/scf.for while the parent |
| 505 | // compute construct is still present (needed to determine conversion |
| 506 | // strategy). |
| 507 | RewritePatternSet loopPatterns(context); |
| 508 | loopPatterns.insert<ACCLoopConversion>(arg&: context, args&: policy, args&: deviceType); |
| 509 | if (failed(Result: applyPatternsGreedily(op, patterns: std::move(loopPatterns)))) |
| 510 | return signalPassFailure(); |
| 511 | |
| 512 | // Part 2: Convert acc.parallel, acc.kernels, and acc.serial to |
| 513 | // acc.kernel_environment { acc.compute_region { ... } }. |
| 514 | RewritePatternSet computePatterns(context); |
| 515 | computePatterns |
| 516 | .insert<ComputeOpConversion<ParallelOp>, ComputeOpConversion<KernelsOp>, |
| 517 | ComputeOpConversion<SerialOp>>(arg&: context, args&: policy, args&: deviceType, |
| 518 | args&: sizedLevelMap); |
| 519 | if (failed(Result: applyPatternsGreedily(op, patterns: std::move(computePatterns)))) |
| 520 | return signalPassFailure(); |
| 521 | } |
| 522 | }; |
| 523 | } // namespace |
| 524 | |