| 1 | //===- ACCCGToGPU.cpp - Lower acc.compute_region to gpu.launch ------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This pass lowers `acc.compute_region` to the GPU dialect. For host-side |
| 10 | // kernels it wraps the region in `gpu.launch`; for specialized acc routines |
| 11 | // already inside a `gpu.func`, the body is lowered in place without emitting |
| 12 | // a launch. |
| 13 | // |
| 14 | // Overview: |
| 15 | // --------- |
| 16 | // `acc.compute_region` is the compute-body representation produced after |
| 17 | // OpenACC compute constructs are decomposed and parallelism has been assigned. |
| 18 | // This pass is the final ACC-to-GPU lowering step for that body: it converts |
| 19 | // nested `scf.parallel` / `scf.for` loops marked with `acc.par_dims` into GPU |
| 20 | // block and thread parallelism, materializes privatization and reductions for |
| 21 | // the device, inserts synchronization where shared state is observed across |
| 22 | // threads, and erases the ACC scaffolding (`acc.compute_region`, |
| 23 | // `acc.par_width`). |
| 24 | // |
| 25 | // Transformations: |
| 26 | // ---------------- |
| 27 | // 1. Launch creation: outside a `gpu.func`, each `acc.compute_region` becomes a |
| 28 | // `gpu.launch` whose grid and block sizes come from `acc.par_width` launch |
| 29 | // operands (defaulting to 1). Kernel/module name attributes are preserved. |
| 30 | // Inside a `gpu.func` (specialized acc routine), no launch is emitted. |
| 31 | // |
| 32 | // 2. Parallel loops: `scf.parallel` with a single `acc.par_dims` entry is |
| 33 | // mapped to the corresponding GPU dimension (`block_*` or `thread_*`). |
| 34 | // Sequential dimensions remain as `scf.parallel`/`scf.for` loops in the |
| 35 | // generated kernel body. |
| 36 | // |
| 37 | // 3. Privatization: `acc.privatize` / `acc.private_local` storage is |
| 38 | // materialized as one of: a per-thread `memref.alloca` (thread-private |
| 39 | // arrays within the stack budget), an `acc.gpu_shared_memory` buffer |
| 40 | // (gang-/worker-private arrays that fit the shared-memory budget), or a |
| 41 | // `memref.alloc` whose pointer is broadcast to the block through a small |
| 42 | // shared-memory slot (the data lives in global memory; shared memory only |
| 43 | // holds the broadcast pointer). |
| 44 | // |
| 45 | // 4. Predication: `acc.predicate_region` becomes `scf.if` guarded by active |
| 46 | // thread/block indices derived from `acc.par_dims` and launch dimensions. |
| 47 | // |
| 48 | // 5. Reductions: `acc.reduction_*` ops are lowered to GPU reduction and |
| 49 | // synchronization primitives according to each reduction's parallel |
| 50 | // dimensions and accumulator storage class. |
| 51 | // |
| 52 | // Example: |
| 53 | // -------- |
| 54 | // Before: |
| 55 | // %c128 = arith.constant 128 : index |
| 56 | // %tx = acc.par_width %c128 par_dim(#acc.par_dim<thread_x>) |
| 57 | // acc.compute_region launch(%arg0 = %tx) { |
| 58 | // %c0 = arith.constant 0 : index |
| 59 | // %c1 = arith.constant 1 : index |
| 60 | // scf.parallel (%iv) = (%c0) to (%c128) step (%c1) { |
| 61 | // ... |
| 62 | // scf.reduce |
| 63 | // } {acc.par_dims = #acc<par_dims[thread_x]>} |
| 64 | // acc.yield |
| 65 | // } <{origin = "acc.parallel"}> |
| 66 | // |
| 67 | // After: |
| 68 | // gpu.launch blocks(%bidx, %bidy, %bidz) in (%gdimx = %c1, ...) |
| 69 | // threads(%tidx, %tidy, %tidz) in (%bdimx = %c128, ...) { |
| 70 | // ... |
| 71 | // } |
| 72 | // |
| 73 | // Requirements: |
| 74 | // ------------- |
| 75 | // - Must run on a GPU device type (`device-type` option); host and multicore |
| 76 | // targets are rejected. |
| 77 | // - Input must already be in the `acc.compute_region` form: nested SCF loops |
| 78 | // carry `acc.par_dims`, privatization is expressed via `acc.privatize` / |
| 79 | // `acc.private_local`, and reductions use the `acc.reduction_*` ops. |
| 80 | // - Each `scf.parallel` processed by this pass is expected to have exactly |
| 81 | // one parallel dimension and one induction variable. |
| 82 | // - For acc routines, the `acc.compute_region` must live inside a `gpu.func` |
| 83 | // in the GPU module. |
| 84 | // - Uses `acc::OpenACCSupport` for NYI reporting and compiler remarks. |
| 85 | // - Pass options: `max-workgroup-shared-memory`, `max-thread-private-stack`, |
| 86 | // and `subgroup-size` (used for reductions and block-dimension alignment). |
| 87 | // |
| 88 | //===----------------------------------------------------------------------===// |
| 89 | |
| 90 | #include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h" |
| 91 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h" |
| 92 | |
| 93 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 94 | #include "mlir/Dialect/Arith/Utils/Utils.h" |
| 95 | #include "mlir/Dialect/Complex/IR/Complex.h" |
| 96 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 97 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 98 | #include "mlir/Dialect/GPU/Utils/GPUUtils.h" |
| 99 | #include "mlir/Dialect/LLVMIR/NVVMDialect.h" |
| 100 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 101 | #include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h" |
| 102 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 103 | #include "mlir/Dialect/OpenACC/OpenACCParMapping.h" |
| 104 | #include "mlir/Dialect/OpenACC/OpenACCUtilsCG.h" |
| 105 | #include "mlir/Dialect/OpenACC/OpenACCUtilsGPU.h" |
| 106 | #include "mlir/Dialect/OpenACC/OpenACCUtilsReduction.h" |
| 107 | #include "mlir/Dialect/OpenACC/OpenACCUtilsType.h" |
| 108 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 109 | #include "mlir/IR/Block.h" |
| 110 | #include "mlir/IR/BuiltinAttributeInterfaces.h" |
| 111 | #include "mlir/IR/BuiltinTypes.h" |
| 112 | #include "mlir/IR/Diagnostics.h" |
| 113 | #include "mlir/IR/Dominance.h" |
| 114 | #include "mlir/IR/IRMapping.h" |
| 115 | #include "mlir/IR/OpDefinition.h" |
| 116 | #include "mlir/IR/PatternMatch.h" |
| 117 | #include "mlir/IR/SymbolTable.h" |
| 118 | #include "mlir/IR/Value.h" |
| 119 | #include "mlir/Interfaces/CallInterfaces.h" |
| 120 | #include "mlir/Interfaces/SideEffectInterfaces.h" |
| 121 | #include "mlir/Interfaces/ViewLikeInterface.h" |
| 122 | #include "mlir/Support/LLVM.h" |
| 123 | #include "mlir/Transforms/DialectConversion.h" |
| 124 | #include "llvm/ADT/ArrayRef.h" |
| 125 | #include "llvm/ADT/DenseMap.h" |
| 126 | #include "llvm/ADT/STLExtras.h" |
| 127 | #include "llvm/ADT/StringExtras.h" |
| 128 | #include "llvm/ADT/Twine.h" |
| 129 | #include "llvm/Support/Debug.h" |
| 130 | #include <algorithm> |
| 131 | #include <optional> |
| 132 | #include <utility> |
| 133 | |
| 134 | namespace mlir { |
| 135 | namespace acc { |
| 136 | #define GEN_PASS_DEF_ACCCGTOGPU |
| 137 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc" |
| 138 | } // namespace acc |
| 139 | } // namespace mlir |
| 140 | |
| 141 | #define DEBUG_TYPE "acc-cg-to-gpu" |
| 142 | |
| 143 | namespace { |
| 144 | using namespace mlir; |
| 145 | using namespace mlir::acc; |
| 146 | |
| 147 | enum class PrivateMemScope { Thread, Worker, Gang, None }; |
| 148 | |
| 149 | /// Device label used in compiler remarks (e.g. "NVIDIA GPU"). |
| 150 | static std::string (DeviceType deviceType) { |
| 151 | switch (deviceType) { |
| 152 | case DeviceType::None: |
| 153 | case DeviceType::Star: |
| 154 | case DeviceType::Default: |
| 155 | return "GPU" ; |
| 156 | default: { |
| 157 | std::string name; |
| 158 | llvm::StringRef deviceName = stringifyDeviceType(deviceType); |
| 159 | name.reserve(res_arg: deviceName.size()); |
| 160 | for (char c : deviceName) |
| 161 | name.push_back(c: llvm::toUpper(x: c)); |
| 162 | return name + " GPU" ; |
| 163 | } |
| 164 | } |
| 165 | } |
| 166 | |
| 167 | /// True when \p op is inside a specialized acc routine function. |
| 168 | static bool isInsideACCSpecializedRoutine(Operation *op) { |
| 169 | FunctionOpInterface funcOp = op->getParentOfType<FunctionOpInterface>(); |
| 170 | return funcOp && acc::isSpecializedAccRoutine(op: funcOp); |
| 171 | } |
| 172 | |
| 173 | /// Maps an acc.routine's parallelism clauses to a GPU parallel dimension. |
| 174 | static GPUParallelDimAttr |
| 175 | getAccRoutineParDim(RoutineOp routineOp, MLIRContext *ctx, |
| 176 | const ACCToGPUMappingPolicy &policy) { |
| 177 | if (routineOp.getGangDimValue() || |
| 178 | routineOp.getGangDimValue(deviceType: DeviceType::Nvidia)) { |
| 179 | int64_t gangDimValue = routineOp.getGangDimValue(deviceType: DeviceType::Nvidia) |
| 180 | ? *routineOp.getGangDimValue(deviceType: DeviceType::Nvidia) |
| 181 | : *routineOp.getGangDimValue(); |
| 182 | ParLevel gangLevel = getGangParLevel(gangDimValue); |
| 183 | return policy.gangDim(ctx, level: gangLevel); |
| 184 | } |
| 185 | if (routineOp.hasGang() || routineOp.hasGang(deviceType: DeviceType::Nvidia)) |
| 186 | return policy.gangDim(ctx, level: ParLevel::gang_dim1); |
| 187 | if (routineOp.hasWorker() || routineOp.hasWorker(deviceType: DeviceType::Nvidia)) |
| 188 | return policy.workerDim(ctx); |
| 189 | if (routineOp.hasVector() || routineOp.hasVector(deviceType: DeviceType::Nvidia)) |
| 190 | return policy.vectorDim(ctx); |
| 191 | return policy.seqDim(ctx); |
| 192 | } |
| 193 | |
| 194 | /// Looks up the acc.routine symbol associated with \p funcOp. |
| 195 | static RoutineOp getRoutineOpForAccRoutineFunction(FunctionOpInterface funcOp, |
| 196 | const SymbolTable &symTab) { |
| 197 | if (isSpecializedAccRoutine(op: funcOp)) { |
| 198 | SpecializedRoutineAttr attr = |
| 199 | funcOp->getDiscardableAttrOfType<SpecializedRoutineAttr>( |
| 200 | name: getSpecializedRoutineAttrName()); |
| 201 | return symTab.lookup<RoutineOp>(name: attr.getRoutine().getLeafReference()); |
| 202 | } |
| 203 | RoutineInfoAttr routineInfo = |
| 204 | funcOp->getDiscardableAttrOfType<RoutineInfoAttr>( |
| 205 | name: getRoutineInfoAttrName()); |
| 206 | if (!routineInfo || routineInfo.getAccRoutines().empty()) |
| 207 | return nullptr; |
| 208 | return symTab.lookup<RoutineOp>( |
| 209 | name: routineInfo.getAccRoutines().front().getLeafReference()); |
| 210 | } |
| 211 | |
| 212 | /// Returns the parallelism level of a specialized acc routine function. |
| 213 | static GPUParallelDimAttr |
| 214 | getSpecializedRoutineDim(FunctionOpInterface funcOp, |
| 215 | const ACCToGPUMappingPolicy &policy) { |
| 216 | SpecializedRoutineAttr specAttr = |
| 217 | funcOp->getDiscardableAttrOfType<SpecializedRoutineAttr>( |
| 218 | name: getSpecializedRoutineAttrName()); |
| 219 | assert(specAttr && "expected specialized routine attribute" ); |
| 220 | return policy.map(ctx: funcOp->getContext(), level: specAttr.getLevel().getValue()); |
| 221 | } |
| 222 | |
| 223 | /// Returns the parallelism dimension of a callee acc routine, if any. |
| 224 | static GPUParallelDimAttr |
| 225 | getAccRoutineCallParDim(CallOpInterface callOp, |
| 226 | const ACCToGPUMappingPolicy &policy) { |
| 227 | std::optional<CallInterfaceCallable> callee = callOp.getCallableForCallee(); |
| 228 | if (!callee) |
| 229 | return nullptr; |
| 230 | SymbolRefAttr calleeSymbolRef = dyn_cast<SymbolRefAttr>(Val&: *callee); |
| 231 | if (!calleeSymbolRef) |
| 232 | return nullptr; |
| 233 | ModuleOp moduleOp = callOp->getParentOfType<ModuleOp>(); |
| 234 | if (!moduleOp) |
| 235 | return nullptr; |
| 236 | |
| 237 | SymbolTable symTab(moduleOp); |
| 238 | FunctionOpInterface funcOp = |
| 239 | symTab.lookup<FunctionOpInterface>(name: calleeSymbolRef.getLeafReference()); |
| 240 | if (!funcOp) |
| 241 | return nullptr; |
| 242 | |
| 243 | if (isSpecializedAccRoutine(op: funcOp)) |
| 244 | return getSpecializedRoutineDim(funcOp, policy); |
| 245 | if (RoutineOp routineOp = getRoutineOpForAccRoutineFunction(funcOp, symTab)) |
| 246 | return getAccRoutineParDim(routineOp, ctx: funcOp.getContext(), policy); |
| 247 | return nullptr; |
| 248 | } |
| 249 | |
| 250 | /// Collects parallel dimensions from enclosing loops and the compute region. |
| 251 | static SmallVector<GPUParallelDimAttr> getAncestorParDims(Operation *op) { |
| 252 | SmallVector<GPUParallelDimAttr> parDimsArray; |
| 253 | ComputeRegionOp computeRegion = op->getParentOfType<ComputeRegionOp>(); |
| 254 | assert(computeRegion && "missing enclosing acc.compute_region" ); |
| 255 | scf::ParallelOp parentLoop = op->getParentOfType<scf::ParallelOp>(); |
| 256 | // True while parentLoop is the innermost parallel ancestor of op. |
| 257 | bool isInnermostParallelParent = true; |
| 258 | while (parentLoop) { |
| 259 | bool hasNonSeqParDim = false; |
| 260 | if (GPUParallelDimsAttr parDimsAttr = getParDimsAttr(op: parentLoop)) { |
| 261 | for (GPUParallelDimAttr parDim : parDimsAttr.getArray()) { |
| 262 | insertParDim(parDims&: parDimsArray, parDim); |
| 263 | if (!parDim.isSeq()) |
| 264 | hasNonSeqParDim = true; |
| 265 | } |
| 266 | } |
| 267 | // Include launch dims for a block-redundant ancestor when: |
| 268 | // - it is itself worksharing (e.g. vector + gpu_block_redundant), or |
| 269 | // - it is the innermost parallel parent (sequential remnant after |
| 270 | // partition-kernel-loops), so the body is not predicated on blockIdx. |
| 271 | // Do not include them for an outer sequential block-redundant wrapper |
| 272 | // around nested gang/worker/vector worksharing: that widens gang-private |
| 273 | // storage to per-thread. |
| 274 | if (hasGPUBlockRedundantAttr(op: parentLoop) && |
| 275 | (hasNonSeqParDim || isInnermostParallelParent)) |
| 276 | for (GPUParallelDimAttr parDim : computeRegion.getLaunchParDims()) |
| 277 | insertParDim(parDims&: parDimsArray, parDim); |
| 278 | isInnermostParallelParent = false; |
| 279 | parentLoop = parentLoop->getParentOfType<scf::ParallelOp>(); |
| 280 | } |
| 281 | |
| 282 | if (GPUParallelDimsAttr parDimsAttr = getParDimsAttr(op: computeRegion)) |
| 283 | for (GPUParallelDimAttr parDim : parDimsAttr.getArray()) |
| 284 | insertParDim(parDims&: parDimsArray, parDim); |
| 285 | return parDimsArray; |
| 286 | } |
| 287 | |
| 288 | /// Strips index casts to reach the underlying defining value. |
| 289 | static Value stripIndexCastsFromValue(Value x) { |
| 290 | Operation *op = x.getDefiningOp(); |
| 291 | if (!op) |
| 292 | return x; |
| 293 | while (arith::IndexCastOp castOp = dyn_cast<arith::IndexCastOp>(Val: op)) { |
| 294 | op = castOp->getOperand(idx: 0).getDefiningOp(); |
| 295 | if (!op) |
| 296 | return x; |
| 297 | } |
| 298 | return op->getResult(idx: 0); |
| 299 | } |
| 300 | |
| 301 | /// Extracts a compile-time integer constant from \p x, when known. |
| 302 | static FailureOr<int64_t> (Value x, |
| 303 | bool stripIndexCasts = false) { |
| 304 | if (stripIndexCasts) |
| 305 | x = stripIndexCastsFromValue(x); |
| 306 | Operation *op = x.getDefiningOp(); |
| 307 | if (op) { |
| 308 | if (arith::ConstantIntOp constOp = dyn_cast<arith::ConstantIntOp>(Val: op)) { |
| 309 | assert(constOp.getType().getIntOrFloatBitWidth() <= 64); |
| 310 | return constOp.value(); |
| 311 | } |
| 312 | if (arith::ConstantIndexOp constOp = dyn_cast<arith::ConstantIndexOp>(Val: op)) |
| 313 | return constOp.value(); |
| 314 | } |
| 315 | return failure(); |
| 316 | } |
| 317 | |
| 318 | /// True when \p x is a constant equal to \p y (modulo index casts). |
| 319 | static bool sameEffectiveValue(Value x, int64_t y) { |
| 320 | x = stripIndexCastsFromValue(x); |
| 321 | FailureOr<int64_t> conX = extractIntConst(x); |
| 322 | if (failed(Result: conX)) |
| 323 | return false; |
| 324 | return *conX == y; |
| 325 | } |
| 326 | |
| 327 | /// Continues tracking a memref through view-like and partial-access ops. |
| 328 | static bool getPassThroughResults(Operation *userOp, Value trackedOperand, |
| 329 | SmallVectorImpl<Value> &passThroughResults) { |
| 330 | if (ViewLikeOpInterface viewLikeOp = dyn_cast<ViewLikeOpInterface>(Val: userOp)) { |
| 331 | if (viewLikeOp.getViewSource() == trackedOperand) { |
| 332 | passThroughResults.push_back(Elt: viewLikeOp.getViewDest()); |
| 333 | return true; |
| 334 | } |
| 335 | return false; |
| 336 | } |
| 337 | |
| 338 | // Partial-entity accesses (e.g. array element or field access) forward the |
| 339 | // base entity through to their results, so treat them as pass-through when |
| 340 | // the base entity is the value being tracked. |
| 341 | if (acc::PartialEntityAccessOpInterface partialAccess = |
| 342 | dyn_cast<acc::PartialEntityAccessOpInterface>(Val: userOp)) { |
| 343 | if (partialAccess.getBaseEntity() == trackedOperand) { |
| 344 | passThroughResults.append(in_start: userOp->result_begin(), in_end: userOp->result_end()); |
| 345 | return true; |
| 346 | } |
| 347 | return false; |
| 348 | } |
| 349 | return false; |
| 350 | } |
| 351 | |
| 352 | /// Skips memref view/cast chains to reach the underlying buffer. |
| 353 | static Value unwrapMemRefConversion(Value v) { |
| 354 | while (Operation *op = v.getDefiningOp()) { |
| 355 | if (ViewLikeOpInterface viewLike = dyn_cast<ViewLikeOpInterface>(Val: op)) { |
| 356 | if (isa<MemRefType>(Val: viewLike.getViewSource().getType()) || |
| 357 | isa<MemRefType>(Val: viewLike.getViewDest().getType())) { |
| 358 | v = viewLike.getViewSource(); |
| 359 | continue; |
| 360 | } |
| 361 | } |
| 362 | break; |
| 363 | } |
| 364 | return v; |
| 365 | } |
| 366 | |
| 367 | /// Walks back from a memref use to its defining `acc.private_local`, if any, |
| 368 | /// looking through view/cast ops. |
| 369 | static acc::PrivateLocalOp getPrivateLocalForMemref(Value memref); |
| 370 | |
| 371 | /// Returns the dimensions that own \p privateLocal. |
| 372 | static GPUParallelDimsAttr |
| 373 | getPrivateParDims(acc::PrivateLocalOp privateLocal, |
| 374 | acc::ComputeRegionOp computeRegion); |
| 375 | |
| 376 | /// True when the storage backing \p privateLocal is thread_x-private. An |
| 377 | /// unknown scope conservatively counts as per-thread. |
| 378 | static bool storageHasThreadX(acc::PrivateLocalOp privateLocal, |
| 379 | acc::ComputeRegionOp computeRegion) { |
| 380 | GPUParallelDimsAttr dims = getPrivateParDims(privateLocal, computeRegion); |
| 381 | return !dims || llvm::any_of(Range: dims.getArray(), P: [](GPUParallelDimAttr d) { |
| 382 | return d.isThreadX(); |
| 383 | }); |
| 384 | } |
| 385 | |
| 386 | /// Returns the sole user of \p v, or null if it has zero or multiple uses. |
| 387 | static Operation *getOnlyUser(Value v) { |
| 388 | if (!v.hasOneUse()) |
| 389 | return nullptr; |
| 390 | return *v.user_begin(); |
| 391 | } |
| 392 | |
| 393 | /// True when \p privatize is privatized at thread_x parallelism. |
| 394 | static bool isThreadXPrivatize(PrivatizeOp privatize) { |
| 395 | if (GPUParallelDimsAttr parDimsAttr = privatize.getParDimsAttr()) |
| 396 | return llvm::any_of(Range: parDimsAttr.getArray(), |
| 397 | P: [](GPUParallelDimAttr d) { return d.isThreadX(); }); |
| 398 | return false; |
| 399 | } |
| 400 | |
| 401 | /// True when \p block is nested in a region that only part of the workgroup |
| 402 | /// enters. |
| 403 | static bool isInsideConditional(Block *block) { |
| 404 | if (!block) |
| 405 | return false; |
| 406 | for (Operation *parent = block->getParentOp(); |
| 407 | parent && !isa<gpu::LaunchOp, FunctionOpInterface>(Val: parent); |
| 408 | parent = parent->getParentOp()) |
| 409 | if (isa<RegionBranchOpInterface>(Val: parent) && |
| 410 | !isa<LoopLikeOpInterface, scf::ExecuteRegionOp>(Val: parent)) |
| 411 | return true; |
| 412 | return false; |
| 413 | } |
| 414 | |
| 415 | /// Emits a workgroup-wide GPU barrier. |
| 416 | static void emitGPUBarrierWorkgroup(OpBuilder &builder, Location loc) { |
| 417 | gpu::BarrierOp::create(builder, location: loc); |
| 418 | } |
| 419 | |
| 420 | /// Emits a subgroup-scoped GPU barrier. |
| 421 | static void emitGPUBarrierSubgroup(OpBuilder &builder, Location loc) { |
| 422 | gpu::BarrierOp::create(builder, location: loc, /*address_spaces=*/ArrayAttr{}, |
| 423 | /*named_barrier=*/Value{}, |
| 424 | scope: gpu::BarrierScope::Subgroup); |
| 425 | } |
| 426 | |
| 427 | /// Emits a barrier over the lanes of this thread's ThreadY row. \p rowWidth |
| 428 | /// (blockDim.x) must divide \p subgroupSize so the rows tile a subgroup. A |
| 429 | /// subgroup-wide barrier would also tie together the other rows sharing it. |
| 430 | static void emitGPUBarrierRow(OpBuilder &builder, Location loc, |
| 431 | int64_t rowWidth, int64_t subgroupSize) { |
| 432 | assert(rowWidth > 0 && subgroupSize % rowWidth == 0 && |
| 433 | "row must tile the subgroup" ); |
| 434 | Type i32Ty = builder.getI32Type(); |
| 435 | int64_t rowsPerSubgroup = subgroupSize / rowWidth; |
| 436 | |
| 437 | // laneBase = (tid.y % rowsPerSubgroup) * rowWidth. Both operands are powers |
| 438 | // of two here, since rowWidth divides the subgroup size. |
| 439 | Value threadY = gpu::ThreadIdOp::create(builder, location: loc, resultType: builder.getIndexType(), |
| 440 | dimension: gpu::Dimension::y); |
| 441 | Value threadY32 = arith::IndexCastOp::create(builder, location: loc, out: i32Ty, in: threadY); |
| 442 | Value rowMask = |
| 443 | arith::ConstantIntOp::create(builder, location: loc, type: i32Ty, value: rowsPerSubgroup - 1); |
| 444 | Value rowIndex = arith::AndIOp::create(builder, location: loc, lhs: threadY32, rhs: rowMask); |
| 445 | Value width = arith::ConstantIntOp::create(builder, location: loc, type: i32Ty, value: rowWidth); |
| 446 | Value laneBase = arith::MulIOp::create(builder, location: loc, lhs: rowIndex, rhs: width); |
| 447 | |
| 448 | Value laneBits = arith::ConstantIntOp::create(builder, location: loc, type: i32Ty, |
| 449 | value: (int64_t{1} << rowWidth) - 1); |
| 450 | Value mask = arith::ShLIOp::create(builder, location: loc, lhs: laneBits, rhs: laneBase); |
| 451 | NVVM::SyncWarpOp::create(builder, location: loc, mask); |
| 452 | } |
| 453 | |
| 454 | /// The storage that a set of memory accesses refers to, identified by the value |
| 455 | /// that names it (see getStorageRoot). \c unknown is set when an access refers |
| 456 | /// to storage that could not be identified, which then may alias anything. |
| 457 | struct SharedStorage { |
| 458 | llvm::SmallPtrSet<Value, 8> roots; |
| 459 | bool unknown = false; |
| 460 | |
| 461 | bool empty() const { return roots.empty() && !unknown; } |
| 462 | void insert(const SharedStorage &other) { |
| 463 | roots.insert(I: other.roots.begin(), E: other.roots.end()); |
| 464 | unknown |= other.unknown; |
| 465 | } |
| 466 | }; |
| 467 | |
| 468 | /// Lowers a single `acc.compute_region` to GPU dialect IR. |
| 469 | class ACCCGToGPULowering { |
| 470 | public: |
| 471 | explicit ACCCGToGPULowering(acc::ComputeRegionOp computeRegion, |
| 472 | RewriterBase &rewriter, |
| 473 | acc::OpenACCSupport &accSupport, |
| 474 | const ACCCGToGPUOptions &options) |
| 475 | : rewriter(rewriter), computeRegion(computeRegion), |
| 476 | accSupport(accSupport), options(options), |
| 477 | sharedMemBudget( |
| 478 | options.maxWorkgroupSharedMemory, |
| 479 | sumExistingSharedMemoryBytes(region&: computeRegion.getRegion())) {} |
| 480 | |
| 481 | /// Main entry point: emit launch (if needed) and lower the region body. |
| 482 | LogicalResult rewrite(); |
| 483 | |
| 484 | gpu::LaunchOp getLaunch() const { return launch; } |
| 485 | |
| 486 | bool hasFailed = false; |
| 487 | bool insideAccumulateGridStride = false; |
| 488 | Value reductionSharedBuf; |
| 489 | // Reduction-accumulator slot (memref) -> the block-reduced value stored into |
| 490 | // it; lets a block combine use the register instead of reloading. |
| 491 | llvm::DenseMap<Value, Value> reductionAccumValue; |
| 492 | // Combine reloads recorded before accumulates are lowered, patched up after. |
| 493 | llvm::SmallVector<std::pair<Value, memref::LoadOp>> pendingCombineReloads; |
| 494 | |
| 495 | private: |
| 496 | /// Lower a parallel loop to the GPU dimension given by its `acc.par_dims`. |
| 497 | void processParallelOp(scf::ParallelOp parallelOp); |
| 498 | /// Lower a sequential loop, including any required post-loop barriers. |
| 499 | template <typename LoopOp> |
| 500 | void processSeqLoop(LoopOp loopOp); |
| 501 | /// Lower an `acc.predicate_region` to a predicated `scf.if`. |
| 502 | void processPredicateRegion(acc::PredicateRegionOp interOp); |
| 503 | /// Materialize storage for an `acc.private_local`. |
| 504 | void |
| 505 | processPrivateLocal(acc::PrivateLocalOp privateLocal, |
| 506 | std::optional<int64_t> sharedMemCopies = std::nullopt); |
| 507 | /// Lower an `acc.privatize` to device storage. |
| 508 | Value processPrivatize(acc::PrivatizeOp privatize); |
| 509 | /// Heap-allocate privatize storage and insert a matching dealloc. |
| 510 | Value allocatePrivatizeHeapStorage(acc::PrivatizeOp privatize, |
| 511 | MemRefType baseTy, |
| 512 | ValueRange mappedDynamicSizes); |
| 513 | /// Clone and lower an `scf.execute_region`. |
| 514 | void processExecuteRegion(scf::ExecuteRegionOp op); |
| 515 | /// Lower `acc.reduction_accumulate`. |
| 516 | void processAccumulateOp(acc::ReductionAccumulateOp op); |
| 517 | /// Lower `acc.reduction_accumulate_array`. |
| 518 | void processAccumulateArrayOp(acc::ReductionAccumulateArrayOp op); |
| 519 | /// Lower `acc.reduction_init`. |
| 520 | void processReductionOp(acc::ReductionInitOp op); |
| 521 | /// Lower `acc.reduction_combine`. |
| 522 | void processReductionCombineOp(acc::ReductionCombineOp op); |
| 523 | /// Lower `acc.reduction_combine_region`. |
| 524 | void processCombineRegionOp(acc::ReductionCombineRegionOp op); |
| 525 | /// Clone a leaf operation into the lowered region. |
| 526 | void processGenericOp(Operation *op); |
| 527 | /// Clone and recursively lower an operation with nested regions. |
| 528 | void processGenericOpWithRegions(Operation *op); |
| 529 | /// Dispatch lowering for one operation in the compute-region body. |
| 530 | void processOp(Operation *op); |
| 531 | |
| 532 | /// Emit an atomic reduction update to \p memref. |
| 533 | void constructAtomicAccumulation(Location loc, Value memref, |
| 534 | ValueRange indices, Value input, |
| 535 | arith::AtomicRMWKind kind); |
| 536 | |
| 537 | /// Map an ACC reduction operator to an atomic RMW kind. |
| 538 | FailureOr<arith::AtomicRMWKind> getReductionKind(acc::ReductionOperator redOp, |
| 539 | Type type, Location loc); |
| 540 | |
| 541 | /// Split launch dimensions into those that execute \p op and those that do |
| 542 | /// not, for predication and barrier placement. |
| 543 | std::pair<SmallVector<mlir::acc::GPUParallelDimAttr>, |
| 544 | SmallVector<mlir::acc::GPUParallelDimAttr>> |
| 545 | computeActiveAndInactiveParDims(Operation *op, Block *block); |
| 546 | |
| 547 | /// Build a predicate that is true only on inactive parallel dimensions. |
| 548 | Value |
| 549 | emitPredicate(Location loc, |
| 550 | SmallVector<mlir::acc::GPUParallelDimAttr> &inactiveParDims); |
| 551 | |
| 552 | /// True when \p privateLocal may be placed in shared memory; returns the |
| 553 | /// number of copies needed, or nullopt if ineligible. |
| 554 | std::optional<int64_t> |
| 555 | isEligibleForSharedMemory(acc::PrivateLocalOp privateLocal, |
| 556 | MemRefType baseTy); |
| 557 | |
| 558 | /// Reserve \p bytes from the shared-memory budget. |
| 559 | bool tryAllocateSharedMemory(int64_t bytes); |
| 560 | |
| 561 | /// Record \p privateLocal as given per-thread storage, for the closing |
| 562 | /// remark. Whether that storage lands in registers or in local memory is |
| 563 | /// decided later, so the remark reports the allocation, not the placement. |
| 564 | /// Privates with no recoverable name or compiler temps are skipped. |
| 565 | void recordThreadPrivate(acc::PrivateLocalOp privateLocal); |
| 566 | |
| 567 | /// Record \p privatizeOp as global memory storage, for the closing remark. |
| 568 | void recordGlobalMemory(acc::PrivatizeOp privatizeOp); |
| 569 | |
| 570 | /// Element size in bytes for \p elementType . |
| 571 | int64_t getElementSizeInBytes(Location loc, Type elementType) const; |
| 572 | |
| 573 | /// True when a static privatization fits in the per-thread stack budget. |
| 574 | bool canUseStackAlloca(MemRefType baseTy, Location loc, |
| 575 | int64_t maxThreadPrivateStack) const; |
| 576 | |
| 577 | /// Emit a barrier scoped to the parallel dimensions in \p parDimsAttr. |
| 578 | void createBarrier(Location loc, mlir::acc::GPUParallelDimsAttr parDimsAttr); |
| 579 | |
| 580 | /// Emit a per-row (per-worker) barrier. |
| 581 | /// Runtime branch on blockDim.y == 1 (workgroup-wide); compile-time choice |
| 582 | /// between gpu.barrier scope<subgroup> (staticBlockDimX <= subgroupSize) |
| 583 | /// and a named gpu.barrier (staticBlockDimX > subgroupSize) with tid.y+1. |
| 584 | void createPerRowBarrier(Location loc); |
| 585 | |
| 586 | /// Insert barriers after a sequential loop when shared private state must be |
| 587 | /// visible to later loops. |
| 588 | void createBarrierAfterSeqLoop(Operation *loopOp); |
| 589 | |
| 590 | /// Flush any deferred post-loop barriers that precede \p beforeOp. |
| 591 | void flushDeferredBarriersBefore(Operation *beforeOp); |
| 592 | |
| 593 | /// True when \p loopOp may write shared memory read by a later sibling loop. |
| 594 | bool mayWriteSharedMemory(Operation *loopOp); |
| 595 | |
| 596 | /// True when the thread-partitioned loop \p loop runs inside a sequential |
| 597 | /// loop nest, writes memory that more than one thread of its row or block |
| 598 | /// can see, and is followed (in the same or a later iteration of the nest) |
| 599 | /// by code that may read that memory. The sequential nest is executed by |
| 600 | /// every thread, so those readers can run before the threads that execute |
| 601 | /// the loop iterations have stored their results unless the loop is followed |
| 602 | /// by a barrier. |
| 603 | bool needsBarrierAfterThreadLoop(scf::ParallelOp loop); |
| 604 | |
| 605 | /// True when \p root may read (\p reads) or write (\p writes) memory that is |
| 606 | /// not private to a single thread. Storage allocated inside \p localScope is |
| 607 | /// private to it. With \p workerLevel, worker-private storage also counts as |
| 608 | /// private, because the lanes of a worker-partitioned loop do not share it |
| 609 | /// with other workers. Operations inside \p ignore are skipped. When |
| 610 | /// \p collect is given, the walk does not stop at the first access and |
| 611 | /// records the storage of every shared access in it. When \p filter is given, |
| 612 | /// accesses to storage known to differ from the storage in \p filter are |
| 613 | /// ignored. |
| 614 | bool mayAccessSharedMemory(Operation *root, bool reads, bool writes, |
| 615 | bool workerLevel, Operation *localScope, |
| 616 | Operation *ignore = nullptr, |
| 617 | SharedStorage *collect = nullptr, |
| 618 | const SharedStorage *filter = nullptr); |
| 619 | |
| 620 | /// Parallelism scope (thread, worker, or gang) of a privatized variable. |
| 621 | PrivateMemScope getPrivateMemScope(acc::PrivatizeOp privatizeOp); |
| 622 | |
| 623 | /// Parallelism scope of the private buffer backing \p memref. |
| 624 | PrivateMemScope getPrivateScopeForMemref(Value memref); |
| 625 | |
| 626 | /// `acc.privatize` that materialized the private buffer for \p memref. |
| 627 | acc::PrivatizeOp getPrivatizeForMemref(Value memref); |
| 628 | |
| 629 | /// Collect privatize ops of the shared private buffers stored to under \p |
| 630 | /// root, keeping the gang- and worker-private ones apart: the two scopes need |
| 631 | /// different barriers, so a store at one scope must not stand in for the |
| 632 | /// other. |
| 633 | void collectSharedPrivateStores( |
| 634 | Operation *root, llvm::SmallPtrSetImpl<Operation *> &gangPrivatizes, |
| 635 | llvm::SmallPtrSetImpl<Operation *> &workerPrivatizes); |
| 636 | |
| 637 | /// True when \p memref is backed by one of \p storePrivatizes at \p |
| 638 | /// storeScope. |
| 639 | bool isMatchingPrivateMemref( |
| 640 | Value memref, PrivateMemScope storeScope, |
| 641 | const llvm::SmallPtrSetImpl<Operation *> &storePrivatizes); |
| 642 | |
| 643 | /// True when a parallel region under \p seqLoopOp, outside \p skipOp, uses a |
| 644 | /// private buffer at \p storeScope backed by one of \p storePrivatizes. |
| 645 | bool hasParallelPrivateReuse( |
| 646 | LoopLikeOpInterface seqLoopOp, Operation *skipOp, |
| 647 | PrivateMemScope storeScope, |
| 648 | const llvm::SmallPtrSetImpl<Operation *> &storePrivatizes); |
| 649 | |
| 650 | /// Whether a predicate region needs a barrier before stores that will be read |
| 651 | /// by a later parallel loop over the same private memory. |
| 652 | PrivateMemScope needsPreStoreReuseBarrier(acc::PredicateRegionOp interOp); |
| 653 | |
| 654 | /// Whether \p loopOp needs a barrier at the top of its body because a |
| 655 | /// gang-private slot it writes is reused by the next iteration. |
| 656 | bool needsInLoopReuseBarrier(Operation *loopOp); |
| 657 | |
| 658 | /// Emit `gpu.all_reduce` for a reduction partial. |
| 659 | void createGPUAllReduceOp(Location loc, Value input, Value memref, |
| 660 | arith::AtomicRMWKind kind, |
| 661 | mlir::acc::GPUParallelDimsAttr parDimsAttr, |
| 662 | ValueRange indices = {}, |
| 663 | bool isPerThreadPrivateTarget = false); |
| 664 | |
| 665 | /// Finish lowering a deferred `acc.reduction_accumulate`. |
| 666 | void postprocessAccumulateOp(acc::ReductionAccumulateOp op); |
| 667 | |
| 668 | /// Finish lowering reductions attached to a parallel loop. |
| 669 | void postprocessLoopReduction(scf::ParallelOp parLoop); |
| 670 | |
| 671 | /// Populate block/thread id and grid/block dimension maps for device |
| 672 | /// routines. |
| 673 | static void |
| 674 | createForAllDimensions(RewriterBase &rewriter, Location loc, |
| 675 | llvm::DenseMap<gpu::Processor, Value> &ids, |
| 676 | llvm::DenseMap<gpu::Processor, Value> &dims) { |
| 677 | ids[gpu::Processor::BlockX] = gpu::BlockIdOp::create( |
| 678 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::x); |
| 679 | ids[gpu::Processor::BlockY] = gpu::BlockIdOp::create( |
| 680 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::y); |
| 681 | ids[gpu::Processor::BlockZ] = gpu::BlockIdOp::create( |
| 682 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::z); |
| 683 | ids[gpu::Processor::ThreadX] = gpu::ThreadIdOp::create( |
| 684 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::x); |
| 685 | ids[gpu::Processor::ThreadY] = gpu::ThreadIdOp::create( |
| 686 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::y); |
| 687 | ids[gpu::Processor::ThreadZ] = gpu::ThreadIdOp::create( |
| 688 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::z); |
| 689 | dims[gpu::Processor::BlockX] = gpu::GridDimOp::create( |
| 690 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::x); |
| 691 | dims[gpu::Processor::BlockY] = gpu::GridDimOp::create( |
| 692 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::y); |
| 693 | dims[gpu::Processor::BlockZ] = gpu::GridDimOp::create( |
| 694 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::z); |
| 695 | dims[gpu::Processor::ThreadX] = gpu::BlockDimOp::create( |
| 696 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::x); |
| 697 | dims[gpu::Processor::ThreadY] = gpu::BlockDimOp::create( |
| 698 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::y); |
| 699 | dims[gpu::Processor::ThreadZ] = gpu::BlockDimOp::create( |
| 700 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::z); |
| 701 | } |
| 702 | |
| 703 | /// Return the compute-region block argument for \p outside, adding an `ins` |
| 704 | /// operand when needed. |
| 705 | BlockArgument getOrAppendInsBlockArg(Value outside) { |
| 706 | if (std::optional<BlockArgument> blockArg = |
| 707 | computeRegion.getBlockArg(value: outside)) { |
| 708 | return *blockArg; |
| 709 | } |
| 710 | return computeRegion.appendInputArg(outside); |
| 711 | } |
| 712 | |
| 713 | /// Wire dynamic privatization extents into the compute region as `ins` args. |
| 714 | void preparePrivatizeExtentInsOperands() { |
| 715 | computeRegion.walk(callback: [&](acc::PrivateLocalOp privateLocal) { |
| 716 | acc::PrivatizeOp privatizeOp = |
| 717 | getPrivatizeOp(privateLocal, computeRegion); |
| 718 | if (privatizeOp->getParentOfType<acc::ComputeRegionOp>() == computeRegion) |
| 719 | return; |
| 720 | for (Value extent : privatizeOp.getDynamicSizes()) |
| 721 | getOrAppendInsBlockArg(outside: extent); |
| 722 | }); |
| 723 | } |
| 724 | |
| 725 | /// Resolve dynamic size operands for a privatized array. |
| 726 | SmallVector<Value> |
| 727 | resolvePrivateLocalDynamicExtents(acc::PrivateLocalOp privateLocal) { |
| 728 | acc::PrivatizeOp privatizeOp = getPrivatizeOp(privateLocal, computeRegion); |
| 729 | SmallVector<Value> extents; |
| 730 | for (Value extent : privatizeOp.getDynamicSizes()) { |
| 731 | if (std::optional<BlockArgument> blockArg = |
| 732 | computeRegion.getBlockArg(value: extent)) { |
| 733 | extents.push_back(Elt: mapping.lookupOrDefault(from: *blockArg)); |
| 734 | continue; |
| 735 | } |
| 736 | extents.push_back(Elt: mapping.lookupOrDefault(from: extent)); |
| 737 | } |
| 738 | return extents; |
| 739 | } |
| 740 | |
| 741 | RewriterBase &rewriter; |
| 742 | acc::ComputeRegionOp computeRegion; |
| 743 | |
| 744 | acc::OpenACCSupport &accSupport; |
| 745 | const ACCCGToGPUOptions &options; |
| 746 | gpu::LaunchOp launch; |
| 747 | IRMapping mapping; |
| 748 | llvm::SmallVector<scf::ParallelOp> loopReductions; |
| 749 | llvm::DenseMap<gpu::Processor, Value> threadIdMap; |
| 750 | llvm::DenseMap<gpu::Processor, Value> dimensionMap; |
| 751 | // True if ThreadY reduction exists, which triggers subgroup alignment |
| 752 | bool hasThreadYReduction = false; |
| 753 | // True if a ThreadX reduction exists, whose shuffles stay inside a row |
| 754 | bool hasThreadXReduction = false; |
| 755 | // True if an array reduction spans any thread dimension |
| 756 | bool hasThreadLevelArrayReduction = false; |
| 757 | // True if any ThreadX routine call exists in the kernel |
| 758 | bool hasThreadLevelRoutineCall = false; |
| 759 | // True when a per-row ThreadY barrier is emitted |
| 760 | bool hasThreadYBarrier = false; |
| 761 | |
| 762 | // Reusable privatize broadcast slots per type; disabled for kernels. |
| 763 | llvm::DenseMap<Type, Value> privatizeBroadcastCache; |
| 764 | |
| 765 | int64_t staticBlockDimX = 1024; |
| 766 | int64_t staticBlockDimY = 1024; |
| 767 | int64_t staticBlockDimZ = 1024; |
| 768 | |
| 769 | /// True when the launch is (1, N, 1) with 1 < N <= subgroupSize, so every |
| 770 | /// worker owns one thread. N is capped by the worker-indexed shared buffers, |
| 771 | /// which hold subgroupSize entries. |
| 772 | bool isSingleThreadWorkerLaunch() const { |
| 773 | return staticBlockDimX == 1 && staticBlockDimZ == 1 && |
| 774 | staticBlockDimY > 1 && staticBlockDimY <= options.subgroupSize; |
| 775 | } |
| 776 | |
| 777 | /// True when only worker reductions would need the rows aligned. They combine |
| 778 | /// their partials in the lowest N threads of the block, not within a row, so |
| 779 | /// a row may sit anywhere in a subgroup. Array accumulates use atomics |
| 780 | /// instead and still need alignment. |
| 781 | bool isWorkerOnlyShuffleLaunch() const { |
| 782 | return hasThreadYReduction && !hasThreadXReduction && |
| 783 | !hasThreadLevelArrayReduction && staticBlockDimZ == 1 && |
| 784 | staticBlockDimY > 1 && staticBlockDimY <= options.subgroupSize && |
| 785 | options.subgroupSize % staticBlockDimX == 0; |
| 786 | } |
| 787 | |
| 788 | acc::DefaultACCToGPUMappingPolicy defaultPolicy; |
| 789 | SharedMemoryBudget sharedMemBudget; |
| 790 | SmallVector<std::string> sharedMemPrivateVarNames; |
| 791 | SmallVector<std::string> threadPrivateVarNames; |
| 792 | SmallVector<std::string> globalMemVarNames; |
| 793 | llvm::SmallVector<Operation *, 4> deferredBarrierSeqLoops; |
| 794 | |
| 795 | Value getThreadId(Location loc, gpu::Dimension dim) { |
| 796 | return gpu::ThreadIdOp::create(builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: dim); |
| 797 | } |
| 798 | |
| 799 | Value getBlockDim(Location loc, gpu::Dimension dim) { |
| 800 | return gpu::BlockDimOp::create(builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: dim); |
| 801 | } |
| 802 | |
| 803 | /// Thread id for \p proc, from the launch op or the routine context map. |
| 804 | Value getGPUThreadIdFor(gpu::Processor proc) { |
| 805 | return getGPUThreadId(processor: proc, launch: getLaunch(), indexOps: threadIdMap); |
| 806 | } |
| 807 | |
| 808 | /// Grid/block dimension for \p proc, from the launch op or routine map. |
| 809 | Value getGPUSizeFor(gpu::Processor proc) { |
| 810 | return getGPUSize(processor: proc, launch: getLaunch(), dimensionOps: dimensionMap); |
| 811 | } |
| 812 | }; |
| 813 | |
| 814 | int64_t ACCCGToGPULowering::getElementSizeInBytes(Location loc, |
| 815 | Type elementType) const { |
| 816 | ModuleOp module = computeRegion->getParentOfType<ModuleOp>(); |
| 817 | if (std::optional<acc::TypeSizeAndAlignment> sizeAndAlignment = |
| 818 | accSupport.getTypeSizeAndAlignment(ty: elementType, module)) { |
| 819 | return sizeAndAlignment->first.getFixedValue(); |
| 820 | } |
| 821 | std::string msg; |
| 822 | llvm::raw_string_ostream os(msg); |
| 823 | os << "element size computation for unsupported type: " << elementType; |
| 824 | (void)accSupport.emitNYI(loc, message: os.str()); |
| 825 | return 0; |
| 826 | } |
| 827 | |
| 828 | bool ACCCGToGPULowering::canUseStackAlloca( |
| 829 | MemRefType baseTy, Location loc, int64_t maxThreadPrivateStack) const { |
| 830 | for (int64_t dim : baseTy.getShape()) |
| 831 | if (dim == ShapedType::kDynamic) |
| 832 | return false; |
| 833 | int64_t elementSize = getElementSizeInBytes(loc, elementType: baseTy.getElementType()); |
| 834 | int64_t numElements = 1; |
| 835 | for (int64_t dim : baseTy.getShape()) { |
| 836 | if (numElements > maxThreadPrivateStack / std::max<int64_t>(a: dim, b: 1)) |
| 837 | return false; |
| 838 | numElements *= dim; |
| 839 | } |
| 840 | return elementSize * numElements < maxThreadPrivateStack; |
| 841 | } |
| 842 | |
| 843 | /// True if the accumulate spans a block dim or is nested in a block-mapped |
| 844 | /// loop, i.e. each block owns the elements it reduces across threads. A |
| 845 | /// thread-only accumulate with no block context grid-strides its element loop |
| 846 | /// onto blocks, so per-thread partials would be dropped; such reductions must |
| 847 | /// stay shared. |
| 848 | static bool reductionHasBlockContext(acc::ReductionAccumulateArrayOp accArr) { |
| 849 | auto hasBlock = [](mlir::acc::GPUParallelDimsAttr parDims) { |
| 850 | return parDims && llvm::any_of(Range: parDims.getArray(), |
| 851 | P: [](auto pd) { return pd.isAnyBlock(); }); |
| 852 | }; |
| 853 | if (hasBlock(accArr.getParDimsAttr())) |
| 854 | return true; |
| 855 | for (scf::ParallelOp loop = accArr->getParentOfType<scf::ParallelOp>(); loop; |
| 856 | loop = loop->getParentOfType<scf::ParallelOp>()) { |
| 857 | if (hasBlock(mlir::acc::getParDimsAttr(op: loop))) |
| 858 | return true; |
| 859 | } |
| 860 | return false; |
| 861 | } |
| 862 | |
| 863 | /// Returns the array reduction accumulate (through cast/view ops) that \p v |
| 864 | /// feeds if it needs per-thread storage: its par_dims include a thread dim |
| 865 | /// and it has block context so the cross-thread all_reduce is well defined. |
| 866 | static acc::ReductionAccumulateArrayOp perThreadArrayReductionAccum(Value v) { |
| 867 | SmallVector<Value> worklist{v}; |
| 868 | DenseSet<Value> seen; |
| 869 | while (!worklist.empty()) { |
| 870 | Value cur = worklist.pop_back_val(); |
| 871 | if (!seen.insert(V: cur).second) |
| 872 | continue; |
| 873 | for (Operation *user : cur.getUsers()) { |
| 874 | if (acc::ReductionAccumulateArrayOp accArr = |
| 875 | dyn_cast<acc::ReductionAccumulateArrayOp>(Val: user)) { |
| 876 | bool hasThread = false; |
| 877 | for (auto pd : accArr.getParDims().getArray()) |
| 878 | hasThread |= pd.isAnyThread(); |
| 879 | if (hasThread && reductionHasBlockContext(accArr)) |
| 880 | return accArr; |
| 881 | continue; |
| 882 | } |
| 883 | SmallVector<Value> through; |
| 884 | if (getPassThroughResults(userOp: user, trackedOperand: cur, passThroughResults&: through)) |
| 885 | worklist.append(in_start: through.begin(), in_end: through.end()); |
| 886 | else if (isa<ViewLikeOpInterface>(Val: user)) |
| 887 | worklist.append(in_start: user->result_begin(), in_end: user->result_end()); |
| 888 | } |
| 889 | } |
| 890 | return nullptr; |
| 891 | } |
| 892 | |
| 893 | /// Store the reduction identity to every element of a freshly allocated |
| 894 | /// per-thread array accumulator so all lanes start from identity (the original |
| 895 | /// init loop may only run on one lane). |
| 896 | static void initPerThreadArrayAccum(OpBuilder &b, Location loc, Value alloca, |
| 897 | MemRefType baseTy, |
| 898 | arith::AtomicRMWKind kind) { |
| 899 | assert(baseTy.getRank() > 0 && baseTy.hasStaticShape() && |
| 900 | "per-thread array reduction accumulator must be static ranked" ); |
| 901 | Value ident = createIdentityValue(b, loc, type: baseTy.getElementType(), kind, |
| 902 | /*useOnlyFiniteValue=*/true); |
| 903 | Value lb = arith::ConstantIndexOp::create(builder&: b, location: loc, value: 0); |
| 904 | Value step = arith::ConstantIndexOp::create(builder&: b, location: loc, value: 1); |
| 905 | SmallVector<Value> indices; |
| 906 | auto buildLoopNest = [&](auto &&self, unsigned dim) -> void { |
| 907 | if (dim == baseTy.getRank()) { |
| 908 | memref::StoreOp::create(builder&: b, location: loc, valueToStore: ident, memref: alloca, indices); |
| 909 | return; |
| 910 | } |
| 911 | |
| 912 | Value ub = arith::ConstantIndexOp::create(builder&: b, location: loc, value: baseTy.getShape()[dim]); |
| 913 | auto forOp = scf::ForOp::create(builder&: b, location: loc, lowerBound: lb, upperBound: ub, step); |
| 914 | OpBuilder::InsertionGuard g(b); |
| 915 | b.setInsertionPoint(forOp.getBody()->getTerminator()); |
| 916 | indices.push_back(Elt: forOp.getInductionVar()); |
| 917 | self(self, dim + 1); |
| 918 | indices.pop_back(); |
| 919 | }; |
| 920 | buildLoopNest(buildLoopNest, 0); |
| 921 | } |
| 922 | |
| 923 | std::optional<int64_t> |
| 924 | ACCCGToGPULowering::isEligibleForSharedMemory(acc::PrivateLocalOp privateLocal, |
| 925 | MemRefType baseTy) { |
| 926 | // Cross-thread array reduction accumulators must stay per-thread when their |
| 927 | // storage scope includes thread_x. Gang-/worker-scoped array temps remain |
| 928 | // eligible for shared memory. |
| 929 | if (perThreadArrayReductionAccum(v: privateLocal.getResult()) && |
| 930 | storageHasThreadX(privateLocal, computeRegion)) |
| 931 | return std::nullopt; |
| 932 | ModuleOp module = computeRegion->getParentOfType<ModuleOp>(); |
| 933 | FailureOr<bool> isCandidate = isPrivateLocalSharedMemoryCandidate( |
| 934 | privateLocal, computeRegion, module, policy: defaultPolicy, support: &accSupport); |
| 935 | if (failed(Result: isCandidate)) { |
| 936 | hasFailed = true; |
| 937 | return std::nullopt; |
| 938 | } |
| 939 | if (!isCandidate.value()) |
| 940 | return std::nullopt; |
| 941 | std::optional<int64_t> upperBound = |
| 942 | getPrivateLocalSharedMemoryUpperBoundBytes(privateLocal, computeRegion, |
| 943 | module, policy: defaultPolicy); |
| 944 | assert(upperBound && "candidate private_local must have an upper bound" ); |
| 945 | int64_t elementSize = |
| 946 | getElementSizeInBytes(loc: privateLocal.getLoc(), elementType: baseTy.getElementType()); |
| 947 | int64_t numElements = 1; |
| 948 | for (int64_t dim : baseTy.getShape()) |
| 949 | numElements *= dim; |
| 950 | return *upperBound / (elementSize * numElements); |
| 951 | } |
| 952 | |
| 953 | bool ACCCGToGPULowering::tryAllocateSharedMemory(int64_t bytes) { |
| 954 | return sharedMemBudget.tryAllocate(bytes); |
| 955 | } |
| 956 | |
| 957 | void ACCCGToGPULowering::recordThreadPrivate(acc::PrivateLocalOp privateLocal) { |
| 958 | std::string varName = accSupport.getVariableName(v: privateLocal.getResult()); |
| 959 | // A name that could not be recovered, or a compiler temp are skipped. |
| 960 | if (varName.empty()) |
| 961 | return; |
| 962 | // The same variable reaches this point once per acc.private_local it has. |
| 963 | if (!llvm::is_contained(Range&: threadPrivateVarNames, Element: varName)) |
| 964 | threadPrivateVarNames.push_back(Elt: varName); |
| 965 | } |
| 966 | |
| 967 | void ACCCGToGPULowering::recordGlobalMemory(acc::PrivatizeOp privatizeOp) { |
| 968 | std::string varName = accSupport.getVariableName(v: privatizeOp.getResult()); |
| 969 | // A name that could not be recovered, or a compiler temp are skipped. |
| 970 | if (varName.empty()) |
| 971 | return; |
| 972 | // The same variable reaches this point once per acc.privatize it has. |
| 973 | if (!llvm::is_contained(Range&: globalMemVarNames, Element: varName)) |
| 974 | globalMemVarNames.push_back(Elt: varName); |
| 975 | } |
| 976 | |
| 977 | FailureOr<arith::AtomicRMWKind> |
| 978 | ACCCGToGPULowering::getReductionKind(acc::ReductionOperator redOp, Type type, |
| 979 | Location loc) { |
| 980 | if (std::optional<arith::AtomicRMWKind> kind = |
| 981 | translateACCReductionOperator(redOp, type)) |
| 982 | return *kind; |
| 983 | |
| 984 | std::string msg; |
| 985 | llvm::raw_string_ostream os(msg); |
| 986 | os << "reduction operator (" << redOp << ") for type " << type; |
| 987 | (void)accSupport.emitNYI(loc, message: os.str()); |
| 988 | return failure(); |
| 989 | } |
| 990 | |
| 991 | LogicalResult ACCCGToGPULowering::rewrite() { |
| 992 | |
| 993 | // Pre-compute if thread-level reductions exist. ThreadY reduction generates |
| 994 | // shuffles which require subgroup alignment (blockDim.x = subgroupSize), |
| 995 | // meaning ThreadX lanes exist even without explicit ThreadX parallelism. |
| 996 | // ThreadX is tracked too, to tell worker-only reductions from row shuffles. |
| 997 | computeRegion->walk(callback: [&](acc::ReductionAccumulateOp op) { |
| 998 | for (auto parDim : op.getParDimsAttr().getArray()) { |
| 999 | hasThreadXReduction |= parDim.isThreadX(); |
| 1000 | hasThreadYReduction |= parDim.isThreadY(); |
| 1001 | } |
| 1002 | }); |
| 1003 | computeRegion->walk(callback: [&](acc::ReductionAccumulateArrayOp op) { |
| 1004 | for (auto parDim : op.getParDimsAttr().getArray()) { |
| 1005 | if (!parDim.isAnyBlock()) |
| 1006 | hasThreadLevelArrayReduction = true; |
| 1007 | } |
| 1008 | }); |
| 1009 | |
| 1010 | // Pre-compute if any thread-level (vector or worker) routine call exists. |
| 1011 | // Such routines partition work across ThreadX/ThreadY and emit workgroup-wide |
| 1012 | // barriers internally (e.g. for shared memory alloca synchronization), so all |
| 1013 | // workgroup threads must reach the call site for those barriers to converge. |
| 1014 | computeRegion->walk(callback: [&](CallOpInterface callOp) -> WalkResult { |
| 1015 | if (mlir::acc::GPUParallelDimAttr parDim = |
| 1016 | getAccRoutineCallParDim(callOp, policy: defaultPolicy)) { |
| 1017 | if (parDim.isThreadX() || parDim.isThreadY()) { |
| 1018 | hasThreadLevelRoutineCall = true; |
| 1019 | return WalkResult::interrupt(); |
| 1020 | } |
| 1021 | } |
| 1022 | return WalkResult::advance(); |
| 1023 | }); |
| 1024 | |
| 1025 | Location loc = computeRegion->getLoc(); |
| 1026 | Value constantOne = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1); |
| 1027 | |
| 1028 | auto launchArgument = [&](gpu::Processor processor) -> Value { |
| 1029 | mlir::acc::GPUParallelDimAttr parDim = mlir::acc::GPUParallelDimAttr::get( |
| 1030 | computeRegion->getContext(), processor); |
| 1031 | std::optional<Value> maybeLaunchArg = |
| 1032 | computeRegion.getKnownLaunchArg(parDim); |
| 1033 | LLVM_DEBUG(llvm::dbgs() << "ACCCGToGPU: launch-arg: " |
| 1034 | << " parDim: " << parDim << " gpu: " << processor |
| 1035 | << " widthValue: " |
| 1036 | << maybeLaunchArg.value_or(constantOne) << "\n" ); |
| 1037 | |
| 1038 | return getValueOrCreateCastToIndexLike( |
| 1039 | b&: rewriter, loc, targetType: rewriter.getIndexType(), |
| 1040 | value: maybeLaunchArg.value_or(u&: constantOne)); |
| 1041 | }; |
| 1042 | LLVM_DEBUG(llvm::dbgs() << "ACCCGToGPU: creating gpu launch op: \n" ); |
| 1043 | |
| 1044 | // acc.compute_region keeps launch argument as block argument, for rewriting |
| 1045 | // we now replace these with gpu.launch dimensions. |
| 1046 | auto mapLaunchArguments = [&](gpu::Processor processor, Value launchArg) { |
| 1047 | mlir::acc::GPUParallelDimAttr parDim = mlir::acc::GPUParallelDimAttr::get( |
| 1048 | computeRegion->getContext(), processor); |
| 1049 | std::optional<Value> kernelArg = computeRegion.getLaunchArg(parDim); |
| 1050 | if (kernelArg) |
| 1051 | mapping.map(from: computeRegion.gpuParWidth(processor), to: launchArg); |
| 1052 | }; |
| 1053 | |
| 1054 | llvm::StringRef blockDimXName = "blockDim.x" ; |
| 1055 | llvm::StringRef blockDimYName = "blockDim.y" ; |
| 1056 | std::string deviceLabel = getDeviceRemarkQualifier(deviceType: options.deviceType); |
| 1057 | |
| 1058 | if (!computeRegion->getParentOfType<gpu::GPUFuncOp>()) { |
| 1059 | Value blockDimX = launchArgument(gpu::Processor::ThreadX); |
| 1060 | APInt bdxVal; |
| 1061 | if (matchPattern(value: blockDimX, pattern: m_ConstantInt(bind_value: &bdxVal))) |
| 1062 | staticBlockDimX = bdxVal.getSExtValue(); |
| 1063 | Value blockDimY = launchArgument(gpu::Processor::ThreadY); |
| 1064 | APInt bdyVal; |
| 1065 | if (matchPattern(value: blockDimY, pattern: m_ConstantInt(bind_value: &bdyVal))) |
| 1066 | staticBlockDimY = bdyVal.getSExtValue(); |
| 1067 | Value blockDimZ = launchArgument(gpu::Processor::ThreadZ); |
| 1068 | APInt bdzVal; |
| 1069 | if (matchPattern(value: blockDimZ, pattern: m_ConstantInt(bind_value: &bdzVal))) |
| 1070 | staticBlockDimZ = bdzVal.getSExtValue(); |
| 1071 | Value gridDimX = launchArgument(gpu::Processor::BlockX); |
| 1072 | Value gridDimY = launchArgument(gpu::Processor::BlockY); |
| 1073 | Value gridDimZ = launchArgument(gpu::Processor::BlockZ); |
| 1074 | |
| 1075 | // The format of the message is: |
| 1076 | // Generating [serial] {deviceLabel} code with gridDim=32x1x1 |
| 1077 | // blockDim=256x1x1 |
| 1078 | accSupport.emitRemark(op: computeRegion, messageFn: [&]() { |
| 1079 | auto getName = [&](Value val) -> std::string { |
| 1080 | // A negative width leaves the size to the launch. |
| 1081 | if (std::optional<int64_t> width = getConstantIntValue(ofr: val); |
| 1082 | width && *width < 0) |
| 1083 | return "(*)" ; |
| 1084 | std::string name = accSupport.getVariableName(v: val); |
| 1085 | return name.empty() ? "(*)" : name; |
| 1086 | }; |
| 1087 | bool isEffectivelySerial = |
| 1088 | sameEffectiveValue(x: blockDimX, y: 1) && |
| 1089 | sameEffectiveValue(x: blockDimY, y: 1) && |
| 1090 | sameEffectiveValue(x: blockDimZ, y: 1) && sameEffectiveValue(x: gridDimX, y: 1) && |
| 1091 | sameEffectiveValue(x: gridDimY, y: 1) && sameEffectiveValue(x: gridDimZ, y: 1); |
| 1092 | return (llvm::Twine("Generating " ) + |
| 1093 | llvm::Twine(isEffectivelySerial ? "serial " : "" ) + deviceLabel + |
| 1094 | " code with gridDim=" + getName(gridDimX) + "x" + |
| 1095 | getName(gridDimY) + "x" + getName(gridDimZ) + |
| 1096 | " blockDim=" + getName(blockDimX) + "x" + getName(blockDimY) + |
| 1097 | "x" + getName(blockDimZ)) |
| 1098 | .str(); |
| 1099 | }); |
| 1100 | |
| 1101 | // Check if kernel has a stream operand for async execution |
| 1102 | if (mlir::Value streamValue = computeRegion.getStream()) { |
| 1103 | LLVM_DEBUG(llvm::dbgs() |
| 1104 | << "\nDEBUG: Creating async gpu.launch with stream: " |
| 1105 | << streamValue << "\n" ); |
| 1106 | launch = gpu::LaunchOp::create( |
| 1107 | builder&: rewriter, location: loc, gridSizeX: gridDimX, gridSizeY: gridDimY, gridSizeZ: gridDimZ, blockSizeX: blockDimX, blockSizeY: blockDimY, |
| 1108 | blockSizeZ: blockDimZ, |
| 1109 | /*dynamicSharedMemorySize=*/mlir::Value{}, |
| 1110 | /*asyncTokenType=*/ |
| 1111 | mlir::gpu::AsyncTokenType::get(ctx: rewriter.getContext())); |
| 1112 | // Add the stream as an async dependency |
| 1113 | launch.getAsyncDependenciesMutable().append(values: streamValue); |
| 1114 | } else { |
| 1115 | LLVM_DEBUG(llvm::dbgs() |
| 1116 | << "\nDEBUG: No stream, creating sync gpu.launch\n" ); |
| 1117 | launch = gpu::LaunchOp::create(builder&: rewriter, location: loc, gridSizeX: gridDimX, gridSizeY: gridDimY, |
| 1118 | gridSizeZ: gridDimZ, blockSizeX: blockDimX, blockSizeY: blockDimY, blockSizeZ: blockDimZ); |
| 1119 | } |
| 1120 | |
| 1121 | // Transfer kernel function name and module name from acc.compute_region to |
| 1122 | // gpu.launch if present |
| 1123 | if (auto kernelFuncName = computeRegion.getKernelFuncNameAttr()) |
| 1124 | launch.setFunctionAttr(kernelFuncName); |
| 1125 | if (auto kernelModuleName = computeRegion.getKernelModuleNameAttr()) |
| 1126 | launch.setModuleAttr(kernelModuleName); |
| 1127 | |
| 1128 | rewriter.setInsertionPointToEnd(&launch.getBody().front()); |
| 1129 | gpu::TerminatorOp::create(builder&: rewriter, location: loc); |
| 1130 | rewriter.setInsertionPointToStart(&launch.getBody().front()); |
| 1131 | mapLaunchArguments(gpu::Processor::BlockX, |
| 1132 | gpu::GridDimOp::create(builder&: rewriter, location: loc, |
| 1133 | resultType: rewriter.getIndexType(), |
| 1134 | dimension: gpu::Dimension::x)); |
| 1135 | mapLaunchArguments(gpu::Processor::BlockY, |
| 1136 | gpu::GridDimOp::create(builder&: rewriter, location: loc, |
| 1137 | resultType: rewriter.getIndexType(), |
| 1138 | dimension: gpu::Dimension::y)); |
| 1139 | mapLaunchArguments(gpu::Processor::BlockZ, |
| 1140 | gpu::GridDimOp::create(builder&: rewriter, location: loc, |
| 1141 | resultType: rewriter.getIndexType(), |
| 1142 | dimension: gpu::Dimension::z)); |
| 1143 | mapLaunchArguments(gpu::Processor::ThreadX, |
| 1144 | gpu::BlockDimOp::create(builder&: rewriter, location: loc, |
| 1145 | resultType: rewriter.getIndexType(), |
| 1146 | dimension: gpu::Dimension::x)); |
| 1147 | mapLaunchArguments(gpu::Processor::ThreadY, |
| 1148 | gpu::BlockDimOp::create(builder&: rewriter, location: loc, |
| 1149 | resultType: rewriter.getIndexType(), |
| 1150 | dimension: gpu::Dimension::y)); |
| 1151 | mapLaunchArguments(gpu::Processor::ThreadZ, |
| 1152 | gpu::BlockDimOp::create(builder&: rewriter, location: loc, |
| 1153 | resultType: rewriter.getIndexType(), |
| 1154 | dimension: gpu::Dimension::z)); |
| 1155 | } else { |
| 1156 | // Do not create gpu.launch for acc routine and map |
| 1157 | // to block/thread index and block/grid size instead |
| 1158 | // of launch arguments, using created maps. |
| 1159 | OpBuilder::InsertionGuard guard(rewriter); |
| 1160 | rewriter.setInsertionPointToStart(computeRegion->getBlock()); |
| 1161 | createForAllDimensions(rewriter, loc, ids&: threadIdMap, dims&: dimensionMap); |
| 1162 | mapLaunchArguments(gpu::Processor::BlockX, |
| 1163 | dimensionMap[gpu::Processor::BlockX]); |
| 1164 | mapLaunchArguments(gpu::Processor::BlockY, |
| 1165 | dimensionMap[gpu::Processor::BlockY]); |
| 1166 | mapLaunchArguments(gpu::Processor::BlockZ, |
| 1167 | dimensionMap[gpu::Processor::BlockZ]); |
| 1168 | mapLaunchArguments(gpu::Processor::ThreadX, |
| 1169 | dimensionMap[gpu::Processor::ThreadX]); |
| 1170 | mapLaunchArguments(gpu::Processor::ThreadY, |
| 1171 | dimensionMap[gpu::Processor::ThreadY]); |
| 1172 | mapLaunchArguments(gpu::Processor::ThreadZ, |
| 1173 | dimensionMap[gpu::Processor::ThreadZ]); |
| 1174 | } |
| 1175 | |
| 1176 | // Map input arguments for compute region; we go from an IsolatedFromAbove |
| 1177 | // operation to gpu.launch which is not IsolatedFromAbove. |
| 1178 | preparePrivatizeExtentInsOperands(); |
| 1179 | Block *body = computeRegion.getBody(); |
| 1180 | unsigned numLaunchArgs = computeRegion.getLaunchArgs().size(); |
| 1181 | ValueRange inputArgs = computeRegion.getInputArgs(); |
| 1182 | for (unsigned i = numLaunchArgs; i < body->getNumArguments(); ++i) |
| 1183 | mapping.map(from: body->getArgument(i), to: inputArgs[i - numLaunchArgs]); |
| 1184 | |
| 1185 | assert(computeRegion.getRegion().hasOneBlock() && |
| 1186 | "compute region only supports one block region for now" ); |
| 1187 | // process all operations inside kernel region |
| 1188 | for (auto &op : computeRegion.getRegion().getBlocks().front().getOperations()) |
| 1189 | processOp(op: &op); |
| 1190 | |
| 1191 | for (auto &parLoop : loopReductions) |
| 1192 | postprocessLoopReduction(parLoop); |
| 1193 | |
| 1194 | // Replace combine reloads of a reduction slot with the block-reduced value. |
| 1195 | // Only when it dominates the reload; otherwise keep the reload. |
| 1196 | if (!pendingCombineReloads.empty() && launch) { |
| 1197 | DominanceInfo domInfo(launch); |
| 1198 | for (auto &[slot, loadOp] : pendingCombineReloads) { |
| 1199 | llvm::DenseMap<Value, Value>::iterator it = |
| 1200 | reductionAccumValue.find(Val: slot); |
| 1201 | if (it == reductionAccumValue.end()) |
| 1202 | continue; |
| 1203 | if (!domInfo.dominates(a: it->second, b: loadOp.getOperation())) |
| 1204 | continue; |
| 1205 | rewriter.replaceOp(op: loadOp, newValues: ValueRange{it->second}); |
| 1206 | } |
| 1207 | } |
| 1208 | |
| 1209 | if (launch) { |
| 1210 | const int64_t subgroupSize = options.subgroupSize; |
| 1211 | const int64_t subgroupAlignMask = subgroupSize - 1; |
| 1212 | |
| 1213 | // Adjust blockDim.x to be a multiple of subgroupSize. This is required |
| 1214 | // because: |
| 1215 | // - Subgroup reductions (gpu.all_reduce) require full subgroups |
| 1216 | // - Per-row workgroup barriers require blockDim.x aligned to subgroupSize |
| 1217 | // Tracked apart because only a ThreadX reduction shuffles within the row, |
| 1218 | // so only it needs the row to start at a subgroup boundary. |
| 1219 | bool needsThreadXAlign = false; |
| 1220 | bool needsThreadYAlign = false; |
| 1221 | bool alignThreadXReduction = |
| 1222 | getConstantIntValue(ofr: launch.getBlockSizeY()) != 1 || |
| 1223 | getConstantIntValue(ofr: launch.getBlockSizeZ()) != 1; |
| 1224 | |
| 1225 | // A reduction over both dimensions counts as a ThreadX one: it still |
| 1226 | // shuffles ThreadX lanes within the row. |
| 1227 | auto classifyAllReduce = [&](gpu::AllReduceOp allReduce) { |
| 1228 | bool hasThreadX = false; |
| 1229 | bool hasThreadY = false; |
| 1230 | for (auto parDim : mlir::acc::getParDimsAttr(op: allReduce).getArray()) { |
| 1231 | hasThreadX |= parDim.isThreadX(); |
| 1232 | hasThreadY |= parDim.isThreadY(); |
| 1233 | } |
| 1234 | if (hasThreadX && alignThreadXReduction) |
| 1235 | needsThreadXAlign = true; |
| 1236 | else if (hasThreadY) |
| 1237 | needsThreadYAlign = true; |
| 1238 | }; |
| 1239 | |
| 1240 | launch.walk(callback&: classifyAllReduce); |
| 1241 | // Also check if called routines have thread-level reductions. |
| 1242 | launch.walk(callback: [&](func::CallOp callOp) { |
| 1243 | if (gpu::GPUFuncOp callee = |
| 1244 | callOp->getParentOfType<ModuleOp>().lookupSymbol<gpu::GPUFuncOp>( |
| 1245 | name: callOp.getCallee())) |
| 1246 | callee.walk(callback&: classifyAllReduce); |
| 1247 | }); |
| 1248 | |
| 1249 | bool isShuffleEnabled = needsThreadXAlign || needsThreadYAlign; |
| 1250 | |
| 1251 | std::optional<int64_t> constBlockDimX = |
| 1252 | getConstantIntValue(ofr: launch.getBlockSizeX()); |
| 1253 | std::optional<int64_t> constBlockDimY = |
| 1254 | getConstantIntValue(ofr: launch.getBlockSizeY()); |
| 1255 | std::optional<int64_t> constBlockDimZ = |
| 1256 | getConstantIntValue(ofr: launch.getBlockSizeZ()); |
| 1257 | |
| 1258 | // Skip subgroup alignment only when the total thread count is already |
| 1259 | // below a subgroup (constant blockDim.x in 2..subgroupSize-1 and |
| 1260 | // constant blockDim.y/z == 1). If blockDim.y/z > 1 or is unknown, |
| 1261 | // padding blockDim.x to a subgroup is still required so subgroups don't |
| 1262 | // cross row boundaries for row-local shuffle/ThreadY-barrier reductions. |
| 1263 | bool skipAlign = constBlockDimX && constBlockDimY && constBlockDimZ && |
| 1264 | *constBlockDimX > 1 && *constBlockDimX < subgroupSize && |
| 1265 | *constBlockDimY == 1 && *constBlockDimZ == 1; |
| 1266 | |
| 1267 | // Padding a worker-only launch divides blockDim.y by the same factor and |
| 1268 | // leaves the folded-away workers as ThreadX lanes redoing each other's |
| 1269 | // work, so keep its shape instead. |
| 1270 | if (isSingleThreadWorkerLaunch()) |
| 1271 | skipAlign = true; |
| 1272 | if (needsThreadYAlign && !needsThreadXAlign && !hasThreadYBarrier && |
| 1273 | isWorkerOnlyShuffleLaunch()) |
| 1274 | skipAlign = true; |
| 1275 | |
| 1276 | if ((isShuffleEnabled || hasThreadYBarrier) && !skipAlign) { |
| 1277 | rewriter.setInsertionPoint(launch); |
| 1278 | |
| 1279 | Value curBlockDimX = launch.getBlockSizeX(); |
| 1280 | Value curBlockDimY = launch.getBlockSizeY(); |
| 1281 | Value curBlockDimZ = launch.getBlockSizeZ(); |
| 1282 | |
| 1283 | // Emit a report on changing parallelism. |
| 1284 | accSupport.emitRemark(op: computeRegion, messageFn: [&]() { |
| 1285 | auto getName = [&](Value val) -> std::string { |
| 1286 | std::string name = accSupport.getVariableName(v: val); |
| 1287 | return name.empty() ? "(*)" : name; |
| 1288 | }; |
| 1289 | std::string blockDimXValStr = getName(curBlockDimX); |
| 1290 | std::string blockDimYValStr = getName(curBlockDimY); |
| 1291 | llvm::StringRef kind = |
| 1292 | isShuffleEnabled ? "Shuffle reduction" : "ThreadY barrier" ; |
| 1293 | return (llvm::Twine(kind) + |
| 1294 | " is generated while adjusting the number of threads into " |
| 1295 | "groups of " + |
| 1296 | llvm::Twine(subgroupSize) + ".\n\t" + blockDimXName + ": `" + |
| 1297 | blockDimXValStr + "` to `((" + blockDimXValStr + " + " + |
| 1298 | llvm::Twine(subgroupAlignMask) + ") / " + |
| 1299 | llvm::Twine(subgroupSize) + ") * " + llvm::Twine(subgroupSize) + |
| 1300 | "`\n" + "\t" + blockDimYName + ": `" + blockDimYValStr + |
| 1301 | "` to `max(1, (new-" + blockDimXName + " * " + blockDimYValStr + |
| 1302 | ") / new-" + blockDimXName + ")`" ) |
| 1303 | .str(); |
| 1304 | }); |
| 1305 | |
| 1306 | // Update the ThreadX length and the numbers of ThreadY and ThreadZ. |
| 1307 | // When the original block dimensions are compile-time |
| 1308 | // constants, compute the adjusted dimensions as constants directly so |
| 1309 | // that the GpuKernelOutliningPass can set `known_block_size` on the |
| 1310 | // outlined gpu.func. |
| 1311 | Value newBlockDimX, newBlockDimY, newBlockDimZ; |
| 1312 | if (constBlockDimX && constBlockDimY && constBlockDimZ) { |
| 1313 | int64_t bdx = *constBlockDimX; |
| 1314 | int64_t bdy = *constBlockDimY; |
| 1315 | int64_t bdz = *constBlockDimZ; |
| 1316 | int64_t alignedBdx = |
| 1317 | ((bdx + subgroupAlignMask) / subgroupSize) * subgroupSize; |
| 1318 | int64_t numXYThreads = bdx * bdy; |
| 1319 | int64_t numThreads = numXYThreads * bdz; |
| 1320 | int64_t newBdy = std::max<int64_t>(a: 1, b: numXYThreads / alignedBdx); |
| 1321 | int64_t newBdz = |
| 1322 | std::max<int64_t>(a: 1, b: numThreads / (alignedBdx * newBdy)); |
| 1323 | newBlockDimX = |
| 1324 | arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: alignedBdx); |
| 1325 | newBlockDimY = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: newBdy); |
| 1326 | newBlockDimZ = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: newBdz); |
| 1327 | } else { |
| 1328 | // numXYThreads = blockDim.x * blockDim.y |
| 1329 | Value numXYThreads = |
| 1330 | arith::MulIOp::create(builder&: rewriter, location: loc, lhs: curBlockDimX, rhs: curBlockDimY); |
| 1331 | Value numThreads = |
| 1332 | arith::MulIOp::create(builder&: rewriter, location: loc, lhs: numXYThreads, rhs: curBlockDimZ); |
| 1333 | // blockDim.x = ((blockDim.x + mask) / subgroupSize) * subgroupSize |
| 1334 | Value cstMask = |
| 1335 | arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: subgroupAlignMask); |
| 1336 | Value cstSubgroupSize = |
| 1337 | arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: subgroupSize); |
| 1338 | Value padded = |
| 1339 | arith::AddIOp::create(builder&: rewriter, location: loc, lhs: curBlockDimX, rhs: cstMask); |
| 1340 | Value subgroupsRequired = |
| 1341 | arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: padded, rhs: cstSubgroupSize); |
| 1342 | newBlockDimX = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: subgroupsRequired, |
| 1343 | rhs: cstSubgroupSize); |
| 1344 | // blockDim.y = max(1, numXYThreads / blockDim.x) |
| 1345 | Value quotient = |
| 1346 | arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: numXYThreads, rhs: newBlockDimX); |
| 1347 | Value cst1 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1); |
| 1348 | newBlockDimY = arith::MaxUIOp::create(builder&: rewriter, location: loc, lhs: cst1, rhs: quotient); |
| 1349 | // blockDim.z = max(1, numThreads / (blockDim.x * blockDim.y)) |
| 1350 | Value newNumXYThreads = |
| 1351 | arith::MulIOp::create(builder&: rewriter, location: loc, lhs: newBlockDimX, rhs: newBlockDimY); |
| 1352 | quotient = |
| 1353 | arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: numThreads, rhs: newNumXYThreads); |
| 1354 | newBlockDimZ = arith::MaxUIOp::create(builder&: rewriter, location: loc, lhs: cst1, rhs: quotient); |
| 1355 | } |
| 1356 | |
| 1357 | launch.getBlockSizeXMutable().assign(value: newBlockDimX); |
| 1358 | launch.getBlockSizeYMutable().assign(value: newBlockDimY); |
| 1359 | launch.getBlockSizeZMutable().assign(value: newBlockDimZ); |
| 1360 | } |
| 1361 | } |
| 1362 | |
| 1363 | if (hasFailed) |
| 1364 | return failure(); |
| 1365 | |
| 1366 | if (!threadPrivateVarNames.empty()) { |
| 1367 | accSupport.emitRemark(op: computeRegion, messageFn: [&]() { |
| 1368 | return (llvm::Twine("Local memory or registers used for " ) + |
| 1369 | llvm::join(R&: threadPrivateVarNames, Separator: "," )) |
| 1370 | .str(); |
| 1371 | }); |
| 1372 | } |
| 1373 | |
| 1374 | if (!sharedMemPrivateVarNames.empty()) { |
| 1375 | accSupport.emitRemark(op: computeRegion, messageFn: [&]() { |
| 1376 | return (llvm::Twine("GPU shared memory used for " ) + |
| 1377 | llvm::join(R&: sharedMemPrivateVarNames, Separator: "," )) |
| 1378 | .str(); |
| 1379 | }); |
| 1380 | } |
| 1381 | |
| 1382 | if (!globalMemVarNames.empty()) { |
| 1383 | accSupport.emitRemark(op: computeRegion, messageFn: [&]() { |
| 1384 | return (llvm::Twine("GPU global memory used for " ) + |
| 1385 | llvm::join(R&: globalMemVarNames, Separator: "," )) |
| 1386 | .str(); |
| 1387 | }); |
| 1388 | } |
| 1389 | |
| 1390 | rewriter.eraseOp(op: computeRegion); |
| 1391 | return success(); |
| 1392 | } |
| 1393 | |
| 1394 | /// True when this accumulate is redundant in a nested reduction chain: the |
| 1395 | /// value is a load of the destination memref and a sibling |
| 1396 | /// acc.reduction_combine with block par_dims has already reduced %M across |
| 1397 | /// threads in the block. |
| 1398 | /// |
| 1399 | /// %v = memref.load %M[] |
| 1400 | /// acc.reduction_accumulate %v to %M ... |
| 1401 | /// acc.reduction_combine %M into %parent ... {block par_dims} |
| 1402 | /// |
| 1403 | /// Lowering the accumulate again would double-count. Detection is structural; |
| 1404 | /// nested reductions into per-thread privates do not match because their |
| 1405 | /// combines are not block-scoped. |
| 1406 | static bool isRedundantChainAccumulate(acc::ReductionAccumulateOp op) { |
| 1407 | Value memref = op.getMemref(); |
| 1408 | memref::LoadOp loadOp = op.getValue().getDefiningOp<memref::LoadOp>(); |
| 1409 | if (!loadOp || loadOp.getMemRef() != memref) |
| 1410 | return false; |
| 1411 | for (Operation *user : memref.getUsers()) { |
| 1412 | acc::ReductionCombineOp combineOp = dyn_cast<acc::ReductionCombineOp>(Val: user); |
| 1413 | if (!combineOp || combineOp.getDestMemref() != memref) |
| 1414 | continue; |
| 1415 | SmallVector<mlir::acc::GPUParallelDimAttr> parDims = |
| 1416 | getReductionCombineParDims(op: combineOp); |
| 1417 | if (llvm::any_of(Range&: parDims, P: [](mlir::acc::GPUParallelDimAttr d) { |
| 1418 | return d.isAnyBlock(); |
| 1419 | })) { |
| 1420 | return true; |
| 1421 | } |
| 1422 | } |
| 1423 | return false; |
| 1424 | } |
| 1425 | |
| 1426 | static GPUParallelDimsAttr |
| 1427 | getPrivateParDims(acc::PrivateLocalOp privateLocal, |
| 1428 | acc::ComputeRegionOp computeRegion) { |
| 1429 | if (GPUParallelDimsAttr parDims = acc::getParDimsAttr(op: privateLocal)) |
| 1430 | return parDims; |
| 1431 | if (acc::PrivatizeOp privatize = getPrivatizeOp(privateLocal, computeRegion)) |
| 1432 | return privatize.getParDimsAttr(); |
| 1433 | return {}; |
| 1434 | } |
| 1435 | |
| 1436 | /// True when \p privateLocal has one private slot per ThreadY row. |
| 1437 | static bool isThreadYPrivate(acc::PrivateLocalOp privateLocal, bool allowBlock, |
| 1438 | acc::ComputeRegionOp computeRegion) { |
| 1439 | if (!privateLocal) |
| 1440 | return false; |
| 1441 | GPUParallelDimsAttr parDims = getPrivateParDims(privateLocal, computeRegion); |
| 1442 | if (!parDims) |
| 1443 | return false; |
| 1444 | return llvm::any_of(Range: parDims.getArray(), |
| 1445 | P: [](auto dim) { return dim.isThreadY(); }) && |
| 1446 | llvm::all_of(Range: parDims.getArray(), P: [=](auto dim) { |
| 1447 | return dim.isThreadY() || (allowBlock && dim.isAnyBlock()); |
| 1448 | }); |
| 1449 | } |
| 1450 | |
| 1451 | struct ThreadYBroadeningInfo { |
| 1452 | bool hasActiveWorkerCombine = false; |
| 1453 | bool hasExplicitInactiveCombine = false; |
| 1454 | bool hasBroadeningConflict = false; |
| 1455 | Operation *diagnosticOp = nullptr; |
| 1456 | |
| 1457 | void merge(const ThreadYBroadeningInfo &other) { |
| 1458 | hasActiveWorkerCombine |= other.hasActiveWorkerCombine; |
| 1459 | hasExplicitInactiveCombine |= other.hasExplicitInactiveCombine; |
| 1460 | hasBroadeningConflict |= other.hasBroadeningConflict; |
| 1461 | if (!diagnosticOp) |
| 1462 | diagnosticOp = other.diagnosticOp; |
| 1463 | } |
| 1464 | }; |
| 1465 | |
| 1466 | /// True when executing \p op on additional ThreadY rows may change behavior. |
| 1467 | static bool hasUnsafeEffectsWhenBroadening(Operation *op) { |
| 1468 | if (auto effectOp = dyn_cast<MemoryEffectOpInterface>(Val: op)) { |
| 1469 | SmallVector<MemoryEffects::EffectInstance> effects; |
| 1470 | effectOp.getEffects(effects); |
| 1471 | return llvm::any_of(Range&: effects, P: [](const auto &effect) { |
| 1472 | return !isa<MemoryEffects::Read>(effect.getEffect()); |
| 1473 | }); |
| 1474 | } |
| 1475 | return !op->hasTrait<OpTrait::HasRecursiveMemoryEffects>(); |
| 1476 | } |
| 1477 | |
| 1478 | /// True when \p accumulator is the destination of a separate block-scoped |
| 1479 | /// combine, so it holds a distinct per-worker partial rather than a broadcast. |
| 1480 | static bool isFedByInnerBlockCombine(acc::PrivateLocalOp accumulator, |
| 1481 | Operation *selfCombine) { |
| 1482 | if (!accumulator) |
| 1483 | return false; |
| 1484 | for (Operation *user : accumulator.getResult().getUsers()) { |
| 1485 | if (user == selfCombine) |
| 1486 | continue; |
| 1487 | auto combineOp = dyn_cast<acc::ReductionCombineOp>(Val: user); |
| 1488 | if (!combineOp || |
| 1489 | unwrapMemRefConversion(v: combineOp.getDestMemref()).getDefiningOp() != |
| 1490 | accumulator.getOperation()) |
| 1491 | continue; |
| 1492 | SmallVector<mlir::acc::GPUParallelDimAttr> parDims = |
| 1493 | getReductionCombineParDims(op: combineOp); |
| 1494 | if (llvm::any_of(Range&: parDims, P: [](mlir::acc::GPUParallelDimAttr d) { |
| 1495 | return d.isAnyBlock(); |
| 1496 | })) |
| 1497 | return true; |
| 1498 | } |
| 1499 | return false; |
| 1500 | } |
| 1501 | |
| 1502 | /// Records whether \p combineOp requires ThreadY to remain active. |
| 1503 | static void classifyThreadYCombine(ThreadYBroadeningInfo &info, |
| 1504 | Operation *combineOp, Value src, Value dest, |
| 1505 | ArrayRef<GPUParallelDimAttr> parDims, |
| 1506 | acc::ComputeRegionOp computeRegion) { |
| 1507 | bool hasThreadY = llvm::any_of( |
| 1508 | Range&: parDims, P: [](GPUParallelDimAttr parDim) { return parDim.isThreadY(); }); |
| 1509 | bool hasBlock = llvm::any_of( |
| 1510 | Range&: parDims, P: [](GPUParallelDimAttr parDim) { return parDim.isAnyBlock(); }); |
| 1511 | acc::PrivateLocalOp srcPrivate = |
| 1512 | unwrapMemRefConversion(v: src).getDefiningOp<acc::PrivateLocalOp>(); |
| 1513 | acc::PrivateLocalOp destPrivate = |
| 1514 | unwrapMemRefConversion(v: dest).getDefiningOp<acc::PrivateLocalOp>(); |
| 1515 | bool hasPrivateDest = isa<acc::ReductionCombineOp>(Val: combineOp) && srcPrivate && |
| 1516 | destPrivate && |
| 1517 | getPrivatizeOp(privateLocal: srcPrivate, computeRegion) != |
| 1518 | getPrivatizeOp(privateLocal: destPrivate, computeRegion); |
| 1519 | if (hasThreadY && hasBlock && |
| 1520 | isThreadYPrivate(privateLocal: srcPrivate, allowBlock: hasPrivateDest, computeRegion)) { |
| 1521 | info.hasActiveWorkerCombine = true; |
| 1522 | return; |
| 1523 | } |
| 1524 | |
| 1525 | // A block_y+thread_y accumulator fed by an inner block-scoped combine holds |
| 1526 | // a distinct partial per worker row, so its combine runs on every row; a |
| 1527 | // plain worker accumulate broadcasts via all_reduce and stays row-zero. |
| 1528 | if (hasThreadY && hasBlock && |
| 1529 | isThreadYPrivate(privateLocal: srcPrivate, /*allowBlock=*/true, computeRegion) && |
| 1530 | isFedByInnerBlockCombine(accumulator: srcPrivate, selfCombine: combineOp)) { |
| 1531 | info.hasActiveWorkerCombine = true; |
| 1532 | return; |
| 1533 | } |
| 1534 | |
| 1535 | info.hasExplicitInactiveCombine = true; |
| 1536 | if (!info.diagnosticOp) |
| 1537 | info.diagnosticOp = combineOp; |
| 1538 | } |
| 1539 | |
| 1540 | /// ThreadY must remain active for hierarchical combines over worker rows. |
| 1541 | /// Broadening is rejected when siblings require inactive ThreadY or have side |
| 1542 | /// effects; nested predicates enforce their own safety. |
| 1543 | static ThreadYBroadeningInfo |
| 1544 | analyzeThreadYBroadening(Block &predicateBlock, |
| 1545 | acc::ComputeRegionOp computeRegion) { |
| 1546 | ThreadYBroadeningInfo info; |
| 1547 | for (Operation &nestedOp : predicateBlock) { |
| 1548 | if (acc::PredicateRegionOp nestedPredicate = |
| 1549 | dyn_cast<acc::PredicateRegionOp>(Val&: nestedOp)) { |
| 1550 | ThreadYBroadeningInfo nestedInfo = analyzeThreadYBroadening( |
| 1551 | predicateBlock&: nestedPredicate.getRegion().front(), computeRegion); |
| 1552 | info.hasActiveWorkerCombine |= nestedInfo.hasActiveWorkerCombine; |
| 1553 | continue; |
| 1554 | } |
| 1555 | if (acc::ReductionCombineOp combineOp = |
| 1556 | dyn_cast<acc::ReductionCombineOp>(Val&: nestedOp)) { |
| 1557 | classifyThreadYCombine( |
| 1558 | info, combineOp, src: combineOp.getSrcMemref(), dest: combineOp.getDestMemref(), |
| 1559 | parDims: getReductionCombineParDims(op: combineOp), computeRegion); |
| 1560 | continue; |
| 1561 | } |
| 1562 | if (acc::ReductionCombineRegionOp combineRegionOp = |
| 1563 | dyn_cast<acc::ReductionCombineRegionOp>(Val&: nestedOp)) { |
| 1564 | classifyThreadYCombine(info, combineOp: combineRegionOp, src: combineRegionOp.getSrcVar(), |
| 1565 | dest: combineRegionOp.getDestVar(), |
| 1566 | parDims: getReductionCombineParDims(op: combineRegionOp), |
| 1567 | computeRegion); |
| 1568 | continue; |
| 1569 | } |
| 1570 | if (nestedOp.getNumRegions() != 0) { |
| 1571 | if (hasUnsafeEffectsWhenBroadening(op: &nestedOp)) { |
| 1572 | info.hasBroadeningConflict = true; |
| 1573 | if (!info.diagnosticOp) |
| 1574 | info.diagnosticOp = &nestedOp; |
| 1575 | } |
| 1576 | for (Region ®ion : nestedOp.getRegions()) |
| 1577 | for (Block &nestedBlock : region) |
| 1578 | info.merge(other: analyzeThreadYBroadening(predicateBlock&: nestedBlock, computeRegion)); |
| 1579 | continue; |
| 1580 | } |
| 1581 | if (hasUnsafeEffectsWhenBroadening(op: &nestedOp)) { |
| 1582 | info.hasBroadeningConflict = true; |
| 1583 | if (!info.diagnosticOp) |
| 1584 | info.diagnosticOp = &nestedOp; |
| 1585 | } |
| 1586 | } |
| 1587 | return info; |
| 1588 | } |
| 1589 | |
| 1590 | std::pair<SmallVector<mlir::acc::GPUParallelDimAttr>, |
| 1591 | SmallVector<mlir::acc::GPUParallelDimAttr>> |
| 1592 | ACCCGToGPULowering::computeActiveAndInactiveParDims(Operation *op, |
| 1593 | Block *block) { |
| 1594 | MLIRContext *ctx = computeRegion->getContext(); |
| 1595 | |
| 1596 | mlir::acc::GPUParallelDimAttr routineParDim; |
| 1597 | if (isInsideACCSpecializedRoutine(op: computeRegion)) { |
| 1598 | FunctionOpInterface funcOp = |
| 1599 | computeRegion->getParentOfType<FunctionOpInterface>(); |
| 1600 | routineParDim = getSpecializedRoutineDim(funcOp, policy: defaultPolicy); |
| 1601 | } |
| 1602 | |
| 1603 | // The active set is precomputed separately from the ownership par_dims for |
| 1604 | // privatizations, and a predicate region may likewise state which dims run |
| 1605 | // it unpredicated; prefer that attribute when present. The inactive dims are |
| 1606 | // the launch dims which are not active. |
| 1607 | if (isa<acc::PrivateLocalOp, acc::PrivatizeOp, acc::PredicateRegionOp>(Val: op)) { |
| 1608 | if (mlir::acc::ActiveParDimsAttr precomputedActiveParDims = |
| 1609 | getActiveParDimsAttr(op)) { |
| 1610 | mlir::acc::GPUParallelDimAttr lowestParDim = |
| 1611 | mlir::acc::GPUParallelDimAttr::threadXDim(ctx); |
| 1612 | |
| 1613 | SmallVector<mlir::acc::GPUParallelDimAttr> launchParDims; |
| 1614 | if (routineParDim) { |
| 1615 | for (mlir::acc::GPUParallelDimAttr parDim = routineParDim; |
| 1616 | parDim.getOrder() >= lowestParDim.getOrder(); |
| 1617 | parDim = parDim.getOneLower()) { |
| 1618 | mlir::acc::insertParDim(parDims&: launchParDims, parDim); |
| 1619 | } |
| 1620 | } else { |
| 1621 | launchParDims = computeRegion.getLaunchParDims(); |
| 1622 | } |
| 1623 | |
| 1624 | SmallVector<mlir::acc::GPUParallelDimAttr> activeParDims( |
| 1625 | precomputedActiveParDims.getArray()); |
| 1626 | SmallVector<mlir::acc::GPUParallelDimAttr> inactiveParDims; |
| 1627 | for (mlir::acc::GPUParallelDimAttr launchParDim : launchParDims) { |
| 1628 | if (launchParDim.getOrder() < lowestParDim.getOrder()) |
| 1629 | break; |
| 1630 | if (!llvm::is_contained(Range&: activeParDims, Element: launchParDim)) |
| 1631 | inactiveParDims.push_back(Elt: launchParDim); |
| 1632 | } |
| 1633 | return std::pair{activeParDims, inactiveParDims}; |
| 1634 | } |
| 1635 | } |
| 1636 | |
| 1637 | SmallVector<mlir::acc::GPUParallelDimAttr> ancestorParDims = |
| 1638 | getAncestorParDims(op); |
| 1639 | // Preserve whether there were any structural ancestor par-dims before |
| 1640 | // we start augmenting them based on inner uses (e.g. private_local). |
| 1641 | // This is needed for gang redundancy check - stores to worker-indexed |
| 1642 | // private_local should not disable redundant gang execution. |
| 1643 | bool noStructuralAncestorParDims = |
| 1644 | llvm::none_of(Range&: ancestorParDims, P: [](auto pd) { return !pd.isSeq(); }); |
| 1645 | |
| 1646 | if (routineParDim) { |
| 1647 | if (routineParDim.isThreadX()) { |
| 1648 | mlir::acc::insertParDim(parDims&: ancestorParDims, |
| 1649 | parDim: mlir::acc::GPUParallelDimAttr::threadYDim(ctx)); |
| 1650 | } |
| 1651 | mlir::acc::insertParDim(parDims&: ancestorParDims, |
| 1652 | parDim: mlir::acc::GPUParallelDimAttr::blockXDim(ctx)); |
| 1653 | } |
| 1654 | |
| 1655 | // acc.private_local should use the same par_dims as acc.reduction_accumulate. |
| 1656 | if (acc::PrivateLocalOp privateLocalOp = dyn_cast<acc::PrivateLocalOp>(Val: op)) { |
| 1657 | for (Operation *user : privateLocalOp.getResult().getUsers()) { |
| 1658 | if (acc::ReductionAccumulateOp accumulateOp = |
| 1659 | dyn_cast<acc::ReductionAccumulateOp>(Val: user)) { |
| 1660 | if (accumulateOp.getMemref() == privateLocalOp.getResult()) { |
| 1661 | for (mlir::acc::GPUParallelDimAttr parDim : |
| 1662 | accumulateOp.getParDims().getArray()) { |
| 1663 | mlir::acc::insertParDim(parDims&: ancestorParDims, parDim); |
| 1664 | } |
| 1665 | } |
| 1666 | } |
| 1667 | // For decomposed complex reductions, the private_local is consumed |
| 1668 | // by an acc.reduction_combine{,_region} (no acc.reduction_accumulate |
| 1669 | // user). Mirror the par_dims so this private_local is treated as the |
| 1670 | // accumulator at the same parallelism level as a scalar reduction |
| 1671 | // would be (per-thread, not block-shared). |
| 1672 | if (acc::ReductionCombineOp combineOp = |
| 1673 | dyn_cast<acc::ReductionCombineOp>(Val: user)) { |
| 1674 | if (combineOp.getSrcMemref() == privateLocalOp.getResult()) { |
| 1675 | for (mlir::acc::GPUParallelDimAttr parDim : |
| 1676 | getReductionCombineParDims(op: combineOp)) { |
| 1677 | mlir::acc::insertParDim(parDims&: ancestorParDims, parDim); |
| 1678 | } |
| 1679 | } |
| 1680 | } |
| 1681 | if (auto combineRegionOp = |
| 1682 | dyn_cast<acc::ReductionCombineRegionOp>(Val: user)) { |
| 1683 | if (combineRegionOp.getSrcVar() == privateLocalOp.getResult()) { |
| 1684 | for (mlir::acc::GPUParallelDimAttr parDim : |
| 1685 | getReductionCombineParDims(op: combineRegionOp)) { |
| 1686 | mlir::acc::insertParDim(parDims&: ancestorParDims, parDim); |
| 1687 | } |
| 1688 | } |
| 1689 | } |
| 1690 | } |
| 1691 | // A dynamically-shaped privatization is materialized as a strided view |
| 1692 | // whose type does not reveal its parallel scope. Keep its own par_dims |
| 1693 | // active so the view is materialized (and predicated) per owning |
| 1694 | // thread/block. Statically-shaped privatizations are left to the stack-fit |
| 1695 | // decision so only do this for dynamic shapes. |
| 1696 | acc::PrivateType privTy = |
| 1697 | cast<acc::PrivateType>(Val: privateLocalOp.getPrivatized().getType()); |
| 1698 | MemRefType baseTy = getPrivateBaseMemRefType( |
| 1699 | baseTy: privTy.getBaseTy(), module: computeRegion->getParentOfType<ModuleOp>()); |
| 1700 | if (!baseTy.hasStaticShape()) { |
| 1701 | GPUParallelDimsAttr ownParDims = |
| 1702 | getPrivateParDims(privateLocal: privateLocalOp, computeRegion); |
| 1703 | if (ownParDims) |
| 1704 | for (GPUParallelDimAttr parDim : ownParDims.getArray()) |
| 1705 | mlir::acc::insertParDim(parDims&: ancestorParDims, parDim); |
| 1706 | } |
| 1707 | } |
| 1708 | |
| 1709 | bool hasBlock = false; |
| 1710 | for (mlir::acc::GPUParallelDimAttr parDim : ancestorParDims) |
| 1711 | if (parDim.isAnyBlock()) |
| 1712 | hasBlock = true; |
| 1713 | |
| 1714 | mlir::acc::GPUParallelDimAttr lowestParDim = |
| 1715 | mlir::acc::GPUParallelDimAttr::threadXDim(ctx); |
| 1716 | if (block) { |
| 1717 | ThreadYBroadeningInfo threadYInfo = |
| 1718 | analyzeThreadYBroadening(predicateBlock&: *block, computeRegion); |
| 1719 | |
| 1720 | auto applyCombineParDims = |
| 1721 | [&](ArrayRef<mlir::acc::GPUParallelDimAttr> combineParDims) { |
| 1722 | for (mlir::acc::GPUParallelDimAttr parDim : combineParDims) |
| 1723 | mlir::acc::removeParDim(parDims&: ancestorParDims, parDim); |
| 1724 | return success(); |
| 1725 | }; |
| 1726 | block->walk(callback: [&](Operation *op) -> WalkResult { |
| 1727 | // Writes to acc.private_local must keep the privatization's par_dims |
| 1728 | // active so the write runs on all owning threads instead of being |
| 1729 | // predicated to a single lane. |
| 1730 | auto addPrivateStoreParDims = [&](Value target) { |
| 1731 | if (auto privateLocalOp = getPrivateLocalForMemref(memref: target)) { |
| 1732 | GPUParallelDimsAttr parDimsAttr = |
| 1733 | getPrivateParDims(privateLocal: privateLocalOp, computeRegion); |
| 1734 | if (parDimsAttr) |
| 1735 | for (auto parDim : parDimsAttr.getArray()) |
| 1736 | mlir::acc::insertParDim(parDims&: ancestorParDims, parDim); |
| 1737 | } |
| 1738 | }; |
| 1739 | if (auto memEffects = dyn_cast<MemoryEffectOpInterface>(Val: op)) { |
| 1740 | SmallVector<MemoryEffects::EffectInstance> effects; |
| 1741 | memEffects.getEffects(effects); |
| 1742 | for (const MemoryEffects::EffectInstance &effect : effects) { |
| 1743 | if (isa<MemoryEffects::Write>(Val: effect.getEffect()) && |
| 1744 | effect.getValue()) |
| 1745 | addPrivateStoreParDims(effect.getValue()); |
| 1746 | } |
| 1747 | } |
| 1748 | // Consider ACC routine calls; routine calls should be predicated up to |
| 1749 | // one level above the parallel dimension of the callee. |
| 1750 | if (CallOpInterface callOp = dyn_cast<CallOpInterface>(Val: op)) { |
| 1751 | if (mlir::acc::GPUParallelDimAttr parDim = |
| 1752 | getAccRoutineCallParDim(callOp, policy: defaultPolicy)) { |
| 1753 | if (parDim.isBlockZ()) |
| 1754 | lowestParDim = parDim; |
| 1755 | else |
| 1756 | lowestParDim = parDim.getOneHigher(); |
| 1757 | } |
| 1758 | } |
| 1759 | // acc.reduction_combine_region should be predicated with the par_dims of |
| 1760 | // acc.reduction_accumulate. This is required when using combine between |
| 1761 | // kernel and loop in combined constructs. |
| 1762 | if (acc::ReductionCombineOp reductionCombineOp = |
| 1763 | dyn_cast<acc::ReductionCombineOp>(Val: op)) { |
| 1764 | if (failed(Result: applyCombineParDims( |
| 1765 | getReductionCombineParDims(op: reductionCombineOp)))) |
| 1766 | return WalkResult::interrupt(); |
| 1767 | } |
| 1768 | if (acc::ReductionCombineRegionOp combineRegionOp = |
| 1769 | dyn_cast<acc::ReductionCombineRegionOp>(Val: op)) { |
| 1770 | if (failed(Result: applyCombineParDims( |
| 1771 | getReductionCombineParDims(op: combineRegionOp)))) |
| 1772 | return WalkResult::interrupt(); |
| 1773 | } |
| 1774 | // An array accumulate reduces across its par_dims via gpu.all_reduce, so |
| 1775 | // all those threads must execute it - treat them as active (unlike the |
| 1776 | // scalar accumulate, which is active through its enclosing scf.parallel). |
| 1777 | if (acc::ReductionAccumulateArrayOp accArrayOp = |
| 1778 | dyn_cast<acc::ReductionAccumulateArrayOp>(Val: op)) { |
| 1779 | for (mlir::acc::GPUParallelDimAttr parDim : |
| 1780 | accArrayOp.getParDims().getArray()) { |
| 1781 | mlir::acc::insertParDim(parDims&: ancestorParDims, parDim); |
| 1782 | } |
| 1783 | } |
| 1784 | return WalkResult::advance(); |
| 1785 | }); |
| 1786 | mlir::acc::GPUParallelDimAttr threadY = |
| 1787 | mlir::acc::GPUParallelDimAttr::threadYDim(ctx); |
| 1788 | bool baselineThreadYActive = llvm::is_contained(Range&: ancestorParDims, Element: threadY); |
| 1789 | if (threadYInfo.hasActiveWorkerCombine && !baselineThreadYActive) { |
| 1790 | if (threadYInfo.hasExplicitInactiveCombine || |
| 1791 | threadYInfo.hasBroadeningConflict) { |
| 1792 | Operation *diagnosticOp = |
| 1793 | threadYInfo.diagnosticOp ? threadYInfo.diagnosticOp : op; |
| 1794 | (void)accSupport.emitNYI( |
| 1795 | loc: diagnosticOp->getLoc(), |
| 1796 | message: "operations in the same predicate region require incompatible " |
| 1797 | "ThreadY predication" ); |
| 1798 | hasFailed = true; |
| 1799 | return {}; |
| 1800 | } |
| 1801 | mlir::acc::insertParDim(parDims&: ancestorParDims, parDim: threadY); |
| 1802 | } |
| 1803 | } |
| 1804 | |
| 1805 | // Obtain launch dimensions |
| 1806 | SmallVector<mlir::acc::GPUParallelDimAttr> launchParDims; |
| 1807 | if (routineParDim) { |
| 1808 | for (mlir::acc::GPUParallelDimAttr parDim = routineParDim; |
| 1809 | parDim.getOrder() >= lowestParDim.getOrder(); |
| 1810 | parDim = parDim.getOneLower()) { |
| 1811 | mlir::acc::insertParDim(parDims&: launchParDims, parDim); |
| 1812 | } |
| 1813 | } else { |
| 1814 | launchParDims = computeRegion.getLaunchParDims(); |
| 1815 | } |
| 1816 | |
| 1817 | // Compute dimensions that execute op |
| 1818 | SmallVector<mlir::acc::GPUParallelDimAttr> activeParDims, inactiveParDims; |
| 1819 | for (mlir::acc::GPUParallelDimAttr launchParDim : launchParDims) { |
| 1820 | if (launchParDim.getOrder() < lowestParDim.getOrder()) |
| 1821 | break; |
| 1822 | if (llvm::find(Range&: ancestorParDims, Val: launchParDim) != ancestorParDims.end() || |
| 1823 | (launchParDim.isAnyBlock() && |
| 1824 | (noStructuralAncestorParDims || hasBlock))) { |
| 1825 | activeParDims.push_back(Elt: launchParDim); |
| 1826 | } else { |
| 1827 | inactiveParDims.push_back(Elt: launchParDim); |
| 1828 | } |
| 1829 | } |
| 1830 | |
| 1831 | return std::pair{activeParDims, inactiveParDims}; |
| 1832 | } |
| 1833 | |
| 1834 | Value ACCCGToGPULowering::emitPredicate( |
| 1835 | Location loc, SmallVector<mlir::acc::GPUParallelDimAttr> &inactiveParDims) { |
| 1836 | Value predicate; |
| 1837 | for (mlir::acc::GPUParallelDimAttr inactiveParDim : inactiveParDims) { |
| 1838 | Value threadId = getGPUThreadIdFor(proc: inactiveParDim.getProcessor()); |
| 1839 | TypedAttr zeroAttr = rewriter.getZeroAttr(type: threadId.getType()); |
| 1840 | Value zero = arith::ConstantOp::create(builder&: rewriter, location: loc, value: zeroAttr); |
| 1841 | Value cmp = arith::CmpIOp::create(builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq, |
| 1842 | lhs: threadId, rhs: zero); |
| 1843 | if (predicate) |
| 1844 | predicate = arith::AndIOp::create(builder&: rewriter, location: loc, lhs: cmp, rhs: predicate); |
| 1845 | else |
| 1846 | predicate = cmp; |
| 1847 | } |
| 1848 | return predicate; |
| 1849 | } |
| 1850 | |
| 1851 | void ACCCGToGPULowering::createBarrier( |
| 1852 | Location loc, mlir::acc::GPUParallelDimsAttr parDimsAttr) { |
| 1853 | bool hasAnyBlock = false, hasThreadY = false, hasThreadX = false; |
| 1854 | for (auto parDim : parDimsAttr.getArray()) { |
| 1855 | if (parDim.isAnyBlock()) |
| 1856 | hasAnyBlock = true; |
| 1857 | if (parDim.isThreadY()) |
| 1858 | hasThreadY = true; |
| 1859 | if (parDim.isThreadX()) |
| 1860 | hasThreadX = true; |
| 1861 | } |
| 1862 | |
| 1863 | if (hasAnyBlock || hasThreadY) |
| 1864 | emitGPUBarrierWorkgroup(builder&: rewriter, loc); |
| 1865 | else if (hasThreadX) |
| 1866 | createPerRowBarrier(loc); |
| 1867 | } |
| 1868 | |
| 1869 | void ACCCGToGPULowering::createPerRowBarrier(Location loc) { |
| 1870 | // A row of a (1, N, 1) launch is a single thread; a subgroup barrier would |
| 1871 | // instead synchronize N workers. |
| 1872 | if (isSingleThreadWorkerLaunch()) |
| 1873 | return; |
| 1874 | |
| 1875 | // A worker-only row narrower than a subgroup covers only part of one, so |
| 1876 | // synchronize just its lanes. Both paths leave hasThreadYBarrier unset, so |
| 1877 | // the launch is not padded on account of this barrier. |
| 1878 | if (isWorkerOnlyShuffleLaunch() && staticBlockDimX < options.subgroupSize) { |
| 1879 | emitGPUBarrierRow(builder&: rewriter, loc, rowWidth: staticBlockDimX, subgroupSize: options.subgroupSize); |
| 1880 | return; |
| 1881 | } |
| 1882 | |
| 1883 | hasThreadYBarrier = true; |
| 1884 | |
| 1885 | if (staticBlockDimX <= options.subgroupSize) { |
| 1886 | emitGPUBarrierSubgroup(builder&: rewriter, loc); |
| 1887 | return; |
| 1888 | } |
| 1889 | |
| 1890 | if (options.deviceType != mlir::acc::DeviceType::Nvidia) { |
| 1891 | (void)accSupport.emitNYI( |
| 1892 | loc, |
| 1893 | message: "per-row barrier to support worker parallelism on non-NVIDIA device" ); |
| 1894 | return; |
| 1895 | } |
| 1896 | |
| 1897 | // Per-row barrier with fully runtime branching. |
| 1898 | // Three mutually exclusive paths: |
| 1899 | // blockDim.y == 1 -> gpu.barrier (workgroup-wide, only one worker) |
| 1900 | // blockDim.x <= subgroupSize -> gpu.barrier scope<subgroup> |
| 1901 | // blockDim.x > subgroupSize -> nvvm.barrier (tid.y + 1), blockDim.x |
| 1902 | // (named) |
| 1903 | // |
| 1904 | // Per-row barriers use tid.y+1 so IDs start at 1, avoiding clash with |
| 1905 | // barrier 0. When blockDim.y >= 16, worker 15's ID (16) wraps to |
| 1906 | // physical barrier 0; this is safe because named barriers are reusable |
| 1907 | // resources - workgroup-wide and per-row barriers on the same physical |
| 1908 | // barrier execute at different program points and never overlap. |
| 1909 | Value blockDimX = gpu::BlockDimOp::create( |
| 1910 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::x); |
| 1911 | Value blockDimY = gpu::BlockDimOp::create( |
| 1912 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::y); |
| 1913 | Value cst1 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1); |
| 1914 | Value isSingleWorker = arith::CmpIOp::create( |
| 1915 | builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq, lhs: blockDimY, rhs: cst1); |
| 1916 | |
| 1917 | auto outerIf = scf::IfOp::create(builder&: rewriter, location: loc, cond: isSingleWorker, |
| 1918 | /*withElseRegion=*/true); |
| 1919 | |
| 1920 | // Then: blockDim.y == 1 -> workgroup-wide barrier (safe, only one worker) |
| 1921 | rewriter.setInsertionPointToStart(&outerIf.getThenRegion().front()); |
| 1922 | emitGPUBarrierWorkgroup(builder&: rewriter, loc); |
| 1923 | |
| 1924 | // Else: blockDim.y > 1 - choose between subgroup sync and named barrier |
| 1925 | rewriter.setInsertionPointToStart(&outerIf.getElseRegion().front()); |
| 1926 | Value cstSubgroupSize = |
| 1927 | arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: options.subgroupSize); |
| 1928 | Value isSubgroupSized = arith::CmpIOp::create( |
| 1929 | builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::ule, lhs: blockDimX, rhs: cstSubgroupSize); |
| 1930 | |
| 1931 | auto innerIf = scf::IfOp::create(builder&: rewriter, location: loc, cond: isSubgroupSized, |
| 1932 | /*withElseRegion=*/true); |
| 1933 | |
| 1934 | // Then: blockDim.x <= subgroupSize -> subgroup barrier (one worker per |
| 1935 | // subgroup) |
| 1936 | rewriter.setInsertionPointToStart(&innerIf.getThenRegion().front()); |
| 1937 | emitGPUBarrierSubgroup(builder&: rewriter, loc); |
| 1938 | |
| 1939 | // Else: blockDim.x > subgroupSize -> per-row named barrier with tid.y + 1. |
| 1940 | // The 1024-thread-per-block hardware limit with subgroup-aligned blockDim.x |
| 1941 | // (>= 64 here) guarantees blockDim.y <= 16, so IDs span at most 16 |
| 1942 | // physical barriers (0-15) with no aliasing across workers. |
| 1943 | rewriter.setInsertionPointToStart(&innerIf.getElseRegion().front()); |
| 1944 | Value threadYId = gpu::ThreadIdOp::create( |
| 1945 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::y); |
| 1946 | Value barrierId = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: threadYId, rhs: cst1); |
| 1947 | Type i32Ty = rewriter.getI32Type(); |
| 1948 | Value barrierId32 = |
| 1949 | arith::IndexCastOp::create(builder&: rewriter, location: loc, out: i32Ty, in: barrierId); |
| 1950 | Value numberOfThreads32 = |
| 1951 | arith::IndexCastOp::create(builder&: rewriter, location: loc, out: i32Ty, in: blockDimX); |
| 1952 | |
| 1953 | // GPU dialect named barriers do not have a means to create a custom barrier |
| 1954 | // id. Thus use nvvm directly. |
| 1955 | assert(options.deviceType == mlir::acc::DeviceType::Nvidia); |
| 1956 | NVVM::BarrierOp::create(builder&: rewriter, location: loc, barrierId: barrierId32, numberOfThreads: numberOfThreads32); |
| 1957 | |
| 1958 | rewriter.setInsertionPointAfter(outerIf); |
| 1959 | } |
| 1960 | |
| 1961 | /// Whether any later sibling of \p loopOp (or a loop nested inside one) is a |
| 1962 | /// loop, i.e. whether some subsequent loop in the same region may read what |
| 1963 | /// \p loopOp wrote. Used to skip a barrier after the last loop, where nothing |
| 1964 | /// reads the data afterward. |
| 1965 | static bool hasSubsequentLoopSibling(Operation *loopOp) { |
| 1966 | for (Operation *next = loopOp->getNextNode(); next; |
| 1967 | next = next->getNextNode()) { |
| 1968 | if (isa<scf::ParallelOp, scf::ForOp>(Val: next)) |
| 1969 | return true; |
| 1970 | bool nested = false; |
| 1971 | next->walk(callback: [&](Operation *op) { |
| 1972 | if (isa<scf::ParallelOp, scf::ForOp>(Val: op)) { |
| 1973 | nested = true; |
| 1974 | return WalkResult::interrupt(); |
| 1975 | } |
| 1976 | return WalkResult::advance(); |
| 1977 | }); |
| 1978 | if (nested) |
| 1979 | return true; |
| 1980 | } |
| 1981 | return false; |
| 1982 | } |
| 1983 | |
| 1984 | /// Nearest enclosing sequential loop ancestor of \p op. |
| 1985 | static LoopLikeOpInterface findFirstSequentialLoop(Operation *op) { |
| 1986 | auto isAllSequentialParDims = [](scf::ParallelOp par) -> bool { |
| 1987 | mlir::acc::GPUParallelDimsAttr pd = mlir::acc::getParDimsAttr(op: par); |
| 1988 | if (!pd || pd.getArray().empty()) |
| 1989 | return false; |
| 1990 | return llvm::all_of(Range: pd.getArray(), P: [](mlir::acc::GPUParallelDimAttr d) { |
| 1991 | return d.isSeq(); |
| 1992 | }); |
| 1993 | }; |
| 1994 | |
| 1995 | for (Operation *p = op->getParentOp(); p; p = p->getParentOp()) { |
| 1996 | // Do not need to check scf.for op's parents |
| 1997 | if (isa<scf::ForOp>(Val: p)) |
| 1998 | return cast<LoopLikeOpInterface>(Val: p); |
| 1999 | if (scf::ParallelOp parOp = dyn_cast<scf::ParallelOp>(Val: p)) { |
| 2000 | if (isAllSequentialParDims(parOp)) |
| 2001 | return cast<LoopLikeOpInterface>(Val: p); |
| 2002 | } |
| 2003 | } |
| 2004 | return nullptr; |
| 2005 | } |
| 2006 | |
| 2007 | // A sequential loop that uses gang-private shared memory needs a |
| 2008 | // workgroup-wide barrier afterward so every thread in the block observes the |
| 2009 | // same state. When more work still lies between the loop and the next |
| 2010 | // thread-reconvergence point in the same block, that barrier must follow that |
| 2011 | // work; not sit immediately after the loop. |
| 2012 | // |
| 2013 | // The helpers below mark loop-body closure and other reconvergence points |
| 2014 | // where any postponed barrier must be inserted. |
| 2015 | |
| 2016 | /// Marks the end of a loop body's iteration in the current block. |
| 2017 | static bool isLoopBodyClosureOp(Operation *op) { |
| 2018 | return isa<scf::ReduceOp, scf::YieldOp, acc::YieldOp>(Val: op); |
| 2019 | } |
| 2020 | |
| 2021 | /// Thread-reconvergence point where any postponed post-loop barrier for earlier |
| 2022 | /// loops in this block must be inserted before proceeding. |
| 2023 | static bool isDeferredBarrierFlushPoint(Operation *op) { |
| 2024 | if (isLoopBodyClosureOp(op)) |
| 2025 | return true; |
| 2026 | // The next sequential loop may consume shared state produced by the prior |
| 2027 | // one. |
| 2028 | if (isa<scf::ForOp>(Val: op)) |
| 2029 | return true; |
| 2030 | if (scf::ParallelOp parallelOp = dyn_cast<scf::ParallelOp>(Val: op)) { |
| 2031 | if (mlir::acc::hasParDimsAttr(op: parallelOp)) { |
| 2032 | if (mlir::acc::GPUParallelDimsAttr parDims = |
| 2033 | mlir::acc::getParDimsAttr(op: parallelOp)) { |
| 2034 | if (parDims.getArray().size() == 1 && |
| 2035 | parDims.getArray().front().isSeq()) { |
| 2036 | return true; |
| 2037 | } |
| 2038 | } |
| 2039 | } |
| 2040 | } |
| 2041 | return false; |
| 2042 | } |
| 2043 | |
| 2044 | /// True when a loop is followed by other work in the same block before the |
| 2045 | /// loop body closes; the post-loop barrier must wait for that reconvergence |
| 2046 | /// point instead of being placed right after the loop. |
| 2047 | static bool hasTrailingSideEffectSiblings(Operation *loopOp) { |
| 2048 | for (Operation *next = loopOp->getNextNode(); next; |
| 2049 | next = next->getNextNode()) { |
| 2050 | return !isLoopBodyClosureOp(op: next); |
| 2051 | } |
| 2052 | return false; |
| 2053 | } |
| 2054 | |
| 2055 | // Insert a barrier after a sequential loop when the surrounding parallel |
| 2056 | // structure requires it, so that block threads observe the loop's writes to |
| 2057 | // gang-/block-private shared memory before a later loop reads them. The barrier |
| 2058 | // must land at an enclosing collective (block- or worker-level) point where all |
| 2059 | // threads converge, so it cannot deadlock. |
| 2060 | // |
| 2061 | // The function walks the parent hierarchy of the loop to decide where (if at |
| 2062 | // all) to place the barrier: |
| 2063 | // - No enclosing parallel loop (top-level worksharing loop): emit a |
| 2064 | // workgroup barrier when the loop writes shared memory and a later sibling |
| 2065 | // loop may read it. |
| 2066 | // - Nearest scf.parallel parent is itself sequential: a barrier *after* it |
| 2067 | // would be unsafe when threads have varying iteration counts. However, when |
| 2068 | // the loop has thread-level sub-loops collaborating on block-private memory |
| 2069 | // (all threads participate every iteration), walk up to the block-level |
| 2070 | // ancestor and insert the barrier there. |
| 2071 | // - Nearest scf.parallel parent is a non-sequential parallel loop: walk up to |
| 2072 | // the block-level (or worker/thread-y) ancestor and insert the barrier |
| 2073 | // there, handling gang-redundant init loops and grid-stride remainders. |
| 2074 | void ACCCGToGPULowering::createBarrierAfterSeqLoop(Operation *loopOp) { |
| 2075 | // Host loops enclosing the compute region are not GPU execution scopes. |
| 2076 | auto getParentParallel = [&](Operation *op) -> scf::ParallelOp { |
| 2077 | scf::ParallelOp parent = op->getParentOfType<scf::ParallelOp>(); |
| 2078 | if (parent && computeRegion->isAncestor(other: parent)) |
| 2079 | return parent; |
| 2080 | return {}; |
| 2081 | }; |
| 2082 | scf::ParallelOp wsLoop = getParentParallel(loopOp); |
| 2083 | if (!wsLoop) { |
| 2084 | // loopOp is a worksharing loop at the kernel-body top level (no enclosing |
| 2085 | // parallel loop). When it writes gang-private shared memory |
| 2086 | // that a later sibling loop reads, a workgroup barrier is needed between |
| 2087 | // them. The point just after a top-level loop is uniform (all threads reach |
| 2088 | // it), so a workgroup-wide barrier here cannot deadlock. Loops writing only |
| 2089 | // global/thread-private memory, or with no subsequent reader, get none. |
| 2090 | if (mayWriteSharedMemory(loopOp) && hasSubsequentLoopSibling(loopOp)) |
| 2091 | emitGPUBarrierWorkgroup(builder&: rewriter, loc: loopOp->getLoc()); |
| 2092 | return; |
| 2093 | } |
| 2094 | |
| 2095 | mlir::acc::GPUParallelDimsAttr wsParDims = mlir::acc::getParDimsAttr(op: wsLoop); |
| 2096 | bool parentIsSeq = |
| 2097 | wsParDims.getArray().size() == 1 && wsParDims.getArray().front().isSeq(); |
| 2098 | |
| 2099 | if (parentIsSeq) { |
| 2100 | // loopOp is nested inside a sequential parent loop. |
| 2101 | // When the loop contains thread-level sub-loops, |
| 2102 | // multiple threads collaborate on shared (block-private) memory |
| 2103 | // within each iteration. Insert a block-level barrier. |
| 2104 | bool hasThreadSubLoop = false; |
| 2105 | loopOp->walk(callback: [&](scf::ParallelOp innerPar) -> WalkResult { |
| 2106 | if (innerPar.getOperation() == loopOp) |
| 2107 | return WalkResult::advance(); |
| 2108 | mlir::acc::GPUParallelDimsAttr dims = mlir::acc::getParDimsAttr(op: innerPar); |
| 2109 | for (auto d : dims.getArray()) { |
| 2110 | if (d.isThreadX() || d.isThreadY()) { |
| 2111 | hasThreadSubLoop = true; |
| 2112 | return WalkResult::interrupt(); |
| 2113 | } |
| 2114 | } |
| 2115 | return WalkResult::advance(); |
| 2116 | }); |
| 2117 | if (!hasThreadSubLoop) |
| 2118 | return; |
| 2119 | scf::ParallelOp threadLoop = getParentParallel(wsLoop); |
| 2120 | if (!threadLoop) |
| 2121 | return; |
| 2122 | scf::ParallelOp blockLoop = getParentParallel(threadLoop); |
| 2123 | if (!blockLoop) |
| 2124 | return; |
| 2125 | mlir::acc::GPUParallelDimsAttr parDimsAttr = |
| 2126 | mlir::acc::getParDimsAttr(op: blockLoop); |
| 2127 | if (parDimsAttr.hasOnlyBlockLevel()) |
| 2128 | createBarrier(loc: loopOp->getLoc(), parDimsAttr); |
| 2129 | return; |
| 2130 | } |
| 2131 | |
| 2132 | // Parent is a non-sequential parallel loop. Walk up to find the block-level |
| 2133 | // ancestor and insert a barrier there. |
| 2134 | scf::ParallelOp seqLoop = getParentParallel(wsLoop); |
| 2135 | if (!seqLoop) { |
| 2136 | // wsLoop is a worksharing loop directly under the compute region with no |
| 2137 | // gang ancestor: a gang-redundant init loop (e.g. a thread-level loop that |
| 2138 | // every gang runs to fill its own copy of gang-private shared memory). If |
| 2139 | // it writes gang-private (block-only) memory that a later sibling loop |
| 2140 | // reads, insert a workgroup barrier between them. The barrier sits just |
| 2141 | // after the top-level loop where all threads converge, so it cannot |
| 2142 | // deadlock. |
| 2143 | if (mayWriteSharedMemory(loopOp) && hasSubsequentLoopSibling(loopOp: wsLoop)) |
| 2144 | emitGPUBarrierWorkgroup(builder&: rewriter, loc: loopOp->getLoc()); |
| 2145 | return; |
| 2146 | } |
| 2147 | if (scf::ParallelOp outerParLoop = getParentParallel(seqLoop)) { |
| 2148 | mlir::acc::GPUParallelDimsAttr parDimsAttr = |
| 2149 | mlir::acc::getParDimsAttr(op: outerParLoop); |
| 2150 | if (parDimsAttr.hasOnlyBlockLevel()) { |
| 2151 | createBarrier(loc: loopOp->getLoc(), parDimsAttr); |
| 2152 | } else if (parDimsAttr.hasOnlyThreadYLevel()) { |
| 2153 | createPerRowBarrier(loc: loopOp->getLoc()); |
| 2154 | } else if (parDimsAttr && parDimsAttr.isSeq()) { |
| 2155 | // outerParLoop is a sequential grid-stride remainder of a partitioned |
| 2156 | // gang loop, not the gang. Walk past the remainder(s) to the block-level |
| 2157 | // gang and barrier there. |
| 2158 | for (scf::ParallelOp gangLoop = getParentParallel(outerParLoop); gangLoop; |
| 2159 | gangLoop = getParentParallel(gangLoop)) { |
| 2160 | mlir::acc::GPUParallelDimsAttr gangDims = |
| 2161 | mlir::acc::getParDimsAttr(op: gangLoop); |
| 2162 | if (gangDims.hasOnlyBlockLevel()) { |
| 2163 | createBarrier(loc: loopOp->getLoc(), parDimsAttr: gangDims); |
| 2164 | break; |
| 2165 | } |
| 2166 | if (!gangDims.isSeq()) |
| 2167 | break; |
| 2168 | } |
| 2169 | } |
| 2170 | return; |
| 2171 | } |
| 2172 | |
| 2173 | // No block-level ancestor above seqLoop: the gang loop is seqLoop itself, a |
| 2174 | // gang(+vector) loop directly under the compute region. The fixed-depth |
| 2175 | // lookup above misses it (issue: a gang+vector init loop that writes |
| 2176 | // gang-private shared memory must be followed by a barrier before another |
| 2177 | // loop reads it). Only needed when the loop writes such shared memory; loops |
| 2178 | // writing global/thread-private memory (e.g. a plain gang-vector saxpy) do |
| 2179 | // not need one here. |
| 2180 | mlir::acc::GPUParallelDimsAttr parDimsAttr = |
| 2181 | mlir::acc::getParDimsAttr(op: seqLoop); |
| 2182 | if (parDimsAttr.hasOnlyBlockLevel() && mayWriteSharedMemory(loopOp)) { |
| 2183 | createBarrier(loc: loopOp->getLoc(), parDimsAttr); |
| 2184 | } |
| 2185 | } |
| 2186 | |
| 2187 | bool ACCCGToGPULowering::mayWriteSharedMemory(Operation *loopOp) { |
| 2188 | bool found = false; |
| 2189 | loopOp->walk(callback: [&](memref::StoreOp storeOp) { |
| 2190 | // Trace the store target back to its backing acc.private_local through |
| 2191 | // view/cast/box ops, then check the privatization is gang-level. |
| 2192 | llvm::SmallVector<Value, 8> worklist{storeOp.getMemref()}; |
| 2193 | llvm::SmallPtrSet<Value, 8> seen; |
| 2194 | while (!worklist.empty()) { |
| 2195 | Value v = worklist.pop_back_val(); |
| 2196 | if (!seen.insert(Ptr: v).second) |
| 2197 | continue; |
| 2198 | Operation *def = v.getDefiningOp(); |
| 2199 | if (!def) |
| 2200 | continue; |
| 2201 | if (acc::PrivateLocalOp privateLocal = |
| 2202 | dyn_cast<acc::PrivateLocalOp>(Val: def)) { |
| 2203 | acc::PrivatizeOp privatizeOp = |
| 2204 | getPrivatizeOp(privateLocal, computeRegion); |
| 2205 | // Only gang-level (block, no thread) private memory is a single copy |
| 2206 | // shared across the workgroup's threads, so a write needs a workgroup |
| 2207 | // barrier before another thread reads it. A [block_x, thread_x] |
| 2208 | // privatization is thread-private (one copy per thread); barriering on |
| 2209 | // it would deadlock when threads take different grid-stride iteration |
| 2210 | // counts. |
| 2211 | if (mlir::acc::GPUParallelDimsAttr parDims = |
| 2212 | privatizeOp.getParDimsAttr()) { |
| 2213 | bool hasBlock = false, hasThread = false; |
| 2214 | for (mlir::acc::GPUParallelDimAttr d : parDims.getArray()) { |
| 2215 | if (d.isAnyBlock()) |
| 2216 | hasBlock = true; |
| 2217 | if (d.isThreadX() || d.isThreadY()) |
| 2218 | hasThread = true; |
| 2219 | } |
| 2220 | if (hasBlock && !hasThread) { |
| 2221 | found = true; |
| 2222 | return WalkResult::interrupt(); |
| 2223 | } |
| 2224 | } |
| 2225 | continue; |
| 2226 | } |
| 2227 | worklist.append(in_start: def->getOperands().begin(), in_end: def->getOperands().end()); |
| 2228 | } |
| 2229 | return WalkResult::advance(); |
| 2230 | }); |
| 2231 | return found; |
| 2232 | } |
| 2233 | |
| 2234 | /// True when \p memref is storage allocated inside \p scope, looking through |
| 2235 | /// effect-free view and conversion ops. Such storage is created by the thread |
| 2236 | /// that runs the code, so no other thread sees it. |
| 2237 | static bool isAllocatedInside(Value memref, Operation *scope) { |
| 2238 | Value v = memref; |
| 2239 | while (Operation *def = v.getDefiningOp()) { |
| 2240 | if (!scope->isAncestor(other: def)) |
| 2241 | return false; |
| 2242 | if (auto effectOp = dyn_cast<MemoryEffectOpInterface>(Val: def)) |
| 2243 | if (effectOp.getEffectOnValue<MemoryEffects::Allocate>(value: v)) |
| 2244 | return true; |
| 2245 | if (!isMemoryEffectFree(op: def) || def->getNumOperands() == 0) |
| 2246 | return false; |
| 2247 | if (ViewLikeOpInterface viewLike = dyn_cast<ViewLikeOpInterface>(Val: def)) |
| 2248 | v = viewLike.getViewSource(); |
| 2249 | else |
| 2250 | v = def->getOperand(idx: 0); |
| 2251 | } |
| 2252 | return false; |
| 2253 | } |
| 2254 | |
| 2255 | /// True when \p op is a loop whose iterations run one after the other and are |
| 2256 | /// repeated by every thread that reaches it. |
| 2257 | static bool isRedundantSequentialLoop(Operation *op) { |
| 2258 | if (isa<scf::ForOp, scf::WhileOp>(Val: op)) |
| 2259 | return true; |
| 2260 | scf::ParallelOp par = dyn_cast<scf::ParallelOp>(Val: op); |
| 2261 | if (!par) |
| 2262 | return false; |
| 2263 | mlir::acc::GPUParallelDimsAttr dims = mlir::acc::getParDimsAttr(op: par); |
| 2264 | return dims && !dims.getArray().empty() && |
| 2265 | llvm::all_of(Range: dims.getArray(), P: [](mlir::acc::GPUParallelDimAttr d) { |
| 2266 | return d.isSeq(); |
| 2267 | }); |
| 2268 | } |
| 2269 | |
| 2270 | /// The value that names the storage \p memref refers to, found by looking |
| 2271 | /// through effect-free view and conversion ops, or null when the storage cannot |
| 2272 | /// be identified. Two accesses with different non-null roots refer to |
| 2273 | /// different storage. The arguments of a sequential loop are not identified: |
| 2274 | /// they can name different storage in different iterations (a buffer swap). |
| 2275 | /// An input of the compute region is traced into the value it is passed from, |
| 2276 | /// so that inputs that are views of the same storage have the same root. |
| 2277 | static Value getStorageRoot(Value memref) { |
| 2278 | // Ops such as fir.declare only have effects on resources that are not |
| 2279 | // memory (e.g. debug info), so they do not hide the storage behind them. |
| 2280 | auto onlyNonMemoryEffects = [](Operation *op) { |
| 2281 | MemoryEffectOpInterface effectOp = dyn_cast<MemoryEffectOpInterface>(Val: op); |
| 2282 | if (!effectOp) |
| 2283 | return false; |
| 2284 | SmallVector<MemoryEffects::EffectInstance> effects; |
| 2285 | effectOp.getEffects(effects); |
| 2286 | return llvm::all_of(Range&: effects, P: [](const MemoryEffects::EffectInstance &e) { |
| 2287 | return e.getResource() != SideEffects::DefaultResource::get(); |
| 2288 | }); |
| 2289 | }; |
| 2290 | Value v = memref; |
| 2291 | while (true) { |
| 2292 | Operation *def = v.getDefiningOp(); |
| 2293 | if (!def) { |
| 2294 | Operation *owner = cast<BlockArgument>(Val&: v).getOwner()->getParentOp(); |
| 2295 | if (isa<scf::ForOp, scf::WhileOp>(Val: owner)) |
| 2296 | return Value(); |
| 2297 | // The region cannot see where its inputs come from, but the values it is |
| 2298 | // passed may be views of the same storage. Keep the argument as the root |
| 2299 | // when that storage cannot be identified. |
| 2300 | if (auto computeOp = dyn_cast<acc::ComputeRegionOp>(Val: owner)) |
| 2301 | if (Value operand = computeOp.getOperand(blockArg: cast<BlockArgument>(Val&: v))) |
| 2302 | if (Value outerRoot = getStorageRoot(memref: operand)) |
| 2303 | return outerRoot; |
| 2304 | return v; |
| 2305 | } |
| 2306 | if (auto effectOp = dyn_cast<MemoryEffectOpInterface>(Val: def)) |
| 2307 | if (effectOp.getEffectOnValue<MemoryEffects::Allocate>(value: v)) |
| 2308 | return v; |
| 2309 | if ((!isMemoryEffectFree(op: def) && !onlyNonMemoryEffects(def)) || |
| 2310 | def->getNumOperands() == 0) |
| 2311 | return Value(); |
| 2312 | if (ViewLikeOpInterface viewLike = dyn_cast<ViewLikeOpInterface>(Val: def)) |
| 2313 | v = viewLike.getViewSource(); |
| 2314 | else |
| 2315 | v = def->getOperand(idx: 0); |
| 2316 | } |
| 2317 | } |
| 2318 | |
| 2319 | bool ACCCGToGPULowering::mayAccessSharedMemory(Operation *root, bool reads, |
| 2320 | bool writes, bool workerLevel, |
| 2321 | Operation *localScope, |
| 2322 | Operation *ignore, |
| 2323 | SharedStorage *collect, |
| 2324 | const SharedStorage *filter) { |
| 2325 | bool found = false; |
| 2326 | // Reports a shared access to \p target (null when unknown). Without |
| 2327 | // \p collect the walk stops here; with it the storage is recorded and the |
| 2328 | // walk goes on. |
| 2329 | auto sharedAccess = [&](Value target) { |
| 2330 | Value storage = target ? getStorageRoot(memref: target) : Value(); |
| 2331 | if (filter && storage && !filter->unknown && |
| 2332 | !filter->roots.contains(Ptr: storage)) |
| 2333 | return WalkResult::advance(); |
| 2334 | found = true; |
| 2335 | if (!collect) |
| 2336 | return WalkResult::interrupt(); |
| 2337 | if (storage) |
| 2338 | collect->roots.insert(Ptr: storage); |
| 2339 | else |
| 2340 | collect->unknown = true; |
| 2341 | return WalkResult::advance(); |
| 2342 | }; |
| 2343 | root->walk(callback: [&](Operation *op) { |
| 2344 | if (op->hasTrait<OpTrait::IsTerminator>()) |
| 2345 | return WalkResult::advance(); |
| 2346 | if (ignore && ignore->isAncestor(other: op)) |
| 2347 | return WalkResult::advance(); |
| 2348 | // Reductions combine their partial results with their own synchronization. |
| 2349 | if (isa<acc::ReductionAccumulateOp, acc::ReductionCombineOp, |
| 2350 | acc::ReductionCombineRegionOp>(Val: op)) |
| 2351 | return WalkResult::advance(); |
| 2352 | MemoryEffectOpInterface effectOp = dyn_cast<MemoryEffectOpInterface>(Val: op); |
| 2353 | if (!effectOp) { |
| 2354 | // Ops that only contain other ops are covered by the walk. Anything else |
| 2355 | // with unknown effects (e.g. a call) may access shared memory. |
| 2356 | if (op->hasTrait<OpTrait::HasRecursiveMemoryEffects>()) |
| 2357 | return WalkResult::advance(); |
| 2358 | return sharedAccess(Value()); |
| 2359 | } |
| 2360 | SmallVector<MemoryEffects::EffectInstance> effects; |
| 2361 | effectOp.getEffects(effects); |
| 2362 | for (const MemoryEffects::EffectInstance &effect : effects) { |
| 2363 | bool relevant = |
| 2364 | (writes && isa<MemoryEffects::Write>(Val: effect.getEffect())) || |
| 2365 | (reads && isa<MemoryEffects::Read>(Val: effect.getEffect())); |
| 2366 | if (!relevant) |
| 2367 | continue; |
| 2368 | Value target = effect.getValue(); |
| 2369 | if (target) { |
| 2370 | PrivateMemScope scope = getPrivateScopeForMemref(memref: target); |
| 2371 | if (scope == PrivateMemScope::Thread || |
| 2372 | (workerLevel && scope == PrivateMemScope::Worker)) |
| 2373 | continue; |
| 2374 | if (localScope && isAllocatedInside(memref: target, scope: localScope)) |
| 2375 | continue; |
| 2376 | } |
| 2377 | if (sharedAccess(target).wasInterrupted()) |
| 2378 | return WalkResult::interrupt(); |
| 2379 | } |
| 2380 | return WalkResult::advance(); |
| 2381 | }); |
| 2382 | return found; |
| 2383 | } |
| 2384 | |
| 2385 | bool ACCCGToGPULowering::needsBarrierAfterThreadLoop(scf::ParallelOp loop) { |
| 2386 | mlir::acc::GPUParallelDimsAttr loopDims = mlir::acc::getParDimsAttr(op: loop); |
| 2387 | if (!loopDims || loopDims.getArray().size() != 1) |
| 2388 | return false; |
| 2389 | mlir::acc::GPUParallelDimAttr loopDim = loopDims.getArray().front(); |
| 2390 | if (!loopDim.isThreadX() && !loopDim.isThreadY()) |
| 2391 | return false; |
| 2392 | bool workerLevel = loopDim.isThreadY(); |
| 2393 | |
| 2394 | // Inside a worker loop, createBarrierAfterSeqLoop already adds a per-row |
| 2395 | // barrier after the sequential remainder of each thread loop. Skip, to avoid |
| 2396 | // a repeat. |
| 2397 | for (Operation *parent = loop->getParentOp(); |
| 2398 | parent && parent != computeRegion.getOperation(); |
| 2399 | parent = parent->getParentOp()) { |
| 2400 | scf::ParallelOp parentLoop = dyn_cast<scf::ParallelOp>(Val: parent); |
| 2401 | if (!parentLoop) |
| 2402 | continue; |
| 2403 | mlir::acc::GPUParallelDimsAttr parentDims = |
| 2404 | mlir::acc::getParDimsAttr(op: parentLoop); |
| 2405 | if (!parentDims) |
| 2406 | continue; |
| 2407 | for (mlir::acc::GPUParallelDimAttr dim : parentDims.getArray()) |
| 2408 | if (dim.isThreadY()) |
| 2409 | return false; |
| 2410 | } |
| 2411 | |
| 2412 | // A loop that works on a shared-memory tile made for it (for instance by the |
| 2413 | // cache directive lowering) fills, uses and writes back that tile with its |
| 2414 | // own synchronization. |
| 2415 | bool hasOwnTile = false; |
| 2416 | loop->walk(callback: [&](acc::GPUSharedMemoryOp) { |
| 2417 | hasOwnTile = true; |
| 2418 | return WalkResult::interrupt(); |
| 2419 | }); |
| 2420 | if (hasOwnTile) |
| 2421 | return false; |
| 2422 | |
| 2423 | // Outside a sequential nest the loop is reached once, and what follows it is |
| 2424 | // handled by the predication and loop-level barriers. |
| 2425 | bool inSequentialNest = false; |
| 2426 | for (Operation *parent = loop->getParentOp(); |
| 2427 | parent && parent != computeRegion.getOperation(); |
| 2428 | parent = parent->getParentOp()) { |
| 2429 | if (isa<scf::ForOp, scf::WhileOp>(Val: parent)) { |
| 2430 | inSequentialNest = true; |
| 2431 | break; |
| 2432 | } |
| 2433 | } |
| 2434 | if (!inSequentialNest) |
| 2435 | return false; |
| 2436 | |
| 2437 | // The iterations are spread over the threads, so a thread cannot rely on |
| 2438 | // what another thread did in the loop. Accesses to thread-private memory |
| 2439 | // never conflict. A later read conflicts with a write in the loop (read |
| 2440 | // after write) and a later write conflicts with a read (write after read) or |
| 2441 | // a write (write after write) in the loop, when both may refer to the same |
| 2442 | // storage. |
| 2443 | SharedStorage written, touched; |
| 2444 | mayAccessSharedMemory(root: loop, /*reads=*/false, /*writes=*/true, workerLevel, |
| 2445 | /*localScope=*/loop, /*ignore=*/nullptr, collect: &written); |
| 2446 | mayAccessSharedMemory(root: loop, /*reads=*/true, /*writes=*/false, workerLevel, |
| 2447 | /*localScope=*/loop, /*ignore=*/nullptr, collect: &touched); |
| 2448 | touched.insert(other: written); |
| 2449 | if (touched.empty()) |
| 2450 | return false; |
| 2451 | bool loopWrites = !written.empty(); |
| 2452 | |
| 2453 | // True when \p op, executed after the loop, conflicts with it. |
| 2454 | auto conflictsWithLoop = [&](Operation *op, Operation *localScope, |
| 2455 | Operation *ignore) { |
| 2456 | if (loopWrites && mayAccessSharedMemory( |
| 2457 | root: op, /*reads=*/true, /*writes=*/false, workerLevel, |
| 2458 | localScope, ignore, /*collect=*/nullptr, filter: &written)) |
| 2459 | return true; |
| 2460 | return mayAccessSharedMemory(root: op, /*reads=*/false, /*writes=*/true, |
| 2461 | workerLevel, localScope, ignore, |
| 2462 | /*collect=*/nullptr, filter: &touched); |
| 2463 | }; |
| 2464 | |
| 2465 | // Look for a conflicting access after the loop: later ops of every enclosing |
| 2466 | // block, and, for each enclosing loop, the ops in its body up to and |
| 2467 | // including the one that contains the loop, which run again in the next |
| 2468 | // iteration. |
| 2469 | for (Operation *cur = loop; cur && cur != computeRegion.getOperation(); |
| 2470 | cur = cur->getParentOp()) { |
| 2471 | for (Operation *next = cur->getNextNode(); next; next = next->getNextNode()) |
| 2472 | if (conflictsWithLoop(next, /*localScope=*/nullptr, /*ignore=*/nullptr)) |
| 2473 | return true; |
| 2474 | Operation *parent = cur->getParentOp(); |
| 2475 | if (!parent || parent == computeRegion.getOperation()) |
| 2476 | break; |
| 2477 | if (!isRedundantSequentialLoop(op: parent)) |
| 2478 | continue; |
| 2479 | // The condition and the other region of a while loop are not scanned. |
| 2480 | if (isa<scf::WhileOp>(Val: parent)) |
| 2481 | return true; |
| 2482 | // Scan the body up to and including cur. Accesses inside cur itself, which |
| 2483 | // contains the loop, also run again in the next iteration. Storage |
| 2484 | // allocated inside cur is private to it. |
| 2485 | for (Operation *prev = &cur->getBlock()->front(); prev; |
| 2486 | prev = prev->getNextNode()) { |
| 2487 | if (prev != cur) { |
| 2488 | if (conflictsWithLoop(prev, /*localScope=*/nullptr, /*ignore=*/nullptr)) |
| 2489 | return true; |
| 2490 | continue; |
| 2491 | } |
| 2492 | // The loop reads again in the next iteration what other threads wrote in |
| 2493 | // this one (e.g. a double-buffered stencil whose only body op is the |
| 2494 | // loop). Its own writes are not treated as conflicting with its writes in |
| 2495 | // the next iteration: the loop is assumed to store the same elements from |
| 2496 | // the same threads each time. |
| 2497 | if (loopWrites && mayAccessSharedMemory( |
| 2498 | root: loop, /*reads=*/true, /*writes=*/false, workerLevel, |
| 2499 | /*localScope=*/cur, |
| 2500 | /*ignore=*/nullptr, /*collect=*/nullptr, filter: &written)) |
| 2501 | return true; |
| 2502 | // The other ops in cur run again too. |
| 2503 | if (cur != loop && conflictsWithLoop(cur, /*localScope=*/cur, loop)) |
| 2504 | return true; |
| 2505 | break; |
| 2506 | } |
| 2507 | } |
| 2508 | return false; |
| 2509 | } |
| 2510 | |
| 2511 | PrivateMemScope |
| 2512 | ACCCGToGPULowering::getPrivateMemScope(acc::PrivatizeOp privatizeOp) { |
| 2513 | bool hasBlock = false; |
| 2514 | bool hasThreadX = false; |
| 2515 | bool hasThreadY = false; |
| 2516 | if (mlir::acc::GPUParallelDimsAttr parDims = privatizeOp.getParDimsAttr()) { |
| 2517 | for (mlir::acc::GPUParallelDimAttr d : parDims.getArray()) { |
| 2518 | if (d.isAnyBlock()) |
| 2519 | hasBlock = true; |
| 2520 | if (d.isThreadX()) |
| 2521 | hasThreadX = true; |
| 2522 | if (d.isThreadY()) |
| 2523 | hasThreadY = true; |
| 2524 | } |
| 2525 | } else { |
| 2526 | for (mlir::acc::GPUParallelDimAttr d : computeRegion.getLaunchParDims()) |
| 2527 | if (d.isAnyBlock()) |
| 2528 | hasBlock = true; |
| 2529 | if (hasBlock) |
| 2530 | return PrivateMemScope::Gang; |
| 2531 | return PrivateMemScope::Thread; |
| 2532 | } |
| 2533 | if (hasThreadX) |
| 2534 | return PrivateMemScope::Thread; |
| 2535 | if (hasBlock && hasThreadY) |
| 2536 | return PrivateMemScope::Worker; |
| 2537 | if (hasBlock) |
| 2538 | return PrivateMemScope::Gang; |
| 2539 | return PrivateMemScope::Thread; |
| 2540 | } |
| 2541 | |
| 2542 | /// Walks back from a memref use to its defining `acc.private_local`, if any. |
| 2543 | static acc::PrivateLocalOp getPrivateLocalForMemref(Value memref) { |
| 2544 | llvm::SmallVector<Value, 8> worklist{memref}; |
| 2545 | llvm::SmallPtrSet<Value, 8> seen; |
| 2546 | while (!worklist.empty()) { |
| 2547 | Value v = worklist.pop_back_val(); |
| 2548 | if (!seen.insert(Ptr: v).second) |
| 2549 | continue; |
| 2550 | Operation *def = v.getDefiningOp(); |
| 2551 | if (!def) |
| 2552 | continue; |
| 2553 | if (acc::PrivateLocalOp privateLocal = dyn_cast<acc::PrivateLocalOp>(Val: def)) |
| 2554 | return privateLocal; |
| 2555 | worklist.append(in_start: def->getOperands().begin(), in_end: def->getOperands().end()); |
| 2556 | } |
| 2557 | return nullptr; |
| 2558 | } |
| 2559 | |
| 2560 | PrivateMemScope ACCCGToGPULowering::getPrivateScopeForMemref(Value memref) { |
| 2561 | if (auto privateLocal = getPrivateLocalForMemref(memref)) |
| 2562 | return getPrivateMemScope(privatizeOp: getPrivatizeOp(privateLocal, computeRegion)); |
| 2563 | return PrivateMemScope::None; |
| 2564 | } |
| 2565 | |
| 2566 | acc::PrivatizeOp ACCCGToGPULowering::getPrivatizeForMemref(Value memref) { |
| 2567 | if (auto privateLocal = getPrivateLocalForMemref(memref)) |
| 2568 | return getPrivatizeOp(privateLocal, computeRegion); |
| 2569 | return acc::PrivatizeOp(); |
| 2570 | } |
| 2571 | |
| 2572 | void ACCCGToGPULowering::collectSharedPrivateStores( |
| 2573 | Operation *root, llvm::SmallPtrSetImpl<Operation *> &gangPrivatizes, |
| 2574 | llvm::SmallPtrSetImpl<Operation *> &workerPrivatizes) { |
| 2575 | root->walk(callback: [&](memref::StoreOp storeOp) { |
| 2576 | PrivateMemScope scope = getPrivateScopeForMemref(memref: storeOp.getMemref()); |
| 2577 | if (scope != PrivateMemScope::Gang && scope != PrivateMemScope::Worker) |
| 2578 | return; |
| 2579 | acc::PrivatizeOp privatize = getPrivatizeForMemref(memref: storeOp.getMemref()); |
| 2580 | if (!privatize) |
| 2581 | return; |
| 2582 | if (scope == PrivateMemScope::Gang) |
| 2583 | gangPrivatizes.insert(Ptr: privatize.getOperation()); |
| 2584 | else |
| 2585 | workerPrivatizes.insert(Ptr: privatize.getOperation()); |
| 2586 | }); |
| 2587 | } |
| 2588 | |
| 2589 | bool ACCCGToGPULowering::isMatchingPrivateMemref( |
| 2590 | Value memref, PrivateMemScope storeScope, |
| 2591 | const llvm::SmallPtrSetImpl<Operation *> &storePrivatizes) { |
| 2592 | if (getPrivateScopeForMemref(memref) != storeScope) |
| 2593 | return false; |
| 2594 | acc::PrivatizeOp usePrivatize = getPrivatizeForMemref(memref); |
| 2595 | return usePrivatize && storePrivatizes.contains(Ptr: usePrivatize.getOperation()); |
| 2596 | } |
| 2597 | |
| 2598 | bool ACCCGToGPULowering::hasParallelPrivateReuse( |
| 2599 | LoopLikeOpInterface seqLoopOp, Operation *skipOp, |
| 2600 | PrivateMemScope storeScope, |
| 2601 | const llvm::SmallPtrSetImpl<Operation *> &storePrivatizes) { |
| 2602 | if (storePrivatizes.empty()) |
| 2603 | return false; |
| 2604 | |
| 2605 | bool hasParallelPrivateUse = false; |
| 2606 | seqLoopOp.getOperation()->walk(callback: [&](Operation *op) { |
| 2607 | // Ignore loads inside the predicate. |
| 2608 | if (skipOp->isAncestor(other: op)) |
| 2609 | return WalkResult::advance(); |
| 2610 | |
| 2611 | Value memref; |
| 2612 | if (memref::LoadOp loadOp = dyn_cast<memref::LoadOp>(Val: op)) |
| 2613 | memref = loadOp.getMemref(); |
| 2614 | else if (memref::StoreOp storeOp = dyn_cast<memref::StoreOp>(Val: op)) |
| 2615 | memref = storeOp.getMemref(); |
| 2616 | else |
| 2617 | return WalkResult::advance(); |
| 2618 | |
| 2619 | if (!isMatchingPrivateMemref(memref, storeScope, storePrivatizes)) |
| 2620 | return WalkResult::advance(); |
| 2621 | |
| 2622 | bool insideNestedParallel = false; |
| 2623 | for (Operation *p = op->getParentOp(); p && p != seqLoopOp.getOperation(); |
| 2624 | p = p->getParentOp()) { |
| 2625 | if (scf::ParallelOp par = dyn_cast<scf::ParallelOp>(Val: p)) { |
| 2626 | if (mlir::acc::GPUParallelDimsAttr pd = |
| 2627 | mlir::acc::getParDimsAttr(op: par)) { |
| 2628 | if (llvm::any_of(Range: pd.getArray(), P: [](mlir::acc::GPUParallelDimAttr d) { |
| 2629 | return !d.isSeq(); |
| 2630 | })) { |
| 2631 | insideNestedParallel = true; |
| 2632 | break; |
| 2633 | } |
| 2634 | } |
| 2635 | } |
| 2636 | } |
| 2637 | if (!insideNestedParallel) |
| 2638 | return WalkResult::advance(); |
| 2639 | // An inner parallel region uses the same private memory. |
| 2640 | hasParallelPrivateUse = true; |
| 2641 | return WalkResult::interrupt(); |
| 2642 | }); |
| 2643 | |
| 2644 | return hasParallelPrivateUse; |
| 2645 | } |
| 2646 | |
| 2647 | PrivateMemScope |
| 2648 | ACCCGToGPULowering::needsPreStoreReuseBarrier(acc::PredicateRegionOp interOp) { |
| 2649 | |
| 2650 | // Check if we need a pre-predicate barrier first. |
| 2651 | // Next, check if the barrier should be gang- or worker-level. |
| 2652 | LoopLikeOpInterface seqLoopOp = findFirstSequentialLoop(op: interOp); |
| 2653 | if (!seqLoopOp) |
| 2654 | return PrivateMemScope::None; |
| 2655 | |
| 2656 | // Check if any op in the predicate region stores to gang- or worker-private |
| 2657 | // memory. If not, no barrier is needed. |
| 2658 | llvm::SmallPtrSet<Operation *, 4> gangPrivatizes, workerPrivatizes; |
| 2659 | collectSharedPrivateStores(root: interOp, gangPrivatizes, workerPrivatizes); |
| 2660 | |
| 2661 | // A reused gang slot needs the workgroup barrier, which also reconverges the |
| 2662 | // worker rows, so look for that hazard first. A gang store whose slot is |
| 2663 | // never read back does not stand in for a worker one: fall through and check |
| 2664 | // the worker slots on their own, otherwise their per-row barrier is lost. |
| 2665 | if (hasParallelPrivateReuse(seqLoopOp, skipOp: interOp, storeScope: PrivateMemScope::Gang, |
| 2666 | storePrivatizes: gangPrivatizes)) |
| 2667 | return PrivateMemScope::Gang; |
| 2668 | if (hasParallelPrivateReuse(seqLoopOp, skipOp: interOp, storeScope: PrivateMemScope::Worker, |
| 2669 | storePrivatizes: workerPrivatizes)) |
| 2670 | return PrivateMemScope::Worker; |
| 2671 | return PrivateMemScope::None; |
| 2672 | } |
| 2673 | |
| 2674 | bool ACCCGToGPULowering::needsInLoopReuseBarrier(Operation *loopOp) { |
| 2675 | // A sequential scf.parallel is the remainder of a partitioned gang loop: the |
| 2676 | // gang iterations this workgroup still runs as a grid stride. Its scalar |
| 2677 | // stores stay predicated and already reconverge between iterations, so the |
| 2678 | // reuse hole is the nested scf.for coming from `acc loop seq`. |
| 2679 | if (scf::ParallelOp par = dyn_cast<scf::ParallelOp>(Val: loopOp)) |
| 2680 | return false; |
| 2681 | |
| 2682 | // The hazard needs a worker- or vector-level routine call in the body: every |
| 2683 | // workgroup thread reaches such a call and the callee synchronizes |
| 2684 | // internally, so threads leave it in no particular order and can then be |
| 2685 | // running different iterations of this loop at the same time. |
| 2686 | if (!hasThreadLevelRoutineCall) |
| 2687 | return false; |
| 2688 | bool callsThreadLevelRoutine = false; |
| 2689 | loopOp->walk(callback: [&](CallOpInterface callOp) -> WalkResult { |
| 2690 | if (mlir::acc::GPUParallelDimAttr parDim = |
| 2691 | getAccRoutineCallParDim(callOp, policy: defaultPolicy)) { |
| 2692 | if (parDim.isThreadX() || parDim.isThreadY()) { |
| 2693 | callsThreadLevelRoutine = true; |
| 2694 | return WalkResult::interrupt(); |
| 2695 | } |
| 2696 | } |
| 2697 | return WalkResult::advance(); |
| 2698 | }); |
| 2699 | if (!callsThreadLevelRoutine) |
| 2700 | return false; |
| 2701 | |
| 2702 | // Only gang-level privatizations are a single copy shared by the whole |
| 2703 | // workgroup and reused by every iteration; worker- and thread-private copies |
| 2704 | // cannot be clobbered by another thread. |
| 2705 | llvm::SmallPtrSet<Operation *, 4> storedPrivatizes, workerPrivatizes; |
| 2706 | collectSharedPrivateStores(root: loopOp, gangPrivatizes&: storedPrivatizes, workerPrivatizes); |
| 2707 | if (storedPrivatizes.empty()) |
| 2708 | return false; |
| 2709 | |
| 2710 | // Storing alone is harmless: the next iteration only clobbers a value someone |
| 2711 | // still needs when the slot is read back in the body, either directly or by a |
| 2712 | // callee it is handed to. |
| 2713 | bool reusesGangPrivate = false; |
| 2714 | loopOp->walk(callback: [&](Operation *op) -> WalkResult { |
| 2715 | if (memref::LoadOp loadOp = dyn_cast<memref::LoadOp>(Val: op)) |
| 2716 | reusesGangPrivate = isMatchingPrivateMemref( |
| 2717 | memref: loadOp.getMemref(), storeScope: PrivateMemScope::Gang, storePrivatizes: storedPrivatizes); |
| 2718 | else if (CallOpInterface callOp = dyn_cast<CallOpInterface>(Val: op)) |
| 2719 | reusesGangPrivate = llvm::any_of(Range: callOp.getArgOperands(), P: [&](Value arg) { |
| 2720 | return isMatchingPrivateMemref(memref: arg, storeScope: PrivateMemScope::Gang, |
| 2721 | storePrivatizes: storedPrivatizes); |
| 2722 | }); |
| 2723 | return reusesGangPrivate ? WalkResult::interrupt() : WalkResult::advance(); |
| 2724 | }); |
| 2725 | if (!reusesGangPrivate) |
| 2726 | return false; |
| 2727 | |
| 2728 | // A barrier inside the body is only safe when the whole workgroup runs every |
| 2729 | // iteration, so bail out under any thread-level worksharing loop, whose trip |
| 2730 | // count varies per thread. |
| 2731 | for (Operation *p = loopOp->getParentOp(); |
| 2732 | p && p != computeRegion.getOperation(); p = p->getParentOp()) { |
| 2733 | scf::ParallelOp par = dyn_cast<scf::ParallelOp>(Val: p); |
| 2734 | if (!par) |
| 2735 | continue; |
| 2736 | if (mlir::acc::GPUParallelDimsAttr parDims = mlir::acc::getParDimsAttr(op: par)) |
| 2737 | for (mlir::acc::GPUParallelDimAttr d : parDims.getArray()) |
| 2738 | if (d.isThreadX() || d.isThreadY()) |
| 2739 | return false; |
| 2740 | } |
| 2741 | return true; |
| 2742 | } |
| 2743 | |
| 2744 | void ACCCGToGPULowering::processPredicateRegion( |
| 2745 | acc::PredicateRegionOp interOp) { |
| 2746 | LLVM_DEBUG(llvm::dbgs() << "processing predicate region: " ; |
| 2747 | interOp->print(llvm::dbgs()); llvm::dbgs() << "\n" ); |
| 2748 | Location loc = interOp->getLoc(); |
| 2749 | |
| 2750 | std::pair<SmallVector<mlir::acc::GPUParallelDimAttr>, |
| 2751 | SmallVector<mlir::acc::GPUParallelDimAttr>> |
| 2752 | parDimsPair = computeActiveAndInactiveParDims( |
| 2753 | op: interOp, block: &interOp.getRegion().front()); |
| 2754 | if (hasFailed) |
| 2755 | return; |
| 2756 | |
| 2757 | // If ThreadY reduction exists, subgroup alignment is applied |
| 2758 | // (blockDim.x = subgroupSize), so ThreadX lanes exist even without explicit |
| 2759 | // ThreadX parallelism. Add ThreadX to inactiveParDims if not already present. |
| 2760 | // Exception: if this region contains a thread-level (vector or worker) |
| 2761 | // routine call, all ThreadX threads must reach the call so the routine's |
| 2762 | // workgroup-wide barriers (e.g. shared memory alloca sync) converge. |
| 2763 | // A single-thread worker launch is left unaligned, so it has no such lanes. |
| 2764 | if (hasThreadYReduction && !isSingleThreadWorkerLaunch()) { |
| 2765 | MLIRContext *ctx = computeRegion->getContext(); |
| 2766 | mlir::acc::GPUParallelDimAttr threadXParDim = |
| 2767 | mlir::acc::GPUParallelDimAttr::threadXDim(ctx); |
| 2768 | bool hasThreadXInActive = |
| 2769 | llvm::any_of(Range&: parDimsPair.first, P: [](mlir::acc::GPUParallelDimAttr pd) { |
| 2770 | return pd.isThreadX(); |
| 2771 | }); |
| 2772 | bool hasThreadXInInactive = |
| 2773 | llvm::any_of(Range&: parDimsPair.second, P: [](mlir::acc::GPUParallelDimAttr pd) { |
| 2774 | return pd.isThreadX(); |
| 2775 | }); |
| 2776 | |
| 2777 | // Check if THIS predicate region contains a thread-level routine call. |
| 2778 | // We use the pre-computed hasThreadLevelRoutineCall as an early-out |
| 2779 | // optimization. |
| 2780 | bool regionHasThreadLevelRoutineCall = false; |
| 2781 | if (hasThreadLevelRoutineCall) { |
| 2782 | interOp.getRegion().walk(callback: [&](CallOpInterface callOp) { |
| 2783 | if (mlir::acc::GPUParallelDimAttr parDim = |
| 2784 | getAccRoutineCallParDim(callOp, policy: defaultPolicy)) { |
| 2785 | if (parDim.isThreadX() || parDim.isThreadY()) { |
| 2786 | regionHasThreadLevelRoutineCall = true; |
| 2787 | return WalkResult::interrupt(); |
| 2788 | } |
| 2789 | } |
| 2790 | return WalkResult::advance(); |
| 2791 | }); |
| 2792 | } |
| 2793 | |
| 2794 | if (!hasThreadXInActive && !hasThreadXInInactive && |
| 2795 | !regionHasThreadLevelRoutineCall) { |
| 2796 | parDimsPair.second.push_back(Elt: threadXParDim); |
| 2797 | } |
| 2798 | } |
| 2799 | |
| 2800 | Value predicate = emitPredicate(loc, inactiveParDims&: parDimsPair.second); |
| 2801 | // With one thread per block nothing is predicated, but a block-level |
| 2802 | // reduction store below must still become a cross-block atomic. |
| 2803 | if (!predicate && llvm::all_of(Range: computeRegion.getLaunchParDims(), |
| 2804 | P: [](mlir::acc::GPUParallelDimAttr pd) { |
| 2805 | return pd.isAnyBlock(); |
| 2806 | })) |
| 2807 | predicate = arith::ConstantIntOp::create(builder&: rewriter, location: loc, value: 1, width: 1); |
| 2808 | if (predicate) { |
| 2809 | LLVM_DEBUG(llvm::dbgs() << "predicate: " << predicate << "\n" ); |
| 2810 | bool isInsideThreadXLoop = false; |
| 2811 | bool isInsideThreadYLoop = false; |
| 2812 | for (auto parDim : parDimsPair.first) { |
| 2813 | if (parDim.isThreadX()) |
| 2814 | isInsideThreadXLoop = true; |
| 2815 | if (parDim.isThreadY()) |
| 2816 | isInsideThreadYLoop = true; |
| 2817 | } |
| 2818 | // Emits the reconvergence barrier matching this region's predication level, |
| 2819 | // at the current insertion point. Called for both before and after the |
| 2820 | // predicated store. Below is the pre-predicate barrier; the post-predicate |
| 2821 | // barrier is emitted after the ifOp. |
| 2822 | auto emitReconvergenceBarrier = [&]() { |
| 2823 | if (isInsideThreadXLoop) { |
| 2824 | // Inside ThreadX loop - skip barrier |
| 2825 | } else if (isInsideThreadYLoop) { |
| 2826 | // Inside ThreadY loop |
| 2827 | if (!isInsideACCSpecializedRoutine(op: computeRegion)) { |
| 2828 | // Add barrier if ThreadX is predicated (lane 0 writes must be |
| 2829 | // visible to all lanes before they read). |
| 2830 | bool predicatesThreadX = llvm::any_of( |
| 2831 | Range&: parDimsPair.second, |
| 2832 | P: [](mlir::acc::GPUParallelDimAttr pd) { return pd.isThreadX(); }); |
| 2833 | if (predicatesThreadX) |
| 2834 | createPerRowBarrier(loc); |
| 2835 | } |
| 2836 | // For acc routine ThreadY routines, skip barrier |
| 2837 | } else if (!parDimsPair.first.empty()) { |
| 2838 | // Inside block loop |
| 2839 | createBarrier(loc, parDimsAttr: mlir::acc::GPUParallelDimsAttr::get( |
| 2840 | context: interOp->getContext(), array: parDimsPair.first)); |
| 2841 | } else { |
| 2842 | // Top level |
| 2843 | createBarrier(loc, parDimsAttr: mlir::acc::GPUParallelDimsAttr::get( |
| 2844 | context: interOp->getContext(), array: parDimsPair.second)); |
| 2845 | } |
| 2846 | }; |
| 2847 | |
| 2848 | // A gang-level (block, no thread) shared slot written here and reused on |
| 2849 | // the next iteration of an enclosing sequential loop must not be |
| 2850 | // overwritten before all threads read the current value. Emit the |
| 2851 | // reconvergence barrier before the store too (the post-store barrier below |
| 2852 | // only orders this iteration's store->read). Restricted to the block-level |
| 2853 | // case: such loops are run uniformly by every workgroup thread, so the |
| 2854 | // workgroup barrier cannot deadlock; worker/vector (thread_y/thread_x) |
| 2855 | // loops may have divergent trip counts. |
| 2856 | PrivateMemScope scope = needsPreStoreReuseBarrier(interOp); |
| 2857 | if (scope == PrivateMemScope::Gang) |
| 2858 | emitGPUBarrierWorkgroup(builder&: rewriter, loc); |
| 2859 | else if (scope == PrivateMemScope::Worker) |
| 2860 | createPerRowBarrier(loc); |
| 2861 | |
| 2862 | auto ifOp = scf::IfOp::create(builder&: rewriter, location: loc, cond: predicate, |
| 2863 | /*withElseRegion=*/false); |
| 2864 | Region &thenRegion = ifOp.getThenRegion(); |
| 2865 | Block &thenBlock = thenRegion.back(); |
| 2866 | rewriter.setInsertionPoint(thenBlock.getTerminator()); |
| 2867 | // Ops in a predicate region may need to be further processed, recurse |
| 2868 | for (auto &bodyOp : interOp.getRegion().front().getOperations()) { |
| 2869 | // If the store's value loads from a block-level reduction |
| 2870 | // memref, convert to atomic for cross-block correctness. |
| 2871 | // Only at kernel top level (no active thread dims). |
| 2872 | if (memref::StoreOp storeOp = dyn_cast<memref::StoreOp>(Val: &bodyOp)) { |
| 2873 | std::optional<arith::AtomicRMWKind> blockReduceKind; |
| 2874 | bool failedReductionKind = false; |
| 2875 | Value storeVal = storeOp.getValueToStore(); |
| 2876 | if (storeVal.getDefiningOp()) { |
| 2877 | // Walk the epilogue def-chain within the enclosing block to find |
| 2878 | // the block-level accumulate load feeding this store. Epilogue ops |
| 2879 | // (type conversions, arithmetic, etc.) are traversed transparently. |
| 2880 | // Non-acc loads and values defined outside the block are treated as |
| 2881 | // loop-invariant and return nullopt, bounding the search naturally. |
| 2882 | Block *epilogueBlock = interOp->getBlock(); |
| 2883 | auto findBlockAccLoad = |
| 2884 | [&](auto &self, |
| 2885 | Value val) -> std::optional<arith::AtomicRMWKind> { |
| 2886 | Operation *def = val.getDefiningOp(); |
| 2887 | if (!def || def->getBlock() != epilogueBlock) |
| 2888 | return std::nullopt; |
| 2889 | if (memref::LoadOp loadOp = dyn_cast<memref::LoadOp>(Val: def)) { |
| 2890 | for (auto *user : loadOp.getMemRef().getUsers()) { |
| 2891 | if (acc::ReductionAccumulateOp accOp = |
| 2892 | dyn_cast<acc::ReductionAccumulateOp>(Val: user)) { |
| 2893 | if (llvm::any_of(accOp.getParDims().getArray(), |
| 2894 | [](mlir::acc::GPUParallelDimAttr pd) { |
| 2895 | return pd.isAnyBlock(); |
| 2896 | })) { |
| 2897 | FailureOr<arith::AtomicRMWKind> kind = getReductionKind( |
| 2898 | redOp: accOp.getReductionOperator(), |
| 2899 | type: accOp.getValue().getType(), loc: accOp.getLoc()); |
| 2900 | if (failed(Result: kind)) { |
| 2901 | failedReductionKind = true; |
| 2902 | return std::nullopt; |
| 2903 | } |
| 2904 | return *kind; |
| 2905 | } |
| 2906 | } |
| 2907 | } |
| 2908 | return std::nullopt; |
| 2909 | } |
| 2910 | for (Value operand : def->getOperands()) |
| 2911 | if (auto kind = self(self, operand)) |
| 2912 | return kind; |
| 2913 | return std::nullopt; |
| 2914 | }; |
| 2915 | blockReduceKind = findBlockAccLoad(findBlockAccLoad, storeVal); |
| 2916 | } |
| 2917 | if (failedReductionKind) |
| 2918 | return; |
| 2919 | // A private destination has a copy per block (or finer), so no other |
| 2920 | // block writes it and the store must stay a plain store. |
| 2921 | if (blockReduceKind && getPrivateScopeForMemref(memref: storeOp.getMemref()) != |
| 2922 | PrivateMemScope::None) |
| 2923 | blockReduceKind.reset(); |
| 2924 | if (blockReduceKind) { |
| 2925 | Value input = mapping.lookupOrDefault(from: storeOp.getValueToStore()); |
| 2926 | Value memref = mapping.lookupOrDefault(from: storeOp.getMemref()); |
| 2927 | bool threadIsActive = llvm::any_of( |
| 2928 | Range&: parDimsPair.first, P: [](mlir::acc::GPUParallelDimAttr pd) { |
| 2929 | return !pd.isAnyBlock(); |
| 2930 | }); |
| 2931 | if (!threadIsActive && |
| 2932 | !isa_and_nonnull<memref::AllocaOp>( |
| 2933 | Val: unwrapMemRefConversion(v: memref).getDefiningOp())) { |
| 2934 | // Initialize the destination to the reduction identity before |
| 2935 | // cross-block atomics so that the final result reflects pure |
| 2936 | // assignment semantics (e.g. r = sum(a)), not accumulation |
| 2937 | // on top of the pre-kernel value. |
| 2938 | if (launch) { |
| 2939 | MemRefType memrefTy = cast<MemRefType>(Val: memref.getType()); |
| 2940 | // Map the store indices for ranked memrefs. |
| 2941 | SmallVector<Value> initIndices; |
| 2942 | for (Value idx : storeOp.getIndices()) |
| 2943 | initIndices.push_back(Elt: mapping.lookupOrDefault(from: idx)); |
| 2944 | |
| 2945 | OpBuilder::InsertionGuard guard(rewriter); |
| 2946 | Block &launchBody = launch.getBody().front(); |
| 2947 | Operation *insertBefore = nullptr; |
| 2948 | |
| 2949 | launchBody.walk(callback: [&](scf::ParallelOp parOp) -> WalkResult { |
| 2950 | for (Operation *parent = parOp->getParentOp(); parent; |
| 2951 | parent = parent->getParentOp()) { |
| 2952 | if (parent == launch.getOperation()) |
| 2953 | break; |
| 2954 | if (isa<scf::ParallelOp>(Val: parent)) |
| 2955 | return WalkResult::advance(); |
| 2956 | } |
| 2957 | insertBefore = parOp.getOperation(); |
| 2958 | return WalkResult::interrupt(); |
| 2959 | }); |
| 2960 | // The identity must be ordered before the atomics of every |
| 2961 | // block, so store it from a launch ahead of this one when the |
| 2962 | // address does not depend on the launch. |
| 2963 | std::function<bool(Value)> fromLaunch = [&](Value v) -> bool { |
| 2964 | if (auto arg = dyn_cast<BlockArgument>(Val&: v)) |
| 2965 | return launch->isAncestor(other: arg.getOwner()->getParentOp()); |
| 2966 | Operation *def = v.getDefiningOp(); |
| 2967 | if (!launch->isAncestor(other: def) || |
| 2968 | def->hasTrait<OpTrait::ConstantLike>()) |
| 2969 | return false; |
| 2970 | // Ids and dims such as gpu.grid_dim take no operands. |
| 2971 | return def->getNumOperands() == 0 || |
| 2972 | llvm::any_of(Range: def->getOperands(), P: fromLaunch); |
| 2973 | }; |
| 2974 | if (!fromLaunch(memref) && |
| 2975 | llvm::none_of(Range&: initIndices, P: fromLaunch)) { |
| 2976 | rewriter.setInsertionPoint(launch); |
| 2977 | Value one = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1); |
| 2978 | Value token = launch.getAsyncToken(); |
| 2979 | auto initLaunch = gpu::LaunchOp::create( |
| 2980 | builder&: rewriter, location: loc, gridSizeX: one, gridSizeY: one, gridSizeZ: one, blockSizeX: one, blockSizeY: one, blockSizeZ: one, |
| 2981 | /*dynamicSharedMemorySize=*/nullptr, |
| 2982 | asyncTokenType: token ? token.getType() : Type(), |
| 2983 | asyncDependencies: launch.getAsyncDependencies()); |
| 2984 | rewriter.setInsertionPointToStart( |
| 2985 | &initLaunch.getBody().front()); |
| 2986 | rewriter.setInsertionPoint( |
| 2987 | gpu::TerminatorOp::create(builder&: rewriter, location: loc)); |
| 2988 | } else if (insertBefore) |
| 2989 | rewriter.setInsertionPoint(insertBefore); |
| 2990 | else |
| 2991 | rewriter.setInsertionPointToStart(&launchBody); |
| 2992 | // Recursively re-materialize operations whose definitions |
| 2993 | // do not dominate the insertion point. A single-level clone |
| 2994 | // is insufficient when the value is produced by a chain of |
| 2995 | // operations (e.g. reinterpret_cast depending on box_dims, |
| 2996 | // divsi, convert, etc.) that are all defined after the |
| 2997 | // insertion point. |
| 2998 | DominanceInfo domInfo(launch); |
| 2999 | IRMapping initMapping; |
| 3000 | std::function<Value(Value)> materialize = |
| 3001 | [&](Value val) -> Value { |
| 3002 | Operation *defOp = val.getDefiningOp(); |
| 3003 | if (!defOp) |
| 3004 | return val; |
| 3005 | if (domInfo.dominates(a: defOp, b: &*rewriter.getInsertionPoint())) |
| 3006 | return val; |
| 3007 | if (auto mapped = initMapping.lookupOrNull(from: val)) |
| 3008 | return mapped; |
| 3009 | // Recurse on operands; the recursive call seeds |
| 3010 | // initMapping for any operand it clones, which the |
| 3011 | // subsequent rewriter.clone(..., initMapping) picks up. |
| 3012 | for (Value operand : defOp->getOperands()) |
| 3013 | materialize(operand); |
| 3014 | Operation *cloned = rewriter.clone(op&: *defOp, mapper&: initMapping); |
| 3015 | for (auto [orig, clonedRes] : |
| 3016 | llvm::zip(t: defOp->getResults(), u: cloned->getResults())) { |
| 3017 | initMapping.map(from: orig, to: clonedRes); |
| 3018 | } |
| 3019 | return initMapping.lookup(from: val); |
| 3020 | }; |
| 3021 | Value initMemref = materialize(memref); |
| 3022 | for (auto &idx : initIndices) |
| 3023 | idx = materialize(idx); |
| 3024 | Value identityVal = createIdentityValue( |
| 3025 | b&: rewriter, loc, type: memrefTy.getElementType(), kind: *blockReduceKind, |
| 3026 | /*useOnlyFiniteValue=*/true); |
| 3027 | Value blockId = gpu::BlockIdOp::create( |
| 3028 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::x); |
| 3029 | Value threadId = gpu::ThreadIdOp::create( |
| 3030 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::x); |
| 3031 | Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 3032 | Value isBlock0 = arith::CmpIOp::create( |
| 3033 | builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq, lhs: blockId, rhs: zero); |
| 3034 | Value isThread0 = arith::CmpIOp::create( |
| 3035 | builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq, lhs: threadId, rhs: zero); |
| 3036 | Value isFirstThread = |
| 3037 | arith::AndIOp::create(builder&: rewriter, location: loc, lhs: isBlock0, rhs: isThread0); |
| 3038 | auto initIf = scf::IfOp::create(builder&: rewriter, location: loc, cond: isFirstThread, |
| 3039 | /*withElseRegion=*/false); |
| 3040 | rewriter.setInsertionPoint( |
| 3041 | initIf.getThenRegion().back().getTerminator()); |
| 3042 | memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: identityVal, memref: initMemref, |
| 3043 | indices: initIndices); |
| 3044 | rewriter.setInsertionPointAfter(initIf); |
| 3045 | gpu::BarrierOp::create(builder&: rewriter, location: loc); |
| 3046 | } |
| 3047 | SmallVector<Value> atomicIndices; |
| 3048 | for (Value idx : storeOp.getIndices()) |
| 3049 | atomicIndices.push_back(Elt: mapping.lookupOrDefault(from: idx)); |
| 3050 | constructAtomicAccumulation(loc, memref, indices: atomicIndices, input, |
| 3051 | kind: *blockReduceKind); |
| 3052 | continue; |
| 3053 | } |
| 3054 | } |
| 3055 | } |
| 3056 | processOp(op: &bodyOp); |
| 3057 | } |
| 3058 | rewriter.setInsertionPointAfter(ifOp); |
| 3059 | emitReconvergenceBarrier(); |
| 3060 | } else { |
| 3061 | // Ops in a predicate region may need to be further processed, recurse |
| 3062 | for (auto &bodyOp : interOp.getRegion().front().getOperations()) |
| 3063 | processOp(op: &bodyOp); |
| 3064 | } |
| 3065 | } |
| 3066 | |
| 3067 | // clang-format off |
| 3068 | // |
| 3069 | // Allocates private storage and broadcasts its pointer through a shared |
| 3070 | // memref-of-memref slot, using the same predication utility as predicate |
| 3071 | // regions. |
| 3072 | // |
| 3073 | // %0 = arith.cmpi eq, %thread_id_y, %c0 : index |
| 3074 | // %1 = arith.cmpi eq, %thread_id_x, %c0 : index |
| 3075 | // %2 = arith.andi %1, %0 : i1 |
| 3076 | // scf.if %2 { |
| 3077 | // %alloc = memref.alloc() : memref<10xi32> |
| 3078 | // memref.store %alloc, %arg1[] : memref<memref<10xi32>, #gpu.address_space<workgroup>> |
| 3079 | // } |
| 3080 | // gpu.barrier scope<subgroup> |
| 3081 | // %4 = memref.load %arg1[] : memref<memref<10xi32>, #gpu.address_space<workgroup>> |
| 3082 | // |
| 3083 | // clang-format on |
| 3084 | |
| 3085 | Value ACCCGToGPULowering::allocatePrivatizeHeapStorage( |
| 3086 | acc::PrivatizeOp privatize, MemRefType baseTy, |
| 3087 | ValueRange mappedDynamicSizes) { |
| 3088 | Location loc = privatize.getLoc(); |
| 3089 | Value mem = |
| 3090 | memref::AllocOp::create(builder&: rewriter, location: loc, memrefType: baseTy, dynamicSizes: mappedDynamicSizes); |
| 3091 | recordGlobalMemory(privatizeOp: privatize); |
| 3092 | OpBuilder::InsertionGuard guard(rewriter); |
| 3093 | Block *allocBlock = mem.getDefiningOp()->getBlock(); |
| 3094 | if (allocBlock->mightHaveTerminator()) { |
| 3095 | rewriter.setInsertionPoint(allocBlock->getTerminator()); |
| 3096 | memref::DeallocOp::create(builder&: rewriter, location: loc, memref: mem); |
| 3097 | } |
| 3098 | mapping.map(from: privatize.getResult(), to: mem); |
| 3099 | // Ops inside the kernel are rewritten from scratch. If privatize is |
| 3100 | // outside the kernel, it must be replaced with the allocated storage. |
| 3101 | if (!privatize->getParentOfType<acc::ComputeRegionOp>()) |
| 3102 | rewriter.replaceOp(op: privatize, newValues: mem); |
| 3103 | return mem; |
| 3104 | } |
| 3105 | |
| 3106 | Value ACCCGToGPULowering::processPrivatize(acc::PrivatizeOp privatize) { |
| 3107 | LLVM_DEBUG(llvm::dbgs() << "processing privatize: " ; |
| 3108 | privatize->print(llvm::dbgs()); llvm::dbgs() << "\n" ); |
| 3109 | Value tracked = privatize.getResult(); |
| 3110 | if (acc::ComputeRegionOp insUser = |
| 3111 | dyn_cast<acc::ComputeRegionOp>(Val: getOnlyUser(v: tracked))) { |
| 3112 | assert(privatize->hasOneUse() && |
| 3113 | "expected acc.privatize op to have one use" ); |
| 3114 | tracked = insUser.getBody()->getArgument( |
| 3115 | i: privatize->use_begin()->getOperandNumber()); |
| 3116 | } |
| 3117 | Operation *privatizeUser = getOnlyUser(v: tracked); |
| 3118 | assert(privatizeUser && "expected PrivateLocalOp user for privatize" ); |
| 3119 | acc::PrivateLocalOp privateLocalUser = |
| 3120 | dyn_cast<acc::PrivateLocalOp>(Val: privatizeUser); |
| 3121 | |
| 3122 | std::pair<SmallVector<mlir::acc::GPUParallelDimAttr>, |
| 3123 | SmallVector<mlir::acc::GPUParallelDimAttr>> |
| 3124 | parDimsPair = computeActiveAndInactiveParDims(op: privatizeUser, block: nullptr); |
| 3125 | // Set `par_dims` only when this `acc.privatize` does not already carry it. |
| 3126 | if (!privatize.getParDimsAttr()) { |
| 3127 | privatize.setParDimsAttr(mlir::acc::GPUParallelDimsAttr::get( |
| 3128 | context: rewriter.getContext(), array: parDimsPair.first)); |
| 3129 | } |
| 3130 | |
| 3131 | Location loc = privatize->getLoc(); |
| 3132 | acc::PrivateType privTy = cast<acc::PrivateType>(Val: privatize.getType()); |
| 3133 | ModuleOp module = computeRegion->getParentOfType<ModuleOp>(); |
| 3134 | MemRefType baseTy = getPrivateBaseMemRefType(baseTy: privTy.getBaseTy(), module); |
| 3135 | |
| 3136 | gpu::GPUFuncOp gpuFuncOp = computeRegion->getParentOfType<gpu::GPUFuncOp>(); |
| 3137 | // acc.privatize is outside this compute_region (e.g. passed via ins). |
| 3138 | // Leave the op unchanged here; processPrivateLocal materializes |
| 3139 | // storage when acc.private_local is lowered. |
| 3140 | if (!gpuFuncOp && |
| 3141 | privatize->getParentOfType<acc::ComputeRegionOp>() != computeRegion) { |
| 3142 | return privatize.getResult(); |
| 3143 | } |
| 3144 | |
| 3145 | bool threadXActive = |
| 3146 | llvm::any_of(Range&: parDimsPair.first, P: [](mlir::acc::GPUParallelDimAttr parDim) { |
| 3147 | return parDim.isThreadX(); |
| 3148 | }); |
| 3149 | if (threadXActive && |
| 3150 | canUseStackAlloca(baseTy, loc, maxThreadPrivateStack: options.maxThreadPrivateStack)) { |
| 3151 | auto alloca = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType: baseTy); |
| 3152 | recordThreadPrivate(privateLocal: privateLocalUser); |
| 3153 | mapping.map(from: privatize.getResult(), to: alloca.getResult()); |
| 3154 | return alloca.getResult(); |
| 3155 | } |
| 3156 | |
| 3157 | if (!gpuFuncOp) |
| 3158 | return privatize.getResult(); |
| 3159 | |
| 3160 | // When ThreadY is active, shared memory must be indexed by ThreadY ID to |
| 3161 | // avoid races between ThreadY threads. |
| 3162 | // For ThreadX acc routines, assume ThreadY may be active since the routine |
| 3163 | // can be called from a ThreadY loop at runtime. |
| 3164 | // For ThreadY/block acc routines, do not force ThreadY indexing - variables |
| 3165 | // outside the ThreadY loop should be shared across ThreadY threads. |
| 3166 | bool threadYIsActive = |
| 3167 | llvm::any_of(Range&: parDimsPair.first, P: [](mlir::acc::GPUParallelDimAttr parDim) { |
| 3168 | return parDim.isThreadY(); |
| 3169 | }); |
| 3170 | // For routines with multiple ThreadY threads, need a workgroup barrier |
| 3171 | // instead of a subgroup barrier. |
| 3172 | // - ThreadX routines: called independently by different ThreadY threads, need |
| 3173 | // per-ThreadY slots (override threadYIsActive) |
| 3174 | // - ThreadY routines: workgroup barrier, but respect parDimsPair for |
| 3175 | // threadYIsActive |
| 3176 | // (variables before ThreadY loop are shared, inside are per-ThreadY) |
| 3177 | // - Block routines: single call, ThreadY threads cooperate within the |
| 3178 | // routine, so |
| 3179 | // variables at routine level are shared across ThreadY threads (single |
| 3180 | // slot) |
| 3181 | bool needsWorkgroupBarrier = false; |
| 3182 | if (isInsideACCSpecializedRoutine(op: computeRegion)) { |
| 3183 | FunctionOpInterface funcOp = |
| 3184 | computeRegion->getParentOfType<FunctionOpInterface>(); |
| 3185 | mlir::acc::GPUParallelDimAttr routineParDim = |
| 3186 | getSpecializedRoutineDim(funcOp, policy: defaultPolicy); |
| 3187 | if (routineParDim.isThreadX()) { |
| 3188 | // ThreadX routine: always per-ThreadY slots since called from ThreadY |
| 3189 | // loops |
| 3190 | threadYIsActive = true; |
| 3191 | } else if (routineParDim.isThreadY()) { |
| 3192 | // ThreadY routine: workgroup barrier, but keep original threadYIsActive |
| 3193 | // (variables before ThreadY loop shared, inside per-ThreadY) |
| 3194 | needsWorkgroupBarrier = true; |
| 3195 | } else if (routineParDim.isAnyBlock()) { |
| 3196 | needsWorkgroupBarrier = true; |
| 3197 | } |
| 3198 | } |
| 3199 | |
| 3200 | llvm::SmallVector<Value> mappedDynamicSizes; |
| 3201 | for (auto dynamicSize : privatize.getDynamicSizes()) { |
| 3202 | Value mappedDynamicSize = mapping.lookupOrDefault(from: dynamicSize); |
| 3203 | mappedDynamicSizes.push_back(Elt: mappedDynamicSize); |
| 3204 | } |
| 3205 | if (isInsideACCSpecializedRoutine(op: computeRegion) && |
| 3206 | computeRegion.isEffectivelySerial()) { |
| 3207 | if (mappedDynamicSizes.empty()) { |
| 3208 | // Static sizes: use alloca (stack allocation) |
| 3209 | auto alloca = |
| 3210 | memref::AllocaOp::create(builder&: rewriter, location: privatize->getLoc(), memrefType: baseTy); |
| 3211 | recordThreadPrivate(privateLocal: privateLocalUser); |
| 3212 | mapping.map(from: privatize.getResult(), to: alloca.getResult()); |
| 3213 | return alloca.getResult(); |
| 3214 | } |
| 3215 | // Dynamic sizes: use alloc (heap allocation) with dealloc |
| 3216 | return allocatePrivatizeHeapStorage(privatize, baseTy, mappedDynamicSizes); |
| 3217 | } |
| 3218 | |
| 3219 | if (threadXActive) { |
| 3220 | // Reached only after canUseStackAlloca failed: either the static size is |
| 3221 | // over the per-thread stack budget, or the shape is dynamic so the size |
| 3222 | // is unknown. Heap-allocate in both cases so a large runtime extent |
| 3223 | // cannot overflow the stack. Each lane still owns its own copy. |
| 3224 | return allocatePrivatizeHeapStorage(privatize, baseTy, mappedDynamicSizes); |
| 3225 | } |
| 3226 | |
| 3227 | // Predication - when threadYIsActive, don't predicate on ThreadY dimension |
| 3228 | // since each ThreadY needs to execute the allocation for its own slot |
| 3229 | SmallVector<mlir::acc::GPUParallelDimAttr> predicateDims; |
| 3230 | for (auto parDim : parDimsPair.second) { |
| 3231 | // Skip ThreadY if threadYIsActive - each ThreadY needs to allocate |
| 3232 | if (threadYIsActive && parDim.isThreadY()) |
| 3233 | continue; |
| 3234 | predicateDims.push_back(Elt: parDim); |
| 3235 | } |
| 3236 | Value predicate = emitPredicate(loc, inactiveParDims&: predicateDims); |
| 3237 | if (!predicate) { |
| 3238 | predicate = arith::ConstantOp::create( |
| 3239 | builder&: rewriter, location: loc, value: rewriter.getIntegerAttr(type: rewriter.getI1Type(), value: 1)); |
| 3240 | } |
| 3241 | auto ifOp = scf::IfOp::create(builder&: rewriter, location: loc, cond: predicate, |
| 3242 | /*withElseRegion=*/false); |
| 3243 | Region &thenRegion = ifOp.getThenRegion(); |
| 3244 | Block &thenBlock = thenRegion.back(); |
| 3245 | rewriter.setInsertionPoint(thenBlock.getTerminator()); |
| 3246 | auto mem = memref::AllocOp::create(builder&: rewriter, location: privatize->getLoc(), memrefType: baseTy, |
| 3247 | dynamicSizes: mappedDynamicSizes); |
| 3248 | recordGlobalMemory(privatizeOp: privatize); |
| 3249 | // Shared memory allocation |
| 3250 | gpu::AddressSpaceAttr sharedMemoryAddressSpace = gpu::AddressSpaceAttr::get( |
| 3251 | context: computeRegion->getContext(), value: gpu::GPUDialect::getWorkgroupAddressSpace()); |
| 3252 | // When ThreadY is active, create a shared memory array indexed by ThreadY ID. |
| 3253 | // Each ThreadY stores to its own slot; ThreadX lanes within a ThreadY share |
| 3254 | // it. |
| 3255 | constexpr int64_t kMaxThreadY = 32; |
| 3256 | MemRefType sharedMemTy = |
| 3257 | threadYIsActive |
| 3258 | ? MemRefType::get(shape: {kMaxThreadY}, elementType: baseTy, layout: MemRefLayoutAttrInterface{}, |
| 3259 | memorySpace: sharedMemoryAddressSpace) |
| 3260 | : MemRefType::get(shape: {}, elementType: baseTy, layout: MemRefLayoutAttrInterface{}, |
| 3261 | memorySpace: sharedMemoryAddressSpace); |
| 3262 | // The slot only transiently broadcasts the storage pointer, so reuse one per |
| 3263 | // type across privatizes, barriering (before the predicated store) on reuse. |
| 3264 | bool reuseBroadcast = !gpuFuncOp.isKernel(); |
| 3265 | Value alloca; |
| 3266 | llvm::DenseMap<Type, Value>::iterator cachedSlot = |
| 3267 | reuseBroadcast ? privatizeBroadcastCache.find(Val: sharedMemTy) |
| 3268 | : privatizeBroadcastCache.end(); |
| 3269 | if (reuseBroadcast && cachedSlot != privatizeBroadcastCache.end()) { |
| 3270 | alloca = cachedSlot->second; |
| 3271 | OpBuilder::InsertionGuard guard(rewriter); |
| 3272 | rewriter.setInsertionPoint(ifOp); |
| 3273 | mlir::acc::GPUParallelDimAttr dim = |
| 3274 | needsWorkgroupBarrier |
| 3275 | ? mlir::acc::GPUParallelDimAttr::threadYDim(rewriter.getContext()) |
| 3276 | : mlir::acc::GPUParallelDimAttr::threadXDim(rewriter.getContext()); |
| 3277 | createBarrier( |
| 3278 | loc, parDimsAttr: mlir::acc::GPUParallelDimsAttr::get(context: rewriter.getContext(), array: {dim})); |
| 3279 | } else { |
| 3280 | alloca = gpuFuncOp.addWorkgroupAttribution(type: sharedMemTy, |
| 3281 | loc: rewriter.getUnknownLoc()); |
| 3282 | // Setting the alignment to 16 because of a bug in the gpu toolchain. |
| 3283 | // The default alignment is 8, but optimizations create a packed store of 16 |
| 3284 | // bytes which cause a misalignment error at runtime. |
| 3285 | unsigned index = gpuFuncOp.getNumWorkgroupAttributions() - 1; |
| 3286 | gpuFuncOp.setWorkgroupAttributionAttr(index, |
| 3287 | name: LLVM::LLVMDialect::getAlignAttrName(), |
| 3288 | value: rewriter.getI32IntegerAttr(value: 16)); |
| 3289 | if (reuseBroadcast) |
| 3290 | privatizeBroadcastCache[sharedMemTy] = alloca; |
| 3291 | } |
| 3292 | // Store to shared memory, indexed by ThreadY ID when ThreadY is active. |
| 3293 | if (threadYIsActive) { |
| 3294 | Value threadYId = getThreadId(loc, dim: gpu::Dimension::y); |
| 3295 | memref::StoreOp::create(builder&: rewriter, location: privatize->getLoc(), valueToStore: mem, memref: alloca, |
| 3296 | indices: ValueRange{threadYId}); |
| 3297 | } else { |
| 3298 | memref::StoreOp::create(builder&: rewriter, location: privatize->getLoc(), valueToStore: mem, memref: alloca); |
| 3299 | } |
| 3300 | |
| 3301 | // Sync and load - use workgroup barrier for Block/ThreadY routines, |
| 3302 | // ThreadX barrier for ThreadX-only routines |
| 3303 | rewriter.setInsertionPointAfter(ifOp); |
| 3304 | if (needsWorkgroupBarrier) { |
| 3305 | mlir::acc::GPUParallelDimsAttr threadYDimsAttr = |
| 3306 | mlir::acc::GPUParallelDimsAttr::get( |
| 3307 | context: rewriter.getContext(), |
| 3308 | array: {mlir::acc::GPUParallelDimAttr::threadYDim(rewriter.getContext())}); |
| 3309 | createBarrier(loc, parDimsAttr: threadYDimsAttr); |
| 3310 | } else { |
| 3311 | // ThreadX-only: use per-ThreadY barrier |
| 3312 | mlir::acc::GPUParallelDimsAttr threadXDimsAttr = |
| 3313 | mlir::acc::GPUParallelDimsAttr::get( |
| 3314 | context: rewriter.getContext(), |
| 3315 | array: {mlir::acc::GPUParallelDimAttr::threadXDim(rewriter.getContext())}); |
| 3316 | createBarrier(loc, parDimsAttr: threadXDimsAttr); |
| 3317 | } |
| 3318 | // Load from shared memory, indexed by ThreadY ID when ThreadY is active. |
| 3319 | Value load; |
| 3320 | if (threadYIsActive) { |
| 3321 | Value threadYId = getThreadId(loc, dim: gpu::Dimension::y); |
| 3322 | load = memref::LoadOp::create(builder&: rewriter, location: privatize->getLoc(), memref: alloca, |
| 3323 | indices: ValueRange{threadYId}); |
| 3324 | } else { |
| 3325 | load = memref::LoadOp::create(builder&: rewriter, location: privatize->getLoc(), memref: alloca, |
| 3326 | indices: ValueRange{}); |
| 3327 | } |
| 3328 | rewriter.setInsertionPointAfter(load.getDefiningOp()); |
| 3329 | mapping.map(from: privatize.getResult(), to: load); |
| 3330 | |
| 3331 | // Operations inside the kernel are all rewritten from scratch. |
| 3332 | // But if the privatize op is outside the kernel, it needs to be replaced. |
| 3333 | if (!privatize->getParentOfType<acc::ComputeRegionOp>()) |
| 3334 | rewriter.replaceOp(op: privatize, newValues: load); |
| 3335 | // Deallocate |
| 3336 | rewriter.setInsertionPoint(ifOp->getBlock()->getTerminator()); |
| 3337 | if (needsWorkgroupBarrier) { |
| 3338 | mlir::acc::GPUParallelDimsAttr workerDimsAttr = |
| 3339 | mlir::acc::GPUParallelDimsAttr::get( |
| 3340 | context: rewriter.getContext(), |
| 3341 | array: {mlir::acc::GPUParallelDimAttr::threadYDim(rewriter.getContext())}); |
| 3342 | createBarrier(loc, parDimsAttr: workerDimsAttr); |
| 3343 | } else { |
| 3344 | mlir::acc::GPUParallelDimsAttr vectorDimsAttr = |
| 3345 | mlir::acc::GPUParallelDimsAttr::get( |
| 3346 | context: rewriter.getContext(), |
| 3347 | array: {mlir::acc::GPUParallelDimAttr::threadXDim(rewriter.getContext())}); |
| 3348 | createBarrier(loc, parDimsAttr: vectorDimsAttr); |
| 3349 | } |
| 3350 | auto ifOp2 = scf::IfOp::create(builder&: rewriter, location: loc, cond: predicate, |
| 3351 | /*withElseRegion=*/false); |
| 3352 | Region &thenRegion2 = ifOp2.getThenRegion(); |
| 3353 | Block &thenBlock2 = thenRegion2.back(); |
| 3354 | rewriter.setInsertionPoint(thenBlock2.getTerminator()); |
| 3355 | memref::DeallocOp::create(builder&: rewriter, location: privatize->getLoc(), memref: load); |
| 3356 | |
| 3357 | // Return the private memory |
| 3358 | rewriter.setInsertionPointAfter(load.getDefiningOp()); |
| 3359 | |
| 3360 | return load; |
| 3361 | } |
| 3362 | |
| 3363 | // Materialize acc.private_local storage from acc.privatize: per-thread alloca |
| 3364 | // when possible, otherwise a shared broadcast slot or acc.gpu_shared_memory. |
| 3365 | |
| 3366 | void ACCCGToGPULowering::processPrivateLocal( |
| 3367 | acc::PrivateLocalOp privateLocal, std::optional<int64_t> sharedMemCopies) { |
| 3368 | LLVM_DEBUG(llvm::dbgs() << "processing private local: " ; |
| 3369 | privateLocal->print(llvm::dbgs()); llvm::dbgs() << "\n" ); |
| 3370 | Location loc = privateLocal.getLoc(); |
| 3371 | acc::PrivateType privTy = |
| 3372 | cast<acc::PrivateType>(Val: privateLocal.getPrivatized().getType()); |
| 3373 | ModuleOp module = computeRegion->getParentOfType<ModuleOp>(); |
| 3374 | MemRefType baseTy = getPrivateBaseMemRefType(baseTy: privTy.getBaseTy(), module); |
| 3375 | MemRefType byteMemrefTy = |
| 3376 | MemRefType::get(shape: {ShapedType::kDynamic}, elementType: rewriter.getI8Type()); |
| 3377 | |
| 3378 | acc::PrivatizeOp privatizeOp = getPrivatizeOp(privateLocal, computeRegion); |
| 3379 | Value inputMem; |
| 3380 | if (privatizeOp->getParentOfType<acc::ComputeRegionOp>() == computeRegion) { |
| 3381 | inputMem = mapping.lookupOrNull(from: privatizeOp); |
| 3382 | if (inputMem) { |
| 3383 | Value result = castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value: inputMem, |
| 3384 | resultType: privateLocal.getType()); |
| 3385 | mapping.map(from: privateLocal.getResult(), to: result); |
| 3386 | return; |
| 3387 | } |
| 3388 | } else { |
| 3389 | // Hoisted acc.privatize: allocate per-thread stack storage in the launch |
| 3390 | // body. Cross-thread array reduction accumulators are per-thread too, so |
| 3391 | // the accumulate can reduce each element across threads. Skip when storage |
| 3392 | // par_dims lack thread_x (gang-/worker-scoped array temp). |
| 3393 | acc::ReductionAccumulateArrayOp arrayAccum = |
| 3394 | perThreadArrayReductionAccum(v: privateLocal.getResult()); |
| 3395 | if ((isThreadXPrivatize(privatize: privatizeOp) || |
| 3396 | (arrayAccum && storageHasThreadX(privateLocal, computeRegion))) && |
| 3397 | canUseStackAlloca(baseTy, loc, maxThreadPrivateStack: options.maxThreadPrivateStack)) { |
| 3398 | Value alloca = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType: baseTy); |
| 3399 | recordThreadPrivate(privateLocal); |
| 3400 | if (arrayAccum) { |
| 3401 | FailureOr<arith::AtomicRMWKind> kind = getReductionKind( |
| 3402 | redOp: arrayAccum.getReductionOperator(), type: baseTy.getElementType(), loc); |
| 3403 | if (failed(Result: kind)) |
| 3404 | return; |
| 3405 | initPerThreadArrayAccum(b&: rewriter, loc, alloca, baseTy, kind: *kind); |
| 3406 | } |
| 3407 | Value mem = castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value: alloca, |
| 3408 | resultType: privateLocal.getType()); |
| 3409 | mapping.map(from: privateLocal.getResult(), to: mem); |
| 3410 | return; |
| 3411 | } |
| 3412 | |
| 3413 | // If acc.privatize is outside the kernel, it needs to be converted |
| 3414 | // explicitly. |
| 3415 | std::optional<int64_t> copies = |
| 3416 | sharedMemCopies ? sharedMemCopies |
| 3417 | : isEligibleForSharedMemory(privateLocal, baseTy); |
| 3418 | if (copies) { |
| 3419 | int64_t numCopies = *copies; |
| 3420 | int64_t elementSize = getElementSizeInBytes(loc, elementType: baseTy.getElementType()); |
| 3421 | int64_t numElements = 1; |
| 3422 | for (int64_t dim : baseTy.getShape()) |
| 3423 | numElements *= dim; |
| 3424 | int64_t upperBound = elementSize * numElements * numCopies; |
| 3425 | |
| 3426 | if (tryAllocateSharedMemory(bytes: upperBound)) { |
| 3427 | std::string varName = |
| 3428 | accSupport.getVariableName(v: privateLocal.getResult()); |
| 3429 | sharedMemPrivateVarNames.push_back(Elt: varName.empty() ? "(*)" : varName); |
| 3430 | |
| 3431 | gpu::AddressSpaceAttr workgroupAS = gpu::AddressSpaceAttr::get( |
| 3432 | context: computeRegion->getContext(), |
| 3433 | value: gpu::GPUDialect::getWorkgroupAddressSpace()); |
| 3434 | MemRefType sharedMemTy = |
| 3435 | MemRefType::get(shape: baseTy.getShape(), elementType: baseTy.getElementType(), |
| 3436 | layout: MemRefLayoutAttrInterface{}, memorySpace: workgroupAS); |
| 3437 | Value sharedMem = acc::GPUSharedMemoryOp::create( |
| 3438 | builder&: rewriter, location: loc, resultType: sharedMemTy, numCopies: rewriter.getI64IntegerAttr(value: numCopies), |
| 3439 | staticUpperBoundBytes: rewriter.getI64IntegerAttr(value: upperBound), dynamicSizes: ValueRange{}, dynamicSharedMemoryScalingBytes: IntegerAttr{}, |
| 3440 | dynamicSharedMemoryFixedBytes: IntegerAttr{}); |
| 3441 | |
| 3442 | Value mem = |
| 3443 | castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value: sharedMem, resultType: baseTy); |
| 3444 | Value result = castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value: mem, |
| 3445 | resultType: privateLocal.getType()); |
| 3446 | |
| 3447 | mapping.map(from: privateLocal.getResult(), to: result); |
| 3448 | return; |
| 3449 | } |
| 3450 | } |
| 3451 | |
| 3452 | OpBuilder::InsertionGuard guard(rewriter); |
| 3453 | rewriter.setInsertionPoint(privatizeOp); |
| 3454 | inputMem = processPrivatize(privatize: privatizeOp); |
| 3455 | } |
| 3456 | |
| 3457 | if (isInsideACCSpecializedRoutine(op: computeRegion)) { |
| 3458 | assert(inputMem && "expected input mem to be mapped" ); |
| 3459 | Value result = castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value: inputMem, |
| 3460 | resultType: privateLocal.getType()); |
| 3461 | mapping.map(from: privateLocal.getResult(), to: result); |
| 3462 | return; |
| 3463 | } |
| 3464 | |
| 3465 | // The private element shape augmented with a dimension for each level of |
| 3466 | // parallelism |
| 3467 | SmallVector<int64_t> viewShape; |
| 3468 | // The dynamic sizes of the view (num_gangs, num_workers, vector_length) |
| 3469 | SmallVector<Value> viewDynSizes; |
| 3470 | // The offset of the subview from gpu.block_id/gpu.thread_id dimensions. |
| 3471 | SmallVector<OpFoldResult> subviewOffset; |
| 3472 | // The sizes of the subview where the dimensionality is brought back to the |
| 3473 | // private element. That is size 1 for each active block/thread dimension. |
| 3474 | SmallVector<OpFoldResult> subviewSizes; |
| 3475 | // The strides of the subview |
| 3476 | SmallVector<int64_t> subviewStrides; |
| 3477 | // The shape of the subview |
| 3478 | SmallVector<int64_t> subviewShape; |
| 3479 | |
| 3480 | std::pair<SmallVector<mlir::acc::GPUParallelDimAttr>, |
| 3481 | SmallVector<mlir::acc::GPUParallelDimAttr>> |
| 3482 | parDimsPair = computeActiveAndInactiveParDims(op: privateLocal, block: nullptr); |
| 3483 | acc::ReductionAccumulateArrayOp arrayAccum = |
| 3484 | perThreadArrayReductionAccum(v: privateLocal.getResult()); |
| 3485 | for (mlir::acc::GPUParallelDimAttr parDim : parDimsPair.first) { |
| 3486 | if ((parDim.isThreadX() || |
| 3487 | (arrayAccum && storageHasThreadX(privateLocal, computeRegion))) && |
| 3488 | canUseStackAlloca(baseTy, loc, maxThreadPrivateStack: options.maxThreadPrivateStack)) { |
| 3489 | Value alloca = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType: baseTy); |
| 3490 | recordThreadPrivate(privateLocal); |
| 3491 | if (arrayAccum) { |
| 3492 | FailureOr<arith::AtomicRMWKind> kind = getReductionKind( |
| 3493 | redOp: arrayAccum.getReductionOperator(), type: baseTy.getElementType(), loc); |
| 3494 | if (failed(Result: kind)) |
| 3495 | return; |
| 3496 | initPerThreadArrayAccum(b&: rewriter, loc, alloca, baseTy, kind: *kind); |
| 3497 | } |
| 3498 | Value mem = castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value: alloca, |
| 3499 | resultType: privateLocal.getType()); |
| 3500 | mapping.map(from: privateLocal.getResult(), to: mem); |
| 3501 | return; |
| 3502 | } |
| 3503 | } |
| 3504 | if (parDimsPair.first.empty()) { |
| 3505 | // No parallelism is found above. It's single block execution. |
| 3506 | mlir::acc::insertParDim( |
| 3507 | parDims&: parDimsPair.first, |
| 3508 | parDim: mlir::acc::GPUParallelDimAttr::blockXDim(privateLocal.getContext())); |
| 3509 | } |
| 3510 | for (mlir::acc::GPUParallelDimAttr parDim : parDimsPair.first) { |
| 3511 | gpu::Processor gpuProc = parDim.getProcessor(); |
| 3512 | Value gpuSize = getGPUSizeFor(proc: gpuProc); |
| 3513 | viewDynSizes.push_back(Elt: gpuSize); |
| 3514 | viewShape.push_back(Elt: ShapedType::kDynamic); |
| 3515 | subviewOffset.push_back(Elt: getGPUThreadIdFor(proc: gpuProc)); |
| 3516 | subviewSizes.push_back(Elt: rewriter.getIndexAttr(value: 1)); |
| 3517 | } |
| 3518 | |
| 3519 | SmallVector<Value> innerDynSizes = |
| 3520 | resolvePrivateLocalDynamicExtents(privateLocal); |
| 3521 | |
| 3522 | unsigned dynIdx = 0; |
| 3523 | for (auto innerDim : baseTy.getShape()) { |
| 3524 | subviewOffset.push_back(Elt: rewriter.getIndexAttr(value: 0)); |
| 3525 | viewShape.push_back(Elt: innerDim); |
| 3526 | subviewShape.push_back(Elt: innerDim); |
| 3527 | if (innerDim == ShapedType::kDynamic) { |
| 3528 | assert(dynIdx < innerDynSizes.size() && |
| 3529 | "not enough dynamic sizes for inner dimensions" ); |
| 3530 | viewDynSizes.push_back(Elt: innerDynSizes[dynIdx]); |
| 3531 | subviewSizes.push_back(Elt: innerDynSizes[dynIdx]); |
| 3532 | ++dynIdx; |
| 3533 | } else { |
| 3534 | subviewSizes.push_back(Elt: rewriter.getIndexAttr(value: innerDim)); |
| 3535 | } |
| 3536 | } |
| 3537 | |
| 3538 | // Do the strides in reverse order. |
| 3539 | int64_t stride = 1; |
| 3540 | for (auto innerDimIt = baseTy.getShape().rbegin(); |
| 3541 | innerDimIt != baseTy.getShape().rend(); ++innerDimIt) { |
| 3542 | int64_t innerDim = *innerDimIt; |
| 3543 | subviewStrides.insert(I: subviewStrides.begin(), Elt: stride); |
| 3544 | if (innerDim == ShapedType::kDynamic) |
| 3545 | stride = ShapedType::kDynamic; |
| 3546 | if (stride != ShapedType::kDynamic) |
| 3547 | stride *= innerDim; |
| 3548 | } |
| 3549 | |
| 3550 | Value memBuffer = |
| 3551 | castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value: inputMem, resultType: byteMemrefTy); |
| 3552 | auto c0 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 3553 | MemRefType viewType = MemRefType::get(shape: viewShape, elementType: baseTy.getElementType()); |
| 3554 | auto view = memref::ViewOp::create(builder&: rewriter, location: loc, resultType0: viewType, source: memBuffer, |
| 3555 | byte_shift: c0.getResult(), sizes: viewDynSizes); |
| 3556 | |
| 3557 | // memref.subview |
| 3558 | StridedLayoutAttr stridedLayout = StridedLayoutAttr::get( |
| 3559 | context: computeRegion->getContext(), offset: ShapedType::kDynamic, strides: subviewStrides); |
| 3560 | MemRefType subviewType = |
| 3561 | MemRefType::get(shape: subviewShape, elementType: baseTy.getElementType(), layout: stridedLayout); |
| 3562 | SmallVector<OpFoldResult> ones(viewType.getRank(), rewriter.getIndexAttr(value: 1)); |
| 3563 | Value subview = memref::SubViewOp::create(builder&: rewriter, location: loc, resultType: subviewType, source: view, |
| 3564 | offsets: subviewOffset, sizes: subviewSizes, strides: ones); |
| 3565 | |
| 3566 | // Pointer-like casts cannot carry memref offsets. Shift the aligned pointer |
| 3567 | // so later zero-offset views retain this thread's private slice. |
| 3568 | auto metadata = |
| 3569 | memref::ExtractStridedMetadataOp::create(builder&: rewriter, location: loc, source: subview); |
| 3570 | Value elementBytes = arith::ConstantIndexOp::create( |
| 3571 | builder&: rewriter, location: loc, value: getElementSizeInBytes(loc, elementType: baseTy.getElementType())); |
| 3572 | Value byteOffset = |
| 3573 | arith::MulIOp::create(builder&: rewriter, location: loc, lhs: metadata.getOffset(), rhs: elementBytes); |
| 3574 | Value privateView = memref::ViewOp::create(builder&: rewriter, location: loc, resultType0: baseTy, source: memBuffer, |
| 3575 | byte_shift: byteOffset, sizes: innerDynSizes); |
| 3576 | Value result = castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value: privateView, |
| 3577 | resultType: privateLocal.getType()); |
| 3578 | mapping.map(from: privateLocal.getResult(), to: result); |
| 3579 | } |
| 3580 | |
| 3581 | // Could be scf::for or scf::parallel |
| 3582 | template <typename LoopOp> |
| 3583 | void ACCCGToGPULowering::processSeqLoop(LoopOp loopOp) { |
| 3584 | // Pre-process shared-memory-eligible private_local ops. Only direct-child |
| 3585 | // ops are considered; nested private_local ops (e.g. inside predicate_region) |
| 3586 | // are handled by recursive body processing. |
| 3587 | LLVM_DEBUG(llvm::dbgs() << "processing seq loop: " ; |
| 3588 | loopOp->print(llvm::dbgs()); llvm::dbgs() << "\n" ); |
| 3589 | llvm::SmallPtrSet<Operation *, 4> preProcessedPrivateLocals; |
| 3590 | ModuleOp module = computeRegion->getParentOfType<ModuleOp>(); |
| 3591 | for (auto &bodyOp : loopOp.getBody()->getOperations()) { |
| 3592 | if (acc::PrivateLocalOp privateLocal = |
| 3593 | dyn_cast<acc::PrivateLocalOp>(&bodyOp)) { |
| 3594 | acc::PrivateType privTy = |
| 3595 | cast<acc::PrivateType>(Val: privateLocal.getPrivatized().getType()); |
| 3596 | MemRefType baseTy = getPrivateBaseMemRefType(baseTy: privTy.getBaseTy(), module); |
| 3597 | if (auto copies = isEligibleForSharedMemory(privateLocal, baseTy)) { |
| 3598 | processPrivateLocal(privateLocal, sharedMemCopies: copies); |
| 3599 | preProcessedPrivateLocals.insert(Ptr: privateLocal.getOperation()); |
| 3600 | } |
| 3601 | } |
| 3602 | } |
| 3603 | |
| 3604 | LoopOp newLoop = |
| 3605 | dyn_cast<LoopOp>(rewriter.cloneWithoutRegions(*loopOp, mapping)); |
| 3606 | rewriter.createBlock( |
| 3607 | &newLoop.getRegion(), newLoop.getRegion().begin(), |
| 3608 | loopOp.getBody()->getArgumentTypes(), |
| 3609 | SmallVector<Location>(loopOp.getBody()->getArgumentTypes().size(), |
| 3610 | loopOp->getLoc())); |
| 3611 | rewriter.setInsertionPointToStart(&newLoop.getRegion().front()); |
| 3612 | |
| 3613 | // Need to clone all block arguments |
| 3614 | Block::BlockArgListType blockArgs = loopOp.getBody()->getArguments(); |
| 3615 | assert(blockArgs.size() && "expected block arguments for loop" ); |
| 3616 | mapping.map(blockArgs, newLoop.getBody()->getArguments()); |
| 3617 | |
| 3618 | // A gang-private slot written here is one copy per workgroup, reused by every |
| 3619 | // iteration, and a thread that has already started the next iteration would |
| 3620 | // overwrite the value another thread has not read yet. Separate the |
| 3621 | // iterations with a barrier at the top of the body. |
| 3622 | if (needsInLoopReuseBarrier(loopOp: loopOp.getOperation()) && |
| 3623 | !isInsideConditional(block: rewriter.getInsertionBlock())) |
| 3624 | emitGPUBarrierWorkgroup(rewriter, loopOp->getLoc()); |
| 3625 | |
| 3626 | for (auto &bodyOp : loopOp.getBody()->getOperations()) { |
| 3627 | if (preProcessedPrivateLocals.contains(Ptr: &bodyOp)) |
| 3628 | continue; |
| 3629 | processOp(op: &bodyOp); |
| 3630 | } |
| 3631 | |
| 3632 | mapping.map(loopOp.getResults(), newLoop.getResults()); |
| 3633 | rewriter.setInsertionPointAfter(newLoop); |
| 3634 | |
| 3635 | // Postpone the barrier when trailing work in this block still separates the |
| 3636 | // loop from the next reconvergence point. |
| 3637 | if (hasTrailingSideEffectSiblings(loopOp.getOperation())) |
| 3638 | deferredBarrierSeqLoops.push_back(Elt: loopOp.getOperation()); |
| 3639 | else |
| 3640 | createBarrierAfterSeqLoop(loopOp: loopOp.getOperation()); |
| 3641 | } |
| 3642 | |
| 3643 | void ACCCGToGPULowering::flushDeferredBarriersBefore(Operation *beforeOp) { |
| 3644 | Block *block = beforeOp->getBlock(); |
| 3645 | SmallVector<Operation *, 4> toFlush; |
| 3646 | for (Operation *loopOp : deferredBarrierSeqLoops) |
| 3647 | if (loopOp->getBlock() == block && loopOp->isBeforeInBlock(other: beforeOp)) |
| 3648 | toFlush.push_back(Elt: loopOp); |
| 3649 | if (toFlush.empty()) |
| 3650 | return; |
| 3651 | llvm::sort(C&: toFlush, |
| 3652 | Comp: [](Operation *a, Operation *b) { return a->isBeforeInBlock(other: b); }); |
| 3653 | for (Operation *loopOp : toFlush) |
| 3654 | createBarrierAfterSeqLoop(loopOp); |
| 3655 | deferredBarrierSeqLoops.erase( |
| 3656 | CS: std::remove_if(first: deferredBarrierSeqLoops.begin(), |
| 3657 | last: deferredBarrierSeqLoops.end(), |
| 3658 | pred: [&](Operation *loopOp) { |
| 3659 | return loopOp->getBlock() == block && |
| 3660 | loopOp->isBeforeInBlock(other: beforeOp); |
| 3661 | }), |
| 3662 | CE: deferredBarrierSeqLoops.end()); |
| 3663 | } |
| 3664 | |
| 3665 | // try to process op as a loop mapped to a gpu parallelism |
| 3666 | // failure signifies not a loop and needs different processing |
| 3667 | void ACCCGToGPULowering::processParallelOp(scf::ParallelOp parallelOp) { |
| 3668 | LLVM_DEBUG(llvm::dbgs() << "processing par loop: " ; |
| 3669 | parallelOp->print(llvm::dbgs()); llvm::dbgs() << "\n" ); |
| 3670 | assert(mlir::acc::hasParDimsAttr(parallelOp) && |
| 3671 | "requires parallel dimensions attribute" ); |
| 3672 | mlir::acc::GPUParallelDimsAttr pDimsAttr = |
| 3673 | mlir::acc::getParDimsAttr(op: parallelOp); |
| 3674 | // both of these should be dealt with before compiler reaches ACCCGToGPU |
| 3675 | // Invalid parallel-loop structure should be rejected before acc-cg-to-gpu. |
| 3676 | assert(pDimsAttr.getArray().size() == 1 && |
| 3677 | "expected a single par dim in acc-cg-to-gpu" ); |
| 3678 | assert(parallelOp.getInductionVars().size() == 1 && |
| 3679 | "expected a single induction variable in acc-cg-to-gpu" ); |
| 3680 | |
| 3681 | mlir::acc::GPUParallelDimAttr parDim = pDimsAttr.getArray().front(); |
| 3682 | |
| 3683 | bool savedGridStrideFlag = insideAccumulateGridStride; |
| 3684 | Value savedReductionBuf = reductionSharedBuf; |
| 3685 | if (parDim.isThreadX()) { |
| 3686 | bool found = false; |
| 3687 | parallelOp.getBody()->walk(callback: [&](acc::ReductionAccumulateOp accOp) { |
| 3688 | bool hasBlockDim = false; |
| 3689 | bool hasThreadDim = false; |
| 3690 | for (auto d : accOp.getParDims().getArray()) { |
| 3691 | if (d.isAnyBlock()) |
| 3692 | hasBlockDim = true; |
| 3693 | if (d.isThreadX() || d.isThreadY()) |
| 3694 | hasThreadDim = true; |
| 3695 | } |
| 3696 | if (hasThreadDim && !hasBlockDim) { |
| 3697 | found = true; |
| 3698 | return WalkResult::interrupt(); |
| 3699 | } |
| 3700 | return WalkResult::advance(); |
| 3701 | }); |
| 3702 | if (found) |
| 3703 | insideAccumulateGridStride = true; |
| 3704 | } |
| 3705 | |
| 3706 | // common loop body processing for both par0 and 1+ |
| 3707 | auto processLoopBody = [&]() { |
| 3708 | // process inner ops recursively |
| 3709 | for (auto &bodyOp : parallelOp.getBody()->getOperations()) { |
| 3710 | if (bodyOp.hasTrait<OpTrait::IsTerminator>()) { |
| 3711 | // Non-cloned parallel loops reconverge at their terminator. |
| 3712 | flushDeferredBarriersBefore(beforeOp: &bodyOp); |
| 3713 | continue; |
| 3714 | } |
| 3715 | // Non-terminator body ops are processed recursively. |
| 3716 | processOp(op: &bodyOp); |
| 3717 | } |
| 3718 | }; |
| 3719 | |
| 3720 | if (parDim.isSeq()) { |
| 3721 | LLVM_DEBUG(llvm::dbgs() << "loop: parDim: " << parDim << " as gpu seq\n" ); |
| 3722 | // Sequential loops here are remainder loops from partitioned parallel |
| 3723 | // loops - clone them as-is but process the body as a parallel region. |
| 3724 | // When blockDim.x >= subgroupSize and the loop contains a thread-level |
| 3725 | // accumulate, inactive grid-stride threads cannot participate in subgroup |
| 3726 | // reductions, so we use atomic-to-shared-memory reduction instead. |
| 3727 | bool needsAtomicReduction = false; |
| 3728 | bool hasAccumulateSibling = false; |
| 3729 | if (scf::ParallelOp parentPar = |
| 3730 | parallelOp->getParentOfType<scf::ParallelOp>()) { |
| 3731 | if (mlir::acc::GPUParallelDimsAttr parentDims = |
| 3732 | mlir::acc::getParDimsAttr(op: parentPar); |
| 3733 | parentDims && llvm::any_of(Range: parentDims.getArray(), |
| 3734 | P: [](auto d) { return d.isThreadX(); })) { |
| 3735 | for (auto &op : parentPar.getBody()->getOperations()) { |
| 3736 | if (acc::ReductionAccumulateOp acc = |
| 3737 | dyn_cast<acc::ReductionAccumulateOp>(Val&: op)) { |
| 3738 | bool hasBlockDim = false; |
| 3739 | bool hasThreadDim = false; |
| 3740 | for (auto d : acc.getParDims().getArray()) { |
| 3741 | if (d.isAnyBlock()) |
| 3742 | hasBlockDim = true; |
| 3743 | if (d.isThreadX() || d.isThreadY()) |
| 3744 | hasThreadDim = true; |
| 3745 | } |
| 3746 | if (hasThreadDim && !hasBlockDim) |
| 3747 | hasAccumulateSibling = true; |
| 3748 | } |
| 3749 | } |
| 3750 | } |
| 3751 | } |
| 3752 | if (insideAccumulateGridStride || hasAccumulateSibling) { |
| 3753 | for (auto launchArg : computeRegion.getLaunchArgs()) { |
| 3754 | if (acc::ParWidthOp pw = launchArg.getDefiningOp<acc::ParWidthOp>()) { |
| 3755 | if (pw.getParDim().isThreadX()) { |
| 3756 | if (auto cval = getConstantIntValue(ofr: pw.getLaunchArg())) |
| 3757 | needsAtomicReduction = (*cval >= options.subgroupSize); |
| 3758 | else |
| 3759 | needsAtomicReduction = true; |
| 3760 | break; |
| 3761 | } |
| 3762 | } |
| 3763 | } |
| 3764 | } |
| 3765 | if (needsAtomicReduction && !reductionSharedBuf) { |
| 3766 | Type elemTy; |
| 3767 | parallelOp.getBody()->walk(callback: [&](acc::ReductionAccumulateOp accOp) { |
| 3768 | Type t = accOp.getValue().getType(); |
| 3769 | if (isa<FloatType, IntegerType>(Val: t)) |
| 3770 | elemTy = t; |
| 3771 | return elemTy ? WalkResult::interrupt() : WalkResult::advance(); |
| 3772 | }); |
| 3773 | if (!elemTy) |
| 3774 | needsAtomicReduction = false; |
| 3775 | } |
| 3776 | if (needsAtomicReduction && !reductionSharedBuf) { |
| 3777 | Location seqLoc = parallelOp->getLoc(); |
| 3778 | gpu::AddressSpaceAttr workgroupAS = gpu::AddressSpaceAttr::get( |
| 3779 | context: computeRegion->getContext(), |
| 3780 | value: gpu::GPUDialect::getWorkgroupAddressSpace()); |
| 3781 | Type elemTy; |
| 3782 | parallelOp.getBody()->walk(callback: [&](acc::ReductionAccumulateOp accOp) { |
| 3783 | Type t = accOp.getValue().getType(); |
| 3784 | if (isa<FloatType, IntegerType>(Val: t)) |
| 3785 | elemTy = t; |
| 3786 | return elemTy ? WalkResult::interrupt() : WalkResult::advance(); |
| 3787 | }); |
| 3788 | assert(elemTy && "expected scalar reduction element type" ); |
| 3789 | unsigned elemBytes = elemTy.getIntOrFloatBitWidth() / 8; |
| 3790 | MemRefType bufTy = MemRefType::get(shape: {options.subgroupSize}, elementType: elemTy, |
| 3791 | map: AffineMap{}, memorySpace: workgroupAS); |
| 3792 | reductionSharedBuf = acc::GPUSharedMemoryOp::create( |
| 3793 | builder&: rewriter, location: seqLoc, resultType: bufTy, numCopies: rewriter.getI64IntegerAttr(value: 1), |
| 3794 | staticUpperBoundBytes: rewriter.getI64IntegerAttr(value: options.subgroupSize * elemBytes), |
| 3795 | dynamicSizes: ValueRange{}, dynamicSharedMemoryScalingBytes: IntegerAttr{}, dynamicSharedMemoryFixedBytes: IntegerAttr{}); |
| 3796 | Value tidY = getThreadId(loc: seqLoc, dim: gpu::Dimension::y); |
| 3797 | Value identity; |
| 3798 | if (isa<FloatType>(Val: elemTy)) { |
| 3799 | identity = arith::ConstantOp::create( |
| 3800 | builder&: rewriter, location: seqLoc, result: elemTy, value: rewriter.getFloatAttr(type: elemTy, value: 0.0)); |
| 3801 | } else { |
| 3802 | identity = arith::ConstantIntOp::create(builder&: rewriter, location: seqLoc, type: elemTy, value: 0); |
| 3803 | } |
| 3804 | memref::StoreOp::create(builder&: rewriter, location: seqLoc, valueToStore: identity, memref: reductionSharedBuf, |
| 3805 | indices: tidY); |
| 3806 | createPerRowBarrier(loc: seqLoc); |
| 3807 | } |
| 3808 | processSeqLoop(loopOp: parallelOp); |
| 3809 | loopReductions.push_back(Elt: parallelOp); |
| 3810 | } else { |
| 3811 | LLVM_DEBUG(llvm::dbgs() |
| 3812 | << "processing loop: parDim: " << parDim << " as gpu par\n" ); |
| 3813 | // actual parallel loops get their iv mapped to gpu hierarchy |
| 3814 | // and the loop construct is not cloned to gpu kernel only the |
| 3815 | // ops are cloned with mapping of gpu id for original loop iv |
| 3816 | Value gpuThreadId = getGPUThreadIdFor(proc: parDim.getProcessor()); |
| 3817 | mapping.map(from: parallelOp.getInductionVars()[0], to: gpuThreadId); |
| 3818 | |
| 3819 | processLoopBody(); |
| 3820 | |
| 3821 | // All threads of the row (or block) must finish their iterations before |
| 3822 | // any of them continues past the loop. The sequential nest around the loop |
| 3823 | // is executed by every thread, so without a barrier its later code can read |
| 3824 | // results that another thread has not stored yet. |
| 3825 | if (needsBarrierAfterThreadLoop(loop: parallelOp)) |
| 3826 | createBarrier(loc: parallelOp->getLoc(), parDimsAttr: pDimsAttr); |
| 3827 | |
| 3828 | // Since the loop is not copied over, create dummy mappings for_each |
| 3829 | // of the loop results. These will ultimately by replaced with a |
| 3830 | // reduction |
| 3831 | llvm::for_each(Range: parallelOp.getResults(), F: [&](Value v) { |
| 3832 | Type valTy = v.getType(); |
| 3833 | TypedAttr zeroAttr = rewriter.getZeroAttr(type: valTy); |
| 3834 | auto zero = arith::ConstantOp::create(builder&: rewriter, location: parallelOp->getLoc(), |
| 3835 | result: valTy, value: zeroAttr); |
| 3836 | mapping.map(from: v, to: zero); |
| 3837 | }); |
| 3838 | loopReductions.push_back(Elt: parallelOp); |
| 3839 | } |
| 3840 | insideAccumulateGridStride = savedGridStrideFlag; |
| 3841 | if (!insideAccumulateGridStride && !savedReductionBuf) |
| 3842 | reductionSharedBuf = Value(); |
| 3843 | } |
| 3844 | |
| 3845 | /// Map an atomic RMW kind to the corresponding `gpu.all_reduce` operation. |
| 3846 | static gpu::AllReduceOperation |
| 3847 | getAllReduceOperation(arith::AtomicRMWKind kind) { |
| 3848 | switch (kind) { |
| 3849 | case arith::AtomicRMWKind::addf: |
| 3850 | case arith::AtomicRMWKind::addi: |
| 3851 | return gpu::AllReduceOperation::ADD; |
| 3852 | case arith::AtomicRMWKind::mulf: |
| 3853 | case arith::AtomicRMWKind::muli: |
| 3854 | return gpu::AllReduceOperation::MUL; |
| 3855 | case arith::AtomicRMWKind::minu: |
| 3856 | return gpu::AllReduceOperation::MINUI; |
| 3857 | case arith::AtomicRMWKind::mins: |
| 3858 | return gpu::AllReduceOperation::MINSI; |
| 3859 | case arith::AtomicRMWKind::minnumf: |
| 3860 | return gpu::AllReduceOperation::MINNUMF; |
| 3861 | case arith::AtomicRMWKind::maxu: |
| 3862 | return gpu::AllReduceOperation::MAXUI; |
| 3863 | case arith::AtomicRMWKind::maxs: |
| 3864 | return gpu::AllReduceOperation::MAXSI; |
| 3865 | case arith::AtomicRMWKind::maxnumf: |
| 3866 | return gpu::AllReduceOperation::MAXNUMF; |
| 3867 | case arith::AtomicRMWKind::ori: |
| 3868 | return gpu::AllReduceOperation::OR; |
| 3869 | case arith::AtomicRMWKind::andi: |
| 3870 | return gpu::AllReduceOperation::AND; |
| 3871 | case arith::AtomicRMWKind::xori: |
| 3872 | return gpu::AllReduceOperation::XOR; |
| 3873 | case arith::AtomicRMWKind::minimumf: |
| 3874 | return gpu::AllReduceOperation::MINIMUMF; |
| 3875 | case arith::AtomicRMWKind::maximumf: |
| 3876 | return gpu::AllReduceOperation::MAXIMUMF; |
| 3877 | case arith::AtomicRMWKind::assign: |
| 3878 | break; |
| 3879 | } |
| 3880 | llvm_unreachable("unsupported atomic kind" ); |
| 3881 | } |
| 3882 | |
| 3883 | void ACCCGToGPULowering::constructAtomicAccumulation( |
| 3884 | Location loc, Value memref, ValueRange indices, Value input, |
| 3885 | arith::AtomicRMWKind kind) { |
| 3886 | assert(!memref.getDefiningOp<memref::AllocaOp>() && |
| 3887 | "cannot lower atomic accumulation on an stack variable" ); |
| 3888 | |
| 3889 | // acc.atomic.update derives the element address from the memref descriptor's |
| 3890 | // base pointer and offset field; it has no subscript operand. When the store |
| 3891 | // being lowered targets a specific array element (e.g. result(idx) = |
| 3892 | // max(...)), fold the indices into the descriptor offset with a subview so |
| 3893 | // the atomic updates the intended element. Otherwise the atomic always hits |
| 3894 | // element 0, so any reduction whose destination index is non-zero is |
| 3895 | // miscompiled (the result lands in element 0 while the intended element keeps |
| 3896 | // its identity-init value). |
| 3897 | Value target = memref; |
| 3898 | if (!indices.empty()) { |
| 3899 | MemRefType memrefTy = cast<MemRefType>(Val: memref.getType()); |
| 3900 | unsigned rank = memrefTy.getRank(); |
| 3901 | assert(indices.size() == rank && "expected one index per memref dimension" ); |
| 3902 | SmallVector<OpFoldResult> offsets(indices.begin(), indices.end()); |
| 3903 | SmallVector<OpFoldResult> sizes(rank, rewriter.getIndexAttr(value: 1)); |
| 3904 | SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(value: 1)); |
| 3905 | target = memref::SubViewOp::create(builder&: rewriter, location: loc, source: memref, offsets, sizes, |
| 3906 | strides); |
| 3907 | } |
| 3908 | |
| 3909 | auto atomicUpdateOp = |
| 3910 | acc::AtomicUpdateOp::create(builder&: rewriter, location: loc, x: target, /*ifCond=*/Value()); |
| 3911 | Region ®ion = atomicUpdateOp->getRegion(index: 0); |
| 3912 | Block *block = |
| 3913 | rewriter.createBlock(parent: ®ion, insertPt: region.begin(), argTypes: {input.getType()}, locs: {loc}); |
| 3914 | rewriter.setInsertionPointToStart(block); |
| 3915 | Value reductionExpr = |
| 3916 | generateReductionOp(b&: rewriter, loc, lhs: input, rhs: block->getArgument(i: 0), kind); |
| 3917 | acc::YieldOp::create(builder&: rewriter, location: loc, operands: reductionExpr); |
| 3918 | rewriter.setInsertionPointAfter(atomicUpdateOp); |
| 3919 | } |
| 3920 | |
| 3921 | void ACCCGToGPULowering::createGPUAllReduceOp( |
| 3922 | Location loc, Value input, Value memref, arith::AtomicRMWKind kind, |
| 3923 | mlir::acc::GPUParallelDimsAttr parDimsAttr, ValueRange indices, |
| 3924 | bool isPerThreadPrivateTarget) { |
| 3925 | gpu::AllReduceOperationAttr attr = gpu::AllReduceOperationAttr::get( |
| 3926 | context: computeRegion->getContext(), value: getAllReduceOperation(kind)); |
| 3927 | auto emitAllReduce = [&](Value operand) -> Value { |
| 3928 | auto allReduceOp = |
| 3929 | gpu::AllReduceOp::create(builder&: rewriter, location: loc, value: operand, op: attr, uniform: true); |
| 3930 | mlir::acc::setParDimsAttr(op: allReduceOp, attr: parDimsAttr); |
| 3931 | return allReduceOp; |
| 3932 | }; |
| 3933 | |
| 3934 | Value reducedValue; |
| 3935 | if (auto complexType = dyn_cast<ComplexType>(Val: input.getType())) { |
| 3936 | // gpu.all_reduce takes only integers and floats, so a complex value has to |
| 3937 | // go through as components. Addition is the one operator reaching here |
| 3938 | // where that is equivalent: multiplication mixes the components, and |
| 3939 | // min/max have no ordering on complex at all. |
| 3940 | if (kind != arith::AtomicRMWKind::addf) { |
| 3941 | std::string msg; |
| 3942 | llvm::raw_string_ostream os(msg); |
| 3943 | os << "reduction: " << arith::stringifyAtomicRMWKind(kind) << " of " |
| 3944 | << complexType << " across threads" ; |
| 3945 | (void)accSupport.emitNYI(loc, message: os.str()); |
| 3946 | hasFailed = true; |
| 3947 | return; |
| 3948 | } |
| 3949 | Type elementType = complexType.getElementType(); |
| 3950 | Value realResult = |
| 3951 | emitAllReduce(complex::ReOp::create(builder&: rewriter, location: loc, real: elementType, complex: input)); |
| 3952 | Value imagResult = |
| 3953 | emitAllReduce(complex::ImOp::create(builder&: rewriter, location: loc, imaginary: elementType, complex: input)); |
| 3954 | reducedValue = complex::CreateOp::create(builder&: rewriter, location: loc, complex: complexType, |
| 3955 | real: realResult, imaginary: imagResult); |
| 3956 | } else { |
| 3957 | reducedValue = emitAllReduce(input); |
| 3958 | } |
| 3959 | // Predicate the store on the thread-level dimensions being reduced so that |
| 3960 | // only one thread per reduced group writes the result. Only dimensions in |
| 3961 | // parDimsAttr are included; sweeping over all dimensions between the highest |
| 3962 | // par_dim and thread_x would incorrectly add unrelated dimensions (e.g. |
| 3963 | // thread_y for a thread_x-only reduction), preventing other rows from |
| 3964 | // storing their independent results. |
| 3965 | SmallVector<mlir::acc::GPUParallelDimAttr> inactiveParDims; |
| 3966 | MLIRContext *ctx = computeRegion->getContext(); |
| 3967 | bool hasThreadX = false; |
| 3968 | for (auto parDim : parDimsAttr.getArray()) { |
| 3969 | if (parDim.isAnyBlock()) |
| 3970 | continue; |
| 3971 | if (parDim.isThreadX()) |
| 3972 | hasThreadX = true; |
| 3973 | if (computeRegion.getLaunchArg(parDim) || |
| 3974 | isInsideACCSpecializedRoutine(op: computeRegion)) { |
| 3975 | inactiveParDims.push_back(Elt: parDim); |
| 3976 | } |
| 3977 | } |
| 3978 | // Subgroup alignment may introduce extra ThreadX lanes even when ThreadX is |
| 3979 | // not part of the reduction. Predicate on ThreadX so only one lane stores. |
| 3980 | // A single-thread worker launch is left unaligned, so it has no such lanes. |
| 3981 | if (!hasThreadX && !isSingleThreadWorkerLaunch()) |
| 3982 | inactiveParDims.push_back(Elt: mlir::acc::GPUParallelDimAttr::threadXDim(ctx)); |
| 3983 | Value predicate = emitPredicate(loc, inactiveParDims); |
| 3984 | // Predication is only needed when the store target is visible to |
| 3985 | // multiple threads (shared/global memory). Per-thread targets like |
| 3986 | // memref.alloca are thread-private: gpu.all_reduce returns the same |
| 3987 | // value on all threads, so each can safely store to its own copy. |
| 3988 | // Detect per-thread storage by walking through conversion ops to |
| 3989 | // find the underlying allocation. |
| 3990 | bool isPerThreadPrivate = isPerThreadPrivateTarget || |
| 3991 | isa_and_nonnull<memref::AllocaOp>( |
| 3992 | Val: unwrapMemRefConversion(v: memref).getDefiningOp()); |
| 3993 | // combine |
| 3994 | scf::IfOp ifOp; |
| 3995 | if (predicate && !isPerThreadPrivate) { |
| 3996 | ifOp = |
| 3997 | scf::IfOp::create(builder&: rewriter, location: loc, cond: predicate, /*withElseRegion=*/false); |
| 3998 | Region &thenRegion = ifOp.getThenRegion(); |
| 3999 | Block &thenBlock = thenRegion.back(); |
| 4000 | rewriter.setInsertionPoint(thenBlock.getTerminator()); |
| 4001 | } |
| 4002 | memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: reducedValue, memref, indices); |
| 4003 | if (predicate && !isPerThreadPrivate) |
| 4004 | rewriter.setInsertionPointAfter(ifOp); |
| 4005 | // A later block combine reuses this instead of reloading the slot. This only |
| 4006 | // applies to scalar accumulators; array elements are indexed individually. |
| 4007 | if (indices.empty()) |
| 4008 | reductionAccumValue[memref] = reducedValue; |
| 4009 | } |
| 4010 | |
| 4011 | void ACCCGToGPULowering::postprocessAccumulateOp( |
| 4012 | acc::ReductionAccumulateOp op) { |
| 4013 | Location loc = op->getLoc(); |
| 4014 | |
| 4015 | rewriter.setInsertionPoint(op); |
| 4016 | |
| 4017 | // Check whether this accumulate has only block-level par dims (no thread |
| 4018 | // dims). gpu.all_reduce reduces across threads within a block, which is |
| 4019 | // wrong for block-only reductions - the loop result is already per-thread |
| 4020 | // and only needs a predicated store to the memref. |
| 4021 | bool hasThreadDim = false; |
| 4022 | SmallVector<mlir::acc::GPUParallelDimAttr> threadParDims; |
| 4023 | for (auto parDim : op.getParDims().getArray()) { |
| 4024 | if (!parDim.isAnyBlock()) { |
| 4025 | hasThreadDim = true; |
| 4026 | threadParDims.push_back(Elt: parDim); |
| 4027 | } |
| 4028 | } |
| 4029 | |
| 4030 | std::optional<arith::AtomicRMWKind> kind; |
| 4031 | if (hasThreadDim) { |
| 4032 | FailureOr<arith::AtomicRMWKind> kindOr = getReductionKind( |
| 4033 | redOp: op.getReductionOperator(), type: op.getValue().getType(), loc); |
| 4034 | if (failed(Result: kindOr)) |
| 4035 | return; |
| 4036 | kind = *kindOr; |
| 4037 | } |
| 4038 | |
| 4039 | if (hasThreadDim && reductionSharedBuf && |
| 4040 | op.getValue().getType() == |
| 4041 | cast<MemRefType>(Val: reductionSharedBuf.getType()).getElementType()) { |
| 4042 | Value val = op.getValue(); |
| 4043 | Value mem = op.getMemref(); |
| 4044 | Value tidY = getThreadId(loc, dim: gpu::Dimension::y); |
| 4045 | memref::AtomicRMWOp::create(builder&: rewriter, location: loc, kind: *kind, value: val, memref: reductionSharedBuf, |
| 4046 | indices: ValueRange{tidY}); |
| 4047 | createPerRowBarrier(loc); |
| 4048 | Value result = |
| 4049 | memref::LoadOp::create(builder&: rewriter, location: loc, memref: reductionSharedBuf, indices: tidY); |
| 4050 | memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: result, memref: mem); |
| 4051 | reductionAccumValue[mem] = result; |
| 4052 | } else if (hasThreadDim) { |
| 4053 | createGPUAllReduceOp(loc, input: op.getValue(), memref: op.getMemref(), kind: *kind, |
| 4054 | parDimsAttr: op.getParDims()); |
| 4055 | // Nothing replaced the accumulate, so leave it in place rather than |
| 4056 | // erasing it below. |
| 4057 | if (hasFailed) |
| 4058 | return; |
| 4059 | } else { |
| 4060 | // Block-only: no gpu.all_reduce needed (all threads have the same |
| 4061 | // value after broadcast). Just store the value to the memref. |
| 4062 | // Predicate on all thread dims when the target is shared memory. |
| 4063 | Value val = mapping.lookupOrDefault(from: op.getValue()); |
| 4064 | Value mem = mapping.lookupOrDefault(from: op.getMemref()); |
| 4065 | bool isPerThreadPrivate = isa_and_nonnull<memref::AllocaOp>( |
| 4066 | Val: unwrapMemRefConversion(v: mem).getDefiningOp()); |
| 4067 | if (!isPerThreadPrivate) { |
| 4068 | SmallVector<mlir::acc::GPUParallelDimAttr> predDims; |
| 4069 | for (auto parDim : computeRegion.getLaunchParDims()) |
| 4070 | if (!parDim.isAnyBlock()) |
| 4071 | predDims.push_back(Elt: parDim); |
| 4072 | if (predDims.empty()) { |
| 4073 | predDims.push_back(Elt: mlir::acc::GPUParallelDimAttr::threadXDim( |
| 4074 | computeRegion->getContext())); |
| 4075 | } |
| 4076 | Value predicate = emitPredicate(loc, inactiveParDims&: predDims); |
| 4077 | auto ifOp = |
| 4078 | scf::IfOp::create(builder&: rewriter, location: loc, cond: predicate, /*withElseRegion=*/false); |
| 4079 | rewriter.setInsertionPoint(ifOp.getThenRegion().back().getTerminator()); |
| 4080 | memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: val, memref: mem); |
| 4081 | rewriter.setInsertionPointAfter(ifOp); |
| 4082 | } else { |
| 4083 | memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: val, memref: mem); |
| 4084 | } |
| 4085 | } |
| 4086 | |
| 4087 | // erase acc.reduction_accumulate |
| 4088 | rewriter.eraseOp(op); |
| 4089 | } |
| 4090 | |
| 4091 | void ACCCGToGPULowering::postprocessLoopReduction(scf::ParallelOp parLoop) { |
| 4092 | if (parLoop.getNumReductions() == 0) |
| 4093 | return; |
| 4094 | |
| 4095 | for (unsigned i = 0; i < parLoop.getNumResults(); ++i) { |
| 4096 | for (Operation *user : |
| 4097 | mapping.lookupOrDefault(from: parLoop.getResult(i)).getUsers()) { |
| 4098 | if (acc::ReductionAccumulateOp accumulateOp = |
| 4099 | dyn_cast<acc::ReductionAccumulateOp>(Val: user)) { |
| 4100 | postprocessAccumulateOp(op: accumulateOp); |
| 4101 | } |
| 4102 | } |
| 4103 | } |
| 4104 | } |
| 4105 | |
| 4106 | void ACCCGToGPULowering::processExecuteRegion(scf::ExecuteRegionOp op) { |
| 4107 | LLVM_DEBUG(llvm::dbgs() << "processing execute region op: " ; |
| 4108 | op->print(llvm::dbgs()); llvm::dbgs() << "\n" ); |
| 4109 | Location loc = op->getLoc(); |
| 4110 | auto types = op.getResultTypes(); |
| 4111 | Region &oldRegion = op.getRegion(); |
| 4112 | // create the executeRegion op inside gpu launch |
| 4113 | auto executeRegionOp = scf::ExecuteRegionOp::create(builder&: rewriter, location: loc, resultType0: types); |
| 4114 | Region ®ion = executeRegionOp.getRegion(); |
| 4115 | rewriter.createBlock(parent: ®ion); |
| 4116 | rewriter.setInsertionPointToEnd(®ion.front()); |
| 4117 | |
| 4118 | llvm::DenseMap<Block *, Block *> blockMap; |
| 4119 | blockMap[&oldRegion.front()] = ®ion.front(); |
| 4120 | |
| 4121 | // Create blocks in the new operation corresponding to all blocks in the |
| 4122 | // original op |
| 4123 | for (auto &oldBlock : llvm::drop_begin(RangeOrContainer&: oldRegion.getBlocks())) { |
| 4124 | TypeRange argTypes = oldBlock.getArgumentTypes(); |
| 4125 | size_t numArgs = argTypes.size(); |
| 4126 | // Create new block with same argument types |
| 4127 | Block *newBlock = rewriter.createBlock(parent: ®ion, insertPt: region.end(), argTypes, |
| 4128 | locs: SmallVector<Location>(numArgs, loc)); |
| 4129 | blockMap[&oldBlock] = newBlock; |
| 4130 | // Map block arguments |
| 4131 | mapping.map(from: oldBlock.getArguments(), to: newBlock->getArguments()); |
| 4132 | } |
| 4133 | |
| 4134 | // Iterate over all blocks of oldRegion and all operations inside them |
| 4135 | // process all the ops except the terminator |
| 4136 | for (auto [oldBlock, newBlock] : |
| 4137 | llvm::zip(t&: oldRegion.getBlocks(), u&: region.getBlocks())) { |
| 4138 | OpBuilder::InsertionGuard blockGuard(rewriter); |
| 4139 | rewriter.setInsertionPointToStart(&newBlock); |
| 4140 | for (auto &bodyOp : oldBlock.getOperations()) { |
| 4141 | // Skip terminators during normal iteration - handle them separately |
| 4142 | if (bodyOp.hasTrait<OpTrait::IsTerminator>()) |
| 4143 | continue; |
| 4144 | processOp(op: &bodyOp); |
| 4145 | } |
| 4146 | |
| 4147 | // Copy the terminator from old block to new block |
| 4148 | Operation *oldTerminator = oldBlock.getTerminator(); |
| 4149 | rewriter.setInsertionPointToEnd(&newBlock); |
| 4150 | Operation *newTerminator = rewriter.clone(op&: *oldTerminator, mapper&: mapping); |
| 4151 | |
| 4152 | // Replace successors with mapped blocks |
| 4153 | for (unsigned i = 0; i < oldTerminator->getNumSuccessors(); ++i) { |
| 4154 | Block *oldDest = oldTerminator->getSuccessor(index: i); |
| 4155 | Block *newDest = blockMap.lookup(Val: oldDest); |
| 4156 | assert(newDest && "Successor block must be in blockMap" ); |
| 4157 | newTerminator->setSuccessor(block: newDest, index: i); |
| 4158 | } |
| 4159 | } |
| 4160 | mapping.map(from: op->getResults(), to: executeRegionOp->getResults()); |
| 4161 | rewriter.setInsertionPointAfter(executeRegionOp); |
| 4162 | } |
| 4163 | |
| 4164 | void ACCCGToGPULowering::processAccumulateOp(acc::ReductionAccumulateOp op) { |
| 4165 | LLVM_DEBUG(llvm::dbgs() << "processing accumulate op: " << *op << "\n" ); |
| 4166 | Value accumulateValue = op.getValue(); |
| 4167 | if (reductionSharedBuf && |
| 4168 | mapping.lookupOrDefault(from: accumulateValue).getType() == |
| 4169 | cast<MemRefType>(Val: reductionSharedBuf.getType()).getElementType()) { |
| 4170 | Location loc = op->getLoc(); |
| 4171 | FailureOr<arith::AtomicRMWKind> kind = getReductionKind( |
| 4172 | redOp: op.getReductionOperator(), type: accumulateValue.getType(), loc); |
| 4173 | if (failed(Result: kind)) |
| 4174 | return; |
| 4175 | Value mappedValue = mapping.lookupOrDefault(from: accumulateValue); |
| 4176 | Value memref = mapping.lookupOrDefault(from: op.getMemref()); |
| 4177 | Value tidY = getThreadId(loc, dim: gpu::Dimension::y); |
| 4178 | memref::AtomicRMWOp::create(builder&: rewriter, location: loc, kind: *kind, value: mappedValue, |
| 4179 | memref: reductionSharedBuf, indices: ValueRange{tidY}); |
| 4180 | createPerRowBarrier(loc); |
| 4181 | Value result = |
| 4182 | memref::LoadOp::create(builder&: rewriter, location: loc, memref: reductionSharedBuf, indices: tidY); |
| 4183 | memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: result, memref); |
| 4184 | reductionAccumValue[memref] = result; |
| 4185 | return; |
| 4186 | } |
| 4187 | if (accumulateValue.getDefiningOp<scf::ParallelOp>()) { |
| 4188 | Operation *newOp = rewriter.clone(op&: *op, mapper&: mapping); |
| 4189 | mapping.map(from: op->getResults(), to: newOp->getResults()); |
| 4190 | } else if (isRedundantChainAccumulate(op)) { |
| 4191 | // The destination memref already holds the correctly aggregated value |
| 4192 | // (atomically reduced by a preceding acc.reduction_combine with a block |
| 4193 | // par_dim). Skip the redundant gpu.all_reduce + atomic.update; emit a |
| 4194 | // Workgroup-wide barrier so downstream readers see all prior atomic |
| 4195 | // updates. |
| 4196 | LLVM_DEBUG(llvm::dbgs() << " skipped: redundant chain accumulate\n" ); |
| 4197 | gpu::BarrierOp::create(builder&: rewriter, location: op->getLoc()); |
| 4198 | } else { |
| 4199 | Value mappedValue = mapping.lookupOrDefault(from: accumulateValue); |
| 4200 | Value memref = mapping.lookupOrDefault(from: op.getMemref()); |
| 4201 | FailureOr<arith::AtomicRMWKind> kind = getReductionKind( |
| 4202 | redOp: op.getReductionOperator(), type: accumulateValue.getType(), loc: op.getLoc()); |
| 4203 | if (failed(Result: kind)) |
| 4204 | return; |
| 4205 | createGPUAllReduceOp(loc: op->getLoc(), input: mappedValue, memref, kind: *kind, |
| 4206 | parDimsAttr: op.getParDims()); |
| 4207 | } |
| 4208 | } |
| 4209 | |
| 4210 | /// True when \p v takes a distinct value per thread: it derives from a thread |
| 4211 | /// id, either directly or through the bounds of an enclosing loop. By this |
| 4212 | /// point a thread-mapped loop carries its mapping in its bounds rather than in |
| 4213 | /// `acc.par_dims`, so the bounds are what must be inspected. |
| 4214 | static bool isThreadVarying(Value v, ArrayRef<Value> threadIds, |
| 4215 | DenseSet<Value> &visited) { |
| 4216 | if (!v || !visited.insert(V: v).second) |
| 4217 | return false; |
| 4218 | if (llvm::is_contained(Range&: threadIds, Element: v)) |
| 4219 | return true; |
| 4220 | if (auto arg = dyn_cast<BlockArgument>(Val&: v)) { |
| 4221 | Operation *owner = arg.getOwner()->getParentOp(); |
| 4222 | unsigned dim = arg.getArgNumber(); |
| 4223 | if (auto loop = dyn_cast<scf::ParallelOp>(Val: owner)) { |
| 4224 | if (dim >= loop.getLowerBound().size()) |
| 4225 | return false; |
| 4226 | return isThreadVarying(v: loop.getLowerBound()[dim], threadIds, visited) || |
| 4227 | isThreadVarying(v: loop.getStep()[dim], threadIds, visited); |
| 4228 | } |
| 4229 | if (auto loop = dyn_cast<scf::ForOp>(Val: owner)) |
| 4230 | return dim == 0 && |
| 4231 | (isThreadVarying(v: loop.getLowerBound(), threadIds, visited) || |
| 4232 | isThreadVarying(v: loop.getStep(), threadIds, visited)); |
| 4233 | return false; |
| 4234 | } |
| 4235 | Operation *def = v.getDefiningOp(); |
| 4236 | if (!def) |
| 4237 | return false; |
| 4238 | if (isa<gpu::ThreadIdOp, gpu::LaneIdOp>(Val: def)) |
| 4239 | return true; |
| 4240 | return llvm::any_of(Range: def->getOperands(), P: [&](Value o) { |
| 4241 | return isThreadVarying(v: o, threadIds, visited); |
| 4242 | }); |
| 4243 | } |
| 4244 | |
| 4245 | /// Strips view and memory-space-cast chains to the underlying buffer. |
| 4246 | static Value accumulatorRoot(Value v) { |
| 4247 | while (Operation *op = v.getDefiningOp()) { |
| 4248 | if (auto cast = dyn_cast<memref::MemorySpaceCastOp>(Val: op)) { |
| 4249 | v = cast.getSource(); |
| 4250 | continue; |
| 4251 | } |
| 4252 | if (auto viewLike = dyn_cast<ViewLikeOpInterface>(Val: op)) { |
| 4253 | if (isa<MemRefType>(Val: viewLike.getViewSource().getType())) { |
| 4254 | v = viewLike.getViewSource(); |
| 4255 | continue; |
| 4256 | } |
| 4257 | } |
| 4258 | break; |
| 4259 | } |
| 4260 | return v; |
| 4261 | } |
| 4262 | |
| 4263 | /// Matches `%l = load %m[%i]` / `%c = combine(%l, %x)` / `store %c, %m[%i]` on |
| 4264 | /// the accumulator \p accum and returns the contributed value `%x`. |
| 4265 | static Value matchAccumulatorUpdate(memref::StoreOp store, Value accum) { |
| 4266 | if (accumulatorRoot(v: store.getMemRef()) != accum) |
| 4267 | return {}; |
| 4268 | Operation *combine = store.getValueToStore().getDefiningOp(); |
| 4269 | if (!combine || combine->getNumOperands() != 2) |
| 4270 | return {}; |
| 4271 | for (unsigned i = 0; i != 2; ++i) { |
| 4272 | auto load = combine->getOperand(idx: i).getDefiningOp<memref::LoadOp>(); |
| 4273 | if (!load || accumulatorRoot(v: load.getMemRef()) != accum) |
| 4274 | continue; |
| 4275 | if (!llvm::equal(LRange: load.getIndices(), RRange: store.getIndices())) |
| 4276 | continue; |
| 4277 | return combine->getOperand(idx: 1 - i); |
| 4278 | } |
| 4279 | return {}; |
| 4280 | } |
| 4281 | |
| 4282 | /// A block-shared accumulator is updated in place by the loop body, so several |
| 4283 | /// threads may hit the same element. Make those updates atomic unless the |
| 4284 | /// element index provably varies across the participating threads. |
| 4285 | static void atomicizeSharedAccumulatorUpdates(Value accum, |
| 4286 | arith::AtomicRMWKind kind, |
| 4287 | ArrayRef<Value> threadIds, |
| 4288 | RewriterBase &rewriter) { |
| 4289 | OpBuilder::InsertionGuard guard(rewriter); |
| 4290 | SmallVector<memref::StoreOp> stores; |
| 4291 | SmallVector<Value> worklist{accum}; |
| 4292 | DenseSet<Value> seen; |
| 4293 | while (!worklist.empty()) { |
| 4294 | Value cur = worklist.pop_back_val(); |
| 4295 | if (!seen.insert(V: cur).second) |
| 4296 | continue; |
| 4297 | for (Operation *user : cur.getUsers()) { |
| 4298 | if (auto store = dyn_cast<memref::StoreOp>(Val: user)) |
| 4299 | stores.push_back(Elt: store); |
| 4300 | else if (isa<ViewLikeOpInterface, memref::MemorySpaceCastOp>(Val: user)) |
| 4301 | llvm::append_range(C&: worklist, R: user->getResults()); |
| 4302 | } |
| 4303 | } |
| 4304 | |
| 4305 | for (memref::StoreOp store : stores) { |
| 4306 | Value contribution = matchAccumulatorUpdate(store, accum); |
| 4307 | if (!contribution) |
| 4308 | continue; |
| 4309 | // A thread-varying index means each thread owns its element, so the |
| 4310 | // existing plain update is already race-free. |
| 4311 | if (llvm::any_of(Range: store.getIndices(), P: [&](Value idx) { |
| 4312 | DenseSet<Value> visited; |
| 4313 | return isThreadVarying(v: idx, threadIds, visited); |
| 4314 | })) |
| 4315 | continue; |
| 4316 | Operation *combine = store.getValueToStore().getDefiningOp(); |
| 4317 | rewriter.setInsertionPoint(store); |
| 4318 | memref::AtomicRMWOp::create(builder&: rewriter, location: store.getLoc(), kind, value: contribution, |
| 4319 | memref: store.getMemRef(), indices: store.getIndices()); |
| 4320 | rewriter.eraseOp(op: store); |
| 4321 | if (combine && combine->use_empty()) |
| 4322 | rewriter.eraseOp(op: combine); |
| 4323 | } |
| 4324 | } |
| 4325 | |
| 4326 | void ACCCGToGPULowering::processAccumulateArrayOp( |
| 4327 | acc::ReductionAccumulateArrayOp op) { |
| 4328 | LLVM_DEBUG(llvm::dbgs() << "processing accumulate array op: " << *op << "\n" ); |
| 4329 | Location loc = op.getLoc(); |
| 4330 | |
| 4331 | Value memref = mapping.lookupOrDefault(from: op.getMemref()); |
| 4332 | MemRefType memrefTy = dyn_cast<MemRefType>(Val: memref.getType()); |
| 4333 | if (!memrefTy) { |
| 4334 | (void)accSupport.emitNYI(loc, message: "reduction: non-MemRefTy accumulate array" ); |
| 4335 | return; |
| 4336 | } |
| 4337 | FailureOr<arith::AtomicRMWKind> kindOr = getReductionKind( |
| 4338 | redOp: op.getReductionOperator(), type: memrefTy.getElementType(), loc); |
| 4339 | if (failed(Result: kindOr)) |
| 4340 | return; |
| 4341 | arith::AtomicRMWKind kind = *kindOr; |
| 4342 | |
| 4343 | // The (already mapped/cloned) acc.bounds op describes the element range; it |
| 4344 | // is dead after lowering since we read its operands directly. |
| 4345 | acc::DataBoundsOp boundsOp = mapping.lookupOrDefault(from: op.getBounds()) |
| 4346 | .getDefiningOp<acc::DataBoundsOp>(); |
| 4347 | assert(boundsOp && "expected acc.bounds defining op for array accumulate" ); |
| 4348 | auto eraseDeadBounds = [&] { |
| 4349 | if (boundsOp->use_empty()) |
| 4350 | rewriter.eraseOp(op: boundsOp); |
| 4351 | }; |
| 4352 | |
| 4353 | auto isUnitExtent = [&](mlir::acc::GPUParallelDimAttr parDim) { |
| 4354 | std::optional<uint64_t> width = |
| 4355 | computeRegion.getKnownConstantLaunchArg(parDim); |
| 4356 | return width && *width == 1; |
| 4357 | }; |
| 4358 | |
| 4359 | // A dimension launched one wide partitions nothing, so skip it. |
| 4360 | bool hasThreadDim = false; |
| 4361 | for (auto pd : op.getParDims().getArray()) { |
| 4362 | if (isUnitExtent(pd)) |
| 4363 | continue; |
| 4364 | hasThreadDim |= pd.isAnyThread(); |
| 4365 | } |
| 4366 | |
| 4367 | // Without a thread dimension no two threads share an accumulator copy, so |
| 4368 | // each copy already holds a complete total and nothing has to be merged |
| 4369 | // here. Merging across gangs, if any, is left to the combine. |
| 4370 | if (!hasThreadDim) { |
| 4371 | eraseDeadBounds(); |
| 4372 | return; |
| 4373 | } |
| 4374 | |
| 4375 | // Per-element gpu.all_reduce is only correct when each thread owns its own |
| 4376 | // accumulator copy. For a statically-shaped accumulator, classify from the |
| 4377 | // operand: an explicit shared/heap allocation is block-shared regardless of |
| 4378 | // size, and a stack alloca (or a view over one) is per-thread when it fits |
| 4379 | // the per-thread stack budget. Storage `acc.par_dims` without thread_x means |
| 4380 | // gang-/worker-scoped privacy (shared among vector lanes) even if the type |
| 4381 | // would fit on the stack. For a dynamically-shaped accumulator the type |
| 4382 | // conveys no size, so classify from storage/accumulate thread_x dims. |
| 4383 | auto storageIsThreadXPrivate = [&](Value v) -> bool { |
| 4384 | acc::PrivateLocalOp privateLocal = getPrivateLocalForMemref(memref: v); |
| 4385 | GPUParallelDimsAttr dims = |
| 4386 | privateLocal ? getPrivateParDims(privateLocal, computeRegion) |
| 4387 | : GPUParallelDimsAttr(); |
| 4388 | if (!dims) { |
| 4389 | if (Operation *root = unwrapMemRefConversion(v).getDefiningOp()) |
| 4390 | dims = getParDimsAttr(op: root); |
| 4391 | } |
| 4392 | return !dims || |
| 4393 | llvm::any_of(Range: dims.getArray(), P: [](auto d) { return d.isThreadX(); }); |
| 4394 | }; |
| 4395 | Operation *rootOp = unwrapMemRefConversion(v: memref).getDefiningOp(); |
| 4396 | bool isSharedStorage = isa_and_nonnull<memref::AllocOp>(Val: rootOp) || |
| 4397 | isa_and_nonnull<acc::GPUSharedMemoryOp>(Val: rootOp); |
| 4398 | if (auto addrSpace = dyn_cast_if_present<gpu::AddressSpaceAttr>( |
| 4399 | Val: memrefTy.getMemorySpace())) { |
| 4400 | isSharedStorage |= |
| 4401 | addrSpace.getValue() == gpu::GPUDialect::getWorkgroupAddressSpace(); |
| 4402 | } |
| 4403 | bool isPerThreadPrivate = |
| 4404 | !isSharedStorage && storageIsThreadXPrivate(op.getMemref()) && |
| 4405 | (memrefTy.hasStaticShape() |
| 4406 | ? canUseStackAlloca(baseTy: memrefTy, loc, maxThreadPrivateStack: options.maxThreadPrivateStack) |
| 4407 | : llvm::any_of(Range: op.getParDims().getArray(), |
| 4408 | P: [](mlir::acc::GPUParallelDimAttr d) { |
| 4409 | return d.isThreadX(); |
| 4410 | })); |
| 4411 | if (!isPerThreadPrivate) { |
| 4412 | // Block-shared accumulator: the body already updated it in place, so the |
| 4413 | // block partial is complete and the atomic combine finishes across blocks. |
| 4414 | // Threads that share an element must not race, so their in-place updates |
| 4415 | // become atomic. |
| 4416 | SmallVector<Value> threadIds; |
| 4417 | if (Value xId = getGPUThreadIdFor(proc: gpu::Processor::ThreadX)) |
| 4418 | threadIds.push_back(Elt: xId); |
| 4419 | if (Value yId = getGPUThreadIdFor(proc: gpu::Processor::ThreadY)) |
| 4420 | threadIds.push_back(Elt: yId); |
| 4421 | if (Value zId = getGPUThreadIdFor(proc: gpu::Processor::ThreadZ)) |
| 4422 | threadIds.push_back(Elt: zId); |
| 4423 | atomicizeSharedAccumulatorUpdates(accum: accumulatorRoot(v: memref), kind, threadIds, |
| 4424 | rewriter); |
| 4425 | eraseDeadBounds(); |
| 4426 | return; |
| 4427 | } |
| 4428 | |
| 4429 | // Bounds are normalized to be zero-based. |
| 4430 | auto toIndex = [&](Value v) -> Value { |
| 4431 | if (v.getType().isIndex()) |
| 4432 | return v; |
| 4433 | return arith::IndexCastOp::create(builder&: rewriter, location: loc, out: rewriter.getIndexType(), |
| 4434 | in: v); |
| 4435 | }; |
| 4436 | |
| 4437 | Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 4438 | Value one = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1); |
| 4439 | Value lb = |
| 4440 | boundsOp.getLowerbound() ? toIndex(boundsOp.getLowerbound()) : zero; |
| 4441 | Value step = boundsOp.getStride() ? toIndex(boundsOp.getStride()) : one; |
| 4442 | // Exclusive upper bound. `extent` counts elements, so the span is |
| 4443 | // `extent * step` (for the common unit-stride case step is 1); fall back to |
| 4444 | // the inclusive upperbound when no extent is given. |
| 4445 | Value ub; |
| 4446 | if (boundsOp.getExtent()) { |
| 4447 | Value span = arith::MulIOp::create(builder&: rewriter, location: loc, |
| 4448 | lhs: toIndex(boundsOp.getExtent()), rhs: step); |
| 4449 | ub = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: lb, rhs: span); |
| 4450 | } else { |
| 4451 | assert(boundsOp.getUpperbound() && |
| 4452 | "acc.bounds must specify an extent or upperbound" ); |
| 4453 | ub = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: toIndex(boundsOp.getUpperbound()), |
| 4454 | rhs: one); |
| 4455 | } |
| 4456 | |
| 4457 | // Reduce each array element across the requested parallel dimensions. |
| 4458 | auto forOp = scf::ForOp::create(builder&: rewriter, location: loc, lowerBound: lb, upperBound: ub, step); |
| 4459 | { |
| 4460 | OpBuilder::InsertionGuard guard(rewriter); |
| 4461 | rewriter.setInsertionPoint(forOp.getBody()->getTerminator()); |
| 4462 | Value iv = forOp.getInductionVar(); |
| 4463 | SmallVector<Value> indices{iv}; |
| 4464 | if (memrefTy.getRank() > 1) { |
| 4465 | indices.resize(N: memrefTy.getRank()); |
| 4466 | Value linearIndex = iv; |
| 4467 | for (int64_t dim = memrefTy.getRank() - 1; dim >= 0; --dim) { |
| 4468 | Value dimSize = |
| 4469 | memrefTy.isDynamicDim(idx: dim) |
| 4470 | ? memref::DimOp::create(builder&: rewriter, location: loc, source: memref, index: dim).getResult() |
| 4471 | : arith::ConstantIndexOp::create(builder&: rewriter, location: loc, |
| 4472 | value: memrefTy.getDimSize(idx: dim)) |
| 4473 | .getResult(); |
| 4474 | indices[dim] = |
| 4475 | arith::RemUIOp::create(builder&: rewriter, location: loc, lhs: linearIndex, rhs: dimSize); |
| 4476 | if (dim != 0) |
| 4477 | linearIndex = |
| 4478 | arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: linearIndex, rhs: dimSize); |
| 4479 | } |
| 4480 | } |
| 4481 | Value elem = memref::LoadOp::create(builder&: rewriter, location: loc, memref, indices); |
| 4482 | createGPUAllReduceOp(loc, input: elem, memref, kind, parDimsAttr: op.getParDims(), indices, |
| 4483 | /*isPerThreadPrivateTarget=*/true); |
| 4484 | } |
| 4485 | |
| 4486 | eraseDeadBounds(); |
| 4487 | } |
| 4488 | |
| 4489 | void ACCCGToGPULowering::processReductionOp(acc::ReductionInitOp op) { |
| 4490 | // Clone the inner ops of the reduction op only |
| 4491 | op.getRegion().walk<WalkOrder::PreOrder>(callback: [&](Operation *innerOp) { |
| 4492 | if (acc::YieldOp yieldOp = dyn_cast<acc::YieldOp>(Val: innerOp)) { |
| 4493 | op.getResult().replaceAllUsesWith(newValue: mapping.lookup(from: yieldOp.getOperand(i: 0))); |
| 4494 | return WalkResult::interrupt(); |
| 4495 | } |
| 4496 | if (innerOp->getNumRegions() > 0) { |
| 4497 | processOp(op: innerOp); |
| 4498 | return WalkResult::skip(); |
| 4499 | } |
| 4500 | rewriter.clone(op&: *innerOp, mapper&: mapping); |
| 4501 | return WalkResult::advance(); |
| 4502 | }); |
| 4503 | } |
| 4504 | |
| 4505 | void ACCCGToGPULowering::processReductionCombineOp(acc::ReductionCombineOp op) { |
| 4506 | LLVM_DEBUG(llvm::dbgs() << "processing reduction combine op: " ; |
| 4507 | op->print(llvm::dbgs()); llvm::dbgs() << "\n" ); |
| 4508 | Location loc = op.getLoc(); |
| 4509 | MemRefType memrefType = dyn_cast<MemRefType>(Val: op.getSrcMemref().getType()); |
| 4510 | assert(memrefType && "expected memref type for reduction combine op" ); |
| 4511 | assert(memrefType.getRank() == 0 && |
| 4512 | "expected scalar memref type for reduction combine op" ); |
| 4513 | Type elTy = memrefType.getElementType(); |
| 4514 | FailureOr<arith::AtomicRMWKind> kindOr = |
| 4515 | getReductionKind(redOp: op.getReductionOperator(), type: elTy, loc); |
| 4516 | if (failed(Result: kindOr)) |
| 4517 | return; |
| 4518 | arith::AtomicRMWKind kind = *kindOr; |
| 4519 | |
| 4520 | Value srcMemref = mapping.lookupOrDefault(from: op.getSrcMemref()); |
| 4521 | Value destMemref = mapping.lookupOrDefault(from: op.getDestMemref()); |
| 4522 | |
| 4523 | // A block par_dim normally means the accumulator is shared across blocks and |
| 4524 | // must be updated atomically. But when the destination resolves to a |
| 4525 | // thread-private stack alloca (e.g. an inner-loop reduction combining one |
| 4526 | // private accumulator into another private accumulator for the same thread), |
| 4527 | // the update is not visible to other threads and must not be atomic. Treat |
| 4528 | // such destinations as a plain load/combine/store below. |
| 4529 | bool destIsPerThreadPrivate = isa_and_nonnull<memref::AllocaOp>( |
| 4530 | Val: unwrapMemRefConversion(v: destMemref).getDefiningOp()); |
| 4531 | |
| 4532 | SmallVector<mlir::acc::GPUParallelDimAttr> parDims = |
| 4533 | getReductionCombineParDims(op); |
| 4534 | // An atomic is needed exactly when more than one block can reach the |
| 4535 | // destination. A block dimension in the reduction's parallel dims implies |
| 4536 | // that; so does a thread-only reduction in a multi-block launch, where every |
| 4537 | // block combines its own partial into the same shared slot. |
| 4538 | bool destIsGridShared = |
| 4539 | !destIsPerThreadPrivate && |
| 4540 | (llvm::any_of(Range&: parDims, |
| 4541 | P: [](mlir::acc::GPUParallelDimAttr parDim) { |
| 4542 | return parDim.isAnyBlock(); |
| 4543 | }) || |
| 4544 | computeRegion.launchesMultipleBlocks()); |
| 4545 | if (destIsGridShared) { |
| 4546 | // The predication (tid.x == 0 when subgroup-aligned) is already handled |
| 4547 | // by the parent predicate_region processing. |
| 4548 | // Reloading a grid-shared slot races with other blocks; record it and |
| 4549 | // replace with the block-reduced register value in the fixup. |
| 4550 | auto srcLoad = |
| 4551 | memref::LoadOp::create(builder&: rewriter, location: loc, memref: srcMemref, indices: ValueRange{}); |
| 4552 | pendingCombineReloads.push_back(Elt: {srcMemref, srcLoad}); |
| 4553 | constructAtomicAccumulation(loc, memref: destMemref, /*indices=*/{}, input: srcLoad, kind); |
| 4554 | return; |
| 4555 | } |
| 4556 | |
| 4557 | // Atomic construction is not needed; lower this operation to typical |
| 4558 | // reduction update operations. E.g. dest = dest <kind> src |
| 4559 | auto srcLoad = memref::LoadOp::create(builder&: rewriter, location: loc, memref: srcMemref, indices: ValueRange{}); |
| 4560 | auto destLoad = |
| 4561 | memref::LoadOp::create(builder&: rewriter, location: loc, memref: destMemref, indices: ValueRange{}); |
| 4562 | Value combine = generateReductionOp(b&: rewriter, loc, lhs: srcLoad, rhs: destLoad, kind); |
| 4563 | memref::StoreOp::create(builder&: rewriter, location: loc, valueToStore: combine, memref: destMemref, indices: ValueRange{}); |
| 4564 | } |
| 4565 | |
| 4566 | void ACCCGToGPULowering::processCombineRegionOp( |
| 4567 | acc::ReductionCombineRegionOp op) { |
| 4568 | LLVM_DEBUG(llvm::dbgs() << "processing combine region op: " ; |
| 4569 | op->print(llvm::dbgs()); llvm::dbgs() << "\n" ); |
| 4570 | // A block par_dim on a combine into a thread-private stack alloca is not a |
| 4571 | // real cross-block accumulation (the alloca is not shared across blocks), so |
| 4572 | // it must use a plain load/combine/store rather than an atomic update. |
| 4573 | bool destIsPerThreadPrivate = isa_and_nonnull<memref::AllocaOp>( |
| 4574 | Val: unwrapMemRefConversion(v: mapping.lookupOrDefault(from: op.getDestVar())) |
| 4575 | .getDefiningOp()); |
| 4576 | SmallVector<mlir::acc::GPUParallelDimAttr> parDims = |
| 4577 | getReductionCombineParDims(op); |
| 4578 | for (auto parDim : parDims) { |
| 4579 | if (parDim.isAnyBlock() && !destIsPerThreadPrivate) { |
| 4580 | // Block reduction directly stores to the accumulator using atomic. |
| 4581 | // The predication (tid.x == 0 when subgroup-aligned) is already handled |
| 4582 | // by the parent predicate_region processing. |
| 4583 | for (Operation *user : op.getSrcVar().getUsers()) { |
| 4584 | if (acc::ReductionAccumulateOp accumulateOp = |
| 4585 | dyn_cast<acc::ReductionAccumulateOp>(Val: user)) { |
| 4586 | Location loc = accumulateOp.getLoc(); |
| 4587 | FailureOr<arith::AtomicRMWKind> kind = |
| 4588 | getReductionKind(redOp: accumulateOp.getReductionOperator(), |
| 4589 | type: accumulateOp.getValue().getType(), loc); |
| 4590 | if (failed(Result: kind)) |
| 4591 | return; |
| 4592 | Value srcMemref = mapping.lookupOrDefault(from: accumulateOp.getMemref()); |
| 4593 | // Recorded and patched in the fixup to avoid the reload race. |
| 4594 | auto reductionLoad = |
| 4595 | memref::LoadOp::create(builder&: rewriter, location: loc, memref: srcMemref, indices: ValueRange{}); |
| 4596 | pendingCombineReloads.push_back(Elt: {srcMemref, reductionLoad}); |
| 4597 | constructAtomicAccumulation(loc, |
| 4598 | memref: mapping.lookupOrDefault(from: op.getDestVar()), |
| 4599 | /*indices=*/{}, input: reductionLoad, kind: *kind); |
| 4600 | return; |
| 4601 | } |
| 4602 | } |
| 4603 | // For decomposed complex reductions, the AccumulateOp was replaced |
| 4604 | // with real/imag AccumulateOps. Load from the private memref which |
| 4605 | // holds the reconstructed complex value. |
| 4606 | Value privateMemref = mapping.lookupOrDefault(from: op.getSrcVar()); |
| 4607 | MemRefType memrefTy = cast<MemRefType>(Val: privateMemref.getType()); |
| 4608 | if (isa<ComplexType>(Val: memrefTy.getElementType())) { |
| 4609 | Location loc = op.getLoc(); |
| 4610 | Value reductionResult = |
| 4611 | memref::LoadOp::create(builder&: rewriter, location: loc, memref: privateMemref, indices: ValueRange{}); |
| 4612 | arith::AtomicRMWKind kind = arith::AtomicRMWKind::addf; |
| 4613 | op.getRegion().walk(callback: [&](Operation *innerOp) { |
| 4614 | if (isa<complex::MulOp>(Val: innerOp)) |
| 4615 | kind = arith::AtomicRMWKind::mulf; |
| 4616 | }); |
| 4617 | constructAtomicAccumulation(loc, |
| 4618 | memref: mapping.lookupOrDefault(from: op.getDestVar()), |
| 4619 | /*indices=*/{}, input: reductionResult, kind); |
| 4620 | return; |
| 4621 | } |
| 4622 | } |
| 4623 | } |
| 4624 | op.getRegion().walk<WalkOrder::PreOrder>(callback: [&](Operation *innerOp) { |
| 4625 | if (acc::YieldOp yieldOp = dyn_cast<acc::YieldOp>(Val: innerOp)) |
| 4626 | return WalkResult::interrupt(); |
| 4627 | if (innerOp->getNumRegions() > 0) { |
| 4628 | processOp(op: innerOp); |
| 4629 | return WalkResult::skip(); |
| 4630 | } |
| 4631 | rewriter.clone(op&: *innerOp, mapper&: mapping); |
| 4632 | return WalkResult::advance(); |
| 4633 | }); |
| 4634 | } |
| 4635 | |
| 4636 | void ACCCGToGPULowering::processGenericOp(Operation *op) { |
| 4637 | // Operations with no regions or operations for which we know |
| 4638 | // no recursive processing is needed can be fully cloned. |
| 4639 | LLVM_DEBUG(llvm::dbgs() << "processing generic op, cloning: " ; |
| 4640 | op->print(llvm::dbgs()); llvm::dbgs() << "\n" ); |
| 4641 | Operation *newOp = rewriter.clone(op&: *op, mapper&: mapping); |
| 4642 | // update mapping as cloning creates different result values |
| 4643 | mapping.map(from: op->getResults(), to: newOp->getResults()); |
| 4644 | } |
| 4645 | |
| 4646 | void ACCCGToGPULowering::processGenericOpWithRegions(Operation *op) { |
| 4647 | // Generic handling for operations with regions |
| 4648 | LLVM_DEBUG(llvm::dbgs() << "processing generic op with regions: " ; |
| 4649 | op->print(llvm::dbgs()); llvm::dbgs() << "\n" ); |
| 4650 | |
| 4651 | // Clone the operation structure without its regions |
| 4652 | Operation *newOp = rewriter.cloneWithoutRegions(op&: *op, mapper&: mapping); |
| 4653 | |
| 4654 | // Process each region recursively |
| 4655 | for (auto [oldRegion, newRegion] : |
| 4656 | llvm::zip(t: op->getRegions(), u: newOp->getRegions())) { |
| 4657 | // Create blocks in the new region corresponding to old region blocks |
| 4658 | for (auto &oldBlock : oldRegion.getBlocks()) { |
| 4659 | TypeRange argTypes = oldBlock.getArgumentTypes(); |
| 4660 | size_t numArgs = argTypes.size(); |
| 4661 | // Create new block with same argument types |
| 4662 | Block *newBlock = |
| 4663 | rewriter.createBlock(parent: &newRegion, insertPt: newRegion.end(), argTypes, |
| 4664 | locs: SmallVector<Location>(numArgs, op->getLoc())); |
| 4665 | |
| 4666 | // Map block arguments |
| 4667 | mapping.map(from: oldBlock.getArguments(), to: newBlock->getArguments()); |
| 4668 | |
| 4669 | // Process each operation in the block |
| 4670 | for (auto &innerOp : oldBlock.getOperations()) { |
| 4671 | OpBuilder::InsertionGuard guard(rewriter); |
| 4672 | rewriter.setInsertionPointToEnd(newBlock); |
| 4673 | processOp(op: &innerOp); |
| 4674 | } |
| 4675 | } |
| 4676 | } |
| 4677 | rewriter.setInsertionPointAfter(newOp); |
| 4678 | |
| 4679 | // Update mapping for results |
| 4680 | mapping.map(from: op->getResults(), to: newOp->getResults()); |
| 4681 | } |
| 4682 | |
| 4683 | // thread through par dim to verify redundant/0 execution modes |
| 4684 | void ACCCGToGPULowering::processOp(Operation *op) { |
| 4685 | if (isDeferredBarrierFlushPoint(op)) |
| 4686 | flushDeferredBarriersBefore(beforeOp: op); |
| 4687 | if (mlir::acc::hasParDimsAttr(op) && isa<scf::ParallelOp>(Val: op)) { |
| 4688 | // parallel loops require special processing based on parallel dimension |
| 4689 | // this is mutually recursive with processOp |
| 4690 | scf::ParallelOp parallelOp = cast<scf::ParallelOp>(Val: op); |
| 4691 | processParallelOp(parallelOp); |
| 4692 | } else if (scf::ForOp seqLoop = dyn_cast<scf::ForOp>(Val: op)) { |
| 4693 | processSeqLoop(loopOp: seqLoop); |
| 4694 | } else if (acc::PrivatizeOp privatize = dyn_cast<acc::PrivatizeOp>(Val: op)) { |
| 4695 | processPrivatize(privatize); |
| 4696 | } else if (acc::PrivateLocalOp privateLocal = |
| 4697 | dyn_cast<acc::PrivateLocalOp>(Val: op)) { |
| 4698 | processPrivateLocal(privateLocal); |
| 4699 | } else if (acc::PredicateRegionOp predicateRegionOp = |
| 4700 | dyn_cast<acc::PredicateRegionOp>(Val: op)) { |
| 4701 | processPredicateRegion(interOp: predicateRegionOp); |
| 4702 | } else if (acc::ReductionAccumulateOp accumulateOp = |
| 4703 | dyn_cast<acc::ReductionAccumulateOp>(Val: op)) { |
| 4704 | processAccumulateOp(op: accumulateOp); |
| 4705 | } else if (auto accumulateArrayOp = |
| 4706 | dyn_cast<acc::ReductionAccumulateArrayOp>(Val: op)) { |
| 4707 | processAccumulateArrayOp(op: accumulateArrayOp); |
| 4708 | } else if (acc::ReductionInitOp reductionInitOp = |
| 4709 | dyn_cast<acc::ReductionInitOp>(Val: op)) { |
| 4710 | processReductionOp(op: reductionInitOp); |
| 4711 | } else if (acc::ReductionCombineOp reductionCombineOp = |
| 4712 | dyn_cast<acc::ReductionCombineOp>(Val: op)) { |
| 4713 | processReductionCombineOp(op: reductionCombineOp); |
| 4714 | } else if (auto combineRegionOp = |
| 4715 | dyn_cast<acc::ReductionCombineRegionOp>(Val: op)) { |
| 4716 | processCombineRegionOp(op: combineRegionOp); |
| 4717 | } else if (acc::ReductionOp accReductionOp = dyn_cast<acc::ReductionOp>(Val: op)) { |
| 4718 | mapping.map(from: accReductionOp->getResult(idx: 0), to: accReductionOp.getVarPtr()); |
| 4719 | } else if (mapping.contains(from: op)) { |
| 4720 | // do nothing, operation in mapping signals it is already taken care of |
| 4721 | LLVM_DEBUG(llvm::dbgs() << "skipping mapped op: " << *op << "\n" ); |
| 4722 | } else if (isa<acc::YieldOp>(Val: op)) { |
| 4723 | for (auto [operand, result] : |
| 4724 | llvm::zip(t: op->getOperands(), u: op->getParentOp()->getResults())) { |
| 4725 | result.replaceAllUsesWith(newValue: mapping.lookup(from: operand)); |
| 4726 | } |
| 4727 | } else if (isa<scf::ExecuteRegionOp>(Val: op)) { |
| 4728 | processExecuteRegion(op: cast<scf::ExecuteRegionOp>(Val: op)); |
| 4729 | } else if (op->getNumRegions() == 0 || |
| 4730 | isa<acc::OpenACCDialect>(Val: op->getDialect())) { |
| 4731 | processGenericOp(op); |
| 4732 | } else { |
| 4733 | processGenericOpWithRegions(op); |
| 4734 | } |
| 4735 | } |
| 4736 | |
| 4737 | /// Fold `acc.par_width` to its launch operand or constant one. |
| 4738 | class RemoveParWidth : public OpRewritePattern<acc::ParWidthOp> { |
| 4739 | using OpRewritePattern<acc::ParWidthOp>::OpRewritePattern; |
| 4740 | LogicalResult matchAndRewrite(acc::ParWidthOp op, |
| 4741 | PatternRewriter &rewriter) const override { |
| 4742 | if (Value launchArg = op.getLaunchArg()) { |
| 4743 | rewriter.replaceOp(op, newValues: launchArg); |
| 4744 | } else { |
| 4745 | Value one = arith::ConstantIndexOp::create(builder&: rewriter, location: op.getLoc(), value: 1); |
| 4746 | rewriter.replaceOp(op, newValues: one); |
| 4747 | } |
| 4748 | return success(); |
| 4749 | } |
| 4750 | }; |
| 4751 | |
| 4752 | /// Rewrite pattern that lowers `acc.compute_region` via ACCCGToGPULowering. |
| 4753 | class ACCComputeRegionToGPUPattern |
| 4754 | : public OpRewritePattern<acc::ComputeRegionOp> { |
| 4755 | public: |
| 4756 | ACCComputeRegionToGPUPattern(MLIRContext *context, |
| 4757 | acc::OpenACCSupport &accSupport, |
| 4758 | const ACCCGToGPUOptions &options) |
| 4759 | : OpRewritePattern<acc::ComputeRegionOp>(context), accSupport(accSupport), |
| 4760 | options(options) {} |
| 4761 | |
| 4762 | LogicalResult matchAndRewrite(acc::ComputeRegionOp op, |
| 4763 | PatternRewriter &rewriter) const override { |
| 4764 | ACCCGToGPULowering kernelOpRewriter(op, rewriter, accSupport, options); |
| 4765 | return kernelOpRewriter.rewrite(); |
| 4766 | } |
| 4767 | |
| 4768 | private: |
| 4769 | acc::OpenACCSupport &accSupport; |
| 4770 | const ACCCGToGPUOptions &options; |
| 4771 | }; |
| 4772 | |
| 4773 | class ACCCGToGPU : public acc::impl::ACCCGToGPUBase<ACCCGToGPU> { |
| 4774 | public: |
| 4775 | using acc::impl::ACCCGToGPUBase<ACCCGToGPU>::ACCCGToGPUBase; |
| 4776 | |
| 4777 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 4778 | acc::impl::ACCCGToGPUBase<ACCCGToGPU>::getDependentDialects(registry); |
| 4779 | acc::registerAllExtensions(registry); |
| 4780 | } |
| 4781 | |
| 4782 | void runOnOperation() override { |
| 4783 | FunctionOpInterface funcOp = getOperation(); |
| 4784 | MLIRContext *context = funcOp->getContext(); |
| 4785 | |
| 4786 | assert(deviceType != mlir::acc::DeviceType::Host && |
| 4787 | deviceType != mlir::acc::DeviceType::Multicore && |
| 4788 | "ACCCGToGPU only supports GPU device types" ); |
| 4789 | |
| 4790 | // Validate parallel-loop mappings before any compute region is rewritten. |
| 4791 | // Loops outside compute regions do not require GPU mappings. |
| 4792 | WalkResult result = funcOp->walk(callback: [](scf::ParallelOp loop) { |
| 4793 | if (!loop->getParentOfType<acc::ComputeRegionOp>() || |
| 4794 | mlir::acc::hasParDimsAttr(op: loop)) |
| 4795 | return WalkResult::advance(); |
| 4796 | loop.emitOpError(message: "requires an 'acc.par_dims' attribute" ); |
| 4797 | return WalkResult::interrupt(); |
| 4798 | }); |
| 4799 | if (result.wasInterrupted()) { |
| 4800 | signalPassFailure(); |
| 4801 | return; |
| 4802 | } |
| 4803 | |
| 4804 | ACCCGToGPUOptions options; |
| 4805 | options.deviceType = deviceType; |
| 4806 | options.maxWorkgroupSharedMemory = maxWorkgroupSharedMemory; |
| 4807 | options.maxThreadPrivateStack = maxThreadPrivateStack; |
| 4808 | options.subgroupSize = subgroupSize; |
| 4809 | |
| 4810 | // Try to get cached parent analysis first, fall back to local analysis. |
| 4811 | std::optional<std::reference_wrapper<acc::OpenACCSupport>> cachedAnalysis = |
| 4812 | getCachedParentAnalysis<acc::OpenACCSupport>(parent: funcOp->getParentOp()); |
| 4813 | acc::OpenACCSupport &accSupport = cachedAnalysis |
| 4814 | ? cachedAnalysis->get() |
| 4815 | : getAnalysis<acc::OpenACCSupport>(); |
| 4816 | |
| 4817 | RewritePatternSet patterns(context); |
| 4818 | patterns.insert<ACCComputeRegionToGPUPattern>(arg&: context, args&: accSupport, args&: options); |
| 4819 | patterns.insert<RemoveParWidth>(arg&: context); |
| 4820 | ConversionTarget target(getContext()); |
| 4821 | target.markUnknownOpDynamicallyLegal(fn: [](Operation *) { return true; }); |
| 4822 | target.addIllegalOp<acc::ComputeRegionOp, acc::ParWidthOp>(); |
| 4823 | if (failed(Result: applyPartialConversion(op: getOperation(), target, |
| 4824 | patterns: std::move(patterns)))) { |
| 4825 | signalPassFailure(); |
| 4826 | } |
| 4827 | } |
| 4828 | }; |
| 4829 | } // namespace |
| 4830 | |